Lines Matching refs:hwq
159 struct hwq *hwq = get_hwq(afu, cmd->hwq_index); in cmd_complete() local
161 spin_lock_irqsave(&hwq->hsq_slock, lock_flags); in cmd_complete()
163 spin_unlock_irqrestore(&hwq->hsq_slock, lock_flags); in cmd_complete()
191 static void flush_pending_cmds(struct hwq *hwq) in flush_pending_cmds() argument
193 struct cxlflash_cfg *cfg = hwq->afu->parent; in flush_pending_cmds()
198 list_for_each_entry_safe(cmd, tmp, &hwq->pending_cmds, list) { in flush_pending_cmds()
235 static int context_reset(struct hwq *hwq, __be64 __iomem *reset_reg) in context_reset() argument
237 struct cxlflash_cfg *cfg = hwq->afu->parent; in context_reset()
244 dev_dbg(dev, "%s: hwq=%p\n", __func__, hwq); in context_reset()
246 spin_lock_irqsave(&hwq->hsq_slock, lock_flags); in context_reset()
261 flush_pending_cmds(hwq); in context_reset()
263 spin_unlock_irqrestore(&hwq->hsq_slock, lock_flags); in context_reset()
276 static int context_reset_ioarrin(struct hwq *hwq) in context_reset_ioarrin() argument
278 return context_reset(hwq, &hwq->host_map->ioarrin); in context_reset_ioarrin()
287 static int context_reset_sq(struct hwq *hwq) in context_reset_sq() argument
289 return context_reset(hwq, &hwq->host_map->sq_ctx_reset); in context_reset_sq()
304 struct hwq *hwq = get_hwq(afu, cmd->hwq_index); in send_cmd_ioarrin() local
313 spin_lock_irqsave(&hwq->hsq_slock, lock_flags); in send_cmd_ioarrin()
314 if (--hwq->room < 0) { in send_cmd_ioarrin()
315 room = readq_be(&hwq->host_map->cmd_room); in send_cmd_ioarrin()
320 hwq->room = 0; in send_cmd_ioarrin()
324 hwq->room = room - 1; in send_cmd_ioarrin()
327 list_add(&cmd->list, &hwq->pending_cmds); in send_cmd_ioarrin()
328 writeq_be((u64)&cmd->rcb, &hwq->host_map->ioarrin); in send_cmd_ioarrin()
330 spin_unlock_irqrestore(&hwq->hsq_slock, lock_flags); in send_cmd_ioarrin()
348 struct hwq *hwq = get_hwq(afu, cmd->hwq_index); in send_cmd_sq() local
353 newval = atomic_dec_if_positive(&hwq->hsq_credits); in send_cmd_sq()
361 spin_lock_irqsave(&hwq->hsq_slock, lock_flags); in send_cmd_sq()
363 *hwq->hsq_curr = cmd->rcb; in send_cmd_sq()
364 if (hwq->hsq_curr < hwq->hsq_end) in send_cmd_sq()
365 hwq->hsq_curr++; in send_cmd_sq()
367 hwq->hsq_curr = hwq->hsq_start; in send_cmd_sq()
369 list_add(&cmd->list, &hwq->pending_cmds); in send_cmd_sq()
370 writeq_be((u64)hwq->hsq_curr, &hwq->host_map->sq_tail); in send_cmd_sq()
372 spin_unlock_irqrestore(&hwq->hsq_slock, lock_flags); in send_cmd_sq()
376 cmd->rcb.data_ea, cmd->rcb.ioasa, rc, hwq->hsq_curr, in send_cmd_sq()
377 readq_be(&hwq->host_map->sq_head), in send_cmd_sq()
378 readq_be(&hwq->host_map->sq_tail)); in send_cmd_sq()
426 u32 hwq = 0; in cmd_to_target_hwq() local
433 hwq = afu->hwq_rr_count++ % afu->num_hwqs; in cmd_to_target_hwq()
437 hwq = blk_mq_unique_tag_to_hwq(tag); in cmd_to_target_hwq()
440 hwq = smp_processor_id() % afu->num_hwqs; in cmd_to_target_hwq()
446 return hwq; in cmd_to_target_hwq()
464 struct hwq *hwq = get_hwq(afu, PRIMARY_HWQ); in send_tmf() local
492 cmd->hwq_index = hwq->index; in send_tmf()
494 cmd->rcb.ctx_id = hwq->ctx_hndl; in send_tmf()
533 spin_lock_irqsave(&hwq->hsq_slock, lock_flags); in send_tmf()
535 spin_unlock_irqrestore(&hwq->hsq_slock, lock_flags); in send_tmf()
568 struct hwq *hwq = get_hwq(afu, hwq_index); in cxlflash_queuecommand() local
622 cmd->rcb.ctx_id = hwq->ctx_hndl; in cxlflash_queuecommand()
693 struct hwq *hwq; in stop_afu() local
706 hwq = get_hwq(afu, i); in stop_afu()
708 irq_poll_disable(&hwq->irqpoll); in stop_afu()
732 struct hwq *hwq; in term_intr() local
739 hwq = get_hwq(afu, index); in term_intr()
741 if (!hwq->ctx_cookie) { in term_intr()
750 cfg->ops->unmap_afu_irq(hwq->ctx_cookie, 3, hwq); in term_intr()
753 cfg->ops->unmap_afu_irq(hwq->ctx_cookie, 2, hwq); in term_intr()
756 cfg->ops->unmap_afu_irq(hwq->ctx_cookie, 1, hwq); in term_intr()
759 cfg->ops->free_afu_irqs(hwq->ctx_cookie); in term_intr()
778 struct hwq *hwq; in term_mc() local
786 hwq = get_hwq(afu, index); in term_mc()
788 if (!hwq->ctx_cookie) { in term_mc()
793 WARN_ON(cfg->ops->stop_context(hwq->ctx_cookie)); in term_mc()
795 WARN_ON(cfg->ops->release_context(hwq->ctx_cookie)); in term_mc()
796 hwq->ctx_cookie = NULL; in term_mc()
798 spin_lock_irqsave(&hwq->hrrq_slock, lock_flags); in term_mc()
799 hwq->hrrq_online = false; in term_mc()
800 spin_unlock_irqrestore(&hwq->hrrq_slock, lock_flags); in term_mc()
802 spin_lock_irqsave(&hwq->hsq_slock, lock_flags); in term_mc()
803 flush_pending_cmds(hwq); in term_mc()
804 spin_unlock_irqrestore(&hwq->hsq_slock, lock_flags); in term_mc()
1265 struct hwq *hwq = get_hwq(afu, PRIMARY_HWQ); in afu_err_intr_init() local
1278 reg = ((u64) (((hwq->ctx_hndl << 8) | SISL_MSI_ASYNC_ERROR)) << 40); in afu_err_intr_init()
1315 hwq = get_hwq(afu, i); in afu_err_intr_init()
1317 reg = readq_be(&hwq->host_map->ctx_ctrl); in afu_err_intr_init()
1320 writeq_be(reg, &hwq->host_map->ctx_ctrl); in afu_err_intr_init()
1321 writeq_be(SISL_ISTATUS_MASK, &hwq->host_map->intr_mask); in afu_err_intr_init()
1334 struct hwq *hwq = (struct hwq *)data; in cxlflash_sync_err_irq() local
1335 struct cxlflash_cfg *cfg = hwq->afu->parent; in cxlflash_sync_err_irq()
1340 reg = readq_be(&hwq->host_map->intr_status); in cxlflash_sync_err_irq()
1352 writeq_be(reg_unmasked, &hwq->host_map->intr_clear); in cxlflash_sync_err_irq()
1368 static int process_hrrq(struct hwq *hwq, struct list_head *doneq, int budget) in process_hrrq() argument
1370 struct afu *afu = hwq->afu; in process_hrrq()
1374 bool toggle = hwq->toggle; in process_hrrq()
1377 *hrrq_start = hwq->hrrq_start, in process_hrrq()
1378 *hrrq_end = hwq->hrrq_end, in process_hrrq()
1379 *hrrq_curr = hwq->hrrq_curr; in process_hrrq()
1408 atomic_inc(&hwq->hsq_credits); in process_hrrq()
1415 hwq->hrrq_curr = hrrq_curr; in process_hrrq()
1416 hwq->toggle = toggle; in process_hrrq()
1446 struct hwq *hwq = container_of(irqpoll, struct hwq, irqpoll); in cxlflash_irqpoll() local
1451 spin_lock_irqsave(&hwq->hrrq_slock, hrrq_flags); in cxlflash_irqpoll()
1453 num_entries = process_hrrq(hwq, &doneq, budget); in cxlflash_irqpoll()
1457 spin_unlock_irqrestore(&hwq->hrrq_slock, hrrq_flags); in cxlflash_irqpoll()
1472 struct hwq *hwq = (struct hwq *)data; in cxlflash_rrq_irq() local
1473 struct afu *afu = hwq->afu; in cxlflash_rrq_irq()
1478 spin_lock_irqsave(&hwq->hrrq_slock, hrrq_flags); in cxlflash_rrq_irq()
1481 if (!hwq->hrrq_online) { in cxlflash_rrq_irq()
1482 spin_unlock_irqrestore(&hwq->hrrq_slock, hrrq_flags); in cxlflash_rrq_irq()
1487 irq_poll_sched(&hwq->irqpoll); in cxlflash_rrq_irq()
1488 spin_unlock_irqrestore(&hwq->hrrq_slock, hrrq_flags); in cxlflash_rrq_irq()
1492 num_entries = process_hrrq(hwq, &doneq, -1); in cxlflash_rrq_irq()
1493 spin_unlock_irqrestore(&hwq->hrrq_slock, hrrq_flags); in cxlflash_rrq_irq()
1540 struct hwq *hwq = (struct hwq *)data; in cxlflash_async_err_irq() local
1541 struct afu *afu = hwq->afu; in cxlflash_async_err_irq()
1711 struct hwq *hwq; in init_pcr() local
1726 hwq = get_hwq(afu, i); in init_pcr()
1727 cookie = hwq->ctx_cookie; in init_pcr()
1729 hwq->ctx_hndl = (u16) cfg->ops->process_element(cookie); in init_pcr()
1730 hwq->host_map = &afu->afu_map->hosts[hwq->ctx_hndl].host; in init_pcr()
1731 hwq->ctrl_map = &afu->afu_map->ctrls[hwq->ctx_hndl].ctrl; in init_pcr()
1734 writeq_be(SISL_ENDIAN_CTRL, &hwq->host_map->endian_ctrl); in init_pcr()
1746 struct hwq *hwq; in init_global() local
1764 hwq = get_hwq(afu, i); in init_global()
1765 hmap = hwq->host_map; in init_global()
1767 writeq_be((u64) hwq->hrrq_start, &hmap->rrq_start); in init_global()
1768 writeq_be((u64) hwq->hrrq_end, &hmap->rrq_end); in init_global()
1769 hwq->hrrq_online = true; in init_global()
1772 writeq_be((u64)hwq->hsq_start, &hmap->sq_start); in init_global()
1773 writeq_be((u64)hwq->hsq_end, &hmap->sq_end); in init_global()
1818 hwq = get_hwq(afu, i); in init_global()
1819 ctx = hwq->ctx_cookie; in init_global()
1821 for (j = 0; j < hwq->num_irqs; j++) { in init_global()
1823 writeq_be(reg, &hwq->ctrl_map->lisn_ea[j]); in init_global()
1826 reg = hwq->ctx_hndl; in init_global()
1828 &hwq->ctrl_map->lisn_pasid[0]); in init_global()
1830 &hwq->ctrl_map->lisn_pasid[1]); in init_global()
1838 hwq = get_hwq(afu, i); in init_global()
1840 (void)readq_be(&hwq->ctrl_map->mbox_r); /* unlock ctx_cap */ in init_global()
1844 &hwq->ctrl_map->ctx_cap); in init_global()
1853 hwq = get_hwq(afu, PRIMARY_HWQ); in init_global()
1854 reg = readq_be(&hwq->host_map->ctx_ctrl); in init_global()
1872 struct hwq *hwq; in start_afu() local
1880 hwq = get_hwq(afu, i); in start_afu()
1883 memset(&hwq->rrq_entry, 0, sizeof(hwq->rrq_entry)); in start_afu()
1886 hwq->hrrq_start = &hwq->rrq_entry[0]; in start_afu()
1887 hwq->hrrq_end = &hwq->rrq_entry[NUM_RRQ_ENTRY - 1]; in start_afu()
1888 hwq->hrrq_curr = hwq->hrrq_start; in start_afu()
1889 hwq->toggle = 1; in start_afu()
1892 spin_lock_init(&hwq->hrrq_slock); in start_afu()
1893 spin_lock_init(&hwq->hsq_slock); in start_afu()
1897 memset(&hwq->sq, 0, sizeof(hwq->sq)); in start_afu()
1898 hwq->hsq_start = &hwq->sq[0]; in start_afu()
1899 hwq->hsq_end = &hwq->sq[NUM_SQ_ENTRY - 1]; in start_afu()
1900 hwq->hsq_curr = hwq->hsq_start; in start_afu()
1902 atomic_set(&hwq->hsq_credits, NUM_SQ_ENTRY - 1); in start_afu()
1907 irq_poll_init(&hwq->irqpoll, afu->irqpoll_weight, in start_afu()
1926 struct hwq *hwq) in init_intr() argument
1929 void *ctx = hwq->ctx_cookie; in init_intr()
1932 bool is_primary_hwq = (hwq->index == PRIMARY_HWQ); in init_intr()
1933 int num_irqs = hwq->num_irqs; in init_intr()
1943 rc = cfg->ops->map_afu_irq(ctx, 1, cxlflash_sync_err_irq, hwq, in init_intr()
1951 rc = cfg->ops->map_afu_irq(ctx, 2, cxlflash_rrq_irq, hwq, in init_intr()
1963 rc = cfg->ops->map_afu_irq(ctx, 3, cxlflash_async_err_irq, hwq, in init_intr()
1985 struct hwq *hwq = get_hwq(cfg->afu, index); in init_mc() local
1990 hwq->afu = cfg->afu; in init_mc()
1991 hwq->index = index; in init_mc()
1992 INIT_LIST_HEAD(&hwq->pending_cmds); in init_mc()
2006 WARN_ON(hwq->ctx_cookie); in init_mc()
2007 hwq->ctx_cookie = ctx; in init_mc()
2008 hwq->num_irqs = num_irqs; in init_mc()
2023 level = init_intr(cfg, hwq); in init_mc()
2030 rc = cfg->ops->start_context(hwq->ctx_cookie); in init_mc()
2045 hwq->ctx_cookie = NULL; in init_mc()
2097 struct hwq *hwq; in init_afu() local
2114 hwq = get_hwq(afu, PRIMARY_HWQ); in init_afu()
2115 afu->afu_map = cfg->ops->psa_map(hwq->ctx_cookie); in init_afu()
2156 hwq = get_hwq(afu, i); in init_afu()
2158 hwq->room = readq_be(&hwq->host_map->cmd_room); in init_afu()
2281 struct hwq *hwq = get_hwq(afu, PRIMARY_HWQ); in send_afu_cmd() local
2310 cmd->hwq_index = hwq->index; in send_afu_cmd()
2311 cmd->rcb.ctx_id = hwq->ctx_hndl; in send_afu_cmd()
2325 rc = afu->context_reset(hwq); in send_afu_cmd()
2328 spin_lock_irqsave(&hwq->hsq_slock, lock_flags); in send_afu_cmd()
2330 spin_unlock_irqrestore(&hwq->hsq_slock, lock_flags); in send_afu_cmd()
2409 struct hwq *hwq = get_hwq(afu, cmd->hwq_index); in cxlflash_eh_abort_handler() local
2428 rc = afu->context_reset(hwq); in cxlflash_eh_abort_handler()
2886 struct hwq *hwq; in irqpoll_weight_store() local
2908 hwq = get_hwq(afu, i); in irqpoll_weight_store()
2910 irq_poll_disable(&hwq->irqpoll); in irqpoll_weight_store()
2918 hwq = get_hwq(afu, i); in irqpoll_weight_store()
2920 irq_poll_init(&hwq->irqpoll, weight, cxlflash_irqpoll); in irqpoll_weight_store()