Lines Matching full:qm
364 struct hisi_qm *qm; member
380 int (*get_vft)(struct hisi_qm *qm, u32 *base, u32 *number);
381 void (*qm_db)(struct hisi_qm *qm, u16 qn,
383 int (*debug_init)(struct hisi_qm *qm);
384 void (*hw_error_init)(struct hisi_qm *qm);
385 void (*hw_error_uninit)(struct hisi_qm *qm);
386 enum acc_err_result (*hw_error_handle)(struct hisi_qm *qm);
387 int (*set_msi)(struct hisi_qm *qm, bool set);
451 static void qm_irqs_unregister(struct hisi_qm *qm);
452 static int qm_reset_device(struct hisi_qm *qm);
454 static bool qm_avail_state(struct hisi_qm *qm, enum qm_state new) in qm_avail_state() argument
456 enum qm_state curr = atomic_read(&qm->status.flags); in qm_avail_state()
476 dev_dbg(&qm->pdev->dev, "change qm state from %s to %s\n", in qm_avail_state()
480 dev_warn(&qm->pdev->dev, "Can not change qm state from %s to %s\n", in qm_avail_state()
486 static bool qm_qp_avail_state(struct hisi_qm *qm, struct hisi_qp *qp, in qm_qp_avail_state() argument
489 enum qm_state qm_curr = atomic_read(&qm->status.flags); in qm_qp_avail_state()
522 dev_dbg(&qm->pdev->dev, "change qp state from %s to %s in QM %s\n", in qm_qp_avail_state()
526 dev_warn(&qm->pdev->dev, in qm_qp_avail_state()
527 "Can not change qp state from %s to %s in QM %s\n", in qm_qp_avail_state()
533 static u32 qm_get_hw_error_status(struct hisi_qm *qm) in qm_get_hw_error_status() argument
535 return readl(qm->io_base + QM_ABNORMAL_INT_STATUS); in qm_get_hw_error_status()
538 static u32 qm_get_dev_err_status(struct hisi_qm *qm) in qm_get_dev_err_status() argument
540 return qm->err_ini->get_dev_hw_err_status(qm); in qm_get_dev_err_status()
544 static bool qm_check_dev_error(struct hisi_qm *qm) in qm_check_dev_error() argument
548 if (qm->fun_type == QM_HW_VF) in qm_check_dev_error()
551 val = qm_get_hw_error_status(qm) & qm->err_info.qm_shutdown_mask; in qm_check_dev_error()
552 dev_val = qm_get_dev_err_status(qm) & qm->err_info.dev_shutdown_mask; in qm_check_dev_error()
557 static int qm_wait_reset_finish(struct hisi_qm *qm) in qm_wait_reset_finish() argument
562 while (test_and_set_bit(QM_RESETTING, &qm->misc_ctl)) { in qm_wait_reset_finish()
571 static int qm_reset_prepare_ready(struct hisi_qm *qm) in qm_reset_prepare_ready() argument
573 struct pci_dev *pdev = qm->pdev; in qm_reset_prepare_ready()
580 if (qm->ver < QM_HW_V3) in qm_reset_prepare_ready()
583 return qm_wait_reset_finish(qm); in qm_reset_prepare_ready()
586 static void qm_reset_bit_clear(struct hisi_qm *qm) in qm_reset_bit_clear() argument
588 struct pci_dev *pdev = qm->pdev; in qm_reset_bit_clear()
591 if (qm->ver < QM_HW_V3) in qm_reset_bit_clear()
594 clear_bit(QM_RESETTING, &qm->misc_ctl); in qm_reset_bit_clear()
610 int hisi_qm_wait_mb_ready(struct hisi_qm *qm) in hisi_qm_wait_mb_ready() argument
614 return readl_relaxed_poll_timeout(qm->io_base + QM_MB_CMD_SEND_BASE, in hisi_qm_wait_mb_ready()
621 static void qm_mb_write(struct hisi_qm *qm, const void *src) in qm_mb_write() argument
623 void __iomem *fun_base = qm->io_base + QM_MB_CMD_SEND_BASE; in qm_mb_write()
647 static int qm_mb_nolock(struct hisi_qm *qm, struct qm_mailbox *mailbox) in qm_mb_nolock() argument
652 if (unlikely(hisi_qm_wait_mb_ready(qm))) { in qm_mb_nolock()
653 dev_err(&qm->pdev->dev, "QM mailbox is busy to start!\n"); in qm_mb_nolock()
658 qm_mb_write(qm, mailbox); in qm_mb_nolock()
660 if (unlikely(hisi_qm_wait_mb_ready(qm))) { in qm_mb_nolock()
661 dev_err(&qm->pdev->dev, "QM mailbox operation timeout!\n"); in qm_mb_nolock()
666 val = readl(qm->io_base + QM_MB_CMD_SEND_BASE); in qm_mb_nolock()
668 dev_err(&qm->pdev->dev, "QM mailbox operation failed!\n"); in qm_mb_nolock()
676 atomic64_inc(&qm->debug.dfx.mb_err_cnt); in qm_mb_nolock()
680 int hisi_qm_mb(struct hisi_qm *qm, u8 cmd, dma_addr_t dma_addr, u16 queue, in hisi_qm_mb() argument
686 dev_dbg(&qm->pdev->dev, "QM mailbox request to q%u: %u-%llx\n", in hisi_qm_mb()
691 mutex_lock(&qm->mailbox_lock); in hisi_qm_mb()
692 ret = qm_mb_nolock(qm, &mailbox); in hisi_qm_mb()
693 mutex_unlock(&qm->mailbox_lock); in hisi_qm_mb()
699 static void qm_db_v1(struct hisi_qm *qm, u16 qn, u8 cmd, u16 index, u8 priority) in qm_db_v1() argument
707 writeq(doorbell, qm->io_base + QM_DOORBELL_BASE_V1); in qm_db_v1()
710 static void qm_db_v2(struct hisi_qm *qm, u16 qn, u8 cmd, u16 index, u8 priority) in qm_db_v2() argument
712 void __iomem *io_base = qm->io_base; in qm_db_v2()
717 io_base = qm->db_io_base + (u64)qn * qm->db_interval + in qm_db_v2()
730 static void qm_db(struct hisi_qm *qm, u16 qn, u8 cmd, u16 index, u8 priority) in qm_db() argument
732 dev_dbg(&qm->pdev->dev, "QM doorbell request: qn=%u, cmd=%u, index=%u\n", in qm_db()
735 qm->ops->qm_db(qm, qn, cmd, index, priority); in qm_db()
738 static void qm_disable_clock_gate(struct hisi_qm *qm) in qm_disable_clock_gate() argument
742 /* if qm enables clock gating in Kunpeng930, qos will be inaccurate. */ in qm_disable_clock_gate()
743 if (qm->ver < QM_HW_V3) in qm_disable_clock_gate()
746 val = readl(qm->io_base + QM_PM_CTRL); in qm_disable_clock_gate()
748 writel(val, qm->io_base + QM_PM_CTRL); in qm_disable_clock_gate()
751 static int qm_dev_mem_reset(struct hisi_qm *qm) in qm_dev_mem_reset() argument
755 writel(0x1, qm->io_base + QM_MEM_START_INIT); in qm_dev_mem_reset()
756 return readl_relaxed_poll_timeout(qm->io_base + QM_MEM_INIT_DONE, val, in qm_dev_mem_reset()
763 * @qm: The qm which want to get information.
770 u32 hisi_qm_get_hw_info(struct hisi_qm *qm, in hisi_qm_get_hw_info() argument
776 switch (qm->ver) { in hisi_qm_get_hw_info()
785 val = readl(qm->io_base + info_table[index].offset); in hisi_qm_get_hw_info()
791 static void qm_get_xqc_depth(struct hisi_qm *qm, u16 *low_bits, in qm_get_xqc_depth() argument
796 depth = hisi_qm_get_hw_info(qm, qm_basic_info, type, qm->cap_ver); in qm_get_xqc_depth()
801 int hisi_qm_set_algs(struct hisi_qm *qm, u64 alg_msk, const struct qm_dev_alg *dev_algs, in hisi_qm_set_algs() argument
804 struct device *dev = &qm->pdev->dev; in hisi_qm_set_algs()
808 if (!qm->uacce) in hisi_qm_set_algs()
828 qm->uacce->algs = algs; in hisi_qm_set_algs()
835 static u32 qm_get_irq_num(struct hisi_qm *qm) in qm_get_irq_num() argument
837 if (qm->fun_type == QM_HW_PF) in qm_get_irq_num()
838 return hisi_qm_get_hw_info(qm, qm_basic_info, QM_PF_IRQ_NUM_CAP, qm->cap_ver); in qm_get_irq_num()
840 return hisi_qm_get_hw_info(qm, qm_basic_info, QM_VF_IRQ_NUM_CAP, qm->cap_ver); in qm_get_irq_num()
843 static int qm_pm_get_sync(struct hisi_qm *qm) in qm_pm_get_sync() argument
845 struct device *dev = &qm->pdev->dev; in qm_pm_get_sync()
848 if (!test_bit(QM_SUPPORT_RPM, &qm->caps)) in qm_pm_get_sync()
860 static void qm_pm_put_sync(struct hisi_qm *qm) in qm_pm_put_sync() argument
862 struct device *dev = &qm->pdev->dev; in qm_pm_put_sync()
864 if (!test_bit(QM_SUPPORT_RPM, &qm->caps)) in qm_pm_put_sync()
884 struct hisi_qm *qm = qp->qm; in qm_poll_req_cb() local
888 qp->req_cb(qp, qp->sqe + qm->sqe_size * in qm_poll_req_cb()
892 qm_db(qm, qp->qp_id, QM_DOORBELL_CMD_CQ, in qm_poll_req_cb()
900 qm_db(qm, qp->qp_id, QM_DOORBELL_CMD_CQ, qp->qp_status.cq_head, 1); in qm_poll_req_cb()
907 struct hisi_qm *qm = poll_data->qm; in qm_work_process() local
913 qp = &qm->qp_array[poll_data->qp_finish_id[i]]; in qm_work_process()
927 static void qm_get_complete_eqe_num(struct hisi_qm *qm) in qm_get_complete_eqe_num() argument
929 struct qm_eqe *eqe = qm->eqe + qm->status.eq_head; in qm_get_complete_eqe_num()
931 u16 eq_depth = qm->eq_depth; in qm_get_complete_eqe_num()
934 if (QM_EQE_PHASE(eqe) != qm->status.eqc_phase) { in qm_get_complete_eqe_num()
935 atomic64_inc(&qm->debug.dfx.err_irq_cnt); in qm_get_complete_eqe_num()
936 qm_db(qm, 0, QM_DOORBELL_CMD_EQ, qm->status.eq_head, 0); in qm_get_complete_eqe_num()
941 if (unlikely(cqn >= qm->qp_num)) in qm_get_complete_eqe_num()
943 poll_data = &qm->poll_data[cqn]; in qm_get_complete_eqe_num()
945 while (QM_EQE_PHASE(eqe) == qm->status.eqc_phase) { in qm_get_complete_eqe_num()
950 if (qm->status.eq_head == eq_depth - 1) { in qm_get_complete_eqe_num()
951 qm->status.eqc_phase = !qm->status.eqc_phase; in qm_get_complete_eqe_num()
952 eqe = qm->eqe; in qm_get_complete_eqe_num()
953 qm->status.eq_head = 0; in qm_get_complete_eqe_num()
956 qm->status.eq_head++; in qm_get_complete_eqe_num()
964 queue_work(qm->wq, &poll_data->work); in qm_get_complete_eqe_num()
965 qm_db(qm, 0, QM_DOORBELL_CMD_EQ, qm->status.eq_head, 0); in qm_get_complete_eqe_num()
970 struct hisi_qm *qm = data; in qm_eq_irq() local
973 qm_get_complete_eqe_num(qm); in qm_eq_irq()
980 struct hisi_qm *qm = data; in qm_mb_cmd_irq() local
983 val = readl(qm->io_base + QM_IFC_INT_STATUS); in qm_mb_cmd_irq()
988 if (test_bit(QM_DRIVER_REMOVING, &qm->misc_ctl)) { in qm_mb_cmd_irq()
989 dev_warn(&qm->pdev->dev, "Driver is down, message cannot be processed!\n"); in qm_mb_cmd_irq()
993 schedule_work(&qm->cmd_process); in qm_mb_cmd_irq()
1012 static void qm_disable_qp(struct hisi_qm *qm, u32 qp_id) in qm_disable_qp() argument
1014 struct hisi_qp *qp = &qm->qp_array[qp_id]; in qm_disable_qp()
1021 static void qm_reset_function(struct hisi_qm *qm) in qm_reset_function() argument
1023 struct hisi_qm *pf_qm = pci_get_drvdata(pci_physfn(qm->pdev)); in qm_reset_function()
1024 struct device *dev = &qm->pdev->dev; in qm_reset_function()
1030 ret = qm_reset_prepare_ready(qm); in qm_reset_function()
1036 ret = hisi_qm_stop(qm, QM_DOWN); in qm_reset_function()
1038 dev_err(dev, "failed to stop qm when reset function\n"); in qm_reset_function()
1042 ret = hisi_qm_start(qm); in qm_reset_function()
1044 dev_err(dev, "failed to start qm when reset function\n"); in qm_reset_function()
1047 qm_reset_bit_clear(qm); in qm_reset_function()
1052 struct hisi_qm *qm = data; in qm_aeq_thread() local
1053 struct qm_aeqe *aeqe = qm->aeqe + qm->status.aeq_head; in qm_aeq_thread()
1054 u16 aeq_depth = qm->aeq_depth; in qm_aeq_thread()
1057 atomic64_inc(&qm->debug.dfx.aeq_irq_cnt); in qm_aeq_thread()
1059 while (QM_AEQE_PHASE(aeqe) == qm->status.aeqc_phase) { in qm_aeq_thread()
1065 dev_err(&qm->pdev->dev, "eq overflow, reset function\n"); in qm_aeq_thread()
1066 qm_reset_function(qm); in qm_aeq_thread()
1069 dev_err(&qm->pdev->dev, "cq overflow, stop qp(%u)\n", in qm_aeq_thread()
1073 qm_disable_qp(qm, qp_id); in qm_aeq_thread()
1076 dev_err(&qm->pdev->dev, "unknown error type %u\n", in qm_aeq_thread()
1081 if (qm->status.aeq_head == aeq_depth - 1) { in qm_aeq_thread()
1082 qm->status.aeqc_phase = !qm->status.aeqc_phase; in qm_aeq_thread()
1083 aeqe = qm->aeqe; in qm_aeq_thread()
1084 qm->status.aeq_head = 0; in qm_aeq_thread()
1087 qm->status.aeq_head++; in qm_aeq_thread()
1091 qm_db(qm, 0, QM_DOORBELL_CMD_AEQ, qm->status.aeq_head, 0); in qm_aeq_thread()
1106 static void qm_init_prefetch(struct hisi_qm *qm) in qm_init_prefetch() argument
1108 struct device *dev = &qm->pdev->dev; in qm_init_prefetch()
1111 if (!test_bit(QM_SUPPORT_SVA_PREFETCH, &qm->caps)) in qm_init_prefetch()
1129 writel(page_type, qm->io_base + QM_PAGE_SIZE); in qm_init_prefetch()
1199 static void qm_vft_data_cfg(struct hisi_qm *qm, enum vft_type type, u32 base, in qm_vft_data_cfg() argument
1207 if (qm->ver == QM_HW_V1) { in qm_vft_data_cfg()
1220 if (qm->ver == QM_HW_V1) { in qm_vft_data_cfg()
1241 writel(lower_32_bits(tmp), qm->io_base + QM_VFT_CFG_DATA_L); in qm_vft_data_cfg()
1242 writel(upper_32_bits(tmp), qm->io_base + QM_VFT_CFG_DATA_H); in qm_vft_data_cfg()
1245 static int qm_set_vft_common(struct hisi_qm *qm, enum vft_type type, in qm_set_vft_common() argument
1252 if (type == SHAPER_VFT && test_bit(QM_SUPPORT_FUNC_QOS, &qm->caps)) in qm_set_vft_common()
1253 factor = &qm->factor[fun_num]; in qm_set_vft_common()
1255 ret = readl_relaxed_poll_timeout(qm->io_base + QM_VFT_CFG_RDY, val, in qm_set_vft_common()
1261 writel(0x0, qm->io_base + QM_VFT_CFG_OP_WR); in qm_set_vft_common()
1262 writel(type, qm->io_base + QM_VFT_CFG_TYPE); in qm_set_vft_common()
1266 writel(fun_num, qm->io_base + QM_VFT_CFG); in qm_set_vft_common()
1268 qm_vft_data_cfg(qm, type, base, number, factor); in qm_set_vft_common()
1270 writel(0x0, qm->io_base + QM_VFT_CFG_RDY); in qm_set_vft_common()
1271 writel(0x1, qm->io_base + QM_VFT_CFG_OP_ENABLE); in qm_set_vft_common()
1273 return readl_relaxed_poll_timeout(qm->io_base + QM_VFT_CFG_RDY, val, in qm_set_vft_common()
1278 static int qm_shaper_init_vft(struct hisi_qm *qm, u32 fun_num) in qm_shaper_init_vft() argument
1280 u32 qos = qm->factor[fun_num].func_qos; in qm_shaper_init_vft()
1283 ret = qm_get_shaper_para(qos * QM_QOS_RATE, &qm->factor[fun_num]); in qm_shaper_init_vft()
1285 dev_err(&qm->pdev->dev, "failed to calculate shaper parameter!\n"); in qm_shaper_init_vft()
1288 writel(qm->type_rate, qm->io_base + QM_SHAPER_CFG); in qm_shaper_init_vft()
1291 ret = qm_set_vft_common(qm, SHAPER_VFT, fun_num, i, 1); in qm_shaper_init_vft()
1300 static int qm_set_sqc_cqc_vft(struct hisi_qm *qm, u32 fun_num, u32 base, in qm_set_sqc_cqc_vft() argument
1306 ret = qm_set_vft_common(qm, i, fun_num, base, number); in qm_set_sqc_cqc_vft()
1312 if (test_bit(QM_SUPPORT_FUNC_QOS, &qm->caps)) { in qm_set_sqc_cqc_vft()
1313 ret = qm_shaper_init_vft(qm, fun_num); in qm_set_sqc_cqc_vft()
1321 qm_set_vft_common(qm, i, fun_num, 0, 0); in qm_set_sqc_cqc_vft()
1326 static int qm_get_vft_v2(struct hisi_qm *qm, u32 *base, u32 *number) in qm_get_vft_v2() argument
1331 ret = hisi_qm_mb(qm, QM_MB_CMD_SQC_VFT_V2, 0, 0, 1); in qm_get_vft_v2()
1335 sqc_vft = readl(qm->io_base + QM_MB_CMD_DATA_ADDR_L) | in qm_get_vft_v2()
1336 ((u64)readl(qm->io_base + QM_MB_CMD_DATA_ADDR_H) << 32); in qm_get_vft_v2()
1344 void *hisi_qm_ctx_alloc(struct hisi_qm *qm, size_t ctx_size, in hisi_qm_ctx_alloc() argument
1347 struct device *dev = &qm->pdev->dev; in hisi_qm_ctx_alloc()
1364 void hisi_qm_ctx_free(struct hisi_qm *qm, size_t ctx_size, in hisi_qm_ctx_free() argument
1367 struct device *dev = &qm->pdev->dev; in hisi_qm_ctx_free()
1373 static int qm_dump_sqc_raw(struct hisi_qm *qm, dma_addr_t dma_addr, u16 qp_id) in qm_dump_sqc_raw() argument
1375 return hisi_qm_mb(qm, QM_MB_CMD_SQC, dma_addr, qp_id, 1); in qm_dump_sqc_raw()
1378 static int qm_dump_cqc_raw(struct hisi_qm *qm, dma_addr_t dma_addr, u16 qp_id) in qm_dump_cqc_raw() argument
1380 return hisi_qm_mb(qm, QM_MB_CMD_CQC, dma_addr, qp_id, 1); in qm_dump_cqc_raw()
1383 static void qm_hw_error_init_v1(struct hisi_qm *qm) in qm_hw_error_init_v1() argument
1385 writel(QM_ABNORMAL_INT_MASK_VALUE, qm->io_base + QM_ABNORMAL_INT_MASK); in qm_hw_error_init_v1()
1388 static void qm_hw_error_cfg(struct hisi_qm *qm) in qm_hw_error_cfg() argument
1390 struct hisi_qm_err_info *err_info = &qm->err_info; in qm_hw_error_cfg()
1392 qm->error_mask = err_info->nfe | err_info->ce | err_info->fe; in qm_hw_error_cfg()
1393 /* clear QM hw residual error source */ in qm_hw_error_cfg()
1394 writel(qm->error_mask, qm->io_base + QM_ABNORMAL_INT_SOURCE); in qm_hw_error_cfg()
1397 writel(err_info->ce, qm->io_base + QM_RAS_CE_ENABLE); in qm_hw_error_cfg()
1398 writel(QM_RAS_CE_TIMES_PER_IRQ, qm->io_base + QM_RAS_CE_THRESHOLD); in qm_hw_error_cfg()
1399 writel(err_info->nfe, qm->io_base + QM_RAS_NFE_ENABLE); in qm_hw_error_cfg()
1400 writel(err_info->fe, qm->io_base + QM_RAS_FE_ENABLE); in qm_hw_error_cfg()
1403 static void qm_hw_error_init_v2(struct hisi_qm *qm) in qm_hw_error_init_v2() argument
1407 qm_hw_error_cfg(qm); in qm_hw_error_init_v2()
1409 irq_unmask = ~qm->error_mask; in qm_hw_error_init_v2()
1410 irq_unmask &= readl(qm->io_base + QM_ABNORMAL_INT_MASK); in qm_hw_error_init_v2()
1411 writel(irq_unmask, qm->io_base + QM_ABNORMAL_INT_MASK); in qm_hw_error_init_v2()
1414 static void qm_hw_error_uninit_v2(struct hisi_qm *qm) in qm_hw_error_uninit_v2() argument
1416 u32 irq_mask = qm->error_mask; in qm_hw_error_uninit_v2()
1418 irq_mask |= readl(qm->io_base + QM_ABNORMAL_INT_MASK); in qm_hw_error_uninit_v2()
1419 writel(irq_mask, qm->io_base + QM_ABNORMAL_INT_MASK); in qm_hw_error_uninit_v2()
1422 static void qm_hw_error_init_v3(struct hisi_qm *qm) in qm_hw_error_init_v3() argument
1426 qm_hw_error_cfg(qm); in qm_hw_error_init_v3()
1429 writel(qm->err_info.qm_shutdown_mask, qm->io_base + QM_OOO_SHUTDOWN_SEL); in qm_hw_error_init_v3()
1431 irq_unmask = ~qm->error_mask; in qm_hw_error_init_v3()
1432 irq_unmask &= readl(qm->io_base + QM_ABNORMAL_INT_MASK); in qm_hw_error_init_v3()
1433 writel(irq_unmask, qm->io_base + QM_ABNORMAL_INT_MASK); in qm_hw_error_init_v3()
1436 static void qm_hw_error_uninit_v3(struct hisi_qm *qm) in qm_hw_error_uninit_v3() argument
1438 u32 irq_mask = qm->error_mask; in qm_hw_error_uninit_v3()
1440 irq_mask |= readl(qm->io_base + QM_ABNORMAL_INT_MASK); in qm_hw_error_uninit_v3()
1441 writel(irq_mask, qm->io_base + QM_ABNORMAL_INT_MASK); in qm_hw_error_uninit_v3()
1444 writel(0x0, qm->io_base + QM_OOO_SHUTDOWN_SEL); in qm_hw_error_uninit_v3()
1447 static void qm_log_hw_error(struct hisi_qm *qm, u32 error_status) in qm_log_hw_error() argument
1450 struct device *dev = &qm->pdev->dev; in qm_log_hw_error()
1463 reg_val = readl(qm->io_base + QM_ABNORMAL_INF01); in qm_log_hw_error()
1467 dev_err(dev, "qm %s doorbell timeout in function %u\n", in qm_log_hw_error()
1470 reg_val = readl(qm->io_base + QM_ABNORMAL_INF00); in qm_log_hw_error()
1476 dev_err(dev, "qm %s fifo overflow in function %u\n", in qm_log_hw_error()
1484 static enum acc_err_result qm_hw_error_handle_v2(struct hisi_qm *qm) in qm_hw_error_handle_v2() argument
1488 error_status = qm_get_hw_error_status(qm); in qm_hw_error_handle_v2()
1489 if (error_status & qm->error_mask) { in qm_hw_error_handle_v2()
1491 qm->err_status.is_qm_ecc_mbit = true; in qm_hw_error_handle_v2()
1493 qm_log_hw_error(qm, error_status); in qm_hw_error_handle_v2()
1494 if (error_status & qm->err_info.qm_reset_mask) { in qm_hw_error_handle_v2()
1496 writel(qm->err_info.nfe & (~error_status), in qm_hw_error_handle_v2()
1497 qm->io_base + QM_RAS_NFE_ENABLE); in qm_hw_error_handle_v2()
1502 writel(error_status, qm->io_base + QM_ABNORMAL_INT_SOURCE); in qm_hw_error_handle_v2()
1503 writel(qm->err_info.nfe, qm->io_base + QM_RAS_NFE_ENABLE); in qm_hw_error_handle_v2()
1504 writel(qm->err_info.ce, qm->io_base + QM_RAS_CE_ENABLE); in qm_hw_error_handle_v2()
1510 static int qm_get_mb_cmd(struct hisi_qm *qm, u64 *msg, u16 fun_num) in qm_get_mb_cmd() argument
1516 mutex_lock(&qm->mailbox_lock); in qm_get_mb_cmd()
1517 ret = qm_mb_nolock(qm, &mailbox); in qm_get_mb_cmd()
1521 *msg = readl(qm->io_base + QM_MB_CMD_DATA_ADDR_L) | in qm_get_mb_cmd()
1522 ((u64)readl(qm->io_base + QM_MB_CMD_DATA_ADDR_H) << 32); in qm_get_mb_cmd()
1525 mutex_unlock(&qm->mailbox_lock); in qm_get_mb_cmd()
1529 static void qm_clear_cmd_interrupt(struct hisi_qm *qm, u64 vf_mask) in qm_clear_cmd_interrupt() argument
1533 if (qm->fun_type == QM_HW_PF) in qm_clear_cmd_interrupt()
1534 writeq(vf_mask, qm->io_base + QM_IFC_INT_SOURCE_P); in qm_clear_cmd_interrupt()
1536 val = readl(qm->io_base + QM_IFC_INT_SOURCE_V); in qm_clear_cmd_interrupt()
1538 writel(val, qm->io_base + QM_IFC_INT_SOURCE_V); in qm_clear_cmd_interrupt()
1541 static void qm_handle_vf_msg(struct hisi_qm *qm, u32 vf_id) in qm_handle_vf_msg() argument
1543 struct device *dev = &qm->pdev->dev; in qm_handle_vf_msg()
1548 ret = qm_get_mb_cmd(qm, &msg, vf_id); in qm_handle_vf_msg()
1571 static int qm_wait_vf_prepare_finish(struct hisi_qm *qm) in qm_wait_vf_prepare_finish() argument
1573 struct device *dev = &qm->pdev->dev; in qm_wait_vf_prepare_finish()
1574 u32 vfs_num = qm->vfs_num; in qm_wait_vf_prepare_finish()
1580 if (!qm->vfs_num || !test_bit(QM_SUPPORT_MB_COMMAND, &qm->caps)) in qm_wait_vf_prepare_finish()
1584 val = readq(qm->io_base + QM_IFC_INT_SOURCE_P); in qm_wait_vf_prepare_finish()
1600 qm_handle_vf_msg(qm, i); in qm_wait_vf_prepare_finish()
1606 qm_clear_cmd_interrupt(qm, val); in qm_wait_vf_prepare_finish()
1611 static void qm_trigger_vf_interrupt(struct hisi_qm *qm, u32 fun_num) in qm_trigger_vf_interrupt() argument
1615 val = readl(qm->io_base + QM_IFC_INT_CFG); in qm_trigger_vf_interrupt()
1618 writel(val, qm->io_base + QM_IFC_INT_CFG); in qm_trigger_vf_interrupt()
1620 val = readl(qm->io_base + QM_IFC_INT_SET_P); in qm_trigger_vf_interrupt()
1622 writel(val, qm->io_base + QM_IFC_INT_SET_P); in qm_trigger_vf_interrupt()
1625 static void qm_trigger_pf_interrupt(struct hisi_qm *qm) in qm_trigger_pf_interrupt() argument
1629 val = readl(qm->io_base + QM_IFC_INT_SET_V); in qm_trigger_pf_interrupt()
1631 writel(val, qm->io_base + QM_IFC_INT_SET_V); in qm_trigger_pf_interrupt()
1634 static int qm_ping_single_vf(struct hisi_qm *qm, u64 cmd, u32 fun_num) in qm_ping_single_vf() argument
1636 struct device *dev = &qm->pdev->dev; in qm_ping_single_vf()
1643 mutex_lock(&qm->mailbox_lock); in qm_ping_single_vf()
1644 ret = qm_mb_nolock(qm, &mailbox); in qm_ping_single_vf()
1650 qm_trigger_vf_interrupt(qm, fun_num); in qm_ping_single_vf()
1653 val = readq(qm->io_base + QM_IFC_READY_STATUS); in qm_ping_single_vf()
1666 mutex_unlock(&qm->mailbox_lock); in qm_ping_single_vf()
1670 static int qm_ping_all_vfs(struct hisi_qm *qm, u64 cmd) in qm_ping_all_vfs() argument
1672 struct device *dev = &qm->pdev->dev; in qm_ping_all_vfs()
1673 u32 vfs_num = qm->vfs_num; in qm_ping_all_vfs()
1681 mutex_lock(&qm->mailbox_lock); in qm_ping_all_vfs()
1683 ret = qm_mb_nolock(qm, &mailbox); in qm_ping_all_vfs()
1686 mutex_unlock(&qm->mailbox_lock); in qm_ping_all_vfs()
1690 qm_trigger_vf_interrupt(qm, QM_IFC_SEND_ALL_VFS); in qm_ping_all_vfs()
1693 val = readq(qm->io_base + QM_IFC_READY_STATUS); in qm_ping_all_vfs()
1696 mutex_unlock(&qm->mailbox_lock); in qm_ping_all_vfs()
1704 mutex_unlock(&qm->mailbox_lock); in qm_ping_all_vfs()
1715 static int qm_ping_pf(struct hisi_qm *qm, u64 cmd) in qm_ping_pf() argument
1723 mutex_lock(&qm->mailbox_lock); in qm_ping_pf()
1724 ret = qm_mb_nolock(qm, &mailbox); in qm_ping_pf()
1726 dev_err(&qm->pdev->dev, "failed to send command to PF!\n"); in qm_ping_pf()
1730 qm_trigger_pf_interrupt(qm); in qm_ping_pf()
1734 val = readl(qm->io_base + QM_IFC_INT_SET_V); in qm_ping_pf()
1745 mutex_unlock(&qm->mailbox_lock); in qm_ping_pf()
1751 return hisi_qm_mb(qp->qm, QM_MB_CMD_STOP_QP, 0, qp->qp_id, 0); in qm_stop_qp()
1754 static int qm_set_msi(struct hisi_qm *qm, bool set) in qm_set_msi() argument
1756 struct pci_dev *pdev = qm->pdev; in qm_set_msi()
1764 if (qm->err_status.is_qm_ecc_mbit || in qm_set_msi()
1765 qm->err_status.is_dev_ecc_mbit) in qm_set_msi()
1769 if (readl(qm->io_base + QM_PEH_DFX_INFO0)) in qm_set_msi()
1776 static void qm_wait_msi_finish(struct hisi_qm *qm) in qm_wait_msi_finish() argument
1778 struct pci_dev *pdev = qm->pdev; in qm_wait_msi_finish()
1798 ret = readl_relaxed_poll_timeout(qm->io_base + QM_PEH_DFX_INFO0, in qm_wait_msi_finish()
1804 ret = readl_relaxed_poll_timeout(qm->io_base + QM_PEH_DFX_INFO1, in qm_wait_msi_finish()
1811 static int qm_set_msi_v3(struct hisi_qm *qm, bool set) in qm_set_msi_v3() argument
1813 struct pci_dev *pdev = qm->pdev; in qm_set_msi_v3()
1834 qm_wait_msi_finish(qm); in qm_set_msi_v3()
1873 return qp->sqe + sq_tail * qp->qm->sqe_size; in qm_get_avail_sqe()
1885 static struct hisi_qp *qm_create_qp_nolock(struct hisi_qm *qm, u8 alg_type) in qm_create_qp_nolock() argument
1887 struct device *dev = &qm->pdev->dev; in qm_create_qp_nolock()
1891 if (!qm_qp_avail_state(qm, NULL, QP_INIT)) in qm_create_qp_nolock()
1894 if (qm->qp_in_used == qm->qp_num) { in qm_create_qp_nolock()
1895 dev_info_ratelimited(dev, "All %u queues of QM are busy!\n", in qm_create_qp_nolock()
1896 qm->qp_num); in qm_create_qp_nolock()
1897 atomic64_inc(&qm->debug.dfx.create_qp_err_cnt); in qm_create_qp_nolock()
1901 qp_id = idr_alloc_cyclic(&qm->qp_idr, NULL, 0, qm->qp_num, GFP_ATOMIC); in qm_create_qp_nolock()
1903 dev_info_ratelimited(dev, "All %u queues of QM are busy!\n", in qm_create_qp_nolock()
1904 qm->qp_num); in qm_create_qp_nolock()
1905 atomic64_inc(&qm->debug.dfx.create_qp_err_cnt); in qm_create_qp_nolock()
1909 qp = &qm->qp_array[qp_id]; in qm_create_qp_nolock()
1918 qm->qp_in_used++; in qm_create_qp_nolock()
1925 * hisi_qm_create_qp() - Create a queue pair from qm.
1926 * @qm: The qm we create a qp from.
1931 static struct hisi_qp *hisi_qm_create_qp(struct hisi_qm *qm, u8 alg_type) in hisi_qm_create_qp() argument
1936 ret = qm_pm_get_sync(qm); in hisi_qm_create_qp()
1940 down_write(&qm->qps_lock); in hisi_qm_create_qp()
1941 qp = qm_create_qp_nolock(qm, alg_type); in hisi_qm_create_qp()
1942 up_write(&qm->qps_lock); in hisi_qm_create_qp()
1945 qm_pm_put_sync(qm); in hisi_qm_create_qp()
1951 * hisi_qm_release_qp() - Release a qp back to its qm.
1958 struct hisi_qm *qm = qp->qm; in hisi_qm_release_qp() local
1960 down_write(&qm->qps_lock); in hisi_qm_release_qp()
1962 if (!qm_qp_avail_state(qm, qp, QP_CLOSE)) { in hisi_qm_release_qp()
1963 up_write(&qm->qps_lock); in hisi_qm_release_qp()
1967 qm->qp_in_used--; in hisi_qm_release_qp()
1968 idr_remove(&qm->qp_idr, qp->qp_id); in hisi_qm_release_qp()
1970 up_write(&qm->qps_lock); in hisi_qm_release_qp()
1972 qm_pm_put_sync(qm); in hisi_qm_release_qp()
1977 struct hisi_qm *qm = qp->qm; in qm_sq_ctx_cfg() local
1978 struct device *dev = &qm->pdev->dev; in qm_sq_ctx_cfg()
1979 enum qm_hw_ver ver = qm->ver; in qm_sq_ctx_cfg()
1990 sqc->dw3 = cpu_to_le32(QM_MK_SQC_DW3_V1(0, 0, 0, qm->sqe_size)); in qm_sq_ctx_cfg()
1993 sqc->dw3 = cpu_to_le32(QM_MK_SQC_DW3_V2(qm->sqe_size, qp->sq_depth)); in qm_sq_ctx_cfg()
1999 if (ver >= QM_HW_V3 && qm->use_sva && !qp->is_in_kernel) in qm_sq_ctx_cfg()
2010 ret = hisi_qm_mb(qm, QM_MB_CMD_SQC, sqc_dma, qp_id, 0); in qm_sq_ctx_cfg()
2019 struct hisi_qm *qm = qp->qm; in qm_cq_ctx_cfg() local
2020 struct device *dev = &qm->pdev->dev; in qm_cq_ctx_cfg()
2021 enum qm_hw_ver ver = qm->ver; in qm_cq_ctx_cfg()
2041 if (ver >= QM_HW_V3 && qm->use_sva && !qp->is_in_kernel) in qm_cq_ctx_cfg()
2051 ret = hisi_qm_mb(qm, QM_MB_CMD_CQC, cqc_dma, qp_id, 0); in qm_cq_ctx_cfg()
2073 struct hisi_qm *qm = qp->qm; in qm_start_qp_nolock() local
2074 struct device *dev = &qm->pdev->dev; in qm_start_qp_nolock()
2079 if (!qm_qp_avail_state(qm, qp, QP_START)) in qm_start_qp_nolock()
2102 struct hisi_qm *qm = qp->qm; in hisi_qm_start_qp() local
2105 down_write(&qm->qps_lock); in hisi_qm_start_qp()
2107 up_write(&qm->qps_lock); in hisi_qm_start_qp()
2125 struct hisi_qm *qm = qp->qm; in qp_stop_fail_cb() local
2131 qp->req_cb(qp, qp->sqe + (u32)(qm->sqe_size * pos)); in qp_stop_fail_cb()
2146 struct hisi_qm *qm = qp->qm; in qm_drain_qp() local
2147 struct device *dev = &qm->pdev->dev; in qm_drain_qp()
2155 if (qm_check_dev_error(qm)) in qm_drain_qp()
2159 if (test_bit(QM_SUPPORT_STOP_QP, &qm->caps)) { in qm_drain_qp()
2166 addr = hisi_qm_ctx_alloc(qm, size, &dma_addr); in qm_drain_qp()
2173 ret = qm_dump_sqc_raw(qm, dma_addr, qp->qp_id); in qm_drain_qp()
2180 ret = qm_dump_cqc_raw(qm, (dma_addr + sizeof(struct qm_sqc)), in qm_drain_qp()
2201 hisi_qm_ctx_free(qm, size, addr, &dma_addr); in qm_drain_qp()
2208 struct device *dev = &qp->qm->pdev->dev; in qm_stop_qp_nolock()
2222 if (!qm_qp_avail_state(qp->qm, qp, QP_STOP)) in qm_stop_qp_nolock()
2232 flush_workqueue(qp->qm->wq); in qm_stop_qp_nolock()
2242 * hisi_qm_stop_qp() - Stop a qp in qm.
2251 down_write(&qp->qm->qps_lock); in hisi_qm_stop_qp()
2253 up_write(&qp->qm->qps_lock); in hisi_qm_stop_qp()
2265 * if qp related qm is resetting.
2270 * causes current qm_db sending fail or can not receive sended sqe. QM
2282 atomic_read(&qp->qm->status.flags) == QM_STOP || in hisi_qp_send()
2284 dev_info_ratelimited(&qp->qm->pdev->dev, "QP is stopped or resetting\n"); in hisi_qp_send()
2291 memcpy(sqe, msg, qp->qm->sqe_size); in hisi_qp_send()
2293 qm_db(qp->qm, qp->qp_id, QM_DOORBELL_CMD_SQ, sq_tail_next, 0); in hisi_qp_send()
2301 static void hisi_qm_cache_wb(struct hisi_qm *qm) in hisi_qm_cache_wb() argument
2305 if (qm->ver == QM_HW_V1) in hisi_qm_cache_wb()
2308 writel(0x1, qm->io_base + QM_CACHE_WB_START); in hisi_qm_cache_wb()
2309 if (readl_relaxed_poll_timeout(qm->io_base + QM_CACHE_WB_DONE, in hisi_qm_cache_wb()
2312 dev_err(&qm->pdev->dev, "QM writeback sqc cache fail!\n"); in hisi_qm_cache_wb()
2320 /* This function returns free number of qp in qm. */
2323 struct hisi_qm *qm = uacce->priv; in hisi_qm_get_available_instances() local
2326 down_read(&qm->qps_lock); in hisi_qm_get_available_instances()
2327 ret = qm->qp_num - qm->qp_in_used; in hisi_qm_get_available_instances()
2328 up_read(&qm->qps_lock); in hisi_qm_get_available_instances()
2333 static void hisi_qm_set_hw_reset(struct hisi_qm *qm, int offset) in hisi_qm_set_hw_reset() argument
2337 for (i = 0; i < qm->qp_num; i++) in hisi_qm_set_hw_reset()
2338 qm_set_qp_disable(&qm->qp_array[i], offset); in hisi_qm_set_hw_reset()
2345 struct hisi_qm *qm = uacce->priv; in hisi_qm_uacce_get_queue() local
2349 qp = hisi_qm_create_qp(qm, alg_type); in hisi_qm_uacce_get_queue()
2376 struct hisi_qm *qm = qp->qm; in hisi_qm_uacce_mmap() local
2377 resource_size_t phys_base = qm->db_phys_base + in hisi_qm_uacce_mmap()
2378 qp->qp_id * qm->db_interval; in hisi_qm_uacce_mmap()
2380 struct pci_dev *pdev = qm->pdev; in hisi_qm_uacce_mmap()
2387 if (qm->ver == QM_HW_V1) { in hisi_qm_uacce_mmap()
2390 } else if (!test_bit(QM_SUPPORT_DB_ISOLATION, &qm->caps)) { in hisi_qm_uacce_mmap()
2395 if (sz > qm->db_interval) in hisi_qm_uacce_mmap()
2455 struct hisi_qm *qm = q->uacce->priv; in qm_set_sqctype() local
2458 down_write(&qm->qps_lock); in qm_set_sqctype()
2460 up_write(&qm->qps_lock); in qm_set_sqctype()
2491 qp_info.sqe_size = qp->qm->sqe_size; in hisi_qm_uacce_ioctl()
2508 * @qm: the uacce device
2510 static int qm_hw_err_isolate(struct hisi_qm *qm) in qm_hw_err_isolate() argument
2516 isolate = &qm->isolate_data; in qm_hw_err_isolate()
2521 if (qm->uacce->is_vf || isolate->is_isolate || !isolate->err_threshold) in qm_hw_err_isolate()
2554 static void qm_hw_err_destroy(struct hisi_qm *qm) in qm_hw_err_destroy() argument
2558 mutex_lock(&qm->isolate_data.isolate_lock); in qm_hw_err_destroy()
2559 list_for_each_entry_safe(err, tmp, &qm->isolate_data.qm_hw_errs, list) { in qm_hw_err_destroy()
2563 mutex_unlock(&qm->isolate_data.isolate_lock); in qm_hw_err_destroy()
2568 struct hisi_qm *qm = uacce->priv; in hisi_qm_get_isolate_state() local
2572 pf_qm = pci_get_drvdata(pci_physfn(qm->pdev)); in hisi_qm_get_isolate_state()
2574 pf_qm = qm; in hisi_qm_get_isolate_state()
2582 struct hisi_qm *qm = uacce->priv; in hisi_qm_isolate_threshold_write() local
2588 if (qm->isolate_data.is_isolate) in hisi_qm_isolate_threshold_write()
2591 qm->isolate_data.err_threshold = num; in hisi_qm_isolate_threshold_write()
2594 qm_hw_err_destroy(qm); in hisi_qm_isolate_threshold_write()
2601 struct hisi_qm *qm = uacce->priv; in hisi_qm_isolate_threshold_read() local
2605 pf_qm = pci_get_drvdata(pci_physfn(qm->pdev)); in hisi_qm_isolate_threshold_read()
2609 return qm->isolate_data.err_threshold; in hisi_qm_isolate_threshold_read()
2626 static void qm_remove_uacce(struct hisi_qm *qm) in qm_remove_uacce() argument
2628 struct uacce_device *uacce = qm->uacce; in qm_remove_uacce()
2630 if (qm->use_sva) { in qm_remove_uacce()
2631 qm_hw_err_destroy(qm); in qm_remove_uacce()
2633 qm->uacce = NULL; in qm_remove_uacce()
2637 static int qm_alloc_uacce(struct hisi_qm *qm) in qm_alloc_uacce() argument
2639 struct pci_dev *pdev = qm->pdev; in qm_alloc_uacce()
2660 qm->use_sva = true; in qm_alloc_uacce()
2663 qm_remove_uacce(qm); in qm_alloc_uacce()
2668 uacce->priv = qm; in qm_alloc_uacce()
2670 if (qm->ver == QM_HW_V1) in qm_alloc_uacce()
2672 else if (qm->ver == QM_HW_V2) in qm_alloc_uacce()
2677 if (qm->ver == QM_HW_V1) in qm_alloc_uacce()
2679 else if (!test_bit(QM_SUPPORT_DB_ISOLATION, &qm->caps)) in qm_alloc_uacce()
2683 mmio_page_nr = qm->db_interval / PAGE_SIZE; in qm_alloc_uacce()
2685 qm_get_xqc_depth(qm, &sq_depth, &cq_depth, QM_QP_DEPTH_CAP); in qm_alloc_uacce()
2688 dus_page_nr = (PAGE_SIZE - 1 + qm->sqe_size * sq_depth + in qm_alloc_uacce()
2695 qm->uacce = uacce; in qm_alloc_uacce()
2696 INIT_LIST_HEAD(&qm->isolate_data.qm_hw_errs); in qm_alloc_uacce()
2697 mutex_init(&qm->isolate_data.isolate_lock); in qm_alloc_uacce()
2703 * qm_frozen() - Try to froze QM to cut continuous queue request. If
2704 * there is user on the QM, return failure without doing anything.
2705 * @qm: The qm needed to be fronzen.
2707 * This function frozes QM, then we can do SRIOV disabling.
2709 static int qm_frozen(struct hisi_qm *qm) in qm_frozen() argument
2711 if (test_bit(QM_DRIVER_REMOVING, &qm->misc_ctl)) in qm_frozen()
2714 down_write(&qm->qps_lock); in qm_frozen()
2716 if (!qm->qp_in_used) { in qm_frozen()
2717 qm->qp_in_used = qm->qp_num; in qm_frozen()
2718 up_write(&qm->qps_lock); in qm_frozen()
2719 set_bit(QM_DRIVER_REMOVING, &qm->misc_ctl); in qm_frozen()
2723 up_write(&qm->qps_lock); in qm_frozen()
2731 struct hisi_qm *qm, *vf_qm; in qm_try_frozen_vfs() local
2740 list_for_each_entry(qm, &qm_list->list, list) { in qm_try_frozen_vfs()
2741 dev = qm->pdev; in qm_try_frozen_vfs()
2761 * @qm: The qm needed to wait for the task to finish.
2764 void hisi_qm_wait_task_finish(struct hisi_qm *qm, struct hisi_qm_list *qm_list) in hisi_qm_wait_task_finish() argument
2766 while (qm_frozen(qm) || in hisi_qm_wait_task_finish()
2767 ((qm->fun_type == QM_HW_PF) && in hisi_qm_wait_task_finish()
2768 qm_try_frozen_vfs(qm->pdev, qm_list))) { in hisi_qm_wait_task_finish()
2772 while (test_bit(QM_RST_SCHED, &qm->misc_ctl) || in hisi_qm_wait_task_finish()
2773 test_bit(QM_RESETTING, &qm->misc_ctl)) in hisi_qm_wait_task_finish()
2776 if (test_bit(QM_SUPPORT_MB_COMMAND, &qm->caps)) in hisi_qm_wait_task_finish()
2777 flush_work(&qm->cmd_process); in hisi_qm_wait_task_finish()
2783 static void hisi_qp_memory_uninit(struct hisi_qm *qm, int num) in hisi_qp_memory_uninit() argument
2785 struct device *dev = &qm->pdev->dev; in hisi_qp_memory_uninit()
2790 qdma = &qm->qp_array[i].qdma; in hisi_qp_memory_uninit()
2792 kfree(qm->poll_data[i].qp_finish_id); in hisi_qp_memory_uninit()
2795 kfree(qm->poll_data); in hisi_qp_memory_uninit()
2796 kfree(qm->qp_array); in hisi_qp_memory_uninit()
2799 static int hisi_qp_memory_init(struct hisi_qm *qm, size_t dma_size, int id, in hisi_qp_memory_init() argument
2802 struct device *dev = &qm->pdev->dev; in hisi_qp_memory_init()
2803 size_t off = qm->sqe_size * sq_depth; in hisi_qp_memory_init()
2807 qm->poll_data[id].qp_finish_id = kcalloc(qm->qp_num, sizeof(u16), in hisi_qp_memory_init()
2809 if (!qm->poll_data[id].qp_finish_id) in hisi_qp_memory_init()
2812 qp = &qm->qp_array[id]; in hisi_qp_memory_init()
2825 qp->qm = qm; in hisi_qp_memory_init()
2831 kfree(qm->poll_data[id].qp_finish_id); in hisi_qp_memory_init()
2835 static void hisi_qm_pre_init(struct hisi_qm *qm) in hisi_qm_pre_init() argument
2837 struct pci_dev *pdev = qm->pdev; in hisi_qm_pre_init()
2839 if (qm->ver == QM_HW_V1) in hisi_qm_pre_init()
2840 qm->ops = &qm_hw_ops_v1; in hisi_qm_pre_init()
2841 else if (qm->ver == QM_HW_V2) in hisi_qm_pre_init()
2842 qm->ops = &qm_hw_ops_v2; in hisi_qm_pre_init()
2844 qm->ops = &qm_hw_ops_v3; in hisi_qm_pre_init()
2846 pci_set_drvdata(pdev, qm); in hisi_qm_pre_init()
2847 mutex_init(&qm->mailbox_lock); in hisi_qm_pre_init()
2848 init_rwsem(&qm->qps_lock); in hisi_qm_pre_init()
2849 qm->qp_in_used = 0; in hisi_qm_pre_init()
2850 if (test_bit(QM_SUPPORT_RPM, &qm->caps)) { in hisi_qm_pre_init()
2856 static void qm_cmd_uninit(struct hisi_qm *qm) in qm_cmd_uninit() argument
2860 if (!test_bit(QM_SUPPORT_MB_COMMAND, &qm->caps)) in qm_cmd_uninit()
2863 val = readl(qm->io_base + QM_IFC_INT_MASK); in qm_cmd_uninit()
2865 writel(val, qm->io_base + QM_IFC_INT_MASK); in qm_cmd_uninit()
2868 static void qm_cmd_init(struct hisi_qm *qm) in qm_cmd_init() argument
2872 if (!test_bit(QM_SUPPORT_MB_COMMAND, &qm->caps)) in qm_cmd_init()
2876 qm_clear_cmd_interrupt(qm, QM_IFC_INT_SOURCE_CLR); in qm_cmd_init()
2879 val = readl(qm->io_base + QM_IFC_INT_MASK); in qm_cmd_init()
2881 writel(val, qm->io_base + QM_IFC_INT_MASK); in qm_cmd_init()
2884 static void qm_put_pci_res(struct hisi_qm *qm) in qm_put_pci_res() argument
2886 struct pci_dev *pdev = qm->pdev; in qm_put_pci_res()
2888 if (test_bit(QM_SUPPORT_DB_ISOLATION, &qm->caps)) in qm_put_pci_res()
2889 iounmap(qm->db_io_base); in qm_put_pci_res()
2891 iounmap(qm->io_base); in qm_put_pci_res()
2895 static void hisi_qm_pci_uninit(struct hisi_qm *qm) in hisi_qm_pci_uninit() argument
2897 struct pci_dev *pdev = qm->pdev; in hisi_qm_pci_uninit()
2900 qm_put_pci_res(qm); in hisi_qm_pci_uninit()
2904 static void hisi_qm_set_state(struct hisi_qm *qm, u8 state) in hisi_qm_set_state() argument
2906 if (qm->ver > QM_HW_V2 && qm->fun_type == QM_HW_VF) in hisi_qm_set_state()
2907 writel(state, qm->io_base + QM_VF_STATE); in hisi_qm_set_state()
2910 static void hisi_qm_unint_work(struct hisi_qm *qm) in hisi_qm_unint_work() argument
2912 destroy_workqueue(qm->wq); in hisi_qm_unint_work()
2915 static void hisi_qm_memory_uninit(struct hisi_qm *qm) in hisi_qm_memory_uninit() argument
2917 struct device *dev = &qm->pdev->dev; in hisi_qm_memory_uninit()
2919 hisi_qp_memory_uninit(qm, qm->qp_num); in hisi_qm_memory_uninit()
2920 if (qm->qdma.va) { in hisi_qm_memory_uninit()
2921 hisi_qm_cache_wb(qm); in hisi_qm_memory_uninit()
2922 dma_free_coherent(dev, qm->qdma.size, in hisi_qm_memory_uninit()
2923 qm->qdma.va, qm->qdma.dma); in hisi_qm_memory_uninit()
2926 idr_destroy(&qm->qp_idr); in hisi_qm_memory_uninit()
2928 if (test_bit(QM_SUPPORT_FUNC_QOS, &qm->caps)) in hisi_qm_memory_uninit()
2929 kfree(qm->factor); in hisi_qm_memory_uninit()
2933 * hisi_qm_uninit() - Uninitialize qm.
2934 * @qm: The qm needed uninit.
2936 * This function uninits qm related device resources.
2938 void hisi_qm_uninit(struct hisi_qm *qm) in hisi_qm_uninit() argument
2940 qm_cmd_uninit(qm); in hisi_qm_uninit()
2941 hisi_qm_unint_work(qm); in hisi_qm_uninit()
2942 down_write(&qm->qps_lock); in hisi_qm_uninit()
2944 if (!qm_avail_state(qm, QM_CLOSE)) { in hisi_qm_uninit()
2945 up_write(&qm->qps_lock); in hisi_qm_uninit()
2949 hisi_qm_memory_uninit(qm); in hisi_qm_uninit()
2950 hisi_qm_set_state(qm, QM_NOT_READY); in hisi_qm_uninit()
2951 up_write(&qm->qps_lock); in hisi_qm_uninit()
2953 qm_remove_uacce(qm); in hisi_qm_uninit()
2954 qm_irqs_unregister(qm); in hisi_qm_uninit()
2955 hisi_qm_pci_uninit(qm); in hisi_qm_uninit()
2960 * hisi_qm_get_vft() - Get vft from a qm.
2961 * @qm: The qm we want to get its vft.
2965 * We can allocate multiple queues to a qm by configuring virtual function
2969 * qm hw v1 does not support this interface.
2971 static int hisi_qm_get_vft(struct hisi_qm *qm, u32 *base, u32 *number) in hisi_qm_get_vft() argument
2976 if (!qm->ops->get_vft) { in hisi_qm_get_vft()
2977 dev_err(&qm->pdev->dev, "Don't support vft read!\n"); in hisi_qm_get_vft()
2981 return qm->ops->get_vft(qm, base, number); in hisi_qm_get_vft()
2985 * hisi_qm_set_vft() - Set vft to a qm.
2986 * @qm: The qm we want to set its vft.
2994 * Assign queues A~B to PF: hisi_qm_set_vft(qm, 0, A, B - A + 1)
2995 * Assign queues A~B to VF: hisi_qm_set_vft(qm, 2, A, B - A + 1)
2998 static int hisi_qm_set_vft(struct hisi_qm *qm, u32 fun_num, u32 base, in hisi_qm_set_vft() argument
3001 u32 max_q_num = qm->ctrl_qp_num; in hisi_qm_set_vft()
3007 return qm_set_sqc_cqc_vft(qm, fun_num, base, number); in hisi_qm_set_vft()
3010 static void qm_init_eq_aeq_status(struct hisi_qm *qm) in qm_init_eq_aeq_status() argument
3012 struct hisi_qm_status *status = &qm->status; in qm_init_eq_aeq_status()
3020 static void qm_enable_eq_aeq_interrupts(struct hisi_qm *qm) in qm_enable_eq_aeq_interrupts() argument
3023 qm_db(qm, 0, QM_DOORBELL_CMD_AEQ, qm->status.aeq_head, 0); in qm_enable_eq_aeq_interrupts()
3024 qm_db(qm, 0, QM_DOORBELL_CMD_EQ, qm->status.eq_head, 0); in qm_enable_eq_aeq_interrupts()
3026 writel(0x0, qm->io_base + QM_VF_EQ_INT_MASK); in qm_enable_eq_aeq_interrupts()
3027 writel(0x0, qm->io_base + QM_VF_AEQ_INT_MASK); in qm_enable_eq_aeq_interrupts()
3030 static void qm_disable_eq_aeq_interrupts(struct hisi_qm *qm) in qm_disable_eq_aeq_interrupts() argument
3032 writel(0x1, qm->io_base + QM_VF_EQ_INT_MASK); in qm_disable_eq_aeq_interrupts()
3033 writel(0x1, qm->io_base + QM_VF_AEQ_INT_MASK); in qm_disable_eq_aeq_interrupts()
3036 static int qm_eq_ctx_cfg(struct hisi_qm *qm) in qm_eq_ctx_cfg() argument
3038 struct device *dev = &qm->pdev->dev; in qm_eq_ctx_cfg()
3047 eqc->base_l = cpu_to_le32(lower_32_bits(qm->eqe_dma)); in qm_eq_ctx_cfg()
3048 eqc->base_h = cpu_to_le32(upper_32_bits(qm->eqe_dma)); in qm_eq_ctx_cfg()
3049 if (qm->ver == QM_HW_V1) in qm_eq_ctx_cfg()
3051 eqc->dw6 = cpu_to_le32(((u32)qm->eq_depth - 1) | (1 << QM_EQC_PHASE_SHIFT)); in qm_eq_ctx_cfg()
3060 ret = hisi_qm_mb(qm, QM_MB_CMD_EQC, eqc_dma, 0, 0); in qm_eq_ctx_cfg()
3067 static int qm_aeq_ctx_cfg(struct hisi_qm *qm) in qm_aeq_ctx_cfg() argument
3069 struct device *dev = &qm->pdev->dev; in qm_aeq_ctx_cfg()
3078 aeqc->base_l = cpu_to_le32(lower_32_bits(qm->aeqe_dma)); in qm_aeq_ctx_cfg()
3079 aeqc->base_h = cpu_to_le32(upper_32_bits(qm->aeqe_dma)); in qm_aeq_ctx_cfg()
3080 aeqc->dw6 = cpu_to_le32(((u32)qm->aeq_depth - 1) | (1 << QM_EQC_PHASE_SHIFT)); in qm_aeq_ctx_cfg()
3089 ret = hisi_qm_mb(qm, QM_MB_CMD_AEQC, aeqc_dma, 0, 0); in qm_aeq_ctx_cfg()
3096 static int qm_eq_aeq_ctx_cfg(struct hisi_qm *qm) in qm_eq_aeq_ctx_cfg() argument
3098 struct device *dev = &qm->pdev->dev; in qm_eq_aeq_ctx_cfg()
3101 qm_init_eq_aeq_status(qm); in qm_eq_aeq_ctx_cfg()
3103 ret = qm_eq_ctx_cfg(qm); in qm_eq_aeq_ctx_cfg()
3109 return qm_aeq_ctx_cfg(qm); in qm_eq_aeq_ctx_cfg()
3112 static int __hisi_qm_start(struct hisi_qm *qm) in __hisi_qm_start() argument
3116 WARN_ON(!qm->qdma.va); in __hisi_qm_start()
3118 if (qm->fun_type == QM_HW_PF) { in __hisi_qm_start()
3119 ret = hisi_qm_set_vft(qm, 0, qm->qp_base, qm->qp_num); in __hisi_qm_start()
3124 ret = qm_eq_aeq_ctx_cfg(qm); in __hisi_qm_start()
3128 ret = hisi_qm_mb(qm, QM_MB_CMD_SQC_BT, qm->sqc_dma, 0, 0); in __hisi_qm_start()
3132 ret = hisi_qm_mb(qm, QM_MB_CMD_CQC_BT, qm->cqc_dma, 0, 0); in __hisi_qm_start()
3136 qm_init_prefetch(qm); in __hisi_qm_start()
3137 qm_enable_eq_aeq_interrupts(qm); in __hisi_qm_start()
3143 * hisi_qm_start() - start qm
3144 * @qm: The qm to be started.
3146 * This function starts a qm, then we can allocate qp from this qm.
3148 int hisi_qm_start(struct hisi_qm *qm) in hisi_qm_start() argument
3150 struct device *dev = &qm->pdev->dev; in hisi_qm_start()
3153 down_write(&qm->qps_lock); in hisi_qm_start()
3155 if (!qm_avail_state(qm, QM_START)) { in hisi_qm_start()
3156 up_write(&qm->qps_lock); in hisi_qm_start()
3160 dev_dbg(dev, "qm start with %u queue pairs\n", qm->qp_num); in hisi_qm_start()
3162 if (!qm->qp_num) { in hisi_qm_start()
3168 ret = __hisi_qm_start(qm); in hisi_qm_start()
3170 atomic_set(&qm->status.flags, QM_START); in hisi_qm_start()
3172 hisi_qm_set_state(qm, QM_READY); in hisi_qm_start()
3174 up_write(&qm->qps_lock); in hisi_qm_start()
3179 static int qm_restart(struct hisi_qm *qm) in qm_restart() argument
3181 struct device *dev = &qm->pdev->dev; in qm_restart()
3185 ret = hisi_qm_start(qm); in qm_restart()
3189 down_write(&qm->qps_lock); in qm_restart()
3190 for (i = 0; i < qm->qp_num; i++) { in qm_restart()
3191 qp = &qm->qp_array[i]; in qm_restart()
3198 up_write(&qm->qps_lock); in qm_restart()
3204 up_write(&qm->qps_lock); in qm_restart()
3210 static int qm_stop_started_qp(struct hisi_qm *qm) in qm_stop_started_qp() argument
3212 struct device *dev = &qm->pdev->dev; in qm_stop_started_qp()
3216 for (i = 0; i < qm->qp_num; i++) { in qm_stop_started_qp()
3217 qp = &qm->qp_array[i]; in qm_stop_started_qp()
3232 * qm_clear_queues() - Clear all queues memory in a qm.
3233 * @qm: The qm in which the queues will be cleared.
3235 * This function clears all queues memory in a qm. Reset of accelerator can
3238 static void qm_clear_queues(struct hisi_qm *qm) in qm_clear_queues() argument
3243 for (i = 0; i < qm->qp_num; i++) { in qm_clear_queues()
3244 qp = &qm->qp_array[i]; in qm_clear_queues()
3249 memset(qm->qdma.va, 0, qm->qdma.size); in qm_clear_queues()
3253 * hisi_qm_stop() - Stop a qm.
3254 * @qm: The qm which will be stopped.
3255 * @r: The reason to stop qm.
3257 * This function stops qm and its qps, then qm can not accept request.
3259 * to let qm start again.
3261 int hisi_qm_stop(struct hisi_qm *qm, enum qm_stop_reason r) in hisi_qm_stop() argument
3263 struct device *dev = &qm->pdev->dev; in hisi_qm_stop()
3266 down_write(&qm->qps_lock); in hisi_qm_stop()
3268 qm->status.stop_reason = r; in hisi_qm_stop()
3269 if (!qm_avail_state(qm, QM_STOP)) { in hisi_qm_stop()
3274 if (qm->status.stop_reason == QM_SOFT_RESET || in hisi_qm_stop()
3275 qm->status.stop_reason == QM_DOWN) { in hisi_qm_stop()
3276 hisi_qm_set_hw_reset(qm, QM_RESET_STOP_TX_OFFSET); in hisi_qm_stop()
3277 ret = qm_stop_started_qp(qm); in hisi_qm_stop()
3282 hisi_qm_set_hw_reset(qm, QM_RESET_STOP_RX_OFFSET); in hisi_qm_stop()
3285 qm_disable_eq_aeq_interrupts(qm); in hisi_qm_stop()
3286 if (qm->fun_type == QM_HW_PF) { in hisi_qm_stop()
3287 ret = hisi_qm_set_vft(qm, 0, 0, 0); in hisi_qm_stop()
3295 qm_clear_queues(qm); in hisi_qm_stop()
3296 atomic_set(&qm->status.flags, QM_STOP); in hisi_qm_stop()
3299 up_write(&qm->qps_lock); in hisi_qm_stop()
3304 static void qm_hw_error_init(struct hisi_qm *qm) in qm_hw_error_init() argument
3306 if (!qm->ops->hw_error_init) { in qm_hw_error_init()
3307 dev_err(&qm->pdev->dev, "QM doesn't support hw error handling!\n"); in qm_hw_error_init()
3311 qm->ops->hw_error_init(qm); in qm_hw_error_init()
3314 static void qm_hw_error_uninit(struct hisi_qm *qm) in qm_hw_error_uninit() argument
3316 if (!qm->ops->hw_error_uninit) { in qm_hw_error_uninit()
3317 dev_err(&qm->pdev->dev, "Unexpected QM hw error uninit!\n"); in qm_hw_error_uninit()
3321 qm->ops->hw_error_uninit(qm); in qm_hw_error_uninit()
3324 static enum acc_err_result qm_hw_error_handle(struct hisi_qm *qm) in qm_hw_error_handle() argument
3326 if (!qm->ops->hw_error_handle) { in qm_hw_error_handle()
3327 dev_err(&qm->pdev->dev, "QM doesn't support hw error report!\n"); in qm_hw_error_handle()
3331 return qm->ops->hw_error_handle(qm); in qm_hw_error_handle()
3336 * @qm: The qm for which we want to do error initialization.
3338 * Initialize QM and device error related configuration.
3340 void hisi_qm_dev_err_init(struct hisi_qm *qm) in hisi_qm_dev_err_init() argument
3342 if (qm->fun_type == QM_HW_VF) in hisi_qm_dev_err_init()
3345 qm_hw_error_init(qm); in hisi_qm_dev_err_init()
3347 if (!qm->err_ini->hw_err_enable) { in hisi_qm_dev_err_init()
3348 dev_err(&qm->pdev->dev, "Device doesn't support hw error init!\n"); in hisi_qm_dev_err_init()
3351 qm->err_ini->hw_err_enable(qm); in hisi_qm_dev_err_init()
3357 * @qm: The qm for which we want to do error uninitialization.
3359 * Uninitialize QM and device error related configuration.
3361 void hisi_qm_dev_err_uninit(struct hisi_qm *qm) in hisi_qm_dev_err_uninit() argument
3363 if (qm->fun_type == QM_HW_VF) in hisi_qm_dev_err_uninit()
3366 qm_hw_error_uninit(qm); in hisi_qm_dev_err_uninit()
3368 if (!qm->err_ini->hw_err_disable) { in hisi_qm_dev_err_uninit()
3369 dev_err(&qm->pdev->dev, "Unexpected device hw error uninit!\n"); in hisi_qm_dev_err_uninit()
3372 qm->err_ini->hw_err_disable(qm); in hisi_qm_dev_err_uninit()
3407 struct hisi_qm *qm; in hisi_qm_sort_devices() local
3412 list_for_each_entry(qm, &qm_list->list, list) { in hisi_qm_sort_devices()
3413 dev = &qm->pdev->dev; in hisi_qm_sort_devices()
3423 res->qm = qm; in hisi_qm_sort_devices()
3469 qps[i] = hisi_qm_create_qp(tmp->qm, alg_type); in hisi_qm_alloc_qps_node()
3493 static int qm_vf_q_assign(struct hisi_qm *qm, u32 num_vfs) in qm_vf_q_assign() argument
3496 u32 max_qp_num = qm->max_qp_num; in qm_vf_q_assign()
3497 u32 q_base = qm->qp_num; in qm_vf_q_assign()
3503 vfs_q_num = qm->ctrl_qp_num - qm->qp_num; in qm_vf_q_assign()
3528 ret = hisi_qm_set_vft(qm, i, q_base, act_q_num); in qm_vf_q_assign()
3531 hisi_qm_set_vft(qm, j, 0, 0); in qm_vf_q_assign()
3540 static int qm_clear_vft_config(struct hisi_qm *qm) in qm_clear_vft_config() argument
3545 for (i = 1; i <= qm->vfs_num; i++) { in qm_clear_vft_config()
3546 ret = hisi_qm_set_vft(qm, i, 0, 0); in qm_clear_vft_config()
3550 qm->vfs_num = 0; in qm_clear_vft_config()
3555 static int qm_func_shaper_enable(struct hisi_qm *qm, u32 fun_index, u32 qos) in qm_func_shaper_enable() argument
3557 struct device *dev = &qm->pdev->dev; in qm_func_shaper_enable()
3561 total_vfs = pci_sriov_get_totalvfs(qm->pdev); in qm_func_shaper_enable()
3565 qm->factor[fun_index].func_qos = qos; in qm_func_shaper_enable()
3567 ret = qm_get_shaper_para(ir, &qm->factor[fun_index]); in qm_func_shaper_enable()
3575 ret = qm_set_vft_common(qm, SHAPER_VFT, fun_index, i, 1); in qm_func_shaper_enable()
3585 static u32 qm_get_shaper_vft_qos(struct hisi_qm *qm, u32 fun_index) in qm_get_shaper_vft_qos() argument
3593 ret = readl_relaxed_poll_timeout(qm->io_base + QM_VFT_CFG_RDY, val, in qm_get_shaper_vft_qos()
3599 writel(0x1, qm->io_base + QM_VFT_CFG_OP_WR); in qm_get_shaper_vft_qos()
3600 writel(SHAPER_VFT, qm->io_base + QM_VFT_CFG_TYPE); in qm_get_shaper_vft_qos()
3601 writel(fun_index, qm->io_base + QM_VFT_CFG); in qm_get_shaper_vft_qos()
3603 writel(0x0, qm->io_base + QM_VFT_CFG_RDY); in qm_get_shaper_vft_qos()
3604 writel(0x1, qm->io_base + QM_VFT_CFG_OP_ENABLE); in qm_get_shaper_vft_qos()
3606 ret = readl_relaxed_poll_timeout(qm->io_base + QM_VFT_CFG_RDY, val, in qm_get_shaper_vft_qos()
3612 shaper_vft = readl(qm->io_base + QM_VFT_CFG_DATA_L) | in qm_get_shaper_vft_qos()
3613 ((u64)readl(qm->io_base + QM_VFT_CFG_DATA_H) << 32); in qm_get_shaper_vft_qos()
3624 ir = qm->factor[fun_index].func_qos * QM_QOS_RATE; in qm_get_shaper_vft_qos()
3628 pci_err(qm->pdev, "error_rate: %u, get function qos is error!\n", error_rate); in qm_get_shaper_vft_qos()
3635 static void qm_vf_get_qos(struct hisi_qm *qm, u32 fun_num) in qm_vf_get_qos() argument
3637 struct device *dev = &qm->pdev->dev; in qm_vf_get_qos()
3642 qos = qm_get_shaper_vft_qos(qm, fun_num); in qm_vf_get_qos()
3649 ret = qm_ping_single_vf(qm, mb_cmd, fun_num); in qm_vf_get_qos()
3654 static int qm_vf_read_qos(struct hisi_qm *qm) in qm_vf_read_qos() argument
3660 qm->mb_qos = 0; in qm_vf_read_qos()
3663 ret = qm_ping_pf(qm, QM_VF_GET_QOS); in qm_vf_read_qos()
3665 pci_err(qm->pdev, "failed to send cmd to PF to get qos!\n"); in qm_vf_read_qos()
3671 if (qm->mb_qos) in qm_vf_read_qos()
3675 pci_err(qm->pdev, "PF ping VF timeout!\n"); in qm_vf_read_qos()
3686 struct hisi_qm *qm = filp->private_data; in qm_algqos_read() local
3691 ret = hisi_qm_get_dfx_access(qm); in qm_algqos_read()
3696 if (test_and_set_bit(QM_RESETTING, &qm->misc_ctl)) { in qm_algqos_read()
3697 pci_err(qm->pdev, "dev resetting, read alg qos failed!\n"); in qm_algqos_read()
3702 if (qm->fun_type == QM_HW_PF) { in qm_algqos_read()
3703 ir = qm_get_shaper_vft_qos(qm, 0); in qm_algqos_read()
3705 ret = qm_vf_read_qos(qm); in qm_algqos_read()
3708 ir = qm->mb_qos; in qm_algqos_read()
3717 clear_bit(QM_RESETTING, &qm->misc_ctl); in qm_algqos_read()
3719 hisi_qm_put_dfx_access(qm); in qm_algqos_read()
3723 static ssize_t qm_get_qos_value(struct hisi_qm *qm, const char *buf, in qm_get_qos_value() argument
3727 const struct bus_type *bus_type = qm->pdev->dev.bus; in qm_get_qos_value()
3740 pci_err(qm->pdev, "input qos value is error, please set 1~1000!\n"); in qm_get_qos_value()
3746 pci_err(qm->pdev, "input pci bdf number is error!\n"); in qm_get_qos_value()
3760 struct hisi_qm *qm = filp->private_data; in qm_algqos_write() local
3777 ret = qm_get_qos_value(qm, tbuf, &val, &fun_index); in qm_algqos_write()
3782 if (test_and_set_bit(QM_RESETTING, &qm->misc_ctl)) { in qm_algqos_write()
3783 pci_err(qm->pdev, "dev resetting, write alg qos failed!\n"); in qm_algqos_write()
3787 ret = qm_pm_get_sync(qm); in qm_algqos_write()
3793 ret = qm_func_shaper_enable(qm, fun_index, val); in qm_algqos_write()
3795 pci_err(qm->pdev, "failed to enable function shaper!\n"); in qm_algqos_write()
3800 pci_info(qm->pdev, "the qos value of function%u is set to %lu.\n", in qm_algqos_write()
3805 qm_pm_put_sync(qm); in qm_algqos_write()
3807 clear_bit(QM_RESETTING, &qm->misc_ctl); in qm_algqos_write()
3820 * @qm: The qm for which we want to add debugfs files.
3824 void hisi_qm_set_algqos_init(struct hisi_qm *qm) in hisi_qm_set_algqos_init() argument
3826 if (qm->fun_type == QM_HW_PF) in hisi_qm_set_algqos_init()
3827 debugfs_create_file("alg_qos", 0644, qm->debug.debug_root, in hisi_qm_set_algqos_init()
3828 qm, &qm_algqos_fops); in hisi_qm_set_algqos_init()
3829 else if (test_bit(QM_SUPPORT_MB_COMMAND, &qm->caps)) in hisi_qm_set_algqos_init()
3830 debugfs_create_file("alg_qos", 0444, qm->debug.debug_root, in hisi_qm_set_algqos_init()
3831 qm, &qm_algqos_fops); in hisi_qm_set_algqos_init()
3834 static void hisi_qm_init_vf_qos(struct hisi_qm *qm, int total_func) in hisi_qm_init_vf_qos() argument
3839 qm->factor[i].func_qos = QM_QOS_MAX_VAL; in hisi_qm_init_vf_qos()
3853 struct hisi_qm *qm = pci_get_drvdata(pdev); in hisi_qm_sriov_enable() local
3856 ret = qm_pm_get_sync(qm); in hisi_qm_sriov_enable()
3876 if (test_bit(QM_SUPPORT_FUNC_QOS, &qm->caps)) in hisi_qm_sriov_enable()
3877 hisi_qm_init_vf_qos(qm, num_vfs); in hisi_qm_sriov_enable()
3879 ret = qm_vf_q_assign(qm, num_vfs); in hisi_qm_sriov_enable()
3885 qm->vfs_num = num_vfs; in hisi_qm_sriov_enable()
3890 qm_clear_vft_config(qm); in hisi_qm_sriov_enable()
3899 qm_pm_put_sync(qm); in hisi_qm_sriov_enable()
3913 struct hisi_qm *qm = pci_get_drvdata(pdev); in hisi_qm_sriov_disable() local
3922 if (!is_frozen && qm_try_frozen_vfs(pdev, qm->qm_list)) { in hisi_qm_sriov_disable()
3929 ret = qm_clear_vft_config(qm); in hisi_qm_sriov_disable()
3933 qm_pm_put_sync(qm); in hisi_qm_sriov_disable()
3955 static enum acc_err_result qm_dev_err_handle(struct hisi_qm *qm) in qm_dev_err_handle() argument
3957 if (!qm->err_ini->get_err_result) { in qm_dev_err_handle()
3958 dev_err(&qm->pdev->dev, "Device doesn't support reset!\n"); in qm_dev_err_handle()
3962 return qm->err_ini->get_err_result(qm); in qm_dev_err_handle()
3965 static enum acc_err_result qm_process_dev_error(struct hisi_qm *qm) in qm_process_dev_error() argument
3969 /* log qm error */ in qm_process_dev_error()
3970 qm_ret = qm_hw_error_handle(qm); in qm_process_dev_error()
3973 dev_ret = qm_dev_err_handle(qm); in qm_process_dev_error()
3981 * hisi_qm_dev_err_detected() - Get device and qm error status then log it.
3986 * qm hardware error status when error occur.
3991 struct hisi_qm *qm = pci_get_drvdata(pdev); in hisi_qm_dev_err_detected() local
4001 ret = qm_process_dev_error(qm); in hisi_qm_dev_err_detected()
4009 static int qm_check_req_recv(struct hisi_qm *qm) in qm_check_req_recv() argument
4011 struct pci_dev *pdev = qm->pdev; in qm_check_req_recv()
4015 if (qm->ver >= QM_HW_V3) in qm_check_req_recv()
4018 writel(ACC_VENDOR_ID_VALUE, qm->io_base + QM_PEH_VENDOR_ID); in qm_check_req_recv()
4019 ret = readl_relaxed_poll_timeout(qm->io_base + QM_PEH_VENDOR_ID, val, in qm_check_req_recv()
4023 dev_err(&pdev->dev, "Fails to read QM reg!\n"); in qm_check_req_recv()
4027 writel(PCI_VENDOR_ID_HUAWEI, qm->io_base + QM_PEH_VENDOR_ID); in qm_check_req_recv()
4028 ret = readl_relaxed_poll_timeout(qm->io_base + QM_PEH_VENDOR_ID, val, in qm_check_req_recv()
4032 dev_err(&pdev->dev, "Fails to read QM reg in the second time!\n"); in qm_check_req_recv()
4037 static int qm_set_pf_mse(struct hisi_qm *qm, bool set) in qm_set_pf_mse() argument
4039 struct pci_dev *pdev = qm->pdev; in qm_set_pf_mse()
4061 static int qm_set_vf_mse(struct hisi_qm *qm, bool set) in qm_set_vf_mse() argument
4063 struct pci_dev *pdev = qm->pdev; in qm_set_vf_mse()
4088 static void qm_dev_ecc_mbit_handle(struct hisi_qm *qm) in qm_dev_ecc_mbit_handle() argument
4093 if (qm->ver >= QM_HW_V3) in qm_dev_ecc_mbit_handle()
4096 if (!qm->err_status.is_dev_ecc_mbit && in qm_dev_ecc_mbit_handle()
4097 qm->err_status.is_qm_ecc_mbit && in qm_dev_ecc_mbit_handle()
4098 qm->err_ini->close_axi_master_ooo) { in qm_dev_ecc_mbit_handle()
4099 qm->err_ini->close_axi_master_ooo(qm); in qm_dev_ecc_mbit_handle()
4100 } else if (qm->err_status.is_dev_ecc_mbit && in qm_dev_ecc_mbit_handle()
4101 !qm->err_status.is_qm_ecc_mbit && in qm_dev_ecc_mbit_handle()
4102 !qm->err_ini->close_axi_master_ooo) { in qm_dev_ecc_mbit_handle()
4103 nfe_enb = readl(qm->io_base + QM_RAS_NFE_ENABLE); in qm_dev_ecc_mbit_handle()
4105 qm->io_base + QM_RAS_NFE_ENABLE); in qm_dev_ecc_mbit_handle()
4106 writel(QM_ECC_MBIT, qm->io_base + QM_ABNORMAL_INT_SET); in qm_dev_ecc_mbit_handle()
4110 static int qm_vf_reset_prepare(struct hisi_qm *qm, in qm_vf_reset_prepare() argument
4113 struct hisi_qm_list *qm_list = qm->qm_list; in qm_vf_reset_prepare()
4114 struct pci_dev *pdev = qm->pdev; in qm_vf_reset_prepare()
4140 static int qm_try_stop_vfs(struct hisi_qm *qm, u64 cmd, in qm_try_stop_vfs() argument
4143 struct pci_dev *pdev = qm->pdev; in qm_try_stop_vfs()
4146 if (!qm->vfs_num) in qm_try_stop_vfs()
4150 if (test_bit(QM_SUPPORT_MB_COMMAND, &qm->caps)) { in qm_try_stop_vfs()
4151 ret = qm_ping_all_vfs(qm, cmd); in qm_try_stop_vfs()
4155 ret = qm_vf_reset_prepare(qm, stop_reason); in qm_try_stop_vfs()
4163 static int qm_controller_reset_prepare(struct hisi_qm *qm) in qm_controller_reset_prepare() argument
4165 struct pci_dev *pdev = qm->pdev; in qm_controller_reset_prepare()
4168 ret = qm_reset_prepare_ready(qm); in qm_controller_reset_prepare()
4174 qm_dev_ecc_mbit_handle(qm); in qm_controller_reset_prepare()
4177 qm_cmd_uninit(qm); in qm_controller_reset_prepare()
4180 ret = qm_try_stop_vfs(qm, QM_PF_SRST_PREPARE, QM_SOFT_RESET); in qm_controller_reset_prepare()
4184 ret = hisi_qm_stop(qm, QM_SOFT_RESET); in qm_controller_reset_prepare()
4186 pci_err(pdev, "Fails to stop QM!\n"); in qm_controller_reset_prepare()
4187 qm_reset_bit_clear(qm); in qm_controller_reset_prepare()
4191 if (qm->use_sva) { in qm_controller_reset_prepare()
4192 ret = qm_hw_err_isolate(qm); in qm_controller_reset_prepare()
4197 ret = qm_wait_vf_prepare_finish(qm); in qm_controller_reset_prepare()
4201 clear_bit(QM_RST_SCHED, &qm->misc_ctl); in qm_controller_reset_prepare()
4206 static int qm_master_ooo_check(struct hisi_qm *qm) in qm_master_ooo_check() argument
4212 writel(ACC_MASTER_GLOBAL_CTRL_SHUTDOWN, qm->io_base + ACC_MASTER_GLOBAL_CTRL); in qm_master_ooo_check()
4213 ret = readl_relaxed_poll_timeout(qm->io_base + ACC_MASTER_TRANS_RETURN, in qm_master_ooo_check()
4217 pci_warn(qm->pdev, "Bus lock! Please reset system.\n"); in qm_master_ooo_check()
4222 static int qm_soft_reset_prepare(struct hisi_qm *qm) in qm_soft_reset_prepare() argument
4224 struct pci_dev *pdev = qm->pdev; in qm_soft_reset_prepare()
4227 /* Ensure all doorbells and mailboxes received by QM */ in qm_soft_reset_prepare()
4228 ret = qm_check_req_recv(qm); in qm_soft_reset_prepare()
4232 if (qm->vfs_num) { in qm_soft_reset_prepare()
4233 ret = qm_set_vf_mse(qm, false); in qm_soft_reset_prepare()
4240 ret = qm->ops->set_msi(qm, false); in qm_soft_reset_prepare()
4246 ret = qm_master_ooo_check(qm); in qm_soft_reset_prepare()
4250 if (qm->err_ini->close_sva_prefetch) in qm_soft_reset_prepare()
4251 qm->err_ini->close_sva_prefetch(qm); in qm_soft_reset_prepare()
4253 ret = qm_set_pf_mse(qm, false); in qm_soft_reset_prepare()
4260 static int qm_reset_device(struct hisi_qm *qm) in qm_reset_device() argument
4262 struct pci_dev *pdev = qm->pdev; in qm_reset_device()
4270 qm->err_info.acpi_rst, in qm_reset_device()
4289 static int qm_soft_reset(struct hisi_qm *qm) in qm_soft_reset() argument
4293 ret = qm_soft_reset_prepare(qm); in qm_soft_reset()
4297 return qm_reset_device(qm); in qm_soft_reset()
4300 static int qm_vf_reset_done(struct hisi_qm *qm) in qm_vf_reset_done() argument
4302 struct hisi_qm_list *qm_list = qm->qm_list; in qm_vf_reset_done()
4303 struct pci_dev *pdev = qm->pdev; in qm_vf_reset_done()
4329 static int qm_try_start_vfs(struct hisi_qm *qm, enum qm_mb_cmd cmd) in qm_try_start_vfs() argument
4331 struct pci_dev *pdev = qm->pdev; in qm_try_start_vfs()
4334 if (!qm->vfs_num) in qm_try_start_vfs()
4337 ret = qm_vf_q_assign(qm, qm->vfs_num); in qm_try_start_vfs()
4344 if (test_bit(QM_SUPPORT_MB_COMMAND, &qm->caps)) { in qm_try_start_vfs()
4345 ret = qm_ping_all_vfs(qm, cmd); in qm_try_start_vfs()
4349 ret = qm_vf_reset_done(qm); in qm_try_start_vfs()
4357 static int qm_dev_hw_init(struct hisi_qm *qm) in qm_dev_hw_init() argument
4359 return qm->err_ini->hw_init(qm); in qm_dev_hw_init()
4362 static void qm_restart_prepare(struct hisi_qm *qm) in qm_restart_prepare() argument
4366 if (qm->err_ini->open_sva_prefetch) in qm_restart_prepare()
4367 qm->err_ini->open_sva_prefetch(qm); in qm_restart_prepare()
4369 if (qm->ver >= QM_HW_V3) in qm_restart_prepare()
4372 if (!qm->err_status.is_qm_ecc_mbit && in qm_restart_prepare()
4373 !qm->err_status.is_dev_ecc_mbit) in qm_restart_prepare()
4377 value = readl(qm->io_base + ACC_AM_CFG_PORT_WR_EN); in qm_restart_prepare()
4378 writel(value & ~qm->err_info.msi_wr_port, in qm_restart_prepare()
4379 qm->io_base + ACC_AM_CFG_PORT_WR_EN); in qm_restart_prepare()
4382 value = qm_get_dev_err_status(qm) & qm->err_info.ecc_2bits_mask; in qm_restart_prepare()
4383 if (value && qm->err_ini->clear_dev_hw_err_status) in qm_restart_prepare()
4384 qm->err_ini->clear_dev_hw_err_status(qm, value); in qm_restart_prepare()
4386 /* clear QM ecc mbit error source */ in qm_restart_prepare()
4387 writel(QM_ECC_MBIT, qm->io_base + QM_ABNORMAL_INT_SOURCE); in qm_restart_prepare()
4390 writel(ACC_ROB_ECC_ERR_MULTPL, qm->io_base + ACC_AM_ROB_ECC_INT_STS); in qm_restart_prepare()
4393 static void qm_restart_done(struct hisi_qm *qm) in qm_restart_done() argument
4397 if (qm->ver >= QM_HW_V3) in qm_restart_done()
4400 if (!qm->err_status.is_qm_ecc_mbit && in qm_restart_done()
4401 !qm->err_status.is_dev_ecc_mbit) in qm_restart_done()
4405 value = readl(qm->io_base + ACC_AM_CFG_PORT_WR_EN); in qm_restart_done()
4406 value |= qm->err_info.msi_wr_port; in qm_restart_done()
4407 writel(value, qm->io_base + ACC_AM_CFG_PORT_WR_EN); in qm_restart_done()
4410 qm->err_status.is_qm_ecc_mbit = false; in qm_restart_done()
4411 qm->err_status.is_dev_ecc_mbit = false; in qm_restart_done()
4414 static int qm_controller_reset_done(struct hisi_qm *qm) in qm_controller_reset_done() argument
4416 struct pci_dev *pdev = qm->pdev; in qm_controller_reset_done()
4419 ret = qm->ops->set_msi(qm, true); in qm_controller_reset_done()
4425 ret = qm_set_pf_mse(qm, true); in qm_controller_reset_done()
4431 if (qm->vfs_num) { in qm_controller_reset_done()
4432 ret = qm_set_vf_mse(qm, true); in qm_controller_reset_done()
4439 ret = qm_dev_hw_init(qm); in qm_controller_reset_done()
4445 qm_restart_prepare(qm); in qm_controller_reset_done()
4446 hisi_qm_dev_err_init(qm); in qm_controller_reset_done()
4447 if (qm->err_ini->open_axi_master_ooo) in qm_controller_reset_done()
4448 qm->err_ini->open_axi_master_ooo(qm); in qm_controller_reset_done()
4450 ret = qm_dev_mem_reset(qm); in qm_controller_reset_done()
4456 ret = qm_restart(qm); in qm_controller_reset_done()
4458 pci_err(pdev, "Failed to start QM!\n"); in qm_controller_reset_done()
4462 ret = qm_try_start_vfs(qm, QM_PF_RESET_DONE); in qm_controller_reset_done()
4466 ret = qm_wait_vf_prepare_finish(qm); in qm_controller_reset_done()
4470 qm_cmd_init(qm); in qm_controller_reset_done()
4471 qm_restart_done(qm); in qm_controller_reset_done()
4473 qm_reset_bit_clear(qm); in qm_controller_reset_done()
4478 static int qm_controller_reset(struct hisi_qm *qm) in qm_controller_reset() argument
4480 struct pci_dev *pdev = qm->pdev; in qm_controller_reset()
4485 ret = qm_controller_reset_prepare(qm); in qm_controller_reset()
4487 hisi_qm_set_hw_reset(qm, QM_RESET_STOP_TX_OFFSET); in qm_controller_reset()
4488 hisi_qm_set_hw_reset(qm, QM_RESET_STOP_RX_OFFSET); in qm_controller_reset()
4489 clear_bit(QM_RST_SCHED, &qm->misc_ctl); in qm_controller_reset()
4493 hisi_qm_show_last_dfx_regs(qm); in qm_controller_reset()
4494 if (qm->err_ini->show_last_dfx_regs) in qm_controller_reset()
4495 qm->err_ini->show_last_dfx_regs(qm); in qm_controller_reset()
4497 ret = qm_soft_reset(qm); in qm_controller_reset()
4501 ret = qm_controller_reset_done(qm); in qm_controller_reset()
4511 qm_reset_bit_clear(qm); in qm_controller_reset()
4514 if (qm->use_sva) in qm_controller_reset()
4515 qm->isolate_data.is_isolate = true; in qm_controller_reset()
4523 * This function offers QM relate PCIe device reset interface. Drivers which
4524 * use QM can use this function as slot_reset in its struct pci_error_handlers.
4528 struct hisi_qm *qm = pci_get_drvdata(pdev); in hisi_qm_dev_slot_reset() local
4535 ret = qm_controller_reset(qm); in hisi_qm_dev_slot_reset()
4548 struct hisi_qm *qm = pci_get_drvdata(pdev); in hisi_qm_reset_prepare() local
4564 ret = qm_reset_prepare_ready(qm); in hisi_qm_reset_prepare()
4571 if (qm->fun_type == QM_HW_PF) in hisi_qm_reset_prepare()
4572 qm_cmd_uninit(qm); in hisi_qm_reset_prepare()
4574 ret = qm_try_stop_vfs(qm, QM_PF_FLR_PREPARE, QM_DOWN); in hisi_qm_reset_prepare()
4578 ret = hisi_qm_stop(qm, QM_DOWN); in hisi_qm_reset_prepare()
4580 pci_err(pdev, "Failed to stop QM, ret = %d.\n", ret); in hisi_qm_reset_prepare()
4581 hisi_qm_set_hw_reset(qm, QM_RESET_STOP_TX_OFFSET); in hisi_qm_reset_prepare()
4582 hisi_qm_set_hw_reset(qm, QM_RESET_STOP_RX_OFFSET); in hisi_qm_reset_prepare()
4586 ret = qm_wait_vf_prepare_finish(qm); in hisi_qm_reset_prepare()
4597 struct hisi_qm *qm = pci_get_drvdata(pf_pdev); in qm_flr_reset_complete() local
4600 pci_read_config_dword(qm->pdev, PCI_COMMAND, &id); in qm_flr_reset_complete()
4612 struct hisi_qm *qm = pci_get_drvdata(pdev); in hisi_qm_reset_done() local
4615 if (qm->fun_type == QM_HW_PF) { in hisi_qm_reset_done()
4616 ret = qm_dev_hw_init(qm); in hisi_qm_reset_done()
4625 ret = qm_restart(qm); in hisi_qm_reset_done()
4627 pci_err(pdev, "Failed to start QM, ret = %d.\n", ret); in hisi_qm_reset_done()
4631 ret = qm_try_start_vfs(qm, QM_PF_RESET_DONE); in hisi_qm_reset_done()
4635 ret = qm_wait_vf_prepare_finish(qm); in hisi_qm_reset_done()
4640 if (qm->fun_type == QM_HW_PF) in hisi_qm_reset_done()
4641 qm_cmd_init(qm); in hisi_qm_reset_done()
4646 qm_reset_bit_clear(qm); in hisi_qm_reset_done()
4652 struct hisi_qm *qm = data; in qm_abnormal_irq() local
4655 atomic64_inc(&qm->debug.dfx.abnormal_irq_cnt); in qm_abnormal_irq()
4656 ret = qm_process_dev_error(qm); in qm_abnormal_irq()
4658 !test_bit(QM_DRIVER_REMOVING, &qm->misc_ctl) && in qm_abnormal_irq()
4659 !test_and_set_bit(QM_RST_SCHED, &qm->misc_ctl)) in qm_abnormal_irq()
4660 schedule_work(&qm->rst_work); in qm_abnormal_irq()
4669 * This function will stop qm when OS shutdown or rebooting.
4673 struct hisi_qm *qm = pci_get_drvdata(pdev); in hisi_qm_dev_shutdown() local
4676 ret = hisi_qm_stop(qm, QM_DOWN); in hisi_qm_dev_shutdown()
4678 dev_err(&pdev->dev, "Fail to stop qm in shutdown!\n"); in hisi_qm_dev_shutdown()
4680 hisi_qm_cache_wb(qm); in hisi_qm_dev_shutdown()
4686 struct hisi_qm *qm = container_of(rst_work, struct hisi_qm, rst_work); in hisi_qm_controller_reset() local
4689 ret = qm_pm_get_sync(qm); in hisi_qm_controller_reset()
4691 clear_bit(QM_RST_SCHED, &qm->misc_ctl); in hisi_qm_controller_reset()
4696 ret = qm_controller_reset(qm); in hisi_qm_controller_reset()
4698 dev_err(&qm->pdev->dev, "controller reset failed (%d)\n", ret); in hisi_qm_controller_reset()
4700 qm_pm_put_sync(qm); in hisi_qm_controller_reset()
4703 static void qm_pf_reset_vf_prepare(struct hisi_qm *qm, in qm_pf_reset_vf_prepare() argument
4707 struct pci_dev *pdev = qm->pdev; in qm_pf_reset_vf_prepare()
4710 ret = qm_reset_prepare_ready(qm); in qm_pf_reset_vf_prepare()
4713 atomic_set(&qm->status.flags, QM_STOP); in qm_pf_reset_vf_prepare()
4718 ret = hisi_qm_stop(qm, stop_reason); in qm_pf_reset_vf_prepare()
4720 dev_err(&pdev->dev, "failed to stop QM, ret = %d.\n", ret); in qm_pf_reset_vf_prepare()
4721 atomic_set(&qm->status.flags, QM_STOP); in qm_pf_reset_vf_prepare()
4729 hisi_qm_set_hw_reset(qm, QM_RESET_STOP_TX_OFFSET); in qm_pf_reset_vf_prepare()
4730 hisi_qm_set_hw_reset(qm, QM_RESET_STOP_RX_OFFSET); in qm_pf_reset_vf_prepare()
4733 ret = qm_ping_pf(qm, cmd); in qm_pf_reset_vf_prepare()
4738 static void qm_pf_reset_vf_done(struct hisi_qm *qm) in qm_pf_reset_vf_done() argument
4741 struct pci_dev *pdev = qm->pdev; in qm_pf_reset_vf_done()
4745 ret = hisi_qm_start(qm); in qm_pf_reset_vf_done()
4747 dev_err(&pdev->dev, "failed to start QM, ret = %d.\n", ret); in qm_pf_reset_vf_done()
4751 qm_cmd_init(qm); in qm_pf_reset_vf_done()
4752 ret = qm_ping_pf(qm, cmd); in qm_pf_reset_vf_done()
4756 qm_reset_bit_clear(qm); in qm_pf_reset_vf_done()
4759 static int qm_wait_pf_reset_finish(struct hisi_qm *qm) in qm_wait_pf_reset_finish() argument
4761 struct device *dev = &qm->pdev->dev; in qm_wait_pf_reset_finish()
4767 ret = readl_relaxed_poll_timeout(qm->io_base + QM_IFC_INT_SOURCE_V, val, in qm_wait_pf_reset_finish()
4780 ret = qm_get_mb_cmd(qm, &msg, 0); in qm_wait_pf_reset_finish()
4781 qm_clear_cmd_interrupt(qm, 0); in qm_wait_pf_reset_finish()
4796 static void qm_pf_reset_vf_process(struct hisi_qm *qm, in qm_pf_reset_vf_process() argument
4799 struct device *dev = &qm->pdev->dev; in qm_pf_reset_vf_process()
4805 qm_cmd_uninit(qm); in qm_pf_reset_vf_process()
4806 qm_pf_reset_vf_prepare(qm, stop_reason); in qm_pf_reset_vf_process()
4808 ret = qm_wait_pf_reset_finish(qm); in qm_pf_reset_vf_process()
4812 qm_pf_reset_vf_done(qm); in qm_pf_reset_vf_process()
4819 qm_cmd_init(qm); in qm_pf_reset_vf_process()
4820 qm_reset_bit_clear(qm); in qm_pf_reset_vf_process()
4823 static void qm_handle_cmd_msg(struct hisi_qm *qm, u32 fun_num) in qm_handle_cmd_msg() argument
4825 struct device *dev = &qm->pdev->dev; in qm_handle_cmd_msg()
4834 ret = qm_get_mb_cmd(qm, &msg, fun_num); in qm_handle_cmd_msg()
4835 qm_clear_cmd_interrupt(qm, BIT(fun_num)); in qm_handle_cmd_msg()
4844 qm_pf_reset_vf_process(qm, QM_DOWN); in qm_handle_cmd_msg()
4847 qm_pf_reset_vf_process(qm, QM_SOFT_RESET); in qm_handle_cmd_msg()
4850 qm_vf_get_qos(qm, fun_num); in qm_handle_cmd_msg()
4853 qm->mb_qos = msg >> QM_MB_CMD_DATA_SHIFT; in qm_handle_cmd_msg()
4863 struct hisi_qm *qm = container_of(cmd_process, in qm_cmd_process() local
4865 u32 vfs_num = qm->vfs_num; in qm_cmd_process()
4869 if (qm->fun_type == QM_HW_PF) { in qm_cmd_process()
4870 val = readq(qm->io_base + QM_IFC_INT_SOURCE_P); in qm_cmd_process()
4876 qm_handle_cmd_msg(qm, i); in qm_cmd_process()
4882 qm_handle_cmd_msg(qm, 0); in qm_cmd_process()
4886 * hisi_qm_alg_register() - Register alg to crypto and add qm to qm_list.
4887 * @qm: The qm needs add.
4888 * @qm_list: The qm list.
4890 * This function adds qm to qm list, and will register algorithm to
4891 * crypto when the qm list is empty.
4893 int hisi_qm_alg_register(struct hisi_qm *qm, struct hisi_qm_list *qm_list) in hisi_qm_alg_register() argument
4895 struct device *dev = &qm->pdev->dev; in hisi_qm_alg_register()
4902 list_add_tail(&qm->list, &qm_list->list); in hisi_qm_alg_register()
4905 if (qm->ver <= QM_HW_V2 && qm->use_sva) { in hisi_qm_alg_register()
4911 ret = qm_list->register_to_crypto(qm); in hisi_qm_alg_register()
4914 list_del(&qm->list); in hisi_qm_alg_register()
4924 * hisi_qm_alg_unregister() - Unregister alg from crypto and delete qm from
4925 * qm list.
4926 * @qm: The qm needs delete.
4927 * @qm_list: The qm list.
4929 * This function deletes qm from qm list, and will unregister algorithm
4930 * from crypto when the qm list is empty.
4932 void hisi_qm_alg_unregister(struct hisi_qm *qm, struct hisi_qm_list *qm_list) in hisi_qm_alg_unregister() argument
4935 list_del(&qm->list); in hisi_qm_alg_unregister()
4938 if (qm->ver <= QM_HW_V2 && qm->use_sva) in hisi_qm_alg_unregister()
4942 qm_list->unregister_from_crypto(qm); in hisi_qm_alg_unregister()
4946 static void qm_unregister_abnormal_irq(struct hisi_qm *qm) in qm_unregister_abnormal_irq() argument
4948 struct pci_dev *pdev = qm->pdev; in qm_unregister_abnormal_irq()
4951 if (qm->fun_type == QM_HW_VF) in qm_unregister_abnormal_irq()
4954 val = qm->cap_tables.qm_cap_table[QM_ABN_IRQ_TYPE_CAP_IDX].cap_val; in qm_unregister_abnormal_irq()
4959 free_irq(pci_irq_vector(pdev, irq_vector), qm); in qm_unregister_abnormal_irq()
4962 static int qm_register_abnormal_irq(struct hisi_qm *qm) in qm_register_abnormal_irq() argument
4964 struct pci_dev *pdev = qm->pdev; in qm_register_abnormal_irq()
4968 if (qm->fun_type == QM_HW_VF) in qm_register_abnormal_irq()
4971 val = qm->cap_tables.qm_cap_table[QM_ABN_IRQ_TYPE_CAP_IDX].cap_val; in qm_register_abnormal_irq()
4976 ret = request_irq(pci_irq_vector(pdev, irq_vector), qm_abnormal_irq, 0, qm->dev_name, qm); in qm_register_abnormal_irq()
4978 dev_err(&qm->pdev->dev, "failed to request abnormal irq, ret = %d", ret); in qm_register_abnormal_irq()
4983 static void qm_unregister_mb_cmd_irq(struct hisi_qm *qm) in qm_unregister_mb_cmd_irq() argument
4985 struct pci_dev *pdev = qm->pdev; in qm_unregister_mb_cmd_irq()
4988 val = qm->cap_tables.qm_cap_table[QM_PF2VF_IRQ_TYPE_CAP_IDX].cap_val; in qm_unregister_mb_cmd_irq()
4993 free_irq(pci_irq_vector(pdev, irq_vector), qm); in qm_unregister_mb_cmd_irq()
4996 static int qm_register_mb_cmd_irq(struct hisi_qm *qm) in qm_register_mb_cmd_irq() argument
4998 struct pci_dev *pdev = qm->pdev; in qm_register_mb_cmd_irq()
5002 val = qm->cap_tables.qm_cap_table[QM_PF2VF_IRQ_TYPE_CAP_IDX].cap_val; in qm_register_mb_cmd_irq()
5007 ret = request_irq(pci_irq_vector(pdev, irq_vector), qm_mb_cmd_irq, 0, qm->dev_name, qm); in qm_register_mb_cmd_irq()
5014 static void qm_unregister_aeq_irq(struct hisi_qm *qm) in qm_unregister_aeq_irq() argument
5016 struct pci_dev *pdev = qm->pdev; in qm_unregister_aeq_irq()
5019 val = qm->cap_tables.qm_cap_table[QM_AEQ_IRQ_TYPE_CAP_IDX].cap_val; in qm_unregister_aeq_irq()
5024 free_irq(pci_irq_vector(pdev, irq_vector), qm); in qm_unregister_aeq_irq()
5027 static int qm_register_aeq_irq(struct hisi_qm *qm) in qm_register_aeq_irq() argument
5029 struct pci_dev *pdev = qm->pdev; in qm_register_aeq_irq()
5033 val = qm->cap_tables.qm_cap_table[QM_AEQ_IRQ_TYPE_CAP_IDX].cap_val; in qm_register_aeq_irq()
5039 qm_aeq_thread, IRQF_ONESHOT, qm->dev_name, qm); in qm_register_aeq_irq()
5046 static void qm_unregister_eq_irq(struct hisi_qm *qm) in qm_unregister_eq_irq() argument
5048 struct pci_dev *pdev = qm->pdev; in qm_unregister_eq_irq()
5051 val = qm->cap_tables.qm_cap_table[QM_EQ_IRQ_TYPE_CAP_IDX].cap_val; in qm_unregister_eq_irq()
5056 free_irq(pci_irq_vector(pdev, irq_vector), qm); in qm_unregister_eq_irq()
5059 static int qm_register_eq_irq(struct hisi_qm *qm) in qm_register_eq_irq() argument
5061 struct pci_dev *pdev = qm->pdev; in qm_register_eq_irq()
5065 val = qm->cap_tables.qm_cap_table[QM_EQ_IRQ_TYPE_CAP_IDX].cap_val; in qm_register_eq_irq()
5070 ret = request_irq(pci_irq_vector(pdev, irq_vector), qm_eq_irq, 0, qm->dev_name, qm); in qm_register_eq_irq()
5077 static void qm_irqs_unregister(struct hisi_qm *qm) in qm_irqs_unregister() argument
5079 qm_unregister_mb_cmd_irq(qm); in qm_irqs_unregister()
5080 qm_unregister_abnormal_irq(qm); in qm_irqs_unregister()
5081 qm_unregister_aeq_irq(qm); in qm_irqs_unregister()
5082 qm_unregister_eq_irq(qm); in qm_irqs_unregister()
5085 static int qm_irqs_register(struct hisi_qm *qm) in qm_irqs_register() argument
5089 ret = qm_register_eq_irq(qm); in qm_irqs_register()
5093 ret = qm_register_aeq_irq(qm); in qm_irqs_register()
5097 ret = qm_register_abnormal_irq(qm); in qm_irqs_register()
5101 ret = qm_register_mb_cmd_irq(qm); in qm_irqs_register()
5108 qm_unregister_abnormal_irq(qm); in qm_irqs_register()
5110 qm_unregister_aeq_irq(qm); in qm_irqs_register()
5112 qm_unregister_eq_irq(qm); in qm_irqs_register()
5116 static int qm_get_qp_num(struct hisi_qm *qm) in qm_get_qp_num() argument
5118 struct device *dev = &qm->pdev->dev; in qm_get_qp_num()
5122 if (qm->fun_type == QM_HW_VF) { in qm_get_qp_num()
5123 if (qm->ver != QM_HW_V1) in qm_get_qp_num()
5125 return hisi_qm_get_vft(qm, &qm->qp_base, &qm->qp_num); in qm_get_qp_num()
5130 is_db_isolation = test_bit(QM_SUPPORT_DB_ISOLATION, &qm->caps); in qm_get_qp_num()
5131 qm->ctrl_qp_num = hisi_qm_get_hw_info(qm, qm_basic_info, QM_TOTAL_QP_NUM_CAP, true); in qm_get_qp_num()
5132 qm->max_qp_num = hisi_qm_get_hw_info(qm, qm_basic_info, in qm_get_qp_num()
5135 if (qm->qp_num <= qm->max_qp_num) in qm_get_qp_num()
5138 if (test_bit(QM_MODULE_PARAM, &qm->misc_ctl)) { in qm_get_qp_num()
5141 qm->qp_num, qm->max_qp_num); in qm_get_qp_num()
5146 qm->qp_num, qm->max_qp_num); in qm_get_qp_num()
5147 qm->qp_num = qm->max_qp_num; in qm_get_qp_num()
5148 qm->debug.curr_qm_qp_num = qm->qp_num; in qm_get_qp_num()
5153 static int qm_pre_store_irq_type_caps(struct hisi_qm *qm) in qm_pre_store_irq_type_caps() argument
5156 struct pci_dev *pdev = qm->pdev; in qm_pre_store_irq_type_caps()
5166 qm_cap[i].cap_val = hisi_qm_get_hw_info(qm, qm_basic_info, in qm_pre_store_irq_type_caps()
5167 qm_pre_store_caps[i], qm->cap_ver); in qm_pre_store_irq_type_caps()
5170 qm->cap_tables.qm_cap_table = qm_cap; in qm_pre_store_irq_type_caps()
5175 static int qm_get_hw_caps(struct hisi_qm *qm) in qm_get_hw_caps() argument
5177 const struct hisi_qm_cap_info *cap_info = qm->fun_type == QM_HW_PF ? in qm_get_hw_caps()
5179 u32 size = qm->fun_type == QM_HW_PF ? ARRAY_SIZE(qm_cap_info_pf) : in qm_get_hw_caps()
5184 val = hisi_qm_get_hw_info(qm, qm_cap_info_comm, QM_SUPPORT_DB_ISOLATION, true); in qm_get_hw_caps()
5186 set_bit(QM_SUPPORT_DB_ISOLATION, &qm->caps); in qm_get_hw_caps()
5188 if (qm->ver >= QM_HW_V3) { in qm_get_hw_caps()
5189 val = readl(qm->io_base + QM_FUNC_CAPS_REG); in qm_get_hw_caps()
5190 qm->cap_ver = val & QM_CAPBILITY_VERSION; in qm_get_hw_caps()
5195 val = hisi_qm_get_hw_info(qm, qm_cap_info_comm, i, qm->cap_ver); in qm_get_hw_caps()
5197 set_bit(qm_cap_info_comm[i].type, &qm->caps); in qm_get_hw_caps()
5202 val = hisi_qm_get_hw_info(qm, cap_info, i, qm->cap_ver); in qm_get_hw_caps()
5204 set_bit(cap_info[i].type, &qm->caps); in qm_get_hw_caps()
5208 return qm_pre_store_irq_type_caps(qm); in qm_get_hw_caps()
5211 static int qm_get_pci_res(struct hisi_qm *qm) in qm_get_pci_res() argument
5213 struct pci_dev *pdev = qm->pdev; in qm_get_pci_res()
5217 ret = pci_request_mem_regions(pdev, qm->dev_name); in qm_get_pci_res()
5223 qm->phys_base = pci_resource_start(pdev, PCI_BAR_2); in qm_get_pci_res()
5224 qm->io_base = ioremap(qm->phys_base, pci_resource_len(pdev, PCI_BAR_2)); in qm_get_pci_res()
5225 if (!qm->io_base) { in qm_get_pci_res()
5230 ret = qm_get_hw_caps(qm); in qm_get_pci_res()
5234 if (test_bit(QM_SUPPORT_DB_ISOLATION, &qm->caps)) { in qm_get_pci_res()
5235 qm->db_interval = QM_QP_DB_INTERVAL; in qm_get_pci_res()
5236 qm->db_phys_base = pci_resource_start(pdev, PCI_BAR_4); in qm_get_pci_res()
5237 qm->db_io_base = ioremap(qm->db_phys_base, in qm_get_pci_res()
5239 if (!qm->db_io_base) { in qm_get_pci_res()
5244 qm->db_phys_base = qm->phys_base; in qm_get_pci_res()
5245 qm->db_io_base = qm->io_base; in qm_get_pci_res()
5246 qm->db_interval = 0; in qm_get_pci_res()
5249 ret = qm_get_qp_num(qm); in qm_get_pci_res()
5256 if (test_bit(QM_SUPPORT_DB_ISOLATION, &qm->caps)) in qm_get_pci_res()
5257 iounmap(qm->db_io_base); in qm_get_pci_res()
5259 iounmap(qm->io_base); in qm_get_pci_res()
5265 static int qm_clear_device(struct hisi_qm *qm) in qm_clear_device() argument
5267 acpi_handle handle = ACPI_HANDLE(&qm->pdev->dev); in qm_clear_device()
5270 if (qm->fun_type == QM_HW_VF) in qm_clear_device()
5274 if (!qm->err_ini->err_info_init) in qm_clear_device()
5276 qm->err_ini->err_info_init(qm); in qm_clear_device()
5282 if (!acpi_has_method(handle, qm->err_info.acpi_rst)) in qm_clear_device()
5285 ret = qm_master_ooo_check(qm); in qm_clear_device()
5287 writel(0x0, qm->io_base + ACC_MASTER_GLOBAL_CTRL); in qm_clear_device()
5291 return qm_reset_device(qm); in qm_clear_device()
5294 static int hisi_qm_pci_init(struct hisi_qm *qm) in hisi_qm_pci_init() argument
5296 struct pci_dev *pdev = qm->pdev; in hisi_qm_pci_init()
5307 ret = qm_get_pci_res(qm); in hisi_qm_pci_init()
5316 num_vec = qm_get_irq_num(qm); in hisi_qm_pci_init()
5323 ret = qm_clear_device(qm); in hisi_qm_pci_init()
5332 qm_put_pci_res(qm); in hisi_qm_pci_init()
5338 static int hisi_qm_init_work(struct hisi_qm *qm) in hisi_qm_init_work() argument
5342 for (i = 0; i < qm->qp_num; i++) in hisi_qm_init_work()
5343 INIT_WORK(&qm->poll_data[i].work, qm_work_process); in hisi_qm_init_work()
5345 if (qm->fun_type == QM_HW_PF) in hisi_qm_init_work()
5346 INIT_WORK(&qm->rst_work, hisi_qm_controller_reset); in hisi_qm_init_work()
5348 if (qm->ver > QM_HW_V2) in hisi_qm_init_work()
5349 INIT_WORK(&qm->cmd_process, qm_cmd_process); in hisi_qm_init_work()
5351 qm->wq = alloc_workqueue("%s", WQ_HIGHPRI | WQ_MEM_RECLAIM | in hisi_qm_init_work()
5353 pci_name(qm->pdev)); in hisi_qm_init_work()
5354 if (!qm->wq) { in hisi_qm_init_work()
5355 pci_err(qm->pdev, "failed to alloc workqueue!\n"); in hisi_qm_init_work()
5362 static int hisi_qp_alloc_memory(struct hisi_qm *qm) in hisi_qp_alloc_memory() argument
5364 struct device *dev = &qm->pdev->dev; in hisi_qp_alloc_memory()
5369 qm->qp_array = kcalloc(qm->qp_num, sizeof(struct hisi_qp), GFP_KERNEL); in hisi_qp_alloc_memory()
5370 if (!qm->qp_array) in hisi_qp_alloc_memory()
5373 qm->poll_data = kcalloc(qm->qp_num, sizeof(struct hisi_qm_poll_data), GFP_KERNEL); in hisi_qp_alloc_memory()
5374 if (!qm->poll_data) { in hisi_qp_alloc_memory()
5375 kfree(qm->qp_array); in hisi_qp_alloc_memory()
5379 qm_get_xqc_depth(qm, &sq_depth, &cq_depth, QM_QP_DEPTH_CAP); in hisi_qp_alloc_memory()
5382 qp_dma_size = qm->sqe_size * sq_depth + sizeof(struct qm_cqe) * cq_depth; in hisi_qp_alloc_memory()
5384 for (i = 0; i < qm->qp_num; i++) { in hisi_qp_alloc_memory()
5385 qm->poll_data[i].qm = qm; in hisi_qp_alloc_memory()
5386 ret = hisi_qp_memory_init(qm, qp_dma_size, i, sq_depth, cq_depth); in hisi_qp_alloc_memory()
5395 hisi_qp_memory_uninit(qm, i); in hisi_qp_alloc_memory()
5400 static int hisi_qm_memory_init(struct hisi_qm *qm) in hisi_qm_memory_init() argument
5402 struct device *dev = &qm->pdev->dev; in hisi_qm_memory_init()
5406 if (test_bit(QM_SUPPORT_FUNC_QOS, &qm->caps)) { in hisi_qm_memory_init()
5407 total_func = pci_sriov_get_totalvfs(qm->pdev) + 1; in hisi_qm_memory_init()
5408 qm->factor = kcalloc(total_func, sizeof(struct qm_shaper_factor), GFP_KERNEL); in hisi_qm_memory_init()
5409 if (!qm->factor) in hisi_qm_memory_init()
5413 qm->factor[0].func_qos = QM_QOS_MAX_VAL; in hisi_qm_memory_init()
5416 #define QM_INIT_BUF(qm, type, num) do { \ in hisi_qm_memory_init() argument
5417 (qm)->type = ((qm)->qdma.va + (off)); \ in hisi_qm_memory_init()
5418 (qm)->type##_dma = (qm)->qdma.dma + (off); \ in hisi_qm_memory_init()
5422 idr_init(&qm->qp_idr); in hisi_qm_memory_init()
5423 qm_get_xqc_depth(qm, &qm->eq_depth, &qm->aeq_depth, QM_XEQ_DEPTH_CAP); in hisi_qm_memory_init()
5424 qm->qdma.size = QMC_ALIGN(sizeof(struct qm_eqe) * qm->eq_depth) + in hisi_qm_memory_init()
5425 QMC_ALIGN(sizeof(struct qm_aeqe) * qm->aeq_depth) + in hisi_qm_memory_init()
5426 QMC_ALIGN(sizeof(struct qm_sqc) * qm->qp_num) + in hisi_qm_memory_init()
5427 QMC_ALIGN(sizeof(struct qm_cqc) * qm->qp_num); in hisi_qm_memory_init()
5428 qm->qdma.va = dma_alloc_coherent(dev, qm->qdma.size, &qm->qdma.dma, in hisi_qm_memory_init()
5430 dev_dbg(dev, "allocate qm dma buf size=%zx)\n", qm->qdma.size); in hisi_qm_memory_init()
5431 if (!qm->qdma.va) { in hisi_qm_memory_init()
5436 QM_INIT_BUF(qm, eqe, qm->eq_depth); in hisi_qm_memory_init()
5437 QM_INIT_BUF(qm, aeqe, qm->aeq_depth); in hisi_qm_memory_init()
5438 QM_INIT_BUF(qm, sqc, qm->qp_num); in hisi_qm_memory_init()
5439 QM_INIT_BUF(qm, cqc, qm->qp_num); in hisi_qm_memory_init()
5441 ret = hisi_qp_alloc_memory(qm); in hisi_qm_memory_init()
5448 dma_free_coherent(dev, qm->qdma.size, qm->qdma.va, qm->qdma.dma); in hisi_qm_memory_init()
5450 idr_destroy(&qm->qp_idr); in hisi_qm_memory_init()
5451 if (test_bit(QM_SUPPORT_FUNC_QOS, &qm->caps)) in hisi_qm_memory_init()
5452 kfree(qm->factor); in hisi_qm_memory_init()
5458 * hisi_qm_init() - Initialize configures about qm.
5459 * @qm: The qm needing init.
5461 * This function init qm, then we can call hisi_qm_start to put qm into work.
5463 int hisi_qm_init(struct hisi_qm *qm) in hisi_qm_init() argument
5465 struct pci_dev *pdev = qm->pdev; in hisi_qm_init()
5469 hisi_qm_pre_init(qm); in hisi_qm_init()
5471 ret = hisi_qm_pci_init(qm); in hisi_qm_init()
5475 ret = qm_irqs_register(qm); in hisi_qm_init()
5479 if (qm->fun_type == QM_HW_PF) { in hisi_qm_init()
5481 writel(QM_DB_TIMEOUT_SET, qm->io_base + QM_DB_TIMEOUT_CFG); in hisi_qm_init()
5482 qm_disable_clock_gate(qm); in hisi_qm_init()
5483 ret = qm_dev_mem_reset(qm); in hisi_qm_init()
5490 if (qm->mode == UACCE_MODE_SVA) { in hisi_qm_init()
5491 ret = qm_alloc_uacce(qm); in hisi_qm_init()
5496 ret = hisi_qm_memory_init(qm); in hisi_qm_init()
5500 ret = hisi_qm_init_work(qm); in hisi_qm_init()
5504 qm_cmd_init(qm); in hisi_qm_init()
5505 atomic_set(&qm->status.flags, QM_INIT); in hisi_qm_init()
5510 hisi_qm_memory_uninit(qm); in hisi_qm_init()
5512 qm_remove_uacce(qm); in hisi_qm_init()
5514 qm_irqs_unregister(qm); in hisi_qm_init()
5516 hisi_qm_pci_uninit(qm); in hisi_qm_init()
5523 * @qm: pointer to accelerator device.
5530 int hisi_qm_get_dfx_access(struct hisi_qm *qm) in hisi_qm_get_dfx_access() argument
5532 struct device *dev = &qm->pdev->dev; in hisi_qm_get_dfx_access()
5539 return qm_pm_get_sync(qm); in hisi_qm_get_dfx_access()
5545 * @qm: pointer to accelerator device.
5549 void hisi_qm_put_dfx_access(struct hisi_qm *qm) in hisi_qm_put_dfx_access() argument
5551 qm_pm_put_sync(qm); in hisi_qm_put_dfx_access()
5556 * hisi_qm_pm_init() - Initialize qm runtime PM.
5557 * @qm: pointer to accelerator device.
5559 * Function that initialize qm runtime PM.
5561 void hisi_qm_pm_init(struct hisi_qm *qm) in hisi_qm_pm_init() argument
5563 struct device *dev = &qm->pdev->dev; in hisi_qm_pm_init()
5565 if (!test_bit(QM_SUPPORT_RPM, &qm->caps)) in hisi_qm_pm_init()
5575 * hisi_qm_pm_uninit() - Uninitialize qm runtime PM.
5576 * @qm: pointer to accelerator device.
5578 * Function that uninitialize qm runtime PM.
5580 void hisi_qm_pm_uninit(struct hisi_qm *qm) in hisi_qm_pm_uninit() argument
5582 struct device *dev = &qm->pdev->dev; in hisi_qm_pm_uninit()
5584 if (!test_bit(QM_SUPPORT_RPM, &qm->caps)) in hisi_qm_pm_uninit()
5592 static int qm_prepare_for_suspend(struct hisi_qm *qm) in qm_prepare_for_suspend() argument
5594 struct pci_dev *pdev = qm->pdev; in qm_prepare_for_suspend()
5597 ret = qm->ops->set_msi(qm, false); in qm_prepare_for_suspend()
5603 ret = qm_master_ooo_check(qm); in qm_prepare_for_suspend()
5607 ret = qm_set_pf_mse(qm, false); in qm_prepare_for_suspend()
5614 static int qm_rebuild_for_resume(struct hisi_qm *qm) in qm_rebuild_for_resume() argument
5616 struct pci_dev *pdev = qm->pdev; in qm_rebuild_for_resume()
5619 ret = qm_set_pf_mse(qm, true); in qm_rebuild_for_resume()
5625 ret = qm->ops->set_msi(qm, true); in qm_rebuild_for_resume()
5631 ret = qm_dev_hw_init(qm); in qm_rebuild_for_resume()
5637 qm_cmd_init(qm); in qm_rebuild_for_resume()
5638 hisi_qm_dev_err_init(qm); in qm_rebuild_for_resume()
5640 writel(QM_DB_TIMEOUT_SET, qm->io_base + QM_DB_TIMEOUT_CFG); in qm_rebuild_for_resume()
5641 qm_disable_clock_gate(qm); in qm_rebuild_for_resume()
5642 ret = qm_dev_mem_reset(qm); in qm_rebuild_for_resume()
5658 struct hisi_qm *qm = pci_get_drvdata(pdev); in hisi_qm_suspend() local
5663 ret = hisi_qm_stop(qm, QM_NORMAL); in hisi_qm_suspend()
5665 pci_err(pdev, "failed to stop qm(%d)\n", ret); in hisi_qm_suspend()
5669 ret = qm_prepare_for_suspend(qm); in hisi_qm_suspend()
5686 struct hisi_qm *qm = pci_get_drvdata(pdev); in hisi_qm_resume() local
5691 ret = qm_rebuild_for_resume(qm); in hisi_qm_resume()
5697 ret = hisi_qm_start(qm); in hisi_qm_resume()
5699 if (qm_check_dev_error(qm)) { in hisi_qm_resume()
5700 pci_info(pdev, "failed to start qm due to device error, device will be reset!\n"); in hisi_qm_resume()
5704 pci_err(pdev, "failed to start qm(%d)!\n", ret); in hisi_qm_resume()