Lines Matching refs:afu
23 static int afu_control(struct cxl_afu *afu, u64 command, in afu_control() argument
26 u64 AFU_Cntl = cxl_p2n_read(afu, CXL_AFU_Cntl_An); in afu_control()
30 spin_lock(&afu->afu_cntl_lock); in afu_control()
33 trace_cxl_afu_ctrl(afu, command); in afu_control()
35 cxl_p2n_write(afu, CXL_AFU_Cntl_An, AFU_Cntl | command); in afu_control()
37 AFU_Cntl = cxl_p2n_read(afu, CXL_AFU_Cntl_An); in afu_control()
40 dev_warn(&afu->dev, "WARNING: AFU control timed out!\n"); in afu_control()
45 if (!cxl_adapter_link_ok(afu->adapter)) { in afu_control()
46 afu->enabled = enabled; in afu_control()
54 AFU_Cntl = cxl_p2n_read(afu, CXL_AFU_Cntl_An); in afu_control()
57 afu->enabled = enabled; in afu_control()
59 trace_cxl_afu_ctrl_done(afu, command, rc); in afu_control()
60 spin_unlock(&afu->afu_cntl_lock); in afu_control()
65 static int afu_enable(struct cxl_afu *afu) in afu_enable() argument
69 return afu_control(afu, CXL_AFU_Cntl_An_E, in afu_enable()
74 int cxl_afu_disable(struct cxl_afu *afu) in cxl_afu_disable() argument
78 return afu_control(afu, 0, CXL_AFU_Cntl_An_ES_Disabled, in cxl_afu_disable()
83 int __cxl_afu_reset(struct cxl_afu *afu) in __cxl_afu_reset() argument
87 return afu_control(afu, CXL_AFU_Cntl_An_RA, in __cxl_afu_reset()
93 int cxl_afu_check_and_enable(struct cxl_afu *afu) in cxl_afu_check_and_enable() argument
95 if (!cxl_adapter_link_ok(afu->adapter)) { in cxl_afu_check_and_enable()
99 if (afu->enabled) in cxl_afu_check_and_enable()
101 return afu_enable(afu); in cxl_afu_check_and_enable()
104 int cxl_psl_purge(struct cxl_afu *afu) in cxl_psl_purge() argument
106 u64 PSL_CNTL = cxl_p1n_read(afu, CXL_PSL_SCNTL_An); in cxl_psl_purge()
107 u64 AFU_Cntl = cxl_p2n_read(afu, CXL_AFU_Cntl_An); in cxl_psl_purge()
113 trace_cxl_psl_ctrl(afu, CXL_PSL_SCNTL_An_Pc); in cxl_psl_purge()
117 if (!cxl_adapter_link_ok(afu->adapter)) { in cxl_psl_purge()
118 dev_warn(&afu->dev, "PSL Purge called with link down, ignoring\n"); in cxl_psl_purge()
125 cxl_afu_disable(afu); in cxl_psl_purge()
128 cxl_p1n_write(afu, CXL_PSL_SCNTL_An, in cxl_psl_purge()
131 PSL_CNTL = cxl_p1n_read(afu, CXL_PSL_SCNTL_An); in cxl_psl_purge()
135 dev_warn(&afu->dev, "WARNING: PSL Purge timed out!\n"); in cxl_psl_purge()
139 if (!cxl_adapter_link_ok(afu->adapter)) { in cxl_psl_purge()
144 dsisr = cxl_p2n_read(afu, CXL_PSL_DSISR_An); in cxl_psl_purge()
147 dar = cxl_p2n_read(afu, CXL_PSL_DAR_An); in cxl_psl_purge()
148 …dev_notice(&afu->dev, "PSL purge terminating pending translation, DSISR: 0x%016llx, DAR: 0x%016llx… in cxl_psl_purge()
149 cxl_p2n_write(afu, CXL_PSL_TFC_An, CXL_PSL_TFC_An_AE); in cxl_psl_purge()
151 …dev_notice(&afu->dev, "PSL purge acknowledging pending non-translation fault, DSISR: 0x%016llx\n",… in cxl_psl_purge()
152 cxl_p2n_write(afu, CXL_PSL_TFC_An, CXL_PSL_TFC_An_A); in cxl_psl_purge()
156 PSL_CNTL = cxl_p1n_read(afu, CXL_PSL_SCNTL_An); in cxl_psl_purge()
161 cxl_p1n_write(afu, CXL_PSL_SCNTL_An, in cxl_psl_purge()
164 trace_cxl_psl_ctrl_done(afu, CXL_PSL_SCNTL_An_Pc, rc); in cxl_psl_purge()
186 int cxl_alloc_spa(struct cxl_afu *afu) in cxl_alloc_spa() argument
189 afu->spa_order = 0; in cxl_alloc_spa()
191 afu->spa_order++; in cxl_alloc_spa()
192 afu->spa_size = (1 << afu->spa_order) * PAGE_SIZE; in cxl_alloc_spa()
193 afu->spa_max_procs = spa_max_procs(afu->spa_size); in cxl_alloc_spa()
194 } while (afu->spa_max_procs < afu->num_procs); in cxl_alloc_spa()
196 WARN_ON(afu->spa_size > 0x100000); /* Max size supported by the hardware */ in cxl_alloc_spa()
198 if (!(afu->spa = (struct cxl_process_element *) in cxl_alloc_spa()
199 __get_free_pages(GFP_KERNEL | __GFP_ZERO, afu->spa_order))) { in cxl_alloc_spa()
204 1<<afu->spa_order, afu->spa_max_procs, afu->num_procs); in cxl_alloc_spa()
209 static void attach_spa(struct cxl_afu *afu) in attach_spa() argument
213 afu->sw_command_status = (__be64 *)((char *)afu->spa + in attach_spa()
214 ((afu->spa_max_procs + 3) * 128)); in attach_spa()
216 spap = virt_to_phys(afu->spa) & CXL_PSL_SPAP_Addr; in attach_spa()
217 spap |= ((afu->spa_size >> (12 - CXL_PSL_SPAP_Size_Shift)) - 1) & CXL_PSL_SPAP_Size; in attach_spa()
219 …: %i, sw_command_status: 0x%p CXL_PSL_SPAP_An=0x%016llx\n", afu->spa, afu->spa_max_procs, afu->sw_… in attach_spa()
220 cxl_p1n_write(afu, CXL_PSL_SPAP_An, spap); in attach_spa()
223 static inline void detach_spa(struct cxl_afu *afu) in detach_spa() argument
225 cxl_p1n_write(afu, CXL_PSL_SPAP_An, 0); in detach_spa()
228 void cxl_release_spa(struct cxl_afu *afu) in cxl_release_spa() argument
230 if (afu->spa) { in cxl_release_spa()
231 free_pages((unsigned long) afu->spa, afu->spa_order); in cxl_release_spa()
232 afu->spa = NULL; in cxl_release_spa()
268 int cxl_afu_slbia(struct cxl_afu *afu) in cxl_afu_slbia() argument
273 cxl_p2n_write(afu, CXL_SLBIA_An, CXL_TLB_SLB_IQ_ALL); in cxl_afu_slbia()
274 while (cxl_p2n_read(afu, CXL_SLBIA_An) & CXL_TLB_SLB_P) { in cxl_afu_slbia()
276 dev_warn(&afu->dev, "WARNING: CXL AFU SLBIA timed out!\n"); in cxl_afu_slbia()
282 if (!cxl_adapter_link_ok(afu->adapter)) in cxl_afu_slbia()
289 static int cxl_write_sstp(struct cxl_afu *afu, u64 sstp0, u64 sstp1) in cxl_write_sstp() argument
294 cxl_p2n_write(afu, CXL_SSTP1_An, 0); in cxl_write_sstp()
297 if ((rc = cxl_afu_slbia(afu))) in cxl_write_sstp()
301 cxl_p2n_write(afu, CXL_SSTP0_An, sstp0); in cxl_write_sstp()
304 cxl_p2n_write(afu, CXL_SSTP1_An, sstp1); in cxl_write_sstp()
312 struct cxl *adapter = ctx->afu->adapter; in slb_invalid()
315 WARN_ON(!mutex_is_locked(&ctx->afu->spa_mutex)); in slb_invalid()
341 WARN_ON(!ctx->afu->enabled); in do_process_element_cmd()
345 *(ctx->afu->sw_command_status) = cpu_to_be64(cmd | 0 | ctx->pe); in do_process_element_cmd()
347 cxl_p1n_write(ctx->afu, CXL_PSL_LLCMD_An, cmd | ctx->pe); in do_process_element_cmd()
350 dev_warn(&ctx->afu->dev, "WARNING: Process Element Command timed out!\n"); in do_process_element_cmd()
354 if (!cxl_adapter_link_ok(ctx->afu->adapter)) { in do_process_element_cmd()
355 dev_warn(&ctx->afu->dev, "WARNING: Device link down, aborting Process Element Command!\n"); in do_process_element_cmd()
359 state = be64_to_cpup(ctx->afu->sw_command_status); in do_process_element_cmd()
387 mutex_lock(&ctx->afu->spa_mutex); in add_process_element()
392 mutex_unlock(&ctx->afu->spa_mutex); in add_process_element()
404 mutex_lock(&ctx->afu->spa_mutex); in terminate_process_element()
410 if (cxl_adapter_link_ok(ctx->afu->adapter)) in terminate_process_element()
415 mutex_unlock(&ctx->afu->spa_mutex); in terminate_process_element()
423 mutex_lock(&ctx->afu->spa_mutex); in remove_process_element()
429 if (cxl_adapter_link_ok(ctx->afu->adapter)) in remove_process_element()
436 mutex_unlock(&ctx->afu->spa_mutex); in remove_process_element()
444 if (!ctx->afu->pp_size || ctx->master) { in cxl_assign_psn_space()
445 ctx->psn_phys = ctx->afu->psn_phys; in cxl_assign_psn_space()
446 ctx->psn_size = ctx->afu->adapter->ps_size; in cxl_assign_psn_space()
448 ctx->psn_phys = ctx->afu->psn_phys + in cxl_assign_psn_space()
449 (ctx->afu->pp_offset + ctx->afu->pp_size * ctx->pe); in cxl_assign_psn_space()
450 ctx->psn_size = ctx->afu->pp_size; in cxl_assign_psn_space()
454 static int activate_afu_directed(struct cxl_afu *afu) in activate_afu_directed() argument
458 dev_info(&afu->dev, "Activating AFU directed mode\n"); in activate_afu_directed()
460 afu->num_procs = afu->max_procs_virtualised; in activate_afu_directed()
461 if (afu->spa == NULL) { in activate_afu_directed()
462 if (cxl_alloc_spa(afu)) in activate_afu_directed()
465 attach_spa(afu); in activate_afu_directed()
467 cxl_p1n_write(afu, CXL_PSL_SCNTL_An, CXL_PSL_SCNTL_An_PM_AFU); in activate_afu_directed()
468 cxl_p1n_write(afu, CXL_PSL_AMOR_An, 0xFFFFFFFFFFFFFFFFULL); in activate_afu_directed()
469 cxl_p1n_write(afu, CXL_PSL_ID_An, CXL_PSL_ID_An_F | CXL_PSL_ID_An_L); in activate_afu_directed()
471 afu->current_mode = CXL_MODE_DIRECTED; in activate_afu_directed()
473 if ((rc = cxl_chardev_m_afu_add(afu))) in activate_afu_directed()
476 if ((rc = cxl_sysfs_afu_m_add(afu))) in activate_afu_directed()
479 if ((rc = cxl_chardev_s_afu_add(afu))) in activate_afu_directed()
484 cxl_sysfs_afu_m_remove(afu); in activate_afu_directed()
486 cxl_chardev_afu_remove(afu); in activate_afu_directed()
555 if ((result = cxl_afu_check_and_enable(ctx->afu))) in attach_afu_directed()
561 static int deactivate_afu_directed(struct cxl_afu *afu) in deactivate_afu_directed() argument
563 dev_info(&afu->dev, "Deactivating AFU directed mode\n"); in deactivate_afu_directed()
565 afu->current_mode = 0; in deactivate_afu_directed()
566 afu->num_procs = 0; in deactivate_afu_directed()
568 cxl_sysfs_afu_m_remove(afu); in deactivate_afu_directed()
569 cxl_chardev_afu_remove(afu); in deactivate_afu_directed()
571 __cxl_afu_reset(afu); in deactivate_afu_directed()
572 cxl_afu_disable(afu); in deactivate_afu_directed()
573 cxl_psl_purge(afu); in deactivate_afu_directed()
578 static int activate_dedicated_process(struct cxl_afu *afu) in activate_dedicated_process() argument
580 dev_info(&afu->dev, "Activating dedicated process mode\n"); in activate_dedicated_process()
582 cxl_p1n_write(afu, CXL_PSL_SCNTL_An, CXL_PSL_SCNTL_An_PM_Process); in activate_dedicated_process()
584 cxl_p1n_write(afu, CXL_PSL_CtxTime_An, 0); /* disable */ in activate_dedicated_process()
585 cxl_p1n_write(afu, CXL_PSL_SPAP_An, 0); /* disable */ in activate_dedicated_process()
586 cxl_p1n_write(afu, CXL_PSL_AMOR_An, 0xFFFFFFFFFFFFFFFFULL); in activate_dedicated_process()
587 cxl_p1n_write(afu, CXL_PSL_LPID_An, mfspr(SPRN_LPID)); in activate_dedicated_process()
588 cxl_p1n_write(afu, CXL_HAURP_An, 0); /* disable */ in activate_dedicated_process()
589 cxl_p1n_write(afu, CXL_PSL_SDR_An, mfspr(SPRN_SDR1)); in activate_dedicated_process()
591 cxl_p2n_write(afu, CXL_CSRP_An, 0); /* disable */ in activate_dedicated_process()
592 cxl_p2n_write(afu, CXL_AURP0_An, 0); /* disable */ in activate_dedicated_process()
593 cxl_p2n_write(afu, CXL_AURP1_An, 0); /* disable */ in activate_dedicated_process()
595 afu->current_mode = CXL_MODE_DEDICATED; in activate_dedicated_process()
596 afu->num_procs = 1; in activate_dedicated_process()
598 return cxl_chardev_d_afu_add(afu); in activate_dedicated_process()
603 struct cxl_afu *afu = ctx->afu; in attach_dedicated() local
610 cxl_p2n_write(afu, CXL_PSL_PID_TID_An, pid); in attach_dedicated()
612 cxl_p1n_write(afu, CXL_PSL_SR_An, calculate_sr(ctx)); in attach_dedicated()
614 if ((rc = cxl_write_sstp(afu, ctx->sstp0, ctx->sstp1))) in attach_dedicated()
619 cxl_p1n_write(afu, CXL_PSL_IVTE_Offset_An, in attach_dedicated()
624 cxl_p1n_write(afu, CXL_PSL_IVTE_Limit_An, (u64) in attach_dedicated()
630 cxl_p2n_write(afu, CXL_PSL_AMR_An, amr); in attach_dedicated()
635 if ((rc = __cxl_afu_reset(afu))) in attach_dedicated()
638 cxl_p2n_write(afu, CXL_PSL_WED_An, wed); in attach_dedicated()
640 return afu_enable(afu); in attach_dedicated()
643 static int deactivate_dedicated_process(struct cxl_afu *afu) in deactivate_dedicated_process() argument
645 dev_info(&afu->dev, "Deactivating dedicated process mode\n"); in deactivate_dedicated_process()
647 afu->current_mode = 0; in deactivate_dedicated_process()
648 afu->num_procs = 0; in deactivate_dedicated_process()
650 cxl_chardev_afu_remove(afu); in deactivate_dedicated_process()
655 int _cxl_afu_deactivate_mode(struct cxl_afu *afu, int mode) in _cxl_afu_deactivate_mode() argument
658 return deactivate_afu_directed(afu); in _cxl_afu_deactivate_mode()
660 return deactivate_dedicated_process(afu); in _cxl_afu_deactivate_mode()
664 int cxl_afu_deactivate_mode(struct cxl_afu *afu) in cxl_afu_deactivate_mode() argument
666 return _cxl_afu_deactivate_mode(afu, afu->current_mode); in cxl_afu_deactivate_mode()
669 int cxl_afu_activate_mode(struct cxl_afu *afu, int mode) in cxl_afu_activate_mode() argument
673 if (!(mode & afu->modes_supported)) in cxl_afu_activate_mode()
676 if (!cxl_adapter_link_ok(afu->adapter)) { in cxl_afu_activate_mode()
682 return activate_afu_directed(afu); in cxl_afu_activate_mode()
684 return activate_dedicated_process(afu); in cxl_afu_activate_mode()
691 if (!cxl_adapter_link_ok(ctx->afu->adapter)) { in cxl_attach_process()
697 if (ctx->afu->current_mode == CXL_MODE_DIRECTED) in cxl_attach_process()
700 if (ctx->afu->current_mode == CXL_MODE_DEDICATED) in cxl_attach_process()
708 __cxl_afu_reset(ctx->afu); in detach_process_native_dedicated()
709 cxl_afu_disable(ctx->afu); in detach_process_native_dedicated()
710 cxl_psl_purge(ctx->afu); in detach_process_native_dedicated()
730 if (ctx->afu->current_mode == CXL_MODE_DEDICATED) in cxl_detach_process()
736 int cxl_get_irq(struct cxl_afu *afu, struct cxl_irq_info *info) in cxl_get_irq() argument
743 if (!cxl_adapter_link_ok(afu->adapter)) in cxl_get_irq()
746 info->dsisr = cxl_p2n_read(afu, CXL_PSL_DSISR_An); in cxl_get_irq()
747 info->dar = cxl_p2n_read(afu, CXL_PSL_DAR_An); in cxl_get_irq()
748 info->dsr = cxl_p2n_read(afu, CXL_PSL_DSR_An); in cxl_get_irq()
749 pidtid = cxl_p2n_read(afu, CXL_PSL_PID_TID_An); in cxl_get_irq()
752 info->afu_err = cxl_p2n_read(afu, CXL_AFU_ERR_An); in cxl_get_irq()
753 info->errstat = cxl_p2n_read(afu, CXL_PSL_ErrStat_An); in cxl_get_irq()
758 static void recover_psl_err(struct cxl_afu *afu, u64 errstat) in recover_psl_err() argument
765 dsisr = cxl_p2n_read(afu, CXL_PSL_DSISR_An); in recover_psl_err()
766 cxl_p2n_write(afu, CXL_PSL_DSISR_An, dsisr & ~CXL_PSL_DSISR_An_PE); in recover_psl_err()
769 cxl_p2n_write(afu, CXL_PSL_ErrStat_An, errstat); in recover_psl_err()
776 cxl_p2n_write(ctx->afu, CXL_PSL_TFC_An, tfc); in cxl_ack_irq()
778 recover_psl_err(ctx->afu, psl_reset_mask); in cxl_ack_irq()
783 int cxl_check_error(struct cxl_afu *afu) in cxl_check_error() argument
785 return (cxl_p1n_read(afu, CXL_PSL_SCNTL_An) == ~0ULL); in cxl_check_error()