Lines Matching refs:mm
549 static void _ppgtt_get_root_entry(struct intel_vgpu_mm *mm, in _ppgtt_get_root_entry() argument
553 const struct intel_gvt_gtt_pte_ops *pte_ops = mm->vgpu->gvt->gtt.pte_ops; in _ppgtt_get_root_entry()
555 GEM_BUG_ON(mm->type != INTEL_GVT_MM_PPGTT); in _ppgtt_get_root_entry()
557 entry->type = mm->ppgtt_mm.root_entry_type; in _ppgtt_get_root_entry()
558 pte_ops->get_entry(guest ? mm->ppgtt_mm.guest_pdps : in _ppgtt_get_root_entry()
559 mm->ppgtt_mm.shadow_pdps, in _ppgtt_get_root_entry()
560 entry, index, false, 0, mm->vgpu); in _ppgtt_get_root_entry()
564 static inline void ppgtt_get_guest_root_entry(struct intel_vgpu_mm *mm, in ppgtt_get_guest_root_entry() argument
567 _ppgtt_get_root_entry(mm, entry, index, true); in ppgtt_get_guest_root_entry()
570 static inline void ppgtt_get_shadow_root_entry(struct intel_vgpu_mm *mm, in ppgtt_get_shadow_root_entry() argument
573 _ppgtt_get_root_entry(mm, entry, index, false); in ppgtt_get_shadow_root_entry()
576 static void _ppgtt_set_root_entry(struct intel_vgpu_mm *mm, in _ppgtt_set_root_entry() argument
580 const struct intel_gvt_gtt_pte_ops *pte_ops = mm->vgpu->gvt->gtt.pte_ops; in _ppgtt_set_root_entry()
582 pte_ops->set_entry(guest ? mm->ppgtt_mm.guest_pdps : in _ppgtt_set_root_entry()
583 mm->ppgtt_mm.shadow_pdps, in _ppgtt_set_root_entry()
584 entry, index, false, 0, mm->vgpu); in _ppgtt_set_root_entry()
587 static inline void ppgtt_set_shadow_root_entry(struct intel_vgpu_mm *mm, in ppgtt_set_shadow_root_entry() argument
590 _ppgtt_set_root_entry(mm, entry, index, false); in ppgtt_set_shadow_root_entry()
593 static void ggtt_get_guest_entry(struct intel_vgpu_mm *mm, in ggtt_get_guest_entry() argument
596 const struct intel_gvt_gtt_pte_ops *pte_ops = mm->vgpu->gvt->gtt.pte_ops; in ggtt_get_guest_entry()
598 GEM_BUG_ON(mm->type != INTEL_GVT_MM_GGTT); in ggtt_get_guest_entry()
601 pte_ops->get_entry(mm->ggtt_mm.virtual_ggtt, entry, index, in ggtt_get_guest_entry()
602 false, 0, mm->vgpu); in ggtt_get_guest_entry()
605 static void ggtt_set_guest_entry(struct intel_vgpu_mm *mm, in ggtt_set_guest_entry() argument
608 const struct intel_gvt_gtt_pte_ops *pte_ops = mm->vgpu->gvt->gtt.pte_ops; in ggtt_set_guest_entry()
610 GEM_BUG_ON(mm->type != INTEL_GVT_MM_GGTT); in ggtt_set_guest_entry()
612 pte_ops->set_entry(mm->ggtt_mm.virtual_ggtt, entry, index, in ggtt_set_guest_entry()
613 false, 0, mm->vgpu); in ggtt_set_guest_entry()
616 static void ggtt_get_host_entry(struct intel_vgpu_mm *mm, in ggtt_get_host_entry() argument
619 const struct intel_gvt_gtt_pte_ops *pte_ops = mm->vgpu->gvt->gtt.pte_ops; in ggtt_get_host_entry()
621 GEM_BUG_ON(mm->type != INTEL_GVT_MM_GGTT); in ggtt_get_host_entry()
623 pte_ops->get_entry(NULL, entry, index, false, 0, mm->vgpu); in ggtt_get_host_entry()
626 static void ggtt_set_host_entry(struct intel_vgpu_mm *mm, in ggtt_set_host_entry() argument
629 const struct intel_gvt_gtt_pte_ops *pte_ops = mm->vgpu->gvt->gtt.pte_ops; in ggtt_set_host_entry()
632 GEM_BUG_ON(mm->type != INTEL_GVT_MM_GGTT); in ggtt_set_host_entry()
634 if (vgpu_gmadr_is_aperture(mm->vgpu, index << I915_GTT_PAGE_SHIFT)) { in ggtt_set_host_entry()
635 offset -= (vgpu_aperture_gmadr_base(mm->vgpu) >> PAGE_SHIFT); in ggtt_set_host_entry()
636 mm->ggtt_mm.host_ggtt_aperture[offset] = entry->val64; in ggtt_set_host_entry()
637 } else if (vgpu_gmadr_is_hidden(mm->vgpu, index << I915_GTT_PAGE_SHIFT)) { in ggtt_set_host_entry()
638 offset -= (vgpu_hidden_gmadr_base(mm->vgpu) >> PAGE_SHIFT); in ggtt_set_host_entry()
639 mm->ggtt_mm.host_ggtt_hidden[offset] = entry->val64; in ggtt_set_host_entry()
642 pte_ops->set_entry(NULL, entry, index, false, 0, mm->vgpu); in ggtt_set_host_entry()
1753 static void invalidate_ppgtt_mm(struct intel_vgpu_mm *mm) in invalidate_ppgtt_mm() argument
1755 struct intel_vgpu *vgpu = mm->vgpu; in invalidate_ppgtt_mm()
1762 if (!mm->ppgtt_mm.shadowed) in invalidate_ppgtt_mm()
1765 for (index = 0; index < ARRAY_SIZE(mm->ppgtt_mm.shadow_pdps); index++) { in invalidate_ppgtt_mm()
1766 ppgtt_get_shadow_root_entry(mm, &se, index); in invalidate_ppgtt_mm()
1773 ppgtt_set_shadow_root_entry(mm, &se, index); in invalidate_ppgtt_mm()
1779 mm->ppgtt_mm.shadowed = false; in invalidate_ppgtt_mm()
1783 static int shadow_ppgtt_mm(struct intel_vgpu_mm *mm) in shadow_ppgtt_mm() argument
1785 struct intel_vgpu *vgpu = mm->vgpu; in shadow_ppgtt_mm()
1793 if (mm->ppgtt_mm.shadowed) in shadow_ppgtt_mm()
1799 mm->ppgtt_mm.shadowed = true; in shadow_ppgtt_mm()
1801 for (index = 0; index < ARRAY_SIZE(mm->ppgtt_mm.guest_pdps); index++) { in shadow_ppgtt_mm()
1802 ppgtt_get_guest_root_entry(mm, &ge, index); in shadow_ppgtt_mm()
1817 ppgtt_set_shadow_root_entry(mm, &se, index); in shadow_ppgtt_mm()
1825 invalidate_ppgtt_mm(mm); in shadow_ppgtt_mm()
1831 struct intel_vgpu_mm *mm; in vgpu_alloc_mm() local
1833 mm = kzalloc(sizeof(*mm), GFP_KERNEL); in vgpu_alloc_mm()
1834 if (!mm) in vgpu_alloc_mm()
1837 mm->vgpu = vgpu; in vgpu_alloc_mm()
1838 kref_init(&mm->ref); in vgpu_alloc_mm()
1839 atomic_set(&mm->pincount, 0); in vgpu_alloc_mm()
1841 return mm; in vgpu_alloc_mm()
1844 static void vgpu_free_mm(struct intel_vgpu_mm *mm) in vgpu_free_mm() argument
1846 kfree(mm); in vgpu_free_mm()
1864 struct intel_vgpu_mm *mm; in intel_vgpu_create_ppgtt_mm() local
1867 mm = vgpu_alloc_mm(vgpu); in intel_vgpu_create_ppgtt_mm()
1868 if (!mm) in intel_vgpu_create_ppgtt_mm()
1871 mm->type = INTEL_GVT_MM_PPGTT; in intel_vgpu_create_ppgtt_mm()
1875 mm->ppgtt_mm.root_entry_type = root_entry_type; in intel_vgpu_create_ppgtt_mm()
1877 INIT_LIST_HEAD(&mm->ppgtt_mm.list); in intel_vgpu_create_ppgtt_mm()
1878 INIT_LIST_HEAD(&mm->ppgtt_mm.lru_list); in intel_vgpu_create_ppgtt_mm()
1879 INIT_LIST_HEAD(&mm->ppgtt_mm.link); in intel_vgpu_create_ppgtt_mm()
1882 mm->ppgtt_mm.guest_pdps[0] = pdps[0]; in intel_vgpu_create_ppgtt_mm()
1884 memcpy(mm->ppgtt_mm.guest_pdps, pdps, in intel_vgpu_create_ppgtt_mm()
1885 sizeof(mm->ppgtt_mm.guest_pdps)); in intel_vgpu_create_ppgtt_mm()
1887 ret = shadow_ppgtt_mm(mm); in intel_vgpu_create_ppgtt_mm()
1890 vgpu_free_mm(mm); in intel_vgpu_create_ppgtt_mm()
1894 list_add_tail(&mm->ppgtt_mm.list, &vgpu->gtt.ppgtt_mm_list_head); in intel_vgpu_create_ppgtt_mm()
1897 list_add_tail(&mm->ppgtt_mm.lru_list, &gvt->gtt.ppgtt_mm_lru_list_head); in intel_vgpu_create_ppgtt_mm()
1900 return mm; in intel_vgpu_create_ppgtt_mm()
1905 struct intel_vgpu_mm *mm; in intel_vgpu_create_ggtt_mm() local
1908 mm = vgpu_alloc_mm(vgpu); in intel_vgpu_create_ggtt_mm()
1909 if (!mm) in intel_vgpu_create_ggtt_mm()
1912 mm->type = INTEL_GVT_MM_GGTT; in intel_vgpu_create_ggtt_mm()
1915 mm->ggtt_mm.virtual_ggtt = in intel_vgpu_create_ggtt_mm()
1918 if (!mm->ggtt_mm.virtual_ggtt) { in intel_vgpu_create_ggtt_mm()
1919 vgpu_free_mm(mm); in intel_vgpu_create_ggtt_mm()
1923 mm->ggtt_mm.host_ggtt_aperture = vzalloc((vgpu_aperture_sz(vgpu) >> PAGE_SHIFT) * sizeof(u64)); in intel_vgpu_create_ggtt_mm()
1924 if (!mm->ggtt_mm.host_ggtt_aperture) { in intel_vgpu_create_ggtt_mm()
1925 vfree(mm->ggtt_mm.virtual_ggtt); in intel_vgpu_create_ggtt_mm()
1926 vgpu_free_mm(mm); in intel_vgpu_create_ggtt_mm()
1930 mm->ggtt_mm.host_ggtt_hidden = vzalloc((vgpu_hidden_sz(vgpu) >> PAGE_SHIFT) * sizeof(u64)); in intel_vgpu_create_ggtt_mm()
1931 if (!mm->ggtt_mm.host_ggtt_hidden) { in intel_vgpu_create_ggtt_mm()
1932 vfree(mm->ggtt_mm.host_ggtt_aperture); in intel_vgpu_create_ggtt_mm()
1933 vfree(mm->ggtt_mm.virtual_ggtt); in intel_vgpu_create_ggtt_mm()
1934 vgpu_free_mm(mm); in intel_vgpu_create_ggtt_mm()
1938 return mm; in intel_vgpu_create_ggtt_mm()
1950 struct intel_vgpu_mm *mm = container_of(mm_ref, typeof(*mm), ref); in _intel_vgpu_mm_release() local
1952 if (GEM_WARN_ON(atomic_read(&mm->pincount))) in _intel_vgpu_mm_release()
1955 if (mm->type == INTEL_GVT_MM_PPGTT) { in _intel_vgpu_mm_release()
1956 list_del(&mm->ppgtt_mm.list); in _intel_vgpu_mm_release()
1958 mutex_lock(&mm->vgpu->gvt->gtt.ppgtt_mm_lock); in _intel_vgpu_mm_release()
1959 list_del(&mm->ppgtt_mm.lru_list); in _intel_vgpu_mm_release()
1960 mutex_unlock(&mm->vgpu->gvt->gtt.ppgtt_mm_lock); in _intel_vgpu_mm_release()
1962 invalidate_ppgtt_mm(mm); in _intel_vgpu_mm_release()
1964 vfree(mm->ggtt_mm.virtual_ggtt); in _intel_vgpu_mm_release()
1965 vfree(mm->ggtt_mm.host_ggtt_aperture); in _intel_vgpu_mm_release()
1966 vfree(mm->ggtt_mm.host_ggtt_hidden); in _intel_vgpu_mm_release()
1969 vgpu_free_mm(mm); in _intel_vgpu_mm_release()
1978 void intel_vgpu_unpin_mm(struct intel_vgpu_mm *mm) in intel_vgpu_unpin_mm() argument
1980 atomic_dec_if_positive(&mm->pincount); in intel_vgpu_unpin_mm()
1994 int intel_vgpu_pin_mm(struct intel_vgpu_mm *mm) in intel_vgpu_pin_mm() argument
1998 atomic_inc(&mm->pincount); in intel_vgpu_pin_mm()
2000 if (mm->type == INTEL_GVT_MM_PPGTT) { in intel_vgpu_pin_mm()
2001 ret = shadow_ppgtt_mm(mm); in intel_vgpu_pin_mm()
2005 mutex_lock(&mm->vgpu->gvt->gtt.ppgtt_mm_lock); in intel_vgpu_pin_mm()
2006 list_move_tail(&mm->ppgtt_mm.lru_list, in intel_vgpu_pin_mm()
2007 &mm->vgpu->gvt->gtt.ppgtt_mm_lru_list_head); in intel_vgpu_pin_mm()
2008 mutex_unlock(&mm->vgpu->gvt->gtt.ppgtt_mm_lock); in intel_vgpu_pin_mm()
2016 struct intel_vgpu_mm *mm; in reclaim_one_ppgtt_mm() local
2022 mm = container_of(pos, struct intel_vgpu_mm, ppgtt_mm.lru_list); in reclaim_one_ppgtt_mm()
2024 if (atomic_read(&mm->pincount)) in reclaim_one_ppgtt_mm()
2027 list_del_init(&mm->ppgtt_mm.lru_list); in reclaim_one_ppgtt_mm()
2029 invalidate_ppgtt_mm(mm); in reclaim_one_ppgtt_mm()
2039 static inline int ppgtt_get_next_level_entry(struct intel_vgpu_mm *mm, in ppgtt_get_next_level_entry() argument
2042 struct intel_vgpu *vgpu = mm->vgpu; in ppgtt_get_next_level_entry()
2068 unsigned long intel_vgpu_gma_to_gpa(struct intel_vgpu_mm *mm, unsigned long gma) in intel_vgpu_gma_to_gpa() argument
2070 struct intel_vgpu *vgpu = mm->vgpu; in intel_vgpu_gma_to_gpa()
2080 GEM_BUG_ON(mm->type != INTEL_GVT_MM_GGTT && in intel_vgpu_gma_to_gpa()
2081 mm->type != INTEL_GVT_MM_PPGTT); in intel_vgpu_gma_to_gpa()
2083 if (mm->type == INTEL_GVT_MM_GGTT) { in intel_vgpu_gma_to_gpa()
2087 ggtt_get_guest_entry(mm, &e, in intel_vgpu_gma_to_gpa()
2095 switch (mm->ppgtt_mm.root_entry_type) { in intel_vgpu_gma_to_gpa()
2097 ppgtt_get_shadow_root_entry(mm, &e, 0); in intel_vgpu_gma_to_gpa()
2106 ppgtt_get_shadow_root_entry(mm, &e, in intel_vgpu_gma_to_gpa()
2119 ret = ppgtt_get_next_level_entry(mm, &e, gma_index[i], in intel_vgpu_gma_to_gpa()
2133 mm->ppgtt_mm.root_entry_type, gma, gpa); in intel_vgpu_gma_to_gpa()
2138 gvt_vgpu_err("invalid mm type: %d gma %lx\n", mm->type, gma); in intel_vgpu_gma_to_gpa()
2489 struct intel_vgpu_mm *mm; in intel_vgpu_destroy_all_ppgtt_mm() local
2492 mm = container_of(pos, struct intel_vgpu_mm, ppgtt_mm.list); in intel_vgpu_destroy_all_ppgtt_mm()
2493 intel_vgpu_destroy_mm(mm); in intel_vgpu_destroy_all_ppgtt_mm()
2604 struct intel_vgpu_mm *mm; in intel_vgpu_find_ppgtt_mm() local
2608 mm = container_of(pos, struct intel_vgpu_mm, ppgtt_mm.list); in intel_vgpu_find_ppgtt_mm()
2610 switch (mm->ppgtt_mm.root_entry_type) { in intel_vgpu_find_ppgtt_mm()
2612 if (pdps[0] == mm->ppgtt_mm.guest_pdps[0]) in intel_vgpu_find_ppgtt_mm()
2613 return mm; in intel_vgpu_find_ppgtt_mm()
2616 if (!memcmp(pdps, mm->ppgtt_mm.guest_pdps, in intel_vgpu_find_ppgtt_mm()
2617 sizeof(mm->ppgtt_mm.guest_pdps))) in intel_vgpu_find_ppgtt_mm()
2618 return mm; in intel_vgpu_find_ppgtt_mm()
2641 struct intel_vgpu_mm *mm; in intel_vgpu_get_ppgtt_mm() local
2643 mm = intel_vgpu_find_ppgtt_mm(vgpu, pdps); in intel_vgpu_get_ppgtt_mm()
2644 if (mm) { in intel_vgpu_get_ppgtt_mm()
2645 intel_vgpu_mm_get(mm); in intel_vgpu_get_ppgtt_mm()
2647 mm = intel_vgpu_create_ppgtt_mm(vgpu, root_entry_type, pdps); in intel_vgpu_get_ppgtt_mm()
2648 if (IS_ERR(mm)) in intel_vgpu_get_ppgtt_mm()
2651 return mm; in intel_vgpu_get_ppgtt_mm()
2666 struct intel_vgpu_mm *mm; in intel_vgpu_put_ppgtt_mm() local
2668 mm = intel_vgpu_find_ppgtt_mm(vgpu, pdps); in intel_vgpu_put_ppgtt_mm()
2669 if (!mm) { in intel_vgpu_put_ppgtt_mm()
2673 intel_vgpu_mm_put(mm); in intel_vgpu_put_ppgtt_mm()
2762 struct intel_vgpu_mm *mm; in intel_vgpu_invalidate_ppgtt() local
2765 mm = container_of(pos, struct intel_vgpu_mm, ppgtt_mm.list); in intel_vgpu_invalidate_ppgtt()
2766 if (mm->type == INTEL_GVT_MM_PPGTT) { in intel_vgpu_invalidate_ppgtt()
2768 list_del_init(&mm->ppgtt_mm.lru_list); in intel_vgpu_invalidate_ppgtt()
2770 if (mm->ppgtt_mm.shadowed) in intel_vgpu_invalidate_ppgtt()
2771 invalidate_ppgtt_mm(mm); in intel_vgpu_invalidate_ppgtt()
2831 struct intel_vgpu_mm *mm; in intel_gvt_restore_ggtt() local
2838 mm = vgpu->gtt.ggtt_mm; in intel_gvt_restore_ggtt()
2843 pte = mm->ggtt_mm.host_ggtt_aperture[idx]; in intel_gvt_restore_ggtt()
2851 pte = mm->ggtt_mm.host_ggtt_hidden[idx]; in intel_gvt_restore_ggtt()