Lines Matching +full:data +full:- +full:addr
1 // SPDX-License-Identifier: GPL-2.0-only
3 * Stand-alone page-table allocator for hyp stage-1 and guest stage-2.
52 u64 addr; member
67 static bool kvm_block_mapping_supported(u64 addr, u64 end, u64 phys, u32 level) in kvm_block_mapping_supported() argument
73 * 52-bit PAs. in kvm_block_mapping_supported()
78 if (granule > (end - addr)) in kvm_block_mapping_supported()
81 return IS_ALIGNED(addr, granule) && IS_ALIGNED(phys, granule); in kvm_block_mapping_supported()
84 static u32 kvm_pgtable_idx(struct kvm_pgtable_walk_data *data, u32 level) in kvm_pgtable_idx() argument
87 u64 mask = BIT(PAGE_SHIFT - 3) - 1; in kvm_pgtable_idx()
89 return (data->addr >> shift) & mask; in kvm_pgtable_idx()
92 static u32 __kvm_pgd_page_idx(struct kvm_pgtable *pgt, u64 addr) in __kvm_pgd_page_idx() argument
94 u64 shift = kvm_granule_shift(pgt->start_level - 1); /* May underflow */ in __kvm_pgd_page_idx()
95 u64 mask = BIT(pgt->ia_bits) - 1; in __kvm_pgd_page_idx()
97 return (addr & mask) >> shift; in __kvm_pgd_page_idx()
100 static u32 kvm_pgd_page_idx(struct kvm_pgtable_walk_data *data) in kvm_pgd_page_idx() argument
102 return __kvm_pgd_page_idx(data->pgt, data->addr); in kvm_pgd_page_idx()
112 return __kvm_pgd_page_idx(&pgt, -1ULL) + 1; in kvm_pgd_pages()
122 if (level == KVM_PGTABLE_MAX_LEVELS - 1) in kvm_pte_table()
177 u64 type = (level == KVM_PGTABLE_MAX_LEVELS - 1) ? KVM_PTE_TYPE_PAGE : in kvm_set_valid_leaf_pte()
192 static int kvm_pgtable_visitor_cb(struct kvm_pgtable_walk_data *data, u64 addr, in kvm_pgtable_visitor_cb() argument
196 struct kvm_pgtable_walker *walker = data->walker; in kvm_pgtable_visitor_cb()
197 return walker->cb(addr, data->end, level, ptep, flag, walker->arg); in kvm_pgtable_visitor_cb()
200 static int __kvm_pgtable_walk(struct kvm_pgtable_walk_data *data,
203 static inline int __kvm_pgtable_visit(struct kvm_pgtable_walk_data *data, in __kvm_pgtable_visit() argument
207 u64 addr = data->addr; in __kvm_pgtable_visit() local
210 enum kvm_pgtable_walk_flags flags = data->walker->flags; in __kvm_pgtable_visit()
213 ret = kvm_pgtable_visitor_cb(data, addr, level, ptep, in __kvm_pgtable_visit()
218 ret = kvm_pgtable_visitor_cb(data, addr, level, ptep, in __kvm_pgtable_visit()
228 data->addr = ALIGN_DOWN(data->addr, kvm_granule_size(level)); in __kvm_pgtable_visit()
229 data->addr += kvm_granule_size(level); in __kvm_pgtable_visit()
234 ret = __kvm_pgtable_walk(data, childp, level + 1); in __kvm_pgtable_visit()
239 ret = kvm_pgtable_visitor_cb(data, addr, level, ptep, in __kvm_pgtable_visit()
247 static int __kvm_pgtable_walk(struct kvm_pgtable_walk_data *data, in __kvm_pgtable_walk() argument
254 return -EINVAL; in __kvm_pgtable_walk()
256 for (idx = kvm_pgtable_idx(data, level); idx < PTRS_PER_PTE; ++idx) { in __kvm_pgtable_walk()
259 if (data->addr >= data->end) in __kvm_pgtable_walk()
262 ret = __kvm_pgtable_visit(data, ptep, level); in __kvm_pgtable_walk()
270 static int _kvm_pgtable_walk(struct kvm_pgtable_walk_data *data) in _kvm_pgtable_walk() argument
274 struct kvm_pgtable *pgt = data->pgt; in _kvm_pgtable_walk()
275 u64 limit = BIT(pgt->ia_bits); in _kvm_pgtable_walk()
277 if (data->addr > limit || data->end > limit) in _kvm_pgtable_walk()
278 return -ERANGE; in _kvm_pgtable_walk()
280 if (!pgt->pgd) in _kvm_pgtable_walk()
281 return -EINVAL; in _kvm_pgtable_walk()
283 for (idx = kvm_pgd_page_idx(data); data->addr < data->end; ++idx) { in _kvm_pgtable_walk()
284 kvm_pte_t *ptep = &pgt->pgd[idx * PTRS_PER_PTE]; in _kvm_pgtable_walk()
286 ret = __kvm_pgtable_walk(data, ptep, pgt->start_level); in _kvm_pgtable_walk()
294 int kvm_pgtable_walk(struct kvm_pgtable *pgt, u64 addr, u64 size, in kvm_pgtable_walk() argument
299 .addr = ALIGN_DOWN(addr, PAGE_SIZE), in kvm_pgtable_walk()
300 .end = PAGE_ALIGN(walk_data.addr + size), in kvm_pgtable_walk()
313 struct hyp_map_data *data) in hyp_map_set_prot_attr() argument
323 return -EINVAL; in hyp_map_set_prot_attr()
327 return -EINVAL; in hyp_map_set_prot_attr()
330 return -EINVAL; in hyp_map_set_prot_attr()
338 data->attr = attr; in hyp_map_set_prot_attr()
342 static bool hyp_map_walker_try_leaf(u64 addr, u64 end, u32 level, in hyp_map_walker_try_leaf() argument
343 kvm_pte_t *ptep, struct hyp_map_data *data) in hyp_map_walker_try_leaf() argument
345 u64 granule = kvm_granule_size(level), phys = data->phys; in hyp_map_walker_try_leaf()
347 if (!kvm_block_mapping_supported(addr, end, phys, level)) in hyp_map_walker_try_leaf()
350 WARN_ON(!kvm_set_valid_leaf_pte(ptep, phys, data->attr, level)); in hyp_map_walker_try_leaf()
351 data->phys += granule; in hyp_map_walker_try_leaf()
355 static int hyp_map_walker(u64 addr, u64 end, u32 level, kvm_pte_t *ptep, in hyp_map_walker() argument
360 if (hyp_map_walker_try_leaf(addr, end, level, ptep, arg)) in hyp_map_walker()
363 if (WARN_ON(level == KVM_PGTABLE_MAX_LEVELS - 1)) in hyp_map_walker()
364 return -EINVAL; in hyp_map_walker()
368 return -ENOMEM; in hyp_map_walker()
374 int kvm_pgtable_hyp_map(struct kvm_pgtable *pgt, u64 addr, u64 size, u64 phys, in kvm_pgtable_hyp_map() argument
391 ret = kvm_pgtable_walk(pgt, addr, size, &walker); in kvm_pgtable_hyp_map()
401 pgt->pgd = (kvm_pte_t *)get_zeroed_page(GFP_KERNEL); in kvm_pgtable_hyp_init()
402 if (!pgt->pgd) in kvm_pgtable_hyp_init()
403 return -ENOMEM; in kvm_pgtable_hyp_init()
405 pgt->ia_bits = va_bits; in kvm_pgtable_hyp_init()
406 pgt->start_level = KVM_PGTABLE_MAX_LEVELS - levels; in kvm_pgtable_hyp_init()
407 pgt->mmu = NULL; in kvm_pgtable_hyp_init()
411 static int hyp_free_walker(u64 addr, u64 end, u32 level, kvm_pte_t *ptep, in hyp_free_walker() argument
425 WARN_ON(kvm_pgtable_walk(pgt, 0, BIT(pgt->ia_bits), &walker)); in kvm_pgtable_hyp_destroy()
426 free_page((unsigned long)pgt->pgd); in kvm_pgtable_hyp_destroy()
427 pgt->pgd = NULL; in kvm_pgtable_hyp_destroy()
441 struct stage2_map_data *data) in stage2_map_set_prot_attr() argument
451 return -EINVAL; in stage2_map_set_prot_attr()
461 data->attr = attr; in stage2_map_set_prot_attr()
465 static bool stage2_map_walker_try_leaf(u64 addr, u64 end, u32 level, in stage2_map_walker_try_leaf() argument
467 struct stage2_map_data *data) in stage2_map_walker_try_leaf() argument
469 u64 granule = kvm_granule_size(level), phys = data->phys; in stage2_map_walker_try_leaf()
471 if (!kvm_block_mapping_supported(addr, end, phys, level)) in stage2_map_walker_try_leaf()
476 * early, as it will be bumped-up again in stage2_map_walk_leaf(). in stage2_map_walker_try_leaf()
483 if (kvm_set_valid_leaf_pte(ptep, phys, data->attr, level)) in stage2_map_walker_try_leaf()
486 /* There's an existing valid leaf entry, so perform break-before-make */ in stage2_map_walker_try_leaf()
488 kvm_call_hyp(__kvm_tlb_flush_vmid_ipa, data->mmu, addr, level); in stage2_map_walker_try_leaf()
489 kvm_set_valid_leaf_pte(ptep, phys, data->attr, level); in stage2_map_walker_try_leaf()
491 data->phys += granule; in stage2_map_walker_try_leaf()
495 static int stage2_map_walk_table_pre(u64 addr, u64 end, u32 level, in stage2_map_walk_table_pre() argument
497 struct stage2_map_data *data) in stage2_map_walk_table_pre() argument
499 if (data->anchor) in stage2_map_walk_table_pre()
502 if (!kvm_block_mapping_supported(addr, end, data->phys, level)) in stage2_map_walk_table_pre()
508 * Invalidate the whole stage-2, as we may have numerous leaf in stage2_map_walk_table_pre()
512 kvm_call_hyp(__kvm_tlb_flush_vmid, data->mmu); in stage2_map_walk_table_pre()
513 data->anchor = ptep; in stage2_map_walk_table_pre()
517 static int stage2_map_walk_leaf(u64 addr, u64 end, u32 level, kvm_pte_t *ptep, in stage2_map_walk_leaf() argument
518 struct stage2_map_data *data) in stage2_map_walk_leaf() argument
523 if (data->anchor) { in stage2_map_walk_leaf()
530 if (stage2_map_walker_try_leaf(addr, end, level, ptep, data)) in stage2_map_walk_leaf()
533 if (WARN_ON(level == KVM_PGTABLE_MAX_LEVELS - 1)) in stage2_map_walk_leaf()
534 return -EINVAL; in stage2_map_walk_leaf()
536 if (!data->memcache) in stage2_map_walk_leaf()
537 return -ENOMEM; in stage2_map_walk_leaf()
539 childp = kvm_mmu_memory_cache_alloc(data->memcache); in stage2_map_walk_leaf()
541 return -ENOMEM; in stage2_map_walk_leaf()
550 kvm_call_hyp(__kvm_tlb_flush_vmid_ipa, data->mmu, addr, level); in stage2_map_walk_leaf()
561 static int stage2_map_walk_table_post(u64 addr, u64 end, u32 level, in stage2_map_walk_table_post() argument
563 struct stage2_map_data *data) in stage2_map_walk_table_post() argument
567 if (!data->anchor) in stage2_map_walk_table_post()
573 if (data->anchor == ptep) { in stage2_map_walk_table_post()
574 data->anchor = NULL; in stage2_map_walk_table_post()
575 ret = stage2_map_walk_leaf(addr, end, level, ptep, data); in stage2_map_walk_table_post()
596 * been set, but otherwise frees the page-table pages while walking back up
597 * the page-table, installing the block entry when it revisits the anchor
600 static int stage2_map_walker(u64 addr, u64 end, u32 level, kvm_pte_t *ptep, in stage2_map_walker() argument
603 struct stage2_map_data *data = arg; in stage2_map_walker() local
607 return stage2_map_walk_table_pre(addr, end, level, ptep, data); in stage2_map_walker()
609 return stage2_map_walk_leaf(addr, end, level, ptep, data); in stage2_map_walker()
611 return stage2_map_walk_table_post(addr, end, level, ptep, data); in stage2_map_walker()
614 return -EINVAL; in stage2_map_walker()
617 int kvm_pgtable_stage2_map(struct kvm_pgtable *pgt, u64 addr, u64 size, in kvm_pgtable_stage2_map() argument
624 .mmu = pgt->mmu, in kvm_pgtable_stage2_map()
639 ret = kvm_pgtable_walk(pgt, addr, size, &walker); in kvm_pgtable_stage2_map()
644 static void stage2_flush_dcache(void *addr, u64 size) in stage2_flush_dcache() argument
649 __flush_dcache_area(addr, size); in stage2_flush_dcache()
658 static int stage2_unmap_walker(u64 addr, u64 end, u32 level, kvm_pte_t *ptep, in stage2_unmap_walker() argument
684 kvm_call_hyp(__kvm_tlb_flush_vmid_ipa, mmu, addr, level); in stage2_unmap_walker()
698 int kvm_pgtable_stage2_unmap(struct kvm_pgtable *pgt, u64 addr, u64 size) in kvm_pgtable_stage2_unmap() argument
702 .arg = pgt->mmu, in kvm_pgtable_stage2_unmap()
706 return kvm_pgtable_walk(pgt, addr, size, &walker); in kvm_pgtable_stage2_unmap()
716 static int stage2_attr_walker(u64 addr, u64 end, u32 level, kvm_pte_t *ptep, in stage2_attr_walker() argument
721 struct stage2_attr_data *data = arg; in stage2_attr_walker() local
726 data->level = level; in stage2_attr_walker()
727 data->pte = pte; in stage2_attr_walker()
728 pte &= ~data->attr_clr; in stage2_attr_walker()
729 pte |= data->attr_set; in stage2_attr_walker()
733 * but worst-case the access flag update gets lost and will be in stage2_attr_walker()
736 if (data->pte != pte) in stage2_attr_walker()
742 static int stage2_update_leaf_attrs(struct kvm_pgtable *pgt, u64 addr, in stage2_update_leaf_attrs() argument
749 struct stage2_attr_data data = { in stage2_update_leaf_attrs() local
755 .arg = &data, in stage2_update_leaf_attrs()
759 ret = kvm_pgtable_walk(pgt, addr, size, &walker); in stage2_update_leaf_attrs()
764 *orig_pte = data.pte; in stage2_update_leaf_attrs()
767 *level = data.level; in stage2_update_leaf_attrs()
771 int kvm_pgtable_stage2_wrprotect(struct kvm_pgtable *pgt, u64 addr, u64 size) in kvm_pgtable_stage2_wrprotect() argument
773 return stage2_update_leaf_attrs(pgt, addr, size, 0, in kvm_pgtable_stage2_wrprotect()
778 kvm_pte_t kvm_pgtable_stage2_mkyoung(struct kvm_pgtable *pgt, u64 addr) in kvm_pgtable_stage2_mkyoung() argument
781 stage2_update_leaf_attrs(pgt, addr, 1, KVM_PTE_LEAF_ATTR_LO_S2_AF, 0, in kvm_pgtable_stage2_mkyoung()
787 kvm_pte_t kvm_pgtable_stage2_mkold(struct kvm_pgtable *pgt, u64 addr) in kvm_pgtable_stage2_mkold() argument
790 stage2_update_leaf_attrs(pgt, addr, 1, 0, KVM_PTE_LEAF_ATTR_LO_S2_AF, in kvm_pgtable_stage2_mkold()
796 * See the '->clear_flush_young()' callback on the KVM mmu notifier. in kvm_pgtable_stage2_mkold()
801 bool kvm_pgtable_stage2_is_young(struct kvm_pgtable *pgt, u64 addr) in kvm_pgtable_stage2_is_young() argument
804 stage2_update_leaf_attrs(pgt, addr, 1, 0, 0, &pte, NULL); in kvm_pgtable_stage2_is_young()
808 int kvm_pgtable_stage2_relax_perms(struct kvm_pgtable *pgt, u64 addr, in kvm_pgtable_stage2_relax_perms() argument
824 ret = stage2_update_leaf_attrs(pgt, addr, 1, set, clr, NULL, &level); in kvm_pgtable_stage2_relax_perms()
826 kvm_call_hyp(__kvm_tlb_flush_vmid_ipa, pgt->mmu, addr, level); in kvm_pgtable_stage2_relax_perms()
830 static int stage2_flush_walker(u64 addr, u64 end, u32 level, kvm_pte_t *ptep, in stage2_flush_walker() argument
843 int kvm_pgtable_stage2_flush(struct kvm_pgtable *pgt, u64 addr, u64 size) in kvm_pgtable_stage2_flush() argument
853 return kvm_pgtable_walk(pgt, addr, size, &walker); in kvm_pgtable_stage2_flush()
859 u64 vtcr = kvm->arch.vtcr; in kvm_pgtable_stage2_init()
862 u32 start_level = VTCR_EL2_TGRAN_SL0_BASE - sl0; in kvm_pgtable_stage2_init()
865 pgt->pgd = alloc_pages_exact(pgd_sz, GFP_KERNEL_ACCOUNT | __GFP_ZERO); in kvm_pgtable_stage2_init()
866 if (!pgt->pgd) in kvm_pgtable_stage2_init()
867 return -ENOMEM; in kvm_pgtable_stage2_init()
869 pgt->ia_bits = ia_bits; in kvm_pgtable_stage2_init()
870 pgt->start_level = start_level; in kvm_pgtable_stage2_init()
871 pgt->mmu = &kvm->arch.mmu; in kvm_pgtable_stage2_init()
878 static int stage2_free_walker(u64 addr, u64 end, u32 level, kvm_pte_t *ptep, in stage2_free_walker() argument
904 WARN_ON(kvm_pgtable_walk(pgt, 0, BIT(pgt->ia_bits), &walker)); in kvm_pgtable_stage2_destroy()
905 pgd_sz = kvm_pgd_pages(pgt->ia_bits, pgt->start_level) * PAGE_SIZE; in kvm_pgtable_stage2_destroy()
906 free_pages_exact(pgt->pgd, pgd_sz); in kvm_pgtable_stage2_destroy()
907 pgt->pgd = NULL; in kvm_pgtable_stage2_destroy()