• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Secure pages management: Migration of pages between normal and secure
4  * memory of KVM guests.
5  *
6  * Copyright 2018 Bharata B Rao, IBM Corp. <bharata@linux.ibm.com>
7  */
8 
9 /*
10  * A pseries guest can be run as secure guest on Ultravisor-enabled
11  * POWER platforms. On such platforms, this driver will be used to manage
12  * the movement of guest pages between the normal memory managed by
13  * hypervisor (HV) and secure memory managed by Ultravisor (UV).
14  *
15  * The page-in or page-out requests from UV will come to HV as hcalls and
16  * HV will call back into UV via ultracalls to satisfy these page requests.
17  *
18  * Private ZONE_DEVICE memory equal to the amount of secure memory
19  * available in the platform for running secure guests is hotplugged.
20  * Whenever a page belonging to the guest becomes secure, a page from this
21  * private device memory is used to represent and track that secure page
22  * on the HV side. Some pages (like virtio buffers, VPA pages etc) are
23  * shared between UV and HV. However such pages aren't represented by
24  * device private memory and mappings to shared memory exist in both
25  * UV and HV page tables.
26  */
27 
28 /*
29  * Notes on locking
30  *
31  * kvm->arch.uvmem_lock is a per-guest lock that prevents concurrent
32  * page-in and page-out requests for the same GPA. Concurrent accesses
33  * can either come via UV (guest vCPUs requesting for same page)
34  * or when HV and guest simultaneously access the same page.
35  * This mutex serializes the migration of page from HV(normal) to
36  * UV(secure) and vice versa. So the serialization points are around
37  * migrate_vma routines and page-in/out routines.
38  *
39  * Per-guest mutex comes with a cost though. Mainly it serializes the
40  * fault path as page-out can occur when HV faults on accessing secure
41  * guest pages. Currently UV issues page-in requests for all the guest
42  * PFNs one at a time during early boot (UV_ESM uvcall), so this is
43  * not a cause for concern. Also currently the number of page-outs caused
44  * by HV touching secure pages is very very low. If an when UV supports
45  * overcommitting, then we might see concurrent guest driven page-outs.
46  *
47  * Locking order
48  *
49  * 1. kvm->srcu - Protects KVM memslots
50  * 2. kvm->mm->mmap_lock - find_vma, migrate_vma_pages and helpers, ksm_madvise
51  * 3. kvm->arch.uvmem_lock - protects read/writes to uvmem slots thus acting
52  *			     as sync-points for page-in/out
53  */
54 
55 /*
56  * Notes on page size
57  *
58  * Currently UV uses 2MB mappings internally, but will issue H_SVM_PAGE_IN
59  * and H_SVM_PAGE_OUT hcalls in PAGE_SIZE(64K) granularity. HV tracks
60  * secure GPAs at 64K page size and maintains one device PFN for each
61  * 64K secure GPA. UV_PAGE_IN and UV_PAGE_OUT calls by HV are also issued
62  * for 64K page at a time.
63  *
64  * HV faulting on secure pages: When HV touches any secure page, it
65  * faults and issues a UV_PAGE_OUT request with 64K page size. Currently
66  * UV splits and remaps the 2MB page if necessary and copies out the
67  * required 64K page contents.
68  *
69  * Shared pages: Whenever guest shares a secure page, UV will split and
70  * remap the 2MB page if required and issue H_SVM_PAGE_IN with 64K page size.
71  *
72  * HV invalidating a page: When a regular page belonging to secure
73  * guest gets unmapped, HV informs UV with UV_PAGE_INVAL of 64K
74  * page size. Using 64K page size is correct here because any non-secure
75  * page will essentially be of 64K page size. Splitting by UV during sharing
76  * and page-out ensures this.
77  *
78  * Page fault handling: When HV handles page fault of a page belonging
79  * to secure guest, it sends that to UV with a 64K UV_PAGE_IN request.
80  * Using 64K size is correct here too as UV would have split the 2MB page
81  * into 64k mappings and would have done page-outs earlier.
82  *
83  * In summary, the current secure pages handling code in HV assumes
84  * 64K page size and in fact fails any page-in/page-out requests of
85  * non-64K size upfront. If and when UV starts supporting multiple
86  * page-sizes, we need to break this assumption.
87  */
88 
89 #include <linux/pagemap.h>
90 #include <linux/migrate.h>
91 #include <linux/kvm_host.h>
92 #include <linux/ksm.h>
93 #include <asm/ultravisor.h>
94 #include <asm/mman.h>
95 #include <asm/kvm_ppc.h>
96 #include <asm/kvm_book3s_uvmem.h>
97 
98 static struct dev_pagemap kvmppc_uvmem_pgmap;
99 static unsigned long *kvmppc_uvmem_bitmap;
100 static DEFINE_SPINLOCK(kvmppc_uvmem_bitmap_lock);
101 
102 /*
103  * States of a GFN
104  * ---------------
105  * The GFN can be in one of the following states.
106  *
107  * (a) Secure - The GFN is secure. The GFN is associated with
108  *	a Secure VM, the contents of the GFN is not accessible
109  *	to the Hypervisor.  This GFN can be backed by a secure-PFN,
110  *	or can be backed by a normal-PFN with contents encrypted.
111  *	The former is true when the GFN is paged-in into the
112  *	ultravisor. The latter is true when the GFN is paged-out
113  *	of the ultravisor.
114  *
115  * (b) Shared - The GFN is shared. The GFN is associated with a
116  *	a secure VM. The contents of the GFN is accessible to
117  *	Hypervisor. This GFN is backed by a normal-PFN and its
118  *	content is un-encrypted.
119  *
120  * (c) Normal - The GFN is a normal. The GFN is associated with
121  *	a normal VM. The contents of the GFN is accesible to
122  *	the Hypervisor. Its content is never encrypted.
123  *
124  * States of a VM.
125  * ---------------
126  *
127  * Normal VM:  A VM whose contents are always accessible to
128  *	the hypervisor.  All its GFNs are normal-GFNs.
129  *
130  * Secure VM: A VM whose contents are not accessible to the
131  *	hypervisor without the VM's consent.  Its GFNs are
132  *	either Shared-GFN or Secure-GFNs.
133  *
134  * Transient VM: A Normal VM that is transitioning to secure VM.
135  *	The transition starts on successful return of
136  *	H_SVM_INIT_START, and ends on successful return
137  *	of H_SVM_INIT_DONE. This transient VM, can have GFNs
138  *	in any of the three states; i.e Secure-GFN, Shared-GFN,
139  *	and Normal-GFN.	The VM never executes in this state
140  *	in supervisor-mode.
141  *
142  * Memory slot State.
143  * -----------------------------
144  *	The state of a memory slot mirrors the state of the
145  *	VM the memory slot is associated with.
146  *
147  * VM State transition.
148  * --------------------
149  *
150  *  A VM always starts in Normal Mode.
151  *
152  *  H_SVM_INIT_START moves the VM into transient state. During this
153  *  time the Ultravisor may request some of its GFNs to be shared or
154  *  secured. So its GFNs can be in one of the three GFN states.
155  *
156  *  H_SVM_INIT_DONE moves the VM entirely from transient state to
157  *  secure-state. At this point any left-over normal-GFNs are
158  *  transitioned to Secure-GFN.
159  *
160  *  H_SVM_INIT_ABORT moves the transient VM back to normal VM.
161  *  All its GFNs are moved to Normal-GFNs.
162  *
163  *  UV_TERMINATE transitions the secure-VM back to normal-VM. All
164  *  the secure-GFN and shared-GFNs are tranistioned to normal-GFN
165  *  Note: The contents of the normal-GFN is undefined at this point.
166  *
167  * GFN state implementation:
168  * -------------------------
169  *
170  * Secure GFN is associated with a secure-PFN; also called uvmem_pfn,
171  * when the GFN is paged-in. Its pfn[] has KVMPPC_GFN_UVMEM_PFN flag
172  * set, and contains the value of the secure-PFN.
173  * It is associated with a normal-PFN; also called mem_pfn, when
174  * the GFN is pagedout. Its pfn[] has KVMPPC_GFN_MEM_PFN flag set.
175  * The value of the normal-PFN is not tracked.
176  *
177  * Shared GFN is associated with a normal-PFN. Its pfn[] has
178  * KVMPPC_UVMEM_SHARED_PFN flag set. The value of the normal-PFN
179  * is not tracked.
180  *
181  * Normal GFN is associated with normal-PFN. Its pfn[] has
182  * no flag set. The value of the normal-PFN is not tracked.
183  *
184  * Life cycle of a GFN
185  * --------------------
186  *
187  * --------------------------------------------------------------
188  * |        |     Share  |  Unshare | SVM       |H_SVM_INIT_DONE|
189  * |        |operation   |operation | abort/    |               |
190  * |        |            |          | terminate |               |
191  * -------------------------------------------------------------
192  * |        |            |          |           |               |
193  * | Secure |     Shared | Secure   |Normal     |Secure         |
194  * |        |            |          |           |               |
195  * | Shared |     Shared | Secure   |Normal     |Shared         |
196  * |        |            |          |           |               |
197  * | Normal |     Shared | Secure   |Normal     |Secure         |
198  * --------------------------------------------------------------
199  *
200  * Life cycle of a VM
201  * --------------------
202  *
203  * --------------------------------------------------------------------
204  * |         |  start    |  H_SVM_  |H_SVM_   |H_SVM_     |UV_SVM_    |
205  * |         |  VM       |INIT_START|INIT_DONE|INIT_ABORT |TERMINATE  |
206  * |         |           |          |         |           |           |
207  * --------- ----------------------------------------------------------
208  * |         |           |          |         |           |           |
209  * | Normal  | Normal    | Transient|Error    |Error      |Normal     |
210  * |         |           |          |         |           |           |
211  * | Secure  |   Error   | Error    |Error    |Error      |Normal     |
212  * |         |           |          |         |           |           |
213  * |Transient|   N/A     | Error    |Secure   |Normal     |Normal     |
214  * --------------------------------------------------------------------
215  */
216 
217 #define KVMPPC_GFN_UVMEM_PFN	(1UL << 63)
218 #define KVMPPC_GFN_MEM_PFN	(1UL << 62)
219 #define KVMPPC_GFN_SHARED	(1UL << 61)
220 #define KVMPPC_GFN_SECURE	(KVMPPC_GFN_UVMEM_PFN | KVMPPC_GFN_MEM_PFN)
221 #define KVMPPC_GFN_FLAG_MASK	(KVMPPC_GFN_SECURE | KVMPPC_GFN_SHARED)
222 #define KVMPPC_GFN_PFN_MASK	(~KVMPPC_GFN_FLAG_MASK)
223 
224 struct kvmppc_uvmem_slot {
225 	struct list_head list;
226 	unsigned long nr_pfns;
227 	unsigned long base_pfn;
228 	unsigned long *pfns;
229 };
230 struct kvmppc_uvmem_page_pvt {
231 	struct kvm *kvm;
232 	unsigned long gpa;
233 	bool skip_page_out;
234 	bool remove_gfn;
235 };
236 
kvmppc_uvmem_available(void)237 bool kvmppc_uvmem_available(void)
238 {
239 	/*
240 	 * If kvmppc_uvmem_bitmap != NULL, then there is an ultravisor
241 	 * and our data structures have been initialized successfully.
242 	 */
243 	return !!kvmppc_uvmem_bitmap;
244 }
245 
kvmppc_uvmem_slot_init(struct kvm * kvm,const struct kvm_memory_slot * slot)246 int kvmppc_uvmem_slot_init(struct kvm *kvm, const struct kvm_memory_slot *slot)
247 {
248 	struct kvmppc_uvmem_slot *p;
249 
250 	p = kzalloc(sizeof(*p), GFP_KERNEL);
251 	if (!p)
252 		return -ENOMEM;
253 	p->pfns = vzalloc(array_size(slot->npages, sizeof(*p->pfns)));
254 	if (!p->pfns) {
255 		kfree(p);
256 		return -ENOMEM;
257 	}
258 	p->nr_pfns = slot->npages;
259 	p->base_pfn = slot->base_gfn;
260 
261 	mutex_lock(&kvm->arch.uvmem_lock);
262 	list_add(&p->list, &kvm->arch.uvmem_pfns);
263 	mutex_unlock(&kvm->arch.uvmem_lock);
264 
265 	return 0;
266 }
267 
268 /*
269  * All device PFNs are already released by the time we come here.
270  */
kvmppc_uvmem_slot_free(struct kvm * kvm,const struct kvm_memory_slot * slot)271 void kvmppc_uvmem_slot_free(struct kvm *kvm, const struct kvm_memory_slot *slot)
272 {
273 	struct kvmppc_uvmem_slot *p, *next;
274 
275 	mutex_lock(&kvm->arch.uvmem_lock);
276 	list_for_each_entry_safe(p, next, &kvm->arch.uvmem_pfns, list) {
277 		if (p->base_pfn == slot->base_gfn) {
278 			vfree(p->pfns);
279 			list_del(&p->list);
280 			kfree(p);
281 			break;
282 		}
283 	}
284 	mutex_unlock(&kvm->arch.uvmem_lock);
285 }
286 
kvmppc_mark_gfn(unsigned long gfn,struct kvm * kvm,unsigned long flag,unsigned long uvmem_pfn)287 static void kvmppc_mark_gfn(unsigned long gfn, struct kvm *kvm,
288 			unsigned long flag, unsigned long uvmem_pfn)
289 {
290 	struct kvmppc_uvmem_slot *p;
291 
292 	list_for_each_entry(p, &kvm->arch.uvmem_pfns, list) {
293 		if (gfn >= p->base_pfn && gfn < p->base_pfn + p->nr_pfns) {
294 			unsigned long index = gfn - p->base_pfn;
295 
296 			if (flag == KVMPPC_GFN_UVMEM_PFN)
297 				p->pfns[index] = uvmem_pfn | flag;
298 			else
299 				p->pfns[index] = flag;
300 			return;
301 		}
302 	}
303 }
304 
305 /* mark the GFN as secure-GFN associated with @uvmem pfn device-PFN. */
kvmppc_gfn_secure_uvmem_pfn(unsigned long gfn,unsigned long uvmem_pfn,struct kvm * kvm)306 static void kvmppc_gfn_secure_uvmem_pfn(unsigned long gfn,
307 			unsigned long uvmem_pfn, struct kvm *kvm)
308 {
309 	kvmppc_mark_gfn(gfn, kvm, KVMPPC_GFN_UVMEM_PFN, uvmem_pfn);
310 }
311 
312 /* mark the GFN as secure-GFN associated with a memory-PFN. */
kvmppc_gfn_secure_mem_pfn(unsigned long gfn,struct kvm * kvm)313 static void kvmppc_gfn_secure_mem_pfn(unsigned long gfn, struct kvm *kvm)
314 {
315 	kvmppc_mark_gfn(gfn, kvm, KVMPPC_GFN_MEM_PFN, 0);
316 }
317 
318 /* mark the GFN as a shared GFN. */
kvmppc_gfn_shared(unsigned long gfn,struct kvm * kvm)319 static void kvmppc_gfn_shared(unsigned long gfn, struct kvm *kvm)
320 {
321 	kvmppc_mark_gfn(gfn, kvm, KVMPPC_GFN_SHARED, 0);
322 }
323 
324 /* mark the GFN as a non-existent GFN. */
kvmppc_gfn_remove(unsigned long gfn,struct kvm * kvm)325 static void kvmppc_gfn_remove(unsigned long gfn, struct kvm *kvm)
326 {
327 	kvmppc_mark_gfn(gfn, kvm, 0, 0);
328 }
329 
330 /* return true, if the GFN is a secure-GFN backed by a secure-PFN */
kvmppc_gfn_is_uvmem_pfn(unsigned long gfn,struct kvm * kvm,unsigned long * uvmem_pfn)331 static bool kvmppc_gfn_is_uvmem_pfn(unsigned long gfn, struct kvm *kvm,
332 				    unsigned long *uvmem_pfn)
333 {
334 	struct kvmppc_uvmem_slot *p;
335 
336 	list_for_each_entry(p, &kvm->arch.uvmem_pfns, list) {
337 		if (gfn >= p->base_pfn && gfn < p->base_pfn + p->nr_pfns) {
338 			unsigned long index = gfn - p->base_pfn;
339 
340 			if (p->pfns[index] & KVMPPC_GFN_UVMEM_PFN) {
341 				if (uvmem_pfn)
342 					*uvmem_pfn = p->pfns[index] &
343 						     KVMPPC_GFN_PFN_MASK;
344 				return true;
345 			} else
346 				return false;
347 		}
348 	}
349 	return false;
350 }
351 
352 /*
353  * starting from *gfn search for the next available GFN that is not yet
354  * transitioned to a secure GFN.  return the value of that GFN in *gfn.  If a
355  * GFN is found, return true, else return false
356  *
357  * Must be called with kvm->arch.uvmem_lock  held.
358  */
kvmppc_next_nontransitioned_gfn(const struct kvm_memory_slot * memslot,struct kvm * kvm,unsigned long * gfn)359 static bool kvmppc_next_nontransitioned_gfn(const struct kvm_memory_slot *memslot,
360 		struct kvm *kvm, unsigned long *gfn)
361 {
362 	struct kvmppc_uvmem_slot *p;
363 	bool ret = false;
364 	unsigned long i;
365 
366 	list_for_each_entry(p, &kvm->arch.uvmem_pfns, list)
367 		if (*gfn >= p->base_pfn && *gfn < p->base_pfn + p->nr_pfns)
368 			break;
369 	if (!p)
370 		return ret;
371 	/*
372 	 * The code below assumes, one to one correspondence between
373 	 * kvmppc_uvmem_slot and memslot.
374 	 */
375 	for (i = *gfn; i < p->base_pfn + p->nr_pfns; i++) {
376 		unsigned long index = i - p->base_pfn;
377 
378 		if (!(p->pfns[index] & KVMPPC_GFN_FLAG_MASK)) {
379 			*gfn = i;
380 			ret = true;
381 			break;
382 		}
383 	}
384 	return ret;
385 }
386 
kvmppc_memslot_page_merge(struct kvm * kvm,const struct kvm_memory_slot * memslot,bool merge)387 static int kvmppc_memslot_page_merge(struct kvm *kvm,
388 		const struct kvm_memory_slot *memslot, bool merge)
389 {
390 	unsigned long gfn = memslot->base_gfn;
391 	unsigned long end, start = gfn_to_hva(kvm, gfn);
392 	int ret = 0;
393 	struct vm_area_struct *vma;
394 	int merge_flag = (merge) ? MADV_MERGEABLE : MADV_UNMERGEABLE;
395 
396 	if (kvm_is_error_hva(start))
397 		return H_STATE;
398 
399 	end = start + (memslot->npages << PAGE_SHIFT);
400 
401 	mmap_write_lock(kvm->mm);
402 	do {
403 		vma = find_vma_intersection(kvm->mm, start, end);
404 		if (!vma) {
405 			ret = H_STATE;
406 			break;
407 		}
408 		ret = ksm_madvise(vma, vma->vm_start, vma->vm_end,
409 			  merge_flag, &vma->vm_flags);
410 		if (ret) {
411 			ret = H_STATE;
412 			break;
413 		}
414 		start = vma->vm_end;
415 	} while (end > vma->vm_end);
416 
417 	mmap_write_unlock(kvm->mm);
418 	return ret;
419 }
420 
__kvmppc_uvmem_memslot_delete(struct kvm * kvm,const struct kvm_memory_slot * memslot)421 static void __kvmppc_uvmem_memslot_delete(struct kvm *kvm,
422 		const struct kvm_memory_slot *memslot)
423 {
424 	uv_unregister_mem_slot(kvm->arch.lpid, memslot->id);
425 	kvmppc_uvmem_slot_free(kvm, memslot);
426 	kvmppc_memslot_page_merge(kvm, memslot, true);
427 }
428 
__kvmppc_uvmem_memslot_create(struct kvm * kvm,const struct kvm_memory_slot * memslot)429 static int __kvmppc_uvmem_memslot_create(struct kvm *kvm,
430 		const struct kvm_memory_slot *memslot)
431 {
432 	int ret = H_PARAMETER;
433 
434 	if (kvmppc_memslot_page_merge(kvm, memslot, false))
435 		return ret;
436 
437 	if (kvmppc_uvmem_slot_init(kvm, memslot))
438 		goto out1;
439 
440 	ret = uv_register_mem_slot(kvm->arch.lpid,
441 				   memslot->base_gfn << PAGE_SHIFT,
442 				   memslot->npages * PAGE_SIZE,
443 				   0, memslot->id);
444 	if (ret < 0) {
445 		ret = H_PARAMETER;
446 		goto out;
447 	}
448 	return 0;
449 out:
450 	kvmppc_uvmem_slot_free(kvm, memslot);
451 out1:
452 	kvmppc_memslot_page_merge(kvm, memslot, true);
453 	return ret;
454 }
455 
kvmppc_h_svm_init_start(struct kvm * kvm)456 unsigned long kvmppc_h_svm_init_start(struct kvm *kvm)
457 {
458 	struct kvm_memslots *slots;
459 	struct kvm_memory_slot *memslot, *m;
460 	int ret = H_SUCCESS;
461 	int srcu_idx;
462 
463 	kvm->arch.secure_guest = KVMPPC_SECURE_INIT_START;
464 
465 	if (!kvmppc_uvmem_bitmap)
466 		return H_UNSUPPORTED;
467 
468 	/* Only radix guests can be secure guests */
469 	if (!kvm_is_radix(kvm))
470 		return H_UNSUPPORTED;
471 
472 	/* NAK the transition to secure if not enabled */
473 	if (!kvm->arch.svm_enabled)
474 		return H_AUTHORITY;
475 
476 	srcu_idx = srcu_read_lock(&kvm->srcu);
477 
478 	/* register the memslot */
479 	slots = kvm_memslots(kvm);
480 	kvm_for_each_memslot(memslot, slots) {
481 		ret = __kvmppc_uvmem_memslot_create(kvm, memslot);
482 		if (ret)
483 			break;
484 	}
485 
486 	if (ret) {
487 		slots = kvm_memslots(kvm);
488 		kvm_for_each_memslot(m, slots) {
489 			if (m == memslot)
490 				break;
491 			__kvmppc_uvmem_memslot_delete(kvm, memslot);
492 		}
493 	}
494 
495 	srcu_read_unlock(&kvm->srcu, srcu_idx);
496 	return ret;
497 }
498 
499 /*
500  * Provision a new page on HV side and copy over the contents
501  * from secure memory using UV_PAGE_OUT uvcall.
502  * Caller must held kvm->arch.uvmem_lock.
503  */
__kvmppc_svm_page_out(struct vm_area_struct * vma,unsigned long start,unsigned long end,unsigned long page_shift,struct kvm * kvm,unsigned long gpa,struct page * fault_page)504 static int __kvmppc_svm_page_out(struct vm_area_struct *vma,
505 		unsigned long start,
506 		unsigned long end, unsigned long page_shift,
507 		struct kvm *kvm, unsigned long gpa, struct page *fault_page)
508 {
509 	unsigned long src_pfn, dst_pfn = 0;
510 	struct migrate_vma mig = { 0 };
511 	struct page *dpage, *spage;
512 	struct kvmppc_uvmem_page_pvt *pvt;
513 	unsigned long pfn;
514 	int ret = U_SUCCESS;
515 
516 	memset(&mig, 0, sizeof(mig));
517 	mig.vma = vma;
518 	mig.start = start;
519 	mig.end = end;
520 	mig.src = &src_pfn;
521 	mig.dst = &dst_pfn;
522 	mig.pgmap_owner = &kvmppc_uvmem_pgmap;
523 	mig.flags = MIGRATE_VMA_SELECT_DEVICE_PRIVATE;
524 	mig.fault_page = fault_page;
525 
526 	/* The requested page is already paged-out, nothing to do */
527 	if (!kvmppc_gfn_is_uvmem_pfn(gpa >> page_shift, kvm, NULL))
528 		return ret;
529 
530 	ret = migrate_vma_setup(&mig);
531 	if (ret)
532 		return -1;
533 
534 	spage = migrate_pfn_to_page(*mig.src);
535 	if (!spage || !(*mig.src & MIGRATE_PFN_MIGRATE))
536 		goto out_finalize;
537 
538 	if (!is_zone_device_page(spage))
539 		goto out_finalize;
540 
541 	dpage = alloc_page_vma(GFP_HIGHUSER, vma, start);
542 	if (!dpage) {
543 		ret = -1;
544 		goto out_finalize;
545 	}
546 
547 	lock_page(dpage);
548 	pvt = spage->zone_device_data;
549 	pfn = page_to_pfn(dpage);
550 
551 	/*
552 	 * This function is used in two cases:
553 	 * - When HV touches a secure page, for which we do UV_PAGE_OUT
554 	 * - When a secure page is converted to shared page, we *get*
555 	 *   the page to essentially unmap the device page. In this
556 	 *   case we skip page-out.
557 	 */
558 	if (!pvt->skip_page_out)
559 		ret = uv_page_out(kvm->arch.lpid, pfn << page_shift,
560 				  gpa, 0, page_shift);
561 
562 	if (ret == U_SUCCESS)
563 		*mig.dst = migrate_pfn(pfn) | MIGRATE_PFN_LOCKED;
564 	else {
565 		unlock_page(dpage);
566 		__free_page(dpage);
567 		goto out_finalize;
568 	}
569 
570 	migrate_vma_pages(&mig);
571 
572 out_finalize:
573 	migrate_vma_finalize(&mig);
574 	return ret;
575 }
576 
kvmppc_svm_page_out(struct vm_area_struct * vma,unsigned long start,unsigned long end,unsigned long page_shift,struct kvm * kvm,unsigned long gpa,struct page * fault_page)577 static inline int kvmppc_svm_page_out(struct vm_area_struct *vma,
578 				      unsigned long start, unsigned long end,
579 				      unsigned long page_shift,
580 				      struct kvm *kvm, unsigned long gpa,
581 				      struct page *fault_page)
582 {
583 	int ret;
584 
585 	mutex_lock(&kvm->arch.uvmem_lock);
586 	ret = __kvmppc_svm_page_out(vma, start, end, page_shift, kvm, gpa,
587 				fault_page);
588 	mutex_unlock(&kvm->arch.uvmem_lock);
589 
590 	return ret;
591 }
592 
593 /*
594  * Drop device pages that we maintain for the secure guest
595  *
596  * We first mark the pages to be skipped from UV_PAGE_OUT when there
597  * is HV side fault on these pages. Next we *get* these pages, forcing
598  * fault on them, do fault time migration to replace the device PTEs in
599  * QEMU page table with normal PTEs from newly allocated pages.
600  */
kvmppc_uvmem_drop_pages(const struct kvm_memory_slot * slot,struct kvm * kvm,bool skip_page_out)601 void kvmppc_uvmem_drop_pages(const struct kvm_memory_slot *slot,
602 			     struct kvm *kvm, bool skip_page_out)
603 {
604 	int i;
605 	struct kvmppc_uvmem_page_pvt *pvt;
606 	struct page *uvmem_page;
607 	struct vm_area_struct *vma = NULL;
608 	unsigned long uvmem_pfn, gfn;
609 	unsigned long addr;
610 
611 	mmap_read_lock(kvm->mm);
612 
613 	addr = slot->userspace_addr;
614 
615 	gfn = slot->base_gfn;
616 	for (i = slot->npages; i; --i, ++gfn, addr += PAGE_SIZE) {
617 
618 		/* Fetch the VMA if addr is not in the latest fetched one */
619 		if (!vma || addr >= vma->vm_end) {
620 			vma = find_vma_intersection(kvm->mm, addr, addr+1);
621 			if (!vma) {
622 				pr_err("Can't find VMA for gfn:0x%lx\n", gfn);
623 				break;
624 			}
625 		}
626 
627 		mutex_lock(&kvm->arch.uvmem_lock);
628 
629 		if (kvmppc_gfn_is_uvmem_pfn(gfn, kvm, &uvmem_pfn)) {
630 			uvmem_page = pfn_to_page(uvmem_pfn);
631 			pvt = uvmem_page->zone_device_data;
632 			pvt->skip_page_out = skip_page_out;
633 			pvt->remove_gfn = true;
634 
635 			if (__kvmppc_svm_page_out(vma, addr, addr + PAGE_SIZE,
636 						  PAGE_SHIFT, kvm, pvt->gpa, NULL))
637 				pr_err("Can't page out gpa:0x%lx addr:0x%lx\n",
638 				       pvt->gpa, addr);
639 		} else {
640 			/* Remove the shared flag if any */
641 			kvmppc_gfn_remove(gfn, kvm);
642 		}
643 
644 		mutex_unlock(&kvm->arch.uvmem_lock);
645 	}
646 
647 	mmap_read_unlock(kvm->mm);
648 }
649 
kvmppc_h_svm_init_abort(struct kvm * kvm)650 unsigned long kvmppc_h_svm_init_abort(struct kvm *kvm)
651 {
652 	int srcu_idx;
653 	struct kvm_memory_slot *memslot;
654 
655 	/*
656 	 * Expect to be called only after INIT_START and before INIT_DONE.
657 	 * If INIT_DONE was completed, use normal VM termination sequence.
658 	 */
659 	if (!(kvm->arch.secure_guest & KVMPPC_SECURE_INIT_START))
660 		return H_UNSUPPORTED;
661 
662 	if (kvm->arch.secure_guest & KVMPPC_SECURE_INIT_DONE)
663 		return H_STATE;
664 
665 	srcu_idx = srcu_read_lock(&kvm->srcu);
666 
667 	kvm_for_each_memslot(memslot, kvm_memslots(kvm))
668 		kvmppc_uvmem_drop_pages(memslot, kvm, false);
669 
670 	srcu_read_unlock(&kvm->srcu, srcu_idx);
671 
672 	kvm->arch.secure_guest = 0;
673 	uv_svm_terminate(kvm->arch.lpid);
674 
675 	return H_PARAMETER;
676 }
677 
678 /*
679  * Get a free device PFN from the pool
680  *
681  * Called when a normal page is moved to secure memory (UV_PAGE_IN). Device
682  * PFN will be used to keep track of the secure page on HV side.
683  *
684  * Called with kvm->arch.uvmem_lock held
685  */
kvmppc_uvmem_get_page(unsigned long gpa,struct kvm * kvm)686 static struct page *kvmppc_uvmem_get_page(unsigned long gpa, struct kvm *kvm)
687 {
688 	struct page *dpage = NULL;
689 	unsigned long bit, uvmem_pfn;
690 	struct kvmppc_uvmem_page_pvt *pvt;
691 	unsigned long pfn_last, pfn_first;
692 
693 	pfn_first = kvmppc_uvmem_pgmap.range.start >> PAGE_SHIFT;
694 	pfn_last = pfn_first +
695 		   (range_len(&kvmppc_uvmem_pgmap.range) >> PAGE_SHIFT);
696 
697 	spin_lock(&kvmppc_uvmem_bitmap_lock);
698 	bit = find_first_zero_bit(kvmppc_uvmem_bitmap,
699 				  pfn_last - pfn_first);
700 	if (bit >= (pfn_last - pfn_first))
701 		goto out;
702 	bitmap_set(kvmppc_uvmem_bitmap, bit, 1);
703 	spin_unlock(&kvmppc_uvmem_bitmap_lock);
704 
705 	pvt = kzalloc(sizeof(*pvt), GFP_KERNEL);
706 	if (!pvt)
707 		goto out_clear;
708 
709 	uvmem_pfn = bit + pfn_first;
710 	kvmppc_gfn_secure_uvmem_pfn(gpa >> PAGE_SHIFT, uvmem_pfn, kvm);
711 
712 	pvt->gpa = gpa;
713 	pvt->kvm = kvm;
714 
715 	dpage = pfn_to_page(uvmem_pfn);
716 	dpage->zone_device_data = pvt;
717 	get_page(dpage);
718 	lock_page(dpage);
719 	return dpage;
720 out_clear:
721 	spin_lock(&kvmppc_uvmem_bitmap_lock);
722 	bitmap_clear(kvmppc_uvmem_bitmap, bit, 1);
723 out:
724 	spin_unlock(&kvmppc_uvmem_bitmap_lock);
725 	return NULL;
726 }
727 
728 /*
729  * Alloc a PFN from private device memory pool. If @pagein is true,
730  * copy page from normal memory to secure memory using UV_PAGE_IN uvcall.
731  */
kvmppc_svm_page_in(struct vm_area_struct * vma,unsigned long start,unsigned long end,unsigned long gpa,struct kvm * kvm,unsigned long page_shift,bool pagein)732 static int kvmppc_svm_page_in(struct vm_area_struct *vma,
733 		unsigned long start,
734 		unsigned long end, unsigned long gpa, struct kvm *kvm,
735 		unsigned long page_shift,
736 		bool pagein)
737 {
738 	unsigned long src_pfn, dst_pfn = 0;
739 	struct migrate_vma mig = { 0 };
740 	struct page *spage;
741 	unsigned long pfn;
742 	struct page *dpage;
743 	int ret = 0;
744 
745 	memset(&mig, 0, sizeof(mig));
746 	mig.vma = vma;
747 	mig.start = start;
748 	mig.end = end;
749 	mig.src = &src_pfn;
750 	mig.dst = &dst_pfn;
751 	mig.flags = MIGRATE_VMA_SELECT_SYSTEM;
752 
753 	ret = migrate_vma_setup(&mig);
754 	if (ret)
755 		return ret;
756 
757 	if (!(*mig.src & MIGRATE_PFN_MIGRATE)) {
758 		ret = -1;
759 		goto out_finalize;
760 	}
761 
762 	dpage = kvmppc_uvmem_get_page(gpa, kvm);
763 	if (!dpage) {
764 		ret = -1;
765 		goto out_finalize;
766 	}
767 
768 	if (pagein) {
769 		pfn = *mig.src >> MIGRATE_PFN_SHIFT;
770 		spage = migrate_pfn_to_page(*mig.src);
771 		if (spage) {
772 			ret = uv_page_in(kvm->arch.lpid, pfn << page_shift,
773 					gpa, 0, page_shift);
774 			if (ret)
775 				goto out_finalize;
776 		}
777 	}
778 
779 	*mig.dst = migrate_pfn(page_to_pfn(dpage)) | MIGRATE_PFN_LOCKED;
780 	migrate_vma_pages(&mig);
781 out_finalize:
782 	migrate_vma_finalize(&mig);
783 	return ret;
784 }
785 
kvmppc_uv_migrate_mem_slot(struct kvm * kvm,const struct kvm_memory_slot * memslot)786 static int kvmppc_uv_migrate_mem_slot(struct kvm *kvm,
787 		const struct kvm_memory_slot *memslot)
788 {
789 	unsigned long gfn = memslot->base_gfn;
790 	struct vm_area_struct *vma;
791 	unsigned long start, end;
792 	int ret = 0;
793 
794 	mmap_read_lock(kvm->mm);
795 	mutex_lock(&kvm->arch.uvmem_lock);
796 	while (kvmppc_next_nontransitioned_gfn(memslot, kvm, &gfn)) {
797 		ret = H_STATE;
798 		start = gfn_to_hva(kvm, gfn);
799 		if (kvm_is_error_hva(start))
800 			break;
801 
802 		end = start + (1UL << PAGE_SHIFT);
803 		vma = find_vma_intersection(kvm->mm, start, end);
804 		if (!vma || vma->vm_start > start || vma->vm_end < end)
805 			break;
806 
807 		ret = kvmppc_svm_page_in(vma, start, end,
808 				(gfn << PAGE_SHIFT), kvm, PAGE_SHIFT, false);
809 		if (ret) {
810 			ret = H_STATE;
811 			break;
812 		}
813 
814 		/* relinquish the cpu if needed */
815 		cond_resched();
816 	}
817 	mutex_unlock(&kvm->arch.uvmem_lock);
818 	mmap_read_unlock(kvm->mm);
819 	return ret;
820 }
821 
kvmppc_h_svm_init_done(struct kvm * kvm)822 unsigned long kvmppc_h_svm_init_done(struct kvm *kvm)
823 {
824 	struct kvm_memslots *slots;
825 	struct kvm_memory_slot *memslot;
826 	int srcu_idx;
827 	long ret = H_SUCCESS;
828 
829 	if (!(kvm->arch.secure_guest & KVMPPC_SECURE_INIT_START))
830 		return H_UNSUPPORTED;
831 
832 	/* migrate any unmoved normal pfn to device pfns*/
833 	srcu_idx = srcu_read_lock(&kvm->srcu);
834 	slots = kvm_memslots(kvm);
835 	kvm_for_each_memslot(memslot, slots) {
836 		ret = kvmppc_uv_migrate_mem_slot(kvm, memslot);
837 		if (ret) {
838 			/*
839 			 * The pages will remain transitioned.
840 			 * Its the callers responsibility to
841 			 * terminate the VM, which will undo
842 			 * all state of the VM. Till then
843 			 * this VM is in a erroneous state.
844 			 * Its KVMPPC_SECURE_INIT_DONE will
845 			 * remain unset.
846 			 */
847 			ret = H_STATE;
848 			goto out;
849 		}
850 	}
851 
852 	kvm->arch.secure_guest |= KVMPPC_SECURE_INIT_DONE;
853 	pr_info("LPID %d went secure\n", kvm->arch.lpid);
854 
855 out:
856 	srcu_read_unlock(&kvm->srcu, srcu_idx);
857 	return ret;
858 }
859 
860 /*
861  * Shares the page with HV, thus making it a normal page.
862  *
863  * - If the page is already secure, then provision a new page and share
864  * - If the page is a normal page, share the existing page
865  *
866  * In the former case, uses dev_pagemap_ops.migrate_to_ram handler
867  * to unmap the device page from QEMU's page tables.
868  */
kvmppc_share_page(struct kvm * kvm,unsigned long gpa,unsigned long page_shift)869 static unsigned long kvmppc_share_page(struct kvm *kvm, unsigned long gpa,
870 		unsigned long page_shift)
871 {
872 
873 	int ret = H_PARAMETER;
874 	struct page *uvmem_page;
875 	struct kvmppc_uvmem_page_pvt *pvt;
876 	unsigned long pfn;
877 	unsigned long gfn = gpa >> page_shift;
878 	int srcu_idx;
879 	unsigned long uvmem_pfn;
880 
881 	srcu_idx = srcu_read_lock(&kvm->srcu);
882 	mutex_lock(&kvm->arch.uvmem_lock);
883 	if (kvmppc_gfn_is_uvmem_pfn(gfn, kvm, &uvmem_pfn)) {
884 		uvmem_page = pfn_to_page(uvmem_pfn);
885 		pvt = uvmem_page->zone_device_data;
886 		pvt->skip_page_out = true;
887 		/*
888 		 * do not drop the GFN. It is a valid GFN
889 		 * that is transitioned to a shared GFN.
890 		 */
891 		pvt->remove_gfn = false;
892 	}
893 
894 retry:
895 	mutex_unlock(&kvm->arch.uvmem_lock);
896 	pfn = gfn_to_pfn(kvm, gfn);
897 	if (is_error_noslot_pfn(pfn))
898 		goto out;
899 
900 	mutex_lock(&kvm->arch.uvmem_lock);
901 	if (kvmppc_gfn_is_uvmem_pfn(gfn, kvm, &uvmem_pfn)) {
902 		uvmem_page = pfn_to_page(uvmem_pfn);
903 		pvt = uvmem_page->zone_device_data;
904 		pvt->skip_page_out = true;
905 		pvt->remove_gfn = false; /* it continues to be a valid GFN */
906 		kvm_release_pfn_clean(pfn);
907 		goto retry;
908 	}
909 
910 	if (!uv_page_in(kvm->arch.lpid, pfn << page_shift, gpa, 0,
911 				page_shift)) {
912 		kvmppc_gfn_shared(gfn, kvm);
913 		ret = H_SUCCESS;
914 	}
915 	kvm_release_pfn_clean(pfn);
916 	mutex_unlock(&kvm->arch.uvmem_lock);
917 out:
918 	srcu_read_unlock(&kvm->srcu, srcu_idx);
919 	return ret;
920 }
921 
922 /*
923  * H_SVM_PAGE_IN: Move page from normal memory to secure memory.
924  *
925  * H_PAGE_IN_SHARED flag makes the page shared which means that the same
926  * memory in is visible from both UV and HV.
927  */
kvmppc_h_svm_page_in(struct kvm * kvm,unsigned long gpa,unsigned long flags,unsigned long page_shift)928 unsigned long kvmppc_h_svm_page_in(struct kvm *kvm, unsigned long gpa,
929 		unsigned long flags,
930 		unsigned long page_shift)
931 {
932 	unsigned long start, end;
933 	struct vm_area_struct *vma;
934 	int srcu_idx;
935 	unsigned long gfn = gpa >> page_shift;
936 	int ret;
937 
938 	if (!(kvm->arch.secure_guest & KVMPPC_SECURE_INIT_START))
939 		return H_UNSUPPORTED;
940 
941 	if (page_shift != PAGE_SHIFT)
942 		return H_P3;
943 
944 	if (flags & ~H_PAGE_IN_SHARED)
945 		return H_P2;
946 
947 	if (flags & H_PAGE_IN_SHARED)
948 		return kvmppc_share_page(kvm, gpa, page_shift);
949 
950 	ret = H_PARAMETER;
951 	srcu_idx = srcu_read_lock(&kvm->srcu);
952 	mmap_read_lock(kvm->mm);
953 
954 	start = gfn_to_hva(kvm, gfn);
955 	if (kvm_is_error_hva(start))
956 		goto out;
957 
958 	mutex_lock(&kvm->arch.uvmem_lock);
959 	/* Fail the page-in request of an already paged-in page */
960 	if (kvmppc_gfn_is_uvmem_pfn(gfn, kvm, NULL))
961 		goto out_unlock;
962 
963 	end = start + (1UL << page_shift);
964 	vma = find_vma_intersection(kvm->mm, start, end);
965 	if (!vma || vma->vm_start > start || vma->vm_end < end)
966 		goto out_unlock;
967 
968 	if (kvmppc_svm_page_in(vma, start, end, gpa, kvm, page_shift,
969 				true))
970 		goto out_unlock;
971 
972 	ret = H_SUCCESS;
973 
974 out_unlock:
975 	mutex_unlock(&kvm->arch.uvmem_lock);
976 out:
977 	mmap_read_unlock(kvm->mm);
978 	srcu_read_unlock(&kvm->srcu, srcu_idx);
979 	return ret;
980 }
981 
982 
983 /*
984  * Fault handler callback that gets called when HV touches any page that
985  * has been moved to secure memory, we ask UV to give back the page by
986  * issuing UV_PAGE_OUT uvcall.
987  *
988  * This eventually results in dropping of device PFN and the newly
989  * provisioned page/PFN gets populated in QEMU page tables.
990  */
kvmppc_uvmem_migrate_to_ram(struct vm_fault * vmf)991 static vm_fault_t kvmppc_uvmem_migrate_to_ram(struct vm_fault *vmf)
992 {
993 	struct kvmppc_uvmem_page_pvt *pvt = vmf->page->zone_device_data;
994 
995 	if (kvmppc_svm_page_out(vmf->vma, vmf->address,
996 				vmf->address + PAGE_SIZE, PAGE_SHIFT,
997 				pvt->kvm, pvt->gpa, vmf->page))
998 		return VM_FAULT_SIGBUS;
999 	else
1000 		return 0;
1001 }
1002 
1003 /*
1004  * Release the device PFN back to the pool
1005  *
1006  * Gets called when secure GFN tranistions from a secure-PFN
1007  * to a normal PFN during H_SVM_PAGE_OUT.
1008  * Gets called with kvm->arch.uvmem_lock held.
1009  */
kvmppc_uvmem_page_free(struct page * page)1010 static void kvmppc_uvmem_page_free(struct page *page)
1011 {
1012 	unsigned long pfn = page_to_pfn(page) -
1013 			(kvmppc_uvmem_pgmap.range.start >> PAGE_SHIFT);
1014 	struct kvmppc_uvmem_page_pvt *pvt;
1015 
1016 	spin_lock(&kvmppc_uvmem_bitmap_lock);
1017 	bitmap_clear(kvmppc_uvmem_bitmap, pfn, 1);
1018 	spin_unlock(&kvmppc_uvmem_bitmap_lock);
1019 
1020 	pvt = page->zone_device_data;
1021 	page->zone_device_data = NULL;
1022 	if (pvt->remove_gfn)
1023 		kvmppc_gfn_remove(pvt->gpa >> PAGE_SHIFT, pvt->kvm);
1024 	else
1025 		kvmppc_gfn_secure_mem_pfn(pvt->gpa >> PAGE_SHIFT, pvt->kvm);
1026 	kfree(pvt);
1027 }
1028 
1029 static const struct dev_pagemap_ops kvmppc_uvmem_ops = {
1030 	.page_free = kvmppc_uvmem_page_free,
1031 	.migrate_to_ram	= kvmppc_uvmem_migrate_to_ram,
1032 };
1033 
1034 /*
1035  * H_SVM_PAGE_OUT: Move page from secure memory to normal memory.
1036  */
1037 unsigned long
kvmppc_h_svm_page_out(struct kvm * kvm,unsigned long gpa,unsigned long flags,unsigned long page_shift)1038 kvmppc_h_svm_page_out(struct kvm *kvm, unsigned long gpa,
1039 		      unsigned long flags, unsigned long page_shift)
1040 {
1041 	unsigned long gfn = gpa >> page_shift;
1042 	unsigned long start, end;
1043 	struct vm_area_struct *vma;
1044 	int srcu_idx;
1045 	int ret;
1046 
1047 	if (!(kvm->arch.secure_guest & KVMPPC_SECURE_INIT_START))
1048 		return H_UNSUPPORTED;
1049 
1050 	if (page_shift != PAGE_SHIFT)
1051 		return H_P3;
1052 
1053 	if (flags)
1054 		return H_P2;
1055 
1056 	ret = H_PARAMETER;
1057 	srcu_idx = srcu_read_lock(&kvm->srcu);
1058 	mmap_read_lock(kvm->mm);
1059 	start = gfn_to_hva(kvm, gfn);
1060 	if (kvm_is_error_hva(start))
1061 		goto out;
1062 
1063 	end = start + (1UL << page_shift);
1064 	vma = find_vma_intersection(kvm->mm, start, end);
1065 	if (!vma || vma->vm_start > start || vma->vm_end < end)
1066 		goto out;
1067 
1068 	if (!kvmppc_svm_page_out(vma, start, end, page_shift, kvm, gpa, NULL))
1069 		ret = H_SUCCESS;
1070 out:
1071 	mmap_read_unlock(kvm->mm);
1072 	srcu_read_unlock(&kvm->srcu, srcu_idx);
1073 	return ret;
1074 }
1075 
kvmppc_send_page_to_uv(struct kvm * kvm,unsigned long gfn)1076 int kvmppc_send_page_to_uv(struct kvm *kvm, unsigned long gfn)
1077 {
1078 	unsigned long pfn;
1079 	int ret = U_SUCCESS;
1080 
1081 	pfn = gfn_to_pfn(kvm, gfn);
1082 	if (is_error_noslot_pfn(pfn))
1083 		return -EFAULT;
1084 
1085 	mutex_lock(&kvm->arch.uvmem_lock);
1086 	if (kvmppc_gfn_is_uvmem_pfn(gfn, kvm, NULL))
1087 		goto out;
1088 
1089 	ret = uv_page_in(kvm->arch.lpid, pfn << PAGE_SHIFT, gfn << PAGE_SHIFT,
1090 			 0, PAGE_SHIFT);
1091 out:
1092 	kvm_release_pfn_clean(pfn);
1093 	mutex_unlock(&kvm->arch.uvmem_lock);
1094 	return (ret == U_SUCCESS) ? RESUME_GUEST : -EFAULT;
1095 }
1096 
kvmppc_uvmem_memslot_create(struct kvm * kvm,const struct kvm_memory_slot * new)1097 int kvmppc_uvmem_memslot_create(struct kvm *kvm, const struct kvm_memory_slot *new)
1098 {
1099 	int ret = __kvmppc_uvmem_memslot_create(kvm, new);
1100 
1101 	if (!ret)
1102 		ret = kvmppc_uv_migrate_mem_slot(kvm, new);
1103 
1104 	return ret;
1105 }
1106 
kvmppc_uvmem_memslot_delete(struct kvm * kvm,const struct kvm_memory_slot * old)1107 void kvmppc_uvmem_memslot_delete(struct kvm *kvm, const struct kvm_memory_slot *old)
1108 {
1109 	__kvmppc_uvmem_memslot_delete(kvm, old);
1110 }
1111 
kvmppc_get_secmem_size(void)1112 static u64 kvmppc_get_secmem_size(void)
1113 {
1114 	struct device_node *np;
1115 	int i, len;
1116 	const __be32 *prop;
1117 	u64 size = 0;
1118 
1119 	/*
1120 	 * First try the new ibm,secure-memory nodes which supersede the
1121 	 * secure-memory-ranges property.
1122 	 * If we found some, no need to read the deprecated ones.
1123 	 */
1124 	for_each_compatible_node(np, NULL, "ibm,secure-memory") {
1125 		prop = of_get_property(np, "reg", &len);
1126 		if (!prop)
1127 			continue;
1128 		size += of_read_number(prop + 2, 2);
1129 	}
1130 	if (size)
1131 		return size;
1132 
1133 	np = of_find_compatible_node(NULL, NULL, "ibm,uv-firmware");
1134 	if (!np)
1135 		goto out;
1136 
1137 	prop = of_get_property(np, "secure-memory-ranges", &len);
1138 	if (!prop)
1139 		goto out_put;
1140 
1141 	for (i = 0; i < len / (sizeof(*prop) * 4); i++)
1142 		size += of_read_number(prop + (i * 4) + 2, 2);
1143 
1144 out_put:
1145 	of_node_put(np);
1146 out:
1147 	return size;
1148 }
1149 
kvmppc_uvmem_init(void)1150 int kvmppc_uvmem_init(void)
1151 {
1152 	int ret = 0;
1153 	unsigned long size;
1154 	struct resource *res;
1155 	void *addr;
1156 	unsigned long pfn_last, pfn_first;
1157 
1158 	size = kvmppc_get_secmem_size();
1159 	if (!size) {
1160 		/*
1161 		 * Don't fail the initialization of kvm-hv module if
1162 		 * the platform doesn't export ibm,uv-firmware node.
1163 		 * Let normal guests run on such PEF-disabled platform.
1164 		 */
1165 		pr_info("KVMPPC-UVMEM: No support for secure guests\n");
1166 		goto out;
1167 	}
1168 
1169 	res = request_free_mem_region(&iomem_resource, size, "kvmppc_uvmem");
1170 	if (IS_ERR(res)) {
1171 		ret = PTR_ERR(res);
1172 		goto out;
1173 	}
1174 
1175 	kvmppc_uvmem_pgmap.type = MEMORY_DEVICE_PRIVATE;
1176 	kvmppc_uvmem_pgmap.range.start = res->start;
1177 	kvmppc_uvmem_pgmap.range.end = res->end;
1178 	kvmppc_uvmem_pgmap.nr_range = 1;
1179 	kvmppc_uvmem_pgmap.ops = &kvmppc_uvmem_ops;
1180 	/* just one global instance: */
1181 	kvmppc_uvmem_pgmap.owner = &kvmppc_uvmem_pgmap;
1182 	addr = memremap_pages(&kvmppc_uvmem_pgmap, NUMA_NO_NODE);
1183 	if (IS_ERR(addr)) {
1184 		ret = PTR_ERR(addr);
1185 		goto out_free_region;
1186 	}
1187 
1188 	pfn_first = res->start >> PAGE_SHIFT;
1189 	pfn_last = pfn_first + (resource_size(res) >> PAGE_SHIFT);
1190 	kvmppc_uvmem_bitmap = kcalloc(BITS_TO_LONGS(pfn_last - pfn_first),
1191 				      sizeof(unsigned long), GFP_KERNEL);
1192 	if (!kvmppc_uvmem_bitmap) {
1193 		ret = -ENOMEM;
1194 		goto out_unmap;
1195 	}
1196 
1197 	pr_info("KVMPPC-UVMEM: Secure Memory size 0x%lx\n", size);
1198 	return ret;
1199 out_unmap:
1200 	memunmap_pages(&kvmppc_uvmem_pgmap);
1201 out_free_region:
1202 	release_mem_region(res->start, size);
1203 out:
1204 	return ret;
1205 }
1206 
kvmppc_uvmem_free(void)1207 void kvmppc_uvmem_free(void)
1208 {
1209 	if (!kvmppc_uvmem_bitmap)
1210 		return;
1211 
1212 	memunmap_pages(&kvmppc_uvmem_pgmap);
1213 	release_mem_region(kvmppc_uvmem_pgmap.range.start,
1214 			   range_len(&kvmppc_uvmem_pgmap.range));
1215 	kfree(kvmppc_uvmem_bitmap);
1216 }
1217