1 // SPDX-License-Identifier: GPL-2.0
3  *  Page table allocation functions
38 	return register_sysctl("vm", page_table_sysctl) ? 0 : -ENOMEM;  in page_table_register_sysctl()
64 	if (current->active_mm == mm) {  in __crst_table_upgrade()
65 		S390_lowcore.user_asce = mm->context.asce;  in __crst_table_upgrade()
74 	unsigned long asce_limit = mm->context.asce_limit;  in crst_table_upgrade()
76 	/* upgrade should only happen from 3 to 4, 3 to 5, or 4 to 5 levels */  in crst_table_upgrade()
95 	spin_lock_bh(&mm->page_table_lock);  in crst_table_upgrade()
102 	VM_BUG_ON(asce_limit != mm->context.asce_limit);  in crst_table_upgrade()
105 		__pgd = (unsigned long *) mm->pgd;  in crst_table_upgrade()
107 		mm->pgd = (pgd_t *) p4d;  in crst_table_upgrade()
108 		mm->context.asce_limit = _REGION1_SIZE;  in crst_table_upgrade()
109 		mm->context.asce = __pa(mm->pgd) | _ASCE_TABLE_LENGTH |  in crst_table_upgrade()
114 		__pgd = (unsigned long *) mm->pgd;  in crst_table_upgrade()
116 		mm->pgd = (pgd_t *) pgd;  in crst_table_upgrade()
117 		mm->context.asce_limit = TASK_SIZE_MAX;  in crst_table_upgrade()
118 		mm->context.asce = __pa(mm->pgd) | _ASCE_TABLE_LENGTH |  in crst_table_upgrade()
122 	spin_unlock_bh(&mm->page_table_lock);  in crst_table_upgrade()
131 	return -ENOMEM;  in crst_table_upgrade()
141 struct page *page_table_alloc_pgste(struct mm_struct *mm)  in page_table_alloc_pgste()
156 void page_table_free_pgste(struct page *page)  in page_table_free_pgste()  argument
158 	pagetable_free(page_ptdesc(page));  in page_table_free_pgste()
164  * A 2KB-pgtable is either upper or lower half of a normal page.
165  * The second half of the page may be unused or used as another
166  * 2KB-pgtable.
168  * Whenever possible the parent page for a new 2KB-pgtable is picked
170  * In case the list is empty a new parent page is allocated and added to
173  * When a parent page gets fully allocated it contains 2KB-pgtables in both
176  * When 2KB-pgtable is freed from to fully allocated parent page that
177  * page turns partially allocated and added to mm_context_t::pgtable_list.
179  * If 2KB-pgtable is freed from the partially allocated parent page that
180  * page turns unused and gets removed from mm_context_t::pgtable_list.
181  * Furthermore, the unused parent page is released.
186  * The upper byte (bits 24-31) of the parent page _refcount is used
187  * for tracking contained 2KB-pgtables and has the following format:
190  * 01234567    upper byte (bits 24-31) of struct page::_refcount
192  *   ||  |+--- upper 2KB-pgtable is allocated
193  *   ||  +---- lower 2KB-pgtable is allocated
194  *   |+------- upper 2KB-pgtable is pending for removal
195  *   +-------- lower 2KB-pgtable is pending for removal
200  * When 2KB-pgtable is allocated the corresponding AA bit is set to 1.
201  * The parent page is either:
202  *   - added to mm_context_t::pgtable_list in case the second half of the
203  *     parent page is still unallocated;
204  *   - removed from mm_context_t::pgtable_list in case both hales of the
205  *     parent page are allocated;
208  * When 2KB-pgtable is deallocated the corresponding AA bit is set to 0
210  * Thus, PP and AA bits corresponding to the same 2KB-pgtable are mutually
212  * The parent page is either:
213  *   - added to mm_context_t::pgtable_list in case the second half of the
214  *     parent page is still allocated;
215  *   - removed from mm_context_t::pgtable_list in case the second half of
216  *     the parent page is unallocated;
220  * mm_context_t::pgtable_list and AA bits, but not the parent page itself
223  * Releasing the parent page happens whenever the PP bit turns from 1 to 0,
225  * parent page does not contain any 2KB-pgtable fragment anymore, and it has
227  * the page therefore.
229  * PGSTE memory spaces use full 4KB-pgtables and do not need most of the
230  * logic described above. Both AA bits are set to 1 to denote a 4KB-pgtable
231  * while the PP bits are never used, nor such a page is added to or removed
234  * pte_free_defer() overrides those rules: it takes the page off pgtable_list,
238  * But for simplicity, because page->rcu_head overlays page->lru, and because
249 	/* Try to get a fragment of a 4K page as a 2K page table */  in page_table_alloc()
252 		spin_lock_bh(&mm->context.lock);  in page_table_alloc()
253 		if (!list_empty(&mm->context.pgtable_list)) {  in page_table_alloc()
254 			ptdesc = list_first_entry(&mm->context.pgtable_list,  in page_table_alloc()
256 			mask = atomic_read(&ptdesc->_refcount) >> 24;  in page_table_alloc()
266 			mask = (mask | (mask >> 4)) & 0x03U;  in page_table_alloc()
269 				bit = mask & 1;		/* =1 -> second 2K */  in page_table_alloc()
272 				atomic_xor_bits(&ptdesc->_refcount,  in page_table_alloc()
274 				list_del_init(&ptdesc->pt_list);  in page_table_alloc()
277 		spin_unlock_bh(&mm->context.lock);  in page_table_alloc()
281 	/* Allocate a fresh page */  in page_table_alloc()
290 	/* Initialize page table */  in page_table_alloc()
293 		/* Return 4K page table with PGSTEs */  in page_table_alloc()
294 		INIT_LIST_HEAD(&ptdesc->pt_list);  in page_table_alloc()
295 		atomic_xor_bits(&ptdesc->_refcount, 0x03U << 24);  in page_table_alloc()
299 		/* Return the first 2K fragment of the page */  in page_table_alloc()
300 		atomic_xor_bits(&ptdesc->_refcount, 0x01U << 24);  in page_table_alloc()
302 		spin_lock_bh(&mm->context.lock);  in page_table_alloc()
303 		list_add(&ptdesc->pt_list, &mm->context.pgtable_list);  in page_table_alloc()
304 		spin_unlock_bh(&mm->context.lock);  in page_table_alloc()
309 static void page_table_release_check(struct page *page, void *table,  in page_table_release_check()  argument
316 	if (!mask && list_empty(&page->lru))  in page_table_release_check()
321 	dump_page(page, msg);  in page_table_release_check()
339 		/* Free 2K page table fragment of a 4K page */  in page_table_free()
341 		spin_lock_bh(&mm->context.lock);  in page_table_free()
343 		 * Mark the page for delayed release. The actual release  in page_table_free()
347 		mask = atomic_xor_bits(&ptdesc->_refcount, 0x11U << (bit + 24));  in page_table_free()
352 			 * its free deferred: add page to head of list, to make  in page_table_free()
355 			list_add(&ptdesc->pt_list, &mm->context.pgtable_list);  in page_table_free()
357 			/* If page is on list, now remove it. */  in page_table_free()
358 			list_del_init(&ptdesc->pt_list);  in page_table_free()
360 		spin_unlock_bh(&mm->context.lock);  in page_table_free()
361 		mask = atomic_xor_bits(&ptdesc->_refcount, 0x10U << (bit + 24));  in page_table_free()
368 		mask = atomic_xor_bits(&ptdesc->_refcount, 0x03U << 24);  in page_table_free()
374 		call_rcu(&ptdesc->pt_rcu_head, pte_free_now);  in page_table_free()
376 		pte_free_now(&ptdesc->pt_rcu_head);  in page_table_free()
386 	mm = tlb->mm;  in page_table_free_rcu()
394 	spin_lock_bh(&mm->context.lock);  in page_table_free_rcu()
396 	 * Mark the page for delayed release. The actual release will happen  in page_table_free_rcu()
400 	mask = atomic_xor_bits(&ptdesc->_refcount, 0x11U << (bit + 24));  in page_table_free_rcu()
405 		 * its free deferred: add page to end of list, to make  in page_table_free_rcu()
409 		list_add_tail(&ptdesc->pt_list, &mm->context.pgtable_list);  in page_table_free_rcu()
411 		/* If page is on list, now remove it. */  in page_table_free_rcu()
412 		list_del_init(&ptdesc->pt_list);  in page_table_free_rcu()
414 	spin_unlock_bh(&mm->context.lock);  in page_table_free_rcu()
429 	case 0x01U:	/* lower 2K of a 4K page table */  in __tlb_remove_table()
430 	case 0x02U:	/* higher 2K of a 4K page table */  in __tlb_remove_table()
431 		mask = atomic_xor_bits(&ptdesc->_refcount, mask << (4 + 24));  in __tlb_remove_table()
436 	case 0x03U:	/* 4K page table with pgstes */  in __tlb_remove_table()
437 		mask = atomic_xor_bits(&ptdesc->_refcount, 0x03U << 24);  in __tlb_remove_table()
444 		call_rcu(&ptdesc->pt_rcu_head, pte_free_now);  in __tlb_remove_table()
446 		pte_free_now(&ptdesc->pt_rcu_head);  in __tlb_remove_table()
452 	struct page *page;  in pte_free_defer()  local
454 	page = virt_to_page(pgtable);  in pte_free_defer()
455 	SetPageActive(page);  in pte_free_defer()
467  * and page tables that do not make use of enhanced features like EDAT1.
510 	unsigned long next = (addr + (SIZE)) & ~((SIZE) - 1);		\
512 	return (next - 1) < (end - 1) ? next : end;			\
515 BASE_ADDR_END_FUNC(page,    _PAGE_SIZE)  in BASE_ADDR_END_FUNC()  argument
562 				return -ENOMEM;  in base_segment_walk()
591 				return -ENOMEM;  in base_region3_walk()
619 				return -ENOMEM;  in base_region2_walk()
647 				return -ENOMEM;  in base_region1_walk()
661  * base_asce_free - free asce and tables returned from base_asce_alloc()
664  * Frees all region, segment, and page tables that were allocated with a
701 	return base_pgt_cache ? 0 : -ENOMEM;  in base_pgt_cache_init()
705  * base_asce_alloc - create kernel mapping without enhanced DAT features
709  * Generate an asce, including all required region, segment and page tables,