1 // SPDX-License-Identifier: GPL-2.0
3  *  Page table allocation functions
38 	return register_sysctl("vm", page_table_sysctl) ? 0 : -ENOMEM;  in page_table_register_sysctl()
66 	if (current->active_mm == mm) {  in __crst_table_upgrade()
67 		S390_lowcore.user_asce = mm->context.asce;  in __crst_table_upgrade()
76 	unsigned long asce_limit = mm->context.asce_limit;  in crst_table_upgrade()
78 	/* upgrade should only happen from 3 to 4, 3 to 5, or 4 to 5 levels */  in crst_table_upgrade()
97 	spin_lock_bh(&mm->page_table_lock);  in crst_table_upgrade()
104 	VM_BUG_ON(asce_limit != mm->context.asce_limit);  in crst_table_upgrade()
107 		__pgd = (unsigned long *) mm->pgd;  in crst_table_upgrade()
109 		mm->pgd = (pgd_t *) p4d;  in crst_table_upgrade()
110 		mm->context.asce_limit = _REGION1_SIZE;  in crst_table_upgrade()
111 		mm->context.asce = __pa(mm->pgd) | _ASCE_TABLE_LENGTH |  in crst_table_upgrade()
116 		__pgd = (unsigned long *) mm->pgd;  in crst_table_upgrade()
118 		mm->pgd = (pgd_t *) pgd;  in crst_table_upgrade()
119 		mm->context.asce_limit = TASK_SIZE_MAX;  in crst_table_upgrade()
120 		mm->context.asce = __pa(mm->pgd) | _ASCE_TABLE_LENGTH |  in crst_table_upgrade()
124 	spin_unlock_bh(&mm->page_table_lock);  in crst_table_upgrade()
133 	return -ENOMEM;  in crst_table_upgrade()
143 struct page *page_table_alloc_pgste(struct mm_struct *mm)  in page_table_alloc_pgste()
158 void page_table_free_pgste(struct page *page)  in page_table_free_pgste()  argument
160 	pagetable_free(page_ptdesc(page));  in page_table_free_pgste()
166  * A 2KB-pgtable is either upper or lower half of a normal page.
167  * The second half of the page may be unused or used as another
168  * 2KB-pgtable.
170  * Whenever possible the parent page for a new 2KB-pgtable is picked
172  * In case the list is empty a new parent page is allocated and added to
175  * When a parent page gets fully allocated it contains 2KB-pgtables in both
178  * When 2KB-pgtable is freed from to fully allocated parent page that
179  * page turns partially allocated and added to mm_context_t::pgtable_list.
181  * If 2KB-pgtable is freed from the partially allocated parent page that
182  * page turns unused and gets removed from mm_context_t::pgtable_list.
183  * Furthermore, the unused parent page is released.
188  * The upper byte (bits 24-31) of the parent page _refcount is used
189  * for tracking contained 2KB-pgtables and has the following format:
192  * 01234567    upper byte (bits 24-31) of struct page::_refcount
194  *   ||  |+--- upper 2KB-pgtable is allocated
195  *   ||  +---- lower 2KB-pgtable is allocated
196  *   |+------- upper 2KB-pgtable is pending for removal
197  *   +-------- lower 2KB-pgtable is pending for removal
202  * When 2KB-pgtable is allocated the corresponding AA bit is set to 1.
203  * The parent page is either:
204  *   - added to mm_context_t::pgtable_list in case the second half of the
205  *     parent page is still unallocated;
206  *   - removed from mm_context_t::pgtable_list in case both hales of the
207  *     parent page are allocated;
210  * When 2KB-pgtable is deallocated the corresponding AA bit is set to 0
212  * Thus, PP and AA bits corresponding to the same 2KB-pgtable are mutually
214  * The parent page is either:
215  *   - added to mm_context_t::pgtable_list in case the second half of the
216  *     parent page is still allocated;
217  *   - removed from mm_context_t::pgtable_list in case the second half of
218  *     the parent page is unallocated;
222  * mm_context_t::pgtable_list and AA bits, but not the parent page itself
225  * Releasing the parent page happens whenever the PP bit turns from 1 to 0,
227  * parent page does not contain any 2KB-pgtable fragment anymore, and it has
229  * the page therefore.
231  * PGSTE memory spaces use full 4KB-pgtables and do not need most of the
232  * logic described above. Both AA bits are set to 1 to denote a 4KB-pgtable
233  * while the PP bits are never used, nor such a page is added to or removed
236  * pte_free_defer() overrides those rules: it takes the page off pgtable_list,
240  * But for simplicity, because page->rcu_head overlays page->lru, and because
251 	/* Try to get a fragment of a 4K page as a 2K page table */  in page_table_alloc()
254 		spin_lock_bh(&mm->context.lock);  in page_table_alloc()
255 		if (!list_empty(&mm->context.pgtable_list)) {  in page_table_alloc()
256 			ptdesc = list_first_entry(&mm->context.pgtable_list,  in page_table_alloc()
258 			mask = atomic_read(&ptdesc->_refcount) >> 24;  in page_table_alloc()
268 			mask = (mask | (mask >> 4)) & 0x03U;  in page_table_alloc()
271 				bit = mask & 1;		/* =1 -> second 2K */  in page_table_alloc()
274 				atomic_xor_bits(&ptdesc->_refcount,  in page_table_alloc()
276 				list_del_init(&ptdesc->pt_list);  in page_table_alloc()
279 		spin_unlock_bh(&mm->context.lock);  in page_table_alloc()
283 	/* Allocate a fresh page */  in page_table_alloc()
292 	/* Initialize page table */  in page_table_alloc()
295 		/* Return 4K page table with PGSTEs */  in page_table_alloc()
296 		INIT_LIST_HEAD(&ptdesc->pt_list);  in page_table_alloc()
297 		atomic_xor_bits(&ptdesc->_refcount, 0x03U << 24);  in page_table_alloc()
301 		/* Return the first 2K fragment of the page */  in page_table_alloc()
302 		atomic_xor_bits(&ptdesc->_refcount, 0x01U << 24);  in page_table_alloc()
304 		spin_lock_bh(&mm->context.lock);  in page_table_alloc()
305 		list_add(&ptdesc->pt_list, &mm->context.pgtable_list);  in page_table_alloc()
306 		spin_unlock_bh(&mm->context.lock);  in page_table_alloc()
311 static void page_table_release_check(struct page *page, void *table,  in page_table_release_check()  argument
318 	if (!mask && list_empty(&page->lru))  in page_table_release_check()
323 	dump_page(page, msg);  in page_table_release_check()
341 		/* Free 2K page table fragment of a 4K page */  in page_table_free()
343 		spin_lock_bh(&mm->context.lock);  in page_table_free()
345 		 * Mark the page for delayed release. The actual release  in page_table_free()
349 		mask = atomic_xor_bits(&ptdesc->_refcount, 0x11U << (bit + 24));  in page_table_free()
354 			 * its free deferred: add page to head of list, to make  in page_table_free()
357 			list_add(&ptdesc->pt_list, &mm->context.pgtable_list);  in page_table_free()
359 			/* If page is on list, now remove it. */  in page_table_free()
360 			list_del_init(&ptdesc->pt_list);  in page_table_free()
362 		spin_unlock_bh(&mm->context.lock);  in page_table_free()
363 		mask = atomic_xor_bits(&ptdesc->_refcount, 0x10U << (bit + 24));  in page_table_free()
370 		mask = atomic_xor_bits(&ptdesc->_refcount, 0x03U << 24);  in page_table_free()
376 		call_rcu(&ptdesc->pt_rcu_head, pte_free_now);  in page_table_free()
378 		pte_free_now(&ptdesc->pt_rcu_head);  in page_table_free()
388 	mm = tlb->mm;  in page_table_free_rcu()
396 	spin_lock_bh(&mm->context.lock);  in page_table_free_rcu()
398 	 * Mark the page for delayed release. The actual release will happen  in page_table_free_rcu()
402 	mask = atomic_xor_bits(&ptdesc->_refcount, 0x11U << (bit + 24));  in page_table_free_rcu()
407 		 * its free deferred: add page to end of list, to make  in page_table_free_rcu()
411 		list_add_tail(&ptdesc->pt_list, &mm->context.pgtable_list);  in page_table_free_rcu()
413 		/* If page is on list, now remove it. */  in page_table_free_rcu()
414 		list_del_init(&ptdesc->pt_list);  in page_table_free_rcu()
416 	spin_unlock_bh(&mm->context.lock);  in page_table_free_rcu()
431 	case 0x01U:	/* lower 2K of a 4K page table */  in __tlb_remove_table()
432 	case 0x02U:	/* higher 2K of a 4K page table */  in __tlb_remove_table()
433 		mask = atomic_xor_bits(&ptdesc->_refcount, mask << (4 + 24));  in __tlb_remove_table()
438 	case 0x03U:	/* 4K page table with pgstes */  in __tlb_remove_table()
439 		mask = atomic_xor_bits(&ptdesc->_refcount, 0x03U << 24);  in __tlb_remove_table()
446 		call_rcu(&ptdesc->pt_rcu_head, pte_free_now);  in __tlb_remove_table()
448 		pte_free_now(&ptdesc->pt_rcu_head);  in __tlb_remove_table()
454 	struct page *page;  in pte_free_defer()  local
456 	page = virt_to_page(pgtable);  in pte_free_defer()
457 	SetPageActive(page);  in pte_free_defer()
469  * and page tables that do not make use of enhanced features like EDAT1.
514 	unsigned long next = (addr + (SIZE)) & ~((SIZE) - 1);		\
516 	return (next - 1) < (end - 1) ? next : end;			\
519 BASE_ADDR_END_FUNC(page,    _PAGE_SIZE)  in BASE_ADDR_END_FUNC()  argument
566 				return -ENOMEM;  in base_segment_walk()
595 				return -ENOMEM;  in base_region3_walk()
623 				return -ENOMEM;  in base_region2_walk()
651 				return -ENOMEM;  in base_region1_walk()
665  * base_asce_free - free asce and tables returned from base_asce_alloc()
668  * Frees all region, segment, and page tables that were allocated with a
705 	return base_pgt_cache ? 0 : -ENOMEM;  in base_pgt_cache_init()
709  * base_asce_alloc - create kernel mapping without enhanced DAT features
713  * Generate an asce, including all required region, segment and page tables,