1442 static int xol_add_vma(struct mm_struct *mm, struct xol_area *area)  in xol_add_vma()  argument
1455 	if (!area->vaddr) {  in xol_add_vma()
1457 		area->vaddr = get_unmapped_area(NULL, TASK_SIZE - PAGE_SIZE,  in xol_add_vma()
1459 		if (IS_ERR_VALUE(area->vaddr)) {  in xol_add_vma()
1460 			ret = area->vaddr;  in xol_add_vma()
1465 	vma = _install_special_mapping(mm, area->vaddr, PAGE_SIZE,  in xol_add_vma()
1467 				&area->xol_mapping);  in xol_add_vma()
1475 	smp_store_release(&mm->uprobes_state.xol_area, area); /* ^^^ */  in xol_add_vma()
1486 	struct xol_area *area;  in __create_xol_area()  local
1488 	area = kmalloc(sizeof(*area), GFP_KERNEL);  in __create_xol_area()
1489 	if (unlikely(!area))  in __create_xol_area()
1492 	area->bitmap = kcalloc(BITS_TO_LONGS(UINSNS_PER_PAGE), sizeof(long),  in __create_xol_area()
1494 	if (!area->bitmap)  in __create_xol_area()
1497 	area->xol_mapping.name = "[uprobes]";  in __create_xol_area()
1498 	area->xol_mapping.fault = NULL;  in __create_xol_area()
1499 	area->xol_mapping.pages = area->pages;  in __create_xol_area()
1500 	area->pages[0] = alloc_page(GFP_HIGHUSER);  in __create_xol_area()
1501 	if (!area->pages[0])  in __create_xol_area()
1503 	area->pages[1] = NULL;  in __create_xol_area()
1505 	area->vaddr = vaddr;  in __create_xol_area()
1506 	init_waitqueue_head(&area->wq);  in __create_xol_area()
1508 	set_bit(0, area->bitmap);  in __create_xol_area()
1509 	atomic_set(&area->slot_count, 1);  in __create_xol_area()
1510 	arch_uprobe_copy_ixol(area->pages[0], 0, &insn, UPROBE_SWBP_INSN_SIZE);  in __create_xol_area()
1512 	if (!xol_add_vma(mm, area))  in __create_xol_area()
1513 		return area;  in __create_xol_area()
1515 	__free_page(area->pages[0]);  in __create_xol_area()
1517 	kfree(area->bitmap);  in __create_xol_area()
1519 	kfree(area);  in __create_xol_area()
1533 	struct xol_area *area;  in get_xol_area()  local
1539 	area = READ_ONCE(mm->uprobes_state.xol_area); /* ^^^ */  in get_xol_area()
1540 	return area;  in get_xol_area()
1548 	struct xol_area *area = mm->uprobes_state.xol_area;  in uprobe_clear_state()  local
1554 	if (!area)  in uprobe_clear_state()
1557 	put_page(area->pages[0]);  in uprobe_clear_state()
1558 	kfree(area->bitmap);  in uprobe_clear_state()
1559 	kfree(area);  in uprobe_clear_state()
1584 static unsigned long xol_take_insn_slot(struct xol_area *area)  in xol_take_insn_slot()  argument
1590 		slot_nr = find_first_zero_bit(area->bitmap, UINSNS_PER_PAGE);  in xol_take_insn_slot()
1592 			if (!test_and_set_bit(slot_nr, area->bitmap))  in xol_take_insn_slot()
1598 		wait_event(area->wq, (atomic_read(&area->slot_count) < UINSNS_PER_PAGE));  in xol_take_insn_slot()
1601 	slot_addr = area->vaddr + (slot_nr * UPROBE_XOL_SLOT_BYTES);  in xol_take_insn_slot()
1602 	atomic_inc(&area->slot_count);  in xol_take_insn_slot()
1613 	struct xol_area *area;  in xol_get_insn_slot()  local
1616 	area = get_xol_area();  in xol_get_insn_slot()
1617 	if (!area)  in xol_get_insn_slot()
1620 	xol_vaddr = xol_take_insn_slot(area);  in xol_get_insn_slot()
1624 	arch_uprobe_copy_ixol(area->pages[0], xol_vaddr,  in xol_get_insn_slot()
1637 	struct xol_area *area;  in xol_free_insn_slot()  local
1648 	area = tsk->mm->uprobes_state.xol_area;  in xol_free_insn_slot()
1649 	vma_end = area->vaddr + PAGE_SIZE;  in xol_free_insn_slot()
1650 	if (area->vaddr <= slot_addr && slot_addr < vma_end) {  in xol_free_insn_slot()
1654 		offset = slot_addr - area->vaddr;  in xol_free_insn_slot()
1659 		clear_bit(slot_nr, area->bitmap);  in xol_free_insn_slot()
1660 		atomic_dec(&area->slot_count);  in xol_free_insn_slot()
1662 		if (waitqueue_active(&area->wq))  in xol_free_insn_slot()
1663 			wake_up(&area->wq);  in xol_free_insn_slot()
1803 	struct xol_area *area;  in uprobe_copy_process()  local
1817 	area = mm->uprobes_state.xol_area;  in uprobe_copy_process()
1818 	if (!area)  in uprobe_copy_process()
1824 	t->utask->dup_xol_addr = area->vaddr;  in uprobe_copy_process()
1837 	struct xol_area *area;  in get_trampoline_vaddr()  local
1841 	area = READ_ONCE(current->mm->uprobes_state.xol_area); /* ^^^ */  in get_trampoline_vaddr()
1842 	if (area)  in get_trampoline_vaddr()
1843 		trampoline_vaddr = area->vaddr;  in get_trampoline_vaddr()