Lines Matching refs:mm
490 struct vm_area_struct *vm_area_alloc(struct mm_struct *mm) in vm_area_alloc() argument
498 vma_init(vma, mm); in vm_area_alloc()
640 static void dup_mm_exe_file(struct mm_struct *mm, struct mm_struct *oldmm) in dup_mm_exe_file() argument
645 RCU_INIT_POINTER(mm->exe_file, exe_file); in dup_mm_exe_file()
655 static __latent_entropy int dup_mmap(struct mm_struct *mm, in dup_mmap() argument
662 MA_STATE(mas, &mm->mm_mt, 0, 0); in dup_mmap()
670 uprobe_dup_mmap(oldmm, mm); in dup_mmap()
674 mmap_write_lock_nested(mm, SINGLE_DEPTH_NESTING); in dup_mmap()
677 dup_mm_exe_file(mm, oldmm); in dup_mmap()
679 mm->total_vm = oldmm->total_vm; in dup_mmap()
680 mm->data_vm = oldmm->data_vm; in dup_mmap()
681 mm->exec_vm = oldmm->exec_vm; in dup_mmap()
682 mm->stack_vm = oldmm->stack_vm; in dup_mmap()
684 retval = ksm_fork(mm, oldmm); in dup_mmap()
687 khugepaged_fork(mm, oldmm); in dup_mmap()
690 retval = __mt_dup(&oldmm->mm_mt, &mm->mm_mt, GFP_KERNEL); in dup_mmap()
706 vm_stat_account(mm, mpnt->vm_flags, -vma_pages(mpnt)); in dup_mmap()
731 tmp->vm_mm = mm; in dup_mmap()
775 mm->map_count++; in dup_mmap()
788 retval = arch_dup_mmap(oldmm, mm); in dup_mmap()
805 mmap_write_unlock(mm); in dup_mmap()
823 static inline int mm_alloc_pgd(struct mm_struct *mm) in mm_alloc_pgd() argument
825 mm->pgd = pgd_alloc(mm); in mm_alloc_pgd()
826 if (unlikely(!mm->pgd)) in mm_alloc_pgd()
831 static inline void mm_free_pgd(struct mm_struct *mm) in mm_free_pgd() argument
833 pgd_free(mm, mm->pgd); in mm_free_pgd()
836 static int dup_mmap(struct mm_struct *mm, struct mm_struct *oldmm) in dup_mmap() argument
839 dup_mm_exe_file(mm, oldmm); in dup_mmap()
843 #define mm_alloc_pgd(mm) (0) argument
844 #define mm_free_pgd(mm) argument
847 static void check_mm(struct mm_struct *mm) in check_mm() argument
855 long x = atomic_long_read(&mm->rss_stat.count[i]); in check_mm()
859 mm, resident_page_types[i], x); in check_mm()
862 if (mm_pgtables_bytes(mm)) in check_mm()
864 mm_pgtables_bytes(mm)); in check_mm()
867 VM_BUG_ON_MM(mm->pmd_huge_pte, mm); in check_mm()
872 #define free_mm(mm) (kmem_cache_free(mm_cachep, (mm))) argument
879 void __mmdrop(struct mm_struct *mm) in __mmdrop() argument
881 BUG_ON(mm == &init_mm); in __mmdrop()
882 WARN_ON_ONCE(mm == current->mm); in __mmdrop()
883 WARN_ON_ONCE(mm == current->active_mm); in __mmdrop()
884 mm_free_pgd(mm); in __mmdrop()
885 destroy_context(mm); in __mmdrop()
886 mmu_notifier_subscriptions_destroy(mm); in __mmdrop()
887 check_mm(mm); in __mmdrop()
888 put_user_ns(mm->user_ns); in __mmdrop()
889 mm_pasid_drop(mm); in __mmdrop()
890 free_mm(mm); in __mmdrop()
896 struct mm_struct *mm; in mmdrop_async_fn() local
898 mm = container_of(work, struct mm_struct, async_put_work); in mmdrop_async_fn()
899 __mmdrop(mm); in mmdrop_async_fn()
902 static void mmdrop_async(struct mm_struct *mm) in mmdrop_async() argument
904 if (unlikely(atomic_dec_and_test(&mm->mm_count))) { in mmdrop_async()
905 INIT_WORK(&mm->async_put_work, mmdrop_async_fn); in mmdrop_async()
906 schedule_work(&mm->async_put_work); in mmdrop_async()
1186 static void mm_init_aio(struct mm_struct *mm) in mm_init_aio() argument
1189 spin_lock_init(&mm->ioctx_lock); in mm_init_aio()
1190 mm->ioctx_table = NULL; in mm_init_aio()
1194 static __always_inline void mm_clear_owner(struct mm_struct *mm, in mm_clear_owner() argument
1198 if (mm->owner == p) in mm_clear_owner()
1199 WRITE_ONCE(mm->owner, NULL); in mm_clear_owner()
1203 static void mm_init_owner(struct mm_struct *mm, struct task_struct *p) in mm_init_owner() argument
1206 mm->owner = p; in mm_init_owner()
1210 static void mm_init_uprobes_state(struct mm_struct *mm) in mm_init_uprobes_state() argument
1213 mm->uprobes_state.xol_area = NULL; in mm_init_uprobes_state()
1217 static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p, in mm_init() argument
1220 mt_init_flags(&mm->mm_mt, MM_MT_FLAGS); in mm_init()
1221 mt_set_external_lock(&mm->mm_mt, &mm->mmap_lock); in mm_init()
1222 atomic_set(&mm->mm_users, 1); in mm_init()
1223 atomic_set(&mm->mm_count, 1); in mm_init()
1224 seqcount_init(&mm->write_protect_seq); in mm_init()
1225 mmap_init_lock(mm); in mm_init()
1226 INIT_LIST_HEAD(&mm->mmlist); in mm_init()
1228 mm->mm_lock_seq = 0; in mm_init()
1230 mm_pgtables_bytes_init(mm); in mm_init()
1231 mm->map_count = 0; in mm_init()
1232 mm->locked_vm = 0; in mm_init()
1233 atomic64_set(&mm->pinned_vm, 0); in mm_init()
1234 memset(&mm->rss_stat, 0, sizeof(mm->rss_stat)); in mm_init()
1235 spin_lock_init(&mm->page_table_lock); in mm_init()
1236 spin_lock_init(&mm->arg_lock); in mm_init()
1237 mm_init_cpumask(mm); in mm_init()
1238 mm_init_aio(mm); in mm_init()
1239 mm_init_owner(mm, p); in mm_init()
1240 mm_pasid_init(mm); in mm_init()
1241 RCU_INIT_POINTER(mm->exe_file, NULL); in mm_init()
1242 mmu_notifier_subscriptions_init(mm); in mm_init()
1243 init_tlb_flush_pending(mm); in mm_init()
1245 mm->pmd_huge_pte = NULL; in mm_init()
1247 mm_init_uprobes_state(mm); in mm_init()
1248 hugetlb_count_init(mm); in mm_init()
1250 if (current->mm) { in mm_init()
1251 mm->flags = current->mm->flags & MMF_INIT_MASK; in mm_init()
1252 mm->def_flags = current->mm->def_flags & VM_INIT_DEF_MASK; in mm_init()
1254 mm->flags = default_dump_filter; in mm_init()
1255 mm->def_flags = 0; in mm_init()
1258 if (mm_alloc_pgd(mm)) in mm_init()
1261 if (init_new_context(p, mm)) in mm_init()
1264 mm->user_ns = get_user_ns(user_ns); in mm_init()
1265 lru_gen_init_mm(mm); in mm_init()
1266 return mm; in mm_init()
1269 mm_free_pgd(mm); in mm_init()
1271 free_mm(mm); in mm_init()
1280 struct mm_struct *mm; in mm_alloc() local
1282 mm = allocate_mm(); in mm_alloc()
1283 if (!mm) in mm_alloc()
1286 memset(mm, 0, sizeof(*mm)); in mm_alloc()
1287 return mm_init(mm, current, current_user_ns()); in mm_alloc()
1290 static inline void __mmput(struct mm_struct *mm) in __mmput() argument
1292 VM_BUG_ON(atomic_read(&mm->mm_users)); in __mmput()
1294 uprobe_clear_state(mm); in __mmput()
1295 exit_aio(mm); in __mmput()
1296 ksm_exit(mm); in __mmput()
1297 khugepaged_exit(mm); /* must run before exit_mmap */ in __mmput()
1298 exit_mmap(mm); in __mmput()
1299 mm_put_huge_zero_page(mm); in __mmput()
1300 set_mm_exe_file(mm, NULL); in __mmput()
1301 if (!list_empty(&mm->mmlist)) { in __mmput()
1303 list_del(&mm->mmlist); in __mmput()
1306 if (mm->binfmt) in __mmput()
1307 module_put(mm->binfmt->module); in __mmput()
1308 lru_gen_del_mm(mm); in __mmput()
1309 mmdrop(mm); in __mmput()
1315 void mmput(struct mm_struct *mm) in mmput() argument
1319 if (atomic_dec_and_test(&mm->mm_users)) { in mmput()
1320 trace_android_vh_mmput(mm); in mmput()
1321 __mmput(mm); in mmput()
1329 struct mm_struct *mm = container_of(work, struct mm_struct, in mmput_async_fn() local
1332 __mmput(mm); in mmput_async_fn()
1335 void mmput_async(struct mm_struct *mm) in mmput_async() argument
1337 if (atomic_dec_and_test(&mm->mm_users)) { in mmput_async()
1338 INIT_WORK(&mm->async_put_work, mmput_async_fn); in mmput_async()
1339 schedule_work(&mm->async_put_work); in mmput_async()
1356 int set_mm_exe_file(struct mm_struct *mm, struct file *new_exe_file) in set_mm_exe_file() argument
1365 old_exe_file = rcu_dereference_raw(mm->exe_file); in set_mm_exe_file()
1376 rcu_assign_pointer(mm->exe_file, new_exe_file); in set_mm_exe_file()
1393 int replace_mm_exe_file(struct mm_struct *mm, struct file *new_exe_file) in replace_mm_exe_file() argument
1400 old_exe_file = get_mm_exe_file(mm); in replace_mm_exe_file()
1402 VMA_ITERATOR(vmi, mm, 0); in replace_mm_exe_file()
1403 mmap_read_lock(mm); in replace_mm_exe_file()
1413 mmap_read_unlock(mm); in replace_mm_exe_file()
1425 old_exe_file = xchg(&mm->exe_file, new_exe_file); in replace_mm_exe_file()
1431 mmap_read_lock(mm); in replace_mm_exe_file()
1434 mmap_read_unlock(mm); in replace_mm_exe_file()
1445 struct file *get_mm_exe_file(struct mm_struct *mm) in get_mm_exe_file() argument
1450 exe_file = rcu_dereference(mm->exe_file); in get_mm_exe_file()
1467 struct mm_struct *mm; in get_task_exe_file() local
1470 mm = task->mm; in get_task_exe_file()
1471 if (mm) { in get_task_exe_file()
1473 exe_file = get_mm_exe_file(mm); in get_task_exe_file()
1490 struct mm_struct *mm; in get_task_mm() local
1493 mm = task->mm; in get_task_mm()
1494 if (mm) { in get_task_mm()
1496 mm = NULL; in get_task_mm()
1498 mmget(mm); in get_task_mm()
1501 return mm; in get_task_mm()
1507 struct mm_struct *mm; in mm_access() local
1514 mm = get_task_mm(task); in mm_access()
1515 if (mm && mm != current->mm && in mm_access()
1517 mmput(mm); in mm_access()
1518 mm = ERR_PTR(-EACCES); in mm_access()
1522 return mm; in mm_access()
1571 static void mm_release(struct task_struct *tsk, struct mm_struct *mm) in mm_release() argument
1576 deactivate_mm(tsk, mm); in mm_release()
1584 if (atomic_read(&mm->mm_users) > 1) { in mm_release()
1604 void exit_mm_release(struct task_struct *tsk, struct mm_struct *mm) in exit_mm_release() argument
1607 mm_release(tsk, mm); in exit_mm_release()
1610 void exec_mm_release(struct task_struct *tsk, struct mm_struct *mm) in exec_mm_release() argument
1613 mm_release(tsk, mm); in exec_mm_release()
1629 struct mm_struct *mm; in dup_mm() local
1632 mm = allocate_mm(); in dup_mm()
1633 if (!mm) in dup_mm()
1636 memcpy(mm, oldmm, sizeof(*mm)); in dup_mm()
1638 if (!mm_init(mm, tsk, mm->user_ns)) in dup_mm()
1641 err = dup_mmap(mm, oldmm); in dup_mm()
1645 mm->hiwater_rss = get_mm_rss(mm); in dup_mm()
1646 mm->hiwater_vm = mm->total_vm; in dup_mm()
1648 if (mm->binfmt && !try_module_get(mm->binfmt->module)) in dup_mm()
1651 return mm; in dup_mm()
1655 mm->binfmt = NULL; in dup_mm()
1656 mm_init_owner(mm, NULL); in dup_mm()
1657 mmput(mm); in dup_mm()
1665 struct mm_struct *mm, *oldmm; in copy_mm() local
1674 tsk->mm = NULL; in copy_mm()
1682 oldmm = current->mm; in copy_mm()
1688 mm = oldmm; in copy_mm()
1690 mm = dup_mm(tsk, current->mm); in copy_mm()
1691 if (!mm) in copy_mm()
1695 tsk->mm = mm; in copy_mm()
1696 tsk->active_mm = mm; in copy_mm()
2067 if (!tsk->mm) in copy_oom_score_adj()
2076 set_bit(MMF_MULTIPROCESS, &tsk->mm->flags); in copy_oom_score_adj()
2636 if (p->mm) { in copy_process()
2637 mm_clear_owner(p->mm, p); in copy_process()
2638 mmput(p->mm); in copy_process()
2815 lru_gen_add_mm(p->mm); in kernel_clone()