Lines Matching refs:p
549 static void inc_cluster_info_page(struct swap_info_struct *p, in inc_cluster_info_page() argument
557 alloc_cluster(p, idx); in inc_cluster_info_page()
569 static void dec_cluster_info_page(struct swap_info_struct *p, in dec_cluster_info_page() argument
582 free_cluster(p, idx); in dec_cluster_info_page()
667 static void __del_from_avail_list(struct swap_info_struct *p) in __del_from_avail_list() argument
671 assert_spin_locked(&p->lock); in __del_from_avail_list()
673 plist_del(&p->avail_lists[nid], &swap_avail_heads[nid]); in __del_from_avail_list()
676 static void del_from_avail_list(struct swap_info_struct *p) in del_from_avail_list() argument
680 trace_android_vh_del_from_avail_list(p, &skip); in del_from_avail_list()
685 __del_from_avail_list(p); in del_from_avail_list()
706 static void add_to_avail_list(struct swap_info_struct *p) in add_to_avail_list() argument
711 trace_android_vh_add_to_avail_list(p, &skip); in add_to_avail_list()
717 WARN_ON(!plist_node_empty(&p->avail_lists[nid])); in add_to_avail_list()
718 plist_add(&p->avail_lists[nid], &swap_avail_heads[nid]); in add_to_avail_list()
1184 struct swap_info_struct *p; in __swap_info_get() local
1189 p = swp_swap_info(entry); in __swap_info_get()
1190 if (!p) in __swap_info_get()
1192 if (data_race(!(p->flags & SWP_USED))) in __swap_info_get()
1195 if (offset >= p->max) in __swap_info_get()
1197 return p; in __swap_info_get()
1213 struct swap_info_struct *p; in _swap_info_get() local
1215 p = __swap_info_get(entry); in _swap_info_get()
1216 if (!p) in _swap_info_get()
1218 if (data_race(!p->swap_map[swp_offset(entry)])) in _swap_info_get()
1220 return p; in _swap_info_get()
1230 struct swap_info_struct *p; in swap_info_get() local
1232 p = _swap_info_get(entry); in swap_info_get()
1233 if (p) in swap_info_get()
1234 spin_lock(&p->lock); in swap_info_get()
1235 return p; in swap_info_get()
1241 struct swap_info_struct *p; in swap_info_get_cont() local
1243 p = _swap_info_get(entry); in swap_info_get_cont()
1245 if (p != q) { in swap_info_get_cont()
1248 if (p != NULL) in swap_info_get_cont()
1249 spin_lock(&p->lock); in swap_info_get_cont()
1251 return p; in swap_info_get_cont()
1254 static unsigned char __swap_entry_free_locked(struct swap_info_struct *p, in __swap_entry_free_locked() argument
1261 count = p->swap_map[offset]; in __swap_entry_free_locked()
1277 if (swap_count_continued(p, offset, count)) in __swap_entry_free_locked()
1287 WRITE_ONCE(p->swap_map[offset], usage); in __swap_entry_free_locked()
1289 WRITE_ONCE(p->swap_map[offset], SWAP_HAS_CACHE); in __swap_entry_free_locked()
1357 static unsigned char __swap_entry_free(struct swap_info_struct *p, in __swap_entry_free() argument
1364 ci = lock_cluster_or_swap_info(p, offset); in __swap_entry_free()
1365 usage = __swap_entry_free_locked(p, offset, 1); in __swap_entry_free()
1366 unlock_cluster_or_swap_info(p, ci); in __swap_entry_free()
1373 static void swap_entry_free(struct swap_info_struct *p, swp_entry_t entry) in swap_entry_free() argument
1379 ci = lock_cluster(p, offset); in swap_entry_free()
1380 count = p->swap_map[offset]; in swap_entry_free()
1382 p->swap_map[offset] = 0; in swap_entry_free()
1383 dec_cluster_info_page(p, p->cluster_info, offset); in swap_entry_free()
1387 swap_range_free(p, offset, 1); in swap_entry_free()
1396 struct swap_info_struct *p; in swap_free() local
1398 p = _swap_info_get(entry); in swap_free()
1399 if (p) in swap_free()
1400 __swap_entry_free(p, entry); in swap_free()
1479 struct swap_info_struct *p, *prev; in swapcache_free_entries() local
1486 p = NULL; in swapcache_free_entries()
1496 p = swap_info_get_cont(entries[i], prev); in swapcache_free_entries()
1497 if (p) in swapcache_free_entries()
1498 swap_entry_free(p, entries[i]); in swapcache_free_entries()
1499 prev = p; in swapcache_free_entries()
1501 if (p) in swapcache_free_entries()
1502 spin_unlock(&p->lock); in swapcache_free_entries()
1514 struct swap_info_struct *p; in page_swapcount() local
1520 p = _swap_info_get(entry); in page_swapcount()
1521 if (p) { in page_swapcount()
1523 ci = lock_cluster_or_swap_info(p, offset); in page_swapcount()
1524 count = swap_count(p->swap_map[offset]); in page_swapcount()
1525 unlock_cluster_or_swap_info(p, ci); in page_swapcount()
1581 struct swap_info_struct *p; in swp_swapcount() local
1587 p = _swap_info_get(entry); in swp_swapcount()
1588 if (!p) in swp_swapcount()
1593 ci = lock_cluster_or_swap_info(p, offset); in swp_swapcount()
1595 count = swap_count(p->swap_map[offset]); in swp_swapcount()
1602 page = vmalloc_to_page(p->swap_map + offset); in swp_swapcount()
1616 unlock_cluster_or_swap_info(p, ci); in swp_swapcount()
1756 struct swap_info_struct *p; in reuse_swap_page() local
1759 p = swap_info_get(entry); in reuse_swap_page()
1760 if (p->flags & SWP_STABLE_WRITES) { in reuse_swap_page()
1761 spin_unlock(&p->lock); in reuse_swap_page()
1764 spin_unlock(&p->lock); in reuse_swap_page()
1816 struct swap_info_struct *p; in free_swap_and_cache() local
1822 p = _swap_info_get(entry); in free_swap_and_cache()
1823 if (p) { in free_swap_and_cache()
1824 count = __swap_entry_free(p, entry); in free_swap_and_cache()
1826 !swap_page_trans_huge_swapped(p, entry)) in free_swap_and_cache()
1827 __try_to_reclaim_swap(p, swp_offset(entry), in free_swap_and_cache()
1830 return p != NULL; in free_swap_and_cache()
2203 struct list_head *p; in try_to_unuse() local
2225 p = &init_mm.mmlist; in try_to_unuse()
2228 (p = p->next) != &init_mm.mmlist) { in try_to_unuse()
2230 mm = list_entry(p, struct mm_struct, mmlist); in try_to_unuse()
2315 struct list_head *p, *next; in drain_mmlist() local
2322 list_for_each_safe(p, next, &init_mm.mmlist) in drain_mmlist()
2323 list_del_init(p); in drain_mmlist()
2485 static int swap_node(struct swap_info_struct *p) in swap_node() argument
2489 if (p->bdev) in swap_node()
2490 bdev = p->bdev; in swap_node()
2492 bdev = p->swap_file->f_inode->i_sb->s_bdev; in swap_node()
2497 static void setup_swap_info(struct swap_info_struct *p, int prio, in setup_swap_info() argument
2504 p->prio = prio; in setup_swap_info()
2506 p->prio = --least_priority; in setup_swap_info()
2511 p->list.prio = -p->prio; in setup_swap_info()
2513 if (p->prio >= 0) in setup_swap_info()
2514 p->avail_lists[i].prio = -p->prio; in setup_swap_info()
2516 if (swap_node(p) == i) in setup_swap_info()
2517 p->avail_lists[i].prio = 1; in setup_swap_info()
2519 p->avail_lists[i].prio = -p->prio; in setup_swap_info()
2522 p->swap_map = swap_map; in setup_swap_info()
2523 p->cluster_info = cluster_info; in setup_swap_info()
2526 static void _enable_swap_info(struct swap_info_struct *p) in _enable_swap_info() argument
2530 p->flags |= SWP_WRITEOK | SWP_VALID; in _enable_swap_info()
2531 trace_android_vh_account_swap_pages(p, &skip); in _enable_swap_info()
2533 atomic_long_add(p->pages, &nr_swap_pages); in _enable_swap_info()
2534 total_swap_pages += p->pages; in _enable_swap_info()
2547 plist_add(&p->list, &swap_active_head); in _enable_swap_info()
2548 add_to_avail_list(p); in _enable_swap_info()
2551 static void enable_swap_info(struct swap_info_struct *p, int prio, in enable_swap_info() argument
2556 frontswap_init(p->type, frontswap_map); in enable_swap_info()
2558 spin_lock(&p->lock); in enable_swap_info()
2559 setup_swap_info(p, prio, swap_map, cluster_info); in enable_swap_info()
2560 spin_unlock(&p->lock); in enable_swap_info()
2568 spin_lock(&p->lock); in enable_swap_info()
2569 _enable_swap_info(p); in enable_swap_info()
2570 spin_unlock(&p->lock); in enable_swap_info()
2574 static void reinsert_swap_info(struct swap_info_struct *p) in reinsert_swap_info() argument
2577 spin_lock(&p->lock); in reinsert_swap_info()
2578 setup_swap_info(p, p->prio, p->swap_map, p->cluster_info); in reinsert_swap_info()
2579 _enable_swap_info(p); in reinsert_swap_info()
2580 spin_unlock(&p->lock); in reinsert_swap_info()
2597 struct swap_info_struct *p = NULL; in SYSCALL_DEFINE1() local
2625 plist_for_each_entry(p, &swap_active_head, list) { in SYSCALL_DEFINE1()
2626 if (p->flags & SWP_WRITEOK) { in SYSCALL_DEFINE1()
2627 if (p->swap_file->f_mapping == mapping) { in SYSCALL_DEFINE1()
2638 if (!security_vm_enough_memory_mm(current->mm, p->pages)) in SYSCALL_DEFINE1()
2639 vm_unacct_memory(p->pages); in SYSCALL_DEFINE1()
2645 spin_lock(&p->lock); in SYSCALL_DEFINE1()
2646 del_from_avail_list(p); in SYSCALL_DEFINE1()
2647 if (p->prio < 0) { in SYSCALL_DEFINE1()
2648 struct swap_info_struct *si = p; in SYSCALL_DEFINE1()
2661 plist_del(&p->list, &swap_active_head); in SYSCALL_DEFINE1()
2662 trace_android_vh_account_swap_pages(p, &skip); in SYSCALL_DEFINE1()
2664 atomic_long_sub(p->pages, &nr_swap_pages); in SYSCALL_DEFINE1()
2665 total_swap_pages -= p->pages; in SYSCALL_DEFINE1()
2667 p->flags &= ~SWP_WRITEOK; in SYSCALL_DEFINE1()
2668 spin_unlock(&p->lock); in SYSCALL_DEFINE1()
2674 err = try_to_unuse(p->type, false, 0); /* force unuse all pages */ in SYSCALL_DEFINE1()
2679 reinsert_swap_info(p); in SYSCALL_DEFINE1()
2687 spin_lock(&p->lock); in SYSCALL_DEFINE1()
2688 p->flags &= ~SWP_VALID; /* mark swap device as invalid */ in SYSCALL_DEFINE1()
2689 spin_unlock(&p->lock); in SYSCALL_DEFINE1()
2697 flush_work(&p->discard_work); in SYSCALL_DEFINE1()
2699 destroy_swap_extents(p); in SYSCALL_DEFINE1()
2700 if (p->flags & SWP_CONTINUED) in SYSCALL_DEFINE1()
2701 free_swap_count_continuations(p); in SYSCALL_DEFINE1()
2703 if (!p->bdev || !blk_queue_nonrot(bdev_get_queue(p->bdev))) in SYSCALL_DEFINE1()
2708 spin_lock(&p->lock); in SYSCALL_DEFINE1()
2712 p->highest_bit = 0; /* cuts scans short */ in SYSCALL_DEFINE1()
2713 while (p->flags >= SWP_SCANNING) { in SYSCALL_DEFINE1()
2714 spin_unlock(&p->lock); in SYSCALL_DEFINE1()
2718 spin_lock(&p->lock); in SYSCALL_DEFINE1()
2721 swap_file = p->swap_file; in SYSCALL_DEFINE1()
2722 old_block_size = p->old_block_size; in SYSCALL_DEFINE1()
2723 p->swap_file = NULL; in SYSCALL_DEFINE1()
2724 p->max = 0; in SYSCALL_DEFINE1()
2725 swap_map = p->swap_map; in SYSCALL_DEFINE1()
2726 p->swap_map = NULL; in SYSCALL_DEFINE1()
2727 cluster_info = p->cluster_info; in SYSCALL_DEFINE1()
2728 p->cluster_info = NULL; in SYSCALL_DEFINE1()
2729 frontswap_map = frontswap_map_get(p); in SYSCALL_DEFINE1()
2730 spin_unlock(&p->lock); in SYSCALL_DEFINE1()
2732 arch_swap_invalidate_area(p->type); in SYSCALL_DEFINE1()
2733 frontswap_invalidate_area(p->type); in SYSCALL_DEFINE1()
2734 frontswap_map_set(p, NULL); in SYSCALL_DEFINE1()
2736 free_percpu(p->percpu_cluster); in SYSCALL_DEFINE1()
2737 p->percpu_cluster = NULL; in SYSCALL_DEFINE1()
2738 free_percpu(p->cluster_next_cpu); in SYSCALL_DEFINE1()
2739 p->cluster_next_cpu = NULL; in SYSCALL_DEFINE1()
2744 swap_cgroup_swapoff(p->type); in SYSCALL_DEFINE1()
2745 exit_swap_address_space(p->type); in SYSCALL_DEFINE1()
2766 p->flags = 0; in SYSCALL_DEFINE1()
2918 struct swap_info_struct *p = NULL; in alloc_swap_info() local
2924 trace_android_rvh_alloc_si(&p, &skip); in alloc_swap_info()
2925 trace_android_vh_alloc_si(&p, &skip); in alloc_swap_info()
2927 p = kvzalloc(struct_size(p, avail_lists, nr_node_ids), GFP_KERNEL); in alloc_swap_info()
2928 if (!p) in alloc_swap_info()
2938 kvfree(p); in alloc_swap_info()
2942 p->type = type; in alloc_swap_info()
2943 WRITE_ONCE(swap_info[type], p); in alloc_swap_info()
2952 defer = p; in alloc_swap_info()
2953 p = swap_info[type]; in alloc_swap_info()
2959 p->swap_extent_root = RB_ROOT; in alloc_swap_info()
2960 plist_node_init(&p->list, 0); in alloc_swap_info()
2962 plist_node_init(&p->avail_lists[i], 0); in alloc_swap_info()
2963 p->flags = SWP_USED; in alloc_swap_info()
2966 spin_lock_init(&p->lock); in alloc_swap_info()
2967 spin_lock_init(&p->cont_lock); in alloc_swap_info()
2969 return p; in alloc_swap_info()
2972 static int claim_swapfile(struct swap_info_struct *p, struct inode *inode) in claim_swapfile() argument
2977 p->bdev = blkdev_get_by_dev(inode->i_rdev, in claim_swapfile()
2978 FMODE_READ | FMODE_WRITE | FMODE_EXCL, p); in claim_swapfile()
2979 if (IS_ERR(p->bdev)) { in claim_swapfile()
2980 error = PTR_ERR(p->bdev); in claim_swapfile()
2981 p->bdev = NULL; in claim_swapfile()
2984 p->old_block_size = block_size(p->bdev); in claim_swapfile()
2985 error = set_blocksize(p->bdev, PAGE_SIZE); in claim_swapfile()
2993 if (blk_queue_is_zoned(p->bdev->bd_disk->queue)) in claim_swapfile()
2995 p->flags |= SWP_BLKDEV; in claim_swapfile()
2997 p->bdev = inode->i_sb->s_bdev; in claim_swapfile()
3032 static unsigned long read_swap_header(struct swap_info_struct *p, in read_swap_header() argument
3063 p->lowest_bit = 1; in read_swap_header()
3064 p->cluster_next = 1; in read_swap_header()
3065 p->cluster_nr = 0; in read_swap_header()
3084 p->highest_bit = maxpages - 1; in read_swap_header()
3108 static int setup_swap_map_and_extents(struct swap_info_struct *p, in setup_swap_map_and_extents() argument
3119 unsigned long col = p->cluster_next / SWAPFILE_CLUSTER % SWAP_CLUSTER_COLS; in setup_swap_map_and_extents()
3124 cluster_list_init(&p->free_clusters); in setup_swap_map_and_extents()
3125 cluster_list_init(&p->discard_clusters); in setup_swap_map_and_extents()
3138 inc_cluster_info_page(p, cluster_info, page_nr); in setup_swap_map_and_extents()
3144 inc_cluster_info_page(p, cluster_info, i); in setup_swap_map_and_extents()
3152 inc_cluster_info_page(p, cluster_info, 0); in setup_swap_map_and_extents()
3153 p->max = maxpages; in setup_swap_map_and_extents()
3154 p->pages = nr_good_pages; in setup_swap_map_and_extents()
3155 nr_extents = setup_swap_extents(p, span); in setup_swap_map_and_extents()
3158 nr_good_pages = p->pages; in setup_swap_map_and_extents()
3182 cluster_list_add_tail(&p->free_clusters, cluster_info, in setup_swap_map_and_extents()
3205 struct swap_info_struct *p; in SYSCALL_DEFINE2() local
3231 p = alloc_swap_info(); in SYSCALL_DEFINE2()
3232 if (IS_ERR(p)) in SYSCALL_DEFINE2()
3233 return PTR_ERR(p); in SYSCALL_DEFINE2()
3235 INIT_WORK(&p->discard_work, swap_discard_work); in SYSCALL_DEFINE2()
3250 p->swap_file = swap_file; in SYSCALL_DEFINE2()
3254 error = claim_swapfile(p, inode); in SYSCALL_DEFINE2()
3278 maxpages = read_swap_header(p, swap_header, inode); in SYSCALL_DEFINE2()
3291 if (p->bdev && blk_queue_stable_writes(p->bdev->bd_disk->queue)) in SYSCALL_DEFINE2()
3292 p->flags |= SWP_STABLE_WRITES; in SYSCALL_DEFINE2()
3294 if (p->bdev && p->bdev->bd_disk->fops->rw_page) in SYSCALL_DEFINE2()
3295 p->flags |= SWP_SYNCHRONOUS_IO; in SYSCALL_DEFINE2()
3297 if (p->bdev && blk_queue_nonrot(bdev_get_queue(p->bdev))) { in SYSCALL_DEFINE2()
3301 p->flags |= SWP_SOLIDSTATE; in SYSCALL_DEFINE2()
3302 p->cluster_next_cpu = alloc_percpu(unsigned int); in SYSCALL_DEFINE2()
3303 if (!p->cluster_next_cpu) { in SYSCALL_DEFINE2()
3312 per_cpu(*p->cluster_next_cpu, cpu) = in SYSCALL_DEFINE2()
3313 1 + prandom_u32_max(p->highest_bit); in SYSCALL_DEFINE2()
3327 p->percpu_cluster = alloc_percpu(struct percpu_cluster); in SYSCALL_DEFINE2()
3328 if (!p->percpu_cluster) { in SYSCALL_DEFINE2()
3334 cluster = per_cpu_ptr(p->percpu_cluster, cpu); in SYSCALL_DEFINE2()
3342 error = swap_cgroup_swapon(p->type, maxpages); in SYSCALL_DEFINE2()
3346 nr_extents = setup_swap_map_and_extents(p, swap_header, swap_map, in SYSCALL_DEFINE2()
3358 if (p->bdev &&(swap_flags & SWAP_FLAG_DISCARD) && swap_discardable(p)) { in SYSCALL_DEFINE2()
3365 p->flags |= (SWP_DISCARDABLE | SWP_AREA_DISCARD | in SYSCALL_DEFINE2()
3375 p->flags &= ~SWP_PAGE_DISCARD; in SYSCALL_DEFINE2()
3377 p->flags &= ~SWP_AREA_DISCARD; in SYSCALL_DEFINE2()
3380 if (p->flags & SWP_AREA_DISCARD) { in SYSCALL_DEFINE2()
3381 int err = discard_swap(p); in SYSCALL_DEFINE2()
3384 p, err); in SYSCALL_DEFINE2()
3388 error = init_swap_address_space(p->type, maxpages); in SYSCALL_DEFINE2()
3410 enable_swap_info(p, prio, swap_map, cluster_info, frontswap_map); in SYSCALL_DEFINE2()
3412 trace_android_vh_init_swap_info_struct(p, swap_avail_heads); in SYSCALL_DEFINE2()
3414 p->pages<<(PAGE_SHIFT-10), name->name, p->prio, in SYSCALL_DEFINE2()
3416 (p->flags & SWP_SOLIDSTATE) ? "SS" : "", in SYSCALL_DEFINE2()
3417 (p->flags & SWP_DISCARDABLE) ? "D" : "", in SYSCALL_DEFINE2()
3418 (p->flags & SWP_AREA_DISCARD) ? "s" : "", in SYSCALL_DEFINE2()
3419 (p->flags & SWP_PAGE_DISCARD) ? "c" : "", in SYSCALL_DEFINE2()
3429 exit_swap_address_space(p->type); in SYSCALL_DEFINE2()
3433 free_percpu(p->percpu_cluster); in SYSCALL_DEFINE2()
3434 p->percpu_cluster = NULL; in SYSCALL_DEFINE2()
3435 free_percpu(p->cluster_next_cpu); in SYSCALL_DEFINE2()
3436 p->cluster_next_cpu = NULL; in SYSCALL_DEFINE2()
3437 if (inode && S_ISBLK(inode->i_mode) && p->bdev) { in SYSCALL_DEFINE2()
3438 set_blocksize(p->bdev, p->old_block_size); in SYSCALL_DEFINE2()
3439 blkdev_put(p->bdev, FMODE_READ | FMODE_WRITE | FMODE_EXCL); in SYSCALL_DEFINE2()
3442 destroy_swap_extents(p); in SYSCALL_DEFINE2()
3443 swap_cgroup_swapoff(p->type); in SYSCALL_DEFINE2()
3445 p->swap_file = NULL; in SYSCALL_DEFINE2()
3446 p->flags = 0; in SYSCALL_DEFINE2()
3502 struct swap_info_struct *p; in __swap_duplicate() local
3509 p = get_swap_device(entry); in __swap_duplicate()
3510 if (!p) in __swap_duplicate()
3514 ci = lock_cluster_or_swap_info(p, offset); in __swap_duplicate()
3516 count = p->swap_map[offset]; in __swap_duplicate()
3547 else if (swap_count_continued(p, offset, count)) in __swap_duplicate()
3554 WRITE_ONCE(p->swap_map[offset], count | has_cache); in __swap_duplicate()
3557 unlock_cluster_or_swap_info(p, ci); in __swap_duplicate()
3559 if (p) in __swap_duplicate()
3560 put_swap_device(p); in __swap_duplicate()