6  *  Swap reorganised 29.12.95, Stephen Tweedie
16 #include <linux/swap.h>
57  * Some modules use swappable objects and may try to swap them out under
59  * check to see if any swap space is available.
66 static const char Bad_file[] = "Bad swap file entry ";
67 static const char Unused_file[] = "Unused swap file entry ";
68 static const char Bad_offset[] = "Bad swap offset entry ";
69 static const char Unused_offset[] = "Unused swap offset entry ";
116 /* Reclaim the swap entry anyway if possible */
119  * Reclaim the swap entry if there are no more mappings of the
123 /* Reclaim the swap entry if swap is getting full*/
126 /* returns 1 if swap entry is freed */
168  * swapon tell device that all the old swap contents can be discarded,
169  * to allow the swap device to optimize its wear-levelling.
178 	/* Do not discard the swap header page! */  in discard_swap()
238  * swap allocation tell device that a cluster of swap can now be discarded,
239  * to allow the swap device to optimize its wear-levelling.
459 	 * taken by scan_swap_map(), mark the swap entries bad (occupied). It  in swap_cluster_schedule_discard()
531 	 * If the swap is discardable, prepare discard the cluster  in free_cluster()
609  * Try to get a swap entry from current cpu's swap entry pool (a cluster). This
755 	 * Cross the swap address space size aligned trunk, choose  in set_cluster_next()
756 	 * another trunk randomly to avoid lock contention on swap  in set_cluster_next()
761 		/* No free swap slots available */  in set_cluster_next()
785 	 * We try to cluster swap pages by allocating them sequentially  in scan_swap_map_slots()
786 	 * in swap.  Once we've allocated SWAPFILE_CLUSTER pages this  in scan_swap_map_slots()
788 	 * a new cluster.  This prevents us from scattering swap pages  in scan_swap_map_slots()
789 	 * all over the entire swap partition, so that we reduce  in scan_swap_map_slots()
790 	 * overall disk seek times between swap pages.  -- sct  in scan_swap_map_slots()
792 	 * And we let swap pages go all over an SSD partition.  Hugh  in scan_swap_map_slots()
798 	 * cluster and swap cache.  For HDD, sequential access is more  in scan_swap_map_slots()
821 		 * start of partition, to minimize the span of allocated swap.  in scan_swap_map_slots()
869 	/* reuse swap entry of cache-only swap if not busy. */  in scan_swap_map_slots()
998 	 * page swap is disabled.  Warn and fail the allocation.  in swap_alloc_cluster()
1149 		/* This is called for allocating swap entry, not cache */  in get_swap_page_of_type()
1276  * after freeing a swap entry. Therefore, immediately after
1277  * __swap_entry_free(), the swap info might become stale and should not
1280  * Check whether swap entry is valid in the swap device.  If so,
1281  * return pointer to swap_info_struct, and keep the swap entry valid
1282  * via preventing the swap device from being swapoff, until
1308  * changing partly because the specified swap entry may be for another
1309  * swap device which has been swapoff.  And in do_swap_page(), after
1310  * the page is read from the swap device, the PTE is verified not
1311  * changed with the page table locked to check whether the swap device
1376  * Caller has made sure that the swap device corresponding to entry
1389  * Called after dropping swapcache to decrease refcnt to swap entries.
1474 	 * Sort swap entries by swap device, so each lock is only taken once.  in swapcache_free_entries()
1492  * This does not give an exact answer when swap count is continued,
1542  * This does not give an exact answer when swap count is continued,
1711  * to it.  And as a side-effect, free up its swap: because the old content
1732 	     /* The remaining swap count will be freed soon */  in reuse_swap_page()
1756  * If swap is getting full, or if there are no more mappings of this page,
1757  * then try_to_free_swap is called to free its swap space.
1774 	 * hibernation is allocating its own swap pages for the image,  in try_to_free_swap()
1776 	 * the swap from a page which has already been recorded in the  in try_to_free_swap()
1777 	 * image as a clean swapcache page, and then reuse its swap for  in try_to_free_swap()
1780 	 * later read back in from swap, now with the wrong data.  in try_to_free_swap()
1795  * Free the swap entry like above, but also try to
1825  * Find the swap type that corresponds to given device (if any).
1828  * from 0, in which the swap header is expected to be located.
1879  * corresponding to given index in swap_info (swap type).
1892  * Return either the total number of swap pages of given type, or the number
1924  * No need to decide whether this PTE shares the swap entry with others,
2254 		 * swap cache just before we acquired the page lock. The page  in try_to_unuse()
2255 		 * might even be back in swap cache on another swap area. But  in try_to_unuse()
2274 	 * Lets check again to see if there are still swap entries in the map.  in try_to_unuse()
2276 	 * Under global memory pressure, swap entries can be reinserted back  in try_to_unuse()
2280 	 * that mm is likely to be freeing swap from exit_mmap(), which proceeds  in try_to_unuse()
2282 	 * been preempted after get_swap_page(), temporarily hiding that swap.  in try_to_unuse()
2295  * After a successful try_to_unuse, if no swap is now in use, we know
2316  * corresponds to page offset for the specified swap entry.
2335  * Returns the page offset into bdev for the specified page's swap entry.
2415  * A `swap extent' is a simple thing which maps a contiguous range of pages
2416  * onto a contiguous range of disk blocks.  An ordered list of swap extents
2422  * swap files identically.
2424  * Whether the swapdev is an S_ISREG file or an S_ISBLK blockdev, the swap
2434  * For all swap devices we set S_SWAPFILE across the life of the swapon.  This
2435  * prevents users from writing to the swap device, which will corrupt memory.
2437  * The amount of disk space which a single swap extent represents varies.
2496 	 * low-to-high, while swap ordering is high-to-low  in setup_swap_info()
2527 	 * which allocates swap pages from the highest available priority  in _enable_swap_info()
2657 		/* re-insert swap space back into swap_list */  in SYSCALL_DEFINE1()
2667 	p->flags &= ~SWP_VALID;		/* mark swap device as invalid */  in SYSCALL_DEFINE1()
2671 	 * wait for swap operations protected by get/put_swap_device()  in SYSCALL_DEFINE1()
2722 	/* Destroy swap account information */  in SYSCALL_DEFINE1()
2775 static void *swap_start(struct seq_file *swap, loff_t *pos)  in swap_start()  argument
2796 static void *swap_next(struct seq_file *swap, void *v, loff_t *pos)  in swap_next()  argument
2816 static void swap_stop(struct seq_file *swap, void *v)  in swap_stop()  argument
2821 static int swap_show(struct seq_file *swap, void *v)  in swap_show()  argument
2829 		seq_puts(swap,"Filename\t\t\t\tType\t\tSize\t\tUsed\t\tPriority\n");  in swap_show()
2837 	len = seq_file_path(swap, file, " \t\n\\");  in swap_show()
2838 	seq_printf(swap, "%*s%s\t%u\t%s%u\t%s%d\n",  in swap_show()
2980  * Find out how many pages are allowed for a single swap device. There
2982  * 1) the number of bits for the swap offset in the swp_entry_t type, and
2983  * 2) the number of bits in the swap pte, as defined by the different
2986  * In order to find the largest possible bit mask, a swap entry with
2987  * swap type 0 and swap offset ~0UL is created, encoded to a swap pte,
2988  * decoded to a swp_entry_t again, and finally the swap offset is
2993  * of a swap pte.
3017 		pr_err("Unable to find swap-space signature\n");  in read_swap_header()
3021 	/* swap partition endianess hack... */  in read_swap_header()
3031 	/* Check the swap header's sub-version */  in read_swap_header()
3033 		pr_warn("Unable to handle swap header version %d\n",  in read_swap_header()
3045 		pr_warn("Empty swap-file\n");  in read_swap_header()
3049 		pr_warn("Truncating oversized swap area, only using %luk out of %luk\n",  in read_swap_header()
3065 		pr_warn("Swap area shorter than signature indicates\n");  in read_swap_header()
3136 		pr_warn("Empty swap-file\n");  in setup_swap_map_and_extents()
3165  * Helper to sys_swapon determining if a given swap
3240 	 * Read the swap header.  in SYSCALL_DEFINE2()
3259 	/* OK, set up the swap map and apply the bad block list */  in SYSCALL_DEFINE2()
3335 		 * When discard is enabled for swap with no particular  in SYSCALL_DEFINE2()
3336 		 * policy flagged, we set all swap discard flags here in  in SYSCALL_DEFINE2()
3346 		 * perform discards for released swap page-clusters.  in SYSCALL_DEFINE2()
3369 	 * swap device.  in SYSCALL_DEFINE2()
3385 	pr_info("Adding %uk swap on %s.  Priority:%d extents:%d across:%lluk %s%s%s%s%s\n",  in SYSCALL_DEFINE2()
3481  * Verify that a swap entry is valid and increment its swap map count.
3487  * - swap-cache reference is requested but there is already one. -> EEXIST
3488  * - swap-cache reference is requested but the entry is not used. -> ENOENT
3489  * - swap-mapped reference requested but needs continued swap count. -> ENOMEM
3510 	 * swapin_readahead() doesn't check if a swap entry is valid, so the  in __swap_duplicate()
3511 	 * swap entry could be SWAP_MAP_BAD. Check here with lock held.  in __swap_duplicate()
3543 		err = -ENOENT;			/* unused swap entry */  in __swap_duplicate()
3556  * Help swapoff by noting that swap entry belongs to shmem/tmpfs
3565  * Increase reference count of swap entry by 1.
3581  * @entry: swap entry for which we allocate swap cache.
3583  * Called when allocating swap cache for existing swap entry,
3585  * -EEXIST means there is a swap cache.
3628 	swp_entry_t swap = { .val = page_private(page) };  in __page_file_index()  local
3629 	return swp_offset(swap);  in __page_file_index()
3634  * add_swap_count_continuation - called when a swap count is duplicated
3637  * (for that entry and for its neighbouring PAGE_SIZE swap entries).  Called
3669 		 * __swap_duplicate(): the swap device may be swapoff  in add_swap_count_continuation()
3683 		 * The higher the swap count, the more likely it is that tasks  in add_swap_count_continuation()
3684 		 * will race to add swap count continuation: we need to avoid  in add_swap_count_continuation()
3757  * Called while __swap_duplicate() or swap_entry_free() holds swap or cluster
3875 	 * We've already scheduled a throttle, avoid taking the global swap  in cgroup_throttle_swaprate()
3900 		pr_emerg("Not enough memory for swap heads, swap is disabled\n");  in swapfile_init()