• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 /*
2  *  linux/mm/swapfile.c
3  *
4  *  Copyright (C) 1991, 1992, 1993, 1994  Linus Torvalds
5  *  Swap reorganised 29.12.95, Stephen Tweedie
6  */
7 
8 #include <linux/mm.h>
9 #include <linux/hugetlb.h>
10 #include <linux/mman.h>
11 #include <linux/slab.h>
12 #include <linux/kernel_stat.h>
13 #include <linux/swap.h>
14 #include <linux/vmalloc.h>
15 #include <linux/pagemap.h>
16 #include <linux/namei.h>
17 #include <linux/shmem_fs.h>
18 #include <linux/blkdev.h>
19 #include <linux/random.h>
20 #include <linux/writeback.h>
21 #include <linux/proc_fs.h>
22 #include <linux/seq_file.h>
23 #include <linux/init.h>
24 #include <linux/ksm.h>
25 #include <linux/rmap.h>
26 #include <linux/security.h>
27 #include <linux/backing-dev.h>
28 #include <linux/mutex.h>
29 #include <linux/capability.h>
30 #include <linux/syscalls.h>
31 #include <linux/memcontrol.h>
32 #include <linux/poll.h>
33 #include <linux/oom.h>
34 #include <linux/frontswap.h>
35 #include <linux/swapfile.h>
36 #include <linux/export.h>
37 
38 #include <asm/pgtable.h>
39 #include <asm/tlbflush.h>
40 #include <linux/swapops.h>
41 #include <linux/swap_cgroup.h>
42 
43 static bool swap_count_continued(struct swap_info_struct *, pgoff_t,
44 				 unsigned char);
45 static void free_swap_count_continuations(struct swap_info_struct *);
46 static sector_t map_swap_entry(swp_entry_t, struct block_device**);
47 
48 DEFINE_SPINLOCK(swap_lock);
49 static unsigned int nr_swapfiles;
50 atomic_long_t nr_swap_pages;
51 /* protected with swap_lock. reading in vm_swap_full() doesn't need lock */
52 long total_swap_pages;
53 static int least_priority;
54 
55 static const char Bad_file[] = "Bad swap file entry ";
56 static const char Unused_file[] = "Unused swap file entry ";
57 static const char Bad_offset[] = "Bad swap offset entry ";
58 static const char Unused_offset[] = "Unused swap offset entry ";
59 
60 /*
61  * all active swap_info_structs
62  * protected with swap_lock, and ordered by priority.
63  */
64 PLIST_HEAD(swap_active_head);
65 
66 /*
67  * all available (active, not full) swap_info_structs
68  * protected with swap_avail_lock, ordered by priority.
69  * This is used by get_swap_page() instead of swap_active_head
70  * because swap_active_head includes all swap_info_structs,
71  * but get_swap_page() doesn't need to look at full ones.
72  * This uses its own lock instead of swap_lock because when a
73  * swap_info_struct changes between not-full/full, it needs to
74  * add/remove itself to/from this list, but the swap_info_struct->lock
75  * is held and the locking order requires swap_lock to be taken
76  * before any swap_info_struct->lock.
77  */
78 static PLIST_HEAD(swap_avail_head);
79 static DEFINE_SPINLOCK(swap_avail_lock);
80 
81 struct swap_info_struct *swap_info[MAX_SWAPFILES];
82 
83 static DEFINE_MUTEX(swapon_mutex);
84 
85 static DECLARE_WAIT_QUEUE_HEAD(proc_poll_wait);
86 /* Activity counter to indicate that a swapon or swapoff has occurred */
87 static atomic_t proc_poll_event = ATOMIC_INIT(0);
88 
swap_count(unsigned char ent)89 static inline unsigned char swap_count(unsigned char ent)
90 {
91 	return ent & ~SWAP_HAS_CACHE;	/* may include SWAP_HAS_CONT flag */
92 }
93 
94 /* returns 1 if swap entry is freed */
95 static int
__try_to_reclaim_swap(struct swap_info_struct * si,unsigned long offset)96 __try_to_reclaim_swap(struct swap_info_struct *si, unsigned long offset)
97 {
98 	swp_entry_t entry = swp_entry(si->type, offset);
99 	struct page *page;
100 	int ret = 0;
101 
102 	page = find_get_page(swap_address_space(entry), entry.val);
103 	if (!page)
104 		return 0;
105 	/*
106 	 * This function is called from scan_swap_map() and it's called
107 	 * by vmscan.c at reclaiming pages. So, we hold a lock on a page, here.
108 	 * We have to use trylock for avoiding deadlock. This is a special
109 	 * case and you should use try_to_free_swap() with explicit lock_page()
110 	 * in usual operations.
111 	 */
112 	if (trylock_page(page)) {
113 		ret = try_to_free_swap(page);
114 		unlock_page(page);
115 	}
116 	page_cache_release(page);
117 	return ret;
118 }
119 
120 /*
121  * swapon tell device that all the old swap contents can be discarded,
122  * to allow the swap device to optimize its wear-levelling.
123  */
discard_swap(struct swap_info_struct * si)124 static int discard_swap(struct swap_info_struct *si)
125 {
126 	struct swap_extent *se;
127 	sector_t start_block;
128 	sector_t nr_blocks;
129 	int err = 0;
130 
131 	/* Do not discard the swap header page! */
132 	se = &si->first_swap_extent;
133 	start_block = (se->start_block + 1) << (PAGE_SHIFT - 9);
134 	nr_blocks = ((sector_t)se->nr_pages - 1) << (PAGE_SHIFT - 9);
135 	if (nr_blocks) {
136 		err = blkdev_issue_discard(si->bdev, start_block,
137 				nr_blocks, GFP_KERNEL, 0);
138 		if (err)
139 			return err;
140 		cond_resched();
141 	}
142 
143 	list_for_each_entry(se, &si->first_swap_extent.list, list) {
144 		start_block = se->start_block << (PAGE_SHIFT - 9);
145 		nr_blocks = (sector_t)se->nr_pages << (PAGE_SHIFT - 9);
146 
147 		err = blkdev_issue_discard(si->bdev, start_block,
148 				nr_blocks, GFP_KERNEL, 0);
149 		if (err)
150 			break;
151 
152 		cond_resched();
153 	}
154 	return err;		/* That will often be -EOPNOTSUPP */
155 }
156 
157 /*
158  * swap allocation tell device that a cluster of swap can now be discarded,
159  * to allow the swap device to optimize its wear-levelling.
160  */
discard_swap_cluster(struct swap_info_struct * si,pgoff_t start_page,pgoff_t nr_pages)161 static void discard_swap_cluster(struct swap_info_struct *si,
162 				 pgoff_t start_page, pgoff_t nr_pages)
163 {
164 	struct swap_extent *se = si->curr_swap_extent;
165 	int found_extent = 0;
166 
167 	while (nr_pages) {
168 		struct list_head *lh;
169 
170 		if (se->start_page <= start_page &&
171 		    start_page < se->start_page + se->nr_pages) {
172 			pgoff_t offset = start_page - se->start_page;
173 			sector_t start_block = se->start_block + offset;
174 			sector_t nr_blocks = se->nr_pages - offset;
175 
176 			if (nr_blocks > nr_pages)
177 				nr_blocks = nr_pages;
178 			start_page += nr_blocks;
179 			nr_pages -= nr_blocks;
180 
181 			if (!found_extent++)
182 				si->curr_swap_extent = se;
183 
184 			start_block <<= PAGE_SHIFT - 9;
185 			nr_blocks <<= PAGE_SHIFT - 9;
186 			if (blkdev_issue_discard(si->bdev, start_block,
187 				    nr_blocks, GFP_NOIO, 0))
188 				break;
189 		}
190 
191 		lh = se->list.next;
192 		se = list_entry(lh, struct swap_extent, list);
193 	}
194 }
195 
196 #define SWAPFILE_CLUSTER	256
197 #define LATENCY_LIMIT		256
198 
cluster_set_flag(struct swap_cluster_info * info,unsigned int flag)199 static inline void cluster_set_flag(struct swap_cluster_info *info,
200 	unsigned int flag)
201 {
202 	info->flags = flag;
203 }
204 
cluster_count(struct swap_cluster_info * info)205 static inline unsigned int cluster_count(struct swap_cluster_info *info)
206 {
207 	return info->data;
208 }
209 
cluster_set_count(struct swap_cluster_info * info,unsigned int c)210 static inline void cluster_set_count(struct swap_cluster_info *info,
211 				     unsigned int c)
212 {
213 	info->data = c;
214 }
215 
cluster_set_count_flag(struct swap_cluster_info * info,unsigned int c,unsigned int f)216 static inline void cluster_set_count_flag(struct swap_cluster_info *info,
217 					 unsigned int c, unsigned int f)
218 {
219 	info->flags = f;
220 	info->data = c;
221 }
222 
cluster_next(struct swap_cluster_info * info)223 static inline unsigned int cluster_next(struct swap_cluster_info *info)
224 {
225 	return info->data;
226 }
227 
cluster_set_next(struct swap_cluster_info * info,unsigned int n)228 static inline void cluster_set_next(struct swap_cluster_info *info,
229 				    unsigned int n)
230 {
231 	info->data = n;
232 }
233 
cluster_set_next_flag(struct swap_cluster_info * info,unsigned int n,unsigned int f)234 static inline void cluster_set_next_flag(struct swap_cluster_info *info,
235 					 unsigned int n, unsigned int f)
236 {
237 	info->flags = f;
238 	info->data = n;
239 }
240 
cluster_is_free(struct swap_cluster_info * info)241 static inline bool cluster_is_free(struct swap_cluster_info *info)
242 {
243 	return info->flags & CLUSTER_FLAG_FREE;
244 }
245 
cluster_is_null(struct swap_cluster_info * info)246 static inline bool cluster_is_null(struct swap_cluster_info *info)
247 {
248 	return info->flags & CLUSTER_FLAG_NEXT_NULL;
249 }
250 
cluster_set_null(struct swap_cluster_info * info)251 static inline void cluster_set_null(struct swap_cluster_info *info)
252 {
253 	info->flags = CLUSTER_FLAG_NEXT_NULL;
254 	info->data = 0;
255 }
256 
257 /* Add a cluster to discard list and schedule it to do discard */
swap_cluster_schedule_discard(struct swap_info_struct * si,unsigned int idx)258 static void swap_cluster_schedule_discard(struct swap_info_struct *si,
259 		unsigned int idx)
260 {
261 	/*
262 	 * If scan_swap_map() can't find a free cluster, it will check
263 	 * si->swap_map directly. To make sure the discarding cluster isn't
264 	 * taken by scan_swap_map(), mark the swap entries bad (occupied). It
265 	 * will be cleared after discard
266 	 */
267 	memset(si->swap_map + idx * SWAPFILE_CLUSTER,
268 			SWAP_MAP_BAD, SWAPFILE_CLUSTER);
269 
270 	if (cluster_is_null(&si->discard_cluster_head)) {
271 		cluster_set_next_flag(&si->discard_cluster_head,
272 						idx, 0);
273 		cluster_set_next_flag(&si->discard_cluster_tail,
274 						idx, 0);
275 	} else {
276 		unsigned int tail = cluster_next(&si->discard_cluster_tail);
277 		cluster_set_next(&si->cluster_info[tail], idx);
278 		cluster_set_next_flag(&si->discard_cluster_tail,
279 						idx, 0);
280 	}
281 
282 	schedule_work(&si->discard_work);
283 }
284 
285 /*
286  * Doing discard actually. After a cluster discard is finished, the cluster
287  * will be added to free cluster list. caller should hold si->lock.
288 */
swap_do_scheduled_discard(struct swap_info_struct * si)289 static void swap_do_scheduled_discard(struct swap_info_struct *si)
290 {
291 	struct swap_cluster_info *info;
292 	unsigned int idx;
293 
294 	info = si->cluster_info;
295 
296 	while (!cluster_is_null(&si->discard_cluster_head)) {
297 		idx = cluster_next(&si->discard_cluster_head);
298 
299 		cluster_set_next_flag(&si->discard_cluster_head,
300 						cluster_next(&info[idx]), 0);
301 		if (cluster_next(&si->discard_cluster_tail) == idx) {
302 			cluster_set_null(&si->discard_cluster_head);
303 			cluster_set_null(&si->discard_cluster_tail);
304 		}
305 		spin_unlock(&si->lock);
306 
307 		discard_swap_cluster(si, idx * SWAPFILE_CLUSTER,
308 				SWAPFILE_CLUSTER);
309 
310 		spin_lock(&si->lock);
311 		cluster_set_flag(&info[idx], CLUSTER_FLAG_FREE);
312 		if (cluster_is_null(&si->free_cluster_head)) {
313 			cluster_set_next_flag(&si->free_cluster_head,
314 						idx, 0);
315 			cluster_set_next_flag(&si->free_cluster_tail,
316 						idx, 0);
317 		} else {
318 			unsigned int tail;
319 
320 			tail = cluster_next(&si->free_cluster_tail);
321 			cluster_set_next(&info[tail], idx);
322 			cluster_set_next_flag(&si->free_cluster_tail,
323 						idx, 0);
324 		}
325 		memset(si->swap_map + idx * SWAPFILE_CLUSTER,
326 				0, SWAPFILE_CLUSTER);
327 	}
328 }
329 
swap_discard_work(struct work_struct * work)330 static void swap_discard_work(struct work_struct *work)
331 {
332 	struct swap_info_struct *si;
333 
334 	si = container_of(work, struct swap_info_struct, discard_work);
335 
336 	spin_lock(&si->lock);
337 	swap_do_scheduled_discard(si);
338 	spin_unlock(&si->lock);
339 }
340 
341 /*
342  * The cluster corresponding to page_nr will be used. The cluster will be
343  * removed from free cluster list and its usage counter will be increased.
344  */
inc_cluster_info_page(struct swap_info_struct * p,struct swap_cluster_info * cluster_info,unsigned long page_nr)345 static void inc_cluster_info_page(struct swap_info_struct *p,
346 	struct swap_cluster_info *cluster_info, unsigned long page_nr)
347 {
348 	unsigned long idx = page_nr / SWAPFILE_CLUSTER;
349 
350 	if (!cluster_info)
351 		return;
352 	if (cluster_is_free(&cluster_info[idx])) {
353 		VM_BUG_ON(cluster_next(&p->free_cluster_head) != idx);
354 		cluster_set_next_flag(&p->free_cluster_head,
355 			cluster_next(&cluster_info[idx]), 0);
356 		if (cluster_next(&p->free_cluster_tail) == idx) {
357 			cluster_set_null(&p->free_cluster_tail);
358 			cluster_set_null(&p->free_cluster_head);
359 		}
360 		cluster_set_count_flag(&cluster_info[idx], 0, 0);
361 	}
362 
363 	VM_BUG_ON(cluster_count(&cluster_info[idx]) >= SWAPFILE_CLUSTER);
364 	cluster_set_count(&cluster_info[idx],
365 		cluster_count(&cluster_info[idx]) + 1);
366 }
367 
368 /*
369  * The cluster corresponding to page_nr decreases one usage. If the usage
370  * counter becomes 0, which means no page in the cluster is in using, we can
371  * optionally discard the cluster and add it to free cluster list.
372  */
dec_cluster_info_page(struct swap_info_struct * p,struct swap_cluster_info * cluster_info,unsigned long page_nr)373 static void dec_cluster_info_page(struct swap_info_struct *p,
374 	struct swap_cluster_info *cluster_info, unsigned long page_nr)
375 {
376 	unsigned long idx = page_nr / SWAPFILE_CLUSTER;
377 
378 	if (!cluster_info)
379 		return;
380 
381 	VM_BUG_ON(cluster_count(&cluster_info[idx]) == 0);
382 	cluster_set_count(&cluster_info[idx],
383 		cluster_count(&cluster_info[idx]) - 1);
384 
385 	if (cluster_count(&cluster_info[idx]) == 0) {
386 		/*
387 		 * If the swap is discardable, prepare discard the cluster
388 		 * instead of free it immediately. The cluster will be freed
389 		 * after discard.
390 		 */
391 		if ((p->flags & (SWP_WRITEOK | SWP_PAGE_DISCARD)) ==
392 				 (SWP_WRITEOK | SWP_PAGE_DISCARD)) {
393 			swap_cluster_schedule_discard(p, idx);
394 			return;
395 		}
396 
397 		cluster_set_flag(&cluster_info[idx], CLUSTER_FLAG_FREE);
398 		if (cluster_is_null(&p->free_cluster_head)) {
399 			cluster_set_next_flag(&p->free_cluster_head, idx, 0);
400 			cluster_set_next_flag(&p->free_cluster_tail, idx, 0);
401 		} else {
402 			unsigned int tail = cluster_next(&p->free_cluster_tail);
403 			cluster_set_next(&cluster_info[tail], idx);
404 			cluster_set_next_flag(&p->free_cluster_tail, idx, 0);
405 		}
406 	}
407 }
408 
409 /*
410  * It's possible scan_swap_map() uses a free cluster in the middle of free
411  * cluster list. Avoiding such abuse to avoid list corruption.
412  */
413 static bool
scan_swap_map_ssd_cluster_conflict(struct swap_info_struct * si,unsigned long offset)414 scan_swap_map_ssd_cluster_conflict(struct swap_info_struct *si,
415 	unsigned long offset)
416 {
417 	struct percpu_cluster *percpu_cluster;
418 	bool conflict;
419 
420 	offset /= SWAPFILE_CLUSTER;
421 	conflict = !cluster_is_null(&si->free_cluster_head) &&
422 		offset != cluster_next(&si->free_cluster_head) &&
423 		cluster_is_free(&si->cluster_info[offset]);
424 
425 	if (!conflict)
426 		return false;
427 
428 	percpu_cluster = this_cpu_ptr(si->percpu_cluster);
429 	cluster_set_null(&percpu_cluster->index);
430 	return true;
431 }
432 
433 /*
434  * Try to get a swap entry from current cpu's swap entry pool (a cluster). This
435  * might involve allocating a new cluster for current CPU too.
436  */
scan_swap_map_try_ssd_cluster(struct swap_info_struct * si,unsigned long * offset,unsigned long * scan_base)437 static void scan_swap_map_try_ssd_cluster(struct swap_info_struct *si,
438 	unsigned long *offset, unsigned long *scan_base)
439 {
440 	struct percpu_cluster *cluster;
441 	bool found_free;
442 	unsigned long tmp;
443 
444 new_cluster:
445 	cluster = this_cpu_ptr(si->percpu_cluster);
446 	if (cluster_is_null(&cluster->index)) {
447 		if (!cluster_is_null(&si->free_cluster_head)) {
448 			cluster->index = si->free_cluster_head;
449 			cluster->next = cluster_next(&cluster->index) *
450 					SWAPFILE_CLUSTER;
451 		} else if (!cluster_is_null(&si->discard_cluster_head)) {
452 			/*
453 			 * we don't have free cluster but have some clusters in
454 			 * discarding, do discard now and reclaim them
455 			 */
456 			swap_do_scheduled_discard(si);
457 			*scan_base = *offset = si->cluster_next;
458 			goto new_cluster;
459 		} else
460 			return;
461 	}
462 
463 	found_free = false;
464 
465 	/*
466 	 * Other CPUs can use our cluster if they can't find a free cluster,
467 	 * check if there is still free entry in the cluster
468 	 */
469 	tmp = cluster->next;
470 	while (tmp < si->max && tmp < (cluster_next(&cluster->index) + 1) *
471 	       SWAPFILE_CLUSTER) {
472 		if (!si->swap_map[tmp]) {
473 			found_free = true;
474 			break;
475 		}
476 		tmp++;
477 	}
478 	if (!found_free) {
479 		cluster_set_null(&cluster->index);
480 		goto new_cluster;
481 	}
482 	cluster->next = tmp + 1;
483 	*offset = tmp;
484 	*scan_base = tmp;
485 }
486 
scan_swap_map(struct swap_info_struct * si,unsigned char usage)487 static unsigned long scan_swap_map(struct swap_info_struct *si,
488 				   unsigned char usage)
489 {
490 	unsigned long offset;
491 	unsigned long scan_base;
492 	unsigned long last_in_cluster = 0;
493 	int latency_ration = LATENCY_LIMIT;
494 
495 	/*
496 	 * We try to cluster swap pages by allocating them sequentially
497 	 * in swap.  Once we've allocated SWAPFILE_CLUSTER pages this
498 	 * way, however, we resort to first-free allocation, starting
499 	 * a new cluster.  This prevents us from scattering swap pages
500 	 * all over the entire swap partition, so that we reduce
501 	 * overall disk seek times between swap pages.  -- sct
502 	 * But we do now try to find an empty cluster.  -Andrea
503 	 * And we let swap pages go all over an SSD partition.  Hugh
504 	 */
505 
506 	si->flags += SWP_SCANNING;
507 	scan_base = offset = si->cluster_next;
508 
509 	/* SSD algorithm */
510 	if (si->cluster_info) {
511 		scan_swap_map_try_ssd_cluster(si, &offset, &scan_base);
512 		goto checks;
513 	}
514 
515 	if (unlikely(!si->cluster_nr--)) {
516 		if (si->pages - si->inuse_pages < SWAPFILE_CLUSTER) {
517 			si->cluster_nr = SWAPFILE_CLUSTER - 1;
518 			goto checks;
519 		}
520 
521 		spin_unlock(&si->lock);
522 
523 		/*
524 		 * If seek is expensive, start searching for new cluster from
525 		 * start of partition, to minimize the span of allocated swap.
526 		 * If seek is cheap, that is the SWP_SOLIDSTATE si->cluster_info
527 		 * case, just handled by scan_swap_map_try_ssd_cluster() above.
528 		 */
529 		scan_base = offset = si->lowest_bit;
530 		last_in_cluster = offset + SWAPFILE_CLUSTER - 1;
531 
532 		/* Locate the first empty (unaligned) cluster */
533 		for (; last_in_cluster <= si->highest_bit; offset++) {
534 			if (si->swap_map[offset])
535 				last_in_cluster = offset + SWAPFILE_CLUSTER;
536 			else if (offset == last_in_cluster) {
537 				spin_lock(&si->lock);
538 				offset -= SWAPFILE_CLUSTER - 1;
539 				si->cluster_next = offset;
540 				si->cluster_nr = SWAPFILE_CLUSTER - 1;
541 				goto checks;
542 			}
543 			if (unlikely(--latency_ration < 0)) {
544 				cond_resched();
545 				latency_ration = LATENCY_LIMIT;
546 			}
547 		}
548 
549 		offset = scan_base;
550 		spin_lock(&si->lock);
551 		si->cluster_nr = SWAPFILE_CLUSTER - 1;
552 	}
553 
554 checks:
555 	if (si->cluster_info) {
556 		while (scan_swap_map_ssd_cluster_conflict(si, offset))
557 			scan_swap_map_try_ssd_cluster(si, &offset, &scan_base);
558 	}
559 	if (!(si->flags & SWP_WRITEOK))
560 		goto no_page;
561 	if (!si->highest_bit)
562 		goto no_page;
563 	if (offset > si->highest_bit)
564 		scan_base = offset = si->lowest_bit;
565 
566 	/* reuse swap entry of cache-only swap if not busy. */
567 	if (vm_swap_full() && si->swap_map[offset] == SWAP_HAS_CACHE) {
568 		int swap_was_freed;
569 		spin_unlock(&si->lock);
570 		swap_was_freed = __try_to_reclaim_swap(si, offset);
571 		spin_lock(&si->lock);
572 		/* entry was freed successfully, try to use this again */
573 		if (swap_was_freed)
574 			goto checks;
575 		goto scan; /* check next one */
576 	}
577 
578 	if (si->swap_map[offset])
579 		goto scan;
580 
581 	if (offset == si->lowest_bit)
582 		si->lowest_bit++;
583 	if (offset == si->highest_bit)
584 		si->highest_bit--;
585 	si->inuse_pages++;
586 	if (si->inuse_pages == si->pages) {
587 		si->lowest_bit = si->max;
588 		si->highest_bit = 0;
589 		spin_lock(&swap_avail_lock);
590 		plist_del(&si->avail_list, &swap_avail_head);
591 		spin_unlock(&swap_avail_lock);
592 	}
593 	si->swap_map[offset] = usage;
594 	inc_cluster_info_page(si, si->cluster_info, offset);
595 	si->cluster_next = offset + 1;
596 	si->flags -= SWP_SCANNING;
597 
598 	return offset;
599 
600 scan:
601 	spin_unlock(&si->lock);
602 	while (++offset <= si->highest_bit) {
603 		if (!si->swap_map[offset]) {
604 			spin_lock(&si->lock);
605 			goto checks;
606 		}
607 		if (vm_swap_full() && si->swap_map[offset] == SWAP_HAS_CACHE) {
608 			spin_lock(&si->lock);
609 			goto checks;
610 		}
611 		if (unlikely(--latency_ration < 0)) {
612 			cond_resched();
613 			latency_ration = LATENCY_LIMIT;
614 		}
615 	}
616 	offset = si->lowest_bit;
617 	while (offset < scan_base) {
618 		if (!si->swap_map[offset]) {
619 			spin_lock(&si->lock);
620 			goto checks;
621 		}
622 		if (vm_swap_full() && si->swap_map[offset] == SWAP_HAS_CACHE) {
623 			spin_lock(&si->lock);
624 			goto checks;
625 		}
626 		if (unlikely(--latency_ration < 0)) {
627 			cond_resched();
628 			latency_ration = LATENCY_LIMIT;
629 		}
630 		offset++;
631 	}
632 	spin_lock(&si->lock);
633 
634 no_page:
635 	si->flags -= SWP_SCANNING;
636 	return 0;
637 }
638 
get_swap_page(void)639 swp_entry_t get_swap_page(void)
640 {
641 	struct swap_info_struct *si, *next;
642 	pgoff_t offset;
643 
644 	if (atomic_long_read(&nr_swap_pages) <= 0)
645 		goto noswap;
646 	atomic_long_dec(&nr_swap_pages);
647 
648 	spin_lock(&swap_avail_lock);
649 
650 start_over:
651 	plist_for_each_entry_safe(si, next, &swap_avail_head, avail_list) {
652 		/* requeue si to after same-priority siblings */
653 		plist_requeue(&si->avail_list, &swap_avail_head);
654 		spin_unlock(&swap_avail_lock);
655 		spin_lock(&si->lock);
656 		if (!si->highest_bit || !(si->flags & SWP_WRITEOK)) {
657 			spin_lock(&swap_avail_lock);
658 			if (plist_node_empty(&si->avail_list)) {
659 				spin_unlock(&si->lock);
660 				goto nextsi;
661 			}
662 			WARN(!si->highest_bit,
663 			     "swap_info %d in list but !highest_bit\n",
664 			     si->type);
665 			WARN(!(si->flags & SWP_WRITEOK),
666 			     "swap_info %d in list but !SWP_WRITEOK\n",
667 			     si->type);
668 			plist_del(&si->avail_list, &swap_avail_head);
669 			spin_unlock(&si->lock);
670 			goto nextsi;
671 		}
672 
673 		/* This is called for allocating swap entry for cache */
674 		offset = scan_swap_map(si, SWAP_HAS_CACHE);
675 		spin_unlock(&si->lock);
676 		if (offset)
677 			return swp_entry(si->type, offset);
678 		pr_debug("scan_swap_map of si %d failed to find offset\n",
679 		       si->type);
680 		spin_lock(&swap_avail_lock);
681 nextsi:
682 		/*
683 		 * if we got here, it's likely that si was almost full before,
684 		 * and since scan_swap_map() can drop the si->lock, multiple
685 		 * callers probably all tried to get a page from the same si
686 		 * and it filled up before we could get one; or, the si filled
687 		 * up between us dropping swap_avail_lock and taking si->lock.
688 		 * Since we dropped the swap_avail_lock, the swap_avail_head
689 		 * list may have been modified; so if next is still in the
690 		 * swap_avail_head list then try it, otherwise start over.
691 		 */
692 		if (plist_node_empty(&next->avail_list))
693 			goto start_over;
694 	}
695 
696 	spin_unlock(&swap_avail_lock);
697 
698 	atomic_long_inc(&nr_swap_pages);
699 noswap:
700 	return (swp_entry_t) {0};
701 }
702 
703 /* The only caller of this function is now suspend routine */
get_swap_page_of_type(int type)704 swp_entry_t get_swap_page_of_type(int type)
705 {
706 	struct swap_info_struct *si;
707 	pgoff_t offset;
708 
709 	si = swap_info[type];
710 	spin_lock(&si->lock);
711 	if (si && (si->flags & SWP_WRITEOK)) {
712 		atomic_long_dec(&nr_swap_pages);
713 		/* This is called for allocating swap entry, not cache */
714 		offset = scan_swap_map(si, 1);
715 		if (offset) {
716 			spin_unlock(&si->lock);
717 			return swp_entry(type, offset);
718 		}
719 		atomic_long_inc(&nr_swap_pages);
720 	}
721 	spin_unlock(&si->lock);
722 	return (swp_entry_t) {0};
723 }
724 
swap_info_get(swp_entry_t entry)725 static struct swap_info_struct *swap_info_get(swp_entry_t entry)
726 {
727 	struct swap_info_struct *p;
728 	unsigned long offset, type;
729 
730 	if (!entry.val)
731 		goto out;
732 	type = swp_type(entry);
733 	if (type >= nr_swapfiles)
734 		goto bad_nofile;
735 	p = swap_info[type];
736 	if (!(p->flags & SWP_USED))
737 		goto bad_device;
738 	offset = swp_offset(entry);
739 	if (offset >= p->max)
740 		goto bad_offset;
741 	if (!p->swap_map[offset])
742 		goto bad_free;
743 	spin_lock(&p->lock);
744 	return p;
745 
746 bad_free:
747 	pr_err("swap_free: %s%08lx\n", Unused_offset, entry.val);
748 	goto out;
749 bad_offset:
750 	pr_err("swap_free: %s%08lx\n", Bad_offset, entry.val);
751 	goto out;
752 bad_device:
753 	pr_err("swap_free: %s%08lx\n", Unused_file, entry.val);
754 	goto out;
755 bad_nofile:
756 	pr_err("swap_free: %s%08lx\n", Bad_file, entry.val);
757 out:
758 	return NULL;
759 }
760 
swap_entry_free(struct swap_info_struct * p,swp_entry_t entry,unsigned char usage)761 static unsigned char swap_entry_free(struct swap_info_struct *p,
762 				     swp_entry_t entry, unsigned char usage)
763 {
764 	unsigned long offset = swp_offset(entry);
765 	unsigned char count;
766 	unsigned char has_cache;
767 
768 	count = p->swap_map[offset];
769 	has_cache = count & SWAP_HAS_CACHE;
770 	count &= ~SWAP_HAS_CACHE;
771 
772 	if (usage == SWAP_HAS_CACHE) {
773 		VM_BUG_ON(!has_cache);
774 		has_cache = 0;
775 	} else if (count == SWAP_MAP_SHMEM) {
776 		/*
777 		 * Or we could insist on shmem.c using a special
778 		 * swap_shmem_free() and free_shmem_swap_and_cache()...
779 		 */
780 		count = 0;
781 	} else if ((count & ~COUNT_CONTINUED) <= SWAP_MAP_MAX) {
782 		if (count == COUNT_CONTINUED) {
783 			if (swap_count_continued(p, offset, count))
784 				count = SWAP_MAP_MAX | COUNT_CONTINUED;
785 			else
786 				count = SWAP_MAP_MAX;
787 		} else
788 			count--;
789 	}
790 
791 	if (!count)
792 		mem_cgroup_uncharge_swap(entry);
793 
794 	usage = count | has_cache;
795 	p->swap_map[offset] = usage;
796 
797 	/* free if no reference */
798 	if (!usage) {
799 		dec_cluster_info_page(p, p->cluster_info, offset);
800 		if (offset < p->lowest_bit)
801 			p->lowest_bit = offset;
802 		if (offset > p->highest_bit) {
803 			bool was_full = !p->highest_bit;
804 			p->highest_bit = offset;
805 			if (was_full && (p->flags & SWP_WRITEOK)) {
806 				spin_lock(&swap_avail_lock);
807 				WARN_ON(!plist_node_empty(&p->avail_list));
808 				if (plist_node_empty(&p->avail_list))
809 					plist_add(&p->avail_list,
810 						  &swap_avail_head);
811 				spin_unlock(&swap_avail_lock);
812 			}
813 		}
814 		atomic_long_inc(&nr_swap_pages);
815 		p->inuse_pages--;
816 		frontswap_invalidate_page(p->type, offset);
817 		if (p->flags & SWP_BLKDEV) {
818 			struct gendisk *disk = p->bdev->bd_disk;
819 			if (disk->fops->swap_slot_free_notify)
820 				disk->fops->swap_slot_free_notify(p->bdev,
821 								  offset);
822 		}
823 	}
824 
825 	return usage;
826 }
827 
828 /*
829  * Caller has made sure that the swap device corresponding to entry
830  * is still around or has not been recycled.
831  */
swap_free(swp_entry_t entry)832 void swap_free(swp_entry_t entry)
833 {
834 	struct swap_info_struct *p;
835 
836 	p = swap_info_get(entry);
837 	if (p) {
838 		swap_entry_free(p, entry, 1);
839 		spin_unlock(&p->lock);
840 	}
841 }
842 
843 /*
844  * Called after dropping swapcache to decrease refcnt to swap entries.
845  */
swapcache_free(swp_entry_t entry)846 void swapcache_free(swp_entry_t entry)
847 {
848 	struct swap_info_struct *p;
849 
850 	p = swap_info_get(entry);
851 	if (p) {
852 		swap_entry_free(p, entry, SWAP_HAS_CACHE);
853 		spin_unlock(&p->lock);
854 	}
855 }
856 
857 /*
858  * How many references to page are currently swapped out?
859  * This does not give an exact answer when swap count is continued,
860  * but does include the high COUNT_CONTINUED flag to allow for that.
861  */
page_swapcount(struct page * page)862 int page_swapcount(struct page *page)
863 {
864 	int count = 0;
865 	struct swap_info_struct *p;
866 	swp_entry_t entry;
867 
868 	entry.val = page_private(page);
869 	p = swap_info_get(entry);
870 	if (p) {
871 		count = swap_count(p->swap_map[swp_offset(entry)]);
872 		spin_unlock(&p->lock);
873 	}
874 	return count;
875 }
876 
877 /*
878  * How many references to @entry are currently swapped out?
879  * This considers COUNT_CONTINUED so it returns exact answer.
880  */
swp_swapcount(swp_entry_t entry)881 int swp_swapcount(swp_entry_t entry)
882 {
883 	int count, tmp_count, n;
884 	struct swap_info_struct *p;
885 	struct page *page;
886 	pgoff_t offset;
887 	unsigned char *map;
888 
889 	p = swap_info_get(entry);
890 	if (!p)
891 		return 0;
892 
893 	count = swap_count(p->swap_map[swp_offset(entry)]);
894 	if (!(count & COUNT_CONTINUED))
895 		goto out;
896 
897 	count &= ~COUNT_CONTINUED;
898 	n = SWAP_MAP_MAX + 1;
899 
900 	offset = swp_offset(entry);
901 	page = vmalloc_to_page(p->swap_map + offset);
902 	offset &= ~PAGE_MASK;
903 	VM_BUG_ON(page_private(page) != SWP_CONTINUED);
904 
905 	do {
906 		page = list_entry(page->lru.next, struct page, lru);
907 		map = kmap_atomic(page);
908 		tmp_count = map[offset];
909 		kunmap_atomic(map);
910 
911 		count += (tmp_count & ~COUNT_CONTINUED) * n;
912 		n *= (SWAP_CONT_MAX + 1);
913 	} while (tmp_count & COUNT_CONTINUED);
914 out:
915 	spin_unlock(&p->lock);
916 	return count;
917 }
918 
919 /*
920  * We can write to an anon page without COW if there are no other references
921  * to it.  And as a side-effect, free up its swap: because the old content
922  * on disk will never be read, and seeking back there to write new content
923  * later would only waste time away from clustering.
924  */
reuse_swap_page(struct page * page)925 int reuse_swap_page(struct page *page)
926 {
927 	int count;
928 
929 	VM_BUG_ON_PAGE(!PageLocked(page), page);
930 	if (unlikely(PageKsm(page)))
931 		return 0;
932 	count = page_mapcount(page);
933 	if (count <= 1 && PageSwapCache(page)) {
934 		count += page_swapcount(page);
935 		if (count == 1 && !PageWriteback(page)) {
936 			delete_from_swap_cache(page);
937 			SetPageDirty(page);
938 		}
939 	}
940 	return count <= 1;
941 }
942 
943 /*
944  * If swap is getting full, or if there are no more mappings of this page,
945  * then try_to_free_swap is called to free its swap space.
946  */
try_to_free_swap(struct page * page)947 int try_to_free_swap(struct page *page)
948 {
949 	VM_BUG_ON_PAGE(!PageLocked(page), page);
950 
951 	if (!PageSwapCache(page))
952 		return 0;
953 	if (PageWriteback(page))
954 		return 0;
955 	if (page_swapcount(page))
956 		return 0;
957 
958 	/*
959 	 * Once hibernation has begun to create its image of memory,
960 	 * there's a danger that one of the calls to try_to_free_swap()
961 	 * - most probably a call from __try_to_reclaim_swap() while
962 	 * hibernation is allocating its own swap pages for the image,
963 	 * but conceivably even a call from memory reclaim - will free
964 	 * the swap from a page which has already been recorded in the
965 	 * image as a clean swapcache page, and then reuse its swap for
966 	 * another page of the image.  On waking from hibernation, the
967 	 * original page might be freed under memory pressure, then
968 	 * later read back in from swap, now with the wrong data.
969 	 *
970 	 * Hibernation suspends storage while it is writing the image
971 	 * to disk so check that here.
972 	 */
973 	if (pm_suspended_storage())
974 		return 0;
975 
976 	delete_from_swap_cache(page);
977 	SetPageDirty(page);
978 	return 1;
979 }
980 
981 /*
982  * Free the swap entry like above, but also try to
983  * free the page cache entry if it is the last user.
984  */
free_swap_and_cache(swp_entry_t entry)985 int free_swap_and_cache(swp_entry_t entry)
986 {
987 	struct swap_info_struct *p;
988 	struct page *page = NULL;
989 
990 	if (non_swap_entry(entry))
991 		return 1;
992 
993 	p = swap_info_get(entry);
994 	if (p) {
995 		if (swap_entry_free(p, entry, 1) == SWAP_HAS_CACHE) {
996 			page = find_get_page(swap_address_space(entry),
997 						entry.val);
998 			if (page && !trylock_page(page)) {
999 				page_cache_release(page);
1000 				page = NULL;
1001 			}
1002 		}
1003 		spin_unlock(&p->lock);
1004 	}
1005 	if (page) {
1006 		/*
1007 		 * Not mapped elsewhere, or swap space full? Free it!
1008 		 * Also recheck PageSwapCache now page is locked (above).
1009 		 */
1010 		if (PageSwapCache(page) && !PageWriteback(page) &&
1011 				(!page_mapped(page) || vm_swap_full())) {
1012 			delete_from_swap_cache(page);
1013 			SetPageDirty(page);
1014 		}
1015 		unlock_page(page);
1016 		page_cache_release(page);
1017 	}
1018 	return p != NULL;
1019 }
1020 
1021 #ifdef CONFIG_HIBERNATION
1022 /*
1023  * Find the swap type that corresponds to given device (if any).
1024  *
1025  * @offset - number of the PAGE_SIZE-sized block of the device, starting
1026  * from 0, in which the swap header is expected to be located.
1027  *
1028  * This is needed for the suspend to disk (aka swsusp).
1029  */
swap_type_of(dev_t device,sector_t offset,struct block_device ** bdev_p)1030 int swap_type_of(dev_t device, sector_t offset, struct block_device **bdev_p)
1031 {
1032 	struct block_device *bdev = NULL;
1033 	int type;
1034 
1035 	if (device)
1036 		bdev = bdget(device);
1037 
1038 	spin_lock(&swap_lock);
1039 	for (type = 0; type < nr_swapfiles; type++) {
1040 		struct swap_info_struct *sis = swap_info[type];
1041 
1042 		if (!(sis->flags & SWP_WRITEOK))
1043 			continue;
1044 
1045 		if (!bdev) {
1046 			if (bdev_p)
1047 				*bdev_p = bdgrab(sis->bdev);
1048 
1049 			spin_unlock(&swap_lock);
1050 			return type;
1051 		}
1052 		if (bdev == sis->bdev) {
1053 			struct swap_extent *se = &sis->first_swap_extent;
1054 
1055 			if (se->start_block == offset) {
1056 				if (bdev_p)
1057 					*bdev_p = bdgrab(sis->bdev);
1058 
1059 				spin_unlock(&swap_lock);
1060 				bdput(bdev);
1061 				return type;
1062 			}
1063 		}
1064 	}
1065 	spin_unlock(&swap_lock);
1066 	if (bdev)
1067 		bdput(bdev);
1068 
1069 	return -ENODEV;
1070 }
1071 
1072 /*
1073  * Get the (PAGE_SIZE) block corresponding to given offset on the swapdev
1074  * corresponding to given index in swap_info (swap type).
1075  */
swapdev_block(int type,pgoff_t offset)1076 sector_t swapdev_block(int type, pgoff_t offset)
1077 {
1078 	struct block_device *bdev;
1079 
1080 	if ((unsigned int)type >= nr_swapfiles)
1081 		return 0;
1082 	if (!(swap_info[type]->flags & SWP_WRITEOK))
1083 		return 0;
1084 	return map_swap_entry(swp_entry(type, offset), &bdev);
1085 }
1086 
1087 /*
1088  * Return either the total number of swap pages of given type, or the number
1089  * of free pages of that type (depending on @free)
1090  *
1091  * This is needed for software suspend
1092  */
count_swap_pages(int type,int free)1093 unsigned int count_swap_pages(int type, int free)
1094 {
1095 	unsigned int n = 0;
1096 
1097 	spin_lock(&swap_lock);
1098 	if ((unsigned int)type < nr_swapfiles) {
1099 		struct swap_info_struct *sis = swap_info[type];
1100 
1101 		spin_lock(&sis->lock);
1102 		if (sis->flags & SWP_WRITEOK) {
1103 			n = sis->pages;
1104 			if (free)
1105 				n -= sis->inuse_pages;
1106 		}
1107 		spin_unlock(&sis->lock);
1108 	}
1109 	spin_unlock(&swap_lock);
1110 	return n;
1111 }
1112 #endif /* CONFIG_HIBERNATION */
1113 
maybe_same_pte(pte_t pte,pte_t swp_pte)1114 static inline int maybe_same_pte(pte_t pte, pte_t swp_pte)
1115 {
1116 #ifdef CONFIG_MEM_SOFT_DIRTY
1117 	/*
1118 	 * When pte keeps soft dirty bit the pte generated
1119 	 * from swap entry does not has it, still it's same
1120 	 * pte from logical point of view.
1121 	 */
1122 	pte_t swp_pte_dirty = pte_swp_mksoft_dirty(swp_pte);
1123 	return pte_same(pte, swp_pte) || pte_same(pte, swp_pte_dirty);
1124 #else
1125 	return pte_same(pte, swp_pte);
1126 #endif
1127 }
1128 
1129 /*
1130  * No need to decide whether this PTE shares the swap entry with others,
1131  * just let do_wp_page work it out if a write is requested later - to
1132  * force COW, vm_page_prot omits write permission from any private vma.
1133  */
unuse_pte(struct vm_area_struct * vma,pmd_t * pmd,unsigned long addr,swp_entry_t entry,struct page * page)1134 static int unuse_pte(struct vm_area_struct *vma, pmd_t *pmd,
1135 		unsigned long addr, swp_entry_t entry, struct page *page)
1136 {
1137 	struct page *swapcache;
1138 	struct mem_cgroup *memcg;
1139 	spinlock_t *ptl;
1140 	pte_t *pte;
1141 	int ret = 1;
1142 
1143 	swapcache = page;
1144 	page = ksm_might_need_to_copy(page, vma, addr);
1145 	if (unlikely(!page))
1146 		return -ENOMEM;
1147 
1148 	if (mem_cgroup_try_charge(page, vma->vm_mm, GFP_KERNEL, &memcg)) {
1149 		ret = -ENOMEM;
1150 		goto out_nolock;
1151 	}
1152 
1153 	pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
1154 	if (unlikely(!maybe_same_pte(*pte, swp_entry_to_pte(entry)))) {
1155 		mem_cgroup_cancel_charge(page, memcg);
1156 		ret = 0;
1157 		goto out;
1158 	}
1159 
1160 	dec_mm_counter(vma->vm_mm, MM_SWAPENTS);
1161 	inc_mm_counter(vma->vm_mm, MM_ANONPAGES);
1162 	get_page(page);
1163 	set_pte_at(vma->vm_mm, addr, pte,
1164 		   pte_mkold(mk_pte(page, vma->vm_page_prot)));
1165 	if (page == swapcache) {
1166 		page_add_anon_rmap(page, vma, addr);
1167 		mem_cgroup_commit_charge(page, memcg, true);
1168 	} else { /* ksm created a completely new copy */
1169 		page_add_new_anon_rmap(page, vma, addr);
1170 		mem_cgroup_commit_charge(page, memcg, false);
1171 		lru_cache_add_active_or_unevictable(page, vma);
1172 	}
1173 	swap_free(entry);
1174 	/*
1175 	 * Move the page to the active list so it is not
1176 	 * immediately swapped out again after swapon.
1177 	 */
1178 	activate_page(page);
1179 out:
1180 	pte_unmap_unlock(pte, ptl);
1181 out_nolock:
1182 	if (page != swapcache) {
1183 		unlock_page(page);
1184 		put_page(page);
1185 	}
1186 	return ret;
1187 }
1188 
unuse_pte_range(struct vm_area_struct * vma,pmd_t * pmd,unsigned long addr,unsigned long end,swp_entry_t entry,struct page * page)1189 static int unuse_pte_range(struct vm_area_struct *vma, pmd_t *pmd,
1190 				unsigned long addr, unsigned long end,
1191 				swp_entry_t entry, struct page *page)
1192 {
1193 	pte_t swp_pte = swp_entry_to_pte(entry);
1194 	pte_t *pte;
1195 	int ret = 0;
1196 
1197 	/*
1198 	 * We don't actually need pte lock while scanning for swp_pte: since
1199 	 * we hold page lock and mmap_sem, swp_pte cannot be inserted into the
1200 	 * page table while we're scanning; though it could get zapped, and on
1201 	 * some architectures (e.g. x86_32 with PAE) we might catch a glimpse
1202 	 * of unmatched parts which look like swp_pte, so unuse_pte must
1203 	 * recheck under pte lock.  Scanning without pte lock lets it be
1204 	 * preemptable whenever CONFIG_PREEMPT but not CONFIG_HIGHPTE.
1205 	 */
1206 	pte = pte_offset_map(pmd, addr);
1207 	do {
1208 		/*
1209 		 * swapoff spends a _lot_ of time in this loop!
1210 		 * Test inline before going to call unuse_pte.
1211 		 */
1212 		if (unlikely(maybe_same_pte(*pte, swp_pte))) {
1213 			pte_unmap(pte);
1214 			ret = unuse_pte(vma, pmd, addr, entry, page);
1215 			if (ret)
1216 				goto out;
1217 			pte = pte_offset_map(pmd, addr);
1218 		}
1219 	} while (pte++, addr += PAGE_SIZE, addr != end);
1220 	pte_unmap(pte - 1);
1221 out:
1222 	return ret;
1223 }
1224 
unuse_pmd_range(struct vm_area_struct * vma,pud_t * pud,unsigned long addr,unsigned long end,swp_entry_t entry,struct page * page)1225 static inline int unuse_pmd_range(struct vm_area_struct *vma, pud_t *pud,
1226 				unsigned long addr, unsigned long end,
1227 				swp_entry_t entry, struct page *page)
1228 {
1229 	pmd_t *pmd;
1230 	unsigned long next;
1231 	int ret;
1232 
1233 	pmd = pmd_offset(pud, addr);
1234 	do {
1235 		next = pmd_addr_end(addr, end);
1236 		if (pmd_none_or_trans_huge_or_clear_bad(pmd))
1237 			continue;
1238 		ret = unuse_pte_range(vma, pmd, addr, next, entry, page);
1239 		if (ret)
1240 			return ret;
1241 	} while (pmd++, addr = next, addr != end);
1242 	return 0;
1243 }
1244 
unuse_pud_range(struct vm_area_struct * vma,pgd_t * pgd,unsigned long addr,unsigned long end,swp_entry_t entry,struct page * page)1245 static inline int unuse_pud_range(struct vm_area_struct *vma, pgd_t *pgd,
1246 				unsigned long addr, unsigned long end,
1247 				swp_entry_t entry, struct page *page)
1248 {
1249 	pud_t *pud;
1250 	unsigned long next;
1251 	int ret;
1252 
1253 	pud = pud_offset(pgd, addr);
1254 	do {
1255 		next = pud_addr_end(addr, end);
1256 		if (pud_none_or_clear_bad(pud))
1257 			continue;
1258 		ret = unuse_pmd_range(vma, pud, addr, next, entry, page);
1259 		if (ret)
1260 			return ret;
1261 	} while (pud++, addr = next, addr != end);
1262 	return 0;
1263 }
1264 
unuse_vma(struct vm_area_struct * vma,swp_entry_t entry,struct page * page)1265 static int unuse_vma(struct vm_area_struct *vma,
1266 				swp_entry_t entry, struct page *page)
1267 {
1268 	pgd_t *pgd;
1269 	unsigned long addr, end, next;
1270 	int ret;
1271 
1272 	if (page_anon_vma(page)) {
1273 		addr = page_address_in_vma(page, vma);
1274 		if (addr == -EFAULT)
1275 			return 0;
1276 		else
1277 			end = addr + PAGE_SIZE;
1278 	} else {
1279 		addr = vma->vm_start;
1280 		end = vma->vm_end;
1281 	}
1282 
1283 	pgd = pgd_offset(vma->vm_mm, addr);
1284 	do {
1285 		next = pgd_addr_end(addr, end);
1286 		if (pgd_none_or_clear_bad(pgd))
1287 			continue;
1288 		ret = unuse_pud_range(vma, pgd, addr, next, entry, page);
1289 		if (ret)
1290 			return ret;
1291 	} while (pgd++, addr = next, addr != end);
1292 	return 0;
1293 }
1294 
unuse_mm(struct mm_struct * mm,swp_entry_t entry,struct page * page)1295 static int unuse_mm(struct mm_struct *mm,
1296 				swp_entry_t entry, struct page *page)
1297 {
1298 	struct vm_area_struct *vma;
1299 	int ret = 0;
1300 
1301 	if (!down_read_trylock(&mm->mmap_sem)) {
1302 		/*
1303 		 * Activate page so shrink_inactive_list is unlikely to unmap
1304 		 * its ptes while lock is dropped, so swapoff can make progress.
1305 		 */
1306 		activate_page(page);
1307 		unlock_page(page);
1308 		down_read(&mm->mmap_sem);
1309 		lock_page(page);
1310 	}
1311 	for (vma = mm->mmap; vma; vma = vma->vm_next) {
1312 		if (vma->anon_vma && (ret = unuse_vma(vma, entry, page)))
1313 			break;
1314 	}
1315 	up_read(&mm->mmap_sem);
1316 	return (ret < 0)? ret: 0;
1317 }
1318 
1319 /*
1320  * Scan swap_map (or frontswap_map if frontswap parameter is true)
1321  * from current position to next entry still in use.
1322  * Recycle to start on reaching the end, returning 0 when empty.
1323  */
find_next_to_unuse(struct swap_info_struct * si,unsigned int prev,bool frontswap)1324 static unsigned int find_next_to_unuse(struct swap_info_struct *si,
1325 					unsigned int prev, bool frontswap)
1326 {
1327 	unsigned int max = si->max;
1328 	unsigned int i = prev;
1329 	unsigned char count;
1330 
1331 	/*
1332 	 * No need for swap_lock here: we're just looking
1333 	 * for whether an entry is in use, not modifying it; false
1334 	 * hits are okay, and sys_swapoff() has already prevented new
1335 	 * allocations from this area (while holding swap_lock).
1336 	 */
1337 	for (;;) {
1338 		if (++i >= max) {
1339 			if (!prev) {
1340 				i = 0;
1341 				break;
1342 			}
1343 			/*
1344 			 * No entries in use at top of swap_map,
1345 			 * loop back to start and recheck there.
1346 			 */
1347 			max = prev + 1;
1348 			prev = 0;
1349 			i = 1;
1350 		}
1351 		if (frontswap) {
1352 			if (frontswap_test(si, i))
1353 				break;
1354 			else
1355 				continue;
1356 		}
1357 		count = READ_ONCE(si->swap_map[i]);
1358 		if (count && swap_count(count) != SWAP_MAP_BAD)
1359 			break;
1360 	}
1361 	return i;
1362 }
1363 
1364 /*
1365  * We completely avoid races by reading each swap page in advance,
1366  * and then search for the process using it.  All the necessary
1367  * page table adjustments can then be made atomically.
1368  *
1369  * if the boolean frontswap is true, only unuse pages_to_unuse pages;
1370  * pages_to_unuse==0 means all pages; ignored if frontswap is false
1371  */
try_to_unuse(unsigned int type,bool frontswap,unsigned long pages_to_unuse)1372 int try_to_unuse(unsigned int type, bool frontswap,
1373 		 unsigned long pages_to_unuse)
1374 {
1375 	struct swap_info_struct *si = swap_info[type];
1376 	struct mm_struct *start_mm;
1377 	volatile unsigned char *swap_map; /* swap_map is accessed without
1378 					   * locking. Mark it as volatile
1379 					   * to prevent compiler doing
1380 					   * something odd.
1381 					   */
1382 	unsigned char swcount;
1383 	struct page *page;
1384 	swp_entry_t entry;
1385 	unsigned int i = 0;
1386 	int retval = 0;
1387 
1388 	/*
1389 	 * When searching mms for an entry, a good strategy is to
1390 	 * start at the first mm we freed the previous entry from
1391 	 * (though actually we don't notice whether we or coincidence
1392 	 * freed the entry).  Initialize this start_mm with a hold.
1393 	 *
1394 	 * A simpler strategy would be to start at the last mm we
1395 	 * freed the previous entry from; but that would take less
1396 	 * advantage of mmlist ordering, which clusters forked mms
1397 	 * together, child after parent.  If we race with dup_mmap(), we
1398 	 * prefer to resolve parent before child, lest we miss entries
1399 	 * duplicated after we scanned child: using last mm would invert
1400 	 * that.
1401 	 */
1402 	start_mm = &init_mm;
1403 	atomic_inc(&init_mm.mm_users);
1404 
1405 	/*
1406 	 * Keep on scanning until all entries have gone.  Usually,
1407 	 * one pass through swap_map is enough, but not necessarily:
1408 	 * there are races when an instance of an entry might be missed.
1409 	 */
1410 	while ((i = find_next_to_unuse(si, i, frontswap)) != 0) {
1411 		if (signal_pending(current)) {
1412 			retval = -EINTR;
1413 			break;
1414 		}
1415 
1416 		/*
1417 		 * Get a page for the entry, using the existing swap
1418 		 * cache page if there is one.  Otherwise, get a clean
1419 		 * page and read the swap into it.
1420 		 */
1421 		swap_map = &si->swap_map[i];
1422 		entry = swp_entry(type, i);
1423 		page = read_swap_cache_async(entry,
1424 					GFP_HIGHUSER_MOVABLE, NULL, 0);
1425 		if (!page) {
1426 			/*
1427 			 * Either swap_duplicate() failed because entry
1428 			 * has been freed independently, and will not be
1429 			 * reused since sys_swapoff() already disabled
1430 			 * allocation from here, or alloc_page() failed.
1431 			 */
1432 			swcount = *swap_map;
1433 			/*
1434 			 * We don't hold lock here, so the swap entry could be
1435 			 * SWAP_MAP_BAD (when the cluster is discarding).
1436 			 * Instead of fail out, We can just skip the swap
1437 			 * entry because swapoff will wait for discarding
1438 			 * finish anyway.
1439 			 */
1440 			if (!swcount || swcount == SWAP_MAP_BAD)
1441 				continue;
1442 			retval = -ENOMEM;
1443 			break;
1444 		}
1445 
1446 		/*
1447 		 * Don't hold on to start_mm if it looks like exiting.
1448 		 */
1449 		if (atomic_read(&start_mm->mm_users) == 1) {
1450 			mmput(start_mm);
1451 			start_mm = &init_mm;
1452 			atomic_inc(&init_mm.mm_users);
1453 		}
1454 
1455 		/*
1456 		 * Wait for and lock page.  When do_swap_page races with
1457 		 * try_to_unuse, do_swap_page can handle the fault much
1458 		 * faster than try_to_unuse can locate the entry.  This
1459 		 * apparently redundant "wait_on_page_locked" lets try_to_unuse
1460 		 * defer to do_swap_page in such a case - in some tests,
1461 		 * do_swap_page and try_to_unuse repeatedly compete.
1462 		 */
1463 		wait_on_page_locked(page);
1464 		wait_on_page_writeback(page);
1465 		lock_page(page);
1466 		wait_on_page_writeback(page);
1467 
1468 		/*
1469 		 * Remove all references to entry.
1470 		 */
1471 		swcount = *swap_map;
1472 		if (swap_count(swcount) == SWAP_MAP_SHMEM) {
1473 			retval = shmem_unuse(entry, page);
1474 			/* page has already been unlocked and released */
1475 			if (retval < 0)
1476 				break;
1477 			continue;
1478 		}
1479 		if (swap_count(swcount) && start_mm != &init_mm)
1480 			retval = unuse_mm(start_mm, entry, page);
1481 
1482 		if (swap_count(*swap_map)) {
1483 			int set_start_mm = (*swap_map >= swcount);
1484 			struct list_head *p = &start_mm->mmlist;
1485 			struct mm_struct *new_start_mm = start_mm;
1486 			struct mm_struct *prev_mm = start_mm;
1487 			struct mm_struct *mm;
1488 
1489 			atomic_inc(&new_start_mm->mm_users);
1490 			atomic_inc(&prev_mm->mm_users);
1491 			spin_lock(&mmlist_lock);
1492 			while (swap_count(*swap_map) && !retval &&
1493 					(p = p->next) != &start_mm->mmlist) {
1494 				mm = list_entry(p, struct mm_struct, mmlist);
1495 				if (!atomic_inc_not_zero(&mm->mm_users))
1496 					continue;
1497 				spin_unlock(&mmlist_lock);
1498 				mmput(prev_mm);
1499 				prev_mm = mm;
1500 
1501 				cond_resched();
1502 
1503 				swcount = *swap_map;
1504 				if (!swap_count(swcount)) /* any usage ? */
1505 					;
1506 				else if (mm == &init_mm)
1507 					set_start_mm = 1;
1508 				else
1509 					retval = unuse_mm(mm, entry, page);
1510 
1511 				if (set_start_mm && *swap_map < swcount) {
1512 					mmput(new_start_mm);
1513 					atomic_inc(&mm->mm_users);
1514 					new_start_mm = mm;
1515 					set_start_mm = 0;
1516 				}
1517 				spin_lock(&mmlist_lock);
1518 			}
1519 			spin_unlock(&mmlist_lock);
1520 			mmput(prev_mm);
1521 			mmput(start_mm);
1522 			start_mm = new_start_mm;
1523 		}
1524 		if (retval) {
1525 			unlock_page(page);
1526 			page_cache_release(page);
1527 			break;
1528 		}
1529 
1530 		/*
1531 		 * If a reference remains (rare), we would like to leave
1532 		 * the page in the swap cache; but try_to_unmap could
1533 		 * then re-duplicate the entry once we drop page lock,
1534 		 * so we might loop indefinitely; also, that page could
1535 		 * not be swapped out to other storage meanwhile.  So:
1536 		 * delete from cache even if there's another reference,
1537 		 * after ensuring that the data has been saved to disk -
1538 		 * since if the reference remains (rarer), it will be
1539 		 * read from disk into another page.  Splitting into two
1540 		 * pages would be incorrect if swap supported "shared
1541 		 * private" pages, but they are handled by tmpfs files.
1542 		 *
1543 		 * Given how unuse_vma() targets one particular offset
1544 		 * in an anon_vma, once the anon_vma has been determined,
1545 		 * this splitting happens to be just what is needed to
1546 		 * handle where KSM pages have been swapped out: re-reading
1547 		 * is unnecessarily slow, but we can fix that later on.
1548 		 */
1549 		if (swap_count(*swap_map) &&
1550 		     PageDirty(page) && PageSwapCache(page)) {
1551 			struct writeback_control wbc = {
1552 				.sync_mode = WB_SYNC_NONE,
1553 			};
1554 
1555 			swap_writepage(page, &wbc);
1556 			lock_page(page);
1557 			wait_on_page_writeback(page);
1558 		}
1559 
1560 		/*
1561 		 * It is conceivable that a racing task removed this page from
1562 		 * swap cache just before we acquired the page lock at the top,
1563 		 * or while we dropped it in unuse_mm().  The page might even
1564 		 * be back in swap cache on another swap area: that we must not
1565 		 * delete, since it may not have been written out to swap yet.
1566 		 */
1567 		if (PageSwapCache(page) &&
1568 		    likely(page_private(page) == entry.val))
1569 			delete_from_swap_cache(page);
1570 
1571 		/*
1572 		 * So we could skip searching mms once swap count went
1573 		 * to 1, we did not mark any present ptes as dirty: must
1574 		 * mark page dirty so shrink_page_list will preserve it.
1575 		 */
1576 		SetPageDirty(page);
1577 		unlock_page(page);
1578 		page_cache_release(page);
1579 
1580 		/*
1581 		 * Make sure that we aren't completely killing
1582 		 * interactive performance.
1583 		 */
1584 		cond_resched();
1585 		if (frontswap && pages_to_unuse > 0) {
1586 			if (!--pages_to_unuse)
1587 				break;
1588 		}
1589 	}
1590 
1591 	mmput(start_mm);
1592 	return retval;
1593 }
1594 
1595 /*
1596  * After a successful try_to_unuse, if no swap is now in use, we know
1597  * we can empty the mmlist.  swap_lock must be held on entry and exit.
1598  * Note that mmlist_lock nests inside swap_lock, and an mm must be
1599  * added to the mmlist just after page_duplicate - before would be racy.
1600  */
drain_mmlist(void)1601 static void drain_mmlist(void)
1602 {
1603 	struct list_head *p, *next;
1604 	unsigned int type;
1605 
1606 	for (type = 0; type < nr_swapfiles; type++)
1607 		if (swap_info[type]->inuse_pages)
1608 			return;
1609 	spin_lock(&mmlist_lock);
1610 	list_for_each_safe(p, next, &init_mm.mmlist)
1611 		list_del_init(p);
1612 	spin_unlock(&mmlist_lock);
1613 }
1614 
1615 /*
1616  * Use this swapdev's extent info to locate the (PAGE_SIZE) block which
1617  * corresponds to page offset for the specified swap entry.
1618  * Note that the type of this function is sector_t, but it returns page offset
1619  * into the bdev, not sector offset.
1620  */
map_swap_entry(swp_entry_t entry,struct block_device ** bdev)1621 static sector_t map_swap_entry(swp_entry_t entry, struct block_device **bdev)
1622 {
1623 	struct swap_info_struct *sis;
1624 	struct swap_extent *start_se;
1625 	struct swap_extent *se;
1626 	pgoff_t offset;
1627 
1628 	sis = swap_info[swp_type(entry)];
1629 	*bdev = sis->bdev;
1630 
1631 	offset = swp_offset(entry);
1632 	start_se = sis->curr_swap_extent;
1633 	se = start_se;
1634 
1635 	for ( ; ; ) {
1636 		struct list_head *lh;
1637 
1638 		if (se->start_page <= offset &&
1639 				offset < (se->start_page + se->nr_pages)) {
1640 			return se->start_block + (offset - se->start_page);
1641 		}
1642 		lh = se->list.next;
1643 		se = list_entry(lh, struct swap_extent, list);
1644 		sis->curr_swap_extent = se;
1645 		BUG_ON(se == start_se);		/* It *must* be present */
1646 	}
1647 }
1648 
1649 /*
1650  * Returns the page offset into bdev for the specified page's swap entry.
1651  */
map_swap_page(struct page * page,struct block_device ** bdev)1652 sector_t map_swap_page(struct page *page, struct block_device **bdev)
1653 {
1654 	swp_entry_t entry;
1655 	entry.val = page_private(page);
1656 	return map_swap_entry(entry, bdev) << (PAGE_SHIFT - 9);
1657 }
1658 
1659 /*
1660  * Free all of a swapdev's extent information
1661  */
destroy_swap_extents(struct swap_info_struct * sis)1662 static void destroy_swap_extents(struct swap_info_struct *sis)
1663 {
1664 	while (!list_empty(&sis->first_swap_extent.list)) {
1665 		struct swap_extent *se;
1666 
1667 		se = list_entry(sis->first_swap_extent.list.next,
1668 				struct swap_extent, list);
1669 		list_del(&se->list);
1670 		kfree(se);
1671 	}
1672 
1673 	if (sis->flags & SWP_FILE) {
1674 		struct file *swap_file = sis->swap_file;
1675 		struct address_space *mapping = swap_file->f_mapping;
1676 
1677 		sis->flags &= ~SWP_FILE;
1678 		mapping->a_ops->swap_deactivate(swap_file);
1679 	}
1680 }
1681 
1682 /*
1683  * Add a block range (and the corresponding page range) into this swapdev's
1684  * extent list.  The extent list is kept sorted in page order.
1685  *
1686  * This function rather assumes that it is called in ascending page order.
1687  */
1688 int
add_swap_extent(struct swap_info_struct * sis,unsigned long start_page,unsigned long nr_pages,sector_t start_block)1689 add_swap_extent(struct swap_info_struct *sis, unsigned long start_page,
1690 		unsigned long nr_pages, sector_t start_block)
1691 {
1692 	struct swap_extent *se;
1693 	struct swap_extent *new_se;
1694 	struct list_head *lh;
1695 
1696 	if (start_page == 0) {
1697 		se = &sis->first_swap_extent;
1698 		sis->curr_swap_extent = se;
1699 		se->start_page = 0;
1700 		se->nr_pages = nr_pages;
1701 		se->start_block = start_block;
1702 		return 1;
1703 	} else {
1704 		lh = sis->first_swap_extent.list.prev;	/* Highest extent */
1705 		se = list_entry(lh, struct swap_extent, list);
1706 		BUG_ON(se->start_page + se->nr_pages != start_page);
1707 		if (se->start_block + se->nr_pages == start_block) {
1708 			/* Merge it */
1709 			se->nr_pages += nr_pages;
1710 			return 0;
1711 		}
1712 	}
1713 
1714 	/*
1715 	 * No merge.  Insert a new extent, preserving ordering.
1716 	 */
1717 	new_se = kmalloc(sizeof(*se), GFP_KERNEL);
1718 	if (new_se == NULL)
1719 		return -ENOMEM;
1720 	new_se->start_page = start_page;
1721 	new_se->nr_pages = nr_pages;
1722 	new_se->start_block = start_block;
1723 
1724 	list_add_tail(&new_se->list, &sis->first_swap_extent.list);
1725 	return 1;
1726 }
1727 
1728 /*
1729  * A `swap extent' is a simple thing which maps a contiguous range of pages
1730  * onto a contiguous range of disk blocks.  An ordered list of swap extents
1731  * is built at swapon time and is then used at swap_writepage/swap_readpage
1732  * time for locating where on disk a page belongs.
1733  *
1734  * If the swapfile is an S_ISBLK block device, a single extent is installed.
1735  * This is done so that the main operating code can treat S_ISBLK and S_ISREG
1736  * swap files identically.
1737  *
1738  * Whether the swapdev is an S_ISREG file or an S_ISBLK blockdev, the swap
1739  * extent list operates in PAGE_SIZE disk blocks.  Both S_ISREG and S_ISBLK
1740  * swapfiles are handled *identically* after swapon time.
1741  *
1742  * For S_ISREG swapfiles, setup_swap_extents() will walk all the file's blocks
1743  * and will parse them into an ordered extent list, in PAGE_SIZE chunks.  If
1744  * some stray blocks are found which do not fall within the PAGE_SIZE alignment
1745  * requirements, they are simply tossed out - we will never use those blocks
1746  * for swapping.
1747  *
1748  * For S_ISREG swapfiles we set S_SWAPFILE across the life of the swapon.  This
1749  * prevents root from shooting her foot off by ftruncating an in-use swapfile,
1750  * which will scribble on the fs.
1751  *
1752  * The amount of disk space which a single swap extent represents varies.
1753  * Typically it is in the 1-4 megabyte range.  So we can have hundreds of
1754  * extents in the list.  To avoid much list walking, we cache the previous
1755  * search location in `curr_swap_extent', and start new searches from there.
1756  * This is extremely effective.  The average number of iterations in
1757  * map_swap_page() has been measured at about 0.3 per page.  - akpm.
1758  */
setup_swap_extents(struct swap_info_struct * sis,sector_t * span)1759 static int setup_swap_extents(struct swap_info_struct *sis, sector_t *span)
1760 {
1761 	struct file *swap_file = sis->swap_file;
1762 	struct address_space *mapping = swap_file->f_mapping;
1763 	struct inode *inode = mapping->host;
1764 	int ret;
1765 
1766 	if (S_ISBLK(inode->i_mode)) {
1767 		ret = add_swap_extent(sis, 0, sis->max, 0);
1768 		*span = sis->pages;
1769 		return ret;
1770 	}
1771 
1772 	if (mapping->a_ops->swap_activate) {
1773 		ret = mapping->a_ops->swap_activate(sis, swap_file, span);
1774 		if (!ret) {
1775 			sis->flags |= SWP_FILE;
1776 			ret = add_swap_extent(sis, 0, sis->max, 0);
1777 			*span = sis->pages;
1778 		}
1779 		return ret;
1780 	}
1781 
1782 	return generic_swapfile_activate(sis, swap_file, span);
1783 }
1784 
_enable_swap_info(struct swap_info_struct * p,int prio,unsigned char * swap_map,struct swap_cluster_info * cluster_info)1785 static void _enable_swap_info(struct swap_info_struct *p, int prio,
1786 				unsigned char *swap_map,
1787 				struct swap_cluster_info *cluster_info)
1788 {
1789 	if (prio >= 0)
1790 		p->prio = prio;
1791 	else
1792 		p->prio = --least_priority;
1793 	/*
1794 	 * the plist prio is negated because plist ordering is
1795 	 * low-to-high, while swap ordering is high-to-low
1796 	 */
1797 	p->list.prio = -p->prio;
1798 	p->avail_list.prio = -p->prio;
1799 	p->swap_map = swap_map;
1800 	p->cluster_info = cluster_info;
1801 	p->flags |= SWP_WRITEOK;
1802 	atomic_long_add(p->pages, &nr_swap_pages);
1803 	total_swap_pages += p->pages;
1804 
1805 	assert_spin_locked(&swap_lock);
1806 	/*
1807 	 * both lists are plists, and thus priority ordered.
1808 	 * swap_active_head needs to be priority ordered for swapoff(),
1809 	 * which on removal of any swap_info_struct with an auto-assigned
1810 	 * (i.e. negative) priority increments the auto-assigned priority
1811 	 * of any lower-priority swap_info_structs.
1812 	 * swap_avail_head needs to be priority ordered for get_swap_page(),
1813 	 * which allocates swap pages from the highest available priority
1814 	 * swap_info_struct.
1815 	 */
1816 	plist_add(&p->list, &swap_active_head);
1817 	spin_lock(&swap_avail_lock);
1818 	plist_add(&p->avail_list, &swap_avail_head);
1819 	spin_unlock(&swap_avail_lock);
1820 }
1821 
enable_swap_info(struct swap_info_struct * p,int prio,unsigned char * swap_map,struct swap_cluster_info * cluster_info,unsigned long * frontswap_map)1822 static void enable_swap_info(struct swap_info_struct *p, int prio,
1823 				unsigned char *swap_map,
1824 				struct swap_cluster_info *cluster_info,
1825 				unsigned long *frontswap_map)
1826 {
1827 	frontswap_init(p->type, frontswap_map);
1828 	spin_lock(&swap_lock);
1829 	spin_lock(&p->lock);
1830 	 _enable_swap_info(p, prio, swap_map, cluster_info);
1831 	spin_unlock(&p->lock);
1832 	spin_unlock(&swap_lock);
1833 }
1834 
reinsert_swap_info(struct swap_info_struct * p)1835 static void reinsert_swap_info(struct swap_info_struct *p)
1836 {
1837 	spin_lock(&swap_lock);
1838 	spin_lock(&p->lock);
1839 	_enable_swap_info(p, p->prio, p->swap_map, p->cluster_info);
1840 	spin_unlock(&p->lock);
1841 	spin_unlock(&swap_lock);
1842 }
1843 
SYSCALL_DEFINE1(swapoff,const char __user *,specialfile)1844 SYSCALL_DEFINE1(swapoff, const char __user *, specialfile)
1845 {
1846 	struct swap_info_struct *p = NULL;
1847 	unsigned char *swap_map;
1848 	struct swap_cluster_info *cluster_info;
1849 	unsigned long *frontswap_map;
1850 	struct file *swap_file, *victim;
1851 	struct address_space *mapping;
1852 	struct inode *inode;
1853 	struct filename *pathname;
1854 	int err, found = 0;
1855 	unsigned int old_block_size;
1856 
1857 	if (!capable(CAP_SYS_ADMIN))
1858 		return -EPERM;
1859 
1860 	BUG_ON(!current->mm);
1861 
1862 	pathname = getname(specialfile);
1863 	if (IS_ERR(pathname))
1864 		return PTR_ERR(pathname);
1865 
1866 	victim = file_open_name(pathname, O_RDWR|O_LARGEFILE, 0);
1867 	err = PTR_ERR(victim);
1868 	if (IS_ERR(victim))
1869 		goto out;
1870 
1871 	mapping = victim->f_mapping;
1872 	spin_lock(&swap_lock);
1873 	plist_for_each_entry(p, &swap_active_head, list) {
1874 		if (p->flags & SWP_WRITEOK) {
1875 			if (p->swap_file->f_mapping == mapping) {
1876 				found = 1;
1877 				break;
1878 			}
1879 		}
1880 	}
1881 	if (!found) {
1882 		err = -EINVAL;
1883 		spin_unlock(&swap_lock);
1884 		goto out_dput;
1885 	}
1886 	if (!security_vm_enough_memory_mm(current->mm, p->pages))
1887 		vm_unacct_memory(p->pages);
1888 	else {
1889 		err = -ENOMEM;
1890 		spin_unlock(&swap_lock);
1891 		goto out_dput;
1892 	}
1893 	spin_lock(&swap_avail_lock);
1894 	plist_del(&p->avail_list, &swap_avail_head);
1895 	spin_unlock(&swap_avail_lock);
1896 	spin_lock(&p->lock);
1897 	if (p->prio < 0) {
1898 		struct swap_info_struct *si = p;
1899 
1900 		plist_for_each_entry_continue(si, &swap_active_head, list) {
1901 			si->prio++;
1902 			si->list.prio--;
1903 			si->avail_list.prio--;
1904 		}
1905 		least_priority++;
1906 	}
1907 	plist_del(&p->list, &swap_active_head);
1908 	atomic_long_sub(p->pages, &nr_swap_pages);
1909 	total_swap_pages -= p->pages;
1910 	p->flags &= ~SWP_WRITEOK;
1911 	spin_unlock(&p->lock);
1912 	spin_unlock(&swap_lock);
1913 
1914 	set_current_oom_origin();
1915 	err = try_to_unuse(p->type, false, 0); /* force unuse all pages */
1916 	clear_current_oom_origin();
1917 
1918 	if (err) {
1919 		/* re-insert swap space back into swap_list */
1920 		reinsert_swap_info(p);
1921 		goto out_dput;
1922 	}
1923 
1924 	flush_work(&p->discard_work);
1925 
1926 	destroy_swap_extents(p);
1927 	if (p->flags & SWP_CONTINUED)
1928 		free_swap_count_continuations(p);
1929 
1930 	mutex_lock(&swapon_mutex);
1931 	spin_lock(&swap_lock);
1932 	spin_lock(&p->lock);
1933 	drain_mmlist();
1934 
1935 	/* wait for anyone still in scan_swap_map */
1936 	p->highest_bit = 0;		/* cuts scans short */
1937 	while (p->flags >= SWP_SCANNING) {
1938 		spin_unlock(&p->lock);
1939 		spin_unlock(&swap_lock);
1940 		schedule_timeout_uninterruptible(1);
1941 		spin_lock(&swap_lock);
1942 		spin_lock(&p->lock);
1943 	}
1944 
1945 	swap_file = p->swap_file;
1946 	old_block_size = p->old_block_size;
1947 	p->swap_file = NULL;
1948 	p->max = 0;
1949 	swap_map = p->swap_map;
1950 	p->swap_map = NULL;
1951 	cluster_info = p->cluster_info;
1952 	p->cluster_info = NULL;
1953 	frontswap_map = frontswap_map_get(p);
1954 	spin_unlock(&p->lock);
1955 	spin_unlock(&swap_lock);
1956 	frontswap_invalidate_area(p->type);
1957 	frontswap_map_set(p, NULL);
1958 	mutex_unlock(&swapon_mutex);
1959 	free_percpu(p->percpu_cluster);
1960 	p->percpu_cluster = NULL;
1961 	vfree(swap_map);
1962 	vfree(cluster_info);
1963 	vfree(frontswap_map);
1964 	/* Destroy swap account information */
1965 	swap_cgroup_swapoff(p->type);
1966 
1967 	inode = mapping->host;
1968 	if (S_ISBLK(inode->i_mode)) {
1969 		struct block_device *bdev = I_BDEV(inode);
1970 		set_blocksize(bdev, old_block_size);
1971 		blkdev_put(bdev, FMODE_READ | FMODE_WRITE | FMODE_EXCL);
1972 	} else {
1973 		mutex_lock(&inode->i_mutex);
1974 		inode->i_flags &= ~S_SWAPFILE;
1975 		mutex_unlock(&inode->i_mutex);
1976 	}
1977 	filp_close(swap_file, NULL);
1978 
1979 	/*
1980 	 * Clear the SWP_USED flag after all resources are freed so that swapon
1981 	 * can reuse this swap_info in alloc_swap_info() safely.  It is ok to
1982 	 * not hold p->lock after we cleared its SWP_WRITEOK.
1983 	 */
1984 	spin_lock(&swap_lock);
1985 	p->flags = 0;
1986 	spin_unlock(&swap_lock);
1987 
1988 	err = 0;
1989 	atomic_inc(&proc_poll_event);
1990 	wake_up_interruptible(&proc_poll_wait);
1991 
1992 out_dput:
1993 	filp_close(victim, NULL);
1994 out:
1995 	putname(pathname);
1996 	return err;
1997 }
1998 
1999 #ifdef CONFIG_PROC_FS
swaps_poll(struct file * file,poll_table * wait)2000 static unsigned swaps_poll(struct file *file, poll_table *wait)
2001 {
2002 	struct seq_file *seq = file->private_data;
2003 
2004 	poll_wait(file, &proc_poll_wait, wait);
2005 
2006 	if (seq->poll_event != atomic_read(&proc_poll_event)) {
2007 		seq->poll_event = atomic_read(&proc_poll_event);
2008 		return POLLIN | POLLRDNORM | POLLERR | POLLPRI;
2009 	}
2010 
2011 	return POLLIN | POLLRDNORM;
2012 }
2013 
2014 /* iterator */
swap_start(struct seq_file * swap,loff_t * pos)2015 static void *swap_start(struct seq_file *swap, loff_t *pos)
2016 {
2017 	struct swap_info_struct *si;
2018 	int type;
2019 	loff_t l = *pos;
2020 
2021 	mutex_lock(&swapon_mutex);
2022 
2023 	if (!l)
2024 		return SEQ_START_TOKEN;
2025 
2026 	for (type = 0; type < nr_swapfiles; type++) {
2027 		smp_rmb();	/* read nr_swapfiles before swap_info[type] */
2028 		si = swap_info[type];
2029 		if (!(si->flags & SWP_USED) || !si->swap_map)
2030 			continue;
2031 		if (!--l)
2032 			return si;
2033 	}
2034 
2035 	return NULL;
2036 }
2037 
swap_next(struct seq_file * swap,void * v,loff_t * pos)2038 static void *swap_next(struct seq_file *swap, void *v, loff_t *pos)
2039 {
2040 	struct swap_info_struct *si = v;
2041 	int type;
2042 
2043 	if (v == SEQ_START_TOKEN)
2044 		type = 0;
2045 	else
2046 		type = si->type + 1;
2047 
2048 	for (; type < nr_swapfiles; type++) {
2049 		smp_rmb();	/* read nr_swapfiles before swap_info[type] */
2050 		si = swap_info[type];
2051 		if (!(si->flags & SWP_USED) || !si->swap_map)
2052 			continue;
2053 		++*pos;
2054 		return si;
2055 	}
2056 
2057 	return NULL;
2058 }
2059 
swap_stop(struct seq_file * swap,void * v)2060 static void swap_stop(struct seq_file *swap, void *v)
2061 {
2062 	mutex_unlock(&swapon_mutex);
2063 }
2064 
swap_show(struct seq_file * swap,void * v)2065 static int swap_show(struct seq_file *swap, void *v)
2066 {
2067 	struct swap_info_struct *si = v;
2068 	struct file *file;
2069 	int len;
2070 
2071 	if (si == SEQ_START_TOKEN) {
2072 		seq_puts(swap,"Filename\t\t\t\tType\t\tSize\tUsed\tPriority\n");
2073 		return 0;
2074 	}
2075 
2076 	file = si->swap_file;
2077 	len = seq_file_path(swap, file, " \t\n\\");
2078 	seq_printf(swap, "%*s%s\t%u\t%u\t%d\n",
2079 			len < 40 ? 40 - len : 1, " ",
2080 			S_ISBLK(file_inode(file)->i_mode) ?
2081 				"partition" : "file\t",
2082 			si->pages << (PAGE_SHIFT - 10),
2083 			si->inuse_pages << (PAGE_SHIFT - 10),
2084 			si->prio);
2085 	return 0;
2086 }
2087 
2088 static const struct seq_operations swaps_op = {
2089 	.start =	swap_start,
2090 	.next =		swap_next,
2091 	.stop =		swap_stop,
2092 	.show =		swap_show
2093 };
2094 
swaps_open(struct inode * inode,struct file * file)2095 static int swaps_open(struct inode *inode, struct file *file)
2096 {
2097 	struct seq_file *seq;
2098 	int ret;
2099 
2100 	ret = seq_open(file, &swaps_op);
2101 	if (ret)
2102 		return ret;
2103 
2104 	seq = file->private_data;
2105 	seq->poll_event = atomic_read(&proc_poll_event);
2106 	return 0;
2107 }
2108 
2109 static const struct file_operations proc_swaps_operations = {
2110 	.open		= swaps_open,
2111 	.read		= seq_read,
2112 	.llseek		= seq_lseek,
2113 	.release	= seq_release,
2114 	.poll		= swaps_poll,
2115 };
2116 
procswaps_init(void)2117 static int __init procswaps_init(void)
2118 {
2119 	proc_create("swaps", 0, NULL, &proc_swaps_operations);
2120 	return 0;
2121 }
2122 __initcall(procswaps_init);
2123 #endif /* CONFIG_PROC_FS */
2124 
2125 #ifdef MAX_SWAPFILES_CHECK
max_swapfiles_check(void)2126 static int __init max_swapfiles_check(void)
2127 {
2128 	MAX_SWAPFILES_CHECK();
2129 	return 0;
2130 }
2131 late_initcall(max_swapfiles_check);
2132 #endif
2133 
alloc_swap_info(void)2134 static struct swap_info_struct *alloc_swap_info(void)
2135 {
2136 	struct swap_info_struct *p;
2137 	unsigned int type;
2138 
2139 	p = kzalloc(sizeof(*p), GFP_KERNEL);
2140 	if (!p)
2141 		return ERR_PTR(-ENOMEM);
2142 
2143 	spin_lock(&swap_lock);
2144 	for (type = 0; type < nr_swapfiles; type++) {
2145 		if (!(swap_info[type]->flags & SWP_USED))
2146 			break;
2147 	}
2148 	if (type >= MAX_SWAPFILES) {
2149 		spin_unlock(&swap_lock);
2150 		kfree(p);
2151 		return ERR_PTR(-EPERM);
2152 	}
2153 	if (type >= nr_swapfiles) {
2154 		p->type = type;
2155 		swap_info[type] = p;
2156 		/*
2157 		 * Write swap_info[type] before nr_swapfiles, in case a
2158 		 * racing procfs swap_start() or swap_next() is reading them.
2159 		 * (We never shrink nr_swapfiles, we never free this entry.)
2160 		 */
2161 		smp_wmb();
2162 		nr_swapfiles++;
2163 	} else {
2164 		kfree(p);
2165 		p = swap_info[type];
2166 		/*
2167 		 * Do not memset this entry: a racing procfs swap_next()
2168 		 * would be relying on p->type to remain valid.
2169 		 */
2170 	}
2171 	INIT_LIST_HEAD(&p->first_swap_extent.list);
2172 	plist_node_init(&p->list, 0);
2173 	plist_node_init(&p->avail_list, 0);
2174 	p->flags = SWP_USED;
2175 	spin_unlock(&swap_lock);
2176 	spin_lock_init(&p->lock);
2177 
2178 	return p;
2179 }
2180 
claim_swapfile(struct swap_info_struct * p,struct inode * inode)2181 static int claim_swapfile(struct swap_info_struct *p, struct inode *inode)
2182 {
2183 	int error;
2184 
2185 	if (S_ISBLK(inode->i_mode)) {
2186 		p->bdev = bdgrab(I_BDEV(inode));
2187 		error = blkdev_get(p->bdev,
2188 				   FMODE_READ | FMODE_WRITE | FMODE_EXCL, p);
2189 		if (error < 0) {
2190 			p->bdev = NULL;
2191 			return error;
2192 		}
2193 		p->old_block_size = block_size(p->bdev);
2194 		error = set_blocksize(p->bdev, PAGE_SIZE);
2195 		if (error < 0)
2196 			return error;
2197 		p->flags |= SWP_BLKDEV;
2198 	} else if (S_ISREG(inode->i_mode)) {
2199 		p->bdev = inode->i_sb->s_bdev;
2200 		mutex_lock(&inode->i_mutex);
2201 		if (IS_SWAPFILE(inode))
2202 			return -EBUSY;
2203 	} else
2204 		return -EINVAL;
2205 
2206 	return 0;
2207 }
2208 
2209 
2210 /*
2211  * Find out how many pages are allowed for a single swap device. There
2212  * are two limiting factors:
2213  * 1) the number of bits for the swap offset in the swp_entry_t type, and
2214  * 2) the number of bits in the swap pte, as defined by the different
2215  * architectures.
2216  *
2217  * In order to find the largest possible bit mask, a swap entry with
2218  * swap type 0 and swap offset ~0UL is created, encoded to a swap pte,
2219  * decoded to a swp_entry_t again, and finally the swap offset is
2220  * extracted.
2221  *
2222  * This will mask all the bits from the initial ~0UL mask that can't
2223  * be encoded in either the swp_entry_t or the architecture definition
2224  * of a swap pte.
2225  */
generic_max_swapfile_size(void)2226 unsigned long generic_max_swapfile_size(void)
2227 {
2228 	return swp_offset(pte_to_swp_entry(
2229 			swp_entry_to_pte(swp_entry(0, ~0UL)))) + 1;
2230 }
2231 
2232 /* Can be overridden by an architecture for additional checks. */
max_swapfile_size(void)2233 __weak unsigned long max_swapfile_size(void)
2234 {
2235 	return generic_max_swapfile_size();
2236 }
2237 
read_swap_header(struct swap_info_struct * p,union swap_header * swap_header,struct inode * inode)2238 static unsigned long read_swap_header(struct swap_info_struct *p,
2239 					union swap_header *swap_header,
2240 					struct inode *inode)
2241 {
2242 	int i;
2243 	unsigned long maxpages;
2244 	unsigned long swapfilepages;
2245 	unsigned long last_page;
2246 
2247 	if (memcmp("SWAPSPACE2", swap_header->magic.magic, 10)) {
2248 		pr_err("Unable to find swap-space signature\n");
2249 		return 0;
2250 	}
2251 
2252 	/* swap partition endianess hack... */
2253 	if (swab32(swap_header->info.version) == 1) {
2254 		swab32s(&swap_header->info.version);
2255 		swab32s(&swap_header->info.last_page);
2256 		swab32s(&swap_header->info.nr_badpages);
2257 		if (swap_header->info.nr_badpages > MAX_SWAP_BADPAGES)
2258 			return 0;
2259 		for (i = 0; i < swap_header->info.nr_badpages; i++)
2260 			swab32s(&swap_header->info.badpages[i]);
2261 	}
2262 	/* Check the swap header's sub-version */
2263 	if (swap_header->info.version != 1) {
2264 		pr_warn("Unable to handle swap header version %d\n",
2265 			swap_header->info.version);
2266 		return 0;
2267 	}
2268 
2269 	p->lowest_bit  = 1;
2270 	p->cluster_next = 1;
2271 	p->cluster_nr = 0;
2272 
2273 	maxpages = max_swapfile_size();
2274 	last_page = swap_header->info.last_page;
2275 	if (!last_page) {
2276 		pr_warn("Empty swap-file\n");
2277 		return 0;
2278 	}
2279 	if (last_page > maxpages) {
2280 		pr_warn("Truncating oversized swap area, only using %luk out of %luk\n",
2281 			maxpages << (PAGE_SHIFT - 10),
2282 			last_page << (PAGE_SHIFT - 10));
2283 	}
2284 	if (maxpages > last_page) {
2285 		maxpages = last_page + 1;
2286 		/* p->max is an unsigned int: don't overflow it */
2287 		if ((unsigned int)maxpages == 0)
2288 			maxpages = UINT_MAX;
2289 	}
2290 	p->highest_bit = maxpages - 1;
2291 
2292 	if (!maxpages)
2293 		return 0;
2294 	swapfilepages = i_size_read(inode) >> PAGE_SHIFT;
2295 	if (swapfilepages && maxpages > swapfilepages) {
2296 		pr_warn("Swap area shorter than signature indicates\n");
2297 		return 0;
2298 	}
2299 	if (swap_header->info.nr_badpages && S_ISREG(inode->i_mode))
2300 		return 0;
2301 	if (swap_header->info.nr_badpages > MAX_SWAP_BADPAGES)
2302 		return 0;
2303 
2304 	return maxpages;
2305 }
2306 
setup_swap_map_and_extents(struct swap_info_struct * p,union swap_header * swap_header,unsigned char * swap_map,struct swap_cluster_info * cluster_info,unsigned long maxpages,sector_t * span)2307 static int setup_swap_map_and_extents(struct swap_info_struct *p,
2308 					union swap_header *swap_header,
2309 					unsigned char *swap_map,
2310 					struct swap_cluster_info *cluster_info,
2311 					unsigned long maxpages,
2312 					sector_t *span)
2313 {
2314 	int i;
2315 	unsigned int nr_good_pages;
2316 	int nr_extents;
2317 	unsigned long nr_clusters = DIV_ROUND_UP(maxpages, SWAPFILE_CLUSTER);
2318 	unsigned long idx = p->cluster_next / SWAPFILE_CLUSTER;
2319 
2320 	nr_good_pages = maxpages - 1;	/* omit header page */
2321 
2322 	cluster_set_null(&p->free_cluster_head);
2323 	cluster_set_null(&p->free_cluster_tail);
2324 	cluster_set_null(&p->discard_cluster_head);
2325 	cluster_set_null(&p->discard_cluster_tail);
2326 
2327 	for (i = 0; i < swap_header->info.nr_badpages; i++) {
2328 		unsigned int page_nr = swap_header->info.badpages[i];
2329 		if (page_nr == 0 || page_nr > swap_header->info.last_page)
2330 			return -EINVAL;
2331 		if (page_nr < maxpages) {
2332 			swap_map[page_nr] = SWAP_MAP_BAD;
2333 			nr_good_pages--;
2334 			/*
2335 			 * Haven't marked the cluster free yet, no list
2336 			 * operation involved
2337 			 */
2338 			inc_cluster_info_page(p, cluster_info, page_nr);
2339 		}
2340 	}
2341 
2342 	/* Haven't marked the cluster free yet, no list operation involved */
2343 	for (i = maxpages; i < round_up(maxpages, SWAPFILE_CLUSTER); i++)
2344 		inc_cluster_info_page(p, cluster_info, i);
2345 
2346 	if (nr_good_pages) {
2347 		swap_map[0] = SWAP_MAP_BAD;
2348 		/*
2349 		 * Not mark the cluster free yet, no list
2350 		 * operation involved
2351 		 */
2352 		inc_cluster_info_page(p, cluster_info, 0);
2353 		p->max = maxpages;
2354 		p->pages = nr_good_pages;
2355 		nr_extents = setup_swap_extents(p, span);
2356 		if (nr_extents < 0)
2357 			return nr_extents;
2358 		nr_good_pages = p->pages;
2359 	}
2360 	if (!nr_good_pages) {
2361 		pr_warn("Empty swap-file\n");
2362 		return -EINVAL;
2363 	}
2364 
2365 	if (!cluster_info)
2366 		return nr_extents;
2367 
2368 	for (i = 0; i < nr_clusters; i++) {
2369 		if (!cluster_count(&cluster_info[idx])) {
2370 			cluster_set_flag(&cluster_info[idx], CLUSTER_FLAG_FREE);
2371 			if (cluster_is_null(&p->free_cluster_head)) {
2372 				cluster_set_next_flag(&p->free_cluster_head,
2373 								idx, 0);
2374 				cluster_set_next_flag(&p->free_cluster_tail,
2375 								idx, 0);
2376 			} else {
2377 				unsigned int tail;
2378 
2379 				tail = cluster_next(&p->free_cluster_tail);
2380 				cluster_set_next(&cluster_info[tail], idx);
2381 				cluster_set_next_flag(&p->free_cluster_tail,
2382 								idx, 0);
2383 			}
2384 		}
2385 		idx++;
2386 		if (idx == nr_clusters)
2387 			idx = 0;
2388 	}
2389 	return nr_extents;
2390 }
2391 
2392 /*
2393  * Helper to sys_swapon determining if a given swap
2394  * backing device queue supports DISCARD operations.
2395  */
swap_discardable(struct swap_info_struct * si)2396 static bool swap_discardable(struct swap_info_struct *si)
2397 {
2398 	struct request_queue *q = bdev_get_queue(si->bdev);
2399 
2400 	if (!q || !blk_queue_discard(q))
2401 		return false;
2402 
2403 	return true;
2404 }
2405 
SYSCALL_DEFINE2(swapon,const char __user *,specialfile,int,swap_flags)2406 SYSCALL_DEFINE2(swapon, const char __user *, specialfile, int, swap_flags)
2407 {
2408 	struct swap_info_struct *p;
2409 	struct filename *name;
2410 	struct file *swap_file = NULL;
2411 	struct address_space *mapping;
2412 	int prio;
2413 	int error;
2414 	union swap_header *swap_header;
2415 	int nr_extents;
2416 	sector_t span;
2417 	unsigned long maxpages;
2418 	unsigned char *swap_map = NULL;
2419 	struct swap_cluster_info *cluster_info = NULL;
2420 	unsigned long *frontswap_map = NULL;
2421 	struct page *page = NULL;
2422 	struct inode *inode = NULL;
2423 
2424 	if (swap_flags & ~SWAP_FLAGS_VALID)
2425 		return -EINVAL;
2426 
2427 	if (!capable(CAP_SYS_ADMIN))
2428 		return -EPERM;
2429 
2430 	p = alloc_swap_info();
2431 	if (IS_ERR(p))
2432 		return PTR_ERR(p);
2433 
2434 	INIT_WORK(&p->discard_work, swap_discard_work);
2435 
2436 	name = getname(specialfile);
2437 	if (IS_ERR(name)) {
2438 		error = PTR_ERR(name);
2439 		name = NULL;
2440 		goto bad_swap;
2441 	}
2442 	swap_file = file_open_name(name, O_RDWR|O_LARGEFILE, 0);
2443 	if (IS_ERR(swap_file)) {
2444 		error = PTR_ERR(swap_file);
2445 		swap_file = NULL;
2446 		goto bad_swap;
2447 	}
2448 
2449 	p->swap_file = swap_file;
2450 	mapping = swap_file->f_mapping;
2451 	inode = mapping->host;
2452 
2453 	/* If S_ISREG(inode->i_mode) will do mutex_lock(&inode->i_mutex); */
2454 	error = claim_swapfile(p, inode);
2455 	if (unlikely(error))
2456 		goto bad_swap;
2457 
2458 	/*
2459 	 * Read the swap header.
2460 	 */
2461 	if (!mapping->a_ops->readpage) {
2462 		error = -EINVAL;
2463 		goto bad_swap;
2464 	}
2465 	page = read_mapping_page(mapping, 0, swap_file);
2466 	if (IS_ERR(page)) {
2467 		error = PTR_ERR(page);
2468 		goto bad_swap;
2469 	}
2470 	swap_header = kmap(page);
2471 
2472 	maxpages = read_swap_header(p, swap_header, inode);
2473 	if (unlikely(!maxpages)) {
2474 		error = -EINVAL;
2475 		goto bad_swap;
2476 	}
2477 
2478 	/* OK, set up the swap map and apply the bad block list */
2479 	swap_map = vzalloc(maxpages);
2480 	if (!swap_map) {
2481 		error = -ENOMEM;
2482 		goto bad_swap;
2483 	}
2484 	if (p->bdev && blk_queue_nonrot(bdev_get_queue(p->bdev))) {
2485 		int cpu;
2486 
2487 		p->flags |= SWP_SOLIDSTATE;
2488 		/*
2489 		 * select a random position to start with to help wear leveling
2490 		 * SSD
2491 		 */
2492 		p->cluster_next = 1 + (prandom_u32() % p->highest_bit);
2493 
2494 		cluster_info = vzalloc(DIV_ROUND_UP(maxpages,
2495 			SWAPFILE_CLUSTER) * sizeof(*cluster_info));
2496 		if (!cluster_info) {
2497 			error = -ENOMEM;
2498 			goto bad_swap;
2499 		}
2500 		p->percpu_cluster = alloc_percpu(struct percpu_cluster);
2501 		if (!p->percpu_cluster) {
2502 			error = -ENOMEM;
2503 			goto bad_swap;
2504 		}
2505 		for_each_possible_cpu(cpu) {
2506 			struct percpu_cluster *cluster;
2507 			cluster = per_cpu_ptr(p->percpu_cluster, cpu);
2508 			cluster_set_null(&cluster->index);
2509 		}
2510 	}
2511 
2512 	error = swap_cgroup_swapon(p->type, maxpages);
2513 	if (error)
2514 		goto bad_swap;
2515 
2516 	nr_extents = setup_swap_map_and_extents(p, swap_header, swap_map,
2517 		cluster_info, maxpages, &span);
2518 	if (unlikely(nr_extents < 0)) {
2519 		error = nr_extents;
2520 		goto bad_swap;
2521 	}
2522 	/* frontswap enabled? set up bit-per-page map for frontswap */
2523 	if (frontswap_enabled)
2524 		frontswap_map = vzalloc(BITS_TO_LONGS(maxpages) * sizeof(long));
2525 
2526 	if (p->bdev &&(swap_flags & SWAP_FLAG_DISCARD) && swap_discardable(p)) {
2527 		/*
2528 		 * When discard is enabled for swap with no particular
2529 		 * policy flagged, we set all swap discard flags here in
2530 		 * order to sustain backward compatibility with older
2531 		 * swapon(8) releases.
2532 		 */
2533 		p->flags |= (SWP_DISCARDABLE | SWP_AREA_DISCARD |
2534 			     SWP_PAGE_DISCARD);
2535 
2536 		/*
2537 		 * By flagging sys_swapon, a sysadmin can tell us to
2538 		 * either do single-time area discards only, or to just
2539 		 * perform discards for released swap page-clusters.
2540 		 * Now it's time to adjust the p->flags accordingly.
2541 		 */
2542 		if (swap_flags & SWAP_FLAG_DISCARD_ONCE)
2543 			p->flags &= ~SWP_PAGE_DISCARD;
2544 		else if (swap_flags & SWAP_FLAG_DISCARD_PAGES)
2545 			p->flags &= ~SWP_AREA_DISCARD;
2546 
2547 		/* issue a swapon-time discard if it's still required */
2548 		if (p->flags & SWP_AREA_DISCARD) {
2549 			int err = discard_swap(p);
2550 			if (unlikely(err))
2551 				pr_err("swapon: discard_swap(%p): %d\n",
2552 					p, err);
2553 		}
2554 	}
2555 
2556 	mutex_lock(&swapon_mutex);
2557 	prio = -1;
2558 	if (swap_flags & SWAP_FLAG_PREFER)
2559 		prio =
2560 		  (swap_flags & SWAP_FLAG_PRIO_MASK) >> SWAP_FLAG_PRIO_SHIFT;
2561 	enable_swap_info(p, prio, swap_map, cluster_info, frontswap_map);
2562 
2563 	pr_info("Adding %uk swap on %s.  Priority:%d extents:%d across:%lluk %s%s%s%s%s\n",
2564 		p->pages<<(PAGE_SHIFT-10), name->name, p->prio,
2565 		nr_extents, (unsigned long long)span<<(PAGE_SHIFT-10),
2566 		(p->flags & SWP_SOLIDSTATE) ? "SS" : "",
2567 		(p->flags & SWP_DISCARDABLE) ? "D" : "",
2568 		(p->flags & SWP_AREA_DISCARD) ? "s" : "",
2569 		(p->flags & SWP_PAGE_DISCARD) ? "c" : "",
2570 		(frontswap_map) ? "FS" : "");
2571 
2572 	mutex_unlock(&swapon_mutex);
2573 	atomic_inc(&proc_poll_event);
2574 	wake_up_interruptible(&proc_poll_wait);
2575 
2576 	if (S_ISREG(inode->i_mode))
2577 		inode->i_flags |= S_SWAPFILE;
2578 	error = 0;
2579 	goto out;
2580 bad_swap:
2581 	free_percpu(p->percpu_cluster);
2582 	p->percpu_cluster = NULL;
2583 	if (inode && S_ISBLK(inode->i_mode) && p->bdev) {
2584 		set_blocksize(p->bdev, p->old_block_size);
2585 		blkdev_put(p->bdev, FMODE_READ | FMODE_WRITE | FMODE_EXCL);
2586 	}
2587 	destroy_swap_extents(p);
2588 	swap_cgroup_swapoff(p->type);
2589 	spin_lock(&swap_lock);
2590 	p->swap_file = NULL;
2591 	p->flags = 0;
2592 	spin_unlock(&swap_lock);
2593 	vfree(swap_map);
2594 	vfree(cluster_info);
2595 	if (swap_file) {
2596 		if (inode && S_ISREG(inode->i_mode)) {
2597 			mutex_unlock(&inode->i_mutex);
2598 			inode = NULL;
2599 		}
2600 		filp_close(swap_file, NULL);
2601 	}
2602 out:
2603 	if (page && !IS_ERR(page)) {
2604 		kunmap(page);
2605 		page_cache_release(page);
2606 	}
2607 	if (name)
2608 		putname(name);
2609 	if (inode && S_ISREG(inode->i_mode))
2610 		mutex_unlock(&inode->i_mutex);
2611 	return error;
2612 }
2613 
si_swapinfo(struct sysinfo * val)2614 void si_swapinfo(struct sysinfo *val)
2615 {
2616 	unsigned int type;
2617 	unsigned long nr_to_be_unused = 0;
2618 
2619 	spin_lock(&swap_lock);
2620 	for (type = 0; type < nr_swapfiles; type++) {
2621 		struct swap_info_struct *si = swap_info[type];
2622 
2623 		if ((si->flags & SWP_USED) && !(si->flags & SWP_WRITEOK))
2624 			nr_to_be_unused += si->inuse_pages;
2625 	}
2626 	val->freeswap = atomic_long_read(&nr_swap_pages) + nr_to_be_unused;
2627 	val->totalswap = total_swap_pages + nr_to_be_unused;
2628 	spin_unlock(&swap_lock);
2629 }
2630 
2631 /*
2632  * Verify that a swap entry is valid and increment its swap map count.
2633  *
2634  * Returns error code in following case.
2635  * - success -> 0
2636  * - swp_entry is invalid -> EINVAL
2637  * - swp_entry is migration entry -> EINVAL
2638  * - swap-cache reference is requested but there is already one. -> EEXIST
2639  * - swap-cache reference is requested but the entry is not used. -> ENOENT
2640  * - swap-mapped reference requested but needs continued swap count. -> ENOMEM
2641  */
__swap_duplicate(swp_entry_t entry,unsigned char usage)2642 static int __swap_duplicate(swp_entry_t entry, unsigned char usage)
2643 {
2644 	struct swap_info_struct *p;
2645 	unsigned long offset, type;
2646 	unsigned char count;
2647 	unsigned char has_cache;
2648 	int err = -EINVAL;
2649 
2650 	if (non_swap_entry(entry))
2651 		goto out;
2652 
2653 	type = swp_type(entry);
2654 	if (type >= nr_swapfiles)
2655 		goto bad_file;
2656 	p = swap_info[type];
2657 	offset = swp_offset(entry);
2658 
2659 	spin_lock(&p->lock);
2660 	if (unlikely(offset >= p->max))
2661 		goto unlock_out;
2662 
2663 	count = p->swap_map[offset];
2664 
2665 	/*
2666 	 * swapin_readahead() doesn't check if a swap entry is valid, so the
2667 	 * swap entry could be SWAP_MAP_BAD. Check here with lock held.
2668 	 */
2669 	if (unlikely(swap_count(count) == SWAP_MAP_BAD)) {
2670 		err = -ENOENT;
2671 		goto unlock_out;
2672 	}
2673 
2674 	has_cache = count & SWAP_HAS_CACHE;
2675 	count &= ~SWAP_HAS_CACHE;
2676 	err = 0;
2677 
2678 	if (usage == SWAP_HAS_CACHE) {
2679 
2680 		/* set SWAP_HAS_CACHE if there is no cache and entry is used */
2681 		if (!has_cache && count)
2682 			has_cache = SWAP_HAS_CACHE;
2683 		else if (has_cache)		/* someone else added cache */
2684 			err = -EEXIST;
2685 		else				/* no users remaining */
2686 			err = -ENOENT;
2687 
2688 	} else if (count || has_cache) {
2689 
2690 		if ((count & ~COUNT_CONTINUED) < SWAP_MAP_MAX)
2691 			count += usage;
2692 		else if ((count & ~COUNT_CONTINUED) > SWAP_MAP_MAX)
2693 			err = -EINVAL;
2694 		else if (swap_count_continued(p, offset, count))
2695 			count = COUNT_CONTINUED;
2696 		else
2697 			err = -ENOMEM;
2698 	} else
2699 		err = -ENOENT;			/* unused swap entry */
2700 
2701 	p->swap_map[offset] = count | has_cache;
2702 
2703 unlock_out:
2704 	spin_unlock(&p->lock);
2705 out:
2706 	return err;
2707 
2708 bad_file:
2709 	pr_err("swap_dup: %s%08lx\n", Bad_file, entry.val);
2710 	goto out;
2711 }
2712 
2713 /*
2714  * Help swapoff by noting that swap entry belongs to shmem/tmpfs
2715  * (in which case its reference count is never incremented).
2716  */
swap_shmem_alloc(swp_entry_t entry)2717 void swap_shmem_alloc(swp_entry_t entry)
2718 {
2719 	__swap_duplicate(entry, SWAP_MAP_SHMEM);
2720 }
2721 
2722 /*
2723  * Increase reference count of swap entry by 1.
2724  * Returns 0 for success, or -ENOMEM if a swap_count_continuation is required
2725  * but could not be atomically allocated.  Returns 0, just as if it succeeded,
2726  * if __swap_duplicate() fails for another reason (-EINVAL or -ENOENT), which
2727  * might occur if a page table entry has got corrupted.
2728  */
swap_duplicate(swp_entry_t entry)2729 int swap_duplicate(swp_entry_t entry)
2730 {
2731 	int err = 0;
2732 
2733 	while (!err && __swap_duplicate(entry, 1) == -ENOMEM)
2734 		err = add_swap_count_continuation(entry, GFP_ATOMIC);
2735 	return err;
2736 }
2737 
2738 /*
2739  * @entry: swap entry for which we allocate swap cache.
2740  *
2741  * Called when allocating swap cache for existing swap entry,
2742  * This can return error codes. Returns 0 at success.
2743  * -EBUSY means there is a swap cache.
2744  * Note: return code is different from swap_duplicate().
2745  */
swapcache_prepare(swp_entry_t entry)2746 int swapcache_prepare(swp_entry_t entry)
2747 {
2748 	return __swap_duplicate(entry, SWAP_HAS_CACHE);
2749 }
2750 
page_swap_info(struct page * page)2751 struct swap_info_struct *page_swap_info(struct page *page)
2752 {
2753 	swp_entry_t swap = { .val = page_private(page) };
2754 	BUG_ON(!PageSwapCache(page));
2755 	return swap_info[swp_type(swap)];
2756 }
2757 
2758 /*
2759  * out-of-line __page_file_ methods to avoid include hell.
2760  */
__page_file_mapping(struct page * page)2761 struct address_space *__page_file_mapping(struct page *page)
2762 {
2763 	VM_BUG_ON_PAGE(!PageSwapCache(page), page);
2764 	return page_swap_info(page)->swap_file->f_mapping;
2765 }
2766 EXPORT_SYMBOL_GPL(__page_file_mapping);
2767 
__page_file_index(struct page * page)2768 pgoff_t __page_file_index(struct page *page)
2769 {
2770 	swp_entry_t swap = { .val = page_private(page) };
2771 	VM_BUG_ON_PAGE(!PageSwapCache(page), page);
2772 	return swp_offset(swap);
2773 }
2774 EXPORT_SYMBOL_GPL(__page_file_index);
2775 
2776 /*
2777  * add_swap_count_continuation - called when a swap count is duplicated
2778  * beyond SWAP_MAP_MAX, it allocates a new page and links that to the entry's
2779  * page of the original vmalloc'ed swap_map, to hold the continuation count
2780  * (for that entry and for its neighbouring PAGE_SIZE swap entries).  Called
2781  * again when count is duplicated beyond SWAP_MAP_MAX * SWAP_CONT_MAX, etc.
2782  *
2783  * These continuation pages are seldom referenced: the common paths all work
2784  * on the original swap_map, only referring to a continuation page when the
2785  * low "digit" of a count is incremented or decremented through SWAP_MAP_MAX.
2786  *
2787  * add_swap_count_continuation(, GFP_ATOMIC) can be called while holding
2788  * page table locks; if it fails, add_swap_count_continuation(, GFP_KERNEL)
2789  * can be called after dropping locks.
2790  */
add_swap_count_continuation(swp_entry_t entry,gfp_t gfp_mask)2791 int add_swap_count_continuation(swp_entry_t entry, gfp_t gfp_mask)
2792 {
2793 	struct swap_info_struct *si;
2794 	struct page *head;
2795 	struct page *page;
2796 	struct page *list_page;
2797 	pgoff_t offset;
2798 	unsigned char count;
2799 
2800 	/*
2801 	 * When debugging, it's easier to use __GFP_ZERO here; but it's better
2802 	 * for latency not to zero a page while GFP_ATOMIC and holding locks.
2803 	 */
2804 	page = alloc_page(gfp_mask | __GFP_HIGHMEM);
2805 
2806 	si = swap_info_get(entry);
2807 	if (!si) {
2808 		/*
2809 		 * An acceptable race has occurred since the failing
2810 		 * __swap_duplicate(): the swap entry has been freed,
2811 		 * perhaps even the whole swap_map cleared for swapoff.
2812 		 */
2813 		goto outer;
2814 	}
2815 
2816 	offset = swp_offset(entry);
2817 	count = si->swap_map[offset] & ~SWAP_HAS_CACHE;
2818 
2819 	if ((count & ~COUNT_CONTINUED) != SWAP_MAP_MAX) {
2820 		/*
2821 		 * The higher the swap count, the more likely it is that tasks
2822 		 * will race to add swap count continuation: we need to avoid
2823 		 * over-provisioning.
2824 		 */
2825 		goto out;
2826 	}
2827 
2828 	if (!page) {
2829 		spin_unlock(&si->lock);
2830 		return -ENOMEM;
2831 	}
2832 
2833 	/*
2834 	 * We are fortunate that although vmalloc_to_page uses pte_offset_map,
2835 	 * no architecture is using highmem pages for kernel page tables: so it
2836 	 * will not corrupt the GFP_ATOMIC caller's atomic page table kmaps.
2837 	 */
2838 	head = vmalloc_to_page(si->swap_map + offset);
2839 	offset &= ~PAGE_MASK;
2840 
2841 	/*
2842 	 * Page allocation does not initialize the page's lru field,
2843 	 * but it does always reset its private field.
2844 	 */
2845 	if (!page_private(head)) {
2846 		BUG_ON(count & COUNT_CONTINUED);
2847 		INIT_LIST_HEAD(&head->lru);
2848 		set_page_private(head, SWP_CONTINUED);
2849 		si->flags |= SWP_CONTINUED;
2850 	}
2851 
2852 	list_for_each_entry(list_page, &head->lru, lru) {
2853 		unsigned char *map;
2854 
2855 		/*
2856 		 * If the previous map said no continuation, but we've found
2857 		 * a continuation page, free our allocation and use this one.
2858 		 */
2859 		if (!(count & COUNT_CONTINUED))
2860 			goto out;
2861 
2862 		map = kmap_atomic(list_page) + offset;
2863 		count = *map;
2864 		kunmap_atomic(map);
2865 
2866 		/*
2867 		 * If this continuation count now has some space in it,
2868 		 * free our allocation and use this one.
2869 		 */
2870 		if ((count & ~COUNT_CONTINUED) != SWAP_CONT_MAX)
2871 			goto out;
2872 	}
2873 
2874 	list_add_tail(&page->lru, &head->lru);
2875 	page = NULL;			/* now it's attached, don't free it */
2876 out:
2877 	spin_unlock(&si->lock);
2878 outer:
2879 	if (page)
2880 		__free_page(page);
2881 	return 0;
2882 }
2883 
2884 /*
2885  * swap_count_continued - when the original swap_map count is incremented
2886  * from SWAP_MAP_MAX, check if there is already a continuation page to carry
2887  * into, carry if so, or else fail until a new continuation page is allocated;
2888  * when the original swap_map count is decremented from 0 with continuation,
2889  * borrow from the continuation and report whether it still holds more.
2890  * Called while __swap_duplicate() or swap_entry_free() holds swap_lock.
2891  */
swap_count_continued(struct swap_info_struct * si,pgoff_t offset,unsigned char count)2892 static bool swap_count_continued(struct swap_info_struct *si,
2893 				 pgoff_t offset, unsigned char count)
2894 {
2895 	struct page *head;
2896 	struct page *page;
2897 	unsigned char *map;
2898 
2899 	head = vmalloc_to_page(si->swap_map + offset);
2900 	if (page_private(head) != SWP_CONTINUED) {
2901 		BUG_ON(count & COUNT_CONTINUED);
2902 		return false;		/* need to add count continuation */
2903 	}
2904 
2905 	offset &= ~PAGE_MASK;
2906 	page = list_entry(head->lru.next, struct page, lru);
2907 	map = kmap_atomic(page) + offset;
2908 
2909 	if (count == SWAP_MAP_MAX)	/* initial increment from swap_map */
2910 		goto init_map;		/* jump over SWAP_CONT_MAX checks */
2911 
2912 	if (count == (SWAP_MAP_MAX | COUNT_CONTINUED)) { /* incrementing */
2913 		/*
2914 		 * Think of how you add 1 to 999
2915 		 */
2916 		while (*map == (SWAP_CONT_MAX | COUNT_CONTINUED)) {
2917 			kunmap_atomic(map);
2918 			page = list_entry(page->lru.next, struct page, lru);
2919 			BUG_ON(page == head);
2920 			map = kmap_atomic(page) + offset;
2921 		}
2922 		if (*map == SWAP_CONT_MAX) {
2923 			kunmap_atomic(map);
2924 			page = list_entry(page->lru.next, struct page, lru);
2925 			if (page == head)
2926 				return false;	/* add count continuation */
2927 			map = kmap_atomic(page) + offset;
2928 init_map:		*map = 0;		/* we didn't zero the page */
2929 		}
2930 		*map += 1;
2931 		kunmap_atomic(map);
2932 		page = list_entry(page->lru.prev, struct page, lru);
2933 		while (page != head) {
2934 			map = kmap_atomic(page) + offset;
2935 			*map = COUNT_CONTINUED;
2936 			kunmap_atomic(map);
2937 			page = list_entry(page->lru.prev, struct page, lru);
2938 		}
2939 		return true;			/* incremented */
2940 
2941 	} else {				/* decrementing */
2942 		/*
2943 		 * Think of how you subtract 1 from 1000
2944 		 */
2945 		BUG_ON(count != COUNT_CONTINUED);
2946 		while (*map == COUNT_CONTINUED) {
2947 			kunmap_atomic(map);
2948 			page = list_entry(page->lru.next, struct page, lru);
2949 			BUG_ON(page == head);
2950 			map = kmap_atomic(page) + offset;
2951 		}
2952 		BUG_ON(*map == 0);
2953 		*map -= 1;
2954 		if (*map == 0)
2955 			count = 0;
2956 		kunmap_atomic(map);
2957 		page = list_entry(page->lru.prev, struct page, lru);
2958 		while (page != head) {
2959 			map = kmap_atomic(page) + offset;
2960 			*map = SWAP_CONT_MAX | count;
2961 			count = COUNT_CONTINUED;
2962 			kunmap_atomic(map);
2963 			page = list_entry(page->lru.prev, struct page, lru);
2964 		}
2965 		return count == COUNT_CONTINUED;
2966 	}
2967 }
2968 
2969 /*
2970  * free_swap_count_continuations - swapoff free all the continuation pages
2971  * appended to the swap_map, after swap_map is quiesced, before vfree'ing it.
2972  */
free_swap_count_continuations(struct swap_info_struct * si)2973 static void free_swap_count_continuations(struct swap_info_struct *si)
2974 {
2975 	pgoff_t offset;
2976 
2977 	for (offset = 0; offset < si->max; offset += PAGE_SIZE) {
2978 		struct page *head;
2979 		head = vmalloc_to_page(si->swap_map + offset);
2980 		if (page_private(head)) {
2981 			struct list_head *this, *next;
2982 			list_for_each_safe(this, next, &head->lru) {
2983 				struct page *page;
2984 				page = list_entry(this, struct page, lru);
2985 				list_del(this);
2986 				__free_page(page);
2987 			}
2988 		}
2989 	}
2990 }
2991