1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * linux/kernel/power/swap.c
4 *
5 * This file provides functions for reading the suspend image from
6 * and writing it to a swap partition.
7 *
8 * Copyright (C) 1998,2001-2005 Pavel Machek <pavel@ucw.cz>
9 * Copyright (C) 2006 Rafael J. Wysocki <rjw@sisk.pl>
10 * Copyright (C) 2010-2012 Bojan Smojver <bojan@rexursive.com>
11 */
12
13 #define pr_fmt(fmt) "PM: " fmt
14
15 #include <linux/module.h>
16 #include <linux/file.h>
17 #include <linux/delay.h>
18 #include <linux/bitops.h>
19 #include <linux/genhd.h>
20 #include <linux/device.h>
21 #include <linux/bio.h>
22 #include <linux/blkdev.h>
23 #include <linux/swap.h>
24 #include <linux/swapops.h>
25 #include <linux/pm.h>
26 #include <linux/slab.h>
27 #include <linux/lzo.h>
28 #include <linux/vmalloc.h>
29 #include <linux/cpumask.h>
30 #include <linux/atomic.h>
31 #include <linux/kthread.h>
32 #include <linux/crc32.h>
33 #include <linux/ktime.h>
34 #include <trace/hooks/bl_hib.h>
35
36 #include "power.h"
37
38 #define HIBERNATE_SIG "S1SUSPEND"
39
40 /*
41 * When reading an {un,}compressed image, we may restore pages in place,
42 * in which case some architectures need these pages cleaning before they
43 * can be executed. We don't know which pages these may be, so clean the lot.
44 */
45 static bool clean_pages_on_read;
46 static bool clean_pages_on_decompress;
47
48 /*
49 * The swap map is a data structure used for keeping track of each page
50 * written to a swap partition. It consists of many swap_map_page
51 * structures that contain each an array of MAP_PAGE_ENTRIES swap entries.
52 * These structures are stored on the swap and linked together with the
53 * help of the .next_swap member.
54 *
55 * The swap map is created during suspend. The swap map pages are
56 * allocated and populated one at a time, so we only need one memory
57 * page to set up the entire structure.
58 *
59 * During resume we pick up all swap_map_page structures into a list.
60 */
61
62 #define MAP_PAGE_ENTRIES (PAGE_SIZE / sizeof(sector_t) - 1)
63
64 /*
65 * Number of free pages that are not high.
66 */
low_free_pages(void)67 static inline unsigned long low_free_pages(void)
68 {
69 return nr_free_pages() - nr_free_highpages();
70 }
71
72 /*
73 * Number of pages required to be kept free while writing the image. Always
74 * half of all available low pages before the writing starts.
75 */
reqd_free_pages(void)76 static inline unsigned long reqd_free_pages(void)
77 {
78 return low_free_pages() / 2;
79 }
80
81 struct swap_map_page {
82 sector_t entries[MAP_PAGE_ENTRIES];
83 sector_t next_swap;
84 };
85
86 struct swap_map_page_list {
87 struct swap_map_page *map;
88 struct swap_map_page_list *next;
89 };
90
91 /**
92 * The swap_map_handle structure is used for handling swap in
93 * a file-alike way
94 */
95
96 struct swap_map_handle {
97 struct swap_map_page *cur;
98 struct swap_map_page_list *maps;
99 sector_t cur_swap;
100 sector_t first_sector;
101 unsigned int k;
102 unsigned long reqd_free_pages;
103 u32 crc32;
104 };
105
106 struct swsusp_header {
107 char reserved[PAGE_SIZE - 20 - sizeof(sector_t) - sizeof(int) -
108 sizeof(u32)];
109 u32 crc32;
110 sector_t image;
111 unsigned int flags; /* Flags to pass to the "boot" kernel */
112 char orig_sig[10];
113 char sig[10];
114 } __packed;
115
116 static struct swsusp_header *swsusp_header;
117
118 /**
119 * The following functions are used for tracing the allocated
120 * swap pages, so that they can be freed in case of an error.
121 */
122
123 struct swsusp_extent {
124 struct rb_node node;
125 unsigned long start;
126 unsigned long end;
127 };
128
129 static struct rb_root swsusp_extents = RB_ROOT;
130
swsusp_extents_insert(unsigned long swap_offset)131 static int swsusp_extents_insert(unsigned long swap_offset)
132 {
133 struct rb_node **new = &(swsusp_extents.rb_node);
134 struct rb_node *parent = NULL;
135 struct swsusp_extent *ext;
136
137 /* Figure out where to put the new node */
138 while (*new) {
139 ext = rb_entry(*new, struct swsusp_extent, node);
140 parent = *new;
141 if (swap_offset < ext->start) {
142 /* Try to merge */
143 if (swap_offset == ext->start - 1) {
144 ext->start--;
145 return 0;
146 }
147 new = &((*new)->rb_left);
148 } else if (swap_offset > ext->end) {
149 /* Try to merge */
150 if (swap_offset == ext->end + 1) {
151 ext->end++;
152 return 0;
153 }
154 new = &((*new)->rb_right);
155 } else {
156 /* It already is in the tree */
157 return -EINVAL;
158 }
159 }
160 /* Add the new node and rebalance the tree. */
161 ext = kzalloc(sizeof(struct swsusp_extent), GFP_KERNEL);
162 if (!ext)
163 return -ENOMEM;
164
165 ext->start = swap_offset;
166 ext->end = swap_offset;
167 rb_link_node(&ext->node, parent, new);
168 rb_insert_color(&ext->node, &swsusp_extents);
169 return 0;
170 }
171
172 /**
173 * alloc_swapdev_block - allocate a swap page and register that it has
174 * been allocated, so that it can be freed in case of an error.
175 */
176
alloc_swapdev_block(int swap)177 sector_t alloc_swapdev_block(int swap)
178 {
179 unsigned long offset;
180
181 offset = swp_offset(get_swap_page_of_type(swap));
182 if (offset) {
183 if (swsusp_extents_insert(offset))
184 swap_free(swp_entry(swap, offset));
185 else
186 return swapdev_block(swap, offset);
187 }
188 return 0;
189 }
190 EXPORT_SYMBOL_GPL(alloc_swapdev_block);
191
192 /**
193 * free_all_swap_pages - free swap pages allocated for saving image data.
194 * It also frees the extents used to register which swap entries had been
195 * allocated.
196 */
197
free_all_swap_pages(int swap)198 void free_all_swap_pages(int swap)
199 {
200 struct rb_node *node;
201
202 while ((node = swsusp_extents.rb_node)) {
203 struct swsusp_extent *ext;
204 unsigned long offset;
205
206 ext = rb_entry(node, struct swsusp_extent, node);
207 rb_erase(node, &swsusp_extents);
208 for (offset = ext->start; offset <= ext->end; offset++)
209 swap_free(swp_entry(swap, offset));
210
211 kfree(ext);
212 }
213 }
214
swsusp_swap_in_use(void)215 int swsusp_swap_in_use(void)
216 {
217 return (swsusp_extents.rb_node != NULL);
218 }
219
220 /*
221 * General things
222 */
223
224 static unsigned short root_swap = 0xffff;
225 static struct block_device *hib_resume_bdev;
226
227 struct hib_bio_batch {
228 atomic_t count;
229 wait_queue_head_t wait;
230 blk_status_t error;
231 struct blk_plug plug;
232 };
233
hib_init_batch(struct hib_bio_batch * hb)234 static void hib_init_batch(struct hib_bio_batch *hb)
235 {
236 atomic_set(&hb->count, 0);
237 init_waitqueue_head(&hb->wait);
238 hb->error = BLK_STS_OK;
239 blk_start_plug(&hb->plug);
240 }
241
hib_finish_batch(struct hib_bio_batch * hb)242 static void hib_finish_batch(struct hib_bio_batch *hb)
243 {
244 blk_finish_plug(&hb->plug);
245 }
246
hib_end_io(struct bio * bio)247 static void hib_end_io(struct bio *bio)
248 {
249 struct hib_bio_batch *hb = bio->bi_private;
250 struct page *page = bio_first_page_all(bio);
251
252 if (bio->bi_status) {
253 pr_alert("Read-error on swap-device (%u:%u:%Lu)\n",
254 MAJOR(bio_dev(bio)), MINOR(bio_dev(bio)),
255 (unsigned long long)bio->bi_iter.bi_sector);
256 }
257
258 if (bio_data_dir(bio) == WRITE)
259 put_page(page);
260 else if (clean_pages_on_read)
261 flush_icache_range((unsigned long)page_address(page),
262 (unsigned long)page_address(page) + PAGE_SIZE);
263
264 if (bio->bi_status && !hb->error)
265 hb->error = bio->bi_status;
266 if (atomic_dec_and_test(&hb->count))
267 wake_up(&hb->wait);
268
269 bio_put(bio);
270 }
271
hib_submit_io(int op,int op_flags,pgoff_t page_off,void * addr,struct hib_bio_batch * hb)272 static int hib_submit_io(int op, int op_flags, pgoff_t page_off, void *addr,
273 struct hib_bio_batch *hb)
274 {
275 struct page *page = virt_to_page(addr);
276 struct bio *bio;
277 int error = 0;
278
279 bio = bio_alloc(GFP_NOIO | __GFP_HIGH, 1);
280 bio->bi_iter.bi_sector = page_off * (PAGE_SIZE >> 9);
281 bio_set_dev(bio, hib_resume_bdev);
282 bio_set_op_attrs(bio, op, op_flags);
283
284 if (bio_add_page(bio, page, PAGE_SIZE, 0) < PAGE_SIZE) {
285 pr_err("Adding page to bio failed at %llu\n",
286 (unsigned long long)bio->bi_iter.bi_sector);
287 bio_put(bio);
288 return -EFAULT;
289 }
290
291 if (hb) {
292 bio->bi_end_io = hib_end_io;
293 bio->bi_private = hb;
294 atomic_inc(&hb->count);
295 submit_bio(bio);
296 } else {
297 error = submit_bio_wait(bio);
298 bio_put(bio);
299 }
300
301 return error;
302 }
303
hib_wait_io(struct hib_bio_batch * hb)304 static int hib_wait_io(struct hib_bio_batch *hb)
305 {
306 /*
307 * We are relying on the behavior of blk_plug that a thread with
308 * a plug will flush the plug list before sleeping.
309 */
310 wait_event(hb->wait, atomic_read(&hb->count) == 0);
311 return blk_status_to_errno(hb->error);
312 }
313
314 /*
315 * Saving part
316 */
317
mark_swapfiles(struct swap_map_handle * handle,unsigned int flags)318 static int mark_swapfiles(struct swap_map_handle *handle, unsigned int flags)
319 {
320 int error;
321
322 hib_submit_io(REQ_OP_READ, 0, swsusp_resume_block,
323 swsusp_header, NULL);
324 if (!memcmp("SWAP-SPACE",swsusp_header->sig, 10) ||
325 !memcmp("SWAPSPACE2",swsusp_header->sig, 10)) {
326 memcpy(swsusp_header->orig_sig,swsusp_header->sig, 10);
327 memcpy(swsusp_header->sig, HIBERNATE_SIG, 10);
328 swsusp_header->image = handle->first_sector;
329 swsusp_header->flags = flags;
330 if (flags & SF_CRC32_MODE)
331 swsusp_header->crc32 = handle->crc32;
332 error = hib_submit_io(REQ_OP_WRITE, REQ_SYNC,
333 swsusp_resume_block, swsusp_header, NULL);
334 } else {
335 pr_err("Swap header not found!\n");
336 error = -ENODEV;
337 }
338 return error;
339 }
340
341 /**
342 * swsusp_swap_check - check if the resume device is a swap device
343 * and get its index (if so)
344 *
345 * This is called before saving image
346 */
swsusp_swap_check(void)347 static int swsusp_swap_check(void)
348 {
349 int res;
350
351 if (swsusp_resume_device)
352 res = swap_type_of(swsusp_resume_device, swsusp_resume_block);
353 else
354 res = find_first_swap(&swsusp_resume_device);
355 if (res < 0)
356 return res;
357 root_swap = res;
358
359 hib_resume_bdev = blkdev_get_by_dev(swsusp_resume_device, FMODE_WRITE,
360 NULL);
361 if (IS_ERR(hib_resume_bdev))
362 return PTR_ERR(hib_resume_bdev);
363
364 res = set_blocksize(hib_resume_bdev, PAGE_SIZE);
365 if (res < 0)
366 blkdev_put(hib_resume_bdev, FMODE_WRITE);
367
368 return res;
369 }
370
371 /**
372 * write_page - Write one page to given swap location.
373 * @buf: Address we're writing.
374 * @offset: Offset of the swap page we're writing to.
375 * @hb: bio completion batch
376 */
377
write_page(void * buf,sector_t offset,struct hib_bio_batch * hb)378 static int write_page(void *buf, sector_t offset, struct hib_bio_batch *hb)
379 {
380 void *src;
381 int ret;
382
383 if (!offset)
384 return -ENOSPC;
385
386 if (hb) {
387 src = (void *)__get_free_page(GFP_NOIO | __GFP_NOWARN |
388 __GFP_NORETRY);
389 if (src) {
390 copy_page(src, buf);
391 } else {
392 ret = hib_wait_io(hb); /* Free pages */
393 if (ret)
394 return ret;
395 src = (void *)__get_free_page(GFP_NOIO |
396 __GFP_NOWARN |
397 __GFP_NORETRY);
398 if (src) {
399 copy_page(src, buf);
400 } else {
401 WARN_ON_ONCE(1);
402 hb = NULL; /* Go synchronous */
403 src = buf;
404 }
405 }
406 } else {
407 src = buf;
408 }
409 return hib_submit_io(REQ_OP_WRITE, REQ_SYNC, offset, src, hb);
410 }
411
release_swap_writer(struct swap_map_handle * handle)412 static void release_swap_writer(struct swap_map_handle *handle)
413 {
414 if (handle->cur)
415 free_page((unsigned long)handle->cur);
416 handle->cur = NULL;
417 }
418
get_swap_writer(struct swap_map_handle * handle)419 static int get_swap_writer(struct swap_map_handle *handle)
420 {
421 int ret;
422
423 ret = swsusp_swap_check();
424 if (ret) {
425 if (ret != -ENOSPC)
426 pr_err("Cannot find swap device, try swapon -a\n");
427 return ret;
428 }
429 handle->cur = (struct swap_map_page *)get_zeroed_page(GFP_KERNEL);
430 if (!handle->cur) {
431 ret = -ENOMEM;
432 goto err_close;
433 }
434 handle->cur_swap = alloc_swapdev_block(root_swap);
435 if (!handle->cur_swap) {
436 ret = -ENOSPC;
437 goto err_rel;
438 }
439 handle->k = 0;
440 handle->reqd_free_pages = reqd_free_pages();
441 handle->first_sector = handle->cur_swap;
442 return 0;
443 err_rel:
444 release_swap_writer(handle);
445 err_close:
446 swsusp_close(FMODE_WRITE);
447 return ret;
448 }
449
swap_write_page(struct swap_map_handle * handle,void * buf,struct hib_bio_batch * hb)450 static int swap_write_page(struct swap_map_handle *handle, void *buf,
451 struct hib_bio_batch *hb)
452 {
453 int error = 0;
454 sector_t offset;
455 bool skip = false;
456
457 if (!handle->cur)
458 return -EINVAL;
459 offset = alloc_swapdev_block(root_swap);
460 error = write_page(buf, offset, hb);
461 if (error)
462 return error;
463 handle->cur->entries[handle->k++] = offset;
464 if (handle->k >= MAP_PAGE_ENTRIES) {
465 offset = alloc_swapdev_block(root_swap);
466 if (!offset)
467 return -ENOSPC;
468 handle->cur->next_swap = offset;
469 trace_android_vh_skip_swap_map_write(&skip);
470 if (!skip) {
471 error = write_page(handle->cur, handle->cur_swap, hb);
472 if (error)
473 goto out;
474 }
475 clear_page(handle->cur);
476 handle->cur_swap = offset;
477 handle->k = 0;
478
479 if (hb && low_free_pages() <= handle->reqd_free_pages) {
480 error = hib_wait_io(hb);
481 if (error)
482 goto out;
483 /*
484 * Recalculate the number of required free pages, to
485 * make sure we never take more than half.
486 */
487 handle->reqd_free_pages = reqd_free_pages();
488 }
489 }
490 out:
491 return error;
492 }
493
flush_swap_writer(struct swap_map_handle * handle)494 static int flush_swap_writer(struct swap_map_handle *handle)
495 {
496 if (handle->cur && handle->cur_swap)
497 return write_page(handle->cur, handle->cur_swap, NULL);
498 else
499 return -EINVAL;
500 }
501
swap_writer_finish(struct swap_map_handle * handle,unsigned int flags,int error)502 static int swap_writer_finish(struct swap_map_handle *handle,
503 unsigned int flags, int error)
504 {
505 if (!error) {
506 pr_info("S");
507 error = mark_swapfiles(handle, flags);
508 pr_cont("|\n");
509 flush_swap_writer(handle);
510 }
511
512 if (error)
513 free_all_swap_pages(root_swap);
514 release_swap_writer(handle);
515 swsusp_close(FMODE_WRITE);
516
517 return error;
518 }
519
520 /* We need to remember how much compressed data we need to read. */
521 #define LZO_HEADER sizeof(size_t)
522
523 /* Number of pages/bytes we'll compress at one time. */
524 #define LZO_UNC_PAGES 32
525 #define LZO_UNC_SIZE (LZO_UNC_PAGES * PAGE_SIZE)
526
527 /* Number of pages/bytes we need for compressed data (worst case). */
528 #define LZO_CMP_PAGES DIV_ROUND_UP(lzo1x_worst_compress(LZO_UNC_SIZE) + \
529 LZO_HEADER, PAGE_SIZE)
530 #define LZO_CMP_SIZE (LZO_CMP_PAGES * PAGE_SIZE)
531
532 /* Maximum number of threads for compression/decompression. */
533 #define LZO_THREADS 3
534
535 /* Minimum/maximum number of pages for read buffering. */
536 #define LZO_MIN_RD_PAGES 1024
537 #define LZO_MAX_RD_PAGES 8192
538
539
540 /**
541 * save_image - save the suspend image data
542 */
543
save_image(struct swap_map_handle * handle,struct snapshot_handle * snapshot,unsigned int nr_to_write)544 static int save_image(struct swap_map_handle *handle,
545 struct snapshot_handle *snapshot,
546 unsigned int nr_to_write)
547 {
548 unsigned int m;
549 int ret;
550 int nr_pages;
551 int err2;
552 struct hib_bio_batch hb;
553 ktime_t start;
554 ktime_t stop;
555
556 hib_init_batch(&hb);
557
558 pr_info("Saving image data pages (%u pages)...\n",
559 nr_to_write);
560 m = nr_to_write / 10;
561 if (!m)
562 m = 1;
563 nr_pages = 0;
564 start = ktime_get();
565 while (1) {
566 ret = snapshot_read_next(snapshot);
567 if (ret <= 0)
568 break;
569 trace_android_vh_encrypt_page(data_of(*snapshot));
570 ret = swap_write_page(handle, data_of(*snapshot), &hb);
571 if (ret)
572 break;
573 if (!(nr_pages % m))
574 pr_info("Image saving progress: %3d%%\n",
575 nr_pages / m * 10);
576 nr_pages++;
577 }
578 err2 = hib_wait_io(&hb);
579 hib_finish_batch(&hb);
580 stop = ktime_get();
581 if (!ret)
582 ret = err2;
583 if (!ret)
584 pr_info("Image saving done\n");
585 swsusp_show_speed(start, stop, nr_to_write, "Wrote");
586 trace_android_vh_post_image_save(root_swap);
587 return ret;
588 }
589
590 /**
591 * Structure used for CRC32.
592 */
593 struct crc_data {
594 struct task_struct *thr; /* thread */
595 atomic_t ready; /* ready to start flag */
596 atomic_t stop; /* ready to stop flag */
597 unsigned run_threads; /* nr current threads */
598 wait_queue_head_t go; /* start crc update */
599 wait_queue_head_t done; /* crc update done */
600 u32 *crc32; /* points to handle's crc32 */
601 size_t *unc_len[LZO_THREADS]; /* uncompressed lengths */
602 unsigned char *unc[LZO_THREADS]; /* uncompressed data */
603 };
604
605 /**
606 * CRC32 update function that runs in its own thread.
607 */
crc32_threadfn(void * data)608 static int crc32_threadfn(void *data)
609 {
610 struct crc_data *d = data;
611 unsigned i;
612
613 while (1) {
614 wait_event(d->go, atomic_read_acquire(&d->ready) ||
615 kthread_should_stop());
616 if (kthread_should_stop()) {
617 d->thr = NULL;
618 atomic_set_release(&d->stop, 1);
619 wake_up(&d->done);
620 break;
621 }
622 atomic_set(&d->ready, 0);
623
624 for (i = 0; i < d->run_threads; i++)
625 *d->crc32 = crc32_le(*d->crc32,
626 d->unc[i], *d->unc_len[i]);
627 atomic_set_release(&d->stop, 1);
628 wake_up(&d->done);
629 }
630 return 0;
631 }
632 /**
633 * Structure used for LZO data compression.
634 */
635 struct cmp_data {
636 struct task_struct *thr; /* thread */
637 atomic_t ready; /* ready to start flag */
638 atomic_t stop; /* ready to stop flag */
639 int ret; /* return code */
640 wait_queue_head_t go; /* start compression */
641 wait_queue_head_t done; /* compression done */
642 size_t unc_len; /* uncompressed length */
643 size_t cmp_len; /* compressed length */
644 unsigned char unc[LZO_UNC_SIZE]; /* uncompressed buffer */
645 unsigned char cmp[LZO_CMP_SIZE]; /* compressed buffer */
646 unsigned char wrk[LZO1X_1_MEM_COMPRESS]; /* compression workspace */
647 };
648
649 /**
650 * Compression function that runs in its own thread.
651 */
lzo_compress_threadfn(void * data)652 static int lzo_compress_threadfn(void *data)
653 {
654 struct cmp_data *d = data;
655
656 while (1) {
657 wait_event(d->go, atomic_read_acquire(&d->ready) ||
658 kthread_should_stop());
659 if (kthread_should_stop()) {
660 d->thr = NULL;
661 d->ret = -1;
662 atomic_set_release(&d->stop, 1);
663 wake_up(&d->done);
664 break;
665 }
666 atomic_set(&d->ready, 0);
667
668 d->ret = lzo1x_1_compress(d->unc, d->unc_len,
669 d->cmp + LZO_HEADER, &d->cmp_len,
670 d->wrk);
671 atomic_set_release(&d->stop, 1);
672 wake_up(&d->done);
673 }
674 return 0;
675 }
676
677 /**
678 * save_image_lzo - Save the suspend image data compressed with LZO.
679 * @handle: Swap map handle to use for saving the image.
680 * @snapshot: Image to read data from.
681 * @nr_to_write: Number of pages to save.
682 */
save_image_lzo(struct swap_map_handle * handle,struct snapshot_handle * snapshot,unsigned int nr_to_write)683 static int save_image_lzo(struct swap_map_handle *handle,
684 struct snapshot_handle *snapshot,
685 unsigned int nr_to_write)
686 {
687 unsigned int m;
688 int ret = 0;
689 int nr_pages;
690 int err2;
691 struct hib_bio_batch hb;
692 ktime_t start;
693 ktime_t stop;
694 size_t off;
695 unsigned thr, run_threads, nr_threads;
696 unsigned char *page = NULL;
697 struct cmp_data *data = NULL;
698 struct crc_data *crc = NULL;
699
700 hib_init_batch(&hb);
701
702 /*
703 * We'll limit the number of threads for compression to limit memory
704 * footprint.
705 */
706 nr_threads = num_online_cpus() - 1;
707 nr_threads = clamp_val(nr_threads, 1, LZO_THREADS);
708
709 page = (void *)__get_free_page(GFP_NOIO | __GFP_HIGH);
710 if (!page) {
711 pr_err("Failed to allocate LZO page\n");
712 ret = -ENOMEM;
713 goto out_clean;
714 }
715
716 data = vmalloc(array_size(nr_threads, sizeof(*data)));
717 if (!data) {
718 pr_err("Failed to allocate LZO data\n");
719 ret = -ENOMEM;
720 goto out_clean;
721 }
722 for (thr = 0; thr < nr_threads; thr++)
723 memset(&data[thr], 0, offsetof(struct cmp_data, go));
724
725 crc = kmalloc(sizeof(*crc), GFP_KERNEL);
726 if (!crc) {
727 pr_err("Failed to allocate crc\n");
728 ret = -ENOMEM;
729 goto out_clean;
730 }
731 memset(crc, 0, offsetof(struct crc_data, go));
732
733 /*
734 * Start the compression threads.
735 */
736 for (thr = 0; thr < nr_threads; thr++) {
737 init_waitqueue_head(&data[thr].go);
738 init_waitqueue_head(&data[thr].done);
739
740 data[thr].thr = kthread_run(lzo_compress_threadfn,
741 &data[thr],
742 "image_compress/%u", thr);
743 if (IS_ERR(data[thr].thr)) {
744 data[thr].thr = NULL;
745 pr_err("Cannot start compression threads\n");
746 ret = -ENOMEM;
747 goto out_clean;
748 }
749 }
750
751 /*
752 * Start the CRC32 thread.
753 */
754 init_waitqueue_head(&crc->go);
755 init_waitqueue_head(&crc->done);
756
757 handle->crc32 = 0;
758 crc->crc32 = &handle->crc32;
759 for (thr = 0; thr < nr_threads; thr++) {
760 crc->unc[thr] = data[thr].unc;
761 crc->unc_len[thr] = &data[thr].unc_len;
762 }
763
764 crc->thr = kthread_run(crc32_threadfn, crc, "image_crc32");
765 if (IS_ERR(crc->thr)) {
766 crc->thr = NULL;
767 pr_err("Cannot start CRC32 thread\n");
768 ret = -ENOMEM;
769 goto out_clean;
770 }
771
772 /*
773 * Adjust the number of required free pages after all allocations have
774 * been done. We don't want to run out of pages when writing.
775 */
776 handle->reqd_free_pages = reqd_free_pages();
777
778 pr_info("Using %u thread(s) for compression\n", nr_threads);
779 pr_info("Compressing and saving image data (%u pages)...\n",
780 nr_to_write);
781 m = nr_to_write / 10;
782 if (!m)
783 m = 1;
784 nr_pages = 0;
785 start = ktime_get();
786 for (;;) {
787 for (thr = 0; thr < nr_threads; thr++) {
788 for (off = 0; off < LZO_UNC_SIZE; off += PAGE_SIZE) {
789 ret = snapshot_read_next(snapshot);
790 if (ret < 0)
791 goto out_finish;
792
793 if (!ret)
794 break;
795
796 memcpy(data[thr].unc + off,
797 data_of(*snapshot), PAGE_SIZE);
798
799 if (!(nr_pages % m))
800 pr_info("Image saving progress: %3d%%\n",
801 nr_pages / m * 10);
802 nr_pages++;
803 }
804 if (!off)
805 break;
806
807 data[thr].unc_len = off;
808
809 atomic_set_release(&data[thr].ready, 1);
810 wake_up(&data[thr].go);
811 }
812
813 if (!thr)
814 break;
815
816 crc->run_threads = thr;
817 atomic_set_release(&crc->ready, 1);
818 wake_up(&crc->go);
819
820 for (run_threads = thr, thr = 0; thr < run_threads; thr++) {
821 wait_event(data[thr].done,
822 atomic_read_acquire(&data[thr].stop));
823 atomic_set(&data[thr].stop, 0);
824
825 ret = data[thr].ret;
826
827 if (ret < 0) {
828 pr_err("LZO compression failed\n");
829 goto out_finish;
830 }
831
832 if (unlikely(!data[thr].cmp_len ||
833 data[thr].cmp_len >
834 lzo1x_worst_compress(data[thr].unc_len))) {
835 pr_err("Invalid LZO compressed length\n");
836 ret = -1;
837 goto out_finish;
838 }
839
840 *(size_t *)data[thr].cmp = data[thr].cmp_len;
841
842 /*
843 * Given we are writing one page at a time to disk, we
844 * copy that much from the buffer, although the last
845 * bit will likely be smaller than full page. This is
846 * OK - we saved the length of the compressed data, so
847 * any garbage at the end will be discarded when we
848 * read it.
849 */
850 for (off = 0;
851 off < LZO_HEADER + data[thr].cmp_len;
852 off += PAGE_SIZE) {
853 memcpy(page, data[thr].cmp + off, PAGE_SIZE);
854
855 ret = swap_write_page(handle, page, &hb);
856 if (ret)
857 goto out_finish;
858 }
859 }
860
861 wait_event(crc->done, atomic_read_acquire(&crc->stop));
862 atomic_set(&crc->stop, 0);
863 }
864
865 out_finish:
866 err2 = hib_wait_io(&hb);
867 stop = ktime_get();
868 if (!ret)
869 ret = err2;
870 if (!ret)
871 pr_info("Image saving done\n");
872 swsusp_show_speed(start, stop, nr_to_write, "Wrote");
873 out_clean:
874 hib_finish_batch(&hb);
875 if (crc) {
876 if (crc->thr)
877 kthread_stop(crc->thr);
878 kfree(crc);
879 }
880 if (data) {
881 for (thr = 0; thr < nr_threads; thr++)
882 if (data[thr].thr)
883 kthread_stop(data[thr].thr);
884 vfree(data);
885 }
886 if (page) free_page((unsigned long)page);
887
888 return ret;
889 }
890
891 /**
892 * enough_swap - Make sure we have enough swap to save the image.
893 *
894 * Returns TRUE or FALSE after checking the total amount of swap
895 * space available from the resume partition.
896 */
897
enough_swap(unsigned int nr_pages)898 static int enough_swap(unsigned int nr_pages)
899 {
900 unsigned int free_swap = count_swap_pages(root_swap, 1);
901 unsigned int required;
902
903 pr_debug("Free swap pages: %u\n", free_swap);
904
905 required = PAGES_FOR_IO + nr_pages;
906 return free_swap > required;
907 }
908
909 /**
910 * swsusp_write - Write entire image and metadata.
911 * @flags: flags to pass to the "boot" kernel in the image header
912 *
913 * It is important _NOT_ to umount filesystems at this point. We want
914 * them synced (in case something goes wrong) but we DO not want to mark
915 * filesystem clean: it is not. (And it does not matter, if we resume
916 * correctly, we'll mark system clean, anyway.)
917 */
918
swsusp_write(unsigned int flags)919 int swsusp_write(unsigned int flags)
920 {
921 struct swap_map_handle handle;
922 struct snapshot_handle snapshot;
923 struct swsusp_info *header;
924 unsigned long pages;
925 int error;
926
927 pages = snapshot_get_image_size();
928 error = get_swap_writer(&handle);
929 if (error) {
930 pr_err("Cannot get swap writer\n");
931 return error;
932 }
933 trace_android_vh_init_aes_encrypt(NULL);
934 if (flags & SF_NOCOMPRESS_MODE) {
935 if (!enough_swap(pages)) {
936 pr_err("Not enough free swap\n");
937 error = -ENOSPC;
938 goto out_finish;
939 }
940 }
941 memset(&snapshot, 0, sizeof(struct snapshot_handle));
942 error = snapshot_read_next(&snapshot);
943 if (error < (int)PAGE_SIZE) {
944 if (error >= 0)
945 error = -EFAULT;
946
947 goto out_finish;
948 }
949 header = (struct swsusp_info *)data_of(snapshot);
950 error = swap_write_page(&handle, header, NULL);
951 if (!error) {
952 error = (flags & SF_NOCOMPRESS_MODE) ?
953 save_image(&handle, &snapshot, pages - 1) :
954 save_image_lzo(&handle, &snapshot, pages - 1);
955 }
956 out_finish:
957 error = swap_writer_finish(&handle, flags, error);
958 return error;
959 }
960
961 /**
962 * The following functions allow us to read data using a swap map
963 * in a file-alike way
964 */
965
release_swap_reader(struct swap_map_handle * handle)966 static void release_swap_reader(struct swap_map_handle *handle)
967 {
968 struct swap_map_page_list *tmp;
969
970 while (handle->maps) {
971 if (handle->maps->map)
972 free_page((unsigned long)handle->maps->map);
973 tmp = handle->maps;
974 handle->maps = handle->maps->next;
975 kfree(tmp);
976 }
977 handle->cur = NULL;
978 }
979
get_swap_reader(struct swap_map_handle * handle,unsigned int * flags_p)980 static int get_swap_reader(struct swap_map_handle *handle,
981 unsigned int *flags_p)
982 {
983 int error;
984 struct swap_map_page_list *tmp, *last;
985 sector_t offset;
986
987 *flags_p = swsusp_header->flags;
988
989 if (!swsusp_header->image) /* how can this happen? */
990 return -EINVAL;
991
992 handle->cur = NULL;
993 last = handle->maps = NULL;
994 offset = swsusp_header->image;
995 while (offset) {
996 tmp = kzalloc(sizeof(*handle->maps), GFP_KERNEL);
997 if (!tmp) {
998 release_swap_reader(handle);
999 return -ENOMEM;
1000 }
1001 if (!handle->maps)
1002 handle->maps = tmp;
1003 if (last)
1004 last->next = tmp;
1005 last = tmp;
1006
1007 tmp->map = (struct swap_map_page *)
1008 __get_free_page(GFP_NOIO | __GFP_HIGH);
1009 if (!tmp->map) {
1010 release_swap_reader(handle);
1011 return -ENOMEM;
1012 }
1013
1014 error = hib_submit_io(REQ_OP_READ, 0, offset, tmp->map, NULL);
1015 if (error) {
1016 release_swap_reader(handle);
1017 return error;
1018 }
1019 offset = tmp->map->next_swap;
1020 }
1021 handle->k = 0;
1022 handle->cur = handle->maps->map;
1023 return 0;
1024 }
1025
swap_read_page(struct swap_map_handle * handle,void * buf,struct hib_bio_batch * hb)1026 static int swap_read_page(struct swap_map_handle *handle, void *buf,
1027 struct hib_bio_batch *hb)
1028 {
1029 sector_t offset;
1030 int error;
1031 struct swap_map_page_list *tmp;
1032
1033 if (!handle->cur)
1034 return -EINVAL;
1035 offset = handle->cur->entries[handle->k];
1036 if (!offset)
1037 return -EFAULT;
1038 error = hib_submit_io(REQ_OP_READ, 0, offset, buf, hb);
1039 if (error)
1040 return error;
1041 if (++handle->k >= MAP_PAGE_ENTRIES) {
1042 handle->k = 0;
1043 free_page((unsigned long)handle->maps->map);
1044 tmp = handle->maps;
1045 handle->maps = handle->maps->next;
1046 kfree(tmp);
1047 if (!handle->maps)
1048 release_swap_reader(handle);
1049 else
1050 handle->cur = handle->maps->map;
1051 }
1052 return error;
1053 }
1054
swap_reader_finish(struct swap_map_handle * handle)1055 static int swap_reader_finish(struct swap_map_handle *handle)
1056 {
1057 release_swap_reader(handle);
1058
1059 return 0;
1060 }
1061
1062 /**
1063 * load_image - load the image using the swap map handle
1064 * @handle and the snapshot handle @snapshot
1065 * (assume there are @nr_pages pages to load)
1066 */
1067
load_image(struct swap_map_handle * handle,struct snapshot_handle * snapshot,unsigned int nr_to_read)1068 static int load_image(struct swap_map_handle *handle,
1069 struct snapshot_handle *snapshot,
1070 unsigned int nr_to_read)
1071 {
1072 unsigned int m;
1073 int ret = 0;
1074 ktime_t start;
1075 ktime_t stop;
1076 struct hib_bio_batch hb;
1077 int err2;
1078 unsigned nr_pages;
1079
1080 hib_init_batch(&hb);
1081
1082 clean_pages_on_read = true;
1083 pr_info("Loading image data pages (%u pages)...\n", nr_to_read);
1084 m = nr_to_read / 10;
1085 if (!m)
1086 m = 1;
1087 nr_pages = 0;
1088 start = ktime_get();
1089 for ( ; ; ) {
1090 ret = snapshot_write_next(snapshot);
1091 if (ret <= 0)
1092 break;
1093 ret = swap_read_page(handle, data_of(*snapshot), &hb);
1094 if (ret)
1095 break;
1096 if (snapshot->sync_read)
1097 ret = hib_wait_io(&hb);
1098 if (ret)
1099 break;
1100 if (!(nr_pages % m))
1101 pr_info("Image loading progress: %3d%%\n",
1102 nr_pages / m * 10);
1103 nr_pages++;
1104 }
1105 err2 = hib_wait_io(&hb);
1106 hib_finish_batch(&hb);
1107 stop = ktime_get();
1108 if (!ret)
1109 ret = err2;
1110 if (!ret) {
1111 pr_info("Image loading done\n");
1112 snapshot_write_finalize(snapshot);
1113 if (!snapshot_image_loaded(snapshot))
1114 ret = -ENODATA;
1115 }
1116 swsusp_show_speed(start, stop, nr_to_read, "Read");
1117 return ret;
1118 }
1119
1120 /**
1121 * Structure used for LZO data decompression.
1122 */
1123 struct dec_data {
1124 struct task_struct *thr; /* thread */
1125 atomic_t ready; /* ready to start flag */
1126 atomic_t stop; /* ready to stop flag */
1127 int ret; /* return code */
1128 wait_queue_head_t go; /* start decompression */
1129 wait_queue_head_t done; /* decompression done */
1130 size_t unc_len; /* uncompressed length */
1131 size_t cmp_len; /* compressed length */
1132 unsigned char unc[LZO_UNC_SIZE]; /* uncompressed buffer */
1133 unsigned char cmp[LZO_CMP_SIZE]; /* compressed buffer */
1134 };
1135
1136 /**
1137 * Decompression function that runs in its own thread.
1138 */
lzo_decompress_threadfn(void * data)1139 static int lzo_decompress_threadfn(void *data)
1140 {
1141 struct dec_data *d = data;
1142
1143 while (1) {
1144 wait_event(d->go, atomic_read_acquire(&d->ready) ||
1145 kthread_should_stop());
1146 if (kthread_should_stop()) {
1147 d->thr = NULL;
1148 d->ret = -1;
1149 atomic_set_release(&d->stop, 1);
1150 wake_up(&d->done);
1151 break;
1152 }
1153 atomic_set(&d->ready, 0);
1154
1155 d->unc_len = LZO_UNC_SIZE;
1156 d->ret = lzo1x_decompress_safe(d->cmp + LZO_HEADER, d->cmp_len,
1157 d->unc, &d->unc_len);
1158 if (clean_pages_on_decompress)
1159 flush_icache_range((unsigned long)d->unc,
1160 (unsigned long)d->unc + d->unc_len);
1161
1162 atomic_set_release(&d->stop, 1);
1163 wake_up(&d->done);
1164 }
1165 return 0;
1166 }
1167
1168 /**
1169 * load_image_lzo - Load compressed image data and decompress them with LZO.
1170 * @handle: Swap map handle to use for loading data.
1171 * @snapshot: Image to copy uncompressed data into.
1172 * @nr_to_read: Number of pages to load.
1173 */
load_image_lzo(struct swap_map_handle * handle,struct snapshot_handle * snapshot,unsigned int nr_to_read)1174 static int load_image_lzo(struct swap_map_handle *handle,
1175 struct snapshot_handle *snapshot,
1176 unsigned int nr_to_read)
1177 {
1178 unsigned int m;
1179 int ret = 0;
1180 int eof = 0;
1181 struct hib_bio_batch hb;
1182 ktime_t start;
1183 ktime_t stop;
1184 unsigned nr_pages;
1185 size_t off;
1186 unsigned i, thr, run_threads, nr_threads;
1187 unsigned ring = 0, pg = 0, ring_size = 0,
1188 have = 0, want, need, asked = 0;
1189 unsigned long read_pages = 0;
1190 unsigned char **page = NULL;
1191 struct dec_data *data = NULL;
1192 struct crc_data *crc = NULL;
1193
1194 hib_init_batch(&hb);
1195
1196 /*
1197 * We'll limit the number of threads for decompression to limit memory
1198 * footprint.
1199 */
1200 nr_threads = num_online_cpus() - 1;
1201 nr_threads = clamp_val(nr_threads, 1, LZO_THREADS);
1202
1203 page = vmalloc(array_size(LZO_MAX_RD_PAGES, sizeof(*page)));
1204 if (!page) {
1205 pr_err("Failed to allocate LZO page\n");
1206 ret = -ENOMEM;
1207 goto out_clean;
1208 }
1209
1210 data = vmalloc(array_size(nr_threads, sizeof(*data)));
1211 if (!data) {
1212 pr_err("Failed to allocate LZO data\n");
1213 ret = -ENOMEM;
1214 goto out_clean;
1215 }
1216 for (thr = 0; thr < nr_threads; thr++)
1217 memset(&data[thr], 0, offsetof(struct dec_data, go));
1218
1219 crc = kmalloc(sizeof(*crc), GFP_KERNEL);
1220 if (!crc) {
1221 pr_err("Failed to allocate crc\n");
1222 ret = -ENOMEM;
1223 goto out_clean;
1224 }
1225 memset(crc, 0, offsetof(struct crc_data, go));
1226
1227 clean_pages_on_decompress = true;
1228
1229 /*
1230 * Start the decompression threads.
1231 */
1232 for (thr = 0; thr < nr_threads; thr++) {
1233 init_waitqueue_head(&data[thr].go);
1234 init_waitqueue_head(&data[thr].done);
1235
1236 data[thr].thr = kthread_run(lzo_decompress_threadfn,
1237 &data[thr],
1238 "image_decompress/%u", thr);
1239 if (IS_ERR(data[thr].thr)) {
1240 data[thr].thr = NULL;
1241 pr_err("Cannot start decompression threads\n");
1242 ret = -ENOMEM;
1243 goto out_clean;
1244 }
1245 }
1246
1247 /*
1248 * Start the CRC32 thread.
1249 */
1250 init_waitqueue_head(&crc->go);
1251 init_waitqueue_head(&crc->done);
1252
1253 handle->crc32 = 0;
1254 crc->crc32 = &handle->crc32;
1255 for (thr = 0; thr < nr_threads; thr++) {
1256 crc->unc[thr] = data[thr].unc;
1257 crc->unc_len[thr] = &data[thr].unc_len;
1258 }
1259
1260 crc->thr = kthread_run(crc32_threadfn, crc, "image_crc32");
1261 if (IS_ERR(crc->thr)) {
1262 crc->thr = NULL;
1263 pr_err("Cannot start CRC32 thread\n");
1264 ret = -ENOMEM;
1265 goto out_clean;
1266 }
1267
1268 /*
1269 * Set the number of pages for read buffering.
1270 * This is complete guesswork, because we'll only know the real
1271 * picture once prepare_image() is called, which is much later on
1272 * during the image load phase. We'll assume the worst case and
1273 * say that none of the image pages are from high memory.
1274 */
1275 if (low_free_pages() > snapshot_get_image_size())
1276 read_pages = (low_free_pages() - snapshot_get_image_size()) / 2;
1277 read_pages = clamp_val(read_pages, LZO_MIN_RD_PAGES, LZO_MAX_RD_PAGES);
1278
1279 for (i = 0; i < read_pages; i++) {
1280 page[i] = (void *)__get_free_page(i < LZO_CMP_PAGES ?
1281 GFP_NOIO | __GFP_HIGH :
1282 GFP_NOIO | __GFP_NOWARN |
1283 __GFP_NORETRY);
1284
1285 if (!page[i]) {
1286 if (i < LZO_CMP_PAGES) {
1287 ring_size = i;
1288 pr_err("Failed to allocate LZO pages\n");
1289 ret = -ENOMEM;
1290 goto out_clean;
1291 } else {
1292 break;
1293 }
1294 }
1295 }
1296 want = ring_size = i;
1297
1298 pr_info("Using %u thread(s) for decompression\n", nr_threads);
1299 pr_info("Loading and decompressing image data (%u pages)...\n",
1300 nr_to_read);
1301 m = nr_to_read / 10;
1302 if (!m)
1303 m = 1;
1304 nr_pages = 0;
1305 start = ktime_get();
1306
1307 ret = snapshot_write_next(snapshot);
1308 if (ret <= 0)
1309 goto out_finish;
1310
1311 for(;;) {
1312 for (i = 0; !eof && i < want; i++) {
1313 ret = swap_read_page(handle, page[ring], &hb);
1314 if (ret) {
1315 /*
1316 * On real read error, finish. On end of data,
1317 * set EOF flag and just exit the read loop.
1318 */
1319 if (handle->cur &&
1320 handle->cur->entries[handle->k]) {
1321 goto out_finish;
1322 } else {
1323 eof = 1;
1324 break;
1325 }
1326 }
1327 if (++ring >= ring_size)
1328 ring = 0;
1329 }
1330 asked += i;
1331 want -= i;
1332
1333 /*
1334 * We are out of data, wait for some more.
1335 */
1336 if (!have) {
1337 if (!asked)
1338 break;
1339
1340 ret = hib_wait_io(&hb);
1341 if (ret)
1342 goto out_finish;
1343 have += asked;
1344 asked = 0;
1345 if (eof)
1346 eof = 2;
1347 }
1348
1349 if (crc->run_threads) {
1350 wait_event(crc->done, atomic_read_acquire(&crc->stop));
1351 atomic_set(&crc->stop, 0);
1352 crc->run_threads = 0;
1353 }
1354
1355 for (thr = 0; have && thr < nr_threads; thr++) {
1356 data[thr].cmp_len = *(size_t *)page[pg];
1357 if (unlikely(!data[thr].cmp_len ||
1358 data[thr].cmp_len >
1359 lzo1x_worst_compress(LZO_UNC_SIZE))) {
1360 pr_err("Invalid LZO compressed length\n");
1361 ret = -1;
1362 goto out_finish;
1363 }
1364
1365 need = DIV_ROUND_UP(data[thr].cmp_len + LZO_HEADER,
1366 PAGE_SIZE);
1367 if (need > have) {
1368 if (eof > 1) {
1369 ret = -1;
1370 goto out_finish;
1371 }
1372 break;
1373 }
1374
1375 for (off = 0;
1376 off < LZO_HEADER + data[thr].cmp_len;
1377 off += PAGE_SIZE) {
1378 memcpy(data[thr].cmp + off,
1379 page[pg], PAGE_SIZE);
1380 have--;
1381 want++;
1382 if (++pg >= ring_size)
1383 pg = 0;
1384 }
1385
1386 atomic_set_release(&data[thr].ready, 1);
1387 wake_up(&data[thr].go);
1388 }
1389
1390 /*
1391 * Wait for more data while we are decompressing.
1392 */
1393 if (have < LZO_CMP_PAGES && asked) {
1394 ret = hib_wait_io(&hb);
1395 if (ret)
1396 goto out_finish;
1397 have += asked;
1398 asked = 0;
1399 if (eof)
1400 eof = 2;
1401 }
1402
1403 for (run_threads = thr, thr = 0; thr < run_threads; thr++) {
1404 wait_event(data[thr].done,
1405 atomic_read_acquire(&data[thr].stop));
1406 atomic_set(&data[thr].stop, 0);
1407
1408 ret = data[thr].ret;
1409
1410 if (ret < 0) {
1411 pr_err("LZO decompression failed\n");
1412 goto out_finish;
1413 }
1414
1415 if (unlikely(!data[thr].unc_len ||
1416 data[thr].unc_len > LZO_UNC_SIZE ||
1417 data[thr].unc_len & (PAGE_SIZE - 1))) {
1418 pr_err("Invalid LZO uncompressed length\n");
1419 ret = -1;
1420 goto out_finish;
1421 }
1422
1423 for (off = 0;
1424 off < data[thr].unc_len; off += PAGE_SIZE) {
1425 memcpy(data_of(*snapshot),
1426 data[thr].unc + off, PAGE_SIZE);
1427
1428 if (!(nr_pages % m))
1429 pr_info("Image loading progress: %3d%%\n",
1430 nr_pages / m * 10);
1431 nr_pages++;
1432
1433 ret = snapshot_write_next(snapshot);
1434 if (ret <= 0) {
1435 crc->run_threads = thr + 1;
1436 atomic_set_release(&crc->ready, 1);
1437 wake_up(&crc->go);
1438 goto out_finish;
1439 }
1440 }
1441 }
1442
1443 crc->run_threads = thr;
1444 atomic_set_release(&crc->ready, 1);
1445 wake_up(&crc->go);
1446 }
1447
1448 out_finish:
1449 if (crc->run_threads) {
1450 wait_event(crc->done, atomic_read_acquire(&crc->stop));
1451 atomic_set(&crc->stop, 0);
1452 }
1453 stop = ktime_get();
1454 if (!ret) {
1455 pr_info("Image loading done\n");
1456 snapshot_write_finalize(snapshot);
1457 if (!snapshot_image_loaded(snapshot))
1458 ret = -ENODATA;
1459 if (!ret) {
1460 if (swsusp_header->flags & SF_CRC32_MODE) {
1461 if(handle->crc32 != swsusp_header->crc32) {
1462 pr_err("Invalid image CRC32!\n");
1463 ret = -ENODATA;
1464 }
1465 }
1466 }
1467 }
1468 swsusp_show_speed(start, stop, nr_to_read, "Read");
1469 out_clean:
1470 hib_finish_batch(&hb);
1471 for (i = 0; i < ring_size; i++)
1472 free_page((unsigned long)page[i]);
1473 if (crc) {
1474 if (crc->thr)
1475 kthread_stop(crc->thr);
1476 kfree(crc);
1477 }
1478 if (data) {
1479 for (thr = 0; thr < nr_threads; thr++)
1480 if (data[thr].thr)
1481 kthread_stop(data[thr].thr);
1482 vfree(data);
1483 }
1484 vfree(page);
1485
1486 return ret;
1487 }
1488
1489 /**
1490 * swsusp_read - read the hibernation image.
1491 * @flags_p: flags passed by the "frozen" kernel in the image header should
1492 * be written into this memory location
1493 */
1494
swsusp_read(unsigned int * flags_p)1495 int swsusp_read(unsigned int *flags_p)
1496 {
1497 int error;
1498 struct swap_map_handle handle;
1499 struct snapshot_handle snapshot;
1500 struct swsusp_info *header;
1501
1502 memset(&snapshot, 0, sizeof(struct snapshot_handle));
1503 error = snapshot_write_next(&snapshot);
1504 if (error < (int)PAGE_SIZE)
1505 return error < 0 ? error : -EFAULT;
1506 header = (struct swsusp_info *)data_of(snapshot);
1507 error = get_swap_reader(&handle, flags_p);
1508 if (error)
1509 goto end;
1510 if (!error)
1511 error = swap_read_page(&handle, header, NULL);
1512 if (!error) {
1513 error = (*flags_p & SF_NOCOMPRESS_MODE) ?
1514 load_image(&handle, &snapshot, header->pages - 1) :
1515 load_image_lzo(&handle, &snapshot, header->pages - 1);
1516 }
1517 swap_reader_finish(&handle);
1518 end:
1519 if (!error)
1520 pr_debug("Image successfully loaded\n");
1521 else
1522 pr_debug("Error %d resuming\n", error);
1523 return error;
1524 }
1525
1526 /**
1527 * swsusp_check - Check for swsusp signature in the resume device
1528 */
1529
swsusp_check(void)1530 int swsusp_check(void)
1531 {
1532 int error;
1533 void *holder;
1534
1535 hib_resume_bdev = blkdev_get_by_dev(swsusp_resume_device,
1536 FMODE_READ | FMODE_EXCL, &holder);
1537 if (!IS_ERR(hib_resume_bdev)) {
1538 set_blocksize(hib_resume_bdev, PAGE_SIZE);
1539 trace_android_vh_save_hib_resume_bdev(hib_resume_bdev);
1540 clear_page(swsusp_header);
1541 error = hib_submit_io(REQ_OP_READ, 0,
1542 swsusp_resume_block,
1543 swsusp_header, NULL);
1544 if (error)
1545 goto put;
1546
1547 if (!memcmp(HIBERNATE_SIG, swsusp_header->sig, 10)) {
1548 memcpy(swsusp_header->sig, swsusp_header->orig_sig, 10);
1549 /* Reset swap signature now */
1550 error = hib_submit_io(REQ_OP_WRITE, REQ_SYNC,
1551 swsusp_resume_block,
1552 swsusp_header, NULL);
1553 } else {
1554 error = -EINVAL;
1555 }
1556
1557 put:
1558 if (error)
1559 blkdev_put(hib_resume_bdev, FMODE_READ | FMODE_EXCL);
1560 else
1561 pr_debug("Image signature found, resuming\n");
1562 } else {
1563 error = PTR_ERR(hib_resume_bdev);
1564 }
1565
1566 if (error)
1567 pr_debug("Image not found (code %d)\n", error);
1568
1569 return error;
1570 }
1571
1572 /**
1573 * swsusp_close - close swap device.
1574 */
1575
swsusp_close(fmode_t mode)1576 void swsusp_close(fmode_t mode)
1577 {
1578 if (IS_ERR(hib_resume_bdev)) {
1579 pr_debug("Image device not initialised\n");
1580 return;
1581 }
1582
1583 blkdev_put(hib_resume_bdev, mode);
1584 }
1585
1586 /**
1587 * swsusp_unmark - Unmark swsusp signature in the resume device
1588 */
1589
1590 #ifdef CONFIG_SUSPEND
swsusp_unmark(void)1591 int swsusp_unmark(void)
1592 {
1593 int error;
1594
1595 hib_submit_io(REQ_OP_READ, 0, swsusp_resume_block,
1596 swsusp_header, NULL);
1597 if (!memcmp(HIBERNATE_SIG,swsusp_header->sig, 10)) {
1598 memcpy(swsusp_header->sig,swsusp_header->orig_sig, 10);
1599 error = hib_submit_io(REQ_OP_WRITE, REQ_SYNC,
1600 swsusp_resume_block,
1601 swsusp_header, NULL);
1602 } else {
1603 pr_err("Cannot find swsusp signature!\n");
1604 error = -ENODEV;
1605 }
1606
1607 /*
1608 * We just returned from suspend, we don't need the image any more.
1609 */
1610 free_all_swap_pages(root_swap);
1611
1612 return error;
1613 }
1614 #endif
1615
swsusp_header_init(void)1616 static int __init swsusp_header_init(void)
1617 {
1618 swsusp_header = (struct swsusp_header*) __get_free_page(GFP_KERNEL);
1619 if (!swsusp_header)
1620 panic("Could not allocate memory for swsusp_header\n");
1621 return 0;
1622 }
1623
1624 core_initcall(swsusp_header_init);
1625