1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * fs/f2fs/checkpoint.c
4 *
5 * Copyright (c) 2012 Samsung Electronics Co., Ltd.
6 * http://www.samsung.com/
7 */
8 #include <linux/fs.h>
9 #include <linux/bio.h>
10 #include <linux/mpage.h>
11 #include <linux/writeback.h>
12 #include <linux/blkdev.h>
13 #include <linux/f2fs_fs.h>
14 #include <linux/pagevec.h>
15 #include <linux/swap.h>
16 #include <linux/kthread.h>
17
18 #include "f2fs.h"
19 #include "node.h"
20 #include "segment.h"
21 #include "iostat.h"
22 #include <trace/events/f2fs.h>
23 #include <trace/hooks/fs.h>
24
25 #define DEFAULT_CHECKPOINT_IOPRIO (IOPRIO_PRIO_VALUE(IOPRIO_CLASS_RT, 3))
26
27 static struct kmem_cache *ino_entry_slab;
28 struct kmem_cache *f2fs_inode_entry_slab;
29
_trace_android_rvh_f2fs_down_read(wait_queue_head_t * read_waiters,struct rw_semaphore * rwsem,bool * skip)30 void _trace_android_rvh_f2fs_down_read(wait_queue_head_t *read_waiters,
31 struct rw_semaphore *rwsem, bool *skip)
32 {
33 trace_android_rvh_f2fs_down_read(read_waiters, rwsem, skip);
34 }
35 EXPORT_SYMBOL_GPL(_trace_android_rvh_f2fs_down_read);
36
f2fs_stop_checkpoint(struct f2fs_sb_info * sbi,bool end_io,unsigned char reason)37 void f2fs_stop_checkpoint(struct f2fs_sb_info *sbi, bool end_io,
38 unsigned char reason)
39 {
40 f2fs_build_fault_attr(sbi, 0, 0);
41 if (!end_io)
42 f2fs_flush_merged_writes(sbi);
43 f2fs_handle_critical_error(sbi, reason);
44 }
45
46 /*
47 * We guarantee no failure on the returned page.
48 */
f2fs_grab_meta_page(struct f2fs_sb_info * sbi,pgoff_t index)49 struct page *f2fs_grab_meta_page(struct f2fs_sb_info *sbi, pgoff_t index)
50 {
51 struct address_space *mapping = META_MAPPING(sbi);
52 struct page *page;
53 repeat:
54 page = f2fs_grab_cache_page(mapping, index, false);
55 if (!page) {
56 cond_resched();
57 goto repeat;
58 }
59 f2fs_wait_on_page_writeback(page, META, true, true);
60 if (!PageUptodate(page))
61 SetPageUptodate(page);
62 return page;
63 }
64
__get_meta_page(struct f2fs_sb_info * sbi,pgoff_t index,bool is_meta)65 static struct page *__get_meta_page(struct f2fs_sb_info *sbi, pgoff_t index,
66 bool is_meta)
67 {
68 struct address_space *mapping = META_MAPPING(sbi);
69 struct folio *folio;
70 struct f2fs_io_info fio = {
71 .sbi = sbi,
72 .type = META,
73 .op = REQ_OP_READ,
74 .op_flags = REQ_META | REQ_PRIO,
75 .old_blkaddr = index,
76 .new_blkaddr = index,
77 .encrypted_page = NULL,
78 .is_por = !is_meta ? 1 : 0,
79 };
80 int err;
81
82 if (unlikely(!is_meta))
83 fio.op_flags &= ~REQ_META;
84 repeat:
85 folio = f2fs_grab_cache_folio(mapping, index, false);
86 if (IS_ERR(folio)) {
87 cond_resched();
88 goto repeat;
89 }
90 if (folio_test_uptodate(folio))
91 goto out;
92
93 fio.page = &folio->page;
94
95 err = f2fs_submit_page_bio(&fio);
96 if (err) {
97 f2fs_folio_put(folio, true);
98 return ERR_PTR(err);
99 }
100
101 f2fs_update_iostat(sbi, NULL, FS_META_READ_IO, F2FS_BLKSIZE);
102
103 folio_lock(folio);
104 if (unlikely(folio->mapping != mapping)) {
105 f2fs_folio_put(folio, true);
106 goto repeat;
107 }
108
109 if (unlikely(!folio_test_uptodate(folio))) {
110 f2fs_handle_page_eio(sbi, folio, META);
111 f2fs_folio_put(folio, true);
112 return ERR_PTR(-EIO);
113 }
114 out:
115 return &folio->page;
116 }
117
f2fs_get_meta_page(struct f2fs_sb_info * sbi,pgoff_t index)118 struct page *f2fs_get_meta_page(struct f2fs_sb_info *sbi, pgoff_t index)
119 {
120 return __get_meta_page(sbi, index, true);
121 }
122
f2fs_get_meta_page_retry(struct f2fs_sb_info * sbi,pgoff_t index)123 struct page *f2fs_get_meta_page_retry(struct f2fs_sb_info *sbi, pgoff_t index)
124 {
125 struct page *page;
126 int count = 0;
127
128 retry:
129 page = __get_meta_page(sbi, index, true);
130 if (IS_ERR(page)) {
131 if (PTR_ERR(page) == -EIO &&
132 ++count <= DEFAULT_RETRY_IO_COUNT)
133 goto retry;
134 f2fs_stop_checkpoint(sbi, false, STOP_CP_REASON_META_PAGE);
135 }
136 return page;
137 }
138
139 /* for POR only */
f2fs_get_tmp_page(struct f2fs_sb_info * sbi,pgoff_t index)140 struct page *f2fs_get_tmp_page(struct f2fs_sb_info *sbi, pgoff_t index)
141 {
142 return __get_meta_page(sbi, index, false);
143 }
144
__is_bitmap_valid(struct f2fs_sb_info * sbi,block_t blkaddr,int type)145 static bool __is_bitmap_valid(struct f2fs_sb_info *sbi, block_t blkaddr,
146 int type)
147 {
148 struct seg_entry *se;
149 unsigned int segno, offset;
150 bool exist;
151
152 if (type == DATA_GENERIC)
153 return true;
154
155 segno = GET_SEGNO(sbi, blkaddr);
156 offset = GET_BLKOFF_FROM_SEG0(sbi, blkaddr);
157 se = get_seg_entry(sbi, segno);
158
159 exist = f2fs_test_bit(offset, se->cur_valid_map);
160
161 /* skip data, if we already have an error in checkpoint. */
162 if (unlikely(f2fs_cp_error(sbi)))
163 return exist;
164
165 if ((exist && type == DATA_GENERIC_ENHANCE_UPDATE) ||
166 (!exist && type == DATA_GENERIC_ENHANCE))
167 goto out_err;
168 if (!exist && type != DATA_GENERIC_ENHANCE_UPDATE)
169 goto out_handle;
170 return exist;
171
172 out_err:
173 f2fs_err(sbi, "Inconsistent error blkaddr:%u, sit bitmap:%d",
174 blkaddr, exist);
175 set_sbi_flag(sbi, SBI_NEED_FSCK);
176 dump_stack();
177 out_handle:
178 f2fs_handle_error(sbi, ERROR_INVALID_BLKADDR);
179 return exist;
180 }
181
__f2fs_is_valid_blkaddr(struct f2fs_sb_info * sbi,block_t blkaddr,int type)182 static bool __f2fs_is_valid_blkaddr(struct f2fs_sb_info *sbi,
183 block_t blkaddr, int type)
184 {
185 switch (type) {
186 case META_NAT:
187 break;
188 case META_SIT:
189 if (unlikely(blkaddr >= SIT_BLK_CNT(sbi)))
190 goto check_only;
191 break;
192 case META_SSA:
193 if (unlikely(blkaddr >= MAIN_BLKADDR(sbi) ||
194 blkaddr < SM_I(sbi)->ssa_blkaddr))
195 goto check_only;
196 break;
197 case META_CP:
198 if (unlikely(blkaddr >= SIT_I(sbi)->sit_base_addr ||
199 blkaddr < __start_cp_addr(sbi)))
200 goto check_only;
201 break;
202 case META_POR:
203 if (unlikely(blkaddr >= MAX_BLKADDR(sbi) ||
204 blkaddr < MAIN_BLKADDR(sbi)))
205 goto check_only;
206 break;
207 case DATA_GENERIC:
208 case DATA_GENERIC_ENHANCE:
209 case DATA_GENERIC_ENHANCE_READ:
210 case DATA_GENERIC_ENHANCE_UPDATE:
211 if (unlikely(blkaddr >= MAX_BLKADDR(sbi) ||
212 blkaddr < MAIN_BLKADDR(sbi))) {
213
214 /* Skip to emit an error message. */
215 if (unlikely(f2fs_cp_error(sbi)))
216 return false;
217
218 f2fs_warn(sbi, "access invalid blkaddr:%u",
219 blkaddr);
220 set_sbi_flag(sbi, SBI_NEED_FSCK);
221 dump_stack();
222 goto err;
223 } else {
224 return __is_bitmap_valid(sbi, blkaddr, type);
225 }
226 break;
227 case META_GENERIC:
228 if (unlikely(blkaddr < SEG0_BLKADDR(sbi) ||
229 blkaddr >= MAIN_BLKADDR(sbi)))
230 goto err;
231 break;
232 default:
233 BUG();
234 }
235
236 return true;
237 err:
238 f2fs_handle_error(sbi, ERROR_INVALID_BLKADDR);
239 check_only:
240 return false;
241 }
242
f2fs_is_valid_blkaddr(struct f2fs_sb_info * sbi,block_t blkaddr,int type)243 bool f2fs_is_valid_blkaddr(struct f2fs_sb_info *sbi,
244 block_t blkaddr, int type)
245 {
246 if (time_to_inject(sbi, FAULT_BLKADDR_VALIDITY))
247 return false;
248 return __f2fs_is_valid_blkaddr(sbi, blkaddr, type);
249 }
250
f2fs_is_valid_blkaddr_raw(struct f2fs_sb_info * sbi,block_t blkaddr,int type)251 bool f2fs_is_valid_blkaddr_raw(struct f2fs_sb_info *sbi,
252 block_t blkaddr, int type)
253 {
254 return __f2fs_is_valid_blkaddr(sbi, blkaddr, type);
255 }
256
257 /*
258 * Readahead CP/NAT/SIT/SSA/POR pages
259 */
f2fs_ra_meta_pages(struct f2fs_sb_info * sbi,block_t start,int nrpages,int type,bool sync)260 int f2fs_ra_meta_pages(struct f2fs_sb_info *sbi, block_t start, int nrpages,
261 int type, bool sync)
262 {
263 struct page *page;
264 block_t blkno = start;
265 struct f2fs_io_info fio = {
266 .sbi = sbi,
267 .type = META,
268 .op = REQ_OP_READ,
269 .op_flags = sync ? (REQ_META | REQ_PRIO) : REQ_RAHEAD,
270 .encrypted_page = NULL,
271 .in_list = 0,
272 .is_por = (type == META_POR) ? 1 : 0,
273 };
274 struct blk_plug plug;
275 int err;
276
277 if (unlikely(type == META_POR))
278 fio.op_flags &= ~REQ_META;
279
280 blk_start_plug(&plug);
281 for (; nrpages-- > 0; blkno++) {
282
283 if (!f2fs_is_valid_blkaddr(sbi, blkno, type))
284 goto out;
285
286 switch (type) {
287 case META_NAT:
288 if (unlikely(blkno >=
289 NAT_BLOCK_OFFSET(NM_I(sbi)->max_nid)))
290 blkno = 0;
291 /* get nat block addr */
292 fio.new_blkaddr = current_nat_addr(sbi,
293 blkno * NAT_ENTRY_PER_BLOCK);
294 break;
295 case META_SIT:
296 if (unlikely(blkno >= TOTAL_SEGS(sbi)))
297 goto out;
298 /* get sit block addr */
299 fio.new_blkaddr = current_sit_addr(sbi,
300 blkno * SIT_ENTRY_PER_BLOCK);
301 break;
302 case META_SSA:
303 case META_CP:
304 case META_POR:
305 fio.new_blkaddr = blkno;
306 break;
307 default:
308 BUG();
309 }
310
311 page = f2fs_grab_cache_page(META_MAPPING(sbi),
312 fio.new_blkaddr, false);
313 if (!page)
314 continue;
315 if (PageUptodate(page)) {
316 f2fs_put_page(page, 1);
317 continue;
318 }
319
320 fio.page = page;
321 err = f2fs_submit_page_bio(&fio);
322 f2fs_put_page(page, err ? 1 : 0);
323
324 if (!err)
325 f2fs_update_iostat(sbi, NULL, FS_META_READ_IO,
326 F2FS_BLKSIZE);
327 }
328 out:
329 blk_finish_plug(&plug);
330 return blkno - start;
331 }
332
f2fs_ra_meta_pages_cond(struct f2fs_sb_info * sbi,pgoff_t index,unsigned int ra_blocks)333 void f2fs_ra_meta_pages_cond(struct f2fs_sb_info *sbi, pgoff_t index,
334 unsigned int ra_blocks)
335 {
336 struct page *page;
337 bool readahead = false;
338
339 if (ra_blocks == RECOVERY_MIN_RA_BLOCKS)
340 return;
341
342 page = find_get_page(META_MAPPING(sbi), index);
343 if (!page || !PageUptodate(page))
344 readahead = true;
345 f2fs_put_page(page, 0);
346
347 if (readahead)
348 f2fs_ra_meta_pages(sbi, index, ra_blocks, META_POR, true);
349 }
350
__f2fs_write_meta_page(struct page * page,struct writeback_control * wbc,enum iostat_type io_type)351 static int __f2fs_write_meta_page(struct page *page,
352 struct writeback_control *wbc,
353 enum iostat_type io_type)
354 {
355 struct f2fs_sb_info *sbi = F2FS_P_SB(page);
356 struct folio *folio = page_folio(page);
357
358 trace_f2fs_writepage(folio, META);
359
360 if (unlikely(f2fs_cp_error(sbi))) {
361 if (is_sbi_flag_set(sbi, SBI_IS_CLOSE)) {
362 folio_clear_uptodate(folio);
363 dec_page_count(sbi, F2FS_DIRTY_META);
364 folio_unlock(folio);
365 return 0;
366 }
367 goto redirty_out;
368 }
369 if (unlikely(is_sbi_flag_set(sbi, SBI_POR_DOING)))
370 goto redirty_out;
371 if (wbc->for_reclaim && folio->index < GET_SUM_BLOCK(sbi, 0))
372 goto redirty_out;
373
374 f2fs_do_write_meta_page(sbi, folio, io_type);
375 dec_page_count(sbi, F2FS_DIRTY_META);
376
377 if (wbc->for_reclaim)
378 f2fs_submit_merged_write_cond(sbi, NULL, page, 0, META);
379
380 folio_unlock(folio);
381
382 if (unlikely(f2fs_cp_error(sbi)))
383 f2fs_submit_merged_write(sbi, META);
384
385 return 0;
386
387 redirty_out:
388 redirty_page_for_writepage(wbc, page);
389 return AOP_WRITEPAGE_ACTIVATE;
390 }
391
f2fs_write_meta_pages(struct address_space * mapping,struct writeback_control * wbc)392 static int f2fs_write_meta_pages(struct address_space *mapping,
393 struct writeback_control *wbc)
394 {
395 struct f2fs_sb_info *sbi = F2FS_M_SB(mapping);
396 long diff, written;
397
398 if (unlikely(is_sbi_flag_set(sbi, SBI_POR_DOING)))
399 goto skip_write;
400
401 /* collect a number of dirty meta pages and write together */
402 if (wbc->sync_mode != WB_SYNC_ALL &&
403 get_pages(sbi, F2FS_DIRTY_META) <
404 nr_pages_to_skip(sbi, META))
405 goto skip_write;
406
407 /* if locked failed, cp will flush dirty pages instead */
408 if (!f2fs_down_write_trylock(&sbi->cp_global_sem))
409 goto skip_write;
410
411 trace_f2fs_writepages(mapping->host, wbc, META);
412 diff = nr_pages_to_write(sbi, META, wbc);
413 written = f2fs_sync_meta_pages(sbi, META, wbc->nr_to_write, FS_META_IO);
414 f2fs_up_write(&sbi->cp_global_sem);
415 wbc->nr_to_write = max((long)0, wbc->nr_to_write - written - diff);
416 return 0;
417
418 skip_write:
419 wbc->pages_skipped += get_pages(sbi, F2FS_DIRTY_META);
420 trace_f2fs_writepages(mapping->host, wbc, META);
421 return 0;
422 }
423
f2fs_sync_meta_pages(struct f2fs_sb_info * sbi,enum page_type type,long nr_to_write,enum iostat_type io_type)424 long f2fs_sync_meta_pages(struct f2fs_sb_info *sbi, enum page_type type,
425 long nr_to_write, enum iostat_type io_type)
426 {
427 struct address_space *mapping = META_MAPPING(sbi);
428 pgoff_t index = 0, prev = ULONG_MAX;
429 struct folio_batch fbatch;
430 long nwritten = 0;
431 int nr_folios;
432 struct writeback_control wbc = {
433 .for_reclaim = 0,
434 };
435 struct blk_plug plug;
436
437 folio_batch_init(&fbatch);
438
439 blk_start_plug(&plug);
440
441 while ((nr_folios = filemap_get_folios_tag(mapping, &index,
442 (pgoff_t)-1,
443 PAGECACHE_TAG_DIRTY, &fbatch))) {
444 int i;
445
446 for (i = 0; i < nr_folios; i++) {
447 struct folio *folio = fbatch.folios[i];
448
449 if (nr_to_write != LONG_MAX && i != 0 &&
450 folio->index != prev +
451 folio_nr_pages(fbatch.folios[i-1])) {
452 folio_batch_release(&fbatch);
453 goto stop;
454 }
455
456 folio_lock(folio);
457
458 if (unlikely(folio->mapping != mapping)) {
459 continue_unlock:
460 folio_unlock(folio);
461 continue;
462 }
463 if (!folio_test_dirty(folio)) {
464 /* someone wrote it for us */
465 goto continue_unlock;
466 }
467
468 f2fs_wait_on_page_writeback(&folio->page, META,
469 true, true);
470
471 if (!folio_clear_dirty_for_io(folio))
472 goto continue_unlock;
473
474 if (__f2fs_write_meta_page(&folio->page, &wbc,
475 io_type)) {
476 folio_unlock(folio);
477 break;
478 }
479 nwritten += folio_nr_pages(folio);
480 prev = folio->index;
481 if (unlikely(nwritten >= nr_to_write))
482 break;
483 }
484 folio_batch_release(&fbatch);
485 cond_resched();
486 }
487 stop:
488 if (nwritten)
489 f2fs_submit_merged_write(sbi, type);
490
491 blk_finish_plug(&plug);
492
493 return nwritten;
494 }
495
f2fs_dirty_meta_folio(struct address_space * mapping,struct folio * folio)496 static bool f2fs_dirty_meta_folio(struct address_space *mapping,
497 struct folio *folio)
498 {
499 trace_f2fs_set_page_dirty(folio, META);
500
501 if (!folio_test_uptodate(folio))
502 folio_mark_uptodate(folio);
503 if (filemap_dirty_folio(mapping, folio)) {
504 inc_page_count(F2FS_M_SB(mapping), F2FS_DIRTY_META);
505 set_page_private_reference(&folio->page);
506 return true;
507 }
508 return false;
509 }
510
511 const struct address_space_operations f2fs_meta_aops = {
512 .writepages = f2fs_write_meta_pages,
513 .dirty_folio = f2fs_dirty_meta_folio,
514 .invalidate_folio = f2fs_invalidate_folio,
515 .release_folio = f2fs_release_folio,
516 .migrate_folio = filemap_migrate_folio,
517 };
518
__add_ino_entry(struct f2fs_sb_info * sbi,nid_t ino,unsigned int devidx,int type)519 static void __add_ino_entry(struct f2fs_sb_info *sbi, nid_t ino,
520 unsigned int devidx, int type)
521 {
522 struct inode_management *im = &sbi->im[type];
523 struct ino_entry *e = NULL, *new = NULL;
524
525 if (type == FLUSH_INO) {
526 rcu_read_lock();
527 e = radix_tree_lookup(&im->ino_root, ino);
528 rcu_read_unlock();
529 }
530
531 retry:
532 if (!e)
533 new = f2fs_kmem_cache_alloc(ino_entry_slab,
534 GFP_NOFS, true, NULL);
535
536 radix_tree_preload(GFP_NOFS | __GFP_NOFAIL);
537
538 spin_lock(&im->ino_lock);
539 e = radix_tree_lookup(&im->ino_root, ino);
540 if (!e) {
541 if (!new) {
542 spin_unlock(&im->ino_lock);
543 radix_tree_preload_end();
544 goto retry;
545 }
546 e = new;
547 if (unlikely(radix_tree_insert(&im->ino_root, ino, e)))
548 f2fs_bug_on(sbi, 1);
549
550 memset(e, 0, sizeof(struct ino_entry));
551 e->ino = ino;
552
553 list_add_tail(&e->list, &im->ino_list);
554 if (type != ORPHAN_INO)
555 im->ino_num++;
556 }
557
558 if (type == FLUSH_INO)
559 f2fs_set_bit(devidx, (char *)&e->dirty_device);
560
561 spin_unlock(&im->ino_lock);
562 radix_tree_preload_end();
563
564 if (new && e != new)
565 kmem_cache_free(ino_entry_slab, new);
566 }
567
__remove_ino_entry(struct f2fs_sb_info * sbi,nid_t ino,int type)568 static void __remove_ino_entry(struct f2fs_sb_info *sbi, nid_t ino, int type)
569 {
570 struct inode_management *im = &sbi->im[type];
571 struct ino_entry *e;
572
573 spin_lock(&im->ino_lock);
574 e = radix_tree_lookup(&im->ino_root, ino);
575 if (e) {
576 list_del(&e->list);
577 radix_tree_delete(&im->ino_root, ino);
578 im->ino_num--;
579 spin_unlock(&im->ino_lock);
580 kmem_cache_free(ino_entry_slab, e);
581 return;
582 }
583 spin_unlock(&im->ino_lock);
584 }
585
f2fs_add_ino_entry(struct f2fs_sb_info * sbi,nid_t ino,int type)586 void f2fs_add_ino_entry(struct f2fs_sb_info *sbi, nid_t ino, int type)
587 {
588 /* add new dirty ino entry into list */
589 __add_ino_entry(sbi, ino, 0, type);
590 }
591
f2fs_remove_ino_entry(struct f2fs_sb_info * sbi,nid_t ino,int type)592 void f2fs_remove_ino_entry(struct f2fs_sb_info *sbi, nid_t ino, int type)
593 {
594 /* remove dirty ino entry from list */
595 __remove_ino_entry(sbi, ino, type);
596 }
597
598 /* mode should be APPEND_INO, UPDATE_INO or TRANS_DIR_INO */
f2fs_exist_written_data(struct f2fs_sb_info * sbi,nid_t ino,int mode)599 bool f2fs_exist_written_data(struct f2fs_sb_info *sbi, nid_t ino, int mode)
600 {
601 struct inode_management *im = &sbi->im[mode];
602 struct ino_entry *e;
603
604 spin_lock(&im->ino_lock);
605 e = radix_tree_lookup(&im->ino_root, ino);
606 spin_unlock(&im->ino_lock);
607 return e ? true : false;
608 }
609
f2fs_release_ino_entry(struct f2fs_sb_info * sbi,bool all)610 void f2fs_release_ino_entry(struct f2fs_sb_info *sbi, bool all)
611 {
612 struct ino_entry *e, *tmp;
613 int i;
614
615 for (i = all ? ORPHAN_INO : APPEND_INO; i < MAX_INO_ENTRY; i++) {
616 struct inode_management *im = &sbi->im[i];
617
618 spin_lock(&im->ino_lock);
619 list_for_each_entry_safe(e, tmp, &im->ino_list, list) {
620 list_del(&e->list);
621 radix_tree_delete(&im->ino_root, e->ino);
622 kmem_cache_free(ino_entry_slab, e);
623 im->ino_num--;
624 }
625 spin_unlock(&im->ino_lock);
626 }
627 }
628
f2fs_set_dirty_device(struct f2fs_sb_info * sbi,nid_t ino,unsigned int devidx,int type)629 void f2fs_set_dirty_device(struct f2fs_sb_info *sbi, nid_t ino,
630 unsigned int devidx, int type)
631 {
632 __add_ino_entry(sbi, ino, devidx, type);
633 }
634
f2fs_is_dirty_device(struct f2fs_sb_info * sbi,nid_t ino,unsigned int devidx,int type)635 bool f2fs_is_dirty_device(struct f2fs_sb_info *sbi, nid_t ino,
636 unsigned int devidx, int type)
637 {
638 struct inode_management *im = &sbi->im[type];
639 struct ino_entry *e;
640 bool is_dirty = false;
641
642 spin_lock(&im->ino_lock);
643 e = radix_tree_lookup(&im->ino_root, ino);
644 if (e && f2fs_test_bit(devidx, (char *)&e->dirty_device))
645 is_dirty = true;
646 spin_unlock(&im->ino_lock);
647 return is_dirty;
648 }
649
f2fs_acquire_orphan_inode(struct f2fs_sb_info * sbi)650 int f2fs_acquire_orphan_inode(struct f2fs_sb_info *sbi)
651 {
652 struct inode_management *im = &sbi->im[ORPHAN_INO];
653 int err = 0;
654
655 spin_lock(&im->ino_lock);
656
657 if (time_to_inject(sbi, FAULT_ORPHAN)) {
658 spin_unlock(&im->ino_lock);
659 return -ENOSPC;
660 }
661
662 if (unlikely(im->ino_num >= sbi->max_orphans))
663 err = -ENOSPC;
664 else
665 im->ino_num++;
666 spin_unlock(&im->ino_lock);
667
668 return err;
669 }
670
f2fs_release_orphan_inode(struct f2fs_sb_info * sbi)671 void f2fs_release_orphan_inode(struct f2fs_sb_info *sbi)
672 {
673 struct inode_management *im = &sbi->im[ORPHAN_INO];
674
675 spin_lock(&im->ino_lock);
676 f2fs_bug_on(sbi, im->ino_num == 0);
677 im->ino_num--;
678 spin_unlock(&im->ino_lock);
679 }
680
f2fs_add_orphan_inode(struct inode * inode)681 void f2fs_add_orphan_inode(struct inode *inode)
682 {
683 /* add new orphan ino entry into list */
684 __add_ino_entry(F2FS_I_SB(inode), inode->i_ino, 0, ORPHAN_INO);
685 f2fs_update_inode_page(inode);
686 }
687
f2fs_remove_orphan_inode(struct f2fs_sb_info * sbi,nid_t ino)688 void f2fs_remove_orphan_inode(struct f2fs_sb_info *sbi, nid_t ino)
689 {
690 /* remove orphan entry from orphan list */
691 __remove_ino_entry(sbi, ino, ORPHAN_INO);
692 }
693
recover_orphan_inode(struct f2fs_sb_info * sbi,nid_t ino)694 static int recover_orphan_inode(struct f2fs_sb_info *sbi, nid_t ino)
695 {
696 struct inode *inode;
697 struct node_info ni;
698 int err;
699
700 inode = f2fs_iget_retry(sbi->sb, ino);
701 if (IS_ERR(inode)) {
702 /*
703 * there should be a bug that we can't find the entry
704 * to orphan inode.
705 */
706 f2fs_bug_on(sbi, PTR_ERR(inode) == -ENOENT);
707 return PTR_ERR(inode);
708 }
709
710 err = f2fs_dquot_initialize(inode);
711 if (err) {
712 iput(inode);
713 goto err_out;
714 }
715
716 clear_nlink(inode);
717
718 /* truncate all the data during iput */
719 iput(inode);
720
721 err = f2fs_get_node_info(sbi, ino, &ni, false);
722 if (err)
723 goto err_out;
724
725 /* ENOMEM was fully retried in f2fs_evict_inode. */
726 if (ni.blk_addr != NULL_ADDR) {
727 err = -EIO;
728 goto err_out;
729 }
730 return 0;
731
732 err_out:
733 set_sbi_flag(sbi, SBI_NEED_FSCK);
734 f2fs_warn(sbi, "%s: orphan failed (ino=%x), run fsck to fix.",
735 __func__, ino);
736 return err;
737 }
738
f2fs_recover_orphan_inodes(struct f2fs_sb_info * sbi)739 int f2fs_recover_orphan_inodes(struct f2fs_sb_info *sbi)
740 {
741 block_t start_blk, orphan_blocks, i, j;
742 int err = 0;
743
744 if (!is_set_ckpt_flags(sbi, CP_ORPHAN_PRESENT_FLAG))
745 return 0;
746
747 if (f2fs_hw_is_readonly(sbi)) {
748 f2fs_info(sbi, "write access unavailable, skipping orphan cleanup");
749 return 0;
750 }
751
752 if (is_sbi_flag_set(sbi, SBI_IS_WRITABLE))
753 f2fs_info(sbi, "orphan cleanup on readonly fs");
754
755 start_blk = __start_cp_addr(sbi) + 1 + __cp_payload(sbi);
756 orphan_blocks = __start_sum_addr(sbi) - 1 - __cp_payload(sbi);
757
758 f2fs_ra_meta_pages(sbi, start_blk, orphan_blocks, META_CP, true);
759
760 for (i = 0; i < orphan_blocks; i++) {
761 struct page *page;
762 struct f2fs_orphan_block *orphan_blk;
763
764 page = f2fs_get_meta_page(sbi, start_blk + i);
765 if (IS_ERR(page)) {
766 err = PTR_ERR(page);
767 goto out;
768 }
769
770 orphan_blk = (struct f2fs_orphan_block *)page_address(page);
771 for (j = 0; j < le32_to_cpu(orphan_blk->entry_count); j++) {
772 nid_t ino = le32_to_cpu(orphan_blk->ino[j]);
773
774 err = recover_orphan_inode(sbi, ino);
775 if (err) {
776 f2fs_put_page(page, 1);
777 goto out;
778 }
779 }
780 f2fs_put_page(page, 1);
781 }
782 /* clear Orphan Flag */
783 clear_ckpt_flags(sbi, CP_ORPHAN_PRESENT_FLAG);
784 out:
785 set_sbi_flag(sbi, SBI_IS_RECOVERED);
786
787 return err;
788 }
789
write_orphan_inodes(struct f2fs_sb_info * sbi,block_t start_blk)790 static void write_orphan_inodes(struct f2fs_sb_info *sbi, block_t start_blk)
791 {
792 struct list_head *head;
793 struct f2fs_orphan_block *orphan_blk = NULL;
794 unsigned int nentries = 0;
795 unsigned short index = 1;
796 unsigned short orphan_blocks;
797 struct page *page = NULL;
798 struct ino_entry *orphan = NULL;
799 struct inode_management *im = &sbi->im[ORPHAN_INO];
800
801 orphan_blocks = GET_ORPHAN_BLOCKS(im->ino_num);
802
803 /*
804 * we don't need to do spin_lock(&im->ino_lock) here, since all the
805 * orphan inode operations are covered under f2fs_lock_op().
806 * And, spin_lock should be avoided due to page operations below.
807 */
808 head = &im->ino_list;
809
810 /* loop for each orphan inode entry and write them in journal block */
811 list_for_each_entry(orphan, head, list) {
812 if (!page) {
813 page = f2fs_grab_meta_page(sbi, start_blk++);
814 orphan_blk =
815 (struct f2fs_orphan_block *)page_address(page);
816 memset(orphan_blk, 0, sizeof(*orphan_blk));
817 }
818
819 orphan_blk->ino[nentries++] = cpu_to_le32(orphan->ino);
820
821 if (nentries == F2FS_ORPHANS_PER_BLOCK) {
822 /*
823 * an orphan block is full of 1020 entries,
824 * then we need to flush current orphan blocks
825 * and bring another one in memory
826 */
827 orphan_blk->blk_addr = cpu_to_le16(index);
828 orphan_blk->blk_count = cpu_to_le16(orphan_blocks);
829 orphan_blk->entry_count = cpu_to_le32(nentries);
830 set_page_dirty(page);
831 f2fs_put_page(page, 1);
832 index++;
833 nentries = 0;
834 page = NULL;
835 }
836 }
837
838 if (page) {
839 orphan_blk->blk_addr = cpu_to_le16(index);
840 orphan_blk->blk_count = cpu_to_le16(orphan_blocks);
841 orphan_blk->entry_count = cpu_to_le32(nentries);
842 set_page_dirty(page);
843 f2fs_put_page(page, 1);
844 }
845 }
846
f2fs_checkpoint_chksum(struct f2fs_sb_info * sbi,struct f2fs_checkpoint * ckpt)847 static __u32 f2fs_checkpoint_chksum(struct f2fs_sb_info *sbi,
848 struct f2fs_checkpoint *ckpt)
849 {
850 unsigned int chksum_ofs = le32_to_cpu(ckpt->checksum_offset);
851 __u32 chksum;
852
853 chksum = f2fs_crc32(sbi, ckpt, chksum_ofs);
854 if (chksum_ofs < CP_CHKSUM_OFFSET) {
855 chksum_ofs += sizeof(chksum);
856 chksum = f2fs_chksum(sbi, chksum, (__u8 *)ckpt + chksum_ofs,
857 F2FS_BLKSIZE - chksum_ofs);
858 }
859 return chksum;
860 }
861
get_checkpoint_version(struct f2fs_sb_info * sbi,block_t cp_addr,struct f2fs_checkpoint ** cp_block,struct page ** cp_page,unsigned long long * version)862 static int get_checkpoint_version(struct f2fs_sb_info *sbi, block_t cp_addr,
863 struct f2fs_checkpoint **cp_block, struct page **cp_page,
864 unsigned long long *version)
865 {
866 size_t crc_offset = 0;
867 __u32 crc;
868
869 *cp_page = f2fs_get_meta_page(sbi, cp_addr);
870 if (IS_ERR(*cp_page))
871 return PTR_ERR(*cp_page);
872
873 *cp_block = (struct f2fs_checkpoint *)page_address(*cp_page);
874
875 crc_offset = le32_to_cpu((*cp_block)->checksum_offset);
876 if (crc_offset < CP_MIN_CHKSUM_OFFSET ||
877 crc_offset > CP_CHKSUM_OFFSET) {
878 f2fs_put_page(*cp_page, 1);
879 f2fs_warn(sbi, "invalid crc_offset: %zu", crc_offset);
880 return -EINVAL;
881 }
882
883 crc = f2fs_checkpoint_chksum(sbi, *cp_block);
884 if (crc != cur_cp_crc(*cp_block)) {
885 f2fs_put_page(*cp_page, 1);
886 f2fs_warn(sbi, "invalid crc value");
887 return -EINVAL;
888 }
889
890 *version = cur_cp_version(*cp_block);
891 return 0;
892 }
893
validate_checkpoint(struct f2fs_sb_info * sbi,block_t cp_addr,unsigned long long * version)894 static struct page *validate_checkpoint(struct f2fs_sb_info *sbi,
895 block_t cp_addr, unsigned long long *version)
896 {
897 struct page *cp_page_1 = NULL, *cp_page_2 = NULL;
898 struct f2fs_checkpoint *cp_block = NULL;
899 unsigned long long cur_version = 0, pre_version = 0;
900 unsigned int cp_blocks;
901 int err;
902
903 err = get_checkpoint_version(sbi, cp_addr, &cp_block,
904 &cp_page_1, version);
905 if (err)
906 return NULL;
907
908 cp_blocks = le32_to_cpu(cp_block->cp_pack_total_block_count);
909
910 if (cp_blocks > BLKS_PER_SEG(sbi) || cp_blocks <= F2FS_CP_PACKS) {
911 f2fs_warn(sbi, "invalid cp_pack_total_block_count:%u",
912 le32_to_cpu(cp_block->cp_pack_total_block_count));
913 goto invalid_cp;
914 }
915 pre_version = *version;
916
917 cp_addr += cp_blocks - 1;
918 err = get_checkpoint_version(sbi, cp_addr, &cp_block,
919 &cp_page_2, version);
920 if (err)
921 goto invalid_cp;
922 cur_version = *version;
923
924 if (cur_version == pre_version) {
925 *version = cur_version;
926 f2fs_put_page(cp_page_2, 1);
927 return cp_page_1;
928 }
929 f2fs_put_page(cp_page_2, 1);
930 invalid_cp:
931 f2fs_put_page(cp_page_1, 1);
932 return NULL;
933 }
934
f2fs_get_valid_checkpoint(struct f2fs_sb_info * sbi)935 int f2fs_get_valid_checkpoint(struct f2fs_sb_info *sbi)
936 {
937 struct f2fs_checkpoint *cp_block;
938 struct f2fs_super_block *fsb = sbi->raw_super;
939 struct page *cp1, *cp2, *cur_page;
940 unsigned long blk_size = sbi->blocksize;
941 unsigned long long cp1_version = 0, cp2_version = 0;
942 unsigned long long cp_start_blk_no;
943 unsigned int cp_blks = 1 + __cp_payload(sbi);
944 block_t cp_blk_no;
945 int i;
946 int err;
947
948 sbi->ckpt = f2fs_kvzalloc(sbi, array_size(blk_size, cp_blks),
949 GFP_KERNEL);
950 if (!sbi->ckpt)
951 return -ENOMEM;
952 /*
953 * Finding out valid cp block involves read both
954 * sets( cp pack 1 and cp pack 2)
955 */
956 cp_start_blk_no = le32_to_cpu(fsb->cp_blkaddr);
957 cp1 = validate_checkpoint(sbi, cp_start_blk_no, &cp1_version);
958
959 /* The second checkpoint pack should start at the next segment */
960 cp_start_blk_no += ((unsigned long long)1) <<
961 le32_to_cpu(fsb->log_blocks_per_seg);
962 cp2 = validate_checkpoint(sbi, cp_start_blk_no, &cp2_version);
963
964 if (cp1 && cp2) {
965 if (ver_after(cp2_version, cp1_version))
966 cur_page = cp2;
967 else
968 cur_page = cp1;
969 } else if (cp1) {
970 cur_page = cp1;
971 } else if (cp2) {
972 cur_page = cp2;
973 } else {
974 err = -EFSCORRUPTED;
975 goto fail_no_cp;
976 }
977
978 cp_block = (struct f2fs_checkpoint *)page_address(cur_page);
979 memcpy(sbi->ckpt, cp_block, blk_size);
980
981 if (cur_page == cp1)
982 sbi->cur_cp_pack = 1;
983 else
984 sbi->cur_cp_pack = 2;
985
986 /* Sanity checking of checkpoint */
987 if (f2fs_sanity_check_ckpt(sbi)) {
988 err = -EFSCORRUPTED;
989 goto free_fail_no_cp;
990 }
991
992 if (cp_blks <= 1)
993 goto done;
994
995 cp_blk_no = le32_to_cpu(fsb->cp_blkaddr);
996 if (cur_page == cp2)
997 cp_blk_no += BIT(le32_to_cpu(fsb->log_blocks_per_seg));
998
999 for (i = 1; i < cp_blks; i++) {
1000 void *sit_bitmap_ptr;
1001 unsigned char *ckpt = (unsigned char *)sbi->ckpt;
1002
1003 cur_page = f2fs_get_meta_page(sbi, cp_blk_no + i);
1004 if (IS_ERR(cur_page)) {
1005 err = PTR_ERR(cur_page);
1006 goto free_fail_no_cp;
1007 }
1008 sit_bitmap_ptr = page_address(cur_page);
1009 memcpy(ckpt + i * blk_size, sit_bitmap_ptr, blk_size);
1010 f2fs_put_page(cur_page, 1);
1011 }
1012 done:
1013 f2fs_put_page(cp1, 1);
1014 f2fs_put_page(cp2, 1);
1015 return 0;
1016
1017 free_fail_no_cp:
1018 f2fs_put_page(cp1, 1);
1019 f2fs_put_page(cp2, 1);
1020 fail_no_cp:
1021 kvfree(sbi->ckpt);
1022 return err;
1023 }
1024
__add_dirty_inode(struct inode * inode,enum inode_type type)1025 static void __add_dirty_inode(struct inode *inode, enum inode_type type)
1026 {
1027 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
1028 int flag = (type == DIR_INODE) ? FI_DIRTY_DIR : FI_DIRTY_FILE;
1029
1030 if (is_inode_flag_set(inode, flag))
1031 return;
1032
1033 set_inode_flag(inode, flag);
1034 list_add_tail(&F2FS_I(inode)->dirty_list, &sbi->inode_list[type]);
1035 stat_inc_dirty_inode(sbi, type);
1036 }
1037
__remove_dirty_inode(struct inode * inode,enum inode_type type)1038 static void __remove_dirty_inode(struct inode *inode, enum inode_type type)
1039 {
1040 int flag = (type == DIR_INODE) ? FI_DIRTY_DIR : FI_DIRTY_FILE;
1041
1042 if (get_dirty_pages(inode) || !is_inode_flag_set(inode, flag))
1043 return;
1044
1045 list_del_init(&F2FS_I(inode)->dirty_list);
1046 clear_inode_flag(inode, flag);
1047 stat_dec_dirty_inode(F2FS_I_SB(inode), type);
1048 }
1049
f2fs_update_dirty_folio(struct inode * inode,struct folio * folio)1050 void f2fs_update_dirty_folio(struct inode *inode, struct folio *folio)
1051 {
1052 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
1053 enum inode_type type = S_ISDIR(inode->i_mode) ? DIR_INODE : FILE_INODE;
1054
1055 if (!S_ISDIR(inode->i_mode) && !S_ISREG(inode->i_mode) &&
1056 !S_ISLNK(inode->i_mode))
1057 return;
1058
1059 spin_lock(&sbi->inode_lock[type]);
1060 if (type != FILE_INODE || test_opt(sbi, DATA_FLUSH))
1061 __add_dirty_inode(inode, type);
1062 inode_inc_dirty_pages(inode);
1063 spin_unlock(&sbi->inode_lock[type]);
1064
1065 set_page_private_reference(&folio->page);
1066 }
1067
f2fs_remove_dirty_inode(struct inode * inode)1068 void f2fs_remove_dirty_inode(struct inode *inode)
1069 {
1070 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
1071 enum inode_type type = S_ISDIR(inode->i_mode) ? DIR_INODE : FILE_INODE;
1072
1073 if (!S_ISDIR(inode->i_mode) && !S_ISREG(inode->i_mode) &&
1074 !S_ISLNK(inode->i_mode))
1075 return;
1076
1077 if (type == FILE_INODE && !test_opt(sbi, DATA_FLUSH))
1078 return;
1079
1080 spin_lock(&sbi->inode_lock[type]);
1081 __remove_dirty_inode(inode, type);
1082 spin_unlock(&sbi->inode_lock[type]);
1083 }
1084
f2fs_sync_dirty_inodes(struct f2fs_sb_info * sbi,enum inode_type type,bool from_cp)1085 int f2fs_sync_dirty_inodes(struct f2fs_sb_info *sbi, enum inode_type type,
1086 bool from_cp)
1087 {
1088 struct list_head *head;
1089 struct inode *inode;
1090 struct f2fs_inode_info *fi;
1091 bool is_dir = (type == DIR_INODE);
1092 unsigned long ino = 0;
1093
1094 trace_f2fs_sync_dirty_inodes_enter(sbi->sb, is_dir,
1095 get_pages(sbi, is_dir ?
1096 F2FS_DIRTY_DENTS : F2FS_DIRTY_DATA));
1097 retry:
1098 if (unlikely(f2fs_cp_error(sbi))) {
1099 trace_f2fs_sync_dirty_inodes_exit(sbi->sb, is_dir,
1100 get_pages(sbi, is_dir ?
1101 F2FS_DIRTY_DENTS : F2FS_DIRTY_DATA));
1102 return -EIO;
1103 }
1104
1105 spin_lock(&sbi->inode_lock[type]);
1106
1107 head = &sbi->inode_list[type];
1108 if (list_empty(head)) {
1109 spin_unlock(&sbi->inode_lock[type]);
1110 trace_f2fs_sync_dirty_inodes_exit(sbi->sb, is_dir,
1111 get_pages(sbi, is_dir ?
1112 F2FS_DIRTY_DENTS : F2FS_DIRTY_DATA));
1113 return 0;
1114 }
1115 fi = list_first_entry(head, struct f2fs_inode_info, dirty_list);
1116 inode = igrab(&fi->vfs_inode);
1117 spin_unlock(&sbi->inode_lock[type]);
1118 if (inode) {
1119 unsigned long cur_ino = inode->i_ino;
1120
1121 if (from_cp)
1122 F2FS_I(inode)->cp_task = current;
1123 F2FS_I(inode)->wb_task = current;
1124
1125 filemap_fdatawrite(inode->i_mapping);
1126
1127 F2FS_I(inode)->wb_task = NULL;
1128 if (from_cp)
1129 F2FS_I(inode)->cp_task = NULL;
1130
1131 iput(inode);
1132 /* We need to give cpu to another writers. */
1133 if (ino == cur_ino)
1134 cond_resched();
1135 else
1136 ino = cur_ino;
1137 } else {
1138 /*
1139 * We should submit bio, since it exists several
1140 * writebacking dentry pages in the freeing inode.
1141 */
1142 f2fs_submit_merged_write(sbi, DATA);
1143 cond_resched();
1144 }
1145 goto retry;
1146 }
1147
f2fs_sync_inode_meta(struct f2fs_sb_info * sbi)1148 static int f2fs_sync_inode_meta(struct f2fs_sb_info *sbi)
1149 {
1150 struct list_head *head = &sbi->inode_list[DIRTY_META];
1151 struct inode *inode;
1152 struct f2fs_inode_info *fi;
1153 s64 total = get_pages(sbi, F2FS_DIRTY_IMETA);
1154
1155 while (total--) {
1156 if (unlikely(f2fs_cp_error(sbi)))
1157 return -EIO;
1158
1159 spin_lock(&sbi->inode_lock[DIRTY_META]);
1160 if (list_empty(head)) {
1161 spin_unlock(&sbi->inode_lock[DIRTY_META]);
1162 return 0;
1163 }
1164 fi = list_first_entry(head, struct f2fs_inode_info,
1165 gdirty_list);
1166 inode = igrab(&fi->vfs_inode);
1167 spin_unlock(&sbi->inode_lock[DIRTY_META]);
1168 if (inode) {
1169 sync_inode_metadata(inode, 0);
1170
1171 /* it's on eviction */
1172 if (is_inode_flag_set(inode, FI_DIRTY_INODE))
1173 f2fs_update_inode_page(inode);
1174 iput(inode);
1175 }
1176 }
1177 return 0;
1178 }
1179
__prepare_cp_block(struct f2fs_sb_info * sbi)1180 static void __prepare_cp_block(struct f2fs_sb_info *sbi)
1181 {
1182 struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi);
1183 struct f2fs_nm_info *nm_i = NM_I(sbi);
1184 nid_t last_nid = nm_i->next_scan_nid;
1185
1186 next_free_nid(sbi, &last_nid);
1187 ckpt->valid_block_count = cpu_to_le64(valid_user_blocks(sbi));
1188 ckpt->valid_node_count = cpu_to_le32(valid_node_count(sbi));
1189 ckpt->valid_inode_count = cpu_to_le32(valid_inode_count(sbi));
1190 ckpt->next_free_nid = cpu_to_le32(last_nid);
1191
1192 /* update user_block_counts */
1193 sbi->last_valid_block_count = sbi->total_valid_block_count;
1194 percpu_counter_set(&sbi->alloc_valid_block_count, 0);
1195 percpu_counter_set(&sbi->rf_node_block_count, 0);
1196 }
1197
__need_flush_quota(struct f2fs_sb_info * sbi)1198 static bool __need_flush_quota(struct f2fs_sb_info *sbi)
1199 {
1200 bool ret = false;
1201
1202 if (!is_journalled_quota(sbi))
1203 return false;
1204
1205 if (!f2fs_down_write_trylock(&sbi->quota_sem))
1206 return true;
1207 if (is_sbi_flag_set(sbi, SBI_QUOTA_SKIP_FLUSH)) {
1208 ret = false;
1209 } else if (is_sbi_flag_set(sbi, SBI_QUOTA_NEED_REPAIR)) {
1210 ret = false;
1211 } else if (is_sbi_flag_set(sbi, SBI_QUOTA_NEED_FLUSH)) {
1212 clear_sbi_flag(sbi, SBI_QUOTA_NEED_FLUSH);
1213 ret = true;
1214 } else if (get_pages(sbi, F2FS_DIRTY_QDATA)) {
1215 ret = true;
1216 }
1217 f2fs_up_write(&sbi->quota_sem);
1218 return ret;
1219 }
1220
1221 /*
1222 * Freeze all the FS-operations for checkpoint.
1223 */
block_operations(struct f2fs_sb_info * sbi)1224 static int block_operations(struct f2fs_sb_info *sbi)
1225 {
1226 struct writeback_control wbc = {
1227 .sync_mode = WB_SYNC_ALL,
1228 .nr_to_write = LONG_MAX,
1229 .for_reclaim = 0,
1230 };
1231 int err = 0, cnt = 0;
1232
1233 /*
1234 * Let's flush inline_data in dirty node pages.
1235 */
1236 f2fs_flush_inline_data(sbi);
1237
1238 retry_flush_quotas:
1239 f2fs_lock_all(sbi);
1240 if (__need_flush_quota(sbi)) {
1241 bool need_lock = sbi->umount_lock_holder != current;
1242
1243 if (++cnt > DEFAULT_RETRY_QUOTA_FLUSH_COUNT) {
1244 set_sbi_flag(sbi, SBI_QUOTA_SKIP_FLUSH);
1245 set_sbi_flag(sbi, SBI_QUOTA_NEED_FLUSH);
1246 goto retry_flush_dents;
1247 }
1248 f2fs_unlock_all(sbi);
1249
1250 /* don't grab s_umount lock during mount/umount/remount/freeze/quotactl */
1251 if (!need_lock) {
1252 f2fs_do_quota_sync(sbi->sb, -1);
1253 } else if (down_read_trylock(&sbi->sb->s_umount)) {
1254 f2fs_do_quota_sync(sbi->sb, -1);
1255 up_read(&sbi->sb->s_umount);
1256 }
1257 cond_resched();
1258 goto retry_flush_quotas;
1259 }
1260
1261 retry_flush_dents:
1262 /* write all the dirty dentry pages */
1263 if (get_pages(sbi, F2FS_DIRTY_DENTS)) {
1264 f2fs_unlock_all(sbi);
1265 err = f2fs_sync_dirty_inodes(sbi, DIR_INODE, true);
1266 if (err)
1267 return err;
1268 cond_resched();
1269 goto retry_flush_quotas;
1270 }
1271
1272 /*
1273 * POR: we should ensure that there are no dirty node pages
1274 * until finishing nat/sit flush. inode->i_blocks can be updated.
1275 */
1276 f2fs_down_write(&sbi->node_change);
1277
1278 if (get_pages(sbi, F2FS_DIRTY_IMETA)) {
1279 f2fs_up_write(&sbi->node_change);
1280 f2fs_unlock_all(sbi);
1281 err = f2fs_sync_inode_meta(sbi);
1282 if (err)
1283 return err;
1284 cond_resched();
1285 goto retry_flush_quotas;
1286 }
1287
1288 retry_flush_nodes:
1289 f2fs_down_write(&sbi->node_write);
1290
1291 if (get_pages(sbi, F2FS_DIRTY_NODES)) {
1292 f2fs_up_write(&sbi->node_write);
1293 atomic_inc(&sbi->wb_sync_req[NODE]);
1294 err = f2fs_sync_node_pages(sbi, &wbc, false, FS_CP_NODE_IO);
1295 atomic_dec(&sbi->wb_sync_req[NODE]);
1296 if (err) {
1297 f2fs_up_write(&sbi->node_change);
1298 f2fs_unlock_all(sbi);
1299 return err;
1300 }
1301 cond_resched();
1302 goto retry_flush_nodes;
1303 }
1304
1305 /*
1306 * sbi->node_change is used only for AIO write_begin path which produces
1307 * dirty node blocks and some checkpoint values by block allocation.
1308 */
1309 __prepare_cp_block(sbi);
1310 f2fs_up_write(&sbi->node_change);
1311 return err;
1312 }
1313
unblock_operations(struct f2fs_sb_info * sbi)1314 static void unblock_operations(struct f2fs_sb_info *sbi)
1315 {
1316 f2fs_up_write(&sbi->node_write);
1317 f2fs_unlock_all(sbi);
1318 }
1319
f2fs_wait_on_all_pages(struct f2fs_sb_info * sbi,int type)1320 void f2fs_wait_on_all_pages(struct f2fs_sb_info *sbi, int type)
1321 {
1322 DEFINE_WAIT(wait);
1323
1324 for (;;) {
1325 if (!get_pages(sbi, type))
1326 break;
1327
1328 if (unlikely(f2fs_cp_error(sbi) &&
1329 !is_sbi_flag_set(sbi, SBI_IS_CLOSE)))
1330 break;
1331
1332 if (type == F2FS_DIRTY_META)
1333 f2fs_sync_meta_pages(sbi, META, LONG_MAX,
1334 FS_CP_META_IO);
1335 else if (type == F2FS_WB_CP_DATA)
1336 f2fs_submit_merged_write(sbi, DATA);
1337
1338 prepare_to_wait(&sbi->cp_wait, &wait, TASK_UNINTERRUPTIBLE);
1339 io_schedule_timeout(DEFAULT_IO_TIMEOUT);
1340 }
1341 finish_wait(&sbi->cp_wait, &wait);
1342 }
1343
update_ckpt_flags(struct f2fs_sb_info * sbi,struct cp_control * cpc)1344 static void update_ckpt_flags(struct f2fs_sb_info *sbi, struct cp_control *cpc)
1345 {
1346 unsigned long orphan_num = sbi->im[ORPHAN_INO].ino_num;
1347 struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi);
1348 unsigned long flags;
1349
1350 spin_lock_irqsave(&sbi->cp_lock, flags);
1351
1352 if ((cpc->reason & CP_UMOUNT) &&
1353 le32_to_cpu(ckpt->cp_pack_total_block_count) >
1354 sbi->blocks_per_seg - NM_I(sbi)->nat_bits_blocks)
1355 disable_nat_bits(sbi, false);
1356
1357 if (cpc->reason & CP_TRIMMED)
1358 __set_ckpt_flags(ckpt, CP_TRIMMED_FLAG);
1359 else
1360 __clear_ckpt_flags(ckpt, CP_TRIMMED_FLAG);
1361
1362 if (cpc->reason & CP_UMOUNT)
1363 __set_ckpt_flags(ckpt, CP_UMOUNT_FLAG);
1364 else
1365 __clear_ckpt_flags(ckpt, CP_UMOUNT_FLAG);
1366
1367 if (cpc->reason & CP_FASTBOOT)
1368 __set_ckpt_flags(ckpt, CP_FASTBOOT_FLAG);
1369 else
1370 __clear_ckpt_flags(ckpt, CP_FASTBOOT_FLAG);
1371
1372 if (orphan_num)
1373 __set_ckpt_flags(ckpt, CP_ORPHAN_PRESENT_FLAG);
1374 else
1375 __clear_ckpt_flags(ckpt, CP_ORPHAN_PRESENT_FLAG);
1376
1377 if (is_sbi_flag_set(sbi, SBI_NEED_FSCK))
1378 __set_ckpt_flags(ckpt, CP_FSCK_FLAG);
1379
1380 if (is_sbi_flag_set(sbi, SBI_IS_RESIZEFS))
1381 __set_ckpt_flags(ckpt, CP_RESIZEFS_FLAG);
1382 else
1383 __clear_ckpt_flags(ckpt, CP_RESIZEFS_FLAG);
1384
1385 if (is_sbi_flag_set(sbi, SBI_CP_DISABLED))
1386 __set_ckpt_flags(ckpt, CP_DISABLED_FLAG);
1387 else
1388 __clear_ckpt_flags(ckpt, CP_DISABLED_FLAG);
1389
1390 if (is_sbi_flag_set(sbi, SBI_CP_DISABLED_QUICK))
1391 __set_ckpt_flags(ckpt, CP_DISABLED_QUICK_FLAG);
1392 else
1393 __clear_ckpt_flags(ckpt, CP_DISABLED_QUICK_FLAG);
1394
1395 if (is_sbi_flag_set(sbi, SBI_QUOTA_SKIP_FLUSH))
1396 __set_ckpt_flags(ckpt, CP_QUOTA_NEED_FSCK_FLAG);
1397 else
1398 __clear_ckpt_flags(ckpt, CP_QUOTA_NEED_FSCK_FLAG);
1399
1400 if (is_sbi_flag_set(sbi, SBI_QUOTA_NEED_REPAIR))
1401 __set_ckpt_flags(ckpt, CP_QUOTA_NEED_FSCK_FLAG);
1402
1403 /* set this flag to activate crc|cp_ver for recovery */
1404 __set_ckpt_flags(ckpt, CP_CRC_RECOVERY_FLAG);
1405 __clear_ckpt_flags(ckpt, CP_NOCRC_RECOVERY_FLAG);
1406
1407 spin_unlock_irqrestore(&sbi->cp_lock, flags);
1408 }
1409
commit_checkpoint(struct f2fs_sb_info * sbi,void * src,block_t blk_addr)1410 static void commit_checkpoint(struct f2fs_sb_info *sbi,
1411 void *src, block_t blk_addr)
1412 {
1413 struct writeback_control wbc = {
1414 .for_reclaim = 0,
1415 };
1416
1417 /*
1418 * filemap_get_folios_tag and lock_page again will take
1419 * some extra time. Therefore, f2fs_update_meta_pages and
1420 * f2fs_sync_meta_pages are combined in this function.
1421 */
1422 struct page *page = f2fs_grab_meta_page(sbi, blk_addr);
1423 int err;
1424
1425 f2fs_wait_on_page_writeback(page, META, true, true);
1426
1427 memcpy(page_address(page), src, PAGE_SIZE);
1428
1429 set_page_dirty(page);
1430 if (unlikely(!clear_page_dirty_for_io(page)))
1431 f2fs_bug_on(sbi, 1);
1432
1433 /* writeout cp pack 2 page */
1434 err = __f2fs_write_meta_page(page, &wbc, FS_CP_META_IO);
1435 if (unlikely(err && f2fs_cp_error(sbi))) {
1436 f2fs_put_page(page, 1);
1437 return;
1438 }
1439
1440 f2fs_bug_on(sbi, err);
1441 f2fs_put_page(page, 0);
1442
1443 /* submit checkpoint (with barrier if NOBARRIER is not set) */
1444 f2fs_submit_merged_write(sbi, META_FLUSH);
1445 }
1446
get_sectors_written(struct block_device * bdev)1447 static inline u64 get_sectors_written(struct block_device *bdev)
1448 {
1449 return (u64)part_stat_read(bdev, sectors[STAT_WRITE]);
1450 }
1451
f2fs_get_sectors_written(struct f2fs_sb_info * sbi)1452 u64 f2fs_get_sectors_written(struct f2fs_sb_info *sbi)
1453 {
1454 if (f2fs_is_multi_device(sbi)) {
1455 u64 sectors = 0;
1456 int i;
1457
1458 for (i = 0; i < sbi->s_ndevs; i++)
1459 sectors += get_sectors_written(FDEV(i).bdev);
1460
1461 return sectors;
1462 }
1463
1464 return get_sectors_written(sbi->sb->s_bdev);
1465 }
1466
do_checkpoint(struct f2fs_sb_info * sbi,struct cp_control * cpc)1467 static int do_checkpoint(struct f2fs_sb_info *sbi, struct cp_control *cpc)
1468 {
1469 struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi);
1470 struct f2fs_nm_info *nm_i = NM_I(sbi);
1471 unsigned long orphan_num = sbi->im[ORPHAN_INO].ino_num, flags;
1472 block_t start_blk;
1473 unsigned int data_sum_blocks, orphan_blocks;
1474 __u32 crc32 = 0;
1475 int i;
1476 int cp_payload_blks = __cp_payload(sbi);
1477 struct curseg_info *seg_i = CURSEG_I(sbi, CURSEG_HOT_NODE);
1478 u64 kbytes_written;
1479 int err;
1480
1481 /* Flush all the NAT/SIT pages */
1482 f2fs_sync_meta_pages(sbi, META, LONG_MAX, FS_CP_META_IO);
1483
1484 /* start to update checkpoint, cp ver is already updated previously */
1485 ckpt->elapsed_time = cpu_to_le64(get_mtime(sbi, true));
1486 ckpt->free_segment_count = cpu_to_le32(free_segments(sbi));
1487 for (i = 0; i < NR_CURSEG_NODE_TYPE; i++) {
1488 struct curseg_info *curseg = CURSEG_I(sbi, i + CURSEG_HOT_NODE);
1489
1490 ckpt->cur_node_segno[i] = cpu_to_le32(curseg->segno);
1491 ckpt->cur_node_blkoff[i] = cpu_to_le16(curseg->next_blkoff);
1492 ckpt->alloc_type[i + CURSEG_HOT_NODE] = curseg->alloc_type;
1493 }
1494 for (i = 0; i < NR_CURSEG_DATA_TYPE; i++) {
1495 struct curseg_info *curseg = CURSEG_I(sbi, i + CURSEG_HOT_DATA);
1496
1497 ckpt->cur_data_segno[i] = cpu_to_le32(curseg->segno);
1498 ckpt->cur_data_blkoff[i] = cpu_to_le16(curseg->next_blkoff);
1499 ckpt->alloc_type[i + CURSEG_HOT_DATA] = curseg->alloc_type;
1500 }
1501
1502 /* 2 cp + n data seg summary + orphan inode blocks */
1503 data_sum_blocks = f2fs_npages_for_summary_flush(sbi, false);
1504 spin_lock_irqsave(&sbi->cp_lock, flags);
1505 if (data_sum_blocks < NR_CURSEG_DATA_TYPE)
1506 __set_ckpt_flags(ckpt, CP_COMPACT_SUM_FLAG);
1507 else
1508 __clear_ckpt_flags(ckpt, CP_COMPACT_SUM_FLAG);
1509 spin_unlock_irqrestore(&sbi->cp_lock, flags);
1510
1511 orphan_blocks = GET_ORPHAN_BLOCKS(orphan_num);
1512 ckpt->cp_pack_start_sum = cpu_to_le32(1 + cp_payload_blks +
1513 orphan_blocks);
1514
1515 if (__remain_node_summaries(cpc->reason))
1516 ckpt->cp_pack_total_block_count = cpu_to_le32(F2FS_CP_PACKS +
1517 cp_payload_blks + data_sum_blocks +
1518 orphan_blocks + NR_CURSEG_NODE_TYPE);
1519 else
1520 ckpt->cp_pack_total_block_count = cpu_to_le32(F2FS_CP_PACKS +
1521 cp_payload_blks + data_sum_blocks +
1522 orphan_blocks);
1523
1524 /* update ckpt flag for checkpoint */
1525 update_ckpt_flags(sbi, cpc);
1526
1527 /* update SIT/NAT bitmap */
1528 get_sit_bitmap(sbi, __bitmap_ptr(sbi, SIT_BITMAP));
1529 get_nat_bitmap(sbi, __bitmap_ptr(sbi, NAT_BITMAP));
1530
1531 crc32 = f2fs_checkpoint_chksum(sbi, ckpt);
1532 *((__le32 *)((unsigned char *)ckpt +
1533 le32_to_cpu(ckpt->checksum_offset)))
1534 = cpu_to_le32(crc32);
1535
1536 start_blk = __start_cp_next_addr(sbi);
1537
1538 /* write nat bits */
1539 if (enabled_nat_bits(sbi, cpc)) {
1540 __u64 cp_ver = cur_cp_version(ckpt);
1541 block_t blk;
1542
1543 cp_ver |= ((__u64)crc32 << 32);
1544 *(__le64 *)nm_i->nat_bits = cpu_to_le64(cp_ver);
1545
1546 blk = start_blk + BLKS_PER_SEG(sbi) - nm_i->nat_bits_blocks;
1547 for (i = 0; i < nm_i->nat_bits_blocks; i++)
1548 f2fs_update_meta_page(sbi, nm_i->nat_bits +
1549 F2FS_BLK_TO_BYTES(i), blk + i);
1550 }
1551
1552 /* write out checkpoint buffer at block 0 */
1553 f2fs_update_meta_page(sbi, ckpt, start_blk++);
1554
1555 for (i = 1; i < 1 + cp_payload_blks; i++)
1556 f2fs_update_meta_page(sbi, (char *)ckpt + i * F2FS_BLKSIZE,
1557 start_blk++);
1558
1559 if (orphan_num) {
1560 write_orphan_inodes(sbi, start_blk);
1561 start_blk += orphan_blocks;
1562 }
1563
1564 f2fs_write_data_summaries(sbi, start_blk);
1565 start_blk += data_sum_blocks;
1566
1567 /* Record write statistics in the hot node summary */
1568 kbytes_written = sbi->kbytes_written;
1569 kbytes_written += (f2fs_get_sectors_written(sbi) -
1570 sbi->sectors_written_start) >> 1;
1571 seg_i->journal->info.kbytes_written = cpu_to_le64(kbytes_written);
1572
1573 if (__remain_node_summaries(cpc->reason)) {
1574 f2fs_write_node_summaries(sbi, start_blk);
1575 start_blk += NR_CURSEG_NODE_TYPE;
1576 }
1577
1578 /* Here, we have one bio having CP pack except cp pack 2 page */
1579 f2fs_sync_meta_pages(sbi, META, LONG_MAX, FS_CP_META_IO);
1580 /* Wait for all dirty meta pages to be submitted for IO */
1581 f2fs_wait_on_all_pages(sbi, F2FS_DIRTY_META);
1582
1583 /* wait for previous submitted meta pages writeback */
1584 f2fs_wait_on_all_pages(sbi, F2FS_WB_CP_DATA);
1585
1586 /* flush all device cache */
1587 err = f2fs_flush_device_cache(sbi);
1588 if (err)
1589 return err;
1590
1591 /* barrier and flush checkpoint cp pack 2 page if it can */
1592 commit_checkpoint(sbi, ckpt, start_blk);
1593 f2fs_wait_on_all_pages(sbi, F2FS_WB_CP_DATA);
1594
1595 /*
1596 * invalidate intermediate page cache borrowed from meta inode which are
1597 * used for migration of encrypted, verity or compressed inode's blocks.
1598 */
1599 if (f2fs_sb_has_encrypt(sbi) || f2fs_sb_has_verity(sbi) ||
1600 f2fs_sb_has_compression(sbi))
1601 f2fs_bug_on(sbi,
1602 invalidate_inode_pages2_range(META_MAPPING(sbi),
1603 MAIN_BLKADDR(sbi), MAX_BLKADDR(sbi) - 1));
1604
1605 f2fs_release_ino_entry(sbi, false);
1606
1607 f2fs_reset_fsync_node_info(sbi);
1608
1609 clear_sbi_flag(sbi, SBI_IS_DIRTY);
1610 clear_sbi_flag(sbi, SBI_NEED_CP);
1611 clear_sbi_flag(sbi, SBI_QUOTA_SKIP_FLUSH);
1612
1613 spin_lock(&sbi->stat_lock);
1614 sbi->unusable_block_count = 0;
1615 spin_unlock(&sbi->stat_lock);
1616
1617 __set_cp_next_pack(sbi);
1618
1619 /*
1620 * redirty superblock if metadata like node page or inode cache is
1621 * updated during writing checkpoint.
1622 */
1623 if (get_pages(sbi, F2FS_DIRTY_NODES) ||
1624 get_pages(sbi, F2FS_DIRTY_IMETA))
1625 set_sbi_flag(sbi, SBI_IS_DIRTY);
1626
1627 f2fs_bug_on(sbi, get_pages(sbi, F2FS_DIRTY_DENTS));
1628
1629 return unlikely(f2fs_cp_error(sbi)) ? -EIO : 0;
1630 }
1631
f2fs_write_checkpoint(struct f2fs_sb_info * sbi,struct cp_control * cpc)1632 int f2fs_write_checkpoint(struct f2fs_sb_info *sbi, struct cp_control *cpc)
1633 {
1634 struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi);
1635 unsigned long long ckpt_ver;
1636 int err = 0;
1637
1638 if (f2fs_readonly(sbi->sb) || f2fs_hw_is_readonly(sbi))
1639 return -EROFS;
1640
1641 if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED))) {
1642 if (cpc->reason != CP_PAUSE)
1643 return 0;
1644 f2fs_warn(sbi, "Start checkpoint disabled!");
1645 }
1646 if (cpc->reason != CP_RESIZE)
1647 f2fs_down_write(&sbi->cp_global_sem);
1648
1649 if (!is_sbi_flag_set(sbi, SBI_IS_DIRTY) &&
1650 ((cpc->reason & CP_FASTBOOT) || (cpc->reason & CP_SYNC) ||
1651 ((cpc->reason & CP_DISCARD) && !sbi->discard_blks)))
1652 goto out;
1653 if (unlikely(f2fs_cp_error(sbi))) {
1654 err = -EIO;
1655 goto out;
1656 }
1657
1658 trace_f2fs_write_checkpoint(sbi->sb, cpc->reason, "start block_ops");
1659
1660 err = block_operations(sbi);
1661 if (err)
1662 goto out;
1663
1664 trace_f2fs_write_checkpoint(sbi->sb, cpc->reason, "finish block_ops");
1665
1666 f2fs_flush_merged_writes(sbi);
1667
1668 /* this is the case of multiple fstrims without any changes */
1669 if (cpc->reason & CP_DISCARD) {
1670 if (!f2fs_exist_trim_candidates(sbi, cpc)) {
1671 unblock_operations(sbi);
1672 goto out;
1673 }
1674
1675 if (NM_I(sbi)->nat_cnt[DIRTY_NAT] == 0 &&
1676 SIT_I(sbi)->dirty_sentries == 0 &&
1677 prefree_segments(sbi) == 0) {
1678 f2fs_flush_sit_entries(sbi, cpc);
1679 f2fs_clear_prefree_segments(sbi, cpc);
1680 unblock_operations(sbi);
1681 goto out;
1682 }
1683 }
1684
1685 /*
1686 * update checkpoint pack index
1687 * Increase the version number so that
1688 * SIT entries and seg summaries are written at correct place
1689 */
1690 ckpt_ver = cur_cp_version(ckpt);
1691 ckpt->checkpoint_ver = cpu_to_le64(++ckpt_ver);
1692
1693 /* write cached NAT/SIT entries to NAT/SIT area */
1694 err = f2fs_flush_nat_entries(sbi, cpc);
1695 if (err) {
1696 f2fs_err(sbi, "f2fs_flush_nat_entries failed err:%d, stop checkpoint", err);
1697 f2fs_bug_on(sbi, !f2fs_cp_error(sbi));
1698 goto stop;
1699 }
1700
1701 f2fs_flush_sit_entries(sbi, cpc);
1702
1703 /* save inmem log status */
1704 f2fs_save_inmem_curseg(sbi);
1705
1706 err = do_checkpoint(sbi, cpc);
1707 if (err) {
1708 f2fs_err(sbi, "do_checkpoint failed err:%d, stop checkpoint", err);
1709 f2fs_bug_on(sbi, !f2fs_cp_error(sbi));
1710 f2fs_release_discard_addrs(sbi);
1711 } else {
1712 f2fs_clear_prefree_segments(sbi, cpc);
1713 }
1714
1715 f2fs_restore_inmem_curseg(sbi);
1716 f2fs_reinit_atgc_curseg(sbi);
1717 stat_inc_cp_count(sbi);
1718 stop:
1719 unblock_operations(sbi);
1720
1721 if (cpc->reason & CP_RECOVERY)
1722 f2fs_notice(sbi, "checkpoint: version = %llx", ckpt_ver);
1723
1724 /* update CP_TIME to trigger checkpoint periodically */
1725 f2fs_update_time(sbi, CP_TIME);
1726 trace_f2fs_write_checkpoint(sbi->sb, cpc->reason, "finish checkpoint");
1727 out:
1728 if (cpc->reason != CP_RESIZE)
1729 f2fs_up_write(&sbi->cp_global_sem);
1730 return err;
1731 }
1732
f2fs_init_ino_entry_info(struct f2fs_sb_info * sbi)1733 void f2fs_init_ino_entry_info(struct f2fs_sb_info *sbi)
1734 {
1735 int i;
1736
1737 for (i = 0; i < MAX_INO_ENTRY; i++) {
1738 struct inode_management *im = &sbi->im[i];
1739
1740 INIT_RADIX_TREE(&im->ino_root, GFP_ATOMIC);
1741 spin_lock_init(&im->ino_lock);
1742 INIT_LIST_HEAD(&im->ino_list);
1743 im->ino_num = 0;
1744 }
1745
1746 sbi->max_orphans = (BLKS_PER_SEG(sbi) - F2FS_CP_PACKS -
1747 NR_CURSEG_PERSIST_TYPE - __cp_payload(sbi)) *
1748 F2FS_ORPHANS_PER_BLOCK;
1749 }
1750
f2fs_create_checkpoint_caches(void)1751 int __init f2fs_create_checkpoint_caches(void)
1752 {
1753 ino_entry_slab = f2fs_kmem_cache_create("f2fs_ino_entry",
1754 sizeof(struct ino_entry));
1755 if (!ino_entry_slab)
1756 return -ENOMEM;
1757 f2fs_inode_entry_slab = f2fs_kmem_cache_create("f2fs_inode_entry",
1758 sizeof(struct inode_entry));
1759 if (!f2fs_inode_entry_slab) {
1760 kmem_cache_destroy(ino_entry_slab);
1761 return -ENOMEM;
1762 }
1763 return 0;
1764 }
1765
f2fs_destroy_checkpoint_caches(void)1766 void f2fs_destroy_checkpoint_caches(void)
1767 {
1768 kmem_cache_destroy(ino_entry_slab);
1769 kmem_cache_destroy(f2fs_inode_entry_slab);
1770 }
1771
__write_checkpoint_sync(struct f2fs_sb_info * sbi)1772 static int __write_checkpoint_sync(struct f2fs_sb_info *sbi)
1773 {
1774 struct cp_control cpc = { .reason = CP_SYNC, };
1775 int err;
1776
1777 f2fs_down_write(&sbi->gc_lock);
1778 err = f2fs_write_checkpoint(sbi, &cpc);
1779 f2fs_up_write(&sbi->gc_lock);
1780
1781 return err;
1782 }
1783
__checkpoint_and_complete_reqs(struct f2fs_sb_info * sbi)1784 static void __checkpoint_and_complete_reqs(struct f2fs_sb_info *sbi)
1785 {
1786 struct ckpt_req_control *cprc = &sbi->cprc_info;
1787 struct ckpt_req *req, *next;
1788 struct llist_node *dispatch_list;
1789 u64 sum_diff = 0, diff, count = 0;
1790 int ret;
1791
1792 dispatch_list = llist_del_all(&cprc->issue_list);
1793 if (!dispatch_list)
1794 return;
1795 dispatch_list = llist_reverse_order(dispatch_list);
1796
1797 ret = __write_checkpoint_sync(sbi);
1798 atomic_inc(&cprc->issued_ckpt);
1799
1800 llist_for_each_entry_safe(req, next, dispatch_list, llnode) {
1801 diff = (u64)ktime_ms_delta(ktime_get(), req->queue_time);
1802 req->ret = ret;
1803 complete(&req->wait);
1804
1805 sum_diff += diff;
1806 count++;
1807 }
1808 atomic_sub(count, &cprc->queued_ckpt);
1809 atomic_add(count, &cprc->total_ckpt);
1810
1811 spin_lock(&cprc->stat_lock);
1812 cprc->cur_time = (unsigned int)div64_u64(sum_diff, count);
1813 if (cprc->peak_time < cprc->cur_time)
1814 cprc->peak_time = cprc->cur_time;
1815 spin_unlock(&cprc->stat_lock);
1816 }
1817
issue_checkpoint_thread(void * data)1818 static int issue_checkpoint_thread(void *data)
1819 {
1820 struct f2fs_sb_info *sbi = data;
1821 struct ckpt_req_control *cprc = &sbi->cprc_info;
1822 wait_queue_head_t *q = &cprc->ckpt_wait_queue;
1823 repeat:
1824 if (kthread_should_stop())
1825 return 0;
1826
1827 if (!llist_empty(&cprc->issue_list))
1828 __checkpoint_and_complete_reqs(sbi);
1829
1830 wait_event_interruptible(*q,
1831 kthread_should_stop() || !llist_empty(&cprc->issue_list));
1832 goto repeat;
1833 }
1834
flush_remained_ckpt_reqs(struct f2fs_sb_info * sbi,struct ckpt_req * wait_req)1835 static void flush_remained_ckpt_reqs(struct f2fs_sb_info *sbi,
1836 struct ckpt_req *wait_req)
1837 {
1838 struct ckpt_req_control *cprc = &sbi->cprc_info;
1839
1840 if (!llist_empty(&cprc->issue_list)) {
1841 __checkpoint_and_complete_reqs(sbi);
1842 } else {
1843 /* already dispatched by issue_checkpoint_thread */
1844 if (wait_req)
1845 wait_for_completion(&wait_req->wait);
1846 }
1847 }
1848
init_ckpt_req(struct ckpt_req * req)1849 static void init_ckpt_req(struct ckpt_req *req)
1850 {
1851 memset(req, 0, sizeof(struct ckpt_req));
1852
1853 init_completion(&req->wait);
1854 req->queue_time = ktime_get();
1855 }
1856
f2fs_issue_checkpoint(struct f2fs_sb_info * sbi)1857 int f2fs_issue_checkpoint(struct f2fs_sb_info *sbi)
1858 {
1859 struct ckpt_req_control *cprc = &sbi->cprc_info;
1860 struct ckpt_req req;
1861 struct cp_control cpc;
1862
1863 cpc.reason = __get_cp_reason(sbi);
1864 if (!test_opt(sbi, MERGE_CHECKPOINT) || cpc.reason != CP_SYNC ||
1865 sbi->umount_lock_holder == current) {
1866 int ret;
1867
1868 f2fs_down_write(&sbi->gc_lock);
1869 ret = f2fs_write_checkpoint(sbi, &cpc);
1870 f2fs_up_write(&sbi->gc_lock);
1871
1872 return ret;
1873 }
1874
1875 if (!cprc->f2fs_issue_ckpt)
1876 return __write_checkpoint_sync(sbi);
1877
1878 init_ckpt_req(&req);
1879
1880 llist_add(&req.llnode, &cprc->issue_list);
1881 atomic_inc(&cprc->queued_ckpt);
1882
1883 /*
1884 * update issue_list before we wake up issue_checkpoint thread,
1885 * this smp_mb() pairs with another barrier in ___wait_event(),
1886 * see more details in comments of waitqueue_active().
1887 */
1888 smp_mb();
1889
1890 if (waitqueue_active(&cprc->ckpt_wait_queue))
1891 wake_up(&cprc->ckpt_wait_queue);
1892
1893 if (cprc->f2fs_issue_ckpt) {
1894 bool prio_changed = false;
1895 int saved_prio;
1896
1897 trace_android_vh_f2fs_improve_priority(cprc->f2fs_issue_ckpt, &saved_prio,
1898 &prio_changed);
1899 wait_for_completion(&req.wait);
1900 if (prio_changed)
1901 trace_android_vh_f2fs_restore_priority(cprc->f2fs_issue_ckpt, saved_prio);
1902 } else {
1903 flush_remained_ckpt_reqs(sbi, &req);
1904 }
1905
1906 return req.ret;
1907 }
1908
f2fs_start_ckpt_thread(struct f2fs_sb_info * sbi)1909 int f2fs_start_ckpt_thread(struct f2fs_sb_info *sbi)
1910 {
1911 dev_t dev = sbi->sb->s_bdev->bd_dev;
1912 struct ckpt_req_control *cprc = &sbi->cprc_info;
1913
1914 if (cprc->f2fs_issue_ckpt)
1915 return 0;
1916
1917 cprc->f2fs_issue_ckpt = kthread_run(issue_checkpoint_thread, sbi,
1918 "f2fs_ckpt-%u:%u", MAJOR(dev), MINOR(dev));
1919 if (IS_ERR(cprc->f2fs_issue_ckpt)) {
1920 int err = PTR_ERR(cprc->f2fs_issue_ckpt);
1921
1922 cprc->f2fs_issue_ckpt = NULL;
1923 return err;
1924 }
1925
1926 set_task_ioprio(cprc->f2fs_issue_ckpt, cprc->ckpt_thread_ioprio);
1927
1928 return 0;
1929 }
1930
f2fs_stop_ckpt_thread(struct f2fs_sb_info * sbi)1931 void f2fs_stop_ckpt_thread(struct f2fs_sb_info *sbi)
1932 {
1933 struct ckpt_req_control *cprc = &sbi->cprc_info;
1934 struct task_struct *ckpt_task;
1935
1936 if (!cprc->f2fs_issue_ckpt)
1937 return;
1938
1939 ckpt_task = cprc->f2fs_issue_ckpt;
1940 cprc->f2fs_issue_ckpt = NULL;
1941 kthread_stop(ckpt_task);
1942
1943 f2fs_flush_ckpt_thread(sbi);
1944 }
1945
f2fs_flush_ckpt_thread(struct f2fs_sb_info * sbi)1946 void f2fs_flush_ckpt_thread(struct f2fs_sb_info *sbi)
1947 {
1948 struct ckpt_req_control *cprc = &sbi->cprc_info;
1949
1950 flush_remained_ckpt_reqs(sbi, NULL);
1951
1952 /* Let's wait for the previous dispatched checkpoint. */
1953 while (atomic_read(&cprc->queued_ckpt))
1954 io_schedule_timeout(DEFAULT_IO_TIMEOUT);
1955 }
1956
f2fs_init_ckpt_req_control(struct f2fs_sb_info * sbi)1957 void f2fs_init_ckpt_req_control(struct f2fs_sb_info *sbi)
1958 {
1959 struct ckpt_req_control *cprc = &sbi->cprc_info;
1960
1961 atomic_set(&cprc->issued_ckpt, 0);
1962 atomic_set(&cprc->total_ckpt, 0);
1963 atomic_set(&cprc->queued_ckpt, 0);
1964 cprc->ckpt_thread_ioprio = DEFAULT_CHECKPOINT_IOPRIO;
1965 init_waitqueue_head(&cprc->ckpt_wait_queue);
1966 init_llist_head(&cprc->issue_list);
1967 spin_lock_init(&cprc->stat_lock);
1968 }
1969