1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * fs/f2fs/gc.c
4 *
5 * Copyright (c) 2012 Samsung Electronics Co., Ltd.
6 * http://www.samsung.com/
7 */
8 #include <linux/fs.h>
9 #include <linux/module.h>
10 #include <linux/backing-dev.h>
11 #include <linux/init.h>
12 #include <linux/f2fs_fs.h>
13 #include <linux/kthread.h>
14 #include <linux/delay.h>
15 #include <linux/freezer.h>
16 #include <linux/sched/signal.h>
17
18 #include "f2fs.h"
19 #include "node.h"
20 #include "segment.h"
21 #include "gc.h"
22 #include <trace/events/f2fs.h>
23
gc_thread_func(void * data)24 static int gc_thread_func(void *data)
25 {
26 struct f2fs_sb_info *sbi = data;
27 struct f2fs_gc_kthread *gc_th = sbi->gc_thread;
28 wait_queue_head_t *wq = &sbi->gc_thread->gc_wait_queue_head;
29 unsigned int wait_ms;
30
31 wait_ms = gc_th->min_sleep_time;
32
33 set_freezable();
34 do {
35 bool sync_mode;
36
37 wait_event_interruptible_timeout(*wq,
38 kthread_should_stop() || freezing(current) ||
39 gc_th->gc_wake,
40 msecs_to_jiffies(wait_ms));
41
42 /* give it a try one time */
43 if (gc_th->gc_wake)
44 gc_th->gc_wake = 0;
45
46 if (try_to_freeze()) {
47 stat_other_skip_bggc_count(sbi);
48 continue;
49 }
50 if (kthread_should_stop())
51 break;
52
53 if (sbi->sb->s_writers.frozen >= SB_FREEZE_WRITE) {
54 increase_sleep_time(gc_th, &wait_ms);
55 stat_other_skip_bggc_count(sbi);
56 continue;
57 }
58
59 if (time_to_inject(sbi, FAULT_CHECKPOINT)) {
60 f2fs_show_injection_info(sbi, FAULT_CHECKPOINT);
61 f2fs_stop_checkpoint(sbi, false);
62 }
63
64 if (!sb_start_write_trylock(sbi->sb)) {
65 stat_other_skip_bggc_count(sbi);
66 continue;
67 }
68
69 /*
70 * [GC triggering condition]
71 * 0. GC is not conducted currently.
72 * 1. There are enough dirty segments.
73 * 2. IO subsystem is idle by checking the # of writeback pages.
74 * 3. IO subsystem is idle by checking the # of requests in
75 * bdev's request list.
76 *
77 * Note) We have to avoid triggering GCs frequently.
78 * Because it is possible that some segments can be
79 * invalidated soon after by user update or deletion.
80 * So, I'd like to wait some time to collect dirty segments.
81 */
82 if (sbi->gc_mode == GC_URGENT) {
83 wait_ms = gc_th->urgent_sleep_time;
84 down_write(&sbi->gc_lock);
85 goto do_gc;
86 }
87
88 if (!down_write_trylock(&sbi->gc_lock)) {
89 stat_other_skip_bggc_count(sbi);
90 goto next;
91 }
92
93 if (!is_idle(sbi, GC_TIME)) {
94 increase_sleep_time(gc_th, &wait_ms);
95 up_write(&sbi->gc_lock);
96 stat_io_skip_bggc_count(sbi);
97 goto next;
98 }
99
100 if (has_enough_invalid_blocks(sbi))
101 decrease_sleep_time(gc_th, &wait_ms);
102 else
103 increase_sleep_time(gc_th, &wait_ms);
104 do_gc:
105 stat_inc_bggc_count(sbi->stat_info);
106
107 sync_mode = F2FS_OPTION(sbi).bggc_mode == BGGC_MODE_SYNC;
108
109 /* if return value is not zero, no victim was selected */
110 if (f2fs_gc(sbi, sync_mode, true, NULL_SEGNO))
111 wait_ms = gc_th->no_gc_sleep_time;
112
113 trace_f2fs_background_gc(sbi->sb, wait_ms,
114 prefree_segments(sbi), free_segments(sbi));
115
116 /* balancing f2fs's metadata periodically */
117 f2fs_balance_fs_bg(sbi, true);
118 next:
119 sb_end_write(sbi->sb);
120
121 } while (!kthread_should_stop());
122 return 0;
123 }
124
f2fs_start_gc_thread(struct f2fs_sb_info * sbi)125 int f2fs_start_gc_thread(struct f2fs_sb_info *sbi)
126 {
127 struct f2fs_gc_kthread *gc_th;
128 dev_t dev = sbi->sb->s_bdev->bd_dev;
129 int err = 0;
130
131 gc_th = f2fs_kmalloc(sbi, sizeof(struct f2fs_gc_kthread), GFP_KERNEL);
132 if (!gc_th) {
133 err = -ENOMEM;
134 goto out;
135 }
136
137 gc_th->urgent_sleep_time = DEF_GC_THREAD_URGENT_SLEEP_TIME;
138 gc_th->min_sleep_time = DEF_GC_THREAD_MIN_SLEEP_TIME;
139 gc_th->max_sleep_time = DEF_GC_THREAD_MAX_SLEEP_TIME;
140 gc_th->no_gc_sleep_time = DEF_GC_THREAD_NOGC_SLEEP_TIME;
141
142 gc_th->gc_wake= 0;
143
144 sbi->gc_thread = gc_th;
145 init_waitqueue_head(&sbi->gc_thread->gc_wait_queue_head);
146 sbi->gc_thread->f2fs_gc_task = kthread_run(gc_thread_func, sbi,
147 "f2fs_gc-%u:%u", MAJOR(dev), MINOR(dev));
148 if (IS_ERR(gc_th->f2fs_gc_task)) {
149 err = PTR_ERR(gc_th->f2fs_gc_task);
150 kvfree(gc_th);
151 sbi->gc_thread = NULL;
152 }
153 out:
154 return err;
155 }
156
f2fs_stop_gc_thread(struct f2fs_sb_info * sbi)157 void f2fs_stop_gc_thread(struct f2fs_sb_info *sbi)
158 {
159 struct f2fs_gc_kthread *gc_th = sbi->gc_thread;
160 if (!gc_th)
161 return;
162 kthread_stop(gc_th->f2fs_gc_task);
163 kvfree(gc_th);
164 sbi->gc_thread = NULL;
165 }
166
select_gc_type(struct f2fs_sb_info * sbi,int gc_type)167 static int select_gc_type(struct f2fs_sb_info *sbi, int gc_type)
168 {
169 int gc_mode = (gc_type == BG_GC) ? GC_CB : GC_GREEDY;
170
171 switch (sbi->gc_mode) {
172 case GC_IDLE_CB:
173 gc_mode = GC_CB;
174 break;
175 case GC_IDLE_GREEDY:
176 case GC_URGENT:
177 gc_mode = GC_GREEDY;
178 break;
179 }
180 return gc_mode;
181 }
182
select_policy(struct f2fs_sb_info * sbi,int gc_type,int type,struct victim_sel_policy * p)183 static void select_policy(struct f2fs_sb_info *sbi, int gc_type,
184 int type, struct victim_sel_policy *p)
185 {
186 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
187
188 if (p->alloc_mode == SSR) {
189 p->gc_mode = GC_GREEDY;
190 p->dirty_segmap = dirty_i->dirty_segmap[type];
191 p->max_search = dirty_i->nr_dirty[type];
192 p->ofs_unit = 1;
193 } else {
194 p->gc_mode = select_gc_type(sbi, gc_type);
195 p->dirty_segmap = dirty_i->dirty_segmap[DIRTY];
196 p->max_search = dirty_i->nr_dirty[DIRTY];
197 p->ofs_unit = sbi->segs_per_sec;
198 }
199
200 /*
201 * adjust candidates range, should select all dirty segments for
202 * foreground GC and urgent GC cases.
203 */
204 if (gc_type != FG_GC &&
205 (sbi->gc_mode != GC_URGENT) &&
206 p->max_search > sbi->max_victim_search)
207 p->max_search = sbi->max_victim_search;
208
209 /* let's select beginning hot/small space first in no_heap mode*/
210 if (test_opt(sbi, NOHEAP) &&
211 (type == CURSEG_HOT_DATA || IS_NODESEG(type)))
212 p->offset = 0;
213 else
214 p->offset = SIT_I(sbi)->last_victim[p->gc_mode];
215 }
216
get_max_cost(struct f2fs_sb_info * sbi,struct victim_sel_policy * p)217 static unsigned int get_max_cost(struct f2fs_sb_info *sbi,
218 struct victim_sel_policy *p)
219 {
220 /* SSR allocates in a segment unit */
221 if (p->alloc_mode == SSR)
222 return sbi->blocks_per_seg;
223 if (p->gc_mode == GC_GREEDY)
224 return 2 * sbi->blocks_per_seg * p->ofs_unit;
225 else if (p->gc_mode == GC_CB)
226 return UINT_MAX;
227 else /* No other gc_mode */
228 return 0;
229 }
230
check_bg_victims(struct f2fs_sb_info * sbi)231 static unsigned int check_bg_victims(struct f2fs_sb_info *sbi)
232 {
233 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
234 unsigned int secno;
235
236 /*
237 * If the gc_type is FG_GC, we can select victim segments
238 * selected by background GC before.
239 * Those segments guarantee they have small valid blocks.
240 */
241 for_each_set_bit(secno, dirty_i->victim_secmap, MAIN_SECS(sbi)) {
242 if (sec_usage_check(sbi, secno))
243 continue;
244 clear_bit(secno, dirty_i->victim_secmap);
245 return GET_SEG_FROM_SEC(sbi, secno);
246 }
247 return NULL_SEGNO;
248 }
249
get_cb_cost(struct f2fs_sb_info * sbi,unsigned int segno)250 static unsigned int get_cb_cost(struct f2fs_sb_info *sbi, unsigned int segno)
251 {
252 struct sit_info *sit_i = SIT_I(sbi);
253 unsigned int secno = GET_SEC_FROM_SEG(sbi, segno);
254 unsigned int start = GET_SEG_FROM_SEC(sbi, secno);
255 unsigned long long mtime = 0;
256 unsigned int vblocks;
257 unsigned char age = 0;
258 unsigned char u;
259 unsigned int i;
260
261 for (i = 0; i < sbi->segs_per_sec; i++)
262 mtime += get_seg_entry(sbi, start + i)->mtime;
263 vblocks = get_valid_blocks(sbi, segno, true);
264
265 mtime = div_u64(mtime, sbi->segs_per_sec);
266 vblocks = div_u64(vblocks, sbi->segs_per_sec);
267
268 u = (vblocks * 100) >> sbi->log_blocks_per_seg;
269
270 /* Handle if the system time has changed by the user */
271 if (mtime < sit_i->min_mtime)
272 sit_i->min_mtime = mtime;
273 if (mtime > sit_i->max_mtime)
274 sit_i->max_mtime = mtime;
275 if (sit_i->max_mtime != sit_i->min_mtime)
276 age = 100 - div64_u64(100 * (mtime - sit_i->min_mtime),
277 sit_i->max_mtime - sit_i->min_mtime);
278
279 return UINT_MAX - ((100 * (100 - u) * age) / (100 + u));
280 }
281
get_gc_cost(struct f2fs_sb_info * sbi,unsigned int segno,struct victim_sel_policy * p)282 static inline unsigned int get_gc_cost(struct f2fs_sb_info *sbi,
283 unsigned int segno, struct victim_sel_policy *p)
284 {
285 if (p->alloc_mode == SSR)
286 return get_seg_entry(sbi, segno)->ckpt_valid_blocks;
287
288 /* alloc_mode == LFS */
289 if (p->gc_mode == GC_GREEDY)
290 return get_valid_blocks(sbi, segno, true);
291 else
292 return get_cb_cost(sbi, segno);
293 }
294
count_bits(const unsigned long * addr,unsigned int offset,unsigned int len)295 static unsigned int count_bits(const unsigned long *addr,
296 unsigned int offset, unsigned int len)
297 {
298 unsigned int end = offset + len, sum = 0;
299
300 while (offset < end) {
301 if (test_bit(offset++, addr))
302 ++sum;
303 }
304 return sum;
305 }
306
307 /*
308 * This function is called from two paths.
309 * One is garbage collection and the other is SSR segment selection.
310 * When it is called during GC, it just gets a victim segment
311 * and it does not remove it from dirty seglist.
312 * When it is called from SSR segment selection, it finds a segment
313 * which has minimum valid blocks and removes it from dirty seglist.
314 */
get_victim_by_default(struct f2fs_sb_info * sbi,unsigned int * result,int gc_type,int type,char alloc_mode)315 static int get_victim_by_default(struct f2fs_sb_info *sbi,
316 unsigned int *result, int gc_type, int type, char alloc_mode)
317 {
318 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
319 struct sit_info *sm = SIT_I(sbi);
320 struct victim_sel_policy p;
321 unsigned int secno, last_victim;
322 unsigned int last_segment;
323 unsigned int nsearched = 0;
324
325 mutex_lock(&dirty_i->seglist_lock);
326 last_segment = MAIN_SECS(sbi) * sbi->segs_per_sec;
327
328 p.alloc_mode = alloc_mode;
329 select_policy(sbi, gc_type, type, &p);
330
331 p.min_segno = NULL_SEGNO;
332 p.min_cost = get_max_cost(sbi, &p);
333
334 if (*result != NULL_SEGNO) {
335 if (get_valid_blocks(sbi, *result, false) &&
336 !sec_usage_check(sbi, GET_SEC_FROM_SEG(sbi, *result)))
337 p.min_segno = *result;
338 goto out;
339 }
340
341 if (p.max_search == 0)
342 goto out;
343
344 if (__is_large_section(sbi) && p.alloc_mode == LFS) {
345 if (sbi->next_victim_seg[BG_GC] != NULL_SEGNO) {
346 p.min_segno = sbi->next_victim_seg[BG_GC];
347 *result = p.min_segno;
348 sbi->next_victim_seg[BG_GC] = NULL_SEGNO;
349 goto got_result;
350 }
351 if (gc_type == FG_GC &&
352 sbi->next_victim_seg[FG_GC] != NULL_SEGNO) {
353 p.min_segno = sbi->next_victim_seg[FG_GC];
354 *result = p.min_segno;
355 sbi->next_victim_seg[FG_GC] = NULL_SEGNO;
356 goto got_result;
357 }
358 }
359
360 last_victim = sm->last_victim[p.gc_mode];
361 if (p.alloc_mode == LFS && gc_type == FG_GC) {
362 p.min_segno = check_bg_victims(sbi);
363 if (p.min_segno != NULL_SEGNO)
364 goto got_it;
365 }
366
367 while (1) {
368 unsigned long cost;
369 unsigned int segno;
370
371 segno = find_next_bit(p.dirty_segmap, last_segment, p.offset);
372 if (segno >= last_segment) {
373 if (sm->last_victim[p.gc_mode]) {
374 last_segment =
375 sm->last_victim[p.gc_mode];
376 sm->last_victim[p.gc_mode] = 0;
377 p.offset = 0;
378 continue;
379 }
380 break;
381 }
382
383 p.offset = segno + p.ofs_unit;
384 if (p.ofs_unit > 1) {
385 p.offset -= segno % p.ofs_unit;
386 nsearched += count_bits(p.dirty_segmap,
387 p.offset - p.ofs_unit,
388 p.ofs_unit);
389 } else {
390 nsearched++;
391 }
392
393 #ifdef CONFIG_F2FS_CHECK_FS
394 /*
395 * skip selecting the invalid segno (that is failed due to block
396 * validity check failure during GC) to avoid endless GC loop in
397 * such cases.
398 */
399 if (test_bit(segno, sm->invalid_segmap))
400 goto next;
401 #endif
402
403 secno = GET_SEC_FROM_SEG(sbi, segno);
404
405 if (sec_usage_check(sbi, secno))
406 goto next;
407 /* Don't touch checkpointed data */
408 if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED) &&
409 get_ckpt_valid_blocks(sbi, segno) &&
410 p.alloc_mode != SSR))
411 goto next;
412 if (gc_type == BG_GC && test_bit(secno, dirty_i->victim_secmap))
413 goto next;
414
415 cost = get_gc_cost(sbi, segno, &p);
416
417 if (p.min_cost > cost) {
418 p.min_segno = segno;
419 p.min_cost = cost;
420 }
421 next:
422 if (nsearched >= p.max_search) {
423 if (!sm->last_victim[p.gc_mode] && segno <= last_victim)
424 sm->last_victim[p.gc_mode] = last_victim + 1;
425 else
426 sm->last_victim[p.gc_mode] = segno + 1;
427 sm->last_victim[p.gc_mode] %=
428 (MAIN_SECS(sbi) * sbi->segs_per_sec);
429 break;
430 }
431 }
432 if (p.min_segno != NULL_SEGNO) {
433 got_it:
434 *result = (p.min_segno / p.ofs_unit) * p.ofs_unit;
435 got_result:
436 if (p.alloc_mode == LFS) {
437 secno = GET_SEC_FROM_SEG(sbi, p.min_segno);
438 if (gc_type == FG_GC)
439 sbi->cur_victim_sec = secno;
440 else
441 set_bit(secno, dirty_i->victim_secmap);
442 }
443
444 }
445 out:
446 if (p.min_segno != NULL_SEGNO)
447 trace_f2fs_get_victim(sbi->sb, type, gc_type, &p,
448 sbi->cur_victim_sec,
449 prefree_segments(sbi), free_segments(sbi));
450 mutex_unlock(&dirty_i->seglist_lock);
451
452 return (p.min_segno == NULL_SEGNO) ? 0 : 1;
453 }
454
455 static const struct victim_selection default_v_ops = {
456 .get_victim = get_victim_by_default,
457 };
458
find_gc_inode(struct gc_inode_list * gc_list,nid_t ino)459 static struct inode *find_gc_inode(struct gc_inode_list *gc_list, nid_t ino)
460 {
461 struct inode_entry *ie;
462
463 ie = radix_tree_lookup(&gc_list->iroot, ino);
464 if (ie)
465 return ie->inode;
466 return NULL;
467 }
468
add_gc_inode(struct gc_inode_list * gc_list,struct inode * inode)469 static void add_gc_inode(struct gc_inode_list *gc_list, struct inode *inode)
470 {
471 struct inode_entry *new_ie;
472
473 if (inode == find_gc_inode(gc_list, inode->i_ino)) {
474 iput(inode);
475 return;
476 }
477 new_ie = f2fs_kmem_cache_alloc(f2fs_inode_entry_slab, GFP_NOFS);
478 new_ie->inode = inode;
479
480 f2fs_radix_tree_insert(&gc_list->iroot, inode->i_ino, new_ie);
481 list_add_tail(&new_ie->list, &gc_list->ilist);
482 }
483
put_gc_inode(struct gc_inode_list * gc_list)484 static void put_gc_inode(struct gc_inode_list *gc_list)
485 {
486 struct inode_entry *ie, *next_ie;
487 list_for_each_entry_safe(ie, next_ie, &gc_list->ilist, list) {
488 radix_tree_delete(&gc_list->iroot, ie->inode->i_ino);
489 iput(ie->inode);
490 list_del(&ie->list);
491 kmem_cache_free(f2fs_inode_entry_slab, ie);
492 }
493 }
494
check_valid_map(struct f2fs_sb_info * sbi,unsigned int segno,int offset)495 static int check_valid_map(struct f2fs_sb_info *sbi,
496 unsigned int segno, int offset)
497 {
498 struct sit_info *sit_i = SIT_I(sbi);
499 struct seg_entry *sentry;
500 int ret;
501
502 down_read(&sit_i->sentry_lock);
503 sentry = get_seg_entry(sbi, segno);
504 ret = f2fs_test_bit(offset, sentry->cur_valid_map);
505 up_read(&sit_i->sentry_lock);
506 return ret;
507 }
508
509 /*
510 * This function compares node address got in summary with that in NAT.
511 * On validity, copy that node with cold status, otherwise (invalid node)
512 * ignore that.
513 */
gc_node_segment(struct f2fs_sb_info * sbi,struct f2fs_summary * sum,unsigned int segno,int gc_type)514 static int gc_node_segment(struct f2fs_sb_info *sbi,
515 struct f2fs_summary *sum, unsigned int segno, int gc_type)
516 {
517 struct f2fs_summary *entry;
518 block_t start_addr;
519 int off;
520 int phase = 0;
521 bool fggc = (gc_type == FG_GC);
522 int submitted = 0;
523
524 start_addr = START_BLOCK(sbi, segno);
525
526 next_step:
527 entry = sum;
528
529 if (fggc && phase == 2)
530 atomic_inc(&sbi->wb_sync_req[NODE]);
531
532 for (off = 0; off < sbi->blocks_per_seg; off++, entry++) {
533 nid_t nid = le32_to_cpu(entry->nid);
534 struct page *node_page;
535 struct node_info ni;
536 int err;
537
538 /* stop BG_GC if there is not enough free sections. */
539 if (gc_type == BG_GC && has_not_enough_free_secs(sbi, 0, 0))
540 return submitted;
541
542 if (check_valid_map(sbi, segno, off) == 0)
543 continue;
544
545 if (phase == 0) {
546 f2fs_ra_meta_pages(sbi, NAT_BLOCK_OFFSET(nid), 1,
547 META_NAT, true);
548 continue;
549 }
550
551 if (phase == 1) {
552 f2fs_ra_node_page(sbi, nid);
553 continue;
554 }
555
556 /* phase == 2 */
557 node_page = f2fs_get_node_page(sbi, nid);
558 if (IS_ERR(node_page))
559 continue;
560
561 /* block may become invalid during f2fs_get_node_page */
562 if (check_valid_map(sbi, segno, off) == 0) {
563 f2fs_put_page(node_page, 1);
564 continue;
565 }
566
567 if (f2fs_get_node_info(sbi, nid, &ni)) {
568 f2fs_put_page(node_page, 1);
569 continue;
570 }
571
572 if (ni.blk_addr != start_addr + off) {
573 f2fs_put_page(node_page, 1);
574 continue;
575 }
576
577 err = f2fs_move_node_page(node_page, gc_type);
578 if (!err && gc_type == FG_GC)
579 submitted++;
580 stat_inc_node_blk_count(sbi, 1, gc_type);
581 }
582
583 if (++phase < 3)
584 goto next_step;
585
586 if (fggc)
587 atomic_dec(&sbi->wb_sync_req[NODE]);
588 return submitted;
589 }
590
591 /*
592 * Calculate start block index indicating the given node offset.
593 * Be careful, caller should give this node offset only indicating direct node
594 * blocks. If any node offsets, which point the other types of node blocks such
595 * as indirect or double indirect node blocks, are given, it must be a caller's
596 * bug.
597 */
f2fs_start_bidx_of_node(unsigned int node_ofs,struct inode * inode)598 block_t f2fs_start_bidx_of_node(unsigned int node_ofs, struct inode *inode)
599 {
600 unsigned int indirect_blks = 2 * NIDS_PER_BLOCK + 4;
601 unsigned int bidx;
602
603 if (node_ofs == 0)
604 return 0;
605
606 if (node_ofs <= 2) {
607 bidx = node_ofs - 1;
608 } else if (node_ofs <= indirect_blks) {
609 int dec = (node_ofs - 4) / (NIDS_PER_BLOCK + 1);
610 bidx = node_ofs - 2 - dec;
611 } else {
612 int dec = (node_ofs - indirect_blks - 3) / (NIDS_PER_BLOCK + 1);
613 bidx = node_ofs - 5 - dec;
614 }
615 return bidx * ADDRS_PER_BLOCK(inode) + ADDRS_PER_INODE(inode);
616 }
617
is_alive(struct f2fs_sb_info * sbi,struct f2fs_summary * sum,struct node_info * dni,block_t blkaddr,unsigned int * nofs)618 static bool is_alive(struct f2fs_sb_info *sbi, struct f2fs_summary *sum,
619 struct node_info *dni, block_t blkaddr, unsigned int *nofs)
620 {
621 struct page *node_page;
622 nid_t nid;
623 unsigned int ofs_in_node, max_addrs, base;
624 block_t source_blkaddr;
625
626 nid = le32_to_cpu(sum->nid);
627 ofs_in_node = le16_to_cpu(sum->ofs_in_node);
628
629 node_page = f2fs_get_node_page(sbi, nid);
630 if (IS_ERR(node_page))
631 return false;
632
633 if (f2fs_get_node_info(sbi, nid, dni)) {
634 f2fs_put_page(node_page, 1);
635 return false;
636 }
637
638 if (sum->version != dni->version) {
639 f2fs_warn(sbi, "%s: valid data with mismatched node version.",
640 __func__);
641 set_sbi_flag(sbi, SBI_NEED_FSCK);
642 }
643
644 if (f2fs_check_nid_range(sbi, dni->ino)) {
645 f2fs_put_page(node_page, 1);
646 return false;
647 }
648
649 if (IS_INODE(node_page)) {
650 base = offset_in_addr(F2FS_INODE(node_page));
651 max_addrs = DEF_ADDRS_PER_INODE;
652 } else {
653 base = 0;
654 max_addrs = DEF_ADDRS_PER_BLOCK;
655 }
656
657 if (base + ofs_in_node >= max_addrs) {
658 f2fs_err(sbi, "Inconsistent blkaddr offset: base:%u, ofs_in_node:%u, max:%u, ino:%u, nid:%u",
659 base, ofs_in_node, max_addrs, dni->ino, dni->nid);
660 f2fs_put_page(node_page, 1);
661 return false;
662 }
663
664 *nofs = ofs_of_node(node_page);
665 source_blkaddr = data_blkaddr(NULL, node_page, ofs_in_node);
666 f2fs_put_page(node_page, 1);
667
668 if (source_blkaddr != blkaddr) {
669 #ifdef CONFIG_F2FS_CHECK_FS
670 unsigned int segno = GET_SEGNO(sbi, blkaddr);
671 unsigned long offset = GET_BLKOFF_FROM_SEG0(sbi, blkaddr);
672
673 if (unlikely(check_valid_map(sbi, segno, offset))) {
674 if (!test_and_set_bit(segno, SIT_I(sbi)->invalid_segmap)) {
675 f2fs_err(sbi, "mismatched blkaddr %u (source_blkaddr %u) in seg %u\n",
676 blkaddr, source_blkaddr, segno);
677 f2fs_bug_on(sbi, 1);
678 }
679 }
680 #endif
681 return false;
682 }
683 return true;
684 }
685
ra_data_block(struct inode * inode,pgoff_t index)686 static int ra_data_block(struct inode *inode, pgoff_t index)
687 {
688 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
689 struct address_space *mapping = inode->i_mapping;
690 struct dnode_of_data dn;
691 struct page *page;
692 struct extent_info ei = {0, 0, 0};
693 struct f2fs_io_info fio = {
694 .sbi = sbi,
695 .ino = inode->i_ino,
696 .type = DATA,
697 .temp = COLD,
698 .op = REQ_OP_READ,
699 .op_flags = 0,
700 .encrypted_page = NULL,
701 .in_list = false,
702 .retry = false,
703 };
704 int err;
705
706 page = f2fs_grab_cache_page(mapping, index, true);
707 if (!page)
708 return -ENOMEM;
709
710 if (f2fs_lookup_extent_cache(inode, index, &ei)) {
711 dn.data_blkaddr = ei.blk + index - ei.fofs;
712 if (unlikely(!f2fs_is_valid_blkaddr(sbi, dn.data_blkaddr,
713 DATA_GENERIC_ENHANCE_READ))) {
714 err = -EFSCORRUPTED;
715 goto put_page;
716 }
717 goto got_it;
718 }
719
720 set_new_dnode(&dn, inode, NULL, NULL, 0);
721 err = f2fs_get_dnode_of_data(&dn, index, LOOKUP_NODE);
722 if (err)
723 goto put_page;
724 f2fs_put_dnode(&dn);
725
726 if (!__is_valid_data_blkaddr(dn.data_blkaddr)) {
727 err = -ENOENT;
728 goto put_page;
729 }
730 if (unlikely(!f2fs_is_valid_blkaddr(sbi, dn.data_blkaddr,
731 DATA_GENERIC_ENHANCE))) {
732 err = -EFSCORRUPTED;
733 goto put_page;
734 }
735 got_it:
736 /* read page */
737 fio.page = page;
738 fio.new_blkaddr = fio.old_blkaddr = dn.data_blkaddr;
739
740 /*
741 * don't cache encrypted data into meta inode until previous dirty
742 * data were writebacked to avoid racing between GC and flush.
743 */
744 f2fs_wait_on_page_writeback(page, DATA, true, true);
745
746 f2fs_wait_on_block_writeback(inode, dn.data_blkaddr);
747
748 fio.encrypted_page = f2fs_pagecache_get_page(META_MAPPING(sbi),
749 dn.data_blkaddr,
750 FGP_LOCK | FGP_CREAT, GFP_NOFS);
751 if (!fio.encrypted_page) {
752 err = -ENOMEM;
753 goto put_page;
754 }
755
756 err = f2fs_submit_page_bio(&fio);
757 if (err)
758 goto put_encrypted_page;
759 f2fs_put_page(fio.encrypted_page, 0);
760 f2fs_put_page(page, 1);
761
762 f2fs_update_iostat(sbi, FS_DATA_READ_IO, F2FS_BLKSIZE);
763 f2fs_update_iostat(sbi, FS_GDATA_READ_IO, F2FS_BLKSIZE);
764
765 return 0;
766 put_encrypted_page:
767 f2fs_put_page(fio.encrypted_page, 1);
768 put_page:
769 f2fs_put_page(page, 1);
770 return err;
771 }
772
773 /*
774 * Move data block via META_MAPPING while keeping locked data page.
775 * This can be used to move blocks, aka LBAs, directly on disk.
776 */
move_data_block(struct inode * inode,block_t bidx,int gc_type,unsigned int segno,int off)777 static int move_data_block(struct inode *inode, block_t bidx,
778 int gc_type, unsigned int segno, int off)
779 {
780 struct f2fs_io_info fio = {
781 .sbi = F2FS_I_SB(inode),
782 .ino = inode->i_ino,
783 .type = DATA,
784 .temp = COLD,
785 .op = REQ_OP_READ,
786 .op_flags = 0,
787 .encrypted_page = NULL,
788 .in_list = false,
789 .retry = false,
790 };
791 struct dnode_of_data dn;
792 struct f2fs_summary sum;
793 struct node_info ni;
794 struct page *page, *mpage;
795 block_t newaddr;
796 int err = 0;
797 bool lfs_mode = f2fs_lfs_mode(fio.sbi);
798
799 /* do not read out */
800 page = f2fs_grab_cache_page(inode->i_mapping, bidx, false);
801 if (!page)
802 return -ENOMEM;
803
804 if (!check_valid_map(F2FS_I_SB(inode), segno, off)) {
805 err = -ENOENT;
806 goto out;
807 }
808
809 if (f2fs_is_atomic_file(inode)) {
810 F2FS_I(inode)->i_gc_failures[GC_FAILURE_ATOMIC]++;
811 F2FS_I_SB(inode)->skipped_atomic_files[gc_type]++;
812 err = -EAGAIN;
813 goto out;
814 }
815
816 if (f2fs_is_pinned_file(inode)) {
817 f2fs_pin_file_control(inode, true);
818 err = -EAGAIN;
819 goto out;
820 }
821
822 set_new_dnode(&dn, inode, NULL, NULL, 0);
823 err = f2fs_get_dnode_of_data(&dn, bidx, LOOKUP_NODE);
824 if (err)
825 goto out;
826
827 if (unlikely(dn.data_blkaddr == NULL_ADDR)) {
828 ClearPageUptodate(page);
829 err = -ENOENT;
830 goto put_out;
831 }
832
833 /*
834 * don't cache encrypted data into meta inode until previous dirty
835 * data were writebacked to avoid racing between GC and flush.
836 */
837 f2fs_wait_on_page_writeback(page, DATA, true, true);
838
839 f2fs_wait_on_block_writeback(inode, dn.data_blkaddr);
840
841 err = f2fs_get_node_info(fio.sbi, dn.nid, &ni);
842 if (err)
843 goto put_out;
844
845 set_summary(&sum, dn.nid, dn.ofs_in_node, ni.version);
846
847 /* read page */
848 fio.page = page;
849 fio.new_blkaddr = fio.old_blkaddr = dn.data_blkaddr;
850
851 if (lfs_mode)
852 down_write(&fio.sbi->io_order_lock);
853
854 mpage = f2fs_grab_cache_page(META_MAPPING(fio.sbi),
855 fio.old_blkaddr, false);
856 if (!mpage)
857 goto up_out;
858
859 fio.encrypted_page = mpage;
860
861 /* read source block in mpage */
862 if (!PageUptodate(mpage)) {
863 err = f2fs_submit_page_bio(&fio);
864 if (err) {
865 f2fs_put_page(mpage, 1);
866 goto up_out;
867 }
868
869 f2fs_update_iostat(fio.sbi, FS_DATA_READ_IO, F2FS_BLKSIZE);
870 f2fs_update_iostat(fio.sbi, FS_GDATA_READ_IO, F2FS_BLKSIZE);
871
872 lock_page(mpage);
873 if (unlikely(mpage->mapping != META_MAPPING(fio.sbi) ||
874 !PageUptodate(mpage))) {
875 err = -EIO;
876 f2fs_put_page(mpage, 1);
877 goto up_out;
878 }
879 }
880
881 f2fs_allocate_data_block(fio.sbi, NULL, fio.old_blkaddr, &newaddr,
882 &sum, CURSEG_COLD_DATA, NULL, false);
883
884 fio.encrypted_page = f2fs_pagecache_get_page(META_MAPPING(fio.sbi),
885 newaddr, FGP_LOCK | FGP_CREAT, GFP_NOFS);
886 if (!fio.encrypted_page) {
887 err = -ENOMEM;
888 f2fs_put_page(mpage, 1);
889 goto recover_block;
890 }
891
892 /* write target block */
893 f2fs_wait_on_page_writeback(fio.encrypted_page, DATA, true, true);
894 memcpy(page_address(fio.encrypted_page),
895 page_address(mpage), PAGE_SIZE);
896 f2fs_put_page(mpage, 1);
897 invalidate_mapping_pages(META_MAPPING(fio.sbi),
898 fio.old_blkaddr, fio.old_blkaddr);
899
900 set_page_dirty(fio.encrypted_page);
901 if (clear_page_dirty_for_io(fio.encrypted_page))
902 dec_page_count(fio.sbi, F2FS_DIRTY_META);
903
904 set_page_writeback(fio.encrypted_page);
905 ClearPageError(page);
906
907 /* allocate block address */
908 f2fs_wait_on_page_writeback(dn.node_page, NODE, true, true);
909
910 fio.op = REQ_OP_WRITE;
911 fio.op_flags = REQ_SYNC;
912 fio.new_blkaddr = newaddr;
913 f2fs_submit_page_write(&fio);
914 if (fio.retry) {
915 err = -EAGAIN;
916 if (PageWriteback(fio.encrypted_page))
917 end_page_writeback(fio.encrypted_page);
918 goto put_page_out;
919 }
920
921 f2fs_update_iostat(fio.sbi, FS_GC_DATA_IO, F2FS_BLKSIZE);
922
923 f2fs_update_data_blkaddr(&dn, newaddr);
924 set_inode_flag(inode, FI_APPEND_WRITE);
925 if (page->index == 0)
926 set_inode_flag(inode, FI_FIRST_BLOCK_WRITTEN);
927 put_page_out:
928 f2fs_put_page(fio.encrypted_page, 1);
929 recover_block:
930 if (err)
931 f2fs_do_replace_block(fio.sbi, &sum, newaddr, fio.old_blkaddr,
932 true, true);
933 up_out:
934 if (lfs_mode)
935 up_write(&fio.sbi->io_order_lock);
936 put_out:
937 f2fs_put_dnode(&dn);
938 out:
939 f2fs_put_page(page, 1);
940 return err;
941 }
942
move_data_page(struct inode * inode,block_t bidx,int gc_type,unsigned int segno,int off)943 static int move_data_page(struct inode *inode, block_t bidx, int gc_type,
944 unsigned int segno, int off)
945 {
946 struct page *page;
947 int err = 0;
948
949 page = f2fs_get_lock_data_page(inode, bidx, true);
950 if (IS_ERR(page))
951 return PTR_ERR(page);
952
953 if (!check_valid_map(F2FS_I_SB(inode), segno, off)) {
954 err = -ENOENT;
955 goto out;
956 }
957
958 if (f2fs_is_atomic_file(inode)) {
959 F2FS_I(inode)->i_gc_failures[GC_FAILURE_ATOMIC]++;
960 F2FS_I_SB(inode)->skipped_atomic_files[gc_type]++;
961 err = -EAGAIN;
962 goto out;
963 }
964 if (f2fs_is_pinned_file(inode)) {
965 if (gc_type == FG_GC)
966 f2fs_pin_file_control(inode, true);
967 err = -EAGAIN;
968 goto out;
969 }
970
971 if (gc_type == BG_GC) {
972 if (PageWriteback(page)) {
973 err = -EAGAIN;
974 goto out;
975 }
976 set_page_dirty(page);
977 set_cold_data(page);
978 } else {
979 struct f2fs_io_info fio = {
980 .sbi = F2FS_I_SB(inode),
981 .ino = inode->i_ino,
982 .type = DATA,
983 .temp = COLD,
984 .op = REQ_OP_WRITE,
985 .op_flags = REQ_SYNC,
986 .old_blkaddr = NULL_ADDR,
987 .page = page,
988 .encrypted_page = NULL,
989 .need_lock = LOCK_REQ,
990 .io_type = FS_GC_DATA_IO,
991 };
992 bool is_dirty = PageDirty(page);
993
994 retry:
995 f2fs_wait_on_page_writeback(page, DATA, true, true);
996
997 set_page_dirty(page);
998 if (clear_page_dirty_for_io(page)) {
999 inode_dec_dirty_pages(inode);
1000 f2fs_remove_dirty_inode(inode);
1001 }
1002
1003 set_cold_data(page);
1004
1005 err = f2fs_do_write_data_page(&fio);
1006 if (err) {
1007 clear_cold_data(page);
1008 if (err == -ENOMEM) {
1009 congestion_wait(BLK_RW_ASYNC,
1010 DEFAULT_IO_TIMEOUT);
1011 goto retry;
1012 }
1013 if (is_dirty)
1014 set_page_dirty(page);
1015 }
1016 }
1017 out:
1018 f2fs_put_page(page, 1);
1019 return err;
1020 }
1021
1022 /*
1023 * This function tries to get parent node of victim data block, and identifies
1024 * data block validity. If the block is valid, copy that with cold status and
1025 * modify parent node.
1026 * If the parent node is not valid or the data block address is different,
1027 * the victim data block is ignored.
1028 */
gc_data_segment(struct f2fs_sb_info * sbi,struct f2fs_summary * sum,struct gc_inode_list * gc_list,unsigned int segno,int gc_type)1029 static int gc_data_segment(struct f2fs_sb_info *sbi, struct f2fs_summary *sum,
1030 struct gc_inode_list *gc_list, unsigned int segno, int gc_type)
1031 {
1032 struct super_block *sb = sbi->sb;
1033 struct f2fs_summary *entry;
1034 block_t start_addr;
1035 int off;
1036 int phase = 0;
1037 int submitted = 0;
1038
1039 start_addr = START_BLOCK(sbi, segno);
1040
1041 next_step:
1042 entry = sum;
1043
1044 for (off = 0; off < sbi->blocks_per_seg; off++, entry++) {
1045 struct page *data_page;
1046 struct inode *inode;
1047 struct node_info dni; /* dnode info for the data */
1048 unsigned int ofs_in_node, nofs;
1049 block_t start_bidx;
1050 nid_t nid = le32_to_cpu(entry->nid);
1051
1052 /*
1053 * stop BG_GC if there is not enough free sections.
1054 * Or, stop GC if the segment becomes fully valid caused by
1055 * race condition along with SSR block allocation.
1056 */
1057 if ((gc_type == BG_GC && has_not_enough_free_secs(sbi, 0, 0)) ||
1058 get_valid_blocks(sbi, segno, true) ==
1059 BLKS_PER_SEC(sbi))
1060 return submitted;
1061
1062 if (check_valid_map(sbi, segno, off) == 0)
1063 continue;
1064
1065 if (phase == 0) {
1066 f2fs_ra_meta_pages(sbi, NAT_BLOCK_OFFSET(nid), 1,
1067 META_NAT, true);
1068 continue;
1069 }
1070
1071 if (phase == 1) {
1072 f2fs_ra_node_page(sbi, nid);
1073 continue;
1074 }
1075
1076 /* Get an inode by ino with checking validity */
1077 if (!is_alive(sbi, entry, &dni, start_addr + off, &nofs))
1078 continue;
1079
1080 if (phase == 2) {
1081 f2fs_ra_node_page(sbi, dni.ino);
1082 continue;
1083 }
1084
1085 ofs_in_node = le16_to_cpu(entry->ofs_in_node);
1086
1087 if (phase == 3) {
1088 inode = f2fs_iget(sb, dni.ino);
1089 if (IS_ERR(inode) || is_bad_inode(inode) ||
1090 special_file(inode->i_mode)) {
1091 set_sbi_flag(sbi, SBI_NEED_FSCK);
1092 continue;
1093 }
1094
1095 if (!down_write_trylock(
1096 &F2FS_I(inode)->i_gc_rwsem[WRITE])) {
1097 iput(inode);
1098 sbi->skipped_gc_rwsem++;
1099 continue;
1100 }
1101
1102 start_bidx = f2fs_start_bidx_of_node(nofs, inode) +
1103 ofs_in_node;
1104
1105 if (f2fs_post_read_required(inode)) {
1106 int err = ra_data_block(inode, start_bidx);
1107
1108 up_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]);
1109 if (err) {
1110 iput(inode);
1111 continue;
1112 }
1113 add_gc_inode(gc_list, inode);
1114 continue;
1115 }
1116
1117 data_page = f2fs_get_read_data_page(inode,
1118 start_bidx, REQ_RAHEAD, true);
1119 up_write(&F2FS_I(inode)->i_gc_rwsem[WRITE]);
1120 if (IS_ERR(data_page)) {
1121 iput(inode);
1122 continue;
1123 }
1124
1125 f2fs_put_page(data_page, 0);
1126 add_gc_inode(gc_list, inode);
1127 continue;
1128 }
1129
1130 /* phase 4 */
1131 inode = find_gc_inode(gc_list, dni.ino);
1132 if (inode) {
1133 struct f2fs_inode_info *fi = F2FS_I(inode);
1134 bool locked = false;
1135 int err;
1136
1137 if (S_ISREG(inode->i_mode)) {
1138 if (!down_write_trylock(&fi->i_gc_rwsem[READ])) {
1139 sbi->skipped_gc_rwsem++;
1140 continue;
1141 }
1142 if (!down_write_trylock(
1143 &fi->i_gc_rwsem[WRITE])) {
1144 sbi->skipped_gc_rwsem++;
1145 up_write(&fi->i_gc_rwsem[READ]);
1146 continue;
1147 }
1148 locked = true;
1149
1150 /* wait for all inflight aio data */
1151 inode_dio_wait(inode);
1152 }
1153
1154 start_bidx = f2fs_start_bidx_of_node(nofs, inode)
1155 + ofs_in_node;
1156 if (f2fs_post_read_required(inode))
1157 err = move_data_block(inode, start_bidx,
1158 gc_type, segno, off);
1159 else
1160 err = move_data_page(inode, start_bidx, gc_type,
1161 segno, off);
1162
1163 if (!err && (gc_type == FG_GC ||
1164 f2fs_post_read_required(inode)))
1165 submitted++;
1166
1167 if (locked) {
1168 up_write(&fi->i_gc_rwsem[WRITE]);
1169 up_write(&fi->i_gc_rwsem[READ]);
1170 }
1171
1172 stat_inc_data_blk_count(sbi, 1, gc_type);
1173 }
1174 }
1175
1176 if (++phase < 5)
1177 goto next_step;
1178
1179 return submitted;
1180 }
1181
__get_victim(struct f2fs_sb_info * sbi,unsigned int * victim,int gc_type)1182 static int __get_victim(struct f2fs_sb_info *sbi, unsigned int *victim,
1183 int gc_type)
1184 {
1185 struct sit_info *sit_i = SIT_I(sbi);
1186 int ret;
1187
1188 down_write(&sit_i->sentry_lock);
1189 ret = DIRTY_I(sbi)->v_ops->get_victim(sbi, victim, gc_type,
1190 NO_CHECK_TYPE, LFS);
1191 up_write(&sit_i->sentry_lock);
1192 return ret;
1193 }
1194
do_garbage_collect(struct f2fs_sb_info * sbi,unsigned int start_segno,struct gc_inode_list * gc_list,int gc_type)1195 static int do_garbage_collect(struct f2fs_sb_info *sbi,
1196 unsigned int start_segno,
1197 struct gc_inode_list *gc_list, int gc_type)
1198 {
1199 struct page *sum_page;
1200 struct f2fs_summary_block *sum;
1201 struct blk_plug plug;
1202 unsigned int segno = start_segno;
1203 unsigned int end_segno = start_segno + sbi->segs_per_sec;
1204 int seg_freed = 0, migrated = 0;
1205 unsigned char type = IS_DATASEG(get_seg_entry(sbi, segno)->type) ?
1206 SUM_TYPE_DATA : SUM_TYPE_NODE;
1207 int submitted = 0;
1208
1209 if (__is_large_section(sbi))
1210 end_segno = rounddown(end_segno, sbi->segs_per_sec);
1211
1212 /* readahead multi ssa blocks those have contiguous address */
1213 if (__is_large_section(sbi))
1214 f2fs_ra_meta_pages(sbi, GET_SUM_BLOCK(sbi, segno),
1215 end_segno - segno, META_SSA, true);
1216
1217 /* reference all summary page */
1218 while (segno < end_segno) {
1219 sum_page = f2fs_get_sum_page(sbi, segno++);
1220 if (IS_ERR(sum_page)) {
1221 int err = PTR_ERR(sum_page);
1222
1223 end_segno = segno - 1;
1224 for (segno = start_segno; segno < end_segno; segno++) {
1225 sum_page = find_get_page(META_MAPPING(sbi),
1226 GET_SUM_BLOCK(sbi, segno));
1227 f2fs_put_page(sum_page, 0);
1228 f2fs_put_page(sum_page, 0);
1229 }
1230 return err;
1231 }
1232 unlock_page(sum_page);
1233 }
1234
1235 blk_start_plug(&plug);
1236
1237 for (segno = start_segno; segno < end_segno; segno++) {
1238
1239 /* find segment summary of victim */
1240 sum_page = find_get_page(META_MAPPING(sbi),
1241 GET_SUM_BLOCK(sbi, segno));
1242 f2fs_put_page(sum_page, 0);
1243
1244 if (get_valid_blocks(sbi, segno, false) == 0)
1245 goto freed;
1246 if (gc_type == BG_GC && __is_large_section(sbi) &&
1247 migrated >= sbi->migration_granularity)
1248 goto skip;
1249 if (!PageUptodate(sum_page) || unlikely(f2fs_cp_error(sbi)))
1250 goto skip;
1251
1252 sum = page_address(sum_page);
1253 if (type != GET_SUM_TYPE((&sum->footer))) {
1254 f2fs_err(sbi, "Inconsistent segment (%u) type [%d, %d] in SSA and SIT",
1255 segno, type, GET_SUM_TYPE((&sum->footer)));
1256 set_sbi_flag(sbi, SBI_NEED_FSCK);
1257 f2fs_stop_checkpoint(sbi, false);
1258 goto skip;
1259 }
1260
1261 /*
1262 * this is to avoid deadlock:
1263 * - lock_page(sum_page) - f2fs_replace_block
1264 * - check_valid_map() - down_write(sentry_lock)
1265 * - down_read(sentry_lock) - change_curseg()
1266 * - lock_page(sum_page)
1267 */
1268 if (type == SUM_TYPE_NODE)
1269 submitted += gc_node_segment(sbi, sum->entries, segno,
1270 gc_type);
1271 else
1272 submitted += gc_data_segment(sbi, sum->entries, gc_list,
1273 segno, gc_type);
1274
1275 stat_inc_seg_count(sbi, type, gc_type);
1276 migrated++;
1277
1278 freed:
1279 if (gc_type == FG_GC &&
1280 get_valid_blocks(sbi, segno, false) == 0)
1281 seg_freed++;
1282
1283 if (__is_large_section(sbi))
1284 sbi->next_victim_seg[gc_type] =
1285 (segno + 1 < end_segno) ? segno + 1 : NULL_SEGNO;
1286 skip:
1287 f2fs_put_page(sum_page, 0);
1288 }
1289
1290 if (submitted)
1291 f2fs_submit_merged_write(sbi,
1292 (type == SUM_TYPE_NODE) ? NODE : DATA);
1293
1294 blk_finish_plug(&plug);
1295
1296 stat_inc_call_count(sbi->stat_info);
1297
1298 return seg_freed;
1299 }
1300
f2fs_gc(struct f2fs_sb_info * sbi,bool sync,bool background,unsigned int segno)1301 int f2fs_gc(struct f2fs_sb_info *sbi, bool sync,
1302 bool background, unsigned int segno)
1303 {
1304 int gc_type = sync ? FG_GC : BG_GC;
1305 int sec_freed = 0, seg_freed = 0, total_freed = 0;
1306 int ret = 0;
1307 struct cp_control cpc;
1308 unsigned int init_segno = segno;
1309 struct gc_inode_list gc_list = {
1310 .ilist = LIST_HEAD_INIT(gc_list.ilist),
1311 .iroot = RADIX_TREE_INIT(gc_list.iroot, GFP_NOFS),
1312 };
1313 unsigned long long last_skipped = sbi->skipped_atomic_files[FG_GC];
1314 unsigned long long first_skipped;
1315 unsigned int skipped_round = 0, round = 0;
1316
1317 trace_f2fs_gc_begin(sbi->sb, sync, background,
1318 get_pages(sbi, F2FS_DIRTY_NODES),
1319 get_pages(sbi, F2FS_DIRTY_DENTS),
1320 get_pages(sbi, F2FS_DIRTY_IMETA),
1321 free_sections(sbi),
1322 free_segments(sbi),
1323 reserved_segments(sbi),
1324 prefree_segments(sbi));
1325
1326 cpc.reason = __get_cp_reason(sbi);
1327 sbi->skipped_gc_rwsem = 0;
1328 first_skipped = last_skipped;
1329 gc_more:
1330 if (unlikely(!(sbi->sb->s_flags & SB_ACTIVE))) {
1331 ret = -EINVAL;
1332 goto stop;
1333 }
1334 if (unlikely(f2fs_cp_error(sbi))) {
1335 ret = -EIO;
1336 goto stop;
1337 }
1338
1339 if (gc_type == BG_GC && has_not_enough_free_secs(sbi, 0, 0)) {
1340 /*
1341 * For example, if there are many prefree_segments below given
1342 * threshold, we can make them free by checkpoint. Then, we
1343 * secure free segments which doesn't need fggc any more.
1344 */
1345 if (prefree_segments(sbi) &&
1346 !is_sbi_flag_set(sbi, SBI_CP_DISABLED)) {
1347 ret = f2fs_write_checkpoint(sbi, &cpc);
1348 if (ret)
1349 goto stop;
1350 }
1351 if (has_not_enough_free_secs(sbi, 0, 0))
1352 gc_type = FG_GC;
1353 }
1354
1355 /* f2fs_balance_fs doesn't need to do BG_GC in critical path. */
1356 if (gc_type == BG_GC && !background) {
1357 ret = -EINVAL;
1358 goto stop;
1359 }
1360 if (!__get_victim(sbi, &segno, gc_type)) {
1361 ret = -ENODATA;
1362 goto stop;
1363 }
1364
1365 seg_freed = do_garbage_collect(sbi, segno, &gc_list, gc_type);
1366 if (gc_type == FG_GC && seg_freed == sbi->segs_per_sec)
1367 sec_freed++;
1368 total_freed += seg_freed;
1369
1370 if (gc_type == FG_GC) {
1371 if (sbi->skipped_atomic_files[FG_GC] > last_skipped ||
1372 sbi->skipped_gc_rwsem)
1373 skipped_round++;
1374 last_skipped = sbi->skipped_atomic_files[FG_GC];
1375 round++;
1376 }
1377
1378 if (gc_type == FG_GC && seg_freed)
1379 sbi->cur_victim_sec = NULL_SEGNO;
1380
1381 if (sync)
1382 goto stop;
1383
1384 if (has_not_enough_free_secs(sbi, sec_freed, 0)) {
1385 if (skipped_round <= MAX_SKIP_GC_COUNT ||
1386 skipped_round * 2 < round) {
1387 segno = NULL_SEGNO;
1388 goto gc_more;
1389 }
1390
1391 if (first_skipped < last_skipped &&
1392 (last_skipped - first_skipped) >
1393 sbi->skipped_gc_rwsem) {
1394 f2fs_drop_inmem_pages_all(sbi, true);
1395 segno = NULL_SEGNO;
1396 goto gc_more;
1397 }
1398 if (gc_type == FG_GC && !is_sbi_flag_set(sbi, SBI_CP_DISABLED))
1399 ret = f2fs_write_checkpoint(sbi, &cpc);
1400 }
1401 stop:
1402 SIT_I(sbi)->last_victim[ALLOC_NEXT] = 0;
1403 SIT_I(sbi)->last_victim[FLUSH_DEVICE] = init_segno;
1404
1405 trace_f2fs_gc_end(sbi->sb, ret, total_freed, sec_freed,
1406 get_pages(sbi, F2FS_DIRTY_NODES),
1407 get_pages(sbi, F2FS_DIRTY_DENTS),
1408 get_pages(sbi, F2FS_DIRTY_IMETA),
1409 free_sections(sbi),
1410 free_segments(sbi),
1411 reserved_segments(sbi),
1412 prefree_segments(sbi));
1413
1414 up_write(&sbi->gc_lock);
1415
1416 put_gc_inode(&gc_list);
1417
1418 if (sync && !ret)
1419 ret = sec_freed ? 0 : -EAGAIN;
1420 return ret;
1421 }
1422
f2fs_build_gc_manager(struct f2fs_sb_info * sbi)1423 void f2fs_build_gc_manager(struct f2fs_sb_info *sbi)
1424 {
1425 DIRTY_I(sbi)->v_ops = &default_v_ops;
1426
1427 sbi->gc_pin_file_threshold = DEF_GC_FAILED_PINNED_FILES;
1428
1429 /* give warm/cold data area from slower device */
1430 if (f2fs_is_multi_device(sbi) && !__is_large_section(sbi))
1431 SIT_I(sbi)->last_victim[ALLOC_NEXT] =
1432 GET_SEGNO(sbi, FDEV(0).end_blk) + 1;
1433 }
1434
free_segment_range(struct f2fs_sb_info * sbi,unsigned int secs,bool gc_only)1435 static int free_segment_range(struct f2fs_sb_info *sbi,
1436 unsigned int secs, bool gc_only)
1437 {
1438 unsigned int segno, next_inuse, start, end;
1439 struct cp_control cpc = { CP_RESIZE, 0, 0, 0 };
1440 int gc_mode, gc_type;
1441 int err = 0;
1442 int type;
1443
1444 /* Force block allocation for GC */
1445 MAIN_SECS(sbi) -= secs;
1446 start = MAIN_SECS(sbi) * sbi->segs_per_sec;
1447 end = MAIN_SEGS(sbi) - 1;
1448
1449 mutex_lock(&DIRTY_I(sbi)->seglist_lock);
1450 for (gc_mode = 0; gc_mode < MAX_GC_POLICY; gc_mode++)
1451 if (SIT_I(sbi)->last_victim[gc_mode] >= start)
1452 SIT_I(sbi)->last_victim[gc_mode] = 0;
1453
1454 for (gc_type = BG_GC; gc_type <= FG_GC; gc_type++)
1455 if (sbi->next_victim_seg[gc_type] >= start)
1456 sbi->next_victim_seg[gc_type] = NULL_SEGNO;
1457 mutex_unlock(&DIRTY_I(sbi)->seglist_lock);
1458
1459 /* Move out cursegs from the target range */
1460 for (type = CURSEG_HOT_DATA; type < NR_CURSEG_TYPE; type++)
1461 allocate_segment_for_resize(sbi, type, start, end);
1462
1463 /* do GC to move out valid blocks in the range */
1464 for (segno = start; segno <= end; segno += sbi->segs_per_sec) {
1465 struct gc_inode_list gc_list = {
1466 .ilist = LIST_HEAD_INIT(gc_list.ilist),
1467 .iroot = RADIX_TREE_INIT(gc_list.iroot, GFP_NOFS),
1468 };
1469
1470 do_garbage_collect(sbi, segno, &gc_list, FG_GC);
1471 put_gc_inode(&gc_list);
1472
1473 if (!gc_only && get_valid_blocks(sbi, segno, true)) {
1474 err = -EAGAIN;
1475 goto out;
1476 }
1477 if (fatal_signal_pending(current)) {
1478 err = -ERESTARTSYS;
1479 goto out;
1480 }
1481 }
1482 if (gc_only)
1483 goto out;
1484
1485 err = f2fs_write_checkpoint(sbi, &cpc);
1486 if (err)
1487 goto out;
1488
1489 next_inuse = find_next_inuse(FREE_I(sbi), end + 1, start);
1490 if (next_inuse <= end) {
1491 f2fs_err(sbi, "segno %u should be free but still inuse!",
1492 next_inuse);
1493 f2fs_bug_on(sbi, 1);
1494 }
1495 out:
1496 MAIN_SECS(sbi) += secs;
1497 return err;
1498 }
1499
update_sb_metadata(struct f2fs_sb_info * sbi,int secs)1500 static void update_sb_metadata(struct f2fs_sb_info *sbi, int secs)
1501 {
1502 struct f2fs_super_block *raw_sb = F2FS_RAW_SUPER(sbi);
1503 int section_count;
1504 int segment_count;
1505 int segment_count_main;
1506 long long block_count;
1507 int segs = secs * sbi->segs_per_sec;
1508
1509 down_write(&sbi->sb_lock);
1510
1511 section_count = le32_to_cpu(raw_sb->section_count);
1512 segment_count = le32_to_cpu(raw_sb->segment_count);
1513 segment_count_main = le32_to_cpu(raw_sb->segment_count_main);
1514 block_count = le64_to_cpu(raw_sb->block_count);
1515
1516 raw_sb->section_count = cpu_to_le32(section_count + secs);
1517 raw_sb->segment_count = cpu_to_le32(segment_count + segs);
1518 raw_sb->segment_count_main = cpu_to_le32(segment_count_main + segs);
1519 raw_sb->block_count = cpu_to_le64(block_count +
1520 (long long)segs * sbi->blocks_per_seg);
1521 if (f2fs_is_multi_device(sbi)) {
1522 int last_dev = sbi->s_ndevs - 1;
1523 int dev_segs =
1524 le32_to_cpu(raw_sb->devs[last_dev].total_segments);
1525
1526 raw_sb->devs[last_dev].total_segments =
1527 cpu_to_le32(dev_segs + segs);
1528 }
1529
1530 up_write(&sbi->sb_lock);
1531 }
1532
update_fs_metadata(struct f2fs_sb_info * sbi,int secs)1533 static void update_fs_metadata(struct f2fs_sb_info *sbi, int secs)
1534 {
1535 int segs = secs * sbi->segs_per_sec;
1536 long long blks = (long long)segs * sbi->blocks_per_seg;
1537 long long user_block_count =
1538 le64_to_cpu(F2FS_CKPT(sbi)->user_block_count);
1539
1540 SM_I(sbi)->segment_count = (int)SM_I(sbi)->segment_count + segs;
1541 MAIN_SEGS(sbi) = (int)MAIN_SEGS(sbi) + segs;
1542 MAIN_SECS(sbi) += secs;
1543 FREE_I(sbi)->free_sections = (int)FREE_I(sbi)->free_sections + secs;
1544 FREE_I(sbi)->free_segments = (int)FREE_I(sbi)->free_segments + segs;
1545 F2FS_CKPT(sbi)->user_block_count = cpu_to_le64(user_block_count + blks);
1546
1547 if (f2fs_is_multi_device(sbi)) {
1548 int last_dev = sbi->s_ndevs - 1;
1549
1550 FDEV(last_dev).total_segments =
1551 (int)FDEV(last_dev).total_segments + segs;
1552 FDEV(last_dev).end_blk =
1553 (long long)FDEV(last_dev).end_blk + blks;
1554 #ifdef CONFIG_BLK_DEV_ZONED
1555 FDEV(last_dev).nr_blkz = (int)FDEV(last_dev).nr_blkz +
1556 (int)(blks >> sbi->log_blocks_per_blkz);
1557 #endif
1558 }
1559 }
1560
f2fs_resize_fs(struct f2fs_sb_info * sbi,__u64 block_count)1561 int f2fs_resize_fs(struct f2fs_sb_info *sbi, __u64 block_count)
1562 {
1563 __u64 old_block_count, shrunk_blocks;
1564 struct cp_control cpc = { CP_RESIZE, 0, 0, 0 };
1565 unsigned int secs;
1566 int err = 0;
1567 __u32 rem;
1568
1569 old_block_count = le64_to_cpu(F2FS_RAW_SUPER(sbi)->block_count);
1570 if (block_count > old_block_count)
1571 return -EINVAL;
1572
1573 if (f2fs_is_multi_device(sbi)) {
1574 int last_dev = sbi->s_ndevs - 1;
1575 __u64 last_segs = FDEV(last_dev).total_segments;
1576
1577 if (block_count + last_segs * sbi->blocks_per_seg <=
1578 old_block_count)
1579 return -EINVAL;
1580 }
1581
1582 /* new fs size should align to section size */
1583 div_u64_rem(block_count, BLKS_PER_SEC(sbi), &rem);
1584 if (rem)
1585 return -EINVAL;
1586
1587 if (block_count == old_block_count)
1588 return 0;
1589
1590 if (is_sbi_flag_set(sbi, SBI_NEED_FSCK)) {
1591 f2fs_err(sbi, "Should run fsck to repair first.");
1592 return -EFSCORRUPTED;
1593 }
1594
1595 if (test_opt(sbi, DISABLE_CHECKPOINT)) {
1596 f2fs_err(sbi, "Checkpoint should be enabled.");
1597 return -EINVAL;
1598 }
1599
1600 shrunk_blocks = old_block_count - block_count;
1601 secs = div_u64(shrunk_blocks, BLKS_PER_SEC(sbi));
1602
1603 /* stop other GC */
1604 if (!down_write_trylock(&sbi->gc_lock))
1605 return -EAGAIN;
1606
1607 /* stop CP to protect MAIN_SEC in free_segment_range */
1608 f2fs_lock_op(sbi);
1609 err = free_segment_range(sbi, secs, true);
1610 f2fs_unlock_op(sbi);
1611 up_write(&sbi->gc_lock);
1612 if (err)
1613 return err;
1614
1615 set_sbi_flag(sbi, SBI_IS_RESIZEFS);
1616
1617 freeze_super(sbi->sb);
1618 down_write(&sbi->gc_lock);
1619 mutex_lock(&sbi->cp_mutex);
1620
1621 spin_lock(&sbi->stat_lock);
1622 if (shrunk_blocks + valid_user_blocks(sbi) +
1623 sbi->current_reserved_blocks + sbi->unusable_block_count +
1624 F2FS_OPTION(sbi).root_reserved_blocks > sbi->user_block_count)
1625 err = -ENOSPC;
1626 else
1627 sbi->user_block_count -= shrunk_blocks;
1628 spin_unlock(&sbi->stat_lock);
1629 if (err)
1630 goto out_err;
1631
1632 err = free_segment_range(sbi, secs, false);
1633 if (err)
1634 goto recover_out;
1635
1636 update_sb_metadata(sbi, -secs);
1637
1638 err = f2fs_commit_super(sbi, false);
1639 if (err) {
1640 update_sb_metadata(sbi, secs);
1641 goto recover_out;
1642 }
1643
1644 update_fs_metadata(sbi, -secs);
1645 clear_sbi_flag(sbi, SBI_IS_RESIZEFS);
1646 set_sbi_flag(sbi, SBI_IS_DIRTY);
1647
1648 err = f2fs_write_checkpoint(sbi, &cpc);
1649 if (err) {
1650 update_fs_metadata(sbi, secs);
1651 update_sb_metadata(sbi, secs);
1652 f2fs_commit_super(sbi, false);
1653 }
1654 recover_out:
1655 if (err) {
1656 set_sbi_flag(sbi, SBI_NEED_FSCK);
1657 f2fs_err(sbi, "resize_fs failed, should run fsck to repair!");
1658
1659 spin_lock(&sbi->stat_lock);
1660 sbi->user_block_count += shrunk_blocks;
1661 spin_unlock(&sbi->stat_lock);
1662 }
1663 out_err:
1664 mutex_unlock(&sbi->cp_mutex);
1665 up_write(&sbi->gc_lock);
1666 thaw_super(sbi->sb);
1667 clear_sbi_flag(sbi, SBI_IS_RESIZEFS);
1668 return err;
1669 }
1670