1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * fs/f2fs/segment.c
4 *
5 * Copyright (c) 2012 Samsung Electronics Co., Ltd.
6 * http://www.samsung.com/
7 */
8 #include <linux/fs.h>
9 #include <linux/f2fs_fs.h>
10 #include <linux/bio.h>
11 #include <linux/blkdev.h>
12 #include <linux/prefetch.h>
13 #include <linux/kthread.h>
14 #include <linux/swap.h>
15 #include <linux/timer.h>
16 #include <linux/freezer.h>
17 #include <linux/sched/signal.h>
18 #include <linux/random.h>
19
20 #include "f2fs.h"
21 #include "segment.h"
22 #include "node.h"
23 #include "gc.h"
24 #include "iostat.h"
25 #include <trace/events/f2fs.h>
26
27 #define __reverse_ffz(x) __reverse_ffs(~(x))
28
29 static struct kmem_cache *discard_entry_slab;
30 static struct kmem_cache *discard_cmd_slab;
31 static struct kmem_cache *sit_entry_set_slab;
32 static struct kmem_cache *inmem_entry_slab;
33
__reverse_ulong(unsigned char * str)34 static unsigned long __reverse_ulong(unsigned char *str)
35 {
36 unsigned long tmp = 0;
37 int shift = 24, idx = 0;
38
39 #if BITS_PER_LONG == 64
40 shift = 56;
41 #endif
42 while (shift >= 0) {
43 tmp |= (unsigned long)str[idx++] << shift;
44 shift -= BITS_PER_BYTE;
45 }
46 return tmp;
47 }
48
49 /*
50 * __reverse_ffs is copied from include/asm-generic/bitops/__ffs.h since
51 * MSB and LSB are reversed in a byte by f2fs_set_bit.
52 */
__reverse_ffs(unsigned long word)53 static inline unsigned long __reverse_ffs(unsigned long word)
54 {
55 int num = 0;
56
57 #if BITS_PER_LONG == 64
58 if ((word & 0xffffffff00000000UL) == 0)
59 num += 32;
60 else
61 word >>= 32;
62 #endif
63 if ((word & 0xffff0000) == 0)
64 num += 16;
65 else
66 word >>= 16;
67
68 if ((word & 0xff00) == 0)
69 num += 8;
70 else
71 word >>= 8;
72
73 if ((word & 0xf0) == 0)
74 num += 4;
75 else
76 word >>= 4;
77
78 if ((word & 0xc) == 0)
79 num += 2;
80 else
81 word >>= 2;
82
83 if ((word & 0x2) == 0)
84 num += 1;
85 return num;
86 }
87
88 /*
89 * __find_rev_next(_zero)_bit is copied from lib/find_next_bit.c because
90 * f2fs_set_bit makes MSB and LSB reversed in a byte.
91 * @size must be integral times of unsigned long.
92 * Example:
93 * MSB <--> LSB
94 * f2fs_set_bit(0, bitmap) => 1000 0000
95 * f2fs_set_bit(7, bitmap) => 0000 0001
96 */
__find_rev_next_bit(const unsigned long * addr,unsigned long size,unsigned long offset)97 static unsigned long __find_rev_next_bit(const unsigned long *addr,
98 unsigned long size, unsigned long offset)
99 {
100 const unsigned long *p = addr + BIT_WORD(offset);
101 unsigned long result = size;
102 unsigned long tmp;
103
104 if (offset >= size)
105 return size;
106
107 size -= (offset & ~(BITS_PER_LONG - 1));
108 offset %= BITS_PER_LONG;
109
110 while (1) {
111 if (*p == 0)
112 goto pass;
113
114 tmp = __reverse_ulong((unsigned char *)p);
115
116 tmp &= ~0UL >> offset;
117 if (size < BITS_PER_LONG)
118 tmp &= (~0UL << (BITS_PER_LONG - size));
119 if (tmp)
120 goto found;
121 pass:
122 if (size <= BITS_PER_LONG)
123 break;
124 size -= BITS_PER_LONG;
125 offset = 0;
126 p++;
127 }
128 return result;
129 found:
130 return result - size + __reverse_ffs(tmp);
131 }
132
__find_rev_next_zero_bit(const unsigned long * addr,unsigned long size,unsigned long offset)133 static unsigned long __find_rev_next_zero_bit(const unsigned long *addr,
134 unsigned long size, unsigned long offset)
135 {
136 const unsigned long *p = addr + BIT_WORD(offset);
137 unsigned long result = size;
138 unsigned long tmp;
139
140 if (offset >= size)
141 return size;
142
143 size -= (offset & ~(BITS_PER_LONG - 1));
144 offset %= BITS_PER_LONG;
145
146 while (1) {
147 if (*p == ~0UL)
148 goto pass;
149
150 tmp = __reverse_ulong((unsigned char *)p);
151
152 if (offset)
153 tmp |= ~0UL << (BITS_PER_LONG - offset);
154 if (size < BITS_PER_LONG)
155 tmp |= ~0UL >> size;
156 if (tmp != ~0UL)
157 goto found;
158 pass:
159 if (size <= BITS_PER_LONG)
160 break;
161 size -= BITS_PER_LONG;
162 offset = 0;
163 p++;
164 }
165 return result;
166 found:
167 return result - size + __reverse_ffz(tmp);
168 }
169
f2fs_need_SSR(struct f2fs_sb_info * sbi)170 bool f2fs_need_SSR(struct f2fs_sb_info *sbi)
171 {
172 int node_secs = get_blocktype_secs(sbi, F2FS_DIRTY_NODES);
173 int dent_secs = get_blocktype_secs(sbi, F2FS_DIRTY_DENTS);
174 int imeta_secs = get_blocktype_secs(sbi, F2FS_DIRTY_IMETA);
175
176 if (f2fs_lfs_mode(sbi))
177 return false;
178 if (sbi->gc_mode == GC_URGENT_HIGH)
179 return true;
180 if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED)))
181 return true;
182
183 return free_sections(sbi) <= (node_secs + 2 * dent_secs + imeta_secs +
184 SM_I(sbi)->min_ssr_sections + reserved_sections(sbi));
185 }
186
f2fs_register_inmem_page(struct inode * inode,struct page * page)187 void f2fs_register_inmem_page(struct inode *inode, struct page *page)
188 {
189 struct inmem_pages *new;
190
191 set_page_private_atomic(page);
192
193 new = f2fs_kmem_cache_alloc(inmem_entry_slab,
194 GFP_NOFS, true, NULL);
195
196 /* add atomic page indices to the list */
197 new->page = page;
198 INIT_LIST_HEAD(&new->list);
199
200 /* increase reference count with clean state */
201 get_page(page);
202 mutex_lock(&F2FS_I(inode)->inmem_lock);
203 list_add_tail(&new->list, &F2FS_I(inode)->inmem_pages);
204 inc_page_count(F2FS_I_SB(inode), F2FS_INMEM_PAGES);
205 mutex_unlock(&F2FS_I(inode)->inmem_lock);
206
207 trace_f2fs_register_inmem_page(page, INMEM);
208 }
209
__revoke_inmem_pages(struct inode * inode,struct list_head * head,bool drop,bool recover,bool trylock)210 static int __revoke_inmem_pages(struct inode *inode,
211 struct list_head *head, bool drop, bool recover,
212 bool trylock)
213 {
214 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
215 struct inmem_pages *cur, *tmp;
216 int err = 0;
217
218 list_for_each_entry_safe(cur, tmp, head, list) {
219 struct page *page = cur->page;
220
221 if (drop)
222 trace_f2fs_commit_inmem_page(page, INMEM_DROP);
223
224 if (trylock) {
225 /*
226 * to avoid deadlock in between page lock and
227 * inmem_lock.
228 */
229 if (!trylock_page(page))
230 continue;
231 } else {
232 lock_page(page);
233 }
234
235 f2fs_wait_on_page_writeback(page, DATA, true, true);
236
237 if (recover) {
238 struct dnode_of_data dn;
239 struct node_info ni;
240
241 trace_f2fs_commit_inmem_page(page, INMEM_REVOKE);
242 retry:
243 set_new_dnode(&dn, inode, NULL, NULL, 0);
244 err = f2fs_get_dnode_of_data(&dn, page->index,
245 LOOKUP_NODE);
246 if (err) {
247 if (err == -ENOMEM) {
248 congestion_wait(BLK_RW_ASYNC,
249 DEFAULT_IO_TIMEOUT);
250 cond_resched();
251 goto retry;
252 }
253 err = -EAGAIN;
254 goto next;
255 }
256
257 err = f2fs_get_node_info(sbi, dn.nid, &ni, false);
258 if (err) {
259 f2fs_put_dnode(&dn);
260 return err;
261 }
262
263 if (cur->old_addr == NEW_ADDR) {
264 f2fs_invalidate_blocks(sbi, dn.data_blkaddr);
265 f2fs_update_data_blkaddr(&dn, NEW_ADDR);
266 } else
267 f2fs_replace_block(sbi, &dn, dn.data_blkaddr,
268 cur->old_addr, ni.version, true, true);
269 f2fs_put_dnode(&dn);
270 }
271 next:
272 /* we don't need to invalidate this in the sccessful status */
273 if (drop || recover) {
274 ClearPageUptodate(page);
275 clear_page_private_gcing(page);
276 }
277 detach_page_private(page);
278 set_page_private(page, 0);
279 f2fs_put_page(page, 1);
280
281 list_del(&cur->list);
282 kmem_cache_free(inmem_entry_slab, cur);
283 dec_page_count(F2FS_I_SB(inode), F2FS_INMEM_PAGES);
284 }
285 return err;
286 }
287
f2fs_drop_inmem_pages_all(struct f2fs_sb_info * sbi,bool gc_failure)288 void f2fs_drop_inmem_pages_all(struct f2fs_sb_info *sbi, bool gc_failure)
289 {
290 struct list_head *head = &sbi->inode_list[ATOMIC_FILE];
291 struct inode *inode;
292 struct f2fs_inode_info *fi;
293 unsigned int count = sbi->atomic_files;
294 unsigned int looped = 0;
295 next:
296 spin_lock(&sbi->inode_lock[ATOMIC_FILE]);
297 if (list_empty(head)) {
298 spin_unlock(&sbi->inode_lock[ATOMIC_FILE]);
299 return;
300 }
301 fi = list_first_entry(head, struct f2fs_inode_info, inmem_ilist);
302 inode = igrab(&fi->vfs_inode);
303 if (inode)
304 list_move_tail(&fi->inmem_ilist, head);
305 spin_unlock(&sbi->inode_lock[ATOMIC_FILE]);
306
307 if (inode) {
308 if (gc_failure) {
309 if (!fi->i_gc_failures[GC_FAILURE_ATOMIC])
310 goto skip;
311 }
312 set_inode_flag(inode, FI_ATOMIC_REVOKE_REQUEST);
313 f2fs_drop_inmem_pages(inode);
314 skip:
315 iput(inode);
316 }
317 f2fs_io_schedule_timeout(DEFAULT_IO_TIMEOUT);
318 if (gc_failure) {
319 if (++looped >= count)
320 return;
321 }
322 goto next;
323 }
324
f2fs_drop_inmem_pages(struct inode * inode)325 void f2fs_drop_inmem_pages(struct inode *inode)
326 {
327 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
328 struct f2fs_inode_info *fi = F2FS_I(inode);
329
330 do {
331 mutex_lock(&fi->inmem_lock);
332 if (list_empty(&fi->inmem_pages)) {
333 fi->i_gc_failures[GC_FAILURE_ATOMIC] = 0;
334
335 spin_lock(&sbi->inode_lock[ATOMIC_FILE]);
336 if (!list_empty(&fi->inmem_ilist))
337 list_del_init(&fi->inmem_ilist);
338 if (f2fs_is_atomic_file(inode)) {
339 clear_inode_flag(inode, FI_ATOMIC_FILE);
340 sbi->atomic_files--;
341 }
342 spin_unlock(&sbi->inode_lock[ATOMIC_FILE]);
343
344 mutex_unlock(&fi->inmem_lock);
345 break;
346 }
347 __revoke_inmem_pages(inode, &fi->inmem_pages,
348 true, false, true);
349 mutex_unlock(&fi->inmem_lock);
350 } while (1);
351 }
352
f2fs_drop_inmem_page(struct inode * inode,struct page * page)353 void f2fs_drop_inmem_page(struct inode *inode, struct page *page)
354 {
355 struct f2fs_inode_info *fi = F2FS_I(inode);
356 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
357 struct list_head *head = &fi->inmem_pages;
358 struct inmem_pages *cur = NULL;
359 struct inmem_pages *tmp;
360
361 f2fs_bug_on(sbi, !page_private_atomic(page));
362
363 mutex_lock(&fi->inmem_lock);
364 list_for_each_entry(tmp, head, list) {
365 if (tmp->page == page) {
366 cur = tmp;
367 break;
368 }
369 }
370
371 f2fs_bug_on(sbi, !cur);
372 list_del(&cur->list);
373 mutex_unlock(&fi->inmem_lock);
374
375 dec_page_count(sbi, F2FS_INMEM_PAGES);
376 kmem_cache_free(inmem_entry_slab, cur);
377
378 ClearPageUptodate(page);
379 clear_page_private_atomic(page);
380 f2fs_put_page(page, 0);
381
382 detach_page_private(page);
383 set_page_private(page, 0);
384
385 trace_f2fs_commit_inmem_page(page, INMEM_INVALIDATE);
386 }
387
__f2fs_commit_inmem_pages(struct inode * inode)388 static int __f2fs_commit_inmem_pages(struct inode *inode)
389 {
390 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
391 struct f2fs_inode_info *fi = F2FS_I(inode);
392 struct inmem_pages *cur, *tmp;
393 struct f2fs_io_info fio = {
394 .sbi = sbi,
395 .ino = inode->i_ino,
396 .type = DATA,
397 .op = REQ_OP_WRITE,
398 .op_flags = REQ_SYNC | REQ_PRIO,
399 .io_type = FS_DATA_IO,
400 };
401 struct list_head revoke_list;
402 bool submit_bio = false;
403 int err = 0;
404
405 INIT_LIST_HEAD(&revoke_list);
406
407 list_for_each_entry_safe(cur, tmp, &fi->inmem_pages, list) {
408 struct page *page = cur->page;
409
410 lock_page(page);
411 if (page->mapping == inode->i_mapping) {
412 trace_f2fs_commit_inmem_page(page, INMEM);
413
414 f2fs_wait_on_page_writeback(page, DATA, true, true);
415
416 set_page_dirty(page);
417 if (clear_page_dirty_for_io(page)) {
418 inode_dec_dirty_pages(inode);
419 f2fs_remove_dirty_inode(inode);
420 }
421 retry:
422 fio.page = page;
423 fio.old_blkaddr = NULL_ADDR;
424 fio.encrypted_page = NULL;
425 fio.need_lock = LOCK_DONE;
426 err = f2fs_do_write_data_page(&fio);
427 if (err) {
428 if (err == -ENOMEM) {
429 congestion_wait(BLK_RW_ASYNC,
430 DEFAULT_IO_TIMEOUT);
431 cond_resched();
432 goto retry;
433 }
434 unlock_page(page);
435 break;
436 }
437 /* record old blkaddr for revoking */
438 cur->old_addr = fio.old_blkaddr;
439 submit_bio = true;
440 }
441 unlock_page(page);
442 list_move_tail(&cur->list, &revoke_list);
443 }
444
445 if (submit_bio)
446 f2fs_submit_merged_write_cond(sbi, inode, NULL, 0, DATA);
447
448 if (err) {
449 /*
450 * try to revoke all committed pages, but still we could fail
451 * due to no memory or other reason, if that happened, EAGAIN
452 * will be returned, which means in such case, transaction is
453 * already not integrity, caller should use journal to do the
454 * recovery or rewrite & commit last transaction. For other
455 * error number, revoking was done by filesystem itself.
456 */
457 err = __revoke_inmem_pages(inode, &revoke_list,
458 false, true, false);
459
460 /* drop all uncommitted pages */
461 __revoke_inmem_pages(inode, &fi->inmem_pages,
462 true, false, false);
463 } else {
464 __revoke_inmem_pages(inode, &revoke_list,
465 false, false, false);
466 }
467
468 return err;
469 }
470
f2fs_commit_inmem_pages(struct inode * inode)471 int f2fs_commit_inmem_pages(struct inode *inode)
472 {
473 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
474 struct f2fs_inode_info *fi = F2FS_I(inode);
475 int err;
476
477 f2fs_balance_fs(sbi, true);
478
479 f2fs_down_write(&fi->i_gc_rwsem[WRITE]);
480
481 f2fs_lock_op(sbi);
482 set_inode_flag(inode, FI_ATOMIC_COMMIT);
483
484 mutex_lock(&fi->inmem_lock);
485 err = __f2fs_commit_inmem_pages(inode);
486 mutex_unlock(&fi->inmem_lock);
487
488 clear_inode_flag(inode, FI_ATOMIC_COMMIT);
489
490 f2fs_unlock_op(sbi);
491 f2fs_up_write(&fi->i_gc_rwsem[WRITE]);
492
493 return err;
494 }
495
496 /*
497 * This function balances dirty node and dentry pages.
498 * In addition, it controls garbage collection.
499 */
f2fs_balance_fs(struct f2fs_sb_info * sbi,bool need)500 void f2fs_balance_fs(struct f2fs_sb_info *sbi, bool need)
501 {
502 if (time_to_inject(sbi, FAULT_CHECKPOINT)) {
503 f2fs_show_injection_info(sbi, FAULT_CHECKPOINT);
504 f2fs_stop_checkpoint(sbi, false, STOP_CP_REASON_FAULT_INJECT);
505 }
506
507 /* balance_fs_bg is able to be pending */
508 if (need && excess_cached_nats(sbi))
509 f2fs_balance_fs_bg(sbi, false);
510
511 if (!f2fs_is_checkpoint_ready(sbi))
512 return;
513
514 /*
515 * We should do GC or end up with checkpoint, if there are so many dirty
516 * dir/node pages without enough free segments.
517 */
518 if (has_not_enough_free_secs(sbi, 0, 0)) {
519 if (test_opt(sbi, GC_MERGE) && sbi->gc_thread &&
520 sbi->gc_thread->f2fs_gc_task) {
521 DEFINE_WAIT(wait);
522
523 prepare_to_wait(&sbi->gc_thread->fggc_wq, &wait,
524 TASK_UNINTERRUPTIBLE);
525 wake_up(&sbi->gc_thread->gc_wait_queue_head);
526 io_schedule();
527 finish_wait(&sbi->gc_thread->fggc_wq, &wait);
528 } else {
529 f2fs_down_write(&sbi->gc_lock);
530 f2fs_gc(sbi, false, false, false, NULL_SEGNO);
531 }
532 }
533 }
534
excess_dirty_threshold(struct f2fs_sb_info * sbi)535 static inline bool excess_dirty_threshold(struct f2fs_sb_info *sbi)
536 {
537 int factor = f2fs_rwsem_is_locked(&sbi->cp_rwsem) ? 3 : 2;
538 unsigned int dents = get_pages(sbi, F2FS_DIRTY_DENTS);
539 unsigned int qdata = get_pages(sbi, F2FS_DIRTY_QDATA);
540 unsigned int nodes = get_pages(sbi, F2FS_DIRTY_NODES);
541 unsigned int meta = get_pages(sbi, F2FS_DIRTY_META);
542 unsigned int imeta = get_pages(sbi, F2FS_DIRTY_IMETA);
543 unsigned int threshold = sbi->blocks_per_seg * factor *
544 DEFAULT_DIRTY_THRESHOLD;
545 unsigned int global_threshold = threshold * 3 / 2;
546
547 if (dents >= threshold || qdata >= threshold ||
548 nodes >= threshold || meta >= threshold ||
549 imeta >= threshold)
550 return true;
551 return dents + qdata + nodes + meta + imeta > global_threshold;
552 }
553
f2fs_balance_fs_bg(struct f2fs_sb_info * sbi,bool from_bg)554 void f2fs_balance_fs_bg(struct f2fs_sb_info *sbi, bool from_bg)
555 {
556 if (unlikely(is_sbi_flag_set(sbi, SBI_POR_DOING)))
557 return;
558
559 /* try to shrink extent cache when there is no enough memory */
560 if (!f2fs_available_free_memory(sbi, EXTENT_CACHE))
561 f2fs_shrink_extent_tree(sbi, EXTENT_CACHE_SHRINK_NUMBER);
562
563 /* check the # of cached NAT entries */
564 if (!f2fs_available_free_memory(sbi, NAT_ENTRIES))
565 f2fs_try_to_free_nats(sbi, NAT_ENTRY_PER_BLOCK);
566
567 if (!f2fs_available_free_memory(sbi, FREE_NIDS))
568 f2fs_try_to_free_nids(sbi, MAX_FREE_NIDS);
569 else
570 f2fs_build_free_nids(sbi, false, false);
571
572 if (excess_dirty_nats(sbi) || excess_dirty_threshold(sbi) ||
573 excess_prefree_segs(sbi) || !f2fs_space_for_roll_forward(sbi))
574 goto do_sync;
575
576 /* there is background inflight IO or foreground operation recently */
577 if (is_inflight_io(sbi, REQ_TIME) ||
578 (!f2fs_time_over(sbi, REQ_TIME) && f2fs_rwsem_is_locked(&sbi->cp_rwsem)))
579 return;
580
581 /* exceed periodical checkpoint timeout threshold */
582 if (f2fs_time_over(sbi, CP_TIME))
583 goto do_sync;
584
585 /* checkpoint is the only way to shrink partial cached entries */
586 if (f2fs_available_free_memory(sbi, NAT_ENTRIES) &&
587 f2fs_available_free_memory(sbi, INO_ENTRIES))
588 return;
589
590 do_sync:
591 if (test_opt(sbi, DATA_FLUSH) && from_bg) {
592 struct blk_plug plug;
593
594 mutex_lock(&sbi->flush_lock);
595
596 blk_start_plug(&plug);
597 f2fs_sync_dirty_inodes(sbi, FILE_INODE, false);
598 blk_finish_plug(&plug);
599
600 mutex_unlock(&sbi->flush_lock);
601 }
602 f2fs_sync_fs(sbi->sb, true);
603 stat_inc_bg_cp_count(sbi->stat_info);
604 }
605
__submit_flush_wait(struct f2fs_sb_info * sbi,struct block_device * bdev)606 static int __submit_flush_wait(struct f2fs_sb_info *sbi,
607 struct block_device *bdev)
608 {
609 int ret = blkdev_issue_flush(bdev, GFP_NOFS);
610
611 trace_f2fs_issue_flush(bdev, test_opt(sbi, NOBARRIER),
612 test_opt(sbi, FLUSH_MERGE), ret);
613 return ret;
614 }
615
submit_flush_wait(struct f2fs_sb_info * sbi,nid_t ino)616 static int submit_flush_wait(struct f2fs_sb_info *sbi, nid_t ino)
617 {
618 int ret = 0;
619 int i;
620
621 if (!f2fs_is_multi_device(sbi))
622 return __submit_flush_wait(sbi, sbi->sb->s_bdev);
623
624 for (i = 0; i < sbi->s_ndevs; i++) {
625 if (!f2fs_is_dirty_device(sbi, ino, i, FLUSH_INO))
626 continue;
627 ret = __submit_flush_wait(sbi, FDEV(i).bdev);
628 if (ret)
629 break;
630 }
631 return ret;
632 }
633
issue_flush_thread(void * data)634 static int issue_flush_thread(void *data)
635 {
636 struct f2fs_sb_info *sbi = data;
637 struct flush_cmd_control *fcc = SM_I(sbi)->fcc_info;
638 wait_queue_head_t *q = &fcc->flush_wait_queue;
639 repeat:
640 if (kthread_should_stop())
641 return 0;
642
643 if (!llist_empty(&fcc->issue_list)) {
644 struct flush_cmd *cmd, *next;
645 int ret;
646
647 fcc->dispatch_list = llist_del_all(&fcc->issue_list);
648 fcc->dispatch_list = llist_reverse_order(fcc->dispatch_list);
649
650 cmd = llist_entry(fcc->dispatch_list, struct flush_cmd, llnode);
651
652 ret = submit_flush_wait(sbi, cmd->ino);
653 atomic_inc(&fcc->issued_flush);
654
655 llist_for_each_entry_safe(cmd, next,
656 fcc->dispatch_list, llnode) {
657 cmd->ret = ret;
658 complete(&cmd->wait);
659 }
660 fcc->dispatch_list = NULL;
661 }
662
663 wait_event_interruptible(*q,
664 kthread_should_stop() || !llist_empty(&fcc->issue_list));
665 goto repeat;
666 }
667
f2fs_issue_flush(struct f2fs_sb_info * sbi,nid_t ino)668 int f2fs_issue_flush(struct f2fs_sb_info *sbi, nid_t ino)
669 {
670 struct flush_cmd_control *fcc = SM_I(sbi)->fcc_info;
671 struct flush_cmd cmd;
672 int ret;
673
674 if (test_opt(sbi, NOBARRIER))
675 return 0;
676
677 if (!test_opt(sbi, FLUSH_MERGE)) {
678 atomic_inc(&fcc->queued_flush);
679 ret = submit_flush_wait(sbi, ino);
680 atomic_dec(&fcc->queued_flush);
681 atomic_inc(&fcc->issued_flush);
682 return ret;
683 }
684
685 if (atomic_inc_return(&fcc->queued_flush) == 1 ||
686 f2fs_is_multi_device(sbi)) {
687 ret = submit_flush_wait(sbi, ino);
688 atomic_dec(&fcc->queued_flush);
689
690 atomic_inc(&fcc->issued_flush);
691 return ret;
692 }
693
694 cmd.ino = ino;
695 init_completion(&cmd.wait);
696
697 llist_add(&cmd.llnode, &fcc->issue_list);
698
699 /*
700 * update issue_list before we wake up issue_flush thread, this
701 * smp_mb() pairs with another barrier in ___wait_event(), see
702 * more details in comments of waitqueue_active().
703 */
704 smp_mb();
705
706 if (waitqueue_active(&fcc->flush_wait_queue))
707 wake_up(&fcc->flush_wait_queue);
708
709 if (fcc->f2fs_issue_flush) {
710 wait_for_completion(&cmd.wait);
711 atomic_dec(&fcc->queued_flush);
712 } else {
713 struct llist_node *list;
714
715 list = llist_del_all(&fcc->issue_list);
716 if (!list) {
717 wait_for_completion(&cmd.wait);
718 atomic_dec(&fcc->queued_flush);
719 } else {
720 struct flush_cmd *tmp, *next;
721
722 ret = submit_flush_wait(sbi, ino);
723
724 llist_for_each_entry_safe(tmp, next, list, llnode) {
725 if (tmp == &cmd) {
726 cmd.ret = ret;
727 atomic_dec(&fcc->queued_flush);
728 continue;
729 }
730 tmp->ret = ret;
731 complete(&tmp->wait);
732 }
733 }
734 }
735
736 return cmd.ret;
737 }
738
f2fs_create_flush_cmd_control(struct f2fs_sb_info * sbi)739 int f2fs_create_flush_cmd_control(struct f2fs_sb_info *sbi)
740 {
741 dev_t dev = sbi->sb->s_bdev->bd_dev;
742 struct flush_cmd_control *fcc;
743 int err = 0;
744
745 if (SM_I(sbi)->fcc_info) {
746 fcc = SM_I(sbi)->fcc_info;
747 if (fcc->f2fs_issue_flush)
748 return err;
749 goto init_thread;
750 }
751
752 fcc = f2fs_kzalloc(sbi, sizeof(struct flush_cmd_control), GFP_KERNEL);
753 if (!fcc)
754 return -ENOMEM;
755 atomic_set(&fcc->issued_flush, 0);
756 atomic_set(&fcc->queued_flush, 0);
757 init_waitqueue_head(&fcc->flush_wait_queue);
758 init_llist_head(&fcc->issue_list);
759 SM_I(sbi)->fcc_info = fcc;
760 if (!test_opt(sbi, FLUSH_MERGE))
761 return err;
762
763 init_thread:
764 fcc->f2fs_issue_flush = kthread_run(issue_flush_thread, sbi,
765 "f2fs_flush-%u:%u", MAJOR(dev), MINOR(dev));
766 if (IS_ERR(fcc->f2fs_issue_flush)) {
767 err = PTR_ERR(fcc->f2fs_issue_flush);
768 kfree(fcc);
769 SM_I(sbi)->fcc_info = NULL;
770 return err;
771 }
772
773 return err;
774 }
775
f2fs_destroy_flush_cmd_control(struct f2fs_sb_info * sbi,bool free)776 void f2fs_destroy_flush_cmd_control(struct f2fs_sb_info *sbi, bool free)
777 {
778 struct flush_cmd_control *fcc = SM_I(sbi)->fcc_info;
779
780 if (fcc && fcc->f2fs_issue_flush) {
781 struct task_struct *flush_thread = fcc->f2fs_issue_flush;
782
783 fcc->f2fs_issue_flush = NULL;
784 kthread_stop(flush_thread);
785 }
786 if (free) {
787 kfree(fcc);
788 SM_I(sbi)->fcc_info = NULL;
789 }
790 }
791
f2fs_flush_device_cache(struct f2fs_sb_info * sbi)792 int f2fs_flush_device_cache(struct f2fs_sb_info *sbi)
793 {
794 int ret = 0, i;
795
796 if (!f2fs_is_multi_device(sbi))
797 return 0;
798
799 if (test_opt(sbi, NOBARRIER))
800 return 0;
801
802 for (i = 1; i < sbi->s_ndevs; i++) {
803 int count = DEFAULT_RETRY_IO_COUNT;
804
805 if (!f2fs_test_bit(i, (char *)&sbi->dirty_device))
806 continue;
807
808 do {
809 ret = __submit_flush_wait(sbi, FDEV(i).bdev);
810 if (ret)
811 f2fs_io_schedule_timeout(DEFAULT_IO_TIMEOUT);
812 } while (ret && --count);
813
814 if (ret) {
815 f2fs_stop_checkpoint(sbi, false,
816 STOP_CP_REASON_FLUSH_FAIL);
817 break;
818 }
819
820 spin_lock(&sbi->dev_lock);
821 f2fs_clear_bit(i, (char *)&sbi->dirty_device);
822 spin_unlock(&sbi->dev_lock);
823 }
824
825 return ret;
826 }
827
__locate_dirty_segment(struct f2fs_sb_info * sbi,unsigned int segno,enum dirty_type dirty_type)828 static void __locate_dirty_segment(struct f2fs_sb_info *sbi, unsigned int segno,
829 enum dirty_type dirty_type)
830 {
831 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
832
833 /* need not be added */
834 if (IS_CURSEG(sbi, segno))
835 return;
836
837 if (!test_and_set_bit(segno, dirty_i->dirty_segmap[dirty_type]))
838 dirty_i->nr_dirty[dirty_type]++;
839
840 if (dirty_type == DIRTY) {
841 struct seg_entry *sentry = get_seg_entry(sbi, segno);
842 enum dirty_type t = sentry->type;
843
844 if (unlikely(t >= DIRTY)) {
845 f2fs_bug_on(sbi, 1);
846 return;
847 }
848 if (!test_and_set_bit(segno, dirty_i->dirty_segmap[t]))
849 dirty_i->nr_dirty[t]++;
850
851 if (__is_large_section(sbi)) {
852 unsigned int secno = GET_SEC_FROM_SEG(sbi, segno);
853 block_t valid_blocks =
854 get_valid_blocks(sbi, segno, true);
855
856 f2fs_bug_on(sbi, unlikely(!valid_blocks ||
857 valid_blocks == BLKS_PER_SEC(sbi)));
858
859 if (!IS_CURSEC(sbi, secno))
860 set_bit(secno, dirty_i->dirty_secmap);
861 }
862 }
863 }
864
__remove_dirty_segment(struct f2fs_sb_info * sbi,unsigned int segno,enum dirty_type dirty_type)865 static void __remove_dirty_segment(struct f2fs_sb_info *sbi, unsigned int segno,
866 enum dirty_type dirty_type)
867 {
868 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
869 block_t valid_blocks;
870
871 if (test_and_clear_bit(segno, dirty_i->dirty_segmap[dirty_type]))
872 dirty_i->nr_dirty[dirty_type]--;
873
874 if (dirty_type == DIRTY) {
875 struct seg_entry *sentry = get_seg_entry(sbi, segno);
876 enum dirty_type t = sentry->type;
877
878 if (test_and_clear_bit(segno, dirty_i->dirty_segmap[t]))
879 dirty_i->nr_dirty[t]--;
880
881 valid_blocks = get_valid_blocks(sbi, segno, true);
882 if (valid_blocks == 0) {
883 clear_bit(GET_SEC_FROM_SEG(sbi, segno),
884 dirty_i->victim_secmap);
885 #ifdef CONFIG_F2FS_CHECK_FS
886 clear_bit(segno, SIT_I(sbi)->invalid_segmap);
887 #endif
888 }
889 if (__is_large_section(sbi)) {
890 unsigned int secno = GET_SEC_FROM_SEG(sbi, segno);
891
892 if (!valid_blocks ||
893 valid_blocks == BLKS_PER_SEC(sbi)) {
894 clear_bit(secno, dirty_i->dirty_secmap);
895 return;
896 }
897
898 if (!IS_CURSEC(sbi, secno))
899 set_bit(secno, dirty_i->dirty_secmap);
900 }
901 }
902 }
903
904 /*
905 * Should not occur error such as -ENOMEM.
906 * Adding dirty entry into seglist is not critical operation.
907 * If a given segment is one of current working segments, it won't be added.
908 */
locate_dirty_segment(struct f2fs_sb_info * sbi,unsigned int segno)909 static void locate_dirty_segment(struct f2fs_sb_info *sbi, unsigned int segno)
910 {
911 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
912 unsigned short valid_blocks, ckpt_valid_blocks;
913 unsigned int usable_blocks;
914
915 if (segno == NULL_SEGNO || IS_CURSEG(sbi, segno))
916 return;
917
918 usable_blocks = f2fs_usable_blks_in_seg(sbi, segno);
919 mutex_lock(&dirty_i->seglist_lock);
920
921 valid_blocks = get_valid_blocks(sbi, segno, false);
922 ckpt_valid_blocks = get_ckpt_valid_blocks(sbi, segno, false);
923
924 if (valid_blocks == 0 && (!is_sbi_flag_set(sbi, SBI_CP_DISABLED) ||
925 ckpt_valid_blocks == usable_blocks)) {
926 __locate_dirty_segment(sbi, segno, PRE);
927 __remove_dirty_segment(sbi, segno, DIRTY);
928 } else if (valid_blocks < usable_blocks) {
929 __locate_dirty_segment(sbi, segno, DIRTY);
930 } else {
931 /* Recovery routine with SSR needs this */
932 __remove_dirty_segment(sbi, segno, DIRTY);
933 }
934
935 mutex_unlock(&dirty_i->seglist_lock);
936 }
937
938 /* This moves currently empty dirty blocks to prefree. Must hold seglist_lock */
f2fs_dirty_to_prefree(struct f2fs_sb_info * sbi)939 void f2fs_dirty_to_prefree(struct f2fs_sb_info *sbi)
940 {
941 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
942 unsigned int segno;
943
944 mutex_lock(&dirty_i->seglist_lock);
945 for_each_set_bit(segno, dirty_i->dirty_segmap[DIRTY], MAIN_SEGS(sbi)) {
946 if (get_valid_blocks(sbi, segno, false))
947 continue;
948 if (IS_CURSEG(sbi, segno))
949 continue;
950 __locate_dirty_segment(sbi, segno, PRE);
951 __remove_dirty_segment(sbi, segno, DIRTY);
952 }
953 mutex_unlock(&dirty_i->seglist_lock);
954 }
955
f2fs_get_unusable_blocks(struct f2fs_sb_info * sbi)956 block_t f2fs_get_unusable_blocks(struct f2fs_sb_info *sbi)
957 {
958 int ovp_hole_segs =
959 (overprovision_segments(sbi) - reserved_segments(sbi));
960 block_t ovp_holes = ovp_hole_segs << sbi->log_blocks_per_seg;
961 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
962 block_t holes[2] = {0, 0}; /* DATA and NODE */
963 block_t unusable;
964 struct seg_entry *se;
965 unsigned int segno;
966
967 mutex_lock(&dirty_i->seglist_lock);
968 for_each_set_bit(segno, dirty_i->dirty_segmap[DIRTY], MAIN_SEGS(sbi)) {
969 se = get_seg_entry(sbi, segno);
970 if (IS_NODESEG(se->type))
971 holes[NODE] += f2fs_usable_blks_in_seg(sbi, segno) -
972 se->valid_blocks;
973 else
974 holes[DATA] += f2fs_usable_blks_in_seg(sbi, segno) -
975 se->valid_blocks;
976 }
977 mutex_unlock(&dirty_i->seglist_lock);
978
979 unusable = holes[DATA] > holes[NODE] ? holes[DATA] : holes[NODE];
980 if (unusable > ovp_holes)
981 return unusable - ovp_holes;
982 return 0;
983 }
984
f2fs_disable_cp_again(struct f2fs_sb_info * sbi,block_t unusable)985 int f2fs_disable_cp_again(struct f2fs_sb_info *sbi, block_t unusable)
986 {
987 int ovp_hole_segs =
988 (overprovision_segments(sbi) - reserved_segments(sbi));
989 if (unusable > F2FS_OPTION(sbi).unusable_cap)
990 return -EAGAIN;
991 if (is_sbi_flag_set(sbi, SBI_CP_DISABLED_QUICK) &&
992 dirty_segments(sbi) > ovp_hole_segs)
993 return -EAGAIN;
994 return 0;
995 }
996
997 /* This is only used by SBI_CP_DISABLED */
get_free_segment(struct f2fs_sb_info * sbi)998 static unsigned int get_free_segment(struct f2fs_sb_info *sbi)
999 {
1000 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
1001 unsigned int segno = 0;
1002
1003 mutex_lock(&dirty_i->seglist_lock);
1004 for_each_set_bit(segno, dirty_i->dirty_segmap[DIRTY], MAIN_SEGS(sbi)) {
1005 if (get_valid_blocks(sbi, segno, false))
1006 continue;
1007 if (get_ckpt_valid_blocks(sbi, segno, false))
1008 continue;
1009 mutex_unlock(&dirty_i->seglist_lock);
1010 return segno;
1011 }
1012 mutex_unlock(&dirty_i->seglist_lock);
1013 return NULL_SEGNO;
1014 }
1015
__create_discard_cmd(struct f2fs_sb_info * sbi,struct block_device * bdev,block_t lstart,block_t start,block_t len)1016 static struct discard_cmd *__create_discard_cmd(struct f2fs_sb_info *sbi,
1017 struct block_device *bdev, block_t lstart,
1018 block_t start, block_t len)
1019 {
1020 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1021 struct list_head *pend_list;
1022 struct discard_cmd *dc;
1023
1024 f2fs_bug_on(sbi, !len);
1025
1026 pend_list = &dcc->pend_list[plist_idx(len)];
1027
1028 dc = f2fs_kmem_cache_alloc(discard_cmd_slab, GFP_NOFS, true, NULL);
1029 INIT_LIST_HEAD(&dc->list);
1030 dc->bdev = bdev;
1031 dc->lstart = lstart;
1032 dc->start = start;
1033 dc->len = len;
1034 dc->ref = 0;
1035 dc->state = D_PREP;
1036 dc->queued = 0;
1037 dc->error = 0;
1038 init_completion(&dc->wait);
1039 list_add_tail(&dc->list, pend_list);
1040 spin_lock_init(&dc->lock);
1041 dc->bio_ref = 0;
1042 atomic_inc(&dcc->discard_cmd_cnt);
1043 dcc->undiscard_blks += len;
1044
1045 return dc;
1046 }
1047
__attach_discard_cmd(struct f2fs_sb_info * sbi,struct block_device * bdev,block_t lstart,block_t start,block_t len,struct rb_node * parent,struct rb_node ** p,bool leftmost)1048 static struct discard_cmd *__attach_discard_cmd(struct f2fs_sb_info *sbi,
1049 struct block_device *bdev, block_t lstart,
1050 block_t start, block_t len,
1051 struct rb_node *parent, struct rb_node **p,
1052 bool leftmost)
1053 {
1054 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1055 struct discard_cmd *dc;
1056
1057 dc = __create_discard_cmd(sbi, bdev, lstart, start, len);
1058
1059 rb_link_node(&dc->rb_node, parent, p);
1060 rb_insert_color_cached(&dc->rb_node, &dcc->root, leftmost);
1061
1062 return dc;
1063 }
1064
__detach_discard_cmd(struct discard_cmd_control * dcc,struct discard_cmd * dc)1065 static void __detach_discard_cmd(struct discard_cmd_control *dcc,
1066 struct discard_cmd *dc)
1067 {
1068 if (dc->state == D_DONE)
1069 atomic_sub(dc->queued, &dcc->queued_discard);
1070
1071 list_del(&dc->list);
1072 rb_erase_cached(&dc->rb_node, &dcc->root);
1073 dcc->undiscard_blks -= dc->len;
1074
1075 kmem_cache_free(discard_cmd_slab, dc);
1076
1077 atomic_dec(&dcc->discard_cmd_cnt);
1078 }
1079
__remove_discard_cmd(struct f2fs_sb_info * sbi,struct discard_cmd * dc)1080 static void __remove_discard_cmd(struct f2fs_sb_info *sbi,
1081 struct discard_cmd *dc)
1082 {
1083 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1084 unsigned long flags;
1085
1086 trace_f2fs_remove_discard(dc->bdev, dc->start, dc->len);
1087
1088 spin_lock_irqsave(&dc->lock, flags);
1089 if (dc->bio_ref) {
1090 spin_unlock_irqrestore(&dc->lock, flags);
1091 return;
1092 }
1093 spin_unlock_irqrestore(&dc->lock, flags);
1094
1095 f2fs_bug_on(sbi, dc->ref);
1096
1097 if (dc->error == -EOPNOTSUPP)
1098 dc->error = 0;
1099
1100 if (dc->error)
1101 printk_ratelimited(
1102 "%sF2FS-fs (%s): Issue discard(%u, %u, %u) failed, ret: %d",
1103 KERN_INFO, sbi->sb->s_id,
1104 dc->lstart, dc->start, dc->len, dc->error);
1105 __detach_discard_cmd(dcc, dc);
1106 }
1107
f2fs_submit_discard_endio(struct bio * bio)1108 static void f2fs_submit_discard_endio(struct bio *bio)
1109 {
1110 struct discard_cmd *dc = (struct discard_cmd *)bio->bi_private;
1111 unsigned long flags;
1112
1113 spin_lock_irqsave(&dc->lock, flags);
1114 if (!dc->error)
1115 dc->error = blk_status_to_errno(bio->bi_status);
1116 dc->bio_ref--;
1117 if (!dc->bio_ref && dc->state == D_SUBMIT) {
1118 dc->state = D_DONE;
1119 complete_all(&dc->wait);
1120 }
1121 spin_unlock_irqrestore(&dc->lock, flags);
1122 bio_put(bio);
1123 }
1124
__check_sit_bitmap(struct f2fs_sb_info * sbi,block_t start,block_t end)1125 static void __check_sit_bitmap(struct f2fs_sb_info *sbi,
1126 block_t start, block_t end)
1127 {
1128 #ifdef CONFIG_F2FS_CHECK_FS
1129 struct seg_entry *sentry;
1130 unsigned int segno;
1131 block_t blk = start;
1132 unsigned long offset, size, max_blocks = sbi->blocks_per_seg;
1133 unsigned long *map;
1134
1135 while (blk < end) {
1136 segno = GET_SEGNO(sbi, blk);
1137 sentry = get_seg_entry(sbi, segno);
1138 offset = GET_BLKOFF_FROM_SEG0(sbi, blk);
1139
1140 if (end < START_BLOCK(sbi, segno + 1))
1141 size = GET_BLKOFF_FROM_SEG0(sbi, end);
1142 else
1143 size = max_blocks;
1144 map = (unsigned long *)(sentry->cur_valid_map);
1145 offset = __find_rev_next_bit(map, size, offset);
1146 f2fs_bug_on(sbi, offset != size);
1147 blk = START_BLOCK(sbi, segno + 1);
1148 }
1149 #endif
1150 }
1151
__init_discard_policy(struct f2fs_sb_info * sbi,struct discard_policy * dpolicy,int discard_type,unsigned int granularity)1152 static void __init_discard_policy(struct f2fs_sb_info *sbi,
1153 struct discard_policy *dpolicy,
1154 int discard_type, unsigned int granularity)
1155 {
1156 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1157
1158 /* common policy */
1159 dpolicy->type = discard_type;
1160 dpolicy->sync = true;
1161 dpolicy->ordered = false;
1162 dpolicy->granularity = granularity;
1163
1164 dpolicy->max_requests = dcc->max_discard_request;
1165 dpolicy->io_aware_gran = MAX_PLIST_NUM;
1166 dpolicy->timeout = false;
1167
1168 if (discard_type == DPOLICY_BG) {
1169 dpolicy->min_interval = dcc->min_discard_issue_time;
1170 dpolicy->mid_interval = dcc->mid_discard_issue_time;
1171 dpolicy->max_interval = dcc->max_discard_issue_time;
1172 dpolicy->io_aware = true;
1173 dpolicy->sync = false;
1174 dpolicy->ordered = true;
1175 if (utilization(sbi) > DEF_DISCARD_URGENT_UTIL) {
1176 dpolicy->granularity = 1;
1177 if (atomic_read(&dcc->discard_cmd_cnt))
1178 dpolicy->max_interval =
1179 dcc->min_discard_issue_time;
1180 }
1181 } else if (discard_type == DPOLICY_FORCE) {
1182 dpolicy->min_interval = dcc->min_discard_issue_time;
1183 dpolicy->mid_interval = dcc->mid_discard_issue_time;
1184 dpolicy->max_interval = dcc->max_discard_issue_time;
1185 dpolicy->io_aware = false;
1186 } else if (discard_type == DPOLICY_FSTRIM) {
1187 dpolicy->io_aware = false;
1188 } else if (discard_type == DPOLICY_UMOUNT) {
1189 dpolicy->io_aware = false;
1190 /* we need to issue all to keep CP_TRIMMED_FLAG */
1191 dpolicy->granularity = 1;
1192 dpolicy->timeout = true;
1193 }
1194 }
1195
1196 static void __update_discard_tree_range(struct f2fs_sb_info *sbi,
1197 struct block_device *bdev, block_t lstart,
1198 block_t start, block_t len);
1199 /* this function is copied from blkdev_issue_discard from block/blk-lib.c */
__submit_discard_cmd(struct f2fs_sb_info * sbi,struct discard_policy * dpolicy,struct discard_cmd * dc,unsigned int * issued)1200 static int __submit_discard_cmd(struct f2fs_sb_info *sbi,
1201 struct discard_policy *dpolicy,
1202 struct discard_cmd *dc,
1203 unsigned int *issued)
1204 {
1205 struct block_device *bdev = dc->bdev;
1206 struct request_queue *q = bdev_get_queue(bdev);
1207 unsigned int max_discard_blocks =
1208 SECTOR_TO_BLOCK(q->limits.max_discard_sectors);
1209 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1210 struct list_head *wait_list = (dpolicy->type == DPOLICY_FSTRIM) ?
1211 &(dcc->fstrim_list) : &(dcc->wait_list);
1212 int flag = dpolicy->sync ? REQ_SYNC : 0;
1213 block_t lstart, start, len, total_len;
1214 int err = 0;
1215
1216 if (dc->state != D_PREP)
1217 return 0;
1218
1219 if (is_sbi_flag_set(sbi, SBI_NEED_FSCK))
1220 return 0;
1221
1222 trace_f2fs_issue_discard(bdev, dc->start, dc->len);
1223
1224 lstart = dc->lstart;
1225 start = dc->start;
1226 len = dc->len;
1227 total_len = len;
1228
1229 dc->len = 0;
1230
1231 while (total_len && *issued < dpolicy->max_requests && !err) {
1232 struct bio *bio = NULL;
1233 unsigned long flags;
1234 bool last = true;
1235
1236 if (len > max_discard_blocks) {
1237 len = max_discard_blocks;
1238 last = false;
1239 }
1240
1241 (*issued)++;
1242 if (*issued == dpolicy->max_requests)
1243 last = true;
1244
1245 dc->len += len;
1246
1247 if (time_to_inject(sbi, FAULT_DISCARD)) {
1248 f2fs_show_injection_info(sbi, FAULT_DISCARD);
1249 err = -EIO;
1250 goto submit;
1251 }
1252 err = __blkdev_issue_discard(bdev,
1253 SECTOR_FROM_BLOCK(start),
1254 SECTOR_FROM_BLOCK(len),
1255 GFP_NOFS, 0, &bio);
1256 submit:
1257 if (err) {
1258 spin_lock_irqsave(&dc->lock, flags);
1259 if (dc->state == D_PARTIAL)
1260 dc->state = D_SUBMIT;
1261 spin_unlock_irqrestore(&dc->lock, flags);
1262
1263 break;
1264 }
1265
1266 f2fs_bug_on(sbi, !bio);
1267
1268 /*
1269 * should keep before submission to avoid D_DONE
1270 * right away
1271 */
1272 spin_lock_irqsave(&dc->lock, flags);
1273 if (last)
1274 dc->state = D_SUBMIT;
1275 else
1276 dc->state = D_PARTIAL;
1277 dc->bio_ref++;
1278 spin_unlock_irqrestore(&dc->lock, flags);
1279
1280 atomic_inc(&dcc->queued_discard);
1281 dc->queued++;
1282 list_move_tail(&dc->list, wait_list);
1283
1284 /* sanity check on discard range */
1285 __check_sit_bitmap(sbi, lstart, lstart + len);
1286
1287 bio->bi_private = dc;
1288 bio->bi_end_io = f2fs_submit_discard_endio;
1289 bio->bi_opf |= flag;
1290 submit_bio(bio);
1291
1292 atomic_inc(&dcc->issued_discard);
1293
1294 f2fs_update_iostat(sbi, FS_DISCARD, 1);
1295
1296 lstart += len;
1297 start += len;
1298 total_len -= len;
1299 len = total_len;
1300 }
1301
1302 if (!err && len) {
1303 dcc->undiscard_blks -= len;
1304 __update_discard_tree_range(sbi, bdev, lstart, start, len);
1305 }
1306 return err;
1307 }
1308
__insert_discard_tree(struct f2fs_sb_info * sbi,struct block_device * bdev,block_t lstart,block_t start,block_t len,struct rb_node ** insert_p,struct rb_node * insert_parent)1309 static void __insert_discard_tree(struct f2fs_sb_info *sbi,
1310 struct block_device *bdev, block_t lstart,
1311 block_t start, block_t len,
1312 struct rb_node **insert_p,
1313 struct rb_node *insert_parent)
1314 {
1315 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1316 struct rb_node **p;
1317 struct rb_node *parent = NULL;
1318 bool leftmost = true;
1319
1320 if (insert_p && insert_parent) {
1321 parent = insert_parent;
1322 p = insert_p;
1323 goto do_insert;
1324 }
1325
1326 p = f2fs_lookup_rb_tree_for_insert(sbi, &dcc->root, &parent,
1327 lstart, &leftmost);
1328 do_insert:
1329 __attach_discard_cmd(sbi, bdev, lstart, start, len, parent,
1330 p, leftmost);
1331 }
1332
__relocate_discard_cmd(struct discard_cmd_control * dcc,struct discard_cmd * dc)1333 static void __relocate_discard_cmd(struct discard_cmd_control *dcc,
1334 struct discard_cmd *dc)
1335 {
1336 list_move_tail(&dc->list, &dcc->pend_list[plist_idx(dc->len)]);
1337 }
1338
__punch_discard_cmd(struct f2fs_sb_info * sbi,struct discard_cmd * dc,block_t blkaddr)1339 static void __punch_discard_cmd(struct f2fs_sb_info *sbi,
1340 struct discard_cmd *dc, block_t blkaddr)
1341 {
1342 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1343 struct discard_info di = dc->di;
1344 bool modified = false;
1345
1346 if (dc->state == D_DONE || dc->len == 1) {
1347 __remove_discard_cmd(sbi, dc);
1348 return;
1349 }
1350
1351 dcc->undiscard_blks -= di.len;
1352
1353 if (blkaddr > di.lstart) {
1354 dc->len = blkaddr - dc->lstart;
1355 dcc->undiscard_blks += dc->len;
1356 __relocate_discard_cmd(dcc, dc);
1357 modified = true;
1358 }
1359
1360 if (blkaddr < di.lstart + di.len - 1) {
1361 if (modified) {
1362 __insert_discard_tree(sbi, dc->bdev, blkaddr + 1,
1363 di.start + blkaddr + 1 - di.lstart,
1364 di.lstart + di.len - 1 - blkaddr,
1365 NULL, NULL);
1366 } else {
1367 dc->lstart++;
1368 dc->len--;
1369 dc->start++;
1370 dcc->undiscard_blks += dc->len;
1371 __relocate_discard_cmd(dcc, dc);
1372 }
1373 }
1374 }
1375
__update_discard_tree_range(struct f2fs_sb_info * sbi,struct block_device * bdev,block_t lstart,block_t start,block_t len)1376 static void __update_discard_tree_range(struct f2fs_sb_info *sbi,
1377 struct block_device *bdev, block_t lstart,
1378 block_t start, block_t len)
1379 {
1380 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1381 struct discard_cmd *prev_dc = NULL, *next_dc = NULL;
1382 struct discard_cmd *dc;
1383 struct discard_info di = {0};
1384 struct rb_node **insert_p = NULL, *insert_parent = NULL;
1385 struct request_queue *q = bdev_get_queue(bdev);
1386 unsigned int max_discard_blocks =
1387 SECTOR_TO_BLOCK(q->limits.max_discard_sectors);
1388 block_t end = lstart + len;
1389
1390 dc = (struct discard_cmd *)f2fs_lookup_rb_tree_ret(&dcc->root,
1391 NULL, lstart,
1392 (struct rb_entry **)&prev_dc,
1393 (struct rb_entry **)&next_dc,
1394 &insert_p, &insert_parent, true, NULL);
1395 if (dc)
1396 prev_dc = dc;
1397
1398 if (!prev_dc) {
1399 di.lstart = lstart;
1400 di.len = next_dc ? next_dc->lstart - lstart : len;
1401 di.len = min(di.len, len);
1402 di.start = start;
1403 }
1404
1405 while (1) {
1406 struct rb_node *node;
1407 bool merged = false;
1408 struct discard_cmd *tdc = NULL;
1409
1410 if (prev_dc) {
1411 di.lstart = prev_dc->lstart + prev_dc->len;
1412 if (di.lstart < lstart)
1413 di.lstart = lstart;
1414 if (di.lstart >= end)
1415 break;
1416
1417 if (!next_dc || next_dc->lstart > end)
1418 di.len = end - di.lstart;
1419 else
1420 di.len = next_dc->lstart - di.lstart;
1421 di.start = start + di.lstart - lstart;
1422 }
1423
1424 if (!di.len)
1425 goto next;
1426
1427 if (prev_dc && prev_dc->state == D_PREP &&
1428 prev_dc->bdev == bdev &&
1429 __is_discard_back_mergeable(&di, &prev_dc->di,
1430 max_discard_blocks)) {
1431 prev_dc->di.len += di.len;
1432 dcc->undiscard_blks += di.len;
1433 __relocate_discard_cmd(dcc, prev_dc);
1434 di = prev_dc->di;
1435 tdc = prev_dc;
1436 merged = true;
1437 }
1438
1439 if (next_dc && next_dc->state == D_PREP &&
1440 next_dc->bdev == bdev &&
1441 __is_discard_front_mergeable(&di, &next_dc->di,
1442 max_discard_blocks)) {
1443 next_dc->di.lstart = di.lstart;
1444 next_dc->di.len += di.len;
1445 next_dc->di.start = di.start;
1446 dcc->undiscard_blks += di.len;
1447 __relocate_discard_cmd(dcc, next_dc);
1448 if (tdc)
1449 __remove_discard_cmd(sbi, tdc);
1450 merged = true;
1451 }
1452
1453 if (!merged) {
1454 __insert_discard_tree(sbi, bdev, di.lstart, di.start,
1455 di.len, NULL, NULL);
1456 }
1457 next:
1458 prev_dc = next_dc;
1459 if (!prev_dc)
1460 break;
1461
1462 node = rb_next(&prev_dc->rb_node);
1463 next_dc = rb_entry_safe(node, struct discard_cmd, rb_node);
1464 }
1465 }
1466
__queue_discard_cmd(struct f2fs_sb_info * sbi,struct block_device * bdev,block_t blkstart,block_t blklen)1467 static int __queue_discard_cmd(struct f2fs_sb_info *sbi,
1468 struct block_device *bdev, block_t blkstart, block_t blklen)
1469 {
1470 block_t lblkstart = blkstart;
1471
1472 if (!f2fs_bdev_support_discard(bdev))
1473 return 0;
1474
1475 trace_f2fs_queue_discard(bdev, blkstart, blklen);
1476
1477 if (f2fs_is_multi_device(sbi)) {
1478 int devi = f2fs_target_device_index(sbi, blkstart);
1479
1480 blkstart -= FDEV(devi).start_blk;
1481 }
1482 mutex_lock(&SM_I(sbi)->dcc_info->cmd_lock);
1483 __update_discard_tree_range(sbi, bdev, lblkstart, blkstart, blklen);
1484 mutex_unlock(&SM_I(sbi)->dcc_info->cmd_lock);
1485 return 0;
1486 }
1487
__issue_discard_cmd_orderly(struct f2fs_sb_info * sbi,struct discard_policy * dpolicy)1488 static unsigned int __issue_discard_cmd_orderly(struct f2fs_sb_info *sbi,
1489 struct discard_policy *dpolicy)
1490 {
1491 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1492 struct discard_cmd *prev_dc = NULL, *next_dc = NULL;
1493 struct rb_node **insert_p = NULL, *insert_parent = NULL;
1494 struct discard_cmd *dc;
1495 struct blk_plug plug;
1496 unsigned int pos = dcc->next_pos;
1497 unsigned int issued = 0;
1498 bool io_interrupted = false;
1499
1500 mutex_lock(&dcc->cmd_lock);
1501 dc = (struct discard_cmd *)f2fs_lookup_rb_tree_ret(&dcc->root,
1502 NULL, pos,
1503 (struct rb_entry **)&prev_dc,
1504 (struct rb_entry **)&next_dc,
1505 &insert_p, &insert_parent, true, NULL);
1506 if (!dc)
1507 dc = next_dc;
1508
1509 blk_start_plug(&plug);
1510
1511 while (dc) {
1512 struct rb_node *node;
1513 int err = 0;
1514
1515 if (dc->state != D_PREP)
1516 goto next;
1517
1518 if (dpolicy->io_aware && !is_idle(sbi, DISCARD_TIME)) {
1519 io_interrupted = true;
1520 break;
1521 }
1522
1523 dcc->next_pos = dc->lstart + dc->len;
1524 err = __submit_discard_cmd(sbi, dpolicy, dc, &issued);
1525
1526 if (issued >= dpolicy->max_requests)
1527 break;
1528 next:
1529 node = rb_next(&dc->rb_node);
1530 if (err)
1531 __remove_discard_cmd(sbi, dc);
1532 dc = rb_entry_safe(node, struct discard_cmd, rb_node);
1533 }
1534
1535 blk_finish_plug(&plug);
1536
1537 if (!dc)
1538 dcc->next_pos = 0;
1539
1540 mutex_unlock(&dcc->cmd_lock);
1541
1542 if (!issued && io_interrupted)
1543 issued = -1;
1544
1545 return issued;
1546 }
1547 static unsigned int __wait_all_discard_cmd(struct f2fs_sb_info *sbi,
1548 struct discard_policy *dpolicy);
1549
__issue_discard_cmd(struct f2fs_sb_info * sbi,struct discard_policy * dpolicy)1550 static int __issue_discard_cmd(struct f2fs_sb_info *sbi,
1551 struct discard_policy *dpolicy)
1552 {
1553 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1554 struct list_head *pend_list;
1555 struct discard_cmd *dc, *tmp;
1556 struct blk_plug plug;
1557 int i, issued;
1558 bool io_interrupted = false;
1559
1560 if (dpolicy->timeout)
1561 f2fs_update_time(sbi, UMOUNT_DISCARD_TIMEOUT);
1562
1563 retry:
1564 issued = 0;
1565 for (i = MAX_PLIST_NUM - 1; i >= 0; i--) {
1566 if (dpolicy->timeout &&
1567 f2fs_time_over(sbi, UMOUNT_DISCARD_TIMEOUT))
1568 break;
1569
1570 if (i + 1 < dpolicy->granularity)
1571 break;
1572
1573 if (i + 1 < DEFAULT_DISCARD_GRANULARITY && dpolicy->ordered)
1574 return __issue_discard_cmd_orderly(sbi, dpolicy);
1575
1576 pend_list = &dcc->pend_list[i];
1577
1578 mutex_lock(&dcc->cmd_lock);
1579 if (list_empty(pend_list))
1580 goto next;
1581 if (unlikely(dcc->rbtree_check))
1582 f2fs_bug_on(sbi, !f2fs_check_rb_tree_consistence(sbi,
1583 &dcc->root, false));
1584 blk_start_plug(&plug);
1585 list_for_each_entry_safe(dc, tmp, pend_list, list) {
1586 f2fs_bug_on(sbi, dc->state != D_PREP);
1587
1588 if (dpolicy->timeout &&
1589 f2fs_time_over(sbi, UMOUNT_DISCARD_TIMEOUT))
1590 break;
1591
1592 if (dpolicy->io_aware && i < dpolicy->io_aware_gran &&
1593 !is_idle(sbi, DISCARD_TIME)) {
1594 io_interrupted = true;
1595 break;
1596 }
1597
1598 __submit_discard_cmd(sbi, dpolicy, dc, &issued);
1599
1600 if (issued >= dpolicy->max_requests)
1601 break;
1602 }
1603 blk_finish_plug(&plug);
1604 next:
1605 mutex_unlock(&dcc->cmd_lock);
1606
1607 if (issued >= dpolicy->max_requests || io_interrupted)
1608 break;
1609 }
1610
1611 if (dpolicy->type == DPOLICY_UMOUNT && issued) {
1612 __wait_all_discard_cmd(sbi, dpolicy);
1613 goto retry;
1614 }
1615
1616 if (!issued && io_interrupted)
1617 issued = -1;
1618
1619 return issued;
1620 }
1621
__drop_discard_cmd(struct f2fs_sb_info * sbi)1622 static bool __drop_discard_cmd(struct f2fs_sb_info *sbi)
1623 {
1624 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1625 struct list_head *pend_list;
1626 struct discard_cmd *dc, *tmp;
1627 int i;
1628 bool dropped = false;
1629
1630 mutex_lock(&dcc->cmd_lock);
1631 for (i = MAX_PLIST_NUM - 1; i >= 0; i--) {
1632 pend_list = &dcc->pend_list[i];
1633 list_for_each_entry_safe(dc, tmp, pend_list, list) {
1634 f2fs_bug_on(sbi, dc->state != D_PREP);
1635 __remove_discard_cmd(sbi, dc);
1636 dropped = true;
1637 }
1638 }
1639 mutex_unlock(&dcc->cmd_lock);
1640
1641 return dropped;
1642 }
1643
f2fs_drop_discard_cmd(struct f2fs_sb_info * sbi)1644 void f2fs_drop_discard_cmd(struct f2fs_sb_info *sbi)
1645 {
1646 __drop_discard_cmd(sbi);
1647 }
1648
__wait_one_discard_bio(struct f2fs_sb_info * sbi,struct discard_cmd * dc)1649 static unsigned int __wait_one_discard_bio(struct f2fs_sb_info *sbi,
1650 struct discard_cmd *dc)
1651 {
1652 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1653 unsigned int len = 0;
1654
1655 wait_for_completion_io(&dc->wait);
1656 mutex_lock(&dcc->cmd_lock);
1657 f2fs_bug_on(sbi, dc->state != D_DONE);
1658 dc->ref--;
1659 if (!dc->ref) {
1660 if (!dc->error)
1661 len = dc->len;
1662 __remove_discard_cmd(sbi, dc);
1663 }
1664 mutex_unlock(&dcc->cmd_lock);
1665
1666 return len;
1667 }
1668
__wait_discard_cmd_range(struct f2fs_sb_info * sbi,struct discard_policy * dpolicy,block_t start,block_t end)1669 static unsigned int __wait_discard_cmd_range(struct f2fs_sb_info *sbi,
1670 struct discard_policy *dpolicy,
1671 block_t start, block_t end)
1672 {
1673 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1674 struct list_head *wait_list = (dpolicy->type == DPOLICY_FSTRIM) ?
1675 &(dcc->fstrim_list) : &(dcc->wait_list);
1676 struct discard_cmd *dc, *tmp;
1677 bool need_wait;
1678 unsigned int trimmed = 0;
1679
1680 next:
1681 need_wait = false;
1682
1683 mutex_lock(&dcc->cmd_lock);
1684 list_for_each_entry_safe(dc, tmp, wait_list, list) {
1685 if (dc->lstart + dc->len <= start || end <= dc->lstart)
1686 continue;
1687 if (dc->len < dpolicy->granularity)
1688 continue;
1689 if (dc->state == D_DONE && !dc->ref) {
1690 wait_for_completion_io(&dc->wait);
1691 if (!dc->error)
1692 trimmed += dc->len;
1693 __remove_discard_cmd(sbi, dc);
1694 } else {
1695 dc->ref++;
1696 need_wait = true;
1697 break;
1698 }
1699 }
1700 mutex_unlock(&dcc->cmd_lock);
1701
1702 if (need_wait) {
1703 trimmed += __wait_one_discard_bio(sbi, dc);
1704 goto next;
1705 }
1706
1707 return trimmed;
1708 }
1709
__wait_all_discard_cmd(struct f2fs_sb_info * sbi,struct discard_policy * dpolicy)1710 static unsigned int __wait_all_discard_cmd(struct f2fs_sb_info *sbi,
1711 struct discard_policy *dpolicy)
1712 {
1713 struct discard_policy dp;
1714 unsigned int discard_blks;
1715
1716 if (dpolicy)
1717 return __wait_discard_cmd_range(sbi, dpolicy, 0, UINT_MAX);
1718
1719 /* wait all */
1720 __init_discard_policy(sbi, &dp, DPOLICY_FSTRIM, 1);
1721 discard_blks = __wait_discard_cmd_range(sbi, &dp, 0, UINT_MAX);
1722 __init_discard_policy(sbi, &dp, DPOLICY_UMOUNT, 1);
1723 discard_blks += __wait_discard_cmd_range(sbi, &dp, 0, UINT_MAX);
1724
1725 return discard_blks;
1726 }
1727
1728 /* This should be covered by global mutex, &sit_i->sentry_lock */
f2fs_wait_discard_bio(struct f2fs_sb_info * sbi,block_t blkaddr)1729 static void f2fs_wait_discard_bio(struct f2fs_sb_info *sbi, block_t blkaddr)
1730 {
1731 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1732 struct discard_cmd *dc;
1733 bool need_wait = false;
1734
1735 mutex_lock(&dcc->cmd_lock);
1736 dc = (struct discard_cmd *)f2fs_lookup_rb_tree(&dcc->root,
1737 NULL, blkaddr);
1738 if (dc) {
1739 if (dc->state == D_PREP) {
1740 __punch_discard_cmd(sbi, dc, blkaddr);
1741 } else {
1742 dc->ref++;
1743 need_wait = true;
1744 }
1745 }
1746 mutex_unlock(&dcc->cmd_lock);
1747
1748 if (need_wait)
1749 __wait_one_discard_bio(sbi, dc);
1750 }
1751
f2fs_stop_discard_thread(struct f2fs_sb_info * sbi)1752 void f2fs_stop_discard_thread(struct f2fs_sb_info *sbi)
1753 {
1754 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1755
1756 if (dcc && dcc->f2fs_issue_discard) {
1757 struct task_struct *discard_thread = dcc->f2fs_issue_discard;
1758
1759 dcc->f2fs_issue_discard = NULL;
1760 kthread_stop(discard_thread);
1761 }
1762 }
1763
1764 /* This comes from f2fs_put_super */
f2fs_issue_discard_timeout(struct f2fs_sb_info * sbi)1765 bool f2fs_issue_discard_timeout(struct f2fs_sb_info *sbi)
1766 {
1767 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1768 struct discard_policy dpolicy;
1769 bool dropped;
1770
1771 __init_discard_policy(sbi, &dpolicy, DPOLICY_UMOUNT,
1772 dcc->discard_granularity);
1773 __issue_discard_cmd(sbi, &dpolicy);
1774 dropped = __drop_discard_cmd(sbi);
1775
1776 /* just to make sure there is no pending discard commands */
1777 __wait_all_discard_cmd(sbi, NULL);
1778
1779 f2fs_bug_on(sbi, atomic_read(&dcc->discard_cmd_cnt));
1780 return dropped;
1781 }
1782
issue_discard_thread(void * data)1783 static int issue_discard_thread(void *data)
1784 {
1785 struct f2fs_sb_info *sbi = data;
1786 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
1787 wait_queue_head_t *q = &dcc->discard_wait_queue;
1788 struct discard_policy dpolicy;
1789 unsigned int wait_ms = dcc->min_discard_issue_time;
1790 int issued;
1791
1792 set_freezable();
1793
1794 do {
1795 if (sbi->gc_mode == GC_URGENT_HIGH ||
1796 !f2fs_available_free_memory(sbi, DISCARD_CACHE))
1797 __init_discard_policy(sbi, &dpolicy, DPOLICY_FORCE, 1);
1798 else
1799 __init_discard_policy(sbi, &dpolicy, DPOLICY_BG,
1800 dcc->discard_granularity);
1801
1802 if (!atomic_read(&dcc->discard_cmd_cnt))
1803 wait_ms = dpolicy.max_interval;
1804
1805 wait_event_interruptible_timeout(*q,
1806 kthread_should_stop() || freezing(current) ||
1807 dcc->discard_wake,
1808 msecs_to_jiffies(wait_ms));
1809
1810 if (dcc->discard_wake)
1811 dcc->discard_wake = 0;
1812
1813 /* clean up pending candidates before going to sleep */
1814 if (atomic_read(&dcc->queued_discard))
1815 __wait_all_discard_cmd(sbi, NULL);
1816
1817 if (try_to_freeze())
1818 continue;
1819 if (f2fs_readonly(sbi->sb))
1820 continue;
1821 if (kthread_should_stop())
1822 return 0;
1823 if (is_sbi_flag_set(sbi, SBI_NEED_FSCK)) {
1824 wait_ms = dpolicy.max_interval;
1825 continue;
1826 }
1827 if (!atomic_read(&dcc->discard_cmd_cnt))
1828 continue;
1829
1830 sb_start_intwrite(sbi->sb);
1831
1832 issued = __issue_discard_cmd(sbi, &dpolicy);
1833 if (issued > 0) {
1834 __wait_all_discard_cmd(sbi, &dpolicy);
1835 wait_ms = dpolicy.min_interval;
1836 } else if (issued == -1) {
1837 wait_ms = f2fs_time_to_wait(sbi, DISCARD_TIME);
1838 if (!wait_ms)
1839 wait_ms = dpolicy.mid_interval;
1840 } else {
1841 wait_ms = dpolicy.max_interval;
1842 }
1843
1844 sb_end_intwrite(sbi->sb);
1845
1846 } while (!kthread_should_stop());
1847 return 0;
1848 }
1849
1850 #ifdef CONFIG_BLK_DEV_ZONED
__f2fs_issue_discard_zone(struct f2fs_sb_info * sbi,struct block_device * bdev,block_t blkstart,block_t blklen)1851 static int __f2fs_issue_discard_zone(struct f2fs_sb_info *sbi,
1852 struct block_device *bdev, block_t blkstart, block_t blklen)
1853 {
1854 sector_t sector, nr_sects;
1855 block_t lblkstart = blkstart;
1856 int devi = 0;
1857
1858 if (f2fs_is_multi_device(sbi)) {
1859 devi = f2fs_target_device_index(sbi, blkstart);
1860 if (blkstart < FDEV(devi).start_blk ||
1861 blkstart > FDEV(devi).end_blk) {
1862 f2fs_err(sbi, "Invalid block %x", blkstart);
1863 return -EIO;
1864 }
1865 blkstart -= FDEV(devi).start_blk;
1866 }
1867
1868 /* For sequential zones, reset the zone write pointer */
1869 if (f2fs_blkz_is_seq(sbi, devi, blkstart)) {
1870 sector = SECTOR_FROM_BLOCK(blkstart);
1871 nr_sects = SECTOR_FROM_BLOCK(blklen);
1872
1873 if (sector & (bdev_zone_sectors(bdev) - 1) ||
1874 nr_sects != bdev_zone_sectors(bdev)) {
1875 f2fs_err(sbi, "(%d) %s: Unaligned zone reset attempted (block %x + %x)",
1876 devi, sbi->s_ndevs ? FDEV(devi).path : "",
1877 blkstart, blklen);
1878 return -EIO;
1879 }
1880 trace_f2fs_issue_reset_zone(bdev, blkstart);
1881 return blkdev_zone_mgmt(bdev, REQ_OP_ZONE_RESET,
1882 sector, nr_sects, GFP_NOFS);
1883 }
1884
1885 /* For conventional zones, use regular discard if supported */
1886 return __queue_discard_cmd(sbi, bdev, lblkstart, blklen);
1887 }
1888 #endif
1889
__issue_discard_async(struct f2fs_sb_info * sbi,struct block_device * bdev,block_t blkstart,block_t blklen)1890 static int __issue_discard_async(struct f2fs_sb_info *sbi,
1891 struct block_device *bdev, block_t blkstart, block_t blklen)
1892 {
1893 #ifdef CONFIG_BLK_DEV_ZONED
1894 if (f2fs_sb_has_blkzoned(sbi) && bdev_is_zoned(bdev))
1895 return __f2fs_issue_discard_zone(sbi, bdev, blkstart, blklen);
1896 #endif
1897 return __queue_discard_cmd(sbi, bdev, blkstart, blklen);
1898 }
1899
f2fs_issue_discard(struct f2fs_sb_info * sbi,block_t blkstart,block_t blklen)1900 static int f2fs_issue_discard(struct f2fs_sb_info *sbi,
1901 block_t blkstart, block_t blklen)
1902 {
1903 sector_t start = blkstart, len = 0;
1904 struct block_device *bdev;
1905 struct seg_entry *se;
1906 unsigned int offset;
1907 block_t i;
1908 int err = 0;
1909
1910 bdev = f2fs_target_device(sbi, blkstart, NULL);
1911
1912 for (i = blkstart; i < blkstart + blklen; i++, len++) {
1913 if (i != start) {
1914 struct block_device *bdev2 =
1915 f2fs_target_device(sbi, i, NULL);
1916
1917 if (bdev2 != bdev) {
1918 err = __issue_discard_async(sbi, bdev,
1919 start, len);
1920 if (err)
1921 return err;
1922 bdev = bdev2;
1923 start = i;
1924 len = 0;
1925 }
1926 }
1927
1928 se = get_seg_entry(sbi, GET_SEGNO(sbi, i));
1929 offset = GET_BLKOFF_FROM_SEG0(sbi, i);
1930
1931 if (f2fs_block_unit_discard(sbi) &&
1932 !f2fs_test_and_set_bit(offset, se->discard_map))
1933 sbi->discard_blks--;
1934 }
1935
1936 if (len)
1937 err = __issue_discard_async(sbi, bdev, start, len);
1938 return err;
1939 }
1940
add_discard_addrs(struct f2fs_sb_info * sbi,struct cp_control * cpc,bool check_only)1941 static bool add_discard_addrs(struct f2fs_sb_info *sbi, struct cp_control *cpc,
1942 bool check_only)
1943 {
1944 int entries = SIT_VBLOCK_MAP_SIZE / sizeof(unsigned long);
1945 int max_blocks = sbi->blocks_per_seg;
1946 struct seg_entry *se = get_seg_entry(sbi, cpc->trim_start);
1947 unsigned long *cur_map = (unsigned long *)se->cur_valid_map;
1948 unsigned long *ckpt_map = (unsigned long *)se->ckpt_valid_map;
1949 unsigned long *discard_map = (unsigned long *)se->discard_map;
1950 unsigned long *dmap = SIT_I(sbi)->tmp_map;
1951 unsigned int start = 0, end = -1;
1952 bool force = (cpc->reason & CP_DISCARD);
1953 struct discard_entry *de = NULL;
1954 struct list_head *head = &SM_I(sbi)->dcc_info->entry_list;
1955 int i;
1956
1957 if (se->valid_blocks == max_blocks || !f2fs_hw_support_discard(sbi) ||
1958 !f2fs_block_unit_discard(sbi))
1959 return false;
1960
1961 if (!force) {
1962 if (!f2fs_realtime_discard_enable(sbi) || !se->valid_blocks ||
1963 SM_I(sbi)->dcc_info->nr_discards >=
1964 SM_I(sbi)->dcc_info->max_discards)
1965 return false;
1966 }
1967
1968 /* SIT_VBLOCK_MAP_SIZE should be multiple of sizeof(unsigned long) */
1969 for (i = 0; i < entries; i++)
1970 dmap[i] = force ? ~ckpt_map[i] & ~discard_map[i] :
1971 (cur_map[i] ^ ckpt_map[i]) & ckpt_map[i];
1972
1973 while (force || SM_I(sbi)->dcc_info->nr_discards <=
1974 SM_I(sbi)->dcc_info->max_discards) {
1975 start = __find_rev_next_bit(dmap, max_blocks, end + 1);
1976 if (start >= max_blocks)
1977 break;
1978
1979 end = __find_rev_next_zero_bit(dmap, max_blocks, start + 1);
1980 if (force && start && end != max_blocks
1981 && (end - start) < cpc->trim_minlen)
1982 continue;
1983
1984 if (check_only)
1985 return true;
1986
1987 if (!de) {
1988 de = f2fs_kmem_cache_alloc(discard_entry_slab,
1989 GFP_F2FS_ZERO, true, NULL);
1990 de->start_blkaddr = START_BLOCK(sbi, cpc->trim_start);
1991 list_add_tail(&de->list, head);
1992 }
1993
1994 for (i = start; i < end; i++)
1995 __set_bit_le(i, (void *)de->discard_map);
1996
1997 SM_I(sbi)->dcc_info->nr_discards += end - start;
1998 }
1999 return false;
2000 }
2001
release_discard_addr(struct discard_entry * entry)2002 static void release_discard_addr(struct discard_entry *entry)
2003 {
2004 list_del(&entry->list);
2005 kmem_cache_free(discard_entry_slab, entry);
2006 }
2007
f2fs_release_discard_addrs(struct f2fs_sb_info * sbi)2008 void f2fs_release_discard_addrs(struct f2fs_sb_info *sbi)
2009 {
2010 struct list_head *head = &(SM_I(sbi)->dcc_info->entry_list);
2011 struct discard_entry *entry, *this;
2012
2013 /* drop caches */
2014 list_for_each_entry_safe(entry, this, head, list)
2015 release_discard_addr(entry);
2016 }
2017
2018 /*
2019 * Should call f2fs_clear_prefree_segments after checkpoint is done.
2020 */
set_prefree_as_free_segments(struct f2fs_sb_info * sbi)2021 static void set_prefree_as_free_segments(struct f2fs_sb_info *sbi)
2022 {
2023 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
2024 unsigned int segno;
2025
2026 mutex_lock(&dirty_i->seglist_lock);
2027 for_each_set_bit(segno, dirty_i->dirty_segmap[PRE], MAIN_SEGS(sbi))
2028 __set_test_and_free(sbi, segno, false);
2029 mutex_unlock(&dirty_i->seglist_lock);
2030 }
2031
f2fs_clear_prefree_segments(struct f2fs_sb_info * sbi,struct cp_control * cpc)2032 void f2fs_clear_prefree_segments(struct f2fs_sb_info *sbi,
2033 struct cp_control *cpc)
2034 {
2035 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
2036 struct list_head *head = &dcc->entry_list;
2037 struct discard_entry *entry, *this;
2038 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
2039 unsigned long *prefree_map = dirty_i->dirty_segmap[PRE];
2040 unsigned int start = 0, end = -1;
2041 unsigned int secno, start_segno;
2042 bool force = (cpc->reason & CP_DISCARD);
2043 bool section_alignment = F2FS_OPTION(sbi).discard_unit ==
2044 DISCARD_UNIT_SECTION;
2045
2046 if (f2fs_lfs_mode(sbi) && __is_large_section(sbi))
2047 section_alignment = true;
2048
2049 mutex_lock(&dirty_i->seglist_lock);
2050
2051 while (1) {
2052 int i;
2053
2054 if (section_alignment && end != -1)
2055 end--;
2056 start = find_next_bit(prefree_map, MAIN_SEGS(sbi), end + 1);
2057 if (start >= MAIN_SEGS(sbi))
2058 break;
2059 end = find_next_zero_bit(prefree_map, MAIN_SEGS(sbi),
2060 start + 1);
2061
2062 if (section_alignment) {
2063 start = rounddown(start, sbi->segs_per_sec);
2064 end = roundup(end, sbi->segs_per_sec);
2065 }
2066
2067 for (i = start; i < end; i++) {
2068 if (test_and_clear_bit(i, prefree_map))
2069 dirty_i->nr_dirty[PRE]--;
2070 }
2071
2072 if (!f2fs_realtime_discard_enable(sbi))
2073 continue;
2074
2075 if (force && start >= cpc->trim_start &&
2076 (end - 1) <= cpc->trim_end)
2077 continue;
2078
2079 if (!f2fs_lfs_mode(sbi) || !__is_large_section(sbi)) {
2080 f2fs_issue_discard(sbi, START_BLOCK(sbi, start),
2081 (end - start) << sbi->log_blocks_per_seg);
2082 continue;
2083 }
2084 next:
2085 secno = GET_SEC_FROM_SEG(sbi, start);
2086 start_segno = GET_SEG_FROM_SEC(sbi, secno);
2087 if (!IS_CURSEC(sbi, secno) &&
2088 !get_valid_blocks(sbi, start, true))
2089 f2fs_issue_discard(sbi, START_BLOCK(sbi, start_segno),
2090 sbi->segs_per_sec << sbi->log_blocks_per_seg);
2091
2092 start = start_segno + sbi->segs_per_sec;
2093 if (start < end)
2094 goto next;
2095 else
2096 end = start - 1;
2097 }
2098 mutex_unlock(&dirty_i->seglist_lock);
2099
2100 if (!f2fs_block_unit_discard(sbi))
2101 goto wakeup;
2102
2103 /* send small discards */
2104 list_for_each_entry_safe(entry, this, head, list) {
2105 unsigned int cur_pos = 0, next_pos, len, total_len = 0;
2106 bool is_valid = test_bit_le(0, entry->discard_map);
2107
2108 find_next:
2109 if (is_valid) {
2110 next_pos = find_next_zero_bit_le(entry->discard_map,
2111 sbi->blocks_per_seg, cur_pos);
2112 len = next_pos - cur_pos;
2113
2114 if (f2fs_sb_has_blkzoned(sbi) ||
2115 (force && len < cpc->trim_minlen))
2116 goto skip;
2117
2118 f2fs_issue_discard(sbi, entry->start_blkaddr + cur_pos,
2119 len);
2120 total_len += len;
2121 } else {
2122 next_pos = find_next_bit_le(entry->discard_map,
2123 sbi->blocks_per_seg, cur_pos);
2124 }
2125 skip:
2126 cur_pos = next_pos;
2127 is_valid = !is_valid;
2128
2129 if (cur_pos < sbi->blocks_per_seg)
2130 goto find_next;
2131
2132 release_discard_addr(entry);
2133 dcc->nr_discards -= total_len;
2134 }
2135
2136 wakeup:
2137 wake_up_discard_thread(sbi, false);
2138 }
2139
f2fs_start_discard_thread(struct f2fs_sb_info * sbi)2140 int f2fs_start_discard_thread(struct f2fs_sb_info *sbi)
2141 {
2142 dev_t dev = sbi->sb->s_bdev->bd_dev;
2143 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
2144 int err = 0;
2145
2146 if (!f2fs_realtime_discard_enable(sbi))
2147 return 0;
2148
2149 dcc->f2fs_issue_discard = kthread_run(issue_discard_thread, sbi,
2150 "f2fs_discard-%u:%u", MAJOR(dev), MINOR(dev));
2151 if (IS_ERR(dcc->f2fs_issue_discard)) {
2152 err = PTR_ERR(dcc->f2fs_issue_discard);
2153 dcc->f2fs_issue_discard = NULL;
2154 }
2155
2156 return err;
2157 }
2158
create_discard_cmd_control(struct f2fs_sb_info * sbi)2159 static int create_discard_cmd_control(struct f2fs_sb_info *sbi)
2160 {
2161 struct discard_cmd_control *dcc;
2162 int err = 0, i;
2163
2164 if (SM_I(sbi)->dcc_info) {
2165 dcc = SM_I(sbi)->dcc_info;
2166 goto init_thread;
2167 }
2168
2169 dcc = f2fs_kzalloc(sbi, sizeof(struct discard_cmd_control), GFP_KERNEL);
2170 if (!dcc)
2171 return -ENOMEM;
2172
2173 dcc->discard_granularity = DEFAULT_DISCARD_GRANULARITY;
2174 if (F2FS_OPTION(sbi).discard_unit == DISCARD_UNIT_SEGMENT)
2175 dcc->discard_granularity = sbi->blocks_per_seg;
2176 else if (F2FS_OPTION(sbi).discard_unit == DISCARD_UNIT_SECTION)
2177 dcc->discard_granularity = BLKS_PER_SEC(sbi);
2178
2179 INIT_LIST_HEAD(&dcc->entry_list);
2180 for (i = 0; i < MAX_PLIST_NUM; i++)
2181 INIT_LIST_HEAD(&dcc->pend_list[i]);
2182 INIT_LIST_HEAD(&dcc->wait_list);
2183 INIT_LIST_HEAD(&dcc->fstrim_list);
2184 mutex_init(&dcc->cmd_lock);
2185 atomic_set(&dcc->issued_discard, 0);
2186 atomic_set(&dcc->queued_discard, 0);
2187 atomic_set(&dcc->discard_cmd_cnt, 0);
2188 dcc->nr_discards = 0;
2189 dcc->max_discards = MAIN_SEGS(sbi) << sbi->log_blocks_per_seg;
2190 dcc->max_discard_request = DEF_MAX_DISCARD_REQUEST;
2191 dcc->min_discard_issue_time = DEF_MIN_DISCARD_ISSUE_TIME;
2192 dcc->mid_discard_issue_time = DEF_MID_DISCARD_ISSUE_TIME;
2193 dcc->max_discard_issue_time = DEF_MAX_DISCARD_ISSUE_TIME;
2194 dcc->undiscard_blks = 0;
2195 dcc->next_pos = 0;
2196 dcc->root = RB_ROOT_CACHED;
2197 dcc->rbtree_check = false;
2198
2199 init_waitqueue_head(&dcc->discard_wait_queue);
2200 SM_I(sbi)->dcc_info = dcc;
2201 init_thread:
2202 err = f2fs_start_discard_thread(sbi);
2203 if (err) {
2204 kfree(dcc);
2205 SM_I(sbi)->dcc_info = NULL;
2206 }
2207
2208 return err;
2209 }
2210
destroy_discard_cmd_control(struct f2fs_sb_info * sbi)2211 static void destroy_discard_cmd_control(struct f2fs_sb_info *sbi)
2212 {
2213 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
2214
2215 if (!dcc)
2216 return;
2217
2218 f2fs_stop_discard_thread(sbi);
2219
2220 /*
2221 * Recovery can cache discard commands, so in error path of
2222 * fill_super(), it needs to give a chance to handle them.
2223 */
2224 if (unlikely(atomic_read(&dcc->discard_cmd_cnt)))
2225 f2fs_issue_discard_timeout(sbi);
2226
2227 kfree(dcc);
2228 SM_I(sbi)->dcc_info = NULL;
2229 }
2230
__mark_sit_entry_dirty(struct f2fs_sb_info * sbi,unsigned int segno)2231 static bool __mark_sit_entry_dirty(struct f2fs_sb_info *sbi, unsigned int segno)
2232 {
2233 struct sit_info *sit_i = SIT_I(sbi);
2234
2235 if (!__test_and_set_bit(segno, sit_i->dirty_sentries_bitmap)) {
2236 sit_i->dirty_sentries++;
2237 return false;
2238 }
2239
2240 return true;
2241 }
2242
__set_sit_entry_type(struct f2fs_sb_info * sbi,int type,unsigned int segno,int modified)2243 static void __set_sit_entry_type(struct f2fs_sb_info *sbi, int type,
2244 unsigned int segno, int modified)
2245 {
2246 struct seg_entry *se = get_seg_entry(sbi, segno);
2247
2248 se->type = type;
2249 if (modified)
2250 __mark_sit_entry_dirty(sbi, segno);
2251 }
2252
get_segment_mtime(struct f2fs_sb_info * sbi,block_t blkaddr)2253 static inline unsigned long long get_segment_mtime(struct f2fs_sb_info *sbi,
2254 block_t blkaddr)
2255 {
2256 unsigned int segno = GET_SEGNO(sbi, blkaddr);
2257
2258 if (segno == NULL_SEGNO)
2259 return 0;
2260 return get_seg_entry(sbi, segno)->mtime;
2261 }
2262
update_segment_mtime(struct f2fs_sb_info * sbi,block_t blkaddr,unsigned long long old_mtime)2263 static void update_segment_mtime(struct f2fs_sb_info *sbi, block_t blkaddr,
2264 unsigned long long old_mtime)
2265 {
2266 struct seg_entry *se;
2267 unsigned int segno = GET_SEGNO(sbi, blkaddr);
2268 unsigned long long ctime = get_mtime(sbi, false);
2269 unsigned long long mtime = old_mtime ? old_mtime : ctime;
2270
2271 if (segno == NULL_SEGNO)
2272 return;
2273
2274 se = get_seg_entry(sbi, segno);
2275
2276 if (!se->mtime)
2277 se->mtime = mtime;
2278 else
2279 se->mtime = div_u64(se->mtime * se->valid_blocks + mtime,
2280 se->valid_blocks + 1);
2281
2282 if (ctime > SIT_I(sbi)->max_mtime)
2283 SIT_I(sbi)->max_mtime = ctime;
2284 }
2285
update_sit_entry(struct f2fs_sb_info * sbi,block_t blkaddr,int del)2286 static void update_sit_entry(struct f2fs_sb_info *sbi, block_t blkaddr, int del)
2287 {
2288 struct seg_entry *se;
2289 unsigned int segno, offset;
2290 long int new_vblocks;
2291 bool exist;
2292 #ifdef CONFIG_F2FS_CHECK_FS
2293 bool mir_exist;
2294 #endif
2295
2296 segno = GET_SEGNO(sbi, blkaddr);
2297
2298 se = get_seg_entry(sbi, segno);
2299 new_vblocks = se->valid_blocks + del;
2300 offset = GET_BLKOFF_FROM_SEG0(sbi, blkaddr);
2301
2302 f2fs_bug_on(sbi, (new_vblocks < 0 ||
2303 (new_vblocks > f2fs_usable_blks_in_seg(sbi, segno))));
2304
2305 se->valid_blocks = new_vblocks;
2306
2307 /* Update valid block bitmap */
2308 if (del > 0) {
2309 exist = f2fs_test_and_set_bit(offset, se->cur_valid_map);
2310 #ifdef CONFIG_F2FS_CHECK_FS
2311 mir_exist = f2fs_test_and_set_bit(offset,
2312 se->cur_valid_map_mir);
2313 if (unlikely(exist != mir_exist)) {
2314 f2fs_err(sbi, "Inconsistent error when setting bitmap, blk:%u, old bit:%d",
2315 blkaddr, exist);
2316 f2fs_bug_on(sbi, 1);
2317 }
2318 #endif
2319 if (unlikely(exist)) {
2320 f2fs_err(sbi, "Bitmap was wrongly set, blk:%u",
2321 blkaddr);
2322 f2fs_bug_on(sbi, 1);
2323 se->valid_blocks--;
2324 del = 0;
2325 }
2326
2327 if (f2fs_block_unit_discard(sbi) &&
2328 !f2fs_test_and_set_bit(offset, se->discard_map))
2329 sbi->discard_blks--;
2330
2331 /*
2332 * SSR should never reuse block which is checkpointed
2333 * or newly invalidated.
2334 */
2335 if (!is_sbi_flag_set(sbi, SBI_CP_DISABLED)) {
2336 if (!f2fs_test_and_set_bit(offset, se->ckpt_valid_map))
2337 se->ckpt_valid_blocks++;
2338 }
2339 } else {
2340 exist = f2fs_test_and_clear_bit(offset, se->cur_valid_map);
2341 #ifdef CONFIG_F2FS_CHECK_FS
2342 mir_exist = f2fs_test_and_clear_bit(offset,
2343 se->cur_valid_map_mir);
2344 if (unlikely(exist != mir_exist)) {
2345 f2fs_err(sbi, "Inconsistent error when clearing bitmap, blk:%u, old bit:%d",
2346 blkaddr, exist);
2347 f2fs_bug_on(sbi, 1);
2348 }
2349 #endif
2350 if (unlikely(!exist)) {
2351 f2fs_err(sbi, "Bitmap was wrongly cleared, blk:%u",
2352 blkaddr);
2353 f2fs_bug_on(sbi, 1);
2354 se->valid_blocks++;
2355 del = 0;
2356 } else if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED))) {
2357 /*
2358 * If checkpoints are off, we must not reuse data that
2359 * was used in the previous checkpoint. If it was used
2360 * before, we must track that to know how much space we
2361 * really have.
2362 */
2363 if (f2fs_test_bit(offset, se->ckpt_valid_map)) {
2364 spin_lock(&sbi->stat_lock);
2365 sbi->unusable_block_count++;
2366 spin_unlock(&sbi->stat_lock);
2367 }
2368 }
2369
2370 if (f2fs_block_unit_discard(sbi) &&
2371 f2fs_test_and_clear_bit(offset, se->discard_map))
2372 sbi->discard_blks++;
2373 }
2374 if (!f2fs_test_bit(offset, se->ckpt_valid_map))
2375 se->ckpt_valid_blocks += del;
2376
2377 __mark_sit_entry_dirty(sbi, segno);
2378
2379 /* update total number of valid blocks to be written in ckpt area */
2380 SIT_I(sbi)->written_valid_blocks += del;
2381
2382 if (__is_large_section(sbi))
2383 get_sec_entry(sbi, segno)->valid_blocks += del;
2384 }
2385
f2fs_invalidate_blocks(struct f2fs_sb_info * sbi,block_t addr)2386 void f2fs_invalidate_blocks(struct f2fs_sb_info *sbi, block_t addr)
2387 {
2388 unsigned int segno = GET_SEGNO(sbi, addr);
2389 struct sit_info *sit_i = SIT_I(sbi);
2390
2391 f2fs_bug_on(sbi, addr == NULL_ADDR);
2392 if (addr == NEW_ADDR || addr == COMPRESS_ADDR)
2393 return;
2394
2395 invalidate_mapping_pages(META_MAPPING(sbi), addr, addr);
2396 f2fs_invalidate_compress_page(sbi, addr);
2397
2398 /* add it into sit main buffer */
2399 down_write(&sit_i->sentry_lock);
2400
2401 update_segment_mtime(sbi, addr, 0);
2402 update_sit_entry(sbi, addr, -1);
2403
2404 /* add it into dirty seglist */
2405 locate_dirty_segment(sbi, segno);
2406
2407 up_write(&sit_i->sentry_lock);
2408 }
2409
f2fs_is_checkpointed_data(struct f2fs_sb_info * sbi,block_t blkaddr)2410 bool f2fs_is_checkpointed_data(struct f2fs_sb_info *sbi, block_t blkaddr)
2411 {
2412 struct sit_info *sit_i = SIT_I(sbi);
2413 unsigned int segno, offset;
2414 struct seg_entry *se;
2415 bool is_cp = false;
2416
2417 if (!__is_valid_data_blkaddr(blkaddr))
2418 return true;
2419
2420 down_read(&sit_i->sentry_lock);
2421
2422 segno = GET_SEGNO(sbi, blkaddr);
2423 se = get_seg_entry(sbi, segno);
2424 offset = GET_BLKOFF_FROM_SEG0(sbi, blkaddr);
2425
2426 if (f2fs_test_bit(offset, se->ckpt_valid_map))
2427 is_cp = true;
2428
2429 up_read(&sit_i->sentry_lock);
2430
2431 return is_cp;
2432 }
2433
2434 /*
2435 * This function should be resided under the curseg_mutex lock
2436 */
__add_sum_entry(struct f2fs_sb_info * sbi,int type,struct f2fs_summary * sum)2437 static void __add_sum_entry(struct f2fs_sb_info *sbi, int type,
2438 struct f2fs_summary *sum)
2439 {
2440 struct curseg_info *curseg = CURSEG_I(sbi, type);
2441 void *addr = curseg->sum_blk;
2442
2443 addr += curseg->next_blkoff * sizeof(struct f2fs_summary);
2444 memcpy(addr, sum, sizeof(struct f2fs_summary));
2445 }
2446
2447 /*
2448 * Calculate the number of current summary pages for writing
2449 */
f2fs_npages_for_summary_flush(struct f2fs_sb_info * sbi,bool for_ra)2450 int f2fs_npages_for_summary_flush(struct f2fs_sb_info *sbi, bool for_ra)
2451 {
2452 int valid_sum_count = 0;
2453 int i, sum_in_page;
2454
2455 for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++) {
2456 if (sbi->ckpt->alloc_type[i] == SSR)
2457 valid_sum_count += sbi->blocks_per_seg;
2458 else {
2459 if (for_ra)
2460 valid_sum_count += le16_to_cpu(
2461 F2FS_CKPT(sbi)->cur_data_blkoff[i]);
2462 else
2463 valid_sum_count += curseg_blkoff(sbi, i);
2464 }
2465 }
2466
2467 sum_in_page = (PAGE_SIZE - 2 * SUM_JOURNAL_SIZE -
2468 SUM_FOOTER_SIZE) / SUMMARY_SIZE;
2469 if (valid_sum_count <= sum_in_page)
2470 return 1;
2471 else if ((valid_sum_count - sum_in_page) <=
2472 (PAGE_SIZE - SUM_FOOTER_SIZE) / SUMMARY_SIZE)
2473 return 2;
2474 return 3;
2475 }
2476
2477 /*
2478 * Caller should put this summary page
2479 */
f2fs_get_sum_page(struct f2fs_sb_info * sbi,unsigned int segno)2480 struct page *f2fs_get_sum_page(struct f2fs_sb_info *sbi, unsigned int segno)
2481 {
2482 if (unlikely(f2fs_cp_error(sbi)))
2483 return ERR_PTR(-EIO);
2484 return f2fs_get_meta_page_retry(sbi, GET_SUM_BLOCK(sbi, segno));
2485 }
2486
f2fs_update_meta_page(struct f2fs_sb_info * sbi,void * src,block_t blk_addr)2487 void f2fs_update_meta_page(struct f2fs_sb_info *sbi,
2488 void *src, block_t blk_addr)
2489 {
2490 struct page *page = f2fs_grab_meta_page(sbi, blk_addr);
2491
2492 memcpy(page_address(page), src, PAGE_SIZE);
2493 set_page_dirty(page);
2494 f2fs_put_page(page, 1);
2495 }
2496
write_sum_page(struct f2fs_sb_info * sbi,struct f2fs_summary_block * sum_blk,block_t blk_addr)2497 static void write_sum_page(struct f2fs_sb_info *sbi,
2498 struct f2fs_summary_block *sum_blk, block_t blk_addr)
2499 {
2500 f2fs_update_meta_page(sbi, (void *)sum_blk, blk_addr);
2501 }
2502
write_current_sum_page(struct f2fs_sb_info * sbi,int type,block_t blk_addr)2503 static void write_current_sum_page(struct f2fs_sb_info *sbi,
2504 int type, block_t blk_addr)
2505 {
2506 struct curseg_info *curseg = CURSEG_I(sbi, type);
2507 struct page *page = f2fs_grab_meta_page(sbi, blk_addr);
2508 struct f2fs_summary_block *src = curseg->sum_blk;
2509 struct f2fs_summary_block *dst;
2510
2511 dst = (struct f2fs_summary_block *)page_address(page);
2512 memset(dst, 0, PAGE_SIZE);
2513
2514 mutex_lock(&curseg->curseg_mutex);
2515
2516 down_read(&curseg->journal_rwsem);
2517 memcpy(&dst->journal, curseg->journal, SUM_JOURNAL_SIZE);
2518 up_read(&curseg->journal_rwsem);
2519
2520 memcpy(dst->entries, src->entries, SUM_ENTRY_SIZE);
2521 memcpy(&dst->footer, &src->footer, SUM_FOOTER_SIZE);
2522
2523 mutex_unlock(&curseg->curseg_mutex);
2524
2525 set_page_dirty(page);
2526 f2fs_put_page(page, 1);
2527 }
2528
is_next_segment_free(struct f2fs_sb_info * sbi,struct curseg_info * curseg,int type)2529 static int is_next_segment_free(struct f2fs_sb_info *sbi,
2530 struct curseg_info *curseg, int type)
2531 {
2532 unsigned int segno = curseg->segno + 1;
2533 struct free_segmap_info *free_i = FREE_I(sbi);
2534
2535 if (segno < MAIN_SEGS(sbi) && segno % sbi->segs_per_sec)
2536 return !test_bit(segno, free_i->free_segmap);
2537 return 0;
2538 }
2539
2540 /*
2541 * Find a new segment from the free segments bitmap to right order
2542 * This function should be returned with success, otherwise BUG
2543 */
get_new_segment(struct f2fs_sb_info * sbi,unsigned int * newseg,bool new_sec,int dir)2544 static void get_new_segment(struct f2fs_sb_info *sbi,
2545 unsigned int *newseg, bool new_sec, int dir)
2546 {
2547 struct free_segmap_info *free_i = FREE_I(sbi);
2548 unsigned int segno, secno, zoneno;
2549 unsigned int total_zones = MAIN_SECS(sbi) / sbi->secs_per_zone;
2550 unsigned int hint = GET_SEC_FROM_SEG(sbi, *newseg);
2551 unsigned int old_zoneno = GET_ZONE_FROM_SEG(sbi, *newseg);
2552 unsigned int left_start = hint;
2553 bool init = true;
2554 int go_left = 0;
2555 int i;
2556
2557 spin_lock(&free_i->segmap_lock);
2558
2559 if (!new_sec && ((*newseg + 1) % sbi->segs_per_sec)) {
2560 segno = find_next_zero_bit(free_i->free_segmap,
2561 GET_SEG_FROM_SEC(sbi, hint + 1), *newseg + 1);
2562 if (segno < GET_SEG_FROM_SEC(sbi, hint + 1))
2563 goto got_it;
2564 }
2565 find_other_zone:
2566 secno = find_next_zero_bit(free_i->free_secmap, MAIN_SECS(sbi), hint);
2567 if (secno >= MAIN_SECS(sbi)) {
2568 if (dir == ALLOC_RIGHT) {
2569 secno = find_next_zero_bit(free_i->free_secmap,
2570 MAIN_SECS(sbi), 0);
2571 f2fs_bug_on(sbi, secno >= MAIN_SECS(sbi));
2572 } else {
2573 go_left = 1;
2574 left_start = hint - 1;
2575 }
2576 }
2577 if (go_left == 0)
2578 goto skip_left;
2579
2580 while (test_bit(left_start, free_i->free_secmap)) {
2581 if (left_start > 0) {
2582 left_start--;
2583 continue;
2584 }
2585 left_start = find_next_zero_bit(free_i->free_secmap,
2586 MAIN_SECS(sbi), 0);
2587 f2fs_bug_on(sbi, left_start >= MAIN_SECS(sbi));
2588 break;
2589 }
2590 secno = left_start;
2591 skip_left:
2592 segno = GET_SEG_FROM_SEC(sbi, secno);
2593 zoneno = GET_ZONE_FROM_SEC(sbi, secno);
2594
2595 /* give up on finding another zone */
2596 if (!init)
2597 goto got_it;
2598 if (sbi->secs_per_zone == 1)
2599 goto got_it;
2600 if (zoneno == old_zoneno)
2601 goto got_it;
2602 if (dir == ALLOC_LEFT) {
2603 if (!go_left && zoneno + 1 >= total_zones)
2604 goto got_it;
2605 if (go_left && zoneno == 0)
2606 goto got_it;
2607 }
2608 for (i = 0; i < NR_CURSEG_TYPE; i++)
2609 if (CURSEG_I(sbi, i)->zone == zoneno)
2610 break;
2611
2612 if (i < NR_CURSEG_TYPE) {
2613 /* zone is in user, try another */
2614 if (go_left)
2615 hint = zoneno * sbi->secs_per_zone - 1;
2616 else if (zoneno + 1 >= total_zones)
2617 hint = 0;
2618 else
2619 hint = (zoneno + 1) * sbi->secs_per_zone;
2620 init = false;
2621 goto find_other_zone;
2622 }
2623 got_it:
2624 /* set it as dirty segment in free segmap */
2625 f2fs_bug_on(sbi, test_bit(segno, free_i->free_segmap));
2626 __set_inuse(sbi, segno);
2627 *newseg = segno;
2628 spin_unlock(&free_i->segmap_lock);
2629 }
2630
reset_curseg(struct f2fs_sb_info * sbi,int type,int modified)2631 static void reset_curseg(struct f2fs_sb_info *sbi, int type, int modified)
2632 {
2633 struct curseg_info *curseg = CURSEG_I(sbi, type);
2634 struct summary_footer *sum_footer;
2635 unsigned short seg_type = curseg->seg_type;
2636
2637 curseg->inited = true;
2638 curseg->segno = curseg->next_segno;
2639 curseg->zone = GET_ZONE_FROM_SEG(sbi, curseg->segno);
2640 curseg->next_blkoff = 0;
2641 curseg->next_segno = NULL_SEGNO;
2642
2643 sum_footer = &(curseg->sum_blk->footer);
2644 memset(sum_footer, 0, sizeof(struct summary_footer));
2645
2646 sanity_check_seg_type(sbi, seg_type);
2647
2648 if (IS_DATASEG(seg_type))
2649 SET_SUM_TYPE(sum_footer, SUM_TYPE_DATA);
2650 if (IS_NODESEG(seg_type))
2651 SET_SUM_TYPE(sum_footer, SUM_TYPE_NODE);
2652 __set_sit_entry_type(sbi, seg_type, curseg->segno, modified);
2653 }
2654
__get_next_segno(struct f2fs_sb_info * sbi,int type)2655 static unsigned int __get_next_segno(struct f2fs_sb_info *sbi, int type)
2656 {
2657 struct curseg_info *curseg = CURSEG_I(sbi, type);
2658 unsigned short seg_type = curseg->seg_type;
2659
2660 sanity_check_seg_type(sbi, seg_type);
2661 if (f2fs_need_rand_seg(sbi))
2662 return prandom_u32() % (MAIN_SECS(sbi) * sbi->segs_per_sec);
2663
2664 /* if segs_per_sec is large than 1, we need to keep original policy. */
2665 if (__is_large_section(sbi))
2666 return curseg->segno;
2667
2668 /* inmem log may not locate on any segment after mount */
2669 if (!curseg->inited)
2670 return 0;
2671
2672 if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED)))
2673 return 0;
2674
2675 if (test_opt(sbi, NOHEAP) &&
2676 (seg_type == CURSEG_HOT_DATA || IS_NODESEG(seg_type)))
2677 return 0;
2678
2679 if (SIT_I(sbi)->last_victim[ALLOC_NEXT])
2680 return SIT_I(sbi)->last_victim[ALLOC_NEXT];
2681
2682 /* find segments from 0 to reuse freed segments */
2683 if (F2FS_OPTION(sbi).alloc_mode == ALLOC_MODE_REUSE)
2684 return 0;
2685
2686 return curseg->segno;
2687 }
2688
2689 /*
2690 * Allocate a current working segment.
2691 * This function always allocates a free segment in LFS manner.
2692 */
new_curseg(struct f2fs_sb_info * sbi,int type,bool new_sec)2693 static void new_curseg(struct f2fs_sb_info *sbi, int type, bool new_sec)
2694 {
2695 struct curseg_info *curseg = CURSEG_I(sbi, type);
2696 unsigned short seg_type = curseg->seg_type;
2697 unsigned int segno = curseg->segno;
2698 int dir = ALLOC_LEFT;
2699
2700 if (curseg->inited)
2701 write_sum_page(sbi, curseg->sum_blk,
2702 GET_SUM_BLOCK(sbi, segno));
2703 if (seg_type == CURSEG_WARM_DATA || seg_type == CURSEG_COLD_DATA)
2704 dir = ALLOC_RIGHT;
2705
2706 if (test_opt(sbi, NOHEAP))
2707 dir = ALLOC_RIGHT;
2708
2709 segno = __get_next_segno(sbi, type);
2710 get_new_segment(sbi, &segno, new_sec, dir);
2711 curseg->next_segno = segno;
2712 reset_curseg(sbi, type, 1);
2713 curseg->alloc_type = LFS;
2714 if (F2FS_OPTION(sbi).fs_mode == FS_MODE_FRAGMENT_BLK)
2715 curseg->fragment_remained_chunk =
2716 prandom_u32() % sbi->max_fragment_chunk + 1;
2717 }
2718
__next_free_blkoff(struct f2fs_sb_info * sbi,int segno,block_t start)2719 static int __next_free_blkoff(struct f2fs_sb_info *sbi,
2720 int segno, block_t start)
2721 {
2722 struct seg_entry *se = get_seg_entry(sbi, segno);
2723 int entries = SIT_VBLOCK_MAP_SIZE / sizeof(unsigned long);
2724 unsigned long *target_map = SIT_I(sbi)->tmp_map;
2725 unsigned long *ckpt_map = (unsigned long *)se->ckpt_valid_map;
2726 unsigned long *cur_map = (unsigned long *)se->cur_valid_map;
2727 int i;
2728
2729 for (i = 0; i < entries; i++)
2730 target_map[i] = ckpt_map[i] | cur_map[i];
2731
2732 return __find_rev_next_zero_bit(target_map, sbi->blocks_per_seg, start);
2733 }
2734
2735 /*
2736 * If a segment is written by LFS manner, next block offset is just obtained
2737 * by increasing the current block offset. However, if a segment is written by
2738 * SSR manner, next block offset obtained by calling __next_free_blkoff
2739 */
__refresh_next_blkoff(struct f2fs_sb_info * sbi,struct curseg_info * seg)2740 static void __refresh_next_blkoff(struct f2fs_sb_info *sbi,
2741 struct curseg_info *seg)
2742 {
2743 if (seg->alloc_type == SSR) {
2744 seg->next_blkoff =
2745 __next_free_blkoff(sbi, seg->segno,
2746 seg->next_blkoff + 1);
2747 } else {
2748 seg->next_blkoff++;
2749 if (F2FS_OPTION(sbi).fs_mode == FS_MODE_FRAGMENT_BLK) {
2750 /* To allocate block chunks in different sizes, use random number */
2751 if (--seg->fragment_remained_chunk <= 0) {
2752 seg->fragment_remained_chunk =
2753 prandom_u32() % sbi->max_fragment_chunk + 1;
2754 seg->next_blkoff +=
2755 prandom_u32() % sbi->max_fragment_hole + 1;
2756 }
2757 }
2758 }
2759 }
2760
f2fs_segment_has_free_slot(struct f2fs_sb_info * sbi,int segno)2761 bool f2fs_segment_has_free_slot(struct f2fs_sb_info *sbi, int segno)
2762 {
2763 return __next_free_blkoff(sbi, segno, 0) < sbi->blocks_per_seg;
2764 }
2765
2766 /*
2767 * This function always allocates a used segment(from dirty seglist) by SSR
2768 * manner, so it should recover the existing segment information of valid blocks
2769 */
change_curseg(struct f2fs_sb_info * sbi,int type,bool flush)2770 static void change_curseg(struct f2fs_sb_info *sbi, int type, bool flush)
2771 {
2772 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
2773 struct curseg_info *curseg = CURSEG_I(sbi, type);
2774 unsigned int new_segno = curseg->next_segno;
2775 struct f2fs_summary_block *sum_node;
2776 struct page *sum_page;
2777
2778 if (flush)
2779 write_sum_page(sbi, curseg->sum_blk,
2780 GET_SUM_BLOCK(sbi, curseg->segno));
2781
2782 __set_test_and_inuse(sbi, new_segno);
2783
2784 mutex_lock(&dirty_i->seglist_lock);
2785 __remove_dirty_segment(sbi, new_segno, PRE);
2786 __remove_dirty_segment(sbi, new_segno, DIRTY);
2787 mutex_unlock(&dirty_i->seglist_lock);
2788
2789 reset_curseg(sbi, type, 1);
2790 curseg->alloc_type = SSR;
2791 curseg->next_blkoff = __next_free_blkoff(sbi, curseg->segno, 0);
2792
2793 sum_page = f2fs_get_sum_page(sbi, new_segno);
2794 if (IS_ERR(sum_page)) {
2795 /* GC won't be able to use stale summary pages by cp_error */
2796 memset(curseg->sum_blk, 0, SUM_ENTRY_SIZE);
2797 return;
2798 }
2799 sum_node = (struct f2fs_summary_block *)page_address(sum_page);
2800 memcpy(curseg->sum_blk, sum_node, SUM_ENTRY_SIZE);
2801 f2fs_put_page(sum_page, 1);
2802 }
2803
2804 static int get_ssr_segment(struct f2fs_sb_info *sbi, int type,
2805 int alloc_mode, unsigned long long age);
2806
get_atssr_segment(struct f2fs_sb_info * sbi,int type,int target_type,int alloc_mode,unsigned long long age)2807 static void get_atssr_segment(struct f2fs_sb_info *sbi, int type,
2808 int target_type, int alloc_mode,
2809 unsigned long long age)
2810 {
2811 struct curseg_info *curseg = CURSEG_I(sbi, type);
2812
2813 curseg->seg_type = target_type;
2814
2815 if (get_ssr_segment(sbi, type, alloc_mode, age)) {
2816 struct seg_entry *se = get_seg_entry(sbi, curseg->next_segno);
2817
2818 curseg->seg_type = se->type;
2819 change_curseg(sbi, type, true);
2820 } else {
2821 /* allocate cold segment by default */
2822 curseg->seg_type = CURSEG_COLD_DATA;
2823 new_curseg(sbi, type, true);
2824 }
2825 stat_inc_seg_type(sbi, curseg);
2826 }
2827
__f2fs_init_atgc_curseg(struct f2fs_sb_info * sbi)2828 static void __f2fs_init_atgc_curseg(struct f2fs_sb_info *sbi)
2829 {
2830 struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_ALL_DATA_ATGC);
2831
2832 if (!sbi->am.atgc_enabled)
2833 return;
2834
2835 f2fs_down_read(&SM_I(sbi)->curseg_lock);
2836
2837 mutex_lock(&curseg->curseg_mutex);
2838 down_write(&SIT_I(sbi)->sentry_lock);
2839
2840 get_atssr_segment(sbi, CURSEG_ALL_DATA_ATGC, CURSEG_COLD_DATA, SSR, 0);
2841
2842 up_write(&SIT_I(sbi)->sentry_lock);
2843 mutex_unlock(&curseg->curseg_mutex);
2844
2845 f2fs_up_read(&SM_I(sbi)->curseg_lock);
2846
2847 }
f2fs_init_inmem_curseg(struct f2fs_sb_info * sbi)2848 void f2fs_init_inmem_curseg(struct f2fs_sb_info *sbi)
2849 {
2850 __f2fs_init_atgc_curseg(sbi);
2851 }
2852
__f2fs_save_inmem_curseg(struct f2fs_sb_info * sbi,int type)2853 static void __f2fs_save_inmem_curseg(struct f2fs_sb_info *sbi, int type)
2854 {
2855 struct curseg_info *curseg = CURSEG_I(sbi, type);
2856
2857 mutex_lock(&curseg->curseg_mutex);
2858 if (!curseg->inited)
2859 goto out;
2860
2861 if (get_valid_blocks(sbi, curseg->segno, false)) {
2862 write_sum_page(sbi, curseg->sum_blk,
2863 GET_SUM_BLOCK(sbi, curseg->segno));
2864 } else {
2865 mutex_lock(&DIRTY_I(sbi)->seglist_lock);
2866 __set_test_and_free(sbi, curseg->segno, true);
2867 mutex_unlock(&DIRTY_I(sbi)->seglist_lock);
2868 }
2869 out:
2870 mutex_unlock(&curseg->curseg_mutex);
2871 }
2872
f2fs_save_inmem_curseg(struct f2fs_sb_info * sbi)2873 void f2fs_save_inmem_curseg(struct f2fs_sb_info *sbi)
2874 {
2875 __f2fs_save_inmem_curseg(sbi, CURSEG_COLD_DATA_PINNED);
2876
2877 if (sbi->am.atgc_enabled)
2878 __f2fs_save_inmem_curseg(sbi, CURSEG_ALL_DATA_ATGC);
2879 }
2880
__f2fs_restore_inmem_curseg(struct f2fs_sb_info * sbi,int type)2881 static void __f2fs_restore_inmem_curseg(struct f2fs_sb_info *sbi, int type)
2882 {
2883 struct curseg_info *curseg = CURSEG_I(sbi, type);
2884
2885 mutex_lock(&curseg->curseg_mutex);
2886 if (!curseg->inited)
2887 goto out;
2888 if (get_valid_blocks(sbi, curseg->segno, false))
2889 goto out;
2890
2891 mutex_lock(&DIRTY_I(sbi)->seglist_lock);
2892 __set_test_and_inuse(sbi, curseg->segno);
2893 mutex_unlock(&DIRTY_I(sbi)->seglist_lock);
2894 out:
2895 mutex_unlock(&curseg->curseg_mutex);
2896 }
2897
f2fs_restore_inmem_curseg(struct f2fs_sb_info * sbi)2898 void f2fs_restore_inmem_curseg(struct f2fs_sb_info *sbi)
2899 {
2900 __f2fs_restore_inmem_curseg(sbi, CURSEG_COLD_DATA_PINNED);
2901
2902 if (sbi->am.atgc_enabled)
2903 __f2fs_restore_inmem_curseg(sbi, CURSEG_ALL_DATA_ATGC);
2904 }
2905
get_ssr_segment(struct f2fs_sb_info * sbi,int type,int alloc_mode,unsigned long long age)2906 static int get_ssr_segment(struct f2fs_sb_info *sbi, int type,
2907 int alloc_mode, unsigned long long age)
2908 {
2909 struct curseg_info *curseg = CURSEG_I(sbi, type);
2910 const struct victim_selection *v_ops = DIRTY_I(sbi)->v_ops;
2911 unsigned segno = NULL_SEGNO;
2912 unsigned short seg_type = curseg->seg_type;
2913 int i, cnt;
2914 bool reversed = false;
2915
2916 sanity_check_seg_type(sbi, seg_type);
2917
2918 /* f2fs_need_SSR() already forces to do this */
2919 if (!v_ops->get_victim(sbi, &segno, BG_GC, seg_type, alloc_mode, age)) {
2920 curseg->next_segno = segno;
2921 return 1;
2922 }
2923
2924 /* For node segments, let's do SSR more intensively */
2925 if (IS_NODESEG(seg_type)) {
2926 if (seg_type >= CURSEG_WARM_NODE) {
2927 reversed = true;
2928 i = CURSEG_COLD_NODE;
2929 } else {
2930 i = CURSEG_HOT_NODE;
2931 }
2932 cnt = NR_CURSEG_NODE_TYPE;
2933 } else {
2934 if (seg_type >= CURSEG_WARM_DATA) {
2935 reversed = true;
2936 i = CURSEG_COLD_DATA;
2937 } else {
2938 i = CURSEG_HOT_DATA;
2939 }
2940 cnt = NR_CURSEG_DATA_TYPE;
2941 }
2942
2943 for (; cnt-- > 0; reversed ? i-- : i++) {
2944 if (i == seg_type)
2945 continue;
2946 if (!v_ops->get_victim(sbi, &segno, BG_GC, i, alloc_mode, age)) {
2947 curseg->next_segno = segno;
2948 return 1;
2949 }
2950 }
2951
2952 /* find valid_blocks=0 in dirty list */
2953 if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED))) {
2954 segno = get_free_segment(sbi);
2955 if (segno != NULL_SEGNO) {
2956 curseg->next_segno = segno;
2957 return 1;
2958 }
2959 }
2960 return 0;
2961 }
2962
2963 /*
2964 * flush out current segment and replace it with new segment
2965 * This function should be returned with success, otherwise BUG
2966 */
allocate_segment_by_default(struct f2fs_sb_info * sbi,int type,bool force)2967 static void allocate_segment_by_default(struct f2fs_sb_info *sbi,
2968 int type, bool force)
2969 {
2970 struct curseg_info *curseg = CURSEG_I(sbi, type);
2971
2972 if (force)
2973 new_curseg(sbi, type, true);
2974 else if (!is_set_ckpt_flags(sbi, CP_CRC_RECOVERY_FLAG) &&
2975 curseg->seg_type == CURSEG_WARM_NODE)
2976 new_curseg(sbi, type, false);
2977 else if (curseg->alloc_type == LFS &&
2978 is_next_segment_free(sbi, curseg, type) &&
2979 likely(!is_sbi_flag_set(sbi, SBI_CP_DISABLED)))
2980 new_curseg(sbi, type, false);
2981 else if (f2fs_need_SSR(sbi) &&
2982 get_ssr_segment(sbi, type, SSR, 0))
2983 change_curseg(sbi, type, true);
2984 else
2985 new_curseg(sbi, type, false);
2986
2987 stat_inc_seg_type(sbi, curseg);
2988 }
2989
f2fs_allocate_segment_for_resize(struct f2fs_sb_info * sbi,int type,unsigned int start,unsigned int end)2990 void f2fs_allocate_segment_for_resize(struct f2fs_sb_info *sbi, int type,
2991 unsigned int start, unsigned int end)
2992 {
2993 struct curseg_info *curseg = CURSEG_I(sbi, type);
2994 unsigned int segno;
2995
2996 f2fs_down_read(&SM_I(sbi)->curseg_lock);
2997 mutex_lock(&curseg->curseg_mutex);
2998 down_write(&SIT_I(sbi)->sentry_lock);
2999
3000 segno = CURSEG_I(sbi, type)->segno;
3001 if (segno < start || segno > end)
3002 goto unlock;
3003
3004 if (f2fs_need_SSR(sbi) && get_ssr_segment(sbi, type, SSR, 0))
3005 change_curseg(sbi, type, true);
3006 else
3007 new_curseg(sbi, type, true);
3008
3009 stat_inc_seg_type(sbi, curseg);
3010
3011 locate_dirty_segment(sbi, segno);
3012 unlock:
3013 up_write(&SIT_I(sbi)->sentry_lock);
3014
3015 if (segno != curseg->segno)
3016 f2fs_notice(sbi, "For resize: curseg of type %d: %u ==> %u",
3017 type, segno, curseg->segno);
3018
3019 mutex_unlock(&curseg->curseg_mutex);
3020 f2fs_up_read(&SM_I(sbi)->curseg_lock);
3021 }
3022
__allocate_new_segment(struct f2fs_sb_info * sbi,int type,bool new_sec,bool force)3023 static void __allocate_new_segment(struct f2fs_sb_info *sbi, int type,
3024 bool new_sec, bool force)
3025 {
3026 struct curseg_info *curseg = CURSEG_I(sbi, type);
3027 unsigned int old_segno;
3028
3029 if (!curseg->inited)
3030 goto alloc;
3031
3032 if (force || curseg->next_blkoff ||
3033 get_valid_blocks(sbi, curseg->segno, new_sec))
3034 goto alloc;
3035
3036 if (!get_ckpt_valid_blocks(sbi, curseg->segno, new_sec))
3037 return;
3038 alloc:
3039 old_segno = curseg->segno;
3040 SIT_I(sbi)->s_ops->allocate_segment(sbi, type, true);
3041 locate_dirty_segment(sbi, old_segno);
3042 }
3043
__allocate_new_section(struct f2fs_sb_info * sbi,int type,bool force)3044 static void __allocate_new_section(struct f2fs_sb_info *sbi,
3045 int type, bool force)
3046 {
3047 __allocate_new_segment(sbi, type, true, force);
3048 }
3049
f2fs_allocate_new_section(struct f2fs_sb_info * sbi,int type,bool force)3050 void f2fs_allocate_new_section(struct f2fs_sb_info *sbi, int type, bool force)
3051 {
3052 f2fs_down_read(&SM_I(sbi)->curseg_lock);
3053 down_write(&SIT_I(sbi)->sentry_lock);
3054 __allocate_new_section(sbi, type, force);
3055 up_write(&SIT_I(sbi)->sentry_lock);
3056 f2fs_up_read(&SM_I(sbi)->curseg_lock);
3057 }
3058
f2fs_allocate_new_segments(struct f2fs_sb_info * sbi)3059 void f2fs_allocate_new_segments(struct f2fs_sb_info *sbi)
3060 {
3061 int i;
3062
3063 f2fs_down_read(&SM_I(sbi)->curseg_lock);
3064 down_write(&SIT_I(sbi)->sentry_lock);
3065 for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++)
3066 __allocate_new_segment(sbi, i, false, false);
3067 up_write(&SIT_I(sbi)->sentry_lock);
3068 f2fs_up_read(&SM_I(sbi)->curseg_lock);
3069 }
3070
3071 static const struct segment_allocation default_salloc_ops = {
3072 .allocate_segment = allocate_segment_by_default,
3073 };
3074
f2fs_exist_trim_candidates(struct f2fs_sb_info * sbi,struct cp_control * cpc)3075 bool f2fs_exist_trim_candidates(struct f2fs_sb_info *sbi,
3076 struct cp_control *cpc)
3077 {
3078 __u64 trim_start = cpc->trim_start;
3079 bool has_candidate = false;
3080
3081 down_write(&SIT_I(sbi)->sentry_lock);
3082 for (; cpc->trim_start <= cpc->trim_end; cpc->trim_start++) {
3083 if (add_discard_addrs(sbi, cpc, true)) {
3084 has_candidate = true;
3085 break;
3086 }
3087 }
3088 up_write(&SIT_I(sbi)->sentry_lock);
3089
3090 cpc->trim_start = trim_start;
3091 return has_candidate;
3092 }
3093
__issue_discard_cmd_range(struct f2fs_sb_info * sbi,struct discard_policy * dpolicy,unsigned int start,unsigned int end)3094 static unsigned int __issue_discard_cmd_range(struct f2fs_sb_info *sbi,
3095 struct discard_policy *dpolicy,
3096 unsigned int start, unsigned int end)
3097 {
3098 struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
3099 struct discard_cmd *prev_dc = NULL, *next_dc = NULL;
3100 struct rb_node **insert_p = NULL, *insert_parent = NULL;
3101 struct discard_cmd *dc;
3102 struct blk_plug plug;
3103 int issued;
3104 unsigned int trimmed = 0;
3105
3106 next:
3107 issued = 0;
3108
3109 mutex_lock(&dcc->cmd_lock);
3110 if (unlikely(dcc->rbtree_check))
3111 f2fs_bug_on(sbi, !f2fs_check_rb_tree_consistence(sbi,
3112 &dcc->root, false));
3113
3114 dc = (struct discard_cmd *)f2fs_lookup_rb_tree_ret(&dcc->root,
3115 NULL, start,
3116 (struct rb_entry **)&prev_dc,
3117 (struct rb_entry **)&next_dc,
3118 &insert_p, &insert_parent, true, NULL);
3119 if (!dc)
3120 dc = next_dc;
3121
3122 blk_start_plug(&plug);
3123
3124 while (dc && dc->lstart <= end) {
3125 struct rb_node *node;
3126 int err = 0;
3127
3128 if (dc->len < dpolicy->granularity)
3129 goto skip;
3130
3131 if (dc->state != D_PREP) {
3132 list_move_tail(&dc->list, &dcc->fstrim_list);
3133 goto skip;
3134 }
3135
3136 err = __submit_discard_cmd(sbi, dpolicy, dc, &issued);
3137
3138 if (issued >= dpolicy->max_requests) {
3139 start = dc->lstart + dc->len;
3140
3141 if (err)
3142 __remove_discard_cmd(sbi, dc);
3143
3144 blk_finish_plug(&plug);
3145 mutex_unlock(&dcc->cmd_lock);
3146 trimmed += __wait_all_discard_cmd(sbi, NULL);
3147 f2fs_io_schedule_timeout(DEFAULT_IO_TIMEOUT);
3148 goto next;
3149 }
3150 skip:
3151 node = rb_next(&dc->rb_node);
3152 if (err)
3153 __remove_discard_cmd(sbi, dc);
3154 dc = rb_entry_safe(node, struct discard_cmd, rb_node);
3155
3156 if (fatal_signal_pending(current))
3157 break;
3158 }
3159
3160 blk_finish_plug(&plug);
3161 mutex_unlock(&dcc->cmd_lock);
3162
3163 return trimmed;
3164 }
3165
f2fs_trim_fs(struct f2fs_sb_info * sbi,struct fstrim_range * range)3166 int f2fs_trim_fs(struct f2fs_sb_info *sbi, struct fstrim_range *range)
3167 {
3168 __u64 start = F2FS_BYTES_TO_BLK(range->start);
3169 __u64 end = start + F2FS_BYTES_TO_BLK(range->len) - 1;
3170 unsigned int start_segno, end_segno;
3171 block_t start_block, end_block;
3172 struct cp_control cpc;
3173 struct discard_policy dpolicy;
3174 unsigned long long trimmed = 0;
3175 int err = 0;
3176 bool need_align = f2fs_lfs_mode(sbi) && __is_large_section(sbi);
3177
3178 if (start >= MAX_BLKADDR(sbi) || range->len < sbi->blocksize)
3179 return -EINVAL;
3180
3181 if (end < MAIN_BLKADDR(sbi))
3182 goto out;
3183
3184 if (is_sbi_flag_set(sbi, SBI_NEED_FSCK)) {
3185 f2fs_warn(sbi, "Found FS corruption, run fsck to fix.");
3186 return -EFSCORRUPTED;
3187 }
3188
3189 /* start/end segment number in main_area */
3190 start_segno = (start <= MAIN_BLKADDR(sbi)) ? 0 : GET_SEGNO(sbi, start);
3191 end_segno = (end >= MAX_BLKADDR(sbi)) ? MAIN_SEGS(sbi) - 1 :
3192 GET_SEGNO(sbi, end);
3193 if (need_align) {
3194 start_segno = rounddown(start_segno, sbi->segs_per_sec);
3195 end_segno = roundup(end_segno + 1, sbi->segs_per_sec) - 1;
3196 }
3197
3198 cpc.reason = CP_DISCARD;
3199 cpc.trim_minlen = max_t(__u64, 1, F2FS_BYTES_TO_BLK(range->minlen));
3200 cpc.trim_start = start_segno;
3201 cpc.trim_end = end_segno;
3202
3203 if (sbi->discard_blks == 0)
3204 goto out;
3205
3206 f2fs_down_write(&sbi->gc_lock);
3207 err = f2fs_write_checkpoint(sbi, &cpc);
3208 f2fs_up_write(&sbi->gc_lock);
3209 if (err)
3210 goto out;
3211
3212 /*
3213 * We filed discard candidates, but actually we don't need to wait for
3214 * all of them, since they'll be issued in idle time along with runtime
3215 * discard option. User configuration looks like using runtime discard
3216 * or periodic fstrim instead of it.
3217 */
3218 if (f2fs_realtime_discard_enable(sbi))
3219 goto out;
3220
3221 start_block = START_BLOCK(sbi, start_segno);
3222 end_block = START_BLOCK(sbi, end_segno + 1);
3223
3224 __init_discard_policy(sbi, &dpolicy, DPOLICY_FSTRIM, cpc.trim_minlen);
3225 trimmed = __issue_discard_cmd_range(sbi, &dpolicy,
3226 start_block, end_block);
3227
3228 trimmed += __wait_discard_cmd_range(sbi, &dpolicy,
3229 start_block, end_block);
3230 out:
3231 if (!err)
3232 range->len = F2FS_BLK_TO_BYTES(trimmed);
3233 return err;
3234 }
3235
__has_curseg_space(struct f2fs_sb_info * sbi,struct curseg_info * curseg)3236 static bool __has_curseg_space(struct f2fs_sb_info *sbi,
3237 struct curseg_info *curseg)
3238 {
3239 return curseg->next_blkoff < f2fs_usable_blks_in_seg(sbi,
3240 curseg->segno);
3241 }
3242
f2fs_rw_hint_to_seg_type(enum rw_hint hint)3243 int f2fs_rw_hint_to_seg_type(enum rw_hint hint)
3244 {
3245 switch (hint) {
3246 case WRITE_LIFE_SHORT:
3247 return CURSEG_HOT_DATA;
3248 case WRITE_LIFE_EXTREME:
3249 return CURSEG_COLD_DATA;
3250 default:
3251 return CURSEG_WARM_DATA;
3252 }
3253 }
3254
3255 /* This returns write hints for each segment type. This hints will be
3256 * passed down to block layer. There are mapping tables which depend on
3257 * the mount option 'whint_mode'.
3258 *
3259 * 1) whint_mode=off. F2FS only passes down WRITE_LIFE_NOT_SET.
3260 *
3261 * 2) whint_mode=user-based. F2FS tries to pass down hints given by users.
3262 *
3263 * User F2FS Block
3264 * ---- ---- -----
3265 * META WRITE_LIFE_NOT_SET
3266 * HOT_NODE "
3267 * WARM_NODE "
3268 * COLD_NODE "
3269 * ioctl(COLD) COLD_DATA WRITE_LIFE_EXTREME
3270 * extension list " "
3271 *
3272 * -- buffered io
3273 * WRITE_LIFE_EXTREME COLD_DATA WRITE_LIFE_EXTREME
3274 * WRITE_LIFE_SHORT HOT_DATA WRITE_LIFE_SHORT
3275 * WRITE_LIFE_NOT_SET WARM_DATA WRITE_LIFE_NOT_SET
3276 * WRITE_LIFE_NONE " "
3277 * WRITE_LIFE_MEDIUM " "
3278 * WRITE_LIFE_LONG " "
3279 *
3280 * -- direct io
3281 * WRITE_LIFE_EXTREME COLD_DATA WRITE_LIFE_EXTREME
3282 * WRITE_LIFE_SHORT HOT_DATA WRITE_LIFE_SHORT
3283 * WRITE_LIFE_NOT_SET WARM_DATA WRITE_LIFE_NOT_SET
3284 * WRITE_LIFE_NONE " WRITE_LIFE_NONE
3285 * WRITE_LIFE_MEDIUM " WRITE_LIFE_MEDIUM
3286 * WRITE_LIFE_LONG " WRITE_LIFE_LONG
3287 *
3288 * 3) whint_mode=fs-based. F2FS passes down hints with its policy.
3289 *
3290 * User F2FS Block
3291 * ---- ---- -----
3292 * META WRITE_LIFE_MEDIUM;
3293 * HOT_NODE WRITE_LIFE_NOT_SET
3294 * WARM_NODE "
3295 * COLD_NODE WRITE_LIFE_NONE
3296 * ioctl(COLD) COLD_DATA WRITE_LIFE_EXTREME
3297 * extension list " "
3298 *
3299 * -- buffered io
3300 * WRITE_LIFE_EXTREME COLD_DATA WRITE_LIFE_EXTREME
3301 * WRITE_LIFE_SHORT HOT_DATA WRITE_LIFE_SHORT
3302 * WRITE_LIFE_NOT_SET WARM_DATA WRITE_LIFE_LONG
3303 * WRITE_LIFE_NONE " "
3304 * WRITE_LIFE_MEDIUM " "
3305 * WRITE_LIFE_LONG " "
3306 *
3307 * -- direct io
3308 * WRITE_LIFE_EXTREME COLD_DATA WRITE_LIFE_EXTREME
3309 * WRITE_LIFE_SHORT HOT_DATA WRITE_LIFE_SHORT
3310 * WRITE_LIFE_NOT_SET WARM_DATA WRITE_LIFE_NOT_SET
3311 * WRITE_LIFE_NONE " WRITE_LIFE_NONE
3312 * WRITE_LIFE_MEDIUM " WRITE_LIFE_MEDIUM
3313 * WRITE_LIFE_LONG " WRITE_LIFE_LONG
3314 */
3315
f2fs_io_type_to_rw_hint(struct f2fs_sb_info * sbi,enum page_type type,enum temp_type temp)3316 enum rw_hint f2fs_io_type_to_rw_hint(struct f2fs_sb_info *sbi,
3317 enum page_type type, enum temp_type temp)
3318 {
3319 if (F2FS_OPTION(sbi).whint_mode == WHINT_MODE_USER) {
3320 if (type == DATA) {
3321 if (temp == WARM)
3322 return WRITE_LIFE_NOT_SET;
3323 else if (temp == HOT)
3324 return WRITE_LIFE_SHORT;
3325 else if (temp == COLD)
3326 return WRITE_LIFE_EXTREME;
3327 } else {
3328 return WRITE_LIFE_NOT_SET;
3329 }
3330 } else if (F2FS_OPTION(sbi).whint_mode == WHINT_MODE_FS) {
3331 if (type == DATA) {
3332 if (temp == WARM)
3333 return WRITE_LIFE_LONG;
3334 else if (temp == HOT)
3335 return WRITE_LIFE_SHORT;
3336 else if (temp == COLD)
3337 return WRITE_LIFE_EXTREME;
3338 } else if (type == NODE) {
3339 if (temp == WARM || temp == HOT)
3340 return WRITE_LIFE_NOT_SET;
3341 else if (temp == COLD)
3342 return WRITE_LIFE_NONE;
3343 } else if (type == META) {
3344 return WRITE_LIFE_MEDIUM;
3345 }
3346 }
3347 return WRITE_LIFE_NOT_SET;
3348 }
3349
__get_segment_type_2(struct f2fs_io_info * fio)3350 static int __get_segment_type_2(struct f2fs_io_info *fio)
3351 {
3352 if (fio->type == DATA)
3353 return CURSEG_HOT_DATA;
3354 else
3355 return CURSEG_HOT_NODE;
3356 }
3357
__get_segment_type_4(struct f2fs_io_info * fio)3358 static int __get_segment_type_4(struct f2fs_io_info *fio)
3359 {
3360 if (fio->type == DATA) {
3361 struct inode *inode = fio->page->mapping->host;
3362
3363 if (S_ISDIR(inode->i_mode))
3364 return CURSEG_HOT_DATA;
3365 else
3366 return CURSEG_COLD_DATA;
3367 } else {
3368 if (IS_DNODE(fio->page) && is_cold_node(fio->page))
3369 return CURSEG_WARM_NODE;
3370 else
3371 return CURSEG_COLD_NODE;
3372 }
3373 }
3374
__get_segment_type_6(struct f2fs_io_info * fio)3375 static int __get_segment_type_6(struct f2fs_io_info *fio)
3376 {
3377 if (fio->type == DATA) {
3378 struct inode *inode = fio->page->mapping->host;
3379
3380 if (is_inode_flag_set(inode, FI_ALIGNED_WRITE))
3381 return CURSEG_COLD_DATA_PINNED;
3382
3383 if (page_private_gcing(fio->page)) {
3384 if (fio->sbi->am.atgc_enabled &&
3385 (fio->io_type == FS_DATA_IO) &&
3386 (fio->sbi->gc_mode != GC_URGENT_HIGH))
3387 return CURSEG_ALL_DATA_ATGC;
3388 else
3389 return CURSEG_COLD_DATA;
3390 }
3391 if (file_is_cold(inode) || f2fs_need_compress_data(inode))
3392 return CURSEG_COLD_DATA;
3393 if (file_is_hot(inode) ||
3394 is_inode_flag_set(inode, FI_HOT_DATA) ||
3395 f2fs_is_atomic_file(inode) ||
3396 f2fs_is_volatile_file(inode))
3397 return CURSEG_HOT_DATA;
3398 return f2fs_rw_hint_to_seg_type(inode->i_write_hint);
3399 } else {
3400 if (IS_DNODE(fio->page))
3401 return is_cold_node(fio->page) ? CURSEG_WARM_NODE :
3402 CURSEG_HOT_NODE;
3403 return CURSEG_COLD_NODE;
3404 }
3405 }
3406
__get_segment_type(struct f2fs_io_info * fio)3407 static int __get_segment_type(struct f2fs_io_info *fio)
3408 {
3409 int type = 0;
3410
3411 switch (F2FS_OPTION(fio->sbi).active_logs) {
3412 case 2:
3413 type = __get_segment_type_2(fio);
3414 break;
3415 case 4:
3416 type = __get_segment_type_4(fio);
3417 break;
3418 case 6:
3419 type = __get_segment_type_6(fio);
3420 break;
3421 default:
3422 f2fs_bug_on(fio->sbi, true);
3423 }
3424
3425 if (IS_HOT(type))
3426 fio->temp = HOT;
3427 else if (IS_WARM(type))
3428 fio->temp = WARM;
3429 else
3430 fio->temp = COLD;
3431 return type;
3432 }
3433
f2fs_allocate_data_block(struct f2fs_sb_info * sbi,struct page * page,block_t old_blkaddr,block_t * new_blkaddr,struct f2fs_summary * sum,int type,struct f2fs_io_info * fio)3434 void f2fs_allocate_data_block(struct f2fs_sb_info *sbi, struct page *page,
3435 block_t old_blkaddr, block_t *new_blkaddr,
3436 struct f2fs_summary *sum, int type,
3437 struct f2fs_io_info *fio)
3438 {
3439 struct sit_info *sit_i = SIT_I(sbi);
3440 struct curseg_info *curseg = CURSEG_I(sbi, type);
3441 unsigned long long old_mtime;
3442 bool from_gc = (type == CURSEG_ALL_DATA_ATGC);
3443 struct seg_entry *se = NULL;
3444
3445 f2fs_down_read(&SM_I(sbi)->curseg_lock);
3446
3447 mutex_lock(&curseg->curseg_mutex);
3448 down_write(&sit_i->sentry_lock);
3449
3450 if (from_gc) {
3451 f2fs_bug_on(sbi, GET_SEGNO(sbi, old_blkaddr) == NULL_SEGNO);
3452 se = get_seg_entry(sbi, GET_SEGNO(sbi, old_blkaddr));
3453 sanity_check_seg_type(sbi, se->type);
3454 f2fs_bug_on(sbi, IS_NODESEG(se->type));
3455 }
3456 *new_blkaddr = NEXT_FREE_BLKADDR(sbi, curseg);
3457
3458 f2fs_bug_on(sbi, curseg->next_blkoff >= sbi->blocks_per_seg);
3459
3460 f2fs_wait_discard_bio(sbi, *new_blkaddr);
3461
3462 /*
3463 * __add_sum_entry should be resided under the curseg_mutex
3464 * because, this function updates a summary entry in the
3465 * current summary block.
3466 */
3467 __add_sum_entry(sbi, type, sum);
3468
3469 __refresh_next_blkoff(sbi, curseg);
3470
3471 stat_inc_block_count(sbi, curseg);
3472
3473 if (from_gc) {
3474 old_mtime = get_segment_mtime(sbi, old_blkaddr);
3475 } else {
3476 update_segment_mtime(sbi, old_blkaddr, 0);
3477 old_mtime = 0;
3478 }
3479 update_segment_mtime(sbi, *new_blkaddr, old_mtime);
3480
3481 /*
3482 * SIT information should be updated before segment allocation,
3483 * since SSR needs latest valid block information.
3484 */
3485 update_sit_entry(sbi, *new_blkaddr, 1);
3486 if (GET_SEGNO(sbi, old_blkaddr) != NULL_SEGNO)
3487 update_sit_entry(sbi, old_blkaddr, -1);
3488
3489 if (!__has_curseg_space(sbi, curseg)) {
3490 if (from_gc)
3491 get_atssr_segment(sbi, type, se->type,
3492 AT_SSR, se->mtime);
3493 else
3494 sit_i->s_ops->allocate_segment(sbi, type, false);
3495 }
3496 /*
3497 * segment dirty status should be updated after segment allocation,
3498 * so we just need to update status only one time after previous
3499 * segment being closed.
3500 */
3501 locate_dirty_segment(sbi, GET_SEGNO(sbi, old_blkaddr));
3502 locate_dirty_segment(sbi, GET_SEGNO(sbi, *new_blkaddr));
3503
3504 up_write(&sit_i->sentry_lock);
3505
3506 if (page && IS_NODESEG(type)) {
3507 fill_node_footer_blkaddr(page, NEXT_FREE_BLKADDR(sbi, curseg));
3508
3509 f2fs_inode_chksum_set(sbi, page);
3510 }
3511
3512 if (fio) {
3513 struct f2fs_bio_info *io;
3514
3515 if (F2FS_IO_ALIGNED(sbi))
3516 fio->retry = false;
3517
3518 INIT_LIST_HEAD(&fio->list);
3519 fio->in_list = true;
3520 io = sbi->write_io[fio->type] + fio->temp;
3521 spin_lock(&io->io_lock);
3522 list_add_tail(&fio->list, &io->io_list);
3523 spin_unlock(&io->io_lock);
3524 }
3525
3526 mutex_unlock(&curseg->curseg_mutex);
3527
3528 f2fs_up_read(&SM_I(sbi)->curseg_lock);
3529 }
3530
f2fs_update_device_state(struct f2fs_sb_info * sbi,nid_t ino,block_t blkaddr,unsigned int blkcnt)3531 void f2fs_update_device_state(struct f2fs_sb_info *sbi, nid_t ino,
3532 block_t blkaddr, unsigned int blkcnt)
3533 {
3534 if (!f2fs_is_multi_device(sbi))
3535 return;
3536
3537 while (1) {
3538 unsigned int devidx = f2fs_target_device_index(sbi, blkaddr);
3539 unsigned int blks = FDEV(devidx).end_blk - blkaddr + 1;
3540
3541 /* update device state for fsync */
3542 f2fs_set_dirty_device(sbi, ino, devidx, FLUSH_INO);
3543
3544 /* update device state for checkpoint */
3545 if (!f2fs_test_bit(devidx, (char *)&sbi->dirty_device)) {
3546 spin_lock(&sbi->dev_lock);
3547 f2fs_set_bit(devidx, (char *)&sbi->dirty_device);
3548 spin_unlock(&sbi->dev_lock);
3549 }
3550
3551 if (blkcnt <= blks)
3552 break;
3553 blkcnt -= blks;
3554 blkaddr += blks;
3555 }
3556 }
3557
do_write_page(struct f2fs_summary * sum,struct f2fs_io_info * fio)3558 static void do_write_page(struct f2fs_summary *sum, struct f2fs_io_info *fio)
3559 {
3560 int type = __get_segment_type(fio);
3561 bool keep_order = (f2fs_lfs_mode(fio->sbi) && type == CURSEG_COLD_DATA);
3562
3563 if (keep_order)
3564 f2fs_down_read(&fio->sbi->io_order_lock);
3565 reallocate:
3566 f2fs_allocate_data_block(fio->sbi, fio->page, fio->old_blkaddr,
3567 &fio->new_blkaddr, sum, type, fio);
3568 if (GET_SEGNO(fio->sbi, fio->old_blkaddr) != NULL_SEGNO) {
3569 invalidate_mapping_pages(META_MAPPING(fio->sbi),
3570 fio->old_blkaddr, fio->old_blkaddr);
3571 f2fs_invalidate_compress_page(fio->sbi, fio->old_blkaddr);
3572 }
3573
3574 /* writeout dirty page into bdev */
3575 f2fs_submit_page_write(fio);
3576 if (fio->retry) {
3577 fio->old_blkaddr = fio->new_blkaddr;
3578 goto reallocate;
3579 }
3580
3581 f2fs_update_device_state(fio->sbi, fio->ino, fio->new_blkaddr, 1);
3582
3583 if (keep_order)
3584 f2fs_up_read(&fio->sbi->io_order_lock);
3585 }
3586
f2fs_do_write_meta_page(struct f2fs_sb_info * sbi,struct page * page,enum iostat_type io_type)3587 void f2fs_do_write_meta_page(struct f2fs_sb_info *sbi, struct page *page,
3588 enum iostat_type io_type)
3589 {
3590 struct f2fs_io_info fio = {
3591 .sbi = sbi,
3592 .type = META,
3593 .temp = HOT,
3594 .op = REQ_OP_WRITE,
3595 .op_flags = REQ_SYNC | REQ_META | REQ_PRIO,
3596 .old_blkaddr = page->index,
3597 .new_blkaddr = page->index,
3598 .page = page,
3599 .encrypted_page = NULL,
3600 .in_list = false,
3601 };
3602
3603 if (unlikely(page->index >= MAIN_BLKADDR(sbi)))
3604 fio.op_flags &= ~REQ_META;
3605
3606 set_page_writeback(page);
3607 ClearPageError(page);
3608 f2fs_submit_page_write(&fio);
3609
3610 stat_inc_meta_count(sbi, page->index);
3611 f2fs_update_iostat(sbi, io_type, F2FS_BLKSIZE);
3612 }
3613
f2fs_do_write_node_page(unsigned int nid,struct f2fs_io_info * fio)3614 void f2fs_do_write_node_page(unsigned int nid, struct f2fs_io_info *fio)
3615 {
3616 struct f2fs_summary sum;
3617
3618 set_summary(&sum, nid, 0, 0);
3619 do_write_page(&sum, fio);
3620
3621 f2fs_update_iostat(fio->sbi, fio->io_type, F2FS_BLKSIZE);
3622 }
3623
f2fs_outplace_write_data(struct dnode_of_data * dn,struct f2fs_io_info * fio)3624 void f2fs_outplace_write_data(struct dnode_of_data *dn,
3625 struct f2fs_io_info *fio)
3626 {
3627 struct f2fs_sb_info *sbi = fio->sbi;
3628 struct f2fs_summary sum;
3629
3630 f2fs_bug_on(sbi, dn->data_blkaddr == NULL_ADDR);
3631 set_summary(&sum, dn->nid, dn->ofs_in_node, fio->version);
3632 do_write_page(&sum, fio);
3633 f2fs_update_data_blkaddr(dn, fio->new_blkaddr);
3634
3635 f2fs_update_iostat(sbi, fio->io_type, F2FS_BLKSIZE);
3636 }
3637
f2fs_inplace_write_data(struct f2fs_io_info * fio)3638 int f2fs_inplace_write_data(struct f2fs_io_info *fio)
3639 {
3640 int err;
3641 struct f2fs_sb_info *sbi = fio->sbi;
3642 unsigned int segno;
3643
3644 fio->new_blkaddr = fio->old_blkaddr;
3645 /* i/o temperature is needed for passing down write hints */
3646 __get_segment_type(fio);
3647
3648 segno = GET_SEGNO(sbi, fio->new_blkaddr);
3649
3650 if (!IS_DATASEG(get_seg_entry(sbi, segno)->type)) {
3651 set_sbi_flag(sbi, SBI_NEED_FSCK);
3652 f2fs_warn(sbi, "%s: incorrect segment(%u) type, run fsck to fix.",
3653 __func__, segno);
3654 err = -EFSCORRUPTED;
3655 goto drop_bio;
3656 }
3657
3658 if (f2fs_cp_error(sbi)) {
3659 err = -EIO;
3660 goto drop_bio;
3661 }
3662
3663 invalidate_mapping_pages(META_MAPPING(sbi),
3664 fio->new_blkaddr, fio->new_blkaddr);
3665
3666 stat_inc_inplace_blocks(fio->sbi);
3667
3668 if (fio->bio && !(SM_I(sbi)->ipu_policy & (1 << F2FS_IPU_NOCACHE)))
3669 err = f2fs_merge_page_bio(fio);
3670 else
3671 err = f2fs_submit_page_bio(fio);
3672 if (!err) {
3673 f2fs_update_device_state(fio->sbi, fio->ino,
3674 fio->new_blkaddr, 1);
3675 f2fs_update_iostat(fio->sbi, fio->io_type, F2FS_BLKSIZE);
3676 }
3677
3678 return err;
3679 drop_bio:
3680 if (fio->bio && *(fio->bio)) {
3681 struct bio *bio = *(fio->bio);
3682
3683 bio->bi_status = BLK_STS_IOERR;
3684 bio_endio(bio);
3685 *(fio->bio) = NULL;
3686 }
3687 return err;
3688 }
3689
__f2fs_get_curseg(struct f2fs_sb_info * sbi,unsigned int segno)3690 static inline int __f2fs_get_curseg(struct f2fs_sb_info *sbi,
3691 unsigned int segno)
3692 {
3693 int i;
3694
3695 for (i = CURSEG_HOT_DATA; i < NO_CHECK_TYPE; i++) {
3696 if (CURSEG_I(sbi, i)->segno == segno)
3697 break;
3698 }
3699 return i;
3700 }
3701
f2fs_do_replace_block(struct f2fs_sb_info * sbi,struct f2fs_summary * sum,block_t old_blkaddr,block_t new_blkaddr,bool recover_curseg,bool recover_newaddr,bool from_gc)3702 void f2fs_do_replace_block(struct f2fs_sb_info *sbi, struct f2fs_summary *sum,
3703 block_t old_blkaddr, block_t new_blkaddr,
3704 bool recover_curseg, bool recover_newaddr,
3705 bool from_gc)
3706 {
3707 struct sit_info *sit_i = SIT_I(sbi);
3708 struct curseg_info *curseg;
3709 unsigned int segno, old_cursegno;
3710 struct seg_entry *se;
3711 int type;
3712 unsigned short old_blkoff;
3713 unsigned char old_alloc_type;
3714
3715 segno = GET_SEGNO(sbi, new_blkaddr);
3716 se = get_seg_entry(sbi, segno);
3717 type = se->type;
3718
3719 f2fs_down_write(&SM_I(sbi)->curseg_lock);
3720
3721 if (!recover_curseg) {
3722 /* for recovery flow */
3723 if (se->valid_blocks == 0 && !IS_CURSEG(sbi, segno)) {
3724 if (old_blkaddr == NULL_ADDR)
3725 type = CURSEG_COLD_DATA;
3726 else
3727 type = CURSEG_WARM_DATA;
3728 }
3729 } else {
3730 if (IS_CURSEG(sbi, segno)) {
3731 /* se->type is volatile as SSR allocation */
3732 type = __f2fs_get_curseg(sbi, segno);
3733 f2fs_bug_on(sbi, type == NO_CHECK_TYPE);
3734 } else {
3735 type = CURSEG_WARM_DATA;
3736 }
3737 }
3738
3739 f2fs_bug_on(sbi, !IS_DATASEG(type));
3740 curseg = CURSEG_I(sbi, type);
3741
3742 mutex_lock(&curseg->curseg_mutex);
3743 down_write(&sit_i->sentry_lock);
3744
3745 old_cursegno = curseg->segno;
3746 old_blkoff = curseg->next_blkoff;
3747 old_alloc_type = curseg->alloc_type;
3748
3749 /* change the current segment */
3750 if (segno != curseg->segno) {
3751 curseg->next_segno = segno;
3752 change_curseg(sbi, type, true);
3753 }
3754
3755 curseg->next_blkoff = GET_BLKOFF_FROM_SEG0(sbi, new_blkaddr);
3756 __add_sum_entry(sbi, type, sum);
3757
3758 if (!recover_curseg || recover_newaddr) {
3759 if (!from_gc)
3760 update_segment_mtime(sbi, new_blkaddr, 0);
3761 update_sit_entry(sbi, new_blkaddr, 1);
3762 }
3763 if (GET_SEGNO(sbi, old_blkaddr) != NULL_SEGNO) {
3764 invalidate_mapping_pages(META_MAPPING(sbi),
3765 old_blkaddr, old_blkaddr);
3766 f2fs_invalidate_compress_page(sbi, old_blkaddr);
3767 if (!from_gc)
3768 update_segment_mtime(sbi, old_blkaddr, 0);
3769 update_sit_entry(sbi, old_blkaddr, -1);
3770 }
3771
3772 locate_dirty_segment(sbi, GET_SEGNO(sbi, old_blkaddr));
3773 locate_dirty_segment(sbi, GET_SEGNO(sbi, new_blkaddr));
3774
3775 locate_dirty_segment(sbi, old_cursegno);
3776
3777 if (recover_curseg) {
3778 if (old_cursegno != curseg->segno) {
3779 curseg->next_segno = old_cursegno;
3780 change_curseg(sbi, type, true);
3781 }
3782 curseg->next_blkoff = old_blkoff;
3783 curseg->alloc_type = old_alloc_type;
3784 }
3785
3786 up_write(&sit_i->sentry_lock);
3787 mutex_unlock(&curseg->curseg_mutex);
3788 f2fs_up_write(&SM_I(sbi)->curseg_lock);
3789 }
3790
f2fs_replace_block(struct f2fs_sb_info * sbi,struct dnode_of_data * dn,block_t old_addr,block_t new_addr,unsigned char version,bool recover_curseg,bool recover_newaddr)3791 void f2fs_replace_block(struct f2fs_sb_info *sbi, struct dnode_of_data *dn,
3792 block_t old_addr, block_t new_addr,
3793 unsigned char version, bool recover_curseg,
3794 bool recover_newaddr)
3795 {
3796 struct f2fs_summary sum;
3797
3798 set_summary(&sum, dn->nid, dn->ofs_in_node, version);
3799
3800 f2fs_do_replace_block(sbi, &sum, old_addr, new_addr,
3801 recover_curseg, recover_newaddr, false);
3802
3803 f2fs_update_data_blkaddr(dn, new_addr);
3804 }
3805
f2fs_wait_on_page_writeback(struct page * page,enum page_type type,bool ordered,bool locked)3806 void f2fs_wait_on_page_writeback(struct page *page,
3807 enum page_type type, bool ordered, bool locked)
3808 {
3809 if (PageWriteback(page)) {
3810 struct f2fs_sb_info *sbi = F2FS_P_SB(page);
3811
3812 /* submit cached LFS IO */
3813 f2fs_submit_merged_write_cond(sbi, NULL, page, 0, type);
3814 /* sbumit cached IPU IO */
3815 f2fs_submit_merged_ipu_write(sbi, NULL, page);
3816 if (ordered) {
3817 wait_on_page_writeback(page);
3818 f2fs_bug_on(sbi, locked && PageWriteback(page));
3819 } else {
3820 wait_for_stable_page(page);
3821 }
3822 }
3823 }
3824
f2fs_wait_on_block_writeback(struct inode * inode,block_t blkaddr)3825 void f2fs_wait_on_block_writeback(struct inode *inode, block_t blkaddr)
3826 {
3827 struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
3828 struct page *cpage;
3829
3830 if (!f2fs_post_read_required(inode))
3831 return;
3832
3833 if (!__is_valid_data_blkaddr(blkaddr))
3834 return;
3835
3836 cpage = find_lock_page(META_MAPPING(sbi), blkaddr);
3837 if (cpage) {
3838 f2fs_wait_on_page_writeback(cpage, DATA, true, true);
3839 f2fs_put_page(cpage, 1);
3840 }
3841 }
3842
f2fs_wait_on_block_writeback_range(struct inode * inode,block_t blkaddr,block_t len)3843 void f2fs_wait_on_block_writeback_range(struct inode *inode, block_t blkaddr,
3844 block_t len)
3845 {
3846 block_t i;
3847
3848 for (i = 0; i < len; i++)
3849 f2fs_wait_on_block_writeback(inode, blkaddr + i);
3850 }
3851
read_compacted_summaries(struct f2fs_sb_info * sbi)3852 static int read_compacted_summaries(struct f2fs_sb_info *sbi)
3853 {
3854 struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi);
3855 struct curseg_info *seg_i;
3856 unsigned char *kaddr;
3857 struct page *page;
3858 block_t start;
3859 int i, j, offset;
3860
3861 start = start_sum_block(sbi);
3862
3863 page = f2fs_get_meta_page(sbi, start++);
3864 if (IS_ERR(page))
3865 return PTR_ERR(page);
3866 kaddr = (unsigned char *)page_address(page);
3867
3868 /* Step 1: restore nat cache */
3869 seg_i = CURSEG_I(sbi, CURSEG_HOT_DATA);
3870 memcpy(seg_i->journal, kaddr, SUM_JOURNAL_SIZE);
3871
3872 /* Step 2: restore sit cache */
3873 seg_i = CURSEG_I(sbi, CURSEG_COLD_DATA);
3874 memcpy(seg_i->journal, kaddr + SUM_JOURNAL_SIZE, SUM_JOURNAL_SIZE);
3875 offset = 2 * SUM_JOURNAL_SIZE;
3876
3877 /* Step 3: restore summary entries */
3878 for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++) {
3879 unsigned short blk_off;
3880 unsigned int segno;
3881
3882 seg_i = CURSEG_I(sbi, i);
3883 segno = le32_to_cpu(ckpt->cur_data_segno[i]);
3884 blk_off = le16_to_cpu(ckpt->cur_data_blkoff[i]);
3885 seg_i->next_segno = segno;
3886 reset_curseg(sbi, i, 0);
3887 seg_i->alloc_type = ckpt->alloc_type[i];
3888 seg_i->next_blkoff = blk_off;
3889
3890 if (seg_i->alloc_type == SSR)
3891 blk_off = sbi->blocks_per_seg;
3892
3893 for (j = 0; j < blk_off; j++) {
3894 struct f2fs_summary *s;
3895
3896 s = (struct f2fs_summary *)(kaddr + offset);
3897 seg_i->sum_blk->entries[j] = *s;
3898 offset += SUMMARY_SIZE;
3899 if (offset + SUMMARY_SIZE <= PAGE_SIZE -
3900 SUM_FOOTER_SIZE)
3901 continue;
3902
3903 f2fs_put_page(page, 1);
3904 page = NULL;
3905
3906 page = f2fs_get_meta_page(sbi, start++);
3907 if (IS_ERR(page))
3908 return PTR_ERR(page);
3909 kaddr = (unsigned char *)page_address(page);
3910 offset = 0;
3911 }
3912 }
3913 f2fs_put_page(page, 1);
3914 return 0;
3915 }
3916
read_normal_summaries(struct f2fs_sb_info * sbi,int type)3917 static int read_normal_summaries(struct f2fs_sb_info *sbi, int type)
3918 {
3919 struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi);
3920 struct f2fs_summary_block *sum;
3921 struct curseg_info *curseg;
3922 struct page *new;
3923 unsigned short blk_off;
3924 unsigned int segno = 0;
3925 block_t blk_addr = 0;
3926 int err = 0;
3927
3928 /* get segment number and block addr */
3929 if (IS_DATASEG(type)) {
3930 segno = le32_to_cpu(ckpt->cur_data_segno[type]);
3931 blk_off = le16_to_cpu(ckpt->cur_data_blkoff[type -
3932 CURSEG_HOT_DATA]);
3933 if (__exist_node_summaries(sbi))
3934 blk_addr = sum_blk_addr(sbi, NR_CURSEG_PERSIST_TYPE, type);
3935 else
3936 blk_addr = sum_blk_addr(sbi, NR_CURSEG_DATA_TYPE, type);
3937 } else {
3938 segno = le32_to_cpu(ckpt->cur_node_segno[type -
3939 CURSEG_HOT_NODE]);
3940 blk_off = le16_to_cpu(ckpt->cur_node_blkoff[type -
3941 CURSEG_HOT_NODE]);
3942 if (__exist_node_summaries(sbi))
3943 blk_addr = sum_blk_addr(sbi, NR_CURSEG_NODE_TYPE,
3944 type - CURSEG_HOT_NODE);
3945 else
3946 blk_addr = GET_SUM_BLOCK(sbi, segno);
3947 }
3948
3949 new = f2fs_get_meta_page(sbi, blk_addr);
3950 if (IS_ERR(new))
3951 return PTR_ERR(new);
3952 sum = (struct f2fs_summary_block *)page_address(new);
3953
3954 if (IS_NODESEG(type)) {
3955 if (__exist_node_summaries(sbi)) {
3956 struct f2fs_summary *ns = &sum->entries[0];
3957 int i;
3958
3959 for (i = 0; i < sbi->blocks_per_seg; i++, ns++) {
3960 ns->version = 0;
3961 ns->ofs_in_node = 0;
3962 }
3963 } else {
3964 err = f2fs_restore_node_summary(sbi, segno, sum);
3965 if (err)
3966 goto out;
3967 }
3968 }
3969
3970 /* set uncompleted segment to curseg */
3971 curseg = CURSEG_I(sbi, type);
3972 mutex_lock(&curseg->curseg_mutex);
3973
3974 /* update journal info */
3975 down_write(&curseg->journal_rwsem);
3976 memcpy(curseg->journal, &sum->journal, SUM_JOURNAL_SIZE);
3977 up_write(&curseg->journal_rwsem);
3978
3979 memcpy(curseg->sum_blk->entries, sum->entries, SUM_ENTRY_SIZE);
3980 memcpy(&curseg->sum_blk->footer, &sum->footer, SUM_FOOTER_SIZE);
3981 curseg->next_segno = segno;
3982 reset_curseg(sbi, type, 0);
3983 curseg->alloc_type = ckpt->alloc_type[type];
3984 curseg->next_blkoff = blk_off;
3985 mutex_unlock(&curseg->curseg_mutex);
3986 out:
3987 f2fs_put_page(new, 1);
3988 return err;
3989 }
3990
restore_curseg_summaries(struct f2fs_sb_info * sbi)3991 static int restore_curseg_summaries(struct f2fs_sb_info *sbi)
3992 {
3993 struct f2fs_journal *sit_j = CURSEG_I(sbi, CURSEG_COLD_DATA)->journal;
3994 struct f2fs_journal *nat_j = CURSEG_I(sbi, CURSEG_HOT_DATA)->journal;
3995 int type = CURSEG_HOT_DATA;
3996 int err;
3997
3998 if (is_set_ckpt_flags(sbi, CP_COMPACT_SUM_FLAG)) {
3999 int npages = f2fs_npages_for_summary_flush(sbi, true);
4000
4001 if (npages >= 2)
4002 f2fs_ra_meta_pages(sbi, start_sum_block(sbi), npages,
4003 META_CP, true);
4004
4005 /* restore for compacted data summary */
4006 err = read_compacted_summaries(sbi);
4007 if (err)
4008 return err;
4009 type = CURSEG_HOT_NODE;
4010 }
4011
4012 if (__exist_node_summaries(sbi))
4013 f2fs_ra_meta_pages(sbi,
4014 sum_blk_addr(sbi, NR_CURSEG_PERSIST_TYPE, type),
4015 NR_CURSEG_PERSIST_TYPE - type, META_CP, true);
4016
4017 for (; type <= CURSEG_COLD_NODE; type++) {
4018 err = read_normal_summaries(sbi, type);
4019 if (err)
4020 return err;
4021 }
4022
4023 /* sanity check for summary blocks */
4024 if (nats_in_cursum(nat_j) > NAT_JOURNAL_ENTRIES ||
4025 sits_in_cursum(sit_j) > SIT_JOURNAL_ENTRIES) {
4026 f2fs_err(sbi, "invalid journal entries nats %u sits %u",
4027 nats_in_cursum(nat_j), sits_in_cursum(sit_j));
4028 return -EINVAL;
4029 }
4030
4031 return 0;
4032 }
4033
write_compacted_summaries(struct f2fs_sb_info * sbi,block_t blkaddr)4034 static void write_compacted_summaries(struct f2fs_sb_info *sbi, block_t blkaddr)
4035 {
4036 struct page *page;
4037 unsigned char *kaddr;
4038 struct f2fs_summary *summary;
4039 struct curseg_info *seg_i;
4040 int written_size = 0;
4041 int i, j;
4042
4043 page = f2fs_grab_meta_page(sbi, blkaddr++);
4044 kaddr = (unsigned char *)page_address(page);
4045 memset(kaddr, 0, PAGE_SIZE);
4046
4047 /* Step 1: write nat cache */
4048 seg_i = CURSEG_I(sbi, CURSEG_HOT_DATA);
4049 memcpy(kaddr, seg_i->journal, SUM_JOURNAL_SIZE);
4050 written_size += SUM_JOURNAL_SIZE;
4051
4052 /* Step 2: write sit cache */
4053 seg_i = CURSEG_I(sbi, CURSEG_COLD_DATA);
4054 memcpy(kaddr + written_size, seg_i->journal, SUM_JOURNAL_SIZE);
4055 written_size += SUM_JOURNAL_SIZE;
4056
4057 /* Step 3: write summary entries */
4058 for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++) {
4059 unsigned short blkoff;
4060
4061 seg_i = CURSEG_I(sbi, i);
4062 if (sbi->ckpt->alloc_type[i] == SSR)
4063 blkoff = sbi->blocks_per_seg;
4064 else
4065 blkoff = curseg_blkoff(sbi, i);
4066
4067 for (j = 0; j < blkoff; j++) {
4068 if (!page) {
4069 page = f2fs_grab_meta_page(sbi, blkaddr++);
4070 kaddr = (unsigned char *)page_address(page);
4071 memset(kaddr, 0, PAGE_SIZE);
4072 written_size = 0;
4073 }
4074 summary = (struct f2fs_summary *)(kaddr + written_size);
4075 *summary = seg_i->sum_blk->entries[j];
4076 written_size += SUMMARY_SIZE;
4077
4078 if (written_size + SUMMARY_SIZE <= PAGE_SIZE -
4079 SUM_FOOTER_SIZE)
4080 continue;
4081
4082 set_page_dirty(page);
4083 f2fs_put_page(page, 1);
4084 page = NULL;
4085 }
4086 }
4087 if (page) {
4088 set_page_dirty(page);
4089 f2fs_put_page(page, 1);
4090 }
4091 }
4092
write_normal_summaries(struct f2fs_sb_info * sbi,block_t blkaddr,int type)4093 static void write_normal_summaries(struct f2fs_sb_info *sbi,
4094 block_t blkaddr, int type)
4095 {
4096 int i, end;
4097
4098 if (IS_DATASEG(type))
4099 end = type + NR_CURSEG_DATA_TYPE;
4100 else
4101 end = type + NR_CURSEG_NODE_TYPE;
4102
4103 for (i = type; i < end; i++)
4104 write_current_sum_page(sbi, i, blkaddr + (i - type));
4105 }
4106
f2fs_write_data_summaries(struct f2fs_sb_info * sbi,block_t start_blk)4107 void f2fs_write_data_summaries(struct f2fs_sb_info *sbi, block_t start_blk)
4108 {
4109 if (is_set_ckpt_flags(sbi, CP_COMPACT_SUM_FLAG))
4110 write_compacted_summaries(sbi, start_blk);
4111 else
4112 write_normal_summaries(sbi, start_blk, CURSEG_HOT_DATA);
4113 }
4114
f2fs_write_node_summaries(struct f2fs_sb_info * sbi,block_t start_blk)4115 void f2fs_write_node_summaries(struct f2fs_sb_info *sbi, block_t start_blk)
4116 {
4117 write_normal_summaries(sbi, start_blk, CURSEG_HOT_NODE);
4118 }
4119
f2fs_lookup_journal_in_cursum(struct f2fs_journal * journal,int type,unsigned int val,int alloc)4120 int f2fs_lookup_journal_in_cursum(struct f2fs_journal *journal, int type,
4121 unsigned int val, int alloc)
4122 {
4123 int i;
4124
4125 if (type == NAT_JOURNAL) {
4126 for (i = 0; i < nats_in_cursum(journal); i++) {
4127 if (le32_to_cpu(nid_in_journal(journal, i)) == val)
4128 return i;
4129 }
4130 if (alloc && __has_cursum_space(journal, 1, NAT_JOURNAL))
4131 return update_nats_in_cursum(journal, 1);
4132 } else if (type == SIT_JOURNAL) {
4133 for (i = 0; i < sits_in_cursum(journal); i++)
4134 if (le32_to_cpu(segno_in_journal(journal, i)) == val)
4135 return i;
4136 if (alloc && __has_cursum_space(journal, 1, SIT_JOURNAL))
4137 return update_sits_in_cursum(journal, 1);
4138 }
4139 return -1;
4140 }
4141
get_current_sit_page(struct f2fs_sb_info * sbi,unsigned int segno)4142 static struct page *get_current_sit_page(struct f2fs_sb_info *sbi,
4143 unsigned int segno)
4144 {
4145 return f2fs_get_meta_page(sbi, current_sit_addr(sbi, segno));
4146 }
4147
get_next_sit_page(struct f2fs_sb_info * sbi,unsigned int start)4148 static struct page *get_next_sit_page(struct f2fs_sb_info *sbi,
4149 unsigned int start)
4150 {
4151 struct sit_info *sit_i = SIT_I(sbi);
4152 struct page *page;
4153 pgoff_t src_off, dst_off;
4154
4155 src_off = current_sit_addr(sbi, start);
4156 dst_off = next_sit_addr(sbi, src_off);
4157
4158 page = f2fs_grab_meta_page(sbi, dst_off);
4159 seg_info_to_sit_page(sbi, page, start);
4160
4161 set_page_dirty(page);
4162 set_to_next_sit(sit_i, start);
4163
4164 return page;
4165 }
4166
grab_sit_entry_set(void)4167 static struct sit_entry_set *grab_sit_entry_set(void)
4168 {
4169 struct sit_entry_set *ses =
4170 f2fs_kmem_cache_alloc(sit_entry_set_slab,
4171 GFP_NOFS, true, NULL);
4172
4173 ses->entry_cnt = 0;
4174 INIT_LIST_HEAD(&ses->set_list);
4175 return ses;
4176 }
4177
release_sit_entry_set(struct sit_entry_set * ses)4178 static void release_sit_entry_set(struct sit_entry_set *ses)
4179 {
4180 list_del(&ses->set_list);
4181 kmem_cache_free(sit_entry_set_slab, ses);
4182 }
4183
adjust_sit_entry_set(struct sit_entry_set * ses,struct list_head * head)4184 static void adjust_sit_entry_set(struct sit_entry_set *ses,
4185 struct list_head *head)
4186 {
4187 struct sit_entry_set *next = ses;
4188
4189 if (list_is_last(&ses->set_list, head))
4190 return;
4191
4192 list_for_each_entry_continue(next, head, set_list)
4193 if (ses->entry_cnt <= next->entry_cnt)
4194 break;
4195
4196 list_move_tail(&ses->set_list, &next->set_list);
4197 }
4198
add_sit_entry(unsigned int segno,struct list_head * head)4199 static void add_sit_entry(unsigned int segno, struct list_head *head)
4200 {
4201 struct sit_entry_set *ses;
4202 unsigned int start_segno = START_SEGNO(segno);
4203
4204 list_for_each_entry(ses, head, set_list) {
4205 if (ses->start_segno == start_segno) {
4206 ses->entry_cnt++;
4207 adjust_sit_entry_set(ses, head);
4208 return;
4209 }
4210 }
4211
4212 ses = grab_sit_entry_set();
4213
4214 ses->start_segno = start_segno;
4215 ses->entry_cnt++;
4216 list_add(&ses->set_list, head);
4217 }
4218
add_sits_in_set(struct f2fs_sb_info * sbi)4219 static void add_sits_in_set(struct f2fs_sb_info *sbi)
4220 {
4221 struct f2fs_sm_info *sm_info = SM_I(sbi);
4222 struct list_head *set_list = &sm_info->sit_entry_set;
4223 unsigned long *bitmap = SIT_I(sbi)->dirty_sentries_bitmap;
4224 unsigned int segno;
4225
4226 for_each_set_bit(segno, bitmap, MAIN_SEGS(sbi))
4227 add_sit_entry(segno, set_list);
4228 }
4229
remove_sits_in_journal(struct f2fs_sb_info * sbi)4230 static void remove_sits_in_journal(struct f2fs_sb_info *sbi)
4231 {
4232 struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_COLD_DATA);
4233 struct f2fs_journal *journal = curseg->journal;
4234 int i;
4235
4236 down_write(&curseg->journal_rwsem);
4237 for (i = 0; i < sits_in_cursum(journal); i++) {
4238 unsigned int segno;
4239 bool dirtied;
4240
4241 segno = le32_to_cpu(segno_in_journal(journal, i));
4242 dirtied = __mark_sit_entry_dirty(sbi, segno);
4243
4244 if (!dirtied)
4245 add_sit_entry(segno, &SM_I(sbi)->sit_entry_set);
4246 }
4247 update_sits_in_cursum(journal, -i);
4248 up_write(&curseg->journal_rwsem);
4249 }
4250
4251 /*
4252 * CP calls this function, which flushes SIT entries including sit_journal,
4253 * and moves prefree segs to free segs.
4254 */
f2fs_flush_sit_entries(struct f2fs_sb_info * sbi,struct cp_control * cpc)4255 void f2fs_flush_sit_entries(struct f2fs_sb_info *sbi, struct cp_control *cpc)
4256 {
4257 struct sit_info *sit_i = SIT_I(sbi);
4258 unsigned long *bitmap = sit_i->dirty_sentries_bitmap;
4259 struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_COLD_DATA);
4260 struct f2fs_journal *journal = curseg->journal;
4261 struct sit_entry_set *ses, *tmp;
4262 struct list_head *head = &SM_I(sbi)->sit_entry_set;
4263 bool to_journal = !is_sbi_flag_set(sbi, SBI_IS_RESIZEFS);
4264 struct seg_entry *se;
4265
4266 down_write(&sit_i->sentry_lock);
4267
4268 if (!sit_i->dirty_sentries)
4269 goto out;
4270
4271 /*
4272 * add and account sit entries of dirty bitmap in sit entry
4273 * set temporarily
4274 */
4275 add_sits_in_set(sbi);
4276
4277 /*
4278 * if there are no enough space in journal to store dirty sit
4279 * entries, remove all entries from journal and add and account
4280 * them in sit entry set.
4281 */
4282 if (!__has_cursum_space(journal, sit_i->dirty_sentries, SIT_JOURNAL) ||
4283 !to_journal)
4284 remove_sits_in_journal(sbi);
4285
4286 /*
4287 * there are two steps to flush sit entries:
4288 * #1, flush sit entries to journal in current cold data summary block.
4289 * #2, flush sit entries to sit page.
4290 */
4291 list_for_each_entry_safe(ses, tmp, head, set_list) {
4292 struct page *page = NULL;
4293 struct f2fs_sit_block *raw_sit = NULL;
4294 unsigned int start_segno = ses->start_segno;
4295 unsigned int end = min(start_segno + SIT_ENTRY_PER_BLOCK,
4296 (unsigned long)MAIN_SEGS(sbi));
4297 unsigned int segno = start_segno;
4298
4299 if (to_journal &&
4300 !__has_cursum_space(journal, ses->entry_cnt, SIT_JOURNAL))
4301 to_journal = false;
4302
4303 if (to_journal) {
4304 down_write(&curseg->journal_rwsem);
4305 } else {
4306 page = get_next_sit_page(sbi, start_segno);
4307 raw_sit = page_address(page);
4308 }
4309
4310 /* flush dirty sit entries in region of current sit set */
4311 for_each_set_bit_from(segno, bitmap, end) {
4312 int offset, sit_offset;
4313
4314 se = get_seg_entry(sbi, segno);
4315 #ifdef CONFIG_F2FS_CHECK_FS
4316 if (memcmp(se->cur_valid_map, se->cur_valid_map_mir,
4317 SIT_VBLOCK_MAP_SIZE))
4318 f2fs_bug_on(sbi, 1);
4319 #endif
4320
4321 /* add discard candidates */
4322 if (!(cpc->reason & CP_DISCARD)) {
4323 cpc->trim_start = segno;
4324 add_discard_addrs(sbi, cpc, false);
4325 }
4326
4327 if (to_journal) {
4328 offset = f2fs_lookup_journal_in_cursum(journal,
4329 SIT_JOURNAL, segno, 1);
4330 f2fs_bug_on(sbi, offset < 0);
4331 segno_in_journal(journal, offset) =
4332 cpu_to_le32(segno);
4333 seg_info_to_raw_sit(se,
4334 &sit_in_journal(journal, offset));
4335 check_block_count(sbi, segno,
4336 &sit_in_journal(journal, offset));
4337 } else {
4338 sit_offset = SIT_ENTRY_OFFSET(sit_i, segno);
4339 seg_info_to_raw_sit(se,
4340 &raw_sit->entries[sit_offset]);
4341 check_block_count(sbi, segno,
4342 &raw_sit->entries[sit_offset]);
4343 }
4344
4345 __clear_bit(segno, bitmap);
4346 sit_i->dirty_sentries--;
4347 ses->entry_cnt--;
4348 }
4349
4350 if (to_journal)
4351 up_write(&curseg->journal_rwsem);
4352 else
4353 f2fs_put_page(page, 1);
4354
4355 f2fs_bug_on(sbi, ses->entry_cnt);
4356 release_sit_entry_set(ses);
4357 }
4358
4359 f2fs_bug_on(sbi, !list_empty(head));
4360 f2fs_bug_on(sbi, sit_i->dirty_sentries);
4361 out:
4362 if (cpc->reason & CP_DISCARD) {
4363 __u64 trim_start = cpc->trim_start;
4364
4365 for (; cpc->trim_start <= cpc->trim_end; cpc->trim_start++)
4366 add_discard_addrs(sbi, cpc, false);
4367
4368 cpc->trim_start = trim_start;
4369 }
4370 up_write(&sit_i->sentry_lock);
4371
4372 set_prefree_as_free_segments(sbi);
4373 }
4374
build_sit_info(struct f2fs_sb_info * sbi)4375 static int build_sit_info(struct f2fs_sb_info *sbi)
4376 {
4377 struct f2fs_super_block *raw_super = F2FS_RAW_SUPER(sbi);
4378 struct sit_info *sit_i;
4379 unsigned int sit_segs, start;
4380 char *src_bitmap, *bitmap;
4381 unsigned int bitmap_size, main_bitmap_size, sit_bitmap_size;
4382 unsigned int discard_map = f2fs_block_unit_discard(sbi) ? 1 : 0;
4383
4384 /* allocate memory for SIT information */
4385 sit_i = f2fs_kzalloc(sbi, sizeof(struct sit_info), GFP_KERNEL);
4386 if (!sit_i)
4387 return -ENOMEM;
4388
4389 SM_I(sbi)->sit_info = sit_i;
4390
4391 sit_i->sentries =
4392 f2fs_kvzalloc(sbi, array_size(sizeof(struct seg_entry),
4393 MAIN_SEGS(sbi)),
4394 GFP_KERNEL);
4395 if (!sit_i->sentries)
4396 return -ENOMEM;
4397
4398 main_bitmap_size = f2fs_bitmap_size(MAIN_SEGS(sbi));
4399 sit_i->dirty_sentries_bitmap = f2fs_kvzalloc(sbi, main_bitmap_size,
4400 GFP_KERNEL);
4401 if (!sit_i->dirty_sentries_bitmap)
4402 return -ENOMEM;
4403
4404 #ifdef CONFIG_F2FS_CHECK_FS
4405 bitmap_size = MAIN_SEGS(sbi) * SIT_VBLOCK_MAP_SIZE * (3 + discard_map);
4406 #else
4407 bitmap_size = MAIN_SEGS(sbi) * SIT_VBLOCK_MAP_SIZE * (2 + discard_map);
4408 #endif
4409 sit_i->bitmap = f2fs_kvzalloc(sbi, bitmap_size, GFP_KERNEL);
4410 if (!sit_i->bitmap)
4411 return -ENOMEM;
4412
4413 bitmap = sit_i->bitmap;
4414
4415 for (start = 0; start < MAIN_SEGS(sbi); start++) {
4416 sit_i->sentries[start].cur_valid_map = bitmap;
4417 bitmap += SIT_VBLOCK_MAP_SIZE;
4418
4419 sit_i->sentries[start].ckpt_valid_map = bitmap;
4420 bitmap += SIT_VBLOCK_MAP_SIZE;
4421
4422 #ifdef CONFIG_F2FS_CHECK_FS
4423 sit_i->sentries[start].cur_valid_map_mir = bitmap;
4424 bitmap += SIT_VBLOCK_MAP_SIZE;
4425 #endif
4426
4427 if (discard_map) {
4428 sit_i->sentries[start].discard_map = bitmap;
4429 bitmap += SIT_VBLOCK_MAP_SIZE;
4430 }
4431 }
4432
4433 sit_i->tmp_map = f2fs_kzalloc(sbi, SIT_VBLOCK_MAP_SIZE, GFP_KERNEL);
4434 if (!sit_i->tmp_map)
4435 return -ENOMEM;
4436
4437 if (__is_large_section(sbi)) {
4438 sit_i->sec_entries =
4439 f2fs_kvzalloc(sbi, array_size(sizeof(struct sec_entry),
4440 MAIN_SECS(sbi)),
4441 GFP_KERNEL);
4442 if (!sit_i->sec_entries)
4443 return -ENOMEM;
4444 }
4445
4446 /* get information related with SIT */
4447 sit_segs = le32_to_cpu(raw_super->segment_count_sit) >> 1;
4448
4449 /* setup SIT bitmap from ckeckpoint pack */
4450 sit_bitmap_size = __bitmap_size(sbi, SIT_BITMAP);
4451 src_bitmap = __bitmap_ptr(sbi, SIT_BITMAP);
4452
4453 sit_i->sit_bitmap = kmemdup(src_bitmap, sit_bitmap_size, GFP_KERNEL);
4454 if (!sit_i->sit_bitmap)
4455 return -ENOMEM;
4456
4457 #ifdef CONFIG_F2FS_CHECK_FS
4458 sit_i->sit_bitmap_mir = kmemdup(src_bitmap,
4459 sit_bitmap_size, GFP_KERNEL);
4460 if (!sit_i->sit_bitmap_mir)
4461 return -ENOMEM;
4462
4463 sit_i->invalid_segmap = f2fs_kvzalloc(sbi,
4464 main_bitmap_size, GFP_KERNEL);
4465 if (!sit_i->invalid_segmap)
4466 return -ENOMEM;
4467 #endif
4468
4469 /* init SIT information */
4470 sit_i->s_ops = &default_salloc_ops;
4471
4472 sit_i->sit_base_addr = le32_to_cpu(raw_super->sit_blkaddr);
4473 sit_i->sit_blocks = sit_segs << sbi->log_blocks_per_seg;
4474 sit_i->written_valid_blocks = 0;
4475 sit_i->bitmap_size = sit_bitmap_size;
4476 sit_i->dirty_sentries = 0;
4477 sit_i->sents_per_block = SIT_ENTRY_PER_BLOCK;
4478 sit_i->elapsed_time = le64_to_cpu(sbi->ckpt->elapsed_time);
4479 sit_i->mounted_time = ktime_get_boottime_seconds();
4480 init_rwsem(&sit_i->sentry_lock);
4481 return 0;
4482 }
4483
build_free_segmap(struct f2fs_sb_info * sbi)4484 static int build_free_segmap(struct f2fs_sb_info *sbi)
4485 {
4486 struct free_segmap_info *free_i;
4487 unsigned int bitmap_size, sec_bitmap_size;
4488
4489 /* allocate memory for free segmap information */
4490 free_i = f2fs_kzalloc(sbi, sizeof(struct free_segmap_info), GFP_KERNEL);
4491 if (!free_i)
4492 return -ENOMEM;
4493
4494 SM_I(sbi)->free_info = free_i;
4495
4496 bitmap_size = f2fs_bitmap_size(MAIN_SEGS(sbi));
4497 free_i->free_segmap = f2fs_kvmalloc(sbi, bitmap_size, GFP_KERNEL);
4498 if (!free_i->free_segmap)
4499 return -ENOMEM;
4500
4501 sec_bitmap_size = f2fs_bitmap_size(MAIN_SECS(sbi));
4502 free_i->free_secmap = f2fs_kvmalloc(sbi, sec_bitmap_size, GFP_KERNEL);
4503 if (!free_i->free_secmap)
4504 return -ENOMEM;
4505
4506 /* set all segments as dirty temporarily */
4507 memset(free_i->free_segmap, 0xff, bitmap_size);
4508 memset(free_i->free_secmap, 0xff, sec_bitmap_size);
4509
4510 /* init free segmap information */
4511 free_i->start_segno = GET_SEGNO_FROM_SEG0(sbi, MAIN_BLKADDR(sbi));
4512 free_i->free_segments = 0;
4513 free_i->free_sections = 0;
4514 spin_lock_init(&free_i->segmap_lock);
4515 return 0;
4516 }
4517
build_curseg(struct f2fs_sb_info * sbi)4518 static int build_curseg(struct f2fs_sb_info *sbi)
4519 {
4520 struct curseg_info *array;
4521 int i;
4522
4523 array = f2fs_kzalloc(sbi, array_size(NR_CURSEG_TYPE,
4524 sizeof(*array)), GFP_KERNEL);
4525 if (!array)
4526 return -ENOMEM;
4527
4528 SM_I(sbi)->curseg_array = array;
4529
4530 for (i = 0; i < NO_CHECK_TYPE; i++) {
4531 mutex_init(&array[i].curseg_mutex);
4532 array[i].sum_blk = f2fs_kzalloc(sbi, PAGE_SIZE, GFP_KERNEL);
4533 if (!array[i].sum_blk)
4534 return -ENOMEM;
4535 init_rwsem(&array[i].journal_rwsem);
4536 array[i].journal = f2fs_kzalloc(sbi,
4537 sizeof(struct f2fs_journal), GFP_KERNEL);
4538 if (!array[i].journal)
4539 return -ENOMEM;
4540 if (i < NR_PERSISTENT_LOG)
4541 array[i].seg_type = CURSEG_HOT_DATA + i;
4542 else if (i == CURSEG_COLD_DATA_PINNED)
4543 array[i].seg_type = CURSEG_COLD_DATA;
4544 else if (i == CURSEG_ALL_DATA_ATGC)
4545 array[i].seg_type = CURSEG_COLD_DATA;
4546 array[i].segno = NULL_SEGNO;
4547 array[i].next_blkoff = 0;
4548 array[i].inited = false;
4549 }
4550 return restore_curseg_summaries(sbi);
4551 }
4552
build_sit_entries(struct f2fs_sb_info * sbi)4553 static int build_sit_entries(struct f2fs_sb_info *sbi)
4554 {
4555 struct sit_info *sit_i = SIT_I(sbi);
4556 struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_COLD_DATA);
4557 struct f2fs_journal *journal = curseg->journal;
4558 struct seg_entry *se;
4559 struct f2fs_sit_entry sit;
4560 int sit_blk_cnt = SIT_BLK_CNT(sbi);
4561 unsigned int i, start, end;
4562 unsigned int readed, start_blk = 0;
4563 int err = 0;
4564 block_t sit_valid_blocks[2] = {0, 0};
4565
4566 do {
4567 readed = f2fs_ra_meta_pages(sbi, start_blk, BIO_MAX_PAGES,
4568 META_SIT, true);
4569
4570 start = start_blk * sit_i->sents_per_block;
4571 end = (start_blk + readed) * sit_i->sents_per_block;
4572
4573 for (; start < end && start < MAIN_SEGS(sbi); start++) {
4574 struct f2fs_sit_block *sit_blk;
4575 struct page *page;
4576
4577 se = &sit_i->sentries[start];
4578 page = get_current_sit_page(sbi, start);
4579 if (IS_ERR(page))
4580 return PTR_ERR(page);
4581 sit_blk = (struct f2fs_sit_block *)page_address(page);
4582 sit = sit_blk->entries[SIT_ENTRY_OFFSET(sit_i, start)];
4583 f2fs_put_page(page, 1);
4584
4585 err = check_block_count(sbi, start, &sit);
4586 if (err)
4587 return err;
4588 seg_info_from_raw_sit(se, &sit);
4589
4590 if (se->type >= NR_PERSISTENT_LOG) {
4591 f2fs_err(sbi, "Invalid segment type: %u, segno: %u",
4592 se->type, start);
4593 return -EFSCORRUPTED;
4594 }
4595
4596 sit_valid_blocks[SE_PAGETYPE(se)] += se->valid_blocks;
4597
4598 if (f2fs_block_unit_discard(sbi)) {
4599 /* build discard map only one time */
4600 if (is_set_ckpt_flags(sbi, CP_TRIMMED_FLAG)) {
4601 memset(se->discard_map, 0xff,
4602 SIT_VBLOCK_MAP_SIZE);
4603 } else {
4604 memcpy(se->discard_map,
4605 se->cur_valid_map,
4606 SIT_VBLOCK_MAP_SIZE);
4607 sbi->discard_blks +=
4608 sbi->blocks_per_seg -
4609 se->valid_blocks;
4610 }
4611 }
4612
4613 if (__is_large_section(sbi))
4614 get_sec_entry(sbi, start)->valid_blocks +=
4615 se->valid_blocks;
4616 }
4617 start_blk += readed;
4618 } while (start_blk < sit_blk_cnt);
4619
4620 down_read(&curseg->journal_rwsem);
4621 for (i = 0; i < sits_in_cursum(journal); i++) {
4622 unsigned int old_valid_blocks;
4623
4624 start = le32_to_cpu(segno_in_journal(journal, i));
4625 if (start >= MAIN_SEGS(sbi)) {
4626 f2fs_err(sbi, "Wrong journal entry on segno %u",
4627 start);
4628 err = -EFSCORRUPTED;
4629 break;
4630 }
4631
4632 se = &sit_i->sentries[start];
4633 sit = sit_in_journal(journal, i);
4634
4635 old_valid_blocks = se->valid_blocks;
4636
4637 sit_valid_blocks[SE_PAGETYPE(se)] -= old_valid_blocks;
4638
4639 err = check_block_count(sbi, start, &sit);
4640 if (err)
4641 break;
4642 seg_info_from_raw_sit(se, &sit);
4643
4644 if (se->type >= NR_PERSISTENT_LOG) {
4645 f2fs_err(sbi, "Invalid segment type: %u, segno: %u",
4646 se->type, start);
4647 err = -EFSCORRUPTED;
4648 break;
4649 }
4650
4651 sit_valid_blocks[SE_PAGETYPE(se)] += se->valid_blocks;
4652
4653 if (f2fs_block_unit_discard(sbi)) {
4654 if (is_set_ckpt_flags(sbi, CP_TRIMMED_FLAG)) {
4655 memset(se->discard_map, 0xff, SIT_VBLOCK_MAP_SIZE);
4656 } else {
4657 memcpy(se->discard_map, se->cur_valid_map,
4658 SIT_VBLOCK_MAP_SIZE);
4659 sbi->discard_blks += old_valid_blocks;
4660 sbi->discard_blks -= se->valid_blocks;
4661 }
4662 }
4663
4664 if (__is_large_section(sbi)) {
4665 get_sec_entry(sbi, start)->valid_blocks +=
4666 se->valid_blocks;
4667 get_sec_entry(sbi, start)->valid_blocks -=
4668 old_valid_blocks;
4669 }
4670 }
4671 up_read(&curseg->journal_rwsem);
4672
4673 if (err)
4674 return err;
4675
4676 if (sit_valid_blocks[NODE] != valid_node_count(sbi)) {
4677 f2fs_err(sbi, "SIT is corrupted node# %u vs %u",
4678 sit_valid_blocks[NODE], valid_node_count(sbi));
4679 return -EFSCORRUPTED;
4680 }
4681
4682 if (sit_valid_blocks[DATA] + sit_valid_blocks[NODE] >
4683 valid_user_blocks(sbi)) {
4684 f2fs_err(sbi, "SIT is corrupted data# %u %u vs %u",
4685 sit_valid_blocks[DATA], sit_valid_blocks[NODE],
4686 valid_user_blocks(sbi));
4687 return -EFSCORRUPTED;
4688 }
4689
4690 return 0;
4691 }
4692
init_free_segmap(struct f2fs_sb_info * sbi)4693 static void init_free_segmap(struct f2fs_sb_info *sbi)
4694 {
4695 unsigned int start;
4696 int type;
4697 struct seg_entry *sentry;
4698
4699 for (start = 0; start < MAIN_SEGS(sbi); start++) {
4700 if (f2fs_usable_blks_in_seg(sbi, start) == 0)
4701 continue;
4702 sentry = get_seg_entry(sbi, start);
4703 if (!sentry->valid_blocks)
4704 __set_free(sbi, start);
4705 else
4706 SIT_I(sbi)->written_valid_blocks +=
4707 sentry->valid_blocks;
4708 }
4709
4710 /* set use the current segments */
4711 for (type = CURSEG_HOT_DATA; type <= CURSEG_COLD_NODE; type++) {
4712 struct curseg_info *curseg_t = CURSEG_I(sbi, type);
4713
4714 __set_test_and_inuse(sbi, curseg_t->segno);
4715 }
4716 }
4717
init_dirty_segmap(struct f2fs_sb_info * sbi)4718 static void init_dirty_segmap(struct f2fs_sb_info *sbi)
4719 {
4720 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
4721 struct free_segmap_info *free_i = FREE_I(sbi);
4722 unsigned int segno = 0, offset = 0, secno;
4723 block_t valid_blocks, usable_blks_in_seg;
4724 block_t blks_per_sec = BLKS_PER_SEC(sbi);
4725
4726 while (1) {
4727 /* find dirty segment based on free segmap */
4728 segno = find_next_inuse(free_i, MAIN_SEGS(sbi), offset);
4729 if (segno >= MAIN_SEGS(sbi))
4730 break;
4731 offset = segno + 1;
4732 valid_blocks = get_valid_blocks(sbi, segno, false);
4733 usable_blks_in_seg = f2fs_usable_blks_in_seg(sbi, segno);
4734 if (valid_blocks == usable_blks_in_seg || !valid_blocks)
4735 continue;
4736 if (valid_blocks > usable_blks_in_seg) {
4737 f2fs_bug_on(sbi, 1);
4738 continue;
4739 }
4740 mutex_lock(&dirty_i->seglist_lock);
4741 __locate_dirty_segment(sbi, segno, DIRTY);
4742 mutex_unlock(&dirty_i->seglist_lock);
4743 }
4744
4745 if (!__is_large_section(sbi))
4746 return;
4747
4748 mutex_lock(&dirty_i->seglist_lock);
4749 for (segno = 0; segno < MAIN_SEGS(sbi); segno += sbi->segs_per_sec) {
4750 valid_blocks = get_valid_blocks(sbi, segno, true);
4751 secno = GET_SEC_FROM_SEG(sbi, segno);
4752
4753 if (!valid_blocks || valid_blocks == blks_per_sec)
4754 continue;
4755 if (IS_CURSEC(sbi, secno))
4756 continue;
4757 set_bit(secno, dirty_i->dirty_secmap);
4758 }
4759 mutex_unlock(&dirty_i->seglist_lock);
4760 }
4761
init_victim_secmap(struct f2fs_sb_info * sbi)4762 static int init_victim_secmap(struct f2fs_sb_info *sbi)
4763 {
4764 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
4765 unsigned int bitmap_size = f2fs_bitmap_size(MAIN_SECS(sbi));
4766
4767 dirty_i->victim_secmap = f2fs_kvzalloc(sbi, bitmap_size, GFP_KERNEL);
4768 if (!dirty_i->victim_secmap)
4769 return -ENOMEM;
4770
4771 dirty_i->pinned_secmap = f2fs_kvzalloc(sbi, bitmap_size, GFP_KERNEL);
4772 if (!dirty_i->pinned_secmap)
4773 return -ENOMEM;
4774
4775 dirty_i->pinned_secmap_cnt = 0;
4776 dirty_i->enable_pin_section = true;
4777 return 0;
4778 }
4779
build_dirty_segmap(struct f2fs_sb_info * sbi)4780 static int build_dirty_segmap(struct f2fs_sb_info *sbi)
4781 {
4782 struct dirty_seglist_info *dirty_i;
4783 unsigned int bitmap_size, i;
4784
4785 /* allocate memory for dirty segments list information */
4786 dirty_i = f2fs_kzalloc(sbi, sizeof(struct dirty_seglist_info),
4787 GFP_KERNEL);
4788 if (!dirty_i)
4789 return -ENOMEM;
4790
4791 SM_I(sbi)->dirty_info = dirty_i;
4792 mutex_init(&dirty_i->seglist_lock);
4793
4794 bitmap_size = f2fs_bitmap_size(MAIN_SEGS(sbi));
4795
4796 for (i = 0; i < NR_DIRTY_TYPE; i++) {
4797 dirty_i->dirty_segmap[i] = f2fs_kvzalloc(sbi, bitmap_size,
4798 GFP_KERNEL);
4799 if (!dirty_i->dirty_segmap[i])
4800 return -ENOMEM;
4801 }
4802
4803 if (__is_large_section(sbi)) {
4804 bitmap_size = f2fs_bitmap_size(MAIN_SECS(sbi));
4805 dirty_i->dirty_secmap = f2fs_kvzalloc(sbi,
4806 bitmap_size, GFP_KERNEL);
4807 if (!dirty_i->dirty_secmap)
4808 return -ENOMEM;
4809 }
4810
4811 init_dirty_segmap(sbi);
4812 return init_victim_secmap(sbi);
4813 }
4814
sanity_check_curseg(struct f2fs_sb_info * sbi)4815 static int sanity_check_curseg(struct f2fs_sb_info *sbi)
4816 {
4817 int i;
4818
4819 /*
4820 * In LFS/SSR curseg, .next_blkoff should point to an unused blkaddr;
4821 * In LFS curseg, all blkaddr after .next_blkoff should be unused.
4822 */
4823 for (i = 0; i < NR_PERSISTENT_LOG; i++) {
4824 struct curseg_info *curseg = CURSEG_I(sbi, i);
4825 struct seg_entry *se = get_seg_entry(sbi, curseg->segno);
4826 unsigned int blkofs = curseg->next_blkoff;
4827
4828 if (f2fs_sb_has_readonly(sbi) &&
4829 i != CURSEG_HOT_DATA && i != CURSEG_HOT_NODE)
4830 continue;
4831
4832 sanity_check_seg_type(sbi, curseg->seg_type);
4833
4834 if (curseg->alloc_type != LFS && curseg->alloc_type != SSR) {
4835 f2fs_err(sbi,
4836 "Current segment has invalid alloc_type:%d",
4837 curseg->alloc_type);
4838 return -EFSCORRUPTED;
4839 }
4840
4841 if (f2fs_test_bit(blkofs, se->cur_valid_map))
4842 goto out;
4843
4844 if (curseg->alloc_type == SSR)
4845 continue;
4846
4847 for (blkofs += 1; blkofs < sbi->blocks_per_seg; blkofs++) {
4848 if (!f2fs_test_bit(blkofs, se->cur_valid_map))
4849 continue;
4850 out:
4851 f2fs_err(sbi,
4852 "Current segment's next free block offset is inconsistent with bitmap, logtype:%u, segno:%u, type:%u, next_blkoff:%u, blkofs:%u",
4853 i, curseg->segno, curseg->alloc_type,
4854 curseg->next_blkoff, blkofs);
4855 return -EFSCORRUPTED;
4856 }
4857 }
4858 return 0;
4859 }
4860
4861 #ifdef CONFIG_BLK_DEV_ZONED
4862
check_zone_write_pointer(struct f2fs_sb_info * sbi,struct f2fs_dev_info * fdev,struct blk_zone * zone)4863 static int check_zone_write_pointer(struct f2fs_sb_info *sbi,
4864 struct f2fs_dev_info *fdev,
4865 struct blk_zone *zone)
4866 {
4867 unsigned int wp_segno, wp_blkoff, zone_secno, zone_segno, segno;
4868 block_t zone_block, wp_block, last_valid_block;
4869 unsigned int log_sectors_per_block = sbi->log_blocksize - SECTOR_SHIFT;
4870 int i, s, b, ret;
4871 struct seg_entry *se;
4872
4873 if (zone->type != BLK_ZONE_TYPE_SEQWRITE_REQ)
4874 return 0;
4875
4876 wp_block = fdev->start_blk + (zone->wp >> log_sectors_per_block);
4877 wp_segno = GET_SEGNO(sbi, wp_block);
4878 wp_blkoff = wp_block - START_BLOCK(sbi, wp_segno);
4879 zone_block = fdev->start_blk + (zone->start >> log_sectors_per_block);
4880 zone_segno = GET_SEGNO(sbi, zone_block);
4881 zone_secno = GET_SEC_FROM_SEG(sbi, zone_segno);
4882
4883 if (zone_segno >= MAIN_SEGS(sbi))
4884 return 0;
4885
4886 /*
4887 * Skip check of zones cursegs point to, since
4888 * fix_curseg_write_pointer() checks them.
4889 */
4890 for (i = 0; i < NO_CHECK_TYPE; i++)
4891 if (zone_secno == GET_SEC_FROM_SEG(sbi,
4892 CURSEG_I(sbi, i)->segno))
4893 return 0;
4894
4895 /*
4896 * Get last valid block of the zone.
4897 */
4898 last_valid_block = zone_block - 1;
4899 for (s = sbi->segs_per_sec - 1; s >= 0; s--) {
4900 segno = zone_segno + s;
4901 se = get_seg_entry(sbi, segno);
4902 for (b = sbi->blocks_per_seg - 1; b >= 0; b--)
4903 if (f2fs_test_bit(b, se->cur_valid_map)) {
4904 last_valid_block = START_BLOCK(sbi, segno) + b;
4905 break;
4906 }
4907 if (last_valid_block >= zone_block)
4908 break;
4909 }
4910
4911 /*
4912 * If last valid block is beyond the write pointer, report the
4913 * inconsistency. This inconsistency does not cause write error
4914 * because the zone will not be selected for write operation until
4915 * it get discarded. Just report it.
4916 */
4917 if (last_valid_block >= wp_block) {
4918 f2fs_notice(sbi, "Valid block beyond write pointer: "
4919 "valid block[0x%x,0x%x] wp[0x%x,0x%x]",
4920 GET_SEGNO(sbi, last_valid_block),
4921 GET_BLKOFF_FROM_SEG0(sbi, last_valid_block),
4922 wp_segno, wp_blkoff);
4923 return 0;
4924 }
4925
4926 /*
4927 * If there is no valid block in the zone and if write pointer is
4928 * not at zone start, reset the write pointer.
4929 */
4930 if (last_valid_block + 1 == zone_block && zone->wp != zone->start) {
4931 f2fs_notice(sbi,
4932 "Zone without valid block has non-zero write "
4933 "pointer. Reset the write pointer: wp[0x%x,0x%x]",
4934 wp_segno, wp_blkoff);
4935 ret = __f2fs_issue_discard_zone(sbi, fdev->bdev, zone_block,
4936 zone->len >> log_sectors_per_block);
4937 if (ret) {
4938 f2fs_err(sbi, "Discard zone failed: %s (errno=%d)",
4939 fdev->path, ret);
4940 return ret;
4941 }
4942 }
4943
4944 return 0;
4945 }
4946
get_target_zoned_dev(struct f2fs_sb_info * sbi,block_t zone_blkaddr)4947 static struct f2fs_dev_info *get_target_zoned_dev(struct f2fs_sb_info *sbi,
4948 block_t zone_blkaddr)
4949 {
4950 int i;
4951
4952 for (i = 0; i < sbi->s_ndevs; i++) {
4953 if (!bdev_is_zoned(FDEV(i).bdev))
4954 continue;
4955 if (sbi->s_ndevs == 1 || (FDEV(i).start_blk <= zone_blkaddr &&
4956 zone_blkaddr <= FDEV(i).end_blk))
4957 return &FDEV(i);
4958 }
4959
4960 return NULL;
4961 }
4962
report_one_zone_cb(struct blk_zone * zone,unsigned int idx,void * data)4963 static int report_one_zone_cb(struct blk_zone *zone, unsigned int idx,
4964 void *data)
4965 {
4966 memcpy(data, zone, sizeof(struct blk_zone));
4967 return 0;
4968 }
4969
fix_curseg_write_pointer(struct f2fs_sb_info * sbi,int type)4970 static int fix_curseg_write_pointer(struct f2fs_sb_info *sbi, int type)
4971 {
4972 struct curseg_info *cs = CURSEG_I(sbi, type);
4973 struct f2fs_dev_info *zbd;
4974 struct blk_zone zone;
4975 unsigned int cs_section, wp_segno, wp_blkoff, wp_sector_off;
4976 block_t cs_zone_block, wp_block;
4977 unsigned int log_sectors_per_block = sbi->log_blocksize - SECTOR_SHIFT;
4978 sector_t zone_sector;
4979 int err;
4980
4981 cs_section = GET_SEC_FROM_SEG(sbi, cs->segno);
4982 cs_zone_block = START_BLOCK(sbi, GET_SEG_FROM_SEC(sbi, cs_section));
4983
4984 zbd = get_target_zoned_dev(sbi, cs_zone_block);
4985 if (!zbd)
4986 return 0;
4987
4988 /* report zone for the sector the curseg points to */
4989 zone_sector = (sector_t)(cs_zone_block - zbd->start_blk)
4990 << log_sectors_per_block;
4991 err = blkdev_report_zones(zbd->bdev, zone_sector, 1,
4992 report_one_zone_cb, &zone);
4993 if (err != 1) {
4994 f2fs_err(sbi, "Report zone failed: %s errno=(%d)",
4995 zbd->path, err);
4996 return err;
4997 }
4998
4999 if (zone.type != BLK_ZONE_TYPE_SEQWRITE_REQ)
5000 return 0;
5001
5002 wp_block = zbd->start_blk + (zone.wp >> log_sectors_per_block);
5003 wp_segno = GET_SEGNO(sbi, wp_block);
5004 wp_blkoff = wp_block - START_BLOCK(sbi, wp_segno);
5005 wp_sector_off = zone.wp & GENMASK(log_sectors_per_block - 1, 0);
5006
5007 if (cs->segno == wp_segno && cs->next_blkoff == wp_blkoff &&
5008 wp_sector_off == 0)
5009 return 0;
5010
5011 f2fs_notice(sbi, "Unaligned curseg[%d] with write pointer: "
5012 "curseg[0x%x,0x%x] wp[0x%x,0x%x]",
5013 type, cs->segno, cs->next_blkoff, wp_segno, wp_blkoff);
5014
5015 f2fs_notice(sbi, "Assign new section to curseg[%d]: "
5016 "curseg[0x%x,0x%x]", type, cs->segno, cs->next_blkoff);
5017
5018 f2fs_allocate_new_section(sbi, type, true);
5019
5020 /* check consistency of the zone curseg pointed to */
5021 if (check_zone_write_pointer(sbi, zbd, &zone))
5022 return -EIO;
5023
5024 /* check newly assigned zone */
5025 cs_section = GET_SEC_FROM_SEG(sbi, cs->segno);
5026 cs_zone_block = START_BLOCK(sbi, GET_SEG_FROM_SEC(sbi, cs_section));
5027
5028 zbd = get_target_zoned_dev(sbi, cs_zone_block);
5029 if (!zbd)
5030 return 0;
5031
5032 zone_sector = (sector_t)(cs_zone_block - zbd->start_blk)
5033 << log_sectors_per_block;
5034 err = blkdev_report_zones(zbd->bdev, zone_sector, 1,
5035 report_one_zone_cb, &zone);
5036 if (err != 1) {
5037 f2fs_err(sbi, "Report zone failed: %s errno=(%d)",
5038 zbd->path, err);
5039 return err;
5040 }
5041
5042 if (zone.type != BLK_ZONE_TYPE_SEQWRITE_REQ)
5043 return 0;
5044
5045 if (zone.wp != zone.start) {
5046 f2fs_notice(sbi,
5047 "New zone for curseg[%d] is not yet discarded. "
5048 "Reset the zone: curseg[0x%x,0x%x]",
5049 type, cs->segno, cs->next_blkoff);
5050 err = __f2fs_issue_discard_zone(sbi, zbd->bdev,
5051 zone_sector >> log_sectors_per_block,
5052 zone.len >> log_sectors_per_block);
5053 if (err) {
5054 f2fs_err(sbi, "Discard zone failed: %s (errno=%d)",
5055 zbd->path, err);
5056 return err;
5057 }
5058 }
5059
5060 return 0;
5061 }
5062
f2fs_fix_curseg_write_pointer(struct f2fs_sb_info * sbi)5063 int f2fs_fix_curseg_write_pointer(struct f2fs_sb_info *sbi)
5064 {
5065 int i, ret;
5066
5067 for (i = 0; i < NR_PERSISTENT_LOG; i++) {
5068 ret = fix_curseg_write_pointer(sbi, i);
5069 if (ret)
5070 return ret;
5071 }
5072
5073 return 0;
5074 }
5075
5076 struct check_zone_write_pointer_args {
5077 struct f2fs_sb_info *sbi;
5078 struct f2fs_dev_info *fdev;
5079 };
5080
check_zone_write_pointer_cb(struct blk_zone * zone,unsigned int idx,void * data)5081 static int check_zone_write_pointer_cb(struct blk_zone *zone, unsigned int idx,
5082 void *data)
5083 {
5084 struct check_zone_write_pointer_args *args;
5085
5086 args = (struct check_zone_write_pointer_args *)data;
5087
5088 return check_zone_write_pointer(args->sbi, args->fdev, zone);
5089 }
5090
f2fs_check_write_pointer(struct f2fs_sb_info * sbi)5091 int f2fs_check_write_pointer(struct f2fs_sb_info *sbi)
5092 {
5093 int i, ret;
5094 struct check_zone_write_pointer_args args;
5095
5096 for (i = 0; i < sbi->s_ndevs; i++) {
5097 if (!bdev_is_zoned(FDEV(i).bdev))
5098 continue;
5099
5100 args.sbi = sbi;
5101 args.fdev = &FDEV(i);
5102 ret = blkdev_report_zones(FDEV(i).bdev, 0, BLK_ALL_ZONES,
5103 check_zone_write_pointer_cb, &args);
5104 if (ret < 0)
5105 return ret;
5106 }
5107
5108 return 0;
5109 }
5110
5111 /*
5112 * Return the number of usable blocks in a segment. The number of blocks
5113 * returned is always equal to the number of blocks in a segment for
5114 * segments fully contained within a sequential zone capacity or a
5115 * conventional zone. For segments partially contained in a sequential
5116 * zone capacity, the number of usable blocks up to the zone capacity
5117 * is returned. 0 is returned in all other cases.
5118 */
f2fs_usable_zone_blks_in_seg(struct f2fs_sb_info * sbi,unsigned int segno)5119 static inline unsigned int f2fs_usable_zone_blks_in_seg(
5120 struct f2fs_sb_info *sbi, unsigned int segno)
5121 {
5122 block_t seg_start, sec_start_blkaddr, sec_cap_blkaddr;
5123 unsigned int secno;
5124
5125 if (!sbi->unusable_blocks_per_sec)
5126 return sbi->blocks_per_seg;
5127
5128 secno = GET_SEC_FROM_SEG(sbi, segno);
5129 seg_start = START_BLOCK(sbi, segno);
5130 sec_start_blkaddr = START_BLOCK(sbi, GET_SEG_FROM_SEC(sbi, secno));
5131 sec_cap_blkaddr = sec_start_blkaddr + CAP_BLKS_PER_SEC(sbi);
5132
5133 /*
5134 * If segment starts before zone capacity and spans beyond
5135 * zone capacity, then usable blocks are from seg start to
5136 * zone capacity. If the segment starts after the zone capacity,
5137 * then there are no usable blocks.
5138 */
5139 if (seg_start >= sec_cap_blkaddr)
5140 return 0;
5141 if (seg_start + sbi->blocks_per_seg > sec_cap_blkaddr)
5142 return sec_cap_blkaddr - seg_start;
5143
5144 return sbi->blocks_per_seg;
5145 }
5146 #else
f2fs_fix_curseg_write_pointer(struct f2fs_sb_info * sbi)5147 int f2fs_fix_curseg_write_pointer(struct f2fs_sb_info *sbi)
5148 {
5149 return 0;
5150 }
5151
f2fs_check_write_pointer(struct f2fs_sb_info * sbi)5152 int f2fs_check_write_pointer(struct f2fs_sb_info *sbi)
5153 {
5154 return 0;
5155 }
5156
f2fs_usable_zone_blks_in_seg(struct f2fs_sb_info * sbi,unsigned int segno)5157 static inline unsigned int f2fs_usable_zone_blks_in_seg(struct f2fs_sb_info *sbi,
5158 unsigned int segno)
5159 {
5160 return 0;
5161 }
5162
5163 #endif
f2fs_usable_blks_in_seg(struct f2fs_sb_info * sbi,unsigned int segno)5164 unsigned int f2fs_usable_blks_in_seg(struct f2fs_sb_info *sbi,
5165 unsigned int segno)
5166 {
5167 if (f2fs_sb_has_blkzoned(sbi))
5168 return f2fs_usable_zone_blks_in_seg(sbi, segno);
5169
5170 return sbi->blocks_per_seg;
5171 }
5172
f2fs_usable_segs_in_sec(struct f2fs_sb_info * sbi,unsigned int segno)5173 unsigned int f2fs_usable_segs_in_sec(struct f2fs_sb_info *sbi,
5174 unsigned int segno)
5175 {
5176 if (f2fs_sb_has_blkzoned(sbi))
5177 return CAP_SEGS_PER_SEC(sbi);
5178
5179 return sbi->segs_per_sec;
5180 }
5181
5182 /*
5183 * Update min, max modified time for cost-benefit GC algorithm
5184 */
init_min_max_mtime(struct f2fs_sb_info * sbi)5185 static void init_min_max_mtime(struct f2fs_sb_info *sbi)
5186 {
5187 struct sit_info *sit_i = SIT_I(sbi);
5188 unsigned int segno;
5189
5190 down_write(&sit_i->sentry_lock);
5191
5192 sit_i->min_mtime = ULLONG_MAX;
5193
5194 for (segno = 0; segno < MAIN_SEGS(sbi); segno += sbi->segs_per_sec) {
5195 unsigned int i;
5196 unsigned long long mtime = 0;
5197
5198 for (i = 0; i < sbi->segs_per_sec; i++)
5199 mtime += get_seg_entry(sbi, segno + i)->mtime;
5200
5201 mtime = div_u64(mtime, sbi->segs_per_sec);
5202
5203 if (sit_i->min_mtime > mtime)
5204 sit_i->min_mtime = mtime;
5205 }
5206 sit_i->max_mtime = get_mtime(sbi, false);
5207 sit_i->dirty_max_mtime = 0;
5208 up_write(&sit_i->sentry_lock);
5209 }
5210
f2fs_build_segment_manager(struct f2fs_sb_info * sbi)5211 int f2fs_build_segment_manager(struct f2fs_sb_info *sbi)
5212 {
5213 struct f2fs_super_block *raw_super = F2FS_RAW_SUPER(sbi);
5214 struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi);
5215 struct f2fs_sm_info *sm_info;
5216 int err;
5217
5218 sm_info = f2fs_kzalloc(sbi, sizeof(struct f2fs_sm_info), GFP_KERNEL);
5219 if (!sm_info)
5220 return -ENOMEM;
5221
5222 /* init sm info */
5223 sbi->sm_info = sm_info;
5224 sm_info->seg0_blkaddr = le32_to_cpu(raw_super->segment0_blkaddr);
5225 sm_info->main_blkaddr = le32_to_cpu(raw_super->main_blkaddr);
5226 sm_info->segment_count = le32_to_cpu(raw_super->segment_count);
5227 sm_info->reserved_segments = le32_to_cpu(ckpt->rsvd_segment_count);
5228 sm_info->ovp_segments = le32_to_cpu(ckpt->overprov_segment_count);
5229 sm_info->main_segments = le32_to_cpu(raw_super->segment_count_main);
5230 sm_info->ssa_blkaddr = le32_to_cpu(raw_super->ssa_blkaddr);
5231 sm_info->rec_prefree_segments = sm_info->main_segments *
5232 DEF_RECLAIM_PREFREE_SEGMENTS / 100;
5233 if (sm_info->rec_prefree_segments > DEF_MAX_RECLAIM_PREFREE_SEGMENTS)
5234 sm_info->rec_prefree_segments = DEF_MAX_RECLAIM_PREFREE_SEGMENTS;
5235
5236 if (!f2fs_lfs_mode(sbi))
5237 sm_info->ipu_policy = 1 << F2FS_IPU_FSYNC;
5238 sm_info->min_ipu_util = DEF_MIN_IPU_UTIL;
5239 sm_info->min_fsync_blocks = DEF_MIN_FSYNC_BLOCKS;
5240 sm_info->min_seq_blocks = sbi->blocks_per_seg;
5241 sm_info->min_hot_blocks = DEF_MIN_HOT_BLOCKS;
5242 sm_info->min_ssr_sections = reserved_sections(sbi);
5243
5244 INIT_LIST_HEAD(&sm_info->sit_entry_set);
5245
5246 init_f2fs_rwsem(&sm_info->curseg_lock);
5247
5248 if (!f2fs_readonly(sbi->sb)) {
5249 err = f2fs_create_flush_cmd_control(sbi);
5250 if (err)
5251 return err;
5252 }
5253
5254 err = create_discard_cmd_control(sbi);
5255 if (err)
5256 return err;
5257
5258 err = build_sit_info(sbi);
5259 if (err)
5260 return err;
5261 err = build_free_segmap(sbi);
5262 if (err)
5263 return err;
5264 err = build_curseg(sbi);
5265 if (err)
5266 return err;
5267
5268 /* reinit free segmap based on SIT */
5269 err = build_sit_entries(sbi);
5270 if (err)
5271 return err;
5272
5273 init_free_segmap(sbi);
5274 err = build_dirty_segmap(sbi);
5275 if (err)
5276 return err;
5277
5278 err = sanity_check_curseg(sbi);
5279 if (err)
5280 return err;
5281
5282 init_min_max_mtime(sbi);
5283 return 0;
5284 }
5285
discard_dirty_segmap(struct f2fs_sb_info * sbi,enum dirty_type dirty_type)5286 static void discard_dirty_segmap(struct f2fs_sb_info *sbi,
5287 enum dirty_type dirty_type)
5288 {
5289 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
5290
5291 mutex_lock(&dirty_i->seglist_lock);
5292 kvfree(dirty_i->dirty_segmap[dirty_type]);
5293 dirty_i->nr_dirty[dirty_type] = 0;
5294 mutex_unlock(&dirty_i->seglist_lock);
5295 }
5296
destroy_victim_secmap(struct f2fs_sb_info * sbi)5297 static void destroy_victim_secmap(struct f2fs_sb_info *sbi)
5298 {
5299 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
5300
5301 kvfree(dirty_i->pinned_secmap);
5302 kvfree(dirty_i->victim_secmap);
5303 }
5304
destroy_dirty_segmap(struct f2fs_sb_info * sbi)5305 static void destroy_dirty_segmap(struct f2fs_sb_info *sbi)
5306 {
5307 struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
5308 int i;
5309
5310 if (!dirty_i)
5311 return;
5312
5313 /* discard pre-free/dirty segments list */
5314 for (i = 0; i < NR_DIRTY_TYPE; i++)
5315 discard_dirty_segmap(sbi, i);
5316
5317 if (__is_large_section(sbi)) {
5318 mutex_lock(&dirty_i->seglist_lock);
5319 kvfree(dirty_i->dirty_secmap);
5320 mutex_unlock(&dirty_i->seglist_lock);
5321 }
5322
5323 destroy_victim_secmap(sbi);
5324 SM_I(sbi)->dirty_info = NULL;
5325 kfree(dirty_i);
5326 }
5327
destroy_curseg(struct f2fs_sb_info * sbi)5328 static void destroy_curseg(struct f2fs_sb_info *sbi)
5329 {
5330 struct curseg_info *array = SM_I(sbi)->curseg_array;
5331 int i;
5332
5333 if (!array)
5334 return;
5335 SM_I(sbi)->curseg_array = NULL;
5336 for (i = 0; i < NR_CURSEG_TYPE; i++) {
5337 kfree(array[i].sum_blk);
5338 kfree(array[i].journal);
5339 }
5340 kfree(array);
5341 }
5342
destroy_free_segmap(struct f2fs_sb_info * sbi)5343 static void destroy_free_segmap(struct f2fs_sb_info *sbi)
5344 {
5345 struct free_segmap_info *free_i = SM_I(sbi)->free_info;
5346
5347 if (!free_i)
5348 return;
5349 SM_I(sbi)->free_info = NULL;
5350 kvfree(free_i->free_segmap);
5351 kvfree(free_i->free_secmap);
5352 kfree(free_i);
5353 }
5354
destroy_sit_info(struct f2fs_sb_info * sbi)5355 static void destroy_sit_info(struct f2fs_sb_info *sbi)
5356 {
5357 struct sit_info *sit_i = SIT_I(sbi);
5358
5359 if (!sit_i)
5360 return;
5361
5362 if (sit_i->sentries)
5363 kvfree(sit_i->bitmap);
5364 kfree(sit_i->tmp_map);
5365
5366 kvfree(sit_i->sentries);
5367 kvfree(sit_i->sec_entries);
5368 kvfree(sit_i->dirty_sentries_bitmap);
5369
5370 SM_I(sbi)->sit_info = NULL;
5371 kvfree(sit_i->sit_bitmap);
5372 #ifdef CONFIG_F2FS_CHECK_FS
5373 kvfree(sit_i->sit_bitmap_mir);
5374 kvfree(sit_i->invalid_segmap);
5375 #endif
5376 kfree(sit_i);
5377 }
5378
f2fs_destroy_segment_manager(struct f2fs_sb_info * sbi)5379 void f2fs_destroy_segment_manager(struct f2fs_sb_info *sbi)
5380 {
5381 struct f2fs_sm_info *sm_info = SM_I(sbi);
5382
5383 if (!sm_info)
5384 return;
5385 f2fs_destroy_flush_cmd_control(sbi, true);
5386 destroy_discard_cmd_control(sbi);
5387 destroy_dirty_segmap(sbi);
5388 destroy_curseg(sbi);
5389 destroy_free_segmap(sbi);
5390 destroy_sit_info(sbi);
5391 sbi->sm_info = NULL;
5392 kfree(sm_info);
5393 }
5394
f2fs_create_segment_manager_caches(void)5395 int __init f2fs_create_segment_manager_caches(void)
5396 {
5397 discard_entry_slab = f2fs_kmem_cache_create("f2fs_discard_entry",
5398 sizeof(struct discard_entry));
5399 if (!discard_entry_slab)
5400 goto fail;
5401
5402 discard_cmd_slab = f2fs_kmem_cache_create("f2fs_discard_cmd",
5403 sizeof(struct discard_cmd));
5404 if (!discard_cmd_slab)
5405 goto destroy_discard_entry;
5406
5407 sit_entry_set_slab = f2fs_kmem_cache_create("f2fs_sit_entry_set",
5408 sizeof(struct sit_entry_set));
5409 if (!sit_entry_set_slab)
5410 goto destroy_discard_cmd;
5411
5412 inmem_entry_slab = f2fs_kmem_cache_create("f2fs_inmem_page_entry",
5413 sizeof(struct inmem_pages));
5414 if (!inmem_entry_slab)
5415 goto destroy_sit_entry_set;
5416 return 0;
5417
5418 destroy_sit_entry_set:
5419 kmem_cache_destroy(sit_entry_set_slab);
5420 destroy_discard_cmd:
5421 kmem_cache_destroy(discard_cmd_slab);
5422 destroy_discard_entry:
5423 kmem_cache_destroy(discard_entry_slab);
5424 fail:
5425 return -ENOMEM;
5426 }
5427
f2fs_destroy_segment_manager_caches(void)5428 void f2fs_destroy_segment_manager_caches(void)
5429 {
5430 kmem_cache_destroy(sit_entry_set_slab);
5431 kmem_cache_destroy(discard_cmd_slab);
5432 kmem_cache_destroy(discard_entry_slab);
5433 kmem_cache_destroy(inmem_entry_slab);
5434 }
5435