Lines Matching +full:boot +full:- +full:blks
1 // SPDX-License-Identifier: GPL-2.0
17 * -----------------
21 * Fast commits are organized as a log of tag-length-value (TLV) structs. (See
29 * - EXT4_FC_TAG_UNLINK - records directory entry unlink
30 * - EXT4_FC_TAG_LINK - records directory entry link
31 * - EXT4_FC_TAG_CREAT - records inode and directory entry creation
35 * - EXT4_FC_TAG_ADD_RANGE - records addition of new blocks to an inode
36 * - EXT4_FC_TAG_DEL_RANGE - records deletion of blocks from an inode
40 * - EXT4_FC_TAG_INODE - record the inode that should be replayed
45 * ----------------
47 * order in which they are issued in an in-memory queue. This queue is flushed
67 * -------------------------
72 * - ext4_fc_mark_ineligible(): This makes next fast commit operation to fall
75 * - ext4_fc_start_ineligible() and ext4_fc_stop_ineligible() - This makes all
85 * --------------------
94 * - Create a new file A and remove existing file B
95 * - fsync()
96 * - Append contents to file A
97 * - Truncate file A
98 * - fsync()
102 * |<--- Fast Commit 1 --->|<--- Fast Commit 2 ---->|
107 * -----
126 ext4_debug("%s: Block %lld up-to-date", in ext4_end_buffer_io_sync()
127 __func__, bh->b_blocknr); in ext4_end_buffer_io_sync()
130 ext4_debug("%s: Block %lld not up-to-date", in ext4_end_buffer_io_sync()
131 __func__, bh->b_blocknr); in ext4_end_buffer_io_sync()
142 ei->i_fc_lblk_start = 0; in ext4_fc_reset_inode()
143 ei->i_fc_lblk_len = 0; in ext4_fc_reset_inode()
152 INIT_LIST_HEAD(&ei->i_fc_list); in ext4_fc_init_inode()
153 init_waitqueue_head(&ei->i_fc_wait); in ext4_fc_init_inode()
154 atomic_set(&ei->i_fc_updates, 0); in ext4_fc_init_inode()
157 /* This function must be called with sbi->s_fc_lock held. */
159 __releases(&EXT4_SB(inode->i_sb)->s_fc_lock) in ext4_fc_wait_committing_inode()
165 DEFINE_WAIT_BIT(wait, &ei->i_state_flags, in ext4_fc_wait_committing_inode()
167 wq = bit_waitqueue(&ei->i_state_flags, in ext4_fc_wait_committing_inode()
170 DEFINE_WAIT_BIT(wait, &ei->i_flags, in ext4_fc_wait_committing_inode()
172 wq = bit_waitqueue(&ei->i_flags, in ext4_fc_wait_committing_inode()
175 lockdep_assert_held(&EXT4_SB(inode->i_sb)->s_fc_lock); in ext4_fc_wait_committing_inode()
177 spin_unlock(&EXT4_SB(inode->i_sb)->s_fc_lock); in ext4_fc_wait_committing_inode()
193 if (!test_opt2(inode->i_sb, JOURNAL_FAST_COMMIT) || in ext4_fc_start_update()
194 (EXT4_SB(inode->i_sb)->s_mount_state & EXT4_FC_REPLAY)) in ext4_fc_start_update()
198 spin_lock(&EXT4_SB(inode->i_sb)->s_fc_lock); in ext4_fc_start_update()
199 if (list_empty(&ei->i_fc_list)) in ext4_fc_start_update()
207 atomic_inc(&ei->i_fc_updates); in ext4_fc_start_update()
208 spin_unlock(&EXT4_SB(inode->i_sb)->s_fc_lock); in ext4_fc_start_update()
218 if (!test_opt2(inode->i_sb, JOURNAL_FAST_COMMIT) || in ext4_fc_stop_update()
219 (EXT4_SB(inode->i_sb)->s_mount_state & EXT4_FC_REPLAY)) in ext4_fc_stop_update()
222 if (atomic_dec_and_test(&ei->i_fc_updates)) in ext4_fc_stop_update()
223 wake_up_all(&ei->i_fc_wait); in ext4_fc_stop_update()
234 if (!test_opt2(inode->i_sb, JOURNAL_FAST_COMMIT) || in ext4_fc_del()
235 (EXT4_SB(inode->i_sb)->s_mount_state & EXT4_FC_REPLAY)) in ext4_fc_del()
239 spin_lock(&EXT4_SB(inode->i_sb)->s_fc_lock); in ext4_fc_del()
240 if (list_empty(&ei->i_fc_list)) { in ext4_fc_del()
241 spin_unlock(&EXT4_SB(inode->i_sb)->s_fc_lock); in ext4_fc_del()
249 list_del_init(&ei->i_fc_list); in ext4_fc_del()
250 spin_unlock(&EXT4_SB(inode->i_sb)->s_fc_lock); in ext4_fc_del()
262 (EXT4_SB(sb)->s_mount_state & EXT4_FC_REPLAY)) in ext4_fc_mark_ineligible()
267 sbi->s_fc_stats.fc_ineligible_reason_count[reason]++; in ext4_fc_mark_ineligible()
279 (EXT4_SB(sb)->s_mount_state & EXT4_FC_REPLAY)) in ext4_fc_start_ineligible()
283 sbi->s_fc_stats.fc_ineligible_reason_count[reason]++; in ext4_fc_start_ineligible()
284 atomic_inc(&sbi->s_fc_ineligible_updates); in ext4_fc_start_ineligible()
295 (EXT4_SB(sb)->s_mount_state & EXT4_FC_REPLAY)) in ext4_fc_stop_ineligible()
299 atomic_dec(&EXT4_SB(sb)->s_fc_ineligible_updates); in ext4_fc_stop_ineligible()
305 atomic_read(&EXT4_SB(sb)->s_fc_ineligible_updates)); in ext4_fc_is_ineligible()
325 struct ext4_sb_info *sbi = EXT4_SB(inode->i_sb); in ext4_fc_track_template()
329 if (!test_opt2(inode->i_sb, JOURNAL_FAST_COMMIT) || in ext4_fc_track_template()
330 (sbi->s_mount_state & EXT4_FC_REPLAY)) in ext4_fc_track_template()
331 return -EOPNOTSUPP; in ext4_fc_track_template()
333 if (ext4_fc_is_ineligible(inode->i_sb)) in ext4_fc_track_template()
334 return -EINVAL; in ext4_fc_track_template()
336 tid = handle->h_transaction->t_tid; in ext4_fc_track_template()
337 mutex_lock(&ei->i_fc_lock); in ext4_fc_track_template()
338 if (tid == ei->i_sync_tid) { in ext4_fc_track_template()
342 ei->i_sync_tid = tid; in ext4_fc_track_template()
345 mutex_unlock(&ei->i_fc_lock); in ext4_fc_track_template()
350 spin_lock(&sbi->s_fc_lock); in ext4_fc_track_template()
351 if (list_empty(&EXT4_I(inode)->i_fc_list)) in ext4_fc_track_template()
352 list_add_tail(&EXT4_I(inode)->i_fc_list, in ext4_fc_track_template()
353 (ext4_test_mount_flag(inode->i_sb, EXT4_MF_FC_COMMITTING)) ? in ext4_fc_track_template()
354 &sbi->s_fc_q[FC_Q_STAGING] : in ext4_fc_track_template()
355 &sbi->s_fc_q[FC_Q_MAIN]); in ext4_fc_track_template()
356 spin_unlock(&sbi->s_fc_lock); in ext4_fc_track_template()
366 /* __track_fn for directory entry updates. Called with ei->i_fc_lock. */
373 struct dentry *dentry = dentry_update->dentry; in __track_dentry_update()
374 struct inode *dir = dentry->d_parent->d_inode; in __track_dentry_update()
375 struct super_block *sb = inode->i_sb; in __track_dentry_update()
378 mutex_unlock(&ei->i_fc_lock); in __track_dentry_update()
382 mutex_lock(&ei->i_fc_lock); in __track_dentry_update()
383 return -EOPNOTSUPP; in __track_dentry_update()
389 mutex_lock(&ei->i_fc_lock); in __track_dentry_update()
390 return -ENOMEM; in __track_dentry_update()
393 node->fcd_op = dentry_update->op; in __track_dentry_update()
394 node->fcd_parent = dir->i_ino; in __track_dentry_update()
395 node->fcd_ino = inode->i_ino; in __track_dentry_update()
396 if (dentry->d_name.len > DNAME_INLINE_LEN) { in __track_dentry_update()
397 node->fcd_name.name = kmalloc(dentry->d_name.len, GFP_NOFS); in __track_dentry_update()
398 if (!node->fcd_name.name) { in __track_dentry_update()
401 mutex_lock(&ei->i_fc_lock); in __track_dentry_update()
402 return -ENOMEM; in __track_dentry_update()
404 memcpy((u8 *)node->fcd_name.name, dentry->d_name.name, in __track_dentry_update()
405 dentry->d_name.len); in __track_dentry_update()
407 memcpy(node->fcd_iname, dentry->d_name.name, in __track_dentry_update()
408 dentry->d_name.len); in __track_dentry_update()
409 node->fcd_name.name = node->fcd_iname; in __track_dentry_update()
411 node->fcd_name.len = dentry->d_name.len; in __track_dentry_update()
413 spin_lock(&sbi->s_fc_lock); in __track_dentry_update()
414 if (ext4_test_mount_flag(inode->i_sb, EXT4_MF_FC_COMMITTING)) in __track_dentry_update()
415 list_add_tail(&node->fcd_list, in __track_dentry_update()
416 &sbi->s_fc_dentry_q[FC_Q_STAGING]); in __track_dentry_update()
418 list_add_tail(&node->fcd_list, &sbi->s_fc_dentry_q[FC_Q_MAIN]); in __track_dentry_update()
419 spin_unlock(&sbi->s_fc_lock); in __track_dentry_update()
420 mutex_lock(&ei->i_fc_lock); in __track_dentry_update()
486 return -EEXIST; in __track_inode()
488 EXT4_I(inode)->i_fc_lblk_len = 0; in __track_inode()
497 if (S_ISDIR(inode->i_mode)) in ext4_fc_track_inode()
501 ext4_fc_mark_ineligible(inode->i_sb, in ext4_fc_track_inode()
522 if (inode->i_ino < EXT4_FIRST_INO(inode->i_sb)) { in __track_range()
523 ext4_debug("Special inode %ld being modified\n", inode->i_ino); in __track_range()
524 return -ECANCELED; in __track_range()
527 oldstart = ei->i_fc_lblk_start; in __track_range()
529 if (update && ei->i_fc_lblk_len > 0) { in __track_range()
530 ei->i_fc_lblk_start = min(ei->i_fc_lblk_start, __arg->start); in __track_range()
531 ei->i_fc_lblk_len = in __track_range()
532 max(oldstart + ei->i_fc_lblk_len - 1, __arg->end) - in __track_range()
533 ei->i_fc_lblk_start + 1; in __track_range()
535 ei->i_fc_lblk_start = __arg->start; in __track_range()
536 ei->i_fc_lblk_len = __arg->end - __arg->start + 1; in __track_range()
548 if (S_ISDIR(inode->i_mode)) in ext4_fc_track_range()
562 struct buffer_head *bh = EXT4_SB(sb)->s_fc_bh; in ext4_fc_submit_bh()
570 bh->b_end_io = ext4_end_buffer_io_sync; in ext4_fc_submit_bh()
572 EXT4_SB(sb)->s_fc_bh = NULL; in ext4_fc_submit_bh()
605 int bsize = sbi->s_journal->j_blocksize; in ext4_fc_reserve_space()
606 int ret, off = sbi->s_fc_bytes % bsize; in ext4_fc_reserve_space()
616 if (bsize - off - 1 > len + sizeof(struct ext4_fc_tl)) { in ext4_fc_reserve_space()
621 if (!sbi->s_fc_bh) { in ext4_fc_reserve_space()
622 ret = jbd2_fc_get_buf(EXT4_SB(sb)->s_journal, &bh); in ext4_fc_reserve_space()
625 sbi->s_fc_bh = bh; in ext4_fc_reserve_space()
627 sbi->s_fc_bytes += len; in ext4_fc_reserve_space()
628 return sbi->s_fc_bh->b_data + off; in ext4_fc_reserve_space()
631 tl = (struct ext4_fc_tl *)(sbi->s_fc_bh->b_data + off); in ext4_fc_reserve_space()
632 tl->fc_tag = cpu_to_le16(EXT4_FC_TAG_PAD); in ext4_fc_reserve_space()
633 pad_len = bsize - off - 1 - sizeof(struct ext4_fc_tl); in ext4_fc_reserve_space()
634 tl->fc_len = cpu_to_le16(pad_len); in ext4_fc_reserve_space()
644 ret = jbd2_fc_get_buf(EXT4_SB(sb)->s_journal, &bh); in ext4_fc_reserve_space()
647 sbi->s_fc_bh = bh; in ext4_fc_reserve_space()
648 sbi->s_fc_bytes = (sbi->s_fc_bytes / bsize + 1) * bsize + len; in ext4_fc_reserve_space()
649 return sbi->s_fc_bh->b_data; in ext4_fc_reserve_space()
674 int off, bsize = sbi->s_journal->j_blocksize; in ext4_fc_write_tail()
683 return -ENOSPC; in ext4_fc_write_tail()
685 off = sbi->s_fc_bytes % bsize; in ext4_fc_write_tail()
688 tl.fc_len = cpu_to_le16(bsize - off - 1 + sizeof(struct ext4_fc_tail)); in ext4_fc_write_tail()
689 sbi->s_fc_bytes = round_up(sbi->s_fc_bytes, bsize); in ext4_fc_write_tail()
693 tail.fc_tid = cpu_to_le32(sbi->s_journal->j_running_transaction->t_tid); in ext4_fc_write_tail()
699 memset(dst, 0, bsize - off); /* Don't leak uninitialized memory. */ in ext4_fc_write_tail()
775 if (EXT4_INODE_SIZE(inode->i_sb) > EXT4_GOOD_OLD_INODE_SIZE) in ext4_fc_write_inode()
776 inode_len += ei->i_extra_isize; in ext4_fc_write_inode()
778 fc_inode.fc_ino = cpu_to_le32(inode->i_ino); in ext4_fc_write_inode()
782 ret = -ECANCELED; in ext4_fc_write_inode()
783 dst = ext4_fc_reserve_space(inode->i_sb, in ext4_fc_write_inode()
788 if (!ext4_fc_memcpy(inode->i_sb, dst, &tl, sizeof(tl), crc)) in ext4_fc_write_inode()
791 if (!ext4_fc_memcpy(inode->i_sb, dst, &fc_inode, sizeof(fc_inode), crc)) in ext4_fc_write_inode()
794 if (!ext4_fc_memcpy(inode->i_sb, dst, (u8 *)ext4_raw_inode(&iloc), in ext4_fc_write_inode()
817 mutex_lock(&ei->i_fc_lock); in ext4_fc_write_inode_data()
818 if (ei->i_fc_lblk_len == 0) { in ext4_fc_write_inode_data()
819 mutex_unlock(&ei->i_fc_lock); in ext4_fc_write_inode_data()
822 old_blk_size = ei->i_fc_lblk_start; in ext4_fc_write_inode_data()
823 new_blk_size = ei->i_fc_lblk_start + ei->i_fc_lblk_len - 1; in ext4_fc_write_inode_data()
824 ei->i_fc_lblk_len = 0; in ext4_fc_write_inode_data()
825 mutex_unlock(&ei->i_fc_lock); in ext4_fc_write_inode_data()
829 __func__, cur_lblk_off, new_blk_size, inode->i_ino); in ext4_fc_write_inode_data()
833 map.m_len = new_blk_size - cur_lblk_off + 1; in ext4_fc_write_inode_data()
836 return -ECANCELED; in ext4_fc_write_inode_data()
844 lrange.fc_ino = cpu_to_le32(inode->i_ino); in ext4_fc_write_inode_data()
847 if (!ext4_fc_add_tlv(inode->i_sb, EXT4_FC_TAG_DEL_RANGE, in ext4_fc_write_inode_data()
849 return -ENOSPC; in ext4_fc_write_inode_data()
857 fc_ext.fc_ino = cpu_to_le32(inode->i_ino); in ext4_fc_write_inode_data()
859 ex->ee_block = cpu_to_le32(map.m_lblk); in ext4_fc_write_inode_data()
860 ex->ee_len = cpu_to_le16(map.m_len); in ext4_fc_write_inode_data()
866 if (!ext4_fc_add_tlv(inode->i_sb, EXT4_FC_TAG_ADD_RANGE, in ext4_fc_write_inode_data()
868 return -ENOSPC; in ext4_fc_write_inode_data()
881 struct super_block *sb = (struct super_block *)(journal->j_private); in ext4_fc_submit_inode_data_all()
887 spin_lock(&sbi->s_fc_lock); in ext4_fc_submit_inode_data_all()
889 list_for_each(pos, &sbi->s_fc_q[FC_Q_MAIN]) { in ext4_fc_submit_inode_data_all()
891 ext4_set_inode_state(&ei->vfs_inode, EXT4_STATE_FC_COMMITTING); in ext4_fc_submit_inode_data_all()
892 while (atomic_read(&ei->i_fc_updates)) { in ext4_fc_submit_inode_data_all()
895 prepare_to_wait(&ei->i_fc_wait, &wait, in ext4_fc_submit_inode_data_all()
897 if (atomic_read(&ei->i_fc_updates)) { in ext4_fc_submit_inode_data_all()
898 spin_unlock(&sbi->s_fc_lock); in ext4_fc_submit_inode_data_all()
900 spin_lock(&sbi->s_fc_lock); in ext4_fc_submit_inode_data_all()
902 finish_wait(&ei->i_fc_wait, &wait); in ext4_fc_submit_inode_data_all()
904 spin_unlock(&sbi->s_fc_lock); in ext4_fc_submit_inode_data_all()
905 ret = jbd2_submit_inode_data(ei->jinode); in ext4_fc_submit_inode_data_all()
908 spin_lock(&sbi->s_fc_lock); in ext4_fc_submit_inode_data_all()
910 spin_unlock(&sbi->s_fc_lock); in ext4_fc_submit_inode_data_all()
918 struct super_block *sb = (struct super_block *)(journal->j_private); in ext4_fc_wait_inode_data_all()
923 spin_lock(&sbi->s_fc_lock); in ext4_fc_wait_inode_data_all()
924 list_for_each_entry_safe(pos, n, &sbi->s_fc_q[FC_Q_MAIN], i_fc_list) { in ext4_fc_wait_inode_data_all()
925 if (!ext4_test_inode_state(&pos->vfs_inode, in ext4_fc_wait_inode_data_all()
928 spin_unlock(&sbi->s_fc_lock); in ext4_fc_wait_inode_data_all()
930 ret = jbd2_wait_inode_data(journal, pos->jinode); in ext4_fc_wait_inode_data_all()
933 spin_lock(&sbi->s_fc_lock); in ext4_fc_wait_inode_data_all()
935 spin_unlock(&sbi->s_fc_lock); in ext4_fc_wait_inode_data_all()
942 __acquires(&sbi->s_fc_lock) in ext4_fc_commit_dentry_updates()
943 __releases(&sbi->s_fc_lock) in ext4_fc_commit_dentry_updates()
945 struct super_block *sb = (struct super_block *)(journal->j_private); in ext4_fc_commit_dentry_updates()
953 if (list_empty(&sbi->s_fc_dentry_q[FC_Q_MAIN])) in ext4_fc_commit_dentry_updates()
955 list_for_each_safe(fcd_pos, fcd_n, &sbi->s_fc_dentry_q[FC_Q_MAIN]) { in ext4_fc_commit_dentry_updates()
958 if (fc_dentry->fcd_op != EXT4_FC_TAG_CREAT) { in ext4_fc_commit_dentry_updates()
959 spin_unlock(&sbi->s_fc_lock); in ext4_fc_commit_dentry_updates()
961 sb, fc_dentry->fcd_op, in ext4_fc_commit_dentry_updates()
962 fc_dentry->fcd_parent, fc_dentry->fcd_ino, in ext4_fc_commit_dentry_updates()
963 fc_dentry->fcd_name.len, in ext4_fc_commit_dentry_updates()
964 fc_dentry->fcd_name.name, crc)) { in ext4_fc_commit_dentry_updates()
965 ret = -ENOSPC; in ext4_fc_commit_dentry_updates()
968 spin_lock(&sbi->s_fc_lock); in ext4_fc_commit_dentry_updates()
973 list_for_each_safe(pos, n, &sbi->s_fc_q[FC_Q_MAIN]) { in ext4_fc_commit_dentry_updates()
975 if (ei->vfs_inode.i_ino == fc_dentry->fcd_ino) { in ext4_fc_commit_dentry_updates()
976 inode = &ei->vfs_inode; in ext4_fc_commit_dentry_updates()
986 spin_unlock(&sbi->s_fc_lock); in ext4_fc_commit_dentry_updates()
1004 sb, fc_dentry->fcd_op, in ext4_fc_commit_dentry_updates()
1005 fc_dentry->fcd_parent, fc_dentry->fcd_ino, in ext4_fc_commit_dentry_updates()
1006 fc_dentry->fcd_name.len, in ext4_fc_commit_dentry_updates()
1007 fc_dentry->fcd_name.name, crc)) { in ext4_fc_commit_dentry_updates()
1008 ret = -ENOSPC; in ext4_fc_commit_dentry_updates()
1012 spin_lock(&sbi->s_fc_lock); in ext4_fc_commit_dentry_updates()
1016 spin_lock(&sbi->s_fc_lock); in ext4_fc_commit_dentry_updates()
1022 struct super_block *sb = (struct super_block *)(journal->j_private); in ext4_fc_perform_commit()
1044 if (journal->j_fs_dev != journal->j_dev) in ext4_fc_perform_commit()
1045 blkdev_issue_flush(journal->j_fs_dev, GFP_NOFS); in ext4_fc_perform_commit()
1048 if (sbi->s_fc_bytes == 0) { in ext4_fc_perform_commit()
1055 sbi->s_journal->j_running_transaction->t_tid); in ext4_fc_perform_commit()
1058 ret = -ENOSPC; in ext4_fc_perform_commit()
1063 spin_lock(&sbi->s_fc_lock); in ext4_fc_perform_commit()
1066 spin_unlock(&sbi->s_fc_lock); in ext4_fc_perform_commit()
1070 list_for_each(pos, &sbi->s_fc_q[FC_Q_MAIN]) { in ext4_fc_perform_commit()
1072 inode = &iter->vfs_inode; in ext4_fc_perform_commit()
1076 spin_unlock(&sbi->s_fc_lock); in ext4_fc_perform_commit()
1083 spin_lock(&sbi->s_fc_lock); in ext4_fc_perform_commit()
1085 spin_unlock(&sbi->s_fc_lock); in ext4_fc_perform_commit()
1102 struct super_block *sb = (struct super_block *)(journal->j_private); in ext4_fc_commit()
1104 int nblks = 0, ret, bsize = journal->j_blocksize; in ext4_fc_commit()
1105 int subtid = atomic_read(&sbi->s_fc_subtid); in ext4_fc_commit()
1121 if (ret == -EALREADY) { in ext4_fc_commit()
1123 if (atomic_read(&sbi->s_fc_subtid) <= subtid && in ext4_fc_commit()
1124 commit_tid > journal->j_commit_sequence) in ext4_fc_commit()
1129 sbi->s_fc_stats.fc_ineligible_reason_count[EXT4_FC_COMMIT_FAILED]++; in ext4_fc_commit()
1134 fc_bufs_before = (sbi->s_fc_bytes + bsize - 1) / bsize; in ext4_fc_commit()
1137 sbi->s_fc_stats.fc_ineligible_reason_count[EXT4_FC_COMMIT_FAILED]++; in ext4_fc_commit()
1141 nblks = (sbi->s_fc_bytes + bsize - 1) / bsize - fc_bufs_before; in ext4_fc_commit()
1144 sbi->s_fc_stats.fc_ineligible_reason_count[EXT4_FC_COMMIT_FAILED]++; in ext4_fc_commit()
1148 atomic_inc(&sbi->s_fc_subtid); in ext4_fc_commit()
1153 sbi->s_fc_stats.fc_ineligible_reason_count[EXT4_FC_COMMIT_FAILED]++; in ext4_fc_commit()
1157 spin_lock(&sbi->s_fc_lock); in ext4_fc_commit()
1160 sbi->s_fc_stats.fc_ineligible_commits++; in ext4_fc_commit()
1162 sbi->s_fc_stats.fc_num_commits++; in ext4_fc_commit()
1163 sbi->s_fc_stats.fc_numblks += nblks; in ext4_fc_commit()
1165 spin_unlock(&sbi->s_fc_lock); in ext4_fc_commit()
1173 if (likely(sbi->s_fc_avg_commit_time)) in ext4_fc_commit()
1174 sbi->s_fc_avg_commit_time = (commit_time + in ext4_fc_commit()
1175 sbi->s_fc_avg_commit_time * 3) / 4; in ext4_fc_commit()
1177 sbi->s_fc_avg_commit_time = commit_time; in ext4_fc_commit()
1179 "Fast commit ended with blks = %d, reason = %d, subtid - %d", in ext4_fc_commit()
1195 struct super_block *sb = journal->j_private; in ext4_fc_cleanup()
1201 if (full && sbi->s_fc_bh) in ext4_fc_cleanup()
1202 sbi->s_fc_bh = NULL; in ext4_fc_cleanup()
1206 spin_lock(&sbi->s_fc_lock); in ext4_fc_cleanup()
1207 list_for_each_safe(pos, n, &sbi->s_fc_q[FC_Q_MAIN]) { in ext4_fc_cleanup()
1209 list_del_init(&iter->i_fc_list); in ext4_fc_cleanup()
1210 ext4_clear_inode_state(&iter->vfs_inode, in ext4_fc_cleanup()
1212 ext4_fc_reset_inode(&iter->vfs_inode); in ext4_fc_cleanup()
1216 wake_up_bit(&iter->i_state_flags, EXT4_STATE_FC_COMMITTING); in ext4_fc_cleanup()
1218 wake_up_bit(&iter->i_flags, EXT4_STATE_FC_COMMITTING); in ext4_fc_cleanup()
1222 while (!list_empty(&sbi->s_fc_dentry_q[FC_Q_MAIN])) { in ext4_fc_cleanup()
1223 fc_dentry = list_first_entry(&sbi->s_fc_dentry_q[FC_Q_MAIN], in ext4_fc_cleanup()
1226 list_del_init(&fc_dentry->fcd_list); in ext4_fc_cleanup()
1227 spin_unlock(&sbi->s_fc_lock); in ext4_fc_cleanup()
1229 if (fc_dentry->fcd_name.name && in ext4_fc_cleanup()
1230 fc_dentry->fcd_name.len > DNAME_INLINE_LEN) in ext4_fc_cleanup()
1231 kfree(fc_dentry->fcd_name.name); in ext4_fc_cleanup()
1233 spin_lock(&sbi->s_fc_lock); in ext4_fc_cleanup()
1236 list_splice_init(&sbi->s_fc_dentry_q[FC_Q_STAGING], in ext4_fc_cleanup()
1237 &sbi->s_fc_dentry_q[FC_Q_MAIN]); in ext4_fc_cleanup()
1238 list_splice_init(&sbi->s_fc_q[FC_Q_STAGING], in ext4_fc_cleanup()
1239 &sbi->s_fc_q[FC_Q_MAIN]); in ext4_fc_cleanup()
1245 sbi->s_fc_bytes = 0; in ext4_fc_cleanup()
1246 spin_unlock(&sbi->s_fc_lock); in ext4_fc_cleanup()
1265 darg->parent_ino = le32_to_cpu(fcd.fc_parent_ino); in tl_to_darg()
1266 darg->ino = le32_to_cpu(fcd.fc_ino); in tl_to_darg()
1267 darg->dname = val + offsetof(struct ext4_fc_dentry_info, fc_dname); in tl_to_darg()
1268 darg->dname_len = le16_to_cpu(tl->fc_len) - in tl_to_darg()
1304 /* -ENOENT ok coz it might not exist anymore. */ in ext4_fc_replay_unlink()
1305 if (ret == -ENOENT) in ext4_fc_replay_unlink()
1318 struct qstr qstr_dname = QSTR_INIT(darg->dname, darg->dname_len); in ext4_fc_replay_link_internal()
1321 dir = ext4_iget(sb, darg->parent_ino, EXT4_IGET_NORMAL); in ext4_fc_replay_link_internal()
1323 jbd_debug(1, "Dir with inode %d not found.", darg->parent_ino); in ext4_fc_replay_link_internal()
1338 ret = -ENOMEM; in ext4_fc_replay_link_internal()
1349 if (ret && ret != -EEXIST) { in ext4_fc_replay_link_internal()
1402 state = &EXT4_SB(sb)->s_fc_replay_state; in ext4_fc_record_modified_inode()
1403 for (i = 0; i < state->fc_modified_inodes_used; i++) in ext4_fc_record_modified_inode()
1404 if (state->fc_modified_inodes[i] == ino) in ext4_fc_record_modified_inode()
1406 if (state->fc_modified_inodes_used == state->fc_modified_inodes_size) { in ext4_fc_record_modified_inode()
1409 fc_modified_inodes = krealloc(state->fc_modified_inodes, in ext4_fc_record_modified_inode()
1410 sizeof(int) * (state->fc_modified_inodes_size + in ext4_fc_record_modified_inode()
1414 return -ENOMEM; in ext4_fc_record_modified_inode()
1415 state->fc_modified_inodes = fc_modified_inodes; in ext4_fc_record_modified_inode()
1416 state->fc_modified_inodes_size += in ext4_fc_record_modified_inode()
1419 state->fc_modified_inodes[state->fc_modified_inodes_used++] = ino; in ext4_fc_record_modified_inode()
1434 int inode_len, ino, ret, tag = le16_to_cpu(tl->fc_tag); in ext4_fc_replay_inode()
1459 inode_len = le16_to_cpu(tl->fc_len) - sizeof(struct ext4_fc_inode); in ext4_fc_replay_inode()
1463 memcpy(&raw_inode->i_generation, &raw_fc_inode->i_generation, in ext4_fc_replay_inode()
1464 inode_len - offsetof(struct ext4_inode, i_generation)); in ext4_fc_replay_inode()
1465 if (le32_to_cpu(raw_inode->i_flags) & EXT4_EXTENTS_FL) { in ext4_fc_replay_inode()
1466 eh = (struct ext4_extent_header *)(&raw_inode->i_block[0]); in ext4_fc_replay_inode()
1467 if (eh->eh_magic != EXT4_EXT_MAGIC) { in ext4_fc_replay_inode()
1469 eh->eh_magic = EXT4_EXT_MAGIC; in ext4_fc_replay_inode()
1470 eh->eh_max = cpu_to_le16( in ext4_fc_replay_inode()
1471 (sizeof(raw_inode->i_block) - in ext4_fc_replay_inode()
1475 } else if (le32_to_cpu(raw_inode->i_flags) & EXT4_INLINE_DATA_FL) { in ext4_fc_replay_inode()
1476 memcpy(raw_inode->i_block, raw_fc_inode->i_block, in ext4_fc_replay_inode()
1477 sizeof(raw_inode->i_block)); in ext4_fc_replay_inode()
1495 return -EFSCORRUPTED; in ext4_fc_replay_inode()
1505 inode->i_generation = le32_to_cpu(ext4_raw_inode(&iloc)->i_generation); in ext4_fc_replay_inode()
1515 blkdev_issue_flush(sb->s_bdev, GFP_KERNEL); in ext4_fc_replay_inode()
1549 ret = -EINVAL; in ext4_fc_replay_create()
1553 if (S_ISDIR(inode->i_mode)) { in ext4_fc_replay_create()
1592 state = &EXT4_SB(sb)->s_fc_replay_state; in ext4_fc_record_regions()
1597 if (replay && state->fc_regions_used != state->fc_regions_valid) in ext4_fc_record_regions()
1598 state->fc_regions_used = state->fc_regions_valid; in ext4_fc_record_regions()
1599 if (state->fc_regions_used == state->fc_regions_size) { in ext4_fc_record_regions()
1602 fc_regions = krealloc(state->fc_regions, in ext4_fc_record_regions()
1604 (state->fc_regions_size + in ext4_fc_record_regions()
1608 return -ENOMEM; in ext4_fc_record_regions()
1609 state->fc_regions_size += in ext4_fc_record_regions()
1611 state->fc_regions = fc_regions; in ext4_fc_record_regions()
1613 region = &state->fc_regions[state->fc_regions_used++]; in ext4_fc_record_regions()
1614 region->ino = ino; in ext4_fc_record_regions()
1615 region->lblk = lblk; in ext4_fc_record_regions()
1616 region->pblk = pblk; in ext4_fc_record_regions()
1617 region->len = len; in ext4_fc_record_regions()
1620 state->fc_regions_valid++; in ext4_fc_record_regions()
1643 le32_to_cpu(fc_add_ex.fc_ino), le32_to_cpu(ex->ee_block), in ext4_fc_replay_add_range()
1652 ret = ext4_fc_record_modified_inode(sb, inode->i_ino); in ext4_fc_replay_add_range()
1656 start = le32_to_cpu(ex->ee_block); in ext4_fc_replay_add_range()
1664 inode->i_ino); in ext4_fc_replay_add_range()
1683 &newex, start_pblk + cur - start); in ext4_fc_replay_add_range()
1687 down_write(&EXT4_I(inode)->i_data_sem); in ext4_fc_replay_add_range()
1690 up_write((&EXT4_I(inode)->i_data_sem)); in ext4_fc_replay_add_range()
1698 if (start_pblk + cur - start != map.m_pblk) { in ext4_fc_replay_add_range()
1706 start_pblk + cur - start); in ext4_fc_replay_add_range()
1718 ext4_mb_mark_bb(inode->i_sb, map.m_pblk, map.m_len, 0); in ext4_fc_replay_add_range()
1737 remaining -= map.m_len; in ext4_fc_replay_add_range()
1740 sb->s_blocksize_bits); in ext4_fc_replay_add_range()
1770 ret = ext4_fc_record_modified_inode(sb, inode->i_ino); in ext4_fc_replay_del_range()
1775 inode->i_ino, le32_to_cpu(lrange.fc_lblk), in ext4_fc_replay_del_range()
1785 remaining -= ret; in ext4_fc_replay_del_range()
1787 ext4_mb_mark_bb(inode->i_sb, map.m_pblk, map.m_len, 0); in ext4_fc_replay_del_range()
1789 remaining -= map.m_len; in ext4_fc_replay_del_range()
1794 down_write(&EXT4_I(inode)->i_data_sem); in ext4_fc_replay_del_range()
1797 le32_to_cpu(lrange.fc_len) - 1); in ext4_fc_replay_del_range()
1798 up_write(&EXT4_I(inode)->i_data_sem); in ext4_fc_replay_del_range()
1802 i_size_read(inode) >> sb->s_blocksize_bits); in ext4_fc_replay_del_range()
1844 state = &EXT4_SB(sb)->s_fc_replay_state; in ext4_fc_set_bitmaps_and_counters()
1845 for (i = 0; i < state->fc_modified_inodes_used; i++) { in ext4_fc_set_bitmaps_and_counters()
1846 inode = ext4_iget(sb, state->fc_modified_inodes[i], in ext4_fc_set_bitmaps_and_counters()
1850 state->fc_modified_inodes[i]); in ext4_fc_set_bitmaps_and_counters()
1857 map.m_len = end - cur; in ext4_fc_set_bitmaps_and_counters()
1866 for (j = 0; j < path->p_depth; j++) in ext4_fc_set_bitmaps_and_counters()
1867 ext4_mb_mark_bb(inode->i_sb, in ext4_fc_set_bitmaps_and_counters()
1873 ext4_mb_mark_bb(inode->i_sb, map.m_pblk, in ext4_fc_set_bitmaps_and_counters()
1893 state = &EXT4_SB(sb)->s_fc_replay_state; in ext4_fc_replay_check_excluded()
1894 for (i = 0; i < state->fc_regions_valid; i++) { in ext4_fc_replay_check_excluded()
1895 if (state->fc_regions[i].ino == 0 || in ext4_fc_replay_check_excluded()
1896 state->fc_regions[i].len == 0) in ext4_fc_replay_check_excluded()
1898 if (blk >= state->fc_regions[i].pblk && in ext4_fc_replay_check_excluded()
1899 blk < state->fc_regions[i].pblk + state->fc_regions[i].len) in ext4_fc_replay_check_excluded()
1910 sbi->s_mount_state &= ~EXT4_FC_REPLAY; in ext4_fc_replay_cleanup()
1911 kfree(sbi->s_fc_replay_state.fc_regions); in ext4_fc_replay_cleanup()
1912 kfree(sbi->s_fc_replay_state.fc_modified_inodes); in ext4_fc_replay_cleanup()
1920 * - Make sure the fast commit area has valid tags for replay
1921 * - Count number of tags that need to be replayed by the replay handler
1922 * - Verify CRC
1923 * - Create a list of excluded blocks for allocation during replay phase
1929 * of a successful scan phase, sbi->s_fc_replay_state.fc_replay_num_tags is set
1936 struct super_block *sb = journal->j_private; in ext4_fc_replay_scan()
1947 state = &sbi->s_fc_replay_state; in ext4_fc_replay_scan()
1949 start = (u8 *)bh->b_data; in ext4_fc_replay_scan()
1950 end = (__u8 *)bh->b_data + journal->j_blocksize - 1; in ext4_fc_replay_scan()
1952 if (state->fc_replay_expected_off == 0) { in ext4_fc_replay_scan()
1953 state->fc_cur_tag = 0; in ext4_fc_replay_scan()
1954 state->fc_replay_num_tags = 0; in ext4_fc_replay_scan()
1955 state->fc_crc = 0; in ext4_fc_replay_scan()
1956 state->fc_regions = NULL; in ext4_fc_replay_scan()
1957 state->fc_regions_valid = state->fc_regions_used = in ext4_fc_replay_scan()
1958 state->fc_regions_size = 0; in ext4_fc_replay_scan()
1960 if (le16_to_cpu(((struct ext4_fc_tl *)start)->fc_tag) in ext4_fc_replay_scan()
1965 if (off != state->fc_replay_expected_off) { in ext4_fc_replay_scan()
1966 ret = -EFSCORRUPTED; in ext4_fc_replay_scan()
1970 state->fc_replay_expected_off++; in ext4_fc_replay_scan()
1975 tag2str(le16_to_cpu(tl.fc_tag)), bh->b_blocknr); in ext4_fc_replay_scan()
1982 le32_to_cpu(ex->ee_block), ext4_ext_pblock(ex), in ext4_fc_replay_scan()
1994 state->fc_cur_tag++; in ext4_fc_replay_scan()
1995 state->fc_crc = ext4_chksum(sbi, state->fc_crc, cur, in ext4_fc_replay_scan()
1999 state->fc_cur_tag++; in ext4_fc_replay_scan()
2001 state->fc_crc = ext4_chksum(sbi, state->fc_crc, cur, in ext4_fc_replay_scan()
2006 le32_to_cpu(tail.fc_crc) == state->fc_crc) { in ext4_fc_replay_scan()
2007 state->fc_replay_num_tags = state->fc_cur_tag; in ext4_fc_replay_scan()
2008 state->fc_regions_valid = in ext4_fc_replay_scan()
2009 state->fc_regions_used; in ext4_fc_replay_scan()
2011 ret = state->fc_replay_num_tags ? in ext4_fc_replay_scan()
2012 JBD2_FC_REPLAY_STOP : -EFSBADCRC; in ext4_fc_replay_scan()
2014 state->fc_crc = 0; in ext4_fc_replay_scan()
2020 ret = -EOPNOTSUPP; in ext4_fc_replay_scan()
2027 state->fc_cur_tag++; in ext4_fc_replay_scan()
2028 state->fc_crc = ext4_chksum(sbi, state->fc_crc, cur, in ext4_fc_replay_scan()
2032 ret = state->fc_replay_num_tags ? in ext4_fc_replay_scan()
2033 JBD2_FC_REPLAY_STOP : -ECANCELED; in ext4_fc_replay_scan()
2051 struct super_block *sb = journal->j_private; in ext4_fc_replay()
2056 struct ext4_fc_replay_state *state = &sbi->s_fc_replay_state; in ext4_fc_replay()
2060 state->fc_current_pass = PASS_SCAN; in ext4_fc_replay()
2064 if (state->fc_current_pass != pass) { in ext4_fc_replay()
2065 state->fc_current_pass = pass; in ext4_fc_replay()
2066 sbi->s_mount_state |= EXT4_FC_REPLAY; in ext4_fc_replay()
2068 if (!sbi->s_fc_replay_state.fc_replay_num_tags) { in ext4_fc_replay()
2075 if (sbi->s_fc_debug_max_replay && off >= sbi->s_fc_debug_max_replay) { in ext4_fc_replay()
2081 start = (u8 *)bh->b_data; in ext4_fc_replay()
2082 end = (__u8 *)bh->b_data + journal->j_blocksize - 1; in ext4_fc_replay()
2088 if (state->fc_replay_num_tags == 0) { in ext4_fc_replay()
2095 state->fc_replay_num_tags--; in ext4_fc_replay()
2130 ret = -ECANCELED; in ext4_fc_replay()
2147 journal->j_fc_replay_callback = ext4_fc_replay; in ext4_fc_init()
2150 journal->j_fc_cleanup_callback = ext4_fc_cleanup; in ext4_fc_init()
2158 [EXT4_FC_REASON_SWAP_BOOT] = "Swap boot",
2168 struct ext4_sb_info *sbi = EXT4_SB((struct super_block *)seq->private); in ext4_fc_info_show()
2169 struct ext4_fc_stats *stats = &sbi->s_fc_stats; in ext4_fc_info_show()
2177 stats->fc_num_commits, stats->fc_ineligible_commits, in ext4_fc_info_show()
2178 stats->fc_numblks, in ext4_fc_info_show()
2179 div_u64(sbi->s_fc_avg_commit_time, 1000)); in ext4_fc_info_show()
2183 stats->fc_ineligible_reason_count[i]); in ext4_fc_info_show()
2194 return -ENOMEM; in ext4_fc_init_dentry_cache()