43 #include "ordered-data.h"
83 	struct btrfs_ordered_extent *ordered;  member
401  * Cleanup all submitted ordered extents in specified range to handle errors
407  * to be released, which we want to happen only when finishing the ordered
429 		 * clear page Ordered and run the ordered extent accounting.  in btrfs_cleanup_ordered_extents()
431 		 * Here we can't just clear the Ordered bit, or  in btrfs_cleanup_ordered_extents()
433 		 * for the page range, and the ordered extent will never finish.  in btrfs_cleanup_ordered_extents()
445 		 * Here we just clear all Ordered bits for every page in the  in btrfs_cleanup_ordered_extents()
447 		 * the ordered extent accounting for the range.  in btrfs_cleanup_ordered_extents()
820  * This is done inside an ordered work queue, and the compression is spread
821  * across many cpus.  The actual IO submission is step two, and the ordered work
1105 	struct btrfs_ordered_extent *ordered;  in submit_one_async_extent()  local
1165 	ordered = btrfs_alloc_ordered_extent(inode, start,	/* file_offset */  in submit_one_async_extent()
1173 	if (IS_ERR(ordered)) {  in submit_one_async_extent()
1175 		ret = PTR_ERR(ordered);  in submit_one_async_extent()
1184 	btrfs_submit_compressed_write(ordered,  in submit_one_async_extent()
1250  * allocate extents on disk for the range, and create ordered data structs
1270  * while-loop, the ordered extents created in previous iterations are kept
1376 		struct btrfs_ordered_extent *ordered;  in cow_file_range()  local
1427 		ordered = btrfs_alloc_ordered_extent(inode, start, ram_size,  in cow_file_range()
1431 		if (IS_ERR(ordered)) {  in cow_file_range()
1432 			ret = PTR_ERR(ordered);  in cow_file_range()
1437 			ret = btrfs_reloc_clone_csums(ordered);  in cow_file_range()
1443 			 * at out_unlock label to free meta of this ordered  in cow_file_range()
1448 			 * skip current ordered extent.  in cow_file_range()
1455 		btrfs_put_ordered_extent(ordered);  in cow_file_range()
1464 		 * Do set the Ordered (Private2) bit so we know this page was  in cow_file_range()
1485 		 * free metadata of current ordered extent, we're OK to exit.  in cow_file_range()
1515 	 * For the range (1). We have already instantiated the ordered extents  in cow_file_range()
1535 	 * (or a subrange) and failed to create the respective ordered extent,  in cow_file_range()
1567  * Phase two of compressed writeback.  This is the ordered portion of the code,
1996 		struct btrfs_ordered_extent *ordered;  in run_delalloc_nocow()  local
2157 		ordered = btrfs_alloc_ordered_extent(inode, cur_offset,  in run_delalloc_nocow()
2165 		if (IS_ERR(ordered)) {  in run_delalloc_nocow()
2170 			ret = PTR_ERR(ordered);  in run_delalloc_nocow()
2178 			 * from freeing metadata of created ordered extent.  in run_delalloc_nocow()
2180 			ret = btrfs_reloc_clone_csums(ordered);  in run_delalloc_nocow()
2181 		btrfs_put_ordered_extent(ordered);  in run_delalloc_nocow()
2193 		 * handler, as metadata for created ordered extent will only  in run_delalloc_nocow()
2550 					struct btrfs_ordered_extent *ordered)  in btrfs_extract_ordered_extent()  argument
2557 	/* Must always be called for the beginning of an ordered extent. */  in btrfs_extract_ordered_extent()
2558 	if (WARN_ON_ONCE(start != ordered->disk_bytenr))  in btrfs_extract_ordered_extent()
2561 	/* No need to split if the ordered extent covers the entire bio. */  in btrfs_extract_ordered_extent()
2562 	if (ordered->disk_num_bytes == len) {  in btrfs_extract_ordered_extent()
2563 		refcount_inc(&ordered->refs);  in btrfs_extract_ordered_extent()
2564 		bbio->ordered = ordered;  in btrfs_extract_ordered_extent()
2572 	if (!test_bit(BTRFS_ORDERED_NOCOW, &ordered->flags)) {  in btrfs_extract_ordered_extent()
2574 				       ordered->num_bytes, len,  in btrfs_extract_ordered_extent()
2575 				       ordered->disk_bytenr);  in btrfs_extract_ordered_extent()
2580 	new = btrfs_split_ordered_extent(ordered, len);  in btrfs_extract_ordered_extent()
2583 	bbio->ordered = new;  in btrfs_extract_ordered_extent()
2588  * given a list of ordered sums record them in the inode.  This happens
2688 	struct btrfs_ordered_extent *ordered;  in btrfs_writepage_fixup_worker()  local
2721 		 *    while waiting for the ordered extent.  in btrfs_writepage_fixup_worker()
2750 	/* already ordered? We're done */  in btrfs_writepage_fixup_worker()
2754 	ordered = btrfs_lookup_ordered_range(inode, page_start, PAGE_SIZE);  in btrfs_writepage_fixup_worker()
2755 	if (ordered) {  in btrfs_writepage_fixup_worker()
2759 		btrfs_start_ordered_extent(ordered);  in btrfs_writepage_fixup_worker()
2760 		btrfs_put_ordered_extent(ordered);  in btrfs_writepage_fixup_worker()
2812  * properly happens and the data=ordered rules are followed.
2814  * In our case any range that doesn't have the ORDERED bit set
2816  * to fix it up.  The async helper will wait for ordered extents, set
2825 	/* This page has ordered extent covering it already */  in btrfs_writepage_cow_fixup()
2925 	 * EXTENT_DELALLOC_BIT bit through the ordered extent completion.  in insert_reserved_file_extent()
2995 	 * For delalloc, when completing an ordered extent we update the inode's  in insert_ordered_extent_file_extent()
2998 	 * except if the ordered extent was truncated.  in insert_ordered_extent_file_extent()
3010  * As ordered data IO finishes, this gets called so we can finish
3011  * an ordered extent if the range of bytes in the file it covers are
3158 		 * If we failed to finish this ordered extent for any reason we  in btrfs_finish_one_ordered()
3159 		 * need to make sure BTRFS_ORDERED_IOERR is set on the ordered  in btrfs_finish_one_ordered()
3163 		 * marking this ordered extent as failed.  in btrfs_finish_one_ordered()
3192 		 * If the ordered extent had an IOERR or something else went  in btrfs_finish_one_ordered()
3193 		 * wrong we need to return the space for this ordered extent  in btrfs_finish_one_ordered()
3219 			 * the ordered extent was created.  in btrfs_finish_one_ordered()
3229 	 * updating everything for this ordered extent.  in btrfs_finish_one_ordered()
3241 int btrfs_finish_ordered_io(struct btrfs_ordered_extent *ordered)  in btrfs_finish_ordered_io()  argument
3243 	if (btrfs_is_zoned(btrfs_sb(ordered->inode->i_sb)) &&  in btrfs_finish_ordered_io()
3244 	    !test_bit(BTRFS_ORDERED_IOERR, &ordered->flags))  in btrfs_finish_ordered_io()
3245 		btrfs_finish_ordered_zoned(ordered);  in btrfs_finish_ordered_io()
3246 	return btrfs_finish_one_ordered(ordered);  in btrfs_finish_ordered_io()
4697 	struct btrfs_ordered_extent *ordered;  in btrfs_truncate_block()  local
4773 	ordered = btrfs_lookup_ordered_extent(inode, block_start);  in btrfs_truncate_block()
4774 	if (ordered) {  in btrfs_truncate_block()
4778 		btrfs_start_ordered_extent(ordered);  in btrfs_truncate_block()
4779 		btrfs_put_ordered_extent(ordered);  in btrfs_truncate_block()
5113  * the ordered io finishing, while we release here the extent_map and
6971 	struct btrfs_ordered_extent *ordered;  in btrfs_create_dio_extent()  local
6981 	ordered = btrfs_alloc_ordered_extent(inode, start, len, len,  in btrfs_create_dio_extent()
6986 	if (IS_ERR(ordered)) {  in btrfs_create_dio_extent()
6992 		em = ERR_CAST(ordered);  in btrfs_create_dio_extent()
6994 		ASSERT(!dio_data->ordered);  in btrfs_create_dio_extent()
6995 		dio_data->ordered = ordered;  in btrfs_create_dio_extent()
7068  *	 any ordered extents.
7177 	struct btrfs_ordered_extent *ordered;  in lock_extent_direct()  local
7190 		 * doing DIO to, so we need to make sure there's no ordered  in lock_extent_direct()
7193 		ordered = btrfs_lookup_ordered_range(BTRFS_I(inode), lockstart,  in lock_extent_direct()
7203 		if (!ordered &&  in lock_extent_direct()
7210 		if (ordered) {  in lock_extent_direct()
7212 				btrfs_put_ordered_extent(ordered);  in lock_extent_direct()
7217 			 * If we are doing a DIO read and the ordered extent we  in lock_extent_direct()
7223 			 * created an ordered extent for a previous extent map  in lock_extent_direct()
7232 			    test_bit(BTRFS_ORDERED_DIRECT, &ordered->flags))  in lock_extent_direct()
7233 				btrfs_start_ordered_extent(ordered);  in lock_extent_direct()
7236 			btrfs_put_ordered_extent(ordered);  in lock_extent_direct()
7245 			 * ordered dio extent we created before but did not have  in lock_extent_direct()
7248 			 * ordered extent to complete while holding a lock on  in lock_extent_direct()
7431 	 * We have created our ordered extent, so we can now release our reservation  in btrfs_get_blocks_direct_write()
7718 			btrfs_finish_ordered_extent(dio_data->ordered, NULL,  in btrfs_dio_iomap_end()
7726 		btrfs_put_ordered_extent(dio_data->ordered);  in btrfs_dio_iomap_end()
7727 		dio_data->ordered = NULL;  in btrfs_dio_iomap_end()
7750 		btrfs_finish_ordered_extent(bbio->ordered, NULL,  in btrfs_dio_end_io()
7782 	 * the ordered extent to match the submitted bio.  Hang on to the  in btrfs_dio_submit_io()
7785 	 * remaining pages is blocked on the outstanding ordered extent.  in btrfs_dio_submit_io()
7790 		ret = btrfs_extract_ordered_extent(bbio, dio_data->ordered);  in btrfs_dio_submit_io()
7792 			btrfs_finish_ordered_extent(dio_data->ordered, NULL,  in btrfs_dio_submit_io()
7848 	 * complete and writeback to start. We also need to wait for ordered  in btrfs_fiemap()
7864 	 * triggering writeback and waiting for the completion of IO and ordered  in btrfs_fiemap()
7978 	 * We have folio locked so no new ordered extent can be created on this  in btrfs_invalidate_folio()
7982 	 * Furthermore, endio function won't skip folio which has Ordered  in btrfs_invalidate_folio()
7984 	 * invalidate_folio to do the same ordered extent accounting twice  in btrfs_invalidate_folio()
7988 	 * do double ordered extent accounting on the same folio.  in btrfs_invalidate_folio()
8003 	 * still safe to wait for ordered extent to finish.  in btrfs_invalidate_folio()
8015 		struct btrfs_ordered_extent *ordered;  in btrfs_invalidate_folio()  local
8020 		ordered = btrfs_lookup_first_ordered_range(inode, cur,  in btrfs_invalidate_folio()
8022 		if (!ordered) {  in btrfs_invalidate_folio()
8025 			 * No ordered extent covering this range, we are safe  in btrfs_invalidate_folio()
8031 		if (ordered->file_offset > cur) {  in btrfs_invalidate_folio()
8034 			 * covered by any ordered extent.  in btrfs_invalidate_folio()
8036 			 * the ordered extent in the next iteration.  in btrfs_invalidate_folio()
8038 			range_end = ordered->file_offset - 1;  in btrfs_invalidate_folio()
8043 		range_end = min(ordered->file_offset + ordered->num_bytes - 1,  in btrfs_invalidate_folio()
8049 			 * If Ordered (Private2) is cleared, it means endio has  in btrfs_invalidate_folio()
8060 		 * for any ordered extents now. Don't clear EXTENT_DELALLOC_NEW  in btrfs_invalidate_folio()
8061 		 * here, must leave that up for the ordered extent completion.  in btrfs_invalidate_folio()
8073 		set_bit(BTRFS_ORDERED_TRUNCATED, &ordered->flags);  in btrfs_invalidate_folio()
8074 		ordered->truncated_len = min(ordered->truncated_len,  in btrfs_invalidate_folio()
8075 					     cur - ordered->file_offset);  in btrfs_invalidate_folio()
8079 		 * If the ordered extent has finished, we're safe to delete all  in btrfs_invalidate_folio()
8084 		if (btrfs_dec_test_ordered_pending(inode, &ordered,  in btrfs_invalidate_folio()
8086 			btrfs_finish_ordered_io(ordered);  in btrfs_invalidate_folio()
8088 			 * The ordered extent has finished, now we're again  in btrfs_invalidate_folio()
8094 		if (ordered)  in btrfs_invalidate_folio()
8095 			btrfs_put_ordered_extent(ordered);  in btrfs_invalidate_folio()
8121 	 * We have iterated through all ordered extents of the page, the page  in btrfs_invalidate_folio()
8122 	 * should not have Ordered (Private2) anymore, or the above iteration  in btrfs_invalidate_folio()
8153 	struct btrfs_ordered_extent *ordered;  in btrfs_page_mkwrite()  local
8216 	 * we can't set the delalloc bits if there are pending ordered  in btrfs_page_mkwrite()
8219 	ordered = btrfs_lookup_ordered_range(BTRFS_I(inode), page_start,  in btrfs_page_mkwrite()
8221 	if (ordered) {  in btrfs_page_mkwrite()
8225 		btrfs_start_ordered_extent(ordered);  in btrfs_page_mkwrite()
8226 		btrfs_put_ordered_extent(ordered);  in btrfs_page_mkwrite()
8582 	struct btrfs_ordered_extent *ordered;  in btrfs_destroy_inode()  local
8608 	 * If this is a free space inode do not take the ordered extents lockdep  in btrfs_destroy_inode()
8614 		ordered = btrfs_lookup_first_ordered_extent(inode, (u64)-1);  in btrfs_destroy_inode()
8615 		if (!ordered)  in btrfs_destroy_inode()
8619 				  "found ordered extent %llu %llu on inode cleanup",  in btrfs_destroy_inode()
8620 				  ordered->file_offset, ordered->num_bytes);  in btrfs_destroy_inode()
8625 			btrfs_remove_ordered_extent(inode, ordered);  in btrfs_destroy_inode()
8626 			btrfs_put_ordered_extent(ordered);  in btrfs_destroy_inode()
8627 			btrfs_put_ordered_extent(ordered);  in btrfs_destroy_inode()
10119 		struct btrfs_ordered_extent *ordered;  in btrfs_encoded_read()  local
10126 		ordered = btrfs_lookup_ordered_range(inode, start,  in btrfs_encoded_read()
10128 		if (!ordered)  in btrfs_encoded_read()
10130 		btrfs_put_ordered_extent(ordered);  in btrfs_encoded_read()
10242 	struct btrfs_ordered_extent *ordered;  in btrfs_do_encoded_write()  local
10362 		struct btrfs_ordered_extent *ordered;  in btrfs_do_encoded_write()  local
10373 		ordered = btrfs_lookup_ordered_range(inode, start, num_bytes);  in btrfs_do_encoded_write()
10374 		if (!ordered &&  in btrfs_do_encoded_write()
10377 		if (ordered)  in btrfs_do_encoded_write()
10378 			btrfs_put_ordered_extent(ordered);  in btrfs_do_encoded_write()
10426 	ordered = btrfs_alloc_ordered_extent(inode, start, num_bytes, ram_bytes,  in btrfs_do_encoded_write()
10432 	if (IS_ERR(ordered)) {  in btrfs_do_encoded_write()
10434 		ret = PTR_ERR(ordered);  in btrfs_do_encoded_write()
10446 	btrfs_submit_compressed_write(ordered, pages, nr_pages, 0, false);  in btrfs_do_encoded_write()
10901  * Verify that there are no ordered extents for a given file range.
10910  * we have flushed all delalloc in the range, we have waited for all ordered
10917 	struct btrfs_ordered_extent *ordered;  in btrfs_assert_inode_range_clean()  local
10922 	ordered = btrfs_lookup_first_ordered_range(inode, start, end + 1 - start);  in btrfs_assert_inode_range_clean()
10923 	if (ordered) {  in btrfs_assert_inode_range_clean()
10925 "found unexpected ordered extent in file range [%llu, %llu] for inode %llu root %llu (ordered range…  in btrfs_assert_inode_range_clean()
10927 			  ordered->file_offset,  in btrfs_assert_inode_range_clean()
10928 			  ordered->file_offset + ordered->num_bytes - 1);  in btrfs_assert_inode_range_clean()
10929 		btrfs_put_ordered_extent(ordered);  in btrfs_assert_inode_range_clean()
10932 	ASSERT(ordered == NULL);  in btrfs_assert_inode_range_clean()