43 #include "ordered-data.h"
83 	struct btrfs_ordered_extent *ordered;  member
401  * Cleanup all submitted ordered extents in specified range to handle errors
407  * to be released, which we want to happen only when finishing the ordered
429 		 * clear page Ordered and run the ordered extent accounting.  in btrfs_cleanup_ordered_extents()
431 		 * Here we can't just clear the Ordered bit, or  in btrfs_cleanup_ordered_extents()
433 		 * for the page range, and the ordered extent will never finish.  in btrfs_cleanup_ordered_extents()
445 		 * Here we just clear all Ordered bits for every page in the  in btrfs_cleanup_ordered_extents()
447 		 * the ordered extent accounting for the range.  in btrfs_cleanup_ordered_extents()
820  * This is done inside an ordered work queue, and the compression is spread
821  * across many cpus.  The actual IO submission is step two, and the ordered work
1105 	struct btrfs_ordered_extent *ordered;  in submit_one_async_extent()  local
1165 	ordered = btrfs_alloc_ordered_extent(inode, start,	/* file_offset */  in submit_one_async_extent()
1173 	if (IS_ERR(ordered)) {  in submit_one_async_extent()
1175 		ret = PTR_ERR(ordered);  in submit_one_async_extent()
1184 	btrfs_submit_compressed_write(ordered,  in submit_one_async_extent()
1250  * allocate extents on disk for the range, and create ordered data structs
1270  * while-loop, the ordered extents created in previous iterations are kept
1376 		struct btrfs_ordered_extent *ordered;  in cow_file_range()  local
1427 		ordered = btrfs_alloc_ordered_extent(inode, start, ram_size,  in cow_file_range()
1431 		if (IS_ERR(ordered)) {  in cow_file_range()
1432 			ret = PTR_ERR(ordered);  in cow_file_range()
1437 			ret = btrfs_reloc_clone_csums(ordered);  in cow_file_range()
1443 			 * at out_unlock label to free meta of this ordered  in cow_file_range()
1448 			 * skip current ordered extent.  in cow_file_range()
1455 		btrfs_put_ordered_extent(ordered);  in cow_file_range()
1464 		 * Do set the Ordered (Private2) bit so we know this page was  in cow_file_range()
1485 		 * free metadata of current ordered extent, we're OK to exit.  in cow_file_range()
1515 	 * For the range (1). We have already instantiated the ordered extents  in cow_file_range()
1535 	 * (or a subrange) and failed to create the respective ordered extent,  in cow_file_range()
1569  * Phase two of compressed writeback.  This is the ordered portion of the code,
1998 		struct btrfs_ordered_extent *ordered;  in run_delalloc_nocow()  local
2160 		ordered = btrfs_alloc_ordered_extent(inode, cur_offset,  in run_delalloc_nocow()
2168 		if (IS_ERR(ordered)) {  in run_delalloc_nocow()
2173 			ret = PTR_ERR(ordered);  in run_delalloc_nocow()
2181 			 * from freeing metadata of created ordered extent.  in run_delalloc_nocow()
2183 			ret = btrfs_reloc_clone_csums(ordered);  in run_delalloc_nocow()
2184 		btrfs_put_ordered_extent(ordered);  in run_delalloc_nocow()
2196 		 * handler, as metadata for created ordered extent will only  in run_delalloc_nocow()
2555 					struct btrfs_ordered_extent *ordered)  in btrfs_extract_ordered_extent()  argument
2562 	/* Must always be called for the beginning of an ordered extent. */  in btrfs_extract_ordered_extent()
2563 	if (WARN_ON_ONCE(start != ordered->disk_bytenr))  in btrfs_extract_ordered_extent()
2566 	/* No need to split if the ordered extent covers the entire bio. */  in btrfs_extract_ordered_extent()
2567 	if (ordered->disk_num_bytes == len) {  in btrfs_extract_ordered_extent()
2568 		refcount_inc(&ordered->refs);  in btrfs_extract_ordered_extent()
2569 		bbio->ordered = ordered;  in btrfs_extract_ordered_extent()
2577 	if (!test_bit(BTRFS_ORDERED_NOCOW, &ordered->flags)) {  in btrfs_extract_ordered_extent()
2579 				       ordered->num_bytes, len,  in btrfs_extract_ordered_extent()
2580 				       ordered->disk_bytenr);  in btrfs_extract_ordered_extent()
2585 	new = btrfs_split_ordered_extent(ordered, len);  in btrfs_extract_ordered_extent()
2588 	bbio->ordered = new;  in btrfs_extract_ordered_extent()
2593  * given a list of ordered sums record them in the inode.  This happens
2693 	struct btrfs_ordered_extent *ordered;  in btrfs_writepage_fixup_worker()  local
2726 		 *    while waiting for the ordered extent.  in btrfs_writepage_fixup_worker()
2755 	/* already ordered? We're done */  in btrfs_writepage_fixup_worker()
2759 	ordered = btrfs_lookup_ordered_range(inode, page_start, PAGE_SIZE);  in btrfs_writepage_fixup_worker()
2760 	if (ordered) {  in btrfs_writepage_fixup_worker()
2764 		btrfs_start_ordered_extent(ordered);  in btrfs_writepage_fixup_worker()
2765 		btrfs_put_ordered_extent(ordered);  in btrfs_writepage_fixup_worker()
2817  * properly happens and the data=ordered rules are followed.
2819  * In our case any range that doesn't have the ORDERED bit set
2821  * to fix it up.  The async helper will wait for ordered extents, set
2830 	/* This page has ordered extent covering it already */  in btrfs_writepage_cow_fixup()
2930 	 * EXTENT_DELALLOC_BIT bit through the ordered extent completion.  in insert_reserved_file_extent()
3000 	 * For delalloc, when completing an ordered extent we update the inode's  in insert_ordered_extent_file_extent()
3003 	 * except if the ordered extent was truncated.  in insert_ordered_extent_file_extent()
3015  * As ordered data IO finishes, this gets called so we can finish
3016  * an ordered extent if the range of bytes in the file it covers are
3163 		 * If we failed to finish this ordered extent for any reason we  in btrfs_finish_one_ordered()
3164 		 * need to make sure BTRFS_ORDERED_IOERR is set on the ordered  in btrfs_finish_one_ordered()
3168 		 * marking this ordered extent as failed.  in btrfs_finish_one_ordered()
3197 		 * If the ordered extent had an IOERR or something else went  in btrfs_finish_one_ordered()
3198 		 * wrong we need to return the space for this ordered extent  in btrfs_finish_one_ordered()
3224 			 * the ordered extent was created.  in btrfs_finish_one_ordered()
3234 	 * updating everything for this ordered extent.  in btrfs_finish_one_ordered()
3246 int btrfs_finish_ordered_io(struct btrfs_ordered_extent *ordered)  in btrfs_finish_ordered_io()  argument
3248 	if (btrfs_is_zoned(btrfs_sb(ordered->inode->i_sb)) &&  in btrfs_finish_ordered_io()
3249 	    !test_bit(BTRFS_ORDERED_IOERR, &ordered->flags))  in btrfs_finish_ordered_io()
3250 		btrfs_finish_ordered_zoned(ordered);  in btrfs_finish_ordered_io()
3251 	return btrfs_finish_one_ordered(ordered);  in btrfs_finish_ordered_io()
4702 	struct btrfs_ordered_extent *ordered;  in btrfs_truncate_block()  local
4778 	ordered = btrfs_lookup_ordered_extent(inode, block_start);  in btrfs_truncate_block()
4779 	if (ordered) {  in btrfs_truncate_block()
4783 		btrfs_start_ordered_extent(ordered);  in btrfs_truncate_block()
4784 		btrfs_put_ordered_extent(ordered);  in btrfs_truncate_block()
5118  * the ordered io finishing, while we release here the extent_map and
6976 	struct btrfs_ordered_extent *ordered;  in btrfs_create_dio_extent()  local
6986 	ordered = btrfs_alloc_ordered_extent(inode, start, len, len,  in btrfs_create_dio_extent()
6991 	if (IS_ERR(ordered)) {  in btrfs_create_dio_extent()
6997 		em = ERR_CAST(ordered);  in btrfs_create_dio_extent()
6999 		ASSERT(!dio_data->ordered);  in btrfs_create_dio_extent()
7000 		dio_data->ordered = ordered;  in btrfs_create_dio_extent()
7073  *	 any ordered extents.
7182 	struct btrfs_ordered_extent *ordered;  in lock_extent_direct()  local
7195 		 * doing DIO to, so we need to make sure there's no ordered  in lock_extent_direct()
7198 		ordered = btrfs_lookup_ordered_range(BTRFS_I(inode), lockstart,  in lock_extent_direct()
7208 		if (!ordered &&  in lock_extent_direct()
7215 		if (ordered) {  in lock_extent_direct()
7217 				btrfs_put_ordered_extent(ordered);  in lock_extent_direct()
7222 			 * If we are doing a DIO read and the ordered extent we  in lock_extent_direct()
7228 			 * created an ordered extent for a previous extent map  in lock_extent_direct()
7237 			    test_bit(BTRFS_ORDERED_DIRECT, &ordered->flags))  in lock_extent_direct()
7238 				btrfs_start_ordered_extent(ordered);  in lock_extent_direct()
7241 			btrfs_put_ordered_extent(ordered);  in lock_extent_direct()
7250 			 * ordered dio extent we created before but did not have  in lock_extent_direct()
7253 			 * ordered extent to complete while holding a lock on  in lock_extent_direct()
7436 	 * We have created our ordered extent, so we can now release our reservation  in btrfs_get_blocks_direct_write()
7723 			btrfs_finish_ordered_extent(dio_data->ordered, NULL,  in btrfs_dio_iomap_end()
7731 		btrfs_put_ordered_extent(dio_data->ordered);  in btrfs_dio_iomap_end()
7732 		dio_data->ordered = NULL;  in btrfs_dio_iomap_end()
7755 		btrfs_finish_ordered_extent(bbio->ordered, NULL,  in btrfs_dio_end_io()
7787 	 * the ordered extent to match the submitted bio.  Hang on to the  in btrfs_dio_submit_io()
7790 	 * remaining pages is blocked on the outstanding ordered extent.  in btrfs_dio_submit_io()
7795 		ret = btrfs_extract_ordered_extent(bbio, dio_data->ordered);  in btrfs_dio_submit_io()
7797 			btrfs_finish_ordered_extent(dio_data->ordered, NULL,  in btrfs_dio_submit_io()
7853 	 * complete and writeback to start. We also need to wait for ordered  in btrfs_fiemap()
7869 	 * triggering writeback and waiting for the completion of IO and ordered  in btrfs_fiemap()
7983 	 * We have folio locked so no new ordered extent can be created on this  in btrfs_invalidate_folio()
7987 	 * Furthermore, endio function won't skip folio which has Ordered  in btrfs_invalidate_folio()
7989 	 * invalidate_folio to do the same ordered extent accounting twice  in btrfs_invalidate_folio()
7993 	 * do double ordered extent accounting on the same folio.  in btrfs_invalidate_folio()
8008 	 * still safe to wait for ordered extent to finish.  in btrfs_invalidate_folio()
8020 		struct btrfs_ordered_extent *ordered;  in btrfs_invalidate_folio()  local
8025 		ordered = btrfs_lookup_first_ordered_range(inode, cur,  in btrfs_invalidate_folio()
8027 		if (!ordered) {  in btrfs_invalidate_folio()
8030 			 * No ordered extent covering this range, we are safe  in btrfs_invalidate_folio()
8036 		if (ordered->file_offset > cur) {  in btrfs_invalidate_folio()
8039 			 * covered by any ordered extent.  in btrfs_invalidate_folio()
8041 			 * the ordered extent in the next iteration.  in btrfs_invalidate_folio()
8043 			range_end = ordered->file_offset - 1;  in btrfs_invalidate_folio()
8048 		range_end = min(ordered->file_offset + ordered->num_bytes - 1,  in btrfs_invalidate_folio()
8054 			 * If Ordered (Private2) is cleared, it means endio has  in btrfs_invalidate_folio()
8065 		 * for any ordered extents now. Don't clear EXTENT_DELALLOC_NEW  in btrfs_invalidate_folio()
8066 		 * here, must leave that up for the ordered extent completion.  in btrfs_invalidate_folio()
8078 		set_bit(BTRFS_ORDERED_TRUNCATED, &ordered->flags);  in btrfs_invalidate_folio()
8079 		ordered->truncated_len = min(ordered->truncated_len,  in btrfs_invalidate_folio()
8080 					     cur - ordered->file_offset);  in btrfs_invalidate_folio()
8084 		 * If the ordered extent has finished, we're safe to delete all  in btrfs_invalidate_folio()
8089 		if (btrfs_dec_test_ordered_pending(inode, &ordered,  in btrfs_invalidate_folio()
8091 			btrfs_finish_ordered_io(ordered);  in btrfs_invalidate_folio()
8093 			 * The ordered extent has finished, now we're again  in btrfs_invalidate_folio()
8099 		if (ordered)  in btrfs_invalidate_folio()
8100 			btrfs_put_ordered_extent(ordered);  in btrfs_invalidate_folio()
8126 	 * We have iterated through all ordered extents of the page, the page  in btrfs_invalidate_folio()
8127 	 * should not have Ordered (Private2) anymore, or the above iteration  in btrfs_invalidate_folio()
8158 	struct btrfs_ordered_extent *ordered;  in btrfs_page_mkwrite()  local
8221 	 * we can't set the delalloc bits if there are pending ordered  in btrfs_page_mkwrite()
8224 	ordered = btrfs_lookup_ordered_range(BTRFS_I(inode), page_start,  in btrfs_page_mkwrite()
8226 	if (ordered) {  in btrfs_page_mkwrite()
8230 		btrfs_start_ordered_extent(ordered);  in btrfs_page_mkwrite()
8231 		btrfs_put_ordered_extent(ordered);  in btrfs_page_mkwrite()
8587 	struct btrfs_ordered_extent *ordered;  in btrfs_destroy_inode()  local
8613 	 * If this is a free space inode do not take the ordered extents lockdep  in btrfs_destroy_inode()
8619 		ordered = btrfs_lookup_first_ordered_extent(inode, (u64)-1);  in btrfs_destroy_inode()
8620 		if (!ordered)  in btrfs_destroy_inode()
8624 				  "found ordered extent %llu %llu on inode cleanup",  in btrfs_destroy_inode()
8625 				  ordered->file_offset, ordered->num_bytes);  in btrfs_destroy_inode()
8630 			btrfs_remove_ordered_extent(inode, ordered);  in btrfs_destroy_inode()
8631 			btrfs_put_ordered_extent(ordered);  in btrfs_destroy_inode()
8632 			btrfs_put_ordered_extent(ordered);  in btrfs_destroy_inode()
10171 		struct btrfs_ordered_extent *ordered;  in btrfs_encoded_read()  local
10178 		ordered = btrfs_lookup_ordered_range(inode, start,  in btrfs_encoded_read()
10180 		if (!ordered)  in btrfs_encoded_read()
10182 		btrfs_put_ordered_extent(ordered);  in btrfs_encoded_read()
10294 	struct btrfs_ordered_extent *ordered;  in btrfs_do_encoded_write()  local
10414 		struct btrfs_ordered_extent *ordered;  in btrfs_do_encoded_write()  local
10425 		ordered = btrfs_lookup_ordered_range(inode, start, num_bytes);  in btrfs_do_encoded_write()
10426 		if (!ordered &&  in btrfs_do_encoded_write()
10429 		if (ordered)  in btrfs_do_encoded_write()
10430 			btrfs_put_ordered_extent(ordered);  in btrfs_do_encoded_write()
10478 	ordered = btrfs_alloc_ordered_extent(inode, start, num_bytes, ram_bytes,  in btrfs_do_encoded_write()
10484 	if (IS_ERR(ordered)) {  in btrfs_do_encoded_write()
10486 		ret = PTR_ERR(ordered);  in btrfs_do_encoded_write()
10498 	btrfs_submit_compressed_write(ordered, pages, nr_pages, 0, false);  in btrfs_do_encoded_write()
10953  * Verify that there are no ordered extents for a given file range.
10962  * we have flushed all delalloc in the range, we have waited for all ordered
10969 	struct btrfs_ordered_extent *ordered;  in btrfs_assert_inode_range_clean()  local
10974 	ordered = btrfs_lookup_first_ordered_range(inode, start, end + 1 - start);  in btrfs_assert_inode_range_clean()
10975 	if (ordered) {  in btrfs_assert_inode_range_clean()
10977 "found unexpected ordered extent in file range [%llu, %llu] for inode %llu root %llu (ordered range…  in btrfs_assert_inode_range_clean()
10979 			  ordered->file_offset,  in btrfs_assert_inode_range_clean()
10980 			  ordered->file_offset + ordered->num_bytes - 1);  in btrfs_assert_inode_range_clean()
10981 		btrfs_put_ordered_extent(ordered);  in btrfs_assert_inode_range_clean()
10984 	ASSERT(ordered == NULL);  in btrfs_assert_inode_range_clean()