• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Copyright (C) 2007 Oracle.  All rights reserved.
4  */
5 
6 #include <linux/fs.h>
7 #include <linux/pagemap.h>
8 #include <linux/time.h>
9 #include <linux/init.h>
10 #include <linux/string.h>
11 #include <linux/backing-dev.h>
12 #include <linux/falloc.h>
13 #include <linux/writeback.h>
14 #include <linux/compat.h>
15 #include <linux/slab.h>
16 #include <linux/btrfs.h>
17 #include <linux/uio.h>
18 #include <linux/iversion.h>
19 #include <linux/fsverity.h>
20 #include "ctree.h"
21 #include "disk-io.h"
22 #include "transaction.h"
23 #include "btrfs_inode.h"
24 #include "print-tree.h"
25 #include "tree-log.h"
26 #include "locking.h"
27 #include "volumes.h"
28 #include "qgroup.h"
29 #include "compression.h"
30 #include "delalloc-space.h"
31 #include "reflink.h"
32 #include "subpage.h"
33 
34 static struct kmem_cache *btrfs_inode_defrag_cachep;
35 /*
36  * when auto defrag is enabled we
37  * queue up these defrag structs to remember which
38  * inodes need defragging passes
39  */
40 struct inode_defrag {
41 	struct rb_node rb_node;
42 	/* objectid */
43 	u64 ino;
44 	/*
45 	 * transid where the defrag was added, we search for
46 	 * extents newer than this
47 	 */
48 	u64 transid;
49 
50 	/* root objectid */
51 	u64 root;
52 
53 	/* last offset we were able to defrag */
54 	u64 last_offset;
55 
56 	/* if we've wrapped around back to zero once already */
57 	int cycled;
58 };
59 
__compare_inode_defrag(struct inode_defrag * defrag1,struct inode_defrag * defrag2)60 static int __compare_inode_defrag(struct inode_defrag *defrag1,
61 				  struct inode_defrag *defrag2)
62 {
63 	if (defrag1->root > defrag2->root)
64 		return 1;
65 	else if (defrag1->root < defrag2->root)
66 		return -1;
67 	else if (defrag1->ino > defrag2->ino)
68 		return 1;
69 	else if (defrag1->ino < defrag2->ino)
70 		return -1;
71 	else
72 		return 0;
73 }
74 
75 /* pop a record for an inode into the defrag tree.  The lock
76  * must be held already
77  *
78  * If you're inserting a record for an older transid than an
79  * existing record, the transid already in the tree is lowered
80  *
81  * If an existing record is found the defrag item you
82  * pass in is freed
83  */
__btrfs_add_inode_defrag(struct btrfs_inode * inode,struct inode_defrag * defrag)84 static int __btrfs_add_inode_defrag(struct btrfs_inode *inode,
85 				    struct inode_defrag *defrag)
86 {
87 	struct btrfs_fs_info *fs_info = inode->root->fs_info;
88 	struct inode_defrag *entry;
89 	struct rb_node **p;
90 	struct rb_node *parent = NULL;
91 	int ret;
92 
93 	p = &fs_info->defrag_inodes.rb_node;
94 	while (*p) {
95 		parent = *p;
96 		entry = rb_entry(parent, struct inode_defrag, rb_node);
97 
98 		ret = __compare_inode_defrag(defrag, entry);
99 		if (ret < 0)
100 			p = &parent->rb_left;
101 		else if (ret > 0)
102 			p = &parent->rb_right;
103 		else {
104 			/* if we're reinserting an entry for
105 			 * an old defrag run, make sure to
106 			 * lower the transid of our existing record
107 			 */
108 			if (defrag->transid < entry->transid)
109 				entry->transid = defrag->transid;
110 			if (defrag->last_offset > entry->last_offset)
111 				entry->last_offset = defrag->last_offset;
112 			return -EEXIST;
113 		}
114 	}
115 	set_bit(BTRFS_INODE_IN_DEFRAG, &inode->runtime_flags);
116 	rb_link_node(&defrag->rb_node, parent, p);
117 	rb_insert_color(&defrag->rb_node, &fs_info->defrag_inodes);
118 	return 0;
119 }
120 
__need_auto_defrag(struct btrfs_fs_info * fs_info)121 static inline int __need_auto_defrag(struct btrfs_fs_info *fs_info)
122 {
123 	if (!btrfs_test_opt(fs_info, AUTO_DEFRAG))
124 		return 0;
125 
126 	if (btrfs_fs_closing(fs_info))
127 		return 0;
128 
129 	return 1;
130 }
131 
132 /*
133  * insert a defrag record for this inode if auto defrag is
134  * enabled
135  */
btrfs_add_inode_defrag(struct btrfs_trans_handle * trans,struct btrfs_inode * inode)136 int btrfs_add_inode_defrag(struct btrfs_trans_handle *trans,
137 			   struct btrfs_inode *inode)
138 {
139 	struct btrfs_root *root = inode->root;
140 	struct btrfs_fs_info *fs_info = root->fs_info;
141 	struct inode_defrag *defrag;
142 	u64 transid;
143 	int ret;
144 
145 	if (!__need_auto_defrag(fs_info))
146 		return 0;
147 
148 	if (test_bit(BTRFS_INODE_IN_DEFRAG, &inode->runtime_flags))
149 		return 0;
150 
151 	if (trans)
152 		transid = trans->transid;
153 	else
154 		transid = inode->root->last_trans;
155 
156 	defrag = kmem_cache_zalloc(btrfs_inode_defrag_cachep, GFP_NOFS);
157 	if (!defrag)
158 		return -ENOMEM;
159 
160 	defrag->ino = btrfs_ino(inode);
161 	defrag->transid = transid;
162 	defrag->root = root->root_key.objectid;
163 
164 	spin_lock(&fs_info->defrag_inodes_lock);
165 	if (!test_bit(BTRFS_INODE_IN_DEFRAG, &inode->runtime_flags)) {
166 		/*
167 		 * If we set IN_DEFRAG flag and evict the inode from memory,
168 		 * and then re-read this inode, this new inode doesn't have
169 		 * IN_DEFRAG flag. At the case, we may find the existed defrag.
170 		 */
171 		ret = __btrfs_add_inode_defrag(inode, defrag);
172 		if (ret)
173 			kmem_cache_free(btrfs_inode_defrag_cachep, defrag);
174 	} else {
175 		kmem_cache_free(btrfs_inode_defrag_cachep, defrag);
176 	}
177 	spin_unlock(&fs_info->defrag_inodes_lock);
178 	return 0;
179 }
180 
181 /*
182  * Requeue the defrag object. If there is a defrag object that points to
183  * the same inode in the tree, we will merge them together (by
184  * __btrfs_add_inode_defrag()) and free the one that we want to requeue.
185  */
btrfs_requeue_inode_defrag(struct btrfs_inode * inode,struct inode_defrag * defrag)186 static void btrfs_requeue_inode_defrag(struct btrfs_inode *inode,
187 				       struct inode_defrag *defrag)
188 {
189 	struct btrfs_fs_info *fs_info = inode->root->fs_info;
190 	int ret;
191 
192 	if (!__need_auto_defrag(fs_info))
193 		goto out;
194 
195 	/*
196 	 * Here we don't check the IN_DEFRAG flag, because we need merge
197 	 * them together.
198 	 */
199 	spin_lock(&fs_info->defrag_inodes_lock);
200 	ret = __btrfs_add_inode_defrag(inode, defrag);
201 	spin_unlock(&fs_info->defrag_inodes_lock);
202 	if (ret)
203 		goto out;
204 	return;
205 out:
206 	kmem_cache_free(btrfs_inode_defrag_cachep, defrag);
207 }
208 
209 /*
210  * pick the defragable inode that we want, if it doesn't exist, we will get
211  * the next one.
212  */
213 static struct inode_defrag *
btrfs_pick_defrag_inode(struct btrfs_fs_info * fs_info,u64 root,u64 ino)214 btrfs_pick_defrag_inode(struct btrfs_fs_info *fs_info, u64 root, u64 ino)
215 {
216 	struct inode_defrag *entry = NULL;
217 	struct inode_defrag tmp;
218 	struct rb_node *p;
219 	struct rb_node *parent = NULL;
220 	int ret;
221 
222 	tmp.ino = ino;
223 	tmp.root = root;
224 
225 	spin_lock(&fs_info->defrag_inodes_lock);
226 	p = fs_info->defrag_inodes.rb_node;
227 	while (p) {
228 		parent = p;
229 		entry = rb_entry(parent, struct inode_defrag, rb_node);
230 
231 		ret = __compare_inode_defrag(&tmp, entry);
232 		if (ret < 0)
233 			p = parent->rb_left;
234 		else if (ret > 0)
235 			p = parent->rb_right;
236 		else
237 			goto out;
238 	}
239 
240 	if (parent && __compare_inode_defrag(&tmp, entry) > 0) {
241 		parent = rb_next(parent);
242 		if (parent)
243 			entry = rb_entry(parent, struct inode_defrag, rb_node);
244 		else
245 			entry = NULL;
246 	}
247 out:
248 	if (entry)
249 		rb_erase(parent, &fs_info->defrag_inodes);
250 	spin_unlock(&fs_info->defrag_inodes_lock);
251 	return entry;
252 }
253 
btrfs_cleanup_defrag_inodes(struct btrfs_fs_info * fs_info)254 void btrfs_cleanup_defrag_inodes(struct btrfs_fs_info *fs_info)
255 {
256 	struct inode_defrag *defrag;
257 	struct rb_node *node;
258 
259 	spin_lock(&fs_info->defrag_inodes_lock);
260 	node = rb_first(&fs_info->defrag_inodes);
261 	while (node) {
262 		rb_erase(node, &fs_info->defrag_inodes);
263 		defrag = rb_entry(node, struct inode_defrag, rb_node);
264 		kmem_cache_free(btrfs_inode_defrag_cachep, defrag);
265 
266 		cond_resched_lock(&fs_info->defrag_inodes_lock);
267 
268 		node = rb_first(&fs_info->defrag_inodes);
269 	}
270 	spin_unlock(&fs_info->defrag_inodes_lock);
271 }
272 
273 #define BTRFS_DEFRAG_BATCH	1024
274 
__btrfs_run_defrag_inode(struct btrfs_fs_info * fs_info,struct inode_defrag * defrag)275 static int __btrfs_run_defrag_inode(struct btrfs_fs_info *fs_info,
276 				    struct inode_defrag *defrag)
277 {
278 	struct btrfs_root *inode_root;
279 	struct inode *inode;
280 	struct btrfs_ioctl_defrag_range_args range;
281 	int num_defrag;
282 	int ret;
283 
284 	/* get the inode */
285 	inode_root = btrfs_get_fs_root(fs_info, defrag->root, true);
286 	if (IS_ERR(inode_root)) {
287 		ret = PTR_ERR(inode_root);
288 		goto cleanup;
289 	}
290 
291 	inode = btrfs_iget(fs_info->sb, defrag->ino, inode_root);
292 	btrfs_put_root(inode_root);
293 	if (IS_ERR(inode)) {
294 		ret = PTR_ERR(inode);
295 		goto cleanup;
296 	}
297 
298 	/* do a chunk of defrag */
299 	clear_bit(BTRFS_INODE_IN_DEFRAG, &BTRFS_I(inode)->runtime_flags);
300 	memset(&range, 0, sizeof(range));
301 	range.len = (u64)-1;
302 	range.start = defrag->last_offset;
303 
304 	sb_start_write(fs_info->sb);
305 	num_defrag = btrfs_defrag_file(inode, NULL, &range, defrag->transid,
306 				       BTRFS_DEFRAG_BATCH);
307 	sb_end_write(fs_info->sb);
308 	/*
309 	 * if we filled the whole defrag batch, there
310 	 * must be more work to do.  Queue this defrag
311 	 * again
312 	 */
313 	if (num_defrag == BTRFS_DEFRAG_BATCH) {
314 		defrag->last_offset = range.start;
315 		btrfs_requeue_inode_defrag(BTRFS_I(inode), defrag);
316 	} else if (defrag->last_offset && !defrag->cycled) {
317 		/*
318 		 * we didn't fill our defrag batch, but
319 		 * we didn't start at zero.  Make sure we loop
320 		 * around to the start of the file.
321 		 */
322 		defrag->last_offset = 0;
323 		defrag->cycled = 1;
324 		btrfs_requeue_inode_defrag(BTRFS_I(inode), defrag);
325 	} else {
326 		kmem_cache_free(btrfs_inode_defrag_cachep, defrag);
327 	}
328 
329 	iput(inode);
330 	return 0;
331 cleanup:
332 	kmem_cache_free(btrfs_inode_defrag_cachep, defrag);
333 	return ret;
334 }
335 
336 /*
337  * run through the list of inodes in the FS that need
338  * defragging
339  */
btrfs_run_defrag_inodes(struct btrfs_fs_info * fs_info)340 int btrfs_run_defrag_inodes(struct btrfs_fs_info *fs_info)
341 {
342 	struct inode_defrag *defrag;
343 	u64 first_ino = 0;
344 	u64 root_objectid = 0;
345 
346 	atomic_inc(&fs_info->defrag_running);
347 	while (1) {
348 		/* Pause the auto defragger. */
349 		if (test_bit(BTRFS_FS_STATE_REMOUNTING,
350 			     &fs_info->fs_state))
351 			break;
352 
353 		if (!__need_auto_defrag(fs_info))
354 			break;
355 
356 		/* find an inode to defrag */
357 		defrag = btrfs_pick_defrag_inode(fs_info, root_objectid,
358 						 first_ino);
359 		if (!defrag) {
360 			if (root_objectid || first_ino) {
361 				root_objectid = 0;
362 				first_ino = 0;
363 				continue;
364 			} else {
365 				break;
366 			}
367 		}
368 
369 		first_ino = defrag->ino + 1;
370 		root_objectid = defrag->root;
371 
372 		__btrfs_run_defrag_inode(fs_info, defrag);
373 	}
374 	atomic_dec(&fs_info->defrag_running);
375 
376 	/*
377 	 * during unmount, we use the transaction_wait queue to
378 	 * wait for the defragger to stop
379 	 */
380 	wake_up(&fs_info->transaction_wait);
381 	return 0;
382 }
383 
384 /* simple helper to fault in pages and copy.  This should go away
385  * and be replaced with calls into generic code.
386  */
btrfs_copy_from_user(loff_t pos,size_t write_bytes,struct page ** prepared_pages,struct iov_iter * i)387 static noinline int btrfs_copy_from_user(loff_t pos, size_t write_bytes,
388 					 struct page **prepared_pages,
389 					 struct iov_iter *i)
390 {
391 	size_t copied = 0;
392 	size_t total_copied = 0;
393 	int pg = 0;
394 	int offset = offset_in_page(pos);
395 
396 	while (write_bytes > 0) {
397 		size_t count = min_t(size_t,
398 				     PAGE_SIZE - offset, write_bytes);
399 		struct page *page = prepared_pages[pg];
400 		/*
401 		 * Copy data from userspace to the current page
402 		 */
403 		copied = copy_page_from_iter_atomic(page, offset, count, i);
404 
405 		/* Flush processor's dcache for this page */
406 		flush_dcache_page(page);
407 
408 		/*
409 		 * if we get a partial write, we can end up with
410 		 * partially up to date pages.  These add
411 		 * a lot of complexity, so make sure they don't
412 		 * happen by forcing this copy to be retried.
413 		 *
414 		 * The rest of the btrfs_file_write code will fall
415 		 * back to page at a time copies after we return 0.
416 		 */
417 		if (unlikely(copied < count)) {
418 			if (!PageUptodate(page)) {
419 				iov_iter_revert(i, copied);
420 				copied = 0;
421 			}
422 			if (!copied)
423 				break;
424 		}
425 
426 		write_bytes -= copied;
427 		total_copied += copied;
428 		offset += copied;
429 		if (offset == PAGE_SIZE) {
430 			pg++;
431 			offset = 0;
432 		}
433 	}
434 	return total_copied;
435 }
436 
437 /*
438  * unlocks pages after btrfs_file_write is done with them
439  */
btrfs_drop_pages(struct page ** pages,size_t num_pages)440 static void btrfs_drop_pages(struct page **pages, size_t num_pages)
441 {
442 	size_t i;
443 	for (i = 0; i < num_pages; i++) {
444 		/* page checked is some magic around finding pages that
445 		 * have been modified without going through btrfs_set_page_dirty
446 		 * clear it here. There should be no need to mark the pages
447 		 * accessed as prepare_pages should have marked them accessed
448 		 * in prepare_pages via find_or_create_page()
449 		 */
450 		ClearPageChecked(pages[i]);
451 		unlock_page(pages[i]);
452 		put_page(pages[i]);
453 	}
454 }
455 
456 /*
457  * After btrfs_copy_from_user(), update the following things for delalloc:
458  * - Mark newly dirtied pages as DELALLOC in the io tree.
459  *   Used to advise which range is to be written back.
460  * - Mark modified pages as Uptodate/Dirty and not needing COW fixup
461  * - Update inode size for past EOF write
462  */
btrfs_dirty_pages(struct btrfs_inode * inode,struct page ** pages,size_t num_pages,loff_t pos,size_t write_bytes,struct extent_state ** cached,bool noreserve)463 int btrfs_dirty_pages(struct btrfs_inode *inode, struct page **pages,
464 		      size_t num_pages, loff_t pos, size_t write_bytes,
465 		      struct extent_state **cached, bool noreserve)
466 {
467 	struct btrfs_fs_info *fs_info = inode->root->fs_info;
468 	int err = 0;
469 	int i;
470 	u64 num_bytes;
471 	u64 start_pos;
472 	u64 end_of_last_block;
473 	u64 end_pos = pos + write_bytes;
474 	loff_t isize = i_size_read(&inode->vfs_inode);
475 	unsigned int extra_bits = 0;
476 
477 	if (write_bytes == 0)
478 		return 0;
479 
480 	if (noreserve)
481 		extra_bits |= EXTENT_NORESERVE;
482 
483 	start_pos = round_down(pos, fs_info->sectorsize);
484 	num_bytes = round_up(write_bytes + pos - start_pos,
485 			     fs_info->sectorsize);
486 	ASSERT(num_bytes <= U32_MAX);
487 
488 	end_of_last_block = start_pos + num_bytes - 1;
489 
490 	/*
491 	 * The pages may have already been dirty, clear out old accounting so
492 	 * we can set things up properly
493 	 */
494 	clear_extent_bit(&inode->io_tree, start_pos, end_of_last_block,
495 			 EXTENT_DELALLOC | EXTENT_DO_ACCOUNTING | EXTENT_DEFRAG,
496 			 0, 0, cached);
497 
498 	err = btrfs_set_extent_delalloc(inode, start_pos, end_of_last_block,
499 					extra_bits, cached);
500 	if (err)
501 		return err;
502 
503 	for (i = 0; i < num_pages; i++) {
504 		struct page *p = pages[i];
505 
506 		btrfs_page_clamp_set_uptodate(fs_info, p, start_pos, num_bytes);
507 		ClearPageChecked(p);
508 		btrfs_page_clamp_set_dirty(fs_info, p, start_pos, num_bytes);
509 	}
510 
511 	/*
512 	 * we've only changed i_size in ram, and we haven't updated
513 	 * the disk i_size.  There is no need to log the inode
514 	 * at this time.
515 	 */
516 	if (end_pos > isize)
517 		i_size_write(&inode->vfs_inode, end_pos);
518 	return 0;
519 }
520 
521 /*
522  * this drops all the extents in the cache that intersect the range
523  * [start, end].  Existing extents are split as required.
524  */
btrfs_drop_extent_cache(struct btrfs_inode * inode,u64 start,u64 end,int skip_pinned)525 void btrfs_drop_extent_cache(struct btrfs_inode *inode, u64 start, u64 end,
526 			     int skip_pinned)
527 {
528 	struct extent_map *em;
529 	struct extent_map *split = NULL;
530 	struct extent_map *split2 = NULL;
531 	struct extent_map_tree *em_tree = &inode->extent_tree;
532 	u64 len = end - start + 1;
533 	u64 gen;
534 	int ret;
535 	int testend = 1;
536 	unsigned long flags;
537 	int compressed = 0;
538 	bool modified;
539 
540 	WARN_ON(end < start);
541 	if (end == (u64)-1) {
542 		len = (u64)-1;
543 		testend = 0;
544 	}
545 	while (1) {
546 		int no_splits = 0;
547 
548 		modified = false;
549 		if (!split)
550 			split = alloc_extent_map();
551 		if (!split2)
552 			split2 = alloc_extent_map();
553 		if (!split || !split2)
554 			no_splits = 1;
555 
556 		write_lock(&em_tree->lock);
557 		em = lookup_extent_mapping(em_tree, start, len);
558 		if (!em) {
559 			write_unlock(&em_tree->lock);
560 			break;
561 		}
562 		flags = em->flags;
563 		gen = em->generation;
564 		if (skip_pinned && test_bit(EXTENT_FLAG_PINNED, &em->flags)) {
565 			if (testend && em->start + em->len >= start + len) {
566 				free_extent_map(em);
567 				write_unlock(&em_tree->lock);
568 				break;
569 			}
570 			start = em->start + em->len;
571 			if (testend)
572 				len = start + len - (em->start + em->len);
573 			free_extent_map(em);
574 			write_unlock(&em_tree->lock);
575 			continue;
576 		}
577 		compressed = test_bit(EXTENT_FLAG_COMPRESSED, &em->flags);
578 		clear_bit(EXTENT_FLAG_PINNED, &em->flags);
579 		clear_bit(EXTENT_FLAG_LOGGING, &flags);
580 		modified = !list_empty(&em->list);
581 		if (no_splits)
582 			goto next;
583 
584 		if (em->start < start) {
585 			split->start = em->start;
586 			split->len = start - em->start;
587 
588 			if (em->block_start < EXTENT_MAP_LAST_BYTE) {
589 				split->orig_start = em->orig_start;
590 				split->block_start = em->block_start;
591 
592 				if (compressed)
593 					split->block_len = em->block_len;
594 				else
595 					split->block_len = split->len;
596 				split->orig_block_len = max(split->block_len,
597 						em->orig_block_len);
598 				split->ram_bytes = em->ram_bytes;
599 			} else {
600 				split->orig_start = split->start;
601 				split->block_len = 0;
602 				split->block_start = em->block_start;
603 				split->orig_block_len = 0;
604 				split->ram_bytes = split->len;
605 			}
606 
607 			split->generation = gen;
608 			split->flags = flags;
609 			split->compress_type = em->compress_type;
610 			replace_extent_mapping(em_tree, em, split, modified);
611 			free_extent_map(split);
612 			split = split2;
613 			split2 = NULL;
614 		}
615 		if (testend && em->start + em->len > start + len) {
616 			u64 diff = start + len - em->start;
617 
618 			split->start = start + len;
619 			split->len = em->start + em->len - (start + len);
620 			split->flags = flags;
621 			split->compress_type = em->compress_type;
622 			split->generation = gen;
623 
624 			if (em->block_start < EXTENT_MAP_LAST_BYTE) {
625 				split->orig_block_len = max(em->block_len,
626 						    em->orig_block_len);
627 
628 				split->ram_bytes = em->ram_bytes;
629 				if (compressed) {
630 					split->block_len = em->block_len;
631 					split->block_start = em->block_start;
632 					split->orig_start = em->orig_start;
633 				} else {
634 					split->block_len = split->len;
635 					split->block_start = em->block_start
636 						+ diff;
637 					split->orig_start = em->orig_start;
638 				}
639 			} else {
640 				split->ram_bytes = split->len;
641 				split->orig_start = split->start;
642 				split->block_len = 0;
643 				split->block_start = em->block_start;
644 				split->orig_block_len = 0;
645 			}
646 
647 			if (extent_map_in_tree(em)) {
648 				replace_extent_mapping(em_tree, em, split,
649 						       modified);
650 			} else {
651 				ret = add_extent_mapping(em_tree, split,
652 							 modified);
653 				ASSERT(ret == 0); /* Logic error */
654 			}
655 			free_extent_map(split);
656 			split = NULL;
657 		}
658 next:
659 		if (extent_map_in_tree(em))
660 			remove_extent_mapping(em_tree, em);
661 		write_unlock(&em_tree->lock);
662 
663 		/* once for us */
664 		free_extent_map(em);
665 		/* once for the tree*/
666 		free_extent_map(em);
667 	}
668 	if (split)
669 		free_extent_map(split);
670 	if (split2)
671 		free_extent_map(split2);
672 }
673 
674 /*
675  * this is very complex, but the basic idea is to drop all extents
676  * in the range start - end.  hint_block is filled in with a block number
677  * that would be a good hint to the block allocator for this file.
678  *
679  * If an extent intersects the range but is not entirely inside the range
680  * it is either truncated or split.  Anything entirely inside the range
681  * is deleted from the tree.
682  *
683  * Note: the VFS' inode number of bytes is not updated, it's up to the caller
684  * to deal with that. We set the field 'bytes_found' of the arguments structure
685  * with the number of allocated bytes found in the target range, so that the
686  * caller can update the inode's number of bytes in an atomic way when
687  * replacing extents in a range to avoid races with stat(2).
688  */
btrfs_drop_extents(struct btrfs_trans_handle * trans,struct btrfs_root * root,struct btrfs_inode * inode,struct btrfs_drop_extents_args * args)689 int btrfs_drop_extents(struct btrfs_trans_handle *trans,
690 		       struct btrfs_root *root, struct btrfs_inode *inode,
691 		       struct btrfs_drop_extents_args *args)
692 {
693 	struct btrfs_fs_info *fs_info = root->fs_info;
694 	struct extent_buffer *leaf;
695 	struct btrfs_file_extent_item *fi;
696 	struct btrfs_ref ref = { 0 };
697 	struct btrfs_key key;
698 	struct btrfs_key new_key;
699 	u64 ino = btrfs_ino(inode);
700 	u64 search_start = args->start;
701 	u64 disk_bytenr = 0;
702 	u64 num_bytes = 0;
703 	u64 extent_offset = 0;
704 	u64 extent_end = 0;
705 	u64 last_end = args->start;
706 	int del_nr = 0;
707 	int del_slot = 0;
708 	int extent_type;
709 	int recow;
710 	int ret;
711 	int modify_tree = -1;
712 	int update_refs;
713 	int found = 0;
714 	int leafs_visited = 0;
715 	struct btrfs_path *path = args->path;
716 
717 	args->bytes_found = 0;
718 	args->extent_inserted = false;
719 
720 	/* Must always have a path if ->replace_extent is true */
721 	ASSERT(!(args->replace_extent && !args->path));
722 
723 	if (!path) {
724 		path = btrfs_alloc_path();
725 		if (!path) {
726 			ret = -ENOMEM;
727 			goto out;
728 		}
729 	}
730 
731 	if (args->drop_cache)
732 		btrfs_drop_extent_cache(inode, args->start, args->end - 1, 0);
733 
734 	if (args->start >= inode->disk_i_size && !args->replace_extent)
735 		modify_tree = 0;
736 
737 	update_refs = (root->root_key.objectid != BTRFS_TREE_LOG_OBJECTID);
738 	while (1) {
739 		recow = 0;
740 		ret = btrfs_lookup_file_extent(trans, root, path, ino,
741 					       search_start, modify_tree);
742 		if (ret < 0)
743 			break;
744 		if (ret > 0 && path->slots[0] > 0 && search_start == args->start) {
745 			leaf = path->nodes[0];
746 			btrfs_item_key_to_cpu(leaf, &key, path->slots[0] - 1);
747 			if (key.objectid == ino &&
748 			    key.type == BTRFS_EXTENT_DATA_KEY)
749 				path->slots[0]--;
750 		}
751 		ret = 0;
752 		leafs_visited++;
753 next_slot:
754 		leaf = path->nodes[0];
755 		if (path->slots[0] >= btrfs_header_nritems(leaf)) {
756 			BUG_ON(del_nr > 0);
757 			ret = btrfs_next_leaf(root, path);
758 			if (ret < 0)
759 				break;
760 			if (ret > 0) {
761 				ret = 0;
762 				break;
763 			}
764 			leafs_visited++;
765 			leaf = path->nodes[0];
766 			recow = 1;
767 		}
768 
769 		btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
770 
771 		if (key.objectid > ino)
772 			break;
773 		if (WARN_ON_ONCE(key.objectid < ino) ||
774 		    key.type < BTRFS_EXTENT_DATA_KEY) {
775 			ASSERT(del_nr == 0);
776 			path->slots[0]++;
777 			goto next_slot;
778 		}
779 		if (key.type > BTRFS_EXTENT_DATA_KEY || key.offset >= args->end)
780 			break;
781 
782 		fi = btrfs_item_ptr(leaf, path->slots[0],
783 				    struct btrfs_file_extent_item);
784 		extent_type = btrfs_file_extent_type(leaf, fi);
785 
786 		if (extent_type == BTRFS_FILE_EXTENT_REG ||
787 		    extent_type == BTRFS_FILE_EXTENT_PREALLOC) {
788 			disk_bytenr = btrfs_file_extent_disk_bytenr(leaf, fi);
789 			num_bytes = btrfs_file_extent_disk_num_bytes(leaf, fi);
790 			extent_offset = btrfs_file_extent_offset(leaf, fi);
791 			extent_end = key.offset +
792 				btrfs_file_extent_num_bytes(leaf, fi);
793 		} else if (extent_type == BTRFS_FILE_EXTENT_INLINE) {
794 			extent_end = key.offset +
795 				btrfs_file_extent_ram_bytes(leaf, fi);
796 		} else {
797 			/* can't happen */
798 			BUG();
799 		}
800 
801 		/*
802 		 * Don't skip extent items representing 0 byte lengths. They
803 		 * used to be created (bug) if while punching holes we hit
804 		 * -ENOSPC condition. So if we find one here, just ensure we
805 		 * delete it, otherwise we would insert a new file extent item
806 		 * with the same key (offset) as that 0 bytes length file
807 		 * extent item in the call to setup_items_for_insert() later
808 		 * in this function.
809 		 */
810 		if (extent_end == key.offset && extent_end >= search_start) {
811 			last_end = extent_end;
812 			goto delete_extent_item;
813 		}
814 
815 		if (extent_end <= search_start) {
816 			path->slots[0]++;
817 			goto next_slot;
818 		}
819 
820 		found = 1;
821 		search_start = max(key.offset, args->start);
822 		if (recow || !modify_tree) {
823 			modify_tree = -1;
824 			btrfs_release_path(path);
825 			continue;
826 		}
827 
828 		/*
829 		 *     | - range to drop - |
830 		 *  | -------- extent -------- |
831 		 */
832 		if (args->start > key.offset && args->end < extent_end) {
833 			BUG_ON(del_nr > 0);
834 			if (extent_type == BTRFS_FILE_EXTENT_INLINE) {
835 				ret = -EOPNOTSUPP;
836 				break;
837 			}
838 
839 			memcpy(&new_key, &key, sizeof(new_key));
840 			new_key.offset = args->start;
841 			ret = btrfs_duplicate_item(trans, root, path,
842 						   &new_key);
843 			if (ret == -EAGAIN) {
844 				btrfs_release_path(path);
845 				continue;
846 			}
847 			if (ret < 0)
848 				break;
849 
850 			leaf = path->nodes[0];
851 			fi = btrfs_item_ptr(leaf, path->slots[0] - 1,
852 					    struct btrfs_file_extent_item);
853 			btrfs_set_file_extent_num_bytes(leaf, fi,
854 							args->start - key.offset);
855 
856 			fi = btrfs_item_ptr(leaf, path->slots[0],
857 					    struct btrfs_file_extent_item);
858 
859 			extent_offset += args->start - key.offset;
860 			btrfs_set_file_extent_offset(leaf, fi, extent_offset);
861 			btrfs_set_file_extent_num_bytes(leaf, fi,
862 							extent_end - args->start);
863 			btrfs_mark_buffer_dirty(leaf);
864 
865 			if (update_refs && disk_bytenr > 0) {
866 				btrfs_init_generic_ref(&ref,
867 						BTRFS_ADD_DELAYED_REF,
868 						disk_bytenr, num_bytes, 0);
869 				btrfs_init_data_ref(&ref,
870 						root->root_key.objectid,
871 						new_key.objectid,
872 						args->start - extent_offset,
873 						0, false);
874 				ret = btrfs_inc_extent_ref(trans, &ref);
875 				if (ret) {
876 					btrfs_abort_transaction(trans, ret);
877 					break;
878 				}
879 			}
880 			key.offset = args->start;
881 		}
882 		/*
883 		 * From here on out we will have actually dropped something, so
884 		 * last_end can be updated.
885 		 */
886 		last_end = extent_end;
887 
888 		/*
889 		 *  | ---- range to drop ----- |
890 		 *      | -------- extent -------- |
891 		 */
892 		if (args->start <= key.offset && args->end < extent_end) {
893 			if (extent_type == BTRFS_FILE_EXTENT_INLINE) {
894 				ret = -EOPNOTSUPP;
895 				break;
896 			}
897 
898 			memcpy(&new_key, &key, sizeof(new_key));
899 			new_key.offset = args->end;
900 			btrfs_set_item_key_safe(fs_info, path, &new_key);
901 
902 			extent_offset += args->end - key.offset;
903 			btrfs_set_file_extent_offset(leaf, fi, extent_offset);
904 			btrfs_set_file_extent_num_bytes(leaf, fi,
905 							extent_end - args->end);
906 			btrfs_mark_buffer_dirty(leaf);
907 			if (update_refs && disk_bytenr > 0)
908 				args->bytes_found += args->end - key.offset;
909 			break;
910 		}
911 
912 		search_start = extent_end;
913 		/*
914 		 *       | ---- range to drop ----- |
915 		 *  | -------- extent -------- |
916 		 */
917 		if (args->start > key.offset && args->end >= extent_end) {
918 			BUG_ON(del_nr > 0);
919 			if (extent_type == BTRFS_FILE_EXTENT_INLINE) {
920 				ret = -EOPNOTSUPP;
921 				break;
922 			}
923 
924 			btrfs_set_file_extent_num_bytes(leaf, fi,
925 							args->start - key.offset);
926 			btrfs_mark_buffer_dirty(leaf);
927 			if (update_refs && disk_bytenr > 0)
928 				args->bytes_found += extent_end - args->start;
929 			if (args->end == extent_end)
930 				break;
931 
932 			path->slots[0]++;
933 			goto next_slot;
934 		}
935 
936 		/*
937 		 *  | ---- range to drop ----- |
938 		 *    | ------ extent ------ |
939 		 */
940 		if (args->start <= key.offset && args->end >= extent_end) {
941 delete_extent_item:
942 			if (del_nr == 0) {
943 				del_slot = path->slots[0];
944 				del_nr = 1;
945 			} else {
946 				BUG_ON(del_slot + del_nr != path->slots[0]);
947 				del_nr++;
948 			}
949 
950 			if (update_refs &&
951 			    extent_type == BTRFS_FILE_EXTENT_INLINE) {
952 				args->bytes_found += extent_end - key.offset;
953 				extent_end = ALIGN(extent_end,
954 						   fs_info->sectorsize);
955 			} else if (update_refs && disk_bytenr > 0) {
956 				btrfs_init_generic_ref(&ref,
957 						BTRFS_DROP_DELAYED_REF,
958 						disk_bytenr, num_bytes, 0);
959 				btrfs_init_data_ref(&ref,
960 						root->root_key.objectid,
961 						key.objectid,
962 						key.offset - extent_offset, 0,
963 						false);
964 				ret = btrfs_free_extent(trans, &ref);
965 				if (ret) {
966 					btrfs_abort_transaction(trans, ret);
967 					break;
968 				}
969 				args->bytes_found += extent_end - key.offset;
970 			}
971 
972 			if (args->end == extent_end)
973 				break;
974 
975 			if (path->slots[0] + 1 < btrfs_header_nritems(leaf)) {
976 				path->slots[0]++;
977 				goto next_slot;
978 			}
979 
980 			ret = btrfs_del_items(trans, root, path, del_slot,
981 					      del_nr);
982 			if (ret) {
983 				btrfs_abort_transaction(trans, ret);
984 				break;
985 			}
986 
987 			del_nr = 0;
988 			del_slot = 0;
989 
990 			btrfs_release_path(path);
991 			continue;
992 		}
993 
994 		BUG();
995 	}
996 
997 	if (!ret && del_nr > 0) {
998 		/*
999 		 * Set path->slots[0] to first slot, so that after the delete
1000 		 * if items are move off from our leaf to its immediate left or
1001 		 * right neighbor leafs, we end up with a correct and adjusted
1002 		 * path->slots[0] for our insertion (if args->replace_extent).
1003 		 */
1004 		path->slots[0] = del_slot;
1005 		ret = btrfs_del_items(trans, root, path, del_slot, del_nr);
1006 		if (ret)
1007 			btrfs_abort_transaction(trans, ret);
1008 	}
1009 
1010 	leaf = path->nodes[0];
1011 	/*
1012 	 * If btrfs_del_items() was called, it might have deleted a leaf, in
1013 	 * which case it unlocked our path, so check path->locks[0] matches a
1014 	 * write lock.
1015 	 */
1016 	if (!ret && args->replace_extent && leafs_visited == 1 &&
1017 	    path->locks[0] == BTRFS_WRITE_LOCK &&
1018 	    btrfs_leaf_free_space(leaf) >=
1019 	    sizeof(struct btrfs_item) + args->extent_item_size) {
1020 
1021 		key.objectid = ino;
1022 		key.type = BTRFS_EXTENT_DATA_KEY;
1023 		key.offset = args->start;
1024 		if (!del_nr && path->slots[0] < btrfs_header_nritems(leaf)) {
1025 			struct btrfs_key slot_key;
1026 
1027 			btrfs_item_key_to_cpu(leaf, &slot_key, path->slots[0]);
1028 			if (btrfs_comp_cpu_keys(&key, &slot_key) > 0)
1029 				path->slots[0]++;
1030 		}
1031 		setup_items_for_insert(root, path, &key,
1032 				       &args->extent_item_size, 1);
1033 		args->extent_inserted = true;
1034 	}
1035 
1036 	if (!args->path)
1037 		btrfs_free_path(path);
1038 	else if (!args->extent_inserted)
1039 		btrfs_release_path(path);
1040 out:
1041 	args->drop_end = found ? min(args->end, last_end) : args->end;
1042 
1043 	return ret;
1044 }
1045 
extent_mergeable(struct extent_buffer * leaf,int slot,u64 objectid,u64 bytenr,u64 orig_offset,u64 * start,u64 * end)1046 static int extent_mergeable(struct extent_buffer *leaf, int slot,
1047 			    u64 objectid, u64 bytenr, u64 orig_offset,
1048 			    u64 *start, u64 *end)
1049 {
1050 	struct btrfs_file_extent_item *fi;
1051 	struct btrfs_key key;
1052 	u64 extent_end;
1053 
1054 	if (slot < 0 || slot >= btrfs_header_nritems(leaf))
1055 		return 0;
1056 
1057 	btrfs_item_key_to_cpu(leaf, &key, slot);
1058 	if (key.objectid != objectid || key.type != BTRFS_EXTENT_DATA_KEY)
1059 		return 0;
1060 
1061 	fi = btrfs_item_ptr(leaf, slot, struct btrfs_file_extent_item);
1062 	if (btrfs_file_extent_type(leaf, fi) != BTRFS_FILE_EXTENT_REG ||
1063 	    btrfs_file_extent_disk_bytenr(leaf, fi) != bytenr ||
1064 	    btrfs_file_extent_offset(leaf, fi) != key.offset - orig_offset ||
1065 	    btrfs_file_extent_compression(leaf, fi) ||
1066 	    btrfs_file_extent_encryption(leaf, fi) ||
1067 	    btrfs_file_extent_other_encoding(leaf, fi))
1068 		return 0;
1069 
1070 	extent_end = key.offset + btrfs_file_extent_num_bytes(leaf, fi);
1071 	if ((*start && *start != key.offset) || (*end && *end != extent_end))
1072 		return 0;
1073 
1074 	*start = key.offset;
1075 	*end = extent_end;
1076 	return 1;
1077 }
1078 
1079 /*
1080  * Mark extent in the range start - end as written.
1081  *
1082  * This changes extent type from 'pre-allocated' to 'regular'. If only
1083  * part of extent is marked as written, the extent will be split into
1084  * two or three.
1085  */
btrfs_mark_extent_written(struct btrfs_trans_handle * trans,struct btrfs_inode * inode,u64 start,u64 end)1086 int btrfs_mark_extent_written(struct btrfs_trans_handle *trans,
1087 			      struct btrfs_inode *inode, u64 start, u64 end)
1088 {
1089 	struct btrfs_fs_info *fs_info = trans->fs_info;
1090 	struct btrfs_root *root = inode->root;
1091 	struct extent_buffer *leaf;
1092 	struct btrfs_path *path;
1093 	struct btrfs_file_extent_item *fi;
1094 	struct btrfs_ref ref = { 0 };
1095 	struct btrfs_key key;
1096 	struct btrfs_key new_key;
1097 	u64 bytenr;
1098 	u64 num_bytes;
1099 	u64 extent_end;
1100 	u64 orig_offset;
1101 	u64 other_start;
1102 	u64 other_end;
1103 	u64 split;
1104 	int del_nr = 0;
1105 	int del_slot = 0;
1106 	int recow;
1107 	int ret = 0;
1108 	u64 ino = btrfs_ino(inode);
1109 
1110 	path = btrfs_alloc_path();
1111 	if (!path)
1112 		return -ENOMEM;
1113 again:
1114 	recow = 0;
1115 	split = start;
1116 	key.objectid = ino;
1117 	key.type = BTRFS_EXTENT_DATA_KEY;
1118 	key.offset = split;
1119 
1120 	ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
1121 	if (ret < 0)
1122 		goto out;
1123 	if (ret > 0 && path->slots[0] > 0)
1124 		path->slots[0]--;
1125 
1126 	leaf = path->nodes[0];
1127 	btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
1128 	if (key.objectid != ino ||
1129 	    key.type != BTRFS_EXTENT_DATA_KEY) {
1130 		ret = -EINVAL;
1131 		btrfs_abort_transaction(trans, ret);
1132 		goto out;
1133 	}
1134 	fi = btrfs_item_ptr(leaf, path->slots[0],
1135 			    struct btrfs_file_extent_item);
1136 	if (btrfs_file_extent_type(leaf, fi) != BTRFS_FILE_EXTENT_PREALLOC) {
1137 		ret = -EINVAL;
1138 		btrfs_abort_transaction(trans, ret);
1139 		goto out;
1140 	}
1141 	extent_end = key.offset + btrfs_file_extent_num_bytes(leaf, fi);
1142 	if (key.offset > start || extent_end < end) {
1143 		ret = -EINVAL;
1144 		btrfs_abort_transaction(trans, ret);
1145 		goto out;
1146 	}
1147 
1148 	bytenr = btrfs_file_extent_disk_bytenr(leaf, fi);
1149 	num_bytes = btrfs_file_extent_disk_num_bytes(leaf, fi);
1150 	orig_offset = key.offset - btrfs_file_extent_offset(leaf, fi);
1151 	memcpy(&new_key, &key, sizeof(new_key));
1152 
1153 	if (start == key.offset && end < extent_end) {
1154 		other_start = 0;
1155 		other_end = start;
1156 		if (extent_mergeable(leaf, path->slots[0] - 1,
1157 				     ino, bytenr, orig_offset,
1158 				     &other_start, &other_end)) {
1159 			new_key.offset = end;
1160 			btrfs_set_item_key_safe(fs_info, path, &new_key);
1161 			fi = btrfs_item_ptr(leaf, path->slots[0],
1162 					    struct btrfs_file_extent_item);
1163 			btrfs_set_file_extent_generation(leaf, fi,
1164 							 trans->transid);
1165 			btrfs_set_file_extent_num_bytes(leaf, fi,
1166 							extent_end - end);
1167 			btrfs_set_file_extent_offset(leaf, fi,
1168 						     end - orig_offset);
1169 			fi = btrfs_item_ptr(leaf, path->slots[0] - 1,
1170 					    struct btrfs_file_extent_item);
1171 			btrfs_set_file_extent_generation(leaf, fi,
1172 							 trans->transid);
1173 			btrfs_set_file_extent_num_bytes(leaf, fi,
1174 							end - other_start);
1175 			btrfs_mark_buffer_dirty(leaf);
1176 			goto out;
1177 		}
1178 	}
1179 
1180 	if (start > key.offset && end == extent_end) {
1181 		other_start = end;
1182 		other_end = 0;
1183 		if (extent_mergeable(leaf, path->slots[0] + 1,
1184 				     ino, bytenr, orig_offset,
1185 				     &other_start, &other_end)) {
1186 			fi = btrfs_item_ptr(leaf, path->slots[0],
1187 					    struct btrfs_file_extent_item);
1188 			btrfs_set_file_extent_num_bytes(leaf, fi,
1189 							start - key.offset);
1190 			btrfs_set_file_extent_generation(leaf, fi,
1191 							 trans->transid);
1192 			path->slots[0]++;
1193 			new_key.offset = start;
1194 			btrfs_set_item_key_safe(fs_info, path, &new_key);
1195 
1196 			fi = btrfs_item_ptr(leaf, path->slots[0],
1197 					    struct btrfs_file_extent_item);
1198 			btrfs_set_file_extent_generation(leaf, fi,
1199 							 trans->transid);
1200 			btrfs_set_file_extent_num_bytes(leaf, fi,
1201 							other_end - start);
1202 			btrfs_set_file_extent_offset(leaf, fi,
1203 						     start - orig_offset);
1204 			btrfs_mark_buffer_dirty(leaf);
1205 			goto out;
1206 		}
1207 	}
1208 
1209 	while (start > key.offset || end < extent_end) {
1210 		if (key.offset == start)
1211 			split = end;
1212 
1213 		new_key.offset = split;
1214 		ret = btrfs_duplicate_item(trans, root, path, &new_key);
1215 		if (ret == -EAGAIN) {
1216 			btrfs_release_path(path);
1217 			goto again;
1218 		}
1219 		if (ret < 0) {
1220 			btrfs_abort_transaction(trans, ret);
1221 			goto out;
1222 		}
1223 
1224 		leaf = path->nodes[0];
1225 		fi = btrfs_item_ptr(leaf, path->slots[0] - 1,
1226 				    struct btrfs_file_extent_item);
1227 		btrfs_set_file_extent_generation(leaf, fi, trans->transid);
1228 		btrfs_set_file_extent_num_bytes(leaf, fi,
1229 						split - key.offset);
1230 
1231 		fi = btrfs_item_ptr(leaf, path->slots[0],
1232 				    struct btrfs_file_extent_item);
1233 
1234 		btrfs_set_file_extent_generation(leaf, fi, trans->transid);
1235 		btrfs_set_file_extent_offset(leaf, fi, split - orig_offset);
1236 		btrfs_set_file_extent_num_bytes(leaf, fi,
1237 						extent_end - split);
1238 		btrfs_mark_buffer_dirty(leaf);
1239 
1240 		btrfs_init_generic_ref(&ref, BTRFS_ADD_DELAYED_REF, bytenr,
1241 				       num_bytes, 0);
1242 		btrfs_init_data_ref(&ref, root->root_key.objectid, ino,
1243 				    orig_offset, 0, false);
1244 		ret = btrfs_inc_extent_ref(trans, &ref);
1245 		if (ret) {
1246 			btrfs_abort_transaction(trans, ret);
1247 			goto out;
1248 		}
1249 
1250 		if (split == start) {
1251 			key.offset = start;
1252 		} else {
1253 			if (start != key.offset) {
1254 				ret = -EINVAL;
1255 				btrfs_abort_transaction(trans, ret);
1256 				goto out;
1257 			}
1258 			path->slots[0]--;
1259 			extent_end = end;
1260 		}
1261 		recow = 1;
1262 	}
1263 
1264 	other_start = end;
1265 	other_end = 0;
1266 	btrfs_init_generic_ref(&ref, BTRFS_DROP_DELAYED_REF, bytenr,
1267 			       num_bytes, 0);
1268 	btrfs_init_data_ref(&ref, root->root_key.objectid, ino, orig_offset,
1269 			    0, false);
1270 	if (extent_mergeable(leaf, path->slots[0] + 1,
1271 			     ino, bytenr, orig_offset,
1272 			     &other_start, &other_end)) {
1273 		if (recow) {
1274 			btrfs_release_path(path);
1275 			goto again;
1276 		}
1277 		extent_end = other_end;
1278 		del_slot = path->slots[0] + 1;
1279 		del_nr++;
1280 		ret = btrfs_free_extent(trans, &ref);
1281 		if (ret) {
1282 			btrfs_abort_transaction(trans, ret);
1283 			goto out;
1284 		}
1285 	}
1286 	other_start = 0;
1287 	other_end = start;
1288 	if (extent_mergeable(leaf, path->slots[0] - 1,
1289 			     ino, bytenr, orig_offset,
1290 			     &other_start, &other_end)) {
1291 		if (recow) {
1292 			btrfs_release_path(path);
1293 			goto again;
1294 		}
1295 		key.offset = other_start;
1296 		del_slot = path->slots[0];
1297 		del_nr++;
1298 		ret = btrfs_free_extent(trans, &ref);
1299 		if (ret) {
1300 			btrfs_abort_transaction(trans, ret);
1301 			goto out;
1302 		}
1303 	}
1304 	if (del_nr == 0) {
1305 		fi = btrfs_item_ptr(leaf, path->slots[0],
1306 			   struct btrfs_file_extent_item);
1307 		btrfs_set_file_extent_type(leaf, fi,
1308 					   BTRFS_FILE_EXTENT_REG);
1309 		btrfs_set_file_extent_generation(leaf, fi, trans->transid);
1310 		btrfs_mark_buffer_dirty(leaf);
1311 	} else {
1312 		fi = btrfs_item_ptr(leaf, del_slot - 1,
1313 			   struct btrfs_file_extent_item);
1314 		btrfs_set_file_extent_type(leaf, fi,
1315 					   BTRFS_FILE_EXTENT_REG);
1316 		btrfs_set_file_extent_generation(leaf, fi, trans->transid);
1317 		btrfs_set_file_extent_num_bytes(leaf, fi,
1318 						extent_end - key.offset);
1319 		btrfs_mark_buffer_dirty(leaf);
1320 
1321 		ret = btrfs_del_items(trans, root, path, del_slot, del_nr);
1322 		if (ret < 0) {
1323 			btrfs_abort_transaction(trans, ret);
1324 			goto out;
1325 		}
1326 	}
1327 out:
1328 	btrfs_free_path(path);
1329 	return ret;
1330 }
1331 
1332 /*
1333  * on error we return an unlocked page and the error value
1334  * on success we return a locked page and 0
1335  */
prepare_uptodate_page(struct inode * inode,struct page * page,u64 pos,bool force_uptodate)1336 static int prepare_uptodate_page(struct inode *inode,
1337 				 struct page *page, u64 pos,
1338 				 bool force_uptodate)
1339 {
1340 	int ret = 0;
1341 
1342 	if (((pos & (PAGE_SIZE - 1)) || force_uptodate) &&
1343 	    !PageUptodate(page)) {
1344 		ret = btrfs_readpage(NULL, page);
1345 		if (ret)
1346 			return ret;
1347 		lock_page(page);
1348 		if (!PageUptodate(page)) {
1349 			unlock_page(page);
1350 			return -EIO;
1351 		}
1352 
1353 		/*
1354 		 * Since btrfs_readpage() will unlock the page before it
1355 		 * returns, there is a window where btrfs_releasepage() can be
1356 		 * called to release the page.  Here we check both inode
1357 		 * mapping and PagePrivate() to make sure the page was not
1358 		 * released.
1359 		 *
1360 		 * The private flag check is essential for subpage as we need
1361 		 * to store extra bitmap using page->private.
1362 		 */
1363 		if (page->mapping != inode->i_mapping || !PagePrivate(page)) {
1364 			unlock_page(page);
1365 			return -EAGAIN;
1366 		}
1367 	}
1368 	return 0;
1369 }
1370 
1371 /*
1372  * this just gets pages into the page cache and locks them down.
1373  */
prepare_pages(struct inode * inode,struct page ** pages,size_t num_pages,loff_t pos,size_t write_bytes,bool force_uptodate)1374 static noinline int prepare_pages(struct inode *inode, struct page **pages,
1375 				  size_t num_pages, loff_t pos,
1376 				  size_t write_bytes, bool force_uptodate)
1377 {
1378 	int i;
1379 	unsigned long index = pos >> PAGE_SHIFT;
1380 	gfp_t mask = btrfs_alloc_write_mask(inode->i_mapping);
1381 	int err = 0;
1382 	int faili;
1383 
1384 	for (i = 0; i < num_pages; i++) {
1385 again:
1386 		pages[i] = find_or_create_page(inode->i_mapping, index + i,
1387 					       mask | __GFP_WRITE);
1388 		if (!pages[i]) {
1389 			faili = i - 1;
1390 			err = -ENOMEM;
1391 			goto fail;
1392 		}
1393 
1394 		err = set_page_extent_mapped(pages[i]);
1395 		if (err < 0) {
1396 			faili = i;
1397 			goto fail;
1398 		}
1399 
1400 		if (i == 0)
1401 			err = prepare_uptodate_page(inode, pages[i], pos,
1402 						    force_uptodate);
1403 		if (!err && i == num_pages - 1)
1404 			err = prepare_uptodate_page(inode, pages[i],
1405 						    pos + write_bytes, false);
1406 		if (err) {
1407 			put_page(pages[i]);
1408 			if (err == -EAGAIN) {
1409 				err = 0;
1410 				goto again;
1411 			}
1412 			faili = i - 1;
1413 			goto fail;
1414 		}
1415 		wait_on_page_writeback(pages[i]);
1416 	}
1417 
1418 	return 0;
1419 fail:
1420 	while (faili >= 0) {
1421 		unlock_page(pages[faili]);
1422 		put_page(pages[faili]);
1423 		faili--;
1424 	}
1425 	return err;
1426 
1427 }
1428 
1429 /*
1430  * This function locks the extent and properly waits for data=ordered extents
1431  * to finish before allowing the pages to be modified if need.
1432  *
1433  * The return value:
1434  * 1 - the extent is locked
1435  * 0 - the extent is not locked, and everything is OK
1436  * -EAGAIN - need re-prepare the pages
1437  * the other < 0 number - Something wrong happens
1438  */
1439 static noinline int
lock_and_cleanup_extent_if_need(struct btrfs_inode * inode,struct page ** pages,size_t num_pages,loff_t pos,size_t write_bytes,u64 * lockstart,u64 * lockend,struct extent_state ** cached_state)1440 lock_and_cleanup_extent_if_need(struct btrfs_inode *inode, struct page **pages,
1441 				size_t num_pages, loff_t pos,
1442 				size_t write_bytes,
1443 				u64 *lockstart, u64 *lockend,
1444 				struct extent_state **cached_state)
1445 {
1446 	struct btrfs_fs_info *fs_info = inode->root->fs_info;
1447 	u64 start_pos;
1448 	u64 last_pos;
1449 	int i;
1450 	int ret = 0;
1451 
1452 	start_pos = round_down(pos, fs_info->sectorsize);
1453 	last_pos = round_up(pos + write_bytes, fs_info->sectorsize) - 1;
1454 
1455 	if (start_pos < inode->vfs_inode.i_size) {
1456 		struct btrfs_ordered_extent *ordered;
1457 
1458 		lock_extent_bits(&inode->io_tree, start_pos, last_pos,
1459 				cached_state);
1460 		ordered = btrfs_lookup_ordered_range(inode, start_pos,
1461 						     last_pos - start_pos + 1);
1462 		if (ordered &&
1463 		    ordered->file_offset + ordered->num_bytes > start_pos &&
1464 		    ordered->file_offset <= last_pos) {
1465 			unlock_extent_cached(&inode->io_tree, start_pos,
1466 					last_pos, cached_state);
1467 			for (i = 0; i < num_pages; i++) {
1468 				unlock_page(pages[i]);
1469 				put_page(pages[i]);
1470 			}
1471 			btrfs_start_ordered_extent(ordered, 1);
1472 			btrfs_put_ordered_extent(ordered);
1473 			return -EAGAIN;
1474 		}
1475 		if (ordered)
1476 			btrfs_put_ordered_extent(ordered);
1477 
1478 		*lockstart = start_pos;
1479 		*lockend = last_pos;
1480 		ret = 1;
1481 	}
1482 
1483 	/*
1484 	 * We should be called after prepare_pages() which should have locked
1485 	 * all pages in the range.
1486 	 */
1487 	for (i = 0; i < num_pages; i++)
1488 		WARN_ON(!PageLocked(pages[i]));
1489 
1490 	return ret;
1491 }
1492 
check_can_nocow(struct btrfs_inode * inode,loff_t pos,size_t * write_bytes,bool nowait)1493 static int check_can_nocow(struct btrfs_inode *inode, loff_t pos,
1494 			   size_t *write_bytes, bool nowait)
1495 {
1496 	struct btrfs_fs_info *fs_info = inode->root->fs_info;
1497 	struct btrfs_root *root = inode->root;
1498 	u64 lockstart, lockend;
1499 	u64 num_bytes;
1500 	int ret;
1501 
1502 	if (!(inode->flags & (BTRFS_INODE_NODATACOW | BTRFS_INODE_PREALLOC)))
1503 		return 0;
1504 
1505 	if (!nowait && !btrfs_drew_try_write_lock(&root->snapshot_lock))
1506 		return -EAGAIN;
1507 
1508 	lockstart = round_down(pos, fs_info->sectorsize);
1509 	lockend = round_up(pos + *write_bytes,
1510 			   fs_info->sectorsize) - 1;
1511 	num_bytes = lockend - lockstart + 1;
1512 
1513 	if (nowait) {
1514 		struct btrfs_ordered_extent *ordered;
1515 
1516 		if (!try_lock_extent(&inode->io_tree, lockstart, lockend))
1517 			return -EAGAIN;
1518 
1519 		ordered = btrfs_lookup_ordered_range(inode, lockstart,
1520 						     num_bytes);
1521 		if (ordered) {
1522 			btrfs_put_ordered_extent(ordered);
1523 			ret = -EAGAIN;
1524 			goto out_unlock;
1525 		}
1526 	} else {
1527 		btrfs_lock_and_flush_ordered_range(inode, lockstart,
1528 						   lockend, NULL);
1529 	}
1530 
1531 	ret = can_nocow_extent(&inode->vfs_inode, lockstart, &num_bytes,
1532 			NULL, NULL, NULL, false);
1533 	if (ret <= 0) {
1534 		ret = 0;
1535 		if (!nowait)
1536 			btrfs_drew_write_unlock(&root->snapshot_lock);
1537 	} else {
1538 		*write_bytes = min_t(size_t, *write_bytes ,
1539 				     num_bytes - pos + lockstart);
1540 	}
1541 out_unlock:
1542 	unlock_extent(&inode->io_tree, lockstart, lockend);
1543 
1544 	return ret;
1545 }
1546 
check_nocow_nolock(struct btrfs_inode * inode,loff_t pos,size_t * write_bytes)1547 static int check_nocow_nolock(struct btrfs_inode *inode, loff_t pos,
1548 			      size_t *write_bytes)
1549 {
1550 	return check_can_nocow(inode, pos, write_bytes, true);
1551 }
1552 
1553 /*
1554  * Check if we can do nocow write into the range [@pos, @pos + @write_bytes)
1555  *
1556  * @pos:	 File offset
1557  * @write_bytes: The length to write, will be updated to the nocow writeable
1558  *		 range
1559  *
1560  * This function will flush ordered extents in the range to ensure proper
1561  * nocow checks.
1562  *
1563  * Return:
1564  * >0		and update @write_bytes if we can do nocow write
1565  *  0		if we can't do nocow write
1566  * -EAGAIN	if we can't get the needed lock or there are ordered extents
1567  * 		for * (nowait == true) case
1568  * <0		if other error happened
1569  *
1570  * NOTE: Callers need to release the lock by btrfs_check_nocow_unlock().
1571  */
btrfs_check_nocow_lock(struct btrfs_inode * inode,loff_t pos,size_t * write_bytes)1572 int btrfs_check_nocow_lock(struct btrfs_inode *inode, loff_t pos,
1573 			   size_t *write_bytes)
1574 {
1575 	return check_can_nocow(inode, pos, write_bytes, false);
1576 }
1577 
btrfs_check_nocow_unlock(struct btrfs_inode * inode)1578 void btrfs_check_nocow_unlock(struct btrfs_inode *inode)
1579 {
1580 	btrfs_drew_write_unlock(&inode->root->snapshot_lock);
1581 }
1582 
update_time_for_write(struct inode * inode)1583 static void update_time_for_write(struct inode *inode)
1584 {
1585 	struct timespec64 now;
1586 
1587 	if (IS_NOCMTIME(inode))
1588 		return;
1589 
1590 	now = current_time(inode);
1591 	if (!timespec64_equal(&inode->i_mtime, &now))
1592 		inode->i_mtime = now;
1593 
1594 	if (!timespec64_equal(&inode->i_ctime, &now))
1595 		inode->i_ctime = now;
1596 
1597 	if (IS_I_VERSION(inode))
1598 		inode_inc_iversion(inode);
1599 }
1600 
btrfs_write_check(struct kiocb * iocb,struct iov_iter * from,size_t count)1601 static int btrfs_write_check(struct kiocb *iocb, struct iov_iter *from,
1602 			     size_t count)
1603 {
1604 	struct file *file = iocb->ki_filp;
1605 	struct inode *inode = file_inode(file);
1606 	struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
1607 	loff_t pos = iocb->ki_pos;
1608 	int ret;
1609 	loff_t oldsize;
1610 	loff_t start_pos;
1611 
1612 	if (iocb->ki_flags & IOCB_NOWAIT) {
1613 		size_t nocow_bytes = count;
1614 
1615 		/* We will allocate space in case nodatacow is not set, so bail */
1616 		if (check_nocow_nolock(BTRFS_I(inode), pos, &nocow_bytes) <= 0)
1617 			return -EAGAIN;
1618 		/*
1619 		 * There are holes in the range or parts of the range that must
1620 		 * be COWed (shared extents, RO block groups, etc), so just bail
1621 		 * out.
1622 		 */
1623 		if (nocow_bytes < count)
1624 			return -EAGAIN;
1625 	}
1626 
1627 	current->backing_dev_info = inode_to_bdi(inode);
1628 	ret = file_remove_privs(file);
1629 	if (ret)
1630 		return ret;
1631 
1632 	/*
1633 	 * We reserve space for updating the inode when we reserve space for the
1634 	 * extent we are going to write, so we will enospc out there.  We don't
1635 	 * need to start yet another transaction to update the inode as we will
1636 	 * update the inode when we finish writing whatever data we write.
1637 	 */
1638 	update_time_for_write(inode);
1639 
1640 	start_pos = round_down(pos, fs_info->sectorsize);
1641 	oldsize = i_size_read(inode);
1642 	if (start_pos > oldsize) {
1643 		/* Expand hole size to cover write data, preventing empty gap */
1644 		loff_t end_pos = round_up(pos + count, fs_info->sectorsize);
1645 
1646 		ret = btrfs_cont_expand(BTRFS_I(inode), oldsize, end_pos);
1647 		if (ret) {
1648 			current->backing_dev_info = NULL;
1649 			return ret;
1650 		}
1651 	}
1652 
1653 	return 0;
1654 }
1655 
btrfs_buffered_write(struct kiocb * iocb,struct iov_iter * i)1656 static noinline ssize_t btrfs_buffered_write(struct kiocb *iocb,
1657 					       struct iov_iter *i)
1658 {
1659 	struct file *file = iocb->ki_filp;
1660 	loff_t pos;
1661 	struct inode *inode = file_inode(file);
1662 	struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
1663 	struct page **pages = NULL;
1664 	struct extent_changeset *data_reserved = NULL;
1665 	u64 release_bytes = 0;
1666 	u64 lockstart;
1667 	u64 lockend;
1668 	size_t num_written = 0;
1669 	int nrptrs;
1670 	ssize_t ret;
1671 	bool only_release_metadata = false;
1672 	bool force_page_uptodate = false;
1673 	loff_t old_isize = i_size_read(inode);
1674 	unsigned int ilock_flags = 0;
1675 
1676 	if (iocb->ki_flags & IOCB_NOWAIT)
1677 		ilock_flags |= BTRFS_ILOCK_TRY;
1678 
1679 	ret = btrfs_inode_lock(inode, ilock_flags);
1680 	if (ret < 0)
1681 		return ret;
1682 
1683 	ret = generic_write_checks(iocb, i);
1684 	if (ret <= 0)
1685 		goto out;
1686 
1687 	ret = btrfs_write_check(iocb, i, ret);
1688 	if (ret < 0)
1689 		goto out;
1690 
1691 	pos = iocb->ki_pos;
1692 	nrptrs = min(DIV_ROUND_UP(iov_iter_count(i), PAGE_SIZE),
1693 			PAGE_SIZE / (sizeof(struct page *)));
1694 	nrptrs = min(nrptrs, current->nr_dirtied_pause - current->nr_dirtied);
1695 	nrptrs = max(nrptrs, 8);
1696 	pages = kmalloc_array(nrptrs, sizeof(struct page *), GFP_KERNEL);
1697 	if (!pages) {
1698 		ret = -ENOMEM;
1699 		goto out;
1700 	}
1701 
1702 	while (iov_iter_count(i) > 0) {
1703 		struct extent_state *cached_state = NULL;
1704 		size_t offset = offset_in_page(pos);
1705 		size_t sector_offset;
1706 		size_t write_bytes = min(iov_iter_count(i),
1707 					 nrptrs * (size_t)PAGE_SIZE -
1708 					 offset);
1709 		size_t num_pages;
1710 		size_t reserve_bytes;
1711 		size_t dirty_pages;
1712 		size_t copied;
1713 		size_t dirty_sectors;
1714 		size_t num_sectors;
1715 		int extents_locked;
1716 
1717 		/*
1718 		 * Fault pages before locking them in prepare_pages
1719 		 * to avoid recursive lock
1720 		 */
1721 		if (unlikely(fault_in_iov_iter_readable(i, write_bytes))) {
1722 			ret = -EFAULT;
1723 			break;
1724 		}
1725 
1726 		only_release_metadata = false;
1727 		sector_offset = pos & (fs_info->sectorsize - 1);
1728 
1729 		extent_changeset_release(data_reserved);
1730 		ret = btrfs_check_data_free_space(BTRFS_I(inode),
1731 						  &data_reserved, pos,
1732 						  write_bytes);
1733 		if (ret < 0) {
1734 			/*
1735 			 * If we don't have to COW at the offset, reserve
1736 			 * metadata only. write_bytes may get smaller than
1737 			 * requested here.
1738 			 */
1739 			if (btrfs_check_nocow_lock(BTRFS_I(inode), pos,
1740 						   &write_bytes) > 0)
1741 				only_release_metadata = true;
1742 			else
1743 				break;
1744 		}
1745 
1746 		num_pages = DIV_ROUND_UP(write_bytes + offset, PAGE_SIZE);
1747 		WARN_ON(num_pages > nrptrs);
1748 		reserve_bytes = round_up(write_bytes + sector_offset,
1749 					 fs_info->sectorsize);
1750 		WARN_ON(reserve_bytes == 0);
1751 		ret = btrfs_delalloc_reserve_metadata(BTRFS_I(inode),
1752 				reserve_bytes);
1753 		if (ret) {
1754 			if (!only_release_metadata)
1755 				btrfs_free_reserved_data_space(BTRFS_I(inode),
1756 						data_reserved, pos,
1757 						write_bytes);
1758 			else
1759 				btrfs_check_nocow_unlock(BTRFS_I(inode));
1760 			break;
1761 		}
1762 
1763 		release_bytes = reserve_bytes;
1764 again:
1765 		/*
1766 		 * This is going to setup the pages array with the number of
1767 		 * pages we want, so we don't really need to worry about the
1768 		 * contents of pages from loop to loop
1769 		 */
1770 		ret = prepare_pages(inode, pages, num_pages,
1771 				    pos, write_bytes,
1772 				    force_page_uptodate);
1773 		if (ret) {
1774 			btrfs_delalloc_release_extents(BTRFS_I(inode),
1775 						       reserve_bytes);
1776 			break;
1777 		}
1778 
1779 		extents_locked = lock_and_cleanup_extent_if_need(
1780 				BTRFS_I(inode), pages,
1781 				num_pages, pos, write_bytes, &lockstart,
1782 				&lockend, &cached_state);
1783 		if (extents_locked < 0) {
1784 			if (extents_locked == -EAGAIN)
1785 				goto again;
1786 			btrfs_delalloc_release_extents(BTRFS_I(inode),
1787 						       reserve_bytes);
1788 			ret = extents_locked;
1789 			break;
1790 		}
1791 
1792 		copied = btrfs_copy_from_user(pos, write_bytes, pages, i);
1793 
1794 		num_sectors = BTRFS_BYTES_TO_BLKS(fs_info, reserve_bytes);
1795 		dirty_sectors = round_up(copied + sector_offset,
1796 					fs_info->sectorsize);
1797 		dirty_sectors = BTRFS_BYTES_TO_BLKS(fs_info, dirty_sectors);
1798 
1799 		/*
1800 		 * if we have trouble faulting in the pages, fall
1801 		 * back to one page at a time
1802 		 */
1803 		if (copied < write_bytes)
1804 			nrptrs = 1;
1805 
1806 		if (copied == 0) {
1807 			force_page_uptodate = true;
1808 			dirty_sectors = 0;
1809 			dirty_pages = 0;
1810 		} else {
1811 			force_page_uptodate = false;
1812 			dirty_pages = DIV_ROUND_UP(copied + offset,
1813 						   PAGE_SIZE);
1814 		}
1815 
1816 		if (num_sectors > dirty_sectors) {
1817 			/* release everything except the sectors we dirtied */
1818 			release_bytes -= dirty_sectors << fs_info->sectorsize_bits;
1819 			if (only_release_metadata) {
1820 				btrfs_delalloc_release_metadata(BTRFS_I(inode),
1821 							release_bytes, true);
1822 			} else {
1823 				u64 __pos;
1824 
1825 				__pos = round_down(pos,
1826 						   fs_info->sectorsize) +
1827 					(dirty_pages << PAGE_SHIFT);
1828 				btrfs_delalloc_release_space(BTRFS_I(inode),
1829 						data_reserved, __pos,
1830 						release_bytes, true);
1831 			}
1832 		}
1833 
1834 		release_bytes = round_up(copied + sector_offset,
1835 					fs_info->sectorsize);
1836 
1837 		ret = btrfs_dirty_pages(BTRFS_I(inode), pages,
1838 					dirty_pages, pos, copied,
1839 					&cached_state, only_release_metadata);
1840 
1841 		/*
1842 		 * If we have not locked the extent range, because the range's
1843 		 * start offset is >= i_size, we might still have a non-NULL
1844 		 * cached extent state, acquired while marking the extent range
1845 		 * as delalloc through btrfs_dirty_pages(). Therefore free any
1846 		 * possible cached extent state to avoid a memory leak.
1847 		 */
1848 		if (extents_locked)
1849 			unlock_extent_cached(&BTRFS_I(inode)->io_tree,
1850 					     lockstart, lockend, &cached_state);
1851 		else
1852 			free_extent_state(cached_state);
1853 
1854 		btrfs_delalloc_release_extents(BTRFS_I(inode), reserve_bytes);
1855 		if (ret) {
1856 			btrfs_drop_pages(pages, num_pages);
1857 			break;
1858 		}
1859 
1860 		release_bytes = 0;
1861 		if (only_release_metadata)
1862 			btrfs_check_nocow_unlock(BTRFS_I(inode));
1863 
1864 		btrfs_drop_pages(pages, num_pages);
1865 
1866 		cond_resched();
1867 
1868 		balance_dirty_pages_ratelimited(inode->i_mapping);
1869 
1870 		pos += copied;
1871 		num_written += copied;
1872 	}
1873 
1874 	kfree(pages);
1875 
1876 	if (release_bytes) {
1877 		if (only_release_metadata) {
1878 			btrfs_check_nocow_unlock(BTRFS_I(inode));
1879 			btrfs_delalloc_release_metadata(BTRFS_I(inode),
1880 					release_bytes, true);
1881 		} else {
1882 			btrfs_delalloc_release_space(BTRFS_I(inode),
1883 					data_reserved,
1884 					round_down(pos, fs_info->sectorsize),
1885 					release_bytes, true);
1886 		}
1887 	}
1888 
1889 	extent_changeset_free(data_reserved);
1890 	if (num_written > 0) {
1891 		pagecache_isize_extended(inode, old_isize, iocb->ki_pos);
1892 		iocb->ki_pos += num_written;
1893 	}
1894 out:
1895 	btrfs_inode_unlock(inode, ilock_flags);
1896 	return num_written ? num_written : ret;
1897 }
1898 
check_direct_IO(struct btrfs_fs_info * fs_info,const struct iov_iter * iter,loff_t offset)1899 static ssize_t check_direct_IO(struct btrfs_fs_info *fs_info,
1900 			       const struct iov_iter *iter, loff_t offset)
1901 {
1902 	const u32 blocksize_mask = fs_info->sectorsize - 1;
1903 
1904 	if (offset & blocksize_mask)
1905 		return -EINVAL;
1906 
1907 	if (iov_iter_alignment(iter) & blocksize_mask)
1908 		return -EINVAL;
1909 
1910 	return 0;
1911 }
1912 
btrfs_direct_write(struct kiocb * iocb,struct iov_iter * from)1913 static ssize_t btrfs_direct_write(struct kiocb *iocb, struct iov_iter *from)
1914 {
1915 	struct file *file = iocb->ki_filp;
1916 	struct inode *inode = file_inode(file);
1917 	struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
1918 	loff_t pos;
1919 	ssize_t written = 0;
1920 	ssize_t written_buffered;
1921 	size_t prev_left = 0;
1922 	loff_t endbyte;
1923 	ssize_t err;
1924 	unsigned int ilock_flags = 0;
1925 	struct iomap_dio *dio;
1926 
1927 	if (iocb->ki_flags & IOCB_NOWAIT)
1928 		ilock_flags |= BTRFS_ILOCK_TRY;
1929 
1930 	/* If the write DIO is within EOF, use a shared lock */
1931 	if (iocb->ki_pos + iov_iter_count(from) <= i_size_read(inode))
1932 		ilock_flags |= BTRFS_ILOCK_SHARED;
1933 
1934 relock:
1935 	err = btrfs_inode_lock(inode, ilock_flags);
1936 	if (err < 0)
1937 		return err;
1938 
1939 	err = generic_write_checks(iocb, from);
1940 	if (err <= 0) {
1941 		btrfs_inode_unlock(inode, ilock_flags);
1942 		return err;
1943 	}
1944 
1945 	err = btrfs_write_check(iocb, from, err);
1946 	if (err < 0) {
1947 		btrfs_inode_unlock(inode, ilock_flags);
1948 		goto out;
1949 	}
1950 
1951 	pos = iocb->ki_pos;
1952 	/*
1953 	 * Re-check since file size may have changed just before taking the
1954 	 * lock or pos may have changed because of O_APPEND in generic_write_check()
1955 	 */
1956 	if ((ilock_flags & BTRFS_ILOCK_SHARED) &&
1957 	    pos + iov_iter_count(from) > i_size_read(inode)) {
1958 		btrfs_inode_unlock(inode, ilock_flags);
1959 		ilock_flags &= ~BTRFS_ILOCK_SHARED;
1960 		goto relock;
1961 	}
1962 
1963 	if (check_direct_IO(fs_info, from, pos)) {
1964 		btrfs_inode_unlock(inode, ilock_flags);
1965 		goto buffered;
1966 	}
1967 
1968 	/*
1969 	 * The iov_iter can be mapped to the same file range we are writing to.
1970 	 * If that's the case, then we will deadlock in the iomap code, because
1971 	 * it first calls our callback btrfs_dio_iomap_begin(), which will create
1972 	 * an ordered extent, and after that it will fault in the pages that the
1973 	 * iov_iter refers to. During the fault in we end up in the readahead
1974 	 * pages code (starting at btrfs_readahead()), which will lock the range,
1975 	 * find that ordered extent and then wait for it to complete (at
1976 	 * btrfs_lock_and_flush_ordered_range()), resulting in a deadlock since
1977 	 * obviously the ordered extent can never complete as we didn't submit
1978 	 * yet the respective bio(s). This always happens when the buffer is
1979 	 * memory mapped to the same file range, since the iomap DIO code always
1980 	 * invalidates pages in the target file range (after starting and waiting
1981 	 * for any writeback).
1982 	 *
1983 	 * So here we disable page faults in the iov_iter and then retry if we
1984 	 * got -EFAULT, faulting in the pages before the retry.
1985 	 */
1986 	from->nofault = true;
1987 	dio = __iomap_dio_rw(iocb, from, &btrfs_dio_iomap_ops, &btrfs_dio_ops,
1988 			     IOMAP_DIO_PARTIAL, written);
1989 	from->nofault = false;
1990 
1991 	/*
1992 	 * iomap_dio_complete() will call btrfs_sync_file() if we have a dsync
1993 	 * iocb, and that needs to lock the inode. So unlock it before calling
1994 	 * iomap_dio_complete() to avoid a deadlock.
1995 	 */
1996 	btrfs_inode_unlock(inode, ilock_flags);
1997 
1998 	if (IS_ERR_OR_NULL(dio))
1999 		err = PTR_ERR_OR_ZERO(dio);
2000 	else
2001 		err = iomap_dio_complete(dio);
2002 
2003 	/* No increment (+=) because iomap returns a cumulative value. */
2004 	if (err > 0)
2005 		written = err;
2006 
2007 	if (iov_iter_count(from) > 0 && (err == -EFAULT || err > 0)) {
2008 		const size_t left = iov_iter_count(from);
2009 		/*
2010 		 * We have more data left to write. Try to fault in as many as
2011 		 * possible of the remainder pages and retry. We do this without
2012 		 * releasing and locking again the inode, to prevent races with
2013 		 * truncate.
2014 		 *
2015 		 * Also, in case the iov refers to pages in the file range of the
2016 		 * file we want to write to (due to a mmap), we could enter an
2017 		 * infinite loop if we retry after faulting the pages in, since
2018 		 * iomap will invalidate any pages in the range early on, before
2019 		 * it tries to fault in the pages of the iov. So we keep track of
2020 		 * how much was left of iov in the previous EFAULT and fallback
2021 		 * to buffered IO in case we haven't made any progress.
2022 		 */
2023 		if (left == prev_left) {
2024 			err = -ENOTBLK;
2025 		} else {
2026 			fault_in_iov_iter_readable(from, left);
2027 			prev_left = left;
2028 			goto relock;
2029 		}
2030 	}
2031 
2032 	/* If 'err' is -ENOTBLK then it means we must fallback to buffered IO. */
2033 	if ((err < 0 && err != -ENOTBLK) || !iov_iter_count(from))
2034 		goto out;
2035 
2036 buffered:
2037 	pos = iocb->ki_pos;
2038 	written_buffered = btrfs_buffered_write(iocb, from);
2039 	if (written_buffered < 0) {
2040 		err = written_buffered;
2041 		goto out;
2042 	}
2043 	/*
2044 	 * Ensure all data is persisted. We want the next direct IO read to be
2045 	 * able to read what was just written.
2046 	 */
2047 	endbyte = pos + written_buffered - 1;
2048 	err = btrfs_fdatawrite_range(inode, pos, endbyte);
2049 	if (err)
2050 		goto out;
2051 	err = filemap_fdatawait_range(inode->i_mapping, pos, endbyte);
2052 	if (err)
2053 		goto out;
2054 	written += written_buffered;
2055 	iocb->ki_pos = pos + written_buffered;
2056 	invalidate_mapping_pages(file->f_mapping, pos >> PAGE_SHIFT,
2057 				 endbyte >> PAGE_SHIFT);
2058 out:
2059 	return err < 0 ? err : written;
2060 }
2061 
btrfs_file_write_iter(struct kiocb * iocb,struct iov_iter * from)2062 static ssize_t btrfs_file_write_iter(struct kiocb *iocb,
2063 				    struct iov_iter *from)
2064 {
2065 	struct file *file = iocb->ki_filp;
2066 	struct btrfs_inode *inode = BTRFS_I(file_inode(file));
2067 	ssize_t num_written = 0;
2068 	const bool sync = iocb->ki_flags & IOCB_DSYNC;
2069 
2070 	/*
2071 	 * If the fs flips readonly due to some impossible error, although we
2072 	 * have opened a file as writable, we have to stop this write operation
2073 	 * to ensure consistency.
2074 	 */
2075 	if (test_bit(BTRFS_FS_STATE_ERROR, &inode->root->fs_info->fs_state))
2076 		return -EROFS;
2077 
2078 	if (!(iocb->ki_flags & IOCB_DIRECT) &&
2079 	    (iocb->ki_flags & IOCB_NOWAIT))
2080 		return -EOPNOTSUPP;
2081 
2082 	if (sync)
2083 		atomic_inc(&inode->sync_writers);
2084 
2085 	if (iocb->ki_flags & IOCB_DIRECT)
2086 		num_written = btrfs_direct_write(iocb, from);
2087 	else
2088 		num_written = btrfs_buffered_write(iocb, from);
2089 
2090 	btrfs_set_inode_last_sub_trans(inode);
2091 
2092 	if (num_written > 0)
2093 		num_written = generic_write_sync(iocb, num_written);
2094 
2095 	if (sync)
2096 		atomic_dec(&inode->sync_writers);
2097 
2098 	current->backing_dev_info = NULL;
2099 	return num_written;
2100 }
2101 
btrfs_release_file(struct inode * inode,struct file * filp)2102 int btrfs_release_file(struct inode *inode, struct file *filp)
2103 {
2104 	struct btrfs_file_private *private = filp->private_data;
2105 
2106 	if (private && private->filldir_buf)
2107 		kfree(private->filldir_buf);
2108 	kfree(private);
2109 	filp->private_data = NULL;
2110 
2111 	/*
2112 	 * Set by setattr when we are about to truncate a file from a non-zero
2113 	 * size to a zero size.  This tries to flush down new bytes that may
2114 	 * have been written if the application were using truncate to replace
2115 	 * a file in place.
2116 	 */
2117 	if (test_and_clear_bit(BTRFS_INODE_FLUSH_ON_CLOSE,
2118 			       &BTRFS_I(inode)->runtime_flags))
2119 			filemap_flush(inode->i_mapping);
2120 	return 0;
2121 }
2122 
start_ordered_ops(struct inode * inode,loff_t start,loff_t end)2123 static int start_ordered_ops(struct inode *inode, loff_t start, loff_t end)
2124 {
2125 	int ret;
2126 	struct blk_plug plug;
2127 
2128 	/*
2129 	 * This is only called in fsync, which would do synchronous writes, so
2130 	 * a plug can merge adjacent IOs as much as possible.  Esp. in case of
2131 	 * multiple disks using raid profile, a large IO can be split to
2132 	 * several segments of stripe length (currently 64K).
2133 	 */
2134 	blk_start_plug(&plug);
2135 	atomic_inc(&BTRFS_I(inode)->sync_writers);
2136 	ret = btrfs_fdatawrite_range(inode, start, end);
2137 	atomic_dec(&BTRFS_I(inode)->sync_writers);
2138 	blk_finish_plug(&plug);
2139 
2140 	return ret;
2141 }
2142 
skip_inode_logging(const struct btrfs_log_ctx * ctx)2143 static inline bool skip_inode_logging(const struct btrfs_log_ctx *ctx)
2144 {
2145 	struct btrfs_inode *inode = BTRFS_I(ctx->inode);
2146 	struct btrfs_fs_info *fs_info = inode->root->fs_info;
2147 
2148 	if (btrfs_inode_in_log(inode, fs_info->generation) &&
2149 	    list_empty(&ctx->ordered_extents))
2150 		return true;
2151 
2152 	/*
2153 	 * If we are doing a fast fsync we can not bail out if the inode's
2154 	 * last_trans is <= then the last committed transaction, because we only
2155 	 * update the last_trans of the inode during ordered extent completion,
2156 	 * and for a fast fsync we don't wait for that, we only wait for the
2157 	 * writeback to complete.
2158 	 */
2159 	if (inode->last_trans <= fs_info->last_trans_committed &&
2160 	    (test_bit(BTRFS_INODE_NEEDS_FULL_SYNC, &inode->runtime_flags) ||
2161 	     list_empty(&ctx->ordered_extents)))
2162 		return true;
2163 
2164 	return false;
2165 }
2166 
2167 /*
2168  * fsync call for both files and directories.  This logs the inode into
2169  * the tree log instead of forcing full commits whenever possible.
2170  *
2171  * It needs to call filemap_fdatawait so that all ordered extent updates are
2172  * in the metadata btree are up to date for copying to the log.
2173  *
2174  * It drops the inode mutex before doing the tree log commit.  This is an
2175  * important optimization for directories because holding the mutex prevents
2176  * new operations on the dir while we write to disk.
2177  */
btrfs_sync_file(struct file * file,loff_t start,loff_t end,int datasync)2178 int btrfs_sync_file(struct file *file, loff_t start, loff_t end, int datasync)
2179 {
2180 	struct dentry *dentry = file_dentry(file);
2181 	struct inode *inode = d_inode(dentry);
2182 	struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
2183 	struct btrfs_root *root = BTRFS_I(inode)->root;
2184 	struct btrfs_trans_handle *trans;
2185 	struct btrfs_log_ctx ctx;
2186 	int ret = 0, err;
2187 	u64 len;
2188 	bool full_sync;
2189 
2190 	trace_btrfs_sync_file(file, datasync);
2191 
2192 	btrfs_init_log_ctx(&ctx, inode);
2193 
2194 	/*
2195 	 * Always set the range to a full range, otherwise we can get into
2196 	 * several problems, from missing file extent items to represent holes
2197 	 * when not using the NO_HOLES feature, to log tree corruption due to
2198 	 * races between hole detection during logging and completion of ordered
2199 	 * extents outside the range, to missing checksums due to ordered extents
2200 	 * for which we flushed only a subset of their pages.
2201 	 */
2202 	start = 0;
2203 	end = LLONG_MAX;
2204 	len = (u64)LLONG_MAX + 1;
2205 
2206 	/*
2207 	 * We write the dirty pages in the range and wait until they complete
2208 	 * out of the ->i_mutex. If so, we can flush the dirty pages by
2209 	 * multi-task, and make the performance up.  See
2210 	 * btrfs_wait_ordered_range for an explanation of the ASYNC check.
2211 	 */
2212 	ret = start_ordered_ops(inode, start, end);
2213 	if (ret)
2214 		goto out;
2215 
2216 	btrfs_inode_lock(inode, BTRFS_ILOCK_MMAP);
2217 
2218 	atomic_inc(&root->log_batch);
2219 
2220 	/*
2221 	 * Always check for the full sync flag while holding the inode's lock,
2222 	 * to avoid races with other tasks. The flag must be either set all the
2223 	 * time during logging or always off all the time while logging.
2224 	 */
2225 	full_sync = test_bit(BTRFS_INODE_NEEDS_FULL_SYNC,
2226 			     &BTRFS_I(inode)->runtime_flags);
2227 
2228 	/*
2229 	 * Before we acquired the inode's lock and the mmap lock, someone may
2230 	 * have dirtied more pages in the target range. We need to make sure
2231 	 * that writeback for any such pages does not start while we are logging
2232 	 * the inode, because if it does, any of the following might happen when
2233 	 * we are not doing a full inode sync:
2234 	 *
2235 	 * 1) We log an extent after its writeback finishes but before its
2236 	 *    checksums are added to the csum tree, leading to -EIO errors
2237 	 *    when attempting to read the extent after a log replay.
2238 	 *
2239 	 * 2) We can end up logging an extent before its writeback finishes.
2240 	 *    Therefore after the log replay we will have a file extent item
2241 	 *    pointing to an unwritten extent (and no data checksums as well).
2242 	 *
2243 	 * So trigger writeback for any eventual new dirty pages and then we
2244 	 * wait for all ordered extents to complete below.
2245 	 */
2246 	ret = start_ordered_ops(inode, start, end);
2247 	if (ret) {
2248 		btrfs_inode_unlock(inode, BTRFS_ILOCK_MMAP);
2249 		goto out;
2250 	}
2251 
2252 	/*
2253 	 * We have to do this here to avoid the priority inversion of waiting on
2254 	 * IO of a lower priority task while holding a transaction open.
2255 	 *
2256 	 * For a full fsync we wait for the ordered extents to complete while
2257 	 * for a fast fsync we wait just for writeback to complete, and then
2258 	 * attach the ordered extents to the transaction so that a transaction
2259 	 * commit waits for their completion, to avoid data loss if we fsync,
2260 	 * the current transaction commits before the ordered extents complete
2261 	 * and a power failure happens right after that.
2262 	 *
2263 	 * For zoned filesystem, if a write IO uses a ZONE_APPEND command, the
2264 	 * logical address recorded in the ordered extent may change. We need
2265 	 * to wait for the IO to stabilize the logical address.
2266 	 */
2267 	if (full_sync || btrfs_is_zoned(fs_info)) {
2268 		ret = btrfs_wait_ordered_range(inode, start, len);
2269 	} else {
2270 		/*
2271 		 * Get our ordered extents as soon as possible to avoid doing
2272 		 * checksum lookups in the csum tree, and use instead the
2273 		 * checksums attached to the ordered extents.
2274 		 */
2275 		btrfs_get_ordered_extents_for_logging(BTRFS_I(inode),
2276 						      &ctx.ordered_extents);
2277 		ret = filemap_fdatawait_range(inode->i_mapping, start, end);
2278 	}
2279 
2280 	if (ret)
2281 		goto out_release_extents;
2282 
2283 	atomic_inc(&root->log_batch);
2284 
2285 	smp_mb();
2286 	if (skip_inode_logging(&ctx)) {
2287 		/*
2288 		 * We've had everything committed since the last time we were
2289 		 * modified so clear this flag in case it was set for whatever
2290 		 * reason, it's no longer relevant.
2291 		 */
2292 		clear_bit(BTRFS_INODE_NEEDS_FULL_SYNC,
2293 			  &BTRFS_I(inode)->runtime_flags);
2294 		/*
2295 		 * An ordered extent might have started before and completed
2296 		 * already with io errors, in which case the inode was not
2297 		 * updated and we end up here. So check the inode's mapping
2298 		 * for any errors that might have happened since we last
2299 		 * checked called fsync.
2300 		 */
2301 		ret = filemap_check_wb_err(inode->i_mapping, file->f_wb_err);
2302 		goto out_release_extents;
2303 	}
2304 
2305 	/*
2306 	 * We use start here because we will need to wait on the IO to complete
2307 	 * in btrfs_sync_log, which could require joining a transaction (for
2308 	 * example checking cross references in the nocow path).  If we use join
2309 	 * here we could get into a situation where we're waiting on IO to
2310 	 * happen that is blocked on a transaction trying to commit.  With start
2311 	 * we inc the extwriter counter, so we wait for all extwriters to exit
2312 	 * before we start blocking joiners.  This comment is to keep somebody
2313 	 * from thinking they are super smart and changing this to
2314 	 * btrfs_join_transaction *cough*Josef*cough*.
2315 	 */
2316 	trans = btrfs_start_transaction(root, 0);
2317 	if (IS_ERR(trans)) {
2318 		ret = PTR_ERR(trans);
2319 		goto out_release_extents;
2320 	}
2321 	trans->in_fsync = true;
2322 
2323 	ret = btrfs_log_dentry_safe(trans, dentry, &ctx);
2324 	btrfs_release_log_ctx_extents(&ctx);
2325 	if (ret < 0) {
2326 		/* Fallthrough and commit/free transaction. */
2327 		ret = 1;
2328 	}
2329 
2330 	/* we've logged all the items and now have a consistent
2331 	 * version of the file in the log.  It is possible that
2332 	 * someone will come in and modify the file, but that's
2333 	 * fine because the log is consistent on disk, and we
2334 	 * have references to all of the file's extents
2335 	 *
2336 	 * It is possible that someone will come in and log the
2337 	 * file again, but that will end up using the synchronization
2338 	 * inside btrfs_sync_log to keep things safe.
2339 	 */
2340 	btrfs_inode_unlock(inode, BTRFS_ILOCK_MMAP);
2341 
2342 	if (ret == BTRFS_NO_LOG_SYNC) {
2343 		ret = btrfs_end_transaction(trans);
2344 		goto out;
2345 	}
2346 
2347 	/* We successfully logged the inode, attempt to sync the log. */
2348 	if (!ret) {
2349 		ret = btrfs_sync_log(trans, root, &ctx);
2350 		if (!ret) {
2351 			ret = btrfs_end_transaction(trans);
2352 			goto out;
2353 		}
2354 	}
2355 
2356 	/*
2357 	 * At this point we need to commit the transaction because we had
2358 	 * btrfs_need_log_full_commit() or some other error.
2359 	 *
2360 	 * If we didn't do a full sync we have to stop the trans handle, wait on
2361 	 * the ordered extents, start it again and commit the transaction.  If
2362 	 * we attempt to wait on the ordered extents here we could deadlock with
2363 	 * something like fallocate() that is holding the extent lock trying to
2364 	 * start a transaction while some other thread is trying to commit the
2365 	 * transaction while we (fsync) are currently holding the transaction
2366 	 * open.
2367 	 */
2368 	if (!full_sync) {
2369 		ret = btrfs_end_transaction(trans);
2370 		if (ret)
2371 			goto out;
2372 		ret = btrfs_wait_ordered_range(inode, start, len);
2373 		if (ret)
2374 			goto out;
2375 
2376 		/*
2377 		 * This is safe to use here because we're only interested in
2378 		 * making sure the transaction that had the ordered extents is
2379 		 * committed.  We aren't waiting on anything past this point,
2380 		 * we're purely getting the transaction and committing it.
2381 		 */
2382 		trans = btrfs_attach_transaction_barrier(root);
2383 		if (IS_ERR(trans)) {
2384 			ret = PTR_ERR(trans);
2385 
2386 			/*
2387 			 * We committed the transaction and there's no currently
2388 			 * running transaction, this means everything we care
2389 			 * about made it to disk and we are done.
2390 			 */
2391 			if (ret == -ENOENT)
2392 				ret = 0;
2393 			goto out;
2394 		}
2395 	}
2396 
2397 	ret = btrfs_commit_transaction(trans);
2398 out:
2399 	ASSERT(list_empty(&ctx.list));
2400 	err = file_check_and_advance_wb_err(file);
2401 	if (!ret)
2402 		ret = err;
2403 	return ret > 0 ? -EIO : ret;
2404 
2405 out_release_extents:
2406 	btrfs_release_log_ctx_extents(&ctx);
2407 	btrfs_inode_unlock(inode, BTRFS_ILOCK_MMAP);
2408 	goto out;
2409 }
2410 
2411 static const struct vm_operations_struct btrfs_file_vm_ops = {
2412 	.fault		= filemap_fault,
2413 	.map_pages	= filemap_map_pages,
2414 	.page_mkwrite	= btrfs_page_mkwrite,
2415 	.speculative	= true,
2416 };
2417 
btrfs_file_mmap(struct file * filp,struct vm_area_struct * vma)2418 static int btrfs_file_mmap(struct file	*filp, struct vm_area_struct *vma)
2419 {
2420 	struct address_space *mapping = filp->f_mapping;
2421 
2422 	if (!mapping->a_ops->readpage)
2423 		return -ENOEXEC;
2424 
2425 	file_accessed(filp);
2426 	vma->vm_ops = &btrfs_file_vm_ops;
2427 
2428 	return 0;
2429 }
2430 
hole_mergeable(struct btrfs_inode * inode,struct extent_buffer * leaf,int slot,u64 start,u64 end)2431 static int hole_mergeable(struct btrfs_inode *inode, struct extent_buffer *leaf,
2432 			  int slot, u64 start, u64 end)
2433 {
2434 	struct btrfs_file_extent_item *fi;
2435 	struct btrfs_key key;
2436 
2437 	if (slot < 0 || slot >= btrfs_header_nritems(leaf))
2438 		return 0;
2439 
2440 	btrfs_item_key_to_cpu(leaf, &key, slot);
2441 	if (key.objectid != btrfs_ino(inode) ||
2442 	    key.type != BTRFS_EXTENT_DATA_KEY)
2443 		return 0;
2444 
2445 	fi = btrfs_item_ptr(leaf, slot, struct btrfs_file_extent_item);
2446 
2447 	if (btrfs_file_extent_type(leaf, fi) != BTRFS_FILE_EXTENT_REG)
2448 		return 0;
2449 
2450 	if (btrfs_file_extent_disk_bytenr(leaf, fi))
2451 		return 0;
2452 
2453 	if (key.offset == end)
2454 		return 1;
2455 	if (key.offset + btrfs_file_extent_num_bytes(leaf, fi) == start)
2456 		return 1;
2457 	return 0;
2458 }
2459 
fill_holes(struct btrfs_trans_handle * trans,struct btrfs_inode * inode,struct btrfs_path * path,u64 offset,u64 end)2460 static int fill_holes(struct btrfs_trans_handle *trans,
2461 		struct btrfs_inode *inode,
2462 		struct btrfs_path *path, u64 offset, u64 end)
2463 {
2464 	struct btrfs_fs_info *fs_info = trans->fs_info;
2465 	struct btrfs_root *root = inode->root;
2466 	struct extent_buffer *leaf;
2467 	struct btrfs_file_extent_item *fi;
2468 	struct extent_map *hole_em;
2469 	struct extent_map_tree *em_tree = &inode->extent_tree;
2470 	struct btrfs_key key;
2471 	int ret;
2472 
2473 	if (btrfs_fs_incompat(fs_info, NO_HOLES))
2474 		goto out;
2475 
2476 	key.objectid = btrfs_ino(inode);
2477 	key.type = BTRFS_EXTENT_DATA_KEY;
2478 	key.offset = offset;
2479 
2480 	ret = btrfs_search_slot(trans, root, &key, path, 0, 1);
2481 	if (ret <= 0) {
2482 		/*
2483 		 * We should have dropped this offset, so if we find it then
2484 		 * something has gone horribly wrong.
2485 		 */
2486 		if (ret == 0)
2487 			ret = -EINVAL;
2488 		return ret;
2489 	}
2490 
2491 	leaf = path->nodes[0];
2492 	if (hole_mergeable(inode, leaf, path->slots[0] - 1, offset, end)) {
2493 		u64 num_bytes;
2494 
2495 		path->slots[0]--;
2496 		fi = btrfs_item_ptr(leaf, path->slots[0],
2497 				    struct btrfs_file_extent_item);
2498 		num_bytes = btrfs_file_extent_num_bytes(leaf, fi) +
2499 			end - offset;
2500 		btrfs_set_file_extent_num_bytes(leaf, fi, num_bytes);
2501 		btrfs_set_file_extent_ram_bytes(leaf, fi, num_bytes);
2502 		btrfs_set_file_extent_offset(leaf, fi, 0);
2503 		btrfs_mark_buffer_dirty(leaf);
2504 		goto out;
2505 	}
2506 
2507 	if (hole_mergeable(inode, leaf, path->slots[0], offset, end)) {
2508 		u64 num_bytes;
2509 
2510 		key.offset = offset;
2511 		btrfs_set_item_key_safe(fs_info, path, &key);
2512 		fi = btrfs_item_ptr(leaf, path->slots[0],
2513 				    struct btrfs_file_extent_item);
2514 		num_bytes = btrfs_file_extent_num_bytes(leaf, fi) + end -
2515 			offset;
2516 		btrfs_set_file_extent_num_bytes(leaf, fi, num_bytes);
2517 		btrfs_set_file_extent_ram_bytes(leaf, fi, num_bytes);
2518 		btrfs_set_file_extent_offset(leaf, fi, 0);
2519 		btrfs_mark_buffer_dirty(leaf);
2520 		goto out;
2521 	}
2522 	btrfs_release_path(path);
2523 
2524 	ret = btrfs_insert_file_extent(trans, root, btrfs_ino(inode),
2525 			offset, 0, 0, end - offset, 0, end - offset, 0, 0, 0);
2526 	if (ret)
2527 		return ret;
2528 
2529 out:
2530 	btrfs_release_path(path);
2531 
2532 	hole_em = alloc_extent_map();
2533 	if (!hole_em) {
2534 		btrfs_drop_extent_cache(inode, offset, end - 1, 0);
2535 		set_bit(BTRFS_INODE_NEEDS_FULL_SYNC, &inode->runtime_flags);
2536 	} else {
2537 		hole_em->start = offset;
2538 		hole_em->len = end - offset;
2539 		hole_em->ram_bytes = hole_em->len;
2540 		hole_em->orig_start = offset;
2541 
2542 		hole_em->block_start = EXTENT_MAP_HOLE;
2543 		hole_em->block_len = 0;
2544 		hole_em->orig_block_len = 0;
2545 		hole_em->compress_type = BTRFS_COMPRESS_NONE;
2546 		hole_em->generation = trans->transid;
2547 
2548 		do {
2549 			btrfs_drop_extent_cache(inode, offset, end - 1, 0);
2550 			write_lock(&em_tree->lock);
2551 			ret = add_extent_mapping(em_tree, hole_em, 1);
2552 			write_unlock(&em_tree->lock);
2553 		} while (ret == -EEXIST);
2554 		free_extent_map(hole_em);
2555 		if (ret)
2556 			set_bit(BTRFS_INODE_NEEDS_FULL_SYNC,
2557 					&inode->runtime_flags);
2558 	}
2559 
2560 	return 0;
2561 }
2562 
2563 /*
2564  * Find a hole extent on given inode and change start/len to the end of hole
2565  * extent.(hole/vacuum extent whose em->start <= start &&
2566  *	   em->start + em->len > start)
2567  * When a hole extent is found, return 1 and modify start/len.
2568  */
find_first_non_hole(struct btrfs_inode * inode,u64 * start,u64 * len)2569 static int find_first_non_hole(struct btrfs_inode *inode, u64 *start, u64 *len)
2570 {
2571 	struct btrfs_fs_info *fs_info = inode->root->fs_info;
2572 	struct extent_map *em;
2573 	int ret = 0;
2574 
2575 	em = btrfs_get_extent(inode, NULL, 0,
2576 			      round_down(*start, fs_info->sectorsize),
2577 			      round_up(*len, fs_info->sectorsize));
2578 	if (IS_ERR(em))
2579 		return PTR_ERR(em);
2580 
2581 	/* Hole or vacuum extent(only exists in no-hole mode) */
2582 	if (em->block_start == EXTENT_MAP_HOLE) {
2583 		ret = 1;
2584 		*len = em->start + em->len > *start + *len ?
2585 		       0 : *start + *len - em->start - em->len;
2586 		*start = em->start + em->len;
2587 	}
2588 	free_extent_map(em);
2589 	return ret;
2590 }
2591 
btrfs_punch_hole_lock_range(struct inode * inode,const u64 lockstart,const u64 lockend,struct extent_state ** cached_state)2592 static int btrfs_punch_hole_lock_range(struct inode *inode,
2593 				       const u64 lockstart,
2594 				       const u64 lockend,
2595 				       struct extent_state **cached_state)
2596 {
2597 	/*
2598 	 * For subpage case, if the range is not at page boundary, we could
2599 	 * have pages at the leading/tailing part of the range.
2600 	 * This could lead to dead loop since filemap_range_has_page()
2601 	 * will always return true.
2602 	 * So here we need to do extra page alignment for
2603 	 * filemap_range_has_page().
2604 	 */
2605 	const u64 page_lockstart = round_up(lockstart, PAGE_SIZE);
2606 	const u64 page_lockend = round_down(lockend + 1, PAGE_SIZE) - 1;
2607 
2608 	while (1) {
2609 		struct btrfs_ordered_extent *ordered;
2610 		int ret;
2611 
2612 		truncate_pagecache_range(inode, lockstart, lockend);
2613 
2614 		lock_extent_bits(&BTRFS_I(inode)->io_tree, lockstart, lockend,
2615 				 cached_state);
2616 		ordered = btrfs_lookup_first_ordered_extent(BTRFS_I(inode),
2617 							    lockend);
2618 
2619 		/*
2620 		 * We need to make sure we have no ordered extents in this range
2621 		 * and nobody raced in and read a page in this range, if we did
2622 		 * we need to try again.
2623 		 */
2624 		if ((!ordered ||
2625 		    (ordered->file_offset + ordered->num_bytes <= lockstart ||
2626 		     ordered->file_offset > lockend)) &&
2627 		     !filemap_range_has_page(inode->i_mapping,
2628 					     page_lockstart, page_lockend)) {
2629 			if (ordered)
2630 				btrfs_put_ordered_extent(ordered);
2631 			break;
2632 		}
2633 		if (ordered)
2634 			btrfs_put_ordered_extent(ordered);
2635 		unlock_extent_cached(&BTRFS_I(inode)->io_tree, lockstart,
2636 				     lockend, cached_state);
2637 		ret = btrfs_wait_ordered_range(inode, lockstart,
2638 					       lockend - lockstart + 1);
2639 		if (ret)
2640 			return ret;
2641 	}
2642 	return 0;
2643 }
2644 
btrfs_insert_replace_extent(struct btrfs_trans_handle * trans,struct btrfs_inode * inode,struct btrfs_path * path,struct btrfs_replace_extent_info * extent_info,const u64 replace_len,const u64 bytes_to_drop)2645 static int btrfs_insert_replace_extent(struct btrfs_trans_handle *trans,
2646 				     struct btrfs_inode *inode,
2647 				     struct btrfs_path *path,
2648 				     struct btrfs_replace_extent_info *extent_info,
2649 				     const u64 replace_len,
2650 				     const u64 bytes_to_drop)
2651 {
2652 	struct btrfs_fs_info *fs_info = trans->fs_info;
2653 	struct btrfs_root *root = inode->root;
2654 	struct btrfs_file_extent_item *extent;
2655 	struct extent_buffer *leaf;
2656 	struct btrfs_key key;
2657 	int slot;
2658 	struct btrfs_ref ref = { 0 };
2659 	int ret;
2660 
2661 	if (replace_len == 0)
2662 		return 0;
2663 
2664 	if (extent_info->disk_offset == 0 &&
2665 	    btrfs_fs_incompat(fs_info, NO_HOLES)) {
2666 		btrfs_update_inode_bytes(inode, 0, bytes_to_drop);
2667 		return 0;
2668 	}
2669 
2670 	key.objectid = btrfs_ino(inode);
2671 	key.type = BTRFS_EXTENT_DATA_KEY;
2672 	key.offset = extent_info->file_offset;
2673 	ret = btrfs_insert_empty_item(trans, root, path, &key,
2674 				      sizeof(struct btrfs_file_extent_item));
2675 	if (ret)
2676 		return ret;
2677 	leaf = path->nodes[0];
2678 	slot = path->slots[0];
2679 	write_extent_buffer(leaf, extent_info->extent_buf,
2680 			    btrfs_item_ptr_offset(leaf, slot),
2681 			    sizeof(struct btrfs_file_extent_item));
2682 	extent = btrfs_item_ptr(leaf, slot, struct btrfs_file_extent_item);
2683 	ASSERT(btrfs_file_extent_type(leaf, extent) != BTRFS_FILE_EXTENT_INLINE);
2684 	btrfs_set_file_extent_offset(leaf, extent, extent_info->data_offset);
2685 	btrfs_set_file_extent_num_bytes(leaf, extent, replace_len);
2686 	if (extent_info->is_new_extent)
2687 		btrfs_set_file_extent_generation(leaf, extent, trans->transid);
2688 	btrfs_mark_buffer_dirty(leaf);
2689 	btrfs_release_path(path);
2690 
2691 	ret = btrfs_inode_set_file_extent_range(inode, extent_info->file_offset,
2692 						replace_len);
2693 	if (ret)
2694 		return ret;
2695 
2696 	/* If it's a hole, nothing more needs to be done. */
2697 	if (extent_info->disk_offset == 0) {
2698 		btrfs_update_inode_bytes(inode, 0, bytes_to_drop);
2699 		return 0;
2700 	}
2701 
2702 	btrfs_update_inode_bytes(inode, replace_len, bytes_to_drop);
2703 
2704 	if (extent_info->is_new_extent && extent_info->insertions == 0) {
2705 		key.objectid = extent_info->disk_offset;
2706 		key.type = BTRFS_EXTENT_ITEM_KEY;
2707 		key.offset = extent_info->disk_len;
2708 		ret = btrfs_alloc_reserved_file_extent(trans, root,
2709 						       btrfs_ino(inode),
2710 						       extent_info->file_offset,
2711 						       extent_info->qgroup_reserved,
2712 						       &key);
2713 	} else {
2714 		u64 ref_offset;
2715 
2716 		btrfs_init_generic_ref(&ref, BTRFS_ADD_DELAYED_REF,
2717 				       extent_info->disk_offset,
2718 				       extent_info->disk_len, 0);
2719 		ref_offset = extent_info->file_offset - extent_info->data_offset;
2720 		btrfs_init_data_ref(&ref, root->root_key.objectid,
2721 				    btrfs_ino(inode), ref_offset, 0, false);
2722 		ret = btrfs_inc_extent_ref(trans, &ref);
2723 	}
2724 
2725 	extent_info->insertions++;
2726 
2727 	return ret;
2728 }
2729 
2730 /*
2731  * The respective range must have been previously locked, as well as the inode.
2732  * The end offset is inclusive (last byte of the range).
2733  * @extent_info is NULL for fallocate's hole punching and non-NULL when replacing
2734  * the file range with an extent.
2735  * When not punching a hole, we don't want to end up in a state where we dropped
2736  * extents without inserting a new one, so we must abort the transaction to avoid
2737  * a corruption.
2738  */
btrfs_replace_file_extents(struct btrfs_inode * inode,struct btrfs_path * path,const u64 start,const u64 end,struct btrfs_replace_extent_info * extent_info,struct btrfs_trans_handle ** trans_out)2739 int btrfs_replace_file_extents(struct btrfs_inode *inode,
2740 			       struct btrfs_path *path, const u64 start,
2741 			       const u64 end,
2742 			       struct btrfs_replace_extent_info *extent_info,
2743 			       struct btrfs_trans_handle **trans_out)
2744 {
2745 	struct btrfs_drop_extents_args drop_args = { 0 };
2746 	struct btrfs_root *root = inode->root;
2747 	struct btrfs_fs_info *fs_info = root->fs_info;
2748 	u64 min_size = btrfs_calc_insert_metadata_size(fs_info, 1);
2749 	u64 ino_size = round_up(inode->vfs_inode.i_size, fs_info->sectorsize);
2750 	struct btrfs_trans_handle *trans = NULL;
2751 	struct btrfs_block_rsv *rsv;
2752 	unsigned int rsv_count;
2753 	u64 cur_offset;
2754 	u64 len = end - start;
2755 	int ret = 0;
2756 
2757 	if (end <= start)
2758 		return -EINVAL;
2759 
2760 	rsv = btrfs_alloc_block_rsv(fs_info, BTRFS_BLOCK_RSV_TEMP);
2761 	if (!rsv) {
2762 		ret = -ENOMEM;
2763 		goto out;
2764 	}
2765 	rsv->size = btrfs_calc_insert_metadata_size(fs_info, 1);
2766 	rsv->failfast = 1;
2767 
2768 	/*
2769 	 * 1 - update the inode
2770 	 * 1 - removing the extents in the range
2771 	 * 1 - adding the hole extent if no_holes isn't set or if we are
2772 	 *     replacing the range with a new extent
2773 	 */
2774 	if (!btrfs_fs_incompat(fs_info, NO_HOLES) || extent_info)
2775 		rsv_count = 3;
2776 	else
2777 		rsv_count = 2;
2778 
2779 	trans = btrfs_start_transaction(root, rsv_count);
2780 	if (IS_ERR(trans)) {
2781 		ret = PTR_ERR(trans);
2782 		trans = NULL;
2783 		goto out_free;
2784 	}
2785 
2786 	ret = btrfs_block_rsv_migrate(&fs_info->trans_block_rsv, rsv,
2787 				      min_size, false);
2788 	BUG_ON(ret);
2789 	trans->block_rsv = rsv;
2790 
2791 	cur_offset = start;
2792 	drop_args.path = path;
2793 	drop_args.end = end + 1;
2794 	drop_args.drop_cache = true;
2795 	while (cur_offset < end) {
2796 		drop_args.start = cur_offset;
2797 		ret = btrfs_drop_extents(trans, root, inode, &drop_args);
2798 		/* If we are punching a hole decrement the inode's byte count */
2799 		if (!extent_info)
2800 			btrfs_update_inode_bytes(inode, 0,
2801 						 drop_args.bytes_found);
2802 		if (ret != -ENOSPC) {
2803 			/*
2804 			 * The only time we don't want to abort is if we are
2805 			 * attempting to clone a partial inline extent, in which
2806 			 * case we'll get EOPNOTSUPP.  However if we aren't
2807 			 * clone we need to abort no matter what, because if we
2808 			 * got EOPNOTSUPP via prealloc then we messed up and
2809 			 * need to abort.
2810 			 */
2811 			if (ret &&
2812 			    (ret != -EOPNOTSUPP ||
2813 			     (extent_info && extent_info->is_new_extent)))
2814 				btrfs_abort_transaction(trans, ret);
2815 			break;
2816 		}
2817 
2818 		trans->block_rsv = &fs_info->trans_block_rsv;
2819 
2820 		if (!extent_info && cur_offset < drop_args.drop_end &&
2821 		    cur_offset < ino_size) {
2822 			ret = fill_holes(trans, inode, path, cur_offset,
2823 					 drop_args.drop_end);
2824 			if (ret) {
2825 				/*
2826 				 * If we failed then we didn't insert our hole
2827 				 * entries for the area we dropped, so now the
2828 				 * fs is corrupted, so we must abort the
2829 				 * transaction.
2830 				 */
2831 				btrfs_abort_transaction(trans, ret);
2832 				break;
2833 			}
2834 		} else if (!extent_info && cur_offset < drop_args.drop_end) {
2835 			/*
2836 			 * We are past the i_size here, but since we didn't
2837 			 * insert holes we need to clear the mapped area so we
2838 			 * know to not set disk_i_size in this area until a new
2839 			 * file extent is inserted here.
2840 			 */
2841 			ret = btrfs_inode_clear_file_extent_range(inode,
2842 					cur_offset,
2843 					drop_args.drop_end - cur_offset);
2844 			if (ret) {
2845 				/*
2846 				 * We couldn't clear our area, so we could
2847 				 * presumably adjust up and corrupt the fs, so
2848 				 * we need to abort.
2849 				 */
2850 				btrfs_abort_transaction(trans, ret);
2851 				break;
2852 			}
2853 		}
2854 
2855 		if (extent_info &&
2856 		    drop_args.drop_end > extent_info->file_offset) {
2857 			u64 replace_len = drop_args.drop_end -
2858 					  extent_info->file_offset;
2859 
2860 			ret = btrfs_insert_replace_extent(trans, inode,	path,
2861 					extent_info, replace_len,
2862 					drop_args.bytes_found);
2863 			if (ret) {
2864 				btrfs_abort_transaction(trans, ret);
2865 				break;
2866 			}
2867 			extent_info->data_len -= replace_len;
2868 			extent_info->data_offset += replace_len;
2869 			extent_info->file_offset += replace_len;
2870 		}
2871 
2872 		ret = btrfs_update_inode(trans, root, inode);
2873 		if (ret)
2874 			break;
2875 
2876 		btrfs_end_transaction(trans);
2877 		btrfs_btree_balance_dirty(fs_info);
2878 
2879 		trans = btrfs_start_transaction(root, rsv_count);
2880 		if (IS_ERR(trans)) {
2881 			ret = PTR_ERR(trans);
2882 			trans = NULL;
2883 			break;
2884 		}
2885 
2886 		ret = btrfs_block_rsv_migrate(&fs_info->trans_block_rsv,
2887 					      rsv, min_size, false);
2888 		BUG_ON(ret);	/* shouldn't happen */
2889 		trans->block_rsv = rsv;
2890 
2891 		cur_offset = drop_args.drop_end;
2892 		len = end - cur_offset;
2893 		if (!extent_info && len) {
2894 			ret = find_first_non_hole(inode, &cur_offset, &len);
2895 			if (unlikely(ret < 0))
2896 				break;
2897 			if (ret && !len) {
2898 				ret = 0;
2899 				break;
2900 			}
2901 		}
2902 	}
2903 
2904 	/*
2905 	 * If we were cloning, force the next fsync to be a full one since we
2906 	 * we replaced (or just dropped in the case of cloning holes when
2907 	 * NO_HOLES is enabled) file extent items and did not setup new extent
2908 	 * maps for the replacement extents (or holes).
2909 	 */
2910 	if (extent_info && !extent_info->is_new_extent)
2911 		set_bit(BTRFS_INODE_NEEDS_FULL_SYNC, &inode->runtime_flags);
2912 
2913 	if (ret)
2914 		goto out_trans;
2915 
2916 	trans->block_rsv = &fs_info->trans_block_rsv;
2917 	/*
2918 	 * If we are using the NO_HOLES feature we might have had already an
2919 	 * hole that overlaps a part of the region [lockstart, lockend] and
2920 	 * ends at (or beyond) lockend. Since we have no file extent items to
2921 	 * represent holes, drop_end can be less than lockend and so we must
2922 	 * make sure we have an extent map representing the existing hole (the
2923 	 * call to __btrfs_drop_extents() might have dropped the existing extent
2924 	 * map representing the existing hole), otherwise the fast fsync path
2925 	 * will not record the existence of the hole region
2926 	 * [existing_hole_start, lockend].
2927 	 */
2928 	if (drop_args.drop_end <= end)
2929 		drop_args.drop_end = end + 1;
2930 	/*
2931 	 * Don't insert file hole extent item if it's for a range beyond eof
2932 	 * (because it's useless) or if it represents a 0 bytes range (when
2933 	 * cur_offset == drop_end).
2934 	 */
2935 	if (!extent_info && cur_offset < ino_size &&
2936 	    cur_offset < drop_args.drop_end) {
2937 		ret = fill_holes(trans, inode, path, cur_offset,
2938 				 drop_args.drop_end);
2939 		if (ret) {
2940 			/* Same comment as above. */
2941 			btrfs_abort_transaction(trans, ret);
2942 			goto out_trans;
2943 		}
2944 	} else if (!extent_info && cur_offset < drop_args.drop_end) {
2945 		/* See the comment in the loop above for the reasoning here. */
2946 		ret = btrfs_inode_clear_file_extent_range(inode, cur_offset,
2947 					drop_args.drop_end - cur_offset);
2948 		if (ret) {
2949 			btrfs_abort_transaction(trans, ret);
2950 			goto out_trans;
2951 		}
2952 
2953 	}
2954 	if (extent_info) {
2955 		ret = btrfs_insert_replace_extent(trans, inode, path,
2956 				extent_info, extent_info->data_len,
2957 				drop_args.bytes_found);
2958 		if (ret) {
2959 			btrfs_abort_transaction(trans, ret);
2960 			goto out_trans;
2961 		}
2962 	}
2963 
2964 out_trans:
2965 	if (!trans)
2966 		goto out_free;
2967 
2968 	trans->block_rsv = &fs_info->trans_block_rsv;
2969 	if (ret)
2970 		btrfs_end_transaction(trans);
2971 	else
2972 		*trans_out = trans;
2973 out_free:
2974 	btrfs_free_block_rsv(fs_info, rsv);
2975 out:
2976 	return ret;
2977 }
2978 
btrfs_punch_hole(struct file * file,loff_t offset,loff_t len)2979 static int btrfs_punch_hole(struct file *file, loff_t offset, loff_t len)
2980 {
2981 	struct inode *inode = file_inode(file);
2982 	struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
2983 	struct btrfs_root *root = BTRFS_I(inode)->root;
2984 	struct extent_state *cached_state = NULL;
2985 	struct btrfs_path *path;
2986 	struct btrfs_trans_handle *trans = NULL;
2987 	u64 lockstart;
2988 	u64 lockend;
2989 	u64 tail_start;
2990 	u64 tail_len;
2991 	u64 orig_start = offset;
2992 	int ret = 0;
2993 	bool same_block;
2994 	u64 ino_size;
2995 	bool truncated_block = false;
2996 	bool updated_inode = false;
2997 
2998 	ret = btrfs_wait_ordered_range(inode, offset, len);
2999 	if (ret)
3000 		return ret;
3001 
3002 	btrfs_inode_lock(inode, BTRFS_ILOCK_MMAP);
3003 	ino_size = round_up(inode->i_size, fs_info->sectorsize);
3004 	ret = find_first_non_hole(BTRFS_I(inode), &offset, &len);
3005 	if (ret < 0)
3006 		goto out_only_mutex;
3007 	if (ret && !len) {
3008 		/* Already in a large hole */
3009 		ret = 0;
3010 		goto out_only_mutex;
3011 	}
3012 
3013 	ret = file_modified(file);
3014 	if (ret)
3015 		goto out_only_mutex;
3016 
3017 	lockstart = round_up(offset, btrfs_inode_sectorsize(BTRFS_I(inode)));
3018 	lockend = round_down(offset + len,
3019 			     btrfs_inode_sectorsize(BTRFS_I(inode))) - 1;
3020 	same_block = (BTRFS_BYTES_TO_BLKS(fs_info, offset))
3021 		== (BTRFS_BYTES_TO_BLKS(fs_info, offset + len - 1));
3022 	/*
3023 	 * We needn't truncate any block which is beyond the end of the file
3024 	 * because we are sure there is no data there.
3025 	 */
3026 	/*
3027 	 * Only do this if we are in the same block and we aren't doing the
3028 	 * entire block.
3029 	 */
3030 	if (same_block && len < fs_info->sectorsize) {
3031 		if (offset < ino_size) {
3032 			truncated_block = true;
3033 			ret = btrfs_truncate_block(BTRFS_I(inode), offset, len,
3034 						   0);
3035 		} else {
3036 			ret = 0;
3037 		}
3038 		goto out_only_mutex;
3039 	}
3040 
3041 	/* zero back part of the first block */
3042 	if (offset < ino_size) {
3043 		truncated_block = true;
3044 		ret = btrfs_truncate_block(BTRFS_I(inode), offset, 0, 0);
3045 		if (ret) {
3046 			btrfs_inode_unlock(inode, BTRFS_ILOCK_MMAP);
3047 			return ret;
3048 		}
3049 	}
3050 
3051 	/* Check the aligned pages after the first unaligned page,
3052 	 * if offset != orig_start, which means the first unaligned page
3053 	 * including several following pages are already in holes,
3054 	 * the extra check can be skipped */
3055 	if (offset == orig_start) {
3056 		/* after truncate page, check hole again */
3057 		len = offset + len - lockstart;
3058 		offset = lockstart;
3059 		ret = find_first_non_hole(BTRFS_I(inode), &offset, &len);
3060 		if (ret < 0)
3061 			goto out_only_mutex;
3062 		if (ret && !len) {
3063 			ret = 0;
3064 			goto out_only_mutex;
3065 		}
3066 		lockstart = offset;
3067 	}
3068 
3069 	/* Check the tail unaligned part is in a hole */
3070 	tail_start = lockend + 1;
3071 	tail_len = offset + len - tail_start;
3072 	if (tail_len) {
3073 		ret = find_first_non_hole(BTRFS_I(inode), &tail_start, &tail_len);
3074 		if (unlikely(ret < 0))
3075 			goto out_only_mutex;
3076 		if (!ret) {
3077 			/* zero the front end of the last page */
3078 			if (tail_start + tail_len < ino_size) {
3079 				truncated_block = true;
3080 				ret = btrfs_truncate_block(BTRFS_I(inode),
3081 							tail_start + tail_len,
3082 							0, 1);
3083 				if (ret)
3084 					goto out_only_mutex;
3085 			}
3086 		}
3087 	}
3088 
3089 	if (lockend < lockstart) {
3090 		ret = 0;
3091 		goto out_only_mutex;
3092 	}
3093 
3094 	ret = btrfs_punch_hole_lock_range(inode, lockstart, lockend,
3095 					  &cached_state);
3096 	if (ret)
3097 		goto out_only_mutex;
3098 
3099 	path = btrfs_alloc_path();
3100 	if (!path) {
3101 		ret = -ENOMEM;
3102 		goto out;
3103 	}
3104 
3105 	ret = btrfs_replace_file_extents(BTRFS_I(inode), path, lockstart,
3106 					 lockend, NULL, &trans);
3107 	btrfs_free_path(path);
3108 	if (ret)
3109 		goto out;
3110 
3111 	ASSERT(trans != NULL);
3112 	inode_inc_iversion(inode);
3113 	inode->i_mtime = inode->i_ctime = current_time(inode);
3114 	ret = btrfs_update_inode(trans, root, BTRFS_I(inode));
3115 	updated_inode = true;
3116 	btrfs_end_transaction(trans);
3117 	btrfs_btree_balance_dirty(fs_info);
3118 out:
3119 	unlock_extent_cached(&BTRFS_I(inode)->io_tree, lockstart, lockend,
3120 			     &cached_state);
3121 out_only_mutex:
3122 	if (!updated_inode && truncated_block && !ret) {
3123 		/*
3124 		 * If we only end up zeroing part of a page, we still need to
3125 		 * update the inode item, so that all the time fields are
3126 		 * updated as well as the necessary btrfs inode in memory fields
3127 		 * for detecting, at fsync time, if the inode isn't yet in the
3128 		 * log tree or it's there but not up to date.
3129 		 */
3130 		struct timespec64 now = current_time(inode);
3131 
3132 		inode_inc_iversion(inode);
3133 		inode->i_mtime = now;
3134 		inode->i_ctime = now;
3135 		trans = btrfs_start_transaction(root, 1);
3136 		if (IS_ERR(trans)) {
3137 			ret = PTR_ERR(trans);
3138 		} else {
3139 			int ret2;
3140 
3141 			ret = btrfs_update_inode(trans, root, BTRFS_I(inode));
3142 			ret2 = btrfs_end_transaction(trans);
3143 			if (!ret)
3144 				ret = ret2;
3145 		}
3146 	}
3147 	btrfs_inode_unlock(inode, BTRFS_ILOCK_MMAP);
3148 	return ret;
3149 }
3150 
3151 /* Helper structure to record which range is already reserved */
3152 struct falloc_range {
3153 	struct list_head list;
3154 	u64 start;
3155 	u64 len;
3156 };
3157 
3158 /*
3159  * Helper function to add falloc range
3160  *
3161  * Caller should have locked the larger range of extent containing
3162  * [start, len)
3163  */
add_falloc_range(struct list_head * head,u64 start,u64 len)3164 static int add_falloc_range(struct list_head *head, u64 start, u64 len)
3165 {
3166 	struct falloc_range *range = NULL;
3167 
3168 	if (!list_empty(head)) {
3169 		/*
3170 		 * As fallocate iterates by bytenr order, we only need to check
3171 		 * the last range.
3172 		 */
3173 		range = list_last_entry(head, struct falloc_range, list);
3174 		if (range->start + range->len == start) {
3175 			range->len += len;
3176 			return 0;
3177 		}
3178 	}
3179 
3180 	range = kmalloc(sizeof(*range), GFP_KERNEL);
3181 	if (!range)
3182 		return -ENOMEM;
3183 	range->start = start;
3184 	range->len = len;
3185 	list_add_tail(&range->list, head);
3186 	return 0;
3187 }
3188 
btrfs_fallocate_update_isize(struct inode * inode,const u64 end,const int mode)3189 static int btrfs_fallocate_update_isize(struct inode *inode,
3190 					const u64 end,
3191 					const int mode)
3192 {
3193 	struct btrfs_trans_handle *trans;
3194 	struct btrfs_root *root = BTRFS_I(inode)->root;
3195 	int ret;
3196 	int ret2;
3197 
3198 	if (mode & FALLOC_FL_KEEP_SIZE || end <= i_size_read(inode))
3199 		return 0;
3200 
3201 	trans = btrfs_start_transaction(root, 1);
3202 	if (IS_ERR(trans))
3203 		return PTR_ERR(trans);
3204 
3205 	inode->i_ctime = current_time(inode);
3206 	i_size_write(inode, end);
3207 	btrfs_inode_safe_disk_i_size_write(BTRFS_I(inode), 0);
3208 	ret = btrfs_update_inode(trans, root, BTRFS_I(inode));
3209 	ret2 = btrfs_end_transaction(trans);
3210 
3211 	return ret ? ret : ret2;
3212 }
3213 
3214 enum {
3215 	RANGE_BOUNDARY_WRITTEN_EXTENT,
3216 	RANGE_BOUNDARY_PREALLOC_EXTENT,
3217 	RANGE_BOUNDARY_HOLE,
3218 };
3219 
btrfs_zero_range_check_range_boundary(struct btrfs_inode * inode,u64 offset)3220 static int btrfs_zero_range_check_range_boundary(struct btrfs_inode *inode,
3221 						 u64 offset)
3222 {
3223 	const u64 sectorsize = btrfs_inode_sectorsize(inode);
3224 	struct extent_map *em;
3225 	int ret;
3226 
3227 	offset = round_down(offset, sectorsize);
3228 	em = btrfs_get_extent(inode, NULL, 0, offset, sectorsize);
3229 	if (IS_ERR(em))
3230 		return PTR_ERR(em);
3231 
3232 	if (em->block_start == EXTENT_MAP_HOLE)
3233 		ret = RANGE_BOUNDARY_HOLE;
3234 	else if (test_bit(EXTENT_FLAG_PREALLOC, &em->flags))
3235 		ret = RANGE_BOUNDARY_PREALLOC_EXTENT;
3236 	else
3237 		ret = RANGE_BOUNDARY_WRITTEN_EXTENT;
3238 
3239 	free_extent_map(em);
3240 	return ret;
3241 }
3242 
btrfs_zero_range(struct inode * inode,loff_t offset,loff_t len,const int mode)3243 static int btrfs_zero_range(struct inode *inode,
3244 			    loff_t offset,
3245 			    loff_t len,
3246 			    const int mode)
3247 {
3248 	struct btrfs_fs_info *fs_info = BTRFS_I(inode)->root->fs_info;
3249 	struct extent_map *em;
3250 	struct extent_changeset *data_reserved = NULL;
3251 	int ret;
3252 	u64 alloc_hint = 0;
3253 	const u64 sectorsize = btrfs_inode_sectorsize(BTRFS_I(inode));
3254 	u64 alloc_start = round_down(offset, sectorsize);
3255 	u64 alloc_end = round_up(offset + len, sectorsize);
3256 	u64 bytes_to_reserve = 0;
3257 	bool space_reserved = false;
3258 
3259 	inode_dio_wait(inode);
3260 
3261 	em = btrfs_get_extent(BTRFS_I(inode), NULL, 0, alloc_start,
3262 			      alloc_end - alloc_start);
3263 	if (IS_ERR(em)) {
3264 		ret = PTR_ERR(em);
3265 		goto out;
3266 	}
3267 
3268 	/*
3269 	 * Avoid hole punching and extent allocation for some cases. More cases
3270 	 * could be considered, but these are unlikely common and we keep things
3271 	 * as simple as possible for now. Also, intentionally, if the target
3272 	 * range contains one or more prealloc extents together with regular
3273 	 * extents and holes, we drop all the existing extents and allocate a
3274 	 * new prealloc extent, so that we get a larger contiguous disk extent.
3275 	 */
3276 	if (em->start <= alloc_start &&
3277 	    test_bit(EXTENT_FLAG_PREALLOC, &em->flags)) {
3278 		const u64 em_end = em->start + em->len;
3279 
3280 		if (em_end >= offset + len) {
3281 			/*
3282 			 * The whole range is already a prealloc extent,
3283 			 * do nothing except updating the inode's i_size if
3284 			 * needed.
3285 			 */
3286 			free_extent_map(em);
3287 			ret = btrfs_fallocate_update_isize(inode, offset + len,
3288 							   mode);
3289 			goto out;
3290 		}
3291 		/*
3292 		 * Part of the range is already a prealloc extent, so operate
3293 		 * only on the remaining part of the range.
3294 		 */
3295 		alloc_start = em_end;
3296 		ASSERT(IS_ALIGNED(alloc_start, sectorsize));
3297 		len = offset + len - alloc_start;
3298 		offset = alloc_start;
3299 		alloc_hint = em->block_start + em->len;
3300 	}
3301 	free_extent_map(em);
3302 
3303 	if (BTRFS_BYTES_TO_BLKS(fs_info, offset) ==
3304 	    BTRFS_BYTES_TO_BLKS(fs_info, offset + len - 1)) {
3305 		em = btrfs_get_extent(BTRFS_I(inode), NULL, 0, alloc_start,
3306 				      sectorsize);
3307 		if (IS_ERR(em)) {
3308 			ret = PTR_ERR(em);
3309 			goto out;
3310 		}
3311 
3312 		if (test_bit(EXTENT_FLAG_PREALLOC, &em->flags)) {
3313 			free_extent_map(em);
3314 			ret = btrfs_fallocate_update_isize(inode, offset + len,
3315 							   mode);
3316 			goto out;
3317 		}
3318 		if (len < sectorsize && em->block_start != EXTENT_MAP_HOLE) {
3319 			free_extent_map(em);
3320 			ret = btrfs_truncate_block(BTRFS_I(inode), offset, len,
3321 						   0);
3322 			if (!ret)
3323 				ret = btrfs_fallocate_update_isize(inode,
3324 								   offset + len,
3325 								   mode);
3326 			return ret;
3327 		}
3328 		free_extent_map(em);
3329 		alloc_start = round_down(offset, sectorsize);
3330 		alloc_end = alloc_start + sectorsize;
3331 		goto reserve_space;
3332 	}
3333 
3334 	alloc_start = round_up(offset, sectorsize);
3335 	alloc_end = round_down(offset + len, sectorsize);
3336 
3337 	/*
3338 	 * For unaligned ranges, check the pages at the boundaries, they might
3339 	 * map to an extent, in which case we need to partially zero them, or
3340 	 * they might map to a hole, in which case we need our allocation range
3341 	 * to cover them.
3342 	 */
3343 	if (!IS_ALIGNED(offset, sectorsize)) {
3344 		ret = btrfs_zero_range_check_range_boundary(BTRFS_I(inode),
3345 							    offset);
3346 		if (ret < 0)
3347 			goto out;
3348 		if (ret == RANGE_BOUNDARY_HOLE) {
3349 			alloc_start = round_down(offset, sectorsize);
3350 			ret = 0;
3351 		} else if (ret == RANGE_BOUNDARY_WRITTEN_EXTENT) {
3352 			ret = btrfs_truncate_block(BTRFS_I(inode), offset, 0, 0);
3353 			if (ret)
3354 				goto out;
3355 		} else {
3356 			ret = 0;
3357 		}
3358 	}
3359 
3360 	if (!IS_ALIGNED(offset + len, sectorsize)) {
3361 		ret = btrfs_zero_range_check_range_boundary(BTRFS_I(inode),
3362 							    offset + len);
3363 		if (ret < 0)
3364 			goto out;
3365 		if (ret == RANGE_BOUNDARY_HOLE) {
3366 			alloc_end = round_up(offset + len, sectorsize);
3367 			ret = 0;
3368 		} else if (ret == RANGE_BOUNDARY_WRITTEN_EXTENT) {
3369 			ret = btrfs_truncate_block(BTRFS_I(inode), offset + len,
3370 						   0, 1);
3371 			if (ret)
3372 				goto out;
3373 		} else {
3374 			ret = 0;
3375 		}
3376 	}
3377 
3378 reserve_space:
3379 	if (alloc_start < alloc_end) {
3380 		struct extent_state *cached_state = NULL;
3381 		const u64 lockstart = alloc_start;
3382 		const u64 lockend = alloc_end - 1;
3383 
3384 		bytes_to_reserve = alloc_end - alloc_start;
3385 		ret = btrfs_alloc_data_chunk_ondemand(BTRFS_I(inode),
3386 						      bytes_to_reserve);
3387 		if (ret < 0)
3388 			goto out;
3389 		space_reserved = true;
3390 		ret = btrfs_punch_hole_lock_range(inode, lockstart, lockend,
3391 						  &cached_state);
3392 		if (ret)
3393 			goto out;
3394 		ret = btrfs_qgroup_reserve_data(BTRFS_I(inode), &data_reserved,
3395 						alloc_start, bytes_to_reserve);
3396 		if (ret) {
3397 			unlock_extent_cached(&BTRFS_I(inode)->io_tree, lockstart,
3398 					     lockend, &cached_state);
3399 			goto out;
3400 		}
3401 		ret = btrfs_prealloc_file_range(inode, mode, alloc_start,
3402 						alloc_end - alloc_start,
3403 						i_blocksize(inode),
3404 						offset + len, &alloc_hint);
3405 		unlock_extent_cached(&BTRFS_I(inode)->io_tree, lockstart,
3406 				     lockend, &cached_state);
3407 		/* btrfs_prealloc_file_range releases reserved space on error */
3408 		if (ret) {
3409 			space_reserved = false;
3410 			goto out;
3411 		}
3412 	}
3413 	ret = btrfs_fallocate_update_isize(inode, offset + len, mode);
3414  out:
3415 	if (ret && space_reserved)
3416 		btrfs_free_reserved_data_space(BTRFS_I(inode), data_reserved,
3417 					       alloc_start, bytes_to_reserve);
3418 	extent_changeset_free(data_reserved);
3419 
3420 	return ret;
3421 }
3422 
btrfs_fallocate(struct file * file,int mode,loff_t offset,loff_t len)3423 static long btrfs_fallocate(struct file *file, int mode,
3424 			    loff_t offset, loff_t len)
3425 {
3426 	struct inode *inode = file_inode(file);
3427 	struct extent_state *cached_state = NULL;
3428 	struct extent_changeset *data_reserved = NULL;
3429 	struct falloc_range *range;
3430 	struct falloc_range *tmp;
3431 	struct list_head reserve_list;
3432 	u64 cur_offset;
3433 	u64 last_byte;
3434 	u64 alloc_start;
3435 	u64 alloc_end;
3436 	u64 alloc_hint = 0;
3437 	u64 locked_end;
3438 	u64 actual_end = 0;
3439 	struct extent_map *em;
3440 	int blocksize = btrfs_inode_sectorsize(BTRFS_I(inode));
3441 	int ret;
3442 
3443 	/* Do not allow fallocate in ZONED mode */
3444 	if (btrfs_is_zoned(btrfs_sb(inode->i_sb)))
3445 		return -EOPNOTSUPP;
3446 
3447 	alloc_start = round_down(offset, blocksize);
3448 	alloc_end = round_up(offset + len, blocksize);
3449 	cur_offset = alloc_start;
3450 
3451 	/* Make sure we aren't being give some crap mode */
3452 	if (mode & ~(FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE |
3453 		     FALLOC_FL_ZERO_RANGE))
3454 		return -EOPNOTSUPP;
3455 
3456 	if (mode & FALLOC_FL_PUNCH_HOLE)
3457 		return btrfs_punch_hole(file, offset, len);
3458 
3459 	/*
3460 	 * Only trigger disk allocation, don't trigger qgroup reserve
3461 	 *
3462 	 * For qgroup space, it will be checked later.
3463 	 */
3464 	if (!(mode & FALLOC_FL_ZERO_RANGE)) {
3465 		ret = btrfs_alloc_data_chunk_ondemand(BTRFS_I(inode),
3466 						      alloc_end - alloc_start);
3467 		if (ret < 0)
3468 			return ret;
3469 	}
3470 
3471 	btrfs_inode_lock(inode, BTRFS_ILOCK_MMAP);
3472 
3473 	if (!(mode & FALLOC_FL_KEEP_SIZE) && offset + len > inode->i_size) {
3474 		ret = inode_newsize_ok(inode, offset + len);
3475 		if (ret)
3476 			goto out;
3477 	}
3478 
3479 	ret = file_modified(file);
3480 	if (ret)
3481 		goto out;
3482 
3483 	/*
3484 	 * TODO: Move these two operations after we have checked
3485 	 * accurate reserved space, or fallocate can still fail but
3486 	 * with page truncated or size expanded.
3487 	 *
3488 	 * But that's a minor problem and won't do much harm BTW.
3489 	 */
3490 	if (alloc_start > inode->i_size) {
3491 		ret = btrfs_cont_expand(BTRFS_I(inode), i_size_read(inode),
3492 					alloc_start);
3493 		if (ret)
3494 			goto out;
3495 	} else if (offset + len > inode->i_size) {
3496 		/*
3497 		 * If we are fallocating from the end of the file onward we
3498 		 * need to zero out the end of the block if i_size lands in the
3499 		 * middle of a block.
3500 		 */
3501 		ret = btrfs_truncate_block(BTRFS_I(inode), inode->i_size, 0, 0);
3502 		if (ret)
3503 			goto out;
3504 	}
3505 
3506 	/*
3507 	 * wait for ordered IO before we have any locks.  We'll loop again
3508 	 * below with the locks held.
3509 	 */
3510 	ret = btrfs_wait_ordered_range(inode, alloc_start,
3511 				       alloc_end - alloc_start);
3512 	if (ret)
3513 		goto out;
3514 
3515 	if (mode & FALLOC_FL_ZERO_RANGE) {
3516 		ret = btrfs_zero_range(inode, offset, len, mode);
3517 		btrfs_inode_unlock(inode, BTRFS_ILOCK_MMAP);
3518 		return ret;
3519 	}
3520 
3521 	locked_end = alloc_end - 1;
3522 	while (1) {
3523 		struct btrfs_ordered_extent *ordered;
3524 
3525 		/* the extent lock is ordered inside the running
3526 		 * transaction
3527 		 */
3528 		lock_extent_bits(&BTRFS_I(inode)->io_tree, alloc_start,
3529 				 locked_end, &cached_state);
3530 		ordered = btrfs_lookup_first_ordered_extent(BTRFS_I(inode),
3531 							    locked_end);
3532 
3533 		if (ordered &&
3534 		    ordered->file_offset + ordered->num_bytes > alloc_start &&
3535 		    ordered->file_offset < alloc_end) {
3536 			btrfs_put_ordered_extent(ordered);
3537 			unlock_extent_cached(&BTRFS_I(inode)->io_tree,
3538 					     alloc_start, locked_end,
3539 					     &cached_state);
3540 			/*
3541 			 * we can't wait on the range with the transaction
3542 			 * running or with the extent lock held
3543 			 */
3544 			ret = btrfs_wait_ordered_range(inode, alloc_start,
3545 						       alloc_end - alloc_start);
3546 			if (ret)
3547 				goto out;
3548 		} else {
3549 			if (ordered)
3550 				btrfs_put_ordered_extent(ordered);
3551 			break;
3552 		}
3553 	}
3554 
3555 	/* First, check if we exceed the qgroup limit */
3556 	INIT_LIST_HEAD(&reserve_list);
3557 	while (cur_offset < alloc_end) {
3558 		em = btrfs_get_extent(BTRFS_I(inode), NULL, 0, cur_offset,
3559 				      alloc_end - cur_offset);
3560 		if (IS_ERR(em)) {
3561 			ret = PTR_ERR(em);
3562 			break;
3563 		}
3564 		last_byte = min(extent_map_end(em), alloc_end);
3565 		actual_end = min_t(u64, extent_map_end(em), offset + len);
3566 		last_byte = ALIGN(last_byte, blocksize);
3567 		if (em->block_start == EXTENT_MAP_HOLE ||
3568 		    (cur_offset >= inode->i_size &&
3569 		     !test_bit(EXTENT_FLAG_PREALLOC, &em->flags))) {
3570 			ret = add_falloc_range(&reserve_list, cur_offset,
3571 					       last_byte - cur_offset);
3572 			if (ret < 0) {
3573 				free_extent_map(em);
3574 				break;
3575 			}
3576 			ret = btrfs_qgroup_reserve_data(BTRFS_I(inode),
3577 					&data_reserved, cur_offset,
3578 					last_byte - cur_offset);
3579 			if (ret < 0) {
3580 				cur_offset = last_byte;
3581 				free_extent_map(em);
3582 				break;
3583 			}
3584 		} else {
3585 			/*
3586 			 * Do not need to reserve unwritten extent for this
3587 			 * range, free reserved data space first, otherwise
3588 			 * it'll result in false ENOSPC error.
3589 			 */
3590 			btrfs_free_reserved_data_space(BTRFS_I(inode),
3591 				data_reserved, cur_offset,
3592 				last_byte - cur_offset);
3593 		}
3594 		free_extent_map(em);
3595 		cur_offset = last_byte;
3596 	}
3597 
3598 	/*
3599 	 * If ret is still 0, means we're OK to fallocate.
3600 	 * Or just cleanup the list and exit.
3601 	 */
3602 	list_for_each_entry_safe(range, tmp, &reserve_list, list) {
3603 		if (!ret)
3604 			ret = btrfs_prealloc_file_range(inode, mode,
3605 					range->start,
3606 					range->len, i_blocksize(inode),
3607 					offset + len, &alloc_hint);
3608 		else
3609 			btrfs_free_reserved_data_space(BTRFS_I(inode),
3610 					data_reserved, range->start,
3611 					range->len);
3612 		list_del(&range->list);
3613 		kfree(range);
3614 	}
3615 	if (ret < 0)
3616 		goto out_unlock;
3617 
3618 	/*
3619 	 * We didn't need to allocate any more space, but we still extended the
3620 	 * size of the file so we need to update i_size and the inode item.
3621 	 */
3622 	ret = btrfs_fallocate_update_isize(inode, actual_end, mode);
3623 out_unlock:
3624 	unlock_extent_cached(&BTRFS_I(inode)->io_tree, alloc_start, locked_end,
3625 			     &cached_state);
3626 out:
3627 	btrfs_inode_unlock(inode, BTRFS_ILOCK_MMAP);
3628 	/* Let go of our reservation. */
3629 	if (ret != 0 && !(mode & FALLOC_FL_ZERO_RANGE))
3630 		btrfs_free_reserved_data_space(BTRFS_I(inode), data_reserved,
3631 				cur_offset, alloc_end - cur_offset);
3632 	extent_changeset_free(data_reserved);
3633 	return ret;
3634 }
3635 
find_desired_extent(struct btrfs_inode * inode,loff_t offset,int whence)3636 static loff_t find_desired_extent(struct btrfs_inode *inode, loff_t offset,
3637 				  int whence)
3638 {
3639 	struct btrfs_fs_info *fs_info = inode->root->fs_info;
3640 	struct extent_map *em = NULL;
3641 	struct extent_state *cached_state = NULL;
3642 	loff_t i_size = inode->vfs_inode.i_size;
3643 	u64 lockstart;
3644 	u64 lockend;
3645 	u64 start;
3646 	u64 len;
3647 	int ret = 0;
3648 
3649 	if (i_size == 0 || offset >= i_size)
3650 		return -ENXIO;
3651 
3652 	/*
3653 	 * offset can be negative, in this case we start finding DATA/HOLE from
3654 	 * the very start of the file.
3655 	 */
3656 	start = max_t(loff_t, 0, offset);
3657 
3658 	lockstart = round_down(start, fs_info->sectorsize);
3659 	lockend = round_up(i_size, fs_info->sectorsize);
3660 	if (lockend <= lockstart)
3661 		lockend = lockstart + fs_info->sectorsize;
3662 	lockend--;
3663 	len = lockend - lockstart + 1;
3664 
3665 	lock_extent_bits(&inode->io_tree, lockstart, lockend, &cached_state);
3666 
3667 	while (start < i_size) {
3668 		em = btrfs_get_extent_fiemap(inode, start, len);
3669 		if (IS_ERR(em)) {
3670 			ret = PTR_ERR(em);
3671 			em = NULL;
3672 			break;
3673 		}
3674 
3675 		if (whence == SEEK_HOLE &&
3676 		    (em->block_start == EXTENT_MAP_HOLE ||
3677 		     test_bit(EXTENT_FLAG_PREALLOC, &em->flags)))
3678 			break;
3679 		else if (whence == SEEK_DATA &&
3680 			   (em->block_start != EXTENT_MAP_HOLE &&
3681 			    !test_bit(EXTENT_FLAG_PREALLOC, &em->flags)))
3682 			break;
3683 
3684 		start = em->start + em->len;
3685 		free_extent_map(em);
3686 		em = NULL;
3687 		cond_resched();
3688 	}
3689 	free_extent_map(em);
3690 	unlock_extent_cached(&inode->io_tree, lockstart, lockend,
3691 			     &cached_state);
3692 	if (ret) {
3693 		offset = ret;
3694 	} else {
3695 		if (whence == SEEK_DATA && start >= i_size)
3696 			offset = -ENXIO;
3697 		else
3698 			offset = min_t(loff_t, start, i_size);
3699 	}
3700 
3701 	return offset;
3702 }
3703 
btrfs_file_llseek(struct file * file,loff_t offset,int whence)3704 static loff_t btrfs_file_llseek(struct file *file, loff_t offset, int whence)
3705 {
3706 	struct inode *inode = file->f_mapping->host;
3707 
3708 	switch (whence) {
3709 	default:
3710 		return generic_file_llseek(file, offset, whence);
3711 	case SEEK_DATA:
3712 	case SEEK_HOLE:
3713 		btrfs_inode_lock(inode, BTRFS_ILOCK_SHARED);
3714 		offset = find_desired_extent(BTRFS_I(inode), offset, whence);
3715 		btrfs_inode_unlock(inode, BTRFS_ILOCK_SHARED);
3716 		break;
3717 	}
3718 
3719 	if (offset < 0)
3720 		return offset;
3721 
3722 	return vfs_setpos(file, offset, inode->i_sb->s_maxbytes);
3723 }
3724 
btrfs_file_open(struct inode * inode,struct file * filp)3725 static int btrfs_file_open(struct inode *inode, struct file *filp)
3726 {
3727 	int ret;
3728 
3729 	filp->f_mode |= FMODE_NOWAIT | FMODE_BUF_RASYNC;
3730 
3731 	ret = fsverity_file_open(inode, filp);
3732 	if (ret)
3733 		return ret;
3734 	return generic_file_open(inode, filp);
3735 }
3736 
check_direct_read(struct btrfs_fs_info * fs_info,const struct iov_iter * iter,loff_t offset)3737 static int check_direct_read(struct btrfs_fs_info *fs_info,
3738 			     const struct iov_iter *iter, loff_t offset)
3739 {
3740 	int ret;
3741 	int i, seg;
3742 
3743 	ret = check_direct_IO(fs_info, iter, offset);
3744 	if (ret < 0)
3745 		return ret;
3746 
3747 	if (!iter_is_iovec(iter))
3748 		return 0;
3749 
3750 	for (seg = 0; seg < iter->nr_segs; seg++)
3751 		for (i = seg + 1; i < iter->nr_segs; i++)
3752 			if (iter->iov[seg].iov_base == iter->iov[i].iov_base)
3753 				return -EINVAL;
3754 	return 0;
3755 }
3756 
btrfs_direct_read(struct kiocb * iocb,struct iov_iter * to)3757 static ssize_t btrfs_direct_read(struct kiocb *iocb, struct iov_iter *to)
3758 {
3759 	struct inode *inode = file_inode(iocb->ki_filp);
3760 	size_t prev_left = 0;
3761 	ssize_t read = 0;
3762 	ssize_t ret;
3763 
3764 	if (fsverity_active(inode))
3765 		return 0;
3766 
3767 	if (check_direct_read(btrfs_sb(inode->i_sb), to, iocb->ki_pos))
3768 		return 0;
3769 
3770 	btrfs_inode_lock(inode, BTRFS_ILOCK_SHARED);
3771 again:
3772 	/*
3773 	 * This is similar to what we do for direct IO writes, see the comment
3774 	 * at btrfs_direct_write(), but we also disable page faults in addition
3775 	 * to disabling them only at the iov_iter level. This is because when
3776 	 * reading from a hole or prealloc extent, iomap calls iov_iter_zero(),
3777 	 * which can still trigger page fault ins despite having set ->nofault
3778 	 * to true of our 'to' iov_iter.
3779 	 *
3780 	 * The difference to direct IO writes is that we deadlock when trying
3781 	 * to lock the extent range in the inode's tree during he page reads
3782 	 * triggered by the fault in (while for writes it is due to waiting for
3783 	 * our own ordered extent). This is because for direct IO reads,
3784 	 * btrfs_dio_iomap_begin() returns with the extent range locked, which
3785 	 * is only unlocked in the endio callback (end_bio_extent_readpage()).
3786 	 */
3787 	pagefault_disable();
3788 	to->nofault = true;
3789 	ret = iomap_dio_rw(iocb, to, &btrfs_dio_iomap_ops, &btrfs_dio_ops,
3790 			   IOMAP_DIO_PARTIAL, read);
3791 	to->nofault = false;
3792 	pagefault_enable();
3793 
3794 	/* No increment (+=) because iomap returns a cumulative value. */
3795 	if (ret > 0)
3796 		read = ret;
3797 
3798 	if (iov_iter_count(to) > 0 && (ret == -EFAULT || ret > 0)) {
3799 		const size_t left = iov_iter_count(to);
3800 
3801 		if (left == prev_left) {
3802 			/*
3803 			 * We didn't make any progress since the last attempt,
3804 			 * fallback to a buffered read for the remainder of the
3805 			 * range. This is just to avoid any possibility of looping
3806 			 * for too long.
3807 			 */
3808 			ret = read;
3809 		} else {
3810 			/*
3811 			 * We made some progress since the last retry or this is
3812 			 * the first time we are retrying. Fault in as many pages
3813 			 * as possible and retry.
3814 			 */
3815 			fault_in_iov_iter_writeable(to, left);
3816 			prev_left = left;
3817 			goto again;
3818 		}
3819 	}
3820 	btrfs_inode_unlock(inode, BTRFS_ILOCK_SHARED);
3821 	return ret < 0 ? ret : read;
3822 }
3823 
btrfs_file_read_iter(struct kiocb * iocb,struct iov_iter * to)3824 static ssize_t btrfs_file_read_iter(struct kiocb *iocb, struct iov_iter *to)
3825 {
3826 	ssize_t ret = 0;
3827 
3828 	if (iocb->ki_flags & IOCB_DIRECT) {
3829 		ret = btrfs_direct_read(iocb, to);
3830 		if (ret < 0 || !iov_iter_count(to) ||
3831 		    iocb->ki_pos >= i_size_read(file_inode(iocb->ki_filp)))
3832 			return ret;
3833 	}
3834 
3835 	return filemap_read(iocb, to, ret);
3836 }
3837 
3838 const struct file_operations btrfs_file_operations = {
3839 	.llseek		= btrfs_file_llseek,
3840 	.read_iter      = btrfs_file_read_iter,
3841 	.splice_read	= generic_file_splice_read,
3842 	.write_iter	= btrfs_file_write_iter,
3843 	.splice_write	= iter_file_splice_write,
3844 	.mmap		= btrfs_file_mmap,
3845 	.open		= btrfs_file_open,
3846 	.release	= btrfs_release_file,
3847 	.fsync		= btrfs_sync_file,
3848 	.fallocate	= btrfs_fallocate,
3849 	.unlocked_ioctl	= btrfs_ioctl,
3850 #ifdef CONFIG_COMPAT
3851 	.compat_ioctl	= btrfs_compat_ioctl,
3852 #endif
3853 	.remap_file_range = btrfs_remap_file_range,
3854 };
3855 
btrfs_auto_defrag_exit(void)3856 void __cold btrfs_auto_defrag_exit(void)
3857 {
3858 	kmem_cache_destroy(btrfs_inode_defrag_cachep);
3859 }
3860 
btrfs_auto_defrag_init(void)3861 int __init btrfs_auto_defrag_init(void)
3862 {
3863 	btrfs_inode_defrag_cachep = kmem_cache_create("btrfs_inode_defrag",
3864 					sizeof(struct inode_defrag), 0,
3865 					SLAB_MEM_SPREAD,
3866 					NULL);
3867 	if (!btrfs_inode_defrag_cachep)
3868 		return -ENOMEM;
3869 
3870 	return 0;
3871 }
3872 
btrfs_fdatawrite_range(struct inode * inode,loff_t start,loff_t end)3873 int btrfs_fdatawrite_range(struct inode *inode, loff_t start, loff_t end)
3874 {
3875 	int ret;
3876 
3877 	/*
3878 	 * So with compression we will find and lock a dirty page and clear the
3879 	 * first one as dirty, setup an async extent, and immediately return
3880 	 * with the entire range locked but with nobody actually marked with
3881 	 * writeback.  So we can't just filemap_write_and_wait_range() and
3882 	 * expect it to work since it will just kick off a thread to do the
3883 	 * actual work.  So we need to call filemap_fdatawrite_range _again_
3884 	 * since it will wait on the page lock, which won't be unlocked until
3885 	 * after the pages have been marked as writeback and so we're good to go
3886 	 * from there.  We have to do this otherwise we'll miss the ordered
3887 	 * extents and that results in badness.  Please Josef, do not think you
3888 	 * know better and pull this out at some point in the future, it is
3889 	 * right and you are wrong.
3890 	 */
3891 	ret = filemap_fdatawrite_range(inode->i_mapping, start, end);
3892 	if (!ret && test_bit(BTRFS_INODE_HAS_ASYNC_EXTENT,
3893 			     &BTRFS_I(inode)->runtime_flags))
3894 		ret = filemap_fdatawrite_range(inode->i_mapping, start, end);
3895 
3896 	return ret;
3897 }
3898