• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 /*
2  * Copyright (C) 2016 Oracle.  All Rights Reserved.
3  *
4  * Author: Darrick J. Wong <darrick.wong@oracle.com>
5  *
6  * This program is free software; you can redistribute it and/or
7  * modify it under the terms of the GNU General Public License
8  * as published by the Free Software Foundation; either version 2
9  * of the License, or (at your option) any later version.
10  *
11  * This program is distributed in the hope that it would be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
14  * GNU General Public License for more details.
15  *
16  * You should have received a copy of the GNU General Public License
17  * along with this program; if not, write the Free Software Foundation,
18  * Inc.,  51 Franklin St, Fifth Floor, Boston, MA  02110-1301, USA.
19  */
20 #include "xfs.h"
21 #include "xfs_fs.h"
22 #include "xfs_shared.h"
23 #include "xfs_format.h"
24 #include "xfs_log_format.h"
25 #include "xfs_trans_resv.h"
26 #include "xfs_sb.h"
27 #include "xfs_mount.h"
28 #include "xfs_defer.h"
29 #include "xfs_btree.h"
30 #include "xfs_bmap.h"
31 #include "xfs_refcount_btree.h"
32 #include "xfs_alloc.h"
33 #include "xfs_error.h"
34 #include "xfs_trace.h"
35 #include "xfs_cksum.h"
36 #include "xfs_trans.h"
37 #include "xfs_bit.h"
38 #include "xfs_refcount.h"
39 #include "xfs_rmap.h"
40 
41 /* Allowable refcount adjustment amounts. */
42 enum xfs_refc_adjust_op {
43 	XFS_REFCOUNT_ADJUST_INCREASE	= 1,
44 	XFS_REFCOUNT_ADJUST_DECREASE	= -1,
45 	XFS_REFCOUNT_ADJUST_COW_ALLOC	= 0,
46 	XFS_REFCOUNT_ADJUST_COW_FREE	= -1,
47 };
48 
49 STATIC int __xfs_refcount_cow_alloc(struct xfs_btree_cur *rcur,
50 		xfs_agblock_t agbno, xfs_extlen_t aglen,
51 		struct xfs_defer_ops *dfops);
52 STATIC int __xfs_refcount_cow_free(struct xfs_btree_cur *rcur,
53 		xfs_agblock_t agbno, xfs_extlen_t aglen,
54 		struct xfs_defer_ops *dfops);
55 
56 /*
57  * Look up the first record less than or equal to [bno, len] in the btree
58  * given by cur.
59  */
60 int
xfs_refcount_lookup_le(struct xfs_btree_cur * cur,xfs_agblock_t bno,int * stat)61 xfs_refcount_lookup_le(
62 	struct xfs_btree_cur	*cur,
63 	xfs_agblock_t		bno,
64 	int			*stat)
65 {
66 	trace_xfs_refcount_lookup(cur->bc_mp, cur->bc_private.a.agno, bno,
67 			XFS_LOOKUP_LE);
68 	cur->bc_rec.rc.rc_startblock = bno;
69 	cur->bc_rec.rc.rc_blockcount = 0;
70 	return xfs_btree_lookup(cur, XFS_LOOKUP_LE, stat);
71 }
72 
73 /*
74  * Look up the first record greater than or equal to [bno, len] in the btree
75  * given by cur.
76  */
77 int
xfs_refcount_lookup_ge(struct xfs_btree_cur * cur,xfs_agblock_t bno,int * stat)78 xfs_refcount_lookup_ge(
79 	struct xfs_btree_cur	*cur,
80 	xfs_agblock_t		bno,
81 	int			*stat)
82 {
83 	trace_xfs_refcount_lookup(cur->bc_mp, cur->bc_private.a.agno, bno,
84 			XFS_LOOKUP_GE);
85 	cur->bc_rec.rc.rc_startblock = bno;
86 	cur->bc_rec.rc.rc_blockcount = 0;
87 	return xfs_btree_lookup(cur, XFS_LOOKUP_GE, stat);
88 }
89 
90 /* Convert on-disk record to in-core format. */
91 static inline void
xfs_refcount_btrec_to_irec(union xfs_btree_rec * rec,struct xfs_refcount_irec * irec)92 xfs_refcount_btrec_to_irec(
93 	union xfs_btree_rec		*rec,
94 	struct xfs_refcount_irec	*irec)
95 {
96 	irec->rc_startblock = be32_to_cpu(rec->refc.rc_startblock);
97 	irec->rc_blockcount = be32_to_cpu(rec->refc.rc_blockcount);
98 	irec->rc_refcount = be32_to_cpu(rec->refc.rc_refcount);
99 }
100 
101 /*
102  * Get the data from the pointed-to record.
103  */
104 int
xfs_refcount_get_rec(struct xfs_btree_cur * cur,struct xfs_refcount_irec * irec,int * stat)105 xfs_refcount_get_rec(
106 	struct xfs_btree_cur		*cur,
107 	struct xfs_refcount_irec	*irec,
108 	int				*stat)
109 {
110 	union xfs_btree_rec		*rec;
111 	int				error;
112 
113 	error = xfs_btree_get_rec(cur, &rec, stat);
114 	if (!error && *stat == 1) {
115 		xfs_refcount_btrec_to_irec(rec, irec);
116 		trace_xfs_refcount_get(cur->bc_mp, cur->bc_private.a.agno,
117 				irec);
118 	}
119 	return error;
120 }
121 
122 /*
123  * Update the record referred to by cur to the value given
124  * by [bno, len, refcount].
125  * This either works (return 0) or gets an EFSCORRUPTED error.
126  */
127 STATIC int
xfs_refcount_update(struct xfs_btree_cur * cur,struct xfs_refcount_irec * irec)128 xfs_refcount_update(
129 	struct xfs_btree_cur		*cur,
130 	struct xfs_refcount_irec	*irec)
131 {
132 	union xfs_btree_rec	rec;
133 	int			error;
134 
135 	trace_xfs_refcount_update(cur->bc_mp, cur->bc_private.a.agno, irec);
136 	rec.refc.rc_startblock = cpu_to_be32(irec->rc_startblock);
137 	rec.refc.rc_blockcount = cpu_to_be32(irec->rc_blockcount);
138 	rec.refc.rc_refcount = cpu_to_be32(irec->rc_refcount);
139 	error = xfs_btree_update(cur, &rec);
140 	if (error)
141 		trace_xfs_refcount_update_error(cur->bc_mp,
142 				cur->bc_private.a.agno, error, _RET_IP_);
143 	return error;
144 }
145 
146 /*
147  * Insert the record referred to by cur to the value given
148  * by [bno, len, refcount].
149  * This either works (return 0) or gets an EFSCORRUPTED error.
150  */
151 STATIC int
xfs_refcount_insert(struct xfs_btree_cur * cur,struct xfs_refcount_irec * irec,int * i)152 xfs_refcount_insert(
153 	struct xfs_btree_cur		*cur,
154 	struct xfs_refcount_irec	*irec,
155 	int				*i)
156 {
157 	int				error;
158 
159 	trace_xfs_refcount_insert(cur->bc_mp, cur->bc_private.a.agno, irec);
160 	cur->bc_rec.rc.rc_startblock = irec->rc_startblock;
161 	cur->bc_rec.rc.rc_blockcount = irec->rc_blockcount;
162 	cur->bc_rec.rc.rc_refcount = irec->rc_refcount;
163 	error = xfs_btree_insert(cur, i);
164 	XFS_WANT_CORRUPTED_GOTO(cur->bc_mp, *i == 1, out_error);
165 out_error:
166 	if (error)
167 		trace_xfs_refcount_insert_error(cur->bc_mp,
168 				cur->bc_private.a.agno, error, _RET_IP_);
169 	return error;
170 }
171 
172 /*
173  * Remove the record referred to by cur, then set the pointer to the spot
174  * where the record could be re-inserted, in case we want to increment or
175  * decrement the cursor.
176  * This either works (return 0) or gets an EFSCORRUPTED error.
177  */
178 STATIC int
xfs_refcount_delete(struct xfs_btree_cur * cur,int * i)179 xfs_refcount_delete(
180 	struct xfs_btree_cur	*cur,
181 	int			*i)
182 {
183 	struct xfs_refcount_irec	irec;
184 	int			found_rec;
185 	int			error;
186 
187 	error = xfs_refcount_get_rec(cur, &irec, &found_rec);
188 	if (error)
189 		goto out_error;
190 	XFS_WANT_CORRUPTED_GOTO(cur->bc_mp, found_rec == 1, out_error);
191 	trace_xfs_refcount_delete(cur->bc_mp, cur->bc_private.a.agno, &irec);
192 	error = xfs_btree_delete(cur, i);
193 	XFS_WANT_CORRUPTED_GOTO(cur->bc_mp, *i == 1, out_error);
194 	if (error)
195 		goto out_error;
196 	error = xfs_refcount_lookup_ge(cur, irec.rc_startblock, &found_rec);
197 out_error:
198 	if (error)
199 		trace_xfs_refcount_delete_error(cur->bc_mp,
200 				cur->bc_private.a.agno, error, _RET_IP_);
201 	return error;
202 }
203 
204 /*
205  * Adjusting the Reference Count
206  *
207  * As stated elsewhere, the reference count btree (refcbt) stores
208  * >1 reference counts for extents of physical blocks.  In this
209  * operation, we're either raising or lowering the reference count of
210  * some subrange stored in the tree:
211  *
212  *      <------ adjustment range ------>
213  * ----+   +---+-----+ +--+--------+---------
214  *  2  |   | 3 |  4  | |17|   55   |   10
215  * ----+   +---+-----+ +--+--------+---------
216  * X axis is physical blocks number;
217  * reference counts are the numbers inside the rectangles
218  *
219  * The first thing we need to do is to ensure that there are no
220  * refcount extents crossing either boundary of the range to be
221  * adjusted.  For any extent that does cross a boundary, split it into
222  * two extents so that we can increment the refcount of one of the
223  * pieces later:
224  *
225  *      <------ adjustment range ------>
226  * ----+   +---+-----+ +--+--------+----+----
227  *  2  |   | 3 |  2  | |17|   55   | 10 | 10
228  * ----+   +---+-----+ +--+--------+----+----
229  *
230  * For this next step, let's assume that all the physical blocks in
231  * the adjustment range are mapped to a file and are therefore in use
232  * at least once.  Therefore, we can infer that any gap in the
233  * refcount tree within the adjustment range represents a physical
234  * extent with refcount == 1:
235  *
236  *      <------ adjustment range ------>
237  * ----+---+---+-----+-+--+--------+----+----
238  *  2  |"1"| 3 |  2  |1|17|   55   | 10 | 10
239  * ----+---+---+-----+-+--+--------+----+----
240  *      ^
241  *
242  * For each extent that falls within the interval range, figure out
243  * which extent is to the left or the right of that extent.  Now we
244  * have a left, current, and right extent.  If the new reference count
245  * of the center extent enables us to merge left, center, and right
246  * into one record covering all three, do so.  If the center extent is
247  * at the left end of the range, abuts the left extent, and its new
248  * reference count matches the left extent's record, then merge them.
249  * If the center extent is at the right end of the range, abuts the
250  * right extent, and the reference counts match, merge those.  In the
251  * example, we can left merge (assuming an increment operation):
252  *
253  *      <------ adjustment range ------>
254  * --------+---+-----+-+--+--------+----+----
255  *    2    | 3 |  2  |1|17|   55   | 10 | 10
256  * --------+---+-----+-+--+--------+----+----
257  *          ^
258  *
259  * For all other extents within the range, adjust the reference count
260  * or delete it if the refcount falls below 2.  If we were
261  * incrementing, the end result looks like this:
262  *
263  *      <------ adjustment range ------>
264  * --------+---+-----+-+--+--------+----+----
265  *    2    | 4 |  3  |2|18|   56   | 11 | 10
266  * --------+---+-----+-+--+--------+----+----
267  *
268  * The result of a decrement operation looks as such:
269  *
270  *      <------ adjustment range ------>
271  * ----+   +---+       +--+--------+----+----
272  *  2  |   | 2 |       |16|   54   |  9 | 10
273  * ----+   +---+       +--+--------+----+----
274  *      DDDD    111111DD
275  *
276  * The blocks marked "D" are freed; the blocks marked "1" are only
277  * referenced once and therefore the record is removed from the
278  * refcount btree.
279  */
280 
281 /* Next block after this extent. */
282 static inline xfs_agblock_t
xfs_refc_next(struct xfs_refcount_irec * rc)283 xfs_refc_next(
284 	struct xfs_refcount_irec	*rc)
285 {
286 	return rc->rc_startblock + rc->rc_blockcount;
287 }
288 
289 /*
290  * Split a refcount extent that crosses agbno.
291  */
292 STATIC int
xfs_refcount_split_extent(struct xfs_btree_cur * cur,xfs_agblock_t agbno,bool * shape_changed)293 xfs_refcount_split_extent(
294 	struct xfs_btree_cur		*cur,
295 	xfs_agblock_t			agbno,
296 	bool				*shape_changed)
297 {
298 	struct xfs_refcount_irec	rcext, tmp;
299 	int				found_rec;
300 	int				error;
301 
302 	*shape_changed = false;
303 	error = xfs_refcount_lookup_le(cur, agbno, &found_rec);
304 	if (error)
305 		goto out_error;
306 	if (!found_rec)
307 		return 0;
308 
309 	error = xfs_refcount_get_rec(cur, &rcext, &found_rec);
310 	if (error)
311 		goto out_error;
312 	XFS_WANT_CORRUPTED_GOTO(cur->bc_mp, found_rec == 1, out_error);
313 	if (rcext.rc_startblock == agbno || xfs_refc_next(&rcext) <= agbno)
314 		return 0;
315 
316 	*shape_changed = true;
317 	trace_xfs_refcount_split_extent(cur->bc_mp, cur->bc_private.a.agno,
318 			&rcext, agbno);
319 
320 	/* Establish the right extent. */
321 	tmp = rcext;
322 	tmp.rc_startblock = agbno;
323 	tmp.rc_blockcount -= (agbno - rcext.rc_startblock);
324 	error = xfs_refcount_update(cur, &tmp);
325 	if (error)
326 		goto out_error;
327 
328 	/* Insert the left extent. */
329 	tmp = rcext;
330 	tmp.rc_blockcount = agbno - rcext.rc_startblock;
331 	error = xfs_refcount_insert(cur, &tmp, &found_rec);
332 	if (error)
333 		goto out_error;
334 	XFS_WANT_CORRUPTED_GOTO(cur->bc_mp, found_rec == 1, out_error);
335 	return error;
336 
337 out_error:
338 	trace_xfs_refcount_split_extent_error(cur->bc_mp,
339 			cur->bc_private.a.agno, error, _RET_IP_);
340 	return error;
341 }
342 
343 /*
344  * Merge the left, center, and right extents.
345  */
346 STATIC int
xfs_refcount_merge_center_extents(struct xfs_btree_cur * cur,struct xfs_refcount_irec * left,struct xfs_refcount_irec * center,struct xfs_refcount_irec * right,unsigned long long extlen,xfs_agblock_t * agbno,xfs_extlen_t * aglen)347 xfs_refcount_merge_center_extents(
348 	struct xfs_btree_cur		*cur,
349 	struct xfs_refcount_irec	*left,
350 	struct xfs_refcount_irec	*center,
351 	struct xfs_refcount_irec	*right,
352 	unsigned long long		extlen,
353 	xfs_agblock_t			*agbno,
354 	xfs_extlen_t			*aglen)
355 {
356 	int				error;
357 	int				found_rec;
358 
359 	trace_xfs_refcount_merge_center_extents(cur->bc_mp,
360 			cur->bc_private.a.agno, left, center, right);
361 
362 	/*
363 	 * Make sure the center and right extents are not in the btree.
364 	 * If the center extent was synthesized, the first delete call
365 	 * removes the right extent and we skip the second deletion.
366 	 * If center and right were in the btree, then the first delete
367 	 * call removes the center and the second one removes the right
368 	 * extent.
369 	 */
370 	error = xfs_refcount_lookup_ge(cur, center->rc_startblock,
371 			&found_rec);
372 	if (error)
373 		goto out_error;
374 	XFS_WANT_CORRUPTED_GOTO(cur->bc_mp, found_rec == 1, out_error);
375 
376 	error = xfs_refcount_delete(cur, &found_rec);
377 	if (error)
378 		goto out_error;
379 	XFS_WANT_CORRUPTED_GOTO(cur->bc_mp, found_rec == 1, out_error);
380 
381 	if (center->rc_refcount > 1) {
382 		error = xfs_refcount_delete(cur, &found_rec);
383 		if (error)
384 			goto out_error;
385 		XFS_WANT_CORRUPTED_GOTO(cur->bc_mp, found_rec == 1,
386 				out_error);
387 	}
388 
389 	/* Enlarge the left extent. */
390 	error = xfs_refcount_lookup_le(cur, left->rc_startblock,
391 			&found_rec);
392 	if (error)
393 		goto out_error;
394 	XFS_WANT_CORRUPTED_GOTO(cur->bc_mp, found_rec == 1, out_error);
395 
396 	left->rc_blockcount = extlen;
397 	error = xfs_refcount_update(cur, left);
398 	if (error)
399 		goto out_error;
400 
401 	*aglen = 0;
402 	return error;
403 
404 out_error:
405 	trace_xfs_refcount_merge_center_extents_error(cur->bc_mp,
406 			cur->bc_private.a.agno, error, _RET_IP_);
407 	return error;
408 }
409 
410 /*
411  * Merge with the left extent.
412  */
413 STATIC int
xfs_refcount_merge_left_extent(struct xfs_btree_cur * cur,struct xfs_refcount_irec * left,struct xfs_refcount_irec * cleft,xfs_agblock_t * agbno,xfs_extlen_t * aglen)414 xfs_refcount_merge_left_extent(
415 	struct xfs_btree_cur		*cur,
416 	struct xfs_refcount_irec	*left,
417 	struct xfs_refcount_irec	*cleft,
418 	xfs_agblock_t			*agbno,
419 	xfs_extlen_t			*aglen)
420 {
421 	int				error;
422 	int				found_rec;
423 
424 	trace_xfs_refcount_merge_left_extent(cur->bc_mp,
425 			cur->bc_private.a.agno, left, cleft);
426 
427 	/* If the extent at agbno (cleft) wasn't synthesized, remove it. */
428 	if (cleft->rc_refcount > 1) {
429 		error = xfs_refcount_lookup_le(cur, cleft->rc_startblock,
430 				&found_rec);
431 		if (error)
432 			goto out_error;
433 		XFS_WANT_CORRUPTED_GOTO(cur->bc_mp, found_rec == 1,
434 				out_error);
435 
436 		error = xfs_refcount_delete(cur, &found_rec);
437 		if (error)
438 			goto out_error;
439 		XFS_WANT_CORRUPTED_GOTO(cur->bc_mp, found_rec == 1,
440 				out_error);
441 	}
442 
443 	/* Enlarge the left extent. */
444 	error = xfs_refcount_lookup_le(cur, left->rc_startblock,
445 			&found_rec);
446 	if (error)
447 		goto out_error;
448 	XFS_WANT_CORRUPTED_GOTO(cur->bc_mp, found_rec == 1, out_error);
449 
450 	left->rc_blockcount += cleft->rc_blockcount;
451 	error = xfs_refcount_update(cur, left);
452 	if (error)
453 		goto out_error;
454 
455 	*agbno += cleft->rc_blockcount;
456 	*aglen -= cleft->rc_blockcount;
457 	return error;
458 
459 out_error:
460 	trace_xfs_refcount_merge_left_extent_error(cur->bc_mp,
461 			cur->bc_private.a.agno, error, _RET_IP_);
462 	return error;
463 }
464 
465 /*
466  * Merge with the right extent.
467  */
468 STATIC int
xfs_refcount_merge_right_extent(struct xfs_btree_cur * cur,struct xfs_refcount_irec * right,struct xfs_refcount_irec * cright,xfs_agblock_t * agbno,xfs_extlen_t * aglen)469 xfs_refcount_merge_right_extent(
470 	struct xfs_btree_cur		*cur,
471 	struct xfs_refcount_irec	*right,
472 	struct xfs_refcount_irec	*cright,
473 	xfs_agblock_t			*agbno,
474 	xfs_extlen_t			*aglen)
475 {
476 	int				error;
477 	int				found_rec;
478 
479 	trace_xfs_refcount_merge_right_extent(cur->bc_mp,
480 			cur->bc_private.a.agno, cright, right);
481 
482 	/*
483 	 * If the extent ending at agbno+aglen (cright) wasn't synthesized,
484 	 * remove it.
485 	 */
486 	if (cright->rc_refcount > 1) {
487 		error = xfs_refcount_lookup_le(cur, cright->rc_startblock,
488 			&found_rec);
489 		if (error)
490 			goto out_error;
491 		XFS_WANT_CORRUPTED_GOTO(cur->bc_mp, found_rec == 1,
492 				out_error);
493 
494 		error = xfs_refcount_delete(cur, &found_rec);
495 		if (error)
496 			goto out_error;
497 		XFS_WANT_CORRUPTED_GOTO(cur->bc_mp, found_rec == 1,
498 				out_error);
499 	}
500 
501 	/* Enlarge the right extent. */
502 	error = xfs_refcount_lookup_le(cur, right->rc_startblock,
503 			&found_rec);
504 	if (error)
505 		goto out_error;
506 	XFS_WANT_CORRUPTED_GOTO(cur->bc_mp, found_rec == 1, out_error);
507 
508 	right->rc_startblock -= cright->rc_blockcount;
509 	right->rc_blockcount += cright->rc_blockcount;
510 	error = xfs_refcount_update(cur, right);
511 	if (error)
512 		goto out_error;
513 
514 	*aglen -= cright->rc_blockcount;
515 	return error;
516 
517 out_error:
518 	trace_xfs_refcount_merge_right_extent_error(cur->bc_mp,
519 			cur->bc_private.a.agno, error, _RET_IP_);
520 	return error;
521 }
522 
523 #define XFS_FIND_RCEXT_SHARED	1
524 #define XFS_FIND_RCEXT_COW	2
525 /*
526  * Find the left extent and the one after it (cleft).  This function assumes
527  * that we've already split any extent crossing agbno.
528  */
529 STATIC int
xfs_refcount_find_left_extents(struct xfs_btree_cur * cur,struct xfs_refcount_irec * left,struct xfs_refcount_irec * cleft,xfs_agblock_t agbno,xfs_extlen_t aglen,int flags)530 xfs_refcount_find_left_extents(
531 	struct xfs_btree_cur		*cur,
532 	struct xfs_refcount_irec	*left,
533 	struct xfs_refcount_irec	*cleft,
534 	xfs_agblock_t			agbno,
535 	xfs_extlen_t			aglen,
536 	int				flags)
537 {
538 	struct xfs_refcount_irec	tmp;
539 	int				error;
540 	int				found_rec;
541 
542 	left->rc_startblock = cleft->rc_startblock = NULLAGBLOCK;
543 	error = xfs_refcount_lookup_le(cur, agbno - 1, &found_rec);
544 	if (error)
545 		goto out_error;
546 	if (!found_rec)
547 		return 0;
548 
549 	error = xfs_refcount_get_rec(cur, &tmp, &found_rec);
550 	if (error)
551 		goto out_error;
552 	XFS_WANT_CORRUPTED_GOTO(cur->bc_mp, found_rec == 1, out_error);
553 
554 	if (xfs_refc_next(&tmp) != agbno)
555 		return 0;
556 	if ((flags & XFS_FIND_RCEXT_SHARED) && tmp.rc_refcount < 2)
557 		return 0;
558 	if ((flags & XFS_FIND_RCEXT_COW) && tmp.rc_refcount > 1)
559 		return 0;
560 	/* We have a left extent; retrieve (or invent) the next right one */
561 	*left = tmp;
562 
563 	error = xfs_btree_increment(cur, 0, &found_rec);
564 	if (error)
565 		goto out_error;
566 	if (found_rec) {
567 		error = xfs_refcount_get_rec(cur, &tmp, &found_rec);
568 		if (error)
569 			goto out_error;
570 		XFS_WANT_CORRUPTED_GOTO(cur->bc_mp, found_rec == 1,
571 				out_error);
572 
573 		/* if tmp starts at the end of our range, just use that */
574 		if (tmp.rc_startblock == agbno)
575 			*cleft = tmp;
576 		else {
577 			/*
578 			 * There's a gap in the refcntbt at the start of the
579 			 * range we're interested in (refcount == 1) so
580 			 * synthesize the implied extent and pass it back.
581 			 * We assume here that the agbno/aglen range was
582 			 * passed in from a data fork extent mapping and
583 			 * therefore is allocated to exactly one owner.
584 			 */
585 			cleft->rc_startblock = agbno;
586 			cleft->rc_blockcount = min(aglen,
587 					tmp.rc_startblock - agbno);
588 			cleft->rc_refcount = 1;
589 		}
590 	} else {
591 		/*
592 		 * No extents, so pretend that there's one covering the whole
593 		 * range.
594 		 */
595 		cleft->rc_startblock = agbno;
596 		cleft->rc_blockcount = aglen;
597 		cleft->rc_refcount = 1;
598 	}
599 	trace_xfs_refcount_find_left_extent(cur->bc_mp, cur->bc_private.a.agno,
600 			left, cleft, agbno);
601 	return error;
602 
603 out_error:
604 	trace_xfs_refcount_find_left_extent_error(cur->bc_mp,
605 			cur->bc_private.a.agno, error, _RET_IP_);
606 	return error;
607 }
608 
609 /*
610  * Find the right extent and the one before it (cright).  This function
611  * assumes that we've already split any extents crossing agbno + aglen.
612  */
613 STATIC int
xfs_refcount_find_right_extents(struct xfs_btree_cur * cur,struct xfs_refcount_irec * right,struct xfs_refcount_irec * cright,xfs_agblock_t agbno,xfs_extlen_t aglen,int flags)614 xfs_refcount_find_right_extents(
615 	struct xfs_btree_cur		*cur,
616 	struct xfs_refcount_irec	*right,
617 	struct xfs_refcount_irec	*cright,
618 	xfs_agblock_t			agbno,
619 	xfs_extlen_t			aglen,
620 	int				flags)
621 {
622 	struct xfs_refcount_irec	tmp;
623 	int				error;
624 	int				found_rec;
625 
626 	right->rc_startblock = cright->rc_startblock = NULLAGBLOCK;
627 	error = xfs_refcount_lookup_ge(cur, agbno + aglen, &found_rec);
628 	if (error)
629 		goto out_error;
630 	if (!found_rec)
631 		return 0;
632 
633 	error = xfs_refcount_get_rec(cur, &tmp, &found_rec);
634 	if (error)
635 		goto out_error;
636 	XFS_WANT_CORRUPTED_GOTO(cur->bc_mp, found_rec == 1, out_error);
637 
638 	if (tmp.rc_startblock != agbno + aglen)
639 		return 0;
640 	if ((flags & XFS_FIND_RCEXT_SHARED) && tmp.rc_refcount < 2)
641 		return 0;
642 	if ((flags & XFS_FIND_RCEXT_COW) && tmp.rc_refcount > 1)
643 		return 0;
644 	/* We have a right extent; retrieve (or invent) the next left one */
645 	*right = tmp;
646 
647 	error = xfs_btree_decrement(cur, 0, &found_rec);
648 	if (error)
649 		goto out_error;
650 	if (found_rec) {
651 		error = xfs_refcount_get_rec(cur, &tmp, &found_rec);
652 		if (error)
653 			goto out_error;
654 		XFS_WANT_CORRUPTED_GOTO(cur->bc_mp, found_rec == 1,
655 				out_error);
656 
657 		/* if tmp ends at the end of our range, just use that */
658 		if (xfs_refc_next(&tmp) == agbno + aglen)
659 			*cright = tmp;
660 		else {
661 			/*
662 			 * There's a gap in the refcntbt at the end of the
663 			 * range we're interested in (refcount == 1) so
664 			 * create the implied extent and pass it back.
665 			 * We assume here that the agbno/aglen range was
666 			 * passed in from a data fork extent mapping and
667 			 * therefore is allocated to exactly one owner.
668 			 */
669 			cright->rc_startblock = max(agbno, xfs_refc_next(&tmp));
670 			cright->rc_blockcount = right->rc_startblock -
671 					cright->rc_startblock;
672 			cright->rc_refcount = 1;
673 		}
674 	} else {
675 		/*
676 		 * No extents, so pretend that there's one covering the whole
677 		 * range.
678 		 */
679 		cright->rc_startblock = agbno;
680 		cright->rc_blockcount = aglen;
681 		cright->rc_refcount = 1;
682 	}
683 	trace_xfs_refcount_find_right_extent(cur->bc_mp, cur->bc_private.a.agno,
684 			cright, right, agbno + aglen);
685 	return error;
686 
687 out_error:
688 	trace_xfs_refcount_find_right_extent_error(cur->bc_mp,
689 			cur->bc_private.a.agno, error, _RET_IP_);
690 	return error;
691 }
692 
693 /* Is this extent valid? */
694 static inline bool
xfs_refc_valid(struct xfs_refcount_irec * rc)695 xfs_refc_valid(
696 	struct xfs_refcount_irec	*rc)
697 {
698 	return rc->rc_startblock != NULLAGBLOCK;
699 }
700 
701 /*
702  * Try to merge with any extents on the boundaries of the adjustment range.
703  */
704 STATIC int
xfs_refcount_merge_extents(struct xfs_btree_cur * cur,xfs_agblock_t * agbno,xfs_extlen_t * aglen,enum xfs_refc_adjust_op adjust,int flags,bool * shape_changed)705 xfs_refcount_merge_extents(
706 	struct xfs_btree_cur	*cur,
707 	xfs_agblock_t		*agbno,
708 	xfs_extlen_t		*aglen,
709 	enum xfs_refc_adjust_op adjust,
710 	int			flags,
711 	bool			*shape_changed)
712 {
713 	struct xfs_refcount_irec	left = {0}, cleft = {0};
714 	struct xfs_refcount_irec	cright = {0}, right = {0};
715 	int				error;
716 	unsigned long long		ulen;
717 	bool				cequal;
718 
719 	*shape_changed = false;
720 	/*
721 	 * Find the extent just below agbno [left], just above agbno [cleft],
722 	 * just below (agbno + aglen) [cright], and just above (agbno + aglen)
723 	 * [right].
724 	 */
725 	error = xfs_refcount_find_left_extents(cur, &left, &cleft, *agbno,
726 			*aglen, flags);
727 	if (error)
728 		return error;
729 	error = xfs_refcount_find_right_extents(cur, &right, &cright, *agbno,
730 			*aglen, flags);
731 	if (error)
732 		return error;
733 
734 	/* No left or right extent to merge; exit. */
735 	if (!xfs_refc_valid(&left) && !xfs_refc_valid(&right))
736 		return 0;
737 
738 	cequal = (cleft.rc_startblock == cright.rc_startblock) &&
739 		 (cleft.rc_blockcount == cright.rc_blockcount);
740 
741 	/* Try to merge left, cleft, and right.  cleft must == cright. */
742 	ulen = (unsigned long long)left.rc_blockcount + cleft.rc_blockcount +
743 			right.rc_blockcount;
744 	if (xfs_refc_valid(&left) && xfs_refc_valid(&right) &&
745 	    xfs_refc_valid(&cleft) && xfs_refc_valid(&cright) && cequal &&
746 	    left.rc_refcount == cleft.rc_refcount + adjust &&
747 	    right.rc_refcount == cleft.rc_refcount + adjust &&
748 	    ulen < MAXREFCEXTLEN) {
749 		*shape_changed = true;
750 		return xfs_refcount_merge_center_extents(cur, &left, &cleft,
751 				&right, ulen, agbno, aglen);
752 	}
753 
754 	/* Try to merge left and cleft. */
755 	ulen = (unsigned long long)left.rc_blockcount + cleft.rc_blockcount;
756 	if (xfs_refc_valid(&left) && xfs_refc_valid(&cleft) &&
757 	    left.rc_refcount == cleft.rc_refcount + adjust &&
758 	    ulen < MAXREFCEXTLEN) {
759 		*shape_changed = true;
760 		error = xfs_refcount_merge_left_extent(cur, &left, &cleft,
761 				agbno, aglen);
762 		if (error)
763 			return error;
764 
765 		/*
766 		 * If we just merged left + cleft and cleft == cright,
767 		 * we no longer have a cright to merge with right.  We're done.
768 		 */
769 		if (cequal)
770 			return 0;
771 	}
772 
773 	/* Try to merge cright and right. */
774 	ulen = (unsigned long long)right.rc_blockcount + cright.rc_blockcount;
775 	if (xfs_refc_valid(&right) && xfs_refc_valid(&cright) &&
776 	    right.rc_refcount == cright.rc_refcount + adjust &&
777 	    ulen < MAXREFCEXTLEN) {
778 		*shape_changed = true;
779 		return xfs_refcount_merge_right_extent(cur, &right, &cright,
780 				agbno, aglen);
781 	}
782 
783 	return error;
784 }
785 
786 /*
787  * XXX: This is a pretty hand-wavy estimate.  The penalty for guessing
788  * true incorrectly is a shutdown FS; the penalty for guessing false
789  * incorrectly is more transaction rolls than might be necessary.
790  * Be conservative here.
791  */
792 static bool
xfs_refcount_still_have_space(struct xfs_btree_cur * cur)793 xfs_refcount_still_have_space(
794 	struct xfs_btree_cur		*cur)
795 {
796 	unsigned long			overhead;
797 
798 	overhead = cur->bc_private.a.priv.refc.shape_changes *
799 			xfs_allocfree_log_count(cur->bc_mp, 1);
800 	overhead *= cur->bc_mp->m_sb.sb_blocksize;
801 
802 	/*
803 	 * Only allow 2 refcount extent updates per transaction if the
804 	 * refcount continue update "error" has been injected.
805 	 */
806 	if (cur->bc_private.a.priv.refc.nr_ops > 2 &&
807 	    XFS_TEST_ERROR(false, cur->bc_mp,
808 			XFS_ERRTAG_REFCOUNT_CONTINUE_UPDATE,
809 			XFS_RANDOM_REFCOUNT_CONTINUE_UPDATE))
810 		return false;
811 
812 	if (cur->bc_private.a.priv.refc.nr_ops == 0)
813 		return true;
814 	else if (overhead > cur->bc_tp->t_log_res)
815 		return false;
816 	return  cur->bc_tp->t_log_res - overhead >
817 		cur->bc_private.a.priv.refc.nr_ops * XFS_REFCOUNT_ITEM_OVERHEAD;
818 }
819 
820 /*
821  * Adjust the refcounts of middle extents.  At this point we should have
822  * split extents that crossed the adjustment range; merged with adjacent
823  * extents; and updated agbno/aglen to reflect the merges.  Therefore,
824  * all we have to do is update the extents inside [agbno, agbno + aglen].
825  */
826 STATIC int
xfs_refcount_adjust_extents(struct xfs_btree_cur * cur,xfs_agblock_t * agbno,xfs_extlen_t * aglen,enum xfs_refc_adjust_op adj,struct xfs_defer_ops * dfops,struct xfs_owner_info * oinfo)827 xfs_refcount_adjust_extents(
828 	struct xfs_btree_cur	*cur,
829 	xfs_agblock_t		*agbno,
830 	xfs_extlen_t		*aglen,
831 	enum xfs_refc_adjust_op	adj,
832 	struct xfs_defer_ops	*dfops,
833 	struct xfs_owner_info	*oinfo)
834 {
835 	struct xfs_refcount_irec	ext, tmp;
836 	int				error;
837 	int				found_rec, found_tmp;
838 	xfs_fsblock_t			fsbno;
839 
840 	/* Merging did all the work already. */
841 	if (*aglen == 0)
842 		return 0;
843 
844 	error = xfs_refcount_lookup_ge(cur, *agbno, &found_rec);
845 	if (error)
846 		goto out_error;
847 
848 	while (*aglen > 0 && xfs_refcount_still_have_space(cur)) {
849 		error = xfs_refcount_get_rec(cur, &ext, &found_rec);
850 		if (error)
851 			goto out_error;
852 		if (!found_rec) {
853 			ext.rc_startblock = cur->bc_mp->m_sb.sb_agblocks;
854 			ext.rc_blockcount = 0;
855 			ext.rc_refcount = 0;
856 		}
857 
858 		/*
859 		 * Deal with a hole in the refcount tree; if a file maps to
860 		 * these blocks and there's no refcountbt record, pretend that
861 		 * there is one with refcount == 1.
862 		 */
863 		if (ext.rc_startblock != *agbno) {
864 			tmp.rc_startblock = *agbno;
865 			tmp.rc_blockcount = min(*aglen,
866 					ext.rc_startblock - *agbno);
867 			tmp.rc_refcount = 1 + adj;
868 			trace_xfs_refcount_modify_extent(cur->bc_mp,
869 					cur->bc_private.a.agno, &tmp);
870 
871 			/*
872 			 * Either cover the hole (increment) or
873 			 * delete the range (decrement).
874 			 */
875 			if (tmp.rc_refcount) {
876 				error = xfs_refcount_insert(cur, &tmp,
877 						&found_tmp);
878 				if (error)
879 					goto out_error;
880 				XFS_WANT_CORRUPTED_GOTO(cur->bc_mp,
881 						found_tmp == 1, out_error);
882 				cur->bc_private.a.priv.refc.nr_ops++;
883 			} else {
884 				fsbno = XFS_AGB_TO_FSB(cur->bc_mp,
885 						cur->bc_private.a.agno,
886 						tmp.rc_startblock);
887 				xfs_bmap_add_free(cur->bc_mp, dfops, fsbno,
888 						tmp.rc_blockcount, oinfo);
889 			}
890 
891 			(*agbno) += tmp.rc_blockcount;
892 			(*aglen) -= tmp.rc_blockcount;
893 
894 			error = xfs_refcount_lookup_ge(cur, *agbno,
895 					&found_rec);
896 			if (error)
897 				goto out_error;
898 		}
899 
900 		/* Stop if there's nothing left to modify */
901 		if (*aglen == 0 || !xfs_refcount_still_have_space(cur))
902 			break;
903 
904 		/*
905 		 * Adjust the reference count and either update the tree
906 		 * (incr) or free the blocks (decr).
907 		 */
908 		if (ext.rc_refcount == MAXREFCOUNT)
909 			goto skip;
910 		ext.rc_refcount += adj;
911 		trace_xfs_refcount_modify_extent(cur->bc_mp,
912 				cur->bc_private.a.agno, &ext);
913 		if (ext.rc_refcount > 1) {
914 			error = xfs_refcount_update(cur, &ext);
915 			if (error)
916 				goto out_error;
917 			cur->bc_private.a.priv.refc.nr_ops++;
918 		} else if (ext.rc_refcount == 1) {
919 			error = xfs_refcount_delete(cur, &found_rec);
920 			if (error)
921 				goto out_error;
922 			XFS_WANT_CORRUPTED_GOTO(cur->bc_mp,
923 					found_rec == 1, out_error);
924 			cur->bc_private.a.priv.refc.nr_ops++;
925 			goto advloop;
926 		} else {
927 			fsbno = XFS_AGB_TO_FSB(cur->bc_mp,
928 					cur->bc_private.a.agno,
929 					ext.rc_startblock);
930 			xfs_bmap_add_free(cur->bc_mp, dfops, fsbno,
931 					ext.rc_blockcount, oinfo);
932 		}
933 
934 skip:
935 		error = xfs_btree_increment(cur, 0, &found_rec);
936 		if (error)
937 			goto out_error;
938 
939 advloop:
940 		(*agbno) += ext.rc_blockcount;
941 		(*aglen) -= ext.rc_blockcount;
942 	}
943 
944 	return error;
945 out_error:
946 	trace_xfs_refcount_modify_extent_error(cur->bc_mp,
947 			cur->bc_private.a.agno, error, _RET_IP_);
948 	return error;
949 }
950 
951 /* Adjust the reference count of a range of AG blocks. */
952 STATIC int
xfs_refcount_adjust(struct xfs_btree_cur * cur,xfs_agblock_t agbno,xfs_extlen_t aglen,xfs_agblock_t * new_agbno,xfs_extlen_t * new_aglen,enum xfs_refc_adjust_op adj,struct xfs_defer_ops * dfops,struct xfs_owner_info * oinfo)953 xfs_refcount_adjust(
954 	struct xfs_btree_cur	*cur,
955 	xfs_agblock_t		agbno,
956 	xfs_extlen_t		aglen,
957 	xfs_agblock_t		*new_agbno,
958 	xfs_extlen_t		*new_aglen,
959 	enum xfs_refc_adjust_op	adj,
960 	struct xfs_defer_ops	*dfops,
961 	struct xfs_owner_info	*oinfo)
962 {
963 	bool			shape_changed;
964 	int			shape_changes = 0;
965 	int			error;
966 
967 	*new_agbno = agbno;
968 	*new_aglen = aglen;
969 	if (adj == XFS_REFCOUNT_ADJUST_INCREASE)
970 		trace_xfs_refcount_increase(cur->bc_mp, cur->bc_private.a.agno,
971 				agbno, aglen);
972 	else
973 		trace_xfs_refcount_decrease(cur->bc_mp, cur->bc_private.a.agno,
974 				agbno, aglen);
975 
976 	/*
977 	 * Ensure that no rcextents cross the boundary of the adjustment range.
978 	 */
979 	error = xfs_refcount_split_extent(cur, agbno, &shape_changed);
980 	if (error)
981 		goto out_error;
982 	if (shape_changed)
983 		shape_changes++;
984 
985 	error = xfs_refcount_split_extent(cur, agbno + aglen, &shape_changed);
986 	if (error)
987 		goto out_error;
988 	if (shape_changed)
989 		shape_changes++;
990 
991 	/*
992 	 * Try to merge with the left or right extents of the range.
993 	 */
994 	error = xfs_refcount_merge_extents(cur, new_agbno, new_aglen, adj,
995 			XFS_FIND_RCEXT_SHARED, &shape_changed);
996 	if (error)
997 		goto out_error;
998 	if (shape_changed)
999 		shape_changes++;
1000 	if (shape_changes)
1001 		cur->bc_private.a.priv.refc.shape_changes++;
1002 
1003 	/* Now that we've taken care of the ends, adjust the middle extents */
1004 	error = xfs_refcount_adjust_extents(cur, new_agbno, new_aglen,
1005 			adj, dfops, oinfo);
1006 	if (error)
1007 		goto out_error;
1008 
1009 	return 0;
1010 
1011 out_error:
1012 	trace_xfs_refcount_adjust_error(cur->bc_mp, cur->bc_private.a.agno,
1013 			error, _RET_IP_);
1014 	return error;
1015 }
1016 
1017 /* Clean up after calling xfs_refcount_finish_one. */
1018 void
xfs_refcount_finish_one_cleanup(struct xfs_trans * tp,struct xfs_btree_cur * rcur,int error)1019 xfs_refcount_finish_one_cleanup(
1020 	struct xfs_trans	*tp,
1021 	struct xfs_btree_cur	*rcur,
1022 	int			error)
1023 {
1024 	struct xfs_buf		*agbp;
1025 
1026 	if (rcur == NULL)
1027 		return;
1028 	agbp = rcur->bc_private.a.agbp;
1029 	xfs_btree_del_cursor(rcur, error ? XFS_BTREE_ERROR : XFS_BTREE_NOERROR);
1030 	if (error)
1031 		xfs_trans_brelse(tp, agbp);
1032 }
1033 
1034 /*
1035  * Process one of the deferred refcount operations.  We pass back the
1036  * btree cursor to maintain our lock on the btree between calls.
1037  * This saves time and eliminates a buffer deadlock between the
1038  * superblock and the AGF because we'll always grab them in the same
1039  * order.
1040  */
1041 int
xfs_refcount_finish_one(struct xfs_trans * tp,struct xfs_defer_ops * dfops,enum xfs_refcount_intent_type type,xfs_fsblock_t startblock,xfs_extlen_t blockcount,xfs_fsblock_t * new_fsb,xfs_extlen_t * new_len,struct xfs_btree_cur ** pcur)1042 xfs_refcount_finish_one(
1043 	struct xfs_trans		*tp,
1044 	struct xfs_defer_ops		*dfops,
1045 	enum xfs_refcount_intent_type	type,
1046 	xfs_fsblock_t			startblock,
1047 	xfs_extlen_t			blockcount,
1048 	xfs_fsblock_t			*new_fsb,
1049 	xfs_extlen_t			*new_len,
1050 	struct xfs_btree_cur		**pcur)
1051 {
1052 	struct xfs_mount		*mp = tp->t_mountp;
1053 	struct xfs_btree_cur		*rcur;
1054 	struct xfs_buf			*agbp = NULL;
1055 	int				error = 0;
1056 	xfs_agnumber_t			agno;
1057 	xfs_agblock_t			bno;
1058 	xfs_agblock_t			new_agbno;
1059 	unsigned long			nr_ops = 0;
1060 	int				shape_changes = 0;
1061 
1062 	agno = XFS_FSB_TO_AGNO(mp, startblock);
1063 	ASSERT(agno != NULLAGNUMBER);
1064 	bno = XFS_FSB_TO_AGBNO(mp, startblock);
1065 
1066 	trace_xfs_refcount_deferred(mp, XFS_FSB_TO_AGNO(mp, startblock),
1067 			type, XFS_FSB_TO_AGBNO(mp, startblock),
1068 			blockcount);
1069 
1070 	if (XFS_TEST_ERROR(false, mp,
1071 			XFS_ERRTAG_REFCOUNT_FINISH_ONE,
1072 			XFS_RANDOM_REFCOUNT_FINISH_ONE))
1073 		return -EIO;
1074 
1075 	/*
1076 	 * If we haven't gotten a cursor or the cursor AG doesn't match
1077 	 * the startblock, get one now.
1078 	 */
1079 	rcur = *pcur;
1080 	if (rcur != NULL && rcur->bc_private.a.agno != agno) {
1081 		nr_ops = rcur->bc_private.a.priv.refc.nr_ops;
1082 		shape_changes = rcur->bc_private.a.priv.refc.shape_changes;
1083 		xfs_refcount_finish_one_cleanup(tp, rcur, 0);
1084 		rcur = NULL;
1085 		*pcur = NULL;
1086 	}
1087 	if (rcur == NULL) {
1088 		error = xfs_alloc_read_agf(tp->t_mountp, tp, agno,
1089 				XFS_ALLOC_FLAG_FREEING, &agbp);
1090 		if (error)
1091 			return error;
1092 		if (!agbp)
1093 			return -EFSCORRUPTED;
1094 
1095 		rcur = xfs_refcountbt_init_cursor(mp, tp, agbp, agno, dfops);
1096 		if (!rcur) {
1097 			error = -ENOMEM;
1098 			goto out_cur;
1099 		}
1100 		rcur->bc_private.a.priv.refc.nr_ops = nr_ops;
1101 		rcur->bc_private.a.priv.refc.shape_changes = shape_changes;
1102 	}
1103 	*pcur = rcur;
1104 
1105 	switch (type) {
1106 	case XFS_REFCOUNT_INCREASE:
1107 		error = xfs_refcount_adjust(rcur, bno, blockcount, &new_agbno,
1108 			new_len, XFS_REFCOUNT_ADJUST_INCREASE, dfops, NULL);
1109 		*new_fsb = XFS_AGB_TO_FSB(mp, agno, new_agbno);
1110 		break;
1111 	case XFS_REFCOUNT_DECREASE:
1112 		error = xfs_refcount_adjust(rcur, bno, blockcount, &new_agbno,
1113 			new_len, XFS_REFCOUNT_ADJUST_DECREASE, dfops, NULL);
1114 		*new_fsb = XFS_AGB_TO_FSB(mp, agno, new_agbno);
1115 		break;
1116 	case XFS_REFCOUNT_ALLOC_COW:
1117 		*new_fsb = startblock + blockcount;
1118 		*new_len = 0;
1119 		error = __xfs_refcount_cow_alloc(rcur, bno, blockcount, dfops);
1120 		break;
1121 	case XFS_REFCOUNT_FREE_COW:
1122 		*new_fsb = startblock + blockcount;
1123 		*new_len = 0;
1124 		error = __xfs_refcount_cow_free(rcur, bno, blockcount, dfops);
1125 		break;
1126 	default:
1127 		ASSERT(0);
1128 		error = -EFSCORRUPTED;
1129 	}
1130 	if (!error && *new_len > 0)
1131 		trace_xfs_refcount_finish_one_leftover(mp, agno, type,
1132 				bno, blockcount, new_agbno, *new_len);
1133 	return error;
1134 
1135 out_cur:
1136 	xfs_trans_brelse(tp, agbp);
1137 
1138 	return error;
1139 }
1140 
1141 /*
1142  * Record a refcount intent for later processing.
1143  */
1144 static int
__xfs_refcount_add(struct xfs_mount * mp,struct xfs_defer_ops * dfops,enum xfs_refcount_intent_type type,xfs_fsblock_t startblock,xfs_extlen_t blockcount)1145 __xfs_refcount_add(
1146 	struct xfs_mount		*mp,
1147 	struct xfs_defer_ops		*dfops,
1148 	enum xfs_refcount_intent_type	type,
1149 	xfs_fsblock_t			startblock,
1150 	xfs_extlen_t			blockcount)
1151 {
1152 	struct xfs_refcount_intent	*ri;
1153 
1154 	trace_xfs_refcount_defer(mp, XFS_FSB_TO_AGNO(mp, startblock),
1155 			type, XFS_FSB_TO_AGBNO(mp, startblock),
1156 			blockcount);
1157 
1158 	ri = kmem_alloc(sizeof(struct xfs_refcount_intent),
1159 			KM_SLEEP | KM_NOFS);
1160 	INIT_LIST_HEAD(&ri->ri_list);
1161 	ri->ri_type = type;
1162 	ri->ri_startblock = startblock;
1163 	ri->ri_blockcount = blockcount;
1164 
1165 	xfs_defer_add(dfops, XFS_DEFER_OPS_TYPE_REFCOUNT, &ri->ri_list);
1166 	return 0;
1167 }
1168 
1169 /*
1170  * Increase the reference count of the blocks backing a file's extent.
1171  */
1172 int
xfs_refcount_increase_extent(struct xfs_mount * mp,struct xfs_defer_ops * dfops,struct xfs_bmbt_irec * PREV)1173 xfs_refcount_increase_extent(
1174 	struct xfs_mount		*mp,
1175 	struct xfs_defer_ops		*dfops,
1176 	struct xfs_bmbt_irec		*PREV)
1177 {
1178 	if (!xfs_sb_version_hasreflink(&mp->m_sb))
1179 		return 0;
1180 
1181 	return __xfs_refcount_add(mp, dfops, XFS_REFCOUNT_INCREASE,
1182 			PREV->br_startblock, PREV->br_blockcount);
1183 }
1184 
1185 /*
1186  * Decrease the reference count of the blocks backing a file's extent.
1187  */
1188 int
xfs_refcount_decrease_extent(struct xfs_mount * mp,struct xfs_defer_ops * dfops,struct xfs_bmbt_irec * PREV)1189 xfs_refcount_decrease_extent(
1190 	struct xfs_mount		*mp,
1191 	struct xfs_defer_ops		*dfops,
1192 	struct xfs_bmbt_irec		*PREV)
1193 {
1194 	if (!xfs_sb_version_hasreflink(&mp->m_sb))
1195 		return 0;
1196 
1197 	return __xfs_refcount_add(mp, dfops, XFS_REFCOUNT_DECREASE,
1198 			PREV->br_startblock, PREV->br_blockcount);
1199 }
1200 
1201 /*
1202  * Given an AG extent, find the lowest-numbered run of shared blocks
1203  * within that range and return the range in fbno/flen.  If
1204  * find_end_of_shared is set, return the longest contiguous extent of
1205  * shared blocks; if not, just return the first extent we find.  If no
1206  * shared blocks are found, fbno and flen will be set to NULLAGBLOCK
1207  * and 0, respectively.
1208  */
1209 int
xfs_refcount_find_shared(struct xfs_btree_cur * cur,xfs_agblock_t agbno,xfs_extlen_t aglen,xfs_agblock_t * fbno,xfs_extlen_t * flen,bool find_end_of_shared)1210 xfs_refcount_find_shared(
1211 	struct xfs_btree_cur		*cur,
1212 	xfs_agblock_t			agbno,
1213 	xfs_extlen_t			aglen,
1214 	xfs_agblock_t			*fbno,
1215 	xfs_extlen_t			*flen,
1216 	bool				find_end_of_shared)
1217 {
1218 	struct xfs_refcount_irec	tmp;
1219 	int				i;
1220 	int				have;
1221 	int				error;
1222 
1223 	trace_xfs_refcount_find_shared(cur->bc_mp, cur->bc_private.a.agno,
1224 			agbno, aglen);
1225 
1226 	/* By default, skip the whole range */
1227 	*fbno = NULLAGBLOCK;
1228 	*flen = 0;
1229 
1230 	/* Try to find a refcount extent that crosses the start */
1231 	error = xfs_refcount_lookup_le(cur, agbno, &have);
1232 	if (error)
1233 		goto out_error;
1234 	if (!have) {
1235 		/* No left extent, look at the next one */
1236 		error = xfs_btree_increment(cur, 0, &have);
1237 		if (error)
1238 			goto out_error;
1239 		if (!have)
1240 			goto done;
1241 	}
1242 	error = xfs_refcount_get_rec(cur, &tmp, &i);
1243 	if (error)
1244 		goto out_error;
1245 	XFS_WANT_CORRUPTED_GOTO(cur->bc_mp, i == 1, out_error);
1246 
1247 	/* If the extent ends before the start, look at the next one */
1248 	if (tmp.rc_startblock + tmp.rc_blockcount <= agbno) {
1249 		error = xfs_btree_increment(cur, 0, &have);
1250 		if (error)
1251 			goto out_error;
1252 		if (!have)
1253 			goto done;
1254 		error = xfs_refcount_get_rec(cur, &tmp, &i);
1255 		if (error)
1256 			goto out_error;
1257 		XFS_WANT_CORRUPTED_GOTO(cur->bc_mp, i == 1, out_error);
1258 	}
1259 
1260 	/* If the extent starts after the range we want, bail out */
1261 	if (tmp.rc_startblock >= agbno + aglen)
1262 		goto done;
1263 
1264 	/* We found the start of a shared extent! */
1265 	if (tmp.rc_startblock < agbno) {
1266 		tmp.rc_blockcount -= (agbno - tmp.rc_startblock);
1267 		tmp.rc_startblock = agbno;
1268 	}
1269 
1270 	*fbno = tmp.rc_startblock;
1271 	*flen = min(tmp.rc_blockcount, agbno + aglen - *fbno);
1272 	if (!find_end_of_shared)
1273 		goto done;
1274 
1275 	/* Otherwise, find the end of this shared extent */
1276 	while (*fbno + *flen < agbno + aglen) {
1277 		error = xfs_btree_increment(cur, 0, &have);
1278 		if (error)
1279 			goto out_error;
1280 		if (!have)
1281 			break;
1282 		error = xfs_refcount_get_rec(cur, &tmp, &i);
1283 		if (error)
1284 			goto out_error;
1285 		XFS_WANT_CORRUPTED_GOTO(cur->bc_mp, i == 1, out_error);
1286 		if (tmp.rc_startblock >= agbno + aglen ||
1287 		    tmp.rc_startblock != *fbno + *flen)
1288 			break;
1289 		*flen = min(*flen + tmp.rc_blockcount, agbno + aglen - *fbno);
1290 	}
1291 
1292 done:
1293 	trace_xfs_refcount_find_shared_result(cur->bc_mp,
1294 			cur->bc_private.a.agno, *fbno, *flen);
1295 
1296 out_error:
1297 	if (error)
1298 		trace_xfs_refcount_find_shared_error(cur->bc_mp,
1299 				cur->bc_private.a.agno, error, _RET_IP_);
1300 	return error;
1301 }
1302 
1303 /*
1304  * Recovering CoW Blocks After a Crash
1305  *
1306  * Due to the way that the copy on write mechanism works, there's a window of
1307  * opportunity in which we can lose track of allocated blocks during a crash.
1308  * Because CoW uses delayed allocation in the in-core CoW fork, writeback
1309  * causes blocks to be allocated and stored in the CoW fork.  The blocks are
1310  * no longer in the free space btree but are not otherwise recorded anywhere
1311  * until the write completes and the blocks are mapped into the file.  A crash
1312  * in between allocation and remapping results in the replacement blocks being
1313  * lost.  This situation is exacerbated by the CoW extent size hint because
1314  * allocations can hang around for long time.
1315  *
1316  * However, there is a place where we can record these allocations before they
1317  * become mappings -- the reference count btree.  The btree does not record
1318  * extents with refcount == 1, so we can record allocations with a refcount of
1319  * 1.  Blocks being used for CoW writeout cannot be shared, so there should be
1320  * no conflict with shared block records.  These mappings should be created
1321  * when we allocate blocks to the CoW fork and deleted when they're removed
1322  * from the CoW fork.
1323  *
1324  * Minor nit: records for in-progress CoW allocations and records for shared
1325  * extents must never be merged, to preserve the property that (except for CoW
1326  * allocations) there are no refcount btree entries with refcount == 1.  The
1327  * only time this could potentially happen is when unsharing a block that's
1328  * adjacent to CoW allocations, so we must be careful to avoid this.
1329  *
1330  * At mount time we recover lost CoW allocations by searching the refcount
1331  * btree for these refcount == 1 mappings.  These represent CoW allocations
1332  * that were in progress at the time the filesystem went down, so we can free
1333  * them to get the space back.
1334  *
1335  * This mechanism is superior to creating EFIs for unmapped CoW extents for
1336  * several reasons -- first, EFIs pin the tail of the log and would have to be
1337  * periodically relogged to avoid filling up the log.  Second, CoW completions
1338  * will have to file an EFD and create new EFIs for whatever remains in the
1339  * CoW fork; this partially takes care of (1) but extent-size reservations
1340  * will have to periodically relog even if there's no writeout in progress.
1341  * This can happen if the CoW extent size hint is set, which you really want.
1342  * Third, EFIs cannot currently be automatically relogged into newer
1343  * transactions to advance the log tail.  Fourth, stuffing the log full of
1344  * EFIs places an upper bound on the number of CoW allocations that can be
1345  * held filesystem-wide at any given time.  Recording them in the refcount
1346  * btree doesn't require us to maintain any state in memory and doesn't pin
1347  * the log.
1348  */
1349 /*
1350  * Adjust the refcounts of CoW allocations.  These allocations are "magic"
1351  * in that they're not referenced anywhere else in the filesystem, so we
1352  * stash them in the refcount btree with a refcount of 1 until either file
1353  * remapping (or CoW cancellation) happens.
1354  */
1355 STATIC int
xfs_refcount_adjust_cow_extents(struct xfs_btree_cur * cur,xfs_agblock_t agbno,xfs_extlen_t aglen,enum xfs_refc_adjust_op adj,struct xfs_defer_ops * dfops,struct xfs_owner_info * oinfo)1356 xfs_refcount_adjust_cow_extents(
1357 	struct xfs_btree_cur	*cur,
1358 	xfs_agblock_t		agbno,
1359 	xfs_extlen_t		aglen,
1360 	enum xfs_refc_adjust_op	adj,
1361 	struct xfs_defer_ops	*dfops,
1362 	struct xfs_owner_info	*oinfo)
1363 {
1364 	struct xfs_refcount_irec	ext, tmp;
1365 	int				error;
1366 	int				found_rec, found_tmp;
1367 
1368 	if (aglen == 0)
1369 		return 0;
1370 
1371 	/* Find any overlapping refcount records */
1372 	error = xfs_refcount_lookup_ge(cur, agbno, &found_rec);
1373 	if (error)
1374 		goto out_error;
1375 	error = xfs_refcount_get_rec(cur, &ext, &found_rec);
1376 	if (error)
1377 		goto out_error;
1378 	if (!found_rec) {
1379 		ext.rc_startblock = cur->bc_mp->m_sb.sb_agblocks +
1380 				XFS_REFC_COW_START;
1381 		ext.rc_blockcount = 0;
1382 		ext.rc_refcount = 0;
1383 	}
1384 
1385 	switch (adj) {
1386 	case XFS_REFCOUNT_ADJUST_COW_ALLOC:
1387 		/* Adding a CoW reservation, there should be nothing here. */
1388 		XFS_WANT_CORRUPTED_GOTO(cur->bc_mp,
1389 				ext.rc_startblock >= agbno + aglen, out_error);
1390 
1391 		tmp.rc_startblock = agbno;
1392 		tmp.rc_blockcount = aglen;
1393 		tmp.rc_refcount = 1;
1394 		trace_xfs_refcount_modify_extent(cur->bc_mp,
1395 				cur->bc_private.a.agno, &tmp);
1396 
1397 		error = xfs_refcount_insert(cur, &tmp,
1398 				&found_tmp);
1399 		if (error)
1400 			goto out_error;
1401 		XFS_WANT_CORRUPTED_GOTO(cur->bc_mp,
1402 				found_tmp == 1, out_error);
1403 		break;
1404 	case XFS_REFCOUNT_ADJUST_COW_FREE:
1405 		/* Removing a CoW reservation, there should be one extent. */
1406 		XFS_WANT_CORRUPTED_GOTO(cur->bc_mp,
1407 			ext.rc_startblock == agbno, out_error);
1408 		XFS_WANT_CORRUPTED_GOTO(cur->bc_mp,
1409 			ext.rc_blockcount == aglen, out_error);
1410 		XFS_WANT_CORRUPTED_GOTO(cur->bc_mp,
1411 			ext.rc_refcount == 1, out_error);
1412 
1413 		ext.rc_refcount = 0;
1414 		trace_xfs_refcount_modify_extent(cur->bc_mp,
1415 				cur->bc_private.a.agno, &ext);
1416 		error = xfs_refcount_delete(cur, &found_rec);
1417 		if (error)
1418 			goto out_error;
1419 		XFS_WANT_CORRUPTED_GOTO(cur->bc_mp,
1420 				found_rec == 1, out_error);
1421 		break;
1422 	default:
1423 		ASSERT(0);
1424 	}
1425 
1426 	return error;
1427 out_error:
1428 	trace_xfs_refcount_modify_extent_error(cur->bc_mp,
1429 			cur->bc_private.a.agno, error, _RET_IP_);
1430 	return error;
1431 }
1432 
1433 /*
1434  * Add or remove refcount btree entries for CoW reservations.
1435  */
1436 STATIC int
xfs_refcount_adjust_cow(struct xfs_btree_cur * cur,xfs_agblock_t agbno,xfs_extlen_t aglen,enum xfs_refc_adjust_op adj,struct xfs_defer_ops * dfops)1437 xfs_refcount_adjust_cow(
1438 	struct xfs_btree_cur	*cur,
1439 	xfs_agblock_t		agbno,
1440 	xfs_extlen_t		aglen,
1441 	enum xfs_refc_adjust_op	adj,
1442 	struct xfs_defer_ops	*dfops)
1443 {
1444 	bool			shape_changed;
1445 	int			error;
1446 
1447 	agbno += XFS_REFC_COW_START;
1448 
1449 	/*
1450 	 * Ensure that no rcextents cross the boundary of the adjustment range.
1451 	 */
1452 	error = xfs_refcount_split_extent(cur, agbno, &shape_changed);
1453 	if (error)
1454 		goto out_error;
1455 
1456 	error = xfs_refcount_split_extent(cur, agbno + aglen, &shape_changed);
1457 	if (error)
1458 		goto out_error;
1459 
1460 	/*
1461 	 * Try to merge with the left or right extents of the range.
1462 	 */
1463 	error = xfs_refcount_merge_extents(cur, &agbno, &aglen, adj,
1464 			XFS_FIND_RCEXT_COW, &shape_changed);
1465 	if (error)
1466 		goto out_error;
1467 
1468 	/* Now that we've taken care of the ends, adjust the middle extents */
1469 	error = xfs_refcount_adjust_cow_extents(cur, agbno, aglen, adj,
1470 			dfops, NULL);
1471 	if (error)
1472 		goto out_error;
1473 
1474 	return 0;
1475 
1476 out_error:
1477 	trace_xfs_refcount_adjust_cow_error(cur->bc_mp, cur->bc_private.a.agno,
1478 			error, _RET_IP_);
1479 	return error;
1480 }
1481 
1482 /*
1483  * Record a CoW allocation in the refcount btree.
1484  */
1485 STATIC int
__xfs_refcount_cow_alloc(struct xfs_btree_cur * rcur,xfs_agblock_t agbno,xfs_extlen_t aglen,struct xfs_defer_ops * dfops)1486 __xfs_refcount_cow_alloc(
1487 	struct xfs_btree_cur	*rcur,
1488 	xfs_agblock_t		agbno,
1489 	xfs_extlen_t		aglen,
1490 	struct xfs_defer_ops	*dfops)
1491 {
1492 	int			error;
1493 
1494 	trace_xfs_refcount_cow_increase(rcur->bc_mp, rcur->bc_private.a.agno,
1495 			agbno, aglen);
1496 
1497 	/* Add refcount btree reservation */
1498 	error = xfs_refcount_adjust_cow(rcur, agbno, aglen,
1499 			XFS_REFCOUNT_ADJUST_COW_ALLOC, dfops);
1500 	if (error)
1501 		return error;
1502 
1503 	/* Add rmap entry */
1504 	if (xfs_sb_version_hasrmapbt(&rcur->bc_mp->m_sb)) {
1505 		error = xfs_rmap_alloc_extent(rcur->bc_mp, dfops,
1506 				rcur->bc_private.a.agno,
1507 				agbno, aglen, XFS_RMAP_OWN_COW);
1508 		if (error)
1509 			return error;
1510 	}
1511 
1512 	return error;
1513 }
1514 
1515 /*
1516  * Remove a CoW allocation from the refcount btree.
1517  */
1518 STATIC int
__xfs_refcount_cow_free(struct xfs_btree_cur * rcur,xfs_agblock_t agbno,xfs_extlen_t aglen,struct xfs_defer_ops * dfops)1519 __xfs_refcount_cow_free(
1520 	struct xfs_btree_cur	*rcur,
1521 	xfs_agblock_t		agbno,
1522 	xfs_extlen_t		aglen,
1523 	struct xfs_defer_ops	*dfops)
1524 {
1525 	int			error;
1526 
1527 	trace_xfs_refcount_cow_decrease(rcur->bc_mp, rcur->bc_private.a.agno,
1528 			agbno, aglen);
1529 
1530 	/* Remove refcount btree reservation */
1531 	error = xfs_refcount_adjust_cow(rcur, agbno, aglen,
1532 			XFS_REFCOUNT_ADJUST_COW_FREE, dfops);
1533 	if (error)
1534 		return error;
1535 
1536 	/* Remove rmap entry */
1537 	if (xfs_sb_version_hasrmapbt(&rcur->bc_mp->m_sb)) {
1538 		error = xfs_rmap_free_extent(rcur->bc_mp, dfops,
1539 				rcur->bc_private.a.agno,
1540 				agbno, aglen, XFS_RMAP_OWN_COW);
1541 		if (error)
1542 			return error;
1543 	}
1544 
1545 	return error;
1546 }
1547 
1548 /* Record a CoW staging extent in the refcount btree. */
1549 int
xfs_refcount_alloc_cow_extent(struct xfs_mount * mp,struct xfs_defer_ops * dfops,xfs_fsblock_t fsb,xfs_extlen_t len)1550 xfs_refcount_alloc_cow_extent(
1551 	struct xfs_mount		*mp,
1552 	struct xfs_defer_ops		*dfops,
1553 	xfs_fsblock_t			fsb,
1554 	xfs_extlen_t			len)
1555 {
1556 	if (!xfs_sb_version_hasreflink(&mp->m_sb))
1557 		return 0;
1558 
1559 	return __xfs_refcount_add(mp, dfops, XFS_REFCOUNT_ALLOC_COW,
1560 			fsb, len);
1561 }
1562 
1563 /* Forget a CoW staging event in the refcount btree. */
1564 int
xfs_refcount_free_cow_extent(struct xfs_mount * mp,struct xfs_defer_ops * dfops,xfs_fsblock_t fsb,xfs_extlen_t len)1565 xfs_refcount_free_cow_extent(
1566 	struct xfs_mount		*mp,
1567 	struct xfs_defer_ops		*dfops,
1568 	xfs_fsblock_t			fsb,
1569 	xfs_extlen_t			len)
1570 {
1571 	if (!xfs_sb_version_hasreflink(&mp->m_sb))
1572 		return 0;
1573 
1574 	return __xfs_refcount_add(mp, dfops, XFS_REFCOUNT_FREE_COW,
1575 			fsb, len);
1576 }
1577 
1578 struct xfs_refcount_recovery {
1579 	struct list_head		rr_list;
1580 	struct xfs_refcount_irec	rr_rrec;
1581 };
1582 
1583 /* Stuff an extent on the recovery list. */
1584 STATIC int
xfs_refcount_recover_extent(struct xfs_btree_cur * cur,union xfs_btree_rec * rec,void * priv)1585 xfs_refcount_recover_extent(
1586 	struct xfs_btree_cur		*cur,
1587 	union xfs_btree_rec		*rec,
1588 	void				*priv)
1589 {
1590 	struct list_head		*debris = priv;
1591 	struct xfs_refcount_recovery	*rr;
1592 
1593 	if (be32_to_cpu(rec->refc.rc_refcount) != 1)
1594 		return -EFSCORRUPTED;
1595 
1596 	rr = kmem_alloc(sizeof(struct xfs_refcount_recovery), KM_SLEEP);
1597 	xfs_refcount_btrec_to_irec(rec, &rr->rr_rrec);
1598 	list_add_tail(&rr->rr_list, debris);
1599 
1600 	return 0;
1601 }
1602 
1603 /* Find and remove leftover CoW reservations. */
1604 int
xfs_refcount_recover_cow_leftovers(struct xfs_mount * mp,xfs_agnumber_t agno)1605 xfs_refcount_recover_cow_leftovers(
1606 	struct xfs_mount		*mp,
1607 	xfs_agnumber_t			agno)
1608 {
1609 	struct xfs_trans		*tp;
1610 	struct xfs_btree_cur		*cur;
1611 	struct xfs_buf			*agbp;
1612 	struct xfs_refcount_recovery	*rr, *n;
1613 	struct list_head		debris;
1614 	union xfs_btree_irec		low;
1615 	union xfs_btree_irec		high;
1616 	struct xfs_defer_ops		dfops;
1617 	xfs_fsblock_t			fsb;
1618 	xfs_agblock_t			agbno;
1619 	int				error;
1620 
1621 	if (mp->m_sb.sb_agblocks >= XFS_REFC_COW_START)
1622 		return -EOPNOTSUPP;
1623 
1624 	INIT_LIST_HEAD(&debris);
1625 
1626 	/*
1627 	 * In this first part, we use an empty transaction to gather up
1628 	 * all the leftover CoW extents so that we can subsequently
1629 	 * delete them.  The empty transaction is used to avoid
1630 	 * a buffer lock deadlock if there happens to be a loop in the
1631 	 * refcountbt because we're allowed to re-grab a buffer that is
1632 	 * already attached to our transaction.  When we're done
1633 	 * recording the CoW debris we cancel the (empty) transaction
1634 	 * and everything goes away cleanly.
1635 	 */
1636 	error = xfs_trans_alloc_empty(mp, &tp);
1637 	if (error)
1638 		return error;
1639 
1640 	error = xfs_alloc_read_agf(mp, tp, agno, 0, &agbp);
1641 	if (error)
1642 		goto out_trans;
1643 	if (!agbp) {
1644 		error = -ENOMEM;
1645 		goto out_trans;
1646 	}
1647 	cur = xfs_refcountbt_init_cursor(mp, tp, agbp, agno, NULL);
1648 
1649 	/* Find all the leftover CoW staging extents. */
1650 	memset(&low, 0, sizeof(low));
1651 	memset(&high, 0, sizeof(high));
1652 	low.rc.rc_startblock = XFS_REFC_COW_START;
1653 	high.rc.rc_startblock = -1U;
1654 	error = xfs_btree_query_range(cur, &low, &high,
1655 			xfs_refcount_recover_extent, &debris);
1656 	if (error)
1657 		goto out_cursor;
1658 	xfs_btree_del_cursor(cur, XFS_BTREE_NOERROR);
1659 	xfs_trans_brelse(tp, agbp);
1660 	xfs_trans_cancel(tp);
1661 
1662 	/* Now iterate the list to free the leftovers */
1663 	list_for_each_entry_safe(rr, n, &debris, rr_list) {
1664 		/* Set up transaction. */
1665 		error = xfs_trans_alloc(mp, &M_RES(mp)->tr_write, 0, 0, 0, &tp);
1666 		if (error)
1667 			goto out_free;
1668 
1669 		trace_xfs_refcount_recover_extent(mp, agno, &rr->rr_rrec);
1670 
1671 		/* Free the orphan record */
1672 		xfs_defer_init(&dfops, &fsb);
1673 		agbno = rr->rr_rrec.rc_startblock - XFS_REFC_COW_START;
1674 		fsb = XFS_AGB_TO_FSB(mp, agno, agbno);
1675 		error = xfs_refcount_free_cow_extent(mp, &dfops, fsb,
1676 				rr->rr_rrec.rc_blockcount);
1677 		if (error)
1678 			goto out_defer;
1679 
1680 		/* Free the block. */
1681 		xfs_bmap_add_free(mp, &dfops, fsb,
1682 				rr->rr_rrec.rc_blockcount, NULL);
1683 
1684 		error = xfs_defer_finish(&tp, &dfops, NULL);
1685 		if (error)
1686 			goto out_defer;
1687 
1688 		error = xfs_trans_commit(tp);
1689 		if (error)
1690 			goto out_free;
1691 
1692 		list_del(&rr->rr_list);
1693 		kmem_free(rr);
1694 	}
1695 
1696 	return error;
1697 out_defer:
1698 	xfs_defer_cancel(&dfops);
1699 out_trans:
1700 	xfs_trans_cancel(tp);
1701 out_free:
1702 	/* Free the leftover list */
1703 	list_for_each_entry_safe(rr, n, &debris, rr_list) {
1704 		list_del(&rr->rr_list);
1705 		kmem_free(rr);
1706 	}
1707 	return error;
1708 
1709 out_cursor:
1710 	xfs_btree_del_cursor(cur, XFS_BTREE_ERROR);
1711 	xfs_trans_brelse(tp, agbp);
1712 	goto out_trans;
1713 }
1714