1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright (C) Sistina Software, Inc. 1997-2003 All rights reserved.
4 * Copyright (C) 2004-2008 Red Hat, Inc. All rights reserved.
5 */
6
7 #include <linux/sched.h>
8 #include <linux/slab.h>
9 #include <linux/spinlock.h>
10 #include <linux/completion.h>
11 #include <linux/buffer_head.h>
12 #include <linux/mm.h>
13 #include <linux/pagemap.h>
14 #include <linux/writeback.h>
15 #include <linux/swap.h>
16 #include <linux/delay.h>
17 #include <linux/bio.h>
18 #include <linux/gfs2_ondisk.h>
19
20 #include "gfs2.h"
21 #include "incore.h"
22 #include "glock.h"
23 #include "glops.h"
24 #include "inode.h"
25 #include "log.h"
26 #include "lops.h"
27 #include "meta_io.h"
28 #include "rgrp.h"
29 #include "trans.h"
30 #include "util.h"
31 #include "trace_gfs2.h"
32
gfs2_aspace_write_folio(struct folio * folio,struct writeback_control * wbc)33 static void gfs2_aspace_write_folio(struct folio *folio,
34 struct writeback_control *wbc)
35 {
36 struct buffer_head *bh, *head;
37 int nr_underway = 0;
38 blk_opf_t write_flags = REQ_META | REQ_PRIO | wbc_to_write_flags(wbc);
39
40 BUG_ON(!folio_test_locked(folio));
41
42 head = folio_buffers(folio);
43 bh = head;
44
45 do {
46 if (!buffer_mapped(bh))
47 continue;
48 /*
49 * If it's a fully non-blocking write attempt and we cannot
50 * lock the buffer then redirty the page. Note that this can
51 * potentially cause a busy-wait loop from flusher thread and kswapd
52 * activity, but those code paths have their own higher-level
53 * throttling.
54 */
55 if (wbc->sync_mode != WB_SYNC_NONE) {
56 lock_buffer(bh);
57 } else if (!trylock_buffer(bh)) {
58 folio_redirty_for_writepage(wbc, folio);
59 continue;
60 }
61 if (test_clear_buffer_dirty(bh)) {
62 mark_buffer_async_write(bh);
63 } else {
64 unlock_buffer(bh);
65 }
66 } while ((bh = bh->b_this_page) != head);
67
68 /*
69 * The folio and its buffers are protected from truncation by
70 * the writeback flag, so we can drop the bh refcounts early.
71 */
72 BUG_ON(folio_test_writeback(folio));
73 folio_start_writeback(folio);
74
75 do {
76 struct buffer_head *next = bh->b_this_page;
77 if (buffer_async_write(bh)) {
78 submit_bh(REQ_OP_WRITE | write_flags, bh);
79 nr_underway++;
80 }
81 bh = next;
82 } while (bh != head);
83 folio_unlock(folio);
84
85 if (nr_underway == 0)
86 folio_end_writeback(folio);
87 }
88
gfs2_aspace_writepages(struct address_space * mapping,struct writeback_control * wbc)89 static int gfs2_aspace_writepages(struct address_space *mapping,
90 struct writeback_control *wbc)
91 {
92 struct folio *folio = NULL;
93 int error;
94
95 while ((folio = writeback_iter(mapping, wbc, folio, &error)))
96 gfs2_aspace_write_folio(folio, wbc);
97
98 return error;
99 }
100
101 const struct address_space_operations gfs2_meta_aops = {
102 .dirty_folio = block_dirty_folio,
103 .invalidate_folio = block_invalidate_folio,
104 .writepages = gfs2_aspace_writepages,
105 .release_folio = gfs2_release_folio,
106 .migrate_folio = buffer_migrate_folio_norefs,
107 };
108
109 const struct address_space_operations gfs2_rgrp_aops = {
110 .dirty_folio = block_dirty_folio,
111 .invalidate_folio = block_invalidate_folio,
112 .writepages = gfs2_aspace_writepages,
113 .release_folio = gfs2_release_folio,
114 .migrate_folio = buffer_migrate_folio_norefs,
115 };
116
117 /**
118 * gfs2_getbuf - Get a buffer with a given address space
119 * @gl: the glock
120 * @blkno: the block number (filesystem scope)
121 * @create: 1 if the buffer should be created
122 *
123 * Returns: the buffer
124 */
125
gfs2_getbuf(struct gfs2_glock * gl,u64 blkno,int create)126 struct buffer_head *gfs2_getbuf(struct gfs2_glock *gl, u64 blkno, int create)
127 {
128 struct address_space *mapping = gfs2_glock2aspace(gl);
129 struct gfs2_sbd *sdp = gl->gl_name.ln_sbd;
130 struct folio *folio;
131 struct buffer_head *bh;
132 unsigned int shift;
133 unsigned long index;
134 unsigned int bufnum;
135
136 if (mapping == NULL)
137 mapping = gfs2_aspace(sdp);
138
139 shift = PAGE_SHIFT - sdp->sd_sb.sb_bsize_shift;
140 index = blkno >> shift; /* convert block to page */
141 bufnum = blkno - (index << shift); /* block buf index within page */
142
143 if (create) {
144 folio = __filemap_get_folio(mapping, index,
145 FGP_LOCK | FGP_ACCESSED | FGP_CREAT,
146 mapping_gfp_mask(mapping) | __GFP_NOFAIL);
147 bh = folio_buffers(folio);
148 if (!bh)
149 bh = create_empty_buffers(folio,
150 sdp->sd_sb.sb_bsize, 0);
151 } else {
152 folio = __filemap_get_folio(mapping, index,
153 FGP_LOCK | FGP_ACCESSED, 0);
154 if (IS_ERR(folio))
155 return NULL;
156 bh = folio_buffers(folio);
157 }
158
159 if (!bh)
160 goto out_unlock;
161
162 bh = get_nth_bh(bh, bufnum);
163 if (!buffer_mapped(bh))
164 map_bh(bh, sdp->sd_vfs, blkno);
165
166 out_unlock:
167 folio_unlock(folio);
168 folio_put(folio);
169
170 return bh;
171 }
172
meta_prep_new(struct buffer_head * bh)173 static void meta_prep_new(struct buffer_head *bh)
174 {
175 struct gfs2_meta_header *mh = (struct gfs2_meta_header *)bh->b_data;
176
177 lock_buffer(bh);
178 clear_buffer_dirty(bh);
179 set_buffer_uptodate(bh);
180 unlock_buffer(bh);
181
182 mh->mh_magic = cpu_to_be32(GFS2_MAGIC);
183 }
184
185 /**
186 * gfs2_meta_new - Get a block
187 * @gl: The glock associated with this block
188 * @blkno: The block number
189 *
190 * Returns: The buffer
191 */
192
gfs2_meta_new(struct gfs2_glock * gl,u64 blkno)193 struct buffer_head *gfs2_meta_new(struct gfs2_glock *gl, u64 blkno)
194 {
195 struct buffer_head *bh;
196 bh = gfs2_getbuf(gl, blkno, CREATE);
197 meta_prep_new(bh);
198 return bh;
199 }
200
gfs2_meta_read_endio(struct bio * bio)201 static void gfs2_meta_read_endio(struct bio *bio)
202 {
203 struct bio_vec *bvec;
204 struct bvec_iter_all iter_all;
205
206 bio_for_each_segment_all(bvec, bio, iter_all) {
207 struct page *page = bvec->bv_page;
208 struct buffer_head *bh = page_buffers(page);
209 unsigned int len = bvec->bv_len;
210
211 while (bh_offset(bh) < bvec->bv_offset)
212 bh = bh->b_this_page;
213 do {
214 struct buffer_head *next = bh->b_this_page;
215 len -= bh->b_size;
216 bh->b_end_io(bh, !bio->bi_status);
217 bh = next;
218 } while (bh && len);
219 }
220 bio_put(bio);
221 }
222
223 /*
224 * Submit several consecutive buffer head I/O requests as a single bio I/O
225 * request. (See submit_bh_wbc.)
226 */
gfs2_submit_bhs(blk_opf_t opf,struct buffer_head * bhs[],int num)227 static void gfs2_submit_bhs(blk_opf_t opf, struct buffer_head *bhs[], int num)
228 {
229 while (num > 0) {
230 struct buffer_head *bh = *bhs;
231 struct bio *bio;
232
233 bio = bio_alloc(bh->b_bdev, num, opf, GFP_NOIO);
234 bio->bi_iter.bi_sector = bh->b_blocknr * (bh->b_size >> 9);
235 while (num > 0) {
236 bh = *bhs;
237 if (!bio_add_page(bio, bh->b_page, bh->b_size, bh_offset(bh))) {
238 BUG_ON(bio->bi_iter.bi_size == 0);
239 break;
240 }
241 bhs++;
242 num--;
243 }
244 bio->bi_end_io = gfs2_meta_read_endio;
245 submit_bio(bio);
246 }
247 }
248
249 /**
250 * gfs2_meta_read - Read a block from disk
251 * @gl: The glock covering the block
252 * @blkno: The block number
253 * @flags: flags
254 * @rahead: Do read-ahead
255 * @bhp: the place where the buffer is returned (NULL on failure)
256 *
257 * Returns: errno
258 */
259
gfs2_meta_read(struct gfs2_glock * gl,u64 blkno,int flags,int rahead,struct buffer_head ** bhp)260 int gfs2_meta_read(struct gfs2_glock *gl, u64 blkno, int flags,
261 int rahead, struct buffer_head **bhp)
262 {
263 struct gfs2_sbd *sdp = gl->gl_name.ln_sbd;
264 struct buffer_head *bh, *bhs[2];
265 int num = 0;
266
267 if (gfs2_withdrawing_or_withdrawn(sdp) &&
268 !gfs2_withdraw_in_prog(sdp)) {
269 *bhp = NULL;
270 return -EIO;
271 }
272
273 *bhp = bh = gfs2_getbuf(gl, blkno, CREATE);
274
275 lock_buffer(bh);
276 if (buffer_uptodate(bh)) {
277 unlock_buffer(bh);
278 flags &= ~DIO_WAIT;
279 } else {
280 bh->b_end_io = end_buffer_read_sync;
281 get_bh(bh);
282 bhs[num++] = bh;
283 }
284
285 if (rahead) {
286 bh = gfs2_getbuf(gl, blkno + 1, CREATE);
287
288 lock_buffer(bh);
289 if (buffer_uptodate(bh)) {
290 unlock_buffer(bh);
291 brelse(bh);
292 } else {
293 bh->b_end_io = end_buffer_read_sync;
294 bhs[num++] = bh;
295 }
296 }
297
298 gfs2_submit_bhs(REQ_OP_READ | REQ_META | REQ_PRIO, bhs, num);
299 if (!(flags & DIO_WAIT))
300 return 0;
301
302 bh = *bhp;
303 wait_on_buffer(bh);
304 if (unlikely(!buffer_uptodate(bh))) {
305 struct gfs2_trans *tr = current->journal_info;
306 if (tr && test_bit(TR_TOUCHED, &tr->tr_flags))
307 gfs2_io_error_bh_wd(sdp, bh);
308 brelse(bh);
309 *bhp = NULL;
310 return -EIO;
311 }
312
313 return 0;
314 }
315
316 /**
317 * gfs2_meta_wait - Reread a block from disk
318 * @sdp: the filesystem
319 * @bh: The block to wait for
320 *
321 * Returns: errno
322 */
323
gfs2_meta_wait(struct gfs2_sbd * sdp,struct buffer_head * bh)324 int gfs2_meta_wait(struct gfs2_sbd *sdp, struct buffer_head *bh)
325 {
326 if (gfs2_withdrawing_or_withdrawn(sdp) &&
327 !gfs2_withdraw_in_prog(sdp))
328 return -EIO;
329
330 wait_on_buffer(bh);
331
332 if (!buffer_uptodate(bh)) {
333 struct gfs2_trans *tr = current->journal_info;
334 if (tr && test_bit(TR_TOUCHED, &tr->tr_flags))
335 gfs2_io_error_bh_wd(sdp, bh);
336 return -EIO;
337 }
338 if (gfs2_withdrawing_or_withdrawn(sdp) &&
339 !gfs2_withdraw_in_prog(sdp))
340 return -EIO;
341
342 return 0;
343 }
344
gfs2_remove_from_journal(struct buffer_head * bh,int meta)345 void gfs2_remove_from_journal(struct buffer_head *bh, int meta)
346 {
347 struct address_space *mapping = bh->b_folio->mapping;
348 struct gfs2_sbd *sdp = gfs2_mapping2sbd(mapping);
349 struct gfs2_bufdata *bd = bh->b_private;
350 struct gfs2_trans *tr = current->journal_info;
351 int was_pinned = 0;
352
353 if (test_clear_buffer_pinned(bh)) {
354 trace_gfs2_pin(bd, 0);
355 atomic_dec(&sdp->sd_log_pinned);
356 list_del_init(&bd->bd_list);
357 if (meta == REMOVE_META)
358 tr->tr_num_buf_rm++;
359 else
360 tr->tr_num_databuf_rm++;
361 set_bit(TR_TOUCHED, &tr->tr_flags);
362 was_pinned = 1;
363 brelse(bh);
364 }
365 if (bd) {
366 if (bd->bd_tr) {
367 gfs2_trans_add_revoke(sdp, bd);
368 } else if (was_pinned) {
369 bh->b_private = NULL;
370 kmem_cache_free(gfs2_bufdata_cachep, bd);
371 } else if (!list_empty(&bd->bd_ail_st_list) &&
372 !list_empty(&bd->bd_ail_gl_list)) {
373 gfs2_remove_from_ail(bd);
374 }
375 }
376 clear_buffer_dirty(bh);
377 clear_buffer_uptodate(bh);
378 }
379
380 /**
381 * gfs2_ail1_wipe - remove deleted/freed buffers from the ail1 list
382 * @sdp: superblock
383 * @bstart: starting block address of buffers to remove
384 * @blen: length of buffers to be removed
385 *
386 * This function is called from gfs2_journal wipe, whose job is to remove
387 * buffers, corresponding to deleted blocks, from the journal. If we find any
388 * bufdata elements on the system ail1 list, they haven't been written to
389 * the journal yet. So we remove them.
390 */
gfs2_ail1_wipe(struct gfs2_sbd * sdp,u64 bstart,u32 blen)391 static void gfs2_ail1_wipe(struct gfs2_sbd *sdp, u64 bstart, u32 blen)
392 {
393 struct gfs2_trans *tr, *s;
394 struct gfs2_bufdata *bd, *bs;
395 struct buffer_head *bh;
396 u64 end = bstart + blen;
397
398 gfs2_log_lock(sdp);
399 spin_lock(&sdp->sd_ail_lock);
400 list_for_each_entry_safe(tr, s, &sdp->sd_ail1_list, tr_list) {
401 list_for_each_entry_safe(bd, bs, &tr->tr_ail1_list,
402 bd_ail_st_list) {
403 bh = bd->bd_bh;
404 if (bh->b_blocknr < bstart || bh->b_blocknr >= end)
405 continue;
406
407 gfs2_remove_from_journal(bh, REMOVE_JDATA);
408 }
409 }
410 spin_unlock(&sdp->sd_ail_lock);
411 gfs2_log_unlock(sdp);
412 }
413
gfs2_getjdatabuf(struct gfs2_inode * ip,u64 blkno)414 static struct buffer_head *gfs2_getjdatabuf(struct gfs2_inode *ip, u64 blkno)
415 {
416 struct address_space *mapping = ip->i_inode.i_mapping;
417 struct gfs2_sbd *sdp = GFS2_SB(&ip->i_inode);
418 struct folio *folio;
419 struct buffer_head *bh;
420 unsigned int shift = PAGE_SHIFT - sdp->sd_sb.sb_bsize_shift;
421 unsigned long index = blkno >> shift; /* convert block to page */
422 unsigned int bufnum = blkno - (index << shift);
423
424 folio = __filemap_get_folio(mapping, index, FGP_LOCK | FGP_ACCESSED, 0);
425 if (IS_ERR(folio))
426 return NULL;
427 bh = folio_buffers(folio);
428 if (bh)
429 bh = get_nth_bh(bh, bufnum);
430 folio_unlock(folio);
431 folio_put(folio);
432 return bh;
433 }
434
435 /**
436 * gfs2_journal_wipe - make inode's buffers so they aren't dirty/pinned anymore
437 * @ip: the inode who owns the buffers
438 * @bstart: the first buffer in the run
439 * @blen: the number of buffers in the run
440 *
441 */
442
gfs2_journal_wipe(struct gfs2_inode * ip,u64 bstart,u32 blen)443 void gfs2_journal_wipe(struct gfs2_inode *ip, u64 bstart, u32 blen)
444 {
445 struct gfs2_sbd *sdp = GFS2_SB(&ip->i_inode);
446 struct buffer_head *bh;
447 int ty;
448
449 if (!ip->i_gl) {
450 /* This can only happen during incomplete inode creation. */
451 BUG_ON(!test_bit(GIF_ALLOC_FAILED, &ip->i_flags));
452 return;
453 }
454
455 gfs2_ail1_wipe(sdp, bstart, blen);
456 while (blen) {
457 ty = REMOVE_META;
458 bh = gfs2_getbuf(ip->i_gl, bstart, NO_CREATE);
459 if (!bh && gfs2_is_jdata(ip)) {
460 bh = gfs2_getjdatabuf(ip, bstart);
461 ty = REMOVE_JDATA;
462 }
463 if (bh) {
464 lock_buffer(bh);
465 gfs2_log_lock(sdp);
466 spin_lock(&sdp->sd_ail_lock);
467 gfs2_remove_from_journal(bh, ty);
468 spin_unlock(&sdp->sd_ail_lock);
469 gfs2_log_unlock(sdp);
470 unlock_buffer(bh);
471 brelse(bh);
472 }
473
474 bstart++;
475 blen--;
476 }
477 }
478
479 /**
480 * gfs2_meta_buffer - Get a metadata buffer
481 * @ip: The GFS2 inode
482 * @mtype: The block type (GFS2_METATYPE_*)
483 * @num: The block number (device relative) of the buffer
484 * @bhp: the buffer is returned here
485 *
486 * Returns: errno
487 */
488
gfs2_meta_buffer(struct gfs2_inode * ip,u32 mtype,u64 num,struct buffer_head ** bhp)489 int gfs2_meta_buffer(struct gfs2_inode *ip, u32 mtype, u64 num,
490 struct buffer_head **bhp)
491 {
492 struct gfs2_sbd *sdp = GFS2_SB(&ip->i_inode);
493 struct gfs2_glock *gl = ip->i_gl;
494 struct buffer_head *bh;
495 int ret = 0;
496 int rahead = 0;
497
498 if (num == ip->i_no_addr)
499 rahead = ip->i_rahead;
500
501 ret = gfs2_meta_read(gl, num, DIO_WAIT, rahead, &bh);
502 if (ret == 0 && gfs2_metatype_check(sdp, bh, mtype)) {
503 brelse(bh);
504 ret = -EIO;
505 } else {
506 *bhp = bh;
507 }
508 return ret;
509 }
510
511 /**
512 * gfs2_meta_ra - start readahead on an extent of a file
513 * @gl: the glock the blocks belong to
514 * @dblock: the starting disk block
515 * @extlen: the number of blocks in the extent
516 *
517 * returns: the first buffer in the extent
518 */
519
gfs2_meta_ra(struct gfs2_glock * gl,u64 dblock,u32 extlen)520 struct buffer_head *gfs2_meta_ra(struct gfs2_glock *gl, u64 dblock, u32 extlen)
521 {
522 struct gfs2_sbd *sdp = gl->gl_name.ln_sbd;
523 struct buffer_head *first_bh, *bh;
524 u32 max_ra = gfs2_tune_get(sdp, gt_max_readahead) >>
525 sdp->sd_sb.sb_bsize_shift;
526
527 BUG_ON(!extlen);
528
529 if (max_ra < 1)
530 max_ra = 1;
531 if (extlen > max_ra)
532 extlen = max_ra;
533
534 first_bh = gfs2_getbuf(gl, dblock, CREATE);
535
536 if (buffer_uptodate(first_bh))
537 goto out;
538 bh_read_nowait(first_bh, REQ_META | REQ_PRIO);
539
540 dblock++;
541 extlen--;
542
543 while (extlen) {
544 bh = gfs2_getbuf(gl, dblock, CREATE);
545
546 bh_readahead(bh, REQ_RAHEAD | REQ_META | REQ_PRIO);
547 brelse(bh);
548 dblock++;
549 extlen--;
550 if (!buffer_locked(first_bh) && buffer_uptodate(first_bh))
551 goto out;
552 }
553
554 wait_on_buffer(first_bh);
555 out:
556 return first_bh;
557 }
558
559