Lines Matching refs:io_ctl
40 struct btrfs_io_ctl *io_ctl,
300 static int io_ctl_init(struct btrfs_io_ctl *io_ctl, struct inode *inode, in io_ctl_init() argument
316 memset(io_ctl, 0, sizeof(struct btrfs_io_ctl)); in io_ctl_init()
318 io_ctl->pages = kcalloc(num_pages, sizeof(struct page *), GFP_NOFS); in io_ctl_init()
319 if (!io_ctl->pages) in io_ctl_init()
322 io_ctl->num_pages = num_pages; in io_ctl_init()
323 io_ctl->fs_info = btrfs_sb(inode->i_sb); in io_ctl_init()
324 io_ctl->check_crcs = check_crcs; in io_ctl_init()
325 io_ctl->inode = inode; in io_ctl_init()
331 static void io_ctl_free(struct btrfs_io_ctl *io_ctl) in io_ctl_free() argument
333 kfree(io_ctl->pages); in io_ctl_free()
334 io_ctl->pages = NULL; in io_ctl_free()
337 static void io_ctl_unmap_page(struct btrfs_io_ctl *io_ctl) in io_ctl_unmap_page() argument
339 if (io_ctl->cur) { in io_ctl_unmap_page()
340 io_ctl->cur = NULL; in io_ctl_unmap_page()
341 io_ctl->orig = NULL; in io_ctl_unmap_page()
345 static void io_ctl_map_page(struct btrfs_io_ctl *io_ctl, int clear) in io_ctl_map_page() argument
347 ASSERT(io_ctl->index < io_ctl->num_pages); in io_ctl_map_page()
348 io_ctl->page = io_ctl->pages[io_ctl->index++]; in io_ctl_map_page()
349 io_ctl->cur = page_address(io_ctl->page); in io_ctl_map_page()
350 io_ctl->orig = io_ctl->cur; in io_ctl_map_page()
351 io_ctl->size = PAGE_SIZE; in io_ctl_map_page()
353 clear_page(io_ctl->cur); in io_ctl_map_page()
356 static void io_ctl_drop_pages(struct btrfs_io_ctl *io_ctl) in io_ctl_drop_pages() argument
360 io_ctl_unmap_page(io_ctl); in io_ctl_drop_pages()
362 for (i = 0; i < io_ctl->num_pages; i++) { in io_ctl_drop_pages()
363 if (io_ctl->pages[i]) { in io_ctl_drop_pages()
364 ClearPageChecked(io_ctl->pages[i]); in io_ctl_drop_pages()
365 unlock_page(io_ctl->pages[i]); in io_ctl_drop_pages()
366 put_page(io_ctl->pages[i]); in io_ctl_drop_pages()
371 static int io_ctl_prepare_pages(struct btrfs_io_ctl *io_ctl, struct inode *inode, in io_ctl_prepare_pages() argument
378 for (i = 0; i < io_ctl->num_pages; i++) { in io_ctl_prepare_pages()
381 io_ctl_drop_pages(io_ctl); in io_ctl_prepare_pages()
384 io_ctl->pages[i] = page; in io_ctl_prepare_pages()
391 io_ctl_drop_pages(io_ctl); in io_ctl_prepare_pages()
397 io_ctl_drop_pages(io_ctl); in io_ctl_prepare_pages()
403 for (i = 0; i < io_ctl->num_pages; i++) { in io_ctl_prepare_pages()
404 clear_page_dirty_for_io(io_ctl->pages[i]); in io_ctl_prepare_pages()
405 set_page_extent_mapped(io_ctl->pages[i]); in io_ctl_prepare_pages()
411 static void io_ctl_set_generation(struct btrfs_io_ctl *io_ctl, u64 generation) in io_ctl_set_generation() argument
415 io_ctl_map_page(io_ctl, 1); in io_ctl_set_generation()
421 if (io_ctl->check_crcs) { in io_ctl_set_generation()
422 io_ctl->cur += (sizeof(u32) * io_ctl->num_pages); in io_ctl_set_generation()
423 io_ctl->size -= sizeof(u64) + (sizeof(u32) * io_ctl->num_pages); in io_ctl_set_generation()
425 io_ctl->cur += sizeof(u64); in io_ctl_set_generation()
426 io_ctl->size -= sizeof(u64) * 2; in io_ctl_set_generation()
429 val = io_ctl->cur; in io_ctl_set_generation()
431 io_ctl->cur += sizeof(u64); in io_ctl_set_generation()
434 static int io_ctl_check_generation(struct btrfs_io_ctl *io_ctl, u64 generation) in io_ctl_check_generation() argument
442 if (io_ctl->check_crcs) { in io_ctl_check_generation()
443 io_ctl->cur += sizeof(u32) * io_ctl->num_pages; in io_ctl_check_generation()
444 io_ctl->size -= sizeof(u64) + in io_ctl_check_generation()
445 (sizeof(u32) * io_ctl->num_pages); in io_ctl_check_generation()
447 io_ctl->cur += sizeof(u64); in io_ctl_check_generation()
448 io_ctl->size -= sizeof(u64) * 2; in io_ctl_check_generation()
451 gen = io_ctl->cur; in io_ctl_check_generation()
453 btrfs_err_rl(io_ctl->fs_info, in io_ctl_check_generation()
456 io_ctl_unmap_page(io_ctl); in io_ctl_check_generation()
459 io_ctl->cur += sizeof(u64); in io_ctl_check_generation()
463 static void io_ctl_set_crc(struct btrfs_io_ctl *io_ctl, int index) in io_ctl_set_crc() argument
469 if (!io_ctl->check_crcs) { in io_ctl_set_crc()
470 io_ctl_unmap_page(io_ctl); in io_ctl_set_crc()
475 offset = sizeof(u32) * io_ctl->num_pages; in io_ctl_set_crc()
477 crc = btrfs_crc32c(crc, io_ctl->orig + offset, PAGE_SIZE - offset); in io_ctl_set_crc()
479 io_ctl_unmap_page(io_ctl); in io_ctl_set_crc()
480 tmp = page_address(io_ctl->pages[0]); in io_ctl_set_crc()
485 static int io_ctl_check_crc(struct btrfs_io_ctl *io_ctl, int index) in io_ctl_check_crc() argument
491 if (!io_ctl->check_crcs) { in io_ctl_check_crc()
492 io_ctl_map_page(io_ctl, 0); in io_ctl_check_crc()
497 offset = sizeof(u32) * io_ctl->num_pages; in io_ctl_check_crc()
499 tmp = page_address(io_ctl->pages[0]); in io_ctl_check_crc()
503 io_ctl_map_page(io_ctl, 0); in io_ctl_check_crc()
504 crc = btrfs_crc32c(crc, io_ctl->orig + offset, PAGE_SIZE - offset); in io_ctl_check_crc()
507 btrfs_err_rl(io_ctl->fs_info, in io_ctl_check_crc()
509 io_ctl_unmap_page(io_ctl); in io_ctl_check_crc()
516 static int io_ctl_add_entry(struct btrfs_io_ctl *io_ctl, u64 offset, u64 bytes, in io_ctl_add_entry() argument
521 if (!io_ctl->cur) in io_ctl_add_entry()
524 entry = io_ctl->cur; in io_ctl_add_entry()
529 io_ctl->cur += sizeof(struct btrfs_free_space_entry); in io_ctl_add_entry()
530 io_ctl->size -= sizeof(struct btrfs_free_space_entry); in io_ctl_add_entry()
532 if (io_ctl->size >= sizeof(struct btrfs_free_space_entry)) in io_ctl_add_entry()
535 io_ctl_set_crc(io_ctl, io_ctl->index - 1); in io_ctl_add_entry()
538 if (io_ctl->index >= io_ctl->num_pages) in io_ctl_add_entry()
542 io_ctl_map_page(io_ctl, 1); in io_ctl_add_entry()
546 static int io_ctl_add_bitmap(struct btrfs_io_ctl *io_ctl, void *bitmap) in io_ctl_add_bitmap() argument
548 if (!io_ctl->cur) in io_ctl_add_bitmap()
555 if (io_ctl->cur != io_ctl->orig) { in io_ctl_add_bitmap()
556 io_ctl_set_crc(io_ctl, io_ctl->index - 1); in io_ctl_add_bitmap()
557 if (io_ctl->index >= io_ctl->num_pages) in io_ctl_add_bitmap()
559 io_ctl_map_page(io_ctl, 0); in io_ctl_add_bitmap()
562 copy_page(io_ctl->cur, bitmap); in io_ctl_add_bitmap()
563 io_ctl_set_crc(io_ctl, io_ctl->index - 1); in io_ctl_add_bitmap()
564 if (io_ctl->index < io_ctl->num_pages) in io_ctl_add_bitmap()
565 io_ctl_map_page(io_ctl, 0); in io_ctl_add_bitmap()
569 static void io_ctl_zero_remaining_pages(struct btrfs_io_ctl *io_ctl) in io_ctl_zero_remaining_pages() argument
575 if (io_ctl->cur != io_ctl->orig) in io_ctl_zero_remaining_pages()
576 io_ctl_set_crc(io_ctl, io_ctl->index - 1); in io_ctl_zero_remaining_pages()
578 io_ctl_unmap_page(io_ctl); in io_ctl_zero_remaining_pages()
580 while (io_ctl->index < io_ctl->num_pages) { in io_ctl_zero_remaining_pages()
581 io_ctl_map_page(io_ctl, 1); in io_ctl_zero_remaining_pages()
582 io_ctl_set_crc(io_ctl, io_ctl->index - 1); in io_ctl_zero_remaining_pages()
586 static int io_ctl_read_entry(struct btrfs_io_ctl *io_ctl, in io_ctl_read_entry() argument
592 if (!io_ctl->cur) { in io_ctl_read_entry()
593 ret = io_ctl_check_crc(io_ctl, io_ctl->index); in io_ctl_read_entry()
598 e = io_ctl->cur; in io_ctl_read_entry()
602 io_ctl->cur += sizeof(struct btrfs_free_space_entry); in io_ctl_read_entry()
603 io_ctl->size -= sizeof(struct btrfs_free_space_entry); in io_ctl_read_entry()
605 if (io_ctl->size >= sizeof(struct btrfs_free_space_entry)) in io_ctl_read_entry()
608 io_ctl_unmap_page(io_ctl); in io_ctl_read_entry()
613 static int io_ctl_read_bitmap(struct btrfs_io_ctl *io_ctl, in io_ctl_read_bitmap() argument
618 ret = io_ctl_check_crc(io_ctl, io_ctl->index); in io_ctl_read_bitmap()
622 copy_page(entry->bitmap, io_ctl->cur); in io_ctl_read_bitmap()
623 io_ctl_unmap_page(io_ctl); in io_ctl_read_bitmap()
673 struct btrfs_io_ctl io_ctl; in __load_free_space_cache() local
726 ret = io_ctl_init(&io_ctl, inode, 0); in __load_free_space_cache()
732 ret = io_ctl_prepare_pages(&io_ctl, inode, 1); in __load_free_space_cache()
736 ret = io_ctl_check_crc(&io_ctl, 0); in __load_free_space_cache()
740 ret = io_ctl_check_generation(&io_ctl, generation); in __load_free_space_cache()
752 ret = io_ctl_read_entry(&io_ctl, e, &type); in __load_free_space_cache()
803 io_ctl_unmap_page(&io_ctl); in __load_free_space_cache()
811 ret = io_ctl_read_bitmap(&io_ctl, e); in __load_free_space_cache()
816 io_ctl_drop_pages(&io_ctl); in __load_free_space_cache()
820 io_ctl_free(&io_ctl); in __load_free_space_cache()
823 io_ctl_drop_pages(&io_ctl); in __load_free_space_cache()
925 int write_cache_extent_entries(struct btrfs_io_ctl *io_ctl, in write_cache_extent_entries() argument
958 ret = io_ctl_add_entry(io_ctl, e->offset, e->bytes, in write_cache_extent_entries()
987 ret = io_ctl_add_entry(io_ctl, trim_entry->start, in write_cache_extent_entries()
1056 struct btrfs_io_ctl *io_ctl, in write_pinned_extent_entries() argument
1095 ret = io_ctl_add_entry(io_ctl, extent_start, len, NULL); in write_pinned_extent_entries()
1106 write_bitmap_entries(struct btrfs_io_ctl *io_ctl, struct list_head *bitmap_list) in write_bitmap_entries() argument
1113 ret = io_ctl_add_bitmap(io_ctl, entry->bitmap); in write_bitmap_entries()
1145 struct btrfs_io_ctl *io_ctl, in cleanup_write_cache_enospc() argument
1148 io_ctl_drop_pages(io_ctl); in cleanup_write_cache_enospc()
1156 struct btrfs_io_ctl *io_ctl, in __btrfs_wait_cache_io() argument
1160 struct inode *inode = io_ctl->inode; in __btrfs_wait_cache_io()
1172 io_ctl->entries, io_ctl->bitmaps); in __btrfs_wait_cache_io()
1206 io_ctl->inode = NULL; in __btrfs_wait_cache_io()
1216 struct btrfs_io_ctl *io_ctl, in btrfs_wait_cache_io_root() argument
1219 return __btrfs_wait_cache_io(root, trans, NULL, io_ctl, path, 0); in btrfs_wait_cache_io_root()
1227 block_group, &block_group->io_ctl, in btrfs_wait_cache_io()
1245 struct btrfs_io_ctl *io_ctl, in __btrfs_write_out_cache() argument
1258 WARN_ON(io_ctl->pages); in __btrfs_write_out_cache()
1259 ret = io_ctl_init(io_ctl, inode, 1); in __btrfs_write_out_cache()
1279 ret = io_ctl_prepare_pages(io_ctl, inode, 0); in __btrfs_write_out_cache()
1286 io_ctl_set_generation(io_ctl, trans->transid); in __btrfs_write_out_cache()
1291 ret = write_cache_extent_entries(io_ctl, ctl, in __btrfs_write_out_cache()
1305 ret = write_pinned_extent_entries(block_group, io_ctl, &entries); in __btrfs_write_out_cache()
1314 ret = write_bitmap_entries(io_ctl, &bitmap_list); in __btrfs_write_out_cache()
1321 io_ctl_zero_remaining_pages(io_ctl); in __btrfs_write_out_cache()
1324 ret = btrfs_dirty_pages(inode, io_ctl->pages, io_ctl->num_pages, 0, in __btrfs_write_out_cache()
1335 io_ctl_drop_pages(io_ctl); in __btrfs_write_out_cache()
1336 io_ctl_free(io_ctl); in __btrfs_write_out_cache()
1346 io_ctl->entries = entries; in __btrfs_write_out_cache()
1347 io_ctl->bitmaps = bitmaps; in __btrfs_write_out_cache()
1356 io_ctl->inode = NULL; in __btrfs_write_out_cache()
1357 io_ctl_free(io_ctl); in __btrfs_write_out_cache()
1373 cleanup_write_cache_enospc(inode, io_ctl, &cached_state); in __btrfs_write_out_cache()
1403 block_group, &block_group->io_ctl, trans); in btrfs_write_out_cache()
1414 block_group->io_ctl.inode = NULL; in btrfs_write_out_cache()
3564 struct btrfs_io_ctl io_ctl; in btrfs_write_out_ino_cache() local
3570 memset(&io_ctl, 0, sizeof(io_ctl)); in btrfs_write_out_ino_cache()
3571 ret = __btrfs_write_out_cache(root, inode, ctl, NULL, &io_ctl, trans); in btrfs_write_out_ino_cache()
3580 ret = btrfs_wait_cache_io_root(root, trans, &io_ctl, path); in btrfs_write_out_ino_cache()