Lines Matching full:cmd
273 static inline int superblock_read_lock(struct dm_clone_metadata *cmd, in superblock_read_lock() argument
276 return dm_bm_read_lock(cmd->bm, SUPERBLOCK_LOCATION, &sb_validator, sblock); in superblock_read_lock()
279 static inline int superblock_write_lock(struct dm_clone_metadata *cmd, in superblock_write_lock() argument
282 return dm_bm_write_lock(cmd->bm, SUPERBLOCK_LOCATION, &sb_validator, sblock); in superblock_write_lock()
285 static inline int superblock_write_lock_zero(struct dm_clone_metadata *cmd, in superblock_write_lock_zero() argument
288 return dm_bm_write_lock_zero(cmd->bm, SUPERBLOCK_LOCATION, &sb_validator, sblock); in superblock_write_lock_zero()
291 static int __copy_sm_root(struct dm_clone_metadata *cmd) in __copy_sm_root() argument
296 r = dm_sm_root_size(cmd->sm, &root_size); in __copy_sm_root()
300 return dm_sm_copy_root(cmd->sm, &cmd->metadata_space_map_root, root_size); in __copy_sm_root()
304 static void __prepare_superblock(struct dm_clone_metadata *cmd, in __prepare_superblock() argument
316 memcpy(&sb->metadata_space_map_root, &cmd->metadata_space_map_root, in __prepare_superblock()
317 sizeof(cmd->metadata_space_map_root)); in __prepare_superblock()
319 sb->region_size = cpu_to_le64(cmd->region_size); in __prepare_superblock()
320 sb->target_size = cpu_to_le64(cmd->target_size); in __prepare_superblock()
321 sb->bitset_root = cpu_to_le64(cmd->bitset_root); in __prepare_superblock()
324 static int __open_metadata(struct dm_clone_metadata *cmd) in __open_metadata() argument
330 r = superblock_read_lock(cmd, &sblock); in __open_metadata()
340 if (cmd->region_size != le64_to_cpu(sb->region_size) || in __open_metadata()
341 cmd->target_size != le64_to_cpu(sb->target_size)) { in __open_metadata()
347 r = dm_tm_open_with_sm(cmd->bm, SUPERBLOCK_LOCATION, in __open_metadata()
350 &cmd->tm, &cmd->sm); in __open_metadata()
357 dm_disk_bitset_init(cmd->tm, &cmd->bitset_info); in __open_metadata()
358 cmd->bitset_root = le64_to_cpu(sb->bitset_root); in __open_metadata()
366 static int __format_metadata(struct dm_clone_metadata *cmd) in __format_metadata() argument
372 r = dm_tm_create_with_sm(cmd->bm, SUPERBLOCK_LOCATION, &cmd->tm, &cmd->sm); in __format_metadata()
378 dm_disk_bitset_init(cmd->tm, &cmd->bitset_info); in __format_metadata()
380 r = dm_bitset_empty(&cmd->bitset_info, &cmd->bitset_root); in __format_metadata()
386 r = dm_bitset_resize(&cmd->bitset_info, cmd->bitset_root, 0, in __format_metadata()
387 cmd->nr_regions, false, &cmd->bitset_root); in __format_metadata()
389 DMERR("Failed to resize on-disk bitset to %lu entries", cmd->nr_regions); in __format_metadata()
394 r = dm_tm_pre_commit(cmd->tm); in __format_metadata()
400 r = __copy_sm_root(cmd); in __format_metadata()
406 r = superblock_write_lock_zero(cmd, &sblock); in __format_metadata()
413 __prepare_superblock(cmd, sb); in __format_metadata()
414 r = dm_tm_commit(cmd->tm, sblock); in __format_metadata()
423 dm_sm_destroy(cmd->sm); in __format_metadata()
424 dm_tm_destroy(cmd->tm); in __format_metadata()
429 static int __open_or_format_metadata(struct dm_clone_metadata *cmd, bool may_format_device) in __open_or_format_metadata() argument
434 r = __superblock_all_zeroes(cmd->bm, &formatted); in __open_or_format_metadata()
439 return may_format_device ? __format_metadata(cmd) : -EPERM; in __open_or_format_metadata()
441 return __open_metadata(cmd); in __open_or_format_metadata()
444 static int __create_persistent_data_structures(struct dm_clone_metadata *cmd, in __create_persistent_data_structures() argument
450 cmd->bm = dm_block_manager_create(cmd->bdev, in __create_persistent_data_structures()
453 if (IS_ERR(cmd->bm)) { in __create_persistent_data_structures()
455 return PTR_ERR(cmd->bm); in __create_persistent_data_structures()
458 r = __open_or_format_metadata(cmd, may_format_device); in __create_persistent_data_structures()
460 dm_block_manager_destroy(cmd->bm); in __create_persistent_data_structures()
465 static void __destroy_persistent_data_structures(struct dm_clone_metadata *cmd) in __destroy_persistent_data_structures() argument
467 dm_sm_destroy(cmd->sm); in __destroy_persistent_data_structures()
468 dm_tm_destroy(cmd->tm); in __destroy_persistent_data_structures()
469 dm_block_manager_destroy(cmd->bm); in __destroy_persistent_data_structures()
503 static int dirty_map_init(struct dm_clone_metadata *cmd) in dirty_map_init() argument
505 if (__dirty_map_init(&cmd->dmap[0], cmd->nr_words, cmd->nr_regions)) { in dirty_map_init()
510 if (__dirty_map_init(&cmd->dmap[1], cmd->nr_words, cmd->nr_regions)) { in dirty_map_init()
512 __dirty_map_exit(&cmd->dmap[0]); in dirty_map_init()
516 cmd->current_dmap = &cmd->dmap[0]; in dirty_map_init()
517 cmd->committing_dmap = NULL; in dirty_map_init()
522 static void dirty_map_exit(struct dm_clone_metadata *cmd) in dirty_map_exit() argument
524 __dirty_map_exit(&cmd->dmap[0]); in dirty_map_exit()
525 __dirty_map_exit(&cmd->dmap[1]); in dirty_map_exit()
528 static int __load_bitset_in_core(struct dm_clone_metadata *cmd) in __load_bitset_in_core() argument
535 r = dm_bitset_flush(&cmd->bitset_info, cmd->bitset_root, &cmd->bitset_root); in __load_bitset_in_core()
539 r = dm_bitset_cursor_begin(&cmd->bitset_info, cmd->bitset_root, cmd->nr_regions, &c); in __load_bitset_in_core()
545 __set_bit(i, cmd->region_map); in __load_bitset_in_core()
547 __clear_bit(i, cmd->region_map); in __load_bitset_in_core()
549 if (i >= (cmd->nr_regions - 1)) in __load_bitset_in_core()
568 struct dm_clone_metadata *cmd; in dm_clone_metadata_open() local
570 cmd = kzalloc(sizeof(*cmd), GFP_KERNEL); in dm_clone_metadata_open()
571 if (!cmd) { in dm_clone_metadata_open()
576 cmd->bdev = bdev; in dm_clone_metadata_open()
577 cmd->target_size = target_size; in dm_clone_metadata_open()
578 cmd->region_size = region_size; in dm_clone_metadata_open()
579 cmd->nr_regions = dm_sector_div_up(cmd->target_size, cmd->region_size); in dm_clone_metadata_open()
580 cmd->nr_words = BITS_TO_LONGS(cmd->nr_regions); in dm_clone_metadata_open()
582 init_rwsem(&cmd->lock); in dm_clone_metadata_open()
583 spin_lock_init(&cmd->bitmap_lock); in dm_clone_metadata_open()
584 cmd->read_only = 0; in dm_clone_metadata_open()
585 cmd->fail_io = false; in dm_clone_metadata_open()
586 cmd->hydration_done = false; in dm_clone_metadata_open()
588 cmd->region_map = kvmalloc(bitmap_size(cmd->nr_regions), GFP_KERNEL); in dm_clone_metadata_open()
589 if (!cmd->region_map) { in dm_clone_metadata_open()
595 r = __create_persistent_data_structures(cmd, true); in dm_clone_metadata_open()
599 r = __load_bitset_in_core(cmd); in dm_clone_metadata_open()
605 r = dirty_map_init(cmd); in dm_clone_metadata_open()
609 if (bitmap_full(cmd->region_map, cmd->nr_regions)) in dm_clone_metadata_open()
610 cmd->hydration_done = true; in dm_clone_metadata_open()
612 return cmd; in dm_clone_metadata_open()
615 __destroy_persistent_data_structures(cmd); in dm_clone_metadata_open()
618 kvfree(cmd->region_map); in dm_clone_metadata_open()
621 kfree(cmd); in dm_clone_metadata_open()
626 void dm_clone_metadata_close(struct dm_clone_metadata *cmd) in dm_clone_metadata_close() argument
628 if (!cmd->fail_io) in dm_clone_metadata_close()
629 __destroy_persistent_data_structures(cmd); in dm_clone_metadata_close()
631 dirty_map_exit(cmd); in dm_clone_metadata_close()
632 kvfree(cmd->region_map); in dm_clone_metadata_close()
633 kfree(cmd); in dm_clone_metadata_close()
636 bool dm_clone_is_hydration_done(struct dm_clone_metadata *cmd) in dm_clone_is_hydration_done() argument
638 return cmd->hydration_done; in dm_clone_is_hydration_done()
641 bool dm_clone_is_region_hydrated(struct dm_clone_metadata *cmd, unsigned long region_nr) in dm_clone_is_region_hydrated() argument
643 return dm_clone_is_hydration_done(cmd) || test_bit(region_nr, cmd->region_map); in dm_clone_is_region_hydrated()
646 bool dm_clone_is_range_hydrated(struct dm_clone_metadata *cmd, in dm_clone_is_range_hydrated() argument
651 if (dm_clone_is_hydration_done(cmd)) in dm_clone_is_range_hydrated()
654 bit = find_next_zero_bit(cmd->region_map, cmd->nr_regions, start); in dm_clone_is_range_hydrated()
659 unsigned int dm_clone_nr_of_hydrated_regions(struct dm_clone_metadata *cmd) in dm_clone_nr_of_hydrated_regions() argument
661 return bitmap_weight(cmd->region_map, cmd->nr_regions); in dm_clone_nr_of_hydrated_regions()
664 unsigned long dm_clone_find_next_unhydrated_region(struct dm_clone_metadata *cmd, in dm_clone_find_next_unhydrated_region() argument
667 return find_next_zero_bit(cmd->region_map, cmd->nr_regions, start); in dm_clone_find_next_unhydrated_region()
670 static int __update_metadata_word(struct dm_clone_metadata *cmd, in __update_metadata_word() argument
676 unsigned long max_index = min(cmd->nr_regions, (word + 1) * BITS_PER_LONG); in __update_metadata_word()
680 r = dm_bitset_set_bit(&cmd->bitset_info, cmd->bitset_root, in __update_metadata_word()
681 index, &cmd->bitset_root); in __update_metadata_word()
694 static int __metadata_commit(struct dm_clone_metadata *cmd) in __metadata_commit() argument
701 r = dm_bitset_flush(&cmd->bitset_info, cmd->bitset_root, &cmd->bitset_root); in __metadata_commit()
708 r = dm_tm_pre_commit(cmd->tm); in __metadata_commit()
714 /* Save the space map root in cmd->metadata_space_map_root */ in __metadata_commit()
715 r = __copy_sm_root(cmd); in __metadata_commit()
722 r = superblock_write_lock_zero(cmd, &sblock); in __metadata_commit()
730 __prepare_superblock(cmd, sb); in __metadata_commit()
733 r = dm_tm_commit(cmd->tm, sblock); in __metadata_commit()
742 if (bitmap_full(cmd->region_map, cmd->nr_regions)) in __metadata_commit()
743 cmd->hydration_done = true; in __metadata_commit()
748 static int __flush_dmap(struct dm_clone_metadata *cmd, struct dirty_map *dmap) in __flush_dmap() argument
755 word = find_next_bit(dmap->dirty_words, cmd->nr_words, word); in __flush_dmap()
757 if (word == cmd->nr_words) in __flush_dmap()
760 r = __update_metadata_word(cmd, dmap->dirty_regions, word); in __flush_dmap()
767 } while (word < cmd->nr_words); in __flush_dmap()
769 r = __metadata_commit(cmd); in __flush_dmap()
775 spin_lock_irq(&cmd->bitmap_lock); in __flush_dmap()
777 spin_unlock_irq(&cmd->bitmap_lock); in __flush_dmap()
782 int dm_clone_metadata_pre_commit(struct dm_clone_metadata *cmd) in dm_clone_metadata_pre_commit() argument
787 down_write(&cmd->lock); in dm_clone_metadata_pre_commit()
789 if (cmd->fail_io || dm_bm_is_read_only(cmd->bm)) { in dm_clone_metadata_pre_commit()
795 dmap = cmd->current_dmap; in dm_clone_metadata_pre_commit()
798 next_dmap = (dmap == &cmd->dmap[0]) ? &cmd->dmap[1] : &cmd->dmap[0]; in dm_clone_metadata_pre_commit()
804 if (WARN_ON(next_dmap->changed || cmd->committing_dmap)) { in dm_clone_metadata_pre_commit()
810 spin_lock_irq(&cmd->bitmap_lock); in dm_clone_metadata_pre_commit()
811 cmd->current_dmap = next_dmap; in dm_clone_metadata_pre_commit()
812 spin_unlock_irq(&cmd->bitmap_lock); in dm_clone_metadata_pre_commit()
815 cmd->committing_dmap = dmap; in dm_clone_metadata_pre_commit()
817 up_write(&cmd->lock); in dm_clone_metadata_pre_commit()
822 int dm_clone_metadata_commit(struct dm_clone_metadata *cmd) in dm_clone_metadata_commit() argument
826 down_write(&cmd->lock); in dm_clone_metadata_commit()
828 if (cmd->fail_io || dm_bm_is_read_only(cmd->bm)) in dm_clone_metadata_commit()
831 if (WARN_ON(!cmd->committing_dmap)) { in dm_clone_metadata_commit()
836 r = __flush_dmap(cmd, cmd->committing_dmap); in dm_clone_metadata_commit()
839 cmd->committing_dmap = NULL; in dm_clone_metadata_commit()
842 up_write(&cmd->lock); in dm_clone_metadata_commit()
847 int dm_clone_set_region_hydrated(struct dm_clone_metadata *cmd, unsigned long region_nr) in dm_clone_set_region_hydrated() argument
853 if (unlikely(region_nr >= cmd->nr_regions)) { in dm_clone_set_region_hydrated()
855 region_nr, cmd->nr_regions); in dm_clone_set_region_hydrated()
861 spin_lock_irqsave(&cmd->bitmap_lock, flags); in dm_clone_set_region_hydrated()
863 if (cmd->read_only) { in dm_clone_set_region_hydrated()
868 dmap = cmd->current_dmap; in dm_clone_set_region_hydrated()
872 __set_bit(region_nr, cmd->region_map); in dm_clone_set_region_hydrated()
876 spin_unlock_irqrestore(&cmd->bitmap_lock, flags); in dm_clone_set_region_hydrated()
881 int dm_clone_cond_set_range(struct dm_clone_metadata *cmd, unsigned long start, in dm_clone_cond_set_range() argument
888 if (unlikely(start >= cmd->nr_regions || (start + nr_regions) < start || in dm_clone_cond_set_range()
889 (start + nr_regions) > cmd->nr_regions)) { in dm_clone_cond_set_range()
891 start, nr_regions, cmd->nr_regions); in dm_clone_cond_set_range()
895 spin_lock_irq(&cmd->bitmap_lock); in dm_clone_cond_set_range()
897 if (cmd->read_only) { in dm_clone_cond_set_range()
902 dmap = cmd->current_dmap; in dm_clone_cond_set_range()
904 if (!test_bit(region_nr, cmd->region_map)) { in dm_clone_cond_set_range()
908 __set_bit(region_nr, cmd->region_map); in dm_clone_cond_set_range()
913 spin_unlock_irq(&cmd->bitmap_lock); in dm_clone_cond_set_range()
921 * cmd->region_map without taking the cmd->bitmap_lock spinlock. The only
928 int dm_clone_reload_in_core_bitset(struct dm_clone_metadata *cmd) in dm_clone_reload_in_core_bitset() argument
932 down_write(&cmd->lock); in dm_clone_reload_in_core_bitset()
934 if (cmd->fail_io) in dm_clone_reload_in_core_bitset()
937 r = __load_bitset_in_core(cmd); in dm_clone_reload_in_core_bitset()
939 up_write(&cmd->lock); in dm_clone_reload_in_core_bitset()
944 bool dm_clone_changed_this_transaction(struct dm_clone_metadata *cmd) in dm_clone_changed_this_transaction() argument
949 spin_lock_irqsave(&cmd->bitmap_lock, flags); in dm_clone_changed_this_transaction()
950 r = cmd->dmap[0].changed || cmd->dmap[1].changed; in dm_clone_changed_this_transaction()
951 spin_unlock_irqrestore(&cmd->bitmap_lock, flags); in dm_clone_changed_this_transaction()
956 int dm_clone_metadata_abort(struct dm_clone_metadata *cmd) in dm_clone_metadata_abort() argument
960 down_write(&cmd->lock); in dm_clone_metadata_abort()
962 if (cmd->fail_io || dm_bm_is_read_only(cmd->bm)) in dm_clone_metadata_abort()
965 __destroy_persistent_data_structures(cmd); in dm_clone_metadata_abort()
967 r = __create_persistent_data_structures(cmd, false); in dm_clone_metadata_abort()
970 cmd->fail_io = true; in dm_clone_metadata_abort()
973 up_write(&cmd->lock); in dm_clone_metadata_abort()
978 void dm_clone_metadata_set_read_only(struct dm_clone_metadata *cmd) in dm_clone_metadata_set_read_only() argument
980 down_write(&cmd->lock); in dm_clone_metadata_set_read_only()
982 spin_lock_irq(&cmd->bitmap_lock); in dm_clone_metadata_set_read_only()
983 cmd->read_only = 1; in dm_clone_metadata_set_read_only()
984 spin_unlock_irq(&cmd->bitmap_lock); in dm_clone_metadata_set_read_only()
986 if (!cmd->fail_io) in dm_clone_metadata_set_read_only()
987 dm_bm_set_read_only(cmd->bm); in dm_clone_metadata_set_read_only()
989 up_write(&cmd->lock); in dm_clone_metadata_set_read_only()
992 void dm_clone_metadata_set_read_write(struct dm_clone_metadata *cmd) in dm_clone_metadata_set_read_write() argument
994 down_write(&cmd->lock); in dm_clone_metadata_set_read_write()
996 spin_lock_irq(&cmd->bitmap_lock); in dm_clone_metadata_set_read_write()
997 cmd->read_only = 0; in dm_clone_metadata_set_read_write()
998 spin_unlock_irq(&cmd->bitmap_lock); in dm_clone_metadata_set_read_write()
1000 if (!cmd->fail_io) in dm_clone_metadata_set_read_write()
1001 dm_bm_set_read_write(cmd->bm); in dm_clone_metadata_set_read_write()
1003 up_write(&cmd->lock); in dm_clone_metadata_set_read_write()
1006 int dm_clone_get_free_metadata_block_count(struct dm_clone_metadata *cmd, in dm_clone_get_free_metadata_block_count() argument
1011 down_read(&cmd->lock); in dm_clone_get_free_metadata_block_count()
1013 if (!cmd->fail_io) in dm_clone_get_free_metadata_block_count()
1014 r = dm_sm_get_nr_free(cmd->sm, result); in dm_clone_get_free_metadata_block_count()
1016 up_read(&cmd->lock); in dm_clone_get_free_metadata_block_count()
1021 int dm_clone_get_metadata_dev_size(struct dm_clone_metadata *cmd, in dm_clone_get_metadata_dev_size() argument
1026 down_read(&cmd->lock); in dm_clone_get_metadata_dev_size()
1028 if (!cmd->fail_io) in dm_clone_get_metadata_dev_size()
1029 r = dm_sm_get_nr_blocks(cmd->sm, result); in dm_clone_get_metadata_dev_size()
1031 up_read(&cmd->lock); in dm_clone_get_metadata_dev_size()