519 static inline void init_inc_context(struct inc_context *ic)  in init_inc_context()  argument
521 	ic->bitmap_block = NULL;  in init_inc_context()
522 	ic->bitmap = NULL;  in init_inc_context()
523 	ic->overflow_leaf = NULL;  in init_inc_context()
526 static inline void exit_inc_context(struct ll_disk *ll, struct inc_context *ic)  in exit_inc_context()  argument
528 	if (ic->bitmap_block)  in exit_inc_context()
529 		dm_tm_unlock(ll->tm, ic->bitmap_block);  in exit_inc_context()
530 	if (ic->overflow_leaf)  in exit_inc_context()
531 		dm_tm_unlock(ll->tm, ic->overflow_leaf);  in exit_inc_context()
534 static inline void reset_inc_context(struct ll_disk *ll, struct inc_context *ic)  in reset_inc_context()  argument
536 	exit_inc_context(ll, ic);  in reset_inc_context()
537 	init_inc_context(ic);  in reset_inc_context()
550 static int __sm_ll_inc_overflow(struct ll_disk *ll, dm_block_t b, struct inc_context *ic)  in __sm_ll_inc_overflow()  argument
562 	reset_inc_context(ll, ic);  in __sm_ll_inc_overflow()
565 				     b, &index, &ll->ref_count_root, &ic->overflow_leaf);  in __sm_ll_inc_overflow()
569 	n = dm_block_data(ic->overflow_leaf);  in __sm_ll_inc_overflow()
583 static int sm_ll_inc_overflow(struct ll_disk *ll, dm_block_t b, struct inc_context *ic)  in sm_ll_inc_overflow()  argument
593 	if (ic->overflow_leaf) {  in sm_ll_inc_overflow()
594 		n = dm_block_data(ic->overflow_leaf);  in sm_ll_inc_overflow()
605 	return __sm_ll_inc_overflow(ll, b, ic);  in sm_ll_inc_overflow()
608 static inline int shadow_bitmap(struct ll_disk *ll, struct inc_context *ic)  in shadow_bitmap()  argument
611 	r = dm_tm_shadow_block(ll->tm, le64_to_cpu(ic->ie_disk.blocknr),  in shadow_bitmap()
612 			       &dm_sm_bitmap_validator, &ic->bitmap_block, &inc);  in shadow_bitmap()
617 	ic->ie_disk.blocknr = cpu_to_le64(dm_block_location(ic->bitmap_block));  in shadow_bitmap()
618 	ic->bitmap = dm_bitmap_data(ic->bitmap_block);  in shadow_bitmap()
627 static inline int ensure_bitmap(struct ll_disk *ll, struct inc_context *ic)  in ensure_bitmap()  argument
629 	if (!ic->bitmap_block) {  in ensure_bitmap()
630 		int r = dm_bm_write_lock(dm_tm_get_bm(ll->tm), le64_to_cpu(ic->ie_disk.blocknr),  in ensure_bitmap()
631 					 &dm_sm_bitmap_validator, &ic->bitmap_block);  in ensure_bitmap()
636 		ic->bitmap = dm_bitmap_data(ic->bitmap_block);  in ensure_bitmap()
648 				   struct inc_context *ic)  in sm_ll_inc_bitmap()  argument
660 		r = ensure_bitmap(ll, ic);  in sm_ll_inc_bitmap()
664 		old = sm_lookup_bitmap(ic->bitmap, bit);  in sm_ll_inc_bitmap()
668 			sm_set_bitmap(ic->bitmap, bit, 1);  in sm_ll_inc_bitmap()
671 			le32_add_cpu(&ic->ie_disk.nr_free, -1);  in sm_ll_inc_bitmap()
672 			if (le32_to_cpu(ic->ie_disk.none_free_before) == bit)  in sm_ll_inc_bitmap()
673 				ic->ie_disk.none_free_before = cpu_to_le32(bit + 1);  in sm_ll_inc_bitmap()
678 			sm_set_bitmap(ic->bitmap, bit, 2);  in sm_ll_inc_bitmap()
683 			sm_set_bitmap(ic->bitmap, bit, 3);  in sm_ll_inc_bitmap()
684 			reset_inc_context(ll, ic);  in sm_ll_inc_bitmap()
700 			r = sm_ll_inc_overflow(ll, b, ic);  in sm_ll_inc_bitmap()
718 	struct inc_context ic;  in __sm_ll_inc()  local
722 	init_inc_context(&ic);  in __sm_ll_inc()
725 	r = ll->load_ie(ll, index, &ic.ie_disk);  in __sm_ll_inc()
729 	r = shadow_bitmap(ll, &ic);  in __sm_ll_inc()
734 	r = sm_ll_inc_bitmap(ll, b, bit, bit_end, nr_allocations, new_b, &ic);  in __sm_ll_inc()
736 	exit_inc_context(ll, &ic);  in __sm_ll_inc()
741 	return ll->save_ie(ll, index, &ic.ie_disk);  in __sm_ll_inc()
760 				struct inc_context *ic)  in __sm_ll_del_overflow()  argument
762 	reset_inc_context(ll, ic);  in __sm_ll_del_overflow()
768 				struct inc_context *ic, uint32_t *old_rc)  in __sm_ll_dec_overflow()  argument
776 	reset_inc_context(ll, ic);  in __sm_ll_dec_overflow()
778 				     b, &index, &ll->ref_count_root, &ic->overflow_leaf);  in __sm_ll_dec_overflow()
782 	n = dm_block_data(ic->overflow_leaf);  in __sm_ll_dec_overflow()
794 		return __sm_ll_del_overflow(ll, b, ic);  in __sm_ll_dec_overflow()
803 			      struct inc_context *ic, uint32_t *old_rc)  in sm_ll_dec_overflow()  argument
808 	if (ic->overflow_leaf) {  in sm_ll_dec_overflow()
814 		n = dm_block_data(ic->overflow_leaf);  in sm_ll_dec_overflow()
826 				return __sm_ll_del_overflow(ll, b, ic);  in sm_ll_dec_overflow()
832 	return __sm_ll_dec_overflow(ll, b, ic, old_rc);  in sm_ll_dec_overflow()
840 				   struct inc_context *ic,  in sm_ll_dec_bitmap()  argument
852 		r = ensure_bitmap(ll, ic);  in sm_ll_dec_bitmap()
856 		old = sm_lookup_bitmap(ic->bitmap, bit);  in sm_ll_dec_bitmap()
864 			sm_set_bitmap(ic->bitmap, bit, 0);  in sm_ll_dec_bitmap()
867 			le32_add_cpu(&ic->ie_disk.nr_free, 1);  in sm_ll_dec_bitmap()
868 			ic->ie_disk.none_free_before =  in sm_ll_dec_bitmap()
869 				cpu_to_le32(min(le32_to_cpu(ic->ie_disk.none_free_before), bit));  in sm_ll_dec_bitmap()
874 			sm_set_bitmap(ic->bitmap, bit, 1);  in sm_ll_dec_bitmap()
878 			r = sm_ll_dec_overflow(ll, b, ic, &old);  in sm_ll_dec_bitmap()
883 				r = ensure_bitmap(ll, ic);  in sm_ll_dec_bitmap()
887 				sm_set_bitmap(ic->bitmap, bit, 2);  in sm_ll_dec_bitmap()
902 	struct inc_context ic;  in __sm_ll_dec()  local
905 	init_inc_context(&ic);  in __sm_ll_dec()
908 	r = ll->load_ie(ll, index, &ic.ie_disk);  in __sm_ll_dec()
912 	r = shadow_bitmap(ll, &ic);  in __sm_ll_dec()
917 	r = sm_ll_dec_bitmap(ll, b, bit, bit_end, &ic, nr_allocations, new_b);  in __sm_ll_dec()
918 	exit_inc_context(ll, &ic);  in __sm_ll_dec()
923 	return ll->save_ie(ll, index, &ic.ie_disk);  in __sm_ll_dec()