44 u32 mlx4_bitmap_alloc(struct mlx4_bitmap *bitmap)  in mlx4_bitmap_alloc()  argument
48 	spin_lock(&bitmap->lock);  in mlx4_bitmap_alloc()
50 	obj = find_next_zero_bit(bitmap->table, bitmap->max, bitmap->last);  in mlx4_bitmap_alloc()
51 	if (obj >= bitmap->max) {  in mlx4_bitmap_alloc()
52 		bitmap->top = (bitmap->top + bitmap->max + bitmap->reserved_top)  in mlx4_bitmap_alloc()
53 				& bitmap->mask;  in mlx4_bitmap_alloc()
54 		obj = find_first_zero_bit(bitmap->table, bitmap->max);  in mlx4_bitmap_alloc()
57 	if (obj < bitmap->max) {  in mlx4_bitmap_alloc()
58 		set_bit(obj, bitmap->table);  in mlx4_bitmap_alloc()
59 		bitmap->last = (obj + 1);  in mlx4_bitmap_alloc()
60 		if (bitmap->last == bitmap->max)  in mlx4_bitmap_alloc()
61 			bitmap->last = 0;  in mlx4_bitmap_alloc()
62 		obj |= bitmap->top;  in mlx4_bitmap_alloc()
67 		--bitmap->avail;  in mlx4_bitmap_alloc()
69 	spin_unlock(&bitmap->lock);  in mlx4_bitmap_alloc()
74 void mlx4_bitmap_free(struct mlx4_bitmap *bitmap, u32 obj, int use_rr)  in mlx4_bitmap_free()  argument
76 	mlx4_bitmap_free_range(bitmap, obj, 1, use_rr);  in mlx4_bitmap_free()
79 static unsigned long find_aligned_range(unsigned long *bitmap,  in find_aligned_range()  argument
88 	while ((start < nbits) && (test_bit(start, bitmap) ||  in find_aligned_range()
100 		if (test_bit(i, bitmap) || ((u32)i & skip_mask)) {  in find_aligned_range()
109 u32 mlx4_bitmap_alloc_range(struct mlx4_bitmap *bitmap, int cnt,  in mlx4_bitmap_alloc_range()  argument
115 		return mlx4_bitmap_alloc(bitmap);  in mlx4_bitmap_alloc_range()
117 	spin_lock(&bitmap->lock);  in mlx4_bitmap_alloc_range()
119 	obj = find_aligned_range(bitmap->table, bitmap->last,  in mlx4_bitmap_alloc_range()
120 				 bitmap->max, cnt, align, skip_mask);  in mlx4_bitmap_alloc_range()
121 	if (obj >= bitmap->max) {  in mlx4_bitmap_alloc_range()
122 		bitmap->top = (bitmap->top + bitmap->max + bitmap->reserved_top)  in mlx4_bitmap_alloc_range()
123 				& bitmap->mask;  in mlx4_bitmap_alloc_range()
124 		obj = find_aligned_range(bitmap->table, 0, bitmap->max,  in mlx4_bitmap_alloc_range()
128 	if (obj < bitmap->max) {  in mlx4_bitmap_alloc_range()
129 		bitmap_set(bitmap->table, obj, cnt);  in mlx4_bitmap_alloc_range()
130 		if (obj == bitmap->last) {  in mlx4_bitmap_alloc_range()
131 			bitmap->last = (obj + cnt);  in mlx4_bitmap_alloc_range()
132 			if (bitmap->last >= bitmap->max)  in mlx4_bitmap_alloc_range()
133 				bitmap->last = 0;  in mlx4_bitmap_alloc_range()
135 		obj |= bitmap->top;  in mlx4_bitmap_alloc_range()
140 		bitmap->avail -= cnt;  in mlx4_bitmap_alloc_range()
142 	spin_unlock(&bitmap->lock);  in mlx4_bitmap_alloc_range()
147 u32 mlx4_bitmap_avail(struct mlx4_bitmap *bitmap)  in mlx4_bitmap_avail()  argument
149 	return bitmap->avail;  in mlx4_bitmap_avail()
152 static u32 mlx4_bitmap_masked_value(struct mlx4_bitmap *bitmap, u32 obj)  in mlx4_bitmap_masked_value()  argument
154 	return obj & (bitmap->max + bitmap->reserved_top - 1);  in mlx4_bitmap_masked_value()
157 void mlx4_bitmap_free_range(struct mlx4_bitmap *bitmap, u32 obj, int cnt,  in mlx4_bitmap_free_range()  argument
160 	obj &= bitmap->max + bitmap->reserved_top - 1;  in mlx4_bitmap_free_range()
162 	spin_lock(&bitmap->lock);  in mlx4_bitmap_free_range()
164 		bitmap->last = min(bitmap->last, obj);  in mlx4_bitmap_free_range()
165 		bitmap->top = (bitmap->top + bitmap->max + bitmap->reserved_top)  in mlx4_bitmap_free_range()
166 				& bitmap->mask;  in mlx4_bitmap_free_range()
168 	bitmap_clear(bitmap->table, obj, cnt);  in mlx4_bitmap_free_range()
169 	bitmap->avail += cnt;  in mlx4_bitmap_free_range()
170 	spin_unlock(&bitmap->lock);  in mlx4_bitmap_free_range()
173 int mlx4_bitmap_init(struct mlx4_bitmap *bitmap, u32 num, u32 mask,  in mlx4_bitmap_init()  argument
180 	bitmap->last = 0;  in mlx4_bitmap_init()
181 	bitmap->top  = 0;  in mlx4_bitmap_init()
182 	bitmap->max  = num - reserved_top;  in mlx4_bitmap_init()
183 	bitmap->mask = mask;  in mlx4_bitmap_init()
184 	bitmap->reserved_top = reserved_top;  in mlx4_bitmap_init()
185 	bitmap->avail = num - reserved_top - reserved_bot;  in mlx4_bitmap_init()
186 	bitmap->effective_len = bitmap->avail;  in mlx4_bitmap_init()
187 	spin_lock_init(&bitmap->lock);  in mlx4_bitmap_init()
188 	bitmap->table = bitmap_zalloc(bitmap->max, GFP_KERNEL);  in mlx4_bitmap_init()
189 	if (!bitmap->table)  in mlx4_bitmap_init()
192 	bitmap_set(bitmap->table, 0, reserved_bot);  in mlx4_bitmap_init()
197 void mlx4_bitmap_cleanup(struct mlx4_bitmap *bitmap)  in mlx4_bitmap_cleanup()  argument
199 	bitmap_free(bitmap->table);  in mlx4_bitmap_cleanup()
217 	struct mlx4_bitmap		*bitmap;  member
242 		      struct mlx4_bitmap *bitmap,  in mlx4_zone_add_one()  argument
248 	u32 mask = mlx4_bitmap_masked_value(bitmap, (u32)-1);  in mlx4_zone_add_one()
256 	zone->bitmap = bitmap;  in mlx4_zone_add_one()
310 			u32 cur_mask = mlx4_bitmap_masked_value(it->bitmap, (u32)-1);  in __mlx4_zone_remove_one_entry()
344 	res = mlx4_bitmap_alloc_range(zone->bitmap, count,  in __mlx4_alloc_from_zone()
362 			res = mlx4_bitmap_alloc_range(it->bitmap, count,  in __mlx4_alloc_from_zone()
382 			res = mlx4_bitmap_alloc_range(it->bitmap, count,  in __mlx4_alloc_from_zone()
401 			res = mlx4_bitmap_alloc_range(curr_node->bitmap, count,  in __mlx4_alloc_from_zone()
421 	mlx4_bitmap_free_range(zone->bitmap, obj - zone->offset, count, zone->use_rr);  in __mlx4_free_from_zone()
441 	struct mlx4_bitmap *bitmap;  in mlx4_zone_get_bitmap()  local
447 	bitmap = zone == NULL ? NULL : zone->bitmap;  in mlx4_zone_get_bitmap()
451 	return bitmap;  in mlx4_zone_get_bitmap()
494 			if (mobj < zone->bitmap->max) {  in __mlx4_find_zone_by_uid_unique()
495 				u32 curr_dist = zone->bitmap->effective_len;  in __mlx4_find_zone_by_uid_unique()