1 // SPDX-License-Identifier: MIT
2 /*
3 * Copyright © 2021 Intel Corporation
4 */
5
6 #include <linux/kmemleak.h>
7
8 #include "i915_buddy.h"
9
10 #include "i915_gem.h"
11 #include "i915_utils.h"
12
13 static struct kmem_cache *slab_blocks;
14
i915_block_alloc(struct i915_buddy_mm * mm,struct i915_buddy_block * parent,unsigned int order,u64 offset)15 static struct i915_buddy_block *i915_block_alloc(struct i915_buddy_mm *mm,
16 struct i915_buddy_block *parent,
17 unsigned int order,
18 u64 offset)
19 {
20 struct i915_buddy_block *block;
21
22 GEM_BUG_ON(order > I915_BUDDY_MAX_ORDER);
23
24 block = kmem_cache_zalloc(slab_blocks, GFP_KERNEL);
25 if (!block)
26 return NULL;
27
28 block->header = offset;
29 block->header |= order;
30 block->parent = parent;
31
32 GEM_BUG_ON(block->header & I915_BUDDY_HEADER_UNUSED);
33 return block;
34 }
35
i915_block_free(struct i915_buddy_mm * mm,struct i915_buddy_block * block)36 static void i915_block_free(struct i915_buddy_mm *mm,
37 struct i915_buddy_block *block)
38 {
39 kmem_cache_free(slab_blocks, block);
40 }
41
mark_allocated(struct i915_buddy_block * block)42 static void mark_allocated(struct i915_buddy_block *block)
43 {
44 block->header &= ~I915_BUDDY_HEADER_STATE;
45 block->header |= I915_BUDDY_ALLOCATED;
46
47 list_del(&block->link);
48 }
49
mark_free(struct i915_buddy_mm * mm,struct i915_buddy_block * block)50 static void mark_free(struct i915_buddy_mm *mm,
51 struct i915_buddy_block *block)
52 {
53 block->header &= ~I915_BUDDY_HEADER_STATE;
54 block->header |= I915_BUDDY_FREE;
55
56 list_add(&block->link,
57 &mm->free_list[i915_buddy_block_order(block)]);
58 }
59
mark_split(struct i915_buddy_block * block)60 static void mark_split(struct i915_buddy_block *block)
61 {
62 block->header &= ~I915_BUDDY_HEADER_STATE;
63 block->header |= I915_BUDDY_SPLIT;
64
65 list_del(&block->link);
66 }
67
i915_buddy_init(struct i915_buddy_mm * mm,u64 size,u64 chunk_size)68 int i915_buddy_init(struct i915_buddy_mm *mm, u64 size, u64 chunk_size)
69 {
70 unsigned int i;
71 u64 offset;
72
73 if (size < chunk_size)
74 return -EINVAL;
75
76 if (chunk_size < PAGE_SIZE)
77 return -EINVAL;
78
79 if (!is_power_of_2(chunk_size))
80 return -EINVAL;
81
82 size = round_down(size, chunk_size);
83
84 mm->size = size;
85 mm->chunk_size = chunk_size;
86 mm->max_order = ilog2(size) - ilog2(chunk_size);
87
88 GEM_BUG_ON(mm->max_order > I915_BUDDY_MAX_ORDER);
89
90 mm->free_list = kmalloc_array(mm->max_order + 1,
91 sizeof(struct list_head),
92 GFP_KERNEL);
93 if (!mm->free_list)
94 return -ENOMEM;
95
96 for (i = 0; i <= mm->max_order; ++i)
97 INIT_LIST_HEAD(&mm->free_list[i]);
98
99 mm->n_roots = hweight64(size);
100
101 mm->roots = kmalloc_array(mm->n_roots,
102 sizeof(struct i915_buddy_block *),
103 GFP_KERNEL);
104 if (!mm->roots)
105 goto out_free_list;
106
107 offset = 0;
108 i = 0;
109
110 /*
111 * Split into power-of-two blocks, in case we are given a size that is
112 * not itself a power-of-two.
113 */
114 do {
115 struct i915_buddy_block *root;
116 unsigned int order;
117 u64 root_size;
118
119 root_size = rounddown_pow_of_two(size);
120 order = ilog2(root_size) - ilog2(chunk_size);
121
122 root = i915_block_alloc(mm, NULL, order, offset);
123 if (!root)
124 goto out_free_roots;
125
126 mark_free(mm, root);
127
128 GEM_BUG_ON(i > mm->max_order);
129 GEM_BUG_ON(i915_buddy_block_size(mm, root) < chunk_size);
130
131 mm->roots[i] = root;
132
133 offset += root_size;
134 size -= root_size;
135 i++;
136 } while (size);
137
138 return 0;
139
140 out_free_roots:
141 while (i--)
142 i915_block_free(mm, mm->roots[i]);
143 kfree(mm->roots);
144 out_free_list:
145 kfree(mm->free_list);
146 return -ENOMEM;
147 }
148
i915_buddy_fini(struct i915_buddy_mm * mm)149 void i915_buddy_fini(struct i915_buddy_mm *mm)
150 {
151 int i;
152
153 for (i = 0; i < mm->n_roots; ++i) {
154 GEM_WARN_ON(!i915_buddy_block_is_free(mm->roots[i]));
155 i915_block_free(mm, mm->roots[i]);
156 }
157
158 kfree(mm->roots);
159 kfree(mm->free_list);
160 }
161
split_block(struct i915_buddy_mm * mm,struct i915_buddy_block * block)162 static int split_block(struct i915_buddy_mm *mm,
163 struct i915_buddy_block *block)
164 {
165 unsigned int block_order = i915_buddy_block_order(block) - 1;
166 u64 offset = i915_buddy_block_offset(block);
167
168 GEM_BUG_ON(!i915_buddy_block_is_free(block));
169 GEM_BUG_ON(!i915_buddy_block_order(block));
170
171 block->left = i915_block_alloc(mm, block, block_order, offset);
172 if (!block->left)
173 return -ENOMEM;
174
175 block->right = i915_block_alloc(mm, block, block_order,
176 offset + (mm->chunk_size << block_order));
177 if (!block->right) {
178 i915_block_free(mm, block->left);
179 return -ENOMEM;
180 }
181
182 mark_free(mm, block->left);
183 mark_free(mm, block->right);
184
185 mark_split(block);
186
187 return 0;
188 }
189
190 static struct i915_buddy_block *
get_buddy(struct i915_buddy_block * block)191 get_buddy(struct i915_buddy_block *block)
192 {
193 struct i915_buddy_block *parent;
194
195 parent = block->parent;
196 if (!parent)
197 return NULL;
198
199 if (parent->left == block)
200 return parent->right;
201
202 return parent->left;
203 }
204
__i915_buddy_free(struct i915_buddy_mm * mm,struct i915_buddy_block * block)205 static void __i915_buddy_free(struct i915_buddy_mm *mm,
206 struct i915_buddy_block *block)
207 {
208 struct i915_buddy_block *parent;
209
210 while ((parent = block->parent)) {
211 struct i915_buddy_block *buddy;
212
213 buddy = get_buddy(block);
214
215 if (!i915_buddy_block_is_free(buddy))
216 break;
217
218 list_del(&buddy->link);
219
220 i915_block_free(mm, block);
221 i915_block_free(mm, buddy);
222
223 block = parent;
224 }
225
226 mark_free(mm, block);
227 }
228
i915_buddy_free(struct i915_buddy_mm * mm,struct i915_buddy_block * block)229 void i915_buddy_free(struct i915_buddy_mm *mm,
230 struct i915_buddy_block *block)
231 {
232 GEM_BUG_ON(!i915_buddy_block_is_allocated(block));
233 __i915_buddy_free(mm, block);
234 }
235
i915_buddy_free_list(struct i915_buddy_mm * mm,struct list_head * objects)236 void i915_buddy_free_list(struct i915_buddy_mm *mm, struct list_head *objects)
237 {
238 struct i915_buddy_block *block, *on;
239
240 list_for_each_entry_safe(block, on, objects, link) {
241 i915_buddy_free(mm, block);
242 cond_resched();
243 }
244 INIT_LIST_HEAD(objects);
245 }
246
247 /*
248 * Allocate power-of-two block. The order value here translates to:
249 *
250 * 0 = 2^0 * mm->chunk_size
251 * 1 = 2^1 * mm->chunk_size
252 * 2 = 2^2 * mm->chunk_size
253 * ...
254 */
255 struct i915_buddy_block *
i915_buddy_alloc(struct i915_buddy_mm * mm,unsigned int order)256 i915_buddy_alloc(struct i915_buddy_mm *mm, unsigned int order)
257 {
258 struct i915_buddy_block *block = NULL;
259 unsigned int i;
260 int err;
261
262 for (i = order; i <= mm->max_order; ++i) {
263 block = list_first_entry_or_null(&mm->free_list[i],
264 struct i915_buddy_block,
265 link);
266 if (block)
267 break;
268 }
269
270 if (!block)
271 return ERR_PTR(-ENOSPC);
272
273 GEM_BUG_ON(!i915_buddy_block_is_free(block));
274
275 while (i != order) {
276 err = split_block(mm, block);
277 if (unlikely(err))
278 goto out_free;
279
280 /* Go low */
281 block = block->left;
282 i--;
283 }
284
285 mark_allocated(block);
286 kmemleak_update_trace(block);
287 return block;
288
289 out_free:
290 if (i != order)
291 __i915_buddy_free(mm, block);
292 return ERR_PTR(err);
293 }
294
overlaps(u64 s1,u64 e1,u64 s2,u64 e2)295 static inline bool overlaps(u64 s1, u64 e1, u64 s2, u64 e2)
296 {
297 return s1 <= e2 && e1 >= s2;
298 }
299
contains(u64 s1,u64 e1,u64 s2,u64 e2)300 static inline bool contains(u64 s1, u64 e1, u64 s2, u64 e2)
301 {
302 return s1 <= s2 && e1 >= e2;
303 }
304
305 /*
306 * Allocate range. Note that it's safe to chain together multiple alloc_ranges
307 * with the same blocks list.
308 *
309 * Intended for pre-allocating portions of the address space, for example to
310 * reserve a block for the initial framebuffer or similar, hence the expectation
311 * here is that i915_buddy_alloc() is still the main vehicle for
312 * allocations, so if that's not the case then the drm_mm range allocator is
313 * probably a much better fit, and so you should probably go use that instead.
314 */
i915_buddy_alloc_range(struct i915_buddy_mm * mm,struct list_head * blocks,u64 start,u64 size)315 int i915_buddy_alloc_range(struct i915_buddy_mm *mm,
316 struct list_head *blocks,
317 u64 start, u64 size)
318 {
319 struct i915_buddy_block *block;
320 struct i915_buddy_block *buddy;
321 LIST_HEAD(allocated);
322 LIST_HEAD(dfs);
323 u64 end;
324 int err;
325 int i;
326
327 if (size < mm->chunk_size)
328 return -EINVAL;
329
330 if (!IS_ALIGNED(size | start, mm->chunk_size))
331 return -EINVAL;
332
333 if (range_overflows(start, size, mm->size))
334 return -EINVAL;
335
336 for (i = 0; i < mm->n_roots; ++i)
337 list_add_tail(&mm->roots[i]->tmp_link, &dfs);
338
339 end = start + size - 1;
340
341 do {
342 u64 block_start;
343 u64 block_end;
344
345 block = list_first_entry_or_null(&dfs,
346 struct i915_buddy_block,
347 tmp_link);
348 if (!block)
349 break;
350
351 list_del(&block->tmp_link);
352
353 block_start = i915_buddy_block_offset(block);
354 block_end = block_start + i915_buddy_block_size(mm, block) - 1;
355
356 if (!overlaps(start, end, block_start, block_end))
357 continue;
358
359 if (i915_buddy_block_is_allocated(block)) {
360 err = -ENOSPC;
361 goto err_free;
362 }
363
364 if (contains(start, end, block_start, block_end)) {
365 if (!i915_buddy_block_is_free(block)) {
366 err = -ENOSPC;
367 goto err_free;
368 }
369
370 mark_allocated(block);
371 list_add_tail(&block->link, &allocated);
372 continue;
373 }
374
375 if (!i915_buddy_block_is_split(block)) {
376 err = split_block(mm, block);
377 if (unlikely(err))
378 goto err_undo;
379 }
380
381 list_add(&block->right->tmp_link, &dfs);
382 list_add(&block->left->tmp_link, &dfs);
383 } while (1);
384
385 list_splice_tail(&allocated, blocks);
386 return 0;
387
388 err_undo:
389 /*
390 * We really don't want to leave around a bunch of split blocks, since
391 * bigger is better, so make sure we merge everything back before we
392 * free the allocated blocks.
393 */
394 buddy = get_buddy(block);
395 if (buddy &&
396 (i915_buddy_block_is_free(block) &&
397 i915_buddy_block_is_free(buddy)))
398 __i915_buddy_free(mm, block);
399
400 err_free:
401 i915_buddy_free_list(mm, &allocated);
402 return err;
403 }
404
405 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
406 #include "selftests/i915_buddy.c"
407 #endif
408
i915_buddy_module_exit(void)409 void i915_buddy_module_exit(void)
410 {
411 kmem_cache_destroy(slab_blocks);
412 }
413
i915_buddy_module_init(void)414 int __init i915_buddy_module_init(void)
415 {
416 slab_blocks = KMEM_CACHE(i915_buddy_block, 0);
417 if (!slab_blocks)
418 return -ENOMEM;
419
420 return 0;
421 }
422