1
2 #include <inttypes.h>
3
4 #include "util/u_inlines.h"
5 #include "util/u_memory.h"
6 #include "util/list.h"
7
8 #include "nouveau_winsys.h"
9 #include "nouveau_screen.h"
10 #include "nouveau_mm.h"
11
12 /* TODO: Higher orders can waste a lot of space for npot size buffers, should
13 * add an extra cache for such buffer objects.
14 *
15 * HACK: Max order == 21 to accommodate TF2's 1.5 MiB, frequently reallocated
16 * vertex buffer (VM flush (?) decreases performance dramatically).
17 */
18
19 #define MM_MIN_ORDER 7 /* >= 6 to not violate ARB_map_buffer_alignment */
20 #define MM_MAX_ORDER 21
21
22 #define MM_NUM_BUCKETS (MM_MAX_ORDER - MM_MIN_ORDER + 1)
23
24 #define MM_MIN_SIZE (1 << MM_MIN_ORDER)
25 #define MM_MAX_SIZE (1 << MM_MAX_ORDER)
26
27 struct mm_bucket {
28 struct list_head free;
29 struct list_head used;
30 struct list_head full;
31 int num_free;
32 };
33
34 struct nouveau_mman {
35 struct nouveau_device *dev;
36 struct mm_bucket bucket[MM_NUM_BUCKETS];
37 uint32_t domain;
38 union nouveau_bo_config config;
39 uint64_t allocated;
40 };
41
42 struct mm_slab {
43 struct list_head head;
44 struct nouveau_bo *bo;
45 struct nouveau_mman *cache;
46 int order;
47 int count;
48 int free;
49 uint32_t bits[0];
50 };
51
52 static int
mm_slab_alloc(struct mm_slab * slab)53 mm_slab_alloc(struct mm_slab *slab)
54 {
55 int i, n, b;
56
57 if (slab->free == 0)
58 return -1;
59
60 for (i = 0; i < (slab->count + 31) / 32; ++i) {
61 b = ffs(slab->bits[i]) - 1;
62 if (b >= 0) {
63 n = i * 32 + b;
64 assert(n < slab->count);
65 slab->free--;
66 slab->bits[i] &= ~(1 << b);
67 return n;
68 }
69 }
70 return -1;
71 }
72
73 static inline void
mm_slab_free(struct mm_slab * slab,int i)74 mm_slab_free(struct mm_slab *slab, int i)
75 {
76 assert(i < slab->count);
77 slab->bits[i / 32] |= 1 << (i % 32);
78 slab->free++;
79 assert(slab->free <= slab->count);
80 }
81
82 static inline int
mm_get_order(uint32_t size)83 mm_get_order(uint32_t size)
84 {
85 int s = __builtin_clz(size) ^ 31;
86
87 if (size > (1 << s))
88 s += 1;
89 return s;
90 }
91
92 static struct mm_bucket *
mm_bucket_by_order(struct nouveau_mman * cache,int order)93 mm_bucket_by_order(struct nouveau_mman *cache, int order)
94 {
95 if (order > MM_MAX_ORDER)
96 return NULL;
97 return &cache->bucket[MAX2(order, MM_MIN_ORDER) - MM_MIN_ORDER];
98 }
99
100 static struct mm_bucket *
mm_bucket_by_size(struct nouveau_mman * cache,unsigned size)101 mm_bucket_by_size(struct nouveau_mman *cache, unsigned size)
102 {
103 return mm_bucket_by_order(cache, mm_get_order(size));
104 }
105
106 /* size of bo allocation for slab with chunks of (1 << chunk_order) bytes */
107 static inline uint32_t
mm_default_slab_size(unsigned chunk_order)108 mm_default_slab_size(unsigned chunk_order)
109 {
110 static const int8_t slab_order[MM_MAX_ORDER - MM_MIN_ORDER + 1] =
111 {
112 12, 12, 13, 14, 14, 17, 17, 17, 17, 19, 19, 20, 21, 22, 22
113 };
114
115 assert(chunk_order <= MM_MAX_ORDER && chunk_order >= MM_MIN_ORDER);
116
117 return 1 << slab_order[chunk_order - MM_MIN_ORDER];
118 }
119
120 static int
mm_slab_new(struct nouveau_mman * cache,int chunk_order)121 mm_slab_new(struct nouveau_mman *cache, int chunk_order)
122 {
123 struct mm_slab *slab;
124 int words, ret;
125 const uint32_t size = mm_default_slab_size(chunk_order);
126
127 words = ((size >> chunk_order) + 31) / 32;
128 assert(words);
129
130 slab = MALLOC(sizeof(struct mm_slab) + words * 4);
131 if (!slab)
132 return PIPE_ERROR_OUT_OF_MEMORY;
133
134 memset(&slab->bits[0], ~0, words * 4);
135
136 slab->bo = NULL;
137
138 ret = nouveau_bo_new(cache->dev, cache->domain, 0, size, &cache->config,
139 &slab->bo);
140 if (ret) {
141 FREE(slab);
142 return PIPE_ERROR_OUT_OF_MEMORY;
143 }
144
145 LIST_INITHEAD(&slab->head);
146
147 slab->cache = cache;
148 slab->order = chunk_order;
149 slab->count = slab->free = size >> chunk_order;
150
151 LIST_ADD(&slab->head, &mm_bucket_by_order(cache, chunk_order)->free);
152
153 cache->allocated += size;
154
155 if (nouveau_mesa_debug)
156 debug_printf("MM: new slab, total memory = %"PRIu64" KiB\n",
157 cache->allocated / 1024);
158
159 return PIPE_OK;
160 }
161
162 /* @return token to identify slab or NULL if we just allocated a new bo */
163 struct nouveau_mm_allocation *
nouveau_mm_allocate(struct nouveau_mman * cache,uint32_t size,struct nouveau_bo ** bo,uint32_t * offset)164 nouveau_mm_allocate(struct nouveau_mman *cache,
165 uint32_t size, struct nouveau_bo **bo, uint32_t *offset)
166 {
167 struct mm_bucket *bucket;
168 struct mm_slab *slab;
169 struct nouveau_mm_allocation *alloc;
170 int ret;
171
172 bucket = mm_bucket_by_size(cache, size);
173 if (!bucket) {
174 ret = nouveau_bo_new(cache->dev, cache->domain, 0, size, &cache->config,
175 bo);
176 if (ret)
177 debug_printf("bo_new(%x, %x): %i\n",
178 size, cache->config.nv50.memtype, ret);
179
180 *offset = 0;
181 return NULL;
182 }
183
184 if (!LIST_IS_EMPTY(&bucket->used)) {
185 slab = LIST_ENTRY(struct mm_slab, bucket->used.next, head);
186 } else {
187 if (LIST_IS_EMPTY(&bucket->free)) {
188 mm_slab_new(cache, MAX2(mm_get_order(size), MM_MIN_ORDER));
189 }
190 slab = LIST_ENTRY(struct mm_slab, bucket->free.next, head);
191
192 LIST_DEL(&slab->head);
193 LIST_ADD(&slab->head, &bucket->used);
194 }
195
196 *offset = mm_slab_alloc(slab) << slab->order;
197
198 alloc = MALLOC_STRUCT(nouveau_mm_allocation);
199 if (!alloc)
200 return NULL;
201
202 nouveau_bo_ref(slab->bo, bo);
203
204 if (slab->free == 0) {
205 LIST_DEL(&slab->head);
206 LIST_ADD(&slab->head, &bucket->full);
207 }
208
209 alloc->next = NULL;
210 alloc->offset = *offset;
211 alloc->priv = (void *)slab;
212
213 return alloc;
214 }
215
216 void
nouveau_mm_free(struct nouveau_mm_allocation * alloc)217 nouveau_mm_free(struct nouveau_mm_allocation *alloc)
218 {
219 struct mm_slab *slab = (struct mm_slab *)alloc->priv;
220 struct mm_bucket *bucket = mm_bucket_by_order(slab->cache, slab->order);
221
222 mm_slab_free(slab, alloc->offset >> slab->order);
223
224 if (slab->free == slab->count) {
225 LIST_DEL(&slab->head);
226 LIST_ADDTAIL(&slab->head, &bucket->free);
227 } else
228 if (slab->free == 1) {
229 LIST_DEL(&slab->head);
230 LIST_ADDTAIL(&slab->head, &bucket->used);
231 }
232
233 FREE(alloc);
234 }
235
236 void
nouveau_mm_free_work(void * data)237 nouveau_mm_free_work(void *data)
238 {
239 nouveau_mm_free(data);
240 }
241
242 struct nouveau_mman *
nouveau_mm_create(struct nouveau_device * dev,uint32_t domain,union nouveau_bo_config * config)243 nouveau_mm_create(struct nouveau_device *dev, uint32_t domain,
244 union nouveau_bo_config *config)
245 {
246 struct nouveau_mman *cache = MALLOC_STRUCT(nouveau_mman);
247 int i;
248
249 if (!cache)
250 return NULL;
251
252 cache->dev = dev;
253 cache->domain = domain;
254 cache->config = *config;
255 cache->allocated = 0;
256
257 for (i = 0; i < MM_NUM_BUCKETS; ++i) {
258 LIST_INITHEAD(&cache->bucket[i].free);
259 LIST_INITHEAD(&cache->bucket[i].used);
260 LIST_INITHEAD(&cache->bucket[i].full);
261 }
262
263 return cache;
264 }
265
266 static inline void
nouveau_mm_free_slabs(struct list_head * head)267 nouveau_mm_free_slabs(struct list_head *head)
268 {
269 struct mm_slab *slab, *next;
270
271 LIST_FOR_EACH_ENTRY_SAFE(slab, next, head, head) {
272 LIST_DEL(&slab->head);
273 nouveau_bo_ref(NULL, &slab->bo);
274 FREE(slab);
275 }
276 }
277
278 void
nouveau_mm_destroy(struct nouveau_mman * cache)279 nouveau_mm_destroy(struct nouveau_mman *cache)
280 {
281 int i;
282
283 if (!cache)
284 return;
285
286 for (i = 0; i < MM_NUM_BUCKETS; ++i) {
287 if (!LIST_IS_EMPTY(&cache->bucket[i].used) ||
288 !LIST_IS_EMPTY(&cache->bucket[i].full))
289 debug_printf("WARNING: destroying GPU memory cache "
290 "with some buffers still in use\n");
291
292 nouveau_mm_free_slabs(&cache->bucket[i].free);
293 nouveau_mm_free_slabs(&cache->bucket[i].used);
294 nouveau_mm_free_slabs(&cache->bucket[i].full);
295 }
296
297 FREE(cache);
298 }
299