1 /* SPDX-License-Identifier: GPL-2.0 OR MIT */
2 /**************************************************************************
3 *
4 * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA
5 * All Rights Reserved.
6 *
7 * Permission is hereby granted, free of charge, to any person obtaining a
8 * copy of this software and associated documentation files (the
9 * "Software"), to deal in the Software without restriction, including
10 * without limitation the rights to use, copy, modify, merge, publish,
11 * distribute, sub license, and/or sell copies of the Software, and to
12 * permit persons to whom the Software is furnished to do so, subject to
13 * the following conditions:
14 *
15 * The above copyright notice and this permission notice (including the
16 * next paragraph) shall be included in all copies or substantial portions
17 * of the Software.
18 *
19 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
20 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
21 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
22 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
23 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
24 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
25 * USE OR OTHER DEALINGS IN THE SOFTWARE.
26 *
27 **************************************************************************/
28
29 #define pr_fmt(fmt) "[TTM] " fmt
30
31 #include <linux/spinlock.h>
32 #include <linux/sched.h>
33 #include <linux/wait.h>
34 #include <linux/mm.h>
35 #include <linux/module.h>
36 #include <linux/slab.h>
37
38 #include <drm/drm_device.h>
39 #include <drm/drm_file.h>
40 #include <drm/ttm/ttm_device.h>
41
42 #include "ttm_memory.h"
43
44 #define TTM_MEMORY_ALLOC_RETRIES 4
45
46 struct ttm_mem_global ttm_mem_glob;
47 EXPORT_SYMBOL(ttm_mem_glob);
48
49 struct ttm_mem_zone {
50 struct kobject kobj;
51 struct ttm_mem_global *glob;
52 const char *name;
53 uint64_t zone_mem;
54 uint64_t emer_mem;
55 uint64_t max_mem;
56 uint64_t swap_limit;
57 uint64_t used_mem;
58 };
59
60 static struct attribute ttm_mem_sys = {
61 .name = "zone_memory",
62 .mode = S_IRUGO
63 };
64 static struct attribute ttm_mem_emer = {
65 .name = "emergency_memory",
66 .mode = S_IRUGO | S_IWUSR
67 };
68 static struct attribute ttm_mem_max = {
69 .name = "available_memory",
70 .mode = S_IRUGO | S_IWUSR
71 };
72 static struct attribute ttm_mem_swap = {
73 .name = "swap_limit",
74 .mode = S_IRUGO | S_IWUSR
75 };
76 static struct attribute ttm_mem_used = {
77 .name = "used_memory",
78 .mode = S_IRUGO
79 };
80
ttm_mem_zone_kobj_release(struct kobject * kobj)81 static void ttm_mem_zone_kobj_release(struct kobject *kobj)
82 {
83 struct ttm_mem_zone *zone =
84 container_of(kobj, struct ttm_mem_zone, kobj);
85
86 pr_info("Zone %7s: Used memory at exit: %llu KiB\n",
87 zone->name, (unsigned long long)zone->used_mem >> 10);
88 kfree(zone);
89 }
90
ttm_mem_zone_show(struct kobject * kobj,struct attribute * attr,char * buffer)91 static ssize_t ttm_mem_zone_show(struct kobject *kobj,
92 struct attribute *attr,
93 char *buffer)
94 {
95 struct ttm_mem_zone *zone =
96 container_of(kobj, struct ttm_mem_zone, kobj);
97 uint64_t val = 0;
98
99 spin_lock(&zone->glob->lock);
100 if (attr == &ttm_mem_sys)
101 val = zone->zone_mem;
102 else if (attr == &ttm_mem_emer)
103 val = zone->emer_mem;
104 else if (attr == &ttm_mem_max)
105 val = zone->max_mem;
106 else if (attr == &ttm_mem_swap)
107 val = zone->swap_limit;
108 else if (attr == &ttm_mem_used)
109 val = zone->used_mem;
110 spin_unlock(&zone->glob->lock);
111
112 return snprintf(buffer, PAGE_SIZE, "%llu\n",
113 (unsigned long long) val >> 10);
114 }
115
116 static void ttm_check_swapping(struct ttm_mem_global *glob);
117
ttm_mem_zone_store(struct kobject * kobj,struct attribute * attr,const char * buffer,size_t size)118 static ssize_t ttm_mem_zone_store(struct kobject *kobj,
119 struct attribute *attr,
120 const char *buffer,
121 size_t size)
122 {
123 struct ttm_mem_zone *zone =
124 container_of(kobj, struct ttm_mem_zone, kobj);
125 int chars;
126 unsigned long val;
127 uint64_t val64;
128
129 chars = sscanf(buffer, "%lu", &val);
130 if (chars == 0)
131 return size;
132
133 val64 = val;
134 val64 <<= 10;
135
136 spin_lock(&zone->glob->lock);
137 if (val64 > zone->zone_mem)
138 val64 = zone->zone_mem;
139 if (attr == &ttm_mem_emer) {
140 zone->emer_mem = val64;
141 if (zone->max_mem > val64)
142 zone->max_mem = val64;
143 } else if (attr == &ttm_mem_max) {
144 zone->max_mem = val64;
145 if (zone->emer_mem < val64)
146 zone->emer_mem = val64;
147 } else if (attr == &ttm_mem_swap)
148 zone->swap_limit = val64;
149 spin_unlock(&zone->glob->lock);
150
151 ttm_check_swapping(zone->glob);
152
153 return size;
154 }
155
156 static struct attribute *ttm_mem_zone_attrs[] = {
157 &ttm_mem_sys,
158 &ttm_mem_emer,
159 &ttm_mem_max,
160 &ttm_mem_swap,
161 &ttm_mem_used,
162 NULL
163 };
164
165 static const struct sysfs_ops ttm_mem_zone_ops = {
166 .show = &ttm_mem_zone_show,
167 .store = &ttm_mem_zone_store
168 };
169
170 static struct kobj_type ttm_mem_zone_kobj_type = {
171 .release = &ttm_mem_zone_kobj_release,
172 .sysfs_ops = &ttm_mem_zone_ops,
173 .default_attrs = ttm_mem_zone_attrs,
174 };
175 static struct kobj_type ttm_mem_glob_kobj_type = {0};
176
ttm_zones_above_swap_target(struct ttm_mem_global * glob,bool from_wq,uint64_t extra)177 static bool ttm_zones_above_swap_target(struct ttm_mem_global *glob,
178 bool from_wq, uint64_t extra)
179 {
180 unsigned int i;
181 struct ttm_mem_zone *zone;
182 uint64_t target;
183
184 for (i = 0; i < glob->num_zones; ++i) {
185 zone = glob->zones[i];
186
187 if (from_wq)
188 target = zone->swap_limit;
189 else if (capable(CAP_SYS_ADMIN))
190 target = zone->emer_mem;
191 else
192 target = zone->max_mem;
193
194 target = (extra > target) ? 0ULL : target;
195
196 if (zone->used_mem > target)
197 return true;
198 }
199 return false;
200 }
201
202 /*
203 * At this point we only support a single shrink callback.
204 * Extend this if needed, perhaps using a linked list of callbacks.
205 * Note that this function is reentrant:
206 * many threads may try to swap out at any given time.
207 */
208
ttm_shrink(struct ttm_mem_global * glob,bool from_wq,uint64_t extra,struct ttm_operation_ctx * ctx)209 static void ttm_shrink(struct ttm_mem_global *glob, bool from_wq,
210 uint64_t extra, struct ttm_operation_ctx *ctx)
211 {
212 int ret;
213
214 spin_lock(&glob->lock);
215
216 while (ttm_zones_above_swap_target(glob, from_wq, extra)) {
217 spin_unlock(&glob->lock);
218 ret = ttm_global_swapout(ctx, GFP_KERNEL);
219 spin_lock(&glob->lock);
220 if (unlikely(ret <= 0))
221 break;
222 }
223
224 spin_unlock(&glob->lock);
225 }
226
ttm_shrink_work(struct work_struct * work)227 static void ttm_shrink_work(struct work_struct *work)
228 {
229 struct ttm_operation_ctx ctx = {
230 .interruptible = false,
231 .no_wait_gpu = false
232 };
233 struct ttm_mem_global *glob =
234 container_of(work, struct ttm_mem_global, work);
235
236 ttm_shrink(glob, true, 0ULL, &ctx);
237 }
238
ttm_mem_init_kernel_zone(struct ttm_mem_global * glob,const struct sysinfo * si)239 static int ttm_mem_init_kernel_zone(struct ttm_mem_global *glob,
240 const struct sysinfo *si)
241 {
242 struct ttm_mem_zone *zone = kzalloc(sizeof(*zone), GFP_KERNEL);
243 uint64_t mem;
244 int ret;
245
246 if (unlikely(!zone))
247 return -ENOMEM;
248
249 mem = si->totalram - si->totalhigh;
250 mem *= si->mem_unit;
251
252 zone->name = "kernel";
253 zone->zone_mem = mem;
254 zone->max_mem = mem >> 1;
255 zone->emer_mem = (mem >> 1) + (mem >> 2);
256 zone->swap_limit = zone->max_mem - (mem >> 3);
257 zone->used_mem = 0;
258 zone->glob = glob;
259 glob->zone_kernel = zone;
260 ret = kobject_init_and_add(
261 &zone->kobj, &ttm_mem_zone_kobj_type, &glob->kobj, zone->name);
262 if (unlikely(ret != 0)) {
263 kobject_put(&zone->kobj);
264 return ret;
265 }
266 glob->zones[glob->num_zones++] = zone;
267 return 0;
268 }
269
270 #ifdef CONFIG_HIGHMEM
ttm_mem_init_highmem_zone(struct ttm_mem_global * glob,const struct sysinfo * si)271 static int ttm_mem_init_highmem_zone(struct ttm_mem_global *glob,
272 const struct sysinfo *si)
273 {
274 struct ttm_mem_zone *zone;
275 uint64_t mem;
276 int ret;
277
278 if (si->totalhigh == 0)
279 return 0;
280
281 zone = kzalloc(sizeof(*zone), GFP_KERNEL);
282 if (unlikely(!zone))
283 return -ENOMEM;
284
285 mem = si->totalram;
286 mem *= si->mem_unit;
287
288 zone->name = "highmem";
289 zone->zone_mem = mem;
290 zone->max_mem = mem >> 1;
291 zone->emer_mem = (mem >> 1) + (mem >> 2);
292 zone->swap_limit = zone->max_mem - (mem >> 3);
293 zone->used_mem = 0;
294 zone->glob = glob;
295 glob->zone_highmem = zone;
296 ret = kobject_init_and_add(
297 &zone->kobj, &ttm_mem_zone_kobj_type, &glob->kobj, "%s",
298 zone->name);
299 if (unlikely(ret != 0)) {
300 kobject_put(&zone->kobj);
301 return ret;
302 }
303 glob->zones[glob->num_zones++] = zone;
304 return 0;
305 }
306 #else
ttm_mem_init_dma32_zone(struct ttm_mem_global * glob,const struct sysinfo * si)307 static int ttm_mem_init_dma32_zone(struct ttm_mem_global *glob,
308 const struct sysinfo *si)
309 {
310 struct ttm_mem_zone *zone = kzalloc(sizeof(*zone), GFP_KERNEL);
311 uint64_t mem;
312 int ret;
313
314 if (unlikely(!zone))
315 return -ENOMEM;
316
317 mem = si->totalram;
318 mem *= si->mem_unit;
319
320 /**
321 * No special dma32 zone needed.
322 */
323
324 if (mem <= ((uint64_t) 1ULL << 32)) {
325 kfree(zone);
326 return 0;
327 }
328
329 /*
330 * Limit max dma32 memory to 4GB for now
331 * until we can figure out how big this
332 * zone really is.
333 */
334
335 mem = ((uint64_t) 1ULL << 32);
336 zone->name = "dma32";
337 zone->zone_mem = mem;
338 zone->max_mem = mem >> 1;
339 zone->emer_mem = (mem >> 1) + (mem >> 2);
340 zone->swap_limit = zone->max_mem - (mem >> 3);
341 zone->used_mem = 0;
342 zone->glob = glob;
343 glob->zone_dma32 = zone;
344 ret = kobject_init_and_add(
345 &zone->kobj, &ttm_mem_zone_kobj_type, &glob->kobj, zone->name);
346 if (unlikely(ret != 0)) {
347 kobject_put(&zone->kobj);
348 return ret;
349 }
350 glob->zones[glob->num_zones++] = zone;
351 return 0;
352 }
353 #endif
354
ttm_mem_global_init(struct ttm_mem_global * glob,struct device * dev)355 int ttm_mem_global_init(struct ttm_mem_global *glob, struct device *dev)
356 {
357 struct sysinfo si;
358 int ret;
359 int i;
360 struct ttm_mem_zone *zone;
361
362 spin_lock_init(&glob->lock);
363 glob->swap_queue = create_singlethread_workqueue("ttm_swap");
364 INIT_WORK(&glob->work, ttm_shrink_work);
365
366 ret = kobject_init_and_add(&glob->kobj, &ttm_mem_glob_kobj_type,
367 &dev->kobj, "memory_accounting");
368 if (unlikely(ret != 0)) {
369 kobject_put(&glob->kobj);
370 return ret;
371 }
372
373 si_meminfo(&si);
374
375 ret = ttm_mem_init_kernel_zone(glob, &si);
376 if (unlikely(ret != 0))
377 goto out_no_zone;
378 #ifdef CONFIG_HIGHMEM
379 ret = ttm_mem_init_highmem_zone(glob, &si);
380 if (unlikely(ret != 0))
381 goto out_no_zone;
382 #else
383 ret = ttm_mem_init_dma32_zone(glob, &si);
384 if (unlikely(ret != 0))
385 goto out_no_zone;
386 #endif
387 for (i = 0; i < glob->num_zones; ++i) {
388 zone = glob->zones[i];
389 pr_info("Zone %7s: Available graphics memory: %llu KiB\n",
390 zone->name, (unsigned long long)zone->max_mem >> 10);
391 }
392 return 0;
393 out_no_zone:
394 ttm_mem_global_release(glob);
395 return ret;
396 }
397
ttm_mem_global_release(struct ttm_mem_global * glob)398 void ttm_mem_global_release(struct ttm_mem_global *glob)
399 {
400 struct ttm_mem_zone *zone;
401 unsigned int i;
402
403 flush_workqueue(glob->swap_queue);
404 destroy_workqueue(glob->swap_queue);
405 glob->swap_queue = NULL;
406 for (i = 0; i < glob->num_zones; ++i) {
407 zone = glob->zones[i];
408 kobject_del(&zone->kobj);
409 kobject_put(&zone->kobj);
410 }
411 kobject_del(&glob->kobj);
412 kobject_put(&glob->kobj);
413 memset(glob, 0, sizeof(*glob));
414 }
415
ttm_check_swapping(struct ttm_mem_global * glob)416 static void ttm_check_swapping(struct ttm_mem_global *glob)
417 {
418 bool needs_swapping = false;
419 unsigned int i;
420 struct ttm_mem_zone *zone;
421
422 spin_lock(&glob->lock);
423 for (i = 0; i < glob->num_zones; ++i) {
424 zone = glob->zones[i];
425 if (zone->used_mem > zone->swap_limit) {
426 needs_swapping = true;
427 break;
428 }
429 }
430
431 spin_unlock(&glob->lock);
432
433 if (unlikely(needs_swapping))
434 (void)queue_work(glob->swap_queue, &glob->work);
435
436 }
437
ttm_mem_global_free_zone(struct ttm_mem_global * glob,struct ttm_mem_zone * single_zone,uint64_t amount)438 static void ttm_mem_global_free_zone(struct ttm_mem_global *glob,
439 struct ttm_mem_zone *single_zone,
440 uint64_t amount)
441 {
442 unsigned int i;
443 struct ttm_mem_zone *zone;
444
445 spin_lock(&glob->lock);
446 for (i = 0; i < glob->num_zones; ++i) {
447 zone = glob->zones[i];
448 if (single_zone && zone != single_zone)
449 continue;
450 zone->used_mem -= amount;
451 }
452 spin_unlock(&glob->lock);
453 }
454
ttm_mem_global_free(struct ttm_mem_global * glob,uint64_t amount)455 void ttm_mem_global_free(struct ttm_mem_global *glob,
456 uint64_t amount)
457 {
458 return ttm_mem_global_free_zone(glob, glob->zone_kernel, amount);
459 }
460 EXPORT_SYMBOL(ttm_mem_global_free);
461
ttm_mem_global_reserve(struct ttm_mem_global * glob,struct ttm_mem_zone * single_zone,uint64_t amount,bool reserve)462 static int ttm_mem_global_reserve(struct ttm_mem_global *glob,
463 struct ttm_mem_zone *single_zone,
464 uint64_t amount, bool reserve)
465 {
466 uint64_t limit;
467 int ret = -ENOMEM;
468 unsigned int i;
469 struct ttm_mem_zone *zone;
470
471 spin_lock(&glob->lock);
472 for (i = 0; i < glob->num_zones; ++i) {
473 zone = glob->zones[i];
474 if (single_zone && zone != single_zone)
475 continue;
476
477 limit = (capable(CAP_SYS_ADMIN)) ?
478 zone->emer_mem : zone->max_mem;
479
480 if (zone->used_mem > limit)
481 goto out_unlock;
482 }
483
484 if (reserve) {
485 for (i = 0; i < glob->num_zones; ++i) {
486 zone = glob->zones[i];
487 if (single_zone && zone != single_zone)
488 continue;
489 zone->used_mem += amount;
490 }
491 }
492
493 ret = 0;
494 out_unlock:
495 spin_unlock(&glob->lock);
496 ttm_check_swapping(glob);
497
498 return ret;
499 }
500
501
ttm_mem_global_alloc_zone(struct ttm_mem_global * glob,struct ttm_mem_zone * single_zone,uint64_t memory,struct ttm_operation_ctx * ctx)502 static int ttm_mem_global_alloc_zone(struct ttm_mem_global *glob,
503 struct ttm_mem_zone *single_zone,
504 uint64_t memory,
505 struct ttm_operation_ctx *ctx)
506 {
507 int count = TTM_MEMORY_ALLOC_RETRIES;
508
509 while (unlikely(ttm_mem_global_reserve(glob,
510 single_zone,
511 memory, true)
512 != 0)) {
513 if (ctx->no_wait_gpu)
514 return -ENOMEM;
515 if (unlikely(count-- == 0))
516 return -ENOMEM;
517 ttm_shrink(glob, false, memory + (memory >> 2) + 16, ctx);
518 }
519
520 return 0;
521 }
522
ttm_mem_global_alloc(struct ttm_mem_global * glob,uint64_t memory,struct ttm_operation_ctx * ctx)523 int ttm_mem_global_alloc(struct ttm_mem_global *glob, uint64_t memory,
524 struct ttm_operation_ctx *ctx)
525 {
526 /**
527 * Normal allocations of kernel memory are registered in
528 * the kernel zone.
529 */
530
531 return ttm_mem_global_alloc_zone(glob, glob->zone_kernel, memory, ctx);
532 }
533 EXPORT_SYMBOL(ttm_mem_global_alloc);
534
ttm_mem_global_alloc_page(struct ttm_mem_global * glob,struct page * page,uint64_t size,struct ttm_operation_ctx * ctx)535 int ttm_mem_global_alloc_page(struct ttm_mem_global *glob,
536 struct page *page, uint64_t size,
537 struct ttm_operation_ctx *ctx)
538 {
539 struct ttm_mem_zone *zone = NULL;
540
541 /**
542 * Page allocations may be registed in a single zone
543 * only if highmem or !dma32.
544 */
545
546 #ifdef CONFIG_HIGHMEM
547 if (PageHighMem(page) && glob->zone_highmem != NULL)
548 zone = glob->zone_highmem;
549 #else
550 if (glob->zone_dma32 && page_to_pfn(page) > 0x00100000UL)
551 zone = glob->zone_kernel;
552 #endif
553 return ttm_mem_global_alloc_zone(glob, zone, size, ctx);
554 }
555
ttm_mem_global_free_page(struct ttm_mem_global * glob,struct page * page,uint64_t size)556 void ttm_mem_global_free_page(struct ttm_mem_global *glob, struct page *page,
557 uint64_t size)
558 {
559 struct ttm_mem_zone *zone = NULL;
560
561 #ifdef CONFIG_HIGHMEM
562 if (PageHighMem(page) && glob->zone_highmem != NULL)
563 zone = glob->zone_highmem;
564 #else
565 if (glob->zone_dma32 && page_to_pfn(page) > 0x00100000UL)
566 zone = glob->zone_kernel;
567 #endif
568 ttm_mem_global_free_zone(glob, zone, size);
569 }
570
ttm_round_pot(size_t size)571 size_t ttm_round_pot(size_t size)
572 {
573 if ((size & (size - 1)) == 0)
574 return size;
575 else if (size > PAGE_SIZE)
576 return PAGE_ALIGN(size);
577 else {
578 size_t tmp_size = 4;
579
580 while (tmp_size < size)
581 tmp_size <<= 1;
582
583 return tmp_size;
584 }
585 return 0;
586 }
587 EXPORT_SYMBOL(ttm_round_pot);
588