1 /*
2 * Copyright © 2009 Corbin Simpson
3 * Copyright © 2011 Marek Olšák <maraeo@gmail.com>
4 *
5 * SPDX-License-Identifier: MIT
6 */
7
8 #include "radeon_drm_bo.h"
9 #include "radeon_drm_cs.h"
10
11 #include "util/os_file.h"
12 #include "util/simple_mtx.h"
13 #include "util/thread_sched.h"
14 #include "util/u_cpu_detect.h"
15 #include "util/u_memory.h"
16 #include "util/u_hash_table.h"
17 #include "util/u_pointer.h"
18
19 #include <xf86drm.h>
20 #include <stdio.h>
21 #include <sys/types.h>
22 #include <sys/stat.h>
23 #include <unistd.h>
24 #include <fcntl.h>
25 #include <radeon_surface.h>
26
27 static struct hash_table *fd_tab = NULL;
28 static simple_mtx_t fd_tab_mutex = SIMPLE_MTX_INITIALIZER;
29
30 /* Enable/disable feature access for one command stream.
31 * If enable == true, return true on success.
32 * Otherwise, return false.
33 *
34 * We basically do the same thing kernel does, because we have to deal
35 * with multiple contexts (here command streams) backed by one winsys. */
radeon_set_fd_access(struct radeon_drm_cs * applier,struct radeon_drm_cs ** owner,mtx_t * mutex,unsigned request,const char * request_name,bool enable)36 static bool radeon_set_fd_access(struct radeon_drm_cs *applier,
37 struct radeon_drm_cs **owner,
38 mtx_t *mutex,
39 unsigned request, const char *request_name,
40 bool enable)
41 {
42 struct drm_radeon_info info;
43 unsigned value = enable ? 1 : 0;
44
45 memset(&info, 0, sizeof(info));
46
47 mtx_lock(&*mutex);
48
49 /* Early exit if we are sure the request will fail. */
50 if (enable) {
51 if (*owner) {
52 mtx_unlock(&*mutex);
53 return false;
54 }
55 } else {
56 if (*owner != applier) {
57 mtx_unlock(&*mutex);
58 return false;
59 }
60 }
61
62 /* Pass through the request to the kernel. */
63 info.value = (unsigned long)&value;
64 info.request = request;
65 if (drmCommandWriteRead(applier->ws->fd, DRM_RADEON_INFO,
66 &info, sizeof(info)) != 0) {
67 mtx_unlock(&*mutex);
68 return false;
69 }
70
71 /* Update the rights in the winsys. */
72 if (enable) {
73 if (value) {
74 *owner = applier;
75 mtx_unlock(&*mutex);
76 return true;
77 }
78 } else {
79 *owner = NULL;
80 }
81
82 mtx_unlock(&*mutex);
83 return false;
84 }
85
radeon_get_drm_value(int fd,unsigned request,const char * errname,uint32_t * out)86 static bool radeon_get_drm_value(int fd, unsigned request,
87 const char *errname, uint32_t *out)
88 {
89 struct drm_radeon_info info;
90 int retval;
91
92 memset(&info, 0, sizeof(info));
93
94 info.value = (unsigned long)out;
95 info.request = request;
96
97 retval = drmCommandWriteRead(fd, DRM_RADEON_INFO, &info, sizeof(info));
98 if (retval) {
99 if (errname) {
100 fprintf(stderr, "radeon: Failed to get %s, error number %d\n",
101 errname, retval);
102 }
103 return false;
104 }
105 return true;
106 }
107
108 /* Helper function to do the ioctls needed for setup and init. */
do_winsys_init(struct radeon_drm_winsys * ws)109 static bool do_winsys_init(struct radeon_drm_winsys *ws)
110 {
111 struct drm_radeon_gem_info gem_info;
112 int retval;
113 drmVersionPtr version;
114
115 memset(&gem_info, 0, sizeof(gem_info));
116
117 /* We do things in a specific order here.
118 *
119 * DRM version first. We need to be sure we're running on a KMS chipset.
120 * This is also for some features.
121 *
122 * Then, the PCI ID. This is essential and should return usable numbers
123 * for all Radeons. If this fails, we probably got handed an FD for some
124 * non-Radeon card.
125 *
126 * The GEM info is actually bogus on the kernel side, as well as our side
127 * (see radeon_gem_info_ioctl in radeon_gem.c) but that's alright because
128 * we don't actually use the info for anything yet.
129 *
130 * The GB and Z pipe requests should always succeed, but they might not
131 * return sensical values for all chipsets, but that's alright because
132 * the pipe drivers already know that.
133 */
134
135 /* Get DRM version. */
136 version = drmGetVersion(ws->fd);
137 if (!version)
138 return false;
139
140 if (version->version_major != 2 ||
141 version->version_minor < 50) {
142 fprintf(stderr, "%s: DRM version is %d.%d.%d but this driver is "
143 "only compatible with 2.50.0 (kernel 4.12) or later.\n",
144 __func__,
145 version->version_major,
146 version->version_minor,
147 version->version_patchlevel);
148 drmFreeVersion(version);
149 return false;
150 }
151
152 ws->info.drm_major = version->version_major;
153 ws->info.drm_minor = version->version_minor;
154 ws->info.drm_patchlevel = version->version_patchlevel;
155 ws->info.is_amdgpu = false;
156 drmFreeVersion(version);
157
158 /* Get PCI ID. */
159 if (!radeon_get_drm_value(ws->fd, RADEON_INFO_DEVICE_ID, "PCI ID",
160 &ws->info.pci_id))
161 return false;
162
163 /* Check PCI ID. */
164 switch (ws->info.pci_id) {
165 #define CHIPSET(pci_id, name, cfamily) case pci_id: ws->info.family = CHIP_##cfamily; ws->gen = DRV_R300; break;
166 #include "pci_ids/r300_pci_ids.h"
167 #undef CHIPSET
168
169 #define CHIPSET(pci_id, name, cfamily) case pci_id: ws->info.family = CHIP_##cfamily; ws->gen = DRV_R600; break;
170 #include "pci_ids/r600_pci_ids.h"
171 #undef CHIPSET
172
173 #define CHIPSET(pci_id, cfamily) \
174 case pci_id: \
175 ws->info.family = CHIP_##cfamily; \
176 ws->info.name = #cfamily; \
177 ws->gen = DRV_SI; \
178 break;
179 #include "pci_ids/radeonsi_pci_ids.h"
180 #undef CHIPSET
181
182 default:
183 fprintf(stderr, "radeon: Invalid PCI ID.\n");
184 return false;
185 }
186
187 switch (ws->info.family) {
188 default:
189 case CHIP_UNKNOWN:
190 fprintf(stderr, "radeon: Unknown family.\n");
191 return false;
192 case CHIP_R300:
193 case CHIP_R350:
194 case CHIP_RV350:
195 case CHIP_RV370:
196 case CHIP_RV380:
197 case CHIP_RS400:
198 case CHIP_RC410:
199 case CHIP_RS480:
200 ws->info.gfx_level = R300;
201 break;
202 case CHIP_R420: /* R4xx-based cores. */
203 case CHIP_R423:
204 case CHIP_R430:
205 case CHIP_R480:
206 case CHIP_R481:
207 case CHIP_RV410:
208 case CHIP_RS600:
209 case CHIP_RS690:
210 case CHIP_RS740:
211 ws->info.gfx_level = R400;
212 break;
213 case CHIP_RV515: /* R5xx-based cores. */
214 case CHIP_R520:
215 case CHIP_RV530:
216 case CHIP_R580:
217 case CHIP_RV560:
218 case CHIP_RV570:
219 ws->info.gfx_level = R500;
220 break;
221 case CHIP_R600:
222 case CHIP_RV610:
223 case CHIP_RV630:
224 case CHIP_RV670:
225 case CHIP_RV620:
226 case CHIP_RV635:
227 case CHIP_RS780:
228 case CHIP_RS880:
229 ws->info.gfx_level = R600;
230 break;
231 case CHIP_RV770:
232 case CHIP_RV730:
233 case CHIP_RV710:
234 case CHIP_RV740:
235 ws->info.gfx_level = R700;
236 break;
237 case CHIP_CEDAR:
238 case CHIP_REDWOOD:
239 case CHIP_JUNIPER:
240 case CHIP_CYPRESS:
241 case CHIP_HEMLOCK:
242 case CHIP_PALM:
243 case CHIP_SUMO:
244 case CHIP_SUMO2:
245 case CHIP_BARTS:
246 case CHIP_TURKS:
247 case CHIP_CAICOS:
248 ws->info.gfx_level = EVERGREEN;
249 break;
250 case CHIP_CAYMAN:
251 case CHIP_ARUBA:
252 ws->info.gfx_level = CAYMAN;
253 break;
254 case CHIP_TAHITI:
255 case CHIP_PITCAIRN:
256 case CHIP_VERDE:
257 case CHIP_OLAND:
258 case CHIP_HAINAN:
259 ws->info.gfx_level = GFX6;
260 break;
261 case CHIP_BONAIRE:
262 case CHIP_KAVERI:
263 case CHIP_KABINI:
264 case CHIP_HAWAII:
265 ws->info.gfx_level = GFX7;
266 break;
267 }
268
269 /* Set which chips don't have dedicated VRAM. */
270 switch (ws->info.family) {
271 case CHIP_RS400:
272 case CHIP_RC410:
273 case CHIP_RS480:
274 case CHIP_RS600:
275 case CHIP_RS690:
276 case CHIP_RS740:
277 case CHIP_RS780:
278 case CHIP_RS880:
279 case CHIP_PALM:
280 case CHIP_SUMO:
281 case CHIP_SUMO2:
282 case CHIP_ARUBA:
283 case CHIP_KAVERI:
284 case CHIP_KABINI:
285 ws->info.has_dedicated_vram = false;
286 break;
287
288 default:
289 ws->info.has_dedicated_vram = true;
290 }
291
292 ws->info.ip[AMD_IP_GFX].num_queues = 1;
293 /* Check for dma */
294 ws->info.ip[AMD_IP_SDMA].num_queues = 0;
295 /* DMA is disabled on R700. There is IB corruption and hangs. */
296 if (ws->info.gfx_level >= EVERGREEN) {
297 ws->info.ip[AMD_IP_SDMA].num_queues = 1;
298 }
299
300 /* Check for UVD and VCE */
301 ws->info.vce_fw_version = 0x00000000;
302
303 uint32_t value = RADEON_CS_RING_UVD;
304 if (radeon_get_drm_value(ws->fd, RADEON_INFO_RING_WORKING,
305 "UVD Ring working", &value)) {
306 ws->info.ip[AMD_IP_UVD].num_queues = 1;
307 }
308
309 value = RADEON_CS_RING_VCE;
310 if (radeon_get_drm_value(ws->fd, RADEON_INFO_RING_WORKING,
311 NULL, &value) && value) {
312
313 if (radeon_get_drm_value(ws->fd, RADEON_INFO_VCE_FW_VERSION,
314 "VCE FW version", &value)) {
315 ws->info.vce_fw_version = value;
316 ws->info.ip[AMD_IP_VCE].num_queues = 1;
317 }
318 }
319
320 /* Check for userptr support. */
321 {
322 struct drm_radeon_gem_userptr args = {0};
323
324 /* If the ioctl doesn't exist, -EINVAL is returned.
325 *
326 * If the ioctl exists, it should return -EACCES
327 * if RADEON_GEM_USERPTR_READONLY or RADEON_GEM_USERPTR_REGISTER
328 * aren't set.
329 */
330 ws->info.has_userptr =
331 drmCommandWriteRead(ws->fd, DRM_RADEON_GEM_USERPTR,
332 &args, sizeof(args)) == -EACCES;
333 }
334
335 /* Get GEM info. */
336 retval = drmCommandWriteRead(ws->fd, DRM_RADEON_GEM_INFO,
337 &gem_info, sizeof(gem_info));
338 if (retval) {
339 fprintf(stderr, "radeon: Failed to get MM info, error number %d\n",
340 retval);
341 return false;
342 }
343 ws->info.gart_size_kb = DIV_ROUND_UP(gem_info.gart_size, 1024);
344 ws->info.vram_size_kb = DIV_ROUND_UP(gem_info.vram_size, 1024);
345 ws->info.vram_vis_size_kb = DIV_ROUND_UP(gem_info.vram_visible, 1024);
346
347 /* Radeon allocates all buffers contiguously, which makes large allocations
348 * unlikely to succeed. */
349 if (ws->info.has_dedicated_vram)
350 ws->info.max_heap_size_kb = ws->info.vram_size_kb;
351 else
352 ws->info.max_heap_size_kb = ws->info.gart_size_kb;
353
354 /* Both 32-bit and 64-bit address spaces only have 4GB.
355 * This is a limitation of the VM allocator in the winsys.
356 */
357 ws->info.max_heap_size_kb = MIN2(ws->info.max_heap_size_kb, 4 * 1024 * 1024); /* 4 GB */
358
359 /* Get max clock frequency info and convert it to MHz */
360 radeon_get_drm_value(ws->fd, RADEON_INFO_MAX_SCLK, NULL,
361 &ws->info.max_gpu_freq_mhz);
362 ws->info.max_gpu_freq_mhz /= 1000;
363
364 ws->num_cpus = sysconf(_SC_NPROCESSORS_ONLN);
365
366 /* Generation-specific queries. */
367 if (ws->gen == DRV_R300) {
368 if (!radeon_get_drm_value(ws->fd, RADEON_INFO_NUM_GB_PIPES,
369 "GB pipe count",
370 &ws->info.r300_num_gb_pipes))
371 return false;
372
373 if (!radeon_get_drm_value(ws->fd, RADEON_INFO_NUM_Z_PIPES,
374 "Z pipe count",
375 &ws->info.r300_num_z_pipes))
376 return false;
377 }
378 else if (ws->gen >= DRV_R600) {
379 uint32_t tiling_config = 0;
380
381 if (!radeon_get_drm_value(ws->fd, RADEON_INFO_NUM_BACKENDS,
382 "num backends",
383 &ws->info.max_render_backends))
384 return false;
385
386 /* get the GPU counter frequency, failure is not fatal */
387 radeon_get_drm_value(ws->fd, RADEON_INFO_CLOCK_CRYSTAL_FREQ, NULL,
388 &ws->info.clock_crystal_freq);
389
390 radeon_get_drm_value(ws->fd, RADEON_INFO_TILING_CONFIG, NULL,
391 &tiling_config);
392
393 ws->info.r600_num_banks =
394 ws->info.gfx_level >= EVERGREEN ?
395 4 << ((tiling_config & 0xf0) >> 4) :
396 4 << ((tiling_config & 0x30) >> 4);
397
398 ws->info.pipe_interleave_bytes =
399 ws->info.gfx_level >= EVERGREEN ?
400 256 << ((tiling_config & 0xf00) >> 8) :
401 256 << ((tiling_config & 0xc0) >> 6);
402
403 if (!ws->info.pipe_interleave_bytes)
404 ws->info.pipe_interleave_bytes =
405 ws->info.gfx_level >= EVERGREEN ? 512 : 256;
406
407 radeon_get_drm_value(ws->fd, RADEON_INFO_NUM_TILE_PIPES, NULL,
408 &ws->info.num_tile_pipes);
409
410 /* "num_tiles_pipes" must be equal to the number of pipes (Px) in the
411 * pipe config field of the GB_TILE_MODE array. Only one card (Tahiti)
412 * reports a different value (12). Fix it by setting what's in the
413 * GB_TILE_MODE array (8).
414 */
415 if (ws->gen == DRV_SI && ws->info.num_tile_pipes == 12)
416 ws->info.num_tile_pipes = 8;
417
418 if (radeon_get_drm_value(ws->fd, RADEON_INFO_BACKEND_MAP, NULL,
419 &ws->info.r600_gb_backend_map))
420 ws->info.r600_gb_backend_map_valid = true;
421
422 /* Default value. */
423 ws->info.enabled_rb_mask = u_bit_consecutive(0, ws->info.max_render_backends);
424 /*
425 * This fails (silently) on non-GCN or older kernels, overwriting the
426 * default enabled_rb_mask with the result of the last query.
427 */
428 if (ws->gen >= DRV_SI) {
429 uint32_t mask;
430
431 radeon_get_drm_value(ws->fd, RADEON_INFO_SI_BACKEND_ENABLED_MASK, NULL, &mask);
432 ws->info.enabled_rb_mask = mask;
433 }
434
435 ws->info.r600_has_virtual_memory = false;
436
437 uint32_t ib_vm_max_size;
438
439 ws->info.r600_has_virtual_memory = true;
440 if (!radeon_get_drm_value(ws->fd, RADEON_INFO_VA_START, NULL,
441 &ws->va_start))
442 ws->info.r600_has_virtual_memory = false;
443 if (!radeon_get_drm_value(ws->fd, RADEON_INFO_IB_VM_MAX_SIZE, NULL,
444 &ib_vm_max_size))
445 ws->info.r600_has_virtual_memory = false;
446 radeon_get_drm_value(ws->fd, RADEON_INFO_VA_UNMAP_WORKING, NULL,
447 &ws->va_unmap_working);
448
449 if (ws->gen == DRV_R600 && !debug_get_bool_option("RADEON_VA", false))
450 ws->info.r600_has_virtual_memory = false;
451 }
452
453 /* Get max pipes, this is only needed for compute shaders. All evergreen+
454 * chips have at least 2 pipes, so we use 2 as a default. */
455 ws->info.r600_max_quad_pipes = 2;
456 radeon_get_drm_value(ws->fd, RADEON_INFO_MAX_PIPES, NULL,
457 &ws->info.r600_max_quad_pipes);
458
459 /* All GPUs have at least one compute unit */
460 ws->info.num_cu = 1;
461 radeon_get_drm_value(ws->fd, RADEON_INFO_ACTIVE_CU_COUNT, NULL,
462 &ws->info.num_cu);
463
464 radeon_get_drm_value(ws->fd, RADEON_INFO_MAX_SE, NULL,
465 &ws->info.max_se);
466
467 switch (ws->info.family) {
468 case CHIP_HAINAN:
469 case CHIP_KABINI:
470 ws->info.max_tcc_blocks = 2;
471 break;
472 case CHIP_VERDE:
473 case CHIP_OLAND:
474 case CHIP_BONAIRE:
475 case CHIP_KAVERI:
476 ws->info.max_tcc_blocks = 4;
477 break;
478 case CHIP_PITCAIRN:
479 ws->info.max_tcc_blocks = 8;
480 break;
481 case CHIP_TAHITI:
482 ws->info.max_tcc_blocks = 12;
483 break;
484 case CHIP_HAWAII:
485 ws->info.max_tcc_blocks = 16;
486 break;
487 default:
488 ws->info.max_tcc_blocks = 0;
489 break;
490 }
491
492 if (!ws->info.max_se) {
493 switch (ws->info.family) {
494 default:
495 ws->info.max_se = 1;
496 break;
497 case CHIP_CYPRESS:
498 case CHIP_HEMLOCK:
499 case CHIP_BARTS:
500 case CHIP_CAYMAN:
501 case CHIP_TAHITI:
502 case CHIP_PITCAIRN:
503 case CHIP_BONAIRE:
504 ws->info.max_se = 2;
505 break;
506 case CHIP_HAWAII:
507 ws->info.max_se = 4;
508 break;
509 }
510 }
511
512 ws->info.num_se = ws->info.max_se;
513
514 radeon_get_drm_value(ws->fd, RADEON_INFO_MAX_SH_PER_SE, NULL,
515 &ws->info.max_sa_per_se);
516 if (ws->gen == DRV_SI) {
517 ws->info.max_good_cu_per_sa =
518 ws->info.min_good_cu_per_sa = ws->info.num_cu /
519 (ws->info.max_se * ws->info.max_sa_per_se);
520 }
521
522 radeon_get_drm_value(ws->fd, RADEON_INFO_ACCEL_WORKING2, NULL,
523 &ws->accel_working2);
524 if (ws->info.family == CHIP_HAWAII && ws->accel_working2 < 2) {
525 fprintf(stderr, "radeon: GPU acceleration for Hawaii disabled, "
526 "returned accel_working2 value %u is smaller than 2. "
527 "Please install a newer kernel.\n",
528 ws->accel_working2);
529 return false;
530 }
531
532 if (ws->info.gfx_level == GFX7) {
533 if (!radeon_get_drm_value(ws->fd, RADEON_INFO_CIK_MACROTILE_MODE_ARRAY, NULL,
534 ws->info.cik_macrotile_mode_array)) {
535 fprintf(stderr, "radeon: Kernel 3.13 is required for Sea Islands support.\n");
536 return false;
537 }
538 }
539
540 if (ws->info.gfx_level >= GFX6) {
541 if (!radeon_get_drm_value(ws->fd, RADEON_INFO_SI_TILE_MODE_ARRAY, NULL,
542 ws->info.si_tile_mode_array)) {
543 fprintf(stderr, "radeon: Kernel 3.10 is required for Southern Islands support.\n");
544 return false;
545 }
546 }
547
548 for (unsigned ip_type = 0; ip_type < AMD_NUM_IP_TYPES; ip_type++)
549 ws->info.ip[ip_type].ib_alignment = 4096;
550
551 /* Hawaii with old firmware needs type2 nop packet.
552 * accel_working2 with value 3 indicates the new firmware.
553 */
554 ws->info.gfx_ib_pad_with_type2 = ws->info.gfx_level <= GFX6 ||
555 (ws->info.family == CHIP_HAWAII &&
556 ws->accel_working2 < 3);
557 ws->info.tcc_cache_line_size = 64; /* TC L2 line size on GCN */
558 ws->info.has_bo_metadata = false;
559 ws->info.has_eqaa_surface_allocator = false;
560 ws->info.has_sparse_vm_mappings = false;
561 ws->info.max_alignment = 1024*1024;
562 ws->info.has_graphics = true;
563 ws->info.cpdma_prefetch_writes_memory = true;
564 ws->info.max_waves_per_simd = 10;
565 ws->info.num_physical_sgprs_per_simd = 512;
566 ws->info.num_physical_wave64_vgprs_per_simd = 256;
567 ws->info.has_3d_cube_border_color_mipmap = true;
568 ws->info.has_image_opcodes = true;
569 ws->info.spi_cu_en_has_effect = false;
570 ws->info.spi_cu_en = 0xffff;
571 ws->info.never_stop_sq_perf_counters = false;
572 ws->info.num_rb = util_bitcount64(ws->info.enabled_rb_mask);
573 ws->info.max_gflops = 128 * ws->info.num_cu * ws->info.max_gpu_freq_mhz / 1000;
574 ws->info.num_tcc_blocks = ws->info.max_tcc_blocks;
575 ws->info.tcp_cache_size = 16 * 1024;
576 ws->info.num_simd_per_compute_unit = 4;
577 ws->info.min_sgpr_alloc = 8;
578 ws->info.max_sgpr_alloc = 104;
579 ws->info.sgpr_alloc_granularity = 8;
580 ws->info.min_wave64_vgpr_alloc = 4;
581 ws->info.max_vgpr_alloc = 256;
582 ws->info.wave64_vgpr_alloc_granularity = 4;
583
584 for (unsigned se = 0; se < ws->info.max_se; se++) {
585 for (unsigned sa = 0; sa < ws->info.max_sa_per_se; sa++)
586 ws->info.cu_mask[se][sa] = BITFIELD_MASK(ws->info.max_good_cu_per_sa);
587 }
588
589 /* The maximum number of scratch waves. The number is only a function of the number of CUs.
590 * It should be large enough to hold at least 1 threadgroup. Use the minimum per-SA CU count.
591 */
592 const unsigned max_waves_per_tg = 1024 / 64; /* LLVM only supports 1024 threads per block */
593 ws->info.max_scratch_waves = MAX2(32 * ws->info.min_good_cu_per_sa * ws->info.max_sa_per_se *
594 ws->info.num_se, max_waves_per_tg);
595
596 switch (ws->info.family) {
597 case CHIP_TAHITI:
598 case CHIP_PITCAIRN:
599 case CHIP_OLAND:
600 case CHIP_HAWAII:
601 case CHIP_KABINI:
602 ws->info.l2_cache_size = ws->info.num_tcc_blocks * 64 * 1024;
603 break;
604 case CHIP_VERDE:
605 case CHIP_HAINAN:
606 case CHIP_BONAIRE:
607 case CHIP_KAVERI:
608 ws->info.l2_cache_size = ws->info.num_tcc_blocks * 128 * 1024;
609 break;
610 default:;
611 }
612
613 ws->info.ip[AMD_IP_GFX].num_queues = 1;
614
615 switch (ws->info.gfx_level) {
616 case R300:
617 case R400:
618 case R500:
619 ws->info.ip[AMD_IP_GFX].ver_major = 2;
620 break;
621 case R600:
622 case R700:
623 ws->info.ip[AMD_IP_GFX].ver_major = 3;
624 break;
625 case EVERGREEN:
626 ws->info.ip[AMD_IP_GFX].ver_major = 4;
627 break;
628 case CAYMAN:
629 ws->info.ip[AMD_IP_GFX].ver_major = 5;
630 break;
631 case GFX6:
632 ws->info.ip[AMD_IP_GFX].ver_major = 6;
633 break;
634 case GFX7:
635 ws->info.ip[AMD_IP_GFX].ver_major = 7;
636 break;
637 default:;
638 }
639
640 ws->check_vm = strstr(debug_get_option("R600_DEBUG", ""), "check_vm") != NULL ||
641 strstr(debug_get_option("AMD_DEBUG", ""), "check_vm") != NULL;
642 ws->noop_cs = debug_get_bool_option("RADEON_NOOP", false);
643
644 return true;
645 }
646
radeon_winsys_destroy(struct radeon_winsys * rws)647 static void radeon_winsys_destroy(struct radeon_winsys *rws)
648 {
649 struct radeon_drm_winsys *ws = (struct radeon_drm_winsys*)rws;
650
651 if (util_queue_is_initialized(&ws->cs_queue))
652 util_queue_destroy(&ws->cs_queue);
653
654 mtx_destroy(&ws->hyperz_owner_mutex);
655 mtx_destroy(&ws->cmask_owner_mutex);
656
657 if (ws->info.r600_has_virtual_memory)
658 pb_slabs_deinit(&ws->bo_slabs);
659 pb_cache_deinit(&ws->bo_cache);
660
661 if (ws->gen >= DRV_R600) {
662 radeon_surface_manager_free(ws->surf_man);
663 }
664
665 _mesa_hash_table_destroy(ws->bo_names, NULL);
666 _mesa_hash_table_destroy(ws->bo_handles, NULL);
667 _mesa_hash_table_u64_destroy(ws->bo_vas);
668 mtx_destroy(&ws->bo_handles_mutex);
669 mtx_destroy(&ws->vm32.mutex);
670 mtx_destroy(&ws->vm64.mutex);
671 mtx_destroy(&ws->bo_fence_lock);
672
673 if (ws->fd >= 0)
674 close(ws->fd);
675
676 FREE(rws);
677 }
678
radeon_query_info(struct radeon_winsys * rws,struct radeon_info * info)679 static void radeon_query_info(struct radeon_winsys *rws, struct radeon_info *info)
680 {
681 *info = ((struct radeon_drm_winsys *)rws)->info;
682 }
683
radeon_cs_request_feature(struct radeon_cmdbuf * rcs,enum radeon_feature_id fid,bool enable)684 static bool radeon_cs_request_feature(struct radeon_cmdbuf *rcs,
685 enum radeon_feature_id fid,
686 bool enable)
687 {
688 struct radeon_drm_cs *cs = radeon_drm_cs(rcs);
689
690 switch (fid) {
691 case RADEON_FID_R300_HYPERZ_ACCESS:
692 return radeon_set_fd_access(cs, &cs->ws->hyperz_owner,
693 &cs->ws->hyperz_owner_mutex,
694 RADEON_INFO_WANT_HYPERZ, "Hyper-Z",
695 enable);
696
697 case RADEON_FID_R300_CMASK_ACCESS:
698 return radeon_set_fd_access(cs, &cs->ws->cmask_owner,
699 &cs->ws->cmask_owner_mutex,
700 RADEON_INFO_WANT_CMASK, "AA optimizations",
701 enable);
702 }
703 return false;
704 }
705
radeon_drm_get_gpu_reset_counter(struct radeon_drm_winsys * ws)706 uint32_t radeon_drm_get_gpu_reset_counter(struct radeon_drm_winsys *ws)
707 {
708 uint64_t retval = 0;
709
710 radeon_get_drm_value(ws->fd, RADEON_INFO_GPU_RESET_COUNTER,
711 "gpu-reset-counter", (uint32_t*)&retval);
712 return retval;
713 }
714
radeon_query_value(struct radeon_winsys * rws,enum radeon_value_id value)715 static uint64_t radeon_query_value(struct radeon_winsys *rws,
716 enum radeon_value_id value)
717 {
718 struct radeon_drm_winsys *ws = (struct radeon_drm_winsys*)rws;
719 uint64_t retval = 0;
720
721 switch (value) {
722 case RADEON_REQUESTED_VRAM_MEMORY:
723 return ws->allocated_vram;
724 case RADEON_REQUESTED_GTT_MEMORY:
725 return ws->allocated_gtt;
726 case RADEON_MAPPED_VRAM:
727 return ws->mapped_vram;
728 case RADEON_MAPPED_GTT:
729 return ws->mapped_gtt;
730 case RADEON_BUFFER_WAIT_TIME_NS:
731 return ws->buffer_wait_time;
732 case RADEON_NUM_MAPPED_BUFFERS:
733 return ws->num_mapped_buffers;
734 case RADEON_TIMESTAMP:
735 if (ws->gen < DRV_R600) {
736 assert(0);
737 return 0;
738 }
739
740 radeon_get_drm_value(ws->fd, RADEON_INFO_TIMESTAMP, "timestamp",
741 (uint32_t*)&retval);
742 return retval;
743 case RADEON_NUM_GFX_IBS:
744 return ws->num_gfx_IBs;
745 case RADEON_NUM_SDMA_IBS:
746 return ws->num_sdma_IBs;
747 case RADEON_NUM_BYTES_MOVED:
748 radeon_get_drm_value(ws->fd, RADEON_INFO_NUM_BYTES_MOVED,
749 "num-bytes-moved", (uint32_t*)&retval);
750 return retval;
751 case RADEON_NUM_EVICTIONS:
752 case RADEON_NUM_VRAM_CPU_PAGE_FAULTS:
753 case RADEON_VRAM_VIS_USAGE:
754 case RADEON_GFX_BO_LIST_COUNTER:
755 case RADEON_GFX_IB_SIZE_COUNTER:
756 case RADEON_SLAB_WASTED_VRAM:
757 case RADEON_SLAB_WASTED_GTT:
758 return 0; /* unimplemented */
759 case RADEON_VRAM_USAGE:
760 radeon_get_drm_value(ws->fd, RADEON_INFO_VRAM_USAGE,
761 "vram-usage", (uint32_t*)&retval);
762 return retval;
763 case RADEON_GTT_USAGE:
764 radeon_get_drm_value(ws->fd, RADEON_INFO_GTT_USAGE,
765 "gtt-usage", (uint32_t*)&retval);
766 return retval;
767 case RADEON_GPU_TEMPERATURE:
768 radeon_get_drm_value(ws->fd, RADEON_INFO_CURRENT_GPU_TEMP,
769 "gpu-temp", (uint32_t*)&retval);
770 return retval;
771 case RADEON_CURRENT_SCLK:
772 radeon_get_drm_value(ws->fd, RADEON_INFO_CURRENT_GPU_SCLK,
773 "current-gpu-sclk", (uint32_t*)&retval);
774 return retval;
775 case RADEON_CURRENT_MCLK:
776 radeon_get_drm_value(ws->fd, RADEON_INFO_CURRENT_GPU_MCLK,
777 "current-gpu-mclk", (uint32_t*)&retval);
778 return retval;
779 case RADEON_CS_THREAD_TIME:
780 return util_queue_get_thread_time_nano(&ws->cs_queue, 0);
781 }
782 return 0;
783 }
784
radeon_read_registers(struct radeon_winsys * rws,unsigned reg_offset,unsigned num_registers,uint32_t * out)785 static bool radeon_read_registers(struct radeon_winsys *rws,
786 unsigned reg_offset,
787 unsigned num_registers, uint32_t *out)
788 {
789 struct radeon_drm_winsys *ws = (struct radeon_drm_winsys*)rws;
790 unsigned i;
791
792 for (i = 0; i < num_registers; i++) {
793 uint32_t reg = reg_offset + i*4;
794
795 if (!radeon_get_drm_value(ws->fd, RADEON_INFO_READ_REG, NULL, ®))
796 return false;
797 out[i] = reg;
798 }
799 return true;
800 }
801
802 DEBUG_GET_ONCE_BOOL_OPTION(thread, "RADEON_THREAD", true)
803
radeon_winsys_unref(struct radeon_winsys * ws)804 static bool radeon_winsys_unref(struct radeon_winsys *ws)
805 {
806 struct radeon_drm_winsys *rws = (struct radeon_drm_winsys*)ws;
807 bool destroy;
808
809 /* When the reference counter drops to zero, remove the fd from the table.
810 * This must happen while the mutex is locked, so that
811 * radeon_drm_winsys_create in another thread doesn't get the winsys
812 * from the table when the counter drops to 0. */
813 simple_mtx_lock(&fd_tab_mutex);
814
815 destroy = pipe_reference(&rws->reference, NULL);
816 if (destroy && fd_tab) {
817 _mesa_hash_table_remove_key(fd_tab, intptr_to_pointer(rws->fd));
818 if (_mesa_hash_table_num_entries(fd_tab) == 0) {
819 _mesa_hash_table_destroy(fd_tab, NULL);
820 fd_tab = NULL;
821 }
822 }
823
824 simple_mtx_unlock(&fd_tab_mutex);
825 return destroy;
826 }
827
radeon_pin_threads_to_L3_cache(struct radeon_winsys * ws,unsigned cpu)828 static void radeon_pin_threads_to_L3_cache(struct radeon_winsys *ws,
829 unsigned cpu)
830 {
831 struct radeon_drm_winsys *rws = (struct radeon_drm_winsys*)ws;
832
833 if (util_queue_is_initialized(&rws->cs_queue)) {
834 util_thread_sched_apply_policy(rws->cs_queue.threads[0],
835 UTIL_THREAD_DRIVER_SUBMIT, cpu, NULL);
836 }
837 }
838
radeon_cs_is_secure(struct radeon_cmdbuf * cs)839 static bool radeon_cs_is_secure(struct radeon_cmdbuf* cs)
840 {
841 return false;
842 }
843
radeon_cs_set_pstate(struct radeon_cmdbuf * cs,enum radeon_ctx_pstate state)844 static bool radeon_cs_set_pstate(struct radeon_cmdbuf* cs, enum radeon_ctx_pstate state)
845 {
846 return false;
847 }
848
849 static int
radeon_drm_winsys_get_fd(struct radeon_winsys * ws)850 radeon_drm_winsys_get_fd(struct radeon_winsys *ws)
851 {
852 struct radeon_drm_winsys *rws = (struct radeon_drm_winsys*)ws;
853
854 return rws->fd;
855 }
856
857 PUBLIC struct radeon_winsys *
radeon_drm_winsys_create(int fd,const struct pipe_screen_config * config,radeon_screen_create_t screen_create)858 radeon_drm_winsys_create(int fd, const struct pipe_screen_config *config,
859 radeon_screen_create_t screen_create)
860 {
861 struct radeon_drm_winsys *ws;
862
863 simple_mtx_lock(&fd_tab_mutex);
864 if (!fd_tab) {
865 fd_tab = util_hash_table_create_fd_keys();
866 }
867
868 ws = util_hash_table_get(fd_tab, intptr_to_pointer(fd));
869 if (ws) {
870 pipe_reference(NULL, &ws->reference);
871 simple_mtx_unlock(&fd_tab_mutex);
872 return &ws->base;
873 }
874
875 ws = CALLOC_STRUCT(radeon_drm_winsys);
876 if (!ws) {
877 simple_mtx_unlock(&fd_tab_mutex);
878 return NULL;
879 }
880
881 ws->fd = os_dupfd_cloexec(fd);
882
883 if (!do_winsys_init(ws))
884 goto fail1;
885
886 pb_cache_init(&ws->bo_cache, RADEON_NUM_HEAPS,
887 500000, ws->check_vm ? 1.0f : 2.0f, 0,
888 (uint64_t)MIN2(ws->info.vram_size_kb, ws->info.gart_size_kb) * 1024,
889 offsetof(struct radeon_bo, u.real.cache_entry),
890 NULL,
891 radeon_bo_destroy,
892 radeon_bo_can_reclaim);
893
894 if (ws->info.r600_has_virtual_memory) {
895 /* There is no fundamental obstacle to using slab buffer allocation
896 * without GPUVM, but enabling it requires making sure that the drivers
897 * honor the address offset.
898 */
899 if (!pb_slabs_init(&ws->bo_slabs,
900 RADEON_SLAB_MIN_SIZE_LOG2, RADEON_SLAB_MAX_SIZE_LOG2,
901 RADEON_NUM_HEAPS, false,
902 ws,
903 radeon_bo_can_reclaim_slab,
904 radeon_bo_slab_alloc,
905 radeon_bo_slab_free))
906 goto fail_cache;
907
908 ws->info.min_alloc_size = 1 << RADEON_SLAB_MIN_SIZE_LOG2;
909 } else {
910 ws->info.min_alloc_size = ws->info.gart_page_size;
911 }
912
913 if (ws->gen >= DRV_R600) {
914 ws->surf_man = radeon_surface_manager_new(ws->fd);
915 if (!ws->surf_man)
916 goto fail_slab;
917 }
918
919 /* init reference */
920 pipe_reference_init(&ws->reference, 1);
921
922 /* Set functions. */
923 ws->base.unref = radeon_winsys_unref;
924 ws->base.destroy = radeon_winsys_destroy;
925 ws->base.get_fd = radeon_drm_winsys_get_fd;
926 ws->base.query_info = radeon_query_info;
927 ws->base.pin_threads_to_L3_cache = radeon_pin_threads_to_L3_cache;
928 ws->base.cs_request_feature = radeon_cs_request_feature;
929 ws->base.query_value = radeon_query_value;
930 ws->base.read_registers = radeon_read_registers;
931 ws->base.cs_is_secure = radeon_cs_is_secure;
932 ws->base.cs_set_pstate = radeon_cs_set_pstate;
933
934 radeon_drm_bo_init_functions(ws);
935 radeon_drm_cs_init_functions(ws);
936 radeon_surface_init_functions(ws);
937
938 (void) mtx_init(&ws->hyperz_owner_mutex, mtx_plain);
939 (void) mtx_init(&ws->cmask_owner_mutex, mtx_plain);
940
941 ws->bo_names = util_hash_table_create_ptr_keys();
942 ws->bo_handles = util_hash_table_create_ptr_keys();
943 ws->bo_vas = _mesa_hash_table_u64_create(NULL);
944 (void) mtx_init(&ws->bo_handles_mutex, mtx_plain);
945 (void) mtx_init(&ws->vm32.mutex, mtx_plain);
946 (void) mtx_init(&ws->vm64.mutex, mtx_plain);
947 (void) mtx_init(&ws->bo_fence_lock, mtx_plain);
948 list_inithead(&ws->vm32.holes);
949 list_inithead(&ws->vm64.holes);
950
951 /* The kernel currently returns 8MB. Make sure this doesn't change. */
952 if (ws->va_start > 8 * 1024 * 1024) {
953 /* Not enough 32-bit address space. */
954 radeon_winsys_destroy(&ws->base);
955 simple_mtx_unlock(&fd_tab_mutex);
956 return NULL;
957 }
958
959 ws->vm32.start = ws->va_start;
960 ws->vm32.end = 1ull << 32;
961
962 ws->vm64.start = 1ull << 32;
963 ws->vm64.end = 1ull << 33;
964
965 /* TTM aligns the BO size to the CPU page size */
966 ws->info.gart_page_size = sysconf(_SC_PAGESIZE);
967 ws->info.pte_fragment_size = 64 * 1024; /* GPUVM page size */
968
969 if (ws->num_cpus > 1 && debug_get_option_thread())
970 util_queue_init(&ws->cs_queue, "rcs", 8, 1, 0, NULL);
971
972 /* Create the screen at the end. The winsys must be initialized
973 * completely.
974 *
975 * Alternatively, we could create the screen based on "ws->gen"
976 * and link all drivers into one binary blob. */
977 ws->base.screen = screen_create(&ws->base, config);
978 if (!ws->base.screen) {
979 radeon_winsys_destroy(&ws->base);
980 simple_mtx_unlock(&fd_tab_mutex);
981 return NULL;
982 }
983
984 _mesa_hash_table_insert(fd_tab, intptr_to_pointer(ws->fd), ws);
985
986 /* We must unlock the mutex once the winsys is fully initialized, so that
987 * other threads attempting to create the winsys from the same fd will
988 * get a fully initialized winsys and not just half-way initialized. */
989 simple_mtx_unlock(&fd_tab_mutex);
990
991 return &ws->base;
992
993 fail_slab:
994 if (ws->info.r600_has_virtual_memory)
995 pb_slabs_deinit(&ws->bo_slabs);
996 fail_cache:
997 pb_cache_deinit(&ws->bo_cache);
998 fail1:
999 simple_mtx_unlock(&fd_tab_mutex);
1000 if (ws->surf_man)
1001 radeon_surface_manager_free(ws->surf_man);
1002 if (ws->fd >= 0)
1003 close(ws->fd);
1004
1005 FREE(ws);
1006 return NULL;
1007 }
1008