1 /*
2 * Copyright 2008 Jerome Glisse.
3 * All Rights Reserved.
4 *
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the "Software"),
7 * to deal in the Software without restriction, including without limitation
8 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9 * and/or sell copies of the Software, and to permit persons to whom the
10 * Software is furnished to do so, subject to the following conditions:
11 *
12 * The above copyright notice and this permission notice (including the next
13 * paragraph) shall be included in all copies or substantial portions of the
14 * Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * PRECISION INSIGHT AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
22 * DEALINGS IN THE SOFTWARE.
23 *
24 * Authors:
25 * Jerome Glisse <glisse@freedesktop.org>
26 */
27 #include <linux/list_sort.h>
28 #include <drm/drmP.h>
29 #include <drm/radeon_drm.h>
30 #include "radeon_reg.h"
31 #include "radeon.h"
32 #include "radeon_trace.h"
33
34 #define RADEON_CS_MAX_PRIORITY 32u
35 #define RADEON_CS_NUM_BUCKETS (RADEON_CS_MAX_PRIORITY + 1)
36
37 /* This is based on the bucket sort with O(n) time complexity.
38 * An item with priority "i" is added to bucket[i]. The lists are then
39 * concatenated in descending order.
40 */
41 struct radeon_cs_buckets {
42 struct list_head bucket[RADEON_CS_NUM_BUCKETS];
43 };
44
radeon_cs_buckets_init(struct radeon_cs_buckets * b)45 static void radeon_cs_buckets_init(struct radeon_cs_buckets *b)
46 {
47 unsigned i;
48
49 for (i = 0; i < RADEON_CS_NUM_BUCKETS; i++)
50 INIT_LIST_HEAD(&b->bucket[i]);
51 }
52
radeon_cs_buckets_add(struct radeon_cs_buckets * b,struct list_head * item,unsigned priority)53 static void radeon_cs_buckets_add(struct radeon_cs_buckets *b,
54 struct list_head *item, unsigned priority)
55 {
56 /* Since buffers which appear sooner in the relocation list are
57 * likely to be used more often than buffers which appear later
58 * in the list, the sort mustn't change the ordering of buffers
59 * with the same priority, i.e. it must be stable.
60 */
61 list_add_tail(item, &b->bucket[min(priority, RADEON_CS_MAX_PRIORITY)]);
62 }
63
radeon_cs_buckets_get_list(struct radeon_cs_buckets * b,struct list_head * out_list)64 static void radeon_cs_buckets_get_list(struct radeon_cs_buckets *b,
65 struct list_head *out_list)
66 {
67 unsigned i;
68
69 /* Connect the sorted buckets in the output list. */
70 for (i = 0; i < RADEON_CS_NUM_BUCKETS; i++) {
71 list_splice(&b->bucket[i], out_list);
72 }
73 }
74
radeon_cs_parser_relocs(struct radeon_cs_parser * p)75 static int radeon_cs_parser_relocs(struct radeon_cs_parser *p)
76 {
77 struct drm_device *ddev = p->rdev->ddev;
78 struct radeon_cs_chunk *chunk;
79 struct radeon_cs_buckets buckets;
80 unsigned i, j;
81 bool duplicate, need_mmap_lock = false;
82 int r;
83
84 if (p->chunk_relocs_idx == -1) {
85 return 0;
86 }
87 chunk = &p->chunks[p->chunk_relocs_idx];
88 p->dma_reloc_idx = 0;
89 /* FIXME: we assume that each relocs use 4 dwords */
90 p->nrelocs = chunk->length_dw / 4;
91 p->relocs_ptr = kcalloc(p->nrelocs, sizeof(void *), GFP_KERNEL);
92 if (p->relocs_ptr == NULL) {
93 return -ENOMEM;
94 }
95 p->relocs = kcalloc(p->nrelocs, sizeof(struct radeon_cs_reloc), GFP_KERNEL);
96 if (p->relocs == NULL) {
97 return -ENOMEM;
98 }
99
100 radeon_cs_buckets_init(&buckets);
101
102 for (i = 0; i < p->nrelocs; i++) {
103 struct drm_radeon_cs_reloc *r;
104 unsigned priority;
105
106 duplicate = false;
107 r = (struct drm_radeon_cs_reloc *)&chunk->kdata[i*4];
108 for (j = 0; j < i; j++) {
109 if (r->handle == p->relocs[j].handle) {
110 p->relocs_ptr[i] = &p->relocs[j];
111 duplicate = true;
112 break;
113 }
114 }
115 if (duplicate) {
116 p->relocs[i].handle = 0;
117 continue;
118 }
119
120 p->relocs[i].gobj = drm_gem_object_lookup(ddev, p->filp,
121 r->handle);
122 if (p->relocs[i].gobj == NULL) {
123 DRM_ERROR("gem object lookup failed 0x%x\n",
124 r->handle);
125 return -ENOENT;
126 }
127 p->relocs_ptr[i] = &p->relocs[i];
128 p->relocs[i].robj = gem_to_radeon_bo(p->relocs[i].gobj);
129
130 /* The userspace buffer priorities are from 0 to 15. A higher
131 * number means the buffer is more important.
132 * Also, the buffers used for write have a higher priority than
133 * the buffers used for read only, which doubles the range
134 * to 0 to 31. 32 is reserved for the kernel driver.
135 */
136 priority = (r->flags & RADEON_RELOC_PRIO_MASK) * 2
137 + !!r->write_domain;
138
139 /* the first reloc of an UVD job is the msg and that must be in
140 VRAM, also but everything into VRAM on AGP cards and older
141 IGP chips to avoid image corruptions */
142 if (p->ring == R600_RING_TYPE_UVD_INDEX &&
143 (i == 0 || drm_pci_device_is_agp(p->rdev->ddev) ||
144 p->rdev->family == CHIP_RS780 ||
145 p->rdev->family == CHIP_RS880)) {
146
147 /* TODO: is this still needed for NI+ ? */
148 p->relocs[i].prefered_domains =
149 RADEON_GEM_DOMAIN_VRAM;
150
151 p->relocs[i].allowed_domains =
152 RADEON_GEM_DOMAIN_VRAM;
153
154 /* prioritize this over any other relocation */
155 priority = RADEON_CS_MAX_PRIORITY;
156 } else {
157 uint32_t domain = r->write_domain ?
158 r->write_domain : r->read_domains;
159
160 if (domain & RADEON_GEM_DOMAIN_CPU) {
161 DRM_ERROR("RADEON_GEM_DOMAIN_CPU is not valid "
162 "for command submission\n");
163 return -EINVAL;
164 }
165
166 p->relocs[i].prefered_domains = domain;
167 if (domain == RADEON_GEM_DOMAIN_VRAM)
168 domain |= RADEON_GEM_DOMAIN_GTT;
169 p->relocs[i].allowed_domains = domain;
170 }
171
172 if (radeon_ttm_tt_has_userptr(p->relocs[i].robj->tbo.ttm)) {
173 uint32_t domain = p->relocs[i].prefered_domains;
174 if (!(domain & RADEON_GEM_DOMAIN_GTT)) {
175 DRM_ERROR("Only RADEON_GEM_DOMAIN_GTT is "
176 "allowed for userptr BOs\n");
177 return -EINVAL;
178 }
179 need_mmap_lock = true;
180 domain = RADEON_GEM_DOMAIN_GTT;
181 p->relocs[i].prefered_domains = domain;
182 p->relocs[i].allowed_domains = domain;
183 }
184
185 p->relocs[i].tv.bo = &p->relocs[i].robj->tbo;
186 p->relocs[i].tv.shared = !r->write_domain;
187 p->relocs[i].handle = r->handle;
188
189 radeon_cs_buckets_add(&buckets, &p->relocs[i].tv.head,
190 priority);
191 }
192
193 radeon_cs_buckets_get_list(&buckets, &p->validated);
194
195 if (p->cs_flags & RADEON_CS_USE_VM)
196 p->vm_bos = radeon_vm_get_bos(p->rdev, p->ib.vm,
197 &p->validated);
198 if (need_mmap_lock)
199 down_read(¤t->mm->mmap_sem);
200
201 r = radeon_bo_list_validate(p->rdev, &p->ticket, &p->validated, p->ring);
202
203 if (need_mmap_lock)
204 up_read(¤t->mm->mmap_sem);
205
206 return r;
207 }
208
radeon_cs_get_ring(struct radeon_cs_parser * p,u32 ring,s32 priority)209 static int radeon_cs_get_ring(struct radeon_cs_parser *p, u32 ring, s32 priority)
210 {
211 p->priority = priority;
212
213 switch (ring) {
214 default:
215 DRM_ERROR("unknown ring id: %d\n", ring);
216 return -EINVAL;
217 case RADEON_CS_RING_GFX:
218 p->ring = RADEON_RING_TYPE_GFX_INDEX;
219 break;
220 case RADEON_CS_RING_COMPUTE:
221 if (p->rdev->family >= CHIP_TAHITI) {
222 if (p->priority > 0)
223 p->ring = CAYMAN_RING_TYPE_CP1_INDEX;
224 else
225 p->ring = CAYMAN_RING_TYPE_CP2_INDEX;
226 } else
227 p->ring = RADEON_RING_TYPE_GFX_INDEX;
228 break;
229 case RADEON_CS_RING_DMA:
230 if (p->rdev->family >= CHIP_CAYMAN) {
231 if (p->priority > 0)
232 p->ring = R600_RING_TYPE_DMA_INDEX;
233 else
234 p->ring = CAYMAN_RING_TYPE_DMA1_INDEX;
235 } else if (p->rdev->family >= CHIP_RV770) {
236 p->ring = R600_RING_TYPE_DMA_INDEX;
237 } else {
238 return -EINVAL;
239 }
240 break;
241 case RADEON_CS_RING_UVD:
242 p->ring = R600_RING_TYPE_UVD_INDEX;
243 break;
244 case RADEON_CS_RING_VCE:
245 /* TODO: only use the low priority ring for now */
246 p->ring = TN_RING_TYPE_VCE1_INDEX;
247 break;
248 }
249 return 0;
250 }
251
radeon_cs_sync_rings(struct radeon_cs_parser * p)252 static int radeon_cs_sync_rings(struct radeon_cs_parser *p)
253 {
254 struct radeon_cs_reloc *reloc;
255 int r;
256
257 list_for_each_entry(reloc, &p->validated, tv.head) {
258 struct reservation_object *resv;
259
260 resv = reloc->robj->tbo.resv;
261 r = radeon_semaphore_sync_resv(p->rdev, p->ib.semaphore, resv,
262 reloc->tv.shared);
263 if (r)
264 return r;
265 }
266 return 0;
267 }
268
269 /* XXX: note that this is called from the legacy UMS CS ioctl as well */
radeon_cs_parser_init(struct radeon_cs_parser * p,void * data)270 int radeon_cs_parser_init(struct radeon_cs_parser *p, void *data)
271 {
272 struct drm_radeon_cs *cs = data;
273 uint64_t *chunk_array_ptr;
274 unsigned size, i;
275 u32 ring = RADEON_CS_RING_GFX;
276 s32 priority = 0;
277
278 INIT_LIST_HEAD(&p->validated);
279
280 if (!cs->num_chunks) {
281 return 0;
282 }
283
284 /* get chunks */
285 p->idx = 0;
286 p->ib.sa_bo = NULL;
287 p->ib.semaphore = NULL;
288 p->const_ib.sa_bo = NULL;
289 p->const_ib.semaphore = NULL;
290 p->chunk_ib_idx = -1;
291 p->chunk_relocs_idx = -1;
292 p->chunk_flags_idx = -1;
293 p->chunk_const_ib_idx = -1;
294 p->chunks_array = kcalloc(cs->num_chunks, sizeof(uint64_t), GFP_KERNEL);
295 if (p->chunks_array == NULL) {
296 return -ENOMEM;
297 }
298 chunk_array_ptr = (uint64_t *)(unsigned long)(cs->chunks);
299 if (copy_from_user(p->chunks_array, chunk_array_ptr,
300 sizeof(uint64_t)*cs->num_chunks)) {
301 return -EFAULT;
302 }
303 p->cs_flags = 0;
304 p->nchunks = cs->num_chunks;
305 p->chunks = kcalloc(p->nchunks, sizeof(struct radeon_cs_chunk), GFP_KERNEL);
306 if (p->chunks == NULL) {
307 return -ENOMEM;
308 }
309 for (i = 0; i < p->nchunks; i++) {
310 struct drm_radeon_cs_chunk __user **chunk_ptr = NULL;
311 struct drm_radeon_cs_chunk user_chunk;
312 uint32_t __user *cdata;
313
314 chunk_ptr = (void __user*)(unsigned long)p->chunks_array[i];
315 if (copy_from_user(&user_chunk, chunk_ptr,
316 sizeof(struct drm_radeon_cs_chunk))) {
317 return -EFAULT;
318 }
319 p->chunks[i].length_dw = user_chunk.length_dw;
320 p->chunks[i].chunk_id = user_chunk.chunk_id;
321 if (p->chunks[i].chunk_id == RADEON_CHUNK_ID_RELOCS) {
322 p->chunk_relocs_idx = i;
323 }
324 if (p->chunks[i].chunk_id == RADEON_CHUNK_ID_IB) {
325 p->chunk_ib_idx = i;
326 /* zero length IB isn't useful */
327 if (p->chunks[i].length_dw == 0)
328 return -EINVAL;
329 }
330 if (p->chunks[i].chunk_id == RADEON_CHUNK_ID_CONST_IB) {
331 p->chunk_const_ib_idx = i;
332 /* zero length CONST IB isn't useful */
333 if (p->chunks[i].length_dw == 0)
334 return -EINVAL;
335 }
336 if (p->chunks[i].chunk_id == RADEON_CHUNK_ID_FLAGS) {
337 p->chunk_flags_idx = i;
338 /* zero length flags aren't useful */
339 if (p->chunks[i].length_dw == 0)
340 return -EINVAL;
341 }
342
343 size = p->chunks[i].length_dw;
344 cdata = (void __user *)(unsigned long)user_chunk.chunk_data;
345 p->chunks[i].user_ptr = cdata;
346 if (p->chunks[i].chunk_id == RADEON_CHUNK_ID_CONST_IB)
347 continue;
348
349 if (p->chunks[i].chunk_id == RADEON_CHUNK_ID_IB) {
350 if (!p->rdev || !(p->rdev->flags & RADEON_IS_AGP))
351 continue;
352 }
353
354 p->chunks[i].kdata = drm_malloc_ab(size, sizeof(uint32_t));
355 size *= sizeof(uint32_t);
356 if (p->chunks[i].kdata == NULL) {
357 return -ENOMEM;
358 }
359 if (copy_from_user(p->chunks[i].kdata, cdata, size)) {
360 return -EFAULT;
361 }
362 if (p->chunks[i].chunk_id == RADEON_CHUNK_ID_FLAGS) {
363 p->cs_flags = p->chunks[i].kdata[0];
364 if (p->chunks[i].length_dw > 1)
365 ring = p->chunks[i].kdata[1];
366 if (p->chunks[i].length_dw > 2)
367 priority = (s32)p->chunks[i].kdata[2];
368 }
369 }
370
371 /* these are KMS only */
372 if (p->rdev) {
373 if ((p->cs_flags & RADEON_CS_USE_VM) &&
374 !p->rdev->vm_manager.enabled) {
375 DRM_ERROR("VM not active on asic!\n");
376 return -EINVAL;
377 }
378
379 if (radeon_cs_get_ring(p, ring, priority))
380 return -EINVAL;
381
382 /* we only support VM on some SI+ rings */
383 if ((p->cs_flags & RADEON_CS_USE_VM) == 0) {
384 if (p->rdev->asic->ring[p->ring]->cs_parse == NULL) {
385 DRM_ERROR("Ring %d requires VM!\n", p->ring);
386 return -EINVAL;
387 }
388 } else {
389 if (p->rdev->asic->ring[p->ring]->ib_parse == NULL) {
390 DRM_ERROR("VM not supported on ring %d!\n",
391 p->ring);
392 return -EINVAL;
393 }
394 }
395 }
396
397 return 0;
398 }
399
cmp_size_smaller_first(void * priv,struct list_head * a,struct list_head * b)400 static int cmp_size_smaller_first(void *priv, struct list_head *a,
401 struct list_head *b)
402 {
403 struct radeon_cs_reloc *la = list_entry(a, struct radeon_cs_reloc, tv.head);
404 struct radeon_cs_reloc *lb = list_entry(b, struct radeon_cs_reloc, tv.head);
405
406 /* Sort A before B if A is smaller. */
407 return (int)la->robj->tbo.num_pages - (int)lb->robj->tbo.num_pages;
408 }
409
410 /**
411 * cs_parser_fini() - clean parser states
412 * @parser: parser structure holding parsing context.
413 * @error: error number
414 *
415 * If error is set than unvalidate buffer, otherwise just free memory
416 * used by parsing context.
417 **/
radeon_cs_parser_fini(struct radeon_cs_parser * parser,int error,bool backoff)418 static void radeon_cs_parser_fini(struct radeon_cs_parser *parser, int error, bool backoff)
419 {
420 unsigned i;
421
422 if (!error) {
423 /* Sort the buffer list from the smallest to largest buffer,
424 * which affects the order of buffers in the LRU list.
425 * This assures that the smallest buffers are added first
426 * to the LRU list, so they are likely to be later evicted
427 * first, instead of large buffers whose eviction is more
428 * expensive.
429 *
430 * This slightly lowers the number of bytes moved by TTM
431 * per frame under memory pressure.
432 */
433 list_sort(NULL, &parser->validated, cmp_size_smaller_first);
434
435 ttm_eu_fence_buffer_objects(&parser->ticket,
436 &parser->validated,
437 &parser->ib.fence->base);
438 } else if (backoff) {
439 ttm_eu_backoff_reservation(&parser->ticket,
440 &parser->validated);
441 }
442
443 if (parser->relocs != NULL) {
444 for (i = 0; i < parser->nrelocs; i++) {
445 if (parser->relocs[i].gobj)
446 drm_gem_object_unreference_unlocked(parser->relocs[i].gobj);
447 }
448 }
449 kfree(parser->track);
450 kfree(parser->relocs);
451 kfree(parser->relocs_ptr);
452 drm_free_large(parser->vm_bos);
453 for (i = 0; i < parser->nchunks; i++)
454 drm_free_large(parser->chunks[i].kdata);
455 kfree(parser->chunks);
456 kfree(parser->chunks_array);
457 radeon_ib_free(parser->rdev, &parser->ib);
458 radeon_ib_free(parser->rdev, &parser->const_ib);
459 }
460
radeon_cs_ib_chunk(struct radeon_device * rdev,struct radeon_cs_parser * parser)461 static int radeon_cs_ib_chunk(struct radeon_device *rdev,
462 struct radeon_cs_parser *parser)
463 {
464 int r;
465
466 if (parser->chunk_ib_idx == -1)
467 return 0;
468
469 if (parser->cs_flags & RADEON_CS_USE_VM)
470 return 0;
471
472 r = radeon_cs_parse(rdev, parser->ring, parser);
473 if (r || parser->parser_error) {
474 DRM_ERROR("Invalid command stream !\n");
475 return r;
476 }
477
478 r = radeon_cs_sync_rings(parser);
479 if (r) {
480 if (r != -ERESTARTSYS)
481 DRM_ERROR("Failed to sync rings: %i\n", r);
482 return r;
483 }
484
485 if (parser->ring == R600_RING_TYPE_UVD_INDEX)
486 radeon_uvd_note_usage(rdev);
487 else if ((parser->ring == TN_RING_TYPE_VCE1_INDEX) ||
488 (parser->ring == TN_RING_TYPE_VCE2_INDEX))
489 radeon_vce_note_usage(rdev);
490
491 r = radeon_ib_schedule(rdev, &parser->ib, NULL, true);
492 if (r) {
493 DRM_ERROR("Failed to schedule IB !\n");
494 }
495 return r;
496 }
497
radeon_bo_vm_update_pte(struct radeon_cs_parser * p,struct radeon_vm * vm)498 static int radeon_bo_vm_update_pte(struct radeon_cs_parser *p,
499 struct radeon_vm *vm)
500 {
501 struct radeon_device *rdev = p->rdev;
502 struct radeon_bo_va *bo_va;
503 int i, r;
504
505 r = radeon_vm_update_page_directory(rdev, vm);
506 if (r)
507 return r;
508
509 r = radeon_vm_clear_freed(rdev, vm);
510 if (r)
511 return r;
512
513 if (vm->ib_bo_va == NULL) {
514 DRM_ERROR("Tmp BO not in VM!\n");
515 return -EINVAL;
516 }
517
518 r = radeon_vm_bo_update(rdev, vm->ib_bo_va,
519 &rdev->ring_tmp_bo.bo->tbo.mem);
520 if (r)
521 return r;
522
523 for (i = 0; i < p->nrelocs; i++) {
524 struct radeon_bo *bo;
525
526 /* ignore duplicates */
527 if (p->relocs_ptr[i] != &p->relocs[i])
528 continue;
529
530 bo = p->relocs[i].robj;
531 bo_va = radeon_vm_bo_find(vm, bo);
532 if (bo_va == NULL) {
533 dev_err(rdev->dev, "bo %p not in vm %p\n", bo, vm);
534 return -EINVAL;
535 }
536
537 r = radeon_vm_bo_update(rdev, bo_va, &bo->tbo.mem);
538 if (r)
539 return r;
540 }
541
542 return radeon_vm_clear_invalids(rdev, vm);
543 }
544
radeon_cs_ib_vm_chunk(struct radeon_device * rdev,struct radeon_cs_parser * parser)545 static int radeon_cs_ib_vm_chunk(struct radeon_device *rdev,
546 struct radeon_cs_parser *parser)
547 {
548 struct radeon_fpriv *fpriv = parser->filp->driver_priv;
549 struct radeon_vm *vm = &fpriv->vm;
550 int r;
551
552 if (parser->chunk_ib_idx == -1)
553 return 0;
554 if ((parser->cs_flags & RADEON_CS_USE_VM) == 0)
555 return 0;
556
557 if (parser->const_ib.length_dw) {
558 r = radeon_ring_ib_parse(rdev, parser->ring, &parser->const_ib);
559 if (r) {
560 return r;
561 }
562 }
563
564 r = radeon_ring_ib_parse(rdev, parser->ring, &parser->ib);
565 if (r) {
566 return r;
567 }
568
569 if (parser->ring == R600_RING_TYPE_UVD_INDEX)
570 radeon_uvd_note_usage(rdev);
571
572 mutex_lock(&vm->mutex);
573 r = radeon_bo_vm_update_pte(parser, vm);
574 if (r) {
575 goto out;
576 }
577
578 r = radeon_cs_sync_rings(parser);
579 if (r) {
580 if (r != -ERESTARTSYS)
581 DRM_ERROR("Failed to sync rings: %i\n", r);
582 goto out;
583 }
584 radeon_semaphore_sync_fence(parser->ib.semaphore, vm->fence);
585
586 if ((rdev->family >= CHIP_TAHITI) &&
587 (parser->chunk_const_ib_idx != -1)) {
588 r = radeon_ib_schedule(rdev, &parser->ib, &parser->const_ib, true);
589 } else {
590 r = radeon_ib_schedule(rdev, &parser->ib, NULL, true);
591 }
592
593 out:
594 mutex_unlock(&vm->mutex);
595 return r;
596 }
597
radeon_cs_handle_lockup(struct radeon_device * rdev,int r)598 static int radeon_cs_handle_lockup(struct radeon_device *rdev, int r)
599 {
600 if (r == -EDEADLK) {
601 r = radeon_gpu_reset(rdev);
602 if (!r)
603 r = -EAGAIN;
604 }
605 return r;
606 }
607
radeon_cs_ib_fill(struct radeon_device * rdev,struct radeon_cs_parser * parser)608 static int radeon_cs_ib_fill(struct radeon_device *rdev, struct radeon_cs_parser *parser)
609 {
610 struct radeon_cs_chunk *ib_chunk;
611 struct radeon_vm *vm = NULL;
612 int r;
613
614 if (parser->chunk_ib_idx == -1)
615 return 0;
616
617 if (parser->cs_flags & RADEON_CS_USE_VM) {
618 struct radeon_fpriv *fpriv = parser->filp->driver_priv;
619 vm = &fpriv->vm;
620
621 if ((rdev->family >= CHIP_TAHITI) &&
622 (parser->chunk_const_ib_idx != -1)) {
623 ib_chunk = &parser->chunks[parser->chunk_const_ib_idx];
624 if (ib_chunk->length_dw > RADEON_IB_VM_MAX_SIZE) {
625 DRM_ERROR("cs IB CONST too big: %d\n", ib_chunk->length_dw);
626 return -EINVAL;
627 }
628 r = radeon_ib_get(rdev, parser->ring, &parser->const_ib,
629 vm, ib_chunk->length_dw * 4);
630 if (r) {
631 DRM_ERROR("Failed to get const ib !\n");
632 return r;
633 }
634 parser->const_ib.is_const_ib = true;
635 parser->const_ib.length_dw = ib_chunk->length_dw;
636 if (copy_from_user(parser->const_ib.ptr,
637 ib_chunk->user_ptr,
638 ib_chunk->length_dw * 4))
639 return -EFAULT;
640 }
641
642 ib_chunk = &parser->chunks[parser->chunk_ib_idx];
643 if (ib_chunk->length_dw > RADEON_IB_VM_MAX_SIZE) {
644 DRM_ERROR("cs IB too big: %d\n", ib_chunk->length_dw);
645 return -EINVAL;
646 }
647 }
648 ib_chunk = &parser->chunks[parser->chunk_ib_idx];
649
650 r = radeon_ib_get(rdev, parser->ring, &parser->ib,
651 vm, ib_chunk->length_dw * 4);
652 if (r) {
653 DRM_ERROR("Failed to get ib !\n");
654 return r;
655 }
656 parser->ib.length_dw = ib_chunk->length_dw;
657 if (ib_chunk->kdata)
658 memcpy(parser->ib.ptr, ib_chunk->kdata, ib_chunk->length_dw * 4);
659 else if (copy_from_user(parser->ib.ptr, ib_chunk->user_ptr, ib_chunk->length_dw * 4))
660 return -EFAULT;
661 return 0;
662 }
663
radeon_cs_ioctl(struct drm_device * dev,void * data,struct drm_file * filp)664 int radeon_cs_ioctl(struct drm_device *dev, void *data, struct drm_file *filp)
665 {
666 struct radeon_device *rdev = dev->dev_private;
667 struct radeon_cs_parser parser;
668 int r;
669
670 down_read(&rdev->exclusive_lock);
671 if (!rdev->accel_working) {
672 up_read(&rdev->exclusive_lock);
673 return -EBUSY;
674 }
675 if (rdev->in_reset) {
676 up_read(&rdev->exclusive_lock);
677 r = radeon_gpu_reset(rdev);
678 if (!r)
679 r = -EAGAIN;
680 return r;
681 }
682 /* initialize parser */
683 memset(&parser, 0, sizeof(struct radeon_cs_parser));
684 parser.filp = filp;
685 parser.rdev = rdev;
686 parser.dev = rdev->dev;
687 parser.family = rdev->family;
688 r = radeon_cs_parser_init(&parser, data);
689 if (r) {
690 DRM_ERROR("Failed to initialize parser !\n");
691 radeon_cs_parser_fini(&parser, r, false);
692 up_read(&rdev->exclusive_lock);
693 r = radeon_cs_handle_lockup(rdev, r);
694 return r;
695 }
696
697 r = radeon_cs_ib_fill(rdev, &parser);
698 if (!r) {
699 r = radeon_cs_parser_relocs(&parser);
700 if (r && r != -ERESTARTSYS)
701 DRM_ERROR("Failed to parse relocation %d!\n", r);
702 }
703
704 if (r) {
705 radeon_cs_parser_fini(&parser, r, false);
706 up_read(&rdev->exclusive_lock);
707 r = radeon_cs_handle_lockup(rdev, r);
708 return r;
709 }
710
711 trace_radeon_cs(&parser);
712
713 r = radeon_cs_ib_chunk(rdev, &parser);
714 if (r) {
715 goto out;
716 }
717 r = radeon_cs_ib_vm_chunk(rdev, &parser);
718 if (r) {
719 goto out;
720 }
721 out:
722 radeon_cs_parser_fini(&parser, r, true);
723 up_read(&rdev->exclusive_lock);
724 r = radeon_cs_handle_lockup(rdev, r);
725 return r;
726 }
727
728 /**
729 * radeon_cs_packet_parse() - parse cp packet and point ib index to next packet
730 * @parser: parser structure holding parsing context.
731 * @pkt: where to store packet information
732 *
733 * Assume that chunk_ib_index is properly set. Will return -EINVAL
734 * if packet is bigger than remaining ib size. or if packets is unknown.
735 **/
radeon_cs_packet_parse(struct radeon_cs_parser * p,struct radeon_cs_packet * pkt,unsigned idx)736 int radeon_cs_packet_parse(struct radeon_cs_parser *p,
737 struct radeon_cs_packet *pkt,
738 unsigned idx)
739 {
740 struct radeon_cs_chunk *ib_chunk = &p->chunks[p->chunk_ib_idx];
741 struct radeon_device *rdev = p->rdev;
742 uint32_t header;
743
744 if (idx >= ib_chunk->length_dw) {
745 DRM_ERROR("Can not parse packet at %d after CS end %d !\n",
746 idx, ib_chunk->length_dw);
747 return -EINVAL;
748 }
749 header = radeon_get_ib_value(p, idx);
750 pkt->idx = idx;
751 pkt->type = RADEON_CP_PACKET_GET_TYPE(header);
752 pkt->count = RADEON_CP_PACKET_GET_COUNT(header);
753 pkt->one_reg_wr = 0;
754 switch (pkt->type) {
755 case RADEON_PACKET_TYPE0:
756 if (rdev->family < CHIP_R600) {
757 pkt->reg = R100_CP_PACKET0_GET_REG(header);
758 pkt->one_reg_wr =
759 RADEON_CP_PACKET0_GET_ONE_REG_WR(header);
760 } else
761 pkt->reg = R600_CP_PACKET0_GET_REG(header);
762 break;
763 case RADEON_PACKET_TYPE3:
764 pkt->opcode = RADEON_CP_PACKET3_GET_OPCODE(header);
765 break;
766 case RADEON_PACKET_TYPE2:
767 pkt->count = -1;
768 break;
769 default:
770 DRM_ERROR("Unknown packet type %d at %d !\n", pkt->type, idx);
771 return -EINVAL;
772 }
773 if ((pkt->count + 1 + pkt->idx) >= ib_chunk->length_dw) {
774 DRM_ERROR("Packet (%d:%d:%d) end after CS buffer (%d) !\n",
775 pkt->idx, pkt->type, pkt->count, ib_chunk->length_dw);
776 return -EINVAL;
777 }
778 return 0;
779 }
780
781 /**
782 * radeon_cs_packet_next_is_pkt3_nop() - test if the next packet is P3 NOP
783 * @p: structure holding the parser context.
784 *
785 * Check if the next packet is NOP relocation packet3.
786 **/
radeon_cs_packet_next_is_pkt3_nop(struct radeon_cs_parser * p)787 bool radeon_cs_packet_next_is_pkt3_nop(struct radeon_cs_parser *p)
788 {
789 struct radeon_cs_packet p3reloc;
790 int r;
791
792 r = radeon_cs_packet_parse(p, &p3reloc, p->idx);
793 if (r)
794 return false;
795 if (p3reloc.type != RADEON_PACKET_TYPE3)
796 return false;
797 if (p3reloc.opcode != RADEON_PACKET3_NOP)
798 return false;
799 return true;
800 }
801
802 /**
803 * radeon_cs_dump_packet() - dump raw packet context
804 * @p: structure holding the parser context.
805 * @pkt: structure holding the packet.
806 *
807 * Used mostly for debugging and error reporting.
808 **/
radeon_cs_dump_packet(struct radeon_cs_parser * p,struct radeon_cs_packet * pkt)809 void radeon_cs_dump_packet(struct radeon_cs_parser *p,
810 struct radeon_cs_packet *pkt)
811 {
812 volatile uint32_t *ib;
813 unsigned i;
814 unsigned idx;
815
816 ib = p->ib.ptr;
817 idx = pkt->idx;
818 for (i = 0; i <= (pkt->count + 1); i++, idx++)
819 DRM_INFO("ib[%d]=0x%08X\n", idx, ib[idx]);
820 }
821
822 /**
823 * radeon_cs_packet_next_reloc() - parse next (should be reloc) packet
824 * @parser: parser structure holding parsing context.
825 * @data: pointer to relocation data
826 * @offset_start: starting offset
827 * @offset_mask: offset mask (to align start offset on)
828 * @reloc: reloc informations
829 *
830 * Check if next packet is relocation packet3, do bo validation and compute
831 * GPU offset using the provided start.
832 **/
radeon_cs_packet_next_reloc(struct radeon_cs_parser * p,struct radeon_cs_reloc ** cs_reloc,int nomm)833 int radeon_cs_packet_next_reloc(struct radeon_cs_parser *p,
834 struct radeon_cs_reloc **cs_reloc,
835 int nomm)
836 {
837 struct radeon_cs_chunk *relocs_chunk;
838 struct radeon_cs_packet p3reloc;
839 unsigned idx;
840 int r;
841
842 if (p->chunk_relocs_idx == -1) {
843 DRM_ERROR("No relocation chunk !\n");
844 return -EINVAL;
845 }
846 *cs_reloc = NULL;
847 relocs_chunk = &p->chunks[p->chunk_relocs_idx];
848 r = radeon_cs_packet_parse(p, &p3reloc, p->idx);
849 if (r)
850 return r;
851 p->idx += p3reloc.count + 2;
852 if (p3reloc.type != RADEON_PACKET_TYPE3 ||
853 p3reloc.opcode != RADEON_PACKET3_NOP) {
854 DRM_ERROR("No packet3 for relocation for packet at %d.\n",
855 p3reloc.idx);
856 radeon_cs_dump_packet(p, &p3reloc);
857 return -EINVAL;
858 }
859 idx = radeon_get_ib_value(p, p3reloc.idx + 1);
860 if (idx >= relocs_chunk->length_dw) {
861 DRM_ERROR("Relocs at %d after relocations chunk end %d !\n",
862 idx, relocs_chunk->length_dw);
863 radeon_cs_dump_packet(p, &p3reloc);
864 return -EINVAL;
865 }
866 /* FIXME: we assume reloc size is 4 dwords */
867 if (nomm) {
868 *cs_reloc = p->relocs;
869 (*cs_reloc)->gpu_offset =
870 (u64)relocs_chunk->kdata[idx + 3] << 32;
871 (*cs_reloc)->gpu_offset |= relocs_chunk->kdata[idx + 0];
872 } else
873 *cs_reloc = p->relocs_ptr[(idx / 4)];
874 return 0;
875 }
876