1 /*
2 * Copyright (C) 2007 Ben Skeggs.
3 * All Rights Reserved.
4 *
5 * Permission is hereby granted, free of charge, to any person obtaining
6 * a copy of this software and associated documentation files (the
7 * "Software"), to deal in the Software without restriction, including
8 * without limitation the rights to use, copy, modify, merge, publish,
9 * distribute, sublicense, and/or sell copies of the Software, and to
10 * permit persons to whom the Software is furnished to do so, subject to
11 * the following conditions:
12 *
13 * The above copyright notice and this permission notice (including the
14 * next paragraph) shall be included in all copies or substantial
15 * portions of the Software.
16 *
17 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
18 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
19 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
20 * IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
21 * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
22 * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
23 * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
24 *
25 */
26
27 #include "drmP.h"
28 #include "drm.h"
29
30 #include <linux/ktime.h>
31 #include <linux/hrtimer.h>
32
33 #include "nouveau_drv.h"
34 #include "nouveau_ramht.h"
35 #include "nouveau_dma.h"
36
37 #define USE_REFCNT(dev) (nouveau_private(dev)->chipset >= 0x10)
38 #define USE_SEMA(dev) (nouveau_private(dev)->chipset >= 0x17)
39
40 struct nouveau_fence {
41 struct nouveau_channel *channel;
42 struct kref refcount;
43 struct list_head entry;
44
45 uint32_t sequence;
46 bool signalled;
47
48 void (*work)(void *priv, bool signalled);
49 void *priv;
50 };
51
52 struct nouveau_semaphore {
53 struct kref ref;
54 struct drm_device *dev;
55 struct drm_mm_node *mem;
56 };
57
58 static inline struct nouveau_fence *
nouveau_fence(void * sync_obj)59 nouveau_fence(void *sync_obj)
60 {
61 return (struct nouveau_fence *)sync_obj;
62 }
63
64 static void
nouveau_fence_del(struct kref * ref)65 nouveau_fence_del(struct kref *ref)
66 {
67 struct nouveau_fence *fence =
68 container_of(ref, struct nouveau_fence, refcount);
69
70 nouveau_channel_ref(NULL, &fence->channel);
71 kfree(fence);
72 }
73
74 void
nouveau_fence_update(struct nouveau_channel * chan)75 nouveau_fence_update(struct nouveau_channel *chan)
76 {
77 struct drm_device *dev = chan->dev;
78 struct nouveau_fence *tmp, *fence;
79 uint32_t sequence;
80
81 spin_lock(&chan->fence.lock);
82
83 /* Fetch the last sequence if the channel is still up and running */
84 if (likely(!list_empty(&chan->fence.pending))) {
85 if (USE_REFCNT(dev))
86 sequence = nvchan_rd32(chan, 0x48);
87 else
88 sequence = atomic_read(&chan->fence.last_sequence_irq);
89
90 if (chan->fence.sequence_ack == sequence)
91 goto out;
92 chan->fence.sequence_ack = sequence;
93 }
94
95 list_for_each_entry_safe(fence, tmp, &chan->fence.pending, entry) {
96 if (fence->sequence > chan->fence.sequence_ack)
97 break;
98
99 fence->signalled = true;
100 list_del(&fence->entry);
101 if (fence->work)
102 fence->work(fence->priv, true);
103
104 kref_put(&fence->refcount, nouveau_fence_del);
105 }
106
107 out:
108 spin_unlock(&chan->fence.lock);
109 }
110
111 int
nouveau_fence_new(struct nouveau_channel * chan,struct nouveau_fence ** pfence,bool emit)112 nouveau_fence_new(struct nouveau_channel *chan, struct nouveau_fence **pfence,
113 bool emit)
114 {
115 struct nouveau_fence *fence;
116 int ret = 0;
117
118 fence = kzalloc(sizeof(*fence), GFP_KERNEL);
119 if (!fence)
120 return -ENOMEM;
121 kref_init(&fence->refcount);
122 nouveau_channel_ref(chan, &fence->channel);
123
124 if (emit)
125 ret = nouveau_fence_emit(fence);
126
127 if (ret)
128 nouveau_fence_unref(&fence);
129 *pfence = fence;
130 return ret;
131 }
132
133 struct nouveau_channel *
nouveau_fence_channel(struct nouveau_fence * fence)134 nouveau_fence_channel(struct nouveau_fence *fence)
135 {
136 return fence ? nouveau_channel_get_unlocked(fence->channel) : NULL;
137 }
138
139 int
nouveau_fence_emit(struct nouveau_fence * fence)140 nouveau_fence_emit(struct nouveau_fence *fence)
141 {
142 struct nouveau_channel *chan = fence->channel;
143 struct drm_device *dev = chan->dev;
144 struct drm_nouveau_private *dev_priv = dev->dev_private;
145 int ret;
146
147 ret = RING_SPACE(chan, 2);
148 if (ret)
149 return ret;
150
151 if (unlikely(chan->fence.sequence == chan->fence.sequence_ack - 1)) {
152 nouveau_fence_update(chan);
153
154 BUG_ON(chan->fence.sequence ==
155 chan->fence.sequence_ack - 1);
156 }
157
158 fence->sequence = ++chan->fence.sequence;
159
160 kref_get(&fence->refcount);
161 spin_lock(&chan->fence.lock);
162 list_add_tail(&fence->entry, &chan->fence.pending);
163 spin_unlock(&chan->fence.lock);
164
165 if (USE_REFCNT(dev)) {
166 if (dev_priv->card_type < NV_C0)
167 BEGIN_RING(chan, 0, NV10_SUBCHAN_REF_CNT, 1);
168 else
169 BEGIN_NVC0(chan, 2, 0, NV10_SUBCHAN_REF_CNT, 1);
170 } else {
171 BEGIN_RING(chan, NvSubSw, 0x0150, 1);
172 }
173 OUT_RING (chan, fence->sequence);
174 FIRE_RING(chan);
175
176 return 0;
177 }
178
179 void
nouveau_fence_work(struct nouveau_fence * fence,void (* work)(void * priv,bool signalled),void * priv)180 nouveau_fence_work(struct nouveau_fence *fence,
181 void (*work)(void *priv, bool signalled),
182 void *priv)
183 {
184 BUG_ON(fence->work);
185
186 spin_lock(&fence->channel->fence.lock);
187
188 if (fence->signalled) {
189 work(priv, true);
190 } else {
191 fence->work = work;
192 fence->priv = priv;
193 }
194
195 spin_unlock(&fence->channel->fence.lock);
196 }
197
198 void
__nouveau_fence_unref(void ** sync_obj)199 __nouveau_fence_unref(void **sync_obj)
200 {
201 struct nouveau_fence *fence = nouveau_fence(*sync_obj);
202
203 if (fence)
204 kref_put(&fence->refcount, nouveau_fence_del);
205 *sync_obj = NULL;
206 }
207
208 void *
__nouveau_fence_ref(void * sync_obj)209 __nouveau_fence_ref(void *sync_obj)
210 {
211 struct nouveau_fence *fence = nouveau_fence(sync_obj);
212
213 kref_get(&fence->refcount);
214 return sync_obj;
215 }
216
217 bool
__nouveau_fence_signalled(void * sync_obj,void * sync_arg)218 __nouveau_fence_signalled(void *sync_obj, void *sync_arg)
219 {
220 struct nouveau_fence *fence = nouveau_fence(sync_obj);
221 struct nouveau_channel *chan = fence->channel;
222
223 if (fence->signalled)
224 return true;
225
226 nouveau_fence_update(chan);
227 return fence->signalled;
228 }
229
230 int
__nouveau_fence_wait(void * sync_obj,void * sync_arg,bool lazy,bool intr)231 __nouveau_fence_wait(void *sync_obj, void *sync_arg, bool lazy, bool intr)
232 {
233 unsigned long timeout = jiffies + (3 * DRM_HZ);
234 unsigned long sleep_time = NSEC_PER_MSEC / 1000;
235 ktime_t t;
236 int ret = 0;
237
238 while (1) {
239 if (__nouveau_fence_signalled(sync_obj, sync_arg))
240 break;
241
242 if (time_after_eq(jiffies, timeout)) {
243 ret = -EBUSY;
244 break;
245 }
246
247 __set_current_state(intr ? TASK_INTERRUPTIBLE
248 : TASK_UNINTERRUPTIBLE);
249 if (lazy) {
250 t = ktime_set(0, sleep_time);
251 schedule_hrtimeout(&t, HRTIMER_MODE_REL);
252 sleep_time *= 2;
253 if (sleep_time > NSEC_PER_MSEC)
254 sleep_time = NSEC_PER_MSEC;
255 }
256
257 if (intr && signal_pending(current)) {
258 ret = -ERESTARTSYS;
259 break;
260 }
261 }
262
263 __set_current_state(TASK_RUNNING);
264
265 return ret;
266 }
267
268 static struct nouveau_semaphore *
semaphore_alloc(struct drm_device * dev)269 semaphore_alloc(struct drm_device *dev)
270 {
271 struct drm_nouveau_private *dev_priv = dev->dev_private;
272 struct nouveau_semaphore *sema;
273 int size = (dev_priv->chipset < 0x84) ? 4 : 16;
274 int ret, i;
275
276 if (!USE_SEMA(dev))
277 return NULL;
278
279 sema = kmalloc(sizeof(*sema), GFP_KERNEL);
280 if (!sema)
281 goto fail;
282
283 ret = drm_mm_pre_get(&dev_priv->fence.heap);
284 if (ret)
285 goto fail;
286
287 spin_lock(&dev_priv->fence.lock);
288 sema->mem = drm_mm_search_free(&dev_priv->fence.heap, size, 0, 0);
289 if (sema->mem)
290 sema->mem = drm_mm_get_block_atomic(sema->mem, size, 0);
291 spin_unlock(&dev_priv->fence.lock);
292
293 if (!sema->mem)
294 goto fail;
295
296 kref_init(&sema->ref);
297 sema->dev = dev;
298 for (i = sema->mem->start; i < sema->mem->start + size; i += 4)
299 nouveau_bo_wr32(dev_priv->fence.bo, i / 4, 0);
300
301 return sema;
302 fail:
303 kfree(sema);
304 return NULL;
305 }
306
307 static void
semaphore_free(struct kref * ref)308 semaphore_free(struct kref *ref)
309 {
310 struct nouveau_semaphore *sema =
311 container_of(ref, struct nouveau_semaphore, ref);
312 struct drm_nouveau_private *dev_priv = sema->dev->dev_private;
313
314 spin_lock(&dev_priv->fence.lock);
315 drm_mm_put_block(sema->mem);
316 spin_unlock(&dev_priv->fence.lock);
317
318 kfree(sema);
319 }
320
321 static void
semaphore_work(void * priv,bool signalled)322 semaphore_work(void *priv, bool signalled)
323 {
324 struct nouveau_semaphore *sema = priv;
325 struct drm_nouveau_private *dev_priv = sema->dev->dev_private;
326
327 if (unlikely(!signalled))
328 nouveau_bo_wr32(dev_priv->fence.bo, sema->mem->start / 4, 1);
329
330 kref_put(&sema->ref, semaphore_free);
331 }
332
333 static int
semaphore_acquire(struct nouveau_channel * chan,struct nouveau_semaphore * sema)334 semaphore_acquire(struct nouveau_channel *chan, struct nouveau_semaphore *sema)
335 {
336 struct drm_nouveau_private *dev_priv = chan->dev->dev_private;
337 struct nouveau_fence *fence = NULL;
338 u64 offset = chan->fence.vma.offset + sema->mem->start;
339 int ret;
340
341 if (dev_priv->chipset < 0x84) {
342 ret = RING_SPACE(chan, 4);
343 if (ret)
344 return ret;
345
346 BEGIN_RING(chan, 0, NV11_SUBCHAN_DMA_SEMAPHORE, 3);
347 OUT_RING (chan, NvSema);
348 OUT_RING (chan, offset);
349 OUT_RING (chan, 1);
350 } else
351 if (dev_priv->chipset < 0xc0) {
352 ret = RING_SPACE(chan, 7);
353 if (ret)
354 return ret;
355
356 BEGIN_RING(chan, 0, NV11_SUBCHAN_DMA_SEMAPHORE, 1);
357 OUT_RING (chan, chan->vram_handle);
358 BEGIN_RING(chan, 0, NV84_SUBCHAN_SEMAPHORE_ADDRESS_HIGH, 4);
359 OUT_RING (chan, upper_32_bits(offset));
360 OUT_RING (chan, lower_32_bits(offset));
361 OUT_RING (chan, 1);
362 OUT_RING (chan, 1); /* ACQUIRE_EQ */
363 } else {
364 ret = RING_SPACE(chan, 5);
365 if (ret)
366 return ret;
367
368 BEGIN_NVC0(chan, 2, 0, NV84_SUBCHAN_SEMAPHORE_ADDRESS_HIGH, 4);
369 OUT_RING (chan, upper_32_bits(offset));
370 OUT_RING (chan, lower_32_bits(offset));
371 OUT_RING (chan, 1);
372 OUT_RING (chan, 0x1001); /* ACQUIRE_EQ */
373 }
374
375 /* Delay semaphore destruction until its work is done */
376 ret = nouveau_fence_new(chan, &fence, true);
377 if (ret)
378 return ret;
379
380 kref_get(&sema->ref);
381 nouveau_fence_work(fence, semaphore_work, sema);
382 nouveau_fence_unref(&fence);
383 return 0;
384 }
385
386 static int
semaphore_release(struct nouveau_channel * chan,struct nouveau_semaphore * sema)387 semaphore_release(struct nouveau_channel *chan, struct nouveau_semaphore *sema)
388 {
389 struct drm_nouveau_private *dev_priv = chan->dev->dev_private;
390 struct nouveau_fence *fence = NULL;
391 u64 offset = chan->fence.vma.offset + sema->mem->start;
392 int ret;
393
394 if (dev_priv->chipset < 0x84) {
395 ret = RING_SPACE(chan, 5);
396 if (ret)
397 return ret;
398
399 BEGIN_RING(chan, 0, NV11_SUBCHAN_DMA_SEMAPHORE, 2);
400 OUT_RING (chan, NvSema);
401 OUT_RING (chan, offset);
402 BEGIN_RING(chan, 0, NV11_SUBCHAN_SEMAPHORE_RELEASE, 1);
403 OUT_RING (chan, 1);
404 } else
405 if (dev_priv->chipset < 0xc0) {
406 ret = RING_SPACE(chan, 7);
407 if (ret)
408 return ret;
409
410 BEGIN_RING(chan, 0, NV11_SUBCHAN_DMA_SEMAPHORE, 1);
411 OUT_RING (chan, chan->vram_handle);
412 BEGIN_RING(chan, 0, NV84_SUBCHAN_SEMAPHORE_ADDRESS_HIGH, 4);
413 OUT_RING (chan, upper_32_bits(offset));
414 OUT_RING (chan, lower_32_bits(offset));
415 OUT_RING (chan, 1);
416 OUT_RING (chan, 2); /* RELEASE */
417 } else {
418 ret = RING_SPACE(chan, 5);
419 if (ret)
420 return ret;
421
422 BEGIN_NVC0(chan, 2, 0, NV84_SUBCHAN_SEMAPHORE_ADDRESS_HIGH, 4);
423 OUT_RING (chan, upper_32_bits(offset));
424 OUT_RING (chan, lower_32_bits(offset));
425 OUT_RING (chan, 1);
426 OUT_RING (chan, 0x1002); /* RELEASE */
427 }
428
429 /* Delay semaphore destruction until its work is done */
430 ret = nouveau_fence_new(chan, &fence, true);
431 if (ret)
432 return ret;
433
434 kref_get(&sema->ref);
435 nouveau_fence_work(fence, semaphore_work, sema);
436 nouveau_fence_unref(&fence);
437 return 0;
438 }
439
440 int
nouveau_fence_sync(struct nouveau_fence * fence,struct nouveau_channel * wchan)441 nouveau_fence_sync(struct nouveau_fence *fence,
442 struct nouveau_channel *wchan)
443 {
444 struct nouveau_channel *chan = nouveau_fence_channel(fence);
445 struct drm_device *dev = wchan->dev;
446 struct nouveau_semaphore *sema;
447 int ret = 0;
448
449 if (likely(!chan || chan == wchan ||
450 nouveau_fence_signalled(fence)))
451 goto out;
452
453 sema = semaphore_alloc(dev);
454 if (!sema) {
455 /* Early card or broken userspace, fall back to
456 * software sync. */
457 ret = nouveau_fence_wait(fence, true, false);
458 goto out;
459 }
460
461 /* try to take chan's mutex, if we can't take it right away
462 * we have to fallback to software sync to prevent locking
463 * order issues
464 */
465 if (!mutex_trylock(&chan->mutex)) {
466 ret = nouveau_fence_wait(fence, true, false);
467 goto out_unref;
468 }
469
470 /* Make wchan wait until it gets signalled */
471 ret = semaphore_acquire(wchan, sema);
472 if (ret)
473 goto out_unlock;
474
475 /* Signal the semaphore from chan */
476 ret = semaphore_release(chan, sema);
477
478 out_unlock:
479 mutex_unlock(&chan->mutex);
480 out_unref:
481 kref_put(&sema->ref, semaphore_free);
482 out:
483 if (chan)
484 nouveau_channel_put_unlocked(&chan);
485 return ret;
486 }
487
488 int
__nouveau_fence_flush(void * sync_obj,void * sync_arg)489 __nouveau_fence_flush(void *sync_obj, void *sync_arg)
490 {
491 return 0;
492 }
493
494 int
nouveau_fence_channel_init(struct nouveau_channel * chan)495 nouveau_fence_channel_init(struct nouveau_channel *chan)
496 {
497 struct drm_device *dev = chan->dev;
498 struct drm_nouveau_private *dev_priv = dev->dev_private;
499 struct nouveau_gpuobj *obj = NULL;
500 int ret;
501
502 if (dev_priv->card_type < NV_C0) {
503 /* Create an NV_SW object for various sync purposes */
504 ret = nouveau_gpuobj_gr_new(chan, NvSw, NV_SW);
505 if (ret)
506 return ret;
507
508 ret = RING_SPACE(chan, 2);
509 if (ret)
510 return ret;
511
512 BEGIN_RING(chan, NvSubSw, NV01_SUBCHAN_OBJECT, 1);
513 OUT_RING (chan, NvSw);
514 FIRE_RING (chan);
515 }
516
517 /* Setup area of memory shared between all channels for x-chan sync */
518 if (USE_SEMA(dev) && dev_priv->chipset < 0x84) {
519 struct ttm_mem_reg *mem = &dev_priv->fence.bo->bo.mem;
520
521 ret = nouveau_gpuobj_dma_new(chan, NV_CLASS_DMA_FROM_MEMORY,
522 mem->start << PAGE_SHIFT,
523 mem->size, NV_MEM_ACCESS_RW,
524 NV_MEM_TARGET_VRAM, &obj);
525 if (ret)
526 return ret;
527
528 ret = nouveau_ramht_insert(chan, NvSema, obj);
529 nouveau_gpuobj_ref(NULL, &obj);
530 if (ret)
531 return ret;
532 } else
533 if (USE_SEMA(dev)) {
534 /* map fence bo into channel's vm */
535 ret = nouveau_bo_vma_add(dev_priv->fence.bo, chan->vm,
536 &chan->fence.vma);
537 if (ret)
538 return ret;
539 }
540
541 atomic_set(&chan->fence.last_sequence_irq, 0);
542 return 0;
543 }
544
545 void
nouveau_fence_channel_fini(struct nouveau_channel * chan)546 nouveau_fence_channel_fini(struct nouveau_channel *chan)
547 {
548 struct drm_nouveau_private *dev_priv = chan->dev->dev_private;
549 struct nouveau_fence *tmp, *fence;
550
551 spin_lock(&chan->fence.lock);
552 list_for_each_entry_safe(fence, tmp, &chan->fence.pending, entry) {
553 fence->signalled = true;
554 list_del(&fence->entry);
555
556 if (unlikely(fence->work))
557 fence->work(fence->priv, false);
558
559 kref_put(&fence->refcount, nouveau_fence_del);
560 }
561 spin_unlock(&chan->fence.lock);
562
563 nouveau_bo_vma_del(dev_priv->fence.bo, &chan->fence.vma);
564 }
565
566 int
nouveau_fence_init(struct drm_device * dev)567 nouveau_fence_init(struct drm_device *dev)
568 {
569 struct drm_nouveau_private *dev_priv = dev->dev_private;
570 int size = (dev_priv->chipset < 0x84) ? 4096 : 16384;
571 int ret;
572
573 /* Create a shared VRAM heap for cross-channel sync. */
574 if (USE_SEMA(dev)) {
575 ret = nouveau_bo_new(dev, size, 0, TTM_PL_FLAG_VRAM,
576 0, 0, &dev_priv->fence.bo);
577 if (ret)
578 return ret;
579
580 ret = nouveau_bo_pin(dev_priv->fence.bo, TTM_PL_FLAG_VRAM);
581 if (ret)
582 goto fail;
583
584 ret = nouveau_bo_map(dev_priv->fence.bo);
585 if (ret)
586 goto fail;
587
588 ret = drm_mm_init(&dev_priv->fence.heap, 0,
589 dev_priv->fence.bo->bo.mem.size);
590 if (ret)
591 goto fail;
592
593 spin_lock_init(&dev_priv->fence.lock);
594 }
595
596 return 0;
597 fail:
598 nouveau_bo_unmap(dev_priv->fence.bo);
599 nouveau_bo_ref(NULL, &dev_priv->fence.bo);
600 return ret;
601 }
602
603 void
nouveau_fence_fini(struct drm_device * dev)604 nouveau_fence_fini(struct drm_device *dev)
605 {
606 struct drm_nouveau_private *dev_priv = dev->dev_private;
607
608 if (USE_SEMA(dev)) {
609 drm_mm_takedown(&dev_priv->fence.heap);
610 nouveau_bo_unmap(dev_priv->fence.bo);
611 nouveau_bo_unpin(dev_priv->fence.bo);
612 nouveau_bo_ref(NULL, &dev_priv->fence.bo);
613 }
614 }
615