1 /**************************************************************************
2 *
3 * Copyright 2006-2008 Tungsten Graphics, Inc., Cedar Park, TX., USA
4 * All Rights Reserved.
5 * Copyright 2009 VMware, Inc., Palo Alto, CA., USA
6 * All Rights Reserved.
7 *
8 * Permission is hereby granted, free of charge, to any person obtaining a
9 * copy of this software and associated documentation files (the
10 * "Software"), to deal in the Software without restriction, including
11 * without limitation the rights to use, copy, modify, merge, publish,
12 * distribute, sub license, and/or sell copies of the Software, and to
13 * permit persons to whom the Software is furnished to do so, subject to
14 * the following conditions:
15 *
16 * The above copyright notice and this permission notice (including the
17 * next paragraph) shall be included in all copies or substantial portions
18 * of the Software.
19 *
20 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
21 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
23 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
24 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
25 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
26 * USE OR OTHER DEALINGS IN THE SOFTWARE.
27 *
28 **************************************************************************/
29 /*
30 * Authors: Thomas Hellstrom <thomas-at-tungstengraphics-dot-com>
31 */
32
33 #ifdef HAVE_CONFIG_H
34 #include "config.h"
35 #endif
36
37 #include <drm/psb_ttm_placement_user.h>
38 #include <stdint.h>
39 #include <sys/time.h>
40 #include <errno.h>
41 #include <unistd.h>
42 #include <assert.h>
43 #include <string.h>
44 #include <sys/mman.h>
45 #include <xf86drm.h>
46 #include "wsbm_pool.h"
47 #include "wsbm_fencemgr.h"
48 #include "wsbm_priv.h"
49 #include "wsbm_manager.h"
50
51 #define WSBM_SLABPOOL_ALLOC_RETRIES 100
52 #define DRMRESTARTCOMMANDWRITE(_fd, _val, _arg, _ret) \
53 do { \
54 (_ret) = drmCommandWrite(_fd, _val, &(_arg), sizeof(_arg)); \
55 } while ((_ret) == -EAGAIN || (_ret) == -ERESTART); \
56
57 #define DRMRESTARTCOMMANDWRITEREAD(_fd, _val, _arg, _ret) \
58 do { \
59 (_ret) = drmCommandWriteRead(_fd, _val, &(_arg), sizeof(_arg)); \
60 } while ((_ret) == -EAGAIN || (_ret) == -ERESTART); \
61
62
63 #ifdef DEBUG_FENCESIGNALED
64 static int createbuffer = 0;
65 static int fencesignaled = 0;
66 #endif
67
68 struct _WsbmSlab;
69
70 struct _WsbmSlabBuffer
71 {
72 struct _WsbmKernelBuf kBuf;
73 struct _WsbmBufStorage storage;
74 struct _WsbmCond event;
75
76 /*
77 * Remains constant after creation.
78 */
79
80 int isSlabBuffer;
81 struct _WsbmSlab *parent;
82 uint32_t start;
83 void *virtual;
84 unsigned long requestedSize;
85 uint64_t mapHandle;
86
87 /*
88 * Protected by struct _WsbmSlabSizeHeader::mutex
89 */
90
91 struct _WsbmListHead head;
92
93 /*
94 * Protected by this::mutex
95 */
96
97 struct _WsbmFenceObject *fence;
98 uint32_t fenceType;
99 struct _WsbmAtomic writers; /* (Only upping) */
100 int unFenced;
101 };
102
103 struct _WsbmSlabPool;
104 struct _WsbmSlabKernelBO
105 {
106
107 /*
108 * Constant at creation
109 */
110
111 struct _WsbmKernelBuf kBuf;
112 uint32_t pageAlignment;
113 void *virtual;
114 unsigned long actualSize;
115 uint64_t mapHandle;
116
117 /*
118 * Protected by struct _WsbmSlabCache::mutex
119 */
120
121 struct _WsbmSlabPool *slabPool;
122 uint32_t proposedPlacement;
123 struct _WsbmListHead timeoutHead;
124 struct _WsbmListHead head;
125 struct timeval timeFreed;
126 };
127
128 struct _WsbmSlab
129 {
130 struct _WsbmListHead head;
131 struct _WsbmListHead freeBuffers;
132 uint32_t numBuffers;
133 uint32_t numFree;
134 struct _WsbmSlabBuffer *buffers;
135 struct _WsbmSlabSizeHeader *header;
136 struct _WsbmSlabKernelBO *kbo;
137 };
138
139 struct _WsbmSlabSizeHeader
140 {
141 /*
142 * Constant at creation.
143 */
144 struct _WsbmSlabPool *slabPool;
145 uint32_t bufSize;
146
147 /*
148 * Protected by this::mutex
149 */
150
151 struct _WsbmListHead slabs;
152 struct _WsbmListHead freeSlabs;
153 struct _WsbmListHead delayedBuffers;
154 uint32_t numDelayed;
155 struct _WsbmMutex mutex;
156 };
157
158 struct _WsbmSlabCache
159 {
160 struct timeval slabTimeout;
161 struct timeval checkInterval;
162 struct timeval nextCheck;
163 struct _WsbmListHead timeoutList;
164 struct _WsbmListHead unCached;
165 struct _WsbmListHead cached;
166 struct _WsbmMutex mutex;
167 };
168
169 struct _WsbmSlabPool
170 {
171 struct _WsbmBufferPool pool;
172
173 /*
174 * The data of this structure remains constant after
175 * initialization and thus needs no mutex protection.
176 */
177
178 unsigned int devOffset;
179 struct _WsbmSlabCache *cache;
180 uint32_t proposedPlacement;
181 uint32_t validMask;
182 uint32_t *bucketSizes;
183 uint32_t numBuckets;
184 uint32_t pageSize;
185 int pageAlignment;
186 int maxSlabSize;
187 int desiredNumBuffers;
188 struct _WsbmSlabSizeHeader *headers;
189 };
190
191 static inline struct _WsbmSlabPool *
slabPoolFromPool(struct _WsbmBufferPool * pool)192 slabPoolFromPool(struct _WsbmBufferPool *pool)
193 {
194 return containerOf(pool, struct _WsbmSlabPool, pool);
195 }
196
197 static inline struct _WsbmSlabPool *
slabPoolFromBuf(struct _WsbmSlabBuffer * sBuf)198 slabPoolFromBuf(struct _WsbmSlabBuffer *sBuf)
199 {
200 return slabPoolFromPool(sBuf->storage.pool);
201 }
202
203 static inline struct _WsbmSlabBuffer *
slabBuffer(struct _WsbmBufStorage * buf)204 slabBuffer(struct _WsbmBufStorage *buf)
205 {
206 return containerOf(buf, struct _WsbmSlabBuffer, storage);
207 }
208
209 /*
210 * FIXME: Perhaps arrange timeout slabs in size buckets for fast
211 * retreival??
212 */
213
214 static inline int
wsbmTimeAfterEq(struct timeval * arg1,struct timeval * arg2)215 wsbmTimeAfterEq(struct timeval *arg1, struct timeval *arg2)
216 {
217 return ((arg1->tv_sec > arg2->tv_sec) ||
218 ((arg1->tv_sec == arg2->tv_sec) &&
219 (arg1->tv_usec > arg2->tv_usec)));
220 }
221
222 static inline void
wsbmTimeAdd(struct timeval * arg,struct timeval * add)223 wsbmTimeAdd(struct timeval *arg, struct timeval *add)
224 {
225 unsigned int sec;
226
227 arg->tv_sec += add->tv_sec;
228 arg->tv_usec += add->tv_usec;
229 sec = arg->tv_usec / 1000000;
230 arg->tv_sec += sec;
231 arg->tv_usec -= sec * 1000000;
232 }
233
234 static void
wsbmFreeKernelBO(struct _WsbmSlabKernelBO * kbo)235 wsbmFreeKernelBO(struct _WsbmSlabKernelBO *kbo)
236 {
237 struct ttm_pl_reference_req arg;
238 struct _WsbmSlabPool *slabPool;
239
240 if (!kbo)
241 return;
242
243 slabPool = kbo->slabPool;
244 arg.handle = kbo->kBuf.handle;
245 (void)munmap(kbo->virtual, kbo->actualSize);
246 (void)drmCommandWrite(slabPool->pool.fd,
247 slabPool->devOffset + TTM_PL_UNREF, &arg,
248 sizeof(arg));
249 free(kbo);
250 }
251
252 static void
wsbmFreeTimeoutKBOsLocked(struct _WsbmSlabCache * cache,struct timeval * time)253 wsbmFreeTimeoutKBOsLocked(struct _WsbmSlabCache *cache, struct timeval *time)
254 {
255 struct _WsbmListHead *list, *next;
256 struct _WsbmSlabKernelBO *kbo;
257
258 if (!wsbmTimeAfterEq(time, &cache->nextCheck))
259 return;
260
261 WSBMLISTFOREACHSAFE(list, next, &cache->timeoutList) {
262 kbo = WSBMLISTENTRY(list, struct _WsbmSlabKernelBO, timeoutHead);
263
264 if (!wsbmTimeAfterEq(time, &kbo->timeFreed))
265 break;
266
267 WSBMLISTDELINIT(&kbo->timeoutHead);
268 WSBMLISTDELINIT(&kbo->head);
269 wsbmFreeKernelBO(kbo);
270 }
271
272 cache->nextCheck = *time;
273 wsbmTimeAdd(&cache->nextCheck, &cache->checkInterval);
274 }
275
276 /*
277 * Add a _SlabKernelBO to the free slab manager.
278 * This means that it is available for reuse, but if it's not
279 * reused in a while, it will be freed.
280 */
281
282 static void
wsbmSetKernelBOFree(struct _WsbmSlabCache * cache,struct _WsbmSlabKernelBO * kbo)283 wsbmSetKernelBOFree(struct _WsbmSlabCache *cache,
284 struct _WsbmSlabKernelBO *kbo)
285 {
286 struct timeval time;
287 struct timeval timeFreed;
288
289 gettimeofday(&time, NULL);
290 timeFreed = time;
291 WSBM_MUTEX_LOCK(&cache->mutex);
292 wsbmTimeAdd(&timeFreed, &cache->slabTimeout);
293 kbo->timeFreed = timeFreed;
294
295 if (kbo->kBuf.placement & TTM_PL_FLAG_CACHED)
296 WSBMLISTADD(&kbo->head, &cache->cached);
297 else
298 WSBMLISTADD(&kbo->head, &cache->unCached);
299
300 WSBMLISTADDTAIL(&kbo->timeoutHead, &cache->timeoutList);
301 wsbmFreeTimeoutKBOsLocked(cache, &time);
302
303 WSBM_MUTEX_UNLOCK(&cache->mutex);
304 }
305
306 /*
307 * Get a _SlabKernelBO for us to use as storage for a slab.
308 */
309
310 static struct _WsbmSlabKernelBO *
wsbmAllocKernelBO(struct _WsbmSlabSizeHeader * header)311 wsbmAllocKernelBO(struct _WsbmSlabSizeHeader *header)
312 {
313 struct _WsbmSlabPool *slabPool = header->slabPool;
314 struct _WsbmSlabCache *cache = slabPool->cache;
315 struct _WsbmListHead *list, *head;
316 uint32_t size = header->bufSize * slabPool->desiredNumBuffers;
317 struct _WsbmSlabKernelBO *kbo;
318 struct _WsbmSlabKernelBO *kboTmp;
319 int ret;
320
321 /*
322 * FIXME: We should perhaps allow some variation in slabsize in order
323 * to efficiently reuse slabs.
324 */
325
326 size = (size <= (uint32_t) slabPool->maxSlabSize) ? size : (uint32_t) slabPool->maxSlabSize;
327 if (size < header->bufSize)
328 size = header->bufSize;
329 size = (size + slabPool->pageSize - 1) & ~(slabPool->pageSize - 1);
330 WSBM_MUTEX_LOCK(&cache->mutex);
331
332 kbo = NULL;
333
334 retry:
335 head = (slabPool->proposedPlacement & TTM_PL_FLAG_CACHED) ?
336 &cache->cached : &cache->unCached;
337
338 WSBMLISTFOREACH(list, head) {
339 kboTmp = WSBMLISTENTRY(list, struct _WsbmSlabKernelBO, head);
340
341 if ((kboTmp->actualSize == size) &&
342 (slabPool->pageAlignment == 0 ||
343 (kboTmp->pageAlignment % slabPool->pageAlignment) == 0)) {
344
345 if (!kbo)
346 kbo = kboTmp;
347
348 if ((kbo->proposedPlacement ^ slabPool->proposedPlacement) == 0)
349 break;
350
351 }
352 }
353
354 if (kbo) {
355 WSBMLISTDELINIT(&kbo->head);
356 WSBMLISTDELINIT(&kbo->timeoutHead);
357 }
358
359 WSBM_MUTEX_UNLOCK(&cache->mutex);
360
361 if (kbo) {
362 uint32_t new_mask =
363 kbo->proposedPlacement ^ slabPool->proposedPlacement;
364
365 ret = 0;
366 if (new_mask) {
367 union ttm_pl_setstatus_arg arg;
368 struct ttm_pl_setstatus_req *req = &arg.req;
369 struct ttm_pl_rep *rep = &arg.rep;
370
371 req->handle = kbo->kBuf.handle;
372 req->set_placement = slabPool->proposedPlacement & new_mask;
373 req->clr_placement = ~slabPool->proposedPlacement & new_mask;
374 DRMRESTARTCOMMANDWRITEREAD(slabPool->pool.fd,
375 slabPool->devOffset + TTM_PL_SETSTATUS,
376 arg, ret);
377 if (ret == 0) {
378 kbo->kBuf.gpuOffset = rep->gpu_offset;
379 kbo->kBuf.placement = rep->placement;
380 }
381 kbo->proposedPlacement = slabPool->proposedPlacement;
382 }
383
384 if (ret == 0)
385 return kbo;
386
387 wsbmFreeKernelBO(kbo);
388 kbo = NULL;
389 goto retry;
390 }
391
392 kbo = calloc(1, sizeof(*kbo));
393 if (!kbo)
394 return NULL;
395
396 {
397 union ttm_pl_create_arg arg;
398
399 kbo->slabPool = slabPool;
400 WSBMINITLISTHEAD(&kbo->head);
401 WSBMINITLISTHEAD(&kbo->timeoutHead);
402
403 arg.req.size = size;
404 arg.req.placement = slabPool->proposedPlacement;
405 arg.req.page_alignment = slabPool->pageAlignment;
406
407 DRMRESTARTCOMMANDWRITEREAD(slabPool->pool.fd,
408 slabPool->devOffset + TTM_PL_CREATE,
409 arg, ret);
410 if (ret)
411 goto out_err0;
412
413 kbo->kBuf.gpuOffset = arg.rep.gpu_offset;
414 kbo->kBuf.placement = arg.rep.placement;
415 kbo->kBuf.handle = arg.rep.handle;
416
417 kbo->actualSize = arg.rep.bo_size;
418 kbo->mapHandle = arg.rep.map_handle;
419 kbo->proposedPlacement = slabPool->proposedPlacement;
420 }
421
422 kbo->virtual = mmap(0, kbo->actualSize,
423 PROT_READ | PROT_WRITE, MAP_SHARED,
424 slabPool->pool.fd, kbo->mapHandle);
425
426 if (kbo->virtual == MAP_FAILED) {
427 ret = -errno;
428 goto out_err1;
429 }
430
431 return kbo;
432
433 out_err1:
434 {
435 struct ttm_pl_reference_req arg = {.handle = kbo->kBuf.handle };
436
437 (void)drmCommandWrite(slabPool->pool.fd,
438 slabPool->devOffset + TTM_PL_UNREF,
439 &arg, sizeof(arg));
440 }
441 out_err0:
442 free(kbo);
443 return NULL;
444 }
445
446 static int
wsbmAllocSlab(struct _WsbmSlabSizeHeader * header)447 wsbmAllocSlab(struct _WsbmSlabSizeHeader *header)
448 {
449 struct _WsbmSlab *slab;
450 struct _WsbmSlabBuffer *sBuf;
451 uint32_t numBuffers;
452 uint32_t ret;
453 uint32_t i;
454
455 slab = calloc(1, sizeof(*slab));
456 if (!slab)
457 return -ENOMEM;
458
459 slab->kbo = wsbmAllocKernelBO(header);
460 if (!slab->kbo) {
461 ret = -ENOMEM;
462 goto out_err0;
463 }
464
465 numBuffers = slab->kbo->actualSize / header->bufSize;
466
467 slab->buffers = calloc(numBuffers, sizeof(*slab->buffers));
468 if (!slab->buffers) {
469 ret = -ENOMEM;
470 goto out_err1;
471 }
472
473 WSBMINITLISTHEAD(&slab->head);
474 WSBMINITLISTHEAD(&slab->freeBuffers);
475 slab->numBuffers = numBuffers;
476 slab->numFree = 0;
477 slab->header = header;
478
479 sBuf = slab->buffers;
480 for (i = 0; i < numBuffers; ++i) {
481 ret = wsbmBufStorageInit(&sBuf->storage, &header->slabPool->pool);
482 if (ret)
483 goto out_err2;
484 sBuf->parent = slab;
485 sBuf->start = i * header->bufSize;
486 sBuf->virtual = (void *)((uint8_t *) slab->kbo->virtual +
487 sBuf->start);
488 wsbmAtomicSet(&sBuf->writers, 0);
489 sBuf->isSlabBuffer = 1;
490 WSBM_COND_INIT(&sBuf->event);
491 WSBMLISTADDTAIL(&sBuf->head, &slab->freeBuffers);
492 slab->numFree++;
493 sBuf++;
494 }
495
496 WSBMLISTADDTAIL(&slab->head, &header->slabs);
497
498 return 0;
499
500 out_err2:
501 sBuf = slab->buffers;
502 for (i = 0; i < numBuffers; ++i) {
503 if (sBuf->parent == slab) {
504 WSBM_COND_FREE(&sBuf->event);
505 wsbmBufStorageTakedown(&sBuf->storage);
506 }
507 sBuf++;
508 }
509 free(slab->buffers);
510 out_err1:
511 wsbmSetKernelBOFree(header->slabPool->cache, slab->kbo);
512 out_err0:
513 free(slab);
514 return ret;
515 }
516
517 /*
518 * Delete a buffer from the slab header delayed list and put
519 * it on the slab free list.
520 */
521
522 static void
wsbmSlabFreeBufferLocked(struct _WsbmSlabBuffer * buf)523 wsbmSlabFreeBufferLocked(struct _WsbmSlabBuffer *buf)
524 {
525 struct _WsbmSlab *slab = buf->parent;
526 struct _WsbmSlabSizeHeader *header = slab->header;
527 struct _WsbmListHead *list = &buf->head;
528
529 WSBMLISTDEL(list);
530 WSBMLISTADDTAIL(list, &slab->freeBuffers);
531 slab->numFree++;
532
533 if (slab->head.next == &slab->head)
534 WSBMLISTADDTAIL(&slab->head, &header->slabs);
535
536 if (slab->numFree == slab->numBuffers) {
537 list = &slab->head;
538 WSBMLISTDEL(list);
539 WSBMLISTADDTAIL(list, &header->freeSlabs);
540 }
541
542 if (header->slabs.next == &header->slabs ||
543 slab->numFree != slab->numBuffers) {
544
545 struct _WsbmListHead *next;
546 struct _WsbmSlabCache *cache = header->slabPool->cache;
547
548 WSBMLISTFOREACHSAFE(list, next, &header->freeSlabs) {
549 uint32_t i;
550 struct _WsbmSlabBuffer *sBuf;
551
552 slab = WSBMLISTENTRY(list, struct _WsbmSlab, head);
553
554 WSBMLISTDELINIT(list);
555
556 sBuf = slab->buffers;
557 for (i = 0; i < slab->numBuffers; ++i) {
558 if (sBuf->parent == slab) {
559 WSBM_COND_FREE(&sBuf->event);
560 wsbmBufStorageTakedown(&sBuf->storage);
561 }
562 sBuf++;
563 }
564 wsbmSetKernelBOFree(cache, slab->kbo);
565 free(slab->buffers);
566 free(slab);
567 }
568 }
569 }
570
571 static void
wsbmSlabCheckFreeLocked(struct _WsbmSlabSizeHeader * header,int wait)572 wsbmSlabCheckFreeLocked(struct _WsbmSlabSizeHeader *header, int wait)
573 {
574 struct _WsbmListHead *list, *prev, *first, *head;
575 struct _WsbmSlabBuffer *sBuf;
576 struct _WsbmSlab *slab;
577 int firstWasSignaled = 1;
578 int signaled;
579 uint32_t i;
580 int ret;
581
582 /*
583 * Rerun the freeing test if the youngest tested buffer
584 * was signaled, since there might be more idle buffers
585 * in the delay list.
586 */
587
588 while (firstWasSignaled) {
589 firstWasSignaled = 0;
590 signaled = 0;
591 first = header->delayedBuffers.next;
592
593 /* Only examine the oldest 1/3 of delayed buffers:
594 */
595 if (header->numDelayed > 3) {
596 for (i = 0; i < header->numDelayed; i += 3) {
597 first = first->next;
598 }
599 }
600
601 /*
602 * No need to take the buffer mutex for each buffer we loop
603 * through since we're currently the only user.
604 */
605
606 head = first->next;
607 WSBMLISTFOREACHPREVSAFE(list, prev, head) {
608
609 if (list == &header->delayedBuffers)
610 break;
611
612 sBuf = WSBMLISTENTRY(list, struct _WsbmSlabBuffer, head);
613
614 slab = sBuf->parent;
615
616 if (!signaled) {
617 if (wait) {
618 ret = wsbmFenceFinish(sBuf->fence, sBuf->fenceType, 0);
619 if (ret)
620 break;
621 signaled = 1;
622 wait = 0;
623 } else {
624 signaled =
625 wsbmFenceSignaled(sBuf->fence, sBuf->fenceType);
626 #ifdef DEBUG_FENCESIGNALED
627 fencesignaled++;
628 #endif
629 }
630 if (signaled) {
631 if (list == first)
632 firstWasSignaled = 1;
633 wsbmFenceUnreference(&sBuf->fence);
634 header->numDelayed--;
635 wsbmSlabFreeBufferLocked(sBuf);
636 } else
637 break;
638 } else if (wsbmFenceSignaledCached(sBuf->fence, sBuf->fenceType)) {
639 wsbmFenceUnreference(&sBuf->fence);
640 header->numDelayed--;
641 wsbmSlabFreeBufferLocked(sBuf);
642 }
643 }
644 }
645 }
646
647 static struct _WsbmSlabBuffer *
wsbmSlabAllocBuffer(struct _WsbmSlabSizeHeader * header)648 wsbmSlabAllocBuffer(struct _WsbmSlabSizeHeader *header)
649 {
650 static struct _WsbmSlabBuffer *buf;
651 struct _WsbmSlab *slab;
652 struct _WsbmListHead *list;
653 int count = WSBM_SLABPOOL_ALLOC_RETRIES;
654
655 WSBM_MUTEX_LOCK(&header->mutex);
656 while (header->slabs.next == &header->slabs && count > 0) {
657 wsbmSlabCheckFreeLocked(header, 0);
658 if (header->slabs.next != &header->slabs)
659 break;
660
661 WSBM_MUTEX_UNLOCK(&header->mutex);
662 if (count != WSBM_SLABPOOL_ALLOC_RETRIES)
663 usleep(1000);
664 WSBM_MUTEX_LOCK(&header->mutex);
665 (void)wsbmAllocSlab(header);
666 count--;
667 }
668
669 list = header->slabs.next;
670 if (list == &header->slabs) {
671 WSBM_MUTEX_UNLOCK(&header->mutex);
672 return NULL;
673 }
674 slab = WSBMLISTENTRY(list, struct _WsbmSlab, head);
675 if (--slab->numFree == 0)
676 WSBMLISTDELINIT(list);
677
678 list = slab->freeBuffers.next;
679 WSBMLISTDELINIT(list);
680
681 WSBM_MUTEX_UNLOCK(&header->mutex);
682 buf = WSBMLISTENTRY(list, struct _WsbmSlabBuffer, head);
683
684 buf->storage.destroyContainer = NULL;
685
686 #ifdef DEBUG_FENCESIGNALED
687 createbuffer++;
688 #endif
689 return buf;
690 }
691
692 static struct _WsbmBufStorage *
pool_create(struct _WsbmBufferPool * pool,unsigned long size,uint32_t placement,unsigned alignment)693 pool_create(struct _WsbmBufferPool *pool, unsigned long size,
694 uint32_t placement, unsigned alignment)
695 {
696 struct _WsbmSlabPool *slabPool = slabPoolFromPool(pool);
697 struct _WsbmSlabSizeHeader *header;
698 struct _WsbmSlabBuffer *sBuf;
699 uint32_t i;
700 int ret;
701
702 /*
703 * FIXME: Check for compatibility.
704 */
705
706 header = slabPool->headers;
707 for (i = 0; i < slabPool->numBuckets; ++i) {
708 if (header->bufSize >= size)
709 break;
710 header++;
711 }
712
713 if (i < slabPool->numBuckets) {
714 sBuf = wsbmSlabAllocBuffer(header);
715 return ((sBuf) ? &sBuf->storage : NULL);
716 }
717
718 /*
719 * Fall back to allocate a buffer object directly from DRM.
720 * and wrap it in a wsbmBO structure.
721 */
722
723 sBuf = calloc(1, sizeof(*sBuf));
724
725 if (!sBuf)
726 return NULL;
727
728 if (alignment) {
729 if ((alignment < slabPool->pageSize)
730 && (slabPool->pageSize % alignment))
731 goto out_err0;
732 if ((alignment > slabPool->pageSize)
733 && (alignment % slabPool->pageSize))
734 goto out_err0;
735 }
736
737 ret = wsbmBufStorageInit(&sBuf->storage, pool);
738 if (ret)
739 goto out_err0;
740
741 ret = WSBM_COND_INIT(&sBuf->event);
742 if (ret)
743 goto out_err1;
744
745 {
746 union ttm_pl_create_arg arg;
747
748 arg.req.size = size;
749 arg.req.placement = placement;
750 arg.req.page_alignment = alignment / slabPool->pageSize;
751
752 DRMRESTARTCOMMANDWRITEREAD(pool->fd,
753 slabPool->devOffset + TTM_PL_CREATE,
754 arg, ret);
755
756 if (ret)
757 goto out_err2;
758
759 sBuf->kBuf.gpuOffset = arg.rep.gpu_offset;
760 sBuf->kBuf.placement = arg.rep.placement;
761 sBuf->kBuf.handle = arg.rep.handle;
762 sBuf->mapHandle = arg.rep.map_handle;
763 sBuf->requestedSize = size;
764
765 sBuf->virtual = mmap(0, size, PROT_READ | PROT_WRITE, MAP_SHARED,
766 pool->fd, sBuf->mapHandle);
767
768 if (sBuf->virtual == MAP_FAILED)
769 goto out_err3;
770 }
771
772 wsbmAtomicSet(&sBuf->writers, 0);
773 return &sBuf->storage;
774 out_err3:
775 {
776 struct ttm_pl_reference_req arg;
777
778 arg.handle = sBuf->kBuf.handle;
779 (void)drmCommandWriteRead(pool->fd,
780 slabPool->devOffset + TTM_PL_UNREF,
781 &arg, sizeof(arg));
782 }
783 out_err2:
784 WSBM_COND_FREE(&sBuf->event);
785 out_err1:
786 wsbmBufStorageTakedown(&sBuf->storage);
787 out_err0:
788 free(sBuf);
789 return NULL;
790 }
791
792 static void
pool_destroy(struct _WsbmBufStorage ** p_buf)793 pool_destroy(struct _WsbmBufStorage **p_buf)
794 {
795 struct _WsbmBufStorage *buf = *p_buf;
796 struct _WsbmSlabBuffer *sBuf = slabBuffer(buf);
797 struct _WsbmSlab *slab;
798 struct _WsbmSlabSizeHeader *header;
799
800 *p_buf = NULL;
801
802 if (!sBuf->isSlabBuffer) {
803 struct _WsbmSlabPool *slabPool = slabPoolFromBuf(sBuf);
804 struct ttm_pl_reference_req arg;
805
806 if (sBuf->virtual != NULL) {
807 (void)munmap(sBuf->virtual, sBuf->requestedSize);
808 sBuf->virtual = NULL;
809 }
810
811 arg.handle = sBuf->kBuf.handle;
812 (void)drmCommandWrite(slabPool->pool.fd,
813 slabPool->devOffset + TTM_PL_UNREF,
814 &arg, sizeof(arg));
815
816 WSBM_COND_FREE(&sBuf->event);
817 wsbmBufStorageTakedown(&sBuf->storage);
818 free(sBuf);
819 return;
820 }
821
822 slab = sBuf->parent;
823 header = slab->header;
824
825 /*
826 * No need to take the buffer mutex below since we're the only user.
827 */
828
829 WSBM_MUTEX_LOCK(&header->mutex);
830 sBuf->unFenced = 0;
831 wsbmAtomicSet(&sBuf->writers, 0);
832 wsbmAtomicSet(&sBuf->storage.refCount, 1);
833
834 if (sBuf->fence && !wsbmFenceSignaledCached(sBuf->fence, sBuf->fenceType)) {
835 WSBMLISTADDTAIL(&sBuf->head, &header->delayedBuffers);
836 header->numDelayed++;
837 } else {
838 if (sBuf->fence)
839 wsbmFenceUnreference(&sBuf->fence);
840 wsbmSlabFreeBufferLocked(sBuf);
841 }
842
843 WSBM_MUTEX_UNLOCK(&header->mutex);
844 }
845
846 static void
waitIdleLocked(struct _WsbmSlabBuffer * sBuf,int lazy)847 waitIdleLocked(struct _WsbmSlabBuffer *sBuf, int lazy)
848 {
849 struct _WsbmBufStorage *storage = &sBuf->storage;
850
851 while (sBuf->unFenced || sBuf->fence != NULL) {
852
853 if (sBuf->unFenced)
854 WSBM_COND_WAIT(&sBuf->event, &storage->mutex);
855
856 if (sBuf->fence != NULL) {
857 if (!wsbmFenceSignaled(sBuf->fence, sBuf->fenceType)) {
858 struct _WsbmFenceObject *fence =
859 wsbmFenceReference(sBuf->fence);
860
861 WSBM_MUTEX_UNLOCK(&storage->mutex);
862 (void)wsbmFenceFinish(fence, sBuf->fenceType, lazy);
863 WSBM_MUTEX_LOCK(&storage->mutex);
864 if (sBuf->fence == fence)
865 wsbmFenceUnreference(&sBuf->fence);
866
867 wsbmFenceUnreference(&fence);
868 } else {
869 wsbmFenceUnreference(&sBuf->fence);
870 }
871 }
872 }
873 }
874
875 static int
pool_waitIdle(struct _WsbmBufStorage * buf,int lazy)876 pool_waitIdle(struct _WsbmBufStorage *buf, int lazy)
877 {
878 struct _WsbmSlabBuffer *sBuf = slabBuffer(buf);
879
880 WSBM_MUTEX_LOCK(&buf->mutex);
881 waitIdleLocked(sBuf, lazy);
882 WSBM_MUTEX_UNLOCK(&buf->mutex);
883
884 return 0;
885 }
886
887 static int
pool_map(struct _WsbmBufStorage * buf,unsigned mode,void ** virtual)888 pool_map(struct _WsbmBufStorage *buf, unsigned mode __attribute__ ((unused)), void **virtual)
889 {
890 struct _WsbmSlabBuffer *sBuf = slabBuffer(buf);
891
892 *virtual = sBuf->virtual;
893
894 return 0;
895 }
896
897 static void
pool_releaseFromCpu(struct _WsbmBufStorage * buf,unsigned mode)898 pool_releaseFromCpu(struct _WsbmBufStorage *buf, unsigned mode __attribute__ ((unused)))
899 {
900 struct _WsbmSlabBuffer *sBuf = slabBuffer(buf);
901
902 if (wsbmAtomicDecZero(&sBuf->writers))
903 WSBM_COND_BROADCAST(&sBuf->event);
904 }
905
906 static int
pool_syncForCpu(struct _WsbmBufStorage * buf,unsigned mode)907 pool_syncForCpu(struct _WsbmBufStorage *buf, unsigned mode)
908 {
909 struct _WsbmSlabBuffer *sBuf = slabBuffer(buf);
910 int ret = 0;
911
912 WSBM_MUTEX_LOCK(&buf->mutex);
913 if ((mode & WSBM_SYNCCPU_DONT_BLOCK)) {
914 int signaled;
915
916 if (sBuf->unFenced) {
917 ret = -EBUSY;
918 goto out_unlock;
919 }
920
921 if (sBuf->isSlabBuffer)
922 signaled = (sBuf->fence == NULL) ||
923 wsbmFenceSignaledCached(sBuf->fence, sBuf->fenceType);
924 else
925 signaled = (sBuf->fence == NULL) ||
926 wsbmFenceSignaled(sBuf->fence, sBuf->fenceType);
927
928 ret = 0;
929 if (signaled) {
930 wsbmFenceUnreference(&sBuf->fence);
931 wsbmAtomicInc(&sBuf->writers);
932 } else
933 ret = -EBUSY;
934 goto out_unlock;
935 }
936 waitIdleLocked(sBuf, 0);
937 wsbmAtomicInc(&sBuf->writers);
938 out_unlock:
939 WSBM_MUTEX_UNLOCK(&buf->mutex);
940 return ret;
941 }
942
943 static void
pool_unmap(struct _WsbmBufStorage * buf)944 pool_unmap(struct _WsbmBufStorage *buf __attribute__ ((unused)))
945 {
946 ;
947 }
948
949 static unsigned long
pool_poolOffset(struct _WsbmBufStorage * buf)950 pool_poolOffset(struct _WsbmBufStorage *buf)
951 {
952 struct _WsbmSlabBuffer *sBuf = slabBuffer(buf);
953
954 return sBuf->start;
955 }
956
957 static unsigned long
pool_size(struct _WsbmBufStorage * buf)958 pool_size(struct _WsbmBufStorage *buf)
959 {
960 struct _WsbmSlabBuffer *sBuf = slabBuffer(buf);
961
962 if (!sBuf->isSlabBuffer)
963 return sBuf->requestedSize;
964
965 return sBuf->parent->header->bufSize;
966 }
967
968 static struct _WsbmKernelBuf *
pool_kernel(struct _WsbmBufStorage * buf)969 pool_kernel(struct _WsbmBufStorage *buf)
970 {
971 struct _WsbmSlabBuffer *sBuf = slabBuffer(buf);
972
973 return (sBuf->isSlabBuffer) ? &sBuf->parent->kbo->kBuf : &sBuf->kBuf;
974 }
975
976 static unsigned long
pool_offset(struct _WsbmBufStorage * buf)977 pool_offset(struct _WsbmBufStorage *buf)
978 {
979 return pool_kernel(buf)->gpuOffset + pool_poolOffset(buf);
980 }
981
982 static void
pool_fence(struct _WsbmBufStorage * buf,struct _WsbmFenceObject * fence)983 pool_fence(struct _WsbmBufStorage *buf, struct _WsbmFenceObject *fence)
984 {
985 struct _WsbmSlabBuffer *sBuf = slabBuffer(buf);
986 struct _WsbmKernelBuf *kBuf;
987
988 WSBM_MUTEX_LOCK(&buf->mutex);
989 if (sBuf->fence)
990 wsbmFenceUnreference(&sBuf->fence);
991
992 kBuf = pool_kernel(buf);
993 sBuf->fenceType = kBuf->fence_type_mask;
994 if (!wsbmFenceSignaledCached(fence, sBuf->fenceType))
995 sBuf->fence = wsbmFenceReference(fence);
996
997 sBuf->unFenced = 0;
998 WSBM_COND_BROADCAST(&sBuf->event);
999 WSBM_MUTEX_UNLOCK(&buf->mutex);
1000 }
1001
1002 static int
pool_validate(struct _WsbmBufStorage * buf,uint64_t set_flags,uint64_t clr_flags)1003 pool_validate(struct _WsbmBufStorage *buf,
1004 uint64_t set_flags __attribute__ ((unused)), uint64_t clr_flags __attribute__ ((unused)))
1005 {
1006 struct _WsbmSlabBuffer *sBuf = slabBuffer(buf);
1007
1008 WSBM_MUTEX_LOCK(&buf->mutex);
1009 while (wsbmAtomicRead(&sBuf->writers) != 0) {
1010 WSBM_COND_WAIT(&sBuf->event, &buf->mutex);
1011 }
1012
1013 sBuf->unFenced = 1;
1014 WSBM_MUTEX_UNLOCK(&buf->mutex);
1015 return 0;
1016 }
1017
1018 static void
pool_unvalidate(struct _WsbmBufStorage * buf)1019 pool_unvalidate(struct _WsbmBufStorage *buf)
1020 {
1021 struct _WsbmSlabBuffer *sBuf = slabBuffer(buf);
1022
1023 WSBM_MUTEX_LOCK(&buf->mutex);
1024 if (sBuf->unFenced) {
1025 sBuf->unFenced = 0;
1026 WSBM_COND_BROADCAST(&sBuf->event);
1027 }
1028 WSBM_MUTEX_UNLOCK(&buf->mutex);
1029 }
1030
1031 struct _WsbmSlabCache *
wsbmSlabCacheInit(uint32_t checkIntervalMsec,uint32_t slabTimeoutMsec)1032 wsbmSlabCacheInit(uint32_t checkIntervalMsec, uint32_t slabTimeoutMsec)
1033 {
1034 struct _WsbmSlabCache *tmp;
1035
1036 tmp = calloc(1, sizeof(*tmp));
1037 if (!tmp)
1038 return NULL;
1039
1040 WSBM_MUTEX_INIT(&tmp->mutex);
1041 WSBM_MUTEX_LOCK(&tmp->mutex);
1042 tmp->slabTimeout.tv_usec = slabTimeoutMsec * 1000;
1043 tmp->slabTimeout.tv_sec = tmp->slabTimeout.tv_usec / 1000000;
1044 tmp->slabTimeout.tv_usec -= tmp->slabTimeout.tv_sec * 1000000;
1045
1046 tmp->checkInterval.tv_usec = checkIntervalMsec * 1000;
1047 tmp->checkInterval.tv_sec = tmp->checkInterval.tv_usec / 1000000;
1048 tmp->checkInterval.tv_usec -= tmp->checkInterval.tv_sec * 1000000;
1049
1050 gettimeofday(&tmp->nextCheck, NULL);
1051 wsbmTimeAdd(&tmp->nextCheck, &tmp->checkInterval);
1052 WSBMINITLISTHEAD(&tmp->timeoutList);
1053 WSBMINITLISTHEAD(&tmp->unCached);
1054 WSBMINITLISTHEAD(&tmp->cached);
1055 WSBM_MUTEX_UNLOCK(&tmp->mutex);
1056
1057 return tmp;
1058 }
1059
1060 void
wsbmSlabCacheFinish(struct _WsbmSlabCache * cache)1061 wsbmSlabCacheFinish(struct _WsbmSlabCache *cache)
1062 {
1063 struct timeval time;
1064
1065 time = cache->nextCheck;
1066 WSBM_MUTEX_LOCK(&cache->mutex);
1067 wsbmTimeAdd(&time, &cache->checkInterval);
1068 wsbmFreeTimeoutKBOsLocked(cache, &time);
1069 WSBM_MUTEX_UNLOCK(&cache->mutex);
1070
1071 assert(cache->timeoutList.next == &cache->timeoutList);
1072 assert(cache->unCached.next == &cache->unCached);
1073 assert(cache->cached.next == &cache->cached);
1074
1075 WSBM_MUTEX_FREE(&cache->mutex);
1076 free(cache);
1077 }
1078
1079 static void
wsbmInitSizeHeader(struct _WsbmSlabPool * slabPool,uint32_t size,struct _WsbmSlabSizeHeader * header)1080 wsbmInitSizeHeader(struct _WsbmSlabPool *slabPool, uint32_t size,
1081 struct _WsbmSlabSizeHeader *header)
1082 {
1083 WSBM_MUTEX_INIT(&header->mutex);
1084 WSBM_MUTEX_LOCK(&header->mutex);
1085
1086 WSBMINITLISTHEAD(&header->slabs);
1087 WSBMINITLISTHEAD(&header->freeSlabs);
1088 WSBMINITLISTHEAD(&header->delayedBuffers);
1089
1090 header->numDelayed = 0;
1091 header->slabPool = slabPool;
1092 header->bufSize = size;
1093
1094 WSBM_MUTEX_UNLOCK(&header->mutex);
1095 }
1096
1097 static void
wsbmFinishSizeHeader(struct _WsbmSlabSizeHeader * header)1098 wsbmFinishSizeHeader(struct _WsbmSlabSizeHeader *header)
1099 {
1100 struct _WsbmListHead *list, *next;
1101 struct _WsbmSlabBuffer *sBuf;
1102
1103 WSBM_MUTEX_LOCK(&header->mutex);
1104 WSBMLISTFOREACHSAFE(list, next, &header->delayedBuffers) {
1105 sBuf = WSBMLISTENTRY(list, struct _WsbmSlabBuffer, head);
1106
1107 if (sBuf->fence) {
1108 (void)wsbmFenceFinish(sBuf->fence, sBuf->fenceType, 0);
1109 wsbmFenceUnreference(&sBuf->fence);
1110 }
1111 header->numDelayed--;
1112 wsbmSlabFreeBufferLocked(sBuf);
1113 }
1114 WSBM_MUTEX_UNLOCK(&header->mutex);
1115 WSBM_MUTEX_FREE(&header->mutex);
1116 }
1117
1118 static void
pool_takedown(struct _WsbmBufferPool * pool)1119 pool_takedown(struct _WsbmBufferPool *pool)
1120 {
1121 struct _WsbmSlabPool *slabPool = slabPoolFromPool(pool);
1122 unsigned int i;
1123
1124 for (i = 0; i < slabPool->numBuckets; ++i) {
1125 wsbmFinishSizeHeader(&slabPool->headers[i]);
1126 }
1127
1128 free(slabPool->headers);
1129 free(slabPool->bucketSizes);
1130 free(slabPool);
1131 }
1132
1133 struct _WsbmBufferPool *
wsbmSlabPoolInit(int fd,uint32_t devOffset,uint32_t placement,uint32_t validMask,uint32_t smallestSize,uint32_t numSizes,uint32_t desiredNumBuffers,uint32_t maxSlabSize,uint32_t pageAlignment,struct _WsbmSlabCache * cache)1134 wsbmSlabPoolInit(int fd,
1135 uint32_t devOffset,
1136 uint32_t placement,
1137 uint32_t validMask,
1138 uint32_t smallestSize,
1139 uint32_t numSizes,
1140 uint32_t desiredNumBuffers,
1141 uint32_t maxSlabSize,
1142 uint32_t pageAlignment, struct _WsbmSlabCache *cache)
1143 {
1144 struct _WsbmBufferPool *pool;
1145 struct _WsbmSlabPool *slabPool;
1146 uint32_t i;
1147
1148 slabPool = calloc(1, sizeof(*slabPool));
1149 if (!slabPool)
1150 return NULL;
1151
1152 pool = &slabPool->pool;
1153
1154 slabPool->bucketSizes = calloc(numSizes, sizeof(*slabPool->bucketSizes));
1155 if (!slabPool->bucketSizes)
1156 goto out_err0;
1157
1158 slabPool->headers = calloc(numSizes, sizeof(*slabPool->headers));
1159 if (!slabPool->headers)
1160 goto out_err1;
1161
1162 slabPool->devOffset = devOffset;
1163 slabPool->cache = cache;
1164 slabPool->proposedPlacement = placement;
1165 slabPool->validMask = validMask;
1166 slabPool->numBuckets = numSizes;
1167 slabPool->pageSize = getpagesize();
1168 slabPool->pageAlignment = pageAlignment;
1169 slabPool->maxSlabSize = maxSlabSize;
1170 slabPool->desiredNumBuffers = desiredNumBuffers;
1171
1172 for (i = 0; i < slabPool->numBuckets; ++i) {
1173 slabPool->bucketSizes[i] = (smallestSize << i);
1174 wsbmInitSizeHeader(slabPool, slabPool->bucketSizes[i],
1175 &slabPool->headers[i]);
1176 }
1177
1178 pool->fd = fd;
1179 pool->map = &pool_map;
1180 pool->unmap = &pool_unmap;
1181 pool->destroy = &pool_destroy;
1182 pool->offset = &pool_offset;
1183 pool->poolOffset = &pool_poolOffset;
1184 pool->size = &pool_size;
1185 pool->create = &pool_create;
1186 pool->fence = &pool_fence;
1187 pool->kernel = &pool_kernel;
1188 pool->validate = &pool_validate;
1189 pool->unvalidate = &pool_unvalidate;
1190 pool->waitIdle = &pool_waitIdle;
1191 pool->takeDown = &pool_takedown;
1192 pool->releasefromcpu = &pool_releaseFromCpu;
1193 pool->syncforcpu = &pool_syncForCpu;
1194
1195 return pool;
1196
1197 out_err1:
1198 free(slabPool->bucketSizes);
1199 out_err0:
1200 free(slabPool);
1201
1202 return NULL;
1203 }
1204