1 /**************************************************************************
2 *
3 * Copyright 2006-2008 Tungsten Graphics, Inc., Cedar Park, TX., USA
4 * All Rights Reserved.
5 * Copyright 2009 Vmware, Inc., Palo Alto, CA., USA
6 * All Rights Reserved.
7 *
8 * Permission is hereby granted, free of charge, to any person obtaining a
9 * copy of this software and associated documentation files (the
10 * "Software"), to deal in the Software without restriction, including
11 * without limitation the rights to use, copy, modify, merge, publish,
12 * distribute, sub license, and/or sell copies of the Software, and to
13 * permit persons to whom the Software is furnished to do so, subject to
14 * the following conditions:
15 *
16 * The above copyright notice and this permission notice (including the
17 * next paragraph) shall be included in all copies or substantial portions
18 * of the Software.
19 *
20 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
21 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
22 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
23 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
24 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
25 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
26 * USE OR OTHER DEALINGS IN THE SOFTWARE.
27 *
28 **************************************************************************/
29 /*
30 * Authors: Thomas Hellstr�m <thomas-at-tungstengraphics-dot-com>
31 * Keith Whitwell <keithw-at-tungstengraphics-dot-com>
32 */
33
34 #ifdef HAVE_CONFIG_H
35 #include "config.h"
36 #endif
37
38 #include <stdlib.h>
39 #include "errno.h"
40 #include "string.h"
41 #include "wsbm_pool.h"
42 #include "wsbm_manager.h"
43 #include "wsbm_fencemgr.h"
44 #include "wsbm_driver.h"
45 #include "wsbm_priv.h"
46 #include "wsbm_util.h"
47 #include "wsbm_atomic.h"
48 #include "assert.h"
49
50 #define WSBM_BODATA_SIZE_ACCEPT 4096
51
52 #define WSBM_BUFFER_COMPLEX 0
53 #define WSBM_BUFFER_SIMPLE 1
54 #define WSBM_BUFFER_REF 2
55
56 struct _ValidateList
57 {
58 unsigned numTarget;
59 unsigned numCurrent;
60 unsigned numOnList;
61 unsigned hashSize;
62 uint32_t hashMask;
63 int driverData;
64 struct _WsbmListHead list;
65 struct _WsbmListHead free;
66 struct _WsbmListHead *hashTable;
67 };
68
69 struct _WsbmBufferObject
70 {
71 /* Left to the client to protect this data for now. */
72
73 struct _WsbmAtomic refCount;
74 struct _WsbmBufStorage *storage;
75
76 uint32_t placement;
77 unsigned alignment;
78 unsigned bufferType;
79 struct _WsbmBufferPool *pool;
80 };
81
82 struct _WsbmBufferList
83 {
84 int hasKernelBuffers;
85
86 struct _ValidateList kernelBuffers; /* List of kernel buffers needing validation */
87 struct _ValidateList userBuffers; /* List of user-space buffers needing validation */
88 };
89
90 static struct _WsbmMutex bmMutex;
91 static struct _WsbmCond bmCond;
92 static int initialized = 0;
93 static void *commonData = NULL;
94
95 static int kernelReaders = 0;
96 static int kernelLocked = 0;
97
98 int
wsbmInit(struct _WsbmThreadFuncs * tf,struct _WsbmVNodeFuncs * vf)99 wsbmInit(struct _WsbmThreadFuncs *tf, struct _WsbmVNodeFuncs *vf)
100 {
101 int ret;
102
103 wsbmCurThreadFunc = tf;
104 wsbmCurVNodeFunc = vf;
105
106 ret = WSBM_MUTEX_INIT(&bmMutex);
107 if (ret)
108 return -ENOMEM;
109 ret = WSBM_COND_INIT(&bmCond);
110 if (ret) {
111 WSBM_MUTEX_FREE(&bmMutex);
112 return -ENOMEM;
113 }
114
115 initialized = 1;
116 return 0;
117 }
118
119 void
wsbmCommonDataSet(void * d)120 wsbmCommonDataSet(void *d)
121 {
122 commonData = d;
123 }
124
125 void *
wsbmCommonDataGet(void)126 wsbmCommonDataGet(void)
127 {
128 return commonData;
129 }
130
131 int
wsbmIsInitialized(void)132 wsbmIsInitialized(void)
133 {
134 return initialized;
135 }
136
137 void
wsbmTakedown(void)138 wsbmTakedown(void)
139 {
140 initialized = 0;
141 commonData = NULL;
142 WSBM_COND_FREE(&bmCond);
143 WSBM_MUTEX_FREE(&bmMutex);
144 }
145
146 static struct _ValidateNode *
validateListAddNode(struct _ValidateList * list,void * item,uint32_t hash,uint64_t flags,uint64_t mask)147 validateListAddNode(struct _ValidateList *list, void *item,
148 uint32_t hash, uint64_t flags, uint64_t mask)
149 {
150 struct _ValidateNode *node;
151 struct _WsbmListHead *l;
152 struct _WsbmListHead *hashHead;
153
154 l = list->free.next;
155 if (l == &list->free) {
156 node = wsbmVNodeFuncs()->alloc(wsbmVNodeFuncs(), 0);
157 if (!node) {
158 return NULL;
159 }
160 list->numCurrent++;
161 } else {
162 WSBMLISTDEL(l);
163 node = WSBMLISTENTRY(l, struct _ValidateNode, head);
164 }
165 node->buf = item;
166 node->set_flags = flags & mask;
167 node->clr_flags = (~flags) & mask;
168 node->listItem = list->numOnList;
169 WSBMLISTADDTAIL(&node->head, &list->list);
170 list->numOnList++;
171 hashHead = list->hashTable + hash;
172 WSBMLISTADDTAIL(&node->hashHead, hashHead);
173
174 return node;
175 }
176
177 static uint32_t
wsbmHashFunc(uint8_t * key,uint32_t len,uint32_t mask)178 wsbmHashFunc(uint8_t * key, uint32_t len, uint32_t mask)
179 {
180 uint32_t hash, i;
181
182 for (hash = 0, i = 0; i < len; ++i) {
183 hash += *key++;
184 hash += (hash << 10);
185 hash ^= (hash >> 6);
186 }
187
188 hash += (hash << 3);
189 hash ^= (hash >> 11);
190 hash += (hash << 15);
191
192 return hash & mask;
193 }
194
195 static void
validateFreeList(struct _ValidateList * list)196 validateFreeList(struct _ValidateList *list)
197 {
198 struct _ValidateNode *node;
199 struct _WsbmListHead *l;
200
201 l = list->list.next;
202 while (l != &list->list) {
203 WSBMLISTDEL(l);
204 node = WSBMLISTENTRY(l, struct _ValidateNode, head);
205
206 WSBMLISTDEL(&node->hashHead);
207 node->func->free(node);
208 l = list->list.next;
209 list->numCurrent--;
210 list->numOnList--;
211 }
212
213 l = list->free.next;
214 while (l != &list->free) {
215 WSBMLISTDEL(l);
216 node = WSBMLISTENTRY(l, struct _ValidateNode, head);
217
218 node->func->free(node);
219 l = list->free.next;
220 list->numCurrent--;
221 }
222 free(list->hashTable);
223 }
224
225 static int
validateListAdjustNodes(struct _ValidateList * list)226 validateListAdjustNodes(struct _ValidateList *list)
227 {
228 struct _ValidateNode *node;
229 struct _WsbmListHead *l;
230 int ret = 0;
231
232 while (list->numCurrent < list->numTarget) {
233 node = wsbmVNodeFuncs()->alloc(wsbmVNodeFuncs(), list->driverData);
234 if (!node) {
235 ret = -ENOMEM;
236 break;
237 }
238 list->numCurrent++;
239 WSBMLISTADD(&node->head, &list->free);
240 }
241
242 while (list->numCurrent > list->numTarget) {
243 l = list->free.next;
244 if (l == &list->free)
245 break;
246 WSBMLISTDEL(l);
247 node = WSBMLISTENTRY(l, struct _ValidateNode, head);
248
249 node->func->free(node);
250 list->numCurrent--;
251 }
252 return ret;
253 }
254
255 static inline int
wsbmPot(unsigned int val)256 wsbmPot(unsigned int val)
257 {
258 unsigned int shift = 0;
259 while(val > (unsigned int)(1 << shift))
260 shift++;
261
262 return shift;
263 }
264
265
266
267 static int
validateCreateList(int numTarget,struct _ValidateList * list,int driverData)268 validateCreateList(int numTarget, struct _ValidateList *list, int driverData)
269 {
270 unsigned int i;
271 unsigned int shift = wsbmPot(numTarget);
272 int ret;
273
274 list->hashSize = (1 << shift);
275 list->hashMask = list->hashSize - 1;
276
277 list->hashTable = malloc(list->hashSize * sizeof(*list->hashTable));
278 if (!list->hashTable)
279 return -ENOMEM;
280
281 for (i = 0; i < list->hashSize; ++i)
282 WSBMINITLISTHEAD(&list->hashTable[i]);
283
284 WSBMINITLISTHEAD(&list->list);
285 WSBMINITLISTHEAD(&list->free);
286 list->numTarget = numTarget;
287 list->numCurrent = 0;
288 list->numOnList = 0;
289 list->driverData = driverData;
290 ret = validateListAdjustNodes(list);
291 if (ret != 0)
292 free(list->hashTable);
293
294 return ret;
295 }
296
297 static int
validateResetList(struct _ValidateList * list)298 validateResetList(struct _ValidateList *list)
299 {
300 struct _WsbmListHead *l;
301 struct _ValidateNode *node;
302 int ret;
303
304 ret = validateListAdjustNodes(list);
305 if (ret)
306 return ret;
307
308 l = list->list.next;
309 while (l != &list->list) {
310 WSBMLISTDEL(l);
311 node = WSBMLISTENTRY(l, struct _ValidateNode, head);
312
313 WSBMLISTDEL(&node->hashHead);
314 WSBMLISTADD(l, &list->free);
315 list->numOnList--;
316 l = list->list.next;
317 }
318 return validateListAdjustNodes(list);
319 }
320
321 void
wsbmWriteLockKernelBO(void)322 wsbmWriteLockKernelBO(void)
323 {
324 WSBM_MUTEX_LOCK(&bmMutex);
325 while (kernelReaders != 0)
326 WSBM_COND_WAIT(&bmCond, &bmMutex);
327 kernelLocked = 1;
328 }
329
330 void
wsbmWriteUnlockKernelBO(void)331 wsbmWriteUnlockKernelBO(void)
332 {
333 kernelLocked = 0;
334 WSBM_MUTEX_UNLOCK(&bmMutex);
335 }
336
337 void
wsbmReadLockKernelBO(void)338 wsbmReadLockKernelBO(void)
339 {
340 WSBM_MUTEX_LOCK(&bmMutex);
341 if (kernelReaders++ == 0)
342 kernelLocked = 1;
343 WSBM_MUTEX_UNLOCK(&bmMutex);
344 }
345
346 void
wsbmReadUnlockKernelBO(void)347 wsbmReadUnlockKernelBO(void)
348 {
349 WSBM_MUTEX_LOCK(&bmMutex);
350 if (--kernelReaders == 0) {
351 kernelLocked = 0;
352 WSBM_COND_BROADCAST(&bmCond);
353 }
354 WSBM_MUTEX_UNLOCK(&bmMutex);
355 }
356
357 void
wsbmBOWaitIdle(struct _WsbmBufferObject * buf,int lazy)358 wsbmBOWaitIdle(struct _WsbmBufferObject *buf, int lazy)
359 {
360 struct _WsbmBufStorage *storage;
361
362 storage = buf->storage;
363 if (!storage)
364 return;
365
366 (void)storage->pool->waitIdle(storage, lazy);
367 }
368
369 void *
wsbmBOMap(struct _WsbmBufferObject * buf,unsigned mode)370 wsbmBOMap(struct _WsbmBufferObject *buf, unsigned mode)
371 {
372 struct _WsbmBufStorage *storage = buf->storage;
373 void *virtual;
374 int retval;
375
376 retval = storage->pool->map(storage, mode, &virtual);
377
378 return (retval == 0) ? virtual : NULL;
379 }
380
381 void
wsbmBOUnmap(struct _WsbmBufferObject * buf)382 wsbmBOUnmap(struct _WsbmBufferObject *buf)
383 {
384 struct _WsbmBufStorage *storage = buf->storage;
385
386 if (!storage)
387 return;
388
389 storage->pool->unmap(storage);
390 }
391
392 int
wsbmBOSyncForCpu(struct _WsbmBufferObject * buf,unsigned mode)393 wsbmBOSyncForCpu(struct _WsbmBufferObject *buf, unsigned mode)
394 {
395 struct _WsbmBufStorage *storage = buf->storage;
396
397 return storage->pool->syncforcpu(storage, mode);
398 }
399
400 void
wsbmBOReleaseFromCpu(struct _WsbmBufferObject * buf,unsigned mode)401 wsbmBOReleaseFromCpu(struct _WsbmBufferObject *buf, unsigned mode)
402 {
403 struct _WsbmBufStorage *storage = buf->storage;
404
405 storage->pool->releasefromcpu(storage, mode);
406 }
407
408 unsigned long
wsbmBOOffsetHint(struct _WsbmBufferObject * buf)409 wsbmBOOffsetHint(struct _WsbmBufferObject *buf)
410 {
411 struct _WsbmBufStorage *storage = buf->storage;
412
413 return storage->pool->offset(storage);
414 }
415
416 unsigned long
wsbmBOPoolOffset(struct _WsbmBufferObject * buf)417 wsbmBOPoolOffset(struct _WsbmBufferObject *buf)
418 {
419 struct _WsbmBufStorage *storage = buf->storage;
420
421 return storage->pool->poolOffset(storage);
422 }
423
424 uint32_t
wsbmBOPlacementHint(struct _WsbmBufferObject * buf)425 wsbmBOPlacementHint(struct _WsbmBufferObject * buf)
426 {
427 struct _WsbmBufStorage *storage = buf->storage;
428
429 assert(buf->storage != NULL);
430
431 return storage->pool->placement(storage);
432 }
433
434 struct _WsbmBufferObject *
wsbmBOReference(struct _WsbmBufferObject * buf)435 wsbmBOReference(struct _WsbmBufferObject *buf)
436 {
437 if (buf->bufferType == WSBM_BUFFER_SIMPLE) {
438 wsbmAtomicInc(&buf->storage->refCount);
439 } else {
440 wsbmAtomicInc(&buf->refCount);
441 }
442 return buf;
443 }
444
445 int
wsbmBOSetStatus(struct _WsbmBufferObject * buf,uint32_t setFlags,uint32_t clrFlags)446 wsbmBOSetStatus(struct _WsbmBufferObject *buf,
447 uint32_t setFlags, uint32_t clrFlags)
448 {
449 struct _WsbmBufStorage *storage = buf->storage;
450
451 if (!storage)
452 return 0;
453
454 if (storage->pool->setStatus == NULL)
455 return -EINVAL;
456
457 return storage->pool->setStatus(storage, setFlags, clrFlags);
458 }
459
460 void
wsbmBOUnreference(struct _WsbmBufferObject ** p_buf)461 wsbmBOUnreference(struct _WsbmBufferObject **p_buf)
462 {
463 struct _WsbmBufferObject *buf = *p_buf;
464
465 *p_buf = NULL;
466
467 if (!buf)
468 return;
469
470 if (buf->bufferType == WSBM_BUFFER_SIMPLE) {
471 struct _WsbmBufStorage *dummy = buf->storage;
472
473 wsbmBufStorageUnref(&dummy);
474 return;
475 }
476
477 if (wsbmAtomicDecZero(&buf->refCount)) {
478 wsbmBufStorageUnref(&buf->storage);
479 free(buf);
480 }
481 }
482
483 int
wsbmBOData(struct _WsbmBufferObject * buf,unsigned size,const void * data,struct _WsbmBufferPool * newPool,uint32_t placement)484 wsbmBOData(struct _WsbmBufferObject *buf,
485 unsigned size, const void *data,
486 struct _WsbmBufferPool *newPool, uint32_t placement)
487 {
488 void *virtual = NULL;
489 int newBuffer;
490 int retval = 0;
491 struct _WsbmBufStorage *storage;
492 int synced = 0;
493 uint32_t placement_diff;
494 struct _WsbmBufferPool *curPool;
495
496 if (buf->bufferType == WSBM_BUFFER_SIMPLE)
497 return -EINVAL;
498
499 storage = buf->storage;
500
501 if (newPool == NULL)
502 newPool = buf->pool;
503
504 if (newPool == NULL)
505 return -EINVAL;
506
507 newBuffer = (!storage || storage->pool != newPool ||
508 storage->pool->size(storage) < size ||
509 storage->pool->size(storage) >
510 size + WSBM_BODATA_SIZE_ACCEPT);
511
512 if (!placement)
513 placement = buf->placement;
514
515 if (newBuffer) {
516 if (buf->bufferType == WSBM_BUFFER_REF)
517 return -EINVAL;
518
519 wsbmBufStorageUnref(&buf->storage);
520
521 if (size == 0) {
522 buf->pool = newPool;
523 buf->placement = placement;
524 retval = 0;
525 goto out;
526 }
527
528 buf->storage =
529 newPool->create(newPool, size, placement, buf->alignment);
530 if (!buf->storage) {
531 retval = -ENOMEM;
532 goto out;
533 }
534
535 buf->placement = placement;
536 buf->pool = newPool;
537 } else if (wsbmAtomicRead(&storage->onList) ||
538 0 != storage->pool->syncforcpu(storage, WSBM_SYNCCPU_WRITE |
539 WSBM_SYNCCPU_DONT_BLOCK)) {
540 /*
541 * Buffer is busy. need to create a new one.
542 */
543
544 struct _WsbmBufStorage *tmp_storage;
545
546 curPool = storage->pool;
547
548 tmp_storage =
549 curPool->create(curPool, size, placement, buf->alignment);
550
551 if (tmp_storage) {
552 wsbmBufStorageUnref(&buf->storage);
553 buf->storage = tmp_storage;
554 buf->placement = placement;
555 } else {
556 retval = curPool->syncforcpu(storage, WSBM_SYNCCPU_WRITE);
557 if (retval)
558 goto out;
559 synced = 1;
560 }
561 } else
562 synced = 1;
563
564 placement_diff = placement ^ buf->placement;
565
566 /*
567 * We might need to change buffer placement.
568 */
569
570 storage = buf->storage;
571 curPool = storage->pool;
572
573 if (placement_diff) {
574 assert(curPool->setStatus != NULL);
575 curPool->releasefromcpu(storage, WSBM_SYNCCPU_WRITE);
576 retval = curPool->setStatus(storage,
577 placement_diff & placement,
578 placement_diff & ~placement);
579 if (retval)
580 goto out;
581
582 buf->placement = placement;
583
584 }
585
586 if (!synced) {
587 retval = curPool->syncforcpu(buf->storage, WSBM_SYNCCPU_WRITE);
588
589 if (retval)
590 goto out;
591 synced = 1;
592 }
593
594 storage = buf->storage;
595 curPool = storage->pool;
596
597 if (data) {
598 retval = curPool->map(storage, WSBM_ACCESS_WRITE, &virtual);
599 if (retval)
600 goto out;
601 memcpy(virtual, data, size);
602 curPool->unmap(storage);
603 }
604
605 out:
606
607 if (synced)
608 curPool->releasefromcpu(storage, WSBM_SYNCCPU_WRITE);
609
610 return retval;
611 }
612
613 int
wsbmBODataUB(struct _WsbmBufferObject * buf,unsigned size,const void * data,struct _WsbmBufferPool * newPool,uint32_t placement,const unsigned long * user_ptr,int fd)614 wsbmBODataUB(struct _WsbmBufferObject *buf,
615 unsigned size, const void *data, struct _WsbmBufferPool *newPool,
616 uint32_t placement, const unsigned long *user_ptr, int fd)
617 {
618 int newBuffer;
619 int retval = 0;
620 struct _WsbmBufStorage *storage;
621 int synced = 0;
622 uint32_t placement_diff;
623 struct _WsbmBufferPool *curPool;
624 extern struct _WsbmBufStorage *
625 ttm_pool_ub_create(struct _WsbmBufferPool *pool,
626 unsigned long size, uint32_t placement, unsigned alignment,
627 const unsigned long *user_ptr, int fd);
628
629 if (buf->bufferType == WSBM_BUFFER_SIMPLE)
630 return -EINVAL;
631
632 storage = buf->storage;
633
634 if (newPool == NULL)
635 newPool = buf->pool;
636
637 if (newPool == NULL)
638 return -EINVAL;
639
640 newBuffer = (!storage || storage->pool != newPool ||
641 storage->pool->size(storage) < size ||
642 storage->pool->size(storage) >
643 size + WSBM_BODATA_SIZE_ACCEPT);
644
645 if (!placement)
646 placement = buf->placement;
647
648 if (newBuffer) {
649 if (buf->bufferType == WSBM_BUFFER_REF)
650 return -EINVAL;
651
652 wsbmBufStorageUnref(&buf->storage);
653
654 if (size == 0) {
655 buf->pool = newPool;
656 buf->placement = placement;
657 retval = 0;
658 goto out;
659 }
660
661 buf->storage =
662 ttm_pool_ub_create(newPool, size, placement, buf->alignment, user_ptr, fd);
663 if (!buf->storage) {
664 retval = -ENOMEM;
665 goto out;
666 }
667
668 buf->placement = placement;
669 buf->pool = newPool;
670 } else if (wsbmAtomicRead(&storage->onList) ||
671 0 != storage->pool->syncforcpu(storage, WSBM_SYNCCPU_WRITE |
672 WSBM_SYNCCPU_DONT_BLOCK)) {
673 /*
674 * Buffer is busy. need to create a new one.
675 * Actually such case will not be encountered for current ICS implementation
676 * TODO: maybe need refine the following code when such usage case is required
677 */
678
679 struct _WsbmBufStorage *tmp_storage;
680
681 curPool = storage->pool;
682
683 tmp_storage =
684 ttm_pool_ub_create(curPool, size, placement, buf->alignment, user_ptr, fd);
685
686 if (tmp_storage) {
687 wsbmBufStorageUnref(&buf->storage);
688 buf->storage = tmp_storage;
689 buf->placement = placement;
690 } else {
691 retval = curPool->syncforcpu(storage, WSBM_SYNCCPU_WRITE);
692 if (retval)
693 goto out;
694 synced = 1;
695 }
696 } else {
697 synced = 1;
698 }
699
700 placement_diff = placement ^ buf->placement;
701
702 /*
703 * We might need to change buffer placement.
704 */
705
706 storage = buf->storage;
707 curPool = storage->pool;
708
709 if (placement_diff) {
710 assert(curPool->setStatus != NULL);
711 curPool->releasefromcpu(storage, WSBM_SYNCCPU_WRITE);
712 retval = curPool->setStatus(storage,
713 placement_diff & placement,
714 placement_diff & ~placement);
715 if (retval)
716 goto out;
717
718 buf->placement = placement;
719 }
720
721 if (!synced) {
722 retval = curPool->syncforcpu(buf->storage, WSBM_SYNCCPU_WRITE);
723 if (retval)
724 goto out;
725 synced = 1;
726 }
727
728 storage = buf->storage;
729 curPool = storage->pool;
730
731 if (data) {
732 memcpy((unsigned long *) user_ptr, data, size);
733 }
734
735 out:
736
737 if (synced)
738 curPool->releasefromcpu(storage, WSBM_SYNCCPU_WRITE);
739
740 return retval;
741 }
742
743 static struct _WsbmBufStorage *
wsbmStorageClone(struct _WsbmBufferObject * buf)744 wsbmStorageClone(struct _WsbmBufferObject *buf)
745 {
746 struct _WsbmBufStorage *storage = buf->storage;
747 struct _WsbmBufferPool *pool = storage->pool;
748
749 return pool->create(pool, pool->size(storage), buf->placement,
750 buf->alignment);
751 }
752
753 struct _WsbmBufferObject *
wsbmBOClone(struct _WsbmBufferObject * buf,int (* accelCopy)(struct _WsbmBufferObject *,struct _WsbmBufferObject *))754 wsbmBOClone(struct _WsbmBufferObject *buf,
755 int (*accelCopy) (struct _WsbmBufferObject *,
756 struct _WsbmBufferObject *))
757 {
758 struct _WsbmBufferObject *newBuf;
759 int ret;
760
761 newBuf = malloc(sizeof(*newBuf));
762 if (!newBuf)
763 return NULL;
764
765 *newBuf = *buf;
766 newBuf->storage = wsbmStorageClone(buf);
767 if (!newBuf->storage)
768 goto out_err0;
769
770 wsbmAtomicSet(&newBuf->refCount, 1);
771 if (!accelCopy || accelCopy(newBuf, buf) != 0) {
772
773 struct _WsbmBufferPool *pool = buf->storage->pool;
774 struct _WsbmBufStorage *storage = buf->storage;
775 struct _WsbmBufStorage *newStorage = newBuf->storage;
776 void *virtual;
777 void *nVirtual;
778
779 ret = pool->syncforcpu(storage, WSBM_SYNCCPU_READ);
780 if (ret)
781 goto out_err1;
782 ret = pool->map(storage, WSBM_ACCESS_READ, &virtual);
783 if (ret)
784 goto out_err2;
785 ret = pool->map(newStorage, WSBM_ACCESS_WRITE, &nVirtual);
786 if (ret)
787 goto out_err3;
788
789 memcpy(nVirtual, virtual, pool->size(storage));
790 pool->unmap(newBuf->storage);
791 pool->unmap(buf->storage);
792 pool->releasefromcpu(storage, WSBM_SYNCCPU_READ);
793 }
794
795 return newBuf;
796 out_err3:
797 buf->pool->unmap(buf->storage);
798 out_err2:
799 buf->pool->releasefromcpu(buf->storage, WSBM_SYNCCPU_READ);
800 out_err1:
801 wsbmBufStorageUnref(&newBuf->storage);
802 out_err0:
803 free(newBuf);
804 return 0;
805 }
806
807 int
wsbmBOSubData(struct _WsbmBufferObject * buf,unsigned long offset,unsigned long size,const void * data,int (* accelCopy)(struct _WsbmBufferObject *,struct _WsbmBufferObject *))808 wsbmBOSubData(struct _WsbmBufferObject *buf,
809 unsigned long offset, unsigned long size, const void *data,
810 int (*accelCopy) (struct _WsbmBufferObject *,
811 struct _WsbmBufferObject *))
812 {
813 int ret = 0;
814
815 if (buf->bufferType == WSBM_BUFFER_SIMPLE)
816 return -EINVAL;
817
818 if (size && data) {
819 void *virtual;
820 struct _WsbmBufStorage *storage = buf->storage;
821 struct _WsbmBufferPool *pool = storage->pool;
822
823 ret = pool->syncforcpu(storage, WSBM_SYNCCPU_WRITE);
824 if (ret)
825 goto out;
826
827 if (wsbmAtomicRead(&storage->onList)) {
828
829 struct _WsbmBufferObject *newBuf;
830
831 /*
832 * Another context has this buffer on its validate list.
833 * This should be a very rare situation, but it can be valid,
834 * and therefore we must deal with it by cloning the storage.
835 */
836
837 pool->releasefromcpu(storage, WSBM_SYNCCPU_WRITE);
838 newBuf = wsbmBOClone(buf, accelCopy);
839
840 /*
841 * If clone fails we have the choice of either bailing.
842 * (The other context will be happy), or go on and update
843 * the old buffer anyway. (We will be happy). We choose the
844 * latter.
845 */
846
847 if (newBuf) {
848 storage = newBuf->storage;
849 wsbmAtomicInc(&storage->refCount);
850 wsbmBufStorageUnref(&buf->storage);
851 buf->storage = storage;
852 wsbmBOUnreference(&newBuf);
853 pool = storage->pool;
854 }
855
856 ret = pool->syncforcpu(storage, WSBM_SYNCCPU_WRITE);
857 if (ret)
858 goto out;
859 }
860
861 ret = pool->map(storage, WSBM_ACCESS_WRITE, &virtual);
862 if (ret) {
863 pool->releasefromcpu(storage, WSBM_SYNCCPU_WRITE);
864 goto out;
865 }
866
867 memcpy((unsigned char *)virtual + offset, data, size);
868 pool->unmap(storage);
869 pool->releasefromcpu(storage, WSBM_SYNCCPU_WRITE);
870 }
871 out:
872 return ret;
873 }
874
875 int
wsbmBOGetSubData(struct _WsbmBufferObject * buf,unsigned long offset,unsigned long size,void * data)876 wsbmBOGetSubData(struct _WsbmBufferObject *buf,
877 unsigned long offset, unsigned long size, void *data)
878 {
879 int ret = 0;
880
881 if (size && data) {
882 void *virtual;
883 struct _WsbmBufStorage *storage = buf->storage;
884 struct _WsbmBufferPool *pool = storage->pool;
885
886 ret = pool->syncforcpu(storage, WSBM_SYNCCPU_READ);
887 if (ret)
888 goto out;
889 ret = pool->map(storage, WSBM_ACCESS_READ, &virtual);
890 if (ret) {
891 pool->releasefromcpu(storage, WSBM_SYNCCPU_WRITE);
892 goto out;
893 }
894 memcpy(data, (unsigned char *)virtual + offset, size);
895 pool->unmap(storage);
896 pool->releasefromcpu(storage, WSBM_SYNCCPU_WRITE);
897 }
898 out:
899 return ret;
900 }
901
902 int
wsbmBOSetReferenced(struct _WsbmBufferObject * buf,unsigned long handle)903 wsbmBOSetReferenced(struct _WsbmBufferObject *buf, unsigned long handle)
904 {
905 int ret = 0;
906
907 wsbmBufStorageUnref(&buf->storage);
908 if (buf->pool->createByReference == NULL) {
909 ret = -EINVAL;
910 goto out;
911 }
912 buf->storage = buf->pool->createByReference(buf->pool, handle);
913 if (!buf->storage) {
914 ret = -EINVAL;
915 goto out;
916 }
917 buf->bufferType = WSBM_BUFFER_REF;
918 out:
919 return ret;
920 }
921
922 void
wsbmBOFreeSimple(void * ptr)923 wsbmBOFreeSimple(void *ptr)
924 {
925 free(ptr);
926 }
927
928 struct _WsbmBufferObject *
wsbmBOCreateSimple(struct _WsbmBufferPool * pool,unsigned long size,uint32_t placement,unsigned alignment,size_t extra_size,size_t * offset)929 wsbmBOCreateSimple(struct _WsbmBufferPool *pool,
930 unsigned long size,
931 uint32_t placement,
932 unsigned alignment, size_t extra_size, size_t * offset)
933 {
934 struct _WsbmBufferObject *buf;
935 struct _WsbmBufStorage *storage;
936
937 *offset = (sizeof(*buf) + 15) & ~15;
938
939 if (extra_size) {
940 extra_size += *offset - sizeof(*buf);
941 }
942
943 buf = (struct _WsbmBufferObject *)calloc(1, sizeof(*buf) + extra_size);
944 if (!buf)
945 return NULL;
946
947 storage = pool->create(pool, size, placement, alignment);
948 if (!storage)
949 goto out_err0;
950
951 storage->destroyContainer = &wsbmBOFreeSimple;
952 storage->destroyArg = buf;
953
954 buf->storage = storage;
955 buf->alignment = alignment;
956 buf->pool = pool;
957 buf->placement = placement;
958 buf->bufferType = WSBM_BUFFER_SIMPLE;
959
960 return buf;
961
962 out_err0:
963 free(buf);
964 return NULL;
965 }
966
967 int
wsbmGenBuffers(struct _WsbmBufferPool * pool,unsigned n,struct _WsbmBufferObject * buffers[],unsigned alignment,uint32_t placement)968 wsbmGenBuffers(struct _WsbmBufferPool *pool,
969 unsigned n,
970 struct _WsbmBufferObject *buffers[],
971 unsigned alignment, uint32_t placement)
972 {
973 struct _WsbmBufferObject *buf;
974 unsigned int i;
975
976 placement = (placement) ? placement :
977 WSBM_PL_FLAG_SYSTEM | WSBM_PL_FLAG_CACHED;
978
979 for (i = 0; i < n; ++i) {
980 buf = (struct _WsbmBufferObject *)calloc(1, sizeof(*buf));
981 if (!buf)
982 return -ENOMEM;
983
984 wsbmAtomicSet(&buf->refCount, 1);
985 buf->placement = placement;
986 buf->alignment = alignment;
987 buf->pool = pool;
988 buf->bufferType = WSBM_BUFFER_COMPLEX;
989 buffers[i] = buf;
990 }
991 return 0;
992 }
993
994 void
wsbmDeleteBuffers(unsigned n,struct _WsbmBufferObject * buffers[])995 wsbmDeleteBuffers(unsigned n, struct _WsbmBufferObject *buffers[])
996 {
997 unsigned int i;
998
999 for (i = 0; i < n; ++i) {
1000 wsbmBOUnreference(&buffers[i]);
1001 }
1002 }
1003
1004 /*
1005 * Note that lists are per-context and don't need mutex protection.
1006 */
1007
1008 struct _WsbmBufferList *
wsbmBOCreateList(int target,int hasKernelBuffers)1009 wsbmBOCreateList(int target, int hasKernelBuffers)
1010 {
1011 struct _WsbmBufferList *list = calloc(sizeof(*list), 1);
1012 int ret;
1013
1014 if (!list)
1015 return NULL;
1016 list->hasKernelBuffers = hasKernelBuffers;
1017 if (hasKernelBuffers) {
1018 ret = validateCreateList(target, &list->kernelBuffers, 0);
1019 if (ret)
1020 return NULL;
1021 }
1022
1023 ret = validateCreateList(target, &list->userBuffers, 1);
1024 if (ret) {
1025 validateFreeList(&list->kernelBuffers);
1026 return NULL;
1027 }
1028
1029 return list;
1030 }
1031
1032 int
wsbmBOResetList(struct _WsbmBufferList * list)1033 wsbmBOResetList(struct _WsbmBufferList *list)
1034 {
1035 int ret;
1036
1037 if (list->hasKernelBuffers) {
1038 ret = validateResetList(&list->kernelBuffers);
1039 if (ret)
1040 return ret;
1041 }
1042 ret = validateResetList(&list->userBuffers);
1043 return ret;
1044 }
1045
1046 void
wsbmBOFreeList(struct _WsbmBufferList * list)1047 wsbmBOFreeList(struct _WsbmBufferList *list)
1048 {
1049 if (list->hasKernelBuffers)
1050 validateFreeList(&list->kernelBuffers);
1051 validateFreeList(&list->userBuffers);
1052 free(list);
1053 }
1054
1055 static int
wsbmAddValidateItem(struct _ValidateList * list,void * buf,uint64_t flags,uint64_t mask,int * itemLoc,struct _ValidateNode ** pnode,int * newItem)1056 wsbmAddValidateItem(struct _ValidateList *list, void *buf, uint64_t flags,
1057 uint64_t mask, int *itemLoc,
1058 struct _ValidateNode **pnode, int *newItem)
1059 {
1060 struct _ValidateNode *node, *cur;
1061 struct _WsbmListHead *l;
1062 struct _WsbmListHead *hashHead;
1063 uint32_t hash;
1064 uint32_t count = 0;
1065 uint32_t key = (unsigned long) buf;
1066
1067 cur = NULL;
1068 hash = wsbmHashFunc((uint8_t *) &key, 4, list->hashMask);
1069 hashHead = list->hashTable + hash;
1070 *newItem = 0;
1071
1072 for (l = hashHead->next; l != hashHead; l = l->next) {
1073 count++;
1074 node = WSBMLISTENTRY(l, struct _ValidateNode, hashHead);
1075
1076 if (node->buf == buf) {
1077 cur = node;
1078 break;
1079 }
1080 }
1081
1082 if (!cur) {
1083 cur = validateListAddNode(list, buf, hash, flags, mask);
1084 if (!cur)
1085 return -ENOMEM;
1086 *newItem = 1;
1087 cur->func->clear(cur);
1088 } else {
1089 uint64_t set_flags = flags & mask;
1090 uint64_t clr_flags = (~flags) & mask;
1091
1092 if (((cur->clr_flags | clr_flags) & WSBM_PL_MASK_MEM) ==
1093 WSBM_PL_MASK_MEM) {
1094 /*
1095 * No available memory type left. Bail.
1096 */
1097 return -EINVAL;
1098 }
1099
1100 if ((cur->set_flags | set_flags) &
1101 (cur->clr_flags | clr_flags) & ~WSBM_PL_MASK_MEM) {
1102 /*
1103 * Conflicting flags. Bail.
1104 */
1105 return -EINVAL;
1106 }
1107
1108 cur->set_flags &= ~(clr_flags & WSBM_PL_MASK_MEM);
1109 cur->set_flags |= (set_flags & ~WSBM_PL_MASK_MEM);
1110 cur->clr_flags |= clr_flags;
1111 }
1112 *itemLoc = cur->listItem;
1113 if (pnode)
1114 *pnode = cur;
1115 return 0;
1116 }
1117
1118 int
wsbmBOAddListItem(struct _WsbmBufferList * list,struct _WsbmBufferObject * buf,uint64_t flags,uint64_t mask,int * itemLoc,struct _ValidateNode ** node)1119 wsbmBOAddListItem(struct _WsbmBufferList *list,
1120 struct _WsbmBufferObject *buf,
1121 uint64_t flags, uint64_t mask, int *itemLoc,
1122 struct _ValidateNode **node)
1123 {
1124 int newItem;
1125 struct _WsbmBufStorage *storage = buf->storage;
1126 int ret;
1127 int dummy;
1128 struct _ValidateNode *dummyNode;
1129
1130 if (list->hasKernelBuffers) {
1131 ret = wsbmAddValidateItem(&list->kernelBuffers,
1132 storage->pool->kernel(storage),
1133 flags, mask, itemLoc, node, &dummy);
1134 if (ret)
1135 goto out_unlock;
1136 } else {
1137 *node = NULL;
1138 *itemLoc = -1000;
1139 }
1140
1141 ret = wsbmAddValidateItem(&list->userBuffers, storage,
1142 flags, mask, &dummy, &dummyNode, &newItem);
1143 if (ret)
1144 goto out_unlock;
1145
1146 if (newItem) {
1147 wsbmAtomicInc(&storage->refCount);
1148 wsbmAtomicInc(&storage->onList);
1149 }
1150
1151 out_unlock:
1152 return ret;
1153 }
1154
1155 void
wsbmBOFence(struct _WsbmBufferObject * buf,struct _WsbmFenceObject * fence)1156 wsbmBOFence(struct _WsbmBufferObject *buf, struct _WsbmFenceObject *fence)
1157 {
1158 struct _WsbmBufStorage *storage;
1159
1160 storage = buf->storage;
1161 if (storage->pool->fence)
1162 storage->pool->fence(storage, fence);
1163
1164 }
1165
1166 int
wsbmBOOnList(const struct _WsbmBufferObject * buf)1167 wsbmBOOnList(const struct _WsbmBufferObject *buf)
1168 {
1169 if (buf->storage == NULL)
1170 return 0;
1171 return wsbmAtomicRead(&buf->storage->onList);
1172 }
1173
1174 int
wsbmBOUnrefUserList(struct _WsbmBufferList * list)1175 wsbmBOUnrefUserList(struct _WsbmBufferList *list)
1176 {
1177 struct _WsbmBufStorage *storage;
1178 void *curBuf;
1179
1180 curBuf = validateListIterator(&list->userBuffers);
1181
1182 while (curBuf) {
1183 storage = (struct _WsbmBufStorage *)(validateListNode(curBuf)->buf);
1184 wsbmAtomicDec(&storage->onList);
1185 wsbmBufStorageUnref(&storage);
1186 curBuf = validateListNext(&list->userBuffers, curBuf);
1187 }
1188
1189 return wsbmBOResetList(list);
1190 }
1191
1192
1193 int
wsbmBOFenceUserList(struct _WsbmBufferList * list,struct _WsbmFenceObject * fence)1194 wsbmBOFenceUserList(struct _WsbmBufferList *list,
1195 struct _WsbmFenceObject *fence)
1196 {
1197 struct _WsbmBufStorage *storage;
1198 void *curBuf;
1199
1200 curBuf = validateListIterator(&list->userBuffers);
1201
1202 /*
1203 * User-space fencing callbacks.
1204 */
1205
1206 while (curBuf) {
1207 storage = (struct _WsbmBufStorage *)(validateListNode(curBuf)->buf);
1208
1209 storage->pool->fence(storage, fence);
1210 wsbmAtomicDec(&storage->onList);
1211 wsbmBufStorageUnref(&storage);
1212 curBuf = validateListNext(&list->userBuffers, curBuf);
1213 }
1214
1215 return wsbmBOResetList(list);
1216 }
1217
1218 int
wsbmBOValidateUserList(struct _WsbmBufferList * list)1219 wsbmBOValidateUserList(struct _WsbmBufferList *list)
1220 {
1221 void *curBuf;
1222 struct _WsbmBufStorage *storage;
1223 struct _ValidateNode *node;
1224 int ret;
1225
1226 curBuf = validateListIterator(&list->userBuffers);
1227
1228 /*
1229 * User-space validation callbacks.
1230 */
1231
1232 while (curBuf) {
1233 node = validateListNode(curBuf);
1234 storage = (struct _WsbmBufStorage *)node->buf;
1235 if (storage->pool->validate) {
1236 ret = storage->pool->validate(storage, node->set_flags,
1237 node->clr_flags);
1238 if (ret)
1239 return ret;
1240 }
1241 curBuf = validateListNext(&list->userBuffers, curBuf);
1242 }
1243 return 0;
1244 }
1245
1246 int
wsbmBOUnvalidateUserList(struct _WsbmBufferList * list)1247 wsbmBOUnvalidateUserList(struct _WsbmBufferList *list)
1248 {
1249 void *curBuf;
1250 struct _WsbmBufStorage *storage;
1251 struct _ValidateNode *node;
1252
1253 curBuf = validateListIterator(&list->userBuffers);
1254
1255 /*
1256 * User-space validation callbacks.
1257 */
1258
1259 while (curBuf) {
1260 node = validateListNode(curBuf);
1261 storage = (struct _WsbmBufStorage *)node->buf;
1262 if (storage->pool->unvalidate) {
1263 storage->pool->unvalidate(storage);
1264 }
1265 wsbmAtomicDec(&storage->onList);
1266 wsbmBufStorageUnref(&storage);
1267 curBuf = validateListNext(&list->userBuffers, curBuf);
1268 }
1269 return wsbmBOResetList(list);
1270 }
1271
1272 void
wsbmPoolTakeDown(struct _WsbmBufferPool * pool)1273 wsbmPoolTakeDown(struct _WsbmBufferPool *pool)
1274 {
1275 pool->takeDown(pool);
1276
1277 }
1278
1279 unsigned long
wsbmBOSize(struct _WsbmBufferObject * buf)1280 wsbmBOSize(struct _WsbmBufferObject *buf)
1281 {
1282 unsigned long size;
1283 struct _WsbmBufStorage *storage;
1284
1285 storage = buf->storage;
1286 size = storage->pool->size(storage);
1287
1288 return size;
1289
1290 }
1291
1292 struct _ValidateList *
wsbmGetKernelValidateList(struct _WsbmBufferList * list)1293 wsbmGetKernelValidateList(struct _WsbmBufferList *list)
1294 {
1295 return (list->hasKernelBuffers) ? &list->kernelBuffers : NULL;
1296 }
1297
1298 struct _ValidateList *
wsbmGetUserValidateList(struct _WsbmBufferList * list)1299 wsbmGetUserValidateList(struct _WsbmBufferList *list)
1300 {
1301 return &list->userBuffers;
1302 }
1303
1304 struct _ValidateNode *
validateListNode(void * iterator)1305 validateListNode(void *iterator)
1306 {
1307 struct _WsbmListHead *l = (struct _WsbmListHead *)iterator;
1308
1309 return WSBMLISTENTRY(l, struct _ValidateNode, head);
1310 }
1311
1312 void *
validateListIterator(struct _ValidateList * list)1313 validateListIterator(struct _ValidateList *list)
1314 {
1315 void *ret = list->list.next;
1316
1317 if (ret == &list->list)
1318 return NULL;
1319 return ret;
1320 }
1321
1322 void *
validateListNext(struct _ValidateList * list,void * iterator)1323 validateListNext(struct _ValidateList *list, void *iterator)
1324 {
1325 void *ret;
1326
1327 struct _WsbmListHead *l = (struct _WsbmListHead *)iterator;
1328
1329 ret = l->next;
1330 if (ret == &list->list)
1331 return NULL;
1332 return ret;
1333 }
1334
1335 uint32_t
wsbmKBufHandle(const struct _WsbmKernelBuf * kBuf)1336 wsbmKBufHandle(const struct _WsbmKernelBuf * kBuf)
1337 {
1338 return kBuf->handle;
1339 }
1340
1341 extern void
wsbmUpdateKBuf(struct _WsbmKernelBuf * kBuf,uint64_t gpuOffset,uint32_t placement,uint32_t fence_type_mask)1342 wsbmUpdateKBuf(struct _WsbmKernelBuf *kBuf,
1343 uint64_t gpuOffset, uint32_t placement,
1344 uint32_t fence_type_mask)
1345 {
1346 kBuf->gpuOffset = gpuOffset;
1347 kBuf->placement = placement;
1348 kBuf->fence_type_mask = fence_type_mask;
1349 }
1350
1351 extern struct _WsbmKernelBuf *
wsbmKBuf(const struct _WsbmBufferObject * buf)1352 wsbmKBuf(const struct _WsbmBufferObject *buf)
1353 {
1354 struct _WsbmBufStorage *storage = buf->storage;
1355
1356 return storage->pool->kernel(storage);
1357 }
1358