• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 /**************************************************************************
2  *
3  * Copyright 2006 Tungsten Graphics, Inc., Cedar Park, Texas.
4  * All Rights Reserved.
5  *
6  * Permission is hereby granted, free of charge, to any person obtaining a
7  * copy of this software and associated documentation files (the
8  * "Software"), to deal in the Software without restriction, including
9  * without limitation the rights to use, copy, modify, merge, publish,
10  * distribute, sub license, and/or sell copies of the Software, and to
11  * permit persons to whom the Software is furnished to do so, subject to
12  * the following conditions:
13  *
14  * The above copyright notice and this permission notice (including the
15  * next paragraph) shall be included in all copies or substantial portions
16  * of the Software.
17  *
18  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
19  * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
20  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT.
21  * IN NO EVENT SHALL TUNGSTEN GRAPHICS AND/OR ITS SUPPLIERS BE LIABLE FOR
22  * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
23  * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
24  * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
25  *
26  **************************************************************************/
27 
28 /* Originally a fake version of the buffer manager so that we can
29  * prototype the changes in a driver fairly quickly, has been fleshed
30  * out to a fully functional interim solution.
31  *
32  * Basically wraps the old style memory management in the new
33  * programming interface, but is more expressive and avoids many of
34  * the bugs in the old texture manager.
35  */
36 
37 #include <stdlib.h>
38 #include <string.h>
39 #include <assert.h>
40 #include <errno.h>
41 #include <strings.h>
42 #include <xf86drm.h>
43 #include <pthread.h>
44 #include "intel_bufmgr.h"
45 #include "intel_bufmgr_priv.h"
46 #include "drm.h"
47 #include "i915_drm.h"
48 #include "mm.h"
49 #include "libdrm_macros.h"
50 #include "libdrm_lists.h"
51 
52 #define DBG(...) do {					\
53 	if (bufmgr_fake->bufmgr.debug)			\
54 		drmMsg(__VA_ARGS__);			\
55 } while (0)
56 
57 /* Internal flags:
58  */
59 #define BM_NO_BACKING_STORE			0x00000001
60 #define BM_NO_FENCE_SUBDATA			0x00000002
61 #define BM_PINNED				0x00000004
62 
63 /* Wrapper around mm.c's mem_block, which understands that you must
64  * wait for fences to expire before memory can be freed.  This is
65  * specific to our use of memcpy for uploads - an upload that was
66  * processed through the command queue wouldn't need to care about
67  * fences.
68  */
69 #define MAX_RELOCS 4096
70 
71 struct fake_buffer_reloc {
72 	/** Buffer object that the relocation points at. */
73 	drm_intel_bo *target_buf;
74 	/** Offset of the relocation entry within reloc_buf. */
75 	uint32_t offset;
76 	/**
77 	 * Cached value of the offset when we last performed this relocation.
78 	 */
79 	uint32_t last_target_offset;
80 	/** Value added to target_buf's offset to get the relocation entry. */
81 	uint32_t delta;
82 	/** Cache domains the target buffer is read into. */
83 	uint32_t read_domains;
84 	/** Cache domain the target buffer will have dirty cachelines in. */
85 	uint32_t write_domain;
86 };
87 
88 struct block {
89 	struct block *next, *prev;
90 	struct mem_block *mem;	/* BM_MEM_AGP */
91 
92 	/**
93 	 * Marks that the block is currently in the aperture and has yet to be
94 	 * fenced.
95 	 */
96 	unsigned on_hardware:1;
97 	/**
98 	 * Marks that the block is currently fenced (being used by rendering)
99 	 * and can't be freed until @fence is passed.
100 	 */
101 	unsigned fenced:1;
102 
103 	/** Fence cookie for the block. */
104 	unsigned fence;		/* Split to read_fence, write_fence */
105 
106 	drm_intel_bo *bo;
107 	void *virtual;
108 };
109 
110 typedef struct _bufmgr_fake {
111 	drm_intel_bufmgr bufmgr;
112 
113 	pthread_mutex_t lock;
114 
115 	unsigned long low_offset;
116 	unsigned long size;
117 	void *virtual;
118 
119 	struct mem_block *heap;
120 
121 	unsigned buf_nr;	/* for generating ids */
122 
123 	/**
124 	 * List of blocks which are currently in the GART but haven't been
125 	 * fenced yet.
126 	 */
127 	struct block on_hardware;
128 	/**
129 	 * List of blocks which are in the GART and have an active fence on
130 	 * them.
131 	 */
132 	struct block fenced;
133 	/**
134 	 * List of blocks which have an expired fence and are ready to be
135 	 * evicted.
136 	 */
137 	struct block lru;
138 
139 	unsigned int last_fence;
140 
141 	unsigned fail:1;
142 	unsigned need_fence:1;
143 	int thrashing;
144 
145 	/**
146 	 * Driver callback to emit a fence, returning the cookie.
147 	 *
148 	 * This allows the driver to hook in a replacement for the DRM usage in
149 	 * bufmgr_fake.
150 	 *
151 	 * Currently, this also requires that a write flush be emitted before
152 	 * emitting the fence, but this should change.
153 	 */
154 	unsigned int (*fence_emit) (void *private);
155 	/** Driver callback to wait for a fence cookie to have passed. */
156 	void (*fence_wait) (unsigned int fence, void *private);
157 	void *fence_priv;
158 
159 	/**
160 	 * Driver callback to execute a buffer.
161 	 *
162 	 * This allows the driver to hook in a replacement for the DRM usage in
163 	 * bufmgr_fake.
164 	 */
165 	int (*exec) (drm_intel_bo *bo, unsigned int used, void *priv);
166 	void *exec_priv;
167 
168 	/** Driver-supplied argument to driver callbacks */
169 	void *driver_priv;
170 	/**
171 	 * Pointer to kernel-updated sarea data for the last completed user irq
172 	 */
173 	volatile int *last_dispatch;
174 
175 	int fd;
176 
177 	int debug;
178 
179 	int performed_rendering;
180 } drm_intel_bufmgr_fake;
181 
182 typedef struct _drm_intel_bo_fake {
183 	drm_intel_bo bo;
184 
185 	unsigned id;		/* debug only */
186 	const char *name;
187 
188 	unsigned dirty:1;
189 	/**
190 	 * has the card written to this buffer - we make need to copy it back
191 	 */
192 	unsigned card_dirty:1;
193 	unsigned int refcount;
194 	/* Flags may consist of any of the DRM_BO flags, plus
195 	 * DRM_BO_NO_BACKING_STORE and BM_NO_FENCE_SUBDATA, which are the
196 	 * first two driver private flags.
197 	 */
198 	uint64_t flags;
199 	/** Cache domains the target buffer is read into. */
200 	uint32_t read_domains;
201 	/** Cache domain the target buffer will have dirty cachelines in. */
202 	uint32_t write_domain;
203 
204 	unsigned int alignment;
205 	int is_static, validated;
206 	unsigned int map_count;
207 
208 	/** relocation list */
209 	struct fake_buffer_reloc *relocs;
210 	int nr_relocs;
211 	/**
212 	 * Total size of the target_bos of this buffer.
213 	 *
214 	 * Used for estimation in check_aperture.
215 	 */
216 	unsigned int child_size;
217 
218 	struct block *block;
219 	void *backing_store;
220 	void (*invalidate_cb) (drm_intel_bo *bo, void *ptr);
221 	void *invalidate_ptr;
222 } drm_intel_bo_fake;
223 
224 static int clear_fenced(drm_intel_bufmgr_fake *bufmgr_fake,
225 			unsigned int fence_cookie);
226 
227 #define MAXFENCE 0x7fffffff
228 
229 static int
FENCE_LTE(unsigned a,unsigned b)230 FENCE_LTE(unsigned a, unsigned b)
231 {
232 	if (a == b)
233 		return 1;
234 
235 	if (a < b && b - a < (1 << 24))
236 		return 1;
237 
238 	if (a > b && MAXFENCE - a + b < (1 << 24))
239 		return 1;
240 
241 	return 0;
242 }
243 
244 void
drm_intel_bufmgr_fake_set_fence_callback(drm_intel_bufmgr * bufmgr,unsigned int (* emit)(void * priv),void (* wait)(unsigned int fence,void * priv),void * priv)245 drm_intel_bufmgr_fake_set_fence_callback(drm_intel_bufmgr *bufmgr,
246 					 unsigned int (*emit) (void *priv),
247 					 void (*wait) (unsigned int fence,
248 						       void *priv),
249 					 void *priv)
250 {
251 	drm_intel_bufmgr_fake *bufmgr_fake = (drm_intel_bufmgr_fake *) bufmgr;
252 
253 	bufmgr_fake->fence_emit = emit;
254 	bufmgr_fake->fence_wait = wait;
255 	bufmgr_fake->fence_priv = priv;
256 }
257 
258 static unsigned int
_fence_emit_internal(drm_intel_bufmgr_fake * bufmgr_fake)259 _fence_emit_internal(drm_intel_bufmgr_fake *bufmgr_fake)
260 {
261 	struct drm_i915_irq_emit ie;
262 	int ret, seq = 1;
263 
264 	if (bufmgr_fake->fence_emit != NULL) {
265 		seq = bufmgr_fake->fence_emit(bufmgr_fake->fence_priv);
266 		return seq;
267 	}
268 
269 	ie.irq_seq = &seq;
270 	ret = drmCommandWriteRead(bufmgr_fake->fd, DRM_I915_IRQ_EMIT,
271 				  &ie, sizeof(ie));
272 	if (ret) {
273 		drmMsg("%s: drm_i915_irq_emit: %d\n", __func__, ret);
274 		abort();
275 	}
276 
277 	DBG("emit 0x%08x\n", seq);
278 	return seq;
279 }
280 
281 static void
_fence_wait_internal(drm_intel_bufmgr_fake * bufmgr_fake,int seq)282 _fence_wait_internal(drm_intel_bufmgr_fake *bufmgr_fake, int seq)
283 {
284 	struct drm_i915_irq_wait iw;
285 	int hw_seq, busy_count = 0;
286 	int ret;
287 	int kernel_lied;
288 
289 	if (bufmgr_fake->fence_wait != NULL) {
290 		bufmgr_fake->fence_wait(seq, bufmgr_fake->fence_priv);
291 		clear_fenced(bufmgr_fake, seq);
292 		return;
293 	}
294 
295 	iw.irq_seq = seq;
296 
297 	DBG("wait 0x%08x\n", iw.irq_seq);
298 
299 	/* The kernel IRQ_WAIT implementation is all sorts of broken.
300 	 * 1) It returns 1 to 0x7fffffff instead of using the full 32-bit
301 	 *    unsigned range.
302 	 * 2) It returns 0 if hw_seq >= seq, not seq - hw_seq < 0 on the 32-bit
303 	 *    signed range.
304 	 * 3) It waits if seq < hw_seq, not seq - hw_seq > 0 on the 32-bit
305 	 *    signed range.
306 	 * 4) It returns -EBUSY in 3 seconds even if the hardware is still
307 	 *    successfully chewing through buffers.
308 	 *
309 	 * Assume that in userland we treat sequence numbers as ints, which
310 	 * makes some of the comparisons convenient, since the sequence
311 	 * numbers are all positive signed integers.
312 	 *
313 	 * From this we get several cases we need to handle.  Here's a timeline.
314 	 * 0x2   0x7                                    0x7ffffff8   0x7ffffffd
315 	 *   |    |                                             |    |
316 	 * ------------------------------------------------------------
317 	 *
318 	 * A) Normal wait for hw to catch up
319 	 * hw_seq seq
320 	 *   |    |
321 	 * ------------------------------------------------------------
322 	 * seq - hw_seq = 5.  If we call IRQ_WAIT, it will wait for hw to
323 	 * catch up.
324 	 *
325 	 * B) Normal wait for a sequence number that's already passed.
326 	 * seq    hw_seq
327 	 *   |    |
328 	 * ------------------------------------------------------------
329 	 * seq - hw_seq = -5.  If we call IRQ_WAIT, it returns 0 quickly.
330 	 *
331 	 * C) Hardware has already wrapped around ahead of us
332 	 * hw_seq                                                    seq
333 	 *   |                                                       |
334 	 * ------------------------------------------------------------
335 	 * seq - hw_seq = 0x80000000 - 5.  If we called IRQ_WAIT, it would wait
336 	 * for hw_seq >= seq, which may never occur.  Thus, we want to catch
337 	 * this in userland and return 0.
338 	 *
339 	 * D) We've wrapped around ahead of the hardware.
340 	 * seq                                                      hw_seq
341 	 *   |                                                       |
342 	 * ------------------------------------------------------------
343 	 * seq - hw_seq = -(0x80000000 - 5).  If we called IRQ_WAIT, it would
344 	 * return 0 quickly because hw_seq >= seq, even though the hardware
345 	 * isn't caught up. Thus, we need to catch this early return in
346 	 * userland and bother the kernel until the hardware really does
347 	 * catch up.
348 	 *
349 	 * E) Hardware might wrap after we test in userland.
350 	 *                                                  hw_seq  seq
351 	 *                                                      |    |
352 	 * ------------------------------------------------------------
353 	 * seq - hw_seq = 5.  If we call IRQ_WAIT, it will likely see seq >=
354 	 * hw_seq and wait.  However, suppose hw_seq wraps before we make it
355 	 * into the kernel.  The kernel sees hw_seq >= seq and waits for 3
356 	 * seconds then returns -EBUSY.  This is case C).  We should catch
357 	 * this and then return successfully.
358 	 *
359 	 * F) Hardware might take a long time on a buffer.
360 	 * hw_seq seq
361 	 *   |    |
362 	 * -------------------------------------------------------------------
363 	 * seq - hw_seq = 5.  If we call IRQ_WAIT, if sequence 2 through 5
364 	 * take too long, it will return -EBUSY.  Batchbuffers in the
365 	 * gltestperf demo were seen to take up to 7 seconds.  We should
366 	 * catch early -EBUSY return and keep trying.
367 	 */
368 
369 	do {
370 		/* Keep a copy of last_dispatch so that if the wait -EBUSYs
371 		 * because the hardware didn't catch up in 3 seconds, we can
372 		 * see if it at least made progress and retry.
373 		 */
374 		hw_seq = *bufmgr_fake->last_dispatch;
375 
376 		/* Catch case C */
377 		if (seq - hw_seq > 0x40000000)
378 			return;
379 
380 		ret = drmCommandWrite(bufmgr_fake->fd, DRM_I915_IRQ_WAIT,
381 				      &iw, sizeof(iw));
382 		/* Catch case D */
383 		kernel_lied = (ret == 0) && (seq - *bufmgr_fake->last_dispatch <
384 					     -0x40000000);
385 
386 		/* Catch case E */
387 		if (ret == -EBUSY
388 		    && (seq - *bufmgr_fake->last_dispatch > 0x40000000))
389 			ret = 0;
390 
391 		/* Catch case F: Allow up to 15 seconds chewing on one buffer. */
392 		if ((ret == -EBUSY) && (hw_seq != *bufmgr_fake->last_dispatch))
393 			busy_count = 0;
394 		else
395 			busy_count++;
396 	} while (kernel_lied || ret == -EAGAIN || ret == -EINTR ||
397 		 (ret == -EBUSY && busy_count < 5));
398 
399 	if (ret != 0) {
400 		drmMsg("%s:%d: Error waiting for fence: %s.\n", __FILE__,
401 		       __LINE__, strerror(-ret));
402 		abort();
403 	}
404 	clear_fenced(bufmgr_fake, seq);
405 }
406 
407 static int
_fence_test(drm_intel_bufmgr_fake * bufmgr_fake,unsigned fence)408 _fence_test(drm_intel_bufmgr_fake *bufmgr_fake, unsigned fence)
409 {
410 	/* Slight problem with wrap-around:
411 	 */
412 	return fence == 0 || FENCE_LTE(fence, bufmgr_fake->last_fence);
413 }
414 
415 /**
416  * Allocate a memory manager block for the buffer.
417  */
418 static int
alloc_block(drm_intel_bo * bo)419 alloc_block(drm_intel_bo *bo)
420 {
421 	drm_intel_bo_fake *bo_fake = (drm_intel_bo_fake *) bo;
422 	drm_intel_bufmgr_fake *bufmgr_fake =
423 	    (drm_intel_bufmgr_fake *) bo->bufmgr;
424 	struct block *block = (struct block *)calloc(sizeof *block, 1);
425 	unsigned int align_log2 = ffs(bo_fake->alignment) - 1;
426 	unsigned int sz;
427 
428 	if (!block)
429 		return 1;
430 
431 	sz = (bo->size + bo_fake->alignment - 1) & ~(bo_fake->alignment - 1);
432 
433 	block->mem = mmAllocMem(bufmgr_fake->heap, sz, align_log2, 0);
434 	if (!block->mem) {
435 		free(block);
436 		return 0;
437 	}
438 
439 	DRMINITLISTHEAD(block);
440 
441 	/* Insert at head or at tail??? */
442 	DRMLISTADDTAIL(block, &bufmgr_fake->lru);
443 
444 	block->virtual = (uint8_t *) bufmgr_fake->virtual +
445 	    block->mem->ofs - bufmgr_fake->low_offset;
446 	block->bo = bo;
447 
448 	bo_fake->block = block;
449 
450 	return 1;
451 }
452 
453 /* Release the card storage associated with buf:
454  */
455 static void
free_block(drm_intel_bufmgr_fake * bufmgr_fake,struct block * block,int skip_dirty_copy)456 free_block(drm_intel_bufmgr_fake *bufmgr_fake, struct block *block,
457 	   int skip_dirty_copy)
458 {
459 	drm_intel_bo_fake *bo_fake;
460 	DBG("free block %p %08x %d %d\n", block, block->mem->ofs,
461 	    block->on_hardware, block->fenced);
462 
463 	if (!block)
464 		return;
465 
466 	bo_fake = (drm_intel_bo_fake *) block->bo;
467 
468 	if (bo_fake->flags & (BM_PINNED | BM_NO_BACKING_STORE))
469 		skip_dirty_copy = 1;
470 
471 	if (!skip_dirty_copy && (bo_fake->card_dirty == 1)) {
472 		memcpy(bo_fake->backing_store, block->virtual, block->bo->size);
473 		bo_fake->card_dirty = 0;
474 		bo_fake->dirty = 1;
475 	}
476 
477 	if (block->on_hardware) {
478 		block->bo = NULL;
479 	} else if (block->fenced) {
480 		block->bo = NULL;
481 	} else {
482 		DBG("    - free immediately\n");
483 		DRMLISTDEL(block);
484 
485 		mmFreeMem(block->mem);
486 		free(block);
487 	}
488 }
489 
490 static void
alloc_backing_store(drm_intel_bo * bo)491 alloc_backing_store(drm_intel_bo *bo)
492 {
493 	drm_intel_bufmgr_fake *bufmgr_fake =
494 	    (drm_intel_bufmgr_fake *) bo->bufmgr;
495 	drm_intel_bo_fake *bo_fake = (drm_intel_bo_fake *) bo;
496 	assert(!bo_fake->backing_store);
497 	assert(!(bo_fake->flags & (BM_PINNED | BM_NO_BACKING_STORE)));
498 
499 	bo_fake->backing_store = malloc(bo->size);
500 
501 	DBG("alloc_backing - buf %d %p %lu\n", bo_fake->id,
502 	    bo_fake->backing_store, bo->size);
503 	assert(bo_fake->backing_store);
504 }
505 
506 static void
free_backing_store(drm_intel_bo * bo)507 free_backing_store(drm_intel_bo *bo)
508 {
509 	drm_intel_bo_fake *bo_fake = (drm_intel_bo_fake *) bo;
510 
511 	if (bo_fake->backing_store) {
512 		assert(!(bo_fake->flags & (BM_PINNED | BM_NO_BACKING_STORE)));
513 		free(bo_fake->backing_store);
514 		bo_fake->backing_store = NULL;
515 	}
516 }
517 
518 static void
set_dirty(drm_intel_bo * bo)519 set_dirty(drm_intel_bo *bo)
520 {
521 	drm_intel_bufmgr_fake *bufmgr_fake =
522 	    (drm_intel_bufmgr_fake *) bo->bufmgr;
523 	drm_intel_bo_fake *bo_fake = (drm_intel_bo_fake *) bo;
524 
525 	if (bo_fake->flags & BM_NO_BACKING_STORE
526 	    && bo_fake->invalidate_cb != NULL)
527 		bo_fake->invalidate_cb(bo, bo_fake->invalidate_ptr);
528 
529 	assert(!(bo_fake->flags & BM_PINNED));
530 
531 	DBG("set_dirty - buf %d\n", bo_fake->id);
532 	bo_fake->dirty = 1;
533 }
534 
535 static int
evict_lru(drm_intel_bufmgr_fake * bufmgr_fake,unsigned int max_fence)536 evict_lru(drm_intel_bufmgr_fake *bufmgr_fake, unsigned int max_fence)
537 {
538 	struct block *block, *tmp;
539 
540 	DBG("%s\n", __func__);
541 
542 	DRMLISTFOREACHSAFE(block, tmp, &bufmgr_fake->lru) {
543 		drm_intel_bo_fake *bo_fake = (drm_intel_bo_fake *) block->bo;
544 
545 		if (bo_fake != NULL && (bo_fake->flags & BM_NO_FENCE_SUBDATA))
546 			continue;
547 
548 		if (block->fence && max_fence && !FENCE_LTE(block->fence,
549 							    max_fence))
550 			return 0;
551 
552 		set_dirty(&bo_fake->bo);
553 		bo_fake->block = NULL;
554 
555 		free_block(bufmgr_fake, block, 0);
556 		return 1;
557 	}
558 
559 	return 0;
560 }
561 
562 static int
evict_mru(drm_intel_bufmgr_fake * bufmgr_fake)563 evict_mru(drm_intel_bufmgr_fake *bufmgr_fake)
564 {
565 	struct block *block, *tmp;
566 
567 	DBG("%s\n", __func__);
568 
569 	DRMLISTFOREACHSAFEREVERSE(block, tmp, &bufmgr_fake->lru) {
570 		drm_intel_bo_fake *bo_fake = (drm_intel_bo_fake *) block->bo;
571 
572 		if (bo_fake && (bo_fake->flags & BM_NO_FENCE_SUBDATA))
573 			continue;
574 
575 		set_dirty(&bo_fake->bo);
576 		bo_fake->block = NULL;
577 
578 		free_block(bufmgr_fake, block, 0);
579 		return 1;
580 	}
581 
582 	return 0;
583 }
584 
585 /**
586  * Removes all objects from the fenced list older than the given fence.
587  */
588 static int
clear_fenced(drm_intel_bufmgr_fake * bufmgr_fake,unsigned int fence_cookie)589 clear_fenced(drm_intel_bufmgr_fake *bufmgr_fake, unsigned int fence_cookie)
590 {
591 	struct block *block, *tmp;
592 	int ret = 0;
593 
594 	bufmgr_fake->last_fence = fence_cookie;
595 	DRMLISTFOREACHSAFE(block, tmp, &bufmgr_fake->fenced) {
596 		assert(block->fenced);
597 
598 		if (_fence_test(bufmgr_fake, block->fence)) {
599 
600 			block->fenced = 0;
601 
602 			if (!block->bo) {
603 				DBG("delayed free: offset %x sz %x\n",
604 				    block->mem->ofs, block->mem->size);
605 				DRMLISTDEL(block);
606 				mmFreeMem(block->mem);
607 				free(block);
608 			} else {
609 				DBG("return to lru: offset %x sz %x\n",
610 				    block->mem->ofs, block->mem->size);
611 				DRMLISTDEL(block);
612 				DRMLISTADDTAIL(block, &bufmgr_fake->lru);
613 			}
614 
615 			ret = 1;
616 		} else {
617 			/* Blocks are ordered by fence, so if one fails, all
618 			 * from here will fail also:
619 			 */
620 			DBG("fence not passed: offset %x sz %x %d %d \n",
621 			    block->mem->ofs, block->mem->size, block->fence,
622 			    bufmgr_fake->last_fence);
623 			break;
624 		}
625 	}
626 
627 	DBG("%s: %d\n", __func__, ret);
628 	return ret;
629 }
630 
631 static void
fence_blocks(drm_intel_bufmgr_fake * bufmgr_fake,unsigned fence)632 fence_blocks(drm_intel_bufmgr_fake *bufmgr_fake, unsigned fence)
633 {
634 	struct block *block, *tmp;
635 
636 	DRMLISTFOREACHSAFE(block, tmp, &bufmgr_fake->on_hardware) {
637 		DBG("Fence block %p (sz 0x%x ofs %x buf %p) with fence %d\n",
638 		    block, block->mem->size, block->mem->ofs, block->bo, fence);
639 		block->fence = fence;
640 
641 		block->on_hardware = 0;
642 		block->fenced = 1;
643 
644 		/* Move to tail of pending list here
645 		 */
646 		DRMLISTDEL(block);
647 		DRMLISTADDTAIL(block, &bufmgr_fake->fenced);
648 	}
649 
650 	assert(DRMLISTEMPTY(&bufmgr_fake->on_hardware));
651 }
652 
653 static int
evict_and_alloc_block(drm_intel_bo * bo)654 evict_and_alloc_block(drm_intel_bo *bo)
655 {
656 	drm_intel_bufmgr_fake *bufmgr_fake =
657 	    (drm_intel_bufmgr_fake *) bo->bufmgr;
658 	drm_intel_bo_fake *bo_fake = (drm_intel_bo_fake *) bo;
659 
660 	assert(bo_fake->block == NULL);
661 
662 	/* Search for already free memory:
663 	 */
664 	if (alloc_block(bo))
665 		return 1;
666 
667 	/* If we're not thrashing, allow lru eviction to dig deeper into
668 	 * recently used textures.  We'll probably be thrashing soon:
669 	 */
670 	if (!bufmgr_fake->thrashing) {
671 		while (evict_lru(bufmgr_fake, 0))
672 			if (alloc_block(bo))
673 				return 1;
674 	}
675 
676 	/* Keep thrashing counter alive?
677 	 */
678 	if (bufmgr_fake->thrashing)
679 		bufmgr_fake->thrashing = 20;
680 
681 	/* Wait on any already pending fences - here we are waiting for any
682 	 * freed memory that has been submitted to hardware and fenced to
683 	 * become available:
684 	 */
685 	while (!DRMLISTEMPTY(&bufmgr_fake->fenced)) {
686 		uint32_t fence = bufmgr_fake->fenced.next->fence;
687 		_fence_wait_internal(bufmgr_fake, fence);
688 
689 		if (alloc_block(bo))
690 			return 1;
691 	}
692 
693 	if (!DRMLISTEMPTY(&bufmgr_fake->on_hardware)) {
694 		while (!DRMLISTEMPTY(&bufmgr_fake->fenced)) {
695 			uint32_t fence = bufmgr_fake->fenced.next->fence;
696 			_fence_wait_internal(bufmgr_fake, fence);
697 		}
698 
699 		if (!bufmgr_fake->thrashing) {
700 			DBG("thrashing\n");
701 		}
702 		bufmgr_fake->thrashing = 20;
703 
704 		if (alloc_block(bo))
705 			return 1;
706 	}
707 
708 	while (evict_mru(bufmgr_fake))
709 		if (alloc_block(bo))
710 			return 1;
711 
712 	DBG("%s 0x%lx bytes failed\n", __func__, bo->size);
713 
714 	return 0;
715 }
716 
717 /***********************************************************************
718  * Public functions
719  */
720 
721 /**
722  * Wait for hardware idle by emitting a fence and waiting for it.
723  */
724 static void
drm_intel_bufmgr_fake_wait_idle(drm_intel_bufmgr_fake * bufmgr_fake)725 drm_intel_bufmgr_fake_wait_idle(drm_intel_bufmgr_fake *bufmgr_fake)
726 {
727 	unsigned int cookie;
728 
729 	cookie = _fence_emit_internal(bufmgr_fake);
730 	_fence_wait_internal(bufmgr_fake, cookie);
731 }
732 
733 /**
734  * Wait for rendering to a buffer to complete.
735  *
736  * It is assumed that the batchbuffer which performed the rendering included
737  * the necessary flushing.
738  */
739 static void
drm_intel_fake_bo_wait_rendering_locked(drm_intel_bo * bo)740 drm_intel_fake_bo_wait_rendering_locked(drm_intel_bo *bo)
741 {
742 	drm_intel_bufmgr_fake *bufmgr_fake =
743 	    (drm_intel_bufmgr_fake *) bo->bufmgr;
744 	drm_intel_bo_fake *bo_fake = (drm_intel_bo_fake *) bo;
745 
746 	if (bo_fake->block == NULL || !bo_fake->block->fenced)
747 		return;
748 
749 	_fence_wait_internal(bufmgr_fake, bo_fake->block->fence);
750 }
751 
752 static void
drm_intel_fake_bo_wait_rendering(drm_intel_bo * bo)753 drm_intel_fake_bo_wait_rendering(drm_intel_bo *bo)
754 {
755 	drm_intel_bufmgr_fake *bufmgr_fake =
756 	    (drm_intel_bufmgr_fake *) bo->bufmgr;
757 
758 	pthread_mutex_lock(&bufmgr_fake->lock);
759 	drm_intel_fake_bo_wait_rendering_locked(bo);
760 	pthread_mutex_unlock(&bufmgr_fake->lock);
761 }
762 
763 /* Specifically ignore texture memory sharing.
764  *  -- just evict everything
765  *  -- and wait for idle
766  */
767 void
drm_intel_bufmgr_fake_contended_lock_take(drm_intel_bufmgr * bufmgr)768 drm_intel_bufmgr_fake_contended_lock_take(drm_intel_bufmgr *bufmgr)
769 {
770 	drm_intel_bufmgr_fake *bufmgr_fake = (drm_intel_bufmgr_fake *) bufmgr;
771 	struct block *block, *tmp;
772 
773 	pthread_mutex_lock(&bufmgr_fake->lock);
774 
775 	bufmgr_fake->need_fence = 1;
776 	bufmgr_fake->fail = 0;
777 
778 	/* Wait for hardware idle.  We don't know where acceleration has been
779 	 * happening, so we'll need to wait anyway before letting anything get
780 	 * put on the card again.
781 	 */
782 	drm_intel_bufmgr_fake_wait_idle(bufmgr_fake);
783 
784 	/* Check that we hadn't released the lock without having fenced the last
785 	 * set of buffers.
786 	 */
787 	assert(DRMLISTEMPTY(&bufmgr_fake->fenced));
788 	assert(DRMLISTEMPTY(&bufmgr_fake->on_hardware));
789 
790 	DRMLISTFOREACHSAFE(block, tmp, &bufmgr_fake->lru) {
791 		assert(_fence_test(bufmgr_fake, block->fence));
792 		set_dirty(block->bo);
793 	}
794 
795 	pthread_mutex_unlock(&bufmgr_fake->lock);
796 }
797 
798 static drm_intel_bo *
drm_intel_fake_bo_alloc(drm_intel_bufmgr * bufmgr,const char * name,unsigned long size,unsigned int alignment)799 drm_intel_fake_bo_alloc(drm_intel_bufmgr *bufmgr,
800 			const char *name,
801 			unsigned long size,
802 			unsigned int alignment)
803 {
804 	drm_intel_bufmgr_fake *bufmgr_fake;
805 	drm_intel_bo_fake *bo_fake;
806 
807 	bufmgr_fake = (drm_intel_bufmgr_fake *) bufmgr;
808 
809 	assert(size != 0);
810 
811 	bo_fake = calloc(1, sizeof(*bo_fake));
812 	if (!bo_fake)
813 		return NULL;
814 
815 	bo_fake->bo.size = size;
816 	bo_fake->bo.offset = -1;
817 	bo_fake->bo.virtual = NULL;
818 	bo_fake->bo.bufmgr = bufmgr;
819 	bo_fake->refcount = 1;
820 
821 	/* Alignment must be a power of two */
822 	assert((alignment & (alignment - 1)) == 0);
823 	if (alignment == 0)
824 		alignment = 1;
825 	bo_fake->alignment = alignment;
826 	bo_fake->id = ++bufmgr_fake->buf_nr;
827 	bo_fake->name = name;
828 	bo_fake->flags = 0;
829 	bo_fake->is_static = 0;
830 
831 	DBG("drm_bo_alloc: (buf %d: %s, %lu kb)\n", bo_fake->id, bo_fake->name,
832 	    bo_fake->bo.size / 1024);
833 
834 	return &bo_fake->bo;
835 }
836 
837 static drm_intel_bo *
drm_intel_fake_bo_alloc_tiled(drm_intel_bufmgr * bufmgr,const char * name,int x,int y,int cpp,uint32_t * tiling_mode,unsigned long * pitch,unsigned long flags)838 drm_intel_fake_bo_alloc_tiled(drm_intel_bufmgr * bufmgr,
839 			      const char *name,
840 			      int x, int y, int cpp,
841 			      uint32_t *tiling_mode,
842 			      unsigned long *pitch,
843 			      unsigned long flags)
844 {
845 	unsigned long stride, aligned_y;
846 
847 	/* No runtime tiling support for fake. */
848 	*tiling_mode = I915_TILING_NONE;
849 
850 	/* Align it for being a render target.  Shouldn't need anything else. */
851 	stride = x * cpp;
852 	stride = ROUND_UP_TO(stride, 64);
853 
854 	/* 965 subspan loading alignment */
855 	aligned_y = ALIGN(y, 2);
856 
857 	*pitch = stride;
858 
859 	return drm_intel_fake_bo_alloc(bufmgr, name, stride * aligned_y,
860 				       4096);
861 }
862 
863 drm_intel_bo *
drm_intel_bo_fake_alloc_static(drm_intel_bufmgr * bufmgr,const char * name,unsigned long offset,unsigned long size,void * virtual)864 drm_intel_bo_fake_alloc_static(drm_intel_bufmgr *bufmgr,
865 			       const char *name,
866 			       unsigned long offset,
867 			       unsigned long size, void *virtual)
868 {
869 	drm_intel_bufmgr_fake *bufmgr_fake;
870 	drm_intel_bo_fake *bo_fake;
871 
872 	bufmgr_fake = (drm_intel_bufmgr_fake *) bufmgr;
873 
874 	assert(size != 0);
875 
876 	bo_fake = calloc(1, sizeof(*bo_fake));
877 	if (!bo_fake)
878 		return NULL;
879 
880 	bo_fake->bo.size = size;
881 	bo_fake->bo.offset = offset;
882 	bo_fake->bo.virtual = virtual;
883 	bo_fake->bo.bufmgr = bufmgr;
884 	bo_fake->refcount = 1;
885 	bo_fake->id = ++bufmgr_fake->buf_nr;
886 	bo_fake->name = name;
887 	bo_fake->flags = BM_PINNED;
888 	bo_fake->is_static = 1;
889 
890 	DBG("drm_bo_alloc_static: (buf %d: %s, %lu kb)\n", bo_fake->id,
891 	    bo_fake->name, bo_fake->bo.size / 1024);
892 
893 	return &bo_fake->bo;
894 }
895 
896 static void
drm_intel_fake_bo_reference(drm_intel_bo * bo)897 drm_intel_fake_bo_reference(drm_intel_bo *bo)
898 {
899 	drm_intel_bufmgr_fake *bufmgr_fake =
900 	    (drm_intel_bufmgr_fake *) bo->bufmgr;
901 	drm_intel_bo_fake *bo_fake = (drm_intel_bo_fake *) bo;
902 
903 	pthread_mutex_lock(&bufmgr_fake->lock);
904 	bo_fake->refcount++;
905 	pthread_mutex_unlock(&bufmgr_fake->lock);
906 }
907 
908 static void
drm_intel_fake_bo_reference_locked(drm_intel_bo * bo)909 drm_intel_fake_bo_reference_locked(drm_intel_bo *bo)
910 {
911 	drm_intel_bo_fake *bo_fake = (drm_intel_bo_fake *) bo;
912 
913 	bo_fake->refcount++;
914 }
915 
916 static void
drm_intel_fake_bo_unreference_locked(drm_intel_bo * bo)917 drm_intel_fake_bo_unreference_locked(drm_intel_bo *bo)
918 {
919 	drm_intel_bufmgr_fake *bufmgr_fake =
920 	    (drm_intel_bufmgr_fake *) bo->bufmgr;
921 	drm_intel_bo_fake *bo_fake = (drm_intel_bo_fake *) bo;
922 	int i;
923 
924 	if (--bo_fake->refcount == 0) {
925 		assert(bo_fake->map_count == 0);
926 		/* No remaining references, so free it */
927 		if (bo_fake->block)
928 			free_block(bufmgr_fake, bo_fake->block, 1);
929 		free_backing_store(bo);
930 
931 		for (i = 0; i < bo_fake->nr_relocs; i++)
932 			drm_intel_fake_bo_unreference_locked(bo_fake->relocs[i].
933 							     target_buf);
934 
935 		DBG("drm_bo_unreference: free buf %d %s\n", bo_fake->id,
936 		    bo_fake->name);
937 
938 		free(bo_fake->relocs);
939 		free(bo);
940 	}
941 }
942 
943 static void
drm_intel_fake_bo_unreference(drm_intel_bo * bo)944 drm_intel_fake_bo_unreference(drm_intel_bo *bo)
945 {
946 	drm_intel_bufmgr_fake *bufmgr_fake =
947 	    (drm_intel_bufmgr_fake *) bo->bufmgr;
948 
949 	pthread_mutex_lock(&bufmgr_fake->lock);
950 	drm_intel_fake_bo_unreference_locked(bo);
951 	pthread_mutex_unlock(&bufmgr_fake->lock);
952 }
953 
954 /**
955  * Set the buffer as not requiring backing store, and instead get the callback
956  * invoked whenever it would be set dirty.
957  */
958 void
drm_intel_bo_fake_disable_backing_store(drm_intel_bo * bo,void (* invalidate_cb)(drm_intel_bo * bo,void * ptr),void * ptr)959 drm_intel_bo_fake_disable_backing_store(drm_intel_bo *bo,
960 					void (*invalidate_cb) (drm_intel_bo *bo,
961 							       void *ptr),
962 					void *ptr)
963 {
964 	drm_intel_bufmgr_fake *bufmgr_fake =
965 	    (drm_intel_bufmgr_fake *) bo->bufmgr;
966 	drm_intel_bo_fake *bo_fake = (drm_intel_bo_fake *) bo;
967 
968 	pthread_mutex_lock(&bufmgr_fake->lock);
969 
970 	if (bo_fake->backing_store)
971 		free_backing_store(bo);
972 
973 	bo_fake->flags |= BM_NO_BACKING_STORE;
974 
975 	DBG("disable_backing_store set buf %d dirty\n", bo_fake->id);
976 	bo_fake->dirty = 1;
977 	bo_fake->invalidate_cb = invalidate_cb;
978 	bo_fake->invalidate_ptr = ptr;
979 
980 	/* Note that it is invalid right from the start.  Also note
981 	 * invalidate_cb is called with the bufmgr locked, so cannot
982 	 * itself make bufmgr calls.
983 	 */
984 	if (invalidate_cb != NULL)
985 		invalidate_cb(bo, ptr);
986 
987 	pthread_mutex_unlock(&bufmgr_fake->lock);
988 }
989 
990 /**
991  * Map a buffer into bo->virtual, allocating either card memory space (If
992  * BM_NO_BACKING_STORE or BM_PINNED) or backing store, as necessary.
993  */
994 static int
drm_intel_fake_bo_map_locked(drm_intel_bo * bo,int write_enable)995  drm_intel_fake_bo_map_locked(drm_intel_bo *bo, int write_enable)
996 {
997 	drm_intel_bufmgr_fake *bufmgr_fake =
998 	    (drm_intel_bufmgr_fake *) bo->bufmgr;
999 	drm_intel_bo_fake *bo_fake = (drm_intel_bo_fake *) bo;
1000 
1001 	/* Static buffers are always mapped. */
1002 	if (bo_fake->is_static) {
1003 		if (bo_fake->card_dirty) {
1004 			drm_intel_bufmgr_fake_wait_idle(bufmgr_fake);
1005 			bo_fake->card_dirty = 0;
1006 		}
1007 		return 0;
1008 	}
1009 
1010 	/* Allow recursive mapping.  Mesa may recursively map buffers with
1011 	 * nested display loops, and it is used internally in bufmgr_fake
1012 	 * for relocation.
1013 	 */
1014 	if (bo_fake->map_count++ != 0)
1015 		return 0;
1016 
1017 	{
1018 		DBG("drm_bo_map: (buf %d: %s, %lu kb)\n", bo_fake->id,
1019 		    bo_fake->name, bo_fake->bo.size / 1024);
1020 
1021 		if (bo->virtual != NULL) {
1022 			drmMsg("%s: already mapped\n", __func__);
1023 			abort();
1024 		} else if (bo_fake->flags & (BM_NO_BACKING_STORE | BM_PINNED)) {
1025 
1026 			if (!bo_fake->block && !evict_and_alloc_block(bo)) {
1027 				DBG("%s: alloc failed\n", __func__);
1028 				bufmgr_fake->fail = 1;
1029 				return 1;
1030 			} else {
1031 				assert(bo_fake->block);
1032 				bo_fake->dirty = 0;
1033 
1034 				if (!(bo_fake->flags & BM_NO_FENCE_SUBDATA) &&
1035 				    bo_fake->block->fenced) {
1036 					drm_intel_fake_bo_wait_rendering_locked
1037 					    (bo);
1038 				}
1039 
1040 				bo->virtual = bo_fake->block->virtual;
1041 			}
1042 		} else {
1043 			if (write_enable)
1044 				set_dirty(bo);
1045 
1046 			if (bo_fake->backing_store == 0)
1047 				alloc_backing_store(bo);
1048 
1049 			if ((bo_fake->card_dirty == 1) && bo_fake->block) {
1050 				if (bo_fake->block->fenced)
1051 					drm_intel_fake_bo_wait_rendering_locked
1052 					    (bo);
1053 
1054 				memcpy(bo_fake->backing_store,
1055 				       bo_fake->block->virtual,
1056 				       bo_fake->block->bo->size);
1057 				bo_fake->card_dirty = 0;
1058 			}
1059 
1060 			bo->virtual = bo_fake->backing_store;
1061 		}
1062 	}
1063 
1064 	return 0;
1065 }
1066 
1067 static int
drm_intel_fake_bo_map(drm_intel_bo * bo,int write_enable)1068  drm_intel_fake_bo_map(drm_intel_bo *bo, int write_enable)
1069 {
1070 	drm_intel_bufmgr_fake *bufmgr_fake =
1071 	    (drm_intel_bufmgr_fake *) bo->bufmgr;
1072 	int ret;
1073 
1074 	pthread_mutex_lock(&bufmgr_fake->lock);
1075 	ret = drm_intel_fake_bo_map_locked(bo, write_enable);
1076 	pthread_mutex_unlock(&bufmgr_fake->lock);
1077 
1078 	return ret;
1079 }
1080 
1081 static int
drm_intel_fake_bo_unmap_locked(drm_intel_bo * bo)1082  drm_intel_fake_bo_unmap_locked(drm_intel_bo *bo)
1083 {
1084 	drm_intel_bufmgr_fake *bufmgr_fake =
1085 	    (drm_intel_bufmgr_fake *) bo->bufmgr;
1086 	drm_intel_bo_fake *bo_fake = (drm_intel_bo_fake *) bo;
1087 
1088 	/* Static buffers are always mapped. */
1089 	if (bo_fake->is_static)
1090 		return 0;
1091 
1092 	assert(bo_fake->map_count != 0);
1093 	if (--bo_fake->map_count != 0)
1094 		return 0;
1095 
1096 	DBG("drm_bo_unmap: (buf %d: %s, %lu kb)\n", bo_fake->id, bo_fake->name,
1097 	    bo_fake->bo.size / 1024);
1098 
1099 	bo->virtual = NULL;
1100 
1101 	return 0;
1102 }
1103 
drm_intel_fake_bo_unmap(drm_intel_bo * bo)1104 static int drm_intel_fake_bo_unmap(drm_intel_bo *bo)
1105 {
1106 	drm_intel_bufmgr_fake *bufmgr_fake =
1107 	    (drm_intel_bufmgr_fake *) bo->bufmgr;
1108 	int ret;
1109 
1110 	pthread_mutex_lock(&bufmgr_fake->lock);
1111 	ret = drm_intel_fake_bo_unmap_locked(bo);
1112 	pthread_mutex_unlock(&bufmgr_fake->lock);
1113 
1114 	return ret;
1115 }
1116 
1117 static int
drm_intel_fake_bo_subdata(drm_intel_bo * bo,unsigned long offset,unsigned long size,const void * data)1118 drm_intel_fake_bo_subdata(drm_intel_bo *bo, unsigned long offset,
1119 			  unsigned long size, const void *data)
1120 {
1121 	int ret;
1122 
1123 	if (size == 0 || data == NULL)
1124 		return 0;
1125 
1126 	ret = drm_intel_bo_map(bo, 1);
1127 	if (ret)
1128 		return ret;
1129 	memcpy((unsigned char *)bo->virtual + offset, data, size);
1130 	drm_intel_bo_unmap(bo);
1131 	return 0;
1132 }
1133 
1134 static void
drm_intel_fake_kick_all_locked(drm_intel_bufmgr_fake * bufmgr_fake)1135  drm_intel_fake_kick_all_locked(drm_intel_bufmgr_fake *bufmgr_fake)
1136 {
1137 	struct block *block, *tmp;
1138 
1139 	bufmgr_fake->performed_rendering = 0;
1140 	/* okay for ever BO that is on the HW kick it off.
1141 	   seriously not afraid of the POLICE right now */
1142 	DRMLISTFOREACHSAFE(block, tmp, &bufmgr_fake->on_hardware) {
1143 		drm_intel_bo_fake *bo_fake = (drm_intel_bo_fake *) block->bo;
1144 
1145 		block->on_hardware = 0;
1146 		free_block(bufmgr_fake, block, 0);
1147 		bo_fake->block = NULL;
1148 		bo_fake->validated = 0;
1149 		if (!(bo_fake->flags & BM_NO_BACKING_STORE))
1150 			bo_fake->dirty = 1;
1151 	}
1152 
1153 }
1154 
1155 static int
drm_intel_fake_bo_validate(drm_intel_bo * bo)1156  drm_intel_fake_bo_validate(drm_intel_bo *bo)
1157 {
1158 	drm_intel_bufmgr_fake *bufmgr_fake;
1159 	drm_intel_bo_fake *bo_fake = (drm_intel_bo_fake *) bo;
1160 
1161 	bufmgr_fake = (drm_intel_bufmgr_fake *) bo->bufmgr;
1162 
1163 	DBG("drm_bo_validate: (buf %d: %s, %lu kb)\n", bo_fake->id,
1164 	    bo_fake->name, bo_fake->bo.size / 1024);
1165 
1166 	/* Sanity check: Buffers should be unmapped before being validated.
1167 	 * This is not so much of a problem for bufmgr_fake, but TTM refuses,
1168 	 * and the problem is harder to debug there.
1169 	 */
1170 	assert(bo_fake->map_count == 0);
1171 
1172 	if (bo_fake->is_static) {
1173 		/* Add it to the needs-fence list */
1174 		bufmgr_fake->need_fence = 1;
1175 		return 0;
1176 	}
1177 
1178 	/* Allocate the card memory */
1179 	if (!bo_fake->block && !evict_and_alloc_block(bo)) {
1180 		bufmgr_fake->fail = 1;
1181 		DBG("Failed to validate buf %d:%s\n", bo_fake->id,
1182 		    bo_fake->name);
1183 		return -1;
1184 	}
1185 
1186 	assert(bo_fake->block);
1187 	assert(bo_fake->block->bo == &bo_fake->bo);
1188 
1189 	bo->offset = bo_fake->block->mem->ofs;
1190 
1191 	/* Upload the buffer contents if necessary */
1192 	if (bo_fake->dirty) {
1193 		DBG("Upload dirty buf %d:%s, sz %lu offset 0x%x\n", bo_fake->id,
1194 		    bo_fake->name, bo->size, bo_fake->block->mem->ofs);
1195 
1196 		assert(!(bo_fake->flags & (BM_NO_BACKING_STORE | BM_PINNED)));
1197 
1198 		/* Actually, should be able to just wait for a fence on the
1199 		 * memory, which we would be tracking when we free it. Waiting
1200 		 * for idle is a sufficiently large hammer for now.
1201 		 */
1202 		drm_intel_bufmgr_fake_wait_idle(bufmgr_fake);
1203 
1204 		/* we may never have mapped this BO so it might not have any
1205 		 * backing store if this happens it should be rare, but 0 the
1206 		 * card memory in any case */
1207 		if (bo_fake->backing_store)
1208 			memcpy(bo_fake->block->virtual, bo_fake->backing_store,
1209 			       bo->size);
1210 		else
1211 			memset(bo_fake->block->virtual, 0, bo->size);
1212 
1213 		bo_fake->dirty = 0;
1214 	}
1215 
1216 	bo_fake->block->fenced = 0;
1217 	bo_fake->block->on_hardware = 1;
1218 	DRMLISTDEL(bo_fake->block);
1219 	DRMLISTADDTAIL(bo_fake->block, &bufmgr_fake->on_hardware);
1220 
1221 	bo_fake->validated = 1;
1222 	bufmgr_fake->need_fence = 1;
1223 
1224 	return 0;
1225 }
1226 
1227 static void
drm_intel_fake_fence_validated(drm_intel_bufmgr * bufmgr)1228 drm_intel_fake_fence_validated(drm_intel_bufmgr *bufmgr)
1229 {
1230 	drm_intel_bufmgr_fake *bufmgr_fake = (drm_intel_bufmgr_fake *) bufmgr;
1231 	unsigned int cookie;
1232 
1233 	cookie = _fence_emit_internal(bufmgr_fake);
1234 	fence_blocks(bufmgr_fake, cookie);
1235 
1236 	DBG("drm_fence_validated: 0x%08x cookie\n", cookie);
1237 }
1238 
1239 static void
drm_intel_fake_destroy(drm_intel_bufmgr * bufmgr)1240 drm_intel_fake_destroy(drm_intel_bufmgr *bufmgr)
1241 {
1242 	drm_intel_bufmgr_fake *bufmgr_fake = (drm_intel_bufmgr_fake *) bufmgr;
1243 
1244 	pthread_mutex_destroy(&bufmgr_fake->lock);
1245 	mmDestroy(bufmgr_fake->heap);
1246 	free(bufmgr);
1247 }
1248 
1249 static int
drm_intel_fake_emit_reloc(drm_intel_bo * bo,uint32_t offset,drm_intel_bo * target_bo,uint32_t target_offset,uint32_t read_domains,uint32_t write_domain)1250 drm_intel_fake_emit_reloc(drm_intel_bo *bo, uint32_t offset,
1251 			  drm_intel_bo *target_bo, uint32_t target_offset,
1252 			  uint32_t read_domains, uint32_t write_domain)
1253 {
1254 	drm_intel_bufmgr_fake *bufmgr_fake =
1255 	    (drm_intel_bufmgr_fake *) bo->bufmgr;
1256 	struct fake_buffer_reloc *r;
1257 	drm_intel_bo_fake *bo_fake = (drm_intel_bo_fake *) bo;
1258 	drm_intel_bo_fake *target_fake = (drm_intel_bo_fake *) target_bo;
1259 	int i;
1260 
1261 	pthread_mutex_lock(&bufmgr_fake->lock);
1262 
1263 	assert(bo);
1264 	assert(target_bo);
1265 
1266 	if (bo_fake->relocs == NULL) {
1267 		bo_fake->relocs =
1268 		    malloc(sizeof(struct fake_buffer_reloc) * MAX_RELOCS);
1269 	}
1270 
1271 	r = &bo_fake->relocs[bo_fake->nr_relocs++];
1272 
1273 	assert(bo_fake->nr_relocs <= MAX_RELOCS);
1274 
1275 	drm_intel_fake_bo_reference_locked(target_bo);
1276 
1277 	if (!target_fake->is_static) {
1278 		bo_fake->child_size +=
1279 		    ALIGN(target_bo->size, target_fake->alignment);
1280 		bo_fake->child_size += target_fake->child_size;
1281 	}
1282 	r->target_buf = target_bo;
1283 	r->offset = offset;
1284 	r->last_target_offset = target_bo->offset;
1285 	r->delta = target_offset;
1286 	r->read_domains = read_domains;
1287 	r->write_domain = write_domain;
1288 
1289 	if (bufmgr_fake->debug) {
1290 		/* Check that a conflicting relocation hasn't already been
1291 		 * emitted.
1292 		 */
1293 		for (i = 0; i < bo_fake->nr_relocs - 1; i++) {
1294 			struct fake_buffer_reloc *r2 = &bo_fake->relocs[i];
1295 
1296 			assert(r->offset != r2->offset);
1297 		}
1298 	}
1299 
1300 	pthread_mutex_unlock(&bufmgr_fake->lock);
1301 
1302 	return 0;
1303 }
1304 
1305 /**
1306  * Incorporates the validation flags associated with each relocation into
1307  * the combined validation flags for the buffer on this batchbuffer submission.
1308  */
1309 static void
drm_intel_fake_calculate_domains(drm_intel_bo * bo)1310 drm_intel_fake_calculate_domains(drm_intel_bo *bo)
1311 {
1312 	drm_intel_bo_fake *bo_fake = (drm_intel_bo_fake *) bo;
1313 	int i;
1314 
1315 	for (i = 0; i < bo_fake->nr_relocs; i++) {
1316 		struct fake_buffer_reloc *r = &bo_fake->relocs[i];
1317 		drm_intel_bo_fake *target_fake =
1318 		    (drm_intel_bo_fake *) r->target_buf;
1319 
1320 		/* Do the same for the tree of buffers we depend on */
1321 		drm_intel_fake_calculate_domains(r->target_buf);
1322 
1323 		target_fake->read_domains |= r->read_domains;
1324 		target_fake->write_domain |= r->write_domain;
1325 	}
1326 }
1327 
1328 static int
drm_intel_fake_reloc_and_validate_buffer(drm_intel_bo * bo)1329 drm_intel_fake_reloc_and_validate_buffer(drm_intel_bo *bo)
1330 {
1331 	drm_intel_bufmgr_fake *bufmgr_fake =
1332 	    (drm_intel_bufmgr_fake *) bo->bufmgr;
1333 	drm_intel_bo_fake *bo_fake = (drm_intel_bo_fake *) bo;
1334 	int i, ret;
1335 
1336 	assert(bo_fake->map_count == 0);
1337 
1338 	for (i = 0; i < bo_fake->nr_relocs; i++) {
1339 		struct fake_buffer_reloc *r = &bo_fake->relocs[i];
1340 		drm_intel_bo_fake *target_fake =
1341 		    (drm_intel_bo_fake *) r->target_buf;
1342 		uint32_t reloc_data;
1343 
1344 		/* Validate the target buffer if that hasn't been done. */
1345 		if (!target_fake->validated) {
1346 			ret =
1347 			    drm_intel_fake_reloc_and_validate_buffer(r->target_buf);
1348 			if (ret != 0) {
1349 				if (bo->virtual != NULL)
1350 					drm_intel_fake_bo_unmap_locked(bo);
1351 				return ret;
1352 			}
1353 		}
1354 
1355 		/* Calculate the value of the relocation entry. */
1356 		if (r->target_buf->offset != r->last_target_offset) {
1357 			reloc_data = r->target_buf->offset + r->delta;
1358 
1359 			if (bo->virtual == NULL)
1360 				drm_intel_fake_bo_map_locked(bo, 1);
1361 
1362 			*(uint32_t *) ((uint8_t *) bo->virtual + r->offset) =
1363 			    reloc_data;
1364 
1365 			r->last_target_offset = r->target_buf->offset;
1366 		}
1367 	}
1368 
1369 	if (bo->virtual != NULL)
1370 		drm_intel_fake_bo_unmap_locked(bo);
1371 
1372 	if (bo_fake->write_domain != 0) {
1373 		if (!(bo_fake->flags & (BM_NO_BACKING_STORE | BM_PINNED))) {
1374 			if (bo_fake->backing_store == 0)
1375 				alloc_backing_store(bo);
1376 		}
1377 		bo_fake->card_dirty = 1;
1378 		bufmgr_fake->performed_rendering = 1;
1379 	}
1380 
1381 	return drm_intel_fake_bo_validate(bo);
1382 }
1383 
1384 static void
drm_intel_bo_fake_post_submit(drm_intel_bo * bo)1385 drm_intel_bo_fake_post_submit(drm_intel_bo *bo)
1386 {
1387 	drm_intel_bufmgr_fake *bufmgr_fake =
1388 	    (drm_intel_bufmgr_fake *) bo->bufmgr;
1389 	drm_intel_bo_fake *bo_fake = (drm_intel_bo_fake *) bo;
1390 	int i;
1391 
1392 	for (i = 0; i < bo_fake->nr_relocs; i++) {
1393 		struct fake_buffer_reloc *r = &bo_fake->relocs[i];
1394 		drm_intel_bo_fake *target_fake =
1395 		    (drm_intel_bo_fake *) r->target_buf;
1396 
1397 		if (target_fake->validated)
1398 			drm_intel_bo_fake_post_submit(r->target_buf);
1399 
1400 		DBG("%s@0x%08x + 0x%08x -> %s@0x%08x + 0x%08x\n",
1401 		    bo_fake->name, (uint32_t) bo->offset, r->offset,
1402 		    target_fake->name, (uint32_t) r->target_buf->offset,
1403 		    r->delta);
1404 	}
1405 
1406 	assert(bo_fake->map_count == 0);
1407 	bo_fake->validated = 0;
1408 	bo_fake->read_domains = 0;
1409 	bo_fake->write_domain = 0;
1410 }
1411 
1412 void
drm_intel_bufmgr_fake_set_exec_callback(drm_intel_bufmgr * bufmgr,int (* exec)(drm_intel_bo * bo,unsigned int used,void * priv),void * priv)1413 drm_intel_bufmgr_fake_set_exec_callback(drm_intel_bufmgr *bufmgr,
1414 					     int (*exec) (drm_intel_bo *bo,
1415 							  unsigned int used,
1416 							  void *priv),
1417 					     void *priv)
1418 {
1419 	drm_intel_bufmgr_fake *bufmgr_fake = (drm_intel_bufmgr_fake *) bufmgr;
1420 
1421 	bufmgr_fake->exec = exec;
1422 	bufmgr_fake->exec_priv = priv;
1423 }
1424 
1425 static int
drm_intel_fake_bo_exec(drm_intel_bo * bo,int used,drm_clip_rect_t * cliprects,int num_cliprects,int DR4)1426 drm_intel_fake_bo_exec(drm_intel_bo *bo, int used,
1427 		       drm_clip_rect_t * cliprects, int num_cliprects, int DR4)
1428 {
1429 	drm_intel_bufmgr_fake *bufmgr_fake =
1430 	    (drm_intel_bufmgr_fake *) bo->bufmgr;
1431 	drm_intel_bo_fake *batch_fake = (drm_intel_bo_fake *) bo;
1432 	struct drm_i915_batchbuffer batch;
1433 	int ret;
1434 	int retry_count = 0;
1435 
1436 	pthread_mutex_lock(&bufmgr_fake->lock);
1437 
1438 	bufmgr_fake->performed_rendering = 0;
1439 
1440 	drm_intel_fake_calculate_domains(bo);
1441 
1442 	batch_fake->read_domains = I915_GEM_DOMAIN_COMMAND;
1443 
1444 	/* we've ran out of RAM so blow the whole lot away and retry */
1445 restart:
1446 	ret = drm_intel_fake_reloc_and_validate_buffer(bo);
1447 	if (bufmgr_fake->fail == 1) {
1448 		if (retry_count == 0) {
1449 			retry_count++;
1450 			drm_intel_fake_kick_all_locked(bufmgr_fake);
1451 			bufmgr_fake->fail = 0;
1452 			goto restart;
1453 		} else		/* dump out the memory here */
1454 			mmDumpMemInfo(bufmgr_fake->heap);
1455 	}
1456 
1457 	assert(ret == 0);
1458 
1459 	if (bufmgr_fake->exec != NULL) {
1460 		ret = bufmgr_fake->exec(bo, used, bufmgr_fake->exec_priv);
1461 		if (ret != 0) {
1462 			pthread_mutex_unlock(&bufmgr_fake->lock);
1463 			return ret;
1464 		}
1465 	} else {
1466 		batch.start = bo->offset;
1467 		batch.used = used;
1468 		batch.cliprects = cliprects;
1469 		batch.num_cliprects = num_cliprects;
1470 		batch.DR1 = 0;
1471 		batch.DR4 = DR4;
1472 
1473 		if (drmCommandWrite
1474 		    (bufmgr_fake->fd, DRM_I915_BATCHBUFFER, &batch,
1475 		     sizeof(batch))) {
1476 			drmMsg("DRM_I915_BATCHBUFFER: %d\n", -errno);
1477 			pthread_mutex_unlock(&bufmgr_fake->lock);
1478 			return -errno;
1479 		}
1480 	}
1481 
1482 	drm_intel_fake_fence_validated(bo->bufmgr);
1483 
1484 	drm_intel_bo_fake_post_submit(bo);
1485 
1486 	pthread_mutex_unlock(&bufmgr_fake->lock);
1487 
1488 	return 0;
1489 }
1490 
1491 /**
1492  * Return an error if the list of BOs will exceed the aperture size.
1493  *
1494  * This is a rough guess and likely to fail, as during the validate sequence we
1495  * may place a buffer in an inopportune spot early on and then fail to fit
1496  * a set smaller than the aperture.
1497  */
1498 static int
drm_intel_fake_check_aperture_space(drm_intel_bo ** bo_array,int count)1499 drm_intel_fake_check_aperture_space(drm_intel_bo ** bo_array, int count)
1500 {
1501 	drm_intel_bufmgr_fake *bufmgr_fake =
1502 	    (drm_intel_bufmgr_fake *) bo_array[0]->bufmgr;
1503 	unsigned int sz = 0;
1504 	int i;
1505 
1506 	for (i = 0; i < count; i++) {
1507 		drm_intel_bo_fake *bo_fake = (drm_intel_bo_fake *) bo_array[i];
1508 
1509 		if (bo_fake == NULL)
1510 			continue;
1511 
1512 		if (!bo_fake->is_static)
1513 			sz += ALIGN(bo_array[i]->size, bo_fake->alignment);
1514 		sz += bo_fake->child_size;
1515 	}
1516 
1517 	if (sz > bufmgr_fake->size) {
1518 		DBG("check_space: overflowed bufmgr size, %ukb vs %lukb\n",
1519 		    sz / 1024, bufmgr_fake->size / 1024);
1520 		return -1;
1521 	}
1522 
1523 	DBG("drm_check_space: sz %ukb vs bufgr %lukb\n", sz / 1024,
1524 	    bufmgr_fake->size / 1024);
1525 	return 0;
1526 }
1527 
1528 /**
1529  * Evicts all buffers, waiting for fences to pass and copying contents out
1530  * as necessary.
1531  *
1532  * Used by the X Server on LeaveVT, when the card memory is no longer our
1533  * own.
1534  */
1535 void
drm_intel_bufmgr_fake_evict_all(drm_intel_bufmgr * bufmgr)1536 drm_intel_bufmgr_fake_evict_all(drm_intel_bufmgr *bufmgr)
1537 {
1538 	drm_intel_bufmgr_fake *bufmgr_fake = (drm_intel_bufmgr_fake *) bufmgr;
1539 	struct block *block, *tmp;
1540 
1541 	pthread_mutex_lock(&bufmgr_fake->lock);
1542 
1543 	bufmgr_fake->need_fence = 1;
1544 	bufmgr_fake->fail = 0;
1545 
1546 	/* Wait for hardware idle.  We don't know where acceleration has been
1547 	 * happening, so we'll need to wait anyway before letting anything get
1548 	 * put on the card again.
1549 	 */
1550 	drm_intel_bufmgr_fake_wait_idle(bufmgr_fake);
1551 
1552 	/* Check that we hadn't released the lock without having fenced the last
1553 	 * set of buffers.
1554 	 */
1555 	assert(DRMLISTEMPTY(&bufmgr_fake->fenced));
1556 	assert(DRMLISTEMPTY(&bufmgr_fake->on_hardware));
1557 
1558 	DRMLISTFOREACHSAFE(block, tmp, &bufmgr_fake->lru) {
1559 		drm_intel_bo_fake *bo_fake = (drm_intel_bo_fake *) block->bo;
1560 		/* Releases the memory, and memcpys dirty contents out if
1561 		 * necessary.
1562 		 */
1563 		free_block(bufmgr_fake, block, 0);
1564 		bo_fake->block = NULL;
1565 	}
1566 
1567 	pthread_mutex_unlock(&bufmgr_fake->lock);
1568 }
1569 
1570 void
drm_intel_bufmgr_fake_set_last_dispatch(drm_intel_bufmgr * bufmgr,volatile unsigned int * last_dispatch)1571 drm_intel_bufmgr_fake_set_last_dispatch(drm_intel_bufmgr *bufmgr,
1572 					volatile unsigned int
1573 					*last_dispatch)
1574 {
1575 	drm_intel_bufmgr_fake *bufmgr_fake = (drm_intel_bufmgr_fake *) bufmgr;
1576 
1577 	bufmgr_fake->last_dispatch = (volatile int *)last_dispatch;
1578 }
1579 
1580 drm_intel_bufmgr *
drm_intel_bufmgr_fake_init(int fd,unsigned long low_offset,void * low_virtual,unsigned long size,volatile unsigned int * last_dispatch)1581 drm_intel_bufmgr_fake_init(int fd, unsigned long low_offset,
1582 			   void *low_virtual, unsigned long size,
1583 			   volatile unsigned int *last_dispatch)
1584 {
1585 	drm_intel_bufmgr_fake *bufmgr_fake;
1586 
1587 	bufmgr_fake = calloc(1, sizeof(*bufmgr_fake));
1588 
1589 	if (pthread_mutex_init(&bufmgr_fake->lock, NULL) != 0) {
1590 		free(bufmgr_fake);
1591 		return NULL;
1592 	}
1593 
1594 	/* Initialize allocator */
1595 	DRMINITLISTHEAD(&bufmgr_fake->fenced);
1596 	DRMINITLISTHEAD(&bufmgr_fake->on_hardware);
1597 	DRMINITLISTHEAD(&bufmgr_fake->lru);
1598 
1599 	bufmgr_fake->low_offset = low_offset;
1600 	bufmgr_fake->virtual = low_virtual;
1601 	bufmgr_fake->size = size;
1602 	bufmgr_fake->heap = mmInit(low_offset, size);
1603 
1604 	/* Hook in methods */
1605 	bufmgr_fake->bufmgr.bo_alloc = drm_intel_fake_bo_alloc;
1606 	bufmgr_fake->bufmgr.bo_alloc_for_render = drm_intel_fake_bo_alloc;
1607 	bufmgr_fake->bufmgr.bo_alloc_tiled = drm_intel_fake_bo_alloc_tiled;
1608 	bufmgr_fake->bufmgr.bo_reference = drm_intel_fake_bo_reference;
1609 	bufmgr_fake->bufmgr.bo_unreference = drm_intel_fake_bo_unreference;
1610 	bufmgr_fake->bufmgr.bo_map = drm_intel_fake_bo_map;
1611 	bufmgr_fake->bufmgr.bo_unmap = drm_intel_fake_bo_unmap;
1612 	bufmgr_fake->bufmgr.bo_subdata = drm_intel_fake_bo_subdata;
1613 	bufmgr_fake->bufmgr.bo_wait_rendering =
1614 	    drm_intel_fake_bo_wait_rendering;
1615 	bufmgr_fake->bufmgr.bo_emit_reloc = drm_intel_fake_emit_reloc;
1616 	bufmgr_fake->bufmgr.destroy = drm_intel_fake_destroy;
1617 	bufmgr_fake->bufmgr.bo_exec = drm_intel_fake_bo_exec;
1618 	bufmgr_fake->bufmgr.check_aperture_space =
1619 	    drm_intel_fake_check_aperture_space;
1620 	bufmgr_fake->bufmgr.debug = 0;
1621 
1622 	bufmgr_fake->fd = fd;
1623 	bufmgr_fake->last_dispatch = (volatile int *)last_dispatch;
1624 
1625 	return &bufmgr_fake->bufmgr;
1626 }
1627