• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 /*
2  * Copyright 2006 VMware, Inc.
3  * All Rights Reserved.
4  *
5  * Permission is hereby granted, free of charge, to any person obtaining a
6  * copy of this software and associated documentation files (the
7  * "Software"), to deal in the Software without restriction, including
8  * without limitation the rights to use, copy, modify, merge, publish,
9  * distribute, sublicense, and/or sell copies of the Software, and to
10  * permit persons to whom the Software is furnished to do so, subject to
11  * the following conditions:
12  *
13  * The above copyright notice and this permission notice (including the
14  * next paragraph) shall be included in all copies or substantial portions
15  * of the Software.
16  *
17  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
18  * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
19  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
20  * IN NO EVENT SHALL VMWARE AND/OR ITS SUPPLIERS BE LIABLE FOR
21  * ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
22  * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
23  * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
24  */
25 
26 #include "intel_batchbuffer.h"
27 #include "intel_buffer_objects.h"
28 #include "brw_bufmgr.h"
29 #include "intel_buffers.h"
30 #include "intel_fbo.h"
31 #include "brw_context.h"
32 #include "brw_defines.h"
33 #include "brw_state.h"
34 #include "common/gen_decoder.h"
35 
36 #include "util/hash_table.h"
37 
38 #include <xf86drm.h>
39 #include <i915_drm.h>
40 
41 #define FILE_DEBUG_FLAG DEBUG_BUFMGR
42 
43 /**
44  * Target sizes of the batch and state buffers.  We create the initial
45  * buffers at these sizes, and flush when they're nearly full.  If we
46  * underestimate how close we are to the end, and suddenly need more space
47  * in the middle of a draw, we can grow the buffers, and finish the draw.
48  * At that point, we'll be over our target size, so the next operation
49  * should flush.  Each time we flush the batch, we recreate both buffers
50  * at the original target size, so it doesn't grow without bound.
51  */
52 #define BATCH_SZ (20 * 1024)
53 #define STATE_SZ (16 * 1024)
54 
55 static void
56 intel_batchbuffer_reset(struct brw_context *brw);
57 
58 static bool
uint_key_compare(const void * a,const void * b)59 uint_key_compare(const void *a, const void *b)
60 {
61    return a == b;
62 }
63 
64 static uint32_t
uint_key_hash(const void * key)65 uint_key_hash(const void *key)
66 {
67    return (uintptr_t) key;
68 }
69 
70 static void
init_reloc_list(struct brw_reloc_list * rlist,int count)71 init_reloc_list(struct brw_reloc_list *rlist, int count)
72 {
73    rlist->reloc_count = 0;
74    rlist->reloc_array_size = count;
75    rlist->relocs = malloc(rlist->reloc_array_size *
76                           sizeof(struct drm_i915_gem_relocation_entry));
77 }
78 
79 void
intel_batchbuffer_init(struct brw_context * brw)80 intel_batchbuffer_init(struct brw_context *brw)
81 {
82    struct intel_screen *screen = brw->screen;
83    struct intel_batchbuffer *batch = &brw->batch;
84    const struct gen_device_info *devinfo = &screen->devinfo;
85 
86    batch->use_shadow_copy = !devinfo->has_llc;
87 
88    if (batch->use_shadow_copy) {
89       batch->batch.map = malloc(BATCH_SZ);
90       batch->map_next = batch->batch.map;
91       batch->state.map = malloc(STATE_SZ);
92    }
93 
94    init_reloc_list(&batch->batch_relocs, 250);
95    init_reloc_list(&batch->state_relocs, 250);
96 
97    batch->exec_count = 0;
98    batch->exec_array_size = 100;
99    batch->exec_bos =
100       malloc(batch->exec_array_size * sizeof(batch->exec_bos[0]));
101    batch->validation_list =
102       malloc(batch->exec_array_size * sizeof(batch->validation_list[0]));
103 
104    if (INTEL_DEBUG & DEBUG_BATCH) {
105       batch->state_batch_sizes =
106          _mesa_hash_table_create(NULL, uint_key_hash, uint_key_compare);
107    }
108 
109    batch->use_batch_first =
110       screen->kernel_features & KERNEL_ALLOWS_EXEC_BATCH_FIRST;
111 
112    /* PIPE_CONTROL needs a w/a but only on gen6 */
113    batch->valid_reloc_flags = EXEC_OBJECT_WRITE;
114    if (devinfo->gen == 6)
115       batch->valid_reloc_flags |= EXEC_OBJECT_NEEDS_GTT;
116 
117    intel_batchbuffer_reset(brw);
118 }
119 
120 #define READ_ONCE(x) (*(volatile __typeof__(x) *)&(x))
121 
122 static unsigned
add_exec_bo(struct intel_batchbuffer * batch,struct brw_bo * bo)123 add_exec_bo(struct intel_batchbuffer *batch, struct brw_bo *bo)
124 {
125    unsigned index = READ_ONCE(bo->index);
126 
127    if (index < batch->exec_count && batch->exec_bos[index] == bo)
128       return index;
129 
130    /* May have been shared between multiple active batches */
131    for (index = 0; index < batch->exec_count; index++) {
132       if (batch->exec_bos[index] == bo)
133          return index;
134    }
135 
136    brw_bo_reference(bo);
137 
138    if (batch->exec_count == batch->exec_array_size) {
139       batch->exec_array_size *= 2;
140       batch->exec_bos =
141          realloc(batch->exec_bos,
142                  batch->exec_array_size * sizeof(batch->exec_bos[0]));
143       batch->validation_list =
144          realloc(batch->validation_list,
145                  batch->exec_array_size * sizeof(batch->validation_list[0]));
146    }
147 
148    batch->validation_list[batch->exec_count] =
149       (struct drm_i915_gem_exec_object2) {
150          .handle = bo->gem_handle,
151          .alignment = bo->align,
152          .offset = bo->gtt_offset,
153          .flags = bo->kflags,
154       };
155 
156    bo->index = batch->exec_count;
157    batch->exec_bos[batch->exec_count] = bo;
158    batch->aperture_space += bo->size;
159 
160    return batch->exec_count++;
161 }
162 
163 static void
recreate_growing_buffer(struct brw_context * brw,struct brw_growing_bo * grow,const char * name,unsigned size)164 recreate_growing_buffer(struct brw_context *brw,
165                         struct brw_growing_bo *grow,
166                         const char *name, unsigned size)
167 {
168    struct intel_screen *screen = brw->screen;
169    struct intel_batchbuffer *batch = &brw->batch;
170    struct brw_bufmgr *bufmgr = screen->bufmgr;
171 
172    grow->bo = brw_bo_alloc(bufmgr, name, size, 4096);
173    grow->bo->kflags = can_do_exec_capture(screen) ? EXEC_OBJECT_CAPTURE : 0;
174    grow->partial_bo = NULL;
175    grow->partial_bo_map = NULL;
176    grow->partial_bytes = 0;
177 
178    if (!batch->use_shadow_copy)
179       grow->map = brw_bo_map(brw, grow->bo, MAP_READ | MAP_WRITE);
180 }
181 
182 static void
intel_batchbuffer_reset(struct brw_context * brw)183 intel_batchbuffer_reset(struct brw_context *brw)
184 {
185    struct intel_batchbuffer *batch = &brw->batch;
186 
187    if (batch->last_bo != NULL) {
188       brw_bo_unreference(batch->last_bo);
189       batch->last_bo = NULL;
190    }
191    batch->last_bo = batch->batch.bo;
192 
193    recreate_growing_buffer(brw, &batch->batch, "batchbuffer", BATCH_SZ);
194    batch->map_next = batch->batch.map;
195 
196    recreate_growing_buffer(brw, &batch->state, "statebuffer", STATE_SZ);
197 
198    /* Avoid making 0 a valid state offset - otherwise the decoder will try
199     * and decode data when we use offset 0 as a null pointer.
200     */
201    batch->state_used = 1;
202 
203    add_exec_bo(batch, batch->batch.bo);
204    assert(batch->batch.bo->index == 0);
205 
206    batch->needs_sol_reset = false;
207    batch->state_base_address_emitted = false;
208 
209    /* We don't know what ring the new batch will be sent to until we see the
210     * first BEGIN_BATCH or BEGIN_BATCH_BLT.  Mark it as unknown.
211     */
212    batch->ring = UNKNOWN_RING;
213 
214    if (batch->state_batch_sizes)
215       _mesa_hash_table_clear(batch->state_batch_sizes, NULL);
216 }
217 
218 static void
intel_batchbuffer_reset_and_clear_render_cache(struct brw_context * brw)219 intel_batchbuffer_reset_and_clear_render_cache(struct brw_context *brw)
220 {
221    intel_batchbuffer_reset(brw);
222    brw_cache_sets_clear(brw);
223 }
224 
225 void
intel_batchbuffer_save_state(struct brw_context * brw)226 intel_batchbuffer_save_state(struct brw_context *brw)
227 {
228    brw->batch.saved.map_next = brw->batch.map_next;
229    brw->batch.saved.batch_reloc_count = brw->batch.batch_relocs.reloc_count;
230    brw->batch.saved.state_reloc_count = brw->batch.state_relocs.reloc_count;
231    brw->batch.saved.exec_count = brw->batch.exec_count;
232 }
233 
234 void
intel_batchbuffer_reset_to_saved(struct brw_context * brw)235 intel_batchbuffer_reset_to_saved(struct brw_context *brw)
236 {
237    for (int i = brw->batch.saved.exec_count;
238         i < brw->batch.exec_count; i++) {
239       brw_bo_unreference(brw->batch.exec_bos[i]);
240    }
241    brw->batch.batch_relocs.reloc_count = brw->batch.saved.batch_reloc_count;
242    brw->batch.state_relocs.reloc_count = brw->batch.saved.state_reloc_count;
243    brw->batch.exec_count = brw->batch.saved.exec_count;
244 
245    brw->batch.map_next = brw->batch.saved.map_next;
246    if (USED_BATCH(brw->batch) == 0)
247       brw->batch.ring = UNKNOWN_RING;
248 }
249 
250 void
intel_batchbuffer_free(struct intel_batchbuffer * batch)251 intel_batchbuffer_free(struct intel_batchbuffer *batch)
252 {
253    if (batch->use_shadow_copy) {
254       free(batch->batch.map);
255       free(batch->state.map);
256    }
257 
258    for (int i = 0; i < batch->exec_count; i++) {
259       brw_bo_unreference(batch->exec_bos[i]);
260    }
261    free(batch->batch_relocs.relocs);
262    free(batch->state_relocs.relocs);
263    free(batch->exec_bos);
264    free(batch->validation_list);
265 
266    brw_bo_unreference(batch->last_bo);
267    brw_bo_unreference(batch->batch.bo);
268    brw_bo_unreference(batch->state.bo);
269    if (batch->state_batch_sizes)
270       _mesa_hash_table_destroy(batch->state_batch_sizes, NULL);
271 }
272 
273 /**
274  * Finish copying the old batch/state buffer's contents to the new one
275  * after we tried to "grow" the buffer in an earlier operation.
276  */
277 static void
finish_growing_bos(struct brw_growing_bo * grow)278 finish_growing_bos(struct brw_growing_bo *grow)
279 {
280    struct brw_bo *old_bo = grow->partial_bo;
281    if (!old_bo)
282       return;
283 
284    memcpy(grow->map, grow->partial_bo_map, grow->partial_bytes);
285 
286    grow->partial_bo = NULL;
287    grow->partial_bo_map = NULL;
288    grow->partial_bytes = 0;
289 
290    brw_bo_unreference(old_bo);
291 }
292 
293 static void
replace_bo_in_reloc_list(struct brw_reloc_list * rlist,uint32_t old_handle,uint32_t new_handle)294 replace_bo_in_reloc_list(struct brw_reloc_list *rlist,
295                          uint32_t old_handle, uint32_t new_handle)
296 {
297    for (int i = 0; i < rlist->reloc_count; i++) {
298       if (rlist->relocs[i].target_handle == old_handle)
299          rlist->relocs[i].target_handle = new_handle;
300    }
301 }
302 
303 /**
304  * Grow either the batch or state buffer to a new larger size.
305  *
306  * We can't actually grow buffers, so we allocate a new one, copy over
307  * the existing contents, and update our lists to refer to the new one.
308  *
309  * Note that this is only temporary - each new batch recreates the buffers
310  * at their original target size (BATCH_SZ or STATE_SZ).
311  */
312 static void
grow_buffer(struct brw_context * brw,struct brw_growing_bo * grow,unsigned existing_bytes,unsigned new_size)313 grow_buffer(struct brw_context *brw,
314             struct brw_growing_bo *grow,
315             unsigned existing_bytes,
316             unsigned new_size)
317 {
318    struct intel_batchbuffer *batch = &brw->batch;
319    struct brw_bufmgr *bufmgr = brw->bufmgr;
320    struct brw_bo *bo = grow->bo;
321 
322    perf_debug("Growing %s - ran out of space\n", bo->name);
323 
324    if (grow->partial_bo) {
325       /* We've already grown once, and now we need to do it again.
326        * Finish our last grow operation so we can start a new one.
327        * This should basically never happen.
328        */
329       perf_debug("Had to grow multiple times");
330       finish_growing_bos(grow);
331    }
332 
333    struct brw_bo *new_bo = brw_bo_alloc(bufmgr, bo->name, new_size, bo->align);
334 
335    /* Copy existing data to the new larger buffer */
336    grow->partial_bo_map = grow->map;
337 
338    if (batch->use_shadow_copy) {
339       /* We can't safely use realloc, as it may move the existing buffer,
340        * breaking existing pointers the caller may still be using.  Just
341        * malloc a new copy and memcpy it like the normal BO path.
342        *
343        * Use bo->size rather than new_size because the bufmgr may have
344        * rounded up the size, and we want the shadow size to match.
345        */
346       grow->map = malloc(new_bo->size);
347    } else {
348       grow->map = brw_bo_map(brw, new_bo, MAP_READ | MAP_WRITE);
349    }
350 
351    /* Try to put the new BO at the same GTT offset as the old BO (which
352     * we're throwing away, so it doesn't need to be there).
353     *
354     * This guarantees that our relocations continue to work: values we've
355     * already written into the buffer, values we're going to write into the
356     * buffer, and the validation/relocation lists all will match.
357     *
358     * Also preserve kflags for EXEC_OBJECT_CAPTURE.
359     */
360    new_bo->gtt_offset = bo->gtt_offset;
361    new_bo->index = bo->index;
362    new_bo->kflags = bo->kflags;
363 
364    /* Batch/state buffers are per-context, and if we've run out of space,
365     * we must have actually used them before, so...they will be in the list.
366     */
367    assert(bo->index < batch->exec_count);
368    assert(batch->exec_bos[bo->index] == bo);
369 
370    /* Update the validation list to use the new BO. */
371    batch->validation_list[bo->index].handle = new_bo->gem_handle;
372 
373    if (!batch->use_batch_first) {
374       /* We're not using I915_EXEC_HANDLE_LUT, which means we need to go
375        * update the relocation list entries to point at the new BO as well.
376        * (With newer kernels, the "handle" is an offset into the validation
377        * list, which remains unchanged, so we can skip this.)
378        */
379       replace_bo_in_reloc_list(&batch->batch_relocs,
380                                bo->gem_handle, new_bo->gem_handle);
381       replace_bo_in_reloc_list(&batch->state_relocs,
382                                bo->gem_handle, new_bo->gem_handle);
383    }
384 
385    /* Exchange the two BOs...without breaking pointers to the old BO.
386     *
387     * Consider this scenario:
388     *
389     * 1. Somebody calls brw_state_batch() to get a region of memory, and
390     *    and then creates a brw_address pointing to brw->batch.state.bo.
391     * 2. They then call brw_state_batch() a second time, which happens to
392     *    grow and replace the state buffer.  They then try to emit a
393     *    relocation to their first section of memory.
394     *
395     * If we replace the brw->batch.state.bo pointer at step 2, we would
396     * break the address created in step 1.  They'd have a pointer to the
397     * old destroyed BO.  Emitting a relocation would add this dead BO to
398     * the validation list...causing /both/ statebuffers to be in the list,
399     * and all kinds of disasters.
400     *
401     * This is not a contrived case - BLORP vertex data upload hits this.
402     *
403     * There are worse scenarios too.  Fences for GL sync objects reference
404     * brw->batch.batch.bo.  If we replaced the batch pointer when growing,
405     * we'd need to chase down every fence and update it to point to the
406     * new BO.  Otherwise, it would refer to a "batch" that never actually
407     * gets submitted, and would fail to trigger.
408     *
409     * To work around both of these issues, we transmutate the buffers in
410     * place, making the existing struct brw_bo represent the new buffer,
411     * and "new_bo" represent the old BO.  This is highly unusual, but it
412     * seems like a necessary evil.
413     *
414     * We also defer the memcpy of the existing batch's contents.  Callers
415     * may make multiple brw_state_batch calls, and retain pointers to the
416     * old BO's map.  We'll perform the memcpy in finish_growing_bo() when
417     * we finally submit the batch, at which point we've finished uploading
418     * state, and nobody should have any old references anymore.
419     *
420     * To do that, we keep a reference to the old BO in grow->partial_bo,
421     * and store the number of bytes to copy in grow->partial_bytes.  We
422     * can monkey with the refcounts directly without atomics because these
423     * are per-context BOs and they can only be touched by this thread.
424     */
425    assert(new_bo->refcount == 1);
426    new_bo->refcount = bo->refcount;
427    bo->refcount = 1;
428 
429    struct brw_bo tmp;
430    memcpy(&tmp, bo, sizeof(struct brw_bo));
431    memcpy(bo, new_bo, sizeof(struct brw_bo));
432    memcpy(new_bo, &tmp, sizeof(struct brw_bo));
433 
434    grow->partial_bo = new_bo; /* the one reference of the OLD bo */
435    grow->partial_bytes = existing_bytes;
436 }
437 
438 void
intel_batchbuffer_require_space(struct brw_context * brw,GLuint sz,enum brw_gpu_ring ring)439 intel_batchbuffer_require_space(struct brw_context *brw, GLuint sz,
440                                 enum brw_gpu_ring ring)
441 {
442    const struct gen_device_info *devinfo = &brw->screen->devinfo;
443    struct intel_batchbuffer *batch = &brw->batch;
444 
445    /* If we're switching rings, implicitly flush the batch. */
446    if (unlikely(ring != brw->batch.ring) && brw->batch.ring != UNKNOWN_RING &&
447        devinfo->gen >= 6) {
448       intel_batchbuffer_flush(brw);
449    }
450 
451    const unsigned batch_used = USED_BATCH(*batch) * 4;
452    if (batch_used + sz >= BATCH_SZ && !batch->no_wrap) {
453       intel_batchbuffer_flush(brw);
454    } else if (batch_used + sz >= batch->batch.bo->size) {
455       const unsigned new_size =
456          MIN2(batch->batch.bo->size + batch->batch.bo->size / 2,
457               MAX_BATCH_SIZE);
458       grow_buffer(brw, &batch->batch, batch_used, new_size);
459       batch->map_next = (void *) batch->batch.map + batch_used;
460       assert(batch_used + sz < batch->batch.bo->size);
461    }
462 
463    /* The intel_batchbuffer_flush() calls above might have changed
464     * brw->batch.ring to UNKNOWN_RING, so we need to set it here at the end.
465     */
466    brw->batch.ring = ring;
467 }
468 
469 #ifdef DEBUG
470 #define CSI "\e["
471 #define BLUE_HEADER  CSI "0;44m"
472 #define NORMAL       CSI "0m"
473 
474 
475 static void
decode_struct(struct brw_context * brw,struct gen_spec * spec,const char * struct_name,uint32_t * data,uint32_t gtt_offset,uint32_t offset,bool color)476 decode_struct(struct brw_context *brw, struct gen_spec *spec,
477               const char *struct_name, uint32_t *data,
478               uint32_t gtt_offset, uint32_t offset, bool color)
479 {
480    struct gen_group *group = gen_spec_find_struct(spec, struct_name);
481    if (!group)
482       return;
483 
484    fprintf(stderr, "%s\n", struct_name);
485    gen_print_group(stderr, group, gtt_offset + offset,
486                    &data[offset / 4], 0, color);
487 }
488 
489 static void
decode_structs(struct brw_context * brw,struct gen_spec * spec,const char * struct_name,uint32_t * data,uint32_t gtt_offset,uint32_t offset,int struct_size,bool color)490 decode_structs(struct brw_context *brw, struct gen_spec *spec,
491                const char *struct_name,
492                uint32_t *data, uint32_t gtt_offset, uint32_t offset,
493                int struct_size, bool color)
494 {
495    struct gen_group *group = gen_spec_find_struct(spec, struct_name);
496    if (!group)
497       return;
498 
499    int entries = brw_state_batch_size(brw, offset) / struct_size;
500    for (int i = 0; i < entries; i++) {
501       fprintf(stderr, "%s %d\n", struct_name, i);
502       gen_print_group(stderr, group, gtt_offset + offset,
503                       &data[(offset + i * struct_size) / 4], 0, color);
504    }
505 }
506 
507 static void
do_batch_dump(struct brw_context * brw)508 do_batch_dump(struct brw_context *brw)
509 {
510    const struct gen_device_info *devinfo = &brw->screen->devinfo;
511    struct intel_batchbuffer *batch = &brw->batch;
512    struct gen_spec *spec = gen_spec_load(&brw->screen->devinfo);
513 
514    if (batch->ring != RENDER_RING)
515       return;
516 
517    uint32_t *batch_data = brw_bo_map(brw, batch->batch.bo, MAP_READ);
518    uint32_t *state = brw_bo_map(brw, batch->state.bo, MAP_READ);
519    if (batch_data == NULL || state == NULL) {
520       fprintf(stderr, "WARNING: failed to map batchbuffer/statebuffer\n");
521       return;
522    }
523 
524    uint32_t *end = batch_data + USED_BATCH(*batch);
525    uint32_t batch_gtt_offset = batch->batch.bo->gtt_offset;
526    uint32_t state_gtt_offset = batch->state.bo->gtt_offset;
527    int length;
528 
529    bool color = INTEL_DEBUG & DEBUG_COLOR;
530    const char *header_color = color ? BLUE_HEADER : "";
531    const char *reset_color  = color ? NORMAL : "";
532 
533    for (uint32_t *p = batch_data; p < end; p += length) {
534       struct gen_group *inst = gen_spec_find_instruction(spec, p);
535       length = gen_group_get_length(inst, p);
536       assert(inst == NULL || length > 0);
537       length = MAX2(1, length);
538       if (inst == NULL) {
539          fprintf(stderr, "unknown instruction %08x\n", p[0]);
540          continue;
541       }
542 
543       uint64_t offset = batch_gtt_offset + 4 * (p - batch_data);
544 
545       fprintf(stderr, "%s0x%08"PRIx64":  0x%08x:  %-80s%s\n", header_color,
546               offset, p[0], gen_group_get_name(inst), reset_color);
547 
548       gen_print_group(stderr, inst, offset, p, 0, color);
549 
550       switch (gen_group_get_opcode(inst) >> 16) {
551       case _3DSTATE_PIPELINED_POINTERS:
552          /* Note: these Gen4-5 pointers are full relocations rather than
553           * offsets from the start of the statebuffer.  So we need to subtract
554           * gtt_offset (the start of the statebuffer) to obtain an offset we
555           * can add to the map and get at the data.
556           */
557          decode_struct(brw, spec, "VS_STATE", state, state_gtt_offset,
558                        (p[1] & ~0x1fu) - state_gtt_offset, color);
559          if (p[2] & 1) {
560             decode_struct(brw, spec, "GS_STATE", state, state_gtt_offset,
561                           (p[2] & ~0x1fu) - state_gtt_offset, color);
562          }
563          if (p[3] & 1) {
564             decode_struct(brw, spec, "CLIP_STATE", state, state_gtt_offset,
565                           (p[3] & ~0x1fu) - state_gtt_offset, color);
566          }
567          decode_struct(brw, spec, "SF_STATE", state, state_gtt_offset,
568                        (p[4] & ~0x1fu) - state_gtt_offset, color);
569          decode_struct(brw, spec, "WM_STATE", state, state_gtt_offset,
570                        (p[5] & ~0x1fu) - state_gtt_offset, color);
571          decode_struct(brw, spec, "COLOR_CALC_STATE", state, state_gtt_offset,
572                        (p[6] & ~0x3fu) - state_gtt_offset, color);
573          break;
574       case _3DSTATE_BINDING_TABLE_POINTERS_VS:
575       case _3DSTATE_BINDING_TABLE_POINTERS_HS:
576       case _3DSTATE_BINDING_TABLE_POINTERS_DS:
577       case _3DSTATE_BINDING_TABLE_POINTERS_GS:
578       case _3DSTATE_BINDING_TABLE_POINTERS_PS: {
579          struct gen_group *group =
580             gen_spec_find_struct(spec, "RENDER_SURFACE_STATE");
581          if (!group)
582             break;
583 
584          uint32_t bt_offset = p[1] & ~0x1fu;
585          int bt_entries = brw_state_batch_size(brw, bt_offset) / 4;
586          uint32_t *bt_pointers = &state[bt_offset / 4];
587          for (int i = 0; i < bt_entries; i++) {
588             fprintf(stderr, "SURFACE_STATE - BTI = %d\n", i);
589             gen_print_group(stderr, group, state_gtt_offset + bt_pointers[i],
590                             &state[bt_pointers[i] / 4], 0, color);
591          }
592          break;
593       }
594       case _3DSTATE_SAMPLER_STATE_POINTERS_VS:
595       case _3DSTATE_SAMPLER_STATE_POINTERS_HS:
596       case _3DSTATE_SAMPLER_STATE_POINTERS_DS:
597       case _3DSTATE_SAMPLER_STATE_POINTERS_GS:
598       case _3DSTATE_SAMPLER_STATE_POINTERS_PS:
599          decode_structs(brw, spec, "SAMPLER_STATE", state,
600                         state_gtt_offset, p[1] & ~0x1fu, 4 * 4, color);
601          break;
602       case _3DSTATE_VIEWPORT_STATE_POINTERS:
603          decode_structs(brw, spec, "CLIP_VIEWPORT", state,
604                         state_gtt_offset, p[1] & ~0x3fu, 4 * 4, color);
605          decode_structs(brw, spec, "SF_VIEWPORT", state,
606                         state_gtt_offset, p[1] & ~0x3fu, 8 * 4, color);
607          decode_structs(brw, spec, "CC_VIEWPORT", state,
608                         state_gtt_offset, p[3] & ~0x3fu, 2 * 4, color);
609          break;
610       case _3DSTATE_VIEWPORT_STATE_POINTERS_CC:
611          decode_structs(brw, spec, "CC_VIEWPORT", state,
612                         state_gtt_offset, p[1] & ~0x3fu, 2 * 4, color);
613          break;
614       case _3DSTATE_VIEWPORT_STATE_POINTERS_SF_CL:
615          decode_structs(brw, spec, "SF_CLIP_VIEWPORT", state,
616                         state_gtt_offset, p[1] & ~0x3fu, 16 * 4, color);
617          break;
618       case _3DSTATE_SCISSOR_STATE_POINTERS:
619          decode_structs(brw, spec, "SCISSOR_RECT", state,
620                         state_gtt_offset, p[1] & ~0x1fu, 2 * 4, color);
621          break;
622       case _3DSTATE_BLEND_STATE_POINTERS:
623          /* TODO: handle Gen8+ extra dword at the beginning */
624          decode_structs(brw, spec, "BLEND_STATE", state,
625                         state_gtt_offset, p[1] & ~0x3fu, 8 * 4, color);
626          break;
627       case _3DSTATE_CC_STATE_POINTERS:
628          if (devinfo->gen >= 7) {
629             decode_struct(brw, spec, "COLOR_CALC_STATE", state,
630                           state_gtt_offset, p[1] & ~0x3fu, color);
631          } else if (devinfo->gen == 6) {
632             decode_structs(brw, spec, "BLEND_STATE", state,
633                            state_gtt_offset, p[1] & ~0x3fu, 2 * 4, color);
634             decode_struct(brw, spec, "DEPTH_STENCIL_STATE", state,
635                           state_gtt_offset, p[2] & ~0x3fu, color);
636             decode_struct(brw, spec, "COLOR_CALC_STATE", state,
637                           state_gtt_offset, p[3] & ~0x3fu, color);
638          }
639          break;
640       case _3DSTATE_DEPTH_STENCIL_STATE_POINTERS:
641          decode_struct(brw, spec, "DEPTH_STENCIL_STATE", state,
642                        state_gtt_offset, p[1] & ~0x3fu, color);
643          break;
644       case MEDIA_INTERFACE_DESCRIPTOR_LOAD: {
645          struct gen_group *group =
646             gen_spec_find_struct(spec, "RENDER_SURFACE_STATE");
647          if (!group)
648             break;
649 
650          uint32_t idd_offset = p[3] & ~0x1fu;
651          decode_struct(brw, spec, "INTERFACE_DESCRIPTOR_DATA", state,
652                        state_gtt_offset, idd_offset, color);
653 
654          uint32_t ss_offset = state[idd_offset / 4 + 3] & ~0x1fu;
655          decode_structs(brw, spec, "SAMPLER_STATE", state,
656                         state_gtt_offset, ss_offset, 4 * 4, color);
657 
658          uint32_t bt_offset = state[idd_offset / 4 + 4] & ~0x1fu;
659          int bt_entries = brw_state_batch_size(brw, bt_offset) / 4;
660          uint32_t *bt_pointers = &state[bt_offset / 4];
661          for (int i = 0; i < bt_entries; i++) {
662             fprintf(stderr, "SURFACE_STATE - BTI = %d\n", i);
663             gen_print_group(stderr, group, state_gtt_offset + bt_pointers[i],
664                             &state[bt_pointers[i] / 4], 0, color);
665          }
666          break;
667       }
668       }
669    }
670 
671    brw_bo_unmap(batch->batch.bo);
672    brw_bo_unmap(batch->state.bo);
673 }
674 #else
do_batch_dump(struct brw_context * brw)675 static void do_batch_dump(struct brw_context *brw) { }
676 #endif
677 
678 /**
679  * Called when starting a new batch buffer.
680  */
681 static void
brw_new_batch(struct brw_context * brw)682 brw_new_batch(struct brw_context *brw)
683 {
684    /* Unreference any BOs held by the previous batch, and reset counts. */
685    for (int i = 0; i < brw->batch.exec_count; i++) {
686       brw_bo_unreference(brw->batch.exec_bos[i]);
687       brw->batch.exec_bos[i] = NULL;
688    }
689    brw->batch.batch_relocs.reloc_count = 0;
690    brw->batch.state_relocs.reloc_count = 0;
691    brw->batch.exec_count = 0;
692    brw->batch.aperture_space = 0;
693 
694    brw_bo_unreference(brw->batch.state.bo);
695 
696    /* Create a new batchbuffer and reset the associated state: */
697    intel_batchbuffer_reset_and_clear_render_cache(brw);
698 
699    /* If the kernel supports hardware contexts, then most hardware state is
700     * preserved between batches; we only need to re-emit state that is required
701     * to be in every batch.  Otherwise we need to re-emit all the state that
702     * would otherwise be stored in the context (which for all intents and
703     * purposes means everything).
704     */
705    if (brw->hw_ctx == 0) {
706       brw->ctx.NewDriverState |= BRW_NEW_CONTEXT;
707       brw_upload_invariant_state(brw);
708    }
709 
710    brw->ctx.NewDriverState |= BRW_NEW_BATCH;
711 
712    brw->ib.index_size = -1;
713 
714    /* We need to periodically reap the shader time results, because rollover
715     * happens every few seconds.  We also want to see results every once in a
716     * while, because many programs won't cleanly destroy our context, so the
717     * end-of-run printout may not happen.
718     */
719    if (INTEL_DEBUG & DEBUG_SHADER_TIME)
720       brw_collect_and_report_shader_time(brw);
721 }
722 
723 /**
724  * Called from intel_batchbuffer_flush before emitting MI_BATCHBUFFER_END and
725  * sending it off.
726  *
727  * This function can emit state (say, to preserve registers that aren't saved
728  * between batches).
729  */
730 static void
brw_finish_batch(struct brw_context * brw)731 brw_finish_batch(struct brw_context *brw)
732 {
733    const struct gen_device_info *devinfo = &brw->screen->devinfo;
734 
735    brw->batch.no_wrap = true;
736 
737    /* Capture the closing pipeline statistics register values necessary to
738     * support query objects (in the non-hardware context world).
739     */
740    brw_emit_query_end(brw);
741 
742    if (brw->batch.ring == RENDER_RING) {
743       /* Work around L3 state leaks into contexts set MI_RESTORE_INHIBIT which
744        * assume that the L3 cache is configured according to the hardware
745        * defaults.
746        */
747       if (devinfo->gen >= 7)
748          gen7_restore_default_l3_config(brw);
749 
750       if (devinfo->is_haswell) {
751          /* From the Haswell PRM, Volume 2b, Command Reference: Instructions,
752           * 3DSTATE_CC_STATE_POINTERS > "Note":
753           *
754           * "SW must program 3DSTATE_CC_STATE_POINTERS command at the end of every
755           *  3D batch buffer followed by a PIPE_CONTROL with RC flush and CS stall."
756           *
757           * From the example in the docs, it seems to expect a regular pipe control
758           * flush here as well. We may have done it already, but meh.
759           *
760           * See also WaAvoidRCZCounterRollover.
761           */
762          brw_emit_mi_flush(brw);
763          BEGIN_BATCH(2);
764          OUT_BATCH(_3DSTATE_CC_STATE_POINTERS << 16 | (2 - 2));
765          OUT_BATCH(brw->cc.state_offset | 1);
766          ADVANCE_BATCH();
767          brw_emit_pipe_control_flush(brw, PIPE_CONTROL_RENDER_TARGET_FLUSH |
768                                           PIPE_CONTROL_CS_STALL);
769       }
770 
771       /* Do not restore push constant packets during context restore. */
772       if (devinfo->gen == 10)
773          gen10_emit_isp_disable(brw);
774    }
775 
776    /* Emit MI_BATCH_BUFFER_END to finish our batch.  Note that execbuf2
777     * requires our batch size to be QWord aligned, so we pad it out if
778     * necessary by emitting an extra MI_NOOP after the end.
779     */
780    intel_batchbuffer_require_space(brw, 8, brw->batch.ring);
781    *brw->batch.map_next++ = MI_BATCH_BUFFER_END;
782    if (USED_BATCH(brw->batch) & 1) {
783       *brw->batch.map_next++ = MI_NOOP;
784    }
785 
786    brw->batch.no_wrap = false;
787 }
788 
789 static void
throttle(struct brw_context * brw)790 throttle(struct brw_context *brw)
791 {
792    /* Wait for the swapbuffers before the one we just emitted, so we
793     * don't get too many swaps outstanding for apps that are GPU-heavy
794     * but not CPU-heavy.
795     *
796     * We're using intelDRI2Flush (called from the loader before
797     * swapbuffer) and glFlush (for front buffer rendering) as the
798     * indicator that a frame is done and then throttle when we get
799     * here as we prepare to render the next frame.  At this point for
800     * round trips for swap/copy and getting new buffers are done and
801     * we'll spend less time waiting on the GPU.
802     *
803     * Unfortunately, we don't have a handle to the batch containing
804     * the swap, and getting our hands on that doesn't seem worth it,
805     * so we just use the first batch we emitted after the last swap.
806     */
807    if (brw->need_swap_throttle && brw->throttle_batch[0]) {
808       if (brw->throttle_batch[1]) {
809          if (!brw->disable_throttling) {
810             /* Pass NULL rather than brw so we avoid perf_debug warnings;
811              * stalling is common and expected here...
812              */
813             brw_bo_wait_rendering(brw->throttle_batch[1]);
814          }
815          brw_bo_unreference(brw->throttle_batch[1]);
816       }
817       brw->throttle_batch[1] = brw->throttle_batch[0];
818       brw->throttle_batch[0] = NULL;
819       brw->need_swap_throttle = false;
820       /* Throttling here is more precise than the throttle ioctl, so skip it */
821       brw->need_flush_throttle = false;
822    }
823 
824    if (brw->need_flush_throttle) {
825       __DRIscreen *dri_screen = brw->screen->driScrnPriv;
826       drmCommandNone(dri_screen->fd, DRM_I915_GEM_THROTTLE);
827       brw->need_flush_throttle = false;
828    }
829 }
830 
831 static int
execbuffer(int fd,struct intel_batchbuffer * batch,uint32_t ctx_id,int used,int in_fence,int * out_fence,int flags)832 execbuffer(int fd,
833            struct intel_batchbuffer *batch,
834            uint32_t ctx_id,
835            int used,
836            int in_fence,
837            int *out_fence,
838            int flags)
839 {
840    struct drm_i915_gem_execbuffer2 execbuf = {
841       .buffers_ptr = (uintptr_t) batch->validation_list,
842       .buffer_count = batch->exec_count,
843       .batch_start_offset = 0,
844       .batch_len = used,
845       .flags = flags,
846       .rsvd1 = ctx_id, /* rsvd1 is actually the context ID */
847    };
848 
849    unsigned long cmd = DRM_IOCTL_I915_GEM_EXECBUFFER2;
850 
851    if (in_fence != -1) {
852       execbuf.rsvd2 = in_fence;
853       execbuf.flags |= I915_EXEC_FENCE_IN;
854    }
855 
856    if (out_fence != NULL) {
857       cmd = DRM_IOCTL_I915_GEM_EXECBUFFER2_WR;
858       *out_fence = -1;
859       execbuf.flags |= I915_EXEC_FENCE_OUT;
860    }
861 
862    int ret = drmIoctl(fd, cmd, &execbuf);
863    if (ret != 0)
864       ret = -errno;
865 
866    for (int i = 0; i < batch->exec_count; i++) {
867       struct brw_bo *bo = batch->exec_bos[i];
868 
869       bo->idle = false;
870       bo->index = -1;
871 
872       /* Update brw_bo::gtt_offset */
873       if (batch->validation_list[i].offset != bo->gtt_offset) {
874          DBG("BO %d migrated: 0x%" PRIx64 " -> 0x%llx\n",
875              bo->gem_handle, bo->gtt_offset,
876              batch->validation_list[i].offset);
877          bo->gtt_offset = batch->validation_list[i].offset;
878       }
879    }
880 
881    if (ret == 0 && out_fence != NULL)
882       *out_fence = execbuf.rsvd2 >> 32;
883 
884    return ret;
885 }
886 
887 static int
submit_batch(struct brw_context * brw,int in_fence_fd,int * out_fence_fd)888 submit_batch(struct brw_context *brw, int in_fence_fd, int *out_fence_fd)
889 {
890    const struct gen_device_info *devinfo = &brw->screen->devinfo;
891    __DRIscreen *dri_screen = brw->screen->driScrnPriv;
892    struct intel_batchbuffer *batch = &brw->batch;
893    int ret = 0;
894 
895    if (batch->use_shadow_copy) {
896       void *bo_map = brw_bo_map(brw, batch->batch.bo, MAP_WRITE);
897       memcpy(bo_map, batch->batch.map, 4 * USED_BATCH(*batch));
898 
899       bo_map = brw_bo_map(brw, batch->state.bo, MAP_WRITE);
900       memcpy(bo_map, batch->state.map, batch->state_used);
901    }
902 
903    brw_bo_unmap(batch->batch.bo);
904    brw_bo_unmap(batch->state.bo);
905 
906    if (!brw->screen->no_hw) {
907       /* The requirement for using I915_EXEC_NO_RELOC are:
908        *
909        *   The addresses written in the objects must match the corresponding
910        *   reloc.gtt_offset which in turn must match the corresponding
911        *   execobject.offset.
912        *
913        *   Any render targets written to in the batch must be flagged with
914        *   EXEC_OBJECT_WRITE.
915        *
916        *   To avoid stalling, execobject.offset should match the current
917        *   address of that object within the active context.
918        */
919       int flags = I915_EXEC_NO_RELOC;
920 
921       if (devinfo->gen >= 6 && batch->ring == BLT_RING) {
922          flags |= I915_EXEC_BLT;
923       } else {
924          flags |= I915_EXEC_RENDER;
925       }
926       if (batch->needs_sol_reset)
927          flags |= I915_EXEC_GEN7_SOL_RESET;
928 
929       uint32_t hw_ctx = batch->ring == RENDER_RING ? brw->hw_ctx : 0;
930 
931       /* Set statebuffer relocations */
932       const unsigned state_index = batch->state.bo->index;
933       if (state_index < batch->exec_count &&
934           batch->exec_bos[state_index] == batch->state.bo) {
935          struct drm_i915_gem_exec_object2 *entry =
936             &batch->validation_list[state_index];
937          assert(entry->handle == batch->state.bo->gem_handle);
938          entry->relocation_count = batch->state_relocs.reloc_count;
939          entry->relocs_ptr = (uintptr_t) batch->state_relocs.relocs;
940       }
941 
942       /* Set batchbuffer relocations */
943       struct drm_i915_gem_exec_object2 *entry = &batch->validation_list[0];
944       assert(entry->handle == batch->batch.bo->gem_handle);
945       entry->relocation_count = batch->batch_relocs.reloc_count;
946       entry->relocs_ptr = (uintptr_t) batch->batch_relocs.relocs;
947 
948       if (batch->use_batch_first) {
949          flags |= I915_EXEC_BATCH_FIRST | I915_EXEC_HANDLE_LUT;
950       } else {
951          /* Move the batch to the end of the validation list */
952          struct drm_i915_gem_exec_object2 tmp;
953          const unsigned index = batch->exec_count - 1;
954 
955          tmp = *entry;
956          *entry = batch->validation_list[index];
957          batch->validation_list[index] = tmp;
958       }
959 
960       ret = execbuffer(dri_screen->fd, batch, hw_ctx,
961                        4 * USED_BATCH(*batch),
962                        in_fence_fd, out_fence_fd, flags);
963 
964       throttle(brw);
965    }
966 
967    if (unlikely(INTEL_DEBUG & DEBUG_BATCH))
968       do_batch_dump(brw);
969 
970    if (brw->ctx.Const.ResetStrategy == GL_LOSE_CONTEXT_ON_RESET_ARB)
971       brw_check_for_reset(brw);
972 
973    if (ret != 0) {
974       fprintf(stderr, "i965: Failed to submit batchbuffer: %s\n",
975               strerror(-ret));
976       exit(1);
977    }
978 
979    return ret;
980 }
981 
982 /**
983  * The in_fence_fd is ignored if -1.  Otherwise this function takes ownership
984  * of the fd.
985  *
986  * The out_fence_fd is ignored if NULL. Otherwise, the caller takes ownership
987  * of the returned fd.
988  */
989 int
_intel_batchbuffer_flush_fence(struct brw_context * brw,int in_fence_fd,int * out_fence_fd,const char * file,int line)990 _intel_batchbuffer_flush_fence(struct brw_context *brw,
991                                int in_fence_fd, int *out_fence_fd,
992                                const char *file, int line)
993 {
994    int ret;
995 
996    if (USED_BATCH(brw->batch) == 0)
997       return 0;
998 
999    /* Check that we didn't just wrap our batchbuffer at a bad time. */
1000    assert(!brw->batch.no_wrap);
1001 
1002    brw_finish_batch(brw);
1003    intel_upload_finish(brw);
1004 
1005    finish_growing_bos(&brw->batch.batch);
1006    finish_growing_bos(&brw->batch.state);
1007 
1008    if (brw->throttle_batch[0] == NULL) {
1009       brw->throttle_batch[0] = brw->batch.batch.bo;
1010       brw_bo_reference(brw->throttle_batch[0]);
1011    }
1012 
1013    if (unlikely(INTEL_DEBUG & (DEBUG_BATCH | DEBUG_SUBMIT))) {
1014       int bytes_for_commands = 4 * USED_BATCH(brw->batch);
1015       int bytes_for_state = brw->batch.state_used;
1016       fprintf(stderr, "%19s:%-3d: Batchbuffer flush with %5db (%0.1f%%) (pkt),"
1017               " %5db (%0.1f%%) (state), %4d BOs (%0.1fMb aperture),"
1018               " %4d batch relocs, %4d state relocs\n", file, line,
1019               bytes_for_commands, 100.0f * bytes_for_commands / BATCH_SZ,
1020               bytes_for_state, 100.0f * bytes_for_state / STATE_SZ,
1021               brw->batch.exec_count,
1022               (float) brw->batch.aperture_space / (1024 * 1024),
1023               brw->batch.batch_relocs.reloc_count,
1024               brw->batch.state_relocs.reloc_count);
1025    }
1026 
1027    ret = submit_batch(brw, in_fence_fd, out_fence_fd);
1028 
1029    if (unlikely(INTEL_DEBUG & DEBUG_SYNC)) {
1030       fprintf(stderr, "waiting for idle\n");
1031       brw_bo_wait_rendering(brw->batch.batch.bo);
1032    }
1033 
1034    /* Start a new batch buffer. */
1035    brw_new_batch(brw);
1036 
1037    return ret;
1038 }
1039 
1040 bool
brw_batch_has_aperture_space(struct brw_context * brw,unsigned extra_space)1041 brw_batch_has_aperture_space(struct brw_context *brw, unsigned extra_space)
1042 {
1043    return brw->batch.aperture_space + extra_space <=
1044           brw->screen->aperture_threshold;
1045 }
1046 
1047 bool
brw_batch_references(struct intel_batchbuffer * batch,struct brw_bo * bo)1048 brw_batch_references(struct intel_batchbuffer *batch, struct brw_bo *bo)
1049 {
1050    unsigned index = READ_ONCE(bo->index);
1051    if (index < batch->exec_count && batch->exec_bos[index] == bo)
1052       return true;
1053 
1054    for (int i = 0; i < batch->exec_count; i++) {
1055       if (batch->exec_bos[i] == bo)
1056          return true;
1057    }
1058    return false;
1059 }
1060 
1061 /*  This is the only way buffers get added to the validate list.
1062  */
1063 static uint64_t
emit_reloc(struct intel_batchbuffer * batch,struct brw_reloc_list * rlist,uint32_t offset,struct brw_bo * target,int32_t target_offset,unsigned int reloc_flags)1064 emit_reloc(struct intel_batchbuffer *batch,
1065            struct brw_reloc_list *rlist, uint32_t offset,
1066            struct brw_bo *target, int32_t target_offset,
1067            unsigned int reloc_flags)
1068 {
1069    assert(target != NULL);
1070 
1071    if (rlist->reloc_count == rlist->reloc_array_size) {
1072       rlist->reloc_array_size *= 2;
1073       rlist->relocs = realloc(rlist->relocs,
1074                               rlist->reloc_array_size *
1075                               sizeof(struct drm_i915_gem_relocation_entry));
1076    }
1077 
1078    unsigned int index = add_exec_bo(batch, target);
1079    struct drm_i915_gem_exec_object2 *entry = &batch->validation_list[index];
1080 
1081    if (reloc_flags)
1082       entry->flags |= reloc_flags & batch->valid_reloc_flags;
1083 
1084    rlist->relocs[rlist->reloc_count++] =
1085       (struct drm_i915_gem_relocation_entry) {
1086          .offset = offset,
1087          .delta = target_offset,
1088          .target_handle = batch->use_batch_first ? index : target->gem_handle,
1089          .presumed_offset = entry->offset,
1090       };
1091 
1092    /* Using the old buffer offset, write in what the right data would be, in
1093     * case the buffer doesn't move and we can short-circuit the relocation
1094     * processing in the kernel
1095     */
1096    return entry->offset + target_offset;
1097 }
1098 
1099 uint64_t
brw_batch_reloc(struct intel_batchbuffer * batch,uint32_t batch_offset,struct brw_bo * target,uint32_t target_offset,unsigned int reloc_flags)1100 brw_batch_reloc(struct intel_batchbuffer *batch, uint32_t batch_offset,
1101                 struct brw_bo *target, uint32_t target_offset,
1102                 unsigned int reloc_flags)
1103 {
1104    assert(batch_offset <= batch->batch.bo->size - sizeof(uint32_t));
1105 
1106    return emit_reloc(batch, &batch->batch_relocs, batch_offset,
1107                      target, target_offset, reloc_flags);
1108 }
1109 
1110 uint64_t
brw_state_reloc(struct intel_batchbuffer * batch,uint32_t state_offset,struct brw_bo * target,uint32_t target_offset,unsigned int reloc_flags)1111 brw_state_reloc(struct intel_batchbuffer *batch, uint32_t state_offset,
1112                 struct brw_bo *target, uint32_t target_offset,
1113                 unsigned int reloc_flags)
1114 {
1115    assert(state_offset <= batch->state.bo->size - sizeof(uint32_t));
1116 
1117    return emit_reloc(batch, &batch->state_relocs, state_offset,
1118                      target, target_offset, reloc_flags);
1119 }
1120 
1121 
1122 uint32_t
brw_state_batch_size(struct brw_context * brw,uint32_t offset)1123 brw_state_batch_size(struct brw_context *brw, uint32_t offset)
1124 {
1125    struct hash_entry *entry =
1126       _mesa_hash_table_search(brw->batch.state_batch_sizes,
1127                               (void *) (uintptr_t) offset);
1128    return entry ? (uintptr_t) entry->data : 0;
1129 }
1130 
1131 /**
1132  * Reserve some space in the statebuffer, or flush.
1133  *
1134  * This is used to estimate when we're near the end of the batch,
1135  * so we can flush early.
1136  */
1137 void
brw_require_statebuffer_space(struct brw_context * brw,int size)1138 brw_require_statebuffer_space(struct brw_context *brw, int size)
1139 {
1140    if (brw->batch.state_used + size >= STATE_SZ)
1141       intel_batchbuffer_flush(brw);
1142 }
1143 
1144 /**
1145  * Allocates a block of space in the batchbuffer for indirect state.
1146  */
1147 void *
brw_state_batch(struct brw_context * brw,int size,int alignment,uint32_t * out_offset)1148 brw_state_batch(struct brw_context *brw,
1149                 int size,
1150                 int alignment,
1151                 uint32_t *out_offset)
1152 {
1153    struct intel_batchbuffer *batch = &brw->batch;
1154 
1155    assert(size < batch->state.bo->size);
1156 
1157    uint32_t offset = ALIGN(batch->state_used, alignment);
1158 
1159    if (offset + size >= STATE_SZ && !batch->no_wrap) {
1160       intel_batchbuffer_flush(brw);
1161       offset = ALIGN(batch->state_used, alignment);
1162    } else if (offset + size >= batch->state.bo->size) {
1163       const unsigned new_size =
1164          MIN2(batch->state.bo->size + batch->state.bo->size / 2,
1165               MAX_STATE_SIZE);
1166       grow_buffer(brw, &batch->state, batch->state_used, new_size);
1167       assert(offset + size < batch->state.bo->size);
1168    }
1169 
1170    if (unlikely(INTEL_DEBUG & DEBUG_BATCH)) {
1171       _mesa_hash_table_insert(batch->state_batch_sizes,
1172                               (void *) (uintptr_t) offset,
1173                               (void *) (uintptr_t) size);
1174    }
1175 
1176    batch->state_used = offset + size;
1177 
1178    *out_offset = offset;
1179    return batch->state.map + (offset >> 2);
1180 }
1181 
1182 void
intel_batchbuffer_data(struct brw_context * brw,const void * data,GLuint bytes,enum brw_gpu_ring ring)1183 intel_batchbuffer_data(struct brw_context *brw,
1184                        const void *data, GLuint bytes, enum brw_gpu_ring ring)
1185 {
1186    assert((bytes & 3) == 0);
1187    intel_batchbuffer_require_space(brw, bytes, ring);
1188    memcpy(brw->batch.map_next, data, bytes);
1189    brw->batch.map_next += bytes >> 2;
1190 }
1191 
1192 static void
load_sized_register_mem(struct brw_context * brw,uint32_t reg,struct brw_bo * bo,uint32_t offset,int size)1193 load_sized_register_mem(struct brw_context *brw,
1194                         uint32_t reg,
1195                         struct brw_bo *bo,
1196                         uint32_t offset,
1197                         int size)
1198 {
1199    const struct gen_device_info *devinfo = &brw->screen->devinfo;
1200    int i;
1201 
1202    /* MI_LOAD_REGISTER_MEM only exists on Gen7+. */
1203    assert(devinfo->gen >= 7);
1204 
1205    if (devinfo->gen >= 8) {
1206       BEGIN_BATCH(4 * size);
1207       for (i = 0; i < size; i++) {
1208          OUT_BATCH(GEN7_MI_LOAD_REGISTER_MEM | (4 - 2));
1209          OUT_BATCH(reg + i * 4);
1210          OUT_RELOC64(bo, 0, offset + i * 4);
1211       }
1212       ADVANCE_BATCH();
1213    } else {
1214       BEGIN_BATCH(3 * size);
1215       for (i = 0; i < size; i++) {
1216          OUT_BATCH(GEN7_MI_LOAD_REGISTER_MEM | (3 - 2));
1217          OUT_BATCH(reg + i * 4);
1218          OUT_RELOC(bo, 0, offset + i * 4);
1219       }
1220       ADVANCE_BATCH();
1221    }
1222 }
1223 
1224 void
brw_load_register_mem(struct brw_context * brw,uint32_t reg,struct brw_bo * bo,uint32_t offset)1225 brw_load_register_mem(struct brw_context *brw,
1226                       uint32_t reg,
1227                       struct brw_bo *bo,
1228                       uint32_t offset)
1229 {
1230    load_sized_register_mem(brw, reg, bo, offset, 1);
1231 }
1232 
1233 void
brw_load_register_mem64(struct brw_context * brw,uint32_t reg,struct brw_bo * bo,uint32_t offset)1234 brw_load_register_mem64(struct brw_context *brw,
1235                         uint32_t reg,
1236                         struct brw_bo *bo,
1237                         uint32_t offset)
1238 {
1239    load_sized_register_mem(brw, reg, bo, offset, 2);
1240 }
1241 
1242 /*
1243  * Write an arbitrary 32-bit register to a buffer via MI_STORE_REGISTER_MEM.
1244  */
1245 void
brw_store_register_mem32(struct brw_context * brw,struct brw_bo * bo,uint32_t reg,uint32_t offset)1246 brw_store_register_mem32(struct brw_context *brw,
1247                          struct brw_bo *bo, uint32_t reg, uint32_t offset)
1248 {
1249    const struct gen_device_info *devinfo = &brw->screen->devinfo;
1250 
1251    assert(devinfo->gen >= 6);
1252 
1253    if (devinfo->gen >= 8) {
1254       BEGIN_BATCH(4);
1255       OUT_BATCH(MI_STORE_REGISTER_MEM | (4 - 2));
1256       OUT_BATCH(reg);
1257       OUT_RELOC64(bo, RELOC_WRITE, offset);
1258       ADVANCE_BATCH();
1259    } else {
1260       BEGIN_BATCH(3);
1261       OUT_BATCH(MI_STORE_REGISTER_MEM | (3 - 2));
1262       OUT_BATCH(reg);
1263       OUT_RELOC(bo, RELOC_WRITE | RELOC_NEEDS_GGTT, offset);
1264       ADVANCE_BATCH();
1265    }
1266 }
1267 
1268 /*
1269  * Write an arbitrary 64-bit register to a buffer via MI_STORE_REGISTER_MEM.
1270  */
1271 void
brw_store_register_mem64(struct brw_context * brw,struct brw_bo * bo,uint32_t reg,uint32_t offset)1272 brw_store_register_mem64(struct brw_context *brw,
1273                          struct brw_bo *bo, uint32_t reg, uint32_t offset)
1274 {
1275    const struct gen_device_info *devinfo = &brw->screen->devinfo;
1276 
1277    assert(devinfo->gen >= 6);
1278 
1279    /* MI_STORE_REGISTER_MEM only stores a single 32-bit value, so to
1280     * read a full 64-bit register, we need to do two of them.
1281     */
1282    if (devinfo->gen >= 8) {
1283       BEGIN_BATCH(8);
1284       OUT_BATCH(MI_STORE_REGISTER_MEM | (4 - 2));
1285       OUT_BATCH(reg);
1286       OUT_RELOC64(bo, RELOC_WRITE, offset);
1287       OUT_BATCH(MI_STORE_REGISTER_MEM | (4 - 2));
1288       OUT_BATCH(reg + sizeof(uint32_t));
1289       OUT_RELOC64(bo, RELOC_WRITE, offset + sizeof(uint32_t));
1290       ADVANCE_BATCH();
1291    } else {
1292       BEGIN_BATCH(6);
1293       OUT_BATCH(MI_STORE_REGISTER_MEM | (3 - 2));
1294       OUT_BATCH(reg);
1295       OUT_RELOC(bo, RELOC_WRITE | RELOC_NEEDS_GGTT, offset);
1296       OUT_BATCH(MI_STORE_REGISTER_MEM | (3 - 2));
1297       OUT_BATCH(reg + sizeof(uint32_t));
1298       OUT_RELOC(bo, RELOC_WRITE | RELOC_NEEDS_GGTT, offset + sizeof(uint32_t));
1299       ADVANCE_BATCH();
1300    }
1301 }
1302 
1303 /*
1304  * Write a 32-bit register using immediate data.
1305  */
1306 void
brw_load_register_imm32(struct brw_context * brw,uint32_t reg,uint32_t imm)1307 brw_load_register_imm32(struct brw_context *brw, uint32_t reg, uint32_t imm)
1308 {
1309    assert(brw->screen->devinfo.gen >= 6);
1310 
1311    BEGIN_BATCH(3);
1312    OUT_BATCH(MI_LOAD_REGISTER_IMM | (3 - 2));
1313    OUT_BATCH(reg);
1314    OUT_BATCH(imm);
1315    ADVANCE_BATCH();
1316 }
1317 
1318 /*
1319  * Write a 64-bit register using immediate data.
1320  */
1321 void
brw_load_register_imm64(struct brw_context * brw,uint32_t reg,uint64_t imm)1322 brw_load_register_imm64(struct brw_context *brw, uint32_t reg, uint64_t imm)
1323 {
1324    assert(brw->screen->devinfo.gen >= 6);
1325 
1326    BEGIN_BATCH(5);
1327    OUT_BATCH(MI_LOAD_REGISTER_IMM | (5 - 2));
1328    OUT_BATCH(reg);
1329    OUT_BATCH(imm & 0xffffffff);
1330    OUT_BATCH(reg + 4);
1331    OUT_BATCH(imm >> 32);
1332    ADVANCE_BATCH();
1333 }
1334 
1335 /*
1336  * Copies a 32-bit register.
1337  */
1338 void
brw_load_register_reg(struct brw_context * brw,uint32_t src,uint32_t dest)1339 brw_load_register_reg(struct brw_context *brw, uint32_t src, uint32_t dest)
1340 {
1341    assert(brw->screen->devinfo.gen >= 8 || brw->screen->devinfo.is_haswell);
1342 
1343    BEGIN_BATCH(3);
1344    OUT_BATCH(MI_LOAD_REGISTER_REG | (3 - 2));
1345    OUT_BATCH(src);
1346    OUT_BATCH(dest);
1347    ADVANCE_BATCH();
1348 }
1349 
1350 /*
1351  * Copies a 64-bit register.
1352  */
1353 void
brw_load_register_reg64(struct brw_context * brw,uint32_t src,uint32_t dest)1354 brw_load_register_reg64(struct brw_context *brw, uint32_t src, uint32_t dest)
1355 {
1356    assert(brw->screen->devinfo.gen >= 8 || brw->screen->devinfo.is_haswell);
1357 
1358    BEGIN_BATCH(6);
1359    OUT_BATCH(MI_LOAD_REGISTER_REG | (3 - 2));
1360    OUT_BATCH(src);
1361    OUT_BATCH(dest);
1362    OUT_BATCH(MI_LOAD_REGISTER_REG | (3 - 2));
1363    OUT_BATCH(src + sizeof(uint32_t));
1364    OUT_BATCH(dest + sizeof(uint32_t));
1365    ADVANCE_BATCH();
1366 }
1367 
1368 /*
1369  * Write 32-bits of immediate data to a GPU memory buffer.
1370  */
1371 void
brw_store_data_imm32(struct brw_context * brw,struct brw_bo * bo,uint32_t offset,uint32_t imm)1372 brw_store_data_imm32(struct brw_context *brw, struct brw_bo *bo,
1373                      uint32_t offset, uint32_t imm)
1374 {
1375    const struct gen_device_info *devinfo = &brw->screen->devinfo;
1376 
1377    assert(devinfo->gen >= 6);
1378 
1379    BEGIN_BATCH(4);
1380    OUT_BATCH(MI_STORE_DATA_IMM | (4 - 2));
1381    if (devinfo->gen >= 8)
1382       OUT_RELOC64(bo, RELOC_WRITE, offset);
1383    else {
1384       OUT_BATCH(0); /* MBZ */
1385       OUT_RELOC(bo, RELOC_WRITE, offset);
1386    }
1387    OUT_BATCH(imm);
1388    ADVANCE_BATCH();
1389 }
1390 
1391 /*
1392  * Write 64-bits of immediate data to a GPU memory buffer.
1393  */
1394 void
brw_store_data_imm64(struct brw_context * brw,struct brw_bo * bo,uint32_t offset,uint64_t imm)1395 brw_store_data_imm64(struct brw_context *brw, struct brw_bo *bo,
1396                      uint32_t offset, uint64_t imm)
1397 {
1398    const struct gen_device_info *devinfo = &brw->screen->devinfo;
1399 
1400    assert(devinfo->gen >= 6);
1401 
1402    BEGIN_BATCH(5);
1403    OUT_BATCH(MI_STORE_DATA_IMM | (5 - 2));
1404    if (devinfo->gen >= 8)
1405       OUT_RELOC64(bo, RELOC_WRITE, offset);
1406    else {
1407       OUT_BATCH(0); /* MBZ */
1408       OUT_RELOC(bo, RELOC_WRITE, offset);
1409    }
1410    OUT_BATCH(imm & 0xffffffffu);
1411    OUT_BATCH(imm >> 32);
1412    ADVANCE_BATCH();
1413 }
1414