1 /*
2 * Copyright © 2010 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 */
23
24 #include "brw_context.h"
25 #include "brw_defines.h"
26 #include "intel_batchbuffer.h"
27 #include "intel_fbo.h"
28
29 /**
30 * According to the latest documentation, any PIPE_CONTROL with the
31 * "Command Streamer Stall" bit set must also have another bit set,
32 * with five different options:
33 *
34 * - Render Target Cache Flush
35 * - Depth Cache Flush
36 * - Stall at Pixel Scoreboard
37 * - Post-Sync Operation
38 * - Depth Stall
39 * - DC Flush Enable
40 *
41 * I chose "Stall at Pixel Scoreboard" since we've used it effectively
42 * in the past, but the choice is fairly arbitrary.
43 */
44 static void
gen8_add_cs_stall_workaround_bits(uint32_t * flags)45 gen8_add_cs_stall_workaround_bits(uint32_t *flags)
46 {
47 uint32_t wa_bits = PIPE_CONTROL_RENDER_TARGET_FLUSH |
48 PIPE_CONTROL_DEPTH_CACHE_FLUSH |
49 PIPE_CONTROL_WRITE_IMMEDIATE |
50 PIPE_CONTROL_WRITE_DEPTH_COUNT |
51 PIPE_CONTROL_WRITE_TIMESTAMP |
52 PIPE_CONTROL_STALL_AT_SCOREBOARD |
53 PIPE_CONTROL_DEPTH_STALL |
54 PIPE_CONTROL_DATA_CACHE_FLUSH;
55
56 /* If we're doing a CS stall, and don't already have one of the
57 * workaround bits set, add "Stall at Pixel Scoreboard."
58 */
59 if ((*flags & PIPE_CONTROL_CS_STALL) != 0 && (*flags & wa_bits) == 0)
60 *flags |= PIPE_CONTROL_STALL_AT_SCOREBOARD;
61 }
62
63 /* Implement the WaCsStallAtEveryFourthPipecontrol workaround on IVB, BYT:
64 *
65 * "Every 4th PIPE_CONTROL command, not counting the PIPE_CONTROL with
66 * only read-cache-invalidate bit(s) set, must have a CS_STALL bit set."
67 *
68 * Note that the kernel does CS stalls between batches, so we only need
69 * to count them within a batch.
70 */
71 static uint32_t
gen7_cs_stall_every_four_pipe_controls(struct brw_context * brw,uint32_t flags)72 gen7_cs_stall_every_four_pipe_controls(struct brw_context *brw, uint32_t flags)
73 {
74 if (brw->gen == 7 && !brw->is_haswell) {
75 if (flags & PIPE_CONTROL_CS_STALL) {
76 /* If we're doing a CS stall, reset the counter and carry on. */
77 brw->pipe_controls_since_last_cs_stall = 0;
78 return 0;
79 }
80
81 /* If this is the fourth pipe control without a CS stall, do one now. */
82 if (++brw->pipe_controls_since_last_cs_stall == 4) {
83 brw->pipe_controls_since_last_cs_stall = 0;
84 return PIPE_CONTROL_CS_STALL;
85 }
86 }
87 return 0;
88 }
89
90 /**
91 * Emit a PIPE_CONTROL with various flushing flags.
92 *
93 * The caller is responsible for deciding what flags are appropriate for the
94 * given generation.
95 */
96 void
brw_emit_pipe_control_flush(struct brw_context * brw,uint32_t flags)97 brw_emit_pipe_control_flush(struct brw_context *brw, uint32_t flags)
98 {
99 if (brw->gen >= 6 &&
100 (flags & PIPE_CONTROL_CACHE_FLUSH_BITS) &&
101 (flags & PIPE_CONTROL_CACHE_INVALIDATE_BITS)) {
102 /* A pipe control command with flush and invalidate bits set
103 * simultaneously is an inherently racy operation on Gen6+ if the
104 * contents of the flushed caches were intended to become visible from
105 * any of the invalidated caches. Split it in two PIPE_CONTROLs, the
106 * first one should stall the pipeline to make sure that the flushed R/W
107 * caches are coherent with memory once the specified R/O caches are
108 * invalidated. On pre-Gen6 hardware the (implicit) R/O cache
109 * invalidation seems to happen at the bottom of the pipeline together
110 * with any write cache flush, so this shouldn't be a concern.
111 */
112 brw_emit_pipe_control_flush(brw, (flags & PIPE_CONTROL_CACHE_FLUSH_BITS) |
113 PIPE_CONTROL_CS_STALL);
114 flags &= ~(PIPE_CONTROL_CACHE_FLUSH_BITS | PIPE_CONTROL_CS_STALL);
115 }
116
117 if (brw->gen >= 8) {
118 if (brw->gen == 8)
119 gen8_add_cs_stall_workaround_bits(&flags);
120
121 if (brw->gen == 9 &&
122 (flags & PIPE_CONTROL_VF_CACHE_INVALIDATE)) {
123 /* Hardware workaround: SKL
124 *
125 * Emit Pipe Control with all bits set to zero before emitting
126 * a Pipe Control with VF Cache Invalidate set.
127 */
128 brw_emit_pipe_control_flush(brw, 0);
129 }
130
131 BEGIN_BATCH(6);
132 OUT_BATCH(_3DSTATE_PIPE_CONTROL | (6 - 2));
133 OUT_BATCH(flags);
134 OUT_BATCH(0);
135 OUT_BATCH(0);
136 OUT_BATCH(0);
137 OUT_BATCH(0);
138 ADVANCE_BATCH();
139 } else if (brw->gen >= 6) {
140 if (brw->gen == 6 &&
141 (flags & PIPE_CONTROL_RENDER_TARGET_FLUSH)) {
142 /* Hardware workaround: SNB B-Spec says:
143 *
144 * [Dev-SNB{W/A}]: Before a PIPE_CONTROL with Write Cache Flush
145 * Enable = 1, a PIPE_CONTROL with any non-zero post-sync-op is
146 * required.
147 */
148 brw_emit_post_sync_nonzero_flush(brw);
149 }
150
151 flags |= gen7_cs_stall_every_four_pipe_controls(brw, flags);
152
153 BEGIN_BATCH(5);
154 OUT_BATCH(_3DSTATE_PIPE_CONTROL | (5 - 2));
155 OUT_BATCH(flags);
156 OUT_BATCH(0);
157 OUT_BATCH(0);
158 OUT_BATCH(0);
159 ADVANCE_BATCH();
160 } else {
161 BEGIN_BATCH(4);
162 OUT_BATCH(_3DSTATE_PIPE_CONTROL | flags | (4 - 2));
163 OUT_BATCH(0);
164 OUT_BATCH(0);
165 OUT_BATCH(0);
166 ADVANCE_BATCH();
167 }
168 }
169
170 /**
171 * Emit a PIPE_CONTROL that writes to a buffer object.
172 *
173 * \p flags should contain one of the following items:
174 * - PIPE_CONTROL_WRITE_IMMEDIATE
175 * - PIPE_CONTROL_WRITE_TIMESTAMP
176 * - PIPE_CONTROL_WRITE_DEPTH_COUNT
177 */
178 void
brw_emit_pipe_control_write(struct brw_context * brw,uint32_t flags,drm_intel_bo * bo,uint32_t offset,uint32_t imm_lower,uint32_t imm_upper)179 brw_emit_pipe_control_write(struct brw_context *brw, uint32_t flags,
180 drm_intel_bo *bo, uint32_t offset,
181 uint32_t imm_lower, uint32_t imm_upper)
182 {
183 if (brw->gen >= 8) {
184 if (brw->gen == 8)
185 gen8_add_cs_stall_workaround_bits(&flags);
186
187 BEGIN_BATCH(6);
188 OUT_BATCH(_3DSTATE_PIPE_CONTROL | (6 - 2));
189 OUT_BATCH(flags);
190 OUT_RELOC64(bo, I915_GEM_DOMAIN_INSTRUCTION, I915_GEM_DOMAIN_INSTRUCTION,
191 offset);
192 OUT_BATCH(imm_lower);
193 OUT_BATCH(imm_upper);
194 ADVANCE_BATCH();
195 } else if (brw->gen >= 6) {
196 flags |= gen7_cs_stall_every_four_pipe_controls(brw, flags);
197
198 /* PPGTT/GGTT is selected by DW2 bit 2 on Sandybridge, but DW1 bit 24
199 * on later platforms. We always use PPGTT on Gen7+.
200 */
201 unsigned gen6_gtt = brw->gen == 6 ? PIPE_CONTROL_GLOBAL_GTT_WRITE : 0;
202
203 BEGIN_BATCH(5);
204 OUT_BATCH(_3DSTATE_PIPE_CONTROL | (5 - 2));
205 OUT_BATCH(flags);
206 OUT_RELOC(bo, I915_GEM_DOMAIN_INSTRUCTION, I915_GEM_DOMAIN_INSTRUCTION,
207 gen6_gtt | offset);
208 OUT_BATCH(imm_lower);
209 OUT_BATCH(imm_upper);
210 ADVANCE_BATCH();
211 } else {
212 BEGIN_BATCH(4);
213 OUT_BATCH(_3DSTATE_PIPE_CONTROL | flags | (4 - 2));
214 OUT_RELOC(bo, I915_GEM_DOMAIN_INSTRUCTION, I915_GEM_DOMAIN_INSTRUCTION,
215 PIPE_CONTROL_GLOBAL_GTT_WRITE | offset);
216 OUT_BATCH(imm_lower);
217 OUT_BATCH(imm_upper);
218 ADVANCE_BATCH();
219 }
220 }
221
222 /**
223 * Restriction [DevSNB, DevIVB]:
224 *
225 * Prior to changing Depth/Stencil Buffer state (i.e. any combination of
226 * 3DSTATE_DEPTH_BUFFER, 3DSTATE_CLEAR_PARAMS, 3DSTATE_STENCIL_BUFFER,
227 * 3DSTATE_HIER_DEPTH_BUFFER) SW must first issue a pipelined depth stall
228 * (PIPE_CONTROL with Depth Stall bit set), followed by a pipelined depth
229 * cache flush (PIPE_CONTROL with Depth Flush Bit set), followed by
230 * another pipelined depth stall (PIPE_CONTROL with Depth Stall bit set),
231 * unless SW can otherwise guarantee that the pipeline from WM onwards is
232 * already flushed (e.g., via a preceding MI_FLUSH).
233 */
234 void
brw_emit_depth_stall_flushes(struct brw_context * brw)235 brw_emit_depth_stall_flushes(struct brw_context *brw)
236 {
237 assert(brw->gen >= 6);
238
239 /* Starting on BDW, these pipe controls are unnecessary.
240 *
241 * WM HW will internally manage the draining pipe and flushing of the caches
242 * when this command is issued. The PIPE_CONTROL restrictions are removed.
243 */
244 if (brw->gen >= 8)
245 return;
246
247 brw_emit_pipe_control_flush(brw, PIPE_CONTROL_DEPTH_STALL);
248 brw_emit_pipe_control_flush(brw, PIPE_CONTROL_DEPTH_CACHE_FLUSH);
249 brw_emit_pipe_control_flush(brw, PIPE_CONTROL_DEPTH_STALL);
250 }
251
252 /**
253 * From the Ivybridge PRM, Volume 2 Part 1, Section 3.2 (VS Stage Input):
254 * "A PIPE_CONTROL with Post-Sync Operation set to 1h and a depth
255 * stall needs to be sent just prior to any 3DSTATE_VS, 3DSTATE_URB_VS,
256 * 3DSTATE_CONSTANT_VS, 3DSTATE_BINDING_TABLE_POINTER_VS,
257 * 3DSTATE_SAMPLER_STATE_POINTER_VS command. Only one PIPE_CONTROL needs
258 * to be sent before any combination of VS associated 3DSTATE."
259 */
260 void
gen7_emit_vs_workaround_flush(struct brw_context * brw)261 gen7_emit_vs_workaround_flush(struct brw_context *brw)
262 {
263 assert(brw->gen == 7);
264 brw_emit_pipe_control_write(brw,
265 PIPE_CONTROL_WRITE_IMMEDIATE
266 | PIPE_CONTROL_DEPTH_STALL,
267 brw->workaround_bo, 0,
268 0, 0);
269 }
270
271
272 /**
273 * Emit a PIPE_CONTROL command for gen7 with the CS Stall bit set.
274 */
275 void
gen7_emit_cs_stall_flush(struct brw_context * brw)276 gen7_emit_cs_stall_flush(struct brw_context *brw)
277 {
278 brw_emit_pipe_control_write(brw,
279 PIPE_CONTROL_CS_STALL
280 | PIPE_CONTROL_WRITE_IMMEDIATE,
281 brw->workaround_bo, 0,
282 0, 0);
283 }
284
285
286 /**
287 * Emits a PIPE_CONTROL with a non-zero post-sync operation, for
288 * implementing two workarounds on gen6. From section 1.4.7.1
289 * "PIPE_CONTROL" of the Sandy Bridge PRM volume 2 part 1:
290 *
291 * [DevSNB-C+{W/A}] Before any depth stall flush (including those
292 * produced by non-pipelined state commands), software needs to first
293 * send a PIPE_CONTROL with no bits set except Post-Sync Operation !=
294 * 0.
295 *
296 * [Dev-SNB{W/A}]: Before a PIPE_CONTROL with Write Cache Flush Enable
297 * =1, a PIPE_CONTROL with any non-zero post-sync-op is required.
298 *
299 * And the workaround for these two requires this workaround first:
300 *
301 * [Dev-SNB{W/A}]: Pipe-control with CS-stall bit set must be sent
302 * BEFORE the pipe-control with a post-sync op and no write-cache
303 * flushes.
304 *
305 * And this last workaround is tricky because of the requirements on
306 * that bit. From section 1.4.7.2.3 "Stall" of the Sandy Bridge PRM
307 * volume 2 part 1:
308 *
309 * "1 of the following must also be set:
310 * - Render Target Cache Flush Enable ([12] of DW1)
311 * - Depth Cache Flush Enable ([0] of DW1)
312 * - Stall at Pixel Scoreboard ([1] of DW1)
313 * - Depth Stall ([13] of DW1)
314 * - Post-Sync Operation ([13] of DW1)
315 * - Notify Enable ([8] of DW1)"
316 *
317 * The cache flushes require the workaround flush that triggered this
318 * one, so we can't use it. Depth stall would trigger the same.
319 * Post-sync nonzero is what triggered this second workaround, so we
320 * can't use that one either. Notify enable is IRQs, which aren't
321 * really our business. That leaves only stall at scoreboard.
322 */
323 void
brw_emit_post_sync_nonzero_flush(struct brw_context * brw)324 brw_emit_post_sync_nonzero_flush(struct brw_context *brw)
325 {
326 brw_emit_pipe_control_flush(brw,
327 PIPE_CONTROL_CS_STALL |
328 PIPE_CONTROL_STALL_AT_SCOREBOARD);
329
330 brw_emit_pipe_control_write(brw, PIPE_CONTROL_WRITE_IMMEDIATE,
331 brw->workaround_bo, 0, 0, 0);
332 }
333
334 /* Emit a pipelined flush to either flush render and texture cache for
335 * reading from a FBO-drawn texture, or flush so that frontbuffer
336 * render appears on the screen in DRI1.
337 *
338 * This is also used for the always_flush_cache driconf debug option.
339 */
340 void
brw_emit_mi_flush(struct brw_context * brw)341 brw_emit_mi_flush(struct brw_context *brw)
342 {
343 if (brw->batch.ring == BLT_RING && brw->gen >= 6) {
344 BEGIN_BATCH_BLT(4);
345 OUT_BATCH(MI_FLUSH_DW);
346 OUT_BATCH(0);
347 OUT_BATCH(0);
348 OUT_BATCH(0);
349 ADVANCE_BATCH();
350 } else {
351 int flags = PIPE_CONTROL_NO_WRITE | PIPE_CONTROL_RENDER_TARGET_FLUSH;
352 if (brw->gen >= 6) {
353 flags |= PIPE_CONTROL_INSTRUCTION_INVALIDATE |
354 PIPE_CONTROL_CONST_CACHE_INVALIDATE |
355 PIPE_CONTROL_DEPTH_CACHE_FLUSH |
356 PIPE_CONTROL_VF_CACHE_INVALIDATE |
357 PIPE_CONTROL_TEXTURE_CACHE_INVALIDATE |
358 PIPE_CONTROL_CS_STALL;
359 }
360 brw_emit_pipe_control_flush(brw, flags);
361 }
362 }
363
364 int
brw_init_pipe_control(struct brw_context * brw,const struct gen_device_info * devinfo)365 brw_init_pipe_control(struct brw_context *brw,
366 const struct gen_device_info *devinfo)
367 {
368 if (devinfo->gen < 6)
369 return 0;
370
371 /* We can't just use brw_state_batch to get a chunk of space for
372 * the gen6 workaround because it involves actually writing to
373 * the buffer, and the kernel doesn't let us write to the batch.
374 */
375 brw->workaround_bo = drm_intel_bo_alloc(brw->bufmgr,
376 "pipe_control workaround",
377 4096, 4096);
378 if (brw->workaround_bo == NULL)
379 return -ENOMEM;
380
381 brw->pipe_controls_since_last_cs_stall = 0;
382
383 return 0;
384 }
385
386 void
brw_fini_pipe_control(struct brw_context * brw)387 brw_fini_pipe_control(struct brw_context *brw)
388 {
389 drm_intel_bo_unreference(brw->workaround_bo);
390 }
391