1 /*
2 Copyright (C) Intel Corp. 2006. All Rights Reserved.
3 Intel funded Tungsten Graphics to
4 develop this 3D driver.
5
6 Permission is hereby granted, free of charge, to any person obtaining
7 a copy of this software and associated documentation files (the
8 "Software"), to deal in the Software without restriction, including
9 without limitation the rights to use, copy, modify, merge, publish,
10 distribute, sublicense, and/or sell copies of the Software, and to
11 permit persons to whom the Software is furnished to do so, subject to
12 the following conditions:
13
14 The above copyright notice and this permission notice (including the
15 next paragraph) shall be included in all copies or substantial
16 portions of the Software.
17
18 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
19 EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
20 MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
21 IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
22 LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
23 OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
24 WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
25
26 **********************************************************************/
27 /*
28 * Authors:
29 * Keith Whitwell <keithw@vmware.com>
30 */
31
32 #pragma once
33
34 #include <stdint.h>
35 #include <stdlib.h>
36 #include "util/macros.h"
37 #include "dev/intel_device_info.h"
38
39 /* The following hunk, up-to "Execution Unit" is used by both the
40 * intel/compiler and i965 codebase. */
41
42 #define INTEL_MASK(high, low) (((1u<<((high)-(low)+1))-1)<<(low))
43 /* Using the GNU statement expression extension */
44 #define SET_FIELD(value, field) \
45 ({ \
46 uint32_t fieldval = (uint32_t)(value) << field ## _SHIFT; \
47 assert((fieldval & ~ field ## _MASK) == 0); \
48 fieldval & field ## _MASK; \
49 })
50
51 #define SET_BITS(value, high, low) \
52 ({ \
53 const uint32_t fieldval = (uint32_t)(value) << (low); \
54 assert((fieldval & ~INTEL_MASK(high, low)) == 0); \
55 fieldval & INTEL_MASK(high, low); \
56 })
57
58 #define GET_BITS(data, high, low) ((data & INTEL_MASK((high), (low))) >> (low))
59 #define GET_FIELD(word, field) (((word) & field ## _MASK) >> field ## _SHIFT)
60
61 # define GFX7_GS_CONTROL_DATA_FORMAT_GSCTL_CUT 0
62 # define GFX7_GS_CONTROL_DATA_FORMAT_GSCTL_SID 1
63
64 /* Execution Unit (EU) defines
65 */
66
67 #define BRW_ALIGN_1 0
68 #define BRW_ALIGN_16 1
69
70 #define BRW_ADDRESS_DIRECT 0
71 #define BRW_ADDRESS_REGISTER_INDIRECT_REGISTER 1
72
73 #define BRW_CHANNEL_X 0
74 #define BRW_CHANNEL_Y 1
75 #define BRW_CHANNEL_Z 2
76 #define BRW_CHANNEL_W 3
77
78 enum brw_compression {
79 BRW_COMPRESSION_NONE = 0,
80 BRW_COMPRESSION_2NDHALF = 1,
81 BRW_COMPRESSION_COMPRESSED = 2,
82 };
83
84 #define GFX6_COMPRESSION_1Q 0
85 #define GFX6_COMPRESSION_2Q 1
86 #define GFX6_COMPRESSION_3Q 2
87 #define GFX6_COMPRESSION_4Q 3
88 #define GFX6_COMPRESSION_1H 0
89 #define GFX6_COMPRESSION_2H 2
90
91 enum ENUM_PACKED brw_conditional_mod {
92 BRW_CONDITIONAL_NONE = 0,
93 BRW_CONDITIONAL_Z = 1,
94 BRW_CONDITIONAL_NZ = 2,
95 BRW_CONDITIONAL_EQ = 1, /* Z */
96 BRW_CONDITIONAL_NEQ = 2, /* NZ */
97 BRW_CONDITIONAL_G = 3,
98 BRW_CONDITIONAL_GE = 4,
99 BRW_CONDITIONAL_L = 5,
100 BRW_CONDITIONAL_LE = 6,
101 BRW_CONDITIONAL_R = 7, /* Gen <= 5 */
102 BRW_CONDITIONAL_O = 8,
103 BRW_CONDITIONAL_U = 9,
104 };
105
106 #define BRW_DEBUG_NONE 0
107 #define BRW_DEBUG_BREAKPOINT 1
108
109 enum ENUM_PACKED brw_execution_size {
110 BRW_EXECUTE_1 = 0,
111 BRW_EXECUTE_2 = 1,
112 BRW_EXECUTE_4 = 2,
113 BRW_EXECUTE_8 = 3,
114 BRW_EXECUTE_16 = 4,
115 BRW_EXECUTE_32 = 5,
116 };
117
118 enum ENUM_PACKED brw_horizontal_stride {
119 BRW_HORIZONTAL_STRIDE_0 = 0,
120 BRW_HORIZONTAL_STRIDE_1 = 1,
121 BRW_HORIZONTAL_STRIDE_2 = 2,
122 BRW_HORIZONTAL_STRIDE_4 = 3,
123 };
124
125 enum ENUM_PACKED gfx10_align1_3src_src_horizontal_stride {
126 BRW_ALIGN1_3SRC_SRC_HORIZONTAL_STRIDE_0 = 0,
127 BRW_ALIGN1_3SRC_SRC_HORIZONTAL_STRIDE_1 = 1,
128 BRW_ALIGN1_3SRC_SRC_HORIZONTAL_STRIDE_2 = 2,
129 BRW_ALIGN1_3SRC_SRC_HORIZONTAL_STRIDE_4 = 3,
130 };
131
132 enum ENUM_PACKED gfx10_align1_3src_dst_horizontal_stride {
133 BRW_ALIGN1_3SRC_DST_HORIZONTAL_STRIDE_1 = 0,
134 BRW_ALIGN1_3SRC_DST_HORIZONTAL_STRIDE_2 = 1,
135 };
136
137 #define BRW_INSTRUCTION_NORMAL 0
138 #define BRW_INSTRUCTION_SATURATE 1
139
140 #define BRW_MASK_ENABLE 0
141 #define BRW_MASK_DISABLE 1
142
143 /** @{
144 *
145 * Gfx6 has replaced "mask enable/disable" with WECtrl, which is
146 * effectively the same but much simpler to think about. Now, there
147 * are two contributors ANDed together to whether channels are
148 * executed: The predication on the instruction, and the channel write
149 * enable.
150 */
151 /**
152 * This is the default value. It means that a channel's write enable is set
153 * if the per-channel IP is pointing at this instruction.
154 */
155 #define BRW_WE_NORMAL 0
156 /**
157 * This is used like BRW_MASK_DISABLE, and causes all channels to have
158 * their write enable set. Note that predication still contributes to
159 * whether the channel actually gets written.
160 */
161 #define BRW_WE_ALL 1
162 /** @} */
163
164 enum opcode {
165 /* These are the actual hardware instructions. */
166 BRW_OPCODE_ILLEGAL,
167 BRW_OPCODE_SYNC,
168 BRW_OPCODE_MOV,
169 BRW_OPCODE_SEL,
170 BRW_OPCODE_MOVI,
171 BRW_OPCODE_NOT,
172 BRW_OPCODE_AND,
173 BRW_OPCODE_OR,
174 BRW_OPCODE_XOR,
175 BRW_OPCODE_SHR,
176 BRW_OPCODE_SHL,
177 BRW_OPCODE_SMOV,
178 BRW_OPCODE_ASR,
179 BRW_OPCODE_ROR, /**< Gfx11+ */
180 BRW_OPCODE_ROL, /**< Gfx11+ */
181 BRW_OPCODE_CMP,
182 BRW_OPCODE_CMPN,
183 BRW_OPCODE_CSEL,
184 BRW_OPCODE_BFREV,
185 BRW_OPCODE_BFE,
186 BRW_OPCODE_BFI1,
187 BRW_OPCODE_BFI2,
188 BRW_OPCODE_JMPI,
189 BRW_OPCODE_BRD,
190 BRW_OPCODE_IF,
191 BRW_OPCODE_BRC,
192 BRW_OPCODE_ELSE,
193 BRW_OPCODE_ENDIF,
194 BRW_OPCODE_DO, /**< Used as pseudo opcode, will be moved later. */
195 BRW_OPCODE_WHILE,
196 BRW_OPCODE_BREAK,
197 BRW_OPCODE_CONTINUE,
198 BRW_OPCODE_HALT,
199 BRW_OPCODE_CALLA,
200 BRW_OPCODE_CALL,
201 BRW_OPCODE_RET,
202 BRW_OPCODE_GOTO,
203 BRW_OPCODE_WAIT,
204 BRW_OPCODE_SEND,
205 BRW_OPCODE_SENDC,
206 BRW_OPCODE_SENDS,
207 BRW_OPCODE_SENDSC,
208 BRW_OPCODE_MATH,
209 BRW_OPCODE_ADD,
210 BRW_OPCODE_MUL,
211 BRW_OPCODE_AVG,
212 BRW_OPCODE_FRC,
213 BRW_OPCODE_RNDU,
214 BRW_OPCODE_RNDD,
215 BRW_OPCODE_RNDE,
216 BRW_OPCODE_RNDZ,
217 BRW_OPCODE_MAC,
218 BRW_OPCODE_MACH,
219 BRW_OPCODE_LZD,
220 BRW_OPCODE_FBH,
221 BRW_OPCODE_FBL,
222 BRW_OPCODE_CBIT,
223 BRW_OPCODE_ADDC,
224 BRW_OPCODE_SUBB,
225 BRW_OPCODE_ADD3, /* Gen12+ only */
226 BRW_OPCODE_DP4,
227 BRW_OPCODE_DPH,
228 BRW_OPCODE_DP3,
229 BRW_OPCODE_DP2,
230 BRW_OPCODE_DP4A, /**< Gfx12+ */
231 BRW_OPCODE_LINE,
232 BRW_OPCODE_DPAS, /**< Gfx12.5+ */
233 BRW_OPCODE_PLN, /**< Up until Gfx9 */
234 BRW_OPCODE_MAD,
235 BRW_OPCODE_LRP,
236 BRW_OPCODE_MADM,
237 BRW_OPCODE_NOP,
238
239 NUM_BRW_OPCODES,
240
241 /**
242 * The position/ordering of the arguments are defined
243 * by the enum fb_write_logical_srcs.
244 */
245 FS_OPCODE_FB_WRITE_LOGICAL = NUM_BRW_OPCODES,
246
247 FS_OPCODE_FB_READ_LOGICAL,
248
249 SHADER_OPCODE_RCP,
250 SHADER_OPCODE_RSQ,
251 SHADER_OPCODE_SQRT,
252 SHADER_OPCODE_EXP2,
253 SHADER_OPCODE_LOG2,
254 SHADER_OPCODE_POW,
255 SHADER_OPCODE_INT_QUOTIENT,
256 SHADER_OPCODE_INT_REMAINDER,
257 SHADER_OPCODE_SIN,
258 SHADER_OPCODE_COS,
259
260 /**
261 * A generic "send" opcode. The first two sources are the message
262 * descriptor and extended message descriptor respectively. The third
263 * and optional fourth sources are the message payload
264 */
265 SHADER_OPCODE_SEND,
266
267 /**
268 * A variant of SEND that collects its sources to form an input.
269 *
270 * Source 0: Message descriptor ("desc").
271 * Source 1: Message extended descriptor ("ex_desc").
272 * Source 2: Before register allocation must be BAD_FILE,
273 * after that, the ARF scalar register containing
274 * the (physical) numbers of the payload sources.
275 * Source 3..n: Payload sources. For this opcode, they must each
276 * have the size of a physical GRF.
277 */
278 SHADER_OPCODE_SEND_GATHER,
279
280 /**
281 * An "undefined" write which does nothing but indicates to liveness that
282 * we don't care about any values in the register which predate this
283 * instruction. Used to prevent partial writes from causing issues with
284 * live ranges.
285 */
286 SHADER_OPCODE_UNDEF,
287
288 /**
289 * Texture sampling opcodes.
290 *
291 * LOGICAL opcodes are eventually translated to SHADER_OPCODE_SEND but
292 * take parameters as individual sources. See enum tex_logical_srcs.
293 */
294 SHADER_OPCODE_TEX_LOGICAL,
295 SHADER_OPCODE_TXD_LOGICAL,
296 SHADER_OPCODE_TXF_LOGICAL,
297 SHADER_OPCODE_TXL_LOGICAL,
298 SHADER_OPCODE_TXS_LOGICAL,
299 FS_OPCODE_TXB_LOGICAL,
300 SHADER_OPCODE_TXF_CMS_W_LOGICAL,
301 SHADER_OPCODE_TXF_CMS_W_GFX12_LOGICAL,
302 SHADER_OPCODE_TXF_MCS_LOGICAL,
303 SHADER_OPCODE_LOD_LOGICAL,
304 SHADER_OPCODE_TG4_LOGICAL,
305 SHADER_OPCODE_TG4_IMPLICIT_LOD_LOGICAL,
306 SHADER_OPCODE_TG4_EXPLICIT_LOD_LOGICAL,
307 SHADER_OPCODE_TG4_BIAS_LOGICAL,
308 SHADER_OPCODE_TG4_OFFSET_LOGICAL,
309 SHADER_OPCODE_TG4_OFFSET_LOD_LOGICAL,
310 SHADER_OPCODE_TG4_OFFSET_BIAS_LOGICAL,
311 SHADER_OPCODE_SAMPLEINFO_LOGICAL,
312
313 SHADER_OPCODE_IMAGE_SIZE_LOGICAL,
314
315 /**
316 * Combines multiple sources of size 1 into a larger virtual GRF.
317 * For example, parameters for a send-from-GRF message. Or, updating
318 * channels of a size 4 VGRF used to store vec4s such as texturing results.
319 *
320 * This will be lowered into MOVs from each source to consecutive offsets
321 * of the destination VGRF.
322 *
323 * src[0] may be BAD_FILE. If so, the lowering pass skips emitting the MOV,
324 * but still reserves the first channel of the destination VGRF. This can be
325 * used to reserve space for, say, a message header set up by the generators.
326 */
327 SHADER_OPCODE_LOAD_PAYLOAD,
328
329 /**
330 * Packs a number of sources into a single value. Unlike LOAD_PAYLOAD, this
331 * acts intra-channel, obtaining the final value for each channel by
332 * combining the sources values for the same channel, the first source
333 * occupying the lowest bits and the last source occupying the highest
334 * bits.
335 */
336 FS_OPCODE_PACK,
337
338 SHADER_OPCODE_RND_MODE,
339 SHADER_OPCODE_FLOAT_CONTROL_MODE,
340
341 /**
342 * Memory fence messages.
343 *
344 * Source 0: Must be register g0, used as header.
345 * Source 1: Immediate bool to indicate whether control is returned to the
346 * thread only after the fence has been honored.
347 * Source 2: Immediate byte indicating which memory to fence. Zero means
348 * global memory; GFX7_BTI_SLM means SLM (for Gfx11+ only).
349 *
350 * Vec4 backend only uses Source 0.
351 */
352 SHADER_OPCODE_MEMORY_FENCE,
353
354 /**
355 * Scheduling-only fence.
356 *
357 * Sources can be used to force a stall until the registers in those are
358 * available. This might generate MOVs or SYNC_NOPs (Gfx12+).
359 */
360 FS_OPCODE_SCHEDULING_FENCE,
361
362 SHADER_OPCODE_SCRATCH_HEADER,
363
364 /**
365 * Gfx8+ SIMD8 URB messages.
366 */
367 SHADER_OPCODE_URB_READ_LOGICAL,
368 SHADER_OPCODE_URB_WRITE_LOGICAL,
369
370 /**
371 * Return the index of the first enabled live channel and assign it to
372 * to the first component of the destination. Frequently used as input
373 * for the BROADCAST pseudo-opcode.
374 */
375 SHADER_OPCODE_FIND_LIVE_CHANNEL,
376
377 /**
378 * Return the index of the last enabled live channel and assign it to
379 * the first component of the destination.
380 */
381 SHADER_OPCODE_FIND_LAST_LIVE_CHANNEL,
382
383 /**
384 * Return the current execution mask and assign it to the first component
385 * of the destination.
386 *
387 * \sa opcode::FS_OPCODE_LOAD_LIVE_CHANNELS
388 */
389 SHADER_OPCODE_LOAD_LIVE_CHANNELS,
390
391 /**
392 * Return the current execution mask in the specified flag subregister.
393 * Can be CSE'ed more easily than a plain MOV from the ce0 ARF register.
394 */
395 FS_OPCODE_LOAD_LIVE_CHANNELS,
396
397 /**
398 * Pick the channel from its first source register given by the index
399 * specified as second source. Useful for variable indexing of surfaces.
400 *
401 * Note that because the result of this instruction is by definition
402 * uniform and it can always be splatted to multiple channels using a
403 * scalar regioning mode, only the first channel of the destination region
404 * is guaranteed to be updated, which implies that BROADCAST instructions
405 * should usually be marked force_writemask_all.
406 */
407 SHADER_OPCODE_BROADCAST,
408
409 /* Pick the channel from its first source register given by the index
410 * specified as second source.
411 *
412 * This is similar to the BROADCAST instruction except that it takes a
413 * dynamic index and potentially puts a different value in each output
414 * channel.
415 */
416 SHADER_OPCODE_SHUFFLE,
417
418 /* Combine all values in each subset (cluster) of channels using an operation,
419 * and broadcast the result to all channels in the subset.
420 *
421 * Source 0: Value.
422 * Source 1: Immediate with brw_reduce_op.
423 * Source 2: Immediate with cluster size.
424 */
425 SHADER_OPCODE_REDUCE,
426
427 /* Combine values of previous channels using an operation. Inclusive scan
428 * will include the value of the channel itself in the channel result.
429 *
430 * Source 0: Value.
431 * Source 1: Immediate with brw_reduce_op.
432 */
433 SHADER_OPCODE_INCLUSIVE_SCAN,
434 SHADER_OPCODE_EXCLUSIVE_SCAN,
435
436 /* Check if any or all values in each subset (cluster) of channels are set,
437 * and broadcast the result to all channels in the subset.
438 *
439 * Source 0: Boolean value.
440 * Source 1: Immediate with cluster size.
441 */
442 SHADER_OPCODE_VOTE_ANY,
443 SHADER_OPCODE_VOTE_ALL,
444
445 /* Check if the values of all channels are equal, and broadcast the result
446 * to all channels.
447 *
448 * Source 0: Value.
449 */
450 SHADER_OPCODE_VOTE_EQUAL,
451
452 /* Produces a mask from the boolean value from all channels, and broadcast
453 * the result to all channels.
454 *
455 * Source 0: Boolean value.
456 */
457 SHADER_OPCODE_BALLOT,
458
459 /* Select between src0 and src1 based on channel enables.
460 *
461 * This instruction copies src0 into the enabled channels of the
462 * destination and copies src1 into the disabled channels.
463 */
464 SHADER_OPCODE_SEL_EXEC,
465
466 /* Swap values inside a quad based on the direction.
467 *
468 * Source 0: Value.
469 * Source 1: Immediate with brw_swap_direction.
470 */
471 SHADER_OPCODE_QUAD_SWAP,
472
473 /* Read value from the first live channel and broadcast the result
474 * to all channels.
475 *
476 * Source 0: Value.
477 */
478 SHADER_OPCODE_READ_FROM_LIVE_CHANNEL,
479
480 /* Read value from a specified channel and broadcast the result
481 * to all channels.
482 *
483 * Source 0: Value.
484 * Source 1: Index of the channel to pick value from.
485 */
486 SHADER_OPCODE_READ_FROM_CHANNEL,
487
488 /* This turns into an align16 mov from src0 to dst with a swizzle
489 * provided as an immediate in src1.
490 */
491 SHADER_OPCODE_QUAD_SWIZZLE,
492
493 /* Take every Nth element in src0 and broadcast it to the group of N
494 * channels in which it lives in the destination. The offset within the
495 * cluster is given by src1 and the cluster size is given by src2.
496 */
497 SHADER_OPCODE_CLUSTER_BROADCAST,
498
499 SHADER_OPCODE_GET_BUFFER_SIZE,
500
501 SHADER_OPCODE_INTERLOCK,
502
503 /** Target for a HALT
504 *
505 * All HALT instructions in a shader must target the same jump point and
506 * that point is denoted by a HALT_TARGET instruction.
507 */
508 SHADER_OPCODE_HALT_TARGET,
509
510 FS_OPCODE_DDX_COARSE,
511 FS_OPCODE_DDX_FINE,
512 /**
513 * Compute dFdy(), dFdyCoarse(), or dFdyFine().
514 */
515 FS_OPCODE_DDY_COARSE,
516 FS_OPCODE_DDY_FINE,
517 FS_OPCODE_PIXEL_X,
518 FS_OPCODE_PIXEL_Y,
519 FS_OPCODE_UNIFORM_PULL_CONSTANT_LOAD,
520 FS_OPCODE_VARYING_PULL_CONSTANT_LOAD_LOGICAL,
521 FS_OPCODE_PACK_HALF_2x16_SPLIT,
522 FS_OPCODE_INTERPOLATE_AT_SAMPLE,
523 FS_OPCODE_INTERPOLATE_AT_SHARED_OFFSET,
524 FS_OPCODE_INTERPOLATE_AT_PER_SLOT_OFFSET,
525
526 /**
527 * GLSL barrier()
528 */
529 SHADER_OPCODE_BARRIER,
530
531 /**
532 * Calculate the high 32-bits of a 32x32 multiply.
533 */
534 SHADER_OPCODE_MULH,
535
536 /** Signed subtraction with saturation. */
537 SHADER_OPCODE_ISUB_SAT,
538
539 /** Unsigned subtraction with saturation. */
540 SHADER_OPCODE_USUB_SAT,
541
542 /**
543 * A MOV that uses VxH indirect addressing.
544 *
545 * Source 0: A register to start from (HW_REG).
546 * Source 1: An indirect offset (in bytes, UD GRF).
547 * Source 2: The length of the region that could be accessed (in bytes,
548 * UD immediate).
549 */
550 SHADER_OPCODE_MOV_INDIRECT,
551
552 /** Fills out a relocatable immediate */
553 SHADER_OPCODE_MOV_RELOC_IMM,
554
555 SHADER_OPCODE_BTD_SPAWN_LOGICAL,
556 SHADER_OPCODE_BTD_RETIRE_LOGICAL,
557
558 SHADER_OPCODE_READ_ARCH_REG,
559
560 SHADER_OPCODE_LOAD_SUBGROUP_INVOCATION,
561
562 RT_OPCODE_TRACE_RAY_LOGICAL,
563
564 SHADER_OPCODE_MEMORY_LOAD_LOGICAL,
565 SHADER_OPCODE_MEMORY_STORE_LOGICAL,
566 SHADER_OPCODE_MEMORY_ATOMIC_LOGICAL,
567 };
568
569 enum fb_write_logical_srcs {
570 FB_WRITE_LOGICAL_SRC_COLOR0, /* REQUIRED */
571 FB_WRITE_LOGICAL_SRC_COLOR1, /* for dual source blend messages */
572 FB_WRITE_LOGICAL_SRC_SRC0_ALPHA,
573 FB_WRITE_LOGICAL_SRC_SRC_DEPTH, /* gl_FragDepth */
574 FB_WRITE_LOGICAL_SRC_DST_DEPTH, /* GFX4-5: passthrough from thread */
575 FB_WRITE_LOGICAL_SRC_SRC_STENCIL, /* gl_FragStencilRefARB */
576 FB_WRITE_LOGICAL_SRC_OMASK, /* Sample Mask (gl_SampleMask) */
577 FB_WRITE_LOGICAL_SRC_COMPONENTS, /* REQUIRED */
578 FB_WRITE_LOGICAL_SRC_NULL_RT, /* Null RT write */
579 FB_WRITE_LOGICAL_NUM_SRCS
580 };
581
582 enum tex_logical_srcs {
583 /** Texture coordinates */
584 TEX_LOGICAL_SRC_COORDINATE,
585 /** Shadow comparator */
586 TEX_LOGICAL_SRC_SHADOW_C,
587 /** dPdx if the operation takes explicit derivatives, otherwise LOD value */
588 TEX_LOGICAL_SRC_LOD,
589 /** dPdy if the operation takes explicit derivatives */
590 TEX_LOGICAL_SRC_LOD2,
591 /** Min LOD */
592 TEX_LOGICAL_SRC_MIN_LOD,
593 /** Sample index */
594 TEX_LOGICAL_SRC_SAMPLE_INDEX,
595 /** MCS data */
596 TEX_LOGICAL_SRC_MCS,
597 /** REQUIRED: Texture surface index */
598 TEX_LOGICAL_SRC_SURFACE,
599 /** Texture sampler index */
600 TEX_LOGICAL_SRC_SAMPLER,
601 /** Texture surface bindless handle */
602 TEX_LOGICAL_SRC_SURFACE_HANDLE,
603 /** Texture sampler bindless handle */
604 TEX_LOGICAL_SRC_SAMPLER_HANDLE,
605 /** Texel offset for gathers */
606 TEX_LOGICAL_SRC_TG4_OFFSET,
607 /** REQUIRED: Number of coordinate components (as UD immediate) */
608 TEX_LOGICAL_SRC_COORD_COMPONENTS,
609 /** REQUIRED: Number of derivative components (as UD immediate) */
610 TEX_LOGICAL_SRC_GRAD_COMPONENTS,
611 /** REQUIRED: request residency (as UD immediate) */
612 TEX_LOGICAL_SRC_RESIDENCY,
613
614 TEX_LOGICAL_NUM_SRCS,
615 };
616
617 enum pull_uniform_constant_srcs {
618 /** Surface binding table index */
619 PULL_UNIFORM_CONSTANT_SRC_SURFACE,
620 /** Surface bindless handle */
621 PULL_UNIFORM_CONSTANT_SRC_SURFACE_HANDLE,
622 /** Surface offset */
623 PULL_UNIFORM_CONSTANT_SRC_OFFSET,
624 /** Pull size */
625 PULL_UNIFORM_CONSTANT_SRC_SIZE,
626
627 PULL_UNIFORM_CONSTANT_SRCS,
628 };
629
630 enum pull_varying_constant_srcs {
631 /** Surface binding table index */
632 PULL_VARYING_CONSTANT_SRC_SURFACE,
633 /** Surface bindless handle */
634 PULL_VARYING_CONSTANT_SRC_SURFACE_HANDLE,
635 /** Surface offset */
636 PULL_VARYING_CONSTANT_SRC_OFFSET,
637 /** Pull alignment */
638 PULL_VARYING_CONSTANT_SRC_ALIGNMENT,
639
640 PULL_VARYING_CONSTANT_SRCS,
641 };
642
643 enum get_buffer_size_srcs {
644 /** Surface binding table index */
645 GET_BUFFER_SIZE_SRC_SURFACE,
646 /** Surface bindless handle */
647 GET_BUFFER_SIZE_SRC_SURFACE_HANDLE,
648 /** LOD */
649 GET_BUFFER_SIZE_SRC_LOD,
650
651 GET_BUFFER_SIZE_SRCS
652 };
653
654 enum memory_logical_mode {
655 MEMORY_MODE_TYPED,
656 MEMORY_MODE_UNTYPED,
657 MEMORY_MODE_SHARED_LOCAL,
658 MEMORY_MODE_SCRATCH,
659 MEMORY_MODE_CONSTANT,
660 };
661
662 enum memory_logical_srcs {
663 /** enum lsc_opcode (as UD immediate) */
664 MEMORY_LOGICAL_OPCODE,
665
666 /** enum memory_logical_mode (as UD immediate) */
667 MEMORY_LOGICAL_MODE,
668
669 /** enum lsc_addr_surface_type (as UD immediate) */
670 MEMORY_LOGICAL_BINDING_TYPE,
671
672 /**
673 * Where to find the surface state. Depends on BINDING_TYPE above:
674 *
675 * - SS: pointer to surface state (relative to surface base address)
676 * - BSS: pointer to surface state (relative to bindless surface base)
677 * - BTI: binding table index
678 * - FLAT: This should should be BAD_FILE
679 */
680 MEMORY_LOGICAL_BINDING,
681
682 /** Coordinate/address/offset for where to access memory */
683 MEMORY_LOGICAL_ADDRESS,
684
685 /** Dimensionality of the "address" source (as UD immediate) */
686 MEMORY_LOGICAL_COORD_COMPONENTS,
687
688 /** Required alignment of address in bytes; 0 for natural alignment */
689 MEMORY_LOGICAL_ALIGNMENT,
690
691 /** Bit-size in the form of enum lsc_data_size (as UD immediate) */
692 MEMORY_LOGICAL_DATA_SIZE,
693
694 /** Number of vector components (as UD immediate) */
695 MEMORY_LOGICAL_COMPONENTS,
696
697 /** memory_flags bitfield (as UD immediate) */
698 MEMORY_LOGICAL_FLAGS,
699
700 /** Data to write for stores or the first operand for atomics */
701 MEMORY_LOGICAL_DATA0,
702
703 /** Second operand for two-source atomics */
704 MEMORY_LOGICAL_DATA1,
705
706 MEMORY_LOGICAL_NUM_SRCS
707 };
708
709 enum memory_flags {
710 /** Whether this is a transposed (i.e. block) memory access */
711 MEMORY_FLAG_TRANSPOSE = 1 << 0,
712 /** Whether this operation should fire for helper invocations */
713 MEMORY_FLAG_INCLUDE_HELPERS = 1 << 1,
714 };
715
716 enum rt_logical_srcs {
717 /** Address of the globals */
718 RT_LOGICAL_SRC_GLOBALS,
719 /** Level at which the tracing should start */
720 RT_LOGICAL_SRC_BVH_LEVEL,
721 /** Type of tracing operation */
722 RT_LOGICAL_SRC_TRACE_RAY_CONTROL,
723 /** Synchronous tracing (ray query) */
724 RT_LOGICAL_SRC_SYNCHRONOUS,
725
726 RT_LOGICAL_NUM_SRCS
727 };
728
729 enum urb_logical_srcs {
730 URB_LOGICAL_SRC_HANDLE,
731 /** Offset in bytes on Xe2+ or OWords on older platforms */
732 URB_LOGICAL_SRC_PER_SLOT_OFFSETS,
733 URB_LOGICAL_SRC_CHANNEL_MASK,
734 /** Data to be written. BAD_FILE for reads. */
735 URB_LOGICAL_SRC_DATA,
736 URB_LOGICAL_SRC_COMPONENTS,
737 URB_LOGICAL_NUM_SRCS
738 };
739
740 enum interpolator_logical_srcs {
741 /** Interpolation offset */
742 INTERP_SRC_OFFSET,
743 /** Message data */
744 INTERP_SRC_MSG_DESC,
745 /** Flag register for dynamic mode */
746 INTERP_SRC_DYNAMIC_MODE,
747
748 INTERP_NUM_SRCS
749 };
750
751 enum brw_reduce_op {
752 BRW_REDUCE_OP_ADD,
753 BRW_REDUCE_OP_MUL,
754 BRW_REDUCE_OP_MIN,
755 BRW_REDUCE_OP_MAX,
756 BRW_REDUCE_OP_AND,
757 BRW_REDUCE_OP_OR,
758 BRW_REDUCE_OP_XOR,
759 };
760
761 enum brw_swap_direction {
762 BRW_SWAP_HORIZONTAL,
763 BRW_SWAP_VERTICAL,
764 BRW_SWAP_DIAGONAL,
765 };
766
767 enum ENUM_PACKED brw_predicate {
768 BRW_PREDICATE_NONE = 0,
769 BRW_PREDICATE_NORMAL = 1,
770 BRW_PREDICATE_ALIGN1_ANYV = 2,
771 BRW_PREDICATE_ALIGN1_ALLV = 3,
772 BRW_PREDICATE_ALIGN1_ANY2H = 4,
773 BRW_PREDICATE_ALIGN1_ALL2H = 5,
774 BRW_PREDICATE_ALIGN1_ANY4H = 6,
775 BRW_PREDICATE_ALIGN1_ALL4H = 7,
776 BRW_PREDICATE_ALIGN1_ANY8H = 8,
777 BRW_PREDICATE_ALIGN1_ALL8H = 9,
778 BRW_PREDICATE_ALIGN1_ANY16H = 10,
779 BRW_PREDICATE_ALIGN1_ALL16H = 11,
780 BRW_PREDICATE_ALIGN1_ANY32H = 12,
781 BRW_PREDICATE_ALIGN1_ALL32H = 13,
782 BRW_PREDICATE_ALIGN16_REPLICATE_X = 2,
783 BRW_PREDICATE_ALIGN16_REPLICATE_Y = 3,
784 BRW_PREDICATE_ALIGN16_REPLICATE_Z = 4,
785 BRW_PREDICATE_ALIGN16_REPLICATE_W = 5,
786 BRW_PREDICATE_ALIGN16_ANY4H = 6,
787 BRW_PREDICATE_ALIGN16_ALL4H = 7,
788 XE2_PREDICATE_ANY = 2,
789 XE2_PREDICATE_ALL = 3
790 };
791
792 enum ENUM_PACKED brw_reg_file {
793 BAD_FILE = 0,
794
795 ARF,
796 FIXED_GRF,
797 IMM,
798
799 ADDRESS,
800 VGRF,
801 ATTR,
802 UNIFORM, /* prog_data->params[reg] */
803 };
804
805 /* CNL adds Align1 support for 3-src instructions. Bit 35 of the instruction
806 * word is "Execution Datatype" which controls whether the instruction operates
807 * on float or integer types. The register arguments have fields that offer
808 * more fine control their respective types.
809 */
810 enum ENUM_PACKED gfx10_align1_3src_exec_type {
811 BRW_ALIGN1_3SRC_EXEC_TYPE_INT = 0,
812 BRW_ALIGN1_3SRC_EXEC_TYPE_FLOAT = 1,
813 };
814
815 #define BRW_ARF_NULL 0x00
816 #define BRW_ARF_ADDRESS 0x10
817 #define BRW_ARF_ACCUMULATOR 0x20
818 #define BRW_ARF_FLAG 0x30
819 #define BRW_ARF_MASK 0x40
820 #define BRW_ARF_SCALAR 0x60
821 #define BRW_ARF_STATE 0x70
822 #define BRW_ARF_CONTROL 0x80
823 #define BRW_ARF_NOTIFICATION_COUNT 0x90
824 #define BRW_ARF_IP 0xA0
825 #define BRW_ARF_TDR 0xB0
826 #define BRW_ARF_TIMESTAMP 0xC0
827
828 #define BRW_THREAD_NORMAL 0
829 #define BRW_THREAD_ATOMIC 1
830 #define BRW_THREAD_SWITCH 2
831
832 /* Subregister of the address register used for particular purposes */
833 enum brw_address_subreg {
834 BRW_ADDRESS_SUBREG_INDIRECT_DESC = 0,
835 BRW_ADDRESS_SUBREG_INDIRECT_EX_DESC = 2,
836 BRW_ADDRESS_SUBREG_INDIRECT_SPILL_DESC = 4,
837 };
838
839 enum ENUM_PACKED brw_vertical_stride {
840 BRW_VERTICAL_STRIDE_0 = 0,
841 BRW_VERTICAL_STRIDE_1 = 1,
842 BRW_VERTICAL_STRIDE_2 = 2,
843 BRW_VERTICAL_STRIDE_4 = 3,
844 BRW_VERTICAL_STRIDE_8 = 4,
845 BRW_VERTICAL_STRIDE_16 = 5,
846 BRW_VERTICAL_STRIDE_32 = 6,
847 BRW_VERTICAL_STRIDE_ONE_DIMENSIONAL = 0xF,
848 };
849
850 enum ENUM_PACKED gfx10_align1_3src_vertical_stride {
851 BRW_ALIGN1_3SRC_VERTICAL_STRIDE_0 = 0,
852 BRW_ALIGN1_3SRC_VERTICAL_STRIDE_1 = 1,
853 BRW_ALIGN1_3SRC_VERTICAL_STRIDE_2 = 1,
854 BRW_ALIGN1_3SRC_VERTICAL_STRIDE_4 = 2,
855 BRW_ALIGN1_3SRC_VERTICAL_STRIDE_8 = 3,
856 };
857
858 enum ENUM_PACKED brw_width {
859 BRW_WIDTH_1 = 0,
860 BRW_WIDTH_2 = 1,
861 BRW_WIDTH_4 = 2,
862 BRW_WIDTH_8 = 3,
863 BRW_WIDTH_16 = 4,
864 };
865
866 /**
867 * Gfx12+ SWSB SBID synchronization mode.
868 *
869 * This is represented as a bitmask including any required SBID token
870 * synchronization modes, used to synchronize out-of-order instructions. Only
871 * the strongest mode of the mask will be provided to the hardware in the SWSB
872 * field of an actual hardware instruction, but virtual instructions may be
873 * able to take into account multiple of them.
874 */
875 enum tgl_sbid_mode {
876 TGL_SBID_NULL = 0,
877 TGL_SBID_SRC = 1,
878 TGL_SBID_DST = 2,
879 TGL_SBID_SET = 4
880 };
881
882
883 enum gfx12_sub_byte_precision {
884 BRW_SUB_BYTE_PRECISION_NONE = 0,
885
886 /** 4 bits. Signedness determined by base type */
887 BRW_SUB_BYTE_PRECISION_4BIT = 1,
888
889 /** 2 bits. Signedness determined by base type */
890 BRW_SUB_BYTE_PRECISION_2BIT = 2,
891 };
892
893 enum gfx12_systolic_depth {
894 BRW_SYSTOLIC_DEPTH_16 = 0,
895 BRW_SYSTOLIC_DEPTH_2 = 1,
896 BRW_SYSTOLIC_DEPTH_4 = 2,
897 BRW_SYSTOLIC_DEPTH_8 = 3,
898 };
899
900 #ifdef __cplusplus
901 /**
902 * Allow bitwise arithmetic of tgl_sbid_mode enums.
903 */
904 inline tgl_sbid_mode
905 operator|(tgl_sbid_mode x, tgl_sbid_mode y)
906 {
907 return tgl_sbid_mode(unsigned(x) | unsigned(y));
908 }
909
910 inline tgl_sbid_mode
911 operator&(tgl_sbid_mode x, tgl_sbid_mode y)
912 {
913 return tgl_sbid_mode(unsigned(x) & unsigned(y));
914 }
915
916 inline tgl_sbid_mode &
917 operator|=(tgl_sbid_mode &x, tgl_sbid_mode y)
918 {
919 return x = x | y;
920 }
921
922 #endif
923
924 /**
925 * TGL+ SWSB RegDist synchronization pipeline.
926 *
927 * On TGL all instructions that use the RegDist synchronization mechanism are
928 * considered to be executed as a single in-order pipeline, therefore only the
929 * TGL_PIPE_FLOAT pipeline is applicable. On XeHP+ platforms there are two
930 * additional asynchronous ALU pipelines (which still execute instructions
931 * in-order and use the RegDist synchronization mechanism). TGL_PIPE_NONE
932 * doesn't provide any RegDist pipeline synchronization information and allows
933 * the hardware to infer the pipeline based on the source types of the
934 * instruction. TGL_PIPE_ALL can be used when synchronization with all ALU
935 * pipelines is intended.
936 *
937 * Xe3 adds TGL_PIPE_SCALAR for a very specific use case (writing immediates
938 * to scalar register).
939 */
940 enum tgl_pipe {
941 TGL_PIPE_NONE = 0,
942 TGL_PIPE_FLOAT,
943 TGL_PIPE_INT,
944 TGL_PIPE_LONG,
945 TGL_PIPE_MATH,
946 TGL_PIPE_SCALAR,
947 TGL_PIPE_ALL
948 };
949
950 /**
951 * Logical representation of the SWSB scheduling information of a hardware
952 * instruction. The binary representation is slightly more compact.
953 */
954 struct tgl_swsb {
955 unsigned regdist : 3;
956 enum tgl_pipe pipe : 3;
957 unsigned sbid : 5;
958 enum tgl_sbid_mode mode : 3;
959 };
960
961 /**
962 * Construct a scheduling annotation with a single RegDist dependency. This
963 * synchronizes with the completion of the d-th previous in-order instruction.
964 * The index is one-based, zero causes a no-op tgl_swsb to be constructed.
965 */
966 static inline struct tgl_swsb
tgl_swsb_regdist(unsigned d)967 tgl_swsb_regdist(unsigned d)
968 {
969 const struct tgl_swsb swsb = { d, d ? TGL_PIPE_ALL : TGL_PIPE_NONE };
970 assert(swsb.regdist == d);
971 return swsb;
972 }
973
974 /**
975 * Construct a scheduling annotation that synchronizes with the specified SBID
976 * token.
977 */
978 static inline struct tgl_swsb
tgl_swsb_sbid(enum tgl_sbid_mode mode,unsigned sbid)979 tgl_swsb_sbid(enum tgl_sbid_mode mode, unsigned sbid)
980 {
981 const struct tgl_swsb swsb = { 0, TGL_PIPE_NONE, sbid, mode };
982 assert(swsb.sbid == sbid);
983 return swsb;
984 }
985
986 /**
987 * Construct a no-op scheduling annotation.
988 */
989 static inline struct tgl_swsb
tgl_swsb_null(void)990 tgl_swsb_null(void)
991 {
992 return tgl_swsb_regdist(0);
993 }
994
995 /**
996 * Return a scheduling annotation that allocates the same SBID synchronization
997 * token as \p swsb. In addition it will synchronize against a previous
998 * in-order instruction if \p regdist is non-zero.
999 */
1000 static inline struct tgl_swsb
tgl_swsb_dst_dep(struct tgl_swsb swsb,unsigned regdist)1001 tgl_swsb_dst_dep(struct tgl_swsb swsb, unsigned regdist)
1002 {
1003 swsb.regdist = regdist;
1004 swsb.mode = swsb.mode & TGL_SBID_SET;
1005 swsb.pipe = (regdist ? TGL_PIPE_ALL : TGL_PIPE_NONE);
1006 return swsb;
1007 }
1008
1009 /**
1010 * Return a scheduling annotation that synchronizes against the same SBID and
1011 * RegDist dependencies as \p swsb, but doesn't allocate any SBID token.
1012 */
1013 static inline struct tgl_swsb
tgl_swsb_src_dep(struct tgl_swsb swsb)1014 tgl_swsb_src_dep(struct tgl_swsb swsb)
1015 {
1016 swsb.mode = swsb.mode & (TGL_SBID_SRC | TGL_SBID_DST);
1017 return swsb;
1018 }
1019
1020 /**
1021 * Convert the provided tgl_swsb to the hardware's binary representation of an
1022 * SWSB annotation.
1023 */
1024 static inline uint32_t
tgl_swsb_encode(const struct intel_device_info * devinfo,struct tgl_swsb swsb,enum opcode opcode)1025 tgl_swsb_encode(const struct intel_device_info *devinfo,
1026 struct tgl_swsb swsb, enum opcode opcode)
1027 {
1028 if (!swsb.mode) {
1029 const unsigned pipe = devinfo->verx10 < 125 ? 0 :
1030 swsb.pipe == TGL_PIPE_FLOAT ? 0x10 :
1031 swsb.pipe == TGL_PIPE_INT ? 0x18 :
1032 swsb.pipe == TGL_PIPE_LONG ? 0x20 :
1033 swsb.pipe == TGL_PIPE_MATH ? 0x28 :
1034 swsb.pipe == TGL_PIPE_SCALAR ? 0x30 :
1035 swsb.pipe == TGL_PIPE_ALL ? 0x8 : 0;
1036 return pipe | swsb.regdist;
1037
1038 } else if (swsb.regdist) {
1039 if (devinfo->ver >= 20) {
1040 unsigned mode = 0;
1041 if (opcode == BRW_OPCODE_DPAS) {
1042 mode = (swsb.mode & TGL_SBID_SET) ? 0b01 :
1043 (swsb.mode & TGL_SBID_SRC) ? 0b10 :
1044 /* swsb.mode & TGL_SBID_DST */ 0b11;
1045 } else if (swsb.mode & TGL_SBID_SET) {
1046 assert(opcode == BRW_OPCODE_SEND || opcode == BRW_OPCODE_SENDC);
1047 assert(swsb.pipe == TGL_PIPE_ALL ||
1048 swsb.pipe == TGL_PIPE_INT ||
1049 swsb.pipe == TGL_PIPE_FLOAT);
1050
1051 mode = swsb.pipe == TGL_PIPE_INT ? 0b11 :
1052 swsb.pipe == TGL_PIPE_FLOAT ? 0b10 :
1053 /* swsb.pipe == TGL_PIPE_ALL */ 0b01;
1054 } else {
1055 assert(!(swsb.mode & ~(TGL_SBID_DST | TGL_SBID_SRC)));
1056 mode = swsb.pipe == TGL_PIPE_ALL ? 0b11 :
1057 swsb.mode == TGL_SBID_SRC ? 0b10 :
1058 /* swsb.mode == TGL_SBID_DST */ 0b01;
1059 }
1060 return mode << 8 | swsb.regdist << 5 | swsb.sbid;
1061 } else {
1062 assert(!(swsb.sbid & ~0xfu));
1063 return 0x80 | swsb.regdist << 4 | swsb.sbid;
1064 }
1065
1066 } else {
1067 if (devinfo->ver >= 20) {
1068 return swsb.sbid | (swsb.mode & TGL_SBID_SET ? 0xc0 :
1069 swsb.mode & TGL_SBID_DST ? 0x80 : 0xa0);
1070 } else {
1071 assert(!(swsb.sbid & ~0xfu));
1072 return swsb.sbid | (swsb.mode & TGL_SBID_SET ? 0x40 :
1073 swsb.mode & TGL_SBID_DST ? 0x20 : 0x30);
1074 }
1075 }
1076 }
1077
1078 /**
1079 * Convert the provided binary representation of an SWSB annotation to a
1080 * tgl_swsb.
1081 */
1082 static inline struct tgl_swsb
tgl_swsb_decode(const struct intel_device_info * devinfo,const bool is_unordered,const uint32_t x,enum opcode opcode)1083 tgl_swsb_decode(const struct intel_device_info *devinfo,
1084 const bool is_unordered, const uint32_t x, enum opcode opcode)
1085 {
1086 if (devinfo->ver >= 20) {
1087 if (x & 0x300) {
1088 /* Mode isn't SingleInfo, there's a tuple */
1089 if (opcode == BRW_OPCODE_SEND || opcode == BRW_OPCODE_SENDC) {
1090 const struct tgl_swsb swsb = {
1091 (x & 0xe0u) >> 5,
1092 ((x & 0x300) == 0x300 ? TGL_PIPE_INT :
1093 (x & 0x300) == 0x200 ? TGL_PIPE_FLOAT :
1094 TGL_PIPE_ALL),
1095 x & 0x1fu,
1096 TGL_SBID_SET
1097 };
1098 return swsb;
1099 } else if (opcode == BRW_OPCODE_DPAS) {
1100 const struct tgl_swsb swsb = {
1101 .regdist = (x & 0xe0u) >> 5,
1102 .pipe = TGL_PIPE_NONE,
1103 .sbid = x & 0x1fu,
1104 .mode = (x & 0x300) == 0x300 ? TGL_SBID_DST :
1105 (x & 0x300) == 0x200 ? TGL_SBID_SRC :
1106 TGL_SBID_SET,
1107 };
1108 return swsb;
1109 } else {
1110 const struct tgl_swsb swsb = {
1111 (x & 0xe0u) >> 5,
1112 ((x & 0x300) == 0x300 ? TGL_PIPE_ALL : TGL_PIPE_NONE),
1113 x & 0x1fu,
1114 ((x & 0x300) == 0x200 ? TGL_SBID_SRC : TGL_SBID_DST)
1115 };
1116 return swsb;
1117 }
1118
1119 } else if ((x & 0xe0) == 0x80) {
1120 return tgl_swsb_sbid(TGL_SBID_DST, x & 0x1f);
1121 } else if ((x & 0xe0) == 0xa0) {
1122 return tgl_swsb_sbid(TGL_SBID_SRC, x & 0x1fu);
1123 } else if ((x & 0xe0) == 0xc0) {
1124 return tgl_swsb_sbid(TGL_SBID_SET, x & 0x1fu);
1125 } else {
1126 const struct tgl_swsb swsb = { x & 0x7u,
1127 ((x & 0x38) == 0x10 ? TGL_PIPE_FLOAT :
1128 (x & 0x38) == 0x18 ? TGL_PIPE_INT :
1129 (x & 0x38) == 0x20 ? TGL_PIPE_LONG :
1130 (x & 0x38) == 0x28 ? TGL_PIPE_MATH :
1131 (x & 0x38) == 0x8 ? TGL_PIPE_ALL :
1132 TGL_PIPE_NONE) };
1133 return swsb;
1134 }
1135
1136 } else {
1137 if (x & 0x80) {
1138 const struct tgl_swsb swsb = { (x & 0x70u) >> 4, TGL_PIPE_NONE,
1139 x & 0xfu,
1140 is_unordered ?
1141 TGL_SBID_SET : TGL_SBID_DST };
1142 return swsb;
1143 } else if ((x & 0x70) == 0x20) {
1144 return tgl_swsb_sbid(TGL_SBID_DST, x & 0xfu);
1145 } else if ((x & 0x70) == 0x30) {
1146 return tgl_swsb_sbid(TGL_SBID_SRC, x & 0xfu);
1147 } else if ((x & 0x70) == 0x40) {
1148 return tgl_swsb_sbid(TGL_SBID_SET, x & 0xfu);
1149 } else {
1150 const struct tgl_swsb swsb = { x & 0x7u,
1151 ((x & 0x78) == 0x10 ? TGL_PIPE_FLOAT :
1152 (x & 0x78) == 0x18 ? TGL_PIPE_INT :
1153 (x & 0x78) == 0x50 ? TGL_PIPE_LONG :
1154 (x & 0x78) == 0x8 ? TGL_PIPE_ALL :
1155 TGL_PIPE_NONE) };
1156 assert(devinfo->verx10 >= 125 || swsb.pipe == TGL_PIPE_NONE);
1157 return swsb;
1158 }
1159 }
1160 }
1161
1162 enum tgl_sync_function {
1163 TGL_SYNC_NOP = 0x0,
1164 TGL_SYNC_ALLRD = 0x2,
1165 TGL_SYNC_ALLWR = 0x3,
1166 TGL_SYNC_FENCE = 0xd,
1167 TGL_SYNC_BAR = 0xe,
1168 TGL_SYNC_HOST = 0xf
1169 };
1170
1171 /**
1172 * Message target: Shared Function ID for where to SEND a message.
1173 *
1174 * These are enumerated in the ISA reference under "send - Send Message".
1175 * In particular, see the following tables:
1176 * - G45 PRM, Volume 4, Table 14-15 "Message Descriptor Definition"
1177 * - Sandybridge PRM, Volume 4 Part 2, Table 8-16 "Extended Message Descriptor"
1178 * - Ivybridge PRM, Volume 1 Part 1, section 3.2.7 "GPE Function IDs"
1179 */
1180 enum brw_message_target {
1181 BRW_SFID_NULL = 0,
1182 BRW_SFID_SAMPLER = 2,
1183 BRW_SFID_MESSAGE_GATEWAY = 3,
1184 BRW_SFID_URB = 6,
1185 BRW_SFID_THREAD_SPAWNER = 7,
1186 BRW_SFID_VME = 8,
1187
1188 GFX6_SFID_DATAPORT_SAMPLER_CACHE = 4,
1189 GFX6_SFID_DATAPORT_RENDER_CACHE = 5,
1190 GFX6_SFID_DATAPORT_CONSTANT_CACHE = 9,
1191
1192 GFX7_SFID_DATAPORT_DATA_CACHE = 10,
1193 GFX7_SFID_PIXEL_INTERPOLATOR = 11,
1194 HSW_SFID_DATAPORT_DATA_CACHE_1 = 12,
1195 HSW_SFID_CRE = 13,
1196
1197 GFX12_SFID_TGM = 13, /* Typed Global Memory */
1198 GFX12_SFID_SLM = 14, /* Shared Local Memory */
1199 GFX12_SFID_UGM = 15, /* Untyped Global Memory */
1200
1201 GEN_RT_SFID_BINDLESS_THREAD_DISPATCH = 7,
1202 GEN_RT_SFID_RAY_TRACE_ACCELERATOR = 8,
1203 };
1204
1205 #define GFX7_MESSAGE_TARGET_DP_DATA_CACHE 10
1206
1207 #define BRW_SAMPLER_RETURN_FORMAT_FLOAT32 0
1208 #define BRW_SAMPLER_RETURN_FORMAT_UINT32 2
1209 #define BRW_SAMPLER_RETURN_FORMAT_SINT32 3
1210
1211 #define GFX8_SAMPLER_RETURN_FORMAT_32BITS 0
1212 #define GFX8_SAMPLER_RETURN_FORMAT_16BITS 1
1213
1214 #define GFX5_SAMPLER_MESSAGE_SAMPLE 0
1215 #define GFX5_SAMPLER_MESSAGE_SAMPLE_BIAS 1
1216 #define GFX5_SAMPLER_MESSAGE_SAMPLE_LOD 2
1217 #define GFX5_SAMPLER_MESSAGE_SAMPLE_COMPARE 3
1218 #define GFX5_SAMPLER_MESSAGE_SAMPLE_DERIVS 4
1219 #define GFX5_SAMPLER_MESSAGE_SAMPLE_BIAS_COMPARE 5
1220 #define GFX5_SAMPLER_MESSAGE_SAMPLE_LOD_COMPARE 6
1221 #define GFX5_SAMPLER_MESSAGE_SAMPLE_LD 7
1222 #define GFX7_SAMPLER_MESSAGE_SAMPLE_GATHER4 8
1223 #define GFX5_SAMPLER_MESSAGE_LOD 9
1224 #define GFX5_SAMPLER_MESSAGE_SAMPLE_RESINFO 10
1225 #define GFX6_SAMPLER_MESSAGE_SAMPLE_SAMPLEINFO 11
1226 #define XE2_SAMPLER_MESSAGE_SAMPLE_GATHER4_L 13
1227 #define XE2_SAMPLER_MESSAGE_SAMPLE_GATHER4_B 14
1228 #define XE2_SAMPLER_MESSAGE_SAMPLE_GATHER4_I 15
1229 #define GFX7_SAMPLER_MESSAGE_SAMPLE_GATHER4_C 16
1230 #define GFX7_SAMPLER_MESSAGE_SAMPLE_GATHER4_PO 17
1231 #define GFX7_SAMPLER_MESSAGE_SAMPLE_GATHER4_PO_C 18
1232 #define XE2_SAMPLER_MESSAGE_SAMPLE_MLOD 18
1233 #define XE2_SAMPLER_MESSAGE_SAMPLE_COMPARE_MLOD 19
1234 #define HSW_SAMPLER_MESSAGE_SAMPLE_DERIV_COMPARE 20
1235 #define XE2_SAMPLER_MESSAGE_SAMPLE_GATHER4_I_C 21
1236 #define XE2_SAMPLER_MESSAGE_SAMPLE_GATHER4_L_C 23
1237 #define GFX9_SAMPLER_MESSAGE_SAMPLE_LZ 24
1238 #define GFX9_SAMPLER_MESSAGE_SAMPLE_C_LZ 25
1239 #define GFX9_SAMPLER_MESSAGE_SAMPLE_LD_LZ 26
1240 #define GFX9_SAMPLER_MESSAGE_SAMPLE_LD2DMS_W 28
1241 #define GFX7_SAMPLER_MESSAGE_SAMPLE_LD_MCS 29
1242 #define GFX7_SAMPLER_MESSAGE_SAMPLE_LD2DMS 30
1243 #define GFX7_SAMPLER_MESSAGE_SAMPLE_LD2DSS 31
1244 #define XE2_SAMPLER_MESSAGE_SAMPLE_GATHER4_PO_L 45
1245 #define XE2_SAMPLER_MESSAGE_SAMPLE_GATHER4_PO_B 46
1246 #define XE2_SAMPLER_MESSAGE_SAMPLE_GATHER4_PO_L_C 55
1247
1248 /* for GFX5 only */
1249 #define BRW_SAMPLER_SIMD_MODE_SIMD4X2 0
1250 #define BRW_SAMPLER_SIMD_MODE_SIMD8 1
1251 #define BRW_SAMPLER_SIMD_MODE_SIMD16 2
1252 #define BRW_SAMPLER_SIMD_MODE_SIMD32_64 3
1253
1254 #define GFX10_SAMPLER_SIMD_MODE_SIMD8H 5
1255 #define GFX10_SAMPLER_SIMD_MODE_SIMD16H 6
1256
1257 #define XE2_SAMPLER_SIMD_MODE_SIMD16 1
1258 #define XE2_SAMPLER_SIMD_MODE_SIMD32 2
1259 #define XE2_SAMPLER_SIMD_MODE_SIMD16H 5
1260 #define XE2_SAMPLER_SIMD_MODE_SIMD32H 6
1261
1262 /* GFX9 changes SIMD mode 0 to mean SIMD8D, but lets us get the SIMD4x2
1263 * behavior by setting bit 22 of dword 2 in the message header. */
1264 #define GFX9_SAMPLER_SIMD_MODE_SIMD8D 0
1265 #define GFX9_SAMPLER_SIMD_MODE_EXTENSION_SIMD4X2 (1 << 22)
1266
1267 #define BRW_DATAPORT_OWORD_BLOCK_1_OWORDLOW 0
1268 #define BRW_DATAPORT_OWORD_BLOCK_1_OWORDHIGH 1
1269 #define BRW_DATAPORT_OWORD_BLOCK_2_OWORDS 2
1270 #define BRW_DATAPORT_OWORD_BLOCK_4_OWORDS 3
1271 #define BRW_DATAPORT_OWORD_BLOCK_8_OWORDS 4
1272 #define GFX12_DATAPORT_OWORD_BLOCK_16_OWORDS 5
1273 #define BRW_DATAPORT_OWORD_BLOCK_OWORDS(n) \
1274 ((n) == 1 ? BRW_DATAPORT_OWORD_BLOCK_1_OWORDLOW : \
1275 (n) == 2 ? BRW_DATAPORT_OWORD_BLOCK_2_OWORDS : \
1276 (n) == 4 ? BRW_DATAPORT_OWORD_BLOCK_4_OWORDS : \
1277 (n) == 8 ? BRW_DATAPORT_OWORD_BLOCK_8_OWORDS : \
1278 (n) == 16 ? GFX12_DATAPORT_OWORD_BLOCK_16_OWORDS : \
1279 (abort(), ~0))
1280 #define BRW_DATAPORT_OWORD_BLOCK_DWORDS(n) \
1281 ((n) == 4 ? BRW_DATAPORT_OWORD_BLOCK_1_OWORDLOW : \
1282 (n) == 8 ? BRW_DATAPORT_OWORD_BLOCK_2_OWORDS : \
1283 (n) == 16 ? BRW_DATAPORT_OWORD_BLOCK_4_OWORDS : \
1284 (n) == 32 ? BRW_DATAPORT_OWORD_BLOCK_8_OWORDS : \
1285 (abort(), ~0))
1286
1287 #define BRW_DATAPORT_OWORD_DUAL_BLOCK_1OWORD 0
1288 #define BRW_DATAPORT_OWORD_DUAL_BLOCK_4OWORDS 2
1289
1290 #define BRW_DATAPORT_DWORD_SCATTERED_BLOCK_8DWORDS 2
1291 #define BRW_DATAPORT_DWORD_SCATTERED_BLOCK_16DWORDS 3
1292
1293 /* This one stays the same across generations. */
1294 #define BRW_DATAPORT_READ_MESSAGE_OWORD_BLOCK_READ 0
1295 /* GFX6 */
1296 #define GFX6_DATAPORT_READ_MESSAGE_RENDER_UNORM_READ 1
1297 #define GFX6_DATAPORT_READ_MESSAGE_OWORD_DUAL_BLOCK_READ 2
1298 #define GFX6_DATAPORT_READ_MESSAGE_MEDIA_BLOCK_READ 4
1299 #define GFX6_DATAPORT_READ_MESSAGE_OWORD_UNALIGN_BLOCK_READ 5
1300 #define GFX6_DATAPORT_READ_MESSAGE_DWORD_SCATTERED_READ 6
1301
1302 #define BRW_DATAPORT_RENDER_TARGET_WRITE_SIMD16_SINGLE_SOURCE 0
1303 #define BRW_DATAPORT_RENDER_TARGET_WRITE_SIMD16_SINGLE_SOURCE_REPLICATED 1
1304 #define BRW_DATAPORT_RENDER_TARGET_WRITE_SIMD8_DUAL_SOURCE_SUBSPAN01 2
1305 #define BRW_DATAPORT_RENDER_TARGET_WRITE_SIMD8_DUAL_SOURCE_SUBSPAN23 3
1306 #define BRW_DATAPORT_RENDER_TARGET_WRITE_SIMD8_SINGLE_SOURCE_SUBSPAN01 4
1307
1308 #define XE2_DATAPORT_RENDER_TARGET_WRITE_SIMD32_SINGLE_SOURCE 1
1309 #define XE2_DATAPORT_RENDER_TARGET_WRITE_SIMD16_DUAL_SOURCE 2
1310
1311 /* GFX6 */
1312 #define GFX6_DATAPORT_WRITE_MESSAGE_DWORD_ATOMIC_WRITE 7
1313 #define GFX6_DATAPORT_WRITE_MESSAGE_OWORD_BLOCK_WRITE 8
1314 #define GFX6_DATAPORT_WRITE_MESSAGE_OWORD_DUAL_BLOCK_WRITE 9
1315 #define GFX6_DATAPORT_WRITE_MESSAGE_MEDIA_BLOCK_WRITE 10
1316 #define GFX6_DATAPORT_WRITE_MESSAGE_DWORD_SCATTERED_WRITE 11
1317 #define GFX6_DATAPORT_WRITE_MESSAGE_RENDER_TARGET_WRITE 12
1318 #define GFX6_DATAPORT_WRITE_MESSAGE_STREAMED_VB_WRITE 13
1319 #define GFX6_DATAPORT_WRITE_MESSAGE_RENDER_TARGET_UNORM_WRITE 14
1320
1321 /* GFX7 */
1322 #define GFX7_DATAPORT_RC_MEDIA_BLOCK_READ 4
1323 #define GFX7_DATAPORT_RC_TYPED_SURFACE_READ 5
1324 #define GFX7_DATAPORT_RC_TYPED_ATOMIC_OP 6
1325 #define GFX7_DATAPORT_RC_MEMORY_FENCE 7
1326 #define GFX7_DATAPORT_RC_MEDIA_BLOCK_WRITE 10
1327 #define GFX7_DATAPORT_RC_RENDER_TARGET_WRITE 12
1328 #define GFX7_DATAPORT_RC_TYPED_SURFACE_WRITE 13
1329 #define GFX7_DATAPORT_DC_OWORD_BLOCK_READ 0
1330 #define GFX7_DATAPORT_DC_UNALIGNED_OWORD_BLOCK_READ 1
1331 #define GFX7_DATAPORT_DC_OWORD_DUAL_BLOCK_READ 2
1332 #define GFX7_DATAPORT_DC_DWORD_SCATTERED_READ 3
1333 #define GFX7_DATAPORT_DC_BYTE_SCATTERED_READ 4
1334 #define GFX7_DATAPORT_DC_UNTYPED_SURFACE_READ 5
1335 #define GFX7_DATAPORT_DC_UNTYPED_ATOMIC_OP 6
1336 #define GFX7_DATAPORT_DC_MEMORY_FENCE 7
1337 #define GFX7_DATAPORT_DC_OWORD_BLOCK_WRITE 8
1338 #define GFX7_DATAPORT_DC_OWORD_DUAL_BLOCK_WRITE 10
1339 #define GFX7_DATAPORT_DC_DWORD_SCATTERED_WRITE 11
1340 #define GFX7_DATAPORT_DC_BYTE_SCATTERED_WRITE 12
1341 #define GFX7_DATAPORT_DC_UNTYPED_SURFACE_WRITE 13
1342
1343 #define GFX7_DATAPORT_SCRATCH_READ ((1 << 18) | \
1344 (0 << 17))
1345 #define GFX7_DATAPORT_SCRATCH_WRITE ((1 << 18) | \
1346 (1 << 17))
1347 #define GFX7_DATAPORT_SCRATCH_NUM_REGS_SHIFT 12
1348
1349 #define GFX7_PIXEL_INTERPOLATOR_LOC_SHARED_OFFSET 0
1350 #define GFX7_PIXEL_INTERPOLATOR_LOC_SAMPLE 1
1351 #define GFX7_PIXEL_INTERPOLATOR_LOC_CENTROID 2
1352 #define GFX7_PIXEL_INTERPOLATOR_LOC_PER_SLOT_OFFSET 3
1353
1354 /* HSW */
1355 #define HSW_DATAPORT_DC_PORT0_OWORD_BLOCK_READ 0
1356 #define HSW_DATAPORT_DC_PORT0_UNALIGNED_OWORD_BLOCK_READ 1
1357 #define HSW_DATAPORT_DC_PORT0_OWORD_DUAL_BLOCK_READ 2
1358 #define HSW_DATAPORT_DC_PORT0_DWORD_SCATTERED_READ 3
1359 #define HSW_DATAPORT_DC_PORT0_BYTE_SCATTERED_READ 4
1360 #define HSW_DATAPORT_DC_PORT0_MEMORY_FENCE 7
1361 #define HSW_DATAPORT_DC_PORT0_OWORD_BLOCK_WRITE 8
1362 #define HSW_DATAPORT_DC_PORT0_OWORD_DUAL_BLOCK_WRITE 10
1363 #define HSW_DATAPORT_DC_PORT0_DWORD_SCATTERED_WRITE 11
1364 #define HSW_DATAPORT_DC_PORT0_BYTE_SCATTERED_WRITE 12
1365
1366 #define HSW_DATAPORT_DC_PORT1_UNTYPED_SURFACE_READ 1
1367 #define HSW_DATAPORT_DC_PORT1_UNTYPED_ATOMIC_OP 2
1368 #define HSW_DATAPORT_DC_PORT1_UNTYPED_ATOMIC_OP_SIMD4X2 3
1369 #define HSW_DATAPORT_DC_PORT1_MEDIA_BLOCK_READ 4
1370 #define HSW_DATAPORT_DC_PORT1_TYPED_SURFACE_READ 5
1371 #define HSW_DATAPORT_DC_PORT1_TYPED_ATOMIC_OP 6
1372 #define HSW_DATAPORT_DC_PORT1_TYPED_ATOMIC_OP_SIMD4X2 7
1373 #define HSW_DATAPORT_DC_PORT1_UNTYPED_SURFACE_WRITE 9
1374 #define HSW_DATAPORT_DC_PORT1_MEDIA_BLOCK_WRITE 10
1375 #define HSW_DATAPORT_DC_PORT1_ATOMIC_COUNTER_OP 11
1376 #define HSW_DATAPORT_DC_PORT1_ATOMIC_COUNTER_OP_SIMD4X2 12
1377 #define HSW_DATAPORT_DC_PORT1_TYPED_SURFACE_WRITE 13
1378 #define GFX9_DATAPORT_DC_PORT1_A64_SCATTERED_READ 0x10
1379 #define GFX8_DATAPORT_DC_PORT1_A64_UNTYPED_SURFACE_READ 0x11
1380 #define GFX8_DATAPORT_DC_PORT1_A64_UNTYPED_ATOMIC_OP 0x12
1381 #define GFX12_DATAPORT_DC_PORT1_A64_UNTYPED_ATOMIC_HALF_INT_OP 0x13
1382 #define GFX9_DATAPORT_DC_PORT1_A64_OWORD_BLOCK_READ 0x14
1383 #define GFX9_DATAPORT_DC_PORT1_A64_OWORD_BLOCK_WRITE 0x15
1384 #define GFX8_DATAPORT_DC_PORT1_A64_UNTYPED_SURFACE_WRITE 0x19
1385 #define GFX8_DATAPORT_DC_PORT1_A64_SCATTERED_WRITE 0x1a
1386 #define GFX9_DATAPORT_DC_PORT1_UNTYPED_ATOMIC_FLOAT_OP 0x1b
1387 #define GFX9_DATAPORT_DC_PORT1_A64_UNTYPED_ATOMIC_FLOAT_OP 0x1d
1388 #define GFX12_DATAPORT_DC_PORT1_A64_UNTYPED_ATOMIC_HALF_FLOAT_OP 0x1e
1389
1390 /* GFX9 */
1391 #define GFX9_DATAPORT_RC_RENDER_TARGET_WRITE 12
1392 #define GFX9_DATAPORT_RC_RENDER_TARGET_READ 13
1393
1394 /* A64 scattered message subtype */
1395 #define GFX8_A64_SCATTERED_SUBTYPE_BYTE 0
1396 #define GFX8_A64_SCATTERED_SUBTYPE_DWORD 1
1397 #define GFX8_A64_SCATTERED_SUBTYPE_QWORD 2
1398 #define GFX8_A64_SCATTERED_SUBTYPE_HWORD 3
1399
1400 /* Dataport special binding table indices: */
1401 #define BRW_BTI_STATELESS 255
1402 #define GFX7_BTI_SLM 254
1403
1404 /* The hardware docs are a bit contradictory here. On Haswell, where they
1405 * first added cache ability control, there were 5 different cache modes (see
1406 * HSW_BTI_STATELESS_* above). On Broadwell, they reduced to two:
1407 *
1408 * - IA-Coherent (BTI=255): Coherent within Gen and coherent within the
1409 * entire IA cache memory hierarchy.
1410 *
1411 * - Non-Coherent (BTI=253): Coherent within Gen, same cache type.
1412 *
1413 * Information about stateless cache coherency can be found in the "A32
1414 * Stateless" section of the "3D Media GPGPU" volume of the PRM for each
1415 * hardware generation.
1416 *
1417 * Unfortunately, the docs for MDC_STATELESS appear to have been copied and
1418 * pasted from Haswell and give the Haswell definitions for the BTI values of
1419 * 255 and 253 including a warning about accessing 253 surfaces from multiple
1420 * threads. This seems to be a copy+paste error and the definitions from the
1421 * "A32 Stateless" section should be trusted instead.
1422 *
1423 * Note that because the DRM sets bit 4 of HDC_CHICKEN0 on BDW, CHV and at
1424 * least some pre-production steppings of SKL due to WaForceEnableNonCoherent,
1425 * HDC memory access may have been overridden by the kernel to be non-coherent
1426 * (matching the behavior of the same BTI on pre-Gfx8 hardware) and BTI 255
1427 * may actually be an alias for BTI 253.
1428 */
1429 #define GFX8_BTI_STATELESS_IA_COHERENT 255
1430 #define GFX8_BTI_STATELESS_NON_COHERENT 253
1431 #define GFX9_BTI_BINDLESS 252
1432
1433 /* Dataport atomic operations for Untyped Atomic Integer Operation message
1434 * (and others).
1435 */
1436 #define BRW_AOP_AND 1
1437 #define BRW_AOP_OR 2
1438 #define BRW_AOP_XOR 3
1439 #define BRW_AOP_MOV 4
1440 #define BRW_AOP_INC 5
1441 #define BRW_AOP_DEC 6
1442 #define BRW_AOP_ADD 7
1443 #define BRW_AOP_SUB 8
1444 #define BRW_AOP_REVSUB 9
1445 #define BRW_AOP_IMAX 10
1446 #define BRW_AOP_IMIN 11
1447 #define BRW_AOP_UMAX 12
1448 #define BRW_AOP_UMIN 13
1449 #define BRW_AOP_CMPWR 14
1450 #define BRW_AOP_PREDEC 15
1451
1452 /* Dataport atomic operations for Untyped Atomic Float Operation message. */
1453 #define BRW_AOP_FMAX 1
1454 #define BRW_AOP_FMIN 2
1455 #define BRW_AOP_FCMPWR 3
1456 #define BRW_AOP_FADD 4
1457
1458 #define BRW_MATH_FUNCTION_INV 1
1459 #define BRW_MATH_FUNCTION_LOG 2
1460 #define BRW_MATH_FUNCTION_EXP 3
1461 #define BRW_MATH_FUNCTION_SQRT 4
1462 #define BRW_MATH_FUNCTION_RSQ 5
1463 #define BRW_MATH_FUNCTION_SIN 6
1464 #define BRW_MATH_FUNCTION_COS 7
1465 #define BRW_MATH_FUNCTION_FDIV 9 /* gfx6+ */
1466 #define BRW_MATH_FUNCTION_POW 10
1467 #define BRW_MATH_FUNCTION_INT_DIV_QUOTIENT_AND_REMAINDER 11
1468 #define BRW_MATH_FUNCTION_INT_DIV_QUOTIENT 12
1469 #define BRW_MATH_FUNCTION_INT_DIV_REMAINDER 13
1470 #define GFX8_MATH_FUNCTION_INVM 14
1471 #define GFX8_MATH_FUNCTION_RSQRTM 15
1472
1473 #define GFX7_URB_OPCODE_ATOMIC_MOV 4
1474 #define GFX7_URB_OPCODE_ATOMIC_INC 5
1475 #define GFX8_URB_OPCODE_ATOMIC_ADD 6
1476 #define GFX8_URB_OPCODE_SIMD8_WRITE 7
1477 #define GFX8_URB_OPCODE_SIMD8_READ 8
1478 #define GFX125_URB_OPCODE_FENCE 9
1479
1480 #define BRW_URB_SWIZZLE_NONE 0
1481 #define BRW_URB_SWIZZLE_INTERLEAVE 1
1482 #define BRW_URB_SWIZZLE_TRANSPOSE 2
1483
1484 #define BRW_MESSAGE_GATEWAY_SFID_OPEN_GATEWAY 0
1485 #define BRW_MESSAGE_GATEWAY_SFID_CLOSE_GATEWAY 1
1486 #define BRW_MESSAGE_GATEWAY_SFID_FORWARD_MSG 2
1487 #define BRW_MESSAGE_GATEWAY_SFID_GET_TIMESTAMP 3
1488 #define BRW_MESSAGE_GATEWAY_SFID_BARRIER_MSG 4
1489 #define BRW_MESSAGE_GATEWAY_SFID_UPDATE_GATEWAY_STATE 5
1490 #define BRW_MESSAGE_GATEWAY_SFID_MMIO_READ_WRITE 6
1491
1492
1493 /* Gfx7 "GS URB Entry Allocation Size" is a U9-1 field, so the maximum gs_size
1494 * is 2^9, or 512. It's counted in multiples of 64 bytes.
1495 *
1496 * Identical for VS, DS, and HS.
1497 */
1498 #define GFX7_MAX_GS_URB_ENTRY_SIZE_BYTES (512*64)
1499 #define GFX7_MAX_DS_URB_ENTRY_SIZE_BYTES (512*64)
1500 #define GFX7_MAX_HS_URB_ENTRY_SIZE_BYTES (512*64)
1501 #define GFX7_MAX_VS_URB_ENTRY_SIZE_BYTES (512*64)
1502
1503 /* GS Thread Payload
1504 */
1505
1506 /* 3DSTATE_GS "Output Vertex Size" has an effective maximum of 62. It's
1507 * counted in multiples of 16 bytes.
1508 */
1509 #define GFX7_MAX_GS_OUTPUT_VERTEX_SIZE_BYTES (62*16)
1510
1511
1512 /* CR0.0[5:4] Floating-Point Rounding Modes
1513 * Skylake PRM, Volume 7 Part 1, "Control Register", page 756
1514 */
1515
1516 #define BRW_CR0_RND_MODE_MASK 0x30
1517 #define BRW_CR0_RND_MODE_SHIFT 4
1518
1519 enum ENUM_PACKED brw_rnd_mode {
1520 BRW_RND_MODE_RTNE = 0, /* Round to Nearest or Even */
1521 BRW_RND_MODE_RU = 1, /* Round Up, toward +inf */
1522 BRW_RND_MODE_RD = 2, /* Round Down, toward -inf */
1523 BRW_RND_MODE_RTZ = 3, /* Round Toward Zero */
1524 BRW_RND_MODE_UNSPECIFIED, /* Unspecified rounding mode */
1525 };
1526
1527 #define BRW_CR0_FP64_DENORM_PRESERVE (1 << 6)
1528 #define BRW_CR0_FP32_DENORM_PRESERVE (1 << 7)
1529 #define BRW_CR0_FP16_DENORM_PRESERVE (1 << 10)
1530
1531 #define BRW_CR0_FP_MODE_MASK (BRW_CR0_FP64_DENORM_PRESERVE | \
1532 BRW_CR0_FP32_DENORM_PRESERVE | \
1533 BRW_CR0_FP16_DENORM_PRESERVE | \
1534 BRW_CR0_RND_MODE_MASK)
1535
1536 /* MDC_DS - Data Size Message Descriptor Control Field
1537 * Skylake PRM, Volume 2d, page 129
1538 *
1539 * Specifies the number of Bytes to be read or written per Dword used at
1540 * byte_scattered read/write and byte_scaled read/write messages.
1541 */
1542 #define GFX7_BYTE_SCATTERED_DATA_ELEMENT_BYTE 0
1543 #define GFX7_BYTE_SCATTERED_DATA_ELEMENT_WORD 1
1544 #define GFX7_BYTE_SCATTERED_DATA_ELEMENT_DWORD 2
1545
1546 #define GEN_RT_BTD_MESSAGE_SPAWN 1
1547
1548 #define GEN_RT_TRACE_RAY_INITAL 0
1549 #define GEN_RT_TRACE_RAY_INSTANCE 1
1550 #define GEN_RT_TRACE_RAY_COMMIT 2
1551 #define GEN_RT_TRACE_RAY_CONTINUE 3
1552
1553 #define GEN_RT_BTD_SHADER_TYPE_ANY_HIT 0
1554 #define GEN_RT_BTD_SHADER_TYPE_CLOSEST_HIT 1
1555 #define GEN_RT_BTD_SHADER_TYPE_MISS 2
1556 #define GEN_RT_BTD_SHADER_TYPE_INTERSECTION 3
1557
1558 /* Starting with Xe-HPG, the old dataport was massively reworked dataport.
1559 * The new thing, called Load/Store Cache or LSC, has a significantly improved
1560 * interface. Instead of bespoke messages for every case, there's basically
1561 * one or two messages with different bits to control things like address
1562 * size, how much data is read/written, etc. It's way nicer but also means we
1563 * get to rewrite all our dataport encoding/decoding code. This patch kicks
1564 * off the party with all of the new enums.
1565 */
1566 enum lsc_opcode {
1567 LSC_OP_LOAD = 0,
1568 LSC_OP_LOAD_CMASK = 2,
1569 LSC_OP_STORE = 4,
1570 LSC_OP_STORE_CMASK = 6,
1571 LSC_OP_ATOMIC_INC = 8,
1572 LSC_OP_ATOMIC_DEC = 9,
1573 LSC_OP_ATOMIC_LOAD = 10,
1574 LSC_OP_ATOMIC_STORE = 11,
1575 LSC_OP_ATOMIC_ADD = 12,
1576 LSC_OP_ATOMIC_SUB = 13,
1577 LSC_OP_ATOMIC_MIN = 14,
1578 LSC_OP_ATOMIC_MAX = 15,
1579 LSC_OP_ATOMIC_UMIN = 16,
1580 LSC_OP_ATOMIC_UMAX = 17,
1581 LSC_OP_ATOMIC_CMPXCHG = 18,
1582 LSC_OP_ATOMIC_FADD = 19,
1583 LSC_OP_ATOMIC_FSUB = 20,
1584 LSC_OP_ATOMIC_FMIN = 21,
1585 LSC_OP_ATOMIC_FMAX = 22,
1586 LSC_OP_ATOMIC_FCMPXCHG = 23,
1587 LSC_OP_ATOMIC_AND = 24,
1588 LSC_OP_ATOMIC_OR = 25,
1589 LSC_OP_ATOMIC_XOR = 26,
1590 LSC_OP_FENCE = 31
1591 };
1592
1593 /*
1594 * Specifies the size of the dataport address payload in registers.
1595 */
1596 enum ENUM_PACKED lsc_addr_reg_size {
1597 LSC_ADDR_REG_SIZE_1 = 1,
1598 LSC_ADDR_REG_SIZE_2 = 2,
1599 LSC_ADDR_REG_SIZE_3 = 3,
1600 LSC_ADDR_REG_SIZE_4 = 4,
1601 LSC_ADDR_REG_SIZE_6 = 6,
1602 LSC_ADDR_REG_SIZE_8 = 8,
1603 };
1604
1605 /*
1606 * Specifies the size of the address payload item in a dataport message.
1607 */
1608 enum ENUM_PACKED lsc_addr_size {
1609 LSC_ADDR_SIZE_A16 = 1, /* 16-bit address offset */
1610 LSC_ADDR_SIZE_A32 = 2, /* 32-bit address offset */
1611 LSC_ADDR_SIZE_A64 = 3, /* 64-bit address offset */
1612 };
1613
1614 /*
1615 * Specifies the type of the address payload item in a dataport message. The
1616 * address type specifies how the dataport message decodes the Extended
1617 * Descriptor for the surface attributes and address calculation.
1618 */
1619 enum ENUM_PACKED lsc_addr_surface_type {
1620 LSC_ADDR_SURFTYPE_FLAT = 0, /* Flat */
1621 LSC_ADDR_SURFTYPE_BSS = 1, /* Bindless surface state */
1622 LSC_ADDR_SURFTYPE_SS = 2, /* Surface state */
1623 LSC_ADDR_SURFTYPE_BTI = 3, /* Binding table index */
1624 };
1625
1626 /*
1627 * Specifies the dataport message override to the default L1 and L3 memory
1628 * cache policies. Dataport L1 cache policies are uncached (UC), cached (C),
1629 * cache streaming (S) and invalidate-after-read (IAR). Dataport L3 cache
1630 * policies are uncached (UC) and cached (C).
1631 */
1632 enum lsc_cache_load {
1633 /* No override. Use the non-pipelined state or surface state cache settings
1634 * for L1 and L3.
1635 */
1636 LSC_CACHE_LOAD_L1STATE_L3MOCS = 0,
1637 /* Override to L1 uncached and L3 uncached */
1638 LSC_CACHE_LOAD_L1UC_L3UC = 1,
1639 /* Override to L1 uncached and L3 cached */
1640 LSC_CACHE_LOAD_L1UC_L3C = 2,
1641 /* Override to L1 cached and L3 uncached */
1642 LSC_CACHE_LOAD_L1C_L3UC = 3,
1643 /* Override to cache at both L1 and L3 */
1644 LSC_CACHE_LOAD_L1C_L3C = 4,
1645 /* Override to L1 streaming load and L3 uncached */
1646 LSC_CACHE_LOAD_L1S_L3UC = 5,
1647 /* Override to L1 streaming load and L3 cached */
1648 LSC_CACHE_LOAD_L1S_L3C = 6,
1649 /* For load messages, override to L1 invalidate-after-read, and L3 cached. */
1650 LSC_CACHE_LOAD_L1IAR_L3C = 7,
1651 };
1652
1653 /*
1654 * Specifies the dataport message override to the default L1 and L3 memory
1655 * cache policies. Dataport L1 cache policies are uncached (UC), cached (C),
1656 * streaming (S) and invalidate-after-read (IAR). Dataport L3 cache policies
1657 * are uncached (UC), cached (C), cached-as-a-constand (CC) and
1658 * invalidate-after-read (IAR).
1659 */
1660 enum PACKED xe2_lsc_cache_load {
1661 /* No override. Use the non-pipelined or surface state cache settings for L1
1662 * and L3.
1663 */
1664 XE2_LSC_CACHE_LOAD_L1STATE_L3MOCS = 0,
1665 /* Override to L1 uncached and L3 uncached */
1666 XE2_LSC_CACHE_LOAD_L1UC_L3UC = 2,
1667 /* Override to L1 uncached and L3 cached */
1668 XE2_LSC_CACHE_LOAD_L1UC_L3C = 4,
1669 /* Override to L1 uncached and L3 cached as a constant */
1670 XE2_LSC_CACHE_LOAD_L1UC_L3CC = 5,
1671 /* Override to L1 cached and L3 uncached */
1672 XE2_LSC_CACHE_LOAD_L1C_L3UC = 6,
1673 /* Override to L1 cached and L3 cached */
1674 XE2_LSC_CACHE_LOAD_L1C_L3C = 8,
1675 /* Override to L1 cached and L3 cached as a constant */
1676 XE2_LSC_CACHE_LOAD_L1C_L3CC = 9,
1677 /* Override to L1 cached as streaming load and L3 uncached */
1678 XE2_LSC_CACHE_LOAD_L1S_L3UC = 10,
1679 /* Override to L1 cached as streaming load and L3 cached */
1680 XE2_LSC_CACHE_LOAD_L1S_L3C = 12,
1681 /* Override to L1 and L3 invalidate after read */
1682 XE2_LSC_CACHE_LOAD_L1IAR_L3IAR = 14,
1683
1684 };
1685
1686 /*
1687 * Specifies the dataport message override to the default L1 and L3 memory
1688 * cache policies. Dataport L1 cache policies are uncached (UC), write-through
1689 * (WT), write-back (WB) and streaming (S). Dataport L3 cache policies are
1690 * uncached (UC) and cached (WB).
1691 */
1692 enum ENUM_PACKED lsc_cache_store {
1693 /* No override. Use the non-pipelined or surface state cache settings for L1
1694 * and L3.
1695 */
1696 LSC_CACHE_STORE_L1STATE_L3MOCS = 0,
1697 /* Override to L1 uncached and L3 uncached */
1698 LSC_CACHE_STORE_L1UC_L3UC = 1,
1699 /* Override to L1 uncached and L3 cached */
1700 LSC_CACHE_STORE_L1UC_L3WB = 2,
1701 /* Override to L1 write-through and L3 uncached */
1702 LSC_CACHE_STORE_L1WT_L3UC = 3,
1703 /* Override to L1 write-through and L3 cached */
1704 LSC_CACHE_STORE_L1WT_L3WB = 4,
1705 /* Override to L1 streaming and L3 uncached */
1706 LSC_CACHE_STORE_L1S_L3UC = 5,
1707 /* Override to L1 streaming and L3 cached */
1708 LSC_CACHE_STORE_L1S_L3WB = 6,
1709 /* Override to L1 write-back, and L3 cached */
1710 LSC_CACHE_STORE_L1WB_L3WB = 7,
1711
1712 };
1713
1714 /*
1715 * Specifies the dataport message override to the default L1 and L3 memory
1716 * cache policies. Dataport L1 cache policies are uncached (UC), write-through
1717 * (WT), write-back (WB) and streaming (S). Dataport L3 cache policies are
1718 * uncached (UC) and cached (WB).
1719 */
1720 enum PACKED xe2_lsc_cache_store {
1721 /* No override. Use the non-pipelined or surface state cache settings for L1
1722 * and L3.
1723 */
1724 XE2_LSC_CACHE_STORE_L1STATE_L3MOCS = 0,
1725 /* Override to L1 uncached and L3 uncached */
1726 XE2_LSC_CACHE_STORE_L1UC_L3UC = 2,
1727 /* Override to L1 uncached and L3 cached */
1728 XE2_LSC_CACHE_STORE_L1UC_L3WB = 4,
1729 /* Override to L1 write-through and L3 uncached */
1730 XE2_LSC_CACHE_STORE_L1WT_L3UC = 6,
1731 /* Override to L1 write-through and L3 cached */
1732 XE2_LSC_CACHE_STORE_L1WT_L3WB = 8,
1733 /* Override to L1 streaming and L3 uncached */
1734 XE2_LSC_CACHE_STORE_L1S_L3UC = 10,
1735 /* Override to L1 streaming and L3 cached */
1736 XE2_LSC_CACHE_STORE_L1S_L3WB = 12,
1737 /* Override to L1 write-back and L3 cached */
1738 XE2_LSC_CACHE_STORE_L1WB_L3WB = 14,
1739
1740 };
1741
1742 #define LSC_CACHE(devinfo, l_or_s, cc) \
1743 ((devinfo)->ver < 20 ? (unsigned)LSC_CACHE_ ## l_or_s ## _ ## cc : \
1744 (unsigned)XE2_LSC_CACHE_ ## l_or_s ## _ ## cc)
1745
1746 /*
1747 * Specifies which components of the data payload 4-element vector (X,Y,Z,W) is
1748 * packed into the register payload.
1749 */
1750 enum ENUM_PACKED lsc_cmask {
1751 LSC_CMASK_X = 0x1,
1752 LSC_CMASK_Y = 0x2,
1753 LSC_CMASK_XY = 0x3,
1754 LSC_CMASK_Z = 0x4,
1755 LSC_CMASK_XZ = 0x5,
1756 LSC_CMASK_YZ = 0x6,
1757 LSC_CMASK_XYZ = 0x7,
1758 LSC_CMASK_W = 0x8,
1759 LSC_CMASK_XW = 0x9,
1760 LSC_CMASK_YW = 0xa,
1761 LSC_CMASK_XYW = 0xb,
1762 LSC_CMASK_ZW = 0xc,
1763 LSC_CMASK_XZW = 0xd,
1764 LSC_CMASK_YZW = 0xe,
1765 LSC_CMASK_XYZW = 0xf,
1766 };
1767
1768 /*
1769 * Specifies the size of the data payload item in a dataport message.
1770 */
1771 enum ENUM_PACKED lsc_data_size {
1772 /* 8-bit scalar data value in memory, packed into a 8-bit data value in
1773 * register.
1774 */
1775 LSC_DATA_SIZE_D8 = 0,
1776 /* 16-bit scalar data value in memory, packed into a 16-bit data value in
1777 * register.
1778 */
1779 LSC_DATA_SIZE_D16 = 1,
1780 /* 32-bit scalar data value in memory, packed into 32-bit data value in
1781 * register.
1782 */
1783 LSC_DATA_SIZE_D32 = 2,
1784 /* 64-bit scalar data value in memory, packed into 64-bit data value in
1785 * register.
1786 */
1787 LSC_DATA_SIZE_D64 = 3,
1788 /* 8-bit scalar data value in memory, packed into 32-bit unsigned data value
1789 * in register.
1790 */
1791 LSC_DATA_SIZE_D8U32 = 4,
1792 /* 16-bit scalar data value in memory, packed into 32-bit unsigned data
1793 * value in register.
1794 */
1795 LSC_DATA_SIZE_D16U32 = 5,
1796 /* 16-bit scalar BigFloat data value in memory, packed into 32-bit float
1797 * value in register.
1798 */
1799 LSC_DATA_SIZE_D16BF32 = 6,
1800 };
1801
1802 /*
1803 * Enum specifies the scope of the fence.
1804 */
1805 enum ENUM_PACKED lsc_fence_scope {
1806 /* Wait until all previous memory transactions from this thread are observed
1807 * within the local thread-group.
1808 */
1809 LSC_FENCE_THREADGROUP = 0,
1810 /* Wait until all previous memory transactions from this thread are observed
1811 * within the local sub-slice.
1812 */
1813 LSC_FENCE_LOCAL = 1,
1814 /* Wait until all previous memory transactions from this thread are observed
1815 * in the local tile.
1816 */
1817 LSC_FENCE_TILE = 2,
1818 /* Wait until all previous memory transactions from this thread are observed
1819 * in the local GPU.
1820 */
1821 LSC_FENCE_GPU = 3,
1822 /* Wait until all previous memory transactions from this thread are observed
1823 * across all GPUs in the system.
1824 */
1825 LSC_FENCE_ALL_GPU = 4,
1826 /* Wait until all previous memory transactions from this thread are observed
1827 * at the "system" level.
1828 */
1829 LSC_FENCE_SYSTEM_RELEASE = 5,
1830 /* For GPUs that do not follow PCIe Write ordering for downstream writes
1831 * targeting device memory, a fence message with scope=System_Acquire will
1832 * commit to device memory all downstream and peer writes that have reached
1833 * the device.
1834 */
1835 LSC_FENCE_SYSTEM_ACQUIRE = 6,
1836 };
1837
1838 /*
1839 * Specifies the type of cache flush operation to perform after a fence is
1840 * complete.
1841 */
1842 enum ENUM_PACKED lsc_flush_type {
1843 LSC_FLUSH_TYPE_NONE = 0,
1844 /*
1845 * For a R/W cache, evict dirty lines (M to I state) and invalidate clean
1846 * lines. For a RO cache, invalidate clean lines.
1847 */
1848 LSC_FLUSH_TYPE_EVICT = 1,
1849 /*
1850 * For both R/W and RO cache, invalidate clean lines in the cache.
1851 */
1852 LSC_FLUSH_TYPE_INVALIDATE = 2,
1853 /*
1854 * For a R/W cache, invalidate dirty lines (M to I state), without
1855 * write-back to next level. This opcode does nothing for a RO cache.
1856 */
1857 LSC_FLUSH_TYPE_DISCARD = 3,
1858 /*
1859 * For a R/W cache, write-back dirty lines to the next level, but kept in
1860 * the cache as "clean" (M to V state). This opcode does nothing for a RO
1861 * cache.
1862 */
1863 LSC_FLUSH_TYPE_CLEAN = 4,
1864 /*
1865 * Flush "RW" section of the L3 cache, but leave L1 and L2 caches untouched.
1866 */
1867 LSC_FLUSH_TYPE_L3ONLY = 5,
1868 /*
1869 * HW maps this flush type internally to NONE.
1870 */
1871 LSC_FLUSH_TYPE_NONE_6 = 6,
1872
1873 };
1874
1875 enum ENUM_PACKED lsc_backup_fence_routing {
1876 /* Normal routing: UGM fence is routed to UGM pipeline. */
1877 LSC_NORMAL_ROUTING,
1878 /* Route UGM fence to LSC unit. */
1879 LSC_ROUTE_TO_LSC,
1880 };
1881
1882 /*
1883 * Specifies the size of the vector in a dataport message.
1884 */
1885 enum ENUM_PACKED lsc_vect_size {
1886 LSC_VECT_SIZE_V1 = 0, /* vector length 1 */
1887 LSC_VECT_SIZE_V2 = 1, /* vector length 2 */
1888 LSC_VECT_SIZE_V3 = 2, /* Vector length 3 */
1889 LSC_VECT_SIZE_V4 = 3, /* Vector length 4 */
1890 LSC_VECT_SIZE_V8 = 4, /* Vector length 8 */
1891 LSC_VECT_SIZE_V16 = 5, /* Vector length 16 */
1892 LSC_VECT_SIZE_V32 = 6, /* Vector length 32 */
1893 LSC_VECT_SIZE_V64 = 7, /* Vector length 64 */
1894 };
1895
1896 #define LSC_ONE_ADDR_REG 1
1897