• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 /*
2  * Copyright 2015 Advanced Micro Devices, Inc.
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice (including the next
12  * paragraph) shall be included in all copies or substantial portions of the
13  * Software.
14  *
15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21  * SOFTWARE.
22  */
23 
24 #include "radeon/r600_cs.h"
25 #include "radeon/r600_query.h"
26 #include "util/u_memory.h"
27 
28 #include "si_pipe.h"
29 #include "sid.h"
30 
31 enum si_pc_reg_layout {
32 	/* All secondary selector dwords follow as one block after the primary
33 	 * selector dwords for the counters that have secondary selectors.
34 	 */
35 	SI_PC_MULTI_BLOCK = 0,
36 
37 	/* Each secondary selector dword follows immediately afters the
38 	 * corresponding primary.
39 	 */
40 	SI_PC_MULTI_ALTERNATE = 1,
41 
42 	/* All secondary selector dwords follow as one block after all primary
43 	 * selector dwords.
44 	 */
45 	SI_PC_MULTI_TAIL = 2,
46 
47 	/* Free-form arrangement of selector registers. */
48 	SI_PC_MULTI_CUSTOM = 3,
49 
50 	SI_PC_MULTI_MASK = 3,
51 
52 	/* Registers are laid out in decreasing rather than increasing order. */
53 	SI_PC_REG_REVERSE = 4,
54 
55 	SI_PC_FAKE = 8,
56 };
57 
58 struct si_pc_block_base {
59 	const char *name;
60 	unsigned num_counters;
61 	unsigned flags;
62 
63 	unsigned select_or;
64 	unsigned select0;
65 	unsigned counter0_lo;
66 	unsigned *select;
67 	unsigned *counters;
68 	unsigned num_multi;
69 	unsigned num_prelude;
70 	unsigned layout;
71 };
72 
73 struct si_pc_block {
74 	struct si_pc_block_base *b;
75 	unsigned selectors;
76 	unsigned instances;
77 };
78 
79 /* The order is chosen to be compatible with GPUPerfStudio's hardcoding of
80  * performance counter group IDs.
81  */
82 static const char * const si_pc_shader_type_suffixes[] = {
83 	"", "_ES", "_GS", "_VS", "_PS", "_LS", "_HS", "_CS"
84 };
85 
86 static const unsigned si_pc_shader_type_bits[] = {
87 	0x7f,
88 	S_036780_ES_EN(1),
89 	S_036780_GS_EN(1),
90 	S_036780_VS_EN(1),
91 	S_036780_PS_EN(1),
92 	S_036780_LS_EN(1),
93 	S_036780_HS_EN(1),
94 	S_036780_CS_EN(1),
95 };
96 
97 static struct si_pc_block_base cik_CB = {
98 	.name = "CB",
99 	.num_counters = 4,
100 	.flags = R600_PC_BLOCK_SE | R600_PC_BLOCK_INSTANCE_GROUPS,
101 
102 	.select0 = R_037000_CB_PERFCOUNTER_FILTER,
103 	.counter0_lo = R_035018_CB_PERFCOUNTER0_LO,
104 	.num_multi = 1,
105 	.num_prelude = 1,
106 	.layout = SI_PC_MULTI_ALTERNATE,
107 };
108 
109 static unsigned cik_CPC_select[] = {
110 	R_036024_CPC_PERFCOUNTER0_SELECT,
111 	R_036010_CPC_PERFCOUNTER0_SELECT1,
112 	R_03600C_CPC_PERFCOUNTER1_SELECT,
113 };
114 static struct si_pc_block_base cik_CPC = {
115 	.name = "CPC",
116 	.num_counters = 2,
117 
118 	.select = cik_CPC_select,
119 	.counter0_lo = R_034018_CPC_PERFCOUNTER0_LO,
120 	.num_multi = 1,
121 	.layout = SI_PC_MULTI_CUSTOM | SI_PC_REG_REVERSE,
122 };
123 
124 static struct si_pc_block_base cik_CPF = {
125 	.name = "CPF",
126 	.num_counters = 2,
127 
128 	.select0 = R_03601C_CPF_PERFCOUNTER0_SELECT,
129 	.counter0_lo = R_034028_CPF_PERFCOUNTER0_LO,
130 	.num_multi = 1,
131 	.layout = SI_PC_MULTI_ALTERNATE | SI_PC_REG_REVERSE,
132 };
133 
134 static struct si_pc_block_base cik_CPG = {
135 	.name = "CPG",
136 	.num_counters = 2,
137 
138 	.select0 = R_036008_CPG_PERFCOUNTER0_SELECT,
139 	.counter0_lo = R_034008_CPG_PERFCOUNTER0_LO,
140 	.num_multi = 1,
141 	.layout = SI_PC_MULTI_ALTERNATE | SI_PC_REG_REVERSE,
142 };
143 
144 static struct si_pc_block_base cik_DB = {
145 	.name = "DB",
146 	.num_counters = 4,
147 	.flags = R600_PC_BLOCK_SE | R600_PC_BLOCK_INSTANCE_GROUPS,
148 
149 	.select0 = R_037100_DB_PERFCOUNTER0_SELECT,
150 	.counter0_lo = R_035100_DB_PERFCOUNTER0_LO,
151 	.num_multi = 3, // really only 2, but there's a gap between registers
152 	.layout = SI_PC_MULTI_ALTERNATE,
153 };
154 
155 static struct si_pc_block_base cik_GDS = {
156 	.name = "GDS",
157 	.num_counters = 4,
158 
159 	.select0 = R_036A00_GDS_PERFCOUNTER0_SELECT,
160 	.counter0_lo = R_034A00_GDS_PERFCOUNTER0_LO,
161 	.num_multi = 1,
162 	.layout = SI_PC_MULTI_TAIL,
163 };
164 
165 static unsigned cik_GRBM_counters[] = {
166 	R_034100_GRBM_PERFCOUNTER0_LO,
167 	R_03410C_GRBM_PERFCOUNTER1_LO,
168 };
169 static struct si_pc_block_base cik_GRBM = {
170 	.name = "GRBM",
171 	.num_counters = 2,
172 
173 	.select0 = R_036100_GRBM_PERFCOUNTER0_SELECT,
174 	.counters = cik_GRBM_counters,
175 };
176 
177 static struct si_pc_block_base cik_GRBMSE = {
178 	.name = "GRBMSE",
179 	.num_counters = 4,
180 
181 	.select0 = R_036108_GRBM_SE0_PERFCOUNTER_SELECT,
182 	.counter0_lo = R_034114_GRBM_SE0_PERFCOUNTER_LO,
183 };
184 
185 static struct si_pc_block_base cik_IA = {
186 	.name = "IA",
187 	.num_counters = 4,
188 
189 	.select0 = R_036210_IA_PERFCOUNTER0_SELECT,
190 	.counter0_lo = R_034220_IA_PERFCOUNTER0_LO,
191 	.num_multi = 1,
192 	.layout = SI_PC_MULTI_TAIL,
193 };
194 
195 static struct si_pc_block_base cik_PA_SC = {
196 	.name = "PA_SC",
197 	.num_counters = 8,
198 	.flags = R600_PC_BLOCK_SE,
199 
200 	.select0 = R_036500_PA_SC_PERFCOUNTER0_SELECT,
201 	.counter0_lo = R_034500_PA_SC_PERFCOUNTER0_LO,
202 	.num_multi = 1,
203 	.layout = SI_PC_MULTI_ALTERNATE,
204 };
205 
206 /* According to docs, PA_SU counters are only 48 bits wide. */
207 static struct si_pc_block_base cik_PA_SU = {
208 	.name = "PA_SU",
209 	.num_counters = 4,
210 	.flags = R600_PC_BLOCK_SE,
211 
212 	.select0 = R_036400_PA_SU_PERFCOUNTER0_SELECT,
213 	.counter0_lo = R_034400_PA_SU_PERFCOUNTER0_LO,
214 	.num_multi = 2,
215 	.layout = SI_PC_MULTI_ALTERNATE,
216 };
217 
218 static struct si_pc_block_base cik_SPI = {
219 	.name = "SPI",
220 	.num_counters = 6,
221 	.flags = R600_PC_BLOCK_SE,
222 
223 	.select0 = R_036600_SPI_PERFCOUNTER0_SELECT,
224 	.counter0_lo = R_034604_SPI_PERFCOUNTER0_LO,
225 	.num_multi = 4,
226 	.layout = SI_PC_MULTI_BLOCK,
227 };
228 
229 static struct si_pc_block_base cik_SQ = {
230 	.name = "SQ",
231 	.num_counters = 16,
232 	.flags = R600_PC_BLOCK_SE | R600_PC_BLOCK_SHADER,
233 
234 	.select0 = R_036700_SQ_PERFCOUNTER0_SELECT,
235 	.select_or = S_036700_SQC_BANK_MASK(15) |
236 			S_036700_SQC_CLIENT_MASK(15) |
237 			S_036700_SIMD_MASK(15),
238 	.counter0_lo = R_034700_SQ_PERFCOUNTER0_LO,
239 };
240 
241 static struct si_pc_block_base cik_SX = {
242 	.name = "SX",
243 	.num_counters = 4,
244 	.flags = R600_PC_BLOCK_SE,
245 
246 	.select0 = R_036900_SX_PERFCOUNTER0_SELECT,
247 	.counter0_lo = R_034900_SX_PERFCOUNTER0_LO,
248 	.num_multi = 2,
249 	.layout = SI_PC_MULTI_TAIL,
250 };
251 
252 static struct si_pc_block_base cik_TA = {
253 	.name = "TA",
254 	.num_counters = 2,
255 	.flags = R600_PC_BLOCK_SE | R600_PC_BLOCK_INSTANCE_GROUPS | R600_PC_BLOCK_SHADER_WINDOWED,
256 
257 	.select0 = R_036B00_TA_PERFCOUNTER0_SELECT,
258 	.counter0_lo = R_034B00_TA_PERFCOUNTER0_LO,
259 	.num_multi = 1,
260 	.layout = SI_PC_MULTI_ALTERNATE,
261 };
262 
263 static struct si_pc_block_base cik_TD = {
264 	.name = "TD",
265 	.num_counters = 2,
266 	.flags = R600_PC_BLOCK_SE | R600_PC_BLOCK_INSTANCE_GROUPS | R600_PC_BLOCK_SHADER_WINDOWED,
267 
268 	.select0 = R_036C00_TD_PERFCOUNTER0_SELECT,
269 	.counter0_lo = R_034C00_TD_PERFCOUNTER0_LO,
270 	.num_multi = 1,
271 	.layout = SI_PC_MULTI_ALTERNATE,
272 };
273 
274 static struct si_pc_block_base cik_TCA = {
275 	.name = "TCA",
276 	.num_counters = 4,
277 	.flags = R600_PC_BLOCK_INSTANCE_GROUPS,
278 
279 	.select0 = R_036E40_TCA_PERFCOUNTER0_SELECT,
280 	.counter0_lo = R_034E40_TCA_PERFCOUNTER0_LO,
281 	.num_multi = 2,
282 	.layout = SI_PC_MULTI_ALTERNATE,
283 };
284 
285 static struct si_pc_block_base cik_TCC = {
286 	.name = "TCC",
287 	.num_counters = 4,
288 	.flags = R600_PC_BLOCK_INSTANCE_GROUPS,
289 
290 	.select0 = R_036E00_TCC_PERFCOUNTER0_SELECT,
291 	.counter0_lo = R_034E00_TCC_PERFCOUNTER0_LO,
292 	.num_multi = 2,
293 	.layout = SI_PC_MULTI_ALTERNATE,
294 };
295 
296 static struct si_pc_block_base cik_TCP = {
297 	.name = "TCP",
298 	.num_counters = 4,
299 	.flags = R600_PC_BLOCK_SE | R600_PC_BLOCK_INSTANCE_GROUPS | R600_PC_BLOCK_SHADER_WINDOWED,
300 
301 	.select0 = R_036D00_TCP_PERFCOUNTER0_SELECT,
302 	.counter0_lo = R_034D00_TCP_PERFCOUNTER0_LO,
303 	.num_multi = 2,
304 	.layout = SI_PC_MULTI_ALTERNATE,
305 };
306 
307 static struct si_pc_block_base cik_VGT = {
308 	.name = "VGT",
309 	.num_counters = 4,
310 	.flags = R600_PC_BLOCK_SE,
311 
312 	.select0 = R_036230_VGT_PERFCOUNTER0_SELECT,
313 	.counter0_lo = R_034240_VGT_PERFCOUNTER0_LO,
314 	.num_multi = 1,
315 	.layout = SI_PC_MULTI_TAIL,
316 };
317 
318 static struct si_pc_block_base cik_WD = {
319 	.name = "WD",
320 	.num_counters = 4,
321 
322 	.select0 = R_036200_WD_PERFCOUNTER0_SELECT,
323 	.counter0_lo = R_034200_WD_PERFCOUNTER0_LO,
324 };
325 
326 static struct si_pc_block_base cik_MC = {
327 	.name = "MC",
328 	.num_counters = 4,
329 
330 	.layout = SI_PC_FAKE,
331 };
332 
333 static struct si_pc_block_base cik_SRBM = {
334 	.name = "SRBM",
335 	.num_counters = 2,
336 
337 	.layout = SI_PC_FAKE,
338 };
339 
340 /* Both the number of instances and selectors varies between chips of the same
341  * class. We only differentiate by class here and simply expose the maximum
342  * number over all chips in a class.
343  *
344  * Unfortunately, GPUPerfStudio uses the order of performance counter groups
345  * blindly once it believes it has identified the hardware, so the order of
346  * blocks here matters.
347  */
348 static struct si_pc_block groups_CIK[] = {
349 	{ &cik_CB, 226, 4 },
350 	{ &cik_CPF, 17 },
351 	{ &cik_DB, 257, 4 },
352 	{ &cik_GRBM, 34 },
353 	{ &cik_GRBMSE, 15 },
354 	{ &cik_PA_SU, 153 },
355 	{ &cik_PA_SC, 395 },
356 	{ &cik_SPI, 186 },
357 	{ &cik_SQ, 252 },
358 	{ &cik_SX, 32 },
359 	{ &cik_TA, 111, 11 },
360 	{ &cik_TCA, 39, 2 },
361 	{ &cik_TCC, 160, 16 },
362 	{ &cik_TD, 55, 11 },
363 	{ &cik_TCP, 154, 11 },
364 	{ &cik_GDS, 121 },
365 	{ &cik_VGT, 140 },
366 	{ &cik_IA, 22 },
367 	{ &cik_MC, 22 },
368 	{ &cik_SRBM, 19 },
369 	{ &cik_WD, 22 },
370 	{ &cik_CPG, 46 },
371 	{ &cik_CPC, 22 },
372 
373 };
374 
375 static struct si_pc_block groups_VI[] = {
376 	{ &cik_CB, 405, 4 },
377 	{ &cik_CPF, 19 },
378 	{ &cik_DB, 257, 4 },
379 	{ &cik_GRBM, 34 },
380 	{ &cik_GRBMSE, 15 },
381 	{ &cik_PA_SU, 153 },
382 	{ &cik_PA_SC, 397 },
383 	{ &cik_SPI, 197 },
384 	{ &cik_SQ, 273 },
385 	{ &cik_SX, 34 },
386 	{ &cik_TA, 119, 16 },
387 	{ &cik_TCA, 35, 2 },
388 	{ &cik_TCC, 192, 16 },
389 	{ &cik_TD, 55, 16 },
390 	{ &cik_TCP, 180, 16 },
391 	{ &cik_GDS, 121 },
392 	{ &cik_VGT, 147 },
393 	{ &cik_IA, 24 },
394 	{ &cik_MC, 22 },
395 	{ &cik_SRBM, 27 },
396 	{ &cik_WD, 37 },
397 	{ &cik_CPG, 48 },
398 	{ &cik_CPC, 24 },
399 
400 };
401 
402 static struct si_pc_block groups_gfx9[] = {
403 	{ &cik_CB, 438, 4 },
404 	{ &cik_CPF, 32 },
405 	{ &cik_DB, 328, 4 },
406 	{ &cik_GRBM, 38 },
407 	{ &cik_GRBMSE, 16 },
408 	{ &cik_PA_SU, 292 },
409 	{ &cik_PA_SC, 491 },
410 	{ &cik_SPI, 196 },
411 	{ &cik_SQ, 374 },
412 	{ &cik_SX, 208 },
413 	{ &cik_TA, 119, 16 },
414 	{ &cik_TCA, 35, 2 },
415 	{ &cik_TCC, 256, 16 },
416 	{ &cik_TD, 57, 16 },
417 	{ &cik_TCP, 85, 16 },
418 	{ &cik_GDS, 121 },
419 	{ &cik_VGT, 148 },
420 	{ &cik_IA, 32 },
421 	{ &cik_WD, 58 },
422 	{ &cik_CPG, 59 },
423 	{ &cik_CPC, 35 },
424 };
425 
si_pc_get_size(struct r600_perfcounter_block * group,unsigned count,unsigned * selectors,unsigned * num_select_dw,unsigned * num_read_dw)426 static void si_pc_get_size(struct r600_perfcounter_block *group,
427 			unsigned count, unsigned *selectors,
428 			unsigned *num_select_dw, unsigned *num_read_dw)
429 {
430 	struct si_pc_block *sigroup = (struct si_pc_block *)group->data;
431 	struct si_pc_block_base *regs = sigroup->b;
432 	unsigned layout_multi = regs->layout & SI_PC_MULTI_MASK;
433 
434 	if (regs->layout & SI_PC_FAKE) {
435 		*num_select_dw = 0;
436 	} else if (layout_multi == SI_PC_MULTI_BLOCK) {
437 		if (count < regs->num_multi)
438 			*num_select_dw = 2 * (count + 2) + regs->num_prelude;
439 		else
440 			*num_select_dw = 2 + count + regs->num_multi + regs->num_prelude;
441 	} else if (layout_multi == SI_PC_MULTI_TAIL) {
442 		*num_select_dw = 4 + count + MIN2(count, regs->num_multi) + regs->num_prelude;
443 	} else if (layout_multi == SI_PC_MULTI_CUSTOM) {
444 		assert(regs->num_prelude == 0);
445 		*num_select_dw = 3 * (count + MIN2(count, regs->num_multi));
446 	} else {
447 		assert(layout_multi == SI_PC_MULTI_ALTERNATE);
448 
449 		*num_select_dw = 2 + count + MIN2(count, regs->num_multi) + regs->num_prelude;
450 	}
451 
452 	*num_read_dw = 6 * count;
453 }
454 
si_pc_emit_instance(struct r600_common_context * ctx,int se,int instance)455 static void si_pc_emit_instance(struct r600_common_context *ctx,
456 				int se, int instance)
457 {
458 	struct radeon_winsys_cs *cs = ctx->gfx.cs;
459 	unsigned value = S_030800_SH_BROADCAST_WRITES(1);
460 
461 	if (se >= 0) {
462 		value |= S_030800_SE_INDEX(se);
463 	} else {
464 		value |= S_030800_SE_BROADCAST_WRITES(1);
465 	}
466 
467 	if (instance >= 0) {
468 		value |= S_030800_INSTANCE_INDEX(instance);
469 	} else {
470 		value |= S_030800_INSTANCE_BROADCAST_WRITES(1);
471 	}
472 
473 	radeon_set_uconfig_reg(cs, R_030800_GRBM_GFX_INDEX, value);
474 }
475 
si_pc_emit_shaders(struct r600_common_context * ctx,unsigned shaders)476 static void si_pc_emit_shaders(struct r600_common_context *ctx,
477 			       unsigned shaders)
478 {
479 	struct radeon_winsys_cs *cs = ctx->gfx.cs;
480 
481 	radeon_set_uconfig_reg_seq(cs, R_036780_SQ_PERFCOUNTER_CTRL, 2);
482 	radeon_emit(cs, shaders & 0x7f);
483 	radeon_emit(cs, 0xffffffff);
484 }
485 
si_pc_emit_select(struct r600_common_context * ctx,struct r600_perfcounter_block * group,unsigned count,unsigned * selectors)486 static void si_pc_emit_select(struct r600_common_context *ctx,
487 		        struct r600_perfcounter_block *group,
488 		        unsigned count, unsigned *selectors)
489 {
490 	struct si_pc_block *sigroup = (struct si_pc_block *)group->data;
491 	struct si_pc_block_base *regs = sigroup->b;
492 	struct radeon_winsys_cs *cs = ctx->gfx.cs;
493 	unsigned idx;
494 	unsigned layout_multi = regs->layout & SI_PC_MULTI_MASK;
495 	unsigned dw;
496 
497 	assert(count <= regs->num_counters);
498 
499 	if (regs->layout & SI_PC_FAKE)
500 		return;
501 
502 	if (layout_multi == SI_PC_MULTI_BLOCK) {
503 		assert(!(regs->layout & SI_PC_REG_REVERSE));
504 
505 		dw = count + regs->num_prelude;
506 		if (count >= regs->num_multi)
507 			dw += regs->num_multi;
508 		radeon_set_uconfig_reg_seq(cs, regs->select0, dw);
509 		for (idx = 0; idx < regs->num_prelude; ++idx)
510 			radeon_emit(cs, 0);
511 		for (idx = 0; idx < MIN2(count, regs->num_multi); ++idx)
512 			radeon_emit(cs, selectors[idx] | regs->select_or);
513 
514 		if (count < regs->num_multi) {
515 			unsigned select1 =
516 				regs->select0 + 4 * regs->num_multi;
517 			radeon_set_uconfig_reg_seq(cs, select1, count);
518 		}
519 
520 		for (idx = 0; idx < MIN2(count, regs->num_multi); ++idx)
521 			radeon_emit(cs, 0);
522 
523 		if (count > regs->num_multi) {
524 			for (idx = regs->num_multi; idx < count; ++idx)
525 				radeon_emit(cs, selectors[idx] | regs->select_or);
526 		}
527 	} else if (layout_multi == SI_PC_MULTI_TAIL) {
528 		unsigned select1, select1_count;
529 
530 		assert(!(regs->layout & SI_PC_REG_REVERSE));
531 
532 		radeon_set_uconfig_reg_seq(cs, regs->select0, count + regs->num_prelude);
533 		for (idx = 0; idx < regs->num_prelude; ++idx)
534 			radeon_emit(cs, 0);
535 		for (idx = 0; idx < count; ++idx)
536 			radeon_emit(cs, selectors[idx] | regs->select_or);
537 
538 		select1 = regs->select0 + 4 * regs->num_counters;
539 		select1_count = MIN2(count, regs->num_multi);
540 		radeon_set_uconfig_reg_seq(cs, select1, select1_count);
541 		for (idx = 0; idx < select1_count; ++idx)
542 			radeon_emit(cs, 0);
543 	} else if (layout_multi == SI_PC_MULTI_CUSTOM) {
544 		unsigned *reg = regs->select;
545 		for (idx = 0; idx < count; ++idx) {
546 			radeon_set_uconfig_reg(cs, *reg++, selectors[idx] | regs->select_or);
547 			if (idx < regs->num_multi)
548 				radeon_set_uconfig_reg(cs, *reg++, 0);
549 		}
550 	} else {
551 		assert(layout_multi == SI_PC_MULTI_ALTERNATE);
552 
553 		unsigned reg_base = regs->select0;
554 		unsigned reg_count = count + MIN2(count, regs->num_multi);
555 		reg_count += regs->num_prelude;
556 
557 		if (!(regs->layout & SI_PC_REG_REVERSE)) {
558 			radeon_set_uconfig_reg_seq(cs, reg_base, reg_count);
559 
560 			for (idx = 0; idx < regs->num_prelude; ++idx)
561 				radeon_emit(cs, 0);
562 			for (idx = 0; idx < count; ++idx) {
563 				radeon_emit(cs, selectors[idx] | regs->select_or);
564 				if (idx < regs->num_multi)
565 					radeon_emit(cs, 0);
566 			}
567 		} else {
568 			reg_base -= (reg_count - 1) * 4;
569 			radeon_set_uconfig_reg_seq(cs, reg_base, reg_count);
570 
571 			for (idx = count; idx > 0; --idx) {
572 				if (idx <= regs->num_multi)
573 					radeon_emit(cs, 0);
574 				radeon_emit(cs, selectors[idx - 1] | regs->select_or);
575 			}
576 			for (idx = 0; idx < regs->num_prelude; ++idx)
577 				radeon_emit(cs, 0);
578 		}
579 	}
580 }
581 
si_pc_emit_start(struct r600_common_context * ctx,struct r600_resource * buffer,uint64_t va)582 static void si_pc_emit_start(struct r600_common_context *ctx,
583 			     struct r600_resource *buffer, uint64_t va)
584 {
585 	struct radeon_winsys_cs *cs = ctx->gfx.cs;
586 
587 	radeon_add_to_buffer_list(ctx, &ctx->gfx, buffer,
588 				  RADEON_USAGE_WRITE, RADEON_PRIO_QUERY);
589 
590 	radeon_emit(cs, PKT3(PKT3_COPY_DATA, 4, 0));
591 	radeon_emit(cs, COPY_DATA_SRC_SEL(COPY_DATA_IMM) |
592 			COPY_DATA_DST_SEL(COPY_DATA_MEM));
593 	radeon_emit(cs, 1); /* immediate */
594 	radeon_emit(cs, 0); /* unused */
595 	radeon_emit(cs, va);
596 	radeon_emit(cs, va >> 32);
597 
598 	radeon_set_uconfig_reg(cs, R_036020_CP_PERFMON_CNTL,
599 			       S_036020_PERFMON_STATE(V_036020_DISABLE_AND_RESET));
600 	radeon_emit(cs, PKT3(PKT3_EVENT_WRITE, 0, 0));
601 	radeon_emit(cs, EVENT_TYPE(V_028A90_PERFCOUNTER_START) | EVENT_INDEX(0));
602 	radeon_set_uconfig_reg(cs, R_036020_CP_PERFMON_CNTL,
603 			       S_036020_PERFMON_STATE(V_036020_START_COUNTING));
604 }
605 
606 /* Note: The buffer was already added in si_pc_emit_start, so we don't have to
607  * do it again in here. */
si_pc_emit_stop(struct r600_common_context * ctx,struct r600_resource * buffer,uint64_t va)608 static void si_pc_emit_stop(struct r600_common_context *ctx,
609 			    struct r600_resource *buffer, uint64_t va)
610 {
611 	struct radeon_winsys_cs *cs = ctx->gfx.cs;
612 
613 	si_gfx_write_event_eop(ctx, V_028A90_BOTTOM_OF_PIPE_TS, 0,
614 				 EOP_DATA_SEL_VALUE_32BIT,
615 				 buffer, va, 0, SI_NOT_QUERY);
616 	si_gfx_wait_fence(ctx, va, 0, 0xffffffff);
617 
618 	radeon_emit(cs, PKT3(PKT3_EVENT_WRITE, 0, 0));
619 	radeon_emit(cs, EVENT_TYPE(V_028A90_PERFCOUNTER_SAMPLE) | EVENT_INDEX(0));
620 	radeon_emit(cs, PKT3(PKT3_EVENT_WRITE, 0, 0));
621 	radeon_emit(cs, EVENT_TYPE(V_028A90_PERFCOUNTER_STOP) | EVENT_INDEX(0));
622 	radeon_set_uconfig_reg(cs, R_036020_CP_PERFMON_CNTL,
623 			       S_036020_PERFMON_STATE(V_036020_STOP_COUNTING) |
624 			       S_036020_PERFMON_SAMPLE_ENABLE(1));
625 }
626 
si_pc_emit_read(struct r600_common_context * ctx,struct r600_perfcounter_block * group,unsigned count,unsigned * selectors,struct r600_resource * buffer,uint64_t va)627 static void si_pc_emit_read(struct r600_common_context *ctx,
628 			    struct r600_perfcounter_block *group,
629 			    unsigned count, unsigned *selectors,
630 			    struct r600_resource *buffer, uint64_t va)
631 {
632 	struct si_pc_block *sigroup = (struct si_pc_block *)group->data;
633 	struct si_pc_block_base *regs = sigroup->b;
634 	struct radeon_winsys_cs *cs = ctx->gfx.cs;
635 	unsigned idx;
636 	unsigned reg = regs->counter0_lo;
637 	unsigned reg_delta = 8;
638 
639 	if (!(regs->layout & SI_PC_FAKE)) {
640 		if (regs->layout & SI_PC_REG_REVERSE)
641 			reg_delta = -reg_delta;
642 
643 		for (idx = 0; idx < count; ++idx) {
644 			if (regs->counters)
645 				reg = regs->counters[idx];
646 
647 			radeon_emit(cs, PKT3(PKT3_COPY_DATA, 4, 0));
648 			radeon_emit(cs, COPY_DATA_SRC_SEL(COPY_DATA_PERF) |
649 					COPY_DATA_DST_SEL(COPY_DATA_MEM) |
650 					COPY_DATA_COUNT_SEL); /* 64 bits */
651 			radeon_emit(cs, reg >> 2);
652 			radeon_emit(cs, 0); /* unused */
653 			radeon_emit(cs, va);
654 			radeon_emit(cs, va >> 32);
655 			va += sizeof(uint64_t);
656 			reg += reg_delta;
657 		}
658 	} else {
659 		for (idx = 0; idx < count; ++idx) {
660 			radeon_emit(cs, PKT3(PKT3_COPY_DATA, 4, 0));
661 			radeon_emit(cs, COPY_DATA_SRC_SEL(COPY_DATA_IMM) |
662 					COPY_DATA_DST_SEL(COPY_DATA_MEM) |
663 					COPY_DATA_COUNT_SEL);
664 			radeon_emit(cs, 0); /* immediate */
665 			radeon_emit(cs, 0);
666 			radeon_emit(cs, va);
667 			radeon_emit(cs, va >> 32);
668 			va += sizeof(uint64_t);
669 		}
670 	}
671 }
672 
si_pc_cleanup(struct si_screen * sscreen)673 static void si_pc_cleanup(struct si_screen *sscreen)
674 {
675 	si_perfcounters_do_destroy(sscreen->perfcounters);
676 	sscreen->perfcounters = NULL;
677 }
678 
si_init_perfcounters(struct si_screen * screen)679 void si_init_perfcounters(struct si_screen *screen)
680 {
681 	struct r600_perfcounters *pc;
682 	struct si_pc_block *blocks;
683 	unsigned num_blocks;
684 	unsigned i;
685 
686 	switch (screen->info.chip_class) {
687 	case CIK:
688 		blocks = groups_CIK;
689 		num_blocks = ARRAY_SIZE(groups_CIK);
690 		break;
691 	case VI:
692 		blocks = groups_VI;
693 		num_blocks = ARRAY_SIZE(groups_VI);
694 		break;
695 	case GFX9:
696 		blocks = groups_gfx9;
697 		num_blocks = ARRAY_SIZE(groups_gfx9);
698 		break;
699 	case SI:
700 	default:
701 		return; /* not implemented */
702 	}
703 
704 	if (screen->info.max_sh_per_se != 1) {
705 		/* This should not happen on non-SI chips. */
706 		fprintf(stderr, "si_init_perfcounters: max_sh_per_se = %d not "
707 			"supported (inaccurate performance counters)\n",
708 			screen->info.max_sh_per_se);
709 	}
710 
711 	pc = CALLOC_STRUCT(r600_perfcounters);
712 	if (!pc)
713 		return;
714 
715 	pc->num_start_cs_dwords = 14;
716 	pc->num_stop_cs_dwords = 14 + si_gfx_write_fence_dwords(screen);
717 	pc->num_instance_cs_dwords = 3;
718 	pc->num_shaders_cs_dwords = 4;
719 
720 	pc->num_shader_types = ARRAY_SIZE(si_pc_shader_type_bits);
721 	pc->shader_type_suffixes = si_pc_shader_type_suffixes;
722 	pc->shader_type_bits = si_pc_shader_type_bits;
723 
724 	pc->get_size = si_pc_get_size;
725 	pc->emit_instance = si_pc_emit_instance;
726 	pc->emit_shaders = si_pc_emit_shaders;
727 	pc->emit_select = si_pc_emit_select;
728 	pc->emit_start = si_pc_emit_start;
729 	pc->emit_stop = si_pc_emit_stop;
730 	pc->emit_read = si_pc_emit_read;
731 	pc->cleanup = si_pc_cleanup;
732 
733 	if (!si_perfcounters_init(pc, num_blocks))
734 		goto error;
735 
736 	for (i = 0; i < num_blocks; ++i) {
737 		struct si_pc_block *block = &blocks[i];
738 		unsigned instances = block->instances;
739 
740 		if (!strcmp(block->b->name, "IA")) {
741 			if (screen->info.max_se > 2)
742 				instances = 2;
743 		}
744 
745 		si_perfcounters_add_block(screen, pc,
746 					    block->b->name,
747 					    block->b->flags,
748 					    block->b->num_counters,
749 					    block->selectors,
750 					    instances,
751 					    block);
752 	}
753 
754 	screen->perfcounters = pc;
755 	return;
756 
757 error:
758 	si_perfcounters_do_destroy(pc);
759 }
760