• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 /*
2  * Copyright (C) 2009 Nicolai Haehnle.
3  * Copyright 2010 Tom Stellard <tstellar@gmail.com>
4  *
5  * All Rights Reserved.
6  *
7  * Permission is hereby granted, free of charge, to any person obtaining
8  * a copy of this software and associated documentation files (the
9  * "Software"), to deal in the Software without restriction, including
10  * without limitation the rights to use, copy, modify, merge, publish,
11  * distribute, sublicense, and/or sell copies of the Software, and to
12  * permit persons to whom the Software is furnished to do so, subject to
13  * the following conditions:
14  *
15  * The above copyright notice and this permission notice (including the
16  * next paragraph) shall be included in all copies or substantial
17  * portions of the Software.
18  *
19  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
20  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
21  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
22  * IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
23  * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
24  * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
25  * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
26  *
27  */
28 
29 #include "radeon_dataflow.h"
30 
31 #include "radeon_compiler.h"
32 #include "radeon_compiler_util.h"
33 #include "radeon_program.h"
34 
35 struct read_write_mask_data {
36 	void * UserData;
37 	rc_read_write_mask_fn Cb;
38 };
39 
reads_normal_callback(void * userdata,struct rc_instruction * fullinst,struct rc_src_register * src)40 static void reads_normal_callback(
41 	void * userdata,
42 	struct rc_instruction * fullinst,
43 	struct rc_src_register * src)
44 {
45 	struct read_write_mask_data * cb_data = userdata;
46 	unsigned int refmask = 0;
47 	unsigned int chan;
48 	for(chan = 0; chan < 4; chan++) {
49 		refmask |= 1 << GET_SWZ(src->Swizzle, chan);
50 	}
51 	refmask &= RC_MASK_XYZW;
52 
53 	if (refmask) {
54 		cb_data->Cb(cb_data->UserData, fullinst, src->File,
55 							src->Index, refmask);
56 	}
57 
58 	if (refmask && src->RelAddr) {
59 		cb_data->Cb(cb_data->UserData, fullinst, RC_FILE_ADDRESS, 0,
60 								RC_MASK_X);
61 	}
62 }
63 
pair_get_src_refmasks(unsigned int * refmasks,struct rc_pair_instruction * inst,unsigned int swz,unsigned int src)64 static void pair_get_src_refmasks(unsigned int * refmasks,
65 					struct rc_pair_instruction * inst,
66 					unsigned int swz, unsigned int src)
67 {
68 	if (swz >= 4)
69 		return;
70 
71 	if (swz == RC_SWIZZLE_X || swz == RC_SWIZZLE_Y || swz == RC_SWIZZLE_Z) {
72 		if(src == RC_PAIR_PRESUB_SRC) {
73 			unsigned int i;
74 			int srcp_regs =
75 				rc_presubtract_src_reg_count(
76 				inst->RGB.Src[src].Index);
77 			for(i = 0; i < srcp_regs; i++) {
78 				refmasks[i] |= 1 << swz;
79 			}
80 		}
81 		else {
82 			refmasks[src] |= 1 << swz;
83 		}
84 	}
85 
86 	if (swz == RC_SWIZZLE_W) {
87 		if (src == RC_PAIR_PRESUB_SRC) {
88 			unsigned int i;
89 			int srcp_regs = rc_presubtract_src_reg_count(
90 					inst->Alpha.Src[src].Index);
91 			for(i = 0; i < srcp_regs; i++) {
92 				refmasks[i] |= 1 << swz;
93 			}
94 		}
95 		else {
96 			refmasks[src] |= 1 << swz;
97 		}
98 	}
99 }
100 
reads_pair(struct rc_instruction * fullinst,rc_read_write_mask_fn cb,void * userdata)101 static void reads_pair(struct rc_instruction * fullinst, rc_read_write_mask_fn cb, void * userdata)
102 {
103 	struct rc_pair_instruction * inst = &fullinst->U.P;
104 	unsigned int refmasks[3] = { 0, 0, 0 };
105 
106 	unsigned int arg;
107 
108 	for(arg = 0; arg < 3; ++arg) {
109 		unsigned int chan;
110 		for(chan = 0; chan < 3; ++chan) {
111 			unsigned int swz_rgb =
112 				GET_SWZ(inst->RGB.Arg[arg].Swizzle, chan);
113 			unsigned int swz_alpha =
114 				GET_SWZ(inst->Alpha.Arg[arg].Swizzle, chan);
115 			pair_get_src_refmasks(refmasks, inst, swz_rgb,
116 						inst->RGB.Arg[arg].Source);
117 			pair_get_src_refmasks(refmasks, inst, swz_alpha,
118 						inst->Alpha.Arg[arg].Source);
119 		}
120 	}
121 
122 	for(unsigned int src = 0; src < 3; ++src) {
123 		if (inst->RGB.Src[src].Used && (refmasks[src] & RC_MASK_XYZ))
124 			cb(userdata, fullinst, inst->RGB.Src[src].File, inst->RGB.Src[src].Index,
125 			   refmasks[src] & RC_MASK_XYZ);
126 
127 		if (inst->Alpha.Src[src].Used && (refmasks[src] & RC_MASK_W))
128 			cb(userdata, fullinst, inst->Alpha.Src[src].File, inst->Alpha.Src[src].Index, RC_MASK_W);
129 	}
130 }
131 
pair_sub_for_all_args(struct rc_instruction * fullinst,struct rc_pair_sub_instruction * sub,rc_pair_read_arg_fn cb,void * userdata)132 static void pair_sub_for_all_args(
133 	struct rc_instruction * fullinst,
134 	struct rc_pair_sub_instruction * sub,
135 	rc_pair_read_arg_fn cb,
136 	void * userdata)
137 {
138 	int i;
139 	const struct rc_opcode_info * info = rc_get_opcode_info(sub->Opcode);
140 
141 	for(i = 0; i < info->NumSrcRegs; i++) {
142 		unsigned int src_type;
143 
144 		src_type = rc_source_type_swz(sub->Arg[i].Swizzle);
145 
146 		if (src_type == RC_SOURCE_NONE)
147 			continue;
148 
149 		if (sub->Arg[i].Source == RC_PAIR_PRESUB_SRC) {
150 			unsigned int presub_type;
151 			unsigned int presub_src_count;
152 			struct rc_pair_instruction_source * src_array;
153 			unsigned int j;
154 
155 			if (src_type & RC_SOURCE_RGB) {
156 				presub_type = fullinst->
157 					U.P.RGB.Src[RC_PAIR_PRESUB_SRC].Index;
158 				src_array = fullinst->U.P.RGB.Src;
159 			} else {
160 				presub_type = fullinst->
161 					U.P.Alpha.Src[RC_PAIR_PRESUB_SRC].Index;
162 				src_array = fullinst->U.P.Alpha.Src;
163 			}
164 			presub_src_count
165 				= rc_presubtract_src_reg_count(presub_type);
166 			for(j = 0; j < presub_src_count; j++) {
167 				cb(userdata, fullinst, &sub->Arg[i],
168 								&src_array[j]);
169 			}
170 		} else {
171 			struct rc_pair_instruction_source * src =
172 				rc_pair_get_src(&fullinst->U.P, &sub->Arg[i]);
173 			if (src) {
174 				cb(userdata, fullinst, &sub->Arg[i], src);
175 			}
176 		}
177 	}
178 }
179 
180 /* This function calls the callback function (cb) for each source used by
181  * the instruction.
182  * */
rc_for_all_reads_src(struct rc_instruction * inst,rc_read_src_fn cb,void * userdata)183 void rc_for_all_reads_src(
184 	struct rc_instruction * inst,
185 	rc_read_src_fn cb,
186 	void * userdata)
187 {
188 	const struct rc_opcode_info * opcode =
189 					rc_get_opcode_info(inst->U.I.Opcode);
190 
191 	/* This function only works with normal instructions. */
192 	if (inst->Type != RC_INSTRUCTION_NORMAL) {
193 		assert(0);
194 		return;
195 	}
196 
197 	for(unsigned int src = 0; src < opcode->NumSrcRegs; ++src) {
198 
199 		if (inst->U.I.SrcReg[src].File == RC_FILE_NONE)
200 			continue;
201 
202 		if (inst->U.I.SrcReg[src].File == RC_FILE_PRESUB) {
203 			unsigned int i;
204 			unsigned int srcp_regs = rc_presubtract_src_reg_count(
205 						inst->U.I.PreSub.Opcode);
206 			for( i = 0; i < srcp_regs; i++) {
207 				cb(userdata, inst, &inst->U.I.PreSub.SrcReg[i]);
208 			}
209 		} else {
210 			cb(userdata, inst, &inst->U.I.SrcReg[src]);
211 		}
212 	}
213 }
214 
215 /**
216  * This function calls the callback function (cb) for each arg of the RGB and
217  * alpha components.
218  */
rc_pair_for_all_reads_arg(struct rc_instruction * inst,rc_pair_read_arg_fn cb,void * userdata)219 void rc_pair_for_all_reads_arg(
220 	struct rc_instruction * inst,
221 	rc_pair_read_arg_fn cb,
222 	void * userdata)
223 {
224 	/* This function only works with pair instructions. */
225 	if (inst->Type != RC_INSTRUCTION_PAIR) {
226 		assert(0);
227 		return;
228 	}
229 
230 	pair_sub_for_all_args(inst, &inst->U.P.RGB, cb, userdata);
231 	pair_sub_for_all_args(inst, &inst->U.P.Alpha, cb, userdata);
232 }
233 
234 /**
235  * Calls a callback function for all register reads.
236  *
237  * This is conservative, i.e. if the same register is referenced multiple times,
238  * the callback may also be called multiple times.
239  * Also, the writemask of the instruction is not taken into account.
240  */
rc_for_all_reads_mask(struct rc_instruction * inst,rc_read_write_mask_fn cb,void * userdata)241 void rc_for_all_reads_mask(struct rc_instruction * inst, rc_read_write_mask_fn cb, void * userdata)
242 {
243 	if (inst->Type == RC_INSTRUCTION_NORMAL) {
244 		struct read_write_mask_data cb_data;
245 		cb_data.UserData = userdata;
246 		cb_data.Cb = cb;
247 
248 		rc_for_all_reads_src(inst, reads_normal_callback, &cb_data);
249 	} else {
250 		reads_pair(inst, cb, userdata);
251 	}
252 }
253 
254 
255 
writes_normal(struct rc_instruction * fullinst,rc_read_write_mask_fn cb,void * userdata)256 static void writes_normal(struct rc_instruction * fullinst, rc_read_write_mask_fn cb, void * userdata)
257 {
258 	struct rc_sub_instruction * inst = &fullinst->U.I;
259 	const struct rc_opcode_info * opcode = rc_get_opcode_info(inst->Opcode);
260 
261 	if (opcode->HasDstReg && inst->DstReg.WriteMask)
262 		cb(userdata, fullinst, inst->DstReg.File, inst->DstReg.Index, inst->DstReg.WriteMask);
263 
264 	if (inst->WriteALUResult)
265 		cb(userdata, fullinst, RC_FILE_SPECIAL, RC_SPECIAL_ALU_RESULT, RC_MASK_X);
266 }
267 
writes_pair(struct rc_instruction * fullinst,rc_read_write_mask_fn cb,void * userdata)268 static void writes_pair(struct rc_instruction * fullinst, rc_read_write_mask_fn cb, void * userdata)
269 {
270 	struct rc_pair_instruction * inst = &fullinst->U.P;
271 
272 	if (inst->RGB.WriteMask)
273 		cb(userdata, fullinst, RC_FILE_TEMPORARY, inst->RGB.DestIndex, inst->RGB.WriteMask);
274 
275 	if (inst->Alpha.WriteMask)
276 		cb(userdata, fullinst, RC_FILE_TEMPORARY, inst->Alpha.DestIndex, RC_MASK_W);
277 
278 	if (inst->WriteALUResult)
279 		cb(userdata, fullinst, RC_FILE_SPECIAL, RC_SPECIAL_ALU_RESULT, RC_MASK_X);
280 }
281 
282 /**
283  * Calls a callback function for all register writes in the instruction,
284  * reporting writemasks to the callback function.
285  *
286  * \warning Does not report output registers for paired instructions!
287  */
rc_for_all_writes_mask(struct rc_instruction * inst,rc_read_write_mask_fn cb,void * userdata)288 void rc_for_all_writes_mask(struct rc_instruction * inst, rc_read_write_mask_fn cb, void * userdata)
289 {
290 	if (inst->Type == RC_INSTRUCTION_NORMAL) {
291 		writes_normal(inst, cb, userdata);
292 	} else {
293 		writes_pair(inst, cb, userdata);
294 	}
295 }
296 
297 
298 struct mask_to_chan_data {
299 	void * UserData;
300 	rc_read_write_chan_fn Fn;
301 };
302 
mask_to_chan_cb(void * data,struct rc_instruction * inst,rc_register_file file,unsigned int index,unsigned int mask)303 static void mask_to_chan_cb(void * data, struct rc_instruction * inst,
304 		rc_register_file file, unsigned int index, unsigned int mask)
305 {
306 	struct mask_to_chan_data * d = data;
307 	for(unsigned int chan = 0; chan < 4; ++chan) {
308 		if (GET_BIT(mask, chan))
309 			d->Fn(d->UserData, inst, file, index, chan);
310 	}
311 }
312 
313 /**
314  * Calls a callback function for all sourced register channels.
315  *
316  * This is conservative, i.e. channels may be called multiple times,
317  * and the writemask of the instruction is not taken into account.
318  */
rc_for_all_reads_chan(struct rc_instruction * inst,rc_read_write_chan_fn cb,void * userdata)319 void rc_for_all_reads_chan(struct rc_instruction * inst, rc_read_write_chan_fn cb, void * userdata)
320 {
321 	struct mask_to_chan_data d;
322 	d.UserData = userdata;
323 	d.Fn = cb;
324 	rc_for_all_reads_mask(inst, &mask_to_chan_cb, &d);
325 }
326 
327 /**
328  * Calls a callback function for all written register channels.
329  *
330  * \warning Does not report output registers for paired instructions!
331  */
rc_for_all_writes_chan(struct rc_instruction * inst,rc_read_write_chan_fn cb,void * userdata)332 void rc_for_all_writes_chan(struct rc_instruction * inst, rc_read_write_chan_fn cb, void * userdata)
333 {
334 	struct mask_to_chan_data d;
335 	d.UserData = userdata;
336 	d.Fn = cb;
337 	rc_for_all_writes_mask(inst, &mask_to_chan_cb, &d);
338 }
339 
remap_normal_instruction(struct rc_instruction * fullinst,rc_remap_register_fn cb,void * userdata)340 static void remap_normal_instruction(struct rc_instruction * fullinst,
341 		rc_remap_register_fn cb, void * userdata)
342 {
343 	struct rc_sub_instruction * inst = &fullinst->U.I;
344 	const struct rc_opcode_info * opcode = rc_get_opcode_info(inst->Opcode);
345 	unsigned int remapped_presub = 0;
346 
347 	if (opcode->HasDstReg) {
348 		rc_register_file file = inst->DstReg.File;
349 		unsigned int index = inst->DstReg.Index;
350 
351 		cb(userdata, fullinst, &file, &index);
352 
353 		inst->DstReg.File = file;
354 		inst->DstReg.Index = index;
355 	}
356 
357 	for(unsigned int src = 0; src < opcode->NumSrcRegs; ++src) {
358 		rc_register_file file = inst->SrcReg[src].File;
359 		unsigned int index = inst->SrcReg[src].Index;
360 
361 		if (file == RC_FILE_PRESUB) {
362 			unsigned int i;
363 			unsigned int srcp_srcs = rc_presubtract_src_reg_count(
364 						inst->PreSub.Opcode);
365 			/* Make sure we only remap presubtract sources once in
366 			 * case more than one source register reads the
367 			 * presubtract result. */
368 			if (remapped_presub)
369 				continue;
370 
371 			for(i = 0; i < srcp_srcs; i++) {
372 				file = inst->PreSub.SrcReg[i].File;
373 				index = inst->PreSub.SrcReg[i].Index;
374 				cb(userdata, fullinst, &file, &index);
375 				inst->PreSub.SrcReg[i].File = file;
376 				inst->PreSub.SrcReg[i].Index = index;
377 			}
378 			remapped_presub = 1;
379 		}
380 		else {
381 			cb(userdata, fullinst, &file, &index);
382 
383 			inst->SrcReg[src].File = file;
384 			inst->SrcReg[src].Index = index;
385 		}
386 	}
387 }
388 
remap_pair_instruction(struct rc_instruction * fullinst,rc_remap_register_fn cb,void * userdata)389 static void remap_pair_instruction(struct rc_instruction * fullinst,
390 		rc_remap_register_fn cb, void * userdata)
391 {
392 	struct rc_pair_instruction * inst = &fullinst->U.P;
393 
394 	if (inst->RGB.WriteMask) {
395 		rc_register_file file = RC_FILE_TEMPORARY;
396 		unsigned int index = inst->RGB.DestIndex;
397 
398 		cb(userdata, fullinst, &file, &index);
399 
400 		inst->RGB.DestIndex = index;
401 	}
402 
403 	if (inst->Alpha.WriteMask) {
404 		rc_register_file file = RC_FILE_TEMPORARY;
405 		unsigned int index = inst->Alpha.DestIndex;
406 
407 		cb(userdata, fullinst, &file, &index);
408 
409 		inst->Alpha.DestIndex = index;
410 	}
411 
412 	for(unsigned int src = 0; src < 3; ++src) {
413 		if (inst->RGB.Src[src].Used) {
414 			rc_register_file file = inst->RGB.Src[src].File;
415 			unsigned int index = inst->RGB.Src[src].Index;
416 
417 			cb(userdata, fullinst, &file, &index);
418 
419 			inst->RGB.Src[src].File = file;
420 			inst->RGB.Src[src].Index = index;
421 		}
422 
423 		if (inst->Alpha.Src[src].Used) {
424 			rc_register_file file = inst->Alpha.Src[src].File;
425 			unsigned int index = inst->Alpha.Src[src].Index;
426 
427 			cb(userdata, fullinst, &file, &index);
428 
429 			inst->Alpha.Src[src].File = file;
430 			inst->Alpha.Src[src].Index = index;
431 		}
432 	}
433 }
434 
435 
436 /**
437  * Remap all register accesses according to the given function.
438  * That is, call the function \p cb for each referenced register (both read and written)
439  * and update the given instruction \p inst accordingly
440  * if it modifies its \ref pfile and \ref pindex contents.
441  */
rc_remap_registers(struct rc_instruction * inst,rc_remap_register_fn cb,void * userdata)442 void rc_remap_registers(struct rc_instruction * inst, rc_remap_register_fn cb, void * userdata)
443 {
444 	if (inst->Type == RC_INSTRUCTION_NORMAL)
445 		remap_normal_instruction(inst, cb, userdata);
446 	else
447 		remap_pair_instruction(inst, cb, userdata);
448 }
449 
450 struct branch_write_mask {
451 	unsigned int IfWriteMask:4;
452 	unsigned int ElseWriteMask:4;
453 	unsigned int HasElse:1;
454 };
455 
456 union get_readers_read_cb {
457 	rc_read_src_fn I;
458 	rc_pair_read_arg_fn P;
459 };
460 
461 struct get_readers_callback_data {
462 	struct radeon_compiler * C;
463 	struct rc_reader_data * ReaderData;
464 	rc_read_src_fn ReadNormalCB;
465 	rc_pair_read_arg_fn ReadPairCB;
466 	rc_read_write_mask_fn WriteCB;
467 	rc_register_file DstFile;
468 	unsigned int DstIndex;
469 	unsigned int DstMask;
470 	unsigned int AliveWriteMask;
471 	/*  For convenience, this is indexed starting at 1 */
472 	struct branch_write_mask BranchMasks[R500_PFS_MAX_BRANCH_DEPTH_FULL + 1];
473 };
474 
add_reader(struct memory_pool * pool,struct rc_reader_data * data,struct rc_instruction * inst,unsigned int mask)475 static struct rc_reader * add_reader(
476 	struct memory_pool * pool,
477 	struct rc_reader_data * data,
478 	struct rc_instruction * inst,
479 	unsigned int mask)
480 {
481 	struct rc_reader * new;
482 	memory_pool_array_reserve(pool, struct rc_reader, data->Readers,
483 				data->ReaderCount, data->ReadersReserved, 1);
484 	new = &data->Readers[data->ReaderCount++];
485 	new->Inst = inst;
486 	new->WriteMask = mask;
487 	return new;
488 }
489 
add_reader_normal(struct memory_pool * pool,struct rc_reader_data * data,struct rc_instruction * inst,unsigned int mask,struct rc_src_register * src)490 static void add_reader_normal(
491 	struct memory_pool * pool,
492 	struct rc_reader_data * data,
493 	struct rc_instruction * inst,
494 	unsigned int mask,
495 	struct rc_src_register * src)
496 {
497 	struct rc_reader * new = add_reader(pool, data, inst, mask);
498 	new->U.I.Src = src;
499 }
500 
501 
add_reader_pair(struct memory_pool * pool,struct rc_reader_data * data,struct rc_instruction * inst,unsigned int mask,struct rc_pair_instruction_arg * arg,struct rc_pair_instruction_source * src)502 static void add_reader_pair(
503 	struct memory_pool * pool,
504 	struct rc_reader_data * data,
505 	struct rc_instruction * inst,
506 	unsigned int mask,
507 	struct rc_pair_instruction_arg * arg,
508 	struct rc_pair_instruction_source * src)
509 {
510 	struct rc_reader * new = add_reader(pool, data, inst, mask);
511 	new->U.P.Src = src;
512 	new->U.P.Arg = arg;
513 }
514 
get_readers_read_callback(struct get_readers_callback_data * cb_data,unsigned int has_rel_addr,rc_register_file file,unsigned int index,unsigned int swizzle)515 static unsigned int get_readers_read_callback(
516 	struct get_readers_callback_data * cb_data,
517 	unsigned int has_rel_addr,
518 	rc_register_file file,
519 	unsigned int index,
520 	unsigned int swizzle)
521 {
522 	unsigned int shared_mask, read_mask;
523 
524 	if (has_rel_addr) {
525 		cb_data->ReaderData->Abort = 1;
526 		return RC_MASK_NONE;
527 	}
528 
529 	shared_mask = rc_src_reads_dst_mask(file, index, swizzle,
530 		cb_data->DstFile, cb_data->DstIndex, cb_data->AliveWriteMask);
531 
532 	if (shared_mask == RC_MASK_NONE)
533 		return shared_mask;
534 
535 	/* If we make it this far, it means that this source reads from the
536 	 * same register written to by d->ReaderData->Writer. */
537 
538 	read_mask = rc_swizzle_to_writemask(swizzle);
539 	if (cb_data->ReaderData->AbortOnRead & read_mask) {
540 		cb_data->ReaderData->Abort = 1;
541 		return shared_mask;
542 	}
543 
544 	if (cb_data->ReaderData->LoopDepth > 0) {
545 		cb_data->ReaderData->AbortOnWrite |=
546 				(read_mask & cb_data->AliveWriteMask);
547 	}
548 
549 	/* XXX The behavior in this case should be configurable. */
550 	if ((read_mask & cb_data->AliveWriteMask) != read_mask) {
551 		cb_data->ReaderData->Abort = 1;
552 		return shared_mask;
553 	}
554 
555 	return shared_mask;
556 }
557 
get_readers_pair_read_callback(void * userdata,struct rc_instruction * inst,struct rc_pair_instruction_arg * arg,struct rc_pair_instruction_source * src)558 static void get_readers_pair_read_callback(
559 	void * userdata,
560 	struct rc_instruction * inst,
561 	struct rc_pair_instruction_arg * arg,
562 	struct rc_pair_instruction_source * src)
563 {
564 	unsigned int shared_mask;
565 	struct get_readers_callback_data * d = userdata;
566 
567 	shared_mask = get_readers_read_callback(d,
568 				0 /*Pair Instructions don't use RelAddr*/,
569 				src->File, src->Index, arg->Swizzle);
570 
571 	if (shared_mask == RC_MASK_NONE)
572 		return;
573 
574 	if (d->ReadPairCB)
575 		d->ReadPairCB(d->ReaderData, inst, arg, src);
576 
577 	if (d->ReaderData->ExitOnAbort && d->ReaderData->Abort)
578 		return;
579 
580 	add_reader_pair(&d->C->Pool, d->ReaderData, inst, shared_mask, arg, src);
581 }
582 
583 /**
584  * This function is used by rc_get_readers_normal() to determine whether inst
585  * is a reader of userdata->ReaderData->Writer
586  */
get_readers_normal_read_callback(void * userdata,struct rc_instruction * inst,struct rc_src_register * src)587 static void get_readers_normal_read_callback(
588 	void * userdata,
589 	struct rc_instruction * inst,
590 	struct rc_src_register * src)
591 {
592 	struct get_readers_callback_data * d = userdata;
593 	unsigned int shared_mask;
594 
595 	shared_mask = get_readers_read_callback(d,
596 			src->RelAddr, src->File, src->Index, src->Swizzle);
597 
598 	if (shared_mask == RC_MASK_NONE)
599 		return;
600 	/* The callback function could potentially clear d->ReaderData->Abort,
601 	 * so we need to call it before we return. */
602 	if (d->ReadNormalCB)
603 		d->ReadNormalCB(d->ReaderData, inst, src);
604 
605 	if (d->ReaderData->ExitOnAbort && d->ReaderData->Abort)
606 		return;
607 
608 	add_reader_normal(&d->C->Pool, d->ReaderData, inst, shared_mask, src);
609 }
610 
611 /**
612  * This function is used by rc_get_readers_normal() to determine when
613  * userdata->ReaderData->Writer is dead (i. e. All compontents of its
614  * destination register have been overwritten by other instructions).
615  */
get_readers_write_callback(void * userdata,struct rc_instruction * inst,rc_register_file file,unsigned int index,unsigned int mask)616 static void get_readers_write_callback(
617 	void *userdata,
618 	struct rc_instruction * inst,
619 	rc_register_file file,
620 	unsigned int index,
621 	unsigned int mask)
622 {
623 	struct get_readers_callback_data * d = userdata;
624 
625 	if (index == d->DstIndex && file == d->DstFile) {
626 		unsigned int shared_mask = mask & d->DstMask;
627 		d->ReaderData->AbortOnRead &= ~shared_mask;
628 		d->AliveWriteMask &= ~shared_mask;
629 		if (d->ReaderData->AbortOnWrite & shared_mask) {
630 			d->ReaderData->Abort = 1;
631 		}
632 	}
633 
634 	if(d->WriteCB)
635 		d->WriteCB(d->ReaderData, inst, file, index, mask);
636 }
637 
push_branch_mask(struct get_readers_callback_data * d,unsigned int * branch_depth)638 static void push_branch_mask(
639 	struct get_readers_callback_data * d,
640 	unsigned int * branch_depth)
641 {
642 	(*branch_depth)++;
643 	if (*branch_depth > R500_PFS_MAX_BRANCH_DEPTH_FULL) {
644 		d->ReaderData->Abort = 1;
645 		return;
646 	}
647 	d->BranchMasks[*branch_depth].IfWriteMask =
648 					d->AliveWriteMask;
649 }
650 
pop_branch_mask(struct get_readers_callback_data * d,unsigned int * branch_depth)651 static void pop_branch_mask(
652 	struct get_readers_callback_data * d,
653 	unsigned int * branch_depth)
654 {
655 	struct branch_write_mask * masks = &d->BranchMasks[*branch_depth];
656 
657 	if (masks->HasElse) {
658 		/* Abort on read for components that were written in the IF
659 		 * block. */
660 		d->ReaderData->AbortOnRead |=
661 				masks->IfWriteMask & ~masks->ElseWriteMask;
662 		/* Abort on read for components that were written in the ELSE
663 		 * block. */
664 		d->ReaderData->AbortOnRead |=
665 				masks->ElseWriteMask & ~d->AliveWriteMask;
666 
667 		d->AliveWriteMask = masks->IfWriteMask
668 			^ ((masks->IfWriteMask ^ masks->ElseWriteMask)
669 			& (masks->IfWriteMask ^ d->AliveWriteMask));
670 	} else {
671 		d->ReaderData->AbortOnRead |=
672 				masks->IfWriteMask & ~d->AliveWriteMask;
673 		d->AliveWriteMask = masks->IfWriteMask;
674 
675 	}
676 	memset(masks, 0, sizeof(struct branch_write_mask));
677 	(*branch_depth)--;
678 }
679 
get_readers_for_single_write(void * userdata,struct rc_instruction * writer,rc_register_file dst_file,unsigned int dst_index,unsigned int dst_mask)680 static void get_readers_for_single_write(
681 	void * userdata,
682 	struct rc_instruction * writer,
683 	rc_register_file dst_file,
684 	unsigned int dst_index,
685 	unsigned int dst_mask)
686 {
687 	struct rc_instruction * tmp;
688 	unsigned int branch_depth = 0;
689 	struct rc_instruction * endloop = NULL;
690 	unsigned int abort_on_read_at_endloop = 0;
691 	struct get_readers_callback_data * d = userdata;
692 
693 	d->ReaderData->Writer = writer;
694 	d->ReaderData->AbortOnRead = 0;
695 	d->ReaderData->AbortOnWrite = 0;
696 	d->ReaderData->LoopDepth = 0;
697 	d->ReaderData->InElse = 0;
698 	d->DstFile = dst_file;
699 	d->DstIndex = dst_index;
700 	d->DstMask = dst_mask;
701 	d->AliveWriteMask = dst_mask;
702 	memset(d->BranchMasks, 0, sizeof(d->BranchMasks));
703 
704 	if (!dst_mask)
705 		return;
706 
707 	for(tmp = writer->Next; tmp != &d->C->Program.Instructions;
708 							tmp = tmp->Next){
709 		rc_opcode opcode = rc_get_flow_control_inst(tmp);
710 		switch(opcode) {
711 		case RC_OPCODE_BGNLOOP:
712 			d->ReaderData->LoopDepth++;
713 			push_branch_mask(d, &branch_depth);
714 			break;
715 		case RC_OPCODE_ENDLOOP:
716 			if (d->ReaderData->LoopDepth > 0) {
717 				d->ReaderData->LoopDepth--;
718 				if (d->ReaderData->LoopDepth == 0) {
719 					d->ReaderData->AbortOnWrite = 0;
720 				}
721 				pop_branch_mask(d, &branch_depth);
722 			} else {
723 				/* Here we have reached an ENDLOOP without
724 				 * seeing its BGNLOOP.  These means that
725 				 * the writer was written inside of a loop,
726 				 * so it could have readers that are above it
727 				 * (i.e. they have a lower IP).  To find these
728 				 * readers we jump to the BGNLOOP instruction
729 				 * and check each instruction until we get
730 				 * back to the writer.
731 				 */
732 				endloop = tmp;
733 				tmp = rc_match_endloop(tmp);
734 				if (!tmp) {
735 					rc_error(d->C, "Failed to match endloop.\n");
736 					d->ReaderData->Abort = 1;
737 					return;
738 				}
739 				abort_on_read_at_endloop = d->ReaderData->AbortOnRead;
740 				d->ReaderData->AbortOnRead |= d->AliveWriteMask;
741 				continue;
742 			}
743 			break;
744 		case RC_OPCODE_IF:
745 			push_branch_mask(d, &branch_depth);
746 			break;
747 		case RC_OPCODE_ELSE:
748 			if (branch_depth == 0) {
749 				d->ReaderData->InElse = 1;
750 			} else {
751 				unsigned int temp_mask = d->AliveWriteMask;
752 				d->AliveWriteMask =
753 					d->BranchMasks[branch_depth].IfWriteMask;
754 				d->BranchMasks[branch_depth].ElseWriteMask =
755 								temp_mask;
756 				d->BranchMasks[branch_depth].HasElse = 1;
757 			}
758 			break;
759 		case RC_OPCODE_ENDIF:
760 			if (branch_depth == 0) {
761 				d->ReaderData->AbortOnRead = d->AliveWriteMask;
762 				d->ReaderData->InElse = 0;
763 			}
764 			else {
765 				pop_branch_mask(d, &branch_depth);
766 			}
767 			break;
768 		default:
769 			break;
770 		}
771 
772 		if (d->ReaderData->InElse)
773 			continue;
774 
775 		if (tmp->Type == RC_INSTRUCTION_NORMAL) {
776 			rc_for_all_reads_src(tmp,
777 				get_readers_normal_read_callback, d);
778 		} else {
779 			rc_pair_for_all_reads_arg(tmp,
780 				get_readers_pair_read_callback, d);
781 		}
782 
783 		/* This can happen when we jump from an ENDLOOP to BGNLOOP */
784 		if (tmp == writer) {
785 			tmp = endloop;
786 			endloop = NULL;
787 			d->ReaderData->AbortOnRead = abort_on_read_at_endloop;
788 			continue;
789 		}
790 		rc_for_all_writes_mask(tmp, get_readers_write_callback, d);
791 
792 		if (d->ReaderData->ExitOnAbort && d->ReaderData->Abort)
793 			return;
794 
795 		if (branch_depth == 0 && !d->AliveWriteMask)
796 			return;
797 	}
798 }
799 
init_get_readers_callback_data(struct get_readers_callback_data * d,struct rc_reader_data * reader_data,struct radeon_compiler * c,rc_read_src_fn read_normal_cb,rc_pair_read_arg_fn read_pair_cb,rc_read_write_mask_fn write_cb)800 static void init_get_readers_callback_data(
801 	struct get_readers_callback_data * d,
802 	struct rc_reader_data * reader_data,
803 	struct radeon_compiler * c,
804 	rc_read_src_fn read_normal_cb,
805 	rc_pair_read_arg_fn read_pair_cb,
806 	rc_read_write_mask_fn write_cb)
807 {
808 	reader_data->Abort = 0;
809 	reader_data->ReaderCount = 0;
810 	reader_data->ReadersReserved = 0;
811 	reader_data->Readers = NULL;
812 
813 	d->C = c;
814 	d->ReaderData = reader_data;
815 	d->ReadNormalCB = read_normal_cb;
816 	d->ReadPairCB = read_pair_cb;
817 	d->WriteCB = write_cb;
818 }
819 
820 /**
821  * This function will create a list of readers via the rc_reader_data struct.
822  * This function will abort (set the flag data->Abort) and return if it
823  * encounters an instruction that reads from @param writer and also a different
824  * instruction.  Here are some examples:
825  *
826  * writer = instruction 0;
827  * 0 MOV TEMP[0].xy, TEMP[1].xy
828  * 1 MOV TEMP[0].zw, TEMP[2].xy
829  * 2 MOV TEMP[3], TEMP[0]
830  * The Abort flag will be set on instruction 2, because it reads values written
831  * by instructions 0 and 1.
832  *
833  * writer = instruction 1;
834  * 0 IF TEMP[0].x
835  * 1 MOV TEMP[1], TEMP[2]
836  * 2 ELSE
837  * 3 MOV TEMP[1], TEMP[2]
838  * 4 ENDIF
839  * 5 MOV TEMP[3], TEMP[1]
840  * The Abort flag will be set on instruction 5, because it could read from the
841  * value written by either instruction 1 or 3, depending on the jump decision
842  * made at instruction 0.
843  *
844  * writer = instruction 0;
845  * 0 MOV TEMP[0], TEMP[1]
846  * 2 BGNLOOP
847  * 3 ADD TEMP[0], TEMP[0], none.1
848  * 4 ENDLOOP
849  * The Abort flag will be set on instruction 3, because in the first iteration
850  * of the loop it reads the value written by instruction 0 and in all other
851  * iterations it reads the value written by instruction 3.
852  *
853  * @param read_cb This function will be called for every instruction that
854  * has been determined to be a reader of writer.
855  * @param write_cb This function will be called for every instruction after
856  * writer.
857  */
rc_get_readers(struct radeon_compiler * c,struct rc_instruction * writer,struct rc_reader_data * data,rc_read_src_fn read_normal_cb,rc_pair_read_arg_fn read_pair_cb,rc_read_write_mask_fn write_cb)858 void rc_get_readers(
859 	struct radeon_compiler * c,
860 	struct rc_instruction * writer,
861 	struct rc_reader_data * data,
862 	rc_read_src_fn read_normal_cb,
863 	rc_pair_read_arg_fn read_pair_cb,
864 	rc_read_write_mask_fn write_cb)
865 {
866 	struct get_readers_callback_data d;
867 
868 	init_get_readers_callback_data(&d, data, c, read_normal_cb,
869 						read_pair_cb, write_cb);
870 
871 	rc_for_all_writes_mask(writer, get_readers_for_single_write, &d);
872 }
873 
rc_get_readers_sub(struct radeon_compiler * c,struct rc_instruction * writer,struct rc_pair_sub_instruction * sub_writer,struct rc_reader_data * data,rc_read_src_fn read_normal_cb,rc_pair_read_arg_fn read_pair_cb,rc_read_write_mask_fn write_cb)874 void rc_get_readers_sub(
875 	struct radeon_compiler * c,
876 	struct rc_instruction * writer,
877 	struct rc_pair_sub_instruction * sub_writer,
878 	struct rc_reader_data * data,
879 	rc_read_src_fn read_normal_cb,
880 	rc_pair_read_arg_fn read_pair_cb,
881 	rc_read_write_mask_fn write_cb)
882 {
883 	struct get_readers_callback_data d;
884 
885 	init_get_readers_callback_data(&d, data, c, read_normal_cb,
886 						read_pair_cb, write_cb);
887 
888 	if (sub_writer->WriteMask) {
889 		get_readers_for_single_write(&d, writer, RC_FILE_TEMPORARY,
890 			sub_writer->DestIndex, sub_writer->WriteMask);
891 	}
892 }
893