1 /*
2 * Tiny Code Generator for QEMU
3 *
4 * Copyright (c) 2008 Fabrice Bellard
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a copy
7 * of this software and associated documentation files (the "Software"), to deal
8 * in the Software without restriction, including without limitation the rights
9 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10 * copies of the Software, and to permit persons to whom the Software is
11 * furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
22 * THE SOFTWARE.
23 */
24
25 /* define it to use liveness analysis (better code) */
26 #define USE_LIVENESS_ANALYSIS
27 #define USE_TCG_OPTIMIZATIONS
28
29 #include "config.h"
30
31 /* Define to jump the ELF file used to communicate with GDB. */
32 #undef DEBUG_JIT
33
34 #if !defined(CONFIG_DEBUG_TCG) && !defined(NDEBUG)
35 /* define it to suppress various consistency checks (faster) */
36 #define NDEBUG
37 #endif
38
39 #include "qemu-common.h"
40 #include "qemu/cache-utils.h"
41 #include "qemu/host-utils.h"
42 #include "qemu/timer.h"
43
44 /* Note: the long term plan is to reduce the dependancies on the QEMU
45 CPU definitions. Currently they are used for qemu_ld/st
46 instructions */
47 #define NO_CPU_IO_DEFS
48 #include "cpu.h"
49
50 #include "tcg-op.h"
51
52 #if UINTPTR_MAX == UINT32_MAX
53 # define ELF_CLASS ELFCLASS32
54 #else
55 # define ELF_CLASS ELFCLASS64
56 #endif
57 #ifdef HOST_WORDS_BIGENDIAN
58 # define ELF_DATA ELFDATA2MSB
59 #else
60 # define ELF_DATA ELFDATA2LSB
61 #endif
62
63 #include "elf.h"
64
65 /* Forward declarations for functions declared in tcg-target.c and used here. */
66 static void tcg_target_init(TCGContext *s);
67 static void tcg_target_qemu_prologue(TCGContext *s);
68 static void patch_reloc(uint8_t *code_ptr, int type,
69 intptr_t value, intptr_t addend);
70
71 /* The CIE and FDE header definitions will be common to all hosts. */
72 typedef struct {
73 uint32_t len __attribute__((aligned((sizeof(void *)))));
74 uint32_t id;
75 uint8_t version;
76 char augmentation[1];
77 uint8_t code_align;
78 uint8_t data_align;
79 uint8_t return_column;
80 } DebugFrameCIE;
81
82 typedef struct QEMU_PACKED {
83 uint32_t len __attribute__((aligned((sizeof(void *)))));
84 uint32_t cie_offset;
85 uintptr_t func_start;
86 uintptr_t func_len;
87 } DebugFrameFDEHeader;
88
89 static void tcg_register_jit_int(void *buf, size_t size,
90 void *debug_frame, size_t debug_frame_size)
91 __attribute__((unused));
92
93 /* Forward declarations for functions declared and used in tcg-target.c. */
94 static int target_parse_constraint(TCGArgConstraint *ct, const char **pct_str);
95 static void tcg_out_ld(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg1,
96 intptr_t arg2);
97 static void tcg_out_mov(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg);
98 static void tcg_out_movi(TCGContext *s, TCGType type,
99 TCGReg ret, tcg_target_long arg);
100 static void tcg_out_op(TCGContext *s, TCGOpcode opc, const TCGArg *args,
101 const int *const_args);
102 static void tcg_out_st(TCGContext *s, TCGType type, TCGReg arg, TCGReg arg1,
103 intptr_t arg2);
104 static int tcg_target_const_match(tcg_target_long val,
105 const TCGArgConstraint *arg_ct);
106 static void tcg_out_tb_init(TCGContext *s);
107 static void tcg_out_tb_finalize(TCGContext *s);
108
109
110 TCGOpDef tcg_op_defs[] = {
111 #define DEF(s, oargs, iargs, cargs, flags) { #s, oargs, iargs, cargs, iargs + oargs + cargs, flags },
112 #include "tcg-opc.h"
113 #undef DEF
114 };
115 const size_t tcg_op_defs_max = ARRAY_SIZE(tcg_op_defs);
116
117 static TCGRegSet tcg_target_available_regs[2];
118 static TCGRegSet tcg_target_call_clobber_regs;
119
tcg_out8(TCGContext * s,uint8_t v)120 static inline void tcg_out8(TCGContext *s, uint8_t v)
121 {
122 *s->code_ptr++ = v;
123 }
124
tcg_out16(TCGContext * s,uint16_t v)125 static inline void tcg_out16(TCGContext *s, uint16_t v)
126 {
127 uint8_t *p = s->code_ptr;
128 *(uint16_t *)p = v;
129 s->code_ptr = p + 2;
130 }
131
tcg_out32(TCGContext * s,uint32_t v)132 static inline void tcg_out32(TCGContext *s, uint32_t v)
133 {
134 uint8_t *p = s->code_ptr;
135 *(uint32_t *)p = v;
136 s->code_ptr = p + 4;
137 }
138
tcg_out64(TCGContext * s,uint64_t v)139 static inline void tcg_out64(TCGContext *s, uint64_t v)
140 {
141 uint8_t *p = s->code_ptr;
142 *(uint64_t *)p = v;
143 s->code_ptr = p + 8;
144 }
145
146 /* label relocation processing */
147
tcg_out_reloc(TCGContext * s,uint8_t * code_ptr,int type,int label_index,intptr_t addend)148 static void tcg_out_reloc(TCGContext *s, uint8_t *code_ptr, int type,
149 int label_index, intptr_t addend)
150 {
151 TCGLabel *l;
152 TCGRelocation *r;
153
154 l = &s->labels[label_index];
155 if (l->has_value) {
156 /* FIXME: This may break relocations on RISC targets that
157 modify instruction fields in place. The caller may not have
158 written the initial value. */
159 patch_reloc(code_ptr, type, l->u.value, addend);
160 } else {
161 /* add a new relocation entry */
162 r = tcg_malloc(sizeof(TCGRelocation));
163 r->type = type;
164 r->ptr = code_ptr;
165 r->addend = addend;
166 r->next = l->u.first_reloc;
167 l->u.first_reloc = r;
168 }
169 }
170
tcg_out_label(TCGContext * s,int label_index,void * ptr)171 static void tcg_out_label(TCGContext *s, int label_index, void *ptr)
172 {
173 TCGLabel *l;
174 TCGRelocation *r;
175 intptr_t value = (intptr_t)ptr;
176
177 l = &s->labels[label_index];
178 if (l->has_value) {
179 tcg_abort();
180 }
181 r = l->u.first_reloc;
182 while (r != NULL) {
183 patch_reloc(r->ptr, r->type, value, r->addend);
184 r = r->next;
185 }
186 l->has_value = 1;
187 l->u.value = value;
188 }
189
gen_new_label(void)190 int gen_new_label(void)
191 {
192 TCGContext *s = &tcg_ctx;
193 int idx;
194 TCGLabel *l;
195
196 if (s->nb_labels >= TCG_MAX_LABELS)
197 tcg_abort();
198 idx = s->nb_labels++;
199 l = &s->labels[idx];
200 l->has_value = 0;
201 l->u.first_reloc = NULL;
202 return idx;
203 }
204
205 #include "tcg-target.c"
206
207 /* pool based memory allocation */
tcg_malloc_internal(TCGContext * s,int size)208 void *tcg_malloc_internal(TCGContext *s, int size)
209 {
210 TCGPool *p;
211 int pool_size;
212
213 if (size > TCG_POOL_CHUNK_SIZE) {
214 /* big malloc: insert a new pool (XXX: could optimize) */
215 p = g_malloc(sizeof(TCGPool) + size);
216 p->size = size;
217 p->next = s->pool_first_large;
218 s->pool_first_large = p;
219 return p->data;
220 } else {
221 p = s->pool_current;
222 if (!p) {
223 p = s->pool_first;
224 if (!p)
225 goto new_pool;
226 } else {
227 if (!p->next) {
228 new_pool:
229 pool_size = TCG_POOL_CHUNK_SIZE;
230 p = g_malloc(sizeof(TCGPool) + pool_size);
231 p->size = pool_size;
232 p->next = NULL;
233 if (s->pool_current)
234 s->pool_current->next = p;
235 else
236 s->pool_first = p;
237 } else {
238 p = p->next;
239 }
240 }
241 }
242 s->pool_current = p;
243 s->pool_cur = p->data + size;
244 s->pool_end = p->data + p->size;
245 return p->data;
246 }
247
tcg_pool_reset(TCGContext * s)248 void tcg_pool_reset(TCGContext *s)
249 {
250 TCGPool *p, *t;
251 for (p = s->pool_first_large; p; p = t) {
252 t = p->next;
253 g_free(p);
254 }
255 s->pool_first_large = NULL;
256 s->pool_cur = s->pool_end = NULL;
257 s->pool_current = NULL;
258 }
259
260 #include "helper.h"
261
262 typedef struct TCGHelperInfo {
263 void *func;
264 const char *name;
265 } TCGHelperInfo;
266
267 static const TCGHelperInfo all_helpers[] = {
268 #define GEN_HELPER 2
269 #include "helper.h"
270
271 /* Include tcg-runtime.c functions. */
272 { tcg_helper_div_i32, "div_i32" },
273 { tcg_helper_rem_i32, "rem_i32" },
274 { tcg_helper_divu_i32, "divu_i32" },
275 { tcg_helper_remu_i32, "remu_i32" },
276
277 { tcg_helper_shl_i64, "shl_i64" },
278 { tcg_helper_shr_i64, "shr_i64" },
279 { tcg_helper_sar_i64, "sar_i64" },
280 { tcg_helper_div_i64, "div_i64" },
281 { tcg_helper_rem_i64, "rem_i64" },
282 { tcg_helper_divu_i64, "divu_i64" },
283 { tcg_helper_remu_i64, "remu_i64" },
284 { tcg_helper_mulsh_i64, "mulsh_i64" },
285 { tcg_helper_muluh_i64, "muluh_i64" },
286 };
287
tcg_context_init(TCGContext * s)288 void tcg_context_init(TCGContext *s)
289 {
290 int op, total_args, n, i;
291 TCGOpDef *def;
292 TCGArgConstraint *args_ct;
293 int *sorted_args;
294 GHashTable *helper_table;
295
296 memset(s, 0, sizeof(*s));
297 s->nb_globals = 0;
298
299 /* Count total number of arguments and allocate the corresponding
300 space */
301 total_args = 0;
302 for(op = 0; op < NB_OPS; op++) {
303 def = &tcg_op_defs[op];
304 n = def->nb_iargs + def->nb_oargs;
305 total_args += n;
306 }
307
308 args_ct = g_malloc(sizeof(TCGArgConstraint) * total_args);
309 sorted_args = g_malloc(sizeof(int) * total_args);
310
311 for(op = 0; op < NB_OPS; op++) {
312 def = &tcg_op_defs[op];
313 def->args_ct = args_ct;
314 def->sorted_args = sorted_args;
315 n = def->nb_iargs + def->nb_oargs;
316 sorted_args += n;
317 args_ct += n;
318 }
319
320 /* Register helpers. */
321 /* Use g_direct_hash/equal for direct pointer comparisons on func. */
322 s->helpers = helper_table = g_hash_table_new(NULL, NULL);
323
324 for (i = 0; i < ARRAY_SIZE(all_helpers); ++i) {
325 g_hash_table_insert(helper_table, (gpointer)all_helpers[i].func,
326 (gpointer)all_helpers[i].name);
327 }
328
329 tcg_target_init(s);
330 }
331
tcg_prologue_init(TCGContext * s)332 void tcg_prologue_init(TCGContext *s)
333 {
334 /* init global prologue and epilogue */
335 s->code_buf = s->code_gen_prologue;
336 s->code_ptr = s->code_buf;
337 tcg_target_qemu_prologue(s);
338 flush_icache_range((uintptr_t)s->code_buf, (uintptr_t)s->code_ptr);
339
340 #ifdef DEBUG_DISAS
341 if (qemu_loglevel_mask(CPU_LOG_TB_OUT_ASM)) {
342 size_t size = s->code_ptr - s->code_buf;
343 qemu_log("PROLOGUE: [size=%zu]\n", size);
344 log_disas(s->code_buf, size);
345 qemu_log("\n");
346 qemu_log_flush();
347 }
348 #endif
349 }
350
tcg_set_frame(TCGContext * s,int reg,intptr_t start,intptr_t size)351 void tcg_set_frame(TCGContext *s, int reg, intptr_t start, intptr_t size)
352 {
353 s->frame_start = start;
354 s->frame_end = start + size;
355 s->frame_reg = reg;
356 }
357
tcg_func_start(TCGContext * s)358 void tcg_func_start(TCGContext *s)
359 {
360 tcg_pool_reset(s);
361 s->nb_temps = s->nb_globals;
362
363 /* No temps have been previously allocated for size or locality. */
364 memset(s->free_temps, 0, sizeof(s->free_temps));
365
366 s->labels = tcg_malloc(sizeof(TCGLabel) * TCG_MAX_LABELS);
367 s->nb_labels = 0;
368 s->current_frame_offset = s->frame_start;
369
370 #ifdef CONFIG_DEBUG_TCG
371 s->goto_tb_issue_mask = 0;
372 #endif
373
374 s->gen_opc_ptr = s->gen_opc_buf;
375 s->gen_opparam_ptr = s->gen_opparam_buf;
376
377 s->be = tcg_malloc(sizeof(TCGBackendData));
378 }
379
tcg_temp_alloc(TCGContext * s,int n)380 static inline void tcg_temp_alloc(TCGContext *s, int n)
381 {
382 if (n > TCG_MAX_TEMPS)
383 tcg_abort();
384 }
385
tcg_global_reg_new_internal(TCGType type,int reg,const char * name)386 static inline int tcg_global_reg_new_internal(TCGType type, int reg,
387 const char *name)
388 {
389 TCGContext *s = &tcg_ctx;
390 TCGTemp *ts;
391 int idx;
392
393 #if TCG_TARGET_REG_BITS == 32
394 if (type != TCG_TYPE_I32)
395 tcg_abort();
396 #endif
397 if (tcg_regset_test_reg(s->reserved_regs, reg))
398 tcg_abort();
399 idx = s->nb_globals;
400 tcg_temp_alloc(s, s->nb_globals + 1);
401 ts = &s->temps[s->nb_globals];
402 ts->base_type = type;
403 ts->type = type;
404 ts->fixed_reg = 1;
405 ts->reg = reg;
406 ts->name = name;
407 s->nb_globals++;
408 tcg_regset_set_reg(s->reserved_regs, reg);
409 return idx;
410 }
411
tcg_global_reg_new_i32(int reg,const char * name)412 TCGv_i32 tcg_global_reg_new_i32(int reg, const char *name)
413 {
414 int idx;
415
416 idx = tcg_global_reg_new_internal(TCG_TYPE_I32, reg, name);
417 return MAKE_TCGV_I32(idx);
418 }
419
tcg_global_reg_new_i64(int reg,const char * name)420 TCGv_i64 tcg_global_reg_new_i64(int reg, const char *name)
421 {
422 int idx;
423
424 idx = tcg_global_reg_new_internal(TCG_TYPE_I64, reg, name);
425 return MAKE_TCGV_I64(idx);
426 }
427
tcg_global_mem_new_internal(TCGType type,int reg,intptr_t offset,const char * name)428 static inline int tcg_global_mem_new_internal(TCGType type, int reg,
429 intptr_t offset,
430 const char *name)
431 {
432 TCGContext *s = &tcg_ctx;
433 TCGTemp *ts;
434 int idx;
435
436 idx = s->nb_globals;
437 #if TCG_TARGET_REG_BITS == 32
438 if (type == TCG_TYPE_I64) {
439 char buf[64];
440 tcg_temp_alloc(s, s->nb_globals + 2);
441 ts = &s->temps[s->nb_globals];
442 ts->base_type = type;
443 ts->type = TCG_TYPE_I32;
444 ts->fixed_reg = 0;
445 ts->mem_allocated = 1;
446 ts->mem_reg = reg;
447 #ifdef TCG_TARGET_WORDS_BIGENDIAN
448 ts->mem_offset = offset + 4;
449 #else
450 ts->mem_offset = offset;
451 #endif
452 pstrcpy(buf, sizeof(buf), name);
453 pstrcat(buf, sizeof(buf), "_0");
454 ts->name = strdup(buf);
455 ts++;
456
457 ts->base_type = type;
458 ts->type = TCG_TYPE_I32;
459 ts->fixed_reg = 0;
460 ts->mem_allocated = 1;
461 ts->mem_reg = reg;
462 #ifdef TCG_TARGET_WORDS_BIGENDIAN
463 ts->mem_offset = offset;
464 #else
465 ts->mem_offset = offset + 4;
466 #endif
467 pstrcpy(buf, sizeof(buf), name);
468 pstrcat(buf, sizeof(buf), "_1");
469 ts->name = strdup(buf);
470
471 s->nb_globals += 2;
472 } else
473 #endif
474 {
475 tcg_temp_alloc(s, s->nb_globals + 1);
476 ts = &s->temps[s->nb_globals];
477 ts->base_type = type;
478 ts->type = type;
479 ts->fixed_reg = 0;
480 ts->mem_allocated = 1;
481 ts->mem_reg = reg;
482 ts->mem_offset = offset;
483 ts->name = name;
484 s->nb_globals++;
485 }
486 return idx;
487 }
488
tcg_global_mem_new_i32(int reg,intptr_t offset,const char * name)489 TCGv_i32 tcg_global_mem_new_i32(int reg, intptr_t offset, const char *name)
490 {
491 int idx = tcg_global_mem_new_internal(TCG_TYPE_I32, reg, offset, name);
492 return MAKE_TCGV_I32(idx);
493 }
494
tcg_global_mem_new_i64(int reg,intptr_t offset,const char * name)495 TCGv_i64 tcg_global_mem_new_i64(int reg, intptr_t offset, const char *name)
496 {
497 int idx = tcg_global_mem_new_internal(TCG_TYPE_I64, reg, offset, name);
498 return MAKE_TCGV_I64(idx);
499 }
500
tcg_temp_new_internal(TCGType type,int temp_local)501 static inline int tcg_temp_new_internal(TCGType type, int temp_local)
502 {
503 TCGContext *s = &tcg_ctx;
504 TCGTemp *ts;
505 int idx, k;
506
507 k = type + (temp_local ? TCG_TYPE_COUNT : 0);
508 idx = find_first_bit(s->free_temps[k].l, TCG_MAX_TEMPS);
509 if (idx < TCG_MAX_TEMPS) {
510 /* There is already an available temp with the right type. */
511 clear_bit(idx, s->free_temps[k].l);
512
513 ts = &s->temps[idx];
514 ts->temp_allocated = 1;
515 assert(ts->base_type == type);
516 assert(ts->temp_local == temp_local);
517 } else {
518 idx = s->nb_temps;
519 #if TCG_TARGET_REG_BITS == 32
520 if (type == TCG_TYPE_I64) {
521 tcg_temp_alloc(s, s->nb_temps + 2);
522 ts = &s->temps[s->nb_temps];
523 ts->base_type = type;
524 ts->type = TCG_TYPE_I32;
525 ts->temp_allocated = 1;
526 ts->temp_local = temp_local;
527 ts->name = NULL;
528 ts++;
529 ts->base_type = TCG_TYPE_I32;
530 ts->type = TCG_TYPE_I32;
531 ts->temp_allocated = 1;
532 ts->temp_local = temp_local;
533 ts->name = NULL;
534 s->nb_temps += 2;
535 } else
536 #endif
537 {
538 tcg_temp_alloc(s, s->nb_temps + 1);
539 ts = &s->temps[s->nb_temps];
540 ts->base_type = type;
541 ts->type = type;
542 ts->temp_allocated = 1;
543 ts->temp_local = temp_local;
544 ts->name = NULL;
545 s->nb_temps++;
546 }
547 }
548
549 #if defined(CONFIG_DEBUG_TCG)
550 s->temps_in_use++;
551 #endif
552 return idx;
553 }
554
tcg_temp_new_internal_i32(int temp_local)555 TCGv_i32 tcg_temp_new_internal_i32(int temp_local)
556 {
557 int idx;
558
559 idx = tcg_temp_new_internal(TCG_TYPE_I32, temp_local);
560 return MAKE_TCGV_I32(idx);
561 }
562
tcg_temp_new_internal_i64(int temp_local)563 TCGv_i64 tcg_temp_new_internal_i64(int temp_local)
564 {
565 int idx;
566
567 idx = tcg_temp_new_internal(TCG_TYPE_I64, temp_local);
568 return MAKE_TCGV_I64(idx);
569 }
570
tcg_temp_free_internal(int idx)571 static void tcg_temp_free_internal(int idx)
572 {
573 TCGContext *s = &tcg_ctx;
574 TCGTemp *ts;
575 int k;
576
577 #if defined(CONFIG_DEBUG_TCG)
578 s->temps_in_use--;
579 if (s->temps_in_use < 0) {
580 fprintf(stderr, "More temporaries freed than allocated!\n");
581 }
582 #endif
583
584 assert(idx >= s->nb_globals && idx < s->nb_temps);
585 ts = &s->temps[idx];
586 assert(ts->temp_allocated != 0);
587 ts->temp_allocated = 0;
588
589 k = ts->type + (ts->temp_local ? TCG_TYPE_COUNT : 0);
590 set_bit(idx, s->free_temps[k].l);
591 }
592
tcg_temp_free_i32(TCGv_i32 arg)593 void tcg_temp_free_i32(TCGv_i32 arg)
594 {
595 tcg_temp_free_internal(GET_TCGV_I32(arg));
596 }
597
tcg_temp_free_i64(TCGv_i64 arg)598 void tcg_temp_free_i64(TCGv_i64 arg)
599 {
600 tcg_temp_free_internal(GET_TCGV_I64(arg));
601 }
602
tcg_const_i32(int32_t val)603 TCGv_i32 tcg_const_i32(int32_t val)
604 {
605 TCGv_i32 t0;
606 t0 = tcg_temp_new_i32();
607 tcg_gen_movi_i32(t0, val);
608 return t0;
609 }
610
tcg_const_i64(int64_t val)611 TCGv_i64 tcg_const_i64(int64_t val)
612 {
613 TCGv_i64 t0;
614 t0 = tcg_temp_new_i64();
615 tcg_gen_movi_i64(t0, val);
616 return t0;
617 }
618
tcg_const_local_i32(int32_t val)619 TCGv_i32 tcg_const_local_i32(int32_t val)
620 {
621 TCGv_i32 t0;
622 t0 = tcg_temp_local_new_i32();
623 tcg_gen_movi_i32(t0, val);
624 return t0;
625 }
626
tcg_const_local_i64(int64_t val)627 TCGv_i64 tcg_const_local_i64(int64_t val)
628 {
629 TCGv_i64 t0;
630 t0 = tcg_temp_local_new_i64();
631 tcg_gen_movi_i64(t0, val);
632 return t0;
633 }
634
635 #if defined(CONFIG_DEBUG_TCG)
tcg_clear_temp_count(void)636 void tcg_clear_temp_count(void)
637 {
638 TCGContext *s = &tcg_ctx;
639 s->temps_in_use = 0;
640 }
641
tcg_check_temp_count(void)642 int tcg_check_temp_count(void)
643 {
644 TCGContext *s = &tcg_ctx;
645 if (s->temps_in_use) {
646 /* Clear the count so that we don't give another
647 * warning immediately next time around.
648 */
649 s->temps_in_use = 0;
650 return 1;
651 }
652 return 0;
653 }
654 #endif
655
656 /* Note: we convert the 64 bit args to 32 bit and do some alignment
657 and endian swap. Maybe it would be better to do the alignment
658 and endian swap in tcg_reg_alloc_call(). */
tcg_gen_callN(TCGContext * s,TCGv_ptr func,unsigned int flags,int sizemask,TCGArg ret,int nargs,TCGArg * args)659 void tcg_gen_callN(TCGContext *s, TCGv_ptr func, unsigned int flags,
660 int sizemask, TCGArg ret, int nargs, TCGArg *args)
661 {
662 int i;
663 int real_args;
664 int nb_rets;
665 TCGArg *nparam;
666
667 #if defined(TCG_TARGET_EXTEND_ARGS) && TCG_TARGET_REG_BITS == 64
668 for (i = 0; i < nargs; ++i) {
669 int is_64bit = sizemask & (1 << (i+1)*2);
670 int is_signed = sizemask & (2 << (i+1)*2);
671 if (!is_64bit) {
672 TCGv_i64 temp = tcg_temp_new_i64();
673 TCGv_i64 orig = MAKE_TCGV_I64(args[i]);
674 if (is_signed) {
675 tcg_gen_ext32s_i64(temp, orig);
676 } else {
677 tcg_gen_ext32u_i64(temp, orig);
678 }
679 args[i] = GET_TCGV_I64(temp);
680 }
681 }
682 #endif /* TCG_TARGET_EXTEND_ARGS */
683
684 *s->gen_opc_ptr++ = INDEX_op_call;
685 nparam = s->gen_opparam_ptr++;
686 if (ret != TCG_CALL_DUMMY_ARG) {
687 #if TCG_TARGET_REG_BITS < 64
688 if (sizemask & 1) {
689 #ifdef TCG_TARGET_WORDS_BIGENDIAN
690 *s->gen_opparam_ptr++ = ret + 1;
691 *s->gen_opparam_ptr++ = ret;
692 #else
693 *s->gen_opparam_ptr++ = ret;
694 *s->gen_opparam_ptr++ = ret + 1;
695 #endif
696 nb_rets = 2;
697 } else
698 #endif
699 {
700 *s->gen_opparam_ptr++ = ret;
701 nb_rets = 1;
702 }
703 } else {
704 nb_rets = 0;
705 }
706 real_args = 0;
707 for (i = 0; i < nargs; i++) {
708 #if TCG_TARGET_REG_BITS < 64
709 int is_64bit = sizemask & (1 << (i+1)*2);
710 if (is_64bit) {
711 #ifdef TCG_TARGET_CALL_ALIGN_ARGS
712 /* some targets want aligned 64 bit args */
713 if (real_args & 1) {
714 *s->gen_opparam_ptr++ = TCG_CALL_DUMMY_ARG;
715 real_args++;
716 }
717 #endif
718 /* If stack grows up, then we will be placing successive
719 arguments at lower addresses, which means we need to
720 reverse the order compared to how we would normally
721 treat either big or little-endian. For those arguments
722 that will wind up in registers, this still works for
723 HPPA (the only current STACK_GROWSUP target) since the
724 argument registers are *also* allocated in decreasing
725 order. If another such target is added, this logic may
726 have to get more complicated to differentiate between
727 stack arguments and register arguments. */
728 #if defined(TCG_TARGET_WORDS_BIGENDIAN) != defined(TCG_TARGET_STACK_GROWSUP)
729 *s->gen_opparam_ptr++ = args[i] + 1;
730 *s->gen_opparam_ptr++ = args[i];
731 #else
732 *s->gen_opparam_ptr++ = args[i];
733 *s->gen_opparam_ptr++ = args[i] + 1;
734 #endif
735 real_args += 2;
736 continue;
737 }
738 #endif /* TCG_TARGET_REG_BITS < 64 */
739
740 *s->gen_opparam_ptr++ = args[i];
741 real_args++;
742 }
743 *s->gen_opparam_ptr++ = GET_TCGV_PTR(func);
744
745 *s->gen_opparam_ptr++ = flags;
746
747 *nparam = (nb_rets << 16) | (real_args + 1);
748
749 /* total parameters, needed to go backward in the instruction stream */
750 *s->gen_opparam_ptr++ = 1 + nb_rets + real_args + 3;
751
752 #if defined(TCG_TARGET_EXTEND_ARGS) && TCG_TARGET_REG_BITS == 64
753 for (i = 0; i < nargs; ++i) {
754 int is_64bit = sizemask & (1 << (i+1)*2);
755 if (!is_64bit) {
756 TCGv_i64 temp = MAKE_TCGV_I64(args[i]);
757 tcg_temp_free_i64(temp);
758 }
759 }
760 #endif /* TCG_TARGET_EXTEND_ARGS */
761 }
762
763 #if TCG_TARGET_REG_BITS == 32
tcg_gen_shifti_i64(TCGv_i64 ret,TCGv_i64 arg1,int c,int right,int arith)764 void tcg_gen_shifti_i64(TCGv_i64 ret, TCGv_i64 arg1,
765 int c, int right, int arith)
766 {
767 if (c == 0) {
768 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg1));
769 tcg_gen_mov_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1));
770 } else if (c >= 32) {
771 c -= 32;
772 if (right) {
773 if (arith) {
774 tcg_gen_sari_i32(TCGV_LOW(ret), TCGV_HIGH(arg1), c);
775 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), 31);
776 } else {
777 tcg_gen_shri_i32(TCGV_LOW(ret), TCGV_HIGH(arg1), c);
778 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
779 }
780 } else {
781 tcg_gen_shli_i32(TCGV_HIGH(ret), TCGV_LOW(arg1), c);
782 tcg_gen_movi_i32(TCGV_LOW(ret), 0);
783 }
784 } else {
785 TCGv_i32 t0, t1;
786
787 t0 = tcg_temp_new_i32();
788 t1 = tcg_temp_new_i32();
789 if (right) {
790 tcg_gen_shli_i32(t0, TCGV_HIGH(arg1), 32 - c);
791 if (arith)
792 tcg_gen_sari_i32(t1, TCGV_HIGH(arg1), c);
793 else
794 tcg_gen_shri_i32(t1, TCGV_HIGH(arg1), c);
795 tcg_gen_shri_i32(TCGV_LOW(ret), TCGV_LOW(arg1), c);
796 tcg_gen_or_i32(TCGV_LOW(ret), TCGV_LOW(ret), t0);
797 tcg_gen_mov_i32(TCGV_HIGH(ret), t1);
798 } else {
799 tcg_gen_shri_i32(t0, TCGV_LOW(arg1), 32 - c);
800 /* Note: ret can be the same as arg1, so we use t1 */
801 tcg_gen_shli_i32(t1, TCGV_LOW(arg1), c);
802 tcg_gen_shli_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), c);
803 tcg_gen_or_i32(TCGV_HIGH(ret), TCGV_HIGH(ret), t0);
804 tcg_gen_mov_i32(TCGV_LOW(ret), t1);
805 }
806 tcg_temp_free_i32(t0);
807 tcg_temp_free_i32(t1);
808 }
809 }
810 #endif
811
tcg_canonicalize_memop(TCGMemOp op,bool is64,bool st)812 static inline TCGMemOp tcg_canonicalize_memop(TCGMemOp op, bool is64, bool st)
813 {
814 switch (op & MO_SIZE) {
815 case MO_8:
816 op &= ~MO_BSWAP;
817 break;
818 case MO_16:
819 break;
820 case MO_32:
821 if (!is64) {
822 op &= ~MO_SIGN;
823 }
824 break;
825 case MO_64:
826 if (!is64) {
827 tcg_abort();
828 }
829 break;
830 }
831 if (st) {
832 op &= ~MO_SIGN;
833 }
834 return op;
835 }
836
837 static const TCGOpcode old_ld_opc[8] = {
838 [MO_UB] = INDEX_op_qemu_ld8u,
839 [MO_SB] = INDEX_op_qemu_ld8s,
840 [MO_UW] = INDEX_op_qemu_ld16u,
841 [MO_SW] = INDEX_op_qemu_ld16s,
842 #if TCG_TARGET_REG_BITS == 32
843 [MO_UL] = INDEX_op_qemu_ld32,
844 [MO_SL] = INDEX_op_qemu_ld32,
845 #else
846 [MO_UL] = INDEX_op_qemu_ld32u,
847 [MO_SL] = INDEX_op_qemu_ld32s,
848 #endif
849 [MO_Q] = INDEX_op_qemu_ld64,
850 };
851
852 static const TCGOpcode old_st_opc[4] = {
853 [MO_UB] = INDEX_op_qemu_st8,
854 [MO_UW] = INDEX_op_qemu_st16,
855 [MO_UL] = INDEX_op_qemu_st32,
856 [MO_Q] = INDEX_op_qemu_st64,
857 };
858
tcg_gen_qemu_ld_i32(TCGv_i32 val,TCGv addr,TCGArg idx,TCGMemOp memop)859 void tcg_gen_qemu_ld_i32(TCGv_i32 val, TCGv addr, TCGArg idx, TCGMemOp memop)
860 {
861 memop = tcg_canonicalize_memop(memop, 0, 0);
862
863 if (TCG_TARGET_HAS_new_ldst) {
864 *tcg_ctx.gen_opc_ptr++ = INDEX_op_qemu_ld_i32;
865 tcg_add_param_i32(val);
866 tcg_add_param_tl(addr);
867 *tcg_ctx.gen_opparam_ptr++ = memop;
868 *tcg_ctx.gen_opparam_ptr++ = idx;
869 return;
870 }
871
872 /* The old opcodes only support target-endian memory operations. */
873 assert((memop & MO_BSWAP) == MO_TE || (memop & MO_SIZE) == MO_8);
874 assert(old_ld_opc[memop & MO_SSIZE] != 0);
875
876 if (TCG_TARGET_REG_BITS == 32) {
877 *tcg_ctx.gen_opc_ptr++ = old_ld_opc[memop & MO_SSIZE];
878 tcg_add_param_i32(val);
879 tcg_add_param_tl(addr);
880 *tcg_ctx.gen_opparam_ptr++ = idx;
881 } else {
882 TCGv_i64 val64 = tcg_temp_new_i64();
883
884 *tcg_ctx.gen_opc_ptr++ = old_ld_opc[memop & MO_SSIZE];
885 tcg_add_param_i64(val64);
886 tcg_add_param_tl(addr);
887 *tcg_ctx.gen_opparam_ptr++ = idx;
888
889 tcg_gen_trunc_i64_i32(val, val64);
890 tcg_temp_free_i64(val64);
891 }
892 }
893
tcg_gen_qemu_st_i32(TCGv_i32 val,TCGv addr,TCGArg idx,TCGMemOp memop)894 void tcg_gen_qemu_st_i32(TCGv_i32 val, TCGv addr, TCGArg idx, TCGMemOp memop)
895 {
896 memop = tcg_canonicalize_memop(memop, 0, 1);
897
898 if (TCG_TARGET_HAS_new_ldst) {
899 *tcg_ctx.gen_opc_ptr++ = INDEX_op_qemu_st_i32;
900 tcg_add_param_i32(val);
901 tcg_add_param_tl(addr);
902 *tcg_ctx.gen_opparam_ptr++ = memop;
903 *tcg_ctx.gen_opparam_ptr++ = idx;
904 return;
905 }
906
907 /* The old opcodes only support target-endian memory operations. */
908 assert((memop & MO_BSWAP) == MO_TE || (memop & MO_SIZE) == MO_8);
909 assert(old_st_opc[memop & MO_SIZE] != 0);
910
911 if (TCG_TARGET_REG_BITS == 32) {
912 *tcg_ctx.gen_opc_ptr++ = old_st_opc[memop & MO_SIZE];
913 tcg_add_param_i32(val);
914 tcg_add_param_tl(addr);
915 *tcg_ctx.gen_opparam_ptr++ = idx;
916 } else {
917 TCGv_i64 val64 = tcg_temp_new_i64();
918
919 tcg_gen_extu_i32_i64(val64, val);
920
921 *tcg_ctx.gen_opc_ptr++ = old_st_opc[memop & MO_SIZE];
922 tcg_add_param_i64(val64);
923 tcg_add_param_tl(addr);
924 *tcg_ctx.gen_opparam_ptr++ = idx;
925
926 tcg_temp_free_i64(val64);
927 }
928 }
929
tcg_gen_qemu_ld_i64(TCGv_i64 val,TCGv addr,TCGArg idx,TCGMemOp memop)930 void tcg_gen_qemu_ld_i64(TCGv_i64 val, TCGv addr, TCGArg idx, TCGMemOp memop)
931 {
932 memop = tcg_canonicalize_memop(memop, 1, 0);
933
934 #if TCG_TARGET_REG_BITS == 32
935 if ((memop & MO_SIZE) < MO_64) {
936 tcg_gen_qemu_ld_i32(TCGV_LOW(val), addr, idx, memop);
937 if (memop & MO_SIGN) {
938 tcg_gen_sari_i32(TCGV_HIGH(val), TCGV_LOW(val), 31);
939 } else {
940 tcg_gen_movi_i32(TCGV_HIGH(val), 0);
941 }
942 return;
943 }
944 #endif
945
946 if (TCG_TARGET_HAS_new_ldst) {
947 *tcg_ctx.gen_opc_ptr++ = INDEX_op_qemu_ld_i64;
948 tcg_add_param_i64(val);
949 tcg_add_param_tl(addr);
950 *tcg_ctx.gen_opparam_ptr++ = memop;
951 *tcg_ctx.gen_opparam_ptr++ = idx;
952 return;
953 }
954
955 /* The old opcodes only support target-endian memory operations. */
956 assert((memop & MO_BSWAP) == MO_TE || (memop & MO_SIZE) == MO_8);
957 assert(old_ld_opc[memop & MO_SSIZE] != 0);
958
959 *tcg_ctx.gen_opc_ptr++ = old_ld_opc[memop & MO_SSIZE];
960 tcg_add_param_i64(val);
961 tcg_add_param_tl(addr);
962 *tcg_ctx.gen_opparam_ptr++ = idx;
963 }
964
tcg_gen_qemu_st_i64(TCGv_i64 val,TCGv addr,TCGArg idx,TCGMemOp memop)965 void tcg_gen_qemu_st_i64(TCGv_i64 val, TCGv addr, TCGArg idx, TCGMemOp memop)
966 {
967 memop = tcg_canonicalize_memop(memop, 1, 1);
968
969 #if TCG_TARGET_REG_BITS == 32
970 if ((memop & MO_SIZE) < MO_64) {
971 tcg_gen_qemu_st_i32(TCGV_LOW(val), addr, idx, memop);
972 return;
973 }
974 #endif
975
976 if (TCG_TARGET_HAS_new_ldst) {
977 *tcg_ctx.gen_opc_ptr++ = INDEX_op_qemu_st_i64;
978 tcg_add_param_i64(val);
979 tcg_add_param_tl(addr);
980 *tcg_ctx.gen_opparam_ptr++ = memop;
981 *tcg_ctx.gen_opparam_ptr++ = idx;
982 return;
983 }
984
985 /* The old opcodes only support target-endian memory operations. */
986 assert((memop & MO_BSWAP) == MO_TE || (memop & MO_SIZE) == MO_8);
987 assert(old_st_opc[memop & MO_SIZE] != 0);
988
989 *tcg_ctx.gen_opc_ptr++ = old_st_opc[memop & MO_SIZE];
990 tcg_add_param_i64(val);
991 tcg_add_param_tl(addr);
992 *tcg_ctx.gen_opparam_ptr++ = idx;
993 }
994
tcg_reg_alloc_start(TCGContext * s)995 static void tcg_reg_alloc_start(TCGContext *s)
996 {
997 int i;
998 TCGTemp *ts;
999 for(i = 0; i < s->nb_globals; i++) {
1000 ts = &s->temps[i];
1001 if (ts->fixed_reg) {
1002 ts->val_type = TEMP_VAL_REG;
1003 } else {
1004 ts->val_type = TEMP_VAL_MEM;
1005 }
1006 }
1007 for(i = s->nb_globals; i < s->nb_temps; i++) {
1008 ts = &s->temps[i];
1009 if (ts->temp_local) {
1010 ts->val_type = TEMP_VAL_MEM;
1011 } else {
1012 ts->val_type = TEMP_VAL_DEAD;
1013 }
1014 ts->mem_allocated = 0;
1015 ts->fixed_reg = 0;
1016 }
1017 for(i = 0; i < TCG_TARGET_NB_REGS; i++) {
1018 s->reg_to_temp[i] = -1;
1019 }
1020 }
1021
tcg_get_arg_str_idx(TCGContext * s,char * buf,int buf_size,int idx)1022 static char *tcg_get_arg_str_idx(TCGContext *s, char *buf, int buf_size,
1023 int idx)
1024 {
1025 TCGTemp *ts;
1026
1027 assert(idx >= 0 && idx < s->nb_temps);
1028 ts = &s->temps[idx];
1029 if (idx < s->nb_globals) {
1030 pstrcpy(buf, buf_size, ts->name);
1031 } else {
1032 if (ts->temp_local)
1033 snprintf(buf, buf_size, "loc%d", idx - s->nb_globals);
1034 else
1035 snprintf(buf, buf_size, "tmp%d", idx - s->nb_globals);
1036 }
1037 return buf;
1038 }
1039
tcg_get_arg_str_i32(TCGContext * s,char * buf,int buf_size,TCGv_i32 arg)1040 char *tcg_get_arg_str_i32(TCGContext *s, char *buf, int buf_size, TCGv_i32 arg)
1041 {
1042 return tcg_get_arg_str_idx(s, buf, buf_size, GET_TCGV_I32(arg));
1043 }
1044
tcg_get_arg_str_i64(TCGContext * s,char * buf,int buf_size,TCGv_i64 arg)1045 char *tcg_get_arg_str_i64(TCGContext *s, char *buf, int buf_size, TCGv_i64 arg)
1046 {
1047 return tcg_get_arg_str_idx(s, buf, buf_size, GET_TCGV_I64(arg));
1048 }
1049
1050 /* Find helper name. */
tcg_find_helper(TCGContext * s,uintptr_t val)1051 static inline const char *tcg_find_helper(TCGContext *s, uintptr_t val)
1052 {
1053 const char *ret = NULL;
1054 if (s->helpers) {
1055 ret = g_hash_table_lookup(s->helpers, (gpointer)val);
1056 }
1057 return ret;
1058 }
1059
1060 static const char * const cond_name[] =
1061 {
1062 [TCG_COND_NEVER] = "never",
1063 [TCG_COND_ALWAYS] = "always",
1064 [TCG_COND_EQ] = "eq",
1065 [TCG_COND_NE] = "ne",
1066 [TCG_COND_LT] = "lt",
1067 [TCG_COND_GE] = "ge",
1068 [TCG_COND_LE] = "le",
1069 [TCG_COND_GT] = "gt",
1070 [TCG_COND_LTU] = "ltu",
1071 [TCG_COND_GEU] = "geu",
1072 [TCG_COND_LEU] = "leu",
1073 [TCG_COND_GTU] = "gtu"
1074 };
1075
1076 static const char * const ldst_name[] =
1077 {
1078 [MO_UB] = "ub",
1079 [MO_SB] = "sb",
1080 [MO_LEUW] = "leuw",
1081 [MO_LESW] = "lesw",
1082 [MO_LEUL] = "leul",
1083 [MO_LESL] = "lesl",
1084 [MO_LEQ] = "leq",
1085 [MO_BEUW] = "beuw",
1086 [MO_BESW] = "besw",
1087 [MO_BEUL] = "beul",
1088 [MO_BESL] = "besl",
1089 [MO_BEQ] = "beq",
1090 };
1091
tcg_dump_ops(TCGContext * s)1092 void tcg_dump_ops(TCGContext *s)
1093 {
1094 const uint16_t *opc_ptr;
1095 const TCGArg *args;
1096 TCGArg arg;
1097 TCGOpcode c;
1098 int i, k, nb_oargs, nb_iargs, nb_cargs, first_insn;
1099 const TCGOpDef *def;
1100 char buf[128];
1101
1102 first_insn = 1;
1103 opc_ptr = s->gen_opc_buf;
1104 args = s->gen_opparam_buf;
1105 while (opc_ptr < s->gen_opc_ptr) {
1106 c = *opc_ptr++;
1107 def = &tcg_op_defs[c];
1108 if (c == INDEX_op_debug_insn_start) {
1109 uint64_t pc;
1110 #if TARGET_LONG_BITS > TCG_TARGET_REG_BITS
1111 pc = ((uint64_t)args[1] << 32) | args[0];
1112 #else
1113 pc = args[0];
1114 #endif
1115 if (!first_insn) {
1116 qemu_log("\n");
1117 }
1118 qemu_log(" ---- 0x%" PRIx64, pc);
1119 first_insn = 0;
1120 nb_oargs = def->nb_oargs;
1121 nb_iargs = def->nb_iargs;
1122 nb_cargs = def->nb_cargs;
1123 } else if (c == INDEX_op_call) {
1124 TCGArg arg;
1125
1126 /* variable number of arguments */
1127 arg = *args++;
1128 nb_oargs = arg >> 16;
1129 nb_iargs = arg & 0xffff;
1130 nb_cargs = def->nb_cargs;
1131
1132 qemu_log(" %s ", def->name);
1133
1134 /* function name */
1135 qemu_log("%s",
1136 tcg_get_arg_str_idx(s, buf, sizeof(buf),
1137 args[nb_oargs + nb_iargs - 1]));
1138 /* flags */
1139 qemu_log(",$0x%" TCG_PRIlx, args[nb_oargs + nb_iargs]);
1140 /* nb out args */
1141 qemu_log(",$%d", nb_oargs);
1142 for(i = 0; i < nb_oargs; i++) {
1143 qemu_log(",");
1144 qemu_log("%s", tcg_get_arg_str_idx(s, buf, sizeof(buf),
1145 args[i]));
1146 }
1147 for(i = 0; i < (nb_iargs - 1); i++) {
1148 qemu_log(",");
1149 if (args[nb_oargs + i] == TCG_CALL_DUMMY_ARG) {
1150 qemu_log("<dummy>");
1151 } else {
1152 qemu_log("%s", tcg_get_arg_str_idx(s, buf, sizeof(buf),
1153 args[nb_oargs + i]));
1154 }
1155 }
1156 } else if (c == INDEX_op_movi_i32 || c == INDEX_op_movi_i64) {
1157 tcg_target_ulong val;
1158 const char *name;
1159
1160 nb_oargs = def->nb_oargs;
1161 nb_iargs = def->nb_iargs;
1162 nb_cargs = def->nb_cargs;
1163 qemu_log(" %s %s,$", def->name,
1164 tcg_get_arg_str_idx(s, buf, sizeof(buf), args[0]));
1165 val = args[1];
1166 name = tcg_find_helper(s, val);
1167 if (name) {
1168 qemu_log("%s", name);
1169 } else {
1170 if (c == INDEX_op_movi_i32) {
1171 qemu_log("0x%x", (uint32_t)val);
1172 } else {
1173 qemu_log("0x%" PRIx64 , (uint64_t)val);
1174 }
1175 }
1176 } else {
1177 qemu_log(" %s ", def->name);
1178 if (c == INDEX_op_nopn) {
1179 /* variable number of arguments */
1180 nb_cargs = *args;
1181 nb_oargs = 0;
1182 nb_iargs = 0;
1183 } else {
1184 nb_oargs = def->nb_oargs;
1185 nb_iargs = def->nb_iargs;
1186 nb_cargs = def->nb_cargs;
1187 }
1188
1189 k = 0;
1190 for(i = 0; i < nb_oargs; i++) {
1191 if (k != 0) {
1192 qemu_log(",");
1193 }
1194 qemu_log("%s", tcg_get_arg_str_idx(s, buf, sizeof(buf),
1195 args[k++]));
1196 }
1197 for(i = 0; i < nb_iargs; i++) {
1198 if (k != 0) {
1199 qemu_log(",");
1200 }
1201 qemu_log("%s", tcg_get_arg_str_idx(s, buf, sizeof(buf),
1202 args[k++]));
1203 }
1204 switch (c) {
1205 case INDEX_op_brcond_i32:
1206 case INDEX_op_setcond_i32:
1207 case INDEX_op_movcond_i32:
1208 case INDEX_op_brcond2_i32:
1209 case INDEX_op_setcond2_i32:
1210 case INDEX_op_brcond_i64:
1211 case INDEX_op_setcond_i64:
1212 case INDEX_op_movcond_i64:
1213 if (args[k] < ARRAY_SIZE(cond_name) && cond_name[args[k]]) {
1214 qemu_log(",%s", cond_name[args[k++]]);
1215 } else {
1216 qemu_log(",$0x%" TCG_PRIlx, args[k++]);
1217 }
1218 i = 1;
1219 break;
1220 case INDEX_op_qemu_ld_i32:
1221 case INDEX_op_qemu_st_i32:
1222 case INDEX_op_qemu_ld_i64:
1223 case INDEX_op_qemu_st_i64:
1224 if (args[k] < ARRAY_SIZE(ldst_name) && ldst_name[args[k]]) {
1225 qemu_log(",%s", ldst_name[args[k++]]);
1226 } else {
1227 qemu_log(",$0x%" TCG_PRIlx, args[k++]);
1228 }
1229 i = 1;
1230 break;
1231 default:
1232 i = 0;
1233 break;
1234 }
1235 for(; i < nb_cargs; i++) {
1236 if (k != 0) {
1237 qemu_log(",");
1238 }
1239 arg = args[k++];
1240 qemu_log("$0x%" TCG_PRIlx, arg);
1241 }
1242 }
1243 qemu_log("\n");
1244 args += nb_iargs + nb_oargs + nb_cargs;
1245 }
1246 }
1247
1248 /* we give more priority to constraints with less registers */
get_constraint_priority(const TCGOpDef * def,int k)1249 static int get_constraint_priority(const TCGOpDef *def, int k)
1250 {
1251 const TCGArgConstraint *arg_ct;
1252
1253 int i, n;
1254 arg_ct = &def->args_ct[k];
1255 if (arg_ct->ct & TCG_CT_ALIAS) {
1256 /* an alias is equivalent to a single register */
1257 n = 1;
1258 } else {
1259 if (!(arg_ct->ct & TCG_CT_REG))
1260 return 0;
1261 n = 0;
1262 for(i = 0; i < TCG_TARGET_NB_REGS; i++) {
1263 if (tcg_regset_test_reg(arg_ct->u.regs, i))
1264 n++;
1265 }
1266 }
1267 return TCG_TARGET_NB_REGS - n + 1;
1268 }
1269
1270 /* sort from highest priority to lowest */
sort_constraints(TCGOpDef * def,int start,int n)1271 static void sort_constraints(TCGOpDef *def, int start, int n)
1272 {
1273 int i, j, p1, p2, tmp;
1274
1275 for(i = 0; i < n; i++)
1276 def->sorted_args[start + i] = start + i;
1277 if (n <= 1)
1278 return;
1279 for(i = 0; i < n - 1; i++) {
1280 for(j = i + 1; j < n; j++) {
1281 p1 = get_constraint_priority(def, def->sorted_args[start + i]);
1282 p2 = get_constraint_priority(def, def->sorted_args[start + j]);
1283 if (p1 < p2) {
1284 tmp = def->sorted_args[start + i];
1285 def->sorted_args[start + i] = def->sorted_args[start + j];
1286 def->sorted_args[start + j] = tmp;
1287 }
1288 }
1289 }
1290 }
1291
tcg_add_target_add_op_defs(const TCGTargetOpDef * tdefs)1292 void tcg_add_target_add_op_defs(const TCGTargetOpDef *tdefs)
1293 {
1294 TCGOpcode op;
1295 TCGOpDef *def;
1296 const char *ct_str;
1297 int i, nb_args;
1298
1299 for(;;) {
1300 if (tdefs->op == (TCGOpcode)-1)
1301 break;
1302 op = tdefs->op;
1303 assert((unsigned)op < NB_OPS);
1304 def = &tcg_op_defs[op];
1305 #if defined(CONFIG_DEBUG_TCG)
1306 /* Duplicate entry in op definitions? */
1307 assert(!def->used);
1308 def->used = 1;
1309 #endif
1310 nb_args = def->nb_iargs + def->nb_oargs;
1311 for(i = 0; i < nb_args; i++) {
1312 ct_str = tdefs->args_ct_str[i];
1313 /* Incomplete TCGTargetOpDef entry? */
1314 assert(ct_str != NULL);
1315 tcg_regset_clear(def->args_ct[i].u.regs);
1316 def->args_ct[i].ct = 0;
1317 if (ct_str[0] >= '0' && ct_str[0] <= '9') {
1318 int oarg;
1319 oarg = ct_str[0] - '0';
1320 assert(oarg < def->nb_oargs);
1321 assert(def->args_ct[oarg].ct & TCG_CT_REG);
1322 /* TCG_CT_ALIAS is for the output arguments. The input
1323 argument is tagged with TCG_CT_IALIAS. */
1324 def->args_ct[i] = def->args_ct[oarg];
1325 def->args_ct[oarg].ct = TCG_CT_ALIAS;
1326 def->args_ct[oarg].alias_index = i;
1327 def->args_ct[i].ct |= TCG_CT_IALIAS;
1328 def->args_ct[i].alias_index = oarg;
1329 } else {
1330 for(;;) {
1331 if (*ct_str == '\0')
1332 break;
1333 switch(*ct_str) {
1334 case 'i':
1335 def->args_ct[i].ct |= TCG_CT_CONST;
1336 ct_str++;
1337 break;
1338 default:
1339 if (target_parse_constraint(&def->args_ct[i], &ct_str) < 0) {
1340 fprintf(stderr, "Invalid constraint '%s' for arg %d of operation '%s'\n",
1341 ct_str, i, def->name);
1342 exit(1);
1343 }
1344 }
1345 }
1346 }
1347 }
1348
1349 /* TCGTargetOpDef entry with too much information? */
1350 assert(i == TCG_MAX_OP_ARGS || tdefs->args_ct_str[i] == NULL);
1351
1352 /* sort the constraints (XXX: this is just an heuristic) */
1353 sort_constraints(def, 0, def->nb_oargs);
1354 sort_constraints(def, def->nb_oargs, def->nb_iargs);
1355
1356 #if 0
1357 {
1358 int i;
1359
1360 printf("%s: sorted=", def->name);
1361 for(i = 0; i < def->nb_oargs + def->nb_iargs; i++)
1362 printf(" %d", def->sorted_args[i]);
1363 printf("\n");
1364 }
1365 #endif
1366 tdefs++;
1367 }
1368
1369 #if defined(CONFIG_DEBUG_TCG)
1370 i = 0;
1371 for (op = 0; op < ARRAY_SIZE(tcg_op_defs); op++) {
1372 const TCGOpDef *def = &tcg_op_defs[op];
1373 if (def->flags & TCG_OPF_NOT_PRESENT) {
1374 /* Wrong entry in op definitions? */
1375 if (def->used) {
1376 fprintf(stderr, "Invalid op definition for %s\n", def->name);
1377 i = 1;
1378 }
1379 } else {
1380 /* Missing entry in op definitions? */
1381 if (!def->used) {
1382 fprintf(stderr, "Missing op definition for %s\n", def->name);
1383 i = 1;
1384 }
1385 }
1386 }
1387 if (i == 1) {
1388 tcg_abort();
1389 }
1390 #endif
1391 }
1392
1393 #ifdef USE_LIVENESS_ANALYSIS
1394
1395 /* set a nop for an operation using 'nb_args' */
tcg_set_nop(TCGContext * s,uint16_t * opc_ptr,TCGArg * args,int nb_args)1396 static inline void tcg_set_nop(TCGContext *s, uint16_t *opc_ptr,
1397 TCGArg *args, int nb_args)
1398 {
1399 if (nb_args == 0) {
1400 *opc_ptr = INDEX_op_nop;
1401 } else {
1402 *opc_ptr = INDEX_op_nopn;
1403 args[0] = nb_args;
1404 args[nb_args - 1] = nb_args;
1405 }
1406 }
1407
1408 /* liveness analysis: end of function: all temps are dead, and globals
1409 should be in memory. */
tcg_la_func_end(TCGContext * s,uint8_t * dead_temps,uint8_t * mem_temps)1410 static inline void tcg_la_func_end(TCGContext *s, uint8_t *dead_temps,
1411 uint8_t *mem_temps)
1412 {
1413 memset(dead_temps, 1, s->nb_temps);
1414 memset(mem_temps, 1, s->nb_globals);
1415 memset(mem_temps + s->nb_globals, 0, s->nb_temps - s->nb_globals);
1416 }
1417
1418 /* liveness analysis: end of basic block: all temps are dead, globals
1419 and local temps should be in memory. */
tcg_la_bb_end(TCGContext * s,uint8_t * dead_temps,uint8_t * mem_temps)1420 static inline void tcg_la_bb_end(TCGContext *s, uint8_t *dead_temps,
1421 uint8_t *mem_temps)
1422 {
1423 int i;
1424
1425 memset(dead_temps, 1, s->nb_temps);
1426 memset(mem_temps, 1, s->nb_globals);
1427 for(i = s->nb_globals; i < s->nb_temps; i++) {
1428 mem_temps[i] = s->temps[i].temp_local;
1429 }
1430 }
1431
1432 /* Liveness analysis : update the opc_dead_args array to tell if a
1433 given input arguments is dead. Instructions updating dead
1434 temporaries are removed. */
tcg_liveness_analysis(TCGContext * s)1435 static void tcg_liveness_analysis(TCGContext *s)
1436 {
1437 int i, op_index, nb_args, nb_iargs, nb_oargs, arg, nb_ops;
1438 TCGOpcode op, op_new, op_new2;
1439 TCGArg *args;
1440 const TCGOpDef *def;
1441 uint8_t *dead_temps, *mem_temps;
1442 uint16_t dead_args;
1443 uint8_t sync_args;
1444 bool have_op_new2;
1445
1446 s->gen_opc_ptr++; /* skip end */
1447
1448 nb_ops = s->gen_opc_ptr - s->gen_opc_buf;
1449
1450 s->op_dead_args = tcg_malloc(nb_ops * sizeof(uint16_t));
1451 s->op_sync_args = tcg_malloc(nb_ops * sizeof(uint8_t));
1452
1453 dead_temps = tcg_malloc(s->nb_temps);
1454 mem_temps = tcg_malloc(s->nb_temps);
1455 tcg_la_func_end(s, dead_temps, mem_temps);
1456
1457 args = s->gen_opparam_ptr;
1458 op_index = nb_ops - 1;
1459 while (op_index >= 0) {
1460 op = s->gen_opc_buf[op_index];
1461 def = &tcg_op_defs[op];
1462 switch(op) {
1463 case INDEX_op_call:
1464 {
1465 int call_flags;
1466
1467 nb_args = args[-1];
1468 args -= nb_args;
1469 nb_iargs = args[0] & 0xffff;
1470 nb_oargs = args[0] >> 16;
1471 args++;
1472 call_flags = args[nb_oargs + nb_iargs];
1473
1474 /* pure functions can be removed if their result is not
1475 used */
1476 if (call_flags & TCG_CALL_NO_SIDE_EFFECTS) {
1477 for(i = 0; i < nb_oargs; i++) {
1478 arg = args[i];
1479 if (!dead_temps[arg] || mem_temps[arg]) {
1480 goto do_not_remove_call;
1481 }
1482 }
1483 tcg_set_nop(s, s->gen_opc_buf + op_index,
1484 args - 1, nb_args);
1485 } else {
1486 do_not_remove_call:
1487
1488 /* output args are dead */
1489 dead_args = 0;
1490 sync_args = 0;
1491 for(i = 0; i < nb_oargs; i++) {
1492 arg = args[i];
1493 if (dead_temps[arg]) {
1494 dead_args |= (1 << i);
1495 }
1496 if (mem_temps[arg]) {
1497 sync_args |= (1 << i);
1498 }
1499 dead_temps[arg] = 1;
1500 mem_temps[arg] = 0;
1501 }
1502
1503 if (!(call_flags & TCG_CALL_NO_READ_GLOBALS)) {
1504 /* globals should be synced to memory */
1505 memset(mem_temps, 1, s->nb_globals);
1506 }
1507 if (!(call_flags & (TCG_CALL_NO_WRITE_GLOBALS |
1508 TCG_CALL_NO_READ_GLOBALS))) {
1509 /* globals should go back to memory */
1510 memset(dead_temps, 1, s->nb_globals);
1511 }
1512
1513 /* input args are live */
1514 for(i = nb_oargs; i < nb_iargs + nb_oargs; i++) {
1515 arg = args[i];
1516 if (arg != TCG_CALL_DUMMY_ARG) {
1517 if (dead_temps[arg]) {
1518 dead_args |= (1 << i);
1519 }
1520 dead_temps[arg] = 0;
1521 }
1522 }
1523 s->op_dead_args[op_index] = dead_args;
1524 s->op_sync_args[op_index] = sync_args;
1525 }
1526 args--;
1527 }
1528 break;
1529 case INDEX_op_debug_insn_start:
1530 args -= def->nb_args;
1531 break;
1532 case INDEX_op_nopn:
1533 nb_args = args[-1];
1534 args -= nb_args;
1535 break;
1536 case INDEX_op_discard:
1537 args--;
1538 /* mark the temporary as dead */
1539 dead_temps[args[0]] = 1;
1540 mem_temps[args[0]] = 0;
1541 break;
1542 case INDEX_op_end:
1543 break;
1544
1545 case INDEX_op_add2_i32:
1546 op_new = INDEX_op_add_i32;
1547 goto do_addsub2;
1548 case INDEX_op_sub2_i32:
1549 op_new = INDEX_op_sub_i32;
1550 goto do_addsub2;
1551 case INDEX_op_add2_i64:
1552 op_new = INDEX_op_add_i64;
1553 goto do_addsub2;
1554 case INDEX_op_sub2_i64:
1555 op_new = INDEX_op_sub_i64;
1556 do_addsub2:
1557 args -= 6;
1558 nb_iargs = 4;
1559 nb_oargs = 2;
1560 /* Test if the high part of the operation is dead, but not
1561 the low part. The result can be optimized to a simple
1562 add or sub. This happens often for x86_64 guest when the
1563 cpu mode is set to 32 bit. */
1564 if (dead_temps[args[1]] && !mem_temps[args[1]]) {
1565 if (dead_temps[args[0]] && !mem_temps[args[0]]) {
1566 goto do_remove;
1567 }
1568 /* Create the single operation plus nop. */
1569 s->gen_opc_buf[op_index] = op = op_new;
1570 args[1] = args[2];
1571 args[2] = args[4];
1572 assert(s->gen_opc_buf[op_index + 1] == INDEX_op_nop);
1573 tcg_set_nop(s, s->gen_opc_buf + op_index + 1, args + 3, 3);
1574 /* Fall through and mark the single-word operation live. */
1575 nb_iargs = 2;
1576 nb_oargs = 1;
1577 }
1578 goto do_not_remove;
1579
1580 case INDEX_op_mulu2_i32:
1581 op_new = INDEX_op_mul_i32;
1582 op_new2 = INDEX_op_muluh_i32;
1583 have_op_new2 = TCG_TARGET_HAS_muluh_i32;
1584 goto do_mul2;
1585 case INDEX_op_muls2_i32:
1586 op_new = INDEX_op_mul_i32;
1587 op_new2 = INDEX_op_mulsh_i32;
1588 have_op_new2 = TCG_TARGET_HAS_mulsh_i32;
1589 goto do_mul2;
1590 case INDEX_op_mulu2_i64:
1591 op_new = INDEX_op_mul_i64;
1592 op_new2 = INDEX_op_muluh_i64;
1593 have_op_new2 = TCG_TARGET_HAS_muluh_i64;
1594 goto do_mul2;
1595 case INDEX_op_muls2_i64:
1596 op_new = INDEX_op_mul_i64;
1597 op_new2 = INDEX_op_mulsh_i64;
1598 have_op_new2 = TCG_TARGET_HAS_mulsh_i64;
1599 goto do_mul2;
1600 do_mul2:
1601 args -= 4;
1602 nb_iargs = 2;
1603 nb_oargs = 2;
1604 if (dead_temps[args[1]] && !mem_temps[args[1]]) {
1605 if (dead_temps[args[0]] && !mem_temps[args[0]]) {
1606 /* Both parts of the operation are dead. */
1607 goto do_remove;
1608 }
1609 /* The high part of the operation is dead; generate the low. */
1610 s->gen_opc_buf[op_index] = op = op_new;
1611 args[1] = args[2];
1612 args[2] = args[3];
1613 } else if (have_op_new2 && dead_temps[args[0]]
1614 && !mem_temps[args[0]]) {
1615 /* The low part of the operation is dead; generate the high. */
1616 s->gen_opc_buf[op_index] = op = op_new2;
1617 args[0] = args[1];
1618 args[1] = args[2];
1619 args[2] = args[3];
1620 } else {
1621 goto do_not_remove;
1622 }
1623 assert(s->gen_opc_buf[op_index + 1] == INDEX_op_nop);
1624 tcg_set_nop(s, s->gen_opc_buf + op_index + 1, args + 3, 1);
1625 /* Mark the single-word operation live. */
1626 nb_oargs = 1;
1627 goto do_not_remove;
1628
1629 default:
1630 /* XXX: optimize by hardcoding common cases (e.g. triadic ops) */
1631 args -= def->nb_args;
1632 nb_iargs = def->nb_iargs;
1633 nb_oargs = def->nb_oargs;
1634
1635 /* Test if the operation can be removed because all
1636 its outputs are dead. We assume that nb_oargs == 0
1637 implies side effects */
1638 if (!(def->flags & TCG_OPF_SIDE_EFFECTS) && nb_oargs != 0) {
1639 for(i = 0; i < nb_oargs; i++) {
1640 arg = args[i];
1641 if (!dead_temps[arg] || mem_temps[arg]) {
1642 goto do_not_remove;
1643 }
1644 }
1645 do_remove:
1646 tcg_set_nop(s, s->gen_opc_buf + op_index, args, def->nb_args);
1647 #ifdef CONFIG_PROFILER
1648 s->del_op_count++;
1649 #endif
1650 } else {
1651 do_not_remove:
1652
1653 /* output args are dead */
1654 dead_args = 0;
1655 sync_args = 0;
1656 for(i = 0; i < nb_oargs; i++) {
1657 arg = args[i];
1658 if (dead_temps[arg]) {
1659 dead_args |= (1 << i);
1660 }
1661 if (mem_temps[arg]) {
1662 sync_args |= (1 << i);
1663 }
1664 dead_temps[arg] = 1;
1665 mem_temps[arg] = 0;
1666 }
1667
1668 /* if end of basic block, update */
1669 if (def->flags & TCG_OPF_BB_END) {
1670 tcg_la_bb_end(s, dead_temps, mem_temps);
1671 } else if (def->flags & TCG_OPF_SIDE_EFFECTS) {
1672 /* globals should be synced to memory */
1673 memset(mem_temps, 1, s->nb_globals);
1674 }
1675
1676 /* input args are live */
1677 for(i = nb_oargs; i < nb_oargs + nb_iargs; i++) {
1678 arg = args[i];
1679 if (dead_temps[arg]) {
1680 dead_args |= (1 << i);
1681 }
1682 dead_temps[arg] = 0;
1683 }
1684 s->op_dead_args[op_index] = dead_args;
1685 s->op_sync_args[op_index] = sync_args;
1686 }
1687 break;
1688 }
1689 op_index--;
1690 }
1691
1692 if (args != s->gen_opparam_buf) {
1693 tcg_abort();
1694 }
1695 }
1696 #else
1697 /* dummy liveness analysis */
tcg_liveness_analysis(TCGContext * s)1698 static void tcg_liveness_analysis(TCGContext *s)
1699 {
1700 int nb_ops;
1701 nb_ops = s->gen_opc_ptr - s->gen_opc_buf;
1702
1703 s->op_dead_args = tcg_malloc(nb_ops * sizeof(uint16_t));
1704 memset(s->op_dead_args, 0, nb_ops * sizeof(uint16_t));
1705 s->op_sync_args = tcg_malloc(nb_ops * sizeof(uint8_t));
1706 memset(s->op_sync_args, 0, nb_ops * sizeof(uint8_t));
1707 }
1708 #endif
1709
1710 #ifndef NDEBUG
dump_regs(TCGContext * s)1711 static void dump_regs(TCGContext *s)
1712 {
1713 TCGTemp *ts;
1714 int i;
1715 char buf[64];
1716
1717 for(i = 0; i < s->nb_temps; i++) {
1718 ts = &s->temps[i];
1719 printf(" %10s: ", tcg_get_arg_str_idx(s, buf, sizeof(buf), i));
1720 switch(ts->val_type) {
1721 case TEMP_VAL_REG:
1722 printf("%s", tcg_target_reg_names[ts->reg]);
1723 break;
1724 case TEMP_VAL_MEM:
1725 printf("%d(%s)", (int)ts->mem_offset, tcg_target_reg_names[ts->mem_reg]);
1726 break;
1727 case TEMP_VAL_CONST:
1728 printf("$0x%" TCG_PRIlx, ts->val);
1729 break;
1730 case TEMP_VAL_DEAD:
1731 printf("D");
1732 break;
1733 default:
1734 printf("???");
1735 break;
1736 }
1737 printf("\n");
1738 }
1739
1740 for(i = 0; i < TCG_TARGET_NB_REGS; i++) {
1741 if (s->reg_to_temp[i] >= 0) {
1742 printf("%s: %s\n",
1743 tcg_target_reg_names[i],
1744 tcg_get_arg_str_idx(s, buf, sizeof(buf), s->reg_to_temp[i]));
1745 }
1746 }
1747 }
1748
check_regs(TCGContext * s)1749 static void check_regs(TCGContext *s)
1750 {
1751 int reg, k;
1752 TCGTemp *ts;
1753 char buf[64];
1754
1755 for(reg = 0; reg < TCG_TARGET_NB_REGS; reg++) {
1756 k = s->reg_to_temp[reg];
1757 if (k >= 0) {
1758 ts = &s->temps[k];
1759 if (ts->val_type != TEMP_VAL_REG ||
1760 ts->reg != reg) {
1761 printf("Inconsistency for register %s:\n",
1762 tcg_target_reg_names[reg]);
1763 goto fail;
1764 }
1765 }
1766 }
1767 for(k = 0; k < s->nb_temps; k++) {
1768 ts = &s->temps[k];
1769 if (ts->val_type == TEMP_VAL_REG &&
1770 !ts->fixed_reg &&
1771 s->reg_to_temp[ts->reg] != k) {
1772 printf("Inconsistency for temp %s:\n",
1773 tcg_get_arg_str_idx(s, buf, sizeof(buf), k));
1774 fail:
1775 printf("reg state:\n");
1776 dump_regs(s);
1777 tcg_abort();
1778 }
1779 }
1780 }
1781 #endif
1782
temp_allocate_frame(TCGContext * s,int temp)1783 static void temp_allocate_frame(TCGContext *s, int temp)
1784 {
1785 TCGTemp *ts;
1786 ts = &s->temps[temp];
1787 #if !(defined(__sparc__) && TCG_TARGET_REG_BITS == 64)
1788 /* Sparc64 stack is accessed with offset of 2047 */
1789 s->current_frame_offset = (s->current_frame_offset +
1790 (tcg_target_long)sizeof(tcg_target_long) - 1) &
1791 ~(sizeof(tcg_target_long) - 1);
1792 #endif
1793 if (s->current_frame_offset + (tcg_target_long)sizeof(tcg_target_long) >
1794 s->frame_end) {
1795 tcg_abort();
1796 }
1797 ts->mem_offset = s->current_frame_offset;
1798 ts->mem_reg = s->frame_reg;
1799 ts->mem_allocated = 1;
1800 s->current_frame_offset += sizeof(tcg_target_long);
1801 }
1802
1803 /* sync register 'reg' by saving it to the corresponding temporary */
tcg_reg_sync(TCGContext * s,int reg)1804 static inline void tcg_reg_sync(TCGContext *s, int reg)
1805 {
1806 TCGTemp *ts;
1807 int temp;
1808
1809 temp = s->reg_to_temp[reg];
1810 ts = &s->temps[temp];
1811 assert(ts->val_type == TEMP_VAL_REG);
1812 if (!ts->mem_coherent && !ts->fixed_reg) {
1813 if (!ts->mem_allocated) {
1814 temp_allocate_frame(s, temp);
1815 }
1816 tcg_out_st(s, ts->type, reg, ts->mem_reg, ts->mem_offset);
1817 }
1818 ts->mem_coherent = 1;
1819 }
1820
1821 /* free register 'reg' by spilling the corresponding temporary if necessary */
tcg_reg_free(TCGContext * s,int reg)1822 static void tcg_reg_free(TCGContext *s, int reg)
1823 {
1824 int temp;
1825
1826 temp = s->reg_to_temp[reg];
1827 if (temp != -1) {
1828 tcg_reg_sync(s, reg);
1829 s->temps[temp].val_type = TEMP_VAL_MEM;
1830 s->reg_to_temp[reg] = -1;
1831 }
1832 }
1833
1834 /* Allocate a register belonging to reg1 & ~reg2 */
tcg_reg_alloc(TCGContext * s,TCGRegSet reg1,TCGRegSet reg2)1835 static int tcg_reg_alloc(TCGContext *s, TCGRegSet reg1, TCGRegSet reg2)
1836 {
1837 int i, reg;
1838 TCGRegSet reg_ct;
1839
1840 tcg_regset_andnot(reg_ct, reg1, reg2);
1841
1842 /* first try free registers */
1843 for(i = 0; i < ARRAY_SIZE(tcg_target_reg_alloc_order); i++) {
1844 reg = tcg_target_reg_alloc_order[i];
1845 if (tcg_regset_test_reg(reg_ct, reg) && s->reg_to_temp[reg] == -1)
1846 return reg;
1847 }
1848
1849 /* XXX: do better spill choice */
1850 for(i = 0; i < ARRAY_SIZE(tcg_target_reg_alloc_order); i++) {
1851 reg = tcg_target_reg_alloc_order[i];
1852 if (tcg_regset_test_reg(reg_ct, reg)) {
1853 tcg_reg_free(s, reg);
1854 return reg;
1855 }
1856 }
1857
1858 tcg_abort();
1859 }
1860
1861 /* mark a temporary as dead. */
temp_dead(TCGContext * s,int temp)1862 static inline void temp_dead(TCGContext *s, int temp)
1863 {
1864 TCGTemp *ts;
1865
1866 ts = &s->temps[temp];
1867 if (!ts->fixed_reg) {
1868 if (ts->val_type == TEMP_VAL_REG) {
1869 s->reg_to_temp[ts->reg] = -1;
1870 }
1871 if (temp < s->nb_globals || ts->temp_local) {
1872 ts->val_type = TEMP_VAL_MEM;
1873 } else {
1874 ts->val_type = TEMP_VAL_DEAD;
1875 }
1876 }
1877 }
1878
1879 /* sync a temporary to memory. 'allocated_regs' is used in case a
1880 temporary registers needs to be allocated to store a constant. */
temp_sync(TCGContext * s,int temp,TCGRegSet allocated_regs)1881 static inline void temp_sync(TCGContext *s, int temp, TCGRegSet allocated_regs)
1882 {
1883 TCGTemp *ts;
1884
1885 ts = &s->temps[temp];
1886 if (!ts->fixed_reg) {
1887 switch(ts->val_type) {
1888 case TEMP_VAL_CONST:
1889 ts->reg = tcg_reg_alloc(s, tcg_target_available_regs[ts->type],
1890 allocated_regs);
1891 ts->val_type = TEMP_VAL_REG;
1892 s->reg_to_temp[ts->reg] = temp;
1893 ts->mem_coherent = 0;
1894 tcg_out_movi(s, ts->type, ts->reg, ts->val);
1895 /* fallthrough*/
1896 case TEMP_VAL_REG:
1897 tcg_reg_sync(s, ts->reg);
1898 break;
1899 case TEMP_VAL_DEAD:
1900 case TEMP_VAL_MEM:
1901 break;
1902 default:
1903 tcg_abort();
1904 }
1905 }
1906 }
1907
1908 /* save a temporary to memory. 'allocated_regs' is used in case a
1909 temporary registers needs to be allocated to store a constant. */
temp_save(TCGContext * s,int temp,TCGRegSet allocated_regs)1910 static inline void temp_save(TCGContext *s, int temp, TCGRegSet allocated_regs)
1911 {
1912 #ifdef USE_LIVENESS_ANALYSIS
1913 /* The liveness analysis already ensures that globals are back
1914 in memory. Keep an assert for safety. */
1915 assert(s->temps[temp].val_type == TEMP_VAL_MEM || s->temps[temp].fixed_reg);
1916 #else
1917 temp_sync(s, temp, allocated_regs);
1918 temp_dead(s, temp);
1919 #endif
1920 }
1921
1922 /* save globals to their canonical location and assume they can be
1923 modified be the following code. 'allocated_regs' is used in case a
1924 temporary registers needs to be allocated to store a constant. */
save_globals(TCGContext * s,TCGRegSet allocated_regs)1925 static void save_globals(TCGContext *s, TCGRegSet allocated_regs)
1926 {
1927 int i;
1928
1929 for(i = 0; i < s->nb_globals; i++) {
1930 temp_save(s, i, allocated_regs);
1931 }
1932 }
1933
1934 /* sync globals to their canonical location and assume they can be
1935 read by the following code. 'allocated_regs' is used in case a
1936 temporary registers needs to be allocated to store a constant. */
sync_globals(TCGContext * s,TCGRegSet allocated_regs)1937 static void sync_globals(TCGContext *s, TCGRegSet allocated_regs)
1938 {
1939 int i;
1940
1941 for (i = 0; i < s->nb_globals; i++) {
1942 #ifdef USE_LIVENESS_ANALYSIS
1943 assert(s->temps[i].val_type != TEMP_VAL_REG || s->temps[i].fixed_reg ||
1944 s->temps[i].mem_coherent);
1945 #else
1946 temp_sync(s, i, allocated_regs);
1947 #endif
1948 }
1949 }
1950
1951 /* at the end of a basic block, we assume all temporaries are dead and
1952 all globals are stored at their canonical location. */
tcg_reg_alloc_bb_end(TCGContext * s,TCGRegSet allocated_regs)1953 static void tcg_reg_alloc_bb_end(TCGContext *s, TCGRegSet allocated_regs)
1954 {
1955 TCGTemp *ts;
1956 int i;
1957
1958 for(i = s->nb_globals; i < s->nb_temps; i++) {
1959 ts = &s->temps[i];
1960 if (ts->temp_local) {
1961 temp_save(s, i, allocated_regs);
1962 } else {
1963 #ifdef USE_LIVENESS_ANALYSIS
1964 /* The liveness analysis already ensures that temps are dead.
1965 Keep an assert for safety. */
1966 assert(ts->val_type == TEMP_VAL_DEAD);
1967 #else
1968 temp_dead(s, i);
1969 #endif
1970 }
1971 }
1972
1973 save_globals(s, allocated_regs);
1974 }
1975
1976 #define IS_DEAD_ARG(n) ((dead_args >> (n)) & 1)
1977 #define NEED_SYNC_ARG(n) ((sync_args >> (n)) & 1)
1978
tcg_reg_alloc_movi(TCGContext * s,const TCGArg * args,uint16_t dead_args,uint8_t sync_args)1979 static void tcg_reg_alloc_movi(TCGContext *s, const TCGArg *args,
1980 uint16_t dead_args, uint8_t sync_args)
1981 {
1982 TCGTemp *ots;
1983 tcg_target_ulong val;
1984
1985 ots = &s->temps[args[0]];
1986 val = args[1];
1987
1988 if (ots->fixed_reg) {
1989 /* for fixed registers, we do not do any constant
1990 propagation */
1991 tcg_out_movi(s, ots->type, ots->reg, val);
1992 } else {
1993 /* The movi is not explicitly generated here */
1994 if (ots->val_type == TEMP_VAL_REG)
1995 s->reg_to_temp[ots->reg] = -1;
1996 ots->val_type = TEMP_VAL_CONST;
1997 ots->val = val;
1998 }
1999 if (NEED_SYNC_ARG(0)) {
2000 temp_sync(s, args[0], s->reserved_regs);
2001 }
2002 if (IS_DEAD_ARG(0)) {
2003 temp_dead(s, args[0]);
2004 }
2005 }
2006
tcg_reg_alloc_mov(TCGContext * s,const TCGOpDef * def,const TCGArg * args,uint16_t dead_args,uint8_t sync_args)2007 static void tcg_reg_alloc_mov(TCGContext *s, const TCGOpDef *def,
2008 const TCGArg *args, uint16_t dead_args,
2009 uint8_t sync_args)
2010 {
2011 TCGRegSet allocated_regs;
2012 TCGTemp *ts, *ots;
2013 const TCGArgConstraint *arg_ct, *oarg_ct;
2014
2015 tcg_regset_set(allocated_regs, s->reserved_regs);
2016 ots = &s->temps[args[0]];
2017 ts = &s->temps[args[1]];
2018 oarg_ct = &def->args_ct[0];
2019 arg_ct = &def->args_ct[1];
2020
2021 /* If the source value is not in a register, and we're going to be
2022 forced to have it in a register in order to perform the copy,
2023 then copy the SOURCE value into its own register first. That way
2024 we don't have to reload SOURCE the next time it is used. */
2025 if (((NEED_SYNC_ARG(0) || ots->fixed_reg) && ts->val_type != TEMP_VAL_REG)
2026 || ts->val_type == TEMP_VAL_MEM) {
2027 ts->reg = tcg_reg_alloc(s, arg_ct->u.regs, allocated_regs);
2028 if (ts->val_type == TEMP_VAL_MEM) {
2029 tcg_out_ld(s, ts->type, ts->reg, ts->mem_reg, ts->mem_offset);
2030 ts->mem_coherent = 1;
2031 } else if (ts->val_type == TEMP_VAL_CONST) {
2032 tcg_out_movi(s, ts->type, ts->reg, ts->val);
2033 }
2034 s->reg_to_temp[ts->reg] = args[1];
2035 ts->val_type = TEMP_VAL_REG;
2036 }
2037
2038 if (IS_DEAD_ARG(0) && !ots->fixed_reg) {
2039 /* mov to a non-saved dead register makes no sense (even with
2040 liveness analysis disabled). */
2041 assert(NEED_SYNC_ARG(0));
2042 /* The code above should have moved the temp to a register. */
2043 assert(ts->val_type == TEMP_VAL_REG);
2044 if (!ots->mem_allocated) {
2045 temp_allocate_frame(s, args[0]);
2046 }
2047 tcg_out_st(s, ots->type, ts->reg, ots->mem_reg, ots->mem_offset);
2048 if (IS_DEAD_ARG(1)) {
2049 temp_dead(s, args[1]);
2050 }
2051 temp_dead(s, args[0]);
2052 } else if (ts->val_type == TEMP_VAL_CONST) {
2053 /* propagate constant */
2054 if (ots->val_type == TEMP_VAL_REG) {
2055 s->reg_to_temp[ots->reg] = -1;
2056 }
2057 ots->val_type = TEMP_VAL_CONST;
2058 ots->val = ts->val;
2059 } else {
2060 /* The code in the first if block should have moved the
2061 temp to a register. */
2062 assert(ts->val_type == TEMP_VAL_REG);
2063 if (IS_DEAD_ARG(1) && !ts->fixed_reg && !ots->fixed_reg) {
2064 /* the mov can be suppressed */
2065 if (ots->val_type == TEMP_VAL_REG) {
2066 s->reg_to_temp[ots->reg] = -1;
2067 }
2068 ots->reg = ts->reg;
2069 temp_dead(s, args[1]);
2070 } else {
2071 if (ots->val_type != TEMP_VAL_REG) {
2072 /* When allocating a new register, make sure to not spill the
2073 input one. */
2074 tcg_regset_set_reg(allocated_regs, ts->reg);
2075 ots->reg = tcg_reg_alloc(s, oarg_ct->u.regs, allocated_regs);
2076 }
2077 tcg_out_mov(s, ots->type, ots->reg, ts->reg);
2078 }
2079 ots->val_type = TEMP_VAL_REG;
2080 ots->mem_coherent = 0;
2081 s->reg_to_temp[ots->reg] = args[0];
2082 if (NEED_SYNC_ARG(0)) {
2083 tcg_reg_sync(s, ots->reg);
2084 }
2085 }
2086 }
2087
tcg_reg_alloc_op(TCGContext * s,const TCGOpDef * def,TCGOpcode opc,const TCGArg * args,uint16_t dead_args,uint8_t sync_args)2088 static void tcg_reg_alloc_op(TCGContext *s,
2089 const TCGOpDef *def, TCGOpcode opc,
2090 const TCGArg *args, uint16_t dead_args,
2091 uint8_t sync_args)
2092 {
2093 TCGRegSet allocated_regs;
2094 int i, k, nb_iargs, nb_oargs, reg;
2095 TCGArg arg;
2096 const TCGArgConstraint *arg_ct;
2097 TCGTemp *ts;
2098 TCGArg new_args[TCG_MAX_OP_ARGS];
2099 int const_args[TCG_MAX_OP_ARGS];
2100
2101 nb_oargs = def->nb_oargs;
2102 nb_iargs = def->nb_iargs;
2103
2104 /* copy constants */
2105 memcpy(new_args + nb_oargs + nb_iargs,
2106 args + nb_oargs + nb_iargs,
2107 sizeof(TCGArg) * def->nb_cargs);
2108
2109 /* satisfy input constraints */
2110 tcg_regset_set(allocated_regs, s->reserved_regs);
2111 for(k = 0; k < nb_iargs; k++) {
2112 i = def->sorted_args[nb_oargs + k];
2113 arg = args[i];
2114 arg_ct = &def->args_ct[i];
2115 ts = &s->temps[arg];
2116 if (ts->val_type == TEMP_VAL_MEM) {
2117 reg = tcg_reg_alloc(s, arg_ct->u.regs, allocated_regs);
2118 tcg_out_ld(s, ts->type, reg, ts->mem_reg, ts->mem_offset);
2119 ts->val_type = TEMP_VAL_REG;
2120 ts->reg = reg;
2121 ts->mem_coherent = 1;
2122 s->reg_to_temp[reg] = arg;
2123 } else if (ts->val_type == TEMP_VAL_CONST) {
2124 if (tcg_target_const_match(ts->val, arg_ct)) {
2125 /* constant is OK for instruction */
2126 const_args[i] = 1;
2127 new_args[i] = ts->val;
2128 goto iarg_end;
2129 } else {
2130 /* need to move to a register */
2131 reg = tcg_reg_alloc(s, arg_ct->u.regs, allocated_regs);
2132 tcg_out_movi(s, ts->type, reg, ts->val);
2133 ts->val_type = TEMP_VAL_REG;
2134 ts->reg = reg;
2135 ts->mem_coherent = 0;
2136 s->reg_to_temp[reg] = arg;
2137 }
2138 }
2139 assert(ts->val_type == TEMP_VAL_REG);
2140 if (arg_ct->ct & TCG_CT_IALIAS) {
2141 if (ts->fixed_reg) {
2142 /* if fixed register, we must allocate a new register
2143 if the alias is not the same register */
2144 if (arg != args[arg_ct->alias_index])
2145 goto allocate_in_reg;
2146 } else {
2147 /* if the input is aliased to an output and if it is
2148 not dead after the instruction, we must allocate
2149 a new register and move it */
2150 if (!IS_DEAD_ARG(i)) {
2151 goto allocate_in_reg;
2152 }
2153 }
2154 }
2155 reg = ts->reg;
2156 if (tcg_regset_test_reg(arg_ct->u.regs, reg)) {
2157 /* nothing to do : the constraint is satisfied */
2158 } else {
2159 allocate_in_reg:
2160 /* allocate a new register matching the constraint
2161 and move the temporary register into it */
2162 reg = tcg_reg_alloc(s, arg_ct->u.regs, allocated_regs);
2163 tcg_out_mov(s, ts->type, reg, ts->reg);
2164 }
2165 new_args[i] = reg;
2166 const_args[i] = 0;
2167 tcg_regset_set_reg(allocated_regs, reg);
2168 iarg_end: ;
2169 }
2170
2171 /* mark dead temporaries and free the associated registers */
2172 for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) {
2173 if (IS_DEAD_ARG(i)) {
2174 temp_dead(s, args[i]);
2175 }
2176 }
2177
2178 if (def->flags & TCG_OPF_BB_END) {
2179 tcg_reg_alloc_bb_end(s, allocated_regs);
2180 } else {
2181 if (def->flags & TCG_OPF_CALL_CLOBBER) {
2182 /* XXX: permit generic clobber register list ? */
2183 for(reg = 0; reg < TCG_TARGET_NB_REGS; reg++) {
2184 if (tcg_regset_test_reg(tcg_target_call_clobber_regs, reg)) {
2185 tcg_reg_free(s, reg);
2186 }
2187 }
2188 }
2189 if (def->flags & TCG_OPF_SIDE_EFFECTS) {
2190 /* sync globals if the op has side effects and might trigger
2191 an exception. */
2192 sync_globals(s, allocated_regs);
2193 }
2194
2195 /* satisfy the output constraints */
2196 tcg_regset_set(allocated_regs, s->reserved_regs);
2197 for(k = 0; k < nb_oargs; k++) {
2198 i = def->sorted_args[k];
2199 arg = args[i];
2200 arg_ct = &def->args_ct[i];
2201 ts = &s->temps[arg];
2202 if (arg_ct->ct & TCG_CT_ALIAS) {
2203 reg = new_args[arg_ct->alias_index];
2204 } else {
2205 /* if fixed register, we try to use it */
2206 reg = ts->reg;
2207 if (ts->fixed_reg &&
2208 tcg_regset_test_reg(arg_ct->u.regs, reg)) {
2209 goto oarg_end;
2210 }
2211 reg = tcg_reg_alloc(s, arg_ct->u.regs, allocated_regs);
2212 }
2213 tcg_regset_set_reg(allocated_regs, reg);
2214 /* if a fixed register is used, then a move will be done afterwards */
2215 if (!ts->fixed_reg) {
2216 if (ts->val_type == TEMP_VAL_REG) {
2217 s->reg_to_temp[ts->reg] = -1;
2218 }
2219 ts->val_type = TEMP_VAL_REG;
2220 ts->reg = reg;
2221 /* temp value is modified, so the value kept in memory is
2222 potentially not the same */
2223 ts->mem_coherent = 0;
2224 s->reg_to_temp[reg] = arg;
2225 }
2226 oarg_end:
2227 new_args[i] = reg;
2228 }
2229 }
2230
2231 /* emit instruction */
2232 tcg_out_op(s, opc, new_args, const_args);
2233
2234 /* move the outputs in the correct register if needed */
2235 for(i = 0; i < nb_oargs; i++) {
2236 ts = &s->temps[args[i]];
2237 reg = new_args[i];
2238 if (ts->fixed_reg && ts->reg != reg) {
2239 tcg_out_mov(s, ts->type, ts->reg, reg);
2240 }
2241 if (NEED_SYNC_ARG(i)) {
2242 tcg_reg_sync(s, reg);
2243 }
2244 if (IS_DEAD_ARG(i)) {
2245 temp_dead(s, args[i]);
2246 }
2247 }
2248 }
2249
2250 #ifdef TCG_TARGET_STACK_GROWSUP
2251 #define STACK_DIR(x) (-(x))
2252 #else
2253 #define STACK_DIR(x) (x)
2254 #endif
2255
tcg_reg_alloc_call(TCGContext * s,const TCGOpDef * def,TCGOpcode opc,const TCGArg * args,uint16_t dead_args,uint8_t sync_args)2256 static int tcg_reg_alloc_call(TCGContext *s, const TCGOpDef *def,
2257 TCGOpcode opc, const TCGArg *args,
2258 uint16_t dead_args, uint8_t sync_args)
2259 {
2260 int nb_iargs, nb_oargs, flags, nb_regs, i, reg, nb_params;
2261 TCGArg arg, func_arg;
2262 TCGTemp *ts;
2263 intptr_t stack_offset;
2264 size_t call_stack_size;
2265 uintptr_t func_addr;
2266 int const_func_arg, allocate_args;
2267 TCGRegSet allocated_regs;
2268 const TCGArgConstraint *arg_ct;
2269
2270 arg = *args++;
2271
2272 nb_oargs = arg >> 16;
2273 nb_iargs = arg & 0xffff;
2274 nb_params = nb_iargs - 1;
2275
2276 flags = args[nb_oargs + nb_iargs];
2277
2278 nb_regs = ARRAY_SIZE(tcg_target_call_iarg_regs);
2279 if (nb_regs > nb_params)
2280 nb_regs = nb_params;
2281
2282 /* assign stack slots first */
2283 call_stack_size = (nb_params - nb_regs) * sizeof(tcg_target_long);
2284 call_stack_size = (call_stack_size + TCG_TARGET_STACK_ALIGN - 1) &
2285 ~(TCG_TARGET_STACK_ALIGN - 1);
2286 allocate_args = (call_stack_size > TCG_STATIC_CALL_ARGS_SIZE);
2287 if (allocate_args) {
2288 /* XXX: if more than TCG_STATIC_CALL_ARGS_SIZE is needed,
2289 preallocate call stack */
2290 tcg_abort();
2291 }
2292
2293 stack_offset = TCG_TARGET_CALL_STACK_OFFSET;
2294 for(i = nb_regs; i < nb_params; i++) {
2295 arg = args[nb_oargs + i];
2296 #ifdef TCG_TARGET_STACK_GROWSUP
2297 stack_offset -= sizeof(tcg_target_long);
2298 #endif
2299 if (arg != TCG_CALL_DUMMY_ARG) {
2300 ts = &s->temps[arg];
2301 if (ts->val_type == TEMP_VAL_REG) {
2302 tcg_out_st(s, ts->type, ts->reg, TCG_REG_CALL_STACK, stack_offset);
2303 } else if (ts->val_type == TEMP_VAL_MEM) {
2304 reg = tcg_reg_alloc(s, tcg_target_available_regs[ts->type],
2305 s->reserved_regs);
2306 /* XXX: not correct if reading values from the stack */
2307 tcg_out_ld(s, ts->type, reg, ts->mem_reg, ts->mem_offset);
2308 tcg_out_st(s, ts->type, reg, TCG_REG_CALL_STACK, stack_offset);
2309 } else if (ts->val_type == TEMP_VAL_CONST) {
2310 reg = tcg_reg_alloc(s, tcg_target_available_regs[ts->type],
2311 s->reserved_regs);
2312 /* XXX: sign extend may be needed on some targets */
2313 tcg_out_movi(s, ts->type, reg, ts->val);
2314 tcg_out_st(s, ts->type, reg, TCG_REG_CALL_STACK, stack_offset);
2315 } else {
2316 tcg_abort();
2317 }
2318 }
2319 #ifndef TCG_TARGET_STACK_GROWSUP
2320 stack_offset += sizeof(tcg_target_long);
2321 #endif
2322 }
2323
2324 /* assign input registers */
2325 tcg_regset_set(allocated_regs, s->reserved_regs);
2326 for(i = 0; i < nb_regs; i++) {
2327 arg = args[nb_oargs + i];
2328 if (arg != TCG_CALL_DUMMY_ARG) {
2329 ts = &s->temps[arg];
2330 reg = tcg_target_call_iarg_regs[i];
2331 tcg_reg_free(s, reg);
2332 if (ts->val_type == TEMP_VAL_REG) {
2333 if (ts->reg != reg) {
2334 tcg_out_mov(s, ts->type, reg, ts->reg);
2335 }
2336 } else if (ts->val_type == TEMP_VAL_MEM) {
2337 tcg_out_ld(s, ts->type, reg, ts->mem_reg, ts->mem_offset);
2338 } else if (ts->val_type == TEMP_VAL_CONST) {
2339 /* XXX: sign extend ? */
2340 tcg_out_movi(s, ts->type, reg, ts->val);
2341 } else {
2342 tcg_abort();
2343 }
2344 tcg_regset_set_reg(allocated_regs, reg);
2345 }
2346 }
2347
2348 /* assign function address */
2349 func_arg = args[nb_oargs + nb_iargs - 1];
2350 arg_ct = &def->args_ct[0];
2351 ts = &s->temps[func_arg];
2352 func_addr = ts->val;
2353 const_func_arg = 0;
2354 if (ts->val_type == TEMP_VAL_MEM) {
2355 reg = tcg_reg_alloc(s, arg_ct->u.regs, allocated_regs);
2356 tcg_out_ld(s, ts->type, reg, ts->mem_reg, ts->mem_offset);
2357 func_arg = reg;
2358 tcg_regset_set_reg(allocated_regs, reg);
2359 } else if (ts->val_type == TEMP_VAL_REG) {
2360 reg = ts->reg;
2361 if (!tcg_regset_test_reg(arg_ct->u.regs, reg)) {
2362 reg = tcg_reg_alloc(s, arg_ct->u.regs, allocated_regs);
2363 tcg_out_mov(s, ts->type, reg, ts->reg);
2364 }
2365 func_arg = reg;
2366 tcg_regset_set_reg(allocated_regs, reg);
2367 } else if (ts->val_type == TEMP_VAL_CONST) {
2368 if (tcg_target_const_match(func_addr, arg_ct)) {
2369 const_func_arg = 1;
2370 func_arg = func_addr;
2371 } else {
2372 reg = tcg_reg_alloc(s, arg_ct->u.regs, allocated_regs);
2373 tcg_out_movi(s, ts->type, reg, func_addr);
2374 func_arg = reg;
2375 tcg_regset_set_reg(allocated_regs, reg);
2376 }
2377 } else {
2378 tcg_abort();
2379 }
2380
2381
2382 /* mark dead temporaries and free the associated registers */
2383 for(i = nb_oargs; i < nb_iargs + nb_oargs; i++) {
2384 if (IS_DEAD_ARG(i)) {
2385 temp_dead(s, args[i]);
2386 }
2387 }
2388
2389 /* clobber call registers */
2390 for(reg = 0; reg < TCG_TARGET_NB_REGS; reg++) {
2391 if (tcg_regset_test_reg(tcg_target_call_clobber_regs, reg)) {
2392 tcg_reg_free(s, reg);
2393 }
2394 }
2395
2396 /* Save globals if they might be written by the helper, sync them if
2397 they might be read. */
2398 if (flags & TCG_CALL_NO_READ_GLOBALS) {
2399 /* Nothing to do */
2400 } else if (flags & TCG_CALL_NO_WRITE_GLOBALS) {
2401 sync_globals(s, allocated_regs);
2402 } else {
2403 save_globals(s, allocated_regs);
2404 }
2405
2406 tcg_out_op(s, opc, &func_arg, &const_func_arg);
2407
2408 /* assign output registers and emit moves if needed */
2409 for(i = 0; i < nb_oargs; i++) {
2410 arg = args[i];
2411 ts = &s->temps[arg];
2412 reg = tcg_target_call_oarg_regs[i];
2413 assert(s->reg_to_temp[reg] == -1);
2414 if (ts->fixed_reg) {
2415 if (ts->reg != reg) {
2416 tcg_out_mov(s, ts->type, ts->reg, reg);
2417 }
2418 } else {
2419 if (ts->val_type == TEMP_VAL_REG) {
2420 s->reg_to_temp[ts->reg] = -1;
2421 }
2422 ts->val_type = TEMP_VAL_REG;
2423 ts->reg = reg;
2424 ts->mem_coherent = 0;
2425 s->reg_to_temp[reg] = arg;
2426 if (NEED_SYNC_ARG(i)) {
2427 tcg_reg_sync(s, reg);
2428 }
2429 if (IS_DEAD_ARG(i)) {
2430 temp_dead(s, args[i]);
2431 }
2432 }
2433 }
2434
2435 return nb_iargs + nb_oargs + def->nb_cargs + 1;
2436 }
2437
2438 #ifdef CONFIG_PROFILER
2439
2440 static int64_t tcg_table_op_count[NB_OPS];
2441
dump_op_count(void)2442 static void dump_op_count(void)
2443 {
2444 int i;
2445 FILE *f;
2446 f = fopen("/tmp/op.log", "w");
2447 for(i = INDEX_op_end; i < NB_OPS; i++) {
2448 fprintf(f, "%s %" PRId64 "\n", tcg_op_defs[i].name, tcg_table_op_count[i]);
2449 }
2450 fclose(f);
2451 }
2452 #endif
2453
2454
tcg_gen_code_common(TCGContext * s,uint8_t * gen_code_buf,long search_pc)2455 static inline int tcg_gen_code_common(TCGContext *s, uint8_t *gen_code_buf,
2456 long search_pc)
2457 {
2458 TCGOpcode opc;
2459 int op_index;
2460 const TCGOpDef *def;
2461 const TCGArg *args;
2462
2463 #ifdef DEBUG_DISAS
2464 if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP))) {
2465 qemu_log("OP:\n");
2466 tcg_dump_ops(s);
2467 qemu_log("\n");
2468 }
2469 #endif
2470
2471 #ifdef CONFIG_PROFILER
2472 s->opt_time -= profile_getclock();
2473 #endif
2474
2475 #ifdef USE_TCG_OPTIMIZATIONS
2476 s->gen_opparam_ptr =
2477 tcg_optimize(s, s->gen_opc_ptr, s->gen_opparam_buf, tcg_op_defs);
2478 #endif
2479
2480 #ifdef CONFIG_PROFILER
2481 s->opt_time += profile_getclock();
2482 s->la_time -= profile_getclock();
2483 #endif
2484
2485 tcg_liveness_analysis(s);
2486
2487 #ifdef CONFIG_PROFILER
2488 s->la_time += profile_getclock();
2489 #endif
2490
2491 #ifdef DEBUG_DISAS
2492 if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP_OPT))) {
2493 qemu_log("OP after optimization and liveness analysis:\n");
2494 tcg_dump_ops(s);
2495 qemu_log("\n");
2496 }
2497 #endif
2498
2499 tcg_reg_alloc_start(s);
2500
2501 s->code_buf = gen_code_buf;
2502 s->code_ptr = gen_code_buf;
2503
2504 tcg_out_tb_init(s);
2505
2506 args = s->gen_opparam_buf;
2507 op_index = 0;
2508
2509 for(;;) {
2510 opc = s->gen_opc_buf[op_index];
2511 #ifdef CONFIG_PROFILER
2512 tcg_table_op_count[opc]++;
2513 #endif
2514 def = &tcg_op_defs[opc];
2515 #if 0
2516 printf("%s: %d %d %d\n", def->name,
2517 def->nb_oargs, def->nb_iargs, def->nb_cargs);
2518 // dump_regs(s);
2519 #endif
2520 switch(opc) {
2521 case INDEX_op_mov_i32:
2522 case INDEX_op_mov_i64:
2523 tcg_reg_alloc_mov(s, def, args, s->op_dead_args[op_index],
2524 s->op_sync_args[op_index]);
2525 break;
2526 case INDEX_op_movi_i32:
2527 case INDEX_op_movi_i64:
2528 tcg_reg_alloc_movi(s, args, s->op_dead_args[op_index],
2529 s->op_sync_args[op_index]);
2530 break;
2531 case INDEX_op_debug_insn_start:
2532 /* debug instruction */
2533 break;
2534 case INDEX_op_nop:
2535 case INDEX_op_nop1:
2536 case INDEX_op_nop2:
2537 case INDEX_op_nop3:
2538 break;
2539 case INDEX_op_nopn:
2540 args += args[0];
2541 goto next;
2542 case INDEX_op_discard:
2543 temp_dead(s, args[0]);
2544 break;
2545 case INDEX_op_set_label:
2546 tcg_reg_alloc_bb_end(s, s->reserved_regs);
2547 tcg_out_label(s, args[0], s->code_ptr);
2548 break;
2549 case INDEX_op_call:
2550 args += tcg_reg_alloc_call(s, def, opc, args,
2551 s->op_dead_args[op_index],
2552 s->op_sync_args[op_index]);
2553 goto next;
2554 case INDEX_op_end:
2555 goto the_end;
2556 default:
2557 /* Sanity check that we've not introduced any unhandled opcodes. */
2558 if (def->flags & TCG_OPF_NOT_PRESENT) {
2559 tcg_abort();
2560 }
2561 /* Note: in order to speed up the code, it would be much
2562 faster to have specialized register allocator functions for
2563 some common argument patterns */
2564 tcg_reg_alloc_op(s, def, opc, args, s->op_dead_args[op_index],
2565 s->op_sync_args[op_index]);
2566 break;
2567 }
2568 args += def->nb_args;
2569 next:
2570 if (search_pc >= 0 && search_pc < s->code_ptr - gen_code_buf) {
2571 return op_index;
2572 }
2573 op_index++;
2574 #ifndef NDEBUG
2575 check_regs(s);
2576 #endif
2577 }
2578 the_end:
2579 /* Generate TB finalization at the end of block */
2580 tcg_out_tb_finalize(s);
2581 return -1;
2582 }
2583
tcg_gen_code(TCGContext * s,uint8_t * gen_code_buf)2584 int tcg_gen_code(TCGContext *s, uint8_t *gen_code_buf)
2585 {
2586 #ifdef CONFIG_PROFILER
2587 {
2588 int n;
2589 n = (s->gen_opc_ptr - s->gen_opc_buf);
2590 s->op_count += n;
2591 if (n > s->op_count_max)
2592 s->op_count_max = n;
2593
2594 s->temp_count += s->nb_temps;
2595 if (s->nb_temps > s->temp_count_max)
2596 s->temp_count_max = s->nb_temps;
2597 }
2598 #endif
2599
2600 tcg_gen_code_common(s, gen_code_buf, -1);
2601
2602 /* flush instruction cache */
2603 flush_icache_range((uintptr_t)gen_code_buf, (uintptr_t)s->code_ptr);
2604
2605 return s->code_ptr - gen_code_buf;
2606 }
2607
2608 /* Return the index of the micro operation such as the pc after is <
2609 offset bytes from the start of the TB. The contents of gen_code_buf must
2610 not be changed, though writing the same values is ok.
2611 Return -1 if not found. */
tcg_gen_code_search_pc(TCGContext * s,uint8_t * gen_code_buf,long offset)2612 int tcg_gen_code_search_pc(TCGContext *s, uint8_t *gen_code_buf, long offset)
2613 {
2614 return tcg_gen_code_common(s, gen_code_buf, offset);
2615 }
2616
2617 #ifdef CONFIG_PROFILER
tcg_dump_info(FILE * f,fprintf_function cpu_fprintf)2618 void tcg_dump_info(FILE *f, fprintf_function cpu_fprintf)
2619 {
2620 TCGContext *s = &tcg_ctx;
2621 int64_t tot;
2622
2623 tot = s->interm_time + s->code_time;
2624 cpu_fprintf(f, "JIT cycles %" PRId64 " (%0.3f s at 2.4 GHz)\n",
2625 tot, tot / 2.4e9);
2626 cpu_fprintf(f, "translated TBs %" PRId64 " (aborted=%" PRId64 " %0.1f%%)\n",
2627 s->tb_count,
2628 s->tb_count1 - s->tb_count,
2629 s->tb_count1 ? (double)(s->tb_count1 - s->tb_count) / s->tb_count1 * 100.0 : 0);
2630 cpu_fprintf(f, "avg ops/TB %0.1f max=%d\n",
2631 s->tb_count ? (double)s->op_count / s->tb_count : 0, s->op_count_max);
2632 cpu_fprintf(f, "deleted ops/TB %0.2f\n",
2633 s->tb_count ?
2634 (double)s->del_op_count / s->tb_count : 0);
2635 cpu_fprintf(f, "avg temps/TB %0.2f max=%d\n",
2636 s->tb_count ?
2637 (double)s->temp_count / s->tb_count : 0,
2638 s->temp_count_max);
2639
2640 cpu_fprintf(f, "cycles/op %0.1f\n",
2641 s->op_count ? (double)tot / s->op_count : 0);
2642 cpu_fprintf(f, "cycles/in byte %0.1f\n",
2643 s->code_in_len ? (double)tot / s->code_in_len : 0);
2644 cpu_fprintf(f, "cycles/out byte %0.1f\n",
2645 s->code_out_len ? (double)tot / s->code_out_len : 0);
2646 if (tot == 0)
2647 tot = 1;
2648 cpu_fprintf(f, " gen_interm time %0.1f%%\n",
2649 (double)s->interm_time / tot * 100.0);
2650 cpu_fprintf(f, " gen_code time %0.1f%%\n",
2651 (double)s->code_time / tot * 100.0);
2652 cpu_fprintf(f, "optim./code time %0.1f%%\n",
2653 (double)s->opt_time / (s->code_time ? s->code_time : 1)
2654 * 100.0);
2655 cpu_fprintf(f, "liveness/code time %0.1f%%\n",
2656 (double)s->la_time / (s->code_time ? s->code_time : 1) * 100.0);
2657 cpu_fprintf(f, "cpu_restore count %" PRId64 "\n",
2658 s->restore_count);
2659 cpu_fprintf(f, " avg cycles %0.1f\n",
2660 s->restore_count ? (double)s->restore_time / s->restore_count : 0);
2661
2662 dump_op_count();
2663 }
2664 #else
tcg_dump_info(FILE * f,fprintf_function cpu_fprintf)2665 void tcg_dump_info(FILE *f, fprintf_function cpu_fprintf)
2666 {
2667 cpu_fprintf(f, "[TCG profiler not compiled]\n");
2668 }
2669 #endif
2670
2671 #ifdef ELF_HOST_MACHINE
2672 /* In order to use this feature, the backend needs to do three things:
2673
2674 (1) Define ELF_HOST_MACHINE to indicate both what value to
2675 put into the ELF image and to indicate support for the feature.
2676
2677 (2) Define tcg_register_jit. This should create a buffer containing
2678 the contents of a .debug_frame section that describes the post-
2679 prologue unwind info for the tcg machine.
2680
2681 (3) Call tcg_register_jit_int, with the constructed .debug_frame.
2682 */
2683
2684 /* Begin GDB interface. THE FOLLOWING MUST MATCH GDB DOCS. */
2685 typedef enum {
2686 JIT_NOACTION = 0,
2687 JIT_REGISTER_FN,
2688 JIT_UNREGISTER_FN
2689 } jit_actions_t;
2690
2691 struct jit_code_entry {
2692 struct jit_code_entry *next_entry;
2693 struct jit_code_entry *prev_entry;
2694 const void *symfile_addr;
2695 uint64_t symfile_size;
2696 };
2697
2698 struct jit_descriptor {
2699 uint32_t version;
2700 uint32_t action_flag;
2701 struct jit_code_entry *relevant_entry;
2702 struct jit_code_entry *first_entry;
2703 };
2704
2705 void __jit_debug_register_code(void) __attribute__((noinline));
__jit_debug_register_code(void)2706 void __jit_debug_register_code(void)
2707 {
2708 asm("");
2709 }
2710
2711 /* Must statically initialize the version, because GDB may check
2712 the version before we can set it. */
2713 struct jit_descriptor __jit_debug_descriptor = { 1, 0, 0, 0 };
2714
2715 /* End GDB interface. */
2716
find_string(const char * strtab,const char * str)2717 static int find_string(const char *strtab, const char *str)
2718 {
2719 const char *p = strtab + 1;
2720
2721 while (1) {
2722 if (strcmp(p, str) == 0) {
2723 return p - strtab;
2724 }
2725 p += strlen(p) + 1;
2726 }
2727 }
2728
tcg_register_jit_int(void * buf_ptr,size_t buf_size,void * debug_frame,size_t debug_frame_size)2729 static void tcg_register_jit_int(void *buf_ptr, size_t buf_size,
2730 void *debug_frame, size_t debug_frame_size)
2731 {
2732 struct __attribute__((packed)) DebugInfo {
2733 uint32_t len;
2734 uint16_t version;
2735 uint32_t abbrev;
2736 uint8_t ptr_size;
2737 uint8_t cu_die;
2738 uint16_t cu_lang;
2739 uintptr_t cu_low_pc;
2740 uintptr_t cu_high_pc;
2741 uint8_t fn_die;
2742 char fn_name[16];
2743 uintptr_t fn_low_pc;
2744 uintptr_t fn_high_pc;
2745 uint8_t cu_eoc;
2746 };
2747
2748 struct ElfImage {
2749 ElfW(Ehdr) ehdr;
2750 ElfW(Phdr) phdr;
2751 ElfW(Shdr) shdr[7];
2752 ElfW(Sym) sym[2];
2753 struct DebugInfo di;
2754 uint8_t da[24];
2755 char str[80];
2756 };
2757
2758 struct ElfImage *img;
2759
2760 static const struct ElfImage img_template = {
2761 .ehdr = {
2762 .e_ident[EI_MAG0] = ELFMAG0,
2763 .e_ident[EI_MAG1] = ELFMAG1,
2764 .e_ident[EI_MAG2] = ELFMAG2,
2765 .e_ident[EI_MAG3] = ELFMAG3,
2766 .e_ident[EI_CLASS] = ELF_CLASS,
2767 .e_ident[EI_DATA] = ELF_DATA,
2768 .e_ident[EI_VERSION] = EV_CURRENT,
2769 .e_type = ET_EXEC,
2770 .e_machine = ELF_HOST_MACHINE,
2771 .e_version = EV_CURRENT,
2772 .e_phoff = offsetof(struct ElfImage, phdr),
2773 .e_shoff = offsetof(struct ElfImage, shdr),
2774 .e_ehsize = sizeof(ElfW(Shdr)),
2775 .e_phentsize = sizeof(ElfW(Phdr)),
2776 .e_phnum = 1,
2777 .e_shentsize = sizeof(ElfW(Shdr)),
2778 .e_shnum = ARRAY_SIZE(img->shdr),
2779 .e_shstrndx = ARRAY_SIZE(img->shdr) - 1,
2780 #ifdef ELF_HOST_FLAGS
2781 .e_flags = ELF_HOST_FLAGS,
2782 #endif
2783 #ifdef ELF_OSABI
2784 .e_ident[EI_OSABI] = ELF_OSABI,
2785 #endif
2786 },
2787 .phdr = {
2788 .p_type = PT_LOAD,
2789 .p_flags = PF_X,
2790 },
2791 .shdr = {
2792 [0] = { .sh_type = SHT_NULL },
2793 /* Trick: The contents of code_gen_buffer are not present in
2794 this fake ELF file; that got allocated elsewhere. Therefore
2795 we mark .text as SHT_NOBITS (similar to .bss) so that readers
2796 will not look for contents. We can record any address. */
2797 [1] = { /* .text */
2798 .sh_type = SHT_NOBITS,
2799 .sh_flags = SHF_EXECINSTR | SHF_ALLOC,
2800 },
2801 [2] = { /* .debug_info */
2802 .sh_type = SHT_PROGBITS,
2803 .sh_offset = offsetof(struct ElfImage, di),
2804 .sh_size = sizeof(struct DebugInfo),
2805 },
2806 [3] = { /* .debug_abbrev */
2807 .sh_type = SHT_PROGBITS,
2808 .sh_offset = offsetof(struct ElfImage, da),
2809 .sh_size = sizeof(img->da),
2810 },
2811 [4] = { /* .debug_frame */
2812 .sh_type = SHT_PROGBITS,
2813 .sh_offset = sizeof(struct ElfImage),
2814 },
2815 [5] = { /* .symtab */
2816 .sh_type = SHT_SYMTAB,
2817 .sh_offset = offsetof(struct ElfImage, sym),
2818 .sh_size = sizeof(img->sym),
2819 .sh_info = 1,
2820 .sh_link = ARRAY_SIZE(img->shdr) - 1,
2821 .sh_entsize = sizeof(ElfW(Sym)),
2822 },
2823 [6] = { /* .strtab */
2824 .sh_type = SHT_STRTAB,
2825 .sh_offset = offsetof(struct ElfImage, str),
2826 .sh_size = sizeof(img->str),
2827 }
2828 },
2829 .sym = {
2830 [1] = { /* code_gen_buffer */
2831 .st_info = ELF_ST_INFO(STB_GLOBAL, STT_FUNC),
2832 .st_shndx = 1,
2833 }
2834 },
2835 .di = {
2836 .len = sizeof(struct DebugInfo) - 4,
2837 .version = 2,
2838 .ptr_size = sizeof(void *),
2839 .cu_die = 1,
2840 .cu_lang = 0x8001, /* DW_LANG_Mips_Assembler */
2841 .fn_die = 2,
2842 .fn_name = "code_gen_buffer"
2843 },
2844 .da = {
2845 1, /* abbrev number (the cu) */
2846 0x11, 1, /* DW_TAG_compile_unit, has children */
2847 0x13, 0x5, /* DW_AT_language, DW_FORM_data2 */
2848 0x11, 0x1, /* DW_AT_low_pc, DW_FORM_addr */
2849 0x12, 0x1, /* DW_AT_high_pc, DW_FORM_addr */
2850 0, 0, /* end of abbrev */
2851 2, /* abbrev number (the fn) */
2852 0x2e, 0, /* DW_TAG_subprogram, no children */
2853 0x3, 0x8, /* DW_AT_name, DW_FORM_string */
2854 0x11, 0x1, /* DW_AT_low_pc, DW_FORM_addr */
2855 0x12, 0x1, /* DW_AT_high_pc, DW_FORM_addr */
2856 0, 0, /* end of abbrev */
2857 0 /* no more abbrev */
2858 },
2859 .str = "\0" ".text\0" ".debug_info\0" ".debug_abbrev\0"
2860 ".debug_frame\0" ".symtab\0" ".strtab\0" "code_gen_buffer",
2861 };
2862
2863 /* We only need a single jit entry; statically allocate it. */
2864 static struct jit_code_entry one_entry;
2865
2866 uintptr_t buf = (uintptr_t)buf_ptr;
2867 size_t img_size = sizeof(struct ElfImage) + debug_frame_size;
2868
2869 img = g_malloc(img_size);
2870 *img = img_template;
2871 memcpy(img + 1, debug_frame, debug_frame_size);
2872
2873 img->phdr.p_vaddr = buf;
2874 img->phdr.p_paddr = buf;
2875 img->phdr.p_memsz = buf_size;
2876
2877 img->shdr[1].sh_name = find_string(img->str, ".text");
2878 img->shdr[1].sh_addr = buf;
2879 img->shdr[1].sh_size = buf_size;
2880
2881 img->shdr[2].sh_name = find_string(img->str, ".debug_info");
2882 img->shdr[3].sh_name = find_string(img->str, ".debug_abbrev");
2883
2884 img->shdr[4].sh_name = find_string(img->str, ".debug_frame");
2885 img->shdr[4].sh_size = debug_frame_size;
2886
2887 img->shdr[5].sh_name = find_string(img->str, ".symtab");
2888 img->shdr[6].sh_name = find_string(img->str, ".strtab");
2889
2890 img->sym[1].st_name = find_string(img->str, "code_gen_buffer");
2891 img->sym[1].st_value = buf;
2892 img->sym[1].st_size = buf_size;
2893
2894 img->di.cu_low_pc = buf;
2895 img->di.cu_high_pc = buf + buf_size;
2896 img->di.fn_low_pc = buf;
2897 img->di.fn_high_pc = buf + buf_size;
2898
2899 #ifdef DEBUG_JIT
2900 /* Enable this block to be able to debug the ELF image file creation.
2901 One can use readelf, objdump, or other inspection utilities. */
2902 {
2903 FILE *f = fopen("/tmp/qemu.jit", "w+b");
2904 if (f) {
2905 if (fwrite(img, img_size, 1, f) != img_size) {
2906 /* Avoid stupid unused return value warning for fwrite. */
2907 }
2908 fclose(f);
2909 }
2910 }
2911 #endif
2912
2913 one_entry.symfile_addr = img;
2914 one_entry.symfile_size = img_size;
2915
2916 __jit_debug_descriptor.action_flag = JIT_REGISTER_FN;
2917 __jit_debug_descriptor.relevant_entry = &one_entry;
2918 __jit_debug_descriptor.first_entry = &one_entry;
2919 __jit_debug_register_code();
2920 }
2921 #else
2922 /* No support for the feature. Provide the entry point expected by exec.c,
2923 and implement the internal function we declared earlier. */
2924
tcg_register_jit_int(void * buf,size_t size,void * debug_frame,size_t debug_frame_size)2925 static void tcg_register_jit_int(void *buf, size_t size,
2926 void *debug_frame, size_t debug_frame_size)
2927 {
2928 }
2929
tcg_register_jit(void * buf,size_t buf_size)2930 void tcg_register_jit(void *buf, size_t buf_size)
2931 {
2932 }
2933 #endif /* ELF_HOST_MACHINE */
2934