]> Git Repo - qemu.git/blame - tcg/tcg.c
projects / qemu.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
tcg/ppc: Ensure _CALL_SYSV is set for 32-bit ELF
[qemu.git] / tcg / tcg.c
CommitLineData
c896fe29
FB		 1/*
2 * Tiny Code Generator for QEMU
3 *
4 * Copyright (c) 2008 Fabrice Bellard
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a copy
7 * of this software and associated documentation files (the "Software"), to deal
8 * in the Software without restriction, including without limitation the rights
9 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10 * copies of the Software, and to permit persons to whom the Software is
11 * furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
22 * THE SOFTWARE.
23 */
24
c896fe29 25/* define it to use liveness analysis (better code) */
8f2e8c07 26#define USE_TCG_OPTIMIZATIONS
c896fe29 27
757e725b 28#include "qemu/osdep.h"
cca82982 29
813da627
RH		 30/* Define to jump the ELF file used to communicate with GDB. */
31#undef DEBUG_JIT
32
72fd2efb 33#include "qemu/error-report.h"
f348b6d1 34#include "qemu/cutils.h"
1de7afc9 35#include "qemu/host-utils.h"
d4c51a0a 36#include "qemu/qemu-print.h"
1de7afc9 37#include "qemu/timer.h"
084cfca1 38#include "qemu/cacheflush.h"
c896fe29 39
c5d3c498 40/* Note: the long term plan is to reduce the dependencies on the QEMU
c896fe29
FB		 41 CPU definitions. Currently they are used for qemu_ld/st
42 instructions */
43#define NO_CPU_IO_DEFS
c896fe29 44
63c91552 45#include "exec/exec-all.h"
dcb32f1d 46#include "tcg/tcg-op.h"
813da627 47
edee2579 48#if UINTPTR_MAX == UINT32_MAX
813da627 49# define ELF_CLASS ELFCLASS32
edee2579
RH		 50#else
51# define ELF_CLASS ELFCLASS64
813da627
RH		 52#endif
53#ifdef HOST_WORDS_BIGENDIAN
54# define ELF_DATA ELFDATA2MSB
55#else
56# define ELF_DATA ELFDATA2LSB
57#endif
58
c896fe29 59#include "elf.h"
508127e2 60#include "exec/log.h"
5ff7258c 61#include "tcg-internal.h"
c896fe29 62
22f15579
RH		 63#ifdef CONFIG_TCG_INTERPRETER
64#include <ffi.h>
65#endif
66
139c1837 67/* Forward declarations for functions declared in tcg-target.c.inc and
ce151109 68 used here. */
e4d58b41
RH		 69static void tcg_target_init(TCGContext *s);
70static void tcg_target_qemu_prologue(TCGContext *s);
6ac17786 71static bool patch_reloc(tcg_insn_unit *code_ptr, int type,
2ba7fae2 72 intptr_t value, intptr_t addend);
c896fe29 73
497a22eb
RH		 74/* The CIE and FDE header definitions will be common to all hosts. */
75typedef struct {
76 uint32_t len __attribute__((aligned((sizeof(void *)))));
77 uint32_t id;
78 uint8_t version;
79 char augmentation[1];
80 uint8_t code_align;
81 uint8_t data_align;
82 uint8_t return_column;
83} DebugFrameCIE;
84
85typedef struct QEMU_PACKED {
86 uint32_t len __attribute__((aligned((sizeof(void *)))));
87 uint32_t cie_offset;
edee2579
RH		 88 uintptr_t func_start;
89 uintptr_t func_len;
497a22eb
RH		 90} DebugFrameFDEHeader;
91
2c90784a
RH		 92typedef struct QEMU_PACKED {
93 DebugFrameCIE cie;
94 DebugFrameFDEHeader fde;
95} DebugFrameHeader;
96
755bf9e5 97static void tcg_register_jit_int(const void *buf, size_t size,
2c90784a
RH		 98 const void *debug_frame,
99 size_t debug_frame_size)
813da627
RH		 100 __attribute__((unused));
101
139c1837 102/* Forward declarations for functions declared and used in tcg-target.c.inc. */
2a534aff 103static void tcg_out_ld(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg1,
a05b5b9b 104 intptr_t arg2);
78113e83 105static bool tcg_out_mov(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg);
c0ad3001 106static void tcg_out_movi(TCGContext *s, TCGType type,
2a534aff 107 TCGReg ret, tcg_target_long arg);
5e8892db
MR		 108static void tcg_out_op(TCGContext *s, TCGOpcode opc,
109 const TCGArg args[TCG_MAX_OP_ARGS],
110 const int const_args[TCG_MAX_OP_ARGS]);
d2fd745f 111#if TCG_TARGET_MAYBE_vec
e7632cfa
RH		 112static bool tcg_out_dup_vec(TCGContext *s, TCGType type, unsigned vece,
113 TCGReg dst, TCGReg src);
d6ecb4a9
RH		 114static bool tcg_out_dupm_vec(TCGContext *s, TCGType type, unsigned vece,
115 TCGReg dst, TCGReg base, intptr_t offset);
4e186175
RH		 116static void tcg_out_dupi_vec(TCGContext *s, TCGType type, unsigned vece,
117 TCGReg dst, int64_t arg);
5e8892db
MR		 118static void tcg_out_vec_op(TCGContext *s, TCGOpcode opc,
119 unsigned vecl, unsigned vece,
120 const TCGArg args[TCG_MAX_OP_ARGS],
121 const int const_args[TCG_MAX_OP_ARGS]);
d2fd745f 122#else
e7632cfa
RH		 123static inline bool tcg_out_dup_vec(TCGContext *s, TCGType type, unsigned vece,
124 TCGReg dst, TCGReg src)
125{
126 g_assert_not_reached();
127}
d6ecb4a9
RH		 128static inline bool tcg_out_dupm_vec(TCGContext *s, TCGType type, unsigned vece,
129 TCGReg dst, TCGReg base, intptr_t offset)
130{
131 g_assert_not_reached();
132}
4e186175
RH		 133static inline void tcg_out_dupi_vec(TCGContext *s, TCGType type, unsigned vece,
134 TCGReg dst, int64_t arg)
e7632cfa
RH		 135{
136 g_assert_not_reached();
137}
5e8892db
MR		 138static inline void tcg_out_vec_op(TCGContext *s, TCGOpcode opc,
139 unsigned vecl, unsigned vece,
140 const TCGArg args[TCG_MAX_OP_ARGS],
141 const int const_args[TCG_MAX_OP_ARGS])
d2fd745f
RH		 142{
143 g_assert_not_reached();
144}
145#endif
2a534aff 146static void tcg_out_st(TCGContext *s, TCGType type, TCGReg arg, TCGReg arg1,
a05b5b9b 147 intptr_t arg2);
59d7c14e
RH		 148static bool tcg_out_sti(TCGContext *s, TCGType type, TCGArg val,
149 TCGReg base, intptr_t ofs);
7b7d8b2d
RH		 150#ifdef CONFIG_TCG_INTERPRETER
151static void tcg_out_call(TCGContext *s, const tcg_insn_unit *target,
152 ffi_cif *cif);
153#else
2be7d76b 154static void tcg_out_call(TCGContext *s, const tcg_insn_unit *target);
7b7d8b2d 155#endif
a4fbbd77 156static bool tcg_target_const_match(int64_t val, TCGType type, int ct);
659ef5cb 157#ifdef TCG_TARGET_NEED_LDST_LABELS
aeee05f5 158static int tcg_out_ldst_finalize(TCGContext *s);
659ef5cb 159#endif
c896fe29 160
42eb6dfc
RH		 161TCGContext tcg_init_ctx;
162__thread TCGContext *tcg_ctx;
163
5ff7258c 164TCGContext **tcg_ctxs;
0e2d61cf
RH		 165unsigned int tcg_cur_ctxs;
166unsigned int tcg_max_ctxs;
1c2adb95 167TCGv_env cpu_env = 0;
c8bc1168 168const void *tcg_code_gen_epilogue;
db0c51a3 169uintptr_t tcg_splitwx_diff;
df2cce29 170
b91ccb31
RH		 171#ifndef CONFIG_TCG_INTERPRETER
172tcg_prologue_fn *tcg_qemu_tb_exec;
173#endif
174
d2fd745f 175static TCGRegSet tcg_target_available_regs[TCG_TYPE_COUNT];
b1d8e52e 176static TCGRegSet tcg_target_call_clobber_regs;
c896fe29 177
1813e175 178#if TCG_TARGET_INSN_UNIT_SIZE == 1
4196dca6 179static __attribute__((unused)) inline void tcg_out8(TCGContext *s, uint8_t v)
c896fe29
FB		 180{
181 *s->code_ptr++ = v;
182}
183
4196dca6
PM		 184static __attribute__((unused)) inline void tcg_patch8(tcg_insn_unit *p,
185 uint8_t v)
5c53bb81 186{
1813e175 187 *p = v;
5c53bb81 188}
1813e175 189#endif
5c53bb81 190
1813e175 191#if TCG_TARGET_INSN_UNIT_SIZE <= 2
4196dca6 192static __attribute__((unused)) inline void tcg_out16(TCGContext *s, uint16_t v)
c896fe29 193{
1813e175
RH		 194 if (TCG_TARGET_INSN_UNIT_SIZE == 2) {
195 *s->code_ptr++ = v;
196 } else {
197 tcg_insn_unit *p = s->code_ptr;
198 memcpy(p, &v, sizeof(v));
199 s->code_ptr = p + (2 / TCG_TARGET_INSN_UNIT_SIZE);
200 }
c896fe29
FB		 201}
202
4196dca6
PM		 203static __attribute__((unused)) inline void tcg_patch16(tcg_insn_unit *p,
204 uint16_t v)
5c53bb81 205{
1813e175
RH		 206 if (TCG_TARGET_INSN_UNIT_SIZE == 2) {
207 *p = v;
208 } else {
209 memcpy(p, &v, sizeof(v));
210 }
5c53bb81 211}
1813e175 212#endif
5c53bb81 213
1813e175 214#if TCG_TARGET_INSN_UNIT_SIZE <= 4
4196dca6 215static __attribute__((unused)) inline void tcg_out32(TCGContext *s, uint32_t v)
c896fe29 216{
1813e175
RH		 217 if (TCG_TARGET_INSN_UNIT_SIZE == 4) {
218 *s->code_ptr++ = v;
219 } else {
220 tcg_insn_unit *p = s->code_ptr;
221 memcpy(p, &v, sizeof(v));
222 s->code_ptr = p + (4 / TCG_TARGET_INSN_UNIT_SIZE);
223 }
c896fe29
FB		 224}
225
4196dca6
PM		 226static __attribute__((unused)) inline void tcg_patch32(tcg_insn_unit *p,
227 uint32_t v)
5c53bb81 228{
1813e175
RH		 229 if (TCG_TARGET_INSN_UNIT_SIZE == 4) {
230 *p = v;
231 } else {
232 memcpy(p, &v, sizeof(v));
233 }
5c53bb81 234}
1813e175 235#endif
5c53bb81 236
1813e175 237#if TCG_TARGET_INSN_UNIT_SIZE <= 8
4196dca6 238static __attribute__((unused)) inline void tcg_out64(TCGContext *s, uint64_t v)
ac26eb69 239{
1813e175
RH		 240 if (TCG_TARGET_INSN_UNIT_SIZE == 8) {
241 *s->code_ptr++ = v;
242 } else {
243 tcg_insn_unit *p = s->code_ptr;
244 memcpy(p, &v, sizeof(v));
245 s->code_ptr = p + (8 / TCG_TARGET_INSN_UNIT_SIZE);
246 }
ac26eb69
RH		 247}
248
4196dca6
PM		 249static __attribute__((unused)) inline void tcg_patch64(tcg_insn_unit *p,
250 uint64_t v)
5c53bb81 251{
1813e175
RH		 252 if (TCG_TARGET_INSN_UNIT_SIZE == 8) {
253 *p = v;
254 } else {
255 memcpy(p, &v, sizeof(v));
256 }
5c53bb81 257}
1813e175 258#endif
5c53bb81 259
c896fe29
FB		 260/* label relocation processing */
261
1813e175 262static void tcg_out_reloc(TCGContext *s, tcg_insn_unit *code_ptr, int type,
bec16311 263 TCGLabel *l, intptr_t addend)
c896fe29 264{
7ecd02a0 265 TCGRelocation *r = tcg_malloc(sizeof(TCGRelocation));
c896fe29 266
7ecd02a0
RH		 267 r->type = type;
268 r->ptr = code_ptr;
269 r->addend = addend;
270 QSIMPLEQ_INSERT_TAIL(&l->relocs, r, next);
c896fe29
FB		 271}
272
92ab8e7d 273static void tcg_out_label(TCGContext *s, TCGLabel *l)
c896fe29 274{
eabb7b91 275 tcg_debug_assert(!l->has_value);
c896fe29 276 l->has_value = 1;
92ab8e7d 277 l->u.value_ptr = tcg_splitwx_to_rx(s->code_ptr);
c896fe29
FB		 278}
279
42a268c2 280TCGLabel *gen_new_label(void)
c896fe29 281{
b1311c4a 282 TCGContext *s = tcg_ctx;
51e3972c 283 TCGLabel *l = tcg_malloc(sizeof(TCGLabel));
c896fe29 284
7ecd02a0
RH		 285 memset(l, 0, sizeof(TCGLabel));
286 l->id = s->nb_labels++;
287 QSIMPLEQ_INIT(&l->relocs);
288
bef16ab4 289 QSIMPLEQ_INSERT_TAIL(&s->labels, l, next);
42a268c2
RH		 290
291 return l;
c896fe29
FB		 292}
293
7ecd02a0
RH		 294static bool tcg_resolve_relocs(TCGContext *s)
295{
296 TCGLabel *l;
297
298 QSIMPLEQ_FOREACH(l, &s->labels, next) {
299 TCGRelocation *r;
300 uintptr_t value = l->u.value;
301
302 QSIMPLEQ_FOREACH(r, &l->relocs, next) {
303 if (!patch_reloc(r->ptr, r->type, value, r->addend)) {
304 return false;
305 }
306 }
307 }
308 return true;
309}
310
9f754620
RH		 311static void set_jmp_reset_offset(TCGContext *s, int which)
312{
f14bed3f
RH		 313 /*
314 * We will check for overflow at the end of the opcode loop in
315 * tcg_gen_code, where we bound tcg_current_code_size to UINT16_MAX.
316 */
317 s->tb_jmp_reset_offset[which] = tcg_current_code_size(s);
9f754620
RH		 318}
319
db6b7d0c
RH		 320/* Signal overflow, starting over with fewer guest insns. */
321static void QEMU_NORETURN tcg_raise_tb_overflow(TCGContext *s)
322{
323 siglongjmp(s->jmp_trans, -2);
324}
325
4c22e840
RH		 326#define C_PFX1(P, A) P##A
327#define C_PFX2(P, A, B) P##A##_##B
328#define C_PFX3(P, A, B, C) P##A##_##B##_##C
329#define C_PFX4(P, A, B, C, D) P##A##_##B##_##C##_##D
330#define C_PFX5(P, A, B, C, D, E) P##A##_##B##_##C##_##D##_##E
331#define C_PFX6(P, A, B, C, D, E, F) P##A##_##B##_##C##_##D##_##E##_##F
332
333/* Define an enumeration for the various combinations. */
334
335#define C_O0_I1(I1) C_PFX1(c_o0_i1_, I1),
336#define C_O0_I2(I1, I2) C_PFX2(c_o0_i2_, I1, I2),
337#define C_O0_I3(I1, I2, I3) C_PFX3(c_o0_i3_, I1, I2, I3),
338#define C_O0_I4(I1, I2, I3, I4) C_PFX4(c_o0_i4_, I1, I2, I3, I4),
339
340#define C_O1_I1(O1, I1) C_PFX2(c_o1_i1_, O1, I1),
341#define C_O1_I2(O1, I1, I2) C_PFX3(c_o1_i2_, O1, I1, I2),
342#define C_O1_I3(O1, I1, I2, I3) C_PFX4(c_o1_i3_, O1, I1, I2, I3),
343#define C_O1_I4(O1, I1, I2, I3, I4) C_PFX5(c_o1_i4_, O1, I1, I2, I3, I4),
344
345#define C_N1_I2(O1, I1, I2) C_PFX3(c_n1_i2_, O1, I1, I2),
346
347#define C_O2_I1(O1, O2, I1) C_PFX3(c_o2_i1_, O1, O2, I1),
348#define C_O2_I2(O1, O2, I1, I2) C_PFX4(c_o2_i2_, O1, O2, I1, I2),
349#define C_O2_I3(O1, O2, I1, I2, I3) C_PFX5(c_o2_i3_, O1, O2, I1, I2, I3),
350#define C_O2_I4(O1, O2, I1, I2, I3, I4) C_PFX6(c_o2_i4_, O1, O2, I1, I2, I3, I4),
351
352typedef enum {
353#include "tcg-target-con-set.h"
354} TCGConstraintSetIndex;
355
356static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode);
357
358#undef C_O0_I1
359#undef C_O0_I2
360#undef C_O0_I3
361#undef C_O0_I4
362#undef C_O1_I1
363#undef C_O1_I2
364#undef C_O1_I3
365#undef C_O1_I4
366#undef C_N1_I2
367#undef C_O2_I1
368#undef C_O2_I2
369#undef C_O2_I3
370#undef C_O2_I4
371
372/* Put all of the constraint sets into an array, indexed by the enum. */
373
374#define C_O0_I1(I1) { .args_ct_str = { #I1 } },
375#define C_O0_I2(I1, I2) { .args_ct_str = { #I1, #I2 } },
376#define C_O0_I3(I1, I2, I3) { .args_ct_str = { #I1, #I2, #I3 } },
377#define C_O0_I4(I1, I2, I3, I4) { .args_ct_str = { #I1, #I2, #I3, #I4 } },
378
379#define C_O1_I1(O1, I1) { .args_ct_str = { #O1, #I1 } },
380#define C_O1_I2(O1, I1, I2) { .args_ct_str = { #O1, #I1, #I2 } },
381#define C_O1_I3(O1, I1, I2, I3) { .args_ct_str = { #O1, #I1, #I2, #I3 } },
382#define C_O1_I4(O1, I1, I2, I3, I4) { .args_ct_str = { #O1, #I1, #I2, #I3, #I4 } },
383
384#define C_N1_I2(O1, I1, I2) { .args_ct_str = { "&" #O1, #I1, #I2 } },
385
386#define C_O2_I1(O1, O2, I1) { .args_ct_str = { #O1, #O2, #I1 } },
387#define C_O2_I2(O1, O2, I1, I2) { .args_ct_str = { #O1, #O2, #I1, #I2 } },
388#define C_O2_I3(O1, O2, I1, I2, I3) { .args_ct_str = { #O1, #O2, #I1, #I2, #I3 } },
389#define C_O2_I4(O1, O2, I1, I2, I3, I4) { .args_ct_str = { #O1, #O2, #I1, #I2, #I3, #I4 } },
390
391static const TCGTargetOpDef constraint_sets[] = {
392#include "tcg-target-con-set.h"
393};
394
395
396#undef C_O0_I1
397#undef C_O0_I2
398#undef C_O0_I3
399#undef C_O0_I4
400#undef C_O1_I1
401#undef C_O1_I2
402#undef C_O1_I3
403#undef C_O1_I4
404#undef C_N1_I2
405#undef C_O2_I1
406#undef C_O2_I2
407#undef C_O2_I3
408#undef C_O2_I4
409
410/* Expand the enumerator to be returned from tcg_target_op_def(). */
411
412#define C_O0_I1(I1) C_PFX1(c_o0_i1_, I1)
413#define C_O0_I2(I1, I2) C_PFX2(c_o0_i2_, I1, I2)
414#define C_O0_I3(I1, I2, I3) C_PFX3(c_o0_i3_, I1, I2, I3)
415#define C_O0_I4(I1, I2, I3, I4) C_PFX4(c_o0_i4_, I1, I2, I3, I4)
416
417#define C_O1_I1(O1, I1) C_PFX2(c_o1_i1_, O1, I1)
418#define C_O1_I2(O1, I1, I2) C_PFX3(c_o1_i2_, O1, I1, I2)
419#define C_O1_I3(O1, I1, I2, I3) C_PFX4(c_o1_i3_, O1, I1, I2, I3)
420#define C_O1_I4(O1, I1, I2, I3, I4) C_PFX5(c_o1_i4_, O1, I1, I2, I3, I4)
421
422#define C_N1_I2(O1, I1, I2) C_PFX3(c_n1_i2_, O1, I1, I2)
423
424#define C_O2_I1(O1, O2, I1) C_PFX3(c_o2_i1_, O1, O2, I1)
425#define C_O2_I2(O1, O2, I1, I2) C_PFX4(c_o2_i2_, O1, O2, I1, I2)
426#define C_O2_I3(O1, O2, I1, I2, I3) C_PFX5(c_o2_i3_, O1, O2, I1, I2, I3)
427#define C_O2_I4(O1, O2, I1, I2, I3, I4) C_PFX6(c_o2_i4_, O1, O2, I1, I2, I3, I4)
428
139c1837 429#include "tcg-target.c.inc"
c896fe29 430
38b47b19
EC		 431static void alloc_tcg_plugin_context(TCGContext *s)
432{
433#ifdef CONFIG_PLUGIN
434 s->plugin_tb = g_new0(struct qemu_plugin_tb, 1);
435 s->plugin_tb->insns =
436 g_ptr_array_new_with_free_func(qemu_plugin_insn_cleanup_fn);
437#endif
438}
439
3468b59e
EC		 440/*
441 * All TCG threads except the parent (i.e. the one that called tcg_context_init
442 * and registered the target's TCG globals) must register with this function
443 * before initiating translation.
444 *
445 * In user-mode we just point tcg_ctx to tcg_init_ctx. See the documentation
446 * of tcg_region_init() for the reasoning behind this.
447 *
448 * In softmmu each caller registers its context in tcg_ctxs[]. Note that in
449 * softmmu tcg_ctxs[] does not track tcg_ctx_init, since the initial context
450 * is not used anymore for translation once this function is called.
451 *
452 * Not tracking tcg_init_ctx in tcg_ctxs[] in softmmu keeps code that iterates
453 * over the array (e.g. tcg_code_size() the same for both softmmu and user-mode.
454 */
455#ifdef CONFIG_USER_ONLY
456void tcg_register_thread(void)
457{
458 tcg_ctx = &tcg_init_ctx;
459}
460#else
461void tcg_register_thread(void)
462{
463 TCGContext *s = g_malloc(sizeof(*s));
464 unsigned int i, n;
3468b59e
EC		 465
466 *s = tcg_init_ctx;
467
468 /* Relink mem_base. */
469 for (i = 0, n = tcg_init_ctx.nb_globals; i < n; ++i) {
470 if (tcg_init_ctx.temps[i].mem_base) {
471 ptrdiff_t b = tcg_init_ctx.temps[i].mem_base - tcg_init_ctx.temps;
472 tcg_debug_assert(b >= 0 && b < n);
473 s->temps[i].mem_base = &s->temps[b];
474 }
475 }
476
477 /* Claim an entry in tcg_ctxs */
0e2d61cf
RH		 478 n = qatomic_fetch_inc(&tcg_cur_ctxs);
479 g_assert(n < tcg_max_ctxs);
d73415a3 480 qatomic_set(&tcg_ctxs[n], s);
3468b59e 481
38b47b19
EC		 482 if (n > 0) {
483 alloc_tcg_plugin_context(s);
bf042e8e 484 tcg_region_initial_alloc(s);
38b47b19
EC		 485 }
486
3468b59e 487 tcg_ctx = s;
e8feb96f 488}
3468b59e 489#endif /* !CONFIG_USER_ONLY */
e8feb96f 490
c896fe29
FB		 491/* pool based memory allocation */
492void *tcg_malloc_internal(TCGContext *s, int size)
493{
494 TCGPool *p;
495 int pool_size;
496
497 if (size > TCG_POOL_CHUNK_SIZE) {
498 /* big malloc: insert a new pool (XXX: could optimize) */
7267c094 499 p = g_malloc(sizeof(TCGPool) + size);
c896fe29 500 p->size = size;
4055299e
KB		 501 p->next = s->pool_first_large;
502 s->pool_first_large = p;
503 return p->data;
c896fe29
FB		 504 } else {
505 p = s->pool_current;
506 if (!p) {
507 p = s->pool_first;
508 if (!p)
509 goto new_pool;
510 } else {
511 if (!p->next) {
512 new_pool:
513 pool_size = TCG_POOL_CHUNK_SIZE;
7267c094 514 p = g_malloc(sizeof(TCGPool) + pool_size);
c896fe29
FB		 515 p->size = pool_size;
516 p->next = NULL;
517 if (s->pool_current)
518 s->pool_current->next = p;
519 else
520 s->pool_first = p;
521 } else {
522 p = p->next;
523 }
524 }
525 }
526 s->pool_current = p;
527 s->pool_cur = p->data + size;
528 s->pool_end = p->data + p->size;
529 return p->data;
530}
531
532void tcg_pool_reset(TCGContext *s)
533{
4055299e
KB		 534 TCGPool *p, *t;
535 for (p = s->pool_first_large; p; p = t) {
536 t = p->next;
537 g_free(p);
538 }
539 s->pool_first_large = NULL;
c896fe29
FB		 540 s->pool_cur = s->pool_end = NULL;
541 s->pool_current = NULL;
542}
543
2ef6175a
RH		 544#include "exec/helper-proto.h"
545
100b5e01 546static const TCGHelperInfo all_helpers[] = {
2ef6175a 547#include "exec/helper-tcg.h"
100b5e01 548};
619205fd 549static GHashTable *helper_table;
100b5e01 550
22f15579
RH		 551#ifdef CONFIG_TCG_INTERPRETER
552static GHashTable *ffi_table;
553
554static ffi_type * const typecode_to_ffi[8] = {
555 [dh_typecode_void] = &ffi_type_void,
556 [dh_typecode_i32] = &ffi_type_uint32,
557 [dh_typecode_s32] = &ffi_type_sint32,
558 [dh_typecode_i64] = &ffi_type_uint64,
559 [dh_typecode_s64] = &ffi_type_sint64,
560 [dh_typecode_ptr] = &ffi_type_pointer,
561};
562#endif
563
91478cef 564static int indirect_reg_alloc_order[ARRAY_SIZE(tcg_target_reg_alloc_order)];
f69d277e 565static void process_op_defs(TCGContext *s);
1c2adb95
RH		 566static TCGTemp *tcg_global_reg_new_internal(TCGContext *s, TCGType type,
567 TCGReg reg, const char *name);
91478cef 568
43b972b7 569static void tcg_context_init(unsigned max_cpus)
c896fe29 570{
a76aabd3 571 TCGContext *s = &tcg_init_ctx;
100b5e01 572 int op, total_args, n, i;
c896fe29
FB		 573 TCGOpDef *def;
574 TCGArgConstraint *args_ct;
1c2adb95 575 TCGTemp *ts;
c896fe29
FB		 576
577 memset(s, 0, sizeof(*s));
c896fe29 578 s->nb_globals = 0;
c70fbf0a 579
c896fe29
FB		 580 /* Count total number of arguments and allocate the corresponding
581 space */
582 total_args = 0;
583 for(op = 0; op < NB_OPS; op++) {
584 def = &tcg_op_defs[op];
585 n = def->nb_iargs + def->nb_oargs;
586 total_args += n;
587 }
588
bc2b17e6 589 args_ct = g_new0(TCGArgConstraint, total_args);
c896fe29
FB		 590
591 for(op = 0; op < NB_OPS; op++) {
592 def = &tcg_op_defs[op];
593 def->args_ct = args_ct;
c896fe29 594 n = def->nb_iargs + def->nb_oargs;
c896fe29
FB		 595 args_ct += n;
596 }
5cd8f621
RH		 597
598 /* Register helpers. */
84fd9dd3 599 /* Use g_direct_hash/equal for direct pointer comparisons on func. */
619205fd 600 helper_table = g_hash_table_new(NULL, NULL);
84fd9dd3 601
100b5e01 602 for (i = 0; i < ARRAY_SIZE(all_helpers); ++i) {
84fd9dd3 603 g_hash_table_insert(helper_table, (gpointer)all_helpers[i].func,
72866e82 604 (gpointer)&all_helpers[i]);
100b5e01 605 }
5cd8f621 606
22f15579
RH		 607#ifdef CONFIG_TCG_INTERPRETER
608 /* g_direct_hash/equal for direct comparisons on uint32_t. */
609 ffi_table = g_hash_table_new(NULL, NULL);
610 for (i = 0; i < ARRAY_SIZE(all_helpers); ++i) {
611 struct {
612 ffi_cif cif;
613 ffi_type *args[];
614 } *ca;
615 uint32_t typemask = all_helpers[i].typemask;
616 gpointer hash = (gpointer)(uintptr_t)typemask;
617 ffi_status status;
618 int nargs;
619
620 if (g_hash_table_lookup(ffi_table, hash)) {
621 continue;
622 }
623
624 /* Ignoring the return type, find the last non-zero field. */
625 nargs = 32 - clz32(typemask >> 3);
626 nargs = DIV_ROUND_UP(nargs, 3);
627
628 ca = g_malloc0(sizeof(*ca) + nargs * sizeof(ffi_type *));
629 ca->cif.rtype = typecode_to_ffi[typemask & 7];
630 ca->cif.nargs = nargs;
631
632 if (nargs != 0) {
633 ca->cif.arg_types = ca->args;
634 for (i = 0; i < nargs; ++i) {
635 int typecode = extract32(typemask, (i + 1) * 3, 3);
636 ca->args[i] = typecode_to_ffi[typecode];
637 }
638 }
639
640 status = ffi_prep_cif(&ca->cif, FFI_DEFAULT_ABI, nargs,
641 ca->cif.rtype, ca->cif.arg_types);
642 assert(status == FFI_OK);
643
644 g_hash_table_insert(ffi_table, hash, (gpointer)&ca->cif);
645 }
646#endif
647
c896fe29 648 tcg_target_init(s);
f69d277e 649 process_op_defs(s);
91478cef
RH		 650
651 /* Reverse the order of the saved registers, assuming they're all at
652 the start of tcg_target_reg_alloc_order. */
653 for (n = 0; n < ARRAY_SIZE(tcg_target_reg_alloc_order); ++n) {
654 int r = tcg_target_reg_alloc_order[n];
655 if (tcg_regset_test_reg(tcg_target_call_clobber_regs, r)) {
656 break;
657 }
658 }
659 for (i = 0; i < n; ++i) {
660 indirect_reg_alloc_order[i] = tcg_target_reg_alloc_order[n - 1 - i];
661 }
662 for (; i < ARRAY_SIZE(tcg_target_reg_alloc_order); ++i) {
663 indirect_reg_alloc_order[i] = tcg_target_reg_alloc_order[i];
664 }
b1311c4a 665
38b47b19
EC		 666 alloc_tcg_plugin_context(s);
667
b1311c4a 668 tcg_ctx = s;
3468b59e
EC		 669 /*
670 * In user-mode we simply share the init context among threads, since we
671 * use a single region. See the documentation tcg_region_init() for the
672 * reasoning behind this.
673 * In softmmu we will have at most max_cpus TCG threads.
674 */
675#ifdef CONFIG_USER_ONLY
df2cce29 676 tcg_ctxs = &tcg_ctx;
0e2d61cf
RH		 677 tcg_cur_ctxs = 1;
678 tcg_max_ctxs = 1;
3468b59e 679#else
0e2d61cf
RH		 680 tcg_max_ctxs = max_cpus;
681 tcg_ctxs = g_new0(TCGContext *, max_cpus);
3468b59e 682#endif
1c2adb95
RH		 683
684 tcg_debug_assert(!tcg_regset_test_reg(s->reserved_regs, TCG_AREG0));
685 ts = tcg_global_reg_new_internal(s, TCG_TYPE_PTR, TCG_AREG0, "env");
686 cpu_env = temp_tcgv_ptr(ts);
9002ec79 687}
b03cce8e 688
43b972b7 689void tcg_init(size_t tb_size, int splitwx, unsigned max_cpus)
a76aabd3 690{
43b972b7
RH		 691 tcg_context_init(max_cpus);
692 tcg_region_init(tb_size, splitwx, max_cpus);
a76aabd3
RH		 693}
694
6e3b2bfd
EC		 695/*
696 * Allocate TBs right before their corresponding translated code, making
697 * sure that TBs and code are on different cache lines.
698 */
699TranslationBlock *tcg_tb_alloc(TCGContext *s)
700{
701 uintptr_t align = qemu_icache_linesize;
702 TranslationBlock *tb;
703 void *next;
704
e8feb96f 705 retry:
6e3b2bfd
EC		 706 tb = (void *)ROUND_UP((uintptr_t)s->code_gen_ptr, align);
707 next = (void *)ROUND_UP((uintptr_t)(tb + 1), align);
708
709 if (unlikely(next > s->code_gen_highwater)) {
e8feb96f
EC		 710 if (tcg_region_alloc(s)) {
711 return NULL;
712 }
713 goto retry;
6e3b2bfd 714 }
d73415a3 715 qatomic_set(&s->code_gen_ptr, next);
57a26946 716 s->data_gen_ptr = NULL;
6e3b2bfd
EC		 717 return tb;
718}
719
9002ec79
RH		 720void tcg_prologue_init(TCGContext *s)
721{
b0a0794a 722 size_t prologue_size;
8163b749 723
b0a0794a
RH		 724 s->code_ptr = s->code_gen_ptr;
725 s->code_buf = s->code_gen_ptr;
5b38ee31 726 s->data_gen_ptr = NULL;
b91ccb31
RH		 727
728#ifndef CONFIG_TCG_INTERPRETER
b0a0794a 729 tcg_qemu_tb_exec = (tcg_prologue_fn *)tcg_splitwx_to_rx(s->code_ptr);
b91ccb31 730#endif
8163b749 731
5b38ee31
RH		 732#ifdef TCG_TARGET_NEED_POOL_LABELS
733 s->pool_labels = NULL;
734#endif
735
653b87eb 736 qemu_thread_jit_write();
8163b749 737 /* Generate the prologue. */
b03cce8e 738 tcg_target_qemu_prologue(s);
5b38ee31
RH		 739
740#ifdef TCG_TARGET_NEED_POOL_LABELS
741 /* Allow the prologue to put e.g. guest_base into a pool entry. */
742 {
1768987b
RH		 743 int result = tcg_out_pool_finalize(s);
744 tcg_debug_assert(result == 0);
5b38ee31
RH		 745 }
746#endif
747
b0a0794a
RH		 748 prologue_size = tcg_current_code_size(s);
749
df5d2b16 750#ifndef CONFIG_TCG_INTERPRETER
b0a0794a
RH		 751 flush_idcache_range((uintptr_t)tcg_splitwx_to_rx(s->code_buf),
752 (uintptr_t)s->code_buf, prologue_size);
df5d2b16 753#endif
8163b749 754
d6b64b2b
RH		 755#ifdef DEBUG_DISAS
756 if (qemu_loglevel_mask(CPU_LOG_TB_OUT_ASM)) {
fc59d2d8 757 FILE *logfile = qemu_log_lock();
8163b749 758 qemu_log("PROLOGUE: [size=%zu]\n", prologue_size);
5b38ee31 759 if (s->data_gen_ptr) {
b0a0794a 760 size_t code_size = s->data_gen_ptr - s->code_gen_ptr;
5b38ee31
RH		 761 size_t data_size = prologue_size - code_size;
762 size_t i;
763
b0a0794a 764 log_disas(s->code_gen_ptr, code_size);
5b38ee31
RH		 765
766 for (i = 0; i < data_size; i += sizeof(tcg_target_ulong)) {
767 if (sizeof(tcg_target_ulong) == 8) {
768 qemu_log("0x%08" PRIxPTR ": .quad 0x%016" PRIx64 "\n",
769 (uintptr_t)s->data_gen_ptr + i,
770 *(uint64_t *)(s->data_gen_ptr + i));
771 } else {
772 qemu_log("0x%08" PRIxPTR ": .long 0x%08x\n",
773 (uintptr_t)s->data_gen_ptr + i,
774 *(uint32_t *)(s->data_gen_ptr + i));
775 }
776 }
777 } else {
b0a0794a 778 log_disas(s->code_gen_ptr, prologue_size);
5b38ee31 779 }
d6b64b2b
RH		 780 qemu_log("\n");
781 qemu_log_flush();
fc59d2d8 782 qemu_log_unlock(logfile);
d6b64b2b
RH		 783 }
784#endif
cedbcb01 785
6eea0434
RH		 786#ifndef CONFIG_TCG_INTERPRETER
787 /*
788 * Assert that goto_ptr is implemented completely, setting an epilogue.
789 * For tci, we use NULL as the signal to return from the interpreter,
790 * so skip this check.
791 */
f4e01e30 792 tcg_debug_assert(tcg_code_gen_epilogue != NULL);
6eea0434 793#endif
d1c74ab3
RH		 794
795 tcg_region_prologue_set(s);
c896fe29
FB		 796}
797
c896fe29
FB		 798void tcg_func_start(TCGContext *s)
799{
800 tcg_pool_reset(s);
801 s->nb_temps = s->nb_globals;
0ec9eabc
RH		 802
803 /* No temps have been previously allocated for size or locality. */
804 memset(s->free_temps, 0, sizeof(s->free_temps));
805
c0522136
RH		 806 /* No constant temps have been previously allocated. */
807 for (int i = 0; i < TCG_TYPE_COUNT; ++i) {
808 if (s->const_table[i]) {
809 g_hash_table_remove_all(s->const_table[i]);
810 }
811 }
812
abebf925 813 s->nb_ops = 0;
c896fe29
FB		 814 s->nb_labels = 0;
815 s->current_frame_offset = s->frame_start;
816
0a209d4b
RH		 817#ifdef CONFIG_DEBUG_TCG
818 s->goto_tb_issue_mask = 0;
819#endif
820
15fa08f8
RH		 821 QTAILQ_INIT(&s->ops);
822 QTAILQ_INIT(&s->free_ops);
bef16ab4 823 QSIMPLEQ_INIT(&s->labels);
c896fe29
FB		 824}
825
ae30e866 826static TCGTemp *tcg_temp_alloc(TCGContext *s)
7ca4b752
RH		 827{
828 int n = s->nb_temps++;
ae30e866
RH		 829
830 if (n >= TCG_MAX_TEMPS) {
db6b7d0c 831 tcg_raise_tb_overflow(s);
ae30e866 832 }
7ca4b752
RH		 833 return memset(&s->temps[n], 0, sizeof(TCGTemp));
834}
835
ae30e866 836static TCGTemp *tcg_global_alloc(TCGContext *s)
7ca4b752 837{
fa477d25
RH		 838 TCGTemp *ts;
839
7ca4b752 840 tcg_debug_assert(s->nb_globals == s->nb_temps);
ae30e866 841 tcg_debug_assert(s->nb_globals < TCG_MAX_TEMPS);
7ca4b752 842 s->nb_globals++;
fa477d25 843 ts = tcg_temp_alloc(s);
ee17db83 844 ts->kind = TEMP_GLOBAL;
fa477d25
RH		 845
846 return ts;
c896fe29
FB		 847}
848
085272b3
RH		 849static TCGTemp *tcg_global_reg_new_internal(TCGContext *s, TCGType type,
850 TCGReg reg, const char *name)
c896fe29 851{
c896fe29 852 TCGTemp *ts;
c896fe29 853
b3a62939 854 if (TCG_TARGET_REG_BITS == 32 && type != TCG_TYPE_I32) {
c896fe29 855 tcg_abort();
b3a62939 856 }
7ca4b752
RH		 857
858 ts = tcg_global_alloc(s);
c896fe29
FB		 859 ts->base_type = type;
860 ts->type = type;
ee17db83 861 ts->kind = TEMP_FIXED;
c896fe29 862 ts->reg = reg;
c896fe29 863 ts->name = name;
c896fe29 864 tcg_regset_set_reg(s->reserved_regs, reg);
7ca4b752 865
085272b3 866 return ts;
a7812ae4
PB		 867}
868
b6638662 869void tcg_set_frame(TCGContext *s, TCGReg reg, intptr_t start, intptr_t size)
b3a62939 870{
b3a62939
RH		 871 s->frame_start = start;
872 s->frame_end = start + size;
085272b3
RH		 873 s->frame_temp
874 = tcg_global_reg_new_internal(s, TCG_TYPE_PTR, reg, "_frame");
b3a62939
RH		 875}
876
085272b3
RH		 877TCGTemp *tcg_global_mem_new_internal(TCGType type, TCGv_ptr base,
878 intptr_t offset, const char *name)
c896fe29 879{
b1311c4a 880 TCGContext *s = tcg_ctx;
dc41aa7d 881 TCGTemp *base_ts = tcgv_ptr_temp(base);
7ca4b752 882 TCGTemp *ts = tcg_global_alloc(s);
b3915dbb 883 int indirect_reg = 0, bigendian = 0;
7ca4b752
RH		 884#ifdef HOST_WORDS_BIGENDIAN
885 bigendian = 1;
886#endif
c896fe29 887
c0522136
RH		 888 switch (base_ts->kind) {
889 case TEMP_FIXED:
890 break;
891 case TEMP_GLOBAL:
5a18407f
RH		 892 /* We do not support double-indirect registers. */
893 tcg_debug_assert(!base_ts->indirect_reg);
b3915dbb 894 base_ts->indirect_base = 1;
5a18407f
RH		 895 s->nb_indirects += (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64
896 ? 2 : 1);
897 indirect_reg = 1;
c0522136
RH		 898 break;
899 default:
900 g_assert_not_reached();
b3915dbb
RH		 901 }
902
7ca4b752
RH		 903 if (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64) {
904 TCGTemp *ts2 = tcg_global_alloc(s);
c896fe29 905 char buf[64];
7ca4b752
RH		 906
907 ts->base_type = TCG_TYPE_I64;
c896fe29 908 ts->type = TCG_TYPE_I32;
b3915dbb 909 ts->indirect_reg = indirect_reg;
c896fe29 910 ts->mem_allocated = 1;
b3a62939 911 ts->mem_base = base_ts;
7ca4b752 912 ts->mem_offset = offset + bigendian * 4;
c896fe29
FB		 913 pstrcpy(buf, sizeof(buf), name);
914 pstrcat(buf, sizeof(buf), "_0");
915 ts->name = strdup(buf);
c896fe29 916
7ca4b752
RH		 917 tcg_debug_assert(ts2 == ts + 1);
918 ts2->base_type = TCG_TYPE_I64;
919 ts2->type = TCG_TYPE_I32;
b3915dbb 920 ts2->indirect_reg = indirect_reg;
7ca4b752
RH		 921 ts2->mem_allocated = 1;
922 ts2->mem_base = base_ts;
923 ts2->mem_offset = offset + (1 - bigendian) * 4;
c896fe29
FB		 924 pstrcpy(buf, sizeof(buf), name);
925 pstrcat(buf, sizeof(buf), "_1");
120c1084 926 ts2->name = strdup(buf);
7ca4b752 927 } else {
c896fe29
FB		 928 ts->base_type = type;
929 ts->type = type;
b3915dbb 930 ts->indirect_reg = indirect_reg;
c896fe29 931 ts->mem_allocated = 1;
b3a62939 932 ts->mem_base = base_ts;
c896fe29 933 ts->mem_offset = offset;
c896fe29 934 ts->name = name;
c896fe29 935 }
085272b3 936 return ts;
a7812ae4
PB		 937}
938
5bfa8034 939TCGTemp *tcg_temp_new_internal(TCGType type, bool temp_local)
c896fe29 940{
b1311c4a 941 TCGContext *s = tcg_ctx;
ee17db83 942 TCGTempKind kind = temp_local ? TEMP_LOCAL : TEMP_NORMAL;
c896fe29 943 TCGTemp *ts;
641d5fbe 944 int idx, k;
c896fe29 945
0ec9eabc
RH		 946 k = type + (temp_local ? TCG_TYPE_COUNT : 0);
947 idx = find_first_bit(s->free_temps[k].l, TCG_MAX_TEMPS);
948 if (idx < TCG_MAX_TEMPS) {
949 /* There is already an available temp with the right type. */
950 clear_bit(idx, s->free_temps[k].l);
951
e8996ee0 952 ts = &s->temps[idx];
e8996ee0 953 ts->temp_allocated = 1;
7ca4b752 954 tcg_debug_assert(ts->base_type == type);
ee17db83 955 tcg_debug_assert(ts->kind == kind);
e8996ee0 956 } else {
7ca4b752
RH		 957 ts = tcg_temp_alloc(s);
958 if (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64) {
959 TCGTemp *ts2 = tcg_temp_alloc(s);
960
f6aa2f7d 961 ts->base_type = type;
e8996ee0
FB		 962 ts->type = TCG_TYPE_I32;
963 ts->temp_allocated = 1;
ee17db83 964 ts->kind = kind;
7ca4b752
RH		 965
966 tcg_debug_assert(ts2 == ts + 1);
967 ts2->base_type = TCG_TYPE_I64;
968 ts2->type = TCG_TYPE_I32;
969 ts2->temp_allocated = 1;
ee17db83 970 ts2->kind = kind;
7ca4b752 971 } else {
e8996ee0
FB		 972 ts->base_type = type;
973 ts->type = type;
974 ts->temp_allocated = 1;
ee17db83 975 ts->kind = kind;
e8996ee0 976 }
c896fe29 977 }
27bfd83c
PM		 978
979#if defined(CONFIG_DEBUG_TCG)
980 s->temps_in_use++;
981#endif
085272b3 982 return ts;
c896fe29
FB		 983}
984
d2fd745f
RH		 985TCGv_vec tcg_temp_new_vec(TCGType type)
986{
987 TCGTemp *t;
988
989#ifdef CONFIG_DEBUG_TCG
990 switch (type) {
991 case TCG_TYPE_V64:
992 assert(TCG_TARGET_HAS_v64);
993 break;
994 case TCG_TYPE_V128:
995 assert(TCG_TARGET_HAS_v128);
996 break;
997 case TCG_TYPE_V256:
998 assert(TCG_TARGET_HAS_v256);
999 break;
1000 default:
1001 g_assert_not_reached();
1002 }
1003#endif
1004
1005 t = tcg_temp_new_internal(type, 0);
1006 return temp_tcgv_vec(t);
1007}
1008
1009/* Create a new temp of the same type as an existing temp. */
1010TCGv_vec tcg_temp_new_vec_matching(TCGv_vec match)
1011{
1012 TCGTemp *t = tcgv_vec_temp(match);
1013
1014 tcg_debug_assert(t->temp_allocated != 0);
1015
1016 t = tcg_temp_new_internal(t->base_type, 0);
1017 return temp_tcgv_vec(t);
1018}
1019
5bfa8034 1020void tcg_temp_free_internal(TCGTemp *ts)
c896fe29 1021{
b1311c4a 1022 TCGContext *s = tcg_ctx;
085272b3 1023 int k, idx;
c896fe29 1024
c0522136
RH		 1025 /* In order to simplify users of tcg_constant_*, silently ignore free. */
1026 if (ts->kind == TEMP_CONST) {
1027 return;
1028 }
1029
27bfd83c
PM		 1030#if defined(CONFIG_DEBUG_TCG)
1031 s->temps_in_use--;
1032 if (s->temps_in_use < 0) {
1033 fprintf(stderr, "More temporaries freed than allocated!\n");
1034 }
1035#endif
1036
ee17db83 1037 tcg_debug_assert(ts->kind < TEMP_GLOBAL);
eabb7b91 1038 tcg_debug_assert(ts->temp_allocated != 0);
e8996ee0 1039 ts->temp_allocated = 0;
0ec9eabc 1040
085272b3 1041 idx = temp_idx(ts);
ee17db83 1042 k = ts->base_type + (ts->kind == TEMP_NORMAL ? 0 : TCG_TYPE_COUNT);
0ec9eabc 1043 set_bit(idx, s->free_temps[k].l);
c896fe29
FB		 1044}
1045
c0522136
RH		 1046TCGTemp *tcg_constant_internal(TCGType type, int64_t val)
1047{
1048 TCGContext *s = tcg_ctx;
1049 GHashTable *h = s->const_table[type];
1050 TCGTemp *ts;
1051
1052 if (h == NULL) {
1053 h = g_hash_table_new(g_int64_hash, g_int64_equal);
1054 s->const_table[type] = h;
1055 }
1056
1057 ts = g_hash_table_lookup(h, &val);
1058 if (ts == NULL) {
1059 ts = tcg_temp_alloc(s);
1060
1061 if (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64) {
1062 TCGTemp *ts2 = tcg_temp_alloc(s);
1063
1064 ts->base_type = TCG_TYPE_I64;
1065 ts->type = TCG_TYPE_I32;
1066 ts->kind = TEMP_CONST;
1067 ts->temp_allocated = 1;
1068 /*
1069 * Retain the full value of the 64-bit constant in the low
1070 * part, so that the hash table works. Actual uses will
1071 * truncate the value to the low part.
1072 */
1073 ts->val = val;
1074
1075 tcg_debug_assert(ts2 == ts + 1);
1076 ts2->base_type = TCG_TYPE_I64;
1077 ts2->type = TCG_TYPE_I32;
1078 ts2->kind = TEMP_CONST;
1079 ts2->temp_allocated = 1;
1080 ts2->val = val >> 32;
1081 } else {
1082 ts->base_type = type;
1083 ts->type = type;
1084 ts->kind = TEMP_CONST;
1085 ts->temp_allocated = 1;
1086 ts->val = val;
1087 }
1088 g_hash_table_insert(h, &ts->val, ts);
1089 }
1090
1091 return ts;
1092}
1093
1094TCGv_vec tcg_constant_vec(TCGType type, unsigned vece, int64_t val)
1095{
1096 val = dup_const(vece, val);
1097 return temp_tcgv_vec(tcg_constant_internal(type, val));
1098}
1099
88d4005b
RH		 1100TCGv_vec tcg_constant_vec_matching(TCGv_vec match, unsigned vece, int64_t val)
1101{
1102 TCGTemp *t = tcgv_vec_temp(match);
1103
1104 tcg_debug_assert(t->temp_allocated != 0);
1105 return tcg_constant_vec(t->base_type, vece, val);
1106}
1107
a7812ae4 1108TCGv_i32 tcg_const_i32(int32_t val)
c896fe29 1109{
a7812ae4
PB		 1110 TCGv_i32 t0;
1111 t0 = tcg_temp_new_i32();
e8996ee0
FB		 1112 tcg_gen_movi_i32(t0, val);
1113 return t0;
1114}
c896fe29 1115
a7812ae4 1116TCGv_i64 tcg_const_i64(int64_t val)
e8996ee0 1117{
a7812ae4
PB		 1118 TCGv_i64 t0;
1119 t0 = tcg_temp_new_i64();
e8996ee0
FB		 1120 tcg_gen_movi_i64(t0, val);
1121 return t0;
c896fe29
FB		 1122}
1123
a7812ae4 1124TCGv_i32 tcg_const_local_i32(int32_t val)
bdffd4a9 1125{
a7812ae4
PB		 1126 TCGv_i32 t0;
1127 t0 = tcg_temp_local_new_i32();
bdffd4a9
AJ		 1128 tcg_gen_movi_i32(t0, val);
1129 return t0;
1130}
1131
a7812ae4 1132TCGv_i64 tcg_const_local_i64(int64_t val)
bdffd4a9 1133{
a7812ae4
PB		 1134 TCGv_i64 t0;
1135 t0 = tcg_temp_local_new_i64();
bdffd4a9
AJ		 1136 tcg_gen_movi_i64(t0, val);
1137 return t0;
1138}
1139
27bfd83c
PM		 1140#if defined(CONFIG_DEBUG_TCG)
1141void tcg_clear_temp_count(void)
1142{
b1311c4a 1143 TCGContext *s = tcg_ctx;
27bfd83c
PM		 1144 s->temps_in_use = 0;
1145}
1146
1147int tcg_check_temp_count(void)
1148{
b1311c4a 1149 TCGContext *s = tcg_ctx;
27bfd83c
PM		 1150 if (s->temps_in_use) {
1151 /* Clear the count so that we don't give another
1152 * warning immediately next time around.
1153 */
1154 s->temps_in_use = 0;
1155 return 1;
1156 }
1157 return 0;
1158}
1159#endif
1160
be0f34b5
RH		 1161/* Return true if OP may appear in the opcode stream.
1162 Test the runtime variable that controls each opcode. */
1163bool tcg_op_supported(TCGOpcode op)
1164{
d2fd745f
RH		 1165 const bool have_vec
1166 = TCG_TARGET_HAS_v64 | TCG_TARGET_HAS_v128 | TCG_TARGET_HAS_v256;
1167
be0f34b5
RH		 1168 switch (op) {
1169 case INDEX_op_discard:
1170 case INDEX_op_set_label:
1171 case INDEX_op_call:
1172 case INDEX_op_br:
1173 case INDEX_op_mb:
1174 case INDEX_op_insn_start:
1175 case INDEX_op_exit_tb:
1176 case INDEX_op_goto_tb:
f4e01e30 1177 case INDEX_op_goto_ptr:
be0f34b5
RH		 1178 case INDEX_op_qemu_ld_i32:
1179 case INDEX_op_qemu_st_i32:
1180 case INDEX_op_qemu_ld_i64:
1181 case INDEX_op_qemu_st_i64:
1182 return true;
1183
07ce0b05
RH		 1184 case INDEX_op_qemu_st8_i32:
1185 return TCG_TARGET_HAS_qemu_st8_i32;
1186
be0f34b5 1187 case INDEX_op_mov_i32:
be0f34b5
RH		 1188 case INDEX_op_setcond_i32:
1189 case INDEX_op_brcond_i32:
1190 case INDEX_op_ld8u_i32:
1191 case INDEX_op_ld8s_i32:
1192 case INDEX_op_ld16u_i32:
1193 case INDEX_op_ld16s_i32:
1194 case INDEX_op_ld_i32:
1195 case INDEX_op_st8_i32:
1196 case INDEX_op_st16_i32:
1197 case INDEX_op_st_i32:
1198 case INDEX_op_add_i32:
1199 case INDEX_op_sub_i32:
1200 case INDEX_op_mul_i32:
1201 case INDEX_op_and_i32:
1202 case INDEX_op_or_i32:
1203 case INDEX_op_xor_i32:
1204 case INDEX_op_shl_i32:
1205 case INDEX_op_shr_i32:
1206 case INDEX_op_sar_i32:
1207 return true;
1208
1209 case INDEX_op_movcond_i32:
1210 return TCG_TARGET_HAS_movcond_i32;
1211 case INDEX_op_div_i32:
1212 case INDEX_op_divu_i32:
1213 return TCG_TARGET_HAS_div_i32;
1214 case INDEX_op_rem_i32:
1215 case INDEX_op_remu_i32:
1216 return TCG_TARGET_HAS_rem_i32;
1217 case INDEX_op_div2_i32:
1218 case INDEX_op_divu2_i32:
1219 return TCG_TARGET_HAS_div2_i32;
1220 case INDEX_op_rotl_i32:
1221 case INDEX_op_rotr_i32:
1222 return TCG_TARGET_HAS_rot_i32;
1223 case INDEX_op_deposit_i32:
1224 return TCG_TARGET_HAS_deposit_i32;
1225 case INDEX_op_extract_i32:
1226 return TCG_TARGET_HAS_extract_i32;
1227 case INDEX_op_sextract_i32:
1228 return TCG_TARGET_HAS_sextract_i32;
fce1296f
RH		 1229 case INDEX_op_extract2_i32:
1230 return TCG_TARGET_HAS_extract2_i32;
be0f34b5
RH		 1231 case INDEX_op_add2_i32:
1232 return TCG_TARGET_HAS_add2_i32;
1233 case INDEX_op_sub2_i32:
1234 return TCG_TARGET_HAS_sub2_i32;
1235 case INDEX_op_mulu2_i32:
1236 return TCG_TARGET_HAS_mulu2_i32;
1237 case INDEX_op_muls2_i32:
1238 return TCG_TARGET_HAS_muls2_i32;
1239 case INDEX_op_muluh_i32:
1240 return TCG_TARGET_HAS_muluh_i32;
1241 case INDEX_op_mulsh_i32:
1242 return TCG_TARGET_HAS_mulsh_i32;
1243 case INDEX_op_ext8s_i32:
1244 return TCG_TARGET_HAS_ext8s_i32;
1245 case INDEX_op_ext16s_i32:
1246 return TCG_TARGET_HAS_ext16s_i32;
1247 case INDEX_op_ext8u_i32:
1248 return TCG_TARGET_HAS_ext8u_i32;
1249 case INDEX_op_ext16u_i32:
1250 return TCG_TARGET_HAS_ext16u_i32;
1251 case INDEX_op_bswap16_i32:
1252 return TCG_TARGET_HAS_bswap16_i32;
1253 case INDEX_op_bswap32_i32:
1254 return TCG_TARGET_HAS_bswap32_i32;
1255 case INDEX_op_not_i32:
1256 return TCG_TARGET_HAS_not_i32;
1257 case INDEX_op_neg_i32:
1258 return TCG_TARGET_HAS_neg_i32;
1259 case INDEX_op_andc_i32:
1260 return TCG_TARGET_HAS_andc_i32;
1261 case INDEX_op_orc_i32:
1262 return TCG_TARGET_HAS_orc_i32;
1263 case INDEX_op_eqv_i32:
1264 return TCG_TARGET_HAS_eqv_i32;
1265 case INDEX_op_nand_i32:
1266 return TCG_TARGET_HAS_nand_i32;
1267 case INDEX_op_nor_i32:
1268 return TCG_TARGET_HAS_nor_i32;
1269 case INDEX_op_clz_i32:
1270 return TCG_TARGET_HAS_clz_i32;
1271 case INDEX_op_ctz_i32:
1272 return TCG_TARGET_HAS_ctz_i32;
1273 case INDEX_op_ctpop_i32:
1274 return TCG_TARGET_HAS_ctpop_i32;
1275
1276 case INDEX_op_brcond2_i32:
1277 case INDEX_op_setcond2_i32:
1278 return TCG_TARGET_REG_BITS == 32;
1279
1280 case INDEX_op_mov_i64:
be0f34b5
RH		 1281 case INDEX_op_setcond_i64:
1282 case INDEX_op_brcond_i64:
1283 case INDEX_op_ld8u_i64:
1284 case INDEX_op_ld8s_i64:
1285 case INDEX_op_ld16u_i64:
1286 case INDEX_op_ld16s_i64:
1287 case INDEX_op_ld32u_i64:
1288 case INDEX_op_ld32s_i64:
1289 case INDEX_op_ld_i64:
1290 case INDEX_op_st8_i64:
1291 case INDEX_op_st16_i64:
1292 case INDEX_op_st32_i64:
1293 case INDEX_op_st_i64:
1294 case INDEX_op_add_i64:
1295 case INDEX_op_sub_i64:
1296 case INDEX_op_mul_i64:
1297 case INDEX_op_and_i64:
1298 case INDEX_op_or_i64:
1299 case INDEX_op_xor_i64:
1300 case INDEX_op_shl_i64:
1301 case INDEX_op_shr_i64:
1302 case INDEX_op_sar_i64:
1303 case INDEX_op_ext_i32_i64:
1304 case INDEX_op_extu_i32_i64:
1305 return TCG_TARGET_REG_BITS == 64;
1306
1307 case INDEX_op_movcond_i64:
1308 return TCG_TARGET_HAS_movcond_i64;
1309 case INDEX_op_div_i64:
1310 case INDEX_op_divu_i64:
1311 return TCG_TARGET_HAS_div_i64;
1312 case INDEX_op_rem_i64:
1313 case INDEX_op_remu_i64:
1314 return TCG_TARGET_HAS_rem_i64;
1315 case INDEX_op_div2_i64:
1316 case INDEX_op_divu2_i64:
1317 return TCG_TARGET_HAS_div2_i64;
1318 case INDEX_op_rotl_i64:
1319 case INDEX_op_rotr_i64:
1320 return TCG_TARGET_HAS_rot_i64;
1321 case INDEX_op_deposit_i64:
1322 return TCG_TARGET_HAS_deposit_i64;
1323 case INDEX_op_extract_i64:
1324 return TCG_TARGET_HAS_extract_i64;
1325 case INDEX_op_sextract_i64:
1326 return TCG_TARGET_HAS_sextract_i64;
fce1296f
RH		 1327 case INDEX_op_extract2_i64:
1328 return TCG_TARGET_HAS_extract2_i64;
be0f34b5
RH		 1329 case INDEX_op_extrl_i64_i32:
1330 return TCG_TARGET_HAS_extrl_i64_i32;
1331 case INDEX_op_extrh_i64_i32:
1332 return TCG_TARGET_HAS_extrh_i64_i32;
1333 case INDEX_op_ext8s_i64:
1334 return TCG_TARGET_HAS_ext8s_i64;
1335 case INDEX_op_ext16s_i64:
1336 return TCG_TARGET_HAS_ext16s_i64;
1337 case INDEX_op_ext32s_i64:
1338 return TCG_TARGET_HAS_ext32s_i64;
1339 case INDEX_op_ext8u_i64:
1340 return TCG_TARGET_HAS_ext8u_i64;
1341 case INDEX_op_ext16u_i64:
1342 return TCG_TARGET_HAS_ext16u_i64;
1343 case INDEX_op_ext32u_i64:
1344 return TCG_TARGET_HAS_ext32u_i64;
1345 case INDEX_op_bswap16_i64:
1346 return TCG_TARGET_HAS_bswap16_i64;
1347 case INDEX_op_bswap32_i64:
1348 return TCG_TARGET_HAS_bswap32_i64;
1349 case INDEX_op_bswap64_i64:
1350 return TCG_TARGET_HAS_bswap64_i64;
1351 case INDEX_op_not_i64:
1352 return TCG_TARGET_HAS_not_i64;
1353 case INDEX_op_neg_i64:
1354 return TCG_TARGET_HAS_neg_i64;
1355 case INDEX_op_andc_i64:
1356 return TCG_TARGET_HAS_andc_i64;
1357 case INDEX_op_orc_i64:
1358 return TCG_TARGET_HAS_orc_i64;
1359 case INDEX_op_eqv_i64:
1360 return TCG_TARGET_HAS_eqv_i64;
1361 case INDEX_op_nand_i64:
1362 return TCG_TARGET_HAS_nand_i64;
1363 case INDEX_op_nor_i64:
1364 return TCG_TARGET_HAS_nor_i64;
1365 case INDEX_op_clz_i64:
1366 return TCG_TARGET_HAS_clz_i64;
1367 case INDEX_op_ctz_i64:
1368 return TCG_TARGET_HAS_ctz_i64;
1369 case INDEX_op_ctpop_i64:
1370 return TCG_TARGET_HAS_ctpop_i64;
1371 case INDEX_op_add2_i64:
1372 return TCG_TARGET_HAS_add2_i64;
1373 case INDEX_op_sub2_i64:
1374 return TCG_TARGET_HAS_sub2_i64;
1375 case INDEX_op_mulu2_i64:
1376 return TCG_TARGET_HAS_mulu2_i64;
1377 case INDEX_op_muls2_i64:
1378 return TCG_TARGET_HAS_muls2_i64;
1379 case INDEX_op_muluh_i64:
1380 return TCG_TARGET_HAS_muluh_i64;
1381 case INDEX_op_mulsh_i64:
1382 return TCG_TARGET_HAS_mulsh_i64;
1383
d2fd745f
RH		 1384 case INDEX_op_mov_vec:
1385 case INDEX_op_dup_vec:
37ee55a0 1386 case INDEX_op_dupm_vec:
d2fd745f
RH		 1387 case INDEX_op_ld_vec:
1388 case INDEX_op_st_vec:
1389 case INDEX_op_add_vec:
1390 case INDEX_op_sub_vec:
1391 case INDEX_op_and_vec:
1392 case INDEX_op_or_vec:
1393 case INDEX_op_xor_vec:
212be173 1394 case INDEX_op_cmp_vec:
d2fd745f
RH		 1395 return have_vec;
1396 case INDEX_op_dup2_vec:
1397 return have_vec && TCG_TARGET_REG_BITS == 32;
1398 case INDEX_op_not_vec:
1399 return have_vec && TCG_TARGET_HAS_not_vec;
1400 case INDEX_op_neg_vec:
1401 return have_vec && TCG_TARGET_HAS_neg_vec;
bcefc902
RH		 1402 case INDEX_op_abs_vec:
1403 return have_vec && TCG_TARGET_HAS_abs_vec;
d2fd745f
RH		 1404 case INDEX_op_andc_vec:
1405 return have_vec && TCG_TARGET_HAS_andc_vec;
1406 case INDEX_op_orc_vec:
1407 return have_vec && TCG_TARGET_HAS_orc_vec;
3774030a
RH		 1408 case INDEX_op_mul_vec:
1409 return have_vec && TCG_TARGET_HAS_mul_vec;
d0ec9796
RH		 1410 case INDEX_op_shli_vec:
1411 case INDEX_op_shri_vec:
1412 case INDEX_op_sari_vec:
1413 return have_vec && TCG_TARGET_HAS_shi_vec;
1414 case INDEX_op_shls_vec:
1415 case INDEX_op_shrs_vec:
1416 case INDEX_op_sars_vec:
1417 return have_vec && TCG_TARGET_HAS_shs_vec;
1418 case INDEX_op_shlv_vec:
1419 case INDEX_op_shrv_vec:
1420 case INDEX_op_sarv_vec:
1421 return have_vec && TCG_TARGET_HAS_shv_vec;
b0f7e744
RH		 1422 case INDEX_op_rotli_vec:
1423 return have_vec && TCG_TARGET_HAS_roti_vec;
23850a74
RH		 1424 case INDEX_op_rotls_vec:
1425 return have_vec && TCG_TARGET_HAS_rots_vec;
5d0ceda9
RH		 1426 case INDEX_op_rotlv_vec:
1427 case INDEX_op_rotrv_vec:
1428 return have_vec && TCG_TARGET_HAS_rotv_vec;
8afaf050
RH		 1429 case INDEX_op_ssadd_vec:
1430 case INDEX_op_usadd_vec:
1431 case INDEX_op_sssub_vec:
1432 case INDEX_op_ussub_vec:
1433 return have_vec && TCG_TARGET_HAS_sat_vec;
dd0a0fcd
RH		 1434 case INDEX_op_smin_vec:
1435 case INDEX_op_umin_vec:
1436 case INDEX_op_smax_vec:
1437 case INDEX_op_umax_vec:
1438 return have_vec && TCG_TARGET_HAS_minmax_vec;
38dc1294
RH		 1439 case INDEX_op_bitsel_vec:
1440 return have_vec && TCG_TARGET_HAS_bitsel_vec;
f75da298
RH		 1441 case INDEX_op_cmpsel_vec:
1442 return have_vec && TCG_TARGET_HAS_cmpsel_vec;
d2fd745f 1443
db432672
RH		 1444 default:
1445 tcg_debug_assert(op > INDEX_op_last_generic && op < NB_OPS);
1446 return true;
be0f34b5 1447 }
be0f34b5
RH		 1448}
1449
39cf05d3
FB		 1450/* Note: we convert the 64 bit args to 32 bit and do some alignment
1451 and endian swap. Maybe it would be better to do the alignment
1452 and endian swap in tcg_reg_alloc_call(). */
ae8b75dc 1453void tcg_gen_callN(void *func, TCGTemp *ret, int nargs, TCGTemp **args)
c896fe29 1454{
75e8b9b7 1455 int i, real_args, nb_rets, pi;
3e92aa34
RH		 1456 unsigned typemask;
1457 const TCGHelperInfo *info;
75e8b9b7 1458 TCGOp *op;
afb49896 1459
619205fd 1460 info = g_hash_table_lookup(helper_table, (gpointer)func);
7319d83a 1461 typemask = info->typemask;
2bece2c8 1462
38b47b19
EC		 1463#ifdef CONFIG_PLUGIN
1464 /* detect non-plugin helpers */
1465 if (tcg_ctx->plugin_insn && unlikely(strncmp(info->name, "plugin_", 7))) {
1466 tcg_ctx->plugin_insn->calls_helpers = true;
1467 }
1468#endif
1469
34b1a49c
RH		 1470#if defined(__sparc__) && !defined(__arch64__) \
1471 && !defined(CONFIG_TCG_INTERPRETER)
1472 /* We have 64-bit values in one register, but need to pass as two
1473 separate parameters. Split them. */
7319d83a 1474 int orig_typemask = typemask;
34b1a49c
RH		 1475 int orig_nargs = nargs;
1476 TCGv_i64 retl, reth;
ae8b75dc 1477 TCGTemp *split_args[MAX_OPC_PARAM];
34b1a49c 1478
f764718d
RH		 1479 retl = NULL;
1480 reth = NULL;
7319d83a
RH		 1481 typemask = 0;
1482 for (i = real_args = 0; i < nargs; ++i) {
1483 int argtype = extract32(orig_typemask, (i + 1) * 3, 3);
1484 bool is_64bit = (argtype & ~1) == dh_typecode_i64;
1485
1486 if (is_64bit) {
1487 TCGv_i64 orig = temp_tcgv_i64(args[i]);
1488 TCGv_i32 h = tcg_temp_new_i32();
1489 TCGv_i32 l = tcg_temp_new_i32();
1490 tcg_gen_extr_i64_i32(l, h, orig);
1491 split_args[real_args++] = tcgv_i32_temp(h);
1492 typemask |= dh_typecode_i32 << (real_args * 3);
1493 split_args[real_args++] = tcgv_i32_temp(l);
1494 typemask |= dh_typecode_i32 << (real_args * 3);
1495 } else {
1496 split_args[real_args++] = args[i];
1497 typemask |= argtype << (real_args * 3);
34b1a49c 1498 }
34b1a49c 1499 }
7319d83a
RH		 1500 nargs = real_args;
1501 args = split_args;
34b1a49c 1502#elif defined(TCG_TARGET_EXTEND_ARGS) && TCG_TARGET_REG_BITS == 64
2bece2c8 1503 for (i = 0; i < nargs; ++i) {
7319d83a
RH		 1504 int argtype = extract32(typemask, (i + 1) * 3, 3);
1505 bool is_32bit = (argtype & ~1) == dh_typecode_i32;
1506 bool is_signed = argtype & 1;
1507
1508 if (is_32bit) {
2bece2c8 1509 TCGv_i64 temp = tcg_temp_new_i64();
085272b3 1510 TCGv_i64 orig = temp_tcgv_i64(args[i]);
2bece2c8
RH		 1511 if (is_signed) {
1512 tcg_gen_ext32s_i64(temp, orig);
1513 } else {
1514 tcg_gen_ext32u_i64(temp, orig);
1515 }
ae8b75dc 1516 args[i] = tcgv_i64_temp(temp);
2bece2c8
RH		 1517 }
1518 }
1519#endif /* TCG_TARGET_EXTEND_ARGS */
1520
15fa08f8 1521 op = tcg_emit_op(INDEX_op_call);
75e8b9b7
RH		 1522
1523 pi = 0;
ae8b75dc 1524 if (ret != NULL) {
34b1a49c
RH		 1525#if defined(__sparc__) && !defined(__arch64__) \
1526 && !defined(CONFIG_TCG_INTERPRETER)
7319d83a 1527 if ((typemask & 6) == dh_typecode_i64) {
34b1a49c
RH		 1528 /* The 32-bit ABI is going to return the 64-bit value in
1529 the %o0/%o1 register pair. Prepare for this by using
1530 two return temporaries, and reassemble below. */
1531 retl = tcg_temp_new_i64();
1532 reth = tcg_temp_new_i64();
ae8b75dc
RH		 1533 op->args[pi++] = tcgv_i64_arg(reth);
1534 op->args[pi++] = tcgv_i64_arg(retl);
34b1a49c
RH		 1535 nb_rets = 2;
1536 } else {
ae8b75dc 1537 op->args[pi++] = temp_arg(ret);
34b1a49c
RH		 1538 nb_rets = 1;
1539 }
1540#else
7319d83a 1541 if (TCG_TARGET_REG_BITS < 64 && (typemask & 6) == dh_typecode_i64) {
02eb19d0 1542#ifdef HOST_WORDS_BIGENDIAN
ae8b75dc
RH		 1543 op->args[pi++] = temp_arg(ret + 1);
1544 op->args[pi++] = temp_arg(ret);
39cf05d3 1545#else
ae8b75dc
RH		 1546 op->args[pi++] = temp_arg(ret);
1547 op->args[pi++] = temp_arg(ret + 1);
39cf05d3 1548#endif
a7812ae4 1549 nb_rets = 2;
34b1a49c 1550 } else {
ae8b75dc 1551 op->args[pi++] = temp_arg(ret);
a7812ae4 1552 nb_rets = 1;
c896fe29 1553 }
34b1a49c 1554#endif
a7812ae4
PB		 1555 } else {
1556 nb_rets = 0;
c896fe29 1557 }
cd9090aa 1558 TCGOP_CALLO(op) = nb_rets;
75e8b9b7 1559
a7812ae4
PB		 1560 real_args = 0;
1561 for (i = 0; i < nargs; i++) {
7319d83a
RH		 1562 int argtype = extract32(typemask, (i + 1) * 3, 3);
1563 bool is_64bit = (argtype & ~1) == dh_typecode_i64;
7b7d8b2d 1564 bool want_align = false;
7319d83a 1565
7b7d8b2d
RH		 1566#if defined(CONFIG_TCG_INTERPRETER)
1567 /*
1568 * Align all arguments, so that they land in predictable places
1569 * for passing off to ffi_call.
1570 */
1571 want_align = true;
1572#elif defined(TCG_TARGET_CALL_ALIGN_ARGS)
1573 /* Some targets want aligned 64 bit args */
1574 want_align = is_64bit;
39cf05d3 1575#endif
7b7d8b2d
RH		 1576
1577 if (TCG_TARGET_REG_BITS < 64 && want_align && (real_args & 1)) {
1578 op->args[pi++] = TCG_CALL_DUMMY_ARG;
1579 real_args++;
1580 }
1581
1582 if (TCG_TARGET_REG_BITS < 64 && is_64bit) {
1583 /*
1584 * If stack grows up, then we will be placing successive
1585 * arguments at lower addresses, which means we need to
1586 * reverse the order compared to how we would normally
1587 * treat either big or little-endian. For those arguments
1588 * that will wind up in registers, this still works for
1589 * HPPA (the only current STACK_GROWSUP target) since the
1590 * argument registers are *also* allocated in decreasing
1591 * order. If another such target is added, this logic may
1592 * have to get more complicated to differentiate between
1593 * stack arguments and register arguments.
1594 */
02eb19d0 1595#if defined(HOST_WORDS_BIGENDIAN) != defined(TCG_TARGET_STACK_GROWSUP)
ae8b75dc
RH		 1596 op->args[pi++] = temp_arg(args[i] + 1);
1597 op->args[pi++] = temp_arg(args[i]);
c896fe29 1598#else
ae8b75dc
RH		 1599 op->args[pi++] = temp_arg(args[i]);
1600 op->args[pi++] = temp_arg(args[i] + 1);
c896fe29 1601#endif
a7812ae4 1602 real_args += 2;
2bece2c8 1603 continue;
c896fe29 1604 }
2bece2c8 1605
ae8b75dc 1606 op->args[pi++] = temp_arg(args[i]);
2bece2c8 1607 real_args++;
c896fe29 1608 }
75e8b9b7 1609 op->args[pi++] = (uintptr_t)func;
3e92aa34 1610 op->args[pi++] = (uintptr_t)info;
cd9090aa 1611 TCGOP_CALLI(op) = real_args;
a7812ae4 1612
75e8b9b7 1613 /* Make sure the fields didn't overflow. */
cd9090aa 1614 tcg_debug_assert(TCGOP_CALLI(op) == real_args);
75e8b9b7 1615 tcg_debug_assert(pi <= ARRAY_SIZE(op->args));
2bece2c8 1616
34b1a49c
RH		 1617#if defined(__sparc__) && !defined(__arch64__) \
1618 && !defined(CONFIG_TCG_INTERPRETER)
1619 /* Free all of the parts we allocated above. */
1620 for (i = real_args = 0; i < orig_nargs; ++i) {
7319d83a
RH		 1621 int argtype = extract32(orig_typemask, (i + 1) * 3, 3);
1622 bool is_64bit = (argtype & ~1) == dh_typecode_i64;
1623
34b1a49c 1624 if (is_64bit) {
085272b3
RH		 1625 tcg_temp_free_internal(args[real_args++]);
1626 tcg_temp_free_internal(args[real_args++]);
34b1a49c
RH		 1627 } else {
1628 real_args++;
1629 }
1630 }
7319d83a 1631 if ((orig_typemask & 6) == dh_typecode_i64) {
34b1a49c
RH		 1632 /* The 32-bit ABI returned two 32-bit pieces. Re-assemble them.
1633 Note that describing these as TCGv_i64 eliminates an unnecessary
1634 zero-extension that tcg_gen_concat_i32_i64 would create. */
085272b3 1635 tcg_gen_concat32_i64(temp_tcgv_i64(ret), retl, reth);
34b1a49c
RH		 1636 tcg_temp_free_i64(retl);
1637 tcg_temp_free_i64(reth);
1638 }
1639#elif defined(TCG_TARGET_EXTEND_ARGS) && TCG_TARGET_REG_BITS == 64
2bece2c8 1640 for (i = 0; i < nargs; ++i) {
7319d83a
RH		 1641 int argtype = extract32(typemask, (i + 1) * 3, 3);
1642 bool is_32bit = (argtype & ~1) == dh_typecode_i32;
1643
1644 if (is_32bit) {
085272b3 1645 tcg_temp_free_internal(args[i]);
2bece2c8
RH		 1646 }
1647 }
1648#endif /* TCG_TARGET_EXTEND_ARGS */
c896fe29 1649}
c896fe29 1650
8fcd3692 1651static void tcg_reg_alloc_start(TCGContext *s)
c896fe29 1652{
ac3b8891 1653 int i, n;
ac3b8891 1654
ee17db83
RH		 1655 for (i = 0, n = s->nb_temps; i < n; i++) {
1656 TCGTemp *ts = &s->temps[i];
1657 TCGTempVal val = TEMP_VAL_MEM;
1658
1659 switch (ts->kind) {
c0522136
RH		 1660 case TEMP_CONST:
1661 val = TEMP_VAL_CONST;
1662 break;
ee17db83
RH		 1663 case TEMP_FIXED:
1664 val = TEMP_VAL_REG;
1665 break;
1666 case TEMP_GLOBAL:
1667 break;
1668 case TEMP_NORMAL:
1669 val = TEMP_VAL_DEAD;
1670 /* fall through */
1671 case TEMP_LOCAL:
1672 ts->mem_allocated = 0;
1673 break;
1674 default:
1675 g_assert_not_reached();
1676 }
1677 ts->val_type = val;
e8996ee0 1678 }
f8b2f202
RH		 1679
1680 memset(s->reg_to_temp, 0, sizeof(s->reg_to_temp));
c896fe29
FB		 1681}
1682
f8b2f202
RH		 1683static char *tcg_get_arg_str_ptr(TCGContext *s, char *buf, int buf_size,
1684 TCGTemp *ts)
c896fe29 1685{
1807f4c4 1686 int idx = temp_idx(ts);
ac56dd48 1687
ee17db83
RH		 1688 switch (ts->kind) {
1689 case TEMP_FIXED:
1690 case TEMP_GLOBAL:
ac56dd48 1691 pstrcpy(buf, buf_size, ts->name);
ee17db83
RH		 1692 break;
1693 case TEMP_LOCAL:
f8b2f202 1694 snprintf(buf, buf_size, "loc%d", idx - s->nb_globals);
ee17db83
RH		 1695 break;
1696 case TEMP_NORMAL:
f8b2f202 1697 snprintf(buf, buf_size, "tmp%d", idx - s->nb_globals);
ee17db83 1698 break;
c0522136
RH		 1699 case TEMP_CONST:
1700 switch (ts->type) {
1701 case TCG_TYPE_I32:
1702 snprintf(buf, buf_size, "$0x%x", (int32_t)ts->val);
1703 break;
1704#if TCG_TARGET_REG_BITS > 32
1705 case TCG_TYPE_I64:
1706 snprintf(buf, buf_size, "$0x%" PRIx64, ts->val);
1707 break;
1708#endif
1709 case TCG_TYPE_V64:
1710 case TCG_TYPE_V128:
1711 case TCG_TYPE_V256:
1712 snprintf(buf, buf_size, "v%d$0x%" PRIx64,
1713 64 << (ts->type - TCG_TYPE_V64), ts->val);
1714 break;
1715 default:
1716 g_assert_not_reached();
1717 }
1718 break;
c896fe29
FB		 1719 }
1720 return buf;
1721}
1722
43439139
RH		 1723static char *tcg_get_arg_str(TCGContext *s, char *buf,
1724 int buf_size, TCGArg arg)
f8b2f202 1725{
43439139 1726 return tcg_get_arg_str_ptr(s, buf, buf_size, arg_temp(arg));
f8b2f202
RH		 1727}
1728
f48f3ede
BS		 1729static const char * const cond_name[] =
1730{
0aed257f
RH		 1731 [TCG_COND_NEVER] = "never",
1732 [TCG_COND_ALWAYS] = "always",
f48f3ede
BS		 1733 [TCG_COND_EQ] = "eq",
1734 [TCG_COND_NE] = "ne",
1735 [TCG_COND_LT] = "lt",
1736 [TCG_COND_GE] = "ge",
1737 [TCG_COND_LE] = "le",
1738 [TCG_COND_GT] = "gt",
1739 [TCG_COND_LTU] = "ltu",
1740 [TCG_COND_GEU] = "geu",
1741 [TCG_COND_LEU] = "leu",
1742 [TCG_COND_GTU] = "gtu"
1743};
1744
f713d6ad
RH		 1745static const char * const ldst_name[] =
1746{
1747 [MO_UB] = "ub",
1748 [MO_SB] = "sb",
1749 [MO_LEUW] = "leuw",
1750 [MO_LESW] = "lesw",
1751 [MO_LEUL] = "leul",
1752 [MO_LESL] = "lesl",
1753 [MO_LEQ] = "leq",
1754 [MO_BEUW] = "beuw",
1755 [MO_BESW] = "besw",
1756 [MO_BEUL] = "beul",
1757 [MO_BESL] = "besl",
1758 [MO_BEQ] = "beq",
1759};
1760
1f00b27f 1761static const char * const alignment_name[(MO_AMASK >> MO_ASHIFT) + 1] = {
52bf9771 1762#ifdef TARGET_ALIGNED_ONLY
1f00b27f
SS		 1763 [MO_UNALN >> MO_ASHIFT] = "un+",
1764 [MO_ALIGN >> MO_ASHIFT] = "",
1765#else
1766 [MO_UNALN >> MO_ASHIFT] = "",
1767 [MO_ALIGN >> MO_ASHIFT] = "al+",
1768#endif
1769 [MO_ALIGN_2 >> MO_ASHIFT] = "al2+",
1770 [MO_ALIGN_4 >> MO_ASHIFT] = "al4+",
1771 [MO_ALIGN_8 >> MO_ASHIFT] = "al8+",
1772 [MO_ALIGN_16 >> MO_ASHIFT] = "al16+",
1773 [MO_ALIGN_32 >> MO_ASHIFT] = "al32+",
1774 [MO_ALIGN_64 >> MO_ASHIFT] = "al64+",
1775};
1776
587195bd
RH		 1777static const char bswap_flag_name[][6] = {
1778 [TCG_BSWAP_IZ] = "iz",
1779 [TCG_BSWAP_OZ] = "oz",
1780 [TCG_BSWAP_OS] = "os",
1781 [TCG_BSWAP_IZ | TCG_BSWAP_OZ] = "iz,oz",
1782 [TCG_BSWAP_IZ | TCG_BSWAP_OS] = "iz,os",
1783};
1784
b016486e
RH		 1785static inline bool tcg_regset_single(TCGRegSet d)
1786{
1787 return (d & (d - 1)) == 0;
1788}
1789
1790static inline TCGReg tcg_regset_first(TCGRegSet d)
1791{
1792 if (TCG_TARGET_NB_REGS <= 32) {
1793 return ctz32(d);
1794 } else {
1795 return ctz64(d);
1796 }
1797}
1798
1894f69a 1799static void tcg_dump_ops(TCGContext *s, bool have_prefs)
c896fe29 1800{
c896fe29 1801 char buf[128];
c45cb8bb 1802 TCGOp *op;
c45cb8bb 1803
15fa08f8 1804 QTAILQ_FOREACH(op, &s->ops, link) {
c45cb8bb
RH		 1805 int i, k, nb_oargs, nb_iargs, nb_cargs;
1806 const TCGOpDef *def;
c45cb8bb 1807 TCGOpcode c;
bdfb460e 1808 int col = 0;
c896fe29 1809
c45cb8bb 1810 c = op->opc;
c896fe29 1811 def = &tcg_op_defs[c];
c45cb8bb 1812
765b842a 1813 if (c == INDEX_op_insn_start) {
b016486e 1814 nb_oargs = 0;
15fa08f8 1815 col += qemu_log("\n ----");
9aef40ed
RH		 1816
1817 for (i = 0; i < TARGET_INSN_START_WORDS; ++i) {
1818 target_ulong a;
7e4597d7 1819#if TARGET_LONG_BITS > TCG_TARGET_REG_BITS
efee3746 1820 a = deposit64(op->args[i * 2], 32, 32, op->args[i * 2 + 1]);
7e4597d7 1821#else
efee3746 1822 a = op->args[i];
7e4597d7 1823#endif
bdfb460e 1824 col += qemu_log(" " TARGET_FMT_lx, a);
eeacee4d 1825 }
7e4597d7 1826 } else if (c == INDEX_op_call) {
3e92aa34 1827 const TCGHelperInfo *info = tcg_call_info(op);
fa52e660 1828 void *func = tcg_call_func(op);
3e92aa34 1829
c896fe29 1830 /* variable number of arguments */
cd9090aa
RH		 1831 nb_oargs = TCGOP_CALLO(op);
1832 nb_iargs = TCGOP_CALLI(op);
c896fe29 1833 nb_cargs = def->nb_cargs;
c896fe29 1834
3e92aa34
RH		 1835 col += qemu_log(" %s ", def->name);
1836
1837 /*
1838 * Print the function name from TCGHelperInfo, if available.
1839 * Note that plugins have a template function for the info,
1840 * but the actual function pointer comes from the plugin.
1841 */
3e92aa34
RH		 1842 if (func == info->func) {
1843 col += qemu_log("%s", info->name);
1844 } else {
1845 col += qemu_log("plugin(%p)", func);
1846 }
1847
8973fe43 1848 col += qemu_log(",$0x%x,$%d", info->flags, nb_oargs);
cf066674 1849 for (i = 0; i < nb_oargs; i++) {
43439139
RH		 1850 col += qemu_log(",%s", tcg_get_arg_str(s, buf, sizeof(buf),
1851 op->args[i]));
b03cce8e 1852 }
cf066674 1853 for (i = 0; i < nb_iargs; i++) {
efee3746 1854 TCGArg arg = op->args[nb_oargs + i];
cf066674
RH		 1855 const char *t = "<dummy>";
1856 if (arg != TCG_CALL_DUMMY_ARG) {
43439139 1857 t = tcg_get_arg_str(s, buf, sizeof(buf), arg);
eeacee4d 1858 }
bdfb460e 1859 col += qemu_log(",%s", t);
e8996ee0 1860 }
b03cce8e 1861 } else {
bdfb460e 1862 col += qemu_log(" %s ", def->name);
c45cb8bb
RH		 1863
1864 nb_oargs = def->nb_oargs;
1865 nb_iargs = def->nb_iargs;
1866 nb_cargs = def->nb_cargs;
1867
d2fd745f
RH		 1868 if (def->flags & TCG_OPF_VECTOR) {
1869 col += qemu_log("v%d,e%d,", 64 << TCGOP_VECL(op),
1870 8 << TCGOP_VECE(op));
1871 }
1872
b03cce8e 1873 k = 0;
c45cb8bb 1874 for (i = 0; i < nb_oargs; i++) {
eeacee4d 1875 if (k != 0) {
bdfb460e 1876 col += qemu_log(",");
eeacee4d 1877 }
43439139
RH		 1878 col += qemu_log("%s", tcg_get_arg_str(s, buf, sizeof(buf),
1879 op->args[k++]));
b03cce8e 1880 }
c45cb8bb 1881 for (i = 0; i < nb_iargs; i++) {
eeacee4d 1882 if (k != 0) {
bdfb460e 1883 col += qemu_log(",");
eeacee4d 1884 }
43439139
RH		 1885 col += qemu_log("%s", tcg_get_arg_str(s, buf, sizeof(buf),
1886 op->args[k++]));
b03cce8e 1887 }
be210acb
RH		 1888 switch (c) {
1889 case INDEX_op_brcond_i32:
be210acb 1890 case INDEX_op_setcond_i32:
ffc5ea09 1891 case INDEX_op_movcond_i32:
ffc5ea09 1892 case INDEX_op_brcond2_i32:
be210acb 1893 case INDEX_op_setcond2_i32:
ffc5ea09 1894 case INDEX_op_brcond_i64:
be210acb 1895 case INDEX_op_setcond_i64:
ffc5ea09 1896 case INDEX_op_movcond_i64:
212be173 1897 case INDEX_op_cmp_vec:
f75da298 1898 case INDEX_op_cmpsel_vec:
efee3746
RH		 1899 if (op->args[k] < ARRAY_SIZE(cond_name)
1900 && cond_name[op->args[k]]) {
1901 col += qemu_log(",%s", cond_name[op->args[k++]]);
eeacee4d 1902 } else {
efee3746 1903 col += qemu_log(",$0x%" TCG_PRIlx, op->args[k++]);
eeacee4d 1904 }
f48f3ede 1905 i = 1;
be210acb 1906 break;
f713d6ad
RH		 1907 case INDEX_op_qemu_ld_i32:
1908 case INDEX_op_qemu_st_i32:
07ce0b05 1909 case INDEX_op_qemu_st8_i32:
f713d6ad
RH		 1910 case INDEX_op_qemu_ld_i64:
1911 case INDEX_op_qemu_st_i64:
59227d5d 1912 {
efee3746 1913 TCGMemOpIdx oi = op->args[k++];
14776ab5 1914 MemOp op = get_memop(oi);
59227d5d
RH		 1915 unsigned ix = get_mmuidx(oi);
1916
59c4b7e8 1917 if (op & ~(MO_AMASK | MO_BSWAP | MO_SSIZE)) {
bdfb460e 1918 col += qemu_log(",$0x%x,%u", op, ix);
59c4b7e8 1919 } else {
1f00b27f
SS		 1920 const char *s_al, *s_op;
1921 s_al = alignment_name[(op & MO_AMASK) >> MO_ASHIFT];
59c4b7e8 1922 s_op = ldst_name[op & (MO_BSWAP | MO_SSIZE)];
bdfb460e 1923 col += qemu_log(",%s%s,%u", s_al, s_op, ix);
59227d5d
RH		 1924 }
1925 i = 1;
f713d6ad 1926 }
f713d6ad 1927 break;
587195bd
RH		 1928 case INDEX_op_bswap16_i32:
1929 case INDEX_op_bswap16_i64:
1930 case INDEX_op_bswap32_i32:
1931 case INDEX_op_bswap32_i64:
1932 case INDEX_op_bswap64_i64:
1933 {
1934 TCGArg flags = op->args[k];
1935 const char *name = NULL;
1936
1937 if (flags < ARRAY_SIZE(bswap_flag_name)) {
1938 name = bswap_flag_name[flags];
1939 }
1940 if (name) {
1941 col += qemu_log(",%s", name);
1942 } else {
1943 col += qemu_log(",$0x%" TCG_PRIlx, flags);
1944 }
1945 i = k = 1;
1946 }
1947 break;
be210acb 1948 default:
f48f3ede 1949 i = 0;
be210acb
RH		 1950 break;
1951 }
51e3972c
RH		 1952 switch (c) {
1953 case INDEX_op_set_label:
1954 case INDEX_op_br:
1955 case INDEX_op_brcond_i32:
1956 case INDEX_op_brcond_i64:
1957 case INDEX_op_brcond2_i32:
efee3746
RH		 1958 col += qemu_log("%s$L%d", k ? "," : "",
1959 arg_label(op->args[k])->id);
51e3972c
RH		 1960 i++, k++;
1961 break;
1962 default:
1963 break;
1964 }
1965 for (; i < nb_cargs; i++, k++) {
efee3746 1966 col += qemu_log("%s$0x%" TCG_PRIlx, k ? "," : "", op->args[k]);
bdfb460e
RH		 1967 }
1968 }
bdfb460e 1969
1894f69a 1970 if (have_prefs || op->life) {
7606488c
RF		 1971
1972 QemuLogFile *logfile;
1973
1974 rcu_read_lock();
d73415a3 1975 logfile = qatomic_rcu_read(&qemu_logfile);
7606488c
RF		 1976 if (logfile) {
1977 for (; col < 40; ++col) {
1978 putc(' ', logfile->fd);
1979 }
bdfb460e 1980 }
7606488c 1981 rcu_read_unlock();
1894f69a
RH		 1982 }
1983
1984 if (op->life) {
1985 unsigned life = op->life;
bdfb460e
RH		 1986
1987 if (life & (SYNC_ARG * 3)) {
1988 qemu_log(" sync:");
1989 for (i = 0; i < 2; ++i) {
1990 if (life & (SYNC_ARG << i)) {
1991 qemu_log(" %d", i);
1992 }
1993 }
1994 }
1995 life /= DEAD_ARG;
1996 if (life) {
1997 qemu_log(" dead:");
1998 for (i = 0; life; ++i, life >>= 1) {
1999 if (life & 1) {
2000 qemu_log(" %d", i);
2001 }
2002 }
b03cce8e 2003 }
c896fe29 2004 }
1894f69a
RH		 2005
2006 if (have_prefs) {
2007 for (i = 0; i < nb_oargs; ++i) {
2008 TCGRegSet set = op->output_pref[i];
2009
2010 if (i == 0) {
2011 qemu_log(" pref=");
2012 } else {
2013 qemu_log(",");
2014 }
2015 if (set == 0) {
2016 qemu_log("none");
2017 } else if (set == MAKE_64BIT_MASK(0, TCG_TARGET_NB_REGS)) {
2018 qemu_log("all");
2019#ifdef CONFIG_DEBUG_TCG
2020 } else if (tcg_regset_single(set)) {
2021 TCGReg reg = tcg_regset_first(set);
2022 qemu_log("%s", tcg_target_reg_names[reg]);
2023#endif
2024 } else if (TCG_TARGET_NB_REGS <= 32) {
2025 qemu_log("%#x", (uint32_t)set);
2026 } else {
2027 qemu_log("%#" PRIx64, (uint64_t)set);
2028 }
2029 }
2030 }
2031
eeacee4d 2032 qemu_log("\n");
c896fe29
FB		 2033 }
2034}
2035
2036/* we give more priority to constraints with less registers */
2037static int get_constraint_priority(const TCGOpDef *def, int k)
2038{
74a11790
RH		 2039 const TCGArgConstraint *arg_ct = &def->args_ct[k];
2040 int n;
c896fe29 2041
bc2b17e6 2042 if (arg_ct->oalias) {
c896fe29
FB		 2043 /* an alias is equivalent to a single register */
2044 n = 1;
2045 } else {
74a11790 2046 n = ctpop64(arg_ct->regs);
c896fe29
FB		 2047 }
2048 return TCG_TARGET_NB_REGS - n + 1;
2049}
2050
2051/* sort from highest priority to lowest */
2052static void sort_constraints(TCGOpDef *def, int start, int n)
2053{
66792f90
RH		 2054 int i, j;
2055 TCGArgConstraint *a = def->args_ct;
c896fe29 2056
66792f90
RH		 2057 for (i = 0; i < n; i++) {
2058 a[start + i].sort_index = start + i;
2059 }
2060 if (n <= 1) {
c896fe29 2061 return;
66792f90
RH		 2062 }
2063 for (i = 0; i < n - 1; i++) {
2064 for (j = i + 1; j < n; j++) {
2065 int p1 = get_constraint_priority(def, a[start + i].sort_index);
2066 int p2 = get_constraint_priority(def, a[start + j].sort_index);
c896fe29 2067 if (p1 < p2) {
66792f90
RH		 2068 int tmp = a[start + i].sort_index;
2069 a[start + i].sort_index = a[start + j].sort_index;
2070 a[start + j].sort_index = tmp;
c896fe29
FB		 2071 }
2072 }
2073 }
2074}
2075
f69d277e 2076static void process_op_defs(TCGContext *s)
c896fe29 2077{
a9751609 2078 TCGOpcode op;
c896fe29 2079
f69d277e
RH		 2080 for (op = 0; op < NB_OPS; op++) {
2081 TCGOpDef *def = &tcg_op_defs[op];
2082 const TCGTargetOpDef *tdefs;
069ea736 2083 int i, nb_args;
f69d277e
RH		 2084
2085 if (def->flags & TCG_OPF_NOT_PRESENT) {
2086 continue;
2087 }
2088
c896fe29 2089 nb_args = def->nb_iargs + def->nb_oargs;
f69d277e
RH		 2090 if (nb_args == 0) {
2091 continue;
2092 }
2093
4c22e840
RH		 2094 /*
2095 * Macro magic should make it impossible, but double-check that
2096 * the array index is in range. Since the signness of an enum
2097 * is implementation defined, force the result to unsigned.
2098 */
2099 unsigned con_set = tcg_target_op_def(op);
2100 tcg_debug_assert(con_set < ARRAY_SIZE(constraint_sets));
2101 tdefs = &constraint_sets[con_set];
f69d277e
RH		 2102
2103 for (i = 0; i < nb_args; i++) {
2104 const char *ct_str = tdefs->args_ct_str[i];
2105 /* Incomplete TCGTargetOpDef entry. */
eabb7b91 2106 tcg_debug_assert(ct_str != NULL);
f69d277e 2107
17280ff4
RH		 2108 while (*ct_str != '\0') {
2109 switch(*ct_str) {
2110 case '0' ... '9':
2111 {
2112 int oarg = *ct_str - '0';
2113 tcg_debug_assert(ct_str == tdefs->args_ct_str[i]);
2114 tcg_debug_assert(oarg < def->nb_oargs);
74a11790 2115 tcg_debug_assert(def->args_ct[oarg].regs != 0);
17280ff4 2116 def->args_ct[i] = def->args_ct[oarg];
bc2b17e6
RH		 2117 /* The output sets oalias. */
2118 def->args_ct[oarg].oalias = true;
17280ff4 2119 def->args_ct[oarg].alias_index = i;
bc2b17e6
RH		 2120 /* The input sets ialias. */
2121 def->args_ct[i].ialias = true;
17280ff4 2122 def->args_ct[i].alias_index = oarg;
c896fe29 2123 }
17280ff4
RH		 2124 ct_str++;
2125 break;
2126 case '&':
bc2b17e6 2127 def->args_ct[i].newreg = true;
17280ff4
RH		 2128 ct_str++;
2129 break;
2130 case 'i':
2131 def->args_ct[i].ct |= TCG_CT_CONST;
2132 ct_str++;
2133 break;
358b4923 2134
358b4923
RH		 2135 /* Include all of the target-specific constraints. */
2136
2137#undef CONST
2138#define CONST(CASE, MASK) \
2139 case CASE: def->args_ct[i].ct |= MASK; ct_str++; break;
2140#define REGS(CASE, MASK) \
2141 case CASE: def->args_ct[i].regs |= MASK; ct_str++; break;
2142
2143#include "tcg-target-con-str.h"
2144
2145#undef REGS
2146#undef CONST
17280ff4 2147 default:
17280ff4 2148 /* Typo in TCGTargetOpDef constraint. */
358b4923 2149 g_assert_not_reached();
c896fe29
FB		 2150 }
2151 }
2152 }
2153
c68aaa18 2154 /* TCGTargetOpDef entry with too much information? */
eabb7b91 2155 tcg_debug_assert(i == TCG_MAX_OP_ARGS || tdefs->args_ct_str[i] == NULL);
c68aaa18 2156
c896fe29
FB		 2157 /* sort the constraints (XXX: this is just an heuristic) */
2158 sort_constraints(def, 0, def->nb_oargs);
2159 sort_constraints(def, def->nb_oargs, def->nb_iargs);
a9751609 2160 }
c896fe29
FB		 2161}
2162
0c627cdc
RH		 2163void tcg_op_remove(TCGContext *s, TCGOp *op)
2164{
d88a117e
RH		 2165 TCGLabel *label;
2166
2167 switch (op->opc) {
2168 case INDEX_op_br:
2169 label = arg_label(op->args[0]);
2170 label->refs--;
2171 break;
2172 case INDEX_op_brcond_i32:
2173 case INDEX_op_brcond_i64:
2174 label = arg_label(op->args[3]);
2175 label->refs--;
2176 break;
2177 case INDEX_op_brcond2_i32:
2178 label = arg_label(op->args[5]);
2179 label->refs--;
2180 break;
2181 default:
2182 break;
2183 }
2184
15fa08f8
RH		 2185 QTAILQ_REMOVE(&s->ops, op, link);
2186 QTAILQ_INSERT_TAIL(&s->free_ops, op, link);
abebf925 2187 s->nb_ops--;
0c627cdc
RH		 2188
2189#ifdef CONFIG_PROFILER
d73415a3 2190 qatomic_set(&s->prof.del_op_count, s->prof.del_op_count + 1);
0c627cdc
RH		 2191#endif
2192}
2193
a80cdd31
RH		 2194void tcg_remove_ops_after(TCGOp *op)
2195{
2196 TCGContext *s = tcg_ctx;
2197
2198 while (true) {
2199 TCGOp *last = tcg_last_op();
2200 if (last == op) {
2201 return;
2202 }
2203 tcg_op_remove(s, last);
2204 }
2205}
2206
15fa08f8 2207static TCGOp *tcg_op_alloc(TCGOpcode opc)
5a18407f 2208{
15fa08f8
RH		 2209 TCGContext *s = tcg_ctx;
2210 TCGOp *op;
5a18407f 2211
15fa08f8
RH		 2212 if (likely(QTAILQ_EMPTY(&s->free_ops))) {
2213 op = tcg_malloc(sizeof(TCGOp));
2214 } else {
2215 op = QTAILQ_FIRST(&s->free_ops);
2216 QTAILQ_REMOVE(&s->free_ops, op, link);
2217 }
2218 memset(op, 0, offsetof(TCGOp, link));
2219 op->opc = opc;
abebf925 2220 s->nb_ops++;
5a18407f 2221
15fa08f8
RH		 2222 return op;
2223}
2224
2225TCGOp *tcg_emit_op(TCGOpcode opc)
2226{
2227 TCGOp *op = tcg_op_alloc(opc);
2228 QTAILQ_INSERT_TAIL(&tcg_ctx->ops, op, link);
2229 return op;
2230}
5a18407f 2231
ac1043f6 2232TCGOp *tcg_op_insert_before(TCGContext *s, TCGOp *old_op, TCGOpcode opc)
15fa08f8
RH		 2233{
2234 TCGOp *new_op = tcg_op_alloc(opc);
2235 QTAILQ_INSERT_BEFORE(old_op, new_op, link);
5a18407f
RH		 2236 return new_op;
2237}
2238
ac1043f6 2239TCGOp *tcg_op_insert_after(TCGContext *s, TCGOp *old_op, TCGOpcode opc)
5a18407f 2240{
15fa08f8
RH		 2241 TCGOp *new_op = tcg_op_alloc(opc);
2242 QTAILQ_INSERT_AFTER(&s->ops, old_op, new_op, link);
5a18407f
RH		 2243 return new_op;
2244}
2245
b4fc67c7
RH		 2246/* Reachable analysis : remove unreachable code. */
2247static void reachable_code_pass(TCGContext *s)
2248{
2249 TCGOp *op, *op_next;
2250 bool dead = false;
2251
2252 QTAILQ_FOREACH_SAFE(op, &s->ops, link, op_next) {
2253 bool remove = dead;
2254 TCGLabel *label;
b4fc67c7
RH		 2255
2256 switch (op->opc) {
2257 case INDEX_op_set_label:
2258 label = arg_label(op->args[0]);
2259 if (label->refs == 0) {
2260 /*
2261 * While there is an occasional backward branch, virtually
2262 * all branches generated by the translators are forward.
2263 * Which means that generally we will have already removed
2264 * all references to the label that will be, and there is
2265 * little to be gained by iterating.
2266 */
2267 remove = true;
2268 } else {
2269 /* Once we see a label, insns become live again. */
2270 dead = false;
2271 remove = false;
2272
2273 /*
2274 * Optimization can fold conditional branches to unconditional.
2275 * If we find a label with one reference which is preceded by
2276 * an unconditional branch to it, remove both. This needed to
2277 * wait until the dead code in between them was removed.
2278 */
2279 if (label->refs == 1) {
eae3eb3e 2280 TCGOp *op_prev = QTAILQ_PREV(op, link);
b4fc67c7
RH		 2281 if (op_prev->opc == INDEX_op_br &&
2282 label == arg_label(op_prev->args[0])) {
2283 tcg_op_remove(s, op_prev);
2284 remove = true;
2285 }
2286 }
2287 }
2288 break;
2289
2290 case INDEX_op_br:
2291 case INDEX_op_exit_tb:
2292 case INDEX_op_goto_ptr:
2293 /* Unconditional branches; everything following is dead. */
2294 dead = true;
2295 break;
2296
2297 case INDEX_op_call:
2298 /* Notice noreturn helper calls, raising exceptions. */
90163900 2299 if (tcg_call_flags(op) & TCG_CALL_NO_RETURN) {
b4fc67c7
RH		 2300 dead = true;
2301 }
2302 break;
2303
2304 case INDEX_op_insn_start:
2305 /* Never remove -- we need to keep these for unwind. */
2306 remove = false;
2307 break;
2308
2309 default:
2310 break;
2311 }
2312
2313 if (remove) {
2314 tcg_op_remove(s, op);
2315 }
2316 }
2317}
2318
c70fbf0a
RH		 2319#define TS_DEAD 1
2320#define TS_MEM 2
2321
5a18407f
RH		 2322#define IS_DEAD_ARG(n) (arg_life & (DEAD_ARG << (n)))
2323#define NEED_SYNC_ARG(n) (arg_life & (SYNC_ARG << (n)))
2324
25f49c5f
RH		 2325/* For liveness_pass_1, the register preferences for a given temp. */
2326static inline TCGRegSet *la_temp_pref(TCGTemp *ts)
2327{
2328 return ts->state_ptr;
2329}
2330
2331/* For liveness_pass_1, reset the preferences for a given temp to the
2332 * maximal regset for its type.
2333 */
2334static inline void la_reset_pref(TCGTemp *ts)
2335{
2336 *la_temp_pref(ts)
2337 = (ts->state == TS_DEAD ? 0 : tcg_target_available_regs[ts->type]);
2338}
2339
9c43b68d
AJ		 2340/* liveness analysis: end of function: all temps are dead, and globals
2341 should be in memory. */
2616c808 2342static void la_func_end(TCGContext *s, int ng, int nt)
c896fe29 2343{
b83eabea
RH		 2344 int i;
2345
2346 for (i = 0; i < ng; ++i) {
2347 s->temps[i].state = TS_DEAD | TS_MEM;
25f49c5f 2348 la_reset_pref(&s->temps[i]);
b83eabea
RH		 2349 }
2350 for (i = ng; i < nt; ++i) {
2351 s->temps[i].state = TS_DEAD;
25f49c5f 2352 la_reset_pref(&s->temps[i]);
b83eabea 2353 }
c896fe29
FB		 2354}
2355
9c43b68d
AJ		 2356/* liveness analysis: end of basic block: all temps are dead, globals
2357 and local temps should be in memory. */
2616c808 2358static void la_bb_end(TCGContext *s, int ng, int nt)
641d5fbe 2359{
b83eabea 2360 int i;
641d5fbe 2361
ee17db83
RH		 2362 for (i = 0; i < nt; ++i) {
2363 TCGTemp *ts = &s->temps[i];
2364 int state;
2365
2366 switch (ts->kind) {
2367 case TEMP_FIXED:
2368 case TEMP_GLOBAL:
2369 case TEMP_LOCAL:
2370 state = TS_DEAD | TS_MEM;
2371 break;
2372 case TEMP_NORMAL:
c0522136 2373 case TEMP_CONST:
ee17db83
RH		 2374 state = TS_DEAD;
2375 break;
2376 default:
2377 g_assert_not_reached();
2378 }
2379 ts->state = state;
2380 la_reset_pref(ts);
641d5fbe
FB		 2381 }
2382}
2383
f65a061c
RH		 2384/* liveness analysis: sync globals back to memory. */
2385static void la_global_sync(TCGContext *s, int ng)
2386{
2387 int i;
2388
2389 for (i = 0; i < ng; ++i) {
25f49c5f
RH		 2390 int state = s->temps[i].state;
2391 s->temps[i].state = state | TS_MEM;
2392 if (state == TS_DEAD) {
2393 /* If the global was previously dead, reset prefs. */
2394 la_reset_pref(&s->temps[i]);
2395 }
f65a061c
RH		 2396 }
2397}
2398
b4cb76e6
RH		 2399/*
2400 * liveness analysis: conditional branch: all temps are dead,
2401 * globals and local temps should be synced.
2402 */
2403static void la_bb_sync(TCGContext *s, int ng, int nt)
2404{
2405 la_global_sync(s, ng);
2406
2407 for (int i = ng; i < nt; ++i) {
c0522136
RH		 2408 TCGTemp *ts = &s->temps[i];
2409 int state;
2410
2411 switch (ts->kind) {
2412 case TEMP_LOCAL:
2413 state = ts->state;
2414 ts->state = state | TS_MEM;
b4cb76e6
RH		 2415 if (state != TS_DEAD) {
2416 continue;
2417 }
c0522136
RH		 2418 break;
2419 case TEMP_NORMAL:
b4cb76e6 2420 s->temps[i].state = TS_DEAD;
c0522136
RH		 2421 break;
2422 case TEMP_CONST:
2423 continue;
2424 default:
2425 g_assert_not_reached();
b4cb76e6
RH		 2426 }
2427 la_reset_pref(&s->temps[i]);
2428 }
2429}
2430
f65a061c
RH		 2431/* liveness analysis: sync globals back to memory and kill. */
2432static void la_global_kill(TCGContext *s, int ng)
2433{
2434 int i;
2435
2436 for (i = 0; i < ng; i++) {
2437 s->temps[i].state = TS_DEAD | TS_MEM;
25f49c5f
RH		 2438 la_reset_pref(&s->temps[i]);
2439 }
2440}
2441
2442/* liveness analysis: note live globals crossing calls. */
2443static void la_cross_call(TCGContext *s, int nt)
2444{
2445 TCGRegSet mask = ~tcg_target_call_clobber_regs;
2446 int i;
2447
2448 for (i = 0; i < nt; i++) {
2449 TCGTemp *ts = &s->temps[i];
2450 if (!(ts->state & TS_DEAD)) {
2451 TCGRegSet *pset = la_temp_pref(ts);
2452 TCGRegSet set = *pset;
2453
2454 set &= mask;
2455 /* If the combination is not possible, restart. */
2456 if (set == 0) {
2457 set = tcg_target_available_regs[ts->type] & mask;
2458 }
2459 *pset = set;
2460 }
f65a061c
RH		 2461 }
2462}
2463
a1b3c48d 2464/* Liveness analysis : update the opc_arg_life array to tell if a
c896fe29
FB		 2465 given input arguments is dead. Instructions updating dead
2466 temporaries are removed. */
b83eabea 2467static void liveness_pass_1(TCGContext *s)
c896fe29 2468{
c70fbf0a 2469 int nb_globals = s->nb_globals;
2616c808 2470 int nb_temps = s->nb_temps;
15fa08f8 2471 TCGOp *op, *op_prev;
25f49c5f
RH		 2472 TCGRegSet *prefs;
2473 int i;
2474
2475 prefs = tcg_malloc(sizeof(TCGRegSet) * nb_temps);
2476 for (i = 0; i < nb_temps; ++i) {
2477 s->temps[i].state_ptr = prefs + i;
2478 }
a1b3c48d 2479
ae36a246 2480 /* ??? Should be redundant with the exit_tb that ends the TB. */
2616c808 2481 la_func_end(s, nb_globals, nb_temps);
c896fe29 2482
eae3eb3e 2483 QTAILQ_FOREACH_REVERSE_SAFE(op, &s->ops, link, op_prev) {
25f49c5f 2484 int nb_iargs, nb_oargs;
c45cb8bb
RH		 2485 TCGOpcode opc_new, opc_new2;
2486 bool have_opc_new2;
a1b3c48d 2487 TCGLifeData arg_life = 0;
25f49c5f 2488 TCGTemp *ts;
c45cb8bb
RH		 2489 TCGOpcode opc = op->opc;
2490 const TCGOpDef *def = &tcg_op_defs[opc];
2491
c45cb8bb 2492 switch (opc) {
c896fe29 2493 case INDEX_op_call:
c6e113f5
FB		 2494 {
2495 int call_flags;
25f49c5f 2496 int nb_call_regs;
c896fe29 2497
cd9090aa
RH		 2498 nb_oargs = TCGOP_CALLO(op);
2499 nb_iargs = TCGOP_CALLI(op);
90163900 2500 call_flags = tcg_call_flags(op);
c6e113f5 2501
c45cb8bb 2502 /* pure functions can be removed if their result is unused */
78505279 2503 if (call_flags & TCG_CALL_NO_SIDE_EFFECTS) {
cf066674 2504 for (i = 0; i < nb_oargs; i++) {
25f49c5f
RH		 2505 ts = arg_temp(op->args[i]);
2506 if (ts->state != TS_DEAD) {
c6e113f5 2507 goto do_not_remove_call;
9c43b68d 2508 }
c6e113f5 2509 }
c45cb8bb 2510 goto do_remove;
152c35aa
RH		 2511 }
2512 do_not_remove_call:
c896fe29 2513
25f49c5f 2514 /* Output args are dead. */
152c35aa 2515 for (i = 0; i < nb_oargs; i++) {
25f49c5f
RH		 2516 ts = arg_temp(op->args[i]);
2517 if (ts->state & TS_DEAD) {
152c35aa
RH		 2518 arg_life |= DEAD_ARG << i;
2519 }
25f49c5f 2520 if (ts->state & TS_MEM) {
152c35aa 2521 arg_life |= SYNC_ARG << i;
c6e113f5 2522 }
25f49c5f
RH		 2523 ts->state = TS_DEAD;
2524 la_reset_pref(ts);
2525
2526 /* Not used -- it will be tcg_target_call_oarg_regs[i]. */
2527 op->output_pref[i] = 0;
152c35aa 2528 }
78505279 2529
152c35aa
RH		 2530 if (!(call_flags & (TCG_CALL_NO_WRITE_GLOBALS |
2531 TCG_CALL_NO_READ_GLOBALS))) {
f65a061c 2532 la_global_kill(s, nb_globals);
152c35aa 2533 } else if (!(call_flags & TCG_CALL_NO_READ_GLOBALS)) {
f65a061c 2534 la_global_sync(s, nb_globals);
152c35aa 2535 }
b9c18f56 2536
25f49c5f 2537 /* Record arguments that die in this helper. */
152c35aa 2538 for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) {
25f49c5f
RH		 2539 ts = arg_temp(op->args[i]);
2540 if (ts && ts->state & TS_DEAD) {
152c35aa 2541 arg_life |= DEAD_ARG << i;
c6e113f5 2542 }
152c35aa 2543 }
25f49c5f
RH		 2544
2545 /* For all live registers, remove call-clobbered prefs. */
2546 la_cross_call(s, nb_temps);
2547
2548 nb_call_regs = ARRAY_SIZE(tcg_target_call_iarg_regs);
2549
2550 /* Input arguments are live for preceding opcodes. */
2551 for (i = 0; i < nb_iargs; i++) {
2552 ts = arg_temp(op->args[i + nb_oargs]);
2553 if (ts && ts->state & TS_DEAD) {
2554 /* For those arguments that die, and will be allocated
2555 * in registers, clear the register set for that arg,
2556 * to be filled in below. For args that will be on
2557 * the stack, reset to any available reg.
2558 */
2559 *la_temp_pref(ts)
2560 = (i < nb_call_regs ? 0 :
2561 tcg_target_available_regs[ts->type]);
2562 ts->state &= ~TS_DEAD;
2563 }
2564 }
2565
2566 /* For each input argument, add its input register to prefs.
2567 If a temp is used once, this produces a single set bit. */
2568 for (i = 0; i < MIN(nb_call_regs, nb_iargs); i++) {
2569 ts = arg_temp(op->args[i + nb_oargs]);
2570 if (ts) {
2571 tcg_regset_set_reg(*la_temp_pref(ts),
2572 tcg_target_call_iarg_regs[i]);
c19f47bf 2573 }
c896fe29 2574 }
c896fe29 2575 }
c896fe29 2576 break;
765b842a 2577 case INDEX_op_insn_start:
c896fe29 2578 break;
5ff9d6a4 2579 case INDEX_op_discard:
5ff9d6a4 2580 /* mark the temporary as dead */
25f49c5f
RH		 2581 ts = arg_temp(op->args[0]);
2582 ts->state = TS_DEAD;
2583 la_reset_pref(ts);
5ff9d6a4 2584 break;
1305c451
RH		 2585
2586 case INDEX_op_add2_i32:
c45cb8bb 2587 opc_new = INDEX_op_add_i32;
f1fae40c 2588 goto do_addsub2;
1305c451 2589 case INDEX_op_sub2_i32:
c45cb8bb 2590 opc_new = INDEX_op_sub_i32;
f1fae40c
RH		 2591 goto do_addsub2;
2592 case INDEX_op_add2_i64:
c45cb8bb 2593 opc_new = INDEX_op_add_i64;
f1fae40c
RH		 2594 goto do_addsub2;
2595 case INDEX_op_sub2_i64:
c45cb8bb 2596 opc_new = INDEX_op_sub_i64;
f1fae40c 2597 do_addsub2:
1305c451
RH		 2598 nb_iargs = 4;
2599 nb_oargs = 2;
2600 /* Test if the high part of the operation is dead, but not
2601 the low part. The result can be optimized to a simple
2602 add or sub. This happens often for x86_64 guest when the
2603 cpu mode is set to 32 bit. */
b83eabea
RH		 2604 if (arg_temp(op->args[1])->state == TS_DEAD) {
2605 if (arg_temp(op->args[0])->state == TS_DEAD) {
1305c451
RH		 2606 goto do_remove;
2607 }
c45cb8bb
RH		 2608 /* Replace the opcode and adjust the args in place,
2609 leaving 3 unused args at the end. */
2610 op->opc = opc = opc_new;
efee3746
RH		 2611 op->args[1] = op->args[2];
2612 op->args[2] = op->args[4];
1305c451
RH		 2613 /* Fall through and mark the single-word operation live. */
2614 nb_iargs = 2;
2615 nb_oargs = 1;
2616 }
2617 goto do_not_remove;
2618
1414968a 2619 case INDEX_op_mulu2_i32:
c45cb8bb
RH		 2620 opc_new = INDEX_op_mul_i32;
2621 opc_new2 = INDEX_op_muluh_i32;
2622 have_opc_new2 = TCG_TARGET_HAS_muluh_i32;
03271524 2623 goto do_mul2;
f1fae40c 2624 case INDEX_op_muls2_i32:
c45cb8bb
RH		 2625 opc_new = INDEX_op_mul_i32;
2626 opc_new2 = INDEX_op_mulsh_i32;
2627 have_opc_new2 = TCG_TARGET_HAS_mulsh_i32;
f1fae40c
RH		 2628 goto do_mul2;
2629 case INDEX_op_mulu2_i64:
c45cb8bb
RH		 2630 opc_new = INDEX_op_mul_i64;
2631 opc_new2 = INDEX_op_muluh_i64;
2632 have_opc_new2 = TCG_TARGET_HAS_muluh_i64;
03271524 2633 goto do_mul2;
f1fae40c 2634 case INDEX_op_muls2_i64:
c45cb8bb
RH		 2635 opc_new = INDEX_op_mul_i64;
2636 opc_new2 = INDEX_op_mulsh_i64;
2637 have_opc_new2 = TCG_TARGET_HAS_mulsh_i64;
03271524 2638 goto do_mul2;
f1fae40c 2639 do_mul2:
1414968a
RH		 2640 nb_iargs = 2;
2641 nb_oargs = 2;
b83eabea
RH		 2642 if (arg_temp(op->args[1])->state == TS_DEAD) {
2643 if (arg_temp(op->args[0])->state == TS_DEAD) {
03271524 2644 /* Both parts of the operation are dead. */
1414968a
RH		 2645 goto do_remove;
2646 }
03271524 2647 /* The high part of the operation is dead; generate the low. */
c45cb8bb 2648 op->opc = opc = opc_new;
efee3746
RH		 2649 op->args[1] = op->args[2];
2650 op->args[2] = op->args[3];
b83eabea 2651 } else if (arg_temp(op->args[0])->state == TS_DEAD && have_opc_new2) {
c45cb8bb
RH		 2652 /* The low part of the operation is dead; generate the high. */
2653 op->opc = opc = opc_new2;
efee3746
RH		 2654 op->args[0] = op->args[1];
2655 op->args[1] = op->args[2];
2656 op->args[2] = op->args[3];
03271524
RH		 2657 } else {
2658 goto do_not_remove;
1414968a 2659 }
03271524
RH		 2660 /* Mark the single-word operation live. */
2661 nb_oargs = 1;
1414968a
RH		 2662 goto do_not_remove;
2663
c896fe29 2664 default:
1305c451 2665 /* XXX: optimize by hardcoding common cases (e.g. triadic ops) */
49516bc0
AJ		 2666 nb_iargs = def->nb_iargs;
2667 nb_oargs = def->nb_oargs;
c896fe29 2668
49516bc0
AJ		 2669 /* Test if the operation can be removed because all
2670 its outputs are dead. We assume that nb_oargs == 0
2671 implies side effects */
2672 if (!(def->flags & TCG_OPF_SIDE_EFFECTS) && nb_oargs != 0) {
c45cb8bb 2673 for (i = 0; i < nb_oargs; i++) {
b83eabea 2674 if (arg_temp(op->args[i])->state != TS_DEAD) {
49516bc0 2675 goto do_not_remove;
9c43b68d 2676 }
49516bc0 2677 }
152c35aa
RH		 2678 goto do_remove;
2679 }
2680 goto do_not_remove;
49516bc0 2681
152c35aa
RH		 2682 do_remove:
2683 tcg_op_remove(s, op);
2684 break;
2685
2686 do_not_remove:
152c35aa 2687 for (i = 0; i < nb_oargs; i++) {
25f49c5f
RH		 2688 ts = arg_temp(op->args[i]);
2689
2690 /* Remember the preference of the uses that followed. */
2691 op->output_pref[i] = *la_temp_pref(ts);
2692
2693 /* Output args are dead. */
2694 if (ts->state & TS_DEAD) {
152c35aa 2695 arg_life |= DEAD_ARG << i;
49516bc0 2696 }
25f49c5f 2697 if (ts->state & TS_MEM) {
152c35aa
RH		 2698 arg_life |= SYNC_ARG << i;
2699 }
25f49c5f
RH		 2700 ts->state = TS_DEAD;
2701 la_reset_pref(ts);
152c35aa 2702 }
49516bc0 2703
25f49c5f 2704 /* If end of basic block, update. */
ae36a246
RH		 2705 if (def->flags & TCG_OPF_BB_EXIT) {
2706 la_func_end(s, nb_globals, nb_temps);
b4cb76e6
RH		 2707 } else if (def->flags & TCG_OPF_COND_BRANCH) {
2708 la_bb_sync(s, nb_globals, nb_temps);
ae36a246 2709 } else if (def->flags & TCG_OPF_BB_END) {
2616c808 2710 la_bb_end(s, nb_globals, nb_temps);
152c35aa 2711 } else if (def->flags & TCG_OPF_SIDE_EFFECTS) {
f65a061c 2712 la_global_sync(s, nb_globals);
25f49c5f
RH		 2713 if (def->flags & TCG_OPF_CALL_CLOBBER) {
2714 la_cross_call(s, nb_temps);
2715 }
152c35aa
RH		 2716 }
2717
25f49c5f 2718 /* Record arguments that die in this opcode. */
152c35aa 2719 for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) {
25f49c5f
RH		 2720 ts = arg_temp(op->args[i]);
2721 if (ts->state & TS_DEAD) {
152c35aa 2722 arg_life |= DEAD_ARG << i;
c896fe29 2723 }
c896fe29 2724 }
25f49c5f
RH		 2725
2726 /* Input arguments are live for preceding opcodes. */
152c35aa 2727 for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) {
25f49c5f
RH		 2728 ts = arg_temp(op->args[i]);
2729 if (ts->state & TS_DEAD) {
2730 /* For operands that were dead, initially allow
2731 all regs for the type. */
2732 *la_temp_pref(ts) = tcg_target_available_regs[ts->type];
2733 ts->state &= ~TS_DEAD;
2734 }
2735 }
2736
2737 /* Incorporate constraints for this operand. */
2738 switch (opc) {
2739 case INDEX_op_mov_i32:
2740 case INDEX_op_mov_i64:
2741 /* Note that these are TCG_OPF_NOT_PRESENT and do not
2742 have proper constraints. That said, special case
2743 moves to propagate preferences backward. */
2744 if (IS_DEAD_ARG(1)) {
2745 *la_temp_pref(arg_temp(op->args[0]))
2746 = *la_temp_pref(arg_temp(op->args[1]));
2747 }
2748 break;
2749
2750 default:
2751 for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) {
2752 const TCGArgConstraint *ct = &def->args_ct[i];
2753 TCGRegSet set, *pset;
2754
2755 ts = arg_temp(op->args[i]);
2756 pset = la_temp_pref(ts);
2757 set = *pset;
2758
9be0d080 2759 set &= ct->regs;
bc2b17e6 2760 if (ct->ialias) {
25f49c5f
RH		 2761 set &= op->output_pref[ct->alias_index];
2762 }
2763 /* If the combination is not possible, restart. */
2764 if (set == 0) {
9be0d080 2765 set = ct->regs;
25f49c5f
RH		 2766 }
2767 *pset = set;
2768 }
2769 break;
152c35aa 2770 }
c896fe29
FB		 2771 break;
2772 }
bee158cb 2773 op->life = arg_life;
1ff0a2c5 2774 }
c896fe29 2775}
c896fe29 2776
5a18407f 2777/* Liveness analysis: Convert indirect regs to direct temporaries. */
b83eabea 2778static bool liveness_pass_2(TCGContext *s)
5a18407f
RH		 2779{
2780 int nb_globals = s->nb_globals;
15fa08f8 2781 int nb_temps, i;
5a18407f 2782 bool changes = false;
15fa08f8 2783 TCGOp *op, *op_next;
5a18407f 2784
5a18407f
RH		 2785 /* Create a temporary for each indirect global. */
2786 for (i = 0; i < nb_globals; ++i) {
2787 TCGTemp *its = &s->temps[i];
2788 if (its->indirect_reg) {
2789 TCGTemp *dts = tcg_temp_alloc(s);
2790 dts->type = its->type;
2791 dts->base_type = its->base_type;
b83eabea
RH		 2792 its->state_ptr = dts;
2793 } else {
2794 its->state_ptr = NULL;
5a18407f 2795 }
b83eabea
RH		 2796 /* All globals begin dead. */
2797 its->state = TS_DEAD;
2798 }
2799 for (nb_temps = s->nb_temps; i < nb_temps; ++i) {
2800 TCGTemp *its = &s->temps[i];
2801 its->state_ptr = NULL;
2802 its->state = TS_DEAD;
5a18407f 2803 }
5a18407f 2804
15fa08f8 2805 QTAILQ_FOREACH_SAFE(op, &s->ops, link, op_next) {
5a18407f
RH		 2806 TCGOpcode opc = op->opc;
2807 const TCGOpDef *def = &tcg_op_defs[opc];
2808 TCGLifeData arg_life = op->life;
2809 int nb_iargs, nb_oargs, call_flags;
b83eabea 2810 TCGTemp *arg_ts, *dir_ts;
5a18407f 2811
5a18407f 2812 if (opc == INDEX_op_call) {
cd9090aa
RH		 2813 nb_oargs = TCGOP_CALLO(op);
2814 nb_iargs = TCGOP_CALLI(op);
90163900 2815 call_flags = tcg_call_flags(op);
5a18407f
RH		 2816 } else {
2817 nb_iargs = def->nb_iargs;
2818 nb_oargs = def->nb_oargs;
2819
2820 /* Set flags similar to how calls require. */
b4cb76e6
RH		 2821 if (def->flags & TCG_OPF_COND_BRANCH) {
2822 /* Like reading globals: sync_globals */
2823 call_flags = TCG_CALL_NO_WRITE_GLOBALS;
2824 } else if (def->flags & TCG_OPF_BB_END) {
5a18407f
RH		 2825 /* Like writing globals: save_globals */
2826 call_flags = 0;
2827 } else if (def->flags & TCG_OPF_SIDE_EFFECTS) {
2828 /* Like reading globals: sync_globals */
2829 call_flags = TCG_CALL_NO_WRITE_GLOBALS;
2830 } else {
2831 /* No effect on globals. */
2832 call_flags = (TCG_CALL_NO_READ_GLOBALS |
2833 TCG_CALL_NO_WRITE_GLOBALS);
2834 }
2835 }
2836
2837 /* Make sure that input arguments are available. */
2838 for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) {
b83eabea
RH		 2839 arg_ts = arg_temp(op->args[i]);
2840 if (arg_ts) {
2841 dir_ts = arg_ts->state_ptr;
2842 if (dir_ts && arg_ts->state == TS_DEAD) {
2843 TCGOpcode lopc = (arg_ts->type == TCG_TYPE_I32
5a18407f
RH		 2844 ? INDEX_op_ld_i32
2845 : INDEX_op_ld_i64);
ac1043f6 2846 TCGOp *lop = tcg_op_insert_before(s, op, lopc);
5a18407f 2847
b83eabea
RH		 2848 lop->args[0] = temp_arg(dir_ts);
2849 lop->args[1] = temp_arg(arg_ts->mem_base);
2850 lop->args[2] = arg_ts->mem_offset;
5a18407f
RH		 2851
2852 /* Loaded, but synced with memory. */
b83eabea 2853 arg_ts->state = TS_MEM;
5a18407f
RH		 2854 }
2855 }
2856 }
2857
2858 /* Perform input replacement, and mark inputs that became dead.
2859 No action is required except keeping temp_state up to date
2860 so that we reload when needed. */
2861 for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) {
b83eabea
RH		 2862 arg_ts = arg_temp(op->args[i]);
2863 if (arg_ts) {
2864 dir_ts = arg_ts->state_ptr;
2865 if (dir_ts) {
2866 op->args[i] = temp_arg(dir_ts);
5a18407f
RH		 2867 changes = true;
2868 if (IS_DEAD_ARG(i)) {
b83eabea 2869 arg_ts->state = TS_DEAD;
5a18407f
RH		 2870 }
2871 }
2872 }
2873 }
2874
2875 /* Liveness analysis should ensure that the following are
2876 all correct, for call sites and basic block end points. */
2877 if (call_flags & TCG_CALL_NO_READ_GLOBALS) {
2878 /* Nothing to do */
2879 } else if (call_flags & TCG_CALL_NO_WRITE_GLOBALS) {
2880 for (i = 0; i < nb_globals; ++i) {
2881 /* Liveness should see that globals are synced back,
2882 that is, either TS_DEAD or TS_MEM. */
b83eabea
RH		 2883 arg_ts = &s->temps[i];
2884 tcg_debug_assert(arg_ts->state_ptr == 0
2885 || arg_ts->state != 0);
5a18407f
RH		 2886 }
2887 } else {
2888 for (i = 0; i < nb_globals; ++i) {
2889 /* Liveness should see that globals are saved back,
2890 that is, TS_DEAD, waiting to be reloaded. */
b83eabea
RH		 2891 arg_ts = &s->temps[i];
2892 tcg_debug_assert(arg_ts->state_ptr == 0
2893 || arg_ts->state == TS_DEAD);
5a18407f
RH		 2894 }
2895 }
2896
2897 /* Outputs become available. */
61f15c48
RH		 2898 if (opc == INDEX_op_mov_i32 || opc == INDEX_op_mov_i64) {
2899 arg_ts = arg_temp(op->args[0]);
b83eabea 2900 dir_ts = arg_ts->state_ptr;
61f15c48
RH		 2901 if (dir_ts) {
2902 op->args[0] = temp_arg(dir_ts);
2903 changes = true;
2904
2905 /* The output is now live and modified. */
2906 arg_ts->state = 0;
2907
2908 if (NEED_SYNC_ARG(0)) {
2909 TCGOpcode sopc = (arg_ts->type == TCG_TYPE_I32
2910 ? INDEX_op_st_i32
2911 : INDEX_op_st_i64);
2912 TCGOp *sop = tcg_op_insert_after(s, op, sopc);
2913 TCGTemp *out_ts = dir_ts;
2914
2915 if (IS_DEAD_ARG(0)) {
2916 out_ts = arg_temp(op->args[1]);
2917 arg_ts->state = TS_DEAD;
2918 tcg_op_remove(s, op);
2919 } else {
2920 arg_ts->state = TS_MEM;
2921 }
2922
2923 sop->args[0] = temp_arg(out_ts);
2924 sop->args[1] = temp_arg(arg_ts->mem_base);
2925 sop->args[2] = arg_ts->mem_offset;
2926 } else {
2927 tcg_debug_assert(!IS_DEAD_ARG(0));
2928 }
5a18407f 2929 }
61f15c48
RH		 2930 } else {
2931 for (i = 0; i < nb_oargs; i++) {
2932 arg_ts = arg_temp(op->args[i]);
2933 dir_ts = arg_ts->state_ptr;
2934 if (!dir_ts) {
2935 continue;
2936 }
2937 op->args[i] = temp_arg(dir_ts);
2938 changes = true;
5a18407f 2939
61f15c48
RH		 2940 /* The output is now live and modified. */
2941 arg_ts->state = 0;
5a18407f 2942
61f15c48
RH		 2943 /* Sync outputs upon their last write. */
2944 if (NEED_SYNC_ARG(i)) {
2945 TCGOpcode sopc = (arg_ts->type == TCG_TYPE_I32
2946 ? INDEX_op_st_i32
2947 : INDEX_op_st_i64);
2948 TCGOp *sop = tcg_op_insert_after(s, op, sopc);
5a18407f 2949
61f15c48
RH		 2950 sop->args[0] = temp_arg(dir_ts);
2951 sop->args[1] = temp_arg(arg_ts->mem_base);
2952 sop->args[2] = arg_ts->mem_offset;
5a18407f 2953
61f15c48
RH		 2954 arg_ts->state = TS_MEM;
2955 }
2956 /* Drop outputs that are dead. */
2957 if (IS_DEAD_ARG(i)) {
2958 arg_ts->state = TS_DEAD;
2959 }
5a18407f
RH		 2960 }
2961 }
2962 }
2963
2964 return changes;
2965}
2966
8d8fdbae 2967#ifdef CONFIG_DEBUG_TCG
c896fe29
FB		 2968static void dump_regs(TCGContext *s)
2969{
2970 TCGTemp *ts;
2971 int i;
2972 char buf[64];
2973
2974 for(i = 0; i < s->nb_temps; i++) {
2975 ts = &s->temps[i];
43439139 2976 printf(" %10s: ", tcg_get_arg_str_ptr(s, buf, sizeof(buf), ts));
c896fe29
FB		 2977 switch(ts->val_type) {
2978 case TEMP_VAL_REG:
2979 printf("%s", tcg_target_reg_names[ts->reg]);
2980 break;
2981 case TEMP_VAL_MEM:
b3a62939
RH		 2982 printf("%d(%s)", (int)ts->mem_offset,
2983 tcg_target_reg_names[ts->mem_base->reg]);
c896fe29
FB		 2984 break;
2985 case TEMP_VAL_CONST:
bdb38b95 2986 printf("$0x%" PRIx64, ts->val);
c896fe29
FB		 2987 break;
2988 case TEMP_VAL_DEAD:
2989 printf("D");
2990 break;
2991 default:
2992 printf("???");
2993 break;
2994 }
2995 printf("\n");
2996 }
2997
2998 for(i = 0; i < TCG_TARGET_NB_REGS; i++) {
f8b2f202 2999 if (s->reg_to_temp[i] != NULL) {
c896fe29
FB		 3000 printf("%s: %s\n",
3001 tcg_target_reg_names[i],
f8b2f202 3002 tcg_get_arg_str_ptr(s, buf, sizeof(buf), s->reg_to_temp[i]));
c896fe29
FB		 3003 }
3004 }
3005}
3006
3007static void check_regs(TCGContext *s)
3008{
869938ae 3009 int reg;
b6638662 3010 int k;
c896fe29
FB		 3011 TCGTemp *ts;
3012 char buf[64];
3013
f8b2f202
RH		 3014 for (reg = 0; reg < TCG_TARGET_NB_REGS; reg++) {
3015 ts = s->reg_to_temp[reg];
3016 if (ts != NULL) {
3017 if (ts->val_type != TEMP_VAL_REG || ts->reg != reg) {
c896fe29
FB		 3018 printf("Inconsistency for register %s:\n",
3019 tcg_target_reg_names[reg]);
b03cce8e 3020 goto fail;
c896fe29
FB		 3021 }
3022 }
3023 }
f8b2f202 3024 for (k = 0; k < s->nb_temps; k++) {
c896fe29 3025 ts = &s->temps[k];
ee17db83
RH		 3026 if (ts->val_type == TEMP_VAL_REG
3027 && ts->kind != TEMP_FIXED
f8b2f202
RH		 3028 && s->reg_to_temp[ts->reg] != ts) {
3029 printf("Inconsistency for temp %s:\n",
3030 tcg_get_arg_str_ptr(s, buf, sizeof(buf), ts));
b03cce8e 3031 fail:
f8b2f202
RH		 3032 printf("reg state:\n");
3033 dump_regs(s);
3034 tcg_abort();
c896fe29
FB		 3035 }
3036 }
3037}
3038#endif
3039
2272e4a7 3040static void temp_allocate_frame(TCGContext *s, TCGTemp *ts)
c896fe29 3041{
c1c09194
RH		 3042 intptr_t off, size, align;
3043
3044 switch (ts->type) {
3045 case TCG_TYPE_I32:
3046 size = align = 4;
3047 break;
3048 case TCG_TYPE_I64:
3049 case TCG_TYPE_V64:
3050 size = align = 8;
3051 break;
3052 case TCG_TYPE_V128:
3053 size = align = 16;
3054 break;
3055 case TCG_TYPE_V256:
3056 /* Note that we do not require aligned storage for V256. */
3057 size = 32, align = 16;
3058 break;
3059 default:
3060 g_assert_not_reached();
b591dc59 3061 }
c1c09194
RH		 3062
3063 assert(align <= TCG_TARGET_STACK_ALIGN);
3064 off = ROUND_UP(s->current_frame_offset, align);
732d5897
RH		 3065
3066 /* If we've exhausted the stack frame, restart with a smaller TB. */
3067 if (off + size > s->frame_end) {
3068 tcg_raise_tb_overflow(s);
3069 }
c1c09194
RH		 3070 s->current_frame_offset = off + size;
3071
3072 ts->mem_offset = off;
9defd1bd
RH		 3073#if defined(__sparc__)
3074 ts->mem_offset += TCG_TARGET_STACK_BIAS;
3075#endif
b3a62939 3076 ts->mem_base = s->frame_temp;
c896fe29 3077 ts->mem_allocated = 1;
c896fe29
FB		 3078}
3079
b722452a 3080static void temp_load(TCGContext *, TCGTemp *, TCGRegSet, TCGRegSet, TCGRegSet);
b3915dbb 3081
59d7c14e
RH		 3082/* Mark a temporary as free or dead. If 'free_or_dead' is negative,
3083 mark it free; otherwise mark it dead. */
3084static void temp_free_or_dead(TCGContext *s, TCGTemp *ts, int free_or_dead)
7f6ceedf 3085{
c0522136
RH		 3086 TCGTempVal new_type;
3087
3088 switch (ts->kind) {
3089 case TEMP_FIXED:
59d7c14e 3090 return;
c0522136
RH		 3091 case TEMP_GLOBAL:
3092 case TEMP_LOCAL:
3093 new_type = TEMP_VAL_MEM;
3094 break;
3095 case TEMP_NORMAL:
3096 new_type = free_or_dead < 0 ? TEMP_VAL_MEM : TEMP_VAL_DEAD;
3097 break;
3098 case TEMP_CONST:
3099 new_type = TEMP_VAL_CONST;
3100 break;
3101 default:
3102 g_assert_not_reached();
59d7c14e
RH		 3103 }
3104 if (ts->val_type == TEMP_VAL_REG) {
3105 s->reg_to_temp[ts->reg] = NULL;
3106 }
c0522136 3107 ts->val_type = new_type;
59d7c14e 3108}
7f6ceedf 3109
59d7c14e
RH		 3110/* Mark a temporary as dead. */
3111static inline void temp_dead(TCGContext *s, TCGTemp *ts)
3112{
3113 temp_free_or_dead(s, ts, 1);
3114}
3115
3116/* Sync a temporary to memory. 'allocated_regs' is used in case a temporary
3117 registers needs to be allocated to store a constant. If 'free_or_dead'
3118 is non-zero, subsequently release the temporary; if it is positive, the
3119 temp is dead; if it is negative, the temp is free. */
98b4e186
RH		 3120static void temp_sync(TCGContext *s, TCGTemp *ts, TCGRegSet allocated_regs,
3121 TCGRegSet preferred_regs, int free_or_dead)
59d7c14e 3122{
c0522136 3123 if (!temp_readonly(ts) && !ts->mem_coherent) {
7f6ceedf 3124 if (!ts->mem_allocated) {
2272e4a7 3125 temp_allocate_frame(s, ts);
59d7c14e 3126 }
59d7c14e
RH		 3127 switch (ts->val_type) {
3128 case TEMP_VAL_CONST:
3129 /* If we're going to free the temp immediately, then we won't
3130 require it later in a register, so attempt to store the
3131 constant to memory directly. */
3132 if (free_or_dead
3133 && tcg_out_sti(s, ts->type, ts->val,
3134 ts->mem_base->reg, ts->mem_offset)) {
3135 break;
3136 }
3137 temp_load(s, ts, tcg_target_available_regs[ts->type],
98b4e186 3138 allocated_regs, preferred_regs);
59d7c14e
RH		 3139 /* fallthrough */
3140
3141 case TEMP_VAL_REG:
3142 tcg_out_st(s, ts->type, ts->reg,
3143 ts->mem_base->reg, ts->mem_offset);
3144 break;
3145
3146 case TEMP_VAL_MEM:
3147 break;
3148
3149 case TEMP_VAL_DEAD:
3150 default:
3151 tcg_abort();
3152 }
3153 ts->mem_coherent = 1;
3154 }
3155 if (free_or_dead) {
3156 temp_free_or_dead(s, ts, free_or_dead);
7f6ceedf 3157 }
7f6ceedf
AJ		 3158}
3159
c896fe29 3160/* free register 'reg' by spilling the corresponding temporary if necessary */
b3915dbb 3161static void tcg_reg_free(TCGContext *s, TCGReg reg, TCGRegSet allocated_regs)
c896fe29 3162{
f8b2f202 3163 TCGTemp *ts = s->reg_to_temp[reg];
f8b2f202 3164 if (ts != NULL) {
98b4e186 3165 temp_sync(s, ts, allocated_regs, 0, -1);
c896fe29
FB		 3166 }
3167}
3168
b016486e
RH		 3169/**
3170 * tcg_reg_alloc:
3171 * @required_regs: Set of registers in which we must allocate.
3172 * @allocated_regs: Set of registers which must be avoided.
3173 * @preferred_regs: Set of registers we should prefer.
3174 * @rev: True if we search the registers in "indirect" order.
3175 *
3176 * The allocated register must be in @required_regs & ~@allocated_regs,
3177 * but if we can put it in @preferred_regs we may save a move later.
3178 */
3179static TCGReg tcg_reg_alloc(TCGContext *s, TCGRegSet required_regs,
3180 TCGRegSet allocated_regs,
3181 TCGRegSet preferred_regs, bool rev)
c896fe29 3182{
b016486e
RH		 3183 int i, j, f, n = ARRAY_SIZE(tcg_target_reg_alloc_order);
3184 TCGRegSet reg_ct[2];
91478cef 3185 const int *order;
c896fe29 3186
b016486e
RH		 3187 reg_ct[1] = required_regs & ~allocated_regs;
3188 tcg_debug_assert(reg_ct[1] != 0);
3189 reg_ct[0] = reg_ct[1] & preferred_regs;
3190
3191 /* Skip the preferred_regs option if it cannot be satisfied,
3192 or if the preference made no difference. */
3193 f = reg_ct[0] == 0 || reg_ct[0] == reg_ct[1];
3194
91478cef 3195 order = rev ? indirect_reg_alloc_order : tcg_target_reg_alloc_order;
c896fe29 3196
b016486e
RH		 3197 /* Try free registers, preferences first. */
3198 for (j = f; j < 2; j++) {
3199 TCGRegSet set = reg_ct[j];
3200
3201 if (tcg_regset_single(set)) {
3202 /* One register in the set. */
3203 TCGReg reg = tcg_regset_first(set);
3204 if (s->reg_to_temp[reg] == NULL) {
3205 return reg;
3206 }
3207 } else {
3208 for (i = 0; i < n; i++) {
3209 TCGReg reg = order[i];
3210 if (s->reg_to_temp[reg] == NULL &&
3211 tcg_regset_test_reg(set, reg)) {
3212 return reg;
3213 }
3214 }
3215 }
c896fe29
FB		 3216 }
3217
b016486e
RH		 3218 /* We must spill something. */
3219 for (j = f; j < 2; j++) {
3220 TCGRegSet set = reg_ct[j];
3221
3222 if (tcg_regset_single(set)) {
3223 /* One register in the set. */
3224 TCGReg reg = tcg_regset_first(set);
b3915dbb 3225 tcg_reg_free(s, reg, allocated_regs);
c896fe29 3226 return reg;
b016486e
RH		 3227 } else {
3228 for (i = 0; i < n; i++) {
3229 TCGReg reg = order[i];
3230 if (tcg_regset_test_reg(set, reg)) {
3231 tcg_reg_free(s, reg, allocated_regs);
3232 return reg;
3233 }
3234 }
c896fe29
FB		 3235 }
3236 }
3237
3238 tcg_abort();
3239}
3240
40ae5c62
RH		 3241/* Make sure the temporary is in a register. If needed, allocate the register
3242 from DESIRED while avoiding ALLOCATED. */
3243static void temp_load(TCGContext *s, TCGTemp *ts, TCGRegSet desired_regs,
b722452a 3244 TCGRegSet allocated_regs, TCGRegSet preferred_regs)
40ae5c62
RH		 3245{
3246 TCGReg reg;
3247
3248 switch (ts->val_type) {
3249 case TEMP_VAL_REG:
3250 return;
3251 case TEMP_VAL_CONST:
b016486e 3252 reg = tcg_reg_alloc(s, desired_regs, allocated_regs,
b722452a 3253 preferred_regs, ts->indirect_base);
0a6a8bc8
RH		 3254 if (ts->type <= TCG_TYPE_I64) {
3255 tcg_out_movi(s, ts->type, reg, ts->val);
3256 } else {
4e186175
RH		 3257 uint64_t val = ts->val;
3258 MemOp vece = MO_64;
3259
3260 /*
3261 * Find the minimal vector element that matches the constant.
3262 * The targets will, in general, have to do this search anyway,
3263 * do this generically.
3264 */
4e186175
RH		 3265 if (val == dup_const(MO_8, val)) {
3266 vece = MO_8;
3267 } else if (val == dup_const(MO_16, val)) {
3268 vece = MO_16;
0b4286dd 3269 } else if (val == dup_const(MO_32, val)) {
4e186175
RH		 3270 vece = MO_32;
3271 }
3272
3273 tcg_out_dupi_vec(s, ts->type, vece, reg, ts->val);
0a6a8bc8 3274 }
40ae5c62
RH		 3275 ts->mem_coherent = 0;
3276 break;
3277 case TEMP_VAL_MEM:
b016486e 3278 reg = tcg_reg_alloc(s, desired_regs, allocated_regs,
b722452a 3279 preferred_regs, ts->indirect_base);
40ae5c62
RH		 3280 tcg_out_ld(s, ts->type, reg, ts->mem_base->reg, ts->mem_offset);
3281 ts->mem_coherent = 1;
3282 break;
3283 case TEMP_VAL_DEAD:
3284 default:
3285 tcg_abort();
3286 }
3287 ts->reg = reg;
3288 ts->val_type = TEMP_VAL_REG;
3289 s->reg_to_temp[reg] = ts;
3290}
3291
59d7c14e
RH		 3292/* Save a temporary to memory. 'allocated_regs' is used in case a
3293 temporary registers needs to be allocated to store a constant. */
3294static void temp_save(TCGContext *s, TCGTemp *ts, TCGRegSet allocated_regs)
1ad80729 3295{
5a18407f
RH		 3296 /* The liveness analysis already ensures that globals are back
3297 in memory. Keep an tcg_debug_assert for safety. */
e01fa97d 3298 tcg_debug_assert(ts->val_type == TEMP_VAL_MEM || temp_readonly(ts));
1ad80729
AJ		 3299}
3300
9814dd27 3301/* save globals to their canonical location and assume they can be
e8996ee0
FB		 3302 modified be the following code. 'allocated_regs' is used in case a
3303 temporary registers needs to be allocated to store a constant. */
3304static void save_globals(TCGContext *s, TCGRegSet allocated_regs)
c896fe29 3305{
ac3b8891 3306 int i, n;
c896fe29 3307
ac3b8891 3308 for (i = 0, n = s->nb_globals; i < n; i++) {
b13eb728 3309 temp_save(s, &s->temps[i], allocated_regs);
c896fe29 3310 }
e5097dc8
FB		 3311}
3312
3d5c5f87
AJ		 3313/* sync globals to their canonical location and assume they can be
3314 read by the following code. 'allocated_regs' is used in case a
3315 temporary registers needs to be allocated to store a constant. */
3316static void sync_globals(TCGContext *s, TCGRegSet allocated_regs)
3317{
ac3b8891 3318 int i, n;
3d5c5f87 3319
ac3b8891 3320 for (i = 0, n = s->nb_globals; i < n; i++) {
12b9b11a 3321 TCGTemp *ts = &s->temps[i];
5a18407f 3322 tcg_debug_assert(ts->val_type != TEMP_VAL_REG
ee17db83 3323 || ts->kind == TEMP_FIXED
5a18407f 3324 || ts->mem_coherent);
3d5c5f87
AJ		 3325 }
3326}
3327
e5097dc8 3328/* at the end of a basic block, we assume all temporaries are dead and
e8996ee0
FB		 3329 all globals are stored at their canonical location. */
3330static void tcg_reg_alloc_bb_end(TCGContext *s, TCGRegSet allocated_regs)
e5097dc8 3331{
e5097dc8
FB		 3332 int i;
3333
b13eb728
RH		 3334 for (i = s->nb_globals; i < s->nb_temps; i++) {
3335 TCGTemp *ts = &s->temps[i];
c0522136
RH		 3336
3337 switch (ts->kind) {
3338 case TEMP_LOCAL:
b13eb728 3339 temp_save(s, ts, allocated_regs);
c0522136
RH		 3340 break;
3341 case TEMP_NORMAL:
5a18407f
RH		 3342 /* The liveness analysis already ensures that temps are dead.
3343 Keep an tcg_debug_assert for safety. */
3344 tcg_debug_assert(ts->val_type == TEMP_VAL_DEAD);
c0522136
RH		 3345 break;
3346 case TEMP_CONST:
3347 /* Similarly, we should have freed any allocated register. */
3348 tcg_debug_assert(ts->val_type == TEMP_VAL_CONST);
3349 break;
3350 default:
3351 g_assert_not_reached();
c896fe29
FB		 3352 }
3353 }
e8996ee0
FB		 3354
3355 save_globals(s, allocated_regs);
c896fe29
FB		 3356}
3357
b4cb76e6
RH		 3358/*
3359 * At a conditional branch, we assume all temporaries are dead and
3360 * all globals and local temps are synced to their location.
3361 */
3362static void tcg_reg_alloc_cbranch(TCGContext *s, TCGRegSet allocated_regs)
3363{
3364 sync_globals(s, allocated_regs);
3365
3366 for (int i = s->nb_globals; i < s->nb_temps; i++) {
3367 TCGTemp *ts = &s->temps[i];
3368 /*
3369 * The liveness analysis already ensures that temps are dead.
3370 * Keep tcg_debug_asserts for safety.
3371 */
c0522136
RH		 3372 switch (ts->kind) {
3373 case TEMP_LOCAL:
b4cb76e6 3374 tcg_debug_assert(ts->val_type != TEMP_VAL_REG || ts->mem_coherent);
c0522136
RH		 3375 break;
3376 case TEMP_NORMAL:
b4cb76e6 3377 tcg_debug_assert(ts->val_type == TEMP_VAL_DEAD);
c0522136
RH		 3378 break;
3379 case TEMP_CONST:
3380 break;
3381 default:
3382 g_assert_not_reached();
b4cb76e6
RH		 3383 }
3384 }
3385}
3386
bab1671f 3387/*
c58f4c97 3388 * Specialized code generation for INDEX_op_mov_* with a constant.
bab1671f 3389 */
0fe4fca4 3390static void tcg_reg_alloc_do_movi(TCGContext *s, TCGTemp *ots,
ba87719c
RH		 3391 tcg_target_ulong val, TCGLifeData arg_life,
3392 TCGRegSet preferred_regs)
e8996ee0 3393{
d63e3b6e 3394 /* ENV should not be modified. */
e01fa97d 3395 tcg_debug_assert(!temp_readonly(ots));
59d7c14e
RH		 3396
3397 /* The movi is not explicitly generated here. */
3398 if (ots->val_type == TEMP_VAL_REG) {
3399 s->reg_to_temp[ots->reg] = NULL;
ec7a869d 3400 }
59d7c14e
RH		 3401 ots->val_type = TEMP_VAL_CONST;
3402 ots->val = val;
3403 ots->mem_coherent = 0;
3404 if (NEED_SYNC_ARG(0)) {
ba87719c 3405 temp_sync(s, ots, s->reserved_regs, preferred_regs, IS_DEAD_ARG(0));
59d7c14e 3406 } else if (IS_DEAD_ARG(0)) {
f8bf00f1 3407 temp_dead(s, ots);
4c4e1ab2 3408 }
e8996ee0
FB		 3409}
3410
bab1671f
RH		 3411/*
3412 * Specialized code generation for INDEX_op_mov_*.
3413 */
dd186292 3414static void tcg_reg_alloc_mov(TCGContext *s, const TCGOp *op)
c896fe29 3415{
dd186292 3416 const TCGLifeData arg_life = op->life;
69e3706d 3417 TCGRegSet allocated_regs, preferred_regs;
c896fe29 3418 TCGTemp *ts, *ots;
450445d5 3419 TCGType otype, itype;
c896fe29 3420
d21369f5 3421 allocated_regs = s->reserved_regs;
69e3706d 3422 preferred_regs = op->output_pref[0];
43439139
RH		 3423 ots = arg_temp(op->args[0]);
3424 ts = arg_temp(op->args[1]);
450445d5 3425
d63e3b6e 3426 /* ENV should not be modified. */
e01fa97d 3427 tcg_debug_assert(!temp_readonly(ots));
d63e3b6e 3428
450445d5
RH		 3429 /* Note that otype != itype for no-op truncation. */
3430 otype = ots->type;
3431 itype = ts->type;
c29c1d7e 3432
0fe4fca4
PB		 3433 if (ts->val_type == TEMP_VAL_CONST) {
3434 /* propagate constant or generate sti */
3435 tcg_target_ulong val = ts->val;
3436 if (IS_DEAD_ARG(1)) {
3437 temp_dead(s, ts);
3438 }
69e3706d 3439 tcg_reg_alloc_do_movi(s, ots, val, arg_life, preferred_regs);
0fe4fca4
PB		 3440 return;
3441 }
3442
3443 /* If the source value is in memory we're going to be forced
3444 to have it in a register in order to perform the copy. Copy
3445 the SOURCE value into its own register first, that way we
3446 don't have to reload SOURCE the next time it is used. */
3447 if (ts->val_type == TEMP_VAL_MEM) {
69e3706d
RH		 3448 temp_load(s, ts, tcg_target_available_regs[itype],
3449 allocated_regs, preferred_regs);
c29c1d7e 3450 }
c896fe29 3451
0fe4fca4 3452 tcg_debug_assert(ts->val_type == TEMP_VAL_REG);
d63e3b6e 3453 if (IS_DEAD_ARG(0)) {
c29c1d7e
AJ		 3454 /* mov to a non-saved dead register makes no sense (even with
3455 liveness analysis disabled). */
eabb7b91 3456 tcg_debug_assert(NEED_SYNC_ARG(0));
c29c1d7e 3457 if (!ots->mem_allocated) {
2272e4a7 3458 temp_allocate_frame(s, ots);
c29c1d7e 3459 }
b3a62939 3460 tcg_out_st(s, otype, ts->reg, ots->mem_base->reg, ots->mem_offset);
c29c1d7e 3461 if (IS_DEAD_ARG(1)) {
f8bf00f1 3462 temp_dead(s, ts);
c29c1d7e 3463 }
f8bf00f1 3464 temp_dead(s, ots);
c29c1d7e 3465 } else {
ee17db83 3466 if (IS_DEAD_ARG(1) && ts->kind != TEMP_FIXED) {
c896fe29 3467 /* the mov can be suppressed */
c29c1d7e 3468 if (ots->val_type == TEMP_VAL_REG) {
f8b2f202 3469 s->reg_to_temp[ots->reg] = NULL;
c29c1d7e
AJ		 3470 }
3471 ots->reg = ts->reg;
f8bf00f1 3472 temp_dead(s, ts);
c896fe29 3473 } else {
c29c1d7e
AJ		 3474 if (ots->val_type != TEMP_VAL_REG) {
3475 /* When allocating a new register, make sure to not spill the
3476 input one. */
3477 tcg_regset_set_reg(allocated_regs, ts->reg);
450445d5 3478 ots->reg = tcg_reg_alloc(s, tcg_target_available_regs[otype],
69e3706d 3479 allocated_regs, preferred_regs,
b016486e 3480 ots->indirect_base);
c896fe29 3481 }
78113e83 3482 if (!tcg_out_mov(s, otype, ots->reg, ts->reg)) {
240c08d0
RH		 3483 /*
3484 * Cross register class move not supported.
3485 * Store the source register into the destination slot
3486 * and leave the destination temp as TEMP_VAL_MEM.
3487 */
e01fa97d 3488 assert(!temp_readonly(ots));
240c08d0
RH		 3489 if (!ts->mem_allocated) {
3490 temp_allocate_frame(s, ots);
3491 }
3492 tcg_out_st(s, ts->type, ts->reg,
3493 ots->mem_base->reg, ots->mem_offset);
3494 ots->mem_coherent = 1;
3495 temp_free_or_dead(s, ots, -1);
3496 return;
78113e83 3497 }
c896fe29 3498 }
c29c1d7e
AJ		 3499 ots->val_type = TEMP_VAL_REG;
3500 ots->mem_coherent = 0;
f8b2f202 3501 s->reg_to_temp[ots->reg] = ots;
c29c1d7e 3502 if (NEED_SYNC_ARG(0)) {
98b4e186 3503 temp_sync(s, ots, allocated_regs, 0, 0);
c896fe29 3504 }
ec7a869d 3505 }
c896fe29
FB		 3506}
3507
bab1671f
RH		 3508/*
3509 * Specialized code generation for INDEX_op_dup_vec.
3510 */
3511static void tcg_reg_alloc_dup(TCGContext *s, const TCGOp *op)
3512{
3513 const TCGLifeData arg_life = op->life;
3514 TCGRegSet dup_out_regs, dup_in_regs;
3515 TCGTemp *its, *ots;
3516 TCGType itype, vtype;
d6ecb4a9 3517 intptr_t endian_fixup;
bab1671f
RH		 3518 unsigned vece;
3519 bool ok;
3520
3521 ots = arg_temp(op->args[0]);
3522 its = arg_temp(op->args[1]);
3523
3524 /* ENV should not be modified. */
e01fa97d 3525 tcg_debug_assert(!temp_readonly(ots));
bab1671f
RH		 3526
3527 itype = its->type;
3528 vece = TCGOP_VECE(op);
3529 vtype = TCGOP_VECL(op) + TCG_TYPE_V64;
3530
3531 if (its->val_type == TEMP_VAL_CONST) {
3532 /* Propagate constant via movi -> dupi. */
3533 tcg_target_ulong val = its->val;
3534 if (IS_DEAD_ARG(1)) {
3535 temp_dead(s, its);
3536 }
3537 tcg_reg_alloc_do_movi(s, ots, val, arg_life, op->output_pref[0]);
3538 return;
3539 }
3540
9be0d080
RH		 3541 dup_out_regs = tcg_op_defs[INDEX_op_dup_vec].args_ct[0].regs;
3542 dup_in_regs = tcg_op_defs[INDEX_op_dup_vec].args_ct[1].regs;
bab1671f
RH		 3543
3544 /* Allocate the output register now. */
3545 if (ots->val_type != TEMP_VAL_REG) {
3546 TCGRegSet allocated_regs = s->reserved_regs;
3547
3548 if (!IS_DEAD_ARG(1) && its->val_type == TEMP_VAL_REG) {
3549 /* Make sure to not spill the input register. */
3550 tcg_regset_set_reg(allocated_regs, its->reg);
3551 }
3552 ots->reg = tcg_reg_alloc(s, dup_out_regs, allocated_regs,
3553 op->output_pref[0], ots->indirect_base);
3554 ots->val_type = TEMP_VAL_REG;
3555 ots->mem_coherent = 0;
3556 s->reg_to_temp[ots->reg] = ots;
3557 }
3558
3559 switch (its->val_type) {
3560 case TEMP_VAL_REG:
3561 /*
3562 * The dup constriaints must be broad, covering all possible VECE.
3563 * However, tcg_op_dup_vec() gets to see the VECE and we allow it
3564 * to fail, indicating that extra moves are required for that case.
3565 */
3566 if (tcg_regset_test_reg(dup_in_regs, its->reg)) {
3567 if (tcg_out_dup_vec(s, vtype, vece, ots->reg, its->reg)) {
3568 goto done;
3569 }
3570 /* Try again from memory or a vector input register. */
3571 }
3572 if (!its->mem_coherent) {
3573 /*
3574 * The input register is not synced, and so an extra store
3575 * would be required to use memory. Attempt an integer-vector
3576 * register move first. We do not have a TCGRegSet for this.
3577 */
3578 if (tcg_out_mov(s, itype, ots->reg, its->reg)) {
3579 break;
3580 }
3581 /* Sync the temp back to its slot and load from there. */
3582 temp_sync(s, its, s->reserved_regs, 0, 0);
3583 }
3584 /* fall through */
3585
3586 case TEMP_VAL_MEM:
d6ecb4a9
RH		 3587#ifdef HOST_WORDS_BIGENDIAN
3588 endian_fixup = itype == TCG_TYPE_I32 ? 4 : 8;
3589 endian_fixup -= 1 << vece;
3590#else
3591 endian_fixup = 0;
3592#endif
3593 if (tcg_out_dupm_vec(s, vtype, vece, ots->reg, its->mem_base->reg,
3594 its->mem_offset + endian_fixup)) {
3595 goto done;
3596 }
bab1671f
RH		 3597 tcg_out_ld(s, itype, ots->reg, its->mem_base->reg, its->mem_offset);
3598 break;
3599
3600 default:
3601 g_assert_not_reached();
3602 }
3603
3604 /* We now have a vector input register, so dup must succeed. */
3605 ok = tcg_out_dup_vec(s, vtype, vece, ots->reg, ots->reg);
3606 tcg_debug_assert(ok);
3607
3608 done:
3609 if (IS_DEAD_ARG(1)) {
3610 temp_dead(s, its);
3611 }
3612 if (NEED_SYNC_ARG(0)) {
3613 temp_sync(s, ots, s->reserved_regs, 0, 0);
3614 }
3615 if (IS_DEAD_ARG(0)) {
3616 temp_dead(s, ots);
3617 }
3618}
3619
dd186292 3620static void tcg_reg_alloc_op(TCGContext *s, const TCGOp *op)
c896fe29 3621{
dd186292
RH		 3622 const TCGLifeData arg_life = op->life;
3623 const TCGOpDef * const def = &tcg_op_defs[op->opc];
82790a87
RH		 3624 TCGRegSet i_allocated_regs;
3625 TCGRegSet o_allocated_regs;
b6638662
RH		 3626 int i, k, nb_iargs, nb_oargs;
3627 TCGReg reg;
c896fe29
FB		 3628 TCGArg arg;
3629 const TCGArgConstraint *arg_ct;
3630 TCGTemp *ts;
3631 TCGArg new_args[TCG_MAX_OP_ARGS];
3632 int const_args[TCG_MAX_OP_ARGS];
3633
3634 nb_oargs = def->nb_oargs;
3635 nb_iargs = def->nb_iargs;
3636
3637 /* copy constants */
3638 memcpy(new_args + nb_oargs + nb_iargs,
dd186292 3639 op->args + nb_oargs + nb_iargs,
c896fe29
FB		 3640 sizeof(TCGArg) * def->nb_cargs);
3641
d21369f5
RH		 3642 i_allocated_regs = s->reserved_regs;
3643 o_allocated_regs = s->reserved_regs;
82790a87 3644
c896fe29 3645 /* satisfy input constraints */
dd186292 3646 for (k = 0; k < nb_iargs; k++) {
d62816f2
RH		 3647 TCGRegSet i_preferred_regs, o_preferred_regs;
3648
66792f90 3649 i = def->args_ct[nb_oargs + k].sort_index;
dd186292 3650 arg = op->args[i];
c896fe29 3651 arg_ct = &def->args_ct[i];
43439139 3652 ts = arg_temp(arg);
40ae5c62
RH		 3653
3654 if (ts->val_type == TEMP_VAL_CONST
a4fbbd77 3655 && tcg_target_const_match(ts->val, ts->type, arg_ct->ct)) {
40ae5c62
RH		 3656 /* constant is OK for instruction */
3657 const_args[i] = 1;
3658 new_args[i] = ts->val;
d62816f2 3659 continue;
c896fe29 3660 }
40ae5c62 3661
d62816f2 3662 i_preferred_regs = o_preferred_regs = 0;
bc2b17e6 3663 if (arg_ct->ialias) {
d62816f2 3664 o_preferred_regs = op->output_pref[arg_ct->alias_index];
d62816f2 3665
c0522136
RH		 3666 /*
3667 * If the input is readonly, then it cannot also be an
3668 * output and aliased to itself. If the input is not
3669 * dead after the instruction, we must allocate a new
3670 * register and move it.
3671 */
3672 if (temp_readonly(ts) || !IS_DEAD_ARG(i)) {
3673 goto allocate_in_reg;
3674 }
3675
3676 /*
3677 * Check if the current register has already been allocated
3678 * for another input aliased to an output.
3679 */
3680 if (ts->val_type == TEMP_VAL_REG) {
3681 reg = ts->reg;
3682 for (int k2 = 0; k2 < k; k2++) {
3683 int i2 = def->args_ct[nb_oargs + k2].sort_index;
3684 if (def->args_ct[i2].ialias && reg == new_args[i2]) {
3685 goto allocate_in_reg;
7e1df267
AJ		 3686 }
3687 }
5ff9d6a4 3688 }
c0522136 3689 i_preferred_regs = o_preferred_regs;
c896fe29 3690 }
d62816f2 3691
9be0d080 3692 temp_load(s, ts, arg_ct->regs, i_allocated_regs, i_preferred_regs);
c896fe29 3693 reg = ts->reg;
d62816f2 3694
c0522136
RH		 3695 if (!tcg_regset_test_reg(arg_ct->regs, reg)) {
3696 allocate_in_reg:
3697 /*
3698 * Allocate a new register matching the constraint
3699 * and move the temporary register into it.
3700 */
d62816f2
RH		 3701 temp_load(s, ts, tcg_target_available_regs[ts->type],
3702 i_allocated_regs, 0);
9be0d080 3703 reg = tcg_reg_alloc(s, arg_ct->regs, i_allocated_regs,
d62816f2 3704 o_preferred_regs, ts->indirect_base);
78113e83 3705 if (!tcg_out_mov(s, ts->type, reg, ts->reg)) {
240c08d0
RH		 3706 /*
3707 * Cross register class move not supported. Sync the
3708 * temp back to its slot and load from there.
3709 */
3710 temp_sync(s, ts, i_allocated_regs, 0, 0);
3711 tcg_out_ld(s, ts->type, reg,
3712 ts->mem_base->reg, ts->mem_offset);
78113e83 3713 }
c896fe29 3714 }
c896fe29
FB		 3715 new_args[i] = reg;
3716 const_args[i] = 0;
82790a87 3717 tcg_regset_set_reg(i_allocated_regs, reg);
c896fe29
FB		 3718 }
3719
a52ad07e
AJ		 3720 /* mark dead temporaries and free the associated registers */
3721 for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) {
3722 if (IS_DEAD_ARG(i)) {
43439139 3723 temp_dead(s, arg_temp(op->args[i]));
a52ad07e
AJ		 3724 }
3725 }
3726
b4cb76e6
RH		 3727 if (def->flags & TCG_OPF_COND_BRANCH) {
3728 tcg_reg_alloc_cbranch(s, i_allocated_regs);
3729 } else if (def->flags & TCG_OPF_BB_END) {
82790a87 3730 tcg_reg_alloc_bb_end(s, i_allocated_regs);
e8996ee0 3731 } else {
e8996ee0
FB		 3732 if (def->flags & TCG_OPF_CALL_CLOBBER) {
3733 /* XXX: permit generic clobber register list ? */
c8074023
RH		 3734 for (i = 0; i < TCG_TARGET_NB_REGS; i++) {
3735 if (tcg_regset_test_reg(tcg_target_call_clobber_regs, i)) {
82790a87 3736 tcg_reg_free(s, i, i_allocated_regs);
e8996ee0 3737 }
c896fe29 3738 }
3d5c5f87
AJ		 3739 }
3740 if (def->flags & TCG_OPF_SIDE_EFFECTS) {
3741 /* sync globals if the op has side effects and might trigger
3742 an exception. */
82790a87 3743 sync_globals(s, i_allocated_regs);
c896fe29 3744 }
e8996ee0
FB		 3745
3746 /* satisfy the output constraints */
e8996ee0 3747 for(k = 0; k < nb_oargs; k++) {
66792f90 3748 i = def->args_ct[k].sort_index;
dd186292 3749 arg = op->args[i];
e8996ee0 3750 arg_ct = &def->args_ct[i];
43439139 3751 ts = arg_temp(arg);
d63e3b6e
RH		 3752
3753 /* ENV should not be modified. */
e01fa97d 3754 tcg_debug_assert(!temp_readonly(ts));
d63e3b6e 3755
bc2b17e6 3756 if (arg_ct->oalias && !const_args[arg_ct->alias_index]) {
e8996ee0 3757 reg = new_args[arg_ct->alias_index];
bc2b17e6 3758 } else if (arg_ct->newreg) {
9be0d080 3759 reg = tcg_reg_alloc(s, arg_ct->regs,
82790a87 3760 i_allocated_regs | o_allocated_regs,
69e3706d 3761 op->output_pref[k], ts->indirect_base);
e8996ee0 3762 } else {
9be0d080 3763 reg = tcg_reg_alloc(s, arg_ct->regs, o_allocated_regs,
69e3706d 3764 op->output_pref[k], ts->indirect_base);
c896fe29 3765 }
82790a87 3766 tcg_regset_set_reg(o_allocated_regs, reg);
d63e3b6e
RH		 3767 if (ts->val_type == TEMP_VAL_REG) {
3768 s->reg_to_temp[ts->reg] = NULL;
e8996ee0 3769 }
d63e3b6e
RH		 3770 ts->val_type = TEMP_VAL_REG;
3771 ts->reg = reg;
3772 /*
3773 * Temp value is modified, so the value kept in memory is
3774 * potentially not the same.
3775 */
3776 ts->mem_coherent = 0;
3777 s->reg_to_temp[reg] = ts;
e8996ee0 3778 new_args[i] = reg;
c896fe29 3779 }
c896fe29
FB		 3780 }
3781
c896fe29 3782 /* emit instruction */
d2fd745f
RH		 3783 if (def->flags & TCG_OPF_VECTOR) {
3784 tcg_out_vec_op(s, op->opc, TCGOP_VECL(op), TCGOP_VECE(op),
3785 new_args, const_args);
3786 } else {
3787 tcg_out_op(s, op->opc, new_args, const_args);
3788 }
3789
c896fe29
FB		 3790 /* move the outputs in the correct register if needed */
3791 for(i = 0; i < nb_oargs; i++) {
43439139 3792 ts = arg_temp(op->args[i]);
d63e3b6e
RH		 3793
3794 /* ENV should not be modified. */
e01fa97d 3795 tcg_debug_assert(!temp_readonly(ts));
d63e3b6e 3796
ec7a869d 3797 if (NEED_SYNC_ARG(i)) {
98b4e186 3798 temp_sync(s, ts, o_allocated_regs, 0, IS_DEAD_ARG(i));
59d7c14e 3799 } else if (IS_DEAD_ARG(i)) {
f8bf00f1 3800 temp_dead(s, ts);
ec7a869d 3801 }
c896fe29
FB		 3802 }
3803}
3804
efe86b21
RH		 3805static bool tcg_reg_alloc_dup2(TCGContext *s, const TCGOp *op)
3806{
3807 const TCGLifeData arg_life = op->life;
3808 TCGTemp *ots, *itsl, *itsh;
3809 TCGType vtype = TCGOP_VECL(op) + TCG_TYPE_V64;
3810
3811 /* This opcode is only valid for 32-bit hosts, for 64-bit elements. */
3812 tcg_debug_assert(TCG_TARGET_REG_BITS == 32);
3813 tcg_debug_assert(TCGOP_VECE(op) == MO_64);
3814
3815 ots = arg_temp(op->args[0]);
3816 itsl = arg_temp(op->args[1]);
3817 itsh = arg_temp(op->args[2]);
3818
3819 /* ENV should not be modified. */
3820 tcg_debug_assert(!temp_readonly(ots));
3821
3822 /* Allocate the output register now. */
3823 if (ots->val_type != TEMP_VAL_REG) {
3824 TCGRegSet allocated_regs = s->reserved_regs;
3825 TCGRegSet dup_out_regs =
3826 tcg_op_defs[INDEX_op_dup_vec].args_ct[0].regs;
3827
3828 /* Make sure to not spill the input registers. */
3829 if (!IS_DEAD_ARG(1) && itsl->val_type == TEMP_VAL_REG) {
3830 tcg_regset_set_reg(allocated_regs, itsl->reg);
3831 }
3832 if (!IS_DEAD_ARG(2) && itsh->val_type == TEMP_VAL_REG) {
3833 tcg_regset_set_reg(allocated_regs, itsh->reg);
3834 }
3835
3836 ots->reg = tcg_reg_alloc(s, dup_out_regs, allocated_regs,
3837 op->output_pref[0], ots->indirect_base);
3838 ots->val_type = TEMP_VAL_REG;
3839 ots->mem_coherent = 0;
3840 s->reg_to_temp[ots->reg] = ots;
3841 }
3842
3843 /* Promote dup2 of immediates to dupi_vec. */
3844 if (itsl->val_type == TEMP_VAL_CONST && itsh->val_type == TEMP_VAL_CONST) {
3845 uint64_t val = deposit64(itsl->val, 32, 32, itsh->val);
3846 MemOp vece = MO_64;
3847
3848 if (val == dup_const(MO_8, val)) {
3849 vece = MO_8;
3850 } else if (val == dup_const(MO_16, val)) {
3851 vece = MO_16;
3852 } else if (val == dup_const(MO_32, val)) {
3853 vece = MO_32;
3854 }
3855
3856 tcg_out_dupi_vec(s, vtype, vece, ots->reg, val);
3857 goto done;
3858 }
3859
3860 /* If the two inputs form one 64-bit value, try dupm_vec. */
3861 if (itsl + 1 == itsh && itsl->base_type == TCG_TYPE_I64) {
3862 if (!itsl->mem_coherent) {
3863 temp_sync(s, itsl, s->reserved_regs, 0, 0);
3864 }
3865 if (!itsh->mem_coherent) {
3866 temp_sync(s, itsh, s->reserved_regs, 0, 0);
3867 }
3868#ifdef HOST_WORDS_BIGENDIAN
3869 TCGTemp *its = itsh;
3870#else
3871 TCGTemp *its = itsl;
3872#endif
3873 if (tcg_out_dupm_vec(s, vtype, MO_64, ots->reg,
3874 its->mem_base->reg, its->mem_offset)) {
3875 goto done;
3876 }
3877 }
3878
3879 /* Fall back to generic expansion. */
3880 return false;
3881
3882 done:
3883 if (IS_DEAD_ARG(1)) {
3884 temp_dead(s, itsl);
3885 }
3886 if (IS_DEAD_ARG(2)) {
3887 temp_dead(s, itsh);
3888 }
3889 if (NEED_SYNC_ARG(0)) {
3890 temp_sync(s, ots, s->reserved_regs, 0, IS_DEAD_ARG(0));
3891 } else if (IS_DEAD_ARG(0)) {
3892 temp_dead(s, ots);
3893 }
3894 return true;
3895}
3896
b03cce8e
FB		 3897#ifdef TCG_TARGET_STACK_GROWSUP
3898#define STACK_DIR(x) (-(x))
3899#else
3900#define STACK_DIR(x) (x)
3901#endif
3902
dd186292 3903static void tcg_reg_alloc_call(TCGContext *s, TCGOp *op)
c896fe29 3904{
cd9090aa
RH		 3905 const int nb_oargs = TCGOP_CALLO(op);
3906 const int nb_iargs = TCGOP_CALLI(op);
dd186292 3907 const TCGLifeData arg_life = op->life;
7b7d8b2d 3908 const TCGHelperInfo *info;
b6638662
RH		 3909 int flags, nb_regs, i;
3910 TCGReg reg;
cf066674 3911 TCGArg arg;
c896fe29 3912 TCGTemp *ts;
d3452f1f
RH		 3913 intptr_t stack_offset;
3914 size_t call_stack_size;
cf066674
RH		 3915 tcg_insn_unit *func_addr;
3916 int allocate_args;
c896fe29 3917 TCGRegSet allocated_regs;
c896fe29 3918
fa52e660 3919 func_addr = tcg_call_func(op);
7b7d8b2d
RH		 3920 info = tcg_call_info(op);
3921 flags = info->flags;
c896fe29 3922
6e17d0c5 3923 nb_regs = ARRAY_SIZE(tcg_target_call_iarg_regs);
c45cb8bb
RH		 3924 if (nb_regs > nb_iargs) {
3925 nb_regs = nb_iargs;
cf066674 3926 }
c896fe29
FB		 3927
3928 /* assign stack slots first */
c45cb8bb 3929 call_stack_size = (nb_iargs - nb_regs) * sizeof(tcg_target_long);
c896fe29
FB		 3930 call_stack_size = (call_stack_size + TCG_TARGET_STACK_ALIGN - 1) &
3931 ~(TCG_TARGET_STACK_ALIGN - 1);
b03cce8e
FB		 3932 allocate_args = (call_stack_size > TCG_STATIC_CALL_ARGS_SIZE);
3933 if (allocate_args) {
345649c0
BS		 3934 /* XXX: if more than TCG_STATIC_CALL_ARGS_SIZE is needed,
3935 preallocate call stack */
3936 tcg_abort();
b03cce8e 3937 }
39cf05d3
FB		 3938
3939 stack_offset = TCG_TARGET_CALL_STACK_OFFSET;
dd186292
RH		 3940 for (i = nb_regs; i < nb_iargs; i++) {
3941 arg = op->args[nb_oargs + i];
39cf05d3
FB		 3942#ifdef TCG_TARGET_STACK_GROWSUP
3943 stack_offset -= sizeof(tcg_target_long);
3944#endif
3945 if (arg != TCG_CALL_DUMMY_ARG) {
43439139 3946 ts = arg_temp(arg);
40ae5c62 3947 temp_load(s, ts, tcg_target_available_regs[ts->type],
b722452a 3948 s->reserved_regs, 0);
40ae5c62 3949 tcg_out_st(s, ts->type, ts->reg, TCG_REG_CALL_STACK, stack_offset);
c896fe29 3950 }
39cf05d3
FB		 3951#ifndef TCG_TARGET_STACK_GROWSUP
3952 stack_offset += sizeof(tcg_target_long);
3953#endif
c896fe29
FB		 3954 }
3955
3956 /* assign input registers */
d21369f5 3957 allocated_regs = s->reserved_regs;
dd186292
RH		 3958 for (i = 0; i < nb_regs; i++) {
3959 arg = op->args[nb_oargs + i];
39cf05d3 3960 if (arg != TCG_CALL_DUMMY_ARG) {
43439139 3961 ts = arg_temp(arg);
39cf05d3 3962 reg = tcg_target_call_iarg_regs[i];
40ae5c62 3963
39cf05d3
FB		 3964 if (ts->val_type == TEMP_VAL_REG) {
3965 if (ts->reg != reg) {
4250da10 3966 tcg_reg_free(s, reg, allocated_regs);
78113e83 3967 if (!tcg_out_mov(s, ts->type, reg, ts->reg)) {
240c08d0
RH		 3968 /*
3969 * Cross register class move not supported. Sync the
3970 * temp back to its slot and load from there.
3971 */
3972 temp_sync(s, ts, allocated_regs, 0, 0);
3973 tcg_out_ld(s, ts->type, reg,
3974 ts->mem_base->reg, ts->mem_offset);
78113e83 3975 }
39cf05d3 3976 }
39cf05d3 3977 } else {
ccb1bb66 3978 TCGRegSet arg_set = 0;
40ae5c62 3979
4250da10 3980 tcg_reg_free(s, reg, allocated_regs);
40ae5c62 3981 tcg_regset_set_reg(arg_set, reg);
b722452a 3982 temp_load(s, ts, arg_set, allocated_regs, 0);
c896fe29 3983 }
40ae5c62 3984
39cf05d3 3985 tcg_regset_set_reg(allocated_regs, reg);
c896fe29 3986 }
c896fe29
FB		 3987 }
3988
c896fe29 3989 /* mark dead temporaries and free the associated registers */
dd186292 3990 for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) {
866cb6cb 3991 if (IS_DEAD_ARG(i)) {
43439139 3992 temp_dead(s, arg_temp(op->args[i]));
c896fe29
FB		 3993 }
3994 }
3995
3996 /* clobber call registers */
c8074023
RH		 3997 for (i = 0; i < TCG_TARGET_NB_REGS; i++) {
3998 if (tcg_regset_test_reg(tcg_target_call_clobber_regs, i)) {
b3915dbb 3999 tcg_reg_free(s, i, allocated_regs);
c896fe29
FB		 4000 }
4001 }
78505279
AJ		 4002
4003 /* Save globals if they might be written by the helper, sync them if
4004 they might be read. */
4005 if (flags & TCG_CALL_NO_READ_GLOBALS) {
4006 /* Nothing to do */
4007 } else if (flags & TCG_CALL_NO_WRITE_GLOBALS) {
4008 sync_globals(s, allocated_regs);
4009 } else {
b9c18f56
AJ		 4010 save_globals(s, allocated_regs);
4011 }
c896fe29 4012
7b7d8b2d
RH		 4013#ifdef CONFIG_TCG_INTERPRETER
4014 {
4015 gpointer hash = (gpointer)(uintptr_t)info->typemask;
4016 ffi_cif *cif = g_hash_table_lookup(ffi_table, hash);
4017 assert(cif != NULL);
4018 tcg_out_call(s, func_addr, cif);
4019 }
4020#else
cf066674 4021 tcg_out_call(s, func_addr);
7b7d8b2d 4022#endif
c896fe29
FB		 4023
4024 /* assign output registers and emit moves if needed */
4025 for(i = 0; i < nb_oargs; i++) {
dd186292 4026 arg = op->args[i];
43439139 4027 ts = arg_temp(arg);
d63e3b6e
RH		 4028
4029 /* ENV should not be modified. */
e01fa97d 4030 tcg_debug_assert(!temp_readonly(ts));
d63e3b6e 4031
c896fe29 4032 reg = tcg_target_call_oarg_regs[i];
eabb7b91 4033 tcg_debug_assert(s->reg_to_temp[reg] == NULL);
d63e3b6e
RH		 4034 if (ts->val_type == TEMP_VAL_REG) {
4035 s->reg_to_temp[ts->reg] = NULL;
4036 }
4037 ts->val_type = TEMP_VAL_REG;
4038 ts->reg = reg;
4039 ts->mem_coherent = 0;
4040 s->reg_to_temp[reg] = ts;
4041 if (NEED_SYNC_ARG(i)) {
4042 temp_sync(s, ts, allocated_regs, 0, IS_DEAD_ARG(i));
4043 } else if (IS_DEAD_ARG(i)) {
4044 temp_dead(s, ts);
c896fe29
FB		 4045 }
4046 }
c896fe29
FB		 4047}
4048
4049#ifdef CONFIG_PROFILER
4050
c3fac113
EC		 4051/* avoid copy/paste errors */
4052#define PROF_ADD(to, from, field) \
4053 do { \
d73415a3 4054 (to)->field += qatomic_read(&((from)->field)); \
c3fac113
EC		 4055 } while (0)
4056
4057#define PROF_MAX(to, from, field) \
4058 do { \
d73415a3 4059 typeof((from)->field) val__ = qatomic_read(&((from)->field)); \
c3fac113
EC		 4060 if (val__ > (to)->field) { \
4061 (to)->field = val__; \
4062 } \
4063 } while (0)
4064
4065/* Pass in a zero'ed @prof */
4066static inline
4067void tcg_profile_snapshot(TCGProfile *prof, bool counters, bool table)
4068{
0e2d61cf 4069 unsigned int n_ctxs = qatomic_read(&tcg_cur_ctxs);
c3fac113
EC		 4070 unsigned int i;
4071
3468b59e 4072 for (i = 0; i < n_ctxs; i++) {
d73415a3 4073 TCGContext *s = qatomic_read(&tcg_ctxs[i]);
3468b59e 4074 const TCGProfile *orig = &s->prof;
c3fac113
EC		 4075
4076 if (counters) {
72fd2efb 4077 PROF_ADD(prof, orig, cpu_exec_time);
c3fac113
EC		 4078 PROF_ADD(prof, orig, tb_count1);
4079 PROF_ADD(prof, orig, tb_count);
4080 PROF_ADD(prof, orig, op_count);
4081 PROF_MAX(prof, orig, op_count_max);
4082 PROF_ADD(prof, orig, temp_count);
4083 PROF_MAX(prof, orig, temp_count_max);
4084 PROF_ADD(prof, orig, del_op_count);
4085 PROF_ADD(prof, orig, code_in_len);
4086 PROF_ADD(prof, orig, code_out_len);
4087 PROF_ADD(prof, orig, search_out_len);
4088 PROF_ADD(prof, orig, interm_time);
4089 PROF_ADD(prof, orig, code_time);
4090 PROF_ADD(prof, orig, la_time);
4091 PROF_ADD(prof, orig, opt_time);
4092 PROF_ADD(prof, orig, restore_count);
4093 PROF_ADD(prof, orig, restore_time);
4094 }
4095 if (table) {
4096 int i;
4097
4098 for (i = 0; i < NB_OPS; i++) {
4099 PROF_ADD(prof, orig, table_op_count[i]);
4100 }
4101 }
4102 }
4103}
4104
4105#undef PROF_ADD
4106#undef PROF_MAX
4107
4108static void tcg_profile_snapshot_counters(TCGProfile *prof)
4109{
4110 tcg_profile_snapshot(prof, true, false);
4111}
4112
4113static void tcg_profile_snapshot_table(TCGProfile *prof)
4114{
4115 tcg_profile_snapshot(prof, false, true);
4116}
c896fe29 4117
d4c51a0a 4118void tcg_dump_op_count(void)
c896fe29 4119{
c3fac113 4120 TCGProfile prof = {};
c896fe29 4121 int i;
d70724ce 4122
c3fac113 4123 tcg_profile_snapshot_table(&prof);
15fc7daa 4124 for (i = 0; i < NB_OPS; i++) {
d4c51a0a 4125 qemu_printf("%s %" PRId64 "\n", tcg_op_defs[i].name,
c3fac113 4126 prof.table_op_count[i]);
c896fe29 4127 }
c896fe29 4128}
72fd2efb
EC		 4129
4130int64_t tcg_cpu_exec_time(void)
4131{
0e2d61cf 4132 unsigned int n_ctxs = qatomic_read(&tcg_cur_ctxs);
72fd2efb
EC		 4133 unsigned int i;
4134 int64_t ret = 0;
4135
4136 for (i = 0; i < n_ctxs; i++) {
d73415a3 4137 const TCGContext *s = qatomic_read(&tcg_ctxs[i]);
72fd2efb
EC		 4138 const TCGProfile *prof = &s->prof;
4139
d73415a3 4140 ret += qatomic_read(&prof->cpu_exec_time);
72fd2efb
EC		 4141 }
4142 return ret;
4143}
246ae24d 4144#else
d4c51a0a 4145void tcg_dump_op_count(void)
246ae24d 4146{
d4c51a0a 4147 qemu_printf("[TCG profiler not compiled]\n");
246ae24d 4148}
72fd2efb
EC		 4149
4150int64_t tcg_cpu_exec_time(void)
4151{
4152 error_report("%s: TCG profiler not compiled", __func__);
4153 exit(EXIT_FAILURE);
4154}
c896fe29
FB		 4155#endif
4156
4157
5bd2ec3d 4158int tcg_gen_code(TCGContext *s, TranslationBlock *tb)
c896fe29 4159{
c3fac113
EC		 4160#ifdef CONFIG_PROFILER
4161 TCGProfile *prof = &s->prof;
4162#endif
15fa08f8
RH		 4163 int i, num_insns;
4164 TCGOp *op;
c896fe29 4165
04fe6400
RH		 4166#ifdef CONFIG_PROFILER
4167 {
c1f543b7 4168 int n = 0;
04fe6400 4169
15fa08f8
RH		 4170 QTAILQ_FOREACH(op, &s->ops, link) {
4171 n++;
4172 }
d73415a3 4173 qatomic_set(&prof->op_count, prof->op_count + n);
c3fac113 4174 if (n > prof->op_count_max) {
d73415a3 4175 qatomic_set(&prof->op_count_max, n);
04fe6400
RH		 4176 }
4177
4178 n = s->nb_temps;
d73415a3 4179 qatomic_set(&prof->temp_count, prof->temp_count + n);
c3fac113 4180 if (n > prof->temp_count_max) {
d73415a3 4181 qatomic_set(&prof->temp_count_max, n);
04fe6400
RH		 4182 }
4183 }
4184#endif
4185
c896fe29 4186#ifdef DEBUG_DISAS
d977e1c2
AB		 4187 if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP)
4188 && qemu_log_in_addr_range(tb->pc))) {
fc59d2d8 4189 FILE *logfile = qemu_log_lock();
93fcfe39 4190 qemu_log("OP:\n");
1894f69a 4191 tcg_dump_ops(s, false);
93fcfe39 4192 qemu_log("\n");
fc59d2d8 4193 qemu_log_unlock(logfile);
c896fe29
FB		 4194 }
4195#endif
4196
bef16ab4
RH		 4197#ifdef CONFIG_DEBUG_TCG
4198 /* Ensure all labels referenced have been emitted. */
4199 {
4200 TCGLabel *l;
4201 bool error = false;
4202
4203 QSIMPLEQ_FOREACH(l, &s->labels, next) {
4204 if (unlikely(!l->present) && l->refs) {
4205 qemu_log_mask(CPU_LOG_TB_OP,
4206 "$L%d referenced but not present.\n", l->id);
4207 error = true;
4208 }
4209 }
4210 assert(!error);
4211 }
4212#endif
4213
c5cc28ff 4214#ifdef CONFIG_PROFILER
d73415a3 4215 qatomic_set(&prof->opt_time, prof->opt_time - profile_getclock());
c5cc28ff
AJ		 4216#endif
4217
8f2e8c07 4218#ifdef USE_TCG_OPTIMIZATIONS
c45cb8bb 4219 tcg_optimize(s);
8f2e8c07
KB		 4220#endif
4221
a23a9ec6 4222#ifdef CONFIG_PROFILER
d73415a3
SH		 4223 qatomic_set(&prof->opt_time, prof->opt_time + profile_getclock());
4224 qatomic_set(&prof->la_time, prof->la_time - profile_getclock());
a23a9ec6 4225#endif
c5cc28ff 4226
b4fc67c7 4227 reachable_code_pass(s);
b83eabea 4228 liveness_pass_1(s);
5a18407f 4229
b83eabea 4230 if (s->nb_indirects > 0) {
5a18407f 4231#ifdef DEBUG_DISAS
b83eabea
RH		 4232 if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP_IND)
4233 && qemu_log_in_addr_range(tb->pc))) {
fc59d2d8 4234 FILE *logfile = qemu_log_lock();
b83eabea 4235 qemu_log("OP before indirect lowering:\n");
1894f69a 4236 tcg_dump_ops(s, false);
b83eabea 4237 qemu_log("\n");
fc59d2d8 4238 qemu_log_unlock(logfile);
b83eabea 4239 }
5a18407f 4240#endif
b83eabea
RH		 4241 /* Replace indirect temps with direct temps. */
4242 if (liveness_pass_2(s)) {
4243 /* If changes were made, re-run liveness. */
4244 liveness_pass_1(s);
5a18407f
RH		 4245 }
4246 }
c5cc28ff 4247
a23a9ec6 4248#ifdef CONFIG_PROFILER
d73415a3 4249 qatomic_set(&prof->la_time, prof->la_time + profile_getclock());
a23a9ec6 4250#endif
c896fe29
FB		 4251
4252#ifdef DEBUG_DISAS
d977e1c2
AB		 4253 if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP_OPT)
4254 && qemu_log_in_addr_range(tb->pc))) {
fc59d2d8 4255 FILE *logfile = qemu_log_lock();
c5cc28ff 4256 qemu_log("OP after optimization and liveness analysis:\n");
1894f69a 4257 tcg_dump_ops(s, true);
93fcfe39 4258 qemu_log("\n");
fc59d2d8 4259 qemu_log_unlock(logfile);
c896fe29
FB		 4260 }
4261#endif
4262
4263 tcg_reg_alloc_start(s);
4264
db0c51a3
RH		 4265 /*
4266 * Reset the buffer pointers when restarting after overflow.
4267 * TODO: Move this into translate-all.c with the rest of the
4268 * buffer management. Having only this done here is confusing.
4269 */
4270 s->code_buf = tcg_splitwx_to_rw(tb->tc.ptr);
4271 s->code_ptr = s->code_buf;
c896fe29 4272
659ef5cb 4273#ifdef TCG_TARGET_NEED_LDST_LABELS
6001f772 4274 QSIMPLEQ_INIT(&s->ldst_labels);
659ef5cb 4275#endif
57a26946
RH		 4276#ifdef TCG_TARGET_NEED_POOL_LABELS
4277 s->pool_labels = NULL;
4278#endif
9ecefc84 4279
fca8a500 4280 num_insns = -1;
15fa08f8 4281 QTAILQ_FOREACH(op, &s->ops, link) {
c45cb8bb 4282 TCGOpcode opc = op->opc;
b3db8758 4283
c896fe29 4284#ifdef CONFIG_PROFILER
d73415a3 4285 qatomic_set(&prof->table_op_count[opc], prof->table_op_count[opc] + 1);
c896fe29 4286#endif
c45cb8bb
RH		 4287
4288 switch (opc) {
c896fe29 4289 case INDEX_op_mov_i32:
c896fe29 4290 case INDEX_op_mov_i64:
d2fd745f 4291 case INDEX_op_mov_vec:
dd186292 4292 tcg_reg_alloc_mov(s, op);
c896fe29 4293 break;
bab1671f
RH		 4294 case INDEX_op_dup_vec:
4295 tcg_reg_alloc_dup(s, op);
4296 break;
765b842a 4297 case INDEX_op_insn_start:
fca8a500 4298 if (num_insns >= 0) {
9f754620
RH		 4299 size_t off = tcg_current_code_size(s);
4300 s->gen_insn_end_off[num_insns] = off;
4301 /* Assert that we do not overflow our stored offset. */
4302 assert(s->gen_insn_end_off[num_insns] == off);
fca8a500
RH		 4303 }
4304 num_insns++;
bad729e2
RH		 4305 for (i = 0; i < TARGET_INSN_START_WORDS; ++i) {
4306 target_ulong a;
4307#if TARGET_LONG_BITS > TCG_TARGET_REG_BITS
efee3746 4308 a = deposit64(op->args[i * 2], 32, 32, op->args[i * 2 + 1]);
bad729e2 4309#else
efee3746 4310 a = op->args[i];
bad729e2 4311#endif
fca8a500 4312 s->gen_insn_data[num_insns][i] = a;
bad729e2 4313 }
c896fe29 4314 break;
5ff9d6a4 4315 case INDEX_op_discard:
43439139 4316 temp_dead(s, arg_temp(op->args[0]));
5ff9d6a4 4317 break;
c896fe29 4318 case INDEX_op_set_label:
e8996ee0 4319 tcg_reg_alloc_bb_end(s, s->reserved_regs);
92ab8e7d 4320 tcg_out_label(s, arg_label(op->args[0]));
c896fe29
FB		 4321 break;
4322 case INDEX_op_call:
dd186292 4323 tcg_reg_alloc_call(s, op);
c45cb8bb 4324 break;
efe86b21
RH		 4325 case INDEX_op_dup2_vec:
4326 if (tcg_reg_alloc_dup2(s, op)) {
4327 break;
4328 }
4329 /* fall through */
c896fe29 4330 default:
25c4d9cc 4331 /* Sanity check that we've not introduced any unhandled opcodes. */
be0f34b5 4332 tcg_debug_assert(tcg_op_supported(opc));
c896fe29
FB		 4333 /* Note: in order to speed up the code, it would be much
4334 faster to have specialized register allocator functions for
4335 some common argument patterns */
dd186292 4336 tcg_reg_alloc_op(s, op);
c896fe29
FB		 4337 break;
4338 }
8d8fdbae 4339#ifdef CONFIG_DEBUG_TCG
c896fe29
FB		 4340 check_regs(s);
4341#endif
b125f9dc
RH		 4342 /* Test for (pending) buffer overflow. The assumption is that any
4343 one operation beginning below the high water mark cannot overrun
4344 the buffer completely. Thus we can test for overflow after
4345 generating code without having to check during generation. */
644da9b3 4346 if (unlikely((void *)s->code_ptr > s->code_gen_highwater)) {
b125f9dc
RH		 4347 return -1;
4348 }
6e6c4efe
RH		 4349 /* Test for TB overflow, as seen by gen_insn_end_off. */
4350 if (unlikely(tcg_current_code_size(s) > UINT16_MAX)) {
4351 return -2;
4352 }
c896fe29 4353 }
fca8a500
RH		 4354 tcg_debug_assert(num_insns >= 0);
4355 s->gen_insn_end_off[num_insns] = tcg_current_code_size(s);
c45cb8bb 4356
b76f0d8c 4357 /* Generate TB finalization at the end of block */
659ef5cb 4358#ifdef TCG_TARGET_NEED_LDST_LABELS
aeee05f5
RH		 4359 i = tcg_out_ldst_finalize(s);
4360 if (i < 0) {
4361 return i;
23dceda6 4362 }
659ef5cb 4363#endif
57a26946 4364#ifdef TCG_TARGET_NEED_POOL_LABELS
1768987b
RH		 4365 i = tcg_out_pool_finalize(s);
4366 if (i < 0) {
4367 return i;
57a26946
RH		 4368 }
4369#endif
7ecd02a0
RH		 4370 if (!tcg_resolve_relocs(s)) {
4371 return -2;
4372 }
c896fe29 4373
df5d2b16 4374#ifndef CONFIG_TCG_INTERPRETER
c896fe29 4375 /* flush instruction cache */
db0c51a3
RH		 4376 flush_idcache_range((uintptr_t)tcg_splitwx_to_rx(s->code_buf),
4377 (uintptr_t)s->code_buf,
1da8de39 4378 tcg_ptr_byte_diff(s->code_ptr, s->code_buf));
df5d2b16 4379#endif
2aeabc08 4380
1813e175 4381 return tcg_current_code_size(s);
c896fe29
FB		 4382}
4383
a23a9ec6 4384#ifdef CONFIG_PROFILER
3de2faa9 4385void tcg_dump_info(void)
a23a9ec6 4386{
c3fac113
EC		 4387 TCGProfile prof = {};
4388 const TCGProfile *s;
4389 int64_t tb_count;
4390 int64_t tb_div_count;
4391 int64_t tot;
4392
4393 tcg_profile_snapshot_counters(&prof);
4394 s = &prof;
4395 tb_count = s->tb_count;
4396 tb_div_count = tb_count ? tb_count : 1;
4397 tot = s->interm_time + s->code_time;
a23a9ec6 4398
3de2faa9 4399 qemu_printf("JIT cycles %" PRId64 " (%0.3f s at 2.4 GHz)\n",
a23a9ec6 4400 tot, tot / 2.4e9);
3de2faa9
MA		 4401 qemu_printf("translated TBs %" PRId64 " (aborted=%" PRId64
4402 " %0.1f%%)\n",
fca8a500
RH		 4403 tb_count, s->tb_count1 - tb_count,
4404 (double)(s->tb_count1 - s->tb_count)
4405 / (s->tb_count1 ? s->tb_count1 : 1) * 100.0);
3de2faa9 4406 qemu_printf("avg ops/TB %0.1f max=%d\n",
fca8a500 4407 (double)s->op_count / tb_div_count, s->op_count_max);
3de2faa9 4408 qemu_printf("deleted ops/TB %0.2f\n",
fca8a500 4409 (double)s->del_op_count / tb_div_count);
3de2faa9 4410 qemu_printf("avg temps/TB %0.2f max=%d\n",
fca8a500 4411 (double)s->temp_count / tb_div_count, s->temp_count_max);
3de2faa9 4412 qemu_printf("avg host code/TB %0.1f\n",
fca8a500 4413 (double)s->code_out_len / tb_div_count);
3de2faa9 4414 qemu_printf("avg search data/TB %0.1f\n",
fca8a500 4415 (double)s->search_out_len / tb_div_count);
a23a9ec6 4416
3de2faa9 4417 qemu_printf("cycles/op %0.1f\n",
a23a9ec6 4418 s->op_count ? (double)tot / s->op_count : 0);
3de2faa9 4419 qemu_printf("cycles/in byte %0.1f\n",
a23a9ec6 4420 s->code_in_len ? (double)tot / s->code_in_len : 0);
3de2faa9 4421 qemu_printf("cycles/out byte %0.1f\n",
a23a9ec6 4422 s->code_out_len ? (double)tot / s->code_out_len : 0);
3de2faa9 4423 qemu_printf("cycles/search byte %0.1f\n",
fca8a500
RH		 4424 s->search_out_len ? (double)tot / s->search_out_len : 0);
4425 if (tot == 0) {
a23a9ec6 4426 tot = 1;
fca8a500 4427 }
3de2faa9 4428 qemu_printf(" gen_interm time %0.1f%%\n",
a23a9ec6 4429 (double)s->interm_time / tot * 100.0);
3de2faa9 4430 qemu_printf(" gen_code time %0.1f%%\n",
a23a9ec6 4431 (double)s->code_time / tot * 100.0);
3de2faa9 4432 qemu_printf("optim./code time %0.1f%%\n",
c5cc28ff
AJ		 4433 (double)s->opt_time / (s->code_time ? s->code_time : 1)
4434 * 100.0);
3de2faa9 4435 qemu_printf("liveness/code time %0.1f%%\n",
a23a9ec6 4436 (double)s->la_time / (s->code_time ? s->code_time : 1) * 100.0);
3de2faa9 4437 qemu_printf("cpu_restore count %" PRId64 "\n",
a23a9ec6 4438 s->restore_count);
3de2faa9 4439 qemu_printf(" avg cycles %0.1f\n",
a23a9ec6 4440 s->restore_count ? (double)s->restore_time / s->restore_count : 0);
a23a9ec6
FB		 4441}
4442#else
3de2faa9 4443void tcg_dump_info(void)
a23a9ec6 4444{
3de2faa9 4445 qemu_printf("[TCG profiler not compiled]\n");
a23a9ec6
FB		 4446}
4447#endif
813da627
RH		 4448
4449#ifdef ELF_HOST_MACHINE
5872bbf2
RH		 4450/* In order to use this feature, the backend needs to do three things:
4451
4452 (1) Define ELF_HOST_MACHINE to indicate both what value to
4453 put into the ELF image and to indicate support for the feature.
4454
4455 (2) Define tcg_register_jit. This should create a buffer containing
4456 the contents of a .debug_frame section that describes the post-
4457 prologue unwind info for the tcg machine.
4458
4459 (3) Call tcg_register_jit_int, with the constructed .debug_frame.
4460*/
813da627
RH		 4461
4462/* Begin GDB interface. THE FOLLOWING MUST MATCH GDB DOCS. */
4463typedef enum {
4464 JIT_NOACTION = 0,
4465 JIT_REGISTER_FN,
4466 JIT_UNREGISTER_FN
4467} jit_actions_t;
4468
4469struct jit_code_entry {
4470 struct jit_code_entry *next_entry;
4471 struct jit_code_entry *prev_entry;
4472 const void *symfile_addr;
4473 uint64_t symfile_size;
4474};
4475
4476struct jit_descriptor {
4477 uint32_t version;
4478 uint32_t action_flag;
4479 struct jit_code_entry *relevant_entry;
4480 struct jit_code_entry *first_entry;
4481};
4482
4483void __jit_debug_register_code(void) __attribute__((noinline));
4484void __jit_debug_register_code(void)
4485{
4486 asm("");
4487}
4488
4489/* Must statically initialize the version, because GDB may check
4490 the version before we can set it. */
4491struct jit_descriptor __jit_debug_descriptor = { 1, 0, 0, 0 };
4492
4493/* End GDB interface. */
4494
4495static int find_string(const char *strtab, const char *str)
4496{
4497 const char *p = strtab + 1;
4498
4499 while (1) {
4500 if (strcmp(p, str) == 0) {
4501 return p - strtab;
4502 }
4503 p += strlen(p) + 1;
4504 }
4505}
4506
755bf9e5 4507static void tcg_register_jit_int(const void *buf_ptr, size_t buf_size,
2c90784a
RH		 4508 const void *debug_frame,
4509 size_t debug_frame_size)
813da627 4510{
5872bbf2
RH		 4511 struct __attribute__((packed)) DebugInfo {
4512 uint32_t len;
4513 uint16_t version;
4514 uint32_t abbrev;
4515 uint8_t ptr_size;
4516 uint8_t cu_die;
4517 uint16_t cu_lang;
4518 uintptr_t cu_low_pc;
4519 uintptr_t cu_high_pc;
4520 uint8_t fn_die;
4521 char fn_name[16];
4522 uintptr_t fn_low_pc;
4523 uintptr_t fn_high_pc;
4524 uint8_t cu_eoc;
4525 };
813da627
RH		 4526
4527 struct ElfImage {
4528 ElfW(Ehdr) ehdr;
4529 ElfW(Phdr) phdr;
5872bbf2
RH		 4530 ElfW(Shdr) shdr[7];
4531 ElfW(Sym) sym[2];
4532 struct DebugInfo di;
4533 uint8_t da[24];
4534 char str[80];
4535 };
4536
4537 struct ElfImage *img;
4538
4539 static const struct ElfImage img_template = {
4540 .ehdr = {
4541 .e_ident[EI_MAG0] = ELFMAG0,
4542 .e_ident[EI_MAG1] = ELFMAG1,
4543 .e_ident[EI_MAG2] = ELFMAG2,
4544 .e_ident[EI_MAG3] = ELFMAG3,
4545 .e_ident[EI_CLASS] = ELF_CLASS,
4546 .e_ident[EI_DATA] = ELF_DATA,
4547 .e_ident[EI_VERSION] = EV_CURRENT,
4548 .e_type = ET_EXEC,
4549 .e_machine = ELF_HOST_MACHINE,
4550 .e_version = EV_CURRENT,
4551 .e_phoff = offsetof(struct ElfImage, phdr),
4552 .e_shoff = offsetof(struct ElfImage, shdr),
4553 .e_ehsize = sizeof(ElfW(Shdr)),
4554 .e_phentsize = sizeof(ElfW(Phdr)),
4555 .e_phnum = 1,
4556 .e_shentsize = sizeof(ElfW(Shdr)),
4557 .e_shnum = ARRAY_SIZE(img->shdr),
4558 .e_shstrndx = ARRAY_SIZE(img->shdr) - 1,
abbb3eae
RH		 4559#ifdef ELF_HOST_FLAGS
4560 .e_flags = ELF_HOST_FLAGS,
4561#endif
4562#ifdef ELF_OSABI
4563 .e_ident[EI_OSABI] = ELF_OSABI,
4564#endif
5872bbf2
RH		 4565 },
4566 .phdr = {
4567 .p_type = PT_LOAD,
4568 .p_flags = PF_X,
4569 },
4570 .shdr = {
4571 [0] = { .sh_type = SHT_NULL },
4572 /* Trick: The contents of code_gen_buffer are not present in
4573 this fake ELF file; that got allocated elsewhere. Therefore
4574 we mark .text as SHT_NOBITS (similar to .bss) so that readers
4575 will not look for contents. We can record any address. */
4576 [1] = { /* .text */
4577 .sh_type = SHT_NOBITS,
4578 .sh_flags = SHF_EXECINSTR | SHF_ALLOC,
4579 },
4580 [2] = { /* .debug_info */
4581 .sh_type = SHT_PROGBITS,
4582 .sh_offset = offsetof(struct ElfImage, di),
4583 .sh_size = sizeof(struct DebugInfo),
4584 },
4585 [3] = { /* .debug_abbrev */
4586 .sh_type = SHT_PROGBITS,
4587 .sh_offset = offsetof(struct ElfImage, da),
4588 .sh_size = sizeof(img->da),
4589 },
4590 [4] = { /* .debug_frame */
4591 .sh_type = SHT_PROGBITS,
4592 .sh_offset = sizeof(struct ElfImage),
4593 },
4594 [5] = { /* .symtab */
4595 .sh_type = SHT_SYMTAB,
4596 .sh_offset = offsetof(struct ElfImage, sym),
4597 .sh_size = sizeof(img->sym),
4598 .sh_info = 1,
4599 .sh_link = ARRAY_SIZE(img->shdr) - 1,
4600 .sh_entsize = sizeof(ElfW(Sym)),
4601 },
4602 [6] = { /* .strtab */
4603 .sh_type = SHT_STRTAB,
4604 .sh_offset = offsetof(struct ElfImage, str),
4605 .sh_size = sizeof(img->str),
4606 }
4607 },
4608 .sym = {
4609 [1] = { /* code_gen_buffer */
4610 .st_info = ELF_ST_INFO(STB_GLOBAL, STT_FUNC),
4611 .st_shndx = 1,
4612 }
4613 },
4614 .di = {
4615 .len = sizeof(struct DebugInfo) - 4,
4616 .version = 2,
4617 .ptr_size = sizeof(void *),
4618 .cu_die = 1,
4619 .cu_lang = 0x8001, /* DW_LANG_Mips_Assembler */
4620 .fn_die = 2,
4621 .fn_name = "code_gen_buffer"
4622 },
4623 .da = {
4624 1, /* abbrev number (the cu) */
4625 0x11, 1, /* DW_TAG_compile_unit, has children */
4626 0x13, 0x5, /* DW_AT_language, DW_FORM_data2 */
4627 0x11, 0x1, /* DW_AT_low_pc, DW_FORM_addr */
4628 0x12, 0x1, /* DW_AT_high_pc, DW_FORM_addr */
4629 0, 0, /* end of abbrev */
4630 2, /* abbrev number (the fn) */
4631 0x2e, 0, /* DW_TAG_subprogram, no children */
4632 0x3, 0x8, /* DW_AT_name, DW_FORM_string */
4633 0x11, 0x1, /* DW_AT_low_pc, DW_FORM_addr */
4634 0x12, 0x1, /* DW_AT_high_pc, DW_FORM_addr */
4635 0, 0, /* end of abbrev */
4636 0 /* no more abbrev */
4637 },
4638 .str = "\0" ".text\0" ".debug_info\0" ".debug_abbrev\0"
4639 ".debug_frame\0" ".symtab\0" ".strtab\0" "code_gen_buffer",
813da627
RH		 4640 };
4641
4642 /* We only need a single jit entry; statically allocate it. */
4643 static struct jit_code_entry one_entry;
4644
5872bbf2 4645 uintptr_t buf = (uintptr_t)buf_ptr;
813da627 4646 size_t img_size = sizeof(struct ElfImage) + debug_frame_size;
2c90784a 4647 DebugFrameHeader *dfh;
813da627 4648
5872bbf2
RH		 4649 img = g_malloc(img_size);
4650 *img = img_template;
813da627 4651
5872bbf2
RH		 4652 img->phdr.p_vaddr = buf;
4653 img->phdr.p_paddr = buf;
4654 img->phdr.p_memsz = buf_size;
813da627 4655
813da627 4656 img->shdr[1].sh_name = find_string(img->str, ".text");
5872bbf2 4657 img->shdr[1].sh_addr = buf;
813da627
RH		 4658 img->shdr[1].sh_size = buf_size;
4659
5872bbf2
RH		 4660 img->shdr[2].sh_name = find_string(img->str, ".debug_info");
4661 img->shdr[3].sh_name = find_string(img->str, ".debug_abbrev");
4662
4663 img->shdr[4].sh_name = find_string(img->str, ".debug_frame");
4664 img->shdr[4].sh_size = debug_frame_size;
4665
4666 img->shdr[5].sh_name = find_string(img->str, ".symtab");
4667 img->shdr[6].sh_name = find_string(img->str, ".strtab");
4668
4669 img->sym[1].st_name = find_string(img->str, "code_gen_buffer");
4670 img->sym[1].st_value = buf;
4671 img->sym[1].st_size = buf_size;
813da627 4672
5872bbf2 4673 img->di.cu_low_pc = buf;
45aba097 4674 img->di.cu_high_pc = buf + buf_size;
5872bbf2 4675 img->di.fn_low_pc = buf;
45aba097 4676 img->di.fn_high_pc = buf + buf_size;
813da627 4677
2c90784a
RH		 4678 dfh = (DebugFrameHeader *)(img + 1);
4679 memcpy(dfh, debug_frame, debug_frame_size);
4680 dfh->fde.func_start = buf;
4681 dfh->fde.func_len = buf_size;
4682
813da627
RH		 4683#ifdef DEBUG_JIT
4684 /* Enable this block to be able to debug the ELF image file creation.
4685 One can use readelf, objdump, or other inspection utilities. */
4686 {
4687 FILE *f = fopen("/tmp/qemu.jit", "w+b");
4688 if (f) {
5872bbf2 4689 if (fwrite(img, img_size, 1, f) != img_size) {
813da627
RH		 4690 /* Avoid stupid unused return value warning for fwrite. */
4691 }
4692 fclose(f);
4693 }
4694 }
4695#endif
4696
4697 one_entry.symfile_addr = img;
4698 one_entry.symfile_size = img_size;
4699
4700 __jit_debug_descriptor.action_flag = JIT_REGISTER_FN;
4701 __jit_debug_descriptor.relevant_entry = &one_entry;
4702 __jit_debug_descriptor.first_entry = &one_entry;
4703 __jit_debug_register_code();
4704}
4705#else
5872bbf2
RH		 4706/* No support for the feature. Provide the entry point expected by exec.c,
4707 and implement the internal function we declared earlier. */
813da627 4708
755bf9e5 4709static void tcg_register_jit_int(const void *buf, size_t size,
2c90784a
RH		 4710 const void *debug_frame,
4711 size_t debug_frame_size)
813da627
RH		 4712{
4713}
4714
755bf9e5 4715void tcg_register_jit(const void *buf, size_t buf_size)
813da627
RH		 4716{
4717}
4718#endif /* ELF_HOST_MACHINE */
db432672
RH		 4719
4720#if !TCG_TARGET_MAYBE_vec
4721void tcg_expand_vec_op(TCGOpcode o, TCGType t, unsigned e, TCGArg a0, ...)
4722{
4723 g_assert_not_reached();
4724}
4725#endif
Empty description
This page took 1.865122 seconds and 4 git commands to generate.