]> Git Repo - qemu.git/blame - tcg/tcg.c
projects / qemu.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
include: Move QEMU_MAP_* constants to mmap-alloc.h
[qemu.git] / tcg / tcg.c
CommitLineData
c896fe29
FB		 1/*
2 * Tiny Code Generator for QEMU
3 *
4 * Copyright (c) 2008 Fabrice Bellard
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a copy
7 * of this software and associated documentation files (the "Software"), to deal
8 * in the Software without restriction, including without limitation the rights
9 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10 * copies of the Software, and to permit persons to whom the Software is
11 * furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
22 * THE SOFTWARE.
23 */
24
c896fe29 25/* define it to use liveness analysis (better code) */
8f2e8c07 26#define USE_TCG_OPTIMIZATIONS
c896fe29 27
757e725b 28#include "qemu/osdep.h"
cca82982 29
813da627
RH		 30/* Define to jump the ELF file used to communicate with GDB. */
31#undef DEBUG_JIT
32
72fd2efb 33#include "qemu/error-report.h"
f348b6d1 34#include "qemu/cutils.h"
1de7afc9 35#include "qemu/host-utils.h"
d4c51a0a 36#include "qemu/qemu-print.h"
1de7afc9 37#include "qemu/timer.h"
084cfca1 38#include "qemu/cacheflush.h"
c896fe29 39
c5d3c498 40/* Note: the long term plan is to reduce the dependencies on the QEMU
c896fe29
FB		 41 CPU definitions. Currently they are used for qemu_ld/st
42 instructions */
43#define NO_CPU_IO_DEFS
c896fe29 44
63c91552 45#include "exec/exec-all.h"
dcb32f1d 46#include "tcg/tcg-op.h"
813da627 47
edee2579 48#if UINTPTR_MAX == UINT32_MAX
813da627 49# define ELF_CLASS ELFCLASS32
edee2579
RH		 50#else
51# define ELF_CLASS ELFCLASS64
813da627
RH		 52#endif
53#ifdef HOST_WORDS_BIGENDIAN
54# define ELF_DATA ELFDATA2MSB
55#else
56# define ELF_DATA ELFDATA2LSB
57#endif
58
c896fe29 59#include "elf.h"
508127e2 60#include "exec/log.h"
d2ba8026 61#include "tcg/tcg-ldst.h"
5ff7258c 62#include "tcg-internal.h"
c896fe29 63
22f15579
RH		 64#ifdef CONFIG_TCG_INTERPRETER
65#include <ffi.h>
66#endif
67
139c1837 68/* Forward declarations for functions declared in tcg-target.c.inc and
ce151109 69 used here. */
e4d58b41
RH		 70static void tcg_target_init(TCGContext *s);
71static void tcg_target_qemu_prologue(TCGContext *s);
6ac17786 72static bool patch_reloc(tcg_insn_unit *code_ptr, int type,
2ba7fae2 73 intptr_t value, intptr_t addend);
c896fe29 74
497a22eb
RH		 75/* The CIE and FDE header definitions will be common to all hosts. */
76typedef struct {
77 uint32_t len __attribute__((aligned((sizeof(void *)))));
78 uint32_t id;
79 uint8_t version;
80 char augmentation[1];
81 uint8_t code_align;
82 uint8_t data_align;
83 uint8_t return_column;
84} DebugFrameCIE;
85
86typedef struct QEMU_PACKED {
87 uint32_t len __attribute__((aligned((sizeof(void *)))));
88 uint32_t cie_offset;
edee2579
RH		 89 uintptr_t func_start;
90 uintptr_t func_len;
497a22eb
RH		 91} DebugFrameFDEHeader;
92
2c90784a
RH		 93typedef struct QEMU_PACKED {
94 DebugFrameCIE cie;
95 DebugFrameFDEHeader fde;
96} DebugFrameHeader;
97
755bf9e5 98static void tcg_register_jit_int(const void *buf, size_t size,
2c90784a
RH		 99 const void *debug_frame,
100 size_t debug_frame_size)
813da627
RH		 101 __attribute__((unused));
102
139c1837 103/* Forward declarations for functions declared and used in tcg-target.c.inc. */
2a534aff 104static void tcg_out_ld(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg1,
a05b5b9b 105 intptr_t arg2);
78113e83 106static bool tcg_out_mov(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg);
c0ad3001 107static void tcg_out_movi(TCGContext *s, TCGType type,
2a534aff 108 TCGReg ret, tcg_target_long arg);
5e8892db
MR		 109static void tcg_out_op(TCGContext *s, TCGOpcode opc,
110 const TCGArg args[TCG_MAX_OP_ARGS],
111 const int const_args[TCG_MAX_OP_ARGS]);
d2fd745f 112#if TCG_TARGET_MAYBE_vec
e7632cfa
RH		 113static bool tcg_out_dup_vec(TCGContext *s, TCGType type, unsigned vece,
114 TCGReg dst, TCGReg src);
d6ecb4a9
RH		 115static bool tcg_out_dupm_vec(TCGContext *s, TCGType type, unsigned vece,
116 TCGReg dst, TCGReg base, intptr_t offset);
4e186175
RH		 117static void tcg_out_dupi_vec(TCGContext *s, TCGType type, unsigned vece,
118 TCGReg dst, int64_t arg);
5e8892db
MR		 119static void tcg_out_vec_op(TCGContext *s, TCGOpcode opc,
120 unsigned vecl, unsigned vece,
121 const TCGArg args[TCG_MAX_OP_ARGS],
122 const int const_args[TCG_MAX_OP_ARGS]);
d2fd745f 123#else
e7632cfa
RH		 124static inline bool tcg_out_dup_vec(TCGContext *s, TCGType type, unsigned vece,
125 TCGReg dst, TCGReg src)
126{
127 g_assert_not_reached();
128}
d6ecb4a9
RH		 129static inline bool tcg_out_dupm_vec(TCGContext *s, TCGType type, unsigned vece,
130 TCGReg dst, TCGReg base, intptr_t offset)
131{
132 g_assert_not_reached();
133}
4e186175
RH		 134static inline void tcg_out_dupi_vec(TCGContext *s, TCGType type, unsigned vece,
135 TCGReg dst, int64_t arg)
e7632cfa
RH		 136{
137 g_assert_not_reached();
138}
5e8892db
MR		 139static inline void tcg_out_vec_op(TCGContext *s, TCGOpcode opc,
140 unsigned vecl, unsigned vece,
141 const TCGArg args[TCG_MAX_OP_ARGS],
142 const int const_args[TCG_MAX_OP_ARGS])
d2fd745f
RH		 143{
144 g_assert_not_reached();
145}
146#endif
2a534aff 147static void tcg_out_st(TCGContext *s, TCGType type, TCGReg arg, TCGReg arg1,
a05b5b9b 148 intptr_t arg2);
59d7c14e
RH		 149static bool tcg_out_sti(TCGContext *s, TCGType type, TCGArg val,
150 TCGReg base, intptr_t ofs);
7b7d8b2d
RH		 151#ifdef CONFIG_TCG_INTERPRETER
152static void tcg_out_call(TCGContext *s, const tcg_insn_unit *target,
153 ffi_cif *cif);
154#else
2be7d76b 155static void tcg_out_call(TCGContext *s, const tcg_insn_unit *target);
7b7d8b2d 156#endif
a4fbbd77 157static bool tcg_target_const_match(int64_t val, TCGType type, int ct);
659ef5cb 158#ifdef TCG_TARGET_NEED_LDST_LABELS
aeee05f5 159static int tcg_out_ldst_finalize(TCGContext *s);
659ef5cb 160#endif
c896fe29 161
42eb6dfc
RH		 162TCGContext tcg_init_ctx;
163__thread TCGContext *tcg_ctx;
164
5ff7258c 165TCGContext **tcg_ctxs;
0e2d61cf
RH		 166unsigned int tcg_cur_ctxs;
167unsigned int tcg_max_ctxs;
1c2adb95 168TCGv_env cpu_env = 0;
c8bc1168 169const void *tcg_code_gen_epilogue;
db0c51a3 170uintptr_t tcg_splitwx_diff;
df2cce29 171
b91ccb31
RH		 172#ifndef CONFIG_TCG_INTERPRETER
173tcg_prologue_fn *tcg_qemu_tb_exec;
174#endif
175
d2fd745f 176static TCGRegSet tcg_target_available_regs[TCG_TYPE_COUNT];
b1d8e52e 177static TCGRegSet tcg_target_call_clobber_regs;
c896fe29 178
1813e175 179#if TCG_TARGET_INSN_UNIT_SIZE == 1
4196dca6 180static __attribute__((unused)) inline void tcg_out8(TCGContext *s, uint8_t v)
c896fe29
FB		 181{
182 *s->code_ptr++ = v;
183}
184
4196dca6
PM		 185static __attribute__((unused)) inline void tcg_patch8(tcg_insn_unit *p,
186 uint8_t v)
5c53bb81 187{
1813e175 188 *p = v;
5c53bb81 189}
1813e175 190#endif
5c53bb81 191
1813e175 192#if TCG_TARGET_INSN_UNIT_SIZE <= 2
4196dca6 193static __attribute__((unused)) inline void tcg_out16(TCGContext *s, uint16_t v)
c896fe29 194{
1813e175
RH		 195 if (TCG_TARGET_INSN_UNIT_SIZE == 2) {
196 *s->code_ptr++ = v;
197 } else {
198 tcg_insn_unit *p = s->code_ptr;
199 memcpy(p, &v, sizeof(v));
200 s->code_ptr = p + (2 / TCG_TARGET_INSN_UNIT_SIZE);
201 }
c896fe29
FB		 202}
203
4196dca6
PM		 204static __attribute__((unused)) inline void tcg_patch16(tcg_insn_unit *p,
205 uint16_t v)
5c53bb81 206{
1813e175
RH		 207 if (TCG_TARGET_INSN_UNIT_SIZE == 2) {
208 *p = v;
209 } else {
210 memcpy(p, &v, sizeof(v));
211 }
5c53bb81 212}
1813e175 213#endif
5c53bb81 214
1813e175 215#if TCG_TARGET_INSN_UNIT_SIZE <= 4
4196dca6 216static __attribute__((unused)) inline void tcg_out32(TCGContext *s, uint32_t v)
c896fe29 217{
1813e175
RH		 218 if (TCG_TARGET_INSN_UNIT_SIZE == 4) {
219 *s->code_ptr++ = v;
220 } else {
221 tcg_insn_unit *p = s->code_ptr;
222 memcpy(p, &v, sizeof(v));
223 s->code_ptr = p + (4 / TCG_TARGET_INSN_UNIT_SIZE);
224 }
c896fe29
FB		 225}
226
4196dca6
PM		 227static __attribute__((unused)) inline void tcg_patch32(tcg_insn_unit *p,
228 uint32_t v)
5c53bb81 229{
1813e175
RH		 230 if (TCG_TARGET_INSN_UNIT_SIZE == 4) {
231 *p = v;
232 } else {
233 memcpy(p, &v, sizeof(v));
234 }
5c53bb81 235}
1813e175 236#endif
5c53bb81 237
1813e175 238#if TCG_TARGET_INSN_UNIT_SIZE <= 8
4196dca6 239static __attribute__((unused)) inline void tcg_out64(TCGContext *s, uint64_t v)
ac26eb69 240{
1813e175
RH		 241 if (TCG_TARGET_INSN_UNIT_SIZE == 8) {
242 *s->code_ptr++ = v;
243 } else {
244 tcg_insn_unit *p = s->code_ptr;
245 memcpy(p, &v, sizeof(v));
246 s->code_ptr = p + (8 / TCG_TARGET_INSN_UNIT_SIZE);
247 }
ac26eb69
RH		 248}
249
4196dca6
PM		 250static __attribute__((unused)) inline void tcg_patch64(tcg_insn_unit *p,
251 uint64_t v)
5c53bb81 252{
1813e175
RH		 253 if (TCG_TARGET_INSN_UNIT_SIZE == 8) {
254 *p = v;
255 } else {
256 memcpy(p, &v, sizeof(v));
257 }
5c53bb81 258}
1813e175 259#endif
5c53bb81 260
c896fe29
FB		 261/* label relocation processing */
262
1813e175 263static void tcg_out_reloc(TCGContext *s, tcg_insn_unit *code_ptr, int type,
bec16311 264 TCGLabel *l, intptr_t addend)
c896fe29 265{
7ecd02a0 266 TCGRelocation *r = tcg_malloc(sizeof(TCGRelocation));
c896fe29 267
7ecd02a0
RH		 268 r->type = type;
269 r->ptr = code_ptr;
270 r->addend = addend;
271 QSIMPLEQ_INSERT_TAIL(&l->relocs, r, next);
c896fe29
FB		 272}
273
92ab8e7d 274static void tcg_out_label(TCGContext *s, TCGLabel *l)
c896fe29 275{
eabb7b91 276 tcg_debug_assert(!l->has_value);
c896fe29 277 l->has_value = 1;
92ab8e7d 278 l->u.value_ptr = tcg_splitwx_to_rx(s->code_ptr);
c896fe29
FB		 279}
280
42a268c2 281TCGLabel *gen_new_label(void)
c896fe29 282{
b1311c4a 283 TCGContext *s = tcg_ctx;
51e3972c 284 TCGLabel *l = tcg_malloc(sizeof(TCGLabel));
c896fe29 285
7ecd02a0
RH		 286 memset(l, 0, sizeof(TCGLabel));
287 l->id = s->nb_labels++;
288 QSIMPLEQ_INIT(&l->relocs);
289
bef16ab4 290 QSIMPLEQ_INSERT_TAIL(&s->labels, l, next);
42a268c2
RH		 291
292 return l;
c896fe29
FB		 293}
294
7ecd02a0
RH		 295static bool tcg_resolve_relocs(TCGContext *s)
296{
297 TCGLabel *l;
298
299 QSIMPLEQ_FOREACH(l, &s->labels, next) {
300 TCGRelocation *r;
301 uintptr_t value = l->u.value;
302
303 QSIMPLEQ_FOREACH(r, &l->relocs, next) {
304 if (!patch_reloc(r->ptr, r->type, value, r->addend)) {
305 return false;
306 }
307 }
308 }
309 return true;
310}
311
9f754620
RH		 312static void set_jmp_reset_offset(TCGContext *s, int which)
313{
f14bed3f
RH		 314 /*
315 * We will check for overflow at the end of the opcode loop in
316 * tcg_gen_code, where we bound tcg_current_code_size to UINT16_MAX.
317 */
318 s->tb_jmp_reset_offset[which] = tcg_current_code_size(s);
9f754620
RH		 319}
320
db6b7d0c
RH		 321/* Signal overflow, starting over with fewer guest insns. */
322static void QEMU_NORETURN tcg_raise_tb_overflow(TCGContext *s)
323{
324 siglongjmp(s->jmp_trans, -2);
325}
326
4c22e840
RH		 327#define C_PFX1(P, A) P##A
328#define C_PFX2(P, A, B) P##A##_##B
329#define C_PFX3(P, A, B, C) P##A##_##B##_##C
330#define C_PFX4(P, A, B, C, D) P##A##_##B##_##C##_##D
331#define C_PFX5(P, A, B, C, D, E) P##A##_##B##_##C##_##D##_##E
332#define C_PFX6(P, A, B, C, D, E, F) P##A##_##B##_##C##_##D##_##E##_##F
333
334/* Define an enumeration for the various combinations. */
335
336#define C_O0_I1(I1) C_PFX1(c_o0_i1_, I1),
337#define C_O0_I2(I1, I2) C_PFX2(c_o0_i2_, I1, I2),
338#define C_O0_I3(I1, I2, I3) C_PFX3(c_o0_i3_, I1, I2, I3),
339#define C_O0_I4(I1, I2, I3, I4) C_PFX4(c_o0_i4_, I1, I2, I3, I4),
340
341#define C_O1_I1(O1, I1) C_PFX2(c_o1_i1_, O1, I1),
342#define C_O1_I2(O1, I1, I2) C_PFX3(c_o1_i2_, O1, I1, I2),
343#define C_O1_I3(O1, I1, I2, I3) C_PFX4(c_o1_i3_, O1, I1, I2, I3),
344#define C_O1_I4(O1, I1, I2, I3, I4) C_PFX5(c_o1_i4_, O1, I1, I2, I3, I4),
345
346#define C_N1_I2(O1, I1, I2) C_PFX3(c_n1_i2_, O1, I1, I2),
347
348#define C_O2_I1(O1, O2, I1) C_PFX3(c_o2_i1_, O1, O2, I1),
349#define C_O2_I2(O1, O2, I1, I2) C_PFX4(c_o2_i2_, O1, O2, I1, I2),
350#define C_O2_I3(O1, O2, I1, I2, I3) C_PFX5(c_o2_i3_, O1, O2, I1, I2, I3),
351#define C_O2_I4(O1, O2, I1, I2, I3, I4) C_PFX6(c_o2_i4_, O1, O2, I1, I2, I3, I4),
352
353typedef enum {
354#include "tcg-target-con-set.h"
355} TCGConstraintSetIndex;
356
357static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode);
358
359#undef C_O0_I1
360#undef C_O0_I2
361#undef C_O0_I3
362#undef C_O0_I4
363#undef C_O1_I1
364#undef C_O1_I2
365#undef C_O1_I3
366#undef C_O1_I4
367#undef C_N1_I2
368#undef C_O2_I1
369#undef C_O2_I2
370#undef C_O2_I3
371#undef C_O2_I4
372
373/* Put all of the constraint sets into an array, indexed by the enum. */
374
375#define C_O0_I1(I1) { .args_ct_str = { #I1 } },
376#define C_O0_I2(I1, I2) { .args_ct_str = { #I1, #I2 } },
377#define C_O0_I3(I1, I2, I3) { .args_ct_str = { #I1, #I2, #I3 } },
378#define C_O0_I4(I1, I2, I3, I4) { .args_ct_str = { #I1, #I2, #I3, #I4 } },
379
380#define C_O1_I1(O1, I1) { .args_ct_str = { #O1, #I1 } },
381#define C_O1_I2(O1, I1, I2) { .args_ct_str = { #O1, #I1, #I2 } },
382#define C_O1_I3(O1, I1, I2, I3) { .args_ct_str = { #O1, #I1, #I2, #I3 } },
383#define C_O1_I4(O1, I1, I2, I3, I4) { .args_ct_str = { #O1, #I1, #I2, #I3, #I4 } },
384
385#define C_N1_I2(O1, I1, I2) { .args_ct_str = { "&" #O1, #I1, #I2 } },
386
387#define C_O2_I1(O1, O2, I1) { .args_ct_str = { #O1, #O2, #I1 } },
388#define C_O2_I2(O1, O2, I1, I2) { .args_ct_str = { #O1, #O2, #I1, #I2 } },
389#define C_O2_I3(O1, O2, I1, I2, I3) { .args_ct_str = { #O1, #O2, #I1, #I2, #I3 } },
390#define C_O2_I4(O1, O2, I1, I2, I3, I4) { .args_ct_str = { #O1, #O2, #I1, #I2, #I3, #I4 } },
391
392static const TCGTargetOpDef constraint_sets[] = {
393#include "tcg-target-con-set.h"
394};
395
396
397#undef C_O0_I1
398#undef C_O0_I2
399#undef C_O0_I3
400#undef C_O0_I4
401#undef C_O1_I1
402#undef C_O1_I2
403#undef C_O1_I3
404#undef C_O1_I4
405#undef C_N1_I2
406#undef C_O2_I1
407#undef C_O2_I2
408#undef C_O2_I3
409#undef C_O2_I4
410
411/* Expand the enumerator to be returned from tcg_target_op_def(). */
412
413#define C_O0_I1(I1) C_PFX1(c_o0_i1_, I1)
414#define C_O0_I2(I1, I2) C_PFX2(c_o0_i2_, I1, I2)
415#define C_O0_I3(I1, I2, I3) C_PFX3(c_o0_i3_, I1, I2, I3)
416#define C_O0_I4(I1, I2, I3, I4) C_PFX4(c_o0_i4_, I1, I2, I3, I4)
417
418#define C_O1_I1(O1, I1) C_PFX2(c_o1_i1_, O1, I1)
419#define C_O1_I2(O1, I1, I2) C_PFX3(c_o1_i2_, O1, I1, I2)
420#define C_O1_I3(O1, I1, I2, I3) C_PFX4(c_o1_i3_, O1, I1, I2, I3)
421#define C_O1_I4(O1, I1, I2, I3, I4) C_PFX5(c_o1_i4_, O1, I1, I2, I3, I4)
422
423#define C_N1_I2(O1, I1, I2) C_PFX3(c_n1_i2_, O1, I1, I2)
424
425#define C_O2_I1(O1, O2, I1) C_PFX3(c_o2_i1_, O1, O2, I1)
426#define C_O2_I2(O1, O2, I1, I2) C_PFX4(c_o2_i2_, O1, O2, I1, I2)
427#define C_O2_I3(O1, O2, I1, I2, I3) C_PFX5(c_o2_i3_, O1, O2, I1, I2, I3)
428#define C_O2_I4(O1, O2, I1, I2, I3, I4) C_PFX6(c_o2_i4_, O1, O2, I1, I2, I3, I4)
429
139c1837 430#include "tcg-target.c.inc"
c896fe29 431
38b47b19
EC		 432static void alloc_tcg_plugin_context(TCGContext *s)
433{
434#ifdef CONFIG_PLUGIN
435 s->plugin_tb = g_new0(struct qemu_plugin_tb, 1);
436 s->plugin_tb->insns =
437 g_ptr_array_new_with_free_func(qemu_plugin_insn_cleanup_fn);
438#endif
439}
440
3468b59e
EC		 441/*
442 * All TCG threads except the parent (i.e. the one that called tcg_context_init
443 * and registered the target's TCG globals) must register with this function
444 * before initiating translation.
445 *
446 * In user-mode we just point tcg_ctx to tcg_init_ctx. See the documentation
447 * of tcg_region_init() for the reasoning behind this.
448 *
449 * In softmmu each caller registers its context in tcg_ctxs[]. Note that in
450 * softmmu tcg_ctxs[] does not track tcg_ctx_init, since the initial context
451 * is not used anymore for translation once this function is called.
452 *
453 * Not tracking tcg_init_ctx in tcg_ctxs[] in softmmu keeps code that iterates
454 * over the array (e.g. tcg_code_size() the same for both softmmu and user-mode.
455 */
456#ifdef CONFIG_USER_ONLY
457void tcg_register_thread(void)
458{
459 tcg_ctx = &tcg_init_ctx;
460}
461#else
462void tcg_register_thread(void)
463{
464 TCGContext *s = g_malloc(sizeof(*s));
465 unsigned int i, n;
3468b59e
EC		 466
467 *s = tcg_init_ctx;
468
469 /* Relink mem_base. */
470 for (i = 0, n = tcg_init_ctx.nb_globals; i < n; ++i) {
471 if (tcg_init_ctx.temps[i].mem_base) {
472 ptrdiff_t b = tcg_init_ctx.temps[i].mem_base - tcg_init_ctx.temps;
473 tcg_debug_assert(b >= 0 && b < n);
474 s->temps[i].mem_base = &s->temps[b];
475 }
476 }
477
478 /* Claim an entry in tcg_ctxs */
0e2d61cf
RH		 479 n = qatomic_fetch_inc(&tcg_cur_ctxs);
480 g_assert(n < tcg_max_ctxs);
d73415a3 481 qatomic_set(&tcg_ctxs[n], s);
3468b59e 482
38b47b19
EC		 483 if (n > 0) {
484 alloc_tcg_plugin_context(s);
bf042e8e 485 tcg_region_initial_alloc(s);
38b47b19
EC		 486 }
487
3468b59e 488 tcg_ctx = s;
e8feb96f 489}
3468b59e 490#endif /* !CONFIG_USER_ONLY */
e8feb96f 491
c896fe29
FB		 492/* pool based memory allocation */
493void *tcg_malloc_internal(TCGContext *s, int size)
494{
495 TCGPool *p;
496 int pool_size;
497
498 if (size > TCG_POOL_CHUNK_SIZE) {
499 /* big malloc: insert a new pool (XXX: could optimize) */
7267c094 500 p = g_malloc(sizeof(TCGPool) + size);
c896fe29 501 p->size = size;
4055299e
KB		 502 p->next = s->pool_first_large;
503 s->pool_first_large = p;
504 return p->data;
c896fe29
FB		 505 } else {
506 p = s->pool_current;
507 if (!p) {
508 p = s->pool_first;
509 if (!p)
510 goto new_pool;
511 } else {
512 if (!p->next) {
513 new_pool:
514 pool_size = TCG_POOL_CHUNK_SIZE;
7267c094 515 p = g_malloc(sizeof(TCGPool) + pool_size);
c896fe29
FB		 516 p->size = pool_size;
517 p->next = NULL;
518 if (s->pool_current)
519 s->pool_current->next = p;
520 else
521 s->pool_first = p;
522 } else {
523 p = p->next;
524 }
525 }
526 }
527 s->pool_current = p;
528 s->pool_cur = p->data + size;
529 s->pool_end = p->data + p->size;
530 return p->data;
531}
532
533void tcg_pool_reset(TCGContext *s)
534{
4055299e
KB		 535 TCGPool *p, *t;
536 for (p = s->pool_first_large; p; p = t) {
537 t = p->next;
538 g_free(p);
539 }
540 s->pool_first_large = NULL;
c896fe29
FB		 541 s->pool_cur = s->pool_end = NULL;
542 s->pool_current = NULL;
543}
544
2ef6175a
RH		 545#include "exec/helper-proto.h"
546
100b5e01 547static const TCGHelperInfo all_helpers[] = {
2ef6175a 548#include "exec/helper-tcg.h"
100b5e01 549};
619205fd 550static GHashTable *helper_table;
100b5e01 551
22f15579
RH		 552#ifdef CONFIG_TCG_INTERPRETER
553static GHashTable *ffi_table;
554
555static ffi_type * const typecode_to_ffi[8] = {
556 [dh_typecode_void] = &ffi_type_void,
557 [dh_typecode_i32] = &ffi_type_uint32,
558 [dh_typecode_s32] = &ffi_type_sint32,
559 [dh_typecode_i64] = &ffi_type_uint64,
560 [dh_typecode_s64] = &ffi_type_sint64,
561 [dh_typecode_ptr] = &ffi_type_pointer,
562};
563#endif
564
91478cef 565static int indirect_reg_alloc_order[ARRAY_SIZE(tcg_target_reg_alloc_order)];
f69d277e 566static void process_op_defs(TCGContext *s);
1c2adb95
RH		 567static TCGTemp *tcg_global_reg_new_internal(TCGContext *s, TCGType type,
568 TCGReg reg, const char *name);
91478cef 569
43b972b7 570static void tcg_context_init(unsigned max_cpus)
c896fe29 571{
a76aabd3 572 TCGContext *s = &tcg_init_ctx;
100b5e01 573 int op, total_args, n, i;
c896fe29
FB		 574 TCGOpDef *def;
575 TCGArgConstraint *args_ct;
1c2adb95 576 TCGTemp *ts;
c896fe29
FB		 577
578 memset(s, 0, sizeof(*s));
c896fe29 579 s->nb_globals = 0;
c70fbf0a 580
c896fe29
FB		 581 /* Count total number of arguments and allocate the corresponding
582 space */
583 total_args = 0;
584 for(op = 0; op < NB_OPS; op++) {
585 def = &tcg_op_defs[op];
586 n = def->nb_iargs + def->nb_oargs;
587 total_args += n;
588 }
589
bc2b17e6 590 args_ct = g_new0(TCGArgConstraint, total_args);
c896fe29
FB		 591
592 for(op = 0; op < NB_OPS; op++) {
593 def = &tcg_op_defs[op];
594 def->args_ct = args_ct;
c896fe29 595 n = def->nb_iargs + def->nb_oargs;
c896fe29
FB		 596 args_ct += n;
597 }
5cd8f621
RH		 598
599 /* Register helpers. */
84fd9dd3 600 /* Use g_direct_hash/equal for direct pointer comparisons on func. */
619205fd 601 helper_table = g_hash_table_new(NULL, NULL);
84fd9dd3 602
100b5e01 603 for (i = 0; i < ARRAY_SIZE(all_helpers); ++i) {
84fd9dd3 604 g_hash_table_insert(helper_table, (gpointer)all_helpers[i].func,
72866e82 605 (gpointer)&all_helpers[i]);
100b5e01 606 }
5cd8f621 607
22f15579
RH		 608#ifdef CONFIG_TCG_INTERPRETER
609 /* g_direct_hash/equal for direct comparisons on uint32_t. */
610 ffi_table = g_hash_table_new(NULL, NULL);
611 for (i = 0; i < ARRAY_SIZE(all_helpers); ++i) {
612 struct {
613 ffi_cif cif;
614 ffi_type *args[];
615 } *ca;
616 uint32_t typemask = all_helpers[i].typemask;
617 gpointer hash = (gpointer)(uintptr_t)typemask;
618 ffi_status status;
619 int nargs;
620
621 if (g_hash_table_lookup(ffi_table, hash)) {
622 continue;
623 }
624
625 /* Ignoring the return type, find the last non-zero field. */
626 nargs = 32 - clz32(typemask >> 3);
627 nargs = DIV_ROUND_UP(nargs, 3);
628
629 ca = g_malloc0(sizeof(*ca) + nargs * sizeof(ffi_type *));
630 ca->cif.rtype = typecode_to_ffi[typemask & 7];
631 ca->cif.nargs = nargs;
632
633 if (nargs != 0) {
634 ca->cif.arg_types = ca->args;
635 for (i = 0; i < nargs; ++i) {
636 int typecode = extract32(typemask, (i + 1) * 3, 3);
637 ca->args[i] = typecode_to_ffi[typecode];
638 }
639 }
640
641 status = ffi_prep_cif(&ca->cif, FFI_DEFAULT_ABI, nargs,
642 ca->cif.rtype, ca->cif.arg_types);
643 assert(status == FFI_OK);
644
645 g_hash_table_insert(ffi_table, hash, (gpointer)&ca->cif);
646 }
647#endif
648
c896fe29 649 tcg_target_init(s);
f69d277e 650 process_op_defs(s);
91478cef
RH		 651
652 /* Reverse the order of the saved registers, assuming they're all at
653 the start of tcg_target_reg_alloc_order. */
654 for (n = 0; n < ARRAY_SIZE(tcg_target_reg_alloc_order); ++n) {
655 int r = tcg_target_reg_alloc_order[n];
656 if (tcg_regset_test_reg(tcg_target_call_clobber_regs, r)) {
657 break;
658 }
659 }
660 for (i = 0; i < n; ++i) {
661 indirect_reg_alloc_order[i] = tcg_target_reg_alloc_order[n - 1 - i];
662 }
663 for (; i < ARRAY_SIZE(tcg_target_reg_alloc_order); ++i) {
664 indirect_reg_alloc_order[i] = tcg_target_reg_alloc_order[i];
665 }
b1311c4a 666
38b47b19
EC		 667 alloc_tcg_plugin_context(s);
668
b1311c4a 669 tcg_ctx = s;
3468b59e
EC		 670 /*
671 * In user-mode we simply share the init context among threads, since we
672 * use a single region. See the documentation tcg_region_init() for the
673 * reasoning behind this.
674 * In softmmu we will have at most max_cpus TCG threads.
675 */
676#ifdef CONFIG_USER_ONLY
df2cce29 677 tcg_ctxs = &tcg_ctx;
0e2d61cf
RH		 678 tcg_cur_ctxs = 1;
679 tcg_max_ctxs = 1;
3468b59e 680#else
0e2d61cf
RH		 681 tcg_max_ctxs = max_cpus;
682 tcg_ctxs = g_new0(TCGContext *, max_cpus);
3468b59e 683#endif
1c2adb95
RH		 684
685 tcg_debug_assert(!tcg_regset_test_reg(s->reserved_regs, TCG_AREG0));
686 ts = tcg_global_reg_new_internal(s, TCG_TYPE_PTR, TCG_AREG0, "env");
687 cpu_env = temp_tcgv_ptr(ts);
9002ec79 688}
b03cce8e 689
43b972b7 690void tcg_init(size_t tb_size, int splitwx, unsigned max_cpus)
a76aabd3 691{
43b972b7
RH		 692 tcg_context_init(max_cpus);
693 tcg_region_init(tb_size, splitwx, max_cpus);
a76aabd3
RH		 694}
695
6e3b2bfd
EC		 696/*
697 * Allocate TBs right before their corresponding translated code, making
698 * sure that TBs and code are on different cache lines.
699 */
700TranslationBlock *tcg_tb_alloc(TCGContext *s)
701{
702 uintptr_t align = qemu_icache_linesize;
703 TranslationBlock *tb;
704 void *next;
705
e8feb96f 706 retry:
6e3b2bfd
EC		 707 tb = (void *)ROUND_UP((uintptr_t)s->code_gen_ptr, align);
708 next = (void *)ROUND_UP((uintptr_t)(tb + 1), align);
709
710 if (unlikely(next > s->code_gen_highwater)) {
e8feb96f
EC		 711 if (tcg_region_alloc(s)) {
712 return NULL;
713 }
714 goto retry;
6e3b2bfd 715 }
d73415a3 716 qatomic_set(&s->code_gen_ptr, next);
57a26946 717 s->data_gen_ptr = NULL;
6e3b2bfd
EC		 718 return tb;
719}
720
9002ec79
RH		 721void tcg_prologue_init(TCGContext *s)
722{
b0a0794a 723 size_t prologue_size;
8163b749 724
b0a0794a
RH		 725 s->code_ptr = s->code_gen_ptr;
726 s->code_buf = s->code_gen_ptr;
5b38ee31 727 s->data_gen_ptr = NULL;
b91ccb31
RH		 728
729#ifndef CONFIG_TCG_INTERPRETER
b0a0794a 730 tcg_qemu_tb_exec = (tcg_prologue_fn *)tcg_splitwx_to_rx(s->code_ptr);
b91ccb31 731#endif
8163b749 732
5b38ee31
RH		 733#ifdef TCG_TARGET_NEED_POOL_LABELS
734 s->pool_labels = NULL;
735#endif
736
653b87eb 737 qemu_thread_jit_write();
8163b749 738 /* Generate the prologue. */
b03cce8e 739 tcg_target_qemu_prologue(s);
5b38ee31
RH		 740
741#ifdef TCG_TARGET_NEED_POOL_LABELS
742 /* Allow the prologue to put e.g. guest_base into a pool entry. */
743 {
1768987b
RH		 744 int result = tcg_out_pool_finalize(s);
745 tcg_debug_assert(result == 0);
5b38ee31
RH		 746 }
747#endif
748
b0a0794a
RH		 749 prologue_size = tcg_current_code_size(s);
750
df5d2b16 751#ifndef CONFIG_TCG_INTERPRETER
b0a0794a
RH		 752 flush_idcache_range((uintptr_t)tcg_splitwx_to_rx(s->code_buf),
753 (uintptr_t)s->code_buf, prologue_size);
df5d2b16 754#endif
8163b749 755
d6b64b2b
RH		 756#ifdef DEBUG_DISAS
757 if (qemu_loglevel_mask(CPU_LOG_TB_OUT_ASM)) {
fc59d2d8 758 FILE *logfile = qemu_log_lock();
8163b749 759 qemu_log("PROLOGUE: [size=%zu]\n", prologue_size);
5b38ee31 760 if (s->data_gen_ptr) {
b0a0794a 761 size_t code_size = s->data_gen_ptr - s->code_gen_ptr;
5b38ee31
RH		 762 size_t data_size = prologue_size - code_size;
763 size_t i;
764
b0a0794a 765 log_disas(s->code_gen_ptr, code_size);
5b38ee31
RH		 766
767 for (i = 0; i < data_size; i += sizeof(tcg_target_ulong)) {
768 if (sizeof(tcg_target_ulong) == 8) {
769 qemu_log("0x%08" PRIxPTR ": .quad 0x%016" PRIx64 "\n",
770 (uintptr_t)s->data_gen_ptr + i,
771 *(uint64_t *)(s->data_gen_ptr + i));
772 } else {
773 qemu_log("0x%08" PRIxPTR ": .long 0x%08x\n",
774 (uintptr_t)s->data_gen_ptr + i,
775 *(uint32_t *)(s->data_gen_ptr + i));
776 }
777 }
778 } else {
b0a0794a 779 log_disas(s->code_gen_ptr, prologue_size);
5b38ee31 780 }
d6b64b2b
RH		 781 qemu_log("\n");
782 qemu_log_flush();
fc59d2d8 783 qemu_log_unlock(logfile);
d6b64b2b
RH		 784 }
785#endif
cedbcb01 786
6eea0434
RH		 787#ifndef CONFIG_TCG_INTERPRETER
788 /*
789 * Assert that goto_ptr is implemented completely, setting an epilogue.
790 * For tci, we use NULL as the signal to return from the interpreter,
791 * so skip this check.
792 */
f4e01e30 793 tcg_debug_assert(tcg_code_gen_epilogue != NULL);
6eea0434 794#endif
d1c74ab3
RH		 795
796 tcg_region_prologue_set(s);
c896fe29
FB		 797}
798
c896fe29
FB		 799void tcg_func_start(TCGContext *s)
800{
801 tcg_pool_reset(s);
802 s->nb_temps = s->nb_globals;
0ec9eabc
RH		 803
804 /* No temps have been previously allocated for size or locality. */
805 memset(s->free_temps, 0, sizeof(s->free_temps));
806
c0522136
RH		 807 /* No constant temps have been previously allocated. */
808 for (int i = 0; i < TCG_TYPE_COUNT; ++i) {
809 if (s->const_table[i]) {
810 g_hash_table_remove_all(s->const_table[i]);
811 }
812 }
813
abebf925 814 s->nb_ops = 0;
c896fe29
FB		 815 s->nb_labels = 0;
816 s->current_frame_offset = s->frame_start;
817
0a209d4b
RH		 818#ifdef CONFIG_DEBUG_TCG
819 s->goto_tb_issue_mask = 0;
820#endif
821
15fa08f8
RH		 822 QTAILQ_INIT(&s->ops);
823 QTAILQ_INIT(&s->free_ops);
bef16ab4 824 QSIMPLEQ_INIT(&s->labels);
c896fe29
FB		 825}
826
ae30e866 827static TCGTemp *tcg_temp_alloc(TCGContext *s)
7ca4b752
RH		 828{
829 int n = s->nb_temps++;
ae30e866
RH		 830
831 if (n >= TCG_MAX_TEMPS) {
db6b7d0c 832 tcg_raise_tb_overflow(s);
ae30e866 833 }
7ca4b752
RH		 834 return memset(&s->temps[n], 0, sizeof(TCGTemp));
835}
836
ae30e866 837static TCGTemp *tcg_global_alloc(TCGContext *s)
7ca4b752 838{
fa477d25
RH		 839 TCGTemp *ts;
840
7ca4b752 841 tcg_debug_assert(s->nb_globals == s->nb_temps);
ae30e866 842 tcg_debug_assert(s->nb_globals < TCG_MAX_TEMPS);
7ca4b752 843 s->nb_globals++;
fa477d25 844 ts = tcg_temp_alloc(s);
ee17db83 845 ts->kind = TEMP_GLOBAL;
fa477d25
RH		 846
847 return ts;
c896fe29
FB		 848}
849
085272b3
RH		 850static TCGTemp *tcg_global_reg_new_internal(TCGContext *s, TCGType type,
851 TCGReg reg, const char *name)
c896fe29 852{
c896fe29 853 TCGTemp *ts;
c896fe29 854
b3a62939 855 if (TCG_TARGET_REG_BITS == 32 && type != TCG_TYPE_I32) {
c896fe29 856 tcg_abort();
b3a62939 857 }
7ca4b752
RH		 858
859 ts = tcg_global_alloc(s);
c896fe29
FB		 860 ts->base_type = type;
861 ts->type = type;
ee17db83 862 ts->kind = TEMP_FIXED;
c896fe29 863 ts->reg = reg;
c896fe29 864 ts->name = name;
c896fe29 865 tcg_regset_set_reg(s->reserved_regs, reg);
7ca4b752 866
085272b3 867 return ts;
a7812ae4
PB		 868}
869
b6638662 870void tcg_set_frame(TCGContext *s, TCGReg reg, intptr_t start, intptr_t size)
b3a62939 871{
b3a62939
RH		 872 s->frame_start = start;
873 s->frame_end = start + size;
085272b3
RH		 874 s->frame_temp
875 = tcg_global_reg_new_internal(s, TCG_TYPE_PTR, reg, "_frame");
b3a62939
RH		 876}
877
085272b3
RH		 878TCGTemp *tcg_global_mem_new_internal(TCGType type, TCGv_ptr base,
879 intptr_t offset, const char *name)
c896fe29 880{
b1311c4a 881 TCGContext *s = tcg_ctx;
dc41aa7d 882 TCGTemp *base_ts = tcgv_ptr_temp(base);
7ca4b752 883 TCGTemp *ts = tcg_global_alloc(s);
b3915dbb 884 int indirect_reg = 0, bigendian = 0;
7ca4b752
RH		 885#ifdef HOST_WORDS_BIGENDIAN
886 bigendian = 1;
887#endif
c896fe29 888
c0522136
RH		 889 switch (base_ts->kind) {
890 case TEMP_FIXED:
891 break;
892 case TEMP_GLOBAL:
5a18407f
RH		 893 /* We do not support double-indirect registers. */
894 tcg_debug_assert(!base_ts->indirect_reg);
b3915dbb 895 base_ts->indirect_base = 1;
5a18407f
RH		 896 s->nb_indirects += (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64
897 ? 2 : 1);
898 indirect_reg = 1;
c0522136
RH		 899 break;
900 default:
901 g_assert_not_reached();
b3915dbb
RH		 902 }
903
7ca4b752
RH		 904 if (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64) {
905 TCGTemp *ts2 = tcg_global_alloc(s);
c896fe29 906 char buf[64];
7ca4b752
RH		 907
908 ts->base_type = TCG_TYPE_I64;
c896fe29 909 ts->type = TCG_TYPE_I32;
b3915dbb 910 ts->indirect_reg = indirect_reg;
c896fe29 911 ts->mem_allocated = 1;
b3a62939 912 ts->mem_base = base_ts;
7ca4b752 913 ts->mem_offset = offset + bigendian * 4;
c896fe29
FB		 914 pstrcpy(buf, sizeof(buf), name);
915 pstrcat(buf, sizeof(buf), "_0");
916 ts->name = strdup(buf);
c896fe29 917
7ca4b752
RH		 918 tcg_debug_assert(ts2 == ts + 1);
919 ts2->base_type = TCG_TYPE_I64;
920 ts2->type = TCG_TYPE_I32;
b3915dbb 921 ts2->indirect_reg = indirect_reg;
7ca4b752
RH		 922 ts2->mem_allocated = 1;
923 ts2->mem_base = base_ts;
924 ts2->mem_offset = offset + (1 - bigendian) * 4;
c896fe29
FB		 925 pstrcpy(buf, sizeof(buf), name);
926 pstrcat(buf, sizeof(buf), "_1");
120c1084 927 ts2->name = strdup(buf);
7ca4b752 928 } else {
c896fe29
FB		 929 ts->base_type = type;
930 ts->type = type;
b3915dbb 931 ts->indirect_reg = indirect_reg;
c896fe29 932 ts->mem_allocated = 1;
b3a62939 933 ts->mem_base = base_ts;
c896fe29 934 ts->mem_offset = offset;
c896fe29 935 ts->name = name;
c896fe29 936 }
085272b3 937 return ts;
a7812ae4
PB		 938}
939
5bfa8034 940TCGTemp *tcg_temp_new_internal(TCGType type, bool temp_local)
c896fe29 941{
b1311c4a 942 TCGContext *s = tcg_ctx;
ee17db83 943 TCGTempKind kind = temp_local ? TEMP_LOCAL : TEMP_NORMAL;
c896fe29 944 TCGTemp *ts;
641d5fbe 945 int idx, k;
c896fe29 946
0ec9eabc
RH		 947 k = type + (temp_local ? TCG_TYPE_COUNT : 0);
948 idx = find_first_bit(s->free_temps[k].l, TCG_MAX_TEMPS);
949 if (idx < TCG_MAX_TEMPS) {
950 /* There is already an available temp with the right type. */
951 clear_bit(idx, s->free_temps[k].l);
952
e8996ee0 953 ts = &s->temps[idx];
e8996ee0 954 ts->temp_allocated = 1;
7ca4b752 955 tcg_debug_assert(ts->base_type == type);
ee17db83 956 tcg_debug_assert(ts->kind == kind);
e8996ee0 957 } else {
7ca4b752
RH		 958 ts = tcg_temp_alloc(s);
959 if (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64) {
960 TCGTemp *ts2 = tcg_temp_alloc(s);
961
f6aa2f7d 962 ts->base_type = type;
e8996ee0
FB		 963 ts->type = TCG_TYPE_I32;
964 ts->temp_allocated = 1;
ee17db83 965 ts->kind = kind;
7ca4b752
RH		 966
967 tcg_debug_assert(ts2 == ts + 1);
968 ts2->base_type = TCG_TYPE_I64;
969 ts2->type = TCG_TYPE_I32;
970 ts2->temp_allocated = 1;
ee17db83 971 ts2->kind = kind;
7ca4b752 972 } else {
e8996ee0
FB		 973 ts->base_type = type;
974 ts->type = type;
975 ts->temp_allocated = 1;
ee17db83 976 ts->kind = kind;
e8996ee0 977 }
c896fe29 978 }
27bfd83c
PM		 979
980#if defined(CONFIG_DEBUG_TCG)
981 s->temps_in_use++;
982#endif
085272b3 983 return ts;
c896fe29
FB		 984}
985
d2fd745f
RH		 986TCGv_vec tcg_temp_new_vec(TCGType type)
987{
988 TCGTemp *t;
989
990#ifdef CONFIG_DEBUG_TCG
991 switch (type) {
992 case TCG_TYPE_V64:
993 assert(TCG_TARGET_HAS_v64);
994 break;
995 case TCG_TYPE_V128:
996 assert(TCG_TARGET_HAS_v128);
997 break;
998 case TCG_TYPE_V256:
999 assert(TCG_TARGET_HAS_v256);
1000 break;
1001 default:
1002 g_assert_not_reached();
1003 }
1004#endif
1005
1006 t = tcg_temp_new_internal(type, 0);
1007 return temp_tcgv_vec(t);
1008}
1009
1010/* Create a new temp of the same type as an existing temp. */
1011TCGv_vec tcg_temp_new_vec_matching(TCGv_vec match)
1012{
1013 TCGTemp *t = tcgv_vec_temp(match);
1014
1015 tcg_debug_assert(t->temp_allocated != 0);
1016
1017 t = tcg_temp_new_internal(t->base_type, 0);
1018 return temp_tcgv_vec(t);
1019}
1020
5bfa8034 1021void tcg_temp_free_internal(TCGTemp *ts)
c896fe29 1022{
b1311c4a 1023 TCGContext *s = tcg_ctx;
085272b3 1024 int k, idx;
c896fe29 1025
c0522136
RH		 1026 /* In order to simplify users of tcg_constant_*, silently ignore free. */
1027 if (ts->kind == TEMP_CONST) {
1028 return;
1029 }
1030
27bfd83c
PM		 1031#if defined(CONFIG_DEBUG_TCG)
1032 s->temps_in_use--;
1033 if (s->temps_in_use < 0) {
1034 fprintf(stderr, "More temporaries freed than allocated!\n");
1035 }
1036#endif
1037
ee17db83 1038 tcg_debug_assert(ts->kind < TEMP_GLOBAL);
eabb7b91 1039 tcg_debug_assert(ts->temp_allocated != 0);
e8996ee0 1040 ts->temp_allocated = 0;
0ec9eabc 1041
085272b3 1042 idx = temp_idx(ts);
ee17db83 1043 k = ts->base_type + (ts->kind == TEMP_NORMAL ? 0 : TCG_TYPE_COUNT);
0ec9eabc 1044 set_bit(idx, s->free_temps[k].l);
c896fe29
FB		 1045}
1046
c0522136
RH		 1047TCGTemp *tcg_constant_internal(TCGType type, int64_t val)
1048{
1049 TCGContext *s = tcg_ctx;
1050 GHashTable *h = s->const_table[type];
1051 TCGTemp *ts;
1052
1053 if (h == NULL) {
1054 h = g_hash_table_new(g_int64_hash, g_int64_equal);
1055 s->const_table[type] = h;
1056 }
1057
1058 ts = g_hash_table_lookup(h, &val);
1059 if (ts == NULL) {
1060 ts = tcg_temp_alloc(s);
1061
1062 if (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64) {
1063 TCGTemp *ts2 = tcg_temp_alloc(s);
1064
1065 ts->base_type = TCG_TYPE_I64;
1066 ts->type = TCG_TYPE_I32;
1067 ts->kind = TEMP_CONST;
1068 ts->temp_allocated = 1;
1069 /*
1070 * Retain the full value of the 64-bit constant in the low
1071 * part, so that the hash table works. Actual uses will
1072 * truncate the value to the low part.
1073 */
1074 ts->val = val;
1075
1076 tcg_debug_assert(ts2 == ts + 1);
1077 ts2->base_type = TCG_TYPE_I64;
1078 ts2->type = TCG_TYPE_I32;
1079 ts2->kind = TEMP_CONST;
1080 ts2->temp_allocated = 1;
1081 ts2->val = val >> 32;
1082 } else {
1083 ts->base_type = type;
1084 ts->type = type;
1085 ts->kind = TEMP_CONST;
1086 ts->temp_allocated = 1;
1087 ts->val = val;
1088 }
1089 g_hash_table_insert(h, &ts->val, ts);
1090 }
1091
1092 return ts;
1093}
1094
1095TCGv_vec tcg_constant_vec(TCGType type, unsigned vece, int64_t val)
1096{
1097 val = dup_const(vece, val);
1098 return temp_tcgv_vec(tcg_constant_internal(type, val));
1099}
1100
88d4005b
RH		 1101TCGv_vec tcg_constant_vec_matching(TCGv_vec match, unsigned vece, int64_t val)
1102{
1103 TCGTemp *t = tcgv_vec_temp(match);
1104
1105 tcg_debug_assert(t->temp_allocated != 0);
1106 return tcg_constant_vec(t->base_type, vece, val);
1107}
1108
a7812ae4 1109TCGv_i32 tcg_const_i32(int32_t val)
c896fe29 1110{
a7812ae4
PB		 1111 TCGv_i32 t0;
1112 t0 = tcg_temp_new_i32();
e8996ee0
FB		 1113 tcg_gen_movi_i32(t0, val);
1114 return t0;
1115}
c896fe29 1116
a7812ae4 1117TCGv_i64 tcg_const_i64(int64_t val)
e8996ee0 1118{
a7812ae4
PB		 1119 TCGv_i64 t0;
1120 t0 = tcg_temp_new_i64();
e8996ee0
FB		 1121 tcg_gen_movi_i64(t0, val);
1122 return t0;
c896fe29
FB		 1123}
1124
a7812ae4 1125TCGv_i32 tcg_const_local_i32(int32_t val)
bdffd4a9 1126{
a7812ae4
PB		 1127 TCGv_i32 t0;
1128 t0 = tcg_temp_local_new_i32();
bdffd4a9
AJ		 1129 tcg_gen_movi_i32(t0, val);
1130 return t0;
1131}
1132
a7812ae4 1133TCGv_i64 tcg_const_local_i64(int64_t val)
bdffd4a9 1134{
a7812ae4
PB		 1135 TCGv_i64 t0;
1136 t0 = tcg_temp_local_new_i64();
bdffd4a9
AJ		 1137 tcg_gen_movi_i64(t0, val);
1138 return t0;
1139}
1140
27bfd83c
PM		 1141#if defined(CONFIG_DEBUG_TCG)
1142void tcg_clear_temp_count(void)
1143{
b1311c4a 1144 TCGContext *s = tcg_ctx;
27bfd83c
PM		 1145 s->temps_in_use = 0;
1146}
1147
1148int tcg_check_temp_count(void)
1149{
b1311c4a 1150 TCGContext *s = tcg_ctx;
27bfd83c
PM		 1151 if (s->temps_in_use) {
1152 /* Clear the count so that we don't give another
1153 * warning immediately next time around.
1154 */
1155 s->temps_in_use = 0;
1156 return 1;
1157 }
1158 return 0;
1159}
1160#endif
1161
be0f34b5
RH		 1162/* Return true if OP may appear in the opcode stream.
1163 Test the runtime variable that controls each opcode. */
1164bool tcg_op_supported(TCGOpcode op)
1165{
d2fd745f
RH		 1166 const bool have_vec
1167 = TCG_TARGET_HAS_v64 | TCG_TARGET_HAS_v128 | TCG_TARGET_HAS_v256;
1168
be0f34b5
RH		 1169 switch (op) {
1170 case INDEX_op_discard:
1171 case INDEX_op_set_label:
1172 case INDEX_op_call:
1173 case INDEX_op_br:
1174 case INDEX_op_mb:
1175 case INDEX_op_insn_start:
1176 case INDEX_op_exit_tb:
1177 case INDEX_op_goto_tb:
f4e01e30 1178 case INDEX_op_goto_ptr:
be0f34b5
RH		 1179 case INDEX_op_qemu_ld_i32:
1180 case INDEX_op_qemu_st_i32:
1181 case INDEX_op_qemu_ld_i64:
1182 case INDEX_op_qemu_st_i64:
1183 return true;
1184
07ce0b05
RH		 1185 case INDEX_op_qemu_st8_i32:
1186 return TCG_TARGET_HAS_qemu_st8_i32;
1187
be0f34b5 1188 case INDEX_op_mov_i32:
be0f34b5
RH		 1189 case INDEX_op_setcond_i32:
1190 case INDEX_op_brcond_i32:
1191 case INDEX_op_ld8u_i32:
1192 case INDEX_op_ld8s_i32:
1193 case INDEX_op_ld16u_i32:
1194 case INDEX_op_ld16s_i32:
1195 case INDEX_op_ld_i32:
1196 case INDEX_op_st8_i32:
1197 case INDEX_op_st16_i32:
1198 case INDEX_op_st_i32:
1199 case INDEX_op_add_i32:
1200 case INDEX_op_sub_i32:
1201 case INDEX_op_mul_i32:
1202 case INDEX_op_and_i32:
1203 case INDEX_op_or_i32:
1204 case INDEX_op_xor_i32:
1205 case INDEX_op_shl_i32:
1206 case INDEX_op_shr_i32:
1207 case INDEX_op_sar_i32:
1208 return true;
1209
1210 case INDEX_op_movcond_i32:
1211 return TCG_TARGET_HAS_movcond_i32;
1212 case INDEX_op_div_i32:
1213 case INDEX_op_divu_i32:
1214 return TCG_TARGET_HAS_div_i32;
1215 case INDEX_op_rem_i32:
1216 case INDEX_op_remu_i32:
1217 return TCG_TARGET_HAS_rem_i32;
1218 case INDEX_op_div2_i32:
1219 case INDEX_op_divu2_i32:
1220 return TCG_TARGET_HAS_div2_i32;
1221 case INDEX_op_rotl_i32:
1222 case INDEX_op_rotr_i32:
1223 return TCG_TARGET_HAS_rot_i32;
1224 case INDEX_op_deposit_i32:
1225 return TCG_TARGET_HAS_deposit_i32;
1226 case INDEX_op_extract_i32:
1227 return TCG_TARGET_HAS_extract_i32;
1228 case INDEX_op_sextract_i32:
1229 return TCG_TARGET_HAS_sextract_i32;
fce1296f
RH		 1230 case INDEX_op_extract2_i32:
1231 return TCG_TARGET_HAS_extract2_i32;
be0f34b5
RH		 1232 case INDEX_op_add2_i32:
1233 return TCG_TARGET_HAS_add2_i32;
1234 case INDEX_op_sub2_i32:
1235 return TCG_TARGET_HAS_sub2_i32;
1236 case INDEX_op_mulu2_i32:
1237 return TCG_TARGET_HAS_mulu2_i32;
1238 case INDEX_op_muls2_i32:
1239 return TCG_TARGET_HAS_muls2_i32;
1240 case INDEX_op_muluh_i32:
1241 return TCG_TARGET_HAS_muluh_i32;
1242 case INDEX_op_mulsh_i32:
1243 return TCG_TARGET_HAS_mulsh_i32;
1244 case INDEX_op_ext8s_i32:
1245 return TCG_TARGET_HAS_ext8s_i32;
1246 case INDEX_op_ext16s_i32:
1247 return TCG_TARGET_HAS_ext16s_i32;
1248 case INDEX_op_ext8u_i32:
1249 return TCG_TARGET_HAS_ext8u_i32;
1250 case INDEX_op_ext16u_i32:
1251 return TCG_TARGET_HAS_ext16u_i32;
1252 case INDEX_op_bswap16_i32:
1253 return TCG_TARGET_HAS_bswap16_i32;
1254 case INDEX_op_bswap32_i32:
1255 return TCG_TARGET_HAS_bswap32_i32;
1256 case INDEX_op_not_i32:
1257 return TCG_TARGET_HAS_not_i32;
1258 case INDEX_op_neg_i32:
1259 return TCG_TARGET_HAS_neg_i32;
1260 case INDEX_op_andc_i32:
1261 return TCG_TARGET_HAS_andc_i32;
1262 case INDEX_op_orc_i32:
1263 return TCG_TARGET_HAS_orc_i32;
1264 case INDEX_op_eqv_i32:
1265 return TCG_TARGET_HAS_eqv_i32;
1266 case INDEX_op_nand_i32:
1267 return TCG_TARGET_HAS_nand_i32;
1268 case INDEX_op_nor_i32:
1269 return TCG_TARGET_HAS_nor_i32;
1270 case INDEX_op_clz_i32:
1271 return TCG_TARGET_HAS_clz_i32;
1272 case INDEX_op_ctz_i32:
1273 return TCG_TARGET_HAS_ctz_i32;
1274 case INDEX_op_ctpop_i32:
1275 return TCG_TARGET_HAS_ctpop_i32;
1276
1277 case INDEX_op_brcond2_i32:
1278 case INDEX_op_setcond2_i32:
1279 return TCG_TARGET_REG_BITS == 32;
1280
1281 case INDEX_op_mov_i64:
be0f34b5
RH		 1282 case INDEX_op_setcond_i64:
1283 case INDEX_op_brcond_i64:
1284 case INDEX_op_ld8u_i64:
1285 case INDEX_op_ld8s_i64:
1286 case INDEX_op_ld16u_i64:
1287 case INDEX_op_ld16s_i64:
1288 case INDEX_op_ld32u_i64:
1289 case INDEX_op_ld32s_i64:
1290 case INDEX_op_ld_i64:
1291 case INDEX_op_st8_i64:
1292 case INDEX_op_st16_i64:
1293 case INDEX_op_st32_i64:
1294 case INDEX_op_st_i64:
1295 case INDEX_op_add_i64:
1296 case INDEX_op_sub_i64:
1297 case INDEX_op_mul_i64:
1298 case INDEX_op_and_i64:
1299 case INDEX_op_or_i64:
1300 case INDEX_op_xor_i64:
1301 case INDEX_op_shl_i64:
1302 case INDEX_op_shr_i64:
1303 case INDEX_op_sar_i64:
1304 case INDEX_op_ext_i32_i64:
1305 case INDEX_op_extu_i32_i64:
1306 return TCG_TARGET_REG_BITS == 64;
1307
1308 case INDEX_op_movcond_i64:
1309 return TCG_TARGET_HAS_movcond_i64;
1310 case INDEX_op_div_i64:
1311 case INDEX_op_divu_i64:
1312 return TCG_TARGET_HAS_div_i64;
1313 case INDEX_op_rem_i64:
1314 case INDEX_op_remu_i64:
1315 return TCG_TARGET_HAS_rem_i64;
1316 case INDEX_op_div2_i64:
1317 case INDEX_op_divu2_i64:
1318 return TCG_TARGET_HAS_div2_i64;
1319 case INDEX_op_rotl_i64:
1320 case INDEX_op_rotr_i64:
1321 return TCG_TARGET_HAS_rot_i64;
1322 case INDEX_op_deposit_i64:
1323 return TCG_TARGET_HAS_deposit_i64;
1324 case INDEX_op_extract_i64:
1325 return TCG_TARGET_HAS_extract_i64;
1326 case INDEX_op_sextract_i64:
1327 return TCG_TARGET_HAS_sextract_i64;
fce1296f
RH		 1328 case INDEX_op_extract2_i64:
1329 return TCG_TARGET_HAS_extract2_i64;
be0f34b5
RH		 1330 case INDEX_op_extrl_i64_i32:
1331 return TCG_TARGET_HAS_extrl_i64_i32;
1332 case INDEX_op_extrh_i64_i32:
1333 return TCG_TARGET_HAS_extrh_i64_i32;
1334 case INDEX_op_ext8s_i64:
1335 return TCG_TARGET_HAS_ext8s_i64;
1336 case INDEX_op_ext16s_i64:
1337 return TCG_TARGET_HAS_ext16s_i64;
1338 case INDEX_op_ext32s_i64:
1339 return TCG_TARGET_HAS_ext32s_i64;
1340 case INDEX_op_ext8u_i64:
1341 return TCG_TARGET_HAS_ext8u_i64;
1342 case INDEX_op_ext16u_i64:
1343 return TCG_TARGET_HAS_ext16u_i64;
1344 case INDEX_op_ext32u_i64:
1345 return TCG_TARGET_HAS_ext32u_i64;
1346 case INDEX_op_bswap16_i64:
1347 return TCG_TARGET_HAS_bswap16_i64;
1348 case INDEX_op_bswap32_i64:
1349 return TCG_TARGET_HAS_bswap32_i64;
1350 case INDEX_op_bswap64_i64:
1351 return TCG_TARGET_HAS_bswap64_i64;
1352 case INDEX_op_not_i64:
1353 return TCG_TARGET_HAS_not_i64;
1354 case INDEX_op_neg_i64:
1355 return TCG_TARGET_HAS_neg_i64;
1356 case INDEX_op_andc_i64:
1357 return TCG_TARGET_HAS_andc_i64;
1358 case INDEX_op_orc_i64:
1359 return TCG_TARGET_HAS_orc_i64;
1360 case INDEX_op_eqv_i64:
1361 return TCG_TARGET_HAS_eqv_i64;
1362 case INDEX_op_nand_i64:
1363 return TCG_TARGET_HAS_nand_i64;
1364 case INDEX_op_nor_i64:
1365 return TCG_TARGET_HAS_nor_i64;
1366 case INDEX_op_clz_i64:
1367 return TCG_TARGET_HAS_clz_i64;
1368 case INDEX_op_ctz_i64:
1369 return TCG_TARGET_HAS_ctz_i64;
1370 case INDEX_op_ctpop_i64:
1371 return TCG_TARGET_HAS_ctpop_i64;
1372 case INDEX_op_add2_i64:
1373 return TCG_TARGET_HAS_add2_i64;
1374 case INDEX_op_sub2_i64:
1375 return TCG_TARGET_HAS_sub2_i64;
1376 case INDEX_op_mulu2_i64:
1377 return TCG_TARGET_HAS_mulu2_i64;
1378 case INDEX_op_muls2_i64:
1379 return TCG_TARGET_HAS_muls2_i64;
1380 case INDEX_op_muluh_i64:
1381 return TCG_TARGET_HAS_muluh_i64;
1382 case INDEX_op_mulsh_i64:
1383 return TCG_TARGET_HAS_mulsh_i64;
1384
d2fd745f
RH		 1385 case INDEX_op_mov_vec:
1386 case INDEX_op_dup_vec:
37ee55a0 1387 case INDEX_op_dupm_vec:
d2fd745f
RH		 1388 case INDEX_op_ld_vec:
1389 case INDEX_op_st_vec:
1390 case INDEX_op_add_vec:
1391 case INDEX_op_sub_vec:
1392 case INDEX_op_and_vec:
1393 case INDEX_op_or_vec:
1394 case INDEX_op_xor_vec:
212be173 1395 case INDEX_op_cmp_vec:
d2fd745f
RH		 1396 return have_vec;
1397 case INDEX_op_dup2_vec:
1398 return have_vec && TCG_TARGET_REG_BITS == 32;
1399 case INDEX_op_not_vec:
1400 return have_vec && TCG_TARGET_HAS_not_vec;
1401 case INDEX_op_neg_vec:
1402 return have_vec && TCG_TARGET_HAS_neg_vec;
bcefc902
RH		 1403 case INDEX_op_abs_vec:
1404 return have_vec && TCG_TARGET_HAS_abs_vec;
d2fd745f
RH		 1405 case INDEX_op_andc_vec:
1406 return have_vec && TCG_TARGET_HAS_andc_vec;
1407 case INDEX_op_orc_vec:
1408 return have_vec && TCG_TARGET_HAS_orc_vec;
3774030a
RH		 1409 case INDEX_op_mul_vec:
1410 return have_vec && TCG_TARGET_HAS_mul_vec;
d0ec9796
RH		 1411 case INDEX_op_shli_vec:
1412 case INDEX_op_shri_vec:
1413 case INDEX_op_sari_vec:
1414 return have_vec && TCG_TARGET_HAS_shi_vec;
1415 case INDEX_op_shls_vec:
1416 case INDEX_op_shrs_vec:
1417 case INDEX_op_sars_vec:
1418 return have_vec && TCG_TARGET_HAS_shs_vec;
1419 case INDEX_op_shlv_vec:
1420 case INDEX_op_shrv_vec:
1421 case INDEX_op_sarv_vec:
1422 return have_vec && TCG_TARGET_HAS_shv_vec;
b0f7e744
RH		 1423 case INDEX_op_rotli_vec:
1424 return have_vec && TCG_TARGET_HAS_roti_vec;
23850a74
RH		 1425 case INDEX_op_rotls_vec:
1426 return have_vec && TCG_TARGET_HAS_rots_vec;
5d0ceda9
RH		 1427 case INDEX_op_rotlv_vec:
1428 case INDEX_op_rotrv_vec:
1429 return have_vec && TCG_TARGET_HAS_rotv_vec;
8afaf050
RH		 1430 case INDEX_op_ssadd_vec:
1431 case INDEX_op_usadd_vec:
1432 case INDEX_op_sssub_vec:
1433 case INDEX_op_ussub_vec:
1434 return have_vec && TCG_TARGET_HAS_sat_vec;
dd0a0fcd
RH		 1435 case INDEX_op_smin_vec:
1436 case INDEX_op_umin_vec:
1437 case INDEX_op_smax_vec:
1438 case INDEX_op_umax_vec:
1439 return have_vec && TCG_TARGET_HAS_minmax_vec;
38dc1294
RH		 1440 case INDEX_op_bitsel_vec:
1441 return have_vec && TCG_TARGET_HAS_bitsel_vec;
f75da298
RH		 1442 case INDEX_op_cmpsel_vec:
1443 return have_vec && TCG_TARGET_HAS_cmpsel_vec;
d2fd745f 1444
db432672
RH		 1445 default:
1446 tcg_debug_assert(op > INDEX_op_last_generic && op < NB_OPS);
1447 return true;
be0f34b5 1448 }
be0f34b5
RH		 1449}
1450
39cf05d3
FB		 1451/* Note: we convert the 64 bit args to 32 bit and do some alignment
1452 and endian swap. Maybe it would be better to do the alignment
1453 and endian swap in tcg_reg_alloc_call(). */
ae8b75dc 1454void tcg_gen_callN(void *func, TCGTemp *ret, int nargs, TCGTemp **args)
c896fe29 1455{
75e8b9b7 1456 int i, real_args, nb_rets, pi;
3e92aa34
RH		 1457 unsigned typemask;
1458 const TCGHelperInfo *info;
75e8b9b7 1459 TCGOp *op;
afb49896 1460
619205fd 1461 info = g_hash_table_lookup(helper_table, (gpointer)func);
7319d83a 1462 typemask = info->typemask;
2bece2c8 1463
38b47b19
EC		 1464#ifdef CONFIG_PLUGIN
1465 /* detect non-plugin helpers */
1466 if (tcg_ctx->plugin_insn && unlikely(strncmp(info->name, "plugin_", 7))) {
1467 tcg_ctx->plugin_insn->calls_helpers = true;
1468 }
1469#endif
1470
34b1a49c
RH		 1471#if defined(__sparc__) && !defined(__arch64__) \
1472 && !defined(CONFIG_TCG_INTERPRETER)
1473 /* We have 64-bit values in one register, but need to pass as two
1474 separate parameters. Split them. */
7319d83a 1475 int orig_typemask = typemask;
34b1a49c
RH		 1476 int orig_nargs = nargs;
1477 TCGv_i64 retl, reth;
ae8b75dc 1478 TCGTemp *split_args[MAX_OPC_PARAM];
34b1a49c 1479
f764718d
RH		 1480 retl = NULL;
1481 reth = NULL;
7319d83a
RH		 1482 typemask = 0;
1483 for (i = real_args = 0; i < nargs; ++i) {
1484 int argtype = extract32(orig_typemask, (i + 1) * 3, 3);
1485 bool is_64bit = (argtype & ~1) == dh_typecode_i64;
1486
1487 if (is_64bit) {
1488 TCGv_i64 orig = temp_tcgv_i64(args[i]);
1489 TCGv_i32 h = tcg_temp_new_i32();
1490 TCGv_i32 l = tcg_temp_new_i32();
1491 tcg_gen_extr_i64_i32(l, h, orig);
1492 split_args[real_args++] = tcgv_i32_temp(h);
1493 typemask |= dh_typecode_i32 << (real_args * 3);
1494 split_args[real_args++] = tcgv_i32_temp(l);
1495 typemask |= dh_typecode_i32 << (real_args * 3);
1496 } else {
1497 split_args[real_args++] = args[i];
1498 typemask |= argtype << (real_args * 3);
34b1a49c 1499 }
34b1a49c 1500 }
7319d83a
RH		 1501 nargs = real_args;
1502 args = split_args;
34b1a49c 1503#elif defined(TCG_TARGET_EXTEND_ARGS) && TCG_TARGET_REG_BITS == 64
2bece2c8 1504 for (i = 0; i < nargs; ++i) {
7319d83a
RH		 1505 int argtype = extract32(typemask, (i + 1) * 3, 3);
1506 bool is_32bit = (argtype & ~1) == dh_typecode_i32;
1507 bool is_signed = argtype & 1;
1508
1509 if (is_32bit) {
2bece2c8 1510 TCGv_i64 temp = tcg_temp_new_i64();
18cf3d07 1511 TCGv_i32 orig = temp_tcgv_i32(args[i]);
2bece2c8 1512 if (is_signed) {
18cf3d07 1513 tcg_gen_ext_i32_i64(temp, orig);
2bece2c8 1514 } else {
18cf3d07 1515 tcg_gen_extu_i32_i64(temp, orig);
2bece2c8 1516 }
ae8b75dc 1517 args[i] = tcgv_i64_temp(temp);
2bece2c8
RH		 1518 }
1519 }
1520#endif /* TCG_TARGET_EXTEND_ARGS */
1521
15fa08f8 1522 op = tcg_emit_op(INDEX_op_call);
75e8b9b7
RH		 1523
1524 pi = 0;
ae8b75dc 1525 if (ret != NULL) {
34b1a49c
RH		 1526#if defined(__sparc__) && !defined(__arch64__) \
1527 && !defined(CONFIG_TCG_INTERPRETER)
7319d83a 1528 if ((typemask & 6) == dh_typecode_i64) {
34b1a49c
RH		 1529 /* The 32-bit ABI is going to return the 64-bit value in
1530 the %o0/%o1 register pair. Prepare for this by using
1531 two return temporaries, and reassemble below. */
1532 retl = tcg_temp_new_i64();
1533 reth = tcg_temp_new_i64();
ae8b75dc
RH		 1534 op->args[pi++] = tcgv_i64_arg(reth);
1535 op->args[pi++] = tcgv_i64_arg(retl);
34b1a49c
RH		 1536 nb_rets = 2;
1537 } else {
ae8b75dc 1538 op->args[pi++] = temp_arg(ret);
34b1a49c
RH		 1539 nb_rets = 1;
1540 }
1541#else
7319d83a 1542 if (TCG_TARGET_REG_BITS < 64 && (typemask & 6) == dh_typecode_i64) {
02eb19d0 1543#ifdef HOST_WORDS_BIGENDIAN
ae8b75dc
RH		 1544 op->args[pi++] = temp_arg(ret + 1);
1545 op->args[pi++] = temp_arg(ret);
39cf05d3 1546#else
ae8b75dc
RH		 1547 op->args[pi++] = temp_arg(ret);
1548 op->args[pi++] = temp_arg(ret + 1);
39cf05d3 1549#endif
a7812ae4 1550 nb_rets = 2;
34b1a49c 1551 } else {
ae8b75dc 1552 op->args[pi++] = temp_arg(ret);
a7812ae4 1553 nb_rets = 1;
c896fe29 1554 }
34b1a49c 1555#endif
a7812ae4
PB		 1556 } else {
1557 nb_rets = 0;
c896fe29 1558 }
cd9090aa 1559 TCGOP_CALLO(op) = nb_rets;
75e8b9b7 1560
a7812ae4
PB		 1561 real_args = 0;
1562 for (i = 0; i < nargs; i++) {
7319d83a
RH		 1563 int argtype = extract32(typemask, (i + 1) * 3, 3);
1564 bool is_64bit = (argtype & ~1) == dh_typecode_i64;
7b7d8b2d 1565 bool want_align = false;
7319d83a 1566
7b7d8b2d
RH		 1567#if defined(CONFIG_TCG_INTERPRETER)
1568 /*
1569 * Align all arguments, so that they land in predictable places
1570 * for passing off to ffi_call.
1571 */
1572 want_align = true;
1573#elif defined(TCG_TARGET_CALL_ALIGN_ARGS)
1574 /* Some targets want aligned 64 bit args */
1575 want_align = is_64bit;
39cf05d3 1576#endif
7b7d8b2d
RH		 1577
1578 if (TCG_TARGET_REG_BITS < 64 && want_align && (real_args & 1)) {
1579 op->args[pi++] = TCG_CALL_DUMMY_ARG;
1580 real_args++;
1581 }
1582
1583 if (TCG_TARGET_REG_BITS < 64 && is_64bit) {
1584 /*
1585 * If stack grows up, then we will be placing successive
1586 * arguments at lower addresses, which means we need to
1587 * reverse the order compared to how we would normally
1588 * treat either big or little-endian. For those arguments
1589 * that will wind up in registers, this still works for
1590 * HPPA (the only current STACK_GROWSUP target) since the
1591 * argument registers are *also* allocated in decreasing
1592 * order. If another such target is added, this logic may
1593 * have to get more complicated to differentiate between
1594 * stack arguments and register arguments.
1595 */
02eb19d0 1596#if defined(HOST_WORDS_BIGENDIAN) != defined(TCG_TARGET_STACK_GROWSUP)
ae8b75dc
RH		 1597 op->args[pi++] = temp_arg(args[i] + 1);
1598 op->args[pi++] = temp_arg(args[i]);
c896fe29 1599#else
ae8b75dc
RH		 1600 op->args[pi++] = temp_arg(args[i]);
1601 op->args[pi++] = temp_arg(args[i] + 1);
c896fe29 1602#endif
a7812ae4 1603 real_args += 2;
2bece2c8 1604 continue;
c896fe29 1605 }
2bece2c8 1606
ae8b75dc 1607 op->args[pi++] = temp_arg(args[i]);
2bece2c8 1608 real_args++;
c896fe29 1609 }
75e8b9b7 1610 op->args[pi++] = (uintptr_t)func;
3e92aa34 1611 op->args[pi++] = (uintptr_t)info;
cd9090aa 1612 TCGOP_CALLI(op) = real_args;
a7812ae4 1613
75e8b9b7 1614 /* Make sure the fields didn't overflow. */
cd9090aa 1615 tcg_debug_assert(TCGOP_CALLI(op) == real_args);
75e8b9b7 1616 tcg_debug_assert(pi <= ARRAY_SIZE(op->args));
2bece2c8 1617
34b1a49c
RH		 1618#if defined(__sparc__) && !defined(__arch64__) \
1619 && !defined(CONFIG_TCG_INTERPRETER)
1620 /* Free all of the parts we allocated above. */
1621 for (i = real_args = 0; i < orig_nargs; ++i) {
7319d83a
RH		 1622 int argtype = extract32(orig_typemask, (i + 1) * 3, 3);
1623 bool is_64bit = (argtype & ~1) == dh_typecode_i64;
1624
34b1a49c 1625 if (is_64bit) {
085272b3
RH		 1626 tcg_temp_free_internal(args[real_args++]);
1627 tcg_temp_free_internal(args[real_args++]);
34b1a49c
RH		 1628 } else {
1629 real_args++;
1630 }
1631 }
7319d83a 1632 if ((orig_typemask & 6) == dh_typecode_i64) {
34b1a49c
RH		 1633 /* The 32-bit ABI returned two 32-bit pieces. Re-assemble them.
1634 Note that describing these as TCGv_i64 eliminates an unnecessary
1635 zero-extension that tcg_gen_concat_i32_i64 would create. */
085272b3 1636 tcg_gen_concat32_i64(temp_tcgv_i64(ret), retl, reth);
34b1a49c
RH		 1637 tcg_temp_free_i64(retl);
1638 tcg_temp_free_i64(reth);
1639 }
1640#elif defined(TCG_TARGET_EXTEND_ARGS) && TCG_TARGET_REG_BITS == 64
2bece2c8 1641 for (i = 0; i < nargs; ++i) {
7319d83a
RH		 1642 int argtype = extract32(typemask, (i + 1) * 3, 3);
1643 bool is_32bit = (argtype & ~1) == dh_typecode_i32;
1644
1645 if (is_32bit) {
085272b3 1646 tcg_temp_free_internal(args[i]);
2bece2c8
RH		 1647 }
1648 }
1649#endif /* TCG_TARGET_EXTEND_ARGS */
c896fe29 1650}
c896fe29 1651
8fcd3692 1652static void tcg_reg_alloc_start(TCGContext *s)
c896fe29 1653{
ac3b8891 1654 int i, n;
ac3b8891 1655
ee17db83
RH		 1656 for (i = 0, n = s->nb_temps; i < n; i++) {
1657 TCGTemp *ts = &s->temps[i];
1658 TCGTempVal val = TEMP_VAL_MEM;
1659
1660 switch (ts->kind) {
c0522136
RH		 1661 case TEMP_CONST:
1662 val = TEMP_VAL_CONST;
1663 break;
ee17db83
RH		 1664 case TEMP_FIXED:
1665 val = TEMP_VAL_REG;
1666 break;
1667 case TEMP_GLOBAL:
1668 break;
1669 case TEMP_NORMAL:
1670 val = TEMP_VAL_DEAD;
1671 /* fall through */
1672 case TEMP_LOCAL:
1673 ts->mem_allocated = 0;
1674 break;
1675 default:
1676 g_assert_not_reached();
1677 }
1678 ts->val_type = val;
e8996ee0 1679 }
f8b2f202
RH		 1680
1681 memset(s->reg_to_temp, 0, sizeof(s->reg_to_temp));
c896fe29
FB		 1682}
1683
f8b2f202
RH		 1684static char *tcg_get_arg_str_ptr(TCGContext *s, char *buf, int buf_size,
1685 TCGTemp *ts)
c896fe29 1686{
1807f4c4 1687 int idx = temp_idx(ts);
ac56dd48 1688
ee17db83
RH		 1689 switch (ts->kind) {
1690 case TEMP_FIXED:
1691 case TEMP_GLOBAL:
ac56dd48 1692 pstrcpy(buf, buf_size, ts->name);
ee17db83
RH		 1693 break;
1694 case TEMP_LOCAL:
f8b2f202 1695 snprintf(buf, buf_size, "loc%d", idx - s->nb_globals);
ee17db83
RH		 1696 break;
1697 case TEMP_NORMAL:
f8b2f202 1698 snprintf(buf, buf_size, "tmp%d", idx - s->nb_globals);
ee17db83 1699 break;
c0522136
RH		 1700 case TEMP_CONST:
1701 switch (ts->type) {
1702 case TCG_TYPE_I32:
1703 snprintf(buf, buf_size, "$0x%x", (int32_t)ts->val);
1704 break;
1705#if TCG_TARGET_REG_BITS > 32
1706 case TCG_TYPE_I64:
1707 snprintf(buf, buf_size, "$0x%" PRIx64, ts->val);
1708 break;
1709#endif
1710 case TCG_TYPE_V64:
1711 case TCG_TYPE_V128:
1712 case TCG_TYPE_V256:
1713 snprintf(buf, buf_size, "v%d$0x%" PRIx64,
1714 64 << (ts->type - TCG_TYPE_V64), ts->val);
1715 break;
1716 default:
1717 g_assert_not_reached();
1718 }
1719 break;
c896fe29
FB		 1720 }
1721 return buf;
1722}
1723
43439139
RH		 1724static char *tcg_get_arg_str(TCGContext *s, char *buf,
1725 int buf_size, TCGArg arg)
f8b2f202 1726{
43439139 1727 return tcg_get_arg_str_ptr(s, buf, buf_size, arg_temp(arg));
f8b2f202
RH		 1728}
1729
f48f3ede
BS		 1730static const char * const cond_name[] =
1731{
0aed257f
RH		 1732 [TCG_COND_NEVER] = "never",
1733 [TCG_COND_ALWAYS] = "always",
f48f3ede
BS		 1734 [TCG_COND_EQ] = "eq",
1735 [TCG_COND_NE] = "ne",
1736 [TCG_COND_LT] = "lt",
1737 [TCG_COND_GE] = "ge",
1738 [TCG_COND_LE] = "le",
1739 [TCG_COND_GT] = "gt",
1740 [TCG_COND_LTU] = "ltu",
1741 [TCG_COND_GEU] = "geu",
1742 [TCG_COND_LEU] = "leu",
1743 [TCG_COND_GTU] = "gtu"
1744};
1745
f713d6ad
RH		 1746static const char * const ldst_name[] =
1747{
1748 [MO_UB] = "ub",
1749 [MO_SB] = "sb",
1750 [MO_LEUW] = "leuw",
1751 [MO_LESW] = "lesw",
1752 [MO_LEUL] = "leul",
1753 [MO_LESL] = "lesl",
fc313c64 1754 [MO_LEUQ] = "leq",
f713d6ad
RH		 1755 [MO_BEUW] = "beuw",
1756 [MO_BESW] = "besw",
1757 [MO_BEUL] = "beul",
1758 [MO_BESL] = "besl",
fc313c64 1759 [MO_BEUQ] = "beq",
f713d6ad
RH		 1760};
1761
1f00b27f 1762static const char * const alignment_name[(MO_AMASK >> MO_ASHIFT) + 1] = {
52bf9771 1763#ifdef TARGET_ALIGNED_ONLY
1f00b27f
SS		 1764 [MO_UNALN >> MO_ASHIFT] = "un+",
1765 [MO_ALIGN >> MO_ASHIFT] = "",
1766#else
1767 [MO_UNALN >> MO_ASHIFT] = "",
1768 [MO_ALIGN >> MO_ASHIFT] = "al+",
1769#endif
1770 [MO_ALIGN_2 >> MO_ASHIFT] = "al2+",
1771 [MO_ALIGN_4 >> MO_ASHIFT] = "al4+",
1772 [MO_ALIGN_8 >> MO_ASHIFT] = "al8+",
1773 [MO_ALIGN_16 >> MO_ASHIFT] = "al16+",
1774 [MO_ALIGN_32 >> MO_ASHIFT] = "al32+",
1775 [MO_ALIGN_64 >> MO_ASHIFT] = "al64+",
1776};
1777
587195bd
RH		 1778static const char bswap_flag_name[][6] = {
1779 [TCG_BSWAP_IZ] = "iz",
1780 [TCG_BSWAP_OZ] = "oz",
1781 [TCG_BSWAP_OS] = "os",
1782 [TCG_BSWAP_IZ | TCG_BSWAP_OZ] = "iz,oz",
1783 [TCG_BSWAP_IZ | TCG_BSWAP_OS] = "iz,os",
1784};
1785
b016486e
RH		 1786static inline bool tcg_regset_single(TCGRegSet d)
1787{
1788 return (d & (d - 1)) == 0;
1789}
1790
1791static inline TCGReg tcg_regset_first(TCGRegSet d)
1792{
1793 if (TCG_TARGET_NB_REGS <= 32) {
1794 return ctz32(d);
1795 } else {
1796 return ctz64(d);
1797 }
1798}
1799
1894f69a 1800static void tcg_dump_ops(TCGContext *s, bool have_prefs)
c896fe29 1801{
c896fe29 1802 char buf[128];
c45cb8bb 1803 TCGOp *op;
c45cb8bb 1804
15fa08f8 1805 QTAILQ_FOREACH(op, &s->ops, link) {
c45cb8bb
RH		 1806 int i, k, nb_oargs, nb_iargs, nb_cargs;
1807 const TCGOpDef *def;
c45cb8bb 1808 TCGOpcode c;
bdfb460e 1809 int col = 0;
c896fe29 1810
c45cb8bb 1811 c = op->opc;
c896fe29 1812 def = &tcg_op_defs[c];
c45cb8bb 1813
765b842a 1814 if (c == INDEX_op_insn_start) {
b016486e 1815 nb_oargs = 0;
15fa08f8 1816 col += qemu_log("\n ----");
9aef40ed
RH		 1817
1818 for (i = 0; i < TARGET_INSN_START_WORDS; ++i) {
1819 target_ulong a;
7e4597d7 1820#if TARGET_LONG_BITS > TCG_TARGET_REG_BITS
efee3746 1821 a = deposit64(op->args[i * 2], 32, 32, op->args[i * 2 + 1]);
7e4597d7 1822#else
efee3746 1823 a = op->args[i];
7e4597d7 1824#endif
bdfb460e 1825 col += qemu_log(" " TARGET_FMT_lx, a);
eeacee4d 1826 }
7e4597d7 1827 } else if (c == INDEX_op_call) {
3e92aa34 1828 const TCGHelperInfo *info = tcg_call_info(op);
fa52e660 1829 void *func = tcg_call_func(op);
3e92aa34 1830
c896fe29 1831 /* variable number of arguments */
cd9090aa
RH		 1832 nb_oargs = TCGOP_CALLO(op);
1833 nb_iargs = TCGOP_CALLI(op);
c896fe29 1834 nb_cargs = def->nb_cargs;
c896fe29 1835
3e92aa34
RH		 1836 col += qemu_log(" %s ", def->name);
1837
1838 /*
1839 * Print the function name from TCGHelperInfo, if available.
1840 * Note that plugins have a template function for the info,
1841 * but the actual function pointer comes from the plugin.
1842 */
3e92aa34
RH		 1843 if (func == info->func) {
1844 col += qemu_log("%s", info->name);
1845 } else {
1846 col += qemu_log("plugin(%p)", func);
1847 }
1848
8973fe43 1849 col += qemu_log(",$0x%x,$%d", info->flags, nb_oargs);
cf066674 1850 for (i = 0; i < nb_oargs; i++) {
43439139
RH		 1851 col += qemu_log(",%s", tcg_get_arg_str(s, buf, sizeof(buf),
1852 op->args[i]));
b03cce8e 1853 }
cf066674 1854 for (i = 0; i < nb_iargs; i++) {
efee3746 1855 TCGArg arg = op->args[nb_oargs + i];
cf066674
RH		 1856 const char *t = "<dummy>";
1857 if (arg != TCG_CALL_DUMMY_ARG) {
43439139 1858 t = tcg_get_arg_str(s, buf, sizeof(buf), arg);
eeacee4d 1859 }
bdfb460e 1860 col += qemu_log(",%s", t);
e8996ee0 1861 }
b03cce8e 1862 } else {
bdfb460e 1863 col += qemu_log(" %s ", def->name);
c45cb8bb
RH		 1864
1865 nb_oargs = def->nb_oargs;
1866 nb_iargs = def->nb_iargs;
1867 nb_cargs = def->nb_cargs;
1868
d2fd745f
RH		 1869 if (def->flags & TCG_OPF_VECTOR) {
1870 col += qemu_log("v%d,e%d,", 64 << TCGOP_VECL(op),
1871 8 << TCGOP_VECE(op));
1872 }
1873
b03cce8e 1874 k = 0;
c45cb8bb 1875 for (i = 0; i < nb_oargs; i++) {
eeacee4d 1876 if (k != 0) {
bdfb460e 1877 col += qemu_log(",");
eeacee4d 1878 }
43439139
RH		 1879 col += qemu_log("%s", tcg_get_arg_str(s, buf, sizeof(buf),
1880 op->args[k++]));
b03cce8e 1881 }
c45cb8bb 1882 for (i = 0; i < nb_iargs; i++) {
eeacee4d 1883 if (k != 0) {
bdfb460e 1884 col += qemu_log(",");
eeacee4d 1885 }
43439139
RH		 1886 col += qemu_log("%s", tcg_get_arg_str(s, buf, sizeof(buf),
1887 op->args[k++]));
b03cce8e 1888 }
be210acb
RH		 1889 switch (c) {
1890 case INDEX_op_brcond_i32:
be210acb 1891 case INDEX_op_setcond_i32:
ffc5ea09 1892 case INDEX_op_movcond_i32:
ffc5ea09 1893 case INDEX_op_brcond2_i32:
be210acb 1894 case INDEX_op_setcond2_i32:
ffc5ea09 1895 case INDEX_op_brcond_i64:
be210acb 1896 case INDEX_op_setcond_i64:
ffc5ea09 1897 case INDEX_op_movcond_i64:
212be173 1898 case INDEX_op_cmp_vec:
f75da298 1899 case INDEX_op_cmpsel_vec:
efee3746
RH		 1900 if (op->args[k] < ARRAY_SIZE(cond_name)
1901 && cond_name[op->args[k]]) {
1902 col += qemu_log(",%s", cond_name[op->args[k++]]);
eeacee4d 1903 } else {
efee3746 1904 col += qemu_log(",$0x%" TCG_PRIlx, op->args[k++]);
eeacee4d 1905 }
f48f3ede 1906 i = 1;
be210acb 1907 break;
f713d6ad
RH		 1908 case INDEX_op_qemu_ld_i32:
1909 case INDEX_op_qemu_st_i32:
07ce0b05 1910 case INDEX_op_qemu_st8_i32:
f713d6ad
RH		 1911 case INDEX_op_qemu_ld_i64:
1912 case INDEX_op_qemu_st_i64:
59227d5d 1913 {
9002ffcb 1914 MemOpIdx oi = op->args[k++];
14776ab5 1915 MemOp op = get_memop(oi);
59227d5d
RH		 1916 unsigned ix = get_mmuidx(oi);
1917
59c4b7e8 1918 if (op & ~(MO_AMASK | MO_BSWAP | MO_SSIZE)) {
bdfb460e 1919 col += qemu_log(",$0x%x,%u", op, ix);
59c4b7e8 1920 } else {
1f00b27f
SS		 1921 const char *s_al, *s_op;
1922 s_al = alignment_name[(op & MO_AMASK) >> MO_ASHIFT];
59c4b7e8 1923 s_op = ldst_name[op & (MO_BSWAP | MO_SSIZE)];
bdfb460e 1924 col += qemu_log(",%s%s,%u", s_al, s_op, ix);
59227d5d
RH		 1925 }
1926 i = 1;
f713d6ad 1927 }
f713d6ad 1928 break;
587195bd
RH		 1929 case INDEX_op_bswap16_i32:
1930 case INDEX_op_bswap16_i64:
1931 case INDEX_op_bswap32_i32:
1932 case INDEX_op_bswap32_i64:
1933 case INDEX_op_bswap64_i64:
1934 {
1935 TCGArg flags = op->args[k];
1936 const char *name = NULL;
1937
1938 if (flags < ARRAY_SIZE(bswap_flag_name)) {
1939 name = bswap_flag_name[flags];
1940 }
1941 if (name) {
1942 col += qemu_log(",%s", name);
1943 } else {
1944 col += qemu_log(",$0x%" TCG_PRIlx, flags);
1945 }
1946 i = k = 1;
1947 }
1948 break;
be210acb 1949 default:
f48f3ede 1950 i = 0;
be210acb
RH		 1951 break;
1952 }
51e3972c
RH		 1953 switch (c) {
1954 case INDEX_op_set_label:
1955 case INDEX_op_br:
1956 case INDEX_op_brcond_i32:
1957 case INDEX_op_brcond_i64:
1958 case INDEX_op_brcond2_i32:
efee3746
RH		 1959 col += qemu_log("%s$L%d", k ? "," : "",
1960 arg_label(op->args[k])->id);
51e3972c
RH		 1961 i++, k++;
1962 break;
1963 default:
1964 break;
1965 }
1966 for (; i < nb_cargs; i++, k++) {
efee3746 1967 col += qemu_log("%s$0x%" TCG_PRIlx, k ? "," : "", op->args[k]);
bdfb460e
RH		 1968 }
1969 }
bdfb460e 1970
1894f69a 1971 if (have_prefs || op->life) {
7606488c
RF		 1972
1973 QemuLogFile *logfile;
1974
1975 rcu_read_lock();
d73415a3 1976 logfile = qatomic_rcu_read(&qemu_logfile);
7606488c
RF		 1977 if (logfile) {
1978 for (; col < 40; ++col) {
1979 putc(' ', logfile->fd);
1980 }
bdfb460e 1981 }
7606488c 1982 rcu_read_unlock();
1894f69a
RH		 1983 }
1984
1985 if (op->life) {
1986 unsigned life = op->life;
bdfb460e
RH		 1987
1988 if (life & (SYNC_ARG * 3)) {
1989 qemu_log(" sync:");
1990 for (i = 0; i < 2; ++i) {
1991 if (life & (SYNC_ARG << i)) {
1992 qemu_log(" %d", i);
1993 }
1994 }
1995 }
1996 life /= DEAD_ARG;
1997 if (life) {
1998 qemu_log(" dead:");
1999 for (i = 0; life; ++i, life >>= 1) {
2000 if (life & 1) {
2001 qemu_log(" %d", i);
2002 }
2003 }
b03cce8e 2004 }
c896fe29 2005 }
1894f69a
RH		 2006
2007 if (have_prefs) {
2008 for (i = 0; i < nb_oargs; ++i) {
2009 TCGRegSet set = op->output_pref[i];
2010
2011 if (i == 0) {
2012 qemu_log(" pref=");
2013 } else {
2014 qemu_log(",");
2015 }
2016 if (set == 0) {
2017 qemu_log("none");
2018 } else if (set == MAKE_64BIT_MASK(0, TCG_TARGET_NB_REGS)) {
2019 qemu_log("all");
2020#ifdef CONFIG_DEBUG_TCG
2021 } else if (tcg_regset_single(set)) {
2022 TCGReg reg = tcg_regset_first(set);
2023 qemu_log("%s", tcg_target_reg_names[reg]);
2024#endif
2025 } else if (TCG_TARGET_NB_REGS <= 32) {
2026 qemu_log("%#x", (uint32_t)set);
2027 } else {
2028 qemu_log("%#" PRIx64, (uint64_t)set);
2029 }
2030 }
2031 }
2032
eeacee4d 2033 qemu_log("\n");
c896fe29
FB		 2034 }
2035}
2036
2037/* we give more priority to constraints with less registers */
2038static int get_constraint_priority(const TCGOpDef *def, int k)
2039{
74a11790
RH		 2040 const TCGArgConstraint *arg_ct = &def->args_ct[k];
2041 int n;
c896fe29 2042
bc2b17e6 2043 if (arg_ct->oalias) {
c896fe29
FB		 2044 /* an alias is equivalent to a single register */
2045 n = 1;
2046 } else {
74a11790 2047 n = ctpop64(arg_ct->regs);
c896fe29
FB		 2048 }
2049 return TCG_TARGET_NB_REGS - n + 1;
2050}
2051
2052/* sort from highest priority to lowest */
2053static void sort_constraints(TCGOpDef *def, int start, int n)
2054{
66792f90
RH		 2055 int i, j;
2056 TCGArgConstraint *a = def->args_ct;
c896fe29 2057
66792f90
RH		 2058 for (i = 0; i < n; i++) {
2059 a[start + i].sort_index = start + i;
2060 }
2061 if (n <= 1) {
c896fe29 2062 return;
66792f90
RH		 2063 }
2064 for (i = 0; i < n - 1; i++) {
2065 for (j = i + 1; j < n; j++) {
2066 int p1 = get_constraint_priority(def, a[start + i].sort_index);
2067 int p2 = get_constraint_priority(def, a[start + j].sort_index);
c896fe29 2068 if (p1 < p2) {
66792f90
RH		 2069 int tmp = a[start + i].sort_index;
2070 a[start + i].sort_index = a[start + j].sort_index;
2071 a[start + j].sort_index = tmp;
c896fe29
FB		 2072 }
2073 }
2074 }
2075}
2076
f69d277e 2077static void process_op_defs(TCGContext *s)
c896fe29 2078{
a9751609 2079 TCGOpcode op;
c896fe29 2080
f69d277e
RH		 2081 for (op = 0; op < NB_OPS; op++) {
2082 TCGOpDef *def = &tcg_op_defs[op];
2083 const TCGTargetOpDef *tdefs;
069ea736 2084 int i, nb_args;
f69d277e
RH		 2085
2086 if (def->flags & TCG_OPF_NOT_PRESENT) {
2087 continue;
2088 }
2089
c896fe29 2090 nb_args = def->nb_iargs + def->nb_oargs;
f69d277e
RH		 2091 if (nb_args == 0) {
2092 continue;
2093 }
2094
4c22e840
RH		 2095 /*
2096 * Macro magic should make it impossible, but double-check that
2097 * the array index is in range. Since the signness of an enum
2098 * is implementation defined, force the result to unsigned.
2099 */
2100 unsigned con_set = tcg_target_op_def(op);
2101 tcg_debug_assert(con_set < ARRAY_SIZE(constraint_sets));
2102 tdefs = &constraint_sets[con_set];
f69d277e
RH		 2103
2104 for (i = 0; i < nb_args; i++) {
2105 const char *ct_str = tdefs->args_ct_str[i];
2106 /* Incomplete TCGTargetOpDef entry. */
eabb7b91 2107 tcg_debug_assert(ct_str != NULL);
f69d277e 2108
17280ff4
RH		 2109 while (*ct_str != '\0') {
2110 switch(*ct_str) {
2111 case '0' ... '9':
2112 {
2113 int oarg = *ct_str - '0';
2114 tcg_debug_assert(ct_str == tdefs->args_ct_str[i]);
2115 tcg_debug_assert(oarg < def->nb_oargs);
74a11790 2116 tcg_debug_assert(def->args_ct[oarg].regs != 0);
17280ff4 2117 def->args_ct[i] = def->args_ct[oarg];
bc2b17e6
RH		 2118 /* The output sets oalias. */
2119 def->args_ct[oarg].oalias = true;
17280ff4 2120 def->args_ct[oarg].alias_index = i;
bc2b17e6
RH		 2121 /* The input sets ialias. */
2122 def->args_ct[i].ialias = true;
17280ff4 2123 def->args_ct[i].alias_index = oarg;
c896fe29 2124 }
17280ff4
RH		 2125 ct_str++;
2126 break;
2127 case '&':
bc2b17e6 2128 def->args_ct[i].newreg = true;
17280ff4
RH		 2129 ct_str++;
2130 break;
2131 case 'i':
2132 def->args_ct[i].ct |= TCG_CT_CONST;
2133 ct_str++;
2134 break;
358b4923 2135
358b4923
RH		 2136 /* Include all of the target-specific constraints. */
2137
2138#undef CONST
2139#define CONST(CASE, MASK) \
2140 case CASE: def->args_ct[i].ct |= MASK; ct_str++; break;
2141#define REGS(CASE, MASK) \
2142 case CASE: def->args_ct[i].regs |= MASK; ct_str++; break;
2143
2144#include "tcg-target-con-str.h"
2145
2146#undef REGS
2147#undef CONST
17280ff4 2148 default:
17280ff4 2149 /* Typo in TCGTargetOpDef constraint. */
358b4923 2150 g_assert_not_reached();
c896fe29
FB		 2151 }
2152 }
2153 }
2154
c68aaa18 2155 /* TCGTargetOpDef entry with too much information? */
eabb7b91 2156 tcg_debug_assert(i == TCG_MAX_OP_ARGS || tdefs->args_ct_str[i] == NULL);
c68aaa18 2157
c896fe29
FB		 2158 /* sort the constraints (XXX: this is just an heuristic) */
2159 sort_constraints(def, 0, def->nb_oargs);
2160 sort_constraints(def, def->nb_oargs, def->nb_iargs);
a9751609 2161 }
c896fe29
FB		 2162}
2163
0c627cdc
RH		 2164void tcg_op_remove(TCGContext *s, TCGOp *op)
2165{
d88a117e
RH		 2166 TCGLabel *label;
2167
2168 switch (op->opc) {
2169 case INDEX_op_br:
2170 label = arg_label(op->args[0]);
2171 label->refs--;
2172 break;
2173 case INDEX_op_brcond_i32:
2174 case INDEX_op_brcond_i64:
2175 label = arg_label(op->args[3]);
2176 label->refs--;
2177 break;
2178 case INDEX_op_brcond2_i32:
2179 label = arg_label(op->args[5]);
2180 label->refs--;
2181 break;
2182 default:
2183 break;
2184 }
2185
15fa08f8
RH		 2186 QTAILQ_REMOVE(&s->ops, op, link);
2187 QTAILQ_INSERT_TAIL(&s->free_ops, op, link);
abebf925 2188 s->nb_ops--;
0c627cdc
RH		 2189
2190#ifdef CONFIG_PROFILER
d73415a3 2191 qatomic_set(&s->prof.del_op_count, s->prof.del_op_count + 1);
0c627cdc
RH		 2192#endif
2193}
2194
a80cdd31
RH		 2195void tcg_remove_ops_after(TCGOp *op)
2196{
2197 TCGContext *s = tcg_ctx;
2198
2199 while (true) {
2200 TCGOp *last = tcg_last_op();
2201 if (last == op) {
2202 return;
2203 }
2204 tcg_op_remove(s, last);
2205 }
2206}
2207
15fa08f8 2208static TCGOp *tcg_op_alloc(TCGOpcode opc)
5a18407f 2209{
15fa08f8
RH		 2210 TCGContext *s = tcg_ctx;
2211 TCGOp *op;
5a18407f 2212
15fa08f8
RH		 2213 if (likely(QTAILQ_EMPTY(&s->free_ops))) {
2214 op = tcg_malloc(sizeof(TCGOp));
2215 } else {
2216 op = QTAILQ_FIRST(&s->free_ops);
2217 QTAILQ_REMOVE(&s->free_ops, op, link);
2218 }
2219 memset(op, 0, offsetof(TCGOp, link));
2220 op->opc = opc;
abebf925 2221 s->nb_ops++;
5a18407f 2222
15fa08f8
RH		 2223 return op;
2224}
2225
2226TCGOp *tcg_emit_op(TCGOpcode opc)
2227{
2228 TCGOp *op = tcg_op_alloc(opc);
2229 QTAILQ_INSERT_TAIL(&tcg_ctx->ops, op, link);
2230 return op;
2231}
5a18407f 2232
ac1043f6 2233TCGOp *tcg_op_insert_before(TCGContext *s, TCGOp *old_op, TCGOpcode opc)
15fa08f8
RH		 2234{
2235 TCGOp *new_op = tcg_op_alloc(opc);
2236 QTAILQ_INSERT_BEFORE(old_op, new_op, link);
5a18407f
RH		 2237 return new_op;
2238}
2239
ac1043f6 2240TCGOp *tcg_op_insert_after(TCGContext *s, TCGOp *old_op, TCGOpcode opc)
5a18407f 2241{
15fa08f8
RH		 2242 TCGOp *new_op = tcg_op_alloc(opc);
2243 QTAILQ_INSERT_AFTER(&s->ops, old_op, new_op, link);
5a18407f
RH		 2244 return new_op;
2245}
2246
b4fc67c7
RH		 2247/* Reachable analysis : remove unreachable code. */
2248static void reachable_code_pass(TCGContext *s)
2249{
2250 TCGOp *op, *op_next;
2251 bool dead = false;
2252
2253 QTAILQ_FOREACH_SAFE(op, &s->ops, link, op_next) {
2254 bool remove = dead;
2255 TCGLabel *label;
b4fc67c7
RH		 2256
2257 switch (op->opc) {
2258 case INDEX_op_set_label:
2259 label = arg_label(op->args[0]);
2260 if (label->refs == 0) {
2261 /*
2262 * While there is an occasional backward branch, virtually
2263 * all branches generated by the translators are forward.
2264 * Which means that generally we will have already removed
2265 * all references to the label that will be, and there is
2266 * little to be gained by iterating.
2267 */
2268 remove = true;
2269 } else {
2270 /* Once we see a label, insns become live again. */
2271 dead = false;
2272 remove = false;
2273
2274 /*
2275 * Optimization can fold conditional branches to unconditional.
2276 * If we find a label with one reference which is preceded by
2277 * an unconditional branch to it, remove both. This needed to
2278 * wait until the dead code in between them was removed.
2279 */
2280 if (label->refs == 1) {
eae3eb3e 2281 TCGOp *op_prev = QTAILQ_PREV(op, link);
b4fc67c7
RH		 2282 if (op_prev->opc == INDEX_op_br &&
2283 label == arg_label(op_prev->args[0])) {
2284 tcg_op_remove(s, op_prev);
2285 remove = true;
2286 }
2287 }
2288 }
2289 break;
2290
2291 case INDEX_op_br:
2292 case INDEX_op_exit_tb:
2293 case INDEX_op_goto_ptr:
2294 /* Unconditional branches; everything following is dead. */
2295 dead = true;
2296 break;
2297
2298 case INDEX_op_call:
2299 /* Notice noreturn helper calls, raising exceptions. */
90163900 2300 if (tcg_call_flags(op) & TCG_CALL_NO_RETURN) {
b4fc67c7
RH		 2301 dead = true;
2302 }
2303 break;
2304
2305 case INDEX_op_insn_start:
2306 /* Never remove -- we need to keep these for unwind. */
2307 remove = false;
2308 break;
2309
2310 default:
2311 break;
2312 }
2313
2314 if (remove) {
2315 tcg_op_remove(s, op);
2316 }
2317 }
2318}
2319
c70fbf0a
RH		 2320#define TS_DEAD 1
2321#define TS_MEM 2
2322
5a18407f
RH		 2323#define IS_DEAD_ARG(n) (arg_life & (DEAD_ARG << (n)))
2324#define NEED_SYNC_ARG(n) (arg_life & (SYNC_ARG << (n)))
2325
25f49c5f
RH		 2326/* For liveness_pass_1, the register preferences for a given temp. */
2327static inline TCGRegSet *la_temp_pref(TCGTemp *ts)
2328{
2329 return ts->state_ptr;
2330}
2331
2332/* For liveness_pass_1, reset the preferences for a given temp to the
2333 * maximal regset for its type.
2334 */
2335static inline void la_reset_pref(TCGTemp *ts)
2336{
2337 *la_temp_pref(ts)
2338 = (ts->state == TS_DEAD ? 0 : tcg_target_available_regs[ts->type]);
2339}
2340
9c43b68d
AJ		 2341/* liveness analysis: end of function: all temps are dead, and globals
2342 should be in memory. */
2616c808 2343static void la_func_end(TCGContext *s, int ng, int nt)
c896fe29 2344{
b83eabea
RH		 2345 int i;
2346
2347 for (i = 0; i < ng; ++i) {
2348 s->temps[i].state = TS_DEAD | TS_MEM;
25f49c5f 2349 la_reset_pref(&s->temps[i]);
b83eabea
RH		 2350 }
2351 for (i = ng; i < nt; ++i) {
2352 s->temps[i].state = TS_DEAD;
25f49c5f 2353 la_reset_pref(&s->temps[i]);
b83eabea 2354 }
c896fe29
FB		 2355}
2356
9c43b68d
AJ		 2357/* liveness analysis: end of basic block: all temps are dead, globals
2358 and local temps should be in memory. */
2616c808 2359static void la_bb_end(TCGContext *s, int ng, int nt)
641d5fbe 2360{
b83eabea 2361 int i;
641d5fbe 2362
ee17db83
RH		 2363 for (i = 0; i < nt; ++i) {
2364 TCGTemp *ts = &s->temps[i];
2365 int state;
2366
2367 switch (ts->kind) {
2368 case TEMP_FIXED:
2369 case TEMP_GLOBAL:
2370 case TEMP_LOCAL:
2371 state = TS_DEAD | TS_MEM;
2372 break;
2373 case TEMP_NORMAL:
c0522136 2374 case TEMP_CONST:
ee17db83
RH		 2375 state = TS_DEAD;
2376 break;
2377 default:
2378 g_assert_not_reached();
2379 }
2380 ts->state = state;
2381 la_reset_pref(ts);
641d5fbe
FB		 2382 }
2383}
2384
f65a061c
RH		 2385/* liveness analysis: sync globals back to memory. */
2386static void la_global_sync(TCGContext *s, int ng)
2387{
2388 int i;
2389
2390 for (i = 0; i < ng; ++i) {
25f49c5f
RH		 2391 int state = s->temps[i].state;
2392 s->temps[i].state = state | TS_MEM;
2393 if (state == TS_DEAD) {
2394 /* If the global was previously dead, reset prefs. */
2395 la_reset_pref(&s->temps[i]);
2396 }
f65a061c
RH		 2397 }
2398}
2399
b4cb76e6
RH		 2400/*
2401 * liveness analysis: conditional branch: all temps are dead,
2402 * globals and local temps should be synced.
2403 */
2404static void la_bb_sync(TCGContext *s, int ng, int nt)
2405{
2406 la_global_sync(s, ng);
2407
2408 for (int i = ng; i < nt; ++i) {
c0522136
RH		 2409 TCGTemp *ts = &s->temps[i];
2410 int state;
2411
2412 switch (ts->kind) {
2413 case TEMP_LOCAL:
2414 state = ts->state;
2415 ts->state = state | TS_MEM;
b4cb76e6
RH		 2416 if (state != TS_DEAD) {
2417 continue;
2418 }
c0522136
RH		 2419 break;
2420 case TEMP_NORMAL:
b4cb76e6 2421 s->temps[i].state = TS_DEAD;
c0522136
RH		 2422 break;
2423 case TEMP_CONST:
2424 continue;
2425 default:
2426 g_assert_not_reached();
b4cb76e6
RH		 2427 }
2428 la_reset_pref(&s->temps[i]);
2429 }
2430}
2431
f65a061c
RH		 2432/* liveness analysis: sync globals back to memory and kill. */
2433static void la_global_kill(TCGContext *s, int ng)
2434{
2435 int i;
2436
2437 for (i = 0; i < ng; i++) {
2438 s->temps[i].state = TS_DEAD | TS_MEM;
25f49c5f
RH		 2439 la_reset_pref(&s->temps[i]);
2440 }
2441}
2442
2443/* liveness analysis: note live globals crossing calls. */
2444static void la_cross_call(TCGContext *s, int nt)
2445{
2446 TCGRegSet mask = ~tcg_target_call_clobber_regs;
2447 int i;
2448
2449 for (i = 0; i < nt; i++) {
2450 TCGTemp *ts = &s->temps[i];
2451 if (!(ts->state & TS_DEAD)) {
2452 TCGRegSet *pset = la_temp_pref(ts);
2453 TCGRegSet set = *pset;
2454
2455 set &= mask;
2456 /* If the combination is not possible, restart. */
2457 if (set == 0) {
2458 set = tcg_target_available_regs[ts->type] & mask;
2459 }
2460 *pset = set;
2461 }
f65a061c
RH		 2462 }
2463}
2464
a1b3c48d 2465/* Liveness analysis : update the opc_arg_life array to tell if a
c896fe29
FB		 2466 given input arguments is dead. Instructions updating dead
2467 temporaries are removed. */
b83eabea 2468static void liveness_pass_1(TCGContext *s)
c896fe29 2469{
c70fbf0a 2470 int nb_globals = s->nb_globals;
2616c808 2471 int nb_temps = s->nb_temps;
15fa08f8 2472 TCGOp *op, *op_prev;
25f49c5f
RH		 2473 TCGRegSet *prefs;
2474 int i;
2475
2476 prefs = tcg_malloc(sizeof(TCGRegSet) * nb_temps);
2477 for (i = 0; i < nb_temps; ++i) {
2478 s->temps[i].state_ptr = prefs + i;
2479 }
a1b3c48d 2480
ae36a246 2481 /* ??? Should be redundant with the exit_tb that ends the TB. */
2616c808 2482 la_func_end(s, nb_globals, nb_temps);
c896fe29 2483
eae3eb3e 2484 QTAILQ_FOREACH_REVERSE_SAFE(op, &s->ops, link, op_prev) {
25f49c5f 2485 int nb_iargs, nb_oargs;
c45cb8bb
RH		 2486 TCGOpcode opc_new, opc_new2;
2487 bool have_opc_new2;
a1b3c48d 2488 TCGLifeData arg_life = 0;
25f49c5f 2489 TCGTemp *ts;
c45cb8bb
RH		 2490 TCGOpcode opc = op->opc;
2491 const TCGOpDef *def = &tcg_op_defs[opc];
2492
c45cb8bb 2493 switch (opc) {
c896fe29 2494 case INDEX_op_call:
c6e113f5
FB		 2495 {
2496 int call_flags;
25f49c5f 2497 int nb_call_regs;
c896fe29 2498
cd9090aa
RH		 2499 nb_oargs = TCGOP_CALLO(op);
2500 nb_iargs = TCGOP_CALLI(op);
90163900 2501 call_flags = tcg_call_flags(op);
c6e113f5 2502
c45cb8bb 2503 /* pure functions can be removed if their result is unused */
78505279 2504 if (call_flags & TCG_CALL_NO_SIDE_EFFECTS) {
cf066674 2505 for (i = 0; i < nb_oargs; i++) {
25f49c5f
RH		 2506 ts = arg_temp(op->args[i]);
2507 if (ts->state != TS_DEAD) {
c6e113f5 2508 goto do_not_remove_call;
9c43b68d 2509 }
c6e113f5 2510 }
c45cb8bb 2511 goto do_remove;
152c35aa
RH		 2512 }
2513 do_not_remove_call:
c896fe29 2514
25f49c5f 2515 /* Output args are dead. */
152c35aa 2516 for (i = 0; i < nb_oargs; i++) {
25f49c5f
RH		 2517 ts = arg_temp(op->args[i]);
2518 if (ts->state & TS_DEAD) {
152c35aa
RH		 2519 arg_life |= DEAD_ARG << i;
2520 }
25f49c5f 2521 if (ts->state & TS_MEM) {
152c35aa 2522 arg_life |= SYNC_ARG << i;
c6e113f5 2523 }
25f49c5f
RH		 2524 ts->state = TS_DEAD;
2525 la_reset_pref(ts);
2526
2527 /* Not used -- it will be tcg_target_call_oarg_regs[i]. */
2528 op->output_pref[i] = 0;
152c35aa 2529 }
78505279 2530
152c35aa
RH		 2531 if (!(call_flags & (TCG_CALL_NO_WRITE_GLOBALS |
2532 TCG_CALL_NO_READ_GLOBALS))) {
f65a061c 2533 la_global_kill(s, nb_globals);
152c35aa 2534 } else if (!(call_flags & TCG_CALL_NO_READ_GLOBALS)) {
f65a061c 2535 la_global_sync(s, nb_globals);
152c35aa 2536 }
b9c18f56 2537
25f49c5f 2538 /* Record arguments that die in this helper. */
152c35aa 2539 for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) {
25f49c5f
RH		 2540 ts = arg_temp(op->args[i]);
2541 if (ts && ts->state & TS_DEAD) {
152c35aa 2542 arg_life |= DEAD_ARG << i;
c6e113f5 2543 }
152c35aa 2544 }
25f49c5f
RH		 2545
2546 /* For all live registers, remove call-clobbered prefs. */
2547 la_cross_call(s, nb_temps);
2548
2549 nb_call_regs = ARRAY_SIZE(tcg_target_call_iarg_regs);
2550
2551 /* Input arguments are live for preceding opcodes. */
2552 for (i = 0; i < nb_iargs; i++) {
2553 ts = arg_temp(op->args[i + nb_oargs]);
2554 if (ts && ts->state & TS_DEAD) {
2555 /* For those arguments that die, and will be allocated
2556 * in registers, clear the register set for that arg,
2557 * to be filled in below. For args that will be on
2558 * the stack, reset to any available reg.
2559 */
2560 *la_temp_pref(ts)
2561 = (i < nb_call_regs ? 0 :
2562 tcg_target_available_regs[ts->type]);
2563 ts->state &= ~TS_DEAD;
2564 }
2565 }
2566
2567 /* For each input argument, add its input register to prefs.
2568 If a temp is used once, this produces a single set bit. */
2569 for (i = 0; i < MIN(nb_call_regs, nb_iargs); i++) {
2570 ts = arg_temp(op->args[i + nb_oargs]);
2571 if (ts) {
2572 tcg_regset_set_reg(*la_temp_pref(ts),
2573 tcg_target_call_iarg_regs[i]);
c19f47bf 2574 }
c896fe29 2575 }
c896fe29 2576 }
c896fe29 2577 break;
765b842a 2578 case INDEX_op_insn_start:
c896fe29 2579 break;
5ff9d6a4 2580 case INDEX_op_discard:
5ff9d6a4 2581 /* mark the temporary as dead */
25f49c5f
RH		 2582 ts = arg_temp(op->args[0]);
2583 ts->state = TS_DEAD;
2584 la_reset_pref(ts);
5ff9d6a4 2585 break;
1305c451
RH		 2586
2587 case INDEX_op_add2_i32:
c45cb8bb 2588 opc_new = INDEX_op_add_i32;
f1fae40c 2589 goto do_addsub2;
1305c451 2590 case INDEX_op_sub2_i32:
c45cb8bb 2591 opc_new = INDEX_op_sub_i32;
f1fae40c
RH		 2592 goto do_addsub2;
2593 case INDEX_op_add2_i64:
c45cb8bb 2594 opc_new = INDEX_op_add_i64;
f1fae40c
RH		 2595 goto do_addsub2;
2596 case INDEX_op_sub2_i64:
c45cb8bb 2597 opc_new = INDEX_op_sub_i64;
f1fae40c 2598 do_addsub2:
1305c451
RH		 2599 nb_iargs = 4;
2600 nb_oargs = 2;
2601 /* Test if the high part of the operation is dead, but not
2602 the low part. The result can be optimized to a simple
2603 add or sub. This happens often for x86_64 guest when the
2604 cpu mode is set to 32 bit. */
b83eabea
RH		 2605 if (arg_temp(op->args[1])->state == TS_DEAD) {
2606 if (arg_temp(op->args[0])->state == TS_DEAD) {
1305c451
RH		 2607 goto do_remove;
2608 }
c45cb8bb
RH		 2609 /* Replace the opcode and adjust the args in place,
2610 leaving 3 unused args at the end. */
2611 op->opc = opc = opc_new;
efee3746
RH		 2612 op->args[1] = op->args[2];
2613 op->args[2] = op->args[4];
1305c451
RH		 2614 /* Fall through and mark the single-word operation live. */
2615 nb_iargs = 2;
2616 nb_oargs = 1;
2617 }
2618 goto do_not_remove;
2619
1414968a 2620 case INDEX_op_mulu2_i32:
c45cb8bb
RH		 2621 opc_new = INDEX_op_mul_i32;
2622 opc_new2 = INDEX_op_muluh_i32;
2623 have_opc_new2 = TCG_TARGET_HAS_muluh_i32;
03271524 2624 goto do_mul2;
f1fae40c 2625 case INDEX_op_muls2_i32:
c45cb8bb
RH		 2626 opc_new = INDEX_op_mul_i32;
2627 opc_new2 = INDEX_op_mulsh_i32;
2628 have_opc_new2 = TCG_TARGET_HAS_mulsh_i32;
f1fae40c
RH		 2629 goto do_mul2;
2630 case INDEX_op_mulu2_i64:
c45cb8bb
RH		 2631 opc_new = INDEX_op_mul_i64;
2632 opc_new2 = INDEX_op_muluh_i64;
2633 have_opc_new2 = TCG_TARGET_HAS_muluh_i64;
03271524 2634 goto do_mul2;
f1fae40c 2635 case INDEX_op_muls2_i64:
c45cb8bb
RH		 2636 opc_new = INDEX_op_mul_i64;
2637 opc_new2 = INDEX_op_mulsh_i64;
2638 have_opc_new2 = TCG_TARGET_HAS_mulsh_i64;
03271524 2639 goto do_mul2;
f1fae40c 2640 do_mul2:
1414968a
RH		 2641 nb_iargs = 2;
2642 nb_oargs = 2;
b83eabea
RH		 2643 if (arg_temp(op->args[1])->state == TS_DEAD) {
2644 if (arg_temp(op->args[0])->state == TS_DEAD) {
03271524 2645 /* Both parts of the operation are dead. */
1414968a
RH		 2646 goto do_remove;
2647 }
03271524 2648 /* The high part of the operation is dead; generate the low. */
c45cb8bb 2649 op->opc = opc = opc_new;
efee3746
RH		 2650 op->args[1] = op->args[2];
2651 op->args[2] = op->args[3];
b83eabea 2652 } else if (arg_temp(op->args[0])->state == TS_DEAD && have_opc_new2) {
c45cb8bb
RH		 2653 /* The low part of the operation is dead; generate the high. */
2654 op->opc = opc = opc_new2;
efee3746
RH		 2655 op->args[0] = op->args[1];
2656 op->args[1] = op->args[2];
2657 op->args[2] = op->args[3];
03271524
RH		 2658 } else {
2659 goto do_not_remove;
1414968a 2660 }
03271524
RH		 2661 /* Mark the single-word operation live. */
2662 nb_oargs = 1;
1414968a
RH		 2663 goto do_not_remove;
2664
c896fe29 2665 default:
1305c451 2666 /* XXX: optimize by hardcoding common cases (e.g. triadic ops) */
49516bc0
AJ		 2667 nb_iargs = def->nb_iargs;
2668 nb_oargs = def->nb_oargs;
c896fe29 2669
49516bc0
AJ		 2670 /* Test if the operation can be removed because all
2671 its outputs are dead. We assume that nb_oargs == 0
2672 implies side effects */
2673 if (!(def->flags & TCG_OPF_SIDE_EFFECTS) && nb_oargs != 0) {
c45cb8bb 2674 for (i = 0; i < nb_oargs; i++) {
b83eabea 2675 if (arg_temp(op->args[i])->state != TS_DEAD) {
49516bc0 2676 goto do_not_remove;
9c43b68d 2677 }
49516bc0 2678 }
152c35aa
RH		 2679 goto do_remove;
2680 }
2681 goto do_not_remove;
49516bc0 2682
152c35aa
RH		 2683 do_remove:
2684 tcg_op_remove(s, op);
2685 break;
2686
2687 do_not_remove:
152c35aa 2688 for (i = 0; i < nb_oargs; i++) {
25f49c5f
RH		 2689 ts = arg_temp(op->args[i]);
2690
2691 /* Remember the preference of the uses that followed. */
2692 op->output_pref[i] = *la_temp_pref(ts);
2693
2694 /* Output args are dead. */
2695 if (ts->state & TS_DEAD) {
152c35aa 2696 arg_life |= DEAD_ARG << i;
49516bc0 2697 }
25f49c5f 2698 if (ts->state & TS_MEM) {
152c35aa
RH		 2699 arg_life |= SYNC_ARG << i;
2700 }
25f49c5f
RH		 2701 ts->state = TS_DEAD;
2702 la_reset_pref(ts);
152c35aa 2703 }
49516bc0 2704
25f49c5f 2705 /* If end of basic block, update. */
ae36a246
RH		 2706 if (def->flags & TCG_OPF_BB_EXIT) {
2707 la_func_end(s, nb_globals, nb_temps);
b4cb76e6
RH		 2708 } else if (def->flags & TCG_OPF_COND_BRANCH) {
2709 la_bb_sync(s, nb_globals, nb_temps);
ae36a246 2710 } else if (def->flags & TCG_OPF_BB_END) {
2616c808 2711 la_bb_end(s, nb_globals, nb_temps);
152c35aa 2712 } else if (def->flags & TCG_OPF_SIDE_EFFECTS) {
f65a061c 2713 la_global_sync(s, nb_globals);
25f49c5f
RH		 2714 if (def->flags & TCG_OPF_CALL_CLOBBER) {
2715 la_cross_call(s, nb_temps);
2716 }
152c35aa
RH		 2717 }
2718
25f49c5f 2719 /* Record arguments that die in this opcode. */
152c35aa 2720 for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) {
25f49c5f
RH		 2721 ts = arg_temp(op->args[i]);
2722 if (ts->state & TS_DEAD) {
152c35aa 2723 arg_life |= DEAD_ARG << i;
c896fe29 2724 }
c896fe29 2725 }
25f49c5f
RH		 2726
2727 /* Input arguments are live for preceding opcodes. */
152c35aa 2728 for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) {
25f49c5f
RH		 2729 ts = arg_temp(op->args[i]);
2730 if (ts->state & TS_DEAD) {
2731 /* For operands that were dead, initially allow
2732 all regs for the type. */
2733 *la_temp_pref(ts) = tcg_target_available_regs[ts->type];
2734 ts->state &= ~TS_DEAD;
2735 }
2736 }
2737
2738 /* Incorporate constraints for this operand. */
2739 switch (opc) {
2740 case INDEX_op_mov_i32:
2741 case INDEX_op_mov_i64:
2742 /* Note that these are TCG_OPF_NOT_PRESENT and do not
2743 have proper constraints. That said, special case
2744 moves to propagate preferences backward. */
2745 if (IS_DEAD_ARG(1)) {
2746 *la_temp_pref(arg_temp(op->args[0]))
2747 = *la_temp_pref(arg_temp(op->args[1]));
2748 }
2749 break;
2750
2751 default:
2752 for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) {
2753 const TCGArgConstraint *ct = &def->args_ct[i];
2754 TCGRegSet set, *pset;
2755
2756 ts = arg_temp(op->args[i]);
2757 pset = la_temp_pref(ts);
2758 set = *pset;
2759
9be0d080 2760 set &= ct->regs;
bc2b17e6 2761 if (ct->ialias) {
25f49c5f
RH		 2762 set &= op->output_pref[ct->alias_index];
2763 }
2764 /* If the combination is not possible, restart. */
2765 if (set == 0) {
9be0d080 2766 set = ct->regs;
25f49c5f
RH		 2767 }
2768 *pset = set;
2769 }
2770 break;
152c35aa 2771 }
c896fe29
FB		 2772 break;
2773 }
bee158cb 2774 op->life = arg_life;
1ff0a2c5 2775 }
c896fe29 2776}
c896fe29 2777
5a18407f 2778/* Liveness analysis: Convert indirect regs to direct temporaries. */
b83eabea 2779static bool liveness_pass_2(TCGContext *s)
5a18407f
RH		 2780{
2781 int nb_globals = s->nb_globals;
15fa08f8 2782 int nb_temps, i;
5a18407f 2783 bool changes = false;
15fa08f8 2784 TCGOp *op, *op_next;
5a18407f 2785
5a18407f
RH		 2786 /* Create a temporary for each indirect global. */
2787 for (i = 0; i < nb_globals; ++i) {
2788 TCGTemp *its = &s->temps[i];
2789 if (its->indirect_reg) {
2790 TCGTemp *dts = tcg_temp_alloc(s);
2791 dts->type = its->type;
2792 dts->base_type = its->base_type;
b83eabea
RH		 2793 its->state_ptr = dts;
2794 } else {
2795 its->state_ptr = NULL;
5a18407f 2796 }
b83eabea
RH		 2797 /* All globals begin dead. */
2798 its->state = TS_DEAD;
2799 }
2800 for (nb_temps = s->nb_temps; i < nb_temps; ++i) {
2801 TCGTemp *its = &s->temps[i];
2802 its->state_ptr = NULL;
2803 its->state = TS_DEAD;
5a18407f 2804 }
5a18407f 2805
15fa08f8 2806 QTAILQ_FOREACH_SAFE(op, &s->ops, link, op_next) {
5a18407f
RH		 2807 TCGOpcode opc = op->opc;
2808 const TCGOpDef *def = &tcg_op_defs[opc];
2809 TCGLifeData arg_life = op->life;
2810 int nb_iargs, nb_oargs, call_flags;
b83eabea 2811 TCGTemp *arg_ts, *dir_ts;
5a18407f 2812
5a18407f 2813 if (opc == INDEX_op_call) {
cd9090aa
RH		 2814 nb_oargs = TCGOP_CALLO(op);
2815 nb_iargs = TCGOP_CALLI(op);
90163900 2816 call_flags = tcg_call_flags(op);
5a18407f
RH		 2817 } else {
2818 nb_iargs = def->nb_iargs;
2819 nb_oargs = def->nb_oargs;
2820
2821 /* Set flags similar to how calls require. */
b4cb76e6
RH		 2822 if (def->flags & TCG_OPF_COND_BRANCH) {
2823 /* Like reading globals: sync_globals */
2824 call_flags = TCG_CALL_NO_WRITE_GLOBALS;
2825 } else if (def->flags & TCG_OPF_BB_END) {
5a18407f
RH		 2826 /* Like writing globals: save_globals */
2827 call_flags = 0;
2828 } else if (def->flags & TCG_OPF_SIDE_EFFECTS) {
2829 /* Like reading globals: sync_globals */
2830 call_flags = TCG_CALL_NO_WRITE_GLOBALS;
2831 } else {
2832 /* No effect on globals. */
2833 call_flags = (TCG_CALL_NO_READ_GLOBALS |
2834 TCG_CALL_NO_WRITE_GLOBALS);
2835 }
2836 }
2837
2838 /* Make sure that input arguments are available. */
2839 for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) {
b83eabea
RH		 2840 arg_ts = arg_temp(op->args[i]);
2841 if (arg_ts) {
2842 dir_ts = arg_ts->state_ptr;
2843 if (dir_ts && arg_ts->state == TS_DEAD) {
2844 TCGOpcode lopc = (arg_ts->type == TCG_TYPE_I32
5a18407f
RH		 2845 ? INDEX_op_ld_i32
2846 : INDEX_op_ld_i64);
ac1043f6 2847 TCGOp *lop = tcg_op_insert_before(s, op, lopc);
5a18407f 2848
b83eabea
RH		 2849 lop->args[0] = temp_arg(dir_ts);
2850 lop->args[1] = temp_arg(arg_ts->mem_base);
2851 lop->args[2] = arg_ts->mem_offset;
5a18407f
RH		 2852
2853 /* Loaded, but synced with memory. */
b83eabea 2854 arg_ts->state = TS_MEM;
5a18407f
RH		 2855 }
2856 }
2857 }
2858
2859 /* Perform input replacement, and mark inputs that became dead.
2860 No action is required except keeping temp_state up to date
2861 so that we reload when needed. */
2862 for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) {
b83eabea
RH		 2863 arg_ts = arg_temp(op->args[i]);
2864 if (arg_ts) {
2865 dir_ts = arg_ts->state_ptr;
2866 if (dir_ts) {
2867 op->args[i] = temp_arg(dir_ts);
5a18407f
RH		 2868 changes = true;
2869 if (IS_DEAD_ARG(i)) {
b83eabea 2870 arg_ts->state = TS_DEAD;
5a18407f
RH		 2871 }
2872 }
2873 }
2874 }
2875
2876 /* Liveness analysis should ensure that the following are
2877 all correct, for call sites and basic block end points. */
2878 if (call_flags & TCG_CALL_NO_READ_GLOBALS) {
2879 /* Nothing to do */
2880 } else if (call_flags & TCG_CALL_NO_WRITE_GLOBALS) {
2881 for (i = 0; i < nb_globals; ++i) {
2882 /* Liveness should see that globals are synced back,
2883 that is, either TS_DEAD or TS_MEM. */
b83eabea
RH		 2884 arg_ts = &s->temps[i];
2885 tcg_debug_assert(arg_ts->state_ptr == 0
2886 || arg_ts->state != 0);
5a18407f
RH		 2887 }
2888 } else {
2889 for (i = 0; i < nb_globals; ++i) {
2890 /* Liveness should see that globals are saved back,
2891 that is, TS_DEAD, waiting to be reloaded. */
b83eabea
RH		 2892 arg_ts = &s->temps[i];
2893 tcg_debug_assert(arg_ts->state_ptr == 0
2894 || arg_ts->state == TS_DEAD);
5a18407f
RH		 2895 }
2896 }
2897
2898 /* Outputs become available. */
61f15c48
RH		 2899 if (opc == INDEX_op_mov_i32 || opc == INDEX_op_mov_i64) {
2900 arg_ts = arg_temp(op->args[0]);
b83eabea 2901 dir_ts = arg_ts->state_ptr;
61f15c48
RH		 2902 if (dir_ts) {
2903 op->args[0] = temp_arg(dir_ts);
2904 changes = true;
2905
2906 /* The output is now live and modified. */
2907 arg_ts->state = 0;
2908
2909 if (NEED_SYNC_ARG(0)) {
2910 TCGOpcode sopc = (arg_ts->type == TCG_TYPE_I32
2911 ? INDEX_op_st_i32
2912 : INDEX_op_st_i64);
2913 TCGOp *sop = tcg_op_insert_after(s, op, sopc);
2914 TCGTemp *out_ts = dir_ts;
2915
2916 if (IS_DEAD_ARG(0)) {
2917 out_ts = arg_temp(op->args[1]);
2918 arg_ts->state = TS_DEAD;
2919 tcg_op_remove(s, op);
2920 } else {
2921 arg_ts->state = TS_MEM;
2922 }
2923
2924 sop->args[0] = temp_arg(out_ts);
2925 sop->args[1] = temp_arg(arg_ts->mem_base);
2926 sop->args[2] = arg_ts->mem_offset;
2927 } else {
2928 tcg_debug_assert(!IS_DEAD_ARG(0));
2929 }
5a18407f 2930 }
61f15c48
RH		 2931 } else {
2932 for (i = 0; i < nb_oargs; i++) {
2933 arg_ts = arg_temp(op->args[i]);
2934 dir_ts = arg_ts->state_ptr;
2935 if (!dir_ts) {
2936 continue;
2937 }
2938 op->args[i] = temp_arg(dir_ts);
2939 changes = true;
5a18407f 2940
61f15c48
RH		 2941 /* The output is now live and modified. */
2942 arg_ts->state = 0;
5a18407f 2943
61f15c48
RH		 2944 /* Sync outputs upon their last write. */
2945 if (NEED_SYNC_ARG(i)) {
2946 TCGOpcode sopc = (arg_ts->type == TCG_TYPE_I32
2947 ? INDEX_op_st_i32
2948 : INDEX_op_st_i64);
2949 TCGOp *sop = tcg_op_insert_after(s, op, sopc);
5a18407f 2950
61f15c48
RH		 2951 sop->args[0] = temp_arg(dir_ts);
2952 sop->args[1] = temp_arg(arg_ts->mem_base);
2953 sop->args[2] = arg_ts->mem_offset;
5a18407f 2954
61f15c48
RH		 2955 arg_ts->state = TS_MEM;
2956 }
2957 /* Drop outputs that are dead. */
2958 if (IS_DEAD_ARG(i)) {
2959 arg_ts->state = TS_DEAD;
2960 }
5a18407f
RH		 2961 }
2962 }
2963 }
2964
2965 return changes;
2966}
2967
8d8fdbae 2968#ifdef CONFIG_DEBUG_TCG
c896fe29
FB		 2969static void dump_regs(TCGContext *s)
2970{
2971 TCGTemp *ts;
2972 int i;
2973 char buf[64];
2974
2975 for(i = 0; i < s->nb_temps; i++) {
2976 ts = &s->temps[i];
43439139 2977 printf(" %10s: ", tcg_get_arg_str_ptr(s, buf, sizeof(buf), ts));
c896fe29
FB		 2978 switch(ts->val_type) {
2979 case TEMP_VAL_REG:
2980 printf("%s", tcg_target_reg_names[ts->reg]);
2981 break;
2982 case TEMP_VAL_MEM:
b3a62939
RH		 2983 printf("%d(%s)", (int)ts->mem_offset,
2984 tcg_target_reg_names[ts->mem_base->reg]);
c896fe29
FB		 2985 break;
2986 case TEMP_VAL_CONST:
bdb38b95 2987 printf("$0x%" PRIx64, ts->val);
c896fe29
FB		 2988 break;
2989 case TEMP_VAL_DEAD:
2990 printf("D");
2991 break;
2992 default:
2993 printf("???");
2994 break;
2995 }
2996 printf("\n");
2997 }
2998
2999 for(i = 0; i < TCG_TARGET_NB_REGS; i++) {
f8b2f202 3000 if (s->reg_to_temp[i] != NULL) {
c896fe29
FB		 3001 printf("%s: %s\n",
3002 tcg_target_reg_names[i],
f8b2f202 3003 tcg_get_arg_str_ptr(s, buf, sizeof(buf), s->reg_to_temp[i]));
c896fe29
FB		 3004 }
3005 }
3006}
3007
3008static void check_regs(TCGContext *s)
3009{
869938ae 3010 int reg;
b6638662 3011 int k;
c896fe29
FB		 3012 TCGTemp *ts;
3013 char buf[64];
3014
f8b2f202
RH		 3015 for (reg = 0; reg < TCG_TARGET_NB_REGS; reg++) {
3016 ts = s->reg_to_temp[reg];
3017 if (ts != NULL) {
3018 if (ts->val_type != TEMP_VAL_REG || ts->reg != reg) {
c896fe29
FB		 3019 printf("Inconsistency for register %s:\n",
3020 tcg_target_reg_names[reg]);
b03cce8e 3021 goto fail;
c896fe29
FB		 3022 }
3023 }
3024 }
f8b2f202 3025 for (k = 0; k < s->nb_temps; k++) {
c896fe29 3026 ts = &s->temps[k];
ee17db83
RH		 3027 if (ts->val_type == TEMP_VAL_REG
3028 && ts->kind != TEMP_FIXED
f8b2f202
RH		 3029 && s->reg_to_temp[ts->reg] != ts) {
3030 printf("Inconsistency for temp %s:\n",
3031 tcg_get_arg_str_ptr(s, buf, sizeof(buf), ts));
b03cce8e 3032 fail:
f8b2f202
RH		 3033 printf("reg state:\n");
3034 dump_regs(s);
3035 tcg_abort();
c896fe29
FB		 3036 }
3037 }
3038}
3039#endif
3040
2272e4a7 3041static void temp_allocate_frame(TCGContext *s, TCGTemp *ts)
c896fe29 3042{
c1c09194
RH		 3043 intptr_t off, size, align;
3044
3045 switch (ts->type) {
3046 case TCG_TYPE_I32:
3047 size = align = 4;
3048 break;
3049 case TCG_TYPE_I64:
3050 case TCG_TYPE_V64:
3051 size = align = 8;
3052 break;
3053 case TCG_TYPE_V128:
3054 size = align = 16;
3055 break;
3056 case TCG_TYPE_V256:
3057 /* Note that we do not require aligned storage for V256. */
3058 size = 32, align = 16;
3059 break;
3060 default:
3061 g_assert_not_reached();
b591dc59 3062 }
c1c09194 3063
b9537d59
RH		 3064 /*
3065 * Assume the stack is sufficiently aligned.
3066 * This affects e.g. ARM NEON, where we have 8 byte stack alignment
3067 * and do not require 16 byte vector alignment. This seems slightly
3068 * easier than fully parameterizing the above switch statement.
3069 */
3070 align = MIN(TCG_TARGET_STACK_ALIGN, align);
c1c09194 3071 off = ROUND_UP(s->current_frame_offset, align);
732d5897
RH		 3072
3073 /* If we've exhausted the stack frame, restart with a smaller TB. */
3074 if (off + size > s->frame_end) {
3075 tcg_raise_tb_overflow(s);
3076 }
c1c09194
RH		 3077 s->current_frame_offset = off + size;
3078
3079 ts->mem_offset = off;
9defd1bd
RH		 3080#if defined(__sparc__)
3081 ts->mem_offset += TCG_TARGET_STACK_BIAS;
3082#endif
b3a62939 3083 ts->mem_base = s->frame_temp;
c896fe29 3084 ts->mem_allocated = 1;
c896fe29
FB		 3085}
3086
b722452a 3087static void temp_load(TCGContext *, TCGTemp *, TCGRegSet, TCGRegSet, TCGRegSet);
b3915dbb 3088
59d7c14e
RH		 3089/* Mark a temporary as free or dead. If 'free_or_dead' is negative,
3090 mark it free; otherwise mark it dead. */
3091static void temp_free_or_dead(TCGContext *s, TCGTemp *ts, int free_or_dead)
7f6ceedf 3092{
c0522136
RH		 3093 TCGTempVal new_type;
3094
3095 switch (ts->kind) {
3096 case TEMP_FIXED:
59d7c14e 3097 return;
c0522136
RH		 3098 case TEMP_GLOBAL:
3099 case TEMP_LOCAL:
3100 new_type = TEMP_VAL_MEM;
3101 break;
3102 case TEMP_NORMAL:
3103 new_type = free_or_dead < 0 ? TEMP_VAL_MEM : TEMP_VAL_DEAD;
3104 break;
3105 case TEMP_CONST:
3106 new_type = TEMP_VAL_CONST;
3107 break;
3108 default:
3109 g_assert_not_reached();
59d7c14e
RH		 3110 }
3111 if (ts->val_type == TEMP_VAL_REG) {
3112 s->reg_to_temp[ts->reg] = NULL;
3113 }
c0522136 3114 ts->val_type = new_type;
59d7c14e 3115}
7f6ceedf 3116
59d7c14e
RH		 3117/* Mark a temporary as dead. */
3118static inline void temp_dead(TCGContext *s, TCGTemp *ts)
3119{
3120 temp_free_or_dead(s, ts, 1);
3121}
3122
3123/* Sync a temporary to memory. 'allocated_regs' is used in case a temporary
3124 registers needs to be allocated to store a constant. If 'free_or_dead'
3125 is non-zero, subsequently release the temporary; if it is positive, the
3126 temp is dead; if it is negative, the temp is free. */
98b4e186
RH		 3127static void temp_sync(TCGContext *s, TCGTemp *ts, TCGRegSet allocated_regs,
3128 TCGRegSet preferred_regs, int free_or_dead)
59d7c14e 3129{
c0522136 3130 if (!temp_readonly(ts) && !ts->mem_coherent) {
7f6ceedf 3131 if (!ts->mem_allocated) {
2272e4a7 3132 temp_allocate_frame(s, ts);
59d7c14e 3133 }
59d7c14e
RH		 3134 switch (ts->val_type) {
3135 case TEMP_VAL_CONST:
3136 /* If we're going to free the temp immediately, then we won't
3137 require it later in a register, so attempt to store the
3138 constant to memory directly. */
3139 if (free_or_dead
3140 && tcg_out_sti(s, ts->type, ts->val,
3141 ts->mem_base->reg, ts->mem_offset)) {
3142 break;
3143 }
3144 temp_load(s, ts, tcg_target_available_regs[ts->type],
98b4e186 3145 allocated_regs, preferred_regs);
59d7c14e
RH		 3146 /* fallthrough */
3147
3148 case TEMP_VAL_REG:
3149 tcg_out_st(s, ts->type, ts->reg,
3150 ts->mem_base->reg, ts->mem_offset);
3151 break;
3152
3153 case TEMP_VAL_MEM:
3154 break;
3155
3156 case TEMP_VAL_DEAD:
3157 default:
3158 tcg_abort();
3159 }
3160 ts->mem_coherent = 1;
3161 }
3162 if (free_or_dead) {
3163 temp_free_or_dead(s, ts, free_or_dead);
7f6ceedf 3164 }
7f6ceedf
AJ		 3165}
3166
c896fe29 3167/* free register 'reg' by spilling the corresponding temporary if necessary */
b3915dbb 3168static void tcg_reg_free(TCGContext *s, TCGReg reg, TCGRegSet allocated_regs)
c896fe29 3169{
f8b2f202 3170 TCGTemp *ts = s->reg_to_temp[reg];
f8b2f202 3171 if (ts != NULL) {
98b4e186 3172 temp_sync(s, ts, allocated_regs, 0, -1);
c896fe29
FB		 3173 }
3174}
3175
b016486e
RH		 3176/**
3177 * tcg_reg_alloc:
3178 * @required_regs: Set of registers in which we must allocate.
3179 * @allocated_regs: Set of registers which must be avoided.
3180 * @preferred_regs: Set of registers we should prefer.
3181 * @rev: True if we search the registers in "indirect" order.
3182 *
3183 * The allocated register must be in @required_regs & ~@allocated_regs,
3184 * but if we can put it in @preferred_regs we may save a move later.
3185 */
3186static TCGReg tcg_reg_alloc(TCGContext *s, TCGRegSet required_regs,
3187 TCGRegSet allocated_regs,
3188 TCGRegSet preferred_regs, bool rev)
c896fe29 3189{
b016486e
RH		 3190 int i, j, f, n = ARRAY_SIZE(tcg_target_reg_alloc_order);
3191 TCGRegSet reg_ct[2];
91478cef 3192 const int *order;
c896fe29 3193
b016486e
RH		 3194 reg_ct[1] = required_regs & ~allocated_regs;
3195 tcg_debug_assert(reg_ct[1] != 0);
3196 reg_ct[0] = reg_ct[1] & preferred_regs;
3197
3198 /* Skip the preferred_regs option if it cannot be satisfied,
3199 or if the preference made no difference. */
3200 f = reg_ct[0] == 0 || reg_ct[0] == reg_ct[1];
3201
91478cef 3202 order = rev ? indirect_reg_alloc_order : tcg_target_reg_alloc_order;
c896fe29 3203
b016486e
RH		 3204 /* Try free registers, preferences first. */
3205 for (j = f; j < 2; j++) {
3206 TCGRegSet set = reg_ct[j];
3207
3208 if (tcg_regset_single(set)) {
3209 /* One register in the set. */
3210 TCGReg reg = tcg_regset_first(set);
3211 if (s->reg_to_temp[reg] == NULL) {
3212 return reg;
3213 }
3214 } else {
3215 for (i = 0; i < n; i++) {
3216 TCGReg reg = order[i];
3217 if (s->reg_to_temp[reg] == NULL &&
3218 tcg_regset_test_reg(set, reg)) {
3219 return reg;
3220 }
3221 }
3222 }
c896fe29
FB		 3223 }
3224
b016486e
RH		 3225 /* We must spill something. */
3226 for (j = f; j < 2; j++) {
3227 TCGRegSet set = reg_ct[j];
3228
3229 if (tcg_regset_single(set)) {
3230 /* One register in the set. */
3231 TCGReg reg = tcg_regset_first(set);
b3915dbb 3232 tcg_reg_free(s, reg, allocated_regs);
c896fe29 3233 return reg;
b016486e
RH		 3234 } else {
3235 for (i = 0; i < n; i++) {
3236 TCGReg reg = order[i];
3237 if (tcg_regset_test_reg(set, reg)) {
3238 tcg_reg_free(s, reg, allocated_regs);
3239 return reg;
3240 }
3241 }
c896fe29
FB		 3242 }
3243 }
3244
3245 tcg_abort();
3246}
3247
40ae5c62
RH		 3248/* Make sure the temporary is in a register. If needed, allocate the register
3249 from DESIRED while avoiding ALLOCATED. */
3250static void temp_load(TCGContext *s, TCGTemp *ts, TCGRegSet desired_regs,
b722452a 3251 TCGRegSet allocated_regs, TCGRegSet preferred_regs)
40ae5c62
RH		 3252{
3253 TCGReg reg;
3254
3255 switch (ts->val_type) {
3256 case TEMP_VAL_REG:
3257 return;
3258 case TEMP_VAL_CONST:
b016486e 3259 reg = tcg_reg_alloc(s, desired_regs, allocated_regs,
b722452a 3260 preferred_regs, ts->indirect_base);
0a6a8bc8
RH		 3261 if (ts->type <= TCG_TYPE_I64) {
3262 tcg_out_movi(s, ts->type, reg, ts->val);
3263 } else {
4e186175
RH		 3264 uint64_t val = ts->val;
3265 MemOp vece = MO_64;
3266
3267 /*
3268 * Find the minimal vector element that matches the constant.
3269 * The targets will, in general, have to do this search anyway,
3270 * do this generically.
3271 */
4e186175
RH		 3272 if (val == dup_const(MO_8, val)) {
3273 vece = MO_8;
3274 } else if (val == dup_const(MO_16, val)) {
3275 vece = MO_16;
0b4286dd 3276 } else if (val == dup_const(MO_32, val)) {
4e186175
RH		 3277 vece = MO_32;
3278 }
3279
3280 tcg_out_dupi_vec(s, ts->type, vece, reg, ts->val);
0a6a8bc8 3281 }
40ae5c62
RH		 3282 ts->mem_coherent = 0;
3283 break;
3284 case TEMP_VAL_MEM:
b016486e 3285 reg = tcg_reg_alloc(s, desired_regs, allocated_regs,
b722452a 3286 preferred_regs, ts->indirect_base);
40ae5c62
RH		 3287 tcg_out_ld(s, ts->type, reg, ts->mem_base->reg, ts->mem_offset);
3288 ts->mem_coherent = 1;
3289 break;
3290 case TEMP_VAL_DEAD:
3291 default:
3292 tcg_abort();
3293 }
3294 ts->reg = reg;
3295 ts->val_type = TEMP_VAL_REG;
3296 s->reg_to_temp[reg] = ts;
3297}
3298
59d7c14e
RH		 3299/* Save a temporary to memory. 'allocated_regs' is used in case a
3300 temporary registers needs to be allocated to store a constant. */
3301static void temp_save(TCGContext *s, TCGTemp *ts, TCGRegSet allocated_regs)
1ad80729 3302{
5a18407f
RH		 3303 /* The liveness analysis already ensures that globals are back
3304 in memory. Keep an tcg_debug_assert for safety. */
e01fa97d 3305 tcg_debug_assert(ts->val_type == TEMP_VAL_MEM || temp_readonly(ts));
1ad80729
AJ		 3306}
3307
9814dd27 3308/* save globals to their canonical location and assume they can be
e8996ee0
FB		 3309 modified be the following code. 'allocated_regs' is used in case a
3310 temporary registers needs to be allocated to store a constant. */
3311static void save_globals(TCGContext *s, TCGRegSet allocated_regs)
c896fe29 3312{
ac3b8891 3313 int i, n;
c896fe29 3314
ac3b8891 3315 for (i = 0, n = s->nb_globals; i < n; i++) {
b13eb728 3316 temp_save(s, &s->temps[i], allocated_regs);
c896fe29 3317 }
e5097dc8
FB		 3318}
3319
3d5c5f87
AJ		 3320/* sync globals to their canonical location and assume they can be
3321 read by the following code. 'allocated_regs' is used in case a
3322 temporary registers needs to be allocated to store a constant. */
3323static void sync_globals(TCGContext *s, TCGRegSet allocated_regs)
3324{
ac3b8891 3325 int i, n;
3d5c5f87 3326
ac3b8891 3327 for (i = 0, n = s->nb_globals; i < n; i++) {
12b9b11a 3328 TCGTemp *ts = &s->temps[i];
5a18407f 3329 tcg_debug_assert(ts->val_type != TEMP_VAL_REG
ee17db83 3330 || ts->kind == TEMP_FIXED
5a18407f 3331 || ts->mem_coherent);
3d5c5f87
AJ		 3332 }
3333}
3334
e5097dc8 3335/* at the end of a basic block, we assume all temporaries are dead and
e8996ee0
FB		 3336 all globals are stored at their canonical location. */
3337static void tcg_reg_alloc_bb_end(TCGContext *s, TCGRegSet allocated_regs)
e5097dc8 3338{
e5097dc8
FB		 3339 int i;
3340
b13eb728
RH		 3341 for (i = s->nb_globals; i < s->nb_temps; i++) {
3342 TCGTemp *ts = &s->temps[i];
c0522136
RH		 3343
3344 switch (ts->kind) {
3345 case TEMP_LOCAL:
b13eb728 3346 temp_save(s, ts, allocated_regs);
c0522136
RH		 3347 break;
3348 case TEMP_NORMAL:
5a18407f
RH		 3349 /* The liveness analysis already ensures that temps are dead.
3350 Keep an tcg_debug_assert for safety. */
3351 tcg_debug_assert(ts->val_type == TEMP_VAL_DEAD);
c0522136
RH		 3352 break;
3353 case TEMP_CONST:
3354 /* Similarly, we should have freed any allocated register. */
3355 tcg_debug_assert(ts->val_type == TEMP_VAL_CONST);
3356 break;
3357 default:
3358 g_assert_not_reached();
c896fe29
FB		 3359 }
3360 }
e8996ee0
FB		 3361
3362 save_globals(s, allocated_regs);
c896fe29
FB		 3363}
3364
b4cb76e6
RH		 3365/*
3366 * At a conditional branch, we assume all temporaries are dead and
3367 * all globals and local temps are synced to their location.
3368 */
3369static void tcg_reg_alloc_cbranch(TCGContext *s, TCGRegSet allocated_regs)
3370{
3371 sync_globals(s, allocated_regs);
3372
3373 for (int i = s->nb_globals; i < s->nb_temps; i++) {
3374 TCGTemp *ts = &s->temps[i];
3375 /*
3376 * The liveness analysis already ensures that temps are dead.
3377 * Keep tcg_debug_asserts for safety.
3378 */
c0522136
RH		 3379 switch (ts->kind) {
3380 case TEMP_LOCAL:
b4cb76e6 3381 tcg_debug_assert(ts->val_type != TEMP_VAL_REG || ts->mem_coherent);
c0522136
RH		 3382 break;
3383 case TEMP_NORMAL:
b4cb76e6 3384 tcg_debug_assert(ts->val_type == TEMP_VAL_DEAD);
c0522136
RH		 3385 break;
3386 case TEMP_CONST:
3387 break;
3388 default:
3389 g_assert_not_reached();
b4cb76e6
RH		 3390 }
3391 }
3392}
3393
bab1671f 3394/*
c58f4c97 3395 * Specialized code generation for INDEX_op_mov_* with a constant.
bab1671f 3396 */
0fe4fca4 3397static void tcg_reg_alloc_do_movi(TCGContext *s, TCGTemp *ots,
ba87719c
RH		 3398 tcg_target_ulong val, TCGLifeData arg_life,
3399 TCGRegSet preferred_regs)
e8996ee0 3400{
d63e3b6e 3401 /* ENV should not be modified. */
e01fa97d 3402 tcg_debug_assert(!temp_readonly(ots));
59d7c14e
RH		 3403
3404 /* The movi is not explicitly generated here. */
3405 if (ots->val_type == TEMP_VAL_REG) {
3406 s->reg_to_temp[ots->reg] = NULL;
ec7a869d 3407 }
59d7c14e
RH		 3408 ots->val_type = TEMP_VAL_CONST;
3409 ots->val = val;
3410 ots->mem_coherent = 0;
3411 if (NEED_SYNC_ARG(0)) {
ba87719c 3412 temp_sync(s, ots, s->reserved_regs, preferred_regs, IS_DEAD_ARG(0));
59d7c14e 3413 } else if (IS_DEAD_ARG(0)) {
f8bf00f1 3414 temp_dead(s, ots);
4c4e1ab2 3415 }
e8996ee0
FB		 3416}
3417
bab1671f
RH		 3418/*
3419 * Specialized code generation for INDEX_op_mov_*.
3420 */
dd186292 3421static void tcg_reg_alloc_mov(TCGContext *s, const TCGOp *op)
c896fe29 3422{
dd186292 3423 const TCGLifeData arg_life = op->life;
69e3706d 3424 TCGRegSet allocated_regs, preferred_regs;
c896fe29 3425 TCGTemp *ts, *ots;
450445d5 3426 TCGType otype, itype;
c896fe29 3427
d21369f5 3428 allocated_regs = s->reserved_regs;
69e3706d 3429 preferred_regs = op->output_pref[0];
43439139
RH		 3430 ots = arg_temp(op->args[0]);
3431 ts = arg_temp(op->args[1]);
450445d5 3432
d63e3b6e 3433 /* ENV should not be modified. */
e01fa97d 3434 tcg_debug_assert(!temp_readonly(ots));
d63e3b6e 3435
450445d5
RH		 3436 /* Note that otype != itype for no-op truncation. */
3437 otype = ots->type;
3438 itype = ts->type;
c29c1d7e 3439
0fe4fca4
PB		 3440 if (ts->val_type == TEMP_VAL_CONST) {
3441 /* propagate constant or generate sti */
3442 tcg_target_ulong val = ts->val;
3443 if (IS_DEAD_ARG(1)) {
3444 temp_dead(s, ts);
3445 }
69e3706d 3446 tcg_reg_alloc_do_movi(s, ots, val, arg_life, preferred_regs);
0fe4fca4
PB		 3447 return;
3448 }
3449
3450 /* If the source value is in memory we're going to be forced
3451 to have it in a register in order to perform the copy. Copy
3452 the SOURCE value into its own register first, that way we
3453 don't have to reload SOURCE the next time it is used. */
3454 if (ts->val_type == TEMP_VAL_MEM) {
69e3706d
RH		 3455 temp_load(s, ts, tcg_target_available_regs[itype],
3456 allocated_regs, preferred_regs);
c29c1d7e 3457 }
c896fe29 3458
0fe4fca4 3459 tcg_debug_assert(ts->val_type == TEMP_VAL_REG);
d63e3b6e 3460 if (IS_DEAD_ARG(0)) {
c29c1d7e
AJ		 3461 /* mov to a non-saved dead register makes no sense (even with
3462 liveness analysis disabled). */
eabb7b91 3463 tcg_debug_assert(NEED_SYNC_ARG(0));
c29c1d7e 3464 if (!ots->mem_allocated) {
2272e4a7 3465 temp_allocate_frame(s, ots);
c29c1d7e 3466 }
b3a62939 3467 tcg_out_st(s, otype, ts->reg, ots->mem_base->reg, ots->mem_offset);
c29c1d7e 3468 if (IS_DEAD_ARG(1)) {
f8bf00f1 3469 temp_dead(s, ts);
c29c1d7e 3470 }
f8bf00f1 3471 temp_dead(s, ots);
c29c1d7e 3472 } else {
ee17db83 3473 if (IS_DEAD_ARG(1) && ts->kind != TEMP_FIXED) {
c896fe29 3474 /* the mov can be suppressed */
c29c1d7e 3475 if (ots->val_type == TEMP_VAL_REG) {
f8b2f202 3476 s->reg_to_temp[ots->reg] = NULL;
c29c1d7e
AJ		 3477 }
3478 ots->reg = ts->reg;
f8bf00f1 3479 temp_dead(s, ts);
c896fe29 3480 } else {
c29c1d7e
AJ		 3481 if (ots->val_type != TEMP_VAL_REG) {
3482 /* When allocating a new register, make sure to not spill the
3483 input one. */
3484 tcg_regset_set_reg(allocated_regs, ts->reg);
450445d5 3485 ots->reg = tcg_reg_alloc(s, tcg_target_available_regs[otype],
69e3706d 3486 allocated_regs, preferred_regs,
b016486e 3487 ots->indirect_base);
c896fe29 3488 }
78113e83 3489 if (!tcg_out_mov(s, otype, ots->reg, ts->reg)) {
240c08d0
RH		 3490 /*
3491 * Cross register class move not supported.
3492 * Store the source register into the destination slot
3493 * and leave the destination temp as TEMP_VAL_MEM.
3494 */
e01fa97d 3495 assert(!temp_readonly(ots));
240c08d0
RH		 3496 if (!ts->mem_allocated) {
3497 temp_allocate_frame(s, ots);
3498 }
3499 tcg_out_st(s, ts->type, ts->reg,
3500 ots->mem_base->reg, ots->mem_offset);
3501 ots->mem_coherent = 1;
3502 temp_free_or_dead(s, ots, -1);
3503 return;
78113e83 3504 }
c896fe29 3505 }
c29c1d7e
AJ		 3506 ots->val_type = TEMP_VAL_REG;
3507 ots->mem_coherent = 0;
f8b2f202 3508 s->reg_to_temp[ots->reg] = ots;
c29c1d7e 3509 if (NEED_SYNC_ARG(0)) {
98b4e186 3510 temp_sync(s, ots, allocated_regs, 0, 0);
c896fe29 3511 }
ec7a869d 3512 }
c896fe29
FB		 3513}
3514
bab1671f
RH		 3515/*
3516 * Specialized code generation for INDEX_op_dup_vec.
3517 */
3518static void tcg_reg_alloc_dup(TCGContext *s, const TCGOp *op)
3519{
3520 const TCGLifeData arg_life = op->life;
3521 TCGRegSet dup_out_regs, dup_in_regs;
3522 TCGTemp *its, *ots;
3523 TCGType itype, vtype;
d6ecb4a9 3524 intptr_t endian_fixup;
bab1671f
RH		 3525 unsigned vece;
3526 bool ok;
3527
3528 ots = arg_temp(op->args[0]);
3529 its = arg_temp(op->args[1]);
3530
3531 /* ENV should not be modified. */
e01fa97d 3532 tcg_debug_assert(!temp_readonly(ots));
bab1671f
RH		 3533
3534 itype = its->type;
3535 vece = TCGOP_VECE(op);
3536 vtype = TCGOP_VECL(op) + TCG_TYPE_V64;
3537
3538 if (its->val_type == TEMP_VAL_CONST) {
3539 /* Propagate constant via movi -> dupi. */
3540 tcg_target_ulong val = its->val;
3541 if (IS_DEAD_ARG(1)) {
3542 temp_dead(s, its);
3543 }
3544 tcg_reg_alloc_do_movi(s, ots, val, arg_life, op->output_pref[0]);
3545 return;
3546 }
3547
9be0d080
RH		 3548 dup_out_regs = tcg_op_defs[INDEX_op_dup_vec].args_ct[0].regs;
3549 dup_in_regs = tcg_op_defs[INDEX_op_dup_vec].args_ct[1].regs;
bab1671f
RH		 3550
3551 /* Allocate the output register now. */
3552 if (ots->val_type != TEMP_VAL_REG) {
3553 TCGRegSet allocated_regs = s->reserved_regs;
3554
3555 if (!IS_DEAD_ARG(1) && its->val_type == TEMP_VAL_REG) {
3556 /* Make sure to not spill the input register. */
3557 tcg_regset_set_reg(allocated_regs, its->reg);
3558 }
3559 ots->reg = tcg_reg_alloc(s, dup_out_regs, allocated_regs,
3560 op->output_pref[0], ots->indirect_base);
3561 ots->val_type = TEMP_VAL_REG;
3562 ots->mem_coherent = 0;
3563 s->reg_to_temp[ots->reg] = ots;
3564 }
3565
3566 switch (its->val_type) {
3567 case TEMP_VAL_REG:
3568 /*
3569 * The dup constriaints must be broad, covering all possible VECE.
3570 * However, tcg_op_dup_vec() gets to see the VECE and we allow it
3571 * to fail, indicating that extra moves are required for that case.
3572 */
3573 if (tcg_regset_test_reg(dup_in_regs, its->reg)) {
3574 if (tcg_out_dup_vec(s, vtype, vece, ots->reg, its->reg)) {
3575 goto done;
3576 }
3577 /* Try again from memory or a vector input register. */
3578 }
3579 if (!its->mem_coherent) {
3580 /*
3581 * The input register is not synced, and so an extra store
3582 * would be required to use memory. Attempt an integer-vector
3583 * register move first. We do not have a TCGRegSet for this.
3584 */
3585 if (tcg_out_mov(s, itype, ots->reg, its->reg)) {
3586 break;
3587 }
3588 /* Sync the temp back to its slot and load from there. */
3589 temp_sync(s, its, s->reserved_regs, 0, 0);
3590 }
3591 /* fall through */
3592
3593 case TEMP_VAL_MEM:
d6ecb4a9
RH		 3594#ifdef HOST_WORDS_BIGENDIAN
3595 endian_fixup = itype == TCG_TYPE_I32 ? 4 : 8;
3596 endian_fixup -= 1 << vece;
3597#else
3598 endian_fixup = 0;
3599#endif
3600 if (tcg_out_dupm_vec(s, vtype, vece, ots->reg, its->mem_base->reg,
3601 its->mem_offset + endian_fixup)) {
3602 goto done;
3603 }
bab1671f
RH		 3604 tcg_out_ld(s, itype, ots->reg, its->mem_base->reg, its->mem_offset);
3605 break;
3606
3607 default:
3608 g_assert_not_reached();
3609 }
3610
3611 /* We now have a vector input register, so dup must succeed. */
3612 ok = tcg_out_dup_vec(s, vtype, vece, ots->reg, ots->reg);
3613 tcg_debug_assert(ok);
3614
3615 done:
3616 if (IS_DEAD_ARG(1)) {
3617 temp_dead(s, its);
3618 }
3619 if (NEED_SYNC_ARG(0)) {
3620 temp_sync(s, ots, s->reserved_regs, 0, 0);
3621 }
3622 if (IS_DEAD_ARG(0)) {
3623 temp_dead(s, ots);
3624 }
3625}
3626
dd186292 3627static void tcg_reg_alloc_op(TCGContext *s, const TCGOp *op)
c896fe29 3628{
dd186292
RH		 3629 const TCGLifeData arg_life = op->life;
3630 const TCGOpDef * const def = &tcg_op_defs[op->opc];
82790a87
RH		 3631 TCGRegSet i_allocated_regs;
3632 TCGRegSet o_allocated_regs;
b6638662
RH		 3633 int i, k, nb_iargs, nb_oargs;
3634 TCGReg reg;
c896fe29
FB		 3635 TCGArg arg;
3636 const TCGArgConstraint *arg_ct;
3637 TCGTemp *ts;
3638 TCGArg new_args[TCG_MAX_OP_ARGS];
3639 int const_args[TCG_MAX_OP_ARGS];
3640
3641 nb_oargs = def->nb_oargs;
3642 nb_iargs = def->nb_iargs;
3643
3644 /* copy constants */
3645 memcpy(new_args + nb_oargs + nb_iargs,
dd186292 3646 op->args + nb_oargs + nb_iargs,
c896fe29
FB		 3647 sizeof(TCGArg) * def->nb_cargs);
3648
d21369f5
RH		 3649 i_allocated_regs = s->reserved_regs;
3650 o_allocated_regs = s->reserved_regs;
82790a87 3651
c896fe29 3652 /* satisfy input constraints */
dd186292 3653 for (k = 0; k < nb_iargs; k++) {
d62816f2
RH		 3654 TCGRegSet i_preferred_regs, o_preferred_regs;
3655
66792f90 3656 i = def->args_ct[nb_oargs + k].sort_index;
dd186292 3657 arg = op->args[i];
c896fe29 3658 arg_ct = &def->args_ct[i];
43439139 3659 ts = arg_temp(arg);
40ae5c62
RH		 3660
3661 if (ts->val_type == TEMP_VAL_CONST
a4fbbd77 3662 && tcg_target_const_match(ts->val, ts->type, arg_ct->ct)) {
40ae5c62
RH		 3663 /* constant is OK for instruction */
3664 const_args[i] = 1;
3665 new_args[i] = ts->val;
d62816f2 3666 continue;
c896fe29 3667 }
40ae5c62 3668
d62816f2 3669 i_preferred_regs = o_preferred_regs = 0;
bc2b17e6 3670 if (arg_ct->ialias) {
d62816f2 3671 o_preferred_regs = op->output_pref[arg_ct->alias_index];
d62816f2 3672
c0522136
RH		 3673 /*
3674 * If the input is readonly, then it cannot also be an
3675 * output and aliased to itself. If the input is not
3676 * dead after the instruction, we must allocate a new
3677 * register and move it.
3678 */
3679 if (temp_readonly(ts) || !IS_DEAD_ARG(i)) {
3680 goto allocate_in_reg;
3681 }
3682
3683 /*
3684 * Check if the current register has already been allocated
3685 * for another input aliased to an output.
3686 */
3687 if (ts->val_type == TEMP_VAL_REG) {
3688 reg = ts->reg;
3689 for (int k2 = 0; k2 < k; k2++) {
3690 int i2 = def->args_ct[nb_oargs + k2].sort_index;
3691 if (def->args_ct[i2].ialias && reg == new_args[i2]) {
3692 goto allocate_in_reg;
7e1df267
AJ		 3693 }
3694 }
5ff9d6a4 3695 }
c0522136 3696 i_preferred_regs = o_preferred_regs;
c896fe29 3697 }
d62816f2 3698
9be0d080 3699 temp_load(s, ts, arg_ct->regs, i_allocated_regs, i_preferred_regs);
c896fe29 3700 reg = ts->reg;
d62816f2 3701
c0522136
RH		 3702 if (!tcg_regset_test_reg(arg_ct->regs, reg)) {
3703 allocate_in_reg:
3704 /*
3705 * Allocate a new register matching the constraint
3706 * and move the temporary register into it.
3707 */
d62816f2
RH		 3708 temp_load(s, ts, tcg_target_available_regs[ts->type],
3709 i_allocated_regs, 0);
9be0d080 3710 reg = tcg_reg_alloc(s, arg_ct->regs, i_allocated_regs,
d62816f2 3711 o_preferred_regs, ts->indirect_base);
78113e83 3712 if (!tcg_out_mov(s, ts->type, reg, ts->reg)) {
240c08d0
RH		 3713 /*
3714 * Cross register class move not supported. Sync the
3715 * temp back to its slot and load from there.
3716 */
3717 temp_sync(s, ts, i_allocated_regs, 0, 0);
3718 tcg_out_ld(s, ts->type, reg,
3719 ts->mem_base->reg, ts->mem_offset);
78113e83 3720 }
c896fe29 3721 }
c896fe29
FB		 3722 new_args[i] = reg;
3723 const_args[i] = 0;
82790a87 3724 tcg_regset_set_reg(i_allocated_regs, reg);
c896fe29
FB		 3725 }
3726
a52ad07e
AJ		 3727 /* mark dead temporaries and free the associated registers */
3728 for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) {
3729 if (IS_DEAD_ARG(i)) {
43439139 3730 temp_dead(s, arg_temp(op->args[i]));
a52ad07e
AJ		 3731 }
3732 }
3733
b4cb76e6
RH		 3734 if (def->flags & TCG_OPF_COND_BRANCH) {
3735 tcg_reg_alloc_cbranch(s, i_allocated_regs);
3736 } else if (def->flags & TCG_OPF_BB_END) {
82790a87 3737 tcg_reg_alloc_bb_end(s, i_allocated_regs);
e8996ee0 3738 } else {
e8996ee0
FB		 3739 if (def->flags & TCG_OPF_CALL_CLOBBER) {
3740 /* XXX: permit generic clobber register list ? */
c8074023
RH		 3741 for (i = 0; i < TCG_TARGET_NB_REGS; i++) {
3742 if (tcg_regset_test_reg(tcg_target_call_clobber_regs, i)) {
82790a87 3743 tcg_reg_free(s, i, i_allocated_regs);
e8996ee0 3744 }
c896fe29 3745 }
3d5c5f87
AJ		 3746 }
3747 if (def->flags & TCG_OPF_SIDE_EFFECTS) {
3748 /* sync globals if the op has side effects and might trigger
3749 an exception. */
82790a87 3750 sync_globals(s, i_allocated_regs);
c896fe29 3751 }
e8996ee0
FB		 3752
3753 /* satisfy the output constraints */
e8996ee0 3754 for(k = 0; k < nb_oargs; k++) {
66792f90 3755 i = def->args_ct[k].sort_index;
dd186292 3756 arg = op->args[i];
e8996ee0 3757 arg_ct = &def->args_ct[i];
43439139 3758 ts = arg_temp(arg);
d63e3b6e
RH		 3759
3760 /* ENV should not be modified. */
e01fa97d 3761 tcg_debug_assert(!temp_readonly(ts));
d63e3b6e 3762
bc2b17e6 3763 if (arg_ct->oalias && !const_args[arg_ct->alias_index]) {
e8996ee0 3764 reg = new_args[arg_ct->alias_index];
bc2b17e6 3765 } else if (arg_ct->newreg) {
9be0d080 3766 reg = tcg_reg_alloc(s, arg_ct->regs,
82790a87 3767 i_allocated_regs | o_allocated_regs,
69e3706d 3768 op->output_pref[k], ts->indirect_base);
e8996ee0 3769 } else {
9be0d080 3770 reg = tcg_reg_alloc(s, arg_ct->regs, o_allocated_regs,
69e3706d 3771 op->output_pref[k], ts->indirect_base);
c896fe29 3772 }
82790a87 3773 tcg_regset_set_reg(o_allocated_regs, reg);
d63e3b6e
RH		 3774 if (ts->val_type == TEMP_VAL_REG) {
3775 s->reg_to_temp[ts->reg] = NULL;
e8996ee0 3776 }
d63e3b6e
RH		 3777 ts->val_type = TEMP_VAL_REG;
3778 ts->reg = reg;
3779 /*
3780 * Temp value is modified, so the value kept in memory is
3781 * potentially not the same.
3782 */
3783 ts->mem_coherent = 0;
3784 s->reg_to_temp[reg] = ts;
e8996ee0 3785 new_args[i] = reg;
c896fe29 3786 }
c896fe29
FB		 3787 }
3788
c896fe29 3789 /* emit instruction */
d2fd745f
RH		 3790 if (def->flags & TCG_OPF_VECTOR) {
3791 tcg_out_vec_op(s, op->opc, TCGOP_VECL(op), TCGOP_VECE(op),
3792 new_args, const_args);
3793 } else {
3794 tcg_out_op(s, op->opc, new_args, const_args);
3795 }
3796
c896fe29
FB		 3797 /* move the outputs in the correct register if needed */
3798 for(i = 0; i < nb_oargs; i++) {
43439139 3799 ts = arg_temp(op->args[i]);
d63e3b6e
RH		 3800
3801 /* ENV should not be modified. */
e01fa97d 3802 tcg_debug_assert(!temp_readonly(ts));
d63e3b6e 3803
ec7a869d 3804 if (NEED_SYNC_ARG(i)) {
98b4e186 3805 temp_sync(s, ts, o_allocated_regs, 0, IS_DEAD_ARG(i));
59d7c14e 3806 } else if (IS_DEAD_ARG(i)) {
f8bf00f1 3807 temp_dead(s, ts);
ec7a869d 3808 }
c896fe29
FB		 3809 }
3810}
3811
efe86b21
RH		 3812static bool tcg_reg_alloc_dup2(TCGContext *s, const TCGOp *op)
3813{
3814 const TCGLifeData arg_life = op->life;
3815 TCGTemp *ots, *itsl, *itsh;
3816 TCGType vtype = TCGOP_VECL(op) + TCG_TYPE_V64;
3817
3818 /* This opcode is only valid for 32-bit hosts, for 64-bit elements. */
3819 tcg_debug_assert(TCG_TARGET_REG_BITS == 32);
3820 tcg_debug_assert(TCGOP_VECE(op) == MO_64);
3821
3822 ots = arg_temp(op->args[0]);
3823 itsl = arg_temp(op->args[1]);
3824 itsh = arg_temp(op->args[2]);
3825
3826 /* ENV should not be modified. */
3827 tcg_debug_assert(!temp_readonly(ots));
3828
3829 /* Allocate the output register now. */
3830 if (ots->val_type != TEMP_VAL_REG) {
3831 TCGRegSet allocated_regs = s->reserved_regs;
3832 TCGRegSet dup_out_regs =
3833 tcg_op_defs[INDEX_op_dup_vec].args_ct[0].regs;
3834
3835 /* Make sure to not spill the input registers. */
3836 if (!IS_DEAD_ARG(1) && itsl->val_type == TEMP_VAL_REG) {
3837 tcg_regset_set_reg(allocated_regs, itsl->reg);
3838 }
3839 if (!IS_DEAD_ARG(2) && itsh->val_type == TEMP_VAL_REG) {
3840 tcg_regset_set_reg(allocated_regs, itsh->reg);
3841 }
3842
3843 ots->reg = tcg_reg_alloc(s, dup_out_regs, allocated_regs,
3844 op->output_pref[0], ots->indirect_base);
3845 ots->val_type = TEMP_VAL_REG;
3846 ots->mem_coherent = 0;
3847 s->reg_to_temp[ots->reg] = ots;
3848 }
3849
3850 /* Promote dup2 of immediates to dupi_vec. */
3851 if (itsl->val_type == TEMP_VAL_CONST && itsh->val_type == TEMP_VAL_CONST) {
3852 uint64_t val = deposit64(itsl->val, 32, 32, itsh->val);
3853 MemOp vece = MO_64;
3854
3855 if (val == dup_const(MO_8, val)) {
3856 vece = MO_8;
3857 } else if (val == dup_const(MO_16, val)) {
3858 vece = MO_16;
3859 } else if (val == dup_const(MO_32, val)) {
3860 vece = MO_32;
3861 }
3862
3863 tcg_out_dupi_vec(s, vtype, vece, ots->reg, val);
3864 goto done;
3865 }
3866
3867 /* If the two inputs form one 64-bit value, try dupm_vec. */
3868 if (itsl + 1 == itsh && itsl->base_type == TCG_TYPE_I64) {
3869 if (!itsl->mem_coherent) {
3870 temp_sync(s, itsl, s->reserved_regs, 0, 0);
3871 }
3872 if (!itsh->mem_coherent) {
3873 temp_sync(s, itsh, s->reserved_regs, 0, 0);
3874 }
3875#ifdef HOST_WORDS_BIGENDIAN
3876 TCGTemp *its = itsh;
3877#else
3878 TCGTemp *its = itsl;
3879#endif
3880 if (tcg_out_dupm_vec(s, vtype, MO_64, ots->reg,
3881 its->mem_base->reg, its->mem_offset)) {
3882 goto done;
3883 }
3884 }
3885
3886 /* Fall back to generic expansion. */
3887 return false;
3888
3889 done:
3890 if (IS_DEAD_ARG(1)) {
3891 temp_dead(s, itsl);
3892 }
3893 if (IS_DEAD_ARG(2)) {
3894 temp_dead(s, itsh);
3895 }
3896 if (NEED_SYNC_ARG(0)) {
3897 temp_sync(s, ots, s->reserved_regs, 0, IS_DEAD_ARG(0));
3898 } else if (IS_DEAD_ARG(0)) {
3899 temp_dead(s, ots);
3900 }
3901 return true;
3902}
3903
b03cce8e
FB		 3904#ifdef TCG_TARGET_STACK_GROWSUP
3905#define STACK_DIR(x) (-(x))
3906#else
3907#define STACK_DIR(x) (x)
3908#endif
3909
dd186292 3910static void tcg_reg_alloc_call(TCGContext *s, TCGOp *op)
c896fe29 3911{
cd9090aa
RH		 3912 const int nb_oargs = TCGOP_CALLO(op);
3913 const int nb_iargs = TCGOP_CALLI(op);
dd186292 3914 const TCGLifeData arg_life = op->life;
7b7d8b2d 3915 const TCGHelperInfo *info;
b6638662
RH		 3916 int flags, nb_regs, i;
3917 TCGReg reg;
cf066674 3918 TCGArg arg;
c896fe29 3919 TCGTemp *ts;
d3452f1f
RH		 3920 intptr_t stack_offset;
3921 size_t call_stack_size;
cf066674
RH		 3922 tcg_insn_unit *func_addr;
3923 int allocate_args;
c896fe29 3924 TCGRegSet allocated_regs;
c896fe29 3925
fa52e660 3926 func_addr = tcg_call_func(op);
7b7d8b2d
RH		 3927 info = tcg_call_info(op);
3928 flags = info->flags;
c896fe29 3929
6e17d0c5 3930 nb_regs = ARRAY_SIZE(tcg_target_call_iarg_regs);
c45cb8bb
RH		 3931 if (nb_regs > nb_iargs) {
3932 nb_regs = nb_iargs;
cf066674 3933 }
c896fe29
FB		 3934
3935 /* assign stack slots first */
c45cb8bb 3936 call_stack_size = (nb_iargs - nb_regs) * sizeof(tcg_target_long);
c896fe29
FB		 3937 call_stack_size = (call_stack_size + TCG_TARGET_STACK_ALIGN - 1) &
3938 ~(TCG_TARGET_STACK_ALIGN - 1);
b03cce8e
FB		 3939 allocate_args = (call_stack_size > TCG_STATIC_CALL_ARGS_SIZE);
3940 if (allocate_args) {
345649c0
BS		 3941 /* XXX: if more than TCG_STATIC_CALL_ARGS_SIZE is needed,
3942 preallocate call stack */
3943 tcg_abort();
b03cce8e 3944 }
39cf05d3
FB		 3945
3946 stack_offset = TCG_TARGET_CALL_STACK_OFFSET;
dd186292
RH		 3947 for (i = nb_regs; i < nb_iargs; i++) {
3948 arg = op->args[nb_oargs + i];
39cf05d3
FB		 3949#ifdef TCG_TARGET_STACK_GROWSUP
3950 stack_offset -= sizeof(tcg_target_long);
3951#endif
3952 if (arg != TCG_CALL_DUMMY_ARG) {
43439139 3953 ts = arg_temp(arg);
40ae5c62 3954 temp_load(s, ts, tcg_target_available_regs[ts->type],
b722452a 3955 s->reserved_regs, 0);
40ae5c62 3956 tcg_out_st(s, ts->type, ts->reg, TCG_REG_CALL_STACK, stack_offset);
c896fe29 3957 }
39cf05d3
FB		 3958#ifndef TCG_TARGET_STACK_GROWSUP
3959 stack_offset += sizeof(tcg_target_long);
3960#endif
c896fe29
FB		 3961 }
3962
3963 /* assign input registers */
d21369f5 3964 allocated_regs = s->reserved_regs;
dd186292
RH		 3965 for (i = 0; i < nb_regs; i++) {
3966 arg = op->args[nb_oargs + i];
39cf05d3 3967 if (arg != TCG_CALL_DUMMY_ARG) {
43439139 3968 ts = arg_temp(arg);
39cf05d3 3969 reg = tcg_target_call_iarg_regs[i];
40ae5c62 3970
39cf05d3
FB		 3971 if (ts->val_type == TEMP_VAL_REG) {
3972 if (ts->reg != reg) {
4250da10 3973 tcg_reg_free(s, reg, allocated_regs);
78113e83 3974 if (!tcg_out_mov(s, ts->type, reg, ts->reg)) {
240c08d0
RH		 3975 /*
3976 * Cross register class move not supported. Sync the
3977 * temp back to its slot and load from there.
3978 */
3979 temp_sync(s, ts, allocated_regs, 0, 0);
3980 tcg_out_ld(s, ts->type, reg,
3981 ts->mem_base->reg, ts->mem_offset);
78113e83 3982 }
39cf05d3 3983 }
39cf05d3 3984 } else {
ccb1bb66 3985 TCGRegSet arg_set = 0;
40ae5c62 3986
4250da10 3987 tcg_reg_free(s, reg, allocated_regs);
40ae5c62 3988 tcg_regset_set_reg(arg_set, reg);
b722452a 3989 temp_load(s, ts, arg_set, allocated_regs, 0);
c896fe29 3990 }
40ae5c62 3991
39cf05d3 3992 tcg_regset_set_reg(allocated_regs, reg);
c896fe29 3993 }
c896fe29
FB		 3994 }
3995
c896fe29 3996 /* mark dead temporaries and free the associated registers */
dd186292 3997 for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) {
866cb6cb 3998 if (IS_DEAD_ARG(i)) {
43439139 3999 temp_dead(s, arg_temp(op->args[i]));
c896fe29
FB		 4000 }
4001 }
4002
4003 /* clobber call registers */
c8074023
RH		 4004 for (i = 0; i < TCG_TARGET_NB_REGS; i++) {
4005 if (tcg_regset_test_reg(tcg_target_call_clobber_regs, i)) {
b3915dbb 4006 tcg_reg_free(s, i, allocated_regs);
c896fe29
FB		 4007 }
4008 }
78505279
AJ		 4009
4010 /* Save globals if they might be written by the helper, sync them if
4011 they might be read. */
4012 if (flags & TCG_CALL_NO_READ_GLOBALS) {
4013 /* Nothing to do */
4014 } else if (flags & TCG_CALL_NO_WRITE_GLOBALS) {
4015 sync_globals(s, allocated_regs);
4016 } else {
b9c18f56
AJ		 4017 save_globals(s, allocated_regs);
4018 }
c896fe29 4019
7b7d8b2d
RH		 4020#ifdef CONFIG_TCG_INTERPRETER
4021 {
4022 gpointer hash = (gpointer)(uintptr_t)info->typemask;
4023 ffi_cif *cif = g_hash_table_lookup(ffi_table, hash);
4024 assert(cif != NULL);
4025 tcg_out_call(s, func_addr, cif);
4026 }
4027#else
cf066674 4028 tcg_out_call(s, func_addr);
7b7d8b2d 4029#endif
c896fe29
FB		 4030
4031 /* assign output registers and emit moves if needed */
4032 for(i = 0; i < nb_oargs; i++) {
dd186292 4033 arg = op->args[i];
43439139 4034 ts = arg_temp(arg);
d63e3b6e
RH		 4035
4036 /* ENV should not be modified. */
e01fa97d 4037 tcg_debug_assert(!temp_readonly(ts));
d63e3b6e 4038
c896fe29 4039 reg = tcg_target_call_oarg_regs[i];
eabb7b91 4040 tcg_debug_assert(s->reg_to_temp[reg] == NULL);
d63e3b6e
RH		 4041 if (ts->val_type == TEMP_VAL_REG) {
4042 s->reg_to_temp[ts->reg] = NULL;
4043 }
4044 ts->val_type = TEMP_VAL_REG;
4045 ts->reg = reg;
4046 ts->mem_coherent = 0;
4047 s->reg_to_temp[reg] = ts;
4048 if (NEED_SYNC_ARG(i)) {
4049 temp_sync(s, ts, allocated_regs, 0, IS_DEAD_ARG(i));
4050 } else if (IS_DEAD_ARG(i)) {
4051 temp_dead(s, ts);
c896fe29
FB		 4052 }
4053 }
c896fe29
FB		 4054}
4055
4056#ifdef CONFIG_PROFILER
4057
c3fac113
EC		 4058/* avoid copy/paste errors */
4059#define PROF_ADD(to, from, field) \
4060 do { \
d73415a3 4061 (to)->field += qatomic_read(&((from)->field)); \
c3fac113
EC		 4062 } while (0)
4063
4064#define PROF_MAX(to, from, field) \
4065 do { \
d73415a3 4066 typeof((from)->field) val__ = qatomic_read(&((from)->field)); \
c3fac113
EC		 4067 if (val__ > (to)->field) { \
4068 (to)->field = val__; \
4069 } \
4070 } while (0)
4071
4072/* Pass in a zero'ed @prof */
4073static inline
4074void tcg_profile_snapshot(TCGProfile *prof, bool counters, bool table)
4075{
0e2d61cf 4076 unsigned int n_ctxs = qatomic_read(&tcg_cur_ctxs);
c3fac113
EC		 4077 unsigned int i;
4078
3468b59e 4079 for (i = 0; i < n_ctxs; i++) {
d73415a3 4080 TCGContext *s = qatomic_read(&tcg_ctxs[i]);
3468b59e 4081 const TCGProfile *orig = &s->prof;
c3fac113
EC		 4082
4083 if (counters) {
72fd2efb 4084 PROF_ADD(prof, orig, cpu_exec_time);
c3fac113
EC		 4085 PROF_ADD(prof, orig, tb_count1);
4086 PROF_ADD(prof, orig, tb_count);
4087 PROF_ADD(prof, orig, op_count);
4088 PROF_MAX(prof, orig, op_count_max);
4089 PROF_ADD(prof, orig, temp_count);
4090 PROF_MAX(prof, orig, temp_count_max);
4091 PROF_ADD(prof, orig, del_op_count);
4092 PROF_ADD(prof, orig, code_in_len);
4093 PROF_ADD(prof, orig, code_out_len);
4094 PROF_ADD(prof, orig, search_out_len);
4095 PROF_ADD(prof, orig, interm_time);
4096 PROF_ADD(prof, orig, code_time);
4097 PROF_ADD(prof, orig, la_time);
4098 PROF_ADD(prof, orig, opt_time);
4099 PROF_ADD(prof, orig, restore_count);
4100 PROF_ADD(prof, orig, restore_time);
4101 }
4102 if (table) {
4103 int i;
4104
4105 for (i = 0; i < NB_OPS; i++) {
4106 PROF_ADD(prof, orig, table_op_count[i]);
4107 }
4108 }
4109 }
4110}
4111
4112#undef PROF_ADD
4113#undef PROF_MAX
4114
4115static void tcg_profile_snapshot_counters(TCGProfile *prof)
4116{
4117 tcg_profile_snapshot(prof, true, false);
4118}
4119
4120static void tcg_profile_snapshot_table(TCGProfile *prof)
4121{
4122 tcg_profile_snapshot(prof, false, true);
4123}
c896fe29 4124
b6a7f3e0 4125void tcg_dump_op_count(GString *buf)
c896fe29 4126{
c3fac113 4127 TCGProfile prof = {};
c896fe29 4128 int i;
d70724ce 4129
c3fac113 4130 tcg_profile_snapshot_table(&prof);
15fc7daa 4131 for (i = 0; i < NB_OPS; i++) {
b6a7f3e0
DB		 4132 g_string_append_printf(buf, "%s %" PRId64 "\n", tcg_op_defs[i].name,
4133 prof.table_op_count[i]);
c896fe29 4134 }
c896fe29 4135}
72fd2efb
EC		 4136
4137int64_t tcg_cpu_exec_time(void)
4138{
0e2d61cf 4139 unsigned int n_ctxs = qatomic_read(&tcg_cur_ctxs);
72fd2efb
EC		 4140 unsigned int i;
4141 int64_t ret = 0;
4142
4143 for (i = 0; i < n_ctxs; i++) {
d73415a3 4144 const TCGContext *s = qatomic_read(&tcg_ctxs[i]);
72fd2efb
EC		 4145 const TCGProfile *prof = &s->prof;
4146
d73415a3 4147 ret += qatomic_read(&prof->cpu_exec_time);
72fd2efb
EC		 4148 }
4149 return ret;
4150}
246ae24d 4151#else
b6a7f3e0 4152void tcg_dump_op_count(GString *buf)
246ae24d 4153{
b6a7f3e0 4154 g_string_append_printf(buf, "[TCG profiler not compiled]\n");
246ae24d 4155}
72fd2efb
EC		 4156
4157int64_t tcg_cpu_exec_time(void)
4158{
4159 error_report("%s: TCG profiler not compiled", __func__);
4160 exit(EXIT_FAILURE);
4161}
c896fe29
FB		 4162#endif
4163
4164
5bd2ec3d 4165int tcg_gen_code(TCGContext *s, TranslationBlock *tb)
c896fe29 4166{
c3fac113
EC		 4167#ifdef CONFIG_PROFILER
4168 TCGProfile *prof = &s->prof;
4169#endif
15fa08f8
RH		 4170 int i, num_insns;
4171 TCGOp *op;
c896fe29 4172
04fe6400
RH		 4173#ifdef CONFIG_PROFILER
4174 {
c1f543b7 4175 int n = 0;
04fe6400 4176
15fa08f8
RH		 4177 QTAILQ_FOREACH(op, &s->ops, link) {
4178 n++;
4179 }
d73415a3 4180 qatomic_set(&prof->op_count, prof->op_count + n);
c3fac113 4181 if (n > prof->op_count_max) {
d73415a3 4182 qatomic_set(&prof->op_count_max, n);
04fe6400
RH		 4183 }
4184
4185 n = s->nb_temps;
d73415a3 4186 qatomic_set(&prof->temp_count, prof->temp_count + n);
c3fac113 4187 if (n > prof->temp_count_max) {
d73415a3 4188 qatomic_set(&prof->temp_count_max, n);
04fe6400
RH		 4189 }
4190 }
4191#endif
4192
c896fe29 4193#ifdef DEBUG_DISAS
d977e1c2
AB		 4194 if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP)
4195 && qemu_log_in_addr_range(tb->pc))) {
fc59d2d8 4196 FILE *logfile = qemu_log_lock();
93fcfe39 4197 qemu_log("OP:\n");
1894f69a 4198 tcg_dump_ops(s, false);
93fcfe39 4199 qemu_log("\n");
fc59d2d8 4200 qemu_log_unlock(logfile);
c896fe29
FB		 4201 }
4202#endif
4203
bef16ab4
RH		 4204#ifdef CONFIG_DEBUG_TCG
4205 /* Ensure all labels referenced have been emitted. */
4206 {
4207 TCGLabel *l;
4208 bool error = false;
4209
4210 QSIMPLEQ_FOREACH(l, &s->labels, next) {
4211 if (unlikely(!l->present) && l->refs) {
4212 qemu_log_mask(CPU_LOG_TB_OP,
4213 "$L%d referenced but not present.\n", l->id);
4214 error = true;
4215 }
4216 }
4217 assert(!error);
4218 }
4219#endif
4220
c5cc28ff 4221#ifdef CONFIG_PROFILER
d73415a3 4222 qatomic_set(&prof->opt_time, prof->opt_time - profile_getclock());
c5cc28ff
AJ		 4223#endif
4224
8f2e8c07 4225#ifdef USE_TCG_OPTIMIZATIONS
c45cb8bb 4226 tcg_optimize(s);
8f2e8c07
KB		 4227#endif
4228
a23a9ec6 4229#ifdef CONFIG_PROFILER
d73415a3
SH		 4230 qatomic_set(&prof->opt_time, prof->opt_time + profile_getclock());
4231 qatomic_set(&prof->la_time, prof->la_time - profile_getclock());
a23a9ec6 4232#endif
c5cc28ff 4233
b4fc67c7 4234 reachable_code_pass(s);
b83eabea 4235 liveness_pass_1(s);
5a18407f 4236
b83eabea 4237 if (s->nb_indirects > 0) {
5a18407f 4238#ifdef DEBUG_DISAS
b83eabea
RH		 4239 if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP_IND)
4240 && qemu_log_in_addr_range(tb->pc))) {
fc59d2d8 4241 FILE *logfile = qemu_log_lock();
b83eabea 4242 qemu_log("OP before indirect lowering:\n");
1894f69a 4243 tcg_dump_ops(s, false);
b83eabea 4244 qemu_log("\n");
fc59d2d8 4245 qemu_log_unlock(logfile);
b83eabea 4246 }
5a18407f 4247#endif
b83eabea
RH		 4248 /* Replace indirect temps with direct temps. */
4249 if (liveness_pass_2(s)) {
4250 /* If changes were made, re-run liveness. */
4251 liveness_pass_1(s);
5a18407f
RH		 4252 }
4253 }
c5cc28ff 4254
a23a9ec6 4255#ifdef CONFIG_PROFILER
d73415a3 4256 qatomic_set(&prof->la_time, prof->la_time + profile_getclock());
a23a9ec6 4257#endif
c896fe29
FB		 4258
4259#ifdef DEBUG_DISAS
d977e1c2
AB		 4260 if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP_OPT)
4261 && qemu_log_in_addr_range(tb->pc))) {
fc59d2d8 4262 FILE *logfile = qemu_log_lock();
c5cc28ff 4263 qemu_log("OP after optimization and liveness analysis:\n");
1894f69a 4264 tcg_dump_ops(s, true);
93fcfe39 4265 qemu_log("\n");
fc59d2d8 4266 qemu_log_unlock(logfile);
c896fe29
FB		 4267 }
4268#endif
4269
4270 tcg_reg_alloc_start(s);
4271
db0c51a3
RH		 4272 /*
4273 * Reset the buffer pointers when restarting after overflow.
4274 * TODO: Move this into translate-all.c with the rest of the
4275 * buffer management. Having only this done here is confusing.
4276 */
4277 s->code_buf = tcg_splitwx_to_rw(tb->tc.ptr);
4278 s->code_ptr = s->code_buf;
c896fe29 4279
659ef5cb 4280#ifdef TCG_TARGET_NEED_LDST_LABELS
6001f772 4281 QSIMPLEQ_INIT(&s->ldst_labels);
659ef5cb 4282#endif
57a26946
RH		 4283#ifdef TCG_TARGET_NEED_POOL_LABELS
4284 s->pool_labels = NULL;
4285#endif
9ecefc84 4286
fca8a500 4287 num_insns = -1;
15fa08f8 4288 QTAILQ_FOREACH(op, &s->ops, link) {
c45cb8bb 4289 TCGOpcode opc = op->opc;
b3db8758 4290
c896fe29 4291#ifdef CONFIG_PROFILER
d73415a3 4292 qatomic_set(&prof->table_op_count[opc], prof->table_op_count[opc] + 1);
c896fe29 4293#endif
c45cb8bb
RH		 4294
4295 switch (opc) {
c896fe29 4296 case INDEX_op_mov_i32:
c896fe29 4297 case INDEX_op_mov_i64:
d2fd745f 4298 case INDEX_op_mov_vec:
dd186292 4299 tcg_reg_alloc_mov(s, op);
c896fe29 4300 break;
bab1671f
RH		 4301 case INDEX_op_dup_vec:
4302 tcg_reg_alloc_dup(s, op);
4303 break;
765b842a 4304 case INDEX_op_insn_start:
fca8a500 4305 if (num_insns >= 0) {
9f754620
RH		 4306 size_t off = tcg_current_code_size(s);
4307 s->gen_insn_end_off[num_insns] = off;
4308 /* Assert that we do not overflow our stored offset. */
4309 assert(s->gen_insn_end_off[num_insns] == off);
fca8a500
RH		 4310 }
4311 num_insns++;
bad729e2
RH		 4312 for (i = 0; i < TARGET_INSN_START_WORDS; ++i) {
4313 target_ulong a;
4314#if TARGET_LONG_BITS > TCG_TARGET_REG_BITS
efee3746 4315 a = deposit64(op->args[i * 2], 32, 32, op->args[i * 2 + 1]);
bad729e2 4316#else
efee3746 4317 a = op->args[i];
bad729e2 4318#endif
fca8a500 4319 s->gen_insn_data[num_insns][i] = a;
bad729e2 4320 }
c896fe29 4321 break;
5ff9d6a4 4322 case INDEX_op_discard:
43439139 4323 temp_dead(s, arg_temp(op->args[0]));
5ff9d6a4 4324 break;
c896fe29 4325 case INDEX_op_set_label:
e8996ee0 4326 tcg_reg_alloc_bb_end(s, s->reserved_regs);
92ab8e7d 4327 tcg_out_label(s, arg_label(op->args[0]));
c896fe29
FB		 4328 break;
4329 case INDEX_op_call:
dd186292 4330 tcg_reg_alloc_call(s, op);
c45cb8bb 4331 break;
efe86b21
RH		 4332 case INDEX_op_dup2_vec:
4333 if (tcg_reg_alloc_dup2(s, op)) {
4334 break;
4335 }
4336 /* fall through */
c896fe29 4337 default:
25c4d9cc 4338 /* Sanity check that we've not introduced any unhandled opcodes. */
be0f34b5 4339 tcg_debug_assert(tcg_op_supported(opc));
c896fe29
FB		 4340 /* Note: in order to speed up the code, it would be much
4341 faster to have specialized register allocator functions for
4342 some common argument patterns */
dd186292 4343 tcg_reg_alloc_op(s, op);
c896fe29
FB		 4344 break;
4345 }
8d8fdbae 4346#ifdef CONFIG_DEBUG_TCG
c896fe29
FB		 4347 check_regs(s);
4348#endif
b125f9dc
RH		 4349 /* Test for (pending) buffer overflow. The assumption is that any
4350 one operation beginning below the high water mark cannot overrun
4351 the buffer completely. Thus we can test for overflow after
4352 generating code without having to check during generation. */
644da9b3 4353 if (unlikely((void *)s->code_ptr > s->code_gen_highwater)) {
b125f9dc
RH		 4354 return -1;
4355 }
6e6c4efe
RH		 4356 /* Test for TB overflow, as seen by gen_insn_end_off. */
4357 if (unlikely(tcg_current_code_size(s) > UINT16_MAX)) {
4358 return -2;
4359 }
c896fe29 4360 }
fca8a500
RH		 4361 tcg_debug_assert(num_insns >= 0);
4362 s->gen_insn_end_off[num_insns] = tcg_current_code_size(s);
c45cb8bb 4363
b76f0d8c 4364 /* Generate TB finalization at the end of block */
659ef5cb 4365#ifdef TCG_TARGET_NEED_LDST_LABELS
aeee05f5
RH		 4366 i = tcg_out_ldst_finalize(s);
4367 if (i < 0) {
4368 return i;
23dceda6 4369 }
659ef5cb 4370#endif
57a26946 4371#ifdef TCG_TARGET_NEED_POOL_LABELS
1768987b
RH		 4372 i = tcg_out_pool_finalize(s);
4373 if (i < 0) {
4374 return i;
57a26946
RH		 4375 }
4376#endif
7ecd02a0
RH		 4377 if (!tcg_resolve_relocs(s)) {
4378 return -2;
4379 }
c896fe29 4380
df5d2b16 4381#ifndef CONFIG_TCG_INTERPRETER
c896fe29 4382 /* flush instruction cache */
db0c51a3
RH		 4383 flush_idcache_range((uintptr_t)tcg_splitwx_to_rx(s->code_buf),
4384 (uintptr_t)s->code_buf,
1da8de39 4385 tcg_ptr_byte_diff(s->code_ptr, s->code_buf));
df5d2b16 4386#endif
2aeabc08 4387
1813e175 4388 return tcg_current_code_size(s);
c896fe29
FB		 4389}
4390
a23a9ec6 4391#ifdef CONFIG_PROFILER
3a841ab5 4392void tcg_dump_info(GString *buf)
a23a9ec6 4393{
c3fac113
EC		 4394 TCGProfile prof = {};
4395 const TCGProfile *s;
4396 int64_t tb_count;
4397 int64_t tb_div_count;
4398 int64_t tot;
4399
4400 tcg_profile_snapshot_counters(&prof);
4401 s = &prof;
4402 tb_count = s->tb_count;
4403 tb_div_count = tb_count ? tb_count : 1;
4404 tot = s->interm_time + s->code_time;
a23a9ec6 4405
3a841ab5
DB		 4406 g_string_append_printf(buf, "JIT cycles %" PRId64
4407 " (%0.3f s at 2.4 GHz)\n",
4408 tot, tot / 2.4e9);
4409 g_string_append_printf(buf, "translated TBs %" PRId64
4410 " (aborted=%" PRId64 " %0.1f%%)\n",
4411 tb_count, s->tb_count1 - tb_count,
4412 (double)(s->tb_count1 - s->tb_count)
4413 / (s->tb_count1 ? s->tb_count1 : 1) * 100.0);
4414 g_string_append_printf(buf, "avg ops/TB %0.1f max=%d\n",
4415 (double)s->op_count / tb_div_count, s->op_count_max);
4416 g_string_append_printf(buf, "deleted ops/TB %0.2f\n",
4417 (double)s->del_op_count / tb_div_count);
4418 g_string_append_printf(buf, "avg temps/TB %0.2f max=%d\n",
4419 (double)s->temp_count / tb_div_count,
4420 s->temp_count_max);
4421 g_string_append_printf(buf, "avg host code/TB %0.1f\n",
4422 (double)s->code_out_len / tb_div_count);
4423 g_string_append_printf(buf, "avg search data/TB %0.1f\n",
4424 (double)s->search_out_len / tb_div_count);
a23a9ec6 4425
3a841ab5
DB		 4426 g_string_append_printf(buf, "cycles/op %0.1f\n",
4427 s->op_count ? (double)tot / s->op_count : 0);
4428 g_string_append_printf(buf, "cycles/in byte %0.1f\n",
4429 s->code_in_len ? (double)tot / s->code_in_len : 0);
4430 g_string_append_printf(buf, "cycles/out byte %0.1f\n",
4431 s->code_out_len ? (double)tot / s->code_out_len : 0);
4432 g_string_append_printf(buf, "cycles/search byte %0.1f\n",
4433 s->search_out_len ?
4434 (double)tot / s->search_out_len : 0);
fca8a500 4435 if (tot == 0) {
a23a9ec6 4436 tot = 1;
fca8a500 4437 }
3a841ab5
DB		 4438 g_string_append_printf(buf, " gen_interm time %0.1f%%\n",
4439 (double)s->interm_time / tot * 100.0);
4440 g_string_append_printf(buf, " gen_code time %0.1f%%\n",
4441 (double)s->code_time / tot * 100.0);
4442 g_string_append_printf(buf, "optim./code time %0.1f%%\n",
4443 (double)s->opt_time / (s->code_time ?
4444 s->code_time : 1)
4445 * 100.0);
4446 g_string_append_printf(buf, "liveness/code time %0.1f%%\n",
4447 (double)s->la_time / (s->code_time ?
4448 s->code_time : 1) * 100.0);
4449 g_string_append_printf(buf, "cpu_restore count %" PRId64 "\n",
4450 s->restore_count);
4451 g_string_append_printf(buf, " avg cycles %0.1f\n",
4452 s->restore_count ?
4453 (double)s->restore_time / s->restore_count : 0);
a23a9ec6
FB		 4454}
4455#else
3a841ab5 4456void tcg_dump_info(GString *buf)
a23a9ec6 4457{
3a841ab5 4458 g_string_append_printf(buf, "[TCG profiler not compiled]\n");
a23a9ec6
FB		 4459}
4460#endif
813da627
RH		 4461
4462#ifdef ELF_HOST_MACHINE
5872bbf2
RH		 4463/* In order to use this feature, the backend needs to do three things:
4464
4465 (1) Define ELF_HOST_MACHINE to indicate both what value to
4466 put into the ELF image and to indicate support for the feature.
4467
4468 (2) Define tcg_register_jit. This should create a buffer containing
4469 the contents of a .debug_frame section that describes the post-
4470 prologue unwind info for the tcg machine.
4471
4472 (3) Call tcg_register_jit_int, with the constructed .debug_frame.
4473*/
813da627
RH		 4474
4475/* Begin GDB interface. THE FOLLOWING MUST MATCH GDB DOCS. */
4476typedef enum {
4477 JIT_NOACTION = 0,
4478 JIT_REGISTER_FN,
4479 JIT_UNREGISTER_FN
4480} jit_actions_t;
4481
4482struct jit_code_entry {
4483 struct jit_code_entry *next_entry;
4484 struct jit_code_entry *prev_entry;
4485 const void *symfile_addr;
4486 uint64_t symfile_size;
4487};
4488
4489struct jit_descriptor {
4490 uint32_t version;
4491 uint32_t action_flag;
4492 struct jit_code_entry *relevant_entry;
4493 struct jit_code_entry *first_entry;
4494};
4495
4496void __jit_debug_register_code(void) __attribute__((noinline));
4497void __jit_debug_register_code(void)
4498{
4499 asm("");
4500}
4501
4502/* Must statically initialize the version, because GDB may check
4503 the version before we can set it. */
4504struct jit_descriptor __jit_debug_descriptor = { 1, 0, 0, 0 };
4505
4506/* End GDB interface. */
4507
4508static int find_string(const char *strtab, const char *str)
4509{
4510 const char *p = strtab + 1;
4511
4512 while (1) {
4513 if (strcmp(p, str) == 0) {
4514 return p - strtab;
4515 }
4516 p += strlen(p) + 1;
4517 }
4518}
4519
755bf9e5 4520static void tcg_register_jit_int(const void *buf_ptr, size_t buf_size,
2c90784a
RH		 4521 const void *debug_frame,
4522 size_t debug_frame_size)
813da627 4523{
5872bbf2
RH		 4524 struct __attribute__((packed)) DebugInfo {
4525 uint32_t len;
4526 uint16_t version;
4527 uint32_t abbrev;
4528 uint8_t ptr_size;
4529 uint8_t cu_die;
4530 uint16_t cu_lang;
4531 uintptr_t cu_low_pc;
4532 uintptr_t cu_high_pc;
4533 uint8_t fn_die;
4534 char fn_name[16];
4535 uintptr_t fn_low_pc;
4536 uintptr_t fn_high_pc;
4537 uint8_t cu_eoc;
4538 };
813da627
RH		 4539
4540 struct ElfImage {
4541 ElfW(Ehdr) ehdr;
4542 ElfW(Phdr) phdr;
5872bbf2
RH		 4543 ElfW(Shdr) shdr[7];
4544 ElfW(Sym) sym[2];
4545 struct DebugInfo di;
4546 uint8_t da[24];
4547 char str[80];
4548 };
4549
4550 struct ElfImage *img;
4551
4552 static const struct ElfImage img_template = {
4553 .ehdr = {
4554 .e_ident[EI_MAG0] = ELFMAG0,
4555 .e_ident[EI_MAG1] = ELFMAG1,
4556 .e_ident[EI_MAG2] = ELFMAG2,
4557 .e_ident[EI_MAG3] = ELFMAG3,
4558 .e_ident[EI_CLASS] = ELF_CLASS,
4559 .e_ident[EI_DATA] = ELF_DATA,
4560 .e_ident[EI_VERSION] = EV_CURRENT,
4561 .e_type = ET_EXEC,
4562 .e_machine = ELF_HOST_MACHINE,
4563 .e_version = EV_CURRENT,
4564 .e_phoff = offsetof(struct ElfImage, phdr),
4565 .e_shoff = offsetof(struct ElfImage, shdr),
4566 .e_ehsize = sizeof(ElfW(Shdr)),
4567 .e_phentsize = sizeof(ElfW(Phdr)),
4568 .e_phnum = 1,
4569 .e_shentsize = sizeof(ElfW(Shdr)),
4570 .e_shnum = ARRAY_SIZE(img->shdr),
4571 .e_shstrndx = ARRAY_SIZE(img->shdr) - 1,
abbb3eae
RH		 4572#ifdef ELF_HOST_FLAGS
4573 .e_flags = ELF_HOST_FLAGS,
4574#endif
4575#ifdef ELF_OSABI
4576 .e_ident[EI_OSABI] = ELF_OSABI,
4577#endif
5872bbf2
RH		 4578 },
4579 .phdr = {
4580 .p_type = PT_LOAD,
4581 .p_flags = PF_X,
4582 },
4583 .shdr = {
4584 [0] = { .sh_type = SHT_NULL },
4585 /* Trick: The contents of code_gen_buffer are not present in
4586 this fake ELF file; that got allocated elsewhere. Therefore
4587 we mark .text as SHT_NOBITS (similar to .bss) so that readers
4588 will not look for contents. We can record any address. */
4589 [1] = { /* .text */
4590 .sh_type = SHT_NOBITS,
4591 .sh_flags = SHF_EXECINSTR | SHF_ALLOC,
4592 },
4593 [2] = { /* .debug_info */
4594 .sh_type = SHT_PROGBITS,
4595 .sh_offset = offsetof(struct ElfImage, di),
4596 .sh_size = sizeof(struct DebugInfo),
4597 },
4598 [3] = { /* .debug_abbrev */
4599 .sh_type = SHT_PROGBITS,
4600 .sh_offset = offsetof(struct ElfImage, da),
4601 .sh_size = sizeof(img->da),
4602 },
4603 [4] = { /* .debug_frame */
4604 .sh_type = SHT_PROGBITS,
4605 .sh_offset = sizeof(struct ElfImage),
4606 },
4607 [5] = { /* .symtab */
4608 .sh_type = SHT_SYMTAB,
4609 .sh_offset = offsetof(struct ElfImage, sym),
4610 .sh_size = sizeof(img->sym),
4611 .sh_info = 1,
4612 .sh_link = ARRAY_SIZE(img->shdr) - 1,
4613 .sh_entsize = sizeof(ElfW(Sym)),
4614 },
4615 [6] = { /* .strtab */
4616 .sh_type = SHT_STRTAB,
4617 .sh_offset = offsetof(struct ElfImage, str),
4618 .sh_size = sizeof(img->str),
4619 }
4620 },
4621 .sym = {
4622 [1] = { /* code_gen_buffer */
4623 .st_info = ELF_ST_INFO(STB_GLOBAL, STT_FUNC),
4624 .st_shndx = 1,
4625 }
4626 },
4627 .di = {
4628 .len = sizeof(struct DebugInfo) - 4,
4629 .version = 2,
4630 .ptr_size = sizeof(void *),
4631 .cu_die = 1,
4632 .cu_lang = 0x8001, /* DW_LANG_Mips_Assembler */
4633 .fn_die = 2,
4634 .fn_name = "code_gen_buffer"
4635 },
4636 .da = {
4637 1, /* abbrev number (the cu) */
4638 0x11, 1, /* DW_TAG_compile_unit, has children */
4639 0x13, 0x5, /* DW_AT_language, DW_FORM_data2 */
4640 0x11, 0x1, /* DW_AT_low_pc, DW_FORM_addr */
4641 0x12, 0x1, /* DW_AT_high_pc, DW_FORM_addr */
4642 0, 0, /* end of abbrev */
4643 2, /* abbrev number (the fn) */
4644 0x2e, 0, /* DW_TAG_subprogram, no children */
4645 0x3, 0x8, /* DW_AT_name, DW_FORM_string */
4646 0x11, 0x1, /* DW_AT_low_pc, DW_FORM_addr */
4647 0x12, 0x1, /* DW_AT_high_pc, DW_FORM_addr */
4648 0, 0, /* end of abbrev */
4649 0 /* no more abbrev */
4650 },
4651 .str = "\0" ".text\0" ".debug_info\0" ".debug_abbrev\0"
4652 ".debug_frame\0" ".symtab\0" ".strtab\0" "code_gen_buffer",
813da627
RH		 4653 };
4654
4655 /* We only need a single jit entry; statically allocate it. */
4656 static struct jit_code_entry one_entry;
4657
5872bbf2 4658 uintptr_t buf = (uintptr_t)buf_ptr;
813da627 4659 size_t img_size = sizeof(struct ElfImage) + debug_frame_size;
2c90784a 4660 DebugFrameHeader *dfh;
813da627 4661
5872bbf2
RH		 4662 img = g_malloc(img_size);
4663 *img = img_template;
813da627 4664
5872bbf2
RH		 4665 img->phdr.p_vaddr = buf;
4666 img->phdr.p_paddr = buf;
4667 img->phdr.p_memsz = buf_size;
813da627 4668
813da627 4669 img->shdr[1].sh_name = find_string(img->str, ".text");
5872bbf2 4670 img->shdr[1].sh_addr = buf;
813da627
RH		 4671 img->shdr[1].sh_size = buf_size;
4672
5872bbf2
RH		 4673 img->shdr[2].sh_name = find_string(img->str, ".debug_info");
4674 img->shdr[3].sh_name = find_string(img->str, ".debug_abbrev");
4675
4676 img->shdr[4].sh_name = find_string(img->str, ".debug_frame");
4677 img->shdr[4].sh_size = debug_frame_size;
4678
4679 img->shdr[5].sh_name = find_string(img->str, ".symtab");
4680 img->shdr[6].sh_name = find_string(img->str, ".strtab");
4681
4682 img->sym[1].st_name = find_string(img->str, "code_gen_buffer");
4683 img->sym[1].st_value = buf;
4684 img->sym[1].st_size = buf_size;
813da627 4685
5872bbf2 4686 img->di.cu_low_pc = buf;
45aba097 4687 img->di.cu_high_pc = buf + buf_size;
5872bbf2 4688 img->di.fn_low_pc = buf;
45aba097 4689 img->di.fn_high_pc = buf + buf_size;
813da627 4690
2c90784a
RH		 4691 dfh = (DebugFrameHeader *)(img + 1);
4692 memcpy(dfh, debug_frame, debug_frame_size);
4693 dfh->fde.func_start = buf;
4694 dfh->fde.func_len = buf_size;
4695
813da627
RH		 4696#ifdef DEBUG_JIT
4697 /* Enable this block to be able to debug the ELF image file creation.
4698 One can use readelf, objdump, or other inspection utilities. */
4699 {
4700 FILE *f = fopen("/tmp/qemu.jit", "w+b");
4701 if (f) {
5872bbf2 4702 if (fwrite(img, img_size, 1, f) != img_size) {
813da627
RH		 4703 /* Avoid stupid unused return value warning for fwrite. */
4704 }
4705 fclose(f);
4706 }
4707 }
4708#endif
4709
4710 one_entry.symfile_addr = img;
4711 one_entry.symfile_size = img_size;
4712
4713 __jit_debug_descriptor.action_flag = JIT_REGISTER_FN;
4714 __jit_debug_descriptor.relevant_entry = &one_entry;
4715 __jit_debug_descriptor.first_entry = &one_entry;
4716 __jit_debug_register_code();
4717}
4718#else
5872bbf2
RH		 4719/* No support for the feature. Provide the entry point expected by exec.c,
4720 and implement the internal function we declared earlier. */
813da627 4721
755bf9e5 4722static void tcg_register_jit_int(const void *buf, size_t size,
2c90784a
RH		 4723 const void *debug_frame,
4724 size_t debug_frame_size)
813da627
RH		 4725{
4726}
4727
755bf9e5 4728void tcg_register_jit(const void *buf, size_t buf_size)
813da627
RH		 4729{
4730}
4731#endif /* ELF_HOST_MACHINE */
db432672
RH		 4732
4733#if !TCG_TARGET_MAYBE_vec
4734void tcg_expand_vec_op(TCGOpcode o, TCGType t, unsigned e, TCGArg a0, ...)
4735{
4736 g_assert_not_reached();
4737}
4738#endif
Empty description
This page took 1.883501 seconds and 4 git commands to generate.