]> Git Repo - qemu.git/blame - tcg/tcg.c
projects / qemu.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
tcg: Fix tcg_reg_alloc_dup*
[qemu.git] / tcg / tcg.c
CommitLineData
c896fe29
FB		 1/*
2 * Tiny Code Generator for QEMU
3 *
4 * Copyright (c) 2008 Fabrice Bellard
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a copy
7 * of this software and associated documentation files (the "Software"), to deal
8 * in the Software without restriction, including without limitation the rights
9 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10 * copies of the Software, and to permit persons to whom the Software is
11 * furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
22 * THE SOFTWARE.
23 */
24
c896fe29 25/* define it to use liveness analysis (better code) */
8f2e8c07 26#define USE_TCG_OPTIMIZATIONS
c896fe29 27
757e725b 28#include "qemu/osdep.h"
cca82982 29
813da627
RH		 30/* Define to jump the ELF file used to communicate with GDB. */
31#undef DEBUG_JIT
32
72fd2efb 33#include "qemu/error-report.h"
f348b6d1 34#include "qemu/cutils.h"
1de7afc9 35#include "qemu/host-utils.h"
d4c51a0a 36#include "qemu/qemu-print.h"
1de7afc9 37#include "qemu/timer.h"
084cfca1 38#include "qemu/cacheflush.h"
ad768e6f 39#include "qemu/cacheinfo.h"
c896fe29 40
c5d3c498 41/* Note: the long term plan is to reduce the dependencies on the QEMU
c896fe29
FB		 42 CPU definitions. Currently they are used for qemu_ld/st
43 instructions */
44#define NO_CPU_IO_DEFS
c896fe29 45
63c91552 46#include "exec/exec-all.h"
dcb32f1d 47#include "tcg/tcg-op.h"
813da627 48
edee2579 49#if UINTPTR_MAX == UINT32_MAX
813da627 50# define ELF_CLASS ELFCLASS32
edee2579
RH		 51#else
52# define ELF_CLASS ELFCLASS64
813da627 53#endif
e03b5686 54#if HOST_BIG_ENDIAN
813da627
RH		 55# define ELF_DATA ELFDATA2MSB
56#else
57# define ELF_DATA ELFDATA2LSB
58#endif
59
c896fe29 60#include "elf.h"
508127e2 61#include "exec/log.h"
d2ba8026 62#include "tcg/tcg-ldst.h"
5ff7258c 63#include "tcg-internal.h"
c896fe29 64
22f15579
RH		 65#ifdef CONFIG_TCG_INTERPRETER
66#include <ffi.h>
67#endif
68
139c1837 69/* Forward declarations for functions declared in tcg-target.c.inc and
ce151109 70 used here. */
e4d58b41
RH		 71static void tcg_target_init(TCGContext *s);
72static void tcg_target_qemu_prologue(TCGContext *s);
6ac17786 73static bool patch_reloc(tcg_insn_unit *code_ptr, int type,
2ba7fae2 74 intptr_t value, intptr_t addend);
c896fe29 75
497a22eb
RH		 76/* The CIE and FDE header definitions will be common to all hosts. */
77typedef struct {
78 uint32_t len __attribute__((aligned((sizeof(void *)))));
79 uint32_t id;
80 uint8_t version;
81 char augmentation[1];
82 uint8_t code_align;
83 uint8_t data_align;
84 uint8_t return_column;
85} DebugFrameCIE;
86
87typedef struct QEMU_PACKED {
88 uint32_t len __attribute__((aligned((sizeof(void *)))));
89 uint32_t cie_offset;
edee2579
RH		 90 uintptr_t func_start;
91 uintptr_t func_len;
497a22eb
RH		 92} DebugFrameFDEHeader;
93
2c90784a
RH		 94typedef struct QEMU_PACKED {
95 DebugFrameCIE cie;
96 DebugFrameFDEHeader fde;
97} DebugFrameHeader;
98
755bf9e5 99static void tcg_register_jit_int(const void *buf, size_t size,
2c90784a
RH		 100 const void *debug_frame,
101 size_t debug_frame_size)
813da627
RH		 102 __attribute__((unused));
103
139c1837 104/* Forward declarations for functions declared and used in tcg-target.c.inc. */
2a534aff 105static void tcg_out_ld(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg1,
a05b5b9b 106 intptr_t arg2);
78113e83 107static bool tcg_out_mov(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg);
c0ad3001 108static void tcg_out_movi(TCGContext *s, TCGType type,
2a534aff 109 TCGReg ret, tcg_target_long arg);
5e8892db
MR		 110static void tcg_out_op(TCGContext *s, TCGOpcode opc,
111 const TCGArg args[TCG_MAX_OP_ARGS],
112 const int const_args[TCG_MAX_OP_ARGS]);
d2fd745f 113#if TCG_TARGET_MAYBE_vec
e7632cfa
RH		 114static bool tcg_out_dup_vec(TCGContext *s, TCGType type, unsigned vece,
115 TCGReg dst, TCGReg src);
d6ecb4a9
RH		 116static bool tcg_out_dupm_vec(TCGContext *s, TCGType type, unsigned vece,
117 TCGReg dst, TCGReg base, intptr_t offset);
4e186175
RH		 118static void tcg_out_dupi_vec(TCGContext *s, TCGType type, unsigned vece,
119 TCGReg dst, int64_t arg);
5e8892db
MR		 120static void tcg_out_vec_op(TCGContext *s, TCGOpcode opc,
121 unsigned vecl, unsigned vece,
122 const TCGArg args[TCG_MAX_OP_ARGS],
123 const int const_args[TCG_MAX_OP_ARGS]);
d2fd745f 124#else
e7632cfa
RH		 125static inline bool tcg_out_dup_vec(TCGContext *s, TCGType type, unsigned vece,
126 TCGReg dst, TCGReg src)
127{
128 g_assert_not_reached();
129}
d6ecb4a9
RH		 130static inline bool tcg_out_dupm_vec(TCGContext *s, TCGType type, unsigned vece,
131 TCGReg dst, TCGReg base, intptr_t offset)
132{
133 g_assert_not_reached();
134}
4e186175
RH		 135static inline void tcg_out_dupi_vec(TCGContext *s, TCGType type, unsigned vece,
136 TCGReg dst, int64_t arg)
e7632cfa
RH		 137{
138 g_assert_not_reached();
139}
5e8892db
MR		 140static inline void tcg_out_vec_op(TCGContext *s, TCGOpcode opc,
141 unsigned vecl, unsigned vece,
142 const TCGArg args[TCG_MAX_OP_ARGS],
143 const int const_args[TCG_MAX_OP_ARGS])
d2fd745f
RH		 144{
145 g_assert_not_reached();
146}
147#endif
2a534aff 148static void tcg_out_st(TCGContext *s, TCGType type, TCGReg arg, TCGReg arg1,
a05b5b9b 149 intptr_t arg2);
59d7c14e
RH		 150static bool tcg_out_sti(TCGContext *s, TCGType type, TCGArg val,
151 TCGReg base, intptr_t ofs);
7b7d8b2d
RH		 152#ifdef CONFIG_TCG_INTERPRETER
153static void tcg_out_call(TCGContext *s, const tcg_insn_unit *target,
154 ffi_cif *cif);
155#else
2be7d76b 156static void tcg_out_call(TCGContext *s, const tcg_insn_unit *target);
7b7d8b2d 157#endif
a4fbbd77 158static bool tcg_target_const_match(int64_t val, TCGType type, int ct);
659ef5cb 159#ifdef TCG_TARGET_NEED_LDST_LABELS
aeee05f5 160static int tcg_out_ldst_finalize(TCGContext *s);
659ef5cb 161#endif
c896fe29 162
42eb6dfc
RH		 163TCGContext tcg_init_ctx;
164__thread TCGContext *tcg_ctx;
165
5ff7258c 166TCGContext **tcg_ctxs;
0e2d61cf
RH		 167unsigned int tcg_cur_ctxs;
168unsigned int tcg_max_ctxs;
1c2adb95 169TCGv_env cpu_env = 0;
c8bc1168 170const void *tcg_code_gen_epilogue;
db0c51a3 171uintptr_t tcg_splitwx_diff;
df2cce29 172
b91ccb31
RH		 173#ifndef CONFIG_TCG_INTERPRETER
174tcg_prologue_fn *tcg_qemu_tb_exec;
175#endif
176
d2fd745f 177static TCGRegSet tcg_target_available_regs[TCG_TYPE_COUNT];
b1d8e52e 178static TCGRegSet tcg_target_call_clobber_regs;
c896fe29 179
1813e175 180#if TCG_TARGET_INSN_UNIT_SIZE == 1
4196dca6 181static __attribute__((unused)) inline void tcg_out8(TCGContext *s, uint8_t v)
c896fe29
FB		 182{
183 *s->code_ptr++ = v;
184}
185
4196dca6
PM		 186static __attribute__((unused)) inline void tcg_patch8(tcg_insn_unit *p,
187 uint8_t v)
5c53bb81 188{
1813e175 189 *p = v;
5c53bb81 190}
1813e175 191#endif
5c53bb81 192
1813e175 193#if TCG_TARGET_INSN_UNIT_SIZE <= 2
4196dca6 194static __attribute__((unused)) inline void tcg_out16(TCGContext *s, uint16_t v)
c896fe29 195{
1813e175
RH		 196 if (TCG_TARGET_INSN_UNIT_SIZE == 2) {
197 *s->code_ptr++ = v;
198 } else {
199 tcg_insn_unit *p = s->code_ptr;
200 memcpy(p, &v, sizeof(v));
201 s->code_ptr = p + (2 / TCG_TARGET_INSN_UNIT_SIZE);
202 }
c896fe29
FB		 203}
204
4196dca6
PM		 205static __attribute__((unused)) inline void tcg_patch16(tcg_insn_unit *p,
206 uint16_t v)
5c53bb81 207{
1813e175
RH		 208 if (TCG_TARGET_INSN_UNIT_SIZE == 2) {
209 *p = v;
210 } else {
211 memcpy(p, &v, sizeof(v));
212 }
5c53bb81 213}
1813e175 214#endif
5c53bb81 215
1813e175 216#if TCG_TARGET_INSN_UNIT_SIZE <= 4
4196dca6 217static __attribute__((unused)) inline void tcg_out32(TCGContext *s, uint32_t v)
c896fe29 218{
1813e175
RH		 219 if (TCG_TARGET_INSN_UNIT_SIZE == 4) {
220 *s->code_ptr++ = v;
221 } else {
222 tcg_insn_unit *p = s->code_ptr;
223 memcpy(p, &v, sizeof(v));
224 s->code_ptr = p + (4 / TCG_TARGET_INSN_UNIT_SIZE);
225 }
c896fe29
FB		 226}
227
4196dca6
PM		 228static __attribute__((unused)) inline void tcg_patch32(tcg_insn_unit *p,
229 uint32_t v)
5c53bb81 230{
1813e175
RH		 231 if (TCG_TARGET_INSN_UNIT_SIZE == 4) {
232 *p = v;
233 } else {
234 memcpy(p, &v, sizeof(v));
235 }
5c53bb81 236}
1813e175 237#endif
5c53bb81 238
1813e175 239#if TCG_TARGET_INSN_UNIT_SIZE <= 8
4196dca6 240static __attribute__((unused)) inline void tcg_out64(TCGContext *s, uint64_t v)
ac26eb69 241{
1813e175
RH		 242 if (TCG_TARGET_INSN_UNIT_SIZE == 8) {
243 *s->code_ptr++ = v;
244 } else {
245 tcg_insn_unit *p = s->code_ptr;
246 memcpy(p, &v, sizeof(v));
247 s->code_ptr = p + (8 / TCG_TARGET_INSN_UNIT_SIZE);
248 }
ac26eb69
RH		 249}
250
4196dca6
PM		 251static __attribute__((unused)) inline void tcg_patch64(tcg_insn_unit *p,
252 uint64_t v)
5c53bb81 253{
1813e175
RH		 254 if (TCG_TARGET_INSN_UNIT_SIZE == 8) {
255 *p = v;
256 } else {
257 memcpy(p, &v, sizeof(v));
258 }
5c53bb81 259}
1813e175 260#endif
5c53bb81 261
c896fe29
FB		 262/* label relocation processing */
263
1813e175 264static void tcg_out_reloc(TCGContext *s, tcg_insn_unit *code_ptr, int type,
bec16311 265 TCGLabel *l, intptr_t addend)
c896fe29 266{
7ecd02a0 267 TCGRelocation *r = tcg_malloc(sizeof(TCGRelocation));
c896fe29 268
7ecd02a0
RH		 269 r->type = type;
270 r->ptr = code_ptr;
271 r->addend = addend;
272 QSIMPLEQ_INSERT_TAIL(&l->relocs, r, next);
c896fe29
FB		 273}
274
92ab8e7d 275static void tcg_out_label(TCGContext *s, TCGLabel *l)
c896fe29 276{
eabb7b91 277 tcg_debug_assert(!l->has_value);
c896fe29 278 l->has_value = 1;
92ab8e7d 279 l->u.value_ptr = tcg_splitwx_to_rx(s->code_ptr);
c896fe29
FB		 280}
281
42a268c2 282TCGLabel *gen_new_label(void)
c896fe29 283{
b1311c4a 284 TCGContext *s = tcg_ctx;
51e3972c 285 TCGLabel *l = tcg_malloc(sizeof(TCGLabel));
c896fe29 286
7ecd02a0
RH		 287 memset(l, 0, sizeof(TCGLabel));
288 l->id = s->nb_labels++;
289 QSIMPLEQ_INIT(&l->relocs);
290
bef16ab4 291 QSIMPLEQ_INSERT_TAIL(&s->labels, l, next);
42a268c2
RH		 292
293 return l;
c896fe29
FB		 294}
295
7ecd02a0
RH		 296static bool tcg_resolve_relocs(TCGContext *s)
297{
298 TCGLabel *l;
299
300 QSIMPLEQ_FOREACH(l, &s->labels, next) {
301 TCGRelocation *r;
302 uintptr_t value = l->u.value;
303
304 QSIMPLEQ_FOREACH(r, &l->relocs, next) {
305 if (!patch_reloc(r->ptr, r->type, value, r->addend)) {
306 return false;
307 }
308 }
309 }
310 return true;
311}
312
9f754620
RH		 313static void set_jmp_reset_offset(TCGContext *s, int which)
314{
f14bed3f
RH		 315 /*
316 * We will check for overflow at the end of the opcode loop in
317 * tcg_gen_code, where we bound tcg_current_code_size to UINT16_MAX.
318 */
319 s->tb_jmp_reset_offset[which] = tcg_current_code_size(s);
9f754620
RH		 320}
321
db6b7d0c 322/* Signal overflow, starting over with fewer guest insns. */
8905770b
MAL		 323static G_NORETURN
324void tcg_raise_tb_overflow(TCGContext *s)
db6b7d0c
RH		 325{
326 siglongjmp(s->jmp_trans, -2);
327}
328
4c22e840
RH		 329#define C_PFX1(P, A) P##A
330#define C_PFX2(P, A, B) P##A##_##B
331#define C_PFX3(P, A, B, C) P##A##_##B##_##C
332#define C_PFX4(P, A, B, C, D) P##A##_##B##_##C##_##D
333#define C_PFX5(P, A, B, C, D, E) P##A##_##B##_##C##_##D##_##E
334#define C_PFX6(P, A, B, C, D, E, F) P##A##_##B##_##C##_##D##_##E##_##F
335
336/* Define an enumeration for the various combinations. */
337
338#define C_O0_I1(I1) C_PFX1(c_o0_i1_, I1),
339#define C_O0_I2(I1, I2) C_PFX2(c_o0_i2_, I1, I2),
340#define C_O0_I3(I1, I2, I3) C_PFX3(c_o0_i3_, I1, I2, I3),
341#define C_O0_I4(I1, I2, I3, I4) C_PFX4(c_o0_i4_, I1, I2, I3, I4),
342
343#define C_O1_I1(O1, I1) C_PFX2(c_o1_i1_, O1, I1),
344#define C_O1_I2(O1, I1, I2) C_PFX3(c_o1_i2_, O1, I1, I2),
345#define C_O1_I3(O1, I1, I2, I3) C_PFX4(c_o1_i3_, O1, I1, I2, I3),
346#define C_O1_I4(O1, I1, I2, I3, I4) C_PFX5(c_o1_i4_, O1, I1, I2, I3, I4),
347
348#define C_N1_I2(O1, I1, I2) C_PFX3(c_n1_i2_, O1, I1, I2),
349
350#define C_O2_I1(O1, O2, I1) C_PFX3(c_o2_i1_, O1, O2, I1),
351#define C_O2_I2(O1, O2, I1, I2) C_PFX4(c_o2_i2_, O1, O2, I1, I2),
352#define C_O2_I3(O1, O2, I1, I2, I3) C_PFX5(c_o2_i3_, O1, O2, I1, I2, I3),
353#define C_O2_I4(O1, O2, I1, I2, I3, I4) C_PFX6(c_o2_i4_, O1, O2, I1, I2, I3, I4),
354
355typedef enum {
356#include "tcg-target-con-set.h"
357} TCGConstraintSetIndex;
358
359static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode);
360
361#undef C_O0_I1
362#undef C_O0_I2
363#undef C_O0_I3
364#undef C_O0_I4
365#undef C_O1_I1
366#undef C_O1_I2
367#undef C_O1_I3
368#undef C_O1_I4
369#undef C_N1_I2
370#undef C_O2_I1
371#undef C_O2_I2
372#undef C_O2_I3
373#undef C_O2_I4
374
375/* Put all of the constraint sets into an array, indexed by the enum. */
376
377#define C_O0_I1(I1) { .args_ct_str = { #I1 } },
378#define C_O0_I2(I1, I2) { .args_ct_str = { #I1, #I2 } },
379#define C_O0_I3(I1, I2, I3) { .args_ct_str = { #I1, #I2, #I3 } },
380#define C_O0_I4(I1, I2, I3, I4) { .args_ct_str = { #I1, #I2, #I3, #I4 } },
381
382#define C_O1_I1(O1, I1) { .args_ct_str = { #O1, #I1 } },
383#define C_O1_I2(O1, I1, I2) { .args_ct_str = { #O1, #I1, #I2 } },
384#define C_O1_I3(O1, I1, I2, I3) { .args_ct_str = { #O1, #I1, #I2, #I3 } },
385#define C_O1_I4(O1, I1, I2, I3, I4) { .args_ct_str = { #O1, #I1, #I2, #I3, #I4 } },
386
387#define C_N1_I2(O1, I1, I2) { .args_ct_str = { "&" #O1, #I1, #I2 } },
388
389#define C_O2_I1(O1, O2, I1) { .args_ct_str = { #O1, #O2, #I1 } },
390#define C_O2_I2(O1, O2, I1, I2) { .args_ct_str = { #O1, #O2, #I1, #I2 } },
391#define C_O2_I3(O1, O2, I1, I2, I3) { .args_ct_str = { #O1, #O2, #I1, #I2, #I3 } },
392#define C_O2_I4(O1, O2, I1, I2, I3, I4) { .args_ct_str = { #O1, #O2, #I1, #I2, #I3, #I4 } },
393
394static const TCGTargetOpDef constraint_sets[] = {
395#include "tcg-target-con-set.h"
396};
397
398
399#undef C_O0_I1
400#undef C_O0_I2
401#undef C_O0_I3
402#undef C_O0_I4
403#undef C_O1_I1
404#undef C_O1_I2
405#undef C_O1_I3
406#undef C_O1_I4
407#undef C_N1_I2
408#undef C_O2_I1
409#undef C_O2_I2
410#undef C_O2_I3
411#undef C_O2_I4
412
413/* Expand the enumerator to be returned from tcg_target_op_def(). */
414
415#define C_O0_I1(I1) C_PFX1(c_o0_i1_, I1)
416#define C_O0_I2(I1, I2) C_PFX2(c_o0_i2_, I1, I2)
417#define C_O0_I3(I1, I2, I3) C_PFX3(c_o0_i3_, I1, I2, I3)
418#define C_O0_I4(I1, I2, I3, I4) C_PFX4(c_o0_i4_, I1, I2, I3, I4)
419
420#define C_O1_I1(O1, I1) C_PFX2(c_o1_i1_, O1, I1)
421#define C_O1_I2(O1, I1, I2) C_PFX3(c_o1_i2_, O1, I1, I2)
422#define C_O1_I3(O1, I1, I2, I3) C_PFX4(c_o1_i3_, O1, I1, I2, I3)
423#define C_O1_I4(O1, I1, I2, I3, I4) C_PFX5(c_o1_i4_, O1, I1, I2, I3, I4)
424
425#define C_N1_I2(O1, I1, I2) C_PFX3(c_n1_i2_, O1, I1, I2)
426
427#define C_O2_I1(O1, O2, I1) C_PFX3(c_o2_i1_, O1, O2, I1)
428#define C_O2_I2(O1, O2, I1, I2) C_PFX4(c_o2_i2_, O1, O2, I1, I2)
429#define C_O2_I3(O1, O2, I1, I2, I3) C_PFX5(c_o2_i3_, O1, O2, I1, I2, I3)
430#define C_O2_I4(O1, O2, I1, I2, I3, I4) C_PFX6(c_o2_i4_, O1, O2, I1, I2, I3, I4)
431
139c1837 432#include "tcg-target.c.inc"
c896fe29 433
38b47b19
EC		 434static void alloc_tcg_plugin_context(TCGContext *s)
435{
436#ifdef CONFIG_PLUGIN
437 s->plugin_tb = g_new0(struct qemu_plugin_tb, 1);
438 s->plugin_tb->insns =
439 g_ptr_array_new_with_free_func(qemu_plugin_insn_cleanup_fn);
440#endif
441}
442
3468b59e
EC		 443/*
444 * All TCG threads except the parent (i.e. the one that called tcg_context_init
445 * and registered the target's TCG globals) must register with this function
446 * before initiating translation.
447 *
448 * In user-mode we just point tcg_ctx to tcg_init_ctx. See the documentation
449 * of tcg_region_init() for the reasoning behind this.
450 *
451 * In softmmu each caller registers its context in tcg_ctxs[]. Note that in
452 * softmmu tcg_ctxs[] does not track tcg_ctx_init, since the initial context
453 * is not used anymore for translation once this function is called.
454 *
455 * Not tracking tcg_init_ctx in tcg_ctxs[] in softmmu keeps code that iterates
456 * over the array (e.g. tcg_code_size() the same for both softmmu and user-mode.
457 */
458#ifdef CONFIG_USER_ONLY
459void tcg_register_thread(void)
460{
461 tcg_ctx = &tcg_init_ctx;
462}
463#else
464void tcg_register_thread(void)
465{
466 TCGContext *s = g_malloc(sizeof(*s));
467 unsigned int i, n;
3468b59e
EC		 468
469 *s = tcg_init_ctx;
470
471 /* Relink mem_base. */
472 for (i = 0, n = tcg_init_ctx.nb_globals; i < n; ++i) {
473 if (tcg_init_ctx.temps[i].mem_base) {
474 ptrdiff_t b = tcg_init_ctx.temps[i].mem_base - tcg_init_ctx.temps;
475 tcg_debug_assert(b >= 0 && b < n);
476 s->temps[i].mem_base = &s->temps[b];
477 }
478 }
479
480 /* Claim an entry in tcg_ctxs */
0e2d61cf
RH		 481 n = qatomic_fetch_inc(&tcg_cur_ctxs);
482 g_assert(n < tcg_max_ctxs);
d73415a3 483 qatomic_set(&tcg_ctxs[n], s);
3468b59e 484
38b47b19
EC		 485 if (n > 0) {
486 alloc_tcg_plugin_context(s);
bf042e8e 487 tcg_region_initial_alloc(s);
38b47b19
EC		 488 }
489
3468b59e 490 tcg_ctx = s;
e8feb96f 491}
3468b59e 492#endif /* !CONFIG_USER_ONLY */
e8feb96f 493
c896fe29
FB		 494/* pool based memory allocation */
495void *tcg_malloc_internal(TCGContext *s, int size)
496{
497 TCGPool *p;
498 int pool_size;
a813e36f 499
c896fe29
FB		 500 if (size > TCG_POOL_CHUNK_SIZE) {
501 /* big malloc: insert a new pool (XXX: could optimize) */
7267c094 502 p = g_malloc(sizeof(TCGPool) + size);
c896fe29 503 p->size = size;
4055299e
KB		 504 p->next = s->pool_first_large;
505 s->pool_first_large = p;
506 return p->data;
c896fe29
FB		 507 } else {
508 p = s->pool_current;
509 if (!p) {
510 p = s->pool_first;
511 if (!p)
512 goto new_pool;
513 } else {
514 if (!p->next) {
515 new_pool:
516 pool_size = TCG_POOL_CHUNK_SIZE;
7267c094 517 p = g_malloc(sizeof(TCGPool) + pool_size);
c896fe29
FB		 518 p->size = pool_size;
519 p->next = NULL;
a813e36f 520 if (s->pool_current) {
c896fe29 521 s->pool_current->next = p;
a813e36f 522 } else {
c896fe29 523 s->pool_first = p;
a813e36f 524 }
c896fe29
FB		 525 } else {
526 p = p->next;
527 }
528 }
529 }
530 s->pool_current = p;
531 s->pool_cur = p->data + size;
532 s->pool_end = p->data + p->size;
533 return p->data;
534}
535
536void tcg_pool_reset(TCGContext *s)
537{
4055299e
KB		 538 TCGPool *p, *t;
539 for (p = s->pool_first_large; p; p = t) {
540 t = p->next;
541 g_free(p);
542 }
543 s->pool_first_large = NULL;
c896fe29
FB		 544 s->pool_cur = s->pool_end = NULL;
545 s->pool_current = NULL;
546}
547
2ef6175a
RH		 548#include "exec/helper-proto.h"
549
100b5e01 550static const TCGHelperInfo all_helpers[] = {
2ef6175a 551#include "exec/helper-tcg.h"
100b5e01 552};
619205fd 553static GHashTable *helper_table;
100b5e01 554
22f15579
RH		 555#ifdef CONFIG_TCG_INTERPRETER
556static GHashTable *ffi_table;
557
558static ffi_type * const typecode_to_ffi[8] = {
559 [dh_typecode_void] = &ffi_type_void,
560 [dh_typecode_i32] = &ffi_type_uint32,
561 [dh_typecode_s32] = &ffi_type_sint32,
562 [dh_typecode_i64] = &ffi_type_uint64,
563 [dh_typecode_s64] = &ffi_type_sint64,
564 [dh_typecode_ptr] = &ffi_type_pointer,
565};
566#endif
567
91478cef 568static int indirect_reg_alloc_order[ARRAY_SIZE(tcg_target_reg_alloc_order)];
f69d277e 569static void process_op_defs(TCGContext *s);
1c2adb95
RH		 570static TCGTemp *tcg_global_reg_new_internal(TCGContext *s, TCGType type,
571 TCGReg reg, const char *name);
91478cef 572
43b972b7 573static void tcg_context_init(unsigned max_cpus)
c896fe29 574{
a76aabd3 575 TCGContext *s = &tcg_init_ctx;
100b5e01 576 int op, total_args, n, i;
c896fe29
FB		 577 TCGOpDef *def;
578 TCGArgConstraint *args_ct;
1c2adb95 579 TCGTemp *ts;
c896fe29
FB		 580
581 memset(s, 0, sizeof(*s));
c896fe29 582 s->nb_globals = 0;
c70fbf0a 583
c896fe29
FB		 584 /* Count total number of arguments and allocate the corresponding
585 space */
586 total_args = 0;
587 for(op = 0; op < NB_OPS; op++) {
588 def = &tcg_op_defs[op];
589 n = def->nb_iargs + def->nb_oargs;
590 total_args += n;
591 }
592
bc2b17e6 593 args_ct = g_new0(TCGArgConstraint, total_args);
c896fe29
FB		 594
595 for(op = 0; op < NB_OPS; op++) {
596 def = &tcg_op_defs[op];
597 def->args_ct = args_ct;
c896fe29 598 n = def->nb_iargs + def->nb_oargs;
c896fe29
FB		 599 args_ct += n;
600 }
5cd8f621
RH		 601
602 /* Register helpers. */
84fd9dd3 603 /* Use g_direct_hash/equal for direct pointer comparisons on func. */
619205fd 604 helper_table = g_hash_table_new(NULL, NULL);
84fd9dd3 605
100b5e01 606 for (i = 0; i < ARRAY_SIZE(all_helpers); ++i) {
84fd9dd3 607 g_hash_table_insert(helper_table, (gpointer)all_helpers[i].func,
72866e82 608 (gpointer)&all_helpers[i]);
100b5e01 609 }
5cd8f621 610
22f15579
RH		 611#ifdef CONFIG_TCG_INTERPRETER
612 /* g_direct_hash/equal for direct comparisons on uint32_t. */
613 ffi_table = g_hash_table_new(NULL, NULL);
614 for (i = 0; i < ARRAY_SIZE(all_helpers); ++i) {
615 struct {
616 ffi_cif cif;
617 ffi_type *args[];
618 } *ca;
619 uint32_t typemask = all_helpers[i].typemask;
620 gpointer hash = (gpointer)(uintptr_t)typemask;
621 ffi_status status;
622 int nargs;
623
624 if (g_hash_table_lookup(ffi_table, hash)) {
625 continue;
626 }
627
628 /* Ignoring the return type, find the last non-zero field. */
629 nargs = 32 - clz32(typemask >> 3);
630 nargs = DIV_ROUND_UP(nargs, 3);
631
632 ca = g_malloc0(sizeof(*ca) + nargs * sizeof(ffi_type *));
633 ca->cif.rtype = typecode_to_ffi[typemask & 7];
634 ca->cif.nargs = nargs;
635
636 if (nargs != 0) {
637 ca->cif.arg_types = ca->args;
9dd1d56e
IZ		 638 for (int j = 0; j < nargs; ++j) {
639 int typecode = extract32(typemask, (j + 1) * 3, 3);
640 ca->args[j] = typecode_to_ffi[typecode];
22f15579
RH		 641 }
642 }
643
644 status = ffi_prep_cif(&ca->cif, FFI_DEFAULT_ABI, nargs,
645 ca->cif.rtype, ca->cif.arg_types);
646 assert(status == FFI_OK);
647
648 g_hash_table_insert(ffi_table, hash, (gpointer)&ca->cif);
649 }
650#endif
651
c896fe29 652 tcg_target_init(s);
f69d277e 653 process_op_defs(s);
91478cef
RH		 654
655 /* Reverse the order of the saved registers, assuming they're all at
656 the start of tcg_target_reg_alloc_order. */
657 for (n = 0; n < ARRAY_SIZE(tcg_target_reg_alloc_order); ++n) {
658 int r = tcg_target_reg_alloc_order[n];
659 if (tcg_regset_test_reg(tcg_target_call_clobber_regs, r)) {
660 break;
661 }
662 }
663 for (i = 0; i < n; ++i) {
664 indirect_reg_alloc_order[i] = tcg_target_reg_alloc_order[n - 1 - i];
665 }
666 for (; i < ARRAY_SIZE(tcg_target_reg_alloc_order); ++i) {
667 indirect_reg_alloc_order[i] = tcg_target_reg_alloc_order[i];
668 }
b1311c4a 669
38b47b19
EC		 670 alloc_tcg_plugin_context(s);
671
b1311c4a 672 tcg_ctx = s;
3468b59e
EC		 673 /*
674 * In user-mode we simply share the init context among threads, since we
675 * use a single region. See the documentation tcg_region_init() for the
676 * reasoning behind this.
677 * In softmmu we will have at most max_cpus TCG threads.
678 */
679#ifdef CONFIG_USER_ONLY
df2cce29 680 tcg_ctxs = &tcg_ctx;
0e2d61cf
RH		 681 tcg_cur_ctxs = 1;
682 tcg_max_ctxs = 1;
3468b59e 683#else
0e2d61cf
RH		 684 tcg_max_ctxs = max_cpus;
685 tcg_ctxs = g_new0(TCGContext *, max_cpus);
3468b59e 686#endif
1c2adb95
RH		 687
688 tcg_debug_assert(!tcg_regset_test_reg(s->reserved_regs, TCG_AREG0));
689 ts = tcg_global_reg_new_internal(s, TCG_TYPE_PTR, TCG_AREG0, "env");
690 cpu_env = temp_tcgv_ptr(ts);
9002ec79 691}
b03cce8e 692
43b972b7 693void tcg_init(size_t tb_size, int splitwx, unsigned max_cpus)
a76aabd3 694{
43b972b7
RH		 695 tcg_context_init(max_cpus);
696 tcg_region_init(tb_size, splitwx, max_cpus);
a76aabd3
RH		 697}
698
6e3b2bfd
EC		 699/*
700 * Allocate TBs right before their corresponding translated code, making
701 * sure that TBs and code are on different cache lines.
702 */
703TranslationBlock *tcg_tb_alloc(TCGContext *s)
704{
705 uintptr_t align = qemu_icache_linesize;
706 TranslationBlock *tb;
707 void *next;
708
e8feb96f 709 retry:
6e3b2bfd
EC		 710 tb = (void *)ROUND_UP((uintptr_t)s->code_gen_ptr, align);
711 next = (void *)ROUND_UP((uintptr_t)(tb + 1), align);
712
713 if (unlikely(next > s->code_gen_highwater)) {
e8feb96f
EC		 714 if (tcg_region_alloc(s)) {
715 return NULL;
716 }
717 goto retry;
6e3b2bfd 718 }
d73415a3 719 qatomic_set(&s->code_gen_ptr, next);
57a26946 720 s->data_gen_ptr = NULL;
6e3b2bfd
EC		 721 return tb;
722}
723
9002ec79
RH		 724void tcg_prologue_init(TCGContext *s)
725{
b0a0794a 726 size_t prologue_size;
8163b749 727
b0a0794a
RH		 728 s->code_ptr = s->code_gen_ptr;
729 s->code_buf = s->code_gen_ptr;
5b38ee31 730 s->data_gen_ptr = NULL;
b91ccb31
RH		 731
732#ifndef CONFIG_TCG_INTERPRETER
b0a0794a 733 tcg_qemu_tb_exec = (tcg_prologue_fn *)tcg_splitwx_to_rx(s->code_ptr);
b91ccb31 734#endif
8163b749 735
5b38ee31
RH		 736#ifdef TCG_TARGET_NEED_POOL_LABELS
737 s->pool_labels = NULL;
738#endif
739
653b87eb 740 qemu_thread_jit_write();
8163b749 741 /* Generate the prologue. */
b03cce8e 742 tcg_target_qemu_prologue(s);
5b38ee31
RH		 743
744#ifdef TCG_TARGET_NEED_POOL_LABELS
745 /* Allow the prologue to put e.g. guest_base into a pool entry. */
746 {
1768987b
RH		 747 int result = tcg_out_pool_finalize(s);
748 tcg_debug_assert(result == 0);
5b38ee31
RH		 749 }
750#endif
751
b0a0794a
RH		 752 prologue_size = tcg_current_code_size(s);
753
df5d2b16 754#ifndef CONFIG_TCG_INTERPRETER
b0a0794a
RH		 755 flush_idcache_range((uintptr_t)tcg_splitwx_to_rx(s->code_buf),
756 (uintptr_t)s->code_buf, prologue_size);
df5d2b16 757#endif
8163b749 758
d6b64b2b
RH		 759#ifdef DEBUG_DISAS
760 if (qemu_loglevel_mask(CPU_LOG_TB_OUT_ASM)) {
c60f599b 761 FILE *logfile = qemu_log_trylock();
78b54858
RH		 762 if (logfile) {
763 fprintf(logfile, "PROLOGUE: [size=%zu]\n", prologue_size);
764 if (s->data_gen_ptr) {
765 size_t code_size = s->data_gen_ptr - s->code_gen_ptr;
766 size_t data_size = prologue_size - code_size;
767 size_t i;
768
769 disas(logfile, s->code_gen_ptr, code_size);
770
771 for (i = 0; i < data_size; i += sizeof(tcg_target_ulong)) {
772 if (sizeof(tcg_target_ulong) == 8) {
773 fprintf(logfile,
774 "0x%08" PRIxPTR ": .quad 0x%016" PRIx64 "\n",
775 (uintptr_t)s->data_gen_ptr + i,
776 *(uint64_t *)(s->data_gen_ptr + i));
777 } else {
778 fprintf(logfile,
779 "0x%08" PRIxPTR ": .long 0x%08x\n",
780 (uintptr_t)s->data_gen_ptr + i,
781 *(uint32_t *)(s->data_gen_ptr + i));
782 }
5b38ee31 783 }
78b54858
RH		 784 } else {
785 disas(logfile, s->code_gen_ptr, prologue_size);
5b38ee31 786 }
78b54858 787 fprintf(logfile, "\n");
78b54858 788 qemu_log_unlock(logfile);
5b38ee31 789 }
d6b64b2b
RH		 790 }
791#endif
cedbcb01 792
6eea0434
RH		 793#ifndef CONFIG_TCG_INTERPRETER
794 /*
795 * Assert that goto_ptr is implemented completely, setting an epilogue.
796 * For tci, we use NULL as the signal to return from the interpreter,
797 * so skip this check.
798 */
f4e01e30 799 tcg_debug_assert(tcg_code_gen_epilogue != NULL);
6eea0434 800#endif
d1c74ab3
RH		 801
802 tcg_region_prologue_set(s);
c896fe29
FB		 803}
804
c896fe29
FB		 805void tcg_func_start(TCGContext *s)
806{
807 tcg_pool_reset(s);
808 s->nb_temps = s->nb_globals;
0ec9eabc
RH		 809
810 /* No temps have been previously allocated for size or locality. */
811 memset(s->free_temps, 0, sizeof(s->free_temps));
812
c0522136
RH		 813 /* No constant temps have been previously allocated. */
814 for (int i = 0; i < TCG_TYPE_COUNT; ++i) {
815 if (s->const_table[i]) {
816 g_hash_table_remove_all(s->const_table[i]);
817 }
818 }
819
abebf925 820 s->nb_ops = 0;
c896fe29
FB		 821 s->nb_labels = 0;
822 s->current_frame_offset = s->frame_start;
823
0a209d4b
RH		 824#ifdef CONFIG_DEBUG_TCG
825 s->goto_tb_issue_mask = 0;
826#endif
827
15fa08f8
RH		 828 QTAILQ_INIT(&s->ops);
829 QTAILQ_INIT(&s->free_ops);
bef16ab4 830 QSIMPLEQ_INIT(&s->labels);
c896fe29
FB		 831}
832
ae30e866 833static TCGTemp *tcg_temp_alloc(TCGContext *s)
7ca4b752
RH		 834{
835 int n = s->nb_temps++;
ae30e866
RH		 836
837 if (n >= TCG_MAX_TEMPS) {
db6b7d0c 838 tcg_raise_tb_overflow(s);
ae30e866 839 }
7ca4b752
RH		 840 return memset(&s->temps[n], 0, sizeof(TCGTemp));
841}
842
ae30e866 843static TCGTemp *tcg_global_alloc(TCGContext *s)
7ca4b752 844{
fa477d25
RH		 845 TCGTemp *ts;
846
7ca4b752 847 tcg_debug_assert(s->nb_globals == s->nb_temps);
ae30e866 848 tcg_debug_assert(s->nb_globals < TCG_MAX_TEMPS);
7ca4b752 849 s->nb_globals++;
fa477d25 850 ts = tcg_temp_alloc(s);
ee17db83 851 ts->kind = TEMP_GLOBAL;
fa477d25
RH		 852
853 return ts;
c896fe29
FB		 854}
855
085272b3
RH		 856static TCGTemp *tcg_global_reg_new_internal(TCGContext *s, TCGType type,
857 TCGReg reg, const char *name)
c896fe29 858{
c896fe29 859 TCGTemp *ts;
c896fe29 860
b3a62939 861 if (TCG_TARGET_REG_BITS == 32 && type != TCG_TYPE_I32) {
c896fe29 862 tcg_abort();
b3a62939 863 }
7ca4b752
RH		 864
865 ts = tcg_global_alloc(s);
c896fe29
FB		 866 ts->base_type = type;
867 ts->type = type;
ee17db83 868 ts->kind = TEMP_FIXED;
c896fe29 869 ts->reg = reg;
c896fe29 870 ts->name = name;
c896fe29 871 tcg_regset_set_reg(s->reserved_regs, reg);
7ca4b752 872
085272b3 873 return ts;
a7812ae4
PB		 874}
875
b6638662 876void tcg_set_frame(TCGContext *s, TCGReg reg, intptr_t start, intptr_t size)
b3a62939 877{
b3a62939
RH		 878 s->frame_start = start;
879 s->frame_end = start + size;
085272b3
RH		 880 s->frame_temp
881 = tcg_global_reg_new_internal(s, TCG_TYPE_PTR, reg, "_frame");
b3a62939
RH		 882}
883
085272b3
RH		 884TCGTemp *tcg_global_mem_new_internal(TCGType type, TCGv_ptr base,
885 intptr_t offset, const char *name)
c896fe29 886{
b1311c4a 887 TCGContext *s = tcg_ctx;
dc41aa7d 888 TCGTemp *base_ts = tcgv_ptr_temp(base);
7ca4b752 889 TCGTemp *ts = tcg_global_alloc(s);
b3915dbb 890 int indirect_reg = 0, bigendian = 0;
e03b5686 891#if HOST_BIG_ENDIAN
7ca4b752
RH		 892 bigendian = 1;
893#endif
c896fe29 894
c0522136
RH		 895 switch (base_ts->kind) {
896 case TEMP_FIXED:
897 break;
898 case TEMP_GLOBAL:
5a18407f
RH		 899 /* We do not support double-indirect registers. */
900 tcg_debug_assert(!base_ts->indirect_reg);
b3915dbb 901 base_ts->indirect_base = 1;
5a18407f
RH		 902 s->nb_indirects += (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64
903 ? 2 : 1);
904 indirect_reg = 1;
c0522136
RH		 905 break;
906 default:
907 g_assert_not_reached();
b3915dbb
RH		 908 }
909
7ca4b752
RH		 910 if (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64) {
911 TCGTemp *ts2 = tcg_global_alloc(s);
c896fe29 912 char buf[64];
7ca4b752
RH		 913
914 ts->base_type = TCG_TYPE_I64;
c896fe29 915 ts->type = TCG_TYPE_I32;
b3915dbb 916 ts->indirect_reg = indirect_reg;
c896fe29 917 ts->mem_allocated = 1;
b3a62939 918 ts->mem_base = base_ts;
7ca4b752 919 ts->mem_offset = offset + bigendian * 4;
c896fe29
FB		 920 pstrcpy(buf, sizeof(buf), name);
921 pstrcat(buf, sizeof(buf), "_0");
922 ts->name = strdup(buf);
c896fe29 923
7ca4b752
RH		 924 tcg_debug_assert(ts2 == ts + 1);
925 ts2->base_type = TCG_TYPE_I64;
926 ts2->type = TCG_TYPE_I32;
b3915dbb 927 ts2->indirect_reg = indirect_reg;
7ca4b752
RH		 928 ts2->mem_allocated = 1;
929 ts2->mem_base = base_ts;
930 ts2->mem_offset = offset + (1 - bigendian) * 4;
c896fe29
FB		 931 pstrcpy(buf, sizeof(buf), name);
932 pstrcat(buf, sizeof(buf), "_1");
120c1084 933 ts2->name = strdup(buf);
7ca4b752 934 } else {
c896fe29
FB		 935 ts->base_type = type;
936 ts->type = type;
b3915dbb 937 ts->indirect_reg = indirect_reg;
c896fe29 938 ts->mem_allocated = 1;
b3a62939 939 ts->mem_base = base_ts;
c896fe29 940 ts->mem_offset = offset;
c896fe29 941 ts->name = name;
c896fe29 942 }
085272b3 943 return ts;
a7812ae4
PB		 944}
945
5bfa8034 946TCGTemp *tcg_temp_new_internal(TCGType type, bool temp_local)
c896fe29 947{
b1311c4a 948 TCGContext *s = tcg_ctx;
ee17db83 949 TCGTempKind kind = temp_local ? TEMP_LOCAL : TEMP_NORMAL;
c896fe29 950 TCGTemp *ts;
641d5fbe 951 int idx, k;
c896fe29 952
0ec9eabc
RH		 953 k = type + (temp_local ? TCG_TYPE_COUNT : 0);
954 idx = find_first_bit(s->free_temps[k].l, TCG_MAX_TEMPS);
955 if (idx < TCG_MAX_TEMPS) {
956 /* There is already an available temp with the right type. */
957 clear_bit(idx, s->free_temps[k].l);
958
e8996ee0 959 ts = &s->temps[idx];
e8996ee0 960 ts->temp_allocated = 1;
7ca4b752 961 tcg_debug_assert(ts->base_type == type);
ee17db83 962 tcg_debug_assert(ts->kind == kind);
e8996ee0 963 } else {
7ca4b752
RH		 964 ts = tcg_temp_alloc(s);
965 if (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64) {
966 TCGTemp *ts2 = tcg_temp_alloc(s);
967
f6aa2f7d 968 ts->base_type = type;
e8996ee0
FB		 969 ts->type = TCG_TYPE_I32;
970 ts->temp_allocated = 1;
ee17db83 971 ts->kind = kind;
7ca4b752
RH		 972
973 tcg_debug_assert(ts2 == ts + 1);
974 ts2->base_type = TCG_TYPE_I64;
975 ts2->type = TCG_TYPE_I32;
976 ts2->temp_allocated = 1;
ee17db83 977 ts2->kind = kind;
7ca4b752 978 } else {
e8996ee0
FB		 979 ts->base_type = type;
980 ts->type = type;
981 ts->temp_allocated = 1;
ee17db83 982 ts->kind = kind;
e8996ee0 983 }
c896fe29 984 }
27bfd83c
PM		 985
986#if defined(CONFIG_DEBUG_TCG)
987 s->temps_in_use++;
988#endif
085272b3 989 return ts;
c896fe29
FB		 990}
991
d2fd745f
RH		 992TCGv_vec tcg_temp_new_vec(TCGType type)
993{
994 TCGTemp *t;
995
996#ifdef CONFIG_DEBUG_TCG
997 switch (type) {
998 case TCG_TYPE_V64:
999 assert(TCG_TARGET_HAS_v64);
1000 break;
1001 case TCG_TYPE_V128:
1002 assert(TCG_TARGET_HAS_v128);
1003 break;
1004 case TCG_TYPE_V256:
1005 assert(TCG_TARGET_HAS_v256);
1006 break;
1007 default:
1008 g_assert_not_reached();
1009 }
1010#endif
1011
1012 t = tcg_temp_new_internal(type, 0);
1013 return temp_tcgv_vec(t);
1014}
1015
1016/* Create a new temp of the same type as an existing temp. */
1017TCGv_vec tcg_temp_new_vec_matching(TCGv_vec match)
1018{
1019 TCGTemp *t = tcgv_vec_temp(match);
1020
1021 tcg_debug_assert(t->temp_allocated != 0);
1022
1023 t = tcg_temp_new_internal(t->base_type, 0);
1024 return temp_tcgv_vec(t);
1025}
1026
5bfa8034 1027void tcg_temp_free_internal(TCGTemp *ts)
c896fe29 1028{
b1311c4a 1029 TCGContext *s = tcg_ctx;
085272b3 1030 int k, idx;
c896fe29 1031
c7482438
RH		 1032 switch (ts->kind) {
1033 case TEMP_CONST:
1034 /*
1035 * In order to simplify users of tcg_constant_*,
1036 * silently ignore free.
1037 */
c0522136 1038 return;
c7482438
RH		 1039 case TEMP_NORMAL:
1040 case TEMP_LOCAL:
1041 break;
1042 default:
1043 g_assert_not_reached();
c0522136
RH		 1044 }
1045
27bfd83c
PM		 1046#if defined(CONFIG_DEBUG_TCG)
1047 s->temps_in_use--;
1048 if (s->temps_in_use < 0) {
1049 fprintf(stderr, "More temporaries freed than allocated!\n");
1050 }
1051#endif
1052
eabb7b91 1053 tcg_debug_assert(ts->temp_allocated != 0);
e8996ee0 1054 ts->temp_allocated = 0;
0ec9eabc 1055
085272b3 1056 idx = temp_idx(ts);
ee17db83 1057 k = ts->base_type + (ts->kind == TEMP_NORMAL ? 0 : TCG_TYPE_COUNT);
0ec9eabc 1058 set_bit(idx, s->free_temps[k].l);
c896fe29
FB		 1059}
1060
c0522136
RH		 1061TCGTemp *tcg_constant_internal(TCGType type, int64_t val)
1062{
1063 TCGContext *s = tcg_ctx;
1064 GHashTable *h = s->const_table[type];
1065 TCGTemp *ts;
1066
1067 if (h == NULL) {
1068 h = g_hash_table_new(g_int64_hash, g_int64_equal);
1069 s->const_table[type] = h;
1070 }
1071
1072 ts = g_hash_table_lookup(h, &val);
1073 if (ts == NULL) {
1074 ts = tcg_temp_alloc(s);
1075
1076 if (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64) {
1077 TCGTemp *ts2 = tcg_temp_alloc(s);
1078
1079 ts->base_type = TCG_TYPE_I64;
1080 ts->type = TCG_TYPE_I32;
1081 ts->kind = TEMP_CONST;
1082 ts->temp_allocated = 1;
1083 /*
1084 * Retain the full value of the 64-bit constant in the low
1085 * part, so that the hash table works. Actual uses will
1086 * truncate the value to the low part.
1087 */
1088 ts->val = val;
1089
1090 tcg_debug_assert(ts2 == ts + 1);
1091 ts2->base_type = TCG_TYPE_I64;
1092 ts2->type = TCG_TYPE_I32;
1093 ts2->kind = TEMP_CONST;
1094 ts2->temp_allocated = 1;
1095 ts2->val = val >> 32;
1096 } else {
1097 ts->base_type = type;
1098 ts->type = type;
1099 ts->kind = TEMP_CONST;
1100 ts->temp_allocated = 1;
1101 ts->val = val;
1102 }
1103 g_hash_table_insert(h, &ts->val, ts);
1104 }
1105
1106 return ts;
1107}
1108
1109TCGv_vec tcg_constant_vec(TCGType type, unsigned vece, int64_t val)
1110{
1111 val = dup_const(vece, val);
1112 return temp_tcgv_vec(tcg_constant_internal(type, val));
1113}
1114
88d4005b
RH		 1115TCGv_vec tcg_constant_vec_matching(TCGv_vec match, unsigned vece, int64_t val)
1116{
1117 TCGTemp *t = tcgv_vec_temp(match);
1118
1119 tcg_debug_assert(t->temp_allocated != 0);
1120 return tcg_constant_vec(t->base_type, vece, val);
1121}
1122
a7812ae4 1123TCGv_i32 tcg_const_i32(int32_t val)
c896fe29 1124{
a7812ae4
PB		 1125 TCGv_i32 t0;
1126 t0 = tcg_temp_new_i32();
e8996ee0
FB		 1127 tcg_gen_movi_i32(t0, val);
1128 return t0;
1129}
c896fe29 1130
a7812ae4 1131TCGv_i64 tcg_const_i64(int64_t val)
e8996ee0 1132{
a7812ae4
PB		 1133 TCGv_i64 t0;
1134 t0 = tcg_temp_new_i64();
e8996ee0
FB		 1135 tcg_gen_movi_i64(t0, val);
1136 return t0;
c896fe29
FB		 1137}
1138
a7812ae4 1139TCGv_i32 tcg_const_local_i32(int32_t val)
bdffd4a9 1140{
a7812ae4
PB		 1141 TCGv_i32 t0;
1142 t0 = tcg_temp_local_new_i32();
bdffd4a9
AJ		 1143 tcg_gen_movi_i32(t0, val);
1144 return t0;
1145}
1146
a7812ae4 1147TCGv_i64 tcg_const_local_i64(int64_t val)
bdffd4a9 1148{
a7812ae4
PB		 1149 TCGv_i64 t0;
1150 t0 = tcg_temp_local_new_i64();
bdffd4a9
AJ		 1151 tcg_gen_movi_i64(t0, val);
1152 return t0;
1153}
1154
27bfd83c
PM		 1155#if defined(CONFIG_DEBUG_TCG)
1156void tcg_clear_temp_count(void)
1157{
b1311c4a 1158 TCGContext *s = tcg_ctx;
27bfd83c
PM		 1159 s->temps_in_use = 0;
1160}
1161
1162int tcg_check_temp_count(void)
1163{
b1311c4a 1164 TCGContext *s = tcg_ctx;
27bfd83c
PM		 1165 if (s->temps_in_use) {
1166 /* Clear the count so that we don't give another
1167 * warning immediately next time around.
1168 */
1169 s->temps_in_use = 0;
1170 return 1;
1171 }
1172 return 0;
1173}
1174#endif
1175
be0f34b5
RH		 1176/* Return true if OP may appear in the opcode stream.
1177 Test the runtime variable that controls each opcode. */
1178bool tcg_op_supported(TCGOpcode op)
1179{
d2fd745f
RH		 1180 const bool have_vec
1181 = TCG_TARGET_HAS_v64 | TCG_TARGET_HAS_v128 | TCG_TARGET_HAS_v256;
1182
be0f34b5
RH		 1183 switch (op) {
1184 case INDEX_op_discard:
1185 case INDEX_op_set_label:
1186 case INDEX_op_call:
1187 case INDEX_op_br:
1188 case INDEX_op_mb:
1189 case INDEX_op_insn_start:
1190 case INDEX_op_exit_tb:
1191 case INDEX_op_goto_tb:
f4e01e30 1192 case INDEX_op_goto_ptr:
be0f34b5
RH		 1193 case INDEX_op_qemu_ld_i32:
1194 case INDEX_op_qemu_st_i32:
1195 case INDEX_op_qemu_ld_i64:
1196 case INDEX_op_qemu_st_i64:
1197 return true;
1198
07ce0b05
RH		 1199 case INDEX_op_qemu_st8_i32:
1200 return TCG_TARGET_HAS_qemu_st8_i32;
1201
be0f34b5 1202 case INDEX_op_mov_i32:
be0f34b5
RH		 1203 case INDEX_op_setcond_i32:
1204 case INDEX_op_brcond_i32:
1205 case INDEX_op_ld8u_i32:
1206 case INDEX_op_ld8s_i32:
1207 case INDEX_op_ld16u_i32:
1208 case INDEX_op_ld16s_i32:
1209 case INDEX_op_ld_i32:
1210 case INDEX_op_st8_i32:
1211 case INDEX_op_st16_i32:
1212 case INDEX_op_st_i32:
1213 case INDEX_op_add_i32:
1214 case INDEX_op_sub_i32:
1215 case INDEX_op_mul_i32:
1216 case INDEX_op_and_i32:
1217 case INDEX_op_or_i32:
1218 case INDEX_op_xor_i32:
1219 case INDEX_op_shl_i32:
1220 case INDEX_op_shr_i32:
1221 case INDEX_op_sar_i32:
1222 return true;
1223
1224 case INDEX_op_movcond_i32:
1225 return TCG_TARGET_HAS_movcond_i32;
1226 case INDEX_op_div_i32:
1227 case INDEX_op_divu_i32:
1228 return TCG_TARGET_HAS_div_i32;
1229 case INDEX_op_rem_i32:
1230 case INDEX_op_remu_i32:
1231 return TCG_TARGET_HAS_rem_i32;
1232 case INDEX_op_div2_i32:
1233 case INDEX_op_divu2_i32:
1234 return TCG_TARGET_HAS_div2_i32;
1235 case INDEX_op_rotl_i32:
1236 case INDEX_op_rotr_i32:
1237 return TCG_TARGET_HAS_rot_i32;
1238 case INDEX_op_deposit_i32:
1239 return TCG_TARGET_HAS_deposit_i32;
1240 case INDEX_op_extract_i32:
1241 return TCG_TARGET_HAS_extract_i32;
1242 case INDEX_op_sextract_i32:
1243 return TCG_TARGET_HAS_sextract_i32;
fce1296f
RH		 1244 case INDEX_op_extract2_i32:
1245 return TCG_TARGET_HAS_extract2_i32;
be0f34b5
RH		 1246 case INDEX_op_add2_i32:
1247 return TCG_TARGET_HAS_add2_i32;
1248 case INDEX_op_sub2_i32:
1249 return TCG_TARGET_HAS_sub2_i32;
1250 case INDEX_op_mulu2_i32:
1251 return TCG_TARGET_HAS_mulu2_i32;
1252 case INDEX_op_muls2_i32:
1253 return TCG_TARGET_HAS_muls2_i32;
1254 case INDEX_op_muluh_i32:
1255 return TCG_TARGET_HAS_muluh_i32;
1256 case INDEX_op_mulsh_i32:
1257 return TCG_TARGET_HAS_mulsh_i32;
1258 case INDEX_op_ext8s_i32:
1259 return TCG_TARGET_HAS_ext8s_i32;
1260 case INDEX_op_ext16s_i32:
1261 return TCG_TARGET_HAS_ext16s_i32;
1262 case INDEX_op_ext8u_i32:
1263 return TCG_TARGET_HAS_ext8u_i32;
1264 case INDEX_op_ext16u_i32:
1265 return TCG_TARGET_HAS_ext16u_i32;
1266 case INDEX_op_bswap16_i32:
1267 return TCG_TARGET_HAS_bswap16_i32;
1268 case INDEX_op_bswap32_i32:
1269 return TCG_TARGET_HAS_bswap32_i32;
1270 case INDEX_op_not_i32:
1271 return TCG_TARGET_HAS_not_i32;
1272 case INDEX_op_neg_i32:
1273 return TCG_TARGET_HAS_neg_i32;
1274 case INDEX_op_andc_i32:
1275 return TCG_TARGET_HAS_andc_i32;
1276 case INDEX_op_orc_i32:
1277 return TCG_TARGET_HAS_orc_i32;
1278 case INDEX_op_eqv_i32:
1279 return TCG_TARGET_HAS_eqv_i32;
1280 case INDEX_op_nand_i32:
1281 return TCG_TARGET_HAS_nand_i32;
1282 case INDEX_op_nor_i32:
1283 return TCG_TARGET_HAS_nor_i32;
1284 case INDEX_op_clz_i32:
1285 return TCG_TARGET_HAS_clz_i32;
1286 case INDEX_op_ctz_i32:
1287 return TCG_TARGET_HAS_ctz_i32;
1288 case INDEX_op_ctpop_i32:
1289 return TCG_TARGET_HAS_ctpop_i32;
1290
1291 case INDEX_op_brcond2_i32:
1292 case INDEX_op_setcond2_i32:
1293 return TCG_TARGET_REG_BITS == 32;
1294
1295 case INDEX_op_mov_i64:
be0f34b5
RH		 1296 case INDEX_op_setcond_i64:
1297 case INDEX_op_brcond_i64:
1298 case INDEX_op_ld8u_i64:
1299 case INDEX_op_ld8s_i64:
1300 case INDEX_op_ld16u_i64:
1301 case INDEX_op_ld16s_i64:
1302 case INDEX_op_ld32u_i64:
1303 case INDEX_op_ld32s_i64:
1304 case INDEX_op_ld_i64:
1305 case INDEX_op_st8_i64:
1306 case INDEX_op_st16_i64:
1307 case INDEX_op_st32_i64:
1308 case INDEX_op_st_i64:
1309 case INDEX_op_add_i64:
1310 case INDEX_op_sub_i64:
1311 case INDEX_op_mul_i64:
1312 case INDEX_op_and_i64:
1313 case INDEX_op_or_i64:
1314 case INDEX_op_xor_i64:
1315 case INDEX_op_shl_i64:
1316 case INDEX_op_shr_i64:
1317 case INDEX_op_sar_i64:
1318 case INDEX_op_ext_i32_i64:
1319 case INDEX_op_extu_i32_i64:
1320 return TCG_TARGET_REG_BITS == 64;
1321
1322 case INDEX_op_movcond_i64:
1323 return TCG_TARGET_HAS_movcond_i64;
1324 case INDEX_op_div_i64:
1325 case INDEX_op_divu_i64:
1326 return TCG_TARGET_HAS_div_i64;
1327 case INDEX_op_rem_i64:
1328 case INDEX_op_remu_i64:
1329 return TCG_TARGET_HAS_rem_i64;
1330 case INDEX_op_div2_i64:
1331 case INDEX_op_divu2_i64:
1332 return TCG_TARGET_HAS_div2_i64;
1333 case INDEX_op_rotl_i64:
1334 case INDEX_op_rotr_i64:
1335 return TCG_TARGET_HAS_rot_i64;
1336 case INDEX_op_deposit_i64:
1337 return TCG_TARGET_HAS_deposit_i64;
1338 case INDEX_op_extract_i64:
1339 return TCG_TARGET_HAS_extract_i64;
1340 case INDEX_op_sextract_i64:
1341 return TCG_TARGET_HAS_sextract_i64;
fce1296f
RH		 1342 case INDEX_op_extract2_i64:
1343 return TCG_TARGET_HAS_extract2_i64;
be0f34b5
RH		 1344 case INDEX_op_extrl_i64_i32:
1345 return TCG_TARGET_HAS_extrl_i64_i32;
1346 case INDEX_op_extrh_i64_i32:
1347 return TCG_TARGET_HAS_extrh_i64_i32;
1348 case INDEX_op_ext8s_i64:
1349 return TCG_TARGET_HAS_ext8s_i64;
1350 case INDEX_op_ext16s_i64:
1351 return TCG_TARGET_HAS_ext16s_i64;
1352 case INDEX_op_ext32s_i64:
1353 return TCG_TARGET_HAS_ext32s_i64;
1354 case INDEX_op_ext8u_i64:
1355 return TCG_TARGET_HAS_ext8u_i64;
1356 case INDEX_op_ext16u_i64:
1357 return TCG_TARGET_HAS_ext16u_i64;
1358 case INDEX_op_ext32u_i64:
1359 return TCG_TARGET_HAS_ext32u_i64;
1360 case INDEX_op_bswap16_i64:
1361 return TCG_TARGET_HAS_bswap16_i64;
1362 case INDEX_op_bswap32_i64:
1363 return TCG_TARGET_HAS_bswap32_i64;
1364 case INDEX_op_bswap64_i64:
1365 return TCG_TARGET_HAS_bswap64_i64;
1366 case INDEX_op_not_i64:
1367 return TCG_TARGET_HAS_not_i64;
1368 case INDEX_op_neg_i64:
1369 return TCG_TARGET_HAS_neg_i64;
1370 case INDEX_op_andc_i64:
1371 return TCG_TARGET_HAS_andc_i64;
1372 case INDEX_op_orc_i64:
1373 return TCG_TARGET_HAS_orc_i64;
1374 case INDEX_op_eqv_i64:
1375 return TCG_TARGET_HAS_eqv_i64;
1376 case INDEX_op_nand_i64:
1377 return TCG_TARGET_HAS_nand_i64;
1378 case INDEX_op_nor_i64:
1379 return TCG_TARGET_HAS_nor_i64;
1380 case INDEX_op_clz_i64:
1381 return TCG_TARGET_HAS_clz_i64;
1382 case INDEX_op_ctz_i64:
1383 return TCG_TARGET_HAS_ctz_i64;
1384 case INDEX_op_ctpop_i64:
1385 return TCG_TARGET_HAS_ctpop_i64;
1386 case INDEX_op_add2_i64:
1387 return TCG_TARGET_HAS_add2_i64;
1388 case INDEX_op_sub2_i64:
1389 return TCG_TARGET_HAS_sub2_i64;
1390 case INDEX_op_mulu2_i64:
1391 return TCG_TARGET_HAS_mulu2_i64;
1392 case INDEX_op_muls2_i64:
1393 return TCG_TARGET_HAS_muls2_i64;
1394 case INDEX_op_muluh_i64:
1395 return TCG_TARGET_HAS_muluh_i64;
1396 case INDEX_op_mulsh_i64:
1397 return TCG_TARGET_HAS_mulsh_i64;
1398
d2fd745f
RH		 1399 case INDEX_op_mov_vec:
1400 case INDEX_op_dup_vec:
37ee55a0 1401 case INDEX_op_dupm_vec:
d2fd745f
RH		 1402 case INDEX_op_ld_vec:
1403 case INDEX_op_st_vec:
1404 case INDEX_op_add_vec:
1405 case INDEX_op_sub_vec:
1406 case INDEX_op_and_vec:
1407 case INDEX_op_or_vec:
1408 case INDEX_op_xor_vec:
212be173 1409 case INDEX_op_cmp_vec:
d2fd745f
RH		 1410 return have_vec;
1411 case INDEX_op_dup2_vec:
1412 return have_vec && TCG_TARGET_REG_BITS == 32;
1413 case INDEX_op_not_vec:
1414 return have_vec && TCG_TARGET_HAS_not_vec;
1415 case INDEX_op_neg_vec:
1416 return have_vec && TCG_TARGET_HAS_neg_vec;
bcefc902
RH		 1417 case INDEX_op_abs_vec:
1418 return have_vec && TCG_TARGET_HAS_abs_vec;
d2fd745f
RH		 1419 case INDEX_op_andc_vec:
1420 return have_vec && TCG_TARGET_HAS_andc_vec;
1421 case INDEX_op_orc_vec:
1422 return have_vec && TCG_TARGET_HAS_orc_vec;
ed523473
RH		 1423 case INDEX_op_nand_vec:
1424 return have_vec && TCG_TARGET_HAS_nand_vec;
1425 case INDEX_op_nor_vec:
1426 return have_vec && TCG_TARGET_HAS_nor_vec;
1427 case INDEX_op_eqv_vec:
1428 return have_vec && TCG_TARGET_HAS_eqv_vec;
3774030a
RH		 1429 case INDEX_op_mul_vec:
1430 return have_vec && TCG_TARGET_HAS_mul_vec;
d0ec9796
RH		 1431 case INDEX_op_shli_vec:
1432 case INDEX_op_shri_vec:
1433 case INDEX_op_sari_vec:
1434 return have_vec && TCG_TARGET_HAS_shi_vec;
1435 case INDEX_op_shls_vec:
1436 case INDEX_op_shrs_vec:
1437 case INDEX_op_sars_vec:
1438 return have_vec && TCG_TARGET_HAS_shs_vec;
1439 case INDEX_op_shlv_vec:
1440 case INDEX_op_shrv_vec:
1441 case INDEX_op_sarv_vec:
1442 return have_vec && TCG_TARGET_HAS_shv_vec;
b0f7e744
RH		 1443 case INDEX_op_rotli_vec:
1444 return have_vec && TCG_TARGET_HAS_roti_vec;
23850a74
RH		 1445 case INDEX_op_rotls_vec:
1446 return have_vec && TCG_TARGET_HAS_rots_vec;
5d0ceda9
RH		 1447 case INDEX_op_rotlv_vec:
1448 case INDEX_op_rotrv_vec:
1449 return have_vec && TCG_TARGET_HAS_rotv_vec;
8afaf050
RH		 1450 case INDEX_op_ssadd_vec:
1451 case INDEX_op_usadd_vec:
1452 case INDEX_op_sssub_vec:
1453 case INDEX_op_ussub_vec:
1454 return have_vec && TCG_TARGET_HAS_sat_vec;
dd0a0fcd
RH		 1455 case INDEX_op_smin_vec:
1456 case INDEX_op_umin_vec:
1457 case INDEX_op_smax_vec:
1458 case INDEX_op_umax_vec:
1459 return have_vec && TCG_TARGET_HAS_minmax_vec;
38dc1294
RH		 1460 case INDEX_op_bitsel_vec:
1461 return have_vec && TCG_TARGET_HAS_bitsel_vec;
f75da298
RH		 1462 case INDEX_op_cmpsel_vec:
1463 return have_vec && TCG_TARGET_HAS_cmpsel_vec;
d2fd745f 1464
db432672
RH		 1465 default:
1466 tcg_debug_assert(op > INDEX_op_last_generic && op < NB_OPS);
1467 return true;
be0f34b5 1468 }
be0f34b5
RH		 1469}
1470
39cf05d3
FB		 1471/* Note: we convert the 64 bit args to 32 bit and do some alignment
1472 and endian swap. Maybe it would be better to do the alignment
1473 and endian swap in tcg_reg_alloc_call(). */
ae8b75dc 1474void tcg_gen_callN(void *func, TCGTemp *ret, int nargs, TCGTemp **args)
c896fe29 1475{
75e8b9b7 1476 int i, real_args, nb_rets, pi;
3e92aa34
RH		 1477 unsigned typemask;
1478 const TCGHelperInfo *info;
75e8b9b7 1479 TCGOp *op;
afb49896 1480
619205fd 1481 info = g_hash_table_lookup(helper_table, (gpointer)func);
7319d83a 1482 typemask = info->typemask;
2bece2c8 1483
38b47b19
EC		 1484#ifdef CONFIG_PLUGIN
1485 /* detect non-plugin helpers */
1486 if (tcg_ctx->plugin_insn && unlikely(strncmp(info->name, "plugin_", 7))) {
1487 tcg_ctx->plugin_insn->calls_helpers = true;
1488 }
1489#endif
1490
3a5f6805 1491#if defined(TCG_TARGET_EXTEND_ARGS) && TCG_TARGET_REG_BITS == 64
2bece2c8 1492 for (i = 0; i < nargs; ++i) {
7319d83a
RH		 1493 int argtype = extract32(typemask, (i + 1) * 3, 3);
1494 bool is_32bit = (argtype & ~1) == dh_typecode_i32;
1495 bool is_signed = argtype & 1;
1496
1497 if (is_32bit) {
2bece2c8 1498 TCGv_i64 temp = tcg_temp_new_i64();
18cf3d07 1499 TCGv_i32 orig = temp_tcgv_i32(args[i]);
2bece2c8 1500 if (is_signed) {
18cf3d07 1501 tcg_gen_ext_i32_i64(temp, orig);
2bece2c8 1502 } else {
18cf3d07 1503 tcg_gen_extu_i32_i64(temp, orig);
2bece2c8 1504 }
ae8b75dc 1505 args[i] = tcgv_i64_temp(temp);
2bece2c8
RH		 1506 }
1507 }
1508#endif /* TCG_TARGET_EXTEND_ARGS */
1509
15fa08f8 1510 op = tcg_emit_op(INDEX_op_call);
75e8b9b7
RH		 1511
1512 pi = 0;
ae8b75dc 1513 if (ret != NULL) {
7319d83a 1514 if (TCG_TARGET_REG_BITS < 64 && (typemask & 6) == dh_typecode_i64) {
e03b5686 1515#if HOST_BIG_ENDIAN
ae8b75dc
RH		 1516 op->args[pi++] = temp_arg(ret + 1);
1517 op->args[pi++] = temp_arg(ret);
39cf05d3 1518#else
ae8b75dc
RH		 1519 op->args[pi++] = temp_arg(ret);
1520 op->args[pi++] = temp_arg(ret + 1);
39cf05d3 1521#endif
a7812ae4 1522 nb_rets = 2;
34b1a49c 1523 } else {
ae8b75dc 1524 op->args[pi++] = temp_arg(ret);
a7812ae4 1525 nb_rets = 1;
c896fe29 1526 }
a7812ae4
PB		 1527 } else {
1528 nb_rets = 0;
c896fe29 1529 }
cd9090aa 1530 TCGOP_CALLO(op) = nb_rets;
75e8b9b7 1531
a7812ae4
PB		 1532 real_args = 0;
1533 for (i = 0; i < nargs; i++) {
7319d83a
RH		 1534 int argtype = extract32(typemask, (i + 1) * 3, 3);
1535 bool is_64bit = (argtype & ~1) == dh_typecode_i64;
7b7d8b2d 1536 bool want_align = false;
7319d83a 1537
7b7d8b2d
RH		 1538#if defined(CONFIG_TCG_INTERPRETER)
1539 /*
1540 * Align all arguments, so that they land in predictable places
1541 * for passing off to ffi_call.
1542 */
1543 want_align = true;
1544#elif defined(TCG_TARGET_CALL_ALIGN_ARGS)
1545 /* Some targets want aligned 64 bit args */
1546 want_align = is_64bit;
39cf05d3 1547#endif
7b7d8b2d
RH		 1548
1549 if (TCG_TARGET_REG_BITS < 64 && want_align && (real_args & 1)) {
1550 op->args[pi++] = TCG_CALL_DUMMY_ARG;
1551 real_args++;
1552 }
1553
1554 if (TCG_TARGET_REG_BITS < 64 && is_64bit) {
8d21de51
RH		 1555 op->args[pi++] = temp_arg(args[i] + HOST_BIG_ENDIAN);
1556 op->args[pi++] = temp_arg(args[i] + !HOST_BIG_ENDIAN);
a7812ae4 1557 real_args += 2;
2bece2c8 1558 continue;
c896fe29 1559 }
2bece2c8 1560
ae8b75dc 1561 op->args[pi++] = temp_arg(args[i]);
2bece2c8 1562 real_args++;
c896fe29 1563 }
75e8b9b7 1564 op->args[pi++] = (uintptr_t)func;
3e92aa34 1565 op->args[pi++] = (uintptr_t)info;
cd9090aa 1566 TCGOP_CALLI(op) = real_args;
a7812ae4 1567
75e8b9b7 1568 /* Make sure the fields didn't overflow. */
cd9090aa 1569 tcg_debug_assert(TCGOP_CALLI(op) == real_args);
75e8b9b7 1570 tcg_debug_assert(pi <= ARRAY_SIZE(op->args));
2bece2c8 1571
3a5f6805 1572#if defined(TCG_TARGET_EXTEND_ARGS) && TCG_TARGET_REG_BITS == 64
2bece2c8 1573 for (i = 0; i < nargs; ++i) {
7319d83a
RH		 1574 int argtype = extract32(typemask, (i + 1) * 3, 3);
1575 bool is_32bit = (argtype & ~1) == dh_typecode_i32;
1576
1577 if (is_32bit) {
085272b3 1578 tcg_temp_free_internal(args[i]);
2bece2c8
RH		 1579 }
1580 }
1581#endif /* TCG_TARGET_EXTEND_ARGS */
c896fe29 1582}
c896fe29 1583
8fcd3692 1584static void tcg_reg_alloc_start(TCGContext *s)
c896fe29 1585{
ac3b8891 1586 int i, n;
ac3b8891 1587
ee17db83
RH		 1588 for (i = 0, n = s->nb_temps; i < n; i++) {
1589 TCGTemp *ts = &s->temps[i];
1590 TCGTempVal val = TEMP_VAL_MEM;
1591
1592 switch (ts->kind) {
c0522136
RH		 1593 case TEMP_CONST:
1594 val = TEMP_VAL_CONST;
1595 break;
ee17db83
RH		 1596 case TEMP_FIXED:
1597 val = TEMP_VAL_REG;
1598 break;
1599 case TEMP_GLOBAL:
1600 break;
1601 case TEMP_NORMAL:
c7482438 1602 case TEMP_EBB:
ee17db83
RH		 1603 val = TEMP_VAL_DEAD;
1604 /* fall through */
1605 case TEMP_LOCAL:
1606 ts->mem_allocated = 0;
1607 break;
1608 default:
1609 g_assert_not_reached();
1610 }
1611 ts->val_type = val;
e8996ee0 1612 }
f8b2f202
RH		 1613
1614 memset(s->reg_to_temp, 0, sizeof(s->reg_to_temp));
c896fe29
FB		 1615}
1616
f8b2f202
RH		 1617static char *tcg_get_arg_str_ptr(TCGContext *s, char *buf, int buf_size,
1618 TCGTemp *ts)
c896fe29 1619{
1807f4c4 1620 int idx = temp_idx(ts);
ac56dd48 1621
ee17db83
RH		 1622 switch (ts->kind) {
1623 case TEMP_FIXED:
1624 case TEMP_GLOBAL:
ac56dd48 1625 pstrcpy(buf, buf_size, ts->name);
ee17db83
RH		 1626 break;
1627 case TEMP_LOCAL:
f8b2f202 1628 snprintf(buf, buf_size, "loc%d", idx - s->nb_globals);
ee17db83 1629 break;
c7482438
RH		 1630 case TEMP_EBB:
1631 snprintf(buf, buf_size, "ebb%d", idx - s->nb_globals);
1632 break;
ee17db83 1633 case TEMP_NORMAL:
f8b2f202 1634 snprintf(buf, buf_size, "tmp%d", idx - s->nb_globals);
ee17db83 1635 break;
c0522136
RH		 1636 case TEMP_CONST:
1637 switch (ts->type) {
1638 case TCG_TYPE_I32:
1639 snprintf(buf, buf_size, "$0x%x", (int32_t)ts->val);
1640 break;
1641#if TCG_TARGET_REG_BITS > 32
1642 case TCG_TYPE_I64:
1643 snprintf(buf, buf_size, "$0x%" PRIx64, ts->val);
1644 break;
1645#endif
1646 case TCG_TYPE_V64:
1647 case TCG_TYPE_V128:
1648 case TCG_TYPE_V256:
1649 snprintf(buf, buf_size, "v%d$0x%" PRIx64,
1650 64 << (ts->type - TCG_TYPE_V64), ts->val);
1651 break;
1652 default:
1653 g_assert_not_reached();
1654 }
1655 break;
c896fe29
FB		 1656 }
1657 return buf;
1658}
1659
43439139
RH		 1660static char *tcg_get_arg_str(TCGContext *s, char *buf,
1661 int buf_size, TCGArg arg)
f8b2f202 1662{
43439139 1663 return tcg_get_arg_str_ptr(s, buf, buf_size, arg_temp(arg));
f8b2f202
RH		 1664}
1665
f48f3ede
BS		 1666static const char * const cond_name[] =
1667{
0aed257f
RH		 1668 [TCG_COND_NEVER] = "never",
1669 [TCG_COND_ALWAYS] = "always",
f48f3ede
BS		 1670 [TCG_COND_EQ] = "eq",
1671 [TCG_COND_NE] = "ne",
1672 [TCG_COND_LT] = "lt",
1673 [TCG_COND_GE] = "ge",
1674 [TCG_COND_LE] = "le",
1675 [TCG_COND_GT] = "gt",
1676 [TCG_COND_LTU] = "ltu",
1677 [TCG_COND_GEU] = "geu",
1678 [TCG_COND_LEU] = "leu",
1679 [TCG_COND_GTU] = "gtu"
1680};
1681
f713d6ad
RH		 1682static const char * const ldst_name[] =
1683{
1684 [MO_UB] = "ub",
1685 [MO_SB] = "sb",
1686 [MO_LEUW] = "leuw",
1687 [MO_LESW] = "lesw",
1688 [MO_LEUL] = "leul",
1689 [MO_LESL] = "lesl",
fc313c64 1690 [MO_LEUQ] = "leq",
f713d6ad
RH		 1691 [MO_BEUW] = "beuw",
1692 [MO_BESW] = "besw",
1693 [MO_BEUL] = "beul",
1694 [MO_BESL] = "besl",
fc313c64 1695 [MO_BEUQ] = "beq",
f713d6ad
RH		 1696};
1697
1f00b27f 1698static const char * const alignment_name[(MO_AMASK >> MO_ASHIFT) + 1] = {
52bf9771 1699#ifdef TARGET_ALIGNED_ONLY
1f00b27f
SS		 1700 [MO_UNALN >> MO_ASHIFT] = "un+",
1701 [MO_ALIGN >> MO_ASHIFT] = "",
1702#else
1703 [MO_UNALN >> MO_ASHIFT] = "",
1704 [MO_ALIGN >> MO_ASHIFT] = "al+",
1705#endif
1706 [MO_ALIGN_2 >> MO_ASHIFT] = "al2+",
1707 [MO_ALIGN_4 >> MO_ASHIFT] = "al4+",
1708 [MO_ALIGN_8 >> MO_ASHIFT] = "al8+",
1709 [MO_ALIGN_16 >> MO_ASHIFT] = "al16+",
1710 [MO_ALIGN_32 >> MO_ASHIFT] = "al32+",
1711 [MO_ALIGN_64 >> MO_ASHIFT] = "al64+",
1712};
1713
587195bd
RH		 1714static const char bswap_flag_name[][6] = {
1715 [TCG_BSWAP_IZ] = "iz",
1716 [TCG_BSWAP_OZ] = "oz",
1717 [TCG_BSWAP_OS] = "os",
1718 [TCG_BSWAP_IZ | TCG_BSWAP_OZ] = "iz,oz",
1719 [TCG_BSWAP_IZ | TCG_BSWAP_OS] = "iz,os",
1720};
1721
b016486e
RH		 1722static inline bool tcg_regset_single(TCGRegSet d)
1723{
1724 return (d & (d - 1)) == 0;
1725}
1726
1727static inline TCGReg tcg_regset_first(TCGRegSet d)
1728{
1729 if (TCG_TARGET_NB_REGS <= 32) {
1730 return ctz32(d);
1731 } else {
1732 return ctz64(d);
1733 }
1734}
1735
b7a83ff8
RH		 1736/* Return only the number of characters output -- no error return. */
1737#define ne_fprintf(...) \
1738 ({ int ret_ = fprintf(__VA_ARGS__); ret_ >= 0 ? ret_ : 0; })
1739
1740static void tcg_dump_ops(TCGContext *s, FILE *f, bool have_prefs)
c896fe29 1741{
c896fe29 1742 char buf[128];
c45cb8bb 1743 TCGOp *op;
c45cb8bb 1744
15fa08f8 1745 QTAILQ_FOREACH(op, &s->ops, link) {
c45cb8bb
RH		 1746 int i, k, nb_oargs, nb_iargs, nb_cargs;
1747 const TCGOpDef *def;
c45cb8bb 1748 TCGOpcode c;
bdfb460e 1749 int col = 0;
c896fe29 1750
c45cb8bb 1751 c = op->opc;
c896fe29 1752 def = &tcg_op_defs[c];
c45cb8bb 1753
765b842a 1754 if (c == INDEX_op_insn_start) {
b016486e 1755 nb_oargs = 0;
b7a83ff8 1756 col += ne_fprintf(f, "\n ----");
9aef40ed
RH		 1757
1758 for (i = 0; i < TARGET_INSN_START_WORDS; ++i) {
1759 target_ulong a;
7e4597d7 1760#if TARGET_LONG_BITS > TCG_TARGET_REG_BITS
efee3746 1761 a = deposit64(op->args[i * 2], 32, 32, op->args[i * 2 + 1]);
7e4597d7 1762#else
efee3746 1763 a = op->args[i];
7e4597d7 1764#endif
b7a83ff8 1765 col += ne_fprintf(f, " " TARGET_FMT_lx, a);
eeacee4d 1766 }
7e4597d7 1767 } else if (c == INDEX_op_call) {
3e92aa34 1768 const TCGHelperInfo *info = tcg_call_info(op);
fa52e660 1769 void *func = tcg_call_func(op);
3e92aa34 1770
c896fe29 1771 /* variable number of arguments */
cd9090aa
RH		 1772 nb_oargs = TCGOP_CALLO(op);
1773 nb_iargs = TCGOP_CALLI(op);
c896fe29 1774 nb_cargs = def->nb_cargs;
c896fe29 1775
b7a83ff8 1776 col += ne_fprintf(f, " %s ", def->name);
3e92aa34
RH		 1777
1778 /*
1779 * Print the function name from TCGHelperInfo, if available.
1780 * Note that plugins have a template function for the info,
1781 * but the actual function pointer comes from the plugin.
1782 */
3e92aa34 1783 if (func == info->func) {
b7a83ff8 1784 col += ne_fprintf(f, "%s", info->name);
3e92aa34 1785 } else {
b7a83ff8 1786 col += ne_fprintf(f, "plugin(%p)", func);
3e92aa34
RH		 1787 }
1788
b7a83ff8 1789 col += ne_fprintf(f, ",$0x%x,$%d", info->flags, nb_oargs);
cf066674 1790 for (i = 0; i < nb_oargs; i++) {
b7a83ff8
RH		 1791 col += ne_fprintf(f, ",%s", tcg_get_arg_str(s, buf, sizeof(buf),
1792 op->args[i]));
b03cce8e 1793 }
cf066674 1794 for (i = 0; i < nb_iargs; i++) {
efee3746 1795 TCGArg arg = op->args[nb_oargs + i];
cf066674
RH		 1796 const char *t = "<dummy>";
1797 if (arg != TCG_CALL_DUMMY_ARG) {
43439139 1798 t = tcg_get_arg_str(s, buf, sizeof(buf), arg);
eeacee4d 1799 }
b7a83ff8 1800 col += ne_fprintf(f, ",%s", t);
e8996ee0 1801 }
b03cce8e 1802 } else {
b7a83ff8 1803 col += ne_fprintf(f, " %s ", def->name);
c45cb8bb
RH		 1804
1805 nb_oargs = def->nb_oargs;
1806 nb_iargs = def->nb_iargs;
1807 nb_cargs = def->nb_cargs;
1808
d2fd745f 1809 if (def->flags & TCG_OPF_VECTOR) {
b7a83ff8
RH		 1810 col += ne_fprintf(f, "v%d,e%d,", 64 << TCGOP_VECL(op),
1811 8 << TCGOP_VECE(op));
d2fd745f
RH		 1812 }
1813
b03cce8e 1814 k = 0;
c45cb8bb 1815 for (i = 0; i < nb_oargs; i++) {
b7a83ff8
RH		 1816 const char *sep = k ? "," : "";
1817 col += ne_fprintf(f, "%s%s", sep,
1818 tcg_get_arg_str(s, buf, sizeof(buf),
1819 op->args[k++]));
b03cce8e 1820 }
c45cb8bb 1821 for (i = 0; i < nb_iargs; i++) {
b7a83ff8
RH		 1822 const char *sep = k ? "," : "";
1823 col += ne_fprintf(f, "%s%s", sep,
1824 tcg_get_arg_str(s, buf, sizeof(buf),
1825 op->args[k++]));
b03cce8e 1826 }
be210acb
RH		 1827 switch (c) {
1828 case INDEX_op_brcond_i32:
be210acb 1829 case INDEX_op_setcond_i32:
ffc5ea09 1830 case INDEX_op_movcond_i32:
ffc5ea09 1831 case INDEX_op_brcond2_i32:
be210acb 1832 case INDEX_op_setcond2_i32:
ffc5ea09 1833 case INDEX_op_brcond_i64:
be210acb 1834 case INDEX_op_setcond_i64:
ffc5ea09 1835 case INDEX_op_movcond_i64:
212be173 1836 case INDEX_op_cmp_vec:
f75da298 1837 case INDEX_op_cmpsel_vec:
efee3746
RH		 1838 if (op->args[k] < ARRAY_SIZE(cond_name)
1839 && cond_name[op->args[k]]) {
b7a83ff8 1840 col += ne_fprintf(f, ",%s", cond_name[op->args[k++]]);
eeacee4d 1841 } else {
b7a83ff8 1842 col += ne_fprintf(f, ",$0x%" TCG_PRIlx, op->args[k++]);
eeacee4d 1843 }
f48f3ede 1844 i = 1;
be210acb 1845 break;
f713d6ad
RH		 1846 case INDEX_op_qemu_ld_i32:
1847 case INDEX_op_qemu_st_i32:
07ce0b05 1848 case INDEX_op_qemu_st8_i32:
f713d6ad
RH		 1849 case INDEX_op_qemu_ld_i64:
1850 case INDEX_op_qemu_st_i64:
59227d5d 1851 {
9002ffcb 1852 MemOpIdx oi = op->args[k++];
14776ab5 1853 MemOp op = get_memop(oi);
59227d5d
RH		 1854 unsigned ix = get_mmuidx(oi);
1855
59c4b7e8 1856 if (op & ~(MO_AMASK | MO_BSWAP | MO_SSIZE)) {
b7a83ff8 1857 col += ne_fprintf(f, ",$0x%x,%u", op, ix);
59c4b7e8 1858 } else {
1f00b27f
SS		 1859 const char *s_al, *s_op;
1860 s_al = alignment_name[(op & MO_AMASK) >> MO_ASHIFT];
59c4b7e8 1861 s_op = ldst_name[op & (MO_BSWAP | MO_SSIZE)];
b7a83ff8 1862 col += ne_fprintf(f, ",%s%s,%u", s_al, s_op, ix);
59227d5d
RH		 1863 }
1864 i = 1;
f713d6ad 1865 }
f713d6ad 1866 break;
587195bd
RH		 1867 case INDEX_op_bswap16_i32:
1868 case INDEX_op_bswap16_i64:
1869 case INDEX_op_bswap32_i32:
1870 case INDEX_op_bswap32_i64:
1871 case INDEX_op_bswap64_i64:
1872 {
1873 TCGArg flags = op->args[k];
1874 const char *name = NULL;
1875
1876 if (flags < ARRAY_SIZE(bswap_flag_name)) {
1877 name = bswap_flag_name[flags];
1878 }
1879 if (name) {
b7a83ff8 1880 col += ne_fprintf(f, ",%s", name);
587195bd 1881 } else {
b7a83ff8 1882 col += ne_fprintf(f, ",$0x%" TCG_PRIlx, flags);
587195bd
RH		 1883 }
1884 i = k = 1;
1885 }
1886 break;
be210acb 1887 default:
f48f3ede 1888 i = 0;
be210acb
RH		 1889 break;
1890 }
51e3972c
RH		 1891 switch (c) {
1892 case INDEX_op_set_label:
1893 case INDEX_op_br:
1894 case INDEX_op_brcond_i32:
1895 case INDEX_op_brcond_i64:
1896 case INDEX_op_brcond2_i32:
b7a83ff8
RH		 1897 col += ne_fprintf(f, "%s$L%d", k ? "," : "",
1898 arg_label(op->args[k])->id);
51e3972c
RH		 1899 i++, k++;
1900 break;
1901 default:
1902 break;
1903 }
1904 for (; i < nb_cargs; i++, k++) {
b7a83ff8
RH		 1905 col += ne_fprintf(f, "%s$0x%" TCG_PRIlx, k ? "," : "",
1906 op->args[k]);
bdfb460e
RH		 1907 }
1908 }
bdfb460e 1909
1894f69a 1910 if (have_prefs || op->life) {
b7a83ff8
RH		 1911 for (; col < 40; ++col) {
1912 putc(' ', f);
bdfb460e 1913 }
1894f69a
RH		 1914 }
1915
1916 if (op->life) {
1917 unsigned life = op->life;
bdfb460e
RH		 1918
1919 if (life & (SYNC_ARG * 3)) {
b7a83ff8 1920 ne_fprintf(f, " sync:");
bdfb460e
RH		 1921 for (i = 0; i < 2; ++i) {
1922 if (life & (SYNC_ARG << i)) {
b7a83ff8 1923 ne_fprintf(f, " %d", i);
bdfb460e
RH		 1924 }
1925 }
1926 }
1927 life /= DEAD_ARG;
1928 if (life) {
b7a83ff8 1929 ne_fprintf(f, " dead:");
bdfb460e
RH		 1930 for (i = 0; life; ++i, life >>= 1) {
1931 if (life & 1) {
b7a83ff8 1932 ne_fprintf(f, " %d", i);
bdfb460e
RH		 1933 }
1934 }
b03cce8e 1935 }
c896fe29 1936 }
1894f69a
RH		 1937
1938 if (have_prefs) {
1939 for (i = 0; i < nb_oargs; ++i) {
1940 TCGRegSet set = op->output_pref[i];
1941
1942 if (i == 0) {
b7a83ff8 1943 ne_fprintf(f, " pref=");
1894f69a 1944 } else {
b7a83ff8 1945 ne_fprintf(f, ",");
1894f69a
RH		 1946 }
1947 if (set == 0) {
b7a83ff8 1948 ne_fprintf(f, "none");
1894f69a 1949 } else if (set == MAKE_64BIT_MASK(0, TCG_TARGET_NB_REGS)) {
b7a83ff8 1950 ne_fprintf(f, "all");
1894f69a
RH		 1951#ifdef CONFIG_DEBUG_TCG
1952 } else if (tcg_regset_single(set)) {
1953 TCGReg reg = tcg_regset_first(set);
b7a83ff8 1954 ne_fprintf(f, "%s", tcg_target_reg_names[reg]);
1894f69a
RH		 1955#endif
1956 } else if (TCG_TARGET_NB_REGS <= 32) {
b7a83ff8 1957 ne_fprintf(f, "0x%x", (uint32_t)set);
1894f69a 1958 } else {
b7a83ff8 1959 ne_fprintf(f, "0x%" PRIx64, (uint64_t)set);
1894f69a
RH		 1960 }
1961 }
1962 }
1963
b7a83ff8 1964 putc('\n', f);
c896fe29
FB		 1965 }
1966}
1967
1968/* we give more priority to constraints with less registers */
1969static int get_constraint_priority(const TCGOpDef *def, int k)
1970{
74a11790
RH		 1971 const TCGArgConstraint *arg_ct = &def->args_ct[k];
1972 int n;
c896fe29 1973
bc2b17e6 1974 if (arg_ct->oalias) {
c896fe29
FB		 1975 /* an alias is equivalent to a single register */
1976 n = 1;
1977 } else {
74a11790 1978 n = ctpop64(arg_ct->regs);
c896fe29
FB		 1979 }
1980 return TCG_TARGET_NB_REGS - n + 1;
1981}
1982
1983/* sort from highest priority to lowest */
1984static void sort_constraints(TCGOpDef *def, int start, int n)
1985{
66792f90
RH		 1986 int i, j;
1987 TCGArgConstraint *a = def->args_ct;
c896fe29 1988
66792f90
RH		 1989 for (i = 0; i < n; i++) {
1990 a[start + i].sort_index = start + i;
1991 }
1992 if (n <= 1) {
c896fe29 1993 return;
66792f90
RH		 1994 }
1995 for (i = 0; i < n - 1; i++) {
1996 for (j = i + 1; j < n; j++) {
1997 int p1 = get_constraint_priority(def, a[start + i].sort_index);
1998 int p2 = get_constraint_priority(def, a[start + j].sort_index);
c896fe29 1999 if (p1 < p2) {
66792f90
RH		 2000 int tmp = a[start + i].sort_index;
2001 a[start + i].sort_index = a[start + j].sort_index;
2002 a[start + j].sort_index = tmp;
c896fe29
FB		 2003 }
2004 }
2005 }
2006}
2007
f69d277e 2008static void process_op_defs(TCGContext *s)
c896fe29 2009{
a9751609 2010 TCGOpcode op;
c896fe29 2011
f69d277e
RH		 2012 for (op = 0; op < NB_OPS; op++) {
2013 TCGOpDef *def = &tcg_op_defs[op];
2014 const TCGTargetOpDef *tdefs;
069ea736 2015 int i, nb_args;
f69d277e
RH		 2016
2017 if (def->flags & TCG_OPF_NOT_PRESENT) {
2018 continue;
2019 }
2020
c896fe29 2021 nb_args = def->nb_iargs + def->nb_oargs;
f69d277e
RH		 2022 if (nb_args == 0) {
2023 continue;
2024 }
2025
4c22e840
RH		 2026 /*
2027 * Macro magic should make it impossible, but double-check that
2028 * the array index is in range. Since the signness of an enum
2029 * is implementation defined, force the result to unsigned.
2030 */
2031 unsigned con_set = tcg_target_op_def(op);
2032 tcg_debug_assert(con_set < ARRAY_SIZE(constraint_sets));
2033 tdefs = &constraint_sets[con_set];
f69d277e
RH		 2034
2035 for (i = 0; i < nb_args; i++) {
2036 const char *ct_str = tdefs->args_ct_str[i];
2037 /* Incomplete TCGTargetOpDef entry. */
eabb7b91 2038 tcg_debug_assert(ct_str != NULL);
f69d277e 2039
17280ff4
RH		 2040 while (*ct_str != '\0') {
2041 switch(*ct_str) {
2042 case '0' ... '9':
2043 {
2044 int oarg = *ct_str - '0';
2045 tcg_debug_assert(ct_str == tdefs->args_ct_str[i]);
2046 tcg_debug_assert(oarg < def->nb_oargs);
74a11790 2047 tcg_debug_assert(def->args_ct[oarg].regs != 0);
17280ff4 2048 def->args_ct[i] = def->args_ct[oarg];
bc2b17e6
RH		 2049 /* The output sets oalias. */
2050 def->args_ct[oarg].oalias = true;
17280ff4 2051 def->args_ct[oarg].alias_index = i;
bc2b17e6
RH		 2052 /* The input sets ialias. */
2053 def->args_ct[i].ialias = true;
17280ff4 2054 def->args_ct[i].alias_index = oarg;
c896fe29 2055 }
17280ff4
RH		 2056 ct_str++;
2057 break;
2058 case '&':
bc2b17e6 2059 def->args_ct[i].newreg = true;
17280ff4
RH		 2060 ct_str++;
2061 break;
2062 case 'i':
2063 def->args_ct[i].ct |= TCG_CT_CONST;
2064 ct_str++;
2065 break;
358b4923 2066
358b4923
RH		 2067 /* Include all of the target-specific constraints. */
2068
2069#undef CONST
2070#define CONST(CASE, MASK) \
2071 case CASE: def->args_ct[i].ct |= MASK; ct_str++; break;
2072#define REGS(CASE, MASK) \
2073 case CASE: def->args_ct[i].regs |= MASK; ct_str++; break;
2074
2075#include "tcg-target-con-str.h"
2076
2077#undef REGS
2078#undef CONST
17280ff4 2079 default:
17280ff4 2080 /* Typo in TCGTargetOpDef constraint. */
358b4923 2081 g_assert_not_reached();
c896fe29
FB		 2082 }
2083 }
2084 }
2085
c68aaa18 2086 /* TCGTargetOpDef entry with too much information? */
eabb7b91 2087 tcg_debug_assert(i == TCG_MAX_OP_ARGS || tdefs->args_ct_str[i] == NULL);
c68aaa18 2088
c896fe29
FB		 2089 /* sort the constraints (XXX: this is just an heuristic) */
2090 sort_constraints(def, 0, def->nb_oargs);
2091 sort_constraints(def, def->nb_oargs, def->nb_iargs);
a9751609 2092 }
c896fe29
FB		 2093}
2094
0c627cdc
RH		 2095void tcg_op_remove(TCGContext *s, TCGOp *op)
2096{
d88a117e
RH		 2097 TCGLabel *label;
2098
2099 switch (op->opc) {
2100 case INDEX_op_br:
2101 label = arg_label(op->args[0]);
2102 label->refs--;
2103 break;
2104 case INDEX_op_brcond_i32:
2105 case INDEX_op_brcond_i64:
2106 label = arg_label(op->args[3]);
2107 label->refs--;
2108 break;
2109 case INDEX_op_brcond2_i32:
2110 label = arg_label(op->args[5]);
2111 label->refs--;
2112 break;
2113 default:
2114 break;
2115 }
2116
15fa08f8
RH		 2117 QTAILQ_REMOVE(&s->ops, op, link);
2118 QTAILQ_INSERT_TAIL(&s->free_ops, op, link);
abebf925 2119 s->nb_ops--;
0c627cdc
RH		 2120
2121#ifdef CONFIG_PROFILER
d73415a3 2122 qatomic_set(&s->prof.del_op_count, s->prof.del_op_count + 1);
0c627cdc
RH		 2123#endif
2124}
2125
a80cdd31
RH		 2126void tcg_remove_ops_after(TCGOp *op)
2127{
2128 TCGContext *s = tcg_ctx;
2129
2130 while (true) {
2131 TCGOp *last = tcg_last_op();
2132 if (last == op) {
2133 return;
2134 }
2135 tcg_op_remove(s, last);
2136 }
2137}
2138
15fa08f8 2139static TCGOp *tcg_op_alloc(TCGOpcode opc)
5a18407f 2140{
15fa08f8
RH		 2141 TCGContext *s = tcg_ctx;
2142 TCGOp *op;
5a18407f 2143
15fa08f8
RH		 2144 if (likely(QTAILQ_EMPTY(&s->free_ops))) {
2145 op = tcg_malloc(sizeof(TCGOp));
2146 } else {
2147 op = QTAILQ_FIRST(&s->free_ops);
2148 QTAILQ_REMOVE(&s->free_ops, op, link);
2149 }
2150 memset(op, 0, offsetof(TCGOp, link));
2151 op->opc = opc;
abebf925 2152 s->nb_ops++;
5a18407f 2153
15fa08f8
RH		 2154 return op;
2155}
2156
2157TCGOp *tcg_emit_op(TCGOpcode opc)
2158{
2159 TCGOp *op = tcg_op_alloc(opc);
2160 QTAILQ_INSERT_TAIL(&tcg_ctx->ops, op, link);
2161 return op;
2162}
5a18407f 2163
ac1043f6 2164TCGOp *tcg_op_insert_before(TCGContext *s, TCGOp *old_op, TCGOpcode opc)
15fa08f8
RH		 2165{
2166 TCGOp *new_op = tcg_op_alloc(opc);
2167 QTAILQ_INSERT_BEFORE(old_op, new_op, link);
5a18407f
RH		 2168 return new_op;
2169}
2170
ac1043f6 2171TCGOp *tcg_op_insert_after(TCGContext *s, TCGOp *old_op, TCGOpcode opc)
5a18407f 2172{
15fa08f8
RH		 2173 TCGOp *new_op = tcg_op_alloc(opc);
2174 QTAILQ_INSERT_AFTER(&s->ops, old_op, new_op, link);
5a18407f
RH		 2175 return new_op;
2176}
2177
b4fc67c7
RH		 2178/* Reachable analysis : remove unreachable code. */
2179static void reachable_code_pass(TCGContext *s)
2180{
2181 TCGOp *op, *op_next;
2182 bool dead = false;
2183
2184 QTAILQ_FOREACH_SAFE(op, &s->ops, link, op_next) {
2185 bool remove = dead;
2186 TCGLabel *label;
b4fc67c7
RH		 2187
2188 switch (op->opc) {
2189 case INDEX_op_set_label:
2190 label = arg_label(op->args[0]);
2191 if (label->refs == 0) {
2192 /*
2193 * While there is an occasional backward branch, virtually
2194 * all branches generated by the translators are forward.
2195 * Which means that generally we will have already removed
2196 * all references to the label that will be, and there is
2197 * little to be gained by iterating.
2198 */
2199 remove = true;
2200 } else {
2201 /* Once we see a label, insns become live again. */
2202 dead = false;
2203 remove = false;
2204
2205 /*
2206 * Optimization can fold conditional branches to unconditional.
2207 * If we find a label with one reference which is preceded by
2208 * an unconditional branch to it, remove both. This needed to
2209 * wait until the dead code in between them was removed.
2210 */
2211 if (label->refs == 1) {
eae3eb3e 2212 TCGOp *op_prev = QTAILQ_PREV(op, link);
b4fc67c7
RH		 2213 if (op_prev->opc == INDEX_op_br &&
2214 label == arg_label(op_prev->args[0])) {
2215 tcg_op_remove(s, op_prev);
2216 remove = true;
2217 }
2218 }
2219 }
2220 break;
2221
2222 case INDEX_op_br:
2223 case INDEX_op_exit_tb:
2224 case INDEX_op_goto_ptr:
2225 /* Unconditional branches; everything following is dead. */
2226 dead = true;
2227 break;
2228
2229 case INDEX_op_call:
2230 /* Notice noreturn helper calls, raising exceptions. */
90163900 2231 if (tcg_call_flags(op) & TCG_CALL_NO_RETURN) {
b4fc67c7
RH		 2232 dead = true;
2233 }
2234 break;
2235
2236 case INDEX_op_insn_start:
2237 /* Never remove -- we need to keep these for unwind. */
2238 remove = false;
2239 break;
2240
2241 default:
2242 break;
2243 }
2244
2245 if (remove) {
2246 tcg_op_remove(s, op);
2247 }
2248 }
2249}
2250
c70fbf0a
RH		 2251#define TS_DEAD 1
2252#define TS_MEM 2
2253
5a18407f
RH		 2254#define IS_DEAD_ARG(n) (arg_life & (DEAD_ARG << (n)))
2255#define NEED_SYNC_ARG(n) (arg_life & (SYNC_ARG << (n)))
2256
25f49c5f
RH		 2257/* For liveness_pass_1, the register preferences for a given temp. */
2258static inline TCGRegSet *la_temp_pref(TCGTemp *ts)
2259{
2260 return ts->state_ptr;
2261}
2262
2263/* For liveness_pass_1, reset the preferences for a given temp to the
2264 * maximal regset for its type.
2265 */
2266static inline void la_reset_pref(TCGTemp *ts)
2267{
2268 *la_temp_pref(ts)
2269 = (ts->state == TS_DEAD ? 0 : tcg_target_available_regs[ts->type]);
2270}
2271
9c43b68d
AJ		 2272/* liveness analysis: end of function: all temps are dead, and globals
2273 should be in memory. */
2616c808 2274static void la_func_end(TCGContext *s, int ng, int nt)
c896fe29 2275{
b83eabea
RH		 2276 int i;
2277
2278 for (i = 0; i < ng; ++i) {
2279 s->temps[i].state = TS_DEAD | TS_MEM;
25f49c5f 2280 la_reset_pref(&s->temps[i]);
b83eabea
RH		 2281 }
2282 for (i = ng; i < nt; ++i) {
2283 s->temps[i].state = TS_DEAD;
25f49c5f 2284 la_reset_pref(&s->temps[i]);
b83eabea 2285 }
c896fe29
FB		 2286}
2287
9c43b68d
AJ		 2288/* liveness analysis: end of basic block: all temps are dead, globals
2289 and local temps should be in memory. */
2616c808 2290static void la_bb_end(TCGContext *s, int ng, int nt)
641d5fbe 2291{
b83eabea 2292 int i;
641d5fbe 2293
ee17db83
RH		 2294 for (i = 0; i < nt; ++i) {
2295 TCGTemp *ts = &s->temps[i];
2296 int state;
2297
2298 switch (ts->kind) {
2299 case TEMP_FIXED:
2300 case TEMP_GLOBAL:
2301 case TEMP_LOCAL:
2302 state = TS_DEAD | TS_MEM;
2303 break;
2304 case TEMP_NORMAL:
c7482438 2305 case TEMP_EBB:
c0522136 2306 case TEMP_CONST:
ee17db83
RH		 2307 state = TS_DEAD;
2308 break;
2309 default:
2310 g_assert_not_reached();
2311 }
2312 ts->state = state;
2313 la_reset_pref(ts);
641d5fbe
FB		 2314 }
2315}
2316
f65a061c
RH		 2317/* liveness analysis: sync globals back to memory. */
2318static void la_global_sync(TCGContext *s, int ng)
2319{
2320 int i;
2321
2322 for (i = 0; i < ng; ++i) {
25f49c5f
RH		 2323 int state = s->temps[i].state;
2324 s->temps[i].state = state | TS_MEM;
2325 if (state == TS_DEAD) {
2326 /* If the global was previously dead, reset prefs. */
2327 la_reset_pref(&s->temps[i]);
2328 }
f65a061c
RH		 2329 }
2330}
2331
b4cb76e6 2332/*
c7482438
RH		 2333 * liveness analysis: conditional branch: all temps are dead unless
2334 * explicitly live-across-conditional-branch, globals and local temps
2335 * should be synced.
b4cb76e6
RH		 2336 */
2337static void la_bb_sync(TCGContext *s, int ng, int nt)
2338{
2339 la_global_sync(s, ng);
2340
2341 for (int i = ng; i < nt; ++i) {
c0522136
RH		 2342 TCGTemp *ts = &s->temps[i];
2343 int state;
2344
2345 switch (ts->kind) {
2346 case TEMP_LOCAL:
2347 state = ts->state;
2348 ts->state = state | TS_MEM;
b4cb76e6
RH		 2349 if (state != TS_DEAD) {
2350 continue;
2351 }
c0522136
RH		 2352 break;
2353 case TEMP_NORMAL:
b4cb76e6 2354 s->temps[i].state = TS_DEAD;
c0522136 2355 break;
c7482438 2356 case TEMP_EBB:
c0522136
RH		 2357 case TEMP_CONST:
2358 continue;
2359 default:
2360 g_assert_not_reached();
b4cb76e6
RH		 2361 }
2362 la_reset_pref(&s->temps[i]);
2363 }
2364}
2365
f65a061c
RH		 2366/* liveness analysis: sync globals back to memory and kill. */
2367static void la_global_kill(TCGContext *s, int ng)
2368{
2369 int i;
2370
2371 for (i = 0; i < ng; i++) {
2372 s->temps[i].state = TS_DEAD | TS_MEM;
25f49c5f
RH		 2373 la_reset_pref(&s->temps[i]);
2374 }
2375}
2376
2377/* liveness analysis: note live globals crossing calls. */
2378static void la_cross_call(TCGContext *s, int nt)
2379{
2380 TCGRegSet mask = ~tcg_target_call_clobber_regs;
2381 int i;
2382
2383 for (i = 0; i < nt; i++) {
2384 TCGTemp *ts = &s->temps[i];
2385 if (!(ts->state & TS_DEAD)) {
2386 TCGRegSet *pset = la_temp_pref(ts);
2387 TCGRegSet set = *pset;
2388
2389 set &= mask;
2390 /* If the combination is not possible, restart. */
2391 if (set == 0) {
2392 set = tcg_target_available_regs[ts->type] & mask;
2393 }
2394 *pset = set;
2395 }
f65a061c
RH		 2396 }
2397}
2398
a1b3c48d 2399/* Liveness analysis : update the opc_arg_life array to tell if a
c896fe29
FB		 2400 given input arguments is dead. Instructions updating dead
2401 temporaries are removed. */
b83eabea 2402static void liveness_pass_1(TCGContext *s)
c896fe29 2403{
c70fbf0a 2404 int nb_globals = s->nb_globals;
2616c808 2405 int nb_temps = s->nb_temps;
15fa08f8 2406 TCGOp *op, *op_prev;
25f49c5f
RH		 2407 TCGRegSet *prefs;
2408 int i;
2409
2410 prefs = tcg_malloc(sizeof(TCGRegSet) * nb_temps);
2411 for (i = 0; i < nb_temps; ++i) {
2412 s->temps[i].state_ptr = prefs + i;
2413 }
a1b3c48d 2414
ae36a246 2415 /* ??? Should be redundant with the exit_tb that ends the TB. */
2616c808 2416 la_func_end(s, nb_globals, nb_temps);
c896fe29 2417
eae3eb3e 2418 QTAILQ_FOREACH_REVERSE_SAFE(op, &s->ops, link, op_prev) {
25f49c5f 2419 int nb_iargs, nb_oargs;
c45cb8bb
RH		 2420 TCGOpcode opc_new, opc_new2;
2421 bool have_opc_new2;
a1b3c48d 2422 TCGLifeData arg_life = 0;
25f49c5f 2423 TCGTemp *ts;
c45cb8bb
RH		 2424 TCGOpcode opc = op->opc;
2425 const TCGOpDef *def = &tcg_op_defs[opc];
2426
c45cb8bb 2427 switch (opc) {
c896fe29 2428 case INDEX_op_call:
c6e113f5
FB		 2429 {
2430 int call_flags;
25f49c5f 2431 int nb_call_regs;
c896fe29 2432
cd9090aa
RH		 2433 nb_oargs = TCGOP_CALLO(op);
2434 nb_iargs = TCGOP_CALLI(op);
90163900 2435 call_flags = tcg_call_flags(op);
c6e113f5 2436
c45cb8bb 2437 /* pure functions can be removed if their result is unused */
78505279 2438 if (call_flags & TCG_CALL_NO_SIDE_EFFECTS) {
cf066674 2439 for (i = 0; i < nb_oargs; i++) {
25f49c5f
RH		 2440 ts = arg_temp(op->args[i]);
2441 if (ts->state != TS_DEAD) {
c6e113f5 2442 goto do_not_remove_call;
9c43b68d 2443 }
c6e113f5 2444 }
c45cb8bb 2445 goto do_remove;
152c35aa
RH		 2446 }
2447 do_not_remove_call:
c896fe29 2448
25f49c5f 2449 /* Output args are dead. */
152c35aa 2450 for (i = 0; i < nb_oargs; i++) {
25f49c5f
RH		 2451 ts = arg_temp(op->args[i]);
2452 if (ts->state & TS_DEAD) {
152c35aa
RH		 2453 arg_life |= DEAD_ARG << i;
2454 }
25f49c5f 2455 if (ts->state & TS_MEM) {
152c35aa 2456 arg_life |= SYNC_ARG << i;
c6e113f5 2457 }
25f49c5f
RH		 2458 ts->state = TS_DEAD;
2459 la_reset_pref(ts);
2460
2461 /* Not used -- it will be tcg_target_call_oarg_regs[i]. */
2462 op->output_pref[i] = 0;
152c35aa 2463 }
78505279 2464
152c35aa
RH		 2465 if (!(call_flags & (TCG_CALL_NO_WRITE_GLOBALS |
2466 TCG_CALL_NO_READ_GLOBALS))) {
f65a061c 2467 la_global_kill(s, nb_globals);
152c35aa 2468 } else if (!(call_flags & TCG_CALL_NO_READ_GLOBALS)) {
f65a061c 2469 la_global_sync(s, nb_globals);
152c35aa 2470 }
b9c18f56 2471
25f49c5f 2472 /* Record arguments that die in this helper. */
152c35aa 2473 for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) {
25f49c5f
RH		 2474 ts = arg_temp(op->args[i]);
2475 if (ts && ts->state & TS_DEAD) {
152c35aa 2476 arg_life |= DEAD_ARG << i;
c6e113f5 2477 }
152c35aa 2478 }
25f49c5f
RH		 2479
2480 /* For all live registers, remove call-clobbered prefs. */
2481 la_cross_call(s, nb_temps);
2482
2483 nb_call_regs = ARRAY_SIZE(tcg_target_call_iarg_regs);
2484
2485 /* Input arguments are live for preceding opcodes. */
2486 for (i = 0; i < nb_iargs; i++) {
2487 ts = arg_temp(op->args[i + nb_oargs]);
2488 if (ts && ts->state & TS_DEAD) {
2489 /* For those arguments that die, and will be allocated
2490 * in registers, clear the register set for that arg,
2491 * to be filled in below. For args that will be on
2492 * the stack, reset to any available reg.
2493 */
2494 *la_temp_pref(ts)
2495 = (i < nb_call_regs ? 0 :
2496 tcg_target_available_regs[ts->type]);
2497 ts->state &= ~TS_DEAD;
2498 }
2499 }
2500
2501 /* For each input argument, add its input register to prefs.
2502 If a temp is used once, this produces a single set bit. */
2503 for (i = 0; i < MIN(nb_call_regs, nb_iargs); i++) {
2504 ts = arg_temp(op->args[i + nb_oargs]);
2505 if (ts) {
2506 tcg_regset_set_reg(*la_temp_pref(ts),
2507 tcg_target_call_iarg_regs[i]);
c19f47bf 2508 }
c896fe29 2509 }
c896fe29 2510 }
c896fe29 2511 break;
765b842a 2512 case INDEX_op_insn_start:
c896fe29 2513 break;
5ff9d6a4 2514 case INDEX_op_discard:
5ff9d6a4 2515 /* mark the temporary as dead */
25f49c5f
RH		 2516 ts = arg_temp(op->args[0]);
2517 ts->state = TS_DEAD;
2518 la_reset_pref(ts);
5ff9d6a4 2519 break;
1305c451
RH		 2520
2521 case INDEX_op_add2_i32:
c45cb8bb 2522 opc_new = INDEX_op_add_i32;
f1fae40c 2523 goto do_addsub2;
1305c451 2524 case INDEX_op_sub2_i32:
c45cb8bb 2525 opc_new = INDEX_op_sub_i32;
f1fae40c
RH		 2526 goto do_addsub2;
2527 case INDEX_op_add2_i64:
c45cb8bb 2528 opc_new = INDEX_op_add_i64;
f1fae40c
RH		 2529 goto do_addsub2;
2530 case INDEX_op_sub2_i64:
c45cb8bb 2531 opc_new = INDEX_op_sub_i64;
f1fae40c 2532 do_addsub2:
1305c451
RH		 2533 nb_iargs = 4;
2534 nb_oargs = 2;
2535 /* Test if the high part of the operation is dead, but not
2536 the low part. The result can be optimized to a simple
2537 add or sub. This happens often for x86_64 guest when the
2538 cpu mode is set to 32 bit. */
b83eabea
RH		 2539 if (arg_temp(op->args[1])->state == TS_DEAD) {
2540 if (arg_temp(op->args[0])->state == TS_DEAD) {
1305c451
RH		 2541 goto do_remove;
2542 }
c45cb8bb
RH		 2543 /* Replace the opcode and adjust the args in place,
2544 leaving 3 unused args at the end. */
2545 op->opc = opc = opc_new;
efee3746
RH		 2546 op->args[1] = op->args[2];
2547 op->args[2] = op->args[4];
1305c451
RH		 2548 /* Fall through and mark the single-word operation live. */
2549 nb_iargs = 2;
2550 nb_oargs = 1;
2551 }
2552 goto do_not_remove;
2553
1414968a 2554 case INDEX_op_mulu2_i32:
c45cb8bb
RH		 2555 opc_new = INDEX_op_mul_i32;
2556 opc_new2 = INDEX_op_muluh_i32;
2557 have_opc_new2 = TCG_TARGET_HAS_muluh_i32;
03271524 2558 goto do_mul2;
f1fae40c 2559 case INDEX_op_muls2_i32:
c45cb8bb
RH		 2560 opc_new = INDEX_op_mul_i32;
2561 opc_new2 = INDEX_op_mulsh_i32;
2562 have_opc_new2 = TCG_TARGET_HAS_mulsh_i32;
f1fae40c
RH		 2563 goto do_mul2;
2564 case INDEX_op_mulu2_i64:
c45cb8bb
RH		 2565 opc_new = INDEX_op_mul_i64;
2566 opc_new2 = INDEX_op_muluh_i64;
2567 have_opc_new2 = TCG_TARGET_HAS_muluh_i64;
03271524 2568 goto do_mul2;
f1fae40c 2569 case INDEX_op_muls2_i64:
c45cb8bb
RH		 2570 opc_new = INDEX_op_mul_i64;
2571 opc_new2 = INDEX_op_mulsh_i64;
2572 have_opc_new2 = TCG_TARGET_HAS_mulsh_i64;
03271524 2573 goto do_mul2;
f1fae40c 2574 do_mul2:
1414968a
RH		 2575 nb_iargs = 2;
2576 nb_oargs = 2;
b83eabea
RH		 2577 if (arg_temp(op->args[1])->state == TS_DEAD) {
2578 if (arg_temp(op->args[0])->state == TS_DEAD) {
03271524 2579 /* Both parts of the operation are dead. */
1414968a
RH		 2580 goto do_remove;
2581 }
03271524 2582 /* The high part of the operation is dead; generate the low. */
c45cb8bb 2583 op->opc = opc = opc_new;
efee3746
RH		 2584 op->args[1] = op->args[2];
2585 op->args[2] = op->args[3];
b83eabea 2586 } else if (arg_temp(op->args[0])->state == TS_DEAD && have_opc_new2) {
c45cb8bb
RH		 2587 /* The low part of the operation is dead; generate the high. */
2588 op->opc = opc = opc_new2;
efee3746
RH		 2589 op->args[0] = op->args[1];
2590 op->args[1] = op->args[2];
2591 op->args[2] = op->args[3];
03271524
RH		 2592 } else {
2593 goto do_not_remove;
1414968a 2594 }
03271524
RH		 2595 /* Mark the single-word operation live. */
2596 nb_oargs = 1;
1414968a
RH		 2597 goto do_not_remove;
2598
c896fe29 2599 default:
1305c451 2600 /* XXX: optimize by hardcoding common cases (e.g. triadic ops) */
49516bc0
AJ		 2601 nb_iargs = def->nb_iargs;
2602 nb_oargs = def->nb_oargs;
c896fe29 2603
49516bc0
AJ		 2604 /* Test if the operation can be removed because all
2605 its outputs are dead. We assume that nb_oargs == 0
2606 implies side effects */
2607 if (!(def->flags & TCG_OPF_SIDE_EFFECTS) && nb_oargs != 0) {
c45cb8bb 2608 for (i = 0; i < nb_oargs; i++) {
b83eabea 2609 if (arg_temp(op->args[i])->state != TS_DEAD) {
49516bc0 2610 goto do_not_remove;
9c43b68d 2611 }
49516bc0 2612 }
152c35aa
RH		 2613 goto do_remove;
2614 }
2615 goto do_not_remove;
49516bc0 2616
152c35aa
RH		 2617 do_remove:
2618 tcg_op_remove(s, op);
2619 break;
2620
2621 do_not_remove:
152c35aa 2622 for (i = 0; i < nb_oargs; i++) {
25f49c5f
RH		 2623 ts = arg_temp(op->args[i]);
2624
2625 /* Remember the preference of the uses that followed. */
2626 op->output_pref[i] = *la_temp_pref(ts);
2627
2628 /* Output args are dead. */
2629 if (ts->state & TS_DEAD) {
152c35aa 2630 arg_life |= DEAD_ARG << i;
49516bc0 2631 }
25f49c5f 2632 if (ts->state & TS_MEM) {
152c35aa
RH		 2633 arg_life |= SYNC_ARG << i;
2634 }
25f49c5f
RH		 2635 ts->state = TS_DEAD;
2636 la_reset_pref(ts);
152c35aa 2637 }
49516bc0 2638
25f49c5f 2639 /* If end of basic block, update. */
ae36a246
RH		 2640 if (def->flags & TCG_OPF_BB_EXIT) {
2641 la_func_end(s, nb_globals, nb_temps);
b4cb76e6
RH		 2642 } else if (def->flags & TCG_OPF_COND_BRANCH) {
2643 la_bb_sync(s, nb_globals, nb_temps);
ae36a246 2644 } else if (def->flags & TCG_OPF_BB_END) {
2616c808 2645 la_bb_end(s, nb_globals, nb_temps);
152c35aa 2646 } else if (def->flags & TCG_OPF_SIDE_EFFECTS) {
f65a061c 2647 la_global_sync(s, nb_globals);
25f49c5f
RH		 2648 if (def->flags & TCG_OPF_CALL_CLOBBER) {
2649 la_cross_call(s, nb_temps);
2650 }
152c35aa
RH		 2651 }
2652
25f49c5f 2653 /* Record arguments that die in this opcode. */
152c35aa 2654 for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) {
25f49c5f
RH		 2655 ts = arg_temp(op->args[i]);
2656 if (ts->state & TS_DEAD) {
152c35aa 2657 arg_life |= DEAD_ARG << i;
c896fe29 2658 }
c896fe29 2659 }
25f49c5f
RH		 2660
2661 /* Input arguments are live for preceding opcodes. */
152c35aa 2662 for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) {
25f49c5f
RH		 2663 ts = arg_temp(op->args[i]);
2664 if (ts->state & TS_DEAD) {
2665 /* For operands that were dead, initially allow
2666 all regs for the type. */
2667 *la_temp_pref(ts) = tcg_target_available_regs[ts->type];
2668 ts->state &= ~TS_DEAD;
2669 }
2670 }
2671
2672 /* Incorporate constraints for this operand. */
2673 switch (opc) {
2674 case INDEX_op_mov_i32:
2675 case INDEX_op_mov_i64:
2676 /* Note that these are TCG_OPF_NOT_PRESENT and do not
2677 have proper constraints. That said, special case
2678 moves to propagate preferences backward. */
2679 if (IS_DEAD_ARG(1)) {
2680 *la_temp_pref(arg_temp(op->args[0]))
2681 = *la_temp_pref(arg_temp(op->args[1]));
2682 }
2683 break;
2684
2685 default:
2686 for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) {
2687 const TCGArgConstraint *ct = &def->args_ct[i];
2688 TCGRegSet set, *pset;
2689
2690 ts = arg_temp(op->args[i]);
2691 pset = la_temp_pref(ts);
2692 set = *pset;
2693
9be0d080 2694 set &= ct->regs;
bc2b17e6 2695 if (ct->ialias) {
25f49c5f
RH		 2696 set &= op->output_pref[ct->alias_index];
2697 }
2698 /* If the combination is not possible, restart. */
2699 if (set == 0) {
9be0d080 2700 set = ct->regs;
25f49c5f
RH		 2701 }
2702 *pset = set;
2703 }
2704 break;
152c35aa 2705 }
c896fe29
FB		 2706 break;
2707 }
bee158cb 2708 op->life = arg_life;
1ff0a2c5 2709 }
c896fe29 2710}
c896fe29 2711
5a18407f 2712/* Liveness analysis: Convert indirect regs to direct temporaries. */
b83eabea 2713static bool liveness_pass_2(TCGContext *s)
5a18407f
RH		 2714{
2715 int nb_globals = s->nb_globals;
15fa08f8 2716 int nb_temps, i;
5a18407f 2717 bool changes = false;
15fa08f8 2718 TCGOp *op, *op_next;
5a18407f 2719
5a18407f
RH		 2720 /* Create a temporary for each indirect global. */
2721 for (i = 0; i < nb_globals; ++i) {
2722 TCGTemp *its = &s->temps[i];
2723 if (its->indirect_reg) {
2724 TCGTemp *dts = tcg_temp_alloc(s);
2725 dts->type = its->type;
2726 dts->base_type = its->base_type;
c7482438 2727 dts->kind = TEMP_EBB;
b83eabea
RH		 2728 its->state_ptr = dts;
2729 } else {
2730 its->state_ptr = NULL;
5a18407f 2731 }
b83eabea
RH		 2732 /* All globals begin dead. */
2733 its->state = TS_DEAD;
2734 }
2735 for (nb_temps = s->nb_temps; i < nb_temps; ++i) {
2736 TCGTemp *its = &s->temps[i];
2737 its->state_ptr = NULL;
2738 its->state = TS_DEAD;
5a18407f 2739 }
5a18407f 2740
15fa08f8 2741 QTAILQ_FOREACH_SAFE(op, &s->ops, link, op_next) {
5a18407f
RH		 2742 TCGOpcode opc = op->opc;
2743 const TCGOpDef *def = &tcg_op_defs[opc];
2744 TCGLifeData arg_life = op->life;
2745 int nb_iargs, nb_oargs, call_flags;
b83eabea 2746 TCGTemp *arg_ts, *dir_ts;
5a18407f 2747
5a18407f 2748 if (opc == INDEX_op_call) {
cd9090aa
RH		 2749 nb_oargs = TCGOP_CALLO(op);
2750 nb_iargs = TCGOP_CALLI(op);
90163900 2751 call_flags = tcg_call_flags(op);
5a18407f
RH		 2752 } else {
2753 nb_iargs = def->nb_iargs;
2754 nb_oargs = def->nb_oargs;
2755
2756 /* Set flags similar to how calls require. */
b4cb76e6
RH		 2757 if (def->flags & TCG_OPF_COND_BRANCH) {
2758 /* Like reading globals: sync_globals */
2759 call_flags = TCG_CALL_NO_WRITE_GLOBALS;
2760 } else if (def->flags & TCG_OPF_BB_END) {
5a18407f
RH		 2761 /* Like writing globals: save_globals */
2762 call_flags = 0;
2763 } else if (def->flags & TCG_OPF_SIDE_EFFECTS) {
2764 /* Like reading globals: sync_globals */
2765 call_flags = TCG_CALL_NO_WRITE_GLOBALS;
2766 } else {
2767 /* No effect on globals. */
2768 call_flags = (TCG_CALL_NO_READ_GLOBALS |
2769 TCG_CALL_NO_WRITE_GLOBALS);
2770 }
2771 }
2772
2773 /* Make sure that input arguments are available. */
2774 for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) {
b83eabea
RH		 2775 arg_ts = arg_temp(op->args[i]);
2776 if (arg_ts) {
2777 dir_ts = arg_ts->state_ptr;
2778 if (dir_ts && arg_ts->state == TS_DEAD) {
2779 TCGOpcode lopc = (arg_ts->type == TCG_TYPE_I32
5a18407f
RH		 2780 ? INDEX_op_ld_i32
2781 : INDEX_op_ld_i64);
ac1043f6 2782 TCGOp *lop = tcg_op_insert_before(s, op, lopc);
5a18407f 2783
b83eabea
RH		 2784 lop->args[0] = temp_arg(dir_ts);
2785 lop->args[1] = temp_arg(arg_ts->mem_base);
2786 lop->args[2] = arg_ts->mem_offset;
5a18407f
RH		 2787
2788 /* Loaded, but synced with memory. */
b83eabea 2789 arg_ts->state = TS_MEM;
5a18407f
RH		 2790 }
2791 }
2792 }
2793
2794 /* Perform input replacement, and mark inputs that became dead.
2795 No action is required except keeping temp_state up to date
2796 so that we reload when needed. */
2797 for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) {
b83eabea
RH		 2798 arg_ts = arg_temp(op->args[i]);
2799 if (arg_ts) {
2800 dir_ts = arg_ts->state_ptr;
2801 if (dir_ts) {
2802 op->args[i] = temp_arg(dir_ts);
5a18407f
RH		 2803 changes = true;
2804 if (IS_DEAD_ARG(i)) {
b83eabea 2805 arg_ts->state = TS_DEAD;
5a18407f
RH		 2806 }
2807 }
2808 }
2809 }
2810
2811 /* Liveness analysis should ensure that the following are
2812 all correct, for call sites and basic block end points. */
2813 if (call_flags & TCG_CALL_NO_READ_GLOBALS) {
2814 /* Nothing to do */
2815 } else if (call_flags & TCG_CALL_NO_WRITE_GLOBALS) {
2816 for (i = 0; i < nb_globals; ++i) {
2817 /* Liveness should see that globals are synced back,
2818 that is, either TS_DEAD or TS_MEM. */
b83eabea
RH		 2819 arg_ts = &s->temps[i];
2820 tcg_debug_assert(arg_ts->state_ptr == 0
2821 || arg_ts->state != 0);
5a18407f
RH		 2822 }
2823 } else {
2824 for (i = 0; i < nb_globals; ++i) {
2825 /* Liveness should see that globals are saved back,
2826 that is, TS_DEAD, waiting to be reloaded. */
b83eabea
RH		 2827 arg_ts = &s->temps[i];
2828 tcg_debug_assert(arg_ts->state_ptr == 0
2829 || arg_ts->state == TS_DEAD);
5a18407f
RH		 2830 }
2831 }
2832
2833 /* Outputs become available. */
61f15c48
RH		 2834 if (opc == INDEX_op_mov_i32 || opc == INDEX_op_mov_i64) {
2835 arg_ts = arg_temp(op->args[0]);
b83eabea 2836 dir_ts = arg_ts->state_ptr;
61f15c48
RH		 2837 if (dir_ts) {
2838 op->args[0] = temp_arg(dir_ts);
2839 changes = true;
2840
2841 /* The output is now live and modified. */
2842 arg_ts->state = 0;
2843
2844 if (NEED_SYNC_ARG(0)) {
2845 TCGOpcode sopc = (arg_ts->type == TCG_TYPE_I32
2846 ? INDEX_op_st_i32
2847 : INDEX_op_st_i64);
2848 TCGOp *sop = tcg_op_insert_after(s, op, sopc);
2849 TCGTemp *out_ts = dir_ts;
2850
2851 if (IS_DEAD_ARG(0)) {
2852 out_ts = arg_temp(op->args[1]);
2853 arg_ts->state = TS_DEAD;
2854 tcg_op_remove(s, op);
2855 } else {
2856 arg_ts->state = TS_MEM;
2857 }
2858
2859 sop->args[0] = temp_arg(out_ts);
2860 sop->args[1] = temp_arg(arg_ts->mem_base);
2861 sop->args[2] = arg_ts->mem_offset;
2862 } else {
2863 tcg_debug_assert(!IS_DEAD_ARG(0));
2864 }
5a18407f 2865 }
61f15c48
RH		 2866 } else {
2867 for (i = 0; i < nb_oargs; i++) {
2868 arg_ts = arg_temp(op->args[i]);
2869 dir_ts = arg_ts->state_ptr;
2870 if (!dir_ts) {
2871 continue;
2872 }
2873 op->args[i] = temp_arg(dir_ts);
2874 changes = true;
5a18407f 2875
61f15c48
RH		 2876 /* The output is now live and modified. */
2877 arg_ts->state = 0;
5a18407f 2878
61f15c48
RH		 2879 /* Sync outputs upon their last write. */
2880 if (NEED_SYNC_ARG(i)) {
2881 TCGOpcode sopc = (arg_ts->type == TCG_TYPE_I32
2882 ? INDEX_op_st_i32
2883 : INDEX_op_st_i64);
2884 TCGOp *sop = tcg_op_insert_after(s, op, sopc);
5a18407f 2885
61f15c48
RH		 2886 sop->args[0] = temp_arg(dir_ts);
2887 sop->args[1] = temp_arg(arg_ts->mem_base);
2888 sop->args[2] = arg_ts->mem_offset;
5a18407f 2889
61f15c48
RH		 2890 arg_ts->state = TS_MEM;
2891 }
2892 /* Drop outputs that are dead. */
2893 if (IS_DEAD_ARG(i)) {
2894 arg_ts->state = TS_DEAD;
2895 }
5a18407f
RH		 2896 }
2897 }
2898 }
2899
2900 return changes;
2901}
2902
8d8fdbae 2903#ifdef CONFIG_DEBUG_TCG
c896fe29
FB		 2904static void dump_regs(TCGContext *s)
2905{
2906 TCGTemp *ts;
2907 int i;
2908 char buf[64];
2909
2910 for(i = 0; i < s->nb_temps; i++) {
2911 ts = &s->temps[i];
43439139 2912 printf(" %10s: ", tcg_get_arg_str_ptr(s, buf, sizeof(buf), ts));
c896fe29
FB		 2913 switch(ts->val_type) {
2914 case TEMP_VAL_REG:
2915 printf("%s", tcg_target_reg_names[ts->reg]);
2916 break;
2917 case TEMP_VAL_MEM:
b3a62939
RH		 2918 printf("%d(%s)", (int)ts->mem_offset,
2919 tcg_target_reg_names[ts->mem_base->reg]);
c896fe29
FB		 2920 break;
2921 case TEMP_VAL_CONST:
bdb38b95 2922 printf("$0x%" PRIx64, ts->val);
c896fe29
FB		 2923 break;
2924 case TEMP_VAL_DEAD:
2925 printf("D");
2926 break;
2927 default:
2928 printf("???");
2929 break;
2930 }
2931 printf("\n");
2932 }
2933
2934 for(i = 0; i < TCG_TARGET_NB_REGS; i++) {
f8b2f202 2935 if (s->reg_to_temp[i] != NULL) {
a813e36f
RH		 2936 printf("%s: %s\n",
2937 tcg_target_reg_names[i],
f8b2f202 2938 tcg_get_arg_str_ptr(s, buf, sizeof(buf), s->reg_to_temp[i]));
c896fe29
FB		 2939 }
2940 }
2941}
2942
2943static void check_regs(TCGContext *s)
2944{
869938ae 2945 int reg;
b6638662 2946 int k;
c896fe29
FB		 2947 TCGTemp *ts;
2948 char buf[64];
2949
f8b2f202
RH		 2950 for (reg = 0; reg < TCG_TARGET_NB_REGS; reg++) {
2951 ts = s->reg_to_temp[reg];
2952 if (ts != NULL) {
2953 if (ts->val_type != TEMP_VAL_REG || ts->reg != reg) {
a813e36f 2954 printf("Inconsistency for register %s:\n",
c896fe29 2955 tcg_target_reg_names[reg]);
b03cce8e 2956 goto fail;
c896fe29
FB		 2957 }
2958 }
2959 }
f8b2f202 2960 for (k = 0; k < s->nb_temps; k++) {
c896fe29 2961 ts = &s->temps[k];
ee17db83
RH		 2962 if (ts->val_type == TEMP_VAL_REG
2963 && ts->kind != TEMP_FIXED
f8b2f202
RH		 2964 && s->reg_to_temp[ts->reg] != ts) {
2965 printf("Inconsistency for temp %s:\n",
2966 tcg_get_arg_str_ptr(s, buf, sizeof(buf), ts));
b03cce8e 2967 fail:
f8b2f202
RH		 2968 printf("reg state:\n");
2969 dump_regs(s);
2970 tcg_abort();
c896fe29
FB		 2971 }
2972 }
2973}
2974#endif
2975
2272e4a7 2976static void temp_allocate_frame(TCGContext *s, TCGTemp *ts)
c896fe29 2977{
c1c09194
RH		 2978 intptr_t off, size, align;
2979
2980 switch (ts->type) {
2981 case TCG_TYPE_I32:
2982 size = align = 4;
2983 break;
2984 case TCG_TYPE_I64:
2985 case TCG_TYPE_V64:
2986 size = align = 8;
2987 break;
2988 case TCG_TYPE_V128:
2989 size = align = 16;
2990 break;
2991 case TCG_TYPE_V256:
2992 /* Note that we do not require aligned storage for V256. */
2993 size = 32, align = 16;
2994 break;
2995 default:
2996 g_assert_not_reached();
b591dc59 2997 }
c1c09194 2998
b9537d59
RH		 2999 /*
3000 * Assume the stack is sufficiently aligned.
3001 * This affects e.g. ARM NEON, where we have 8 byte stack alignment
3002 * and do not require 16 byte vector alignment. This seems slightly
3003 * easier than fully parameterizing the above switch statement.
3004 */
3005 align = MIN(TCG_TARGET_STACK_ALIGN, align);
c1c09194 3006 off = ROUND_UP(s->current_frame_offset, align);
732d5897
RH		 3007
3008 /* If we've exhausted the stack frame, restart with a smaller TB. */
3009 if (off + size > s->frame_end) {
3010 tcg_raise_tb_overflow(s);
3011 }
c1c09194
RH		 3012 s->current_frame_offset = off + size;
3013
3014 ts->mem_offset = off;
9defd1bd
RH		 3015#if defined(__sparc__)
3016 ts->mem_offset += TCG_TARGET_STACK_BIAS;
3017#endif
b3a62939 3018 ts->mem_base = s->frame_temp;
c896fe29 3019 ts->mem_allocated = 1;
c896fe29
FB		 3020}
3021
b722452a 3022static void temp_load(TCGContext *, TCGTemp *, TCGRegSet, TCGRegSet, TCGRegSet);
b3915dbb 3023
59d7c14e
RH		 3024/* Mark a temporary as free or dead. If 'free_or_dead' is negative,
3025 mark it free; otherwise mark it dead. */
3026static void temp_free_or_dead(TCGContext *s, TCGTemp *ts, int free_or_dead)
7f6ceedf 3027{
c0522136
RH		 3028 TCGTempVal new_type;
3029
3030 switch (ts->kind) {
3031 case TEMP_FIXED:
59d7c14e 3032 return;
c0522136
RH		 3033 case TEMP_GLOBAL:
3034 case TEMP_LOCAL:
3035 new_type = TEMP_VAL_MEM;
3036 break;
3037 case TEMP_NORMAL:
c7482438 3038 case TEMP_EBB:
c0522136
RH		 3039 new_type = free_or_dead < 0 ? TEMP_VAL_MEM : TEMP_VAL_DEAD;
3040 break;
3041 case TEMP_CONST:
3042 new_type = TEMP_VAL_CONST;
3043 break;
3044 default:
3045 g_assert_not_reached();
59d7c14e
RH		 3046 }
3047 if (ts->val_type == TEMP_VAL_REG) {
3048 s->reg_to_temp[ts->reg] = NULL;
3049 }
c0522136 3050 ts->val_type = new_type;
59d7c14e 3051}
7f6ceedf 3052
59d7c14e
RH		 3053/* Mark a temporary as dead. */
3054static inline void temp_dead(TCGContext *s, TCGTemp *ts)
3055{
3056 temp_free_or_dead(s, ts, 1);
3057}
3058
3059/* Sync a temporary to memory. 'allocated_regs' is used in case a temporary
3060 registers needs to be allocated to store a constant. If 'free_or_dead'
3061 is non-zero, subsequently release the temporary; if it is positive, the
3062 temp is dead; if it is negative, the temp is free. */
98b4e186
RH		 3063static void temp_sync(TCGContext *s, TCGTemp *ts, TCGRegSet allocated_regs,
3064 TCGRegSet preferred_regs, int free_or_dead)
59d7c14e 3065{
c0522136 3066 if (!temp_readonly(ts) && !ts->mem_coherent) {
7f6ceedf 3067 if (!ts->mem_allocated) {
2272e4a7 3068 temp_allocate_frame(s, ts);
59d7c14e 3069 }
59d7c14e
RH		 3070 switch (ts->val_type) {
3071 case TEMP_VAL_CONST:
3072 /* If we're going to free the temp immediately, then we won't
3073 require it later in a register, so attempt to store the
3074 constant to memory directly. */
3075 if (free_or_dead
3076 && tcg_out_sti(s, ts->type, ts->val,
3077 ts->mem_base->reg, ts->mem_offset)) {
3078 break;
3079 }
3080 temp_load(s, ts, tcg_target_available_regs[ts->type],
98b4e186 3081 allocated_regs, preferred_regs);
59d7c14e
RH		 3082 /* fallthrough */
3083
3084 case TEMP_VAL_REG:
3085 tcg_out_st(s, ts->type, ts->reg,
3086 ts->mem_base->reg, ts->mem_offset);
3087 break;
3088
3089 case TEMP_VAL_MEM:
3090 break;
3091
3092 case TEMP_VAL_DEAD:
3093 default:
3094 tcg_abort();
3095 }
3096 ts->mem_coherent = 1;
3097 }
3098 if (free_or_dead) {
3099 temp_free_or_dead(s, ts, free_or_dead);
7f6ceedf 3100 }
7f6ceedf
AJ		 3101}
3102
c896fe29 3103/* free register 'reg' by spilling the corresponding temporary if necessary */
b3915dbb 3104static void tcg_reg_free(TCGContext *s, TCGReg reg, TCGRegSet allocated_regs)
c896fe29 3105{
f8b2f202 3106 TCGTemp *ts = s->reg_to_temp[reg];
f8b2f202 3107 if (ts != NULL) {
98b4e186 3108 temp_sync(s, ts, allocated_regs, 0, -1);
c896fe29
FB		 3109 }
3110}
3111
b016486e
RH		 3112/**
3113 * tcg_reg_alloc:
3114 * @required_regs: Set of registers in which we must allocate.
3115 * @allocated_regs: Set of registers which must be avoided.
3116 * @preferred_regs: Set of registers we should prefer.
3117 * @rev: True if we search the registers in "indirect" order.
3118 *
3119 * The allocated register must be in @required_regs & ~@allocated_regs,
3120 * but if we can put it in @preferred_regs we may save a move later.
3121 */
3122static TCGReg tcg_reg_alloc(TCGContext *s, TCGRegSet required_regs,
3123 TCGRegSet allocated_regs,
3124 TCGRegSet preferred_regs, bool rev)
c896fe29 3125{
b016486e
RH		 3126 int i, j, f, n = ARRAY_SIZE(tcg_target_reg_alloc_order);
3127 TCGRegSet reg_ct[2];
91478cef 3128 const int *order;
c896fe29 3129
b016486e
RH		 3130 reg_ct[1] = required_regs & ~allocated_regs;
3131 tcg_debug_assert(reg_ct[1] != 0);
3132 reg_ct[0] = reg_ct[1] & preferred_regs;
3133
3134 /* Skip the preferred_regs option if it cannot be satisfied,
3135 or if the preference made no difference. */
3136 f = reg_ct[0] == 0 || reg_ct[0] == reg_ct[1];
3137
91478cef 3138 order = rev ? indirect_reg_alloc_order : tcg_target_reg_alloc_order;
c896fe29 3139
b016486e
RH		 3140 /* Try free registers, preferences first. */
3141 for (j = f; j < 2; j++) {
3142 TCGRegSet set = reg_ct[j];
3143
3144 if (tcg_regset_single(set)) {
3145 /* One register in the set. */
3146 TCGReg reg = tcg_regset_first(set);
3147 if (s->reg_to_temp[reg] == NULL) {
3148 return reg;
3149 }
3150 } else {
3151 for (i = 0; i < n; i++) {
3152 TCGReg reg = order[i];
3153 if (s->reg_to_temp[reg] == NULL &&
3154 tcg_regset_test_reg(set, reg)) {
3155 return reg;
3156 }
3157 }
3158 }
c896fe29
FB		 3159 }
3160
b016486e
RH		 3161 /* We must spill something. */
3162 for (j = f; j < 2; j++) {
3163 TCGRegSet set = reg_ct[j];
3164
3165 if (tcg_regset_single(set)) {
3166 /* One register in the set. */
3167 TCGReg reg = tcg_regset_first(set);
b3915dbb 3168 tcg_reg_free(s, reg, allocated_regs);
c896fe29 3169 return reg;
b016486e
RH		 3170 } else {
3171 for (i = 0; i < n; i++) {
3172 TCGReg reg = order[i];
3173 if (tcg_regset_test_reg(set, reg)) {
3174 tcg_reg_free(s, reg, allocated_regs);
3175 return reg;
3176 }
3177 }
c896fe29
FB		 3178 }
3179 }
3180
3181 tcg_abort();
3182}
3183
40ae5c62
RH		 3184/* Make sure the temporary is in a register. If needed, allocate the register
3185 from DESIRED while avoiding ALLOCATED. */
3186static void temp_load(TCGContext *s, TCGTemp *ts, TCGRegSet desired_regs,
b722452a 3187 TCGRegSet allocated_regs, TCGRegSet preferred_regs)
40ae5c62
RH		 3188{
3189 TCGReg reg;
3190
3191 switch (ts->val_type) {
3192 case TEMP_VAL_REG:
3193 return;
3194 case TEMP_VAL_CONST:
b016486e 3195 reg = tcg_reg_alloc(s, desired_regs, allocated_regs,
b722452a 3196 preferred_regs, ts->indirect_base);
0a6a8bc8
RH		 3197 if (ts->type <= TCG_TYPE_I64) {
3198 tcg_out_movi(s, ts->type, reg, ts->val);
3199 } else {
4e186175
RH		 3200 uint64_t val = ts->val;
3201 MemOp vece = MO_64;
3202
3203 /*
3204 * Find the minimal vector element that matches the constant.
3205 * The targets will, in general, have to do this search anyway,
3206 * do this generically.
3207 */
4e186175
RH		 3208 if (val == dup_const(MO_8, val)) {
3209 vece = MO_8;
3210 } else if (val == dup_const(MO_16, val)) {
3211 vece = MO_16;
0b4286dd 3212 } else if (val == dup_const(MO_32, val)) {
4e186175
RH		 3213 vece = MO_32;
3214 }
3215
3216 tcg_out_dupi_vec(s, ts->type, vece, reg, ts->val);
0a6a8bc8 3217 }
40ae5c62
RH		 3218 ts->mem_coherent = 0;
3219 break;
3220 case TEMP_VAL_MEM:
b016486e 3221 reg = tcg_reg_alloc(s, desired_regs, allocated_regs,
b722452a 3222 preferred_regs, ts->indirect_base);
40ae5c62
RH		 3223 tcg_out_ld(s, ts->type, reg, ts->mem_base->reg, ts->mem_offset);
3224 ts->mem_coherent = 1;
3225 break;
3226 case TEMP_VAL_DEAD:
3227 default:
3228 tcg_abort();
3229 }
3230 ts->reg = reg;
3231 ts->val_type = TEMP_VAL_REG;
3232 s->reg_to_temp[reg] = ts;
3233}
3234
59d7c14e
RH		 3235/* Save a temporary to memory. 'allocated_regs' is used in case a
3236 temporary registers needs to be allocated to store a constant. */
3237static void temp_save(TCGContext *s, TCGTemp *ts, TCGRegSet allocated_regs)
1ad80729 3238{
5a18407f
RH		 3239 /* The liveness analysis already ensures that globals are back
3240 in memory. Keep an tcg_debug_assert for safety. */
e01fa97d 3241 tcg_debug_assert(ts->val_type == TEMP_VAL_MEM || temp_readonly(ts));
1ad80729
AJ		 3242}
3243
9814dd27 3244/* save globals to their canonical location and assume they can be
e8996ee0
FB		 3245 modified be the following code. 'allocated_regs' is used in case a
3246 temporary registers needs to be allocated to store a constant. */
3247static void save_globals(TCGContext *s, TCGRegSet allocated_regs)
c896fe29 3248{
ac3b8891 3249 int i, n;
c896fe29 3250
ac3b8891 3251 for (i = 0, n = s->nb_globals; i < n; i++) {
b13eb728 3252 temp_save(s, &s->temps[i], allocated_regs);
c896fe29 3253 }
e5097dc8
FB		 3254}
3255
3d5c5f87
AJ		 3256/* sync globals to their canonical location and assume they can be
3257 read by the following code. 'allocated_regs' is used in case a
3258 temporary registers needs to be allocated to store a constant. */
3259static void sync_globals(TCGContext *s, TCGRegSet allocated_regs)
3260{
ac3b8891 3261 int i, n;
3d5c5f87 3262
ac3b8891 3263 for (i = 0, n = s->nb_globals; i < n; i++) {
12b9b11a 3264 TCGTemp *ts = &s->temps[i];
5a18407f 3265 tcg_debug_assert(ts->val_type != TEMP_VAL_REG
ee17db83 3266 || ts->kind == TEMP_FIXED
5a18407f 3267 || ts->mem_coherent);
3d5c5f87
AJ		 3268 }
3269}
3270
e5097dc8 3271/* at the end of a basic block, we assume all temporaries are dead and
e8996ee0
FB		 3272 all globals are stored at their canonical location. */
3273static void tcg_reg_alloc_bb_end(TCGContext *s, TCGRegSet allocated_regs)
e5097dc8 3274{
e5097dc8
FB		 3275 int i;
3276
b13eb728
RH		 3277 for (i = s->nb_globals; i < s->nb_temps; i++) {
3278 TCGTemp *ts = &s->temps[i];
c0522136
RH		 3279
3280 switch (ts->kind) {
3281 case TEMP_LOCAL:
b13eb728 3282 temp_save(s, ts, allocated_regs);
c0522136
RH		 3283 break;
3284 case TEMP_NORMAL:
c7482438 3285 case TEMP_EBB:
5a18407f
RH		 3286 /* The liveness analysis already ensures that temps are dead.
3287 Keep an tcg_debug_assert for safety. */
3288 tcg_debug_assert(ts->val_type == TEMP_VAL_DEAD);
c0522136
RH		 3289 break;
3290 case TEMP_CONST:
3291 /* Similarly, we should have freed any allocated register. */
3292 tcg_debug_assert(ts->val_type == TEMP_VAL_CONST);
3293 break;
3294 default:
3295 g_assert_not_reached();
c896fe29
FB		 3296 }
3297 }
e8996ee0
FB		 3298
3299 save_globals(s, allocated_regs);
c896fe29
FB		 3300}
3301
b4cb76e6 3302/*
c7482438
RH		 3303 * At a conditional branch, we assume all temporaries are dead unless
3304 * explicitly live-across-conditional-branch; all globals and local
3305 * temps are synced to their location.
b4cb76e6
RH		 3306 */
3307static void tcg_reg_alloc_cbranch(TCGContext *s, TCGRegSet allocated_regs)
3308{
3309 sync_globals(s, allocated_regs);
3310
3311 for (int i = s->nb_globals; i < s->nb_temps; i++) {
3312 TCGTemp *ts = &s->temps[i];
3313 /*
3314 * The liveness analysis already ensures that temps are dead.
3315 * Keep tcg_debug_asserts for safety.
3316 */
c0522136
RH		 3317 switch (ts->kind) {
3318 case TEMP_LOCAL:
b4cb76e6 3319 tcg_debug_assert(ts->val_type != TEMP_VAL_REG || ts->mem_coherent);
c0522136
RH		 3320 break;
3321 case TEMP_NORMAL:
b4cb76e6 3322 tcg_debug_assert(ts->val_type == TEMP_VAL_DEAD);
c0522136 3323 break;
c7482438 3324 case TEMP_EBB:
c0522136
RH		 3325 case TEMP_CONST:
3326 break;
3327 default:
3328 g_assert_not_reached();
b4cb76e6
RH		 3329 }
3330 }
3331}
3332
bab1671f 3333/*
c58f4c97 3334 * Specialized code generation for INDEX_op_mov_* with a constant.
bab1671f 3335 */
0fe4fca4 3336static void tcg_reg_alloc_do_movi(TCGContext *s, TCGTemp *ots,
ba87719c
RH		 3337 tcg_target_ulong val, TCGLifeData arg_life,
3338 TCGRegSet preferred_regs)
e8996ee0 3339{
d63e3b6e 3340 /* ENV should not be modified. */
e01fa97d 3341 tcg_debug_assert(!temp_readonly(ots));
59d7c14e
RH		 3342
3343 /* The movi is not explicitly generated here. */
3344 if (ots->val_type == TEMP_VAL_REG) {
3345 s->reg_to_temp[ots->reg] = NULL;
ec7a869d 3346 }
59d7c14e
RH		 3347 ots->val_type = TEMP_VAL_CONST;
3348 ots->val = val;
3349 ots->mem_coherent = 0;
3350 if (NEED_SYNC_ARG(0)) {
ba87719c 3351 temp_sync(s, ots, s->reserved_regs, preferred_regs, IS_DEAD_ARG(0));
59d7c14e 3352 } else if (IS_DEAD_ARG(0)) {
f8bf00f1 3353 temp_dead(s, ots);
4c4e1ab2 3354 }
e8996ee0
FB		 3355}
3356
bab1671f
RH		 3357/*
3358 * Specialized code generation for INDEX_op_mov_*.
3359 */
dd186292 3360static void tcg_reg_alloc_mov(TCGContext *s, const TCGOp *op)
c896fe29 3361{
dd186292 3362 const TCGLifeData arg_life = op->life;
69e3706d 3363 TCGRegSet allocated_regs, preferred_regs;
c896fe29 3364 TCGTemp *ts, *ots;
450445d5 3365 TCGType otype, itype;
c896fe29 3366
d21369f5 3367 allocated_regs = s->reserved_regs;
69e3706d 3368 preferred_regs = op->output_pref[0];
43439139
RH		 3369 ots = arg_temp(op->args[0]);
3370 ts = arg_temp(op->args[1]);
450445d5 3371
d63e3b6e 3372 /* ENV should not be modified. */
e01fa97d 3373 tcg_debug_assert(!temp_readonly(ots));
d63e3b6e 3374
450445d5
RH		 3375 /* Note that otype != itype for no-op truncation. */
3376 otype = ots->type;
3377 itype = ts->type;
c29c1d7e 3378
0fe4fca4
PB		 3379 if (ts->val_type == TEMP_VAL_CONST) {
3380 /* propagate constant or generate sti */
3381 tcg_target_ulong val = ts->val;
3382 if (IS_DEAD_ARG(1)) {
3383 temp_dead(s, ts);
3384 }
69e3706d 3385 tcg_reg_alloc_do_movi(s, ots, val, arg_life, preferred_regs);
0fe4fca4
PB		 3386 return;
3387 }
3388
3389 /* If the source value is in memory we're going to be forced
3390 to have it in a register in order to perform the copy. Copy
3391 the SOURCE value into its own register first, that way we
3392 don't have to reload SOURCE the next time it is used. */
3393 if (ts->val_type == TEMP_VAL_MEM) {
69e3706d
RH		 3394 temp_load(s, ts, tcg_target_available_regs[itype],
3395 allocated_regs, preferred_regs);
c29c1d7e 3396 }
c896fe29 3397
0fe4fca4 3398 tcg_debug_assert(ts->val_type == TEMP_VAL_REG);
d63e3b6e 3399 if (IS_DEAD_ARG(0)) {
c29c1d7e
AJ		 3400 /* mov to a non-saved dead register makes no sense (even with
3401 liveness analysis disabled). */
eabb7b91 3402 tcg_debug_assert(NEED_SYNC_ARG(0));
c29c1d7e 3403 if (!ots->mem_allocated) {
2272e4a7 3404 temp_allocate_frame(s, ots);
c29c1d7e 3405 }
b3a62939 3406 tcg_out_st(s, otype, ts->reg, ots->mem_base->reg, ots->mem_offset);
c29c1d7e 3407 if (IS_DEAD_ARG(1)) {
f8bf00f1 3408 temp_dead(s, ts);
c29c1d7e 3409 }
f8bf00f1 3410 temp_dead(s, ots);
c29c1d7e 3411 } else {
ee17db83 3412 if (IS_DEAD_ARG(1) && ts->kind != TEMP_FIXED) {
c896fe29 3413 /* the mov can be suppressed */
c29c1d7e 3414 if (ots->val_type == TEMP_VAL_REG) {
f8b2f202 3415 s->reg_to_temp[ots->reg] = NULL;
c29c1d7e
AJ		 3416 }
3417 ots->reg = ts->reg;
f8bf00f1 3418 temp_dead(s, ts);
c896fe29 3419 } else {
c29c1d7e
AJ		 3420 if (ots->val_type != TEMP_VAL_REG) {
3421 /* When allocating a new register, make sure to not spill the
3422 input one. */
3423 tcg_regset_set_reg(allocated_regs, ts->reg);
450445d5 3424 ots->reg = tcg_reg_alloc(s, tcg_target_available_regs[otype],
69e3706d 3425 allocated_regs, preferred_regs,
b016486e 3426 ots->indirect_base);
c896fe29 3427 }
78113e83 3428 if (!tcg_out_mov(s, otype, ots->reg, ts->reg)) {
240c08d0
RH		 3429 /*
3430 * Cross register class move not supported.
3431 * Store the source register into the destination slot
3432 * and leave the destination temp as TEMP_VAL_MEM.
3433 */
e01fa97d 3434 assert(!temp_readonly(ots));
240c08d0
RH		 3435 if (!ts->mem_allocated) {
3436 temp_allocate_frame(s, ots);
3437 }
3438 tcg_out_st(s, ts->type, ts->reg,
3439 ots->mem_base->reg, ots->mem_offset);
3440 ots->mem_coherent = 1;
3441 temp_free_or_dead(s, ots, -1);
3442 return;
78113e83 3443 }
c896fe29 3444 }
c29c1d7e
AJ		 3445 ots->val_type = TEMP_VAL_REG;
3446 ots->mem_coherent = 0;
f8b2f202 3447 s->reg_to_temp[ots->reg] = ots;
c29c1d7e 3448 if (NEED_SYNC_ARG(0)) {
98b4e186 3449 temp_sync(s, ots, allocated_regs, 0, 0);
c896fe29 3450 }
ec7a869d 3451 }
c896fe29
FB		 3452}
3453
bab1671f
RH		 3454/*
3455 * Specialized code generation for INDEX_op_dup_vec.
3456 */
3457static void tcg_reg_alloc_dup(TCGContext *s, const TCGOp *op)
3458{
3459 const TCGLifeData arg_life = op->life;
3460 TCGRegSet dup_out_regs, dup_in_regs;
3461 TCGTemp *its, *ots;
3462 TCGType itype, vtype;
d6ecb4a9 3463 intptr_t endian_fixup;
bab1671f
RH		 3464 unsigned vece;
3465 bool ok;
3466
3467 ots = arg_temp(op->args[0]);
3468 its = arg_temp(op->args[1]);
3469
3470 /* ENV should not be modified. */
e01fa97d 3471 tcg_debug_assert(!temp_readonly(ots));
bab1671f
RH		 3472
3473 itype = its->type;
3474 vece = TCGOP_VECE(op);
3475 vtype = TCGOP_VECL(op) + TCG_TYPE_V64;
3476
3477 if (its->val_type == TEMP_VAL_CONST) {
3478 /* Propagate constant via movi -> dupi. */
3479 tcg_target_ulong val = its->val;
3480 if (IS_DEAD_ARG(1)) {
3481 temp_dead(s, its);
3482 }
3483 tcg_reg_alloc_do_movi(s, ots, val, arg_life, op->output_pref[0]);
3484 return;
3485 }
3486
9be0d080
RH		 3487 dup_out_regs = tcg_op_defs[INDEX_op_dup_vec].args_ct[0].regs;
3488 dup_in_regs = tcg_op_defs[INDEX_op_dup_vec].args_ct[1].regs;
bab1671f
RH		 3489
3490 /* Allocate the output register now. */
3491 if (ots->val_type != TEMP_VAL_REG) {
3492 TCGRegSet allocated_regs = s->reserved_regs;
3493
3494 if (!IS_DEAD_ARG(1) && its->val_type == TEMP_VAL_REG) {
3495 /* Make sure to not spill the input register. */
3496 tcg_regset_set_reg(allocated_regs, its->reg);
3497 }
3498 ots->reg = tcg_reg_alloc(s, dup_out_regs, allocated_regs,
3499 op->output_pref[0], ots->indirect_base);
3500 ots->val_type = TEMP_VAL_REG;
bab1671f
RH		 3501 s->reg_to_temp[ots->reg] = ots;
3502 }
3503
3504 switch (its->val_type) {
3505 case TEMP_VAL_REG:
3506 /*
3507 * The dup constriaints must be broad, covering all possible VECE.
3508 * However, tcg_op_dup_vec() gets to see the VECE and we allow it
3509 * to fail, indicating that extra moves are required for that case.
3510 */
3511 if (tcg_regset_test_reg(dup_in_regs, its->reg)) {
3512 if (tcg_out_dup_vec(s, vtype, vece, ots->reg, its->reg)) {
3513 goto done;
3514 }
3515 /* Try again from memory or a vector input register. */
3516 }
3517 if (!its->mem_coherent) {
3518 /*
3519 * The input register is not synced, and so an extra store
3520 * would be required to use memory. Attempt an integer-vector
3521 * register move first. We do not have a TCGRegSet for this.
3522 */
3523 if (tcg_out_mov(s, itype, ots->reg, its->reg)) {
3524 break;
3525 }
3526 /* Sync the temp back to its slot and load from there. */
3527 temp_sync(s, its, s->reserved_regs, 0, 0);
3528 }
3529 /* fall through */
3530
3531 case TEMP_VAL_MEM:
e03b5686 3532#if HOST_BIG_ENDIAN
d6ecb4a9
RH		 3533 endian_fixup = itype == TCG_TYPE_I32 ? 4 : 8;
3534 endian_fixup -= 1 << vece;
3535#else
3536 endian_fixup = 0;
3537#endif
3538 if (tcg_out_dupm_vec(s, vtype, vece, ots->reg, its->mem_base->reg,
3539 its->mem_offset + endian_fixup)) {
3540 goto done;
3541 }
bab1671f
RH		 3542 tcg_out_ld(s, itype, ots->reg, its->mem_base->reg, its->mem_offset);
3543 break;
3544
3545 default:
3546 g_assert_not_reached();
3547 }
3548
3549 /* We now have a vector input register, so dup must succeed. */
3550 ok = tcg_out_dup_vec(s, vtype, vece, ots->reg, ots->reg);
3551 tcg_debug_assert(ok);
3552
3553 done:
36f5539c 3554 ots->mem_coherent = 0;
bab1671f
RH		 3555 if (IS_DEAD_ARG(1)) {
3556 temp_dead(s, its);
3557 }
3558 if (NEED_SYNC_ARG(0)) {
3559 temp_sync(s, ots, s->reserved_regs, 0, 0);
3560 }
3561 if (IS_DEAD_ARG(0)) {
3562 temp_dead(s, ots);
3563 }
3564}
3565
dd186292 3566static void tcg_reg_alloc_op(TCGContext *s, const TCGOp *op)
c896fe29 3567{
dd186292
RH		 3568 const TCGLifeData arg_life = op->life;
3569 const TCGOpDef * const def = &tcg_op_defs[op->opc];
82790a87
RH		 3570 TCGRegSet i_allocated_regs;
3571 TCGRegSet o_allocated_regs;
b6638662
RH		 3572 int i, k, nb_iargs, nb_oargs;
3573 TCGReg reg;
c896fe29
FB		 3574 TCGArg arg;
3575 const TCGArgConstraint *arg_ct;
3576 TCGTemp *ts;
3577 TCGArg new_args[TCG_MAX_OP_ARGS];
3578 int const_args[TCG_MAX_OP_ARGS];
3579
3580 nb_oargs = def->nb_oargs;
3581 nb_iargs = def->nb_iargs;
3582
3583 /* copy constants */
a813e36f 3584 memcpy(new_args + nb_oargs + nb_iargs,
dd186292 3585 op->args + nb_oargs + nb_iargs,
c896fe29
FB		 3586 sizeof(TCGArg) * def->nb_cargs);
3587
d21369f5
RH		 3588 i_allocated_regs = s->reserved_regs;
3589 o_allocated_regs = s->reserved_regs;
82790a87 3590
a813e36f 3591 /* satisfy input constraints */
dd186292 3592 for (k = 0; k < nb_iargs; k++) {
1c1824dc
RH		 3593 TCGRegSet i_preferred_regs;
3594 bool allocate_new_reg;
d62816f2 3595
66792f90 3596 i = def->args_ct[nb_oargs + k].sort_index;
dd186292 3597 arg = op->args[i];
c896fe29 3598 arg_ct = &def->args_ct[i];
43439139 3599 ts = arg_temp(arg);
40ae5c62
RH		 3600
3601 if (ts->val_type == TEMP_VAL_CONST
a4fbbd77 3602 && tcg_target_const_match(ts->val, ts->type, arg_ct->ct)) {
40ae5c62
RH		 3603 /* constant is OK for instruction */
3604 const_args[i] = 1;
3605 new_args[i] = ts->val;
d62816f2 3606 continue;
c896fe29 3607 }
40ae5c62 3608
1c1824dc
RH		 3609 reg = ts->reg;
3610 i_preferred_regs = 0;
3611 allocate_new_reg = false;
3612
bc2b17e6 3613 if (arg_ct->ialias) {
1c1824dc 3614 i_preferred_regs = op->output_pref[arg_ct->alias_index];
d62816f2 3615
c0522136
RH		 3616 /*
3617 * If the input is readonly, then it cannot also be an
3618 * output and aliased to itself. If the input is not
3619 * dead after the instruction, we must allocate a new
3620 * register and move it.
3621 */
3622 if (temp_readonly(ts) || !IS_DEAD_ARG(i)) {
1c1824dc
RH		 3623 allocate_new_reg = true;
3624 } else if (ts->val_type == TEMP_VAL_REG) {
3625 /*
3626 * Check if the current register has already been
3627 * allocated for another input.
3628 */
3629 allocate_new_reg = tcg_regset_test_reg(i_allocated_regs, reg);
5ff9d6a4 3630 }
c896fe29 3631 }
d62816f2 3632
1c1824dc
RH		 3633 if (!allocate_new_reg) {
3634 temp_load(s, ts, arg_ct->regs, i_allocated_regs, i_preferred_regs);
3635 reg = ts->reg;
3636 allocate_new_reg = !tcg_regset_test_reg(arg_ct->regs, reg);
3637 }
d62816f2 3638
1c1824dc 3639 if (allocate_new_reg) {
c0522136
RH		 3640 /*
3641 * Allocate a new register matching the constraint
3642 * and move the temporary register into it.
3643 */
d62816f2
RH		 3644 temp_load(s, ts, tcg_target_available_regs[ts->type],
3645 i_allocated_regs, 0);
9be0d080 3646 reg = tcg_reg_alloc(s, arg_ct->regs, i_allocated_regs,
1c1824dc 3647 i_preferred_regs, ts->indirect_base);
78113e83 3648 if (!tcg_out_mov(s, ts->type, reg, ts->reg)) {
240c08d0
RH		 3649 /*
3650 * Cross register class move not supported. Sync the
3651 * temp back to its slot and load from there.
3652 */
3653 temp_sync(s, ts, i_allocated_regs, 0, 0);
3654 tcg_out_ld(s, ts->type, reg,
3655 ts->mem_base->reg, ts->mem_offset);
78113e83 3656 }
c896fe29 3657 }
c896fe29
FB		 3658 new_args[i] = reg;
3659 const_args[i] = 0;
82790a87 3660 tcg_regset_set_reg(i_allocated_regs, reg);
c896fe29 3661 }
a813e36f 3662
a52ad07e
AJ		 3663 /* mark dead temporaries and free the associated registers */
3664 for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) {
3665 if (IS_DEAD_ARG(i)) {
43439139 3666 temp_dead(s, arg_temp(op->args[i]));
a52ad07e
AJ		 3667 }
3668 }
3669
b4cb76e6
RH		 3670 if (def->flags & TCG_OPF_COND_BRANCH) {
3671 tcg_reg_alloc_cbranch(s, i_allocated_regs);
3672 } else if (def->flags & TCG_OPF_BB_END) {
82790a87 3673 tcg_reg_alloc_bb_end(s, i_allocated_regs);
e8996ee0 3674 } else {
e8996ee0 3675 if (def->flags & TCG_OPF_CALL_CLOBBER) {
a813e36f 3676 /* XXX: permit generic clobber register list ? */
c8074023
RH		 3677 for (i = 0; i < TCG_TARGET_NB_REGS; i++) {
3678 if (tcg_regset_test_reg(tcg_target_call_clobber_regs, i)) {
82790a87 3679 tcg_reg_free(s, i, i_allocated_regs);
e8996ee0 3680 }
c896fe29 3681 }
3d5c5f87
AJ		 3682 }
3683 if (def->flags & TCG_OPF_SIDE_EFFECTS) {
3684 /* sync globals if the op has side effects and might trigger
3685 an exception. */
82790a87 3686 sync_globals(s, i_allocated_regs);
c896fe29 3687 }
a813e36f 3688
e8996ee0 3689 /* satisfy the output constraints */
e8996ee0 3690 for(k = 0; k < nb_oargs; k++) {
66792f90 3691 i = def->args_ct[k].sort_index;
dd186292 3692 arg = op->args[i];
e8996ee0 3693 arg_ct = &def->args_ct[i];
43439139 3694 ts = arg_temp(arg);
d63e3b6e
RH		 3695
3696 /* ENV should not be modified. */
e01fa97d 3697 tcg_debug_assert(!temp_readonly(ts));
d63e3b6e 3698
bc2b17e6 3699 if (arg_ct->oalias && !const_args[arg_ct->alias_index]) {
e8996ee0 3700 reg = new_args[arg_ct->alias_index];
bc2b17e6 3701 } else if (arg_ct->newreg) {
9be0d080 3702 reg = tcg_reg_alloc(s, arg_ct->regs,
82790a87 3703 i_allocated_regs | o_allocated_regs,
69e3706d 3704 op->output_pref[k], ts->indirect_base);
e8996ee0 3705 } else {
9be0d080 3706 reg = tcg_reg_alloc(s, arg_ct->regs, o_allocated_regs,
69e3706d 3707 op->output_pref[k], ts->indirect_base);
c896fe29 3708 }
82790a87 3709 tcg_regset_set_reg(o_allocated_regs, reg);
d63e3b6e
RH		 3710 if (ts->val_type == TEMP_VAL_REG) {
3711 s->reg_to_temp[ts->reg] = NULL;
e8996ee0 3712 }
d63e3b6e
RH		 3713 ts->val_type = TEMP_VAL_REG;
3714 ts->reg = reg;
3715 /*
3716 * Temp value is modified, so the value kept in memory is
3717 * potentially not the same.
3718 */
3719 ts->mem_coherent = 0;
3720 s->reg_to_temp[reg] = ts;
e8996ee0 3721 new_args[i] = reg;
c896fe29 3722 }
c896fe29
FB		 3723 }
3724
c896fe29 3725 /* emit instruction */
d2fd745f
RH		 3726 if (def->flags & TCG_OPF_VECTOR) {
3727 tcg_out_vec_op(s, op->opc, TCGOP_VECL(op), TCGOP_VECE(op),
3728 new_args, const_args);
3729 } else {
3730 tcg_out_op(s, op->opc, new_args, const_args);
3731 }
3732
c896fe29
FB		 3733 /* move the outputs in the correct register if needed */
3734 for(i = 0; i < nb_oargs; i++) {
43439139 3735 ts = arg_temp(op->args[i]);
d63e3b6e
RH		 3736
3737 /* ENV should not be modified. */
e01fa97d 3738 tcg_debug_assert(!temp_readonly(ts));
d63e3b6e 3739
ec7a869d 3740 if (NEED_SYNC_ARG(i)) {
98b4e186 3741 temp_sync(s, ts, o_allocated_regs, 0, IS_DEAD_ARG(i));
59d7c14e 3742 } else if (IS_DEAD_ARG(i)) {
f8bf00f1 3743 temp_dead(s, ts);
ec7a869d 3744 }
c896fe29
FB		 3745 }
3746}
3747
efe86b21
RH		 3748static bool tcg_reg_alloc_dup2(TCGContext *s, const TCGOp *op)
3749{
3750 const TCGLifeData arg_life = op->life;
3751 TCGTemp *ots, *itsl, *itsh;
3752 TCGType vtype = TCGOP_VECL(op) + TCG_TYPE_V64;
3753
3754 /* This opcode is only valid for 32-bit hosts, for 64-bit elements. */
3755 tcg_debug_assert(TCG_TARGET_REG_BITS == 32);
3756 tcg_debug_assert(TCGOP_VECE(op) == MO_64);
3757
3758 ots = arg_temp(op->args[0]);
3759 itsl = arg_temp(op->args[1]);
3760 itsh = arg_temp(op->args[2]);
3761
3762 /* ENV should not be modified. */
3763 tcg_debug_assert(!temp_readonly(ots));
3764
3765 /* Allocate the output register now. */
3766 if (ots->val_type != TEMP_VAL_REG) {
3767 TCGRegSet allocated_regs = s->reserved_regs;
3768 TCGRegSet dup_out_regs =
3769 tcg_op_defs[INDEX_op_dup_vec].args_ct[0].regs;
3770
3771 /* Make sure to not spill the input registers. */
3772 if (!IS_DEAD_ARG(1) && itsl->val_type == TEMP_VAL_REG) {
3773 tcg_regset_set_reg(allocated_regs, itsl->reg);
3774 }
3775 if (!IS_DEAD_ARG(2) && itsh->val_type == TEMP_VAL_REG) {
3776 tcg_regset_set_reg(allocated_regs, itsh->reg);
3777 }
3778
3779 ots->reg = tcg_reg_alloc(s, dup_out_regs, allocated_regs,
3780 op->output_pref[0], ots->indirect_base);
3781 ots->val_type = TEMP_VAL_REG;
efe86b21
RH		 3782 s->reg_to_temp[ots->reg] = ots;
3783 }
3784
3785 /* Promote dup2 of immediates to dupi_vec. */
3786 if (itsl->val_type == TEMP_VAL_CONST && itsh->val_type == TEMP_VAL_CONST) {
3787 uint64_t val = deposit64(itsl->val, 32, 32, itsh->val);
3788 MemOp vece = MO_64;
3789
3790 if (val == dup_const(MO_8, val)) {
3791 vece = MO_8;
3792 } else if (val == dup_const(MO_16, val)) {
3793 vece = MO_16;
3794 } else if (val == dup_const(MO_32, val)) {
3795 vece = MO_32;
3796 }
3797
3798 tcg_out_dupi_vec(s, vtype, vece, ots->reg, val);
3799 goto done;
3800 }
3801
3802 /* If the two inputs form one 64-bit value, try dupm_vec. */
3803 if (itsl + 1 == itsh && itsl->base_type == TCG_TYPE_I64) {
3804 if (!itsl->mem_coherent) {
3805 temp_sync(s, itsl, s->reserved_regs, 0, 0);
3806 }
3807 if (!itsh->mem_coherent) {
3808 temp_sync(s, itsh, s->reserved_regs, 0, 0);
3809 }
e03b5686 3810#if HOST_BIG_ENDIAN
efe86b21
RH		 3811 TCGTemp *its = itsh;
3812#else
3813 TCGTemp *its = itsl;
3814#endif
3815 if (tcg_out_dupm_vec(s, vtype, MO_64, ots->reg,
3816 its->mem_base->reg, its->mem_offset)) {
3817 goto done;
3818 }
3819 }
3820
3821 /* Fall back to generic expansion. */
3822 return false;
3823
3824 done:
36f5539c 3825 ots->mem_coherent = 0;
efe86b21
RH		 3826 if (IS_DEAD_ARG(1)) {
3827 temp_dead(s, itsl);
3828 }
3829 if (IS_DEAD_ARG(2)) {
3830 temp_dead(s, itsh);
3831 }
3832 if (NEED_SYNC_ARG(0)) {
3833 temp_sync(s, ots, s->reserved_regs, 0, IS_DEAD_ARG(0));
3834 } else if (IS_DEAD_ARG(0)) {
3835 temp_dead(s, ots);
3836 }
3837 return true;
3838}
3839
dd186292 3840static void tcg_reg_alloc_call(TCGContext *s, TCGOp *op)
c896fe29 3841{
cd9090aa
RH		 3842 const int nb_oargs = TCGOP_CALLO(op);
3843 const int nb_iargs = TCGOP_CALLI(op);
dd186292 3844 const TCGLifeData arg_life = op->life;
7b7d8b2d 3845 const TCGHelperInfo *info;
b6638662
RH		 3846 int flags, nb_regs, i;
3847 TCGReg reg;
cf066674 3848 TCGArg arg;
c896fe29 3849 TCGTemp *ts;
d3452f1f
RH		 3850 intptr_t stack_offset;
3851 size_t call_stack_size;
cf066674
RH		 3852 tcg_insn_unit *func_addr;
3853 int allocate_args;
c896fe29 3854 TCGRegSet allocated_regs;
c896fe29 3855
fa52e660 3856 func_addr = tcg_call_func(op);
7b7d8b2d
RH		 3857 info = tcg_call_info(op);
3858 flags = info->flags;
c896fe29 3859
6e17d0c5 3860 nb_regs = ARRAY_SIZE(tcg_target_call_iarg_regs);
c45cb8bb
RH		 3861 if (nb_regs > nb_iargs) {
3862 nb_regs = nb_iargs;
cf066674 3863 }
c896fe29
FB		 3864
3865 /* assign stack slots first */
c45cb8bb 3866 call_stack_size = (nb_iargs - nb_regs) * sizeof(tcg_target_long);
a813e36f 3867 call_stack_size = (call_stack_size + TCG_TARGET_STACK_ALIGN - 1) &
c896fe29 3868 ~(TCG_TARGET_STACK_ALIGN - 1);
b03cce8e
FB		 3869 allocate_args = (call_stack_size > TCG_STATIC_CALL_ARGS_SIZE);
3870 if (allocate_args) {
345649c0
BS		 3871 /* XXX: if more than TCG_STATIC_CALL_ARGS_SIZE is needed,
3872 preallocate call stack */
3873 tcg_abort();
b03cce8e 3874 }
39cf05d3
FB		 3875
3876 stack_offset = TCG_TARGET_CALL_STACK_OFFSET;
dd186292
RH		 3877 for (i = nb_regs; i < nb_iargs; i++) {
3878 arg = op->args[nb_oargs + i];
39cf05d3 3879 if (arg != TCG_CALL_DUMMY_ARG) {
43439139 3880 ts = arg_temp(arg);
40ae5c62 3881 temp_load(s, ts, tcg_target_available_regs[ts->type],
b722452a 3882 s->reserved_regs, 0);
40ae5c62 3883 tcg_out_st(s, ts->type, ts->reg, TCG_REG_CALL_STACK, stack_offset);
c896fe29 3884 }
39cf05d3 3885 stack_offset += sizeof(tcg_target_long);
c896fe29 3886 }
a813e36f 3887
c896fe29 3888 /* assign input registers */
d21369f5 3889 allocated_regs = s->reserved_regs;
dd186292
RH		 3890 for (i = 0; i < nb_regs; i++) {
3891 arg = op->args[nb_oargs + i];
39cf05d3 3892 if (arg != TCG_CALL_DUMMY_ARG) {
43439139 3893 ts = arg_temp(arg);
39cf05d3 3894 reg = tcg_target_call_iarg_regs[i];
40ae5c62 3895
39cf05d3
FB		 3896 if (ts->val_type == TEMP_VAL_REG) {
3897 if (ts->reg != reg) {
4250da10 3898 tcg_reg_free(s, reg, allocated_regs);
78113e83 3899 if (!tcg_out_mov(s, ts->type, reg, ts->reg)) {
240c08d0
RH		 3900 /*
3901 * Cross register class move not supported. Sync the
3902 * temp back to its slot and load from there.
3903 */
3904 temp_sync(s, ts, allocated_regs, 0, 0);
3905 tcg_out_ld(s, ts->type, reg,
3906 ts->mem_base->reg, ts->mem_offset);
78113e83 3907 }
39cf05d3 3908 }
39cf05d3 3909 } else {
ccb1bb66 3910 TCGRegSet arg_set = 0;
40ae5c62 3911
4250da10 3912 tcg_reg_free(s, reg, allocated_regs);
40ae5c62 3913 tcg_regset_set_reg(arg_set, reg);
b722452a 3914 temp_load(s, ts, arg_set, allocated_regs, 0);
c896fe29 3915 }
40ae5c62 3916
39cf05d3 3917 tcg_regset_set_reg(allocated_regs, reg);
c896fe29 3918 }
c896fe29 3919 }
a813e36f 3920
c896fe29 3921 /* mark dead temporaries and free the associated registers */
dd186292 3922 for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) {
866cb6cb 3923 if (IS_DEAD_ARG(i)) {
43439139 3924 temp_dead(s, arg_temp(op->args[i]));
c896fe29
FB		 3925 }
3926 }
a813e36f 3927
c896fe29 3928 /* clobber call registers */
c8074023
RH		 3929 for (i = 0; i < TCG_TARGET_NB_REGS; i++) {
3930 if (tcg_regset_test_reg(tcg_target_call_clobber_regs, i)) {
b3915dbb 3931 tcg_reg_free(s, i, allocated_regs);
c896fe29
FB		 3932 }
3933 }
78505279
AJ		 3934
3935 /* Save globals if they might be written by the helper, sync them if
3936 they might be read. */
3937 if (flags & TCG_CALL_NO_READ_GLOBALS) {
3938 /* Nothing to do */
3939 } else if (flags & TCG_CALL_NO_WRITE_GLOBALS) {
3940 sync_globals(s, allocated_regs);
3941 } else {
b9c18f56
AJ		 3942 save_globals(s, allocated_regs);
3943 }
c896fe29 3944
7b7d8b2d
RH		 3945#ifdef CONFIG_TCG_INTERPRETER
3946 {
3947 gpointer hash = (gpointer)(uintptr_t)info->typemask;
3948 ffi_cif *cif = g_hash_table_lookup(ffi_table, hash);
3949 assert(cif != NULL);
3950 tcg_out_call(s, func_addr, cif);
3951 }
3952#else
cf066674 3953 tcg_out_call(s, func_addr);
7b7d8b2d 3954#endif
c896fe29
FB		 3955
3956 /* assign output registers and emit moves if needed */
3957 for(i = 0; i < nb_oargs; i++) {
dd186292 3958 arg = op->args[i];
43439139 3959 ts = arg_temp(arg);
d63e3b6e
RH		 3960
3961 /* ENV should not be modified. */
e01fa97d 3962 tcg_debug_assert(!temp_readonly(ts));
d63e3b6e 3963
c896fe29 3964 reg = tcg_target_call_oarg_regs[i];
eabb7b91 3965 tcg_debug_assert(s->reg_to_temp[reg] == NULL);
d63e3b6e
RH		 3966 if (ts->val_type == TEMP_VAL_REG) {
3967 s->reg_to_temp[ts->reg] = NULL;
3968 }
3969 ts->val_type = TEMP_VAL_REG;
3970 ts->reg = reg;
3971 ts->mem_coherent = 0;
3972 s->reg_to_temp[reg] = ts;
3973 if (NEED_SYNC_ARG(i)) {
3974 temp_sync(s, ts, allocated_regs, 0, IS_DEAD_ARG(i));
3975 } else if (IS_DEAD_ARG(i)) {
3976 temp_dead(s, ts);
c896fe29
FB		 3977 }
3978 }
c896fe29
FB		 3979}
3980
3981#ifdef CONFIG_PROFILER
3982
c3fac113
EC		 3983/* avoid copy/paste errors */
3984#define PROF_ADD(to, from, field) \
3985 do { \
d73415a3 3986 (to)->field += qatomic_read(&((from)->field)); \
c3fac113
EC		 3987 } while (0)
3988
3989#define PROF_MAX(to, from, field) \
3990 do { \
d73415a3 3991 typeof((from)->field) val__ = qatomic_read(&((from)->field)); \
c3fac113
EC		 3992 if (val__ > (to)->field) { \
3993 (to)->field = val__; \
3994 } \
3995 } while (0)
3996
3997/* Pass in a zero'ed @prof */
3998static inline
3999void tcg_profile_snapshot(TCGProfile *prof, bool counters, bool table)
4000{
0e2d61cf 4001 unsigned int n_ctxs = qatomic_read(&tcg_cur_ctxs);
c3fac113
EC		 4002 unsigned int i;
4003
3468b59e 4004 for (i = 0; i < n_ctxs; i++) {
d73415a3 4005 TCGContext *s = qatomic_read(&tcg_ctxs[i]);
3468b59e 4006 const TCGProfile *orig = &s->prof;
c3fac113
EC		 4007
4008 if (counters) {
72fd2efb 4009 PROF_ADD(prof, orig, cpu_exec_time);
c3fac113
EC		 4010 PROF_ADD(prof, orig, tb_count1);
4011 PROF_ADD(prof, orig, tb_count);
4012 PROF_ADD(prof, orig, op_count);
4013 PROF_MAX(prof, orig, op_count_max);
4014 PROF_ADD(prof, orig, temp_count);
4015 PROF_MAX(prof, orig, temp_count_max);
4016 PROF_ADD(prof, orig, del_op_count);
4017 PROF_ADD(prof, orig, code_in_len);
4018 PROF_ADD(prof, orig, code_out_len);
4019 PROF_ADD(prof, orig, search_out_len);
4020 PROF_ADD(prof, orig, interm_time);
4021 PROF_ADD(prof, orig, code_time);
4022 PROF_ADD(prof, orig, la_time);
4023 PROF_ADD(prof, orig, opt_time);
4024 PROF_ADD(prof, orig, restore_count);
4025 PROF_ADD(prof, orig, restore_time);
4026 }
4027 if (table) {
4028 int i;
4029
4030 for (i = 0; i < NB_OPS; i++) {
4031 PROF_ADD(prof, orig, table_op_count[i]);
4032 }
4033 }
4034 }
4035}
4036
4037#undef PROF_ADD
4038#undef PROF_MAX
4039
4040static void tcg_profile_snapshot_counters(TCGProfile *prof)
4041{
4042 tcg_profile_snapshot(prof, true, false);
4043}
4044
4045static void tcg_profile_snapshot_table(TCGProfile *prof)
4046{
4047 tcg_profile_snapshot(prof, false, true);
4048}
c896fe29 4049
b6a7f3e0 4050void tcg_dump_op_count(GString *buf)
c896fe29 4051{
c3fac113 4052 TCGProfile prof = {};
c896fe29 4053 int i;
d70724ce 4054
c3fac113 4055 tcg_profile_snapshot_table(&prof);
15fc7daa 4056 for (i = 0; i < NB_OPS; i++) {
b6a7f3e0
DB		 4057 g_string_append_printf(buf, "%s %" PRId64 "\n", tcg_op_defs[i].name,
4058 prof.table_op_count[i]);
c896fe29 4059 }
c896fe29 4060}
72fd2efb
EC		 4061
4062int64_t tcg_cpu_exec_time(void)
4063{
0e2d61cf 4064 unsigned int n_ctxs = qatomic_read(&tcg_cur_ctxs);
72fd2efb
EC		 4065 unsigned int i;
4066 int64_t ret = 0;
4067
4068 for (i = 0; i < n_ctxs; i++) {
d73415a3 4069 const TCGContext *s = qatomic_read(&tcg_ctxs[i]);
72fd2efb
EC		 4070 const TCGProfile *prof = &s->prof;
4071
d73415a3 4072 ret += qatomic_read(&prof->cpu_exec_time);
72fd2efb
EC		 4073 }
4074 return ret;
4075}
246ae24d 4076#else
b6a7f3e0 4077void tcg_dump_op_count(GString *buf)
246ae24d 4078{
b6a7f3e0 4079 g_string_append_printf(buf, "[TCG profiler not compiled]\n");
246ae24d 4080}
72fd2efb
EC		 4081
4082int64_t tcg_cpu_exec_time(void)
4083{
4084 error_report("%s: TCG profiler not compiled", __func__);
4085 exit(EXIT_FAILURE);
4086}
c896fe29
FB		 4087#endif
4088
4089
fbf59aad 4090int tcg_gen_code(TCGContext *s, TranslationBlock *tb, target_ulong pc_start)
c896fe29 4091{
c3fac113
EC		 4092#ifdef CONFIG_PROFILER
4093 TCGProfile *prof = &s->prof;
4094#endif
15fa08f8
RH		 4095 int i, num_insns;
4096 TCGOp *op;
c896fe29 4097
04fe6400
RH		 4098#ifdef CONFIG_PROFILER
4099 {
c1f543b7 4100 int n = 0;
04fe6400 4101
15fa08f8
RH		 4102 QTAILQ_FOREACH(op, &s->ops, link) {
4103 n++;
4104 }
d73415a3 4105 qatomic_set(&prof->op_count, prof->op_count + n);
c3fac113 4106 if (n > prof->op_count_max) {
d73415a3 4107 qatomic_set(&prof->op_count_max, n);
04fe6400
RH		 4108 }
4109
4110 n = s->nb_temps;
d73415a3 4111 qatomic_set(&prof->temp_count, prof->temp_count + n);
c3fac113 4112 if (n > prof->temp_count_max) {
d73415a3 4113 qatomic_set(&prof->temp_count_max, n);
04fe6400
RH		 4114 }
4115 }
4116#endif
4117
c896fe29 4118#ifdef DEBUG_DISAS
d977e1c2 4119 if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP)
fbf59aad 4120 && qemu_log_in_addr_range(pc_start))) {
c60f599b 4121 FILE *logfile = qemu_log_trylock();
78b54858
RH		 4122 if (logfile) {
4123 fprintf(logfile, "OP:\n");
b7a83ff8 4124 tcg_dump_ops(s, logfile, false);
78b54858
RH		 4125 fprintf(logfile, "\n");
4126 qemu_log_unlock(logfile);
4127 }
c896fe29
FB		 4128 }
4129#endif
4130
bef16ab4
RH		 4131#ifdef CONFIG_DEBUG_TCG
4132 /* Ensure all labels referenced have been emitted. */
4133 {
4134 TCGLabel *l;
4135 bool error = false;
4136
4137 QSIMPLEQ_FOREACH(l, &s->labels, next) {
4138 if (unlikely(!l->present) && l->refs) {
4139 qemu_log_mask(CPU_LOG_TB_OP,
4140 "$L%d referenced but not present.\n", l->id);
4141 error = true;
4142 }
4143 }
4144 assert(!error);
4145 }
4146#endif
4147
c5cc28ff 4148#ifdef CONFIG_PROFILER
d73415a3 4149 qatomic_set(&prof->opt_time, prof->opt_time - profile_getclock());
c5cc28ff
AJ		 4150#endif
4151
8f2e8c07 4152#ifdef USE_TCG_OPTIMIZATIONS
c45cb8bb 4153 tcg_optimize(s);
8f2e8c07
KB		 4154#endif
4155
a23a9ec6 4156#ifdef CONFIG_PROFILER
d73415a3
SH		 4157 qatomic_set(&prof->opt_time, prof->opt_time + profile_getclock());
4158 qatomic_set(&prof->la_time, prof->la_time - profile_getclock());
a23a9ec6 4159#endif
c5cc28ff 4160
b4fc67c7 4161 reachable_code_pass(s);
b83eabea 4162 liveness_pass_1(s);
5a18407f 4163
b83eabea 4164 if (s->nb_indirects > 0) {
5a18407f 4165#ifdef DEBUG_DISAS
b83eabea 4166 if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP_IND)
fbf59aad 4167 && qemu_log_in_addr_range(pc_start))) {
c60f599b 4168 FILE *logfile = qemu_log_trylock();
78b54858
RH		 4169 if (logfile) {
4170 fprintf(logfile, "OP before indirect lowering:\n");
b7a83ff8 4171 tcg_dump_ops(s, logfile, false);
78b54858
RH		 4172 fprintf(logfile, "\n");
4173 qemu_log_unlock(logfile);
4174 }
b83eabea 4175 }
5a18407f 4176#endif
b83eabea
RH		 4177 /* Replace indirect temps with direct temps. */
4178 if (liveness_pass_2(s)) {
4179 /* If changes were made, re-run liveness. */
4180 liveness_pass_1(s);
5a18407f
RH		 4181 }
4182 }
c5cc28ff 4183
a23a9ec6 4184#ifdef CONFIG_PROFILER
d73415a3 4185 qatomic_set(&prof->la_time, prof->la_time + profile_getclock());
a23a9ec6 4186#endif
c896fe29
FB		 4187
4188#ifdef DEBUG_DISAS
d977e1c2 4189 if (unlikely(qemu_loglevel_mask(CPU_LOG_TB_OP_OPT)
fbf59aad 4190 && qemu_log_in_addr_range(pc_start))) {
c60f599b 4191 FILE *logfile = qemu_log_trylock();
78b54858
RH		 4192 if (logfile) {
4193 fprintf(logfile, "OP after optimization and liveness analysis:\n");
b7a83ff8 4194 tcg_dump_ops(s, logfile, true);
78b54858
RH		 4195 fprintf(logfile, "\n");
4196 qemu_log_unlock(logfile);
4197 }
c896fe29
FB		 4198 }
4199#endif
4200
35abb009
RH		 4201 /* Initialize goto_tb jump offsets. */
4202 tb->jmp_reset_offset[0] = TB_JMP_RESET_OFFSET_INVALID;
4203 tb->jmp_reset_offset[1] = TB_JMP_RESET_OFFSET_INVALID;
4204 tcg_ctx->tb_jmp_reset_offset = tb->jmp_reset_offset;
4205 if (TCG_TARGET_HAS_direct_jump) {
4206 tcg_ctx->tb_jmp_insn_offset = tb->jmp_target_arg;
4207 tcg_ctx->tb_jmp_target_addr = NULL;
4208 } else {
4209 tcg_ctx->tb_jmp_insn_offset = NULL;
4210 tcg_ctx->tb_jmp_target_addr = tb->jmp_target_arg;
4211 }
4212
c896fe29
FB		 4213 tcg_reg_alloc_start(s);
4214
db0c51a3
RH		 4215 /*
4216 * Reset the buffer pointers when restarting after overflow.
4217 * TODO: Move this into translate-all.c with the rest of the
4218 * buffer management. Having only this done here is confusing.
4219 */
4220 s->code_buf = tcg_splitwx_to_rw(tb->tc.ptr);
4221 s->code_ptr = s->code_buf;
c896fe29 4222
659ef5cb 4223#ifdef TCG_TARGET_NEED_LDST_LABELS
6001f772 4224 QSIMPLEQ_INIT(&s->ldst_labels);
659ef5cb 4225#endif
57a26946
RH		 4226#ifdef TCG_TARGET_NEED_POOL_LABELS
4227 s->pool_labels = NULL;
4228#endif
9ecefc84 4229
fca8a500 4230 num_insns = -1;
15fa08f8 4231 QTAILQ_FOREACH(op, &s->ops, link) {
c45cb8bb 4232 TCGOpcode opc = op->opc;
b3db8758 4233
c896fe29 4234#ifdef CONFIG_PROFILER
d73415a3 4235 qatomic_set(&prof->table_op_count[opc], prof->table_op_count[opc] + 1);
c896fe29 4236#endif
c45cb8bb
RH		 4237
4238 switch (opc) {
c896fe29 4239 case INDEX_op_mov_i32:
c896fe29 4240 case INDEX_op_mov_i64:
d2fd745f 4241 case INDEX_op_mov_vec:
dd186292 4242 tcg_reg_alloc_mov(s, op);
c896fe29 4243 break;
bab1671f
RH		 4244 case INDEX_op_dup_vec:
4245 tcg_reg_alloc_dup(s, op);
4246 break;
765b842a 4247 case INDEX_op_insn_start:
fca8a500 4248 if (num_insns >= 0) {
9f754620
RH		 4249 size_t off = tcg_current_code_size(s);
4250 s->gen_insn_end_off[num_insns] = off;
4251 /* Assert that we do not overflow our stored offset. */
4252 assert(s->gen_insn_end_off[num_insns] == off);
fca8a500
RH		 4253 }
4254 num_insns++;
bad729e2
RH		 4255 for (i = 0; i < TARGET_INSN_START_WORDS; ++i) {
4256 target_ulong a;
4257#if TARGET_LONG_BITS > TCG_TARGET_REG_BITS
efee3746 4258 a = deposit64(op->args[i * 2], 32, 32, op->args[i * 2 + 1]);
bad729e2 4259#else
efee3746 4260 a = op->args[i];
bad729e2 4261#endif
fca8a500 4262 s->gen_insn_data[num_insns][i] = a;
bad729e2 4263 }
c896fe29 4264 break;
5ff9d6a4 4265 case INDEX_op_discard:
43439139 4266 temp_dead(s, arg_temp(op->args[0]));
5ff9d6a4 4267 break;
c896fe29 4268 case INDEX_op_set_label:
e8996ee0 4269 tcg_reg_alloc_bb_end(s, s->reserved_regs);
92ab8e7d 4270 tcg_out_label(s, arg_label(op->args[0]));
c896fe29
FB		 4271 break;
4272 case INDEX_op_call:
dd186292 4273 tcg_reg_alloc_call(s, op);
c45cb8bb 4274 break;
efe86b21
RH		 4275 case INDEX_op_dup2_vec:
4276 if (tcg_reg_alloc_dup2(s, op)) {
4277 break;
4278 }
4279 /* fall through */
c896fe29 4280 default:
25c4d9cc 4281 /* Sanity check that we've not introduced any unhandled opcodes. */
be0f34b5 4282 tcg_debug_assert(tcg_op_supported(opc));
c896fe29
FB		 4283 /* Note: in order to speed up the code, it would be much
4284 faster to have specialized register allocator functions for
4285 some common argument patterns */
dd186292 4286 tcg_reg_alloc_op(s, op);
c896fe29
FB		 4287 break;
4288 }
8d8fdbae 4289#ifdef CONFIG_DEBUG_TCG
c896fe29
FB		 4290 check_regs(s);
4291#endif
b125f9dc
RH		 4292 /* Test for (pending) buffer overflow. The assumption is that any
4293 one operation beginning below the high water mark cannot overrun
4294 the buffer completely. Thus we can test for overflow after
4295 generating code without having to check during generation. */
644da9b3 4296 if (unlikely((void *)s->code_ptr > s->code_gen_highwater)) {
b125f9dc
RH		 4297 return -1;
4298 }
6e6c4efe
RH		 4299 /* Test for TB overflow, as seen by gen_insn_end_off. */
4300 if (unlikely(tcg_current_code_size(s) > UINT16_MAX)) {
4301 return -2;
4302 }
c896fe29 4303 }
fca8a500
RH		 4304 tcg_debug_assert(num_insns >= 0);
4305 s->gen_insn_end_off[num_insns] = tcg_current_code_size(s);
c45cb8bb 4306
b76f0d8c 4307 /* Generate TB finalization at the end of block */
659ef5cb 4308#ifdef TCG_TARGET_NEED_LDST_LABELS
aeee05f5
RH		 4309 i = tcg_out_ldst_finalize(s);
4310 if (i < 0) {
4311 return i;
23dceda6 4312 }
659ef5cb 4313#endif
57a26946 4314#ifdef TCG_TARGET_NEED_POOL_LABELS
1768987b
RH		 4315 i = tcg_out_pool_finalize(s);
4316 if (i < 0) {
4317 return i;
57a26946
RH		 4318 }
4319#endif
7ecd02a0
RH		 4320 if (!tcg_resolve_relocs(s)) {
4321 return -2;
4322 }
c896fe29 4323
df5d2b16 4324#ifndef CONFIG_TCG_INTERPRETER
c896fe29 4325 /* flush instruction cache */
db0c51a3
RH		 4326 flush_idcache_range((uintptr_t)tcg_splitwx_to_rx(s->code_buf),
4327 (uintptr_t)s->code_buf,
1da8de39 4328 tcg_ptr_byte_diff(s->code_ptr, s->code_buf));
df5d2b16 4329#endif
2aeabc08 4330
1813e175 4331 return tcg_current_code_size(s);
c896fe29
FB		 4332}
4333
a23a9ec6 4334#ifdef CONFIG_PROFILER
3a841ab5 4335void tcg_dump_info(GString *buf)
a23a9ec6 4336{
c3fac113
EC		 4337 TCGProfile prof = {};
4338 const TCGProfile *s;
4339 int64_t tb_count;
4340 int64_t tb_div_count;
4341 int64_t tot;
4342
4343 tcg_profile_snapshot_counters(&prof);
4344 s = &prof;
4345 tb_count = s->tb_count;
4346 tb_div_count = tb_count ? tb_count : 1;
4347 tot = s->interm_time + s->code_time;
a23a9ec6 4348
3a841ab5
DB		 4349 g_string_append_printf(buf, "JIT cycles %" PRId64
4350 " (%0.3f s at 2.4 GHz)\n",
4351 tot, tot / 2.4e9);
4352 g_string_append_printf(buf, "translated TBs %" PRId64
4353 " (aborted=%" PRId64 " %0.1f%%)\n",
4354 tb_count, s->tb_count1 - tb_count,
4355 (double)(s->tb_count1 - s->tb_count)
4356 / (s->tb_count1 ? s->tb_count1 : 1) * 100.0);
4357 g_string_append_printf(buf, "avg ops/TB %0.1f max=%d\n",
4358 (double)s->op_count / tb_div_count, s->op_count_max);
4359 g_string_append_printf(buf, "deleted ops/TB %0.2f\n",
4360 (double)s->del_op_count / tb_div_count);
4361 g_string_append_printf(buf, "avg temps/TB %0.2f max=%d\n",
4362 (double)s->temp_count / tb_div_count,
4363 s->temp_count_max);
4364 g_string_append_printf(buf, "avg host code/TB %0.1f\n",
4365 (double)s->code_out_len / tb_div_count);
4366 g_string_append_printf(buf, "avg search data/TB %0.1f\n",
4367 (double)s->search_out_len / tb_div_count);
a813e36f 4368
3a841ab5
DB		 4369 g_string_append_printf(buf, "cycles/op %0.1f\n",
4370 s->op_count ? (double)tot / s->op_count : 0);
4371 g_string_append_printf(buf, "cycles/in byte %0.1f\n",
4372 s->code_in_len ? (double)tot / s->code_in_len : 0);
4373 g_string_append_printf(buf, "cycles/out byte %0.1f\n",
4374 s->code_out_len ? (double)tot / s->code_out_len : 0);
4375 g_string_append_printf(buf, "cycles/search byte %0.1f\n",
4376 s->search_out_len ?
4377 (double)tot / s->search_out_len : 0);
fca8a500 4378 if (tot == 0) {
a23a9ec6 4379 tot = 1;
fca8a500 4380 }
3a841ab5
DB		 4381 g_string_append_printf(buf, " gen_interm time %0.1f%%\n",
4382 (double)s->interm_time / tot * 100.0);
4383 g_string_append_printf(buf, " gen_code time %0.1f%%\n",
4384 (double)s->code_time / tot * 100.0);
4385 g_string_append_printf(buf, "optim./code time %0.1f%%\n",
4386 (double)s->opt_time / (s->code_time ?
4387 s->code_time : 1)
4388 * 100.0);
4389 g_string_append_printf(buf, "liveness/code time %0.1f%%\n",
4390 (double)s->la_time / (s->code_time ?
4391 s->code_time : 1) * 100.0);
4392 g_string_append_printf(buf, "cpu_restore count %" PRId64 "\n",
4393 s->restore_count);
4394 g_string_append_printf(buf, " avg cycles %0.1f\n",
4395 s->restore_count ?
4396 (double)s->restore_time / s->restore_count : 0);
a23a9ec6
FB		 4397}
4398#else
3a841ab5 4399void tcg_dump_info(GString *buf)
a23a9ec6 4400{
3a841ab5 4401 g_string_append_printf(buf, "[TCG profiler not compiled]\n");
a23a9ec6
FB		 4402}
4403#endif
813da627
RH		 4404
4405#ifdef ELF_HOST_MACHINE
5872bbf2
RH		 4406/* In order to use this feature, the backend needs to do three things:
4407
4408 (1) Define ELF_HOST_MACHINE to indicate both what value to
4409 put into the ELF image and to indicate support for the feature.
4410
4411 (2) Define tcg_register_jit. This should create a buffer containing
4412 the contents of a .debug_frame section that describes the post-
4413 prologue unwind info for the tcg machine.
4414
4415 (3) Call tcg_register_jit_int, with the constructed .debug_frame.
4416*/
813da627
RH		 4417
4418/* Begin GDB interface. THE FOLLOWING MUST MATCH GDB DOCS. */
4419typedef enum {
4420 JIT_NOACTION = 0,
4421 JIT_REGISTER_FN,
4422 JIT_UNREGISTER_FN
4423} jit_actions_t;
4424
4425struct jit_code_entry {
4426 struct jit_code_entry *next_entry;
4427 struct jit_code_entry *prev_entry;
4428 const void *symfile_addr;
4429 uint64_t symfile_size;
4430};
4431
4432struct jit_descriptor {
4433 uint32_t version;
4434 uint32_t action_flag;
4435 struct jit_code_entry *relevant_entry;
4436 struct jit_code_entry *first_entry;
4437};
4438
4439void __jit_debug_register_code(void) __attribute__((noinline));
4440void __jit_debug_register_code(void)
4441{
4442 asm("");
4443}
4444
4445/* Must statically initialize the version, because GDB may check
4446 the version before we can set it. */
4447struct jit_descriptor __jit_debug_descriptor = { 1, 0, 0, 0 };
4448
4449/* End GDB interface. */
4450
4451static int find_string(const char *strtab, const char *str)
4452{
4453 const char *p = strtab + 1;
4454
4455 while (1) {
4456 if (strcmp(p, str) == 0) {
4457 return p - strtab;
4458 }
4459 p += strlen(p) + 1;
4460 }
4461}
4462
755bf9e5 4463static void tcg_register_jit_int(const void *buf_ptr, size_t buf_size,
2c90784a
RH		 4464 const void *debug_frame,
4465 size_t debug_frame_size)
813da627 4466{
5872bbf2
RH		 4467 struct __attribute__((packed)) DebugInfo {
4468 uint32_t len;
4469 uint16_t version;
4470 uint32_t abbrev;
4471 uint8_t ptr_size;
4472 uint8_t cu_die;
4473 uint16_t cu_lang;
4474 uintptr_t cu_low_pc;
4475 uintptr_t cu_high_pc;
4476 uint8_t fn_die;
4477 char fn_name[16];
4478 uintptr_t fn_low_pc;
4479 uintptr_t fn_high_pc;
4480 uint8_t cu_eoc;
4481 };
813da627
RH		 4482
4483 struct ElfImage {
4484 ElfW(Ehdr) ehdr;
4485 ElfW(Phdr) phdr;
5872bbf2
RH		 4486 ElfW(Shdr) shdr[7];
4487 ElfW(Sym) sym[2];
4488 struct DebugInfo di;
4489 uint8_t da[24];
4490 char str[80];
4491 };
4492
4493 struct ElfImage *img;
4494
4495 static const struct ElfImage img_template = {
4496 .ehdr = {
4497 .e_ident[EI_MAG0] = ELFMAG0,
4498 .e_ident[EI_MAG1] = ELFMAG1,
4499 .e_ident[EI_MAG2] = ELFMAG2,
4500 .e_ident[EI_MAG3] = ELFMAG3,
4501 .e_ident[EI_CLASS] = ELF_CLASS,
4502 .e_ident[EI_DATA] = ELF_DATA,
4503 .e_ident[EI_VERSION] = EV_CURRENT,
4504 .e_type = ET_EXEC,
4505 .e_machine = ELF_HOST_MACHINE,
4506 .e_version = EV_CURRENT,
4507 .e_phoff = offsetof(struct ElfImage, phdr),
4508 .e_shoff = offsetof(struct ElfImage, shdr),
4509 .e_ehsize = sizeof(ElfW(Shdr)),
4510 .e_phentsize = sizeof(ElfW(Phdr)),
4511 .e_phnum = 1,
4512 .e_shentsize = sizeof(ElfW(Shdr)),
4513 .e_shnum = ARRAY_SIZE(img->shdr),
4514 .e_shstrndx = ARRAY_SIZE(img->shdr) - 1,
abbb3eae
RH		 4515#ifdef ELF_HOST_FLAGS
4516 .e_flags = ELF_HOST_FLAGS,
4517#endif
4518#ifdef ELF_OSABI
4519 .e_ident[EI_OSABI] = ELF_OSABI,
4520#endif
5872bbf2
RH		 4521 },
4522 .phdr = {
4523 .p_type = PT_LOAD,
4524 .p_flags = PF_X,
4525 },
4526 .shdr = {
4527 [0] = { .sh_type = SHT_NULL },
4528 /* Trick: The contents of code_gen_buffer are not present in
4529 this fake ELF file; that got allocated elsewhere. Therefore
4530 we mark .text as SHT_NOBITS (similar to .bss) so that readers
4531 will not look for contents. We can record any address. */
4532 [1] = { /* .text */
4533 .sh_type = SHT_NOBITS,
4534 .sh_flags = SHF_EXECINSTR | SHF_ALLOC,
4535 },
4536 [2] = { /* .debug_info */
4537 .sh_type = SHT_PROGBITS,
4538 .sh_offset = offsetof(struct ElfImage, di),
4539 .sh_size = sizeof(struct DebugInfo),
4540 },
4541 [3] = { /* .debug_abbrev */
4542 .sh_type = SHT_PROGBITS,
4543 .sh_offset = offsetof(struct ElfImage, da),
4544 .sh_size = sizeof(img->da),
4545 },
4546 [4] = { /* .debug_frame */
4547 .sh_type = SHT_PROGBITS,
4548 .sh_offset = sizeof(struct ElfImage),
4549 },
4550 [5] = { /* .symtab */
4551 .sh_type = SHT_SYMTAB,
4552 .sh_offset = offsetof(struct ElfImage, sym),
4553 .sh_size = sizeof(img->sym),
4554 .sh_info = 1,
4555 .sh_link = ARRAY_SIZE(img->shdr) - 1,
4556 .sh_entsize = sizeof(ElfW(Sym)),
4557 },
4558 [6] = { /* .strtab */
4559 .sh_type = SHT_STRTAB,
4560 .sh_offset = offsetof(struct ElfImage, str),
4561 .sh_size = sizeof(img->str),
4562 }
4563 },
4564 .sym = {
4565 [1] = { /* code_gen_buffer */
4566 .st_info = ELF_ST_INFO(STB_GLOBAL, STT_FUNC),
4567 .st_shndx = 1,
4568 }
4569 },
4570 .di = {
4571 .len = sizeof(struct DebugInfo) - 4,
4572 .version = 2,
4573 .ptr_size = sizeof(void *),
4574 .cu_die = 1,
4575 .cu_lang = 0x8001, /* DW_LANG_Mips_Assembler */
4576 .fn_die = 2,
4577 .fn_name = "code_gen_buffer"
4578 },
4579 .da = {
4580 1, /* abbrev number (the cu) */
4581 0x11, 1, /* DW_TAG_compile_unit, has children */
4582 0x13, 0x5, /* DW_AT_language, DW_FORM_data2 */
4583 0x11, 0x1, /* DW_AT_low_pc, DW_FORM_addr */
4584 0x12, 0x1, /* DW_AT_high_pc, DW_FORM_addr */
4585 0, 0, /* end of abbrev */
4586 2, /* abbrev number (the fn) */
4587 0x2e, 0, /* DW_TAG_subprogram, no children */
4588 0x3, 0x8, /* DW_AT_name, DW_FORM_string */
4589 0x11, 0x1, /* DW_AT_low_pc, DW_FORM_addr */
4590 0x12, 0x1, /* DW_AT_high_pc, DW_FORM_addr */
4591 0, 0, /* end of abbrev */
4592 0 /* no more abbrev */
4593 },
4594 .str = "\0" ".text\0" ".debug_info\0" ".debug_abbrev\0"
4595 ".debug_frame\0" ".symtab\0" ".strtab\0" "code_gen_buffer",
813da627
RH		 4596 };
4597
4598 /* We only need a single jit entry; statically allocate it. */
4599 static struct jit_code_entry one_entry;
4600
5872bbf2 4601 uintptr_t buf = (uintptr_t)buf_ptr;
813da627 4602 size_t img_size = sizeof(struct ElfImage) + debug_frame_size;
2c90784a 4603 DebugFrameHeader *dfh;
813da627 4604
5872bbf2
RH		 4605 img = g_malloc(img_size);
4606 *img = img_template;
813da627 4607
5872bbf2
RH		 4608 img->phdr.p_vaddr = buf;
4609 img->phdr.p_paddr = buf;
4610 img->phdr.p_memsz = buf_size;
813da627 4611
813da627 4612 img->shdr[1].sh_name = find_string(img->str, ".text");
5872bbf2 4613 img->shdr[1].sh_addr = buf;
813da627
RH		 4614 img->shdr[1].sh_size = buf_size;
4615
5872bbf2
RH		 4616 img->shdr[2].sh_name = find_string(img->str, ".debug_info");
4617 img->shdr[3].sh_name = find_string(img->str, ".debug_abbrev");
4618
4619 img->shdr[4].sh_name = find_string(img->str, ".debug_frame");
4620 img->shdr[4].sh_size = debug_frame_size;
4621
4622 img->shdr[5].sh_name = find_string(img->str, ".symtab");
4623 img->shdr[6].sh_name = find_string(img->str, ".strtab");
4624
4625 img->sym[1].st_name = find_string(img->str, "code_gen_buffer");
4626 img->sym[1].st_value = buf;
4627 img->sym[1].st_size = buf_size;
813da627 4628
5872bbf2 4629 img->di.cu_low_pc = buf;
45aba097 4630 img->di.cu_high_pc = buf + buf_size;
5872bbf2 4631 img->di.fn_low_pc = buf;
45aba097 4632 img->di.fn_high_pc = buf + buf_size;
813da627 4633
2c90784a
RH		 4634 dfh = (DebugFrameHeader *)(img + 1);
4635 memcpy(dfh, debug_frame, debug_frame_size);
4636 dfh->fde.func_start = buf;
4637 dfh->fde.func_len = buf_size;
4638
813da627
RH		 4639#ifdef DEBUG_JIT
4640 /* Enable this block to be able to debug the ELF image file creation.
4641 One can use readelf, objdump, or other inspection utilities. */
4642 {
eb6b2edf
BM		 4643 g_autofree char *jit = g_strdup_printf("%s/qemu.jit", g_get_tmp_dir());
4644 FILE *f = fopen(jit, "w+b");
813da627 4645 if (f) {
5872bbf2 4646 if (fwrite(img, img_size, 1, f) != img_size) {
813da627
RH		 4647 /* Avoid stupid unused return value warning for fwrite. */
4648 }
4649 fclose(f);
4650 }
4651 }
4652#endif
4653
4654 one_entry.symfile_addr = img;
4655 one_entry.symfile_size = img_size;
4656
4657 __jit_debug_descriptor.action_flag = JIT_REGISTER_FN;
4658 __jit_debug_descriptor.relevant_entry = &one_entry;
4659 __jit_debug_descriptor.first_entry = &one_entry;
4660 __jit_debug_register_code();
4661}
4662#else
5872bbf2
RH		 4663/* No support for the feature. Provide the entry point expected by exec.c,
4664 and implement the internal function we declared earlier. */
813da627 4665
755bf9e5 4666static void tcg_register_jit_int(const void *buf, size_t size,
2c90784a
RH		 4667 const void *debug_frame,
4668 size_t debug_frame_size)
813da627
RH		 4669{
4670}
4671
755bf9e5 4672void tcg_register_jit(const void *buf, size_t buf_size)
813da627
RH		 4673{
4674}
4675#endif /* ELF_HOST_MACHINE */
db432672
RH		 4676
4677#if !TCG_TARGET_MAYBE_vec
4678void tcg_expand_vec_op(TCGOpcode o, TCGType t, unsigned e, TCGArg a0, ...)
4679{
4680 g_assert_not_reached();
4681}
4682#endif
Empty description
This page took 1.933385 seconds and 4 git commands to generate.