CPU definitions. Currently they are used for qemu_ld/st
instructions */
#define NO_CPU_IO_DEFS
-#include "cpu.h"
#include "exec/exec-all.h"
#include "elf.h"
#include "exec/log.h"
-#include "sysemu/sysemu.h"
/* Forward declarations for functions declared in tcg-target.c.inc and
used here. */
static void tcg_target_init(TCGContext *s);
-static const TCGTargetOpDef *tcg_target_op_def(TCGOpcode);
static void tcg_target_qemu_prologue(TCGContext *s);
static bool patch_reloc(tcg_insn_unit *code_ptr, int type,
intptr_t value, intptr_t addend);
DebugFrameFDEHeader fde;
} DebugFrameHeader;
-static void tcg_register_jit_int(void *buf, size_t size,
+static void tcg_register_jit_int(const void *buf, size_t size,
const void *debug_frame,
size_t debug_frame_size)
__attribute__((unused));
/* Forward declarations for functions declared and used in tcg-target.c.inc. */
-static const char *target_parse_constraint(TCGArgConstraint *ct,
- const char *ct_str, TCGType type);
static void tcg_out_ld(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg1,
intptr_t arg2);
static bool tcg_out_mov(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg);
static void tcg_out_movi(TCGContext *s, TCGType type,
TCGReg ret, tcg_target_long arg);
-static void tcg_out_op(TCGContext *s, TCGOpcode opc, const TCGArg *args,
- const int *const_args);
+static void tcg_out_op(TCGContext *s, TCGOpcode opc,
+ const TCGArg args[TCG_MAX_OP_ARGS],
+ const int const_args[TCG_MAX_OP_ARGS]);
#if TCG_TARGET_MAYBE_vec
static bool tcg_out_dup_vec(TCGContext *s, TCGType type, unsigned vece,
TCGReg dst, TCGReg src);
static bool tcg_out_dupm_vec(TCGContext *s, TCGType type, unsigned vece,
TCGReg dst, TCGReg base, intptr_t offset);
-static void tcg_out_dupi_vec(TCGContext *s, TCGType type,
- TCGReg dst, tcg_target_long arg);
-static void tcg_out_vec_op(TCGContext *s, TCGOpcode opc, unsigned vecl,
- unsigned vece, const TCGArg *args,
- const int *const_args);
+static void tcg_out_dupi_vec(TCGContext *s, TCGType type, unsigned vece,
+ TCGReg dst, int64_t arg);
+static void tcg_out_vec_op(TCGContext *s, TCGOpcode opc,
+ unsigned vecl, unsigned vece,
+ const TCGArg args[TCG_MAX_OP_ARGS],
+ const int const_args[TCG_MAX_OP_ARGS]);
#else
static inline bool tcg_out_dup_vec(TCGContext *s, TCGType type, unsigned vece,
TCGReg dst, TCGReg src)
{
g_assert_not_reached();
}
-static inline void tcg_out_dupi_vec(TCGContext *s, TCGType type,
- TCGReg dst, tcg_target_long arg)
+static inline void tcg_out_dupi_vec(TCGContext *s, TCGType type, unsigned vece,
+ TCGReg dst, int64_t arg)
{
g_assert_not_reached();
}
-static inline void tcg_out_vec_op(TCGContext *s, TCGOpcode opc, unsigned vecl,
- unsigned vece, const TCGArg *args,
- const int *const_args)
+static inline void tcg_out_vec_op(TCGContext *s, TCGOpcode opc,
+ unsigned vecl, unsigned vece,
+ const TCGArg args[TCG_MAX_OP_ARGS],
+ const int const_args[TCG_MAX_OP_ARGS])
{
g_assert_not_reached();
}
intptr_t arg2);
static bool tcg_out_sti(TCGContext *s, TCGType type, TCGArg val,
TCGReg base, intptr_t ofs);
-static void tcg_out_call(TCGContext *s, tcg_insn_unit *target);
-static int tcg_target_const_match(tcg_target_long val, TCGType type,
- const TCGArgConstraint *arg_ct);
+static void tcg_out_call(TCGContext *s, const tcg_insn_unit *target);
+static bool tcg_target_const_match(int64_t val, TCGType type, int ct);
#ifdef TCG_TARGET_NEED_LDST_LABELS
static int tcg_out_ldst_finalize(TCGContext *s);
#endif
static TCGContext **tcg_ctxs;
static unsigned int n_tcg_ctxs;
TCGv_env cpu_env = 0;
+const void *tcg_code_gen_epilogue;
+uintptr_t tcg_splitwx_diff;
#ifndef CONFIG_TCG_INTERPRETER
tcg_prologue_fn *tcg_qemu_tb_exec;
QSIMPLEQ_INSERT_TAIL(&l->relocs, r, next);
}
-static void tcg_out_label(TCGContext *s, TCGLabel *l, tcg_insn_unit *ptr)
+static void tcg_out_label(TCGContext *s, TCGLabel *l)
{
tcg_debug_assert(!l->has_value);
l->has_value = 1;
- l->u.value_ptr = ptr;
+ l->u.value_ptr = tcg_splitwx_to_rx(s->code_ptr);
}
TCGLabel *gen_new_label(void)
s->tb_jmp_reset_offset[which] = tcg_current_code_size(s);
}
+/* Signal overflow, starting over with fewer guest insns. */
+static void QEMU_NORETURN tcg_raise_tb_overflow(TCGContext *s)
+{
+ siglongjmp(s->jmp_trans, -2);
+}
+
+#define C_PFX1(P, A) P##A
+#define C_PFX2(P, A, B) P##A##_##B
+#define C_PFX3(P, A, B, C) P##A##_##B##_##C
+#define C_PFX4(P, A, B, C, D) P##A##_##B##_##C##_##D
+#define C_PFX5(P, A, B, C, D, E) P##A##_##B##_##C##_##D##_##E
+#define C_PFX6(P, A, B, C, D, E, F) P##A##_##B##_##C##_##D##_##E##_##F
+
+/* Define an enumeration for the various combinations. */
+
+#define C_O0_I1(I1) C_PFX1(c_o0_i1_, I1),
+#define C_O0_I2(I1, I2) C_PFX2(c_o0_i2_, I1, I2),
+#define C_O0_I3(I1, I2, I3) C_PFX3(c_o0_i3_, I1, I2, I3),
+#define C_O0_I4(I1, I2, I3, I4) C_PFX4(c_o0_i4_, I1, I2, I3, I4),
+
+#define C_O1_I1(O1, I1) C_PFX2(c_o1_i1_, O1, I1),
+#define C_O1_I2(O1, I1, I2) C_PFX3(c_o1_i2_, O1, I1, I2),
+#define C_O1_I3(O1, I1, I2, I3) C_PFX4(c_o1_i3_, O1, I1, I2, I3),
+#define C_O1_I4(O1, I1, I2, I3, I4) C_PFX5(c_o1_i4_, O1, I1, I2, I3, I4),
+
+#define C_N1_I2(O1, I1, I2) C_PFX3(c_n1_i2_, O1, I1, I2),
+
+#define C_O2_I1(O1, O2, I1) C_PFX3(c_o2_i1_, O1, O2, I1),
+#define C_O2_I2(O1, O2, I1, I2) C_PFX4(c_o2_i2_, O1, O2, I1, I2),
+#define C_O2_I3(O1, O2, I1, I2, I3) C_PFX5(c_o2_i3_, O1, O2, I1, I2, I3),
+#define C_O2_I4(O1, O2, I1, I2, I3, I4) C_PFX6(c_o2_i4_, O1, O2, I1, I2, I3, I4),
+
+typedef enum {
+#include "tcg-target-con-set.h"
+} TCGConstraintSetIndex;
+
+static TCGConstraintSetIndex tcg_target_op_def(TCGOpcode);
+
+#undef C_O0_I1
+#undef C_O0_I2
+#undef C_O0_I3
+#undef C_O0_I4
+#undef C_O1_I1
+#undef C_O1_I2
+#undef C_O1_I3
+#undef C_O1_I4
+#undef C_N1_I2
+#undef C_O2_I1
+#undef C_O2_I2
+#undef C_O2_I3
+#undef C_O2_I4
+
+/* Put all of the constraint sets into an array, indexed by the enum. */
+
+#define C_O0_I1(I1) { .args_ct_str = { #I1 } },
+#define C_O0_I2(I1, I2) { .args_ct_str = { #I1, #I2 } },
+#define C_O0_I3(I1, I2, I3) { .args_ct_str = { #I1, #I2, #I3 } },
+#define C_O0_I4(I1, I2, I3, I4) { .args_ct_str = { #I1, #I2, #I3, #I4 } },
+
+#define C_O1_I1(O1, I1) { .args_ct_str = { #O1, #I1 } },
+#define C_O1_I2(O1, I1, I2) { .args_ct_str = { #O1, #I1, #I2 } },
+#define C_O1_I3(O1, I1, I2, I3) { .args_ct_str = { #O1, #I1, #I2, #I3 } },
+#define C_O1_I4(O1, I1, I2, I3, I4) { .args_ct_str = { #O1, #I1, #I2, #I3, #I4 } },
+
+#define C_N1_I2(O1, I1, I2) { .args_ct_str = { "&" #O1, #I1, #I2 } },
+
+#define C_O2_I1(O1, O2, I1) { .args_ct_str = { #O1, #O2, #I1 } },
+#define C_O2_I2(O1, O2, I1, I2) { .args_ct_str = { #O1, #O2, #I1, #I2 } },
+#define C_O2_I3(O1, O2, I1, I2, I3) { .args_ct_str = { #O1, #O2, #I1, #I2, #I3 } },
+#define C_O2_I4(O1, O2, I1, I2, I3, I4) { .args_ct_str = { #O1, #O2, #I1, #I2, #I3, #I4 } },
+
+static const TCGTargetOpDef constraint_sets[] = {
+#include "tcg-target-con-set.h"
+};
+
+
+#undef C_O0_I1
+#undef C_O0_I2
+#undef C_O0_I3
+#undef C_O0_I4
+#undef C_O1_I1
+#undef C_O1_I2
+#undef C_O1_I3
+#undef C_O1_I4
+#undef C_N1_I2
+#undef C_O2_I1
+#undef C_O2_I2
+#undef C_O2_I3
+#undef C_O2_I4
+
+/* Expand the enumerator to be returned from tcg_target_op_def(). */
+
+#define C_O0_I1(I1) C_PFX1(c_o0_i1_, I1)
+#define C_O0_I2(I1, I2) C_PFX2(c_o0_i2_, I1, I2)
+#define C_O0_I3(I1, I2, I3) C_PFX3(c_o0_i3_, I1, I2, I3)
+#define C_O0_I4(I1, I2, I3, I4) C_PFX4(c_o0_i4_, I1, I2, I3, I4)
+
+#define C_O1_I1(O1, I1) C_PFX2(c_o1_i1_, O1, I1)
+#define C_O1_I2(O1, I1, I2) C_PFX3(c_o1_i2_, O1, I1, I2)
+#define C_O1_I3(O1, I1, I2, I3) C_PFX4(c_o1_i3_, O1, I1, I2, I3)
+#define C_O1_I4(O1, I1, I2, I3, I4) C_PFX5(c_o1_i4_, O1, I1, I2, I3, I4)
+
+#define C_N1_I2(O1, I1, I2) C_PFX3(c_n1_i2_, O1, I1, I2)
+
+#define C_O2_I1(O1, O2, I1) C_PFX3(c_o2_i1_, O1, O2, I1)
+#define C_O2_I2(O1, O2, I1, I2) C_PFX4(c_o2_i2_, O1, O2, I1, I2)
+#define C_O2_I3(O1, O2, I1, I2, I3) C_PFX5(c_o2_i3_, O1, O2, I1, I2, I3)
+#define C_O2_I4(O1, O2, I1, I2, I3, I4) C_PFX6(c_o2_i4_, O1, O2, I1, I2, I3, I4)
+
#include "tcg-target.c.inc"
/* compare a pointer @ptr and a tb_tc @s */
}
}
-static struct tcg_region_tree *tc_ptr_to_region_tree(void *p)
+static struct tcg_region_tree *tc_ptr_to_region_tree(const void *p)
{
size_t region_idx;
+ /*
+ * Like tcg_splitwx_to_rw, with no assert. The pc may come from
+ * a signal handler over which the caller has no control.
+ */
+ if (!in_code_gen_buffer(p)) {
+ p -= tcg_splitwx_diff;
+ if (!in_code_gen_buffer(p)) {
+ return NULL;
+ }
+ }
+
if (p < region.start_aligned) {
region_idx = 0;
} else {
{
struct tcg_region_tree *rt = tc_ptr_to_region_tree(tb->tc.ptr);
+ g_assert(rt != NULL);
qemu_mutex_lock(&rt->lock);
g_tree_insert(rt->tree, &tb->tc, tb);
qemu_mutex_unlock(&rt->lock);
{
struct tcg_region_tree *rt = tc_ptr_to_region_tree(tb->tc.ptr);
+ g_assert(rt != NULL);
qemu_mutex_lock(&rt->lock);
g_tree_remove(rt->tree, &tb->tc);
qemu_mutex_unlock(&rt->lock);
TranslationBlock *tb;
struct tb_tc s = { .ptr = (void *)tc_ptr };
+ if (rt == NULL) {
+ return NULL;
+ }
+
qemu_mutex_lock(&rt->lock);
tb = g_tree_lookup(rt->tree, &s);
qemu_mutex_unlock(&rt->lock);
/* account for that last guard page */
region.end -= page_size;
- /* set guard pages */
+ /*
+ * Set guard pages in the rw buffer, as that's the one into which
+ * buffer overruns could occur. Do not set guard pages in the rx
+ * buffer -- let that one use hugepages throughout.
+ */
for (i = 0; i < region.n; i++) {
void *start, *end;
- int rc;
tcg_region_bounds(i, &start, &end);
- rc = qemu_mprotect_none(end, page_size);
- g_assert(!rc);
+
+ /*
+ * macOS 11.2 has a bug (Apple Feedback FB8994773) in which mprotect
+ * rejects a permission change from RWX -> NONE. Guard pages are
+ * nice for bug detection but are not essential; ignore any failure.
+ */
+ (void)qemu_mprotect_none(end, page_size);
}
tcg_region_trees_init();
#endif
}
+#ifdef CONFIG_DEBUG_TCG
+const void *tcg_splitwx_to_rx(void *rw)
+{
+ /* Pass NULL pointers unchanged. */
+ if (rw) {
+ g_assert(in_code_gen_buffer(rw));
+ rw += tcg_splitwx_diff;
+ }
+ return rw;
+}
+
+void *tcg_splitwx_to_rw(const void *rx)
+{
+ /* Pass NULL pointers unchanged. */
+ if (rx) {
+ rx -= tcg_splitwx_diff;
+ /* Assert that we end with a pointer in the rw region. */
+ g_assert(in_code_gen_buffer(rx));
+ }
+ return (void *)rx;
+}
+#endif /* CONFIG_DEBUG_TCG */
+
static void alloc_tcg_plugin_context(TCGContext *s)
{
#ifdef CONFIG_PLUGIN
s->code_buf = buf0;
s->data_gen_ptr = NULL;
+ /*
+ * The region trees are not yet configured, but tcg_splitwx_to_rx
+ * needs the bounds for an assert.
+ */
+ region.start = buf0;
+ region.end = buf0 + total_size;
+
#ifndef CONFIG_TCG_INTERPRETER
- tcg_qemu_tb_exec = (tcg_prologue_fn *)buf0;
+ tcg_qemu_tb_exec = (tcg_prologue_fn *)tcg_splitwx_to_rx(buf0);
#endif
/* Compute a high-water mark, at which we voluntarily flush the buffer
s->pool_labels = NULL;
#endif
+ qemu_thread_jit_write();
/* Generate the prologue. */
tcg_target_qemu_prologue(s);
buf1 = s->code_ptr;
#ifndef CONFIG_TCG_INTERPRETER
- flush_idcache_range((uintptr_t)buf0, (uintptr_t)buf0,
+ flush_idcache_range((uintptr_t)tcg_splitwx_to_rx(buf0), (uintptr_t)buf0,
tcg_ptr_byte_diff(buf1, buf0));
#endif
total_size -= prologue_size;
s->code_gen_buffer_size = total_size;
- tcg_register_jit(s->code_gen_buffer, total_size);
+ tcg_register_jit(tcg_splitwx_to_rx(s->code_gen_buffer), total_size);
#ifdef DEBUG_DISAS
if (qemu_loglevel_mask(CPU_LOG_TB_OUT_ASM)) {
/* Assert that goto_ptr is implemented completely. */
if (TCG_TARGET_HAS_goto_ptr) {
- tcg_debug_assert(s->code_gen_epilogue != NULL);
+ tcg_debug_assert(tcg_code_gen_epilogue != NULL);
}
}
/* No temps have been previously allocated for size or locality. */
memset(s->free_temps, 0, sizeof(s->free_temps));
+ /* No constant temps have been previously allocated. */
+ for (int i = 0; i < TCG_TYPE_COUNT; ++i) {
+ if (s->const_table[i]) {
+ g_hash_table_remove_all(s->const_table[i]);
+ }
+ }
+
s->nb_ops = 0;
s->nb_labels = 0;
s->current_frame_offset = s->frame_start;
QSIMPLEQ_INIT(&s->labels);
}
-static inline TCGTemp *tcg_temp_alloc(TCGContext *s)
+static TCGTemp *tcg_temp_alloc(TCGContext *s)
{
int n = s->nb_temps++;
- tcg_debug_assert(n < TCG_MAX_TEMPS);
+
+ if (n >= TCG_MAX_TEMPS) {
+ tcg_raise_tb_overflow(s);
+ }
return memset(&s->temps[n], 0, sizeof(TCGTemp));
}
-static inline TCGTemp *tcg_global_alloc(TCGContext *s)
+static TCGTemp *tcg_global_alloc(TCGContext *s)
{
TCGTemp *ts;
tcg_debug_assert(s->nb_globals == s->nb_temps);
+ tcg_debug_assert(s->nb_globals < TCG_MAX_TEMPS);
s->nb_globals++;
ts = tcg_temp_alloc(s);
- ts->temp_global = 1;
+ ts->kind = TEMP_GLOBAL;
return ts;
}
ts = tcg_global_alloc(s);
ts->base_type = type;
ts->type = type;
- ts->fixed_reg = 1;
+ ts->kind = TEMP_FIXED;
ts->reg = reg;
ts->name = name;
tcg_regset_set_reg(s->reserved_regs, reg);
bigendian = 1;
#endif
- if (!base_ts->fixed_reg) {
+ switch (base_ts->kind) {
+ case TEMP_FIXED:
+ break;
+ case TEMP_GLOBAL:
/* We do not support double-indirect registers. */
tcg_debug_assert(!base_ts->indirect_reg);
base_ts->indirect_base = 1;
s->nb_indirects += (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64
? 2 : 1);
indirect_reg = 1;
+ break;
+ default:
+ g_assert_not_reached();
}
if (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64) {
TCGTemp *tcg_temp_new_internal(TCGType type, bool temp_local)
{
TCGContext *s = tcg_ctx;
+ TCGTempKind kind = temp_local ? TEMP_LOCAL : TEMP_NORMAL;
TCGTemp *ts;
int idx, k;
ts = &s->temps[idx];
ts->temp_allocated = 1;
tcg_debug_assert(ts->base_type == type);
- tcg_debug_assert(ts->temp_local == temp_local);
+ tcg_debug_assert(ts->kind == kind);
} else {
ts = tcg_temp_alloc(s);
if (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64) {
ts->base_type = type;
ts->type = TCG_TYPE_I32;
ts->temp_allocated = 1;
- ts->temp_local = temp_local;
+ ts->kind = kind;
tcg_debug_assert(ts2 == ts + 1);
ts2->base_type = TCG_TYPE_I64;
ts2->type = TCG_TYPE_I32;
ts2->temp_allocated = 1;
- ts2->temp_local = temp_local;
+ ts2->kind = kind;
} else {
ts->base_type = type;
ts->type = type;
ts->temp_allocated = 1;
- ts->temp_local = temp_local;
+ ts->kind = kind;
}
}
TCGContext *s = tcg_ctx;
int k, idx;
+ /* In order to simplify users of tcg_constant_*, silently ignore free. */
+ if (ts->kind == TEMP_CONST) {
+ return;
+ }
+
#if defined(CONFIG_DEBUG_TCG)
s->temps_in_use--;
if (s->temps_in_use < 0) {
}
#endif
- tcg_debug_assert(ts->temp_global == 0);
+ tcg_debug_assert(ts->kind < TEMP_GLOBAL);
tcg_debug_assert(ts->temp_allocated != 0);
ts->temp_allocated = 0;
idx = temp_idx(ts);
- k = ts->base_type + (ts->temp_local ? TCG_TYPE_COUNT : 0);
+ k = ts->base_type + (ts->kind == TEMP_NORMAL ? 0 : TCG_TYPE_COUNT);
set_bit(idx, s->free_temps[k].l);
}
+TCGTemp *tcg_constant_internal(TCGType type, int64_t val)
+{
+ TCGContext *s = tcg_ctx;
+ GHashTable *h = s->const_table[type];
+ TCGTemp *ts;
+
+ if (h == NULL) {
+ h = g_hash_table_new(g_int64_hash, g_int64_equal);
+ s->const_table[type] = h;
+ }
+
+ ts = g_hash_table_lookup(h, &val);
+ if (ts == NULL) {
+ ts = tcg_temp_alloc(s);
+
+ if (TCG_TARGET_REG_BITS == 32 && type == TCG_TYPE_I64) {
+ TCGTemp *ts2 = tcg_temp_alloc(s);
+
+ ts->base_type = TCG_TYPE_I64;
+ ts->type = TCG_TYPE_I32;
+ ts->kind = TEMP_CONST;
+ ts->temp_allocated = 1;
+ /*
+ * Retain the full value of the 64-bit constant in the low
+ * part, so that the hash table works. Actual uses will
+ * truncate the value to the low part.
+ */
+ ts->val = val;
+
+ tcg_debug_assert(ts2 == ts + 1);
+ ts2->base_type = TCG_TYPE_I64;
+ ts2->type = TCG_TYPE_I32;
+ ts2->kind = TEMP_CONST;
+ ts2->temp_allocated = 1;
+ ts2->val = val >> 32;
+ } else {
+ ts->base_type = type;
+ ts->type = type;
+ ts->kind = TEMP_CONST;
+ ts->temp_allocated = 1;
+ ts->val = val;
+ }
+ g_hash_table_insert(h, &ts->val, ts);
+ }
+
+ return ts;
+}
+
+TCGv_vec tcg_constant_vec(TCGType type, unsigned vece, int64_t val)
+{
+ val = dup_const(vece, val);
+ return temp_tcgv_vec(tcg_constant_internal(type, val));
+}
+
+TCGv_vec tcg_constant_vec_matching(TCGv_vec match, unsigned vece, int64_t val)
+{
+ TCGTemp *t = tcgv_vec_temp(match);
+
+ tcg_debug_assert(t->temp_allocated != 0);
+ return tcg_constant_vec(t->base_type, vece, val);
+}
+
TCGv_i32 tcg_const_i32(int32_t val)
{
TCGv_i32 t0;
return TCG_TARGET_HAS_goto_ptr;
case INDEX_op_mov_i32:
- case INDEX_op_movi_i32:
case INDEX_op_setcond_i32:
case INDEX_op_brcond_i32:
case INDEX_op_ld8u_i32:
return TCG_TARGET_REG_BITS == 32;
case INDEX_op_mov_i64:
- case INDEX_op_movi_i64:
case INDEX_op_setcond_i64:
case INDEX_op_brcond_i64:
case INDEX_op_ld8u_i64:
case INDEX_op_mov_vec:
case INDEX_op_dup_vec:
- case INDEX_op_dupi_vec:
case INDEX_op_dupm_vec:
case INDEX_op_ld_vec:
case INDEX_op_st_vec:
static void tcg_reg_alloc_start(TCGContext *s)
{
int i, n;
- TCGTemp *ts;
- for (i = 0, n = s->nb_globals; i < n; i++) {
- ts = &s->temps[i];
- ts->val_type = (ts->fixed_reg ? TEMP_VAL_REG : TEMP_VAL_MEM);
- }
- for (n = s->nb_temps; i < n; i++) {
- ts = &s->temps[i];
- ts->val_type = (ts->temp_local ? TEMP_VAL_MEM : TEMP_VAL_DEAD);
- ts->mem_allocated = 0;
- ts->fixed_reg = 0;
+ for (i = 0, n = s->nb_temps; i < n; i++) {
+ TCGTemp *ts = &s->temps[i];
+ TCGTempVal val = TEMP_VAL_MEM;
+
+ switch (ts->kind) {
+ case TEMP_CONST:
+ val = TEMP_VAL_CONST;
+ break;
+ case TEMP_FIXED:
+ val = TEMP_VAL_REG;
+ break;
+ case TEMP_GLOBAL:
+ break;
+ case TEMP_NORMAL:
+ val = TEMP_VAL_DEAD;
+ /* fall through */
+ case TEMP_LOCAL:
+ ts->mem_allocated = 0;
+ break;
+ default:
+ g_assert_not_reached();
+ }
+ ts->val_type = val;
}
memset(s->reg_to_temp, 0, sizeof(s->reg_to_temp));
{
int idx = temp_idx(ts);
- if (ts->temp_global) {
+ switch (ts->kind) {
+ case TEMP_FIXED:
+ case TEMP_GLOBAL:
pstrcpy(buf, buf_size, ts->name);
- } else if (ts->temp_local) {
+ break;
+ case TEMP_LOCAL:
snprintf(buf, buf_size, "loc%d", idx - s->nb_globals);
- } else {
+ break;
+ case TEMP_NORMAL:
snprintf(buf, buf_size, "tmp%d", idx - s->nb_globals);
+ break;
+ case TEMP_CONST:
+ switch (ts->type) {
+ case TCG_TYPE_I32:
+ snprintf(buf, buf_size, "$0x%x", (int32_t)ts->val);
+ break;
+#if TCG_TARGET_REG_BITS > 32
+ case TCG_TYPE_I64:
+ snprintf(buf, buf_size, "$0x%" PRIx64, ts->val);
+ break;
+#endif
+ case TCG_TYPE_V64:
+ case TCG_TYPE_V128:
+ case TCG_TYPE_V256:
+ snprintf(buf, buf_size, "v%d$0x%" PRIx64,
+ 64 << (ts->type - TCG_TYPE_V64), ts->val);
+ break;
+ default:
+ g_assert_not_reached();
+ }
+ break;
}
return buf;
}
for (op = 0; op < NB_OPS; op++) {
TCGOpDef *def = &tcg_op_defs[op];
const TCGTargetOpDef *tdefs;
- TCGType type;
int i, nb_args;
if (def->flags & TCG_OPF_NOT_PRESENT) {
continue;
}
- tdefs = tcg_target_op_def(op);
- /* Missing TCGTargetOpDef entry. */
- tcg_debug_assert(tdefs != NULL);
+ /*
+ * Macro magic should make it impossible, but double-check that
+ * the array index is in range. Since the signness of an enum
+ * is implementation defined, force the result to unsigned.
+ */
+ unsigned con_set = tcg_target_op_def(op);
+ tcg_debug_assert(con_set < ARRAY_SIZE(constraint_sets));
+ tdefs = &constraint_sets[con_set];
- type = (def->flags & TCG_OPF_64BIT ? TCG_TYPE_I64 : TCG_TYPE_I32);
for (i = 0; i < nb_args; i++) {
const char *ct_str = tdefs->args_ct_str[i];
/* Incomplete TCGTargetOpDef entry. */
def->args_ct[i].ct |= TCG_CT_CONST;
ct_str++;
break;
+
+ /* Include all of the target-specific constraints. */
+
+#undef CONST
+#define CONST(CASE, MASK) \
+ case CASE: def->args_ct[i].ct |= MASK; ct_str++; break;
+#define REGS(CASE, MASK) \
+ case CASE: def->args_ct[i].regs |= MASK; ct_str++; break;
+
+#include "tcg-target-con-str.h"
+
+#undef REGS
+#undef CONST
default:
- ct_str = target_parse_constraint(&def->args_ct[i],
- ct_str, type);
/* Typo in TCGTargetOpDef constraint. */
- tcg_debug_assert(ct_str != NULL);
+ g_assert_not_reached();
}
}
}
{
int i;
- for (i = 0; i < ng; ++i) {
- s->temps[i].state = TS_DEAD | TS_MEM;
- la_reset_pref(&s->temps[i]);
- }
- for (i = ng; i < nt; ++i) {
- s->temps[i].state = (s->temps[i].temp_local
- ? TS_DEAD | TS_MEM
- : TS_DEAD);
- la_reset_pref(&s->temps[i]);
+ for (i = 0; i < nt; ++i) {
+ TCGTemp *ts = &s->temps[i];
+ int state;
+
+ switch (ts->kind) {
+ case TEMP_FIXED:
+ case TEMP_GLOBAL:
+ case TEMP_LOCAL:
+ state = TS_DEAD | TS_MEM;
+ break;
+ case TEMP_NORMAL:
+ case TEMP_CONST:
+ state = TS_DEAD;
+ break;
+ default:
+ g_assert_not_reached();
+ }
+ ts->state = state;
+ la_reset_pref(ts);
}
}
la_global_sync(s, ng);
for (int i = ng; i < nt; ++i) {
- if (s->temps[i].temp_local) {
- int state = s->temps[i].state;
- s->temps[i].state = state | TS_MEM;
+ TCGTemp *ts = &s->temps[i];
+ int state;
+
+ switch (ts->kind) {
+ case TEMP_LOCAL:
+ state = ts->state;
+ ts->state = state | TS_MEM;
if (state != TS_DEAD) {
continue;
}
- } else {
+ break;
+ case TEMP_NORMAL:
s->temps[i].state = TS_DEAD;
+ break;
+ case TEMP_CONST:
+ continue;
+ default:
+ g_assert_not_reached();
}
la_reset_pref(&s->temps[i]);
}
tcg_target_reg_names[ts->mem_base->reg]);
break;
case TEMP_VAL_CONST:
- printf("$0x%" TCG_PRIlx, ts->val);
+ printf("$0x%" PRIx64, ts->val);
break;
case TEMP_VAL_DEAD:
printf("D");
}
for (k = 0; k < s->nb_temps; k++) {
ts = &s->temps[k];
- if (ts->val_type == TEMP_VAL_REG && !ts->fixed_reg
+ if (ts->val_type == TEMP_VAL_REG
+ && ts->kind != TEMP_FIXED
&& s->reg_to_temp[ts->reg] != ts) {
printf("Inconsistency for temp %s:\n",
tcg_get_arg_str_ptr(s, buf, sizeof(buf), ts));
mark it free; otherwise mark it dead. */
static void temp_free_or_dead(TCGContext *s, TCGTemp *ts, int free_or_dead)
{
- if (ts->fixed_reg) {
+ TCGTempVal new_type;
+
+ switch (ts->kind) {
+ case TEMP_FIXED:
return;
+ case TEMP_GLOBAL:
+ case TEMP_LOCAL:
+ new_type = TEMP_VAL_MEM;
+ break;
+ case TEMP_NORMAL:
+ new_type = free_or_dead < 0 ? TEMP_VAL_MEM : TEMP_VAL_DEAD;
+ break;
+ case TEMP_CONST:
+ new_type = TEMP_VAL_CONST;
+ break;
+ default:
+ g_assert_not_reached();
}
if (ts->val_type == TEMP_VAL_REG) {
s->reg_to_temp[ts->reg] = NULL;
}
- ts->val_type = (free_or_dead < 0
- || ts->temp_local
- || ts->temp_global
- ? TEMP_VAL_MEM : TEMP_VAL_DEAD);
+ ts->val_type = new_type;
}
/* Mark a temporary as dead. */
static void temp_sync(TCGContext *s, TCGTemp *ts, TCGRegSet allocated_regs,
TCGRegSet preferred_regs, int free_or_dead)
{
- if (ts->fixed_reg) {
- return;
- }
- if (!ts->mem_coherent) {
+ if (!temp_readonly(ts) && !ts->mem_coherent) {
if (!ts->mem_allocated) {
temp_allocate_frame(s, ts);
}
case TEMP_VAL_CONST:
reg = tcg_reg_alloc(s, desired_regs, allocated_regs,
preferred_regs, ts->indirect_base);
- tcg_out_movi(s, ts->type, reg, ts->val);
+ if (ts->type <= TCG_TYPE_I64) {
+ tcg_out_movi(s, ts->type, reg, ts->val);
+ } else {
+ uint64_t val = ts->val;
+ MemOp vece = MO_64;
+
+ /*
+ * Find the minimal vector element that matches the constant.
+ * The targets will, in general, have to do this search anyway,
+ * do this generically.
+ */
+ if (val == dup_const(MO_8, val)) {
+ vece = MO_8;
+ } else if (val == dup_const(MO_16, val)) {
+ vece = MO_16;
+ } else if (val == dup_const(MO_32, val)) {
+ vece = MO_32;
+ }
+
+ tcg_out_dupi_vec(s, ts->type, vece, reg, ts->val);
+ }
ts->mem_coherent = 0;
break;
case TEMP_VAL_MEM:
{
/* The liveness analysis already ensures that globals are back
in memory. Keep an tcg_debug_assert for safety. */
- tcg_debug_assert(ts->val_type == TEMP_VAL_MEM || ts->fixed_reg);
+ tcg_debug_assert(ts->val_type == TEMP_VAL_MEM || temp_readonly(ts));
}
/* save globals to their canonical location and assume they can be
for (i = 0, n = s->nb_globals; i < n; i++) {
TCGTemp *ts = &s->temps[i];
tcg_debug_assert(ts->val_type != TEMP_VAL_REG
- || ts->fixed_reg
+ || ts->kind == TEMP_FIXED
|| ts->mem_coherent);
}
}
for (i = s->nb_globals; i < s->nb_temps; i++) {
TCGTemp *ts = &s->temps[i];
- if (ts->temp_local) {
+
+ switch (ts->kind) {
+ case TEMP_LOCAL:
temp_save(s, ts, allocated_regs);
- } else {
+ break;
+ case TEMP_NORMAL:
/* The liveness analysis already ensures that temps are dead.
Keep an tcg_debug_assert for safety. */
tcg_debug_assert(ts->val_type == TEMP_VAL_DEAD);
+ break;
+ case TEMP_CONST:
+ /* Similarly, we should have freed any allocated register. */
+ tcg_debug_assert(ts->val_type == TEMP_VAL_CONST);
+ break;
+ default:
+ g_assert_not_reached();
}
}
* The liveness analysis already ensures that temps are dead.
* Keep tcg_debug_asserts for safety.
*/
- if (ts->temp_local) {
+ switch (ts->kind) {
+ case TEMP_LOCAL:
tcg_debug_assert(ts->val_type != TEMP_VAL_REG || ts->mem_coherent);
- } else {
+ break;
+ case TEMP_NORMAL:
tcg_debug_assert(ts->val_type == TEMP_VAL_DEAD);
+ break;
+ case TEMP_CONST:
+ break;
+ default:
+ g_assert_not_reached();
}
}
}
/*
- * Specialized code generation for INDEX_op_movi_*.
+ * Specialized code generation for INDEX_op_mov_* with a constant.
*/
static void tcg_reg_alloc_do_movi(TCGContext *s, TCGTemp *ots,
tcg_target_ulong val, TCGLifeData arg_life,
TCGRegSet preferred_regs)
{
/* ENV should not be modified. */
- tcg_debug_assert(!ots->fixed_reg);
+ tcg_debug_assert(!temp_readonly(ots));
/* The movi is not explicitly generated here. */
if (ots->val_type == TEMP_VAL_REG) {
}
}
-static void tcg_reg_alloc_movi(TCGContext *s, const TCGOp *op)
-{
- TCGTemp *ots = arg_temp(op->args[0]);
- tcg_target_ulong val = op->args[1];
-
- tcg_reg_alloc_do_movi(s, ots, val, op->life, op->output_pref[0]);
-}
-
/*
* Specialized code generation for INDEX_op_mov_*.
*/
ts = arg_temp(op->args[1]);
/* ENV should not be modified. */
- tcg_debug_assert(!ots->fixed_reg);
+ tcg_debug_assert(!temp_readonly(ots));
/* Note that otype != itype for no-op truncation. */
otype = ots->type;
}
temp_dead(s, ots);
} else {
- if (IS_DEAD_ARG(1) && !ts->fixed_reg) {
+ if (IS_DEAD_ARG(1) && ts->kind != TEMP_FIXED) {
/* the mov can be suppressed */
if (ots->val_type == TEMP_VAL_REG) {
s->reg_to_temp[ots->reg] = NULL;
* Store the source register into the destination slot
* and leave the destination temp as TEMP_VAL_MEM.
*/
- assert(!ots->fixed_reg);
+ assert(!temp_readonly(ots));
if (!ts->mem_allocated) {
temp_allocate_frame(s, ots);
}
its = arg_temp(op->args[1]);
/* ENV should not be modified. */
- tcg_debug_assert(!ots->fixed_reg);
+ tcg_debug_assert(!temp_readonly(ots));
itype = its->type;
vece = TCGOP_VECE(op);
ts = arg_temp(arg);
if (ts->val_type == TEMP_VAL_CONST
- && tcg_target_const_match(ts->val, ts->type, arg_ct)) {
+ && tcg_target_const_match(ts->val, ts->type, arg_ct->ct)) {
/* constant is OK for instruction */
const_args[i] = 1;
new_args[i] = ts->val;
i_preferred_regs = o_preferred_regs = 0;
if (arg_ct->ialias) {
o_preferred_regs = op->output_pref[arg_ct->alias_index];
- if (ts->fixed_reg) {
- /* if fixed register, we must allocate a new register
- if the alias is not the same register */
- if (arg != op->args[arg_ct->alias_index]) {
- goto allocate_in_reg;
- }
- } else {
- /* if the input is aliased to an output and if it is
- not dead after the instruction, we must allocate
- a new register and move it */
- if (!IS_DEAD_ARG(i)) {
- goto allocate_in_reg;
- }
- /* check if the current register has already been allocated
- for another input aliased to an output */
- if (ts->val_type == TEMP_VAL_REG) {
- int k2, i2;
- reg = ts->reg;
- for (k2 = 0 ; k2 < k ; k2++) {
- i2 = def->args_ct[nb_oargs + k2].sort_index;
- if (def->args_ct[i2].ialias && reg == new_args[i2]) {
- goto allocate_in_reg;
- }
+ /*
+ * If the input is readonly, then it cannot also be an
+ * output and aliased to itself. If the input is not
+ * dead after the instruction, we must allocate a new
+ * register and move it.
+ */
+ if (temp_readonly(ts) || !IS_DEAD_ARG(i)) {
+ goto allocate_in_reg;
+ }
+
+ /*
+ * Check if the current register has already been allocated
+ * for another input aliased to an output.
+ */
+ if (ts->val_type == TEMP_VAL_REG) {
+ reg = ts->reg;
+ for (int k2 = 0; k2 < k; k2++) {
+ int i2 = def->args_ct[nb_oargs + k2].sort_index;
+ if (def->args_ct[i2].ialias && reg == new_args[i2]) {
+ goto allocate_in_reg;
}
}
- i_preferred_regs = o_preferred_regs;
}
+ i_preferred_regs = o_preferred_regs;
}
temp_load(s, ts, arg_ct->regs, i_allocated_regs, i_preferred_regs);
reg = ts->reg;
- if (tcg_regset_test_reg(arg_ct->regs, reg)) {
- /* nothing to do : the constraint is satisfied */
- } else {
- allocate_in_reg:
- /* allocate a new register matching the constraint
- and move the temporary register into it */
+ if (!tcg_regset_test_reg(arg_ct->regs, reg)) {
+ allocate_in_reg:
+ /*
+ * Allocate a new register matching the constraint
+ * and move the temporary register into it.
+ */
temp_load(s, ts, tcg_target_available_regs[ts->type],
i_allocated_regs, 0);
reg = tcg_reg_alloc(s, arg_ct->regs, i_allocated_regs,
ts = arg_temp(arg);
/* ENV should not be modified. */
- tcg_debug_assert(!ts->fixed_reg);
+ tcg_debug_assert(!temp_readonly(ts));
if (arg_ct->oalias && !const_args[arg_ct->alias_index]) {
reg = new_args[arg_ct->alias_index];
ts = arg_temp(op->args[i]);
/* ENV should not be modified. */
- tcg_debug_assert(!ts->fixed_reg);
+ tcg_debug_assert(!temp_readonly(ts));
if (NEED_SYNC_ARG(i)) {
temp_sync(s, ts, o_allocated_regs, 0, IS_DEAD_ARG(i));
}
}
+static bool tcg_reg_alloc_dup2(TCGContext *s, const TCGOp *op)
+{
+ const TCGLifeData arg_life = op->life;
+ TCGTemp *ots, *itsl, *itsh;
+ TCGType vtype = TCGOP_VECL(op) + TCG_TYPE_V64;
+
+ /* This opcode is only valid for 32-bit hosts, for 64-bit elements. */
+ tcg_debug_assert(TCG_TARGET_REG_BITS == 32);
+ tcg_debug_assert(TCGOP_VECE(op) == MO_64);
+
+ ots = arg_temp(op->args[0]);
+ itsl = arg_temp(op->args[1]);
+ itsh = arg_temp(op->args[2]);
+
+ /* ENV should not be modified. */
+ tcg_debug_assert(!temp_readonly(ots));
+
+ /* Allocate the output register now. */
+ if (ots->val_type != TEMP_VAL_REG) {
+ TCGRegSet allocated_regs = s->reserved_regs;
+ TCGRegSet dup_out_regs =
+ tcg_op_defs[INDEX_op_dup_vec].args_ct[0].regs;
+
+ /* Make sure to not spill the input registers. */
+ if (!IS_DEAD_ARG(1) && itsl->val_type == TEMP_VAL_REG) {
+ tcg_regset_set_reg(allocated_regs, itsl->reg);
+ }
+ if (!IS_DEAD_ARG(2) && itsh->val_type == TEMP_VAL_REG) {
+ tcg_regset_set_reg(allocated_regs, itsh->reg);
+ }
+
+ ots->reg = tcg_reg_alloc(s, dup_out_regs, allocated_regs,
+ op->output_pref[0], ots->indirect_base);
+ ots->val_type = TEMP_VAL_REG;
+ ots->mem_coherent = 0;
+ s->reg_to_temp[ots->reg] = ots;
+ }
+
+ /* Promote dup2 of immediates to dupi_vec. */
+ if (itsl->val_type == TEMP_VAL_CONST && itsh->val_type == TEMP_VAL_CONST) {
+ uint64_t val = deposit64(itsl->val, 32, 32, itsh->val);
+ MemOp vece = MO_64;
+
+ if (val == dup_const(MO_8, val)) {
+ vece = MO_8;
+ } else if (val == dup_const(MO_16, val)) {
+ vece = MO_16;
+ } else if (val == dup_const(MO_32, val)) {
+ vece = MO_32;
+ }
+
+ tcg_out_dupi_vec(s, vtype, vece, ots->reg, val);
+ goto done;
+ }
+
+ /* If the two inputs form one 64-bit value, try dupm_vec. */
+ if (itsl + 1 == itsh && itsl->base_type == TCG_TYPE_I64) {
+ if (!itsl->mem_coherent) {
+ temp_sync(s, itsl, s->reserved_regs, 0, 0);
+ }
+ if (!itsh->mem_coherent) {
+ temp_sync(s, itsh, s->reserved_regs, 0, 0);
+ }
+#ifdef HOST_WORDS_BIGENDIAN
+ TCGTemp *its = itsh;
+#else
+ TCGTemp *its = itsl;
+#endif
+ if (tcg_out_dupm_vec(s, vtype, MO_64, ots->reg,
+ its->mem_base->reg, its->mem_offset)) {
+ goto done;
+ }
+ }
+
+ /* Fall back to generic expansion. */
+ return false;
+
+ done:
+ if (IS_DEAD_ARG(1)) {
+ temp_dead(s, itsl);
+ }
+ if (IS_DEAD_ARG(2)) {
+ temp_dead(s, itsh);
+ }
+ if (NEED_SYNC_ARG(0)) {
+ temp_sync(s, ots, s->reserved_regs, 0, IS_DEAD_ARG(0));
+ } else if (IS_DEAD_ARG(0)) {
+ temp_dead(s, ots);
+ }
+ return true;
+}
+
#ifdef TCG_TARGET_STACK_GROWSUP
#define STACK_DIR(x) (-(x))
#else
ts = arg_temp(arg);
/* ENV should not be modified. */
- tcg_debug_assert(!ts->fixed_reg);
+ tcg_debug_assert(!temp_readonly(ts));
reg = tcg_target_call_oarg_regs[i];
tcg_debug_assert(s->reg_to_temp[reg] == NULL);
tcg_reg_alloc_start(s);
- s->code_buf = tb->tc.ptr;
- s->code_ptr = tb->tc.ptr;
+ /*
+ * Reset the buffer pointers when restarting after overflow.
+ * TODO: Move this into translate-all.c with the rest of the
+ * buffer management. Having only this done here is confusing.
+ */
+ s->code_buf = tcg_splitwx_to_rw(tb->tc.ptr);
+ s->code_ptr = s->code_buf;
#ifdef TCG_TARGET_NEED_LDST_LABELS
QSIMPLEQ_INIT(&s->ldst_labels);
case INDEX_op_mov_vec:
tcg_reg_alloc_mov(s, op);
break;
- case INDEX_op_movi_i32:
- case INDEX_op_movi_i64:
- case INDEX_op_dupi_vec:
- tcg_reg_alloc_movi(s, op);
- break;
case INDEX_op_dup_vec:
tcg_reg_alloc_dup(s, op);
break;
break;
case INDEX_op_set_label:
tcg_reg_alloc_bb_end(s, s->reserved_regs);
- tcg_out_label(s, arg_label(op->args[0]), s->code_ptr);
+ tcg_out_label(s, arg_label(op->args[0]));
break;
case INDEX_op_call:
tcg_reg_alloc_call(s, op);
break;
+ case INDEX_op_dup2_vec:
+ if (tcg_reg_alloc_dup2(s, op)) {
+ break;
+ }
+ /* fall through */
default:
/* Sanity check that we've not introduced any unhandled opcodes. */
tcg_debug_assert(tcg_op_supported(opc));
#ifndef CONFIG_TCG_INTERPRETER
/* flush instruction cache */
- flush_idcache_range((uintptr_t)s->code_buf, (uintptr_t)s->code_buf,
+ flush_idcache_range((uintptr_t)tcg_splitwx_to_rx(s->code_buf),
+ (uintptr_t)s->code_buf,
tcg_ptr_byte_diff(s->code_ptr, s->code_buf));
#endif
}
}
-static void tcg_register_jit_int(void *buf_ptr, size_t buf_size,
+static void tcg_register_jit_int(const void *buf_ptr, size_t buf_size,
const void *debug_frame,
size_t debug_frame_size)
{
/* No support for the feature. Provide the entry point expected by exec.c,
and implement the internal function we declared earlier. */
-static void tcg_register_jit_int(void *buf, size_t size,
+static void tcg_register_jit_int(const void *buf, size_t size,
const void *debug_frame,
size_t debug_frame_size)
{
}
-void tcg_register_jit(void *buf, size_t buf_size)
+void tcg_register_jit(const void *buf, size_t buf_size)
{
}
#endif /* ELF_HOST_MACHINE */
projects / qemu.git / blobdiff