#endif
#include "qemu-common.h"
-#include "cache-utils.h"
-#include "host-utils.h"
-#include "qemu-timer.h"
+#include "qemu/cache-utils.h"
+#include "qemu/host-utils.h"
+#include "qemu/timer.h"
/* Note: the long term plan is to reduce the dependancies on the QEMU
CPU definitions. Currently they are used for qemu_ld/st
#include "tcg-op.h"
-#if TCG_TARGET_REG_BITS == 64
-# define ELF_CLASS ELFCLASS64
-#else
+#if UINTPTR_MAX == UINT32_MAX
# define ELF_CLASS ELFCLASS32
+#else
+# define ELF_CLASS ELFCLASS64
#endif
#ifdef HOST_WORDS_BIGENDIAN
# define ELF_DATA ELFDATA2MSB
static void tcg_target_init(TCGContext *s);
static void tcg_target_qemu_prologue(TCGContext *s);
static void patch_reloc(uint8_t *code_ptr, int type,
- tcg_target_long value, tcg_target_long addend);
+ intptr_t value, intptr_t addend);
+
+/* The CIE and FDE header definitions will be common to all hosts. */
+typedef struct {
+ uint32_t len __attribute__((aligned((sizeof(void *)))));
+ uint32_t id;
+ uint8_t version;
+ char augmentation[1];
+ uint8_t code_align;
+ uint8_t data_align;
+ uint8_t return_column;
+} DebugFrameCIE;
+
+typedef struct QEMU_PACKED {
+ uint32_t len __attribute__((aligned((sizeof(void *)))));
+ uint32_t cie_offset;
+ uintptr_t func_start;
+ uintptr_t func_len;
+} DebugFrameFDEHeader;
static void tcg_register_jit_int(void *buf, size_t size,
void *debug_frame, size_t debug_frame_size)
/* Forward declarations for functions declared and used in tcg-target.c. */
static int target_parse_constraint(TCGArgConstraint *ct, const char **pct_str);
static void tcg_out_ld(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg1,
- tcg_target_long arg2);
+ intptr_t arg2);
static void tcg_out_mov(TCGContext *s, TCGType type, TCGReg ret, TCGReg arg);
static void tcg_out_movi(TCGContext *s, TCGType type,
TCGReg ret, tcg_target_long arg);
static void tcg_out_op(TCGContext *s, TCGOpcode opc, const TCGArg *args,
const int *const_args);
static void tcg_out_st(TCGContext *s, TCGType type, TCGReg arg, TCGReg arg1,
- tcg_target_long arg2);
+ intptr_t arg2);
static int tcg_target_const_match(tcg_target_long val,
const TCGArgConstraint *arg_ct);
static TCGRegSet tcg_target_available_regs[2];
static TCGRegSet tcg_target_call_clobber_regs;
-/* XXX: move that inside the context */
-uint16_t *gen_opc_ptr;
-TCGArg *gen_opparam_ptr;
-
static inline void tcg_out8(TCGContext *s, uint8_t v)
{
*s->code_ptr++ = v;
static inline void tcg_out16(TCGContext *s, uint16_t v)
{
- *(uint16_t *)s->code_ptr = v;
- s->code_ptr += 2;
+ uint8_t *p = s->code_ptr;
+ *(uint16_t *)p = v;
+ s->code_ptr = p + 2;
}
static inline void tcg_out32(TCGContext *s, uint32_t v)
{
- *(uint32_t *)s->code_ptr = v;
- s->code_ptr += 4;
+ uint8_t *p = s->code_ptr;
+ *(uint32_t *)p = v;
+ s->code_ptr = p + 4;
+}
+
+static inline void tcg_out64(TCGContext *s, uint64_t v)
+{
+ uint8_t *p = s->code_ptr;
+ *(uint64_t *)p = v;
+ s->code_ptr = p + 8;
}
/* label relocation processing */
static void tcg_out_reloc(TCGContext *s, uint8_t *code_ptr, int type,
- int label_index, long addend)
+ int label_index, intptr_t addend)
{
TCGLabel *l;
TCGRelocation *r;
{
TCGLabel *l;
TCGRelocation *r;
- tcg_target_long value = (tcg_target_long)ptr;
+ intptr_t value = (intptr_t)ptr;
l = &s->labels[label_index];
- if (l->has_value)
+ if (l->has_value) {
tcg_abort();
+ }
r = l->u.first_reloc;
while (r != NULL) {
patch_reloc(r->ptr, r->type, value, r->addend);
void tcg_prologue_init(TCGContext *s)
{
/* init global prologue and epilogue */
- s->code_buf = code_gen_prologue;
+ s->code_buf = s->code_gen_prologue;
s->code_ptr = s->code_buf;
tcg_target_qemu_prologue(s);
- flush_icache_range((tcg_target_ulong)s->code_buf,
- (tcg_target_ulong)s->code_ptr);
+ flush_icache_range((uintptr_t)s->code_buf, (uintptr_t)s->code_ptr);
+
+#ifdef DEBUG_DISAS
+ if (qemu_loglevel_mask(CPU_LOG_TB_OUT_ASM)) {
+ size_t size = s->code_ptr - s->code_buf;
+ qemu_log("PROLOGUE: [size=%zu]\n", size);
+ log_disas(s->code_buf, size);
+ qemu_log("\n");
+ qemu_log_flush();
+ }
+#endif
}
-void tcg_set_frame(TCGContext *s, int reg,
- tcg_target_long start, tcg_target_long size)
+void tcg_set_frame(TCGContext *s, int reg, intptr_t start, intptr_t size)
{
s->frame_start = start;
s->frame_end = start + size;
s->goto_tb_issue_mask = 0;
#endif
- gen_opc_ptr = gen_opc_buf;
- gen_opparam_ptr = gen_opparam_buf;
+ s->gen_opc_ptr = s->gen_opc_buf;
+ s->gen_opparam_ptr = s->gen_opparam_buf;
+
+#if defined(CONFIG_QEMU_LDST_OPTIMIZATION) && defined(CONFIG_SOFTMMU)
+ /* Initialize qemu_ld/st labels to assist code generation at the end of TB
+ for TLB miss cases at the end of TB */
+ s->qemu_ldst_labels = tcg_malloc(sizeof(TCGLabelQemuLdst) *
+ TCG_MAX_QEMU_LDST);
+ s->nb_qemu_ldst_labels = 0;
+#endif
}
static inline void tcg_temp_alloc(TCGContext *s, int n)
}
static inline int tcg_global_mem_new_internal(TCGType type, int reg,
- tcg_target_long offset,
+ intptr_t offset,
const char *name)
{
TCGContext *s = &tcg_ctx;
return idx;
}
-TCGv_i32 tcg_global_mem_new_i32(int reg, tcg_target_long offset,
- const char *name)
+TCGv_i32 tcg_global_mem_new_i32(int reg, intptr_t offset, const char *name)
{
- int idx;
-
- idx = tcg_global_mem_new_internal(TCG_TYPE_I32, reg, offset, name);
+ int idx = tcg_global_mem_new_internal(TCG_TYPE_I32, reg, offset, name);
return MAKE_TCGV_I32(idx);
}
-TCGv_i64 tcg_global_mem_new_i64(int reg, tcg_target_long offset,
- const char *name)
+TCGv_i64 tcg_global_mem_new_i64(int reg, intptr_t offset, const char *name)
{
- int idx;
-
- idx = tcg_global_mem_new_internal(TCG_TYPE_I64, reg, offset, name);
+ int idx = tcg_global_mem_new_internal(TCG_TYPE_I64, reg, offset, name);
return MAKE_TCGV_I64(idx);
}
s->helpers = realloc(s->helpers, n * sizeof(TCGHelperInfo));
s->allocated_helpers = n;
}
- s->helpers[s->nb_helpers].func = (tcg_target_ulong)func;
+ s->helpers[s->nb_helpers].func = (uintptr_t)func;
s->helpers[s->nb_helpers].name = name;
s->nb_helpers++;
}
}
#endif /* TCG_TARGET_EXTEND_ARGS */
- *gen_opc_ptr++ = INDEX_op_call;
- nparam = gen_opparam_ptr++;
+ *s->gen_opc_ptr++ = INDEX_op_call;
+ nparam = s->gen_opparam_ptr++;
if (ret != TCG_CALL_DUMMY_ARG) {
#if TCG_TARGET_REG_BITS < 64
if (sizemask & 1) {
#ifdef TCG_TARGET_WORDS_BIGENDIAN
- *gen_opparam_ptr++ = ret + 1;
- *gen_opparam_ptr++ = ret;
+ *s->gen_opparam_ptr++ = ret + 1;
+ *s->gen_opparam_ptr++ = ret;
#else
- *gen_opparam_ptr++ = ret;
- *gen_opparam_ptr++ = ret + 1;
+ *s->gen_opparam_ptr++ = ret;
+ *s->gen_opparam_ptr++ = ret + 1;
#endif
nb_rets = 2;
} else
#endif
{
- *gen_opparam_ptr++ = ret;
+ *s->gen_opparam_ptr++ = ret;
nb_rets = 1;
}
} else {
#ifdef TCG_TARGET_CALL_ALIGN_ARGS
/* some targets want aligned 64 bit args */
if (real_args & 1) {
- *gen_opparam_ptr++ = TCG_CALL_DUMMY_ARG;
+ *s->gen_opparam_ptr++ = TCG_CALL_DUMMY_ARG;
real_args++;
}
#endif
have to get more complicated to differentiate between
stack arguments and register arguments. */
#if defined(TCG_TARGET_WORDS_BIGENDIAN) != defined(TCG_TARGET_STACK_GROWSUP)
- *gen_opparam_ptr++ = args[i] + 1;
- *gen_opparam_ptr++ = args[i];
+ *s->gen_opparam_ptr++ = args[i] + 1;
+ *s->gen_opparam_ptr++ = args[i];
#else
- *gen_opparam_ptr++ = args[i];
- *gen_opparam_ptr++ = args[i] + 1;
+ *s->gen_opparam_ptr++ = args[i];
+ *s->gen_opparam_ptr++ = args[i] + 1;
#endif
real_args += 2;
continue;
}
#endif /* TCG_TARGET_REG_BITS < 64 */
- *gen_opparam_ptr++ = args[i];
+ *s->gen_opparam_ptr++ = args[i];
real_args++;
}
- *gen_opparam_ptr++ = GET_TCGV_PTR(func);
+ *s->gen_opparam_ptr++ = GET_TCGV_PTR(func);
- *gen_opparam_ptr++ = flags;
+ *s->gen_opparam_ptr++ = flags;
*nparam = (nb_rets << 16) | (real_args + 1);
/* total parameters, needed to go backward in the instruction stream */
- *gen_opparam_ptr++ = 1 + nb_rets + real_args + 3;
+ *s->gen_opparam_ptr++ = 1 + nb_rets + real_args + 3;
#if defined(TCG_TARGET_EXTEND_ARGS) && TCG_TARGET_REG_BITS == 64
for (i = 0; i < nargs; ++i) {
assert(idx >= 0 && idx < s->nb_temps);
ts = &s->temps[idx];
- assert(ts);
if (idx < s->nb_globals) {
pstrcpy(buf, buf_size, ts->name);
} else {
}
/* find helper definition (Note: A hash table would be better) */
-static TCGHelperInfo *tcg_find_helper(TCGContext *s, tcg_target_ulong val)
+static TCGHelperInfo *tcg_find_helper(TCGContext *s, uintptr_t val)
{
int m, m_min, m_max;
TCGHelperInfo *th;
- tcg_target_ulong v;
+ uintptr_t v;
if (unlikely(!s->helpers_sorted)) {
qsort(s->helpers, s->nb_helpers, sizeof(TCGHelperInfo),
char buf[128];
first_insn = 1;
- opc_ptr = gen_opc_buf;
- args = gen_opparam_buf;
- while (opc_ptr < gen_opc_ptr) {
+ opc_ptr = s->gen_opc_buf;
+ args = s->gen_opparam_buf;
+ while (opc_ptr < s->gen_opc_ptr) {
c = *opc_ptr++;
def = &tcg_op_defs[c];
if (c == INDEX_op_debug_insn_start) {
i = 0;
for (op = 0; op < ARRAY_SIZE(tcg_op_defs); op++) {
const TCGOpDef *def = &tcg_op_defs[op];
- if (op < INDEX_op_call
- || op == INDEX_op_debug_insn_start
- || (def->flags & TCG_OPF_NOT_PRESENT)) {
+ if (def->flags & TCG_OPF_NOT_PRESENT) {
/* Wrong entry in op definitions? */
if (def->used) {
fprintf(stderr, "Invalid op definition for %s\n", def->name);
static void tcg_liveness_analysis(TCGContext *s)
{
int i, op_index, nb_args, nb_iargs, nb_oargs, arg, nb_ops;
- TCGOpcode op;
+ TCGOpcode op, op_new, op_new2;
TCGArg *args;
const TCGOpDef *def;
uint8_t *dead_temps, *mem_temps;
uint16_t dead_args;
uint8_t sync_args;
+ bool have_op_new2;
- gen_opc_ptr++; /* skip end */
+ s->gen_opc_ptr++; /* skip end */
- nb_ops = gen_opc_ptr - gen_opc_buf;
+ nb_ops = s->gen_opc_ptr - s->gen_opc_buf;
s->op_dead_args = tcg_malloc(nb_ops * sizeof(uint16_t));
s->op_sync_args = tcg_malloc(nb_ops * sizeof(uint8_t));
mem_temps = tcg_malloc(s->nb_temps);
tcg_la_func_end(s, dead_temps, mem_temps);
- args = gen_opparam_ptr;
+ args = s->gen_opparam_ptr;
op_index = nb_ops - 1;
while (op_index >= 0) {
- op = gen_opc_buf[op_index];
+ op = s->gen_opc_buf[op_index];
def = &tcg_op_defs[op];
switch(op) {
case INDEX_op_call:
/* pure functions can be removed if their result is not
used */
- if (call_flags & TCG_CALL_PURE) {
+ if (call_flags & TCG_CALL_NO_SIDE_EFFECTS) {
for(i = 0; i < nb_oargs; i++) {
arg = args[i];
if (!dead_temps[arg] || mem_temps[arg]) {
goto do_not_remove_call;
}
}
- tcg_set_nop(s, gen_opc_buf + op_index,
+ tcg_set_nop(s, s->gen_opc_buf + op_index,
args - 1, nb_args);
} else {
do_not_remove_call:
dead_temps[arg] = 1;
mem_temps[arg] = 0;
}
-
- if (!(call_flags & TCG_CALL_CONST)) {
+
+ if (!(call_flags & TCG_CALL_NO_READ_GLOBALS)) {
+ /* globals should be synced to memory */
+ memset(mem_temps, 1, s->nb_globals);
+ }
+ if (!(call_flags & (TCG_CALL_NO_WRITE_GLOBALS |
+ TCG_CALL_NO_READ_GLOBALS))) {
/* globals should go back to memory */
memset(dead_temps, 1, s->nb_globals);
- memset(mem_temps, 1, s->nb_globals);
}
/* input args are live */
break;
case INDEX_op_add2_i32:
+ op_new = INDEX_op_add_i32;
+ goto do_addsub2;
case INDEX_op_sub2_i32:
+ op_new = INDEX_op_sub_i32;
+ goto do_addsub2;
+ case INDEX_op_add2_i64:
+ op_new = INDEX_op_add_i64;
+ goto do_addsub2;
+ case INDEX_op_sub2_i64:
+ op_new = INDEX_op_sub_i64;
+ do_addsub2:
args -= 6;
nb_iargs = 4;
nb_oargs = 2;
the low part. The result can be optimized to a simple
add or sub. This happens often for x86_64 guest when the
cpu mode is set to 32 bit. */
- if (dead_temps[args[1]]) {
- if (dead_temps[args[0]]) {
+ if (dead_temps[args[1]] && !mem_temps[args[1]]) {
+ if (dead_temps[args[0]] && !mem_temps[args[0]]) {
goto do_remove;
}
/* Create the single operation plus nop. */
- if (op == INDEX_op_add2_i32) {
- op = INDEX_op_add_i32;
- } else {
- op = INDEX_op_sub_i32;
- }
- gen_opc_buf[op_index] = op;
+ s->gen_opc_buf[op_index] = op = op_new;
args[1] = args[2];
args[2] = args[4];
- assert(gen_opc_buf[op_index + 1] == INDEX_op_nop);
- tcg_set_nop(s, gen_opc_buf + op_index + 1, args + 3, 3);
+ assert(s->gen_opc_buf[op_index + 1] == INDEX_op_nop);
+ tcg_set_nop(s, s->gen_opc_buf + op_index + 1, args + 3, 3);
/* Fall through and mark the single-word operation live. */
nb_iargs = 2;
nb_oargs = 1;
goto do_not_remove;
case INDEX_op_mulu2_i32:
+ op_new = INDEX_op_mul_i32;
+ op_new2 = INDEX_op_muluh_i32;
+ have_op_new2 = TCG_TARGET_HAS_muluh_i32;
+ goto do_mul2;
+ case INDEX_op_muls2_i32:
+ op_new = INDEX_op_mul_i32;
+ op_new2 = INDEX_op_mulsh_i32;
+ have_op_new2 = TCG_TARGET_HAS_mulsh_i32;
+ goto do_mul2;
+ case INDEX_op_mulu2_i64:
+ op_new = INDEX_op_mul_i64;
+ op_new2 = INDEX_op_muluh_i64;
+ have_op_new2 = TCG_TARGET_HAS_muluh_i64;
+ goto do_mul2;
+ case INDEX_op_muls2_i64:
+ op_new = INDEX_op_mul_i64;
+ op_new2 = INDEX_op_mulsh_i64;
+ have_op_new2 = TCG_TARGET_HAS_mulsh_i64;
+ goto do_mul2;
+ do_mul2:
args -= 4;
nb_iargs = 2;
nb_oargs = 2;
- /* Likewise, test for the high part of the operation dead. */
- if (dead_temps[args[1]]) {
- if (dead_temps[args[0]]) {
+ if (dead_temps[args[1]] && !mem_temps[args[1]]) {
+ if (dead_temps[args[0]] && !mem_temps[args[0]]) {
+ /* Both parts of the operation are dead. */
goto do_remove;
}
- gen_opc_buf[op_index] = op = INDEX_op_mul_i32;
+ /* The high part of the operation is dead; generate the low. */
+ s->gen_opc_buf[op_index] = op = op_new;
args[1] = args[2];
args[2] = args[3];
- assert(gen_opc_buf[op_index + 1] == INDEX_op_nop);
- tcg_set_nop(s, gen_opc_buf + op_index + 1, args + 3, 1);
- /* Fall through and mark the single-word operation live. */
- nb_oargs = 1;
+ } else if (have_op_new2 && dead_temps[args[0]]
+ && !mem_temps[args[0]]) {
+ /* The low part of the operation is dead; generate the high. */
+ s->gen_opc_buf[op_index] = op = op_new2;
+ args[0] = args[1];
+ args[1] = args[2];
+ args[2] = args[3];
+ } else {
+ goto do_not_remove;
}
+ assert(s->gen_opc_buf[op_index + 1] == INDEX_op_nop);
+ tcg_set_nop(s, s->gen_opc_buf + op_index + 1, args + 3, 1);
+ /* Mark the single-word operation live. */
+ nb_oargs = 1;
goto do_not_remove;
default:
}
}
do_remove:
- tcg_set_nop(s, gen_opc_buf + op_index, args, def->nb_args);
+ tcg_set_nop(s, s->gen_opc_buf + op_index, args, def->nb_args);
#ifdef CONFIG_PROFILER
s->del_op_count++;
#endif
op_index--;
}
- if (args != gen_opparam_buf)
+ if (args != s->gen_opparam_buf) {
tcg_abort();
+ }
}
#else
/* dummy liveness analysis */
static void tcg_liveness_analysis(TCGContext *s)
{
int nb_ops;
- nb_ops = gen_opc_ptr - gen_opc_buf;
+ nb_ops = s->gen_opc_ptr - s->gen_opc_buf;
s->op_dead_args = tcg_malloc(nb_ops * sizeof(uint16_t));
memset(s->op_dead_args, 0, nb_ops * sizeof(uint16_t));
ts->mem_offset = s->current_frame_offset;
ts->mem_reg = s->frame_reg;
ts->mem_allocated = 1;
- s->current_frame_offset += (tcg_target_long)sizeof(tcg_target_long);
+ s->current_frame_offset += sizeof(tcg_target_long);
}
/* sync register 'reg' by saving it to the corresponding temporary */
if (ts->val_type == TEMP_VAL_REG) {
s->reg_to_temp[ts->reg] = -1;
}
- if (temp < s->nb_globals || (ts->temp_local && ts->mem_allocated)) {
+ if (temp < s->nb_globals || ts->temp_local) {
ts->val_type = TEMP_VAL_MEM;
} else {
ts->val_type = TEMP_VAL_DEAD;
int nb_iargs, nb_oargs, flags, nb_regs, i, reg, nb_params;
TCGArg arg, func_arg;
TCGTemp *ts;
- tcg_target_long stack_offset, call_stack_size, func_addr;
+ intptr_t stack_offset;
+ size_t call_stack_size;
+ uintptr_t func_addr;
int const_func_arg, allocate_args;
TCGRegSet allocated_regs;
const TCGArgConstraint *arg_ct;
tcg_reg_free(s, reg);
}
}
-
- /* store globals and free associated registers (we assume the call
- can modify any global. */
- if (!(flags & TCG_CALL_CONST)) {
+
+ /* Save globals if they might be written by the helper, sync them if
+ they might be read. */
+ if (flags & TCG_CALL_NO_READ_GLOBALS) {
+ /* Nothing to do */
+ } else if (flags & TCG_CALL_NO_WRITE_GLOBALS) {
+ sync_globals(s, allocated_regs);
+ } else {
save_globals(s, allocated_regs);
}
#endif
#ifdef USE_TCG_OPTIMIZATIONS
- gen_opparam_ptr =
- tcg_optimize(s, gen_opc_ptr, gen_opparam_buf, tcg_op_defs);
+ s->gen_opparam_ptr =
+ tcg_optimize(s, s->gen_opc_ptr, s->gen_opparam_buf, tcg_op_defs);
#endif
#ifdef CONFIG_PROFILER
s->code_buf = gen_code_buf;
s->code_ptr = gen_code_buf;
- args = gen_opparam_buf;
+ args = s->gen_opparam_buf;
op_index = 0;
for(;;) {
- opc = gen_opc_buf[op_index];
+ opc = s->gen_opc_buf[op_index];
#ifdef CONFIG_PROFILER
tcg_table_op_count[opc]++;
#endif
#endif
}
the_end:
+#if defined(CONFIG_QEMU_LDST_OPTIMIZATION) && defined(CONFIG_SOFTMMU)
+ /* Generate TB finalization at the end of block */
+ tcg_out_tb_finalize(s);
+#endif
return -1;
}
#ifdef CONFIG_PROFILER
{
int n;
- n = (gen_opc_ptr - gen_opc_buf);
+ n = (s->gen_opc_ptr - s->gen_opc_buf);
s->op_count += n;
if (n > s->op_count_max)
s->op_count_max = n;
tcg_gen_code_common(s, gen_code_buf, -1);
/* flush instruction cache */
- flush_icache_range((tcg_target_ulong)gen_code_buf,
- (tcg_target_ulong)s->code_ptr);
+ flush_icache_range((uintptr_t)gen_code_buf, (uintptr_t)s->code_ptr);
return s->code_ptr - gen_code_buf;
}
img->sym[1].st_size = buf_size;
img->di.cu_low_pc = buf;
- img->di.cu_high_pc = buf_size;
+ img->di.cu_high_pc = buf + buf_size;
img->di.fn_low_pc = buf;
- img->di.fn_high_pc = buf_size;
+ img->di.fn_high_pc = buf + buf_size;
#ifdef DEBUG_JIT
/* Enable this block to be able to debug the ELF image file creation.
projects / qemu.git / blobdiff