X-Git-Url: https://repo.jachan.dev/qemu.git/blobdiff_plain/5a93d5c2abc719bd44f6c9fbeed88d3cae712606..7f25c469c74ce7f1c2f7c9b4e81ced53ac0a1887:/tcg/tcg.c diff --git a/tcg/tcg.c b/tcg/tcg.c index fd7fb6b85e..dae4b7cb00 100644 --- a/tcg/tcg.c +++ b/tcg/tcg.c @@ -41,7 +41,7 @@ #include "qemu/host-utils.h" #include "qemu/timer.h" -/* Note: the long term plan is to reduce the dependancies on the QEMU +/* Note: the long term plan is to reduce the dependencies on the QEMU CPU definitions. Currently they are used for qemu_ld/st instructions */ #define NO_CPU_IO_DEFS @@ -65,7 +65,7 @@ /* Forward declarations for functions declared in tcg-target.c and used here. */ static void tcg_target_init(TCGContext *s); static void tcg_target_qemu_prologue(TCGContext *s); -static void patch_reloc(uint8_t *code_ptr, int type, +static void patch_reloc(tcg_insn_unit *code_ptr, int type, intptr_t value, intptr_t addend); /* The CIE and FDE header definitions will be common to all hosts. */ @@ -86,8 +86,14 @@ typedef struct QEMU_PACKED { uintptr_t func_len; } DebugFrameFDEHeader; +typedef struct QEMU_PACKED { + DebugFrameCIE cie; + DebugFrameFDEHeader fde; +} DebugFrameHeader; + static void tcg_register_jit_int(void *buf, size_t size, - void *debug_frame, size_t debug_frame_size) + const void *debug_frame, + size_t debug_frame_size) __attribute__((unused)); /* Forward declarations for functions declared and used in tcg-target.c. */ @@ -101,8 +107,12 @@ static void tcg_out_op(TCGContext *s, TCGOpcode opc, const TCGArg *args, const int *const_args); static void tcg_out_st(TCGContext *s, TCGType type, TCGReg arg, TCGReg arg1, intptr_t arg2); -static int tcg_target_const_match(tcg_target_long val, +static void tcg_out_call(TCGContext *s, tcg_insn_unit *target); +static int tcg_target_const_match(tcg_target_long val, TCGType type, const TCGArgConstraint *arg_ct); +static void tcg_out_tb_init(TCGContext *s); +static void tcg_out_tb_finalize(TCGContext *s); + TCGOpDef tcg_op_defs[] = { #define DEF(s, oargs, iargs, cargs, flags) { #s, oargs, iargs, cargs, iargs + oargs + cargs, flags }, @@ -114,35 +124,87 @@ const size_t tcg_op_defs_max = ARRAY_SIZE(tcg_op_defs); static TCGRegSet tcg_target_available_regs[2]; static TCGRegSet tcg_target_call_clobber_regs; +#if TCG_TARGET_INSN_UNIT_SIZE == 1 static inline void tcg_out8(TCGContext *s, uint8_t v) { *s->code_ptr++ = v; } +static inline void tcg_patch8(tcg_insn_unit *p, uint8_t v) +{ + *p = v; +} +#endif + +#if TCG_TARGET_INSN_UNIT_SIZE <= 2 static inline void tcg_out16(TCGContext *s, uint16_t v) { - uint8_t *p = s->code_ptr; - *(uint16_t *)p = v; - s->code_ptr = p + 2; + if (TCG_TARGET_INSN_UNIT_SIZE == 2) { + *s->code_ptr++ = v; + } else { + tcg_insn_unit *p = s->code_ptr; + memcpy(p, &v, sizeof(v)); + s->code_ptr = p + (2 / TCG_TARGET_INSN_UNIT_SIZE); + } } +static inline void tcg_patch16(tcg_insn_unit *p, uint16_t v) +{ + if (TCG_TARGET_INSN_UNIT_SIZE == 2) { + *p = v; + } else { + memcpy(p, &v, sizeof(v)); + } +} +#endif + +#if TCG_TARGET_INSN_UNIT_SIZE <= 4 static inline void tcg_out32(TCGContext *s, uint32_t v) { - uint8_t *p = s->code_ptr; - *(uint32_t *)p = v; - s->code_ptr = p + 4; + if (TCG_TARGET_INSN_UNIT_SIZE == 4) { + *s->code_ptr++ = v; + } else { + tcg_insn_unit *p = s->code_ptr; + memcpy(p, &v, sizeof(v)); + s->code_ptr = p + (4 / TCG_TARGET_INSN_UNIT_SIZE); + } } +static inline void tcg_patch32(tcg_insn_unit *p, uint32_t v) +{ + if (TCG_TARGET_INSN_UNIT_SIZE == 4) { + *p = v; + } else { + memcpy(p, &v, sizeof(v)); + } +} +#endif + +#if TCG_TARGET_INSN_UNIT_SIZE <= 8 static inline void tcg_out64(TCGContext *s, uint64_t v) { - uint8_t *p = s->code_ptr; - *(uint64_t *)p = v; - s->code_ptr = p + 8; + if (TCG_TARGET_INSN_UNIT_SIZE == 8) { + *s->code_ptr++ = v; + } else { + tcg_insn_unit *p = s->code_ptr; + memcpy(p, &v, sizeof(v)); + s->code_ptr = p + (8 / TCG_TARGET_INSN_UNIT_SIZE); + } } +static inline void tcg_patch64(tcg_insn_unit *p, uint64_t v) +{ + if (TCG_TARGET_INSN_UNIT_SIZE == 8) { + *p = v; + } else { + memcpy(p, &v, sizeof(v)); + } +} +#endif + /* label relocation processing */ -static void tcg_out_reloc(TCGContext *s, uint8_t *code_ptr, int type, +static void tcg_out_reloc(TCGContext *s, tcg_insn_unit *code_ptr, int type, int label_index, intptr_t addend) { TCGLabel *l; @@ -165,23 +227,20 @@ static void tcg_out_reloc(TCGContext *s, uint8_t *code_ptr, int type, } } -static void tcg_out_label(TCGContext *s, int label_index, void *ptr) +static void tcg_out_label(TCGContext *s, int label_index, tcg_insn_unit *ptr) { - TCGLabel *l; - TCGRelocation *r; + TCGLabel *l = &s->labels[label_index]; intptr_t value = (intptr_t)ptr; + TCGRelocation *r; - l = &s->labels[label_index]; - if (l->has_value) { - tcg_abort(); - } - r = l->u.first_reloc; - while (r != NULL) { + assert(!l->has_value); + + for (r = l->u.first_reloc; r != NULL; r = r->next) { patch_reloc(r->ptr, r->type, value, r->addend); - r = r->next; } + l->has_value = 1; - l->u.value = value; + l->u.value_ptr = ptr; } int gen_new_label(void) @@ -254,12 +313,26 @@ void tcg_pool_reset(TCGContext *s) s->pool_current = NULL; } +typedef struct TCGHelperInfo { + void *func; + const char *name; + unsigned flags; + unsigned sizemask; +} TCGHelperInfo; + +#include "exec/helper-proto.h" + +static const TCGHelperInfo all_helpers[] = { +#include "exec/helper-tcg.h" +}; + void tcg_context_init(TCGContext *s) { - int op, total_args, n; + int op, total_args, n, i; TCGOpDef *def; TCGArgConstraint *args_ct; int *sorted_args; + GHashTable *helper_table; memset(s, 0, sizeof(*s)); s->nb_globals = 0; @@ -284,7 +357,16 @@ void tcg_context_init(TCGContext *s) sorted_args += n; args_ct += n; } - + + /* Register helpers. */ + /* Use g_direct_hash/equal for direct pointer comparisons on func. */ + s->helpers = helper_table = g_hash_table_new(NULL, NULL); + + for (i = 0; i < ARRAY_SIZE(all_helpers); ++i) { + g_hash_table_insert(helper_table, (gpointer)all_helpers[i].func, + (gpointer)&all_helpers[i]); + } + tcg_target_init(s); } @@ -298,7 +380,7 @@ void tcg_prologue_init(TCGContext *s) #ifdef DEBUG_DISAS if (qemu_loglevel_mask(CPU_LOG_TB_OUT_ASM)) { - size_t size = s->code_ptr - s->code_buf; + size_t size = tcg_current_code_size(s); qemu_log("PROLOGUE: [size=%zu]\n", size); log_disas(s->code_buf, size); qemu_log("\n"); @@ -316,11 +398,12 @@ void tcg_set_frame(TCGContext *s, int reg, intptr_t start, intptr_t size) void tcg_func_start(TCGContext *s) { - int i; tcg_pool_reset(s); s->nb_temps = s->nb_globals; - for(i = 0; i < (TCG_TYPE_COUNT * 2); i++) - s->first_free_temp[i] = -1; + + /* No temps have been previously allocated for size or locality. */ + memset(s->free_temps, 0, sizeof(s->free_temps)); + s->labels = tcg_malloc(sizeof(TCGLabel) * TCG_MAX_LABELS); s->nb_labels = 0; s->current_frame_offset = s->frame_start; @@ -332,13 +415,7 @@ void tcg_func_start(TCGContext *s) s->gen_opc_ptr = s->gen_opc_buf; s->gen_opparam_ptr = s->gen_opparam_buf; -#if defined(CONFIG_QEMU_LDST_OPTIMIZATION) && defined(CONFIG_SOFTMMU) - /* Initialize qemu_ld/st labels to assist code generation at the end of TB - for TLB miss cases at the end of TB */ - s->qemu_ldst_labels = tcg_malloc(sizeof(TCGLabelQemuLdst) * - TCG_MAX_QEMU_LDST); - s->nb_qemu_ldst_labels = 0; -#endif + s->be = tcg_malloc(sizeof(TCGBackendData)); } static inline void tcg_temp_alloc(TCGContext *s, int n) @@ -408,7 +485,7 @@ static inline int tcg_global_mem_new_internal(TCGType type, int reg, ts->fixed_reg = 0; ts->mem_allocated = 1; ts->mem_reg = reg; -#ifdef TCG_TARGET_WORDS_BIGENDIAN +#ifdef HOST_WORDS_BIGENDIAN ts->mem_offset = offset + 4; #else ts->mem_offset = offset; @@ -423,7 +500,7 @@ static inline int tcg_global_mem_new_internal(TCGType type, int reg, ts->fixed_reg = 0; ts->mem_allocated = 1; ts->mem_reg = reg; -#ifdef TCG_TARGET_WORDS_BIGENDIAN +#ifdef HOST_WORDS_BIGENDIAN ts->mem_offset = offset; #else ts->mem_offset = offset + 4; @@ -468,16 +545,15 @@ static inline int tcg_temp_new_internal(TCGType type, int temp_local) TCGTemp *ts; int idx, k; - k = type; - if (temp_local) - k += TCG_TYPE_COUNT; - idx = s->first_free_temp[k]; - if (idx != -1) { - /* There is already an available temp with the - right type */ + k = type + (temp_local ? TCG_TYPE_COUNT : 0); + idx = find_first_bit(s->free_temps[k].l, TCG_MAX_TEMPS); + if (idx < TCG_MAX_TEMPS) { + /* There is already an available temp with the right type. */ + clear_bit(idx, s->free_temps[k].l); + ts = &s->temps[idx]; - s->first_free_temp[k] = ts->next_free_temp; ts->temp_allocated = 1; + assert(ts->base_type == type); assert(ts->temp_local == temp_local); } else { idx = s->nb_temps; @@ -491,7 +567,7 @@ static inline int tcg_temp_new_internal(TCGType type, int temp_local) ts->temp_local = temp_local; ts->name = NULL; ts++; - ts->base_type = TCG_TYPE_I32; + ts->base_type = type; ts->type = TCG_TYPE_I32; ts->temp_allocated = 1; ts->temp_local = temp_local; @@ -533,7 +609,7 @@ TCGv_i64 tcg_temp_new_internal_i64(int temp_local) return MAKE_TCGV_I64(idx); } -static inline void tcg_temp_free_internal(int idx) +static void tcg_temp_free_internal(int idx) { TCGContext *s = &tcg_ctx; TCGTemp *ts; @@ -550,11 +626,9 @@ static inline void tcg_temp_free_internal(int idx) ts = &s->temps[idx]; assert(ts->temp_allocated != 0); ts->temp_allocated = 0; - k = ts->base_type; - if (ts->temp_local) - k += TCG_TYPE_COUNT; - ts->next_free_temp = s->first_free_temp[k]; - s->first_free_temp[k] = idx; + + k = ts->base_type + (ts->temp_local ? TCG_TYPE_COUNT : 0); + set_bit(idx, s->free_temps[k].l); } void tcg_temp_free_i32(TCGv_i32 arg) @@ -620,37 +694,51 @@ int tcg_check_temp_count(void) } #endif -void tcg_register_helper(void *func, const char *name) -{ - TCGContext *s = &tcg_ctx; - int n; - if ((s->nb_helpers + 1) > s->allocated_helpers) { - n = s->allocated_helpers; - if (n == 0) { - n = 4; - } else { - n *= 2; - } - s->helpers = realloc(s->helpers, n * sizeof(TCGHelperInfo)); - s->allocated_helpers = n; - } - s->helpers[s->nb_helpers].func = (uintptr_t)func; - s->helpers[s->nb_helpers].name = name; - s->nb_helpers++; -} - /* Note: we convert the 64 bit args to 32 bit and do some alignment and endian swap. Maybe it would be better to do the alignment and endian swap in tcg_reg_alloc_call(). */ -void tcg_gen_callN(TCGContext *s, TCGv_ptr func, unsigned int flags, - int sizemask, TCGArg ret, int nargs, TCGArg *args) +void tcg_gen_callN(TCGContext *s, void *func, TCGArg ret, + int nargs, TCGArg *args) { - int i; - int real_args; - int nb_rets; + int i, real_args, nb_rets; + unsigned sizemask, flags; TCGArg *nparam; - -#if defined(TCG_TARGET_EXTEND_ARGS) && TCG_TARGET_REG_BITS == 64 + TCGHelperInfo *info; + + info = g_hash_table_lookup(s->helpers, (gpointer)func); + flags = info->flags; + sizemask = info->sizemask; + +#if defined(__sparc__) && !defined(__arch64__) \ + && !defined(CONFIG_TCG_INTERPRETER) + /* We have 64-bit values in one register, but need to pass as two + separate parameters. Split them. */ + int orig_sizemask = sizemask; + int orig_nargs = nargs; + TCGv_i64 retl, reth; + + TCGV_UNUSED_I64(retl); + TCGV_UNUSED_I64(reth); + if (sizemask != 0) { + TCGArg *split_args = __builtin_alloca(sizeof(TCGArg) * nargs * 2); + for (i = real_args = 0; i < nargs; ++i) { + int is_64bit = sizemask & (1 << (i+1)*2); + if (is_64bit) { + TCGv_i64 orig = MAKE_TCGV_I64(args[i]); + TCGv_i32 h = tcg_temp_new_i32(); + TCGv_i32 l = tcg_temp_new_i32(); + tcg_gen_extr_i64_i32(l, h, orig); + split_args[real_args++] = GET_TCGV_I32(h); + split_args[real_args++] = GET_TCGV_I32(l); + } else { + split_args[real_args++] = args[i]; + } + } + nargs = real_args; + args = split_args; + sizemask = 0; + } +#elif defined(TCG_TARGET_EXTEND_ARGS) && TCG_TARGET_REG_BITS == 64 for (i = 0; i < nargs; ++i) { int is_64bit = sizemask & (1 << (i+1)*2); int is_signed = sizemask & (2 << (i+1)*2); @@ -670,9 +758,24 @@ void tcg_gen_callN(TCGContext *s, TCGv_ptr func, unsigned int flags, *s->gen_opc_ptr++ = INDEX_op_call; nparam = s->gen_opparam_ptr++; if (ret != TCG_CALL_DUMMY_ARG) { -#if TCG_TARGET_REG_BITS < 64 - if (sizemask & 1) { -#ifdef TCG_TARGET_WORDS_BIGENDIAN +#if defined(__sparc__) && !defined(__arch64__) \ + && !defined(CONFIG_TCG_INTERPRETER) + if (orig_sizemask & 1) { + /* The 32-bit ABI is going to return the 64-bit value in + the %o0/%o1 register pair. Prepare for this by using + two return temporaries, and reassemble below. */ + retl = tcg_temp_new_i64(); + reth = tcg_temp_new_i64(); + *s->gen_opparam_ptr++ = GET_TCGV_I64(reth); + *s->gen_opparam_ptr++ = GET_TCGV_I64(retl); + nb_rets = 2; + } else { + *s->gen_opparam_ptr++ = ret; + nb_rets = 1; + } +#else + if (TCG_TARGET_REG_BITS < 64 && (sizemask & 1)) { +#ifdef HOST_WORDS_BIGENDIAN *s->gen_opparam_ptr++ = ret + 1; *s->gen_opparam_ptr++ = ret; #else @@ -680,20 +783,18 @@ void tcg_gen_callN(TCGContext *s, TCGv_ptr func, unsigned int flags, *s->gen_opparam_ptr++ = ret + 1; #endif nb_rets = 2; - } else -#endif - { + } else { *s->gen_opparam_ptr++ = ret; nb_rets = 1; } +#endif } else { nb_rets = 0; } real_args = 0; for (i = 0; i < nargs; i++) { -#if TCG_TARGET_REG_BITS < 64 int is_64bit = sizemask & (1 << (i+1)*2); - if (is_64bit) { + if (TCG_TARGET_REG_BITS < 64 && is_64bit) { #ifdef TCG_TARGET_CALL_ALIGN_ARGS /* some targets want aligned 64 bit args */ if (real_args & 1) { @@ -711,7 +812,7 @@ void tcg_gen_callN(TCGContext *s, TCGv_ptr func, unsigned int flags, order. If another such target is added, this logic may have to get more complicated to differentiate between stack arguments and register arguments. */ -#if defined(TCG_TARGET_WORDS_BIGENDIAN) != defined(TCG_TARGET_STACK_GROWSUP) +#if defined(HOST_WORDS_BIGENDIAN) != defined(TCG_TARGET_STACK_GROWSUP) *s->gen_opparam_ptr++ = args[i] + 1; *s->gen_opparam_ptr++ = args[i]; #else @@ -721,21 +822,41 @@ void tcg_gen_callN(TCGContext *s, TCGv_ptr func, unsigned int flags, real_args += 2; continue; } -#endif /* TCG_TARGET_REG_BITS < 64 */ *s->gen_opparam_ptr++ = args[i]; real_args++; } - *s->gen_opparam_ptr++ = GET_TCGV_PTR(func); - + *s->gen_opparam_ptr++ = (uintptr_t)func; *s->gen_opparam_ptr++ = flags; - *nparam = (nb_rets << 16) | (real_args + 1); + *nparam = (nb_rets << 16) | real_args; /* total parameters, needed to go backward in the instruction stream */ *s->gen_opparam_ptr++ = 1 + nb_rets + real_args + 3; -#if defined(TCG_TARGET_EXTEND_ARGS) && TCG_TARGET_REG_BITS == 64 +#if defined(__sparc__) && !defined(__arch64__) \ + && !defined(CONFIG_TCG_INTERPRETER) + /* Free all of the parts we allocated above. */ + for (i = real_args = 0; i < orig_nargs; ++i) { + int is_64bit = orig_sizemask & (1 << (i+1)*2); + if (is_64bit) { + TCGv_i32 h = MAKE_TCGV_I32(args[real_args++]); + TCGv_i32 l = MAKE_TCGV_I32(args[real_args++]); + tcg_temp_free_i32(h); + tcg_temp_free_i32(l); + } else { + real_args++; + } + } + if (orig_sizemask & 1) { + /* The 32-bit ABI returned two 32-bit pieces. Re-assemble them. + Note that describing these as TCGv_i64 eliminates an unnecessary + zero-extension that tcg_gen_concat_i32_i64 would create. */ + tcg_gen_concat32_i64(MAKE_TCGV_I64(ret), retl, reth); + tcg_temp_free_i64(retl); + tcg_temp_free_i64(reth); + } +#elif defined(TCG_TARGET_EXTEND_ARGS) && TCG_TARGET_REG_BITS == 64 for (i = 0; i < nargs; ++i) { int is_64bit = sizemask & (1 << (i+1)*2); if (!is_64bit) { @@ -795,6 +916,93 @@ void tcg_gen_shifti_i64(TCGv_i64 ret, TCGv_i64 arg1, } #endif +static inline TCGMemOp tcg_canonicalize_memop(TCGMemOp op, bool is64, bool st) +{ + switch (op & MO_SIZE) { + case MO_8: + op &= ~MO_BSWAP; + break; + case MO_16: + break; + case MO_32: + if (!is64) { + op &= ~MO_SIGN; + } + break; + case MO_64: + if (!is64) { + tcg_abort(); + } + break; + } + if (st) { + op &= ~MO_SIGN; + } + return op; +} + +void tcg_gen_qemu_ld_i32(TCGv_i32 val, TCGv addr, TCGArg idx, TCGMemOp memop) +{ + memop = tcg_canonicalize_memop(memop, 0, 0); + + *tcg_ctx.gen_opc_ptr++ = INDEX_op_qemu_ld_i32; + tcg_add_param_i32(val); + tcg_add_param_tl(addr); + *tcg_ctx.gen_opparam_ptr++ = memop; + *tcg_ctx.gen_opparam_ptr++ = idx; +} + +void tcg_gen_qemu_st_i32(TCGv_i32 val, TCGv addr, TCGArg idx, TCGMemOp memop) +{ + memop = tcg_canonicalize_memop(memop, 0, 1); + + *tcg_ctx.gen_opc_ptr++ = INDEX_op_qemu_st_i32; + tcg_add_param_i32(val); + tcg_add_param_tl(addr); + *tcg_ctx.gen_opparam_ptr++ = memop; + *tcg_ctx.gen_opparam_ptr++ = idx; +} + +void tcg_gen_qemu_ld_i64(TCGv_i64 val, TCGv addr, TCGArg idx, TCGMemOp memop) +{ + memop = tcg_canonicalize_memop(memop, 1, 0); + +#if TCG_TARGET_REG_BITS == 32 + if ((memop & MO_SIZE) < MO_64) { + tcg_gen_qemu_ld_i32(TCGV_LOW(val), addr, idx, memop); + if (memop & MO_SIGN) { + tcg_gen_sari_i32(TCGV_HIGH(val), TCGV_LOW(val), 31); + } else { + tcg_gen_movi_i32(TCGV_HIGH(val), 0); + } + return; + } +#endif + + *tcg_ctx.gen_opc_ptr++ = INDEX_op_qemu_ld_i64; + tcg_add_param_i64(val); + tcg_add_param_tl(addr); + *tcg_ctx.gen_opparam_ptr++ = memop; + *tcg_ctx.gen_opparam_ptr++ = idx; +} + +void tcg_gen_qemu_st_i64(TCGv_i64 val, TCGv addr, TCGArg idx, TCGMemOp memop) +{ + memop = tcg_canonicalize_memop(memop, 1, 1); + +#if TCG_TARGET_REG_BITS == 32 + if ((memop & MO_SIZE) < MO_64) { + tcg_gen_qemu_st_i32(TCGV_LOW(val), addr, idx, memop); + return; + } +#endif + + *tcg_ctx.gen_opc_ptr++ = INDEX_op_qemu_st_i64; + tcg_add_param_i64(val); + tcg_add_param_tl(addr); + *tcg_ctx.gen_opparam_ptr++ = memop; + *tcg_ctx.gen_opparam_ptr++ = idx; +} static void tcg_reg_alloc_start(TCGContext *s) { @@ -851,47 +1059,17 @@ char *tcg_get_arg_str_i64(TCGContext *s, char *buf, int buf_size, TCGv_i64 arg) return tcg_get_arg_str_idx(s, buf, buf_size, GET_TCGV_I64(arg)); } -static int helper_cmp(const void *p1, const void *p2) -{ - const TCGHelperInfo *th1 = p1; - const TCGHelperInfo *th2 = p2; - if (th1->func < th2->func) - return -1; - else if (th1->func == th2->func) - return 0; - else - return 1; -} - -/* find helper definition (Note: A hash table would be better) */ -static TCGHelperInfo *tcg_find_helper(TCGContext *s, uintptr_t val) +/* Find helper name. */ +static inline const char *tcg_find_helper(TCGContext *s, uintptr_t val) { - int m, m_min, m_max; - TCGHelperInfo *th; - uintptr_t v; - - if (unlikely(!s->helpers_sorted)) { - qsort(s->helpers, s->nb_helpers, sizeof(TCGHelperInfo), - helper_cmp); - s->helpers_sorted = 1; - } - - /* binary search */ - m_min = 0; - m_max = s->nb_helpers - 1; - while (m_min <= m_max) { - m = (m_min + m_max) >> 1; - th = &s->helpers[m]; - v = th->func; - if (v == val) - return th; - else if (val < v) { - m_max = m - 1; - } else { - m_min = m + 1; + const char *ret = NULL; + if (s->helpers) { + TCGHelperInfo *info = g_hash_table_lookup(s->helpers, (gpointer)val); + if (info) { + ret = info->name; } } - return NULL; + return ret; } static const char * const cond_name[] = @@ -910,6 +1088,22 @@ static const char * const cond_name[] = [TCG_COND_GTU] = "gtu" }; +static const char * const ldst_name[] = +{ + [MO_UB] = "ub", + [MO_SB] = "sb", + [MO_LEUW] = "leuw", + [MO_LESW] = "lesw", + [MO_LEUL] = "leul", + [MO_LESL] = "lesl", + [MO_LEQ] = "leq", + [MO_BEUW] = "beuw", + [MO_BESW] = "besw", + [MO_BEUL] = "beul", + [MO_BESL] = "besl", + [MO_BEQ] = "beq", +}; + void tcg_dump_ops(TCGContext *s) { const uint16_t *opc_ptr; @@ -950,49 +1144,21 @@ void tcg_dump_ops(TCGContext *s) nb_iargs = arg & 0xffff; nb_cargs = def->nb_cargs; - qemu_log(" %s ", def->name); - - /* function name */ - qemu_log("%s", - tcg_get_arg_str_idx(s, buf, sizeof(buf), - args[nb_oargs + nb_iargs - 1])); - /* flags */ - qemu_log(",$0x%" TCG_PRIlx, args[nb_oargs + nb_iargs]); - /* nb out args */ - qemu_log(",$%d", nb_oargs); - for(i = 0; i < nb_oargs; i++) { - qemu_log(","); - qemu_log("%s", tcg_get_arg_str_idx(s, buf, sizeof(buf), + /* function name, flags, out args */ + qemu_log(" %s %s,$0x%" TCG_PRIlx ",$%d", def->name, + tcg_find_helper(s, args[nb_oargs + nb_iargs]), + args[nb_oargs + nb_iargs + 1], nb_oargs); + for (i = 0; i < nb_oargs; i++) { + qemu_log(",%s", tcg_get_arg_str_idx(s, buf, sizeof(buf), args[i])); } - for(i = 0; i < (nb_iargs - 1); i++) { - qemu_log(","); - if (args[nb_oargs + i] == TCG_CALL_DUMMY_ARG) { - qemu_log(""); - } else { - qemu_log("%s", tcg_get_arg_str_idx(s, buf, sizeof(buf), - args[nb_oargs + i])); - } - } - } else if (c == INDEX_op_movi_i32 || c == INDEX_op_movi_i64) { - tcg_target_ulong val; - TCGHelperInfo *th; - - nb_oargs = def->nb_oargs; - nb_iargs = def->nb_iargs; - nb_cargs = def->nb_cargs; - qemu_log(" %s %s,$", def->name, - tcg_get_arg_str_idx(s, buf, sizeof(buf), args[0])); - val = args[1]; - th = tcg_find_helper(s, val); - if (th) { - qemu_log("%s", th->name); - } else { - if (c == INDEX_op_movi_i32) { - qemu_log("0x%x", (uint32_t)val); - } else { - qemu_log("0x%" PRIx64 , (uint64_t)val); + for (i = 0; i < nb_iargs; i++) { + TCGArg arg = args[nb_oargs + i]; + const char *t = ""; + if (arg != TCG_CALL_DUMMY_ARG) { + t = tcg_get_arg_str_idx(s, buf, sizeof(buf), arg); } + qemu_log(",%s", t); } } else { qemu_log(" %s ", def->name); @@ -1038,6 +1204,17 @@ void tcg_dump_ops(TCGContext *s) } i = 1; break; + case INDEX_op_qemu_ld_i32: + case INDEX_op_qemu_st_i32: + case INDEX_op_qemu_ld_i64: + case INDEX_op_qemu_st_i64: + if (args[k] < ARRAY_SIZE(ldst_name) && ldst_name[args[k]]) { + qemu_log(",%s", ldst_name[args[k++]]); + } else { + qemu_log(",$0x%" TCG_PRIlx, args[k++]); + } + i = 1; + break; default: i = 0; break; @@ -1244,9 +1421,9 @@ static inline void tcg_la_bb_end(TCGContext *s, uint8_t *dead_temps, temporaries are removed. */ static void tcg_liveness_analysis(TCGContext *s) { - int i, op_index, nb_args, nb_iargs, nb_oargs, arg, nb_ops; + int i, op_index, nb_args, nb_iargs, nb_oargs, nb_ops; TCGOpcode op, op_new, op_new2; - TCGArg *args; + TCGArg *args, arg; const TCGOpDef *def; uint8_t *dead_temps, *mem_temps; uint16_t dead_args; @@ -1276,15 +1453,15 @@ static void tcg_liveness_analysis(TCGContext *s) nb_args = args[-1]; args -= nb_args; - nb_iargs = args[0] & 0xffff; - nb_oargs = args[0] >> 16; - args++; - call_flags = args[nb_oargs + nb_iargs]; + arg = *args++; + nb_iargs = arg & 0xffff; + nb_oargs = arg >> 16; + call_flags = args[nb_oargs + nb_iargs + 1]; /* pure functions can be removed if their result is not used */ if (call_flags & TCG_CALL_NO_SIDE_EFFECTS) { - for(i = 0; i < nb_oargs; i++) { + for (i = 0; i < nb_oargs; i++) { arg = args[i]; if (!dead_temps[arg] || mem_temps[arg]) { goto do_not_remove_call; @@ -1298,7 +1475,7 @@ static void tcg_liveness_analysis(TCGContext *s) /* output args are dead */ dead_args = 0; sync_args = 0; - for(i = 0; i < nb_oargs; i++) { + for (i = 0; i < nb_oargs; i++) { arg = args[i]; if (dead_temps[arg]) { dead_args |= (1 << i); @@ -1321,7 +1498,7 @@ static void tcg_liveness_analysis(TCGContext *s) } /* input args are live */ - for(i = nb_oargs; i < nb_iargs + nb_oargs; i++) { + for (i = nb_oargs; i < nb_iargs + nb_oargs; i++) { arg = args[i]; if (arg != TCG_CALL_DUMMY_ARG) { if (dead_temps[arg]) { @@ -1820,13 +1997,15 @@ static void tcg_reg_alloc_mov(TCGContext *s, const TCGOpDef *def, { TCGRegSet allocated_regs; TCGTemp *ts, *ots; - const TCGArgConstraint *arg_ct, *oarg_ct; + TCGType otype, itype; tcg_regset_set(allocated_regs, s->reserved_regs); ots = &s->temps[args[0]]; ts = &s->temps[args[1]]; - oarg_ct = &def->args_ct[0]; - arg_ct = &def->args_ct[1]; + + /* Note that otype != itype for no-op truncation. */ + otype = ots->type; + itype = ts->type; /* If the source value is not in a register, and we're going to be forced to have it in a register in order to perform the copy, @@ -1834,12 +2013,13 @@ static void tcg_reg_alloc_mov(TCGContext *s, const TCGOpDef *def, we don't have to reload SOURCE the next time it is used. */ if (((NEED_SYNC_ARG(0) || ots->fixed_reg) && ts->val_type != TEMP_VAL_REG) || ts->val_type == TEMP_VAL_MEM) { - ts->reg = tcg_reg_alloc(s, arg_ct->u.regs, allocated_regs); + ts->reg = tcg_reg_alloc(s, tcg_target_available_regs[itype], + allocated_regs); if (ts->val_type == TEMP_VAL_MEM) { - tcg_out_ld(s, ts->type, ts->reg, ts->mem_reg, ts->mem_offset); + tcg_out_ld(s, itype, ts->reg, ts->mem_reg, ts->mem_offset); ts->mem_coherent = 1; } else if (ts->val_type == TEMP_VAL_CONST) { - tcg_out_movi(s, ts->type, ts->reg, ts->val); + tcg_out_movi(s, itype, ts->reg, ts->val); } s->reg_to_temp[ts->reg] = args[1]; ts->val_type = TEMP_VAL_REG; @@ -1854,7 +2034,7 @@ static void tcg_reg_alloc_mov(TCGContext *s, const TCGOpDef *def, if (!ots->mem_allocated) { temp_allocate_frame(s, args[0]); } - tcg_out_st(s, ots->type, ts->reg, ots->mem_reg, ots->mem_offset); + tcg_out_st(s, otype, ts->reg, ots->mem_reg, ots->mem_offset); if (IS_DEAD_ARG(1)) { temp_dead(s, args[1]); } @@ -1882,9 +2062,10 @@ static void tcg_reg_alloc_mov(TCGContext *s, const TCGOpDef *def, /* When allocating a new register, make sure to not spill the input one. */ tcg_regset_set_reg(allocated_regs, ts->reg); - ots->reg = tcg_reg_alloc(s, oarg_ct->u.regs, allocated_regs); + ots->reg = tcg_reg_alloc(s, tcg_target_available_regs[otype], + allocated_regs); } - tcg_out_mov(s, ots->type, ots->reg, ts->reg); + tcg_out_mov(s, otype, ots->reg, ts->reg); } ots->val_type = TEMP_VAL_REG; ots->mem_coherent = 0; @@ -1931,7 +2112,7 @@ static void tcg_reg_alloc_op(TCGContext *s, ts->mem_coherent = 1; s->reg_to_temp[reg] = arg; } else if (ts->val_type == TEMP_VAL_CONST) { - if (tcg_target_const_match(ts->val, arg_ct)) { + if (tcg_target_const_match(ts->val, ts->type, arg_ct)) { /* constant is OK for instruction */ const_args[i] = 1; new_args[i] = ts->val; @@ -2068,26 +2249,27 @@ static int tcg_reg_alloc_call(TCGContext *s, const TCGOpDef *def, uint16_t dead_args, uint8_t sync_args) { int nb_iargs, nb_oargs, flags, nb_regs, i, reg, nb_params; - TCGArg arg, func_arg; + TCGArg arg; TCGTemp *ts; intptr_t stack_offset; size_t call_stack_size; - uintptr_t func_addr; - int const_func_arg, allocate_args; + tcg_insn_unit *func_addr; + int allocate_args; TCGRegSet allocated_regs; - const TCGArgConstraint *arg_ct; arg = *args++; nb_oargs = arg >> 16; nb_iargs = arg & 0xffff; - nb_params = nb_iargs - 1; + nb_params = nb_iargs; - flags = args[nb_oargs + nb_iargs]; + func_addr = (tcg_insn_unit *)(intptr_t)args[nb_oargs + nb_iargs]; + flags = args[nb_oargs + nb_iargs + 1]; nb_regs = ARRAY_SIZE(tcg_target_call_iarg_regs); - if (nb_regs > nb_params) + if (nb_regs > nb_params) { nb_regs = nb_params; + } /* assign stack slots first */ call_stack_size = (nb_params - nb_regs) * sizeof(tcg_target_long); @@ -2155,40 +2337,6 @@ static int tcg_reg_alloc_call(TCGContext *s, const TCGOpDef *def, } } - /* assign function address */ - func_arg = args[nb_oargs + nb_iargs - 1]; - arg_ct = &def->args_ct[0]; - ts = &s->temps[func_arg]; - func_addr = ts->val; - const_func_arg = 0; - if (ts->val_type == TEMP_VAL_MEM) { - reg = tcg_reg_alloc(s, arg_ct->u.regs, allocated_regs); - tcg_out_ld(s, ts->type, reg, ts->mem_reg, ts->mem_offset); - func_arg = reg; - tcg_regset_set_reg(allocated_regs, reg); - } else if (ts->val_type == TEMP_VAL_REG) { - reg = ts->reg; - if (!tcg_regset_test_reg(arg_ct->u.regs, reg)) { - reg = tcg_reg_alloc(s, arg_ct->u.regs, allocated_regs); - tcg_out_mov(s, ts->type, reg, ts->reg); - } - func_arg = reg; - tcg_regset_set_reg(allocated_regs, reg); - } else if (ts->val_type == TEMP_VAL_CONST) { - if (tcg_target_const_match(func_addr, arg_ct)) { - const_func_arg = 1; - func_arg = func_addr; - } else { - reg = tcg_reg_alloc(s, arg_ct->u.regs, allocated_regs); - tcg_out_movi(s, ts->type, reg, func_addr); - func_arg = reg; - tcg_regset_set_reg(allocated_regs, reg); - } - } else { - tcg_abort(); - } - - /* mark dead temporaries and free the associated registers */ for(i = nb_oargs; i < nb_iargs + nb_oargs; i++) { if (IS_DEAD_ARG(i)) { @@ -2213,7 +2361,7 @@ static int tcg_reg_alloc_call(TCGContext *s, const TCGOpDef *def, save_globals(s, allocated_regs); } - tcg_out_op(s, opc, &func_arg, &const_func_arg); + tcg_out_call(s, func_addr); /* assign output registers and emit moves if needed */ for(i = 0; i < nb_oargs; i++) { @@ -2221,6 +2369,7 @@ static int tcg_reg_alloc_call(TCGContext *s, const TCGOpDef *def, ts = &s->temps[arg]; reg = tcg_target_call_oarg_regs[i]; assert(s->reg_to_temp[reg] == -1); + if (ts->fixed_reg) { if (ts->reg != reg) { tcg_out_mov(s, ts->type, ts->reg, reg); @@ -2262,7 +2411,8 @@ static void dump_op_count(void) #endif -static inline int tcg_gen_code_common(TCGContext *s, uint8_t *gen_code_buf, +static inline int tcg_gen_code_common(TCGContext *s, + tcg_insn_unit *gen_code_buf, long search_pc) { TCGOpcode opc; @@ -2311,6 +2461,8 @@ static inline int tcg_gen_code_common(TCGContext *s, uint8_t *gen_code_buf, s->code_buf = gen_code_buf; s->code_ptr = gen_code_buf; + tcg_out_tb_init(s); + args = s->gen_opparam_buf; op_index = 0; @@ -2375,7 +2527,7 @@ static inline int tcg_gen_code_common(TCGContext *s, uint8_t *gen_code_buf, } args += def->nb_args; next: - if (search_pc >= 0 && search_pc < s->code_ptr - gen_code_buf) { + if (search_pc >= 0 && search_pc < tcg_current_code_size(s)) { return op_index; } op_index++; @@ -2384,14 +2536,12 @@ static inline int tcg_gen_code_common(TCGContext *s, uint8_t *gen_code_buf, #endif } the_end: -#if defined(CONFIG_QEMU_LDST_OPTIMIZATION) && defined(CONFIG_SOFTMMU) /* Generate TB finalization at the end of block */ tcg_out_tb_finalize(s); -#endif return -1; } -int tcg_gen_code(TCGContext *s, uint8_t *gen_code_buf) +int tcg_gen_code(TCGContext *s, tcg_insn_unit *gen_code_buf) { #ifdef CONFIG_PROFILER { @@ -2410,16 +2560,17 @@ int tcg_gen_code(TCGContext *s, uint8_t *gen_code_buf) tcg_gen_code_common(s, gen_code_buf, -1); /* flush instruction cache */ - flush_icache_range((uintptr_t)gen_code_buf, (uintptr_t)s->code_ptr); + flush_icache_range((uintptr_t)s->code_buf, (uintptr_t)s->code_ptr); - return s->code_ptr - gen_code_buf; + return tcg_current_code_size(s); } /* Return the index of the micro operation such as the pc after is < offset bytes from the start of the TB. The contents of gen_code_buf must not be changed, though writing the same values is ok. Return -1 if not found. */ -int tcg_gen_code_search_pc(TCGContext *s, uint8_t *gen_code_buf, long offset) +int tcg_gen_code_search_pc(TCGContext *s, tcg_insn_unit *gen_code_buf, + long offset) { return tcg_gen_code_common(s, gen_code_buf, offset); } @@ -2537,7 +2688,8 @@ static int find_string(const char *strtab, const char *str) } static void tcg_register_jit_int(void *buf_ptr, size_t buf_size, - void *debug_frame, size_t debug_frame_size) + const void *debug_frame, + size_t debug_frame_size) { struct __attribute__((packed)) DebugInfo { uint32_t len; @@ -2675,10 +2827,10 @@ static void tcg_register_jit_int(void *buf_ptr, size_t buf_size, uintptr_t buf = (uintptr_t)buf_ptr; size_t img_size = sizeof(struct ElfImage) + debug_frame_size; + DebugFrameHeader *dfh; img = g_malloc(img_size); *img = img_template; - memcpy(img + 1, debug_frame, debug_frame_size); img->phdr.p_vaddr = buf; img->phdr.p_paddr = buf; @@ -2706,6 +2858,11 @@ static void tcg_register_jit_int(void *buf_ptr, size_t buf_size, img->di.fn_low_pc = buf; img->di.fn_high_pc = buf + buf_size; + dfh = (DebugFrameHeader *)(img + 1); + memcpy(dfh, debug_frame, debug_frame_size); + dfh->fde.func_start = buf; + dfh->fde.func_len = buf_size; + #ifdef DEBUG_JIT /* Enable this block to be able to debug the ELF image file creation. One can use readelf, objdump, or other inspection utilities. */ @@ -2733,7 +2890,8 @@ static void tcg_register_jit_int(void *buf_ptr, size_t buf_size, and implement the internal function we declared earlier. */ static void tcg_register_jit_int(void *buf, size_t size, - void *debug_frame, size_t debug_frame_size) + const void *debug_frame, + size_t debug_frame_size) { }