X-Git-Url: https://repo.jachan.dev/qemu.git/blobdiff_plain/29f3ff8d6cbc28f79933aeaa25805408d0984a8f..2e4a0b176642c5ffcca063880ad2aa8e44c78c8a:/tcg/optimize.c

diff --git a/tcg/optimize.c b/tcg/optimize.c
index cd0e793a9d..5dbe11c3c8 100644
--- a/tcg/optimize.c
+++ b/tcg/optimize.c
@@ -23,169 +23,139 @@
  * THE SOFTWARE.
  */
 
-#include "config.h"
-
-#include <stdlib.h>
-#include <stdio.h>
-
+#include "qemu/osdep.h"
 #include "qemu-common.h"
+#include "exec/cpu-common.h"
 #include "tcg-op.h"
 
 #define CASE_OP_32_64(x)                        \
         glue(glue(case INDEX_op_, x), _i32):    \
         glue(glue(case INDEX_op_, x), _i64)
 
-typedef enum {
-    TCG_TEMP_UNDEF = 0,
-    TCG_TEMP_CONST,
-    TCG_TEMP_COPY,
-} tcg_temp_state;
+#define CASE_OP_32_64_VEC(x)                    \
+        glue(glue(case INDEX_op_, x), _i32):    \
+        glue(glue(case INDEX_op_, x), _i64):    \
+        glue(glue(case INDEX_op_, x), _vec)
 
 struct tcg_temp_info {
-    tcg_temp_state state;
-    uint16_t prev_copy;
-    uint16_t next_copy;
+    bool is_const;
+    TCGTemp *prev_copy;
+    TCGTemp *next_copy;
     tcg_target_ulong val;
     tcg_target_ulong mask;
 };
 
-static struct tcg_temp_info temps[TCG_MAX_TEMPS];
+static inline struct tcg_temp_info *ts_info(TCGTemp *ts)
+{
+    return ts->state_ptr;
+}
 
-/* Reset TEMP's state to TCG_TEMP_UNDEF.  If TEMP only had one copy, remove
-   the copy flag from the left temp.  */
-static void reset_temp(TCGArg temp)
+static inline struct tcg_temp_info *arg_info(TCGArg arg)
 {
-    if (temps[temp].state == TCG_TEMP_COPY) {
-        if (temps[temp].prev_copy == temps[temp].next_copy) {
-            temps[temps[temp].next_copy].state = TCG_TEMP_UNDEF;
-        } else {
-            temps[temps[temp].next_copy].prev_copy = temps[temp].prev_copy;
-            temps[temps[temp].prev_copy].next_copy = temps[temp].next_copy;
-        }
-    }
-    temps[temp].state = TCG_TEMP_UNDEF;
-    temps[temp].mask = -1;
+    return ts_info(arg_temp(arg));
 }
 
-static TCGOp *insert_op_before(TCGContext *s, TCGOp *old_op,
-                                TCGOpcode opc, int nargs)
+static inline bool ts_is_const(TCGTemp *ts)
 {
-    int oi = s->gen_next_op_idx;
-    int pi = s->gen_next_parm_idx;
-    int prev = old_op->prev;
-    int next = old_op - s->gen_op_buf;
-    TCGOp *new_op;
-
-    tcg_debug_assert(oi < OPC_BUF_SIZE);
-    tcg_debug_assert(pi + nargs <= OPPARAM_BUF_SIZE);
-    s->gen_next_op_idx = oi + 1;
-    s->gen_next_parm_idx = pi + nargs;
-
-    new_op = &s->gen_op_buf[oi];
-    *new_op = (TCGOp){
-        .opc = opc,
-        .args = pi,
-        .prev = prev,
-        .next = next
-    };
-    if (prev >= 0) {
-        s->gen_op_buf[prev].next = oi;
-    } else {
-        s->gen_first_op_idx = oi;
-    }
-    old_op->prev = oi;
+    return ts_info(ts)->is_const;
+}
 
-    return new_op;
+static inline bool arg_is_const(TCGArg arg)
+{
+    return ts_is_const(arg_temp(arg));
 }
 
-/* Reset all temporaries, given that there are NB_TEMPS of them.  */
-static void reset_all_temps(int nb_temps)
+static inline bool ts_is_copy(TCGTemp *ts)
 {
-    int i;
-    for (i = 0; i < nb_temps; i++) {
-        temps[i].state = TCG_TEMP_UNDEF;
-        temps[i].mask = -1;
-    }
+    return ts_info(ts)->next_copy != ts;
 }
 
-static int op_bits(TCGOpcode op)
+/* Reset TEMP's state, possibly removing the temp for the list of copies.  */
+static void reset_ts(TCGTemp *ts)
 {
-    const TCGOpDef *def = &tcg_op_defs[op];
-    return def->flags & TCG_OPF_64BIT ? 64 : 32;
+    struct tcg_temp_info *ti = ts_info(ts);
+    struct tcg_temp_info *pi = ts_info(ti->prev_copy);
+    struct tcg_temp_info *ni = ts_info(ti->next_copy);
+
+    ni->prev_copy = ti->prev_copy;
+    pi->next_copy = ti->next_copy;
+    ti->next_copy = ts;
+    ti->prev_copy = ts;
+    ti->is_const = false;
+    ti->mask = -1;
 }
 
-static TCGOpcode op_to_mov(TCGOpcode op)
+static void reset_temp(TCGArg arg)
 {
-    switch (op_bits(op)) {
-    case 32:
-        return INDEX_op_mov_i32;
-    case 64:
-        return INDEX_op_mov_i64;
-    default:
-        fprintf(stderr, "op_to_mov: unexpected return value of "
-                "function op_bits.\n");
-        tcg_abort();
-    }
+    reset_ts(arg_temp(arg));
 }
 
-static TCGOpcode op_to_movi(TCGOpcode op)
+/* Initialize and activate a temporary.  */
+static void init_ts_info(struct tcg_temp_info *infos,
+                         TCGTempSet *temps_used, TCGTemp *ts)
 {
-    switch (op_bits(op)) {
-    case 32:
-        return INDEX_op_movi_i32;
-    case 64:
-        return INDEX_op_movi_i64;
-    default:
-        fprintf(stderr, "op_to_movi: unexpected return value of "
-                "function op_bits.\n");
-        tcg_abort();
+    size_t idx = temp_idx(ts);
+    if (!test_bit(idx, temps_used->l)) {
+        struct tcg_temp_info *ti = &infos[idx];
+
+        ts->state_ptr = ti;
+        ti->next_copy = ts;
+        ti->prev_copy = ts;
+        ti->is_const = false;
+        ti->mask = -1;
+        set_bit(idx, temps_used->l);
     }
 }
 
-static TCGArg find_better_copy(TCGContext *s, TCGArg temp)
+static void init_arg_info(struct tcg_temp_info *infos,
+                          TCGTempSet *temps_used, TCGArg arg)
+{
+    init_ts_info(infos, temps_used, arg_temp(arg));
+}
+
+static TCGTemp *find_better_copy(TCGContext *s, TCGTemp *ts)
 {
-    TCGArg i;
+    TCGTemp *i;
 
     /* If this is already a global, we can't do better. */
-    if (temp < s->nb_globals) {
-        return temp;
+    if (ts->temp_global) {
+        return ts;
     }
 
     /* Search for a global first. */
-    for (i = temps[temp].next_copy ; i != temp ; i = temps[i].next_copy) {
-        if (i < s->nb_globals) {
+    for (i = ts_info(ts)->next_copy; i != ts; i = ts_info(i)->next_copy) {
+        if (i->temp_global) {
             return i;
         }
     }
 
     /* If it is a temp, search for a temp local. */
-    if (!s->temps[temp].temp_local) {
-        for (i = temps[temp].next_copy ; i != temp ; i = temps[i].next_copy) {
-            if (s->temps[i].temp_local) {
+    if (!ts->temp_local) {
+        for (i = ts_info(ts)->next_copy; i != ts; i = ts_info(i)->next_copy) {
+            if (ts->temp_local) {
                 return i;
             }
         }
     }
 
     /* Failure to find a better representation, return the same temp. */
-    return temp;
+    return ts;
 }
 
-static bool temps_are_copies(TCGArg arg1, TCGArg arg2)
+static bool ts_are_copies(TCGTemp *ts1, TCGTemp *ts2)
 {
-    TCGArg i;
+    TCGTemp *i;
 
-    if (arg1 == arg2) {
+    if (ts1 == ts2) {
         return true;
     }
 
-    if (temps[arg1].state != TCG_TEMP_COPY
-        || temps[arg2].state != TCG_TEMP_COPY) {
+    if (!ts_is_copy(ts1) || !ts_is_copy(ts2)) {
         return false;
     }
 
-    for (i = temps[arg1].next_copy ; i != arg1 ; i = temps[i].next_copy) {
-        if (i == arg2) {
+    for (i = ts_info(ts1)->next_copy; i != ts1; i = ts_info(i)->next_copy) {
+        if (i == ts2) {
             return true;
         }
     }
@@ -193,71 +163,90 @@ static bool temps_are_copies(TCGArg arg1, TCGArg arg2)
     return false;
 }
 
-static void tcg_opt_gen_movi(TCGContext *s, TCGOp *op, TCGArg *args,
-                             TCGArg dst, TCGArg val)
+static bool args_are_copies(TCGArg arg1, TCGArg arg2)
 {
-    TCGOpcode new_op = op_to_movi(op->opc);
+    return ts_are_copies(arg_temp(arg1), arg_temp(arg2));
+}
+
+static void tcg_opt_gen_movi(TCGContext *s, TCGOp *op, TCGArg dst, TCGArg val)
+{
+    const TCGOpDef *def;
+    TCGOpcode new_op;
     tcg_target_ulong mask;
+    struct tcg_temp_info *di = arg_info(dst);
 
+    def = &tcg_op_defs[op->opc];
+    if (def->flags & TCG_OPF_VECTOR) {
+        new_op = INDEX_op_dupi_vec;
+    } else if (def->flags & TCG_OPF_64BIT) {
+        new_op = INDEX_op_movi_i64;
+    } else {
+        new_op = INDEX_op_movi_i32;
+    }
     op->opc = new_op;
+    /* TCGOP_VECL and TCGOP_VECE remain unchanged.  */
+    op->args[0] = dst;
+    op->args[1] = val;
 
     reset_temp(dst);
-    temps[dst].state = TCG_TEMP_CONST;
-    temps[dst].val = val;
+    di->is_const = true;
+    di->val = val;
     mask = val;
     if (TCG_TARGET_REG_BITS > 32 && new_op == INDEX_op_movi_i32) {
         /* High bits of the destination are now garbage.  */
         mask |= ~0xffffffffull;
     }
-    temps[dst].mask = mask;
-
-    args[0] = dst;
-    args[1] = val;
+    di->mask = mask;
 }
 
-static void tcg_opt_gen_mov(TCGContext *s, TCGOp *op, TCGArg *args,
-                            TCGArg dst, TCGArg src)
+static void tcg_opt_gen_mov(TCGContext *s, TCGOp *op, TCGArg dst, TCGArg src)
 {
-    if (temps_are_copies(dst, src)) {
+    TCGTemp *dst_ts = arg_temp(dst);
+    TCGTemp *src_ts = arg_temp(src);
+    const TCGOpDef *def;
+    struct tcg_temp_info *di;
+    struct tcg_temp_info *si;
+    tcg_target_ulong mask;
+    TCGOpcode new_op;
+
+    if (ts_are_copies(dst_ts, src_ts)) {
         tcg_op_remove(s, op);
         return;
     }
 
-    if (temps[src].state == TCG_TEMP_CONST) {
-        tcg_opt_gen_movi(s, op, args, dst, temps[src].val);
-        return;
+    reset_ts(dst_ts);
+    di = ts_info(dst_ts);
+    si = ts_info(src_ts);
+    def = &tcg_op_defs[op->opc];
+    if (def->flags & TCG_OPF_VECTOR) {
+        new_op = INDEX_op_mov_vec;
+    } else if (def->flags & TCG_OPF_64BIT) {
+        new_op = INDEX_op_mov_i64;
+    } else {
+        new_op = INDEX_op_mov_i32;
     }
-
-    TCGOpcode new_op = op_to_mov(op->opc);
-    tcg_target_ulong mask;
-
     op->opc = new_op;
+    /* TCGOP_VECL and TCGOP_VECE remain unchanged.  */
+    op->args[0] = dst;
+    op->args[1] = src;
 
-    reset_temp(dst);
-    mask = temps[src].mask;
+    mask = si->mask;
     if (TCG_TARGET_REG_BITS > 32 && new_op == INDEX_op_mov_i32) {
         /* High bits of the destination are now garbage.  */
         mask |= ~0xffffffffull;
     }
-    temps[dst].mask = mask;
+    di->mask = mask;
 
-    assert(temps[src].state != TCG_TEMP_CONST);
+    if (src_ts->type == dst_ts->type) {
+        struct tcg_temp_info *ni = ts_info(si->next_copy);
 
-    if (s->temps[src].type == s->temps[dst].type) {
-        if (temps[src].state != TCG_TEMP_COPY) {
-            temps[src].state = TCG_TEMP_COPY;
-            temps[src].next_copy = src;
-            temps[src].prev_copy = src;
-        }
-        temps[dst].state = TCG_TEMP_COPY;
-        temps[dst].next_copy = temps[src].next_copy;
-        temps[dst].prev_copy = src;
-        temps[temps[dst].next_copy].prev_copy = dst;
-        temps[src].next_copy = dst;
+        di->next_copy = si->next_copy;
+        di->prev_copy = src_ts;
+        ni->prev_copy = dst_ts;
+        si->next_copy = dst_ts;
+        di->is_const = si->is_const;
+        di->val = si->val;
     }
-
-    args[0] = dst;
-    args[1] = src;
 }
 
 static TCGArg do_constant_folding_2(TCGOpcode op, TCGArg x, TCGArg y)
@@ -292,7 +281,6 @@ static TCGArg do_constant_folding_2(TCGOpcode op, TCGArg x, TCGArg y)
     case INDEX_op_shr_i32:
         return (uint32_t)x >> (y & 31);
 
-    case INDEX_op_trunc_shr_i32:
     case INDEX_op_shr_i64:
         return (uint64_t)x >> (y & 63);
 
@@ -335,6 +323,24 @@ static TCGArg do_constant_folding_2(TCGOpcode op, TCGArg x, TCGArg y)
     CASE_OP_32_64(nor):
         return ~(x | y);
 
+    case INDEX_op_clz_i32:
+        return (uint32_t)x ? clz32(x) : y;
+
+    case INDEX_op_clz_i64:
+        return x ? clz64(x) : y;
+
+    case INDEX_op_ctz_i32:
+        return (uint32_t)x ? ctz32(x) : y;
+
+    case INDEX_op_ctz_i64:
+        return x ? ctz64(x) : y;
+
+    case INDEX_op_ctpop_i32:
+        return ctpop32(x);
+
+    case INDEX_op_ctpop_i64:
+        return ctpop64(x);
+
     CASE_OP_32_64(ext8s):
         return (int8_t)x;
 
@@ -347,12 +353,18 @@ static TCGArg do_constant_folding_2(TCGOpcode op, TCGArg x, TCGArg y)
     CASE_OP_32_64(ext16u):
         return (uint16_t)x;
 
+    case INDEX_op_ext_i32_i64:
     case INDEX_op_ext32s_i64:
         return (int32_t)x;
 
+    case INDEX_op_extu_i32_i64:
+    case INDEX_op_extrl_i64_i32:
     case INDEX_op_ext32u_i64:
         return (uint32_t)x;
 
+    case INDEX_op_extrh_i64_i32:
+        return (uint64_t)x >> 32;
+
     case INDEX_op_muluh_i32:
         return ((uint64_t)(uint32_t)x * (uint32_t)y) >> 32;
     case INDEX_op_mulsh_i32:
@@ -393,8 +405,9 @@ static TCGArg do_constant_folding_2(TCGOpcode op, TCGArg x, TCGArg y)
 
 static TCGArg do_constant_folding(TCGOpcode op, TCGArg x, TCGArg y)
 {
+    const TCGOpDef *def = &tcg_op_defs[op];
     TCGArg res = do_constant_folding_2(op, x, y);
-    if (op_bits(op) == 32) {
+    if (!(def->flags & TCG_OPF_64BIT)) {
         res = (int32_t)res;
     }
     return res;
@@ -481,18 +494,19 @@ static bool do_constant_folding_cond_eq(TCGCond c)
 static TCGArg do_constant_folding_cond(TCGOpcode op, TCGArg x,
                                        TCGArg y, TCGCond c)
 {
-    if (temps[x].state == TCG_TEMP_CONST && temps[y].state == TCG_TEMP_CONST) {
-        switch (op_bits(op)) {
-        case 32:
-            return do_constant_folding_cond_32(temps[x].val, temps[y].val, c);
-        case 64:
-            return do_constant_folding_cond_64(temps[x].val, temps[y].val, c);
-        default:
-            tcg_abort();
+    tcg_target_ulong xv = arg_info(x)->val;
+    tcg_target_ulong yv = arg_info(y)->val;
+    if (arg_is_const(x) && arg_is_const(y)) {
+        const TCGOpDef *def = &tcg_op_defs[op];
+        tcg_debug_assert(!(def->flags & TCG_OPF_VECTOR));
+        if (def->flags & TCG_OPF_64BIT) {
+            return do_constant_folding_cond_64(xv, yv, c);
+        } else {
+            return do_constant_folding_cond_32(xv, yv, c);
         }
-    } else if (temps_are_copies(x, y)) {
+    } else if (args_are_copies(x, y)) {
         return do_constant_folding_cond_eq(c);
-    } else if (temps[y].state == TCG_TEMP_CONST && temps[y].val == 0) {
+    } else if (arg_is_const(y) && yv == 0) {
         switch (c) {
         case TCG_COND_LTU:
             return 0;
@@ -501,9 +515,8 @@ static TCGArg do_constant_folding_cond(TCGOpcode op, TCGArg x,
         default:
             return 2;
         }
-    } else {
-        return 2;
     }
+    return 2;
 }
 
 /* Return 2 if the condition can't be simplified, and the result
@@ -513,14 +526,15 @@ static TCGArg do_constant_folding_cond2(TCGArg *p1, TCGArg *p2, TCGCond c)
     TCGArg al = p1[0], ah = p1[1];
     TCGArg bl = p2[0], bh = p2[1];
 
-    if (temps[bl].state == TCG_TEMP_CONST
-        && temps[bh].state == TCG_TEMP_CONST) {
-        uint64_t b = ((uint64_t)temps[bh].val << 32) | (uint32_t)temps[bl].val;
+    if (arg_is_const(bl) && arg_is_const(bh)) {
+        tcg_target_ulong blv = arg_info(bl)->val;
+        tcg_target_ulong bhv = arg_info(bh)->val;
+        uint64_t b = deposit64(blv, 32, 32, bhv);
 
-        if (temps[al].state == TCG_TEMP_CONST
-            && temps[ah].state == TCG_TEMP_CONST) {
-            uint64_t a;
-            a = ((uint64_t)temps[ah].val << 32) | (uint32_t)temps[al].val;
+        if (arg_is_const(al) && arg_is_const(ah)) {
+            tcg_target_ulong alv = arg_info(al)->val;
+            tcg_target_ulong ahv = arg_info(ah)->val;
+            uint64_t a = deposit64(alv, 32, 32, ahv);
             return do_constant_folding_cond_64(a, b, c);
         }
         if (b == 0) {
@@ -534,7 +548,7 @@ static TCGArg do_constant_folding_cond2(TCGArg *p1, TCGArg *p2, TCGCond c)
             }
         }
     }
-    if (temps_are_copies(al, bl) && temps_are_copies(ah, bh)) {
+    if (args_are_copies(al, bl) && args_are_copies(ah, bh)) {
         return do_constant_folding_cond_eq(c);
     }
     return 2;
@@ -544,8 +558,8 @@ static bool swap_commutative(TCGArg dest, TCGArg *p1, TCGArg *p2)
 {
     TCGArg a1 = *p1, a2 = *p2;
     int sum = 0;
-    sum += temps[a1].state == TCG_TEMP_CONST;
-    sum -= temps[a2].state == TCG_TEMP_CONST;
+    sum += arg_is_const(a1);
+    sum -= arg_is_const(a2);
 
     /* Prefer the constant in second argument, and then the form
        op a, a, b, which is better handled on non-RISC hosts. */
@@ -560,10 +574,10 @@ static bool swap_commutative(TCGArg dest, TCGArg *p1, TCGArg *p2)
 static bool swap_commutative2(TCGArg *p1, TCGArg *p2)
 {
     int sum = 0;
-    sum += temps[p1[0]].state == TCG_TEMP_CONST;
-    sum += temps[p1[1]].state == TCG_TEMP_CONST;
-    sum -= temps[p2[0]].state == TCG_TEMP_CONST;
-    sum -= temps[p2[1]].state == TCG_TEMP_CONST;
+    sum += arg_is_const(p1[0]);
+    sum += arg_is_const(p1[1]);
+    sum -= arg_is_const(p2[0]);
+    sum -= arg_is_const(p2[1]);
     if (sum > 0) {
         TCGArg t;
         t = p1[0], p1[0] = p2[0], p2[0] = t;
@@ -576,7 +590,10 @@ static bool swap_commutative2(TCGArg *p1, TCGArg *p2)
 /* Propagate constants and copies, fold constant expressions. */
 void tcg_optimize(TCGContext *s)
 {
-    int oi, oi_next, nb_temps, nb_globals;
+    int nb_temps, nb_globals;
+    TCGOp *op, *op_next, *prev_mb = NULL;
+    struct tcg_temp_info *infos;
+    TCGTempSet temps_used;
 
     /* Array VALS has an element for each temp.
        If this temp holds a constant then its value is kept in VALS' element.
@@ -585,85 +602,94 @@ void tcg_optimize(TCGContext *s)
 
     nb_temps = s->nb_temps;
     nb_globals = s->nb_globals;
-    reset_all_temps(nb_temps);
+    bitmap_zero(temps_used.l, nb_temps);
+    infos = tcg_malloc(sizeof(struct tcg_temp_info) * nb_temps);
 
-    for (oi = s->gen_first_op_idx; oi >= 0; oi = oi_next) {
+    QTAILQ_FOREACH_SAFE(op, &s->ops, link, op_next) {
         tcg_target_ulong mask, partmask, affected;
         int nb_oargs, nb_iargs, i;
         TCGArg tmp;
-
-        TCGOp * const op = &s->gen_op_buf[oi];
-        TCGArg * const args = &s->gen_opparam_buf[op->args];
         TCGOpcode opc = op->opc;
         const TCGOpDef *def = &tcg_op_defs[opc];
 
-        oi_next = op->next;
+        /* Count the arguments, and initialize the temps that are
+           going to be used */
         if (opc == INDEX_op_call) {
-            nb_oargs = op->callo;
-            nb_iargs = op->calli;
+            nb_oargs = TCGOP_CALLO(op);
+            nb_iargs = TCGOP_CALLI(op);
+            for (i = 0; i < nb_oargs + nb_iargs; i++) {
+                TCGTemp *ts = arg_temp(op->args[i]);
+                if (ts) {
+                    init_ts_info(infos, &temps_used, ts);
+                }
+            }
         } else {
             nb_oargs = def->nb_oargs;
             nb_iargs = def->nb_iargs;
+            for (i = 0; i < nb_oargs + nb_iargs; i++) {
+                init_arg_info(infos, &temps_used, op->args[i]);
+            }
         }
 
         /* Do copy propagation */
         for (i = nb_oargs; i < nb_oargs + nb_iargs; i++) {
-            if (temps[args[i]].state == TCG_TEMP_COPY) {
-                args[i] = find_better_copy(s, args[i]);
+            TCGTemp *ts = arg_temp(op->args[i]);
+            if (ts && ts_is_copy(ts)) {
+                op->args[i] = temp_arg(find_better_copy(s, ts));
             }
         }
 
         /* For commutative operations make constant second argument */
         switch (opc) {
-        CASE_OP_32_64(add):
-        CASE_OP_32_64(mul):
-        CASE_OP_32_64(and):
-        CASE_OP_32_64(or):
-        CASE_OP_32_64(xor):
+        CASE_OP_32_64_VEC(add):
+        CASE_OP_32_64_VEC(mul):
+        CASE_OP_32_64_VEC(and):
+        CASE_OP_32_64_VEC(or):
+        CASE_OP_32_64_VEC(xor):
         CASE_OP_32_64(eqv):
         CASE_OP_32_64(nand):
         CASE_OP_32_64(nor):
         CASE_OP_32_64(muluh):
         CASE_OP_32_64(mulsh):
-            swap_commutative(args[0], &args[1], &args[2]);
+            swap_commutative(op->args[0], &op->args[1], &op->args[2]);
             break;
         CASE_OP_32_64(brcond):
-            if (swap_commutative(-1, &args[0], &args[1])) {
-                args[2] = tcg_swap_cond(args[2]);
+            if (swap_commutative(-1, &op->args[0], &op->args[1])) {
+                op->args[2] = tcg_swap_cond(op->args[2]);
             }
             break;
         CASE_OP_32_64(setcond):
-            if (swap_commutative(args[0], &args[1], &args[2])) {
-                args[3] = tcg_swap_cond(args[3]);
+            if (swap_commutative(op->args[0], &op->args[1], &op->args[2])) {
+                op->args[3] = tcg_swap_cond(op->args[3]);
             }
             break;
         CASE_OP_32_64(movcond):
-            if (swap_commutative(-1, &args[1], &args[2])) {
-                args[5] = tcg_swap_cond(args[5]);
+            if (swap_commutative(-1, &op->args[1], &op->args[2])) {
+                op->args[5] = tcg_swap_cond(op->args[5]);
             }
             /* For movcond, we canonicalize the "false" input reg to match
                the destination reg so that the tcg backend can implement
                a "move if true" operation.  */
-            if (swap_commutative(args[0], &args[4], &args[3])) {
-                args[5] = tcg_invert_cond(args[5]);
+            if (swap_commutative(op->args[0], &op->args[4], &op->args[3])) {
+                op->args[5] = tcg_invert_cond(op->args[5]);
             }
             break;
         CASE_OP_32_64(add2):
-            swap_commutative(args[0], &args[2], &args[4]);
-            swap_commutative(args[1], &args[3], &args[5]);
+            swap_commutative(op->args[0], &op->args[2], &op->args[4]);
+            swap_commutative(op->args[1], &op->args[3], &op->args[5]);
             break;
         CASE_OP_32_64(mulu2):
         CASE_OP_32_64(muls2):
-            swap_commutative(args[0], &args[2], &args[3]);
+            swap_commutative(op->args[0], &op->args[2], &op->args[3]);
             break;
         case INDEX_op_brcond2_i32:
-            if (swap_commutative2(&args[0], &args[2])) {
-                args[4] = tcg_swap_cond(args[4]);
+            if (swap_commutative2(&op->args[0], &op->args[2])) {
+                op->args[4] = tcg_swap_cond(op->args[4]);
             }
             break;
         case INDEX_op_setcond2_i32:
-            if (swap_commutative2(&args[1], &args[3])) {
-                args[5] = tcg_swap_cond(args[5]);
+            if (swap_commutative2(&op->args[1], &op->args[3])) {
+                op->args[5] = tcg_swap_cond(op->args[5]);
             }
             break;
         default:
@@ -678,70 +704,73 @@ void tcg_optimize(TCGContext *s)
         CASE_OP_32_64(sar):
         CASE_OP_32_64(rotl):
         CASE_OP_32_64(rotr):
-            if (temps[args[1]].state == TCG_TEMP_CONST
-                && temps[args[1]].val == 0) {
-                tcg_opt_gen_movi(s, op, args, args[0], 0);
+            if (arg_is_const(op->args[1])
+                && arg_info(op->args[1])->val == 0) {
+                tcg_opt_gen_movi(s, op, op->args[0], 0);
                 continue;
             }
             break;
-        CASE_OP_32_64(sub):
+        CASE_OP_32_64_VEC(sub):
             {
                 TCGOpcode neg_op;
                 bool have_neg;
 
-                if (temps[args[2]].state == TCG_TEMP_CONST) {
+                if (arg_is_const(op->args[2])) {
                     /* Proceed with possible constant folding. */
                     break;
                 }
                 if (opc == INDEX_op_sub_i32) {
                     neg_op = INDEX_op_neg_i32;
                     have_neg = TCG_TARGET_HAS_neg_i32;
-                } else {
+                } else if (opc == INDEX_op_sub_i64) {
                     neg_op = INDEX_op_neg_i64;
                     have_neg = TCG_TARGET_HAS_neg_i64;
+                } else {
+                    neg_op = INDEX_op_neg_vec;
+                    have_neg = TCG_TARGET_HAS_neg_vec;
                 }
                 if (!have_neg) {
                     break;
                 }
-                if (temps[args[1]].state == TCG_TEMP_CONST
-                    && temps[args[1]].val == 0) {
+                if (arg_is_const(op->args[1])
+                    && arg_info(op->args[1])->val == 0) {
                     op->opc = neg_op;
-                    reset_temp(args[0]);
-                    args[1] = args[2];
+                    reset_temp(op->args[0]);
+                    op->args[1] = op->args[2];
                     continue;
                 }
             }
             break;
-        CASE_OP_32_64(xor):
+        CASE_OP_32_64_VEC(xor):
         CASE_OP_32_64(nand):
-            if (temps[args[1]].state != TCG_TEMP_CONST
-                && temps[args[2]].state == TCG_TEMP_CONST
-                && temps[args[2]].val == -1) {
+            if (!arg_is_const(op->args[1])
+                && arg_is_const(op->args[2])
+                && arg_info(op->args[2])->val == -1) {
                 i = 1;
                 goto try_not;
             }
             break;
         CASE_OP_32_64(nor):
-            if (temps[args[1]].state != TCG_TEMP_CONST
-                && temps[args[2]].state == TCG_TEMP_CONST
-                && temps[args[2]].val == 0) {
+            if (!arg_is_const(op->args[1])
+                && arg_is_const(op->args[2])
+                && arg_info(op->args[2])->val == 0) {
                 i = 1;
                 goto try_not;
             }
             break;
-        CASE_OP_32_64(andc):
-            if (temps[args[2]].state != TCG_TEMP_CONST
-                && temps[args[1]].state == TCG_TEMP_CONST
-                && temps[args[1]].val == -1) {
+        CASE_OP_32_64_VEC(andc):
+            if (!arg_is_const(op->args[2])
+                && arg_is_const(op->args[1])
+                && arg_info(op->args[1])->val == -1) {
                 i = 2;
                 goto try_not;
             }
             break;
-        CASE_OP_32_64(orc):
+        CASE_OP_32_64_VEC(orc):
         CASE_OP_32_64(eqv):
-            if (temps[args[2]].state != TCG_TEMP_CONST
-                && temps[args[1]].state == TCG_TEMP_CONST
-                && temps[args[1]].val == 0) {
+            if (!arg_is_const(op->args[2])
+                && arg_is_const(op->args[1])
+                && arg_info(op->args[1])->val == 0) {
                 i = 2;
                 goto try_not;
             }
@@ -751,7 +780,10 @@ void tcg_optimize(TCGContext *s)
                 TCGOpcode not_op;
                 bool have_not;
 
-                if (def->flags & TCG_OPF_64BIT) {
+                if (def->flags & TCG_OPF_VECTOR) {
+                    not_op = INDEX_op_not_vec;
+                    have_not = TCG_TARGET_HAS_not_vec;
+                } else if (def->flags & TCG_OPF_64BIT) {
                     not_op = INDEX_op_not_i64;
                     have_not = TCG_TARGET_HAS_not_i64;
                 } else {
@@ -762,8 +794,8 @@ void tcg_optimize(TCGContext *s)
                     break;
                 }
                 op->opc = not_op;
-                reset_temp(args[0]);
-                args[1] = args[i];
+                reset_temp(op->args[0]);
+                op->args[1] = op->args[i];
                 continue;
             }
         default:
@@ -772,30 +804,30 @@ void tcg_optimize(TCGContext *s)
 
         /* Simplify expression for "op r, a, const => mov r, a" cases */
         switch (opc) {
-        CASE_OP_32_64(add):
-        CASE_OP_32_64(sub):
+        CASE_OP_32_64_VEC(add):
+        CASE_OP_32_64_VEC(sub):
+        CASE_OP_32_64_VEC(or):
+        CASE_OP_32_64_VEC(xor):
+        CASE_OP_32_64_VEC(andc):
         CASE_OP_32_64(shl):
         CASE_OP_32_64(shr):
         CASE_OP_32_64(sar):
         CASE_OP_32_64(rotl):
         CASE_OP_32_64(rotr):
-        CASE_OP_32_64(or):
-        CASE_OP_32_64(xor):
-        CASE_OP_32_64(andc):
-            if (temps[args[1]].state != TCG_TEMP_CONST
-                && temps[args[2]].state == TCG_TEMP_CONST
-                && temps[args[2]].val == 0) {
-                tcg_opt_gen_mov(s, op, args, args[0], args[1]);
+            if (!arg_is_const(op->args[1])
+                && arg_is_const(op->args[2])
+                && arg_info(op->args[2])->val == 0) {
+                tcg_opt_gen_mov(s, op, op->args[0], op->args[1]);
                 continue;
             }
             break;
-        CASE_OP_32_64(and):
-        CASE_OP_32_64(orc):
+        CASE_OP_32_64_VEC(and):
+        CASE_OP_32_64_VEC(orc):
         CASE_OP_32_64(eqv):
-            if (temps[args[1]].state != TCG_TEMP_CONST
-                && temps[args[2]].state == TCG_TEMP_CONST
-                && temps[args[2]].val == -1) {
-                tcg_opt_gen_mov(s, op, args, args[0], args[1]);
+            if (!arg_is_const(op->args[1])
+                && arg_is_const(op->args[2])
+                && arg_info(op->args[2])->val == -1) {
+                tcg_opt_gen_mov(s, op, op->args[0], op->args[1]);
                 continue;
             }
             break;
@@ -809,21 +841,21 @@ void tcg_optimize(TCGContext *s)
         affected = -1;
         switch (opc) {
         CASE_OP_32_64(ext8s):
-            if ((temps[args[1]].mask & 0x80) != 0) {
+            if ((arg_info(op->args[1])->mask & 0x80) != 0) {
                 break;
             }
         CASE_OP_32_64(ext8u):
             mask = 0xff;
             goto and_const;
         CASE_OP_32_64(ext16s):
-            if ((temps[args[1]].mask & 0x8000) != 0) {
+            if ((arg_info(op->args[1])->mask & 0x8000) != 0) {
                 break;
             }
         CASE_OP_32_64(ext16u):
             mask = 0xffff;
             goto and_const;
         case INDEX_op_ext32s_i64:
-            if ((temps[args[1]].mask & 0x80000000) != 0) {
+            if ((arg_info(op->args[1])->mask & 0x80000000) != 0) {
                 break;
             }
         case INDEX_op_ext32u_i64:
@@ -831,75 +863,121 @@ void tcg_optimize(TCGContext *s)
             goto and_const;
 
         CASE_OP_32_64(and):
-            mask = temps[args[2]].mask;
-            if (temps[args[2]].state == TCG_TEMP_CONST) {
+            mask = arg_info(op->args[2])->mask;
+            if (arg_is_const(op->args[2])) {
         and_const:
-                affected = temps[args[1]].mask & ~mask;
+                affected = arg_info(op->args[1])->mask & ~mask;
+            }
+            mask = arg_info(op->args[1])->mask & mask;
+            break;
+
+        case INDEX_op_ext_i32_i64:
+            if ((arg_info(op->args[1])->mask & 0x80000000) != 0) {
+                break;
             }
-            mask = temps[args[1]].mask & mask;
+        case INDEX_op_extu_i32_i64:
+            /* We do not compute affected as it is a size changing op.  */
+            mask = (uint32_t)arg_info(op->args[1])->mask;
             break;
 
         CASE_OP_32_64(andc):
             /* Known-zeros does not imply known-ones.  Therefore unless
-               args[2] is constant, we can't infer anything from it.  */
-            if (temps[args[2]].state == TCG_TEMP_CONST) {
-                mask = ~temps[args[2]].mask;
+               op->args[2] is constant, we can't infer anything from it.  */
+            if (arg_is_const(op->args[2])) {
+                mask = ~arg_info(op->args[2])->mask;
                 goto and_const;
             }
             /* But we certainly know nothing outside args[1] may be set. */
-            mask = temps[args[1]].mask;
+            mask = arg_info(op->args[1])->mask;
             break;
 
         case INDEX_op_sar_i32:
-            if (temps[args[2]].state == TCG_TEMP_CONST) {
-                tmp = temps[args[2]].val & 31;
-                mask = (int32_t)temps[args[1]].mask >> tmp;
+            if (arg_is_const(op->args[2])) {
+                tmp = arg_info(op->args[2])->val & 31;
+                mask = (int32_t)arg_info(op->args[1])->mask >> tmp;
             }
             break;
         case INDEX_op_sar_i64:
-            if (temps[args[2]].state == TCG_TEMP_CONST) {
-                tmp = temps[args[2]].val & 63;
-                mask = (int64_t)temps[args[1]].mask >> tmp;
+            if (arg_is_const(op->args[2])) {
+                tmp = arg_info(op->args[2])->val & 63;
+                mask = (int64_t)arg_info(op->args[1])->mask >> tmp;
             }
             break;
 
         case INDEX_op_shr_i32:
-            if (temps[args[2]].state == TCG_TEMP_CONST) {
-                tmp = temps[args[2]].val & 31;
-                mask = (uint32_t)temps[args[1]].mask >> tmp;
+            if (arg_is_const(op->args[2])) {
+                tmp = arg_info(op->args[2])->val & 31;
+                mask = (uint32_t)arg_info(op->args[1])->mask >> tmp;
             }
             break;
         case INDEX_op_shr_i64:
-            if (temps[args[2]].state == TCG_TEMP_CONST) {
-                tmp = temps[args[2]].val & 63;
-                mask = (uint64_t)temps[args[1]].mask >> tmp;
+            if (arg_is_const(op->args[2])) {
+                tmp = arg_info(op->args[2])->val & 63;
+                mask = (uint64_t)arg_info(op->args[1])->mask >> tmp;
             }
             break;
 
-        case INDEX_op_trunc_shr_i32:
-            mask = (uint64_t)temps[args[1]].mask >> args[2];
+        case INDEX_op_extrl_i64_i32:
+            mask = (uint32_t)arg_info(op->args[1])->mask;
+            break;
+        case INDEX_op_extrh_i64_i32:
+            mask = (uint64_t)arg_info(op->args[1])->mask >> 32;
             break;
 
         CASE_OP_32_64(shl):
-            if (temps[args[2]].state == TCG_TEMP_CONST) {
-                tmp = temps[args[2]].val & (TCG_TARGET_REG_BITS - 1);
-                mask = temps[args[1]].mask << tmp;
+            if (arg_is_const(op->args[2])) {
+                tmp = arg_info(op->args[2])->val & (TCG_TARGET_REG_BITS - 1);
+                mask = arg_info(op->args[1])->mask << tmp;
             }
             break;
 
         CASE_OP_32_64(neg):
             /* Set to 1 all bits to the left of the rightmost.  */
-            mask = -(temps[args[1]].mask & -temps[args[1]].mask);
+            mask = -(arg_info(op->args[1])->mask
+                     & -arg_info(op->args[1])->mask);
             break;
 
         CASE_OP_32_64(deposit):
-            mask = deposit64(temps[args[1]].mask, args[3], args[4],
-                             temps[args[2]].mask);
+            mask = deposit64(arg_info(op->args[1])->mask,
+                             op->args[3], op->args[4],
+                             arg_info(op->args[2])->mask);
+            break;
+
+        CASE_OP_32_64(extract):
+            mask = extract64(arg_info(op->args[1])->mask,
+                             op->args[2], op->args[3]);
+            if (op->args[2] == 0) {
+                affected = arg_info(op->args[1])->mask & ~mask;
+            }
+            break;
+        CASE_OP_32_64(sextract):
+            mask = sextract64(arg_info(op->args[1])->mask,
+                              op->args[2], op->args[3]);
+            if (op->args[2] == 0 && (tcg_target_long)mask >= 0) {
+                affected = arg_info(op->args[1])->mask & ~mask;
+            }
             break;
 
         CASE_OP_32_64(or):
         CASE_OP_32_64(xor):
-            mask = temps[args[1]].mask | temps[args[2]].mask;
+            mask = arg_info(op->args[1])->mask | arg_info(op->args[2])->mask;
+            break;
+
+        case INDEX_op_clz_i32:
+        case INDEX_op_ctz_i32:
+            mask = arg_info(op->args[2])->mask | 31;
+            break;
+
+        case INDEX_op_clz_i64:
+        case INDEX_op_ctz_i64:
+            mask = arg_info(op->args[2])->mask | 63;
+            break;
+
+        case INDEX_op_ctpop_i32:
+            mask = 32 | 31;
+            break;
+        case INDEX_op_ctpop_i64:
+            mask = 64 | 63;
             break;
 
         CASE_OP_32_64(setcond):
@@ -908,7 +986,7 @@ void tcg_optimize(TCGContext *s)
             break;
 
         CASE_OP_32_64(movcond):
-            mask = temps[args[3]].mask | temps[args[4]].mask;
+            mask = arg_info(op->args[3])->mask | arg_info(op->args[4])->mask;
             break;
 
         CASE_OP_32_64(ld8u):
@@ -923,7 +1001,7 @@ void tcg_optimize(TCGContext *s)
 
         CASE_OP_32_64(qemu_ld):
             {
-                TCGMemOpIdx oi = args[nb_oargs + nb_iargs];
+                TCGMemOpIdx oi = op->args[nb_oargs + nb_iargs];
                 TCGMemOp mop = get_memop(oi);
                 if (!(mop & MO_SIGN)) {
                     mask = (2ULL << ((8 << (mop & MO_SIZE)) - 1)) - 1;
@@ -946,25 +1024,25 @@ void tcg_optimize(TCGContext *s)
         }
 
         if (partmask == 0) {
-            assert(nb_oargs == 1);
-            tcg_opt_gen_movi(s, op, args, args[0], 0);
+            tcg_debug_assert(nb_oargs == 1);
+            tcg_opt_gen_movi(s, op, op->args[0], 0);
             continue;
         }
         if (affected == 0) {
-            assert(nb_oargs == 1);
-            tcg_opt_gen_mov(s, op, args, args[0], args[1]);
+            tcg_debug_assert(nb_oargs == 1);
+            tcg_opt_gen_mov(s, op, op->args[0], op->args[1]);
             continue;
         }
 
         /* Simplify expression for "op r, a, 0 => movi r, 0" cases */
         switch (opc) {
-        CASE_OP_32_64(and):
-        CASE_OP_32_64(mul):
+        CASE_OP_32_64_VEC(and):
+        CASE_OP_32_64_VEC(mul):
         CASE_OP_32_64(muluh):
         CASE_OP_32_64(mulsh):
-            if ((temps[args[2]].state == TCG_TEMP_CONST
-                && temps[args[2]].val == 0)) {
-                tcg_opt_gen_movi(s, op, args, args[0], 0);
+            if (arg_is_const(op->args[2])
+                && arg_info(op->args[2])->val == 0) {
+                tcg_opt_gen_movi(s, op, op->args[0], 0);
                 continue;
             }
             break;
@@ -974,10 +1052,10 @@ void tcg_optimize(TCGContext *s)
 
         /* Simplify expression for "op r, a, a => mov r, a" cases */
         switch (opc) {
-        CASE_OP_32_64(or):
-        CASE_OP_32_64(and):
-            if (temps_are_copies(args[1], args[2])) {
-                tcg_opt_gen_mov(s, op, args, args[0], args[1]);
+        CASE_OP_32_64_VEC(or):
+        CASE_OP_32_64_VEC(and):
+            if (args_are_copies(op->args[1], op->args[2])) {
+                tcg_opt_gen_mov(s, op, op->args[0], op->args[1]);
                 continue;
             }
             break;
@@ -987,11 +1065,11 @@ void tcg_optimize(TCGContext *s)
 
         /* Simplify expression for "op r, a, a => movi r, 0" cases */
         switch (opc) {
-        CASE_OP_32_64(andc):
-        CASE_OP_32_64(sub):
-        CASE_OP_32_64(xor):
-            if (temps_are_copies(args[1], args[2])) {
-                tcg_opt_gen_movi(s, op, args, args[0], 0);
+        CASE_OP_32_64_VEC(andc):
+        CASE_OP_32_64_VEC(sub):
+        CASE_OP_32_64_VEC(xor):
+            if (args_are_copies(op->args[1], op->args[2])) {
+                tcg_opt_gen_movi(s, op, op->args[0], 0);
                 continue;
             }
             break;
@@ -1003,32 +1081,39 @@ void tcg_optimize(TCGContext *s)
            folding.  Constants will be substituted to arguments by register
            allocator where needed and possible.  Also detect copies. */
         switch (opc) {
-        CASE_OP_32_64(mov):
-            tcg_opt_gen_mov(s, op, args, args[0], args[1]);
+        CASE_OP_32_64_VEC(mov):
+            tcg_opt_gen_mov(s, op, op->args[0], op->args[1]);
             break;
         CASE_OP_32_64(movi):
-            tcg_opt_gen_movi(s, op, args, args[0], args[1]);
+        case INDEX_op_dupi_vec:
+            tcg_opt_gen_movi(s, op, op->args[0], op->args[1]);
             break;
 
+        case INDEX_op_dup_vec:
+            if (arg_is_const(op->args[1])) {
+                tmp = arg_info(op->args[1])->val;
+                tmp = dup_const(TCGOP_VECE(op), tmp);
+                tcg_opt_gen_movi(s, op, op->args[0], tmp);
+                break;
+            }
+            goto do_default;
+
         CASE_OP_32_64(not):
         CASE_OP_32_64(neg):
         CASE_OP_32_64(ext8s):
         CASE_OP_32_64(ext8u):
         CASE_OP_32_64(ext16s):
         CASE_OP_32_64(ext16u):
+        CASE_OP_32_64(ctpop):
         case INDEX_op_ext32s_i64:
         case INDEX_op_ext32u_i64:
-            if (temps[args[1]].state == TCG_TEMP_CONST) {
-                tmp = do_constant_folding(opc, temps[args[1]].val, 0);
-                tcg_opt_gen_movi(s, op, args, args[0], tmp);
-                break;
-            }
-            goto do_default;
-
-        case INDEX_op_trunc_shr_i32:
-            if (temps[args[1]].state == TCG_TEMP_CONST) {
-                tmp = do_constant_folding(opc, temps[args[1]].val, args[2]);
-                tcg_opt_gen_movi(s, op, args, args[0], tmp);
+        case INDEX_op_ext_i32_i64:
+        case INDEX_op_extu_i32_i64:
+        case INDEX_op_extrl_i64_i32:
+        case INDEX_op_extrh_i64_i32:
+            if (arg_is_const(op->args[1])) {
+                tmp = do_constant_folding(opc, arg_info(op->args[1])->val, 0);
+                tcg_opt_gen_movi(s, op, op->args[0], tmp);
                 break;
             }
             goto do_default;
@@ -1055,40 +1140,73 @@ void tcg_optimize(TCGContext *s)
         CASE_OP_32_64(divu):
         CASE_OP_32_64(rem):
         CASE_OP_32_64(remu):
-            if (temps[args[1]].state == TCG_TEMP_CONST
-                && temps[args[2]].state == TCG_TEMP_CONST) {
-                tmp = do_constant_folding(opc, temps[args[1]].val,
-                                          temps[args[2]].val);
-                tcg_opt_gen_movi(s, op, args, args[0], tmp);
+            if (arg_is_const(op->args[1]) && arg_is_const(op->args[2])) {
+                tmp = do_constant_folding(opc, arg_info(op->args[1])->val,
+                                          arg_info(op->args[2])->val);
+                tcg_opt_gen_movi(s, op, op->args[0], tmp);
+                break;
+            }
+            goto do_default;
+
+        CASE_OP_32_64(clz):
+        CASE_OP_32_64(ctz):
+            if (arg_is_const(op->args[1])) {
+                TCGArg v = arg_info(op->args[1])->val;
+                if (v != 0) {
+                    tmp = do_constant_folding(opc, v, 0);
+                    tcg_opt_gen_movi(s, op, op->args[0], tmp);
+                } else {
+                    tcg_opt_gen_mov(s, op, op->args[0], op->args[2]);
+                }
                 break;
             }
             goto do_default;
 
         CASE_OP_32_64(deposit):
-            if (temps[args[1]].state == TCG_TEMP_CONST
-                && temps[args[2]].state == TCG_TEMP_CONST) {
-                tmp = deposit64(temps[args[1]].val, args[3], args[4],
-                                temps[args[2]].val);
-                tcg_opt_gen_movi(s, op, args, args[0], tmp);
+            if (arg_is_const(op->args[1]) && arg_is_const(op->args[2])) {
+                tmp = deposit64(arg_info(op->args[1])->val,
+                                op->args[3], op->args[4],
+                                arg_info(op->args[2])->val);
+                tcg_opt_gen_movi(s, op, op->args[0], tmp);
+                break;
+            }
+            goto do_default;
+
+        CASE_OP_32_64(extract):
+            if (arg_is_const(op->args[1])) {
+                tmp = extract64(arg_info(op->args[1])->val,
+                                op->args[2], op->args[3]);
+                tcg_opt_gen_movi(s, op, op->args[0], tmp);
+                break;
+            }
+            goto do_default;
+
+        CASE_OP_32_64(sextract):
+            if (arg_is_const(op->args[1])) {
+                tmp = sextract64(arg_info(op->args[1])->val,
+                                 op->args[2], op->args[3]);
+                tcg_opt_gen_movi(s, op, op->args[0], tmp);
                 break;
             }
             goto do_default;
 
         CASE_OP_32_64(setcond):
-            tmp = do_constant_folding_cond(opc, args[1], args[2], args[3]);
+            tmp = do_constant_folding_cond(opc, op->args[1],
+                                           op->args[2], op->args[3]);
             if (tmp != 2) {
-                tcg_opt_gen_movi(s, op, args, args[0], tmp);
+                tcg_opt_gen_movi(s, op, op->args[0], tmp);
                 break;
             }
             goto do_default;
 
         CASE_OP_32_64(brcond):
-            tmp = do_constant_folding_cond(opc, args[0], args[1], args[2]);
+            tmp = do_constant_folding_cond(opc, op->args[0],
+                                           op->args[1], op->args[2]);
             if (tmp != 2) {
                 if (tmp) {
-                    reset_all_temps(nb_temps);
+                    bitmap_zero(temps_used.l, nb_temps);
                     op->opc = INDEX_op_br;
-                    args[0] = args[3];
+                    op->args[0] = op->args[3];
                 } else {
                     tcg_op_remove(s, op);
                 }
@@ -1097,28 +1215,41 @@ void tcg_optimize(TCGContext *s)
             goto do_default;
 
         CASE_OP_32_64(movcond):
-            tmp = do_constant_folding_cond(opc, args[1], args[2], args[5]);
+            tmp = do_constant_folding_cond(opc, op->args[1],
+                                           op->args[2], op->args[5]);
             if (tmp != 2) {
-                tcg_opt_gen_mov(s, op, args, args[0], args[4-tmp]);
+                tcg_opt_gen_mov(s, op, op->args[0], op->args[4-tmp]);
                 break;
             }
+            if (arg_is_const(op->args[3]) && arg_is_const(op->args[4])) {
+                tcg_target_ulong tv = arg_info(op->args[3])->val;
+                tcg_target_ulong fv = arg_info(op->args[4])->val;
+                TCGCond cond = op->args[5];
+                if (fv == 1 && tv == 0) {
+                    cond = tcg_invert_cond(cond);
+                } else if (!(tv == 1 && fv == 0)) {
+                    goto do_default;
+                }
+                op->args[3] = cond;
+                op->opc = opc = (opc == INDEX_op_movcond_i32
+                                 ? INDEX_op_setcond_i32
+                                 : INDEX_op_setcond_i64);
+                nb_iargs = 2;
+            }
             goto do_default;
 
         case INDEX_op_add2_i32:
         case INDEX_op_sub2_i32:
-            if (temps[args[2]].state == TCG_TEMP_CONST
-                && temps[args[3]].state == TCG_TEMP_CONST
-                && temps[args[4]].state == TCG_TEMP_CONST
-                && temps[args[5]].state == TCG_TEMP_CONST) {
-                uint32_t al = temps[args[2]].val;
-                uint32_t ah = temps[args[3]].val;
-                uint32_t bl = temps[args[4]].val;
-                uint32_t bh = temps[args[5]].val;
+            if (arg_is_const(op->args[2]) && arg_is_const(op->args[3])
+                && arg_is_const(op->args[4]) && arg_is_const(op->args[5])) {
+                uint32_t al = arg_info(op->args[2])->val;
+                uint32_t ah = arg_info(op->args[3])->val;
+                uint32_t bl = arg_info(op->args[4])->val;
+                uint32_t bh = arg_info(op->args[5])->val;
                 uint64_t a = ((uint64_t)ah << 32) | al;
                 uint64_t b = ((uint64_t)bh << 32) | bl;
                 TCGArg rl, rh;
-                TCGOp *op2 = insert_op_before(s, op, INDEX_op_movi_i32, 2);
-                TCGArg *args2 = &s->gen_opparam_buf[op2->args];
+                TCGOp *op2 = tcg_op_insert_before(s, op, INDEX_op_movi_i32, 2);
 
                 if (opc == INDEX_op_add2_i32) {
                     a += b;
@@ -1126,99 +1257,97 @@ void tcg_optimize(TCGContext *s)
                     a -= b;
                 }
 
-                rl = args[0];
-                rh = args[1];
-                tcg_opt_gen_movi(s, op, args, rl, (int32_t)a);
-                tcg_opt_gen_movi(s, op2, args2, rh, (int32_t)(a >> 32));
-
-                /* We've done all we need to do with the movi.  Skip it.  */
-                oi_next = op2->next;
+                rl = op->args[0];
+                rh = op->args[1];
+                tcg_opt_gen_movi(s, op, rl, (int32_t)a);
+                tcg_opt_gen_movi(s, op2, rh, (int32_t)(a >> 32));
                 break;
             }
             goto do_default;
 
         case INDEX_op_mulu2_i32:
-            if (temps[args[2]].state == TCG_TEMP_CONST
-                && temps[args[3]].state == TCG_TEMP_CONST) {
-                uint32_t a = temps[args[2]].val;
-                uint32_t b = temps[args[3]].val;
+            if (arg_is_const(op->args[2]) && arg_is_const(op->args[3])) {
+                uint32_t a = arg_info(op->args[2])->val;
+                uint32_t b = arg_info(op->args[3])->val;
                 uint64_t r = (uint64_t)a * b;
                 TCGArg rl, rh;
-                TCGOp *op2 = insert_op_before(s, op, INDEX_op_movi_i32, 2);
-                TCGArg *args2 = &s->gen_opparam_buf[op2->args];
-
-                rl = args[0];
-                rh = args[1];
-                tcg_opt_gen_movi(s, op, args, rl, (int32_t)r);
-                tcg_opt_gen_movi(s, op2, args2, rh, (int32_t)(r >> 32));
+                TCGOp *op2 = tcg_op_insert_before(s, op, INDEX_op_movi_i32, 2);
 
-                /* We've done all we need to do with the movi.  Skip it.  */
-                oi_next = op2->next;
+                rl = op->args[0];
+                rh = op->args[1];
+                tcg_opt_gen_movi(s, op, rl, (int32_t)r);
+                tcg_opt_gen_movi(s, op2, rh, (int32_t)(r >> 32));
                 break;
             }
             goto do_default;
 
         case INDEX_op_brcond2_i32:
-            tmp = do_constant_folding_cond2(&args[0], &args[2], args[4]);
+            tmp = do_constant_folding_cond2(&op->args[0], &op->args[2],
+                                            op->args[4]);
             if (tmp != 2) {
                 if (tmp) {
             do_brcond_true:
-                    reset_all_temps(nb_temps);
+                    bitmap_zero(temps_used.l, nb_temps);
                     op->opc = INDEX_op_br;
-                    args[0] = args[5];
+                    op->args[0] = op->args[5];
                 } else {
             do_brcond_false:
                     tcg_op_remove(s, op);
                 }
-            } else if ((args[4] == TCG_COND_LT || args[4] == TCG_COND_GE)
-                       && temps[args[2]].state == TCG_TEMP_CONST
-                       && temps[args[3]].state == TCG_TEMP_CONST
-                       && temps[args[2]].val == 0
-                       && temps[args[3]].val == 0) {
+            } else if ((op->args[4] == TCG_COND_LT
+                        || op->args[4] == TCG_COND_GE)
+                       && arg_is_const(op->args[2])
+                       && arg_info(op->args[2])->val == 0
+                       && arg_is_const(op->args[3])
+                       && arg_info(op->args[3])->val == 0) {
                 /* Simplify LT/GE comparisons vs zero to a single compare
                    vs the high word of the input.  */
             do_brcond_high:
-                reset_all_temps(nb_temps);
+                bitmap_zero(temps_used.l, nb_temps);
                 op->opc = INDEX_op_brcond_i32;
-                args[0] = args[1];
-                args[1] = args[3];
-                args[2] = args[4];
-                args[3] = args[5];
-            } else if (args[4] == TCG_COND_EQ) {
+                op->args[0] = op->args[1];
+                op->args[1] = op->args[3];
+                op->args[2] = op->args[4];
+                op->args[3] = op->args[5];
+            } else if (op->args[4] == TCG_COND_EQ) {
                 /* Simplify EQ comparisons where one of the pairs
                    can be simplified.  */
                 tmp = do_constant_folding_cond(INDEX_op_brcond_i32,
-                                               args[0], args[2], TCG_COND_EQ);
+                                               op->args[0], op->args[2],
+                                               TCG_COND_EQ);
                 if (tmp == 0) {
                     goto do_brcond_false;
                 } else if (tmp == 1) {
                     goto do_brcond_high;
                 }
                 tmp = do_constant_folding_cond(INDEX_op_brcond_i32,
-                                               args[1], args[3], TCG_COND_EQ);
+                                               op->args[1], op->args[3],
+                                               TCG_COND_EQ);
                 if (tmp == 0) {
                     goto do_brcond_false;
                 } else if (tmp != 1) {
                     goto do_default;
                 }
             do_brcond_low:
-                reset_all_temps(nb_temps);
+                bitmap_zero(temps_used.l, nb_temps);
                 op->opc = INDEX_op_brcond_i32;
-                args[1] = args[2];
-                args[2] = args[4];
-                args[3] = args[5];
-            } else if (args[4] == TCG_COND_NE) {
+                op->args[1] = op->args[2];
+                op->args[2] = op->args[4];
+                op->args[3] = op->args[5];
+            } else if (op->args[4] == TCG_COND_NE) {
                 /* Simplify NE comparisons where one of the pairs
                    can be simplified.  */
                 tmp = do_constant_folding_cond(INDEX_op_brcond_i32,
-                                               args[0], args[2], TCG_COND_NE);
+                                               op->args[0], op->args[2],
+                                               TCG_COND_NE);
                 if (tmp == 0) {
                     goto do_brcond_high;
                 } else if (tmp == 1) {
                     goto do_brcond_true;
                 }
                 tmp = do_constant_folding_cond(INDEX_op_brcond_i32,
-                                               args[1], args[3], TCG_COND_NE);
+                                               op->args[1], op->args[3],
+                                               TCG_COND_NE);
                 if (tmp == 0) {
                     goto do_brcond_low;
                 } else if (tmp == 1) {
@@ -1231,59 +1360,65 @@ void tcg_optimize(TCGContext *s)
             break;
 
         case INDEX_op_setcond2_i32:
-            tmp = do_constant_folding_cond2(&args[1], &args[3], args[5]);
+            tmp = do_constant_folding_cond2(&op->args[1], &op->args[3],
+                                            op->args[5]);
             if (tmp != 2) {
             do_setcond_const:
-                tcg_opt_gen_movi(s, op, args, args[0], tmp);
-            } else if ((args[5] == TCG_COND_LT || args[5] == TCG_COND_GE)
-                       && temps[args[3]].state == TCG_TEMP_CONST
-                       && temps[args[4]].state == TCG_TEMP_CONST
-                       && temps[args[3]].val == 0
-                       && temps[args[4]].val == 0) {
+                tcg_opt_gen_movi(s, op, op->args[0], tmp);
+            } else if ((op->args[5] == TCG_COND_LT
+                        || op->args[5] == TCG_COND_GE)
+                       && arg_is_const(op->args[3])
+                       && arg_info(op->args[3])->val == 0
+                       && arg_is_const(op->args[4])
+                       && arg_info(op->args[4])->val == 0) {
                 /* Simplify LT/GE comparisons vs zero to a single compare
                    vs the high word of the input.  */
             do_setcond_high:
-                reset_temp(args[0]);
-                temps[args[0]].mask = 1;
+                reset_temp(op->args[0]);
+                arg_info(op->args[0])->mask = 1;
                 op->opc = INDEX_op_setcond_i32;
-                args[1] = args[2];
-                args[2] = args[4];
-                args[3] = args[5];
-            } else if (args[5] == TCG_COND_EQ) {
+                op->args[1] = op->args[2];
+                op->args[2] = op->args[4];
+                op->args[3] = op->args[5];
+            } else if (op->args[5] == TCG_COND_EQ) {
                 /* Simplify EQ comparisons where one of the pairs
                    can be simplified.  */
                 tmp = do_constant_folding_cond(INDEX_op_setcond_i32,
-                                               args[1], args[3], TCG_COND_EQ);
+                                               op->args[1], op->args[3],
+                                               TCG_COND_EQ);
                 if (tmp == 0) {
                     goto do_setcond_const;
                 } else if (tmp == 1) {
                     goto do_setcond_high;
                 }
                 tmp = do_constant_folding_cond(INDEX_op_setcond_i32,
-                                               args[2], args[4], TCG_COND_EQ);
+                                               op->args[2], op->args[4],
+                                               TCG_COND_EQ);
                 if (tmp == 0) {
                     goto do_setcond_high;
                 } else if (tmp != 1) {
                     goto do_default;
                 }
             do_setcond_low:
-                reset_temp(args[0]);
-                temps[args[0]].mask = 1;
+                reset_temp(op->args[0]);
+                arg_info(op->args[0])->mask = 1;
                 op->opc = INDEX_op_setcond_i32;
-                args[2] = args[3];
-                args[3] = args[5];
-            } else if (args[5] == TCG_COND_NE) {
+                op->args[2] = op->args[3];
+                op->args[3] = op->args[5];
+            } else if (op->args[5] == TCG_COND_NE) {
                 /* Simplify NE comparisons where one of the pairs
                    can be simplified.  */
                 tmp = do_constant_folding_cond(INDEX_op_setcond_i32,
-                                               args[1], args[3], TCG_COND_NE);
+                                               op->args[1], op->args[3],
+                                               TCG_COND_NE);
                 if (tmp == 0) {
                     goto do_setcond_high;
                 } else if (tmp == 1) {
                     goto do_setcond_const;
                 }
                 tmp = do_constant_folding_cond(INDEX_op_setcond_i32,
-                                               args[2], args[4], TCG_COND_NE);
+                                               op->args[2], op->args[4],
+                                               TCG_COND_NE);
                 if (tmp == 0) {
                     goto do_setcond_low;
                 } else if (tmp == 1) {
@@ -1296,10 +1431,12 @@ void tcg_optimize(TCGContext *s)
             break;
 
         case INDEX_op_call:
-            if (!(args[nb_oargs + nb_iargs + 1]
+            if (!(op->args[nb_oargs + nb_iargs + 1]
                   & (TCG_CALL_NO_READ_GLOBALS | TCG_CALL_NO_WRITE_GLOBALS))) {
                 for (i = 0; i < nb_globals; i++) {
-                    reset_temp(i);
+                    if (test_bit(i, temps_used.l)) {
+                        reset_ts(&s->temps[i]);
+                    }
                 }
             }
             goto do_reset_output;
@@ -1312,19 +1449,57 @@ void tcg_optimize(TCGContext *s)
                block, otherwise we only trash the output args.  "mask" is
                the non-zero bits mask for the first output arg.  */
             if (def->flags & TCG_OPF_BB_END) {
-                reset_all_temps(nb_temps);
+                bitmap_zero(temps_used.l, nb_temps);
             } else {
         do_reset_output:
                 for (i = 0; i < nb_oargs; i++) {
-                    reset_temp(args[i]);
+                    reset_temp(op->args[i]);
                     /* Save the corresponding known-zero bits mask for the
                        first output argument (only one supported so far). */
                     if (i == 0) {
-                        temps[args[i]].mask = mask;
+                        arg_info(op->args[i])->mask = mask;
                     }
                 }
             }
             break;
         }
+
+        /* Eliminate duplicate and redundant fence instructions.  */
+        if (prev_mb) {
+            switch (opc) {
+            case INDEX_op_mb:
+                /* Merge two barriers of the same type into one,
+                 * or a weaker barrier into a stronger one,
+                 * or two weaker barriers into a stronger one.
+                 *   mb X; mb Y => mb X|Y
+                 *   mb; strl => mb; st
+                 *   ldaq; mb => ld; mb
+                 *   ldaq; strl => ld; mb; st
+                 * Other combinations are also merged into a strong
+                 * barrier.  This is stricter than specified but for
+                 * the purposes of TCG is better than not optimizing.
+                 */
+                prev_mb->args[0] |= op->args[0];
+                tcg_op_remove(s, op);
+                break;
+
+            default:
+                /* Opcodes that end the block stop the optimization.  */
+                if ((def->flags & TCG_OPF_BB_END) == 0) {
+                    break;
+                }
+                /* fallthru */
+            case INDEX_op_qemu_ld_i32:
+            case INDEX_op_qemu_ld_i64:
+            case INDEX_op_qemu_st_i32:
+            case INDEX_op_qemu_st_i64:
+            case INDEX_op_call:
+                /* Opcodes that touch guest memory stop the optimization.  */
+                prev_mb = NULL;
+                break;
+            }
+        } else if (opc == INDEX_op_mb) {
+            prev_mb = op;
+        }
     }
 }