tcg/tcg-op.c

   1 /*
   2  * Tiny Code Generator for QEMU
   3  *
   4  * Copyright (c) 2008 Fabrice Bellard
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a copy
   7  * of this software and associated documentation files (the "Software"), to deal
   8  * in the Software without restriction, including without limitation the rights
   9  * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
  10  * copies of the Software, and to permit persons to whom the Software is
  11  * furnished to do so, subject to the following conditions:
  12  *
  13  * The above copyright notice and this permission notice shall be included in
  14  * all copies or substantial portions of the Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
  19  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  20  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  21  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  22  * THE SOFTWARE.
  23  */
  24
  25 #include "qemu/osdep.h"
  26 #include "qemu-common.h"
  27 #include "cpu.h"
  28 #include "exec/exec-all.h"
  29 #include "tcg.h"
  30 #include "tcg-op.h"
  31 #include "tcg-mo.h"
  32 #include "trace-tcg.h"
  33 #include "trace/mem.h"
  34
  35 /* Reduce the number of ifdefs below.  This assumes that all uses of
  36    TCGV_HIGH and TCGV_LOW are properly protected by a conditional that
  37    the compiler can eliminate.  */
  38 #if TCG_TARGET_REG_BITS == 64
  39 extern TCGv_i32 TCGV_LOW_link_error(TCGv_i64);
  40 extern TCGv_i32 TCGV_HIGH_link_error(TCGv_i64);
  41 #define TCGV_LOW  TCGV_LOW_link_error
  42 #define TCGV_HIGH TCGV_HIGH_link_error
  43 #endif
  44
  45 void tcg_gen_op1(TCGOpcode opc, TCGArg a1)
  46 {
  47     TCGOp *op = tcg_emit_op(opc);
  48     op->args[0] = a1;
  49 }
  50
  51 void tcg_gen_op2(TCGOpcode opc, TCGArg a1, TCGArg a2)
  52 {
  53     TCGOp *op = tcg_emit_op(opc);
  54     op->args[0] = a1;
  55     op->args[1] = a2;
  56 }
  57
  58 void tcg_gen_op3(TCGOpcode opc, TCGArg a1, TCGArg a2, TCGArg a3)
  59 {
  60     TCGOp *op = tcg_emit_op(opc);
  61     op->args[0] = a1;
  62     op->args[1] = a2;
  63     op->args[2] = a3;
  64 }
  65
  66 void tcg_gen_op4(TCGOpcode opc, TCGArg a1, TCGArg a2, TCGArg a3, TCGArg a4)
  67 {
  68     TCGOp *op = tcg_emit_op(opc);
  69     op->args[0] = a1;
  70     op->args[1] = a2;
  71     op->args[2] = a3;
  72     op->args[3] = a4;
  73 }
  74
  75 void tcg_gen_op5(TCGOpcode opc, TCGArg a1, TCGArg a2, TCGArg a3,
  76                  TCGArg a4, TCGArg a5)
  77 {
  78     TCGOp *op = tcg_emit_op(opc);
  79     op->args[0] = a1;
  80     op->args[1] = a2;
  81     op->args[2] = a3;
  82     op->args[3] = a4;
  83     op->args[4] = a5;
  84 }
  85
  86 void tcg_gen_op6(TCGOpcode opc, TCGArg a1, TCGArg a2, TCGArg a3,
  87                  TCGArg a4, TCGArg a5, TCGArg a6)
  88 {
  89     TCGOp *op = tcg_emit_op(opc);
  90     op->args[0] = a1;
  91     op->args[1] = a2;
  92     op->args[2] = a3;
  93     op->args[3] = a4;
  94     op->args[4] = a5;
  95     op->args[5] = a6;
  96 }
  97
  98 void tcg_gen_mb(TCGBar mb_type)
  99 {
 100     if (tcg_ctx->tb_cflags & CF_PARALLEL) {
 101         tcg_gen_op1(INDEX_op_mb, mb_type);
 102     }
 103 }
 104
 105 /* 32 bit ops */
 106
 107 void tcg_gen_addi_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
 108 {
 109     /* some cases can be optimized here */
 110     if (arg2 == 0) {
 111         tcg_gen_mov_i32(ret, arg1);
 112     } else {
 113         TCGv_i32 t0 = tcg_const_i32(arg2);
 114         tcg_gen_add_i32(ret, arg1, t0);
 115         tcg_temp_free_i32(t0);
 116     }
 117 }
 118
 119 void tcg_gen_subfi_i32(TCGv_i32 ret, int32_t arg1, TCGv_i32 arg2)
 120 {
 121     if (arg1 == 0 && TCG_TARGET_HAS_neg_i32) {
 122         /* Don't recurse with tcg_gen_neg_i32.  */
 123         tcg_gen_op2_i32(INDEX_op_neg_i32, ret, arg2);
 124     } else {
 125         TCGv_i32 t0 = tcg_const_i32(arg1);
 126         tcg_gen_sub_i32(ret, t0, arg2);
 127         tcg_temp_free_i32(t0);
 128     }
 129 }
 130
 131 void tcg_gen_subi_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
 132 {
 133     /* some cases can be optimized here */
 134     if (arg2 == 0) {
 135         tcg_gen_mov_i32(ret, arg1);
 136     } else {
 137         TCGv_i32 t0 = tcg_const_i32(arg2);
 138         tcg_gen_sub_i32(ret, arg1, t0);
 139         tcg_temp_free_i32(t0);
 140     }
 141 }
 142
 143 void tcg_gen_andi_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
 144 {
 145     TCGv_i32 t0;
 146     /* Some cases can be optimized here.  */
 147     switch (arg2) {
 148     case 0:
 149         tcg_gen_movi_i32(ret, 0);
 150         return;
 151     case -1:
 152         tcg_gen_mov_i32(ret, arg1);
 153         return;
 154     case 0xff:
 155         /* Don't recurse with tcg_gen_ext8u_i32.  */
 156         if (TCG_TARGET_HAS_ext8u_i32) {
 157             tcg_gen_op2_i32(INDEX_op_ext8u_i32, ret, arg1);
 158             return;
 159         }
 160         break;
 161     case 0xffff:
 162         if (TCG_TARGET_HAS_ext16u_i32) {
 163             tcg_gen_op2_i32(INDEX_op_ext16u_i32, ret, arg1);
 164             return;
 165         }
 166         break;
 167     }
 168     t0 = tcg_const_i32(arg2);
 169     tcg_gen_and_i32(ret, arg1, t0);
 170     tcg_temp_free_i32(t0);
 171 }
 172
 173 void tcg_gen_ori_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
 174 {
 175     /* Some cases can be optimized here.  */
 176     if (arg2 == -1) {
 177         tcg_gen_movi_i32(ret, -1);
 178     } else if (arg2 == 0) {
 179         tcg_gen_mov_i32(ret, arg1);
 180     } else {
 181         TCGv_i32 t0 = tcg_const_i32(arg2);
 182         tcg_gen_or_i32(ret, arg1, t0);
 183         tcg_temp_free_i32(t0);
 184     }
 185 }
 186
 187 void tcg_gen_xori_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
 188 {
 189     /* Some cases can be optimized here.  */
 190     if (arg2 == 0) {
 191         tcg_gen_mov_i32(ret, arg1);
 192     } else if (arg2 == -1 && TCG_TARGET_HAS_not_i32) {
 193         /* Don't recurse with tcg_gen_not_i32.  */
 194         tcg_gen_op2_i32(INDEX_op_not_i32, ret, arg1);
 195     } else {
 196         TCGv_i32 t0 = tcg_const_i32(arg2);
 197         tcg_gen_xor_i32(ret, arg1, t0);
 198         tcg_temp_free_i32(t0);
 199     }
 200 }
 201
 202 void tcg_gen_shli_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
 203 {
 204     tcg_debug_assert(arg2 >= 0 && arg2 < 32);
 205     if (arg2 == 0) {
 206         tcg_gen_mov_i32(ret, arg1);
 207     } else {
 208         TCGv_i32 t0 = tcg_const_i32(arg2);
 209         tcg_gen_shl_i32(ret, arg1, t0);
 210         tcg_temp_free_i32(t0);
 211     }
 212 }
 213
 214 void tcg_gen_shri_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
 215 {
 216     tcg_debug_assert(arg2 >= 0 && arg2 < 32);
 217     if (arg2 == 0) {
 218         tcg_gen_mov_i32(ret, arg1);
 219     } else {
 220         TCGv_i32 t0 = tcg_const_i32(arg2);
 221         tcg_gen_shr_i32(ret, arg1, t0);
 222         tcg_temp_free_i32(t0);
 223     }
 224 }
 225
 226 void tcg_gen_sari_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
 227 {
 228     tcg_debug_assert(arg2 >= 0 && arg2 < 32);
 229     if (arg2 == 0) {
 230         tcg_gen_mov_i32(ret, arg1);
 231     } else {
 232         TCGv_i32 t0 = tcg_const_i32(arg2);
 233         tcg_gen_sar_i32(ret, arg1, t0);
 234         tcg_temp_free_i32(t0);
 235     }
 236 }
 237
 238 void tcg_gen_brcond_i32(TCGCond cond, TCGv_i32 arg1, TCGv_i32 arg2, TCGLabel *l)
 239 {
 240     if (cond == TCG_COND_ALWAYS) {
 241         tcg_gen_br(l);
 242     } else if (cond != TCG_COND_NEVER) {
 243         tcg_gen_op4ii_i32(INDEX_op_brcond_i32, arg1, arg2, cond, label_arg(l));
 244     }
 245 }
 246
 247 void tcg_gen_brcondi_i32(TCGCond cond, TCGv_i32 arg1, int32_t arg2, TCGLabel *l)
 248 {
 249     if (cond == TCG_COND_ALWAYS) {
 250         tcg_gen_br(l);
 251     } else if (cond != TCG_COND_NEVER) {
 252         TCGv_i32 t0 = tcg_const_i32(arg2);
 253         tcg_gen_brcond_i32(cond, arg1, t0, l);
 254         tcg_temp_free_i32(t0);
 255     }
 256 }
 257
 258 void tcg_gen_setcond_i32(TCGCond cond, TCGv_i32 ret,
 259                          TCGv_i32 arg1, TCGv_i32 arg2)
 260 {
 261     if (cond == TCG_COND_ALWAYS) {
 262         tcg_gen_movi_i32(ret, 1);
 263     } else if (cond == TCG_COND_NEVER) {
 264         tcg_gen_movi_i32(ret, 0);
 265     } else {
 266         tcg_gen_op4i_i32(INDEX_op_setcond_i32, ret, arg1, arg2, cond);
 267     }
 268 }
 269
 270 void tcg_gen_setcondi_i32(TCGCond cond, TCGv_i32 ret,
 271                           TCGv_i32 arg1, int32_t arg2)
 272 {
 273     TCGv_i32 t0 = tcg_const_i32(arg2);
 274     tcg_gen_setcond_i32(cond, ret, arg1, t0);
 275     tcg_temp_free_i32(t0);
 276 }
 277
 278 void tcg_gen_muli_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
 279 {
 280     if (arg2 == 0) {
 281         tcg_gen_movi_i32(ret, 0);
 282     } else if (is_power_of_2(arg2)) {
 283         tcg_gen_shli_i32(ret, arg1, ctz32(arg2));
 284     } else {
 285         TCGv_i32 t0 = tcg_const_i32(arg2);
 286         tcg_gen_mul_i32(ret, arg1, t0);
 287         tcg_temp_free_i32(t0);
 288     }
 289 }
 290
 291 void tcg_gen_div_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
 292 {
 293     if (TCG_TARGET_HAS_div_i32) {
 294         tcg_gen_op3_i32(INDEX_op_div_i32, ret, arg1, arg2);
 295     } else if (TCG_TARGET_HAS_div2_i32) {
 296         TCGv_i32 t0 = tcg_temp_new_i32();
 297         tcg_gen_sari_i32(t0, arg1, 31);
 298         tcg_gen_op5_i32(INDEX_op_div2_i32, ret, t0, arg1, t0, arg2);
 299         tcg_temp_free_i32(t0);
 300     } else {
 301         gen_helper_div_i32(ret, arg1, arg2);
 302     }
 303 }
 304
 305 void tcg_gen_rem_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
 306 {
 307     if (TCG_TARGET_HAS_rem_i32) {
 308         tcg_gen_op3_i32(INDEX_op_rem_i32, ret, arg1, arg2);
 309     } else if (TCG_TARGET_HAS_div_i32) {
 310         TCGv_i32 t0 = tcg_temp_new_i32();
 311         tcg_gen_op3_i32(INDEX_op_div_i32, t0, arg1, arg2);
 312         tcg_gen_mul_i32(t0, t0, arg2);
 313         tcg_gen_sub_i32(ret, arg1, t0);
 314         tcg_temp_free_i32(t0);
 315     } else if (TCG_TARGET_HAS_div2_i32) {
 316         TCGv_i32 t0 = tcg_temp_new_i32();
 317         tcg_gen_sari_i32(t0, arg1, 31);
 318         tcg_gen_op5_i32(INDEX_op_div2_i32, t0, ret, arg1, t0, arg2);
 319         tcg_temp_free_i32(t0);
 320     } else {
 321         gen_helper_rem_i32(ret, arg1, arg2);
 322     }
 323 }
 324
 325 void tcg_gen_divu_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
 326 {
 327     if (TCG_TARGET_HAS_div_i32) {
 328         tcg_gen_op3_i32(INDEX_op_divu_i32, ret, arg1, arg2);
 329     } else if (TCG_TARGET_HAS_div2_i32) {
 330         TCGv_i32 t0 = tcg_temp_new_i32();
 331         tcg_gen_movi_i32(t0, 0);
 332         tcg_gen_op5_i32(INDEX_op_divu2_i32, ret, t0, arg1, t0, arg2);
 333         tcg_temp_free_i32(t0);
 334     } else {
 335         gen_helper_divu_i32(ret, arg1, arg2);
 336     }
 337 }
 338
 339 void tcg_gen_remu_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
 340 {
 341     if (TCG_TARGET_HAS_rem_i32) {
 342         tcg_gen_op3_i32(INDEX_op_remu_i32, ret, arg1, arg2);
 343     } else if (TCG_TARGET_HAS_div_i32) {
 344         TCGv_i32 t0 = tcg_temp_new_i32();
 345         tcg_gen_op3_i32(INDEX_op_divu_i32, t0, arg1, arg2);
 346         tcg_gen_mul_i32(t0, t0, arg2);
 347         tcg_gen_sub_i32(ret, arg1, t0);
 348         tcg_temp_free_i32(t0);
 349     } else if (TCG_TARGET_HAS_div2_i32) {
 350         TCGv_i32 t0 = tcg_temp_new_i32();
 351         tcg_gen_movi_i32(t0, 0);
 352         tcg_gen_op5_i32(INDEX_op_divu2_i32, t0, ret, arg1, t0, arg2);
 353         tcg_temp_free_i32(t0);
 354     } else {
 355         gen_helper_remu_i32(ret, arg1, arg2);
 356     }
 357 }
 358
 359 void tcg_gen_andc_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
 360 {
 361     if (TCG_TARGET_HAS_andc_i32) {
 362         tcg_gen_op3_i32(INDEX_op_andc_i32, ret, arg1, arg2);
 363     } else {
 364         TCGv_i32 t0 = tcg_temp_new_i32();
 365         tcg_gen_not_i32(t0, arg2);
 366         tcg_gen_and_i32(ret, arg1, t0);
 367         tcg_temp_free_i32(t0);
 368     }
 369 }
 370
 371 void tcg_gen_eqv_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
 372 {
 373     if (TCG_TARGET_HAS_eqv_i32) {
 374         tcg_gen_op3_i32(INDEX_op_eqv_i32, ret, arg1, arg2);
 375     } else {
 376         tcg_gen_xor_i32(ret, arg1, arg2);
 377         tcg_gen_not_i32(ret, ret);
 378     }
 379 }
 380
 381 void tcg_gen_nand_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
 382 {
 383     if (TCG_TARGET_HAS_nand_i32) {
 384         tcg_gen_op3_i32(INDEX_op_nand_i32, ret, arg1, arg2);
 385     } else {
 386         tcg_gen_and_i32(ret, arg1, arg2);
 387         tcg_gen_not_i32(ret, ret);
 388     }
 389 }
 390
 391 void tcg_gen_nor_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
 392 {
 393     if (TCG_TARGET_HAS_nor_i32) {
 394         tcg_gen_op3_i32(INDEX_op_nor_i32, ret, arg1, arg2);
 395     } else {
 396         tcg_gen_or_i32(ret, arg1, arg2);
 397         tcg_gen_not_i32(ret, ret);
 398     }
 399 }
 400
 401 void tcg_gen_orc_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
 402 {
 403     if (TCG_TARGET_HAS_orc_i32) {
 404         tcg_gen_op3_i32(INDEX_op_orc_i32, ret, arg1, arg2);
 405     } else {
 406         TCGv_i32 t0 = tcg_temp_new_i32();
 407         tcg_gen_not_i32(t0, arg2);
 408         tcg_gen_or_i32(ret, arg1, t0);
 409         tcg_temp_free_i32(t0);
 410     }
 411 }
 412
 413 void tcg_gen_clz_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
 414 {
 415     if (TCG_TARGET_HAS_clz_i32) {
 416         tcg_gen_op3_i32(INDEX_op_clz_i32, ret, arg1, arg2);
 417     } else if (TCG_TARGET_HAS_clz_i64) {
 418         TCGv_i64 t1 = tcg_temp_new_i64();
 419         TCGv_i64 t2 = tcg_temp_new_i64();
 420         tcg_gen_extu_i32_i64(t1, arg1);
 421         tcg_gen_extu_i32_i64(t2, arg2);
 422         tcg_gen_addi_i64(t2, t2, 32);
 423         tcg_gen_clz_i64(t1, t1, t2);
 424         tcg_gen_extrl_i64_i32(ret, t1);
 425         tcg_temp_free_i64(t1);
 426         tcg_temp_free_i64(t2);
 427         tcg_gen_subi_i32(ret, ret, 32);
 428     } else {
 429         gen_helper_clz_i32(ret, arg1, arg2);
 430     }
 431 }
 432
 433 void tcg_gen_clzi_i32(TCGv_i32 ret, TCGv_i32 arg1, uint32_t arg2)
 434 {
 435     TCGv_i32 t = tcg_const_i32(arg2);
 436     tcg_gen_clz_i32(ret, arg1, t);
 437     tcg_temp_free_i32(t);
 438 }
 439
 440 void tcg_gen_ctz_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
 441 {
 442     if (TCG_TARGET_HAS_ctz_i32) {
 443         tcg_gen_op3_i32(INDEX_op_ctz_i32, ret, arg1, arg2);
 444     } else if (TCG_TARGET_HAS_ctz_i64) {
 445         TCGv_i64 t1 = tcg_temp_new_i64();
 446         TCGv_i64 t2 = tcg_temp_new_i64();
 447         tcg_gen_extu_i32_i64(t1, arg1);
 448         tcg_gen_extu_i32_i64(t2, arg2);
 449         tcg_gen_ctz_i64(t1, t1, t2);
 450         tcg_gen_extrl_i64_i32(ret, t1);
 451         tcg_temp_free_i64(t1);
 452         tcg_temp_free_i64(t2);
 453     } else if (TCG_TARGET_HAS_ctpop_i32
 454                || TCG_TARGET_HAS_ctpop_i64
 455                || TCG_TARGET_HAS_clz_i32
 456                || TCG_TARGET_HAS_clz_i64) {
 457         TCGv_i32 z, t = tcg_temp_new_i32();
 458
 459         if (TCG_TARGET_HAS_ctpop_i32 || TCG_TARGET_HAS_ctpop_i64) {
 460             tcg_gen_subi_i32(t, arg1, 1);
 461             tcg_gen_andc_i32(t, t, arg1);
 462             tcg_gen_ctpop_i32(t, t);
 463         } else {
 464             /* Since all non-x86 hosts have clz(0) == 32, don't fight it.  */
 465             tcg_gen_neg_i32(t, arg1);
 466             tcg_gen_and_i32(t, t, arg1);
 467             tcg_gen_clzi_i32(t, t, 32);
 468             tcg_gen_xori_i32(t, t, 31);
 469         }
 470         z = tcg_const_i32(0);
 471         tcg_gen_movcond_i32(TCG_COND_EQ, ret, arg1, z, arg2, t);
 472         tcg_temp_free_i32(t);
 473         tcg_temp_free_i32(z);
 474     } else {
 475         gen_helper_ctz_i32(ret, arg1, arg2);
 476     }
 477 }
 478
 479 void tcg_gen_ctzi_i32(TCGv_i32 ret, TCGv_i32 arg1, uint32_t arg2)
 480 {
 481     if (!TCG_TARGET_HAS_ctz_i32 && TCG_TARGET_HAS_ctpop_i32 && arg2 == 32) {
 482         /* This equivalence has the advantage of not requiring a fixup.  */
 483         TCGv_i32 t = tcg_temp_new_i32();
 484         tcg_gen_subi_i32(t, arg1, 1);
 485         tcg_gen_andc_i32(t, t, arg1);
 486         tcg_gen_ctpop_i32(ret, t);
 487         tcg_temp_free_i32(t);
 488     } else {
 489         TCGv_i32 t = tcg_const_i32(arg2);
 490         tcg_gen_ctz_i32(ret, arg1, t);
 491         tcg_temp_free_i32(t);
 492     }
 493 }
 494
 495 void tcg_gen_clrsb_i32(TCGv_i32 ret, TCGv_i32 arg)
 496 {
 497     if (TCG_TARGET_HAS_clz_i32) {
 498         TCGv_i32 t = tcg_temp_new_i32();
 499         tcg_gen_sari_i32(t, arg, 31);
 500         tcg_gen_xor_i32(t, t, arg);
 501         tcg_gen_clzi_i32(t, t, 32);
 502         tcg_gen_subi_i32(ret, t, 1);
 503         tcg_temp_free_i32(t);
 504     } else {
 505         gen_helper_clrsb_i32(ret, arg);
 506     }
 507 }
 508
 509 void tcg_gen_ctpop_i32(TCGv_i32 ret, TCGv_i32 arg1)
 510 {
 511     if (TCG_TARGET_HAS_ctpop_i32) {
 512         tcg_gen_op2_i32(INDEX_op_ctpop_i32, ret, arg1);
 513     } else if (TCG_TARGET_HAS_ctpop_i64) {
 514         TCGv_i64 t = tcg_temp_new_i64();
 515         tcg_gen_extu_i32_i64(t, arg1);
 516         tcg_gen_ctpop_i64(t, t);
 517         tcg_gen_extrl_i64_i32(ret, t);
 518         tcg_temp_free_i64(t);
 519     } else {
 520         gen_helper_ctpop_i32(ret, arg1);
 521     }
 522 }
 523
 524 void tcg_gen_rotl_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
 525 {
 526     if (TCG_TARGET_HAS_rot_i32) {
 527         tcg_gen_op3_i32(INDEX_op_rotl_i32, ret, arg1, arg2);
 528     } else {
 529         TCGv_i32 t0, t1;
 530
 531         t0 = tcg_temp_new_i32();
 532         t1 = tcg_temp_new_i32();
 533         tcg_gen_shl_i32(t0, arg1, arg2);
 534         tcg_gen_subfi_i32(t1, 32, arg2);
 535         tcg_gen_shr_i32(t1, arg1, t1);
 536         tcg_gen_or_i32(ret, t0, t1);
 537         tcg_temp_free_i32(t0);
 538         tcg_temp_free_i32(t1);
 539     }
 540 }
 541
 542 void tcg_gen_rotli_i32(TCGv_i32 ret, TCGv_i32 arg1, unsigned arg2)
 543 {
 544     tcg_debug_assert(arg2 < 32);
 545     /* some cases can be optimized here */
 546     if (arg2 == 0) {
 547         tcg_gen_mov_i32(ret, arg1);
 548     } else if (TCG_TARGET_HAS_rot_i32) {
 549         TCGv_i32 t0 = tcg_const_i32(arg2);
 550         tcg_gen_rotl_i32(ret, arg1, t0);
 551         tcg_temp_free_i32(t0);
 552     } else {
 553         TCGv_i32 t0, t1;
 554         t0 = tcg_temp_new_i32();
 555         t1 = tcg_temp_new_i32();
 556         tcg_gen_shli_i32(t0, arg1, arg2);
 557         tcg_gen_shri_i32(t1, arg1, 32 - arg2);
 558         tcg_gen_or_i32(ret, t0, t1);
 559         tcg_temp_free_i32(t0);
 560         tcg_temp_free_i32(t1);
 561     }
 562 }
 563
 564 void tcg_gen_rotr_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
 565 {
 566     if (TCG_TARGET_HAS_rot_i32) {
 567         tcg_gen_op3_i32(INDEX_op_rotr_i32, ret, arg1, arg2);
 568     } else {
 569         TCGv_i32 t0, t1;
 570
 571         t0 = tcg_temp_new_i32();
 572         t1 = tcg_temp_new_i32();
 573         tcg_gen_shr_i32(t0, arg1, arg2);
 574         tcg_gen_subfi_i32(t1, 32, arg2);
 575         tcg_gen_shl_i32(t1, arg1, t1);
 576         tcg_gen_or_i32(ret, t0, t1);
 577         tcg_temp_free_i32(t0);
 578         tcg_temp_free_i32(t1);
 579     }
 580 }
 581
 582 void tcg_gen_rotri_i32(TCGv_i32 ret, TCGv_i32 arg1, unsigned arg2)
 583 {
 584     tcg_debug_assert(arg2 < 32);
 585     /* some cases can be optimized here */
 586     if (arg2 == 0) {
 587         tcg_gen_mov_i32(ret, arg1);
 588     } else {
 589         tcg_gen_rotli_i32(ret, arg1, 32 - arg2);
 590     }
 591 }
 592
 593 void tcg_gen_deposit_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2,
 594                          unsigned int ofs, unsigned int len)
 595 {
 596     uint32_t mask;
 597     TCGv_i32 t1;
 598
 599     tcg_debug_assert(ofs < 32);
 600     tcg_debug_assert(len > 0);
 601     tcg_debug_assert(len <= 32);
 602     tcg_debug_assert(ofs + len <= 32);
 603
 604     if (len == 32) {
 605         tcg_gen_mov_i32(ret, arg2);
 606         return;
 607     }
 608     if (TCG_TARGET_HAS_deposit_i32 && TCG_TARGET_deposit_i32_valid(ofs, len)) {
 609         tcg_gen_op5ii_i32(INDEX_op_deposit_i32, ret, arg1, arg2, ofs, len);
 610         return;
 611     }
 612
 613     mask = (1u << len) - 1;
 614     t1 = tcg_temp_new_i32();
 615
 616     if (ofs + len < 32) {
 617         tcg_gen_andi_i32(t1, arg2, mask);
 618         tcg_gen_shli_i32(t1, t1, ofs);
 619     } else {
 620         tcg_gen_shli_i32(t1, arg2, ofs);
 621     }
 622     tcg_gen_andi_i32(ret, arg1, ~(mask << ofs));
 623     tcg_gen_or_i32(ret, ret, t1);
 624
 625     tcg_temp_free_i32(t1);
 626 }
 627
 628 void tcg_gen_deposit_z_i32(TCGv_i32 ret, TCGv_i32 arg,
 629                            unsigned int ofs, unsigned int len)
 630 {
 631     tcg_debug_assert(ofs < 32);
 632     tcg_debug_assert(len > 0);
 633     tcg_debug_assert(len <= 32);
 634     tcg_debug_assert(ofs + len <= 32);
 635
 636     if (ofs + len == 32) {
 637         tcg_gen_shli_i32(ret, arg, ofs);
 638     } else if (ofs == 0) {
 639         tcg_gen_andi_i32(ret, arg, (1u << len) - 1);
 640     } else if (TCG_TARGET_HAS_deposit_i32
 641                && TCG_TARGET_deposit_i32_valid(ofs, len)) {
 642         TCGv_i32 zero = tcg_const_i32(0);
 643         tcg_gen_op5ii_i32(INDEX_op_deposit_i32, ret, zero, arg, ofs, len);
 644         tcg_temp_free_i32(zero);
 645     } else {
 646         /* To help two-operand hosts we prefer to zero-extend first,
 647            which allows ARG to stay live.  */
 648         switch (len) {
 649         case 16:
 650             if (TCG_TARGET_HAS_ext16u_i32) {
 651                 tcg_gen_ext16u_i32(ret, arg);
 652                 tcg_gen_shli_i32(ret, ret, ofs);
 653                 return;
 654             }
 655             break;
 656         case 8:
 657             if (TCG_TARGET_HAS_ext8u_i32) {
 658                 tcg_gen_ext8u_i32(ret, arg);
 659                 tcg_gen_shli_i32(ret, ret, ofs);
 660                 return;
 661             }
 662             break;
 663         }
 664         /* Otherwise prefer zero-extension over AND for code size.  */
 665         switch (ofs + len) {
 666         case 16:
 667             if (TCG_TARGET_HAS_ext16u_i32) {
 668                 tcg_gen_shli_i32(ret, arg, ofs);
 669                 tcg_gen_ext16u_i32(ret, ret);
 670                 return;
 671             }
 672             break;
 673         case 8:
 674             if (TCG_TARGET_HAS_ext8u_i32) {
 675                 tcg_gen_shli_i32(ret, arg, ofs);
 676                 tcg_gen_ext8u_i32(ret, ret);
 677                 return;
 678             }
 679             break;
 680         }
 681         tcg_gen_andi_i32(ret, arg, (1u << len) - 1);
 682         tcg_gen_shli_i32(ret, ret, ofs);
 683     }
 684 }
 685
 686 void tcg_gen_extract_i32(TCGv_i32 ret, TCGv_i32 arg,
 687                          unsigned int ofs, unsigned int len)
 688 {
 689     tcg_debug_assert(ofs < 32);
 690     tcg_debug_assert(len > 0);
 691     tcg_debug_assert(len <= 32);
 692     tcg_debug_assert(ofs + len <= 32);
 693
 694     /* Canonicalize certain special cases, even if extract is supported.  */
 695     if (ofs + len == 32) {
 696         tcg_gen_shri_i32(ret, arg, 32 - len);
 697         return;
 698     }
 699     if (ofs == 0) {
 700         tcg_gen_andi_i32(ret, arg, (1u << len) - 1);
 701         return;
 702     }
 703
 704     if (TCG_TARGET_HAS_extract_i32
 705         && TCG_TARGET_extract_i32_valid(ofs, len)) {
 706         tcg_gen_op4ii_i32(INDEX_op_extract_i32, ret, arg, ofs, len);
 707         return;
 708     }
 709
 710     /* Assume that zero-extension, if available, is cheaper than a shift.  */
 711     switch (ofs + len) {
 712     case 16:
 713         if (TCG_TARGET_HAS_ext16u_i32) {
 714             tcg_gen_ext16u_i32(ret, arg);
 715             tcg_gen_shri_i32(ret, ret, ofs);
 716             return;
 717         }
 718         break;
 719     case 8:
 720         if (TCG_TARGET_HAS_ext8u_i32) {
 721             tcg_gen_ext8u_i32(ret, arg);
 722             tcg_gen_shri_i32(ret, ret, ofs);
 723             return;
 724         }
 725         break;
 726     }
 727
 728     /* ??? Ideally we'd know what values are available for immediate AND.
 729        Assume that 8 bits are available, plus the special case of 16,
 730        so that we get ext8u, ext16u.  */
 731     switch (len) {
 732     case 1 ... 8: case 16:
 733         tcg_gen_shri_i32(ret, arg, ofs);
 734         tcg_gen_andi_i32(ret, ret, (1u << len) - 1);
 735         break;
 736     default:
 737         tcg_gen_shli_i32(ret, arg, 32 - len - ofs);
 738         tcg_gen_shri_i32(ret, ret, 32 - len);
 739         break;
 740     }
 741 }
 742
 743 void tcg_gen_sextract_i32(TCGv_i32 ret, TCGv_i32 arg,
 744                           unsigned int ofs, unsigned int len)
 745 {
 746     tcg_debug_assert(ofs < 32);
 747     tcg_debug_assert(len > 0);
 748     tcg_debug_assert(len <= 32);
 749     tcg_debug_assert(ofs + len <= 32);
 750
 751     /* Canonicalize certain special cases, even if extract is supported.  */
 752     if (ofs + len == 32) {
 753         tcg_gen_sari_i32(ret, arg, 32 - len);
 754         return;
 755     }
 756     if (ofs == 0) {
 757         switch (len) {
 758         case 16:
 759             tcg_gen_ext16s_i32(ret, arg);
 760             return;
 761         case 8:
 762             tcg_gen_ext8s_i32(ret, arg);
 763             return;
 764         }
 765     }
 766
 767     if (TCG_TARGET_HAS_sextract_i32
 768         && TCG_TARGET_extract_i32_valid(ofs, len)) {
 769         tcg_gen_op4ii_i32(INDEX_op_sextract_i32, ret, arg, ofs, len);
 770         return;
 771     }
 772
 773     /* Assume that sign-extension, if available, is cheaper than a shift.  */
 774     switch (ofs + len) {
 775     case 16:
 776         if (TCG_TARGET_HAS_ext16s_i32) {
 777             tcg_gen_ext16s_i32(ret, arg);
 778             tcg_gen_sari_i32(ret, ret, ofs);
 779             return;
 780         }
 781         break;
 782     case 8:
 783         if (TCG_TARGET_HAS_ext8s_i32) {
 784             tcg_gen_ext8s_i32(ret, arg);
 785             tcg_gen_sari_i32(ret, ret, ofs);
 786             return;
 787         }
 788         break;
 789     }
 790     switch (len) {
 791     case 16:
 792         if (TCG_TARGET_HAS_ext16s_i32) {
 793             tcg_gen_shri_i32(ret, arg, ofs);
 794             tcg_gen_ext16s_i32(ret, ret);
 795             return;
 796         }
 797         break;
 798     case 8:
 799         if (TCG_TARGET_HAS_ext8s_i32) {
 800             tcg_gen_shri_i32(ret, arg, ofs);
 801             tcg_gen_ext8s_i32(ret, ret);
 802             return;
 803         }
 804         break;
 805     }
 806
 807     tcg_gen_shli_i32(ret, arg, 32 - len - ofs);
 808     tcg_gen_sari_i32(ret, ret, 32 - len);
 809 }
 810
 811 void tcg_gen_movcond_i32(TCGCond cond, TCGv_i32 ret, TCGv_i32 c1,
 812                          TCGv_i32 c2, TCGv_i32 v1, TCGv_i32 v2)
 813 {
 814     if (cond == TCG_COND_ALWAYS) {
 815         tcg_gen_mov_i32(ret, v1);
 816     } else if (cond == TCG_COND_NEVER) {
 817         tcg_gen_mov_i32(ret, v2);
 818     } else if (TCG_TARGET_HAS_movcond_i32) {
 819         tcg_gen_op6i_i32(INDEX_op_movcond_i32, ret, c1, c2, v1, v2, cond);
 820     } else {
 821         TCGv_i32 t0 = tcg_temp_new_i32();
 822         TCGv_i32 t1 = tcg_temp_new_i32();
 823         tcg_gen_setcond_i32(cond, t0, c1, c2);
 824         tcg_gen_neg_i32(t0, t0);
 825         tcg_gen_and_i32(t1, v1, t0);
 826         tcg_gen_andc_i32(ret, v2, t0);
 827         tcg_gen_or_i32(ret, ret, t1);
 828         tcg_temp_free_i32(t0);
 829         tcg_temp_free_i32(t1);
 830     }
 831 }
 832
 833 void tcg_gen_add2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 al,
 834                       TCGv_i32 ah, TCGv_i32 bl, TCGv_i32 bh)
 835 {
 836     if (TCG_TARGET_HAS_add2_i32) {
 837         tcg_gen_op6_i32(INDEX_op_add2_i32, rl, rh, al, ah, bl, bh);
 838     } else {
 839         TCGv_i64 t0 = tcg_temp_new_i64();
 840         TCGv_i64 t1 = tcg_temp_new_i64();
 841         tcg_gen_concat_i32_i64(t0, al, ah);
 842         tcg_gen_concat_i32_i64(t1, bl, bh);
 843         tcg_gen_add_i64(t0, t0, t1);
 844         tcg_gen_extr_i64_i32(rl, rh, t0);
 845         tcg_temp_free_i64(t0);
 846         tcg_temp_free_i64(t1);
 847     }
 848 }
 849
 850 void tcg_gen_sub2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 al,
 851                       TCGv_i32 ah, TCGv_i32 bl, TCGv_i32 bh)
 852 {
 853     if (TCG_TARGET_HAS_sub2_i32) {
 854         tcg_gen_op6_i32(INDEX_op_sub2_i32, rl, rh, al, ah, bl, bh);
 855     } else {
 856         TCGv_i64 t0 = tcg_temp_new_i64();
 857         TCGv_i64 t1 = tcg_temp_new_i64();
 858         tcg_gen_concat_i32_i64(t0, al, ah);
 859         tcg_gen_concat_i32_i64(t1, bl, bh);
 860         tcg_gen_sub_i64(t0, t0, t1);
 861         tcg_gen_extr_i64_i32(rl, rh, t0);
 862         tcg_temp_free_i64(t0);
 863         tcg_temp_free_i64(t1);
 864     }
 865 }
 866
 867 void tcg_gen_mulu2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 arg1, TCGv_i32 arg2)
 868 {
 869     if (TCG_TARGET_HAS_mulu2_i32) {
 870         tcg_gen_op4_i32(INDEX_op_mulu2_i32, rl, rh, arg1, arg2);
 871     } else if (TCG_TARGET_HAS_muluh_i32) {
 872         TCGv_i32 t = tcg_temp_new_i32();
 873         tcg_gen_op3_i32(INDEX_op_mul_i32, t, arg1, arg2);
 874         tcg_gen_op3_i32(INDEX_op_muluh_i32, rh, arg1, arg2);
 875         tcg_gen_mov_i32(rl, t);
 876         tcg_temp_free_i32(t);
 877     } else {
 878         TCGv_i64 t0 = tcg_temp_new_i64();
 879         TCGv_i64 t1 = tcg_temp_new_i64();
 880         tcg_gen_extu_i32_i64(t0, arg1);
 881         tcg_gen_extu_i32_i64(t1, arg2);
 882         tcg_gen_mul_i64(t0, t0, t1);
 883         tcg_gen_extr_i64_i32(rl, rh, t0);
 884         tcg_temp_free_i64(t0);
 885         tcg_temp_free_i64(t1);
 886     }
 887 }
 888
 889 void tcg_gen_muls2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 arg1, TCGv_i32 arg2)
 890 {
 891     if (TCG_TARGET_HAS_muls2_i32) {
 892         tcg_gen_op4_i32(INDEX_op_muls2_i32, rl, rh, arg1, arg2);
 893     } else if (TCG_TARGET_HAS_mulsh_i32) {
 894         TCGv_i32 t = tcg_temp_new_i32();
 895         tcg_gen_op3_i32(INDEX_op_mul_i32, t, arg1, arg2);
 896         tcg_gen_op3_i32(INDEX_op_mulsh_i32, rh, arg1, arg2);
 897         tcg_gen_mov_i32(rl, t);
 898         tcg_temp_free_i32(t);
 899     } else if (TCG_TARGET_REG_BITS == 32) {
 900         TCGv_i32 t0 = tcg_temp_new_i32();
 901         TCGv_i32 t1 = tcg_temp_new_i32();
 902         TCGv_i32 t2 = tcg_temp_new_i32();
 903         TCGv_i32 t3 = tcg_temp_new_i32();
 904         tcg_gen_mulu2_i32(t0, t1, arg1, arg2);
 905         /* Adjust for negative inputs.  */
 906         tcg_gen_sari_i32(t2, arg1, 31);
 907         tcg_gen_sari_i32(t3, arg2, 31);
 908         tcg_gen_and_i32(t2, t2, arg2);
 909         tcg_gen_and_i32(t3, t3, arg1);
 910         tcg_gen_sub_i32(rh, t1, t2);
 911         tcg_gen_sub_i32(rh, rh, t3);
 912         tcg_gen_mov_i32(rl, t0);
 913         tcg_temp_free_i32(t0);
 914         tcg_temp_free_i32(t1);
 915         tcg_temp_free_i32(t2);
 916         tcg_temp_free_i32(t3);
 917     } else {
 918         TCGv_i64 t0 = tcg_temp_new_i64();
 919         TCGv_i64 t1 = tcg_temp_new_i64();
 920         tcg_gen_ext_i32_i64(t0, arg1);
 921         tcg_gen_ext_i32_i64(t1, arg2);
 922         tcg_gen_mul_i64(t0, t0, t1);
 923         tcg_gen_extr_i64_i32(rl, rh, t0);
 924         tcg_temp_free_i64(t0);
 925         tcg_temp_free_i64(t1);
 926     }
 927 }
 928
 929 void tcg_gen_mulsu2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 arg1, TCGv_i32 arg2)
 930 {
 931     if (TCG_TARGET_REG_BITS == 32) {
 932         TCGv_i32 t0 = tcg_temp_new_i32();
 933         TCGv_i32 t1 = tcg_temp_new_i32();
 934         TCGv_i32 t2 = tcg_temp_new_i32();
 935         tcg_gen_mulu2_i32(t0, t1, arg1, arg2);
 936         /* Adjust for negative input for the signed arg1.  */
 937         tcg_gen_sari_i32(t2, arg1, 31);
 938         tcg_gen_and_i32(t2, t2, arg2);
 939         tcg_gen_sub_i32(rh, t1, t2);
 940         tcg_gen_mov_i32(rl, t0);
 941         tcg_temp_free_i32(t0);
 942         tcg_temp_free_i32(t1);
 943         tcg_temp_free_i32(t2);
 944     } else {
 945         TCGv_i64 t0 = tcg_temp_new_i64();
 946         TCGv_i64 t1 = tcg_temp_new_i64();
 947         tcg_gen_ext_i32_i64(t0, arg1);
 948         tcg_gen_extu_i32_i64(t1, arg2);
 949         tcg_gen_mul_i64(t0, t0, t1);
 950         tcg_gen_extr_i64_i32(rl, rh, t0);
 951         tcg_temp_free_i64(t0);
 952         tcg_temp_free_i64(t1);
 953     }
 954 }
 955
 956 void tcg_gen_ext8s_i32(TCGv_i32 ret, TCGv_i32 arg)
 957 {
 958     if (TCG_TARGET_HAS_ext8s_i32) {
 959         tcg_gen_op2_i32(INDEX_op_ext8s_i32, ret, arg);
 960     } else {
 961         tcg_gen_shli_i32(ret, arg, 24);
 962         tcg_gen_sari_i32(ret, ret, 24);
 963     }
 964 }
 965
 966 void tcg_gen_ext16s_i32(TCGv_i32 ret, TCGv_i32 arg)
 967 {
 968     if (TCG_TARGET_HAS_ext16s_i32) {
 969         tcg_gen_op2_i32(INDEX_op_ext16s_i32, ret, arg);
 970     } else {
 971         tcg_gen_shli_i32(ret, arg, 16);
 972         tcg_gen_sari_i32(ret, ret, 16);
 973     }
 974 }
 975
 976 void tcg_gen_ext8u_i32(TCGv_i32 ret, TCGv_i32 arg)
 977 {
 978     if (TCG_TARGET_HAS_ext8u_i32) {
 979         tcg_gen_op2_i32(INDEX_op_ext8u_i32, ret, arg);
 980     } else {
 981         tcg_gen_andi_i32(ret, arg, 0xffu);
 982     }
 983 }
 984
 985 void tcg_gen_ext16u_i32(TCGv_i32 ret, TCGv_i32 arg)
 986 {
 987     if (TCG_TARGET_HAS_ext16u_i32) {
 988         tcg_gen_op2_i32(INDEX_op_ext16u_i32, ret, arg);
 989     } else {
 990         tcg_gen_andi_i32(ret, arg, 0xffffu);
 991     }
 992 }
 993
 994 /* Note: we assume the two high bytes are set to zero */
 995 void tcg_gen_bswap16_i32(TCGv_i32 ret, TCGv_i32 arg)
 996 {
 997     if (TCG_TARGET_HAS_bswap16_i32) {
 998         tcg_gen_op2_i32(INDEX_op_bswap16_i32, ret, arg);
 999     } else {
1000         TCGv_i32 t0 = tcg_temp_new_i32();
1001
1002         tcg_gen_ext8u_i32(t0, arg);
1003         tcg_gen_shli_i32(t0, t0, 8);
1004         tcg_gen_shri_i32(ret, arg, 8);
1005         tcg_gen_or_i32(ret, ret, t0);
1006         tcg_temp_free_i32(t0);
1007     }
1008 }
1009
1010 void tcg_gen_bswap32_i32(TCGv_i32 ret, TCGv_i32 arg)
1011 {
1012     if (TCG_TARGET_HAS_bswap32_i32) {
1013         tcg_gen_op2_i32(INDEX_op_bswap32_i32, ret, arg);
1014     } else {
1015         TCGv_i32 t0, t1;
1016         t0 = tcg_temp_new_i32();
1017         t1 = tcg_temp_new_i32();
1018
1019         tcg_gen_shli_i32(t0, arg, 24);
1020
1021         tcg_gen_andi_i32(t1, arg, 0x0000ff00);
1022         tcg_gen_shli_i32(t1, t1, 8);
1023         tcg_gen_or_i32(t0, t0, t1);
1024
1025         tcg_gen_shri_i32(t1, arg, 8);
1026         tcg_gen_andi_i32(t1, t1, 0x0000ff00);
1027         tcg_gen_or_i32(t0, t0, t1);
1028
1029         tcg_gen_shri_i32(t1, arg, 24);
1030         tcg_gen_or_i32(ret, t0, t1);
1031         tcg_temp_free_i32(t0);
1032         tcg_temp_free_i32(t1);
1033     }
1034 }
1035
1036 void tcg_gen_smin_i32(TCGv_i32 ret, TCGv_i32 a, TCGv_i32 b)
1037 {
1038     tcg_gen_movcond_i32(TCG_COND_LT, ret, a, b, a, b);
1039 }
1040
1041 void tcg_gen_umin_i32(TCGv_i32 ret, TCGv_i32 a, TCGv_i32 b)
1042 {
1043     tcg_gen_movcond_i32(TCG_COND_LTU, ret, a, b, a, b);
1044 }
1045
1046 void tcg_gen_smax_i32(TCGv_i32 ret, TCGv_i32 a, TCGv_i32 b)
1047 {
1048     tcg_gen_movcond_i32(TCG_COND_LT, ret, a, b, b, a);
1049 }
1050
1051 void tcg_gen_umax_i32(TCGv_i32 ret, TCGv_i32 a, TCGv_i32 b)
1052 {
1053     tcg_gen_movcond_i32(TCG_COND_LTU, ret, a, b, b, a);
1054 }
1055
1056 /* 64-bit ops */
1057
1058 #if TCG_TARGET_REG_BITS == 32
1059 /* These are all inline for TCG_TARGET_REG_BITS == 64.  */
1060
1061 void tcg_gen_discard_i64(TCGv_i64 arg)
1062 {
1063     tcg_gen_discard_i32(TCGV_LOW(arg));
1064     tcg_gen_discard_i32(TCGV_HIGH(arg));
1065 }
1066
1067 void tcg_gen_mov_i64(TCGv_i64 ret, TCGv_i64 arg)
1068 {
1069     tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1070     tcg_gen_mov_i32(TCGV_HIGH(ret), TCGV_HIGH(arg));
1071 }
1072
1073 void tcg_gen_movi_i64(TCGv_i64 ret, int64_t arg)
1074 {
1075     tcg_gen_movi_i32(TCGV_LOW(ret), arg);
1076     tcg_gen_movi_i32(TCGV_HIGH(ret), arg >> 32);
1077 }
1078
1079 void tcg_gen_ld8u_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
1080 {
1081     tcg_gen_ld8u_i32(TCGV_LOW(ret), arg2, offset);
1082     tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1083 }
1084
1085 void tcg_gen_ld8s_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
1086 {
1087     tcg_gen_ld8s_i32(TCGV_LOW(ret), arg2, offset);
1088     tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
1089 }
1090
1091 void tcg_gen_ld16u_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
1092 {
1093     tcg_gen_ld16u_i32(TCGV_LOW(ret), arg2, offset);
1094     tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1095 }
1096
1097 void tcg_gen_ld16s_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
1098 {
1099     tcg_gen_ld16s_i32(TCGV_LOW(ret), arg2, offset);
1100     tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
1101 }
1102
1103 void tcg_gen_ld32u_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
1104 {
1105     tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset);
1106     tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1107 }
1108
1109 void tcg_gen_ld32s_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
1110 {
1111     tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset);
1112     tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
1113 }
1114
1115 void tcg_gen_ld_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
1116 {
1117     /* Since arg2 and ret have different types,
1118        they cannot be the same temporary */
1119 #ifdef HOST_WORDS_BIGENDIAN
1120     tcg_gen_ld_i32(TCGV_HIGH(ret), arg2, offset);
1121     tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset + 4);
1122 #else
1123     tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset);
1124     tcg_gen_ld_i32(TCGV_HIGH(ret), arg2, offset + 4);
1125 #endif
1126 }
1127
1128 void tcg_gen_st_i64(TCGv_i64 arg1, TCGv_ptr arg2, tcg_target_long offset)
1129 {
1130 #ifdef HOST_WORDS_BIGENDIAN
1131     tcg_gen_st_i32(TCGV_HIGH(arg1), arg2, offset);
1132     tcg_gen_st_i32(TCGV_LOW(arg1), arg2, offset + 4);
1133 #else
1134     tcg_gen_st_i32(TCGV_LOW(arg1), arg2, offset);
1135     tcg_gen_st_i32(TCGV_HIGH(arg1), arg2, offset + 4);
1136 #endif
1137 }
1138
1139 void tcg_gen_and_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1140 {
1141     tcg_gen_and_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
1142     tcg_gen_and_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
1143 }
1144
1145 void tcg_gen_or_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1146 {
1147     tcg_gen_or_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
1148     tcg_gen_or_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
1149 }
1150
1151 void tcg_gen_xor_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1152 {
1153     tcg_gen_xor_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
1154     tcg_gen_xor_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
1155 }
1156
1157 void tcg_gen_shl_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1158 {
1159     gen_helper_shl_i64(ret, arg1, arg2);
1160 }
1161
1162 void tcg_gen_shr_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1163 {
1164     gen_helper_shr_i64(ret, arg1, arg2);
1165 }
1166
1167 void tcg_gen_sar_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1168 {
1169     gen_helper_sar_i64(ret, arg1, arg2);
1170 }
1171
1172 void tcg_gen_mul_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1173 {
1174     TCGv_i64 t0;
1175     TCGv_i32 t1;
1176
1177     t0 = tcg_temp_new_i64();
1178     t1 = tcg_temp_new_i32();
1179
1180     tcg_gen_mulu2_i32(TCGV_LOW(t0), TCGV_HIGH(t0),
1181                       TCGV_LOW(arg1), TCGV_LOW(arg2));
1182
1183     tcg_gen_mul_i32(t1, TCGV_LOW(arg1), TCGV_HIGH(arg2));
1184     tcg_gen_add_i32(TCGV_HIGH(t0), TCGV_HIGH(t0), t1);
1185     tcg_gen_mul_i32(t1, TCGV_HIGH(arg1), TCGV_LOW(arg2));
1186     tcg_gen_add_i32(TCGV_HIGH(t0), TCGV_HIGH(t0), t1);
1187
1188     tcg_gen_mov_i64(ret, t0);
1189     tcg_temp_free_i64(t0);
1190     tcg_temp_free_i32(t1);
1191 }
1192 #endif /* TCG_TARGET_REG_SIZE == 32 */
1193
1194 void tcg_gen_addi_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
1195 {
1196     /* some cases can be optimized here */
1197     if (arg2 == 0) {
1198         tcg_gen_mov_i64(ret, arg1);
1199     } else {
1200         TCGv_i64 t0 = tcg_const_i64(arg2);
1201         tcg_gen_add_i64(ret, arg1, t0);
1202         tcg_temp_free_i64(t0);
1203     }
1204 }
1205
1206 void tcg_gen_subfi_i64(TCGv_i64 ret, int64_t arg1, TCGv_i64 arg2)
1207 {
1208     if (arg1 == 0 && TCG_TARGET_HAS_neg_i64) {
1209         /* Don't recurse with tcg_gen_neg_i64.  */
1210         tcg_gen_op2_i64(INDEX_op_neg_i64, ret, arg2);
1211     } else {
1212         TCGv_i64 t0 = tcg_const_i64(arg1);
1213         tcg_gen_sub_i64(ret, t0, arg2);
1214         tcg_temp_free_i64(t0);
1215     }
1216 }
1217
1218 void tcg_gen_subi_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
1219 {
1220     /* some cases can be optimized here */
1221     if (arg2 == 0) {
1222         tcg_gen_mov_i64(ret, arg1);
1223     } else {
1224         TCGv_i64 t0 = tcg_const_i64(arg2);
1225         tcg_gen_sub_i64(ret, arg1, t0);
1226         tcg_temp_free_i64(t0);
1227     }
1228 }
1229
1230 void tcg_gen_andi_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
1231 {
1232     TCGv_i64 t0;
1233
1234     if (TCG_TARGET_REG_BITS == 32) {
1235         tcg_gen_andi_i32(TCGV_LOW(ret), TCGV_LOW(arg1), arg2);
1236         tcg_gen_andi_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), arg2 >> 32);
1237         return;
1238     }
1239
1240     /* Some cases can be optimized here.  */
1241     switch (arg2) {
1242     case 0:
1243         tcg_gen_movi_i64(ret, 0);
1244         return;
1245     case -1:
1246         tcg_gen_mov_i64(ret, arg1);
1247         return;
1248     case 0xff:
1249         /* Don't recurse with tcg_gen_ext8u_i64.  */
1250         if (TCG_TARGET_HAS_ext8u_i64) {
1251             tcg_gen_op2_i64(INDEX_op_ext8u_i64, ret, arg1);
1252             return;
1253         }
1254         break;
1255     case 0xffff:
1256         if (TCG_TARGET_HAS_ext16u_i64) {
1257             tcg_gen_op2_i64(INDEX_op_ext16u_i64, ret, arg1);
1258             return;
1259         }
1260         break;
1261     case 0xffffffffu:
1262         if (TCG_TARGET_HAS_ext32u_i64) {
1263             tcg_gen_op2_i64(INDEX_op_ext32u_i64, ret, arg1);
1264             return;
1265         }
1266         break;
1267     }
1268     t0 = tcg_const_i64(arg2);
1269     tcg_gen_and_i64(ret, arg1, t0);
1270     tcg_temp_free_i64(t0);
1271 }
1272
1273 void tcg_gen_ori_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
1274 {
1275     if (TCG_TARGET_REG_BITS == 32) {
1276         tcg_gen_ori_i32(TCGV_LOW(ret), TCGV_LOW(arg1), arg2);
1277         tcg_gen_ori_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), arg2 >> 32);
1278         return;
1279     }
1280     /* Some cases can be optimized here.  */
1281     if (arg2 == -1) {
1282         tcg_gen_movi_i64(ret, -1);
1283     } else if (arg2 == 0) {
1284         tcg_gen_mov_i64(ret, arg1);
1285     } else {
1286         TCGv_i64 t0 = tcg_const_i64(arg2);
1287         tcg_gen_or_i64(ret, arg1, t0);
1288         tcg_temp_free_i64(t0);
1289     }
1290 }
1291
1292 void tcg_gen_xori_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
1293 {
1294     if (TCG_TARGET_REG_BITS == 32) {
1295         tcg_gen_xori_i32(TCGV_LOW(ret), TCGV_LOW(arg1), arg2);
1296         tcg_gen_xori_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), arg2 >> 32);
1297         return;
1298     }
1299     /* Some cases can be optimized here.  */
1300     if (arg2 == 0) {
1301         tcg_gen_mov_i64(ret, arg1);
1302     } else if (arg2 == -1 && TCG_TARGET_HAS_not_i64) {
1303         /* Don't recurse with tcg_gen_not_i64.  */
1304         tcg_gen_op2_i64(INDEX_op_not_i64, ret, arg1);
1305     } else {
1306         TCGv_i64 t0 = tcg_const_i64(arg2);
1307         tcg_gen_xor_i64(ret, arg1, t0);
1308         tcg_temp_free_i64(t0);
1309     }
1310 }
1311
1312 static inline void tcg_gen_shifti_i64(TCGv_i64 ret, TCGv_i64 arg1,
1313                                       unsigned c, bool right, bool arith)
1314 {
1315     tcg_debug_assert(c < 64);
1316     if (c == 0) {
1317         tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg1));
1318         tcg_gen_mov_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1));
1319     } else if (c >= 32) {
1320         c -= 32;
1321         if (right) {
1322             if (arith) {
1323                 tcg_gen_sari_i32(TCGV_LOW(ret), TCGV_HIGH(arg1), c);
1324                 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), 31);
1325             } else {
1326                 tcg_gen_shri_i32(TCGV_LOW(ret), TCGV_HIGH(arg1), c);
1327                 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1328             }
1329         } else {
1330             tcg_gen_shli_i32(TCGV_HIGH(ret), TCGV_LOW(arg1), c);
1331             tcg_gen_movi_i32(TCGV_LOW(ret), 0);
1332         }
1333     } else {
1334         TCGv_i32 t0, t1;
1335
1336         t0 = tcg_temp_new_i32();
1337         t1 = tcg_temp_new_i32();
1338         if (right) {
1339             tcg_gen_shli_i32(t0, TCGV_HIGH(arg1), 32 - c);
1340             if (arith) {
1341                 tcg_gen_sari_i32(t1, TCGV_HIGH(arg1), c);
1342             } else {
1343                 tcg_gen_shri_i32(t1, TCGV_HIGH(arg1), c);
1344             }
1345             tcg_gen_shri_i32(TCGV_LOW(ret), TCGV_LOW(arg1), c);
1346             tcg_gen_or_i32(TCGV_LOW(ret), TCGV_LOW(ret), t0);
1347             tcg_gen_mov_i32(TCGV_HIGH(ret), t1);
1348         } else {
1349             tcg_gen_shri_i32(t0, TCGV_LOW(arg1), 32 - c);
1350             /* Note: ret can be the same as arg1, so we use t1 */
1351             tcg_gen_shli_i32(t1, TCGV_LOW(arg1), c);
1352             tcg_gen_shli_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), c);
1353             tcg_gen_or_i32(TCGV_HIGH(ret), TCGV_HIGH(ret), t0);
1354             tcg_gen_mov_i32(TCGV_LOW(ret), t1);
1355         }
1356         tcg_temp_free_i32(t0);
1357         tcg_temp_free_i32(t1);
1358     }
1359 }
1360
1361 void tcg_gen_shli_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
1362 {
1363     tcg_debug_assert(arg2 >= 0 && arg2 < 64);
1364     if (TCG_TARGET_REG_BITS == 32) {
1365         tcg_gen_shifti_i64(ret, arg1, arg2, 0, 0);
1366     } else if (arg2 == 0) {
1367         tcg_gen_mov_i64(ret, arg1);
1368     } else {
1369         TCGv_i64 t0 = tcg_const_i64(arg2);
1370         tcg_gen_shl_i64(ret, arg1, t0);
1371         tcg_temp_free_i64(t0);
1372     }
1373 }
1374
1375 void tcg_gen_shri_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
1376 {
1377     tcg_debug_assert(arg2 >= 0 && arg2 < 64);
1378     if (TCG_TARGET_REG_BITS == 32) {
1379         tcg_gen_shifti_i64(ret, arg1, arg2, 1, 0);
1380     } else if (arg2 == 0) {
1381         tcg_gen_mov_i64(ret, arg1);
1382     } else {
1383         TCGv_i64 t0 = tcg_const_i64(arg2);
1384         tcg_gen_shr_i64(ret, arg1, t0);
1385         tcg_temp_free_i64(t0);
1386     }
1387 }
1388
1389 void tcg_gen_sari_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
1390 {
1391     tcg_debug_assert(arg2 >= 0 && arg2 < 64);
1392     if (TCG_TARGET_REG_BITS == 32) {
1393         tcg_gen_shifti_i64(ret, arg1, arg2, 1, 1);
1394     } else if (arg2 == 0) {
1395         tcg_gen_mov_i64(ret, arg1);
1396     } else {
1397         TCGv_i64 t0 = tcg_const_i64(arg2);
1398         tcg_gen_sar_i64(ret, arg1, t0);
1399         tcg_temp_free_i64(t0);
1400     }
1401 }
1402
1403 void tcg_gen_brcond_i64(TCGCond cond, TCGv_i64 arg1, TCGv_i64 arg2, TCGLabel *l)
1404 {
1405     if (cond == TCG_COND_ALWAYS) {
1406         tcg_gen_br(l);
1407     } else if (cond != TCG_COND_NEVER) {
1408         if (TCG_TARGET_REG_BITS == 32) {
1409             tcg_gen_op6ii_i32(INDEX_op_brcond2_i32, TCGV_LOW(arg1),
1410                               TCGV_HIGH(arg1), TCGV_LOW(arg2),
1411                               TCGV_HIGH(arg2), cond, label_arg(l));
1412         } else {
1413             tcg_gen_op4ii_i64(INDEX_op_brcond_i64, arg1, arg2, cond,
1414                               label_arg(l));
1415         }
1416     }
1417 }
1418
1419 void tcg_gen_brcondi_i64(TCGCond cond, TCGv_i64 arg1, int64_t arg2, TCGLabel *l)
1420 {
1421     if (cond == TCG_COND_ALWAYS) {
1422         tcg_gen_br(l);
1423     } else if (cond != TCG_COND_NEVER) {
1424         TCGv_i64 t0 = tcg_const_i64(arg2);
1425         tcg_gen_brcond_i64(cond, arg1, t0, l);
1426         tcg_temp_free_i64(t0);
1427     }
1428 }
1429
1430 void tcg_gen_setcond_i64(TCGCond cond, TCGv_i64 ret,
1431                          TCGv_i64 arg1, TCGv_i64 arg2)
1432 {
1433     if (cond == TCG_COND_ALWAYS) {
1434         tcg_gen_movi_i64(ret, 1);
1435     } else if (cond == TCG_COND_NEVER) {
1436         tcg_gen_movi_i64(ret, 0);
1437     } else {
1438         if (TCG_TARGET_REG_BITS == 32) {
1439             tcg_gen_op6i_i32(INDEX_op_setcond2_i32, TCGV_LOW(ret),
1440                              TCGV_LOW(arg1), TCGV_HIGH(arg1),
1441                              TCGV_LOW(arg2), TCGV_HIGH(arg2), cond);
1442             tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1443         } else {
1444             tcg_gen_op4i_i64(INDEX_op_setcond_i64, ret, arg1, arg2, cond);
1445         }
1446     }
1447 }
1448
1449 void tcg_gen_setcondi_i64(TCGCond cond, TCGv_i64 ret,
1450                           TCGv_i64 arg1, int64_t arg2)
1451 {
1452     TCGv_i64 t0 = tcg_const_i64(arg2);
1453     tcg_gen_setcond_i64(cond, ret, arg1, t0);
1454     tcg_temp_free_i64(t0);
1455 }
1456
1457 void tcg_gen_muli_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
1458 {
1459     if (arg2 == 0) {
1460         tcg_gen_movi_i64(ret, 0);
1461     } else if (is_power_of_2(arg2)) {
1462         tcg_gen_shli_i64(ret, arg1, ctz64(arg2));
1463     } else {
1464         TCGv_i64 t0 = tcg_const_i64(arg2);
1465         tcg_gen_mul_i64(ret, arg1, t0);
1466         tcg_temp_free_i64(t0);
1467     }
1468 }
1469
1470 void tcg_gen_div_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1471 {
1472     if (TCG_TARGET_HAS_div_i64) {
1473         tcg_gen_op3_i64(INDEX_op_div_i64, ret, arg1, arg2);
1474     } else if (TCG_TARGET_HAS_div2_i64) {
1475         TCGv_i64 t0 = tcg_temp_new_i64();
1476         tcg_gen_sari_i64(t0, arg1, 63);
1477         tcg_gen_op5_i64(INDEX_op_div2_i64, ret, t0, arg1, t0, arg2);
1478         tcg_temp_free_i64(t0);
1479     } else {
1480         gen_helper_div_i64(ret, arg1, arg2);
1481     }
1482 }
1483
1484 void tcg_gen_rem_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1485 {
1486     if (TCG_TARGET_HAS_rem_i64) {
1487         tcg_gen_op3_i64(INDEX_op_rem_i64, ret, arg1, arg2);
1488     } else if (TCG_TARGET_HAS_div_i64) {
1489         TCGv_i64 t0 = tcg_temp_new_i64();
1490         tcg_gen_op3_i64(INDEX_op_div_i64, t0, arg1, arg2);
1491         tcg_gen_mul_i64(t0, t0, arg2);
1492         tcg_gen_sub_i64(ret, arg1, t0);
1493         tcg_temp_free_i64(t0);
1494     } else if (TCG_TARGET_HAS_div2_i64) {
1495         TCGv_i64 t0 = tcg_temp_new_i64();
1496         tcg_gen_sari_i64(t0, arg1, 63);
1497         tcg_gen_op5_i64(INDEX_op_div2_i64, t0, ret, arg1, t0, arg2);
1498         tcg_temp_free_i64(t0);
1499     } else {
1500         gen_helper_rem_i64(ret, arg1, arg2);
1501     }
1502 }
1503
1504 void tcg_gen_divu_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1505 {
1506     if (TCG_TARGET_HAS_div_i64) {
1507         tcg_gen_op3_i64(INDEX_op_divu_i64, ret, arg1, arg2);
1508     } else if (TCG_TARGET_HAS_div2_i64) {
1509         TCGv_i64 t0 = tcg_temp_new_i64();
1510         tcg_gen_movi_i64(t0, 0);
1511         tcg_gen_op5_i64(INDEX_op_divu2_i64, ret, t0, arg1, t0, arg2);
1512         tcg_temp_free_i64(t0);
1513     } else {
1514         gen_helper_divu_i64(ret, arg1, arg2);
1515     }
1516 }
1517
1518 void tcg_gen_remu_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1519 {
1520     if (TCG_TARGET_HAS_rem_i64) {
1521         tcg_gen_op3_i64(INDEX_op_remu_i64, ret, arg1, arg2);
1522     } else if (TCG_TARGET_HAS_div_i64) {
1523         TCGv_i64 t0 = tcg_temp_new_i64();
1524         tcg_gen_op3_i64(INDEX_op_divu_i64, t0, arg1, arg2);
1525         tcg_gen_mul_i64(t0, t0, arg2);
1526         tcg_gen_sub_i64(ret, arg1, t0);
1527         tcg_temp_free_i64(t0);
1528     } else if (TCG_TARGET_HAS_div2_i64) {
1529         TCGv_i64 t0 = tcg_temp_new_i64();
1530         tcg_gen_movi_i64(t0, 0);
1531         tcg_gen_op5_i64(INDEX_op_divu2_i64, t0, ret, arg1, t0, arg2);
1532         tcg_temp_free_i64(t0);
1533     } else {
1534         gen_helper_remu_i64(ret, arg1, arg2);
1535     }
1536 }
1537
1538 void tcg_gen_ext8s_i64(TCGv_i64 ret, TCGv_i64 arg)
1539 {
1540     if (TCG_TARGET_REG_BITS == 32) {
1541         tcg_gen_ext8s_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1542         tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
1543     } else if (TCG_TARGET_HAS_ext8s_i64) {
1544         tcg_gen_op2_i64(INDEX_op_ext8s_i64, ret, arg);
1545     } else {
1546         tcg_gen_shli_i64(ret, arg, 56);
1547         tcg_gen_sari_i64(ret, ret, 56);
1548     }
1549 }
1550
1551 void tcg_gen_ext16s_i64(TCGv_i64 ret, TCGv_i64 arg)
1552 {
1553     if (TCG_TARGET_REG_BITS == 32) {
1554         tcg_gen_ext16s_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1555         tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
1556     } else if (TCG_TARGET_HAS_ext16s_i64) {
1557         tcg_gen_op2_i64(INDEX_op_ext16s_i64, ret, arg);
1558     } else {
1559         tcg_gen_shli_i64(ret, arg, 48);
1560         tcg_gen_sari_i64(ret, ret, 48);
1561     }
1562 }
1563
1564 void tcg_gen_ext32s_i64(TCGv_i64 ret, TCGv_i64 arg)
1565 {
1566     if (TCG_TARGET_REG_BITS == 32) {
1567         tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1568         tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
1569     } else if (TCG_TARGET_HAS_ext32s_i64) {
1570         tcg_gen_op2_i64(INDEX_op_ext32s_i64, ret, arg);
1571     } else {
1572         tcg_gen_shli_i64(ret, arg, 32);
1573         tcg_gen_sari_i64(ret, ret, 32);
1574     }
1575 }
1576
1577 void tcg_gen_ext8u_i64(TCGv_i64 ret, TCGv_i64 arg)
1578 {
1579     if (TCG_TARGET_REG_BITS == 32) {
1580         tcg_gen_ext8u_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1581         tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1582     } else if (TCG_TARGET_HAS_ext8u_i64) {
1583         tcg_gen_op2_i64(INDEX_op_ext8u_i64, ret, arg);
1584     } else {
1585         tcg_gen_andi_i64(ret, arg, 0xffu);
1586     }
1587 }
1588
1589 void tcg_gen_ext16u_i64(TCGv_i64 ret, TCGv_i64 arg)
1590 {
1591     if (TCG_TARGET_REG_BITS == 32) {
1592         tcg_gen_ext16u_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1593         tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1594     } else if (TCG_TARGET_HAS_ext16u_i64) {
1595         tcg_gen_op2_i64(INDEX_op_ext16u_i64, ret, arg);
1596     } else {
1597         tcg_gen_andi_i64(ret, arg, 0xffffu);
1598     }
1599 }
1600
1601 void tcg_gen_ext32u_i64(TCGv_i64 ret, TCGv_i64 arg)
1602 {
1603     if (TCG_TARGET_REG_BITS == 32) {
1604         tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1605         tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1606     } else if (TCG_TARGET_HAS_ext32u_i64) {
1607         tcg_gen_op2_i64(INDEX_op_ext32u_i64, ret, arg);
1608     } else {
1609         tcg_gen_andi_i64(ret, arg, 0xffffffffu);
1610     }
1611 }
1612
1613 /* Note: we assume the six high bytes are set to zero */
1614 void tcg_gen_bswap16_i64(TCGv_i64 ret, TCGv_i64 arg)
1615 {
1616     if (TCG_TARGET_REG_BITS == 32) {
1617         tcg_gen_bswap16_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1618         tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1619     } else if (TCG_TARGET_HAS_bswap16_i64) {
1620         tcg_gen_op2_i64(INDEX_op_bswap16_i64, ret, arg);
1621     } else {
1622         TCGv_i64 t0 = tcg_temp_new_i64();
1623
1624         tcg_gen_ext8u_i64(t0, arg);
1625         tcg_gen_shli_i64(t0, t0, 8);
1626         tcg_gen_shri_i64(ret, arg, 8);
1627         tcg_gen_or_i64(ret, ret, t0);
1628         tcg_temp_free_i64(t0);
1629     }
1630 }
1631
1632 /* Note: we assume the four high bytes are set to zero */
1633 void tcg_gen_bswap32_i64(TCGv_i64 ret, TCGv_i64 arg)
1634 {
1635     if (TCG_TARGET_REG_BITS == 32) {
1636         tcg_gen_bswap32_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1637         tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1638     } else if (TCG_TARGET_HAS_bswap32_i64) {
1639         tcg_gen_op2_i64(INDEX_op_bswap32_i64, ret, arg);
1640     } else {
1641         TCGv_i64 t0, t1;
1642         t0 = tcg_temp_new_i64();
1643         t1 = tcg_temp_new_i64();
1644
1645         tcg_gen_shli_i64(t0, arg, 24);
1646         tcg_gen_ext32u_i64(t0, t0);
1647
1648         tcg_gen_andi_i64(t1, arg, 0x0000ff00);
1649         tcg_gen_shli_i64(t1, t1, 8);
1650         tcg_gen_or_i64(t0, t0, t1);
1651
1652         tcg_gen_shri_i64(t1, arg, 8);
1653         tcg_gen_andi_i64(t1, t1, 0x0000ff00);
1654         tcg_gen_or_i64(t0, t0, t1);
1655
1656         tcg_gen_shri_i64(t1, arg, 24);
1657         tcg_gen_or_i64(ret, t0, t1);
1658         tcg_temp_free_i64(t0);
1659         tcg_temp_free_i64(t1);
1660     }
1661 }
1662
1663 void tcg_gen_bswap64_i64(TCGv_i64 ret, TCGv_i64 arg)
1664 {
1665     if (TCG_TARGET_REG_BITS == 32) {
1666         TCGv_i32 t0, t1;
1667         t0 = tcg_temp_new_i32();
1668         t1 = tcg_temp_new_i32();
1669
1670         tcg_gen_bswap32_i32(t0, TCGV_LOW(arg));
1671         tcg_gen_bswap32_i32(t1, TCGV_HIGH(arg));
1672         tcg_gen_mov_i32(TCGV_LOW(ret), t1);
1673         tcg_gen_mov_i32(TCGV_HIGH(ret), t0);
1674         tcg_temp_free_i32(t0);
1675         tcg_temp_free_i32(t1);
1676     } else if (TCG_TARGET_HAS_bswap64_i64) {
1677         tcg_gen_op2_i64(INDEX_op_bswap64_i64, ret, arg);
1678     } else {
1679         TCGv_i64 t0 = tcg_temp_new_i64();
1680         TCGv_i64 t1 = tcg_temp_new_i64();
1681
1682         tcg_gen_shli_i64(t0, arg, 56);
1683
1684         tcg_gen_andi_i64(t1, arg, 0x0000ff00);
1685         tcg_gen_shli_i64(t1, t1, 40);
1686         tcg_gen_or_i64(t0, t0, t1);
1687
1688         tcg_gen_andi_i64(t1, arg, 0x00ff0000);
1689         tcg_gen_shli_i64(t1, t1, 24);
1690         tcg_gen_or_i64(t0, t0, t1);
1691
1692         tcg_gen_andi_i64(t1, arg, 0xff000000);
1693         tcg_gen_shli_i64(t1, t1, 8);
1694         tcg_gen_or_i64(t0, t0, t1);
1695
1696         tcg_gen_shri_i64(t1, arg, 8);
1697         tcg_gen_andi_i64(t1, t1, 0xff000000);
1698         tcg_gen_or_i64(t0, t0, t1);
1699
1700         tcg_gen_shri_i64(t1, arg, 24);
1701         tcg_gen_andi_i64(t1, t1, 0x00ff0000);
1702         tcg_gen_or_i64(t0, t0, t1);
1703
1704         tcg_gen_shri_i64(t1, arg, 40);
1705         tcg_gen_andi_i64(t1, t1, 0x0000ff00);
1706         tcg_gen_or_i64(t0, t0, t1);
1707
1708         tcg_gen_shri_i64(t1, arg, 56);
1709         tcg_gen_or_i64(ret, t0, t1);
1710         tcg_temp_free_i64(t0);
1711         tcg_temp_free_i64(t1);
1712     }
1713 }
1714
1715 void tcg_gen_not_i64(TCGv_i64 ret, TCGv_i64 arg)
1716 {
1717     if (TCG_TARGET_REG_BITS == 32) {
1718         tcg_gen_not_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1719         tcg_gen_not_i32(TCGV_HIGH(ret), TCGV_HIGH(arg));
1720     } else if (TCG_TARGET_HAS_not_i64) {
1721         tcg_gen_op2_i64(INDEX_op_not_i64, ret, arg);
1722     } else {
1723         tcg_gen_xori_i64(ret, arg, -1);
1724     }
1725 }
1726
1727 void tcg_gen_andc_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1728 {
1729     if (TCG_TARGET_REG_BITS == 32) {
1730         tcg_gen_andc_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
1731         tcg_gen_andc_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
1732     } else if (TCG_TARGET_HAS_andc_i64) {
1733         tcg_gen_op3_i64(INDEX_op_andc_i64, ret, arg1, arg2);
1734     } else {
1735         TCGv_i64 t0 = tcg_temp_new_i64();
1736         tcg_gen_not_i64(t0, arg2);
1737         tcg_gen_and_i64(ret, arg1, t0);
1738         tcg_temp_free_i64(t0);
1739     }
1740 }
1741
1742 void tcg_gen_eqv_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1743 {
1744     if (TCG_TARGET_REG_BITS == 32) {
1745         tcg_gen_eqv_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
1746         tcg_gen_eqv_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
1747     } else if (TCG_TARGET_HAS_eqv_i64) {
1748         tcg_gen_op3_i64(INDEX_op_eqv_i64, ret, arg1, arg2);
1749     } else {
1750         tcg_gen_xor_i64(ret, arg1, arg2);
1751         tcg_gen_not_i64(ret, ret);
1752     }
1753 }
1754
1755 void tcg_gen_nand_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1756 {
1757     if (TCG_TARGET_REG_BITS == 32) {
1758         tcg_gen_nand_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
1759         tcg_gen_nand_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
1760     } else if (TCG_TARGET_HAS_nand_i64) {
1761         tcg_gen_op3_i64(INDEX_op_nand_i64, ret, arg1, arg2);
1762     } else {
1763         tcg_gen_and_i64(ret, arg1, arg2);
1764         tcg_gen_not_i64(ret, ret);
1765     }
1766 }
1767
1768 void tcg_gen_nor_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1769 {
1770     if (TCG_TARGET_REG_BITS == 32) {
1771         tcg_gen_nor_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
1772         tcg_gen_nor_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
1773     } else if (TCG_TARGET_HAS_nor_i64) {
1774         tcg_gen_op3_i64(INDEX_op_nor_i64, ret, arg1, arg2);
1775     } else {
1776         tcg_gen_or_i64(ret, arg1, arg2);
1777         tcg_gen_not_i64(ret, ret);
1778     }
1779 }
1780
1781 void tcg_gen_orc_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1782 {
1783     if (TCG_TARGET_REG_BITS == 32) {
1784         tcg_gen_orc_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
1785         tcg_gen_orc_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
1786     } else if (TCG_TARGET_HAS_orc_i64) {
1787         tcg_gen_op3_i64(INDEX_op_orc_i64, ret, arg1, arg2);
1788     } else {
1789         TCGv_i64 t0 = tcg_temp_new_i64();
1790         tcg_gen_not_i64(t0, arg2);
1791         tcg_gen_or_i64(ret, arg1, t0);
1792         tcg_temp_free_i64(t0);
1793     }
1794 }
1795
1796 void tcg_gen_clz_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1797 {
1798     if (TCG_TARGET_HAS_clz_i64) {
1799         tcg_gen_op3_i64(INDEX_op_clz_i64, ret, arg1, arg2);
1800     } else {
1801         gen_helper_clz_i64(ret, arg1, arg2);
1802     }
1803 }
1804
1805 void tcg_gen_clzi_i64(TCGv_i64 ret, TCGv_i64 arg1, uint64_t arg2)
1806 {
1807     if (TCG_TARGET_REG_BITS == 32
1808         && TCG_TARGET_HAS_clz_i32
1809         && arg2 <= 0xffffffffu) {
1810         TCGv_i32 t = tcg_const_i32((uint32_t)arg2 - 32);
1811         tcg_gen_clz_i32(t, TCGV_LOW(arg1), t);
1812         tcg_gen_addi_i32(t, t, 32);
1813         tcg_gen_clz_i32(TCGV_LOW(ret), TCGV_HIGH(arg1), t);
1814         tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1815         tcg_temp_free_i32(t);
1816     } else {
1817         TCGv_i64 t = tcg_const_i64(arg2);
1818         tcg_gen_clz_i64(ret, arg1, t);
1819         tcg_temp_free_i64(t);
1820     }
1821 }
1822
1823 void tcg_gen_ctz_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1824 {
1825     if (TCG_TARGET_HAS_ctz_i64) {
1826         tcg_gen_op3_i64(INDEX_op_ctz_i64, ret, arg1, arg2);
1827     } else if (TCG_TARGET_HAS_ctpop_i64 || TCG_TARGET_HAS_clz_i64) {
1828         TCGv_i64 z, t = tcg_temp_new_i64();
1829
1830         if (TCG_TARGET_HAS_ctpop_i64) {
1831             tcg_gen_subi_i64(t, arg1, 1);
1832             tcg_gen_andc_i64(t, t, arg1);
1833             tcg_gen_ctpop_i64(t, t);
1834         } else {
1835             /* Since all non-x86 hosts have clz(0) == 64, don't fight it.  */
1836             tcg_gen_neg_i64(t, arg1);
1837             tcg_gen_and_i64(t, t, arg1);
1838             tcg_gen_clzi_i64(t, t, 64);
1839             tcg_gen_xori_i64(t, t, 63);
1840         }
1841         z = tcg_const_i64(0);
1842         tcg_gen_movcond_i64(TCG_COND_EQ, ret, arg1, z, arg2, t);
1843         tcg_temp_free_i64(t);
1844         tcg_temp_free_i64(z);
1845     } else {
1846         gen_helper_ctz_i64(ret, arg1, arg2);
1847     }
1848 }
1849
1850 void tcg_gen_ctzi_i64(TCGv_i64 ret, TCGv_i64 arg1, uint64_t arg2)
1851 {
1852     if (TCG_TARGET_REG_BITS == 32
1853         && TCG_TARGET_HAS_ctz_i32
1854         && arg2 <= 0xffffffffu) {
1855         TCGv_i32 t32 = tcg_const_i32((uint32_t)arg2 - 32);
1856         tcg_gen_ctz_i32(t32, TCGV_HIGH(arg1), t32);
1857         tcg_gen_addi_i32(t32, t32, 32);
1858         tcg_gen_ctz_i32(TCGV_LOW(ret), TCGV_LOW(arg1), t32);
1859         tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1860         tcg_temp_free_i32(t32);
1861     } else if (!TCG_TARGET_HAS_ctz_i64
1862                && TCG_TARGET_HAS_ctpop_i64
1863                && arg2 == 64) {
1864         /* This equivalence has the advantage of not requiring a fixup.  */
1865         TCGv_i64 t = tcg_temp_new_i64();
1866         tcg_gen_subi_i64(t, arg1, 1);
1867         tcg_gen_andc_i64(t, t, arg1);
1868         tcg_gen_ctpop_i64(ret, t);
1869         tcg_temp_free_i64(t);
1870     } else {
1871         TCGv_i64 t64 = tcg_const_i64(arg2);
1872         tcg_gen_ctz_i64(ret, arg1, t64);
1873         tcg_temp_free_i64(t64);
1874     }
1875 }
1876
1877 void tcg_gen_clrsb_i64(TCGv_i64 ret, TCGv_i64 arg)
1878 {
1879     if (TCG_TARGET_HAS_clz_i64 || TCG_TARGET_HAS_clz_i32) {
1880         TCGv_i64 t = tcg_temp_new_i64();
1881         tcg_gen_sari_i64(t, arg, 63);
1882         tcg_gen_xor_i64(t, t, arg);
1883         tcg_gen_clzi_i64(t, t, 64);
1884         tcg_gen_subi_i64(ret, t, 1);
1885         tcg_temp_free_i64(t);
1886     } else {
1887         gen_helper_clrsb_i64(ret, arg);
1888     }
1889 }
1890
1891 void tcg_gen_ctpop_i64(TCGv_i64 ret, TCGv_i64 arg1)
1892 {
1893     if (TCG_TARGET_HAS_ctpop_i64) {
1894         tcg_gen_op2_i64(INDEX_op_ctpop_i64, ret, arg1);
1895     } else if (TCG_TARGET_REG_BITS == 32 && TCG_TARGET_HAS_ctpop_i32) {
1896         tcg_gen_ctpop_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1));
1897         tcg_gen_ctpop_i32(TCGV_LOW(ret), TCGV_LOW(arg1));
1898         tcg_gen_add_i32(TCGV_LOW(ret), TCGV_LOW(ret), TCGV_HIGH(ret));
1899         tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1900     } else {
1901         gen_helper_ctpop_i64(ret, arg1);
1902     }
1903 }
1904
1905 void tcg_gen_rotl_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1906 {
1907     if (TCG_TARGET_HAS_rot_i64) {
1908         tcg_gen_op3_i64(INDEX_op_rotl_i64, ret, arg1, arg2);
1909     } else {
1910         TCGv_i64 t0, t1;
1911         t0 = tcg_temp_new_i64();
1912         t1 = tcg_temp_new_i64();
1913         tcg_gen_shl_i64(t0, arg1, arg2);
1914         tcg_gen_subfi_i64(t1, 64, arg2);
1915         tcg_gen_shr_i64(t1, arg1, t1);
1916         tcg_gen_or_i64(ret, t0, t1);
1917         tcg_temp_free_i64(t0);
1918         tcg_temp_free_i64(t1);
1919     }
1920 }
1921
1922 void tcg_gen_rotli_i64(TCGv_i64 ret, TCGv_i64 arg1, unsigned arg2)
1923 {
1924     tcg_debug_assert(arg2 < 64);
1925     /* some cases can be optimized here */
1926     if (arg2 == 0) {
1927         tcg_gen_mov_i64(ret, arg1);
1928     } else if (TCG_TARGET_HAS_rot_i64) {
1929         TCGv_i64 t0 = tcg_const_i64(arg2);
1930         tcg_gen_rotl_i64(ret, arg1, t0);
1931         tcg_temp_free_i64(t0);
1932     } else {
1933         TCGv_i64 t0, t1;
1934         t0 = tcg_temp_new_i64();
1935         t1 = tcg_temp_new_i64();
1936         tcg_gen_shli_i64(t0, arg1, arg2);
1937         tcg_gen_shri_i64(t1, arg1, 64 - arg2);
1938         tcg_gen_or_i64(ret, t0, t1);
1939         tcg_temp_free_i64(t0);
1940         tcg_temp_free_i64(t1);
1941     }
1942 }
1943
1944 void tcg_gen_rotr_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1945 {
1946     if (TCG_TARGET_HAS_rot_i64) {
1947         tcg_gen_op3_i64(INDEX_op_rotr_i64, ret, arg1, arg2);
1948     } else {
1949         TCGv_i64 t0, t1;
1950         t0 = tcg_temp_new_i64();
1951         t1 = tcg_temp_new_i64();
1952         tcg_gen_shr_i64(t0, arg1, arg2);
1953         tcg_gen_subfi_i64(t1, 64, arg2);
1954         tcg_gen_shl_i64(t1, arg1, t1);
1955         tcg_gen_or_i64(ret, t0, t1);
1956         tcg_temp_free_i64(t0);
1957         tcg_temp_free_i64(t1);
1958     }
1959 }
1960
1961 void tcg_gen_rotri_i64(TCGv_i64 ret, TCGv_i64 arg1, unsigned arg2)
1962 {
1963     tcg_debug_assert(arg2 < 64);
1964     /* some cases can be optimized here */
1965     if (arg2 == 0) {
1966         tcg_gen_mov_i64(ret, arg1);
1967     } else {
1968         tcg_gen_rotli_i64(ret, arg1, 64 - arg2);
1969     }
1970 }
1971
1972 void tcg_gen_deposit_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2,
1973                          unsigned int ofs, unsigned int len)
1974 {
1975     uint64_t mask;
1976     TCGv_i64 t1;
1977
1978     tcg_debug_assert(ofs < 64);
1979     tcg_debug_assert(len > 0);
1980     tcg_debug_assert(len <= 64);
1981     tcg_debug_assert(ofs + len <= 64);
1982
1983     if (len == 64) {
1984         tcg_gen_mov_i64(ret, arg2);
1985         return;
1986     }
1987     if (TCG_TARGET_HAS_deposit_i64 && TCG_TARGET_deposit_i64_valid(ofs, len)) {
1988         tcg_gen_op5ii_i64(INDEX_op_deposit_i64, ret, arg1, arg2, ofs, len);
1989         return;
1990     }
1991
1992     if (TCG_TARGET_REG_BITS == 32) {
1993         if (ofs >= 32) {
1994             tcg_gen_deposit_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1),
1995                                 TCGV_LOW(arg2), ofs - 32, len);
1996             tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg1));
1997             return;
1998         }
1999         if (ofs + len <= 32) {
2000             tcg_gen_deposit_i32(TCGV_LOW(ret), TCGV_LOW(arg1),
2001                                 TCGV_LOW(arg2), ofs, len);
2002             tcg_gen_mov_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1));
2003             return;
2004         }
2005     }
2006
2007     mask = (1ull << len) - 1;
2008     t1 = tcg_temp_new_i64();
2009
2010     if (ofs + len < 64) {
2011         tcg_gen_andi_i64(t1, arg2, mask);
2012         tcg_gen_shli_i64(t1, t1, ofs);
2013     } else {
2014         tcg_gen_shli_i64(t1, arg2, ofs);
2015     }
2016     tcg_gen_andi_i64(ret, arg1, ~(mask << ofs));
2017     tcg_gen_or_i64(ret, ret, t1);
2018
2019     tcg_temp_free_i64(t1);
2020 }
2021
2022 void tcg_gen_deposit_z_i64(TCGv_i64 ret, TCGv_i64 arg,
2023                            unsigned int ofs, unsigned int len)
2024 {
2025     tcg_debug_assert(ofs < 64);
2026     tcg_debug_assert(len > 0);
2027     tcg_debug_assert(len <= 64);
2028     tcg_debug_assert(ofs + len <= 64);
2029
2030     if (ofs + len == 64) {
2031         tcg_gen_shli_i64(ret, arg, ofs);
2032     } else if (ofs == 0) {
2033         tcg_gen_andi_i64(ret, arg, (1ull << len) - 1);
2034     } else if (TCG_TARGET_HAS_deposit_i64
2035                && TCG_TARGET_deposit_i64_valid(ofs, len)) {
2036         TCGv_i64 zero = tcg_const_i64(0);
2037         tcg_gen_op5ii_i64(INDEX_op_deposit_i64, ret, zero, arg, ofs, len);
2038         tcg_temp_free_i64(zero);
2039     } else {
2040         if (TCG_TARGET_REG_BITS == 32) {
2041             if (ofs >= 32) {
2042                 tcg_gen_deposit_z_i32(TCGV_HIGH(ret), TCGV_LOW(arg),
2043                                       ofs - 32, len);
2044                 tcg_gen_movi_i32(TCGV_LOW(ret), 0);
2045                 return;
2046             }
2047             if (ofs + len <= 32) {
2048                 tcg_gen_deposit_z_i32(TCGV_LOW(ret), TCGV_LOW(arg), ofs, len);
2049                 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
2050                 return;
2051             }
2052         }
2053         /* To help two-operand hosts we prefer to zero-extend first,
2054            which allows ARG to stay live.  */
2055         switch (len) {
2056         case 32:
2057             if (TCG_TARGET_HAS_ext32u_i64) {
2058                 tcg_gen_ext32u_i64(ret, arg);
2059                 tcg_gen_shli_i64(ret, ret, ofs);
2060                 return;
2061             }
2062             break;
2063         case 16:
2064             if (TCG_TARGET_HAS_ext16u_i64) {
2065                 tcg_gen_ext16u_i64(ret, arg);
2066                 tcg_gen_shli_i64(ret, ret, ofs);
2067                 return;
2068             }
2069             break;
2070         case 8:
2071             if (TCG_TARGET_HAS_ext8u_i64) {
2072                 tcg_gen_ext8u_i64(ret, arg);
2073                 tcg_gen_shli_i64(ret, ret, ofs);
2074                 return;
2075             }
2076             break;
2077         }
2078         /* Otherwise prefer zero-extension over AND for code size.  */
2079         switch (ofs + len) {
2080         case 32:
2081             if (TCG_TARGET_HAS_ext32u_i64) {
2082                 tcg_gen_shli_i64(ret, arg, ofs);
2083                 tcg_gen_ext32u_i64(ret, ret);
2084                 return;
2085             }
2086             break;
2087         case 16:
2088             if (TCG_TARGET_HAS_ext16u_i64) {
2089                 tcg_gen_shli_i64(ret, arg, ofs);
2090                 tcg_gen_ext16u_i64(ret, ret);
2091                 return;
2092             }
2093             break;
2094         case 8:
2095             if (TCG_TARGET_HAS_ext8u_i64) {
2096                 tcg_gen_shli_i64(ret, arg, ofs);
2097                 tcg_gen_ext8u_i64(ret, ret);
2098                 return;
2099             }
2100             break;
2101         }
2102         tcg_gen_andi_i64(ret, arg, (1ull << len) - 1);
2103         tcg_gen_shli_i64(ret, ret, ofs);
2104     }
2105 }
2106
2107 void tcg_gen_extract_i64(TCGv_i64 ret, TCGv_i64 arg,
2108                          unsigned int ofs, unsigned int len)
2109 {
2110     tcg_debug_assert(ofs < 64);
2111     tcg_debug_assert(len > 0);
2112     tcg_debug_assert(len <= 64);
2113     tcg_debug_assert(ofs + len <= 64);
2114
2115     /* Canonicalize certain special cases, even if extract is supported.  */
2116     if (ofs + len == 64) {
2117         tcg_gen_shri_i64(ret, arg, 64 - len);
2118         return;
2119     }
2120     if (ofs == 0) {
2121         tcg_gen_andi_i64(ret, arg, (1ull << len) - 1);
2122         return;
2123     }
2124
2125     if (TCG_TARGET_REG_BITS == 32) {
2126         /* Look for a 32-bit extract within one of the two words.  */
2127         if (ofs >= 32) {
2128             tcg_gen_extract_i32(TCGV_LOW(ret), TCGV_HIGH(arg), ofs - 32, len);
2129             tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
2130             return;
2131         }
2132         if (ofs + len <= 32) {
2133             tcg_gen_extract_i32(TCGV_LOW(ret), TCGV_LOW(arg), ofs, len);
2134             tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
2135             return;
2136         }
2137         /* The field is split across two words.  One double-word
2138            shift is better than two double-word shifts.  */
2139         goto do_shift_and;
2140     }
2141
2142     if (TCG_TARGET_HAS_extract_i64
2143         && TCG_TARGET_extract_i64_valid(ofs, len)) {
2144         tcg_gen_op4ii_i64(INDEX_op_extract_i64, ret, arg, ofs, len);
2145         return;
2146     }
2147
2148     /* Assume that zero-extension, if available, is cheaper than a shift.  */
2149     switch (ofs + len) {
2150     case 32:
2151         if (TCG_TARGET_HAS_ext32u_i64) {
2152             tcg_gen_ext32u_i64(ret, arg);
2153             tcg_gen_shri_i64(ret, ret, ofs);
2154             return;
2155         }
2156         break;
2157     case 16:
2158         if (TCG_TARGET_HAS_ext16u_i64) {
2159             tcg_gen_ext16u_i64(ret, arg);
2160             tcg_gen_shri_i64(ret, ret, ofs);
2161             return;
2162         }
2163         break;
2164     case 8:
2165         if (TCG_TARGET_HAS_ext8u_i64) {
2166             tcg_gen_ext8u_i64(ret, arg);
2167             tcg_gen_shri_i64(ret, ret, ofs);
2168             return;
2169         }
2170         break;
2171     }
2172
2173     /* ??? Ideally we'd know what values are available for immediate AND.
2174        Assume that 8 bits are available, plus the special cases of 16 and 32,
2175        so that we get ext8u, ext16u, and ext32u.  */
2176     switch (len) {
2177     case 1 ... 8: case 16: case 32:
2178     do_shift_and:
2179         tcg_gen_shri_i64(ret, arg, ofs);
2180         tcg_gen_andi_i64(ret, ret, (1ull << len) - 1);
2181         break;
2182     default:
2183         tcg_gen_shli_i64(ret, arg, 64 - len - ofs);
2184         tcg_gen_shri_i64(ret, ret, 64 - len);
2185         break;
2186     }
2187 }
2188
2189 void tcg_gen_sextract_i64(TCGv_i64 ret, TCGv_i64 arg,
2190                           unsigned int ofs, unsigned int len)
2191 {
2192     tcg_debug_assert(ofs < 64);
2193     tcg_debug_assert(len > 0);
2194     tcg_debug_assert(len <= 64);
2195     tcg_debug_assert(ofs + len <= 64);
2196
2197     /* Canonicalize certain special cases, even if sextract is supported.  */
2198     if (ofs + len == 64) {
2199         tcg_gen_sari_i64(ret, arg, 64 - len);
2200         return;
2201     }
2202     if (ofs == 0) {
2203         switch (len) {
2204         case 32:
2205             tcg_gen_ext32s_i64(ret, arg);
2206             return;
2207         case 16:
2208             tcg_gen_ext16s_i64(ret, arg);
2209             return;
2210         case 8:
2211             tcg_gen_ext8s_i64(ret, arg);
2212             return;
2213         }
2214     }
2215
2216     if (TCG_TARGET_REG_BITS == 32) {
2217         /* Look for a 32-bit extract within one of the two words.  */
2218         if (ofs >= 32) {
2219             tcg_gen_sextract_i32(TCGV_LOW(ret), TCGV_HIGH(arg), ofs - 32, len);
2220         } else if (ofs + len <= 32) {
2221             tcg_gen_sextract_i32(TCGV_LOW(ret), TCGV_LOW(arg), ofs, len);
2222         } else if (ofs == 0) {
2223             tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg));
2224             tcg_gen_sextract_i32(TCGV_HIGH(ret), TCGV_HIGH(arg), 0, len - 32);
2225             return;
2226         } else if (len > 32) {
2227             TCGv_i32 t = tcg_temp_new_i32();
2228             /* Extract the bits for the high word normally.  */
2229             tcg_gen_sextract_i32(t, TCGV_HIGH(arg), ofs + 32, len - 32);
2230             /* Shift the field down for the low part.  */
2231             tcg_gen_shri_i64(ret, arg, ofs);
2232             /* Overwrite the shift into the high part.  */
2233             tcg_gen_mov_i32(TCGV_HIGH(ret), t);
2234             tcg_temp_free_i32(t);
2235             return;
2236         } else {
2237             /* Shift the field down for the low part, such that the
2238                field sits at the MSB.  */
2239             tcg_gen_shri_i64(ret, arg, ofs + len - 32);
2240             /* Shift the field down from the MSB, sign extending.  */
2241             tcg_gen_sari_i32(TCGV_LOW(ret), TCGV_LOW(ret), 32 - len);
2242         }
2243         /* Sign-extend the field from 32 bits.  */
2244         tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
2245         return;
2246     }
2247
2248     if (TCG_TARGET_HAS_sextract_i64
2249         && TCG_TARGET_extract_i64_valid(ofs, len)) {
2250         tcg_gen_op4ii_i64(INDEX_op_sextract_i64, ret, arg, ofs, len);
2251         return;
2252     }
2253
2254     /* Assume that sign-extension, if available, is cheaper than a shift.  */
2255     switch (ofs + len) {
2256     case 32:
2257         if (TCG_TARGET_HAS_ext32s_i64) {
2258             tcg_gen_ext32s_i64(ret, arg);
2259             tcg_gen_sari_i64(ret, ret, ofs);
2260             return;
2261         }
2262         break;
2263     case 16:
2264         if (TCG_TARGET_HAS_ext16s_i64) {
2265             tcg_gen_ext16s_i64(ret, arg);
2266             tcg_gen_sari_i64(ret, ret, ofs);
2267             return;
2268         }
2269         break;
2270     case 8:
2271         if (TCG_TARGET_HAS_ext8s_i64) {
2272             tcg_gen_ext8s_i64(ret, arg);
2273             tcg_gen_sari_i64(ret, ret, ofs);
2274             return;
2275         }
2276         break;
2277     }
2278     switch (len) {
2279     case 32:
2280         if (TCG_TARGET_HAS_ext32s_i64) {
2281             tcg_gen_shri_i64(ret, arg, ofs);
2282             tcg_gen_ext32s_i64(ret, ret);
2283             return;
2284         }
2285         break;
2286     case 16:
2287         if (TCG_TARGET_HAS_ext16s_i64) {
2288             tcg_gen_shri_i64(ret, arg, ofs);
2289             tcg_gen_ext16s_i64(ret, ret);
2290             return;
2291         }
2292         break;
2293     case 8:
2294         if (TCG_TARGET_HAS_ext8s_i64) {
2295             tcg_gen_shri_i64(ret, arg, ofs);
2296             tcg_gen_ext8s_i64(ret, ret);
2297             return;
2298         }
2299         break;
2300     }
2301     tcg_gen_shli_i64(ret, arg, 64 - len - ofs);
2302     tcg_gen_sari_i64(ret, ret, 64 - len);
2303 }
2304
2305 void tcg_gen_movcond_i64(TCGCond cond, TCGv_i64 ret, TCGv_i64 c1,
2306                          TCGv_i64 c2, TCGv_i64 v1, TCGv_i64 v2)
2307 {
2308     if (cond == TCG_COND_ALWAYS) {
2309         tcg_gen_mov_i64(ret, v1);
2310     } else if (cond == TCG_COND_NEVER) {
2311         tcg_gen_mov_i64(ret, v2);
2312     } else if (TCG_TARGET_REG_BITS == 32) {
2313         TCGv_i32 t0 = tcg_temp_new_i32();
2314         TCGv_i32 t1 = tcg_temp_new_i32();
2315         tcg_gen_op6i_i32(INDEX_op_setcond2_i32, t0,
2316                          TCGV_LOW(c1), TCGV_HIGH(c1),
2317                          TCGV_LOW(c2), TCGV_HIGH(c2), cond);
2318
2319         if (TCG_TARGET_HAS_movcond_i32) {
2320             tcg_gen_movi_i32(t1, 0);
2321             tcg_gen_movcond_i32(TCG_COND_NE, TCGV_LOW(ret), t0, t1,
2322                                 TCGV_LOW(v1), TCGV_LOW(v2));
2323             tcg_gen_movcond_i32(TCG_COND_NE, TCGV_HIGH(ret), t0, t1,
2324                                 TCGV_HIGH(v1), TCGV_HIGH(v2));
2325         } else {
2326             tcg_gen_neg_i32(t0, t0);
2327
2328             tcg_gen_and_i32(t1, TCGV_LOW(v1), t0);
2329             tcg_gen_andc_i32(TCGV_LOW(ret), TCGV_LOW(v2), t0);
2330             tcg_gen_or_i32(TCGV_LOW(ret), TCGV_LOW(ret), t1);
2331
2332             tcg_gen_and_i32(t1, TCGV_HIGH(v1), t0);
2333             tcg_gen_andc_i32(TCGV_HIGH(ret), TCGV_HIGH(v2), t0);
2334             tcg_gen_or_i32(TCGV_HIGH(ret), TCGV_HIGH(ret), t1);
2335         }
2336         tcg_temp_free_i32(t0);
2337         tcg_temp_free_i32(t1);
2338     } else if (TCG_TARGET_HAS_movcond_i64) {
2339         tcg_gen_op6i_i64(INDEX_op_movcond_i64, ret, c1, c2, v1, v2, cond);
2340     } else {
2341         TCGv_i64 t0 = tcg_temp_new_i64();
2342         TCGv_i64 t1 = tcg_temp_new_i64();
2343         tcg_gen_setcond_i64(cond, t0, c1, c2);
2344         tcg_gen_neg_i64(t0, t0);
2345         tcg_gen_and_i64(t1, v1, t0);
2346         tcg_gen_andc_i64(ret, v2, t0);
2347         tcg_gen_or_i64(ret, ret, t1);
2348         tcg_temp_free_i64(t0);
2349         tcg_temp_free_i64(t1);
2350     }
2351 }
2352
2353 void tcg_gen_add2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 al,
2354                       TCGv_i64 ah, TCGv_i64 bl, TCGv_i64 bh)
2355 {
2356     if (TCG_TARGET_HAS_add2_i64) {
2357         tcg_gen_op6_i64(INDEX_op_add2_i64, rl, rh, al, ah, bl, bh);
2358     } else {
2359         TCGv_i64 t0 = tcg_temp_new_i64();
2360         TCGv_i64 t1 = tcg_temp_new_i64();
2361         tcg_gen_add_i64(t0, al, bl);
2362         tcg_gen_setcond_i64(TCG_COND_LTU, t1, t0, al);
2363         tcg_gen_add_i64(rh, ah, bh);
2364         tcg_gen_add_i64(rh, rh, t1);
2365         tcg_gen_mov_i64(rl, t0);
2366         tcg_temp_free_i64(t0);
2367         tcg_temp_free_i64(t1);
2368     }
2369 }
2370
2371 void tcg_gen_sub2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 al,
2372                       TCGv_i64 ah, TCGv_i64 bl, TCGv_i64 bh)
2373 {
2374     if (TCG_TARGET_HAS_sub2_i64) {
2375         tcg_gen_op6_i64(INDEX_op_sub2_i64, rl, rh, al, ah, bl, bh);
2376     } else {
2377         TCGv_i64 t0 = tcg_temp_new_i64();
2378         TCGv_i64 t1 = tcg_temp_new_i64();
2379         tcg_gen_sub_i64(t0, al, bl);
2380         tcg_gen_setcond_i64(TCG_COND_LTU, t1, al, bl);
2381         tcg_gen_sub_i64(rh, ah, bh);
2382         tcg_gen_sub_i64(rh, rh, t1);
2383         tcg_gen_mov_i64(rl, t0);
2384         tcg_temp_free_i64(t0);
2385         tcg_temp_free_i64(t1);
2386     }
2387 }
2388
2389 void tcg_gen_mulu2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 arg1, TCGv_i64 arg2)
2390 {
2391     if (TCG_TARGET_HAS_mulu2_i64) {
2392         tcg_gen_op4_i64(INDEX_op_mulu2_i64, rl, rh, arg1, arg2);
2393     } else if (TCG_TARGET_HAS_muluh_i64) {
2394         TCGv_i64 t = tcg_temp_new_i64();
2395         tcg_gen_op3_i64(INDEX_op_mul_i64, t, arg1, arg2);
2396         tcg_gen_op3_i64(INDEX_op_muluh_i64, rh, arg1, arg2);
2397         tcg_gen_mov_i64(rl, t);
2398         tcg_temp_free_i64(t);
2399     } else {
2400         TCGv_i64 t0 = tcg_temp_new_i64();
2401         tcg_gen_mul_i64(t0, arg1, arg2);
2402         gen_helper_muluh_i64(rh, arg1, arg2);
2403         tcg_gen_mov_i64(rl, t0);
2404         tcg_temp_free_i64(t0);
2405     }
2406 }
2407
2408 void tcg_gen_muls2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 arg1, TCGv_i64 arg2)
2409 {
2410     if (TCG_TARGET_HAS_muls2_i64) {
2411         tcg_gen_op4_i64(INDEX_op_muls2_i64, rl, rh, arg1, arg2);
2412     } else if (TCG_TARGET_HAS_mulsh_i64) {
2413         TCGv_i64 t = tcg_temp_new_i64();
2414         tcg_gen_op3_i64(INDEX_op_mul_i64, t, arg1, arg2);
2415         tcg_gen_op3_i64(INDEX_op_mulsh_i64, rh, arg1, arg2);
2416         tcg_gen_mov_i64(rl, t);
2417         tcg_temp_free_i64(t);
2418     } else if (TCG_TARGET_HAS_mulu2_i64 || TCG_TARGET_HAS_muluh_i64) {
2419         TCGv_i64 t0 = tcg_temp_new_i64();
2420         TCGv_i64 t1 = tcg_temp_new_i64();
2421         TCGv_i64 t2 = tcg_temp_new_i64();
2422         TCGv_i64 t3 = tcg_temp_new_i64();
2423         tcg_gen_mulu2_i64(t0, t1, arg1, arg2);
2424         /* Adjust for negative inputs.  */
2425         tcg_gen_sari_i64(t2, arg1, 63);
2426         tcg_gen_sari_i64(t3, arg2, 63);
2427         tcg_gen_and_i64(t2, t2, arg2);
2428         tcg_gen_and_i64(t3, t3, arg1);
2429         tcg_gen_sub_i64(rh, t1, t2);
2430         tcg_gen_sub_i64(rh, rh, t3);
2431         tcg_gen_mov_i64(rl, t0);
2432         tcg_temp_free_i64(t0);
2433         tcg_temp_free_i64(t1);
2434         tcg_temp_free_i64(t2);
2435         tcg_temp_free_i64(t3);
2436     } else {
2437         TCGv_i64 t0 = tcg_temp_new_i64();
2438         tcg_gen_mul_i64(t0, arg1, arg2);
2439         gen_helper_mulsh_i64(rh, arg1, arg2);
2440         tcg_gen_mov_i64(rl, t0);
2441         tcg_temp_free_i64(t0);
2442     }
2443 }
2444
2445 void tcg_gen_mulsu2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 arg1, TCGv_i64 arg2)
2446 {
2447     TCGv_i64 t0 = tcg_temp_new_i64();
2448     TCGv_i64 t1 = tcg_temp_new_i64();
2449     TCGv_i64 t2 = tcg_temp_new_i64();
2450     tcg_gen_mulu2_i64(t0, t1, arg1, arg2);
2451     /* Adjust for negative input for the signed arg1.  */
2452     tcg_gen_sari_i64(t2, arg1, 63);
2453     tcg_gen_and_i64(t2, t2, arg2);
2454     tcg_gen_sub_i64(rh, t1, t2);
2455     tcg_gen_mov_i64(rl, t0);
2456     tcg_temp_free_i64(t0);
2457     tcg_temp_free_i64(t1);
2458     tcg_temp_free_i64(t2);
2459 }
2460
2461 void tcg_gen_smin_i64(TCGv_i64 ret, TCGv_i64 a, TCGv_i64 b)
2462 {
2463     tcg_gen_movcond_i64(TCG_COND_LT, ret, a, b, a, b);
2464 }
2465
2466 void tcg_gen_umin_i64(TCGv_i64 ret, TCGv_i64 a, TCGv_i64 b)
2467 {
2468     tcg_gen_movcond_i64(TCG_COND_LTU, ret, a, b, a, b);
2469 }
2470
2471 void tcg_gen_smax_i64(TCGv_i64 ret, TCGv_i64 a, TCGv_i64 b)
2472 {
2473     tcg_gen_movcond_i64(TCG_COND_LT, ret, a, b, b, a);
2474 }
2475
2476 void tcg_gen_umax_i64(TCGv_i64 ret, TCGv_i64 a, TCGv_i64 b)
2477 {
2478     tcg_gen_movcond_i64(TCG_COND_LTU, ret, a, b, b, a);
2479 }
2480
2481 /* Size changing operations.  */
2482
2483 void tcg_gen_extrl_i64_i32(TCGv_i32 ret, TCGv_i64 arg)
2484 {
2485     if (TCG_TARGET_REG_BITS == 32) {
2486         tcg_gen_mov_i32(ret, TCGV_LOW(arg));
2487     } else if (TCG_TARGET_HAS_extrl_i64_i32) {
2488         tcg_gen_op2(INDEX_op_extrl_i64_i32,
2489                     tcgv_i32_arg(ret), tcgv_i64_arg(arg));
2490     } else {
2491         tcg_gen_mov_i32(ret, (TCGv_i32)arg);
2492     }
2493 }
2494
2495 void tcg_gen_extrh_i64_i32(TCGv_i32 ret, TCGv_i64 arg)
2496 {
2497     if (TCG_TARGET_REG_BITS == 32) {
2498         tcg_gen_mov_i32(ret, TCGV_HIGH(arg));
2499     } else if (TCG_TARGET_HAS_extrh_i64_i32) {
2500         tcg_gen_op2(INDEX_op_extrh_i64_i32,
2501                     tcgv_i32_arg(ret), tcgv_i64_arg(arg));
2502     } else {
2503         TCGv_i64 t = tcg_temp_new_i64();
2504         tcg_gen_shri_i64(t, arg, 32);
2505         tcg_gen_mov_i32(ret, (TCGv_i32)t);
2506         tcg_temp_free_i64(t);
2507     }
2508 }
2509
2510 void tcg_gen_extu_i32_i64(TCGv_i64 ret, TCGv_i32 arg)
2511 {
2512     if (TCG_TARGET_REG_BITS == 32) {
2513         tcg_gen_mov_i32(TCGV_LOW(ret), arg);
2514         tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
2515     } else {
2516         tcg_gen_op2(INDEX_op_extu_i32_i64,
2517                     tcgv_i64_arg(ret), tcgv_i32_arg(arg));
2518     }
2519 }
2520
2521 void tcg_gen_ext_i32_i64(TCGv_i64 ret, TCGv_i32 arg)
2522 {
2523     if (TCG_TARGET_REG_BITS == 32) {
2524         tcg_gen_mov_i32(TCGV_LOW(ret), arg);
2525         tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
2526     } else {
2527         tcg_gen_op2(INDEX_op_ext_i32_i64,
2528                     tcgv_i64_arg(ret), tcgv_i32_arg(arg));
2529     }
2530 }
2531
2532 void tcg_gen_concat_i32_i64(TCGv_i64 dest, TCGv_i32 low, TCGv_i32 high)
2533 {
2534     TCGv_i64 tmp;
2535
2536     if (TCG_TARGET_REG_BITS == 32) {
2537         tcg_gen_mov_i32(TCGV_LOW(dest), low);
2538         tcg_gen_mov_i32(TCGV_HIGH(dest), high);
2539         return;
2540     }
2541
2542     tmp = tcg_temp_new_i64();
2543     /* These extensions are only needed for type correctness.
2544        We may be able to do better given target specific information.  */
2545     tcg_gen_extu_i32_i64(tmp, high);
2546     tcg_gen_extu_i32_i64(dest, low);
2547     /* If deposit is available, use it.  Otherwise use the extra
2548        knowledge that we have of the zero-extensions above.  */
2549     if (TCG_TARGET_HAS_deposit_i64 && TCG_TARGET_deposit_i64_valid(32, 32)) {
2550         tcg_gen_deposit_i64(dest, dest, tmp, 32, 32);
2551     } else {
2552         tcg_gen_shli_i64(tmp, tmp, 32);
2553         tcg_gen_or_i64(dest, dest, tmp);
2554     }
2555     tcg_temp_free_i64(tmp);
2556 }
2557
2558 void tcg_gen_extr_i64_i32(TCGv_i32 lo, TCGv_i32 hi, TCGv_i64 arg)
2559 {
2560     if (TCG_TARGET_REG_BITS == 32) {
2561         tcg_gen_mov_i32(lo, TCGV_LOW(arg));
2562         tcg_gen_mov_i32(hi, TCGV_HIGH(arg));
2563     } else {
2564         tcg_gen_extrl_i64_i32(lo, arg);
2565         tcg_gen_extrh_i64_i32(hi, arg);
2566     }
2567 }
2568
2569 void tcg_gen_extr32_i64(TCGv_i64 lo, TCGv_i64 hi, TCGv_i64 arg)
2570 {
2571     tcg_gen_ext32u_i64(lo, arg);
2572     tcg_gen_shri_i64(hi, arg, 32);
2573 }
2574
2575 /* QEMU specific operations.  */
2576
2577 void tcg_gen_exit_tb(TranslationBlock *tb, unsigned idx)
2578 {
2579     uintptr_t val = (uintptr_t)tb + idx;
2580
2581     if (tb == NULL) {
2582         tcg_debug_assert(idx == 0);
2583     } else if (idx <= TB_EXIT_IDXMAX) {
2584 #ifdef CONFIG_DEBUG_TCG
2585         /* This is an exit following a goto_tb.  Verify that we have
2586            seen this numbered exit before, via tcg_gen_goto_tb.  */
2587         tcg_debug_assert(tcg_ctx->goto_tb_issue_mask & (1 << idx));
2588 #endif
2589     } else {
2590         /* This is an exit via the exitreq label.  */
2591         tcg_debug_assert(idx == TB_EXIT_REQUESTED);
2592     }
2593
2594     tcg_gen_op1i(INDEX_op_exit_tb, val);
2595 }
2596
2597 void tcg_gen_goto_tb(unsigned idx)
2598 {
2599     /* We only support two chained exits.  */
2600     tcg_debug_assert(idx <= TB_EXIT_IDXMAX);
2601 #ifdef CONFIG_DEBUG_TCG
2602     /* Verify that we havn't seen this numbered exit before.  */
2603     tcg_debug_assert((tcg_ctx->goto_tb_issue_mask & (1 << idx)) == 0);
2604     tcg_ctx->goto_tb_issue_mask |= 1 << idx;
2605 #endif
2606     tcg_gen_op1i(INDEX_op_goto_tb, idx);
2607 }
2608
2609 void tcg_gen_lookup_and_goto_ptr(void)
2610 {
2611     if (TCG_TARGET_HAS_goto_ptr && !qemu_loglevel_mask(CPU_LOG_TB_NOCHAIN)) {
2612         TCGv_ptr ptr = tcg_temp_new_ptr();
2613         gen_helper_lookup_tb_ptr(ptr, cpu_env);
2614         tcg_gen_op1i(INDEX_op_goto_ptr, tcgv_ptr_arg(ptr));
2615         tcg_temp_free_ptr(ptr);
2616     } else {
2617         tcg_gen_exit_tb(NULL, 0);
2618     }
2619 }
2620
2621 static inline TCGMemOp tcg_canonicalize_memop(TCGMemOp op, bool is64, bool st)
2622 {
2623     /* Trigger the asserts within as early as possible.  */
2624     (void)get_alignment_bits(op);
2625
2626     switch (op & MO_SIZE) {
2627     case MO_8:
2628         op &= ~MO_BSWAP;
2629         break;
2630     case MO_16:
2631         break;
2632     case MO_32:
2633         if (!is64) {
2634             op &= ~MO_SIGN;
2635         }
2636         break;
2637     case MO_64:
2638         if (!is64) {
2639             tcg_abort();
2640         }
2641         break;
2642     }
2643     if (st) {
2644         op &= ~MO_SIGN;
2645     }
2646     return op;
2647 }
2648
2649 static void gen_ldst_i32(TCGOpcode opc, TCGv_i32 val, TCGv addr,
2650                          TCGMemOp memop, TCGArg idx)
2651 {
2652     TCGMemOpIdx oi = make_memop_idx(memop, idx);
2653 #if TARGET_LONG_BITS == 32
2654     tcg_gen_op3i_i32(opc, val, addr, oi);
2655 #else
2656     if (TCG_TARGET_REG_BITS == 32) {
2657         tcg_gen_op4i_i32(opc, val, TCGV_LOW(addr), TCGV_HIGH(addr), oi);
2658     } else {
2659         tcg_gen_op3(opc, tcgv_i32_arg(val), tcgv_i64_arg(addr), oi);
2660     }
2661 #endif
2662 }
2663
2664 static void gen_ldst_i64(TCGOpcode opc, TCGv_i64 val, TCGv addr,
2665                          TCGMemOp memop, TCGArg idx)
2666 {
2667     TCGMemOpIdx oi = make_memop_idx(memop, idx);
2668 #if TARGET_LONG_BITS == 32
2669     if (TCG_TARGET_REG_BITS == 32) {
2670         tcg_gen_op4i_i32(opc, TCGV_LOW(val), TCGV_HIGH(val), addr, oi);
2671     } else {
2672         tcg_gen_op3(opc, tcgv_i64_arg(val), tcgv_i32_arg(addr), oi);
2673     }
2674 #else
2675     if (TCG_TARGET_REG_BITS == 32) {
2676         tcg_gen_op5i_i32(opc, TCGV_LOW(val), TCGV_HIGH(val),
2677                          TCGV_LOW(addr), TCGV_HIGH(addr), oi);
2678     } else {
2679         tcg_gen_op3i_i64(opc, val, addr, oi);
2680     }
2681 #endif
2682 }
2683
2684 static void tcg_gen_req_mo(TCGBar type)
2685 {
2686 #ifdef TCG_GUEST_DEFAULT_MO
2687     type &= TCG_GUEST_DEFAULT_MO;
2688 #endif
2689     type &= ~TCG_TARGET_DEFAULT_MO;
2690     if (type) {
2691         tcg_gen_mb(type | TCG_BAR_SC);
2692     }
2693 }
2694
2695 void tcg_gen_qemu_ld_i32(TCGv_i32 val, TCGv addr, TCGArg idx, TCGMemOp memop)
2696 {
2697     tcg_gen_req_mo(TCG_MO_LD_LD | TCG_MO_ST_LD);
2698     memop = tcg_canonicalize_memop(memop, 0, 0);
2699     trace_guest_mem_before_tcg(tcg_ctx->cpu, cpu_env,
2700                                addr, trace_mem_get_info(memop, 0));
2701     gen_ldst_i32(INDEX_op_qemu_ld_i32, val, addr, memop, idx);
2702 }
2703
2704 void tcg_gen_qemu_st_i32(TCGv_i32 val, TCGv addr, TCGArg idx, TCGMemOp memop)
2705 {
2706     tcg_gen_req_mo(TCG_MO_LD_ST | TCG_MO_ST_ST);
2707     memop = tcg_canonicalize_memop(memop, 0, 1);
2708     trace_guest_mem_before_tcg(tcg_ctx->cpu, cpu_env,
2709                                addr, trace_mem_get_info(memop, 1));
2710     gen_ldst_i32(INDEX_op_qemu_st_i32, val, addr, memop, idx);
2711 }
2712
2713 void tcg_gen_qemu_ld_i64(TCGv_i64 val, TCGv addr, TCGArg idx, TCGMemOp memop)
2714 {
2715     tcg_gen_req_mo(TCG_MO_LD_LD | TCG_MO_ST_LD);
2716     if (TCG_TARGET_REG_BITS == 32 && (memop & MO_SIZE) < MO_64) {
2717         tcg_gen_qemu_ld_i32(TCGV_LOW(val), addr, idx, memop);
2718         if (memop & MO_SIGN) {
2719             tcg_gen_sari_i32(TCGV_HIGH(val), TCGV_LOW(val), 31);
2720         } else {
2721             tcg_gen_movi_i32(TCGV_HIGH(val), 0);
2722         }
2723         return;
2724     }
2725
2726     memop = tcg_canonicalize_memop(memop, 1, 0);
2727     trace_guest_mem_before_tcg(tcg_ctx->cpu, cpu_env,
2728                                addr, trace_mem_get_info(memop, 0));
2729     gen_ldst_i64(INDEX_op_qemu_ld_i64, val, addr, memop, idx);
2730 }
2731
2732 void tcg_gen_qemu_st_i64(TCGv_i64 val, TCGv addr, TCGArg idx, TCGMemOp memop)
2733 {
2734     tcg_gen_req_mo(TCG_MO_LD_ST | TCG_MO_ST_ST);
2735     if (TCG_TARGET_REG_BITS == 32 && (memop & MO_SIZE) < MO_64) {
2736         tcg_gen_qemu_st_i32(TCGV_LOW(val), addr, idx, memop);
2737         return;
2738     }
2739
2740     memop = tcg_canonicalize_memop(memop, 1, 1);
2741     trace_guest_mem_before_tcg(tcg_ctx->cpu, cpu_env,
2742                                addr, trace_mem_get_info(memop, 1));
2743     gen_ldst_i64(INDEX_op_qemu_st_i64, val, addr, memop, idx);
2744 }
2745
2746 static void tcg_gen_ext_i32(TCGv_i32 ret, TCGv_i32 val, TCGMemOp opc)
2747 {
2748     switch (opc & MO_SSIZE) {
2749     case MO_SB:
2750         tcg_gen_ext8s_i32(ret, val);
2751         break;
2752     case MO_UB:
2753         tcg_gen_ext8u_i32(ret, val);
2754         break;
2755     case MO_SW:
2756         tcg_gen_ext16s_i32(ret, val);
2757         break;
2758     case MO_UW:
2759         tcg_gen_ext16u_i32(ret, val);
2760         break;
2761     default:
2762         tcg_gen_mov_i32(ret, val);
2763         break;
2764     }
2765 }
2766
2767 static void tcg_gen_ext_i64(TCGv_i64 ret, TCGv_i64 val, TCGMemOp opc)
2768 {
2769     switch (opc & MO_SSIZE) {
2770     case MO_SB:
2771         tcg_gen_ext8s_i64(ret, val);
2772         break;
2773     case MO_UB:
2774         tcg_gen_ext8u_i64(ret, val);
2775         break;
2776     case MO_SW:
2777         tcg_gen_ext16s_i64(ret, val);
2778         break;
2779     case MO_UW:
2780         tcg_gen_ext16u_i64(ret, val);
2781         break;
2782     case MO_SL:
2783         tcg_gen_ext32s_i64(ret, val);
2784         break;
2785     case MO_UL:
2786         tcg_gen_ext32u_i64(ret, val);
2787         break;
2788     default:
2789         tcg_gen_mov_i64(ret, val);
2790         break;
2791     }
2792 }
2793
2794 #ifdef CONFIG_SOFTMMU
2795 typedef void (*gen_atomic_cx_i32)(TCGv_i32, TCGv_env, TCGv,
2796                                   TCGv_i32, TCGv_i32, TCGv_i32);
2797 typedef void (*gen_atomic_cx_i64)(TCGv_i64, TCGv_env, TCGv,
2798                                   TCGv_i64, TCGv_i64, TCGv_i32);
2799 typedef void (*gen_atomic_op_i32)(TCGv_i32, TCGv_env, TCGv,
2800                                   TCGv_i32, TCGv_i32);
2801 typedef void (*gen_atomic_op_i64)(TCGv_i64, TCGv_env, TCGv,
2802                                   TCGv_i64, TCGv_i32);
2803 #else
2804 typedef void (*gen_atomic_cx_i32)(TCGv_i32, TCGv_env, TCGv, TCGv_i32, TCGv_i32);
2805 typedef void (*gen_atomic_cx_i64)(TCGv_i64, TCGv_env, TCGv, TCGv_i64, TCGv_i64);
2806 typedef void (*gen_atomic_op_i32)(TCGv_i32, TCGv_env, TCGv, TCGv_i32);
2807 typedef void (*gen_atomic_op_i64)(TCGv_i64, TCGv_env, TCGv, TCGv_i64);
2808 #endif
2809
2810 #ifdef CONFIG_ATOMIC64
2811 # define WITH_ATOMIC64(X) X,
2812 #else
2813 # define WITH_ATOMIC64(X)
2814 #endif
2815
2816 static void * const table_cmpxchg[16] = {
2817     [MO_8] = gen_helper_atomic_cmpxchgb,
2818     [MO_16 | MO_LE] = gen_helper_atomic_cmpxchgw_le,
2819     [MO_16 | MO_BE] = gen_helper_atomic_cmpxchgw_be,
2820     [MO_32 | MO_LE] = gen_helper_atomic_cmpxchgl_le,
2821     [MO_32 | MO_BE] = gen_helper_atomic_cmpxchgl_be,
2822     WITH_ATOMIC64([MO_64 | MO_LE] = gen_helper_atomic_cmpxchgq_le)
2823     WITH_ATOMIC64([MO_64 | MO_BE] = gen_helper_atomic_cmpxchgq_be)
2824 };
2825
2826 void tcg_gen_atomic_cmpxchg_i32(TCGv_i32 retv, TCGv addr, TCGv_i32 cmpv,
2827                                 TCGv_i32 newv, TCGArg idx, TCGMemOp memop)
2828 {
2829     memop = tcg_canonicalize_memop(memop, 0, 0);
2830
2831     if (!(tcg_ctx->tb_cflags & CF_PARALLEL)) {
2832         TCGv_i32 t1 = tcg_temp_new_i32();
2833         TCGv_i32 t2 = tcg_temp_new_i32();
2834
2835         tcg_gen_ext_i32(t2, cmpv, memop & MO_SIZE);
2836
2837         tcg_gen_qemu_ld_i32(t1, addr, idx, memop & ~MO_SIGN);
2838         tcg_gen_movcond_i32(TCG_COND_EQ, t2, t1, t2, newv, t1);
2839         tcg_gen_qemu_st_i32(t2, addr, idx, memop);
2840         tcg_temp_free_i32(t2);
2841
2842         if (memop & MO_SIGN) {
2843             tcg_gen_ext_i32(retv, t1, memop);
2844         } else {
2845             tcg_gen_mov_i32(retv, t1);
2846         }
2847         tcg_temp_free_i32(t1);
2848     } else {
2849         gen_atomic_cx_i32 gen;
2850
2851         gen = table_cmpxchg[memop & (MO_SIZE | MO_BSWAP)];
2852         tcg_debug_assert(gen != NULL);
2853
2854 #ifdef CONFIG_SOFTMMU
2855         {
2856             TCGv_i32 oi = tcg_const_i32(make_memop_idx(memop & ~MO_SIGN, idx));
2857             gen(retv, cpu_env, addr, cmpv, newv, oi);
2858             tcg_temp_free_i32(oi);
2859         }
2860 #else
2861         gen(retv, cpu_env, addr, cmpv, newv);
2862 #endif
2863
2864         if (memop & MO_SIGN) {
2865             tcg_gen_ext_i32(retv, retv, memop);
2866         }
2867     }
2868 }
2869
2870 void tcg_gen_atomic_cmpxchg_i64(TCGv_i64 retv, TCGv addr, TCGv_i64 cmpv,
2871                                 TCGv_i64 newv, TCGArg idx, TCGMemOp memop)
2872 {
2873     memop = tcg_canonicalize_memop(memop, 1, 0);
2874
2875     if (!(tcg_ctx->tb_cflags & CF_PARALLEL)) {
2876         TCGv_i64 t1 = tcg_temp_new_i64();
2877         TCGv_i64 t2 = tcg_temp_new_i64();
2878
2879         tcg_gen_ext_i64(t2, cmpv, memop & MO_SIZE);
2880
2881         tcg_gen_qemu_ld_i64(t1, addr, idx, memop & ~MO_SIGN);
2882         tcg_gen_movcond_i64(TCG_COND_EQ, t2, t1, t2, newv, t1);
2883         tcg_gen_qemu_st_i64(t2, addr, idx, memop);
2884         tcg_temp_free_i64(t2);
2885
2886         if (memop & MO_SIGN) {
2887             tcg_gen_ext_i64(retv, t1, memop);
2888         } else {
2889             tcg_gen_mov_i64(retv, t1);
2890         }
2891         tcg_temp_free_i64(t1);
2892     } else if ((memop & MO_SIZE) == MO_64) {
2893 #ifdef CONFIG_ATOMIC64
2894         gen_atomic_cx_i64 gen;
2895
2896         gen = table_cmpxchg[memop & (MO_SIZE | MO_BSWAP)];
2897         tcg_debug_assert(gen != NULL);
2898
2899 #ifdef CONFIG_SOFTMMU
2900         {
2901             TCGv_i32 oi = tcg_const_i32(make_memop_idx(memop, idx));
2902             gen(retv, cpu_env, addr, cmpv, newv, oi);
2903             tcg_temp_free_i32(oi);
2904         }
2905 #else
2906         gen(retv, cpu_env, addr, cmpv, newv);
2907 #endif
2908 #else
2909         gen_helper_exit_atomic(cpu_env);
2910         /* Produce a result, so that we have a well-formed opcode stream
2911            with respect to uses of the result in the (dead) code following.  */
2912         tcg_gen_movi_i64(retv, 0);
2913 #endif /* CONFIG_ATOMIC64 */
2914     } else {
2915         TCGv_i32 c32 = tcg_temp_new_i32();
2916         TCGv_i32 n32 = tcg_temp_new_i32();
2917         TCGv_i32 r32 = tcg_temp_new_i32();
2918
2919         tcg_gen_extrl_i64_i32(c32, cmpv);
2920         tcg_gen_extrl_i64_i32(n32, newv);
2921         tcg_gen_atomic_cmpxchg_i32(r32, addr, c32, n32, idx, memop & ~MO_SIGN);
2922         tcg_temp_free_i32(c32);
2923         tcg_temp_free_i32(n32);
2924
2925         tcg_gen_extu_i32_i64(retv, r32);
2926         tcg_temp_free_i32(r32);
2927
2928         if (memop & MO_SIGN) {
2929             tcg_gen_ext_i64(retv, retv, memop);
2930         }
2931     }
2932 }
2933
2934 static void do_nonatomic_op_i32(TCGv_i32 ret, TCGv addr, TCGv_i32 val,
2935                                 TCGArg idx, TCGMemOp memop, bool new_val,
2936                                 void (*gen)(TCGv_i32, TCGv_i32, TCGv_i32))
2937 {
2938     TCGv_i32 t1 = tcg_temp_new_i32();
2939     TCGv_i32 t2 = tcg_temp_new_i32();
2940
2941     memop = tcg_canonicalize_memop(memop, 0, 0);
2942
2943     tcg_gen_qemu_ld_i32(t1, addr, idx, memop & ~MO_SIGN);
2944     gen(t2, t1, val);
2945     tcg_gen_qemu_st_i32(t2, addr, idx, memop);
2946
2947     tcg_gen_ext_i32(ret, (new_val ? t2 : t1), memop);
2948     tcg_temp_free_i32(t1);
2949     tcg_temp_free_i32(t2);
2950 }
2951
2952 static void do_atomic_op_i32(TCGv_i32 ret, TCGv addr, TCGv_i32 val,
2953                              TCGArg idx, TCGMemOp memop, void * const table[])
2954 {
2955     gen_atomic_op_i32 gen;
2956
2957     memop = tcg_canonicalize_memop(memop, 0, 0);
2958
2959     gen = table[memop & (MO_SIZE | MO_BSWAP)];
2960     tcg_debug_assert(gen != NULL);
2961
2962 #ifdef CONFIG_SOFTMMU
2963     {
2964         TCGv_i32 oi = tcg_const_i32(make_memop_idx(memop & ~MO_SIGN, idx));
2965         gen(ret, cpu_env, addr, val, oi);
2966         tcg_temp_free_i32(oi);
2967     }
2968 #else
2969     gen(ret, cpu_env, addr, val);
2970 #endif
2971
2972     if (memop & MO_SIGN) {
2973         tcg_gen_ext_i32(ret, ret, memop);
2974     }
2975 }
2976
2977 static void do_nonatomic_op_i64(TCGv_i64 ret, TCGv addr, TCGv_i64 val,
2978                                 TCGArg idx, TCGMemOp memop, bool new_val,
2979                                 void (*gen)(TCGv_i64, TCGv_i64, TCGv_i64))
2980 {
2981     TCGv_i64 t1 = tcg_temp_new_i64();
2982     TCGv_i64 t2 = tcg_temp_new_i64();
2983
2984     memop = tcg_canonicalize_memop(memop, 1, 0);
2985
2986     tcg_gen_qemu_ld_i64(t1, addr, idx, memop & ~MO_SIGN);
2987     gen(t2, t1, val);
2988     tcg_gen_qemu_st_i64(t2, addr, idx, memop);
2989
2990     tcg_gen_ext_i64(ret, (new_val ? t2 : t1), memop);
2991     tcg_temp_free_i64(t1);
2992     tcg_temp_free_i64(t2);
2993 }
2994
2995 static void do_atomic_op_i64(TCGv_i64 ret, TCGv addr, TCGv_i64 val,
2996                              TCGArg idx, TCGMemOp memop, void * const table[])
2997 {
2998     memop = tcg_canonicalize_memop(memop, 1, 0);
2999
3000     if ((memop & MO_SIZE) == MO_64) {
3001 #ifdef CONFIG_ATOMIC64
3002         gen_atomic_op_i64 gen;
3003
3004         gen = table[memop & (MO_SIZE | MO_BSWAP)];
3005         tcg_debug_assert(gen != NULL);
3006
3007 #ifdef CONFIG_SOFTMMU
3008         {
3009             TCGv_i32 oi = tcg_const_i32(make_memop_idx(memop & ~MO_SIGN, idx));
3010             gen(ret, cpu_env, addr, val, oi);
3011             tcg_temp_free_i32(oi);
3012         }
3013 #else
3014         gen(ret, cpu_env, addr, val);
3015 #endif
3016 #else
3017         gen_helper_exit_atomic(cpu_env);
3018         /* Produce a result, so that we have a well-formed opcode stream
3019            with respect to uses of the result in the (dead) code following.  */
3020         tcg_gen_movi_i64(ret, 0);
3021 #endif /* CONFIG_ATOMIC64 */
3022     } else {
3023         TCGv_i32 v32 = tcg_temp_new_i32();
3024         TCGv_i32 r32 = tcg_temp_new_i32();
3025
3026         tcg_gen_extrl_i64_i32(v32, val);
3027         do_atomic_op_i32(r32, addr, v32, idx, memop & ~MO_SIGN, table);
3028         tcg_temp_free_i32(v32);
3029
3030         tcg_gen_extu_i32_i64(ret, r32);
3031         tcg_temp_free_i32(r32);
3032
3033         if (memop & MO_SIGN) {
3034             tcg_gen_ext_i64(ret, ret, memop);
3035         }
3036     }
3037 }
3038
3039 #define GEN_ATOMIC_HELPER(NAME, OP, NEW)                                \
3040 static void * const table_##NAME[16] = {                                \
3041     [MO_8] = gen_helper_atomic_##NAME##b,                               \
3042     [MO_16 | MO_LE] = gen_helper_atomic_##NAME##w_le,                   \
3043     [MO_16 | MO_BE] = gen_helper_atomic_##NAME##w_be,                   \
3044     [MO_32 | MO_LE] = gen_helper_atomic_##NAME##l_le,                   \
3045     [MO_32 | MO_BE] = gen_helper_atomic_##NAME##l_be,                   \
3046     WITH_ATOMIC64([MO_64 | MO_LE] = gen_helper_atomic_##NAME##q_le)     \
3047     WITH_ATOMIC64([MO_64 | MO_BE] = gen_helper_atomic_##NAME##q_be)     \
3048 };                                                                      \
3049 void tcg_gen_atomic_##NAME##_i32                                        \
3050     (TCGv_i32 ret, TCGv addr, TCGv_i32 val, TCGArg idx, TCGMemOp memop) \
3051 {                                                                       \
3052     if (tcg_ctx->tb_cflags & CF_PARALLEL) {                             \
3053         do_atomic_op_i32(ret, addr, val, idx, memop, table_##NAME);     \
3054     } else {                                                            \
3055         do_nonatomic_op_i32(ret, addr, val, idx, memop, NEW,            \
3056                             tcg_gen_##OP##_i32);                        \
3057     }                                                                   \
3058 }                                                                       \
3059 void tcg_gen_atomic_##NAME##_i64                                        \
3060     (TCGv_i64 ret, TCGv addr, TCGv_i64 val, TCGArg idx, TCGMemOp memop) \
3061 {                                                                       \
3062     if (tcg_ctx->tb_cflags & CF_PARALLEL) {                             \
3063         do_atomic_op_i64(ret, addr, val, idx, memop, table_##NAME);     \
3064     } else {                                                            \
3065         do_nonatomic_op_i64(ret, addr, val, idx, memop, NEW,            \
3066                             tcg_gen_##OP##_i64);                        \
3067     }                                                                   \
3068 }
3069
3070 GEN_ATOMIC_HELPER(fetch_add, add, 0)
3071 GEN_ATOMIC_HELPER(fetch_and, and, 0)
3072 GEN_ATOMIC_HELPER(fetch_or, or, 0)
3073 GEN_ATOMIC_HELPER(fetch_xor, xor, 0)
3074 GEN_ATOMIC_HELPER(fetch_smin, smin, 0)
3075 GEN_ATOMIC_HELPER(fetch_umin, umin, 0)
3076 GEN_ATOMIC_HELPER(fetch_smax, smax, 0)
3077 GEN_ATOMIC_HELPER(fetch_umax, umax, 0)
3078
3079 GEN_ATOMIC_HELPER(add_fetch, add, 1)
3080 GEN_ATOMIC_HELPER(and_fetch, and, 1)
3081 GEN_ATOMIC_HELPER(or_fetch, or, 1)
3082 GEN_ATOMIC_HELPER(xor_fetch, xor, 1)
3083 GEN_ATOMIC_HELPER(smin_fetch, smin, 1)
3084 GEN_ATOMIC_HELPER(umin_fetch, umin, 1)
3085 GEN_ATOMIC_HELPER(smax_fetch, smax, 1)
3086 GEN_ATOMIC_HELPER(umax_fetch, umax, 1)
3087
3088 static void tcg_gen_mov2_i32(TCGv_i32 r, TCGv_i32 a, TCGv_i32 b)
3089 {
3090     tcg_gen_mov_i32(r, b);
3091 }
3092
3093 static void tcg_gen_mov2_i64(TCGv_i64 r, TCGv_i64 a, TCGv_i64 b)
3094 {
3095     tcg_gen_mov_i64(r, b);
3096 }
3097
3098 GEN_ATOMIC_HELPER(xchg, mov2, 0)
3099
3100 #undef GEN_ATOMIC_HELPER