tcg/tcg-op.c

   1 /*
   2  * Tiny Code Generator for QEMU
   3  *
   4  * Copyright (c) 2008 Fabrice Bellard
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a copy
   7  * of this software and associated documentation files (the "Software"), to deal
   8  * in the Software without restriction, including without limitation the rights
   9  * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
  10  * copies of the Software, and to permit persons to whom the Software is
  11  * furnished to do so, subject to the following conditions:
  12  *
  13  * The above copyright notice and this permission notice shall be included in
  14  * all copies or substantial portions of the Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
  19  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  20  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
  21  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
  22  * THE SOFTWARE.
  23  */
  24
  25 #include "qemu/osdep.h"
  26 #include "exec/exec-all.h"
  27 #include "tcg/tcg.h"
  28 #include "tcg/tcg-op.h"
  29 #include "tcg/tcg-mo.h"
  30 #include "trace-tcg.h"
  31 #include "trace/mem.h"
  32 #include "exec/plugin-gen.h"
  33
  34 /* Reduce the number of ifdefs below.  This assumes that all uses of
  35    TCGV_HIGH and TCGV_LOW are properly protected by a conditional that
  36    the compiler can eliminate.  */
  37 #if TCG_TARGET_REG_BITS == 64
  38 extern TCGv_i32 TCGV_LOW_link_error(TCGv_i64);
  39 extern TCGv_i32 TCGV_HIGH_link_error(TCGv_i64);
  40 #define TCGV_LOW  TCGV_LOW_link_error
  41 #define TCGV_HIGH TCGV_HIGH_link_error
  42 #endif
  43
  44 void tcg_gen_op1(TCGOpcode opc, TCGArg a1)
  45 {
  46     TCGOp *op = tcg_emit_op(opc);
  47     op->args[0] = a1;
  48 }
  49
  50 void tcg_gen_op2(TCGOpcode opc, TCGArg a1, TCGArg a2)
  51 {
  52     TCGOp *op = tcg_emit_op(opc);
  53     op->args[0] = a1;
  54     op->args[1] = a2;
  55 }
  56
  57 void tcg_gen_op3(TCGOpcode opc, TCGArg a1, TCGArg a2, TCGArg a3)
  58 {
  59     TCGOp *op = tcg_emit_op(opc);
  60     op->args[0] = a1;
  61     op->args[1] = a2;
  62     op->args[2] = a3;
  63 }
  64
  65 void tcg_gen_op4(TCGOpcode opc, TCGArg a1, TCGArg a2, TCGArg a3, TCGArg a4)
  66 {
  67     TCGOp *op = tcg_emit_op(opc);
  68     op->args[0] = a1;
  69     op->args[1] = a2;
  70     op->args[2] = a3;
  71     op->args[3] = a4;
  72 }
  73
  74 void tcg_gen_op5(TCGOpcode opc, TCGArg a1, TCGArg a2, TCGArg a3,
  75                  TCGArg a4, TCGArg a5)
  76 {
  77     TCGOp *op = tcg_emit_op(opc);
  78     op->args[0] = a1;
  79     op->args[1] = a2;
  80     op->args[2] = a3;
  81     op->args[3] = a4;
  82     op->args[4] = a5;
  83 }
  84
  85 void tcg_gen_op6(TCGOpcode opc, TCGArg a1, TCGArg a2, TCGArg a3,
  86                  TCGArg a4, TCGArg a5, TCGArg a6)
  87 {
  88     TCGOp *op = tcg_emit_op(opc);
  89     op->args[0] = a1;
  90     op->args[1] = a2;
  91     op->args[2] = a3;
  92     op->args[3] = a4;
  93     op->args[4] = a5;
  94     op->args[5] = a6;
  95 }
  96
  97 void tcg_gen_mb(TCGBar mb_type)
  98 {
  99     if (tcg_ctx->tb_cflags & CF_PARALLEL) {
 100         tcg_gen_op1(INDEX_op_mb, mb_type);
 101     }
 102 }
 103
 104 /* 32 bit ops */
 105
 106 void tcg_gen_movi_i32(TCGv_i32 ret, int32_t arg)
 107 {
 108     tcg_gen_mov_i32(ret, tcg_constant_i32(arg));
 109 }
 110
 111 void tcg_gen_addi_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
 112 {
 113     /* some cases can be optimized here */
 114     if (arg2 == 0) {
 115         tcg_gen_mov_i32(ret, arg1);
 116     } else {
 117         tcg_gen_add_i32(ret, arg1, tcg_constant_i32(arg2));
 118     }
 119 }
 120
 121 void tcg_gen_subfi_i32(TCGv_i32 ret, int32_t arg1, TCGv_i32 arg2)
 122 {
 123     if (arg1 == 0 && TCG_TARGET_HAS_neg_i32) {
 124         /* Don't recurse with tcg_gen_neg_i32.  */
 125         tcg_gen_op2_i32(INDEX_op_neg_i32, ret, arg2);
 126     } else {
 127         tcg_gen_sub_i32(ret, tcg_constant_i32(arg1), arg2);
 128     }
 129 }
 130
 131 void tcg_gen_subi_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
 132 {
 133     /* some cases can be optimized here */
 134     if (arg2 == 0) {
 135         tcg_gen_mov_i32(ret, arg1);
 136     } else {
 137         tcg_gen_sub_i32(ret, arg1, tcg_constant_i32(arg2));
 138     }
 139 }
 140
 141 void tcg_gen_andi_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
 142 {
 143     /* Some cases can be optimized here.  */
 144     switch (arg2) {
 145     case 0:
 146         tcg_gen_movi_i32(ret, 0);
 147         return;
 148     case -1:
 149         tcg_gen_mov_i32(ret, arg1);
 150         return;
 151     case 0xff:
 152         /* Don't recurse with tcg_gen_ext8u_i32.  */
 153         if (TCG_TARGET_HAS_ext8u_i32) {
 154             tcg_gen_op2_i32(INDEX_op_ext8u_i32, ret, arg1);
 155             return;
 156         }
 157         break;
 158     case 0xffff:
 159         if (TCG_TARGET_HAS_ext16u_i32) {
 160             tcg_gen_op2_i32(INDEX_op_ext16u_i32, ret, arg1);
 161             return;
 162         }
 163         break;
 164     }
 165
 166     tcg_gen_and_i32(ret, arg1, tcg_constant_i32(arg2));
 167 }
 168
 169 void tcg_gen_ori_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
 170 {
 171     /* Some cases can be optimized here.  */
 172     if (arg2 == -1) {
 173         tcg_gen_movi_i32(ret, -1);
 174     } else if (arg2 == 0) {
 175         tcg_gen_mov_i32(ret, arg1);
 176     } else {
 177         tcg_gen_or_i32(ret, arg1, tcg_constant_i32(arg2));
 178     }
 179 }
 180
 181 void tcg_gen_xori_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
 182 {
 183     /* Some cases can be optimized here.  */
 184     if (arg2 == 0) {
 185         tcg_gen_mov_i32(ret, arg1);
 186     } else if (arg2 == -1 && TCG_TARGET_HAS_not_i32) {
 187         /* Don't recurse with tcg_gen_not_i32.  */
 188         tcg_gen_op2_i32(INDEX_op_not_i32, ret, arg1);
 189     } else {
 190         tcg_gen_xor_i32(ret, arg1, tcg_constant_i32(arg2));
 191     }
 192 }
 193
 194 void tcg_gen_shli_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
 195 {
 196     tcg_debug_assert(arg2 >= 0 && arg2 < 32);
 197     if (arg2 == 0) {
 198         tcg_gen_mov_i32(ret, arg1);
 199     } else {
 200         tcg_gen_shl_i32(ret, arg1, tcg_constant_i32(arg2));
 201     }
 202 }
 203
 204 void tcg_gen_shri_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
 205 {
 206     tcg_debug_assert(arg2 >= 0 && arg2 < 32);
 207     if (arg2 == 0) {
 208         tcg_gen_mov_i32(ret, arg1);
 209     } else {
 210         tcg_gen_shr_i32(ret, arg1, tcg_constant_i32(arg2));
 211     }
 212 }
 213
 214 void tcg_gen_sari_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
 215 {
 216     tcg_debug_assert(arg2 >= 0 && arg2 < 32);
 217     if (arg2 == 0) {
 218         tcg_gen_mov_i32(ret, arg1);
 219     } else {
 220         tcg_gen_sar_i32(ret, arg1, tcg_constant_i32(arg2));
 221     }
 222 }
 223
 224 void tcg_gen_brcond_i32(TCGCond cond, TCGv_i32 arg1, TCGv_i32 arg2, TCGLabel *l)
 225 {
 226     if (cond == TCG_COND_ALWAYS) {
 227         tcg_gen_br(l);
 228     } else if (cond != TCG_COND_NEVER) {
 229         l->refs++;
 230         tcg_gen_op4ii_i32(INDEX_op_brcond_i32, arg1, arg2, cond, label_arg(l));
 231     }
 232 }
 233
 234 void tcg_gen_brcondi_i32(TCGCond cond, TCGv_i32 arg1, int32_t arg2, TCGLabel *l)
 235 {
 236     if (cond == TCG_COND_ALWAYS) {
 237         tcg_gen_br(l);
 238     } else if (cond != TCG_COND_NEVER) {
 239         tcg_gen_brcond_i32(cond, arg1, tcg_constant_i32(arg2), l);
 240     }
 241 }
 242
 243 void tcg_gen_setcond_i32(TCGCond cond, TCGv_i32 ret,
 244                          TCGv_i32 arg1, TCGv_i32 arg2)
 245 {
 246     if (cond == TCG_COND_ALWAYS) {
 247         tcg_gen_movi_i32(ret, 1);
 248     } else if (cond == TCG_COND_NEVER) {
 249         tcg_gen_movi_i32(ret, 0);
 250     } else {
 251         tcg_gen_op4i_i32(INDEX_op_setcond_i32, ret, arg1, arg2, cond);
 252     }
 253 }
 254
 255 void tcg_gen_setcondi_i32(TCGCond cond, TCGv_i32 ret,
 256                           TCGv_i32 arg1, int32_t arg2)
 257 {
 258     tcg_gen_setcond_i32(cond, ret, arg1, tcg_constant_i32(arg2));
 259 }
 260
 261 void tcg_gen_muli_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
 262 {
 263     if (arg2 == 0) {
 264         tcg_gen_movi_i32(ret, 0);
 265     } else if (is_power_of_2(arg2)) {
 266         tcg_gen_shli_i32(ret, arg1, ctz32(arg2));
 267     } else {
 268         tcg_gen_mul_i32(ret, arg1, tcg_constant_i32(arg2));
 269     }
 270 }
 271
 272 void tcg_gen_div_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
 273 {
 274     if (TCG_TARGET_HAS_div_i32) {
 275         tcg_gen_op3_i32(INDEX_op_div_i32, ret, arg1, arg2);
 276     } else if (TCG_TARGET_HAS_div2_i32) {
 277         TCGv_i32 t0 = tcg_temp_new_i32();
 278         tcg_gen_sari_i32(t0, arg1, 31);
 279         tcg_gen_op5_i32(INDEX_op_div2_i32, ret, t0, arg1, t0, arg2);
 280         tcg_temp_free_i32(t0);
 281     } else {
 282         gen_helper_div_i32(ret, arg1, arg2);
 283     }
 284 }
 285
 286 void tcg_gen_rem_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
 287 {
 288     if (TCG_TARGET_HAS_rem_i32) {
 289         tcg_gen_op3_i32(INDEX_op_rem_i32, ret, arg1, arg2);
 290     } else if (TCG_TARGET_HAS_div_i32) {
 291         TCGv_i32 t0 = tcg_temp_new_i32();
 292         tcg_gen_op3_i32(INDEX_op_div_i32, t0, arg1, arg2);
 293         tcg_gen_mul_i32(t0, t0, arg2);
 294         tcg_gen_sub_i32(ret, arg1, t0);
 295         tcg_temp_free_i32(t0);
 296     } else if (TCG_TARGET_HAS_div2_i32) {
 297         TCGv_i32 t0 = tcg_temp_new_i32();
 298         tcg_gen_sari_i32(t0, arg1, 31);
 299         tcg_gen_op5_i32(INDEX_op_div2_i32, t0, ret, arg1, t0, arg2);
 300         tcg_temp_free_i32(t0);
 301     } else {
 302         gen_helper_rem_i32(ret, arg1, arg2);
 303     }
 304 }
 305
 306 void tcg_gen_divu_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
 307 {
 308     if (TCG_TARGET_HAS_div_i32) {
 309         tcg_gen_op3_i32(INDEX_op_divu_i32, ret, arg1, arg2);
 310     } else if (TCG_TARGET_HAS_div2_i32) {
 311         TCGv_i32 t0 = tcg_temp_new_i32();
 312         tcg_gen_movi_i32(t0, 0);
 313         tcg_gen_op5_i32(INDEX_op_divu2_i32, ret, t0, arg1, t0, arg2);
 314         tcg_temp_free_i32(t0);
 315     } else {
 316         gen_helper_divu_i32(ret, arg1, arg2);
 317     }
 318 }
 319
 320 void tcg_gen_remu_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
 321 {
 322     if (TCG_TARGET_HAS_rem_i32) {
 323         tcg_gen_op3_i32(INDEX_op_remu_i32, ret, arg1, arg2);
 324     } else if (TCG_TARGET_HAS_div_i32) {
 325         TCGv_i32 t0 = tcg_temp_new_i32();
 326         tcg_gen_op3_i32(INDEX_op_divu_i32, t0, arg1, arg2);
 327         tcg_gen_mul_i32(t0, t0, arg2);
 328         tcg_gen_sub_i32(ret, arg1, t0);
 329         tcg_temp_free_i32(t0);
 330     } else if (TCG_TARGET_HAS_div2_i32) {
 331         TCGv_i32 t0 = tcg_temp_new_i32();
 332         tcg_gen_movi_i32(t0, 0);
 333         tcg_gen_op5_i32(INDEX_op_divu2_i32, t0, ret, arg1, t0, arg2);
 334         tcg_temp_free_i32(t0);
 335     } else {
 336         gen_helper_remu_i32(ret, arg1, arg2);
 337     }
 338 }
 339
 340 void tcg_gen_andc_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
 341 {
 342     if (TCG_TARGET_HAS_andc_i32) {
 343         tcg_gen_op3_i32(INDEX_op_andc_i32, ret, arg1, arg2);
 344     } else {
 345         TCGv_i32 t0 = tcg_temp_new_i32();
 346         tcg_gen_not_i32(t0, arg2);
 347         tcg_gen_and_i32(ret, arg1, t0);
 348         tcg_temp_free_i32(t0);
 349     }
 350 }
 351
 352 void tcg_gen_eqv_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
 353 {
 354     if (TCG_TARGET_HAS_eqv_i32) {
 355         tcg_gen_op3_i32(INDEX_op_eqv_i32, ret, arg1, arg2);
 356     } else {
 357         tcg_gen_xor_i32(ret, arg1, arg2);
 358         tcg_gen_not_i32(ret, ret);
 359     }
 360 }
 361
 362 void tcg_gen_nand_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
 363 {
 364     if (TCG_TARGET_HAS_nand_i32) {
 365         tcg_gen_op3_i32(INDEX_op_nand_i32, ret, arg1, arg2);
 366     } else {
 367         tcg_gen_and_i32(ret, arg1, arg2);
 368         tcg_gen_not_i32(ret, ret);
 369     }
 370 }
 371
 372 void tcg_gen_nor_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
 373 {
 374     if (TCG_TARGET_HAS_nor_i32) {
 375         tcg_gen_op3_i32(INDEX_op_nor_i32, ret, arg1, arg2);
 376     } else {
 377         tcg_gen_or_i32(ret, arg1, arg2);
 378         tcg_gen_not_i32(ret, ret);
 379     }
 380 }
 381
 382 void tcg_gen_orc_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
 383 {
 384     if (TCG_TARGET_HAS_orc_i32) {
 385         tcg_gen_op3_i32(INDEX_op_orc_i32, ret, arg1, arg2);
 386     } else {
 387         TCGv_i32 t0 = tcg_temp_new_i32();
 388         tcg_gen_not_i32(t0, arg2);
 389         tcg_gen_or_i32(ret, arg1, t0);
 390         tcg_temp_free_i32(t0);
 391     }
 392 }
 393
 394 void tcg_gen_clz_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
 395 {
 396     if (TCG_TARGET_HAS_clz_i32) {
 397         tcg_gen_op3_i32(INDEX_op_clz_i32, ret, arg1, arg2);
 398     } else if (TCG_TARGET_HAS_clz_i64) {
 399         TCGv_i64 t1 = tcg_temp_new_i64();
 400         TCGv_i64 t2 = tcg_temp_new_i64();
 401         tcg_gen_extu_i32_i64(t1, arg1);
 402         tcg_gen_extu_i32_i64(t2, arg2);
 403         tcg_gen_addi_i64(t2, t2, 32);
 404         tcg_gen_clz_i64(t1, t1, t2);
 405         tcg_gen_extrl_i64_i32(ret, t1);
 406         tcg_temp_free_i64(t1);
 407         tcg_temp_free_i64(t2);
 408         tcg_gen_subi_i32(ret, ret, 32);
 409     } else {
 410         gen_helper_clz_i32(ret, arg1, arg2);
 411     }
 412 }
 413
 414 void tcg_gen_clzi_i32(TCGv_i32 ret, TCGv_i32 arg1, uint32_t arg2)
 415 {
 416     tcg_gen_clz_i32(ret, arg1, tcg_constant_i32(arg2));
 417 }
 418
 419 void tcg_gen_ctz_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
 420 {
 421     if (TCG_TARGET_HAS_ctz_i32) {
 422         tcg_gen_op3_i32(INDEX_op_ctz_i32, ret, arg1, arg2);
 423     } else if (TCG_TARGET_HAS_ctz_i64) {
 424         TCGv_i64 t1 = tcg_temp_new_i64();
 425         TCGv_i64 t2 = tcg_temp_new_i64();
 426         tcg_gen_extu_i32_i64(t1, arg1);
 427         tcg_gen_extu_i32_i64(t2, arg2);
 428         tcg_gen_ctz_i64(t1, t1, t2);
 429         tcg_gen_extrl_i64_i32(ret, t1);
 430         tcg_temp_free_i64(t1);
 431         tcg_temp_free_i64(t2);
 432     } else if (TCG_TARGET_HAS_ctpop_i32
 433                || TCG_TARGET_HAS_ctpop_i64
 434                || TCG_TARGET_HAS_clz_i32
 435                || TCG_TARGET_HAS_clz_i64) {
 436         TCGv_i32 z, t = tcg_temp_new_i32();
 437
 438         if (TCG_TARGET_HAS_ctpop_i32 || TCG_TARGET_HAS_ctpop_i64) {
 439             tcg_gen_subi_i32(t, arg1, 1);
 440             tcg_gen_andc_i32(t, t, arg1);
 441             tcg_gen_ctpop_i32(t, t);
 442         } else {
 443             /* Since all non-x86 hosts have clz(0) == 32, don't fight it.  */
 444             tcg_gen_neg_i32(t, arg1);
 445             tcg_gen_and_i32(t, t, arg1);
 446             tcg_gen_clzi_i32(t, t, 32);
 447             tcg_gen_xori_i32(t, t, 31);
 448         }
 449         z = tcg_constant_i32(0);
 450         tcg_gen_movcond_i32(TCG_COND_EQ, ret, arg1, z, arg2, t);
 451         tcg_temp_free_i32(t);
 452     } else {
 453         gen_helper_ctz_i32(ret, arg1, arg2);
 454     }
 455 }
 456
 457 void tcg_gen_ctzi_i32(TCGv_i32 ret, TCGv_i32 arg1, uint32_t arg2)
 458 {
 459     if (!TCG_TARGET_HAS_ctz_i32 && TCG_TARGET_HAS_ctpop_i32 && arg2 == 32) {
 460         /* This equivalence has the advantage of not requiring a fixup.  */
 461         TCGv_i32 t = tcg_temp_new_i32();
 462         tcg_gen_subi_i32(t, arg1, 1);
 463         tcg_gen_andc_i32(t, t, arg1);
 464         tcg_gen_ctpop_i32(ret, t);
 465         tcg_temp_free_i32(t);
 466     } else {
 467         tcg_gen_ctz_i32(ret, arg1, tcg_constant_i32(arg2));
 468     }
 469 }
 470
 471 void tcg_gen_clrsb_i32(TCGv_i32 ret, TCGv_i32 arg)
 472 {
 473     if (TCG_TARGET_HAS_clz_i32) {
 474         TCGv_i32 t = tcg_temp_new_i32();
 475         tcg_gen_sari_i32(t, arg, 31);
 476         tcg_gen_xor_i32(t, t, arg);
 477         tcg_gen_clzi_i32(t, t, 32);
 478         tcg_gen_subi_i32(ret, t, 1);
 479         tcg_temp_free_i32(t);
 480     } else {
 481         gen_helper_clrsb_i32(ret, arg);
 482     }
 483 }
 484
 485 void tcg_gen_ctpop_i32(TCGv_i32 ret, TCGv_i32 arg1)
 486 {
 487     if (TCG_TARGET_HAS_ctpop_i32) {
 488         tcg_gen_op2_i32(INDEX_op_ctpop_i32, ret, arg1);
 489     } else if (TCG_TARGET_HAS_ctpop_i64) {
 490         TCGv_i64 t = tcg_temp_new_i64();
 491         tcg_gen_extu_i32_i64(t, arg1);
 492         tcg_gen_ctpop_i64(t, t);
 493         tcg_gen_extrl_i64_i32(ret, t);
 494         tcg_temp_free_i64(t);
 495     } else {
 496         gen_helper_ctpop_i32(ret, arg1);
 497     }
 498 }
 499
 500 void tcg_gen_rotl_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
 501 {
 502     if (TCG_TARGET_HAS_rot_i32) {
 503         tcg_gen_op3_i32(INDEX_op_rotl_i32, ret, arg1, arg2);
 504     } else {
 505         TCGv_i32 t0, t1;
 506
 507         t0 = tcg_temp_new_i32();
 508         t1 = tcg_temp_new_i32();
 509         tcg_gen_shl_i32(t0, arg1, arg2);
 510         tcg_gen_subfi_i32(t1, 32, arg2);
 511         tcg_gen_shr_i32(t1, arg1, t1);
 512         tcg_gen_or_i32(ret, t0, t1);
 513         tcg_temp_free_i32(t0);
 514         tcg_temp_free_i32(t1);
 515     }
 516 }
 517
 518 void tcg_gen_rotli_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
 519 {
 520     tcg_debug_assert(arg2 >= 0 && arg2 < 32);
 521     /* some cases can be optimized here */
 522     if (arg2 == 0) {
 523         tcg_gen_mov_i32(ret, arg1);
 524     } else if (TCG_TARGET_HAS_rot_i32) {
 525         tcg_gen_rotl_i32(ret, arg1, tcg_constant_i32(arg2));
 526     } else {
 527         TCGv_i32 t0, t1;
 528         t0 = tcg_temp_new_i32();
 529         t1 = tcg_temp_new_i32();
 530         tcg_gen_shli_i32(t0, arg1, arg2);
 531         tcg_gen_shri_i32(t1, arg1, 32 - arg2);
 532         tcg_gen_or_i32(ret, t0, t1);
 533         tcg_temp_free_i32(t0);
 534         tcg_temp_free_i32(t1);
 535     }
 536 }
 537
 538 void tcg_gen_rotr_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
 539 {
 540     if (TCG_TARGET_HAS_rot_i32) {
 541         tcg_gen_op3_i32(INDEX_op_rotr_i32, ret, arg1, arg2);
 542     } else {
 543         TCGv_i32 t0, t1;
 544
 545         t0 = tcg_temp_new_i32();
 546         t1 = tcg_temp_new_i32();
 547         tcg_gen_shr_i32(t0, arg1, arg2);
 548         tcg_gen_subfi_i32(t1, 32, arg2);
 549         tcg_gen_shl_i32(t1, arg1, t1);
 550         tcg_gen_or_i32(ret, t0, t1);
 551         tcg_temp_free_i32(t0);
 552         tcg_temp_free_i32(t1);
 553     }
 554 }
 555
 556 void tcg_gen_rotri_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
 557 {
 558     tcg_debug_assert(arg2 >= 0 && arg2 < 32);
 559     /* some cases can be optimized here */
 560     if (arg2 == 0) {
 561         tcg_gen_mov_i32(ret, arg1);
 562     } else {
 563         tcg_gen_rotli_i32(ret, arg1, 32 - arg2);
 564     }
 565 }
 566
 567 void tcg_gen_deposit_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2,
 568                          unsigned int ofs, unsigned int len)
 569 {
 570     uint32_t mask;
 571     TCGv_i32 t1;
 572
 573     tcg_debug_assert(ofs < 32);
 574     tcg_debug_assert(len > 0);
 575     tcg_debug_assert(len <= 32);
 576     tcg_debug_assert(ofs + len <= 32);
 577
 578     if (len == 32) {
 579         tcg_gen_mov_i32(ret, arg2);
 580         return;
 581     }
 582     if (TCG_TARGET_HAS_deposit_i32 && TCG_TARGET_deposit_i32_valid(ofs, len)) {
 583         tcg_gen_op5ii_i32(INDEX_op_deposit_i32, ret, arg1, arg2, ofs, len);
 584         return;
 585     }
 586
 587     t1 = tcg_temp_new_i32();
 588
 589     if (TCG_TARGET_HAS_extract2_i32) {
 590         if (ofs + len == 32) {
 591             tcg_gen_shli_i32(t1, arg1, len);
 592             tcg_gen_extract2_i32(ret, t1, arg2, len);
 593             goto done;
 594         }
 595         if (ofs == 0) {
 596             tcg_gen_extract2_i32(ret, arg1, arg2, len);
 597             tcg_gen_rotli_i32(ret, ret, len);
 598             goto done;
 599         }
 600     }
 601
 602     mask = (1u << len) - 1;
 603     if (ofs + len < 32) {
 604         tcg_gen_andi_i32(t1, arg2, mask);
 605         tcg_gen_shli_i32(t1, t1, ofs);
 606     } else {
 607         tcg_gen_shli_i32(t1, arg2, ofs);
 608     }
 609     tcg_gen_andi_i32(ret, arg1, ~(mask << ofs));
 610     tcg_gen_or_i32(ret, ret, t1);
 611  done:
 612     tcg_temp_free_i32(t1);
 613 }
 614
 615 void tcg_gen_deposit_z_i32(TCGv_i32 ret, TCGv_i32 arg,
 616                            unsigned int ofs, unsigned int len)
 617 {
 618     tcg_debug_assert(ofs < 32);
 619     tcg_debug_assert(len > 0);
 620     tcg_debug_assert(len <= 32);
 621     tcg_debug_assert(ofs + len <= 32);
 622
 623     if (ofs + len == 32) {
 624         tcg_gen_shli_i32(ret, arg, ofs);
 625     } else if (ofs == 0) {
 626         tcg_gen_andi_i32(ret, arg, (1u << len) - 1);
 627     } else if (TCG_TARGET_HAS_deposit_i32
 628                && TCG_TARGET_deposit_i32_valid(ofs, len)) {
 629         TCGv_i32 zero = tcg_constant_i32(0);
 630         tcg_gen_op5ii_i32(INDEX_op_deposit_i32, ret, zero, arg, ofs, len);
 631     } else {
 632         /* To help two-operand hosts we prefer to zero-extend first,
 633            which allows ARG to stay live.  */
 634         switch (len) {
 635         case 16:
 636             if (TCG_TARGET_HAS_ext16u_i32) {
 637                 tcg_gen_ext16u_i32(ret, arg);
 638                 tcg_gen_shli_i32(ret, ret, ofs);
 639                 return;
 640             }
 641             break;
 642         case 8:
 643             if (TCG_TARGET_HAS_ext8u_i32) {
 644                 tcg_gen_ext8u_i32(ret, arg);
 645                 tcg_gen_shli_i32(ret, ret, ofs);
 646                 return;
 647             }
 648             break;
 649         }
 650         /* Otherwise prefer zero-extension over AND for code size.  */
 651         switch (ofs + len) {
 652         case 16:
 653             if (TCG_TARGET_HAS_ext16u_i32) {
 654                 tcg_gen_shli_i32(ret, arg, ofs);
 655                 tcg_gen_ext16u_i32(ret, ret);
 656                 return;
 657             }
 658             break;
 659         case 8:
 660             if (TCG_TARGET_HAS_ext8u_i32) {
 661                 tcg_gen_shli_i32(ret, arg, ofs);
 662                 tcg_gen_ext8u_i32(ret, ret);
 663                 return;
 664             }
 665             break;
 666         }
 667         tcg_gen_andi_i32(ret, arg, (1u << len) - 1);
 668         tcg_gen_shli_i32(ret, ret, ofs);
 669     }
 670 }
 671
 672 void tcg_gen_extract_i32(TCGv_i32 ret, TCGv_i32 arg,
 673                          unsigned int ofs, unsigned int len)
 674 {
 675     tcg_debug_assert(ofs < 32);
 676     tcg_debug_assert(len > 0);
 677     tcg_debug_assert(len <= 32);
 678     tcg_debug_assert(ofs + len <= 32);
 679
 680     /* Canonicalize certain special cases, even if extract is supported.  */
 681     if (ofs + len == 32) {
 682         tcg_gen_shri_i32(ret, arg, 32 - len);
 683         return;
 684     }
 685     if (ofs == 0) {
 686         tcg_gen_andi_i32(ret, arg, (1u << len) - 1);
 687         return;
 688     }
 689
 690     if (TCG_TARGET_HAS_extract_i32
 691         && TCG_TARGET_extract_i32_valid(ofs, len)) {
 692         tcg_gen_op4ii_i32(INDEX_op_extract_i32, ret, arg, ofs, len);
 693         return;
 694     }
 695
 696     /* Assume that zero-extension, if available, is cheaper than a shift.  */
 697     switch (ofs + len) {
 698     case 16:
 699         if (TCG_TARGET_HAS_ext16u_i32) {
 700             tcg_gen_ext16u_i32(ret, arg);
 701             tcg_gen_shri_i32(ret, ret, ofs);
 702             return;
 703         }
 704         break;
 705     case 8:
 706         if (TCG_TARGET_HAS_ext8u_i32) {
 707             tcg_gen_ext8u_i32(ret, arg);
 708             tcg_gen_shri_i32(ret, ret, ofs);
 709             return;
 710         }
 711         break;
 712     }
 713
 714     /* ??? Ideally we'd know what values are available for immediate AND.
 715        Assume that 8 bits are available, plus the special case of 16,
 716        so that we get ext8u, ext16u.  */
 717     switch (len) {
 718     case 1 ... 8: case 16:
 719         tcg_gen_shri_i32(ret, arg, ofs);
 720         tcg_gen_andi_i32(ret, ret, (1u << len) - 1);
 721         break;
 722     default:
 723         tcg_gen_shli_i32(ret, arg, 32 - len - ofs);
 724         tcg_gen_shri_i32(ret, ret, 32 - len);
 725         break;
 726     }
 727 }
 728
 729 void tcg_gen_sextract_i32(TCGv_i32 ret, TCGv_i32 arg,
 730                           unsigned int ofs, unsigned int len)
 731 {
 732     tcg_debug_assert(ofs < 32);
 733     tcg_debug_assert(len > 0);
 734     tcg_debug_assert(len <= 32);
 735     tcg_debug_assert(ofs + len <= 32);
 736
 737     /* Canonicalize certain special cases, even if extract is supported.  */
 738     if (ofs + len == 32) {
 739         tcg_gen_sari_i32(ret, arg, 32 - len);
 740         return;
 741     }
 742     if (ofs == 0) {
 743         switch (len) {
 744         case 16:
 745             tcg_gen_ext16s_i32(ret, arg);
 746             return;
 747         case 8:
 748             tcg_gen_ext8s_i32(ret, arg);
 749             return;
 750         }
 751     }
 752
 753     if (TCG_TARGET_HAS_sextract_i32
 754         && TCG_TARGET_extract_i32_valid(ofs, len)) {
 755         tcg_gen_op4ii_i32(INDEX_op_sextract_i32, ret, arg, ofs, len);
 756         return;
 757     }
 758
 759     /* Assume that sign-extension, if available, is cheaper than a shift.  */
 760     switch (ofs + len) {
 761     case 16:
 762         if (TCG_TARGET_HAS_ext16s_i32) {
 763             tcg_gen_ext16s_i32(ret, arg);
 764             tcg_gen_sari_i32(ret, ret, ofs);
 765             return;
 766         }
 767         break;
 768     case 8:
 769         if (TCG_TARGET_HAS_ext8s_i32) {
 770             tcg_gen_ext8s_i32(ret, arg);
 771             tcg_gen_sari_i32(ret, ret, ofs);
 772             return;
 773         }
 774         break;
 775     }
 776     switch (len) {
 777     case 16:
 778         if (TCG_TARGET_HAS_ext16s_i32) {
 779             tcg_gen_shri_i32(ret, arg, ofs);
 780             tcg_gen_ext16s_i32(ret, ret);
 781             return;
 782         }
 783         break;
 784     case 8:
 785         if (TCG_TARGET_HAS_ext8s_i32) {
 786             tcg_gen_shri_i32(ret, arg, ofs);
 787             tcg_gen_ext8s_i32(ret, ret);
 788             return;
 789         }
 790         break;
 791     }
 792
 793     tcg_gen_shli_i32(ret, arg, 32 - len - ofs);
 794     tcg_gen_sari_i32(ret, ret, 32 - len);
 795 }
 796
 797 /*
 798  * Extract 32-bits from a 64-bit input, ah:al, starting from ofs.
 799  * Unlike tcg_gen_extract_i32 above, len is fixed at 32.
 800  */
 801 void tcg_gen_extract2_i32(TCGv_i32 ret, TCGv_i32 al, TCGv_i32 ah,
 802                           unsigned int ofs)
 803 {
 804     tcg_debug_assert(ofs <= 32);
 805     if (ofs == 0) {
 806         tcg_gen_mov_i32(ret, al);
 807     } else if (ofs == 32) {
 808         tcg_gen_mov_i32(ret, ah);
 809     } else if (al == ah) {
 810         tcg_gen_rotri_i32(ret, al, ofs);
 811     } else if (TCG_TARGET_HAS_extract2_i32) {
 812         tcg_gen_op4i_i32(INDEX_op_extract2_i32, ret, al, ah, ofs);
 813     } else {
 814         TCGv_i32 t0 = tcg_temp_new_i32();
 815         tcg_gen_shri_i32(t0, al, ofs);
 816         tcg_gen_deposit_i32(ret, t0, ah, 32 - ofs, ofs);
 817         tcg_temp_free_i32(t0);
 818     }
 819 }
 820
 821 void tcg_gen_movcond_i32(TCGCond cond, TCGv_i32 ret, TCGv_i32 c1,
 822                          TCGv_i32 c2, TCGv_i32 v1, TCGv_i32 v2)
 823 {
 824     if (cond == TCG_COND_ALWAYS) {
 825         tcg_gen_mov_i32(ret, v1);
 826     } else if (cond == TCG_COND_NEVER) {
 827         tcg_gen_mov_i32(ret, v2);
 828     } else if (TCG_TARGET_HAS_movcond_i32) {
 829         tcg_gen_op6i_i32(INDEX_op_movcond_i32, ret, c1, c2, v1, v2, cond);
 830     } else {
 831         TCGv_i32 t0 = tcg_temp_new_i32();
 832         TCGv_i32 t1 = tcg_temp_new_i32();
 833         tcg_gen_setcond_i32(cond, t0, c1, c2);
 834         tcg_gen_neg_i32(t0, t0);
 835         tcg_gen_and_i32(t1, v1, t0);
 836         tcg_gen_andc_i32(ret, v2, t0);
 837         tcg_gen_or_i32(ret, ret, t1);
 838         tcg_temp_free_i32(t0);
 839         tcg_temp_free_i32(t1);
 840     }
 841 }
 842
 843 void tcg_gen_add2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 al,
 844                       TCGv_i32 ah, TCGv_i32 bl, TCGv_i32 bh)
 845 {
 846     if (TCG_TARGET_HAS_add2_i32) {
 847         tcg_gen_op6_i32(INDEX_op_add2_i32, rl, rh, al, ah, bl, bh);
 848     } else {
 849         TCGv_i64 t0 = tcg_temp_new_i64();
 850         TCGv_i64 t1 = tcg_temp_new_i64();
 851         tcg_gen_concat_i32_i64(t0, al, ah);
 852         tcg_gen_concat_i32_i64(t1, bl, bh);
 853         tcg_gen_add_i64(t0, t0, t1);
 854         tcg_gen_extr_i64_i32(rl, rh, t0);
 855         tcg_temp_free_i64(t0);
 856         tcg_temp_free_i64(t1);
 857     }
 858 }
 859
 860 void tcg_gen_sub2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 al,
 861                       TCGv_i32 ah, TCGv_i32 bl, TCGv_i32 bh)
 862 {
 863     if (TCG_TARGET_HAS_sub2_i32) {
 864         tcg_gen_op6_i32(INDEX_op_sub2_i32, rl, rh, al, ah, bl, bh);
 865     } else {
 866         TCGv_i64 t0 = tcg_temp_new_i64();
 867         TCGv_i64 t1 = tcg_temp_new_i64();
 868         tcg_gen_concat_i32_i64(t0, al, ah);
 869         tcg_gen_concat_i32_i64(t1, bl, bh);
 870         tcg_gen_sub_i64(t0, t0, t1);
 871         tcg_gen_extr_i64_i32(rl, rh, t0);
 872         tcg_temp_free_i64(t0);
 873         tcg_temp_free_i64(t1);
 874     }
 875 }
 876
 877 void tcg_gen_mulu2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 arg1, TCGv_i32 arg2)
 878 {
 879     if (TCG_TARGET_HAS_mulu2_i32) {
 880         tcg_gen_op4_i32(INDEX_op_mulu2_i32, rl, rh, arg1, arg2);
 881     } else if (TCG_TARGET_HAS_muluh_i32) {
 882         TCGv_i32 t = tcg_temp_new_i32();
 883         tcg_gen_op3_i32(INDEX_op_mul_i32, t, arg1, arg2);
 884         tcg_gen_op3_i32(INDEX_op_muluh_i32, rh, arg1, arg2);
 885         tcg_gen_mov_i32(rl, t);
 886         tcg_temp_free_i32(t);
 887     } else {
 888         TCGv_i64 t0 = tcg_temp_new_i64();
 889         TCGv_i64 t1 = tcg_temp_new_i64();
 890         tcg_gen_extu_i32_i64(t0, arg1);
 891         tcg_gen_extu_i32_i64(t1, arg2);
 892         tcg_gen_mul_i64(t0, t0, t1);
 893         tcg_gen_extr_i64_i32(rl, rh, t0);
 894         tcg_temp_free_i64(t0);
 895         tcg_temp_free_i64(t1);
 896     }
 897 }
 898
 899 void tcg_gen_muls2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 arg1, TCGv_i32 arg2)
 900 {
 901     if (TCG_TARGET_HAS_muls2_i32) {
 902         tcg_gen_op4_i32(INDEX_op_muls2_i32, rl, rh, arg1, arg2);
 903     } else if (TCG_TARGET_HAS_mulsh_i32) {
 904         TCGv_i32 t = tcg_temp_new_i32();
 905         tcg_gen_op3_i32(INDEX_op_mul_i32, t, arg1, arg2);
 906         tcg_gen_op3_i32(INDEX_op_mulsh_i32, rh, arg1, arg2);
 907         tcg_gen_mov_i32(rl, t);
 908         tcg_temp_free_i32(t);
 909     } else if (TCG_TARGET_REG_BITS == 32) {
 910         TCGv_i32 t0 = tcg_temp_new_i32();
 911         TCGv_i32 t1 = tcg_temp_new_i32();
 912         TCGv_i32 t2 = tcg_temp_new_i32();
 913         TCGv_i32 t3 = tcg_temp_new_i32();
 914         tcg_gen_mulu2_i32(t0, t1, arg1, arg2);
 915         /* Adjust for negative inputs.  */
 916         tcg_gen_sari_i32(t2, arg1, 31);
 917         tcg_gen_sari_i32(t3, arg2, 31);
 918         tcg_gen_and_i32(t2, t2, arg2);
 919         tcg_gen_and_i32(t3, t3, arg1);
 920         tcg_gen_sub_i32(rh, t1, t2);
 921         tcg_gen_sub_i32(rh, rh, t3);
 922         tcg_gen_mov_i32(rl, t0);
 923         tcg_temp_free_i32(t0);
 924         tcg_temp_free_i32(t1);
 925         tcg_temp_free_i32(t2);
 926         tcg_temp_free_i32(t3);
 927     } else {
 928         TCGv_i64 t0 = tcg_temp_new_i64();
 929         TCGv_i64 t1 = tcg_temp_new_i64();
 930         tcg_gen_ext_i32_i64(t0, arg1);
 931         tcg_gen_ext_i32_i64(t1, arg2);
 932         tcg_gen_mul_i64(t0, t0, t1);
 933         tcg_gen_extr_i64_i32(rl, rh, t0);
 934         tcg_temp_free_i64(t0);
 935         tcg_temp_free_i64(t1);
 936     }
 937 }
 938
 939 void tcg_gen_mulsu2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 arg1, TCGv_i32 arg2)
 940 {
 941     if (TCG_TARGET_REG_BITS == 32) {
 942         TCGv_i32 t0 = tcg_temp_new_i32();
 943         TCGv_i32 t1 = tcg_temp_new_i32();
 944         TCGv_i32 t2 = tcg_temp_new_i32();
 945         tcg_gen_mulu2_i32(t0, t1, arg1, arg2);
 946         /* Adjust for negative input for the signed arg1.  */
 947         tcg_gen_sari_i32(t2, arg1, 31);
 948         tcg_gen_and_i32(t2, t2, arg2);
 949         tcg_gen_sub_i32(rh, t1, t2);
 950         tcg_gen_mov_i32(rl, t0);
 951         tcg_temp_free_i32(t0);
 952         tcg_temp_free_i32(t1);
 953         tcg_temp_free_i32(t2);
 954     } else {
 955         TCGv_i64 t0 = tcg_temp_new_i64();
 956         TCGv_i64 t1 = tcg_temp_new_i64();
 957         tcg_gen_ext_i32_i64(t0, arg1);
 958         tcg_gen_extu_i32_i64(t1, arg2);
 959         tcg_gen_mul_i64(t0, t0, t1);
 960         tcg_gen_extr_i64_i32(rl, rh, t0);
 961         tcg_temp_free_i64(t0);
 962         tcg_temp_free_i64(t1);
 963     }
 964 }
 965
 966 void tcg_gen_ext8s_i32(TCGv_i32 ret, TCGv_i32 arg)
 967 {
 968     if (TCG_TARGET_HAS_ext8s_i32) {
 969         tcg_gen_op2_i32(INDEX_op_ext8s_i32, ret, arg);
 970     } else {
 971         tcg_gen_shli_i32(ret, arg, 24);
 972         tcg_gen_sari_i32(ret, ret, 24);
 973     }
 974 }
 975
 976 void tcg_gen_ext16s_i32(TCGv_i32 ret, TCGv_i32 arg)
 977 {
 978     if (TCG_TARGET_HAS_ext16s_i32) {
 979         tcg_gen_op2_i32(INDEX_op_ext16s_i32, ret, arg);
 980     } else {
 981         tcg_gen_shli_i32(ret, arg, 16);
 982         tcg_gen_sari_i32(ret, ret, 16);
 983     }
 984 }
 985
 986 void tcg_gen_ext8u_i32(TCGv_i32 ret, TCGv_i32 arg)
 987 {
 988     if (TCG_TARGET_HAS_ext8u_i32) {
 989         tcg_gen_op2_i32(INDEX_op_ext8u_i32, ret, arg);
 990     } else {
 991         tcg_gen_andi_i32(ret, arg, 0xffu);
 992     }
 993 }
 994
 995 void tcg_gen_ext16u_i32(TCGv_i32 ret, TCGv_i32 arg)
 996 {
 997     if (TCG_TARGET_HAS_ext16u_i32) {
 998         tcg_gen_op2_i32(INDEX_op_ext16u_i32, ret, arg);
 999     } else {
1000         tcg_gen_andi_i32(ret, arg, 0xffffu);
1001     }
1002 }
1003
1004 void tcg_gen_bswap16_i32(TCGv_i32 ret, TCGv_i32 arg, int flags)
1005 {
1006     /* Only one extension flag may be present. */
1007     tcg_debug_assert(!(flags & TCG_BSWAP_OS) || !(flags & TCG_BSWAP_OZ));
1008
1009     if (TCG_TARGET_HAS_bswap16_i32) {
1010         tcg_gen_op3i_i32(INDEX_op_bswap16_i32, ret, arg, flags);
1011     } else {
1012         TCGv_i32 t0 = tcg_temp_new_i32();
1013         TCGv_i32 t1 = tcg_temp_new_i32();
1014
1015         tcg_gen_shri_i32(t0, arg, 8);
1016         if (!(flags & TCG_BSWAP_IZ)) {
1017             tcg_gen_ext8u_i32(t0, t0);
1018         }
1019
1020         if (flags & TCG_BSWAP_OS) {
1021             tcg_gen_shli_i32(t1, arg, 24);
1022             tcg_gen_sari_i32(t1, t1, 16);
1023         } else if (flags & TCG_BSWAP_OZ) {
1024             tcg_gen_ext8u_i32(t1, arg);
1025             tcg_gen_shli_i32(t1, t1, 8);
1026         } else {
1027             tcg_gen_shli_i32(t1, arg, 8);
1028         }
1029
1030         tcg_gen_or_i32(ret, t0, t1);
1031         tcg_temp_free_i32(t0);
1032         tcg_temp_free_i32(t1);
1033     }
1034 }
1035
1036 void tcg_gen_bswap32_i32(TCGv_i32 ret, TCGv_i32 arg)
1037 {
1038     if (TCG_TARGET_HAS_bswap32_i32) {
1039         tcg_gen_op3i_i32(INDEX_op_bswap32_i32, ret, arg, 0);
1040     } else {
1041         TCGv_i32 t0 = tcg_temp_new_i32();
1042         TCGv_i32 t1 = tcg_temp_new_i32();
1043         TCGv_i32 t2 = tcg_constant_i32(0x00ff00ff);
1044
1045                                         /* arg = abcd */
1046         tcg_gen_shri_i32(t0, arg, 8);   /*  t0 = .abc */
1047         tcg_gen_and_i32(t1, arg, t2);   /*  t1 = .b.d */
1048         tcg_gen_and_i32(t0, t0, t2);    /*  t0 = .a.c */
1049         tcg_gen_shli_i32(t1, t1, 8);    /*  t1 = b.d. */
1050         tcg_gen_or_i32(ret, t0, t1);    /* ret = badc */
1051
1052         tcg_gen_shri_i32(t0, ret, 16);  /*  t0 = ..ba */
1053         tcg_gen_shli_i32(t1, ret, 16);  /*  t1 = dc.. */
1054         tcg_gen_or_i32(ret, t0, t1);    /* ret = dcba */
1055
1056         tcg_temp_free_i32(t0);
1057         tcg_temp_free_i32(t1);
1058     }
1059 }
1060
1061 void tcg_gen_smin_i32(TCGv_i32 ret, TCGv_i32 a, TCGv_i32 b)
1062 {
1063     tcg_gen_movcond_i32(TCG_COND_LT, ret, a, b, a, b);
1064 }
1065
1066 void tcg_gen_umin_i32(TCGv_i32 ret, TCGv_i32 a, TCGv_i32 b)
1067 {
1068     tcg_gen_movcond_i32(TCG_COND_LTU, ret, a, b, a, b);
1069 }
1070
1071 void tcg_gen_smax_i32(TCGv_i32 ret, TCGv_i32 a, TCGv_i32 b)
1072 {
1073     tcg_gen_movcond_i32(TCG_COND_LT, ret, a, b, b, a);
1074 }
1075
1076 void tcg_gen_umax_i32(TCGv_i32 ret, TCGv_i32 a, TCGv_i32 b)
1077 {
1078     tcg_gen_movcond_i32(TCG_COND_LTU, ret, a, b, b, a);
1079 }
1080
1081 void tcg_gen_abs_i32(TCGv_i32 ret, TCGv_i32 a)
1082 {
1083     TCGv_i32 t = tcg_temp_new_i32();
1084
1085     tcg_gen_sari_i32(t, a, 31);
1086     tcg_gen_xor_i32(ret, a, t);
1087     tcg_gen_sub_i32(ret, ret, t);
1088     tcg_temp_free_i32(t);
1089 }
1090
1091 /* 64-bit ops */
1092
1093 #if TCG_TARGET_REG_BITS == 32
1094 /* These are all inline for TCG_TARGET_REG_BITS == 64.  */
1095
1096 void tcg_gen_discard_i64(TCGv_i64 arg)
1097 {
1098     tcg_gen_discard_i32(TCGV_LOW(arg));
1099     tcg_gen_discard_i32(TCGV_HIGH(arg));
1100 }
1101
1102 void tcg_gen_mov_i64(TCGv_i64 ret, TCGv_i64 arg)
1103 {
1104     TCGTemp *ts = tcgv_i64_temp(arg);
1105
1106     /* Canonicalize TCGv_i64 TEMP_CONST into TCGv_i32 TEMP_CONST. */
1107     if (ts->kind == TEMP_CONST) {
1108         tcg_gen_movi_i64(ret, ts->val);
1109     } else {
1110         tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1111         tcg_gen_mov_i32(TCGV_HIGH(ret), TCGV_HIGH(arg));
1112     }
1113 }
1114
1115 void tcg_gen_movi_i64(TCGv_i64 ret, int64_t arg)
1116 {
1117     tcg_gen_movi_i32(TCGV_LOW(ret), arg);
1118     tcg_gen_movi_i32(TCGV_HIGH(ret), arg >> 32);
1119 }
1120
1121 void tcg_gen_ld8u_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
1122 {
1123     tcg_gen_ld8u_i32(TCGV_LOW(ret), arg2, offset);
1124     tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1125 }
1126
1127 void tcg_gen_ld8s_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
1128 {
1129     tcg_gen_ld8s_i32(TCGV_LOW(ret), arg2, offset);
1130     tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
1131 }
1132
1133 void tcg_gen_ld16u_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
1134 {
1135     tcg_gen_ld16u_i32(TCGV_LOW(ret), arg2, offset);
1136     tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1137 }
1138
1139 void tcg_gen_ld16s_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
1140 {
1141     tcg_gen_ld16s_i32(TCGV_LOW(ret), arg2, offset);
1142     tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
1143 }
1144
1145 void tcg_gen_ld32u_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
1146 {
1147     tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset);
1148     tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1149 }
1150
1151 void tcg_gen_ld32s_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
1152 {
1153     tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset);
1154     tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
1155 }
1156
1157 void tcg_gen_ld_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
1158 {
1159     /* Since arg2 and ret have different types,
1160        they cannot be the same temporary */
1161 #ifdef HOST_WORDS_BIGENDIAN
1162     tcg_gen_ld_i32(TCGV_HIGH(ret), arg2, offset);
1163     tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset + 4);
1164 #else
1165     tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset);
1166     tcg_gen_ld_i32(TCGV_HIGH(ret), arg2, offset + 4);
1167 #endif
1168 }
1169
1170 void tcg_gen_st_i64(TCGv_i64 arg1, TCGv_ptr arg2, tcg_target_long offset)
1171 {
1172 #ifdef HOST_WORDS_BIGENDIAN
1173     tcg_gen_st_i32(TCGV_HIGH(arg1), arg2, offset);
1174     tcg_gen_st_i32(TCGV_LOW(arg1), arg2, offset + 4);
1175 #else
1176     tcg_gen_st_i32(TCGV_LOW(arg1), arg2, offset);
1177     tcg_gen_st_i32(TCGV_HIGH(arg1), arg2, offset + 4);
1178 #endif
1179 }
1180
1181 void tcg_gen_and_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1182 {
1183     tcg_gen_and_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
1184     tcg_gen_and_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
1185 }
1186
1187 void tcg_gen_or_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1188 {
1189     tcg_gen_or_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
1190     tcg_gen_or_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
1191 }
1192
1193 void tcg_gen_xor_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1194 {
1195     tcg_gen_xor_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
1196     tcg_gen_xor_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
1197 }
1198
1199 void tcg_gen_shl_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1200 {
1201     gen_helper_shl_i64(ret, arg1, arg2);
1202 }
1203
1204 void tcg_gen_shr_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1205 {
1206     gen_helper_shr_i64(ret, arg1, arg2);
1207 }
1208
1209 void tcg_gen_sar_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1210 {
1211     gen_helper_sar_i64(ret, arg1, arg2);
1212 }
1213
1214 void tcg_gen_mul_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1215 {
1216     TCGv_i64 t0;
1217     TCGv_i32 t1;
1218
1219     t0 = tcg_temp_new_i64();
1220     t1 = tcg_temp_new_i32();
1221
1222     tcg_gen_mulu2_i32(TCGV_LOW(t0), TCGV_HIGH(t0),
1223                       TCGV_LOW(arg1), TCGV_LOW(arg2));
1224
1225     tcg_gen_mul_i32(t1, TCGV_LOW(arg1), TCGV_HIGH(arg2));
1226     tcg_gen_add_i32(TCGV_HIGH(t0), TCGV_HIGH(t0), t1);
1227     tcg_gen_mul_i32(t1, TCGV_HIGH(arg1), TCGV_LOW(arg2));
1228     tcg_gen_add_i32(TCGV_HIGH(t0), TCGV_HIGH(t0), t1);
1229
1230     tcg_gen_mov_i64(ret, t0);
1231     tcg_temp_free_i64(t0);
1232     tcg_temp_free_i32(t1);
1233 }
1234
1235 #else
1236
1237 void tcg_gen_movi_i64(TCGv_i64 ret, int64_t arg)
1238 {
1239     tcg_gen_mov_i64(ret, tcg_constant_i64(arg));
1240 }
1241
1242 #endif /* TCG_TARGET_REG_SIZE == 32 */
1243
1244 void tcg_gen_addi_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
1245 {
1246     /* some cases can be optimized here */
1247     if (arg2 == 0) {
1248         tcg_gen_mov_i64(ret, arg1);
1249     } else if (TCG_TARGET_REG_BITS == 64) {
1250         tcg_gen_add_i64(ret, arg1, tcg_constant_i64(arg2));
1251     } else {
1252         tcg_gen_add2_i32(TCGV_LOW(ret), TCGV_HIGH(ret),
1253                          TCGV_LOW(arg1), TCGV_HIGH(arg1),
1254                          tcg_constant_i32(arg2), tcg_constant_i32(arg2 >> 32));
1255     }
1256 }
1257
1258 void tcg_gen_subfi_i64(TCGv_i64 ret, int64_t arg1, TCGv_i64 arg2)
1259 {
1260     if (arg1 == 0 && TCG_TARGET_HAS_neg_i64) {
1261         /* Don't recurse with tcg_gen_neg_i64.  */
1262         tcg_gen_op2_i64(INDEX_op_neg_i64, ret, arg2);
1263     } else if (TCG_TARGET_REG_BITS == 64) {
1264         tcg_gen_sub_i64(ret, tcg_constant_i64(arg1), arg2);
1265     } else {
1266         tcg_gen_sub2_i32(TCGV_LOW(ret), TCGV_HIGH(ret),
1267                          tcg_constant_i32(arg1), tcg_constant_i32(arg1 >> 32),
1268                          TCGV_LOW(arg2), TCGV_HIGH(arg2));
1269     }
1270 }
1271
1272 void tcg_gen_subi_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
1273 {
1274     /* some cases can be optimized here */
1275     if (arg2 == 0) {
1276         tcg_gen_mov_i64(ret, arg1);
1277     } else if (TCG_TARGET_REG_BITS == 64) {
1278         tcg_gen_sub_i64(ret, arg1, tcg_constant_i64(arg2));
1279     } else {
1280         tcg_gen_sub2_i32(TCGV_LOW(ret), TCGV_HIGH(ret),
1281                          TCGV_LOW(arg1), TCGV_HIGH(arg1),
1282                          tcg_constant_i32(arg2), tcg_constant_i32(arg2 >> 32));
1283     }
1284 }
1285
1286 void tcg_gen_andi_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
1287 {
1288     if (TCG_TARGET_REG_BITS == 32) {
1289         tcg_gen_andi_i32(TCGV_LOW(ret), TCGV_LOW(arg1), arg2);
1290         tcg_gen_andi_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), arg2 >> 32);
1291         return;
1292     }
1293
1294     /* Some cases can be optimized here.  */
1295     switch (arg2) {
1296     case 0:
1297         tcg_gen_movi_i64(ret, 0);
1298         return;
1299     case -1:
1300         tcg_gen_mov_i64(ret, arg1);
1301         return;
1302     case 0xff:
1303         /* Don't recurse with tcg_gen_ext8u_i64.  */
1304         if (TCG_TARGET_HAS_ext8u_i64) {
1305             tcg_gen_op2_i64(INDEX_op_ext8u_i64, ret, arg1);
1306             return;
1307         }
1308         break;
1309     case 0xffff:
1310         if (TCG_TARGET_HAS_ext16u_i64) {
1311             tcg_gen_op2_i64(INDEX_op_ext16u_i64, ret, arg1);
1312             return;
1313         }
1314         break;
1315     case 0xffffffffu:
1316         if (TCG_TARGET_HAS_ext32u_i64) {
1317             tcg_gen_op2_i64(INDEX_op_ext32u_i64, ret, arg1);
1318             return;
1319         }
1320         break;
1321     }
1322
1323     tcg_gen_and_i64(ret, arg1, tcg_constant_i64(arg2));
1324 }
1325
1326 void tcg_gen_ori_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
1327 {
1328     if (TCG_TARGET_REG_BITS == 32) {
1329         tcg_gen_ori_i32(TCGV_LOW(ret), TCGV_LOW(arg1), arg2);
1330         tcg_gen_ori_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), arg2 >> 32);
1331         return;
1332     }
1333     /* Some cases can be optimized here.  */
1334     if (arg2 == -1) {
1335         tcg_gen_movi_i64(ret, -1);
1336     } else if (arg2 == 0) {
1337         tcg_gen_mov_i64(ret, arg1);
1338     } else {
1339         tcg_gen_or_i64(ret, arg1, tcg_constant_i64(arg2));
1340     }
1341 }
1342
1343 void tcg_gen_xori_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
1344 {
1345     if (TCG_TARGET_REG_BITS == 32) {
1346         tcg_gen_xori_i32(TCGV_LOW(ret), TCGV_LOW(arg1), arg2);
1347         tcg_gen_xori_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), arg2 >> 32);
1348         return;
1349     }
1350     /* Some cases can be optimized here.  */
1351     if (arg2 == 0) {
1352         tcg_gen_mov_i64(ret, arg1);
1353     } else if (arg2 == -1 && TCG_TARGET_HAS_not_i64) {
1354         /* Don't recurse with tcg_gen_not_i64.  */
1355         tcg_gen_op2_i64(INDEX_op_not_i64, ret, arg1);
1356     } else {
1357         tcg_gen_xor_i64(ret, arg1, tcg_constant_i64(arg2));
1358     }
1359 }
1360
1361 static inline void tcg_gen_shifti_i64(TCGv_i64 ret, TCGv_i64 arg1,
1362                                       unsigned c, bool right, bool arith)
1363 {
1364     tcg_debug_assert(c < 64);
1365     if (c == 0) {
1366         tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg1));
1367         tcg_gen_mov_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1));
1368     } else if (c >= 32) {
1369         c -= 32;
1370         if (right) {
1371             if (arith) {
1372                 tcg_gen_sari_i32(TCGV_LOW(ret), TCGV_HIGH(arg1), c);
1373                 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), 31);
1374             } else {
1375                 tcg_gen_shri_i32(TCGV_LOW(ret), TCGV_HIGH(arg1), c);
1376                 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1377             }
1378         } else {
1379             tcg_gen_shli_i32(TCGV_HIGH(ret), TCGV_LOW(arg1), c);
1380             tcg_gen_movi_i32(TCGV_LOW(ret), 0);
1381         }
1382     } else if (right) {
1383         if (TCG_TARGET_HAS_extract2_i32) {
1384             tcg_gen_extract2_i32(TCGV_LOW(ret),
1385                                  TCGV_LOW(arg1), TCGV_HIGH(arg1), c);
1386         } else {
1387             tcg_gen_shri_i32(TCGV_LOW(ret), TCGV_LOW(arg1), c);
1388             tcg_gen_deposit_i32(TCGV_LOW(ret), TCGV_LOW(ret),
1389                                 TCGV_HIGH(arg1), 32 - c, c);
1390         }
1391         if (arith) {
1392             tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), c);
1393         } else {
1394             tcg_gen_shri_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), c);
1395         }
1396     } else {
1397         if (TCG_TARGET_HAS_extract2_i32) {
1398             tcg_gen_extract2_i32(TCGV_HIGH(ret),
1399                                  TCGV_LOW(arg1), TCGV_HIGH(arg1), 32 - c);
1400         } else {
1401             TCGv_i32 t0 = tcg_temp_new_i32();
1402             tcg_gen_shri_i32(t0, TCGV_LOW(arg1), 32 - c);
1403             tcg_gen_deposit_i32(TCGV_HIGH(ret), t0,
1404                                 TCGV_HIGH(arg1), c, 32 - c);
1405             tcg_temp_free_i32(t0);
1406         }
1407         tcg_gen_shli_i32(TCGV_LOW(ret), TCGV_LOW(arg1), c);
1408     }
1409 }
1410
1411 void tcg_gen_shli_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
1412 {
1413     tcg_debug_assert(arg2 >= 0 && arg2 < 64);
1414     if (TCG_TARGET_REG_BITS == 32) {
1415         tcg_gen_shifti_i64(ret, arg1, arg2, 0, 0);
1416     } else if (arg2 == 0) {
1417         tcg_gen_mov_i64(ret, arg1);
1418     } else {
1419         tcg_gen_shl_i64(ret, arg1, tcg_constant_i64(arg2));
1420     }
1421 }
1422
1423 void tcg_gen_shri_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
1424 {
1425     tcg_debug_assert(arg2 >= 0 && arg2 < 64);
1426     if (TCG_TARGET_REG_BITS == 32) {
1427         tcg_gen_shifti_i64(ret, arg1, arg2, 1, 0);
1428     } else if (arg2 == 0) {
1429         tcg_gen_mov_i64(ret, arg1);
1430     } else {
1431         tcg_gen_shr_i64(ret, arg1, tcg_constant_i64(arg2));
1432     }
1433 }
1434
1435 void tcg_gen_sari_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
1436 {
1437     tcg_debug_assert(arg2 >= 0 && arg2 < 64);
1438     if (TCG_TARGET_REG_BITS == 32) {
1439         tcg_gen_shifti_i64(ret, arg1, arg2, 1, 1);
1440     } else if (arg2 == 0) {
1441         tcg_gen_mov_i64(ret, arg1);
1442     } else {
1443         tcg_gen_sar_i64(ret, arg1, tcg_constant_i64(arg2));
1444     }
1445 }
1446
1447 void tcg_gen_brcond_i64(TCGCond cond, TCGv_i64 arg1, TCGv_i64 arg2, TCGLabel *l)
1448 {
1449     if (cond == TCG_COND_ALWAYS) {
1450         tcg_gen_br(l);
1451     } else if (cond != TCG_COND_NEVER) {
1452         l->refs++;
1453         if (TCG_TARGET_REG_BITS == 32) {
1454             tcg_gen_op6ii_i32(INDEX_op_brcond2_i32, TCGV_LOW(arg1),
1455                               TCGV_HIGH(arg1), TCGV_LOW(arg2),
1456                               TCGV_HIGH(arg2), cond, label_arg(l));
1457         } else {
1458             tcg_gen_op4ii_i64(INDEX_op_brcond_i64, arg1, arg2, cond,
1459                               label_arg(l));
1460         }
1461     }
1462 }
1463
1464 void tcg_gen_brcondi_i64(TCGCond cond, TCGv_i64 arg1, int64_t arg2, TCGLabel *l)
1465 {
1466     if (TCG_TARGET_REG_BITS == 64) {
1467         tcg_gen_brcond_i64(cond, arg1, tcg_constant_i64(arg2), l);
1468     } else if (cond == TCG_COND_ALWAYS) {
1469         tcg_gen_br(l);
1470     } else if (cond != TCG_COND_NEVER) {
1471         l->refs++;
1472         tcg_gen_op6ii_i32(INDEX_op_brcond2_i32,
1473                           TCGV_LOW(arg1), TCGV_HIGH(arg1),
1474                           tcg_constant_i32(arg2),
1475                           tcg_constant_i32(arg2 >> 32),
1476                           cond, label_arg(l));
1477     }
1478 }
1479
1480 void tcg_gen_setcond_i64(TCGCond cond, TCGv_i64 ret,
1481                          TCGv_i64 arg1, TCGv_i64 arg2)
1482 {
1483     if (cond == TCG_COND_ALWAYS) {
1484         tcg_gen_movi_i64(ret, 1);
1485     } else if (cond == TCG_COND_NEVER) {
1486         tcg_gen_movi_i64(ret, 0);
1487     } else {
1488         if (TCG_TARGET_REG_BITS == 32) {
1489             tcg_gen_op6i_i32(INDEX_op_setcond2_i32, TCGV_LOW(ret),
1490                              TCGV_LOW(arg1), TCGV_HIGH(arg1),
1491                              TCGV_LOW(arg2), TCGV_HIGH(arg2), cond);
1492             tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1493         } else {
1494             tcg_gen_op4i_i64(INDEX_op_setcond_i64, ret, arg1, arg2, cond);
1495         }
1496     }
1497 }
1498
1499 void tcg_gen_setcondi_i64(TCGCond cond, TCGv_i64 ret,
1500                           TCGv_i64 arg1, int64_t arg2)
1501 {
1502     if (TCG_TARGET_REG_BITS == 64) {
1503         tcg_gen_setcond_i64(cond, ret, arg1, tcg_constant_i64(arg2));
1504     } else if (cond == TCG_COND_ALWAYS) {
1505         tcg_gen_movi_i64(ret, 1);
1506     } else if (cond == TCG_COND_NEVER) {
1507         tcg_gen_movi_i64(ret, 0);
1508     } else {
1509         tcg_gen_op6i_i32(INDEX_op_setcond2_i32, TCGV_LOW(ret),
1510                          TCGV_LOW(arg1), TCGV_HIGH(arg1),
1511                          tcg_constant_i32(arg2),
1512                          tcg_constant_i32(arg2 >> 32), cond);
1513         tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1514     }
1515 }
1516
1517 void tcg_gen_muli_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
1518 {
1519     if (arg2 == 0) {
1520         tcg_gen_movi_i64(ret, 0);
1521     } else if (is_power_of_2(arg2)) {
1522         tcg_gen_shli_i64(ret, arg1, ctz64(arg2));
1523     } else {
1524         TCGv_i64 t0 = tcg_const_i64(arg2);
1525         tcg_gen_mul_i64(ret, arg1, t0);
1526         tcg_temp_free_i64(t0);
1527     }
1528 }
1529
1530 void tcg_gen_div_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1531 {
1532     if (TCG_TARGET_HAS_div_i64) {
1533         tcg_gen_op3_i64(INDEX_op_div_i64, ret, arg1, arg2);
1534     } else if (TCG_TARGET_HAS_div2_i64) {
1535         TCGv_i64 t0 = tcg_temp_new_i64();
1536         tcg_gen_sari_i64(t0, arg1, 63);
1537         tcg_gen_op5_i64(INDEX_op_div2_i64, ret, t0, arg1, t0, arg2);
1538         tcg_temp_free_i64(t0);
1539     } else {
1540         gen_helper_div_i64(ret, arg1, arg2);
1541     }
1542 }
1543
1544 void tcg_gen_rem_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1545 {
1546     if (TCG_TARGET_HAS_rem_i64) {
1547         tcg_gen_op3_i64(INDEX_op_rem_i64, ret, arg1, arg2);
1548     } else if (TCG_TARGET_HAS_div_i64) {
1549         TCGv_i64 t0 = tcg_temp_new_i64();
1550         tcg_gen_op3_i64(INDEX_op_div_i64, t0, arg1, arg2);
1551         tcg_gen_mul_i64(t0, t0, arg2);
1552         tcg_gen_sub_i64(ret, arg1, t0);
1553         tcg_temp_free_i64(t0);
1554     } else if (TCG_TARGET_HAS_div2_i64) {
1555         TCGv_i64 t0 = tcg_temp_new_i64();
1556         tcg_gen_sari_i64(t0, arg1, 63);
1557         tcg_gen_op5_i64(INDEX_op_div2_i64, t0, ret, arg1, t0, arg2);
1558         tcg_temp_free_i64(t0);
1559     } else {
1560         gen_helper_rem_i64(ret, arg1, arg2);
1561     }
1562 }
1563
1564 void tcg_gen_divu_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1565 {
1566     if (TCG_TARGET_HAS_div_i64) {
1567         tcg_gen_op3_i64(INDEX_op_divu_i64, ret, arg1, arg2);
1568     } else if (TCG_TARGET_HAS_div2_i64) {
1569         TCGv_i64 t0 = tcg_temp_new_i64();
1570         tcg_gen_movi_i64(t0, 0);
1571         tcg_gen_op5_i64(INDEX_op_divu2_i64, ret, t0, arg1, t0, arg2);
1572         tcg_temp_free_i64(t0);
1573     } else {
1574         gen_helper_divu_i64(ret, arg1, arg2);
1575     }
1576 }
1577
1578 void tcg_gen_remu_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1579 {
1580     if (TCG_TARGET_HAS_rem_i64) {
1581         tcg_gen_op3_i64(INDEX_op_remu_i64, ret, arg1, arg2);
1582     } else if (TCG_TARGET_HAS_div_i64) {
1583         TCGv_i64 t0 = tcg_temp_new_i64();
1584         tcg_gen_op3_i64(INDEX_op_divu_i64, t0, arg1, arg2);
1585         tcg_gen_mul_i64(t0, t0, arg2);
1586         tcg_gen_sub_i64(ret, arg1, t0);
1587         tcg_temp_free_i64(t0);
1588     } else if (TCG_TARGET_HAS_div2_i64) {
1589         TCGv_i64 t0 = tcg_temp_new_i64();
1590         tcg_gen_movi_i64(t0, 0);
1591         tcg_gen_op5_i64(INDEX_op_divu2_i64, t0, ret, arg1, t0, arg2);
1592         tcg_temp_free_i64(t0);
1593     } else {
1594         gen_helper_remu_i64(ret, arg1, arg2);
1595     }
1596 }
1597
1598 void tcg_gen_ext8s_i64(TCGv_i64 ret, TCGv_i64 arg)
1599 {
1600     if (TCG_TARGET_REG_BITS == 32) {
1601         tcg_gen_ext8s_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1602         tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
1603     } else if (TCG_TARGET_HAS_ext8s_i64) {
1604         tcg_gen_op2_i64(INDEX_op_ext8s_i64, ret, arg);
1605     } else {
1606         tcg_gen_shli_i64(ret, arg, 56);
1607         tcg_gen_sari_i64(ret, ret, 56);
1608     }
1609 }
1610
1611 void tcg_gen_ext16s_i64(TCGv_i64 ret, TCGv_i64 arg)
1612 {
1613     if (TCG_TARGET_REG_BITS == 32) {
1614         tcg_gen_ext16s_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1615         tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
1616     } else if (TCG_TARGET_HAS_ext16s_i64) {
1617         tcg_gen_op2_i64(INDEX_op_ext16s_i64, ret, arg);
1618     } else {
1619         tcg_gen_shli_i64(ret, arg, 48);
1620         tcg_gen_sari_i64(ret, ret, 48);
1621     }
1622 }
1623
1624 void tcg_gen_ext32s_i64(TCGv_i64 ret, TCGv_i64 arg)
1625 {
1626     if (TCG_TARGET_REG_BITS == 32) {
1627         tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1628         tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
1629     } else if (TCG_TARGET_HAS_ext32s_i64) {
1630         tcg_gen_op2_i64(INDEX_op_ext32s_i64, ret, arg);
1631     } else {
1632         tcg_gen_shli_i64(ret, arg, 32);
1633         tcg_gen_sari_i64(ret, ret, 32);
1634     }
1635 }
1636
1637 void tcg_gen_ext8u_i64(TCGv_i64 ret, TCGv_i64 arg)
1638 {
1639     if (TCG_TARGET_REG_BITS == 32) {
1640         tcg_gen_ext8u_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1641         tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1642     } else if (TCG_TARGET_HAS_ext8u_i64) {
1643         tcg_gen_op2_i64(INDEX_op_ext8u_i64, ret, arg);
1644     } else {
1645         tcg_gen_andi_i64(ret, arg, 0xffu);
1646     }
1647 }
1648
1649 void tcg_gen_ext16u_i64(TCGv_i64 ret, TCGv_i64 arg)
1650 {
1651     if (TCG_TARGET_REG_BITS == 32) {
1652         tcg_gen_ext16u_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1653         tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1654     } else if (TCG_TARGET_HAS_ext16u_i64) {
1655         tcg_gen_op2_i64(INDEX_op_ext16u_i64, ret, arg);
1656     } else {
1657         tcg_gen_andi_i64(ret, arg, 0xffffu);
1658     }
1659 }
1660
1661 void tcg_gen_ext32u_i64(TCGv_i64 ret, TCGv_i64 arg)
1662 {
1663     if (TCG_TARGET_REG_BITS == 32) {
1664         tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1665         tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1666     } else if (TCG_TARGET_HAS_ext32u_i64) {
1667         tcg_gen_op2_i64(INDEX_op_ext32u_i64, ret, arg);
1668     } else {
1669         tcg_gen_andi_i64(ret, arg, 0xffffffffu);
1670     }
1671 }
1672
1673 void tcg_gen_bswap16_i64(TCGv_i64 ret, TCGv_i64 arg, int flags)
1674 {
1675     /* Only one extension flag may be present. */
1676     tcg_debug_assert(!(flags & TCG_BSWAP_OS) || !(flags & TCG_BSWAP_OZ));
1677
1678     if (TCG_TARGET_REG_BITS == 32) {
1679         tcg_gen_bswap16_i32(TCGV_LOW(ret), TCGV_LOW(arg), flags);
1680         if (flags & TCG_BSWAP_OS) {
1681             tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
1682         } else {
1683             tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1684         }
1685     } else if (TCG_TARGET_HAS_bswap16_i64) {
1686         tcg_gen_op3i_i64(INDEX_op_bswap16_i64, ret, arg, flags);
1687     } else {
1688         TCGv_i64 t0 = tcg_temp_new_i64();
1689         TCGv_i64 t1 = tcg_temp_new_i64();
1690
1691         tcg_gen_shri_i64(t0, arg, 8);
1692         if (!(flags & TCG_BSWAP_IZ)) {
1693             tcg_gen_ext8u_i64(t0, t0);
1694         }
1695
1696         if (flags & TCG_BSWAP_OS) {
1697             tcg_gen_shli_i64(t1, arg, 56);
1698             tcg_gen_sari_i64(t1, t1, 48);
1699         } else if (flags & TCG_BSWAP_OZ) {
1700             tcg_gen_ext8u_i64(t1, arg);
1701             tcg_gen_shli_i64(t1, t1, 8);
1702         } else {
1703             tcg_gen_shli_i64(t1, arg, 8);
1704         }
1705
1706         tcg_gen_or_i64(ret, t0, t1);
1707         tcg_temp_free_i64(t0);
1708         tcg_temp_free_i64(t1);
1709     }
1710 }
1711
1712 void tcg_gen_bswap32_i64(TCGv_i64 ret, TCGv_i64 arg, int flags)
1713 {
1714     /* Only one extension flag may be present. */
1715     tcg_debug_assert(!(flags & TCG_BSWAP_OS) || !(flags & TCG_BSWAP_OZ));
1716
1717     if (TCG_TARGET_REG_BITS == 32) {
1718         tcg_gen_bswap32_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1719         if (flags & TCG_BSWAP_OS) {
1720             tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
1721         } else {
1722             tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1723         }
1724     } else if (TCG_TARGET_HAS_bswap32_i64) {
1725         tcg_gen_op3i_i64(INDEX_op_bswap32_i64, ret, arg, flags);
1726     } else {
1727         TCGv_i64 t0 = tcg_temp_new_i64();
1728         TCGv_i64 t1 = tcg_temp_new_i64();
1729         TCGv_i64 t2 = tcg_constant_i64(0x00ff00ff);
1730
1731                                             /* arg = xxxxabcd */
1732         tcg_gen_shri_i64(t0, arg, 8);       /*  t0 = .xxxxabc */
1733         tcg_gen_and_i64(t1, arg, t2);       /*  t1 = .....b.d */
1734         tcg_gen_and_i64(t0, t0, t2);        /*  t0 = .....a.c */
1735         tcg_gen_shli_i64(t1, t1, 8);        /*  t1 = ....b.d. */
1736         tcg_gen_or_i64(ret, t0, t1);        /* ret = ....badc */
1737
1738         tcg_gen_shli_i64(t1, ret, 48);      /*  t1 = dc...... */
1739         tcg_gen_shri_i64(t0, ret, 16);      /*  t0 = ......ba */
1740         if (flags & TCG_BSWAP_OS) {
1741             tcg_gen_sari_i64(t1, t1, 32);   /*  t1 = ssssdc.. */
1742         } else {
1743             tcg_gen_shri_i64(t1, t1, 32);   /*  t1 = ....dc.. */
1744         }
1745         tcg_gen_or_i64(ret, t0, t1);        /* ret = ssssdcba */
1746
1747         tcg_temp_free_i64(t0);
1748         tcg_temp_free_i64(t1);
1749     }
1750 }
1751
1752 void tcg_gen_bswap64_i64(TCGv_i64 ret, TCGv_i64 arg)
1753 {
1754     if (TCG_TARGET_REG_BITS == 32) {
1755         TCGv_i32 t0, t1;
1756         t0 = tcg_temp_new_i32();
1757         t1 = tcg_temp_new_i32();
1758
1759         tcg_gen_bswap32_i32(t0, TCGV_LOW(arg));
1760         tcg_gen_bswap32_i32(t1, TCGV_HIGH(arg));
1761         tcg_gen_mov_i32(TCGV_LOW(ret), t1);
1762         tcg_gen_mov_i32(TCGV_HIGH(ret), t0);
1763         tcg_temp_free_i32(t0);
1764         tcg_temp_free_i32(t1);
1765     } else if (TCG_TARGET_HAS_bswap64_i64) {
1766         tcg_gen_op3i_i64(INDEX_op_bswap64_i64, ret, arg, 0);
1767     } else {
1768         TCGv_i64 t0 = tcg_temp_new_i64();
1769         TCGv_i64 t1 = tcg_temp_new_i64();
1770         TCGv_i64 t2 = tcg_temp_new_i64();
1771
1772                                         /* arg = abcdefgh */
1773         tcg_gen_movi_i64(t2, 0x00ff00ff00ff00ffull);
1774         tcg_gen_shri_i64(t0, arg, 8);   /*  t0 = .abcdefg */
1775         tcg_gen_and_i64(t1, arg, t2);   /*  t1 = .b.d.f.h */
1776         tcg_gen_and_i64(t0, t0, t2);    /*  t0 = .a.c.e.g */
1777         tcg_gen_shli_i64(t1, t1, 8);    /*  t1 = b.d.f.h. */
1778         tcg_gen_or_i64(ret, t0, t1);    /* ret = badcfehg */
1779
1780         tcg_gen_movi_i64(t2, 0x0000ffff0000ffffull);
1781         tcg_gen_shri_i64(t0, ret, 16);  /*  t0 = ..badcfe */
1782         tcg_gen_and_i64(t1, ret, t2);   /*  t1 = ..dc..hg */
1783         tcg_gen_and_i64(t0, t0, t2);    /*  t0 = ..ba..fe */
1784         tcg_gen_shli_i64(t1, t1, 16);   /*  t1 = dc..hg.. */
1785         tcg_gen_or_i64(ret, t0, t1);    /* ret = dcbahgfe */
1786
1787         tcg_gen_shri_i64(t0, ret, 32);  /*  t0 = ....dcba */
1788         tcg_gen_shli_i64(t1, ret, 32);  /*  t1 = hgfe.... */
1789         tcg_gen_or_i64(ret, t0, t1);    /* ret = hgfedcba */
1790
1791         tcg_temp_free_i64(t0);
1792         tcg_temp_free_i64(t1);
1793         tcg_temp_free_i64(t2);
1794     }
1795 }
1796
1797 void tcg_gen_not_i64(TCGv_i64 ret, TCGv_i64 arg)
1798 {
1799     if (TCG_TARGET_REG_BITS == 32) {
1800         tcg_gen_not_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1801         tcg_gen_not_i32(TCGV_HIGH(ret), TCGV_HIGH(arg));
1802     } else if (TCG_TARGET_HAS_not_i64) {
1803         tcg_gen_op2_i64(INDEX_op_not_i64, ret, arg);
1804     } else {
1805         tcg_gen_xori_i64(ret, arg, -1);
1806     }
1807 }
1808
1809 void tcg_gen_andc_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1810 {
1811     if (TCG_TARGET_REG_BITS == 32) {
1812         tcg_gen_andc_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
1813         tcg_gen_andc_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
1814     } else if (TCG_TARGET_HAS_andc_i64) {
1815         tcg_gen_op3_i64(INDEX_op_andc_i64, ret, arg1, arg2);
1816     } else {
1817         TCGv_i64 t0 = tcg_temp_new_i64();
1818         tcg_gen_not_i64(t0, arg2);
1819         tcg_gen_and_i64(ret, arg1, t0);
1820         tcg_temp_free_i64(t0);
1821     }
1822 }
1823
1824 void tcg_gen_eqv_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1825 {
1826     if (TCG_TARGET_REG_BITS == 32) {
1827         tcg_gen_eqv_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
1828         tcg_gen_eqv_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
1829     } else if (TCG_TARGET_HAS_eqv_i64) {
1830         tcg_gen_op3_i64(INDEX_op_eqv_i64, ret, arg1, arg2);
1831     } else {
1832         tcg_gen_xor_i64(ret, arg1, arg2);
1833         tcg_gen_not_i64(ret, ret);
1834     }
1835 }
1836
1837 void tcg_gen_nand_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1838 {
1839     if (TCG_TARGET_REG_BITS == 32) {
1840         tcg_gen_nand_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
1841         tcg_gen_nand_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
1842     } else if (TCG_TARGET_HAS_nand_i64) {
1843         tcg_gen_op3_i64(INDEX_op_nand_i64, ret, arg1, arg2);
1844     } else {
1845         tcg_gen_and_i64(ret, arg1, arg2);
1846         tcg_gen_not_i64(ret, ret);
1847     }
1848 }
1849
1850 void tcg_gen_nor_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1851 {
1852     if (TCG_TARGET_REG_BITS == 32) {
1853         tcg_gen_nor_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
1854         tcg_gen_nor_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
1855     } else if (TCG_TARGET_HAS_nor_i64) {
1856         tcg_gen_op3_i64(INDEX_op_nor_i64, ret, arg1, arg2);
1857     } else {
1858         tcg_gen_or_i64(ret, arg1, arg2);
1859         tcg_gen_not_i64(ret, ret);
1860     }
1861 }
1862
1863 void tcg_gen_orc_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1864 {
1865     if (TCG_TARGET_REG_BITS == 32) {
1866         tcg_gen_orc_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
1867         tcg_gen_orc_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
1868     } else if (TCG_TARGET_HAS_orc_i64) {
1869         tcg_gen_op3_i64(INDEX_op_orc_i64, ret, arg1, arg2);
1870     } else {
1871         TCGv_i64 t0 = tcg_temp_new_i64();
1872         tcg_gen_not_i64(t0, arg2);
1873         tcg_gen_or_i64(ret, arg1, t0);
1874         tcg_temp_free_i64(t0);
1875     }
1876 }
1877
1878 void tcg_gen_clz_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1879 {
1880     if (TCG_TARGET_HAS_clz_i64) {
1881         tcg_gen_op3_i64(INDEX_op_clz_i64, ret, arg1, arg2);
1882     } else {
1883         gen_helper_clz_i64(ret, arg1, arg2);
1884     }
1885 }
1886
1887 void tcg_gen_clzi_i64(TCGv_i64 ret, TCGv_i64 arg1, uint64_t arg2)
1888 {
1889     if (TCG_TARGET_REG_BITS == 32
1890         && TCG_TARGET_HAS_clz_i32
1891         && arg2 <= 0xffffffffu) {
1892         TCGv_i32 t = tcg_temp_new_i32();
1893         tcg_gen_clzi_i32(t, TCGV_LOW(arg1), arg2 - 32);
1894         tcg_gen_addi_i32(t, t, 32);
1895         tcg_gen_clz_i32(TCGV_LOW(ret), TCGV_HIGH(arg1), t);
1896         tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1897         tcg_temp_free_i32(t);
1898     } else {
1899         TCGv_i64 t0 = tcg_const_i64(arg2);
1900         tcg_gen_clz_i64(ret, arg1, t0);
1901         tcg_temp_free_i64(t0);
1902     }
1903 }
1904
1905 void tcg_gen_ctz_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1906 {
1907     if (TCG_TARGET_HAS_ctz_i64) {
1908         tcg_gen_op3_i64(INDEX_op_ctz_i64, ret, arg1, arg2);
1909     } else if (TCG_TARGET_HAS_ctpop_i64 || TCG_TARGET_HAS_clz_i64) {
1910         TCGv_i64 z, t = tcg_temp_new_i64();
1911
1912         if (TCG_TARGET_HAS_ctpop_i64) {
1913             tcg_gen_subi_i64(t, arg1, 1);
1914             tcg_gen_andc_i64(t, t, arg1);
1915             tcg_gen_ctpop_i64(t, t);
1916         } else {
1917             /* Since all non-x86 hosts have clz(0) == 64, don't fight it.  */
1918             tcg_gen_neg_i64(t, arg1);
1919             tcg_gen_and_i64(t, t, arg1);
1920             tcg_gen_clzi_i64(t, t, 64);
1921             tcg_gen_xori_i64(t, t, 63);
1922         }
1923         z = tcg_constant_i64(0);
1924         tcg_gen_movcond_i64(TCG_COND_EQ, ret, arg1, z, arg2, t);
1925         tcg_temp_free_i64(t);
1926         tcg_temp_free_i64(z);
1927     } else {
1928         gen_helper_ctz_i64(ret, arg1, arg2);
1929     }
1930 }
1931
1932 void tcg_gen_ctzi_i64(TCGv_i64 ret, TCGv_i64 arg1, uint64_t arg2)
1933 {
1934     if (TCG_TARGET_REG_BITS == 32
1935         && TCG_TARGET_HAS_ctz_i32
1936         && arg2 <= 0xffffffffu) {
1937         TCGv_i32 t32 = tcg_temp_new_i32();
1938         tcg_gen_ctzi_i32(t32, TCGV_HIGH(arg1), arg2 - 32);
1939         tcg_gen_addi_i32(t32, t32, 32);
1940         tcg_gen_ctz_i32(TCGV_LOW(ret), TCGV_LOW(arg1), t32);
1941         tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1942         tcg_temp_free_i32(t32);
1943     } else if (!TCG_TARGET_HAS_ctz_i64
1944                && TCG_TARGET_HAS_ctpop_i64
1945                && arg2 == 64) {
1946         /* This equivalence has the advantage of not requiring a fixup.  */
1947         TCGv_i64 t = tcg_temp_new_i64();
1948         tcg_gen_subi_i64(t, arg1, 1);
1949         tcg_gen_andc_i64(t, t, arg1);
1950         tcg_gen_ctpop_i64(ret, t);
1951         tcg_temp_free_i64(t);
1952     } else {
1953         TCGv_i64 t0 = tcg_const_i64(arg2);
1954         tcg_gen_ctz_i64(ret, arg1, t0);
1955         tcg_temp_free_i64(t0);
1956     }
1957 }
1958
1959 void tcg_gen_clrsb_i64(TCGv_i64 ret, TCGv_i64 arg)
1960 {
1961     if (TCG_TARGET_HAS_clz_i64 || TCG_TARGET_HAS_clz_i32) {
1962         TCGv_i64 t = tcg_temp_new_i64();
1963         tcg_gen_sari_i64(t, arg, 63);
1964         tcg_gen_xor_i64(t, t, arg);
1965         tcg_gen_clzi_i64(t, t, 64);
1966         tcg_gen_subi_i64(ret, t, 1);
1967         tcg_temp_free_i64(t);
1968     } else {
1969         gen_helper_clrsb_i64(ret, arg);
1970     }
1971 }
1972
1973 void tcg_gen_ctpop_i64(TCGv_i64 ret, TCGv_i64 arg1)
1974 {
1975     if (TCG_TARGET_HAS_ctpop_i64) {
1976         tcg_gen_op2_i64(INDEX_op_ctpop_i64, ret, arg1);
1977     } else if (TCG_TARGET_REG_BITS == 32 && TCG_TARGET_HAS_ctpop_i32) {
1978         tcg_gen_ctpop_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1));
1979         tcg_gen_ctpop_i32(TCGV_LOW(ret), TCGV_LOW(arg1));
1980         tcg_gen_add_i32(TCGV_LOW(ret), TCGV_LOW(ret), TCGV_HIGH(ret));
1981         tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1982     } else {
1983         gen_helper_ctpop_i64(ret, arg1);
1984     }
1985 }
1986
1987 void tcg_gen_rotl_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1988 {
1989     if (TCG_TARGET_HAS_rot_i64) {
1990         tcg_gen_op3_i64(INDEX_op_rotl_i64, ret, arg1, arg2);
1991     } else {
1992         TCGv_i64 t0, t1;
1993         t0 = tcg_temp_new_i64();
1994         t1 = tcg_temp_new_i64();
1995         tcg_gen_shl_i64(t0, arg1, arg2);
1996         tcg_gen_subfi_i64(t1, 64, arg2);
1997         tcg_gen_shr_i64(t1, arg1, t1);
1998         tcg_gen_or_i64(ret, t0, t1);
1999         tcg_temp_free_i64(t0);
2000         tcg_temp_free_i64(t1);
2001     }
2002 }
2003
2004 void tcg_gen_rotli_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
2005 {
2006     tcg_debug_assert(arg2 >= 0 && arg2 < 64);
2007     /* some cases can be optimized here */
2008     if (arg2 == 0) {
2009         tcg_gen_mov_i64(ret, arg1);
2010     } else if (TCG_TARGET_HAS_rot_i64) {
2011         tcg_gen_rotl_i64(ret, arg1, tcg_constant_i64(arg2));
2012     } else {
2013         TCGv_i64 t0, t1;
2014         t0 = tcg_temp_new_i64();
2015         t1 = tcg_temp_new_i64();
2016         tcg_gen_shli_i64(t0, arg1, arg2);
2017         tcg_gen_shri_i64(t1, arg1, 64 - arg2);
2018         tcg_gen_or_i64(ret, t0, t1);
2019         tcg_temp_free_i64(t0);
2020         tcg_temp_free_i64(t1);
2021     }
2022 }
2023
2024 void tcg_gen_rotr_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
2025 {
2026     if (TCG_TARGET_HAS_rot_i64) {
2027         tcg_gen_op3_i64(INDEX_op_rotr_i64, ret, arg1, arg2);
2028     } else {
2029         TCGv_i64 t0, t1;
2030         t0 = tcg_temp_new_i64();
2031         t1 = tcg_temp_new_i64();
2032         tcg_gen_shr_i64(t0, arg1, arg2);
2033         tcg_gen_subfi_i64(t1, 64, arg2);
2034         tcg_gen_shl_i64(t1, arg1, t1);
2035         tcg_gen_or_i64(ret, t0, t1);
2036         tcg_temp_free_i64(t0);
2037         tcg_temp_free_i64(t1);
2038     }
2039 }
2040
2041 void tcg_gen_rotri_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
2042 {
2043     tcg_debug_assert(arg2 >= 0 && arg2 < 64);
2044     /* some cases can be optimized here */
2045     if (arg2 == 0) {
2046         tcg_gen_mov_i64(ret, arg1);
2047     } else {
2048         tcg_gen_rotli_i64(ret, arg1, 64 - arg2);
2049     }
2050 }
2051
2052 void tcg_gen_deposit_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2,
2053                          unsigned int ofs, unsigned int len)
2054 {
2055     uint64_t mask;
2056     TCGv_i64 t1;
2057
2058     tcg_debug_assert(ofs < 64);
2059     tcg_debug_assert(len > 0);
2060     tcg_debug_assert(len <= 64);
2061     tcg_debug_assert(ofs + len <= 64);
2062
2063     if (len == 64) {
2064         tcg_gen_mov_i64(ret, arg2);
2065         return;
2066     }
2067     if (TCG_TARGET_HAS_deposit_i64 && TCG_TARGET_deposit_i64_valid(ofs, len)) {
2068         tcg_gen_op5ii_i64(INDEX_op_deposit_i64, ret, arg1, arg2, ofs, len);
2069         return;
2070     }
2071
2072     if (TCG_TARGET_REG_BITS == 32) {
2073         if (ofs >= 32) {
2074             tcg_gen_deposit_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1),
2075                                 TCGV_LOW(arg2), ofs - 32, len);
2076             tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg1));
2077             return;
2078         }
2079         if (ofs + len <= 32) {
2080             tcg_gen_deposit_i32(TCGV_LOW(ret), TCGV_LOW(arg1),
2081                                 TCGV_LOW(arg2), ofs, len);
2082             tcg_gen_mov_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1));
2083             return;
2084         }
2085     }
2086
2087     t1 = tcg_temp_new_i64();
2088
2089     if (TCG_TARGET_HAS_extract2_i64) {
2090         if (ofs + len == 64) {
2091             tcg_gen_shli_i64(t1, arg1, len);
2092             tcg_gen_extract2_i64(ret, t1, arg2, len);
2093             goto done;
2094         }
2095         if (ofs == 0) {
2096             tcg_gen_extract2_i64(ret, arg1, arg2, len);
2097             tcg_gen_rotli_i64(ret, ret, len);
2098             goto done;
2099         }
2100     }
2101
2102     mask = (1ull << len) - 1;
2103     if (ofs + len < 64) {
2104         tcg_gen_andi_i64(t1, arg2, mask);
2105         tcg_gen_shli_i64(t1, t1, ofs);
2106     } else {
2107         tcg_gen_shli_i64(t1, arg2, ofs);
2108     }
2109     tcg_gen_andi_i64(ret, arg1, ~(mask << ofs));
2110     tcg_gen_or_i64(ret, ret, t1);
2111  done:
2112     tcg_temp_free_i64(t1);
2113 }
2114
2115 void tcg_gen_deposit_z_i64(TCGv_i64 ret, TCGv_i64 arg,
2116                            unsigned int ofs, unsigned int len)
2117 {
2118     tcg_debug_assert(ofs < 64);
2119     tcg_debug_assert(len > 0);
2120     tcg_debug_assert(len <= 64);
2121     tcg_debug_assert(ofs + len <= 64);
2122
2123     if (ofs + len == 64) {
2124         tcg_gen_shli_i64(ret, arg, ofs);
2125     } else if (ofs == 0) {
2126         tcg_gen_andi_i64(ret, arg, (1ull << len) - 1);
2127     } else if (TCG_TARGET_HAS_deposit_i64
2128                && TCG_TARGET_deposit_i64_valid(ofs, len)) {
2129         TCGv_i64 zero = tcg_constant_i64(0);
2130         tcg_gen_op5ii_i64(INDEX_op_deposit_i64, ret, zero, arg, ofs, len);
2131     } else {
2132         if (TCG_TARGET_REG_BITS == 32) {
2133             if (ofs >= 32) {
2134                 tcg_gen_deposit_z_i32(TCGV_HIGH(ret), TCGV_LOW(arg),
2135                                       ofs - 32, len);
2136                 tcg_gen_movi_i32(TCGV_LOW(ret), 0);
2137                 return;
2138             }
2139             if (ofs + len <= 32) {
2140                 tcg_gen_deposit_z_i32(TCGV_LOW(ret), TCGV_LOW(arg), ofs, len);
2141                 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
2142                 return;
2143             }
2144         }
2145         /* To help two-operand hosts we prefer to zero-extend first,
2146            which allows ARG to stay live.  */
2147         switch (len) {
2148         case 32:
2149             if (TCG_TARGET_HAS_ext32u_i64) {
2150                 tcg_gen_ext32u_i64(ret, arg);
2151                 tcg_gen_shli_i64(ret, ret, ofs);
2152                 return;
2153             }
2154             break;
2155         case 16:
2156             if (TCG_TARGET_HAS_ext16u_i64) {
2157                 tcg_gen_ext16u_i64(ret, arg);
2158                 tcg_gen_shli_i64(ret, ret, ofs);
2159                 return;
2160             }
2161             break;
2162         case 8:
2163             if (TCG_TARGET_HAS_ext8u_i64) {
2164                 tcg_gen_ext8u_i64(ret, arg);
2165                 tcg_gen_shli_i64(ret, ret, ofs);
2166                 return;
2167             }
2168             break;
2169         }
2170         /* Otherwise prefer zero-extension over AND for code size.  */
2171         switch (ofs + len) {
2172         case 32:
2173             if (TCG_TARGET_HAS_ext32u_i64) {
2174                 tcg_gen_shli_i64(ret, arg, ofs);
2175                 tcg_gen_ext32u_i64(ret, ret);
2176                 return;
2177             }
2178             break;
2179         case 16:
2180             if (TCG_TARGET_HAS_ext16u_i64) {
2181                 tcg_gen_shli_i64(ret, arg, ofs);
2182                 tcg_gen_ext16u_i64(ret, ret);
2183                 return;
2184             }
2185             break;
2186         case 8:
2187             if (TCG_TARGET_HAS_ext8u_i64) {
2188                 tcg_gen_shli_i64(ret, arg, ofs);
2189                 tcg_gen_ext8u_i64(ret, ret);
2190                 return;
2191             }
2192             break;
2193         }
2194         tcg_gen_andi_i64(ret, arg, (1ull << len) - 1);
2195         tcg_gen_shli_i64(ret, ret, ofs);
2196     }
2197 }
2198
2199 void tcg_gen_extract_i64(TCGv_i64 ret, TCGv_i64 arg,
2200                          unsigned int ofs, unsigned int len)
2201 {
2202     tcg_debug_assert(ofs < 64);
2203     tcg_debug_assert(len > 0);
2204     tcg_debug_assert(len <= 64);
2205     tcg_debug_assert(ofs + len <= 64);
2206
2207     /* Canonicalize certain special cases, even if extract is supported.  */
2208     if (ofs + len == 64) {
2209         tcg_gen_shri_i64(ret, arg, 64 - len);
2210         return;
2211     }
2212     if (ofs == 0) {
2213         tcg_gen_andi_i64(ret, arg, (1ull << len) - 1);
2214         return;
2215     }
2216
2217     if (TCG_TARGET_REG_BITS == 32) {
2218         /* Look for a 32-bit extract within one of the two words.  */
2219         if (ofs >= 32) {
2220             tcg_gen_extract_i32(TCGV_LOW(ret), TCGV_HIGH(arg), ofs - 32, len);
2221             tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
2222             return;
2223         }
2224         if (ofs + len <= 32) {
2225             tcg_gen_extract_i32(TCGV_LOW(ret), TCGV_LOW(arg), ofs, len);
2226             tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
2227             return;
2228         }
2229         /* The field is split across two words.  One double-word
2230            shift is better than two double-word shifts.  */
2231         goto do_shift_and;
2232     }
2233
2234     if (TCG_TARGET_HAS_extract_i64
2235         && TCG_TARGET_extract_i64_valid(ofs, len)) {
2236         tcg_gen_op4ii_i64(INDEX_op_extract_i64, ret, arg, ofs, len);
2237         return;
2238     }
2239
2240     /* Assume that zero-extension, if available, is cheaper than a shift.  */
2241     switch (ofs + len) {
2242     case 32:
2243         if (TCG_TARGET_HAS_ext32u_i64) {
2244             tcg_gen_ext32u_i64(ret, arg);
2245             tcg_gen_shri_i64(ret, ret, ofs);
2246             return;
2247         }
2248         break;
2249     case 16:
2250         if (TCG_TARGET_HAS_ext16u_i64) {
2251             tcg_gen_ext16u_i64(ret, arg);
2252             tcg_gen_shri_i64(ret, ret, ofs);
2253             return;
2254         }
2255         break;
2256     case 8:
2257         if (TCG_TARGET_HAS_ext8u_i64) {
2258             tcg_gen_ext8u_i64(ret, arg);
2259             tcg_gen_shri_i64(ret, ret, ofs);
2260             return;
2261         }
2262         break;
2263     }
2264
2265     /* ??? Ideally we'd know what values are available for immediate AND.
2266        Assume that 8 bits are available, plus the special cases of 16 and 32,
2267        so that we get ext8u, ext16u, and ext32u.  */
2268     switch (len) {
2269     case 1 ... 8: case 16: case 32:
2270     do_shift_and:
2271         tcg_gen_shri_i64(ret, arg, ofs);
2272         tcg_gen_andi_i64(ret, ret, (1ull << len) - 1);
2273         break;
2274     default:
2275         tcg_gen_shli_i64(ret, arg, 64 - len - ofs);
2276         tcg_gen_shri_i64(ret, ret, 64 - len);
2277         break;
2278     }
2279 }
2280
2281 void tcg_gen_sextract_i64(TCGv_i64 ret, TCGv_i64 arg,
2282                           unsigned int ofs, unsigned int len)
2283 {
2284     tcg_debug_assert(ofs < 64);
2285     tcg_debug_assert(len > 0);
2286     tcg_debug_assert(len <= 64);
2287     tcg_debug_assert(ofs + len <= 64);
2288
2289     /* Canonicalize certain special cases, even if sextract is supported.  */
2290     if (ofs + len == 64) {
2291         tcg_gen_sari_i64(ret, arg, 64 - len);
2292         return;
2293     }
2294     if (ofs == 0) {
2295         switch (len) {
2296         case 32:
2297             tcg_gen_ext32s_i64(ret, arg);
2298             return;
2299         case 16:
2300             tcg_gen_ext16s_i64(ret, arg);
2301             return;
2302         case 8:
2303             tcg_gen_ext8s_i64(ret, arg);
2304             return;
2305         }
2306     }
2307
2308     if (TCG_TARGET_REG_BITS == 32) {
2309         /* Look for a 32-bit extract within one of the two words.  */
2310         if (ofs >= 32) {
2311             tcg_gen_sextract_i32(TCGV_LOW(ret), TCGV_HIGH(arg), ofs - 32, len);
2312         } else if (ofs + len <= 32) {
2313             tcg_gen_sextract_i32(TCGV_LOW(ret), TCGV_LOW(arg), ofs, len);
2314         } else if (ofs == 0) {
2315             tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg));
2316             tcg_gen_sextract_i32(TCGV_HIGH(ret), TCGV_HIGH(arg), 0, len - 32);
2317             return;
2318         } else if (len > 32) {
2319             TCGv_i32 t = tcg_temp_new_i32();
2320             /* Extract the bits for the high word normally.  */
2321             tcg_gen_sextract_i32(t, TCGV_HIGH(arg), ofs + 32, len - 32);
2322             /* Shift the field down for the low part.  */
2323             tcg_gen_shri_i64(ret, arg, ofs);
2324             /* Overwrite the shift into the high part.  */
2325             tcg_gen_mov_i32(TCGV_HIGH(ret), t);
2326             tcg_temp_free_i32(t);
2327             return;
2328         } else {
2329             /* Shift the field down for the low part, such that the
2330                field sits at the MSB.  */
2331             tcg_gen_shri_i64(ret, arg, ofs + len - 32);
2332             /* Shift the field down from the MSB, sign extending.  */
2333             tcg_gen_sari_i32(TCGV_LOW(ret), TCGV_LOW(ret), 32 - len);
2334         }
2335         /* Sign-extend the field from 32 bits.  */
2336         tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
2337         return;
2338     }
2339
2340     if (TCG_TARGET_HAS_sextract_i64
2341         && TCG_TARGET_extract_i64_valid(ofs, len)) {
2342         tcg_gen_op4ii_i64(INDEX_op_sextract_i64, ret, arg, ofs, len);
2343         return;
2344     }
2345
2346     /* Assume that sign-extension, if available, is cheaper than a shift.  */
2347     switch (ofs + len) {
2348     case 32:
2349         if (TCG_TARGET_HAS_ext32s_i64) {
2350             tcg_gen_ext32s_i64(ret, arg);
2351             tcg_gen_sari_i64(ret, ret, ofs);
2352             return;
2353         }
2354         break;
2355     case 16:
2356         if (TCG_TARGET_HAS_ext16s_i64) {
2357             tcg_gen_ext16s_i64(ret, arg);
2358             tcg_gen_sari_i64(ret, ret, ofs);
2359             return;
2360         }
2361         break;
2362     case 8:
2363         if (TCG_TARGET_HAS_ext8s_i64) {
2364             tcg_gen_ext8s_i64(ret, arg);
2365             tcg_gen_sari_i64(ret, ret, ofs);
2366             return;
2367         }
2368         break;
2369     }
2370     switch (len) {
2371     case 32:
2372         if (TCG_TARGET_HAS_ext32s_i64) {
2373             tcg_gen_shri_i64(ret, arg, ofs);
2374             tcg_gen_ext32s_i64(ret, ret);
2375             return;
2376         }
2377         break;
2378     case 16:
2379         if (TCG_TARGET_HAS_ext16s_i64) {
2380             tcg_gen_shri_i64(ret, arg, ofs);
2381             tcg_gen_ext16s_i64(ret, ret);
2382             return;
2383         }
2384         break;
2385     case 8:
2386         if (TCG_TARGET_HAS_ext8s_i64) {
2387             tcg_gen_shri_i64(ret, arg, ofs);
2388             tcg_gen_ext8s_i64(ret, ret);
2389             return;
2390         }
2391         break;
2392     }
2393     tcg_gen_shli_i64(ret, arg, 64 - len - ofs);
2394     tcg_gen_sari_i64(ret, ret, 64 - len);
2395 }
2396
2397 /*
2398  * Extract 64 bits from a 128-bit input, ah:al, starting from ofs.
2399  * Unlike tcg_gen_extract_i64 above, len is fixed at 64.
2400  */
2401 void tcg_gen_extract2_i64(TCGv_i64 ret, TCGv_i64 al, TCGv_i64 ah,
2402                           unsigned int ofs)
2403 {
2404     tcg_debug_assert(ofs <= 64);
2405     if (ofs == 0) {
2406         tcg_gen_mov_i64(ret, al);
2407     } else if (ofs == 64) {
2408         tcg_gen_mov_i64(ret, ah);
2409     } else if (al == ah) {
2410         tcg_gen_rotri_i64(ret, al, ofs);
2411     } else if (TCG_TARGET_HAS_extract2_i64) {
2412         tcg_gen_op4i_i64(INDEX_op_extract2_i64, ret, al, ah, ofs);
2413     } else {
2414         TCGv_i64 t0 = tcg_temp_new_i64();
2415         tcg_gen_shri_i64(t0, al, ofs);
2416         tcg_gen_deposit_i64(ret, t0, ah, 64 - ofs, ofs);
2417         tcg_temp_free_i64(t0);
2418     }
2419 }
2420
2421 void tcg_gen_movcond_i64(TCGCond cond, TCGv_i64 ret, TCGv_i64 c1,
2422                          TCGv_i64 c2, TCGv_i64 v1, TCGv_i64 v2)
2423 {
2424     if (cond == TCG_COND_ALWAYS) {
2425         tcg_gen_mov_i64(ret, v1);
2426     } else if (cond == TCG_COND_NEVER) {
2427         tcg_gen_mov_i64(ret, v2);
2428     } else if (TCG_TARGET_REG_BITS == 32) {
2429         TCGv_i32 t0 = tcg_temp_new_i32();
2430         TCGv_i32 t1 = tcg_temp_new_i32();
2431         tcg_gen_op6i_i32(INDEX_op_setcond2_i32, t0,
2432                          TCGV_LOW(c1), TCGV_HIGH(c1),
2433                          TCGV_LOW(c2), TCGV_HIGH(c2), cond);
2434
2435         if (TCG_TARGET_HAS_movcond_i32) {
2436             tcg_gen_movi_i32(t1, 0);
2437             tcg_gen_movcond_i32(TCG_COND_NE, TCGV_LOW(ret), t0, t1,
2438                                 TCGV_LOW(v1), TCGV_LOW(v2));
2439             tcg_gen_movcond_i32(TCG_COND_NE, TCGV_HIGH(ret), t0, t1,
2440                                 TCGV_HIGH(v1), TCGV_HIGH(v2));
2441         } else {
2442             tcg_gen_neg_i32(t0, t0);
2443
2444             tcg_gen_and_i32(t1, TCGV_LOW(v1), t0);
2445             tcg_gen_andc_i32(TCGV_LOW(ret), TCGV_LOW(v2), t0);
2446             tcg_gen_or_i32(TCGV_LOW(ret), TCGV_LOW(ret), t1);
2447
2448             tcg_gen_and_i32(t1, TCGV_HIGH(v1), t0);
2449             tcg_gen_andc_i32(TCGV_HIGH(ret), TCGV_HIGH(v2), t0);
2450             tcg_gen_or_i32(TCGV_HIGH(ret), TCGV_HIGH(ret), t1);
2451         }
2452         tcg_temp_free_i32(t0);
2453         tcg_temp_free_i32(t1);
2454     } else if (TCG_TARGET_HAS_movcond_i64) {
2455         tcg_gen_op6i_i64(INDEX_op_movcond_i64, ret, c1, c2, v1, v2, cond);
2456     } else {
2457         TCGv_i64 t0 = tcg_temp_new_i64();
2458         TCGv_i64 t1 = tcg_temp_new_i64();
2459         tcg_gen_setcond_i64(cond, t0, c1, c2);
2460         tcg_gen_neg_i64(t0, t0);
2461         tcg_gen_and_i64(t1, v1, t0);
2462         tcg_gen_andc_i64(ret, v2, t0);
2463         tcg_gen_or_i64(ret, ret, t1);
2464         tcg_temp_free_i64(t0);
2465         tcg_temp_free_i64(t1);
2466     }
2467 }
2468
2469 void tcg_gen_add2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 al,
2470                       TCGv_i64 ah, TCGv_i64 bl, TCGv_i64 bh)
2471 {
2472     if (TCG_TARGET_HAS_add2_i64) {
2473         tcg_gen_op6_i64(INDEX_op_add2_i64, rl, rh, al, ah, bl, bh);
2474     } else {
2475         TCGv_i64 t0 = tcg_temp_new_i64();
2476         TCGv_i64 t1 = tcg_temp_new_i64();
2477         tcg_gen_add_i64(t0, al, bl);
2478         tcg_gen_setcond_i64(TCG_COND_LTU, t1, t0, al);
2479         tcg_gen_add_i64(rh, ah, bh);
2480         tcg_gen_add_i64(rh, rh, t1);
2481         tcg_gen_mov_i64(rl, t0);
2482         tcg_temp_free_i64(t0);
2483         tcg_temp_free_i64(t1);
2484     }
2485 }
2486
2487 void tcg_gen_sub2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 al,
2488                       TCGv_i64 ah, TCGv_i64 bl, TCGv_i64 bh)
2489 {
2490     if (TCG_TARGET_HAS_sub2_i64) {
2491         tcg_gen_op6_i64(INDEX_op_sub2_i64, rl, rh, al, ah, bl, bh);
2492     } else {
2493         TCGv_i64 t0 = tcg_temp_new_i64();
2494         TCGv_i64 t1 = tcg_temp_new_i64();
2495         tcg_gen_sub_i64(t0, al, bl);
2496         tcg_gen_setcond_i64(TCG_COND_LTU, t1, al, bl);
2497         tcg_gen_sub_i64(rh, ah, bh);
2498         tcg_gen_sub_i64(rh, rh, t1);
2499         tcg_gen_mov_i64(rl, t0);
2500         tcg_temp_free_i64(t0);
2501         tcg_temp_free_i64(t1);
2502     }
2503 }
2504
2505 void tcg_gen_mulu2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 arg1, TCGv_i64 arg2)
2506 {
2507     if (TCG_TARGET_HAS_mulu2_i64) {
2508         tcg_gen_op4_i64(INDEX_op_mulu2_i64, rl, rh, arg1, arg2);
2509     } else if (TCG_TARGET_HAS_muluh_i64) {
2510         TCGv_i64 t = tcg_temp_new_i64();
2511         tcg_gen_op3_i64(INDEX_op_mul_i64, t, arg1, arg2);
2512         tcg_gen_op3_i64(INDEX_op_muluh_i64, rh, arg1, arg2);
2513         tcg_gen_mov_i64(rl, t);
2514         tcg_temp_free_i64(t);
2515     } else {
2516         TCGv_i64 t0 = tcg_temp_new_i64();
2517         tcg_gen_mul_i64(t0, arg1, arg2);
2518         gen_helper_muluh_i64(rh, arg1, arg2);
2519         tcg_gen_mov_i64(rl, t0);
2520         tcg_temp_free_i64(t0);
2521     }
2522 }
2523
2524 void tcg_gen_muls2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 arg1, TCGv_i64 arg2)
2525 {
2526     if (TCG_TARGET_HAS_muls2_i64) {
2527         tcg_gen_op4_i64(INDEX_op_muls2_i64, rl, rh, arg1, arg2);
2528     } else if (TCG_TARGET_HAS_mulsh_i64) {
2529         TCGv_i64 t = tcg_temp_new_i64();
2530         tcg_gen_op3_i64(INDEX_op_mul_i64, t, arg1, arg2);
2531         tcg_gen_op3_i64(INDEX_op_mulsh_i64, rh, arg1, arg2);
2532         tcg_gen_mov_i64(rl, t);
2533         tcg_temp_free_i64(t);
2534     } else if (TCG_TARGET_HAS_mulu2_i64 || TCG_TARGET_HAS_muluh_i64) {
2535         TCGv_i64 t0 = tcg_temp_new_i64();
2536         TCGv_i64 t1 = tcg_temp_new_i64();
2537         TCGv_i64 t2 = tcg_temp_new_i64();
2538         TCGv_i64 t3 = tcg_temp_new_i64();
2539         tcg_gen_mulu2_i64(t0, t1, arg1, arg2);
2540         /* Adjust for negative inputs.  */
2541         tcg_gen_sari_i64(t2, arg1, 63);
2542         tcg_gen_sari_i64(t3, arg2, 63);
2543         tcg_gen_and_i64(t2, t2, arg2);
2544         tcg_gen_and_i64(t3, t3, arg1);
2545         tcg_gen_sub_i64(rh, t1, t2);
2546         tcg_gen_sub_i64(rh, rh, t3);
2547         tcg_gen_mov_i64(rl, t0);
2548         tcg_temp_free_i64(t0);
2549         tcg_temp_free_i64(t1);
2550         tcg_temp_free_i64(t2);
2551         tcg_temp_free_i64(t3);
2552     } else {
2553         TCGv_i64 t0 = tcg_temp_new_i64();
2554         tcg_gen_mul_i64(t0, arg1, arg2);
2555         gen_helper_mulsh_i64(rh, arg1, arg2);
2556         tcg_gen_mov_i64(rl, t0);
2557         tcg_temp_free_i64(t0);
2558     }
2559 }
2560
2561 void tcg_gen_mulsu2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 arg1, TCGv_i64 arg2)
2562 {
2563     TCGv_i64 t0 = tcg_temp_new_i64();
2564     TCGv_i64 t1 = tcg_temp_new_i64();
2565     TCGv_i64 t2 = tcg_temp_new_i64();
2566     tcg_gen_mulu2_i64(t0, t1, arg1, arg2);
2567     /* Adjust for negative input for the signed arg1.  */
2568     tcg_gen_sari_i64(t2, arg1, 63);
2569     tcg_gen_and_i64(t2, t2, arg2);
2570     tcg_gen_sub_i64(rh, t1, t2);
2571     tcg_gen_mov_i64(rl, t0);
2572     tcg_temp_free_i64(t0);
2573     tcg_temp_free_i64(t1);
2574     tcg_temp_free_i64(t2);
2575 }
2576
2577 void tcg_gen_smin_i64(TCGv_i64 ret, TCGv_i64 a, TCGv_i64 b)
2578 {
2579     tcg_gen_movcond_i64(TCG_COND_LT, ret, a, b, a, b);
2580 }
2581
2582 void tcg_gen_umin_i64(TCGv_i64 ret, TCGv_i64 a, TCGv_i64 b)
2583 {
2584     tcg_gen_movcond_i64(TCG_COND_LTU, ret, a, b, a, b);
2585 }
2586
2587 void tcg_gen_smax_i64(TCGv_i64 ret, TCGv_i64 a, TCGv_i64 b)
2588 {
2589     tcg_gen_movcond_i64(TCG_COND_LT, ret, a, b, b, a);
2590 }
2591
2592 void tcg_gen_umax_i64(TCGv_i64 ret, TCGv_i64 a, TCGv_i64 b)
2593 {
2594     tcg_gen_movcond_i64(TCG_COND_LTU, ret, a, b, b, a);
2595 }
2596
2597 void tcg_gen_abs_i64(TCGv_i64 ret, TCGv_i64 a)
2598 {
2599     TCGv_i64 t = tcg_temp_new_i64();
2600
2601     tcg_gen_sari_i64(t, a, 63);
2602     tcg_gen_xor_i64(ret, a, t);
2603     tcg_gen_sub_i64(ret, ret, t);
2604     tcg_temp_free_i64(t);
2605 }
2606
2607 /* Size changing operations.  */
2608
2609 void tcg_gen_extrl_i64_i32(TCGv_i32 ret, TCGv_i64 arg)
2610 {
2611     if (TCG_TARGET_REG_BITS == 32) {
2612         tcg_gen_mov_i32(ret, TCGV_LOW(arg));
2613     } else if (TCG_TARGET_HAS_extrl_i64_i32) {
2614         tcg_gen_op2(INDEX_op_extrl_i64_i32,
2615                     tcgv_i32_arg(ret), tcgv_i64_arg(arg));
2616     } else {
2617         tcg_gen_mov_i32(ret, (TCGv_i32)arg);
2618     }
2619 }
2620
2621 void tcg_gen_extrh_i64_i32(TCGv_i32 ret, TCGv_i64 arg)
2622 {
2623     if (TCG_TARGET_REG_BITS == 32) {
2624         tcg_gen_mov_i32(ret, TCGV_HIGH(arg));
2625     } else if (TCG_TARGET_HAS_extrh_i64_i32) {
2626         tcg_gen_op2(INDEX_op_extrh_i64_i32,
2627                     tcgv_i32_arg(ret), tcgv_i64_arg(arg));
2628     } else {
2629         TCGv_i64 t = tcg_temp_new_i64();
2630         tcg_gen_shri_i64(t, arg, 32);
2631         tcg_gen_mov_i32(ret, (TCGv_i32)t);
2632         tcg_temp_free_i64(t);
2633     }
2634 }
2635
2636 void tcg_gen_extu_i32_i64(TCGv_i64 ret, TCGv_i32 arg)
2637 {
2638     if (TCG_TARGET_REG_BITS == 32) {
2639         tcg_gen_mov_i32(TCGV_LOW(ret), arg);
2640         tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
2641     } else {
2642         tcg_gen_op2(INDEX_op_extu_i32_i64,
2643                     tcgv_i64_arg(ret), tcgv_i32_arg(arg));
2644     }
2645 }
2646
2647 void tcg_gen_ext_i32_i64(TCGv_i64 ret, TCGv_i32 arg)
2648 {
2649     if (TCG_TARGET_REG_BITS == 32) {
2650         tcg_gen_mov_i32(TCGV_LOW(ret), arg);
2651         tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
2652     } else {
2653         tcg_gen_op2(INDEX_op_ext_i32_i64,
2654                     tcgv_i64_arg(ret), tcgv_i32_arg(arg));
2655     }
2656 }
2657
2658 void tcg_gen_concat_i32_i64(TCGv_i64 dest, TCGv_i32 low, TCGv_i32 high)
2659 {
2660     TCGv_i64 tmp;
2661
2662     if (TCG_TARGET_REG_BITS == 32) {
2663         tcg_gen_mov_i32(TCGV_LOW(dest), low);
2664         tcg_gen_mov_i32(TCGV_HIGH(dest), high);
2665         return;
2666     }
2667
2668     tmp = tcg_temp_new_i64();
2669     /* These extensions are only needed for type correctness.
2670        We may be able to do better given target specific information.  */
2671     tcg_gen_extu_i32_i64(tmp, high);
2672     tcg_gen_extu_i32_i64(dest, low);
2673     /* If deposit is available, use it.  Otherwise use the extra
2674        knowledge that we have of the zero-extensions above.  */
2675     if (TCG_TARGET_HAS_deposit_i64 && TCG_TARGET_deposit_i64_valid(32, 32)) {
2676         tcg_gen_deposit_i64(dest, dest, tmp, 32, 32);
2677     } else {
2678         tcg_gen_shli_i64(tmp, tmp, 32);
2679         tcg_gen_or_i64(dest, dest, tmp);
2680     }
2681     tcg_temp_free_i64(tmp);
2682 }
2683
2684 void tcg_gen_extr_i64_i32(TCGv_i32 lo, TCGv_i32 hi, TCGv_i64 arg)
2685 {
2686     if (TCG_TARGET_REG_BITS == 32) {
2687         tcg_gen_mov_i32(lo, TCGV_LOW(arg));
2688         tcg_gen_mov_i32(hi, TCGV_HIGH(arg));
2689     } else {
2690         tcg_gen_extrl_i64_i32(lo, arg);
2691         tcg_gen_extrh_i64_i32(hi, arg);
2692     }
2693 }
2694
2695 void tcg_gen_extr32_i64(TCGv_i64 lo, TCGv_i64 hi, TCGv_i64 arg)
2696 {
2697     tcg_gen_ext32u_i64(lo, arg);
2698     tcg_gen_shri_i64(hi, arg, 32);
2699 }
2700
2701 /* QEMU specific operations.  */
2702
2703 void tcg_gen_exit_tb(const TranslationBlock *tb, unsigned idx)
2704 {
2705     /*
2706      * Let the jit code return the read-only version of the
2707      * TranslationBlock, so that we minimize the pc-relative
2708      * distance of the address of the exit_tb code to TB.
2709      * This will improve utilization of pc-relative address loads.
2710      *
2711      * TODO: Move this to translator_loop, so that all const
2712      * TranslationBlock pointers refer to read-only memory.
2713      * This requires coordination with targets that do not use
2714      * the translator_loop.
2715      */
2716     uintptr_t val = (uintptr_t)tcg_splitwx_to_rx((void *)tb) + idx;
2717
2718     if (tb == NULL) {
2719         tcg_debug_assert(idx == 0);
2720     } else if (idx <= TB_EXIT_IDXMAX) {
2721 #ifdef CONFIG_DEBUG_TCG
2722         /* This is an exit following a goto_tb.  Verify that we have
2723            seen this numbered exit before, via tcg_gen_goto_tb.  */
2724         tcg_debug_assert(tcg_ctx->goto_tb_issue_mask & (1 << idx));
2725 #endif
2726         /* When not chaining, exit without indicating a link.  */
2727         if (qemu_loglevel_mask(CPU_LOG_TB_NOCHAIN)) {
2728             val = 0;
2729         }
2730     } else {
2731         /* This is an exit via the exitreq label.  */
2732         tcg_debug_assert(idx == TB_EXIT_REQUESTED);
2733     }
2734
2735     plugin_gen_disable_mem_helpers();
2736     tcg_gen_op1i(INDEX_op_exit_tb, val);
2737 }
2738
2739 void tcg_gen_goto_tb(unsigned idx)
2740 {
2741     /* We only support two chained exits.  */
2742     tcg_debug_assert(idx <= TB_EXIT_IDXMAX);
2743 #ifdef CONFIG_DEBUG_TCG
2744     /* Verify that we havn't seen this numbered exit before.  */
2745     tcg_debug_assert((tcg_ctx->goto_tb_issue_mask & (1 << idx)) == 0);
2746     tcg_ctx->goto_tb_issue_mask |= 1 << idx;
2747 #endif
2748     plugin_gen_disable_mem_helpers();
2749     /* When not chaining, we simply fall through to the "fallback" exit.  */
2750     if (!qemu_loglevel_mask(CPU_LOG_TB_NOCHAIN)) {
2751         tcg_gen_op1i(INDEX_op_goto_tb, idx);
2752     }
2753 }
2754
2755 void tcg_gen_lookup_and_goto_ptr(void)
2756 {
2757     if (TCG_TARGET_HAS_goto_ptr && !qemu_loglevel_mask(CPU_LOG_TB_NOCHAIN)) {
2758         TCGv_ptr ptr;
2759
2760         plugin_gen_disable_mem_helpers();
2761         ptr = tcg_temp_new_ptr();
2762         gen_helper_lookup_tb_ptr(ptr, cpu_env);
2763         tcg_gen_op1i(INDEX_op_goto_ptr, tcgv_ptr_arg(ptr));
2764         tcg_temp_free_ptr(ptr);
2765     } else {
2766         tcg_gen_exit_tb(NULL, 0);
2767     }
2768 }
2769
2770 static inline MemOp tcg_canonicalize_memop(MemOp op, bool is64, bool st)
2771 {
2772     /* Trigger the asserts within as early as possible.  */
2773     (void)get_alignment_bits(op);
2774
2775     switch (op & MO_SIZE) {
2776     case MO_8:
2777         op &= ~MO_BSWAP;
2778         break;
2779     case MO_16:
2780         break;
2781     case MO_32:
2782         if (!is64) {
2783             op &= ~MO_SIGN;
2784         }
2785         break;
2786     case MO_64:
2787         if (!is64) {
2788             tcg_abort();
2789         }
2790         break;
2791     }
2792     if (st) {
2793         op &= ~MO_SIGN;
2794     }
2795     return op;
2796 }
2797
2798 static void gen_ldst_i32(TCGOpcode opc, TCGv_i32 val, TCGv addr,
2799                          MemOp memop, TCGArg idx)
2800 {
2801     TCGMemOpIdx oi = make_memop_idx(memop, idx);
2802 #if TARGET_LONG_BITS == 32
2803     tcg_gen_op3i_i32(opc, val, addr, oi);
2804 #else
2805     if (TCG_TARGET_REG_BITS == 32) {
2806         tcg_gen_op4i_i32(opc, val, TCGV_LOW(addr), TCGV_HIGH(addr), oi);
2807     } else {
2808         tcg_gen_op3(opc, tcgv_i32_arg(val), tcgv_i64_arg(addr), oi);
2809     }
2810 #endif
2811 }
2812
2813 static void gen_ldst_i64(TCGOpcode opc, TCGv_i64 val, TCGv addr,
2814                          MemOp memop, TCGArg idx)
2815 {
2816     TCGMemOpIdx oi = make_memop_idx(memop, idx);
2817 #if TARGET_LONG_BITS == 32
2818     if (TCG_TARGET_REG_BITS == 32) {
2819         tcg_gen_op4i_i32(opc, TCGV_LOW(val), TCGV_HIGH(val), addr, oi);
2820     } else {
2821         tcg_gen_op3(opc, tcgv_i64_arg(val), tcgv_i32_arg(addr), oi);
2822     }
2823 #else
2824     if (TCG_TARGET_REG_BITS == 32) {
2825         tcg_gen_op5i_i32(opc, TCGV_LOW(val), TCGV_HIGH(val),
2826                          TCGV_LOW(addr), TCGV_HIGH(addr), oi);
2827     } else {
2828         tcg_gen_op3i_i64(opc, val, addr, oi);
2829     }
2830 #endif
2831 }
2832
2833 static void tcg_gen_req_mo(TCGBar type)
2834 {
2835 #ifdef TCG_GUEST_DEFAULT_MO
2836     type &= TCG_GUEST_DEFAULT_MO;
2837 #endif
2838     type &= ~TCG_TARGET_DEFAULT_MO;
2839     if (type) {
2840         tcg_gen_mb(type | TCG_BAR_SC);
2841     }
2842 }
2843
2844 static inline TCGv plugin_prep_mem_callbacks(TCGv vaddr)
2845 {
2846 #ifdef CONFIG_PLUGIN
2847     if (tcg_ctx->plugin_insn != NULL) {
2848         /* Save a copy of the vaddr for use after a load.  */
2849         TCGv temp = tcg_temp_new();
2850         tcg_gen_mov_tl(temp, vaddr);
2851         return temp;
2852     }
2853 #endif
2854     return vaddr;
2855 }
2856
2857 static inline void plugin_gen_mem_callbacks(TCGv vaddr, uint16_t info)
2858 {
2859 #ifdef CONFIG_PLUGIN
2860     if (tcg_ctx->plugin_insn != NULL) {
2861         plugin_gen_empty_mem_callback(vaddr, info);
2862         tcg_temp_free(vaddr);
2863     }
2864 #endif
2865 }
2866
2867 void tcg_gen_qemu_ld_i32(TCGv_i32 val, TCGv addr, TCGArg idx, MemOp memop)
2868 {
2869     MemOp orig_memop;
2870     uint16_t info = trace_mem_get_info(memop, idx, 0);
2871
2872     tcg_gen_req_mo(TCG_MO_LD_LD | TCG_MO_ST_LD);
2873     memop = tcg_canonicalize_memop(memop, 0, 0);
2874     trace_guest_mem_before_tcg(tcg_ctx->cpu, cpu_env, addr, info);
2875
2876     orig_memop = memop;
2877     if (!TCG_TARGET_HAS_MEMORY_BSWAP && (memop & MO_BSWAP)) {
2878         memop &= ~MO_BSWAP;
2879         /* The bswap primitive benefits from zero-extended input.  */
2880         if ((memop & MO_SSIZE) == MO_SW) {
2881             memop &= ~MO_SIGN;
2882         }
2883     }
2884
2885     addr = plugin_prep_mem_callbacks(addr);
2886     gen_ldst_i32(INDEX_op_qemu_ld_i32, val, addr, memop, idx);
2887     plugin_gen_mem_callbacks(addr, info);
2888
2889     if ((orig_memop ^ memop) & MO_BSWAP) {
2890         switch (orig_memop & MO_SIZE) {
2891         case MO_16:
2892             tcg_gen_bswap16_i32(val, val, (orig_memop & MO_SIGN
2893                                            ? TCG_BSWAP_IZ | TCG_BSWAP_OS
2894                                            : TCG_BSWAP_IZ | TCG_BSWAP_OZ));
2895             break;
2896         case MO_32:
2897             tcg_gen_bswap32_i32(val, val);
2898             break;
2899         default:
2900             g_assert_not_reached();
2901         }
2902     }
2903 }
2904
2905 void tcg_gen_qemu_st_i32(TCGv_i32 val, TCGv addr, TCGArg idx, MemOp memop)
2906 {
2907     TCGv_i32 swap = NULL;
2908     uint16_t info = trace_mem_get_info(memop, idx, 1);
2909
2910     tcg_gen_req_mo(TCG_MO_LD_ST | TCG_MO_ST_ST);
2911     memop = tcg_canonicalize_memop(memop, 0, 1);
2912     trace_guest_mem_before_tcg(tcg_ctx->cpu, cpu_env, addr, info);
2913
2914     if (!TCG_TARGET_HAS_MEMORY_BSWAP && (memop & MO_BSWAP)) {
2915         swap = tcg_temp_new_i32();
2916         switch (memop & MO_SIZE) {
2917         case MO_16:
2918             tcg_gen_bswap16_i32(swap, val, 0);
2919             break;
2920         case MO_32:
2921             tcg_gen_bswap32_i32(swap, val);
2922             break;
2923         default:
2924             g_assert_not_reached();
2925         }
2926         val = swap;
2927         memop &= ~MO_BSWAP;
2928     }
2929
2930     addr = plugin_prep_mem_callbacks(addr);
2931     if (TCG_TARGET_HAS_qemu_st8_i32 && (memop & MO_SIZE) == MO_8) {
2932         gen_ldst_i32(INDEX_op_qemu_st8_i32, val, addr, memop, idx);
2933     } else {
2934         gen_ldst_i32(INDEX_op_qemu_st_i32, val, addr, memop, idx);
2935     }
2936     plugin_gen_mem_callbacks(addr, info);
2937
2938     if (swap) {
2939         tcg_temp_free_i32(swap);
2940     }
2941 }
2942
2943 void tcg_gen_qemu_ld_i64(TCGv_i64 val, TCGv addr, TCGArg idx, MemOp memop)
2944 {
2945     MemOp orig_memop;
2946     uint16_t info;
2947
2948     if (TCG_TARGET_REG_BITS == 32 && (memop & MO_SIZE) < MO_64) {
2949         tcg_gen_qemu_ld_i32(TCGV_LOW(val), addr, idx, memop);
2950         if (memop & MO_SIGN) {
2951             tcg_gen_sari_i32(TCGV_HIGH(val), TCGV_LOW(val), 31);
2952         } else {
2953             tcg_gen_movi_i32(TCGV_HIGH(val), 0);
2954         }
2955         return;
2956     }
2957
2958     tcg_gen_req_mo(TCG_MO_LD_LD | TCG_MO_ST_LD);
2959     memop = tcg_canonicalize_memop(memop, 1, 0);
2960     info = trace_mem_get_info(memop, idx, 0);
2961     trace_guest_mem_before_tcg(tcg_ctx->cpu, cpu_env, addr, info);
2962
2963     orig_memop = memop;
2964     if (!TCG_TARGET_HAS_MEMORY_BSWAP && (memop & MO_BSWAP)) {
2965         memop &= ~MO_BSWAP;
2966         /* The bswap primitive benefits from zero-extended input.  */
2967         if ((memop & MO_SIGN) && (memop & MO_SIZE) < MO_64) {
2968             memop &= ~MO_SIGN;
2969         }
2970     }
2971
2972     addr = plugin_prep_mem_callbacks(addr);
2973     gen_ldst_i64(INDEX_op_qemu_ld_i64, val, addr, memop, idx);
2974     plugin_gen_mem_callbacks(addr, info);
2975
2976     if ((orig_memop ^ memop) & MO_BSWAP) {
2977         int flags = (orig_memop & MO_SIGN
2978                      ? TCG_BSWAP_IZ | TCG_BSWAP_OS
2979                      : TCG_BSWAP_IZ | TCG_BSWAP_OZ);
2980         switch (orig_memop & MO_SIZE) {
2981         case MO_16:
2982             tcg_gen_bswap16_i64(val, val, flags);
2983             break;
2984         case MO_32:
2985             tcg_gen_bswap32_i64(val, val, flags);
2986             break;
2987         case MO_64:
2988             tcg_gen_bswap64_i64(val, val);
2989             break;
2990         default:
2991             g_assert_not_reached();
2992         }
2993     }
2994 }
2995
2996 void tcg_gen_qemu_st_i64(TCGv_i64 val, TCGv addr, TCGArg idx, MemOp memop)
2997 {
2998     TCGv_i64 swap = NULL;
2999     uint16_t info;
3000
3001     if (TCG_TARGET_REG_BITS == 32 && (memop & MO_SIZE) < MO_64) {
3002         tcg_gen_qemu_st_i32(TCGV_LOW(val), addr, idx, memop);
3003         return;
3004     }
3005
3006     tcg_gen_req_mo(TCG_MO_LD_ST | TCG_MO_ST_ST);
3007     memop = tcg_canonicalize_memop(memop, 1, 1);
3008     info = trace_mem_get_info(memop, idx, 1);
3009     trace_guest_mem_before_tcg(tcg_ctx->cpu, cpu_env, addr, info);
3010
3011     if (!TCG_TARGET_HAS_MEMORY_BSWAP && (memop & MO_BSWAP)) {
3012         swap = tcg_temp_new_i64();
3013         switch (memop & MO_SIZE) {
3014         case MO_16:
3015             tcg_gen_bswap16_i64(swap, val, 0);
3016             break;
3017         case MO_32:
3018             tcg_gen_bswap32_i64(swap, val, 0);
3019             break;
3020         case MO_64:
3021             tcg_gen_bswap64_i64(swap, val);
3022             break;
3023         default:
3024             g_assert_not_reached();
3025         }
3026         val = swap;
3027         memop &= ~MO_BSWAP;
3028     }
3029
3030     addr = plugin_prep_mem_callbacks(addr);
3031     gen_ldst_i64(INDEX_op_qemu_st_i64, val, addr, memop, idx);
3032     plugin_gen_mem_callbacks(addr, info);
3033
3034     if (swap) {
3035         tcg_temp_free_i64(swap);
3036     }
3037 }
3038
3039 static void tcg_gen_ext_i32(TCGv_i32 ret, TCGv_i32 val, MemOp opc)
3040 {
3041     switch (opc & MO_SSIZE) {
3042     case MO_SB:
3043         tcg_gen_ext8s_i32(ret, val);
3044         break;
3045     case MO_UB:
3046         tcg_gen_ext8u_i32(ret, val);
3047         break;
3048     case MO_SW:
3049         tcg_gen_ext16s_i32(ret, val);
3050         break;
3051     case MO_UW:
3052         tcg_gen_ext16u_i32(ret, val);
3053         break;
3054     default:
3055         tcg_gen_mov_i32(ret, val);
3056         break;
3057     }
3058 }
3059
3060 static void tcg_gen_ext_i64(TCGv_i64 ret, TCGv_i64 val, MemOp opc)
3061 {
3062     switch (opc & MO_SSIZE) {
3063     case MO_SB:
3064         tcg_gen_ext8s_i64(ret, val);
3065         break;
3066     case MO_UB:
3067         tcg_gen_ext8u_i64(ret, val);
3068         break;
3069     case MO_SW:
3070         tcg_gen_ext16s_i64(ret, val);
3071         break;
3072     case MO_UW:
3073         tcg_gen_ext16u_i64(ret, val);
3074         break;
3075     case MO_SL:
3076         tcg_gen_ext32s_i64(ret, val);
3077         break;
3078     case MO_UL:
3079         tcg_gen_ext32u_i64(ret, val);
3080         break;
3081     default:
3082         tcg_gen_mov_i64(ret, val);
3083         break;
3084     }
3085 }
3086
3087 #ifdef CONFIG_SOFTMMU
3088 typedef void (*gen_atomic_cx_i32)(TCGv_i32, TCGv_env, TCGv,
3089                                   TCGv_i32, TCGv_i32, TCGv_i32);
3090 typedef void (*gen_atomic_cx_i64)(TCGv_i64, TCGv_env, TCGv,
3091                                   TCGv_i64, TCGv_i64, TCGv_i32);
3092 typedef void (*gen_atomic_op_i32)(TCGv_i32, TCGv_env, TCGv,
3093                                   TCGv_i32, TCGv_i32);
3094 typedef void (*gen_atomic_op_i64)(TCGv_i64, TCGv_env, TCGv,
3095                                   TCGv_i64, TCGv_i32);
3096 #else
3097 typedef void (*gen_atomic_cx_i32)(TCGv_i32, TCGv_env, TCGv, TCGv_i32, TCGv_i32);
3098 typedef void (*gen_atomic_cx_i64)(TCGv_i64, TCGv_env, TCGv, TCGv_i64, TCGv_i64);
3099 typedef void (*gen_atomic_op_i32)(TCGv_i32, TCGv_env, TCGv, TCGv_i32);
3100 typedef void (*gen_atomic_op_i64)(TCGv_i64, TCGv_env, TCGv, TCGv_i64);
3101 #endif
3102
3103 #ifdef CONFIG_ATOMIC64
3104 # define WITH_ATOMIC64(X) X,
3105 #else
3106 # define WITH_ATOMIC64(X)
3107 #endif
3108
3109 static void * const table_cmpxchg[16] = {
3110     [MO_8] = gen_helper_atomic_cmpxchgb,
3111     [MO_16 | MO_LE] = gen_helper_atomic_cmpxchgw_le,
3112     [MO_16 | MO_BE] = gen_helper_atomic_cmpxchgw_be,
3113     [MO_32 | MO_LE] = gen_helper_atomic_cmpxchgl_le,
3114     [MO_32 | MO_BE] = gen_helper_atomic_cmpxchgl_be,
3115     WITH_ATOMIC64([MO_64 | MO_LE] = gen_helper_atomic_cmpxchgq_le)
3116     WITH_ATOMIC64([MO_64 | MO_BE] = gen_helper_atomic_cmpxchgq_be)
3117 };
3118
3119 void tcg_gen_atomic_cmpxchg_i32(TCGv_i32 retv, TCGv addr, TCGv_i32 cmpv,
3120                                 TCGv_i32 newv, TCGArg idx, MemOp memop)
3121 {
3122     memop = tcg_canonicalize_memop(memop, 0, 0);
3123
3124     if (!(tcg_ctx->tb_cflags & CF_PARALLEL)) {
3125         TCGv_i32 t1 = tcg_temp_new_i32();
3126         TCGv_i32 t2 = tcg_temp_new_i32();
3127
3128         tcg_gen_ext_i32(t2, cmpv, memop & MO_SIZE);
3129
3130         tcg_gen_qemu_ld_i32(t1, addr, idx, memop & ~MO_SIGN);
3131         tcg_gen_movcond_i32(TCG_COND_EQ, t2, t1, t2, newv, t1);
3132         tcg_gen_qemu_st_i32(t2, addr, idx, memop);
3133         tcg_temp_free_i32(t2);
3134
3135         if (memop & MO_SIGN) {
3136             tcg_gen_ext_i32(retv, t1, memop);
3137         } else {
3138             tcg_gen_mov_i32(retv, t1);
3139         }
3140         tcg_temp_free_i32(t1);
3141     } else {
3142         gen_atomic_cx_i32 gen;
3143
3144         gen = table_cmpxchg[memop & (MO_SIZE | MO_BSWAP)];
3145         tcg_debug_assert(gen != NULL);
3146
3147 #ifdef CONFIG_SOFTMMU
3148         {
3149             TCGMemOpIdx oi = make_memop_idx(memop & ~MO_SIGN, idx);
3150             gen(retv, cpu_env, addr, cmpv, newv, tcg_constant_i32(oi));
3151         }
3152 #else
3153         gen(retv, cpu_env, addr, cmpv, newv);
3154 #endif
3155
3156         if (memop & MO_SIGN) {
3157             tcg_gen_ext_i32(retv, retv, memop);
3158         }
3159     }
3160 }
3161
3162 void tcg_gen_atomic_cmpxchg_i64(TCGv_i64 retv, TCGv addr, TCGv_i64 cmpv,
3163                                 TCGv_i64 newv, TCGArg idx, MemOp memop)
3164 {
3165     memop = tcg_canonicalize_memop(memop, 1, 0);
3166
3167     if (!(tcg_ctx->tb_cflags & CF_PARALLEL)) {
3168         TCGv_i64 t1 = tcg_temp_new_i64();
3169         TCGv_i64 t2 = tcg_temp_new_i64();
3170
3171         tcg_gen_ext_i64(t2, cmpv, memop & MO_SIZE);
3172
3173         tcg_gen_qemu_ld_i64(t1, addr, idx, memop & ~MO_SIGN);
3174         tcg_gen_movcond_i64(TCG_COND_EQ, t2, t1, t2, newv, t1);
3175         tcg_gen_qemu_st_i64(t2, addr, idx, memop);
3176         tcg_temp_free_i64(t2);
3177
3178         if (memop & MO_SIGN) {
3179             tcg_gen_ext_i64(retv, t1, memop);
3180         } else {
3181             tcg_gen_mov_i64(retv, t1);
3182         }
3183         tcg_temp_free_i64(t1);
3184     } else if ((memop & MO_SIZE) == MO_64) {
3185 #ifdef CONFIG_ATOMIC64
3186         gen_atomic_cx_i64 gen;
3187
3188         gen = table_cmpxchg[memop & (MO_SIZE | MO_BSWAP)];
3189         tcg_debug_assert(gen != NULL);
3190
3191 #ifdef CONFIG_SOFTMMU
3192         {
3193             TCGMemOpIdx oi = make_memop_idx(memop, idx);
3194             gen(retv, cpu_env, addr, cmpv, newv, tcg_constant_i32(oi));
3195         }
3196 #else
3197         gen(retv, cpu_env, addr, cmpv, newv);
3198 #endif
3199 #else
3200         gen_helper_exit_atomic(cpu_env);
3201         /* Produce a result, so that we have a well-formed opcode stream
3202            with respect to uses of the result in the (dead) code following.  */
3203         tcg_gen_movi_i64(retv, 0);
3204 #endif /* CONFIG_ATOMIC64 */
3205     } else {
3206         TCGv_i32 c32 = tcg_temp_new_i32();
3207         TCGv_i32 n32 = tcg_temp_new_i32();
3208         TCGv_i32 r32 = tcg_temp_new_i32();
3209
3210         tcg_gen_extrl_i64_i32(c32, cmpv);
3211         tcg_gen_extrl_i64_i32(n32, newv);
3212         tcg_gen_atomic_cmpxchg_i32(r32, addr, c32, n32, idx, memop & ~MO_SIGN);
3213         tcg_temp_free_i32(c32);
3214         tcg_temp_free_i32(n32);
3215
3216         tcg_gen_extu_i32_i64(retv, r32);
3217         tcg_temp_free_i32(r32);
3218
3219         if (memop & MO_SIGN) {
3220             tcg_gen_ext_i64(retv, retv, memop);
3221         }
3222     }
3223 }
3224
3225 static void do_nonatomic_op_i32(TCGv_i32 ret, TCGv addr, TCGv_i32 val,
3226                                 TCGArg idx, MemOp memop, bool new_val,
3227                                 void (*gen)(TCGv_i32, TCGv_i32, TCGv_i32))
3228 {
3229     TCGv_i32 t1 = tcg_temp_new_i32();
3230     TCGv_i32 t2 = tcg_temp_new_i32();
3231
3232     memop = tcg_canonicalize_memop(memop, 0, 0);
3233
3234     tcg_gen_qemu_ld_i32(t1, addr, idx, memop);
3235     tcg_gen_ext_i32(t2, val, memop);
3236     gen(t2, t1, t2);
3237     tcg_gen_qemu_st_i32(t2, addr, idx, memop);
3238
3239     tcg_gen_ext_i32(ret, (new_val ? t2 : t1), memop);
3240     tcg_temp_free_i32(t1);
3241     tcg_temp_free_i32(t2);
3242 }
3243
3244 static void do_atomic_op_i32(TCGv_i32 ret, TCGv addr, TCGv_i32 val,
3245                              TCGArg idx, MemOp memop, void * const table[])
3246 {
3247     gen_atomic_op_i32 gen;
3248
3249     memop = tcg_canonicalize_memop(memop, 0, 0);
3250
3251     gen = table[memop & (MO_SIZE | MO_BSWAP)];
3252     tcg_debug_assert(gen != NULL);
3253
3254 #ifdef CONFIG_SOFTMMU
3255     {
3256         TCGMemOpIdx oi = make_memop_idx(memop & ~MO_SIGN, idx);
3257         gen(ret, cpu_env, addr, val, tcg_constant_i32(oi));
3258     }
3259 #else
3260     gen(ret, cpu_env, addr, val);
3261 #endif
3262
3263     if (memop & MO_SIGN) {
3264         tcg_gen_ext_i32(ret, ret, memop);
3265     }
3266 }
3267
3268 static void do_nonatomic_op_i64(TCGv_i64 ret, TCGv addr, TCGv_i64 val,
3269                                 TCGArg idx, MemOp memop, bool new_val,
3270                                 void (*gen)(TCGv_i64, TCGv_i64, TCGv_i64))
3271 {
3272     TCGv_i64 t1 = tcg_temp_new_i64();
3273     TCGv_i64 t2 = tcg_temp_new_i64();
3274
3275     memop = tcg_canonicalize_memop(memop, 1, 0);
3276
3277     tcg_gen_qemu_ld_i64(t1, addr, idx, memop);
3278     tcg_gen_ext_i64(t2, val, memop);
3279     gen(t2, t1, t2);
3280     tcg_gen_qemu_st_i64(t2, addr, idx, memop);
3281
3282     tcg_gen_ext_i64(ret, (new_val ? t2 : t1), memop);
3283     tcg_temp_free_i64(t1);
3284     tcg_temp_free_i64(t2);
3285 }
3286
3287 static void do_atomic_op_i64(TCGv_i64 ret, TCGv addr, TCGv_i64 val,
3288                              TCGArg idx, MemOp memop, void * const table[])
3289 {
3290     memop = tcg_canonicalize_memop(memop, 1, 0);
3291
3292     if ((memop & MO_SIZE) == MO_64) {
3293 #ifdef CONFIG_ATOMIC64
3294         gen_atomic_op_i64 gen;
3295
3296         gen = table[memop & (MO_SIZE | MO_BSWAP)];
3297         tcg_debug_assert(gen != NULL);
3298
3299 #ifdef CONFIG_SOFTMMU
3300         {
3301             TCGMemOpIdx oi = make_memop_idx(memop & ~MO_SIGN, idx);
3302             gen(ret, cpu_env, addr, val, tcg_constant_i32(oi));
3303         }
3304 #else
3305         gen(ret, cpu_env, addr, val);
3306 #endif
3307 #else
3308         gen_helper_exit_atomic(cpu_env);
3309         /* Produce a result, so that we have a well-formed opcode stream
3310            with respect to uses of the result in the (dead) code following.  */
3311         tcg_gen_movi_i64(ret, 0);
3312 #endif /* CONFIG_ATOMIC64 */
3313     } else {
3314         TCGv_i32 v32 = tcg_temp_new_i32();
3315         TCGv_i32 r32 = tcg_temp_new_i32();
3316
3317         tcg_gen_extrl_i64_i32(v32, val);
3318         do_atomic_op_i32(r32, addr, v32, idx, memop & ~MO_SIGN, table);
3319         tcg_temp_free_i32(v32);
3320
3321         tcg_gen_extu_i32_i64(ret, r32);
3322         tcg_temp_free_i32(r32);
3323
3324         if (memop & MO_SIGN) {
3325             tcg_gen_ext_i64(ret, ret, memop);
3326         }
3327     }
3328 }
3329
3330 #define GEN_ATOMIC_HELPER(NAME, OP, NEW)                                \
3331 static void * const table_##NAME[16] = {                                \
3332     [MO_8] = gen_helper_atomic_##NAME##b,                               \
3333     [MO_16 | MO_LE] = gen_helper_atomic_##NAME##w_le,                   \
3334     [MO_16 | MO_BE] = gen_helper_atomic_##NAME##w_be,                   \
3335     [MO_32 | MO_LE] = gen_helper_atomic_##NAME##l_le,                   \
3336     [MO_32 | MO_BE] = gen_helper_atomic_##NAME##l_be,                   \
3337     WITH_ATOMIC64([MO_64 | MO_LE] = gen_helper_atomic_##NAME##q_le)     \
3338     WITH_ATOMIC64([MO_64 | MO_BE] = gen_helper_atomic_##NAME##q_be)     \
3339 };                                                                      \
3340 void tcg_gen_atomic_##NAME##_i32                                        \
3341     (TCGv_i32 ret, TCGv addr, TCGv_i32 val, TCGArg idx, MemOp memop)    \
3342 {                                                                       \
3343     if (tcg_ctx->tb_cflags & CF_PARALLEL) {                             \
3344         do_atomic_op_i32(ret, addr, val, idx, memop, table_##NAME);     \
3345     } else {                                                            \
3346         do_nonatomic_op_i32(ret, addr, val, idx, memop, NEW,            \
3347                             tcg_gen_##OP##_i32);                        \
3348     }                                                                   \
3349 }                                                                       \
3350 void tcg_gen_atomic_##NAME##_i64                                        \
3351     (TCGv_i64 ret, TCGv addr, TCGv_i64 val, TCGArg idx, MemOp memop)    \
3352 {                                                                       \
3353     if (tcg_ctx->tb_cflags & CF_PARALLEL) {                             \
3354         do_atomic_op_i64(ret, addr, val, idx, memop, table_##NAME);     \
3355     } else {                                                            \
3356         do_nonatomic_op_i64(ret, addr, val, idx, memop, NEW,            \
3357                             tcg_gen_##OP##_i64);                        \
3358     }                                                                   \
3359 }
3360
3361 GEN_ATOMIC_HELPER(fetch_add, add, 0)
3362 GEN_ATOMIC_HELPER(fetch_and, and, 0)
3363 GEN_ATOMIC_HELPER(fetch_or, or, 0)
3364 GEN_ATOMIC_HELPER(fetch_xor, xor, 0)
3365 GEN_ATOMIC_HELPER(fetch_smin, smin, 0)
3366 GEN_ATOMIC_HELPER(fetch_umin, umin, 0)
3367 GEN_ATOMIC_HELPER(fetch_smax, smax, 0)
3368 GEN_ATOMIC_HELPER(fetch_umax, umax, 0)
3369
3370 GEN_ATOMIC_HELPER(add_fetch, add, 1)
3371 GEN_ATOMIC_HELPER(and_fetch, and, 1)
3372 GEN_ATOMIC_HELPER(or_fetch, or, 1)
3373 GEN_ATOMIC_HELPER(xor_fetch, xor, 1)
3374 GEN_ATOMIC_HELPER(smin_fetch, smin, 1)
3375 GEN_ATOMIC_HELPER(umin_fetch, umin, 1)
3376 GEN_ATOMIC_HELPER(smax_fetch, smax, 1)
3377 GEN_ATOMIC_HELPER(umax_fetch, umax, 1)
3378
3379 static void tcg_gen_mov2_i32(TCGv_i32 r, TCGv_i32 a, TCGv_i32 b)
3380 {
3381     tcg_gen_mov_i32(r, b);
3382 }
3383
3384 static void tcg_gen_mov2_i64(TCGv_i64 r, TCGv_i64 a, TCGv_i64 b)
3385 {
3386     tcg_gen_mov_i64(r, b);
3387 }
3388
3389 GEN_ATOMIC_HELPER(xchg, mov2, 0)
3390
3391 #undef GEN_ATOMIC_HELPER