]> Git Repo - qemu.git/blame - tcg/tcg-op.c
projects / qemu.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Merge remote-tracking branch 'remotes/kraxel/tags/usb-20170913-pull-request' into...
[qemu.git] / tcg / tcg-op.c
CommitLineData
951c6300
RH		 1/*
2 * Tiny Code Generator for QEMU
3 *
4 * Copyright (c) 2008 Fabrice Bellard
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a copy
7 * of this software and associated documentation files (the "Software"), to deal
8 * in the Software without restriction, including without limitation the rights
9 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10 * copies of the Software, and to permit persons to whom the Software is
11 * furnished to do so, subject to the following conditions:
12 *
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
22 * THE SOFTWARE.
23 */
24
757e725b 25#include "qemu/osdep.h"
33c11879
PB		 26#include "qemu-common.h"
27#include "cpu.h"
63c91552 28#include "exec/exec-all.h"
951c6300
RH		 29#include "tcg.h"
30#include "tcg-op.h"
b32dc337 31#include "tcg-mo.h"
dcdaadb6
LV		 32#include "trace-tcg.h"
33#include "trace/mem.h"
951c6300 34
3a13c3f3
RH		 35/* Reduce the number of ifdefs below. This assumes that all uses of
36 TCGV_HIGH and TCGV_LOW are properly protected by a conditional that
37 the compiler can eliminate. */
38#if TCG_TARGET_REG_BITS == 64
39extern TCGv_i32 TCGV_LOW_link_error(TCGv_i64);
40extern TCGv_i32 TCGV_HIGH_link_error(TCGv_i64);
41#define TCGV_LOW TCGV_LOW_link_error
42#define TCGV_HIGH TCGV_HIGH_link_error
43#endif
951c6300 44
c45cb8bb
RH		 45/* Note that this is optimized for sequential allocation during translate.
46 Up to and including filling in the forward link immediately. We'll do
47 proper termination of the end of the list after we finish translation. */
48
49static void tcg_emit_op(TCGContext *ctx, TCGOpcode opc, int args)
50{
51 int oi = ctx->gen_next_op_idx;
52 int ni = oi + 1;
53 int pi = oi - 1;
54
55 tcg_debug_assert(oi < OPC_BUF_SIZE);
dcb8e758 56 ctx->gen_op_buf[0].prev = oi;
c45cb8bb
RH		 57 ctx->gen_next_op_idx = ni;
58
59 ctx->gen_op_buf[oi] = (TCGOp){
60 .opc = opc,
61 .args = args,
62 .prev = pi,
63 .next = ni
64 };
65}
66
951c6300
RH		 67void tcg_gen_op1(TCGContext *ctx, TCGOpcode opc, TCGArg a1)
68{
c45cb8bb 69 int pi = ctx->gen_next_parm_idx;
951c6300 70
c45cb8bb
RH		 71 tcg_debug_assert(pi + 1 <= OPPARAM_BUF_SIZE);
72 ctx->gen_next_parm_idx = pi + 1;
73 ctx->gen_opparam_buf[pi] = a1;
951c6300 74
c45cb8bb 75 tcg_emit_op(ctx, opc, pi);
951c6300
RH		 76}
77
78void tcg_gen_op2(TCGContext *ctx, TCGOpcode opc, TCGArg a1, TCGArg a2)
79{
c45cb8bb 80 int pi = ctx->gen_next_parm_idx;
951c6300 81
c45cb8bb
RH		 82 tcg_debug_assert(pi + 2 <= OPPARAM_BUF_SIZE);
83 ctx->gen_next_parm_idx = pi + 2;
84 ctx->gen_opparam_buf[pi + 0] = a1;
85 ctx->gen_opparam_buf[pi + 1] = a2;
951c6300 86
c45cb8bb 87 tcg_emit_op(ctx, opc, pi);
951c6300
RH		 88}
89
90void tcg_gen_op3(TCGContext *ctx, TCGOpcode opc, TCGArg a1,
91 TCGArg a2, TCGArg a3)
92{
c45cb8bb 93 int pi = ctx->gen_next_parm_idx;
951c6300 94
c45cb8bb
RH		 95 tcg_debug_assert(pi + 3 <= OPPARAM_BUF_SIZE);
96 ctx->gen_next_parm_idx = pi + 3;
97 ctx->gen_opparam_buf[pi + 0] = a1;
98 ctx->gen_opparam_buf[pi + 1] = a2;
99 ctx->gen_opparam_buf[pi + 2] = a3;
951c6300 100
c45cb8bb 101 tcg_emit_op(ctx, opc, pi);
951c6300
RH		 102}
103
104void tcg_gen_op4(TCGContext *ctx, TCGOpcode opc, TCGArg a1,
105 TCGArg a2, TCGArg a3, TCGArg a4)
106{
c45cb8bb 107 int pi = ctx->gen_next_parm_idx;
951c6300 108
c45cb8bb
RH		 109 tcg_debug_assert(pi + 4 <= OPPARAM_BUF_SIZE);
110 ctx->gen_next_parm_idx = pi + 4;
111 ctx->gen_opparam_buf[pi + 0] = a1;
112 ctx->gen_opparam_buf[pi + 1] = a2;
113 ctx->gen_opparam_buf[pi + 2] = a3;
114 ctx->gen_opparam_buf[pi + 3] = a4;
951c6300 115
c45cb8bb 116 tcg_emit_op(ctx, opc, pi);
951c6300
RH		 117}
118
119void tcg_gen_op5(TCGContext *ctx, TCGOpcode opc, TCGArg a1,
120 TCGArg a2, TCGArg a3, TCGArg a4, TCGArg a5)
121{
c45cb8bb 122 int pi = ctx->gen_next_parm_idx;
951c6300 123
c45cb8bb
RH		 124 tcg_debug_assert(pi + 5 <= OPPARAM_BUF_SIZE);
125 ctx->gen_next_parm_idx = pi + 5;
126 ctx->gen_opparam_buf[pi + 0] = a1;
127 ctx->gen_opparam_buf[pi + 1] = a2;
128 ctx->gen_opparam_buf[pi + 2] = a3;
129 ctx->gen_opparam_buf[pi + 3] = a4;
130 ctx->gen_opparam_buf[pi + 4] = a5;
951c6300 131
c45cb8bb 132 tcg_emit_op(ctx, opc, pi);
951c6300
RH		 133}
134
135void tcg_gen_op6(TCGContext *ctx, TCGOpcode opc, TCGArg a1, TCGArg a2,
136 TCGArg a3, TCGArg a4, TCGArg a5, TCGArg a6)
137{
c45cb8bb 138 int pi = ctx->gen_next_parm_idx;
951c6300 139
c45cb8bb
RH		 140 tcg_debug_assert(pi + 6 <= OPPARAM_BUF_SIZE);
141 ctx->gen_next_parm_idx = pi + 6;
142 ctx->gen_opparam_buf[pi + 0] = a1;
143 ctx->gen_opparam_buf[pi + 1] = a2;
144 ctx->gen_opparam_buf[pi + 2] = a3;
145 ctx->gen_opparam_buf[pi + 3] = a4;
146 ctx->gen_opparam_buf[pi + 4] = a5;
147 ctx->gen_opparam_buf[pi + 5] = a6;
951c6300 148
c45cb8bb 149 tcg_emit_op(ctx, opc, pi);
951c6300
RH		 150}
151
f65e19bc
PK		 152void tcg_gen_mb(TCGBar mb_type)
153{
91682118 154 if (parallel_cpus) {
f65e19bc
PK		 155 tcg_gen_op1(&tcg_ctx, INDEX_op_mb, mb_type);
156 }
157}
158
951c6300
RH		 159/* 32 bit ops */
160
161void tcg_gen_addi_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
162{
163 /* some cases can be optimized here */
164 if (arg2 == 0) {
165 tcg_gen_mov_i32(ret, arg1);
166 } else {
167 TCGv_i32 t0 = tcg_const_i32(arg2);
168 tcg_gen_add_i32(ret, arg1, t0);
169 tcg_temp_free_i32(t0);
170 }
171}
172
173void tcg_gen_subfi_i32(TCGv_i32 ret, int32_t arg1, TCGv_i32 arg2)
174{
175 if (arg1 == 0 && TCG_TARGET_HAS_neg_i32) {
176 /* Don't recurse with tcg_gen_neg_i32. */
177 tcg_gen_op2_i32(INDEX_op_neg_i32, ret, arg2);
178 } else {
179 TCGv_i32 t0 = tcg_const_i32(arg1);
180 tcg_gen_sub_i32(ret, t0, arg2);
181 tcg_temp_free_i32(t0);
182 }
183}
184
185void tcg_gen_subi_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
186{
187 /* some cases can be optimized here */
188 if (arg2 == 0) {
189 tcg_gen_mov_i32(ret, arg1);
190 } else {
191 TCGv_i32 t0 = tcg_const_i32(arg2);
192 tcg_gen_sub_i32(ret, arg1, t0);
193 tcg_temp_free_i32(t0);
194 }
195}
196
197void tcg_gen_andi_i32(TCGv_i32 ret, TCGv_i32 arg1, uint32_t arg2)
198{
199 TCGv_i32 t0;
200 /* Some cases can be optimized here. */
201 switch (arg2) {
202 case 0:
203 tcg_gen_movi_i32(ret, 0);
204 return;
205 case 0xffffffffu:
206 tcg_gen_mov_i32(ret, arg1);
207 return;
208 case 0xffu:
209 /* Don't recurse with tcg_gen_ext8u_i32. */
210 if (TCG_TARGET_HAS_ext8u_i32) {
211 tcg_gen_op2_i32(INDEX_op_ext8u_i32, ret, arg1);
212 return;
213 }
214 break;
215 case 0xffffu:
216 if (TCG_TARGET_HAS_ext16u_i32) {
217 tcg_gen_op2_i32(INDEX_op_ext16u_i32, ret, arg1);
218 return;
219 }
220 break;
221 }
222 t0 = tcg_const_i32(arg2);
223 tcg_gen_and_i32(ret, arg1, t0);
224 tcg_temp_free_i32(t0);
225}
226
227void tcg_gen_ori_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
228{
229 /* Some cases can be optimized here. */
230 if (arg2 == -1) {
231 tcg_gen_movi_i32(ret, -1);
232 } else if (arg2 == 0) {
233 tcg_gen_mov_i32(ret, arg1);
234 } else {
235 TCGv_i32 t0 = tcg_const_i32(arg2);
236 tcg_gen_or_i32(ret, arg1, t0);
237 tcg_temp_free_i32(t0);
238 }
239}
240
241void tcg_gen_xori_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
242{
243 /* Some cases can be optimized here. */
244 if (arg2 == 0) {
245 tcg_gen_mov_i32(ret, arg1);
246 } else if (arg2 == -1 && TCG_TARGET_HAS_not_i32) {
247 /* Don't recurse with tcg_gen_not_i32. */
248 tcg_gen_op2_i32(INDEX_op_not_i32, ret, arg1);
249 } else {
250 TCGv_i32 t0 = tcg_const_i32(arg2);
251 tcg_gen_xor_i32(ret, arg1, t0);
252 tcg_temp_free_i32(t0);
253 }
254}
255
256void tcg_gen_shli_i32(TCGv_i32 ret, TCGv_i32 arg1, unsigned arg2)
257{
258 tcg_debug_assert(arg2 < 32);
259 if (arg2 == 0) {
260 tcg_gen_mov_i32(ret, arg1);
261 } else {
262 TCGv_i32 t0 = tcg_const_i32(arg2);
263 tcg_gen_shl_i32(ret, arg1, t0);
264 tcg_temp_free_i32(t0);
265 }
266}
267
268void tcg_gen_shri_i32(TCGv_i32 ret, TCGv_i32 arg1, unsigned arg2)
269{
270 tcg_debug_assert(arg2 < 32);
271 if (arg2 == 0) {
272 tcg_gen_mov_i32(ret, arg1);
273 } else {
274 TCGv_i32 t0 = tcg_const_i32(arg2);
275 tcg_gen_shr_i32(ret, arg1, t0);
276 tcg_temp_free_i32(t0);
277 }
278}
279
280void tcg_gen_sari_i32(TCGv_i32 ret, TCGv_i32 arg1, unsigned arg2)
281{
282 tcg_debug_assert(arg2 < 32);
283 if (arg2 == 0) {
284 tcg_gen_mov_i32(ret, arg1);
285 } else {
286 TCGv_i32 t0 = tcg_const_i32(arg2);
287 tcg_gen_sar_i32(ret, arg1, t0);
288 tcg_temp_free_i32(t0);
289 }
290}
291
42a268c2 292void tcg_gen_brcond_i32(TCGCond cond, TCGv_i32 arg1, TCGv_i32 arg2, TCGLabel *l)
951c6300
RH		 293{
294 if (cond == TCG_COND_ALWAYS) {
42a268c2 295 tcg_gen_br(l);
951c6300 296 } else if (cond != TCG_COND_NEVER) {
42a268c2 297 tcg_gen_op4ii_i32(INDEX_op_brcond_i32, arg1, arg2, cond, label_arg(l));
951c6300
RH		 298 }
299}
300
42a268c2 301void tcg_gen_brcondi_i32(TCGCond cond, TCGv_i32 arg1, int32_t arg2, TCGLabel *l)
951c6300 302{
37ed3bf1
RH		 303 if (cond == TCG_COND_ALWAYS) {
304 tcg_gen_br(l);
305 } else if (cond != TCG_COND_NEVER) {
306 TCGv_i32 t0 = tcg_const_i32(arg2);
307 tcg_gen_brcond_i32(cond, arg1, t0, l);
308 tcg_temp_free_i32(t0);
309 }
951c6300
RH		 310}
311
312void tcg_gen_setcond_i32(TCGCond cond, TCGv_i32 ret,
313 TCGv_i32 arg1, TCGv_i32 arg2)
314{
315 if (cond == TCG_COND_ALWAYS) {
316 tcg_gen_movi_i32(ret, 1);
317 } else if (cond == TCG_COND_NEVER) {
318 tcg_gen_movi_i32(ret, 0);
319 } else {
320 tcg_gen_op4i_i32(INDEX_op_setcond_i32, ret, arg1, arg2, cond);
321 }
322}
323
324void tcg_gen_setcondi_i32(TCGCond cond, TCGv_i32 ret,
325 TCGv_i32 arg1, int32_t arg2)
326{
327 TCGv_i32 t0 = tcg_const_i32(arg2);
328 tcg_gen_setcond_i32(cond, ret, arg1, t0);
329 tcg_temp_free_i32(t0);
330}
331
332void tcg_gen_muli_i32(TCGv_i32 ret, TCGv_i32 arg1, int32_t arg2)
333{
334 TCGv_i32 t0 = tcg_const_i32(arg2);
335 tcg_gen_mul_i32(ret, arg1, t0);
336 tcg_temp_free_i32(t0);
337}
338
339void tcg_gen_div_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
340{
341 if (TCG_TARGET_HAS_div_i32) {
342 tcg_gen_op3_i32(INDEX_op_div_i32, ret, arg1, arg2);
343 } else if (TCG_TARGET_HAS_div2_i32) {
344 TCGv_i32 t0 = tcg_temp_new_i32();
345 tcg_gen_sari_i32(t0, arg1, 31);
346 tcg_gen_op5_i32(INDEX_op_div2_i32, ret, t0, arg1, t0, arg2);
347 tcg_temp_free_i32(t0);
348 } else {
349 gen_helper_div_i32(ret, arg1, arg2);
350 }
351}
352
353void tcg_gen_rem_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
354{
355 if (TCG_TARGET_HAS_rem_i32) {
356 tcg_gen_op3_i32(INDEX_op_rem_i32, ret, arg1, arg2);
357 } else if (TCG_TARGET_HAS_div_i32) {
358 TCGv_i32 t0 = tcg_temp_new_i32();
359 tcg_gen_op3_i32(INDEX_op_div_i32, t0, arg1, arg2);
360 tcg_gen_mul_i32(t0, t0, arg2);
361 tcg_gen_sub_i32(ret, arg1, t0);
362 tcg_temp_free_i32(t0);
363 } else if (TCG_TARGET_HAS_div2_i32) {
364 TCGv_i32 t0 = tcg_temp_new_i32();
365 tcg_gen_sari_i32(t0, arg1, 31);
366 tcg_gen_op5_i32(INDEX_op_div2_i32, t0, ret, arg1, t0, arg2);
367 tcg_temp_free_i32(t0);
368 } else {
369 gen_helper_rem_i32(ret, arg1, arg2);
370 }
371}
372
373void tcg_gen_divu_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
374{
375 if (TCG_TARGET_HAS_div_i32) {
376 tcg_gen_op3_i32(INDEX_op_divu_i32, ret, arg1, arg2);
377 } else if (TCG_TARGET_HAS_div2_i32) {
378 TCGv_i32 t0 = tcg_temp_new_i32();
379 tcg_gen_movi_i32(t0, 0);
380 tcg_gen_op5_i32(INDEX_op_divu2_i32, ret, t0, arg1, t0, arg2);
381 tcg_temp_free_i32(t0);
382 } else {
383 gen_helper_divu_i32(ret, arg1, arg2);
384 }
385}
386
387void tcg_gen_remu_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
388{
389 if (TCG_TARGET_HAS_rem_i32) {
390 tcg_gen_op3_i32(INDEX_op_remu_i32, ret, arg1, arg2);
391 } else if (TCG_TARGET_HAS_div_i32) {
392 TCGv_i32 t0 = tcg_temp_new_i32();
393 tcg_gen_op3_i32(INDEX_op_divu_i32, t0, arg1, arg2);
394 tcg_gen_mul_i32(t0, t0, arg2);
395 tcg_gen_sub_i32(ret, arg1, t0);
396 tcg_temp_free_i32(t0);
397 } else if (TCG_TARGET_HAS_div2_i32) {
398 TCGv_i32 t0 = tcg_temp_new_i32();
399 tcg_gen_movi_i32(t0, 0);
400 tcg_gen_op5_i32(INDEX_op_divu2_i32, t0, ret, arg1, t0, arg2);
401 tcg_temp_free_i32(t0);
402 } else {
403 gen_helper_remu_i32(ret, arg1, arg2);
404 }
405}
406
407void tcg_gen_andc_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
408{
409 if (TCG_TARGET_HAS_andc_i32) {
410 tcg_gen_op3_i32(INDEX_op_andc_i32, ret, arg1, arg2);
411 } else {
412 TCGv_i32 t0 = tcg_temp_new_i32();
413 tcg_gen_not_i32(t0, arg2);
414 tcg_gen_and_i32(ret, arg1, t0);
415 tcg_temp_free_i32(t0);
416 }
417}
418
419void tcg_gen_eqv_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
420{
421 if (TCG_TARGET_HAS_eqv_i32) {
422 tcg_gen_op3_i32(INDEX_op_eqv_i32, ret, arg1, arg2);
423 } else {
424 tcg_gen_xor_i32(ret, arg1, arg2);
425 tcg_gen_not_i32(ret, ret);
426 }
427}
428
429void tcg_gen_nand_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
430{
431 if (TCG_TARGET_HAS_nand_i32) {
432 tcg_gen_op3_i32(INDEX_op_nand_i32, ret, arg1, arg2);
433 } else {
434 tcg_gen_and_i32(ret, arg1, arg2);
435 tcg_gen_not_i32(ret, ret);
436 }
437}
438
439void tcg_gen_nor_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
440{
441 if (TCG_TARGET_HAS_nor_i32) {
442 tcg_gen_op3_i32(INDEX_op_nor_i32, ret, arg1, arg2);
443 } else {
444 tcg_gen_or_i32(ret, arg1, arg2);
445 tcg_gen_not_i32(ret, ret);
446 }
447}
448
449void tcg_gen_orc_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
450{
451 if (TCG_TARGET_HAS_orc_i32) {
452 tcg_gen_op3_i32(INDEX_op_orc_i32, ret, arg1, arg2);
453 } else {
454 TCGv_i32 t0 = tcg_temp_new_i32();
455 tcg_gen_not_i32(t0, arg2);
456 tcg_gen_or_i32(ret, arg1, t0);
457 tcg_temp_free_i32(t0);
458 }
459}
460
0e28d006
RH		 461void tcg_gen_clz_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
462{
463 if (TCG_TARGET_HAS_clz_i32) {
464 tcg_gen_op3_i32(INDEX_op_clz_i32, ret, arg1, arg2);
465 } else if (TCG_TARGET_HAS_clz_i64) {
466 TCGv_i64 t1 = tcg_temp_new_i64();
467 TCGv_i64 t2 = tcg_temp_new_i64();
468 tcg_gen_extu_i32_i64(t1, arg1);
469 tcg_gen_extu_i32_i64(t2, arg2);
470 tcg_gen_addi_i64(t2, t2, 32);
471 tcg_gen_clz_i64(t1, t1, t2);
472 tcg_gen_extrl_i64_i32(ret, t1);
473 tcg_temp_free_i64(t1);
474 tcg_temp_free_i64(t2);
475 tcg_gen_subi_i32(ret, ret, 32);
476 } else {
477 gen_helper_clz_i32(ret, arg1, arg2);
478 }
479}
480
481void tcg_gen_clzi_i32(TCGv_i32 ret, TCGv_i32 arg1, uint32_t arg2)
482{
483 TCGv_i32 t = tcg_const_i32(arg2);
484 tcg_gen_clz_i32(ret, arg1, t);
485 tcg_temp_free_i32(t);
486}
487
488void tcg_gen_ctz_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
489{
490 if (TCG_TARGET_HAS_ctz_i32) {
491 tcg_gen_op3_i32(INDEX_op_ctz_i32, ret, arg1, arg2);
492 } else if (TCG_TARGET_HAS_ctz_i64) {
493 TCGv_i64 t1 = tcg_temp_new_i64();
494 TCGv_i64 t2 = tcg_temp_new_i64();
495 tcg_gen_extu_i32_i64(t1, arg1);
496 tcg_gen_extu_i32_i64(t2, arg2);
497 tcg_gen_ctz_i64(t1, t1, t2);
498 tcg_gen_extrl_i64_i32(ret, t1);
499 tcg_temp_free_i64(t1);
500 tcg_temp_free_i64(t2);
14e99210
RH		 501 } else if (TCG_TARGET_HAS_ctpop_i32
502 || TCG_TARGET_HAS_ctpop_i64
503 || TCG_TARGET_HAS_clz_i32
504 || TCG_TARGET_HAS_clz_i64) {
505 TCGv_i32 z, t = tcg_temp_new_i32();
506
507 if (TCG_TARGET_HAS_ctpop_i32 || TCG_TARGET_HAS_ctpop_i64) {
508 tcg_gen_subi_i32(t, arg1, 1);
509 tcg_gen_andc_i32(t, t, arg1);
510 tcg_gen_ctpop_i32(t, t);
511 } else {
512 /* Since all non-x86 hosts have clz(0) == 32, don't fight it. */
513 tcg_gen_neg_i32(t, arg1);
514 tcg_gen_and_i32(t, t, arg1);
515 tcg_gen_clzi_i32(t, t, 32);
516 tcg_gen_xori_i32(t, t, 31);
517 }
518 z = tcg_const_i32(0);
519 tcg_gen_movcond_i32(TCG_COND_EQ, ret, arg1, z, arg2, t);
520 tcg_temp_free_i32(t);
521 tcg_temp_free_i32(z);
0e28d006
RH		 522 } else {
523 gen_helper_ctz_i32(ret, arg1, arg2);
524 }
525}
526
527void tcg_gen_ctzi_i32(TCGv_i32 ret, TCGv_i32 arg1, uint32_t arg2)
528{
14e99210
RH		 529 if (!TCG_TARGET_HAS_ctz_i32 && TCG_TARGET_HAS_ctpop_i32 && arg2 == 32) {
530 /* This equivalence has the advantage of not requiring a fixup. */
531 TCGv_i32 t = tcg_temp_new_i32();
532 tcg_gen_subi_i32(t, arg1, 1);
533 tcg_gen_andc_i32(t, t, arg1);
534 tcg_gen_ctpop_i32(ret, t);
535 tcg_temp_free_i32(t);
536 } else {
537 TCGv_i32 t = tcg_const_i32(arg2);
538 tcg_gen_ctz_i32(ret, arg1, t);
539 tcg_temp_free_i32(t);
540 }
0e28d006
RH		 541}
542
086920c2
RH		 543void tcg_gen_clrsb_i32(TCGv_i32 ret, TCGv_i32 arg)
544{
545 if (TCG_TARGET_HAS_clz_i32) {
546 TCGv_i32 t = tcg_temp_new_i32();
547 tcg_gen_sari_i32(t, arg, 31);
548 tcg_gen_xor_i32(t, t, arg);
549 tcg_gen_clzi_i32(t, t, 32);
550 tcg_gen_subi_i32(ret, t, 1);
551 tcg_temp_free_i32(t);
552 } else {
553 gen_helper_clrsb_i32(ret, arg);
554 }
555}
556
a768e4e9
RH		 557void tcg_gen_ctpop_i32(TCGv_i32 ret, TCGv_i32 arg1)
558{
559 if (TCG_TARGET_HAS_ctpop_i32) {
560 tcg_gen_op2_i32(INDEX_op_ctpop_i32, ret, arg1);
561 } else if (TCG_TARGET_HAS_ctpop_i64) {
562 TCGv_i64 t = tcg_temp_new_i64();
563 tcg_gen_extu_i32_i64(t, arg1);
564 tcg_gen_ctpop_i64(t, t);
565 tcg_gen_extrl_i64_i32(ret, t);
566 tcg_temp_free_i64(t);
567 } else {
568 gen_helper_ctpop_i32(ret, arg1);
569 }
570}
571
951c6300
RH		 572void tcg_gen_rotl_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
573{
574 if (TCG_TARGET_HAS_rot_i32) {
575 tcg_gen_op3_i32(INDEX_op_rotl_i32, ret, arg1, arg2);
576 } else {
577 TCGv_i32 t0, t1;
578
579 t0 = tcg_temp_new_i32();
580 t1 = tcg_temp_new_i32();
581 tcg_gen_shl_i32(t0, arg1, arg2);
582 tcg_gen_subfi_i32(t1, 32, arg2);
583 tcg_gen_shr_i32(t1, arg1, t1);
584 tcg_gen_or_i32(ret, t0, t1);
585 tcg_temp_free_i32(t0);
586 tcg_temp_free_i32(t1);
587 }
588}
589
590void tcg_gen_rotli_i32(TCGv_i32 ret, TCGv_i32 arg1, unsigned arg2)
591{
592 tcg_debug_assert(arg2 < 32);
593 /* some cases can be optimized here */
594 if (arg2 == 0) {
595 tcg_gen_mov_i32(ret, arg1);
596 } else if (TCG_TARGET_HAS_rot_i32) {
597 TCGv_i32 t0 = tcg_const_i32(arg2);
598 tcg_gen_rotl_i32(ret, arg1, t0);
599 tcg_temp_free_i32(t0);
600 } else {
601 TCGv_i32 t0, t1;
602 t0 = tcg_temp_new_i32();
603 t1 = tcg_temp_new_i32();
604 tcg_gen_shli_i32(t0, arg1, arg2);
605 tcg_gen_shri_i32(t1, arg1, 32 - arg2);
606 tcg_gen_or_i32(ret, t0, t1);
607 tcg_temp_free_i32(t0);
608 tcg_temp_free_i32(t1);
609 }
610}
611
612void tcg_gen_rotr_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2)
613{
614 if (TCG_TARGET_HAS_rot_i32) {
615 tcg_gen_op3_i32(INDEX_op_rotr_i32, ret, arg1, arg2);
616 } else {
617 TCGv_i32 t0, t1;
618
619 t0 = tcg_temp_new_i32();
620 t1 = tcg_temp_new_i32();
621 tcg_gen_shr_i32(t0, arg1, arg2);
622 tcg_gen_subfi_i32(t1, 32, arg2);
623 tcg_gen_shl_i32(t1, arg1, t1);
624 tcg_gen_or_i32(ret, t0, t1);
625 tcg_temp_free_i32(t0);
626 tcg_temp_free_i32(t1);
627 }
628}
629
630void tcg_gen_rotri_i32(TCGv_i32 ret, TCGv_i32 arg1, unsigned arg2)
631{
632 tcg_debug_assert(arg2 < 32);
633 /* some cases can be optimized here */
634 if (arg2 == 0) {
635 tcg_gen_mov_i32(ret, arg1);
636 } else {
637 tcg_gen_rotli_i32(ret, arg1, 32 - arg2);
638 }
639}
640
641void tcg_gen_deposit_i32(TCGv_i32 ret, TCGv_i32 arg1, TCGv_i32 arg2,
642 unsigned int ofs, unsigned int len)
643{
644 uint32_t mask;
645 TCGv_i32 t1;
646
647 tcg_debug_assert(ofs < 32);
0d0d309d 648 tcg_debug_assert(len > 0);
951c6300
RH		 649 tcg_debug_assert(len <= 32);
650 tcg_debug_assert(ofs + len <= 32);
651
0d0d309d 652 if (len == 32) {
951c6300
RH		 653 tcg_gen_mov_i32(ret, arg2);
654 return;
655 }
656 if (TCG_TARGET_HAS_deposit_i32 && TCG_TARGET_deposit_i32_valid(ofs, len)) {
657 tcg_gen_op5ii_i32(INDEX_op_deposit_i32, ret, arg1, arg2, ofs, len);
658 return;
659 }
660
661 mask = (1u << len) - 1;
662 t1 = tcg_temp_new_i32();
663
664 if (ofs + len < 32) {
665 tcg_gen_andi_i32(t1, arg2, mask);
666 tcg_gen_shli_i32(t1, t1, ofs);
667 } else {
668 tcg_gen_shli_i32(t1, arg2, ofs);
669 }
670 tcg_gen_andi_i32(ret, arg1, ~(mask << ofs));
671 tcg_gen_or_i32(ret, ret, t1);
672
673 tcg_temp_free_i32(t1);
674}
675
07cc68d5
RH		 676void tcg_gen_deposit_z_i32(TCGv_i32 ret, TCGv_i32 arg,
677 unsigned int ofs, unsigned int len)
678{
679 tcg_debug_assert(ofs < 32);
680 tcg_debug_assert(len > 0);
681 tcg_debug_assert(len <= 32);
682 tcg_debug_assert(ofs + len <= 32);
683
684 if (ofs + len == 32) {
685 tcg_gen_shli_i32(ret, arg, ofs);
686 } else if (ofs == 0) {
687 tcg_gen_andi_i32(ret, arg, (1u << len) - 1);
688 } else if (TCG_TARGET_HAS_deposit_i32
689 && TCG_TARGET_deposit_i32_valid(ofs, len)) {
690 TCGv_i32 zero = tcg_const_i32(0);
691 tcg_gen_op5ii_i32(INDEX_op_deposit_i32, ret, zero, arg, ofs, len);
692 tcg_temp_free_i32(zero);
693 } else {
694 /* To help two-operand hosts we prefer to zero-extend first,
695 which allows ARG to stay live. */
696 switch (len) {
697 case 16:
698 if (TCG_TARGET_HAS_ext16u_i32) {
699 tcg_gen_ext16u_i32(ret, arg);
700 tcg_gen_shli_i32(ret, ret, ofs);
701 return;
702 }
703 break;
704 case 8:
705 if (TCG_TARGET_HAS_ext8u_i32) {
706 tcg_gen_ext8u_i32(ret, arg);
707 tcg_gen_shli_i32(ret, ret, ofs);
708 return;
709 }
710 break;
711 }
712 /* Otherwise prefer zero-extension over AND for code size. */
713 switch (ofs + len) {
714 case 16:
715 if (TCG_TARGET_HAS_ext16u_i32) {
716 tcg_gen_shli_i32(ret, arg, ofs);
717 tcg_gen_ext16u_i32(ret, ret);
718 return;
719 }
720 break;
721 case 8:
722 if (TCG_TARGET_HAS_ext8u_i32) {
723 tcg_gen_shli_i32(ret, arg, ofs);
724 tcg_gen_ext8u_i32(ret, ret);
725 return;
726 }
727 break;
728 }
729 tcg_gen_andi_i32(ret, arg, (1u << len) - 1);
730 tcg_gen_shli_i32(ret, ret, ofs);
731 }
732}
733
7ec8bab3
RH		 734void tcg_gen_extract_i32(TCGv_i32 ret, TCGv_i32 arg,
735 unsigned int ofs, unsigned int len)
736{
737 tcg_debug_assert(ofs < 32);
738 tcg_debug_assert(len > 0);
739 tcg_debug_assert(len <= 32);
740 tcg_debug_assert(ofs + len <= 32);
741
742 /* Canonicalize certain special cases, even if extract is supported. */
743 if (ofs + len == 32) {
744 tcg_gen_shri_i32(ret, arg, 32 - len);
745 return;
746 }
747 if (ofs == 0) {
748 tcg_gen_andi_i32(ret, arg, (1u << len) - 1);
749 return;
750 }
751
752 if (TCG_TARGET_HAS_extract_i32
753 && TCG_TARGET_extract_i32_valid(ofs, len)) {
754 tcg_gen_op4ii_i32(INDEX_op_extract_i32, ret, arg, ofs, len);
755 return;
756 }
757
758 /* Assume that zero-extension, if available, is cheaper than a shift. */
759 switch (ofs + len) {
760 case 16:
761 if (TCG_TARGET_HAS_ext16u_i32) {
762 tcg_gen_ext16u_i32(ret, arg);
763 tcg_gen_shri_i32(ret, ret, ofs);
764 return;
765 }
766 break;
767 case 8:
768 if (TCG_TARGET_HAS_ext8u_i32) {
769 tcg_gen_ext8u_i32(ret, arg);
770 tcg_gen_shri_i32(ret, ret, ofs);
771 return;
772 }
773 break;
774 }
775
776 /* ??? Ideally we'd know what values are available for immediate AND.
777 Assume that 8 bits are available, plus the special case of 16,
778 so that we get ext8u, ext16u. */
779 switch (len) {
780 case 1 ... 8: case 16:
781 tcg_gen_shri_i32(ret, arg, ofs);
782 tcg_gen_andi_i32(ret, ret, (1u << len) - 1);
783 break;
784 default:
785 tcg_gen_shli_i32(ret, arg, 32 - len - ofs);
786 tcg_gen_shri_i32(ret, ret, 32 - len);
787 break;
788 }
789}
790
791void tcg_gen_sextract_i32(TCGv_i32 ret, TCGv_i32 arg,
792 unsigned int ofs, unsigned int len)
793{
794 tcg_debug_assert(ofs < 32);
795 tcg_debug_assert(len > 0);
796 tcg_debug_assert(len <= 32);
797 tcg_debug_assert(ofs + len <= 32);
798
799 /* Canonicalize certain special cases, even if extract is supported. */
800 if (ofs + len == 32) {
801 tcg_gen_sari_i32(ret, arg, 32 - len);
802 return;
803 }
804 if (ofs == 0) {
805 switch (len) {
806 case 16:
807 tcg_gen_ext16s_i32(ret, arg);
808 return;
809 case 8:
810 tcg_gen_ext8s_i32(ret, arg);
811 return;
812 }
813 }
814
815 if (TCG_TARGET_HAS_sextract_i32
816 && TCG_TARGET_extract_i32_valid(ofs, len)) {
817 tcg_gen_op4ii_i32(INDEX_op_sextract_i32, ret, arg, ofs, len);
818 return;
819 }
820
821 /* Assume that sign-extension, if available, is cheaper than a shift. */
822 switch (ofs + len) {
823 case 16:
824 if (TCG_TARGET_HAS_ext16s_i32) {
825 tcg_gen_ext16s_i32(ret, arg);
826 tcg_gen_sari_i32(ret, ret, ofs);
827 return;
828 }
829 break;
830 case 8:
831 if (TCG_TARGET_HAS_ext8s_i32) {
832 tcg_gen_ext8s_i32(ret, arg);
833 tcg_gen_sari_i32(ret, ret, ofs);
834 return;
835 }
836 break;
837 }
838 switch (len) {
839 case 16:
840 if (TCG_TARGET_HAS_ext16s_i32) {
841 tcg_gen_shri_i32(ret, arg, ofs);
842 tcg_gen_ext16s_i32(ret, ret);
843 return;
844 }
845 break;
846 case 8:
847 if (TCG_TARGET_HAS_ext8s_i32) {
848 tcg_gen_shri_i32(ret, arg, ofs);
849 tcg_gen_ext8s_i32(ret, ret);
850 return;
851 }
852 break;
853 }
854
855 tcg_gen_shli_i32(ret, arg, 32 - len - ofs);
856 tcg_gen_sari_i32(ret, ret, 32 - len);
857}
858
951c6300
RH		 859void tcg_gen_movcond_i32(TCGCond cond, TCGv_i32 ret, TCGv_i32 c1,
860 TCGv_i32 c2, TCGv_i32 v1, TCGv_i32 v2)
861{
37ed3bf1
RH		 862 if (cond == TCG_COND_ALWAYS) {
863 tcg_gen_mov_i32(ret, v1);
864 } else if (cond == TCG_COND_NEVER) {
865 tcg_gen_mov_i32(ret, v2);
866 } else if (TCG_TARGET_HAS_movcond_i32) {
951c6300
RH		 867 tcg_gen_op6i_i32(INDEX_op_movcond_i32, ret, c1, c2, v1, v2, cond);
868 } else {
869 TCGv_i32 t0 = tcg_temp_new_i32();
870 TCGv_i32 t1 = tcg_temp_new_i32();
871 tcg_gen_setcond_i32(cond, t0, c1, c2);
872 tcg_gen_neg_i32(t0, t0);
873 tcg_gen_and_i32(t1, v1, t0);
874 tcg_gen_andc_i32(ret, v2, t0);
875 tcg_gen_or_i32(ret, ret, t1);
876 tcg_temp_free_i32(t0);
877 tcg_temp_free_i32(t1);
878 }
879}
880
881void tcg_gen_add2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 al,
882 TCGv_i32 ah, TCGv_i32 bl, TCGv_i32 bh)
883{
884 if (TCG_TARGET_HAS_add2_i32) {
885 tcg_gen_op6_i32(INDEX_op_add2_i32, rl, rh, al, ah, bl, bh);
951c6300
RH		 886 } else {
887 TCGv_i64 t0 = tcg_temp_new_i64();
888 TCGv_i64 t1 = tcg_temp_new_i64();
889 tcg_gen_concat_i32_i64(t0, al, ah);
890 tcg_gen_concat_i32_i64(t1, bl, bh);
891 tcg_gen_add_i64(t0, t0, t1);
892 tcg_gen_extr_i64_i32(rl, rh, t0);
893 tcg_temp_free_i64(t0);
894 tcg_temp_free_i64(t1);
895 }
896}
897
898void tcg_gen_sub2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 al,
899 TCGv_i32 ah, TCGv_i32 bl, TCGv_i32 bh)
900{
901 if (TCG_TARGET_HAS_sub2_i32) {
902 tcg_gen_op6_i32(INDEX_op_sub2_i32, rl, rh, al, ah, bl, bh);
951c6300
RH		 903 } else {
904 TCGv_i64 t0 = tcg_temp_new_i64();
905 TCGv_i64 t1 = tcg_temp_new_i64();
906 tcg_gen_concat_i32_i64(t0, al, ah);
907 tcg_gen_concat_i32_i64(t1, bl, bh);
908 tcg_gen_sub_i64(t0, t0, t1);
909 tcg_gen_extr_i64_i32(rl, rh, t0);
910 tcg_temp_free_i64(t0);
911 tcg_temp_free_i64(t1);
912 }
913}
914
915void tcg_gen_mulu2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 arg1, TCGv_i32 arg2)
916{
917 if (TCG_TARGET_HAS_mulu2_i32) {
918 tcg_gen_op4_i32(INDEX_op_mulu2_i32, rl, rh, arg1, arg2);
951c6300
RH		 919 } else if (TCG_TARGET_HAS_muluh_i32) {
920 TCGv_i32 t = tcg_temp_new_i32();
921 tcg_gen_op3_i32(INDEX_op_mul_i32, t, arg1, arg2);
922 tcg_gen_op3_i32(INDEX_op_muluh_i32, rh, arg1, arg2);
923 tcg_gen_mov_i32(rl, t);
924 tcg_temp_free_i32(t);
925 } else {
926 TCGv_i64 t0 = tcg_temp_new_i64();
927 TCGv_i64 t1 = tcg_temp_new_i64();
928 tcg_gen_extu_i32_i64(t0, arg1);
929 tcg_gen_extu_i32_i64(t1, arg2);
930 tcg_gen_mul_i64(t0, t0, t1);
931 tcg_gen_extr_i64_i32(rl, rh, t0);
932 tcg_temp_free_i64(t0);
933 tcg_temp_free_i64(t1);
934 }
935}
936
937void tcg_gen_muls2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 arg1, TCGv_i32 arg2)
938{
939 if (TCG_TARGET_HAS_muls2_i32) {
940 tcg_gen_op4_i32(INDEX_op_muls2_i32, rl, rh, arg1, arg2);
951c6300
RH		 941 } else if (TCG_TARGET_HAS_mulsh_i32) {
942 TCGv_i32 t = tcg_temp_new_i32();
943 tcg_gen_op3_i32(INDEX_op_mul_i32, t, arg1, arg2);
944 tcg_gen_op3_i32(INDEX_op_mulsh_i32, rh, arg1, arg2);
945 tcg_gen_mov_i32(rl, t);
946 tcg_temp_free_i32(t);
947 } else if (TCG_TARGET_REG_BITS == 32) {
948 TCGv_i32 t0 = tcg_temp_new_i32();
949 TCGv_i32 t1 = tcg_temp_new_i32();
950 TCGv_i32 t2 = tcg_temp_new_i32();
951 TCGv_i32 t3 = tcg_temp_new_i32();
952 tcg_gen_mulu2_i32(t0, t1, arg1, arg2);
953 /* Adjust for negative inputs. */
954 tcg_gen_sari_i32(t2, arg1, 31);
955 tcg_gen_sari_i32(t3, arg2, 31);
956 tcg_gen_and_i32(t2, t2, arg2);
957 tcg_gen_and_i32(t3, t3, arg1);
958 tcg_gen_sub_i32(rh, t1, t2);
959 tcg_gen_sub_i32(rh, rh, t3);
960 tcg_gen_mov_i32(rl, t0);
961 tcg_temp_free_i32(t0);
962 tcg_temp_free_i32(t1);
963 tcg_temp_free_i32(t2);
964 tcg_temp_free_i32(t3);
965 } else {
966 TCGv_i64 t0 = tcg_temp_new_i64();
967 TCGv_i64 t1 = tcg_temp_new_i64();
968 tcg_gen_ext_i32_i64(t0, arg1);
969 tcg_gen_ext_i32_i64(t1, arg2);
970 tcg_gen_mul_i64(t0, t0, t1);
971 tcg_gen_extr_i64_i32(rl, rh, t0);
972 tcg_temp_free_i64(t0);
973 tcg_temp_free_i64(t1);
974 }
975}
976
5087abfb
RH		 977void tcg_gen_mulsu2_i32(TCGv_i32 rl, TCGv_i32 rh, TCGv_i32 arg1, TCGv_i32 arg2)
978{
979 if (TCG_TARGET_REG_BITS == 32) {
980 TCGv_i32 t0 = tcg_temp_new_i32();
981 TCGv_i32 t1 = tcg_temp_new_i32();
982 TCGv_i32 t2 = tcg_temp_new_i32();
983 tcg_gen_mulu2_i32(t0, t1, arg1, arg2);
984 /* Adjust for negative input for the signed arg1. */
985 tcg_gen_sari_i32(t2, arg1, 31);
986 tcg_gen_and_i32(t2, t2, arg2);
987 tcg_gen_sub_i32(rh, t1, t2);
988 tcg_gen_mov_i32(rl, t0);
989 tcg_temp_free_i32(t0);
990 tcg_temp_free_i32(t1);
991 tcg_temp_free_i32(t2);
992 } else {
993 TCGv_i64 t0 = tcg_temp_new_i64();
994 TCGv_i64 t1 = tcg_temp_new_i64();
995 tcg_gen_ext_i32_i64(t0, arg1);
996 tcg_gen_extu_i32_i64(t1, arg2);
997 tcg_gen_mul_i64(t0, t0, t1);
998 tcg_gen_extr_i64_i32(rl, rh, t0);
999 tcg_temp_free_i64(t0);
1000 tcg_temp_free_i64(t1);
1001 }
1002}
1003
951c6300
RH		 1004void tcg_gen_ext8s_i32(TCGv_i32 ret, TCGv_i32 arg)
1005{
1006 if (TCG_TARGET_HAS_ext8s_i32) {
1007 tcg_gen_op2_i32(INDEX_op_ext8s_i32, ret, arg);
1008 } else {
1009 tcg_gen_shli_i32(ret, arg, 24);
1010 tcg_gen_sari_i32(ret, ret, 24);
1011 }
1012}
1013
1014void tcg_gen_ext16s_i32(TCGv_i32 ret, TCGv_i32 arg)
1015{
1016 if (TCG_TARGET_HAS_ext16s_i32) {
1017 tcg_gen_op2_i32(INDEX_op_ext16s_i32, ret, arg);
1018 } else {
1019 tcg_gen_shli_i32(ret, arg, 16);
1020 tcg_gen_sari_i32(ret, ret, 16);
1021 }
1022}
1023
1024void tcg_gen_ext8u_i32(TCGv_i32 ret, TCGv_i32 arg)
1025{
1026 if (TCG_TARGET_HAS_ext8u_i32) {
1027 tcg_gen_op2_i32(INDEX_op_ext8u_i32, ret, arg);
1028 } else {
1029 tcg_gen_andi_i32(ret, arg, 0xffu);
1030 }
1031}
1032
1033void tcg_gen_ext16u_i32(TCGv_i32 ret, TCGv_i32 arg)
1034{
1035 if (TCG_TARGET_HAS_ext16u_i32) {
1036 tcg_gen_op2_i32(INDEX_op_ext16u_i32, ret, arg);
1037 } else {
1038 tcg_gen_andi_i32(ret, arg, 0xffffu);
1039 }
1040}
1041
1042/* Note: we assume the two high bytes are set to zero */
1043void tcg_gen_bswap16_i32(TCGv_i32 ret, TCGv_i32 arg)
1044{
1045 if (TCG_TARGET_HAS_bswap16_i32) {
1046 tcg_gen_op2_i32(INDEX_op_bswap16_i32, ret, arg);
1047 } else {
1048 TCGv_i32 t0 = tcg_temp_new_i32();
1049
1050 tcg_gen_ext8u_i32(t0, arg);
1051 tcg_gen_shli_i32(t0, t0, 8);
1052 tcg_gen_shri_i32(ret, arg, 8);
1053 tcg_gen_or_i32(ret, ret, t0);
1054 tcg_temp_free_i32(t0);
1055 }
1056}
1057
1058void tcg_gen_bswap32_i32(TCGv_i32 ret, TCGv_i32 arg)
1059{
1060 if (TCG_TARGET_HAS_bswap32_i32) {
1061 tcg_gen_op2_i32(INDEX_op_bswap32_i32, ret, arg);
1062 } else {
1063 TCGv_i32 t0, t1;
1064 t0 = tcg_temp_new_i32();
1065 t1 = tcg_temp_new_i32();
1066
1067 tcg_gen_shli_i32(t0, arg, 24);
1068
1069 tcg_gen_andi_i32(t1, arg, 0x0000ff00);
1070 tcg_gen_shli_i32(t1, t1, 8);
1071 tcg_gen_or_i32(t0, t0, t1);
1072
1073 tcg_gen_shri_i32(t1, arg, 8);
1074 tcg_gen_andi_i32(t1, t1, 0x0000ff00);
1075 tcg_gen_or_i32(t0, t0, t1);
1076
1077 tcg_gen_shri_i32(t1, arg, 24);
1078 tcg_gen_or_i32(ret, t0, t1);
1079 tcg_temp_free_i32(t0);
1080 tcg_temp_free_i32(t1);
1081 }
1082}
1083
1084/* 64-bit ops */
1085
1086#if TCG_TARGET_REG_BITS == 32
1087/* These are all inline for TCG_TARGET_REG_BITS == 64. */
1088
1089void tcg_gen_discard_i64(TCGv_i64 arg)
1090{
1091 tcg_gen_discard_i32(TCGV_LOW(arg));
1092 tcg_gen_discard_i32(TCGV_HIGH(arg));
1093}
1094
1095void tcg_gen_mov_i64(TCGv_i64 ret, TCGv_i64 arg)
1096{
1097 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1098 tcg_gen_mov_i32(TCGV_HIGH(ret), TCGV_HIGH(arg));
1099}
1100
1101void tcg_gen_movi_i64(TCGv_i64 ret, int64_t arg)
1102{
1103 tcg_gen_movi_i32(TCGV_LOW(ret), arg);
1104 tcg_gen_movi_i32(TCGV_HIGH(ret), arg >> 32);
1105}
1106
1107void tcg_gen_ld8u_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
1108{
1109 tcg_gen_ld8u_i32(TCGV_LOW(ret), arg2, offset);
1110 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1111}
1112
1113void tcg_gen_ld8s_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
1114{
1115 tcg_gen_ld8s_i32(TCGV_LOW(ret), arg2, offset);
3ff91d7e 1116 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
951c6300
RH		 1117}
1118
1119void tcg_gen_ld16u_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
1120{
1121 tcg_gen_ld16u_i32(TCGV_LOW(ret), arg2, offset);
1122 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1123}
1124
1125void tcg_gen_ld16s_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
1126{
1127 tcg_gen_ld16s_i32(TCGV_LOW(ret), arg2, offset);
1128 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
1129}
1130
1131void tcg_gen_ld32u_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
1132{
1133 tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset);
1134 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1135}
1136
1137void tcg_gen_ld32s_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
1138{
1139 tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset);
1140 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
1141}
1142
1143void tcg_gen_ld_i64(TCGv_i64 ret, TCGv_ptr arg2, tcg_target_long offset)
1144{
1145 /* Since arg2 and ret have different types,
1146 they cannot be the same temporary */
cf811fff 1147#ifdef HOST_WORDS_BIGENDIAN
951c6300
RH		 1148 tcg_gen_ld_i32(TCGV_HIGH(ret), arg2, offset);
1149 tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset + 4);
1150#else
1151 tcg_gen_ld_i32(TCGV_LOW(ret), arg2, offset);
1152 tcg_gen_ld_i32(TCGV_HIGH(ret), arg2, offset + 4);
1153#endif
1154}
1155
1156void tcg_gen_st_i64(TCGv_i64 arg1, TCGv_ptr arg2, tcg_target_long offset)
1157{
cf811fff 1158#ifdef HOST_WORDS_BIGENDIAN
951c6300
RH		 1159 tcg_gen_st_i32(TCGV_HIGH(arg1), arg2, offset);
1160 tcg_gen_st_i32(TCGV_LOW(arg1), arg2, offset + 4);
1161#else
1162 tcg_gen_st_i32(TCGV_LOW(arg1), arg2, offset);
1163 tcg_gen_st_i32(TCGV_HIGH(arg1), arg2, offset + 4);
1164#endif
1165}
1166
1167void tcg_gen_and_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1168{
1169 tcg_gen_and_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
1170 tcg_gen_and_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
1171}
1172
1173void tcg_gen_or_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1174{
1175 tcg_gen_or_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
1176 tcg_gen_or_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
1177}
1178
1179void tcg_gen_xor_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1180{
1181 tcg_gen_xor_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
1182 tcg_gen_xor_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
1183}
1184
1185void tcg_gen_shl_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1186{
1187 gen_helper_shl_i64(ret, arg1, arg2);
1188}
1189
1190void tcg_gen_shr_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1191{
1192 gen_helper_shr_i64(ret, arg1, arg2);
1193}
1194
1195void tcg_gen_sar_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1196{
1197 gen_helper_sar_i64(ret, arg1, arg2);
1198}
1199
1200void tcg_gen_mul_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1201{
1202 TCGv_i64 t0;
1203 TCGv_i32 t1;
1204
1205 t0 = tcg_temp_new_i64();
1206 t1 = tcg_temp_new_i32();
1207
1208 tcg_gen_mulu2_i32(TCGV_LOW(t0), TCGV_HIGH(t0),
1209 TCGV_LOW(arg1), TCGV_LOW(arg2));
1210
1211 tcg_gen_mul_i32(t1, TCGV_LOW(arg1), TCGV_HIGH(arg2));
1212 tcg_gen_add_i32(TCGV_HIGH(t0), TCGV_HIGH(t0), t1);
1213 tcg_gen_mul_i32(t1, TCGV_HIGH(arg1), TCGV_LOW(arg2));
1214 tcg_gen_add_i32(TCGV_HIGH(t0), TCGV_HIGH(t0), t1);
1215
1216 tcg_gen_mov_i64(ret, t0);
1217 tcg_temp_free_i64(t0);
1218 tcg_temp_free_i32(t1);
1219}
1220#endif /* TCG_TARGET_REG_SIZE == 32 */
1221
1222void tcg_gen_addi_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
1223{
1224 /* some cases can be optimized here */
1225 if (arg2 == 0) {
1226 tcg_gen_mov_i64(ret, arg1);
1227 } else {
1228 TCGv_i64 t0 = tcg_const_i64(arg2);
1229 tcg_gen_add_i64(ret, arg1, t0);
1230 tcg_temp_free_i64(t0);
1231 }
1232}
1233
1234void tcg_gen_subfi_i64(TCGv_i64 ret, int64_t arg1, TCGv_i64 arg2)
1235{
1236 if (arg1 == 0 && TCG_TARGET_HAS_neg_i64) {
1237 /* Don't recurse with tcg_gen_neg_i64. */
1238 tcg_gen_op2_i64(INDEX_op_neg_i64, ret, arg2);
1239 } else {
1240 TCGv_i64 t0 = tcg_const_i64(arg1);
1241 tcg_gen_sub_i64(ret, t0, arg2);
1242 tcg_temp_free_i64(t0);
1243 }
1244}
1245
1246void tcg_gen_subi_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
1247{
1248 /* some cases can be optimized here */
1249 if (arg2 == 0) {
1250 tcg_gen_mov_i64(ret, arg1);
1251 } else {
1252 TCGv_i64 t0 = tcg_const_i64(arg2);
1253 tcg_gen_sub_i64(ret, arg1, t0);
1254 tcg_temp_free_i64(t0);
1255 }
1256}
1257
1258void tcg_gen_andi_i64(TCGv_i64 ret, TCGv_i64 arg1, uint64_t arg2)
1259{
951c6300 1260 TCGv_i64 t0;
3a13c3f3
RH		 1261
1262 if (TCG_TARGET_REG_BITS == 32) {
1263 tcg_gen_andi_i32(TCGV_LOW(ret), TCGV_LOW(arg1), arg2);
1264 tcg_gen_andi_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), arg2 >> 32);
1265 return;
1266 }
1267
951c6300
RH		 1268 /* Some cases can be optimized here. */
1269 switch (arg2) {
1270 case 0:
1271 tcg_gen_movi_i64(ret, 0);
1272 return;
1273 case 0xffffffffffffffffull:
1274 tcg_gen_mov_i64(ret, arg1);
1275 return;
1276 case 0xffull:
1277 /* Don't recurse with tcg_gen_ext8u_i64. */
1278 if (TCG_TARGET_HAS_ext8u_i64) {
1279 tcg_gen_op2_i64(INDEX_op_ext8u_i64, ret, arg1);
1280 return;
1281 }
1282 break;
1283 case 0xffffu:
1284 if (TCG_TARGET_HAS_ext16u_i64) {
1285 tcg_gen_op2_i64(INDEX_op_ext16u_i64, ret, arg1);
1286 return;
1287 }
1288 break;
1289 case 0xffffffffull:
1290 if (TCG_TARGET_HAS_ext32u_i64) {
1291 tcg_gen_op2_i64(INDEX_op_ext32u_i64, ret, arg1);
1292 return;
1293 }
1294 break;
1295 }
1296 t0 = tcg_const_i64(arg2);
1297 tcg_gen_and_i64(ret, arg1, t0);
1298 tcg_temp_free_i64(t0);
951c6300
RH		 1299}
1300
1301void tcg_gen_ori_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
1302{
3a13c3f3
RH		 1303 if (TCG_TARGET_REG_BITS == 32) {
1304 tcg_gen_ori_i32(TCGV_LOW(ret), TCGV_LOW(arg1), arg2);
1305 tcg_gen_ori_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), arg2 >> 32);
1306 return;
1307 }
951c6300
RH		 1308 /* Some cases can be optimized here. */
1309 if (arg2 == -1) {
1310 tcg_gen_movi_i64(ret, -1);
1311 } else if (arg2 == 0) {
1312 tcg_gen_mov_i64(ret, arg1);
1313 } else {
1314 TCGv_i64 t0 = tcg_const_i64(arg2);
1315 tcg_gen_or_i64(ret, arg1, t0);
1316 tcg_temp_free_i64(t0);
1317 }
951c6300
RH		 1318}
1319
1320void tcg_gen_xori_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
1321{
3a13c3f3
RH		 1322 if (TCG_TARGET_REG_BITS == 32) {
1323 tcg_gen_xori_i32(TCGV_LOW(ret), TCGV_LOW(arg1), arg2);
1324 tcg_gen_xori_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), arg2 >> 32);
1325 return;
1326 }
951c6300
RH		 1327 /* Some cases can be optimized here. */
1328 if (arg2 == 0) {
1329 tcg_gen_mov_i64(ret, arg1);
1330 } else if (arg2 == -1 && TCG_TARGET_HAS_not_i64) {
1331 /* Don't recurse with tcg_gen_not_i64. */
1332 tcg_gen_op2_i64(INDEX_op_not_i64, ret, arg1);
1333 } else {
1334 TCGv_i64 t0 = tcg_const_i64(arg2);
1335 tcg_gen_xor_i64(ret, arg1, t0);
1336 tcg_temp_free_i64(t0);
1337 }
951c6300
RH		 1338}
1339
951c6300
RH		 1340static inline void tcg_gen_shifti_i64(TCGv_i64 ret, TCGv_i64 arg1,
1341 unsigned c, bool right, bool arith)
1342{
1343 tcg_debug_assert(c < 64);
1344 if (c == 0) {
1345 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg1));
1346 tcg_gen_mov_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1));
1347 } else if (c >= 32) {
1348 c -= 32;
1349 if (right) {
1350 if (arith) {
1351 tcg_gen_sari_i32(TCGV_LOW(ret), TCGV_HIGH(arg1), c);
1352 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), 31);
1353 } else {
1354 tcg_gen_shri_i32(TCGV_LOW(ret), TCGV_HIGH(arg1), c);
1355 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1356 }
1357 } else {
1358 tcg_gen_shli_i32(TCGV_HIGH(ret), TCGV_LOW(arg1), c);
1359 tcg_gen_movi_i32(TCGV_LOW(ret), 0);
1360 }
1361 } else {
1362 TCGv_i32 t0, t1;
1363
1364 t0 = tcg_temp_new_i32();
1365 t1 = tcg_temp_new_i32();
1366 if (right) {
1367 tcg_gen_shli_i32(t0, TCGV_HIGH(arg1), 32 - c);
1368 if (arith) {
1369 tcg_gen_sari_i32(t1, TCGV_HIGH(arg1), c);
1370 } else {
1371 tcg_gen_shri_i32(t1, TCGV_HIGH(arg1), c);
1372 }
1373 tcg_gen_shri_i32(TCGV_LOW(ret), TCGV_LOW(arg1), c);
1374 tcg_gen_or_i32(TCGV_LOW(ret), TCGV_LOW(ret), t0);
1375 tcg_gen_mov_i32(TCGV_HIGH(ret), t1);
1376 } else {
1377 tcg_gen_shri_i32(t0, TCGV_LOW(arg1), 32 - c);
1378 /* Note: ret can be the same as arg1, so we use t1 */
1379 tcg_gen_shli_i32(t1, TCGV_LOW(arg1), c);
1380 tcg_gen_shli_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), c);
1381 tcg_gen_or_i32(TCGV_HIGH(ret), TCGV_HIGH(ret), t0);
1382 tcg_gen_mov_i32(TCGV_LOW(ret), t1);
1383 }
1384 tcg_temp_free_i32(t0);
1385 tcg_temp_free_i32(t1);
1386 }
1387}
1388
951c6300
RH		 1389void tcg_gen_shli_i64(TCGv_i64 ret, TCGv_i64 arg1, unsigned arg2)
1390{
1391 tcg_debug_assert(arg2 < 64);
3a13c3f3
RH		 1392 if (TCG_TARGET_REG_BITS == 32) {
1393 tcg_gen_shifti_i64(ret, arg1, arg2, 0, 0);
1394 } else if (arg2 == 0) {
951c6300
RH		 1395 tcg_gen_mov_i64(ret, arg1);
1396 } else {
1397 TCGv_i64 t0 = tcg_const_i64(arg2);
1398 tcg_gen_shl_i64(ret, arg1, t0);
1399 tcg_temp_free_i64(t0);
1400 }
1401}
1402
1403void tcg_gen_shri_i64(TCGv_i64 ret, TCGv_i64 arg1, unsigned arg2)
1404{
1405 tcg_debug_assert(arg2 < 64);
3a13c3f3
RH		 1406 if (TCG_TARGET_REG_BITS == 32) {
1407 tcg_gen_shifti_i64(ret, arg1, arg2, 1, 0);
1408 } else if (arg2 == 0) {
951c6300
RH		 1409 tcg_gen_mov_i64(ret, arg1);
1410 } else {
1411 TCGv_i64 t0 = tcg_const_i64(arg2);
1412 tcg_gen_shr_i64(ret, arg1, t0);
1413 tcg_temp_free_i64(t0);
1414 }
1415}
1416
1417void tcg_gen_sari_i64(TCGv_i64 ret, TCGv_i64 arg1, unsigned arg2)
1418{
1419 tcg_debug_assert(arg2 < 64);
3a13c3f3
RH		 1420 if (TCG_TARGET_REG_BITS == 32) {
1421 tcg_gen_shifti_i64(ret, arg1, arg2, 1, 1);
1422 } else if (arg2 == 0) {
951c6300
RH		 1423 tcg_gen_mov_i64(ret, arg1);
1424 } else {
1425 TCGv_i64 t0 = tcg_const_i64(arg2);
1426 tcg_gen_sar_i64(ret, arg1, t0);
1427 tcg_temp_free_i64(t0);
1428 }
1429}
951c6300 1430
42a268c2 1431void tcg_gen_brcond_i64(TCGCond cond, TCGv_i64 arg1, TCGv_i64 arg2, TCGLabel *l)
951c6300
RH		 1432{
1433 if (cond == TCG_COND_ALWAYS) {
42a268c2 1434 tcg_gen_br(l);
951c6300 1435 } else if (cond != TCG_COND_NEVER) {
3a13c3f3
RH		 1436 if (TCG_TARGET_REG_BITS == 32) {
1437 tcg_gen_op6ii_i32(INDEX_op_brcond2_i32, TCGV_LOW(arg1),
1438 TCGV_HIGH(arg1), TCGV_LOW(arg2),
42a268c2 1439 TCGV_HIGH(arg2), cond, label_arg(l));
3a13c3f3 1440 } else {
42a268c2
RH		 1441 tcg_gen_op4ii_i64(INDEX_op_brcond_i64, arg1, arg2, cond,
1442 label_arg(l));
3a13c3f3 1443 }
951c6300
RH		 1444 }
1445}
1446
42a268c2 1447void tcg_gen_brcondi_i64(TCGCond cond, TCGv_i64 arg1, int64_t arg2, TCGLabel *l)
951c6300
RH		 1448{
1449 if (cond == TCG_COND_ALWAYS) {
42a268c2 1450 tcg_gen_br(l);
951c6300
RH		 1451 } else if (cond != TCG_COND_NEVER) {
1452 TCGv_i64 t0 = tcg_const_i64(arg2);
42a268c2 1453 tcg_gen_brcond_i64(cond, arg1, t0, l);
951c6300
RH		 1454 tcg_temp_free_i64(t0);
1455 }
1456}
1457
1458void tcg_gen_setcond_i64(TCGCond cond, TCGv_i64 ret,
1459 TCGv_i64 arg1, TCGv_i64 arg2)
1460{
1461 if (cond == TCG_COND_ALWAYS) {
1462 tcg_gen_movi_i64(ret, 1);
1463 } else if (cond == TCG_COND_NEVER) {
1464 tcg_gen_movi_i64(ret, 0);
1465 } else {
3a13c3f3
RH		 1466 if (TCG_TARGET_REG_BITS == 32) {
1467 tcg_gen_op6i_i32(INDEX_op_setcond2_i32, TCGV_LOW(ret),
1468 TCGV_LOW(arg1), TCGV_HIGH(arg1),
1469 TCGV_LOW(arg2), TCGV_HIGH(arg2), cond);
1470 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1471 } else {
1472 tcg_gen_op4i_i64(INDEX_op_setcond_i64, ret, arg1, arg2, cond);
1473 }
951c6300
RH		 1474 }
1475}
1476
1477void tcg_gen_setcondi_i64(TCGCond cond, TCGv_i64 ret,
1478 TCGv_i64 arg1, int64_t arg2)
1479{
1480 TCGv_i64 t0 = tcg_const_i64(arg2);
1481 tcg_gen_setcond_i64(cond, ret, arg1, t0);
1482 tcg_temp_free_i64(t0);
1483}
1484
1485void tcg_gen_muli_i64(TCGv_i64 ret, TCGv_i64 arg1, int64_t arg2)
1486{
1487 TCGv_i64 t0 = tcg_const_i64(arg2);
1488 tcg_gen_mul_i64(ret, arg1, t0);
1489 tcg_temp_free_i64(t0);
1490}
1491
1492void tcg_gen_div_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1493{
1494 if (TCG_TARGET_HAS_div_i64) {
1495 tcg_gen_op3_i64(INDEX_op_div_i64, ret, arg1, arg2);
1496 } else if (TCG_TARGET_HAS_div2_i64) {
1497 TCGv_i64 t0 = tcg_temp_new_i64();
1498 tcg_gen_sari_i64(t0, arg1, 63);
1499 tcg_gen_op5_i64(INDEX_op_div2_i64, ret, t0, arg1, t0, arg2);
1500 tcg_temp_free_i64(t0);
1501 } else {
1502 gen_helper_div_i64(ret, arg1, arg2);
1503 }
1504}
1505
1506void tcg_gen_rem_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1507{
1508 if (TCG_TARGET_HAS_rem_i64) {
1509 tcg_gen_op3_i64(INDEX_op_rem_i64, ret, arg1, arg2);
1510 } else if (TCG_TARGET_HAS_div_i64) {
1511 TCGv_i64 t0 = tcg_temp_new_i64();
1512 tcg_gen_op3_i64(INDEX_op_div_i64, t0, arg1, arg2);
1513 tcg_gen_mul_i64(t0, t0, arg2);
1514 tcg_gen_sub_i64(ret, arg1, t0);
1515 tcg_temp_free_i64(t0);
1516 } else if (TCG_TARGET_HAS_div2_i64) {
1517 TCGv_i64 t0 = tcg_temp_new_i64();
1518 tcg_gen_sari_i64(t0, arg1, 63);
1519 tcg_gen_op5_i64(INDEX_op_div2_i64, t0, ret, arg1, t0, arg2);
1520 tcg_temp_free_i64(t0);
1521 } else {
1522 gen_helper_rem_i64(ret, arg1, arg2);
1523 }
1524}
1525
1526void tcg_gen_divu_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1527{
1528 if (TCG_TARGET_HAS_div_i64) {
1529 tcg_gen_op3_i64(INDEX_op_divu_i64, ret, arg1, arg2);
1530 } else if (TCG_TARGET_HAS_div2_i64) {
1531 TCGv_i64 t0 = tcg_temp_new_i64();
1532 tcg_gen_movi_i64(t0, 0);
1533 tcg_gen_op5_i64(INDEX_op_divu2_i64, ret, t0, arg1, t0, arg2);
1534 tcg_temp_free_i64(t0);
1535 } else {
1536 gen_helper_divu_i64(ret, arg1, arg2);
1537 }
1538}
1539
1540void tcg_gen_remu_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1541{
1542 if (TCG_TARGET_HAS_rem_i64) {
1543 tcg_gen_op3_i64(INDEX_op_remu_i64, ret, arg1, arg2);
1544 } else if (TCG_TARGET_HAS_div_i64) {
1545 TCGv_i64 t0 = tcg_temp_new_i64();
1546 tcg_gen_op3_i64(INDEX_op_divu_i64, t0, arg1, arg2);
1547 tcg_gen_mul_i64(t0, t0, arg2);
1548 tcg_gen_sub_i64(ret, arg1, t0);
1549 tcg_temp_free_i64(t0);
1550 } else if (TCG_TARGET_HAS_div2_i64) {
1551 TCGv_i64 t0 = tcg_temp_new_i64();
1552 tcg_gen_movi_i64(t0, 0);
1553 tcg_gen_op5_i64(INDEX_op_divu2_i64, t0, ret, arg1, t0, arg2);
1554 tcg_temp_free_i64(t0);
1555 } else {
1556 gen_helper_remu_i64(ret, arg1, arg2);
1557 }
1558}
1559
1560void tcg_gen_ext8s_i64(TCGv_i64 ret, TCGv_i64 arg)
1561{
3a13c3f3
RH		 1562 if (TCG_TARGET_REG_BITS == 32) {
1563 tcg_gen_ext8s_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1564 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
1565 } else if (TCG_TARGET_HAS_ext8s_i64) {
951c6300
RH		 1566 tcg_gen_op2_i64(INDEX_op_ext8s_i64, ret, arg);
1567 } else {
1568 tcg_gen_shli_i64(ret, arg, 56);
1569 tcg_gen_sari_i64(ret, ret, 56);
1570 }
951c6300
RH		 1571}
1572
1573void tcg_gen_ext16s_i64(TCGv_i64 ret, TCGv_i64 arg)
1574{
3a13c3f3
RH		 1575 if (TCG_TARGET_REG_BITS == 32) {
1576 tcg_gen_ext16s_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1577 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
1578 } else if (TCG_TARGET_HAS_ext16s_i64) {
951c6300
RH		 1579 tcg_gen_op2_i64(INDEX_op_ext16s_i64, ret, arg);
1580 } else {
1581 tcg_gen_shli_i64(ret, arg, 48);
1582 tcg_gen_sari_i64(ret, ret, 48);
1583 }
951c6300
RH		 1584}
1585
1586void tcg_gen_ext32s_i64(TCGv_i64 ret, TCGv_i64 arg)
1587{
3a13c3f3
RH		 1588 if (TCG_TARGET_REG_BITS == 32) {
1589 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1590 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
1591 } else if (TCG_TARGET_HAS_ext32s_i64) {
951c6300
RH		 1592 tcg_gen_op2_i64(INDEX_op_ext32s_i64, ret, arg);
1593 } else {
1594 tcg_gen_shli_i64(ret, arg, 32);
1595 tcg_gen_sari_i64(ret, ret, 32);
1596 }
951c6300
RH		 1597}
1598
1599void tcg_gen_ext8u_i64(TCGv_i64 ret, TCGv_i64 arg)
1600{
3a13c3f3
RH		 1601 if (TCG_TARGET_REG_BITS == 32) {
1602 tcg_gen_ext8u_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1603 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1604 } else if (TCG_TARGET_HAS_ext8u_i64) {
951c6300
RH		 1605 tcg_gen_op2_i64(INDEX_op_ext8u_i64, ret, arg);
1606 } else {
1607 tcg_gen_andi_i64(ret, arg, 0xffu);
1608 }
951c6300
RH		 1609}
1610
1611void tcg_gen_ext16u_i64(TCGv_i64 ret, TCGv_i64 arg)
1612{
3a13c3f3
RH		 1613 if (TCG_TARGET_REG_BITS == 32) {
1614 tcg_gen_ext16u_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1615 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1616 } else if (TCG_TARGET_HAS_ext16u_i64) {
951c6300
RH		 1617 tcg_gen_op2_i64(INDEX_op_ext16u_i64, ret, arg);
1618 } else {
1619 tcg_gen_andi_i64(ret, arg, 0xffffu);
1620 }
951c6300
RH		 1621}
1622
1623void tcg_gen_ext32u_i64(TCGv_i64 ret, TCGv_i64 arg)
1624{
3a13c3f3
RH		 1625 if (TCG_TARGET_REG_BITS == 32) {
1626 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1627 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1628 } else if (TCG_TARGET_HAS_ext32u_i64) {
951c6300
RH		 1629 tcg_gen_op2_i64(INDEX_op_ext32u_i64, ret, arg);
1630 } else {
1631 tcg_gen_andi_i64(ret, arg, 0xffffffffu);
1632 }
951c6300
RH		 1633}
1634
1635/* Note: we assume the six high bytes are set to zero */
1636void tcg_gen_bswap16_i64(TCGv_i64 ret, TCGv_i64 arg)
1637{
3a13c3f3
RH		 1638 if (TCG_TARGET_REG_BITS == 32) {
1639 tcg_gen_bswap16_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1640 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1641 } else if (TCG_TARGET_HAS_bswap16_i64) {
951c6300
RH		 1642 tcg_gen_op2_i64(INDEX_op_bswap16_i64, ret, arg);
1643 } else {
1644 TCGv_i64 t0 = tcg_temp_new_i64();
1645
1646 tcg_gen_ext8u_i64(t0, arg);
1647 tcg_gen_shli_i64(t0, t0, 8);
1648 tcg_gen_shri_i64(ret, arg, 8);
1649 tcg_gen_or_i64(ret, ret, t0);
1650 tcg_temp_free_i64(t0);
1651 }
951c6300
RH		 1652}
1653
1654/* Note: we assume the four high bytes are set to zero */
1655void tcg_gen_bswap32_i64(TCGv_i64 ret, TCGv_i64 arg)
1656{
3a13c3f3
RH		 1657 if (TCG_TARGET_REG_BITS == 32) {
1658 tcg_gen_bswap32_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1659 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1660 } else if (TCG_TARGET_HAS_bswap32_i64) {
951c6300
RH		 1661 tcg_gen_op2_i64(INDEX_op_bswap32_i64, ret, arg);
1662 } else {
1663 TCGv_i64 t0, t1;
1664 t0 = tcg_temp_new_i64();
1665 t1 = tcg_temp_new_i64();
1666
1667 tcg_gen_shli_i64(t0, arg, 24);
1668 tcg_gen_ext32u_i64(t0, t0);
1669
1670 tcg_gen_andi_i64(t1, arg, 0x0000ff00);
1671 tcg_gen_shli_i64(t1, t1, 8);
1672 tcg_gen_or_i64(t0, t0, t1);
1673
1674 tcg_gen_shri_i64(t1, arg, 8);
1675 tcg_gen_andi_i64(t1, t1, 0x0000ff00);
1676 tcg_gen_or_i64(t0, t0, t1);
1677
1678 tcg_gen_shri_i64(t1, arg, 24);
1679 tcg_gen_or_i64(ret, t0, t1);
1680 tcg_temp_free_i64(t0);
1681 tcg_temp_free_i64(t1);
1682 }
951c6300
RH		 1683}
1684
1685void tcg_gen_bswap64_i64(TCGv_i64 ret, TCGv_i64 arg)
1686{
3a13c3f3
RH		 1687 if (TCG_TARGET_REG_BITS == 32) {
1688 TCGv_i32 t0, t1;
1689 t0 = tcg_temp_new_i32();
1690 t1 = tcg_temp_new_i32();
951c6300 1691
3a13c3f3
RH		 1692 tcg_gen_bswap32_i32(t0, TCGV_LOW(arg));
1693 tcg_gen_bswap32_i32(t1, TCGV_HIGH(arg));
1694 tcg_gen_mov_i32(TCGV_LOW(ret), t1);
1695 tcg_gen_mov_i32(TCGV_HIGH(ret), t0);
1696 tcg_temp_free_i32(t0);
1697 tcg_temp_free_i32(t1);
1698 } else if (TCG_TARGET_HAS_bswap64_i64) {
951c6300
RH		 1699 tcg_gen_op2_i64(INDEX_op_bswap64_i64, ret, arg);
1700 } else {
1701 TCGv_i64 t0 = tcg_temp_new_i64();
1702 TCGv_i64 t1 = tcg_temp_new_i64();
1703
1704 tcg_gen_shli_i64(t0, arg, 56);
1705
1706 tcg_gen_andi_i64(t1, arg, 0x0000ff00);
1707 tcg_gen_shli_i64(t1, t1, 40);
1708 tcg_gen_or_i64(t0, t0, t1);
1709
1710 tcg_gen_andi_i64(t1, arg, 0x00ff0000);
1711 tcg_gen_shli_i64(t1, t1, 24);
1712 tcg_gen_or_i64(t0, t0, t1);
1713
1714 tcg_gen_andi_i64(t1, arg, 0xff000000);
1715 tcg_gen_shli_i64(t1, t1, 8);
1716 tcg_gen_or_i64(t0, t0, t1);
1717
1718 tcg_gen_shri_i64(t1, arg, 8);
1719 tcg_gen_andi_i64(t1, t1, 0xff000000);
1720 tcg_gen_or_i64(t0, t0, t1);
1721
1722 tcg_gen_shri_i64(t1, arg, 24);
1723 tcg_gen_andi_i64(t1, t1, 0x00ff0000);
1724 tcg_gen_or_i64(t0, t0, t1);
1725
1726 tcg_gen_shri_i64(t1, arg, 40);
1727 tcg_gen_andi_i64(t1, t1, 0x0000ff00);
1728 tcg_gen_or_i64(t0, t0, t1);
1729
1730 tcg_gen_shri_i64(t1, arg, 56);
1731 tcg_gen_or_i64(ret, t0, t1);
1732 tcg_temp_free_i64(t0);
1733 tcg_temp_free_i64(t1);
1734 }
951c6300
RH		 1735}
1736
1737void tcg_gen_not_i64(TCGv_i64 ret, TCGv_i64 arg)
1738{
3a13c3f3
RH		 1739 if (TCG_TARGET_REG_BITS == 32) {
1740 tcg_gen_not_i32(TCGV_LOW(ret), TCGV_LOW(arg));
1741 tcg_gen_not_i32(TCGV_HIGH(ret), TCGV_HIGH(arg));
1742 } else if (TCG_TARGET_HAS_not_i64) {
951c6300
RH		 1743 tcg_gen_op2_i64(INDEX_op_not_i64, ret, arg);
1744 } else {
1745 tcg_gen_xori_i64(ret, arg, -1);
1746 }
951c6300
RH		 1747}
1748
1749void tcg_gen_andc_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1750{
3a13c3f3
RH		 1751 if (TCG_TARGET_REG_BITS == 32) {
1752 tcg_gen_andc_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
1753 tcg_gen_andc_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
1754 } else if (TCG_TARGET_HAS_andc_i64) {
951c6300
RH		 1755 tcg_gen_op3_i64(INDEX_op_andc_i64, ret, arg1, arg2);
1756 } else {
1757 TCGv_i64 t0 = tcg_temp_new_i64();
1758 tcg_gen_not_i64(t0, arg2);
1759 tcg_gen_and_i64(ret, arg1, t0);
1760 tcg_temp_free_i64(t0);
1761 }
951c6300
RH		 1762}
1763
1764void tcg_gen_eqv_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1765{
3a13c3f3
RH		 1766 if (TCG_TARGET_REG_BITS == 32) {
1767 tcg_gen_eqv_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
1768 tcg_gen_eqv_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
1769 } else if (TCG_TARGET_HAS_eqv_i64) {
951c6300
RH		 1770 tcg_gen_op3_i64(INDEX_op_eqv_i64, ret, arg1, arg2);
1771 } else {
1772 tcg_gen_xor_i64(ret, arg1, arg2);
1773 tcg_gen_not_i64(ret, ret);
1774 }
951c6300
RH		 1775}
1776
1777void tcg_gen_nand_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1778{
3a13c3f3
RH		 1779 if (TCG_TARGET_REG_BITS == 32) {
1780 tcg_gen_nand_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
1781 tcg_gen_nand_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
1782 } else if (TCG_TARGET_HAS_nand_i64) {
951c6300
RH		 1783 tcg_gen_op3_i64(INDEX_op_nand_i64, ret, arg1, arg2);
1784 } else {
1785 tcg_gen_and_i64(ret, arg1, arg2);
1786 tcg_gen_not_i64(ret, ret);
1787 }
951c6300
RH		 1788}
1789
1790void tcg_gen_nor_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1791{
3a13c3f3
RH		 1792 if (TCG_TARGET_REG_BITS == 32) {
1793 tcg_gen_nor_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
1794 tcg_gen_nor_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
1795 } else if (TCG_TARGET_HAS_nor_i64) {
951c6300
RH		 1796 tcg_gen_op3_i64(INDEX_op_nor_i64, ret, arg1, arg2);
1797 } else {
1798 tcg_gen_or_i64(ret, arg1, arg2);
1799 tcg_gen_not_i64(ret, ret);
1800 }
951c6300
RH		 1801}
1802
1803void tcg_gen_orc_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1804{
3a13c3f3
RH		 1805 if (TCG_TARGET_REG_BITS == 32) {
1806 tcg_gen_orc_i32(TCGV_LOW(ret), TCGV_LOW(arg1), TCGV_LOW(arg2));
1807 tcg_gen_orc_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1), TCGV_HIGH(arg2));
1808 } else if (TCG_TARGET_HAS_orc_i64) {
951c6300
RH		 1809 tcg_gen_op3_i64(INDEX_op_orc_i64, ret, arg1, arg2);
1810 } else {
1811 TCGv_i64 t0 = tcg_temp_new_i64();
1812 tcg_gen_not_i64(t0, arg2);
1813 tcg_gen_or_i64(ret, arg1, t0);
1814 tcg_temp_free_i64(t0);
1815 }
951c6300
RH		 1816}
1817
0e28d006
RH		 1818void tcg_gen_clz_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1819{
1820 if (TCG_TARGET_HAS_clz_i64) {
1821 tcg_gen_op3_i64(INDEX_op_clz_i64, ret, arg1, arg2);
1822 } else {
1823 gen_helper_clz_i64(ret, arg1, arg2);
1824 }
1825}
1826
1827void tcg_gen_clzi_i64(TCGv_i64 ret, TCGv_i64 arg1, uint64_t arg2)
1828{
1829 if (TCG_TARGET_REG_BITS == 32
1830 && TCG_TARGET_HAS_clz_i32
1831 && arg2 <= 0xffffffffu) {
1832 TCGv_i32 t = tcg_const_i32((uint32_t)arg2 - 32);
1833 tcg_gen_clz_i32(t, TCGV_LOW(arg1), t);
1834 tcg_gen_addi_i32(t, t, 32);
1835 tcg_gen_clz_i32(TCGV_LOW(ret), TCGV_HIGH(arg1), t);
1836 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1837 tcg_temp_free_i32(t);
1838 } else {
1839 TCGv_i64 t = tcg_const_i64(arg2);
1840 tcg_gen_clz_i64(ret, arg1, t);
1841 tcg_temp_free_i64(t);
1842 }
1843}
1844
1845void tcg_gen_ctz_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1846{
1847 if (TCG_TARGET_HAS_ctz_i64) {
1848 tcg_gen_op3_i64(INDEX_op_ctz_i64, ret, arg1, arg2);
14e99210
RH		 1849 } else if (TCG_TARGET_HAS_ctpop_i64 || TCG_TARGET_HAS_clz_i64) {
1850 TCGv_i64 z, t = tcg_temp_new_i64();
1851
1852 if (TCG_TARGET_HAS_ctpop_i64) {
1853 tcg_gen_subi_i64(t, arg1, 1);
1854 tcg_gen_andc_i64(t, t, arg1);
1855 tcg_gen_ctpop_i64(t, t);
1856 } else {
1857 /* Since all non-x86 hosts have clz(0) == 64, don't fight it. */
1858 tcg_gen_neg_i64(t, arg1);
1859 tcg_gen_and_i64(t, t, arg1);
1860 tcg_gen_clzi_i64(t, t, 64);
1861 tcg_gen_xori_i64(t, t, 63);
1862 }
1863 z = tcg_const_i64(0);
1864 tcg_gen_movcond_i64(TCG_COND_EQ, ret, arg1, z, arg2, t);
1865 tcg_temp_free_i64(t);
1866 tcg_temp_free_i64(z);
0e28d006
RH		 1867 } else {
1868 gen_helper_ctz_i64(ret, arg1, arg2);
1869 }
1870}
1871
1872void tcg_gen_ctzi_i64(TCGv_i64 ret, TCGv_i64 arg1, uint64_t arg2)
1873{
1874 if (TCG_TARGET_REG_BITS == 32
1875 && TCG_TARGET_HAS_ctz_i32
1876 && arg2 <= 0xffffffffu) {
1877 TCGv_i32 t32 = tcg_const_i32((uint32_t)arg2 - 32);
1878 tcg_gen_ctz_i32(t32, TCGV_HIGH(arg1), t32);
1879 tcg_gen_addi_i32(t32, t32, 32);
1880 tcg_gen_ctz_i32(TCGV_LOW(ret), TCGV_LOW(arg1), t32);
1881 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1882 tcg_temp_free_i32(t32);
14e99210
RH		 1883 } else if (!TCG_TARGET_HAS_ctz_i64
1884 && TCG_TARGET_HAS_ctpop_i64
1885 && arg2 == 64) {
1886 /* This equivalence has the advantage of not requiring a fixup. */
1887 TCGv_i64 t = tcg_temp_new_i64();
1888 tcg_gen_subi_i64(t, arg1, 1);
1889 tcg_gen_andc_i64(t, t, arg1);
1890 tcg_gen_ctpop_i64(ret, t);
1891 tcg_temp_free_i64(t);
0e28d006
RH		 1892 } else {
1893 TCGv_i64 t64 = tcg_const_i64(arg2);
1894 tcg_gen_ctz_i64(ret, arg1, t64);
1895 tcg_temp_free_i64(t64);
1896 }
1897}
1898
086920c2
RH		 1899void tcg_gen_clrsb_i64(TCGv_i64 ret, TCGv_i64 arg)
1900{
1901 if (TCG_TARGET_HAS_clz_i64 || TCG_TARGET_HAS_clz_i32) {
1902 TCGv_i64 t = tcg_temp_new_i64();
1903 tcg_gen_sari_i64(t, arg, 63);
1904 tcg_gen_xor_i64(t, t, arg);
1905 tcg_gen_clzi_i64(t, t, 64);
1906 tcg_gen_subi_i64(ret, t, 1);
1907 tcg_temp_free_i64(t);
1908 } else {
1909 gen_helper_clrsb_i64(ret, arg);
1910 }
1911}
1912
a768e4e9
RH		 1913void tcg_gen_ctpop_i64(TCGv_i64 ret, TCGv_i64 arg1)
1914{
1915 if (TCG_TARGET_HAS_ctpop_i64) {
1916 tcg_gen_op2_i64(INDEX_op_ctpop_i64, ret, arg1);
1917 } else if (TCG_TARGET_REG_BITS == 32 && TCG_TARGET_HAS_ctpop_i32) {
1918 tcg_gen_ctpop_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1));
1919 tcg_gen_ctpop_i32(TCGV_LOW(ret), TCGV_LOW(arg1));
1920 tcg_gen_add_i32(TCGV_LOW(ret), TCGV_LOW(ret), TCGV_HIGH(ret));
1921 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
1922 } else {
1923 gen_helper_ctpop_i64(ret, arg1);
1924 }
1925}
1926
951c6300
RH		 1927void tcg_gen_rotl_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1928{
1929 if (TCG_TARGET_HAS_rot_i64) {
1930 tcg_gen_op3_i64(INDEX_op_rotl_i64, ret, arg1, arg2);
1931 } else {
1932 TCGv_i64 t0, t1;
1933 t0 = tcg_temp_new_i64();
1934 t1 = tcg_temp_new_i64();
1935 tcg_gen_shl_i64(t0, arg1, arg2);
1936 tcg_gen_subfi_i64(t1, 64, arg2);
1937 tcg_gen_shr_i64(t1, arg1, t1);
1938 tcg_gen_or_i64(ret, t0, t1);
1939 tcg_temp_free_i64(t0);
1940 tcg_temp_free_i64(t1);
1941 }
1942}
1943
1944void tcg_gen_rotli_i64(TCGv_i64 ret, TCGv_i64 arg1, unsigned arg2)
1945{
1946 tcg_debug_assert(arg2 < 64);
1947 /* some cases can be optimized here */
1948 if (arg2 == 0) {
1949 tcg_gen_mov_i64(ret, arg1);
1950 } else if (TCG_TARGET_HAS_rot_i64) {
1951 TCGv_i64 t0 = tcg_const_i64(arg2);
1952 tcg_gen_rotl_i64(ret, arg1, t0);
1953 tcg_temp_free_i64(t0);
1954 } else {
1955 TCGv_i64 t0, t1;
1956 t0 = tcg_temp_new_i64();
1957 t1 = tcg_temp_new_i64();
1958 tcg_gen_shli_i64(t0, arg1, arg2);
1959 tcg_gen_shri_i64(t1, arg1, 64 - arg2);
1960 tcg_gen_or_i64(ret, t0, t1);
1961 tcg_temp_free_i64(t0);
1962 tcg_temp_free_i64(t1);
1963 }
1964}
1965
1966void tcg_gen_rotr_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2)
1967{
1968 if (TCG_TARGET_HAS_rot_i64) {
1969 tcg_gen_op3_i64(INDEX_op_rotr_i64, ret, arg1, arg2);
1970 } else {
1971 TCGv_i64 t0, t1;
1972 t0 = tcg_temp_new_i64();
1973 t1 = tcg_temp_new_i64();
1974 tcg_gen_shr_i64(t0, arg1, arg2);
1975 tcg_gen_subfi_i64(t1, 64, arg2);
1976 tcg_gen_shl_i64(t1, arg1, t1);
1977 tcg_gen_or_i64(ret, t0, t1);
1978 tcg_temp_free_i64(t0);
1979 tcg_temp_free_i64(t1);
1980 }
1981}
1982
1983void tcg_gen_rotri_i64(TCGv_i64 ret, TCGv_i64 arg1, unsigned arg2)
1984{
1985 tcg_debug_assert(arg2 < 64);
1986 /* some cases can be optimized here */
1987 if (arg2 == 0) {
1988 tcg_gen_mov_i64(ret, arg1);
1989 } else {
1990 tcg_gen_rotli_i64(ret, arg1, 64 - arg2);
1991 }
1992}
1993
1994void tcg_gen_deposit_i64(TCGv_i64 ret, TCGv_i64 arg1, TCGv_i64 arg2,
1995 unsigned int ofs, unsigned int len)
1996{
1997 uint64_t mask;
1998 TCGv_i64 t1;
1999
2000 tcg_debug_assert(ofs < 64);
0d0d309d 2001 tcg_debug_assert(len > 0);
951c6300
RH		 2002 tcg_debug_assert(len <= 64);
2003 tcg_debug_assert(ofs + len <= 64);
2004
0d0d309d 2005 if (len == 64) {
951c6300
RH		 2006 tcg_gen_mov_i64(ret, arg2);
2007 return;
2008 }
2009 if (TCG_TARGET_HAS_deposit_i64 && TCG_TARGET_deposit_i64_valid(ofs, len)) {
2010 tcg_gen_op5ii_i64(INDEX_op_deposit_i64, ret, arg1, arg2, ofs, len);
2011 return;
2012 }
2013
3a13c3f3
RH		 2014 if (TCG_TARGET_REG_BITS == 32) {
2015 if (ofs >= 32) {
2016 tcg_gen_deposit_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1),
2017 TCGV_LOW(arg2), ofs - 32, len);
2018 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg1));
2019 return;
2020 }
2021 if (ofs + len <= 32) {
2022 tcg_gen_deposit_i32(TCGV_LOW(ret), TCGV_LOW(arg1),
2023 TCGV_LOW(arg2), ofs, len);
2024 tcg_gen_mov_i32(TCGV_HIGH(ret), TCGV_HIGH(arg1));
2025 return;
2026 }
951c6300 2027 }
951c6300
RH		 2028
2029 mask = (1ull << len) - 1;
2030 t1 = tcg_temp_new_i64();
2031
2032 if (ofs + len < 64) {
2033 tcg_gen_andi_i64(t1, arg2, mask);
2034 tcg_gen_shli_i64(t1, t1, ofs);
2035 } else {
2036 tcg_gen_shli_i64(t1, arg2, ofs);
2037 }
2038 tcg_gen_andi_i64(ret, arg1, ~(mask << ofs));
2039 tcg_gen_or_i64(ret, ret, t1);
2040
2041 tcg_temp_free_i64(t1);
2042}
2043
07cc68d5
RH		 2044void tcg_gen_deposit_z_i64(TCGv_i64 ret, TCGv_i64 arg,
2045 unsigned int ofs, unsigned int len)
2046{
2047 tcg_debug_assert(ofs < 64);
2048 tcg_debug_assert(len > 0);
2049 tcg_debug_assert(len <= 64);
2050 tcg_debug_assert(ofs + len <= 64);
2051
2052 if (ofs + len == 64) {
2053 tcg_gen_shli_i64(ret, arg, ofs);
2054 } else if (ofs == 0) {
2055 tcg_gen_andi_i64(ret, arg, (1ull << len) - 1);
2056 } else if (TCG_TARGET_HAS_deposit_i64
2057 && TCG_TARGET_deposit_i64_valid(ofs, len)) {
2058 TCGv_i64 zero = tcg_const_i64(0);
2059 tcg_gen_op5ii_i64(INDEX_op_deposit_i64, ret, zero, arg, ofs, len);
2060 tcg_temp_free_i64(zero);
2061 } else {
2062 if (TCG_TARGET_REG_BITS == 32) {
2063 if (ofs >= 32) {
2064 tcg_gen_deposit_z_i32(TCGV_HIGH(ret), TCGV_LOW(arg),
2065 ofs - 32, len);
2066 tcg_gen_movi_i32(TCGV_LOW(ret), 0);
2067 return;
2068 }
2069 if (ofs + len <= 32) {
2070 tcg_gen_deposit_z_i32(TCGV_LOW(ret), TCGV_LOW(arg), ofs, len);
2071 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
2072 return;
2073 }
2074 }
2075 /* To help two-operand hosts we prefer to zero-extend first,
2076 which allows ARG to stay live. */
2077 switch (len) {
2078 case 32:
2079 if (TCG_TARGET_HAS_ext32u_i64) {
2080 tcg_gen_ext32u_i64(ret, arg);
2081 tcg_gen_shli_i64(ret, ret, ofs);
2082 return;
2083 }
2084 break;
2085 case 16:
2086 if (TCG_TARGET_HAS_ext16u_i64) {
2087 tcg_gen_ext16u_i64(ret, arg);
2088 tcg_gen_shli_i64(ret, ret, ofs);
2089 return;
2090 }
2091 break;
2092 case 8:
2093 if (TCG_TARGET_HAS_ext8u_i64) {
2094 tcg_gen_ext8u_i64(ret, arg);
2095 tcg_gen_shli_i64(ret, ret, ofs);
2096 return;
2097 }
2098 break;
2099 }
2100 /* Otherwise prefer zero-extension over AND for code size. */
2101 switch (ofs + len) {
2102 case 32:
2103 if (TCG_TARGET_HAS_ext32u_i64) {
2104 tcg_gen_shli_i64(ret, arg, ofs);
2105 tcg_gen_ext32u_i64(ret, ret);
2106 return;
2107 }
2108 break;
2109 case 16:
2110 if (TCG_TARGET_HAS_ext16u_i64) {
2111 tcg_gen_shli_i64(ret, arg, ofs);
2112 tcg_gen_ext16u_i64(ret, ret);
2113 return;
2114 }
2115 break;
2116 case 8:
2117 if (TCG_TARGET_HAS_ext8u_i64) {
2118 tcg_gen_shli_i64(ret, arg, ofs);
2119 tcg_gen_ext8u_i64(ret, ret);
2120 return;
2121 }
2122 break;
2123 }
2124 tcg_gen_andi_i64(ret, arg, (1ull << len) - 1);
2125 tcg_gen_shli_i64(ret, ret, ofs);
2126 }
2127}
2128
7ec8bab3
RH		 2129void tcg_gen_extract_i64(TCGv_i64 ret, TCGv_i64 arg,
2130 unsigned int ofs, unsigned int len)
2131{
2132 tcg_debug_assert(ofs < 64);
2133 tcg_debug_assert(len > 0);
2134 tcg_debug_assert(len <= 64);
2135 tcg_debug_assert(ofs + len <= 64);
2136
2137 /* Canonicalize certain special cases, even if extract is supported. */
2138 if (ofs + len == 64) {
2139 tcg_gen_shri_i64(ret, arg, 64 - len);
2140 return;
2141 }
2142 if (ofs == 0) {
2143 tcg_gen_andi_i64(ret, arg, (1ull << len) - 1);
2144 return;
2145 }
2146
2147 if (TCG_TARGET_REG_BITS == 32) {
2148 /* Look for a 32-bit extract within one of the two words. */
2149 if (ofs >= 32) {
2150 tcg_gen_extract_i32(TCGV_LOW(ret), TCGV_HIGH(arg), ofs - 32, len);
2151 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
2152 return;
2153 }
2154 if (ofs + len <= 32) {
2155 tcg_gen_extract_i32(TCGV_LOW(ret), TCGV_LOW(arg), ofs, len);
2156 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
2157 return;
2158 }
2159 /* The field is split across two words. One double-word
2160 shift is better than two double-word shifts. */
2161 goto do_shift_and;
2162 }
2163
2164 if (TCG_TARGET_HAS_extract_i64
2165 && TCG_TARGET_extract_i64_valid(ofs, len)) {
2166 tcg_gen_op4ii_i64(INDEX_op_extract_i64, ret, arg, ofs, len);
2167 return;
2168 }
2169
2170 /* Assume that zero-extension, if available, is cheaper than a shift. */
2171 switch (ofs + len) {
2172 case 32:
2173 if (TCG_TARGET_HAS_ext32u_i64) {
2174 tcg_gen_ext32u_i64(ret, arg);
2175 tcg_gen_shri_i64(ret, ret, ofs);
2176 return;
2177 }
2178 break;
2179 case 16:
2180 if (TCG_TARGET_HAS_ext16u_i64) {
2181 tcg_gen_ext16u_i64(ret, arg);
2182 tcg_gen_shri_i64(ret, ret, ofs);
2183 return;
2184 }
2185 break;
2186 case 8:
2187 if (TCG_TARGET_HAS_ext8u_i64) {
2188 tcg_gen_ext8u_i64(ret, arg);
2189 tcg_gen_shri_i64(ret, ret, ofs);
2190 return;
2191 }
2192 break;
2193 }
2194
2195 /* ??? Ideally we'd know what values are available for immediate AND.
2196 Assume that 8 bits are available, plus the special cases of 16 and 32,
2197 so that we get ext8u, ext16u, and ext32u. */
2198 switch (len) {
2199 case 1 ... 8: case 16: case 32:
2200 do_shift_and:
2201 tcg_gen_shri_i64(ret, arg, ofs);
2202 tcg_gen_andi_i64(ret, ret, (1ull << len) - 1);
2203 break;
2204 default:
2205 tcg_gen_shli_i64(ret, arg, 64 - len - ofs);
2206 tcg_gen_shri_i64(ret, ret, 64 - len);
2207 break;
2208 }
2209}
2210
2211void tcg_gen_sextract_i64(TCGv_i64 ret, TCGv_i64 arg,
2212 unsigned int ofs, unsigned int len)
2213{
2214 tcg_debug_assert(ofs < 64);
2215 tcg_debug_assert(len > 0);
2216 tcg_debug_assert(len <= 64);
2217 tcg_debug_assert(ofs + len <= 64);
2218
2219 /* Canonicalize certain special cases, even if sextract is supported. */
2220 if (ofs + len == 64) {
2221 tcg_gen_sari_i64(ret, arg, 64 - len);
2222 return;
2223 }
2224 if (ofs == 0) {
2225 switch (len) {
2226 case 32:
2227 tcg_gen_ext32s_i64(ret, arg);
2228 return;
2229 case 16:
2230 tcg_gen_ext16s_i64(ret, arg);
2231 return;
2232 case 8:
2233 tcg_gen_ext8s_i64(ret, arg);
2234 return;
2235 }
2236 }
2237
2238 if (TCG_TARGET_REG_BITS == 32) {
2239 /* Look for a 32-bit extract within one of the two words. */
2240 if (ofs >= 32) {
2241 tcg_gen_sextract_i32(TCGV_LOW(ret), TCGV_HIGH(arg), ofs - 32, len);
2242 } else if (ofs + len <= 32) {
2243 tcg_gen_sextract_i32(TCGV_LOW(ret), TCGV_LOW(arg), ofs, len);
2244 } else if (ofs == 0) {
2245 tcg_gen_mov_i32(TCGV_LOW(ret), TCGV_LOW(arg));
2246 tcg_gen_sextract_i32(TCGV_HIGH(ret), TCGV_HIGH(arg), 0, len - 32);
2247 return;
2248 } else if (len > 32) {
2249 TCGv_i32 t = tcg_temp_new_i32();
2250 /* Extract the bits for the high word normally. */
2251 tcg_gen_sextract_i32(t, TCGV_HIGH(arg), ofs + 32, len - 32);
2252 /* Shift the field down for the low part. */
2253 tcg_gen_shri_i64(ret, arg, ofs);
2254 /* Overwrite the shift into the high part. */
2255 tcg_gen_mov_i32(TCGV_HIGH(ret), t);
2256 tcg_temp_free_i32(t);
2257 return;
2258 } else {
2259 /* Shift the field down for the low part, such that the
2260 field sits at the MSB. */
2261 tcg_gen_shri_i64(ret, arg, ofs + len - 32);
2262 /* Shift the field down from the MSB, sign extending. */
2263 tcg_gen_sari_i32(TCGV_LOW(ret), TCGV_LOW(ret), 32 - len);
2264 }
2265 /* Sign-extend the field from 32 bits. */
2266 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
2267 return;
2268 }
2269
2270 if (TCG_TARGET_HAS_sextract_i64
2271 && TCG_TARGET_extract_i64_valid(ofs, len)) {
2272 tcg_gen_op4ii_i64(INDEX_op_sextract_i64, ret, arg, ofs, len);
2273 return;
2274 }
2275
2276 /* Assume that sign-extension, if available, is cheaper than a shift. */
2277 switch (ofs + len) {
2278 case 32:
2279 if (TCG_TARGET_HAS_ext32s_i64) {
2280 tcg_gen_ext32s_i64(ret, arg);
2281 tcg_gen_sari_i64(ret, ret, ofs);
2282 return;
2283 }
2284 break;
2285 case 16:
2286 if (TCG_TARGET_HAS_ext16s_i64) {
2287 tcg_gen_ext16s_i64(ret, arg);
2288 tcg_gen_sari_i64(ret, ret, ofs);
2289 return;
2290 }
2291 break;
2292 case 8:
2293 if (TCG_TARGET_HAS_ext8s_i64) {
2294 tcg_gen_ext8s_i64(ret, arg);
2295 tcg_gen_sari_i64(ret, ret, ofs);
2296 return;
2297 }
2298 break;
2299 }
2300 switch (len) {
2301 case 32:
2302 if (TCG_TARGET_HAS_ext32s_i64) {
2303 tcg_gen_shri_i64(ret, arg, ofs);
2304 tcg_gen_ext32s_i64(ret, ret);
2305 return;
2306 }
2307 break;
2308 case 16:
2309 if (TCG_TARGET_HAS_ext16s_i64) {
2310 tcg_gen_shri_i64(ret, arg, ofs);
2311 tcg_gen_ext16s_i64(ret, ret);
2312 return;
2313 }
2314 break;
2315 case 8:
2316 if (TCG_TARGET_HAS_ext8s_i64) {
2317 tcg_gen_shri_i64(ret, arg, ofs);
2318 tcg_gen_ext8s_i64(ret, ret);
2319 return;
2320 }
2321 break;
2322 }
2323 tcg_gen_shli_i64(ret, arg, 64 - len - ofs);
2324 tcg_gen_sari_i64(ret, ret, 64 - len);
2325}
2326
951c6300
RH		 2327void tcg_gen_movcond_i64(TCGCond cond, TCGv_i64 ret, TCGv_i64 c1,
2328 TCGv_i64 c2, TCGv_i64 v1, TCGv_i64 v2)
2329{
37ed3bf1
RH		 2330 if (cond == TCG_COND_ALWAYS) {
2331 tcg_gen_mov_i64(ret, v1);
2332 } else if (cond == TCG_COND_NEVER) {
2333 tcg_gen_mov_i64(ret, v2);
2334 } else if (TCG_TARGET_REG_BITS == 32) {
3a13c3f3
RH		 2335 TCGv_i32 t0 = tcg_temp_new_i32();
2336 TCGv_i32 t1 = tcg_temp_new_i32();
2337 tcg_gen_op6i_i32(INDEX_op_setcond2_i32, t0,
2338 TCGV_LOW(c1), TCGV_HIGH(c1),
2339 TCGV_LOW(c2), TCGV_HIGH(c2), cond);
2340
2341 if (TCG_TARGET_HAS_movcond_i32) {
2342 tcg_gen_movi_i32(t1, 0);
2343 tcg_gen_movcond_i32(TCG_COND_NE, TCGV_LOW(ret), t0, t1,
2344 TCGV_LOW(v1), TCGV_LOW(v2));
2345 tcg_gen_movcond_i32(TCG_COND_NE, TCGV_HIGH(ret), t0, t1,
2346 TCGV_HIGH(v1), TCGV_HIGH(v2));
2347 } else {
2348 tcg_gen_neg_i32(t0, t0);
951c6300 2349
3a13c3f3
RH		 2350 tcg_gen_and_i32(t1, TCGV_LOW(v1), t0);
2351 tcg_gen_andc_i32(TCGV_LOW(ret), TCGV_LOW(v2), t0);
2352 tcg_gen_or_i32(TCGV_LOW(ret), TCGV_LOW(ret), t1);
951c6300 2353
3a13c3f3
RH		 2354 tcg_gen_and_i32(t1, TCGV_HIGH(v1), t0);
2355 tcg_gen_andc_i32(TCGV_HIGH(ret), TCGV_HIGH(v2), t0);
2356 tcg_gen_or_i32(TCGV_HIGH(ret), TCGV_HIGH(ret), t1);
2357 }
2358 tcg_temp_free_i32(t0);
2359 tcg_temp_free_i32(t1);
2360 } else if (TCG_TARGET_HAS_movcond_i64) {
951c6300
RH		 2361 tcg_gen_op6i_i64(INDEX_op_movcond_i64, ret, c1, c2, v1, v2, cond);
2362 } else {
2363 TCGv_i64 t0 = tcg_temp_new_i64();
2364 TCGv_i64 t1 = tcg_temp_new_i64();
2365 tcg_gen_setcond_i64(cond, t0, c1, c2);
2366 tcg_gen_neg_i64(t0, t0);
2367 tcg_gen_and_i64(t1, v1, t0);
2368 tcg_gen_andc_i64(ret, v2, t0);
2369 tcg_gen_or_i64(ret, ret, t1);
2370 tcg_temp_free_i64(t0);
2371 tcg_temp_free_i64(t1);
2372 }
951c6300
RH		 2373}
2374
2375void tcg_gen_add2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 al,
2376 TCGv_i64 ah, TCGv_i64 bl, TCGv_i64 bh)
2377{
2378 if (TCG_TARGET_HAS_add2_i64) {
2379 tcg_gen_op6_i64(INDEX_op_add2_i64, rl, rh, al, ah, bl, bh);
951c6300
RH		 2380 } else {
2381 TCGv_i64 t0 = tcg_temp_new_i64();
2382 TCGv_i64 t1 = tcg_temp_new_i64();
2383 tcg_gen_add_i64(t0, al, bl);
2384 tcg_gen_setcond_i64(TCG_COND_LTU, t1, t0, al);
2385 tcg_gen_add_i64(rh, ah, bh);
2386 tcg_gen_add_i64(rh, rh, t1);
2387 tcg_gen_mov_i64(rl, t0);
2388 tcg_temp_free_i64(t0);
2389 tcg_temp_free_i64(t1);
2390 }
2391}
2392
2393void tcg_gen_sub2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 al,
2394 TCGv_i64 ah, TCGv_i64 bl, TCGv_i64 bh)
2395{
2396 if (TCG_TARGET_HAS_sub2_i64) {
2397 tcg_gen_op6_i64(INDEX_op_sub2_i64, rl, rh, al, ah, bl, bh);
951c6300
RH		 2398 } else {
2399 TCGv_i64 t0 = tcg_temp_new_i64();
2400 TCGv_i64 t1 = tcg_temp_new_i64();
2401 tcg_gen_sub_i64(t0, al, bl);
2402 tcg_gen_setcond_i64(TCG_COND_LTU, t1, al, bl);
2403 tcg_gen_sub_i64(rh, ah, bh);
2404 tcg_gen_sub_i64(rh, rh, t1);
2405 tcg_gen_mov_i64(rl, t0);
2406 tcg_temp_free_i64(t0);
2407 tcg_temp_free_i64(t1);
2408 }
2409}
2410
2411void tcg_gen_mulu2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 arg1, TCGv_i64 arg2)
2412{
2413 if (TCG_TARGET_HAS_mulu2_i64) {
2414 tcg_gen_op4_i64(INDEX_op_mulu2_i64, rl, rh, arg1, arg2);
951c6300
RH		 2415 } else if (TCG_TARGET_HAS_muluh_i64) {
2416 TCGv_i64 t = tcg_temp_new_i64();
2417 tcg_gen_op3_i64(INDEX_op_mul_i64, t, arg1, arg2);
2418 tcg_gen_op3_i64(INDEX_op_muluh_i64, rh, arg1, arg2);
2419 tcg_gen_mov_i64(rl, t);
2420 tcg_temp_free_i64(t);
2421 } else {
2422 TCGv_i64 t0 = tcg_temp_new_i64();
2423 tcg_gen_mul_i64(t0, arg1, arg2);
2424 gen_helper_muluh_i64(rh, arg1, arg2);
2425 tcg_gen_mov_i64(rl, t0);
2426 tcg_temp_free_i64(t0);
2427 }
2428}
2429
2430void tcg_gen_muls2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 arg1, TCGv_i64 arg2)
2431{
2432 if (TCG_TARGET_HAS_muls2_i64) {
2433 tcg_gen_op4_i64(INDEX_op_muls2_i64, rl, rh, arg1, arg2);
951c6300
RH		 2434 } else if (TCG_TARGET_HAS_mulsh_i64) {
2435 TCGv_i64 t = tcg_temp_new_i64();
2436 tcg_gen_op3_i64(INDEX_op_mul_i64, t, arg1, arg2);
2437 tcg_gen_op3_i64(INDEX_op_mulsh_i64, rh, arg1, arg2);
2438 tcg_gen_mov_i64(rl, t);
2439 tcg_temp_free_i64(t);
2440 } else if (TCG_TARGET_HAS_mulu2_i64 || TCG_TARGET_HAS_muluh_i64) {
2441 TCGv_i64 t0 = tcg_temp_new_i64();
2442 TCGv_i64 t1 = tcg_temp_new_i64();
2443 TCGv_i64 t2 = tcg_temp_new_i64();
2444 TCGv_i64 t3 = tcg_temp_new_i64();
2445 tcg_gen_mulu2_i64(t0, t1, arg1, arg2);
2446 /* Adjust for negative inputs. */
2447 tcg_gen_sari_i64(t2, arg1, 63);
2448 tcg_gen_sari_i64(t3, arg2, 63);
2449 tcg_gen_and_i64(t2, t2, arg2);
2450 tcg_gen_and_i64(t3, t3, arg1);
2451 tcg_gen_sub_i64(rh, t1, t2);
2452 tcg_gen_sub_i64(rh, rh, t3);
2453 tcg_gen_mov_i64(rl, t0);
2454 tcg_temp_free_i64(t0);
2455 tcg_temp_free_i64(t1);
2456 tcg_temp_free_i64(t2);
2457 tcg_temp_free_i64(t3);
2458 } else {
2459 TCGv_i64 t0 = tcg_temp_new_i64();
2460 tcg_gen_mul_i64(t0, arg1, arg2);
2461 gen_helper_mulsh_i64(rh, arg1, arg2);
2462 tcg_gen_mov_i64(rl, t0);
2463 tcg_temp_free_i64(t0);
2464 }
2465}
2466
5087abfb
RH		 2467void tcg_gen_mulsu2_i64(TCGv_i64 rl, TCGv_i64 rh, TCGv_i64 arg1, TCGv_i64 arg2)
2468{
2469 TCGv_i64 t0 = tcg_temp_new_i64();
2470 TCGv_i64 t1 = tcg_temp_new_i64();
2471 TCGv_i64 t2 = tcg_temp_new_i64();
2472 tcg_gen_mulu2_i64(t0, t1, arg1, arg2);
2473 /* Adjust for negative input for the signed arg1. */
2474 tcg_gen_sari_i64(t2, arg1, 63);
2475 tcg_gen_and_i64(t2, t2, arg2);
2476 tcg_gen_sub_i64(rh, t1, t2);
2477 tcg_gen_mov_i64(rl, t0);
2478 tcg_temp_free_i64(t0);
2479 tcg_temp_free_i64(t1);
2480 tcg_temp_free_i64(t2);
2481}
2482
951c6300
RH		 2483/* Size changing operations. */
2484
609ad705 2485void tcg_gen_extrl_i64_i32(TCGv_i32 ret, TCGv_i64 arg)
951c6300 2486{
3a13c3f3 2487 if (TCG_TARGET_REG_BITS == 32) {
609ad705
RH		 2488 tcg_gen_mov_i32(ret, TCGV_LOW(arg));
2489 } else if (TCG_TARGET_HAS_extrl_i64_i32) {
2490 tcg_gen_op2(&tcg_ctx, INDEX_op_extrl_i64_i32,
2491 GET_TCGV_I32(ret), GET_TCGV_I64(arg));
2492 } else {
951c6300 2493 tcg_gen_mov_i32(ret, MAKE_TCGV_I32(GET_TCGV_I64(arg)));
609ad705
RH		 2494 }
2495}
2496
2497void tcg_gen_extrh_i64_i32(TCGv_i32 ret, TCGv_i64 arg)
2498{
2499 if (TCG_TARGET_REG_BITS == 32) {
2500 tcg_gen_mov_i32(ret, TCGV_HIGH(arg));
2501 } else if (TCG_TARGET_HAS_extrh_i64_i32) {
2502 tcg_gen_op2(&tcg_ctx, INDEX_op_extrh_i64_i32,
2503 GET_TCGV_I32(ret), GET_TCGV_I64(arg));
951c6300
RH		 2504 } else {
2505 TCGv_i64 t = tcg_temp_new_i64();
609ad705 2506 tcg_gen_shri_i64(t, arg, 32);
951c6300
RH		 2507 tcg_gen_mov_i32(ret, MAKE_TCGV_I32(GET_TCGV_I64(t)));
2508 tcg_temp_free_i64(t);
2509 }
951c6300
RH		 2510}
2511
2512void tcg_gen_extu_i32_i64(TCGv_i64 ret, TCGv_i32 arg)
2513{
3a13c3f3
RH		 2514 if (TCG_TARGET_REG_BITS == 32) {
2515 tcg_gen_mov_i32(TCGV_LOW(ret), arg);
2516 tcg_gen_movi_i32(TCGV_HIGH(ret), 0);
2517 } else {
4f2331e5
AJ		 2518 tcg_gen_op2(&tcg_ctx, INDEX_op_extu_i32_i64,
2519 GET_TCGV_I64(ret), GET_TCGV_I32(arg));
3a13c3f3 2520 }
951c6300
RH		 2521}
2522
2523void tcg_gen_ext_i32_i64(TCGv_i64 ret, TCGv_i32 arg)
2524{
3a13c3f3
RH		 2525 if (TCG_TARGET_REG_BITS == 32) {
2526 tcg_gen_mov_i32(TCGV_LOW(ret), arg);
2527 tcg_gen_sari_i32(TCGV_HIGH(ret), TCGV_LOW(ret), 31);
2528 } else {
4f2331e5
AJ		 2529 tcg_gen_op2(&tcg_ctx, INDEX_op_ext_i32_i64,
2530 GET_TCGV_I64(ret), GET_TCGV_I32(arg));
3a13c3f3 2531 }
951c6300
RH		 2532}
2533
2534void tcg_gen_concat_i32_i64(TCGv_i64 dest, TCGv_i32 low, TCGv_i32 high)
2535{
3a13c3f3
RH		 2536 TCGv_i64 tmp;
2537
2538 if (TCG_TARGET_REG_BITS == 32) {
2539 tcg_gen_mov_i32(TCGV_LOW(dest), low);
2540 tcg_gen_mov_i32(TCGV_HIGH(dest), high);
2541 return;
2542 }
2543
2544 tmp = tcg_temp_new_i64();
951c6300
RH		 2545 /* These extensions are only needed for type correctness.
2546 We may be able to do better given target specific information. */
2547 tcg_gen_extu_i32_i64(tmp, high);
2548 tcg_gen_extu_i32_i64(dest, low);
2549 /* If deposit is available, use it. Otherwise use the extra
2550 knowledge that we have of the zero-extensions above. */
2551 if (TCG_TARGET_HAS_deposit_i64 && TCG_TARGET_deposit_i64_valid(32, 32)) {
2552 tcg_gen_deposit_i64(dest, dest, tmp, 32, 32);
2553 } else {
2554 tcg_gen_shli_i64(tmp, tmp, 32);
2555 tcg_gen_or_i64(dest, dest, tmp);
2556 }
2557 tcg_temp_free_i64(tmp);
951c6300
RH		 2558}
2559
2560void tcg_gen_extr_i64_i32(TCGv_i32 lo, TCGv_i32 hi, TCGv_i64 arg)
2561{
3a13c3f3
RH		 2562 if (TCG_TARGET_REG_BITS == 32) {
2563 tcg_gen_mov_i32(lo, TCGV_LOW(arg));
2564 tcg_gen_mov_i32(hi, TCGV_HIGH(arg));
2565 } else {
609ad705
RH		 2566 tcg_gen_extrl_i64_i32(lo, arg);
2567 tcg_gen_extrh_i64_i32(hi, arg);
3a13c3f3 2568 }
951c6300
RH		 2569}
2570
2571void tcg_gen_extr32_i64(TCGv_i64 lo, TCGv_i64 hi, TCGv_i64 arg)
2572{
2573 tcg_gen_ext32u_i64(lo, arg);
2574 tcg_gen_shri_i64(hi, arg, 32);
2575}
2576
2577/* QEMU specific operations. */
2578
2579void tcg_gen_goto_tb(unsigned idx)
2580{
2581 /* We only support two chained exits. */
2582 tcg_debug_assert(idx <= 1);
2583#ifdef CONFIG_DEBUG_TCG
2584 /* Verify that we havn't seen this numbered exit before. */
2585 tcg_debug_assert((tcg_ctx.goto_tb_issue_mask & (1 << idx)) == 0);
2586 tcg_ctx.goto_tb_issue_mask |= 1 << idx;
2587#endif
2588 tcg_gen_op1i(INDEX_op_goto_tb, idx);
2589}
2590
cedbcb01
EC		 2591void tcg_gen_lookup_and_goto_ptr(TCGv addr)
2592{
2593 if (TCG_TARGET_HAS_goto_ptr && !qemu_loglevel_mask(CPU_LOG_TB_NOCHAIN)) {
2594 TCGv_ptr ptr = tcg_temp_new_ptr();
2595 gen_helper_lookup_tb_ptr(ptr, tcg_ctx.tcg_env, addr);
2596 tcg_gen_op1i(INDEX_op_goto_ptr, GET_TCGV_PTR(ptr));
2597 tcg_temp_free_ptr(ptr);
2598 } else {
2599 tcg_gen_exit_tb(0);
2600 }
2601}
2602
951c6300
RH		 2603static inline TCGMemOp tcg_canonicalize_memop(TCGMemOp op, bool is64, bool st)
2604{
1f00b27f
SS		 2605 /* Trigger the asserts within as early as possible. */
2606 (void)get_alignment_bits(op);
2607
951c6300
RH		 2608 switch (op & MO_SIZE) {
2609 case MO_8:
2610 op &= ~MO_BSWAP;
2611 break;
2612 case MO_16:
2613 break;
2614 case MO_32:
2615 if (!is64) {
2616 op &= ~MO_SIGN;
2617 }
2618 break;
2619 case MO_64:
2620 if (!is64) {
2621 tcg_abort();
2622 }
2623 break;
2624 }
2625 if (st) {
2626 op &= ~MO_SIGN;
2627 }
2628 return op;
2629}
2630
c45cb8bb
RH		 2631static void gen_ldst_i32(TCGOpcode opc, TCGv_i32 val, TCGv addr,
2632 TCGMemOp memop, TCGArg idx)
951c6300 2633{
59227d5d 2634 TCGMemOpIdx oi = make_memop_idx(memop, idx);
c45cb8bb 2635#if TARGET_LONG_BITS == 32
59227d5d 2636 tcg_gen_op3i_i32(opc, val, addr, oi);
c45cb8bb 2637#else
3a13c3f3 2638 if (TCG_TARGET_REG_BITS == 32) {
59227d5d 2639 tcg_gen_op4i_i32(opc, val, TCGV_LOW(addr), TCGV_HIGH(addr), oi);
3a13c3f3 2640 } else {
59227d5d 2641 tcg_gen_op3(&tcg_ctx, opc, GET_TCGV_I32(val), GET_TCGV_I64(addr), oi);
3a13c3f3 2642 }
c45cb8bb 2643#endif
951c6300
RH		 2644}
2645
c45cb8bb
RH		 2646static void gen_ldst_i64(TCGOpcode opc, TCGv_i64 val, TCGv addr,
2647 TCGMemOp memop, TCGArg idx)
2648{
59227d5d 2649 TCGMemOpIdx oi = make_memop_idx(memop, idx);
951c6300 2650#if TARGET_LONG_BITS == 32
c45cb8bb 2651 if (TCG_TARGET_REG_BITS == 32) {
59227d5d 2652 tcg_gen_op4i_i32(opc, TCGV_LOW(val), TCGV_HIGH(val), addr, oi);
c45cb8bb 2653 } else {
59227d5d 2654 tcg_gen_op3(&tcg_ctx, opc, GET_TCGV_I64(val), GET_TCGV_I32(addr), oi);
c45cb8bb 2655 }
951c6300 2656#else
c45cb8bb 2657 if (TCG_TARGET_REG_BITS == 32) {
59227d5d
RH		 2658 tcg_gen_op5i_i32(opc, TCGV_LOW(val), TCGV_HIGH(val),
2659 TCGV_LOW(addr), TCGV_HIGH(addr), oi);
c45cb8bb 2660 } else {
59227d5d 2661 tcg_gen_op3i_i64(opc, val, addr, oi);
c45cb8bb 2662 }
951c6300 2663#endif
c45cb8bb 2664}
951c6300 2665
b32dc337
PK		 2666static void tcg_gen_req_mo(TCGBar type)
2667{
2668#ifdef TCG_GUEST_DEFAULT_MO
2669 type &= TCG_GUEST_DEFAULT_MO;
2670#endif
2671 type &= ~TCG_TARGET_DEFAULT_MO;
2672 if (type) {
2673 tcg_gen_mb(type | TCG_BAR_SC);
2674 }
2675}
2676
951c6300
RH		 2677void tcg_gen_qemu_ld_i32(TCGv_i32 val, TCGv addr, TCGArg idx, TCGMemOp memop)
2678{
b32dc337 2679 tcg_gen_req_mo(TCG_MO_LD_LD | TCG_MO_ST_LD);
951c6300 2680 memop = tcg_canonicalize_memop(memop, 0, 0);
dcdaadb6
LV		 2681 trace_guest_mem_before_tcg(tcg_ctx.cpu, tcg_ctx.tcg_env,
2682 addr, trace_mem_get_info(memop, 0));
c45cb8bb 2683 gen_ldst_i32(INDEX_op_qemu_ld_i32, val, addr, memop, idx);
951c6300
RH		 2684}
2685
2686void tcg_gen_qemu_st_i32(TCGv_i32 val, TCGv addr, TCGArg idx, TCGMemOp memop)
2687{
b32dc337 2688 tcg_gen_req_mo(TCG_MO_LD_ST | TCG_MO_ST_ST);
951c6300 2689 memop = tcg_canonicalize_memop(memop, 0, 1);
dcdaadb6
LV		 2690 trace_guest_mem_before_tcg(tcg_ctx.cpu, tcg_ctx.tcg_env,
2691 addr, trace_mem_get_info(memop, 1));
c45cb8bb 2692 gen_ldst_i32(INDEX_op_qemu_st_i32, val, addr, memop, idx);
951c6300
RH		 2693}
2694
2695void tcg_gen_qemu_ld_i64(TCGv_i64 val, TCGv addr, TCGArg idx, TCGMemOp memop)
2696{
b32dc337 2697 tcg_gen_req_mo(TCG_MO_LD_LD | TCG_MO_ST_LD);
3a13c3f3 2698 if (TCG_TARGET_REG_BITS == 32 && (memop & MO_SIZE) < MO_64) {
951c6300
RH		 2699 tcg_gen_qemu_ld_i32(TCGV_LOW(val), addr, idx, memop);
2700 if (memop & MO_SIGN) {
2701 tcg_gen_sari_i32(TCGV_HIGH(val), TCGV_LOW(val), 31);
2702 } else {
2703 tcg_gen_movi_i32(TCGV_HIGH(val), 0);
2704 }
2705 return;
2706 }
951c6300 2707
c45cb8bb 2708 memop = tcg_canonicalize_memop(memop, 1, 0);
dcdaadb6
LV		 2709 trace_guest_mem_before_tcg(tcg_ctx.cpu, tcg_ctx.tcg_env,
2710 addr, trace_mem_get_info(memop, 0));
c45cb8bb 2711 gen_ldst_i64(INDEX_op_qemu_ld_i64, val, addr, memop, idx);
951c6300
RH		 2712}
2713
2714void tcg_gen_qemu_st_i64(TCGv_i64 val, TCGv addr, TCGArg idx, TCGMemOp memop)
2715{
b32dc337 2716 tcg_gen_req_mo(TCG_MO_LD_ST | TCG_MO_ST_ST);
3a13c3f3 2717 if (TCG_TARGET_REG_BITS == 32 && (memop & MO_SIZE) < MO_64) {
951c6300
RH		 2718 tcg_gen_qemu_st_i32(TCGV_LOW(val), addr, idx, memop);
2719 return;
2720 }
951c6300 2721
c45cb8bb 2722 memop = tcg_canonicalize_memop(memop, 1, 1);
dcdaadb6
LV		 2723 trace_guest_mem_before_tcg(tcg_ctx.cpu, tcg_ctx.tcg_env,
2724 addr, trace_mem_get_info(memop, 1));
c45cb8bb 2725 gen_ldst_i64(INDEX_op_qemu_st_i64, val, addr, memop, idx);
951c6300 2726}
c482cb11
RH		 2727
2728static void tcg_gen_ext_i32(TCGv_i32 ret, TCGv_i32 val, TCGMemOp opc)
2729{
2730 switch (opc & MO_SSIZE) {
2731 case MO_SB:
2732 tcg_gen_ext8s_i32(ret, val);
2733 break;
2734 case MO_UB:
2735 tcg_gen_ext8u_i32(ret, val);
2736 break;
2737 case MO_SW:
2738 tcg_gen_ext16s_i32(ret, val);
2739 break;
2740 case MO_UW:
2741 tcg_gen_ext16u_i32(ret, val);
2742 break;
2743 default:
2744 tcg_gen_mov_i32(ret, val);
2745 break;
2746 }
2747}
2748
2749static void tcg_gen_ext_i64(TCGv_i64 ret, TCGv_i64 val, TCGMemOp opc)
2750{
2751 switch (opc & MO_SSIZE) {
2752 case MO_SB:
2753 tcg_gen_ext8s_i64(ret, val);
2754 break;
2755 case MO_UB:
2756 tcg_gen_ext8u_i64(ret, val);
2757 break;
2758 case MO_SW:
2759 tcg_gen_ext16s_i64(ret, val);
2760 break;
2761 case MO_UW:
2762 tcg_gen_ext16u_i64(ret, val);
2763 break;
2764 case MO_SL:
2765 tcg_gen_ext32s_i64(ret, val);
2766 break;
2767 case MO_UL:
2768 tcg_gen_ext32u_i64(ret, val);
2769 break;
2770 default:
2771 tcg_gen_mov_i64(ret, val);
2772 break;
2773 }
2774}
2775
2776#ifdef CONFIG_SOFTMMU
2777typedef void (*gen_atomic_cx_i32)(TCGv_i32, TCGv_env, TCGv,
2778 TCGv_i32, TCGv_i32, TCGv_i32);
2779typedef void (*gen_atomic_cx_i64)(TCGv_i64, TCGv_env, TCGv,
2780 TCGv_i64, TCGv_i64, TCGv_i32);
2781typedef void (*gen_atomic_op_i32)(TCGv_i32, TCGv_env, TCGv,
2782 TCGv_i32, TCGv_i32);
2783typedef void (*gen_atomic_op_i64)(TCGv_i64, TCGv_env, TCGv,
2784 TCGv_i64, TCGv_i32);
2785#else
2786typedef void (*gen_atomic_cx_i32)(TCGv_i32, TCGv_env, TCGv, TCGv_i32, TCGv_i32);
2787typedef void (*gen_atomic_cx_i64)(TCGv_i64, TCGv_env, TCGv, TCGv_i64, TCGv_i64);
2788typedef void (*gen_atomic_op_i32)(TCGv_i32, TCGv_env, TCGv, TCGv_i32);
2789typedef void (*gen_atomic_op_i64)(TCGv_i64, TCGv_env, TCGv, TCGv_i64);
2790#endif
2791
df79b996
RH		 2792#ifdef CONFIG_ATOMIC64
2793# define WITH_ATOMIC64(X) X,
2794#else
2795# define WITH_ATOMIC64(X)
2796#endif
2797
c482cb11
RH		 2798static void * const table_cmpxchg[16] = {
2799 [MO_8] = gen_helper_atomic_cmpxchgb,
2800 [MO_16 | MO_LE] = gen_helper_atomic_cmpxchgw_le,
2801 [MO_16 | MO_BE] = gen_helper_atomic_cmpxchgw_be,
2802 [MO_32 | MO_LE] = gen_helper_atomic_cmpxchgl_le,
2803 [MO_32 | MO_BE] = gen_helper_atomic_cmpxchgl_be,
df79b996
RH		 2804 WITH_ATOMIC64([MO_64 | MO_LE] = gen_helper_atomic_cmpxchgq_le)
2805 WITH_ATOMIC64([MO_64 | MO_BE] = gen_helper_atomic_cmpxchgq_be)
c482cb11
RH		 2806};
2807
2808void tcg_gen_atomic_cmpxchg_i32(TCGv_i32 retv, TCGv addr, TCGv_i32 cmpv,
2809 TCGv_i32 newv, TCGArg idx, TCGMemOp memop)
2810{
2811 memop = tcg_canonicalize_memop(memop, 0, 0);
2812
2813 if (!parallel_cpus) {
2814 TCGv_i32 t1 = tcg_temp_new_i32();
2815 TCGv_i32 t2 = tcg_temp_new_i32();
2816
2817 tcg_gen_ext_i32(t2, cmpv, memop & MO_SIZE);
2818
2819 tcg_gen_qemu_ld_i32(t1, addr, idx, memop & ~MO_SIGN);
2820 tcg_gen_movcond_i32(TCG_COND_EQ, t2, t1, t2, newv, t1);
2821 tcg_gen_qemu_st_i32(t2, addr, idx, memop);
2822 tcg_temp_free_i32(t2);
2823
2824 if (memop & MO_SIGN) {
2825 tcg_gen_ext_i32(retv, t1, memop);
2826 } else {
2827 tcg_gen_mov_i32(retv, t1);
2828 }
2829 tcg_temp_free_i32(t1);
2830 } else {
2831 gen_atomic_cx_i32 gen;
2832
2833 gen = table_cmpxchg[memop & (MO_SIZE | MO_BSWAP)];
2834 tcg_debug_assert(gen != NULL);
2835
2836#ifdef CONFIG_SOFTMMU
2837 {
2838 TCGv_i32 oi = tcg_const_i32(make_memop_idx(memop & ~MO_SIGN, idx));
2839 gen(retv, tcg_ctx.tcg_env, addr, cmpv, newv, oi);
2840 tcg_temp_free_i32(oi);
2841 }
2842#else
2843 gen(retv, tcg_ctx.tcg_env, addr, cmpv, newv);
2844#endif
2845
2846 if (memop & MO_SIGN) {
2847 tcg_gen_ext_i32(retv, retv, memop);
2848 }
2849 }
2850}
2851
2852void tcg_gen_atomic_cmpxchg_i64(TCGv_i64 retv, TCGv addr, TCGv_i64 cmpv,
2853 TCGv_i64 newv, TCGArg idx, TCGMemOp memop)
2854{
2855 memop = tcg_canonicalize_memop(memop, 1, 0);
2856
2857 if (!parallel_cpus) {
2858 TCGv_i64 t1 = tcg_temp_new_i64();
2859 TCGv_i64 t2 = tcg_temp_new_i64();
2860
2861 tcg_gen_ext_i64(t2, cmpv, memop & MO_SIZE);
2862
2863 tcg_gen_qemu_ld_i64(t1, addr, idx, memop & ~MO_SIGN);
2864 tcg_gen_movcond_i64(TCG_COND_EQ, t2, t1, t2, newv, t1);
2865 tcg_gen_qemu_st_i64(t2, addr, idx, memop);
2866 tcg_temp_free_i64(t2);
2867
2868 if (memop & MO_SIGN) {
2869 tcg_gen_ext_i64(retv, t1, memop);
2870 } else {
2871 tcg_gen_mov_i64(retv, t1);
2872 }
2873 tcg_temp_free_i64(t1);
2874 } else if ((memop & MO_SIZE) == MO_64) {
df79b996 2875#ifdef CONFIG_ATOMIC64
c482cb11
RH		 2876 gen_atomic_cx_i64 gen;
2877
2878 gen = table_cmpxchg[memop & (MO_SIZE | MO_BSWAP)];
2879 tcg_debug_assert(gen != NULL);
2880
2881#ifdef CONFIG_SOFTMMU
2882 {
2883 TCGv_i32 oi = tcg_const_i32(make_memop_idx(memop, idx));
2884 gen(retv, tcg_ctx.tcg_env, addr, cmpv, newv, oi);
2885 tcg_temp_free_i32(oi);
2886 }
2887#else
2888 gen(retv, tcg_ctx.tcg_env, addr, cmpv, newv);
2889#endif
df79b996
RH		 2890#else
2891 gen_helper_exit_atomic(tcg_ctx.tcg_env);
79b1af90
RH		 2892 /* Produce a result, so that we have a well-formed opcode stream
2893 with respect to uses of the result in the (dead) code following. */
2894 tcg_gen_movi_i64(retv, 0);
df79b996 2895#endif /* CONFIG_ATOMIC64 */
c482cb11
RH		 2896 } else {
2897 TCGv_i32 c32 = tcg_temp_new_i32();
2898 TCGv_i32 n32 = tcg_temp_new_i32();
2899 TCGv_i32 r32 = tcg_temp_new_i32();
2900
2901 tcg_gen_extrl_i64_i32(c32, cmpv);
2902 tcg_gen_extrl_i64_i32(n32, newv);
2903 tcg_gen_atomic_cmpxchg_i32(r32, addr, c32, n32, idx, memop & ~MO_SIGN);
2904 tcg_temp_free_i32(c32);
2905 tcg_temp_free_i32(n32);
2906
2907 tcg_gen_extu_i32_i64(retv, r32);
2908 tcg_temp_free_i32(r32);
2909
2910 if (memop & MO_SIGN) {
2911 tcg_gen_ext_i64(retv, retv, memop);
2912 }
2913 }
2914}
2915
2916static void do_nonatomic_op_i32(TCGv_i32 ret, TCGv addr, TCGv_i32 val,
2917 TCGArg idx, TCGMemOp memop, bool new_val,
2918 void (*gen)(TCGv_i32, TCGv_i32, TCGv_i32))
2919{
2920 TCGv_i32 t1 = tcg_temp_new_i32();
2921 TCGv_i32 t2 = tcg_temp_new_i32();
2922
2923 memop = tcg_canonicalize_memop(memop, 0, 0);
2924
2925 tcg_gen_qemu_ld_i32(t1, addr, idx, memop & ~MO_SIGN);
2926 gen(t2, t1, val);
2927 tcg_gen_qemu_st_i32(t2, addr, idx, memop);
2928
2929 tcg_gen_ext_i32(ret, (new_val ? t2 : t1), memop);
2930 tcg_temp_free_i32(t1);
2931 tcg_temp_free_i32(t2);
2932}
2933
2934static void do_atomic_op_i32(TCGv_i32 ret, TCGv addr, TCGv_i32 val,
2935 TCGArg idx, TCGMemOp memop, void * const table[])
2936{
2937 gen_atomic_op_i32 gen;
2938
2939 memop = tcg_canonicalize_memop(memop, 0, 0);
2940
2941 gen = table[memop & (MO_SIZE | MO_BSWAP)];
2942 tcg_debug_assert(gen != NULL);
2943
2944#ifdef CONFIG_SOFTMMU
2945 {
2946 TCGv_i32 oi = tcg_const_i32(make_memop_idx(memop & ~MO_SIGN, idx));
2947 gen(ret, tcg_ctx.tcg_env, addr, val, oi);
2948 tcg_temp_free_i32(oi);
2949 }
2950#else
2951 gen(ret, tcg_ctx.tcg_env, addr, val);
2952#endif
2953
2954 if (memop & MO_SIGN) {
2955 tcg_gen_ext_i32(ret, ret, memop);
2956 }
2957}
2958
2959static void do_nonatomic_op_i64(TCGv_i64 ret, TCGv addr, TCGv_i64 val,
2960 TCGArg idx, TCGMemOp memop, bool new_val,
2961 void (*gen)(TCGv_i64, TCGv_i64, TCGv_i64))
2962{
2963 TCGv_i64 t1 = tcg_temp_new_i64();
2964 TCGv_i64 t2 = tcg_temp_new_i64();
2965
2966 memop = tcg_canonicalize_memop(memop, 1, 0);
2967
2968 tcg_gen_qemu_ld_i64(t1, addr, idx, memop & ~MO_SIGN);
2969 gen(t2, t1, val);
2970 tcg_gen_qemu_st_i64(t2, addr, idx, memop);
2971
2972 tcg_gen_ext_i64(ret, (new_val ? t2 : t1), memop);
2973 tcg_temp_free_i64(t1);
2974 tcg_temp_free_i64(t2);
2975}
2976
2977static void do_atomic_op_i64(TCGv_i64 ret, TCGv addr, TCGv_i64 val,
2978 TCGArg idx, TCGMemOp memop, void * const table[])
2979{
2980 memop = tcg_canonicalize_memop(memop, 1, 0);
2981
2982 if ((memop & MO_SIZE) == MO_64) {
df79b996 2983#ifdef CONFIG_ATOMIC64
c482cb11
RH		 2984 gen_atomic_op_i64 gen;
2985
2986 gen = table[memop & (MO_SIZE | MO_BSWAP)];
2987 tcg_debug_assert(gen != NULL);
2988
2989#ifdef CONFIG_SOFTMMU
2990 {
2991 TCGv_i32 oi = tcg_const_i32(make_memop_idx(memop & ~MO_SIGN, idx));
2992 gen(ret, tcg_ctx.tcg_env, addr, val, oi);
2993 tcg_temp_free_i32(oi);
2994 }
2995#else
2996 gen(ret, tcg_ctx.tcg_env, addr, val);
2997#endif
df79b996
RH		 2998#else
2999 gen_helper_exit_atomic(tcg_ctx.tcg_env);
79b1af90
RH		 3000 /* Produce a result, so that we have a well-formed opcode stream
3001 with respect to uses of the result in the (dead) code following. */
3002 tcg_gen_movi_i64(ret, 0);
df79b996 3003#endif /* CONFIG_ATOMIC64 */
c482cb11
RH		 3004 } else {
3005 TCGv_i32 v32 = tcg_temp_new_i32();
3006 TCGv_i32 r32 = tcg_temp_new_i32();
3007
3008 tcg_gen_extrl_i64_i32(v32, val);
3009 do_atomic_op_i32(r32, addr, v32, idx, memop & ~MO_SIGN, table);
3010 tcg_temp_free_i32(v32);
3011
3012 tcg_gen_extu_i32_i64(ret, r32);
3013 tcg_temp_free_i32(r32);
3014
3015 if (memop & MO_SIGN) {
3016 tcg_gen_ext_i64(ret, ret, memop);
3017 }
3018 }
3019}
3020
3021#define GEN_ATOMIC_HELPER(NAME, OP, NEW) \
3022static void * const table_##NAME[16] = { \
3023 [MO_8] = gen_helper_atomic_##NAME##b, \
3024 [MO_16 | MO_LE] = gen_helper_atomic_##NAME##w_le, \
3025 [MO_16 | MO_BE] = gen_helper_atomic_##NAME##w_be, \
3026 [MO_32 | MO_LE] = gen_helper_atomic_##NAME##l_le, \
3027 [MO_32 | MO_BE] = gen_helper_atomic_##NAME##l_be, \
df79b996
RH		 3028 WITH_ATOMIC64([MO_64 | MO_LE] = gen_helper_atomic_##NAME##q_le) \
3029 WITH_ATOMIC64([MO_64 | MO_BE] = gen_helper_atomic_##NAME##q_be) \
c482cb11
RH		 3030}; \
3031void tcg_gen_atomic_##NAME##_i32 \
3032 (TCGv_i32 ret, TCGv addr, TCGv_i32 val, TCGArg idx, TCGMemOp memop) \
3033{ \
3034 if (parallel_cpus) { \
3035 do_atomic_op_i32(ret, addr, val, idx, memop, table_##NAME); \
3036 } else { \
3037 do_nonatomic_op_i32(ret, addr, val, idx, memop, NEW, \
3038 tcg_gen_##OP##_i32); \
3039 } \
3040} \
3041void tcg_gen_atomic_##NAME##_i64 \
3042 (TCGv_i64 ret, TCGv addr, TCGv_i64 val, TCGArg idx, TCGMemOp memop) \
3043{ \
3044 if (parallel_cpus) { \
3045 do_atomic_op_i64(ret, addr, val, idx, memop, table_##NAME); \
3046 } else { \
3047 do_nonatomic_op_i64(ret, addr, val, idx, memop, NEW, \
3048 tcg_gen_##OP##_i64); \
3049 } \
3050}
3051
3052GEN_ATOMIC_HELPER(fetch_add, add, 0)
3053GEN_ATOMIC_HELPER(fetch_and, and, 0)
3054GEN_ATOMIC_HELPER(fetch_or, or, 0)
3055GEN_ATOMIC_HELPER(fetch_xor, xor, 0)
3056
3057GEN_ATOMIC_HELPER(add_fetch, add, 1)
3058GEN_ATOMIC_HELPER(and_fetch, and, 1)
3059GEN_ATOMIC_HELPER(or_fetch, or, 1)
3060GEN_ATOMIC_HELPER(xor_fetch, xor, 1)
3061
3062static void tcg_gen_mov2_i32(TCGv_i32 r, TCGv_i32 a, TCGv_i32 b)
3063{
3064 tcg_gen_mov_i32(r, b);
3065}
3066
3067static void tcg_gen_mov2_i64(TCGv_i64 r, TCGv_i64 a, TCGv_i64 b)
3068{
3069 tcg_gen_mov_i64(r, b);
3070}
3071
3072GEN_ATOMIC_HELPER(xchg, mov2, 0)
3073
3074#undef GEN_ATOMIC_HELPER
Empty description
This page took 0.4866 seconds and 4 git commands to generate.