]> Git Repo - qemu.git/blame - tcg/aarch64/tcg-target.c
projects / qemu.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
tcg-aarch64: Change all ext variables to TCGType
[qemu.git] / tcg / aarch64 / tcg-target.c
CommitLineData
4a136e0a
CF		 1/*
2 * Initial TCG Implementation for aarch64
3 *
4 * Copyright (c) 2013 Huawei Technologies Duesseldorf GmbH
5 * Written by Claudio Fontana
6 *
7 * This work is licensed under the terms of the GNU GPL, version 2 or
8 * (at your option) any later version.
9 *
10 * See the COPYING file in the top-level directory for details.
11 */
12
9ecefc84 13#include "tcg-be-ldst.h"
4a136e0a
CF		 14#include "qemu/bitops.h"
15
7763ffa0
RH		 16/* We're going to re-use TCGType in setting of the SF bit, which controls
17 the size of the operation performed. If we know the values match, it
18 makes things much cleaner. */
19QEMU_BUILD_BUG_ON(TCG_TYPE_I32 != 0 || TCG_TYPE_I64 != 1);
20
4a136e0a
CF		 21#ifndef NDEBUG
22static const char * const tcg_target_reg_names[TCG_TARGET_NB_REGS] = {
23 "%x0", "%x1", "%x2", "%x3", "%x4", "%x5", "%x6", "%x7",
24 "%x8", "%x9", "%x10", "%x11", "%x12", "%x13", "%x14", "%x15",
25 "%x16", "%x17", "%x18", "%x19", "%x20", "%x21", "%x22", "%x23",
26 "%x24", "%x25", "%x26", "%x27", "%x28",
27 "%fp", /* frame pointer */
28 "%lr", /* link register */
29 "%sp", /* stack pointer */
30};
31#endif /* NDEBUG */
32
6a91c7c9
JK		 33#ifdef TARGET_WORDS_BIGENDIAN
34 #define TCG_LDST_BSWAP 1
35#else
36 #define TCG_LDST_BSWAP 0
37#endif
38
4a136e0a
CF		 39static const int tcg_target_reg_alloc_order[] = {
40 TCG_REG_X20, TCG_REG_X21, TCG_REG_X22, TCG_REG_X23,
41 TCG_REG_X24, TCG_REG_X25, TCG_REG_X26, TCG_REG_X27,
6a91c7c9 42 TCG_REG_X28, /* we will reserve this for GUEST_BASE if configured */
4a136e0a
CF		 43
44 TCG_REG_X9, TCG_REG_X10, TCG_REG_X11, TCG_REG_X12,
45 TCG_REG_X13, TCG_REG_X14, TCG_REG_X15,
46 TCG_REG_X16, TCG_REG_X17,
47
48 TCG_REG_X18, TCG_REG_X19, /* will not use these, see tcg_target_init */
49
50 TCG_REG_X0, TCG_REG_X1, TCG_REG_X2, TCG_REG_X3,
51 TCG_REG_X4, TCG_REG_X5, TCG_REG_X6, TCG_REG_X7,
52
53 TCG_REG_X8, /* will not use, see tcg_target_init */
54};
55
56static const int tcg_target_call_iarg_regs[8] = {
57 TCG_REG_X0, TCG_REG_X1, TCG_REG_X2, TCG_REG_X3,
58 TCG_REG_X4, TCG_REG_X5, TCG_REG_X6, TCG_REG_X7
59};
60static const int tcg_target_call_oarg_regs[1] = {
61 TCG_REG_X0
62};
63
64#define TCG_REG_TMP TCG_REG_X8
65
6a91c7c9
JK		 66#ifndef CONFIG_SOFTMMU
67# if defined(CONFIG_USE_GUEST_BASE)
68# define TCG_REG_GUEST_BASE TCG_REG_X28
69# else
70# define TCG_REG_GUEST_BASE TCG_REG_XZR
71# endif
72#endif
73
4a136e0a
CF		 74static inline void reloc_pc26(void *code_ptr, tcg_target_long target)
75{
76 tcg_target_long offset; uint32_t insn;
77 offset = (target - (tcg_target_long)code_ptr) / 4;
78 /* read instruction, mask away previous PC_REL26 parameter contents,
79 set the proper offset, then write back the instruction. */
80 insn = *(uint32_t *)code_ptr;
81 insn = deposit32(insn, 0, 26, offset);
82 *(uint32_t *)code_ptr = insn;
83}
84
85static inline void reloc_pc19(void *code_ptr, tcg_target_long target)
86{
87 tcg_target_long offset; uint32_t insn;
88 offset = (target - (tcg_target_long)code_ptr) / 4;
89 /* read instruction, mask away previous PC_REL19 parameter contents,
90 set the proper offset, then write back the instruction. */
91 insn = *(uint32_t *)code_ptr;
92 insn = deposit32(insn, 5, 19, offset);
93 *(uint32_t *)code_ptr = insn;
94}
95
96static inline void patch_reloc(uint8_t *code_ptr, int type,
2ba7fae2 97 intptr_t value, intptr_t addend)
4a136e0a
CF		 98{
99 value += addend;
100
101 switch (type) {
102 case R_AARCH64_JUMP26:
103 case R_AARCH64_CALL26:
104 reloc_pc26(code_ptr, value);
105 break;
106 case R_AARCH64_CONDBR19:
107 reloc_pc19(code_ptr, value);
108 break;
109
110 default:
111 tcg_abort();
112 }
113}
114
115/* parse target specific constraints */
116static int target_parse_constraint(TCGArgConstraint *ct,
117 const char **pct_str)
118{
119 const char *ct_str = *pct_str;
120
121 switch (ct_str[0]) {
122 case 'r':
123 ct->ct |= TCG_CT_REG;
124 tcg_regset_set32(ct->u.regs, 0, (1ULL << TCG_TARGET_NB_REGS) - 1);
125 break;
126 case 'l': /* qemu_ld / qemu_st address, data_reg */
127 ct->ct |= TCG_CT_REG;
128 tcg_regset_set32(ct->u.regs, 0, (1ULL << TCG_TARGET_NB_REGS) - 1);
129#ifdef CONFIG_SOFTMMU
130 /* x0 and x1 will be overwritten when reading the tlb entry,
131 and x2, and x3 for helper args, better to avoid using them. */
132 tcg_regset_reset_reg(ct->u.regs, TCG_REG_X0);
133 tcg_regset_reset_reg(ct->u.regs, TCG_REG_X1);
134 tcg_regset_reset_reg(ct->u.regs, TCG_REG_X2);
135 tcg_regset_reset_reg(ct->u.regs, TCG_REG_X3);
136#endif
137 break;
138 default:
139 return -1;
140 }
141
142 ct_str++;
143 *pct_str = ct_str;
144 return 0;
145}
146
147static inline int tcg_target_const_match(tcg_target_long val,
148 const TCGArgConstraint *arg_ct)
149{
150 int ct = arg_ct->ct;
151
152 if (ct & TCG_CT_CONST) {
153 return 1;
154 }
155
156 return 0;
157}
158
159enum aarch64_cond_code {
160 COND_EQ = 0x0,
161 COND_NE = 0x1,
162 COND_CS = 0x2, /* Unsigned greater or equal */
163 COND_HS = COND_CS, /* ALIAS greater or equal */
164 COND_CC = 0x3, /* Unsigned less than */
165 COND_LO = COND_CC, /* ALIAS Lower */
166 COND_MI = 0x4, /* Negative */
167 COND_PL = 0x5, /* Zero or greater */
168 COND_VS = 0x6, /* Overflow */
169 COND_VC = 0x7, /* No overflow */
170 COND_HI = 0x8, /* Unsigned greater than */
171 COND_LS = 0x9, /* Unsigned less or equal */
172 COND_GE = 0xa,
173 COND_LT = 0xb,
174 COND_GT = 0xc,
175 COND_LE = 0xd,
176 COND_AL = 0xe,
177 COND_NV = 0xf, /* behaves like COND_AL here */
178};
179
180static const enum aarch64_cond_code tcg_cond_to_aarch64[] = {
181 [TCG_COND_EQ] = COND_EQ,
182 [TCG_COND_NE] = COND_NE,
183 [TCG_COND_LT] = COND_LT,
184 [TCG_COND_GE] = COND_GE,
185 [TCG_COND_LE] = COND_LE,
186 [TCG_COND_GT] = COND_GT,
187 /* unsigned */
188 [TCG_COND_LTU] = COND_LO,
189 [TCG_COND_GTU] = COND_HI,
190 [TCG_COND_GEU] = COND_HS,
191 [TCG_COND_LEU] = COND_LS,
192};
193
194/* opcodes for LDR / STR instructions with base + simm9 addressing */
195enum aarch64_ldst_op_data { /* size of the data moved */
196 LDST_8 = 0x38,
197 LDST_16 = 0x78,
198 LDST_32 = 0xb8,
199 LDST_64 = 0xf8,
200};
201enum aarch64_ldst_op_type { /* type of operation */
202 LDST_ST = 0x0, /* store */
203 LDST_LD = 0x4, /* load */
204 LDST_LD_S_X = 0x8, /* load and sign-extend into Xt */
205 LDST_LD_S_W = 0xc, /* load and sign-extend into Wt */
206};
207
208enum aarch64_arith_opc {
4a136e0a 209 ARITH_AND = 0x0a,
36fac14a 210 ARITH_ADD = 0x0b,
4a136e0a 211 ARITH_OR = 0x2a,
36fac14a
CF		 212 ARITH_ADDS = 0x2b,
213 ARITH_XOR = 0x4a,
214 ARITH_SUB = 0x4b,
215 ARITH_ANDS = 0x6a,
216 ARITH_SUBS = 0x6b,
4a136e0a
CF		 217};
218
219enum aarch64_srr_opc {
220 SRR_SHL = 0x0,
221 SRR_SHR = 0x4,
222 SRR_SAR = 0x8,
223 SRR_ROR = 0xc
224};
225
226static inline enum aarch64_ldst_op_data
227aarch64_ldst_get_data(TCGOpcode tcg_op)
228{
229 switch (tcg_op) {
230 case INDEX_op_ld8u_i32:
231 case INDEX_op_ld8s_i32:
232 case INDEX_op_ld8u_i64:
233 case INDEX_op_ld8s_i64:
234 case INDEX_op_st8_i32:
235 case INDEX_op_st8_i64:
236 return LDST_8;
237
238 case INDEX_op_ld16u_i32:
239 case INDEX_op_ld16s_i32:
240 case INDEX_op_ld16u_i64:
241 case INDEX_op_ld16s_i64:
242 case INDEX_op_st16_i32:
243 case INDEX_op_st16_i64:
244 return LDST_16;
245
246 case INDEX_op_ld_i32:
247 case INDEX_op_st_i32:
248 case INDEX_op_ld32u_i64:
249 case INDEX_op_ld32s_i64:
250 case INDEX_op_st32_i64:
251 return LDST_32;
252
253 case INDEX_op_ld_i64:
254 case INDEX_op_st_i64:
255 return LDST_64;
256
257 default:
258 tcg_abort();
259 }
260}
261
262static inline enum aarch64_ldst_op_type
263aarch64_ldst_get_type(TCGOpcode tcg_op)
264{
265 switch (tcg_op) {
266 case INDEX_op_st8_i32:
267 case INDEX_op_st16_i32:
268 case INDEX_op_st8_i64:
269 case INDEX_op_st16_i64:
270 case INDEX_op_st_i32:
271 case INDEX_op_st32_i64:
272 case INDEX_op_st_i64:
273 return LDST_ST;
274
275 case INDEX_op_ld8u_i32:
276 case INDEX_op_ld16u_i32:
277 case INDEX_op_ld8u_i64:
278 case INDEX_op_ld16u_i64:
279 case INDEX_op_ld_i32:
280 case INDEX_op_ld32u_i64:
281 case INDEX_op_ld_i64:
282 return LDST_LD;
283
284 case INDEX_op_ld8s_i32:
285 case INDEX_op_ld16s_i32:
286 return LDST_LD_S_W;
287
288 case INDEX_op_ld8s_i64:
289 case INDEX_op_ld16s_i64:
290 case INDEX_op_ld32s_i64:
291 return LDST_LD_S_X;
292
293 default:
294 tcg_abort();
295 }
296}
297
298static inline uint32_t tcg_in32(TCGContext *s)
299{
300 uint32_t v = *(uint32_t *)s->code_ptr;
301 return v;
302}
303
304static inline void tcg_out_ldst_9(TCGContext *s,
305 enum aarch64_ldst_op_data op_data,
306 enum aarch64_ldst_op_type op_type,
307 TCGReg rd, TCGReg rn, tcg_target_long offset)
308{
309 /* use LDUR with BASE register with 9bit signed unscaled offset */
310 unsigned int mod, off;
311
312 if (offset < 0) {
313 off = (256 + offset);
314 mod = 0x1;
315 } else {
316 off = offset;
317 mod = 0x0;
318 }
319
320 mod |= op_type;
321 tcg_out32(s, op_data << 24 | mod << 20 | off << 12 | rn << 5 | rd);
322}
323
b1f6dc0d
CF		 324/* tcg_out_ldst_12 expects a scaled unsigned immediate offset */
325static inline void tcg_out_ldst_12(TCGContext *s,
326 enum aarch64_ldst_op_data op_data,
327 enum aarch64_ldst_op_type op_type,
328 TCGReg rd, TCGReg rn,
329 tcg_target_ulong scaled_uimm)
330{
331 tcg_out32(s, (op_data | 1) << 24
332 | op_type << 20 | scaled_uimm << 10 | rn << 5 | rd);
333}
334
7763ffa0
RH		 335static inline void tcg_out_movr(TCGContext *s, TCGType ext,
336 TCGReg rd, TCGReg src)
4a136e0a
CF		 337{
338 /* register to register move using MOV (shifted register with no shift) */
339 /* using MOV 0x2a0003e0 | (shift).. */
340 unsigned int base = ext ? 0xaa0003e0 : 0x2a0003e0;
341 tcg_out32(s, base | src << 16 | rd);
342}
343
344static inline void tcg_out_movi_aux(TCGContext *s,
345 TCGReg rd, uint64_t value)
346{
347 uint32_t half, base, shift, movk = 0;
348 /* construct halfwords of the immediate with MOVZ/MOVK with LSL */
349 /* using MOVZ 0x52800000 | extended reg.. */
350 base = (value > 0xffffffff) ? 0xd2800000 : 0x52800000;
351 /* count trailing zeros in 16 bit steps, mapping 64 to 0. Emit the
352 first MOVZ with the half-word immediate skipping the zeros, with a shift
353 (LSL) equal to this number. Then morph all next instructions into MOVKs.
354 Zero the processed half-word in the value, continue until empty.
355 We build the final result 16bits at a time with up to 4 instructions,
356 but do not emit instructions for 16bit zero holes. */
357 do {
358 shift = ctz64(value) & (63 & -16);
359 half = (value >> shift) & 0xffff;
360 tcg_out32(s, base | movk | shift << 17 | half << 5 | rd);
361 movk = 0x20000000; /* morph next MOVZs into MOVKs */
362 value &= ~(0xffffUL << shift);
363 } while (value);
364}
365
366static inline void tcg_out_movi(TCGContext *s, TCGType type,
367 TCGReg rd, tcg_target_long value)
368{
369 if (type == TCG_TYPE_I64) {
370 tcg_out_movi_aux(s, rd, value);
371 } else {
372 tcg_out_movi_aux(s, rd, value & 0xffffffff);
373 }
374}
375
376static inline void tcg_out_ldst_r(TCGContext *s,
377 enum aarch64_ldst_op_data op_data,
378 enum aarch64_ldst_op_type op_type,
379 TCGReg rd, TCGReg base, TCGReg regoff)
380{
381 /* load from memory to register using base + 64bit register offset */
382 /* using f.e. STR Wt, [Xn, Xm] 0xb8600800|(regoff << 16)|(base << 5)|rd */
383 /* the 0x6000 is for the "no extend field" */
384 tcg_out32(s, 0x00206800
385 | op_data << 24 | op_type << 20 | regoff << 16 | base << 5 | rd);
386}
387
388/* solve the whole ldst problem */
389static inline void tcg_out_ldst(TCGContext *s, enum aarch64_ldst_op_data data,
390 enum aarch64_ldst_op_type type,
391 TCGReg rd, TCGReg rn, tcg_target_long offset)
392{
393 if (offset >= -256 && offset < 256) {
394 tcg_out_ldst_9(s, data, type, rd, rn, offset);
b1f6dc0d 395 return;
4a136e0a 396 }
b1f6dc0d
CF		 397
398 if (offset >= 256) {
399 /* if the offset is naturally aligned and in range,
400 then we can use the scaled uimm12 encoding */
401 unsigned int s_bits = data >> 6;
402 if (!(offset & ((1 << s_bits) - 1))) {
403 tcg_target_ulong scaled_uimm = offset >> s_bits;
404 if (scaled_uimm <= 0xfff) {
405 tcg_out_ldst_12(s, data, type, rd, rn, scaled_uimm);
406 return;
407 }
408 }
409 }
410
411 /* worst-case scenario, move offset to temp register, use reg offset */
412 tcg_out_movi(s, TCG_TYPE_I64, TCG_REG_TMP, offset);
413 tcg_out_ldst_r(s, data, type, rd, rn, TCG_REG_TMP);
4a136e0a
CF		 414}
415
416/* mov alias implemented with add immediate, useful to move to/from SP */
7763ffa0
RH		 417static inline void tcg_out_movr_sp(TCGContext *s, TCGType ext,
418 TCGReg rd, TCGReg rn)
4a136e0a
CF		 419{
420 /* using ADD 0x11000000 | (ext) | rn << 5 | rd */
421 unsigned int base = ext ? 0x91000000 : 0x11000000;
422 tcg_out32(s, base | rn << 5 | rd);
423}
424
425static inline void tcg_out_mov(TCGContext *s,
426 TCGType type, TCGReg ret, TCGReg arg)
427{
428 if (ret != arg) {
429 tcg_out_movr(s, type == TCG_TYPE_I64, ret, arg);
430 }
431}
432
433static inline void tcg_out_ld(TCGContext *s, TCGType type, TCGReg arg,
a05b5b9b 434 TCGReg arg1, intptr_t arg2)
4a136e0a
CF		 435{
436 tcg_out_ldst(s, (type == TCG_TYPE_I64) ? LDST_64 : LDST_32, LDST_LD,
437 arg, arg1, arg2);
438}
439
440static inline void tcg_out_st(TCGContext *s, TCGType type, TCGReg arg,
a05b5b9b 441 TCGReg arg1, intptr_t arg2)
4a136e0a
CF		 442{
443 tcg_out_ldst(s, (type == TCG_TYPE_I64) ? LDST_64 : LDST_32, LDST_ST,
444 arg, arg1, arg2);
445}
446
447static inline void tcg_out_arith(TCGContext *s, enum aarch64_arith_opc opc,
7763ffa0 448 TCGType ext, TCGReg rd, TCGReg rn, TCGReg rm,
36fac14a 449 int shift_imm)
4a136e0a
CF		 450{
451 /* Using shifted register arithmetic operations */
36fac14a
CF		 452 /* if extended register operation (64bit) just OR with 0x80 << 24 */
453 unsigned int shift, base = ext ? (0x80 | opc) << 24 : opc << 24;
454 if (shift_imm == 0) {
455 shift = 0;
456 } else if (shift_imm > 0) {
457 shift = shift_imm << 10 | 1 << 22;
458 } else /* (shift_imm < 0) */ {
459 shift = (-shift_imm) << 10;
460 }
461 tcg_out32(s, base | rm << 16 | shift | rn << 5 | rd);
4a136e0a
CF		 462}
463
7763ffa0 464static inline void tcg_out_mul(TCGContext *s, TCGType ext,
4a136e0a
CF		 465 TCGReg rd, TCGReg rn, TCGReg rm)
466{
467 /* Using MADD 0x1b000000 with Ra = wzr alias MUL 0x1b007c00 */
468 unsigned int base = ext ? 0x9b007c00 : 0x1b007c00;
469 tcg_out32(s, base | rm << 16 | rn << 5 | rd);
470}
471
472static inline void tcg_out_shiftrot_reg(TCGContext *s,
7763ffa0 473 enum aarch64_srr_opc opc, TCGType ext,
4a136e0a
CF		 474 TCGReg rd, TCGReg rn, TCGReg rm)
475{
476 /* using 2-source data processing instructions 0x1ac02000 */
477 unsigned int base = ext ? 0x9ac02000 : 0x1ac02000;
478 tcg_out32(s, base | rm << 16 | opc << 8 | rn << 5 | rd);
479}
480
7763ffa0
RH		 481static inline void tcg_out_ubfm(TCGContext *s, TCGType ext, TCGReg rd,
482 TCGReg rn, unsigned int a, unsigned int b)
4a136e0a
CF		 483{
484 /* Using UBFM 0x53000000 Wd, Wn, a, b */
485 unsigned int base = ext ? 0xd3400000 : 0x53000000;
486 tcg_out32(s, base | a << 16 | b << 10 | rn << 5 | rd);
487}
488
7763ffa0
RH		 489static inline void tcg_out_sbfm(TCGContext *s, TCGType ext, TCGReg rd,
490 TCGReg rn, unsigned int a, unsigned int b)
4a136e0a
CF		 491{
492 /* Using SBFM 0x13000000 Wd, Wn, a, b */
493 unsigned int base = ext ? 0x93400000 : 0x13000000;
494 tcg_out32(s, base | a << 16 | b << 10 | rn << 5 | rd);
495}
496
7763ffa0 497static inline void tcg_out_extr(TCGContext *s, TCGType ext, TCGReg rd,
4a136e0a
CF		 498 TCGReg rn, TCGReg rm, unsigned int a)
499{
500 /* Using EXTR 0x13800000 Wd, Wn, Wm, a */
501 unsigned int base = ext ? 0x93c00000 : 0x13800000;
502 tcg_out32(s, base | rm << 16 | a << 10 | rn << 5 | rd);
503}
504
7763ffa0 505static inline void tcg_out_shl(TCGContext *s, TCGType ext,
4a136e0a
CF		 506 TCGReg rd, TCGReg rn, unsigned int m)
507{
508 int bits, max;
509 bits = ext ? 64 : 32;
510 max = bits - 1;
511 tcg_out_ubfm(s, ext, rd, rn, bits - (m & max), max - (m & max));
512}
513
7763ffa0 514static inline void tcg_out_shr(TCGContext *s, TCGType ext,
4a136e0a
CF		 515 TCGReg rd, TCGReg rn, unsigned int m)
516{
517 int max = ext ? 63 : 31;
518 tcg_out_ubfm(s, ext, rd, rn, m & max, max);
519}
520
7763ffa0 521static inline void tcg_out_sar(TCGContext *s, TCGType ext,
4a136e0a
CF		 522 TCGReg rd, TCGReg rn, unsigned int m)
523{
524 int max = ext ? 63 : 31;
525 tcg_out_sbfm(s, ext, rd, rn, m & max, max);
526}
527
7763ffa0 528static inline void tcg_out_rotr(TCGContext *s, TCGType ext,
4a136e0a
CF		 529 TCGReg rd, TCGReg rn, unsigned int m)
530{
531 int max = ext ? 63 : 31;
532 tcg_out_extr(s, ext, rd, rn, rn, m & max);
533}
534
7763ffa0 535static inline void tcg_out_rotl(TCGContext *s, TCGType ext,
4a136e0a
CF		 536 TCGReg rd, TCGReg rn, unsigned int m)
537{
538 int bits, max;
539 bits = ext ? 64 : 32;
540 max = bits - 1;
541 tcg_out_extr(s, ext, rd, rn, rn, bits - (m & max));
542}
543
7763ffa0
RH		 544static inline void tcg_out_cmp(TCGContext *s, TCGType ext, TCGReg rn,
545 TCGReg rm, int shift_imm)
4a136e0a
CF		 546{
547 /* Using CMP alias SUBS wzr, Wn, Wm */
36fac14a 548 tcg_out_arith(s, ARITH_SUBS, ext, TCG_REG_XZR, rn, rm, shift_imm);
4a136e0a
CF		 549}
550
7763ffa0
RH		 551static inline void tcg_out_cset(TCGContext *s, TCGType ext,
552 TCGReg rd, TCGCond c)
4a136e0a
CF		 553{
554 /* Using CSET alias of CSINC 0x1a800400 Xd, XZR, XZR, invert(cond) */
555 unsigned int base = ext ? 0x9a9f07e0 : 0x1a9f07e0;
556 tcg_out32(s, base | tcg_cond_to_aarch64[tcg_invert_cond(c)] << 12 | rd);
557}
558
559static inline void tcg_out_goto(TCGContext *s, tcg_target_long target)
560{
561 tcg_target_long offset;
562 offset = (target - (tcg_target_long)s->code_ptr) / 4;
563
564 if (offset < -0x02000000 || offset >= 0x02000000) {
565 /* out of 26bit range */
566 tcg_abort();
567 }
568
569 tcg_out32(s, 0x14000000 | (offset & 0x03ffffff));
570}
571
572static inline void tcg_out_goto_noaddr(TCGContext *s)
573{
574 /* We pay attention here to not modify the branch target by
575 reading from the buffer. This ensure that caches and memory are
576 kept coherent during retranslation.
577 Mask away possible garbage in the high bits for the first translation,
578 while keeping the offset bits for retranslation. */
579 uint32_t insn;
580 insn = (tcg_in32(s) & 0x03ffffff) | 0x14000000;
581 tcg_out32(s, insn);
582}
583
584static inline void tcg_out_goto_cond_noaddr(TCGContext *s, TCGCond c)
585{
586 /* see comments in tcg_out_goto_noaddr */
587 uint32_t insn;
588 insn = tcg_in32(s) & (0x07ffff << 5);
589 insn |= 0x54000000 | tcg_cond_to_aarch64[c];
590 tcg_out32(s, insn);
591}
592
593static inline void tcg_out_goto_cond(TCGContext *s, TCGCond c,
594 tcg_target_long target)
595{
596 tcg_target_long offset;
597 offset = (target - (tcg_target_long)s->code_ptr) / 4;
598
599 if (offset < -0x40000 || offset >= 0x40000) {
600 /* out of 19bit range */
601 tcg_abort();
602 }
603
604 offset &= 0x7ffff;
605 tcg_out32(s, 0x54000000 | tcg_cond_to_aarch64[c] | offset << 5);
606}
607
608static inline void tcg_out_callr(TCGContext *s, TCGReg reg)
609{
610 tcg_out32(s, 0xd63f0000 | reg << 5);
611}
612
613static inline void tcg_out_gotor(TCGContext *s, TCGReg reg)
614{
615 tcg_out32(s, 0xd61f0000 | reg << 5);
616}
617
618static inline void tcg_out_call(TCGContext *s, tcg_target_long target)
619{
620 tcg_target_long offset;
621
622 offset = (target - (tcg_target_long)s->code_ptr) / 4;
623
624 if (offset < -0x02000000 || offset >= 0x02000000) { /* out of 26bit rng */
625 tcg_out_movi(s, TCG_TYPE_I64, TCG_REG_TMP, target);
626 tcg_out_callr(s, TCG_REG_TMP);
627 } else {
628 tcg_out32(s, 0x94000000 | (offset & 0x03ffffff));
629 }
630}
631
7deea126
CF		 632/* encode a logical immediate, mapping user parameter
633 M=set bits pattern length to S=M-1 */
634static inline unsigned int
635aarch64_limm(unsigned int m, unsigned int r)
636{
637 assert(m > 0);
638 return r << 16 | (m - 1) << 10;
639}
640
641/* test a register against an immediate bit pattern made of
642 M set bits rotated right by R.
643 Examples:
644 to test a 32/64 reg against 0x00000007, pass M = 3, R = 0.
645 to test a 32/64 reg against 0x000000ff, pass M = 8, R = 0.
646 to test a 32bit reg against 0xff000000, pass M = 8, R = 8.
647 to test a 32bit reg against 0xff0000ff, pass M = 16, R = 8.
648 */
7763ffa0 649static inline void tcg_out_tst(TCGContext *s, TCGType ext, TCGReg rn,
7deea126
CF		 650 unsigned int m, unsigned int r)
651{
652 /* using TST alias of ANDS XZR, Xn,#bimm64 0x7200001f */
653 unsigned int base = ext ? 0xf240001f : 0x7200001f;
654 tcg_out32(s, base | aarch64_limm(m, r) | rn << 5);
655}
656
657/* and a register with a bit pattern, similarly to TST, no flags change */
7763ffa0
RH		 658static inline void tcg_out_andi(TCGContext *s, TCGType ext, TCGReg rd,
659 TCGReg rn, unsigned int m, unsigned int r)
7deea126
CF		 660{
661 /* using AND 0x12000000 */
662 unsigned int base = ext ? 0x92400000 : 0x12000000;
663 tcg_out32(s, base | aarch64_limm(m, r) | rn << 5 | rd);
664}
665
4a136e0a
CF		 666static inline void tcg_out_ret(TCGContext *s)
667{
668 /* emit RET { LR } */
669 tcg_out32(s, 0xd65f03c0);
670}
671
672void aarch64_tb_set_jmp_target(uintptr_t jmp_addr, uintptr_t addr)
673{
674 tcg_target_long target, offset;
675 target = (tcg_target_long)addr;
676 offset = (target - (tcg_target_long)jmp_addr) / 4;
677
678 if (offset < -0x02000000 || offset >= 0x02000000) {
679 /* out of 26bit range */
680 tcg_abort();
681 }
682
683 patch_reloc((uint8_t *)jmp_addr, R_AARCH64_JUMP26, target, 0);
684 flush_icache_range(jmp_addr, jmp_addr + 4);
685}
686
687static inline void tcg_out_goto_label(TCGContext *s, int label_index)
688{
689 TCGLabel *l = &s->labels[label_index];
690
691 if (!l->has_value) {
692 tcg_out_reloc(s, s->code_ptr, R_AARCH64_JUMP26, label_index, 0);
693 tcg_out_goto_noaddr(s);
694 } else {
695 tcg_out_goto(s, l->u.value);
696 }
697}
698
699static inline void tcg_out_goto_label_cond(TCGContext *s,
700 TCGCond c, int label_index)
701{
702 TCGLabel *l = &s->labels[label_index];
703
704 if (!l->has_value) {
705 tcg_out_reloc(s, s->code_ptr, R_AARCH64_CONDBR19, label_index, 0);
706 tcg_out_goto_cond_noaddr(s, c);
707 } else {
708 tcg_out_goto_cond(s, c, l->u.value);
709 }
710}
711
7763ffa0
RH		 712static inline void tcg_out_rev(TCGContext *s, TCGType ext,
713 TCGReg rd, TCGReg rm)
9c4a059d
CF		 714{
715 /* using REV 0x5ac00800 */
716 unsigned int base = ext ? 0xdac00c00 : 0x5ac00800;
717 tcg_out32(s, base | rm << 5 | rd);
718}
719
7763ffa0
RH		 720static inline void tcg_out_rev16(TCGContext *s, TCGType ext,
721 TCGReg rd, TCGReg rm)
9c4a059d
CF		 722{
723 /* using REV16 0x5ac00400 */
724 unsigned int base = ext ? 0xdac00400 : 0x5ac00400;
725 tcg_out32(s, base | rm << 5 | rd);
726}
727
7763ffa0 728static inline void tcg_out_sxt(TCGContext *s, TCGType ext, int s_bits,
31f1275b
CF		 729 TCGReg rd, TCGReg rn)
730{
731 /* using ALIASes SXTB 0x13001c00, SXTH 0x13003c00, SXTW 0x93407c00
732 of SBFM Xd, Xn, #0, #7|15|31 */
733 int bits = 8 * (1 << s_bits) - 1;
734 tcg_out_sbfm(s, ext, rd, rn, 0, bits);
735}
736
737static inline void tcg_out_uxt(TCGContext *s, int s_bits,
738 TCGReg rd, TCGReg rn)
739{
740 /* using ALIASes UXTB 0x53001c00, UXTH 0x53003c00
741 of UBFM Wd, Wn, #0, #7|15 */
742 int bits = 8 * (1 << s_bits) - 1;
743 tcg_out_ubfm(s, 0, rd, rn, 0, bits);
744}
745
7763ffa0 746static inline void tcg_out_addi(TCGContext *s, TCGType ext,
c6d8ed24
JK		 747 TCGReg rd, TCGReg rn, unsigned int aimm)
748{
749 /* add immediate aimm unsigned 12bit value (with LSL 0 or 12) */
750 /* using ADD 0x11000000 | (ext) | (aimm << 10) | (rn << 5) | rd */
751 unsigned int base = ext ? 0x91000000 : 0x11000000;
752
753 if (aimm <= 0xfff) {
754 aimm <<= 10;
755 } else {
756 /* we can only shift left by 12, on assert we cannot represent */
757 assert(!(aimm & 0xfff));
758 assert(aimm <= 0xfff000);
759 base |= 1 << 22; /* apply LSL 12 */
760 aimm >>= 2;
761 }
762
763 tcg_out32(s, base | aimm | (rn << 5) | rd);
764}
765
7763ffa0 766static inline void tcg_out_subi(TCGContext *s, TCGType ext,
c6d8ed24
JK		 767 TCGReg rd, TCGReg rn, unsigned int aimm)
768{
769 /* sub immediate aimm unsigned 12bit value (with LSL 0 or 12) */
770 /* using SUB 0x51000000 | (ext) | (aimm << 10) | (rn << 5) | rd */
771 unsigned int base = ext ? 0xd1000000 : 0x51000000;
772
773 if (aimm <= 0xfff) {
774 aimm <<= 10;
775 } else {
776 /* we can only shift left by 12, on assert we cannot represent */
777 assert(!(aimm & 0xfff));
778 assert(aimm <= 0xfff000);
779 base |= 1 << 22; /* apply LSL 12 */
780 aimm >>= 2;
781 }
782
783 tcg_out32(s, base | aimm | (rn << 5) | rd);
784}
785
786static inline void tcg_out_nop(TCGContext *s)
787{
788 tcg_out32(s, 0xd503201f);
789}
790
4a136e0a 791#ifdef CONFIG_SOFTMMU
023261ef
RH		 792/* helper signature: helper_ret_ld_mmu(CPUState *env, target_ulong addr,
793 * int mmu_idx, uintptr_t ra)
794 */
4a136e0a 795static const void * const qemu_ld_helpers[4] = {
023261ef
RH		 796 helper_ret_ldub_mmu,
797 helper_ret_lduw_mmu,
798 helper_ret_ldul_mmu,
799 helper_ret_ldq_mmu,
4a136e0a
CF		 800};
801
023261ef
RH		 802/* helper signature: helper_ret_st_mmu(CPUState *env, target_ulong addr,
803 * uintxx_t val, int mmu_idx, uintptr_t ra)
804 */
4a136e0a 805static const void * const qemu_st_helpers[4] = {
023261ef
RH		 806 helper_ret_stb_mmu,
807 helper_ret_stw_mmu,
808 helper_ret_stl_mmu,
809 helper_ret_stq_mmu,
4a136e0a
CF		 810};
811
c6d8ed24
JK		 812static void tcg_out_qemu_ld_slow_path(TCGContext *s, TCGLabelQemuLdst *lb)
813{
814 reloc_pc19(lb->label_ptr[0], (tcg_target_long)s->code_ptr);
815 tcg_out_movr(s, 1, TCG_REG_X0, TCG_AREG0);
816 tcg_out_movr(s, (TARGET_LONG_BITS == 64), TCG_REG_X1, lb->addrlo_reg);
817 tcg_out_movi(s, TCG_TYPE_I32, TCG_REG_X2, lb->mem_index);
023261ef 818 tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_X3, (tcg_target_long)lb->raddr);
c6d8ed24
JK		 819 tcg_out_movi(s, TCG_TYPE_I64, TCG_REG_TMP,
820 (tcg_target_long)qemu_ld_helpers[lb->opc & 3]);
821 tcg_out_callr(s, TCG_REG_TMP);
822 if (lb->opc & 0x04) {
823 tcg_out_sxt(s, 1, lb->opc & 3, lb->datalo_reg, TCG_REG_X0);
824 } else {
825 tcg_out_movr(s, 1, lb->datalo_reg, TCG_REG_X0);
826 }
827
828 tcg_out_goto(s, (tcg_target_long)lb->raddr);
829}
830
831static void tcg_out_qemu_st_slow_path(TCGContext *s, TCGLabelQemuLdst *lb)
832{
833 reloc_pc19(lb->label_ptr[0], (tcg_target_long)s->code_ptr);
834
835 tcg_out_movr(s, 1, TCG_REG_X0, TCG_AREG0);
836 tcg_out_movr(s, (TARGET_LONG_BITS == 64), TCG_REG_X1, lb->addrlo_reg);
837 tcg_out_movr(s, 1, TCG_REG_X2, lb->datalo_reg);
838 tcg_out_movi(s, TCG_TYPE_I32, TCG_REG_X3, lb->mem_index);
023261ef 839 tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_X4, (tcg_target_long)lb->raddr);
c6d8ed24
JK		 840 tcg_out_movi(s, TCG_TYPE_I64, TCG_REG_TMP,
841 (tcg_target_long)qemu_st_helpers[lb->opc & 3]);
842 tcg_out_callr(s, TCG_REG_TMP);
843
844 tcg_out_nop(s);
845 tcg_out_goto(s, (tcg_target_long)lb->raddr);
846}
847
c6d8ed24
JK		 848static void add_qemu_ldst_label(TCGContext *s, int is_ld, int opc,
849 TCGReg data_reg, TCGReg addr_reg,
850 int mem_index,
851 uint8_t *raddr, uint8_t *label_ptr)
852{
9ecefc84 853 TCGLabelQemuLdst *label = new_ldst_label(s);
c6d8ed24 854
c6d8ed24
JK		 855 label->is_ld = is_ld;
856 label->opc = opc;
857 label->datalo_reg = data_reg;
858 label->addrlo_reg = addr_reg;
859 label->mem_index = mem_index;
860 label->raddr = raddr;
861 label->label_ptr[0] = label_ptr;
862}
863
864/* Load and compare a TLB entry, emitting the conditional jump to the
865 slow path for the failure case, which will be patched later when finalizing
866 the slow path. Generated code returns the host addend in X1,
867 clobbers X0,X2,X3,TMP. */
868static void tcg_out_tlb_read(TCGContext *s, TCGReg addr_reg,
869 int s_bits, uint8_t **label_ptr, int mem_index, int is_read)
870{
871 TCGReg base = TCG_AREG0;
872 int tlb_offset = is_read ?
873 offsetof(CPUArchState, tlb_table[mem_index][0].addr_read)
874 : offsetof(CPUArchState, tlb_table[mem_index][0].addr_write);
875 /* Extract the TLB index from the address into X0.
876 X0<CPU_TLB_BITS:0> =
877 addr_reg<TARGET_PAGE_BITS+CPU_TLB_BITS:TARGET_PAGE_BITS> */
878 tcg_out_ubfm(s, (TARGET_LONG_BITS == 64), TCG_REG_X0, addr_reg,
879 TARGET_PAGE_BITS, TARGET_PAGE_BITS + CPU_TLB_BITS);
880 /* Store the page mask part of the address and the low s_bits into X3.
881 Later this allows checking for equality and alignment at the same time.
882 X3 = addr_reg & (PAGE_MASK | ((1 << s_bits) - 1)) */
883 tcg_out_andi(s, (TARGET_LONG_BITS == 64), TCG_REG_X3, addr_reg,
884 (TARGET_LONG_BITS - TARGET_PAGE_BITS) + s_bits,
885 (TARGET_LONG_BITS - TARGET_PAGE_BITS));
886 /* Add any "high bits" from the tlb offset to the env address into X2,
887 to take advantage of the LSL12 form of the addi instruction.
888 X2 = env + (tlb_offset & 0xfff000) */
889 tcg_out_addi(s, 1, TCG_REG_X2, base, tlb_offset & 0xfff000);
890 /* Merge the tlb index contribution into X2.
891 X2 = X2 + (X0 << CPU_TLB_ENTRY_BITS) */
892 tcg_out_arith(s, ARITH_ADD, 1, TCG_REG_X2, TCG_REG_X2,
893 TCG_REG_X0, -CPU_TLB_ENTRY_BITS);
894 /* Merge "low bits" from tlb offset, load the tlb comparator into X0.
895 X0 = load [X2 + (tlb_offset & 0x000fff)] */
896 tcg_out_ldst(s, TARGET_LONG_BITS == 64 ? LDST_64 : LDST_32,
897 LDST_LD, TCG_REG_X0, TCG_REG_X2,
898 (tlb_offset & 0xfff));
899 /* Load the tlb addend. Do that early to avoid stalling.
900 X1 = load [X2 + (tlb_offset & 0xfff) + offsetof(addend)] */
901 tcg_out_ldst(s, LDST_64, LDST_LD, TCG_REG_X1, TCG_REG_X2,
902 (tlb_offset & 0xfff) + (offsetof(CPUTLBEntry, addend)) -
903 (is_read ? offsetof(CPUTLBEntry, addr_read)
904 : offsetof(CPUTLBEntry, addr_write)));
905 /* Perform the address comparison. */
906 tcg_out_cmp(s, (TARGET_LONG_BITS == 64), TCG_REG_X0, TCG_REG_X3, 0);
907 *label_ptr = s->code_ptr;
908 /* If not equal, we jump to the slow path. */
909 tcg_out_goto_cond_noaddr(s, TCG_COND_NE);
910}
911
912#endif /* CONFIG_SOFTMMU */
6a91c7c9
JK		 913
914static void tcg_out_qemu_ld_direct(TCGContext *s, int opc, TCGReg data_r,
915 TCGReg addr_r, TCGReg off_r)
916{
917 switch (opc) {
918 case 0:
919 tcg_out_ldst_r(s, LDST_8, LDST_LD, data_r, addr_r, off_r);
920 break;
921 case 0 | 4:
922 tcg_out_ldst_r(s, LDST_8, LDST_LD_S_X, data_r, addr_r, off_r);
923 break;
924 case 1:
925 tcg_out_ldst_r(s, LDST_16, LDST_LD, data_r, addr_r, off_r);
926 if (TCG_LDST_BSWAP) {
927 tcg_out_rev16(s, 0, data_r, data_r);
928 }
929 break;
930 case 1 | 4:
931 if (TCG_LDST_BSWAP) {
932 tcg_out_ldst_r(s, LDST_16, LDST_LD, data_r, addr_r, off_r);
933 tcg_out_rev16(s, 0, data_r, data_r);
934 tcg_out_sxt(s, 1, 1, data_r, data_r);
935 } else {
936 tcg_out_ldst_r(s, LDST_16, LDST_LD_S_X, data_r, addr_r, off_r);
937 }
938 break;
939 case 2:
940 tcg_out_ldst_r(s, LDST_32, LDST_LD, data_r, addr_r, off_r);
941 if (TCG_LDST_BSWAP) {
942 tcg_out_rev(s, 0, data_r, data_r);
943 }
944 break;
945 case 2 | 4:
946 if (TCG_LDST_BSWAP) {
947 tcg_out_ldst_r(s, LDST_32, LDST_LD, data_r, addr_r, off_r);
948 tcg_out_rev(s, 0, data_r, data_r);
949 tcg_out_sxt(s, 1, 2, data_r, data_r);
950 } else {
951 tcg_out_ldst_r(s, LDST_32, LDST_LD_S_X, data_r, addr_r, off_r);
952 }
953 break;
954 case 3:
955 tcg_out_ldst_r(s, LDST_64, LDST_LD, data_r, addr_r, off_r);
956 if (TCG_LDST_BSWAP) {
957 tcg_out_rev(s, 1, data_r, data_r);
958 }
959 break;
960 default:
961 tcg_abort();
962 }
963}
964
965static void tcg_out_qemu_st_direct(TCGContext *s, int opc, TCGReg data_r,
966 TCGReg addr_r, TCGReg off_r)
967{
968 switch (opc) {
969 case 0:
970 tcg_out_ldst_r(s, LDST_8, LDST_ST, data_r, addr_r, off_r);
971 break;
972 case 1:
973 if (TCG_LDST_BSWAP) {
974 tcg_out_rev16(s, 0, TCG_REG_TMP, data_r);
975 tcg_out_ldst_r(s, LDST_16, LDST_ST, TCG_REG_TMP, addr_r, off_r);
976 } else {
977 tcg_out_ldst_r(s, LDST_16, LDST_ST, data_r, addr_r, off_r);
978 }
979 break;
980 case 2:
981 if (TCG_LDST_BSWAP) {
982 tcg_out_rev(s, 0, TCG_REG_TMP, data_r);
983 tcg_out_ldst_r(s, LDST_32, LDST_ST, TCG_REG_TMP, addr_r, off_r);
984 } else {
985 tcg_out_ldst_r(s, LDST_32, LDST_ST, data_r, addr_r, off_r);
986 }
987 break;
988 case 3:
989 if (TCG_LDST_BSWAP) {
990 tcg_out_rev(s, 1, TCG_REG_TMP, data_r);
991 tcg_out_ldst_r(s, LDST_64, LDST_ST, TCG_REG_TMP, addr_r, off_r);
992 } else {
993 tcg_out_ldst_r(s, LDST_64, LDST_ST, data_r, addr_r, off_r);
994 }
995 break;
996 default:
997 tcg_abort();
998 }
999}
4a136e0a
CF		 1000
1001static void tcg_out_qemu_ld(TCGContext *s, const TCGArg *args, int opc)
1002{
1003 TCGReg addr_reg, data_reg;
1004#ifdef CONFIG_SOFTMMU
1005 int mem_index, s_bits;
c6d8ed24 1006 uint8_t *label_ptr;
4a136e0a
CF		 1007#endif
1008 data_reg = args[0];
1009 addr_reg = args[1];
1010
1011#ifdef CONFIG_SOFTMMU
1012 mem_index = args[2];
1013 s_bits = opc & 3;
c6d8ed24
JK		 1014 tcg_out_tlb_read(s, addr_reg, s_bits, &label_ptr, mem_index, 1);
1015 tcg_out_qemu_ld_direct(s, opc, data_reg, addr_reg, TCG_REG_X1);
1016 add_qemu_ldst_label(s, 1, opc, data_reg, addr_reg,
1017 mem_index, s->code_ptr, label_ptr);
4a136e0a 1018#else /* !CONFIG_SOFTMMU */
6a91c7c9
JK		 1019 tcg_out_qemu_ld_direct(s, opc, data_reg, addr_reg,
1020 GUEST_BASE ? TCG_REG_GUEST_BASE : TCG_REG_XZR);
1021#endif /* CONFIG_SOFTMMU */
4a136e0a
CF		 1022}
1023
1024static void tcg_out_qemu_st(TCGContext *s, const TCGArg *args, int opc)
1025{
1026 TCGReg addr_reg, data_reg;
1027#ifdef CONFIG_SOFTMMU
1028 int mem_index, s_bits;
c6d8ed24 1029 uint8_t *label_ptr;
4a136e0a
CF		 1030#endif
1031 data_reg = args[0];
1032 addr_reg = args[1];
1033
1034#ifdef CONFIG_SOFTMMU
1035 mem_index = args[2];
1036 s_bits = opc & 3;
1037
c6d8ed24
JK		 1038 tcg_out_tlb_read(s, addr_reg, s_bits, &label_ptr, mem_index, 0);
1039 tcg_out_qemu_st_direct(s, opc, data_reg, addr_reg, TCG_REG_X1);
1040 add_qemu_ldst_label(s, 0, opc, data_reg, addr_reg,
1041 mem_index, s->code_ptr, label_ptr);
4a136e0a 1042#else /* !CONFIG_SOFTMMU */
6a91c7c9
JK		 1043 tcg_out_qemu_st_direct(s, opc, data_reg, addr_reg,
1044 GUEST_BASE ? TCG_REG_GUEST_BASE : TCG_REG_XZR);
1045#endif /* CONFIG_SOFTMMU */
4a136e0a
CF		 1046}
1047
1048static uint8_t *tb_ret_addr;
1049
1050/* callee stack use example:
1051 stp x29, x30, [sp,#-32]!
1052 mov x29, sp
1053 stp x1, x2, [sp,#16]
1054 ...
1055 ldp x1, x2, [sp,#16]
1056 ldp x29, x30, [sp],#32
1057 ret
1058*/
1059
1060/* push r1 and r2, and alloc stack space for a total of
1061 alloc_n elements (1 element=16 bytes, must be between 1 and 31. */
1062static inline void tcg_out_push_pair(TCGContext *s, TCGReg addr,
1063 TCGReg r1, TCGReg r2, int alloc_n)
1064{
1065 /* using indexed scaled simm7 STP 0x28800000 | (ext) | 0x01000000 (pre-idx)
1066 | alloc_n * (-1) << 16 | r2 << 10 | addr << 5 | r1 */
1067 assert(alloc_n > 0 && alloc_n < 0x20);
1068 alloc_n = (-alloc_n) & 0x3f;
1069 tcg_out32(s, 0xa9800000 | alloc_n << 16 | r2 << 10 | addr << 5 | r1);
1070}
1071
1072/* dealloc stack space for a total of alloc_n elements and pop r1, r2. */
1073static inline void tcg_out_pop_pair(TCGContext *s, TCGReg addr,
1074 TCGReg r1, TCGReg r2, int alloc_n)
1075{
1076 /* using indexed scaled simm7 LDP 0x28c00000 | (ext) | nothing (post-idx)
1077 | alloc_n << 16 | r2 << 10 | addr << 5 | r1 */
1078 assert(alloc_n > 0 && alloc_n < 0x20);
1079 tcg_out32(s, 0xa8c00000 | alloc_n << 16 | r2 << 10 | addr << 5 | r1);
1080}
1081
1082static inline void tcg_out_store_pair(TCGContext *s, TCGReg addr,
1083 TCGReg r1, TCGReg r2, int idx)
1084{
1085 /* using register pair offset simm7 STP 0x29000000 | (ext)
1086 | idx << 16 | r2 << 10 | addr << 5 | r1 */
1087 assert(idx > 0 && idx < 0x20);
1088 tcg_out32(s, 0xa9000000 | idx << 16 | r2 << 10 | addr << 5 | r1);
1089}
1090
1091static inline void tcg_out_load_pair(TCGContext *s, TCGReg addr,
1092 TCGReg r1, TCGReg r2, int idx)
1093{
1094 /* using register pair offset simm7 LDP 0x29400000 | (ext)
1095 | idx << 16 | r2 << 10 | addr << 5 | r1 */
1096 assert(idx > 0 && idx < 0x20);
1097 tcg_out32(s, 0xa9400000 | idx << 16 | r2 << 10 | addr << 5 | r1);
1098}
1099
1100static void tcg_out_op(TCGContext *s, TCGOpcode opc,
1101 const TCGArg *args, const int *const_args)
1102{
1103 /* ext will be set in the switch below, which will fall through to the
1104 common code. It triggers the use of extended regs where appropriate. */
7763ffa0 1105 TCGType ext = 0;
4a136e0a
CF		 1106
1107 switch (opc) {
1108 case INDEX_op_exit_tb:
1109 tcg_out_movi(s, TCG_TYPE_I64, TCG_REG_X0, args[0]);
1110 tcg_out_goto(s, (tcg_target_long)tb_ret_addr);
1111 break;
1112
1113 case INDEX_op_goto_tb:
1114#ifndef USE_DIRECT_JUMP
1115#error "USE_DIRECT_JUMP required for aarch64"
1116#endif
1117 assert(s->tb_jmp_offset != NULL); /* consistency for USE_DIRECT_JUMP */
1118 s->tb_jmp_offset[args[0]] = s->code_ptr - s->code_buf;
1119 /* actual branch destination will be patched by
1120 aarch64_tb_set_jmp_target later, beware retranslation. */
1121 tcg_out_goto_noaddr(s);
1122 s->tb_next_offset[args[0]] = s->code_ptr - s->code_buf;
1123 break;
1124
1125 case INDEX_op_call:
1126 if (const_args[0]) {
1127 tcg_out_call(s, args[0]);
1128 } else {
1129 tcg_out_callr(s, args[0]);
1130 }
1131 break;
1132
1133 case INDEX_op_br:
1134 tcg_out_goto_label(s, args[0]);
1135 break;
1136
1137 case INDEX_op_ld_i32:
1138 case INDEX_op_ld_i64:
1139 case INDEX_op_st_i32:
1140 case INDEX_op_st_i64:
1141 case INDEX_op_ld8u_i32:
1142 case INDEX_op_ld8s_i32:
1143 case INDEX_op_ld16u_i32:
1144 case INDEX_op_ld16s_i32:
1145 case INDEX_op_ld8u_i64:
1146 case INDEX_op_ld8s_i64:
1147 case INDEX_op_ld16u_i64:
1148 case INDEX_op_ld16s_i64:
1149 case INDEX_op_ld32u_i64:
1150 case INDEX_op_ld32s_i64:
1151 case INDEX_op_st8_i32:
1152 case INDEX_op_st8_i64:
1153 case INDEX_op_st16_i32:
1154 case INDEX_op_st16_i64:
1155 case INDEX_op_st32_i64:
1156 tcg_out_ldst(s, aarch64_ldst_get_data(opc), aarch64_ldst_get_type(opc),
1157 args[0], args[1], args[2]);
1158 break;
1159
1160 case INDEX_op_mov_i64:
1161 ext = 1; /* fall through */
1162 case INDEX_op_mov_i32:
1163 tcg_out_movr(s, ext, args[0], args[1]);
1164 break;
1165
1166 case INDEX_op_movi_i64:
1167 tcg_out_movi(s, TCG_TYPE_I64, args[0], args[1]);
1168 break;
1169 case INDEX_op_movi_i32:
1170 tcg_out_movi(s, TCG_TYPE_I32, args[0], args[1]);
1171 break;
1172
1173 case INDEX_op_add_i64:
1174 ext = 1; /* fall through */
1175 case INDEX_op_add_i32:
36fac14a 1176 tcg_out_arith(s, ARITH_ADD, ext, args[0], args[1], args[2], 0);
4a136e0a
CF		 1177 break;
1178
1179 case INDEX_op_sub_i64:
1180 ext = 1; /* fall through */
1181 case INDEX_op_sub_i32:
36fac14a 1182 tcg_out_arith(s, ARITH_SUB, ext, args[0], args[1], args[2], 0);
4a136e0a
CF		 1183 break;
1184
1185 case INDEX_op_and_i64:
1186 ext = 1; /* fall through */
1187 case INDEX_op_and_i32:
36fac14a 1188 tcg_out_arith(s, ARITH_AND, ext, args[0], args[1], args[2], 0);
4a136e0a
CF		 1189 break;
1190
1191 case INDEX_op_or_i64:
1192 ext = 1; /* fall through */
1193 case INDEX_op_or_i32:
36fac14a 1194 tcg_out_arith(s, ARITH_OR, ext, args[0], args[1], args[2], 0);
4a136e0a
CF		 1195 break;
1196
1197 case INDEX_op_xor_i64:
1198 ext = 1; /* fall through */
1199 case INDEX_op_xor_i32:
36fac14a 1200 tcg_out_arith(s, ARITH_XOR, ext, args[0], args[1], args[2], 0);
4a136e0a
CF		 1201 break;
1202
1203 case INDEX_op_mul_i64:
1204 ext = 1; /* fall through */
1205 case INDEX_op_mul_i32:
1206 tcg_out_mul(s, ext, args[0], args[1], args[2]);
1207 break;
1208
1209 case INDEX_op_shl_i64:
1210 ext = 1; /* fall through */
1211 case INDEX_op_shl_i32:
1212 if (const_args[2]) { /* LSL / UBFM Wd, Wn, (32 - m) */
1213 tcg_out_shl(s, ext, args[0], args[1], args[2]);
1214 } else { /* LSL / LSLV */
1215 tcg_out_shiftrot_reg(s, SRR_SHL, ext, args[0], args[1], args[2]);
1216 }
1217 break;
1218
1219 case INDEX_op_shr_i64:
1220 ext = 1; /* fall through */
1221 case INDEX_op_shr_i32:
1222 if (const_args[2]) { /* LSR / UBFM Wd, Wn, m, 31 */
1223 tcg_out_shr(s, ext, args[0], args[1], args[2]);
1224 } else { /* LSR / LSRV */
1225 tcg_out_shiftrot_reg(s, SRR_SHR, ext, args[0], args[1], args[2]);
1226 }
1227 break;
1228
1229 case INDEX_op_sar_i64:
1230 ext = 1; /* fall through */
1231 case INDEX_op_sar_i32:
1232 if (const_args[2]) { /* ASR / SBFM Wd, Wn, m, 31 */
1233 tcg_out_sar(s, ext, args[0], args[1], args[2]);
1234 } else { /* ASR / ASRV */
1235 tcg_out_shiftrot_reg(s, SRR_SAR, ext, args[0], args[1], args[2]);
1236 }
1237 break;
1238
1239 case INDEX_op_rotr_i64:
1240 ext = 1; /* fall through */
1241 case INDEX_op_rotr_i32:
1242 if (const_args[2]) { /* ROR / EXTR Wd, Wm, Wm, m */
1243 tcg_out_rotr(s, ext, args[0], args[1], args[2]);
1244 } else { /* ROR / RORV */
1245 tcg_out_shiftrot_reg(s, SRR_ROR, ext, args[0], args[1], args[2]);
1246 }
1247 break;
1248
1249 case INDEX_op_rotl_i64:
1250 ext = 1; /* fall through */
1251 case INDEX_op_rotl_i32: /* same as rotate right by (32 - m) */
1252 if (const_args[2]) { /* ROR / EXTR Wd, Wm, Wm, 32 - m */
1253 tcg_out_rotl(s, ext, args[0], args[1], args[2]);
1254 } else {
36fac14a
CF		 1255 tcg_out_arith(s, ARITH_SUB, 0,
1256 TCG_REG_TMP, TCG_REG_XZR, args[2], 0);
4a136e0a
CF		 1257 tcg_out_shiftrot_reg(s, SRR_ROR, ext,
1258 args[0], args[1], TCG_REG_TMP);
1259 }
1260 break;
1261
1262 case INDEX_op_brcond_i64:
1263 ext = 1; /* fall through */
1264 case INDEX_op_brcond_i32: /* CMP 0, 1, cond(2), label 3 */
36fac14a 1265 tcg_out_cmp(s, ext, args[0], args[1], 0);
4a136e0a
CF		 1266 tcg_out_goto_label_cond(s, args[2], args[3]);
1267 break;
1268
1269 case INDEX_op_setcond_i64:
1270 ext = 1; /* fall through */
1271 case INDEX_op_setcond_i32:
36fac14a 1272 tcg_out_cmp(s, ext, args[1], args[2], 0);
4a136e0a
CF		 1273 tcg_out_cset(s, 0, args[0], args[3]);
1274 break;
1275
1276 case INDEX_op_qemu_ld8u:
1277 tcg_out_qemu_ld(s, args, 0 | 0);
1278 break;
1279 case INDEX_op_qemu_ld8s:
1280 tcg_out_qemu_ld(s, args, 4 | 0);
1281 break;
1282 case INDEX_op_qemu_ld16u:
1283 tcg_out_qemu_ld(s, args, 0 | 1);
1284 break;
1285 case INDEX_op_qemu_ld16s:
1286 tcg_out_qemu_ld(s, args, 4 | 1);
1287 break;
1288 case INDEX_op_qemu_ld32u:
1289 tcg_out_qemu_ld(s, args, 0 | 2);
1290 break;
1291 case INDEX_op_qemu_ld32s:
1292 tcg_out_qemu_ld(s, args, 4 | 2);
1293 break;
1294 case INDEX_op_qemu_ld32:
1295 tcg_out_qemu_ld(s, args, 0 | 2);
1296 break;
1297 case INDEX_op_qemu_ld64:
1298 tcg_out_qemu_ld(s, args, 0 | 3);
1299 break;
1300 case INDEX_op_qemu_st8:
1301 tcg_out_qemu_st(s, args, 0);
1302 break;
1303 case INDEX_op_qemu_st16:
1304 tcg_out_qemu_st(s, args, 1);
1305 break;
1306 case INDEX_op_qemu_st32:
1307 tcg_out_qemu_st(s, args, 2);
1308 break;
1309 case INDEX_op_qemu_st64:
1310 tcg_out_qemu_st(s, args, 3);
1311 break;
1312
9c4a059d
CF		 1313 case INDEX_op_bswap64_i64:
1314 ext = 1; /* fall through */
1315 case INDEX_op_bswap32_i64:
1316 case INDEX_op_bswap32_i32:
1317 tcg_out_rev(s, ext, args[0], args[1]);
1318 break;
1319 case INDEX_op_bswap16_i64:
1320 case INDEX_op_bswap16_i32:
1321 tcg_out_rev16(s, 0, args[0], args[1]);
1322 break;
1323
31f1275b
CF		 1324 case INDEX_op_ext8s_i64:
1325 ext = 1; /* fall through */
1326 case INDEX_op_ext8s_i32:
1327 tcg_out_sxt(s, ext, 0, args[0], args[1]);
1328 break;
1329 case INDEX_op_ext16s_i64:
1330 ext = 1; /* fall through */
1331 case INDEX_op_ext16s_i32:
1332 tcg_out_sxt(s, ext, 1, args[0], args[1]);
1333 break;
1334 case INDEX_op_ext32s_i64:
1335 tcg_out_sxt(s, 1, 2, args[0], args[1]);
1336 break;
1337 case INDEX_op_ext8u_i64:
1338 case INDEX_op_ext8u_i32:
1339 tcg_out_uxt(s, 0, args[0], args[1]);
1340 break;
1341 case INDEX_op_ext16u_i64:
1342 case INDEX_op_ext16u_i32:
1343 tcg_out_uxt(s, 1, args[0], args[1]);
1344 break;
1345 case INDEX_op_ext32u_i64:
1346 tcg_out_movr(s, 0, args[0], args[1]);
1347 break;
1348
4a136e0a
CF		 1349 default:
1350 tcg_abort(); /* opcode not implemented */
1351 }
1352}
1353
1354static const TCGTargetOpDef aarch64_op_defs[] = {
1355 { INDEX_op_exit_tb, { } },
1356 { INDEX_op_goto_tb, { } },
1357 { INDEX_op_call, { "ri" } },
1358 { INDEX_op_br, { } },
1359
1360 { INDEX_op_mov_i32, { "r", "r" } },
1361 { INDEX_op_mov_i64, { "r", "r" } },
1362
1363 { INDEX_op_movi_i32, { "r" } },
1364 { INDEX_op_movi_i64, { "r" } },
1365
1366 { INDEX_op_ld8u_i32, { "r", "r" } },
1367 { INDEX_op_ld8s_i32, { "r", "r" } },
1368 { INDEX_op_ld16u_i32, { "r", "r" } },
1369 { INDEX_op_ld16s_i32, { "r", "r" } },
1370 { INDEX_op_ld_i32, { "r", "r" } },
1371 { INDEX_op_ld8u_i64, { "r", "r" } },
1372 { INDEX_op_ld8s_i64, { "r", "r" } },
1373 { INDEX_op_ld16u_i64, { "r", "r" } },
1374 { INDEX_op_ld16s_i64, { "r", "r" } },
1375 { INDEX_op_ld32u_i64, { "r", "r" } },
1376 { INDEX_op_ld32s_i64, { "r", "r" } },
1377 { INDEX_op_ld_i64, { "r", "r" } },
1378
1379 { INDEX_op_st8_i32, { "r", "r" } },
1380 { INDEX_op_st16_i32, { "r", "r" } },
1381 { INDEX_op_st_i32, { "r", "r" } },
1382 { INDEX_op_st8_i64, { "r", "r" } },
1383 { INDEX_op_st16_i64, { "r", "r" } },
1384 { INDEX_op_st32_i64, { "r", "r" } },
1385 { INDEX_op_st_i64, { "r", "r" } },
1386
1387 { INDEX_op_add_i32, { "r", "r", "r" } },
1388 { INDEX_op_add_i64, { "r", "r", "r" } },
1389 { INDEX_op_sub_i32, { "r", "r", "r" } },
1390 { INDEX_op_sub_i64, { "r", "r", "r" } },
1391 { INDEX_op_mul_i32, { "r", "r", "r" } },
1392 { INDEX_op_mul_i64, { "r", "r", "r" } },
1393 { INDEX_op_and_i32, { "r", "r", "r" } },
1394 { INDEX_op_and_i64, { "r", "r", "r" } },
1395 { INDEX_op_or_i32, { "r", "r", "r" } },
1396 { INDEX_op_or_i64, { "r", "r", "r" } },
1397 { INDEX_op_xor_i32, { "r", "r", "r" } },
1398 { INDEX_op_xor_i64, { "r", "r", "r" } },
1399
1400 { INDEX_op_shl_i32, { "r", "r", "ri" } },
1401 { INDEX_op_shr_i32, { "r", "r", "ri" } },
1402 { INDEX_op_sar_i32, { "r", "r", "ri" } },
1403 { INDEX_op_rotl_i32, { "r", "r", "ri" } },
1404 { INDEX_op_rotr_i32, { "r", "r", "ri" } },
1405 { INDEX_op_shl_i64, { "r", "r", "ri" } },
1406 { INDEX_op_shr_i64, { "r", "r", "ri" } },
1407 { INDEX_op_sar_i64, { "r", "r", "ri" } },
1408 { INDEX_op_rotl_i64, { "r", "r", "ri" } },
1409 { INDEX_op_rotr_i64, { "r", "r", "ri" } },
1410
1411 { INDEX_op_brcond_i32, { "r", "r" } },
1412 { INDEX_op_setcond_i32, { "r", "r", "r" } },
1413 { INDEX_op_brcond_i64, { "r", "r" } },
1414 { INDEX_op_setcond_i64, { "r", "r", "r" } },
1415
1416 { INDEX_op_qemu_ld8u, { "r", "l" } },
1417 { INDEX_op_qemu_ld8s, { "r", "l" } },
1418 { INDEX_op_qemu_ld16u, { "r", "l" } },
1419 { INDEX_op_qemu_ld16s, { "r", "l" } },
1420 { INDEX_op_qemu_ld32u, { "r", "l" } },
1421 { INDEX_op_qemu_ld32s, { "r", "l" } },
1422
1423 { INDEX_op_qemu_ld32, { "r", "l" } },
1424 { INDEX_op_qemu_ld64, { "r", "l" } },
1425
1426 { INDEX_op_qemu_st8, { "l", "l" } },
1427 { INDEX_op_qemu_st16, { "l", "l" } },
1428 { INDEX_op_qemu_st32, { "l", "l" } },
1429 { INDEX_op_qemu_st64, { "l", "l" } },
9c4a059d
CF		 1430
1431 { INDEX_op_bswap16_i32, { "r", "r" } },
1432 { INDEX_op_bswap32_i32, { "r", "r" } },
1433 { INDEX_op_bswap16_i64, { "r", "r" } },
1434 { INDEX_op_bswap32_i64, { "r", "r" } },
1435 { INDEX_op_bswap64_i64, { "r", "r" } },
1436
31f1275b
CF		 1437 { INDEX_op_ext8s_i32, { "r", "r" } },
1438 { INDEX_op_ext16s_i32, { "r", "r" } },
1439 { INDEX_op_ext8u_i32, { "r", "r" } },
1440 { INDEX_op_ext16u_i32, { "r", "r" } },
1441
1442 { INDEX_op_ext8s_i64, { "r", "r" } },
1443 { INDEX_op_ext16s_i64, { "r", "r" } },
1444 { INDEX_op_ext32s_i64, { "r", "r" } },
1445 { INDEX_op_ext8u_i64, { "r", "r" } },
1446 { INDEX_op_ext16u_i64, { "r", "r" } },
1447 { INDEX_op_ext32u_i64, { "r", "r" } },
1448
4a136e0a
CF		 1449 { -1 },
1450};
1451
1452static void tcg_target_init(TCGContext *s)
1453{
4a136e0a
CF		 1454 tcg_regset_set32(tcg_target_available_regs[TCG_TYPE_I32], 0, 0xffffffff);
1455 tcg_regset_set32(tcg_target_available_regs[TCG_TYPE_I64], 0, 0xffffffff);
1456
1457 tcg_regset_set32(tcg_target_call_clobber_regs, 0,
1458 (1 << TCG_REG_X0) | (1 << TCG_REG_X1) |
1459 (1 << TCG_REG_X2) | (1 << TCG_REG_X3) |
1460 (1 << TCG_REG_X4) | (1 << TCG_REG_X5) |
1461 (1 << TCG_REG_X6) | (1 << TCG_REG_X7) |
1462 (1 << TCG_REG_X8) | (1 << TCG_REG_X9) |
1463 (1 << TCG_REG_X10) | (1 << TCG_REG_X11) |
1464 (1 << TCG_REG_X12) | (1 << TCG_REG_X13) |
1465 (1 << TCG_REG_X14) | (1 << TCG_REG_X15) |
1466 (1 << TCG_REG_X16) | (1 << TCG_REG_X17) |
1467 (1 << TCG_REG_X18));
1468
1469 tcg_regset_clear(s->reserved_regs);
1470 tcg_regset_set_reg(s->reserved_regs, TCG_REG_SP);
1471 tcg_regset_set_reg(s->reserved_regs, TCG_REG_FP);
1472 tcg_regset_set_reg(s->reserved_regs, TCG_REG_TMP);
1473 tcg_regset_set_reg(s->reserved_regs, TCG_REG_X18); /* platform register */
1474
1475 tcg_add_target_add_op_defs(aarch64_op_defs);
1476}
1477
4a136e0a
CF		 1478static void tcg_target_qemu_prologue(TCGContext *s)
1479{
1480 /* NB: frame sizes are in 16 byte stack units! */
1481 int frame_size_callee_saved, frame_size_tcg_locals;
1482 TCGReg r;
1483
1484 /* save pairs (FP, LR) and (X19, X20) .. (X27, X28) */
1485 frame_size_callee_saved = (1) + (TCG_REG_X28 - TCG_REG_X19) / 2 + 1;
1486
1487 /* frame size requirement for TCG local variables */
1488 frame_size_tcg_locals = TCG_STATIC_CALL_ARGS_SIZE
1489 + CPU_TEMP_BUF_NLONGS * sizeof(long)
1490 + (TCG_TARGET_STACK_ALIGN - 1);
1491 frame_size_tcg_locals &= ~(TCG_TARGET_STACK_ALIGN - 1);
1492 frame_size_tcg_locals /= TCG_TARGET_STACK_ALIGN;
1493
1494 /* push (FP, LR) and update sp */
1495 tcg_out_push_pair(s, TCG_REG_SP,
1496 TCG_REG_FP, TCG_REG_LR, frame_size_callee_saved);
1497
1498 /* FP -> callee_saved */
1499 tcg_out_movr_sp(s, 1, TCG_REG_FP, TCG_REG_SP);
1500
1501 /* store callee-preserved regs x19..x28 using FP -> callee_saved */
1502 for (r = TCG_REG_X19; r <= TCG_REG_X27; r += 2) {
1503 int idx = (r - TCG_REG_X19) / 2 + 1;
1504 tcg_out_store_pair(s, TCG_REG_FP, r, r + 1, idx);
1505 }
1506
1507 /* make stack space for TCG locals */
1508 tcg_out_subi(s, 1, TCG_REG_SP, TCG_REG_SP,
1509 frame_size_tcg_locals * TCG_TARGET_STACK_ALIGN);
1510 /* inform TCG about how to find TCG locals with register, offset, size */
1511 tcg_set_frame(s, TCG_REG_SP, TCG_STATIC_CALL_ARGS_SIZE,
1512 CPU_TEMP_BUF_NLONGS * sizeof(long));
1513
6a91c7c9
JK		 1514#if defined(CONFIG_USE_GUEST_BASE)
1515 if (GUEST_BASE) {
1516 tcg_out_movi(s, TCG_TYPE_PTR, TCG_REG_GUEST_BASE, GUEST_BASE);
1517 tcg_regset_set_reg(s->reserved_regs, TCG_REG_GUEST_BASE);
1518 }
1519#endif
1520
4a136e0a
CF		 1521 tcg_out_mov(s, TCG_TYPE_PTR, TCG_AREG0, tcg_target_call_iarg_regs[0]);
1522 tcg_out_gotor(s, tcg_target_call_iarg_regs[1]);
1523
1524 tb_ret_addr = s->code_ptr;
1525
1526 /* remove TCG locals stack space */
1527 tcg_out_addi(s, 1, TCG_REG_SP, TCG_REG_SP,
1528 frame_size_tcg_locals * TCG_TARGET_STACK_ALIGN);
1529
1530 /* restore registers x19..x28.
1531 FP must be preserved, so it still points to callee_saved area */
1532 for (r = TCG_REG_X19; r <= TCG_REG_X27; r += 2) {
1533 int idx = (r - TCG_REG_X19) / 2 + 1;
1534 tcg_out_load_pair(s, TCG_REG_FP, r, r + 1, idx);
1535 }
1536
1537 /* pop (FP, LR), restore SP to previous frame, return */
1538 tcg_out_pop_pair(s, TCG_REG_SP,
1539 TCG_REG_FP, TCG_REG_LR, frame_size_callee_saved);
1540 tcg_out_ret(s);
1541}
Empty description
This page took 0.293047 seconds and 4 git commands to generate.