]> Git Repo - qemu.git/blame - tcg/tcg-op-gvec.c
projects / qemu.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Include qemu/main-loop.h less
[qemu.git] / tcg / tcg-op-gvec.c
CommitLineData
db432672
RH		 1/*
2 * Generic vector operation expansion
3 *
4 * Copyright (c) 2018 Linaro
5 *
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
fb0343d5 9 * version 2.1 of the License, or (at your option) any later version.
db432672
RH		 10 *
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
15 *
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
18 */
19
20#include "qemu/osdep.h"
db432672
RH		 21#include "tcg.h"
22#include "tcg-op.h"
23#include "tcg-op-gvec.h"
db725815 24#include "qemu/main-loop.h"
db432672
RH		 25#include "tcg-gvec-desc.h"
26
27#define MAX_UNROLL 4
28
53229a77
RH		 29#ifdef CONFIG_DEBUG_TCG
30static const TCGOpcode vecop_list_empty[1] = { 0 };
31#else
32#define vecop_list_empty NULL
33#endif
34
35
db432672
RH		 36/* Verify vector size and alignment rules. OFS should be the OR of all
37 of the operand offsets so that we can check them all at once. */
38static void check_size_align(uint32_t oprsz, uint32_t maxsz, uint32_t ofs)
39{
40 uint32_t opr_align = oprsz >= 16 ? 15 : 7;
41 uint32_t max_align = maxsz >= 16 || oprsz >= 16 ? 15 : 7;
42 tcg_debug_assert(oprsz > 0);
43 tcg_debug_assert(oprsz <= maxsz);
44 tcg_debug_assert((oprsz & opr_align) == 0);
45 tcg_debug_assert((maxsz & max_align) == 0);
46 tcg_debug_assert((ofs & max_align) == 0);
47}
48
49/* Verify vector overlap rules for two operands. */
50static void check_overlap_2(uint32_t d, uint32_t a, uint32_t s)
51{
52 tcg_debug_assert(d == a || d + s <= a || a + s <= d);
53}
54
55/* Verify vector overlap rules for three operands. */
56static void check_overlap_3(uint32_t d, uint32_t a, uint32_t b, uint32_t s)
57{
58 check_overlap_2(d, a, s);
59 check_overlap_2(d, b, s);
60 check_overlap_2(a, b, s);
61}
62
63/* Verify vector overlap rules for four operands. */
64static void check_overlap_4(uint32_t d, uint32_t a, uint32_t b,
65 uint32_t c, uint32_t s)
66{
67 check_overlap_2(d, a, s);
68 check_overlap_2(d, b, s);
69 check_overlap_2(d, c, s);
70 check_overlap_2(a, b, s);
71 check_overlap_2(a, c, s);
72 check_overlap_2(b, c, s);
73}
74
75/* Create a descriptor from components. */
76uint32_t simd_desc(uint32_t oprsz, uint32_t maxsz, int32_t data)
77{
78 uint32_t desc = 0;
79
80 assert(oprsz % 8 == 0 && oprsz <= (8 << SIMD_OPRSZ_BITS));
81 assert(maxsz % 8 == 0 && maxsz <= (8 << SIMD_MAXSZ_BITS));
82 assert(data == sextract32(data, 0, SIMD_DATA_BITS));
83
84 oprsz = (oprsz / 8) - 1;
85 maxsz = (maxsz / 8) - 1;
86 desc = deposit32(desc, SIMD_OPRSZ_SHIFT, SIMD_OPRSZ_BITS, oprsz);
87 desc = deposit32(desc, SIMD_MAXSZ_SHIFT, SIMD_MAXSZ_BITS, maxsz);
88 desc = deposit32(desc, SIMD_DATA_SHIFT, SIMD_DATA_BITS, data);
89
90 return desc;
91}
92
93/* Generate a call to a gvec-style helper with two vector operands. */
94void tcg_gen_gvec_2_ool(uint32_t dofs, uint32_t aofs,
95 uint32_t oprsz, uint32_t maxsz, int32_t data,
96 gen_helper_gvec_2 *fn)
97{
98 TCGv_ptr a0, a1;
99 TCGv_i32 desc = tcg_const_i32(simd_desc(oprsz, maxsz, data));
100
101 a0 = tcg_temp_new_ptr();
102 a1 = tcg_temp_new_ptr();
103
104 tcg_gen_addi_ptr(a0, cpu_env, dofs);
105 tcg_gen_addi_ptr(a1, cpu_env, aofs);
106
107 fn(a0, a1, desc);
108
109 tcg_temp_free_ptr(a0);
110 tcg_temp_free_ptr(a1);
111 tcg_temp_free_i32(desc);
112}
113
22fc3527
RH		 114/* Generate a call to a gvec-style helper with two vector operands
115 and one scalar operand. */
116void tcg_gen_gvec_2i_ool(uint32_t dofs, uint32_t aofs, TCGv_i64 c,
117 uint32_t oprsz, uint32_t maxsz, int32_t data,
118 gen_helper_gvec_2i *fn)
119{
120 TCGv_ptr a0, a1;
121 TCGv_i32 desc = tcg_const_i32(simd_desc(oprsz, maxsz, data));
122
123 a0 = tcg_temp_new_ptr();
124 a1 = tcg_temp_new_ptr();
125
126 tcg_gen_addi_ptr(a0, cpu_env, dofs);
127 tcg_gen_addi_ptr(a1, cpu_env, aofs);
128
129 fn(a0, a1, c, desc);
130
131 tcg_temp_free_ptr(a0);
132 tcg_temp_free_ptr(a1);
133 tcg_temp_free_i32(desc);
134}
135
db432672
RH		 136/* Generate a call to a gvec-style helper with three vector operands. */
137void tcg_gen_gvec_3_ool(uint32_t dofs, uint32_t aofs, uint32_t bofs,
138 uint32_t oprsz, uint32_t maxsz, int32_t data,
139 gen_helper_gvec_3 *fn)
140{
141 TCGv_ptr a0, a1, a2;
142 TCGv_i32 desc = tcg_const_i32(simd_desc(oprsz, maxsz, data));
143
144 a0 = tcg_temp_new_ptr();
145 a1 = tcg_temp_new_ptr();
146 a2 = tcg_temp_new_ptr();
147
148 tcg_gen_addi_ptr(a0, cpu_env, dofs);
149 tcg_gen_addi_ptr(a1, cpu_env, aofs);
150 tcg_gen_addi_ptr(a2, cpu_env, bofs);
151
152 fn(a0, a1, a2, desc);
153
154 tcg_temp_free_ptr(a0);
155 tcg_temp_free_ptr(a1);
156 tcg_temp_free_ptr(a2);
157 tcg_temp_free_i32(desc);
158}
159
160/* Generate a call to a gvec-style helper with four vector operands. */
161void tcg_gen_gvec_4_ool(uint32_t dofs, uint32_t aofs, uint32_t bofs,
162 uint32_t cofs, uint32_t oprsz, uint32_t maxsz,
163 int32_t data, gen_helper_gvec_4 *fn)
164{
165 TCGv_ptr a0, a1, a2, a3;
166 TCGv_i32 desc = tcg_const_i32(simd_desc(oprsz, maxsz, data));
167
168 a0 = tcg_temp_new_ptr();
169 a1 = tcg_temp_new_ptr();
170 a2 = tcg_temp_new_ptr();
171 a3 = tcg_temp_new_ptr();
172
173 tcg_gen_addi_ptr(a0, cpu_env, dofs);
174 tcg_gen_addi_ptr(a1, cpu_env, aofs);
175 tcg_gen_addi_ptr(a2, cpu_env, bofs);
176 tcg_gen_addi_ptr(a3, cpu_env, cofs);
177
178 fn(a0, a1, a2, a3, desc);
179
180 tcg_temp_free_ptr(a0);
181 tcg_temp_free_ptr(a1);
182 tcg_temp_free_ptr(a2);
183 tcg_temp_free_ptr(a3);
184 tcg_temp_free_i32(desc);
185}
186
187/* Generate a call to a gvec-style helper with five vector operands. */
188void tcg_gen_gvec_5_ool(uint32_t dofs, uint32_t aofs, uint32_t bofs,
189 uint32_t cofs, uint32_t xofs, uint32_t oprsz,
190 uint32_t maxsz, int32_t data, gen_helper_gvec_5 *fn)
191{
192 TCGv_ptr a0, a1, a2, a3, a4;
193 TCGv_i32 desc = tcg_const_i32(simd_desc(oprsz, maxsz, data));
194
195 a0 = tcg_temp_new_ptr();
196 a1 = tcg_temp_new_ptr();
197 a2 = tcg_temp_new_ptr();
198 a3 = tcg_temp_new_ptr();
199 a4 = tcg_temp_new_ptr();
200
201 tcg_gen_addi_ptr(a0, cpu_env, dofs);
202 tcg_gen_addi_ptr(a1, cpu_env, aofs);
203 tcg_gen_addi_ptr(a2, cpu_env, bofs);
204 tcg_gen_addi_ptr(a3, cpu_env, cofs);
205 tcg_gen_addi_ptr(a4, cpu_env, xofs);
206
207 fn(a0, a1, a2, a3, a4, desc);
208
209 tcg_temp_free_ptr(a0);
210 tcg_temp_free_ptr(a1);
211 tcg_temp_free_ptr(a2);
212 tcg_temp_free_ptr(a3);
213 tcg_temp_free_ptr(a4);
214 tcg_temp_free_i32(desc);
215}
216
217/* Generate a call to a gvec-style helper with three vector operands
218 and an extra pointer operand. */
219void tcg_gen_gvec_2_ptr(uint32_t dofs, uint32_t aofs,
220 TCGv_ptr ptr, uint32_t oprsz, uint32_t maxsz,
221 int32_t data, gen_helper_gvec_2_ptr *fn)
222{
223 TCGv_ptr a0, a1;
224 TCGv_i32 desc = tcg_const_i32(simd_desc(oprsz, maxsz, data));
225
226 a0 = tcg_temp_new_ptr();
227 a1 = tcg_temp_new_ptr();
228
229 tcg_gen_addi_ptr(a0, cpu_env, dofs);
230 tcg_gen_addi_ptr(a1, cpu_env, aofs);
231
232 fn(a0, a1, ptr, desc);
233
234 tcg_temp_free_ptr(a0);
235 tcg_temp_free_ptr(a1);
236 tcg_temp_free_i32(desc);
237}
238
239/* Generate a call to a gvec-style helper with three vector operands
240 and an extra pointer operand. */
241void tcg_gen_gvec_3_ptr(uint32_t dofs, uint32_t aofs, uint32_t bofs,
242 TCGv_ptr ptr, uint32_t oprsz, uint32_t maxsz,
243 int32_t data, gen_helper_gvec_3_ptr *fn)
244{
245 TCGv_ptr a0, a1, a2;
246 TCGv_i32 desc = tcg_const_i32(simd_desc(oprsz, maxsz, data));
247
248 a0 = tcg_temp_new_ptr();
249 a1 = tcg_temp_new_ptr();
250 a2 = tcg_temp_new_ptr();
251
252 tcg_gen_addi_ptr(a0, cpu_env, dofs);
253 tcg_gen_addi_ptr(a1, cpu_env, aofs);
254 tcg_gen_addi_ptr(a2, cpu_env, bofs);
255
256 fn(a0, a1, a2, ptr, desc);
257
258 tcg_temp_free_ptr(a0);
259 tcg_temp_free_ptr(a1);
260 tcg_temp_free_ptr(a2);
261 tcg_temp_free_i32(desc);
262}
263
264/* Generate a call to a gvec-style helper with four vector operands
265 and an extra pointer operand. */
266void tcg_gen_gvec_4_ptr(uint32_t dofs, uint32_t aofs, uint32_t bofs,
267 uint32_t cofs, TCGv_ptr ptr, uint32_t oprsz,
268 uint32_t maxsz, int32_t data,
269 gen_helper_gvec_4_ptr *fn)
270{
271 TCGv_ptr a0, a1, a2, a3;
272 TCGv_i32 desc = tcg_const_i32(simd_desc(oprsz, maxsz, data));
273
274 a0 = tcg_temp_new_ptr();
275 a1 = tcg_temp_new_ptr();
276 a2 = tcg_temp_new_ptr();
277 a3 = tcg_temp_new_ptr();
278
279 tcg_gen_addi_ptr(a0, cpu_env, dofs);
280 tcg_gen_addi_ptr(a1, cpu_env, aofs);
281 tcg_gen_addi_ptr(a2, cpu_env, bofs);
282 tcg_gen_addi_ptr(a3, cpu_env, cofs);
283
284 fn(a0, a1, a2, a3, ptr, desc);
285
286 tcg_temp_free_ptr(a0);
287 tcg_temp_free_ptr(a1);
288 tcg_temp_free_ptr(a2);
289 tcg_temp_free_ptr(a3);
290 tcg_temp_free_i32(desc);
291}
292
293/* Return true if we want to implement something of OPRSZ bytes
294 in units of LNSZ. This limits the expansion of inline code. */
295static inline bool check_size_impl(uint32_t oprsz, uint32_t lnsz)
296{
499748d7
RH		 297 if (oprsz % lnsz == 0) {
298 uint32_t lnct = oprsz / lnsz;
299 return lnct >= 1 && lnct <= MAX_UNROLL;
300 }
301 return false;
db432672
RH		 302}
303
304static void expand_clr(uint32_t dofs, uint32_t maxsz);
305
306/* Duplicate C as per VECE. */
307uint64_t (dup_const)(unsigned vece, uint64_t c)
308{
309 switch (vece) {
310 case MO_8:
311 return 0x0101010101010101ull * (uint8_t)c;
312 case MO_16:
313 return 0x0001000100010001ull * (uint16_t)c;
314 case MO_32:
315 return 0x0000000100000001ull * (uint32_t)c;
316 case MO_64:
317 return c;
318 default:
319 g_assert_not_reached();
320 }
321}
322
323/* Duplicate IN into OUT as per VECE. */
324static void gen_dup_i32(unsigned vece, TCGv_i32 out, TCGv_i32 in)
325{
326 switch (vece) {
327 case MO_8:
328 tcg_gen_ext8u_i32(out, in);
329 tcg_gen_muli_i32(out, out, 0x01010101);
330 break;
331 case MO_16:
332 tcg_gen_deposit_i32(out, in, in, 16, 16);
333 break;
334 case MO_32:
335 tcg_gen_mov_i32(out, in);
336 break;
337 default:
338 g_assert_not_reached();
339 }
340}
341
342static void gen_dup_i64(unsigned vece, TCGv_i64 out, TCGv_i64 in)
343{
344 switch (vece) {
345 case MO_8:
346 tcg_gen_ext8u_i64(out, in);
347 tcg_gen_muli_i64(out, out, 0x0101010101010101ull);
348 break;
349 case MO_16:
350 tcg_gen_ext16u_i64(out, in);
351 tcg_gen_muli_i64(out, out, 0x0001000100010001ull);
352 break;
353 case MO_32:
354 tcg_gen_deposit_i64(out, in, in, 32, 32);
355 break;
356 case MO_64:
357 tcg_gen_mov_i64(out, in);
358 break;
359 default:
360 g_assert_not_reached();
361 }
362}
363
adb196cb
RH		 364/* Select a supported vector type for implementing an operation on SIZE
365 * bytes. If OP is 0, assume that the real operation to be performed is
366 * required by all backends. Otherwise, make sure than OP can be performed
367 * on elements of size VECE in the selected type. Do not select V64 if
368 * PREFER_I64 is true. Return 0 if no vector type is selected.
369 */
53229a77
RH		 370static TCGType choose_vector_type(const TCGOpcode *list, unsigned vece,
371 uint32_t size, bool prefer_i64)
adb196cb
RH		 372{
373 if (TCG_TARGET_HAS_v256 && check_size_impl(size, 32)) {
53229a77
RH		 374 /*
375 * Recall that ARM SVE allows vector sizes that are not a
adb196cb
RH		 376 * power of 2, but always a multiple of 16. The intent is
377 * that e.g. size == 80 would be expanded with 2x32 + 1x16.
378 * It is hard to imagine a case in which v256 is supported
379 * but v128 is not, but check anyway.
380 */
53229a77 381 if (tcg_can_emit_vecop_list(list, TCG_TYPE_V256, vece)
adb196cb 382 && (size % 32 == 0
53229a77 383 || tcg_can_emit_vecop_list(list, TCG_TYPE_V128, vece))) {
adb196cb
RH		 384 return TCG_TYPE_V256;
385 }
386 }
387 if (TCG_TARGET_HAS_v128 && check_size_impl(size, 16)
53229a77 388 && tcg_can_emit_vecop_list(list, TCG_TYPE_V128, vece)) {
adb196cb
RH		 389 return TCG_TYPE_V128;
390 }
391 if (TCG_TARGET_HAS_v64 && !prefer_i64 && check_size_impl(size, 8)
53229a77 392 && tcg_can_emit_vecop_list(list, TCG_TYPE_V64, vece)) {
adb196cb
RH		 393 return TCG_TYPE_V64;
394 }
395 return 0;
396}
397
37ee55a0
RH		 398static void do_dup_store(TCGType type, uint32_t dofs, uint32_t oprsz,
399 uint32_t maxsz, TCGv_vec t_vec)
400{
401 uint32_t i = 0;
402
403 switch (type) {
404 case TCG_TYPE_V256:
405 /*
406 * Recall that ARM SVE allows vector sizes that are not a
407 * power of 2, but always a multiple of 16. The intent is
408 * that e.g. size == 80 would be expanded with 2x32 + 1x16.
409 */
410 for (; i + 32 <= oprsz; i += 32) {
411 tcg_gen_stl_vec(t_vec, cpu_env, dofs + i, TCG_TYPE_V256);
412 }
413 /* fallthru */
414 case TCG_TYPE_V128:
415 for (; i + 16 <= oprsz; i += 16) {
416 tcg_gen_stl_vec(t_vec, cpu_env, dofs + i, TCG_TYPE_V128);
417 }
418 break;
419 case TCG_TYPE_V64:
420 for (; i < oprsz; i += 8) {
421 tcg_gen_stl_vec(t_vec, cpu_env, dofs + i, TCG_TYPE_V64);
422 }
423 break;
424 default:
425 g_assert_not_reached();
426 }
427
428 if (oprsz < maxsz) {
429 expand_clr(dofs + oprsz, maxsz - oprsz);
430 }
431}
432
db432672
RH		 433/* Set OPRSZ bytes at DOFS to replications of IN_32, IN_64 or IN_C.
434 * Only one of IN_32 or IN_64 may be set;
435 * IN_C is used if IN_32 and IN_64 are unset.
436 */
437static void do_dup(unsigned vece, uint32_t dofs, uint32_t oprsz,
438 uint32_t maxsz, TCGv_i32 in_32, TCGv_i64 in_64,
439 uint64_t in_c)
440{
441 TCGType type;
442 TCGv_i64 t_64;
443 TCGv_i32 t_32, t_desc;
444 TCGv_ptr t_ptr;
445 uint32_t i;
446
447 assert(vece <= (in_32 ? MO_32 : MO_64));
448 assert(in_32 == NULL || in_64 == NULL);
449
450 /* If we're storing 0, expand oprsz to maxsz. */
451 if (in_32 == NULL && in_64 == NULL) {
452 in_c = dup_const(vece, in_c);
453 if (in_c == 0) {
454 oprsz = maxsz;
455 }
456 }
457
adb196cb
RH		 458 /* Implement inline with a vector type, if possible.
459 * Prefer integer when 64-bit host and no variable dup.
460 */
53229a77 461 type = choose_vector_type(NULL, vece, oprsz,
adb196cb
RH		 462 (TCG_TARGET_REG_BITS == 64 && in_32 == NULL
463 && (in_64 == NULL || vece == MO_64)));
db432672
RH		 464 if (type != 0) {
465 TCGv_vec t_vec = tcg_temp_new_vec(type);
466
467 if (in_32) {
468 tcg_gen_dup_i32_vec(vece, t_vec, in_32);
469 } else if (in_64) {
470 tcg_gen_dup_i64_vec(vece, t_vec, in_64);
471 } else {
37ee55a0 472 tcg_gen_dupi_vec(vece, t_vec, in_c);
db432672 473 }
37ee55a0 474 do_dup_store(type, dofs, oprsz, maxsz, t_vec);
db432672 475 tcg_temp_free_vec(t_vec);
37ee55a0 476 return;
db432672
RH		 477 }
478
479 /* Otherwise, inline with an integer type, unless "large". */
480 if (check_size_impl(oprsz, TCG_TARGET_REG_BITS / 8)) {
481 t_64 = NULL;
482 t_32 = NULL;
483
484 if (in_32) {
485 /* We are given a 32-bit variable input. For a 64-bit host,
486 use a 64-bit operation unless the 32-bit operation would
487 be simple enough. */
488 if (TCG_TARGET_REG_BITS == 64
489 && (vece != MO_32 || !check_size_impl(oprsz, 4))) {
490 t_64 = tcg_temp_new_i64();
491 tcg_gen_extu_i32_i64(t_64, in_32);
492 gen_dup_i64(vece, t_64, t_64);
493 } else {
494 t_32 = tcg_temp_new_i32();
495 gen_dup_i32(vece, t_32, in_32);
496 }
497 } else if (in_64) {
498 /* We are given a 64-bit variable input. */
499 t_64 = tcg_temp_new_i64();
500 gen_dup_i64(vece, t_64, in_64);
501 } else {
502 /* We are given a constant input. */
503 /* For 64-bit hosts, use 64-bit constants for "simple" constants
504 or when we'd need too many 32-bit stores, or when a 64-bit
505 constant is really required. */
506 if (vece == MO_64
507 || (TCG_TARGET_REG_BITS == 64
508 && (in_c == 0 || in_c == -1
509 || !check_size_impl(oprsz, 4)))) {
510 t_64 = tcg_const_i64(in_c);
511 } else {
512 t_32 = tcg_const_i32(in_c);
513 }
514 }
515
516 /* Implement inline if we picked an implementation size above. */
517 if (t_32) {
518 for (i = 0; i < oprsz; i += 4) {
519 tcg_gen_st_i32(t_32, cpu_env, dofs + i);
520 }
521 tcg_temp_free_i32(t_32);
522 goto done;
523 }
524 if (t_64) {
525 for (i = 0; i < oprsz; i += 8) {
526 tcg_gen_st_i64(t_64, cpu_env, dofs + i);
527 }
528 tcg_temp_free_i64(t_64);
529 goto done;
adb196cb 530 }
db432672
RH		 531 }
532
533 /* Otherwise implement out of line. */
534 t_ptr = tcg_temp_new_ptr();
535 tcg_gen_addi_ptr(t_ptr, cpu_env, dofs);
536 t_desc = tcg_const_i32(simd_desc(oprsz, maxsz, 0));
537
538 if (vece == MO_64) {
539 if (in_64) {
540 gen_helper_gvec_dup64(t_ptr, t_desc, in_64);
541 } else {
542 t_64 = tcg_const_i64(in_c);
543 gen_helper_gvec_dup64(t_ptr, t_desc, t_64);
544 tcg_temp_free_i64(t_64);
545 }
546 } else {
547 typedef void dup_fn(TCGv_ptr, TCGv_i32, TCGv_i32);
548 static dup_fn * const fns[3] = {
549 gen_helper_gvec_dup8,
550 gen_helper_gvec_dup16,
551 gen_helper_gvec_dup32
552 };
553
554 if (in_32) {
555 fns[vece](t_ptr, t_desc, in_32);
556 } else {
557 t_32 = tcg_temp_new_i32();
558 if (in_64) {
559 tcg_gen_extrl_i64_i32(t_32, in_64);
560 } else if (vece == MO_8) {
561 tcg_gen_movi_i32(t_32, in_c & 0xff);
562 } else if (vece == MO_16) {
563 tcg_gen_movi_i32(t_32, in_c & 0xffff);
564 } else {
565 tcg_gen_movi_i32(t_32, in_c);
566 }
567 fns[vece](t_ptr, t_desc, t_32);
568 tcg_temp_free_i32(t_32);
569 }
570 }
571
572 tcg_temp_free_ptr(t_ptr);
573 tcg_temp_free_i32(t_desc);
574 return;
575
576 done:
577 if (oprsz < maxsz) {
578 expand_clr(dofs + oprsz, maxsz - oprsz);
579 }
580}
581
582/* Likewise, but with zero. */
583static void expand_clr(uint32_t dofs, uint32_t maxsz)
584{
585 do_dup(MO_8, dofs, maxsz, maxsz, NULL, NULL, 0);
586}
587
588/* Expand OPSZ bytes worth of two-operand operations using i32 elements. */
589static void expand_2_i32(uint32_t dofs, uint32_t aofs, uint32_t oprsz,
590 void (*fni)(TCGv_i32, TCGv_i32))
591{
592 TCGv_i32 t0 = tcg_temp_new_i32();
593 uint32_t i;
594
595 for (i = 0; i < oprsz; i += 4) {
596 tcg_gen_ld_i32(t0, cpu_env, aofs + i);
597 fni(t0, t0);
598 tcg_gen_st_i32(t0, cpu_env, dofs + i);
599 }
600 tcg_temp_free_i32(t0);
601}
602
d0ec9796
RH		 603static void expand_2i_i32(uint32_t dofs, uint32_t aofs, uint32_t oprsz,
604 int32_t c, bool load_dest,
605 void (*fni)(TCGv_i32, TCGv_i32, int32_t))
606{
607 TCGv_i32 t0 = tcg_temp_new_i32();
608 TCGv_i32 t1 = tcg_temp_new_i32();
609 uint32_t i;
610
611 for (i = 0; i < oprsz; i += 4) {
612 tcg_gen_ld_i32(t0, cpu_env, aofs + i);
613 if (load_dest) {
614 tcg_gen_ld_i32(t1, cpu_env, dofs + i);
615 }
616 fni(t1, t0, c);
617 tcg_gen_st_i32(t1, cpu_env, dofs + i);
618 }
619 tcg_temp_free_i32(t0);
620 tcg_temp_free_i32(t1);
621}
622
22fc3527
RH		 623static void expand_2s_i32(uint32_t dofs, uint32_t aofs, uint32_t oprsz,
624 TCGv_i32 c, bool scalar_first,
625 void (*fni)(TCGv_i32, TCGv_i32, TCGv_i32))
626{
627 TCGv_i32 t0 = tcg_temp_new_i32();
628 TCGv_i32 t1 = tcg_temp_new_i32();
629 uint32_t i;
630
631 for (i = 0; i < oprsz; i += 4) {
632 tcg_gen_ld_i32(t0, cpu_env, aofs + i);
633 if (scalar_first) {
634 fni(t1, c, t0);
635 } else {
636 fni(t1, t0, c);
637 }
638 tcg_gen_st_i32(t1, cpu_env, dofs + i);
639 }
640 tcg_temp_free_i32(t0);
641 tcg_temp_free_i32(t1);
642}
643
db432672
RH		 644/* Expand OPSZ bytes worth of three-operand operations using i32 elements. */
645static void expand_3_i32(uint32_t dofs, uint32_t aofs,
646 uint32_t bofs, uint32_t oprsz, bool load_dest,
647 void (*fni)(TCGv_i32, TCGv_i32, TCGv_i32))
648{
649 TCGv_i32 t0 = tcg_temp_new_i32();
650 TCGv_i32 t1 = tcg_temp_new_i32();
651 TCGv_i32 t2 = tcg_temp_new_i32();
652 uint32_t i;
653
654 for (i = 0; i < oprsz; i += 4) {
655 tcg_gen_ld_i32(t0, cpu_env, aofs + i);
656 tcg_gen_ld_i32(t1, cpu_env, bofs + i);
657 if (load_dest) {
658 tcg_gen_ld_i32(t2, cpu_env, dofs + i);
659 }
660 fni(t2, t0, t1);
661 tcg_gen_st_i32(t2, cpu_env, dofs + i);
662 }
663 tcg_temp_free_i32(t2);
664 tcg_temp_free_i32(t1);
665 tcg_temp_free_i32(t0);
666}
667
e1227bb6
DH		 668static void expand_3i_i32(uint32_t dofs, uint32_t aofs, uint32_t bofs,
669 uint32_t oprsz, int32_t c, bool load_dest,
670 void (*fni)(TCGv_i32, TCGv_i32, TCGv_i32, int32_t))
671{
672 TCGv_i32 t0 = tcg_temp_new_i32();
673 TCGv_i32 t1 = tcg_temp_new_i32();
674 TCGv_i32 t2 = tcg_temp_new_i32();
675 uint32_t i;
676
677 for (i = 0; i < oprsz; i += 4) {
678 tcg_gen_ld_i32(t0, cpu_env, aofs + i);
679 tcg_gen_ld_i32(t1, cpu_env, bofs + i);
680 if (load_dest) {
681 tcg_gen_ld_i32(t2, cpu_env, dofs + i);
682 }
683 fni(t2, t0, t1, c);
684 tcg_gen_st_i32(t2, cpu_env, dofs + i);
685 }
686 tcg_temp_free_i32(t0);
687 tcg_temp_free_i32(t1);
688 tcg_temp_free_i32(t2);
689}
690
db432672
RH		 691/* Expand OPSZ bytes worth of three-operand operations using i32 elements. */
692static void expand_4_i32(uint32_t dofs, uint32_t aofs, uint32_t bofs,
5d6acdd4 693 uint32_t cofs, uint32_t oprsz, bool write_aofs,
db432672
RH		 694 void (*fni)(TCGv_i32, TCGv_i32, TCGv_i32, TCGv_i32))
695{
696 TCGv_i32 t0 = tcg_temp_new_i32();
697 TCGv_i32 t1 = tcg_temp_new_i32();
698 TCGv_i32 t2 = tcg_temp_new_i32();
699 TCGv_i32 t3 = tcg_temp_new_i32();
700 uint32_t i;
701
702 for (i = 0; i < oprsz; i += 4) {
703 tcg_gen_ld_i32(t1, cpu_env, aofs + i);
704 tcg_gen_ld_i32(t2, cpu_env, bofs + i);
705 tcg_gen_ld_i32(t3, cpu_env, cofs + i);
706 fni(t0, t1, t2, t3);
707 tcg_gen_st_i32(t0, cpu_env, dofs + i);
5d6acdd4
RH		 708 if (write_aofs) {
709 tcg_gen_st_i32(t1, cpu_env, aofs + i);
710 }
db432672
RH		 711 }
712 tcg_temp_free_i32(t3);
713 tcg_temp_free_i32(t2);
714 tcg_temp_free_i32(t1);
715 tcg_temp_free_i32(t0);
716}
717
718/* Expand OPSZ bytes worth of two-operand operations using i64 elements. */
719static void expand_2_i64(uint32_t dofs, uint32_t aofs, uint32_t oprsz,
720 void (*fni)(TCGv_i64, TCGv_i64))
721{
722 TCGv_i64 t0 = tcg_temp_new_i64();
723 uint32_t i;
724
725 for (i = 0; i < oprsz; i += 8) {
726 tcg_gen_ld_i64(t0, cpu_env, aofs + i);
727 fni(t0, t0);
728 tcg_gen_st_i64(t0, cpu_env, dofs + i);
729 }
730 tcg_temp_free_i64(t0);
731}
732
d0ec9796
RH		 733static void expand_2i_i64(uint32_t dofs, uint32_t aofs, uint32_t oprsz,
734 int64_t c, bool load_dest,
735 void (*fni)(TCGv_i64, TCGv_i64, int64_t))
736{
737 TCGv_i64 t0 = tcg_temp_new_i64();
738 TCGv_i64 t1 = tcg_temp_new_i64();
739 uint32_t i;
740
741 for (i = 0; i < oprsz; i += 8) {
742 tcg_gen_ld_i64(t0, cpu_env, aofs + i);
743 if (load_dest) {
744 tcg_gen_ld_i64(t1, cpu_env, dofs + i);
745 }
746 fni(t1, t0, c);
747 tcg_gen_st_i64(t1, cpu_env, dofs + i);
748 }
749 tcg_temp_free_i64(t0);
750 tcg_temp_free_i64(t1);
751}
752
22fc3527
RH		 753static void expand_2s_i64(uint32_t dofs, uint32_t aofs, uint32_t oprsz,
754 TCGv_i64 c, bool scalar_first,
755 void (*fni)(TCGv_i64, TCGv_i64, TCGv_i64))
756{
757 TCGv_i64 t0 = tcg_temp_new_i64();
758 TCGv_i64 t1 = tcg_temp_new_i64();
759 uint32_t i;
760
761 for (i = 0; i < oprsz; i += 8) {
762 tcg_gen_ld_i64(t0, cpu_env, aofs + i);
763 if (scalar_first) {
764 fni(t1, c, t0);
765 } else {
766 fni(t1, t0, c);
767 }
768 tcg_gen_st_i64(t1, cpu_env, dofs + i);
769 }
770 tcg_temp_free_i64(t0);
771 tcg_temp_free_i64(t1);
772}
773
db432672
RH		 774/* Expand OPSZ bytes worth of three-operand operations using i64 elements. */
775static void expand_3_i64(uint32_t dofs, uint32_t aofs,
776 uint32_t bofs, uint32_t oprsz, bool load_dest,
777 void (*fni)(TCGv_i64, TCGv_i64, TCGv_i64))
778{
779 TCGv_i64 t0 = tcg_temp_new_i64();
780 TCGv_i64 t1 = tcg_temp_new_i64();
781 TCGv_i64 t2 = tcg_temp_new_i64();
782 uint32_t i;
783
784 for (i = 0; i < oprsz; i += 8) {
785 tcg_gen_ld_i64(t0, cpu_env, aofs + i);
786 tcg_gen_ld_i64(t1, cpu_env, bofs + i);
787 if (load_dest) {
788 tcg_gen_ld_i64(t2, cpu_env, dofs + i);
789 }
790 fni(t2, t0, t1);
791 tcg_gen_st_i64(t2, cpu_env, dofs + i);
792 }
793 tcg_temp_free_i64(t2);
794 tcg_temp_free_i64(t1);
795 tcg_temp_free_i64(t0);
796}
797
e1227bb6
DH		 798static void expand_3i_i64(uint32_t dofs, uint32_t aofs, uint32_t bofs,
799 uint32_t oprsz, int64_t c, bool load_dest,
800 void (*fni)(TCGv_i64, TCGv_i64, TCGv_i64, int64_t))
801{
802 TCGv_i64 t0 = tcg_temp_new_i64();
803 TCGv_i64 t1 = tcg_temp_new_i64();
804 TCGv_i64 t2 = tcg_temp_new_i64();
805 uint32_t i;
806
807 for (i = 0; i < oprsz; i += 8) {
808 tcg_gen_ld_i64(t0, cpu_env, aofs + i);
809 tcg_gen_ld_i64(t1, cpu_env, bofs + i);
810 if (load_dest) {
811 tcg_gen_ld_i64(t2, cpu_env, dofs + i);
812 }
813 fni(t2, t0, t1, c);
814 tcg_gen_st_i64(t2, cpu_env, dofs + i);
815 }
816 tcg_temp_free_i64(t0);
817 tcg_temp_free_i64(t1);
818 tcg_temp_free_i64(t2);
819}
820
db432672
RH		 821/* Expand OPSZ bytes worth of three-operand operations using i64 elements. */
822static void expand_4_i64(uint32_t dofs, uint32_t aofs, uint32_t bofs,
5d6acdd4 823 uint32_t cofs, uint32_t oprsz, bool write_aofs,
db432672
RH		 824 void (*fni)(TCGv_i64, TCGv_i64, TCGv_i64, TCGv_i64))
825{
826 TCGv_i64 t0 = tcg_temp_new_i64();
827 TCGv_i64 t1 = tcg_temp_new_i64();
828 TCGv_i64 t2 = tcg_temp_new_i64();
829 TCGv_i64 t3 = tcg_temp_new_i64();
830 uint32_t i;
831
832 for (i = 0; i < oprsz; i += 8) {
833 tcg_gen_ld_i64(t1, cpu_env, aofs + i);
834 tcg_gen_ld_i64(t2, cpu_env, bofs + i);
835 tcg_gen_ld_i64(t3, cpu_env, cofs + i);
836 fni(t0, t1, t2, t3);
837 tcg_gen_st_i64(t0, cpu_env, dofs + i);
5d6acdd4
RH		 838 if (write_aofs) {
839 tcg_gen_st_i64(t1, cpu_env, aofs + i);
840 }
db432672
RH		 841 }
842 tcg_temp_free_i64(t3);
843 tcg_temp_free_i64(t2);
844 tcg_temp_free_i64(t1);
845 tcg_temp_free_i64(t0);
846}
847
848/* Expand OPSZ bytes worth of two-operand operations using host vectors. */
849static void expand_2_vec(unsigned vece, uint32_t dofs, uint32_t aofs,
850 uint32_t oprsz, uint32_t tysz, TCGType type,
851 void (*fni)(unsigned, TCGv_vec, TCGv_vec))
852{
853 TCGv_vec t0 = tcg_temp_new_vec(type);
854 uint32_t i;
855
856 for (i = 0; i < oprsz; i += tysz) {
857 tcg_gen_ld_vec(t0, cpu_env, aofs + i);
858 fni(vece, t0, t0);
859 tcg_gen_st_vec(t0, cpu_env, dofs + i);
860 }
861 tcg_temp_free_vec(t0);
862}
863
d0ec9796
RH		 864/* Expand OPSZ bytes worth of two-vector operands and an immediate operand
865 using host vectors. */
866static void expand_2i_vec(unsigned vece, uint32_t dofs, uint32_t aofs,
867 uint32_t oprsz, uint32_t tysz, TCGType type,
868 int64_t c, bool load_dest,
869 void (*fni)(unsigned, TCGv_vec, TCGv_vec, int64_t))
870{
871 TCGv_vec t0 = tcg_temp_new_vec(type);
872 TCGv_vec t1 = tcg_temp_new_vec(type);
873 uint32_t i;
874
875 for (i = 0; i < oprsz; i += tysz) {
876 tcg_gen_ld_vec(t0, cpu_env, aofs + i);
877 if (load_dest) {
878 tcg_gen_ld_vec(t1, cpu_env, dofs + i);
879 }
880 fni(vece, t1, t0, c);
881 tcg_gen_st_vec(t1, cpu_env, dofs + i);
882 }
883 tcg_temp_free_vec(t0);
884 tcg_temp_free_vec(t1);
885}
886
22fc3527
RH		 887static void expand_2s_vec(unsigned vece, uint32_t dofs, uint32_t aofs,
888 uint32_t oprsz, uint32_t tysz, TCGType type,
889 TCGv_vec c, bool scalar_first,
890 void (*fni)(unsigned, TCGv_vec, TCGv_vec, TCGv_vec))
891{
892 TCGv_vec t0 = tcg_temp_new_vec(type);
893 TCGv_vec t1 = tcg_temp_new_vec(type);
894 uint32_t i;
895
896 for (i = 0; i < oprsz; i += tysz) {
897 tcg_gen_ld_vec(t0, cpu_env, aofs + i);
898 if (scalar_first) {
899 fni(vece, t1, c, t0);
900 } else {
901 fni(vece, t1, t0, c);
902 }
903 tcg_gen_st_vec(t1, cpu_env, dofs + i);
904 }
905 tcg_temp_free_vec(t0);
906 tcg_temp_free_vec(t1);
907}
908
db432672
RH		 909/* Expand OPSZ bytes worth of three-operand operations using host vectors. */
910static void expand_3_vec(unsigned vece, uint32_t dofs, uint32_t aofs,
911 uint32_t bofs, uint32_t oprsz,
912 uint32_t tysz, TCGType type, bool load_dest,
913 void (*fni)(unsigned, TCGv_vec, TCGv_vec, TCGv_vec))
914{
915 TCGv_vec t0 = tcg_temp_new_vec(type);
916 TCGv_vec t1 = tcg_temp_new_vec(type);
917 TCGv_vec t2 = tcg_temp_new_vec(type);
918 uint32_t i;
919
920 for (i = 0; i < oprsz; i += tysz) {
921 tcg_gen_ld_vec(t0, cpu_env, aofs + i);
922 tcg_gen_ld_vec(t1, cpu_env, bofs + i);
923 if (load_dest) {
924 tcg_gen_ld_vec(t2, cpu_env, dofs + i);
925 }
926 fni(vece, t2, t0, t1);
927 tcg_gen_st_vec(t2, cpu_env, dofs + i);
928 }
929 tcg_temp_free_vec(t2);
930 tcg_temp_free_vec(t1);
931 tcg_temp_free_vec(t0);
932}
933
e1227bb6
DH		 934/*
935 * Expand OPSZ bytes worth of three-vector operands and an immediate operand
936 * using host vectors.
937 */
938static void expand_3i_vec(unsigned vece, uint32_t dofs, uint32_t aofs,
939 uint32_t bofs, uint32_t oprsz, uint32_t tysz,
940 TCGType type, int64_t c, bool load_dest,
941 void (*fni)(unsigned, TCGv_vec, TCGv_vec, TCGv_vec,
942 int64_t))
943{
944 TCGv_vec t0 = tcg_temp_new_vec(type);
945 TCGv_vec t1 = tcg_temp_new_vec(type);
946 TCGv_vec t2 = tcg_temp_new_vec(type);
947 uint32_t i;
948
949 for (i = 0; i < oprsz; i += tysz) {
950 tcg_gen_ld_vec(t0, cpu_env, aofs + i);
951 tcg_gen_ld_vec(t1, cpu_env, bofs + i);
952 if (load_dest) {
953 tcg_gen_ld_vec(t2, cpu_env, dofs + i);
954 }
955 fni(vece, t2, t0, t1, c);
956 tcg_gen_st_vec(t2, cpu_env, dofs + i);
957 }
958 tcg_temp_free_vec(t0);
959 tcg_temp_free_vec(t1);
960 tcg_temp_free_vec(t2);
961}
962
db432672
RH		 963/* Expand OPSZ bytes worth of four-operand operations using host vectors. */
964static void expand_4_vec(unsigned vece, uint32_t dofs, uint32_t aofs,
965 uint32_t bofs, uint32_t cofs, uint32_t oprsz,
5d6acdd4 966 uint32_t tysz, TCGType type, bool write_aofs,
db432672
RH		 967 void (*fni)(unsigned, TCGv_vec, TCGv_vec,
968 TCGv_vec, TCGv_vec))
969{
970 TCGv_vec t0 = tcg_temp_new_vec(type);
971 TCGv_vec t1 = tcg_temp_new_vec(type);
972 TCGv_vec t2 = tcg_temp_new_vec(type);
973 TCGv_vec t3 = tcg_temp_new_vec(type);
974 uint32_t i;
975
976 for (i = 0; i < oprsz; i += tysz) {
977 tcg_gen_ld_vec(t1, cpu_env, aofs + i);
978 tcg_gen_ld_vec(t2, cpu_env, bofs + i);
979 tcg_gen_ld_vec(t3, cpu_env, cofs + i);
980 fni(vece, t0, t1, t2, t3);
981 tcg_gen_st_vec(t0, cpu_env, dofs + i);
5d6acdd4
RH		 982 if (write_aofs) {
983 tcg_gen_st_vec(t1, cpu_env, aofs + i);
984 }
db432672
RH		 985 }
986 tcg_temp_free_vec(t3);
987 tcg_temp_free_vec(t2);
988 tcg_temp_free_vec(t1);
989 tcg_temp_free_vec(t0);
990}
991
992/* Expand a vector two-operand operation. */
993void tcg_gen_gvec_2(uint32_t dofs, uint32_t aofs,
994 uint32_t oprsz, uint32_t maxsz, const GVecGen2 *g)
995{
53229a77
RH		 996 const TCGOpcode *this_list = g->opt_opc ? : vecop_list_empty;
997 const TCGOpcode *hold_list = tcg_swap_vecop_list(this_list);
adb196cb
RH		 998 TCGType type;
999 uint32_t some;
1000
db432672
RH		 1001 check_size_align(oprsz, maxsz, dofs | aofs);
1002 check_overlap_2(dofs, aofs, maxsz);
1003
adb196cb
RH		 1004 type = 0;
1005 if (g->fniv) {
53229a77 1006 type = choose_vector_type(g->opt_opc, g->vece, oprsz, g->prefer_i64);
adb196cb
RH		 1007 }
1008 switch (type) {
1009 case TCG_TYPE_V256:
1010 /* Recall that ARM SVE allows vector sizes that are not a
1011 * power of 2, but always a multiple of 16. The intent is
1012 * that e.g. size == 80 would be expanded with 2x32 + 1x16.
1013 */
1014 some = QEMU_ALIGN_DOWN(oprsz, 32);
db432672
RH		 1015 expand_2_vec(g->vece, dofs, aofs, some, 32, TCG_TYPE_V256, g->fniv);
1016 if (some == oprsz) {
adb196cb 1017 break;
db432672
RH		 1018 }
1019 dofs += some;
1020 aofs += some;
1021 oprsz -= some;
1022 maxsz -= some;
adb196cb
RH		 1023 /* fallthru */
1024 case TCG_TYPE_V128:
db432672 1025 expand_2_vec(g->vece, dofs, aofs, oprsz, 16, TCG_TYPE_V128, g->fniv);
adb196cb
RH		 1026 break;
1027 case TCG_TYPE_V64:
db432672 1028 expand_2_vec(g->vece, dofs, aofs, oprsz, 8, TCG_TYPE_V64, g->fniv);
adb196cb
RH		 1029 break;
1030
1031 case 0:
1032 if (g->fni8 && check_size_impl(oprsz, 8)) {
1033 expand_2_i64(dofs, aofs, oprsz, g->fni8);
1034 } else if (g->fni4 && check_size_impl(oprsz, 4)) {
1035 expand_2_i32(dofs, aofs, oprsz, g->fni4);
1036 } else {
1037 assert(g->fno != NULL);
1038 tcg_gen_gvec_2_ool(dofs, aofs, oprsz, maxsz, g->data, g->fno);
53229a77 1039 oprsz = maxsz;
adb196cb
RH		 1040 }
1041 break;
1042
1043 default:
1044 g_assert_not_reached();
db432672 1045 }
53229a77 1046 tcg_swap_vecop_list(hold_list);
db432672 1047
db432672
RH		 1048 if (oprsz < maxsz) {
1049 expand_clr(dofs + oprsz, maxsz - oprsz);
1050 }
1051}
1052
22fc3527 1053/* Expand a vector operation with two vectors and an immediate. */
d0ec9796
RH		 1054void tcg_gen_gvec_2i(uint32_t dofs, uint32_t aofs, uint32_t oprsz,
1055 uint32_t maxsz, int64_t c, const GVecGen2i *g)
1056{
53229a77
RH		 1057 const TCGOpcode *this_list = g->opt_opc ? : vecop_list_empty;
1058 const TCGOpcode *hold_list = tcg_swap_vecop_list(this_list);
adb196cb
RH		 1059 TCGType type;
1060 uint32_t some;
1061
d0ec9796
RH		 1062 check_size_align(oprsz, maxsz, dofs | aofs);
1063 check_overlap_2(dofs, aofs, maxsz);
1064
adb196cb
RH		 1065 type = 0;
1066 if (g->fniv) {
53229a77 1067 type = choose_vector_type(g->opt_opc, g->vece, oprsz, g->prefer_i64);
adb196cb
RH		 1068 }
1069 switch (type) {
1070 case TCG_TYPE_V256:
1071 /* Recall that ARM SVE allows vector sizes that are not a
1072 * power of 2, but always a multiple of 16. The intent is
1073 * that e.g. size == 80 would be expanded with 2x32 + 1x16.
1074 */
1075 some = QEMU_ALIGN_DOWN(oprsz, 32);
d0ec9796
RH		 1076 expand_2i_vec(g->vece, dofs, aofs, some, 32, TCG_TYPE_V256,
1077 c, g->load_dest, g->fniv);
1078 if (some == oprsz) {
adb196cb 1079 break;
d0ec9796
RH		 1080 }
1081 dofs += some;
1082 aofs += some;
1083 oprsz -= some;
1084 maxsz -= some;
adb196cb
RH		 1085 /* fallthru */
1086 case TCG_TYPE_V128:
d0ec9796
RH		 1087 expand_2i_vec(g->vece, dofs, aofs, oprsz, 16, TCG_TYPE_V128,
1088 c, g->load_dest, g->fniv);
adb196cb
RH		 1089 break;
1090 case TCG_TYPE_V64:
d0ec9796
RH		 1091 expand_2i_vec(g->vece, dofs, aofs, oprsz, 8, TCG_TYPE_V64,
1092 c, g->load_dest, g->fniv);
adb196cb
RH		 1093 break;
1094
1095 case 0:
1096 if (g->fni8 && check_size_impl(oprsz, 8)) {
1097 expand_2i_i64(dofs, aofs, oprsz, c, g->load_dest, g->fni8);
1098 } else if (g->fni4 && check_size_impl(oprsz, 4)) {
1099 expand_2i_i32(dofs, aofs, oprsz, c, g->load_dest, g->fni4);
22fc3527 1100 } else {
adb196cb
RH		 1101 if (g->fno) {
1102 tcg_gen_gvec_2_ool(dofs, aofs, oprsz, maxsz, c, g->fno);
1103 } else {
1104 TCGv_i64 tcg_c = tcg_const_i64(c);
1105 tcg_gen_gvec_2i_ool(dofs, aofs, tcg_c, oprsz,
1106 maxsz, c, g->fnoi);
1107 tcg_temp_free_i64(tcg_c);
1108 }
53229a77 1109 oprsz = maxsz;
22fc3527 1110 }
adb196cb
RH		 1111 break;
1112
1113 default:
1114 g_assert_not_reached();
d0ec9796 1115 }
53229a77 1116 tcg_swap_vecop_list(hold_list);
d0ec9796 1117
d0ec9796
RH		 1118 if (oprsz < maxsz) {
1119 expand_clr(dofs + oprsz, maxsz - oprsz);
1120 }
1121}
1122
22fc3527
RH		 1123/* Expand a vector operation with two vectors and a scalar. */
1124void tcg_gen_gvec_2s(uint32_t dofs, uint32_t aofs, uint32_t oprsz,
1125 uint32_t maxsz, TCGv_i64 c, const GVecGen2s *g)
1126{
1127 TCGType type;
1128
1129 check_size_align(oprsz, maxsz, dofs | aofs);
1130 check_overlap_2(dofs, aofs, maxsz);
1131
1132 type = 0;
1133 if (g->fniv) {
53229a77 1134 type = choose_vector_type(g->opt_opc, g->vece, oprsz, g->prefer_i64);
22fc3527
RH		 1135 }
1136 if (type != 0) {
53229a77
RH		 1137 const TCGOpcode *this_list = g->opt_opc ? : vecop_list_empty;
1138 const TCGOpcode *hold_list = tcg_swap_vecop_list(this_list);
22fc3527 1139 TCGv_vec t_vec = tcg_temp_new_vec(type);
adb196cb 1140 uint32_t some;
22fc3527
RH		 1141
1142 tcg_gen_dup_i64_vec(g->vece, t_vec, c);
1143
22fc3527
RH		 1144 switch (type) {
1145 case TCG_TYPE_V256:
adb196cb
RH		 1146 /* Recall that ARM SVE allows vector sizes that are not a
1147 * power of 2, but always a multiple of 16. The intent is
1148 * that e.g. size == 80 would be expanded with 2x32 + 1x16.
1149 */
1150 some = QEMU_ALIGN_DOWN(oprsz, 32);
1151 expand_2s_vec(g->vece, dofs, aofs, some, 32, TCG_TYPE_V256,
1152 t_vec, g->scalar_first, g->fniv);
1153 if (some == oprsz) {
1154 break;
22fc3527 1155 }
adb196cb
RH		 1156 dofs += some;
1157 aofs += some;
1158 oprsz -= some;
1159 maxsz -= some;
22fc3527
RH		 1160 /* fallthru */
1161
1162 case TCG_TYPE_V128:
1163 expand_2s_vec(g->vece, dofs, aofs, oprsz, 16, TCG_TYPE_V128,
1164 t_vec, g->scalar_first, g->fniv);
1165 break;
1166
1167 case TCG_TYPE_V64:
1168 expand_2s_vec(g->vece, dofs, aofs, oprsz, 8, TCG_TYPE_V64,
1169 t_vec, g->scalar_first, g->fniv);
1170 break;
1171
1172 default:
1173 g_assert_not_reached();
1174 }
1175 tcg_temp_free_vec(t_vec);
53229a77 1176 tcg_swap_vecop_list(hold_list);
22fc3527
RH		 1177 } else if (g->fni8 && check_size_impl(oprsz, 8)) {
1178 TCGv_i64 t64 = tcg_temp_new_i64();
1179
1180 gen_dup_i64(g->vece, t64, c);
1181 expand_2s_i64(dofs, aofs, oprsz, t64, g->scalar_first, g->fni8);
1182 tcg_temp_free_i64(t64);
1183 } else if (g->fni4 && check_size_impl(oprsz, 4)) {
1184 TCGv_i32 t32 = tcg_temp_new_i32();
1185
1186 tcg_gen_extrl_i64_i32(t32, c);
1187 gen_dup_i32(g->vece, t32, t32);
1188 expand_2s_i32(dofs, aofs, oprsz, t32, g->scalar_first, g->fni4);
1189 tcg_temp_free_i32(t32);
1190 } else {
1191 tcg_gen_gvec_2i_ool(dofs, aofs, c, oprsz, maxsz, 0, g->fno);
1192 return;
1193 }
1194
1195 if (oprsz < maxsz) {
1196 expand_clr(dofs + oprsz, maxsz - oprsz);
1197 }
1198}
1199
db432672
RH		 1200/* Expand a vector three-operand operation. */
1201void tcg_gen_gvec_3(uint32_t dofs, uint32_t aofs, uint32_t bofs,
1202 uint32_t oprsz, uint32_t maxsz, const GVecGen3 *g)
1203{
53229a77
RH		 1204 const TCGOpcode *this_list = g->opt_opc ? : vecop_list_empty;
1205 const TCGOpcode *hold_list = tcg_swap_vecop_list(this_list);
adb196cb
RH		 1206 TCGType type;
1207 uint32_t some;
1208
db432672
RH		 1209 check_size_align(oprsz, maxsz, dofs | aofs | bofs);
1210 check_overlap_3(dofs, aofs, bofs, maxsz);
1211
adb196cb
RH		 1212 type = 0;
1213 if (g->fniv) {
53229a77 1214 type = choose_vector_type(g->opt_opc, g->vece, oprsz, g->prefer_i64);
adb196cb
RH		 1215 }
1216 switch (type) {
1217 case TCG_TYPE_V256:
1218 /* Recall that ARM SVE allows vector sizes that are not a
1219 * power of 2, but always a multiple of 16. The intent is
1220 * that e.g. size == 80 would be expanded with 2x32 + 1x16.
1221 */
1222 some = QEMU_ALIGN_DOWN(oprsz, 32);
db432672
RH		 1223 expand_3_vec(g->vece, dofs, aofs, bofs, some, 32, TCG_TYPE_V256,
1224 g->load_dest, g->fniv);
1225 if (some == oprsz) {
adb196cb 1226 break;
db432672
RH		 1227 }
1228 dofs += some;
1229 aofs += some;
1230 bofs += some;
1231 oprsz -= some;
1232 maxsz -= some;
adb196cb
RH		 1233 /* fallthru */
1234 case TCG_TYPE_V128:
db432672
RH		 1235 expand_3_vec(g->vece, dofs, aofs, bofs, oprsz, 16, TCG_TYPE_V128,
1236 g->load_dest, g->fniv);
adb196cb
RH		 1237 break;
1238 case TCG_TYPE_V64:
db432672
RH		 1239 expand_3_vec(g->vece, dofs, aofs, bofs, oprsz, 8, TCG_TYPE_V64,
1240 g->load_dest, g->fniv);
adb196cb
RH		 1241 break;
1242
1243 case 0:
1244 if (g->fni8 && check_size_impl(oprsz, 8)) {
1245 expand_3_i64(dofs, aofs, bofs, oprsz, g->load_dest, g->fni8);
1246 } else if (g->fni4 && check_size_impl(oprsz, 4)) {
1247 expand_3_i32(dofs, aofs, bofs, oprsz, g->load_dest, g->fni4);
1248 } else {
1249 assert(g->fno != NULL);
1250 tcg_gen_gvec_3_ool(dofs, aofs, bofs, oprsz,
1251 maxsz, g->data, g->fno);
53229a77 1252 oprsz = maxsz;
adb196cb
RH		 1253 }
1254 break;
1255
1256 default:
1257 g_assert_not_reached();
db432672 1258 }
53229a77 1259 tcg_swap_vecop_list(hold_list);
db432672 1260
db432672
RH		 1261 if (oprsz < maxsz) {
1262 expand_clr(dofs + oprsz, maxsz - oprsz);
1263 }
1264}
1265
e1227bb6
DH		 1266/* Expand a vector operation with three vectors and an immediate. */
1267void tcg_gen_gvec_3i(uint32_t dofs, uint32_t aofs, uint32_t bofs,
1268 uint32_t oprsz, uint32_t maxsz, int64_t c,
1269 const GVecGen3i *g)
1270{
53229a77
RH		 1271 const TCGOpcode *this_list = g->opt_opc ? : vecop_list_empty;
1272 const TCGOpcode *hold_list = tcg_swap_vecop_list(this_list);
e1227bb6
DH		 1273 TCGType type;
1274 uint32_t some;
1275
1276 check_size_align(oprsz, maxsz, dofs | aofs | bofs);
1277 check_overlap_3(dofs, aofs, bofs, maxsz);
1278
1279 type = 0;
1280 if (g->fniv) {
53229a77 1281 type = choose_vector_type(g->opt_opc, g->vece, oprsz, g->prefer_i64);
e1227bb6
DH		 1282 }
1283 switch (type) {
1284 case TCG_TYPE_V256:
1285 /*
1286 * Recall that ARM SVE allows vector sizes that are not a
1287 * power of 2, but always a multiple of 16. The intent is
1288 * that e.g. size == 80 would be expanded with 2x32 + 1x16.
1289 */
1290 some = QEMU_ALIGN_DOWN(oprsz, 32);
1291 expand_3i_vec(g->vece, dofs, aofs, bofs, some, 32, TCG_TYPE_V256,
1292 c, g->load_dest, g->fniv);
1293 if (some == oprsz) {
1294 break;
1295 }
1296 dofs += some;
1297 aofs += some;
1298 bofs += some;
1299 oprsz -= some;
1300 maxsz -= some;
1301 /* fallthru */
1302 case TCG_TYPE_V128:
1303 expand_3i_vec(g->vece, dofs, aofs, bofs, oprsz, 16, TCG_TYPE_V128,
1304 c, g->load_dest, g->fniv);
1305 break;
1306 case TCG_TYPE_V64:
1307 expand_3i_vec(g->vece, dofs, aofs, bofs, oprsz, 8, TCG_TYPE_V64,
1308 c, g->load_dest, g->fniv);
1309 break;
1310
1311 case 0:
1312 if (g->fni8 && check_size_impl(oprsz, 8)) {
1313 expand_3i_i64(dofs, aofs, bofs, oprsz, c, g->load_dest, g->fni8);
1314 } else if (g->fni4 && check_size_impl(oprsz, 4)) {
1315 expand_3i_i32(dofs, aofs, bofs, oprsz, c, g->load_dest, g->fni4);
1316 } else {
1317 assert(g->fno != NULL);
1318 tcg_gen_gvec_3_ool(dofs, aofs, bofs, oprsz, maxsz, c, g->fno);
53229a77 1319 oprsz = maxsz;
e1227bb6
DH		 1320 }
1321 break;
1322
1323 default:
1324 g_assert_not_reached();
1325 }
53229a77 1326 tcg_swap_vecop_list(hold_list);
e1227bb6
DH		 1327
1328 if (oprsz < maxsz) {
1329 expand_clr(dofs + oprsz, maxsz - oprsz);
1330 }
1331}
1332
db432672
RH		 1333/* Expand a vector four-operand operation. */
1334void tcg_gen_gvec_4(uint32_t dofs, uint32_t aofs, uint32_t bofs, uint32_t cofs,
1335 uint32_t oprsz, uint32_t maxsz, const GVecGen4 *g)
1336{
53229a77
RH		 1337 const TCGOpcode *this_list = g->opt_opc ? : vecop_list_empty;
1338 const TCGOpcode *hold_list = tcg_swap_vecop_list(this_list);
adb196cb
RH		 1339 TCGType type;
1340 uint32_t some;
1341
db432672
RH		 1342 check_size_align(oprsz, maxsz, dofs | aofs | bofs | cofs);
1343 check_overlap_4(dofs, aofs, bofs, cofs, maxsz);
1344
adb196cb
RH		 1345 type = 0;
1346 if (g->fniv) {
53229a77 1347 type = choose_vector_type(g->opt_opc, g->vece, oprsz, g->prefer_i64);
adb196cb
RH		 1348 }
1349 switch (type) {
1350 case TCG_TYPE_V256:
1351 /* Recall that ARM SVE allows vector sizes that are not a
1352 * power of 2, but always a multiple of 16. The intent is
1353 * that e.g. size == 80 would be expanded with 2x32 + 1x16.
1354 */
1355 some = QEMU_ALIGN_DOWN(oprsz, 32);
db432672 1356 expand_4_vec(g->vece, dofs, aofs, bofs, cofs, some,
5d6acdd4 1357 32, TCG_TYPE_V256, g->write_aofs, g->fniv);
db432672 1358 if (some == oprsz) {
adb196cb 1359 break;
db432672
RH		 1360 }
1361 dofs += some;
1362 aofs += some;
1363 bofs += some;
1364 cofs += some;
1365 oprsz -= some;
1366 maxsz -= some;
adb196cb
RH		 1367 /* fallthru */
1368 case TCG_TYPE_V128:
db432672 1369 expand_4_vec(g->vece, dofs, aofs, bofs, cofs, oprsz,
5d6acdd4 1370 16, TCG_TYPE_V128, g->write_aofs, g->fniv);
adb196cb
RH		 1371 break;
1372 case TCG_TYPE_V64:
db432672 1373 expand_4_vec(g->vece, dofs, aofs, bofs, cofs, oprsz,
5d6acdd4 1374 8, TCG_TYPE_V64, g->write_aofs, g->fniv);
adb196cb
RH		 1375 break;
1376
1377 case 0:
1378 if (g->fni8 && check_size_impl(oprsz, 8)) {
5d6acdd4
RH		 1379 expand_4_i64(dofs, aofs, bofs, cofs, oprsz,
1380 g->write_aofs, g->fni8);
adb196cb 1381 } else if (g->fni4 && check_size_impl(oprsz, 4)) {
5d6acdd4
RH		 1382 expand_4_i32(dofs, aofs, bofs, cofs, oprsz,
1383 g->write_aofs, g->fni4);
adb196cb
RH		 1384 } else {
1385 assert(g->fno != NULL);
1386 tcg_gen_gvec_4_ool(dofs, aofs, bofs, cofs,
1387 oprsz, maxsz, g->data, g->fno);
53229a77 1388 oprsz = maxsz;
adb196cb
RH		 1389 }
1390 break;
1391
1392 default:
1393 g_assert_not_reached();
db432672 1394 }
53229a77 1395 tcg_swap_vecop_list(hold_list);
db432672 1396
db432672
RH		 1397 if (oprsz < maxsz) {
1398 expand_clr(dofs + oprsz, maxsz - oprsz);
1399 }
1400}
1401
1402/*
1403 * Expand specific vector operations.
1404 */
1405
1406static void vec_mov2(unsigned vece, TCGv_vec a, TCGv_vec b)
1407{
1408 tcg_gen_mov_vec(a, b);
1409}
1410
1411void tcg_gen_gvec_mov(unsigned vece, uint32_t dofs, uint32_t aofs,
1412 uint32_t oprsz, uint32_t maxsz)
1413{
1414 static const GVecGen2 g = {
1415 .fni8 = tcg_gen_mov_i64,
1416 .fniv = vec_mov2,
1417 .fno = gen_helper_gvec_mov,
1418 .prefer_i64 = TCG_TARGET_REG_BITS == 64,
1419 };
1420 if (dofs != aofs) {
1421 tcg_gen_gvec_2(dofs, aofs, oprsz, maxsz, &g);
1422 } else {
1423 check_size_align(oprsz, maxsz, dofs);
1424 if (oprsz < maxsz) {
1425 expand_clr(dofs + oprsz, maxsz - oprsz);
1426 }
1427 }
1428}
1429
1430void tcg_gen_gvec_dup_i32(unsigned vece, uint32_t dofs, uint32_t oprsz,
1431 uint32_t maxsz, TCGv_i32 in)
1432{
1433 check_size_align(oprsz, maxsz, dofs);
1434 tcg_debug_assert(vece <= MO_32);
1435 do_dup(vece, dofs, oprsz, maxsz, in, NULL, 0);
1436}
1437
1438void tcg_gen_gvec_dup_i64(unsigned vece, uint32_t dofs, uint32_t oprsz,
1439 uint32_t maxsz, TCGv_i64 in)
1440{
1441 check_size_align(oprsz, maxsz, dofs);
1442 tcg_debug_assert(vece <= MO_64);
1443 do_dup(vece, dofs, oprsz, maxsz, NULL, in, 0);
1444}
1445
1446void tcg_gen_gvec_dup_mem(unsigned vece, uint32_t dofs, uint32_t aofs,
1447 uint32_t oprsz, uint32_t maxsz)
1448{
532ba368 1449 check_size_align(oprsz, maxsz, dofs);
37ee55a0 1450 if (vece <= MO_64) {
532ba368 1451 TCGType type = choose_vector_type(NULL, vece, oprsz, 0);
37ee55a0
RH		 1452 if (type != 0) {
1453 TCGv_vec t_vec = tcg_temp_new_vec(type);
1454 tcg_gen_dup_mem_vec(vece, t_vec, cpu_env, aofs);
1455 do_dup_store(type, dofs, oprsz, maxsz, t_vec);
1456 tcg_temp_free_vec(t_vec);
532ba368
RH		 1457 } else if (vece <= MO_32) {
1458 TCGv_i32 in = tcg_temp_new_i32();
1459 switch (vece) {
1460 case MO_8:
1461 tcg_gen_ld8u_i32(in, cpu_env, aofs);
1462 break;
1463 case MO_16:
1464 tcg_gen_ld16u_i32(in, cpu_env, aofs);
1465 break;
1466 default:
1467 tcg_gen_ld_i32(in, cpu_env, aofs);
1468 break;
1469 }
1470 do_dup(vece, dofs, oprsz, maxsz, in, NULL, 0);
1471 tcg_temp_free_i32(in);
1472 } else {
1473 TCGv_i64 in = tcg_temp_new_i64();
1474 tcg_gen_ld_i64(in, cpu_env, aofs);
1475 do_dup(vece, dofs, oprsz, maxsz, NULL, in, 0);
1476 tcg_temp_free_i64(in);
db432672 1477 }
db432672
RH		 1478 } else {
1479 /* 128-bit duplicate. */
1480 /* ??? Dup to 256-bit vector. */
1481 int i;
1482
1483 tcg_debug_assert(vece == 4);
1484 tcg_debug_assert(oprsz >= 16);
1485 if (TCG_TARGET_HAS_v128) {
1486 TCGv_vec in = tcg_temp_new_vec(TCG_TYPE_V128);
1487
1488 tcg_gen_ld_vec(in, cpu_env, aofs);
1489 for (i = 0; i < oprsz; i += 16) {
1490 tcg_gen_st_vec(in, cpu_env, dofs + i);
1491 }
1492 tcg_temp_free_vec(in);
1493 } else {
1494 TCGv_i64 in0 = tcg_temp_new_i64();
1495 TCGv_i64 in1 = tcg_temp_new_i64();
1496
1497 tcg_gen_ld_i64(in0, cpu_env, aofs);
1498 tcg_gen_ld_i64(in1, cpu_env, aofs + 8);
1499 for (i = 0; i < oprsz; i += 16) {
1500 tcg_gen_st_i64(in0, cpu_env, dofs + i);
1501 tcg_gen_st_i64(in1, cpu_env, dofs + i + 8);
1502 }
1503 tcg_temp_free_i64(in0);
1504 tcg_temp_free_i64(in1);
1505 }
532ba368
RH		 1506 if (oprsz < maxsz) {
1507 expand_clr(dofs + oprsz, maxsz - oprsz);
1508 }
db432672
RH		 1509 }
1510}
1511
1512void tcg_gen_gvec_dup64i(uint32_t dofs, uint32_t oprsz,
1513 uint32_t maxsz, uint64_t x)
1514{
1515 check_size_align(oprsz, maxsz, dofs);
1516 do_dup(MO_64, dofs, oprsz, maxsz, NULL, NULL, x);
1517}
1518
1519void tcg_gen_gvec_dup32i(uint32_t dofs, uint32_t oprsz,
1520 uint32_t maxsz, uint32_t x)
1521{
1522 check_size_align(oprsz, maxsz, dofs);
1523 do_dup(MO_32, dofs, oprsz, maxsz, NULL, NULL, x);
1524}
1525
1526void tcg_gen_gvec_dup16i(uint32_t dofs, uint32_t oprsz,
1527 uint32_t maxsz, uint16_t x)
1528{
1529 check_size_align(oprsz, maxsz, dofs);
1530 do_dup(MO_16, dofs, oprsz, maxsz, NULL, NULL, x);
1531}
1532
1533void tcg_gen_gvec_dup8i(uint32_t dofs, uint32_t oprsz,
1534 uint32_t maxsz, uint8_t x)
1535{
1536 check_size_align(oprsz, maxsz, dofs);
1537 do_dup(MO_8, dofs, oprsz, maxsz, NULL, NULL, x);
1538}
1539
1540void tcg_gen_gvec_not(unsigned vece, uint32_t dofs, uint32_t aofs,
1541 uint32_t oprsz, uint32_t maxsz)
1542{
1543 static const GVecGen2 g = {
1544 .fni8 = tcg_gen_not_i64,
1545 .fniv = tcg_gen_not_vec,
1546 .fno = gen_helper_gvec_not,
1547 .prefer_i64 = TCG_TARGET_REG_BITS == 64,
1548 };
1549 tcg_gen_gvec_2(dofs, aofs, oprsz, maxsz, &g);
1550}
1551
1552/* Perform a vector addition using normal addition and a mask. The mask
1553 should be the sign bit of each lane. This 6-operation form is more
1554 efficient than separate additions when there are 4 or more lanes in
1555 the 64-bit operation. */
1556static void gen_addv_mask(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b, TCGv_i64 m)
1557{
1558 TCGv_i64 t1 = tcg_temp_new_i64();
1559 TCGv_i64 t2 = tcg_temp_new_i64();
1560 TCGv_i64 t3 = tcg_temp_new_i64();
1561
1562 tcg_gen_andc_i64(t1, a, m);
1563 tcg_gen_andc_i64(t2, b, m);
1564 tcg_gen_xor_i64(t3, a, b);
1565 tcg_gen_add_i64(d, t1, t2);
1566 tcg_gen_and_i64(t3, t3, m);
1567 tcg_gen_xor_i64(d, d, t3);
1568
1569 tcg_temp_free_i64(t1);
1570 tcg_temp_free_i64(t2);
1571 tcg_temp_free_i64(t3);
1572}
1573
1574void tcg_gen_vec_add8_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b)
1575{
1576 TCGv_i64 m = tcg_const_i64(dup_const(MO_8, 0x80));
1577 gen_addv_mask(d, a, b, m);
1578 tcg_temp_free_i64(m);
1579}
1580
1581void tcg_gen_vec_add16_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b)
1582{
1583 TCGv_i64 m = tcg_const_i64(dup_const(MO_16, 0x8000));
1584 gen_addv_mask(d, a, b, m);
1585 tcg_temp_free_i64(m);
1586}
1587
1588void tcg_gen_vec_add32_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b)
1589{
1590 TCGv_i64 t1 = tcg_temp_new_i64();
1591 TCGv_i64 t2 = tcg_temp_new_i64();
1592
1593 tcg_gen_andi_i64(t1, a, ~0xffffffffull);
1594 tcg_gen_add_i64(t2, a, b);
1595 tcg_gen_add_i64(t1, t1, b);
1596 tcg_gen_deposit_i64(d, t1, t2, 0, 32);
1597
1598 tcg_temp_free_i64(t1);
1599 tcg_temp_free_i64(t2);
1600}
1601
53229a77
RH		 1602static const TCGOpcode vecop_list_add[] = { INDEX_op_add_vec, 0 };
1603
db432672
RH		 1604void tcg_gen_gvec_add(unsigned vece, uint32_t dofs, uint32_t aofs,
1605 uint32_t bofs, uint32_t oprsz, uint32_t maxsz)
1606{
1607 static const GVecGen3 g[4] = {
1608 { .fni8 = tcg_gen_vec_add8_i64,
1609 .fniv = tcg_gen_add_vec,
1610 .fno = gen_helper_gvec_add8,
53229a77 1611 .opt_opc = vecop_list_add,
db432672
RH		 1612 .vece = MO_8 },
1613 { .fni8 = tcg_gen_vec_add16_i64,
1614 .fniv = tcg_gen_add_vec,
1615 .fno = gen_helper_gvec_add16,
53229a77 1616 .opt_opc = vecop_list_add,
db432672
RH		 1617 .vece = MO_16 },
1618 { .fni4 = tcg_gen_add_i32,
1619 .fniv = tcg_gen_add_vec,
1620 .fno = gen_helper_gvec_add32,
53229a77 1621 .opt_opc = vecop_list_add,
db432672
RH		 1622 .vece = MO_32 },
1623 { .fni8 = tcg_gen_add_i64,
1624 .fniv = tcg_gen_add_vec,
1625 .fno = gen_helper_gvec_add64,
53229a77 1626 .opt_opc = vecop_list_add,
db432672
RH		 1627 .prefer_i64 = TCG_TARGET_REG_BITS == 64,
1628 .vece = MO_64 },
1629 };
1630
1631 tcg_debug_assert(vece <= MO_64);
1632 tcg_gen_gvec_3(dofs, aofs, bofs, oprsz, maxsz, &g[vece]);
1633}
1634
22fc3527
RH		 1635void tcg_gen_gvec_adds(unsigned vece, uint32_t dofs, uint32_t aofs,
1636 TCGv_i64 c, uint32_t oprsz, uint32_t maxsz)
1637{
1638 static const GVecGen2s g[4] = {
1639 { .fni8 = tcg_gen_vec_add8_i64,
1640 .fniv = tcg_gen_add_vec,
1641 .fno = gen_helper_gvec_adds8,
53229a77 1642 .opt_opc = vecop_list_add,
22fc3527
RH		 1643 .vece = MO_8 },
1644 { .fni8 = tcg_gen_vec_add16_i64,
1645 .fniv = tcg_gen_add_vec,
1646 .fno = gen_helper_gvec_adds16,
53229a77 1647 .opt_opc = vecop_list_add,
22fc3527
RH		 1648 .vece = MO_16 },
1649 { .fni4 = tcg_gen_add_i32,
1650 .fniv = tcg_gen_add_vec,
1651 .fno = gen_helper_gvec_adds32,
53229a77 1652 .opt_opc = vecop_list_add,
22fc3527
RH		 1653 .vece = MO_32 },
1654 { .fni8 = tcg_gen_add_i64,
1655 .fniv = tcg_gen_add_vec,
1656 .fno = gen_helper_gvec_adds64,
53229a77 1657 .opt_opc = vecop_list_add,
22fc3527
RH		 1658 .prefer_i64 = TCG_TARGET_REG_BITS == 64,
1659 .vece = MO_64 },
1660 };
1661
1662 tcg_debug_assert(vece <= MO_64);
1663 tcg_gen_gvec_2s(dofs, aofs, oprsz, maxsz, c, &g[vece]);
1664}
1665
1666void tcg_gen_gvec_addi(unsigned vece, uint32_t dofs, uint32_t aofs,
1667 int64_t c, uint32_t oprsz, uint32_t maxsz)
1668{
1669 TCGv_i64 tmp = tcg_const_i64(c);
1670 tcg_gen_gvec_adds(vece, dofs, aofs, tmp, oprsz, maxsz);
1671 tcg_temp_free_i64(tmp);
1672}
1673
53229a77
RH		 1674static const TCGOpcode vecop_list_sub[] = { INDEX_op_sub_vec, 0 };
1675
22fc3527
RH		 1676void tcg_gen_gvec_subs(unsigned vece, uint32_t dofs, uint32_t aofs,
1677 TCGv_i64 c, uint32_t oprsz, uint32_t maxsz)
1678{
1679 static const GVecGen2s g[4] = {
1680 { .fni8 = tcg_gen_vec_sub8_i64,
1681 .fniv = tcg_gen_sub_vec,
1682 .fno = gen_helper_gvec_subs8,
53229a77 1683 .opt_opc = vecop_list_sub,
22fc3527
RH		 1684 .vece = MO_8 },
1685 { .fni8 = tcg_gen_vec_sub16_i64,
1686 .fniv = tcg_gen_sub_vec,
1687 .fno = gen_helper_gvec_subs16,
53229a77 1688 .opt_opc = vecop_list_sub,
22fc3527
RH		 1689 .vece = MO_16 },
1690 { .fni4 = tcg_gen_sub_i32,
1691 .fniv = tcg_gen_sub_vec,
1692 .fno = gen_helper_gvec_subs32,
53229a77 1693 .opt_opc = vecop_list_sub,
22fc3527
RH		 1694 .vece = MO_32 },
1695 { .fni8 = tcg_gen_sub_i64,
1696 .fniv = tcg_gen_sub_vec,
1697 .fno = gen_helper_gvec_subs64,
53229a77 1698 .opt_opc = vecop_list_sub,
22fc3527
RH		 1699 .prefer_i64 = TCG_TARGET_REG_BITS == 64,
1700 .vece = MO_64 },
1701 };
1702
1703 tcg_debug_assert(vece <= MO_64);
1704 tcg_gen_gvec_2s(dofs, aofs, oprsz, maxsz, c, &g[vece]);
1705}
1706
db432672
RH		 1707/* Perform a vector subtraction using normal subtraction and a mask.
1708 Compare gen_addv_mask above. */
1709static void gen_subv_mask(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b, TCGv_i64 m)
1710{
1711 TCGv_i64 t1 = tcg_temp_new_i64();
1712 TCGv_i64 t2 = tcg_temp_new_i64();
1713 TCGv_i64 t3 = tcg_temp_new_i64();
1714
1715 tcg_gen_or_i64(t1, a, m);
1716 tcg_gen_andc_i64(t2, b, m);
1717 tcg_gen_eqv_i64(t3, a, b);
1718 tcg_gen_sub_i64(d, t1, t2);
1719 tcg_gen_and_i64(t3, t3, m);
1720 tcg_gen_xor_i64(d, d, t3);
1721
1722 tcg_temp_free_i64(t1);
1723 tcg_temp_free_i64(t2);
1724 tcg_temp_free_i64(t3);
1725}
1726
1727void tcg_gen_vec_sub8_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b)
1728{
1729 TCGv_i64 m = tcg_const_i64(dup_const(MO_8, 0x80));
1730 gen_subv_mask(d, a, b, m);
1731 tcg_temp_free_i64(m);
1732}
1733
1734void tcg_gen_vec_sub16_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b)
1735{
1736 TCGv_i64 m = tcg_const_i64(dup_const(MO_16, 0x8000));
1737 gen_subv_mask(d, a, b, m);
1738 tcg_temp_free_i64(m);
1739}
1740
1741void tcg_gen_vec_sub32_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b)
1742{
1743 TCGv_i64 t1 = tcg_temp_new_i64();
1744 TCGv_i64 t2 = tcg_temp_new_i64();
1745
1746 tcg_gen_andi_i64(t1, b, ~0xffffffffull);
1747 tcg_gen_sub_i64(t2, a, b);
1748 tcg_gen_sub_i64(t1, a, t1);
1749 tcg_gen_deposit_i64(d, t1, t2, 0, 32);
1750
1751 tcg_temp_free_i64(t1);
1752 tcg_temp_free_i64(t2);
1753}
1754
1755void tcg_gen_gvec_sub(unsigned vece, uint32_t dofs, uint32_t aofs,
1756 uint32_t bofs, uint32_t oprsz, uint32_t maxsz)
1757{
1758 static const GVecGen3 g[4] = {
1759 { .fni8 = tcg_gen_vec_sub8_i64,
1760 .fniv = tcg_gen_sub_vec,
1761 .fno = gen_helper_gvec_sub8,
53229a77 1762 .opt_opc = vecop_list_sub,
db432672
RH		 1763 .vece = MO_8 },
1764 { .fni8 = tcg_gen_vec_sub16_i64,
1765 .fniv = tcg_gen_sub_vec,
1766 .fno = gen_helper_gvec_sub16,
53229a77 1767 .opt_opc = vecop_list_sub,
db432672
RH		 1768 .vece = MO_16 },
1769 { .fni4 = tcg_gen_sub_i32,
1770 .fniv = tcg_gen_sub_vec,
1771 .fno = gen_helper_gvec_sub32,
53229a77 1772 .opt_opc = vecop_list_sub,
db432672
RH		 1773 .vece = MO_32 },
1774 { .fni8 = tcg_gen_sub_i64,
1775 .fniv = tcg_gen_sub_vec,
1776 .fno = gen_helper_gvec_sub64,
53229a77 1777 .opt_opc = vecop_list_sub,
db432672
RH		 1778 .prefer_i64 = TCG_TARGET_REG_BITS == 64,
1779 .vece = MO_64 },
1780 };
1781
1782 tcg_debug_assert(vece <= MO_64);
1783 tcg_gen_gvec_3(dofs, aofs, bofs, oprsz, maxsz, &g[vece]);
1784}
1785
53229a77
RH		 1786static const TCGOpcode vecop_list_mul[] = { INDEX_op_mul_vec, 0 };
1787
3774030a
RH		 1788void tcg_gen_gvec_mul(unsigned vece, uint32_t dofs, uint32_t aofs,
1789 uint32_t bofs, uint32_t oprsz, uint32_t maxsz)
1790{
1791 static const GVecGen3 g[4] = {
1792 { .fniv = tcg_gen_mul_vec,
1793 .fno = gen_helper_gvec_mul8,
53229a77 1794 .opt_opc = vecop_list_mul,
3774030a
RH		 1795 .vece = MO_8 },
1796 { .fniv = tcg_gen_mul_vec,
1797 .fno = gen_helper_gvec_mul16,
53229a77 1798 .opt_opc = vecop_list_mul,
3774030a
RH		 1799 .vece = MO_16 },
1800 { .fni4 = tcg_gen_mul_i32,
1801 .fniv = tcg_gen_mul_vec,
1802 .fno = gen_helper_gvec_mul32,
53229a77 1803 .opt_opc = vecop_list_mul,
3774030a
RH		 1804 .vece = MO_32 },
1805 { .fni8 = tcg_gen_mul_i64,
1806 .fniv = tcg_gen_mul_vec,
1807 .fno = gen_helper_gvec_mul64,
53229a77 1808 .opt_opc = vecop_list_mul,
3774030a
RH		 1809 .prefer_i64 = TCG_TARGET_REG_BITS == 64,
1810 .vece = MO_64 },
1811 };
1812
1813 tcg_debug_assert(vece <= MO_64);
1814 tcg_gen_gvec_3(dofs, aofs, bofs, oprsz, maxsz, &g[vece]);
1815}
1816
22fc3527
RH		 1817void tcg_gen_gvec_muls(unsigned vece, uint32_t dofs, uint32_t aofs,
1818 TCGv_i64 c, uint32_t oprsz, uint32_t maxsz)
1819{
1820 static const GVecGen2s g[4] = {
1821 { .fniv = tcg_gen_mul_vec,
1822 .fno = gen_helper_gvec_muls8,
53229a77 1823 .opt_opc = vecop_list_mul,
22fc3527
RH		 1824 .vece = MO_8 },
1825 { .fniv = tcg_gen_mul_vec,
1826 .fno = gen_helper_gvec_muls16,
53229a77 1827 .opt_opc = vecop_list_mul,
22fc3527
RH		 1828 .vece = MO_16 },
1829 { .fni4 = tcg_gen_mul_i32,
1830 .fniv = tcg_gen_mul_vec,
1831 .fno = gen_helper_gvec_muls32,
53229a77 1832 .opt_opc = vecop_list_mul,
22fc3527
RH		 1833 .vece = MO_32 },
1834 { .fni8 = tcg_gen_mul_i64,
1835 .fniv = tcg_gen_mul_vec,
1836 .fno = gen_helper_gvec_muls64,
53229a77 1837 .opt_opc = vecop_list_mul,
22fc3527
RH		 1838 .prefer_i64 = TCG_TARGET_REG_BITS == 64,
1839 .vece = MO_64 },
1840 };
1841
1842 tcg_debug_assert(vece <= MO_64);
1843 tcg_gen_gvec_2s(dofs, aofs, oprsz, maxsz, c, &g[vece]);
1844}
1845
1846void tcg_gen_gvec_muli(unsigned vece, uint32_t dofs, uint32_t aofs,
1847 int64_t c, uint32_t oprsz, uint32_t maxsz)
1848{
1849 TCGv_i64 tmp = tcg_const_i64(c);
1850 tcg_gen_gvec_muls(vece, dofs, aofs, tmp, oprsz, maxsz);
1851 tcg_temp_free_i64(tmp);
1852}
1853
f49b12c6
RH		 1854void tcg_gen_gvec_ssadd(unsigned vece, uint32_t dofs, uint32_t aofs,
1855 uint32_t bofs, uint32_t oprsz, uint32_t maxsz)
1856{
53229a77 1857 static const TCGOpcode vecop_list[] = { INDEX_op_ssadd_vec, 0 };
f49b12c6 1858 static const GVecGen3 g[4] = {
8afaf050
RH		 1859 { .fniv = tcg_gen_ssadd_vec,
1860 .fno = gen_helper_gvec_ssadd8,
53229a77 1861 .opt_opc = vecop_list,
8afaf050
RH		 1862 .vece = MO_8 },
1863 { .fniv = tcg_gen_ssadd_vec,
1864 .fno = gen_helper_gvec_ssadd16,
53229a77 1865 .opt_opc = vecop_list,
8afaf050
RH		 1866 .vece = MO_16 },
1867 { .fniv = tcg_gen_ssadd_vec,
1868 .fno = gen_helper_gvec_ssadd32,
53229a77 1869 .opt_opc = vecop_list,
8afaf050
RH		 1870 .vece = MO_32 },
1871 { .fniv = tcg_gen_ssadd_vec,
1872 .fno = gen_helper_gvec_ssadd64,
53229a77 1873 .opt_opc = vecop_list,
8afaf050 1874 .vece = MO_64 },
f49b12c6
RH		 1875 };
1876 tcg_debug_assert(vece <= MO_64);
1877 tcg_gen_gvec_3(dofs, aofs, bofs, oprsz, maxsz, &g[vece]);
1878}
1879
1880void tcg_gen_gvec_sssub(unsigned vece, uint32_t dofs, uint32_t aofs,
1881 uint32_t bofs, uint32_t oprsz, uint32_t maxsz)
1882{
53229a77 1883 static const TCGOpcode vecop_list[] = { INDEX_op_sssub_vec, 0 };
f49b12c6 1884 static const GVecGen3 g[4] = {
8afaf050
RH		 1885 { .fniv = tcg_gen_sssub_vec,
1886 .fno = gen_helper_gvec_sssub8,
53229a77 1887 .opt_opc = vecop_list,
8afaf050
RH		 1888 .vece = MO_8 },
1889 { .fniv = tcg_gen_sssub_vec,
1890 .fno = gen_helper_gvec_sssub16,
53229a77 1891 .opt_opc = vecop_list,
8afaf050
RH		 1892 .vece = MO_16 },
1893 { .fniv = tcg_gen_sssub_vec,
1894 .fno = gen_helper_gvec_sssub32,
53229a77 1895 .opt_opc = vecop_list,
8afaf050
RH		 1896 .vece = MO_32 },
1897 { .fniv = tcg_gen_sssub_vec,
1898 .fno = gen_helper_gvec_sssub64,
53229a77 1899 .opt_opc = vecop_list,
8afaf050 1900 .vece = MO_64 },
f49b12c6
RH		 1901 };
1902 tcg_debug_assert(vece <= MO_64);
1903 tcg_gen_gvec_3(dofs, aofs, bofs, oprsz, maxsz, &g[vece]);
1904}
1905
8afaf050 1906static void tcg_gen_usadd_i32(TCGv_i32 d, TCGv_i32 a, TCGv_i32 b)
f49b12c6
RH		 1907{
1908 TCGv_i32 max = tcg_const_i32(-1);
1909 tcg_gen_add_i32(d, a, b);
1910 tcg_gen_movcond_i32(TCG_COND_LTU, d, d, a, max, d);
1911 tcg_temp_free_i32(max);
1912}
1913
8afaf050 1914static void tcg_gen_usadd_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b)
f49b12c6
RH		 1915{
1916 TCGv_i64 max = tcg_const_i64(-1);
1917 tcg_gen_add_i64(d, a, b);
1918 tcg_gen_movcond_i64(TCG_COND_LTU, d, d, a, max, d);
1919 tcg_temp_free_i64(max);
1920}
1921
1922void tcg_gen_gvec_usadd(unsigned vece, uint32_t dofs, uint32_t aofs,
1923 uint32_t bofs, uint32_t oprsz, uint32_t maxsz)
1924{
53229a77 1925 static const TCGOpcode vecop_list[] = { INDEX_op_usadd_vec, 0 };
f49b12c6 1926 static const GVecGen3 g[4] = {
8afaf050
RH		 1927 { .fniv = tcg_gen_usadd_vec,
1928 .fno = gen_helper_gvec_usadd8,
53229a77 1929 .opt_opc = vecop_list,
8afaf050
RH		 1930 .vece = MO_8 },
1931 { .fniv = tcg_gen_usadd_vec,
1932 .fno = gen_helper_gvec_usadd16,
53229a77 1933 .opt_opc = vecop_list,
8afaf050
RH		 1934 .vece = MO_16 },
1935 { .fni4 = tcg_gen_usadd_i32,
1936 .fniv = tcg_gen_usadd_vec,
f49b12c6 1937 .fno = gen_helper_gvec_usadd32,
53229a77 1938 .opt_opc = vecop_list,
f49b12c6 1939 .vece = MO_32 },
8afaf050
RH		 1940 { .fni8 = tcg_gen_usadd_i64,
1941 .fniv = tcg_gen_usadd_vec,
f49b12c6 1942 .fno = gen_helper_gvec_usadd64,
53229a77 1943 .opt_opc = vecop_list,
f49b12c6
RH		 1944 .vece = MO_64 }
1945 };
1946 tcg_debug_assert(vece <= MO_64);
1947 tcg_gen_gvec_3(dofs, aofs, bofs, oprsz, maxsz, &g[vece]);
1948}
1949
8afaf050 1950static void tcg_gen_ussub_i32(TCGv_i32 d, TCGv_i32 a, TCGv_i32 b)
f49b12c6
RH		 1951{
1952 TCGv_i32 min = tcg_const_i32(0);
1953 tcg_gen_sub_i32(d, a, b);
1954 tcg_gen_movcond_i32(TCG_COND_LTU, d, a, b, min, d);
1955 tcg_temp_free_i32(min);
1956}
1957
8afaf050 1958static void tcg_gen_ussub_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b)
f49b12c6
RH		 1959{
1960 TCGv_i64 min = tcg_const_i64(0);
1961 tcg_gen_sub_i64(d, a, b);
1962 tcg_gen_movcond_i64(TCG_COND_LTU, d, a, b, min, d);
1963 tcg_temp_free_i64(min);
1964}
1965
1966void tcg_gen_gvec_ussub(unsigned vece, uint32_t dofs, uint32_t aofs,
1967 uint32_t bofs, uint32_t oprsz, uint32_t maxsz)
1968{
53229a77 1969 static const TCGOpcode vecop_list[] = { INDEX_op_ussub_vec, 0 };
f49b12c6 1970 static const GVecGen3 g[4] = {
8afaf050
RH		 1971 { .fniv = tcg_gen_ussub_vec,
1972 .fno = gen_helper_gvec_ussub8,
53229a77 1973 .opt_opc = vecop_list,
8afaf050
RH		 1974 .vece = MO_8 },
1975 { .fniv = tcg_gen_ussub_vec,
1976 .fno = gen_helper_gvec_ussub16,
53229a77 1977 .opt_opc = vecop_list,
8afaf050
RH		 1978 .vece = MO_16 },
1979 { .fni4 = tcg_gen_ussub_i32,
1980 .fniv = tcg_gen_ussub_vec,
f49b12c6 1981 .fno = gen_helper_gvec_ussub32,
53229a77 1982 .opt_opc = vecop_list,
f49b12c6 1983 .vece = MO_32 },
8afaf050
RH		 1984 { .fni8 = tcg_gen_ussub_i64,
1985 .fniv = tcg_gen_ussub_vec,
f49b12c6 1986 .fno = gen_helper_gvec_ussub64,
53229a77 1987 .opt_opc = vecop_list,
f49b12c6
RH		 1988 .vece = MO_64 }
1989 };
1990 tcg_debug_assert(vece <= MO_64);
1991 tcg_gen_gvec_3(dofs, aofs, bofs, oprsz, maxsz, &g[vece]);
dd0a0fcd
RH		 1992}
1993
1994void tcg_gen_gvec_smin(unsigned vece, uint32_t dofs, uint32_t aofs,
1995 uint32_t bofs, uint32_t oprsz, uint32_t maxsz)
1996{
53229a77 1997 static const TCGOpcode vecop_list[] = { INDEX_op_smin_vec, 0 };
dd0a0fcd
RH		 1998 static const GVecGen3 g[4] = {
1999 { .fniv = tcg_gen_smin_vec,
2000 .fno = gen_helper_gvec_smin8,
53229a77 2001 .opt_opc = vecop_list,
dd0a0fcd
RH		 2002 .vece = MO_8 },
2003 { .fniv = tcg_gen_smin_vec,
2004 .fno = gen_helper_gvec_smin16,
53229a77 2005 .opt_opc = vecop_list,
dd0a0fcd
RH		 2006 .vece = MO_16 },
2007 { .fni4 = tcg_gen_smin_i32,
2008 .fniv = tcg_gen_smin_vec,
2009 .fno = gen_helper_gvec_smin32,
53229a77 2010 .opt_opc = vecop_list,
dd0a0fcd
RH		 2011 .vece = MO_32 },
2012 { .fni8 = tcg_gen_smin_i64,
2013 .fniv = tcg_gen_smin_vec,
2014 .fno = gen_helper_gvec_smin64,
53229a77 2015 .opt_opc = vecop_list,
dd0a0fcd
RH		 2016 .vece = MO_64 }
2017 };
2018 tcg_debug_assert(vece <= MO_64);
2019 tcg_gen_gvec_3(dofs, aofs, bofs, oprsz, maxsz, &g[vece]);
2020}
2021
2022void tcg_gen_gvec_umin(unsigned vece, uint32_t dofs, uint32_t aofs,
2023 uint32_t bofs, uint32_t oprsz, uint32_t maxsz)
2024{
53229a77 2025 static const TCGOpcode vecop_list[] = { INDEX_op_umin_vec, 0 };
dd0a0fcd
RH		 2026 static const GVecGen3 g[4] = {
2027 { .fniv = tcg_gen_umin_vec,
2028 .fno = gen_helper_gvec_umin8,
53229a77 2029 .opt_opc = vecop_list,
dd0a0fcd
RH		 2030 .vece = MO_8 },
2031 { .fniv = tcg_gen_umin_vec,
2032 .fno = gen_helper_gvec_umin16,
53229a77 2033 .opt_opc = vecop_list,
dd0a0fcd
RH		 2034 .vece = MO_16 },
2035 { .fni4 = tcg_gen_umin_i32,
2036 .fniv = tcg_gen_umin_vec,
2037 .fno = gen_helper_gvec_umin32,
53229a77 2038 .opt_opc = vecop_list,
dd0a0fcd
RH		 2039 .vece = MO_32 },
2040 { .fni8 = tcg_gen_umin_i64,
2041 .fniv = tcg_gen_umin_vec,
2042 .fno = gen_helper_gvec_umin64,
53229a77 2043 .opt_opc = vecop_list,
dd0a0fcd
RH		 2044 .vece = MO_64 }
2045 };
2046 tcg_debug_assert(vece <= MO_64);
2047 tcg_gen_gvec_3(dofs, aofs, bofs, oprsz, maxsz, &g[vece]);
2048}
2049
2050void tcg_gen_gvec_smax(unsigned vece, uint32_t dofs, uint32_t aofs,
2051 uint32_t bofs, uint32_t oprsz, uint32_t maxsz)
2052{
53229a77 2053 static const TCGOpcode vecop_list[] = { INDEX_op_smax_vec, 0 };
dd0a0fcd
RH		 2054 static const GVecGen3 g[4] = {
2055 { .fniv = tcg_gen_smax_vec,
2056 .fno = gen_helper_gvec_smax8,
53229a77 2057 .opt_opc = vecop_list,
dd0a0fcd
RH		 2058 .vece = MO_8 },
2059 { .fniv = tcg_gen_smax_vec,
2060 .fno = gen_helper_gvec_smax16,
53229a77 2061 .opt_opc = vecop_list,
dd0a0fcd
RH		 2062 .vece = MO_16 },
2063 { .fni4 = tcg_gen_smax_i32,
2064 .fniv = tcg_gen_smax_vec,
2065 .fno = gen_helper_gvec_smax32,
53229a77 2066 .opt_opc = vecop_list,
dd0a0fcd
RH		 2067 .vece = MO_32 },
2068 { .fni8 = tcg_gen_smax_i64,
2069 .fniv = tcg_gen_smax_vec,
2070 .fno = gen_helper_gvec_smax64,
53229a77 2071 .opt_opc = vecop_list,
dd0a0fcd
RH		 2072 .vece = MO_64 }
2073 };
2074 tcg_debug_assert(vece <= MO_64);
2075 tcg_gen_gvec_3(dofs, aofs, bofs, oprsz, maxsz, &g[vece]);
2076}
2077
2078void tcg_gen_gvec_umax(unsigned vece, uint32_t dofs, uint32_t aofs,
2079 uint32_t bofs, uint32_t oprsz, uint32_t maxsz)
2080{
53229a77 2081 static const TCGOpcode vecop_list[] = { INDEX_op_umax_vec, 0 };
dd0a0fcd
RH		 2082 static const GVecGen3 g[4] = {
2083 { .fniv = tcg_gen_umax_vec,
2084 .fno = gen_helper_gvec_umax8,
53229a77 2085 .opt_opc = vecop_list,
dd0a0fcd
RH		 2086 .vece = MO_8 },
2087 { .fniv = tcg_gen_umax_vec,
2088 .fno = gen_helper_gvec_umax16,
53229a77 2089 .opt_opc = vecop_list,
dd0a0fcd
RH		 2090 .vece = MO_16 },
2091 { .fni4 = tcg_gen_umax_i32,
2092 .fniv = tcg_gen_umax_vec,
2093 .fno = gen_helper_gvec_umax32,
53229a77 2094 .opt_opc = vecop_list,
dd0a0fcd
RH		 2095 .vece = MO_32 },
2096 { .fni8 = tcg_gen_umax_i64,
2097 .fniv = tcg_gen_umax_vec,
2098 .fno = gen_helper_gvec_umax64,
53229a77 2099 .opt_opc = vecop_list,
dd0a0fcd
RH		 2100 .vece = MO_64 }
2101 };
2102 tcg_debug_assert(vece <= MO_64);
2103 tcg_gen_gvec_3(dofs, aofs, bofs, oprsz, maxsz, &g[vece]);
f49b12c6
RH		 2104}
2105
db432672
RH		 2106/* Perform a vector negation using normal negation and a mask.
2107 Compare gen_subv_mask above. */
2108static void gen_negv_mask(TCGv_i64 d, TCGv_i64 b, TCGv_i64 m)
2109{
2110 TCGv_i64 t2 = tcg_temp_new_i64();
2111 TCGv_i64 t3 = tcg_temp_new_i64();
2112
2113 tcg_gen_andc_i64(t3, m, b);
2114 tcg_gen_andc_i64(t2, b, m);
2115 tcg_gen_sub_i64(d, m, t2);
2116 tcg_gen_xor_i64(d, d, t3);
2117
2118 tcg_temp_free_i64(t2);
2119 tcg_temp_free_i64(t3);
2120}
2121
2122void tcg_gen_vec_neg8_i64(TCGv_i64 d, TCGv_i64 b)
2123{
2124 TCGv_i64 m = tcg_const_i64(dup_const(MO_8, 0x80));
2125 gen_negv_mask(d, b, m);
2126 tcg_temp_free_i64(m);
2127}
2128
2129void tcg_gen_vec_neg16_i64(TCGv_i64 d, TCGv_i64 b)
2130{
2131 TCGv_i64 m = tcg_const_i64(dup_const(MO_16, 0x8000));
2132 gen_negv_mask(d, b, m);
2133 tcg_temp_free_i64(m);
2134}
2135
2136void tcg_gen_vec_neg32_i64(TCGv_i64 d, TCGv_i64 b)
2137{
2138 TCGv_i64 t1 = tcg_temp_new_i64();
2139 TCGv_i64 t2 = tcg_temp_new_i64();
2140
2141 tcg_gen_andi_i64(t1, b, ~0xffffffffull);
2142 tcg_gen_neg_i64(t2, b);
2143 tcg_gen_neg_i64(t1, t1);
2144 tcg_gen_deposit_i64(d, t1, t2, 0, 32);
2145
2146 tcg_temp_free_i64(t1);
2147 tcg_temp_free_i64(t2);
2148}
2149
2150void tcg_gen_gvec_neg(unsigned vece, uint32_t dofs, uint32_t aofs,
2151 uint32_t oprsz, uint32_t maxsz)
2152{
53229a77 2153 static const TCGOpcode vecop_list[] = { INDEX_op_neg_vec, 0 };
db432672
RH		 2154 static const GVecGen2 g[4] = {
2155 { .fni8 = tcg_gen_vec_neg8_i64,
2156 .fniv = tcg_gen_neg_vec,
2157 .fno = gen_helper_gvec_neg8,
53229a77 2158 .opt_opc = vecop_list,
db432672
RH		 2159 .vece = MO_8 },
2160 { .fni8 = tcg_gen_vec_neg16_i64,
2161 .fniv = tcg_gen_neg_vec,
2162 .fno = gen_helper_gvec_neg16,
53229a77 2163 .opt_opc = vecop_list,
db432672
RH		 2164 .vece = MO_16 },
2165 { .fni4 = tcg_gen_neg_i32,
2166 .fniv = tcg_gen_neg_vec,
2167 .fno = gen_helper_gvec_neg32,
53229a77 2168 .opt_opc = vecop_list,
db432672
RH		 2169 .vece = MO_32 },
2170 { .fni8 = tcg_gen_neg_i64,
2171 .fniv = tcg_gen_neg_vec,
2172 .fno = gen_helper_gvec_neg64,
53229a77 2173 .opt_opc = vecop_list,
db432672
RH		 2174 .prefer_i64 = TCG_TARGET_REG_BITS == 64,
2175 .vece = MO_64 },
2176 };
2177
2178 tcg_debug_assert(vece <= MO_64);
2179 tcg_gen_gvec_2(dofs, aofs, oprsz, maxsz, &g[vece]);
2180}
2181
bcefc902
RH		 2182static void gen_absv_mask(TCGv_i64 d, TCGv_i64 b, unsigned vece)
2183{
2184 TCGv_i64 t = tcg_temp_new_i64();
2185 int nbit = 8 << vece;
2186
2187 /* Create -1 for each negative element. */
2188 tcg_gen_shri_i64(t, b, nbit - 1);
2189 tcg_gen_andi_i64(t, t, dup_const(vece, 1));
2190 tcg_gen_muli_i64(t, t, (1 << nbit) - 1);
2191
2192 /*
2193 * Invert (via xor -1) and add one (via sub -1).
2194 * Because of the ordering the msb is cleared,
2195 * so we never have carry into the next element.
2196 */
2197 tcg_gen_xor_i64(d, b, t);
2198 tcg_gen_sub_i64(d, d, t);
2199
2200 tcg_temp_free_i64(t);
2201}
2202
2203static void tcg_gen_vec_abs8_i64(TCGv_i64 d, TCGv_i64 b)
2204{
2205 gen_absv_mask(d, b, MO_8);
2206}
2207
2208static void tcg_gen_vec_abs16_i64(TCGv_i64 d, TCGv_i64 b)
2209{
2210 gen_absv_mask(d, b, MO_16);
2211}
2212
2213void tcg_gen_gvec_abs(unsigned vece, uint32_t dofs, uint32_t aofs,
2214 uint32_t oprsz, uint32_t maxsz)
2215{
2216 static const TCGOpcode vecop_list[] = { INDEX_op_abs_vec, 0 };
2217 static const GVecGen2 g[4] = {
2218 { .fni8 = tcg_gen_vec_abs8_i64,
2219 .fniv = tcg_gen_abs_vec,
2220 .fno = gen_helper_gvec_abs8,
2221 .opt_opc = vecop_list,
2222 .vece = MO_8 },
2223 { .fni8 = tcg_gen_vec_abs16_i64,
2224 .fniv = tcg_gen_abs_vec,
2225 .fno = gen_helper_gvec_abs16,
2226 .opt_opc = vecop_list,
2227 .vece = MO_16 },
2228 { .fni4 = tcg_gen_abs_i32,
2229 .fniv = tcg_gen_abs_vec,
2230 .fno = gen_helper_gvec_abs32,
2231 .opt_opc = vecop_list,
2232 .vece = MO_32 },
2233 { .fni8 = tcg_gen_abs_i64,
2234 .fniv = tcg_gen_abs_vec,
2235 .fno = gen_helper_gvec_abs64,
2236 .opt_opc = vecop_list,
2237 .prefer_i64 = TCG_TARGET_REG_BITS == 64,
2238 .vece = MO_64 },
2239 };
2240
2241 tcg_debug_assert(vece <= MO_64);
2242 tcg_gen_gvec_2(dofs, aofs, oprsz, maxsz, &g[vece]);
2243}
2244
db432672
RH		 2245void tcg_gen_gvec_and(unsigned vece, uint32_t dofs, uint32_t aofs,
2246 uint32_t bofs, uint32_t oprsz, uint32_t maxsz)
2247{
2248 static const GVecGen3 g = {
2249 .fni8 = tcg_gen_and_i64,
2250 .fniv = tcg_gen_and_vec,
2251 .fno = gen_helper_gvec_and,
db432672
RH		 2252 .prefer_i64 = TCG_TARGET_REG_BITS == 64,
2253 };
9a9eda78
RH		 2254
2255 if (aofs == bofs) {
2256 tcg_gen_gvec_mov(vece, dofs, aofs, oprsz, maxsz);
2257 } else {
2258 tcg_gen_gvec_3(dofs, aofs, bofs, oprsz, maxsz, &g);
2259 }
db432672
RH		 2260}
2261
2262void tcg_gen_gvec_or(unsigned vece, uint32_t dofs, uint32_t aofs,
2263 uint32_t bofs, uint32_t oprsz, uint32_t maxsz)
2264{
2265 static const GVecGen3 g = {
2266 .fni8 = tcg_gen_or_i64,
2267 .fniv = tcg_gen_or_vec,
2268 .fno = gen_helper_gvec_or,
db432672
RH		 2269 .prefer_i64 = TCG_TARGET_REG_BITS == 64,
2270 };
9a9eda78
RH		 2271
2272 if (aofs == bofs) {
2273 tcg_gen_gvec_mov(vece, dofs, aofs, oprsz, maxsz);
2274 } else {
2275 tcg_gen_gvec_3(dofs, aofs, bofs, oprsz, maxsz, &g);
2276 }
db432672
RH		 2277}
2278
2279void tcg_gen_gvec_xor(unsigned vece, uint32_t dofs, uint32_t aofs,
2280 uint32_t bofs, uint32_t oprsz, uint32_t maxsz)
2281{
2282 static const GVecGen3 g = {
2283 .fni8 = tcg_gen_xor_i64,
2284 .fniv = tcg_gen_xor_vec,
2285 .fno = gen_helper_gvec_xor,
db432672
RH		 2286 .prefer_i64 = TCG_TARGET_REG_BITS == 64,
2287 };
9a9eda78
RH		 2288
2289 if (aofs == bofs) {
2290 tcg_gen_gvec_dup8i(dofs, oprsz, maxsz, 0);
2291 } else {
2292 tcg_gen_gvec_3(dofs, aofs, bofs, oprsz, maxsz, &g);
2293 }
db432672
RH		 2294}
2295
2296void tcg_gen_gvec_andc(unsigned vece, uint32_t dofs, uint32_t aofs,
2297 uint32_t bofs, uint32_t oprsz, uint32_t maxsz)
2298{
2299 static const GVecGen3 g = {
2300 .fni8 = tcg_gen_andc_i64,
2301 .fniv = tcg_gen_andc_vec,
2302 .fno = gen_helper_gvec_andc,
db432672
RH		 2303 .prefer_i64 = TCG_TARGET_REG_BITS == 64,
2304 };
9a9eda78
RH		 2305
2306 if (aofs == bofs) {
2307 tcg_gen_gvec_dup8i(dofs, oprsz, maxsz, 0);
2308 } else {
2309 tcg_gen_gvec_3(dofs, aofs, bofs, oprsz, maxsz, &g);
2310 }
db432672
RH		 2311}
2312
2313void tcg_gen_gvec_orc(unsigned vece, uint32_t dofs, uint32_t aofs,
2314 uint32_t bofs, uint32_t oprsz, uint32_t maxsz)
2315{
2316 static const GVecGen3 g = {
2317 .fni8 = tcg_gen_orc_i64,
2318 .fniv = tcg_gen_orc_vec,
2319 .fno = gen_helper_gvec_orc,
db432672
RH		 2320 .prefer_i64 = TCG_TARGET_REG_BITS == 64,
2321 };
9a9eda78
RH		 2322
2323 if (aofs == bofs) {
2324 tcg_gen_gvec_dup8i(dofs, oprsz, maxsz, -1);
2325 } else {
f550805d
RH		 2326 tcg_gen_gvec_3(dofs, aofs, bofs, oprsz, maxsz, &g);
2327 }
2328}
2329
2330void tcg_gen_gvec_nand(unsigned vece, uint32_t dofs, uint32_t aofs,
2331 uint32_t bofs, uint32_t oprsz, uint32_t maxsz)
2332{
2333 static const GVecGen3 g = {
2334 .fni8 = tcg_gen_nand_i64,
2335 .fniv = tcg_gen_nand_vec,
2336 .fno = gen_helper_gvec_nand,
2337 .prefer_i64 = TCG_TARGET_REG_BITS == 64,
2338 };
2339
2340 if (aofs == bofs) {
2341 tcg_gen_gvec_not(vece, dofs, aofs, oprsz, maxsz);
2342 } else {
2343 tcg_gen_gvec_3(dofs, aofs, bofs, oprsz, maxsz, &g);
2344 }
2345}
2346
2347void tcg_gen_gvec_nor(unsigned vece, uint32_t dofs, uint32_t aofs,
2348 uint32_t bofs, uint32_t oprsz, uint32_t maxsz)
2349{
2350 static const GVecGen3 g = {
2351 .fni8 = tcg_gen_nor_i64,
2352 .fniv = tcg_gen_nor_vec,
2353 .fno = gen_helper_gvec_nor,
2354 .prefer_i64 = TCG_TARGET_REG_BITS == 64,
2355 };
2356
2357 if (aofs == bofs) {
2358 tcg_gen_gvec_not(vece, dofs, aofs, oprsz, maxsz);
2359 } else {
2360 tcg_gen_gvec_3(dofs, aofs, bofs, oprsz, maxsz, &g);
2361 }
2362}
2363
2364void tcg_gen_gvec_eqv(unsigned vece, uint32_t dofs, uint32_t aofs,
2365 uint32_t bofs, uint32_t oprsz, uint32_t maxsz)
2366{
2367 static const GVecGen3 g = {
2368 .fni8 = tcg_gen_eqv_i64,
2369 .fniv = tcg_gen_eqv_vec,
2370 .fno = gen_helper_gvec_eqv,
2371 .prefer_i64 = TCG_TARGET_REG_BITS == 64,
2372 };
2373
2374 if (aofs == bofs) {
2375 tcg_gen_gvec_dup8i(dofs, oprsz, maxsz, -1);
2376 } else {
9a9eda78
RH		 2377 tcg_gen_gvec_3(dofs, aofs, bofs, oprsz, maxsz, &g);
2378 }
db432672 2379}
d0ec9796 2380
22fc3527
RH		 2381static const GVecGen2s gop_ands = {
2382 .fni8 = tcg_gen_and_i64,
2383 .fniv = tcg_gen_and_vec,
2384 .fno = gen_helper_gvec_ands,
22fc3527
RH		 2385 .prefer_i64 = TCG_TARGET_REG_BITS == 64,
2386 .vece = MO_64
2387};
2388
2389void tcg_gen_gvec_ands(unsigned vece, uint32_t dofs, uint32_t aofs,
2390 TCGv_i64 c, uint32_t oprsz, uint32_t maxsz)
2391{
2392 TCGv_i64 tmp = tcg_temp_new_i64();
2393 gen_dup_i64(vece, tmp, c);
2394 tcg_gen_gvec_2s(dofs, aofs, oprsz, maxsz, tmp, &gop_ands);
2395 tcg_temp_free_i64(tmp);
2396}
2397
2398void tcg_gen_gvec_andi(unsigned vece, uint32_t dofs, uint32_t aofs,
2399 int64_t c, uint32_t oprsz, uint32_t maxsz)
2400{
2401 TCGv_i64 tmp = tcg_const_i64(dup_const(vece, c));
2402 tcg_gen_gvec_2s(dofs, aofs, oprsz, maxsz, tmp, &gop_ands);
2403 tcg_temp_free_i64(tmp);
2404}
2405
2406static const GVecGen2s gop_xors = {
2407 .fni8 = tcg_gen_xor_i64,
2408 .fniv = tcg_gen_xor_vec,
2409 .fno = gen_helper_gvec_xors,
22fc3527
RH		 2410 .prefer_i64 = TCG_TARGET_REG_BITS == 64,
2411 .vece = MO_64
2412};
2413
2414void tcg_gen_gvec_xors(unsigned vece, uint32_t dofs, uint32_t aofs,
2415 TCGv_i64 c, uint32_t oprsz, uint32_t maxsz)
2416{
2417 TCGv_i64 tmp = tcg_temp_new_i64();
2418 gen_dup_i64(vece, tmp, c);
2419 tcg_gen_gvec_2s(dofs, aofs, oprsz, maxsz, tmp, &gop_xors);
2420 tcg_temp_free_i64(tmp);
2421}
2422
2423void tcg_gen_gvec_xori(unsigned vece, uint32_t dofs, uint32_t aofs,
2424 int64_t c, uint32_t oprsz, uint32_t maxsz)
2425{
2426 TCGv_i64 tmp = tcg_const_i64(dup_const(vece, c));
2427 tcg_gen_gvec_2s(dofs, aofs, oprsz, maxsz, tmp, &gop_xors);
2428 tcg_temp_free_i64(tmp);
2429}
2430
2431static const GVecGen2s gop_ors = {
2432 .fni8 = tcg_gen_or_i64,
2433 .fniv = tcg_gen_or_vec,
2434 .fno = gen_helper_gvec_ors,
22fc3527
RH		 2435 .prefer_i64 = TCG_TARGET_REG_BITS == 64,
2436 .vece = MO_64
2437};
2438
2439void tcg_gen_gvec_ors(unsigned vece, uint32_t dofs, uint32_t aofs,
2440 TCGv_i64 c, uint32_t oprsz, uint32_t maxsz)
2441{
2442 TCGv_i64 tmp = tcg_temp_new_i64();
2443 gen_dup_i64(vece, tmp, c);
2444 tcg_gen_gvec_2s(dofs, aofs, oprsz, maxsz, tmp, &gop_ors);
2445 tcg_temp_free_i64(tmp);
2446}
2447
2448void tcg_gen_gvec_ori(unsigned vece, uint32_t dofs, uint32_t aofs,
2449 int64_t c, uint32_t oprsz, uint32_t maxsz)
2450{
2451 TCGv_i64 tmp = tcg_const_i64(dup_const(vece, c));
2452 tcg_gen_gvec_2s(dofs, aofs, oprsz, maxsz, tmp, &gop_ors);
2453 tcg_temp_free_i64(tmp);
2454}
2455
d0ec9796
RH		 2456void tcg_gen_vec_shl8i_i64(TCGv_i64 d, TCGv_i64 a, int64_t c)
2457{
2458 uint64_t mask = dup_const(MO_8, 0xff << c);
2459 tcg_gen_shli_i64(d, a, c);
2460 tcg_gen_andi_i64(d, d, mask);
2461}
2462
2463void tcg_gen_vec_shl16i_i64(TCGv_i64 d, TCGv_i64 a, int64_t c)
2464{
2465 uint64_t mask = dup_const(MO_16, 0xffff << c);
2466 tcg_gen_shli_i64(d, a, c);
2467 tcg_gen_andi_i64(d, d, mask);
2468}
2469
2470void tcg_gen_gvec_shli(unsigned vece, uint32_t dofs, uint32_t aofs,
2471 int64_t shift, uint32_t oprsz, uint32_t maxsz)
2472{
53229a77 2473 static const TCGOpcode vecop_list[] = { INDEX_op_shli_vec, 0 };
d0ec9796
RH		 2474 static const GVecGen2i g[4] = {
2475 { .fni8 = tcg_gen_vec_shl8i_i64,
2476 .fniv = tcg_gen_shli_vec,
2477 .fno = gen_helper_gvec_shl8i,
53229a77 2478 .opt_opc = vecop_list,
d0ec9796
RH		 2479 .vece = MO_8 },
2480 { .fni8 = tcg_gen_vec_shl16i_i64,
2481 .fniv = tcg_gen_shli_vec,
2482 .fno = gen_helper_gvec_shl16i,
53229a77 2483 .opt_opc = vecop_list,
d0ec9796
RH		 2484 .vece = MO_16 },
2485 { .fni4 = tcg_gen_shli_i32,
2486 .fniv = tcg_gen_shli_vec,
2487 .fno = gen_helper_gvec_shl32i,
53229a77 2488 .opt_opc = vecop_list,
d0ec9796
RH		 2489 .vece = MO_32 },
2490 { .fni8 = tcg_gen_shli_i64,
2491 .fniv = tcg_gen_shli_vec,
2492 .fno = gen_helper_gvec_shl64i,
53229a77 2493 .opt_opc = vecop_list,
d0ec9796
RH		 2494 .prefer_i64 = TCG_TARGET_REG_BITS == 64,
2495 .vece = MO_64 },
2496 };
2497
2498 tcg_debug_assert(vece <= MO_64);
2499 tcg_debug_assert(shift >= 0 && shift < (8 << vece));
2500 if (shift == 0) {
2501 tcg_gen_gvec_mov(vece, dofs, aofs, oprsz, maxsz);
2502 } else {
2503 tcg_gen_gvec_2i(dofs, aofs, oprsz, maxsz, shift, &g[vece]);
2504 }
2505}
2506
2507void tcg_gen_vec_shr8i_i64(TCGv_i64 d, TCGv_i64 a, int64_t c)
2508{
2509 uint64_t mask = dup_const(MO_8, 0xff >> c);
2510 tcg_gen_shri_i64(d, a, c);
2511 tcg_gen_andi_i64(d, d, mask);
2512}
2513
2514void tcg_gen_vec_shr16i_i64(TCGv_i64 d, TCGv_i64 a, int64_t c)
2515{
2516 uint64_t mask = dup_const(MO_16, 0xffff >> c);
2517 tcg_gen_shri_i64(d, a, c);
2518 tcg_gen_andi_i64(d, d, mask);
2519}
2520
2521void tcg_gen_gvec_shri(unsigned vece, uint32_t dofs, uint32_t aofs,
2522 int64_t shift, uint32_t oprsz, uint32_t maxsz)
2523{
53229a77 2524 static const TCGOpcode vecop_list[] = { INDEX_op_shri_vec, 0 };
d0ec9796
RH		 2525 static const GVecGen2i g[4] = {
2526 { .fni8 = tcg_gen_vec_shr8i_i64,
2527 .fniv = tcg_gen_shri_vec,
2528 .fno = gen_helper_gvec_shr8i,
53229a77 2529 .opt_opc = vecop_list,
d0ec9796
RH		 2530 .vece = MO_8 },
2531 { .fni8 = tcg_gen_vec_shr16i_i64,
2532 .fniv = tcg_gen_shri_vec,
2533 .fno = gen_helper_gvec_shr16i,
53229a77 2534 .opt_opc = vecop_list,
d0ec9796
RH		 2535 .vece = MO_16 },
2536 { .fni4 = tcg_gen_shri_i32,
2537 .fniv = tcg_gen_shri_vec,
2538 .fno = gen_helper_gvec_shr32i,
53229a77 2539 .opt_opc = vecop_list,
d0ec9796
RH		 2540 .vece = MO_32 },
2541 { .fni8 = tcg_gen_shri_i64,
2542 .fniv = tcg_gen_shri_vec,
2543 .fno = gen_helper_gvec_shr64i,
53229a77 2544 .opt_opc = vecop_list,
d0ec9796
RH		 2545 .prefer_i64 = TCG_TARGET_REG_BITS == 64,
2546 .vece = MO_64 },
2547 };
2548
2549 tcg_debug_assert(vece <= MO_64);
2550 tcg_debug_assert(shift >= 0 && shift < (8 << vece));
2551 if (shift == 0) {
2552 tcg_gen_gvec_mov(vece, dofs, aofs, oprsz, maxsz);
2553 } else {
2554 tcg_gen_gvec_2i(dofs, aofs, oprsz, maxsz, shift, &g[vece]);
2555 }
2556}
2557
2558void tcg_gen_vec_sar8i_i64(TCGv_i64 d, TCGv_i64 a, int64_t c)
2559{
2560 uint64_t s_mask = dup_const(MO_8, 0x80 >> c);
2561 uint64_t c_mask = dup_const(MO_8, 0xff >> c);
2562 TCGv_i64 s = tcg_temp_new_i64();
2563
2564 tcg_gen_shri_i64(d, a, c);
2565 tcg_gen_andi_i64(s, d, s_mask); /* isolate (shifted) sign bit */
2566 tcg_gen_muli_i64(s, s, (2 << c) - 2); /* replicate isolated signs */
2567 tcg_gen_andi_i64(d, d, c_mask); /* clear out bits above sign */
2568 tcg_gen_or_i64(d, d, s); /* include sign extension */
2569 tcg_temp_free_i64(s);
2570}
2571
2572void tcg_gen_vec_sar16i_i64(TCGv_i64 d, TCGv_i64 a, int64_t c)
2573{
2574 uint64_t s_mask = dup_const(MO_16, 0x8000 >> c);
2575 uint64_t c_mask = dup_const(MO_16, 0xffff >> c);
2576 TCGv_i64 s = tcg_temp_new_i64();
2577
2578 tcg_gen_shri_i64(d, a, c);
2579 tcg_gen_andi_i64(s, d, s_mask); /* isolate (shifted) sign bit */
2580 tcg_gen_andi_i64(d, d, c_mask); /* clear out bits above sign */
2581 tcg_gen_muli_i64(s, s, (2 << c) - 2); /* replicate isolated signs */
2582 tcg_gen_or_i64(d, d, s); /* include sign extension */
2583 tcg_temp_free_i64(s);
2584}
2585
2586void tcg_gen_gvec_sari(unsigned vece, uint32_t dofs, uint32_t aofs,
2587 int64_t shift, uint32_t oprsz, uint32_t maxsz)
2588{
53229a77 2589 static const TCGOpcode vecop_list[] = { INDEX_op_sari_vec, 0 };
d0ec9796
RH		 2590 static const GVecGen2i g[4] = {
2591 { .fni8 = tcg_gen_vec_sar8i_i64,
2592 .fniv = tcg_gen_sari_vec,
2593 .fno = gen_helper_gvec_sar8i,
53229a77 2594 .opt_opc = vecop_list,
d0ec9796
RH		 2595 .vece = MO_8 },
2596 { .fni8 = tcg_gen_vec_sar16i_i64,
2597 .fniv = tcg_gen_sari_vec,
2598 .fno = gen_helper_gvec_sar16i,
53229a77 2599 .opt_opc = vecop_list,
d0ec9796
RH		 2600 .vece = MO_16 },
2601 { .fni4 = tcg_gen_sari_i32,
2602 .fniv = tcg_gen_sari_vec,
2603 .fno = gen_helper_gvec_sar32i,
53229a77 2604 .opt_opc = vecop_list,
d0ec9796
RH		 2605 .vece = MO_32 },
2606 { .fni8 = tcg_gen_sari_i64,
2607 .fniv = tcg_gen_sari_vec,
2608 .fno = gen_helper_gvec_sar64i,
53229a77 2609 .opt_opc = vecop_list,
d0ec9796
RH		 2610 .prefer_i64 = TCG_TARGET_REG_BITS == 64,
2611 .vece = MO_64 },
2612 };
2613
2614 tcg_debug_assert(vece <= MO_64);
2615 tcg_debug_assert(shift >= 0 && shift < (8 << vece));
2616 if (shift == 0) {
2617 tcg_gen_gvec_mov(vece, dofs, aofs, oprsz, maxsz);
2618 } else {
2619 tcg_gen_gvec_2i(dofs, aofs, oprsz, maxsz, shift, &g[vece]);
2620 }
2621}
212be173 2622
b4578cd9
RH		 2623/*
2624 * Specialized generation vector shifts by a non-constant scalar.
2625 */
2626
2627typedef struct {
2628 void (*fni4)(TCGv_i32, TCGv_i32, TCGv_i32);
2629 void (*fni8)(TCGv_i64, TCGv_i64, TCGv_i64);
2630 void (*fniv_s)(unsigned, TCGv_vec, TCGv_vec, TCGv_i32);
2631 void (*fniv_v)(unsigned, TCGv_vec, TCGv_vec, TCGv_vec);
2632 gen_helper_gvec_2 *fno[4];
2633 TCGOpcode s_list[2];
2634 TCGOpcode v_list[2];
2635} GVecGen2sh;
2636
2637static void expand_2sh_vec(unsigned vece, uint32_t dofs, uint32_t aofs,
2638 uint32_t oprsz, uint32_t tysz, TCGType type,
2639 TCGv_i32 shift,
2640 void (*fni)(unsigned, TCGv_vec, TCGv_vec, TCGv_i32))
2641{
2642 TCGv_vec t0 = tcg_temp_new_vec(type);
2643 uint32_t i;
2644
2645 for (i = 0; i < oprsz; i += tysz) {
2646 tcg_gen_ld_vec(t0, cpu_env, aofs + i);
2647 fni(vece, t0, t0, shift);
2648 tcg_gen_st_vec(t0, cpu_env, dofs + i);
2649 }
2650 tcg_temp_free_vec(t0);
2651}
2652
2653static void
2654do_gvec_shifts(unsigned vece, uint32_t dofs, uint32_t aofs, TCGv_i32 shift,
2655 uint32_t oprsz, uint32_t maxsz, const GVecGen2sh *g)
2656{
2657 TCGType type;
2658 uint32_t some;
2659
2660 check_size_align(oprsz, maxsz, dofs | aofs);
2661 check_overlap_2(dofs, aofs, maxsz);
2662
2663 /* If the backend has a scalar expansion, great. */
2664 type = choose_vector_type(g->s_list, vece, oprsz, vece == MO_64);
2665 if (type) {
2666 const TCGOpcode *hold_list = tcg_swap_vecop_list(NULL);
2667 switch (type) {
2668 case TCG_TYPE_V256:
2669 some = QEMU_ALIGN_DOWN(oprsz, 32);
2670 expand_2sh_vec(vece, dofs, aofs, some, 32,
2671 TCG_TYPE_V256, shift, g->fniv_s);
2672 if (some == oprsz) {
2673 break;
2674 }
2675 dofs += some;
2676 aofs += some;
2677 oprsz -= some;
2678 maxsz -= some;
2679 /* fallthru */
2680 case TCG_TYPE_V128:
2681 expand_2sh_vec(vece, dofs, aofs, oprsz, 16,
2682 TCG_TYPE_V128, shift, g->fniv_s);
2683 break;
2684 case TCG_TYPE_V64:
2685 expand_2sh_vec(vece, dofs, aofs, oprsz, 8,
2686 TCG_TYPE_V64, shift, g->fniv_s);
2687 break;
2688 default:
2689 g_assert_not_reached();
2690 }
2691 tcg_swap_vecop_list(hold_list);
2692 goto clear_tail;
2693 }
2694
2695 /* If the backend supports variable vector shifts, also cool. */
2696 type = choose_vector_type(g->v_list, vece, oprsz, vece == MO_64);
2697 if (type) {
2698 const TCGOpcode *hold_list = tcg_swap_vecop_list(NULL);
2699 TCGv_vec v_shift = tcg_temp_new_vec(type);
2700
2701 if (vece == MO_64) {
2702 TCGv_i64 sh64 = tcg_temp_new_i64();
2703 tcg_gen_extu_i32_i64(sh64, shift);
2704 tcg_gen_dup_i64_vec(MO_64, v_shift, sh64);
2705 tcg_temp_free_i64(sh64);
2706 } else {
2707 tcg_gen_dup_i32_vec(vece, v_shift, shift);
2708 }
2709
2710 switch (type) {
2711 case TCG_TYPE_V256:
2712 some = QEMU_ALIGN_DOWN(oprsz, 32);
2713 expand_2s_vec(vece, dofs, aofs, some, 32, TCG_TYPE_V256,
2714 v_shift, false, g->fniv_v);
2715 if (some == oprsz) {
2716 break;
2717 }
2718 dofs += some;
2719 aofs += some;
2720 oprsz -= some;
2721 maxsz -= some;
2722 /* fallthru */
2723 case TCG_TYPE_V128:
2724 expand_2s_vec(vece, dofs, aofs, oprsz, 16, TCG_TYPE_V128,
2725 v_shift, false, g->fniv_v);
2726 break;
2727 case TCG_TYPE_V64:
2728 expand_2s_vec(vece, dofs, aofs, oprsz, 8, TCG_TYPE_V64,
2729 v_shift, false, g->fniv_v);
2730 break;
2731 default:
2732 g_assert_not_reached();
2733 }
2734 tcg_temp_free_vec(v_shift);
2735 tcg_swap_vecop_list(hold_list);
2736 goto clear_tail;
2737 }
2738
2739 /* Otherwise fall back to integral... */
2740 if (vece == MO_32 && check_size_impl(oprsz, 4)) {
2741 expand_2s_i32(dofs, aofs, oprsz, shift, false, g->fni4);
2742 } else if (vece == MO_64 && check_size_impl(oprsz, 8)) {
2743 TCGv_i64 sh64 = tcg_temp_new_i64();
2744 tcg_gen_extu_i32_i64(sh64, shift);
2745 expand_2s_i64(dofs, aofs, oprsz, sh64, false, g->fni8);
2746 tcg_temp_free_i64(sh64);
2747 } else {
2748 TCGv_ptr a0 = tcg_temp_new_ptr();
2749 TCGv_ptr a1 = tcg_temp_new_ptr();
2750 TCGv_i32 desc = tcg_temp_new_i32();
2751
2752 tcg_gen_shli_i32(desc, shift, SIMD_DATA_SHIFT);
2753 tcg_gen_ori_i32(desc, desc, simd_desc(oprsz, maxsz, 0));
2754 tcg_gen_addi_ptr(a0, cpu_env, dofs);
2755 tcg_gen_addi_ptr(a1, cpu_env, aofs);
2756
2757 g->fno[vece](a0, a1, desc);
2758
2759 tcg_temp_free_ptr(a0);
2760 tcg_temp_free_ptr(a1);
2761 tcg_temp_free_i32(desc);
2762 return;
2763 }
2764
2765 clear_tail:
2766 if (oprsz < maxsz) {
2767 expand_clr(dofs + oprsz, maxsz - oprsz);
2768 }
2769}
2770
2771void tcg_gen_gvec_shls(unsigned vece, uint32_t dofs, uint32_t aofs,
2772 TCGv_i32 shift, uint32_t oprsz, uint32_t maxsz)
2773{
2774 static const GVecGen2sh g = {
2775 .fni4 = tcg_gen_shl_i32,
2776 .fni8 = tcg_gen_shl_i64,
2777 .fniv_s = tcg_gen_shls_vec,
2778 .fniv_v = tcg_gen_shlv_vec,
2779 .fno = {
2780 gen_helper_gvec_shl8i,
2781 gen_helper_gvec_shl16i,
2782 gen_helper_gvec_shl32i,
2783 gen_helper_gvec_shl64i,
2784 },
2785 .s_list = { INDEX_op_shls_vec, 0 },
2786 .v_list = { INDEX_op_shlv_vec, 0 },
2787 };
2788
2789 tcg_debug_assert(vece <= MO_64);
2790 do_gvec_shifts(vece, dofs, aofs, shift, oprsz, maxsz, &g);
2791}
2792
2793void tcg_gen_gvec_shrs(unsigned vece, uint32_t dofs, uint32_t aofs,
2794 TCGv_i32 shift, uint32_t oprsz, uint32_t maxsz)
2795{
2796 static const GVecGen2sh g = {
2797 .fni4 = tcg_gen_shr_i32,
2798 .fni8 = tcg_gen_shr_i64,
2799 .fniv_s = tcg_gen_shrs_vec,
2800 .fniv_v = tcg_gen_shrv_vec,
2801 .fno = {
2802 gen_helper_gvec_shr8i,
2803 gen_helper_gvec_shr16i,
2804 gen_helper_gvec_shr32i,
2805 gen_helper_gvec_shr64i,
2806 },
2807 .s_list = { INDEX_op_shrs_vec, 0 },
2808 .v_list = { INDEX_op_shrv_vec, 0 },
2809 };
2810
2811 tcg_debug_assert(vece <= MO_64);
2812 do_gvec_shifts(vece, dofs, aofs, shift, oprsz, maxsz, &g);
2813}
2814
2815void tcg_gen_gvec_sars(unsigned vece, uint32_t dofs, uint32_t aofs,
2816 TCGv_i32 shift, uint32_t oprsz, uint32_t maxsz)
2817{
2818 static const GVecGen2sh g = {
2819 .fni4 = tcg_gen_sar_i32,
2820 .fni8 = tcg_gen_sar_i64,
2821 .fniv_s = tcg_gen_sars_vec,
2822 .fniv_v = tcg_gen_sarv_vec,
2823 .fno = {
2824 gen_helper_gvec_sar8i,
2825 gen_helper_gvec_sar16i,
2826 gen_helper_gvec_sar32i,
2827 gen_helper_gvec_sar64i,
2828 },
2829 .s_list = { INDEX_op_sars_vec, 0 },
2830 .v_list = { INDEX_op_sarv_vec, 0 },
2831 };
2832
2833 tcg_debug_assert(vece <= MO_64);
2834 do_gvec_shifts(vece, dofs, aofs, shift, oprsz, maxsz, &g);
2835}
2836
5ee5c14c
RH		 2837/*
2838 * Expand D = A << (B % element bits)
2839 *
2840 * Unlike scalar shifts, where it is easy for the target front end
2841 * to include the modulo as part of the expansion. If the target
2842 * naturally includes the modulo as part of the operation, great!
2843 * If the target has some other behaviour from out-of-range shifts,
2844 * then it could not use this function anyway, and would need to
2845 * do it's own expansion with custom functions.
2846 */
2847static void tcg_gen_shlv_mod_vec(unsigned vece, TCGv_vec d,
2848 TCGv_vec a, TCGv_vec b)
2849{
2850 TCGv_vec t = tcg_temp_new_vec_matching(d);
2851
2852 tcg_gen_dupi_vec(vece, t, (8 << vece) - 1);
2853 tcg_gen_and_vec(vece, t, t, b);
2854 tcg_gen_shlv_vec(vece, d, a, t);
2855 tcg_temp_free_vec(t);
2856}
2857
2858static void tcg_gen_shl_mod_i32(TCGv_i32 d, TCGv_i32 a, TCGv_i32 b)
2859{
2860 TCGv_i32 t = tcg_temp_new_i32();
2861
2862 tcg_gen_andi_i32(t, b, 31);
2863 tcg_gen_shl_i32(d, a, t);
2864 tcg_temp_free_i32(t);
2865}
2866
2867static void tcg_gen_shl_mod_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b)
2868{
2869 TCGv_i64 t = tcg_temp_new_i64();
2870
2871 tcg_gen_andi_i64(t, b, 63);
2872 tcg_gen_shl_i64(d, a, t);
2873 tcg_temp_free_i64(t);
2874}
2875
2876void tcg_gen_gvec_shlv(unsigned vece, uint32_t dofs, uint32_t aofs,
2877 uint32_t bofs, uint32_t oprsz, uint32_t maxsz)
2878{
2879 static const TCGOpcode vecop_list[] = { INDEX_op_shlv_vec, 0 };
2880 static const GVecGen3 g[4] = {
2881 { .fniv = tcg_gen_shlv_mod_vec,
2882 .fno = gen_helper_gvec_shl8v,
2883 .opt_opc = vecop_list,
2884 .vece = MO_8 },
2885 { .fniv = tcg_gen_shlv_mod_vec,
2886 .fno = gen_helper_gvec_shl16v,
2887 .opt_opc = vecop_list,
2888 .vece = MO_16 },
2889 { .fni4 = tcg_gen_shl_mod_i32,
2890 .fniv = tcg_gen_shlv_mod_vec,
2891 .fno = gen_helper_gvec_shl32v,
2892 .opt_opc = vecop_list,
2893 .vece = MO_32 },
2894 { .fni8 = tcg_gen_shl_mod_i64,
2895 .fniv = tcg_gen_shlv_mod_vec,
2896 .fno = gen_helper_gvec_shl64v,
2897 .opt_opc = vecop_list,
2898 .prefer_i64 = TCG_TARGET_REG_BITS == 64,
2899 .vece = MO_64 },
2900 };
2901
2902 tcg_debug_assert(vece <= MO_64);
2903 tcg_gen_gvec_3(dofs, aofs, bofs, oprsz, maxsz, &g[vece]);
2904}
2905
2906/*
2907 * Similarly for logical right shifts.
2908 */
2909
2910static void tcg_gen_shrv_mod_vec(unsigned vece, TCGv_vec d,
2911 TCGv_vec a, TCGv_vec b)
2912{
2913 TCGv_vec t = tcg_temp_new_vec_matching(d);
2914
2915 tcg_gen_dupi_vec(vece, t, (8 << vece) - 1);
2916 tcg_gen_and_vec(vece, t, t, b);
2917 tcg_gen_shrv_vec(vece, d, a, t);
2918 tcg_temp_free_vec(t);
2919}
2920
2921static void tcg_gen_shr_mod_i32(TCGv_i32 d, TCGv_i32 a, TCGv_i32 b)
2922{
2923 TCGv_i32 t = tcg_temp_new_i32();
2924
2925 tcg_gen_andi_i32(t, b, 31);
2926 tcg_gen_shr_i32(d, a, t);
2927 tcg_temp_free_i32(t);
2928}
2929
2930static void tcg_gen_shr_mod_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b)
2931{
2932 TCGv_i64 t = tcg_temp_new_i64();
2933
2934 tcg_gen_andi_i64(t, b, 63);
2935 tcg_gen_shr_i64(d, a, t);
2936 tcg_temp_free_i64(t);
2937}
2938
2939void tcg_gen_gvec_shrv(unsigned vece, uint32_t dofs, uint32_t aofs,
2940 uint32_t bofs, uint32_t oprsz, uint32_t maxsz)
2941{
2942 static const TCGOpcode vecop_list[] = { INDEX_op_shrv_vec, 0 };
2943 static const GVecGen3 g[4] = {
2944 { .fniv = tcg_gen_shrv_mod_vec,
2945 .fno = gen_helper_gvec_shr8v,
2946 .opt_opc = vecop_list,
2947 .vece = MO_8 },
2948 { .fniv = tcg_gen_shrv_mod_vec,
2949 .fno = gen_helper_gvec_shr16v,
2950 .opt_opc = vecop_list,
2951 .vece = MO_16 },
2952 { .fni4 = tcg_gen_shr_mod_i32,
2953 .fniv = tcg_gen_shrv_mod_vec,
2954 .fno = gen_helper_gvec_shr32v,
2955 .opt_opc = vecop_list,
2956 .vece = MO_32 },
2957 { .fni8 = tcg_gen_shr_mod_i64,
2958 .fniv = tcg_gen_shrv_mod_vec,
2959 .fno = gen_helper_gvec_shr64v,
2960 .opt_opc = vecop_list,
2961 .prefer_i64 = TCG_TARGET_REG_BITS == 64,
2962 .vece = MO_64 },
2963 };
2964
2965 tcg_debug_assert(vece <= MO_64);
2966 tcg_gen_gvec_3(dofs, aofs, bofs, oprsz, maxsz, &g[vece]);
2967}
2968
2969/*
2970 * Similarly for arithmetic right shifts.
2971 */
2972
2973static void tcg_gen_sarv_mod_vec(unsigned vece, TCGv_vec d,
2974 TCGv_vec a, TCGv_vec b)
2975{
2976 TCGv_vec t = tcg_temp_new_vec_matching(d);
2977
2978 tcg_gen_dupi_vec(vece, t, (8 << vece) - 1);
2979 tcg_gen_and_vec(vece, t, t, b);
2980 tcg_gen_sarv_vec(vece, d, a, t);
2981 tcg_temp_free_vec(t);
2982}
2983
2984static void tcg_gen_sar_mod_i32(TCGv_i32 d, TCGv_i32 a, TCGv_i32 b)
2985{
2986 TCGv_i32 t = tcg_temp_new_i32();
2987
2988 tcg_gen_andi_i32(t, b, 31);
2989 tcg_gen_sar_i32(d, a, t);
2990 tcg_temp_free_i32(t);
2991}
2992
2993static void tcg_gen_sar_mod_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b)
2994{
2995 TCGv_i64 t = tcg_temp_new_i64();
2996
2997 tcg_gen_andi_i64(t, b, 63);
2998 tcg_gen_sar_i64(d, a, t);
2999 tcg_temp_free_i64(t);
3000}
3001
3002void tcg_gen_gvec_sarv(unsigned vece, uint32_t dofs, uint32_t aofs,
3003 uint32_t bofs, uint32_t oprsz, uint32_t maxsz)
3004{
3005 static const TCGOpcode vecop_list[] = { INDEX_op_sarv_vec, 0 };
3006 static const GVecGen3 g[4] = {
3007 { .fniv = tcg_gen_sarv_mod_vec,
3008 .fno = gen_helper_gvec_sar8v,
3009 .opt_opc = vecop_list,
3010 .vece = MO_8 },
3011 { .fniv = tcg_gen_sarv_mod_vec,
3012 .fno = gen_helper_gvec_sar16v,
3013 .opt_opc = vecop_list,
3014 .vece = MO_16 },
3015 { .fni4 = tcg_gen_sar_mod_i32,
3016 .fniv = tcg_gen_sarv_mod_vec,
3017 .fno = gen_helper_gvec_sar32v,
3018 .opt_opc = vecop_list,
3019 .vece = MO_32 },
3020 { .fni8 = tcg_gen_sar_mod_i64,
3021 .fniv = tcg_gen_sarv_mod_vec,
3022 .fno = gen_helper_gvec_sar64v,
3023 .opt_opc = vecop_list,
3024 .prefer_i64 = TCG_TARGET_REG_BITS == 64,
3025 .vece = MO_64 },
3026 };
3027
3028 tcg_debug_assert(vece <= MO_64);
3029 tcg_gen_gvec_3(dofs, aofs, bofs, oprsz, maxsz, &g[vece]);
3030}
3031
212be173
RH		 3032/* Expand OPSZ bytes worth of three-operand operations using i32 elements. */
3033static void expand_cmp_i32(uint32_t dofs, uint32_t aofs, uint32_t bofs,
3034 uint32_t oprsz, TCGCond cond)
3035{
3036 TCGv_i32 t0 = tcg_temp_new_i32();
3037 TCGv_i32 t1 = tcg_temp_new_i32();
3038 uint32_t i;
3039
3040 for (i = 0; i < oprsz; i += 4) {
3041 tcg_gen_ld_i32(t0, cpu_env, aofs + i);
3042 tcg_gen_ld_i32(t1, cpu_env, bofs + i);
3043 tcg_gen_setcond_i32(cond, t0, t0, t1);
3044 tcg_gen_neg_i32(t0, t0);
3045 tcg_gen_st_i32(t0, cpu_env, dofs + i);
3046 }
3047 tcg_temp_free_i32(t1);
3048 tcg_temp_free_i32(t0);
3049}
3050
3051static void expand_cmp_i64(uint32_t dofs, uint32_t aofs, uint32_t bofs,
3052 uint32_t oprsz, TCGCond cond)
3053{
3054 TCGv_i64 t0 = tcg_temp_new_i64();
3055 TCGv_i64 t1 = tcg_temp_new_i64();
3056 uint32_t i;
3057
3058 for (i = 0; i < oprsz; i += 8) {
3059 tcg_gen_ld_i64(t0, cpu_env, aofs + i);
3060 tcg_gen_ld_i64(t1, cpu_env, bofs + i);
3061 tcg_gen_setcond_i64(cond, t0, t0, t1);
3062 tcg_gen_neg_i64(t0, t0);
3063 tcg_gen_st_i64(t0, cpu_env, dofs + i);
3064 }
3065 tcg_temp_free_i64(t1);
3066 tcg_temp_free_i64(t0);
3067}
3068
3069static void expand_cmp_vec(unsigned vece, uint32_t dofs, uint32_t aofs,
3070 uint32_t bofs, uint32_t oprsz, uint32_t tysz,
3071 TCGType type, TCGCond cond)
3072{
3073 TCGv_vec t0 = tcg_temp_new_vec(type);
3074 TCGv_vec t1 = tcg_temp_new_vec(type);
3075 uint32_t i;
3076
3077 for (i = 0; i < oprsz; i += tysz) {
3078 tcg_gen_ld_vec(t0, cpu_env, aofs + i);
3079 tcg_gen_ld_vec(t1, cpu_env, bofs + i);
3080 tcg_gen_cmp_vec(cond, vece, t0, t0, t1);
3081 tcg_gen_st_vec(t0, cpu_env, dofs + i);
3082 }
3083 tcg_temp_free_vec(t1);
3084 tcg_temp_free_vec(t0);
3085}
3086
3087void tcg_gen_gvec_cmp(TCGCond cond, unsigned vece, uint32_t dofs,
3088 uint32_t aofs, uint32_t bofs,
3089 uint32_t oprsz, uint32_t maxsz)
3090{
53229a77 3091 static const TCGOpcode cmp_list[] = { INDEX_op_cmp_vec, 0 };
212be173
RH		 3092 static gen_helper_gvec_3 * const eq_fn[4] = {
3093 gen_helper_gvec_eq8, gen_helper_gvec_eq16,
3094 gen_helper_gvec_eq32, gen_helper_gvec_eq64
3095 };
3096 static gen_helper_gvec_3 * const ne_fn[4] = {
3097 gen_helper_gvec_ne8, gen_helper_gvec_ne16,
3098 gen_helper_gvec_ne32, gen_helper_gvec_ne64
3099 };
3100 static gen_helper_gvec_3 * const lt_fn[4] = {
3101 gen_helper_gvec_lt8, gen_helper_gvec_lt16,
3102 gen_helper_gvec_lt32, gen_helper_gvec_lt64
3103 };
3104 static gen_helper_gvec_3 * const le_fn[4] = {
3105 gen_helper_gvec_le8, gen_helper_gvec_le16,
3106 gen_helper_gvec_le32, gen_helper_gvec_le64
3107 };
3108 static gen_helper_gvec_3 * const ltu_fn[4] = {
3109 gen_helper_gvec_ltu8, gen_helper_gvec_ltu16,
3110 gen_helper_gvec_ltu32, gen_helper_gvec_ltu64
3111 };
3112 static gen_helper_gvec_3 * const leu_fn[4] = {
3113 gen_helper_gvec_leu8, gen_helper_gvec_leu16,
3114 gen_helper_gvec_leu32, gen_helper_gvec_leu64
3115 };
3116 static gen_helper_gvec_3 * const * const fns[16] = {
3117 [TCG_COND_EQ] = eq_fn,
3118 [TCG_COND_NE] = ne_fn,
3119 [TCG_COND_LT] = lt_fn,
3120 [TCG_COND_LE] = le_fn,
3121 [TCG_COND_LTU] = ltu_fn,
3122 [TCG_COND_LEU] = leu_fn,
3123 };
53229a77
RH		 3124
3125 const TCGOpcode *hold_list;
adb196cb
RH		 3126 TCGType type;
3127 uint32_t some;
212be173
RH		 3128
3129 check_size_align(oprsz, maxsz, dofs | aofs | bofs);
3130 check_overlap_3(dofs, aofs, bofs, maxsz);
3131
3132 if (cond == TCG_COND_NEVER || cond == TCG_COND_ALWAYS) {
3133 do_dup(MO_8, dofs, oprsz, maxsz,
3134 NULL, NULL, -(cond == TCG_COND_ALWAYS));
3135 return;
3136 }
3137
53229a77
RH		 3138 /*
3139 * Implement inline with a vector type, if possible.
adb196cb
RH		 3140 * Prefer integer when 64-bit host and 64-bit comparison.
3141 */
53229a77
RH		 3142 hold_list = tcg_swap_vecop_list(cmp_list);
3143 type = choose_vector_type(cmp_list, vece, oprsz,
adb196cb
RH		 3144 TCG_TARGET_REG_BITS == 64 && vece == MO_64);
3145 switch (type) {
3146 case TCG_TYPE_V256:
3147 /* Recall that ARM SVE allows vector sizes that are not a
3148 * power of 2, but always a multiple of 16. The intent is
3149 * that e.g. size == 80 would be expanded with 2x32 + 1x16.
3150 */
3151 some = QEMU_ALIGN_DOWN(oprsz, 32);
212be173
RH		 3152 expand_cmp_vec(vece, dofs, aofs, bofs, some, 32, TCG_TYPE_V256, cond);
3153 if (some == oprsz) {
adb196cb 3154 break;
212be173
RH		 3155 }
3156 dofs += some;
3157 aofs += some;
3158 bofs += some;
3159 oprsz -= some;
3160 maxsz -= some;
adb196cb
RH		 3161 /* fallthru */
3162 case TCG_TYPE_V128:
212be173 3163 expand_cmp_vec(vece, dofs, aofs, bofs, oprsz, 16, TCG_TYPE_V128, cond);
adb196cb
RH		 3164 break;
3165 case TCG_TYPE_V64:
212be173 3166 expand_cmp_vec(vece, dofs, aofs, bofs, oprsz, 8, TCG_TYPE_V64, cond);
adb196cb
RH		 3167 break;
3168
3169 case 0:
3170 if (vece == MO_64 && check_size_impl(oprsz, 8)) {
3171 expand_cmp_i64(dofs, aofs, bofs, oprsz, cond);
3172 } else if (vece == MO_32 && check_size_impl(oprsz, 4)) {
3173 expand_cmp_i32(dofs, aofs, bofs, oprsz, cond);
3174 } else {
3175 gen_helper_gvec_3 * const *fn = fns[cond];
3176
3177 if (fn == NULL) {
3178 uint32_t tmp;
3179 tmp = aofs, aofs = bofs, bofs = tmp;
3180 cond = tcg_swap_cond(cond);
3181 fn = fns[cond];
3182 assert(fn != NULL);
3183 }
3184 tcg_gen_gvec_3_ool(dofs, aofs, bofs, oprsz, maxsz, 0, fn[vece]);
53229a77 3185 oprsz = maxsz;
212be173 3186 }
adb196cb
RH		 3187 break;
3188
3189 default:
3190 g_assert_not_reached();
212be173 3191 }
53229a77 3192 tcg_swap_vecop_list(hold_list);
212be173 3193
212be173
RH		 3194 if (oprsz < maxsz) {
3195 expand_clr(dofs + oprsz, maxsz - oprsz);
3196 }
3197}
38dc1294
RH		 3198
3199static void tcg_gen_bitsel_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b, TCGv_i64 c)
3200{
3201 TCGv_i64 t = tcg_temp_new_i64();
3202
3203 tcg_gen_and_i64(t, b, a);
3204 tcg_gen_andc_i64(d, c, a);
3205 tcg_gen_or_i64(d, d, t);
3206 tcg_temp_free_i64(t);
3207}
3208
3209void tcg_gen_gvec_bitsel(unsigned vece, uint32_t dofs, uint32_t aofs,
3210 uint32_t bofs, uint32_t cofs,
3211 uint32_t oprsz, uint32_t maxsz)
3212{
3213 static const GVecGen4 g = {
3214 .fni8 = tcg_gen_bitsel_i64,
3215 .fniv = tcg_gen_bitsel_vec,
3216 .fno = gen_helper_gvec_bitsel,
3217 };
3218
3219 tcg_gen_gvec_4(dofs, aofs, bofs, cofs, oprsz, maxsz, &g);
3220}
Empty description
This page took 0.492393 seconds and 4 git commands to generate.