]> Git Repo - qemu.git/blame - tcg/tcg-op-gvec.c
projects / qemu.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Merge remote-tracking branch 'remotes/rth-gitlab/tags/pull-tcg-20210104' into staging
[qemu.git] / tcg / tcg-op-gvec.c
CommitLineData
db432672
RH		 1/*
2 * Generic vector operation expansion
3 *
4 * Copyright (c) 2018 Linaro
5 *
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
fb0343d5 9 * version 2.1 of the License, or (at your option) any later version.
db432672
RH		 10 *
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
15 *
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
18 */
19
20#include "qemu/osdep.h"
dcb32f1d
PMD		 21#include "tcg/tcg.h"
22#include "tcg/tcg-op.h"
23#include "tcg/tcg-op-gvec.h"
db725815 24#include "qemu/main-loop.h"
dcb32f1d 25#include "tcg/tcg-gvec-desc.h"
db432672
RH		 26
27#define MAX_UNROLL 4
28
53229a77
RH		 29#ifdef CONFIG_DEBUG_TCG
30static const TCGOpcode vecop_list_empty[1] = { 0 };
31#else
32#define vecop_list_empty NULL
33#endif
34
35
db432672
RH		 36/* Verify vector size and alignment rules. OFS should be the OR of all
37 of the operand offsets so that we can check them all at once. */
38static void check_size_align(uint32_t oprsz, uint32_t maxsz, uint32_t ofs)
39{
e2e7168a
RH		 40 uint32_t max_align;
41
42 switch (oprsz) {
43 case 8:
44 case 16:
45 case 32:
46 tcg_debug_assert(oprsz <= maxsz);
47 break;
48 default:
49 tcg_debug_assert(oprsz == maxsz);
50 break;
51 }
52 tcg_debug_assert(maxsz <= (8 << SIMD_MAXSZ_BITS));
53
54 max_align = maxsz >= 16 ? 15 : 7;
db432672
RH		 55 tcg_debug_assert((maxsz & max_align) == 0);
56 tcg_debug_assert((ofs & max_align) == 0);
57}
58
59/* Verify vector overlap rules for two operands. */
60static void check_overlap_2(uint32_t d, uint32_t a, uint32_t s)
61{
62 tcg_debug_assert(d == a || d + s <= a || a + s <= d);
63}
64
65/* Verify vector overlap rules for three operands. */
66static void check_overlap_3(uint32_t d, uint32_t a, uint32_t b, uint32_t s)
67{
68 check_overlap_2(d, a, s);
69 check_overlap_2(d, b, s);
70 check_overlap_2(a, b, s);
71}
72
73/* Verify vector overlap rules for four operands. */
74static void check_overlap_4(uint32_t d, uint32_t a, uint32_t b,
75 uint32_t c, uint32_t s)
76{
77 check_overlap_2(d, a, s);
78 check_overlap_2(d, b, s);
79 check_overlap_2(d, c, s);
80 check_overlap_2(a, b, s);
81 check_overlap_2(a, c, s);
82 check_overlap_2(b, c, s);
83}
84
85/* Create a descriptor from components. */
86uint32_t simd_desc(uint32_t oprsz, uint32_t maxsz, int32_t data)
87{
88 uint32_t desc = 0;
89
e2e7168a
RH		 90 check_size_align(oprsz, maxsz, 0);
91 tcg_debug_assert(data == sextract32(data, 0, SIMD_DATA_BITS));
db432672
RH		 92
93 oprsz = (oprsz / 8) - 1;
94 maxsz = (maxsz / 8) - 1;
e2e7168a
RH		 95
96 /*
97 * We have just asserted in check_size_align that either
98 * oprsz is {8,16,32} or matches maxsz. Encode the final
99 * case with '2', as that would otherwise map to 24.
100 */
101 if (oprsz == maxsz) {
102 oprsz = 2;
103 }
104
db432672
RH		 105 desc = deposit32(desc, SIMD_OPRSZ_SHIFT, SIMD_OPRSZ_BITS, oprsz);
106 desc = deposit32(desc, SIMD_MAXSZ_SHIFT, SIMD_MAXSZ_BITS, maxsz);
107 desc = deposit32(desc, SIMD_DATA_SHIFT, SIMD_DATA_BITS, data);
108
109 return desc;
110}
111
112/* Generate a call to a gvec-style helper with two vector operands. */
113void tcg_gen_gvec_2_ool(uint32_t dofs, uint32_t aofs,
114 uint32_t oprsz, uint32_t maxsz, int32_t data,
115 gen_helper_gvec_2 *fn)
116{
117 TCGv_ptr a0, a1;
118 TCGv_i32 desc = tcg_const_i32(simd_desc(oprsz, maxsz, data));
119
120 a0 = tcg_temp_new_ptr();
121 a1 = tcg_temp_new_ptr();
122
123 tcg_gen_addi_ptr(a0, cpu_env, dofs);
124 tcg_gen_addi_ptr(a1, cpu_env, aofs);
125
126 fn(a0, a1, desc);
127
128 tcg_temp_free_ptr(a0);
129 tcg_temp_free_ptr(a1);
130 tcg_temp_free_i32(desc);
131}
132
22fc3527
RH		 133/* Generate a call to a gvec-style helper with two vector operands
134 and one scalar operand. */
135void tcg_gen_gvec_2i_ool(uint32_t dofs, uint32_t aofs, TCGv_i64 c,
136 uint32_t oprsz, uint32_t maxsz, int32_t data,
137 gen_helper_gvec_2i *fn)
138{
139 TCGv_ptr a0, a1;
140 TCGv_i32 desc = tcg_const_i32(simd_desc(oprsz, maxsz, data));
141
142 a0 = tcg_temp_new_ptr();
143 a1 = tcg_temp_new_ptr();
144
145 tcg_gen_addi_ptr(a0, cpu_env, dofs);
146 tcg_gen_addi_ptr(a1, cpu_env, aofs);
147
148 fn(a0, a1, c, desc);
149
150 tcg_temp_free_ptr(a0);
151 tcg_temp_free_ptr(a1);
152 tcg_temp_free_i32(desc);
153}
154
db432672
RH		 155/* Generate a call to a gvec-style helper with three vector operands. */
156void tcg_gen_gvec_3_ool(uint32_t dofs, uint32_t aofs, uint32_t bofs,
157 uint32_t oprsz, uint32_t maxsz, int32_t data,
158 gen_helper_gvec_3 *fn)
159{
160 TCGv_ptr a0, a1, a2;
161 TCGv_i32 desc = tcg_const_i32(simd_desc(oprsz, maxsz, data));
162
163 a0 = tcg_temp_new_ptr();
164 a1 = tcg_temp_new_ptr();
165 a2 = tcg_temp_new_ptr();
166
167 tcg_gen_addi_ptr(a0, cpu_env, dofs);
168 tcg_gen_addi_ptr(a1, cpu_env, aofs);
169 tcg_gen_addi_ptr(a2, cpu_env, bofs);
170
171 fn(a0, a1, a2, desc);
172
173 tcg_temp_free_ptr(a0);
174 tcg_temp_free_ptr(a1);
175 tcg_temp_free_ptr(a2);
176 tcg_temp_free_i32(desc);
177}
178
179/* Generate a call to a gvec-style helper with four vector operands. */
180void tcg_gen_gvec_4_ool(uint32_t dofs, uint32_t aofs, uint32_t bofs,
181 uint32_t cofs, uint32_t oprsz, uint32_t maxsz,
182 int32_t data, gen_helper_gvec_4 *fn)
183{
184 TCGv_ptr a0, a1, a2, a3;
185 TCGv_i32 desc = tcg_const_i32(simd_desc(oprsz, maxsz, data));
186
187 a0 = tcg_temp_new_ptr();
188 a1 = tcg_temp_new_ptr();
189 a2 = tcg_temp_new_ptr();
190 a3 = tcg_temp_new_ptr();
191
192 tcg_gen_addi_ptr(a0, cpu_env, dofs);
193 tcg_gen_addi_ptr(a1, cpu_env, aofs);
194 tcg_gen_addi_ptr(a2, cpu_env, bofs);
195 tcg_gen_addi_ptr(a3, cpu_env, cofs);
196
197 fn(a0, a1, a2, a3, desc);
198
199 tcg_temp_free_ptr(a0);
200 tcg_temp_free_ptr(a1);
201 tcg_temp_free_ptr(a2);
202 tcg_temp_free_ptr(a3);
203 tcg_temp_free_i32(desc);
204}
205
206/* Generate a call to a gvec-style helper with five vector operands. */
207void tcg_gen_gvec_5_ool(uint32_t dofs, uint32_t aofs, uint32_t bofs,
208 uint32_t cofs, uint32_t xofs, uint32_t oprsz,
209 uint32_t maxsz, int32_t data, gen_helper_gvec_5 *fn)
210{
211 TCGv_ptr a0, a1, a2, a3, a4;
212 TCGv_i32 desc = tcg_const_i32(simd_desc(oprsz, maxsz, data));
213
214 a0 = tcg_temp_new_ptr();
215 a1 = tcg_temp_new_ptr();
216 a2 = tcg_temp_new_ptr();
217 a3 = tcg_temp_new_ptr();
218 a4 = tcg_temp_new_ptr();
219
220 tcg_gen_addi_ptr(a0, cpu_env, dofs);
221 tcg_gen_addi_ptr(a1, cpu_env, aofs);
222 tcg_gen_addi_ptr(a2, cpu_env, bofs);
223 tcg_gen_addi_ptr(a3, cpu_env, cofs);
224 tcg_gen_addi_ptr(a4, cpu_env, xofs);
225
226 fn(a0, a1, a2, a3, a4, desc);
227
228 tcg_temp_free_ptr(a0);
229 tcg_temp_free_ptr(a1);
230 tcg_temp_free_ptr(a2);
231 tcg_temp_free_ptr(a3);
232 tcg_temp_free_ptr(a4);
233 tcg_temp_free_i32(desc);
234}
235
236/* Generate a call to a gvec-style helper with three vector operands
237 and an extra pointer operand. */
238void tcg_gen_gvec_2_ptr(uint32_t dofs, uint32_t aofs,
239 TCGv_ptr ptr, uint32_t oprsz, uint32_t maxsz,
240 int32_t data, gen_helper_gvec_2_ptr *fn)
241{
242 TCGv_ptr a0, a1;
243 TCGv_i32 desc = tcg_const_i32(simd_desc(oprsz, maxsz, data));
244
245 a0 = tcg_temp_new_ptr();
246 a1 = tcg_temp_new_ptr();
247
248 tcg_gen_addi_ptr(a0, cpu_env, dofs);
249 tcg_gen_addi_ptr(a1, cpu_env, aofs);
250
251 fn(a0, a1, ptr, desc);
252
253 tcg_temp_free_ptr(a0);
254 tcg_temp_free_ptr(a1);
255 tcg_temp_free_i32(desc);
256}
257
258/* Generate a call to a gvec-style helper with three vector operands
259 and an extra pointer operand. */
260void tcg_gen_gvec_3_ptr(uint32_t dofs, uint32_t aofs, uint32_t bofs,
261 TCGv_ptr ptr, uint32_t oprsz, uint32_t maxsz,
262 int32_t data, gen_helper_gvec_3_ptr *fn)
263{
264 TCGv_ptr a0, a1, a2;
265 TCGv_i32 desc = tcg_const_i32(simd_desc(oprsz, maxsz, data));
266
267 a0 = tcg_temp_new_ptr();
268 a1 = tcg_temp_new_ptr();
269 a2 = tcg_temp_new_ptr();
270
271 tcg_gen_addi_ptr(a0, cpu_env, dofs);
272 tcg_gen_addi_ptr(a1, cpu_env, aofs);
273 tcg_gen_addi_ptr(a2, cpu_env, bofs);
274
275 fn(a0, a1, a2, ptr, desc);
276
277 tcg_temp_free_ptr(a0);
278 tcg_temp_free_ptr(a1);
279 tcg_temp_free_ptr(a2);
280 tcg_temp_free_i32(desc);
281}
282
283/* Generate a call to a gvec-style helper with four vector operands
284 and an extra pointer operand. */
285void tcg_gen_gvec_4_ptr(uint32_t dofs, uint32_t aofs, uint32_t bofs,
286 uint32_t cofs, TCGv_ptr ptr, uint32_t oprsz,
287 uint32_t maxsz, int32_t data,
288 gen_helper_gvec_4_ptr *fn)
289{
290 TCGv_ptr a0, a1, a2, a3;
291 TCGv_i32 desc = tcg_const_i32(simd_desc(oprsz, maxsz, data));
292
293 a0 = tcg_temp_new_ptr();
294 a1 = tcg_temp_new_ptr();
295 a2 = tcg_temp_new_ptr();
296 a3 = tcg_temp_new_ptr();
297
298 tcg_gen_addi_ptr(a0, cpu_env, dofs);
299 tcg_gen_addi_ptr(a1, cpu_env, aofs);
300 tcg_gen_addi_ptr(a2, cpu_env, bofs);
301 tcg_gen_addi_ptr(a3, cpu_env, cofs);
302
303 fn(a0, a1, a2, a3, ptr, desc);
304
305 tcg_temp_free_ptr(a0);
306 tcg_temp_free_ptr(a1);
307 tcg_temp_free_ptr(a2);
308 tcg_temp_free_ptr(a3);
309 tcg_temp_free_i32(desc);
310}
311
24459716
RH		 312/* Generate a call to a gvec-style helper with five vector operands
313 and an extra pointer operand. */
314void tcg_gen_gvec_5_ptr(uint32_t dofs, uint32_t aofs, uint32_t bofs,
315 uint32_t cofs, uint32_t eofs, TCGv_ptr ptr,
316 uint32_t oprsz, uint32_t maxsz, int32_t data,
317 gen_helper_gvec_5_ptr *fn)
318{
319 TCGv_ptr a0, a1, a2, a3, a4;
320 TCGv_i32 desc = tcg_const_i32(simd_desc(oprsz, maxsz, data));
321
322 a0 = tcg_temp_new_ptr();
323 a1 = tcg_temp_new_ptr();
324 a2 = tcg_temp_new_ptr();
325 a3 = tcg_temp_new_ptr();
326 a4 = tcg_temp_new_ptr();
327
328 tcg_gen_addi_ptr(a0, cpu_env, dofs);
329 tcg_gen_addi_ptr(a1, cpu_env, aofs);
330 tcg_gen_addi_ptr(a2, cpu_env, bofs);
331 tcg_gen_addi_ptr(a3, cpu_env, cofs);
332 tcg_gen_addi_ptr(a4, cpu_env, eofs);
333
334 fn(a0, a1, a2, a3, a4, ptr, desc);
335
336 tcg_temp_free_ptr(a0);
337 tcg_temp_free_ptr(a1);
338 tcg_temp_free_ptr(a2);
339 tcg_temp_free_ptr(a3);
340 tcg_temp_free_ptr(a4);
341 tcg_temp_free_i32(desc);
342}
343
db432672
RH		 344/* Return true if we want to implement something of OPRSZ bytes
345 in units of LNSZ. This limits the expansion of inline code. */
346static inline bool check_size_impl(uint32_t oprsz, uint32_t lnsz)
347{
f47db80c
RH		 348 uint32_t q, r;
349
350 if (oprsz < lnsz) {
351 return false;
352 }
353
354 q = oprsz / lnsz;
355 r = oprsz % lnsz;
356 tcg_debug_assert((r & 7) == 0);
357
358 if (lnsz < 16) {
359 /* For sizes below 16, accept no remainder. */
360 if (r != 0) {
361 return false;
362 }
363 } else {
364 /*
365 * Recall that ARM SVE allows vector sizes that are not a
366 * power of 2, but always a multiple of 16. The intent is
367 * that e.g. size == 80 would be expanded with 2x32 + 1x16.
368 * In addition, expand_clr needs to handle a multiple of 8.
369 * Thus we can handle the tail with one more operation per
370 * diminishing power of 2.
371 */
372 q += ctpop32(r);
499748d7 373 }
f47db80c
RH		 374
375 return q <= MAX_UNROLL;
db432672
RH		 376}
377
378static void expand_clr(uint32_t dofs, uint32_t maxsz);
379
380/* Duplicate C as per VECE. */
381uint64_t (dup_const)(unsigned vece, uint64_t c)
382{
383 switch (vece) {
384 case MO_8:
385 return 0x0101010101010101ull * (uint8_t)c;
386 case MO_16:
387 return 0x0001000100010001ull * (uint16_t)c;
388 case MO_32:
389 return 0x0000000100000001ull * (uint32_t)c;
390 case MO_64:
391 return c;
392 default:
393 g_assert_not_reached();
394 }
395}
396
397/* Duplicate IN into OUT as per VECE. */
398static void gen_dup_i32(unsigned vece, TCGv_i32 out, TCGv_i32 in)
399{
400 switch (vece) {
401 case MO_8:
402 tcg_gen_ext8u_i32(out, in);
403 tcg_gen_muli_i32(out, out, 0x01010101);
404 break;
405 case MO_16:
406 tcg_gen_deposit_i32(out, in, in, 16, 16);
407 break;
408 case MO_32:
409 tcg_gen_mov_i32(out, in);
410 break;
411 default:
412 g_assert_not_reached();
413 }
414}
415
416static void gen_dup_i64(unsigned vece, TCGv_i64 out, TCGv_i64 in)
417{
418 switch (vece) {
419 case MO_8:
420 tcg_gen_ext8u_i64(out, in);
421 tcg_gen_muli_i64(out, out, 0x0101010101010101ull);
422 break;
423 case MO_16:
424 tcg_gen_ext16u_i64(out, in);
425 tcg_gen_muli_i64(out, out, 0x0001000100010001ull);
426 break;
427 case MO_32:
428 tcg_gen_deposit_i64(out, in, in, 32, 32);
429 break;
430 case MO_64:
431 tcg_gen_mov_i64(out, in);
432 break;
433 default:
434 g_assert_not_reached();
435 }
436}
437
adb196cb
RH		 438/* Select a supported vector type for implementing an operation on SIZE
439 * bytes. If OP is 0, assume that the real operation to be performed is
440 * required by all backends. Otherwise, make sure than OP can be performed
441 * on elements of size VECE in the selected type. Do not select V64 if
442 * PREFER_I64 is true. Return 0 if no vector type is selected.
443 */
53229a77
RH		 444static TCGType choose_vector_type(const TCGOpcode *list, unsigned vece,
445 uint32_t size, bool prefer_i64)
adb196cb 446{
f47db80c
RH		 447 /*
448 * Recall that ARM SVE allows vector sizes that are not a
449 * power of 2, but always a multiple of 16. The intent is
450 * that e.g. size == 80 would be expanded with 2x32 + 1x16.
451 * It is hard to imagine a case in which v256 is supported
452 * but v128 is not, but check anyway.
453 * In addition, expand_clr needs to handle a multiple of 8.
454 */
455 if (TCG_TARGET_HAS_v256 &&
456 check_size_impl(size, 32) &&
457 tcg_can_emit_vecop_list(list, TCG_TYPE_V256, vece) &&
458 (!(size & 16) ||
459 (TCG_TARGET_HAS_v128 &&
460 tcg_can_emit_vecop_list(list, TCG_TYPE_V128, vece))) &&
461 (!(size & 8) ||
462 (TCG_TARGET_HAS_v64 &&
463 tcg_can_emit_vecop_list(list, TCG_TYPE_V64, vece)))) {
464 return TCG_TYPE_V256;
adb196cb 465 }
f47db80c
RH		 466 if (TCG_TARGET_HAS_v128 &&
467 check_size_impl(size, 16) &&
468 tcg_can_emit_vecop_list(list, TCG_TYPE_V128, vece) &&
469 (!(size & 8) ||
470 (TCG_TARGET_HAS_v64 &&
471 tcg_can_emit_vecop_list(list, TCG_TYPE_V64, vece)))) {
adb196cb
RH		 472 return TCG_TYPE_V128;
473 }
474 if (TCG_TARGET_HAS_v64 && !prefer_i64 && check_size_impl(size, 8)
53229a77 475 && tcg_can_emit_vecop_list(list, TCG_TYPE_V64, vece)) {
adb196cb
RH		 476 return TCG_TYPE_V64;
477 }
478 return 0;
479}
480
37ee55a0
RH		 481static void do_dup_store(TCGType type, uint32_t dofs, uint32_t oprsz,
482 uint32_t maxsz, TCGv_vec t_vec)
483{
484 uint32_t i = 0;
485
f47db80c
RH		 486 tcg_debug_assert(oprsz >= 8);
487
488 /*
489 * This may be expand_clr for the tail of an operation, e.g.
490 * oprsz == 8 && maxsz == 64. The first 8 bytes of this store
491 * are misaligned wrt the maximum vector size, so do that first.
492 */
493 if (dofs & 8) {
494 tcg_gen_stl_vec(t_vec, cpu_env, dofs + i, TCG_TYPE_V64);
495 i += 8;
496 }
497
37ee55a0
RH		 498 switch (type) {
499 case TCG_TYPE_V256:
500 /*
501 * Recall that ARM SVE allows vector sizes that are not a
502 * power of 2, but always a multiple of 16. The intent is
503 * that e.g. size == 80 would be expanded with 2x32 + 1x16.
504 */
505 for (; i + 32 <= oprsz; i += 32) {
506 tcg_gen_stl_vec(t_vec, cpu_env, dofs + i, TCG_TYPE_V256);
507 }
508 /* fallthru */
509 case TCG_TYPE_V128:
510 for (; i + 16 <= oprsz; i += 16) {
511 tcg_gen_stl_vec(t_vec, cpu_env, dofs + i, TCG_TYPE_V128);
512 }
513 break;
514 case TCG_TYPE_V64:
515 for (; i < oprsz; i += 8) {
516 tcg_gen_stl_vec(t_vec, cpu_env, dofs + i, TCG_TYPE_V64);
517 }
518 break;
519 default:
520 g_assert_not_reached();
521 }
522
523 if (oprsz < maxsz) {
524 expand_clr(dofs + oprsz, maxsz - oprsz);
525 }
526}
527
db432672
RH		 528/* Set OPRSZ bytes at DOFS to replications of IN_32, IN_64 or IN_C.
529 * Only one of IN_32 or IN_64 may be set;
530 * IN_C is used if IN_32 and IN_64 are unset.
531 */
532static void do_dup(unsigned vece, uint32_t dofs, uint32_t oprsz,
533 uint32_t maxsz, TCGv_i32 in_32, TCGv_i64 in_64,
534 uint64_t in_c)
535{
536 TCGType type;
537 TCGv_i64 t_64;
538 TCGv_i32 t_32, t_desc;
539 TCGv_ptr t_ptr;
540 uint32_t i;
541
542 assert(vece <= (in_32 ? MO_32 : MO_64));
543 assert(in_32 == NULL || in_64 == NULL);
544
545 /* If we're storing 0, expand oprsz to maxsz. */
546 if (in_32 == NULL && in_64 == NULL) {
547 in_c = dup_const(vece, in_c);
548 if (in_c == 0) {
549 oprsz = maxsz;
6d3ef048
RH		 550 vece = MO_8;
551 } else if (in_c == dup_const(MO_8, in_c)) {
552 vece = MO_8;
db432672
RH		 553 }
554 }
555
adb196cb
RH		 556 /* Implement inline with a vector type, if possible.
557 * Prefer integer when 64-bit host and no variable dup.
558 */
53229a77 559 type = choose_vector_type(NULL, vece, oprsz,
adb196cb
RH		 560 (TCG_TARGET_REG_BITS == 64 && in_32 == NULL
561 && (in_64 == NULL || vece == MO_64)));
db432672
RH		 562 if (type != 0) {
563 TCGv_vec t_vec = tcg_temp_new_vec(type);
564
565 if (in_32) {
566 tcg_gen_dup_i32_vec(vece, t_vec, in_32);
567 } else if (in_64) {
568 tcg_gen_dup_i64_vec(vece, t_vec, in_64);
569 } else {
37ee55a0 570 tcg_gen_dupi_vec(vece, t_vec, in_c);
db432672 571 }
37ee55a0 572 do_dup_store(type, dofs, oprsz, maxsz, t_vec);
db432672 573 tcg_temp_free_vec(t_vec);
37ee55a0 574 return;
db432672
RH		 575 }
576
577 /* Otherwise, inline with an integer type, unless "large". */
578 if (check_size_impl(oprsz, TCG_TARGET_REG_BITS / 8)) {
579 t_64 = NULL;
580 t_32 = NULL;
581
582 if (in_32) {
583 /* We are given a 32-bit variable input. For a 64-bit host,
584 use a 64-bit operation unless the 32-bit operation would
585 be simple enough. */
586 if (TCG_TARGET_REG_BITS == 64
587 && (vece != MO_32 || !check_size_impl(oprsz, 4))) {
588 t_64 = tcg_temp_new_i64();
589 tcg_gen_extu_i32_i64(t_64, in_32);
590 gen_dup_i64(vece, t_64, t_64);
591 } else {
592 t_32 = tcg_temp_new_i32();
593 gen_dup_i32(vece, t_32, in_32);
594 }
595 } else if (in_64) {
596 /* We are given a 64-bit variable input. */
597 t_64 = tcg_temp_new_i64();
598 gen_dup_i64(vece, t_64, in_64);
599 } else {
600 /* We are given a constant input. */
601 /* For 64-bit hosts, use 64-bit constants for "simple" constants
602 or when we'd need too many 32-bit stores, or when a 64-bit
603 constant is really required. */
604 if (vece == MO_64
605 || (TCG_TARGET_REG_BITS == 64
606 && (in_c == 0 || in_c == -1
607 || !check_size_impl(oprsz, 4)))) {
608 t_64 = tcg_const_i64(in_c);
609 } else {
610 t_32 = tcg_const_i32(in_c);
611 }
612 }
613
614 /* Implement inline if we picked an implementation size above. */
615 if (t_32) {
616 for (i = 0; i < oprsz; i += 4) {
617 tcg_gen_st_i32(t_32, cpu_env, dofs + i);
618 }
619 tcg_temp_free_i32(t_32);
620 goto done;
621 }
622 if (t_64) {
623 for (i = 0; i < oprsz; i += 8) {
624 tcg_gen_st_i64(t_64, cpu_env, dofs + i);
625 }
626 tcg_temp_free_i64(t_64);
627 goto done;
adb196cb 628 }
db432672
RH		 629 }
630
631 /* Otherwise implement out of line. */
632 t_ptr = tcg_temp_new_ptr();
633 tcg_gen_addi_ptr(t_ptr, cpu_env, dofs);
6d3ef048
RH		 634
635 /*
636 * This may be expand_clr for the tail of an operation, e.g.
637 * oprsz == 8 && maxsz == 64. The size of the clear is misaligned
638 * wrt simd_desc and will assert. Simply pass all replicated byte
639 * stores through to memset.
640 */
641 if (oprsz == maxsz && vece == MO_8) {
642 TCGv_ptr t_size = tcg_const_ptr(oprsz);
643 TCGv_i32 t_val;
644
645 if (in_32) {
646 t_val = in_32;
647 } else if (in_64) {
648 t_val = tcg_temp_new_i32();
649 tcg_gen_extrl_i64_i32(t_val, in_64);
650 } else {
651 t_val = tcg_const_i32(in_c);
652 }
653 gen_helper_memset(t_ptr, t_ptr, t_val, t_size);
654
655 if (!in_32) {
656 tcg_temp_free_i32(t_val);
657 }
658 tcg_temp_free_ptr(t_size);
659 tcg_temp_free_ptr(t_ptr);
660 return;
661 }
662
db432672
RH		 663 t_desc = tcg_const_i32(simd_desc(oprsz, maxsz, 0));
664
665 if (vece == MO_64) {
666 if (in_64) {
667 gen_helper_gvec_dup64(t_ptr, t_desc, in_64);
668 } else {
669 t_64 = tcg_const_i64(in_c);
670 gen_helper_gvec_dup64(t_ptr, t_desc, t_64);
671 tcg_temp_free_i64(t_64);
672 }
673 } else {
674 typedef void dup_fn(TCGv_ptr, TCGv_i32, TCGv_i32);
675 static dup_fn * const fns[3] = {
676 gen_helper_gvec_dup8,
677 gen_helper_gvec_dup16,
678 gen_helper_gvec_dup32
679 };
680
681 if (in_32) {
682 fns[vece](t_ptr, t_desc, in_32);
683 } else {
684 t_32 = tcg_temp_new_i32();
685 if (in_64) {
686 tcg_gen_extrl_i64_i32(t_32, in_64);
687 } else if (vece == MO_8) {
688 tcg_gen_movi_i32(t_32, in_c & 0xff);
689 } else if (vece == MO_16) {
690 tcg_gen_movi_i32(t_32, in_c & 0xffff);
691 } else {
692 tcg_gen_movi_i32(t_32, in_c);
693 }
694 fns[vece](t_ptr, t_desc, t_32);
695 tcg_temp_free_i32(t_32);
696 }
697 }
698
699 tcg_temp_free_ptr(t_ptr);
700 tcg_temp_free_i32(t_desc);
701 return;
702
703 done:
704 if (oprsz < maxsz) {
705 expand_clr(dofs + oprsz, maxsz - oprsz);
706 }
707}
708
709/* Likewise, but with zero. */
710static void expand_clr(uint32_t dofs, uint32_t maxsz)
711{
712 do_dup(MO_8, dofs, maxsz, maxsz, NULL, NULL, 0);
713}
714
715/* Expand OPSZ bytes worth of two-operand operations using i32 elements. */
716static void expand_2_i32(uint32_t dofs, uint32_t aofs, uint32_t oprsz,
ac09ae62 717 bool load_dest, void (*fni)(TCGv_i32, TCGv_i32))
db432672
RH		 718{
719 TCGv_i32 t0 = tcg_temp_new_i32();
ac09ae62 720 TCGv_i32 t1 = tcg_temp_new_i32();
db432672
RH		 721 uint32_t i;
722
723 for (i = 0; i < oprsz; i += 4) {
724 tcg_gen_ld_i32(t0, cpu_env, aofs + i);
ac09ae62
RH		 725 if (load_dest) {
726 tcg_gen_ld_i32(t1, cpu_env, dofs + i);
727 }
728 fni(t1, t0);
729 tcg_gen_st_i32(t1, cpu_env, dofs + i);
db432672
RH		 730 }
731 tcg_temp_free_i32(t0);
ac09ae62 732 tcg_temp_free_i32(t1);
db432672
RH		 733}
734
d0ec9796
RH		 735static void expand_2i_i32(uint32_t dofs, uint32_t aofs, uint32_t oprsz,
736 int32_t c, bool load_dest,
737 void (*fni)(TCGv_i32, TCGv_i32, int32_t))
738{
739 TCGv_i32 t0 = tcg_temp_new_i32();
740 TCGv_i32 t1 = tcg_temp_new_i32();
741 uint32_t i;
742
743 for (i = 0; i < oprsz; i += 4) {
744 tcg_gen_ld_i32(t0, cpu_env, aofs + i);
745 if (load_dest) {
746 tcg_gen_ld_i32(t1, cpu_env, dofs + i);
747 }
748 fni(t1, t0, c);
749 tcg_gen_st_i32(t1, cpu_env, dofs + i);
750 }
751 tcg_temp_free_i32(t0);
752 tcg_temp_free_i32(t1);
753}
754
22fc3527
RH		 755static void expand_2s_i32(uint32_t dofs, uint32_t aofs, uint32_t oprsz,
756 TCGv_i32 c, bool scalar_first,
757 void (*fni)(TCGv_i32, TCGv_i32, TCGv_i32))
758{
759 TCGv_i32 t0 = tcg_temp_new_i32();
760 TCGv_i32 t1 = tcg_temp_new_i32();
761 uint32_t i;
762
763 for (i = 0; i < oprsz; i += 4) {
764 tcg_gen_ld_i32(t0, cpu_env, aofs + i);
765 if (scalar_first) {
766 fni(t1, c, t0);
767 } else {
768 fni(t1, t0, c);
769 }
770 tcg_gen_st_i32(t1, cpu_env, dofs + i);
771 }
772 tcg_temp_free_i32(t0);
773 tcg_temp_free_i32(t1);
774}
775
db432672
RH		 776/* Expand OPSZ bytes worth of three-operand operations using i32 elements. */
777static void expand_3_i32(uint32_t dofs, uint32_t aofs,
778 uint32_t bofs, uint32_t oprsz, bool load_dest,
779 void (*fni)(TCGv_i32, TCGv_i32, TCGv_i32))
780{
781 TCGv_i32 t0 = tcg_temp_new_i32();
782 TCGv_i32 t1 = tcg_temp_new_i32();
783 TCGv_i32 t2 = tcg_temp_new_i32();
784 uint32_t i;
785
786 for (i = 0; i < oprsz; i += 4) {
787 tcg_gen_ld_i32(t0, cpu_env, aofs + i);
788 tcg_gen_ld_i32(t1, cpu_env, bofs + i);
789 if (load_dest) {
790 tcg_gen_ld_i32(t2, cpu_env, dofs + i);
791 }
792 fni(t2, t0, t1);
793 tcg_gen_st_i32(t2, cpu_env, dofs + i);
794 }
795 tcg_temp_free_i32(t2);
796 tcg_temp_free_i32(t1);
797 tcg_temp_free_i32(t0);
798}
799
e1227bb6
DH		 800static void expand_3i_i32(uint32_t dofs, uint32_t aofs, uint32_t bofs,
801 uint32_t oprsz, int32_t c, bool load_dest,
802 void (*fni)(TCGv_i32, TCGv_i32, TCGv_i32, int32_t))
803{
804 TCGv_i32 t0 = tcg_temp_new_i32();
805 TCGv_i32 t1 = tcg_temp_new_i32();
806 TCGv_i32 t2 = tcg_temp_new_i32();
807 uint32_t i;
808
809 for (i = 0; i < oprsz; i += 4) {
810 tcg_gen_ld_i32(t0, cpu_env, aofs + i);
811 tcg_gen_ld_i32(t1, cpu_env, bofs + i);
812 if (load_dest) {
813 tcg_gen_ld_i32(t2, cpu_env, dofs + i);
814 }
815 fni(t2, t0, t1, c);
816 tcg_gen_st_i32(t2, cpu_env, dofs + i);
817 }
818 tcg_temp_free_i32(t0);
819 tcg_temp_free_i32(t1);
820 tcg_temp_free_i32(t2);
821}
822
db432672
RH		 823/* Expand OPSZ bytes worth of three-operand operations using i32 elements. */
824static void expand_4_i32(uint32_t dofs, uint32_t aofs, uint32_t bofs,
5d6acdd4 825 uint32_t cofs, uint32_t oprsz, bool write_aofs,
db432672
RH		 826 void (*fni)(TCGv_i32, TCGv_i32, TCGv_i32, TCGv_i32))
827{
828 TCGv_i32 t0 = tcg_temp_new_i32();
829 TCGv_i32 t1 = tcg_temp_new_i32();
830 TCGv_i32 t2 = tcg_temp_new_i32();
831 TCGv_i32 t3 = tcg_temp_new_i32();
832 uint32_t i;
833
834 for (i = 0; i < oprsz; i += 4) {
835 tcg_gen_ld_i32(t1, cpu_env, aofs + i);
836 tcg_gen_ld_i32(t2, cpu_env, bofs + i);
837 tcg_gen_ld_i32(t3, cpu_env, cofs + i);
838 fni(t0, t1, t2, t3);
839 tcg_gen_st_i32(t0, cpu_env, dofs + i);
5d6acdd4
RH		 840 if (write_aofs) {
841 tcg_gen_st_i32(t1, cpu_env, aofs + i);
842 }
db432672
RH		 843 }
844 tcg_temp_free_i32(t3);
845 tcg_temp_free_i32(t2);
846 tcg_temp_free_i32(t1);
847 tcg_temp_free_i32(t0);
848}
849
850/* Expand OPSZ bytes worth of two-operand operations using i64 elements. */
851static void expand_2_i64(uint32_t dofs, uint32_t aofs, uint32_t oprsz,
ac09ae62 852 bool load_dest, void (*fni)(TCGv_i64, TCGv_i64))
db432672
RH		 853{
854 TCGv_i64 t0 = tcg_temp_new_i64();
ac09ae62 855 TCGv_i64 t1 = tcg_temp_new_i64();
db432672
RH		 856 uint32_t i;
857
858 for (i = 0; i < oprsz; i += 8) {
859 tcg_gen_ld_i64(t0, cpu_env, aofs + i);
ac09ae62
RH		 860 if (load_dest) {
861 tcg_gen_ld_i64(t1, cpu_env, dofs + i);
862 }
863 fni(t1, t0);
864 tcg_gen_st_i64(t1, cpu_env, dofs + i);
db432672
RH		 865 }
866 tcg_temp_free_i64(t0);
ac09ae62 867 tcg_temp_free_i64(t1);
db432672
RH		 868}
869
d0ec9796
RH		 870static void expand_2i_i64(uint32_t dofs, uint32_t aofs, uint32_t oprsz,
871 int64_t c, bool load_dest,
872 void (*fni)(TCGv_i64, TCGv_i64, int64_t))
873{
874 TCGv_i64 t0 = tcg_temp_new_i64();
875 TCGv_i64 t1 = tcg_temp_new_i64();
876 uint32_t i;
877
878 for (i = 0; i < oprsz; i += 8) {
879 tcg_gen_ld_i64(t0, cpu_env, aofs + i);
880 if (load_dest) {
881 tcg_gen_ld_i64(t1, cpu_env, dofs + i);
882 }
883 fni(t1, t0, c);
884 tcg_gen_st_i64(t1, cpu_env, dofs + i);
885 }
886 tcg_temp_free_i64(t0);
887 tcg_temp_free_i64(t1);
888}
889
22fc3527
RH		 890static void expand_2s_i64(uint32_t dofs, uint32_t aofs, uint32_t oprsz,
891 TCGv_i64 c, bool scalar_first,
892 void (*fni)(TCGv_i64, TCGv_i64, TCGv_i64))
893{
894 TCGv_i64 t0 = tcg_temp_new_i64();
895 TCGv_i64 t1 = tcg_temp_new_i64();
896 uint32_t i;
897
898 for (i = 0; i < oprsz; i += 8) {
899 tcg_gen_ld_i64(t0, cpu_env, aofs + i);
900 if (scalar_first) {
901 fni(t1, c, t0);
902 } else {
903 fni(t1, t0, c);
904 }
905 tcg_gen_st_i64(t1, cpu_env, dofs + i);
906 }
907 tcg_temp_free_i64(t0);
908 tcg_temp_free_i64(t1);
909}
910
db432672
RH		 911/* Expand OPSZ bytes worth of three-operand operations using i64 elements. */
912static void expand_3_i64(uint32_t dofs, uint32_t aofs,
913 uint32_t bofs, uint32_t oprsz, bool load_dest,
914 void (*fni)(TCGv_i64, TCGv_i64, TCGv_i64))
915{
916 TCGv_i64 t0 = tcg_temp_new_i64();
917 TCGv_i64 t1 = tcg_temp_new_i64();
918 TCGv_i64 t2 = tcg_temp_new_i64();
919 uint32_t i;
920
921 for (i = 0; i < oprsz; i += 8) {
922 tcg_gen_ld_i64(t0, cpu_env, aofs + i);
923 tcg_gen_ld_i64(t1, cpu_env, bofs + i);
924 if (load_dest) {
925 tcg_gen_ld_i64(t2, cpu_env, dofs + i);
926 }
927 fni(t2, t0, t1);
928 tcg_gen_st_i64(t2, cpu_env, dofs + i);
929 }
930 tcg_temp_free_i64(t2);
931 tcg_temp_free_i64(t1);
932 tcg_temp_free_i64(t0);
933}
934
e1227bb6
DH		 935static void expand_3i_i64(uint32_t dofs, uint32_t aofs, uint32_t bofs,
936 uint32_t oprsz, int64_t c, bool load_dest,
937 void (*fni)(TCGv_i64, TCGv_i64, TCGv_i64, int64_t))
938{
939 TCGv_i64 t0 = tcg_temp_new_i64();
940 TCGv_i64 t1 = tcg_temp_new_i64();
941 TCGv_i64 t2 = tcg_temp_new_i64();
942 uint32_t i;
943
944 for (i = 0; i < oprsz; i += 8) {
945 tcg_gen_ld_i64(t0, cpu_env, aofs + i);
946 tcg_gen_ld_i64(t1, cpu_env, bofs + i);
947 if (load_dest) {
948 tcg_gen_ld_i64(t2, cpu_env, dofs + i);
949 }
950 fni(t2, t0, t1, c);
951 tcg_gen_st_i64(t2, cpu_env, dofs + i);
952 }
953 tcg_temp_free_i64(t0);
954 tcg_temp_free_i64(t1);
955 tcg_temp_free_i64(t2);
956}
957
db432672
RH		 958/* Expand OPSZ bytes worth of three-operand operations using i64 elements. */
959static void expand_4_i64(uint32_t dofs, uint32_t aofs, uint32_t bofs,
5d6acdd4 960 uint32_t cofs, uint32_t oprsz, bool write_aofs,
db432672
RH		 961 void (*fni)(TCGv_i64, TCGv_i64, TCGv_i64, TCGv_i64))
962{
963 TCGv_i64 t0 = tcg_temp_new_i64();
964 TCGv_i64 t1 = tcg_temp_new_i64();
965 TCGv_i64 t2 = tcg_temp_new_i64();
966 TCGv_i64 t3 = tcg_temp_new_i64();
967 uint32_t i;
968
969 for (i = 0; i < oprsz; i += 8) {
970 tcg_gen_ld_i64(t1, cpu_env, aofs + i);
971 tcg_gen_ld_i64(t2, cpu_env, bofs + i);
972 tcg_gen_ld_i64(t3, cpu_env, cofs + i);
973 fni(t0, t1, t2, t3);
974 tcg_gen_st_i64(t0, cpu_env, dofs + i);
5d6acdd4
RH		 975 if (write_aofs) {
976 tcg_gen_st_i64(t1, cpu_env, aofs + i);
977 }
db432672
RH		 978 }
979 tcg_temp_free_i64(t3);
980 tcg_temp_free_i64(t2);
981 tcg_temp_free_i64(t1);
982 tcg_temp_free_i64(t0);
983}
984
985/* Expand OPSZ bytes worth of two-operand operations using host vectors. */
986static void expand_2_vec(unsigned vece, uint32_t dofs, uint32_t aofs,
987 uint32_t oprsz, uint32_t tysz, TCGType type,
ac09ae62 988 bool load_dest,
db432672
RH		 989 void (*fni)(unsigned, TCGv_vec, TCGv_vec))
990{
991 TCGv_vec t0 = tcg_temp_new_vec(type);
ac09ae62 992 TCGv_vec t1 = tcg_temp_new_vec(type);
db432672
RH		 993 uint32_t i;
994
995 for (i = 0; i < oprsz; i += tysz) {
996 tcg_gen_ld_vec(t0, cpu_env, aofs + i);
ac09ae62
RH		 997 if (load_dest) {
998 tcg_gen_ld_vec(t1, cpu_env, dofs + i);
999 }
1000 fni(vece, t1, t0);
1001 tcg_gen_st_vec(t1, cpu_env, dofs + i);
db432672
RH		 1002 }
1003 tcg_temp_free_vec(t0);
ac09ae62 1004 tcg_temp_free_vec(t1);
db432672
RH		 1005}
1006
d0ec9796
RH		 1007/* Expand OPSZ bytes worth of two-vector operands and an immediate operand
1008 using host vectors. */
1009static void expand_2i_vec(unsigned vece, uint32_t dofs, uint32_t aofs,
1010 uint32_t oprsz, uint32_t tysz, TCGType type,
1011 int64_t c, bool load_dest,
1012 void (*fni)(unsigned, TCGv_vec, TCGv_vec, int64_t))
1013{
1014 TCGv_vec t0 = tcg_temp_new_vec(type);
1015 TCGv_vec t1 = tcg_temp_new_vec(type);
1016 uint32_t i;
1017
1018 for (i = 0; i < oprsz; i += tysz) {
1019 tcg_gen_ld_vec(t0, cpu_env, aofs + i);
1020 if (load_dest) {
1021 tcg_gen_ld_vec(t1, cpu_env, dofs + i);
1022 }
1023 fni(vece, t1, t0, c);
1024 tcg_gen_st_vec(t1, cpu_env, dofs + i);
1025 }
1026 tcg_temp_free_vec(t0);
1027 tcg_temp_free_vec(t1);
1028}
1029
22fc3527
RH		 1030static void expand_2s_vec(unsigned vece, uint32_t dofs, uint32_t aofs,
1031 uint32_t oprsz, uint32_t tysz, TCGType type,
1032 TCGv_vec c, bool scalar_first,
1033 void (*fni)(unsigned, TCGv_vec, TCGv_vec, TCGv_vec))
1034{
1035 TCGv_vec t0 = tcg_temp_new_vec(type);
1036 TCGv_vec t1 = tcg_temp_new_vec(type);
1037 uint32_t i;
1038
1039 for (i = 0; i < oprsz; i += tysz) {
1040 tcg_gen_ld_vec(t0, cpu_env, aofs + i);
1041 if (scalar_first) {
1042 fni(vece, t1, c, t0);
1043 } else {
1044 fni(vece, t1, t0, c);
1045 }
1046 tcg_gen_st_vec(t1, cpu_env, dofs + i);
1047 }
1048 tcg_temp_free_vec(t0);
1049 tcg_temp_free_vec(t1);
1050}
1051
db432672
RH		 1052/* Expand OPSZ bytes worth of three-operand operations using host vectors. */
1053static void expand_3_vec(unsigned vece, uint32_t dofs, uint32_t aofs,
1054 uint32_t bofs, uint32_t oprsz,
1055 uint32_t tysz, TCGType type, bool load_dest,
1056 void (*fni)(unsigned, TCGv_vec, TCGv_vec, TCGv_vec))
1057{
1058 TCGv_vec t0 = tcg_temp_new_vec(type);
1059 TCGv_vec t1 = tcg_temp_new_vec(type);
1060 TCGv_vec t2 = tcg_temp_new_vec(type);
1061 uint32_t i;
1062
1063 for (i = 0; i < oprsz; i += tysz) {
1064 tcg_gen_ld_vec(t0, cpu_env, aofs + i);
1065 tcg_gen_ld_vec(t1, cpu_env, bofs + i);
1066 if (load_dest) {
1067 tcg_gen_ld_vec(t2, cpu_env, dofs + i);
1068 }
1069 fni(vece, t2, t0, t1);
1070 tcg_gen_st_vec(t2, cpu_env, dofs + i);
1071 }
1072 tcg_temp_free_vec(t2);
1073 tcg_temp_free_vec(t1);
1074 tcg_temp_free_vec(t0);
1075}
1076
e1227bb6
DH		 1077/*
1078 * Expand OPSZ bytes worth of three-vector operands and an immediate operand
1079 * using host vectors.
1080 */
1081static void expand_3i_vec(unsigned vece, uint32_t dofs, uint32_t aofs,
1082 uint32_t bofs, uint32_t oprsz, uint32_t tysz,
1083 TCGType type, int64_t c, bool load_dest,
1084 void (*fni)(unsigned, TCGv_vec, TCGv_vec, TCGv_vec,
1085 int64_t))
1086{
1087 TCGv_vec t0 = tcg_temp_new_vec(type);
1088 TCGv_vec t1 = tcg_temp_new_vec(type);
1089 TCGv_vec t2 = tcg_temp_new_vec(type);
1090 uint32_t i;
1091
1092 for (i = 0; i < oprsz; i += tysz) {
1093 tcg_gen_ld_vec(t0, cpu_env, aofs + i);
1094 tcg_gen_ld_vec(t1, cpu_env, bofs + i);
1095 if (load_dest) {
1096 tcg_gen_ld_vec(t2, cpu_env, dofs + i);
1097 }
1098 fni(vece, t2, t0, t1, c);
1099 tcg_gen_st_vec(t2, cpu_env, dofs + i);
1100 }
1101 tcg_temp_free_vec(t0);
1102 tcg_temp_free_vec(t1);
1103 tcg_temp_free_vec(t2);
1104}
1105
db432672
RH		 1106/* Expand OPSZ bytes worth of four-operand operations using host vectors. */
1107static void expand_4_vec(unsigned vece, uint32_t dofs, uint32_t aofs,
1108 uint32_t bofs, uint32_t cofs, uint32_t oprsz,
5d6acdd4 1109 uint32_t tysz, TCGType type, bool write_aofs,
db432672
RH		 1110 void (*fni)(unsigned, TCGv_vec, TCGv_vec,
1111 TCGv_vec, TCGv_vec))
1112{
1113 TCGv_vec t0 = tcg_temp_new_vec(type);
1114 TCGv_vec t1 = tcg_temp_new_vec(type);
1115 TCGv_vec t2 = tcg_temp_new_vec(type);
1116 TCGv_vec t3 = tcg_temp_new_vec(type);
1117 uint32_t i;
1118
1119 for (i = 0; i < oprsz; i += tysz) {
1120 tcg_gen_ld_vec(t1, cpu_env, aofs + i);
1121 tcg_gen_ld_vec(t2, cpu_env, bofs + i);
1122 tcg_gen_ld_vec(t3, cpu_env, cofs + i);
1123 fni(vece, t0, t1, t2, t3);
1124 tcg_gen_st_vec(t0, cpu_env, dofs + i);
5d6acdd4
RH		 1125 if (write_aofs) {
1126 tcg_gen_st_vec(t1, cpu_env, aofs + i);
1127 }
db432672
RH		 1128 }
1129 tcg_temp_free_vec(t3);
1130 tcg_temp_free_vec(t2);
1131 tcg_temp_free_vec(t1);
1132 tcg_temp_free_vec(t0);
1133}
1134
1135/* Expand a vector two-operand operation. */
1136void tcg_gen_gvec_2(uint32_t dofs, uint32_t aofs,
1137 uint32_t oprsz, uint32_t maxsz, const GVecGen2 *g)
1138{
53229a77
RH		 1139 const TCGOpcode *this_list = g->opt_opc ? : vecop_list_empty;
1140 const TCGOpcode *hold_list = tcg_swap_vecop_list(this_list);
adb196cb
RH		 1141 TCGType type;
1142 uint32_t some;
1143
db432672
RH		 1144 check_size_align(oprsz, maxsz, dofs | aofs);
1145 check_overlap_2(dofs, aofs, maxsz);
1146
adb196cb
RH		 1147 type = 0;
1148 if (g->fniv) {
53229a77 1149 type = choose_vector_type(g->opt_opc, g->vece, oprsz, g->prefer_i64);
adb196cb
RH		 1150 }
1151 switch (type) {
1152 case TCG_TYPE_V256:
1153 /* Recall that ARM SVE allows vector sizes that are not a
1154 * power of 2, but always a multiple of 16. The intent is
1155 * that e.g. size == 80 would be expanded with 2x32 + 1x16.
1156 */
1157 some = QEMU_ALIGN_DOWN(oprsz, 32);
ac09ae62
RH		 1158 expand_2_vec(g->vece, dofs, aofs, some, 32, TCG_TYPE_V256,
1159 g->load_dest, g->fniv);
db432672 1160 if (some == oprsz) {
adb196cb 1161 break;
db432672
RH		 1162 }
1163 dofs += some;
1164 aofs += some;
1165 oprsz -= some;
1166 maxsz -= some;
adb196cb
RH		 1167 /* fallthru */
1168 case TCG_TYPE_V128:
ac09ae62
RH		 1169 expand_2_vec(g->vece, dofs, aofs, oprsz, 16, TCG_TYPE_V128,
1170 g->load_dest, g->fniv);
adb196cb
RH		 1171 break;
1172 case TCG_TYPE_V64:
ac09ae62
RH		 1173 expand_2_vec(g->vece, dofs, aofs, oprsz, 8, TCG_TYPE_V64,
1174 g->load_dest, g->fniv);
adb196cb
RH		 1175 break;
1176
1177 case 0:
1178 if (g->fni8 && check_size_impl(oprsz, 8)) {
ac09ae62 1179 expand_2_i64(dofs, aofs, oprsz, g->load_dest, g->fni8);
adb196cb 1180 } else if (g->fni4 && check_size_impl(oprsz, 4)) {
ac09ae62 1181 expand_2_i32(dofs, aofs, oprsz, g->load_dest, g->fni4);
adb196cb
RH		 1182 } else {
1183 assert(g->fno != NULL);
1184 tcg_gen_gvec_2_ool(dofs, aofs, oprsz, maxsz, g->data, g->fno);
53229a77 1185 oprsz = maxsz;
adb196cb
RH		 1186 }
1187 break;
1188
1189 default:
1190 g_assert_not_reached();
db432672 1191 }
53229a77 1192 tcg_swap_vecop_list(hold_list);
db432672 1193
db432672
RH		 1194 if (oprsz < maxsz) {
1195 expand_clr(dofs + oprsz, maxsz - oprsz);
1196 }
1197}
1198
22fc3527 1199/* Expand a vector operation with two vectors and an immediate. */
d0ec9796
RH		 1200void tcg_gen_gvec_2i(uint32_t dofs, uint32_t aofs, uint32_t oprsz,
1201 uint32_t maxsz, int64_t c, const GVecGen2i *g)
1202{
53229a77
RH		 1203 const TCGOpcode *this_list = g->opt_opc ? : vecop_list_empty;
1204 const TCGOpcode *hold_list = tcg_swap_vecop_list(this_list);
adb196cb
RH		 1205 TCGType type;
1206 uint32_t some;
1207
d0ec9796
RH		 1208 check_size_align(oprsz, maxsz, dofs | aofs);
1209 check_overlap_2(dofs, aofs, maxsz);
1210
adb196cb
RH		 1211 type = 0;
1212 if (g->fniv) {
53229a77 1213 type = choose_vector_type(g->opt_opc, g->vece, oprsz, g->prefer_i64);
adb196cb
RH		 1214 }
1215 switch (type) {
1216 case TCG_TYPE_V256:
1217 /* Recall that ARM SVE allows vector sizes that are not a
1218 * power of 2, but always a multiple of 16. The intent is
1219 * that e.g. size == 80 would be expanded with 2x32 + 1x16.
1220 */
1221 some = QEMU_ALIGN_DOWN(oprsz, 32);
d0ec9796
RH		 1222 expand_2i_vec(g->vece, dofs, aofs, some, 32, TCG_TYPE_V256,
1223 c, g->load_dest, g->fniv);
1224 if (some == oprsz) {
adb196cb 1225 break;
d0ec9796
RH		 1226 }
1227 dofs += some;
1228 aofs += some;
1229 oprsz -= some;
1230 maxsz -= some;
adb196cb
RH		 1231 /* fallthru */
1232 case TCG_TYPE_V128:
d0ec9796
RH		 1233 expand_2i_vec(g->vece, dofs, aofs, oprsz, 16, TCG_TYPE_V128,
1234 c, g->load_dest, g->fniv);
adb196cb
RH		 1235 break;
1236 case TCG_TYPE_V64:
d0ec9796
RH		 1237 expand_2i_vec(g->vece, dofs, aofs, oprsz, 8, TCG_TYPE_V64,
1238 c, g->load_dest, g->fniv);
adb196cb
RH		 1239 break;
1240
1241 case 0:
1242 if (g->fni8 && check_size_impl(oprsz, 8)) {
1243 expand_2i_i64(dofs, aofs, oprsz, c, g->load_dest, g->fni8);
1244 } else if (g->fni4 && check_size_impl(oprsz, 4)) {
1245 expand_2i_i32(dofs, aofs, oprsz, c, g->load_dest, g->fni4);
22fc3527 1246 } else {
adb196cb
RH		 1247 if (g->fno) {
1248 tcg_gen_gvec_2_ool(dofs, aofs, oprsz, maxsz, c, g->fno);
1249 } else {
1250 TCGv_i64 tcg_c = tcg_const_i64(c);
1251 tcg_gen_gvec_2i_ool(dofs, aofs, tcg_c, oprsz,
1252 maxsz, c, g->fnoi);
1253 tcg_temp_free_i64(tcg_c);
1254 }
53229a77 1255 oprsz = maxsz;
22fc3527 1256 }
adb196cb
RH		 1257 break;
1258
1259 default:
1260 g_assert_not_reached();
d0ec9796 1261 }
53229a77 1262 tcg_swap_vecop_list(hold_list);
d0ec9796 1263
d0ec9796
RH		 1264 if (oprsz < maxsz) {
1265 expand_clr(dofs + oprsz, maxsz - oprsz);
1266 }
1267}
1268
22fc3527
RH		 1269/* Expand a vector operation with two vectors and a scalar. */
1270void tcg_gen_gvec_2s(uint32_t dofs, uint32_t aofs, uint32_t oprsz,
1271 uint32_t maxsz, TCGv_i64 c, const GVecGen2s *g)
1272{
1273 TCGType type;
1274
1275 check_size_align(oprsz, maxsz, dofs | aofs);
1276 check_overlap_2(dofs, aofs, maxsz);
1277
1278 type = 0;
1279 if (g->fniv) {
53229a77 1280 type = choose_vector_type(g->opt_opc, g->vece, oprsz, g->prefer_i64);
22fc3527
RH		 1281 }
1282 if (type != 0) {
53229a77
RH		 1283 const TCGOpcode *this_list = g->opt_opc ? : vecop_list_empty;
1284 const TCGOpcode *hold_list = tcg_swap_vecop_list(this_list);
22fc3527 1285 TCGv_vec t_vec = tcg_temp_new_vec(type);
adb196cb 1286 uint32_t some;
22fc3527
RH		 1287
1288 tcg_gen_dup_i64_vec(g->vece, t_vec, c);
1289
22fc3527
RH		 1290 switch (type) {
1291 case TCG_TYPE_V256:
adb196cb
RH		 1292 /* Recall that ARM SVE allows vector sizes that are not a
1293 * power of 2, but always a multiple of 16. The intent is
1294 * that e.g. size == 80 would be expanded with 2x32 + 1x16.
1295 */
1296 some = QEMU_ALIGN_DOWN(oprsz, 32);
1297 expand_2s_vec(g->vece, dofs, aofs, some, 32, TCG_TYPE_V256,
1298 t_vec, g->scalar_first, g->fniv);
1299 if (some == oprsz) {
1300 break;
22fc3527 1301 }
adb196cb
RH		 1302 dofs += some;
1303 aofs += some;
1304 oprsz -= some;
1305 maxsz -= some;
22fc3527
RH		 1306 /* fallthru */
1307
1308 case TCG_TYPE_V128:
1309 expand_2s_vec(g->vece, dofs, aofs, oprsz, 16, TCG_TYPE_V128,
1310 t_vec, g->scalar_first, g->fniv);
1311 break;
1312
1313 case TCG_TYPE_V64:
1314 expand_2s_vec(g->vece, dofs, aofs, oprsz, 8, TCG_TYPE_V64,
1315 t_vec, g->scalar_first, g->fniv);
1316 break;
1317
1318 default:
1319 g_assert_not_reached();
1320 }
1321 tcg_temp_free_vec(t_vec);
53229a77 1322 tcg_swap_vecop_list(hold_list);
22fc3527
RH		 1323 } else if (g->fni8 && check_size_impl(oprsz, 8)) {
1324 TCGv_i64 t64 = tcg_temp_new_i64();
1325
1326 gen_dup_i64(g->vece, t64, c);
1327 expand_2s_i64(dofs, aofs, oprsz, t64, g->scalar_first, g->fni8);
1328 tcg_temp_free_i64(t64);
1329 } else if (g->fni4 && check_size_impl(oprsz, 4)) {
1330 TCGv_i32 t32 = tcg_temp_new_i32();
1331
1332 tcg_gen_extrl_i64_i32(t32, c);
1333 gen_dup_i32(g->vece, t32, t32);
1334 expand_2s_i32(dofs, aofs, oprsz, t32, g->scalar_first, g->fni4);
1335 tcg_temp_free_i32(t32);
1336 } else {
1337 tcg_gen_gvec_2i_ool(dofs, aofs, c, oprsz, maxsz, 0, g->fno);
1338 return;
1339 }
1340
1341 if (oprsz < maxsz) {
1342 expand_clr(dofs + oprsz, maxsz - oprsz);
1343 }
1344}
1345
db432672
RH		 1346/* Expand a vector three-operand operation. */
1347void tcg_gen_gvec_3(uint32_t dofs, uint32_t aofs, uint32_t bofs,
1348 uint32_t oprsz, uint32_t maxsz, const GVecGen3 *g)
1349{
53229a77
RH		 1350 const TCGOpcode *this_list = g->opt_opc ? : vecop_list_empty;
1351 const TCGOpcode *hold_list = tcg_swap_vecop_list(this_list);
adb196cb
RH		 1352 TCGType type;
1353 uint32_t some;
1354
db432672
RH		 1355 check_size_align(oprsz, maxsz, dofs | aofs | bofs);
1356 check_overlap_3(dofs, aofs, bofs, maxsz);
1357
adb196cb
RH		 1358 type = 0;
1359 if (g->fniv) {
53229a77 1360 type = choose_vector_type(g->opt_opc, g->vece, oprsz, g->prefer_i64);
adb196cb
RH		 1361 }
1362 switch (type) {
1363 case TCG_TYPE_V256:
1364 /* Recall that ARM SVE allows vector sizes that are not a
1365 * power of 2, but always a multiple of 16. The intent is
1366 * that e.g. size == 80 would be expanded with 2x32 + 1x16.
1367 */
1368 some = QEMU_ALIGN_DOWN(oprsz, 32);
db432672
RH		 1369 expand_3_vec(g->vece, dofs, aofs, bofs, some, 32, TCG_TYPE_V256,
1370 g->load_dest, g->fniv);
1371 if (some == oprsz) {
adb196cb 1372 break;
db432672
RH		 1373 }
1374 dofs += some;
1375 aofs += some;
1376 bofs += some;
1377 oprsz -= some;
1378 maxsz -= some;
adb196cb
RH		 1379 /* fallthru */
1380 case TCG_TYPE_V128:
db432672
RH		 1381 expand_3_vec(g->vece, dofs, aofs, bofs, oprsz, 16, TCG_TYPE_V128,
1382 g->load_dest, g->fniv);
adb196cb
RH		 1383 break;
1384 case TCG_TYPE_V64:
db432672
RH		 1385 expand_3_vec(g->vece, dofs, aofs, bofs, oprsz, 8, TCG_TYPE_V64,
1386 g->load_dest, g->fniv);
adb196cb
RH		 1387 break;
1388
1389 case 0:
1390 if (g->fni8 && check_size_impl(oprsz, 8)) {
1391 expand_3_i64(dofs, aofs, bofs, oprsz, g->load_dest, g->fni8);
1392 } else if (g->fni4 && check_size_impl(oprsz, 4)) {
1393 expand_3_i32(dofs, aofs, bofs, oprsz, g->load_dest, g->fni4);
1394 } else {
1395 assert(g->fno != NULL);
1396 tcg_gen_gvec_3_ool(dofs, aofs, bofs, oprsz,
1397 maxsz, g->data, g->fno);
53229a77 1398 oprsz = maxsz;
adb196cb
RH		 1399 }
1400 break;
1401
1402 default:
1403 g_assert_not_reached();
db432672 1404 }
53229a77 1405 tcg_swap_vecop_list(hold_list);
db432672 1406
db432672
RH		 1407 if (oprsz < maxsz) {
1408 expand_clr(dofs + oprsz, maxsz - oprsz);
1409 }
1410}
1411
e1227bb6
DH		 1412/* Expand a vector operation with three vectors and an immediate. */
1413void tcg_gen_gvec_3i(uint32_t dofs, uint32_t aofs, uint32_t bofs,
1414 uint32_t oprsz, uint32_t maxsz, int64_t c,
1415 const GVecGen3i *g)
1416{
53229a77
RH		 1417 const TCGOpcode *this_list = g->opt_opc ? : vecop_list_empty;
1418 const TCGOpcode *hold_list = tcg_swap_vecop_list(this_list);
e1227bb6
DH		 1419 TCGType type;
1420 uint32_t some;
1421
1422 check_size_align(oprsz, maxsz, dofs | aofs | bofs);
1423 check_overlap_3(dofs, aofs, bofs, maxsz);
1424
1425 type = 0;
1426 if (g->fniv) {
53229a77 1427 type = choose_vector_type(g->opt_opc, g->vece, oprsz, g->prefer_i64);
e1227bb6
DH		 1428 }
1429 switch (type) {
1430 case TCG_TYPE_V256:
1431 /*
1432 * Recall that ARM SVE allows vector sizes that are not a
1433 * power of 2, but always a multiple of 16. The intent is
1434 * that e.g. size == 80 would be expanded with 2x32 + 1x16.
1435 */
1436 some = QEMU_ALIGN_DOWN(oprsz, 32);
1437 expand_3i_vec(g->vece, dofs, aofs, bofs, some, 32, TCG_TYPE_V256,
1438 c, g->load_dest, g->fniv);
1439 if (some == oprsz) {
1440 break;
1441 }
1442 dofs += some;
1443 aofs += some;
1444 bofs += some;
1445 oprsz -= some;
1446 maxsz -= some;
1447 /* fallthru */
1448 case TCG_TYPE_V128:
1449 expand_3i_vec(g->vece, dofs, aofs, bofs, oprsz, 16, TCG_TYPE_V128,
1450 c, g->load_dest, g->fniv);
1451 break;
1452 case TCG_TYPE_V64:
1453 expand_3i_vec(g->vece, dofs, aofs, bofs, oprsz, 8, TCG_TYPE_V64,
1454 c, g->load_dest, g->fniv);
1455 break;
1456
1457 case 0:
1458 if (g->fni8 && check_size_impl(oprsz, 8)) {
1459 expand_3i_i64(dofs, aofs, bofs, oprsz, c, g->load_dest, g->fni8);
1460 } else if (g->fni4 && check_size_impl(oprsz, 4)) {
1461 expand_3i_i32(dofs, aofs, bofs, oprsz, c, g->load_dest, g->fni4);
1462 } else {
1463 assert(g->fno != NULL);
1464 tcg_gen_gvec_3_ool(dofs, aofs, bofs, oprsz, maxsz, c, g->fno);
53229a77 1465 oprsz = maxsz;
e1227bb6
DH		 1466 }
1467 break;
1468
1469 default:
1470 g_assert_not_reached();
1471 }
53229a77 1472 tcg_swap_vecop_list(hold_list);
e1227bb6
DH		 1473
1474 if (oprsz < maxsz) {
1475 expand_clr(dofs + oprsz, maxsz - oprsz);
1476 }
1477}
1478
db432672
RH		 1479/* Expand a vector four-operand operation. */
1480void tcg_gen_gvec_4(uint32_t dofs, uint32_t aofs, uint32_t bofs, uint32_t cofs,
1481 uint32_t oprsz, uint32_t maxsz, const GVecGen4 *g)
1482{
53229a77
RH		 1483 const TCGOpcode *this_list = g->opt_opc ? : vecop_list_empty;
1484 const TCGOpcode *hold_list = tcg_swap_vecop_list(this_list);
adb196cb
RH		 1485 TCGType type;
1486 uint32_t some;
1487
db432672
RH		 1488 check_size_align(oprsz, maxsz, dofs | aofs | bofs | cofs);
1489 check_overlap_4(dofs, aofs, bofs, cofs, maxsz);
1490
adb196cb
RH		 1491 type = 0;
1492 if (g->fniv) {
53229a77 1493 type = choose_vector_type(g->opt_opc, g->vece, oprsz, g->prefer_i64);
adb196cb
RH		 1494 }
1495 switch (type) {
1496 case TCG_TYPE_V256:
1497 /* Recall that ARM SVE allows vector sizes that are not a
1498 * power of 2, but always a multiple of 16. The intent is
1499 * that e.g. size == 80 would be expanded with 2x32 + 1x16.
1500 */
1501 some = QEMU_ALIGN_DOWN(oprsz, 32);
db432672 1502 expand_4_vec(g->vece, dofs, aofs, bofs, cofs, some,
5d6acdd4 1503 32, TCG_TYPE_V256, g->write_aofs, g->fniv);
db432672 1504 if (some == oprsz) {
adb196cb 1505 break;
db432672
RH		 1506 }
1507 dofs += some;
1508 aofs += some;
1509 bofs += some;
1510 cofs += some;
1511 oprsz -= some;
1512 maxsz -= some;
adb196cb
RH		 1513 /* fallthru */
1514 case TCG_TYPE_V128:
db432672 1515 expand_4_vec(g->vece, dofs, aofs, bofs, cofs, oprsz,
5d6acdd4 1516 16, TCG_TYPE_V128, g->write_aofs, g->fniv);
adb196cb
RH		 1517 break;
1518 case TCG_TYPE_V64:
db432672 1519 expand_4_vec(g->vece, dofs, aofs, bofs, cofs, oprsz,
5d6acdd4 1520 8, TCG_TYPE_V64, g->write_aofs, g->fniv);
adb196cb
RH		 1521 break;
1522
1523 case 0:
1524 if (g->fni8 && check_size_impl(oprsz, 8)) {
5d6acdd4
RH		 1525 expand_4_i64(dofs, aofs, bofs, cofs, oprsz,
1526 g->write_aofs, g->fni8);
adb196cb 1527 } else if (g->fni4 && check_size_impl(oprsz, 4)) {
5d6acdd4
RH		 1528 expand_4_i32(dofs, aofs, bofs, cofs, oprsz,
1529 g->write_aofs, g->fni4);
adb196cb
RH		 1530 } else {
1531 assert(g->fno != NULL);
1532 tcg_gen_gvec_4_ool(dofs, aofs, bofs, cofs,
1533 oprsz, maxsz, g->data, g->fno);
53229a77 1534 oprsz = maxsz;
adb196cb
RH		 1535 }
1536 break;
1537
1538 default:
1539 g_assert_not_reached();
db432672 1540 }
53229a77 1541 tcg_swap_vecop_list(hold_list);
db432672 1542
db432672
RH		 1543 if (oprsz < maxsz) {
1544 expand_clr(dofs + oprsz, maxsz - oprsz);
1545 }
1546}
1547
1548/*
1549 * Expand specific vector operations.
1550 */
1551
1552static void vec_mov2(unsigned vece, TCGv_vec a, TCGv_vec b)
1553{
1554 tcg_gen_mov_vec(a, b);
1555}
1556
1557void tcg_gen_gvec_mov(unsigned vece, uint32_t dofs, uint32_t aofs,
1558 uint32_t oprsz, uint32_t maxsz)
1559{
1560 static const GVecGen2 g = {
1561 .fni8 = tcg_gen_mov_i64,
1562 .fniv = vec_mov2,
1563 .fno = gen_helper_gvec_mov,
1564 .prefer_i64 = TCG_TARGET_REG_BITS == 64,
1565 };
1566 if (dofs != aofs) {
1567 tcg_gen_gvec_2(dofs, aofs, oprsz, maxsz, &g);
1568 } else {
1569 check_size_align(oprsz, maxsz, dofs);
1570 if (oprsz < maxsz) {
1571 expand_clr(dofs + oprsz, maxsz - oprsz);
1572 }
1573 }
1574}
1575
1576void tcg_gen_gvec_dup_i32(unsigned vece, uint32_t dofs, uint32_t oprsz,
1577 uint32_t maxsz, TCGv_i32 in)
1578{
1579 check_size_align(oprsz, maxsz, dofs);
1580 tcg_debug_assert(vece <= MO_32);
1581 do_dup(vece, dofs, oprsz, maxsz, in, NULL, 0);
1582}
1583
1584void tcg_gen_gvec_dup_i64(unsigned vece, uint32_t dofs, uint32_t oprsz,
1585 uint32_t maxsz, TCGv_i64 in)
1586{
1587 check_size_align(oprsz, maxsz, dofs);
1588 tcg_debug_assert(vece <= MO_64);
1589 do_dup(vece, dofs, oprsz, maxsz, NULL, in, 0);
1590}
1591
1592void tcg_gen_gvec_dup_mem(unsigned vece, uint32_t dofs, uint32_t aofs,
1593 uint32_t oprsz, uint32_t maxsz)
1594{
532ba368 1595 check_size_align(oprsz, maxsz, dofs);
37ee55a0 1596 if (vece <= MO_64) {
532ba368 1597 TCGType type = choose_vector_type(NULL, vece, oprsz, 0);
37ee55a0
RH		 1598 if (type != 0) {
1599 TCGv_vec t_vec = tcg_temp_new_vec(type);
1600 tcg_gen_dup_mem_vec(vece, t_vec, cpu_env, aofs);
1601 do_dup_store(type, dofs, oprsz, maxsz, t_vec);
1602 tcg_temp_free_vec(t_vec);
532ba368
RH		 1603 } else if (vece <= MO_32) {
1604 TCGv_i32 in = tcg_temp_new_i32();
1605 switch (vece) {
1606 case MO_8:
1607 tcg_gen_ld8u_i32(in, cpu_env, aofs);
1608 break;
1609 case MO_16:
1610 tcg_gen_ld16u_i32(in, cpu_env, aofs);
1611 break;
1612 default:
1613 tcg_gen_ld_i32(in, cpu_env, aofs);
1614 break;
1615 }
1616 do_dup(vece, dofs, oprsz, maxsz, in, NULL, 0);
1617 tcg_temp_free_i32(in);
1618 } else {
1619 TCGv_i64 in = tcg_temp_new_i64();
1620 tcg_gen_ld_i64(in, cpu_env, aofs);
1621 do_dup(vece, dofs, oprsz, maxsz, NULL, in, 0);
1622 tcg_temp_free_i64(in);
db432672 1623 }
fe4b0b5b 1624 } else if (vece == 4) {
db432672 1625 /* 128-bit duplicate. */
db432672
RH		 1626 int i;
1627
db432672
RH		 1628 tcg_debug_assert(oprsz >= 16);
1629 if (TCG_TARGET_HAS_v128) {
1630 TCGv_vec in = tcg_temp_new_vec(TCG_TYPE_V128);
1631
1632 tcg_gen_ld_vec(in, cpu_env, aofs);
6a176461 1633 for (i = (aofs == dofs) * 16; i < oprsz; i += 16) {
db432672
RH		 1634 tcg_gen_st_vec(in, cpu_env, dofs + i);
1635 }
1636 tcg_temp_free_vec(in);
1637 } else {
1638 TCGv_i64 in0 = tcg_temp_new_i64();
1639 TCGv_i64 in1 = tcg_temp_new_i64();
1640
1641 tcg_gen_ld_i64(in0, cpu_env, aofs);
1642 tcg_gen_ld_i64(in1, cpu_env, aofs + 8);
6a176461 1643 for (i = (aofs == dofs) * 16; i < oprsz; i += 16) {
db432672
RH		 1644 tcg_gen_st_i64(in0, cpu_env, dofs + i);
1645 tcg_gen_st_i64(in1, cpu_env, dofs + i + 8);
1646 }
1647 tcg_temp_free_i64(in0);
1648 tcg_temp_free_i64(in1);
1649 }
532ba368
RH		 1650 if (oprsz < maxsz) {
1651 expand_clr(dofs + oprsz, maxsz - oprsz);
1652 }
fe4b0b5b
RH		 1653 } else if (vece == 5) {
1654 /* 256-bit duplicate. */
1655 int i;
1656
1657 tcg_debug_assert(oprsz >= 32);
1658 tcg_debug_assert(oprsz % 32 == 0);
1659 if (TCG_TARGET_HAS_v256) {
1660 TCGv_vec in = tcg_temp_new_vec(TCG_TYPE_V256);
1661
1662 tcg_gen_ld_vec(in, cpu_env, aofs);
1663 for (i = (aofs == dofs) * 32; i < oprsz; i += 32) {
1664 tcg_gen_st_vec(in, cpu_env, dofs + i);
1665 }
1666 tcg_temp_free_vec(in);
1667 } else if (TCG_TARGET_HAS_v128) {
1668 TCGv_vec in0 = tcg_temp_new_vec(TCG_TYPE_V128);
1669 TCGv_vec in1 = tcg_temp_new_vec(TCG_TYPE_V128);
1670
1671 tcg_gen_ld_vec(in0, cpu_env, aofs);
1672 tcg_gen_ld_vec(in1, cpu_env, aofs + 16);
1673 for (i = (aofs == dofs) * 32; i < oprsz; i += 32) {
1674 tcg_gen_st_vec(in0, cpu_env, dofs + i);
1675 tcg_gen_st_vec(in1, cpu_env, dofs + i + 16);
1676 }
1677 tcg_temp_free_vec(in0);
1678 tcg_temp_free_vec(in1);
1679 } else {
1680 TCGv_i64 in[4];
1681 int j;
1682
1683 for (j = 0; j < 4; ++j) {
1684 in[j] = tcg_temp_new_i64();
1685 tcg_gen_ld_i64(in[j], cpu_env, aofs + j * 8);
1686 }
1687 for (i = (aofs == dofs) * 32; i < oprsz; i += 32) {
1688 for (j = 0; j < 4; ++j) {
1689 tcg_gen_st_i64(in[j], cpu_env, dofs + i + j * 8);
1690 }
1691 }
1692 for (j = 0; j < 4; ++j) {
1693 tcg_temp_free_i64(in[j]);
1694 }
1695 }
1696 if (oprsz < maxsz) {
1697 expand_clr(dofs + oprsz, maxsz - oprsz);
1698 }
1699 } else {
1700 g_assert_not_reached();
db432672
RH		 1701 }
1702}
1703
44c94677
RH		 1704void tcg_gen_gvec_dup_imm(unsigned vece, uint32_t dofs, uint32_t oprsz,
1705 uint32_t maxsz, uint64_t x)
1706{
1707 check_size_align(oprsz, maxsz, dofs);
1708 do_dup(vece, dofs, oprsz, maxsz, NULL, NULL, x);
1709}
1710
db432672
RH		 1711void tcg_gen_gvec_not(unsigned vece, uint32_t dofs, uint32_t aofs,
1712 uint32_t oprsz, uint32_t maxsz)
1713{
1714 static const GVecGen2 g = {
1715 .fni8 = tcg_gen_not_i64,
1716 .fniv = tcg_gen_not_vec,
1717 .fno = gen_helper_gvec_not,
1718 .prefer_i64 = TCG_TARGET_REG_BITS == 64,
1719 };
1720 tcg_gen_gvec_2(dofs, aofs, oprsz, maxsz, &g);
1721}
1722
1723/* Perform a vector addition using normal addition and a mask. The mask
1724 should be the sign bit of each lane. This 6-operation form is more
1725 efficient than separate additions when there are 4 or more lanes in
1726 the 64-bit operation. */
1727static void gen_addv_mask(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b, TCGv_i64 m)
1728{
1729 TCGv_i64 t1 = tcg_temp_new_i64();
1730 TCGv_i64 t2 = tcg_temp_new_i64();
1731 TCGv_i64 t3 = tcg_temp_new_i64();
1732
1733 tcg_gen_andc_i64(t1, a, m);
1734 tcg_gen_andc_i64(t2, b, m);
1735 tcg_gen_xor_i64(t3, a, b);
1736 tcg_gen_add_i64(d, t1, t2);
1737 tcg_gen_and_i64(t3, t3, m);
1738 tcg_gen_xor_i64(d, d, t3);
1739
1740 tcg_temp_free_i64(t1);
1741 tcg_temp_free_i64(t2);
1742 tcg_temp_free_i64(t3);
1743}
1744
1745void tcg_gen_vec_add8_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b)
1746{
1747 TCGv_i64 m = tcg_const_i64(dup_const(MO_8, 0x80));
1748 gen_addv_mask(d, a, b, m);
1749 tcg_temp_free_i64(m);
1750}
1751
1752void tcg_gen_vec_add16_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b)
1753{
1754 TCGv_i64 m = tcg_const_i64(dup_const(MO_16, 0x8000));
1755 gen_addv_mask(d, a, b, m);
1756 tcg_temp_free_i64(m);
1757}
1758
1759void tcg_gen_vec_add32_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b)
1760{
1761 TCGv_i64 t1 = tcg_temp_new_i64();
1762 TCGv_i64 t2 = tcg_temp_new_i64();
1763
1764 tcg_gen_andi_i64(t1, a, ~0xffffffffull);
1765 tcg_gen_add_i64(t2, a, b);
1766 tcg_gen_add_i64(t1, t1, b);
1767 tcg_gen_deposit_i64(d, t1, t2, 0, 32);
1768
1769 tcg_temp_free_i64(t1);
1770 tcg_temp_free_i64(t2);
1771}
1772
53229a77
RH		 1773static const TCGOpcode vecop_list_add[] = { INDEX_op_add_vec, 0 };
1774
db432672
RH		 1775void tcg_gen_gvec_add(unsigned vece, uint32_t dofs, uint32_t aofs,
1776 uint32_t bofs, uint32_t oprsz, uint32_t maxsz)
1777{
1778 static const GVecGen3 g[4] = {
1779 { .fni8 = tcg_gen_vec_add8_i64,
1780 .fniv = tcg_gen_add_vec,
1781 .fno = gen_helper_gvec_add8,
53229a77 1782 .opt_opc = vecop_list_add,
db432672
RH		 1783 .vece = MO_8 },
1784 { .fni8 = tcg_gen_vec_add16_i64,
1785 .fniv = tcg_gen_add_vec,
1786 .fno = gen_helper_gvec_add16,
53229a77 1787 .opt_opc = vecop_list_add,
db432672
RH		 1788 .vece = MO_16 },
1789 { .fni4 = tcg_gen_add_i32,
1790 .fniv = tcg_gen_add_vec,
1791 .fno = gen_helper_gvec_add32,
53229a77 1792 .opt_opc = vecop_list_add,
db432672
RH		 1793 .vece = MO_32 },
1794 { .fni8 = tcg_gen_add_i64,
1795 .fniv = tcg_gen_add_vec,
1796 .fno = gen_helper_gvec_add64,
53229a77 1797 .opt_opc = vecop_list_add,
db432672
RH		 1798 .prefer_i64 = TCG_TARGET_REG_BITS == 64,
1799 .vece = MO_64 },
1800 };
1801
1802 tcg_debug_assert(vece <= MO_64);
1803 tcg_gen_gvec_3(dofs, aofs, bofs, oprsz, maxsz, &g[vece]);
1804}
1805
22fc3527
RH		 1806void tcg_gen_gvec_adds(unsigned vece, uint32_t dofs, uint32_t aofs,
1807 TCGv_i64 c, uint32_t oprsz, uint32_t maxsz)
1808{
1809 static const GVecGen2s g[4] = {
1810 { .fni8 = tcg_gen_vec_add8_i64,
1811 .fniv = tcg_gen_add_vec,
1812 .fno = gen_helper_gvec_adds8,
53229a77 1813 .opt_opc = vecop_list_add,
22fc3527
RH		 1814 .vece = MO_8 },
1815 { .fni8 = tcg_gen_vec_add16_i64,
1816 .fniv = tcg_gen_add_vec,
1817 .fno = gen_helper_gvec_adds16,
53229a77 1818 .opt_opc = vecop_list_add,
22fc3527
RH		 1819 .vece = MO_16 },
1820 { .fni4 = tcg_gen_add_i32,
1821 .fniv = tcg_gen_add_vec,
1822 .fno = gen_helper_gvec_adds32,
53229a77 1823 .opt_opc = vecop_list_add,
22fc3527
RH		 1824 .vece = MO_32 },
1825 { .fni8 = tcg_gen_add_i64,
1826 .fniv = tcg_gen_add_vec,
1827 .fno = gen_helper_gvec_adds64,
53229a77 1828 .opt_opc = vecop_list_add,
22fc3527
RH		 1829 .prefer_i64 = TCG_TARGET_REG_BITS == 64,
1830 .vece = MO_64 },
1831 };
1832
1833 tcg_debug_assert(vece <= MO_64);
1834 tcg_gen_gvec_2s(dofs, aofs, oprsz, maxsz, c, &g[vece]);
1835}
1836
1837void tcg_gen_gvec_addi(unsigned vece, uint32_t dofs, uint32_t aofs,
1838 int64_t c, uint32_t oprsz, uint32_t maxsz)
1839{
1840 TCGv_i64 tmp = tcg_const_i64(c);
1841 tcg_gen_gvec_adds(vece, dofs, aofs, tmp, oprsz, maxsz);
1842 tcg_temp_free_i64(tmp);
1843}
1844
53229a77
RH		 1845static const TCGOpcode vecop_list_sub[] = { INDEX_op_sub_vec, 0 };
1846
22fc3527
RH		 1847void tcg_gen_gvec_subs(unsigned vece, uint32_t dofs, uint32_t aofs,
1848 TCGv_i64 c, uint32_t oprsz, uint32_t maxsz)
1849{
1850 static const GVecGen2s g[4] = {
1851 { .fni8 = tcg_gen_vec_sub8_i64,
1852 .fniv = tcg_gen_sub_vec,
1853 .fno = gen_helper_gvec_subs8,
53229a77 1854 .opt_opc = vecop_list_sub,
22fc3527
RH		 1855 .vece = MO_8 },
1856 { .fni8 = tcg_gen_vec_sub16_i64,
1857 .fniv = tcg_gen_sub_vec,
1858 .fno = gen_helper_gvec_subs16,
53229a77 1859 .opt_opc = vecop_list_sub,
22fc3527
RH		 1860 .vece = MO_16 },
1861 { .fni4 = tcg_gen_sub_i32,
1862 .fniv = tcg_gen_sub_vec,
1863 .fno = gen_helper_gvec_subs32,
53229a77 1864 .opt_opc = vecop_list_sub,
22fc3527
RH		 1865 .vece = MO_32 },
1866 { .fni8 = tcg_gen_sub_i64,
1867 .fniv = tcg_gen_sub_vec,
1868 .fno = gen_helper_gvec_subs64,
53229a77 1869 .opt_opc = vecop_list_sub,
22fc3527
RH		 1870 .prefer_i64 = TCG_TARGET_REG_BITS == 64,
1871 .vece = MO_64 },
1872 };
1873
1874 tcg_debug_assert(vece <= MO_64);
1875 tcg_gen_gvec_2s(dofs, aofs, oprsz, maxsz, c, &g[vece]);
1876}
1877
db432672
RH		 1878/* Perform a vector subtraction using normal subtraction and a mask.
1879 Compare gen_addv_mask above. */
1880static void gen_subv_mask(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b, TCGv_i64 m)
1881{
1882 TCGv_i64 t1 = tcg_temp_new_i64();
1883 TCGv_i64 t2 = tcg_temp_new_i64();
1884 TCGv_i64 t3 = tcg_temp_new_i64();
1885
1886 tcg_gen_or_i64(t1, a, m);
1887 tcg_gen_andc_i64(t2, b, m);
1888 tcg_gen_eqv_i64(t3, a, b);
1889 tcg_gen_sub_i64(d, t1, t2);
1890 tcg_gen_and_i64(t3, t3, m);
1891 tcg_gen_xor_i64(d, d, t3);
1892
1893 tcg_temp_free_i64(t1);
1894 tcg_temp_free_i64(t2);
1895 tcg_temp_free_i64(t3);
1896}
1897
1898void tcg_gen_vec_sub8_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b)
1899{
1900 TCGv_i64 m = tcg_const_i64(dup_const(MO_8, 0x80));
1901 gen_subv_mask(d, a, b, m);
1902 tcg_temp_free_i64(m);
1903}
1904
1905void tcg_gen_vec_sub16_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b)
1906{
1907 TCGv_i64 m = tcg_const_i64(dup_const(MO_16, 0x8000));
1908 gen_subv_mask(d, a, b, m);
1909 tcg_temp_free_i64(m);
1910}
1911
1912void tcg_gen_vec_sub32_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b)
1913{
1914 TCGv_i64 t1 = tcg_temp_new_i64();
1915 TCGv_i64 t2 = tcg_temp_new_i64();
1916
1917 tcg_gen_andi_i64(t1, b, ~0xffffffffull);
1918 tcg_gen_sub_i64(t2, a, b);
1919 tcg_gen_sub_i64(t1, a, t1);
1920 tcg_gen_deposit_i64(d, t1, t2, 0, 32);
1921
1922 tcg_temp_free_i64(t1);
1923 tcg_temp_free_i64(t2);
1924}
1925
1926void tcg_gen_gvec_sub(unsigned vece, uint32_t dofs, uint32_t aofs,
1927 uint32_t bofs, uint32_t oprsz, uint32_t maxsz)
1928{
1929 static const GVecGen3 g[4] = {
1930 { .fni8 = tcg_gen_vec_sub8_i64,
1931 .fniv = tcg_gen_sub_vec,
1932 .fno = gen_helper_gvec_sub8,
53229a77 1933 .opt_opc = vecop_list_sub,
db432672
RH		 1934 .vece = MO_8 },
1935 { .fni8 = tcg_gen_vec_sub16_i64,
1936 .fniv = tcg_gen_sub_vec,
1937 .fno = gen_helper_gvec_sub16,
53229a77 1938 .opt_opc = vecop_list_sub,
db432672
RH		 1939 .vece = MO_16 },
1940 { .fni4 = tcg_gen_sub_i32,
1941 .fniv = tcg_gen_sub_vec,
1942 .fno = gen_helper_gvec_sub32,
53229a77 1943 .opt_opc = vecop_list_sub,
db432672
RH		 1944 .vece = MO_32 },
1945 { .fni8 = tcg_gen_sub_i64,
1946 .fniv = tcg_gen_sub_vec,
1947 .fno = gen_helper_gvec_sub64,
53229a77 1948 .opt_opc = vecop_list_sub,
db432672
RH		 1949 .prefer_i64 = TCG_TARGET_REG_BITS == 64,
1950 .vece = MO_64 },
1951 };
1952
1953 tcg_debug_assert(vece <= MO_64);
1954 tcg_gen_gvec_3(dofs, aofs, bofs, oprsz, maxsz, &g[vece]);
1955}
1956
53229a77
RH		 1957static const TCGOpcode vecop_list_mul[] = { INDEX_op_mul_vec, 0 };
1958
3774030a
RH		 1959void tcg_gen_gvec_mul(unsigned vece, uint32_t dofs, uint32_t aofs,
1960 uint32_t bofs, uint32_t oprsz, uint32_t maxsz)
1961{
1962 static const GVecGen3 g[4] = {
1963 { .fniv = tcg_gen_mul_vec,
1964 .fno = gen_helper_gvec_mul8,
53229a77 1965 .opt_opc = vecop_list_mul,
3774030a
RH		 1966 .vece = MO_8 },
1967 { .fniv = tcg_gen_mul_vec,
1968 .fno = gen_helper_gvec_mul16,
53229a77 1969 .opt_opc = vecop_list_mul,
3774030a
RH		 1970 .vece = MO_16 },
1971 { .fni4 = tcg_gen_mul_i32,
1972 .fniv = tcg_gen_mul_vec,
1973 .fno = gen_helper_gvec_mul32,
53229a77 1974 .opt_opc = vecop_list_mul,
3774030a
RH		 1975 .vece = MO_32 },
1976 { .fni8 = tcg_gen_mul_i64,
1977 .fniv = tcg_gen_mul_vec,
1978 .fno = gen_helper_gvec_mul64,
53229a77 1979 .opt_opc = vecop_list_mul,
3774030a
RH		 1980 .prefer_i64 = TCG_TARGET_REG_BITS == 64,
1981 .vece = MO_64 },
1982 };
1983
1984 tcg_debug_assert(vece <= MO_64);
1985 tcg_gen_gvec_3(dofs, aofs, bofs, oprsz, maxsz, &g[vece]);
1986}
1987
22fc3527
RH		 1988void tcg_gen_gvec_muls(unsigned vece, uint32_t dofs, uint32_t aofs,
1989 TCGv_i64 c, uint32_t oprsz, uint32_t maxsz)
1990{
1991 static const GVecGen2s g[4] = {
1992 { .fniv = tcg_gen_mul_vec,
1993 .fno = gen_helper_gvec_muls8,
53229a77 1994 .opt_opc = vecop_list_mul,
22fc3527
RH		 1995 .vece = MO_8 },
1996 { .fniv = tcg_gen_mul_vec,
1997 .fno = gen_helper_gvec_muls16,
53229a77 1998 .opt_opc = vecop_list_mul,
22fc3527
RH		 1999 .vece = MO_16 },
2000 { .fni4 = tcg_gen_mul_i32,
2001 .fniv = tcg_gen_mul_vec,
2002 .fno = gen_helper_gvec_muls32,
53229a77 2003 .opt_opc = vecop_list_mul,
22fc3527
RH		 2004 .vece = MO_32 },
2005 { .fni8 = tcg_gen_mul_i64,
2006 .fniv = tcg_gen_mul_vec,
2007 .fno = gen_helper_gvec_muls64,
53229a77 2008 .opt_opc = vecop_list_mul,
22fc3527
RH		 2009 .prefer_i64 = TCG_TARGET_REG_BITS == 64,
2010 .vece = MO_64 },
2011 };
2012
2013 tcg_debug_assert(vece <= MO_64);
2014 tcg_gen_gvec_2s(dofs, aofs, oprsz, maxsz, c, &g[vece]);
2015}
2016
2017void tcg_gen_gvec_muli(unsigned vece, uint32_t dofs, uint32_t aofs,
2018 int64_t c, uint32_t oprsz, uint32_t maxsz)
2019{
2020 TCGv_i64 tmp = tcg_const_i64(c);
2021 tcg_gen_gvec_muls(vece, dofs, aofs, tmp, oprsz, maxsz);
2022 tcg_temp_free_i64(tmp);
2023}
2024
f49b12c6
RH		 2025void tcg_gen_gvec_ssadd(unsigned vece, uint32_t dofs, uint32_t aofs,
2026 uint32_t bofs, uint32_t oprsz, uint32_t maxsz)
2027{
53229a77 2028 static const TCGOpcode vecop_list[] = { INDEX_op_ssadd_vec, 0 };
f49b12c6 2029 static const GVecGen3 g[4] = {
8afaf050
RH		 2030 { .fniv = tcg_gen_ssadd_vec,
2031 .fno = gen_helper_gvec_ssadd8,
53229a77 2032 .opt_opc = vecop_list,
8afaf050
RH		 2033 .vece = MO_8 },
2034 { .fniv = tcg_gen_ssadd_vec,
2035 .fno = gen_helper_gvec_ssadd16,
53229a77 2036 .opt_opc = vecop_list,
8afaf050
RH		 2037 .vece = MO_16 },
2038 { .fniv = tcg_gen_ssadd_vec,
2039 .fno = gen_helper_gvec_ssadd32,
53229a77 2040 .opt_opc = vecop_list,
8afaf050
RH		 2041 .vece = MO_32 },
2042 { .fniv = tcg_gen_ssadd_vec,
2043 .fno = gen_helper_gvec_ssadd64,
53229a77 2044 .opt_opc = vecop_list,
8afaf050 2045 .vece = MO_64 },
f49b12c6
RH		 2046 };
2047 tcg_debug_assert(vece <= MO_64);
2048 tcg_gen_gvec_3(dofs, aofs, bofs, oprsz, maxsz, &g[vece]);
2049}
2050
2051void tcg_gen_gvec_sssub(unsigned vece, uint32_t dofs, uint32_t aofs,
2052 uint32_t bofs, uint32_t oprsz, uint32_t maxsz)
2053{
53229a77 2054 static const TCGOpcode vecop_list[] = { INDEX_op_sssub_vec, 0 };
f49b12c6 2055 static const GVecGen3 g[4] = {
8afaf050
RH		 2056 { .fniv = tcg_gen_sssub_vec,
2057 .fno = gen_helper_gvec_sssub8,
53229a77 2058 .opt_opc = vecop_list,
8afaf050
RH		 2059 .vece = MO_8 },
2060 { .fniv = tcg_gen_sssub_vec,
2061 .fno = gen_helper_gvec_sssub16,
53229a77 2062 .opt_opc = vecop_list,
8afaf050
RH		 2063 .vece = MO_16 },
2064 { .fniv = tcg_gen_sssub_vec,
2065 .fno = gen_helper_gvec_sssub32,
53229a77 2066 .opt_opc = vecop_list,
8afaf050
RH		 2067 .vece = MO_32 },
2068 { .fniv = tcg_gen_sssub_vec,
2069 .fno = gen_helper_gvec_sssub64,
53229a77 2070 .opt_opc = vecop_list,
8afaf050 2071 .vece = MO_64 },
f49b12c6
RH		 2072 };
2073 tcg_debug_assert(vece <= MO_64);
2074 tcg_gen_gvec_3(dofs, aofs, bofs, oprsz, maxsz, &g[vece]);
2075}
2076
8afaf050 2077static void tcg_gen_usadd_i32(TCGv_i32 d, TCGv_i32 a, TCGv_i32 b)
f49b12c6
RH		 2078{
2079 TCGv_i32 max = tcg_const_i32(-1);
2080 tcg_gen_add_i32(d, a, b);
2081 tcg_gen_movcond_i32(TCG_COND_LTU, d, d, a, max, d);
2082 tcg_temp_free_i32(max);
2083}
2084
8afaf050 2085static void tcg_gen_usadd_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b)
f49b12c6
RH		 2086{
2087 TCGv_i64 max = tcg_const_i64(-1);
2088 tcg_gen_add_i64(d, a, b);
2089 tcg_gen_movcond_i64(TCG_COND_LTU, d, d, a, max, d);
2090 tcg_temp_free_i64(max);
2091}
2092
2093void tcg_gen_gvec_usadd(unsigned vece, uint32_t dofs, uint32_t aofs,
2094 uint32_t bofs, uint32_t oprsz, uint32_t maxsz)
2095{
53229a77 2096 static const TCGOpcode vecop_list[] = { INDEX_op_usadd_vec, 0 };
f49b12c6 2097 static const GVecGen3 g[4] = {
8afaf050
RH		 2098 { .fniv = tcg_gen_usadd_vec,
2099 .fno = gen_helper_gvec_usadd8,
53229a77 2100 .opt_opc = vecop_list,
8afaf050
RH		 2101 .vece = MO_8 },
2102 { .fniv = tcg_gen_usadd_vec,
2103 .fno = gen_helper_gvec_usadd16,
53229a77 2104 .opt_opc = vecop_list,
8afaf050
RH		 2105 .vece = MO_16 },
2106 { .fni4 = tcg_gen_usadd_i32,
2107 .fniv = tcg_gen_usadd_vec,
f49b12c6 2108 .fno = gen_helper_gvec_usadd32,
53229a77 2109 .opt_opc = vecop_list,
f49b12c6 2110 .vece = MO_32 },
8afaf050
RH		 2111 { .fni8 = tcg_gen_usadd_i64,
2112 .fniv = tcg_gen_usadd_vec,
f49b12c6 2113 .fno = gen_helper_gvec_usadd64,
53229a77 2114 .opt_opc = vecop_list,
f49b12c6
RH		 2115 .vece = MO_64 }
2116 };
2117 tcg_debug_assert(vece <= MO_64);
2118 tcg_gen_gvec_3(dofs, aofs, bofs, oprsz, maxsz, &g[vece]);
2119}
2120
8afaf050 2121static void tcg_gen_ussub_i32(TCGv_i32 d, TCGv_i32 a, TCGv_i32 b)
f49b12c6
RH		 2122{
2123 TCGv_i32 min = tcg_const_i32(0);
2124 tcg_gen_sub_i32(d, a, b);
2125 tcg_gen_movcond_i32(TCG_COND_LTU, d, a, b, min, d);
2126 tcg_temp_free_i32(min);
2127}
2128
8afaf050 2129static void tcg_gen_ussub_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b)
f49b12c6
RH		 2130{
2131 TCGv_i64 min = tcg_const_i64(0);
2132 tcg_gen_sub_i64(d, a, b);
2133 tcg_gen_movcond_i64(TCG_COND_LTU, d, a, b, min, d);
2134 tcg_temp_free_i64(min);
2135}
2136
2137void tcg_gen_gvec_ussub(unsigned vece, uint32_t dofs, uint32_t aofs,
2138 uint32_t bofs, uint32_t oprsz, uint32_t maxsz)
2139{
53229a77 2140 static const TCGOpcode vecop_list[] = { INDEX_op_ussub_vec, 0 };
f49b12c6 2141 static const GVecGen3 g[4] = {
8afaf050
RH		 2142 { .fniv = tcg_gen_ussub_vec,
2143 .fno = gen_helper_gvec_ussub8,
53229a77 2144 .opt_opc = vecop_list,
8afaf050
RH		 2145 .vece = MO_8 },
2146 { .fniv = tcg_gen_ussub_vec,
2147 .fno = gen_helper_gvec_ussub16,
53229a77 2148 .opt_opc = vecop_list,
8afaf050
RH		 2149 .vece = MO_16 },
2150 { .fni4 = tcg_gen_ussub_i32,
2151 .fniv = tcg_gen_ussub_vec,
f49b12c6 2152 .fno = gen_helper_gvec_ussub32,
53229a77 2153 .opt_opc = vecop_list,
f49b12c6 2154 .vece = MO_32 },
8afaf050
RH		 2155 { .fni8 = tcg_gen_ussub_i64,
2156 .fniv = tcg_gen_ussub_vec,
f49b12c6 2157 .fno = gen_helper_gvec_ussub64,
53229a77 2158 .opt_opc = vecop_list,
f49b12c6
RH		 2159 .vece = MO_64 }
2160 };
2161 tcg_debug_assert(vece <= MO_64);
2162 tcg_gen_gvec_3(dofs, aofs, bofs, oprsz, maxsz, &g[vece]);
dd0a0fcd
RH		 2163}
2164
2165void tcg_gen_gvec_smin(unsigned vece, uint32_t dofs, uint32_t aofs,
2166 uint32_t bofs, uint32_t oprsz, uint32_t maxsz)
2167{
53229a77 2168 static const TCGOpcode vecop_list[] = { INDEX_op_smin_vec, 0 };
dd0a0fcd
RH		 2169 static const GVecGen3 g[4] = {
2170 { .fniv = tcg_gen_smin_vec,
2171 .fno = gen_helper_gvec_smin8,
53229a77 2172 .opt_opc = vecop_list,
dd0a0fcd
RH		 2173 .vece = MO_8 },
2174 { .fniv = tcg_gen_smin_vec,
2175 .fno = gen_helper_gvec_smin16,
53229a77 2176 .opt_opc = vecop_list,
dd0a0fcd
RH		 2177 .vece = MO_16 },
2178 { .fni4 = tcg_gen_smin_i32,
2179 .fniv = tcg_gen_smin_vec,
2180 .fno = gen_helper_gvec_smin32,
53229a77 2181 .opt_opc = vecop_list,
dd0a0fcd
RH		 2182 .vece = MO_32 },
2183 { .fni8 = tcg_gen_smin_i64,
2184 .fniv = tcg_gen_smin_vec,
2185 .fno = gen_helper_gvec_smin64,
53229a77 2186 .opt_opc = vecop_list,
dd0a0fcd
RH		 2187 .vece = MO_64 }
2188 };
2189 tcg_debug_assert(vece <= MO_64);
2190 tcg_gen_gvec_3(dofs, aofs, bofs, oprsz, maxsz, &g[vece]);
2191}
2192
2193void tcg_gen_gvec_umin(unsigned vece, uint32_t dofs, uint32_t aofs,
2194 uint32_t bofs, uint32_t oprsz, uint32_t maxsz)
2195{
53229a77 2196 static const TCGOpcode vecop_list[] = { INDEX_op_umin_vec, 0 };
dd0a0fcd
RH		 2197 static const GVecGen3 g[4] = {
2198 { .fniv = tcg_gen_umin_vec,
2199 .fno = gen_helper_gvec_umin8,
53229a77 2200 .opt_opc = vecop_list,
dd0a0fcd
RH		 2201 .vece = MO_8 },
2202 { .fniv = tcg_gen_umin_vec,
2203 .fno = gen_helper_gvec_umin16,
53229a77 2204 .opt_opc = vecop_list,
dd0a0fcd
RH		 2205 .vece = MO_16 },
2206 { .fni4 = tcg_gen_umin_i32,
2207 .fniv = tcg_gen_umin_vec,
2208 .fno = gen_helper_gvec_umin32,
53229a77 2209 .opt_opc = vecop_list,
dd0a0fcd
RH		 2210 .vece = MO_32 },
2211 { .fni8 = tcg_gen_umin_i64,
2212 .fniv = tcg_gen_umin_vec,
2213 .fno = gen_helper_gvec_umin64,
53229a77 2214 .opt_opc = vecop_list,
dd0a0fcd
RH		 2215 .vece = MO_64 }
2216 };
2217 tcg_debug_assert(vece <= MO_64);
2218 tcg_gen_gvec_3(dofs, aofs, bofs, oprsz, maxsz, &g[vece]);
2219}
2220
2221void tcg_gen_gvec_smax(unsigned vece, uint32_t dofs, uint32_t aofs,
2222 uint32_t bofs, uint32_t oprsz, uint32_t maxsz)
2223{
53229a77 2224 static const TCGOpcode vecop_list[] = { INDEX_op_smax_vec, 0 };
dd0a0fcd
RH		 2225 static const GVecGen3 g[4] = {
2226 { .fniv = tcg_gen_smax_vec,
2227 .fno = gen_helper_gvec_smax8,
53229a77 2228 .opt_opc = vecop_list,
dd0a0fcd
RH		 2229 .vece = MO_8 },
2230 { .fniv = tcg_gen_smax_vec,
2231 .fno = gen_helper_gvec_smax16,
53229a77 2232 .opt_opc = vecop_list,
dd0a0fcd
RH		 2233 .vece = MO_16 },
2234 { .fni4 = tcg_gen_smax_i32,
2235 .fniv = tcg_gen_smax_vec,
2236 .fno = gen_helper_gvec_smax32,
53229a77 2237 .opt_opc = vecop_list,
dd0a0fcd
RH		 2238 .vece = MO_32 },
2239 { .fni8 = tcg_gen_smax_i64,
2240 .fniv = tcg_gen_smax_vec,
2241 .fno = gen_helper_gvec_smax64,
53229a77 2242 .opt_opc = vecop_list,
dd0a0fcd
RH		 2243 .vece = MO_64 }
2244 };
2245 tcg_debug_assert(vece <= MO_64);
2246 tcg_gen_gvec_3(dofs, aofs, bofs, oprsz, maxsz, &g[vece]);
2247}
2248
2249void tcg_gen_gvec_umax(unsigned vece, uint32_t dofs, uint32_t aofs,
2250 uint32_t bofs, uint32_t oprsz, uint32_t maxsz)
2251{
53229a77 2252 static const TCGOpcode vecop_list[] = { INDEX_op_umax_vec, 0 };
dd0a0fcd
RH		 2253 static const GVecGen3 g[4] = {
2254 { .fniv = tcg_gen_umax_vec,
2255 .fno = gen_helper_gvec_umax8,
53229a77 2256 .opt_opc = vecop_list,
dd0a0fcd
RH		 2257 .vece = MO_8 },
2258 { .fniv = tcg_gen_umax_vec,
2259 .fno = gen_helper_gvec_umax16,
53229a77 2260 .opt_opc = vecop_list,
dd0a0fcd
RH		 2261 .vece = MO_16 },
2262 { .fni4 = tcg_gen_umax_i32,
2263 .fniv = tcg_gen_umax_vec,
2264 .fno = gen_helper_gvec_umax32,
53229a77 2265 .opt_opc = vecop_list,
dd0a0fcd
RH		 2266 .vece = MO_32 },
2267 { .fni8 = tcg_gen_umax_i64,
2268 .fniv = tcg_gen_umax_vec,
2269 .fno = gen_helper_gvec_umax64,
53229a77 2270 .opt_opc = vecop_list,
dd0a0fcd
RH		 2271 .vece = MO_64 }
2272 };
2273 tcg_debug_assert(vece <= MO_64);
2274 tcg_gen_gvec_3(dofs, aofs, bofs, oprsz, maxsz, &g[vece]);
f49b12c6
RH		 2275}
2276
db432672
RH		 2277/* Perform a vector negation using normal negation and a mask.
2278 Compare gen_subv_mask above. */
2279static void gen_negv_mask(TCGv_i64 d, TCGv_i64 b, TCGv_i64 m)
2280{
2281 TCGv_i64 t2 = tcg_temp_new_i64();
2282 TCGv_i64 t3 = tcg_temp_new_i64();
2283
2284 tcg_gen_andc_i64(t3, m, b);
2285 tcg_gen_andc_i64(t2, b, m);
2286 tcg_gen_sub_i64(d, m, t2);
2287 tcg_gen_xor_i64(d, d, t3);
2288
2289 tcg_temp_free_i64(t2);
2290 tcg_temp_free_i64(t3);
2291}
2292
2293void tcg_gen_vec_neg8_i64(TCGv_i64 d, TCGv_i64 b)
2294{
2295 TCGv_i64 m = tcg_const_i64(dup_const(MO_8, 0x80));
2296 gen_negv_mask(d, b, m);
2297 tcg_temp_free_i64(m);
2298}
2299
2300void tcg_gen_vec_neg16_i64(TCGv_i64 d, TCGv_i64 b)
2301{
2302 TCGv_i64 m = tcg_const_i64(dup_const(MO_16, 0x8000));
2303 gen_negv_mask(d, b, m);
2304 tcg_temp_free_i64(m);
2305}
2306
2307void tcg_gen_vec_neg32_i64(TCGv_i64 d, TCGv_i64 b)
2308{
2309 TCGv_i64 t1 = tcg_temp_new_i64();
2310 TCGv_i64 t2 = tcg_temp_new_i64();
2311
2312 tcg_gen_andi_i64(t1, b, ~0xffffffffull);
2313 tcg_gen_neg_i64(t2, b);
2314 tcg_gen_neg_i64(t1, t1);
2315 tcg_gen_deposit_i64(d, t1, t2, 0, 32);
2316
2317 tcg_temp_free_i64(t1);
2318 tcg_temp_free_i64(t2);
2319}
2320
2321void tcg_gen_gvec_neg(unsigned vece, uint32_t dofs, uint32_t aofs,
2322 uint32_t oprsz, uint32_t maxsz)
2323{
53229a77 2324 static const TCGOpcode vecop_list[] = { INDEX_op_neg_vec, 0 };
db432672
RH		 2325 static const GVecGen2 g[4] = {
2326 { .fni8 = tcg_gen_vec_neg8_i64,
2327 .fniv = tcg_gen_neg_vec,
2328 .fno = gen_helper_gvec_neg8,
53229a77 2329 .opt_opc = vecop_list,
db432672
RH		 2330 .vece = MO_8 },
2331 { .fni8 = tcg_gen_vec_neg16_i64,
2332 .fniv = tcg_gen_neg_vec,
2333 .fno = gen_helper_gvec_neg16,
53229a77 2334 .opt_opc = vecop_list,
db432672
RH		 2335 .vece = MO_16 },
2336 { .fni4 = tcg_gen_neg_i32,
2337 .fniv = tcg_gen_neg_vec,
2338 .fno = gen_helper_gvec_neg32,
53229a77 2339 .opt_opc = vecop_list,
db432672
RH		 2340 .vece = MO_32 },
2341 { .fni8 = tcg_gen_neg_i64,
2342 .fniv = tcg_gen_neg_vec,
2343 .fno = gen_helper_gvec_neg64,
53229a77 2344 .opt_opc = vecop_list,
db432672
RH		 2345 .prefer_i64 = TCG_TARGET_REG_BITS == 64,
2346 .vece = MO_64 },
2347 };
2348
2349 tcg_debug_assert(vece <= MO_64);
2350 tcg_gen_gvec_2(dofs, aofs, oprsz, maxsz, &g[vece]);
2351}
2352
bcefc902
RH		 2353static void gen_absv_mask(TCGv_i64 d, TCGv_i64 b, unsigned vece)
2354{
2355 TCGv_i64 t = tcg_temp_new_i64();
2356 int nbit = 8 << vece;
2357
2358 /* Create -1 for each negative element. */
2359 tcg_gen_shri_i64(t, b, nbit - 1);
2360 tcg_gen_andi_i64(t, t, dup_const(vece, 1));
2361 tcg_gen_muli_i64(t, t, (1 << nbit) - 1);
2362
2363 /*
e7e8f33f 2364 * Invert (via xor -1) and add one.
bcefc902
RH		 2365 * Because of the ordering the msb is cleared,
2366 * so we never have carry into the next element.
2367 */
2368 tcg_gen_xor_i64(d, b, t);
e7e8f33f
SL		 2369 tcg_gen_andi_i64(t, t, dup_const(vece, 1));
2370 tcg_gen_add_i64(d, d, t);
bcefc902
RH		 2371
2372 tcg_temp_free_i64(t);
2373}
2374
2375static void tcg_gen_vec_abs8_i64(TCGv_i64 d, TCGv_i64 b)
2376{
2377 gen_absv_mask(d, b, MO_8);
2378}
2379
2380static void tcg_gen_vec_abs16_i64(TCGv_i64 d, TCGv_i64 b)
2381{
2382 gen_absv_mask(d, b, MO_16);
2383}
2384
2385void tcg_gen_gvec_abs(unsigned vece, uint32_t dofs, uint32_t aofs,
2386 uint32_t oprsz, uint32_t maxsz)
2387{
2388 static const TCGOpcode vecop_list[] = { INDEX_op_abs_vec, 0 };
2389 static const GVecGen2 g[4] = {
2390 { .fni8 = tcg_gen_vec_abs8_i64,
2391 .fniv = tcg_gen_abs_vec,
2392 .fno = gen_helper_gvec_abs8,
2393 .opt_opc = vecop_list,
2394 .vece = MO_8 },
2395 { .fni8 = tcg_gen_vec_abs16_i64,
2396 .fniv = tcg_gen_abs_vec,
2397 .fno = gen_helper_gvec_abs16,
2398 .opt_opc = vecop_list,
2399 .vece = MO_16 },
2400 { .fni4 = tcg_gen_abs_i32,
2401 .fniv = tcg_gen_abs_vec,
2402 .fno = gen_helper_gvec_abs32,
2403 .opt_opc = vecop_list,
2404 .vece = MO_32 },
2405 { .fni8 = tcg_gen_abs_i64,
2406 .fniv = tcg_gen_abs_vec,
2407 .fno = gen_helper_gvec_abs64,
2408 .opt_opc = vecop_list,
2409 .prefer_i64 = TCG_TARGET_REG_BITS == 64,
2410 .vece = MO_64 },
2411 };
2412
2413 tcg_debug_assert(vece <= MO_64);
2414 tcg_gen_gvec_2(dofs, aofs, oprsz, maxsz, &g[vece]);
2415}
2416
db432672
RH		 2417void tcg_gen_gvec_and(unsigned vece, uint32_t dofs, uint32_t aofs,
2418 uint32_t bofs, uint32_t oprsz, uint32_t maxsz)
2419{
2420 static const GVecGen3 g = {
2421 .fni8 = tcg_gen_and_i64,
2422 .fniv = tcg_gen_and_vec,
2423 .fno = gen_helper_gvec_and,
db432672
RH		 2424 .prefer_i64 = TCG_TARGET_REG_BITS == 64,
2425 };
9a9eda78
RH		 2426
2427 if (aofs == bofs) {
2428 tcg_gen_gvec_mov(vece, dofs, aofs, oprsz, maxsz);
2429 } else {
2430 tcg_gen_gvec_3(dofs, aofs, bofs, oprsz, maxsz, &g);
2431 }
db432672
RH		 2432}
2433
2434void tcg_gen_gvec_or(unsigned vece, uint32_t dofs, uint32_t aofs,
2435 uint32_t bofs, uint32_t oprsz, uint32_t maxsz)
2436{
2437 static const GVecGen3 g = {
2438 .fni8 = tcg_gen_or_i64,
2439 .fniv = tcg_gen_or_vec,
2440 .fno = gen_helper_gvec_or,
db432672
RH		 2441 .prefer_i64 = TCG_TARGET_REG_BITS == 64,
2442 };
9a9eda78
RH		 2443
2444 if (aofs == bofs) {
2445 tcg_gen_gvec_mov(vece, dofs, aofs, oprsz, maxsz);
2446 } else {
2447 tcg_gen_gvec_3(dofs, aofs, bofs, oprsz, maxsz, &g);
2448 }
db432672
RH		 2449}
2450
2451void tcg_gen_gvec_xor(unsigned vece, uint32_t dofs, uint32_t aofs,
2452 uint32_t bofs, uint32_t oprsz, uint32_t maxsz)
2453{
2454 static const GVecGen3 g = {
2455 .fni8 = tcg_gen_xor_i64,
2456 .fniv = tcg_gen_xor_vec,
2457 .fno = gen_helper_gvec_xor,
db432672
RH		 2458 .prefer_i64 = TCG_TARGET_REG_BITS == 64,
2459 };
9a9eda78
RH		 2460
2461 if (aofs == bofs) {
03ddb6f3 2462 tcg_gen_gvec_dup_imm(MO_64, dofs, oprsz, maxsz, 0);
9a9eda78
RH		 2463 } else {
2464 tcg_gen_gvec_3(dofs, aofs, bofs, oprsz, maxsz, &g);
2465 }
db432672
RH		 2466}
2467
2468void tcg_gen_gvec_andc(unsigned vece, uint32_t dofs, uint32_t aofs,
2469 uint32_t bofs, uint32_t oprsz, uint32_t maxsz)
2470{
2471 static const GVecGen3 g = {
2472 .fni8 = tcg_gen_andc_i64,
2473 .fniv = tcg_gen_andc_vec,
2474 .fno = gen_helper_gvec_andc,
db432672
RH		 2475 .prefer_i64 = TCG_TARGET_REG_BITS == 64,
2476 };
9a9eda78
RH		 2477
2478 if (aofs == bofs) {
03ddb6f3 2479 tcg_gen_gvec_dup_imm(MO_64, dofs, oprsz, maxsz, 0);
9a9eda78
RH		 2480 } else {
2481 tcg_gen_gvec_3(dofs, aofs, bofs, oprsz, maxsz, &g);
2482 }
db432672
RH		 2483}
2484
2485void tcg_gen_gvec_orc(unsigned vece, uint32_t dofs, uint32_t aofs,
2486 uint32_t bofs, uint32_t oprsz, uint32_t maxsz)
2487{
2488 static const GVecGen3 g = {
2489 .fni8 = tcg_gen_orc_i64,
2490 .fniv = tcg_gen_orc_vec,
2491 .fno = gen_helper_gvec_orc,
db432672
RH		 2492 .prefer_i64 = TCG_TARGET_REG_BITS == 64,
2493 };
9a9eda78
RH		 2494
2495 if (aofs == bofs) {
03ddb6f3 2496 tcg_gen_gvec_dup_imm(MO_64, dofs, oprsz, maxsz, -1);
9a9eda78 2497 } else {
f550805d
RH		 2498 tcg_gen_gvec_3(dofs, aofs, bofs, oprsz, maxsz, &g);
2499 }
2500}
2501
2502void tcg_gen_gvec_nand(unsigned vece, uint32_t dofs, uint32_t aofs,
2503 uint32_t bofs, uint32_t oprsz, uint32_t maxsz)
2504{
2505 static const GVecGen3 g = {
2506 .fni8 = tcg_gen_nand_i64,
2507 .fniv = tcg_gen_nand_vec,
2508 .fno = gen_helper_gvec_nand,
2509 .prefer_i64 = TCG_TARGET_REG_BITS == 64,
2510 };
2511
2512 if (aofs == bofs) {
2513 tcg_gen_gvec_not(vece, dofs, aofs, oprsz, maxsz);
2514 } else {
2515 tcg_gen_gvec_3(dofs, aofs, bofs, oprsz, maxsz, &g);
2516 }
2517}
2518
2519void tcg_gen_gvec_nor(unsigned vece, uint32_t dofs, uint32_t aofs,
2520 uint32_t bofs, uint32_t oprsz, uint32_t maxsz)
2521{
2522 static const GVecGen3 g = {
2523 .fni8 = tcg_gen_nor_i64,
2524 .fniv = tcg_gen_nor_vec,
2525 .fno = gen_helper_gvec_nor,
2526 .prefer_i64 = TCG_TARGET_REG_BITS == 64,
2527 };
2528
2529 if (aofs == bofs) {
2530 tcg_gen_gvec_not(vece, dofs, aofs, oprsz, maxsz);
2531 } else {
2532 tcg_gen_gvec_3(dofs, aofs, bofs, oprsz, maxsz, &g);
2533 }
2534}
2535
2536void tcg_gen_gvec_eqv(unsigned vece, uint32_t dofs, uint32_t aofs,
2537 uint32_t bofs, uint32_t oprsz, uint32_t maxsz)
2538{
2539 static const GVecGen3 g = {
2540 .fni8 = tcg_gen_eqv_i64,
2541 .fniv = tcg_gen_eqv_vec,
2542 .fno = gen_helper_gvec_eqv,
2543 .prefer_i64 = TCG_TARGET_REG_BITS == 64,
2544 };
2545
2546 if (aofs == bofs) {
03ddb6f3 2547 tcg_gen_gvec_dup_imm(MO_64, dofs, oprsz, maxsz, -1);
f550805d 2548 } else {
9a9eda78
RH		 2549 tcg_gen_gvec_3(dofs, aofs, bofs, oprsz, maxsz, &g);
2550 }
db432672 2551}
d0ec9796 2552
22fc3527
RH		 2553static const GVecGen2s gop_ands = {
2554 .fni8 = tcg_gen_and_i64,
2555 .fniv = tcg_gen_and_vec,
2556 .fno = gen_helper_gvec_ands,
22fc3527
RH		 2557 .prefer_i64 = TCG_TARGET_REG_BITS == 64,
2558 .vece = MO_64
2559};
2560
2561void tcg_gen_gvec_ands(unsigned vece, uint32_t dofs, uint32_t aofs,
2562 TCGv_i64 c, uint32_t oprsz, uint32_t maxsz)
2563{
2564 TCGv_i64 tmp = tcg_temp_new_i64();
2565 gen_dup_i64(vece, tmp, c);
2566 tcg_gen_gvec_2s(dofs, aofs, oprsz, maxsz, tmp, &gop_ands);
2567 tcg_temp_free_i64(tmp);
2568}
2569
2570void tcg_gen_gvec_andi(unsigned vece, uint32_t dofs, uint32_t aofs,
2571 int64_t c, uint32_t oprsz, uint32_t maxsz)
2572{
2573 TCGv_i64 tmp = tcg_const_i64(dup_const(vece, c));
2574 tcg_gen_gvec_2s(dofs, aofs, oprsz, maxsz, tmp, &gop_ands);
2575 tcg_temp_free_i64(tmp);
2576}
2577
2578static const GVecGen2s gop_xors = {
2579 .fni8 = tcg_gen_xor_i64,
2580 .fniv = tcg_gen_xor_vec,
2581 .fno = gen_helper_gvec_xors,
22fc3527
RH		 2582 .prefer_i64 = TCG_TARGET_REG_BITS == 64,
2583 .vece = MO_64
2584};
2585
2586void tcg_gen_gvec_xors(unsigned vece, uint32_t dofs, uint32_t aofs,
2587 TCGv_i64 c, uint32_t oprsz, uint32_t maxsz)
2588{
2589 TCGv_i64 tmp = tcg_temp_new_i64();
2590 gen_dup_i64(vece, tmp, c);
2591 tcg_gen_gvec_2s(dofs, aofs, oprsz, maxsz, tmp, &gop_xors);
2592 tcg_temp_free_i64(tmp);
2593}
2594
2595void tcg_gen_gvec_xori(unsigned vece, uint32_t dofs, uint32_t aofs,
2596 int64_t c, uint32_t oprsz, uint32_t maxsz)
2597{
2598 TCGv_i64 tmp = tcg_const_i64(dup_const(vece, c));
2599 tcg_gen_gvec_2s(dofs, aofs, oprsz, maxsz, tmp, &gop_xors);
2600 tcg_temp_free_i64(tmp);
2601}
2602
2603static const GVecGen2s gop_ors = {
2604 .fni8 = tcg_gen_or_i64,
2605 .fniv = tcg_gen_or_vec,
2606 .fno = gen_helper_gvec_ors,
22fc3527
RH		 2607 .prefer_i64 = TCG_TARGET_REG_BITS == 64,
2608 .vece = MO_64
2609};
2610
2611void tcg_gen_gvec_ors(unsigned vece, uint32_t dofs, uint32_t aofs,
2612 TCGv_i64 c, uint32_t oprsz, uint32_t maxsz)
2613{
2614 TCGv_i64 tmp = tcg_temp_new_i64();
2615 gen_dup_i64(vece, tmp, c);
2616 tcg_gen_gvec_2s(dofs, aofs, oprsz, maxsz, tmp, &gop_ors);
2617 tcg_temp_free_i64(tmp);
2618}
2619
2620void tcg_gen_gvec_ori(unsigned vece, uint32_t dofs, uint32_t aofs,
2621 int64_t c, uint32_t oprsz, uint32_t maxsz)
2622{
2623 TCGv_i64 tmp = tcg_const_i64(dup_const(vece, c));
2624 tcg_gen_gvec_2s(dofs, aofs, oprsz, maxsz, tmp, &gop_ors);
2625 tcg_temp_free_i64(tmp);
2626}
2627
d0ec9796
RH		 2628void tcg_gen_vec_shl8i_i64(TCGv_i64 d, TCGv_i64 a, int64_t c)
2629{
2630 uint64_t mask = dup_const(MO_8, 0xff << c);
2631 tcg_gen_shli_i64(d, a, c);
2632 tcg_gen_andi_i64(d, d, mask);
2633}
2634
2635void tcg_gen_vec_shl16i_i64(TCGv_i64 d, TCGv_i64 a, int64_t c)
2636{
2637 uint64_t mask = dup_const(MO_16, 0xffff << c);
2638 tcg_gen_shli_i64(d, a, c);
2639 tcg_gen_andi_i64(d, d, mask);
2640}
2641
2642void tcg_gen_gvec_shli(unsigned vece, uint32_t dofs, uint32_t aofs,
2643 int64_t shift, uint32_t oprsz, uint32_t maxsz)
2644{
53229a77 2645 static const TCGOpcode vecop_list[] = { INDEX_op_shli_vec, 0 };
d0ec9796
RH		 2646 static const GVecGen2i g[4] = {
2647 { .fni8 = tcg_gen_vec_shl8i_i64,
2648 .fniv = tcg_gen_shli_vec,
2649 .fno = gen_helper_gvec_shl8i,
53229a77 2650 .opt_opc = vecop_list,
d0ec9796
RH		 2651 .vece = MO_8 },
2652 { .fni8 = tcg_gen_vec_shl16i_i64,
2653 .fniv = tcg_gen_shli_vec,
2654 .fno = gen_helper_gvec_shl16i,
53229a77 2655 .opt_opc = vecop_list,
d0ec9796
RH		 2656 .vece = MO_16 },
2657 { .fni4 = tcg_gen_shli_i32,
2658 .fniv = tcg_gen_shli_vec,
2659 .fno = gen_helper_gvec_shl32i,
53229a77 2660 .opt_opc = vecop_list,
d0ec9796
RH		 2661 .vece = MO_32 },
2662 { .fni8 = tcg_gen_shli_i64,
2663 .fniv = tcg_gen_shli_vec,
2664 .fno = gen_helper_gvec_shl64i,
53229a77 2665 .opt_opc = vecop_list,
d0ec9796
RH		 2666 .prefer_i64 = TCG_TARGET_REG_BITS == 64,
2667 .vece = MO_64 },
2668 };
2669
2670 tcg_debug_assert(vece <= MO_64);
2671 tcg_debug_assert(shift >= 0 && shift < (8 << vece));
2672 if (shift == 0) {
2673 tcg_gen_gvec_mov(vece, dofs, aofs, oprsz, maxsz);
2674 } else {
2675 tcg_gen_gvec_2i(dofs, aofs, oprsz, maxsz, shift, &g[vece]);
2676 }
2677}
2678
2679void tcg_gen_vec_shr8i_i64(TCGv_i64 d, TCGv_i64 a, int64_t c)
2680{
2681 uint64_t mask = dup_const(MO_8, 0xff >> c);
2682 tcg_gen_shri_i64(d, a, c);
2683 tcg_gen_andi_i64(d, d, mask);
2684}
2685
2686void tcg_gen_vec_shr16i_i64(TCGv_i64 d, TCGv_i64 a, int64_t c)
2687{
2688 uint64_t mask = dup_const(MO_16, 0xffff >> c);
2689 tcg_gen_shri_i64(d, a, c);
2690 tcg_gen_andi_i64(d, d, mask);
2691}
2692
2693void tcg_gen_gvec_shri(unsigned vece, uint32_t dofs, uint32_t aofs,
2694 int64_t shift, uint32_t oprsz, uint32_t maxsz)
2695{
53229a77 2696 static const TCGOpcode vecop_list[] = { INDEX_op_shri_vec, 0 };
d0ec9796
RH		 2697 static const GVecGen2i g[4] = {
2698 { .fni8 = tcg_gen_vec_shr8i_i64,
2699 .fniv = tcg_gen_shri_vec,
2700 .fno = gen_helper_gvec_shr8i,
53229a77 2701 .opt_opc = vecop_list,
d0ec9796
RH		 2702 .vece = MO_8 },
2703 { .fni8 = tcg_gen_vec_shr16i_i64,
2704 .fniv = tcg_gen_shri_vec,
2705 .fno = gen_helper_gvec_shr16i,
53229a77 2706 .opt_opc = vecop_list,
d0ec9796
RH		 2707 .vece = MO_16 },
2708 { .fni4 = tcg_gen_shri_i32,
2709 .fniv = tcg_gen_shri_vec,
2710 .fno = gen_helper_gvec_shr32i,
53229a77 2711 .opt_opc = vecop_list,
d0ec9796
RH		 2712 .vece = MO_32 },
2713 { .fni8 = tcg_gen_shri_i64,
2714 .fniv = tcg_gen_shri_vec,
2715 .fno = gen_helper_gvec_shr64i,
53229a77 2716 .opt_opc = vecop_list,
d0ec9796
RH		 2717 .prefer_i64 = TCG_TARGET_REG_BITS == 64,
2718 .vece = MO_64 },
2719 };
2720
2721 tcg_debug_assert(vece <= MO_64);
2722 tcg_debug_assert(shift >= 0 && shift < (8 << vece));
2723 if (shift == 0) {
2724 tcg_gen_gvec_mov(vece, dofs, aofs, oprsz, maxsz);
2725 } else {
2726 tcg_gen_gvec_2i(dofs, aofs, oprsz, maxsz, shift, &g[vece]);
2727 }
2728}
2729
2730void tcg_gen_vec_sar8i_i64(TCGv_i64 d, TCGv_i64 a, int64_t c)
2731{
2732 uint64_t s_mask = dup_const(MO_8, 0x80 >> c);
2733 uint64_t c_mask = dup_const(MO_8, 0xff >> c);
2734 TCGv_i64 s = tcg_temp_new_i64();
2735
2736 tcg_gen_shri_i64(d, a, c);
2737 tcg_gen_andi_i64(s, d, s_mask); /* isolate (shifted) sign bit */
2738 tcg_gen_muli_i64(s, s, (2 << c) - 2); /* replicate isolated signs */
2739 tcg_gen_andi_i64(d, d, c_mask); /* clear out bits above sign */
2740 tcg_gen_or_i64(d, d, s); /* include sign extension */
2741 tcg_temp_free_i64(s);
2742}
2743
2744void tcg_gen_vec_sar16i_i64(TCGv_i64 d, TCGv_i64 a, int64_t c)
2745{
2746 uint64_t s_mask = dup_const(MO_16, 0x8000 >> c);
2747 uint64_t c_mask = dup_const(MO_16, 0xffff >> c);
2748 TCGv_i64 s = tcg_temp_new_i64();
2749
2750 tcg_gen_shri_i64(d, a, c);
2751 tcg_gen_andi_i64(s, d, s_mask); /* isolate (shifted) sign bit */
2752 tcg_gen_andi_i64(d, d, c_mask); /* clear out bits above sign */
2753 tcg_gen_muli_i64(s, s, (2 << c) - 2); /* replicate isolated signs */
2754 tcg_gen_or_i64(d, d, s); /* include sign extension */
2755 tcg_temp_free_i64(s);
2756}
2757
2758void tcg_gen_gvec_sari(unsigned vece, uint32_t dofs, uint32_t aofs,
2759 int64_t shift, uint32_t oprsz, uint32_t maxsz)
2760{
53229a77 2761 static const TCGOpcode vecop_list[] = { INDEX_op_sari_vec, 0 };
d0ec9796
RH		 2762 static const GVecGen2i g[4] = {
2763 { .fni8 = tcg_gen_vec_sar8i_i64,
2764 .fniv = tcg_gen_sari_vec,
2765 .fno = gen_helper_gvec_sar8i,
53229a77 2766 .opt_opc = vecop_list,
d0ec9796
RH		 2767 .vece = MO_8 },
2768 { .fni8 = tcg_gen_vec_sar16i_i64,
2769 .fniv = tcg_gen_sari_vec,
2770 .fno = gen_helper_gvec_sar16i,
53229a77 2771 .opt_opc = vecop_list,
d0ec9796
RH		 2772 .vece = MO_16 },
2773 { .fni4 = tcg_gen_sari_i32,
2774 .fniv = tcg_gen_sari_vec,
2775 .fno = gen_helper_gvec_sar32i,
53229a77 2776 .opt_opc = vecop_list,
d0ec9796
RH		 2777 .vece = MO_32 },
2778 { .fni8 = tcg_gen_sari_i64,
2779 .fniv = tcg_gen_sari_vec,
2780 .fno = gen_helper_gvec_sar64i,
53229a77 2781 .opt_opc = vecop_list,
d0ec9796
RH		 2782 .prefer_i64 = TCG_TARGET_REG_BITS == 64,
2783 .vece = MO_64 },
2784 };
2785
2786 tcg_debug_assert(vece <= MO_64);
2787 tcg_debug_assert(shift >= 0 && shift < (8 << vece));
2788 if (shift == 0) {
2789 tcg_gen_gvec_mov(vece, dofs, aofs, oprsz, maxsz);
2790 } else {
2791 tcg_gen_gvec_2i(dofs, aofs, oprsz, maxsz, shift, &g[vece]);
2792 }
2793}
212be173 2794
b0f7e744
RH		 2795void tcg_gen_vec_rotl8i_i64(TCGv_i64 d, TCGv_i64 a, int64_t c)
2796{
2797 uint64_t mask = dup_const(MO_8, 0xff << c);
2798
2799 tcg_gen_shli_i64(d, a, c);
2800 tcg_gen_shri_i64(a, a, 8 - c);
2801 tcg_gen_andi_i64(d, d, mask);
2802 tcg_gen_andi_i64(a, a, ~mask);
2803 tcg_gen_or_i64(d, d, a);
2804}
2805
2806void tcg_gen_vec_rotl16i_i64(TCGv_i64 d, TCGv_i64 a, int64_t c)
2807{
2808 uint64_t mask = dup_const(MO_16, 0xffff << c);
2809
2810 tcg_gen_shli_i64(d, a, c);
2811 tcg_gen_shri_i64(a, a, 16 - c);
2812 tcg_gen_andi_i64(d, d, mask);
2813 tcg_gen_andi_i64(a, a, ~mask);
2814 tcg_gen_or_i64(d, d, a);
2815}
2816
2817void tcg_gen_gvec_rotli(unsigned vece, uint32_t dofs, uint32_t aofs,
2818 int64_t shift, uint32_t oprsz, uint32_t maxsz)
2819{
2820 static const TCGOpcode vecop_list[] = { INDEX_op_rotli_vec, 0 };
2821 static const GVecGen2i g[4] = {
2822 { .fni8 = tcg_gen_vec_rotl8i_i64,
2823 .fniv = tcg_gen_rotli_vec,
2824 .fno = gen_helper_gvec_rotl8i,
2825 .opt_opc = vecop_list,
2826 .vece = MO_8 },
2827 { .fni8 = tcg_gen_vec_rotl16i_i64,
2828 .fniv = tcg_gen_rotli_vec,
2829 .fno = gen_helper_gvec_rotl16i,
2830 .opt_opc = vecop_list,
2831 .vece = MO_16 },
2832 { .fni4 = tcg_gen_rotli_i32,
2833 .fniv = tcg_gen_rotli_vec,
2834 .fno = gen_helper_gvec_rotl32i,
2835 .opt_opc = vecop_list,
2836 .vece = MO_32 },
2837 { .fni8 = tcg_gen_rotli_i64,
2838 .fniv = tcg_gen_rotli_vec,
2839 .fno = gen_helper_gvec_rotl64i,
2840 .opt_opc = vecop_list,
2841 .prefer_i64 = TCG_TARGET_REG_BITS == 64,
2842 .vece = MO_64 },
2843 };
2844
2845 tcg_debug_assert(vece <= MO_64);
2846 tcg_debug_assert(shift >= 0 && shift < (8 << vece));
2847 if (shift == 0) {
2848 tcg_gen_gvec_mov(vece, dofs, aofs, oprsz, maxsz);
2849 } else {
2850 tcg_gen_gvec_2i(dofs, aofs, oprsz, maxsz, shift, &g[vece]);
2851 }
2852}
2853
2854void tcg_gen_gvec_rotri(unsigned vece, uint32_t dofs, uint32_t aofs,
2855 int64_t shift, uint32_t oprsz, uint32_t maxsz)
2856{
2857 tcg_debug_assert(vece <= MO_64);
2858 tcg_debug_assert(shift >= 0 && shift < (8 << vece));
2859 tcg_gen_gvec_rotli(vece, dofs, aofs, -shift & ((8 << vece) - 1),
2860 oprsz, maxsz);
2861}
2862
b4578cd9
RH		 2863/*
2864 * Specialized generation vector shifts by a non-constant scalar.
2865 */
2866
2867typedef struct {
2868 void (*fni4)(TCGv_i32, TCGv_i32, TCGv_i32);
2869 void (*fni8)(TCGv_i64, TCGv_i64, TCGv_i64);
2870 void (*fniv_s)(unsigned, TCGv_vec, TCGv_vec, TCGv_i32);
2871 void (*fniv_v)(unsigned, TCGv_vec, TCGv_vec, TCGv_vec);
2872 gen_helper_gvec_2 *fno[4];
2873 TCGOpcode s_list[2];
2874 TCGOpcode v_list[2];
2875} GVecGen2sh;
2876
2877static void expand_2sh_vec(unsigned vece, uint32_t dofs, uint32_t aofs,
2878 uint32_t oprsz, uint32_t tysz, TCGType type,
2879 TCGv_i32 shift,
2880 void (*fni)(unsigned, TCGv_vec, TCGv_vec, TCGv_i32))
2881{
2882 TCGv_vec t0 = tcg_temp_new_vec(type);
2883 uint32_t i;
2884
2885 for (i = 0; i < oprsz; i += tysz) {
2886 tcg_gen_ld_vec(t0, cpu_env, aofs + i);
2887 fni(vece, t0, t0, shift);
2888 tcg_gen_st_vec(t0, cpu_env, dofs + i);
2889 }
2890 tcg_temp_free_vec(t0);
2891}
2892
2893static void
2894do_gvec_shifts(unsigned vece, uint32_t dofs, uint32_t aofs, TCGv_i32 shift,
2895 uint32_t oprsz, uint32_t maxsz, const GVecGen2sh *g)
2896{
2897 TCGType type;
2898 uint32_t some;
2899
2900 check_size_align(oprsz, maxsz, dofs | aofs);
2901 check_overlap_2(dofs, aofs, maxsz);
2902
2903 /* If the backend has a scalar expansion, great. */
2904 type = choose_vector_type(g->s_list, vece, oprsz, vece == MO_64);
2905 if (type) {
2906 const TCGOpcode *hold_list = tcg_swap_vecop_list(NULL);
2907 switch (type) {
2908 case TCG_TYPE_V256:
2909 some = QEMU_ALIGN_DOWN(oprsz, 32);
2910 expand_2sh_vec(vece, dofs, aofs, some, 32,
2911 TCG_TYPE_V256, shift, g->fniv_s);
2912 if (some == oprsz) {
2913 break;
2914 }
2915 dofs += some;
2916 aofs += some;
2917 oprsz -= some;
2918 maxsz -= some;
2919 /* fallthru */
2920 case TCG_TYPE_V128:
2921 expand_2sh_vec(vece, dofs, aofs, oprsz, 16,
2922 TCG_TYPE_V128, shift, g->fniv_s);
2923 break;
2924 case TCG_TYPE_V64:
2925 expand_2sh_vec(vece, dofs, aofs, oprsz, 8,
2926 TCG_TYPE_V64, shift, g->fniv_s);
2927 break;
2928 default:
2929 g_assert_not_reached();
2930 }
2931 tcg_swap_vecop_list(hold_list);
2932 goto clear_tail;
2933 }
2934
2935 /* If the backend supports variable vector shifts, also cool. */
2936 type = choose_vector_type(g->v_list, vece, oprsz, vece == MO_64);
2937 if (type) {
2938 const TCGOpcode *hold_list = tcg_swap_vecop_list(NULL);
2939 TCGv_vec v_shift = tcg_temp_new_vec(type);
2940
2941 if (vece == MO_64) {
2942 TCGv_i64 sh64 = tcg_temp_new_i64();
2943 tcg_gen_extu_i32_i64(sh64, shift);
2944 tcg_gen_dup_i64_vec(MO_64, v_shift, sh64);
2945 tcg_temp_free_i64(sh64);
2946 } else {
2947 tcg_gen_dup_i32_vec(vece, v_shift, shift);
2948 }
2949
2950 switch (type) {
2951 case TCG_TYPE_V256:
2952 some = QEMU_ALIGN_DOWN(oprsz, 32);
2953 expand_2s_vec(vece, dofs, aofs, some, 32, TCG_TYPE_V256,
2954 v_shift, false, g->fniv_v);
2955 if (some == oprsz) {
2956 break;
2957 }
2958 dofs += some;
2959 aofs += some;
2960 oprsz -= some;
2961 maxsz -= some;
2962 /* fallthru */
2963 case TCG_TYPE_V128:
2964 expand_2s_vec(vece, dofs, aofs, oprsz, 16, TCG_TYPE_V128,
2965 v_shift, false, g->fniv_v);
2966 break;
2967 case TCG_TYPE_V64:
2968 expand_2s_vec(vece, dofs, aofs, oprsz, 8, TCG_TYPE_V64,
2969 v_shift, false, g->fniv_v);
2970 break;
2971 default:
2972 g_assert_not_reached();
2973 }
2974 tcg_temp_free_vec(v_shift);
2975 tcg_swap_vecop_list(hold_list);
2976 goto clear_tail;
2977 }
2978
2979 /* Otherwise fall back to integral... */
2980 if (vece == MO_32 && check_size_impl(oprsz, 4)) {
2981 expand_2s_i32(dofs, aofs, oprsz, shift, false, g->fni4);
2982 } else if (vece == MO_64 && check_size_impl(oprsz, 8)) {
2983 TCGv_i64 sh64 = tcg_temp_new_i64();
2984 tcg_gen_extu_i32_i64(sh64, shift);
2985 expand_2s_i64(dofs, aofs, oprsz, sh64, false, g->fni8);
2986 tcg_temp_free_i64(sh64);
2987 } else {
2988 TCGv_ptr a0 = tcg_temp_new_ptr();
2989 TCGv_ptr a1 = tcg_temp_new_ptr();
2990 TCGv_i32 desc = tcg_temp_new_i32();
2991
2992 tcg_gen_shli_i32(desc, shift, SIMD_DATA_SHIFT);
2993 tcg_gen_ori_i32(desc, desc, simd_desc(oprsz, maxsz, 0));
2994 tcg_gen_addi_ptr(a0, cpu_env, dofs);
2995 tcg_gen_addi_ptr(a1, cpu_env, aofs);
2996
2997 g->fno[vece](a0, a1, desc);
2998
2999 tcg_temp_free_ptr(a0);
3000 tcg_temp_free_ptr(a1);
3001 tcg_temp_free_i32(desc);
3002 return;
3003 }
3004
3005 clear_tail:
3006 if (oprsz < maxsz) {
3007 expand_clr(dofs + oprsz, maxsz - oprsz);
3008 }
3009}
3010
3011void tcg_gen_gvec_shls(unsigned vece, uint32_t dofs, uint32_t aofs,
3012 TCGv_i32 shift, uint32_t oprsz, uint32_t maxsz)
3013{
3014 static const GVecGen2sh g = {
3015 .fni4 = tcg_gen_shl_i32,
3016 .fni8 = tcg_gen_shl_i64,
3017 .fniv_s = tcg_gen_shls_vec,
3018 .fniv_v = tcg_gen_shlv_vec,
3019 .fno = {
3020 gen_helper_gvec_shl8i,
3021 gen_helper_gvec_shl16i,
3022 gen_helper_gvec_shl32i,
3023 gen_helper_gvec_shl64i,
3024 },
3025 .s_list = { INDEX_op_shls_vec, 0 },
3026 .v_list = { INDEX_op_shlv_vec, 0 },
3027 };
3028
3029 tcg_debug_assert(vece <= MO_64);
3030 do_gvec_shifts(vece, dofs, aofs, shift, oprsz, maxsz, &g);
3031}
3032
3033void tcg_gen_gvec_shrs(unsigned vece, uint32_t dofs, uint32_t aofs,
3034 TCGv_i32 shift, uint32_t oprsz, uint32_t maxsz)
3035{
3036 static const GVecGen2sh g = {
3037 .fni4 = tcg_gen_shr_i32,
3038 .fni8 = tcg_gen_shr_i64,
3039 .fniv_s = tcg_gen_shrs_vec,
3040 .fniv_v = tcg_gen_shrv_vec,
3041 .fno = {
3042 gen_helper_gvec_shr8i,
3043 gen_helper_gvec_shr16i,
3044 gen_helper_gvec_shr32i,
3045 gen_helper_gvec_shr64i,
3046 },
3047 .s_list = { INDEX_op_shrs_vec, 0 },
3048 .v_list = { INDEX_op_shrv_vec, 0 },
3049 };
3050
3051 tcg_debug_assert(vece <= MO_64);
3052 do_gvec_shifts(vece, dofs, aofs, shift, oprsz, maxsz, &g);
3053}
3054
3055void tcg_gen_gvec_sars(unsigned vece, uint32_t dofs, uint32_t aofs,
3056 TCGv_i32 shift, uint32_t oprsz, uint32_t maxsz)
3057{
3058 static const GVecGen2sh g = {
3059 .fni4 = tcg_gen_sar_i32,
3060 .fni8 = tcg_gen_sar_i64,
3061 .fniv_s = tcg_gen_sars_vec,
3062 .fniv_v = tcg_gen_sarv_vec,
3063 .fno = {
3064 gen_helper_gvec_sar8i,
3065 gen_helper_gvec_sar16i,
3066 gen_helper_gvec_sar32i,
3067 gen_helper_gvec_sar64i,
3068 },
3069 .s_list = { INDEX_op_sars_vec, 0 },
3070 .v_list = { INDEX_op_sarv_vec, 0 },
3071 };
3072
3073 tcg_debug_assert(vece <= MO_64);
3074 do_gvec_shifts(vece, dofs, aofs, shift, oprsz, maxsz, &g);
3075}
3076
23850a74
RH		 3077void tcg_gen_gvec_rotls(unsigned vece, uint32_t dofs, uint32_t aofs,
3078 TCGv_i32 shift, uint32_t oprsz, uint32_t maxsz)
3079{
3080 static const GVecGen2sh g = {
3081 .fni4 = tcg_gen_rotl_i32,
3082 .fni8 = tcg_gen_rotl_i64,
3083 .fniv_s = tcg_gen_rotls_vec,
3084 .fniv_v = tcg_gen_rotlv_vec,
3085 .fno = {
3086 gen_helper_gvec_rotl8i,
3087 gen_helper_gvec_rotl16i,
3088 gen_helper_gvec_rotl32i,
3089 gen_helper_gvec_rotl64i,
3090 },
3091 .s_list = { INDEX_op_rotls_vec, 0 },
3092 .v_list = { INDEX_op_rotlv_vec, 0 },
3093 };
3094
3095 tcg_debug_assert(vece <= MO_64);
3096 do_gvec_shifts(vece, dofs, aofs, shift, oprsz, maxsz, &g);
3097}
3098
5ee5c14c
RH		 3099/*
3100 * Expand D = A << (B % element bits)
3101 *
3102 * Unlike scalar shifts, where it is easy for the target front end
3103 * to include the modulo as part of the expansion. If the target
3104 * naturally includes the modulo as part of the operation, great!
3105 * If the target has some other behaviour from out-of-range shifts,
3106 * then it could not use this function anyway, and would need to
3107 * do it's own expansion with custom functions.
3108 */
3109static void tcg_gen_shlv_mod_vec(unsigned vece, TCGv_vec d,
3110 TCGv_vec a, TCGv_vec b)
3111{
3112 TCGv_vec t = tcg_temp_new_vec_matching(d);
3113
3114 tcg_gen_dupi_vec(vece, t, (8 << vece) - 1);
3115 tcg_gen_and_vec(vece, t, t, b);
3116 tcg_gen_shlv_vec(vece, d, a, t);
3117 tcg_temp_free_vec(t);
3118}
3119
3120static void tcg_gen_shl_mod_i32(TCGv_i32 d, TCGv_i32 a, TCGv_i32 b)
3121{
3122 TCGv_i32 t = tcg_temp_new_i32();
3123
3124 tcg_gen_andi_i32(t, b, 31);
3125 tcg_gen_shl_i32(d, a, t);
3126 tcg_temp_free_i32(t);
3127}
3128
3129static void tcg_gen_shl_mod_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b)
3130{
3131 TCGv_i64 t = tcg_temp_new_i64();
3132
3133 tcg_gen_andi_i64(t, b, 63);
3134 tcg_gen_shl_i64(d, a, t);
3135 tcg_temp_free_i64(t);
3136}
3137
3138void tcg_gen_gvec_shlv(unsigned vece, uint32_t dofs, uint32_t aofs,
3139 uint32_t bofs, uint32_t oprsz, uint32_t maxsz)
3140{
3141 static const TCGOpcode vecop_list[] = { INDEX_op_shlv_vec, 0 };
3142 static const GVecGen3 g[4] = {
3143 { .fniv = tcg_gen_shlv_mod_vec,
3144 .fno = gen_helper_gvec_shl8v,
3145 .opt_opc = vecop_list,
3146 .vece = MO_8 },
3147 { .fniv = tcg_gen_shlv_mod_vec,
3148 .fno = gen_helper_gvec_shl16v,
3149 .opt_opc = vecop_list,
3150 .vece = MO_16 },
3151 { .fni4 = tcg_gen_shl_mod_i32,
3152 .fniv = tcg_gen_shlv_mod_vec,
3153 .fno = gen_helper_gvec_shl32v,
3154 .opt_opc = vecop_list,
3155 .vece = MO_32 },
3156 { .fni8 = tcg_gen_shl_mod_i64,
3157 .fniv = tcg_gen_shlv_mod_vec,
3158 .fno = gen_helper_gvec_shl64v,
3159 .opt_opc = vecop_list,
3160 .prefer_i64 = TCG_TARGET_REG_BITS == 64,
3161 .vece = MO_64 },
3162 };
3163
3164 tcg_debug_assert(vece <= MO_64);
3165 tcg_gen_gvec_3(dofs, aofs, bofs, oprsz, maxsz, &g[vece]);
3166}
3167
3168/*
3169 * Similarly for logical right shifts.
3170 */
3171
3172static void tcg_gen_shrv_mod_vec(unsigned vece, TCGv_vec d,
3173 TCGv_vec a, TCGv_vec b)
3174{
3175 TCGv_vec t = tcg_temp_new_vec_matching(d);
3176
3177 tcg_gen_dupi_vec(vece, t, (8 << vece) - 1);
3178 tcg_gen_and_vec(vece, t, t, b);
3179 tcg_gen_shrv_vec(vece, d, a, t);
3180 tcg_temp_free_vec(t);
3181}
3182
3183static void tcg_gen_shr_mod_i32(TCGv_i32 d, TCGv_i32 a, TCGv_i32 b)
3184{
3185 TCGv_i32 t = tcg_temp_new_i32();
3186
3187 tcg_gen_andi_i32(t, b, 31);
3188 tcg_gen_shr_i32(d, a, t);
3189 tcg_temp_free_i32(t);
3190}
3191
3192static void tcg_gen_shr_mod_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b)
3193{
3194 TCGv_i64 t = tcg_temp_new_i64();
3195
3196 tcg_gen_andi_i64(t, b, 63);
3197 tcg_gen_shr_i64(d, a, t);
3198 tcg_temp_free_i64(t);
3199}
3200
3201void tcg_gen_gvec_shrv(unsigned vece, uint32_t dofs, uint32_t aofs,
3202 uint32_t bofs, uint32_t oprsz, uint32_t maxsz)
3203{
3204 static const TCGOpcode vecop_list[] = { INDEX_op_shrv_vec, 0 };
3205 static const GVecGen3 g[4] = {
3206 { .fniv = tcg_gen_shrv_mod_vec,
3207 .fno = gen_helper_gvec_shr8v,
3208 .opt_opc = vecop_list,
3209 .vece = MO_8 },
3210 { .fniv = tcg_gen_shrv_mod_vec,
3211 .fno = gen_helper_gvec_shr16v,
3212 .opt_opc = vecop_list,
3213 .vece = MO_16 },
3214 { .fni4 = tcg_gen_shr_mod_i32,
3215 .fniv = tcg_gen_shrv_mod_vec,
3216 .fno = gen_helper_gvec_shr32v,
3217 .opt_opc = vecop_list,
3218 .vece = MO_32 },
3219 { .fni8 = tcg_gen_shr_mod_i64,
3220 .fniv = tcg_gen_shrv_mod_vec,
3221 .fno = gen_helper_gvec_shr64v,
3222 .opt_opc = vecop_list,
3223 .prefer_i64 = TCG_TARGET_REG_BITS == 64,
3224 .vece = MO_64 },
3225 };
3226
3227 tcg_debug_assert(vece <= MO_64);
3228 tcg_gen_gvec_3(dofs, aofs, bofs, oprsz, maxsz, &g[vece]);
3229}
3230
3231/*
3232 * Similarly for arithmetic right shifts.
3233 */
3234
3235static void tcg_gen_sarv_mod_vec(unsigned vece, TCGv_vec d,
3236 TCGv_vec a, TCGv_vec b)
3237{
3238 TCGv_vec t = tcg_temp_new_vec_matching(d);
3239
3240 tcg_gen_dupi_vec(vece, t, (8 << vece) - 1);
3241 tcg_gen_and_vec(vece, t, t, b);
3242 tcg_gen_sarv_vec(vece, d, a, t);
3243 tcg_temp_free_vec(t);
3244}
3245
3246static void tcg_gen_sar_mod_i32(TCGv_i32 d, TCGv_i32 a, TCGv_i32 b)
3247{
3248 TCGv_i32 t = tcg_temp_new_i32();
3249
3250 tcg_gen_andi_i32(t, b, 31);
3251 tcg_gen_sar_i32(d, a, t);
3252 tcg_temp_free_i32(t);
3253}
3254
3255static void tcg_gen_sar_mod_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b)
3256{
3257 TCGv_i64 t = tcg_temp_new_i64();
3258
3259 tcg_gen_andi_i64(t, b, 63);
3260 tcg_gen_sar_i64(d, a, t);
3261 tcg_temp_free_i64(t);
3262}
3263
3264void tcg_gen_gvec_sarv(unsigned vece, uint32_t dofs, uint32_t aofs,
3265 uint32_t bofs, uint32_t oprsz, uint32_t maxsz)
3266{
3267 static const TCGOpcode vecop_list[] = { INDEX_op_sarv_vec, 0 };
3268 static const GVecGen3 g[4] = {
3269 { .fniv = tcg_gen_sarv_mod_vec,
3270 .fno = gen_helper_gvec_sar8v,
3271 .opt_opc = vecop_list,
3272 .vece = MO_8 },
3273 { .fniv = tcg_gen_sarv_mod_vec,
3274 .fno = gen_helper_gvec_sar16v,
3275 .opt_opc = vecop_list,
3276 .vece = MO_16 },
3277 { .fni4 = tcg_gen_sar_mod_i32,
3278 .fniv = tcg_gen_sarv_mod_vec,
3279 .fno = gen_helper_gvec_sar32v,
3280 .opt_opc = vecop_list,
3281 .vece = MO_32 },
3282 { .fni8 = tcg_gen_sar_mod_i64,
3283 .fniv = tcg_gen_sarv_mod_vec,
3284 .fno = gen_helper_gvec_sar64v,
3285 .opt_opc = vecop_list,
3286 .prefer_i64 = TCG_TARGET_REG_BITS == 64,
3287 .vece = MO_64 },
3288 };
3289
3290 tcg_debug_assert(vece <= MO_64);
3291 tcg_gen_gvec_3(dofs, aofs, bofs, oprsz, maxsz, &g[vece]);
3292}
3293
5d0ceda9
RH		 3294/*
3295 * Similarly for rotates.
3296 */
3297
3298static void tcg_gen_rotlv_mod_vec(unsigned vece, TCGv_vec d,
3299 TCGv_vec a, TCGv_vec b)
3300{
3301 TCGv_vec t = tcg_temp_new_vec_matching(d);
3302
3303 tcg_gen_dupi_vec(vece, t, (8 << vece) - 1);
3304 tcg_gen_and_vec(vece, t, t, b);
3305 tcg_gen_rotlv_vec(vece, d, a, t);
3306 tcg_temp_free_vec(t);
3307}
3308
3309static void tcg_gen_rotl_mod_i32(TCGv_i32 d, TCGv_i32 a, TCGv_i32 b)
3310{
3311 TCGv_i32 t = tcg_temp_new_i32();
3312
3313 tcg_gen_andi_i32(t, b, 31);
3314 tcg_gen_rotl_i32(d, a, t);
3315 tcg_temp_free_i32(t);
3316}
3317
3318static void tcg_gen_rotl_mod_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b)
3319{
3320 TCGv_i64 t = tcg_temp_new_i64();
3321
3322 tcg_gen_andi_i64(t, b, 63);
3323 tcg_gen_rotl_i64(d, a, t);
3324 tcg_temp_free_i64(t);
3325}
3326
3327void tcg_gen_gvec_rotlv(unsigned vece, uint32_t dofs, uint32_t aofs,
3328 uint32_t bofs, uint32_t oprsz, uint32_t maxsz)
3329{
3330 static const TCGOpcode vecop_list[] = { INDEX_op_rotlv_vec, 0 };
3331 static const GVecGen3 g[4] = {
3332 { .fniv = tcg_gen_rotlv_mod_vec,
3333 .fno = gen_helper_gvec_rotl8v,
3334 .opt_opc = vecop_list,
3335 .vece = MO_8 },
3336 { .fniv = tcg_gen_rotlv_mod_vec,
3337 .fno = gen_helper_gvec_rotl16v,
3338 .opt_opc = vecop_list,
3339 .vece = MO_16 },
3340 { .fni4 = tcg_gen_rotl_mod_i32,
3341 .fniv = tcg_gen_rotlv_mod_vec,
3342 .fno = gen_helper_gvec_rotl32v,
3343 .opt_opc = vecop_list,
3344 .vece = MO_32 },
3345 { .fni8 = tcg_gen_rotl_mod_i64,
3346 .fniv = tcg_gen_rotlv_mod_vec,
3347 .fno = gen_helper_gvec_rotl64v,
3348 .opt_opc = vecop_list,
3349 .prefer_i64 = TCG_TARGET_REG_BITS == 64,
3350 .vece = MO_64 },
3351 };
3352
3353 tcg_debug_assert(vece <= MO_64);
3354 tcg_gen_gvec_3(dofs, aofs, bofs, oprsz, maxsz, &g[vece]);
3355}
3356
3357static void tcg_gen_rotrv_mod_vec(unsigned vece, TCGv_vec d,
3358 TCGv_vec a, TCGv_vec b)
3359{
3360 TCGv_vec t = tcg_temp_new_vec_matching(d);
3361
3362 tcg_gen_dupi_vec(vece, t, (8 << vece) - 1);
3363 tcg_gen_and_vec(vece, t, t, b);
3364 tcg_gen_rotrv_vec(vece, d, a, t);
3365 tcg_temp_free_vec(t);
3366}
3367
3368static void tcg_gen_rotr_mod_i32(TCGv_i32 d, TCGv_i32 a, TCGv_i32 b)
3369{
3370 TCGv_i32 t = tcg_temp_new_i32();
3371
3372 tcg_gen_andi_i32(t, b, 31);
3373 tcg_gen_rotr_i32(d, a, t);
3374 tcg_temp_free_i32(t);
3375}
3376
3377static void tcg_gen_rotr_mod_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b)
3378{
3379 TCGv_i64 t = tcg_temp_new_i64();
3380
3381 tcg_gen_andi_i64(t, b, 63);
3382 tcg_gen_rotr_i64(d, a, t);
3383 tcg_temp_free_i64(t);
3384}
3385
3386void tcg_gen_gvec_rotrv(unsigned vece, uint32_t dofs, uint32_t aofs,
3387 uint32_t bofs, uint32_t oprsz, uint32_t maxsz)
3388{
3389 static const TCGOpcode vecop_list[] = { INDEX_op_rotrv_vec, 0 };
3390 static const GVecGen3 g[4] = {
3391 { .fniv = tcg_gen_rotrv_mod_vec,
3392 .fno = gen_helper_gvec_rotr8v,
3393 .opt_opc = vecop_list,
3394 .vece = MO_8 },
3395 { .fniv = tcg_gen_rotrv_mod_vec,
3396 .fno = gen_helper_gvec_rotr16v,
3397 .opt_opc = vecop_list,
3398 .vece = MO_16 },
3399 { .fni4 = tcg_gen_rotr_mod_i32,
3400 .fniv = tcg_gen_rotrv_mod_vec,
3401 .fno = gen_helper_gvec_rotr32v,
3402 .opt_opc = vecop_list,
3403 .vece = MO_32 },
3404 { .fni8 = tcg_gen_rotr_mod_i64,
3405 .fniv = tcg_gen_rotrv_mod_vec,
3406 .fno = gen_helper_gvec_rotr64v,
3407 .opt_opc = vecop_list,
3408 .prefer_i64 = TCG_TARGET_REG_BITS == 64,
3409 .vece = MO_64 },
3410 };
3411
3412 tcg_debug_assert(vece <= MO_64);
3413 tcg_gen_gvec_3(dofs, aofs, bofs, oprsz, maxsz, &g[vece]);
3414}
3415
212be173
RH		 3416/* Expand OPSZ bytes worth of three-operand operations using i32 elements. */
3417static void expand_cmp_i32(uint32_t dofs, uint32_t aofs, uint32_t bofs,
3418 uint32_t oprsz, TCGCond cond)
3419{
3420 TCGv_i32 t0 = tcg_temp_new_i32();
3421 TCGv_i32 t1 = tcg_temp_new_i32();
3422 uint32_t i;
3423
3424 for (i = 0; i < oprsz; i += 4) {
3425 tcg_gen_ld_i32(t0, cpu_env, aofs + i);
3426 tcg_gen_ld_i32(t1, cpu_env, bofs + i);
3427 tcg_gen_setcond_i32(cond, t0, t0, t1);
3428 tcg_gen_neg_i32(t0, t0);
3429 tcg_gen_st_i32(t0, cpu_env, dofs + i);
3430 }
3431 tcg_temp_free_i32(t1);
3432 tcg_temp_free_i32(t0);
3433}
3434
3435static void expand_cmp_i64(uint32_t dofs, uint32_t aofs, uint32_t bofs,
3436 uint32_t oprsz, TCGCond cond)
3437{
3438 TCGv_i64 t0 = tcg_temp_new_i64();
3439 TCGv_i64 t1 = tcg_temp_new_i64();
3440 uint32_t i;
3441
3442 for (i = 0; i < oprsz; i += 8) {
3443 tcg_gen_ld_i64(t0, cpu_env, aofs + i);
3444 tcg_gen_ld_i64(t1, cpu_env, bofs + i);
3445 tcg_gen_setcond_i64(cond, t0, t0, t1);
3446 tcg_gen_neg_i64(t0, t0);
3447 tcg_gen_st_i64(t0, cpu_env, dofs + i);
3448 }
3449 tcg_temp_free_i64(t1);
3450 tcg_temp_free_i64(t0);
3451}
3452
3453static void expand_cmp_vec(unsigned vece, uint32_t dofs, uint32_t aofs,
3454 uint32_t bofs, uint32_t oprsz, uint32_t tysz,
3455 TCGType type, TCGCond cond)
3456{
3457 TCGv_vec t0 = tcg_temp_new_vec(type);
3458 TCGv_vec t1 = tcg_temp_new_vec(type);
3459 uint32_t i;
3460
3461 for (i = 0; i < oprsz; i += tysz) {
3462 tcg_gen_ld_vec(t0, cpu_env, aofs + i);
3463 tcg_gen_ld_vec(t1, cpu_env, bofs + i);
3464 tcg_gen_cmp_vec(cond, vece, t0, t0, t1);
3465 tcg_gen_st_vec(t0, cpu_env, dofs + i);
3466 }
3467 tcg_temp_free_vec(t1);
3468 tcg_temp_free_vec(t0);
3469}
3470
3471void tcg_gen_gvec_cmp(TCGCond cond, unsigned vece, uint32_t dofs,
3472 uint32_t aofs, uint32_t bofs,
3473 uint32_t oprsz, uint32_t maxsz)
3474{
53229a77 3475 static const TCGOpcode cmp_list[] = { INDEX_op_cmp_vec, 0 };
212be173
RH		 3476 static gen_helper_gvec_3 * const eq_fn[4] = {
3477 gen_helper_gvec_eq8, gen_helper_gvec_eq16,
3478 gen_helper_gvec_eq32, gen_helper_gvec_eq64
3479 };
3480 static gen_helper_gvec_3 * const ne_fn[4] = {
3481 gen_helper_gvec_ne8, gen_helper_gvec_ne16,
3482 gen_helper_gvec_ne32, gen_helper_gvec_ne64
3483 };
3484 static gen_helper_gvec_3 * const lt_fn[4] = {
3485 gen_helper_gvec_lt8, gen_helper_gvec_lt16,
3486 gen_helper_gvec_lt32, gen_helper_gvec_lt64
3487 };
3488 static gen_helper_gvec_3 * const le_fn[4] = {
3489 gen_helper_gvec_le8, gen_helper_gvec_le16,
3490 gen_helper_gvec_le32, gen_helper_gvec_le64
3491 };
3492 static gen_helper_gvec_3 * const ltu_fn[4] = {
3493 gen_helper_gvec_ltu8, gen_helper_gvec_ltu16,
3494 gen_helper_gvec_ltu32, gen_helper_gvec_ltu64
3495 };
3496 static gen_helper_gvec_3 * const leu_fn[4] = {
3497 gen_helper_gvec_leu8, gen_helper_gvec_leu16,
3498 gen_helper_gvec_leu32, gen_helper_gvec_leu64
3499 };
3500 static gen_helper_gvec_3 * const * const fns[16] = {
3501 [TCG_COND_EQ] = eq_fn,
3502 [TCG_COND_NE] = ne_fn,
3503 [TCG_COND_LT] = lt_fn,
3504 [TCG_COND_LE] = le_fn,
3505 [TCG_COND_LTU] = ltu_fn,
3506 [TCG_COND_LEU] = leu_fn,
3507 };
53229a77
RH		 3508
3509 const TCGOpcode *hold_list;
adb196cb
RH		 3510 TCGType type;
3511 uint32_t some;
212be173
RH		 3512
3513 check_size_align(oprsz, maxsz, dofs | aofs | bofs);
3514 check_overlap_3(dofs, aofs, bofs, maxsz);
3515
3516 if (cond == TCG_COND_NEVER || cond == TCG_COND_ALWAYS) {
3517 do_dup(MO_8, dofs, oprsz, maxsz,
3518 NULL, NULL, -(cond == TCG_COND_ALWAYS));
3519 return;
3520 }
3521
53229a77
RH		 3522 /*
3523 * Implement inline with a vector type, if possible.
adb196cb
RH		 3524 * Prefer integer when 64-bit host and 64-bit comparison.
3525 */
53229a77
RH		 3526 hold_list = tcg_swap_vecop_list(cmp_list);
3527 type = choose_vector_type(cmp_list, vece, oprsz,
adb196cb
RH		 3528 TCG_TARGET_REG_BITS == 64 && vece == MO_64);
3529 switch (type) {
3530 case TCG_TYPE_V256:
3531 /* Recall that ARM SVE allows vector sizes that are not a
3532 * power of 2, but always a multiple of 16. The intent is
3533 * that e.g. size == 80 would be expanded with 2x32 + 1x16.
3534 */
3535 some = QEMU_ALIGN_DOWN(oprsz, 32);
212be173
RH		 3536 expand_cmp_vec(vece, dofs, aofs, bofs, some, 32, TCG_TYPE_V256, cond);
3537 if (some == oprsz) {
adb196cb 3538 break;
212be173
RH		 3539 }
3540 dofs += some;
3541 aofs += some;
3542 bofs += some;
3543 oprsz -= some;
3544 maxsz -= some;
adb196cb
RH		 3545 /* fallthru */
3546 case TCG_TYPE_V128:
212be173 3547 expand_cmp_vec(vece, dofs, aofs, bofs, oprsz, 16, TCG_TYPE_V128, cond);
adb196cb
RH		 3548 break;
3549 case TCG_TYPE_V64:
212be173 3550 expand_cmp_vec(vece, dofs, aofs, bofs, oprsz, 8, TCG_TYPE_V64, cond);
adb196cb
RH		 3551 break;
3552
3553 case 0:
3554 if (vece == MO_64 && check_size_impl(oprsz, 8)) {
3555 expand_cmp_i64(dofs, aofs, bofs, oprsz, cond);
3556 } else if (vece == MO_32 && check_size_impl(oprsz, 4)) {
3557 expand_cmp_i32(dofs, aofs, bofs, oprsz, cond);
3558 } else {
3559 gen_helper_gvec_3 * const *fn = fns[cond];
3560
3561 if (fn == NULL) {
3562 uint32_t tmp;
3563 tmp = aofs, aofs = bofs, bofs = tmp;
3564 cond = tcg_swap_cond(cond);
3565 fn = fns[cond];
3566 assert(fn != NULL);
3567 }
3568 tcg_gen_gvec_3_ool(dofs, aofs, bofs, oprsz, maxsz, 0, fn[vece]);
53229a77 3569 oprsz = maxsz;
212be173 3570 }
adb196cb
RH		 3571 break;
3572
3573 default:
3574 g_assert_not_reached();
212be173 3575 }
53229a77 3576 tcg_swap_vecop_list(hold_list);
212be173 3577
212be173
RH		 3578 if (oprsz < maxsz) {
3579 expand_clr(dofs + oprsz, maxsz - oprsz);
3580 }
3581}
38dc1294
RH		 3582
3583static void tcg_gen_bitsel_i64(TCGv_i64 d, TCGv_i64 a, TCGv_i64 b, TCGv_i64 c)
3584{
3585 TCGv_i64 t = tcg_temp_new_i64();
3586
3587 tcg_gen_and_i64(t, b, a);
3588 tcg_gen_andc_i64(d, c, a);
3589 tcg_gen_or_i64(d, d, t);
3590 tcg_temp_free_i64(t);
3591}
3592
3593void tcg_gen_gvec_bitsel(unsigned vece, uint32_t dofs, uint32_t aofs,
3594 uint32_t bofs, uint32_t cofs,
3595 uint32_t oprsz, uint32_t maxsz)
3596{
3597 static const GVecGen4 g = {
3598 .fni8 = tcg_gen_bitsel_i64,
3599 .fniv = tcg_gen_bitsel_vec,
3600 .fno = gen_helper_gvec_bitsel,
3601 };
3602
3603 tcg_gen_gvec_4(dofs, aofs, bofs, cofs, oprsz, maxsz, &g);
3604}
Empty description
This page took 0.619739 seconds and 4 git commands to generate.