]> Git Repo - qemu.git/blame - target/arm/neon_helper.c
projects / qemu.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Merge remote-tracking branch 'remotes/mst/tags/for_upstream' into staging
[qemu.git] / target / arm / neon_helper.c
CommitLineData
e677137d
PB		 1/*
2 * ARM NEON vector operations.
3 *
4 * Copyright (c) 2007, 2008 CodeSourcery.
5 * Written by Paul Brook
6 *
8e31bf38 7 * This code is licensed under the GNU GPL v2.
e677137d 8 */
74c21bd0 9#include "qemu/osdep.h"
ad69471c
PB		 10
11#include "cpu.h"
2ef6175a 12#include "exec/helper-proto.h"
24f91e81 13#include "fpu/softfloat.h"
ad69471c
PB		 14
15#define SIGNBIT (uint32_t)0x80000000
16#define SIGNBIT64 ((uint64_t)1 << 63)
17
a4d58462 18#define SET_QC() env->vfp.qc[0] = 1
ad69471c 19
ad69471c
PB		 20#define NEON_TYPE1(name, type) \
21typedef struct \
22{ \
23 type v1; \
24} neon_##name;
e2542fe2 25#ifdef HOST_WORDS_BIGENDIAN
ad69471c
PB		 26#define NEON_TYPE2(name, type) \
27typedef struct \
28{ \
29 type v2; \
30 type v1; \
31} neon_##name;
32#define NEON_TYPE4(name, type) \
33typedef struct \
34{ \
35 type v4; \
36 type v3; \
37 type v2; \
38 type v1; \
39} neon_##name;
40#else
41#define NEON_TYPE2(name, type) \
42typedef struct \
43{ \
44 type v1; \
45 type v2; \
46} neon_##name;
47#define NEON_TYPE4(name, type) \
48typedef struct \
49{ \
50 type v1; \
51 type v2; \
52 type v3; \
53 type v4; \
54} neon_##name;
55#endif
56
57NEON_TYPE4(s8, int8_t)
58NEON_TYPE4(u8, uint8_t)
59NEON_TYPE2(s16, int16_t)
60NEON_TYPE2(u16, uint16_t)
61NEON_TYPE1(s32, int32_t)
62NEON_TYPE1(u32, uint32_t)
63#undef NEON_TYPE4
64#undef NEON_TYPE2
65#undef NEON_TYPE1
66
67/* Copy from a uint32_t to a vector structure type. */
68#define NEON_UNPACK(vtype, dest, val) do { \
69 union { \
70 vtype v; \
71 uint32_t i; \
72 } conv_u; \
73 conv_u.i = (val); \
74 dest = conv_u.v; \
75 } while(0)
76
77/* Copy from a vector structure type to a uint32_t. */
78#define NEON_PACK(vtype, dest, val) do { \
79 union { \
80 vtype v; \
81 uint32_t i; \
82 } conv_u; \
83 conv_u.v = (val); \
84 dest = conv_u.i; \
85 } while(0)
86
87#define NEON_DO1 \
88 NEON_FN(vdest.v1, vsrc1.v1, vsrc2.v1);
89#define NEON_DO2 \
90 NEON_FN(vdest.v1, vsrc1.v1, vsrc2.v1); \
91 NEON_FN(vdest.v2, vsrc1.v2, vsrc2.v2);
92#define NEON_DO4 \
93 NEON_FN(vdest.v1, vsrc1.v1, vsrc2.v1); \
94 NEON_FN(vdest.v2, vsrc1.v2, vsrc2.v2); \
95 NEON_FN(vdest.v3, vsrc1.v3, vsrc2.v3); \
96 NEON_FN(vdest.v4, vsrc1.v4, vsrc2.v4);
97
98#define NEON_VOP_BODY(vtype, n) \
99{ \
100 uint32_t res; \
101 vtype vsrc1; \
102 vtype vsrc2; \
103 vtype vdest; \
104 NEON_UNPACK(vtype, vsrc1, arg1); \
105 NEON_UNPACK(vtype, vsrc2, arg2); \
106 NEON_DO##n; \
107 NEON_PACK(vtype, res, vdest); \
108 return res; \
109}
110
111#define NEON_VOP(name, vtype, n) \
112uint32_t HELPER(glue(neon_,name))(uint32_t arg1, uint32_t arg2) \
113NEON_VOP_BODY(vtype, n)
114
02da0b2d 115#define NEON_VOP_ENV(name, vtype, n) \
0ecb72a5 116uint32_t HELPER(glue(neon_,name))(CPUARMState *env, uint32_t arg1, uint32_t arg2) \
02da0b2d
PM		 117NEON_VOP_BODY(vtype, n)
118
ad69471c
PB		 119/* Pairwise operations. */
120/* For 32-bit elements each segment only contains a single element, so
121 the elementwise and pairwise operations are the same. */
122#define NEON_PDO2 \
123 NEON_FN(vdest.v1, vsrc1.v1, vsrc1.v2); \
124 NEON_FN(vdest.v2, vsrc2.v1, vsrc2.v2);
125#define NEON_PDO4 \
126 NEON_FN(vdest.v1, vsrc1.v1, vsrc1.v2); \
127 NEON_FN(vdest.v2, vsrc1.v3, vsrc1.v4); \
128 NEON_FN(vdest.v3, vsrc2.v1, vsrc2.v2); \
129 NEON_FN(vdest.v4, vsrc2.v3, vsrc2.v4); \
130
131#define NEON_POP(name, vtype, n) \
132uint32_t HELPER(glue(neon_,name))(uint32_t arg1, uint32_t arg2) \
133{ \
134 uint32_t res; \
135 vtype vsrc1; \
136 vtype vsrc2; \
137 vtype vdest; \
138 NEON_UNPACK(vtype, vsrc1, arg1); \
139 NEON_UNPACK(vtype, vsrc2, arg2); \
140 NEON_PDO##n; \
141 NEON_PACK(vtype, res, vdest); \
142 return res; \
143}
144
145/* Unary operators. */
146#define NEON_VOP1(name, vtype, n) \
147uint32_t HELPER(glue(neon_,name))(uint32_t arg) \
148{ \
149 vtype vsrc1; \
150 vtype vdest; \
151 NEON_UNPACK(vtype, vsrc1, arg); \
152 NEON_DO##n; \
153 NEON_PACK(vtype, arg, vdest); \
154 return arg; \
155}
156
157
158#define NEON_USAT(dest, src1, src2, type) do { \
159 uint32_t tmp = (uint32_t)src1 + (uint32_t)src2; \
160 if (tmp != (type)tmp) { \
161 SET_QC(); \
162 dest = ~0; \
163 } else { \
164 dest = tmp; \
165 }} while(0)
166#define NEON_FN(dest, src1, src2) NEON_USAT(dest, src1, src2, uint8_t)
02da0b2d 167NEON_VOP_ENV(qadd_u8, neon_u8, 4)
ad69471c
PB		 168#undef NEON_FN
169#define NEON_FN(dest, src1, src2) NEON_USAT(dest, src1, src2, uint16_t)
02da0b2d 170NEON_VOP_ENV(qadd_u16, neon_u16, 2)
ad69471c
PB		 171#undef NEON_FN
172#undef NEON_USAT
173
0ecb72a5 174uint32_t HELPER(neon_qadd_u32)(CPUARMState *env, uint32_t a, uint32_t b)
72902672
CL		 175{
176 uint32_t res = a + b;
177 if (res < a) {
178 SET_QC();
179 res = ~0;
180 }
181 return res;
182}
183
0ecb72a5 184uint64_t HELPER(neon_qadd_u64)(CPUARMState *env, uint64_t src1, uint64_t src2)
72902672
CL		 185{
186 uint64_t res;
187
188 res = src1 + src2;
189 if (res < src1) {
190 SET_QC();
191 res = ~(uint64_t)0;
192 }
193 return res;
194}
195
ad69471c
PB		 196#define NEON_SSAT(dest, src1, src2, type) do { \
197 int32_t tmp = (uint32_t)src1 + (uint32_t)src2; \
198 if (tmp != (type)tmp) { \
199 SET_QC(); \
200 if (src2 > 0) { \
201 tmp = (1 << (sizeof(type) * 8 - 1)) - 1; \
202 } else { \
203 tmp = 1 << (sizeof(type) * 8 - 1); \
204 } \
205 } \
206 dest = tmp; \
207 } while(0)
208#define NEON_FN(dest, src1, src2) NEON_SSAT(dest, src1, src2, int8_t)
02da0b2d 209NEON_VOP_ENV(qadd_s8, neon_s8, 4)
ad69471c
PB		 210#undef NEON_FN
211#define NEON_FN(dest, src1, src2) NEON_SSAT(dest, src1, src2, int16_t)
02da0b2d 212NEON_VOP_ENV(qadd_s16, neon_s16, 2)
ad69471c
PB		 213#undef NEON_FN
214#undef NEON_SSAT
215
0ecb72a5 216uint32_t HELPER(neon_qadd_s32)(CPUARMState *env, uint32_t a, uint32_t b)
72902672
CL		 217{
218 uint32_t res = a + b;
219 if (((res ^ a) & SIGNBIT) && !((a ^ b) & SIGNBIT)) {
220 SET_QC();
221 res = ~(((int32_t)a >> 31) ^ SIGNBIT);
222 }
223 return res;
224}
225
0ecb72a5 226uint64_t HELPER(neon_qadd_s64)(CPUARMState *env, uint64_t src1, uint64_t src2)
72902672
CL		 227{
228 uint64_t res;
229
230 res = src1 + src2;
231 if (((res ^ src1) & SIGNBIT64) && !((src1 ^ src2) & SIGNBIT64)) {
232 SET_QC();
233 res = ((int64_t)src1 >> 63) ^ ~SIGNBIT64;
234 }
235 return res;
236}
237
09e03735
AB		 238/* Unsigned saturating accumulate of signed value
239 *
240 * Op1/Rn is treated as signed
241 * Op2/Rd is treated as unsigned
242 *
243 * Explicit casting is used to ensure the correct sign extension of
244 * inputs. The result is treated as a unsigned value and saturated as such.
245 *
246 * We use a macro for the 8/16 bit cases which expects signed integers of va,
247 * vb, and vr for interim calculation and an unsigned 32 bit result value r.
248 */
249
250#define USATACC(bits, shift) \
251 do { \
252 va = sextract32(a, shift, bits); \
253 vb = extract32(b, shift, bits); \
254 vr = va + vb; \
255 if (vr > UINT##bits##_MAX) { \
256 SET_QC(); \
257 vr = UINT##bits##_MAX; \
258 } else if (vr < 0) { \
259 SET_QC(); \
260 vr = 0; \
261 } \
262 r = deposit32(r, shift, bits, vr); \
263 } while (0)
264
265uint32_t HELPER(neon_uqadd_s8)(CPUARMState *env, uint32_t a, uint32_t b)
266{
267 int16_t va, vb, vr;
268 uint32_t r = 0;
269
270 USATACC(8, 0);
271 USATACC(8, 8);
272 USATACC(8, 16);
273 USATACC(8, 24);
274 return r;
275}
276
277uint32_t HELPER(neon_uqadd_s16)(CPUARMState *env, uint32_t a, uint32_t b)
278{
279 int32_t va, vb, vr;
280 uint64_t r = 0;
281
282 USATACC(16, 0);
283 USATACC(16, 16);
284 return r;
285}
286
287#undef USATACC
288
289uint32_t HELPER(neon_uqadd_s32)(CPUARMState *env, uint32_t a, uint32_t b)
290{
291 int64_t va = (int32_t)a;
292 int64_t vb = (uint32_t)b;
293 int64_t vr = va + vb;
294 if (vr > UINT32_MAX) {
295 SET_QC();
296 vr = UINT32_MAX;
297 } else if (vr < 0) {
298 SET_QC();
299 vr = 0;
300 }
301 return vr;
302}
303
304uint64_t HELPER(neon_uqadd_s64)(CPUARMState *env, uint64_t a, uint64_t b)
305{
306 uint64_t res;
307 res = a + b;
308 /* We only need to look at the pattern of SIGN bits to detect
309 * +ve/-ve saturation
310 */
311 if (~a & b & ~res & SIGNBIT64) {
312 SET_QC();
313 res = UINT64_MAX;
314 } else if (a & ~b & res & SIGNBIT64) {
315 SET_QC();
316 res = 0;
317 }
318 return res;
319}
320
321/* Signed saturating accumulate of unsigned value
322 *
323 * Op1/Rn is treated as unsigned
324 * Op2/Rd is treated as signed
325 *
326 * The result is treated as a signed value and saturated as such
327 *
328 * We use a macro for the 8/16 bit cases which expects signed integers of va,
329 * vb, and vr for interim calculation and an unsigned 32 bit result value r.
330 */
331
332#define SSATACC(bits, shift) \
333 do { \
334 va = extract32(a, shift, bits); \
335 vb = sextract32(b, shift, bits); \
336 vr = va + vb; \
337 if (vr > INT##bits##_MAX) { \
338 SET_QC(); \
339 vr = INT##bits##_MAX; \
340 } else if (vr < INT##bits##_MIN) { \
341 SET_QC(); \
342 vr = INT##bits##_MIN; \
343 } \
344 r = deposit32(r, shift, bits, vr); \
345 } while (0)
346
347uint32_t HELPER(neon_sqadd_u8)(CPUARMState *env, uint32_t a, uint32_t b)
348{
349 int16_t va, vb, vr;
350 uint32_t r = 0;
351
352 SSATACC(8, 0);
353 SSATACC(8, 8);
354 SSATACC(8, 16);
355 SSATACC(8, 24);
356 return r;
357}
358
359uint32_t HELPER(neon_sqadd_u16)(CPUARMState *env, uint32_t a, uint32_t b)
360{
361 int32_t va, vb, vr;
362 uint32_t r = 0;
363
364 SSATACC(16, 0);
365 SSATACC(16, 16);
366
367 return r;
368}
369
370#undef SSATACC
371
372uint32_t HELPER(neon_sqadd_u32)(CPUARMState *env, uint32_t a, uint32_t b)
373{
374 int64_t res;
375 int64_t op1 = (uint32_t)a;
376 int64_t op2 = (int32_t)b;
377 res = op1 + op2;
378 if (res > INT32_MAX) {
379 SET_QC();
380 res = INT32_MAX;
381 } else if (res < INT32_MIN) {
382 SET_QC();
383 res = INT32_MIN;
384 }
385 return res;
386}
387
388uint64_t HELPER(neon_sqadd_u64)(CPUARMState *env, uint64_t a, uint64_t b)
389{
390 uint64_t res;
391 res = a + b;
392 /* We only need to look at the pattern of SIGN bits to detect an overflow */
393 if (((a & res)
394 | (~b & res)
395 | (a & ~b)) & SIGNBIT64) {
396 SET_QC();
397 res = INT64_MAX;
398 }
399 return res;
400}
401
402
ad69471c
PB		 403#define NEON_USAT(dest, src1, src2, type) do { \
404 uint32_t tmp = (uint32_t)src1 - (uint32_t)src2; \
405 if (tmp != (type)tmp) { \
406 SET_QC(); \
407 dest = 0; \
408 } else { \
409 dest = tmp; \
410 }} while(0)
411#define NEON_FN(dest, src1, src2) NEON_USAT(dest, src1, src2, uint8_t)
02da0b2d 412NEON_VOP_ENV(qsub_u8, neon_u8, 4)
ad69471c
PB		 413#undef NEON_FN
414#define NEON_FN(dest, src1, src2) NEON_USAT(dest, src1, src2, uint16_t)
02da0b2d 415NEON_VOP_ENV(qsub_u16, neon_u16, 2)
ad69471c
PB		 416#undef NEON_FN
417#undef NEON_USAT
418
0ecb72a5 419uint32_t HELPER(neon_qsub_u32)(CPUARMState *env, uint32_t a, uint32_t b)
72902672
CL		 420{
421 uint32_t res = a - b;
422 if (res > a) {
423 SET_QC();
424 res = 0;
425 }
426 return res;
427}
428
0ecb72a5 429uint64_t HELPER(neon_qsub_u64)(CPUARMState *env, uint64_t src1, uint64_t src2)
72902672
CL		 430{
431 uint64_t res;
432
433 if (src1 < src2) {
434 SET_QC();
435 res = 0;
436 } else {
437 res = src1 - src2;
438 }
439 return res;
440}
441
ad69471c
PB		 442#define NEON_SSAT(dest, src1, src2, type) do { \
443 int32_t tmp = (uint32_t)src1 - (uint32_t)src2; \
444 if (tmp != (type)tmp) { \
445 SET_QC(); \
446 if (src2 < 0) { \
447 tmp = (1 << (sizeof(type) * 8 - 1)) - 1; \
448 } else { \
449 tmp = 1 << (sizeof(type) * 8 - 1); \
450 } \
451 } \
452 dest = tmp; \
453 } while(0)
454#define NEON_FN(dest, src1, src2) NEON_SSAT(dest, src1, src2, int8_t)
02da0b2d 455NEON_VOP_ENV(qsub_s8, neon_s8, 4)
ad69471c
PB		 456#undef NEON_FN
457#define NEON_FN(dest, src1, src2) NEON_SSAT(dest, src1, src2, int16_t)
02da0b2d 458NEON_VOP_ENV(qsub_s16, neon_s16, 2)
ad69471c
PB		 459#undef NEON_FN
460#undef NEON_SSAT
461
0ecb72a5 462uint32_t HELPER(neon_qsub_s32)(CPUARMState *env, uint32_t a, uint32_t b)
72902672
CL		 463{
464 uint32_t res = a - b;
465 if (((res ^ a) & SIGNBIT) && ((a ^ b) & SIGNBIT)) {
466 SET_QC();
467 res = ~(((int32_t)a >> 31) ^ SIGNBIT);
468 }
469 return res;
470}
471
0ecb72a5 472uint64_t HELPER(neon_qsub_s64)(CPUARMState *env, uint64_t src1, uint64_t src2)
72902672
CL		 473{
474 uint64_t res;
475
476 res = src1 - src2;
477 if (((res ^ src1) & SIGNBIT64) && ((src1 ^ src2) & SIGNBIT64)) {
478 SET_QC();
479 res = ((int64_t)src1 >> 63) ^ ~SIGNBIT64;
480 }
481 return res;
482}
483
ad69471c
PB		 484#define NEON_FN(dest, src1, src2) dest = (src1 + src2) >> 1
485NEON_VOP(hadd_s8, neon_s8, 4)
486NEON_VOP(hadd_u8, neon_u8, 4)
487NEON_VOP(hadd_s16, neon_s16, 2)
488NEON_VOP(hadd_u16, neon_u16, 2)
489#undef NEON_FN
490
491int32_t HELPER(neon_hadd_s32)(int32_t src1, int32_t src2)
492{
493 int32_t dest;
494
495 dest = (src1 >> 1) + (src2 >> 1);
496 if (src1 & src2 & 1)
497 dest++;
498 return dest;
499}
500
501uint32_t HELPER(neon_hadd_u32)(uint32_t src1, uint32_t src2)
502{
503 uint32_t dest;
504
505 dest = (src1 >> 1) + (src2 >> 1);
506 if (src1 & src2 & 1)
507 dest++;
508 return dest;
509}
510
511#define NEON_FN(dest, src1, src2) dest = (src1 + src2 + 1) >> 1
512NEON_VOP(rhadd_s8, neon_s8, 4)
513NEON_VOP(rhadd_u8, neon_u8, 4)
514NEON_VOP(rhadd_s16, neon_s16, 2)
515NEON_VOP(rhadd_u16, neon_u16, 2)
516#undef NEON_FN
517
518int32_t HELPER(neon_rhadd_s32)(int32_t src1, int32_t src2)
519{
520 int32_t dest;
521
522 dest = (src1 >> 1) + (src2 >> 1);
523 if ((src1 | src2) & 1)
524 dest++;
525 return dest;
526}
527
528uint32_t HELPER(neon_rhadd_u32)(uint32_t src1, uint32_t src2)
529{
530 uint32_t dest;
531
532 dest = (src1 >> 1) + (src2 >> 1);
533 if ((src1 | src2) & 1)
534 dest++;
535 return dest;
536}
537
538#define NEON_FN(dest, src1, src2) dest = (src1 - src2) >> 1
539NEON_VOP(hsub_s8, neon_s8, 4)
540NEON_VOP(hsub_u8, neon_u8, 4)
541NEON_VOP(hsub_s16, neon_s16, 2)
542NEON_VOP(hsub_u16, neon_u16, 2)
543#undef NEON_FN
544
545int32_t HELPER(neon_hsub_s32)(int32_t src1, int32_t src2)
546{
547 int32_t dest;
548
549 dest = (src1 >> 1) - (src2 >> 1);
550 if ((~src1) & src2 & 1)
551 dest--;
552 return dest;
553}
554
555uint32_t HELPER(neon_hsub_u32)(uint32_t src1, uint32_t src2)
556{
557 uint32_t dest;
558
559 dest = (src1 >> 1) - (src2 >> 1);
560 if ((~src1) & src2 & 1)
561 dest--;
562 return dest;
563}
564
565#define NEON_FN(dest, src1, src2) dest = (src1 > src2) ? ~0 : 0
566NEON_VOP(cgt_s8, neon_s8, 4)
567NEON_VOP(cgt_u8, neon_u8, 4)
568NEON_VOP(cgt_s16, neon_s16, 2)
569NEON_VOP(cgt_u16, neon_u16, 2)
570NEON_VOP(cgt_s32, neon_s32, 1)
571NEON_VOP(cgt_u32, neon_u32, 1)
572#undef NEON_FN
573
574#define NEON_FN(dest, src1, src2) dest = (src1 >= src2) ? ~0 : 0
575NEON_VOP(cge_s8, neon_s8, 4)
576NEON_VOP(cge_u8, neon_u8, 4)
577NEON_VOP(cge_s16, neon_s16, 2)
578NEON_VOP(cge_u16, neon_u16, 2)
579NEON_VOP(cge_s32, neon_s32, 1)
580NEON_VOP(cge_u32, neon_u32, 1)
581#undef NEON_FN
582
583#define NEON_FN(dest, src1, src2) dest = (src1 < src2) ? src1 : src2
ad69471c
PB		 584NEON_POP(pmin_s8, neon_s8, 4)
585NEON_POP(pmin_u8, neon_u8, 4)
586NEON_POP(pmin_s16, neon_s16, 2)
587NEON_POP(pmin_u16, neon_u16, 2)
588#undef NEON_FN
589
590#define NEON_FN(dest, src1, src2) dest = (src1 > src2) ? src1 : src2
ad69471c
PB		 591NEON_POP(pmax_s8, neon_s8, 4)
592NEON_POP(pmax_u8, neon_u8, 4)
593NEON_POP(pmax_s16, neon_s16, 2)
594NEON_POP(pmax_u16, neon_u16, 2)
595#undef NEON_FN
596
597#define NEON_FN(dest, src1, src2) \
598 dest = (src1 > src2) ? (src1 - src2) : (src2 - src1)
599NEON_VOP(abd_s8, neon_s8, 4)
600NEON_VOP(abd_u8, neon_u8, 4)
601NEON_VOP(abd_s16, neon_s16, 2)
602NEON_VOP(abd_u16, neon_u16, 2)
603NEON_VOP(abd_s32, neon_s32, 1)
604NEON_VOP(abd_u32, neon_u32, 1)
605#undef NEON_FN
606
607#define NEON_FN(dest, src1, src2) do { \
608 int8_t tmp; \
609 tmp = (int8_t)src2; \
50f67e95
JR		 610 if (tmp >= (ssize_t)sizeof(src1) * 8 || \
611 tmp <= -(ssize_t)sizeof(src1) * 8) { \
ad69471c
PB		 612 dest = 0; \
613 } else if (tmp < 0) { \
614 dest = src1 >> -tmp; \
615 } else { \
616 dest = src1 << tmp; \
617 }} while (0)
ad69471c 618NEON_VOP(shl_u16, neon_u16, 2)
ad69471c
PB		 619#undef NEON_FN
620
ad69471c
PB		 621#define NEON_FN(dest, src1, src2) do { \
622 int8_t tmp; \
623 tmp = (int8_t)src2; \
50f67e95 624 if (tmp >= (ssize_t)sizeof(src1) * 8) { \
ad69471c 625 dest = 0; \
50f67e95 626 } else if (tmp <= -(ssize_t)sizeof(src1) * 8) { \
ad69471c
PB		 627 dest = src1 >> (sizeof(src1) * 8 - 1); \
628 } else if (tmp < 0) { \
629 dest = src1 >> -tmp; \
630 } else { \
631 dest = src1 << tmp; \
632 }} while (0)
ad69471c 633NEON_VOP(shl_s16, neon_s16, 2)
ad69471c
PB		 634#undef NEON_FN
635
ad69471c
PB		 636#define NEON_FN(dest, src1, src2) do { \
637 int8_t tmp; \
638 tmp = (int8_t)src2; \
0670a7b6
PM		 639 if ((tmp >= (ssize_t)sizeof(src1) * 8) \
640 || (tmp <= -(ssize_t)sizeof(src1) * 8)) { \
ad69471c 641 dest = 0; \
ad69471c
PB		 642 } else if (tmp < 0) { \
643 dest = (src1 + (1 << (-1 - tmp))) >> -tmp; \
644 } else { \
645 dest = src1 << tmp; \
646 }} while (0)
647NEON_VOP(rshl_s8, neon_s8, 4)
648NEON_VOP(rshl_s16, neon_s16, 2)
ad69471c
PB		 649#undef NEON_FN
650
4bd4ee07 651/* The addition of the rounding constant may overflow, so we use an
b90372ad 652 * intermediate 64 bit accumulator. */
4bd4ee07
CL		 653uint32_t HELPER(neon_rshl_s32)(uint32_t valop, uint32_t shiftop)
654{
655 int32_t dest;
656 int32_t val = (int32_t)valop;
657 int8_t shift = (int8_t)shiftop;
658 if ((shift >= 32) || (shift <= -32)) {
659 dest = 0;
660 } else if (shift < 0) {
661 int64_t big_dest = ((int64_t)val + (1 << (-1 - shift)));
662 dest = big_dest >> -shift;
663 } else {
664 dest = val << shift;
665 }
666 return dest;
667}
668
b90372ad
PM		 669/* Handling addition overflow with 64 bit input values is more
670 * tricky than with 32 bit values. */
ad69471c
PB		 671uint64_t HELPER(neon_rshl_s64)(uint64_t valop, uint64_t shiftop)
672{
673 int8_t shift = (int8_t)shiftop;
674 int64_t val = valop;
0670a7b6 675 if ((shift >= 64) || (shift <= -64)) {
ad69471c 676 val = 0;
ad69471c 677 } else if (shift < 0) {
4bd4ee07
CL		 678 val >>= (-shift - 1);
679 if (val == INT64_MAX) {
680 /* In this case, it means that the rounding constant is 1,
681 * and the addition would overflow. Return the actual
682 * result directly. */
683 val = 0x4000000000000000LL;
684 } else {
685 val++;
686 val >>= 1;
687 }
ad69471c
PB		 688 } else {
689 val <<= shift;
690 }
691 return val;
692}
693
694#define NEON_FN(dest, src1, src2) do { \
695 int8_t tmp; \
696 tmp = (int8_t)src2; \
50f67e95
JR		 697 if (tmp >= (ssize_t)sizeof(src1) * 8 || \
698 tmp < -(ssize_t)sizeof(src1) * 8) { \
ad69471c 699 dest = 0; \
50f67e95 700 } else if (tmp == -(ssize_t)sizeof(src1) * 8) { \
b6c63b98 701 dest = src1 >> (-tmp - 1); \
ad69471c
PB		 702 } else if (tmp < 0) { \
703 dest = (src1 + (1 << (-1 - tmp))) >> -tmp; \
704 } else { \
705 dest = src1 << tmp; \
706 }} while (0)
707NEON_VOP(rshl_u8, neon_u8, 4)
708NEON_VOP(rshl_u16, neon_u16, 2)
ad69471c
PB		 709#undef NEON_FN
710
4bd4ee07 711/* The addition of the rounding constant may overflow, so we use an
b90372ad 712 * intermediate 64 bit accumulator. */
4bd4ee07
CL		 713uint32_t HELPER(neon_rshl_u32)(uint32_t val, uint32_t shiftop)
714{
715 uint32_t dest;
716 int8_t shift = (int8_t)shiftop;
717 if (shift >= 32 || shift < -32) {
718 dest = 0;
719 } else if (shift == -32) {
720 dest = val >> 31;
721 } else if (shift < 0) {
722 uint64_t big_dest = ((uint64_t)val + (1 << (-1 - shift)));
723 dest = big_dest >> -shift;
724 } else {
725 dest = val << shift;
726 }
727 return dest;
728}
729
b90372ad
PM		 730/* Handling addition overflow with 64 bit input values is more
731 * tricky than with 32 bit values. */
ad69471c
PB		 732uint64_t HELPER(neon_rshl_u64)(uint64_t val, uint64_t shiftop)
733{
734 int8_t shift = (uint8_t)shiftop;
51e3930f 735 if (shift >= 64 || shift < -64) {
ad69471c
PB		 736 val = 0;
737 } else if (shift == -64) {
738 /* Rounding a 1-bit result just preserves that bit. */
739 val >>= 63;
4bd4ee07
CL		 740 } else if (shift < 0) {
741 val >>= (-shift - 1);
742 if (val == UINT64_MAX) {
743 /* In this case, it means that the rounding constant is 1,
744 * and the addition would overflow. Return the actual
745 * result directly. */
746 val = 0x8000000000000000ULL;
747 } else {
748 val++;
749 val >>= 1;
750 }
ad69471c
PB		 751 } else {
752 val <<= shift;
753 }
754 return val;
755}
756
757#define NEON_FN(dest, src1, src2) do { \
758 int8_t tmp; \
759 tmp = (int8_t)src2; \
50f67e95 760 if (tmp >= (ssize_t)sizeof(src1) * 8) { \
ad69471c
PB		 761 if (src1) { \
762 SET_QC(); \
763 dest = ~0; \
764 } else { \
765 dest = 0; \
766 } \
50f67e95 767 } else if (tmp <= -(ssize_t)sizeof(src1) * 8) { \
ad69471c
PB		 768 dest = 0; \
769 } else if (tmp < 0) { \
770 dest = src1 >> -tmp; \
771 } else { \
772 dest = src1 << tmp; \
773 if ((dest >> tmp) != src1) { \
774 SET_QC(); \
775 dest = ~0; \
776 } \
777 }} while (0)
02da0b2d
PM		 778NEON_VOP_ENV(qshl_u8, neon_u8, 4)
779NEON_VOP_ENV(qshl_u16, neon_u16, 2)
780NEON_VOP_ENV(qshl_u32, neon_u32, 1)
ad69471c
PB		 781#undef NEON_FN
782
0ecb72a5 783uint64_t HELPER(neon_qshl_u64)(CPUARMState *env, uint64_t val, uint64_t shiftop)
ad69471c
PB		 784{
785 int8_t shift = (int8_t)shiftop;
786 if (shift >= 64) {
787 if (val) {
788 val = ~(uint64_t)0;
789 SET_QC();
ad69471c
PB		 790 }
791 } else if (shift <= -64) {
792 val = 0;
793 } else if (shift < 0) {
794 val >>= -shift;
795 } else {
796 uint64_t tmp = val;
797 val <<= shift;
798 if ((val >> shift) != tmp) {
799 SET_QC();
800 val = ~(uint64_t)0;
801 }
802 }
803 return val;
804}
805
806#define NEON_FN(dest, src1, src2) do { \
807 int8_t tmp; \
808 tmp = (int8_t)src2; \
50f67e95 809 if (tmp >= (ssize_t)sizeof(src1) * 8) { \
a5d88f3e 810 if (src1) { \
ad69471c 811 SET_QC(); \
a5d88f3e
PM		 812 dest = (uint32_t)(1 << (sizeof(src1) * 8 - 1)); \
813 if (src1 > 0) { \
814 dest--; \
815 } \
816 } else { \
817 dest = src1; \
818 } \
50f67e95 819 } else if (tmp <= -(ssize_t)sizeof(src1) * 8) { \
ad69471c
PB		 820 dest = src1 >> 31; \
821 } else if (tmp < 0) { \
822 dest = src1 >> -tmp; \
823 } else { \
824 dest = src1 << tmp; \
825 if ((dest >> tmp) != src1) { \
826 SET_QC(); \
a5d88f3e
PM		 827 dest = (uint32_t)(1 << (sizeof(src1) * 8 - 1)); \
828 if (src1 > 0) { \
829 dest--; \
830 } \
ad69471c
PB		 831 } \
832 }} while (0)
02da0b2d
PM		 833NEON_VOP_ENV(qshl_s8, neon_s8, 4)
834NEON_VOP_ENV(qshl_s16, neon_s16, 2)
835NEON_VOP_ENV(qshl_s32, neon_s32, 1)
ad69471c
PB		 836#undef NEON_FN
837
0ecb72a5 838uint64_t HELPER(neon_qshl_s64)(CPUARMState *env, uint64_t valop, uint64_t shiftop)
ad69471c
PB		 839{
840 int8_t shift = (uint8_t)shiftop;
841 int64_t val = valop;
842 if (shift >= 64) {
843 if (val) {
844 SET_QC();
eb7a3d79 845 val = (val >> 63) ^ ~SIGNBIT64;
ad69471c 846 }
4c9b70ae 847 } else if (shift <= -64) {
ad69471c
PB		 848 val >>= 63;
849 } else if (shift < 0) {
850 val >>= -shift;
851 } else {
852 int64_t tmp = val;
853 val <<= shift;
854 if ((val >> shift) != tmp) {
855 SET_QC();
856 val = (tmp >> 63) ^ ~SIGNBIT64;
857 }
858 }
859 return val;
860}
861
4ca4502c
JR		 862#define NEON_FN(dest, src1, src2) do { \
863 if (src1 & (1 << (sizeof(src1) * 8 - 1))) { \
864 SET_QC(); \
865 dest = 0; \
866 } else { \
867 int8_t tmp; \
868 tmp = (int8_t)src2; \
869 if (tmp >= (ssize_t)sizeof(src1) * 8) { \
870 if (src1) { \
871 SET_QC(); \
872 dest = ~0; \
873 } else { \
874 dest = 0; \
875 } \
876 } else if (tmp <= -(ssize_t)sizeof(src1) * 8) { \
877 dest = 0; \
878 } else if (tmp < 0) { \
879 dest = src1 >> -tmp; \
880 } else { \
881 dest = src1 << tmp; \
882 if ((dest >> tmp) != src1) { \
883 SET_QC(); \
884 dest = ~0; \
885 } \
886 } \
887 }} while (0)
02da0b2d
PM		 888NEON_VOP_ENV(qshlu_s8, neon_u8, 4)
889NEON_VOP_ENV(qshlu_s16, neon_u16, 2)
4ca4502c
JR		 890#undef NEON_FN
891
0ecb72a5 892uint32_t HELPER(neon_qshlu_s32)(CPUARMState *env, uint32_t valop, uint32_t shiftop)
4ca4502c
JR		 893{
894 if ((int32_t)valop < 0) {
895 SET_QC();
896 return 0;
897 }
02da0b2d 898 return helper_neon_qshl_u32(env, valop, shiftop);
4ca4502c
JR		 899}
900
0ecb72a5 901uint64_t HELPER(neon_qshlu_s64)(CPUARMState *env, uint64_t valop, uint64_t shiftop)
4ca4502c
JR		 902{
903 if ((int64_t)valop < 0) {
904 SET_QC();
905 return 0;
906 }
02da0b2d 907 return helper_neon_qshl_u64(env, valop, shiftop);
4ca4502c 908}
ad69471c 909
ad69471c
PB		 910#define NEON_FN(dest, src1, src2) do { \
911 int8_t tmp; \
912 tmp = (int8_t)src2; \
33ebc293
PM		 913 if (tmp >= (ssize_t)sizeof(src1) * 8) { \
914 if (src1) { \
915 SET_QC(); \
916 dest = ~0; \
917 } else { \
918 dest = 0; \
919 } \
920 } else if (tmp < -(ssize_t)sizeof(src1) * 8) { \
921 dest = 0; \
922 } else if (tmp == -(ssize_t)sizeof(src1) * 8) { \
923 dest = src1 >> (sizeof(src1) * 8 - 1); \
924 } else if (tmp < 0) { \
ad69471c
PB		 925 dest = (src1 + (1 << (-1 - tmp))) >> -tmp; \
926 } else { \
927 dest = src1 << tmp; \
928 if ((dest >> tmp) != src1) { \
929 SET_QC(); \
930 dest = ~0; \
931 } \
932 }} while (0)
02da0b2d
PM		 933NEON_VOP_ENV(qrshl_u8, neon_u8, 4)
934NEON_VOP_ENV(qrshl_u16, neon_u16, 2)
ad69471c
PB		 935#undef NEON_FN
936
4bd4ee07 937/* The addition of the rounding constant may overflow, so we use an
b90372ad 938 * intermediate 64 bit accumulator. */
0ecb72a5 939uint32_t HELPER(neon_qrshl_u32)(CPUARMState *env, uint32_t val, uint32_t shiftop)
4bd4ee07
CL		 940{
941 uint32_t dest;
942 int8_t shift = (int8_t)shiftop;
33ebc293
PM		 943 if (shift >= 32) {
944 if (val) {
945 SET_QC();
946 dest = ~0;
947 } else {
948 dest = 0;
949 }
950 } else if (shift < -32) {
951 dest = 0;
952 } else if (shift == -32) {
953 dest = val >> 31;
954 } else if (shift < 0) {
4bd4ee07
CL		 955 uint64_t big_dest = ((uint64_t)val + (1 << (-1 - shift)));
956 dest = big_dest >> -shift;
957 } else {
958 dest = val << shift;
959 if ((dest >> shift) != val) {
960 SET_QC();
961 dest = ~0;
962 }
963 }
964 return dest;
965}
966
b90372ad
PM		 967/* Handling addition overflow with 64 bit input values is more
968 * tricky than with 32 bit values. */
0ecb72a5 969uint64_t HELPER(neon_qrshl_u64)(CPUARMState *env, uint64_t val, uint64_t shiftop)
ad69471c
PB		 970{
971 int8_t shift = (int8_t)shiftop;
33ebc293
PM		 972 if (shift >= 64) {
973 if (val) {
974 SET_QC();
975 val = ~0;
976 }
977 } else if (shift < -64) {
978 val = 0;
979 } else if (shift == -64) {
980 val >>= 63;
981 } else if (shift < 0) {
4bd4ee07
CL		 982 val >>= (-shift - 1);
983 if (val == UINT64_MAX) {
984 /* In this case, it means that the rounding constant is 1,
985 * and the addition would overflow. Return the actual
986 * result directly. */
987 val = 0x8000000000000000ULL;
988 } else {
989 val++;
990 val >>= 1;
991 }
ad69471c
PB		 992 } else { \
993 uint64_t tmp = val;
994 val <<= shift;
995 if ((val >> shift) != tmp) {
996 SET_QC();
997 val = ~0;
998 }
999 }
1000 return val;
1001}
1002
1003#define NEON_FN(dest, src1, src2) do { \
1004 int8_t tmp; \
1005 tmp = (int8_t)src2; \
7b6ecf5b
PM		 1006 if (tmp >= (ssize_t)sizeof(src1) * 8) { \
1007 if (src1) { \
1008 SET_QC(); \
6bbbb0ac 1009 dest = (typeof(dest))(1 << (sizeof(src1) * 8 - 1)); \
7b6ecf5b
PM		 1010 if (src1 > 0) { \
1011 dest--; \
1012 } \
1013 } else { \
1014 dest = 0; \
1015 } \
1016 } else if (tmp <= -(ssize_t)sizeof(src1) * 8) { \
1017 dest = 0; \
1018 } else if (tmp < 0) { \
ad69471c
PB		 1019 dest = (src1 + (1 << (-1 - tmp))) >> -tmp; \
1020 } else { \
1021 dest = src1 << tmp; \
1022 if ((dest >> tmp) != src1) { \
1023 SET_QC(); \
960e623b
PM		 1024 dest = (uint32_t)(1 << (sizeof(src1) * 8 - 1)); \
1025 if (src1 > 0) { \
1026 dest--; \
1027 } \
ad69471c
PB		 1028 } \
1029 }} while (0)
02da0b2d
PM		 1030NEON_VOP_ENV(qrshl_s8, neon_s8, 4)
1031NEON_VOP_ENV(qrshl_s16, neon_s16, 2)
ad69471c
PB		 1032#undef NEON_FN
1033
4bd4ee07 1034/* The addition of the rounding constant may overflow, so we use an
b90372ad 1035 * intermediate 64 bit accumulator. */
0ecb72a5 1036uint32_t HELPER(neon_qrshl_s32)(CPUARMState *env, uint32_t valop, uint32_t shiftop)
4bd4ee07
CL		 1037{
1038 int32_t dest;
1039 int32_t val = (int32_t)valop;
1040 int8_t shift = (int8_t)shiftop;
7b6ecf5b
PM		 1041 if (shift >= 32) {
1042 if (val) {
1043 SET_QC();
1044 dest = (val >> 31) ^ ~SIGNBIT;
1045 } else {
1046 dest = 0;
1047 }
1048 } else if (shift <= -32) {
1049 dest = 0;
1050 } else if (shift < 0) {
4bd4ee07
CL		 1051 int64_t big_dest = ((int64_t)val + (1 << (-1 - shift)));
1052 dest = big_dest >> -shift;
1053 } else {
1054 dest = val << shift;
1055 if ((dest >> shift) != val) {
1056 SET_QC();
1057 dest = (val >> 31) ^ ~SIGNBIT;
1058 }
1059 }
1060 return dest;
1061}
1062
b90372ad
PM		 1063/* Handling addition overflow with 64 bit input values is more
1064 * tricky than with 32 bit values. */
0ecb72a5 1065uint64_t HELPER(neon_qrshl_s64)(CPUARMState *env, uint64_t valop, uint64_t shiftop)
ad69471c
PB		 1066{
1067 int8_t shift = (uint8_t)shiftop;
1068 int64_t val = valop;
1069
7b6ecf5b
PM		 1070 if (shift >= 64) {
1071 if (val) {
1072 SET_QC();
1073 val = (val >> 63) ^ ~SIGNBIT64;
1074 }
1075 } else if (shift <= -64) {
1076 val = 0;
1077 } else if (shift < 0) {
4bd4ee07
CL		 1078 val >>= (-shift - 1);
1079 if (val == INT64_MAX) {
1080 /* In this case, it means that the rounding constant is 1,
1081 * and the addition would overflow. Return the actual
1082 * result directly. */
1083 val = 0x4000000000000000ULL;
1084 } else {
1085 val++;
1086 val >>= 1;
1087 }
ad69471c 1088 } else {
4bd4ee07 1089 int64_t tmp = val;
ad69471c
PB		 1090 val <<= shift;
1091 if ((val >> shift) != tmp) {
1092 SET_QC();
4bd4ee07 1093 val = (tmp >> 63) ^ ~SIGNBIT64;
ad69471c
PB		 1094 }
1095 }
1096 return val;
1097}
1098
1099uint32_t HELPER(neon_add_u8)(uint32_t a, uint32_t b)
1100{
1101 uint32_t mask;
1102 mask = (a ^ b) & 0x80808080u;
1103 a &= ~0x80808080u;
1104 b &= ~0x80808080u;
1105 return (a + b) ^ mask;
1106}
1107
1108uint32_t HELPER(neon_add_u16)(uint32_t a, uint32_t b)
1109{
1110 uint32_t mask;
1111 mask = (a ^ b) & 0x80008000u;
1112 a &= ~0x80008000u;
1113 b &= ~0x80008000u;
1114 return (a + b) ^ mask;
1115}
1116
1117#define NEON_FN(dest, src1, src2) dest = src1 + src2
1118NEON_POP(padd_u8, neon_u8, 4)
1119NEON_POP(padd_u16, neon_u16, 2)
1120#undef NEON_FN
1121
1122#define NEON_FN(dest, src1, src2) dest = src1 - src2
1123NEON_VOP(sub_u8, neon_u8, 4)
1124NEON_VOP(sub_u16, neon_u16, 2)
1125#undef NEON_FN
1126
1127#define NEON_FN(dest, src1, src2) dest = src1 * src2
1128NEON_VOP(mul_u8, neon_u8, 4)
1129NEON_VOP(mul_u16, neon_u16, 2)
1130#undef NEON_FN
1131
ad69471c
PB		 1132#define NEON_FN(dest, src1, src2) dest = (src1 & src2) ? -1 : 0
1133NEON_VOP(tst_u8, neon_u8, 4)
1134NEON_VOP(tst_u16, neon_u16, 2)
1135NEON_VOP(tst_u32, neon_u32, 1)
1136#undef NEON_FN
1137
1138#define NEON_FN(dest, src1, src2) dest = (src1 == src2) ? -1 : 0
1139NEON_VOP(ceq_u8, neon_u8, 4)
1140NEON_VOP(ceq_u16, neon_u16, 2)
1141NEON_VOP(ceq_u32, neon_u32, 1)
1142#undef NEON_FN
1143
ad69471c
PB		 1144/* Count Leading Sign/Zero Bits. */
1145static inline int do_clz8(uint8_t x)
1146{
1147 int n;
1148 for (n = 8; x; n--)
1149 x >>= 1;
1150 return n;
1151}
1152
1153static inline int do_clz16(uint16_t x)
1154{
1155 int n;
1156 for (n = 16; x; n--)
1157 x >>= 1;
1158 return n;
1159}
1160
1161#define NEON_FN(dest, src, dummy) dest = do_clz8(src)
1162NEON_VOP1(clz_u8, neon_u8, 4)
1163#undef NEON_FN
1164
1165#define NEON_FN(dest, src, dummy) dest = do_clz16(src)
1166NEON_VOP1(clz_u16, neon_u16, 2)
1167#undef NEON_FN
1168
1169#define NEON_FN(dest, src, dummy) dest = do_clz8((src < 0) ? ~src : src) - 1
1170NEON_VOP1(cls_s8, neon_s8, 4)
1171#undef NEON_FN
1172
1173#define NEON_FN(dest, src, dummy) dest = do_clz16((src < 0) ? ~src : src) - 1
1174NEON_VOP1(cls_s16, neon_s16, 2)
1175#undef NEON_FN
1176
1177uint32_t HELPER(neon_cls_s32)(uint32_t x)
1178{
1179 int count;
1180 if ((int32_t)x < 0)
1181 x = ~x;
1182 for (count = 32; x; count--)
1183 x = x >> 1;
1184 return count - 1;
1185}
1186
1187/* Bit count. */
1188uint32_t HELPER(neon_cnt_u8)(uint32_t x)
1189{
1190 x = (x & 0x55555555) + ((x >> 1) & 0x55555555);
1191 x = (x & 0x33333333) + ((x >> 2) & 0x33333333);
1192 x = (x & 0x0f0f0f0f) + ((x >> 4) & 0x0f0f0f0f);
1193 return x;
1194}
1195
86cbc418
PM		 1196/* Reverse bits in each 8 bit word */
1197uint32_t HELPER(neon_rbit_u8)(uint32_t x)
1198{
1199 x = ((x & 0xf0f0f0f0) >> 4)
1200 | ((x & 0x0f0f0f0f) << 4);
1201 x = ((x & 0x88888888) >> 3)
1202 | ((x & 0x44444444) >> 1)
1203 | ((x & 0x22222222) << 1)
1204 | ((x & 0x11111111) << 3);
1205 return x;
1206}
1207
ad69471c
PB		 1208#define NEON_QDMULH16(dest, src1, src2, round) do { \
1209 uint32_t tmp = (int32_t)(int16_t) src1 * (int16_t) src2; \
1210 if ((tmp ^ (tmp << 1)) & SIGNBIT) { \
1211 SET_QC(); \
1212 tmp = (tmp >> 31) ^ ~SIGNBIT; \
46eece9d
JR		 1213 } else { \
1214 tmp <<= 1; \
ad69471c 1215 } \
ad69471c
PB		 1216 if (round) { \
1217 int32_t old = tmp; \
1218 tmp += 1 << 15; \
1219 if ((int32_t)tmp < old) { \
1220 SET_QC(); \
1221 tmp = SIGNBIT - 1; \
1222 } \
1223 } \
1224 dest = tmp >> 16; \
1225 } while(0)
1226#define NEON_FN(dest, src1, src2) NEON_QDMULH16(dest, src1, src2, 0)
02da0b2d 1227NEON_VOP_ENV(qdmulh_s16, neon_s16, 2)
ad69471c
PB		 1228#undef NEON_FN
1229#define NEON_FN(dest, src1, src2) NEON_QDMULH16(dest, src1, src2, 1)
02da0b2d 1230NEON_VOP_ENV(qrdmulh_s16, neon_s16, 2)
ad69471c
PB		 1231#undef NEON_FN
1232#undef NEON_QDMULH16
1233
1234#define NEON_QDMULH32(dest, src1, src2, round) do { \
1235 uint64_t tmp = (int64_t)(int32_t) src1 * (int32_t) src2; \
1236 if ((tmp ^ (tmp << 1)) & SIGNBIT64) { \
1237 SET_QC(); \
1238 tmp = (tmp >> 63) ^ ~SIGNBIT64; \
1239 } else { \
1240 tmp <<= 1; \
1241 } \
1242 if (round) { \
1243 int64_t old = tmp; \
1244 tmp += (int64_t)1 << 31; \
1245 if ((int64_t)tmp < old) { \
1246 SET_QC(); \
1247 tmp = SIGNBIT64 - 1; \
1248 } \
1249 } \
1250 dest = tmp >> 32; \
1251 } while(0)
1252#define NEON_FN(dest, src1, src2) NEON_QDMULH32(dest, src1, src2, 0)
02da0b2d 1253NEON_VOP_ENV(qdmulh_s32, neon_s32, 1)
ad69471c
PB		 1254#undef NEON_FN
1255#define NEON_FN(dest, src1, src2) NEON_QDMULH32(dest, src1, src2, 1)
02da0b2d 1256NEON_VOP_ENV(qrdmulh_s32, neon_s32, 1)
ad69471c
PB		 1257#undef NEON_FN
1258#undef NEON_QDMULH32
1259
1260uint32_t HELPER(neon_narrow_u8)(uint64_t x)
1261{
1262 return (x & 0xffu) | ((x >> 8) & 0xff00u) | ((x >> 16) & 0xff0000u)
1263 | ((x >> 24) & 0xff000000u);
1264}
1265
1266uint32_t HELPER(neon_narrow_u16)(uint64_t x)
1267{
1268 return (x & 0xffffu) | ((x >> 16) & 0xffff0000u);
1269}
1270
1271uint32_t HELPER(neon_narrow_high_u8)(uint64_t x)
1272{
1273 return ((x >> 8) & 0xff) | ((x >> 16) & 0xff00)
1274 | ((x >> 24) & 0xff0000) | ((x >> 32) & 0xff000000);
1275}
1276
1277uint32_t HELPER(neon_narrow_high_u16)(uint64_t x)
1278{
1279 return ((x >> 16) & 0xffff) | ((x >> 32) & 0xffff0000);
1280}
1281
1282uint32_t HELPER(neon_narrow_round_high_u8)(uint64_t x)
1283{
1284 x &= 0xff80ff80ff80ff80ull;
1285 x += 0x0080008000800080ull;
1286 return ((x >> 8) & 0xff) | ((x >> 16) & 0xff00)
1287 | ((x >> 24) & 0xff0000) | ((x >> 32) & 0xff000000);
1288}
1289
1290uint32_t HELPER(neon_narrow_round_high_u16)(uint64_t x)
1291{
1292 x &= 0xffff8000ffff8000ull;
1293 x += 0x0000800000008000ull;
1294 return ((x >> 16) & 0xffff) | ((x >> 32) & 0xffff0000);
1295}
1296
0ecb72a5 1297uint32_t HELPER(neon_unarrow_sat8)(CPUARMState *env, uint64_t x)
af1bbf30
JR		 1298{
1299 uint16_t s;
1300 uint8_t d;
1301 uint32_t res = 0;
1302#define SAT8(n) \
1303 s = x >> n; \
1304 if (s & 0x8000) { \
1305 SET_QC(); \
1306 } else { \
1307 if (s > 0xff) { \
1308 d = 0xff; \
1309 SET_QC(); \
1310 } else { \
1311 d = s; \
1312 } \
1313 res |= (uint32_t)d << (n / 2); \
1314 }
1315
1316 SAT8(0);
1317 SAT8(16);
1318 SAT8(32);
1319 SAT8(48);
1320#undef SAT8
1321 return res;
1322}
1323
0ecb72a5 1324uint32_t HELPER(neon_narrow_sat_u8)(CPUARMState *env, uint64_t x)
ad69471c
PB		 1325{
1326 uint16_t s;
1327 uint8_t d;
1328 uint32_t res = 0;
1329#define SAT8(n) \
1330 s = x >> n; \
1331 if (s > 0xff) { \
1332 d = 0xff; \
1333 SET_QC(); \
1334 } else { \
1335 d = s; \
1336 } \
1337 res |= (uint32_t)d << (n / 2);
1338
1339 SAT8(0);
1340 SAT8(16);
1341 SAT8(32);
1342 SAT8(48);
1343#undef SAT8
1344 return res;
1345}
1346
0ecb72a5 1347uint32_t HELPER(neon_narrow_sat_s8)(CPUARMState *env, uint64_t x)
ad69471c
PB		 1348{
1349 int16_t s;
1350 uint8_t d;
1351 uint32_t res = 0;
1352#define SAT8(n) \
1353 s = x >> n; \
1354 if (s != (int8_t)s) { \
1355 d = (s >> 15) ^ 0x7f; \
1356 SET_QC(); \
1357 } else { \
1358 d = s; \
1359 } \
1360 res |= (uint32_t)d << (n / 2);
1361
1362 SAT8(0);
1363 SAT8(16);
1364 SAT8(32);
1365 SAT8(48);
1366#undef SAT8
1367 return res;
1368}
1369
0ecb72a5 1370uint32_t HELPER(neon_unarrow_sat16)(CPUARMState *env, uint64_t x)
af1bbf30
JR		 1371{
1372 uint32_t high;
1373 uint32_t low;
1374 low = x;
1375 if (low & 0x80000000) {
1376 low = 0;
1377 SET_QC();
1378 } else if (low > 0xffff) {
1379 low = 0xffff;
1380 SET_QC();
1381 }
1382 high = x >> 32;
1383 if (high & 0x80000000) {
1384 high = 0;
1385 SET_QC();
1386 } else if (high > 0xffff) {
1387 high = 0xffff;
1388 SET_QC();
1389 }
1390 return low | (high << 16);
1391}
1392
0ecb72a5 1393uint32_t HELPER(neon_narrow_sat_u16)(CPUARMState *env, uint64_t x)
ad69471c
PB		 1394{
1395 uint32_t high;
1396 uint32_t low;
1397 low = x;
1398 if (low > 0xffff) {
1399 low = 0xffff;
1400 SET_QC();
1401 }
1402 high = x >> 32;
1403 if (high > 0xffff) {
1404 high = 0xffff;
1405 SET_QC();
1406 }
1407 return low | (high << 16);
1408}
1409
0ecb72a5 1410uint32_t HELPER(neon_narrow_sat_s16)(CPUARMState *env, uint64_t x)
ad69471c
PB		 1411{
1412 int32_t low;
1413 int32_t high;
1414 low = x;
1415 if (low != (int16_t)low) {
1416 low = (low >> 31) ^ 0x7fff;
1417 SET_QC();
1418 }
1419 high = x >> 32;
1420 if (high != (int16_t)high) {
1421 high = (high >> 31) ^ 0x7fff;
1422 SET_QC();
1423 }
1424 return (uint16_t)low | (high << 16);
1425}
1426
0ecb72a5 1427uint32_t HELPER(neon_unarrow_sat32)(CPUARMState *env, uint64_t x)
af1bbf30
JR		 1428{
1429 if (x & 0x8000000000000000ull) {
1430 SET_QC();
1431 return 0;
1432 }
1433 if (x > 0xffffffffu) {
1434 SET_QC();
1435 return 0xffffffffu;
1436 }
1437 return x;
1438}
1439
0ecb72a5 1440uint32_t HELPER(neon_narrow_sat_u32)(CPUARMState *env, uint64_t x)
ad69471c
PB		 1441{
1442 if (x > 0xffffffffu) {
1443 SET_QC();
1444 return 0xffffffffu;
1445 }
1446 return x;
1447}
1448
0ecb72a5 1449uint32_t HELPER(neon_narrow_sat_s32)(CPUARMState *env, uint64_t x)
ad69471c
PB		 1450{
1451 if ((int64_t)x != (int32_t)x) {
1452 SET_QC();
cc2212c2 1453 return ((int64_t)x >> 63) ^ 0x7fffffff;
ad69471c
PB		 1454 }
1455 return x;
1456}
1457
1458uint64_t HELPER(neon_widen_u8)(uint32_t x)
1459{
1460 uint64_t tmp;
1461 uint64_t ret;
1462 ret = (uint8_t)x;
1463 tmp = (uint8_t)(x >> 8);
1464 ret |= tmp << 16;
1465 tmp = (uint8_t)(x >> 16);
1466 ret |= tmp << 32;
1467 tmp = (uint8_t)(x >> 24);
1468 ret |= tmp << 48;
1469 return ret;
1470}
1471
1472uint64_t HELPER(neon_widen_s8)(uint32_t x)
1473{
1474 uint64_t tmp;
1475 uint64_t ret;
1476 ret = (uint16_t)(int8_t)x;
1477 tmp = (uint16_t)(int8_t)(x >> 8);
1478 ret |= tmp << 16;
1479 tmp = (uint16_t)(int8_t)(x >> 16);
1480 ret |= tmp << 32;
1481 tmp = (uint16_t)(int8_t)(x >> 24);
1482 ret |= tmp << 48;
1483 return ret;
1484}
1485
1486uint64_t HELPER(neon_widen_u16)(uint32_t x)
1487{
1488 uint64_t high = (uint16_t)(x >> 16);
1489 return ((uint16_t)x) | (high << 32);
1490}
1491
1492uint64_t HELPER(neon_widen_s16)(uint32_t x)
1493{
1494 uint64_t high = (int16_t)(x >> 16);
1495 return ((uint32_t)(int16_t)x) | (high << 32);
1496}
1497
1498uint64_t HELPER(neon_addl_u16)(uint64_t a, uint64_t b)
1499{
1500 uint64_t mask;
1501 mask = (a ^ b) & 0x8000800080008000ull;
1502 a &= ~0x8000800080008000ull;
1503 b &= ~0x8000800080008000ull;
1504 return (a + b) ^ mask;
1505}
1506
1507uint64_t HELPER(neon_addl_u32)(uint64_t a, uint64_t b)
1508{
1509 uint64_t mask;
1510 mask = (a ^ b) & 0x8000000080000000ull;
1511 a &= ~0x8000000080000000ull;
1512 b &= ~0x8000000080000000ull;
1513 return (a + b) ^ mask;
1514}
1515
1516uint64_t HELPER(neon_paddl_u16)(uint64_t a, uint64_t b)
1517{
1518 uint64_t tmp;
1519 uint64_t tmp2;
1520
1521 tmp = a & 0x0000ffff0000ffffull;
1522 tmp += (a >> 16) & 0x0000ffff0000ffffull;
1523 tmp2 = b & 0xffff0000ffff0000ull;
1524 tmp2 += (b << 16) & 0xffff0000ffff0000ull;
1525 return ( tmp & 0xffff)
1526 | ((tmp >> 16) & 0xffff0000ull)
1527 | ((tmp2 << 16) & 0xffff00000000ull)
1528 | ( tmp2 & 0xffff000000000000ull);
1529}
1530
1531uint64_t HELPER(neon_paddl_u32)(uint64_t a, uint64_t b)
1532{
1533 uint32_t low = a + (a >> 32);
1534 uint32_t high = b + (b >> 32);
1535 return low + ((uint64_t)high << 32);
1536}
1537
1538uint64_t HELPER(neon_subl_u16)(uint64_t a, uint64_t b)
1539{
1540 uint64_t mask;
1541 mask = (a ^ ~b) & 0x8000800080008000ull;
1542 a |= 0x8000800080008000ull;
1543 b &= ~0x8000800080008000ull;
1544 return (a - b) ^ mask;
1545}
1546
1547uint64_t HELPER(neon_subl_u32)(uint64_t a, uint64_t b)
1548{
1549 uint64_t mask;
1550 mask = (a ^ ~b) & 0x8000000080000000ull;
1551 a |= 0x8000000080000000ull;
1552 b &= ~0x8000000080000000ull;
1553 return (a - b) ^ mask;
1554}
1555
0ecb72a5 1556uint64_t HELPER(neon_addl_saturate_s32)(CPUARMState *env, uint64_t a, uint64_t b)
ad69471c
PB		 1557{
1558 uint32_t x, y;
1559 uint32_t low, high;
1560
1561 x = a;
1562 y = b;
1563 low = x + y;
1564 if (((low ^ x) & SIGNBIT) && !((x ^ y) & SIGNBIT)) {
1565 SET_QC();
1566 low = ((int32_t)x >> 31) ^ ~SIGNBIT;
1567 }
1568 x = a >> 32;
1569 y = b >> 32;
1570 high = x + y;
1571 if (((high ^ x) & SIGNBIT) && !((x ^ y) & SIGNBIT)) {
1572 SET_QC();
1573 high = ((int32_t)x >> 31) ^ ~SIGNBIT;
1574 }
1575 return low | ((uint64_t)high << 32);
1576}
1577
0ecb72a5 1578uint64_t HELPER(neon_addl_saturate_s64)(CPUARMState *env, uint64_t a, uint64_t b)
ad69471c
PB		 1579{
1580 uint64_t result;
1581
1582 result = a + b;
1583 if (((result ^ a) & SIGNBIT64) && !((a ^ b) & SIGNBIT64)) {
1584 SET_QC();
1585 result = ((int64_t)a >> 63) ^ ~SIGNBIT64;
1586 }
1587 return result;
1588}
1589
4d9ad7f7
PM		 1590/* We have to do the arithmetic in a larger type than
1591 * the input type, because for example with a signed 32 bit
1592 * op the absolute difference can overflow a signed 32 bit value.
1593 */
1594#define DO_ABD(dest, x, y, intype, arithtype) do { \
1595 arithtype tmp_x = (intype)(x); \
1596 arithtype tmp_y = (intype)(y); \
ad69471c
PB		 1597 dest = ((tmp_x > tmp_y) ? tmp_x - tmp_y : tmp_y - tmp_x); \
1598 } while(0)
1599
1600uint64_t HELPER(neon_abdl_u16)(uint32_t a, uint32_t b)
1601{
1602 uint64_t tmp;
1603 uint64_t result;
4d9ad7f7
PM		 1604 DO_ABD(result, a, b, uint8_t, uint32_t);
1605 DO_ABD(tmp, a >> 8, b >> 8, uint8_t, uint32_t);
ad69471c 1606 result |= tmp << 16;
4d9ad7f7 1607 DO_ABD(tmp, a >> 16, b >> 16, uint8_t, uint32_t);
ad69471c 1608 result |= tmp << 32;
4d9ad7f7 1609 DO_ABD(tmp, a >> 24, b >> 24, uint8_t, uint32_t);
ad69471c
PB		 1610 result |= tmp << 48;
1611 return result;
1612}
1613
1614uint64_t HELPER(neon_abdl_s16)(uint32_t a, uint32_t b)
1615{
1616 uint64_t tmp;
1617 uint64_t result;
4d9ad7f7
PM		 1618 DO_ABD(result, a, b, int8_t, int32_t);
1619 DO_ABD(tmp, a >> 8, b >> 8, int8_t, int32_t);
ad69471c 1620 result |= tmp << 16;
4d9ad7f7 1621 DO_ABD(tmp, a >> 16, b >> 16, int8_t, int32_t);
ad69471c 1622 result |= tmp << 32;
4d9ad7f7 1623 DO_ABD(tmp, a >> 24, b >> 24, int8_t, int32_t);
ad69471c
PB		 1624 result |= tmp << 48;
1625 return result;
1626}
1627
1628uint64_t HELPER(neon_abdl_u32)(uint32_t a, uint32_t b)
1629{
1630 uint64_t tmp;
1631 uint64_t result;
4d9ad7f7
PM		 1632 DO_ABD(result, a, b, uint16_t, uint32_t);
1633 DO_ABD(tmp, a >> 16, b >> 16, uint16_t, uint32_t);
ad69471c
PB		 1634 return result | (tmp << 32);
1635}
1636
1637uint64_t HELPER(neon_abdl_s32)(uint32_t a, uint32_t b)
1638{
1639 uint64_t tmp;
1640 uint64_t result;
4d9ad7f7
PM		 1641 DO_ABD(result, a, b, int16_t, int32_t);
1642 DO_ABD(tmp, a >> 16, b >> 16, int16_t, int32_t);
ad69471c
PB		 1643 return result | (tmp << 32);
1644}
1645
1646uint64_t HELPER(neon_abdl_u64)(uint32_t a, uint32_t b)
1647{
1648 uint64_t result;
4d9ad7f7 1649 DO_ABD(result, a, b, uint32_t, uint64_t);
ad69471c
PB		 1650 return result;
1651}
1652
1653uint64_t HELPER(neon_abdl_s64)(uint32_t a, uint32_t b)
1654{
1655 uint64_t result;
4d9ad7f7 1656 DO_ABD(result, a, b, int32_t, int64_t);
ad69471c
PB		 1657 return result;
1658}
1659#undef DO_ABD
1660
1661/* Widening multiply. Named type is the source type. */
1662#define DO_MULL(dest, x, y, type1, type2) do { \
1663 type1 tmp_x = x; \
1664 type1 tmp_y = y; \
1665 dest = (type2)((type2)tmp_x * (type2)tmp_y); \
1666 } while(0)
1667
1668uint64_t HELPER(neon_mull_u8)(uint32_t a, uint32_t b)
1669{
1670 uint64_t tmp;
1671 uint64_t result;
1672
1673 DO_MULL(result, a, b, uint8_t, uint16_t);
1674 DO_MULL(tmp, a >> 8, b >> 8, uint8_t, uint16_t);
1675 result |= tmp << 16;
1676 DO_MULL(tmp, a >> 16, b >> 16, uint8_t, uint16_t);
1677 result |= tmp << 32;
1678 DO_MULL(tmp, a >> 24, b >> 24, uint8_t, uint16_t);
1679 result |= tmp << 48;
1680 return result;
1681}
1682
1683uint64_t HELPER(neon_mull_s8)(uint32_t a, uint32_t b)
1684{
1685 uint64_t tmp;
1686 uint64_t result;
1687
1688 DO_MULL(result, a, b, int8_t, uint16_t);
1689 DO_MULL(tmp, a >> 8, b >> 8, int8_t, uint16_t);
1690 result |= tmp << 16;
1691 DO_MULL(tmp, a >> 16, b >> 16, int8_t, uint16_t);
1692 result |= tmp << 32;
1693 DO_MULL(tmp, a >> 24, b >> 24, int8_t, uint16_t);
1694 result |= tmp << 48;
1695 return result;
1696}
1697
1698uint64_t HELPER(neon_mull_u16)(uint32_t a, uint32_t b)
1699{
1700 uint64_t tmp;
1701 uint64_t result;
1702
1703 DO_MULL(result, a, b, uint16_t, uint32_t);
1704 DO_MULL(tmp, a >> 16, b >> 16, uint16_t, uint32_t);
1705 return result | (tmp << 32);
1706}
1707
1708uint64_t HELPER(neon_mull_s16)(uint32_t a, uint32_t b)
1709{
1710 uint64_t tmp;
1711 uint64_t result;
1712
1713 DO_MULL(result, a, b, int16_t, uint32_t);
1714 DO_MULL(tmp, a >> 16, b >> 16, int16_t, uint32_t);
1715 return result | (tmp << 32);
1716}
1717
1718uint64_t HELPER(neon_negl_u16)(uint64_t x)
1719{
1720 uint16_t tmp;
1721 uint64_t result;
1722 result = (uint16_t)-x;
1723 tmp = -(x >> 16);
1724 result |= (uint64_t)tmp << 16;
1725 tmp = -(x >> 32);
1726 result |= (uint64_t)tmp << 32;
1727 tmp = -(x >> 48);
1728 result |= (uint64_t)tmp << 48;
1729 return result;
1730}
1731
ad69471c
PB		 1732uint64_t HELPER(neon_negl_u32)(uint64_t x)
1733{
1734 uint32_t low = -x;
1735 uint32_t high = -(x >> 32);
1736 return low | ((uint64_t)high << 32);
1737}
1738
b90372ad 1739/* Saturating sign manipulation. */
ad69471c
PB		 1740/* ??? Make these use NEON_VOP1 */
1741#define DO_QABS8(x) do { \
1742 if (x == (int8_t)0x80) { \
1743 x = 0x7f; \
1744 SET_QC(); \
1745 } else if (x < 0) { \
1746 x = -x; \
1747 }} while (0)
0ecb72a5 1748uint32_t HELPER(neon_qabs_s8)(CPUARMState *env, uint32_t x)
ad69471c
PB		 1749{
1750 neon_s8 vec;
1751 NEON_UNPACK(neon_s8, vec, x);
1752 DO_QABS8(vec.v1);
1753 DO_QABS8(vec.v2);
1754 DO_QABS8(vec.v3);
1755 DO_QABS8(vec.v4);
1756 NEON_PACK(neon_s8, x, vec);
1757 return x;
1758}
1759#undef DO_QABS8
1760
1761#define DO_QNEG8(x) do { \
1762 if (x == (int8_t)0x80) { \
1763 x = 0x7f; \
1764 SET_QC(); \
1765 } else { \
1766 x = -x; \
1767 }} while (0)
0ecb72a5 1768uint32_t HELPER(neon_qneg_s8)(CPUARMState *env, uint32_t x)
ad69471c
PB		 1769{
1770 neon_s8 vec;
1771 NEON_UNPACK(neon_s8, vec, x);
1772 DO_QNEG8(vec.v1);
1773 DO_QNEG8(vec.v2);
1774 DO_QNEG8(vec.v3);
1775 DO_QNEG8(vec.v4);
1776 NEON_PACK(neon_s8, x, vec);
1777 return x;
1778}
1779#undef DO_QNEG8
1780
1781#define DO_QABS16(x) do { \
1782 if (x == (int16_t)0x8000) { \
1783 x = 0x7fff; \
1784 SET_QC(); \
1785 } else if (x < 0) { \
1786 x = -x; \
1787 }} while (0)
0ecb72a5 1788uint32_t HELPER(neon_qabs_s16)(CPUARMState *env, uint32_t x)
ad69471c
PB		 1789{
1790 neon_s16 vec;
1791 NEON_UNPACK(neon_s16, vec, x);
1792 DO_QABS16(vec.v1);
1793 DO_QABS16(vec.v2);
1794 NEON_PACK(neon_s16, x, vec);
1795 return x;
1796}
1797#undef DO_QABS16
1798
1799#define DO_QNEG16(x) do { \
1800 if (x == (int16_t)0x8000) { \
1801 x = 0x7fff; \
1802 SET_QC(); \
1803 } else { \
1804 x = -x; \
1805 }} while (0)
0ecb72a5 1806uint32_t HELPER(neon_qneg_s16)(CPUARMState *env, uint32_t x)
ad69471c
PB		 1807{
1808 neon_s16 vec;
1809 NEON_UNPACK(neon_s16, vec, x);
1810 DO_QNEG16(vec.v1);
1811 DO_QNEG16(vec.v2);
1812 NEON_PACK(neon_s16, x, vec);
1813 return x;
1814}
1815#undef DO_QNEG16
1816
0ecb72a5 1817uint32_t HELPER(neon_qabs_s32)(CPUARMState *env, uint32_t x)
ad69471c
PB		 1818{
1819 if (x == SIGNBIT) {
1820 SET_QC();
1821 x = ~SIGNBIT;
1822 } else if ((int32_t)x < 0) {
1823 x = -x;
1824 }
1825 return x;
1826}
1827
0ecb72a5 1828uint32_t HELPER(neon_qneg_s32)(CPUARMState *env, uint32_t x)
ad69471c
PB		 1829{
1830 if (x == SIGNBIT) {
1831 SET_QC();
1832 x = ~SIGNBIT;
1833 } else {
1834 x = -x;
1835 }
1836 return x;
1837}
1838
0a79bc87
AB		 1839uint64_t HELPER(neon_qabs_s64)(CPUARMState *env, uint64_t x)
1840{
1841 if (x == SIGNBIT64) {
1842 SET_QC();
1843 x = ~SIGNBIT64;
1844 } else if ((int64_t)x < 0) {
1845 x = -x;
1846 }
1847 return x;
1848}
1849
1850uint64_t HELPER(neon_qneg_s64)(CPUARMState *env, uint64_t x)
1851{
1852 if (x == SIGNBIT64) {
1853 SET_QC();
1854 x = ~SIGNBIT64;
1855 } else {
1856 x = -x;
1857 }
1858 return x;
1859}
1860
ad69471c 1861/* NEON Float helpers. */
aa47cfdd 1862uint32_t HELPER(neon_abd_f32)(uint32_t a, uint32_t b, void *fpstp)
ad69471c 1863{
aa47cfdd 1864 float_status *fpst = fpstp;
51d85267
PM		 1865 float32 f0 = make_float32(a);
1866 float32 f1 = make_float32(b);
aa47cfdd 1867 return float32_val(float32_abs(float32_sub(f0, f1, fpst)));
ad69471c
PB		 1868}
1869
cab565c4
PM		 1870/* Floating point comparisons produce an integer result.
1871 * Note that EQ doesn't signal InvalidOp for QNaNs but GE and GT do.
1872 * Softfloat routines return 0/1, which we convert to the 0/-1 Neon requires.
1873 */
aa47cfdd 1874uint32_t HELPER(neon_ceq_f32)(uint32_t a, uint32_t b, void *fpstp)
cab565c4 1875{
aa47cfdd
PM		 1876 float_status *fpst = fpstp;
1877 return -float32_eq_quiet(make_float32(a), make_float32(b), fpst);
cab565c4
PM		 1878}
1879
aa47cfdd 1880uint32_t HELPER(neon_cge_f32)(uint32_t a, uint32_t b, void *fpstp)
cab565c4 1881{
aa47cfdd
PM		 1882 float_status *fpst = fpstp;
1883 return -float32_le(make_float32(b), make_float32(a), fpst);
ad69471c
PB		 1884}
1885
aa47cfdd 1886uint32_t HELPER(neon_cgt_f32)(uint32_t a, uint32_t b, void *fpstp)
cab565c4 1887{
aa47cfdd
PM		 1888 float_status *fpst = fpstp;
1889 return -float32_lt(make_float32(b), make_float32(a), fpst);
cab565c4 1890}
ad69471c 1891
aa47cfdd 1892uint32_t HELPER(neon_acge_f32)(uint32_t a, uint32_t b, void *fpstp)
ad69471c 1893{
aa47cfdd 1894 float_status *fpst = fpstp;
51d85267
PM		 1895 float32 f0 = float32_abs(make_float32(a));
1896 float32 f1 = float32_abs(make_float32(b));
aa47cfdd 1897 return -float32_le(f1, f0, fpst);
ad69471c
PB		 1898}
1899
aa47cfdd 1900uint32_t HELPER(neon_acgt_f32)(uint32_t a, uint32_t b, void *fpstp)
ad69471c 1901{
aa47cfdd 1902 float_status *fpst = fpstp;
51d85267
PM		 1903 float32 f0 = float32_abs(make_float32(a));
1904 float32 f1 = float32_abs(make_float32(b));
aa47cfdd 1905 return -float32_lt(f1, f0, fpst);
ad69471c 1906}
02acedf9 1907
057d5f62
PM		 1908uint64_t HELPER(neon_acge_f64)(uint64_t a, uint64_t b, void *fpstp)
1909{
1910 float_status *fpst = fpstp;
1911 float64 f0 = float64_abs(make_float64(a));
1912 float64 f1 = float64_abs(make_float64(b));
1913 return -float64_le(f1, f0, fpst);
1914}
1915
1916uint64_t HELPER(neon_acgt_f64)(uint64_t a, uint64_t b, void *fpstp)
1917{
1918 float_status *fpst = fpstp;
1919 float64 f0 = float64_abs(make_float64(a));
1920 float64 f1 = float64_abs(make_float64(b));
1921 return -float64_lt(f1, f0, fpst);
1922}
1923
02acedf9
PM		 1924#define ELEM(V, N, SIZE) (((V) >> ((N) * (SIZE))) & ((1ull << (SIZE)) - 1))
1925
b13708bb 1926void HELPER(neon_qunzip8)(void *vd, void *vm)
02acedf9 1927{
b13708bb
RH		 1928 uint64_t *rd = vd, *rm = vm;
1929 uint64_t zd0 = rd[0], zd1 = rd[1];
1930 uint64_t zm0 = rm[0], zm1 = rm[1];
1931
02acedf9
PM		 1932 uint64_t d0 = ELEM(zd0, 0, 8) | (ELEM(zd0, 2, 8) << 8)
1933 | (ELEM(zd0, 4, 8) << 16) | (ELEM(zd0, 6, 8) << 24)
1934 | (ELEM(zd1, 0, 8) << 32) | (ELEM(zd1, 2, 8) << 40)
1935 | (ELEM(zd1, 4, 8) << 48) | (ELEM(zd1, 6, 8) << 56);
1936 uint64_t d1 = ELEM(zm0, 0, 8) | (ELEM(zm0, 2, 8) << 8)
1937 | (ELEM(zm0, 4, 8) << 16) | (ELEM(zm0, 6, 8) << 24)
1938 | (ELEM(zm1, 0, 8) << 32) | (ELEM(zm1, 2, 8) << 40)
1939 | (ELEM(zm1, 4, 8) << 48) | (ELEM(zm1, 6, 8) << 56);
1940 uint64_t m0 = ELEM(zd0, 1, 8) | (ELEM(zd0, 3, 8) << 8)
1941 | (ELEM(zd0, 5, 8) << 16) | (ELEM(zd0, 7, 8) << 24)
1942 | (ELEM(zd1, 1, 8) << 32) | (ELEM(zd1, 3, 8) << 40)
1943 | (ELEM(zd1, 5, 8) << 48) | (ELEM(zd1, 7, 8) << 56);
1944 uint64_t m1 = ELEM(zm0, 1, 8) | (ELEM(zm0, 3, 8) << 8)
1945 | (ELEM(zm0, 5, 8) << 16) | (ELEM(zm0, 7, 8) << 24)
1946 | (ELEM(zm1, 1, 8) << 32) | (ELEM(zm1, 3, 8) << 40)
1947 | (ELEM(zm1, 5, 8) << 48) | (ELEM(zm1, 7, 8) << 56);
b13708bb
RH		 1948
1949 rm[0] = m0;
1950 rm[1] = m1;
1951 rd[0] = d0;
1952 rd[1] = d1;
02acedf9
PM		 1953}
1954
b13708bb 1955void HELPER(neon_qunzip16)(void *vd, void *vm)
02acedf9 1956{
b13708bb
RH		 1957 uint64_t *rd = vd, *rm = vm;
1958 uint64_t zd0 = rd[0], zd1 = rd[1];
1959 uint64_t zm0 = rm[0], zm1 = rm[1];
1960
02acedf9
PM		 1961 uint64_t d0 = ELEM(zd0, 0, 16) | (ELEM(zd0, 2, 16) << 16)
1962 | (ELEM(zd1, 0, 16) << 32) | (ELEM(zd1, 2, 16) << 48);
1963 uint64_t d1 = ELEM(zm0, 0, 16) | (ELEM(zm0, 2, 16) << 16)
1964 | (ELEM(zm1, 0, 16) << 32) | (ELEM(zm1, 2, 16) << 48);
1965 uint64_t m0 = ELEM(zd0, 1, 16) | (ELEM(zd0, 3, 16) << 16)
1966 | (ELEM(zd1, 1, 16) << 32) | (ELEM(zd1, 3, 16) << 48);
1967 uint64_t m1 = ELEM(zm0, 1, 16) | (ELEM(zm0, 3, 16) << 16)
1968 | (ELEM(zm1, 1, 16) << 32) | (ELEM(zm1, 3, 16) << 48);
b13708bb
RH		 1969
1970 rm[0] = m0;
1971 rm[1] = m1;
1972 rd[0] = d0;
1973 rd[1] = d1;
02acedf9
PM		 1974}
1975
b13708bb 1976void HELPER(neon_qunzip32)(void *vd, void *vm)
02acedf9 1977{
b13708bb
RH		 1978 uint64_t *rd = vd, *rm = vm;
1979 uint64_t zd0 = rd[0], zd1 = rd[1];
1980 uint64_t zm0 = rm[0], zm1 = rm[1];
1981
02acedf9
PM		 1982 uint64_t d0 = ELEM(zd0, 0, 32) | (ELEM(zd1, 0, 32) << 32);
1983 uint64_t d1 = ELEM(zm0, 0, 32) | (ELEM(zm1, 0, 32) << 32);
1984 uint64_t m0 = ELEM(zd0, 1, 32) | (ELEM(zd1, 1, 32) << 32);
1985 uint64_t m1 = ELEM(zm0, 1, 32) | (ELEM(zm1, 1, 32) << 32);
b13708bb
RH		 1986
1987 rm[0] = m0;
1988 rm[1] = m1;
1989 rd[0] = d0;
1990 rd[1] = d1;
02acedf9
PM		 1991}
1992
b13708bb 1993void HELPER(neon_unzip8)(void *vd, void *vm)
02acedf9 1994{
b13708bb
RH		 1995 uint64_t *rd = vd, *rm = vm;
1996 uint64_t zd = rd[0], zm = rm[0];
1997
02acedf9
PM		 1998 uint64_t d0 = ELEM(zd, 0, 8) | (ELEM(zd, 2, 8) << 8)
1999 | (ELEM(zd, 4, 8) << 16) | (ELEM(zd, 6, 8) << 24)
2000 | (ELEM(zm, 0, 8) << 32) | (ELEM(zm, 2, 8) << 40)
2001 | (ELEM(zm, 4, 8) << 48) | (ELEM(zm, 6, 8) << 56);
2002 uint64_t m0 = ELEM(zd, 1, 8) | (ELEM(zd, 3, 8) << 8)
2003 | (ELEM(zd, 5, 8) << 16) | (ELEM(zd, 7, 8) << 24)
2004 | (ELEM(zm, 1, 8) << 32) | (ELEM(zm, 3, 8) << 40)
2005 | (ELEM(zm, 5, 8) << 48) | (ELEM(zm, 7, 8) << 56);
b13708bb
RH		 2006
2007 rm[0] = m0;
2008 rd[0] = d0;
02acedf9
PM		 2009}
2010
b13708bb 2011void HELPER(neon_unzip16)(void *vd, void *vm)
02acedf9 2012{
b13708bb
RH		 2013 uint64_t *rd = vd, *rm = vm;
2014 uint64_t zd = rd[0], zm = rm[0];
2015
02acedf9
PM		 2016 uint64_t d0 = ELEM(zd, 0, 16) | (ELEM(zd, 2, 16) << 16)
2017 | (ELEM(zm, 0, 16) << 32) | (ELEM(zm, 2, 16) << 48);
2018 uint64_t m0 = ELEM(zd, 1, 16) | (ELEM(zd, 3, 16) << 16)
2019 | (ELEM(zm, 1, 16) << 32) | (ELEM(zm, 3, 16) << 48);
b13708bb
RH		 2020
2021 rm[0] = m0;
2022 rd[0] = d0;
02acedf9 2023}
d68a6f3a 2024
b13708bb 2025void HELPER(neon_qzip8)(void *vd, void *vm)
d68a6f3a 2026{
b13708bb
RH		 2027 uint64_t *rd = vd, *rm = vm;
2028 uint64_t zd0 = rd[0], zd1 = rd[1];
2029 uint64_t zm0 = rm[0], zm1 = rm[1];
2030
d68a6f3a
PM		 2031 uint64_t d0 = ELEM(zd0, 0, 8) | (ELEM(zm0, 0, 8) << 8)
2032 | (ELEM(zd0, 1, 8) << 16) | (ELEM(zm0, 1, 8) << 24)
2033 | (ELEM(zd0, 2, 8) << 32) | (ELEM(zm0, 2, 8) << 40)
2034 | (ELEM(zd0, 3, 8) << 48) | (ELEM(zm0, 3, 8) << 56);
2035 uint64_t d1 = ELEM(zd0, 4, 8) | (ELEM(zm0, 4, 8) << 8)
2036 | (ELEM(zd0, 5, 8) << 16) | (ELEM(zm0, 5, 8) << 24)
2037 | (ELEM(zd0, 6, 8) << 32) | (ELEM(zm0, 6, 8) << 40)
2038 | (ELEM(zd0, 7, 8) << 48) | (ELEM(zm0, 7, 8) << 56);
2039 uint64_t m0 = ELEM(zd1, 0, 8) | (ELEM(zm1, 0, 8) << 8)
2040 | (ELEM(zd1, 1, 8) << 16) | (ELEM(zm1, 1, 8) << 24)
2041 | (ELEM(zd1, 2, 8) << 32) | (ELEM(zm1, 2, 8) << 40)
2042 | (ELEM(zd1, 3, 8) << 48) | (ELEM(zm1, 3, 8) << 56);
2043 uint64_t m1 = ELEM(zd1, 4, 8) | (ELEM(zm1, 4, 8) << 8)
2044 | (ELEM(zd1, 5, 8) << 16) | (ELEM(zm1, 5, 8) << 24)
2045 | (ELEM(zd1, 6, 8) << 32) | (ELEM(zm1, 6, 8) << 40)
2046 | (ELEM(zd1, 7, 8) << 48) | (ELEM(zm1, 7, 8) << 56);
b13708bb
RH		 2047
2048 rm[0] = m0;
2049 rm[1] = m1;
2050 rd[0] = d0;
2051 rd[1] = d1;
d68a6f3a
PM		 2052}
2053
b13708bb 2054void HELPER(neon_qzip16)(void *vd, void *vm)
d68a6f3a 2055{
b13708bb
RH		 2056 uint64_t *rd = vd, *rm = vm;
2057 uint64_t zd0 = rd[0], zd1 = rd[1];
2058 uint64_t zm0 = rm[0], zm1 = rm[1];
2059
d68a6f3a
PM		 2060 uint64_t d0 = ELEM(zd0, 0, 16) | (ELEM(zm0, 0, 16) << 16)
2061 | (ELEM(zd0, 1, 16) << 32) | (ELEM(zm0, 1, 16) << 48);
2062 uint64_t d1 = ELEM(zd0, 2, 16) | (ELEM(zm0, 2, 16) << 16)
2063 | (ELEM(zd0, 3, 16) << 32) | (ELEM(zm0, 3, 16) << 48);
2064 uint64_t m0 = ELEM(zd1, 0, 16) | (ELEM(zm1, 0, 16) << 16)
2065 | (ELEM(zd1, 1, 16) << 32) | (ELEM(zm1, 1, 16) << 48);
2066 uint64_t m1 = ELEM(zd1, 2, 16) | (ELEM(zm1, 2, 16) << 16)
2067 | (ELEM(zd1, 3, 16) << 32) | (ELEM(zm1, 3, 16) << 48);
b13708bb
RH		 2068
2069 rm[0] = m0;
2070 rm[1] = m1;
2071 rd[0] = d0;
2072 rd[1] = d1;
d68a6f3a
PM		 2073}
2074
b13708bb 2075void HELPER(neon_qzip32)(void *vd, void *vm)
d68a6f3a 2076{
b13708bb
RH		 2077 uint64_t *rd = vd, *rm = vm;
2078 uint64_t zd0 = rd[0], zd1 = rd[1];
2079 uint64_t zm0 = rm[0], zm1 = rm[1];
2080
d68a6f3a
PM		 2081 uint64_t d0 = ELEM(zd0, 0, 32) | (ELEM(zm0, 0, 32) << 32);
2082 uint64_t d1 = ELEM(zd0, 1, 32) | (ELEM(zm0, 1, 32) << 32);
2083 uint64_t m0 = ELEM(zd1, 0, 32) | (ELEM(zm1, 0, 32) << 32);
2084 uint64_t m1 = ELEM(zd1, 1, 32) | (ELEM(zm1, 1, 32) << 32);
b13708bb
RH		 2085
2086 rm[0] = m0;
2087 rm[1] = m1;
2088 rd[0] = d0;
2089 rd[1] = d1;
d68a6f3a
PM		 2090}
2091
b13708bb 2092void HELPER(neon_zip8)(void *vd, void *vm)
d68a6f3a 2093{
b13708bb
RH		 2094 uint64_t *rd = vd, *rm = vm;
2095 uint64_t zd = rd[0], zm = rm[0];
2096
d68a6f3a
PM		 2097 uint64_t d0 = ELEM(zd, 0, 8) | (ELEM(zm, 0, 8) << 8)
2098 | (ELEM(zd, 1, 8) << 16) | (ELEM(zm, 1, 8) << 24)
2099 | (ELEM(zd, 2, 8) << 32) | (ELEM(zm, 2, 8) << 40)
2100 | (ELEM(zd, 3, 8) << 48) | (ELEM(zm, 3, 8) << 56);
2101 uint64_t m0 = ELEM(zd, 4, 8) | (ELEM(zm, 4, 8) << 8)
2102 | (ELEM(zd, 5, 8) << 16) | (ELEM(zm, 5, 8) << 24)
2103 | (ELEM(zd, 6, 8) << 32) | (ELEM(zm, 6, 8) << 40)
2104 | (ELEM(zd, 7, 8) << 48) | (ELEM(zm, 7, 8) << 56);
b13708bb
RH		 2105
2106 rm[0] = m0;
2107 rd[0] = d0;
d68a6f3a
PM		 2108}
2109
b13708bb 2110void HELPER(neon_zip16)(void *vd, void *vm)
d68a6f3a 2111{
b13708bb
RH		 2112 uint64_t *rd = vd, *rm = vm;
2113 uint64_t zd = rd[0], zm = rm[0];
2114
d68a6f3a
PM		 2115 uint64_t d0 = ELEM(zd, 0, 16) | (ELEM(zm, 0, 16) << 16)
2116 | (ELEM(zd, 1, 16) << 32) | (ELEM(zm, 1, 16) << 48);
2117 uint64_t m0 = ELEM(zd, 2, 16) | (ELEM(zm, 2, 16) << 16)
2118 | (ELEM(zd, 3, 16) << 32) | (ELEM(zm, 3, 16) << 48);
b13708bb
RH		 2119
2120 rm[0] = m0;
2121 rd[0] = d0;
d68a6f3a 2122}
Empty description
This page took 1.166663 seconds and 4 git commands to generate.