]> Git Repo - qemu.git/blame - target/arm/neon_helper.c
projects / qemu.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
target/arm/vfp_helper: Call set_fpscr_to_host before updating to FPSCR
[qemu.git] / target / arm / neon_helper.c
CommitLineData
e677137d
PB		 1/*
2 * ARM NEON vector operations.
3 *
4 * Copyright (c) 2007, 2008 CodeSourcery.
5 * Written by Paul Brook
6 *
8e31bf38 7 * This code is licensed under the GNU GPL v2.
e677137d 8 */
74c21bd0 9#include "qemu/osdep.h"
ad69471c
PB		 10
11#include "cpu.h"
2ef6175a 12#include "exec/helper-proto.h"
24f91e81 13#include "fpu/softfloat.h"
ad69471c
PB		 14
15#define SIGNBIT (uint32_t)0x80000000
16#define SIGNBIT64 ((uint64_t)1 << 63)
17
a4d58462 18#define SET_QC() env->vfp.qc[0] = 1
ad69471c 19
ad69471c
PB		 20#define NEON_TYPE1(name, type) \
21typedef struct \
22{ \
23 type v1; \
24} neon_##name;
e2542fe2 25#ifdef HOST_WORDS_BIGENDIAN
ad69471c
PB		 26#define NEON_TYPE2(name, type) \
27typedef struct \
28{ \
29 type v2; \
30 type v1; \
31} neon_##name;
32#define NEON_TYPE4(name, type) \
33typedef struct \
34{ \
35 type v4; \
36 type v3; \
37 type v2; \
38 type v1; \
39} neon_##name;
40#else
41#define NEON_TYPE2(name, type) \
42typedef struct \
43{ \
44 type v1; \
45 type v2; \
46} neon_##name;
47#define NEON_TYPE4(name, type) \
48typedef struct \
49{ \
50 type v1; \
51 type v2; \
52 type v3; \
53 type v4; \
54} neon_##name;
55#endif
56
57NEON_TYPE4(s8, int8_t)
58NEON_TYPE4(u8, uint8_t)
59NEON_TYPE2(s16, int16_t)
60NEON_TYPE2(u16, uint16_t)
61NEON_TYPE1(s32, int32_t)
62NEON_TYPE1(u32, uint32_t)
63#undef NEON_TYPE4
64#undef NEON_TYPE2
65#undef NEON_TYPE1
66
67/* Copy from a uint32_t to a vector structure type. */
68#define NEON_UNPACK(vtype, dest, val) do { \
69 union { \
70 vtype v; \
71 uint32_t i; \
72 } conv_u; \
73 conv_u.i = (val); \
74 dest = conv_u.v; \
75 } while(0)
76
77/* Copy from a vector structure type to a uint32_t. */
78#define NEON_PACK(vtype, dest, val) do { \
79 union { \
80 vtype v; \
81 uint32_t i; \
82 } conv_u; \
83 conv_u.v = (val); \
84 dest = conv_u.i; \
85 } while(0)
86
87#define NEON_DO1 \
88 NEON_FN(vdest.v1, vsrc1.v1, vsrc2.v1);
89#define NEON_DO2 \
90 NEON_FN(vdest.v1, vsrc1.v1, vsrc2.v1); \
91 NEON_FN(vdest.v2, vsrc1.v2, vsrc2.v2);
92#define NEON_DO4 \
93 NEON_FN(vdest.v1, vsrc1.v1, vsrc2.v1); \
94 NEON_FN(vdest.v2, vsrc1.v2, vsrc2.v2); \
95 NEON_FN(vdest.v3, vsrc1.v3, vsrc2.v3); \
96 NEON_FN(vdest.v4, vsrc1.v4, vsrc2.v4);
97
98#define NEON_VOP_BODY(vtype, n) \
99{ \
100 uint32_t res; \
101 vtype vsrc1; \
102 vtype vsrc2; \
103 vtype vdest; \
104 NEON_UNPACK(vtype, vsrc1, arg1); \
105 NEON_UNPACK(vtype, vsrc2, arg2); \
106 NEON_DO##n; \
107 NEON_PACK(vtype, res, vdest); \
108 return res; \
109}
110
111#define NEON_VOP(name, vtype, n) \
112uint32_t HELPER(glue(neon_,name))(uint32_t arg1, uint32_t arg2) \
113NEON_VOP_BODY(vtype, n)
114
02da0b2d 115#define NEON_VOP_ENV(name, vtype, n) \
0ecb72a5 116uint32_t HELPER(glue(neon_,name))(CPUARMState *env, uint32_t arg1, uint32_t arg2) \
02da0b2d
PM		 117NEON_VOP_BODY(vtype, n)
118
ad69471c
PB		 119/* Pairwise operations. */
120/* For 32-bit elements each segment only contains a single element, so
121 the elementwise and pairwise operations are the same. */
122#define NEON_PDO2 \
123 NEON_FN(vdest.v1, vsrc1.v1, vsrc1.v2); \
124 NEON_FN(vdest.v2, vsrc2.v1, vsrc2.v2);
125#define NEON_PDO4 \
126 NEON_FN(vdest.v1, vsrc1.v1, vsrc1.v2); \
127 NEON_FN(vdest.v2, vsrc1.v3, vsrc1.v4); \
128 NEON_FN(vdest.v3, vsrc2.v1, vsrc2.v2); \
129 NEON_FN(vdest.v4, vsrc2.v3, vsrc2.v4); \
130
131#define NEON_POP(name, vtype, n) \
132uint32_t HELPER(glue(neon_,name))(uint32_t arg1, uint32_t arg2) \
133{ \
134 uint32_t res; \
135 vtype vsrc1; \
136 vtype vsrc2; \
137 vtype vdest; \
138 NEON_UNPACK(vtype, vsrc1, arg1); \
139 NEON_UNPACK(vtype, vsrc2, arg2); \
140 NEON_PDO##n; \
141 NEON_PACK(vtype, res, vdest); \
142 return res; \
143}
144
145/* Unary operators. */
146#define NEON_VOP1(name, vtype, n) \
147uint32_t HELPER(glue(neon_,name))(uint32_t arg) \
148{ \
149 vtype vsrc1; \
150 vtype vdest; \
151 NEON_UNPACK(vtype, vsrc1, arg); \
152 NEON_DO##n; \
153 NEON_PACK(vtype, arg, vdest); \
154 return arg; \
155}
156
157
158#define NEON_USAT(dest, src1, src2, type) do { \
159 uint32_t tmp = (uint32_t)src1 + (uint32_t)src2; \
160 if (tmp != (type)tmp) { \
161 SET_QC(); \
162 dest = ~0; \
163 } else { \
164 dest = tmp; \
165 }} while(0)
166#define NEON_FN(dest, src1, src2) NEON_USAT(dest, src1, src2, uint8_t)
02da0b2d 167NEON_VOP_ENV(qadd_u8, neon_u8, 4)
ad69471c
PB		 168#undef NEON_FN
169#define NEON_FN(dest, src1, src2) NEON_USAT(dest, src1, src2, uint16_t)
02da0b2d 170NEON_VOP_ENV(qadd_u16, neon_u16, 2)
ad69471c
PB		 171#undef NEON_FN
172#undef NEON_USAT
173
0ecb72a5 174uint32_t HELPER(neon_qadd_u32)(CPUARMState *env, uint32_t a, uint32_t b)
72902672
CL		 175{
176 uint32_t res = a + b;
177 if (res < a) {
178 SET_QC();
179 res = ~0;
180 }
181 return res;
182}
183
0ecb72a5 184uint64_t HELPER(neon_qadd_u64)(CPUARMState *env, uint64_t src1, uint64_t src2)
72902672
CL		 185{
186 uint64_t res;
187
188 res = src1 + src2;
189 if (res < src1) {
190 SET_QC();
191 res = ~(uint64_t)0;
192 }
193 return res;
194}
195
ad69471c
PB		 196#define NEON_SSAT(dest, src1, src2, type) do { \
197 int32_t tmp = (uint32_t)src1 + (uint32_t)src2; \
198 if (tmp != (type)tmp) { \
199 SET_QC(); \
200 if (src2 > 0) { \
201 tmp = (1 << (sizeof(type) * 8 - 1)) - 1; \
202 } else { \
203 tmp = 1 << (sizeof(type) * 8 - 1); \
204 } \
205 } \
206 dest = tmp; \
207 } while(0)
208#define NEON_FN(dest, src1, src2) NEON_SSAT(dest, src1, src2, int8_t)
02da0b2d 209NEON_VOP_ENV(qadd_s8, neon_s8, 4)
ad69471c
PB		 210#undef NEON_FN
211#define NEON_FN(dest, src1, src2) NEON_SSAT(dest, src1, src2, int16_t)
02da0b2d 212NEON_VOP_ENV(qadd_s16, neon_s16, 2)
ad69471c
PB		 213#undef NEON_FN
214#undef NEON_SSAT
215
0ecb72a5 216uint32_t HELPER(neon_qadd_s32)(CPUARMState *env, uint32_t a, uint32_t b)
72902672
CL		 217{
218 uint32_t res = a + b;
219 if (((res ^ a) & SIGNBIT) && !((a ^ b) & SIGNBIT)) {
220 SET_QC();
221 res = ~(((int32_t)a >> 31) ^ SIGNBIT);
222 }
223 return res;
224}
225
0ecb72a5 226uint64_t HELPER(neon_qadd_s64)(CPUARMState *env, uint64_t src1, uint64_t src2)
72902672
CL		 227{
228 uint64_t res;
229
230 res = src1 + src2;
231 if (((res ^ src1) & SIGNBIT64) && !((src1 ^ src2) & SIGNBIT64)) {
232 SET_QC();
233 res = ((int64_t)src1 >> 63) ^ ~SIGNBIT64;
234 }
235 return res;
236}
237
09e03735
AB		 238/* Unsigned saturating accumulate of signed value
239 *
240 * Op1/Rn is treated as signed
241 * Op2/Rd is treated as unsigned
242 *
243 * Explicit casting is used to ensure the correct sign extension of
244 * inputs. The result is treated as a unsigned value and saturated as such.
245 *
246 * We use a macro for the 8/16 bit cases which expects signed integers of va,
247 * vb, and vr for interim calculation and an unsigned 32 bit result value r.
248 */
249
250#define USATACC(bits, shift) \
251 do { \
252 va = sextract32(a, shift, bits); \
253 vb = extract32(b, shift, bits); \
254 vr = va + vb; \
255 if (vr > UINT##bits##_MAX) { \
256 SET_QC(); \
257 vr = UINT##bits##_MAX; \
258 } else if (vr < 0) { \
259 SET_QC(); \
260 vr = 0; \
261 } \
262 r = deposit32(r, shift, bits, vr); \
263 } while (0)
264
265uint32_t HELPER(neon_uqadd_s8)(CPUARMState *env, uint32_t a, uint32_t b)
266{
267 int16_t va, vb, vr;
268 uint32_t r = 0;
269
270 USATACC(8, 0);
271 USATACC(8, 8);
272 USATACC(8, 16);
273 USATACC(8, 24);
274 return r;
275}
276
277uint32_t HELPER(neon_uqadd_s16)(CPUARMState *env, uint32_t a, uint32_t b)
278{
279 int32_t va, vb, vr;
280 uint64_t r = 0;
281
282 USATACC(16, 0);
283 USATACC(16, 16);
284 return r;
285}
286
287#undef USATACC
288
289uint32_t HELPER(neon_uqadd_s32)(CPUARMState *env, uint32_t a, uint32_t b)
290{
291 int64_t va = (int32_t)a;
292 int64_t vb = (uint32_t)b;
293 int64_t vr = va + vb;
294 if (vr > UINT32_MAX) {
295 SET_QC();
296 vr = UINT32_MAX;
297 } else if (vr < 0) {
298 SET_QC();
299 vr = 0;
300 }
301 return vr;
302}
303
304uint64_t HELPER(neon_uqadd_s64)(CPUARMState *env, uint64_t a, uint64_t b)
305{
306 uint64_t res;
307 res = a + b;
308 /* We only need to look at the pattern of SIGN bits to detect
309 * +ve/-ve saturation
310 */
311 if (~a & b & ~res & SIGNBIT64) {
312 SET_QC();
313 res = UINT64_MAX;
314 } else if (a & ~b & res & SIGNBIT64) {
315 SET_QC();
316 res = 0;
317 }
318 return res;
319}
320
321/* Signed saturating accumulate of unsigned value
322 *
323 * Op1/Rn is treated as unsigned
324 * Op2/Rd is treated as signed
325 *
326 * The result is treated as a signed value and saturated as such
327 *
328 * We use a macro for the 8/16 bit cases which expects signed integers of va,
329 * vb, and vr for interim calculation and an unsigned 32 bit result value r.
330 */
331
332#define SSATACC(bits, shift) \
333 do { \
334 va = extract32(a, shift, bits); \
335 vb = sextract32(b, shift, bits); \
336 vr = va + vb; \
337 if (vr > INT##bits##_MAX) { \
338 SET_QC(); \
339 vr = INT##bits##_MAX; \
340 } else if (vr < INT##bits##_MIN) { \
341 SET_QC(); \
342 vr = INT##bits##_MIN; \
343 } \
344 r = deposit32(r, shift, bits, vr); \
345 } while (0)
346
347uint32_t HELPER(neon_sqadd_u8)(CPUARMState *env, uint32_t a, uint32_t b)
348{
349 int16_t va, vb, vr;
350 uint32_t r = 0;
351
352 SSATACC(8, 0);
353 SSATACC(8, 8);
354 SSATACC(8, 16);
355 SSATACC(8, 24);
356 return r;
357}
358
359uint32_t HELPER(neon_sqadd_u16)(CPUARMState *env, uint32_t a, uint32_t b)
360{
361 int32_t va, vb, vr;
362 uint32_t r = 0;
363
364 SSATACC(16, 0);
365 SSATACC(16, 16);
366
367 return r;
368}
369
370#undef SSATACC
371
372uint32_t HELPER(neon_sqadd_u32)(CPUARMState *env, uint32_t a, uint32_t b)
373{
374 int64_t res;
375 int64_t op1 = (uint32_t)a;
376 int64_t op2 = (int32_t)b;
377 res = op1 + op2;
378 if (res > INT32_MAX) {
379 SET_QC();
380 res = INT32_MAX;
381 } else if (res < INT32_MIN) {
382 SET_QC();
383 res = INT32_MIN;
384 }
385 return res;
386}
387
388uint64_t HELPER(neon_sqadd_u64)(CPUARMState *env, uint64_t a, uint64_t b)
389{
390 uint64_t res;
391 res = a + b;
392 /* We only need to look at the pattern of SIGN bits to detect an overflow */
393 if (((a & res)
394 | (~b & res)
395 | (a & ~b)) & SIGNBIT64) {
396 SET_QC();
397 res = INT64_MAX;
398 }
399 return res;
400}
401
402
ad69471c
PB		 403#define NEON_USAT(dest, src1, src2, type) do { \
404 uint32_t tmp = (uint32_t)src1 - (uint32_t)src2; \
405 if (tmp != (type)tmp) { \
406 SET_QC(); \
407 dest = 0; \
408 } else { \
409 dest = tmp; \
410 }} while(0)
411#define NEON_FN(dest, src1, src2) NEON_USAT(dest, src1, src2, uint8_t)
02da0b2d 412NEON_VOP_ENV(qsub_u8, neon_u8, 4)
ad69471c
PB		 413#undef NEON_FN
414#define NEON_FN(dest, src1, src2) NEON_USAT(dest, src1, src2, uint16_t)
02da0b2d 415NEON_VOP_ENV(qsub_u16, neon_u16, 2)
ad69471c
PB		 416#undef NEON_FN
417#undef NEON_USAT
418
0ecb72a5 419uint32_t HELPER(neon_qsub_u32)(CPUARMState *env, uint32_t a, uint32_t b)
72902672
CL		 420{
421 uint32_t res = a - b;
422 if (res > a) {
423 SET_QC();
424 res = 0;
425 }
426 return res;
427}
428
0ecb72a5 429uint64_t HELPER(neon_qsub_u64)(CPUARMState *env, uint64_t src1, uint64_t src2)
72902672
CL		 430{
431 uint64_t res;
432
433 if (src1 < src2) {
434 SET_QC();
435 res = 0;
436 } else {
437 res = src1 - src2;
438 }
439 return res;
440}
441
ad69471c
PB		 442#define NEON_SSAT(dest, src1, src2, type) do { \
443 int32_t tmp = (uint32_t)src1 - (uint32_t)src2; \
444 if (tmp != (type)tmp) { \
445 SET_QC(); \
446 if (src2 < 0) { \
447 tmp = (1 << (sizeof(type) * 8 - 1)) - 1; \
448 } else { \
449 tmp = 1 << (sizeof(type) * 8 - 1); \
450 } \
451 } \
452 dest = tmp; \
453 } while(0)
454#define NEON_FN(dest, src1, src2) NEON_SSAT(dest, src1, src2, int8_t)
02da0b2d 455NEON_VOP_ENV(qsub_s8, neon_s8, 4)
ad69471c
PB		 456#undef NEON_FN
457#define NEON_FN(dest, src1, src2) NEON_SSAT(dest, src1, src2, int16_t)
02da0b2d 458NEON_VOP_ENV(qsub_s16, neon_s16, 2)
ad69471c
PB		 459#undef NEON_FN
460#undef NEON_SSAT
461
0ecb72a5 462uint32_t HELPER(neon_qsub_s32)(CPUARMState *env, uint32_t a, uint32_t b)
72902672
CL		 463{
464 uint32_t res = a - b;
465 if (((res ^ a) & SIGNBIT) && ((a ^ b) & SIGNBIT)) {
466 SET_QC();
467 res = ~(((int32_t)a >> 31) ^ SIGNBIT);
468 }
469 return res;
470}
471
0ecb72a5 472uint64_t HELPER(neon_qsub_s64)(CPUARMState *env, uint64_t src1, uint64_t src2)
72902672
CL		 473{
474 uint64_t res;
475
476 res = src1 - src2;
477 if (((res ^ src1) & SIGNBIT64) && ((src1 ^ src2) & SIGNBIT64)) {
478 SET_QC();
479 res = ((int64_t)src1 >> 63) ^ ~SIGNBIT64;
480 }
481 return res;
482}
483
ad69471c
PB		 484#define NEON_FN(dest, src1, src2) dest = (src1 + src2) >> 1
485NEON_VOP(hadd_s8, neon_s8, 4)
486NEON_VOP(hadd_u8, neon_u8, 4)
487NEON_VOP(hadd_s16, neon_s16, 2)
488NEON_VOP(hadd_u16, neon_u16, 2)
489#undef NEON_FN
490
491int32_t HELPER(neon_hadd_s32)(int32_t src1, int32_t src2)
492{
493 int32_t dest;
494
495 dest = (src1 >> 1) + (src2 >> 1);
496 if (src1 & src2 & 1)
497 dest++;
498 return dest;
499}
500
501uint32_t HELPER(neon_hadd_u32)(uint32_t src1, uint32_t src2)
502{
503 uint32_t dest;
504
505 dest = (src1 >> 1) + (src2 >> 1);
506 if (src1 & src2 & 1)
507 dest++;
508 return dest;
509}
510
511#define NEON_FN(dest, src1, src2) dest = (src1 + src2 + 1) >> 1
512NEON_VOP(rhadd_s8, neon_s8, 4)
513NEON_VOP(rhadd_u8, neon_u8, 4)
514NEON_VOP(rhadd_s16, neon_s16, 2)
515NEON_VOP(rhadd_u16, neon_u16, 2)
516#undef NEON_FN
517
518int32_t HELPER(neon_rhadd_s32)(int32_t src1, int32_t src2)
519{
520 int32_t dest;
521
522 dest = (src1 >> 1) + (src2 >> 1);
523 if ((src1 | src2) & 1)
524 dest++;
525 return dest;
526}
527
528uint32_t HELPER(neon_rhadd_u32)(uint32_t src1, uint32_t src2)
529{
530 uint32_t dest;
531
532 dest = (src1 >> 1) + (src2 >> 1);
533 if ((src1 | src2) & 1)
534 dest++;
535 return dest;
536}
537
538#define NEON_FN(dest, src1, src2) dest = (src1 - src2) >> 1
539NEON_VOP(hsub_s8, neon_s8, 4)
540NEON_VOP(hsub_u8, neon_u8, 4)
541NEON_VOP(hsub_s16, neon_s16, 2)
542NEON_VOP(hsub_u16, neon_u16, 2)
543#undef NEON_FN
544
545int32_t HELPER(neon_hsub_s32)(int32_t src1, int32_t src2)
546{
547 int32_t dest;
548
549 dest = (src1 >> 1) - (src2 >> 1);
550 if ((~src1) & src2 & 1)
551 dest--;
552 return dest;
553}
554
555uint32_t HELPER(neon_hsub_u32)(uint32_t src1, uint32_t src2)
556{
557 uint32_t dest;
558
559 dest = (src1 >> 1) - (src2 >> 1);
560 if ((~src1) & src2 & 1)
561 dest--;
562 return dest;
563}
564
565#define NEON_FN(dest, src1, src2) dest = (src1 > src2) ? ~0 : 0
566NEON_VOP(cgt_s8, neon_s8, 4)
567NEON_VOP(cgt_u8, neon_u8, 4)
568NEON_VOP(cgt_s16, neon_s16, 2)
569NEON_VOP(cgt_u16, neon_u16, 2)
570NEON_VOP(cgt_s32, neon_s32, 1)
571NEON_VOP(cgt_u32, neon_u32, 1)
572#undef NEON_FN
573
574#define NEON_FN(dest, src1, src2) dest = (src1 >= src2) ? ~0 : 0
575NEON_VOP(cge_s8, neon_s8, 4)
576NEON_VOP(cge_u8, neon_u8, 4)
577NEON_VOP(cge_s16, neon_s16, 2)
578NEON_VOP(cge_u16, neon_u16, 2)
579NEON_VOP(cge_s32, neon_s32, 1)
580NEON_VOP(cge_u32, neon_u32, 1)
581#undef NEON_FN
582
583#define NEON_FN(dest, src1, src2) dest = (src1 < src2) ? src1 : src2
ad69471c
PB		 584NEON_POP(pmin_s8, neon_s8, 4)
585NEON_POP(pmin_u8, neon_u8, 4)
586NEON_POP(pmin_s16, neon_s16, 2)
587NEON_POP(pmin_u16, neon_u16, 2)
588#undef NEON_FN
589
590#define NEON_FN(dest, src1, src2) dest = (src1 > src2) ? src1 : src2
ad69471c
PB		 591NEON_POP(pmax_s8, neon_s8, 4)
592NEON_POP(pmax_u8, neon_u8, 4)
593NEON_POP(pmax_s16, neon_s16, 2)
594NEON_POP(pmax_u16, neon_u16, 2)
595#undef NEON_FN
596
597#define NEON_FN(dest, src1, src2) \
598 dest = (src1 > src2) ? (src1 - src2) : (src2 - src1)
599NEON_VOP(abd_s8, neon_s8, 4)
600NEON_VOP(abd_u8, neon_u8, 4)
601NEON_VOP(abd_s16, neon_s16, 2)
602NEON_VOP(abd_u16, neon_u16, 2)
603NEON_VOP(abd_s32, neon_s32, 1)
604NEON_VOP(abd_u32, neon_u32, 1)
605#undef NEON_FN
606
607#define NEON_FN(dest, src1, src2) do { \
608 int8_t tmp; \
609 tmp = (int8_t)src2; \
50f67e95
JR		 610 if (tmp >= (ssize_t)sizeof(src1) * 8 || \
611 tmp <= -(ssize_t)sizeof(src1) * 8) { \
ad69471c
PB		 612 dest = 0; \
613 } else if (tmp < 0) { \
614 dest = src1 >> -tmp; \
615 } else { \
616 dest = src1 << tmp; \
617 }} while (0)
618NEON_VOP(shl_u8, neon_u8, 4)
619NEON_VOP(shl_u16, neon_u16, 2)
620NEON_VOP(shl_u32, neon_u32, 1)
621#undef NEON_FN
622
623uint64_t HELPER(neon_shl_u64)(uint64_t val, uint64_t shiftop)
624{
625 int8_t shift = (int8_t)shiftop;
626 if (shift >= 64 || shift <= -64) {
627 val = 0;
628 } else if (shift < 0) {
629 val >>= -shift;
630 } else {
631 val <<= shift;
632 }
633 return val;
634}
635
636#define NEON_FN(dest, src1, src2) do { \
637 int8_t tmp; \
638 tmp = (int8_t)src2; \
50f67e95 639 if (tmp >= (ssize_t)sizeof(src1) * 8) { \
ad69471c 640 dest = 0; \
50f67e95 641 } else if (tmp <= -(ssize_t)sizeof(src1) * 8) { \
ad69471c
PB		 642 dest = src1 >> (sizeof(src1) * 8 - 1); \
643 } else if (tmp < 0) { \
644 dest = src1 >> -tmp; \
645 } else { \
646 dest = src1 << tmp; \
647 }} while (0)
648NEON_VOP(shl_s8, neon_s8, 4)
649NEON_VOP(shl_s16, neon_s16, 2)
650NEON_VOP(shl_s32, neon_s32, 1)
651#undef NEON_FN
652
653uint64_t HELPER(neon_shl_s64)(uint64_t valop, uint64_t shiftop)
654{
655 int8_t shift = (int8_t)shiftop;
656 int64_t val = valop;
657 if (shift >= 64) {
658 val = 0;
659 } else if (shift <= -64) {
660 val >>= 63;
661 } else if (shift < 0) {
662 val >>= -shift;
663 } else {
664 val <<= shift;
665 }
666 return val;
667}
668
669#define NEON_FN(dest, src1, src2) do { \
670 int8_t tmp; \
671 tmp = (int8_t)src2; \
0670a7b6
PM		 672 if ((tmp >= (ssize_t)sizeof(src1) * 8) \
673 || (tmp <= -(ssize_t)sizeof(src1) * 8)) { \
ad69471c 674 dest = 0; \
ad69471c
PB		 675 } else if (tmp < 0) { \
676 dest = (src1 + (1 << (-1 - tmp))) >> -tmp; \
677 } else { \
678 dest = src1 << tmp; \
679 }} while (0)
680NEON_VOP(rshl_s8, neon_s8, 4)
681NEON_VOP(rshl_s16, neon_s16, 2)
ad69471c
PB		 682#undef NEON_FN
683
4bd4ee07 684/* The addition of the rounding constant may overflow, so we use an
b90372ad 685 * intermediate 64 bit accumulator. */
4bd4ee07
CL		 686uint32_t HELPER(neon_rshl_s32)(uint32_t valop, uint32_t shiftop)
687{
688 int32_t dest;
689 int32_t val = (int32_t)valop;
690 int8_t shift = (int8_t)shiftop;
691 if ((shift >= 32) || (shift <= -32)) {
692 dest = 0;
693 } else if (shift < 0) {
694 int64_t big_dest = ((int64_t)val + (1 << (-1 - shift)));
695 dest = big_dest >> -shift;
696 } else {
697 dest = val << shift;
698 }
699 return dest;
700}
701
b90372ad
PM		 702/* Handling addition overflow with 64 bit input values is more
703 * tricky than with 32 bit values. */
ad69471c
PB		 704uint64_t HELPER(neon_rshl_s64)(uint64_t valop, uint64_t shiftop)
705{
706 int8_t shift = (int8_t)shiftop;
707 int64_t val = valop;
0670a7b6 708 if ((shift >= 64) || (shift <= -64)) {
ad69471c 709 val = 0;
ad69471c 710 } else if (shift < 0) {
4bd4ee07
CL		 711 val >>= (-shift - 1);
712 if (val == INT64_MAX) {
713 /* In this case, it means that the rounding constant is 1,
714 * and the addition would overflow. Return the actual
715 * result directly. */
716 val = 0x4000000000000000LL;
717 } else {
718 val++;
719 val >>= 1;
720 }
ad69471c
PB		 721 } else {
722 val <<= shift;
723 }
724 return val;
725}
726
727#define NEON_FN(dest, src1, src2) do { \
728 int8_t tmp; \
729 tmp = (int8_t)src2; \
50f67e95
JR		 730 if (tmp >= (ssize_t)sizeof(src1) * 8 || \
731 tmp < -(ssize_t)sizeof(src1) * 8) { \
ad69471c 732 dest = 0; \
50f67e95 733 } else if (tmp == -(ssize_t)sizeof(src1) * 8) { \
b6c63b98 734 dest = src1 >> (-tmp - 1); \
ad69471c
PB		 735 } else if (tmp < 0) { \
736 dest = (src1 + (1 << (-1 - tmp))) >> -tmp; \
737 } else { \
738 dest = src1 << tmp; \
739 }} while (0)
740NEON_VOP(rshl_u8, neon_u8, 4)
741NEON_VOP(rshl_u16, neon_u16, 2)
ad69471c
PB		 742#undef NEON_FN
743
4bd4ee07 744/* The addition of the rounding constant may overflow, so we use an
b90372ad 745 * intermediate 64 bit accumulator. */
4bd4ee07
CL		 746uint32_t HELPER(neon_rshl_u32)(uint32_t val, uint32_t shiftop)
747{
748 uint32_t dest;
749 int8_t shift = (int8_t)shiftop;
750 if (shift >= 32 || shift < -32) {
751 dest = 0;
752 } else if (shift == -32) {
753 dest = val >> 31;
754 } else if (shift < 0) {
755 uint64_t big_dest = ((uint64_t)val + (1 << (-1 - shift)));
756 dest = big_dest >> -shift;
757 } else {
758 dest = val << shift;
759 }
760 return dest;
761}
762
b90372ad
PM		 763/* Handling addition overflow with 64 bit input values is more
764 * tricky than with 32 bit values. */
ad69471c
PB		 765uint64_t HELPER(neon_rshl_u64)(uint64_t val, uint64_t shiftop)
766{
767 int8_t shift = (uint8_t)shiftop;
51e3930f 768 if (shift >= 64 || shift < -64) {
ad69471c
PB		 769 val = 0;
770 } else if (shift == -64) {
771 /* Rounding a 1-bit result just preserves that bit. */
772 val >>= 63;
4bd4ee07
CL		 773 } else if (shift < 0) {
774 val >>= (-shift - 1);
775 if (val == UINT64_MAX) {
776 /* In this case, it means that the rounding constant is 1,
777 * and the addition would overflow. Return the actual
778 * result directly. */
779 val = 0x8000000000000000ULL;
780 } else {
781 val++;
782 val >>= 1;
783 }
ad69471c
PB		 784 } else {
785 val <<= shift;
786 }
787 return val;
788}
789
790#define NEON_FN(dest, src1, src2) do { \
791 int8_t tmp; \
792 tmp = (int8_t)src2; \
50f67e95 793 if (tmp >= (ssize_t)sizeof(src1) * 8) { \
ad69471c
PB		 794 if (src1) { \
795 SET_QC(); \
796 dest = ~0; \
797 } else { \
798 dest = 0; \
799 } \
50f67e95 800 } else if (tmp <= -(ssize_t)sizeof(src1) * 8) { \
ad69471c
PB		 801 dest = 0; \
802 } else if (tmp < 0) { \
803 dest = src1 >> -tmp; \
804 } else { \
805 dest = src1 << tmp; \
806 if ((dest >> tmp) != src1) { \
807 SET_QC(); \
808 dest = ~0; \
809 } \
810 }} while (0)
02da0b2d
PM		 811NEON_VOP_ENV(qshl_u8, neon_u8, 4)
812NEON_VOP_ENV(qshl_u16, neon_u16, 2)
813NEON_VOP_ENV(qshl_u32, neon_u32, 1)
ad69471c
PB		 814#undef NEON_FN
815
0ecb72a5 816uint64_t HELPER(neon_qshl_u64)(CPUARMState *env, uint64_t val, uint64_t shiftop)
ad69471c
PB		 817{
818 int8_t shift = (int8_t)shiftop;
819 if (shift >= 64) {
820 if (val) {
821 val = ~(uint64_t)0;
822 SET_QC();
ad69471c
PB		 823 }
824 } else if (shift <= -64) {
825 val = 0;
826 } else if (shift < 0) {
827 val >>= -shift;
828 } else {
829 uint64_t tmp = val;
830 val <<= shift;
831 if ((val >> shift) != tmp) {
832 SET_QC();
833 val = ~(uint64_t)0;
834 }
835 }
836 return val;
837}
838
839#define NEON_FN(dest, src1, src2) do { \
840 int8_t tmp; \
841 tmp = (int8_t)src2; \
50f67e95 842 if (tmp >= (ssize_t)sizeof(src1) * 8) { \
a5d88f3e 843 if (src1) { \
ad69471c 844 SET_QC(); \
a5d88f3e
PM		 845 dest = (uint32_t)(1 << (sizeof(src1) * 8 - 1)); \
846 if (src1 > 0) { \
847 dest--; \
848 } \
849 } else { \
850 dest = src1; \
851 } \
50f67e95 852 } else if (tmp <= -(ssize_t)sizeof(src1) * 8) { \
ad69471c
PB		 853 dest = src1 >> 31; \
854 } else if (tmp < 0) { \
855 dest = src1 >> -tmp; \
856 } else { \
857 dest = src1 << tmp; \
858 if ((dest >> tmp) != src1) { \
859 SET_QC(); \
a5d88f3e
PM		 860 dest = (uint32_t)(1 << (sizeof(src1) * 8 - 1)); \
861 if (src1 > 0) { \
862 dest--; \
863 } \
ad69471c
PB		 864 } \
865 }} while (0)
02da0b2d
PM		 866NEON_VOP_ENV(qshl_s8, neon_s8, 4)
867NEON_VOP_ENV(qshl_s16, neon_s16, 2)
868NEON_VOP_ENV(qshl_s32, neon_s32, 1)
ad69471c
PB		 869#undef NEON_FN
870
0ecb72a5 871uint64_t HELPER(neon_qshl_s64)(CPUARMState *env, uint64_t valop, uint64_t shiftop)
ad69471c
PB		 872{
873 int8_t shift = (uint8_t)shiftop;
874 int64_t val = valop;
875 if (shift >= 64) {
876 if (val) {
877 SET_QC();
eb7a3d79 878 val = (val >> 63) ^ ~SIGNBIT64;
ad69471c 879 }
4c9b70ae 880 } else if (shift <= -64) {
ad69471c
PB		 881 val >>= 63;
882 } else if (shift < 0) {
883 val >>= -shift;
884 } else {
885 int64_t tmp = val;
886 val <<= shift;
887 if ((val >> shift) != tmp) {
888 SET_QC();
889 val = (tmp >> 63) ^ ~SIGNBIT64;
890 }
891 }
892 return val;
893}
894
4ca4502c
JR		 895#define NEON_FN(dest, src1, src2) do { \
896 if (src1 & (1 << (sizeof(src1) * 8 - 1))) { \
897 SET_QC(); \
898 dest = 0; \
899 } else { \
900 int8_t tmp; \
901 tmp = (int8_t)src2; \
902 if (tmp >= (ssize_t)sizeof(src1) * 8) { \
903 if (src1) { \
904 SET_QC(); \
905 dest = ~0; \
906 } else { \
907 dest = 0; \
908 } \
909 } else if (tmp <= -(ssize_t)sizeof(src1) * 8) { \
910 dest = 0; \
911 } else if (tmp < 0) { \
912 dest = src1 >> -tmp; \
913 } else { \
914 dest = src1 << tmp; \
915 if ((dest >> tmp) != src1) { \
916 SET_QC(); \
917 dest = ~0; \
918 } \
919 } \
920 }} while (0)
02da0b2d
PM		 921NEON_VOP_ENV(qshlu_s8, neon_u8, 4)
922NEON_VOP_ENV(qshlu_s16, neon_u16, 2)
4ca4502c
JR		 923#undef NEON_FN
924
0ecb72a5 925uint32_t HELPER(neon_qshlu_s32)(CPUARMState *env, uint32_t valop, uint32_t shiftop)
4ca4502c
JR		 926{
927 if ((int32_t)valop < 0) {
928 SET_QC();
929 return 0;
930 }
02da0b2d 931 return helper_neon_qshl_u32(env, valop, shiftop);
4ca4502c
JR		 932}
933
0ecb72a5 934uint64_t HELPER(neon_qshlu_s64)(CPUARMState *env, uint64_t valop, uint64_t shiftop)
4ca4502c
JR		 935{
936 if ((int64_t)valop < 0) {
937 SET_QC();
938 return 0;
939 }
02da0b2d 940 return helper_neon_qshl_u64(env, valop, shiftop);
4ca4502c 941}
ad69471c 942
ad69471c
PB		 943#define NEON_FN(dest, src1, src2) do { \
944 int8_t tmp; \
945 tmp = (int8_t)src2; \
33ebc293
PM		 946 if (tmp >= (ssize_t)sizeof(src1) * 8) { \
947 if (src1) { \
948 SET_QC(); \
949 dest = ~0; \
950 } else { \
951 dest = 0; \
952 } \
953 } else if (tmp < -(ssize_t)sizeof(src1) * 8) { \
954 dest = 0; \
955 } else if (tmp == -(ssize_t)sizeof(src1) * 8) { \
956 dest = src1 >> (sizeof(src1) * 8 - 1); \
957 } else if (tmp < 0) { \
ad69471c
PB		 958 dest = (src1 + (1 << (-1 - tmp))) >> -tmp; \
959 } else { \
960 dest = src1 << tmp; \
961 if ((dest >> tmp) != src1) { \
962 SET_QC(); \
963 dest = ~0; \
964 } \
965 }} while (0)
02da0b2d
PM		 966NEON_VOP_ENV(qrshl_u8, neon_u8, 4)
967NEON_VOP_ENV(qrshl_u16, neon_u16, 2)
ad69471c
PB		 968#undef NEON_FN
969
4bd4ee07 970/* The addition of the rounding constant may overflow, so we use an
b90372ad 971 * intermediate 64 bit accumulator. */
0ecb72a5 972uint32_t HELPER(neon_qrshl_u32)(CPUARMState *env, uint32_t val, uint32_t shiftop)
4bd4ee07
CL		 973{
974 uint32_t dest;
975 int8_t shift = (int8_t)shiftop;
33ebc293
PM		 976 if (shift >= 32) {
977 if (val) {
978 SET_QC();
979 dest = ~0;
980 } else {
981 dest = 0;
982 }
983 } else if (shift < -32) {
984 dest = 0;
985 } else if (shift == -32) {
986 dest = val >> 31;
987 } else if (shift < 0) {
4bd4ee07
CL		 988 uint64_t big_dest = ((uint64_t)val + (1 << (-1 - shift)));
989 dest = big_dest >> -shift;
990 } else {
991 dest = val << shift;
992 if ((dest >> shift) != val) {
993 SET_QC();
994 dest = ~0;
995 }
996 }
997 return dest;
998}
999
b90372ad
PM		 1000/* Handling addition overflow with 64 bit input values is more
1001 * tricky than with 32 bit values. */
0ecb72a5 1002uint64_t HELPER(neon_qrshl_u64)(CPUARMState *env, uint64_t val, uint64_t shiftop)
ad69471c
PB		 1003{
1004 int8_t shift = (int8_t)shiftop;
33ebc293
PM		 1005 if (shift >= 64) {
1006 if (val) {
1007 SET_QC();
1008 val = ~0;
1009 }
1010 } else if (shift < -64) {
1011 val = 0;
1012 } else if (shift == -64) {
1013 val >>= 63;
1014 } else if (shift < 0) {
4bd4ee07
CL		 1015 val >>= (-shift - 1);
1016 if (val == UINT64_MAX) {
1017 /* In this case, it means that the rounding constant is 1,
1018 * and the addition would overflow. Return the actual
1019 * result directly. */
1020 val = 0x8000000000000000ULL;
1021 } else {
1022 val++;
1023 val >>= 1;
1024 }
ad69471c
PB		 1025 } else { \
1026 uint64_t tmp = val;
1027 val <<= shift;
1028 if ((val >> shift) != tmp) {
1029 SET_QC();
1030 val = ~0;
1031 }
1032 }
1033 return val;
1034}
1035
1036#define NEON_FN(dest, src1, src2) do { \
1037 int8_t tmp; \
1038 tmp = (int8_t)src2; \
7b6ecf5b
PM		 1039 if (tmp >= (ssize_t)sizeof(src1) * 8) { \
1040 if (src1) { \
1041 SET_QC(); \
6bbbb0ac 1042 dest = (typeof(dest))(1 << (sizeof(src1) * 8 - 1)); \
7b6ecf5b
PM		 1043 if (src1 > 0) { \
1044 dest--; \
1045 } \
1046 } else { \
1047 dest = 0; \
1048 } \
1049 } else if (tmp <= -(ssize_t)sizeof(src1) * 8) { \
1050 dest = 0; \
1051 } else if (tmp < 0) { \
ad69471c
PB		 1052 dest = (src1 + (1 << (-1 - tmp))) >> -tmp; \
1053 } else { \
1054 dest = src1 << tmp; \
1055 if ((dest >> tmp) != src1) { \
1056 SET_QC(); \
960e623b
PM		 1057 dest = (uint32_t)(1 << (sizeof(src1) * 8 - 1)); \
1058 if (src1 > 0) { \
1059 dest--; \
1060 } \
ad69471c
PB		 1061 } \
1062 }} while (0)
02da0b2d
PM		 1063NEON_VOP_ENV(qrshl_s8, neon_s8, 4)
1064NEON_VOP_ENV(qrshl_s16, neon_s16, 2)
ad69471c
PB		 1065#undef NEON_FN
1066
4bd4ee07 1067/* The addition of the rounding constant may overflow, so we use an
b90372ad 1068 * intermediate 64 bit accumulator. */
0ecb72a5 1069uint32_t HELPER(neon_qrshl_s32)(CPUARMState *env, uint32_t valop, uint32_t shiftop)
4bd4ee07
CL		 1070{
1071 int32_t dest;
1072 int32_t val = (int32_t)valop;
1073 int8_t shift = (int8_t)shiftop;
7b6ecf5b
PM		 1074 if (shift >= 32) {
1075 if (val) {
1076 SET_QC();
1077 dest = (val >> 31) ^ ~SIGNBIT;
1078 } else {
1079 dest = 0;
1080 }
1081 } else if (shift <= -32) {
1082 dest = 0;
1083 } else if (shift < 0) {
4bd4ee07
CL		 1084 int64_t big_dest = ((int64_t)val + (1 << (-1 - shift)));
1085 dest = big_dest >> -shift;
1086 } else {
1087 dest = val << shift;
1088 if ((dest >> shift) != val) {
1089 SET_QC();
1090 dest = (val >> 31) ^ ~SIGNBIT;
1091 }
1092 }
1093 return dest;
1094}
1095
b90372ad
PM		 1096/* Handling addition overflow with 64 bit input values is more
1097 * tricky than with 32 bit values. */
0ecb72a5 1098uint64_t HELPER(neon_qrshl_s64)(CPUARMState *env, uint64_t valop, uint64_t shiftop)
ad69471c
PB		 1099{
1100 int8_t shift = (uint8_t)shiftop;
1101 int64_t val = valop;
1102
7b6ecf5b
PM		 1103 if (shift >= 64) {
1104 if (val) {
1105 SET_QC();
1106 val = (val >> 63) ^ ~SIGNBIT64;
1107 }
1108 } else if (shift <= -64) {
1109 val = 0;
1110 } else if (shift < 0) {
4bd4ee07
CL		 1111 val >>= (-shift - 1);
1112 if (val == INT64_MAX) {
1113 /* In this case, it means that the rounding constant is 1,
1114 * and the addition would overflow. Return the actual
1115 * result directly. */
1116 val = 0x4000000000000000ULL;
1117 } else {
1118 val++;
1119 val >>= 1;
1120 }
ad69471c 1121 } else {
4bd4ee07 1122 int64_t tmp = val;
ad69471c
PB		 1123 val <<= shift;
1124 if ((val >> shift) != tmp) {
1125 SET_QC();
4bd4ee07 1126 val = (tmp >> 63) ^ ~SIGNBIT64;
ad69471c
PB		 1127 }
1128 }
1129 return val;
1130}
1131
1132uint32_t HELPER(neon_add_u8)(uint32_t a, uint32_t b)
1133{
1134 uint32_t mask;
1135 mask = (a ^ b) & 0x80808080u;
1136 a &= ~0x80808080u;
1137 b &= ~0x80808080u;
1138 return (a + b) ^ mask;
1139}
1140
1141uint32_t HELPER(neon_add_u16)(uint32_t a, uint32_t b)
1142{
1143 uint32_t mask;
1144 mask = (a ^ b) & 0x80008000u;
1145 a &= ~0x80008000u;
1146 b &= ~0x80008000u;
1147 return (a + b) ^ mask;
1148}
1149
1150#define NEON_FN(dest, src1, src2) dest = src1 + src2
1151NEON_POP(padd_u8, neon_u8, 4)
1152NEON_POP(padd_u16, neon_u16, 2)
1153#undef NEON_FN
1154
1155#define NEON_FN(dest, src1, src2) dest = src1 - src2
1156NEON_VOP(sub_u8, neon_u8, 4)
1157NEON_VOP(sub_u16, neon_u16, 2)
1158#undef NEON_FN
1159
1160#define NEON_FN(dest, src1, src2) dest = src1 * src2
1161NEON_VOP(mul_u8, neon_u8, 4)
1162NEON_VOP(mul_u16, neon_u16, 2)
1163#undef NEON_FN
1164
1654b2d6 1165/* Polynomial multiplication is like integer multiplication except the
ad69471c
PB		 1166 partial products are XORed, not added. */
1167uint32_t HELPER(neon_mul_p8)(uint32_t op1, uint32_t op2)
1168{
1169 uint32_t mask;
1170 uint32_t result;
1171 result = 0;
1172 while (op1) {
1173 mask = 0;
1174 if (op1 & 1)
1175 mask |= 0xff;
1176 if (op1 & (1 << 8))
1177 mask |= (0xff << 8);
1178 if (op1 & (1 << 16))
1179 mask |= (0xff << 16);
1180 if (op1 & (1 << 24))
1181 mask |= (0xff << 24);
1182 result ^= op2 & mask;
1183 op1 = (op1 >> 1) & 0x7f7f7f7f;
1184 op2 = (op2 << 1) & 0xfefefefe;
1185 }
1186 return result;
1187}
1188
e5ca24cb
PM		 1189uint64_t HELPER(neon_mull_p8)(uint32_t op1, uint32_t op2)
1190{
1191 uint64_t result = 0;
1192 uint64_t mask;
1193 uint64_t op2ex = op2;
1194 op2ex = (op2ex & 0xff) |
1195 ((op2ex & 0xff00) << 8) |
1196 ((op2ex & 0xff0000) << 16) |
1197 ((op2ex & 0xff000000) << 24);
1198 while (op1) {
1199 mask = 0;
1200 if (op1 & 1) {
1201 mask |= 0xffff;
1202 }
1203 if (op1 & (1 << 8)) {
1204 mask |= (0xffffU << 16);
1205 }
1206 if (op1 & (1 << 16)) {
1207 mask |= (0xffffULL << 32);
1208 }
1209 if (op1 & (1 << 24)) {
1210 mask |= (0xffffULL << 48);
1211 }
1212 result ^= op2ex & mask;
1213 op1 = (op1 >> 1) & 0x7f7f7f7f;
1214 op2ex <<= 1;
1215 }
1216 return result;
1217}
1218
ad69471c
PB		 1219#define NEON_FN(dest, src1, src2) dest = (src1 & src2) ? -1 : 0
1220NEON_VOP(tst_u8, neon_u8, 4)
1221NEON_VOP(tst_u16, neon_u16, 2)
1222NEON_VOP(tst_u32, neon_u32, 1)
1223#undef NEON_FN
1224
1225#define NEON_FN(dest, src1, src2) dest = (src1 == src2) ? -1 : 0
1226NEON_VOP(ceq_u8, neon_u8, 4)
1227NEON_VOP(ceq_u16, neon_u16, 2)
1228NEON_VOP(ceq_u32, neon_u32, 1)
1229#undef NEON_FN
1230
ad69471c
PB		 1231/* Count Leading Sign/Zero Bits. */
1232static inline int do_clz8(uint8_t x)
1233{
1234 int n;
1235 for (n = 8; x; n--)
1236 x >>= 1;
1237 return n;
1238}
1239
1240static inline int do_clz16(uint16_t x)
1241{
1242 int n;
1243 for (n = 16; x; n--)
1244 x >>= 1;
1245 return n;
1246}
1247
1248#define NEON_FN(dest, src, dummy) dest = do_clz8(src)
1249NEON_VOP1(clz_u8, neon_u8, 4)
1250#undef NEON_FN
1251
1252#define NEON_FN(dest, src, dummy) dest = do_clz16(src)
1253NEON_VOP1(clz_u16, neon_u16, 2)
1254#undef NEON_FN
1255
1256#define NEON_FN(dest, src, dummy) dest = do_clz8((src < 0) ? ~src : src) - 1
1257NEON_VOP1(cls_s8, neon_s8, 4)
1258#undef NEON_FN
1259
1260#define NEON_FN(dest, src, dummy) dest = do_clz16((src < 0) ? ~src : src) - 1
1261NEON_VOP1(cls_s16, neon_s16, 2)
1262#undef NEON_FN
1263
1264uint32_t HELPER(neon_cls_s32)(uint32_t x)
1265{
1266 int count;
1267 if ((int32_t)x < 0)
1268 x = ~x;
1269 for (count = 32; x; count--)
1270 x = x >> 1;
1271 return count - 1;
1272}
1273
1274/* Bit count. */
1275uint32_t HELPER(neon_cnt_u8)(uint32_t x)
1276{
1277 x = (x & 0x55555555) + ((x >> 1) & 0x55555555);
1278 x = (x & 0x33333333) + ((x >> 2) & 0x33333333);
1279 x = (x & 0x0f0f0f0f) + ((x >> 4) & 0x0f0f0f0f);
1280 return x;
1281}
1282
86cbc418
PM		 1283/* Reverse bits in each 8 bit word */
1284uint32_t HELPER(neon_rbit_u8)(uint32_t x)
1285{
1286 x = ((x & 0xf0f0f0f0) >> 4)
1287 | ((x & 0x0f0f0f0f) << 4);
1288 x = ((x & 0x88888888) >> 3)
1289 | ((x & 0x44444444) >> 1)
1290 | ((x & 0x22222222) << 1)
1291 | ((x & 0x11111111) << 3);
1292 return x;
1293}
1294
ad69471c
PB		 1295#define NEON_QDMULH16(dest, src1, src2, round) do { \
1296 uint32_t tmp = (int32_t)(int16_t) src1 * (int16_t) src2; \
1297 if ((tmp ^ (tmp << 1)) & SIGNBIT) { \
1298 SET_QC(); \
1299 tmp = (tmp >> 31) ^ ~SIGNBIT; \
46eece9d
JR		 1300 } else { \
1301 tmp <<= 1; \
ad69471c 1302 } \
ad69471c
PB		 1303 if (round) { \
1304 int32_t old = tmp; \
1305 tmp += 1 << 15; \
1306 if ((int32_t)tmp < old) { \
1307 SET_QC(); \
1308 tmp = SIGNBIT - 1; \
1309 } \
1310 } \
1311 dest = tmp >> 16; \
1312 } while(0)
1313#define NEON_FN(dest, src1, src2) NEON_QDMULH16(dest, src1, src2, 0)
02da0b2d 1314NEON_VOP_ENV(qdmulh_s16, neon_s16, 2)
ad69471c
PB		 1315#undef NEON_FN
1316#define NEON_FN(dest, src1, src2) NEON_QDMULH16(dest, src1, src2, 1)
02da0b2d 1317NEON_VOP_ENV(qrdmulh_s16, neon_s16, 2)
ad69471c
PB		 1318#undef NEON_FN
1319#undef NEON_QDMULH16
1320
1321#define NEON_QDMULH32(dest, src1, src2, round) do { \
1322 uint64_t tmp = (int64_t)(int32_t) src1 * (int32_t) src2; \
1323 if ((tmp ^ (tmp << 1)) & SIGNBIT64) { \
1324 SET_QC(); \
1325 tmp = (tmp >> 63) ^ ~SIGNBIT64; \
1326 } else { \
1327 tmp <<= 1; \
1328 } \
1329 if (round) { \
1330 int64_t old = tmp; \
1331 tmp += (int64_t)1 << 31; \
1332 if ((int64_t)tmp < old) { \
1333 SET_QC(); \
1334 tmp = SIGNBIT64 - 1; \
1335 } \
1336 } \
1337 dest = tmp >> 32; \
1338 } while(0)
1339#define NEON_FN(dest, src1, src2) NEON_QDMULH32(dest, src1, src2, 0)
02da0b2d 1340NEON_VOP_ENV(qdmulh_s32, neon_s32, 1)
ad69471c
PB		 1341#undef NEON_FN
1342#define NEON_FN(dest, src1, src2) NEON_QDMULH32(dest, src1, src2, 1)
02da0b2d 1343NEON_VOP_ENV(qrdmulh_s32, neon_s32, 1)
ad69471c
PB		 1344#undef NEON_FN
1345#undef NEON_QDMULH32
1346
1347uint32_t HELPER(neon_narrow_u8)(uint64_t x)
1348{
1349 return (x & 0xffu) | ((x >> 8) & 0xff00u) | ((x >> 16) & 0xff0000u)
1350 | ((x >> 24) & 0xff000000u);
1351}
1352
1353uint32_t HELPER(neon_narrow_u16)(uint64_t x)
1354{
1355 return (x & 0xffffu) | ((x >> 16) & 0xffff0000u);
1356}
1357
1358uint32_t HELPER(neon_narrow_high_u8)(uint64_t x)
1359{
1360 return ((x >> 8) & 0xff) | ((x >> 16) & 0xff00)
1361 | ((x >> 24) & 0xff0000) | ((x >> 32) & 0xff000000);
1362}
1363
1364uint32_t HELPER(neon_narrow_high_u16)(uint64_t x)
1365{
1366 return ((x >> 16) & 0xffff) | ((x >> 32) & 0xffff0000);
1367}
1368
1369uint32_t HELPER(neon_narrow_round_high_u8)(uint64_t x)
1370{
1371 x &= 0xff80ff80ff80ff80ull;
1372 x += 0x0080008000800080ull;
1373 return ((x >> 8) & 0xff) | ((x >> 16) & 0xff00)
1374 | ((x >> 24) & 0xff0000) | ((x >> 32) & 0xff000000);
1375}
1376
1377uint32_t HELPER(neon_narrow_round_high_u16)(uint64_t x)
1378{
1379 x &= 0xffff8000ffff8000ull;
1380 x += 0x0000800000008000ull;
1381 return ((x >> 16) & 0xffff) | ((x >> 32) & 0xffff0000);
1382}
1383
0ecb72a5 1384uint32_t HELPER(neon_unarrow_sat8)(CPUARMState *env, uint64_t x)
af1bbf30
JR		 1385{
1386 uint16_t s;
1387 uint8_t d;
1388 uint32_t res = 0;
1389#define SAT8(n) \
1390 s = x >> n; \
1391 if (s & 0x8000) { \
1392 SET_QC(); \
1393 } else { \
1394 if (s > 0xff) { \
1395 d = 0xff; \
1396 SET_QC(); \
1397 } else { \
1398 d = s; \
1399 } \
1400 res |= (uint32_t)d << (n / 2); \
1401 }
1402
1403 SAT8(0);
1404 SAT8(16);
1405 SAT8(32);
1406 SAT8(48);
1407#undef SAT8
1408 return res;
1409}
1410
0ecb72a5 1411uint32_t HELPER(neon_narrow_sat_u8)(CPUARMState *env, uint64_t x)
ad69471c
PB		 1412{
1413 uint16_t s;
1414 uint8_t d;
1415 uint32_t res = 0;
1416#define SAT8(n) \
1417 s = x >> n; \
1418 if (s > 0xff) { \
1419 d = 0xff; \
1420 SET_QC(); \
1421 } else { \
1422 d = s; \
1423 } \
1424 res |= (uint32_t)d << (n / 2);
1425
1426 SAT8(0);
1427 SAT8(16);
1428 SAT8(32);
1429 SAT8(48);
1430#undef SAT8
1431 return res;
1432}
1433
0ecb72a5 1434uint32_t HELPER(neon_narrow_sat_s8)(CPUARMState *env, uint64_t x)
ad69471c
PB		 1435{
1436 int16_t s;
1437 uint8_t d;
1438 uint32_t res = 0;
1439#define SAT8(n) \
1440 s = x >> n; \
1441 if (s != (int8_t)s) { \
1442 d = (s >> 15) ^ 0x7f; \
1443 SET_QC(); \
1444 } else { \
1445 d = s; \
1446 } \
1447 res |= (uint32_t)d << (n / 2);
1448
1449 SAT8(0);
1450 SAT8(16);
1451 SAT8(32);
1452 SAT8(48);
1453#undef SAT8
1454 return res;
1455}
1456
0ecb72a5 1457uint32_t HELPER(neon_unarrow_sat16)(CPUARMState *env, uint64_t x)
af1bbf30
JR		 1458{
1459 uint32_t high;
1460 uint32_t low;
1461 low = x;
1462 if (low & 0x80000000) {
1463 low = 0;
1464 SET_QC();
1465 } else if (low > 0xffff) {
1466 low = 0xffff;
1467 SET_QC();
1468 }
1469 high = x >> 32;
1470 if (high & 0x80000000) {
1471 high = 0;
1472 SET_QC();
1473 } else if (high > 0xffff) {
1474 high = 0xffff;
1475 SET_QC();
1476 }
1477 return low | (high << 16);
1478}
1479
0ecb72a5 1480uint32_t HELPER(neon_narrow_sat_u16)(CPUARMState *env, uint64_t x)
ad69471c
PB		 1481{
1482 uint32_t high;
1483 uint32_t low;
1484 low = x;
1485 if (low > 0xffff) {
1486 low = 0xffff;
1487 SET_QC();
1488 }
1489 high = x >> 32;
1490 if (high > 0xffff) {
1491 high = 0xffff;
1492 SET_QC();
1493 }
1494 return low | (high << 16);
1495}
1496
0ecb72a5 1497uint32_t HELPER(neon_narrow_sat_s16)(CPUARMState *env, uint64_t x)
ad69471c
PB		 1498{
1499 int32_t low;
1500 int32_t high;
1501 low = x;
1502 if (low != (int16_t)low) {
1503 low = (low >> 31) ^ 0x7fff;
1504 SET_QC();
1505 }
1506 high = x >> 32;
1507 if (high != (int16_t)high) {
1508 high = (high >> 31) ^ 0x7fff;
1509 SET_QC();
1510 }
1511 return (uint16_t)low | (high << 16);
1512}
1513
0ecb72a5 1514uint32_t HELPER(neon_unarrow_sat32)(CPUARMState *env, uint64_t x)
af1bbf30
JR		 1515{
1516 if (x & 0x8000000000000000ull) {
1517 SET_QC();
1518 return 0;
1519 }
1520 if (x > 0xffffffffu) {
1521 SET_QC();
1522 return 0xffffffffu;
1523 }
1524 return x;
1525}
1526
0ecb72a5 1527uint32_t HELPER(neon_narrow_sat_u32)(CPUARMState *env, uint64_t x)
ad69471c
PB		 1528{
1529 if (x > 0xffffffffu) {
1530 SET_QC();
1531 return 0xffffffffu;
1532 }
1533 return x;
1534}
1535
0ecb72a5 1536uint32_t HELPER(neon_narrow_sat_s32)(CPUARMState *env, uint64_t x)
ad69471c
PB		 1537{
1538 if ((int64_t)x != (int32_t)x) {
1539 SET_QC();
cc2212c2 1540 return ((int64_t)x >> 63) ^ 0x7fffffff;
ad69471c
PB		 1541 }
1542 return x;
1543}
1544
1545uint64_t HELPER(neon_widen_u8)(uint32_t x)
1546{
1547 uint64_t tmp;
1548 uint64_t ret;
1549 ret = (uint8_t)x;
1550 tmp = (uint8_t)(x >> 8);
1551 ret |= tmp << 16;
1552 tmp = (uint8_t)(x >> 16);
1553 ret |= tmp << 32;
1554 tmp = (uint8_t)(x >> 24);
1555 ret |= tmp << 48;
1556 return ret;
1557}
1558
1559uint64_t HELPER(neon_widen_s8)(uint32_t x)
1560{
1561 uint64_t tmp;
1562 uint64_t ret;
1563 ret = (uint16_t)(int8_t)x;
1564 tmp = (uint16_t)(int8_t)(x >> 8);
1565 ret |= tmp << 16;
1566 tmp = (uint16_t)(int8_t)(x >> 16);
1567 ret |= tmp << 32;
1568 tmp = (uint16_t)(int8_t)(x >> 24);
1569 ret |= tmp << 48;
1570 return ret;
1571}
1572
1573uint64_t HELPER(neon_widen_u16)(uint32_t x)
1574{
1575 uint64_t high = (uint16_t)(x >> 16);
1576 return ((uint16_t)x) | (high << 32);
1577}
1578
1579uint64_t HELPER(neon_widen_s16)(uint32_t x)
1580{
1581 uint64_t high = (int16_t)(x >> 16);
1582 return ((uint32_t)(int16_t)x) | (high << 32);
1583}
1584
1585uint64_t HELPER(neon_addl_u16)(uint64_t a, uint64_t b)
1586{
1587 uint64_t mask;
1588 mask = (a ^ b) & 0x8000800080008000ull;
1589 a &= ~0x8000800080008000ull;
1590 b &= ~0x8000800080008000ull;
1591 return (a + b) ^ mask;
1592}
1593
1594uint64_t HELPER(neon_addl_u32)(uint64_t a, uint64_t b)
1595{
1596 uint64_t mask;
1597 mask = (a ^ b) & 0x8000000080000000ull;
1598 a &= ~0x8000000080000000ull;
1599 b &= ~0x8000000080000000ull;
1600 return (a + b) ^ mask;
1601}
1602
1603uint64_t HELPER(neon_paddl_u16)(uint64_t a, uint64_t b)
1604{
1605 uint64_t tmp;
1606 uint64_t tmp2;
1607
1608 tmp = a & 0x0000ffff0000ffffull;
1609 tmp += (a >> 16) & 0x0000ffff0000ffffull;
1610 tmp2 = b & 0xffff0000ffff0000ull;
1611 tmp2 += (b << 16) & 0xffff0000ffff0000ull;
1612 return ( tmp & 0xffff)
1613 | ((tmp >> 16) & 0xffff0000ull)
1614 | ((tmp2 << 16) & 0xffff00000000ull)
1615 | ( tmp2 & 0xffff000000000000ull);
1616}
1617
1618uint64_t HELPER(neon_paddl_u32)(uint64_t a, uint64_t b)
1619{
1620 uint32_t low = a + (a >> 32);
1621 uint32_t high = b + (b >> 32);
1622 return low + ((uint64_t)high << 32);
1623}
1624
1625uint64_t HELPER(neon_subl_u16)(uint64_t a, uint64_t b)
1626{
1627 uint64_t mask;
1628 mask = (a ^ ~b) & 0x8000800080008000ull;
1629 a |= 0x8000800080008000ull;
1630 b &= ~0x8000800080008000ull;
1631 return (a - b) ^ mask;
1632}
1633
1634uint64_t HELPER(neon_subl_u32)(uint64_t a, uint64_t b)
1635{
1636 uint64_t mask;
1637 mask = (a ^ ~b) & 0x8000000080000000ull;
1638 a |= 0x8000000080000000ull;
1639 b &= ~0x8000000080000000ull;
1640 return (a - b) ^ mask;
1641}
1642
0ecb72a5 1643uint64_t HELPER(neon_addl_saturate_s32)(CPUARMState *env, uint64_t a, uint64_t b)
ad69471c
PB		 1644{
1645 uint32_t x, y;
1646 uint32_t low, high;
1647
1648 x = a;
1649 y = b;
1650 low = x + y;
1651 if (((low ^ x) & SIGNBIT) && !((x ^ y) & SIGNBIT)) {
1652 SET_QC();
1653 low = ((int32_t)x >> 31) ^ ~SIGNBIT;
1654 }
1655 x = a >> 32;
1656 y = b >> 32;
1657 high = x + y;
1658 if (((high ^ x) & SIGNBIT) && !((x ^ y) & SIGNBIT)) {
1659 SET_QC();
1660 high = ((int32_t)x >> 31) ^ ~SIGNBIT;
1661 }
1662 return low | ((uint64_t)high << 32);
1663}
1664
0ecb72a5 1665uint64_t HELPER(neon_addl_saturate_s64)(CPUARMState *env, uint64_t a, uint64_t b)
ad69471c
PB		 1666{
1667 uint64_t result;
1668
1669 result = a + b;
1670 if (((result ^ a) & SIGNBIT64) && !((a ^ b) & SIGNBIT64)) {
1671 SET_QC();
1672 result = ((int64_t)a >> 63) ^ ~SIGNBIT64;
1673 }
1674 return result;
1675}
1676
4d9ad7f7
PM		 1677/* We have to do the arithmetic in a larger type than
1678 * the input type, because for example with a signed 32 bit
1679 * op the absolute difference can overflow a signed 32 bit value.
1680 */
1681#define DO_ABD(dest, x, y, intype, arithtype) do { \
1682 arithtype tmp_x = (intype)(x); \
1683 arithtype tmp_y = (intype)(y); \
ad69471c
PB		 1684 dest = ((tmp_x > tmp_y) ? tmp_x - tmp_y : tmp_y - tmp_x); \
1685 } while(0)
1686
1687uint64_t HELPER(neon_abdl_u16)(uint32_t a, uint32_t b)
1688{
1689 uint64_t tmp;
1690 uint64_t result;
4d9ad7f7
PM		 1691 DO_ABD(result, a, b, uint8_t, uint32_t);
1692 DO_ABD(tmp, a >> 8, b >> 8, uint8_t, uint32_t);
ad69471c 1693 result |= tmp << 16;
4d9ad7f7 1694 DO_ABD(tmp, a >> 16, b >> 16, uint8_t, uint32_t);
ad69471c 1695 result |= tmp << 32;
4d9ad7f7 1696 DO_ABD(tmp, a >> 24, b >> 24, uint8_t, uint32_t);
ad69471c
PB		 1697 result |= tmp << 48;
1698 return result;
1699}
1700
1701uint64_t HELPER(neon_abdl_s16)(uint32_t a, uint32_t b)
1702{
1703 uint64_t tmp;
1704 uint64_t result;
4d9ad7f7
PM		 1705 DO_ABD(result, a, b, int8_t, int32_t);
1706 DO_ABD(tmp, a >> 8, b >> 8, int8_t, int32_t);
ad69471c 1707 result |= tmp << 16;
4d9ad7f7 1708 DO_ABD(tmp, a >> 16, b >> 16, int8_t, int32_t);
ad69471c 1709 result |= tmp << 32;
4d9ad7f7 1710 DO_ABD(tmp, a >> 24, b >> 24, int8_t, int32_t);
ad69471c
PB		 1711 result |= tmp << 48;
1712 return result;
1713}
1714
1715uint64_t HELPER(neon_abdl_u32)(uint32_t a, uint32_t b)
1716{
1717 uint64_t tmp;
1718 uint64_t result;
4d9ad7f7
PM		 1719 DO_ABD(result, a, b, uint16_t, uint32_t);
1720 DO_ABD(tmp, a >> 16, b >> 16, uint16_t, uint32_t);
ad69471c
PB		 1721 return result | (tmp << 32);
1722}
1723
1724uint64_t HELPER(neon_abdl_s32)(uint32_t a, uint32_t b)
1725{
1726 uint64_t tmp;
1727 uint64_t result;
4d9ad7f7
PM		 1728 DO_ABD(result, a, b, int16_t, int32_t);
1729 DO_ABD(tmp, a >> 16, b >> 16, int16_t, int32_t);
ad69471c
PB		 1730 return result | (tmp << 32);
1731}
1732
1733uint64_t HELPER(neon_abdl_u64)(uint32_t a, uint32_t b)
1734{
1735 uint64_t result;
4d9ad7f7 1736 DO_ABD(result, a, b, uint32_t, uint64_t);
ad69471c
PB		 1737 return result;
1738}
1739
1740uint64_t HELPER(neon_abdl_s64)(uint32_t a, uint32_t b)
1741{
1742 uint64_t result;
4d9ad7f7 1743 DO_ABD(result, a, b, int32_t, int64_t);
ad69471c
PB		 1744 return result;
1745}
1746#undef DO_ABD
1747
1748/* Widening multiply. Named type is the source type. */
1749#define DO_MULL(dest, x, y, type1, type2) do { \
1750 type1 tmp_x = x; \
1751 type1 tmp_y = y; \
1752 dest = (type2)((type2)tmp_x * (type2)tmp_y); \
1753 } while(0)
1754
1755uint64_t HELPER(neon_mull_u8)(uint32_t a, uint32_t b)
1756{
1757 uint64_t tmp;
1758 uint64_t result;
1759
1760 DO_MULL(result, a, b, uint8_t, uint16_t);
1761 DO_MULL(tmp, a >> 8, b >> 8, uint8_t, uint16_t);
1762 result |= tmp << 16;
1763 DO_MULL(tmp, a >> 16, b >> 16, uint8_t, uint16_t);
1764 result |= tmp << 32;
1765 DO_MULL(tmp, a >> 24, b >> 24, uint8_t, uint16_t);
1766 result |= tmp << 48;
1767 return result;
1768}
1769
1770uint64_t HELPER(neon_mull_s8)(uint32_t a, uint32_t b)
1771{
1772 uint64_t tmp;
1773 uint64_t result;
1774
1775 DO_MULL(result, a, b, int8_t, uint16_t);
1776 DO_MULL(tmp, a >> 8, b >> 8, int8_t, uint16_t);
1777 result |= tmp << 16;
1778 DO_MULL(tmp, a >> 16, b >> 16, int8_t, uint16_t);
1779 result |= tmp << 32;
1780 DO_MULL(tmp, a >> 24, b >> 24, int8_t, uint16_t);
1781 result |= tmp << 48;
1782 return result;
1783}
1784
1785uint64_t HELPER(neon_mull_u16)(uint32_t a, uint32_t b)
1786{
1787 uint64_t tmp;
1788 uint64_t result;
1789
1790 DO_MULL(result, a, b, uint16_t, uint32_t);
1791 DO_MULL(tmp, a >> 16, b >> 16, uint16_t, uint32_t);
1792 return result | (tmp << 32);
1793}
1794
1795uint64_t HELPER(neon_mull_s16)(uint32_t a, uint32_t b)
1796{
1797 uint64_t tmp;
1798 uint64_t result;
1799
1800 DO_MULL(result, a, b, int16_t, uint32_t);
1801 DO_MULL(tmp, a >> 16, b >> 16, int16_t, uint32_t);
1802 return result | (tmp << 32);
1803}
1804
1805uint64_t HELPER(neon_negl_u16)(uint64_t x)
1806{
1807 uint16_t tmp;
1808 uint64_t result;
1809 result = (uint16_t)-x;
1810 tmp = -(x >> 16);
1811 result |= (uint64_t)tmp << 16;
1812 tmp = -(x >> 32);
1813 result |= (uint64_t)tmp << 32;
1814 tmp = -(x >> 48);
1815 result |= (uint64_t)tmp << 48;
1816 return result;
1817}
1818
ad69471c
PB		 1819uint64_t HELPER(neon_negl_u32)(uint64_t x)
1820{
1821 uint32_t low = -x;
1822 uint32_t high = -(x >> 32);
1823 return low | ((uint64_t)high << 32);
1824}
1825
b90372ad 1826/* Saturating sign manipulation. */
ad69471c
PB		 1827/* ??? Make these use NEON_VOP1 */
1828#define DO_QABS8(x) do { \
1829 if (x == (int8_t)0x80) { \
1830 x = 0x7f; \
1831 SET_QC(); \
1832 } else if (x < 0) { \
1833 x = -x; \
1834 }} while (0)
0ecb72a5 1835uint32_t HELPER(neon_qabs_s8)(CPUARMState *env, uint32_t x)
ad69471c
PB		 1836{
1837 neon_s8 vec;
1838 NEON_UNPACK(neon_s8, vec, x);
1839 DO_QABS8(vec.v1);
1840 DO_QABS8(vec.v2);
1841 DO_QABS8(vec.v3);
1842 DO_QABS8(vec.v4);
1843 NEON_PACK(neon_s8, x, vec);
1844 return x;
1845}
1846#undef DO_QABS8
1847
1848#define DO_QNEG8(x) do { \
1849 if (x == (int8_t)0x80) { \
1850 x = 0x7f; \
1851 SET_QC(); \
1852 } else { \
1853 x = -x; \
1854 }} while (0)
0ecb72a5 1855uint32_t HELPER(neon_qneg_s8)(CPUARMState *env, uint32_t x)
ad69471c
PB		 1856{
1857 neon_s8 vec;
1858 NEON_UNPACK(neon_s8, vec, x);
1859 DO_QNEG8(vec.v1);
1860 DO_QNEG8(vec.v2);
1861 DO_QNEG8(vec.v3);
1862 DO_QNEG8(vec.v4);
1863 NEON_PACK(neon_s8, x, vec);
1864 return x;
1865}
1866#undef DO_QNEG8
1867
1868#define DO_QABS16(x) do { \
1869 if (x == (int16_t)0x8000) { \
1870 x = 0x7fff; \
1871 SET_QC(); \
1872 } else if (x < 0) { \
1873 x = -x; \
1874 }} while (0)
0ecb72a5 1875uint32_t HELPER(neon_qabs_s16)(CPUARMState *env, uint32_t x)
ad69471c
PB		 1876{
1877 neon_s16 vec;
1878 NEON_UNPACK(neon_s16, vec, x);
1879 DO_QABS16(vec.v1);
1880 DO_QABS16(vec.v2);
1881 NEON_PACK(neon_s16, x, vec);
1882 return x;
1883}
1884#undef DO_QABS16
1885
1886#define DO_QNEG16(x) do { \
1887 if (x == (int16_t)0x8000) { \
1888 x = 0x7fff; \
1889 SET_QC(); \
1890 } else { \
1891 x = -x; \
1892 }} while (0)
0ecb72a5 1893uint32_t HELPER(neon_qneg_s16)(CPUARMState *env, uint32_t x)
ad69471c
PB		 1894{
1895 neon_s16 vec;
1896 NEON_UNPACK(neon_s16, vec, x);
1897 DO_QNEG16(vec.v1);
1898 DO_QNEG16(vec.v2);
1899 NEON_PACK(neon_s16, x, vec);
1900 return x;
1901}
1902#undef DO_QNEG16
1903
0ecb72a5 1904uint32_t HELPER(neon_qabs_s32)(CPUARMState *env, uint32_t x)
ad69471c
PB		 1905{
1906 if (x == SIGNBIT) {
1907 SET_QC();
1908 x = ~SIGNBIT;
1909 } else if ((int32_t)x < 0) {
1910 x = -x;
1911 }
1912 return x;
1913}
1914
0ecb72a5 1915uint32_t HELPER(neon_qneg_s32)(CPUARMState *env, uint32_t x)
ad69471c
PB		 1916{
1917 if (x == SIGNBIT) {
1918 SET_QC();
1919 x = ~SIGNBIT;
1920 } else {
1921 x = -x;
1922 }
1923 return x;
1924}
1925
0a79bc87
AB		 1926uint64_t HELPER(neon_qabs_s64)(CPUARMState *env, uint64_t x)
1927{
1928 if (x == SIGNBIT64) {
1929 SET_QC();
1930 x = ~SIGNBIT64;
1931 } else if ((int64_t)x < 0) {
1932 x = -x;
1933 }
1934 return x;
1935}
1936
1937uint64_t HELPER(neon_qneg_s64)(CPUARMState *env, uint64_t x)
1938{
1939 if (x == SIGNBIT64) {
1940 SET_QC();
1941 x = ~SIGNBIT64;
1942 } else {
1943 x = -x;
1944 }
1945 return x;
1946}
1947
ad69471c 1948/* NEON Float helpers. */
aa47cfdd 1949uint32_t HELPER(neon_abd_f32)(uint32_t a, uint32_t b, void *fpstp)
ad69471c 1950{
aa47cfdd 1951 float_status *fpst = fpstp;
51d85267
PM		 1952 float32 f0 = make_float32(a);
1953 float32 f1 = make_float32(b);
aa47cfdd 1954 return float32_val(float32_abs(float32_sub(f0, f1, fpst)));
ad69471c
PB		 1955}
1956
cab565c4
PM		 1957/* Floating point comparisons produce an integer result.
1958 * Note that EQ doesn't signal InvalidOp for QNaNs but GE and GT do.
1959 * Softfloat routines return 0/1, which we convert to the 0/-1 Neon requires.
1960 */
aa47cfdd 1961uint32_t HELPER(neon_ceq_f32)(uint32_t a, uint32_t b, void *fpstp)
cab565c4 1962{
aa47cfdd
PM		 1963 float_status *fpst = fpstp;
1964 return -float32_eq_quiet(make_float32(a), make_float32(b), fpst);
cab565c4
PM		 1965}
1966
aa47cfdd 1967uint32_t HELPER(neon_cge_f32)(uint32_t a, uint32_t b, void *fpstp)
cab565c4 1968{
aa47cfdd
PM		 1969 float_status *fpst = fpstp;
1970 return -float32_le(make_float32(b), make_float32(a), fpst);
ad69471c
PB		 1971}
1972
aa47cfdd 1973uint32_t HELPER(neon_cgt_f32)(uint32_t a, uint32_t b, void *fpstp)
cab565c4 1974{
aa47cfdd
PM		 1975 float_status *fpst = fpstp;
1976 return -float32_lt(make_float32(b), make_float32(a), fpst);
cab565c4 1977}
ad69471c 1978
aa47cfdd 1979uint32_t HELPER(neon_acge_f32)(uint32_t a, uint32_t b, void *fpstp)
ad69471c 1980{
aa47cfdd 1981 float_status *fpst = fpstp;
51d85267
PM		 1982 float32 f0 = float32_abs(make_float32(a));
1983 float32 f1 = float32_abs(make_float32(b));
aa47cfdd 1984 return -float32_le(f1, f0, fpst);
ad69471c
PB		 1985}
1986
aa47cfdd 1987uint32_t HELPER(neon_acgt_f32)(uint32_t a, uint32_t b, void *fpstp)
ad69471c 1988{
aa47cfdd 1989 float_status *fpst = fpstp;
51d85267
PM		 1990 float32 f0 = float32_abs(make_float32(a));
1991 float32 f1 = float32_abs(make_float32(b));
aa47cfdd 1992 return -float32_lt(f1, f0, fpst);
ad69471c 1993}
02acedf9 1994
057d5f62
PM		 1995uint64_t HELPER(neon_acge_f64)(uint64_t a, uint64_t b, void *fpstp)
1996{
1997 float_status *fpst = fpstp;
1998 float64 f0 = float64_abs(make_float64(a));
1999 float64 f1 = float64_abs(make_float64(b));
2000 return -float64_le(f1, f0, fpst);
2001}
2002
2003uint64_t HELPER(neon_acgt_f64)(uint64_t a, uint64_t b, void *fpstp)
2004{
2005 float_status *fpst = fpstp;
2006 float64 f0 = float64_abs(make_float64(a));
2007 float64 f1 = float64_abs(make_float64(b));
2008 return -float64_lt(f1, f0, fpst);
2009}
2010
02acedf9
PM		 2011#define ELEM(V, N, SIZE) (((V) >> ((N) * (SIZE))) & ((1ull << (SIZE)) - 1))
2012
b13708bb 2013void HELPER(neon_qunzip8)(void *vd, void *vm)
02acedf9 2014{
b13708bb
RH		 2015 uint64_t *rd = vd, *rm = vm;
2016 uint64_t zd0 = rd[0], zd1 = rd[1];
2017 uint64_t zm0 = rm[0], zm1 = rm[1];
2018
02acedf9
PM		 2019 uint64_t d0 = ELEM(zd0, 0, 8) | (ELEM(zd0, 2, 8) << 8)
2020 | (ELEM(zd0, 4, 8) << 16) | (ELEM(zd0, 6, 8) << 24)
2021 | (ELEM(zd1, 0, 8) << 32) | (ELEM(zd1, 2, 8) << 40)
2022 | (ELEM(zd1, 4, 8) << 48) | (ELEM(zd1, 6, 8) << 56);
2023 uint64_t d1 = ELEM(zm0, 0, 8) | (ELEM(zm0, 2, 8) << 8)
2024 | (ELEM(zm0, 4, 8) << 16) | (ELEM(zm0, 6, 8) << 24)
2025 | (ELEM(zm1, 0, 8) << 32) | (ELEM(zm1, 2, 8) << 40)
2026 | (ELEM(zm1, 4, 8) << 48) | (ELEM(zm1, 6, 8) << 56);
2027 uint64_t m0 = ELEM(zd0, 1, 8) | (ELEM(zd0, 3, 8) << 8)
2028 | (ELEM(zd0, 5, 8) << 16) | (ELEM(zd0, 7, 8) << 24)
2029 | (ELEM(zd1, 1, 8) << 32) | (ELEM(zd1, 3, 8) << 40)
2030 | (ELEM(zd1, 5, 8) << 48) | (ELEM(zd1, 7, 8) << 56);
2031 uint64_t m1 = ELEM(zm0, 1, 8) | (ELEM(zm0, 3, 8) << 8)
2032 | (ELEM(zm0, 5, 8) << 16) | (ELEM(zm0, 7, 8) << 24)
2033 | (ELEM(zm1, 1, 8) << 32) | (ELEM(zm1, 3, 8) << 40)
2034 | (ELEM(zm1, 5, 8) << 48) | (ELEM(zm1, 7, 8) << 56);
b13708bb
RH		 2035
2036 rm[0] = m0;
2037 rm[1] = m1;
2038 rd[0] = d0;
2039 rd[1] = d1;
02acedf9
PM		 2040}
2041
b13708bb 2042void HELPER(neon_qunzip16)(void *vd, void *vm)
02acedf9 2043{
b13708bb
RH		 2044 uint64_t *rd = vd, *rm = vm;
2045 uint64_t zd0 = rd[0], zd1 = rd[1];
2046 uint64_t zm0 = rm[0], zm1 = rm[1];
2047
02acedf9
PM		 2048 uint64_t d0 = ELEM(zd0, 0, 16) | (ELEM(zd0, 2, 16) << 16)
2049 | (ELEM(zd1, 0, 16) << 32) | (ELEM(zd1, 2, 16) << 48);
2050 uint64_t d1 = ELEM(zm0, 0, 16) | (ELEM(zm0, 2, 16) << 16)
2051 | (ELEM(zm1, 0, 16) << 32) | (ELEM(zm1, 2, 16) << 48);
2052 uint64_t m0 = ELEM(zd0, 1, 16) | (ELEM(zd0, 3, 16) << 16)
2053 | (ELEM(zd1, 1, 16) << 32) | (ELEM(zd1, 3, 16) << 48);
2054 uint64_t m1 = ELEM(zm0, 1, 16) | (ELEM(zm0, 3, 16) << 16)
2055 | (ELEM(zm1, 1, 16) << 32) | (ELEM(zm1, 3, 16) << 48);
b13708bb
RH		 2056
2057 rm[0] = m0;
2058 rm[1] = m1;
2059 rd[0] = d0;
2060 rd[1] = d1;
02acedf9
PM		 2061}
2062
b13708bb 2063void HELPER(neon_qunzip32)(void *vd, void *vm)
02acedf9 2064{
b13708bb
RH		 2065 uint64_t *rd = vd, *rm = vm;
2066 uint64_t zd0 = rd[0], zd1 = rd[1];
2067 uint64_t zm0 = rm[0], zm1 = rm[1];
2068
02acedf9
PM		 2069 uint64_t d0 = ELEM(zd0, 0, 32) | (ELEM(zd1, 0, 32) << 32);
2070 uint64_t d1 = ELEM(zm0, 0, 32) | (ELEM(zm1, 0, 32) << 32);
2071 uint64_t m0 = ELEM(zd0, 1, 32) | (ELEM(zd1, 1, 32) << 32);
2072 uint64_t m1 = ELEM(zm0, 1, 32) | (ELEM(zm1, 1, 32) << 32);
b13708bb
RH		 2073
2074 rm[0] = m0;
2075 rm[1] = m1;
2076 rd[0] = d0;
2077 rd[1] = d1;
02acedf9
PM		 2078}
2079
b13708bb 2080void HELPER(neon_unzip8)(void *vd, void *vm)
02acedf9 2081{
b13708bb
RH		 2082 uint64_t *rd = vd, *rm = vm;
2083 uint64_t zd = rd[0], zm = rm[0];
2084
02acedf9
PM		 2085 uint64_t d0 = ELEM(zd, 0, 8) | (ELEM(zd, 2, 8) << 8)
2086 | (ELEM(zd, 4, 8) << 16) | (ELEM(zd, 6, 8) << 24)
2087 | (ELEM(zm, 0, 8) << 32) | (ELEM(zm, 2, 8) << 40)
2088 | (ELEM(zm, 4, 8) << 48) | (ELEM(zm, 6, 8) << 56);
2089 uint64_t m0 = ELEM(zd, 1, 8) | (ELEM(zd, 3, 8) << 8)
2090 | (ELEM(zd, 5, 8) << 16) | (ELEM(zd, 7, 8) << 24)
2091 | (ELEM(zm, 1, 8) << 32) | (ELEM(zm, 3, 8) << 40)
2092 | (ELEM(zm, 5, 8) << 48) | (ELEM(zm, 7, 8) << 56);
b13708bb
RH		 2093
2094 rm[0] = m0;
2095 rd[0] = d0;
02acedf9
PM		 2096}
2097
b13708bb 2098void HELPER(neon_unzip16)(void *vd, void *vm)
02acedf9 2099{
b13708bb
RH		 2100 uint64_t *rd = vd, *rm = vm;
2101 uint64_t zd = rd[0], zm = rm[0];
2102
02acedf9
PM		 2103 uint64_t d0 = ELEM(zd, 0, 16) | (ELEM(zd, 2, 16) << 16)
2104 | (ELEM(zm, 0, 16) << 32) | (ELEM(zm, 2, 16) << 48);
2105 uint64_t m0 = ELEM(zd, 1, 16) | (ELEM(zd, 3, 16) << 16)
2106 | (ELEM(zm, 1, 16) << 32) | (ELEM(zm, 3, 16) << 48);
b13708bb
RH		 2107
2108 rm[0] = m0;
2109 rd[0] = d0;
02acedf9 2110}
d68a6f3a 2111
b13708bb 2112void HELPER(neon_qzip8)(void *vd, void *vm)
d68a6f3a 2113{
b13708bb
RH		 2114 uint64_t *rd = vd, *rm = vm;
2115 uint64_t zd0 = rd[0], zd1 = rd[1];
2116 uint64_t zm0 = rm[0], zm1 = rm[1];
2117
d68a6f3a
PM		 2118 uint64_t d0 = ELEM(zd0, 0, 8) | (ELEM(zm0, 0, 8) << 8)
2119 | (ELEM(zd0, 1, 8) << 16) | (ELEM(zm0, 1, 8) << 24)
2120 | (ELEM(zd0, 2, 8) << 32) | (ELEM(zm0, 2, 8) << 40)
2121 | (ELEM(zd0, 3, 8) << 48) | (ELEM(zm0, 3, 8) << 56);
2122 uint64_t d1 = ELEM(zd0, 4, 8) | (ELEM(zm0, 4, 8) << 8)
2123 | (ELEM(zd0, 5, 8) << 16) | (ELEM(zm0, 5, 8) << 24)
2124 | (ELEM(zd0, 6, 8) << 32) | (ELEM(zm0, 6, 8) << 40)
2125 | (ELEM(zd0, 7, 8) << 48) | (ELEM(zm0, 7, 8) << 56);
2126 uint64_t m0 = ELEM(zd1, 0, 8) | (ELEM(zm1, 0, 8) << 8)
2127 | (ELEM(zd1, 1, 8) << 16) | (ELEM(zm1, 1, 8) << 24)
2128 | (ELEM(zd1, 2, 8) << 32) | (ELEM(zm1, 2, 8) << 40)
2129 | (ELEM(zd1, 3, 8) << 48) | (ELEM(zm1, 3, 8) << 56);
2130 uint64_t m1 = ELEM(zd1, 4, 8) | (ELEM(zm1, 4, 8) << 8)
2131 | (ELEM(zd1, 5, 8) << 16) | (ELEM(zm1, 5, 8) << 24)
2132 | (ELEM(zd1, 6, 8) << 32) | (ELEM(zm1, 6, 8) << 40)
2133 | (ELEM(zd1, 7, 8) << 48) | (ELEM(zm1, 7, 8) << 56);
b13708bb
RH		 2134
2135 rm[0] = m0;
2136 rm[1] = m1;
2137 rd[0] = d0;
2138 rd[1] = d1;
d68a6f3a
PM		 2139}
2140
b13708bb 2141void HELPER(neon_qzip16)(void *vd, void *vm)
d68a6f3a 2142{
b13708bb
RH		 2143 uint64_t *rd = vd, *rm = vm;
2144 uint64_t zd0 = rd[0], zd1 = rd[1];
2145 uint64_t zm0 = rm[0], zm1 = rm[1];
2146
d68a6f3a
PM		 2147 uint64_t d0 = ELEM(zd0, 0, 16) | (ELEM(zm0, 0, 16) << 16)
2148 | (ELEM(zd0, 1, 16) << 32) | (ELEM(zm0, 1, 16) << 48);
2149 uint64_t d1 = ELEM(zd0, 2, 16) | (ELEM(zm0, 2, 16) << 16)
2150 | (ELEM(zd0, 3, 16) << 32) | (ELEM(zm0, 3, 16) << 48);
2151 uint64_t m0 = ELEM(zd1, 0, 16) | (ELEM(zm1, 0, 16) << 16)
2152 | (ELEM(zd1, 1, 16) << 32) | (ELEM(zm1, 1, 16) << 48);
2153 uint64_t m1 = ELEM(zd1, 2, 16) | (ELEM(zm1, 2, 16) << 16)
2154 | (ELEM(zd1, 3, 16) << 32) | (ELEM(zm1, 3, 16) << 48);
b13708bb
RH		 2155
2156 rm[0] = m0;
2157 rm[1] = m1;
2158 rd[0] = d0;
2159 rd[1] = d1;
d68a6f3a
PM		 2160}
2161
b13708bb 2162void HELPER(neon_qzip32)(void *vd, void *vm)
d68a6f3a 2163{
b13708bb
RH		 2164 uint64_t *rd = vd, *rm = vm;
2165 uint64_t zd0 = rd[0], zd1 = rd[1];
2166 uint64_t zm0 = rm[0], zm1 = rm[1];
2167
d68a6f3a
PM		 2168 uint64_t d0 = ELEM(zd0, 0, 32) | (ELEM(zm0, 0, 32) << 32);
2169 uint64_t d1 = ELEM(zd0, 1, 32) | (ELEM(zm0, 1, 32) << 32);
2170 uint64_t m0 = ELEM(zd1, 0, 32) | (ELEM(zm1, 0, 32) << 32);
2171 uint64_t m1 = ELEM(zd1, 1, 32) | (ELEM(zm1, 1, 32) << 32);
b13708bb
RH		 2172
2173 rm[0] = m0;
2174 rm[1] = m1;
2175 rd[0] = d0;
2176 rd[1] = d1;
d68a6f3a
PM		 2177}
2178
b13708bb 2179void HELPER(neon_zip8)(void *vd, void *vm)
d68a6f3a 2180{
b13708bb
RH		 2181 uint64_t *rd = vd, *rm = vm;
2182 uint64_t zd = rd[0], zm = rm[0];
2183
d68a6f3a
PM		 2184 uint64_t d0 = ELEM(zd, 0, 8) | (ELEM(zm, 0, 8) << 8)
2185 | (ELEM(zd, 1, 8) << 16) | (ELEM(zm, 1, 8) << 24)
2186 | (ELEM(zd, 2, 8) << 32) | (ELEM(zm, 2, 8) << 40)
2187 | (ELEM(zd, 3, 8) << 48) | (ELEM(zm, 3, 8) << 56);
2188 uint64_t m0 = ELEM(zd, 4, 8) | (ELEM(zm, 4, 8) << 8)
2189 | (ELEM(zd, 5, 8) << 16) | (ELEM(zm, 5, 8) << 24)
2190 | (ELEM(zd, 6, 8) << 32) | (ELEM(zm, 6, 8) << 40)
2191 | (ELEM(zd, 7, 8) << 48) | (ELEM(zm, 7, 8) << 56);
b13708bb
RH		 2192
2193 rm[0] = m0;
2194 rd[0] = d0;
d68a6f3a
PM		 2195}
2196
b13708bb 2197void HELPER(neon_zip16)(void *vd, void *vm)
d68a6f3a 2198{
b13708bb
RH		 2199 uint64_t *rd = vd, *rm = vm;
2200 uint64_t zd = rd[0], zm = rm[0];
2201
d68a6f3a
PM		 2202 uint64_t d0 = ELEM(zd, 0, 16) | (ELEM(zm, 0, 16) << 16)
2203 | (ELEM(zd, 1, 16) << 32) | (ELEM(zm, 1, 16) << 48);
2204 uint64_t m0 = ELEM(zd, 2, 16) | (ELEM(zm, 2, 16) << 16)
2205 | (ELEM(zd, 3, 16) << 32) | (ELEM(zm, 3, 16) << 48);
b13708bb
RH		 2206
2207 rm[0] = m0;
2208 rd[0] = d0;
d68a6f3a 2209}
4e624eda
PM		 2210
2211/* Helper function for 64 bit polynomial multiply case:
2212 * perform PolynomialMult(op1, op2) and return either the top or
2213 * bottom half of the 128 bit result.
2214 */
2215uint64_t HELPER(neon_pmull_64_lo)(uint64_t op1, uint64_t op2)
2216{
2217 int bitnum;
2218 uint64_t res = 0;
2219
2220 for (bitnum = 0; bitnum < 64; bitnum++) {
2221 if (op1 & (1ULL << bitnum)) {
2222 res ^= op2 << bitnum;
2223 }
2224 }
2225 return res;
2226}
2227uint64_t HELPER(neon_pmull_64_hi)(uint64_t op1, uint64_t op2)
2228{
2229 int bitnum;
2230 uint64_t res = 0;
2231
2232 /* bit 0 of op1 can't influence the high 64 bits at all */
2233 for (bitnum = 1; bitnum < 64; bitnum++) {
2234 if (op1 & (1ULL << bitnum)) {
2235 res ^= op2 >> (64 - bitnum);
2236 }
2237 }
2238 return res;
2239}
Empty description
This page took 1.159017 seconds and 4 git commands to generate.