Git Repo - qemu.git/blame_incremental

... / ...

Commit	Line	Data
	1	/*
	2	* MMX/3DNow!/SSE/SSE2/SSE3/SSSE3/SSE4/PNI support
	3	*
	4	* Copyright (c) 2005 Fabrice Bellard
	5	* Copyright (c) 2008 Intel Corporation <[email protected]>
	6	*
	7	* This library is free software; you can redistribute it and/or
	8	* modify it under the terms of the GNU Lesser General Public
	9	* License as published by the Free Software Foundation; either
	10	* version 2 of the License, or (at your option) any later version.
	11	*
	12	* This library is distributed in the hope that it will be useful,
	13	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	14	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	15	* Lesser General Public License for more details.
	16	*
	17	* You should have received a copy of the GNU Lesser General Public
	18	* License along with this library; if not, see <http://www.gnu.org/licenses/>.
	19	*/
	20
	21	#include "crypto/aes.h"
	22
	23	#if SHIFT == 0
	24	#define Reg MMXReg
	25	#define XMM_ONLY(...)
	26	#define B(n) MMX_B(n)
	27	#define W(n) MMX_W(n)
	28	#define L(n) MMX_L(n)
	29	#define Q(n) q
	30	#define SUFFIX _mmx
	31	#else
	32	#define Reg XMMReg
	33	#define XMM_ONLY(...) __VA_ARGS__
	34	#define B(n) XMM_B(n)
	35	#define W(n) XMM_W(n)
	36	#define L(n) XMM_L(n)
	37	#define Q(n) XMM_Q(n)
	38	#define SUFFIX _xmm
	39	#endif
	40
	41	void glue(helper_psrlw, SUFFIX)(CPUX86State env, Reg d, Reg *s)
	42	{
	43	int shift;
	44
	45	if (s->Q(0) > 15) {
	46	d->Q(0) = 0;
	47	#if SHIFT == 1
	48	d->Q(1) = 0;
	49	#endif
	50	} else {
	51	shift = s->B(0);
	52	d->W(0) >>= shift;
	53	d->W(1) >>= shift;
	54	d->W(2) >>= shift;
	55	d->W(3) >>= shift;
	56	#if SHIFT == 1
	57	d->W(4) >>= shift;
	58	d->W(5) >>= shift;
	59	d->W(6) >>= shift;
	60	d->W(7) >>= shift;
	61	#endif
	62	}
	63	}
	64
	65	void glue(helper_psraw, SUFFIX)(CPUX86State env, Reg d, Reg *s)
	66	{
	67	int shift;
	68
	69	if (s->Q(0) > 15) {
	70	shift = 15;
	71	} else {
	72	shift = s->B(0);
	73	}
	74	d->W(0) = (int16_t)d->W(0) >> shift;
	75	d->W(1) = (int16_t)d->W(1) >> shift;
	76	d->W(2) = (int16_t)d->W(2) >> shift;
	77	d->W(3) = (int16_t)d->W(3) >> shift;
	78	#if SHIFT == 1
	79	d->W(4) = (int16_t)d->W(4) >> shift;
	80	d->W(5) = (int16_t)d->W(5) >> shift;
	81	d->W(6) = (int16_t)d->W(6) >> shift;
	82	d->W(7) = (int16_t)d->W(7) >> shift;
	83	#endif
	84	}
	85
	86	void glue(helper_psllw, SUFFIX)(CPUX86State env, Reg d, Reg *s)
	87	{
	88	int shift;
	89
	90	if (s->Q(0) > 15) {
	91	d->Q(0) = 0;
	92	#if SHIFT == 1
	93	d->Q(1) = 0;
	94	#endif
	95	} else {
	96	shift = s->B(0);
	97	d->W(0) <<= shift;
	98	d->W(1) <<= shift;
	99	d->W(2) <<= shift;
	100	d->W(3) <<= shift;
	101	#if SHIFT == 1
	102	d->W(4) <<= shift;
	103	d->W(5) <<= shift;
	104	d->W(6) <<= shift;
	105	d->W(7) <<= shift;
	106	#endif
	107	}
	108	}
	109
	110	void glue(helper_psrld, SUFFIX)(CPUX86State env, Reg d, Reg *s)
	111	{
	112	int shift;
	113
	114	if (s->Q(0) > 31) {
	115	d->Q(0) = 0;
	116	#if SHIFT == 1
	117	d->Q(1) = 0;
	118	#endif
	119	} else {
	120	shift = s->B(0);
	121	d->L(0) >>= shift;
	122	d->L(1) >>= shift;
	123	#if SHIFT == 1
	124	d->L(2) >>= shift;
	125	d->L(3) >>= shift;
	126	#endif
	127	}
	128	}
	129
	130	void glue(helper_psrad, SUFFIX)(CPUX86State env, Reg d, Reg *s)
	131	{
	132	int shift;
	133
	134	if (s->Q(0) > 31) {
	135	shift = 31;
	136	} else {
	137	shift = s->B(0);
	138	}
	139	d->L(0) = (int32_t)d->L(0) >> shift;
	140	d->L(1) = (int32_t)d->L(1) >> shift;
	141	#if SHIFT == 1
	142	d->L(2) = (int32_t)d->L(2) >> shift;
	143	d->L(3) = (int32_t)d->L(3) >> shift;
	144	#endif
	145	}
	146
	147	void glue(helper_pslld, SUFFIX)(CPUX86State env, Reg d, Reg *s)
	148	{
	149	int shift;
	150
	151	if (s->Q(0) > 31) {
	152	d->Q(0) = 0;
	153	#if SHIFT == 1
	154	d->Q(1) = 0;
	155	#endif
	156	} else {
	157	shift = s->B(0);
	158	d->L(0) <<= shift;
	159	d->L(1) <<= shift;
	160	#if SHIFT == 1
	161	d->L(2) <<= shift;
	162	d->L(3) <<= shift;
	163	#endif
	164	}
	165	}
	166
	167	void glue(helper_psrlq, SUFFIX)(CPUX86State env, Reg d, Reg *s)
	168	{
	169	int shift;
	170
	171	if (s->Q(0) > 63) {
	172	d->Q(0) = 0;
	173	#if SHIFT == 1
	174	d->Q(1) = 0;
	175	#endif
	176	} else {
	177	shift = s->B(0);
	178	d->Q(0) >>= shift;
	179	#if SHIFT == 1
	180	d->Q(1) >>= shift;
	181	#endif
	182	}
	183	}
	184
	185	void glue(helper_psllq, SUFFIX)(CPUX86State env, Reg d, Reg *s)
	186	{
	187	int shift;
	188
	189	if (s->Q(0) > 63) {
	190	d->Q(0) = 0;
	191	#if SHIFT == 1
	192	d->Q(1) = 0;
	193	#endif
	194	} else {
	195	shift = s->B(0);
	196	d->Q(0) <<= shift;
	197	#if SHIFT == 1
	198	d->Q(1) <<= shift;
	199	#endif
	200	}
	201	}
	202
	203	#if SHIFT == 1
	204	void glue(helper_psrldq, SUFFIX)(CPUX86State env, Reg d, Reg *s)
	205	{
	206	int shift, i;
	207
	208	shift = s->L(0);
	209	if (shift > 16) {
	210	shift = 16;
	211	}
	212	for (i = 0; i < 16 - shift; i++) {
	213	d->B(i) = d->B(i + shift);
	214	}
	215	for (i = 16 - shift; i < 16; i++) {
	216	d->B(i) = 0;
	217	}
	218	}
	219
	220	void glue(helper_pslldq, SUFFIX)(CPUX86State env, Reg d, Reg *s)
	221	{
	222	int shift, i;
	223
	224	shift = s->L(0);
	225	if (shift > 16) {
	226	shift = 16;
	227	}
	228	for (i = 15; i >= shift; i--) {
	229	d->B(i) = d->B(i - shift);
	230	}
	231	for (i = 0; i < shift; i++) {
	232	d->B(i) = 0;
	233	}
	234	}
	235	#endif
	236
	237	#define SSE_HELPER_B(name, F) \
	238	void glue(name, SUFFIX)(CPUX86State env, Reg d, Reg *s) \
	239	{ \
	240	d->B(0) = F(d->B(0), s->B(0)); \
	241	d->B(1) = F(d->B(1), s->B(1)); \
	242	d->B(2) = F(d->B(2), s->B(2)); \
	243	d->B(3) = F(d->B(3), s->B(3)); \
	244	d->B(4) = F(d->B(4), s->B(4)); \
	245	d->B(5) = F(d->B(5), s->B(5)); \
	246	d->B(6) = F(d->B(6), s->B(6)); \
	247	d->B(7) = F(d->B(7), s->B(7)); \
	248	XMM_ONLY( \
	249	d->B(8) = F(d->B(8), s->B(8)); \
	250	d->B(9) = F(d->B(9), s->B(9)); \
	251	d->B(10) = F(d->B(10), s->B(10)); \
	252	d->B(11) = F(d->B(11), s->B(11)); \
	253	d->B(12) = F(d->B(12), s->B(12)); \
	254	d->B(13) = F(d->B(13), s->B(13)); \
	255	d->B(14) = F(d->B(14), s->B(14)); \
	256	d->B(15) = F(d->B(15), s->B(15)); \
	257	) \
	258	}
	259
	260	#define SSE_HELPER_W(name, F) \
	261	void glue(name, SUFFIX)(CPUX86State env, Reg d, Reg *s) \
	262	{ \
	263	d->W(0) = F(d->W(0), s->W(0)); \
	264	d->W(1) = F(d->W(1), s->W(1)); \
	265	d->W(2) = F(d->W(2), s->W(2)); \
	266	d->W(3) = F(d->W(3), s->W(3)); \
	267	XMM_ONLY( \
	268	d->W(4) = F(d->W(4), s->W(4)); \
	269	d->W(5) = F(d->W(5), s->W(5)); \
	270	d->W(6) = F(d->W(6), s->W(6)); \
	271	d->W(7) = F(d->W(7), s->W(7)); \
	272	) \
	273	}
	274
	275	#define SSE_HELPER_L(name, F) \
	276	void glue(name, SUFFIX)(CPUX86State env, Reg d, Reg *s) \
	277	{ \
	278	d->L(0) = F(d->L(0), s->L(0)); \
	279	d->L(1) = F(d->L(1), s->L(1)); \
	280	XMM_ONLY( \
	281	d->L(2) = F(d->L(2), s->L(2)); \
	282	d->L(3) = F(d->L(3), s->L(3)); \
	283	) \
	284	}
	285
	286	#define SSE_HELPER_Q(name, F) \
	287	void glue(name, SUFFIX)(CPUX86State env, Reg d, Reg *s) \
	288	{ \
	289	d->Q(0) = F(d->Q(0), s->Q(0)); \
	290	XMM_ONLY( \
	291	d->Q(1) = F(d->Q(1), s->Q(1)); \
	292	) \
	293	}
	294
	295	#if SHIFT == 0
	296	static inline int satub(int x)
	297	{
	298	if (x < 0) {
	299	return 0;
	300	} else if (x > 255) {
	301	return 255;
	302	} else {
	303	return x;
	304	}
	305	}
	306
	307	static inline int satuw(int x)
	308	{
	309	if (x < 0) {
	310	return 0;
	311	} else if (x > 65535) {
	312	return 65535;
	313	} else {
	314	return x;
	315	}
	316	}
	317
	318	static inline int satsb(int x)
	319	{
	320	if (x < -128) {
	321	return -128;
	322	} else if (x > 127) {
	323	return 127;
	324	} else {
	325	return x;
	326	}
	327	}
	328
	329	static inline int satsw(int x)
	330	{
	331	if (x < -32768) {
	332	return -32768;
	333	} else if (x > 32767) {
	334	return 32767;
	335	} else {
	336	return x;
	337	}
	338	}
	339
	340	#define FADD(a, b) ((a) + (b))
	341	#define FADDUB(a, b) satub((a) + (b))
	342	#define FADDUW(a, b) satuw((a) + (b))
	343	#define FADDSB(a, b) satsb((int8_t)(a) + (int8_t)(b))
	344	#define FADDSW(a, b) satsw((int16_t)(a) + (int16_t)(b))
	345
	346	#define FSUB(a, b) ((a) - (b))
	347	#define FSUBUB(a, b) satub((a) - (b))
	348	#define FSUBUW(a, b) satuw((a) - (b))
	349	#define FSUBSB(a, b) satsb((int8_t)(a) - (int8_t)(b))
	350	#define FSUBSW(a, b) satsw((int16_t)(a) - (int16_t)(b))
	351	#define FMINUB(a, b) ((a) < (b)) ? (a) : (b)
	352	#define FMINSW(a, b) ((int16_t)(a) < (int16_t)(b)) ? (a) : (b)
	353	#define FMAXUB(a, b) ((a) > (b)) ? (a) : (b)
	354	#define FMAXSW(a, b) ((int16_t)(a) > (int16_t)(b)) ? (a) : (b)
	355
	356	#define FAND(a, b) ((a) & (b))
	357	#define FANDN(a, b) ((~(a)) & (b))
	358	#define FOR(a, b) ((a) \| (b))
	359	#define FXOR(a, b) ((a) ^ (b))
	360
	361	#define FCMPGTB(a, b) ((int8_t)(a) > (int8_t)(b) ? -1 : 0)
	362	#define FCMPGTW(a, b) ((int16_t)(a) > (int16_t)(b) ? -1 : 0)
	363	#define FCMPGTL(a, b) ((int32_t)(a) > (int32_t)(b) ? -1 : 0)
	364	#define FCMPEQ(a, b) ((a) == (b) ? -1 : 0)
	365
	366	#define FMULLW(a, b) ((a) * (b))
	367	#define FMULHRW(a, b) (((int16_t)(a) * (int16_t)(b) + 0x8000) >> 16)
	368	#define FMULHUW(a, b) ((a) * (b) >> 16)
	369	#define FMULHW(a, b) ((int16_t)(a) * (int16_t)(b) >> 16)
	370
	371	#define FAVG(a, b) (((a) + (b) + 1) >> 1)
	372	#endif
	373
	374	SSE_HELPER_B(helper_paddb, FADD)
	375	SSE_HELPER_W(helper_paddw, FADD)
	376	SSE_HELPER_L(helper_paddl, FADD)
	377	SSE_HELPER_Q(helper_paddq, FADD)
	378
	379	SSE_HELPER_B(helper_psubb, FSUB)
	380	SSE_HELPER_W(helper_psubw, FSUB)
	381	SSE_HELPER_L(helper_psubl, FSUB)
	382	SSE_HELPER_Q(helper_psubq, FSUB)
	383
	384	SSE_HELPER_B(helper_paddusb, FADDUB)
	385	SSE_HELPER_B(helper_paddsb, FADDSB)
	386	SSE_HELPER_B(helper_psubusb, FSUBUB)
	387	SSE_HELPER_B(helper_psubsb, FSUBSB)
	388
	389	SSE_HELPER_W(helper_paddusw, FADDUW)
	390	SSE_HELPER_W(helper_paddsw, FADDSW)
	391	SSE_HELPER_W(helper_psubusw, FSUBUW)
	392	SSE_HELPER_W(helper_psubsw, FSUBSW)
	393
	394	SSE_HELPER_B(helper_pminub, FMINUB)
	395	SSE_HELPER_B(helper_pmaxub, FMAXUB)
	396
	397	SSE_HELPER_W(helper_pminsw, FMINSW)
	398	SSE_HELPER_W(helper_pmaxsw, FMAXSW)
	399
	400	SSE_HELPER_Q(helper_pand, FAND)
	401	SSE_HELPER_Q(helper_pandn, FANDN)
	402	SSE_HELPER_Q(helper_por, FOR)
	403	SSE_HELPER_Q(helper_pxor, FXOR)
	404
	405	SSE_HELPER_B(helper_pcmpgtb, FCMPGTB)
	406	SSE_HELPER_W(helper_pcmpgtw, FCMPGTW)
	407	SSE_HELPER_L(helper_pcmpgtl, FCMPGTL)
	408
	409	SSE_HELPER_B(helper_pcmpeqb, FCMPEQ)
	410	SSE_HELPER_W(helper_pcmpeqw, FCMPEQ)
	411	SSE_HELPER_L(helper_pcmpeql, FCMPEQ)
	412
	413	SSE_HELPER_W(helper_pmullw, FMULLW)
	414	#if SHIFT == 0
	415	SSE_HELPER_W(helper_pmulhrw, FMULHRW)
	416	#endif
	417	SSE_HELPER_W(helper_pmulhuw, FMULHUW)
	418	SSE_HELPER_W(helper_pmulhw, FMULHW)
	419
	420	SSE_HELPER_B(helper_pavgb, FAVG)
	421	SSE_HELPER_W(helper_pavgw, FAVG)
	422
	423	void glue(helper_pmuludq, SUFFIX)(CPUX86State env, Reg d, Reg *s)
	424	{
	425	d->Q(0) = (uint64_t)s->L(0) * (uint64_t)d->L(0);
	426	#if SHIFT == 1
	427	d->Q(1) = (uint64_t)s->L(2) * (uint64_t)d->L(2);
	428	#endif
	429	}
	430
	431	void glue(helper_pmaddwd, SUFFIX)(CPUX86State env, Reg d, Reg *s)
	432	{
	433	int i;
	434
	435	for (i = 0; i < (2 << SHIFT); i++) {
	436	d->L(i) = (int16_t)s->W(2 * i) * (int16_t)d->W(2 * i) +
	437	(int16_t)s->W(2 * i + 1) * (int16_t)d->W(2 * i + 1);
	438	}
	439	}
	440
	441	#if SHIFT == 0
	442	static inline int abs1(int a)
	443	{
	444	if (a < 0) {
	445	return -a;
	446	} else {
	447	return a;
	448	}
	449	}
	450	#endif
	451	void glue(helper_psadbw, SUFFIX)(CPUX86State env, Reg d, Reg *s)
	452	{
	453	unsigned int val;
	454
	455	val = 0;
	456	val += abs1(d->B(0) - s->B(0));
	457	val += abs1(d->B(1) - s->B(1));
	458	val += abs1(d->B(2) - s->B(2));
	459	val += abs1(d->B(3) - s->B(3));
	460	val += abs1(d->B(4) - s->B(4));
	461	val += abs1(d->B(5) - s->B(5));
	462	val += abs1(d->B(6) - s->B(6));
	463	val += abs1(d->B(7) - s->B(7));
	464	d->Q(0) = val;
	465	#if SHIFT == 1
	466	val = 0;
	467	val += abs1(d->B(8) - s->B(8));
	468	val += abs1(d->B(9) - s->B(9));
	469	val += abs1(d->B(10) - s->B(10));
	470	val += abs1(d->B(11) - s->B(11));
	471	val += abs1(d->B(12) - s->B(12));
	472	val += abs1(d->B(13) - s->B(13));
	473	val += abs1(d->B(14) - s->B(14));
	474	val += abs1(d->B(15) - s->B(15));
	475	d->Q(1) = val;
	476	#endif
	477	}
	478
	479	void glue(helper_maskmov, SUFFIX)(CPUX86State env, Reg d, Reg *s,
	480	target_ulong a0)
	481	{
	482	int i;
	483
	484	for (i = 0; i < (8 << SHIFT); i++) {
	485	if (s->B(i) & 0x80) {
	486	cpu_stb_data_ra(env, a0 + i, d->B(i), GETPC());
	487	}
	488	}
	489	}
	490
	491	void glue(helper_movl_mm_T0, SUFFIX)(Reg *d, uint32_t val)
	492	{
	493	d->L(0) = val;
	494	d->L(1) = 0;
	495	#if SHIFT == 1
	496	d->Q(1) = 0;
	497	#endif
	498	}
	499
	500	#ifdef TARGET_X86_64
	501	void glue(helper_movq_mm_T0, SUFFIX)(Reg *d, uint64_t val)
	502	{
	503	d->Q(0) = val;
	504	#if SHIFT == 1
	505	d->Q(1) = 0;
	506	#endif
	507	}
	508	#endif
	509
	510	#if SHIFT == 0
	511	void glue(helper_pshufw, SUFFIX)(Reg d, Reg s, int order)
	512	{
	513	Reg r;
	514
	515	r.W(0) = s->W(order & 3);
	516	r.W(1) = s->W((order >> 2) & 3);
	517	r.W(2) = s->W((order >> 4) & 3);
	518	r.W(3) = s->W((order >> 6) & 3);
	519	*d = r;
	520	}
	521	#else
	522	void helper_shufps(Reg d, Reg s, int order)
	523	{
	524	Reg r;
	525
	526	r.L(0) = d->L(order & 3);
	527	r.L(1) = d->L((order >> 2) & 3);
	528	r.L(2) = s->L((order >> 4) & 3);
	529	r.L(3) = s->L((order >> 6) & 3);
	530	*d = r;
	531	}
	532
	533	void helper_shufpd(Reg d, Reg s, int order)
	534	{
	535	Reg r;
	536
	537	r.Q(0) = d->Q(order & 1);
	538	r.Q(1) = s->Q((order >> 1) & 1);
	539	*d = r;
	540	}
	541
	542	void glue(helper_pshufd, SUFFIX)(Reg d, Reg s, int order)
	543	{
	544	Reg r;
	545
	546	r.L(0) = s->L(order & 3);
	547	r.L(1) = s->L((order >> 2) & 3);
	548	r.L(2) = s->L((order >> 4) & 3);
	549	r.L(3) = s->L((order >> 6) & 3);
	550	*d = r;
	551	}
	552
	553	void glue(helper_pshuflw, SUFFIX)(Reg d, Reg s, int order)
	554	{
	555	Reg r;
	556
	557	r.W(0) = s->W(order & 3);
	558	r.W(1) = s->W((order >> 2) & 3);
	559	r.W(2) = s->W((order >> 4) & 3);
	560	r.W(3) = s->W((order >> 6) & 3);
	561	r.Q(1) = s->Q(1);
	562	*d = r;
	563	}
	564
	565	void glue(helper_pshufhw, SUFFIX)(Reg d, Reg s, int order)
	566	{
	567	Reg r;
	568
	569	r.Q(0) = s->Q(0);
	570	r.W(4) = s->W(4 + (order & 3));
	571	r.W(5) = s->W(4 + ((order >> 2) & 3));
	572	r.W(6) = s->W(4 + ((order >> 4) & 3));
	573	r.W(7) = s->W(4 + ((order >> 6) & 3));
	574	*d = r;
	575	}
	576	#endif
	577
	578	#if SHIFT == 1
	579	/* FPU ops */
	580	/* XXX: not accurate */
	581
	582	#define SSE_HELPER_S(name, F) \
	583	void helper_ ## name ## ps(CPUX86State env, Reg d, Reg *s) \
	584	{ \
	585	d->XMM_S(0) = F(32, d->XMM_S(0), s->XMM_S(0)); \
	586	d->XMM_S(1) = F(32, d->XMM_S(1), s->XMM_S(1)); \
	587	d->XMM_S(2) = F(32, d->XMM_S(2), s->XMM_S(2)); \
	588	d->XMM_S(3) = F(32, d->XMM_S(3), s->XMM_S(3)); \
	589	} \
	590	\
	591	void helper_ ## name ## ss(CPUX86State env, Reg d, Reg *s) \
	592	{ \
	593	d->XMM_S(0) = F(32, d->XMM_S(0), s->XMM_S(0)); \
	594	} \
	595	\
	596	void helper_ ## name ## pd(CPUX86State env, Reg d, Reg *s) \
	597	{ \
	598	d->XMM_D(0) = F(64, d->XMM_D(0), s->XMM_D(0)); \
	599	d->XMM_D(1) = F(64, d->XMM_D(1), s->XMM_D(1)); \
	600	} \
	601	\
	602	void helper_ ## name ## sd(CPUX86State env, Reg d, Reg *s) \
	603	{ \
	604	d->XMM_D(0) = F(64, d->XMM_D(0), s->XMM_D(0)); \
	605	}
	606
	607	#define FPU_ADD(size, a, b) float ## size ## _add(a, b, &env->sse_status)
	608	#define FPU_SUB(size, a, b) float ## size ## _sub(a, b, &env->sse_status)
	609	#define FPU_MUL(size, a, b) float ## size ## _mul(a, b, &env->sse_status)
	610	#define FPU_DIV(size, a, b) float ## size ## _div(a, b, &env->sse_status)
	611	#define FPU_SQRT(size, a, b) float ## size ## _sqrt(b, &env->sse_status)
	612
	613	/* Note that the choice of comparison op here is important to get the
	614	* special cases right: for min and max Intel specifies that (-0,0),
	615	* (NaN, anything) and (anything, NaN) return the second argument.
	616	*/
	617	#define FPU_MIN(size, a, b) \
	618	(float ## size ## _lt(a, b, &env->sse_status) ? (a) : (b))
	619	#define FPU_MAX(size, a, b) \
	620	(float ## size ## _lt(b, a, &env->sse_status) ? (a) : (b))
	621
	622	SSE_HELPER_S(add, FPU_ADD)
	623	SSE_HELPER_S(sub, FPU_SUB)
	624	SSE_HELPER_S(mul, FPU_MUL)
	625	SSE_HELPER_S(div, FPU_DIV)
	626	SSE_HELPER_S(min, FPU_MIN)
	627	SSE_HELPER_S(max, FPU_MAX)
	628	SSE_HELPER_S(sqrt, FPU_SQRT)
	629
	630
	631	/* float to float conversions */
	632	void helper_cvtps2pd(CPUX86State env, Reg d, Reg *s)
	633	{
	634	float32 s0, s1;
	635
	636	s0 = s->XMM_S(0);
	637	s1 = s->XMM_S(1);
	638	d->XMM_D(0) = float32_to_float64(s0, &env->sse_status);
	639	d->XMM_D(1) = float32_to_float64(s1, &env->sse_status);
	640	}
	641
	642	void helper_cvtpd2ps(CPUX86State env, Reg d, Reg *s)
	643	{
	644	d->XMM_S(0) = float64_to_float32(s->XMM_D(0), &env->sse_status);
	645	d->XMM_S(1) = float64_to_float32(s->XMM_D(1), &env->sse_status);
	646	d->Q(1) = 0;
	647	}
	648
	649	void helper_cvtss2sd(CPUX86State env, Reg d, Reg *s)
	650	{
	651	d->XMM_D(0) = float32_to_float64(s->XMM_S(0), &env->sse_status);
	652	}
	653
	654	void helper_cvtsd2ss(CPUX86State env, Reg d, Reg *s)
	655	{
	656	d->XMM_S(0) = float64_to_float32(s->XMM_D(0), &env->sse_status);
	657	}
	658
	659	/* integer to float */
	660	void helper_cvtdq2ps(CPUX86State env, Reg d, Reg *s)
	661	{
	662	d->XMM_S(0) = int32_to_float32(s->XMM_L(0), &env->sse_status);
	663	d->XMM_S(1) = int32_to_float32(s->XMM_L(1), &env->sse_status);
	664	d->XMM_S(2) = int32_to_float32(s->XMM_L(2), &env->sse_status);
	665	d->XMM_S(3) = int32_to_float32(s->XMM_L(3), &env->sse_status);
	666	}
	667
	668	void helper_cvtdq2pd(CPUX86State env, Reg d, Reg *s)
	669	{
	670	int32_t l0, l1;
	671
	672	l0 = (int32_t)s->XMM_L(0);
	673	l1 = (int32_t)s->XMM_L(1);
	674	d->XMM_D(0) = int32_to_float64(l0, &env->sse_status);
	675	d->XMM_D(1) = int32_to_float64(l1, &env->sse_status);
	676	}
	677
	678	void helper_cvtpi2ps(CPUX86State env, XMMReg d, MMXReg *s)
	679	{
	680	d->XMM_S(0) = int32_to_float32(s->MMX_L(0), &env->sse_status);
	681	d->XMM_S(1) = int32_to_float32(s->MMX_L(1), &env->sse_status);
	682	}
	683
	684	void helper_cvtpi2pd(CPUX86State env, XMMReg d, MMXReg *s)
	685	{
	686	d->XMM_D(0) = int32_to_float64(s->MMX_L(0), &env->sse_status);
	687	d->XMM_D(1) = int32_to_float64(s->MMX_L(1), &env->sse_status);
	688	}
	689
	690	void helper_cvtsi2ss(CPUX86State env, XMMReg d, uint32_t val)
	691	{
	692	d->XMM_S(0) = int32_to_float32(val, &env->sse_status);
	693	}
	694
	695	void helper_cvtsi2sd(CPUX86State env, XMMReg d, uint32_t val)
	696	{
	697	d->XMM_D(0) = int32_to_float64(val, &env->sse_status);
	698	}
	699
	700	#ifdef TARGET_X86_64
	701	void helper_cvtsq2ss(CPUX86State env, XMMReg d, uint64_t val)
	702	{
	703	d->XMM_S(0) = int64_to_float32(val, &env->sse_status);
	704	}
	705
	706	void helper_cvtsq2sd(CPUX86State env, XMMReg d, uint64_t val)
	707	{
	708	d->XMM_D(0) = int64_to_float64(val, &env->sse_status);
	709	}
	710	#endif
	711
	712	/* float to integer */
	713	void helper_cvtps2dq(CPUX86State env, XMMReg d, XMMReg *s)
	714	{
	715	d->XMM_L(0) = float32_to_int32(s->XMM_S(0), &env->sse_status);
	716	d->XMM_L(1) = float32_to_int32(s->XMM_S(1), &env->sse_status);
	717	d->XMM_L(2) = float32_to_int32(s->XMM_S(2), &env->sse_status);
	718	d->XMM_L(3) = float32_to_int32(s->XMM_S(3), &env->sse_status);
	719	}
	720
	721	void helper_cvtpd2dq(CPUX86State env, XMMReg d, XMMReg *s)
	722	{
	723	d->XMM_L(0) = float64_to_int32(s->XMM_D(0), &env->sse_status);
	724	d->XMM_L(1) = float64_to_int32(s->XMM_D(1), &env->sse_status);
	725	d->XMM_Q(1) = 0;
	726	}
	727
	728	void helper_cvtps2pi(CPUX86State env, MMXReg d, XMMReg *s)
	729	{
	730	d->MMX_L(0) = float32_to_int32(s->XMM_S(0), &env->sse_status);
	731	d->MMX_L(1) = float32_to_int32(s->XMM_S(1), &env->sse_status);
	732	}
	733
	734	void helper_cvtpd2pi(CPUX86State env, MMXReg d, XMMReg *s)
	735	{
	736	d->MMX_L(0) = float64_to_int32(s->XMM_D(0), &env->sse_status);
	737	d->MMX_L(1) = float64_to_int32(s->XMM_D(1), &env->sse_status);
	738	}
	739
	740	int32_t helper_cvtss2si(CPUX86State env, XMMReg s)
	741	{
	742	return float32_to_int32(s->XMM_S(0), &env->sse_status);
	743	}
	744
	745	int32_t helper_cvtsd2si(CPUX86State env, XMMReg s)
	746	{
	747	return float64_to_int32(s->XMM_D(0), &env->sse_status);
	748	}
	749
	750	#ifdef TARGET_X86_64
	751	int64_t helper_cvtss2sq(CPUX86State env, XMMReg s)
	752	{
	753	return float32_to_int64(s->XMM_S(0), &env->sse_status);
	754	}
	755
	756	int64_t helper_cvtsd2sq(CPUX86State env, XMMReg s)
	757	{
	758	return float64_to_int64(s->XMM_D(0), &env->sse_status);
	759	}
	760	#endif
	761
	762	/* float to integer truncated */
	763	void helper_cvttps2dq(CPUX86State env, XMMReg d, XMMReg *s)
	764	{
	765	d->XMM_L(0) = float32_to_int32_round_to_zero(s->XMM_S(0), &env->sse_status);
	766	d->XMM_L(1) = float32_to_int32_round_to_zero(s->XMM_S(1), &env->sse_status);
	767	d->XMM_L(2) = float32_to_int32_round_to_zero(s->XMM_S(2), &env->sse_status);
	768	d->XMM_L(3) = float32_to_int32_round_to_zero(s->XMM_S(3), &env->sse_status);
	769	}
	770
	771	void helper_cvttpd2dq(CPUX86State env, XMMReg d, XMMReg *s)
	772	{
	773	d->XMM_L(0) = float64_to_int32_round_to_zero(s->XMM_D(0), &env->sse_status);
	774	d->XMM_L(1) = float64_to_int32_round_to_zero(s->XMM_D(1), &env->sse_status);
	775	d->XMM_Q(1) = 0;
	776	}
	777
	778	void helper_cvttps2pi(CPUX86State env, MMXReg d, XMMReg *s)
	779	{
	780	d->MMX_L(0) = float32_to_int32_round_to_zero(s->XMM_S(0), &env->sse_status);
	781	d->MMX_L(1) = float32_to_int32_round_to_zero(s->XMM_S(1), &env->sse_status);
	782	}
	783
	784	void helper_cvttpd2pi(CPUX86State env, MMXReg d, XMMReg *s)
	785	{
	786	d->MMX_L(0) = float64_to_int32_round_to_zero(s->XMM_D(0), &env->sse_status);
	787	d->MMX_L(1) = float64_to_int32_round_to_zero(s->XMM_D(1), &env->sse_status);
	788	}
	789
	790	int32_t helper_cvttss2si(CPUX86State env, XMMReg s)
	791	{
	792	return float32_to_int32_round_to_zero(s->XMM_S(0), &env->sse_status);
	793	}
	794
	795	int32_t helper_cvttsd2si(CPUX86State env, XMMReg s)
	796	{
	797	return float64_to_int32_round_to_zero(s->XMM_D(0), &env->sse_status);
	798	}
	799
	800	#ifdef TARGET_X86_64
	801	int64_t helper_cvttss2sq(CPUX86State env, XMMReg s)
	802	{
	803	return float32_to_int64_round_to_zero(s->XMM_S(0), &env->sse_status);
	804	}
	805
	806	int64_t helper_cvttsd2sq(CPUX86State env, XMMReg s)
	807	{
	808	return float64_to_int64_round_to_zero(s->XMM_D(0), &env->sse_status);
	809	}
	810	#endif
	811
	812	void helper_rsqrtps(CPUX86State env, XMMReg d, XMMReg *s)
	813	{
	814	d->XMM_S(0) = float32_div(float32_one,
	815	float32_sqrt(s->XMM_S(0), &env->sse_status),
	816	&env->sse_status);
	817	d->XMM_S(1) = float32_div(float32_one,
	818	float32_sqrt(s->XMM_S(1), &env->sse_status),
	819	&env->sse_status);
	820	d->XMM_S(2) = float32_div(float32_one,
	821	float32_sqrt(s->XMM_S(2), &env->sse_status),
	822	&env->sse_status);
	823	d->XMM_S(3) = float32_div(float32_one,
	824	float32_sqrt(s->XMM_S(3), &env->sse_status),
	825	&env->sse_status);
	826	}
	827
	828	void helper_rsqrtss(CPUX86State env, XMMReg d, XMMReg *s)
	829	{
	830	d->XMM_S(0) = float32_div(float32_one,
	831	float32_sqrt(s->XMM_S(0), &env->sse_status),
	832	&env->sse_status);
	833	}
	834
	835	void helper_rcpps(CPUX86State env, XMMReg d, XMMReg *s)
	836	{
	837	d->XMM_S(0) = float32_div(float32_one, s->XMM_S(0), &env->sse_status);
	838	d->XMM_S(1) = float32_div(float32_one, s->XMM_S(1), &env->sse_status);
	839	d->XMM_S(2) = float32_div(float32_one, s->XMM_S(2), &env->sse_status);
	840	d->XMM_S(3) = float32_div(float32_one, s->XMM_S(3), &env->sse_status);
	841	}
	842
	843	void helper_rcpss(CPUX86State env, XMMReg d, XMMReg *s)
	844	{
	845	d->XMM_S(0) = float32_div(float32_one, s->XMM_S(0), &env->sse_status);
	846	}
	847
	848	static inline uint64_t helper_extrq(uint64_t src, int shift, int len)
	849	{
	850	uint64_t mask;
	851
	852	if (len == 0) {
	853	mask = ~0LL;
	854	} else {
	855	mask = (1ULL << len) - 1;
	856	}
	857	return (src >> shift) & mask;
	858	}
	859
	860	void helper_extrq_r(CPUX86State env, XMMReg d, XMMReg *s)
	861	{
	862	d->XMM_Q(0) = helper_extrq(d->XMM_Q(0), s->XMM_B(1), s->XMM_B(0));
	863	}
	864
	865	void helper_extrq_i(CPUX86State env, XMMReg d, int index, int length)
	866	{
	867	d->XMM_Q(0) = helper_extrq(d->XMM_Q(0), index, length);
	868	}
	869
	870	static inline uint64_t helper_insertq(uint64_t src, int shift, int len)
	871	{
	872	uint64_t mask;
	873
	874	if (len == 0) {
	875	mask = ~0ULL;
	876	} else {
	877	mask = (1ULL << len) - 1;
	878	}
	879	return (src & ~(mask << shift)) \| ((src & mask) << shift);
	880	}
	881
	882	void helper_insertq_r(CPUX86State env, XMMReg d, XMMReg *s)
	883	{
	884	d->XMM_Q(0) = helper_insertq(s->XMM_Q(0), s->XMM_B(9), s->XMM_B(8));
	885	}
	886
	887	void helper_insertq_i(CPUX86State env, XMMReg d, int index, int length)
	888	{
	889	d->XMM_Q(0) = helper_insertq(d->XMM_Q(0), index, length);
	890	}
	891
	892	void helper_haddps(CPUX86State env, XMMReg d, XMMReg *s)
	893	{
	894	XMMReg r;
	895
	896	r.XMM_S(0) = float32_add(d->XMM_S(0), d->XMM_S(1), &env->sse_status);
	897	r.XMM_S(1) = float32_add(d->XMM_S(2), d->XMM_S(3), &env->sse_status);
	898	r.XMM_S(2) = float32_add(s->XMM_S(0), s->XMM_S(1), &env->sse_status);
	899	r.XMM_S(3) = float32_add(s->XMM_S(2), s->XMM_S(3), &env->sse_status);
	900	*d = r;
	901	}
	902
	903	void helper_haddpd(CPUX86State env, XMMReg d, XMMReg *s)
	904	{
	905	XMMReg r;
	906
	907	r.XMM_D(0) = float64_add(d->XMM_D(0), d->XMM_D(1), &env->sse_status);
	908	r.XMM_D(1) = float64_add(s->XMM_D(0), s->XMM_D(1), &env->sse_status);
	909	*d = r;
	910	}
	911
	912	void helper_hsubps(CPUX86State env, XMMReg d, XMMReg *s)
	913	{
	914	XMMReg r;
	915
	916	r.XMM_S(0) = float32_sub(d->XMM_S(0), d->XMM_S(1), &env->sse_status);
	917	r.XMM_S(1) = float32_sub(d->XMM_S(2), d->XMM_S(3), &env->sse_status);
	918	r.XMM_S(2) = float32_sub(s->XMM_S(0), s->XMM_S(1), &env->sse_status);
	919	r.XMM_S(3) = float32_sub(s->XMM_S(2), s->XMM_S(3), &env->sse_status);
	920	*d = r;
	921	}
	922
	923	void helper_hsubpd(CPUX86State env, XMMReg d, XMMReg *s)
	924	{
	925	XMMReg r;
	926
	927	r.XMM_D(0) = float64_sub(d->XMM_D(0), d->XMM_D(1), &env->sse_status);
	928	r.XMM_D(1) = float64_sub(s->XMM_D(0), s->XMM_D(1), &env->sse_status);
	929	*d = r;
	930	}
	931
	932	void helper_addsubps(CPUX86State env, XMMReg d, XMMReg *s)
	933	{
	934	d->XMM_S(0) = float32_sub(d->XMM_S(0), s->XMM_S(0), &env->sse_status);
	935	d->XMM_S(1) = float32_add(d->XMM_S(1), s->XMM_S(1), &env->sse_status);
	936	d->XMM_S(2) = float32_sub(d->XMM_S(2), s->XMM_S(2), &env->sse_status);
	937	d->XMM_S(3) = float32_add(d->XMM_S(3), s->XMM_S(3), &env->sse_status);
	938	}
	939
	940	void helper_addsubpd(CPUX86State env, XMMReg d, XMMReg *s)
	941	{
	942	d->XMM_D(0) = float64_sub(d->XMM_D(0), s->XMM_D(0), &env->sse_status);
	943	d->XMM_D(1) = float64_add(d->XMM_D(1), s->XMM_D(1), &env->sse_status);
	944	}
	945
	946	/* XXX: unordered */
	947	#define SSE_HELPER_CMP(name, F) \
	948	void helper_ ## name ## ps(CPUX86State env, Reg d, Reg *s) \
	949	{ \
	950	d->XMM_L(0) = F(32, d->XMM_S(0), s->XMM_S(0)); \
	951	d->XMM_L(1) = F(32, d->XMM_S(1), s->XMM_S(1)); \
	952	d->XMM_L(2) = F(32, d->XMM_S(2), s->XMM_S(2)); \
	953	d->XMM_L(3) = F(32, d->XMM_S(3), s->XMM_S(3)); \
	954	} \
	955	\
	956	void helper_ ## name ## ss(CPUX86State env, Reg d, Reg *s) \
	957	{ \
	958	d->XMM_L(0) = F(32, d->XMM_S(0), s->XMM_S(0)); \
	959	} \
	960	\
	961	void helper_ ## name ## pd(CPUX86State env, Reg d, Reg *s) \
	962	{ \
	963	d->XMM_Q(0) = F(64, d->XMM_D(0), s->XMM_D(0)); \
	964	d->XMM_Q(1) = F(64, d->XMM_D(1), s->XMM_D(1)); \
	965	} \
	966	\
	967	void helper_ ## name ## sd(CPUX86State env, Reg d, Reg *s) \
	968	{ \
	969	d->XMM_Q(0) = F(64, d->XMM_D(0), s->XMM_D(0)); \
	970	}
	971
	972	#define FPU_CMPEQ(size, a, b) \
	973	(float ## size ## _eq_quiet(a, b, &env->sse_status) ? -1 : 0)
	974	#define FPU_CMPLT(size, a, b) \
	975	(float ## size ## _lt(a, b, &env->sse_status) ? -1 : 0)
	976	#define FPU_CMPLE(size, a, b) \
	977	(float ## size ## _le(a, b, &env->sse_status) ? -1 : 0)
	978	#define FPU_CMPUNORD(size, a, b) \
	979	(float ## size ## _unordered_quiet(a, b, &env->sse_status) ? -1 : 0)
	980	#define FPU_CMPNEQ(size, a, b) \
	981	(float ## size ## _eq_quiet(a, b, &env->sse_status) ? 0 : -1)
	982	#define FPU_CMPNLT(size, a, b) \
	983	(float ## size ## _lt(a, b, &env->sse_status) ? 0 : -1)
	984	#define FPU_CMPNLE(size, a, b) \
	985	(float ## size ## _le(a, b, &env->sse_status) ? 0 : -1)
	986	#define FPU_CMPORD(size, a, b) \
	987	(float ## size ## _unordered_quiet(a, b, &env->sse_status) ? 0 : -1)
	988
	989	SSE_HELPER_CMP(cmpeq, FPU_CMPEQ)
	990	SSE_HELPER_CMP(cmplt, FPU_CMPLT)
	991	SSE_HELPER_CMP(cmple, FPU_CMPLE)
	992	SSE_HELPER_CMP(cmpunord, FPU_CMPUNORD)
	993	SSE_HELPER_CMP(cmpneq, FPU_CMPNEQ)
	994	SSE_HELPER_CMP(cmpnlt, FPU_CMPNLT)
	995	SSE_HELPER_CMP(cmpnle, FPU_CMPNLE)
	996	SSE_HELPER_CMP(cmpord, FPU_CMPORD)
	997
	998	static const int comis_eflags[4] = {CC_C, CC_Z, 0, CC_Z \| CC_P \| CC_C};
	999
	1000	void helper_ucomiss(CPUX86State env, Reg d, Reg *s)
	1001	{
	1002	int ret;
	1003	float32 s0, s1;
	1004
	1005	s0 = d->XMM_S(0);
	1006	s1 = s->XMM_S(0);
	1007	ret = float32_compare_quiet(s0, s1, &env->sse_status);
	1008	CC_SRC = comis_eflags[ret + 1];
	1009	}
	1010
	1011	void helper_comiss(CPUX86State env, Reg d, Reg *s)
	1012	{
	1013	int ret;
	1014	float32 s0, s1;
	1015
	1016	s0 = d->XMM_S(0);
	1017	s1 = s->XMM_S(0);
	1018	ret = float32_compare(s0, s1, &env->sse_status);
	1019	CC_SRC = comis_eflags[ret + 1];
	1020	}
	1021
	1022	void helper_ucomisd(CPUX86State env, Reg d, Reg *s)
	1023	{
	1024	int ret;
	1025	float64 d0, d1;
	1026
	1027	d0 = d->XMM_D(0);
	1028	d1 = s->XMM_D(0);
	1029	ret = float64_compare_quiet(d0, d1, &env->sse_status);
	1030	CC_SRC = comis_eflags[ret + 1];
	1031	}
	1032
	1033	void helper_comisd(CPUX86State env, Reg d, Reg *s)
	1034	{
	1035	int ret;
	1036	float64 d0, d1;
	1037
	1038	d0 = d->XMM_D(0);
	1039	d1 = s->XMM_D(0);
	1040	ret = float64_compare(d0, d1, &env->sse_status);
	1041	CC_SRC = comis_eflags[ret + 1];
	1042	}
	1043
	1044	uint32_t helper_movmskps(CPUX86State env, Reg s)
	1045	{
	1046	int b0, b1, b2, b3;
	1047
	1048	b0 = s->XMM_L(0) >> 31;
	1049	b1 = s->XMM_L(1) >> 31;
	1050	b2 = s->XMM_L(2) >> 31;
	1051	b3 = s->XMM_L(3) >> 31;
	1052	return b0 \| (b1 << 1) \| (b2 << 2) \| (b3 << 3);
	1053	}
	1054
	1055	uint32_t helper_movmskpd(CPUX86State env, Reg s)
	1056	{
	1057	int b0, b1;
	1058
	1059	b0 = s->XMM_L(1) >> 31;
	1060	b1 = s->XMM_L(3) >> 31;
	1061	return b0 \| (b1 << 1);
	1062	}
	1063
	1064	#endif
	1065
	1066	uint32_t glue(helper_pmovmskb, SUFFIX)(CPUX86State env, Reg s)
	1067	{
	1068	uint32_t val;
	1069
	1070	val = 0;
	1071	val \|= (s->B(0) >> 7);
	1072	val \|= (s->B(1) >> 6) & 0x02;
	1073	val \|= (s->B(2) >> 5) & 0x04;
	1074	val \|= (s->B(3) >> 4) & 0x08;
	1075	val \|= (s->B(4) >> 3) & 0x10;
	1076	val \|= (s->B(5) >> 2) & 0x20;
	1077	val \|= (s->B(6) >> 1) & 0x40;
	1078	val \|= (s->B(7)) & 0x80;
	1079	#if SHIFT == 1
	1080	val \|= (s->B(8) << 1) & 0x0100;
	1081	val \|= (s->B(9) << 2) & 0x0200;
	1082	val \|= (s->B(10) << 3) & 0x0400;
	1083	val \|= (s->B(11) << 4) & 0x0800;
	1084	val \|= (s->B(12) << 5) & 0x1000;
	1085	val \|= (s->B(13) << 6) & 0x2000;
	1086	val \|= (s->B(14) << 7) & 0x4000;
	1087	val \|= (s->B(15) << 8) & 0x8000;
	1088	#endif
	1089	return val;
	1090	}
	1091
	1092	void glue(helper_packsswb, SUFFIX)(CPUX86State env, Reg d, Reg *s)
	1093	{
	1094	Reg r;
	1095
	1096	r.B(0) = satsb((int16_t)d->W(0));
	1097	r.B(1) = satsb((int16_t)d->W(1));
	1098	r.B(2) = satsb((int16_t)d->W(2));
	1099	r.B(3) = satsb((int16_t)d->W(3));
	1100	#if SHIFT == 1
	1101	r.B(4) = satsb((int16_t)d->W(4));
	1102	r.B(5) = satsb((int16_t)d->W(5));
	1103	r.B(6) = satsb((int16_t)d->W(6));
	1104	r.B(7) = satsb((int16_t)d->W(7));
	1105	#endif
	1106	r.B((4 << SHIFT) + 0) = satsb((int16_t)s->W(0));
	1107	r.B((4 << SHIFT) + 1) = satsb((int16_t)s->W(1));
	1108	r.B((4 << SHIFT) + 2) = satsb((int16_t)s->W(2));
	1109	r.B((4 << SHIFT) + 3) = satsb((int16_t)s->W(3));
	1110	#if SHIFT == 1
	1111	r.B(12) = satsb((int16_t)s->W(4));
	1112	r.B(13) = satsb((int16_t)s->W(5));
	1113	r.B(14) = satsb((int16_t)s->W(6));
	1114	r.B(15) = satsb((int16_t)s->W(7));
	1115	#endif
	1116	*d = r;
	1117	}
	1118
	1119	void glue(helper_packuswb, SUFFIX)(CPUX86State env, Reg d, Reg *s)
	1120	{
	1121	Reg r;
	1122
	1123	r.B(0) = satub((int16_t)d->W(0));
	1124	r.B(1) = satub((int16_t)d->W(1));
	1125	r.B(2) = satub((int16_t)d->W(2));
	1126	r.B(3) = satub((int16_t)d->W(3));
	1127	#if SHIFT == 1
	1128	r.B(4) = satub((int16_t)d->W(4));
	1129	r.B(5) = satub((int16_t)d->W(5));
	1130	r.B(6) = satub((int16_t)d->W(6));
	1131	r.B(7) = satub((int16_t)d->W(7));
	1132	#endif
	1133	r.B((4 << SHIFT) + 0) = satub((int16_t)s->W(0));
	1134	r.B((4 << SHIFT) + 1) = satub((int16_t)s->W(1));
	1135	r.B((4 << SHIFT) + 2) = satub((int16_t)s->W(2));
	1136	r.B((4 << SHIFT) + 3) = satub((int16_t)s->W(3));
	1137	#if SHIFT == 1
	1138	r.B(12) = satub((int16_t)s->W(4));
	1139	r.B(13) = satub((int16_t)s->W(5));
	1140	r.B(14) = satub((int16_t)s->W(6));
	1141	r.B(15) = satub((int16_t)s->W(7));
	1142	#endif
	1143	*d = r;
	1144	}
	1145
	1146	void glue(helper_packssdw, SUFFIX)(CPUX86State env, Reg d, Reg *s)
	1147	{
	1148	Reg r;
	1149
	1150	r.W(0) = satsw(d->L(0));
	1151	r.W(1) = satsw(d->L(1));
	1152	#if SHIFT == 1
	1153	r.W(2) = satsw(d->L(2));
	1154	r.W(3) = satsw(d->L(3));
	1155	#endif
	1156	r.W((2 << SHIFT) + 0) = satsw(s->L(0));
	1157	r.W((2 << SHIFT) + 1) = satsw(s->L(1));
	1158	#if SHIFT == 1
	1159	r.W(6) = satsw(s->L(2));
	1160	r.W(7) = satsw(s->L(3));
	1161	#endif
	1162	*d = r;
	1163	}
	1164
	1165	#define UNPCK_OP(base_name, base) \
	1166	\
	1167	void glue(helper_punpck ## base_name ## bw, SUFFIX)(CPUX86State *env,\
	1168	Reg d, Reg s) \
	1169	{ \
	1170	Reg r; \
	1171	\
	1172	r.B(0) = d->B((base << (SHIFT + 2)) + 0); \
	1173	r.B(1) = s->B((base << (SHIFT + 2)) + 0); \
	1174	r.B(2) = d->B((base << (SHIFT + 2)) + 1); \
	1175	r.B(3) = s->B((base << (SHIFT + 2)) + 1); \
	1176	r.B(4) = d->B((base << (SHIFT + 2)) + 2); \
	1177	r.B(5) = s->B((base << (SHIFT + 2)) + 2); \
	1178	r.B(6) = d->B((base << (SHIFT + 2)) + 3); \
	1179	r.B(7) = s->B((base << (SHIFT + 2)) + 3); \
	1180	XMM_ONLY( \
	1181	r.B(8) = d->B((base << (SHIFT + 2)) + 4); \
	1182	r.B(9) = s->B((base << (SHIFT + 2)) + 4); \
	1183	r.B(10) = d->B((base << (SHIFT + 2)) + 5); \
	1184	r.B(11) = s->B((base << (SHIFT + 2)) + 5); \
	1185	r.B(12) = d->B((base << (SHIFT + 2)) + 6); \
	1186	r.B(13) = s->B((base << (SHIFT + 2)) + 6); \
	1187	r.B(14) = d->B((base << (SHIFT + 2)) + 7); \
	1188	r.B(15) = s->B((base << (SHIFT + 2)) + 7); \
	1189	) \
	1190	*d = r; \
	1191	} \
	1192	\
	1193	void glue(helper_punpck ## base_name ## wd, SUFFIX)(CPUX86State *env,\
	1194	Reg d, Reg s) \
	1195	{ \
	1196	Reg r; \
	1197	\
	1198	r.W(0) = d->W((base << (SHIFT + 1)) + 0); \
	1199	r.W(1) = s->W((base << (SHIFT + 1)) + 0); \
	1200	r.W(2) = d->W((base << (SHIFT + 1)) + 1); \
	1201	r.W(3) = s->W((base << (SHIFT + 1)) + 1); \
	1202	XMM_ONLY( \
	1203	r.W(4) = d->W((base << (SHIFT + 1)) + 2); \
	1204	r.W(5) = s->W((base << (SHIFT + 1)) + 2); \
	1205	r.W(6) = d->W((base << (SHIFT + 1)) + 3); \
	1206	r.W(7) = s->W((base << (SHIFT + 1)) + 3); \
	1207	) \
	1208	*d = r; \
	1209	} \
	1210	\
	1211	void glue(helper_punpck ## base_name ## dq, SUFFIX)(CPUX86State *env,\
	1212	Reg d, Reg s) \
	1213	{ \
	1214	Reg r; \
	1215	\
	1216	r.L(0) = d->L((base << SHIFT) + 0); \
	1217	r.L(1) = s->L((base << SHIFT) + 0); \
	1218	XMM_ONLY( \
	1219	r.L(2) = d->L((base << SHIFT) + 1); \
	1220	r.L(3) = s->L((base << SHIFT) + 1); \
	1221	) \
	1222	*d = r; \
	1223	} \
	1224	\
	1225	XMM_ONLY( \
	1226	void glue(helper_punpck ## base_name ## qdq, SUFFIX)(CPUX86State \
	1227	*env, \
	1228	Reg *d, \
	1229	Reg *s) \
	1230	{ \
	1231	Reg r; \
	1232	\
	1233	r.Q(0) = d->Q(base); \
	1234	r.Q(1) = s->Q(base); \
	1235	*d = r; \
	1236	} \
	1237	)
	1238
	1239	UNPCK_OP(l, 0)
	1240	UNPCK_OP(h, 1)
	1241
	1242	/* 3DNow! float ops */
	1243	#if SHIFT == 0
	1244	void helper_pi2fd(CPUX86State env, MMXReg d, MMXReg *s)
	1245	{
	1246	d->MMX_S(0) = int32_to_float32(s->MMX_L(0), &env->mmx_status);
	1247	d->MMX_S(1) = int32_to_float32(s->MMX_L(1), &env->mmx_status);
	1248	}
	1249
	1250	void helper_pi2fw(CPUX86State env, MMXReg d, MMXReg *s)
	1251	{
	1252	d->MMX_S(0) = int32_to_float32((int16_t)s->MMX_W(0), &env->mmx_status);
	1253	d->MMX_S(1) = int32_to_float32((int16_t)s->MMX_W(2), &env->mmx_status);
	1254	}
	1255
	1256	void helper_pf2id(CPUX86State env, MMXReg d, MMXReg *s)
	1257	{
	1258	d->MMX_L(0) = float32_to_int32_round_to_zero(s->MMX_S(0), &env->mmx_status);
	1259	d->MMX_L(1) = float32_to_int32_round_to_zero(s->MMX_S(1), &env->mmx_status);
	1260	}
	1261
	1262	void helper_pf2iw(CPUX86State env, MMXReg d, MMXReg *s)
	1263	{
	1264	d->MMX_L(0) = satsw(float32_to_int32_round_to_zero(s->MMX_S(0),
	1265	&env->mmx_status));
	1266	d->MMX_L(1) = satsw(float32_to_int32_round_to_zero(s->MMX_S(1),
	1267	&env->mmx_status));
	1268	}
	1269
	1270	void helper_pfacc(CPUX86State env, MMXReg d, MMXReg *s)
	1271	{
	1272	MMXReg r;
	1273
	1274	r.MMX_S(0) = float32_add(d->MMX_S(0), d->MMX_S(1), &env->mmx_status);
	1275	r.MMX_S(1) = float32_add(s->MMX_S(0), s->MMX_S(1), &env->mmx_status);
	1276	*d = r;
	1277	}
	1278
	1279	void helper_pfadd(CPUX86State env, MMXReg d, MMXReg *s)
	1280	{
	1281	d->MMX_S(0) = float32_add(d->MMX_S(0), s->MMX_S(0), &env->mmx_status);
	1282	d->MMX_S(1) = float32_add(d->MMX_S(1), s->MMX_S(1), &env->mmx_status);
	1283	}
	1284
	1285	void helper_pfcmpeq(CPUX86State env, MMXReg d, MMXReg *s)
	1286	{
	1287	d->MMX_L(0) = float32_eq_quiet(d->MMX_S(0), s->MMX_S(0),
	1288	&env->mmx_status) ? -1 : 0;
	1289	d->MMX_L(1) = float32_eq_quiet(d->MMX_S(1), s->MMX_S(1),
	1290	&env->mmx_status) ? -1 : 0;
	1291	}
	1292
	1293	void helper_pfcmpge(CPUX86State env, MMXReg d, MMXReg *s)
	1294	{
	1295	d->MMX_L(0) = float32_le(s->MMX_S(0), d->MMX_S(0),
	1296	&env->mmx_status) ? -1 : 0;
	1297	d->MMX_L(1) = float32_le(s->MMX_S(1), d->MMX_S(1),
	1298	&env->mmx_status) ? -1 : 0;
	1299	}
	1300
	1301	void helper_pfcmpgt(CPUX86State env, MMXReg d, MMXReg *s)
	1302	{
	1303	d->MMX_L(0) = float32_lt(s->MMX_S(0), d->MMX_S(0),
	1304	&env->mmx_status) ? -1 : 0;
	1305	d->MMX_L(1) = float32_lt(s->MMX_S(1), d->MMX_S(1),
	1306	&env->mmx_status) ? -1 : 0;
	1307	}
	1308
	1309	void helper_pfmax(CPUX86State env, MMXReg d, MMXReg *s)
	1310	{
	1311	if (float32_lt(d->MMX_S(0), s->MMX_S(0), &env->mmx_status)) {
	1312	d->MMX_S(0) = s->MMX_S(0);
	1313	}
	1314	if (float32_lt(d->MMX_S(1), s->MMX_S(1), &env->mmx_status)) {
	1315	d->MMX_S(1) = s->MMX_S(1);
	1316	}
	1317	}
	1318
	1319	void helper_pfmin(CPUX86State env, MMXReg d, MMXReg *s)
	1320	{
	1321	if (float32_lt(s->MMX_S(0), d->MMX_S(0), &env->mmx_status)) {
	1322	d->MMX_S(0) = s->MMX_S(0);
	1323	}
	1324	if (float32_lt(s->MMX_S(1), d->MMX_S(1), &env->mmx_status)) {
	1325	d->MMX_S(1) = s->MMX_S(1);
	1326	}
	1327	}
	1328
	1329	void helper_pfmul(CPUX86State env, MMXReg d, MMXReg *s)
	1330	{
	1331	d->MMX_S(0) = float32_mul(d->MMX_S(0), s->MMX_S(0), &env->mmx_status);
	1332	d->MMX_S(1) = float32_mul(d->MMX_S(1), s->MMX_S(1), &env->mmx_status);
	1333	}
	1334
	1335	void helper_pfnacc(CPUX86State env, MMXReg d, MMXReg *s)
	1336	{
	1337	MMXReg r;
	1338
	1339	r.MMX_S(0) = float32_sub(d->MMX_S(0), d->MMX_S(1), &env->mmx_status);
	1340	r.MMX_S(1) = float32_sub(s->MMX_S(0), s->MMX_S(1), &env->mmx_status);
	1341	*d = r;
	1342	}
	1343
	1344	void helper_pfpnacc(CPUX86State env, MMXReg d, MMXReg *s)
	1345	{
	1346	MMXReg r;
	1347
	1348	r.MMX_S(0) = float32_sub(d->MMX_S(0), d->MMX_S(1), &env->mmx_status);
	1349	r.MMX_S(1) = float32_add(s->MMX_S(0), s->MMX_S(1), &env->mmx_status);
	1350	*d = r;
	1351	}
	1352
	1353	void helper_pfrcp(CPUX86State env, MMXReg d, MMXReg *s)
	1354	{
	1355	d->MMX_S(0) = float32_div(float32_one, s->MMX_S(0), &env->mmx_status);
	1356	d->MMX_S(1) = d->MMX_S(0);
	1357	}
	1358
	1359	void helper_pfrsqrt(CPUX86State env, MMXReg d, MMXReg *s)
	1360	{
	1361	d->MMX_L(1) = s->MMX_L(0) & 0x7fffffff;
	1362	d->MMX_S(1) = float32_div(float32_one,
	1363	float32_sqrt(d->MMX_S(1), &env->mmx_status),
	1364	&env->mmx_status);
	1365	d->MMX_L(1) \|= s->MMX_L(0) & 0x80000000;
	1366	d->MMX_L(0) = d->MMX_L(1);
	1367	}
	1368
	1369	void helper_pfsub(CPUX86State env, MMXReg d, MMXReg *s)
	1370	{
	1371	d->MMX_S(0) = float32_sub(d->MMX_S(0), s->MMX_S(0), &env->mmx_status);
	1372	d->MMX_S(1) = float32_sub(d->MMX_S(1), s->MMX_S(1), &env->mmx_status);
	1373	}
	1374
	1375	void helper_pfsubr(CPUX86State env, MMXReg d, MMXReg *s)
	1376	{
	1377	d->MMX_S(0) = float32_sub(s->MMX_S(0), d->MMX_S(0), &env->mmx_status);
	1378	d->MMX_S(1) = float32_sub(s->MMX_S(1), d->MMX_S(1), &env->mmx_status);
	1379	}
	1380
	1381	void helper_pswapd(CPUX86State env, MMXReg d, MMXReg *s)
	1382	{
	1383	MMXReg r;
	1384
	1385	r.MMX_L(0) = s->MMX_L(1);
	1386	r.MMX_L(1) = s->MMX_L(0);
	1387	*d = r;
	1388	}
	1389	#endif
	1390
	1391	/* SSSE3 op helpers */
	1392	void glue(helper_pshufb, SUFFIX)(CPUX86State env, Reg d, Reg *s)
	1393	{
	1394	int i;
	1395	Reg r;
	1396
	1397	for (i = 0; i < (8 << SHIFT); i++) {
	1398	r.B(i) = (s->B(i) & 0x80) ? 0 : (d->B(s->B(i) & ((8 << SHIFT) - 1)));
	1399	}
	1400
	1401	*d = r;
	1402	}
	1403
	1404	void glue(helper_phaddw, SUFFIX)(CPUX86State env, Reg d, Reg *s)
	1405	{
	1406	d->W(0) = (int16_t)d->W(0) + (int16_t)d->W(1);
	1407	d->W(1) = (int16_t)d->W(2) + (int16_t)d->W(3);
	1408	XMM_ONLY(d->W(2) = (int16_t)d->W(4) + (int16_t)d->W(5));
	1409	XMM_ONLY(d->W(3) = (int16_t)d->W(6) + (int16_t)d->W(7));
	1410	d->W((2 << SHIFT) + 0) = (int16_t)s->W(0) + (int16_t)s->W(1);
	1411	d->W((2 << SHIFT) + 1) = (int16_t)s->W(2) + (int16_t)s->W(3);
	1412	XMM_ONLY(d->W(6) = (int16_t)s->W(4) + (int16_t)s->W(5));
	1413	XMM_ONLY(d->W(7) = (int16_t)s->W(6) + (int16_t)s->W(7));
	1414	}
	1415
	1416	void glue(helper_phaddd, SUFFIX)(CPUX86State env, Reg d, Reg *s)
	1417	{
	1418	d->L(0) = (int32_t)d->L(0) + (int32_t)d->L(1);
	1419	XMM_ONLY(d->L(1) = (int32_t)d->L(2) + (int32_t)d->L(3));
	1420	d->L((1 << SHIFT) + 0) = (int32_t)s->L(0) + (int32_t)s->L(1);
	1421	XMM_ONLY(d->L(3) = (int32_t)s->L(2) + (int32_t)s->L(3));
	1422	}
	1423
	1424	void glue(helper_phaddsw, SUFFIX)(CPUX86State env, Reg d, Reg *s)
	1425	{
	1426	d->W(0) = satsw((int16_t)d->W(0) + (int16_t)d->W(1));
	1427	d->W(1) = satsw((int16_t)d->W(2) + (int16_t)d->W(3));
	1428	XMM_ONLY(d->W(2) = satsw((int16_t)d->W(4) + (int16_t)d->W(5)));
	1429	XMM_ONLY(d->W(3) = satsw((int16_t)d->W(6) + (int16_t)d->W(7)));
	1430	d->W((2 << SHIFT) + 0) = satsw((int16_t)s->W(0) + (int16_t)s->W(1));
	1431	d->W((2 << SHIFT) + 1) = satsw((int16_t)s->W(2) + (int16_t)s->W(3));
	1432	XMM_ONLY(d->W(6) = satsw((int16_t)s->W(4) + (int16_t)s->W(5)));
	1433	XMM_ONLY(d->W(7) = satsw((int16_t)s->W(6) + (int16_t)s->W(7)));
	1434	}
	1435
	1436	void glue(helper_pmaddubsw, SUFFIX)(CPUX86State env, Reg d, Reg *s)
	1437	{
	1438	d->W(0) = satsw((int8_t)s->B(0) * (uint8_t)d->B(0) +
	1439	(int8_t)s->B(1) * (uint8_t)d->B(1));
	1440	d->W(1) = satsw((int8_t)s->B(2) * (uint8_t)d->B(2) +
	1441	(int8_t)s->B(3) * (uint8_t)d->B(3));
	1442	d->W(2) = satsw((int8_t)s->B(4) * (uint8_t)d->B(4) +
	1443	(int8_t)s->B(5) * (uint8_t)d->B(5));
	1444	d->W(3) = satsw((int8_t)s->B(6) * (uint8_t)d->B(6) +
	1445	(int8_t)s->B(7) * (uint8_t)d->B(7));
	1446	#if SHIFT == 1
	1447	d->W(4) = satsw((int8_t)s->B(8) * (uint8_t)d->B(8) +
	1448	(int8_t)s->B(9) * (uint8_t)d->B(9));
	1449	d->W(5) = satsw((int8_t)s->B(10) * (uint8_t)d->B(10) +
	1450	(int8_t)s->B(11) * (uint8_t)d->B(11));
	1451	d->W(6) = satsw((int8_t)s->B(12) * (uint8_t)d->B(12) +
	1452	(int8_t)s->B(13) * (uint8_t)d->B(13));
	1453	d->W(7) = satsw((int8_t)s->B(14) * (uint8_t)d->B(14) +
	1454	(int8_t)s->B(15) * (uint8_t)d->B(15));
	1455	#endif
	1456	}
	1457
	1458	void glue(helper_phsubw, SUFFIX)(CPUX86State env, Reg d, Reg *s)
	1459	{
	1460	d->W(0) = (int16_t)d->W(0) - (int16_t)d->W(1);
	1461	d->W(1) = (int16_t)d->W(2) - (int16_t)d->W(3);
	1462	XMM_ONLY(d->W(2) = (int16_t)d->W(4) - (int16_t)d->W(5));
	1463	XMM_ONLY(d->W(3) = (int16_t)d->W(6) - (int16_t)d->W(7));
	1464	d->W((2 << SHIFT) + 0) = (int16_t)s->W(0) - (int16_t)s->W(1);
	1465	d->W((2 << SHIFT) + 1) = (int16_t)s->W(2) - (int16_t)s->W(3);
	1466	XMM_ONLY(d->W(6) = (int16_t)s->W(4) - (int16_t)s->W(5));
	1467	XMM_ONLY(d->W(7) = (int16_t)s->W(6) - (int16_t)s->W(7));
	1468	}
	1469
	1470	void glue(helper_phsubd, SUFFIX)(CPUX86State env, Reg d, Reg *s)
	1471	{
	1472	d->L(0) = (int32_t)d->L(0) - (int32_t)d->L(1);
	1473	XMM_ONLY(d->L(1) = (int32_t)d->L(2) - (int32_t)d->L(3));
	1474	d->L((1 << SHIFT) + 0) = (int32_t)s->L(0) - (int32_t)s->L(1);
	1475	XMM_ONLY(d->L(3) = (int32_t)s->L(2) - (int32_t)s->L(3));
	1476	}
	1477
	1478	void glue(helper_phsubsw, SUFFIX)(CPUX86State env, Reg d, Reg *s)
	1479	{
	1480	d->W(0) = satsw((int16_t)d->W(0) - (int16_t)d->W(1));
	1481	d->W(1) = satsw((int16_t)d->W(2) - (int16_t)d->W(3));
	1482	XMM_ONLY(d->W(2) = satsw((int16_t)d->W(4) - (int16_t)d->W(5)));
	1483	XMM_ONLY(d->W(3) = satsw((int16_t)d->W(6) - (int16_t)d->W(7)));
	1484	d->W((2 << SHIFT) + 0) = satsw((int16_t)s->W(0) - (int16_t)s->W(1));
	1485	d->W((2 << SHIFT) + 1) = satsw((int16_t)s->W(2) - (int16_t)s->W(3));
	1486	XMM_ONLY(d->W(6) = satsw((int16_t)s->W(4) - (int16_t)s->W(5)));
	1487	XMM_ONLY(d->W(7) = satsw((int16_t)s->W(6) - (int16_t)s->W(7)));
	1488	}
	1489
	1490	#define FABSB(_, x) (x > INT8_MAX ? -(int8_t)x : x)
	1491	#define FABSW(_, x) (x > INT16_MAX ? -(int16_t)x : x)
	1492	#define FABSL(_, x) (x > INT32_MAX ? -(int32_t)x : x)
	1493	SSE_HELPER_B(helper_pabsb, FABSB)
	1494	SSE_HELPER_W(helper_pabsw, FABSW)
	1495	SSE_HELPER_L(helper_pabsd, FABSL)
	1496
	1497	#define FMULHRSW(d, s) (((int16_t) d * (int16_t)s + 0x4000) >> 15)
	1498	SSE_HELPER_W(helper_pmulhrsw, FMULHRSW)
	1499
	1500	#define FSIGNB(d, s) (s <= INT8_MAX ? s ? d : 0 : -(int8_t)d)
	1501	#define FSIGNW(d, s) (s <= INT16_MAX ? s ? d : 0 : -(int16_t)d)
	1502	#define FSIGNL(d, s) (s <= INT32_MAX ? s ? d : 0 : -(int32_t)d)
	1503	SSE_HELPER_B(helper_psignb, FSIGNB)
	1504	SSE_HELPER_W(helper_psignw, FSIGNW)
	1505	SSE_HELPER_L(helper_psignd, FSIGNL)
	1506
	1507	void glue(helper_palignr, SUFFIX)(CPUX86State env, Reg d, Reg *s,
	1508	int32_t shift)
	1509	{
	1510	Reg r;
	1511
	1512	/* XXX could be checked during translation */
	1513	if (shift >= (16 << SHIFT)) {
	1514	r.Q(0) = 0;
	1515	XMM_ONLY(r.Q(1) = 0);
	1516	} else {
	1517	shift <<= 3;
	1518	#define SHR(v, i) (i < 64 && i > -64 ? i > 0 ? v >> (i) : (v << -(i)) : 0)
	1519	#if SHIFT == 0
	1520	r.Q(0) = SHR(s->Q(0), shift - 0) \|
	1521	SHR(d->Q(0), shift - 64);
	1522	#else
	1523	r.Q(0) = SHR(s->Q(0), shift - 0) \|
	1524	SHR(s->Q(1), shift - 64) \|
	1525	SHR(d->Q(0), shift - 128) \|
	1526	SHR(d->Q(1), shift - 192);
	1527	r.Q(1) = SHR(s->Q(0), shift + 64) \|
	1528	SHR(s->Q(1), shift - 0) \|
	1529	SHR(d->Q(0), shift - 64) \|
	1530	SHR(d->Q(1), shift - 128);
	1531	#endif
	1532	#undef SHR
	1533	}
	1534
	1535	*d = r;
	1536	}
	1537
	1538	#define XMM0 (env->xmm_regs[0])
	1539
	1540	#if SHIFT == 1
	1541	#define SSE_HELPER_V(name, elem, num, F) \
	1542	void glue(name, SUFFIX)(CPUX86State env, Reg d, Reg *s) \
	1543	{ \
	1544	d->elem(0) = F(d->elem(0), s->elem(0), XMM0.elem(0)); \
	1545	d->elem(1) = F(d->elem(1), s->elem(1), XMM0.elem(1)); \
	1546	if (num > 2) { \
	1547	d->elem(2) = F(d->elem(2), s->elem(2), XMM0.elem(2)); \
	1548	d->elem(3) = F(d->elem(3), s->elem(3), XMM0.elem(3)); \
	1549	if (num > 4) { \
	1550	d->elem(4) = F(d->elem(4), s->elem(4), XMM0.elem(4)); \
	1551	d->elem(5) = F(d->elem(5), s->elem(5), XMM0.elem(5)); \
	1552	d->elem(6) = F(d->elem(6), s->elem(6), XMM0.elem(6)); \
	1553	d->elem(7) = F(d->elem(7), s->elem(7), XMM0.elem(7)); \
	1554	if (num > 8) { \
	1555	d->elem(8) = F(d->elem(8), s->elem(8), XMM0.elem(8)); \
	1556	d->elem(9) = F(d->elem(9), s->elem(9), XMM0.elem(9)); \
	1557	d->elem(10) = F(d->elem(10), s->elem(10), XMM0.elem(10)); \
	1558	d->elem(11) = F(d->elem(11), s->elem(11), XMM0.elem(11)); \
	1559	d->elem(12) = F(d->elem(12), s->elem(12), XMM0.elem(12)); \
	1560	d->elem(13) = F(d->elem(13), s->elem(13), XMM0.elem(13)); \
	1561	d->elem(14) = F(d->elem(14), s->elem(14), XMM0.elem(14)); \
	1562	d->elem(15) = F(d->elem(15), s->elem(15), XMM0.elem(15)); \
	1563	} \
	1564	} \
	1565	} \
	1566	}
	1567
	1568	#define SSE_HELPER_I(name, elem, num, F) \
	1569	void glue(name, SUFFIX)(CPUX86State env, Reg d, Reg *s, uint32_t imm) \
	1570	{ \
	1571	d->elem(0) = F(d->elem(0), s->elem(0), ((imm >> 0) & 1)); \
	1572	d->elem(1) = F(d->elem(1), s->elem(1), ((imm >> 1) & 1)); \
	1573	if (num > 2) { \
	1574	d->elem(2) = F(d->elem(2), s->elem(2), ((imm >> 2) & 1)); \
	1575	d->elem(3) = F(d->elem(3), s->elem(3), ((imm >> 3) & 1)); \
	1576	if (num > 4) { \
	1577	d->elem(4) = F(d->elem(4), s->elem(4), ((imm >> 4) & 1)); \
	1578	d->elem(5) = F(d->elem(5), s->elem(5), ((imm >> 5) & 1)); \
	1579	d->elem(6) = F(d->elem(6), s->elem(6), ((imm >> 6) & 1)); \
	1580	d->elem(7) = F(d->elem(7), s->elem(7), ((imm >> 7) & 1)); \
	1581	if (num > 8) { \
	1582	d->elem(8) = F(d->elem(8), s->elem(8), ((imm >> 8) & 1)); \
	1583	d->elem(9) = F(d->elem(9), s->elem(9), ((imm >> 9) & 1)); \
	1584	d->elem(10) = F(d->elem(10), s->elem(10), \
	1585	((imm >> 10) & 1)); \
	1586	d->elem(11) = F(d->elem(11), s->elem(11), \
	1587	((imm >> 11) & 1)); \
	1588	d->elem(12) = F(d->elem(12), s->elem(12), \
	1589	((imm >> 12) & 1)); \
	1590	d->elem(13) = F(d->elem(13), s->elem(13), \
	1591	((imm >> 13) & 1)); \
	1592	d->elem(14) = F(d->elem(14), s->elem(14), \
	1593	((imm >> 14) & 1)); \
	1594	d->elem(15) = F(d->elem(15), s->elem(15), \
	1595	((imm >> 15) & 1)); \
	1596	} \
	1597	} \
	1598	} \
	1599	}
	1600
	1601	/* SSE4.1 op helpers */
	1602	#define FBLENDVB(d, s, m) ((m & 0x80) ? s : d)
	1603	#define FBLENDVPS(d, s, m) ((m & 0x80000000) ? s : d)
	1604	#define FBLENDVPD(d, s, m) ((m & 0x8000000000000000LL) ? s : d)
	1605	SSE_HELPER_V(helper_pblendvb, B, 16, FBLENDVB)
	1606	SSE_HELPER_V(helper_blendvps, L, 4, FBLENDVPS)
	1607	SSE_HELPER_V(helper_blendvpd, Q, 2, FBLENDVPD)
	1608
	1609	void glue(helper_ptest, SUFFIX)(CPUX86State env, Reg d, Reg *s)
	1610	{
	1611	uint64_t zf = (s->Q(0) & d->Q(0)) \| (s->Q(1) & d->Q(1));
	1612	uint64_t cf = (s->Q(0) & ~d->Q(0)) \| (s->Q(1) & ~d->Q(1));
	1613
	1614	CC_SRC = (zf ? 0 : CC_Z) \| (cf ? 0 : CC_C);
	1615	}
	1616
	1617	#define SSE_HELPER_F(name, elem, num, F) \
	1618	void glue(name, SUFFIX)(CPUX86State env, Reg d, Reg *s) \
	1619	{ \
	1620	d->elem(0) = F(0); \
	1621	d->elem(1) = F(1); \
	1622	if (num > 2) { \
	1623	d->elem(2) = F(2); \
	1624	d->elem(3) = F(3); \
	1625	if (num > 4) { \
	1626	d->elem(4) = F(4); \
	1627	d->elem(5) = F(5); \
	1628	d->elem(6) = F(6); \
	1629	d->elem(7) = F(7); \
	1630	} \
	1631	} \
	1632	}
	1633
	1634	SSE_HELPER_F(helper_pmovsxbw, W, 8, (int8_t) s->B)
	1635	SSE_HELPER_F(helper_pmovsxbd, L, 4, (int8_t) s->B)
	1636	SSE_HELPER_F(helper_pmovsxbq, Q, 2, (int8_t) s->B)
	1637	SSE_HELPER_F(helper_pmovsxwd, L, 4, (int16_t) s->W)
	1638	SSE_HELPER_F(helper_pmovsxwq, Q, 2, (int16_t) s->W)
	1639	SSE_HELPER_F(helper_pmovsxdq, Q, 2, (int32_t) s->L)
	1640	SSE_HELPER_F(helper_pmovzxbw, W, 8, s->B)
	1641	SSE_HELPER_F(helper_pmovzxbd, L, 4, s->B)
	1642	SSE_HELPER_F(helper_pmovzxbq, Q, 2, s->B)
	1643	SSE_HELPER_F(helper_pmovzxwd, L, 4, s->W)
	1644	SSE_HELPER_F(helper_pmovzxwq, Q, 2, s->W)
	1645	SSE_HELPER_F(helper_pmovzxdq, Q, 2, s->L)
	1646
	1647	void glue(helper_pmuldq, SUFFIX)(CPUX86State env, Reg d, Reg *s)
	1648	{
	1649	d->Q(0) = (int64_t)(int32_t) d->L(0) * (int32_t) s->L(0);
	1650	d->Q(1) = (int64_t)(int32_t) d->L(2) * (int32_t) s->L(2);
	1651	}
	1652
	1653	#define FCMPEQQ(d, s) (d == s ? -1 : 0)
	1654	SSE_HELPER_Q(helper_pcmpeqq, FCMPEQQ)
	1655
	1656	void glue(helper_packusdw, SUFFIX)(CPUX86State env, Reg d, Reg *s)
	1657	{
	1658	d->W(0) = satuw((int32_t) d->L(0));
	1659	d->W(1) = satuw((int32_t) d->L(1));
	1660	d->W(2) = satuw((int32_t) d->L(2));
	1661	d->W(3) = satuw((int32_t) d->L(3));
	1662	d->W(4) = satuw((int32_t) s->L(0));
	1663	d->W(5) = satuw((int32_t) s->L(1));
	1664	d->W(6) = satuw((int32_t) s->L(2));
	1665	d->W(7) = satuw((int32_t) s->L(3));
	1666	}
	1667
	1668	#define FMINSB(d, s) MIN((int8_t)d, (int8_t)s)
	1669	#define FMINSD(d, s) MIN((int32_t)d, (int32_t)s)
	1670	#define FMAXSB(d, s) MAX((int8_t)d, (int8_t)s)
	1671	#define FMAXSD(d, s) MAX((int32_t)d, (int32_t)s)
	1672	SSE_HELPER_B(helper_pminsb, FMINSB)
	1673	SSE_HELPER_L(helper_pminsd, FMINSD)
	1674	SSE_HELPER_W(helper_pminuw, MIN)
	1675	SSE_HELPER_L(helper_pminud, MIN)
	1676	SSE_HELPER_B(helper_pmaxsb, FMAXSB)
	1677	SSE_HELPER_L(helper_pmaxsd, FMAXSD)
	1678	SSE_HELPER_W(helper_pmaxuw, MAX)
	1679	SSE_HELPER_L(helper_pmaxud, MAX)
	1680
	1681	#define FMULLD(d, s) ((int32_t)d * (int32_t)s)
	1682	SSE_HELPER_L(helper_pmulld, FMULLD)
	1683
	1684	void glue(helper_phminposuw, SUFFIX)(CPUX86State env, Reg d, Reg *s)
	1685	{
	1686	int idx = 0;
	1687
	1688	if (s->W(1) < s->W(idx)) {
	1689	idx = 1;
	1690	}
	1691	if (s->W(2) < s->W(idx)) {
	1692	idx = 2;
	1693	}
	1694	if (s->W(3) < s->W(idx)) {
	1695	idx = 3;
	1696	}
	1697	if (s->W(4) < s->W(idx)) {
	1698	idx = 4;
	1699	}
	1700	if (s->W(5) < s->W(idx)) {
	1701	idx = 5;
	1702	}
	1703	if (s->W(6) < s->W(idx)) {
	1704	idx = 6;
	1705	}
	1706	if (s->W(7) < s->W(idx)) {
	1707	idx = 7;
	1708	}
	1709
	1710	d->Q(1) = 0;
	1711	d->L(1) = 0;
	1712	d->W(1) = idx;
	1713	d->W(0) = s->W(idx);
	1714	}
	1715
	1716	void glue(helper_roundps, SUFFIX)(CPUX86State env, Reg d, Reg *s,
	1717	uint32_t mode)
	1718	{
	1719	signed char prev_rounding_mode;
	1720
	1721	prev_rounding_mode = env->sse_status.float_rounding_mode;
	1722	if (!(mode & (1 << 2))) {
	1723	switch (mode & 3) {
	1724	case 0:
	1725	set_float_rounding_mode(float_round_nearest_even, &env->sse_status);
	1726	break;
	1727	case 1:
	1728	set_float_rounding_mode(float_round_down, &env->sse_status);
	1729	break;
	1730	case 2:
	1731	set_float_rounding_mode(float_round_up, &env->sse_status);
	1732	break;
	1733	case 3:
	1734	set_float_rounding_mode(float_round_to_zero, &env->sse_status);
	1735	break;
	1736	}
	1737	}
	1738
	1739	d->XMM_S(0) = float32_round_to_int(s->XMM_S(0), &env->sse_status);
	1740	d->XMM_S(1) = float32_round_to_int(s->XMM_S(1), &env->sse_status);
	1741	d->XMM_S(2) = float32_round_to_int(s->XMM_S(2), &env->sse_status);
	1742	d->XMM_S(3) = float32_round_to_int(s->XMM_S(3), &env->sse_status);
	1743
	1744	#if 0 /* TODO */
	1745	if (mode & (1 << 3)) {
	1746	set_float_exception_flags(get_float_exception_flags(&env->sse_status) &
	1747	~float_flag_inexact,
	1748	&env->sse_status);
	1749	}
	1750	#endif
	1751	env->sse_status.float_rounding_mode = prev_rounding_mode;
	1752	}
	1753
	1754	void glue(helper_roundpd, SUFFIX)(CPUX86State env, Reg d, Reg *s,
	1755	uint32_t mode)
	1756	{
	1757	signed char prev_rounding_mode;
	1758
	1759	prev_rounding_mode = env->sse_status.float_rounding_mode;
	1760	if (!(mode & (1 << 2))) {
	1761	switch (mode & 3) {
	1762	case 0:
	1763	set_float_rounding_mode(float_round_nearest_even, &env->sse_status);
	1764	break;
	1765	case 1:
	1766	set_float_rounding_mode(float_round_down, &env->sse_status);
	1767	break;
	1768	case 2:
	1769	set_float_rounding_mode(float_round_up, &env->sse_status);
	1770	break;
	1771	case 3:
	1772	set_float_rounding_mode(float_round_to_zero, &env->sse_status);
	1773	break;
	1774	}
	1775	}
	1776
	1777	d->XMM_D(0) = float64_round_to_int(s->XMM_D(0), &env->sse_status);
	1778	d->XMM_D(1) = float64_round_to_int(s->XMM_D(1), &env->sse_status);
	1779
	1780	#if 0 /* TODO */
	1781	if (mode & (1 << 3)) {
	1782	set_float_exception_flags(get_float_exception_flags(&env->sse_status) &
	1783	~float_flag_inexact,
	1784	&env->sse_status);
	1785	}
	1786	#endif
	1787	env->sse_status.float_rounding_mode = prev_rounding_mode;
	1788	}
	1789
	1790	void glue(helper_roundss, SUFFIX)(CPUX86State env, Reg d, Reg *s,
	1791	uint32_t mode)
	1792	{
	1793	signed char prev_rounding_mode;
	1794
	1795	prev_rounding_mode = env->sse_status.float_rounding_mode;
	1796	if (!(mode & (1 << 2))) {
	1797	switch (mode & 3) {
	1798	case 0:
	1799	set_float_rounding_mode(float_round_nearest_even, &env->sse_status);
	1800	break;
	1801	case 1:
	1802	set_float_rounding_mode(float_round_down, &env->sse_status);
	1803	break;
	1804	case 2:
	1805	set_float_rounding_mode(float_round_up, &env->sse_status);
	1806	break;
	1807	case 3:
	1808	set_float_rounding_mode(float_round_to_zero, &env->sse_status);
	1809	break;
	1810	}
	1811	}
	1812
	1813	d->XMM_S(0) = float32_round_to_int(s->XMM_S(0), &env->sse_status);
	1814
	1815	#if 0 /* TODO */
	1816	if (mode & (1 << 3)) {
	1817	set_float_exception_flags(get_float_exception_flags(&env->sse_status) &
	1818	~float_flag_inexact,
	1819	&env->sse_status);
	1820	}
	1821	#endif
	1822	env->sse_status.float_rounding_mode = prev_rounding_mode;
	1823	}
	1824
	1825	void glue(helper_roundsd, SUFFIX)(CPUX86State env, Reg d, Reg *s,
	1826	uint32_t mode)
	1827	{
	1828	signed char prev_rounding_mode;
	1829
	1830	prev_rounding_mode = env->sse_status.float_rounding_mode;
	1831	if (!(mode & (1 << 2))) {
	1832	switch (mode & 3) {
	1833	case 0:
	1834	set_float_rounding_mode(float_round_nearest_even, &env->sse_status);
	1835	break;
	1836	case 1:
	1837	set_float_rounding_mode(float_round_down, &env->sse_status);
	1838	break;
	1839	case 2:
	1840	set_float_rounding_mode(float_round_up, &env->sse_status);
	1841	break;
	1842	case 3:
	1843	set_float_rounding_mode(float_round_to_zero, &env->sse_status);
	1844	break;
	1845	}
	1846	}
	1847
	1848	d->XMM_D(0) = float64_round_to_int(s->XMM_D(0), &env->sse_status);
	1849
	1850	#if 0 /* TODO */
	1851	if (mode & (1 << 3)) {
	1852	set_float_exception_flags(get_float_exception_flags(&env->sse_status) &
	1853	~float_flag_inexact,
	1854	&env->sse_status);
	1855	}
	1856	#endif
	1857	env->sse_status.float_rounding_mode = prev_rounding_mode;
	1858	}
	1859
	1860	#define FBLENDP(d, s, m) (m ? s : d)
	1861	SSE_HELPER_I(helper_blendps, L, 4, FBLENDP)
	1862	SSE_HELPER_I(helper_blendpd, Q, 2, FBLENDP)
	1863	SSE_HELPER_I(helper_pblendw, W, 8, FBLENDP)
	1864
	1865	void glue(helper_dpps, SUFFIX)(CPUX86State env, Reg d, Reg *s, uint32_t mask)
	1866	{
	1867	float32 iresult = float32_zero;
	1868
	1869	if (mask & (1 << 4)) {
	1870	iresult = float32_add(iresult,
	1871	float32_mul(d->XMM_S(0), s->XMM_S(0),
	1872	&env->sse_status),
	1873	&env->sse_status);
	1874	}
	1875	if (mask & (1 << 5)) {
	1876	iresult = float32_add(iresult,
	1877	float32_mul(d->XMM_S(1), s->XMM_S(1),
	1878	&env->sse_status),
	1879	&env->sse_status);
	1880	}
	1881	if (mask & (1 << 6)) {
	1882	iresult = float32_add(iresult,
	1883	float32_mul(d->XMM_S(2), s->XMM_S(2),
	1884	&env->sse_status),
	1885	&env->sse_status);
	1886	}
	1887	if (mask & (1 << 7)) {
	1888	iresult = float32_add(iresult,
	1889	float32_mul(d->XMM_S(3), s->XMM_S(3),
	1890	&env->sse_status),
	1891	&env->sse_status);
	1892	}
	1893	d->XMM_S(0) = (mask & (1 << 0)) ? iresult : float32_zero;
	1894	d->XMM_S(1) = (mask & (1 << 1)) ? iresult : float32_zero;
	1895	d->XMM_S(2) = (mask & (1 << 2)) ? iresult : float32_zero;
	1896	d->XMM_S(3) = (mask & (1 << 3)) ? iresult : float32_zero;
	1897	}
	1898
	1899	void glue(helper_dppd, SUFFIX)(CPUX86State env, Reg d, Reg *s, uint32_t mask)
	1900	{
	1901	float64 iresult = float64_zero;
	1902
	1903	if (mask & (1 << 4)) {
	1904	iresult = float64_add(iresult,
	1905	float64_mul(d->XMM_D(0), s->XMM_D(0),
	1906	&env->sse_status),
	1907	&env->sse_status);
	1908	}
	1909	if (mask & (1 << 5)) {
	1910	iresult = float64_add(iresult,
	1911	float64_mul(d->XMM_D(1), s->XMM_D(1),
	1912	&env->sse_status),
	1913	&env->sse_status);
	1914	}
	1915	d->XMM_D(0) = (mask & (1 << 0)) ? iresult : float64_zero;
	1916	d->XMM_D(1) = (mask & (1 << 1)) ? iresult : float64_zero;
	1917	}
	1918
	1919	void glue(helper_mpsadbw, SUFFIX)(CPUX86State env, Reg d, Reg *s,
	1920	uint32_t offset)
	1921	{
	1922	int s0 = (offset & 3) << 2;
	1923	int d0 = (offset & 4) << 0;
	1924	int i;
	1925	Reg r;
	1926
	1927	for (i = 0; i < 8; i++, d0++) {
	1928	r.W(i) = 0;
	1929	r.W(i) += abs1(d->B(d0 + 0) - s->B(s0 + 0));
	1930	r.W(i) += abs1(d->B(d0 + 1) - s->B(s0 + 1));
	1931	r.W(i) += abs1(d->B(d0 + 2) - s->B(s0 + 2));
	1932	r.W(i) += abs1(d->B(d0 + 3) - s->B(s0 + 3));
	1933	}
	1934
	1935	*d = r;
	1936	}
	1937
	1938	/* SSE4.2 op helpers */
	1939	#define FCMPGTQ(d, s) ((int64_t)d > (int64_t)s ? -1 : 0)
	1940	SSE_HELPER_Q(helper_pcmpgtq, FCMPGTQ)
	1941
	1942	static inline int pcmp_elen(CPUX86State *env, int reg, uint32_t ctrl)
	1943	{
	1944	int val;
	1945
	1946	/* Presence of REX.W is indicated by a bit higher than 7 set */
	1947	if (ctrl >> 8) {
	1948	val = abs1((int64_t)env->regs[reg]);
	1949	} else {
	1950	val = abs1((int32_t)env->regs[reg]);
	1951	}
	1952
	1953	if (ctrl & 1) {
	1954	if (val > 8) {
	1955	return 8;
	1956	}
	1957	} else {
	1958	if (val > 16) {
	1959	return 16;
	1960	}
	1961	}
	1962	return val;
	1963	}
	1964
	1965	static inline int pcmp_ilen(Reg *r, uint8_t ctrl)
	1966	{
	1967	int val = 0;
	1968
	1969	if (ctrl & 1) {
	1970	while (val < 8 && r->W(val)) {
	1971	val++;
	1972	}
	1973	} else {
	1974	while (val < 16 && r->B(val)) {
	1975	val++;
	1976	}
	1977	}
	1978
	1979	return val;
	1980	}
	1981
	1982	static inline int pcmp_val(Reg *r, uint8_t ctrl, int i)
	1983	{
	1984	switch ((ctrl >> 0) & 3) {
	1985	case 0:
	1986	return r->B(i);
	1987	case 1:
	1988	return r->W(i);
	1989	case 2:
	1990	return (int8_t)r->B(i);
	1991	case 3:
	1992	default:
	1993	return (int16_t)r->W(i);
	1994	}
	1995	}
	1996
	1997	static inline unsigned pcmpxstrx(CPUX86State env, Reg d, Reg *s,
	1998	int8_t ctrl, int valids, int validd)
	1999	{
	2000	unsigned int res = 0;
	2001	int v;
	2002	int j, i;
	2003	int upper = (ctrl & 1) ? 7 : 15;
	2004
	2005	valids--;
	2006	validd--;
	2007
	2008	CC_SRC = (valids < upper ? CC_Z : 0) \| (validd < upper ? CC_S : 0);
	2009
	2010	switch ((ctrl >> 2) & 3) {
	2011	case 0:
	2012	for (j = valids; j >= 0; j--) {
	2013	res <<= 1;
	2014	v = pcmp_val(s, ctrl, j);
	2015	for (i = validd; i >= 0; i--) {
	2016	res \|= (v == pcmp_val(d, ctrl, i));
	2017	}
	2018	}
	2019	break;
	2020	case 1:
	2021	for (j = valids; j >= 0; j--) {
	2022	res <<= 1;
	2023	v = pcmp_val(s, ctrl, j);
	2024	for (i = ((validd - 1) \| 1); i >= 0; i -= 2) {
	2025	res \|= (pcmp_val(d, ctrl, i - 0) >= v &&
	2026	pcmp_val(d, ctrl, i - 1) <= v);
	2027	}
	2028	}
	2029	break;
	2030	case 2:
	2031	res = (1 << (upper - MAX(valids, validd))) - 1;
	2032	res <<= MAX(valids, validd) - MIN(valids, validd);
	2033	for (i = MIN(valids, validd); i >= 0; i--) {
	2034	res <<= 1;
	2035	v = pcmp_val(s, ctrl, i);
	2036	res \|= (v == pcmp_val(d, ctrl, i));
	2037	}
	2038	break;
	2039	case 3:
	2040	for (j = valids; j >= 0; j--) {
	2041	res <<= 1;
	2042	v = 1;
	2043	for (i = MIN(valids - j, validd); i >= 0; i--) {
	2044	v &= (pcmp_val(s, ctrl, i + j) == pcmp_val(d, ctrl, i));
	2045	}
	2046	res \|= v;
	2047	}
	2048	break;
	2049	}
	2050
	2051	switch ((ctrl >> 4) & 3) {
	2052	case 1:
	2053	res ^= (2 << upper) - 1;
	2054	break;
	2055	case 3:
	2056	res ^= (1 << (valids + 1)) - 1;
	2057	break;
	2058	}
	2059
	2060	if (res) {
	2061	CC_SRC \|= CC_C;
	2062	}
	2063	if (res & 1) {
	2064	CC_SRC \|= CC_O;
	2065	}
	2066
	2067	return res;
	2068	}
	2069
	2070	void glue(helper_pcmpestri, SUFFIX)(CPUX86State env, Reg d, Reg *s,
	2071	uint32_t ctrl)
	2072	{
	2073	unsigned int res = pcmpxstrx(env, d, s, ctrl,
	2074	pcmp_elen(env, R_EDX, ctrl),
	2075	pcmp_elen(env, R_EAX, ctrl));
	2076
	2077	if (res) {
	2078	env->regs[R_ECX] = (ctrl & (1 << 6)) ? 31 - clz32(res) : ctz32(res);
	2079	} else {
	2080	env->regs[R_ECX] = 16 >> (ctrl & (1 << 0));
	2081	}
	2082	}
	2083
	2084	void glue(helper_pcmpestrm, SUFFIX)(CPUX86State env, Reg d, Reg *s,
	2085	uint32_t ctrl)
	2086	{
	2087	int i;
	2088	unsigned int res = pcmpxstrx(env, d, s, ctrl,
	2089	pcmp_elen(env, R_EDX, ctrl),
	2090	pcmp_elen(env, R_EAX, ctrl));
	2091
	2092	if ((ctrl >> 6) & 1) {
	2093	if (ctrl & 1) {
	2094	for (i = 0; i < 8; i++, res >>= 1) {
	2095	env->xmm_regs[0].W(i) = (res & 1) ? ~0 : 0;
	2096	}
	2097	} else {
	2098	for (i = 0; i < 16; i++, res >>= 1) {
	2099	env->xmm_regs[0].B(i) = (res & 1) ? ~0 : 0;
	2100	}
	2101	}
	2102	} else {
	2103	env->xmm_regs[0].Q(1) = 0;
	2104	env->xmm_regs[0].Q(0) = res;
	2105	}
	2106	}
	2107
	2108	void glue(helper_pcmpistri, SUFFIX)(CPUX86State env, Reg d, Reg *s,
	2109	uint32_t ctrl)
	2110	{
	2111	unsigned int res = pcmpxstrx(env, d, s, ctrl,
	2112	pcmp_ilen(s, ctrl),
	2113	pcmp_ilen(d, ctrl));
	2114
	2115	if (res) {
	2116	env->regs[R_ECX] = (ctrl & (1 << 6)) ? 31 - clz32(res) : ctz32(res);
	2117	} else {
	2118	env->regs[R_ECX] = 16 >> (ctrl & (1 << 0));
	2119	}
	2120	}
	2121
	2122	void glue(helper_pcmpistrm, SUFFIX)(CPUX86State env, Reg d, Reg *s,
	2123	uint32_t ctrl)
	2124	{
	2125	int i;
	2126	unsigned int res = pcmpxstrx(env, d, s, ctrl,
	2127	pcmp_ilen(s, ctrl),
	2128	pcmp_ilen(d, ctrl));
	2129
	2130	if ((ctrl >> 6) & 1) {
	2131	if (ctrl & 1) {
	2132	for (i = 0; i < 8; i++, res >>= 1) {
	2133	env->xmm_regs[0].W(i) = (res & 1) ? ~0 : 0;
	2134	}
	2135	} else {
	2136	for (i = 0; i < 16; i++, res >>= 1) {
	2137	env->xmm_regs[0].B(i) = (res & 1) ? ~0 : 0;
	2138	}
	2139	}
	2140	} else {
	2141	env->xmm_regs[0].Q(1) = 0;
	2142	env->xmm_regs[0].Q(0) = res;
	2143	}
	2144	}
	2145
	2146	#define CRCPOLY 0x1edc6f41
	2147	#define CRCPOLY_BITREV 0x82f63b78
	2148	target_ulong helper_crc32(uint32_t crc1, target_ulong msg, uint32_t len)
	2149	{
	2150	target_ulong crc = (msg & ((target_ulong) -1 >>
	2151	(TARGET_LONG_BITS - len))) ^ crc1;
	2152
	2153	while (len--) {
	2154	crc = (crc >> 1) ^ ((crc & 1) ? CRCPOLY_BITREV : 0);
	2155	}
	2156
	2157	return crc;
	2158	}
	2159
	2160	#define POPMASK(i) ((target_ulong) -1 / ((1LL << (1 << i)) + 1))
	2161	#define POPCOUNT(n, i) ((n & POPMASK(i)) + ((n >> (1 << i)) & POPMASK(i)))
	2162	target_ulong helper_popcnt(CPUX86State *env, target_ulong n, uint32_t type)
	2163	{
	2164	CC_SRC = n ? 0 : CC_Z;
	2165
	2166	n = POPCOUNT(n, 0);
	2167	n = POPCOUNT(n, 1);
	2168	n = POPCOUNT(n, 2);
	2169	n = POPCOUNT(n, 3);
	2170	if (type == 1) {
	2171	return n & 0xff;
	2172	}
	2173
	2174	n = POPCOUNT(n, 4);
	2175	#ifndef TARGET_X86_64
	2176	return n;
	2177	#else
	2178	if (type == 2) {
	2179	return n & 0xff;
	2180	}
	2181
	2182	return POPCOUNT(n, 5);
	2183	#endif
	2184	}
	2185
	2186	void glue(helper_pclmulqdq, SUFFIX)(CPUX86State env, Reg d, Reg *s,
	2187	uint32_t ctrl)
	2188	{
	2189	uint64_t ah, al, b, resh, resl;
	2190
	2191	ah = 0;
	2192	al = d->Q((ctrl & 1) != 0);
	2193	b = s->Q((ctrl & 16) != 0);
	2194	resh = resl = 0;
	2195
	2196	while (b) {
	2197	if (b & 1) {
	2198	resl ^= al;
	2199	resh ^= ah;
	2200	}
	2201	ah = (ah << 1) \| (al >> 63);
	2202	al <<= 1;
	2203	b >>= 1;
	2204	}
	2205
	2206	d->Q(0) = resl;
	2207	d->Q(1) = resh;
	2208	}
	2209
	2210	void glue(helper_aesdec, SUFFIX)(CPUX86State env, Reg d, Reg *s)
	2211	{
	2212	int i;
	2213	Reg st = *d;
	2214	Reg rk = *s;
	2215
	2216	for (i = 0 ; i < 4 ; i++) {
	2217	d->L(i) = rk.L(i) ^ bswap32(AES_Td0[st.B(AES_ishifts[4*i+0])] ^
	2218	AES_Td1[st.B(AES_ishifts[4*i+1])] ^
	2219	AES_Td2[st.B(AES_ishifts[4*i+2])] ^
	2220	AES_Td3[st.B(AES_ishifts[4*i+3])]);
	2221	}
	2222	}
	2223
	2224	void glue(helper_aesdeclast, SUFFIX)(CPUX86State env, Reg d, Reg *s)
	2225	{
	2226	int i;
	2227	Reg st = *d;
	2228	Reg rk = *s;
	2229
	2230	for (i = 0; i < 16; i++) {
	2231	d->B(i) = rk.B(i) ^ (AES_isbox[st.B(AES_ishifts[i])]);
	2232	}
	2233	}
	2234
	2235	void glue(helper_aesenc, SUFFIX)(CPUX86State env, Reg d, Reg *s)
	2236	{
	2237	int i;
	2238	Reg st = *d;
	2239	Reg rk = *s;
	2240
	2241	for (i = 0 ; i < 4 ; i++) {
	2242	d->L(i) = rk.L(i) ^ bswap32(AES_Te0[st.B(AES_shifts[4*i+0])] ^
	2243	AES_Te1[st.B(AES_shifts[4*i+1])] ^
	2244	AES_Te2[st.B(AES_shifts[4*i+2])] ^
	2245	AES_Te3[st.B(AES_shifts[4*i+3])]);
	2246	}
	2247	}
	2248
	2249	void glue(helper_aesenclast, SUFFIX)(CPUX86State env, Reg d, Reg *s)
	2250	{
	2251	int i;
	2252	Reg st = *d;
	2253	Reg rk = *s;
	2254
	2255	for (i = 0; i < 16; i++) {
	2256	d->B(i) = rk.B(i) ^ (AES_sbox[st.B(AES_shifts[i])]);
	2257	}
	2258
	2259	}
	2260
	2261	void glue(helper_aesimc, SUFFIX)(CPUX86State env, Reg d, Reg *s)
	2262	{
	2263	int i;
	2264	Reg tmp = *s;
	2265
	2266	for (i = 0 ; i < 4 ; i++) {
	2267	d->L(i) = bswap32(AES_imc[tmp.B(4*i+0)][0] ^
	2268	AES_imc[tmp.B(4*i+1)][1] ^
	2269	AES_imc[tmp.B(4*i+2)][2] ^
	2270	AES_imc[tmp.B(4*i+3)][3]);
	2271	}
	2272	}
	2273
	2274	void glue(helper_aeskeygenassist, SUFFIX)(CPUX86State env, Reg d, Reg *s,
	2275	uint32_t ctrl)
	2276	{
	2277	int i;
	2278	Reg tmp = *s;
	2279
	2280	for (i = 0 ; i < 4 ; i++) {
	2281	d->B(i) = AES_sbox[tmp.B(i + 4)];
	2282	d->B(i + 8) = AES_sbox[tmp.B(i + 12)];
	2283	}
	2284	d->L(1) = (d->L(0) << 24 \| d->L(0) >> 8) ^ ctrl;
	2285	d->L(3) = (d->L(2) << 24 \| d->L(2) >> 8) ^ ctrl;
	2286	}
	2287	#endif
	2288
	2289	#undef SHIFT
	2290	#undef XMM_ONLY
	2291	#undef Reg
	2292	#undef B
	2293	#undef W
	2294	#undef L
	2295	#undef Q
	2296	#undef SUFFIX