[linux.git] / arch / powerpc / lib / copyuser_power7.S

/*
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
 *
 * Copyright (C) IBM Corporation, 2011
 *
 * Author: Anton Blanchard <[email protected]>
 */
#include <asm/ppc_asm.h>

#ifdef __BIG_ENDIAN__
#define LVS(VRT,RA,RB)		lvsl	VRT,RA,RB
#define VPERM(VRT,VRA,VRB,VRC)	vperm	VRT,VRA,VRB,VRC
#else
#define LVS(VRT,RA,RB)		lvsr	VRT,RA,RB
#define VPERM(VRT,VRA,VRB,VRC)	vperm	VRT,VRB,VRA,VRC
#endif

	.macro err1
100:
	EX_TABLE(100b,.Ldo_err1)
	.endm

	.macro err2
200:
	EX_TABLE(200b,.Ldo_err2)
	.endm

#ifdef CONFIG_ALTIVEC
	.macro err3
300:
	EX_TABLE(300b,.Ldo_err3)
	.endm

	.macro err4
400:
	EX_TABLE(400b,.Ldo_err4)
	.endm


.Ldo_err4:
	ld	r16,STK_REG(R16)(r1)
	ld	r15,STK_REG(R15)(r1)
	ld	r14,STK_REG(R14)(r1)
.Ldo_err3:
	bl	exit_vmx_usercopy
	ld	r0,STACKFRAMESIZE+16(r1)
	mtlr	r0
	b	.Lexit
#endif /* CONFIG_ALTIVEC */

.Ldo_err2:
	ld	r22,STK_REG(R22)(r1)
	ld	r21,STK_REG(R21)(r1)
	ld	r20,STK_REG(R20)(r1)
	ld	r19,STK_REG(R19)(r1)
	ld	r18,STK_REG(R18)(r1)
	ld	r17,STK_REG(R17)(r1)
	ld	r16,STK_REG(R16)(r1)
	ld	r15,STK_REG(R15)(r1)
	ld	r14,STK_REG(R14)(r1)
.Lexit:
	addi	r1,r1,STACKFRAMESIZE
.Ldo_err1:
	ld	r3,-STACKFRAMESIZE+STK_REG(R31)(r1)
	ld	r4,-STACKFRAMESIZE+STK_REG(R30)(r1)
	ld	r5,-STACKFRAMESIZE+STK_REG(R29)(r1)
	b	__copy_tofrom_user_base


_GLOBAL(__copy_tofrom_user_power7)
#ifdef CONFIG_ALTIVEC
	cmpldi	r5,16
	cmpldi	cr1,r5,3328

	std	r3,-STACKFRAMESIZE+STK_REG(R31)(r1)
	std	r4,-STACKFRAMESIZE+STK_REG(R30)(r1)
	std	r5,-STACKFRAMESIZE+STK_REG(R29)(r1)

	blt	.Lshort_copy
	bge	cr1,.Lvmx_copy
#else
	cmpldi	r5,16

	std	r3,-STACKFRAMESIZE+STK_REG(R31)(r1)
	std	r4,-STACKFRAMESIZE+STK_REG(R30)(r1)
	std	r5,-STACKFRAMESIZE+STK_REG(R29)(r1)

	blt	.Lshort_copy
#endif

.Lnonvmx_copy:
	/* Get the source 8B aligned */
	neg	r6,r4
	mtocrf	0x01,r6
	clrldi	r6,r6,(64-3)

	bf	cr7*4+3,1f
err1;	lbz	r0,0(r4)
	addi	r4,r4,1
err1;	stb	r0,0(r3)
	addi	r3,r3,1

1:	bf	cr7*4+2,2f
err1;	lhz	r0,0(r4)
	addi	r4,r4,2
err1;	sth	r0,0(r3)
	addi	r3,r3,2

2:	bf	cr7*4+1,3f
err1;	lwz	r0,0(r4)
	addi	r4,r4,4
err1;	stw	r0,0(r3)
	addi	r3,r3,4

3:	sub	r5,r5,r6
	cmpldi	r5,128
	blt	5f

	mflr	r0
	stdu	r1,-STACKFRAMESIZE(r1)
	std	r14,STK_REG(R14)(r1)
	std	r15,STK_REG(R15)(r1)
	std	r16,STK_REG(R16)(r1)
	std	r17,STK_REG(R17)(r1)
	std	r18,STK_REG(R18)(r1)
	std	r19,STK_REG(R19)(r1)
	std	r20,STK_REG(R20)(r1)
	std	r21,STK_REG(R21)(r1)
	std	r22,STK_REG(R22)(r1)
	std	r0,STACKFRAMESIZE+16(r1)

	srdi	r6,r5,7
	mtctr	r6

	/* Now do cacheline (128B) sized loads and stores. */
	.align	5
4:
err2;	ld	r0,0(r4)
err2;	ld	r6,8(r4)
err2;	ld	r7,16(r4)
err2;	ld	r8,24(r4)
err2;	ld	r9,32(r4)
err2;	ld	r10,40(r4)
err2;	ld	r11,48(r4)
err2;	ld	r12,56(r4)
err2;	ld	r14,64(r4)
err2;	ld	r15,72(r4)
err2;	ld	r16,80(r4)
err2;	ld	r17,88(r4)
err2;	ld	r18,96(r4)
err2;	ld	r19,104(r4)
err2;	ld	r20,112(r4)
err2;	ld	r21,120(r4)
	addi	r4,r4,128
err2;	std	r0,0(r3)
err2;	std	r6,8(r3)
err2;	std	r7,16(r3)
err2;	std	r8,24(r3)
err2;	std	r9,32(r3)
err2;	std	r10,40(r3)
err2;	std	r11,48(r3)
err2;	std	r12,56(r3)
err2;	std	r14,64(r3)
err2;	std	r15,72(r3)
err2;	std	r16,80(r3)
err2;	std	r17,88(r3)
err2;	std	r18,96(r3)
err2;	std	r19,104(r3)
err2;	std	r20,112(r3)
err2;	std	r21,120(r3)
	addi	r3,r3,128
	bdnz	4b

	clrldi	r5,r5,(64-7)

	ld	r14,STK_REG(R14)(r1)
	ld	r15,STK_REG(R15)(r1)
	ld	r16,STK_REG(R16)(r1)
	ld	r17,STK_REG(R17)(r1)
	ld	r18,STK_REG(R18)(r1)
	ld	r19,STK_REG(R19)(r1)
	ld	r20,STK_REG(R20)(r1)
	ld	r21,STK_REG(R21)(r1)
	ld	r22,STK_REG(R22)(r1)
	addi	r1,r1,STACKFRAMESIZE

	/* Up to 127B to go */
5:	srdi	r6,r5,4
	mtocrf	0x01,r6

6:	bf	cr7*4+1,7f
err1;	ld	r0,0(r4)
err1;	ld	r6,8(r4)
err1;	ld	r7,16(r4)
err1;	ld	r8,24(r4)
err1;	ld	r9,32(r4)
err1;	ld	r10,40(r4)
err1;	ld	r11,48(r4)
err1;	ld	r12,56(r4)
	addi	r4,r4,64
err1;	std	r0,0(r3)
err1;	std	r6,8(r3)
err1;	std	r7,16(r3)
err1;	std	r8,24(r3)
err1;	std	r9,32(r3)
err1;	std	r10,40(r3)
err1;	std	r11,48(r3)
err1;	std	r12,56(r3)
	addi	r3,r3,64

	/* Up to 63B to go */
7:	bf	cr7*4+2,8f
err1;	ld	r0,0(r4)
err1;	ld	r6,8(r4)
err1;	ld	r7,16(r4)
err1;	ld	r8,24(r4)
	addi	r4,r4,32
err1;	std	r0,0(r3)
err1;	std	r6,8(r3)
err1;	std	r7,16(r3)
err1;	std	r8,24(r3)
	addi	r3,r3,32

	/* Up to 31B to go */
8:	bf	cr7*4+3,9f
err1;	ld	r0,0(r4)
err1;	ld	r6,8(r4)
	addi	r4,r4,16
err1;	std	r0,0(r3)
err1;	std	r6,8(r3)
	addi	r3,r3,16

9:	clrldi	r5,r5,(64-4)

	/* Up to 15B to go */
.Lshort_copy:
	mtocrf	0x01,r5
	bf	cr7*4+0,12f
err1;	lwz	r0,0(r4)	/* Less chance of a reject with word ops */
err1;	lwz	r6,4(r4)
	addi	r4,r4,8
err1;	stw	r0,0(r3)
err1;	stw	r6,4(r3)
	addi	r3,r3,8

12:	bf	cr7*4+1,13f
err1;	lwz	r0,0(r4)
	addi	r4,r4,4
err1;	stw	r0,0(r3)
	addi	r3,r3,4

13:	bf	cr7*4+2,14f
err1;	lhz	r0,0(r4)
	addi	r4,r4,2
err1;	sth	r0,0(r3)
	addi	r3,r3,2

14:	bf	cr7*4+3,15f
err1;	lbz	r0,0(r4)
err1;	stb	r0,0(r3)

15:	li	r3,0
	blr

.Lunwind_stack_nonvmx_copy:
	addi	r1,r1,STACKFRAMESIZE
	b	.Lnonvmx_copy

#ifdef CONFIG_ALTIVEC
.Lvmx_copy:
	mflr	r0
	std	r0,16(r1)
	stdu	r1,-STACKFRAMESIZE(r1)
	bl	enter_vmx_usercopy
	cmpwi	cr1,r3,0
	ld	r0,STACKFRAMESIZE+16(r1)
	ld	r3,STK_REG(R31)(r1)
	ld	r4,STK_REG(R30)(r1)
	ld	r5,STK_REG(R29)(r1)
	mtlr	r0

	/*
	 * We prefetch both the source and destination using enhanced touch
	 * instructions. We use a stream ID of 0 for the load side and
	 * 1 for the store side.
	 */
	clrrdi	r6,r4,7
	clrrdi	r9,r3,7
	ori	r9,r9,1		/* stream=1 */

	srdi	r7,r5,7		/* length in cachelines, capped at 0x3FF */
	cmpldi	r7,0x3FF
	ble	1f
	li	r7,0x3FF
1:	lis	r0,0x0E00	/* depth=7 */
	sldi	r7,r7,7
	or	r7,r7,r0
	ori	r10,r7,1	/* stream=1 */

	lis	r8,0x8000	/* GO=1 */
	clrldi	r8,r8,32

	/* setup read stream 0 */
	dcbt	0,r6,0b01000   /* addr from */
	dcbt	0,r7,0b01010   /* length and depth from */
	/* setup write stream 1 */
	dcbtst	0,r9,0b01000   /* addr to */
	dcbtst	0,r10,0b01010  /* length and depth to */
	eieio
	dcbt	0,r8,0b01010	/* all streams GO */

	beq	cr1,.Lunwind_stack_nonvmx_copy

	/*
	 * If source and destination are not relatively aligned we use a
	 * slower permute loop.
	 */
	xor	r6,r4,r3
	rldicl.	r6,r6,0,(64-4)
	bne	.Lvmx_unaligned_copy

	/* Get the destination 16B aligned */
	neg	r6,r3
	mtocrf	0x01,r6
	clrldi	r6,r6,(64-4)

	bf	cr7*4+3,1f
err3;	lbz	r0,0(r4)
	addi	r4,r4,1
err3;	stb	r0,0(r3)
	addi	r3,r3,1

1:	bf	cr7*4+2,2f
err3;	lhz	r0,0(r4)
	addi	r4,r4,2
err3;	sth	r0,0(r3)
	addi	r3,r3,2

2:	bf	cr7*4+1,3f
err3;	lwz	r0,0(r4)
	addi	r4,r4,4
err3;	stw	r0,0(r3)
	addi	r3,r3,4

3:	bf	cr7*4+0,4f
err3;	ld	r0,0(r4)
	addi	r4,r4,8
err3;	std	r0,0(r3)
	addi	r3,r3,8

4:	sub	r5,r5,r6

	/* Get the desination 128B aligned */
	neg	r6,r3
	srdi	r7,r6,4
	mtocrf	0x01,r7
	clrldi	r6,r6,(64-7)

	li	r9,16
	li	r10,32
	li	r11,48

	bf	cr7*4+3,5f
err3;	lvx	v1,0,r4
	addi	r4,r4,16
err3;	stvx	v1,0,r3
	addi	r3,r3,16

5:	bf	cr7*4+2,6f
err3;	lvx	v1,0,r4
err3;	lvx	v0,r4,r9
	addi	r4,r4,32
err3;	stvx	v1,0,r3
err3;	stvx	v0,r3,r9
	addi	r3,r3,32

6:	bf	cr7*4+1,7f
err3;	lvx	v3,0,r4
err3;	lvx	v2,r4,r9
err3;	lvx	v1,r4,r10
err3;	lvx	v0,r4,r11
	addi	r4,r4,64
err3;	stvx	v3,0,r3
err3;	stvx	v2,r3,r9
err3;	stvx	v1,r3,r10
err3;	stvx	v0,r3,r11
	addi	r3,r3,64

7:	sub	r5,r5,r6
	srdi	r6,r5,7

	std	r14,STK_REG(R14)(r1)
	std	r15,STK_REG(R15)(r1)
	std	r16,STK_REG(R16)(r1)

	li	r12,64
	li	r14,80
	li	r15,96
	li	r16,112

	mtctr	r6

	/*
	 * Now do cacheline sized loads and stores. By this stage the
	 * cacheline stores are also cacheline aligned.
	 */
	.align	5
8:
err4;	lvx	v7,0,r4
err4;	lvx	v6,r4,r9
err4;	lvx	v5,r4,r10
err4;	lvx	v4,r4,r11
err4;	lvx	v3,r4,r12
err4;	lvx	v2,r4,r14
err4;	lvx	v1,r4,r15
err4;	lvx	v0,r4,r16
	addi	r4,r4,128
err4;	stvx	v7,0,r3
err4;	stvx	v6,r3,r9
err4;	stvx	v5,r3,r10
err4;	stvx	v4,r3,r11
err4;	stvx	v3,r3,r12
err4;	stvx	v2,r3,r14
err4;	stvx	v1,r3,r15
err4;	stvx	v0,r3,r16
	addi	r3,r3,128
	bdnz	8b

	ld	r14,STK_REG(R14)(r1)
	ld	r15,STK_REG(R15)(r1)
	ld	r16,STK_REG(R16)(r1)

	/* Up to 127B to go */
	clrldi	r5,r5,(64-7)
	srdi	r6,r5,4
	mtocrf	0x01,r6

	bf	cr7*4+1,9f
err3;	lvx	v3,0,r4
err3;	lvx	v2,r4,r9
err3;	lvx	v1,r4,r10
err3;	lvx	v0,r4,r11
	addi	r4,r4,64
err3;	stvx	v3,0,r3
err3;	stvx	v2,r3,r9
err3;	stvx	v1,r3,r10
err3;	stvx	v0,r3,r11
	addi	r3,r3,64

9:	bf	cr7*4+2,10f
err3;	lvx	v1,0,r4
err3;	lvx	v0,r4,r9
	addi	r4,r4,32
err3;	stvx	v1,0,r3
err3;	stvx	v0,r3,r9
	addi	r3,r3,32

10:	bf	cr7*4+3,11f
err3;	lvx	v1,0,r4
	addi	r4,r4,16
err3;	stvx	v1,0,r3
	addi	r3,r3,16

	/* Up to 15B to go */
11:	clrldi	r5,r5,(64-4)
	mtocrf	0x01,r5
	bf	cr7*4+0,12f
err3;	ld	r0,0(r4)
	addi	r4,r4,8
err3;	std	r0,0(r3)
	addi	r3,r3,8

12:	bf	cr7*4+1,13f
err3;	lwz	r0,0(r4)
	addi	r4,r4,4
err3;	stw	r0,0(r3)
	addi	r3,r3,4

13:	bf	cr7*4+2,14f
err3;	lhz	r0,0(r4)
	addi	r4,r4,2
err3;	sth	r0,0(r3)
	addi	r3,r3,2

14:	bf	cr7*4+3,15f
err3;	lbz	r0,0(r4)
err3;	stb	r0,0(r3)

15:	addi	r1,r1,STACKFRAMESIZE
	b	exit_vmx_usercopy	/* tail call optimise */

.Lvmx_unaligned_copy:
	/* Get the destination 16B aligned */
	neg	r6,r3
	mtocrf	0x01,r6
	clrldi	r6,r6,(64-4)

	bf	cr7*4+3,1f
err3;	lbz	r0,0(r4)
	addi	r4,r4,1
err3;	stb	r0,0(r3)
	addi	r3,r3,1

1:	bf	cr7*4+2,2f
err3;	lhz	r0,0(r4)
	addi	r4,r4,2
err3;	sth	r0,0(r3)
	addi	r3,r3,2

2:	bf	cr7*4+1,3f
err3;	lwz	r0,0(r4)
	addi	r4,r4,4
err3;	stw	r0,0(r3)
	addi	r3,r3,4

3:	bf	cr7*4+0,4f
err3;	lwz	r0,0(r4)	/* Less chance of a reject with word ops */
err3;	lwz	r7,4(r4)
	addi	r4,r4,8
err3;	stw	r0,0(r3)
err3;	stw	r7,4(r3)
	addi	r3,r3,8

4:	sub	r5,r5,r6

	/* Get the desination 128B aligned */
	neg	r6,r3
	srdi	r7,r6,4
	mtocrf	0x01,r7
	clrldi	r6,r6,(64-7)

	li	r9,16
	li	r10,32
	li	r11,48

	LVS(v16,0,r4)		/* Setup permute control vector */
err3;	lvx	v0,0,r4
	addi	r4,r4,16

	bf	cr7*4+3,5f
err3;	lvx	v1,0,r4
	VPERM(v8,v0,v1,v16)
	addi	r4,r4,16
err3;	stvx	v8,0,r3
	addi	r3,r3,16
	vor	v0,v1,v1

5:	bf	cr7*4+2,6f
err3;	lvx	v1,0,r4
	VPERM(v8,v0,v1,v16)
err3;	lvx	v0,r4,r9
	VPERM(v9,v1,v0,v16)
	addi	r4,r4,32
err3;	stvx	v8,0,r3
err3;	stvx	v9,r3,r9
	addi	r3,r3,32

6:	bf	cr7*4+1,7f
err3;	lvx	v3,0,r4
	VPERM(v8,v0,v3,v16)
err3;	lvx	v2,r4,r9
	VPERM(v9,v3,v2,v16)
err3;	lvx	v1,r4,r10
	VPERM(v10,v2,v1,v16)
err3;	lvx	v0,r4,r11
	VPERM(v11,v1,v0,v16)
	addi	r4,r4,64
err3;	stvx	v8,0,r3
err3;	stvx	v9,r3,r9
err3;	stvx	v10,r3,r10
err3;	stvx	v11,r3,r11
	addi	r3,r3,64

7:	sub	r5,r5,r6
	srdi	r6,r5,7

	std	r14,STK_REG(R14)(r1)
	std	r15,STK_REG(R15)(r1)
	std	r16,STK_REG(R16)(r1)

	li	r12,64
	li	r14,80
	li	r15,96
	li	r16,112

	mtctr	r6

	/*
	 * Now do cacheline sized loads and stores. By this stage the
	 * cacheline stores are also cacheline aligned.
	 */
	.align	5
8:
err4;	lvx	v7,0,r4
	VPERM(v8,v0,v7,v16)
err4;	lvx	v6,r4,r9
	VPERM(v9,v7,v6,v16)
err4;	lvx	v5,r4,r10
	VPERM(v10,v6,v5,v16)
err4;	lvx	v4,r4,r11
	VPERM(v11,v5,v4,v16)
err4;	lvx	v3,r4,r12
	VPERM(v12,v4,v3,v16)
err4;	lvx	v2,r4,r14
	VPERM(v13,v3,v2,v16)
err4;	lvx	v1,r4,r15
	VPERM(v14,v2,v1,v16)
err4;	lvx	v0,r4,r16
	VPERM(v15,v1,v0,v16)
	addi	r4,r4,128
err4;	stvx	v8,0,r3
err4;	stvx	v9,r3,r9
err4;	stvx	v10,r3,r10
err4;	stvx	v11,r3,r11
err4;	stvx	v12,r3,r12
err4;	stvx	v13,r3,r14
err4;	stvx	v14,r3,r15
err4;	stvx	v15,r3,r16
	addi	r3,r3,128
	bdnz	8b

	ld	r14,STK_REG(R14)(r1)
	ld	r15,STK_REG(R15)(r1)
	ld	r16,STK_REG(R16)(r1)

	/* Up to 127B to go */
	clrldi	r5,r5,(64-7)
	srdi	r6,r5,4
	mtocrf	0x01,r6

	bf	cr7*4+1,9f
err3;	lvx	v3,0,r4
	VPERM(v8,v0,v3,v16)
err3;	lvx	v2,r4,r9
	VPERM(v9,v3,v2,v16)
err3;	lvx	v1,r4,r10
	VPERM(v10,v2,v1,v16)
err3;	lvx	v0,r4,r11
	VPERM(v11,v1,v0,v16)
	addi	r4,r4,64
err3;	stvx	v8,0,r3
err3;	stvx	v9,r3,r9
err3;	stvx	v10,r3,r10
err3;	stvx	v11,r3,r11
	addi	r3,r3,64

9:	bf	cr7*4+2,10f
err3;	lvx	v1,0,r4
	VPERM(v8,v0,v1,v16)
err3;	lvx	v0,r4,r9
	VPERM(v9,v1,v0,v16)
	addi	r4,r4,32
err3;	stvx	v8,0,r3
err3;	stvx	v9,r3,r9
	addi	r3,r3,32

10:	bf	cr7*4+3,11f
err3;	lvx	v1,0,r4
	VPERM(v8,v0,v1,v16)
	addi	r4,r4,16
err3;	stvx	v8,0,r3
	addi	r3,r3,16

	/* Up to 15B to go */
11:	clrldi	r5,r5,(64-4)
	addi	r4,r4,-16	/* Unwind the +16 load offset */
	mtocrf	0x01,r5
	bf	cr7*4+0,12f
err3;	lwz	r0,0(r4)	/* Less chance of a reject with word ops */
err3;	lwz	r6,4(r4)
	addi	r4,r4,8
err3;	stw	r0,0(r3)
err3;	stw	r6,4(r3)
	addi	r3,r3,8

12:	bf	cr7*4+1,13f
err3;	lwz	r0,0(r4)
	addi	r4,r4,4
err3;	stw	r0,0(r3)
	addi	r3,r3,4

13:	bf	cr7*4+2,14f
err3;	lhz	r0,0(r4)
	addi	r4,r4,2
err3;	sth	r0,0(r3)
	addi	r3,r3,2

14:	bf	cr7*4+3,15f
err3;	lbz	r0,0(r4)
err3;	stb	r0,0(r3)

15:	addi	r1,r1,STACKFRAMESIZE
	b	exit_vmx_usercopy	/* tail call optimise */
#endif /* CONFIG_ALTIVEC */
Commit	Line	Data
a66086b8 AB	1	/*
	2	* This program is free software; you can redistribute it and/or modify
	3	* it under the terms of the GNU General Public License as published by
	4	* the Free Software Foundation; either version 2 of the License, or
	5	* (at your option) any later version.
	6	*
	7	* This program is distributed in the hope that it will be useful,
	8	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	9	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	10	* GNU General Public License for more details.
	11	*
	12	* You should have received a copy of the GNU General Public License
	13	* along with this program; if not, write to the Free Software
	14	* Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
	15	*
	16	* Copyright (C) IBM Corporation, 2011
	17	*
	18	* Author: Anton Blanchard <[email protected]>
	19	*/
	20	#include <asm/ppc_asm.h>
	21
32ee1e18 AB	22	#ifdef __BIG_ENDIAN__
	23	#define LVS(VRT,RA,RB) lvsl VRT,RA,RB
	24	#define VPERM(VRT,VRA,VRB,VRC) vperm VRT,VRA,VRB,VRC
	25	#else
	26	#define LVS(VRT,RA,RB) lvsr VRT,RA,RB
	27	#define VPERM(VRT,VRA,VRB,VRC) vperm VRT,VRB,VRA,VRC
	28	#endif
	29
a66086b8 AB	30	.macro err1
a66086b8 AB	31	100:
24bfa6a9	32	EX_TABLE(100b,.Ldo_err1)
a66086b8 AB	33	.endm
	34
	35	.macro err2
	36	200:
24bfa6a9	37	EX_TABLE(200b,.Ldo_err2)
a66086b8 AB	38	.endm
	39
	40	#ifdef CONFIG_ALTIVEC
	41	.macro err3
	42	300:
24bfa6a9	43	EX_TABLE(300b,.Ldo_err3)
a66086b8 AB	44	.endm
	45
	46	.macro err4
	47	400:
24bfa6a9	48	EX_TABLE(400b,.Ldo_err4)
a66086b8 AB	49	.endm
	50
	51
	52	.Ldo_err4:
c75df6f9 MN	53	ld r16,STK_REG(R16)(r1)
	54	ld r15,STK_REG(R15)(r1)
	55	ld r14,STK_REG(R14)(r1)
a66086b8	56	.Ldo_err3:
b1576fec	57	bl exit_vmx_usercopy
a66086b8 AB	58	ld r0,STACKFRAMESIZE+16(r1)
	59	mtlr r0
	60	b .Lexit
	61	#endif /* CONFIG_ALTIVEC */
	62
	63	.Ldo_err2:
c75df6f9 MN	64	ld r22,STK_REG(R22)(r1)
	65	ld r21,STK_REG(R21)(r1)
	66	ld r20,STK_REG(R20)(r1)
	67	ld r19,STK_REG(R19)(r1)
	68	ld r18,STK_REG(R18)(r1)
	69	ld r17,STK_REG(R17)(r1)
	70	ld r16,STK_REG(R16)(r1)
	71	ld r15,STK_REG(R15)(r1)
	72	ld r14,STK_REG(R14)(r1)
a66086b8 AB	73	.Lexit:
	74	addi r1,r1,STACKFRAMESIZE
	75	.Ldo_err1:
752a6422 UW	76	ld r3,-STACKFRAMESIZE+STK_REG(R31)(r1)
	77	ld r4,-STACKFRAMESIZE+STK_REG(R30)(r1)
	78	ld r5,-STACKFRAMESIZE+STK_REG(R29)(r1)
a66086b8 AB	79	b __copy_tofrom_user_base
	80
	81
	82	_GLOBAL(__copy_tofrom_user_power7)
	83	#ifdef CONFIG_ALTIVEC
	84	cmpldi r5,16
a3f952df	85	cmpldi cr1,r5,3328
a66086b8	86
752a6422 UW	87	std r3,-STACKFRAMESIZE+STK_REG(R31)(r1)
	88	std r4,-STACKFRAMESIZE+STK_REG(R30)(r1)
	89	std r5,-STACKFRAMESIZE+STK_REG(R29)(r1)
a66086b8 AB	90
a66086b8 AB	91	blt .Lshort_copy
a3f952df	92	bge cr1,.Lvmx_copy
a66086b8 AB	93	#else
	94	cmpldi r5,16
	95
752a6422 UW	96	std r3,-STACKFRAMESIZE+STK_REG(R31)(r1)
	97	std r4,-STACKFRAMESIZE+STK_REG(R30)(r1)
	98	std r5,-STACKFRAMESIZE+STK_REG(R29)(r1)
a66086b8 AB	99
	100	blt .Lshort_copy
	101	#endif
	102
	103	.Lnonvmx_copy:
	104	/* Get the source 8B aligned */
	105	neg r6,r4
	106	mtocrf 0x01,r6
	107	clrldi r6,r6,(64-3)
	108
	109	bf cr7*4+3,1f
	110	err1; lbz r0,0(r4)
	111	addi r4,r4,1
	112	err1; stb r0,0(r3)
	113	addi r3,r3,1
	114
	115	1: bf cr7*4+2,2f
	116	err1; lhz r0,0(r4)
	117	addi r4,r4,2
	118	err1; sth r0,0(r3)
	119	addi r3,r3,2
	120
	121	2: bf cr7*4+1,3f
	122	err1; lwz r0,0(r4)
	123	addi r4,r4,4
	124	err1; stw r0,0(r3)
	125	addi r3,r3,4
	126
	127	3: sub r5,r5,r6
	128	cmpldi r5,128
	129	blt 5f
	130
	131	mflr r0
	132	stdu r1,-STACKFRAMESIZE(r1)
c75df6f9 MN	133	std r14,STK_REG(R14)(r1)
	134	std r15,STK_REG(R15)(r1)
	135	std r16,STK_REG(R16)(r1)
	136	std r17,STK_REG(R17)(r1)
	137	std r18,STK_REG(R18)(r1)
	138	std r19,STK_REG(R19)(r1)
	139	std r20,STK_REG(R20)(r1)
	140	std r21,STK_REG(R21)(r1)
	141	std r22,STK_REG(R22)(r1)
a66086b8 AB	142	std r0,STACKFRAMESIZE+16(r1)
	143
	144	srdi r6,r5,7
	145	mtctr r6
	146
	147	/* Now do cacheline (128B) sized loads and stores. */
	148	.align 5
	149	4:
	150	err2; ld r0,0(r4)
	151	err2; ld r6,8(r4)
	152	err2; ld r7,16(r4)
	153	err2; ld r8,24(r4)
	154	err2; ld r9,32(r4)
	155	err2; ld r10,40(r4)
	156	err2; ld r11,48(r4)
	157	err2; ld r12,56(r4)
	158	err2; ld r14,64(r4)
	159	err2; ld r15,72(r4)
	160	err2; ld r16,80(r4)
	161	err2; ld r17,88(r4)
	162	err2; ld r18,96(r4)
	163	err2; ld r19,104(r4)
	164	err2; ld r20,112(r4)
	165	err2; ld r21,120(r4)
	166	addi r4,r4,128
	167	err2; std r0,0(r3)
	168	err2; std r6,8(r3)
	169	err2; std r7,16(r3)
	170	err2; std r8,24(r3)
	171	err2; std r9,32(r3)
	172	err2; std r10,40(r3)
	173	err2; std r11,48(r3)
	174	err2; std r12,56(r3)
	175	err2; std r14,64(r3)
	176	err2; std r15,72(r3)
	177	err2; std r16,80(r3)
	178	err2; std r17,88(r3)
	179	err2; std r18,96(r3)
	180	err2; std r19,104(r3)
	181	err2; std r20,112(r3)
	182	err2; std r21,120(r3)
	183	addi r3,r3,128
	184	bdnz 4b
	185
	186	clrldi r5,r5,(64-7)
	187
c75df6f9 MN	188	ld r14,STK_REG(R14)(r1)
	189	ld r15,STK_REG(R15)(r1)
	190	ld r16,STK_REG(R16)(r1)
	191	ld r17,STK_REG(R17)(r1)
	192	ld r18,STK_REG(R18)(r1)
	193	ld r19,STK_REG(R19)(r1)
	194	ld r20,STK_REG(R20)(r1)
	195	ld r21,STK_REG(R21)(r1)
	196	ld r22,STK_REG(R22)(r1)
a66086b8 AB	197	addi r1,r1,STACKFRAMESIZE
	198
	199	/* Up to 127B to go */
	200	5: srdi r6,r5,4
	201	mtocrf 0x01,r6
	202
	203	6: bf cr7*4+1,7f
	204	err1; ld r0,0(r4)
	205	err1; ld r6,8(r4)
	206	err1; ld r7,16(r4)
	207	err1; ld r8,24(r4)
	208	err1; ld r9,32(r4)
	209	err1; ld r10,40(r4)
	210	err1; ld r11,48(r4)
	211	err1; ld r12,56(r4)
	212	addi r4,r4,64
	213	err1; std r0,0(r3)
	214	err1; std r6,8(r3)
	215	err1; std r7,16(r3)
	216	err1; std r8,24(r3)
	217	err1; std r9,32(r3)
	218	err1; std r10,40(r3)
	219	err1; std r11,48(r3)
	220	err1; std r12,56(r3)
	221	addi r3,r3,64
	222
	223	/* Up to 63B to go */
	224	7: bf cr7*4+2,8f
	225	err1; ld r0,0(r4)
	226	err1; ld r6,8(r4)
	227	err1; ld r7,16(r4)
	228	err1; ld r8,24(r4)
	229	addi r4,r4,32
	230	err1; std r0,0(r3)
	231	err1; std r6,8(r3)
	232	err1; std r7,16(r3)
	233	err1; std r8,24(r3)
	234	addi r3,r3,32
	235
	236	/* Up to 31B to go */
	237	8: bf cr7*4+3,9f
	238	err1; ld r0,0(r4)
	239	err1; ld r6,8(r4)
	240	addi r4,r4,16
	241	err1; std r0,0(r3)
	242	err1; std r6,8(r3)
	243	addi r3,r3,16
	244
	245	9: clrldi r5,r5,(64-4)
	246
	247	/* Up to 15B to go */
	248	.Lshort_copy:
	249	mtocrf 0x01,r5
	250	bf cr7*4+0,12f
	251	err1; lwz r0,0(r4) /* Less chance of a reject with word ops */
	252	err1; lwz r6,4(r4)
	253	addi r4,r4,8
	254	err1; stw r0,0(r3)
	255	err1; stw r6,4(r3)
	256	addi r3,r3,8
	257
	258	12: bf cr7*4+1,13f
	259	err1; lwz r0,0(r4)
	260	addi r4,r4,4
261	err1; stw r0,0(r3)
262	addi r3,r3,4
263
264	13: bf cr7*4+2,14f
265	err1; lhz r0,0(r4)
266	addi r4,r4,2
267	err1; sth r0,0(r3)
268	addi r3,r3,2
269
270	14: bf cr7*4+3,15f
271	err1; lbz r0,0(r4)
272	err1; stb r0,0(r3)
273
274	15: li r3,0
275	blr
276
277	.Lunwind_stack_nonvmx_copy:
278	addi r1,r1,STACKFRAMESIZE
279	b .Lnonvmx_copy
280
281	#ifdef CONFIG_ALTIVEC
282	.Lvmx_copy:
283	mflr r0
284	std r0,16(r1)
285	stdu r1,-STACKFRAMESIZE(r1)
b1576fec	286	bl enter_vmx_usercopy
2fae7cdb	287	cmpwi cr1,r3,0
a66086b8	288	ld r0,STACKFRAMESIZE+16(r1)
752a6422 UW	289	ld r3,STK_REG(R31)(r1)
	290	ld r4,STK_REG(R30)(r1)
	291	ld r5,STK_REG(R29)(r1)
a66086b8 AB	292	mtlr r0
a66086b8 AB	293
a9514dc6 AB	294	/*
	295	* We prefetch both the source and destination using enhanced touch
	296	* instructions. We use a stream ID of 0 for the load side and
	297	* 1 for the store side.
	298	*/
	299	clrrdi r6,r4,7
	300	clrrdi r9,r3,7
	301	ori r9,r9,1 /* stream=1 */
	302
	303	srdi r7,r5,7 /* length in cachelines, capped at 0x3FF */
	304	cmpldi r7,0x3FF
	305	ble 1f
	306	li r7,0x3FF
	307	1: lis r0,0x0E00 /* depth=7 */
	308	sldi r7,r7,7
	309	or r7,r7,r0
	310	ori r10,r7,1 /* stream=1 */
	311
	312	lis r8,0x8000 /* GO=1 */
	313	clrldi r8,r8,32
	314
280a5ba2	315	/* setup read stream 0 */
8a583c0a AS	316	dcbt 0,r6,0b01000 /* addr from */
8a583c0a AS	317	dcbt 0,r7,0b01010 /* length and depth from */
280a5ba2	318	/* setup write stream 1 */
8a583c0a AS	319	dcbtst 0,r9,0b01000 /* addr to */
8a583c0a AS	320	dcbtst 0,r10,0b01010 /* length and depth to */
a9514dc6	321	eieio
8a583c0a	322	dcbt 0,r8,0b01010 /* all streams GO */
a9514dc6	323
2fae7cdb	324	beq cr1,.Lunwind_stack_nonvmx_copy
a66086b8 AB	325
	326	/*
	327	* If source and destination are not relatively aligned we use a
	328	* slower permute loop.
	329	*/
	330	xor r6,r4,r3
	331	rldicl. r6,r6,0,(64-4)
	332	bne .Lvmx_unaligned_copy
	333
	334	/* Get the destination 16B aligned */
	335	neg r6,r3
	336	mtocrf 0x01,r6
	337	clrldi r6,r6,(64-4)
	338
	339	bf cr7*4+3,1f
	340	err3; lbz r0,0(r4)
	341	addi r4,r4,1
	342	err3; stb r0,0(r3)
	343	addi r3,r3,1
	344
	345	1: bf cr7*4+2,2f
	346	err3; lhz r0,0(r4)
	347	addi r4,r4,2
	348	err3; sth r0,0(r3)
	349	addi r3,r3,2
	350
	351	2: bf cr7*4+1,3f
	352	err3; lwz r0,0(r4)
	353	addi r4,r4,4
	354	err3; stw r0,0(r3)
	355	addi r3,r3,4
	356
	357	3: bf cr7*4+0,4f
	358	err3; ld r0,0(r4)
	359	addi r4,r4,8
	360	err3; std r0,0(r3)
	361	addi r3,r3,8
	362
	363	4: sub r5,r5,r6
	364
	365	/* Get the desination 128B aligned */
	366	neg r6,r3
	367	srdi r7,r6,4
	368	mtocrf 0x01,r7
	369	clrldi r6,r6,(64-7)
	370
	371	li r9,16
	372	li r10,32
	373	li r11,48
	374
	375	bf cr7*4+3,5f
8a583c0a	376	err3; lvx v1,0,r4
a66086b8	377	addi r4,r4,16
8a583c0a	378	err3; stvx v1,0,r3
a66086b8 AB	379	addi r3,r3,16
	380
	381	5: bf cr7*4+2,6f
8a583c0a	382	err3; lvx v1,0,r4
c2ce6f9f	383	err3; lvx v0,r4,r9
a66086b8	384	addi r4,r4,32
8a583c0a	385	err3; stvx v1,0,r3
c2ce6f9f	386	err3; stvx v0,r3,r9
a66086b8 AB	387	addi r3,r3,32
	388
	389	6: bf cr7*4+1,7f
8a583c0a	390	err3; lvx v3,0,r4
c2ce6f9f AB	391	err3; lvx v2,r4,r9
	392	err3; lvx v1,r4,r10
	393	err3; lvx v0,r4,r11
a66086b8	394	addi r4,r4,64
8a583c0a	395	err3; stvx v3,0,r3
c2ce6f9f AB	396	err3; stvx v2,r3,r9
	397	err3; stvx v1,r3,r10
	398	err3; stvx v0,r3,r11
a66086b8 AB	399	addi r3,r3,64
	400
	401	7: sub r5,r5,r6
	402	srdi r6,r5,7
	403
c75df6f9 MN	404	std r14,STK_REG(R14)(r1)
	405	std r15,STK_REG(R15)(r1)
	406	std r16,STK_REG(R16)(r1)
a66086b8 AB	407
	408	li r12,64
	409	li r14,80
	410	li r15,96
	411	li r16,112
	412
	413	mtctr r6
	414
	415	/*
	416	* Now do cacheline sized loads and stores. By this stage the
	417	* cacheline stores are also cacheline aligned.
	418	*/
	419	.align 5
	420	8:
8a583c0a	421	err4; lvx v7,0,r4
c2ce6f9f AB	422	err4; lvx v6,r4,r9
	423	err4; lvx v5,r4,r10
	424	err4; lvx v4,r4,r11
	425	err4; lvx v3,r4,r12
	426	err4; lvx v2,r4,r14
	427	err4; lvx v1,r4,r15
	428	err4; lvx v0,r4,r16
a66086b8	429	addi r4,r4,128
8a583c0a	430	err4; stvx v7,0,r3
c2ce6f9f AB	431	err4; stvx v6,r3,r9
	432	err4; stvx v5,r3,r10
	433	err4; stvx v4,r3,r11
	434	err4; stvx v3,r3,r12
	435	err4; stvx v2,r3,r14
	436	err4; stvx v1,r3,r15
	437	err4; stvx v0,r3,r16
a66086b8 AB	438	addi r3,r3,128
	439	bdnz 8b
	440
c75df6f9 MN	441	ld r14,STK_REG(R14)(r1)
	442	ld r15,STK_REG(R15)(r1)
	443	ld r16,STK_REG(R16)(r1)
a66086b8 AB	444
	445	/* Up to 127B to go */
	446	clrldi r5,r5,(64-7)
	447	srdi r6,r5,4
	448	mtocrf 0x01,r6
	449
	450	bf cr7*4+1,9f
8a583c0a	451	err3; lvx v3,0,r4
c2ce6f9f AB	452	err3; lvx v2,r4,r9
	453	err3; lvx v1,r4,r10
	454	err3; lvx v0,r4,r11
a66086b8	455	addi r4,r4,64
8a583c0a	456	err3; stvx v3,0,r3
c2ce6f9f AB	457	err3; stvx v2,r3,r9
	458	err3; stvx v1,r3,r10
	459	err3; stvx v0,r3,r11
a66086b8 AB	460	addi r3,r3,64
	461
	462	9: bf cr7*4+2,10f
8a583c0a	463	err3; lvx v1,0,r4
c2ce6f9f	464	err3; lvx v0,r4,r9
a66086b8	465	addi r4,r4,32
8a583c0a	466	err3; stvx v1,0,r3
c2ce6f9f	467	err3; stvx v0,r3,r9
a66086b8 AB	468	addi r3,r3,32
	469
	470	10: bf cr7*4+3,11f
8a583c0a	471	err3; lvx v1,0,r4
a66086b8	472	addi r4,r4,16
8a583c0a	473	err3; stvx v1,0,r3
a66086b8 AB	474	addi r3,r3,16
	475
	476	/* Up to 15B to go */
	477	11: clrldi r5,r5,(64-4)
	478	mtocrf 0x01,r5
	479	bf cr7*4+0,12f
	480	err3; ld r0,0(r4)
	481	addi r4,r4,8
	482	err3; std r0,0(r3)
	483	addi r3,r3,8
	484
	485	12: bf cr7*4+1,13f
	486	err3; lwz r0,0(r4)
	487	addi r4,r4,4
	488	err3; stw r0,0(r3)
	489	addi r3,r3,4
	490
	491	13: bf cr7*4+2,14f
	492	err3; lhz r0,0(r4)
	493	addi r4,r4,2
	494	err3; sth r0,0(r3)
	495	addi r3,r3,2
	496
	497	14: bf cr7*4+3,15f
	498	err3; lbz r0,0(r4)
	499	err3; stb r0,0(r3)
	500
	501	15: addi r1,r1,STACKFRAMESIZE
b1576fec	502	b exit_vmx_usercopy /* tail call optimise */
a66086b8 AB	503
	504	.Lvmx_unaligned_copy:
	505	/* Get the destination 16B aligned */
	506	neg r6,r3
	507	mtocrf 0x01,r6
	508	clrldi r6,r6,(64-4)
	509
	510	bf cr7*4+3,1f
	511	err3; lbz r0,0(r4)
	512	addi r4,r4,1
	513	err3; stb r0,0(r3)
	514	addi r3,r3,1
	515
	516	1: bf cr7*4+2,2f
	517	err3; lhz r0,0(r4)
	518	addi r4,r4,2
	519	err3; sth r0,0(r3)
	520	addi r3,r3,2
	521
	522	2: bf cr7*4+1,3f
	523	err3; lwz r0,0(r4)
	524	addi r4,r4,4
	525	err3; stw r0,0(r3)
	526	addi r3,r3,4
	527
	528	3: bf cr7*4+0,4f
	529	err3; lwz r0,0(r4) /* Less chance of a reject with word ops */
	530	err3; lwz r7,4(r4)
	531	addi r4,r4,8
	532	err3; stw r0,0(r3)
	533	err3; stw r7,4(r3)
	534	addi r3,r3,8
	535
	536	4: sub r5,r5,r6
	537
	538	/* Get the desination 128B aligned */
	539	neg r6,r3
	540	srdi r7,r6,4
	541	mtocrf 0x01,r7
	542	clrldi r6,r6,(64-7)
	543
	544	li r9,16
	545	li r10,32
	546	li r11,48
	547
c2ce6f9f AB	548	LVS(v16,0,r4) /* Setup permute control vector */
c2ce6f9f AB	549	err3; lvx v0,0,r4
a66086b8 AB	550	addi r4,r4,16
	551
	552	bf cr7*4+3,5f
8a583c0a	553	err3; lvx v1,0,r4
c2ce6f9f	554	VPERM(v8,v0,v1,v16)
a66086b8	555	addi r4,r4,16
8a583c0a	556	err3; stvx v8,0,r3
a66086b8	557	addi r3,r3,16
c2ce6f9f	558	vor v0,v1,v1
a66086b8 AB	559
a66086b8 AB	560	5: bf cr7*4+2,6f
8a583c0a	561	err3; lvx v1,0,r4
c2ce6f9f AB	562	VPERM(v8,v0,v1,v16)
	563	err3; lvx v0,r4,r9
	564	VPERM(v9,v1,v0,v16)
a66086b8	565	addi r4,r4,32
8a583c0a	566	err3; stvx v8,0,r3
c2ce6f9f	567	err3; stvx v9,r3,r9
a66086b8 AB	568	addi r3,r3,32
	569
	570	6: bf cr7*4+1,7f
8a583c0a	571	err3; lvx v3,0,r4
c2ce6f9f AB	572	VPERM(v8,v0,v3,v16)
	573	err3; lvx v2,r4,r9
	574	VPERM(v9,v3,v2,v16)
	575	err3; lvx v1,r4,r10
	576	VPERM(v10,v2,v1,v16)
	577	err3; lvx v0,r4,r11
	578	VPERM(v11,v1,v0,v16)
a66086b8	579	addi r4,r4,64
8a583c0a	580	err3; stvx v8,0,r3
c2ce6f9f AB	581	err3; stvx v9,r3,r9
	582	err3; stvx v10,r3,r10
	583	err3; stvx v11,r3,r11
a66086b8 AB	584	addi r3,r3,64
	585
	586	7: sub r5,r5,r6
	587	srdi r6,r5,7
	588
c75df6f9 MN	589	std r14,STK_REG(R14)(r1)
	590	std r15,STK_REG(R15)(r1)
	591	std r16,STK_REG(R16)(r1)
a66086b8 AB	592
	593	li r12,64
	594	li r14,80
	595	li r15,96
	596	li r16,112
	597
	598	mtctr r6
	599
	600	/*
	601	* Now do cacheline sized loads and stores. By this stage the
	602	* cacheline stores are also cacheline aligned.
	603	*/
	604	.align 5
	605	8:
8a583c0a	606	err4; lvx v7,0,r4
c2ce6f9f AB	607	VPERM(v8,v0,v7,v16)
	608	err4; lvx v6,r4,r9
	609	VPERM(v9,v7,v6,v16)
	610	err4; lvx v5,r4,r10
	611	VPERM(v10,v6,v5,v16)
	612	err4; lvx v4,r4,r11
	613	VPERM(v11,v5,v4,v16)
	614	err4; lvx v3,r4,r12
	615	VPERM(v12,v4,v3,v16)
	616	err4; lvx v2,r4,r14
	617	VPERM(v13,v3,v2,v16)
	618	err4; lvx v1,r4,r15
	619	VPERM(v14,v2,v1,v16)
	620	err4; lvx v0,r4,r16
	621	VPERM(v15,v1,v0,v16)
a66086b8	622	addi r4,r4,128
8a583c0a	623	err4; stvx v8,0,r3
c2ce6f9f AB	624	err4; stvx v9,r3,r9
	625	err4; stvx v10,r3,r10
	626	err4; stvx v11,r3,r11
	627	err4; stvx v12,r3,r12
	628	err4; stvx v13,r3,r14
	629	err4; stvx v14,r3,r15
	630	err4; stvx v15,r3,r16
a66086b8 AB	631	addi r3,r3,128
	632	bdnz 8b
	633
c75df6f9 MN	634	ld r14,STK_REG(R14)(r1)
	635	ld r15,STK_REG(R15)(r1)
	636	ld r16,STK_REG(R16)(r1)
a66086b8 AB	637
	638	/* Up to 127B to go */
	639	clrldi r5,r5,(64-7)
	640	srdi r6,r5,4
	641	mtocrf 0x01,r6
	642
	643	bf cr7*4+1,9f
8a583c0a	644	err3; lvx v3,0,r4
c2ce6f9f AB	645	VPERM(v8,v0,v3,v16)
	646	err3; lvx v2,r4,r9
	647	VPERM(v9,v3,v2,v16)
	648	err3; lvx v1,r4,r10
	649	VPERM(v10,v2,v1,v16)
	650	err3; lvx v0,r4,r11
	651	VPERM(v11,v1,v0,v16)
a66086b8	652	addi r4,r4,64
8a583c0a	653	err3; stvx v8,0,r3
c2ce6f9f AB	654	err3; stvx v9,r3,r9
	655	err3; stvx v10,r3,r10
	656	err3; stvx v11,r3,r11
a66086b8 AB	657	addi r3,r3,64
	658
	659	9: bf cr7*4+2,10f
8a583c0a	660	err3; lvx v1,0,r4
c2ce6f9f AB	661	VPERM(v8,v0,v1,v16)
	662	err3; lvx v0,r4,r9
	663	VPERM(v9,v1,v0,v16)
a66086b8	664	addi r4,r4,32
8a583c0a	665	err3; stvx v8,0,r3
c2ce6f9f	666	err3; stvx v9,r3,r9
a66086b8 AB	667	addi r3,r3,32
	668
	669	10: bf cr7*4+3,11f
8a583c0a	670	err3; lvx v1,0,r4
c2ce6f9f	671	VPERM(v8,v0,v1,v16)
a66086b8	672	addi r4,r4,16
8a583c0a	673	err3; stvx v8,0,r3
a66086b8 AB	674	addi r3,r3,16
	675
	676	/* Up to 15B to go */
	677	11: clrldi r5,r5,(64-4)
	678	addi r4,r4,-16 /* Unwind the +16 load offset */
	679	mtocrf 0x01,r5
	680	bf cr7*4+0,12f
	681	err3; lwz r0,0(r4) /* Less chance of a reject with word ops */
	682	err3; lwz r6,4(r4)
	683	addi r4,r4,8
	684	err3; stw r0,0(r3)
	685	err3; stw r6,4(r3)
	686	addi r3,r3,8
	687
	688	12: bf cr7*4+1,13f
	689	err3; lwz r0,0(r4)
	690	addi r4,r4,4
	691	err3; stw r0,0(r3)
	692	addi r3,r3,4
	693
	694	13: bf cr7*4+2,14f
	695	err3; lhz r0,0(r4)
	696	addi r4,r4,2
	697	err3; sth r0,0(r3)
	698	addi r3,r3,2
	699
	700	14: bf cr7*4+3,15f
	701	err3; lbz r0,0(r4)
	702	err3; stb r0,0(r3)
	703
	704	15: addi r1,r1,STACKFRAMESIZE
b1576fec	705	b exit_vmx_usercopy /* tail call optimise */
c2522dcd	706	#endif /* CONFIG_ALTIVEC */