[linux.git] / arch / x86 / lib / atomic64_cx8_32.S

/*
 * atomic64_t for 586+
 *
 * Copyright © 2010  Luca Barbieri
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 */

#include <linux/linkage.h>
#include <asm/alternative-asm.h>
#include <asm/dwarf2.h>

.macro SAVE reg
	pushl_cfi %\reg
	CFI_REL_OFFSET \reg, 0
.endm

.macro RESTORE reg
	popl_cfi %\reg
	CFI_RESTORE \reg
.endm

.macro read64 reg
	movl %ebx, %eax
	movl %ecx, %edx
/* we need LOCK_PREFIX since otherwise cmpxchg8b always does the write */
	LOCK_PREFIX
	cmpxchg8b (\reg)
.endm

ENTRY(atomic64_read_cx8)
	CFI_STARTPROC

	read64 %ecx
	ret
	CFI_ENDPROC
ENDPROC(atomic64_read_cx8)

ENTRY(atomic64_set_cx8)
	CFI_STARTPROC

1:
/* we don't need LOCK_PREFIX since aligned 64-bit writes
 * are atomic on 586 and newer */
	cmpxchg8b (%esi)
	jne 1b

	ret
	CFI_ENDPROC
ENDPROC(atomic64_set_cx8)

ENTRY(atomic64_xchg_cx8)
	CFI_STARTPROC

1:
	LOCK_PREFIX
	cmpxchg8b (%esi)
	jne 1b

	ret
	CFI_ENDPROC
ENDPROC(atomic64_xchg_cx8)

.macro addsub_return func ins insc
ENTRY(atomic64_\func\()_return_cx8)
	CFI_STARTPROC
	SAVE ebp
	SAVE ebx
	SAVE esi
	SAVE edi

	movl %eax, %esi
	movl %edx, %edi
	movl %ecx, %ebp

	read64 %ecx
1:
	movl %eax, %ebx
	movl %edx, %ecx
	\ins\()l %esi, %ebx
	\insc\()l %edi, %ecx
	LOCK_PREFIX
	cmpxchg8b (%ebp)
	jne 1b

10:
	movl %ebx, %eax
	movl %ecx, %edx
	RESTORE edi
	RESTORE esi
	RESTORE ebx
	RESTORE ebp
	ret
	CFI_ENDPROC
ENDPROC(atomic64_\func\()_return_cx8)
.endm

addsub_return add add adc
addsub_return sub sub sbb

.macro incdec_return func ins insc
ENTRY(atomic64_\func\()_return_cx8)
	CFI_STARTPROC
	SAVE ebx

	read64 %esi
1:
	movl %eax, %ebx
	movl %edx, %ecx
	\ins\()l $1, %ebx
	\insc\()l $0, %ecx
	LOCK_PREFIX
	cmpxchg8b (%esi)
	jne 1b

10:
	movl %ebx, %eax
	movl %ecx, %edx
	RESTORE ebx
	ret
	CFI_ENDPROC
ENDPROC(atomic64_\func\()_return_cx8)
.endm

incdec_return inc add adc
incdec_return dec sub sbb

ENTRY(atomic64_dec_if_positive_cx8)
	CFI_STARTPROC
	SAVE ebx

	read64 %esi
1:
	movl %eax, %ebx
	movl %edx, %ecx
	subl $1, %ebx
	sbb $0, %ecx
	js 2f
	LOCK_PREFIX
	cmpxchg8b (%esi)
	jne 1b

2:
	movl %ebx, %eax
	movl %ecx, %edx
	RESTORE ebx
	ret
	CFI_ENDPROC
ENDPROC(atomic64_dec_if_positive_cx8)

ENTRY(atomic64_add_unless_cx8)
	CFI_STARTPROC
	SAVE ebp
	SAVE ebx
/* these just push these two parameters on the stack */
	SAVE edi
	SAVE ecx

	movl %eax, %ebp
	movl %edx, %edi

	read64 %esi
1:
	cmpl %eax, 0(%esp)
	je 4f
2:
	movl %eax, %ebx
	movl %edx, %ecx
	addl %ebp, %ebx
	adcl %edi, %ecx
	LOCK_PREFIX
	cmpxchg8b (%esi)
	jne 1b

	movl $1, %eax
3:
	addl $8, %esp
	CFI_ADJUST_CFA_OFFSET -8
	RESTORE ebx
	RESTORE ebp
	ret
4:
	cmpl %edx, 4(%esp)
	jne 2b
	xorl %eax, %eax
	jmp 3b
	CFI_ENDPROC
ENDPROC(atomic64_add_unless_cx8)

ENTRY(atomic64_inc_not_zero_cx8)
	CFI_STARTPROC
	SAVE ebx

	read64 %esi
1:
	movl %eax, %ecx
	orl %edx, %ecx
	jz 3f
	movl %eax, %ebx
	xorl %ecx, %ecx
	addl $1, %ebx
	adcl %edx, %ecx
	LOCK_PREFIX
	cmpxchg8b (%esi)
	jne 1b

	movl $1, %eax
3:
	RESTORE ebx
	ret
	CFI_ENDPROC
ENDPROC(atomic64_inc_not_zero_cx8)
Commit	Line	Data
a7e926ab LB	1	/*
	2	* atomic64_t for 586+
	3	*
	4	* Copyright © 2010 Luca Barbieri
	5	*
	6	* This program is free software; you can redistribute it and/or modify
	7	* it under the terms of the GNU General Public License as published by
	8	* the Free Software Foundation; either version 2 of the License, or
	9	* (at your option) any later version.
	10	*/
	11
	12	#include <linux/linkage.h>
	13	#include <asm/alternative-asm.h>
	14	#include <asm/dwarf2.h>
	15
	16	.macro SAVE reg
60cf637a	17	pushl_cfi %\reg
a7e926ab LB	18	CFI_REL_OFFSET \reg, 0
	19	.endm
	20
	21	.macro RESTORE reg
60cf637a	22	popl_cfi %\reg
a7e926ab LB	23	CFI_RESTORE \reg
	24	.endm
	25
	26	.macro read64 reg
	27	movl %ebx, %eax
	28	movl %ecx, %edx
	29	/* we need LOCK_PREFIX since otherwise cmpxchg8b always does the write */
	30	LOCK_PREFIX
	31	cmpxchg8b (\reg)
	32	.endm
	33
	34	ENTRY(atomic64_read_cx8)
	35	CFI_STARTPROC
	36
	37	read64 %ecx
	38	ret
	39	CFI_ENDPROC
	40	ENDPROC(atomic64_read_cx8)
	41
	42	ENTRY(atomic64_set_cx8)
	43	CFI_STARTPROC
	44
	45	1:
	46	/* we don't need LOCK_PREFIX since aligned 64-bit writes
	47	* are atomic on 586 and newer */
	48	cmpxchg8b (%esi)
	49	jne 1b
	50
	51	ret
	52	CFI_ENDPROC
	53	ENDPROC(atomic64_set_cx8)
	54
	55	ENTRY(atomic64_xchg_cx8)
	56	CFI_STARTPROC
	57
a7e926ab LB	58	1:
	59	LOCK_PREFIX
	60	cmpxchg8b (%esi)
	61	jne 1b
	62
	63	ret
	64	CFI_ENDPROC
	65	ENDPROC(atomic64_xchg_cx8)
	66
	67	.macro addsub_return func ins insc
	68	ENTRY(atomic64_\func\()_return_cx8)
	69	CFI_STARTPROC
	70	SAVE ebp
	71	SAVE ebx
	72	SAVE esi
	73	SAVE edi
	74
	75	movl %eax, %esi
	76	movl %edx, %edi
	77	movl %ecx, %ebp
	78
cb8095bb	79	read64 %ecx
a7e926ab LB	80	1:
	81	movl %eax, %ebx
	82	movl %edx, %ecx
	83	\ins\()l %esi, %ebx
	84	\insc\()l %edi, %ecx
	85	LOCK_PREFIX
	86	cmpxchg8b (%ebp)
	87	jne 1b
	88
	89	10:
	90	movl %ebx, %eax
	91	movl %ecx, %edx
	92	RESTORE edi
	93	RESTORE esi
	94	RESTORE ebx
	95	RESTORE ebp
	96	ret
	97	CFI_ENDPROC
	98	ENDPROC(atomic64_\func\()_return_cx8)
	99	.endm
	100
	101	addsub_return add add adc
	102	addsub_return sub sub sbb
	103
	104	.macro incdec_return func ins insc
	105	ENTRY(atomic64_\func\()_return_cx8)
	106	CFI_STARTPROC
	107	SAVE ebx
	108
	109	read64 %esi
	110	1:
	111	movl %eax, %ebx
	112	movl %edx, %ecx
	113	\ins\()l $1, %ebx
	114	\insc\()l $0, %ecx
	115	LOCK_PREFIX
	116	cmpxchg8b (%esi)
	117	jne 1b
	118
	119	10:
	120	movl %ebx, %eax
	121	movl %ecx, %edx
	122	RESTORE ebx
	123	ret
	124	CFI_ENDPROC
	125	ENDPROC(atomic64_\func\()_return_cx8)
	126	.endm
	127
	128	incdec_return inc add adc
	129	incdec_return dec sub sbb
	130
	131	ENTRY(atomic64_dec_if_positive_cx8)
	132	CFI_STARTPROC
	133	SAVE ebx
	134
	135	read64 %esi
	136	1:
	137	movl %eax, %ebx
	138	movl %edx, %ecx
	139	subl $1, %ebx
	140	sbb $0, %ecx
	141	js 2f
	142	LOCK_PREFIX
	143	cmpxchg8b (%esi)
144	jne 1b
145
146	2:
147	movl %ebx, %eax
148	movl %ecx, %edx
149	RESTORE ebx
150	ret
151	CFI_ENDPROC
152	ENDPROC(atomic64_dec_if_positive_cx8)
153
154	ENTRY(atomic64_add_unless_cx8)
155	CFI_STARTPROC
156	SAVE ebp
157	SAVE ebx
158	/* these just push these two parameters on the stack */
159	SAVE edi
cb8095bb	160	SAVE ecx
a7e926ab	161
cb8095bb	162	movl %eax, %ebp
a7e926ab LB	163	movl %edx, %edi
a7e926ab LB	164
cb8095bb	165	read64 %esi
a7e926ab LB	166	1:
	167	cmpl %eax, 0(%esp)
	168	je 4f
	169	2:
	170	movl %eax, %ebx
	171	movl %edx, %ecx
cb8095bb	172	addl %ebp, %ebx
a7e926ab LB	173	adcl %edi, %ecx
a7e926ab LB	174	LOCK_PREFIX
cb8095bb	175	cmpxchg8b (%esi)
a7e926ab LB	176	jne 1b
a7e926ab LB	177
6e6104fe	178	movl $1, %eax
a7e926ab LB	179	3:
	180	addl $8, %esp
	181	CFI_ADJUST_CFA_OFFSET -8
	182	RESTORE ebx
	183	RESTORE ebp
	184	ret
	185	4:
	186	cmpl %edx, 4(%esp)
	187	jne 2b
6e6104fe	188	xorl %eax, %eax
a7e926ab LB	189	jmp 3b
	190	CFI_ENDPROC
	191	ENDPROC(atomic64_add_unless_cx8)
	192
	193	ENTRY(atomic64_inc_not_zero_cx8)
	194	CFI_STARTPROC
	195	SAVE ebx
	196
	197	read64 %esi
	198	1:
cb8095bb JB	199	movl %eax, %ecx
	200	orl %edx, %ecx
	201	jz 3f
a7e926ab	202	movl %eax, %ebx
cb8095bb	203	xorl %ecx, %ecx
a7e926ab	204	addl $1, %ebx
cb8095bb	205	adcl %edx, %ecx
a7e926ab LB	206	LOCK_PREFIX
	207	cmpxchg8b (%esi)
	208	jne 1b
	209
f3e83131	210	movl $1, %eax
a7e926ab LB	211	3:
	212	RESTORE ebx
	213	ret
a7e926ab LB	214	CFI_ENDPROC
a7e926ab LB	215	ENDPROC(atomic64_inc_not_zero_cx8)