From ecaf45ee5ce60afe7cc46e91d82c1b0cbda09387 Mon Sep 17 00:00:00 2001 From: Andi Kleen Date: Tue, 26 Sep 2006 10:52:29 +0200 Subject: [PATCH] i386: Redo semaphore and rwlock assembly helpers - Move them to a pure assembly file. Previously they were in a C file that only consisted of inline assembly. Doing it in pure assembler is much nicer. - Add a frame.i include with FRAME/ENDFRAME macros to easily add frame pointers to assembly functions - Add dwarf2 annotation to them so that the new dwarf2 unwinder doesn't get stuck on them - Random cleanups Includes feedback from Jan Beulich and a UML build fix from Andrew Morton. Cc: jbeulich@novell.com Cc: jdike@addtoit.com Signed-off-by: Andi Kleen --- arch/i386/lib/semaphore.S | 154 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 154 insertions(+) create mode 100644 arch/i386/lib/semaphore.S (limited to 'arch/i386/lib/semaphore.S') diff --git a/arch/i386/lib/semaphore.S b/arch/i386/lib/semaphore.S new file mode 100644 index 000000000000..e16ff1696390 --- /dev/null +++ b/arch/i386/lib/semaphore.S @@ -0,0 +1,154 @@ +/* + * i386 semaphore implementation. + * + * (C) Copyright 1999 Linus Torvalds + * + * Portions Copyright 1999 Red Hat, Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version + * 2 of the License, or (at your option) any later version. + * + * rw semaphores implemented November 1999 by Benjamin LaHaise + */ + +#include +#include +#include +#include +#include +#include + +/* + * The semaphore operations have a special calling sequence that + * allow us to do a simpler in-line version of them. These routines + * need to convert that sequence back into the C sequence when + * there is contention on the semaphore. + * + * %eax contains the semaphore pointer on entry. Save the C-clobbered + * registers (%eax, %edx and %ecx) except %eax whish is either a return + * value or just clobbered.. + */ + .section .sched.text +ENTRY(__down_failed) + CFI_STARTPROC + FRAME + pushl %edx + CFI_ADJUST_CFA_OFFSET 4 + CFI_REL_OFFSET edx,0 + pushl %ecx + CFI_ADJUST_CFA_OFFSET 4 + CFI_REL_OFFSET ecx,0 + call __down + popl %ecx + CFI_ADJUST_CFA_OFFSET -4 + CFI_RESTORE ecx + popl %edx + CFI_ADJUST_CFA_OFFSET -4 + CFI_RESTORE edx + ENDFRAME + ret + CFI_ENDPROC + END(__down_failed) + +ENTRY(__down_failed_interruptible) + CFI_STARTPROC + FRAME + pushl %edx + CFI_ADJUST_CFA_OFFSET 4 + CFI_REL_OFFSET edx,0 + pushl %ecx + CFI_ADJUST_CFA_OFFSET 4 + CFI_REL_OFFSET ecx,0 + call __down_interruptible + popl %ecx + CFI_ADJUST_CFA_OFFSET -4 + CFI_RESTORE ecx + popl %edx + CFI_ADJUST_CFA_OFFSET -4 + CFI_RESTORE edx + ENDFRAME + ret + CFI_ENDPROC + END(__down_failed_interruptible) + +ENTRY(__down_failed_trylock) + CFI_STARTPROC + FRAME + pushl %edx + CFI_ADJUST_CFA_OFFSET 4 + CFI_REL_OFFSET edx,0 + pushl %ecx + CFI_ADJUST_CFA_OFFSET 4 + CFI_REL_OFFSET ecx,0 + call __down_trylock + popl %ecx + CFI_ADJUST_CFA_OFFSET -4 + CFI_RESTORE ecx + popl %edx + CFI_ADJUST_CFA_OFFSET -4 + CFI_RESTORE edx + ENDFRAME + ret + CFI_ENDPROC + END(__down_failed_trylock) + +ENTRY(__up_wakeup) + CFI_STARTPROC + FRAME + pushl %edx + CFI_ADJUST_CFA_OFFSET 4 + CFI_REL_OFFSET edx,0 + pushl %ecx + CFI_ADJUST_CFA_OFFSET 4 + CFI_REL_OFFSET ecx,0 + call __up + popl %ecx + CFI_ADJUST_CFA_OFFSET -4 + CFI_RESTORE ecx + popl %edx + CFI_ADJUST_CFA_OFFSET -4 + CFI_RESTORE edx + ENDFRAME + ret + CFI_ENDPROC + END(__up_wakeup) + +/* + * rw spinlock fallbacks + */ +#ifdef CONFIG_SMP +ENTRY(__write_lock_failed) + CFI_STARTPROC simple + FRAME +2: LOCK_PREFIX + addl $ RW_LOCK_BIAS,(%eax) +1: rep; nop + cmpl $ RW_LOCK_BIAS,(%eax) + jne 1b + LOCK_PREFIX + subl $ RW_LOCK_BIAS,(%eax) + jnz 2b + ENDFRAME + ret + CFI_ENDPROC + END(__write_lock_failed) + +ENTRY(__read_lock_failed) + CFI_STARTPROC + FRAME +2: LOCK_PREFIX + incl (%eax) +1: rep; nop + cmpl $1,(%eax) + js 1b + LOCK_PREFIX + decl (%eax) + js 2b + ENDFRAME + ret + CFI_ENDPROC + END(__read_lock_failed) + +#endif -- cgit From add659bf8aa92f8b3f01a8c0220557c959507fb1 Mon Sep 17 00:00:00 2001 From: Andi Kleen Date: Tue, 26 Sep 2006 10:52:31 +0200 Subject: [PATCH] i386: Remove lock section support in rwsem.h Lock sections don't work the new dwarf2 unwinder This generates slightly smaller code. It adds one more taken jump to the fast path. Also move the trampolines into semaphore.S and add proper CFI annotations. Cc: jbeulich@novell.com Signed-off-by: Andi Kleen --- arch/i386/lib/semaphore.S | 63 +++++++++++++++++++++++++++++++++++++++++++++++ include/asm-i386/rwsem.h | 62 +++++++++------------------------------------- 2 files changed, 75 insertions(+), 50 deletions(-) (limited to 'arch/i386/lib/semaphore.S') diff --git a/arch/i386/lib/semaphore.S b/arch/i386/lib/semaphore.S index e16ff1696390..01f80b5c45d2 100644 --- a/arch/i386/lib/semaphore.S +++ b/arch/i386/lib/semaphore.S @@ -152,3 +152,66 @@ ENTRY(__read_lock_failed) END(__read_lock_failed) #endif + +/* Fix up special calling conventions */ +ENTRY(call_rwsem_down_read_failed) + CFI_STARTPROC + push %ecx + CFI_ADJUST_CFA_OFFSET 4 + CFI_REL_OFFSET ecx,0 + push %edx + CFI_ADJUST_CFA_OFFSET 4 + CFI_REL_OFFSET edx,0 + call rwsem_down_read_failed + pop %edx + CFI_ADJUST_CFA_OFFSET -4 + pop %ecx + CFI_ADJUST_CFA_OFFSET -4 + ret + CFI_ENDPROC + END(call_rwsem_down_read_failed) + +ENTRY(call_rwsem_down_write_failed) + CFI_STARTPROC + push %ecx + CFI_ADJUST_CFA_OFFSET 4 + CFI_REL_OFFSET ecx,0 + calll rwsem_down_write_failed + pop %ecx + CFI_ADJUST_CFA_OFFSET -4 + ret + CFI_ENDPROC + END(call_rwsem_down_write_failed) + +ENTRY(call_rwsem_wake) + CFI_STARTPROC + decw %dx /* do nothing if still outstanding active readers */ + jnz 1f + push %ecx + CFI_ADJUST_CFA_OFFSET 4 + CFI_REL_OFFSET ecx,0 + call rwsem_wake + pop %ecx + CFI_ADJUST_CFA_OFFSET -4 +1: ret + CFI_ENDPROC + END(call_rwsem_wake) + +/* Fix up special calling conventions */ +ENTRY(call_rwsem_downgrade_wake) + CFI_STARTPROC + push %ecx + CFI_ADJUST_CFA_OFFSET 4 + CFI_REL_OFFSET ecx,0 + push %edx + CFI_ADJUST_CFA_OFFSET 4 + CFI_REL_OFFSET edx,0 + call rwsem_downgrade_wake + pop %edx + CFI_ADJUST_CFA_OFFSET -4 + pop %ecx + CFI_ADJUST_CFA_OFFSET -4 + ret + CFI_ENDPROC + END(call_rwsem_downgrade_wake) + diff --git a/include/asm-i386/rwsem.h b/include/asm-i386/rwsem.h index 43113f5608eb..bc598d6388e3 100644 --- a/include/asm-i386/rwsem.h +++ b/include/asm-i386/rwsem.h @@ -99,17 +99,9 @@ static inline void __down_read(struct rw_semaphore *sem) __asm__ __volatile__( "# beginning down_read\n\t" LOCK_PREFIX " incl (%%eax)\n\t" /* adds 0x00000001, returns the old value */ - " js 2f\n\t" /* jump if we weren't granted the lock */ + " jns 1f\n" + " call call_rwsem_down_read_failed\n" "1:\n\t" - LOCK_SECTION_START("") - "2:\n\t" - " pushl %%ecx\n\t" - " pushl %%edx\n\t" - " call rwsem_down_read_failed\n\t" - " popl %%edx\n\t" - " popl %%ecx\n\t" - " jmp 1b\n" - LOCK_SECTION_END "# ending down_read\n\t" : "+m" (sem->count) : "a" (sem) @@ -151,15 +143,9 @@ static inline void __down_write_nested(struct rw_semaphore *sem, int subclass) "# beginning down_write\n\t" LOCK_PREFIX " xadd %%edx,(%%eax)\n\t" /* subtract 0x0000ffff, returns the old value */ " testl %%edx,%%edx\n\t" /* was the count 0 before? */ - " jnz 2f\n\t" /* jump if we weren't granted the lock */ - "1:\n\t" - LOCK_SECTION_START("") - "2:\n\t" - " pushl %%ecx\n\t" - " call rwsem_down_write_failed\n\t" - " popl %%ecx\n\t" - " jmp 1b\n" - LOCK_SECTION_END + " jz 1f\n" + " call call_rwsem_down_write_failed\n" + "1:\n" "# ending down_write" : "+m" (sem->count), "=d" (tmp) : "a" (sem), "1" (tmp) @@ -193,17 +179,9 @@ static inline void __up_read(struct rw_semaphore *sem) __asm__ __volatile__( "# beginning __up_read\n\t" LOCK_PREFIX " xadd %%edx,(%%eax)\n\t" /* subtracts 1, returns the old value */ - " js 2f\n\t" /* jump if the lock is being waited upon */ - "1:\n\t" - LOCK_SECTION_START("") - "2:\n\t" - " decw %%dx\n\t" /* do nothing if still outstanding active readers */ - " jnz 1b\n\t" - " pushl %%ecx\n\t" - " call rwsem_wake\n\t" - " popl %%ecx\n\t" - " jmp 1b\n" - LOCK_SECTION_END + " jns 1f\n\t" + " call call_rwsem_wake\n" + "1:\n" "# ending __up_read\n" : "+m" (sem->count), "=d" (tmp) : "a" (sem), "1" (tmp) @@ -219,17 +197,9 @@ static inline void __up_write(struct rw_semaphore *sem) "# beginning __up_write\n\t" " movl %2,%%edx\n\t" LOCK_PREFIX " xaddl %%edx,(%%eax)\n\t" /* tries to transition 0xffff0001 -> 0x00000000 */ - " jnz 2f\n\t" /* jump if the lock is being waited upon */ + " jz 1f\n" + " call call_rwsem_wake\n" "1:\n\t" - LOCK_SECTION_START("") - "2:\n\t" - " decw %%dx\n\t" /* did the active count reduce to 0? */ - " jnz 1b\n\t" /* jump back if not */ - " pushl %%ecx\n\t" - " call rwsem_wake\n\t" - " popl %%ecx\n\t" - " jmp 1b\n" - LOCK_SECTION_END "# ending __up_write\n" : "+m" (sem->count) : "a" (sem), "i" (-RWSEM_ACTIVE_WRITE_BIAS) @@ -244,17 +214,9 @@ static inline void __downgrade_write(struct rw_semaphore *sem) __asm__ __volatile__( "# beginning __downgrade_write\n\t" LOCK_PREFIX " addl %2,(%%eax)\n\t" /* transitions 0xZZZZ0001 -> 0xYYYY0001 */ - " js 2f\n\t" /* jump if the lock is being waited upon */ + " jns 1f\n\t" + " call call_rwsem_downgrade_wake\n" "1:\n\t" - LOCK_SECTION_START("") - "2:\n\t" - " pushl %%ecx\n\t" - " pushl %%edx\n\t" - " call rwsem_downgrade_wake\n\t" - " popl %%edx\n\t" - " popl %%ecx\n\t" - " jmp 1b\n" - LOCK_SECTION_END "# ending __downgrade_write\n" : "+m" (sem->count) : "a" (sem), "i" (-RWSEM_WAITING_BIAS) -- cgit