aboutsummaryrefslogtreecommitdiff
path: root/arch/s390/lib
diff options
context:
space:
mode:
Diffstat (limited to 'arch/s390/lib')
-rw-r--r--arch/s390/lib/spinlock.c84
-rw-r--r--arch/s390/lib/uaccess.c68
2 files changed, 52 insertions, 100 deletions
diff --git a/arch/s390/lib/spinlock.c b/arch/s390/lib/spinlock.c
index ba427eb6f14c..ffb15bd4c593 100644
--- a/arch/s390/lib/spinlock.c
+++ b/arch/s390/lib/spinlock.c
@@ -17,7 +17,7 @@ int spin_retry = -1;
static int __init spin_retry_init(void)
{
if (spin_retry < 0)
- spin_retry = MACHINE_HAS_CAD ? 10 : 1000;
+ spin_retry = 1000;
return 0;
}
early_initcall(spin_retry_init);
@@ -32,23 +32,17 @@ static int __init spin_retry_setup(char *str)
}
__setup("spin_retry=", spin_retry_setup);
-static inline void _raw_compare_and_delay(unsigned int *lock, unsigned int old)
-{
- asm(".insn rsy,0xeb0000000022,%0,0,%1" : : "d" (old), "Q" (*lock));
-}
-
void arch_spin_lock_wait(arch_spinlock_t *lp)
{
- unsigned int cpu = SPINLOCK_LOCKVAL;
- unsigned int owner;
- int count, first_diag;
+ int cpu = SPINLOCK_LOCKVAL;
+ int owner, count, first_diag;
first_diag = 1;
while (1) {
owner = ACCESS_ONCE(lp->lock);
/* Try to get the lock if it is free. */
if (!owner) {
- if (_raw_compare_and_swap(&lp->lock, 0, cpu))
+ if (__atomic_cmpxchg_bool(&lp->lock, 0, cpu))
return;
continue;
}
@@ -61,8 +55,6 @@ void arch_spin_lock_wait(arch_spinlock_t *lp)
/* Loop for a while on the lock value. */
count = spin_retry;
do {
- if (MACHINE_HAS_CAD)
- _raw_compare_and_delay(&lp->lock, owner);
owner = ACCESS_ONCE(lp->lock);
} while (owner && count-- > 0);
if (!owner)
@@ -82,9 +74,8 @@ EXPORT_SYMBOL(arch_spin_lock_wait);
void arch_spin_lock_wait_flags(arch_spinlock_t *lp, unsigned long flags)
{
- unsigned int cpu = SPINLOCK_LOCKVAL;
- unsigned int owner;
- int count, first_diag;
+ int cpu = SPINLOCK_LOCKVAL;
+ int owner, count, first_diag;
local_irq_restore(flags);
first_diag = 1;
@@ -93,7 +84,7 @@ void arch_spin_lock_wait_flags(arch_spinlock_t *lp, unsigned long flags)
/* Try to get the lock if it is free. */
if (!owner) {
local_irq_disable();
- if (_raw_compare_and_swap(&lp->lock, 0, cpu))
+ if (__atomic_cmpxchg_bool(&lp->lock, 0, cpu))
return;
local_irq_restore(flags);
continue;
@@ -107,8 +98,6 @@ void arch_spin_lock_wait_flags(arch_spinlock_t *lp, unsigned long flags)
/* Loop for a while on the lock value. */
count = spin_retry;
do {
- if (MACHINE_HAS_CAD)
- _raw_compare_and_delay(&lp->lock, owner);
owner = ACCESS_ONCE(lp->lock);
} while (owner && count-- > 0);
if (!owner)
@@ -128,18 +117,16 @@ EXPORT_SYMBOL(arch_spin_lock_wait_flags);
int arch_spin_trylock_retry(arch_spinlock_t *lp)
{
- unsigned int cpu = SPINLOCK_LOCKVAL;
- unsigned int owner;
- int count;
+ int cpu = SPINLOCK_LOCKVAL;
+ int owner, count;
for (count = spin_retry; count > 0; count--) {
owner = READ_ONCE(lp->lock);
/* Try to get the lock if it is free. */
if (!owner) {
- if (_raw_compare_and_swap(&lp->lock, 0, cpu))
+ if (__atomic_cmpxchg_bool(&lp->lock, 0, cpu))
return 1;
- } else if (MACHINE_HAS_CAD)
- _raw_compare_and_delay(&lp->lock, owner);
+ }
}
return 0;
}
@@ -147,8 +134,8 @@ EXPORT_SYMBOL(arch_spin_trylock_retry);
void _raw_read_lock_wait(arch_rwlock_t *rw)
{
- unsigned int owner, old;
int count = spin_retry;
+ int owner, old;
#ifdef CONFIG_HAVE_MARCH_Z196_FEATURES
__RAW_LOCK(&rw->lock, -1, __RAW_OP_ADD);
@@ -162,12 +149,9 @@ void _raw_read_lock_wait(arch_rwlock_t *rw)
}
old = ACCESS_ONCE(rw->lock);
owner = ACCESS_ONCE(rw->owner);
- if ((int) old < 0) {
- if (MACHINE_HAS_CAD)
- _raw_compare_and_delay(&rw->lock, old);
+ if (old < 0)
continue;
- }
- if (_raw_compare_and_swap(&rw->lock, old, old + 1))
+ if (__atomic_cmpxchg_bool(&rw->lock, old, old + 1))
return;
}
}
@@ -175,17 +159,14 @@ EXPORT_SYMBOL(_raw_read_lock_wait);
int _raw_read_trylock_retry(arch_rwlock_t *rw)
{
- unsigned int old;
int count = spin_retry;
+ int old;
while (count-- > 0) {
old = ACCESS_ONCE(rw->lock);
- if ((int) old < 0) {
- if (MACHINE_HAS_CAD)
- _raw_compare_and_delay(&rw->lock, old);
+ if (old < 0)
continue;
- }
- if (_raw_compare_and_swap(&rw->lock, old, old + 1))
+ if (__atomic_cmpxchg_bool(&rw->lock, old, old + 1))
return 1;
}
return 0;
@@ -194,10 +175,10 @@ EXPORT_SYMBOL(_raw_read_trylock_retry);
#ifdef CONFIG_HAVE_MARCH_Z196_FEATURES
-void _raw_write_lock_wait(arch_rwlock_t *rw, unsigned int prev)
+void _raw_write_lock_wait(arch_rwlock_t *rw, int prev)
{
- unsigned int owner, old;
int count = spin_retry;
+ int owner, old;
owner = 0;
while (1) {
@@ -209,14 +190,12 @@ void _raw_write_lock_wait(arch_rwlock_t *rw, unsigned int prev)
old = ACCESS_ONCE(rw->lock);
owner = ACCESS_ONCE(rw->owner);
smp_mb();
- if ((int) old >= 0) {
+ if (old >= 0) {
prev = __RAW_LOCK(&rw->lock, 0x80000000, __RAW_OP_OR);
old = prev;
}
- if ((old & 0x7fffffff) == 0 && (int) prev >= 0)
+ if ((old & 0x7fffffff) == 0 && prev >= 0)
break;
- if (MACHINE_HAS_CAD)
- _raw_compare_and_delay(&rw->lock, old);
}
}
EXPORT_SYMBOL(_raw_write_lock_wait);
@@ -225,8 +204,8 @@ EXPORT_SYMBOL(_raw_write_lock_wait);
void _raw_write_lock_wait(arch_rwlock_t *rw)
{
- unsigned int owner, old, prev;
int count = spin_retry;
+ int owner, old, prev;
prev = 0x80000000;
owner = 0;
@@ -238,15 +217,13 @@ void _raw_write_lock_wait(arch_rwlock_t *rw)
}
old = ACCESS_ONCE(rw->lock);
owner = ACCESS_ONCE(rw->owner);
- if ((int) old >= 0 &&
- _raw_compare_and_swap(&rw->lock, old, old | 0x80000000))
+ if (old >= 0 &&
+ __atomic_cmpxchg_bool(&rw->lock, old, old | 0x80000000))
prev = old;
else
smp_mb();
- if ((old & 0x7fffffff) == 0 && (int) prev >= 0)
+ if ((old & 0x7fffffff) == 0 && prev >= 0)
break;
- if (MACHINE_HAS_CAD)
- _raw_compare_and_delay(&rw->lock, old);
}
}
EXPORT_SYMBOL(_raw_write_lock_wait);
@@ -255,24 +232,21 @@ EXPORT_SYMBOL(_raw_write_lock_wait);
int _raw_write_trylock_retry(arch_rwlock_t *rw)
{
- unsigned int old;
int count = spin_retry;
+ int old;
while (count-- > 0) {
old = ACCESS_ONCE(rw->lock);
- if (old) {
- if (MACHINE_HAS_CAD)
- _raw_compare_and_delay(&rw->lock, old);
+ if (old)
continue;
- }
- if (_raw_compare_and_swap(&rw->lock, 0, 0x80000000))
+ if (__atomic_cmpxchg_bool(&rw->lock, 0, 0x80000000))
return 1;
}
return 0;
}
EXPORT_SYMBOL(_raw_write_trylock_retry);
-void arch_lock_relax(unsigned int cpu)
+void arch_lock_relax(int cpu)
{
if (!cpu)
return;
diff --git a/arch/s390/lib/uaccess.c b/arch/s390/lib/uaccess.c
index f481fcde067b..1e5bb2b86c42 100644
--- a/arch/s390/lib/uaccess.c
+++ b/arch/s390/lib/uaccess.c
@@ -26,7 +26,7 @@ static inline unsigned long copy_from_user_mvcos(void *x, const void __user *ptr
tmp1 = -4096UL;
asm volatile(
"0: .insn ss,0xc80000000000,0(%0,%2),0(%1),0\n"
- "9: jz 7f\n"
+ "6: jz 4f\n"
"1: algr %0,%3\n"
" slgr %1,%3\n"
" slgr %2,%3\n"
@@ -35,23 +35,13 @@ static inline unsigned long copy_from_user_mvcos(void *x, const void __user *ptr
" nr %4,%3\n" /* %4 = (ptr + 4095) & -4096 */
" slgr %4,%1\n"
" clgr %0,%4\n" /* copy crosses next page boundary? */
- " jnh 4f\n"
+ " jnh 5f\n"
"3: .insn ss,0xc80000000000,0(%4,%2),0(%1),0\n"
- "10:slgr %0,%4\n"
- " algr %2,%4\n"
- "4: lghi %4,-1\n"
- " algr %4,%0\n" /* copy remaining size, subtract 1 */
- " bras %3,6f\n" /* memset loop */
- " xc 0(1,%2),0(%2)\n"
- "5: xc 0(256,%2),0(%2)\n"
- " la %2,256(%2)\n"
- "6: aghi %4,-256\n"
- " jnm 5b\n"
- " ex %4,0(%3)\n"
- " j 8f\n"
- "7: slgr %0,%0\n"
- "8:\n"
- EX_TABLE(0b,2b) EX_TABLE(3b,4b) EX_TABLE(9b,2b) EX_TABLE(10b,4b)
+ "7: slgr %0,%4\n"
+ " j 5f\n"
+ "4: slgr %0,%0\n"
+ "5:\n"
+ EX_TABLE(0b,2b) EX_TABLE(3b,5b) EX_TABLE(6b,2b) EX_TABLE(7b,5b)
: "+a" (size), "+a" (ptr), "+a" (x), "+a" (tmp1), "=a" (tmp2)
: "d" (reg0) : "cc", "memory");
return size;
@@ -67,49 +57,38 @@ static inline unsigned long copy_from_user_mvcp(void *x, const void __user *ptr,
asm volatile(
" sacf 0\n"
"0: mvcp 0(%0,%2),0(%1),%3\n"
- "10:jz 8f\n"
+ "7: jz 5f\n"
"1: algr %0,%3\n"
" la %1,256(%1)\n"
" la %2,256(%2)\n"
"2: mvcp 0(%0,%2),0(%1),%3\n"
- "11:jnz 1b\n"
- " j 8f\n"
+ "8: jnz 1b\n"
+ " j 5f\n"
"3: la %4,255(%1)\n" /* %4 = ptr + 255 */
" lghi %3,-4096\n"
" nr %4,%3\n" /* %4 = (ptr + 255) & -4096 */
" slgr %4,%1\n"
" clgr %0,%4\n" /* copy crosses next page boundary? */
- " jnh 5f\n"
+ " jnh 6f\n"
"4: mvcp 0(%4,%2),0(%1),%3\n"
- "12:slgr %0,%4\n"
- " algr %2,%4\n"
- "5: lghi %4,-1\n"
- " algr %4,%0\n" /* copy remaining size, subtract 1 */
- " bras %3,7f\n" /* memset loop */
- " xc 0(1,%2),0(%2)\n"
- "6: xc 0(256,%2),0(%2)\n"
- " la %2,256(%2)\n"
- "7: aghi %4,-256\n"
- " jnm 6b\n"
- " ex %4,0(%3)\n"
- " j 9f\n"
- "8: slgr %0,%0\n"
- "9: sacf 768\n"
- EX_TABLE(0b,3b) EX_TABLE(2b,3b) EX_TABLE(4b,5b)
- EX_TABLE(10b,3b) EX_TABLE(11b,3b) EX_TABLE(12b,5b)
+ "9: slgr %0,%4\n"
+ " j 6f\n"
+ "5: slgr %0,%0\n"
+ "6: sacf 768\n"
+ EX_TABLE(0b,3b) EX_TABLE(2b,3b) EX_TABLE(4b,6b)
+ EX_TABLE(7b,3b) EX_TABLE(8b,3b) EX_TABLE(9b,6b)
: "+a" (size), "+a" (ptr), "+a" (x), "+a" (tmp1), "=a" (tmp2)
: : "cc", "memory");
return size;
}
-unsigned long __copy_from_user(void *to, const void __user *from, unsigned long n)
+unsigned long raw_copy_from_user(void *to, const void __user *from, unsigned long n)
{
- check_object_size(to, n, false);
if (static_branch_likely(&have_mvcos))
return copy_from_user_mvcos(to, from, n);
return copy_from_user_mvcp(to, from, n);
}
-EXPORT_SYMBOL(__copy_from_user);
+EXPORT_SYMBOL(raw_copy_from_user);
static inline unsigned long copy_to_user_mvcos(void __user *ptr, const void *x,
unsigned long size)
@@ -176,14 +155,13 @@ static inline unsigned long copy_to_user_mvcs(void __user *ptr, const void *x,
return size;
}
-unsigned long __copy_to_user(void __user *to, const void *from, unsigned long n)
+unsigned long raw_copy_to_user(void __user *to, const void *from, unsigned long n)
{
- check_object_size(from, n, true);
if (static_branch_likely(&have_mvcos))
return copy_to_user_mvcos(to, from, n);
return copy_to_user_mvcs(to, from, n);
}
-EXPORT_SYMBOL(__copy_to_user);
+EXPORT_SYMBOL(raw_copy_to_user);
static inline unsigned long copy_in_user_mvcos(void __user *to, const void __user *from,
unsigned long size)
@@ -240,13 +218,13 @@ static inline unsigned long copy_in_user_mvc(void __user *to, const void __user
return size;
}
-unsigned long __copy_in_user(void __user *to, const void __user *from, unsigned long n)
+unsigned long raw_copy_in_user(void __user *to, const void __user *from, unsigned long n)
{
if (static_branch_likely(&have_mvcos))
return copy_in_user_mvcos(to, from, n);
return copy_in_user_mvc(to, from, n);
}
-EXPORT_SYMBOL(__copy_in_user);
+EXPORT_SYMBOL(raw_copy_in_user);
static inline unsigned long clear_user_mvcos(void __user *to, unsigned long size)
{
Powered by cgit, Gruvboxed by Blaster4385.