diff options
Diffstat (limited to 'lib/percpu-refcount.c')
| -rw-r--r-- | lib/percpu-refcount.c | 18 |
1 files changed, 14 insertions, 4 deletions
diff --git a/lib/percpu-refcount.c b/lib/percpu-refcount.c index 9877682e49c7..4f6c6ebbbbde 100644 --- a/lib/percpu-refcount.c +++ b/lib/percpu-refcount.c @@ -1,3 +1,4 @@ +// SPDX-License-Identifier: GPL-2.0-only #define pr_fmt(fmt) "%s: " fmt "\n", __func__ #include <linux/kernel.h> @@ -69,11 +70,14 @@ int percpu_ref_init(struct percpu_ref *ref, percpu_ref_func_t *release, return -ENOMEM; ref->force_atomic = flags & PERCPU_REF_INIT_ATOMIC; + ref->allow_reinit = flags & PERCPU_REF_ALLOW_REINIT; - if (flags & (PERCPU_REF_INIT_ATOMIC | PERCPU_REF_INIT_DEAD)) + if (flags & (PERCPU_REF_INIT_ATOMIC | PERCPU_REF_INIT_DEAD)) { ref->percpu_count_ptr |= __PERCPU_REF_ATOMIC; - else + ref->allow_reinit = true; + } else { start_count += PERCPU_COUNT_BIAS; + } if (flags & PERCPU_REF_INIT_DEAD) ref->percpu_count_ptr |= __PERCPU_REF_DEAD; @@ -119,6 +123,9 @@ static void percpu_ref_call_confirm_rcu(struct rcu_head *rcu) ref->confirm_switch = NULL; wake_up_all(&percpu_ref_switch_waitq); + if (!ref->allow_reinit) + percpu_ref_exit(ref); + /* drop ref from percpu_ref_switch_to_atomic() */ percpu_ref_put(ref); } @@ -151,7 +158,7 @@ static void percpu_ref_switch_to_atomic_rcu(struct rcu_head *rcu) atomic_long_add((long)count - PERCPU_COUNT_BIAS, &ref->count); WARN_ONCE(atomic_long_read(&ref->count) <= 0, - "percpu ref (%pf) <= 0 (%ld) after switching to atomic", + "percpu ref (%ps) <= 0 (%ld) after switching to atomic", ref->release, atomic_long_read(&ref->count)); /* @ref is viewed as dead on all CPUs, send out switch confirmation */ @@ -194,6 +201,9 @@ static void __percpu_ref_switch_to_percpu(struct percpu_ref *ref) if (!(ref->percpu_count_ptr & __PERCPU_REF_ATOMIC)) return; + if (WARN_ON_ONCE(!ref->allow_reinit)) + return; + atomic_long_add(PERCPU_COUNT_BIAS, &ref->count); /* @@ -333,7 +343,7 @@ void percpu_ref_kill_and_confirm(struct percpu_ref *ref, spin_lock_irqsave(&percpu_ref_switch_lock, flags); WARN_ONCE(ref->percpu_count_ptr & __PERCPU_REF_DEAD, - "%s called more than once on %pf!", __func__, ref->release); + "%s called more than once on %ps!", __func__, ref->release); ref->percpu_count_ptr |= __PERCPU_REF_DEAD; __percpu_ref_switch_mode(ref, confirm_kill); |