diff options
| -rw-r--r-- | include/linux/sched.h | 1 | ||||
| -rw-r--r-- | kernel/rcu/tasks.h | 24 | ||||
| -rw-r--r-- | kernel/sched/core.c | 32 |
3 files changed, 40 insertions, 17 deletions
diff --git a/include/linux/sched.h b/include/linux/sched.h index b88caf54e168..72242bc73d85 100644 --- a/include/linux/sched.h +++ b/include/linux/sched.h @@ -2224,6 +2224,7 @@ static inline void set_task_cpu(struct task_struct *p, unsigned int cpu) extern bool sched_task_on_rq(struct task_struct *p); extern unsigned long get_wchan(struct task_struct *p); +extern struct task_struct *cpu_curr_snapshot(int cpu); /* * In order to reduce various lock holder preemption latencies provide an diff --git a/kernel/rcu/tasks.h b/kernel/rcu/tasks.h index 9d7d6fd4b8a7..c2aae2643a0b 100644 --- a/kernel/rcu/tasks.h +++ b/kernel/rcu/tasks.h @@ -1479,21 +1479,6 @@ static void rcu_tasks_trace_pertask(struct task_struct *t, struct list_head *hop trc_wait_for_one_reader(t, hop); } -/* - * Get the current CPU's current task on the holdout list. - * Calls to this function must be serialized. - */ -static void rcu_tasks_trace_pertask_handler(void *hop_in) -{ - struct list_head *hop = hop_in; - struct task_struct *t = current; - - // Pull in the currently running task, but only if it is currently - // in an RCU tasks trace read-side critical section. - if (rcu_tasks_trace_pertask_prep(t, false)) - trc_add_holdout(t, hop); -} - /* Initialize for a new RCU-tasks-trace grace period. */ static void rcu_tasks_trace_pregp_step(struct list_head *hop) { @@ -1513,8 +1498,13 @@ static void rcu_tasks_trace_pregp_step(struct list_head *hop) // These smp_call_function_single() calls are serialized to // allow safe access to the hop list. - for_each_online_cpu(cpu) - smp_call_function_single(cpu, rcu_tasks_trace_pertask_handler, hop, 1); + for_each_online_cpu(cpu) { + rcu_read_lock(); + t = cpu_curr_snapshot(cpu); + if (rcu_tasks_trace_pertask_prep(t, true)) + trc_add_holdout(t, hop); + rcu_read_unlock(); + } // Only after all running tasks have been accounted for is it // safe to take care of the tasks that have blocked within their diff --git a/kernel/sched/core.c b/kernel/sched/core.c index da0bf6fe9ecd..9568019be124 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c @@ -4264,6 +4264,38 @@ int task_call_func(struct task_struct *p, task_call_f func, void *arg) } /** + * cpu_curr_snapshot - Return a snapshot of the currently running task + * @cpu: The CPU on which to snapshot the task. + * + * Returns the task_struct pointer of the task "currently" running on + * the specified CPU. If the same task is running on that CPU throughout, + * the return value will be a pointer to that task's task_struct structure. + * If the CPU did any context switches even vaguely concurrently with the + * execution of this function, the return value will be a pointer to the + * task_struct structure of a randomly chosen task that was running on + * that CPU somewhere around the time that this function was executing. + * + * If the specified CPU was offline, the return value is whatever it + * is, perhaps a pointer to the task_struct structure of that CPU's idle + * task, but there is no guarantee. Callers wishing a useful return + * value must take some action to ensure that the specified CPU remains + * online throughout. + * + * This function executes full memory barriers before and after fetching + * the pointer, which permits the caller to confine this function's fetch + * with respect to the caller's accesses to other shared variables. + */ +struct task_struct *cpu_curr_snapshot(int cpu) +{ + struct task_struct *t; + + smp_mb(); /* Pairing determined by caller's synchronization design. */ + t = rcu_dereference(cpu_curr(cpu)); + smp_mb(); /* Pairing determined by caller's synchronization design. */ + return t; +} + +/** * wake_up_process - Wake up a specific process * @p: The process to be woken up. * |