diff options
| author | Ingo Molnar <mingo@kernel.org> | 2017-04-27 07:41:37 +0200 |
|---|---|---|
| committer | Ingo Molnar <mingo@kernel.org> | 2017-04-27 07:41:37 +0200 |
| commit | 96fd20cf3f1620d699db43cd08745f1e57e1072a (patch) | |
| tree | 88b0caa1cd8bfa66e6a4b00ad21108f03369eb03 /include/linux/srcutree.h | |
| parent | d160a727c40e7175aa642137910a3fda46262fc8 (diff) | |
| parent | 22607d66bbc3e81140d3bcf08894f4378eb36428 (diff) | |
Merge branch 'rcu/urgent' of git://git.kernel.org/pub/scm/linux/kernel/git/paulmck/linux-rcu into core/rcu
Pull RCU updates from Paul E. McKenney:
"This series greatly reduces the performance degradation of Tree SRCU
on a CPU-hotplug stress test. The effect was not subtle: Mike Galbraith
measured Classic SRCU at 55 seconds and Tree SRCU at more than 16 -minutes-
for this test. Mike collected ftrace data that showed that Classic SRCU
was auto-expediting invocations of synchronize_srcu() that found SRCU
completely idle. This series therefore adds this auto-expedite capability
to Tree SRCU, bringing the performance shortfall to less than ten seconds,
which is a great improvement over the initial shortfall of 15 minutes."
Signed-off-by: Ingo Molnar <mingo@kernel.org>
Diffstat (limited to 'include/linux/srcutree.h')
| -rw-r--r-- | include/linux/srcutree.h | 13 |
1 files changed, 12 insertions, 1 deletions
diff --git a/include/linux/srcutree.h b/include/linux/srcutree.h index 0400e211aa44..32e86d85fd11 100644 --- a/include/linux/srcutree.h +++ b/include/linux/srcutree.h @@ -43,10 +43,13 @@ struct srcu_data { spinlock_t lock ____cacheline_internodealigned_in_smp; struct rcu_segcblist srcu_cblist; /* List of callbacks.*/ unsigned long srcu_gp_seq_needed; /* Furthest future GP needed. */ + unsigned long srcu_gp_seq_needed_exp; /* Furthest future exp GP. */ bool srcu_cblist_invoking; /* Invoking these CBs? */ struct delayed_work work; /* Context for CB invoking. */ struct rcu_head srcu_barrier_head; /* For srcu_barrier() use. */ struct srcu_node *mynode; /* Leaf srcu_node. */ + unsigned long grpmask; /* Mask for leaf srcu_node */ + /* ->srcu_data_have_cbs[]. */ int cpu; struct srcu_struct *sp; }; @@ -59,6 +62,9 @@ struct srcu_node { unsigned long srcu_have_cbs[4]; /* GP seq for children */ /* having CBs, but only */ /* is > ->srcu_gq_seq. */ + unsigned long srcu_data_have_cbs[4]; /* Which srcu_data structs */ + /* have CBs for given GP? */ + unsigned long srcu_gp_seq_needed_exp; /* Furthest future exp GP. */ struct srcu_node *srcu_parent; /* Next up in tree. */ int grplo; /* Least CPU for node. */ int grphi; /* Biggest CPU for node. */ @@ -77,7 +83,8 @@ struct srcu_struct { unsigned int srcu_idx; /* Current rdr array element. */ unsigned long srcu_gp_seq; /* Grace-period seq #. */ unsigned long srcu_gp_seq_needed; /* Latest gp_seq needed. */ - atomic_t srcu_exp_cnt; /* # ongoing expedited GPs. */ + unsigned long srcu_gp_seq_needed_exp; /* Furthest future exp GP. */ + unsigned long srcu_last_gp_end; /* Last GP end timestamp (ns) */ struct srcu_data __percpu *sda; /* Per-CPU srcu_data array. */ unsigned long srcu_barrier_seq; /* srcu_barrier seq #. */ struct mutex srcu_barrier_mutex; /* Serialize barrier ops. */ @@ -136,4 +143,8 @@ void synchronize_srcu_expedited(struct srcu_struct *sp); void srcu_barrier(struct srcu_struct *sp); unsigned long srcu_batches_completed(struct srcu_struct *sp); +void srcutorture_get_gp_data(enum rcutorture_type test_type, + struct srcu_struct *sp, int *flags, + unsigned long *gpnum, unsigned long *completed); + #endif |