kevin/linux
1
Fork 0
Code

rcuscale: Count outstanding callbacks per-task rather than per-CPU

Browse Source 
The current rcu_scale_writer() asynchronous grace-period testing uses a
per-CPU counter to track the number of outstanding callbacks.  This is
subject to CPU-imbalance errors when tasks migrate from one CPU to another
between the time that the counter is incremented and the callback is
queued, and additionally in kernels configured such that callbacks can
be invoked on some CPU other than the one that queued it.

This commit therefore arranges for per-task callback counts, thus avoiding
any issues with migration of either tasks or callbacks.

Reported-by: Vlastimil Babka <vbabka@suse.cz>
Signed-off-by: "Paul E. McKenney" <paulmck@kernel.org>
Signed-off-by: Neeraj Upadhyay <neeraj.upadhyay@kernel.org>
This commit is contained in:
Paul E. McKenney 2024-08-01 17:43:08 -07:00 committed by Neeraj Upadhyay
parent 554f07a119
commit f1fd0e0bb1
1 changed files with 6 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
kernel/rcu/rcuscale.c
View File

@ -114,6 +114,7 @@ struct writer_mblock {
struct writer_freelist { struct writer_freelist {
struct llist_head ws_lhg; struct llist_head ws_lhg;
atomic_t ws_inflight;
struct llist_head ____cacheline_internodealigned_in_smp ws_lhp; struct llist_head ____cacheline_internodealigned_in_smp ws_lhp;
struct writer_mblock *ws_mblocks; struct writer_mblock *ws_mblocks;
}; };
@ -136,7 +137,6 @@ static u64 t_rcu_scale_writer_started;
static u64 t_rcu_scale_writer_finished; static u64 t_rcu_scale_writer_finished;
static unsigned long b_rcu_gp_test_started; static unsigned long b_rcu_gp_test_started;
static unsigned long b_rcu_gp_test_finished; static unsigned long b_rcu_gp_test_finished;
static DEFINE_PER_CPU(atomic_t, n_async_inflight);
#define MAX_MEAS 10000 #define MAX_MEAS 10000
#define MIN_MEAS 100 #define MIN_MEAS 100
@ -520,8 +520,9 @@ static void rcu_scale_free(struct writer_mblock *wmbp)
static void rcu_scale_async_cb(struct rcu_head *rhp) static void rcu_scale_async_cb(struct rcu_head *rhp)
{ {
struct writer_mblock *wmbp = container_of(rhp, struct writer_mblock, wmb_rh); struct writer_mblock *wmbp = container_of(rhp, struct writer_mblock, wmb_rh);
struct writer_freelist *wflp = wmbp->wmb_wfl;
atomic_dec(this_cpu_ptr(&n_async_inflight)); atomic_dec(&wflp->ws_inflight);
rcu_scale_free(wmbp); rcu_scale_free(wmbp);
} }
@ -541,6 +542,7 @@ rcu_scale_writer(void *arg)
DEFINE_TORTURE_RANDOM(tr); DEFINE_TORTURE_RANDOM(tr);
u64 *wdp; u64 *wdp;
u64 *wdpp = writer_durations[me]; u64 *wdpp = writer_durations[me];
struct writer_freelist *wflp = &writer_freelists[me];
struct writer_mblock *wmbp = NULL; struct writer_mblock *wmbp = NULL;
VERBOSE_SCALEOUT_STRING("rcu_scale_writer task started"); VERBOSE_SCALEOUT_STRING("rcu_scale_writer task started");
@ -584,8 +586,8 @@ rcu_scale_writer(void *arg)
if (gp_async && !WARN_ON_ONCE(!cur_ops->async)) { if (gp_async && !WARN_ON_ONCE(!cur_ops->async)) {
if (!wmbp) if (!wmbp)
wmbp = rcu_scale_alloc(me); wmbp = rcu_scale_alloc(me);
if (wmbp && atomic_read(this_cpu_ptr(&n_async_inflight)) < gp_async_max) { if (wmbp && atomic_read(&wflp->ws_inflight) < gp_async_max) {
atomic_inc(this_cpu_ptr(&n_async_inflight)); atomic_inc(&wflp->ws_inflight);
cur_ops->async(&wmbp->wmb_rh, rcu_scale_async_cb); cur_ops->async(&wmbp->wmb_rh, rcu_scale_async_cb);
wmbp = NULL; wmbp = NULL;
gp_succeeded = true; gp_succeeded = true;