2020-03-02 12:59:20 -07:00
|
|
|
/* SPDX-License-Identifier: GPL-2.0+ */
|
|
|
|
|
/*
|
|
|
|
|
* Task-based RCU implementations.
|
|
|
|
|
*
|
|
|
|
|
* Copyright (C) 2020 Paul E. McKenney
|
|
|
|
|
*/
|
|
|
|
|
|
2020-03-15 14:51:20 -07:00
|
|
|
#ifdef CONFIG_TASKS_RCU_GENERIC
|
2021-11-08 17:18:57 -07:00
|
|
|
#include "rcu_segcblist.h"
|
2020-03-03 12:49:21 -07:00
|
|
|
|
|
|
|
|
////////////////////////////////////////////////////////////////////////
|
|
|
|
|
//
|
|
|
|
|
// Generic data structures.
|
|
|
|
|
|
|
|
|
|
struct rcu_tasks;
|
|
|
|
|
typedef void (*rcu_tasks_gp_func_t)(struct rcu_tasks *rtp);
|
2022-05-18 16:06:55 -07:00
|
|
|
typedef void (*pregp_func_t)(struct list_head *hop);
|
2020-03-04 18:31:43 -07:00
|
|
|
typedef void (*pertask_func_t)(struct task_struct *t, struct list_head *hop);
|
2020-03-22 13:18:54 -07:00
|
|
|
typedef void (*postscan_func_t)(struct list_head *hop);
|
2020-03-04 18:31:43 -07:00
|
|
|
typedef void (*holdouts_func_t)(struct list_head *hop, bool ndrpt, bool *frptp);
|
2020-03-16 12:13:33 -07:00
|
|
|
typedef void (*postgp_func_t)(struct rcu_tasks *rtp);
|
2020-03-02 12:59:20 -07:00
|
|
|
|
2020-03-02 16:16:57 -07:00
|
|
|
/**
|
2021-11-05 21:52:00 -07:00
|
|
|
* struct rcu_tasks_percpu - Per-CPU component of definition for a Tasks-RCU-like mechanism.
|
2021-11-08 17:18:57 -07:00
|
|
|
* @cblist: Callback list.
|
2021-11-08 17:52:02 -07:00
|
|
|
* @lock: Lock protecting per-CPU callback list.
|
2021-11-22 14:38:42 -07:00
|
|
|
* @rtp_jiffies: Jiffies counter value for statistics.
|
rcu-tasks: Treat only synchronous grace periods urgently
The performance requirements on RCU Tasks, and in particular on RCU
Tasks Trace, have evolved over time as the workloads have evolved.
The current implementation is designed to provide low grace-period
latencies, and also to accommodate short-duration floods of callbacks.
However, current workloads can also provide a constant background
callback-queuing rate of a few hundred call_rcu_tasks_trace() invocations
per second. This results in continuous back-to-back RCU Tasks Trace
grace periods, which in turn can consume the better part of 10% of a CPU.
One could take the attitude that there are several tens of other CPUs on
the systems running such workloads, but energy efficiency is a thing.
On these systems, although asynchronous grace-period requests happen
every few milliseconds, synchronous grace-period requests are quite rare.
This commit therefore arrranges for grace periods to be initiated
immediately in response to calls to synchronize_rcu_tasks*() and
also to calls to synchronize_rcu_mult() that are passed one of the
call_rcu_tasks*() functions. These are recognized by the tell-tale
wakeme_after_rcu callback function.
In other cases, callbacks are gathered up for up to about 250 milliseconds
before a grace period is initiated. This results in more than an order of
magnitude reduction in RCU Tasks Trace grace periods, with corresponding
reduction in consumption of CPU time.
Reported-by: Alexei Starovoitov <ast@kernel.org>
Reported-by: Martin KaFai Lau <kafai@fb.com>
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
2023-05-14 18:06:12 -07:00
|
|
|
* @lazy_timer: Timer to unlazify callbacks.
|
|
|
|
|
* @urgent_gp: Number of additional non-lazy grace periods.
|
2021-11-22 14:38:42 -07:00
|
|
|
* @rtp_n_lock_retries: Rough lock-contention statistic.
|
2021-11-10 16:56:40 -07:00
|
|
|
* @rtp_work: Work queue for invoking callbacks.
|
2021-11-23 17:16:50 -07:00
|
|
|
* @rtp_irq_work: IRQ work queue for deferred wakeups.
|
2021-11-11 15:53:43 -07:00
|
|
|
* @barrier_q_head: RCU callback for barrier operation.
|
2022-05-16 17:56:16 -07:00
|
|
|
* @rtp_blkd_tasks: List of tasks blocked as readers.
|
2024-02-05 14:08:22 -07:00
|
|
|
* @rtp_exit_list: List of tasks in the latter portion of do_exit().
|
2021-11-11 15:53:43 -07:00
|
|
|
* @cpu: CPU number corresponding to this entry.
|
2024-07-09 21:45:42 -07:00
|
|
|
* @index: Index of this CPU in rtpcp_array of the rcu_tasks structure.
|
2021-11-11 15:53:43 -07:00
|
|
|
* @rtpp: Pointer to the rcu_tasks structure.
|
2021-11-05 21:52:00 -07:00
|
|
|
*/
|
|
|
|
|
struct rcu_tasks_percpu {
|
2021-11-08 17:18:57 -07:00
|
|
|
struct rcu_segcblist cblist;
|
2021-11-08 17:52:02 -07:00
|
|
|
raw_spinlock_t __private lock;
|
2021-11-22 14:38:42 -07:00
|
|
|
unsigned long rtp_jiffies;
|
|
|
|
|
unsigned long rtp_n_lock_retries;
|
rcu-tasks: Treat only synchronous grace periods urgently
The performance requirements on RCU Tasks, and in particular on RCU
Tasks Trace, have evolved over time as the workloads have evolved.
The current implementation is designed to provide low grace-period
latencies, and also to accommodate short-duration floods of callbacks.
However, current workloads can also provide a constant background
callback-queuing rate of a few hundred call_rcu_tasks_trace() invocations
per second. This results in continuous back-to-back RCU Tasks Trace
grace periods, which in turn can consume the better part of 10% of a CPU.
One could take the attitude that there are several tens of other CPUs on
the systems running such workloads, but energy efficiency is a thing.
On these systems, although asynchronous grace-period requests happen
every few milliseconds, synchronous grace-period requests are quite rare.
This commit therefore arrranges for grace periods to be initiated
immediately in response to calls to synchronize_rcu_tasks*() and
also to calls to synchronize_rcu_mult() that are passed one of the
call_rcu_tasks*() functions. These are recognized by the tell-tale
wakeme_after_rcu callback function.
In other cases, callbacks are gathered up for up to about 250 milliseconds
before a grace period is initiated. This results in more than an order of
magnitude reduction in RCU Tasks Trace grace periods, with corresponding
reduction in consumption of CPU time.
Reported-by: Alexei Starovoitov <ast@kernel.org>
Reported-by: Martin KaFai Lau <kafai@fb.com>
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
2023-05-14 18:06:12 -07:00
|
|
|
struct timer_list lazy_timer;
|
|
|
|
|
unsigned int urgent_gp;
|
2021-11-10 16:56:40 -07:00
|
|
|
struct work_struct rtp_work;
|
2021-11-23 17:16:50 -07:00
|
|
|
struct irq_work rtp_irq_work;
|
2021-11-11 15:53:43 -07:00
|
|
|
struct rcu_head barrier_q_head;
|
2022-05-16 17:56:16 -07:00
|
|
|
struct list_head rtp_blkd_tasks;
|
2024-02-05 14:08:22 -07:00
|
|
|
struct list_head rtp_exit_list;
|
2021-11-10 16:56:40 -07:00
|
|
|
int cpu;
|
2024-07-09 21:45:42 -07:00
|
|
|
int index;
|
2021-11-10 16:56:40 -07:00
|
|
|
struct rcu_tasks *rtpp;
|
2021-11-05 21:52:00 -07:00
|
|
|
};
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* struct rcu_tasks - Definition for a Tasks-RCU-like mechanism.
|
2022-03-04 04:07:25 -07:00
|
|
|
* @cbs_wait: RCU wait allowing a new callback to get kthread's attention.
|
2021-11-05 21:52:00 -07:00
|
|
|
* @cbs_gbl_lock: Lock protecting callback list.
|
2022-04-19 10:47:28 -07:00
|
|
|
* @tasks_gp_mutex: Mutex protecting grace period, needed during mid-boot dead zone.
|
2020-03-03 12:49:21 -07:00
|
|
|
* @gp_func: This flavor's grace-period-wait function.
|
2020-03-16 12:13:33 -07:00
|
|
|
* @gp_state: Grace period's most recent state transition (debugging).
|
2020-09-09 22:05:41 -07:00
|
|
|
* @gp_sleep: Per-grace-period sleep to prevent CPU-bound looping.
|
2020-09-09 21:36:34 -07:00
|
|
|
* @init_fract: Initial backoff sleep interval.
|
2020-03-16 12:13:33 -07:00
|
|
|
* @gp_jiffies: Time of last @gp_state transition.
|
|
|
|
|
* @gp_start: Most recent grace-period start in jiffies.
|
2024-08-01 17:34:22 -07:00
|
|
|
* @tasks_gp_seq: Number of grace periods completed since boot in upper bits.
|
2020-03-18 10:54:05 -07:00
|
|
|
* @n_ipis: Number of IPIs sent to encourage grace periods to end.
|
2020-03-25 14:36:05 -07:00
|
|
|
* @n_ipis_fails: Number of IPI-send failures.
|
rcu-tasks: Treat only synchronous grace periods urgently
The performance requirements on RCU Tasks, and in particular on RCU
Tasks Trace, have evolved over time as the workloads have evolved.
The current implementation is designed to provide low grace-period
latencies, and also to accommodate short-duration floods of callbacks.
However, current workloads can also provide a constant background
callback-queuing rate of a few hundred call_rcu_tasks_trace() invocations
per second. This results in continuous back-to-back RCU Tasks Trace
grace periods, which in turn can consume the better part of 10% of a CPU.
One could take the attitude that there are several tens of other CPUs on
the systems running such workloads, but energy efficiency is a thing.
On these systems, although asynchronous grace-period requests happen
every few milliseconds, synchronous grace-period requests are quite rare.
This commit therefore arrranges for grace periods to be initiated
immediately in response to calls to synchronize_rcu_tasks*() and
also to calls to synchronize_rcu_mult() that are passed one of the
call_rcu_tasks*() functions. These are recognized by the tell-tale
wakeme_after_rcu callback function.
In other cases, callbacks are gathered up for up to about 250 milliseconds
before a grace period is initiated. This results in more than an order of
magnitude reduction in RCU Tasks Trace grace periods, with corresponding
reduction in consumption of CPU time.
Reported-by: Alexei Starovoitov <ast@kernel.org>
Reported-by: Martin KaFai Lau <kafai@fb.com>
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
2023-05-14 18:06:12 -07:00
|
|
|
* @kthread_ptr: This flavor's grace-period/callback-invocation kthread.
|
|
|
|
|
* @lazy_jiffies: Number of jiffies to allow callbacks to be lazy.
|
2020-03-04 18:31:43 -07:00
|
|
|
* @pregp_func: This flavor's pre-grace-period function (optional).
|
|
|
|
|
* @pertask_func: This flavor's per-task scan function (optional).
|
|
|
|
|
* @postscan_func: This flavor's post-task scan function (optional).
|
2021-01-25 00:41:05 -07:00
|
|
|
* @holdouts_func: This flavor's holdout-list scan function (optional).
|
2020-03-04 18:31:43 -07:00
|
|
|
* @postgp_func: This flavor's post-grace-period function (optional).
|
2020-03-03 12:49:21 -07:00
|
|
|
* @call_func: This flavor's call_rcu()-equivalent function.
|
2024-02-23 16:16:20 -07:00
|
|
|
* @wait_state: Task state for synchronous grace-period waits (default TASK_UNINTERRUPTIBLE).
|
2021-11-05 21:52:00 -07:00
|
|
|
* @rtpcpu: This flavor's rcu_tasks_percpu structure.
|
2024-07-09 21:45:42 -07:00
|
|
|
* @rtpcp_array: Array of pointers to rcu_tasks_percpu structure of CPUs in cpu_possible_mask.
|
2021-11-08 11:51:13 -07:00
|
|
|
* @percpu_enqueue_shift: Shift down CPU ID this much when enqueuing callbacks.
|
2021-11-29 12:46:33 -07:00
|
|
|
* @percpu_enqueue_lim: Number of per-CPU callback queues in use for enqueuing.
|
|
|
|
|
* @percpu_dequeue_lim: Number of per-CPU callback queues in use for dequeuing.
|
rcu-tasks: Use fewer callbacks queues if callback flood ends
By default, when lock contention is encountered, the RCU Tasks flavors
of RCU switch to using per-CPU queueing. However, if the callback
flood ends, per-CPU queueing continues to be used, which introduces
significant additional overhead, especially for callback invocation,
which fans out a series of workqueue handlers.
This commit therefore switches back to single-queue operation if at the
beginning of a grace period there are very few callbacks. The definition
of "very few" is set by the rcupdate.rcu_task_collapse_lim module
parameter, which defaults to 10. This switch happens in two phases,
with the first phase causing future callbacks to be enqueued on CPU 0's
queue, but with all queues continuing to be checked for grace periods
and callback invocation. The second phase checks to see if an RCU grace
period has elapsed and if all remaining RCU-Tasks callbacks are queued
on CPU 0. If so, only CPU 0 is checked for future grace periods and
callback operation.
Of course, the return of contention anywhere during this process will
result in returning to per-CPU callback queueing.
Reported-by: Martin Lau <kafai@fb.com>
Cc: Neeraj Upadhyay <neeraj.iitr10@gmail.com>
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
2021-11-29 17:52:31 -07:00
|
|
|
* @percpu_dequeue_gpseq: RCU grace-period number to propagate enqueue limit to dequeuers.
|
2021-11-11 15:53:43 -07:00
|
|
|
* @barrier_q_mutex: Serialize barrier operations.
|
|
|
|
|
* @barrier_q_count: Number of queues being waited on.
|
|
|
|
|
* @barrier_q_completion: Barrier wait/wakeup mechanism.
|
|
|
|
|
* @barrier_q_seq: Sequence number for barrier operations.
|
2024-08-01 17:34:26 -07:00
|
|
|
* @barrier_q_start: Most recent barrier start in jiffies.
|
2020-03-03 16:50:31 -07:00
|
|
|
* @name: This flavor's textual name.
|
|
|
|
|
* @kname: This flavor's kthread name.
|
2020-03-02 16:16:57 -07:00
|
|
|
*/
|
|
|
|
|
struct rcu_tasks {
|
2022-03-04 04:07:25 -07:00
|
|
|
struct rcuwait cbs_wait;
|
2021-11-05 21:52:00 -07:00
|
|
|
raw_spinlock_t cbs_gbl_lock;
|
2022-04-19 10:47:28 -07:00
|
|
|
struct mutex tasks_gp_mutex;
|
2020-03-16 12:13:33 -07:00
|
|
|
int gp_state;
|
2020-09-09 22:05:41 -07:00
|
|
|
int gp_sleep;
|
2020-09-09 21:36:34 -07:00
|
|
|
int init_fract;
|
2020-03-16 12:13:33 -07:00
|
|
|
unsigned long gp_jiffies;
|
2020-03-17 08:57:02 -07:00
|
|
|
unsigned long gp_start;
|
2021-11-08 15:14:43 -07:00
|
|
|
unsigned long tasks_gp_seq;
|
2020-03-18 10:54:05 -07:00
|
|
|
unsigned long n_ipis;
|
2020-03-25 14:36:05 -07:00
|
|
|
unsigned long n_ipis_fails;
|
2020-03-02 16:16:57 -07:00
|
|
|
struct task_struct *kthread_ptr;
|
rcu-tasks: Treat only synchronous grace periods urgently
The performance requirements on RCU Tasks, and in particular on RCU
Tasks Trace, have evolved over time as the workloads have evolved.
The current implementation is designed to provide low grace-period
latencies, and also to accommodate short-duration floods of callbacks.
However, current workloads can also provide a constant background
callback-queuing rate of a few hundred call_rcu_tasks_trace() invocations
per second. This results in continuous back-to-back RCU Tasks Trace
grace periods, which in turn can consume the better part of 10% of a CPU.
One could take the attitude that there are several tens of other CPUs on
the systems running such workloads, but energy efficiency is a thing.
On these systems, although asynchronous grace-period requests happen
every few milliseconds, synchronous grace-period requests are quite rare.
This commit therefore arrranges for grace periods to be initiated
immediately in response to calls to synchronize_rcu_tasks*() and
also to calls to synchronize_rcu_mult() that are passed one of the
call_rcu_tasks*() functions. These are recognized by the tell-tale
wakeme_after_rcu callback function.
In other cases, callbacks are gathered up for up to about 250 milliseconds
before a grace period is initiated. This results in more than an order of
magnitude reduction in RCU Tasks Trace grace periods, with corresponding
reduction in consumption of CPU time.
Reported-by: Alexei Starovoitov <ast@kernel.org>
Reported-by: Martin KaFai Lau <kafai@fb.com>
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
2023-05-14 18:06:12 -07:00
|
|
|
unsigned long lazy_jiffies;
|
2020-03-03 12:49:21 -07:00
|
|
|
rcu_tasks_gp_func_t gp_func;
|
2020-03-04 18:31:43 -07:00
|
|
|
pregp_func_t pregp_func;
|
|
|
|
|
pertask_func_t pertask_func;
|
|
|
|
|
postscan_func_t postscan_func;
|
|
|
|
|
holdouts_func_t holdouts_func;
|
|
|
|
|
postgp_func_t postgp_func;
|
2020-03-03 12:49:21 -07:00
|
|
|
call_rcu_func_t call_func;
|
2024-02-23 16:16:20 -07:00
|
|
|
unsigned int wait_state;
|
2021-11-05 21:52:00 -07:00
|
|
|
struct rcu_tasks_percpu __percpu *rtpcpu;
|
2024-07-09 21:45:42 -07:00
|
|
|
struct rcu_tasks_percpu **rtpcp_array;
|
2021-11-08 11:51:13 -07:00
|
|
|
int percpu_enqueue_shift;
|
2021-11-09 12:11:32 -07:00
|
|
|
int percpu_enqueue_lim;
|
2021-11-29 12:46:33 -07:00
|
|
|
int percpu_dequeue_lim;
|
rcu-tasks: Use fewer callbacks queues if callback flood ends
By default, when lock contention is encountered, the RCU Tasks flavors
of RCU switch to using per-CPU queueing. However, if the callback
flood ends, per-CPU queueing continues to be used, which introduces
significant additional overhead, especially for callback invocation,
which fans out a series of workqueue handlers.
This commit therefore switches back to single-queue operation if at the
beginning of a grace period there are very few callbacks. The definition
of "very few" is set by the rcupdate.rcu_task_collapse_lim module
parameter, which defaults to 10. This switch happens in two phases,
with the first phase causing future callbacks to be enqueued on CPU 0's
queue, but with all queues continuing to be checked for grace periods
and callback invocation. The second phase checks to see if an RCU grace
period has elapsed and if all remaining RCU-Tasks callbacks are queued
on CPU 0. If so, only CPU 0 is checked for future grace periods and
callback operation.
Of course, the return of contention anywhere during this process will
result in returning to per-CPU callback queueing.
Reported-by: Martin Lau <kafai@fb.com>
Cc: Neeraj Upadhyay <neeraj.iitr10@gmail.com>
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
2021-11-29 17:52:31 -07:00
|
|
|
unsigned long percpu_dequeue_gpseq;
|
2021-11-11 15:53:43 -07:00
|
|
|
struct mutex barrier_q_mutex;
|
|
|
|
|
atomic_t barrier_q_count;
|
|
|
|
|
struct completion barrier_q_completion;
|
|
|
|
|
unsigned long barrier_q_seq;
|
2024-08-01 17:34:26 -07:00
|
|
|
unsigned long barrier_q_start;
|
2020-03-03 16:50:31 -07:00
|
|
|
char *name;
|
|
|
|
|
char *kname;
|
2020-03-02 16:16:57 -07:00
|
|
|
};
|
|
|
|
|
|
2021-11-23 17:16:50 -07:00
|
|
|
static void call_rcu_tasks_iw_wakeup(struct irq_work *iwp);
|
|
|
|
|
|
2021-11-05 21:52:00 -07:00
|
|
|
#define DEFINE_RCU_TASKS(rt_name, gp, call, n) \
|
|
|
|
|
static DEFINE_PER_CPU(struct rcu_tasks_percpu, rt_name ## __percpu) = { \
|
2021-11-08 17:52:02 -07:00
|
|
|
.lock = __RAW_SPIN_LOCK_UNLOCKED(rt_name ## __percpu.cbs_pcpu_lock), \
|
2022-03-04 04:07:25 -07:00
|
|
|
.rtp_irq_work = IRQ_WORK_INIT_HARD(call_rcu_tasks_iw_wakeup), \
|
2021-11-05 21:52:00 -07:00
|
|
|
}; \
|
|
|
|
|
static struct rcu_tasks rt_name = \
|
|
|
|
|
{ \
|
2022-03-04 04:07:25 -07:00
|
|
|
.cbs_wait = __RCUWAIT_INITIALIZER(rt_name.wait), \
|
2021-11-05 21:52:00 -07:00
|
|
|
.cbs_gbl_lock = __RAW_SPIN_LOCK_UNLOCKED(rt_name.cbs_gbl_lock), \
|
2022-04-19 10:47:28 -07:00
|
|
|
.tasks_gp_mutex = __MUTEX_INITIALIZER(rt_name.tasks_gp_mutex), \
|
2021-11-05 21:52:00 -07:00
|
|
|
.gp_func = gp, \
|
|
|
|
|
.call_func = call, \
|
2024-02-23 16:16:20 -07:00
|
|
|
.wait_state = TASK_UNINTERRUPTIBLE, \
|
2021-11-05 21:52:00 -07:00
|
|
|
.rtpcpu = &rt_name ## __percpu, \
|
rcu-tasks: Treat only synchronous grace periods urgently
The performance requirements on RCU Tasks, and in particular on RCU
Tasks Trace, have evolved over time as the workloads have evolved.
The current implementation is designed to provide low grace-period
latencies, and also to accommodate short-duration floods of callbacks.
However, current workloads can also provide a constant background
callback-queuing rate of a few hundred call_rcu_tasks_trace() invocations
per second. This results in continuous back-to-back RCU Tasks Trace
grace periods, which in turn can consume the better part of 10% of a CPU.
One could take the attitude that there are several tens of other CPUs on
the systems running such workloads, but energy efficiency is a thing.
On these systems, although asynchronous grace-period requests happen
every few milliseconds, synchronous grace-period requests are quite rare.
This commit therefore arrranges for grace periods to be initiated
immediately in response to calls to synchronize_rcu_tasks*() and
also to calls to synchronize_rcu_mult() that are passed one of the
call_rcu_tasks*() functions. These are recognized by the tell-tale
wakeme_after_rcu callback function.
In other cases, callbacks are gathered up for up to about 250 milliseconds
before a grace period is initiated. This results in more than an order of
magnitude reduction in RCU Tasks Trace grace periods, with corresponding
reduction in consumption of CPU time.
Reported-by: Alexei Starovoitov <ast@kernel.org>
Reported-by: Martin KaFai Lau <kafai@fb.com>
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
2023-05-14 18:06:12 -07:00
|
|
|
.lazy_jiffies = DIV_ROUND_UP(HZ, 4), \
|
2021-11-05 21:52:00 -07:00
|
|
|
.name = n, \
|
2022-02-02 16:42:36 -07:00
|
|
|
.percpu_enqueue_shift = order_base_2(CONFIG_NR_CPUS), \
|
2021-11-09 12:11:32 -07:00
|
|
|
.percpu_enqueue_lim = 1, \
|
2021-11-29 12:46:33 -07:00
|
|
|
.percpu_dequeue_lim = 1, \
|
2021-11-11 15:53:43 -07:00
|
|
|
.barrier_q_mutex = __MUTEX_INITIALIZER(rt_name.barrier_q_mutex), \
|
|
|
|
|
.barrier_q_seq = (0UL - 50UL) << RCU_SEQ_CTR_SHIFT, \
|
2021-11-05 21:52:00 -07:00
|
|
|
.kname = #rt_name, \
|
2020-03-02 16:16:57 -07:00
|
|
|
}
|
|
|
|
|
|
2023-03-19 17:29:20 -07:00
|
|
|
#ifdef CONFIG_TASKS_RCU
|
2020-03-02 12:59:20 -07:00
|
|
|
|
2024-02-25 20:24:39 -07:00
|
|
|
/* Report delay of scan exiting tasklist in rcu_tasks_postscan(). */
|
2023-01-11 00:45:00 -07:00
|
|
|
static void tasks_rcu_exit_srcu_stall(struct timer_list *unused);
|
|
|
|
|
static DEFINE_TIMER(tasks_rcu_exit_srcu_stall_timer, tasks_rcu_exit_srcu_stall);
|
|
|
|
|
#endif
|
|
|
|
|
|
2020-03-17 11:39:26 -07:00
|
|
|
/* Avoid IPIing CPUs early in the grace period. */
|
2020-09-09 21:51:09 -07:00
|
|
|
#define RCU_TASK_IPI_DELAY (IS_ENABLED(CONFIG_TASKS_TRACE_RCU_READ_MB) ? HZ / 2 : 0)
|
2020-03-17 11:39:26 -07:00
|
|
|
static int rcu_task_ipi_delay __read_mostly = RCU_TASK_IPI_DELAY;
|
|
|
|
|
module_param(rcu_task_ipi_delay, int, 0644);
|
|
|
|
|
|
2020-03-02 12:59:20 -07:00
|
|
|
/* Control stall timeouts. Disable with <= 0, otherwise jiffies till stall. */
|
2022-06-07 15:23:52 -07:00
|
|
|
#define RCU_TASK_BOOT_STALL_TIMEOUT (HZ * 30)
|
2020-03-02 12:59:20 -07:00
|
|
|
#define RCU_TASK_STALL_TIMEOUT (HZ * 60 * 10)
|
|
|
|
|
static int rcu_task_stall_timeout __read_mostly = RCU_TASK_STALL_TIMEOUT;
|
|
|
|
|
module_param(rcu_task_stall_timeout, int, 0644);
|
rcu-tasks: Print pre-stall-warning informational messages
RCU-tasks stall-warning messages are printed after the grace period is ten
minutes old. Unfortunately, most of us will have rebooted the system in
response to an apparently-hung command long before the ten minutes is up,
and will thus see what looks to be a silent hang.
This commit therefore adds pr_info() messages that are printed earlier.
These should avoid being classified as errors, but should give impatient
users a hint. These are controlled by new rcupdate.rcu_task_stall_info
and rcupdate.rcu_task_stall_info_mult kernel-boot parameters. The former
defines the initial delay in jiffies (defaulting to 10 seconds) and the
latter defines the multiplier (defaulting to 3). Thus, by default, the
first message will appear 10 seconds into the RCU-tasks grace period,
the second 40 seconds in, and the third 160 seconds in. There would be
a fourth at 640 seconds in, but the stall warning message appears 600
seconds in, and once a stall warning is printed for a given grace period,
no further informational messages are printed.
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
2022-02-25 17:01:12 -07:00
|
|
|
#define RCU_TASK_STALL_INFO (HZ * 10)
|
|
|
|
|
static int rcu_task_stall_info __read_mostly = RCU_TASK_STALL_INFO;
|
|
|
|
|
module_param(rcu_task_stall_info, int, 0644);
|
|
|
|
|
static int rcu_task_stall_info_mult __read_mostly = 3;
|
|
|
|
|
module_param(rcu_task_stall_info_mult, int, 0444);
|
2020-03-02 12:59:20 -07:00
|
|
|
|
2021-11-12 08:33:40 -07:00
|
|
|
static int rcu_task_enqueue_lim __read_mostly = -1;
|
|
|
|
|
module_param(rcu_task_enqueue_lim, int, 0444);
|
|
|
|
|
|
2021-11-24 16:12:15 -07:00
|
|
|
static bool rcu_task_cb_adjust;
|
|
|
|
|
static int rcu_task_contend_lim __read_mostly = 100;
|
|
|
|
|
module_param(rcu_task_contend_lim, int, 0444);
|
rcu-tasks: Use fewer callbacks queues if callback flood ends
By default, when lock contention is encountered, the RCU Tasks flavors
of RCU switch to using per-CPU queueing. However, if the callback
flood ends, per-CPU queueing continues to be used, which introduces
significant additional overhead, especially for callback invocation,
which fans out a series of workqueue handlers.
This commit therefore switches back to single-queue operation if at the
beginning of a grace period there are very few callbacks. The definition
of "very few" is set by the rcupdate.rcu_task_collapse_lim module
parameter, which defaults to 10. This switch happens in two phases,
with the first phase causing future callbacks to be enqueued on CPU 0's
queue, but with all queues continuing to be checked for grace periods
and callback invocation. The second phase checks to see if an RCU grace
period has elapsed and if all remaining RCU-Tasks callbacks are queued
on CPU 0. If so, only CPU 0 is checked for future grace periods and
callback operation.
Of course, the return of contention anywhere during this process will
result in returning to per-CPU callback queueing.
Reported-by: Martin Lau <kafai@fb.com>
Cc: Neeraj Upadhyay <neeraj.iitr10@gmail.com>
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
2021-11-29 17:52:31 -07:00
|
|
|
static int rcu_task_collapse_lim __read_mostly = 10;
|
|
|
|
|
module_param(rcu_task_collapse_lim, int, 0444);
|
2023-06-26 13:45:13 -07:00
|
|
|
static int rcu_task_lazy_lim __read_mostly = 32;
|
|
|
|
|
module_param(rcu_task_lazy_lim, int, 0444);
|
2021-11-24 16:12:15 -07:00
|
|
|
|
2024-07-09 21:45:42 -07:00
|
|
|
static int rcu_task_cpu_ids;
|
|
|
|
|
|
2020-03-16 12:13:33 -07:00
|
|
|
/* RCU tasks grace-period state for debugging. */
|
|
|
|
|
#define RTGS_INIT 0
|
|
|
|
|
#define RTGS_WAIT_WAIT_CBS 1
|
|
|
|
|
#define RTGS_WAIT_GP 2
|
|
|
|
|
#define RTGS_PRE_WAIT_GP 3
|
|
|
|
|
#define RTGS_SCAN_TASKLIST 4
|
|
|
|
|
#define RTGS_POST_SCAN_TASKLIST 5
|
|
|
|
|
#define RTGS_WAIT_SCAN_HOLDOUTS 6
|
|
|
|
|
#define RTGS_SCAN_HOLDOUTS 7
|
|
|
|
|
#define RTGS_POST_GP 8
|
|
|
|
|
#define RTGS_WAIT_READERS 9
|
|
|
|
|
#define RTGS_INVOKE_CBS 10
|
|
|
|
|
#define RTGS_WAIT_CBS 11
|
2020-05-28 20:03:48 -07:00
|
|
|
#ifndef CONFIG_TINY_RCU
|
2020-03-16 12:13:33 -07:00
|
|
|
static const char * const rcu_tasks_gp_state_names[] = {
|
|
|
|
|
"RTGS_INIT",
|
|
|
|
|
"RTGS_WAIT_WAIT_CBS",
|
|
|
|
|
"RTGS_WAIT_GP",
|
|
|
|
|
"RTGS_PRE_WAIT_GP",
|
|
|
|
|
"RTGS_SCAN_TASKLIST",
|
|
|
|
|
"RTGS_POST_SCAN_TASKLIST",
|
|
|
|
|
"RTGS_WAIT_SCAN_HOLDOUTS",
|
|
|
|
|
"RTGS_SCAN_HOLDOUTS",
|
|
|
|
|
"RTGS_POST_GP",
|
|
|
|
|
"RTGS_WAIT_READERS",
|
|
|
|
|
"RTGS_INVOKE_CBS",
|
|
|
|
|
"RTGS_WAIT_CBS",
|
|
|
|
|
};
|
2020-05-28 20:03:48 -07:00
|
|
|
#endif /* #ifndef CONFIG_TINY_RCU */
|
2020-03-16 12:13:33 -07:00
|
|
|
|
2020-03-03 12:49:21 -07:00
|
|
|
////////////////////////////////////////////////////////////////////////
|
|
|
|
|
//
|
|
|
|
|
// Generic code.
|
|
|
|
|
|
2021-11-10 16:56:40 -07:00
|
|
|
static void rcu_tasks_invoke_cbs_wq(struct work_struct *wp);
|
|
|
|
|
|
2020-03-16 12:13:33 -07:00
|
|
|
/* Record grace-period phase and time. */
|
|
|
|
|
static void set_tasks_gp_state(struct rcu_tasks *rtp, int newstate)
|
|
|
|
|
{
|
|
|
|
|
rtp->gp_state = newstate;
|
|
|
|
|
rtp->gp_jiffies = jiffies;
|
|
|
|
|
}
|
|
|
|
|
|
2020-05-28 20:03:48 -07:00
|
|
|
#ifndef CONFIG_TINY_RCU
|
2020-03-16 12:13:33 -07:00
|
|
|
/* Return state name. */
|
|
|
|
|
static const char *tasks_gp_state_getname(struct rcu_tasks *rtp)
|
|
|
|
|
{
|
|
|
|
|
int i = data_race(rtp->gp_state); // Let KCSAN detect update races
|
|
|
|
|
int j = READ_ONCE(i); // Prevent the compiler from reading twice
|
|
|
|
|
|
|
|
|
|
if (j >= ARRAY_SIZE(rcu_tasks_gp_state_names))
|
|
|
|
|
return "???";
|
|
|
|
|
return rcu_tasks_gp_state_names[j];
|
|
|
|
|
}
|
2020-05-28 20:03:48 -07:00
|
|
|
#endif /* #ifndef CONFIG_TINY_RCU */
|
2020-03-16 12:13:33 -07:00
|
|
|
|
2021-11-05 21:52:00 -07:00
|
|
|
// Initialize per-CPU callback lists for the specified flavor of
|
2023-07-28 13:33:02 -07:00
|
|
|
// Tasks RCU. Do not enqueue callbacks before this function is invoked.
|
2021-11-05 21:52:00 -07:00
|
|
|
static void cblist_init_generic(struct rcu_tasks *rtp)
|
|
|
|
|
{
|
|
|
|
|
int cpu;
|
2021-11-12 08:33:40 -07:00
|
|
|
int lim;
|
2022-01-26 11:42:58 -07:00
|
|
|
int shift;
|
2024-07-09 21:45:42 -07:00
|
|
|
int maxcpu;
|
|
|
|
|
int index = 0;
|
2021-11-05 21:52:00 -07:00
|
|
|
|
2021-11-24 16:12:15 -07:00
|
|
|
if (rcu_task_enqueue_lim < 0) {
|
|
|
|
|
rcu_task_enqueue_lim = 1;
|
|
|
|
|
rcu_task_cb_adjust = true;
|
|
|
|
|
} else if (rcu_task_enqueue_lim == 0) {
|
2021-11-12 08:33:40 -07:00
|
|
|
rcu_task_enqueue_lim = 1;
|
2021-11-24 16:12:15 -07:00
|
|
|
}
|
2021-11-12 08:33:40 -07:00
|
|
|
lim = rcu_task_enqueue_lim;
|
|
|
|
|
|
2024-07-09 21:45:42 -07:00
|
|
|
rtp->rtpcp_array = kcalloc(num_possible_cpus(), sizeof(struct rcu_tasks_percpu *), GFP_KERNEL);
|
|
|
|
|
BUG_ON(!rtp->rtpcp_array);
|
|
|
|
|
|
2021-11-05 21:52:00 -07:00
|
|
|
for_each_possible_cpu(cpu) {
|
|
|
|
|
struct rcu_tasks_percpu *rtpcp = per_cpu_ptr(rtp->rtpcpu, cpu);
|
|
|
|
|
|
|
|
|
|
WARN_ON_ONCE(!rtpcp);
|
|
|
|
|
if (cpu)
|
2021-11-08 17:52:02 -07:00
|
|
|
raw_spin_lock_init(&ACCESS_PRIVATE(rtpcp, lock));
|
2021-11-08 17:18:57 -07:00
|
|
|
if (rcu_segcblist_empty(&rtpcp->cblist))
|
|
|
|
|
rcu_segcblist_init(&rtpcp->cblist);
|
2021-11-10 16:56:40 -07:00
|
|
|
INIT_WORK(&rtpcp->rtp_work, rcu_tasks_invoke_cbs_wq);
|
|
|
|
|
rtpcp->cpu = cpu;
|
|
|
|
|
rtpcp->rtpp = rtp;
|
2024-07-09 21:45:42 -07:00
|
|
|
rtpcp->index = index;
|
|
|
|
|
rtp->rtpcp_array[index] = rtpcp;
|
|
|
|
|
index++;
|
2022-05-16 17:56:16 -07:00
|
|
|
if (!rtpcp->rtp_blkd_tasks.next)
|
|
|
|
|
INIT_LIST_HEAD(&rtpcp->rtp_blkd_tasks);
|
2024-02-05 14:10:19 -07:00
|
|
|
if (!rtpcp->rtp_exit_list.next)
|
|
|
|
|
INIT_LIST_HEAD(&rtpcp->rtp_exit_list);
|
2024-08-01 17:34:24 -07:00
|
|
|
rtpcp->barrier_q_head.next = &rtpcp->barrier_q_head;
|
2024-07-09 21:45:42 -07:00
|
|
|
maxcpu = cpu;
|
2021-11-05 21:52:00 -07:00
|
|
|
}
|
2022-08-02 09:22:05 -07:00
|
|
|
|
2024-07-09 21:45:42 -07:00
|
|
|
rcu_task_cpu_ids = maxcpu + 1;
|
|
|
|
|
if (lim > rcu_task_cpu_ids)
|
|
|
|
|
lim = rcu_task_cpu_ids;
|
|
|
|
|
shift = ilog2(rcu_task_cpu_ids / lim);
|
|
|
|
|
if (((rcu_task_cpu_ids - 1) >> shift) >= lim)
|
|
|
|
|
shift++;
|
|
|
|
|
WRITE_ONCE(rtp->percpu_enqueue_shift, shift);
|
|
|
|
|
WRITE_ONCE(rtp->percpu_dequeue_lim, lim);
|
|
|
|
|
smp_store_release(&rtp->percpu_enqueue_lim, lim);
|
|
|
|
|
|
|
|
|
|
pr_info("%s: Setting shift to %d and lim to %d rcu_task_cb_adjust=%d rcu_task_cpu_ids=%d.\n",
|
|
|
|
|
rtp->name, data_race(rtp->percpu_enqueue_shift), data_race(rtp->percpu_enqueue_lim),
|
|
|
|
|
rcu_task_cb_adjust, rcu_task_cpu_ids);
|
2021-11-05 21:52:00 -07:00
|
|
|
}
|
|
|
|
|
|
rcu-tasks: Treat only synchronous grace periods urgently
The performance requirements on RCU Tasks, and in particular on RCU
Tasks Trace, have evolved over time as the workloads have evolved.
The current implementation is designed to provide low grace-period
latencies, and also to accommodate short-duration floods of callbacks.
However, current workloads can also provide a constant background
callback-queuing rate of a few hundred call_rcu_tasks_trace() invocations
per second. This results in continuous back-to-back RCU Tasks Trace
grace periods, which in turn can consume the better part of 10% of a CPU.
One could take the attitude that there are several tens of other CPUs on
the systems running such workloads, but energy efficiency is a thing.
On these systems, although asynchronous grace-period requests happen
every few milliseconds, synchronous grace-period requests are quite rare.
This commit therefore arrranges for grace periods to be initiated
immediately in response to calls to synchronize_rcu_tasks*() and
also to calls to synchronize_rcu_mult() that are passed one of the
call_rcu_tasks*() functions. These are recognized by the tell-tale
wakeme_after_rcu callback function.
In other cases, callbacks are gathered up for up to about 250 milliseconds
before a grace period is initiated. This results in more than an order of
magnitude reduction in RCU Tasks Trace grace periods, with corresponding
reduction in consumption of CPU time.
Reported-by: Alexei Starovoitov <ast@kernel.org>
Reported-by: Martin KaFai Lau <kafai@fb.com>
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
2023-05-14 18:06:12 -07:00
|
|
|
// Compute wakeup time for lazy callback timer.
|
|
|
|
|
static unsigned long rcu_tasks_lazy_time(struct rcu_tasks *rtp)
|
|
|
|
|
{
|
|
|
|
|
return jiffies + rtp->lazy_jiffies;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Timer handler that unlazifies lazy callbacks.
|
|
|
|
|
static void call_rcu_tasks_generic_timer(struct timer_list *tlp)
|
|
|
|
|
{
|
|
|
|
|
unsigned long flags;
|
|
|
|
|
bool needwake = false;
|
|
|
|
|
struct rcu_tasks *rtp;
|
|
|
|
|
struct rcu_tasks_percpu *rtpcp = from_timer(rtpcp, tlp, lazy_timer);
|
|
|
|
|
|
|
|
|
|
rtp = rtpcp->rtpp;
|
|
|
|
|
raw_spin_lock_irqsave_rcu_node(rtpcp, flags);
|
|
|
|
|
if (!rcu_segcblist_empty(&rtpcp->cblist) && rtp->lazy_jiffies) {
|
|
|
|
|
if (!rtpcp->urgent_gp)
|
|
|
|
|
rtpcp->urgent_gp = 1;
|
|
|
|
|
needwake = true;
|
|
|
|
|
mod_timer(&rtpcp->lazy_timer, rcu_tasks_lazy_time(rtp));
|
|
|
|
|
}
|
|
|
|
|
raw_spin_unlock_irqrestore_rcu_node(rtpcp, flags);
|
|
|
|
|
if (needwake)
|
|
|
|
|
rcuwait_wake_up(&rtp->cbs_wait);
|
|
|
|
|
}
|
|
|
|
|
|
2021-11-23 17:16:50 -07:00
|
|
|
// IRQ-work handler that does deferred wakeup for call_rcu_tasks_generic().
|
|
|
|
|
static void call_rcu_tasks_iw_wakeup(struct irq_work *iwp)
|
|
|
|
|
{
|
|
|
|
|
struct rcu_tasks *rtp;
|
|
|
|
|
struct rcu_tasks_percpu *rtpcp = container_of(iwp, struct rcu_tasks_percpu, rtp_irq_work);
|
|
|
|
|
|
|
|
|
|
rtp = rtpcp->rtpp;
|
2022-03-04 04:07:25 -07:00
|
|
|
rcuwait_wake_up(&rtp->cbs_wait);
|
2021-11-23 17:16:50 -07:00
|
|
|
}
|
|
|
|
|
|
2020-03-03 12:49:21 -07:00
|
|
|
// Enqueue a callback for the specified flavor of Tasks RCU.
|
|
|
|
|
static void call_rcu_tasks_generic(struct rcu_head *rhp, rcu_callback_t func,
|
|
|
|
|
struct rcu_tasks *rtp)
|
2020-03-02 12:59:20 -07:00
|
|
|
{
|
rcu-tasks: Handle sparse cpu_possible_mask
If the rcupdate.rcu_task_enqueue_lim kernel boot parameter is set to
something greater than 1 and less than nr_cpu_ids, the code attempts to
use a subset of the CPU's RCU Tasks callback lists. This works, but only
if the cpu_possible_mask is contiguous. If there are "holes" in this
mask, the callback-enqueue code might attempt to access a non-existent
per-CPU ->rtcpu variable for a non-existent CPU. For example, if only
CPUs 0, 4, 8, 12, 16 and so on are in cpu_possible_mask, specifying
rcupdate.rcu_task_enqueue_lim=4 would cause the code to attempt to
use callback queues for non-existent CPUs 1, 2, and 3. Because such
systems have existed in the past and might still exist, the code needs
to gracefully handle this situation.
This commit therefore checks to see whether the desired CPU is present
in cpu_possible_mask, and, if not, searches for the next CPU. This means
that the systems administrator of a system with a sparse cpu_possible_mask
will need to account for this sparsity when specifying the value of
the rcupdate.rcu_task_enqueue_lim kernel boot parameter. For example,
setting this parameter to the value 4 will use only CPUs 0 and 4, which
CPU 4 getting three times the callback load of CPU 0.
This commit assumes that bit (nr_cpu_ids - 1) is always set in
cpu_possible_mask.
Link: https://lore.kernel.org/lkml/CANn89iKaNEwyNZ=L_PQnkH0LP_XjLYrr_dpyRKNNoDJaWKdrmg@mail.gmail.com/
Signed-off-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
2022-04-04 12:30:18 -07:00
|
|
|
int chosen_cpu;
|
2020-03-02 12:59:20 -07:00
|
|
|
unsigned long flags;
|
rcu-tasks: Treat only synchronous grace periods urgently
The performance requirements on RCU Tasks, and in particular on RCU
Tasks Trace, have evolved over time as the workloads have evolved.
The current implementation is designed to provide low grace-period
latencies, and also to accommodate short-duration floods of callbacks.
However, current workloads can also provide a constant background
callback-queuing rate of a few hundred call_rcu_tasks_trace() invocations
per second. This results in continuous back-to-back RCU Tasks Trace
grace periods, which in turn can consume the better part of 10% of a CPU.
One could take the attitude that there are several tens of other CPUs on
the systems running such workloads, but energy efficiency is a thing.
On these systems, although asynchronous grace-period requests happen
every few milliseconds, synchronous grace-period requests are quite rare.
This commit therefore arrranges for grace periods to be initiated
immediately in response to calls to synchronize_rcu_tasks*() and
also to calls to synchronize_rcu_mult() that are passed one of the
call_rcu_tasks*() functions. These are recognized by the tell-tale
wakeme_after_rcu callback function.
In other cases, callbacks are gathered up for up to about 250 milliseconds
before a grace period is initiated. This results in more than an order of
magnitude reduction in RCU Tasks Trace grace periods, with corresponding
reduction in consumption of CPU time.
Reported-by: Alexei Starovoitov <ast@kernel.org>
Reported-by: Martin KaFai Lau <kafai@fb.com>
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
2023-05-14 18:06:12 -07:00
|
|
|
bool havekthread = smp_load_acquire(&rtp->kthread_ptr);
|
rcu-tasks: Handle sparse cpu_possible_mask
If the rcupdate.rcu_task_enqueue_lim kernel boot parameter is set to
something greater than 1 and less than nr_cpu_ids, the code attempts to
use a subset of the CPU's RCU Tasks callback lists. This works, but only
if the cpu_possible_mask is contiguous. If there are "holes" in this
mask, the callback-enqueue code might attempt to access a non-existent
per-CPU ->rtcpu variable for a non-existent CPU. For example, if only
CPUs 0, 4, 8, 12, 16 and so on are in cpu_possible_mask, specifying
rcupdate.rcu_task_enqueue_lim=4 would cause the code to attempt to
use callback queues for non-existent CPUs 1, 2, and 3. Because such
systems have existed in the past and might still exist, the code needs
to gracefully handle this situation.
This commit therefore checks to see whether the desired CPU is present
in cpu_possible_mask, and, if not, searches for the next CPU. This means
that the systems administrator of a system with a sparse cpu_possible_mask
will need to account for this sparsity when specifying the value of
the rcupdate.rcu_task_enqueue_lim kernel boot parameter. For example,
setting this parameter to the value 4 will use only CPUs 0 and 4, which
CPU 4 getting three times the callback load of CPU 0.
This commit assumes that bit (nr_cpu_ids - 1) is always set in
cpu_possible_mask.
Link: https://lore.kernel.org/lkml/CANn89iKaNEwyNZ=L_PQnkH0LP_XjLYrr_dpyRKNNoDJaWKdrmg@mail.gmail.com/
Signed-off-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
2022-04-04 12:30:18 -07:00
|
|
|
int ideal_cpu;
|
2021-11-22 14:38:42 -07:00
|
|
|
unsigned long j;
|
2021-11-24 16:12:15 -07:00
|
|
|
bool needadjust = false;
|
2020-03-02 12:59:20 -07:00
|
|
|
bool needwake;
|
2021-11-05 21:52:00 -07:00
|
|
|
struct rcu_tasks_percpu *rtpcp;
|
2020-03-02 12:59:20 -07:00
|
|
|
|
|
|
|
|
rhp->next = NULL;
|
|
|
|
|
rhp->func = func;
|
2021-11-05 21:52:00 -07:00
|
|
|
local_irq_save(flags);
|
rcu-tasks: Use fewer callbacks queues if callback flood ends
By default, when lock contention is encountered, the RCU Tasks flavors
of RCU switch to using per-CPU queueing. However, if the callback
flood ends, per-CPU queueing continues to be used, which introduces
significant additional overhead, especially for callback invocation,
which fans out a series of workqueue handlers.
This commit therefore switches back to single-queue operation if at the
beginning of a grace period there are very few callbacks. The definition
of "very few" is set by the rcupdate.rcu_task_collapse_lim module
parameter, which defaults to 10. This switch happens in two phases,
with the first phase causing future callbacks to be enqueued on CPU 0's
queue, but with all queues continuing to be checked for grace periods
and callback invocation. The second phase checks to see if an RCU grace
period has elapsed and if all remaining RCU-Tasks callbacks are queued
on CPU 0. If so, only CPU 0 is checked for future grace periods and
callback operation.
Of course, the return of contention anywhere during this process will
result in returning to per-CPU callback queueing.
Reported-by: Martin Lau <kafai@fb.com>
Cc: Neeraj Upadhyay <neeraj.iitr10@gmail.com>
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
2021-11-29 17:52:31 -07:00
|
|
|
rcu_read_lock();
|
rcu-tasks: Handle sparse cpu_possible_mask
If the rcupdate.rcu_task_enqueue_lim kernel boot parameter is set to
something greater than 1 and less than nr_cpu_ids, the code attempts to
use a subset of the CPU's RCU Tasks callback lists. This works, but only
if the cpu_possible_mask is contiguous. If there are "holes" in this
mask, the callback-enqueue code might attempt to access a non-existent
per-CPU ->rtcpu variable for a non-existent CPU. For example, if only
CPUs 0, 4, 8, 12, 16 and so on are in cpu_possible_mask, specifying
rcupdate.rcu_task_enqueue_lim=4 would cause the code to attempt to
use callback queues for non-existent CPUs 1, 2, and 3. Because such
systems have existed in the past and might still exist, the code needs
to gracefully handle this situation.
This commit therefore checks to see whether the desired CPU is present
in cpu_possible_mask, and, if not, searches for the next CPU. This means
that the systems administrator of a system with a sparse cpu_possible_mask
will need to account for this sparsity when specifying the value of
the rcupdate.rcu_task_enqueue_lim kernel boot parameter. For example,
setting this parameter to the value 4 will use only CPUs 0 and 4, which
CPU 4 getting three times the callback load of CPU 0.
This commit assumes that bit (nr_cpu_ids - 1) is always set in
cpu_possible_mask.
Link: https://lore.kernel.org/lkml/CANn89iKaNEwyNZ=L_PQnkH0LP_XjLYrr_dpyRKNNoDJaWKdrmg@mail.gmail.com/
Signed-off-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
2022-04-04 12:30:18 -07:00
|
|
|
ideal_cpu = smp_processor_id() >> READ_ONCE(rtp->percpu_enqueue_shift);
|
|
|
|
|
chosen_cpu = cpumask_next(ideal_cpu - 1, cpu_possible_mask);
|
2024-08-01 17:34:21 -07:00
|
|
|
WARN_ON_ONCE(chosen_cpu >= rcu_task_cpu_ids);
|
rcu-tasks: Handle sparse cpu_possible_mask
If the rcupdate.rcu_task_enqueue_lim kernel boot parameter is set to
something greater than 1 and less than nr_cpu_ids, the code attempts to
use a subset of the CPU's RCU Tasks callback lists. This works, but only
if the cpu_possible_mask is contiguous. If there are "holes" in this
mask, the callback-enqueue code might attempt to access a non-existent
per-CPU ->rtcpu variable for a non-existent CPU. For example, if only
CPUs 0, 4, 8, 12, 16 and so on are in cpu_possible_mask, specifying
rcupdate.rcu_task_enqueue_lim=4 would cause the code to attempt to
use callback queues for non-existent CPUs 1, 2, and 3. Because such
systems have existed in the past and might still exist, the code needs
to gracefully handle this situation.
This commit therefore checks to see whether the desired CPU is present
in cpu_possible_mask, and, if not, searches for the next CPU. This means
that the systems administrator of a system with a sparse cpu_possible_mask
will need to account for this sparsity when specifying the value of
the rcupdate.rcu_task_enqueue_lim kernel boot parameter. For example,
setting this parameter to the value 4 will use only CPUs 0 and 4, which
CPU 4 getting three times the callback load of CPU 0.
This commit assumes that bit (nr_cpu_ids - 1) is always set in
cpu_possible_mask.
Link: https://lore.kernel.org/lkml/CANn89iKaNEwyNZ=L_PQnkH0LP_XjLYrr_dpyRKNNoDJaWKdrmg@mail.gmail.com/
Signed-off-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
2022-04-04 12:30:18 -07:00
|
|
|
rtpcp = per_cpu_ptr(rtp->rtpcpu, chosen_cpu);
|
2021-11-22 14:38:42 -07:00
|
|
|
if (!raw_spin_trylock_rcu_node(rtpcp)) { // irqs already disabled.
|
|
|
|
|
raw_spin_lock_rcu_node(rtpcp); // irqs already disabled.
|
|
|
|
|
j = jiffies;
|
|
|
|
|
if (rtpcp->rtp_jiffies != j) {
|
|
|
|
|
rtpcp->rtp_jiffies = j;
|
|
|
|
|
rtpcp->rtp_n_lock_retries = 0;
|
|
|
|
|
}
|
2021-11-24 16:12:15 -07:00
|
|
|
if (rcu_task_cb_adjust && ++rtpcp->rtp_n_lock_retries > rcu_task_contend_lim &&
|
2024-07-09 21:45:42 -07:00
|
|
|
READ_ONCE(rtp->percpu_enqueue_lim) != rcu_task_cpu_ids)
|
2021-11-24 16:12:15 -07:00
|
|
|
needadjust = true; // Defer adjustment to avoid deadlock.
|
2021-11-22 14:38:42 -07:00
|
|
|
}
|
2023-07-28 13:33:02 -07:00
|
|
|
// Queuing callbacks before initialization not yet supported.
|
|
|
|
|
if (WARN_ON_ONCE(!rcu_segcblist_is_enabled(&rtpcp->cblist)))
|
|
|
|
|
rcu_segcblist_init(&rtpcp->cblist);
|
2023-06-26 13:45:13 -07:00
|
|
|
needwake = (func == wakeme_after_rcu) ||
|
|
|
|
|
(rcu_segcblist_n_cbs(&rtpcp->cblist) == rcu_task_lazy_lim);
|
|
|
|
|
if (havekthread && !needwake && !timer_pending(&rtpcp->lazy_timer)) {
|
rcu-tasks: Treat only synchronous grace periods urgently
The performance requirements on RCU Tasks, and in particular on RCU
Tasks Trace, have evolved over time as the workloads have evolved.
The current implementation is designed to provide low grace-period
latencies, and also to accommodate short-duration floods of callbacks.
However, current workloads can also provide a constant background
callback-queuing rate of a few hundred call_rcu_tasks_trace() invocations
per second. This results in continuous back-to-back RCU Tasks Trace
grace periods, which in turn can consume the better part of 10% of a CPU.
One could take the attitude that there are several tens of other CPUs on
the systems running such workloads, but energy efficiency is a thing.
On these systems, although asynchronous grace-period requests happen
every few milliseconds, synchronous grace-period requests are quite rare.
This commit therefore arrranges for grace periods to be initiated
immediately in response to calls to synchronize_rcu_tasks*() and
also to calls to synchronize_rcu_mult() that are passed one of the
call_rcu_tasks*() functions. These are recognized by the tell-tale
wakeme_after_rcu callback function.
In other cases, callbacks are gathered up for up to about 250 milliseconds
before a grace period is initiated. This results in more than an order of
magnitude reduction in RCU Tasks Trace grace periods, with corresponding
reduction in consumption of CPU time.
Reported-by: Alexei Starovoitov <ast@kernel.org>
Reported-by: Martin KaFai Lau <kafai@fb.com>
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
2023-05-14 18:06:12 -07:00
|
|
|
if (rtp->lazy_jiffies)
|
|
|
|
|
mod_timer(&rtpcp->lazy_timer, rcu_tasks_lazy_time(rtp));
|
|
|
|
|
else
|
|
|
|
|
needwake = rcu_segcblist_empty(&rtpcp->cblist);
|
2021-11-05 21:52:00 -07:00
|
|
|
}
|
rcu-tasks: Treat only synchronous grace periods urgently
The performance requirements on RCU Tasks, and in particular on RCU
Tasks Trace, have evolved over time as the workloads have evolved.
The current implementation is designed to provide low grace-period
latencies, and also to accommodate short-duration floods of callbacks.
However, current workloads can also provide a constant background
callback-queuing rate of a few hundred call_rcu_tasks_trace() invocations
per second. This results in continuous back-to-back RCU Tasks Trace
grace periods, which in turn can consume the better part of 10% of a CPU.
One could take the attitude that there are several tens of other CPUs on
the systems running such workloads, but energy efficiency is a thing.
On these systems, although asynchronous grace-period requests happen
every few milliseconds, synchronous grace-period requests are quite rare.
This commit therefore arrranges for grace periods to be initiated
immediately in response to calls to synchronize_rcu_tasks*() and
also to calls to synchronize_rcu_mult() that are passed one of the
call_rcu_tasks*() functions. These are recognized by the tell-tale
wakeme_after_rcu callback function.
In other cases, callbacks are gathered up for up to about 250 milliseconds
before a grace period is initiated. This results in more than an order of
magnitude reduction in RCU Tasks Trace grace periods, with corresponding
reduction in consumption of CPU time.
Reported-by: Alexei Starovoitov <ast@kernel.org>
Reported-by: Martin KaFai Lau <kafai@fb.com>
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
2023-05-14 18:06:12 -07:00
|
|
|
if (needwake)
|
|
|
|
|
rtpcp->urgent_gp = 3;
|
2021-11-08 17:18:57 -07:00
|
|
|
rcu_segcblist_enqueue(&rtpcp->cblist, rhp);
|
2021-11-08 17:52:02 -07:00
|
|
|
raw_spin_unlock_irqrestore_rcu_node(rtpcp, flags);
|
2021-11-24 16:12:15 -07:00
|
|
|
if (unlikely(needadjust)) {
|
|
|
|
|
raw_spin_lock_irqsave(&rtp->cbs_gbl_lock, flags);
|
2024-07-09 21:45:42 -07:00
|
|
|
if (rtp->percpu_enqueue_lim != rcu_task_cpu_ids) {
|
2022-02-02 17:34:40 -07:00
|
|
|
WRITE_ONCE(rtp->percpu_enqueue_shift, 0);
|
2024-07-09 21:45:42 -07:00
|
|
|
WRITE_ONCE(rtp->percpu_dequeue_lim, rcu_task_cpu_ids);
|
|
|
|
|
smp_store_release(&rtp->percpu_enqueue_lim, rcu_task_cpu_ids);
|
2021-11-24 16:12:15 -07:00
|
|
|
pr_info("Switching %s to per-CPU callback queuing.\n", rtp->name);
|
|
|
|
|
}
|
|
|
|
|
raw_spin_unlock_irqrestore(&rtp->cbs_gbl_lock, flags);
|
|
|
|
|
}
|
rcu-tasks: Use fewer callbacks queues if callback flood ends
By default, when lock contention is encountered, the RCU Tasks flavors
of RCU switch to using per-CPU queueing. However, if the callback
flood ends, per-CPU queueing continues to be used, which introduces
significant additional overhead, especially for callback invocation,
which fans out a series of workqueue handlers.
This commit therefore switches back to single-queue operation if at the
beginning of a grace period there are very few callbacks. The definition
of "very few" is set by the rcupdate.rcu_task_collapse_lim module
parameter, which defaults to 10. This switch happens in two phases,
with the first phase causing future callbacks to be enqueued on CPU 0's
queue, but with all queues continuing to be checked for grace periods
and callback invocation. The second phase checks to see if an RCU grace
period has elapsed and if all remaining RCU-Tasks callbacks are queued
on CPU 0. If so, only CPU 0 is checked for future grace periods and
callback operation.
Of course, the return of contention anywhere during this process will
result in returning to per-CPU callback queueing.
Reported-by: Martin Lau <kafai@fb.com>
Cc: Neeraj Upadhyay <neeraj.iitr10@gmail.com>
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
2021-11-29 17:52:31 -07:00
|
|
|
rcu_read_unlock();
|
2020-03-02 12:59:20 -07:00
|
|
|
/* We can't create the thread unless interrupts are enabled. */
|
2020-03-02 16:16:57 -07:00
|
|
|
if (needwake && READ_ONCE(rtp->kthread_ptr))
|
2021-11-23 17:16:50 -07:00
|
|
|
irq_work_queue(&rtpcp->rtp_irq_work);
|
2020-03-02 12:59:20 -07:00
|
|
|
}
|
|
|
|
|
|
2021-11-11 15:53:43 -07:00
|
|
|
// RCU callback function for rcu_barrier_tasks_generic().
|
|
|
|
|
static void rcu_barrier_tasks_generic_cb(struct rcu_head *rhp)
|
|
|
|
|
{
|
|
|
|
|
struct rcu_tasks *rtp;
|
|
|
|
|
struct rcu_tasks_percpu *rtpcp;
|
|
|
|
|
|
2024-08-01 17:34:24 -07:00
|
|
|
rhp->next = rhp; // Mark the callback as having been invoked.
|
2021-11-11 15:53:43 -07:00
|
|
|
rtpcp = container_of(rhp, struct rcu_tasks_percpu, barrier_q_head);
|
|
|
|
|
rtp = rtpcp->rtpp;
|
|
|
|
|
if (atomic_dec_and_test(&rtp->barrier_q_count))
|
|
|
|
|
complete(&rtp->barrier_q_completion);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Wait for all in-flight callbacks for the specified RCU Tasks flavor.
|
|
|
|
|
// Operates in a manner similar to rcu_barrier().
|
2024-07-03 19:47:47 -07:00
|
|
|
static void __maybe_unused rcu_barrier_tasks_generic(struct rcu_tasks *rtp)
|
2021-11-11 15:53:43 -07:00
|
|
|
{
|
|
|
|
|
int cpu;
|
|
|
|
|
unsigned long flags;
|
|
|
|
|
struct rcu_tasks_percpu *rtpcp;
|
|
|
|
|
unsigned long s = rcu_seq_snap(&rtp->barrier_q_seq);
|
|
|
|
|
|
|
|
|
|
mutex_lock(&rtp->barrier_q_mutex);
|
|
|
|
|
if (rcu_seq_done(&rtp->barrier_q_seq, s)) {
|
|
|
|
|
smp_mb();
|
|
|
|
|
mutex_unlock(&rtp->barrier_q_mutex);
|
|
|
|
|
return;
|
|
|
|
|
}
|
2024-08-01 17:34:26 -07:00
|
|
|
rtp->barrier_q_start = jiffies;
|
2021-11-11 15:53:43 -07:00
|
|
|
rcu_seq_start(&rtp->barrier_q_seq);
|
|
|
|
|
init_completion(&rtp->barrier_q_completion);
|
|
|
|
|
atomic_set(&rtp->barrier_q_count, 2);
|
|
|
|
|
for_each_possible_cpu(cpu) {
|
2021-11-29 12:46:33 -07:00
|
|
|
if (cpu >= smp_load_acquire(&rtp->percpu_dequeue_lim))
|
2021-11-11 15:53:43 -07:00
|
|
|
break;
|
|
|
|
|
rtpcp = per_cpu_ptr(rtp->rtpcpu, cpu);
|
|
|
|
|
rtpcp->barrier_q_head.func = rcu_barrier_tasks_generic_cb;
|
|
|
|
|
raw_spin_lock_irqsave_rcu_node(rtpcp, flags);
|
|
|
|
|
if (rcu_segcblist_entrain(&rtpcp->cblist, &rtpcp->barrier_q_head))
|
|
|
|
|
atomic_inc(&rtp->barrier_q_count);
|
|
|
|
|
raw_spin_unlock_irqrestore_rcu_node(rtpcp, flags);
|
|
|
|
|
}
|
|
|
|
|
if (atomic_sub_and_test(2, &rtp->barrier_q_count))
|
|
|
|
|
complete(&rtp->barrier_q_completion);
|
|
|
|
|
wait_for_completion(&rtp->barrier_q_completion);
|
|
|
|
|
rcu_seq_end(&rtp->barrier_q_seq);
|
|
|
|
|
mutex_unlock(&rtp->barrier_q_mutex);
|
|
|
|
|
}
|
|
|
|
|
|
2021-11-09 14:37:34 -07:00
|
|
|
// Advance callbacks and indicate whether either a grace period or
|
|
|
|
|
// callback invocation is needed.
|
|
|
|
|
static int rcu_tasks_need_gpcb(struct rcu_tasks *rtp)
|
|
|
|
|
{
|
|
|
|
|
int cpu;
|
rcu-tasks: Pull sampling of ->percpu_dequeue_lim out of loop
The rcu_tasks_need_gpcb() samples ->percpu_dequeue_lim as part of the
condition clause of a "for" loop, which is a bit confusing. This commit
therefore hoists this sampling out of the loop, using the result loaded
in the condition clause.
So why does this work in the face of a concurrent switch from single-CPU
queueing to per-CPU queueing?
o The call_rcu_tasks_generic() that makes the change has already
enqueued its callback, which means that all of the other CPU's
callback queues are empty.
o For the call_rcu_tasks_generic() that first notices
the switch to per-CPU queues, the smp_store_release()
used to update ->percpu_enqueue_lim pairs with the
raw_spin_trylock_rcu_node()'s full barrier that is
between the READ_ONCE(rtp->percpu_enqueue_shift) and the
rcu_segcblist_enqueue() that enqueues the callback.
o Because this CPU's queue is empty (unless it happens to
be the original single queue, in which case there is no
need for synchronization), this call_rcu_tasks_generic()
will do an irq_work_queue() to schedule a handler for the
needed rcuwait_wake_up() call. This call will be ordered
after the first call_rcu_tasks_generic() function's change to
->percpu_dequeue_lim.
o This rcuwait_wake_up() will either happen before or after the
set_current_state() in rcuwait_wait_event(). If it happens
before, the "condition" argument's call to rcu_tasks_need_gpcb()
will be ordered after the original change, and all callbacks on
all CPUs will be visible. Otherwise, if it happens after, then
the grace-period kthread's state will be set back to running,
which will result in a later call to rcuwait_wait_event() and
thus to rcu_tasks_need_gpcb(), which will again see the change.
So it all works out.
Suggested-by: Linus Torvalds <torvalds@linux-foundation.org>
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
Signed-off-by: Frederic Weisbecker <frederic@kernel.org>
2023-08-02 13:42:00 -07:00
|
|
|
int dequeue_limit;
|
2021-11-09 14:37:34 -07:00
|
|
|
unsigned long flags;
|
rcu-tasks: Handle queue-shrink/callback-enqueue race condition
The rcu_tasks_need_gpcb() determines whether or not: (1) There are
callbacks needing another grace period, (2) There are callbacks ready
to be invoked, and (3) It would be a good time to shrink back down to a
single-CPU callback list. This third case is interesting because some
other CPU might be adding new callbacks, which might suddenly make this
a very bad time to be shrinking.
This is currently handled by requiring call_rcu_tasks_generic() to
enqueue callbacks under the protection of rcu_read_lock() and requiring
rcu_tasks_need_gpcb() to wait for an RCU grace period to elapse before
finalizing the transition. This works well in practice.
Unfortunately, the current code assumes that a grace period whose end is
detected by the poll_state_synchronize_rcu() in the second "if" condition
actually ended before the earlier code counted the callbacks queued on
CPUs other than CPU 0 (local variable "ncbsnz"). Given the current code,
it is possible that a long-delayed call_rcu_tasks_generic() invocation
will queue a callback on a non-zero CPU after these CPUs have had their
callbacks counted and zero has been stored to ncbsnz. Such a callback
would trigger the WARN_ON_ONCE() in the second "if" statement.
To see this, consider the following sequence of events:
o CPU 0 invokes rcu_tasks_one_gp(), and counts fewer than
rcu_task_collapse_lim callbacks. It sees at least one
callback queued on some other CPU, thus setting ncbsnz
to a non-zero value.
o CPU 1 invokes call_rcu_tasks_generic() and loads 42 from
->percpu_enqueue_lim. It therefore decides to enqueue its
callback onto CPU 1's callback list, but is delayed.
o CPU 0 sees the rcu_task_cb_adjust is non-zero and that the number
of callbacks does not exceed rcu_task_collapse_lim. It therefore
checks percpu_enqueue_lim, and sees that its value is greater
than the value one. CPU 0 therefore starts the shift back
to a single callback list. It sets ->percpu_enqueue_lim to 1,
but CPU 1 has already read the old value of 42. It also gets
a grace-period state value from get_state_synchronize_rcu().
o CPU 0 sees that ncbsnz is non-zero in its second "if" statement,
so it declines to finalize the shrink operation.
o CPU 0 again invokes rcu_tasks_one_gp(), and counts fewer than
rcu_task_collapse_lim callbacks. It also sees that there are
no callback queued on any other CPU, and thus sets ncbsnz to zero.
o CPU 1 resumes execution and enqueues its callback onto its own
list. This invalidates the value of ncbsnz.
o CPU 0 sees the rcu_task_cb_adjust is non-zero and that the number
of callbacks does not exceed rcu_task_collapse_lim. It therefore
checks percpu_enqueue_lim, but sees that its value is already
unity. It therefore does not get a new grace-period state value.
o CPU 0 sees that rcu_task_cb_adjust is non-zero, ncbsnz is zero,
and that poll_state_synchronize_rcu() says that the grace period
has completed. it therefore finalizes the shrink operation,
setting ->percpu_dequeue_lim to the value one.
o CPU 0 does a debug check, scanning the other CPUs' callback lists.
It sees that CPU 1's list has a callback, so it (rightly)
triggers the WARN_ON_ONCE(). After all, the new value of
->percpu_dequeue_lim says to not bother looking at CPU 1's
callback list, which means that this callback will never be
invoked. This can result in hangs and maybe even OOMs.
Based on long experience with rcutorture, this is an extremely
low-probability race condition, but it really can happen, especially in
preemptible kernels or within guest OSes.
This commit therefore checks for completion of the grace period
before counting callbacks. With this change, in the above failure
scenario CPU 0 would know not to prematurely end the shrink operation
because the grace period would not have completed before the count
operation started.
[ paulmck: Adjust grace-period end rather than adding RCU reader. ]
[ paulmck: Avoid spurious WARN_ON_ONCE() with ->percpu_dequeue_lim check. ]
Signed-off-by: Zqiang <qiang1.zhang@intel.com>
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
2022-12-02 19:25:03 -07:00
|
|
|
bool gpdone = poll_state_synchronize_rcu(rtp->percpu_dequeue_gpseq);
|
rcu-tasks: Use fewer callbacks queues if callback flood ends
By default, when lock contention is encountered, the RCU Tasks flavors
of RCU switch to using per-CPU queueing. However, if the callback
flood ends, per-CPU queueing continues to be used, which introduces
significant additional overhead, especially for callback invocation,
which fans out a series of workqueue handlers.
This commit therefore switches back to single-queue operation if at the
beginning of a grace period there are very few callbacks. The definition
of "very few" is set by the rcupdate.rcu_task_collapse_lim module
parameter, which defaults to 10. This switch happens in two phases,
with the first phase causing future callbacks to be enqueued on CPU 0's
queue, but with all queues continuing to be checked for grace periods
and callback invocation. The second phase checks to see if an RCU grace
period has elapsed and if all remaining RCU-Tasks callbacks are queued
on CPU 0. If so, only CPU 0 is checked for future grace periods and
callback operation.
Of course, the return of contention anywhere during this process will
result in returning to per-CPU callback queueing.
Reported-by: Martin Lau <kafai@fb.com>
Cc: Neeraj Upadhyay <neeraj.iitr10@gmail.com>
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
2021-11-29 17:52:31 -07:00
|
|
|
long n;
|
|
|
|
|
long ncbs = 0;
|
|
|
|
|
long ncbsnz = 0;
|
2021-11-09 14:37:34 -07:00
|
|
|
int needgpcb = 0;
|
|
|
|
|
|
rcu-tasks: Pull sampling of ->percpu_dequeue_lim out of loop
The rcu_tasks_need_gpcb() samples ->percpu_dequeue_lim as part of the
condition clause of a "for" loop, which is a bit confusing. This commit
therefore hoists this sampling out of the loop, using the result loaded
in the condition clause.
So why does this work in the face of a concurrent switch from single-CPU
queueing to per-CPU queueing?
o The call_rcu_tasks_generic() that makes the change has already
enqueued its callback, which means that all of the other CPU's
callback queues are empty.
o For the call_rcu_tasks_generic() that first notices
the switch to per-CPU queues, the smp_store_release()
used to update ->percpu_enqueue_lim pairs with the
raw_spin_trylock_rcu_node()'s full barrier that is
between the READ_ONCE(rtp->percpu_enqueue_shift) and the
rcu_segcblist_enqueue() that enqueues the callback.
o Because this CPU's queue is empty (unless it happens to
be the original single queue, in which case there is no
need for synchronization), this call_rcu_tasks_generic()
will do an irq_work_queue() to schedule a handler for the
needed rcuwait_wake_up() call. This call will be ordered
after the first call_rcu_tasks_generic() function's change to
->percpu_dequeue_lim.
o This rcuwait_wake_up() will either happen before or after the
set_current_state() in rcuwait_wait_event(). If it happens
before, the "condition" argument's call to rcu_tasks_need_gpcb()
will be ordered after the original change, and all callbacks on
all CPUs will be visible. Otherwise, if it happens after, then
the grace-period kthread's state will be set back to running,
which will result in a later call to rcuwait_wait_event() and
thus to rcu_tasks_need_gpcb(), which will again see the change.
So it all works out.
Suggested-by: Linus Torvalds <torvalds@linux-foundation.org>
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
Signed-off-by: Frederic Weisbecker <frederic@kernel.org>
2023-08-02 13:42:00 -07:00
|
|
|
dequeue_limit = smp_load_acquire(&rtp->percpu_dequeue_lim);
|
|
|
|
|
for (cpu = 0; cpu < dequeue_limit; cpu++) {
|
2024-07-09 21:45:42 -07:00
|
|
|
if (!cpu_possible(cpu))
|
|
|
|
|
continue;
|
2021-11-09 14:37:34 -07:00
|
|
|
struct rcu_tasks_percpu *rtpcp = per_cpu_ptr(rtp->rtpcpu, cpu);
|
|
|
|
|
|
|
|
|
|
/* Advance and accelerate any new callbacks. */
|
rcu-tasks: Use fewer callbacks queues if callback flood ends
By default, when lock contention is encountered, the RCU Tasks flavors
of RCU switch to using per-CPU queueing. However, if the callback
flood ends, per-CPU queueing continues to be used, which introduces
significant additional overhead, especially for callback invocation,
which fans out a series of workqueue handlers.
This commit therefore switches back to single-queue operation if at the
beginning of a grace period there are very few callbacks. The definition
of "very few" is set by the rcupdate.rcu_task_collapse_lim module
parameter, which defaults to 10. This switch happens in two phases,
with the first phase causing future callbacks to be enqueued on CPU 0's
queue, but with all queues continuing to be checked for grace periods
and callback invocation. The second phase checks to see if an RCU grace
period has elapsed and if all remaining RCU-Tasks callbacks are queued
on CPU 0. If so, only CPU 0 is checked for future grace periods and
callback operation.
Of course, the return of contention anywhere during this process will
result in returning to per-CPU callback queueing.
Reported-by: Martin Lau <kafai@fb.com>
Cc: Neeraj Upadhyay <neeraj.iitr10@gmail.com>
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
2021-11-29 17:52:31 -07:00
|
|
|
if (!rcu_segcblist_n_cbs(&rtpcp->cblist))
|
2021-11-09 14:37:34 -07:00
|
|
|
continue;
|
|
|
|
|
raw_spin_lock_irqsave_rcu_node(rtpcp, flags);
|
rcu-tasks: Use fewer callbacks queues if callback flood ends
By default, when lock contention is encountered, the RCU Tasks flavors
of RCU switch to using per-CPU queueing. However, if the callback
flood ends, per-CPU queueing continues to be used, which introduces
significant additional overhead, especially for callback invocation,
which fans out a series of workqueue handlers.
This commit therefore switches back to single-queue operation if at the
beginning of a grace period there are very few callbacks. The definition
of "very few" is set by the rcupdate.rcu_task_collapse_lim module
parameter, which defaults to 10. This switch happens in two phases,
with the first phase causing future callbacks to be enqueued on CPU 0's
queue, but with all queues continuing to be checked for grace periods
and callback invocation. The second phase checks to see if an RCU grace
period has elapsed and if all remaining RCU-Tasks callbacks are queued
on CPU 0. If so, only CPU 0 is checked for future grace periods and
callback operation.
Of course, the return of contention anywhere during this process will
result in returning to per-CPU callback queueing.
Reported-by: Martin Lau <kafai@fb.com>
Cc: Neeraj Upadhyay <neeraj.iitr10@gmail.com>
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
2021-11-29 17:52:31 -07:00
|
|
|
// Should we shrink down to a single callback queue?
|
|
|
|
|
n = rcu_segcblist_n_cbs(&rtpcp->cblist);
|
|
|
|
|
if (n) {
|
|
|
|
|
ncbs += n;
|
|
|
|
|
if (cpu > 0)
|
|
|
|
|
ncbsnz += n;
|
|
|
|
|
}
|
2021-11-09 14:37:34 -07:00
|
|
|
rcu_segcblist_advance(&rtpcp->cblist, rcu_seq_current(&rtp->tasks_gp_seq));
|
|
|
|
|
(void)rcu_segcblist_accelerate(&rtpcp->cblist, rcu_seq_snap(&rtp->tasks_gp_seq));
|
rcu-tasks: Treat only synchronous grace periods urgently
The performance requirements on RCU Tasks, and in particular on RCU
Tasks Trace, have evolved over time as the workloads have evolved.
The current implementation is designed to provide low grace-period
latencies, and also to accommodate short-duration floods of callbacks.
However, current workloads can also provide a constant background
callback-queuing rate of a few hundred call_rcu_tasks_trace() invocations
per second. This results in continuous back-to-back RCU Tasks Trace
grace periods, which in turn can consume the better part of 10% of a CPU.
One could take the attitude that there are several tens of other CPUs on
the systems running such workloads, but energy efficiency is a thing.
On these systems, although asynchronous grace-period requests happen
every few milliseconds, synchronous grace-period requests are quite rare.
This commit therefore arrranges for grace periods to be initiated
immediately in response to calls to synchronize_rcu_tasks*() and
also to calls to synchronize_rcu_mult() that are passed one of the
call_rcu_tasks*() functions. These are recognized by the tell-tale
wakeme_after_rcu callback function.
In other cases, callbacks are gathered up for up to about 250 milliseconds
before a grace period is initiated. This results in more than an order of
magnitude reduction in RCU Tasks Trace grace periods, with corresponding
reduction in consumption of CPU time.
Reported-by: Alexei Starovoitov <ast@kernel.org>
Reported-by: Martin KaFai Lau <kafai@fb.com>
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
2023-05-14 18:06:12 -07:00
|
|
|
if (rtpcp->urgent_gp > 0 && rcu_segcblist_pend_cbs(&rtpcp->cblist)) {
|
|
|
|
|
if (rtp->lazy_jiffies)
|
|
|
|
|
rtpcp->urgent_gp--;
|
2021-11-09 14:37:34 -07:00
|
|
|
needgpcb |= 0x3;
|
rcu-tasks: Treat only synchronous grace periods urgently
The performance requirements on RCU Tasks, and in particular on RCU
Tasks Trace, have evolved over time as the workloads have evolved.
The current implementation is designed to provide low grace-period
latencies, and also to accommodate short-duration floods of callbacks.
However, current workloads can also provide a constant background
callback-queuing rate of a few hundred call_rcu_tasks_trace() invocations
per second. This results in continuous back-to-back RCU Tasks Trace
grace periods, which in turn can consume the better part of 10% of a CPU.
One could take the attitude that there are several tens of other CPUs on
the systems running such workloads, but energy efficiency is a thing.
On these systems, although asynchronous grace-period requests happen
every few milliseconds, synchronous grace-period requests are quite rare.
This commit therefore arrranges for grace periods to be initiated
immediately in response to calls to synchronize_rcu_tasks*() and
also to calls to synchronize_rcu_mult() that are passed one of the
call_rcu_tasks*() functions. These are recognized by the tell-tale
wakeme_after_rcu callback function.
In other cases, callbacks are gathered up for up to about 250 milliseconds
before a grace period is initiated. This results in more than an order of
magnitude reduction in RCU Tasks Trace grace periods, with corresponding
reduction in consumption of CPU time.
Reported-by: Alexei Starovoitov <ast@kernel.org>
Reported-by: Martin KaFai Lau <kafai@fb.com>
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
2023-05-14 18:06:12 -07:00
|
|
|
} else if (rcu_segcblist_empty(&rtpcp->cblist)) {
|
|
|
|
|
rtpcp->urgent_gp = 0;
|
|
|
|
|
}
|
|
|
|
|
if (rcu_segcblist_ready_cbs(&rtpcp->cblist))
|
2021-11-09 14:37:34 -07:00
|
|
|
needgpcb |= 0x1;
|
|
|
|
|
raw_spin_unlock_irqrestore_rcu_node(rtpcp, flags);
|
|
|
|
|
}
|
rcu-tasks: Use fewer callbacks queues if callback flood ends
By default, when lock contention is encountered, the RCU Tasks flavors
of RCU switch to using per-CPU queueing. However, if the callback
flood ends, per-CPU queueing continues to be used, which introduces
significant additional overhead, especially for callback invocation,
which fans out a series of workqueue handlers.
This commit therefore switches back to single-queue operation if at the
beginning of a grace period there are very few callbacks. The definition
of "very few" is set by the rcupdate.rcu_task_collapse_lim module
parameter, which defaults to 10. This switch happens in two phases,
with the first phase causing future callbacks to be enqueued on CPU 0's
queue, but with all queues continuing to be checked for grace periods
and callback invocation. The second phase checks to see if an RCU grace
period has elapsed and if all remaining RCU-Tasks callbacks are queued
on CPU 0. If so, only CPU 0 is checked for future grace periods and
callback operation.
Of course, the return of contention anywhere during this process will
result in returning to per-CPU callback queueing.
Reported-by: Martin Lau <kafai@fb.com>
Cc: Neeraj Upadhyay <neeraj.iitr10@gmail.com>
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
2021-11-29 17:52:31 -07:00
|
|
|
|
|
|
|
|
// Shrink down to a single callback queue if appropriate.
|
|
|
|
|
// This is done in two stages: (1) If there are no more than
|
|
|
|
|
// rcu_task_collapse_lim callbacks on CPU 0 and none on any other
|
|
|
|
|
// CPU, limit enqueueing to CPU 0. (2) After an RCU grace period,
|
|
|
|
|
// if there has not been an increase in callbacks, limit dequeuing
|
|
|
|
|
// to CPU 0. Note the matching RCU read-side critical section in
|
|
|
|
|
// call_rcu_tasks_generic().
|
|
|
|
|
if (rcu_task_cb_adjust && ncbs <= rcu_task_collapse_lim) {
|
|
|
|
|
raw_spin_lock_irqsave(&rtp->cbs_gbl_lock, flags);
|
|
|
|
|
if (rtp->percpu_enqueue_lim > 1) {
|
2024-07-09 21:45:42 -07:00
|
|
|
WRITE_ONCE(rtp->percpu_enqueue_shift, order_base_2(rcu_task_cpu_ids));
|
rcu-tasks: Use fewer callbacks queues if callback flood ends
By default, when lock contention is encountered, the RCU Tasks flavors
of RCU switch to using per-CPU queueing. However, if the callback
flood ends, per-CPU queueing continues to be used, which introduces
significant additional overhead, especially for callback invocation,
which fans out a series of workqueue handlers.
This commit therefore switches back to single-queue operation if at the
beginning of a grace period there are very few callbacks. The definition
of "very few" is set by the rcupdate.rcu_task_collapse_lim module
parameter, which defaults to 10. This switch happens in two phases,
with the first phase causing future callbacks to be enqueued on CPU 0's
queue, but with all queues continuing to be checked for grace periods
and callback invocation. The second phase checks to see if an RCU grace
period has elapsed and if all remaining RCU-Tasks callbacks are queued
on CPU 0. If so, only CPU 0 is checked for future grace periods and
callback operation.
Of course, the return of contention anywhere during this process will
result in returning to per-CPU callback queueing.
Reported-by: Martin Lau <kafai@fb.com>
Cc: Neeraj Upadhyay <neeraj.iitr10@gmail.com>
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
2021-11-29 17:52:31 -07:00
|
|
|
smp_store_release(&rtp->percpu_enqueue_lim, 1);
|
|
|
|
|
rtp->percpu_dequeue_gpseq = get_state_synchronize_rcu();
|
rcu-tasks: Handle queue-shrink/callback-enqueue race condition
The rcu_tasks_need_gpcb() determines whether or not: (1) There are
callbacks needing another grace period, (2) There are callbacks ready
to be invoked, and (3) It would be a good time to shrink back down to a
single-CPU callback list. This third case is interesting because some
other CPU might be adding new callbacks, which might suddenly make this
a very bad time to be shrinking.
This is currently handled by requiring call_rcu_tasks_generic() to
enqueue callbacks under the protection of rcu_read_lock() and requiring
rcu_tasks_need_gpcb() to wait for an RCU grace period to elapse before
finalizing the transition. This works well in practice.
Unfortunately, the current code assumes that a grace period whose end is
detected by the poll_state_synchronize_rcu() in the second "if" condition
actually ended before the earlier code counted the callbacks queued on
CPUs other than CPU 0 (local variable "ncbsnz"). Given the current code,
it is possible that a long-delayed call_rcu_tasks_generic() invocation
will queue a callback on a non-zero CPU after these CPUs have had their
callbacks counted and zero has been stored to ncbsnz. Such a callback
would trigger the WARN_ON_ONCE() in the second "if" statement.
To see this, consider the following sequence of events:
o CPU 0 invokes rcu_tasks_one_gp(), and counts fewer than
rcu_task_collapse_lim callbacks. It sees at least one
callback queued on some other CPU, thus setting ncbsnz
to a non-zero value.
o CPU 1 invokes call_rcu_tasks_generic() and loads 42 from
->percpu_enqueue_lim. It therefore decides to enqueue its
callback onto CPU 1's callback list, but is delayed.
o CPU 0 sees the rcu_task_cb_adjust is non-zero and that the number
of callbacks does not exceed rcu_task_collapse_lim. It therefore
checks percpu_enqueue_lim, and sees that its value is greater
than the value one. CPU 0 therefore starts the shift back
to a single callback list. It sets ->percpu_enqueue_lim to 1,
but CPU 1 has already read the old value of 42. It also gets
a grace-period state value from get_state_synchronize_rcu().
o CPU 0 sees that ncbsnz is non-zero in its second "if" statement,
so it declines to finalize the shrink operation.
o CPU 0 again invokes rcu_tasks_one_gp(), and counts fewer than
rcu_task_collapse_lim callbacks. It also sees that there are
no callback queued on any other CPU, and thus sets ncbsnz to zero.
o CPU 1 resumes execution and enqueues its callback onto its own
list. This invalidates the value of ncbsnz.
o CPU 0 sees the rcu_task_cb_adjust is non-zero and that the number
of callbacks does not exceed rcu_task_collapse_lim. It therefore
checks percpu_enqueue_lim, but sees that its value is already
unity. It therefore does not get a new grace-period state value.
o CPU 0 sees that rcu_task_cb_adjust is non-zero, ncbsnz is zero,
and that poll_state_synchronize_rcu() says that the grace period
has completed. it therefore finalizes the shrink operation,
setting ->percpu_dequeue_lim to the value one.
o CPU 0 does a debug check, scanning the other CPUs' callback lists.
It sees that CPU 1's list has a callback, so it (rightly)
triggers the WARN_ON_ONCE(). After all, the new value of
->percpu_dequeue_lim says to not bother looking at CPU 1's
callback list, which means that this callback will never be
invoked. This can result in hangs and maybe even OOMs.
Based on long experience with rcutorture, this is an extremely
low-probability race condition, but it really can happen, especially in
preemptible kernels or within guest OSes.
This commit therefore checks for completion of the grace period
before counting callbacks. With this change, in the above failure
scenario CPU 0 would know not to prematurely end the shrink operation
because the grace period would not have completed before the count
operation started.
[ paulmck: Adjust grace-period end rather than adding RCU reader. ]
[ paulmck: Avoid spurious WARN_ON_ONCE() with ->percpu_dequeue_lim check. ]
Signed-off-by: Zqiang <qiang1.zhang@intel.com>
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
2022-12-02 19:25:03 -07:00
|
|
|
gpdone = false;
|
rcu-tasks: Use fewer callbacks queues if callback flood ends
By default, when lock contention is encountered, the RCU Tasks flavors
of RCU switch to using per-CPU queueing. However, if the callback
flood ends, per-CPU queueing continues to be used, which introduces
significant additional overhead, especially for callback invocation,
which fans out a series of workqueue handlers.
This commit therefore switches back to single-queue operation if at the
beginning of a grace period there are very few callbacks. The definition
of "very few" is set by the rcupdate.rcu_task_collapse_lim module
parameter, which defaults to 10. This switch happens in two phases,
with the first phase causing future callbacks to be enqueued on CPU 0's
queue, but with all queues continuing to be checked for grace periods
and callback invocation. The second phase checks to see if an RCU grace
period has elapsed and if all remaining RCU-Tasks callbacks are queued
on CPU 0. If so, only CPU 0 is checked for future grace periods and
callback operation.
Of course, the return of contention anywhere during this process will
result in returning to per-CPU callback queueing.
Reported-by: Martin Lau <kafai@fb.com>
Cc: Neeraj Upadhyay <neeraj.iitr10@gmail.com>
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
2021-11-29 17:52:31 -07:00
|
|
|
pr_info("Starting switch %s to CPU-0 callback queuing.\n", rtp->name);
|
|
|
|
|
}
|
|
|
|
|
raw_spin_unlock_irqrestore(&rtp->cbs_gbl_lock, flags);
|
|
|
|
|
}
|
rcu-tasks: Handle queue-shrink/callback-enqueue race condition
The rcu_tasks_need_gpcb() determines whether or not: (1) There are
callbacks needing another grace period, (2) There are callbacks ready
to be invoked, and (3) It would be a good time to shrink back down to a
single-CPU callback list. This third case is interesting because some
other CPU might be adding new callbacks, which might suddenly make this
a very bad time to be shrinking.
This is currently handled by requiring call_rcu_tasks_generic() to
enqueue callbacks under the protection of rcu_read_lock() and requiring
rcu_tasks_need_gpcb() to wait for an RCU grace period to elapse before
finalizing the transition. This works well in practice.
Unfortunately, the current code assumes that a grace period whose end is
detected by the poll_state_synchronize_rcu() in the second "if" condition
actually ended before the earlier code counted the callbacks queued on
CPUs other than CPU 0 (local variable "ncbsnz"). Given the current code,
it is possible that a long-delayed call_rcu_tasks_generic() invocation
will queue a callback on a non-zero CPU after these CPUs have had their
callbacks counted and zero has been stored to ncbsnz. Such a callback
would trigger the WARN_ON_ONCE() in the second "if" statement.
To see this, consider the following sequence of events:
o CPU 0 invokes rcu_tasks_one_gp(), and counts fewer than
rcu_task_collapse_lim callbacks. It sees at least one
callback queued on some other CPU, thus setting ncbsnz
to a non-zero value.
o CPU 1 invokes call_rcu_tasks_generic() and loads 42 from
->percpu_enqueue_lim. It therefore decides to enqueue its
callback onto CPU 1's callback list, but is delayed.
o CPU 0 sees the rcu_task_cb_adjust is non-zero and that the number
of callbacks does not exceed rcu_task_collapse_lim. It therefore
checks percpu_enqueue_lim, and sees that its value is greater
than the value one. CPU 0 therefore starts the shift back
to a single callback list. It sets ->percpu_enqueue_lim to 1,
but CPU 1 has already read the old value of 42. It also gets
a grace-period state value from get_state_synchronize_rcu().
o CPU 0 sees that ncbsnz is non-zero in its second "if" statement,
so it declines to finalize the shrink operation.
o CPU 0 again invokes rcu_tasks_one_gp(), and counts fewer than
rcu_task_collapse_lim callbacks. It also sees that there are
no callback queued on any other CPU, and thus sets ncbsnz to zero.
o CPU 1 resumes execution and enqueues its callback onto its own
list. This invalidates the value of ncbsnz.
o CPU 0 sees the rcu_task_cb_adjust is non-zero and that the number
of callbacks does not exceed rcu_task_collapse_lim. It therefore
checks percpu_enqueue_lim, but sees that its value is already
unity. It therefore does not get a new grace-period state value.
o CPU 0 sees that rcu_task_cb_adjust is non-zero, ncbsnz is zero,
and that poll_state_synchronize_rcu() says that the grace period
has completed. it therefore finalizes the shrink operation,
setting ->percpu_dequeue_lim to the value one.
o CPU 0 does a debug check, scanning the other CPUs' callback lists.
It sees that CPU 1's list has a callback, so it (rightly)
triggers the WARN_ON_ONCE(). After all, the new value of
->percpu_dequeue_lim says to not bother looking at CPU 1's
callback list, which means that this callback will never be
invoked. This can result in hangs and maybe even OOMs.
Based on long experience with rcutorture, this is an extremely
low-probability race condition, but it really can happen, especially in
preemptible kernels or within guest OSes.
This commit therefore checks for completion of the grace period
before counting callbacks. With this change, in the above failure
scenario CPU 0 would know not to prematurely end the shrink operation
because the grace period would not have completed before the count
operation started.
[ paulmck: Adjust grace-period end rather than adding RCU reader. ]
[ paulmck: Avoid spurious WARN_ON_ONCE() with ->percpu_dequeue_lim check. ]
Signed-off-by: Zqiang <qiang1.zhang@intel.com>
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
2022-12-02 19:25:03 -07:00
|
|
|
if (rcu_task_cb_adjust && !ncbsnz && gpdone) {
|
rcu-tasks: Use fewer callbacks queues if callback flood ends
By default, when lock contention is encountered, the RCU Tasks flavors
of RCU switch to using per-CPU queueing. However, if the callback
flood ends, per-CPU queueing continues to be used, which introduces
significant additional overhead, especially for callback invocation,
which fans out a series of workqueue handlers.
This commit therefore switches back to single-queue operation if at the
beginning of a grace period there are very few callbacks. The definition
of "very few" is set by the rcupdate.rcu_task_collapse_lim module
parameter, which defaults to 10. This switch happens in two phases,
with the first phase causing future callbacks to be enqueued on CPU 0's
queue, but with all queues continuing to be checked for grace periods
and callback invocation. The second phase checks to see if an RCU grace
period has elapsed and if all remaining RCU-Tasks callbacks are queued
on CPU 0. If so, only CPU 0 is checked for future grace periods and
callback operation.
Of course, the return of contention anywhere during this process will
result in returning to per-CPU callback queueing.
Reported-by: Martin Lau <kafai@fb.com>
Cc: Neeraj Upadhyay <neeraj.iitr10@gmail.com>
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
2021-11-29 17:52:31 -07:00
|
|
|
raw_spin_lock_irqsave(&rtp->cbs_gbl_lock, flags);
|
|
|
|
|
if (rtp->percpu_enqueue_lim < rtp->percpu_dequeue_lim) {
|
|
|
|
|
WRITE_ONCE(rtp->percpu_dequeue_lim, 1);
|
|
|
|
|
pr_info("Completing switch %s to CPU-0 callback queuing.\n", rtp->name);
|
|
|
|
|
}
|
rcu-tasks: Handle queue-shrink/callback-enqueue race condition
The rcu_tasks_need_gpcb() determines whether or not: (1) There are
callbacks needing another grace period, (2) There are callbacks ready
to be invoked, and (3) It would be a good time to shrink back down to a
single-CPU callback list. This third case is interesting because some
other CPU might be adding new callbacks, which might suddenly make this
a very bad time to be shrinking.
This is currently handled by requiring call_rcu_tasks_generic() to
enqueue callbacks under the protection of rcu_read_lock() and requiring
rcu_tasks_need_gpcb() to wait for an RCU grace period to elapse before
finalizing the transition. This works well in practice.
Unfortunately, the current code assumes that a grace period whose end is
detected by the poll_state_synchronize_rcu() in the second "if" condition
actually ended before the earlier code counted the callbacks queued on
CPUs other than CPU 0 (local variable "ncbsnz"). Given the current code,
it is possible that a long-delayed call_rcu_tasks_generic() invocation
will queue a callback on a non-zero CPU after these CPUs have had their
callbacks counted and zero has been stored to ncbsnz. Such a callback
would trigger the WARN_ON_ONCE() in the second "if" statement.
To see this, consider the following sequence of events:
o CPU 0 invokes rcu_tasks_one_gp(), and counts fewer than
rcu_task_collapse_lim callbacks. It sees at least one
callback queued on some other CPU, thus setting ncbsnz
to a non-zero value.
o CPU 1 invokes call_rcu_tasks_generic() and loads 42 from
->percpu_enqueue_lim. It therefore decides to enqueue its
callback onto CPU 1's callback list, but is delayed.
o CPU 0 sees the rcu_task_cb_adjust is non-zero and that the number
of callbacks does not exceed rcu_task_collapse_lim. It therefore
checks percpu_enqueue_lim, and sees that its value is greater
than the value one. CPU 0 therefore starts the shift back
to a single callback list. It sets ->percpu_enqueue_lim to 1,
but CPU 1 has already read the old value of 42. It also gets
a grace-period state value from get_state_synchronize_rcu().
o CPU 0 sees that ncbsnz is non-zero in its second "if" statement,
so it declines to finalize the shrink operation.
o CPU 0 again invokes rcu_tasks_one_gp(), and counts fewer than
rcu_task_collapse_lim callbacks. It also sees that there are
no callback queued on any other CPU, and thus sets ncbsnz to zero.
o CPU 1 resumes execution and enqueues its callback onto its own
list. This invalidates the value of ncbsnz.
o CPU 0 sees the rcu_task_cb_adjust is non-zero and that the number
of callbacks does not exceed rcu_task_collapse_lim. It therefore
checks percpu_enqueue_lim, but sees that its value is already
unity. It therefore does not get a new grace-period state value.
o CPU 0 sees that rcu_task_cb_adjust is non-zero, ncbsnz is zero,
and that poll_state_synchronize_rcu() says that the grace period
has completed. it therefore finalizes the shrink operation,
setting ->percpu_dequeue_lim to the value one.
o CPU 0 does a debug check, scanning the other CPUs' callback lists.
It sees that CPU 1's list has a callback, so it (rightly)
triggers the WARN_ON_ONCE(). After all, the new value of
->percpu_dequeue_lim says to not bother looking at CPU 1's
callback list, which means that this callback will never be
invoked. This can result in hangs and maybe even OOMs.
Based on long experience with rcutorture, this is an extremely
low-probability race condition, but it really can happen, especially in
preemptible kernels or within guest OSes.
This commit therefore checks for completion of the grace period
before counting callbacks. With this change, in the above failure
scenario CPU 0 would know not to prematurely end the shrink operation
because the grace period would not have completed before the count
operation started.
[ paulmck: Adjust grace-period end rather than adding RCU reader. ]
[ paulmck: Avoid spurious WARN_ON_ONCE() with ->percpu_dequeue_lim check. ]
Signed-off-by: Zqiang <qiang1.zhang@intel.com>
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
2022-12-02 19:25:03 -07:00
|
|
|
if (rtp->percpu_dequeue_lim == 1) {
|
2024-07-09 21:45:42 -07:00
|
|
|
for (cpu = rtp->percpu_dequeue_lim; cpu < rcu_task_cpu_ids; cpu++) {
|
|
|
|
|
if (!cpu_possible(cpu))
|
|
|
|
|
continue;
|
rcu-tasks: Handle queue-shrink/callback-enqueue race condition
The rcu_tasks_need_gpcb() determines whether or not: (1) There are
callbacks needing another grace period, (2) There are callbacks ready
to be invoked, and (3) It would be a good time to shrink back down to a
single-CPU callback list. This third case is interesting because some
other CPU might be adding new callbacks, which might suddenly make this
a very bad time to be shrinking.
This is currently handled by requiring call_rcu_tasks_generic() to
enqueue callbacks under the protection of rcu_read_lock() and requiring
rcu_tasks_need_gpcb() to wait for an RCU grace period to elapse before
finalizing the transition. This works well in practice.
Unfortunately, the current code assumes that a grace period whose end is
detected by the poll_state_synchronize_rcu() in the second "if" condition
actually ended before the earlier code counted the callbacks queued on
CPUs other than CPU 0 (local variable "ncbsnz"). Given the current code,
it is possible that a long-delayed call_rcu_tasks_generic() invocation
will queue a callback on a non-zero CPU after these CPUs have had their
callbacks counted and zero has been stored to ncbsnz. Such a callback
would trigger the WARN_ON_ONCE() in the second "if" statement.
To see this, consider the following sequence of events:
o CPU 0 invokes rcu_tasks_one_gp(), and counts fewer than
rcu_task_collapse_lim callbacks. It sees at least one
callback queued on some other CPU, thus setting ncbsnz
to a non-zero value.
o CPU 1 invokes call_rcu_tasks_generic() and loads 42 from
->percpu_enqueue_lim. It therefore decides to enqueue its
callback onto CPU 1's callback list, but is delayed.
o CPU 0 sees the rcu_task_cb_adjust is non-zero and that the number
of callbacks does not exceed rcu_task_collapse_lim. It therefore
checks percpu_enqueue_lim, and sees that its value is greater
than the value one. CPU 0 therefore starts the shift back
to a single callback list. It sets ->percpu_enqueue_lim to 1,
but CPU 1 has already read the old value of 42. It also gets
a grace-period state value from get_state_synchronize_rcu().
o CPU 0 sees that ncbsnz is non-zero in its second "if" statement,
so it declines to finalize the shrink operation.
o CPU 0 again invokes rcu_tasks_one_gp(), and counts fewer than
rcu_task_collapse_lim callbacks. It also sees that there are
no callback queued on any other CPU, and thus sets ncbsnz to zero.
o CPU 1 resumes execution and enqueues its callback onto its own
list. This invalidates the value of ncbsnz.
o CPU 0 sees the rcu_task_cb_adjust is non-zero and that the number
of callbacks does not exceed rcu_task_collapse_lim. It therefore
checks percpu_enqueue_lim, but sees that its value is already
unity. It therefore does not get a new grace-period state value.
o CPU 0 sees that rcu_task_cb_adjust is non-zero, ncbsnz is zero,
and that poll_state_synchronize_rcu() says that the grace period
has completed. it therefore finalizes the shrink operation,
setting ->percpu_dequeue_lim to the value one.
o CPU 0 does a debug check, scanning the other CPUs' callback lists.
It sees that CPU 1's list has a callback, so it (rightly)
triggers the WARN_ON_ONCE(). After all, the new value of
->percpu_dequeue_lim says to not bother looking at CPU 1's
callback list, which means that this callback will never be
invoked. This can result in hangs and maybe even OOMs.
Based on long experience with rcutorture, this is an extremely
low-probability race condition, but it really can happen, especially in
preemptible kernels or within guest OSes.
This commit therefore checks for completion of the grace period
before counting callbacks. With this change, in the above failure
scenario CPU 0 would know not to prematurely end the shrink operation
because the grace period would not have completed before the count
operation started.
[ paulmck: Adjust grace-period end rather than adding RCU reader. ]
[ paulmck: Avoid spurious WARN_ON_ONCE() with ->percpu_dequeue_lim check. ]
Signed-off-by: Zqiang <qiang1.zhang@intel.com>
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
2022-12-02 19:25:03 -07:00
|
|
|
struct rcu_tasks_percpu *rtpcp = per_cpu_ptr(rtp->rtpcpu, cpu);
|
2021-12-06 17:19:40 -07:00
|
|
|
|
rcu-tasks: Handle queue-shrink/callback-enqueue race condition
The rcu_tasks_need_gpcb() determines whether or not: (1) There are
callbacks needing another grace period, (2) There are callbacks ready
to be invoked, and (3) It would be a good time to shrink back down to a
single-CPU callback list. This third case is interesting because some
other CPU might be adding new callbacks, which might suddenly make this
a very bad time to be shrinking.
This is currently handled by requiring call_rcu_tasks_generic() to
enqueue callbacks under the protection of rcu_read_lock() and requiring
rcu_tasks_need_gpcb() to wait for an RCU grace period to elapse before
finalizing the transition. This works well in practice.
Unfortunately, the current code assumes that a grace period whose end is
detected by the poll_state_synchronize_rcu() in the second "if" condition
actually ended before the earlier code counted the callbacks queued on
CPUs other than CPU 0 (local variable "ncbsnz"). Given the current code,
it is possible that a long-delayed call_rcu_tasks_generic() invocation
will queue a callback on a non-zero CPU after these CPUs have had their
callbacks counted and zero has been stored to ncbsnz. Such a callback
would trigger the WARN_ON_ONCE() in the second "if" statement.
To see this, consider the following sequence of events:
o CPU 0 invokes rcu_tasks_one_gp(), and counts fewer than
rcu_task_collapse_lim callbacks. It sees at least one
callback queued on some other CPU, thus setting ncbsnz
to a non-zero value.
o CPU 1 invokes call_rcu_tasks_generic() and loads 42 from
->percpu_enqueue_lim. It therefore decides to enqueue its
callback onto CPU 1's callback list, but is delayed.
o CPU 0 sees the rcu_task_cb_adjust is non-zero and that the number
of callbacks does not exceed rcu_task_collapse_lim. It therefore
checks percpu_enqueue_lim, and sees that its value is greater
than the value one. CPU 0 therefore starts the shift back
to a single callback list. It sets ->percpu_enqueue_lim to 1,
but CPU 1 has already read the old value of 42. It also gets
a grace-period state value from get_state_synchronize_rcu().
o CPU 0 sees that ncbsnz is non-zero in its second "if" statement,
so it declines to finalize the shrink operation.
o CPU 0 again invokes rcu_tasks_one_gp(), and counts fewer than
rcu_task_collapse_lim callbacks. It also sees that there are
no callback queued on any other CPU, and thus sets ncbsnz to zero.
o CPU 1 resumes execution and enqueues its callback onto its own
list. This invalidates the value of ncbsnz.
o CPU 0 sees the rcu_task_cb_adjust is non-zero and that the number
of callbacks does not exceed rcu_task_collapse_lim. It therefore
checks percpu_enqueue_lim, but sees that its value is already
unity. It therefore does not get a new grace-period state value.
o CPU 0 sees that rcu_task_cb_adjust is non-zero, ncbsnz is zero,
and that poll_state_synchronize_rcu() says that the grace period
has completed. it therefore finalizes the shrink operation,
setting ->percpu_dequeue_lim to the value one.
o CPU 0 does a debug check, scanning the other CPUs' callback lists.
It sees that CPU 1's list has a callback, so it (rightly)
triggers the WARN_ON_ONCE(). After all, the new value of
->percpu_dequeue_lim says to not bother looking at CPU 1's
callback list, which means that this callback will never be
invoked. This can result in hangs and maybe even OOMs.
Based on long experience with rcutorture, this is an extremely
low-probability race condition, but it really can happen, especially in
preemptible kernels or within guest OSes.
This commit therefore checks for completion of the grace period
before counting callbacks. With this change, in the above failure
scenario CPU 0 would know not to prematurely end the shrink operation
because the grace period would not have completed before the count
operation started.
[ paulmck: Adjust grace-period end rather than adding RCU reader. ]
[ paulmck: Avoid spurious WARN_ON_ONCE() with ->percpu_dequeue_lim check. ]
Signed-off-by: Zqiang <qiang1.zhang@intel.com>
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
2022-12-02 19:25:03 -07:00
|
|
|
WARN_ON_ONCE(rcu_segcblist_n_cbs(&rtpcp->cblist));
|
|
|
|
|
}
|
2021-12-06 17:19:40 -07:00
|
|
|
}
|
rcu-tasks: Use fewer callbacks queues if callback flood ends
By default, when lock contention is encountered, the RCU Tasks flavors
of RCU switch to using per-CPU queueing. However, if the callback
flood ends, per-CPU queueing continues to be used, which introduces
significant additional overhead, especially for callback invocation,
which fans out a series of workqueue handlers.
This commit therefore switches back to single-queue operation if at the
beginning of a grace period there are very few callbacks. The definition
of "very few" is set by the rcupdate.rcu_task_collapse_lim module
parameter, which defaults to 10. This switch happens in two phases,
with the first phase causing future callbacks to be enqueued on CPU 0's
queue, but with all queues continuing to be checked for grace periods
and callback invocation. The second phase checks to see if an RCU grace
period has elapsed and if all remaining RCU-Tasks callbacks are queued
on CPU 0. If so, only CPU 0 is checked for future grace periods and
callback operation.
Of course, the return of contention anywhere during this process will
result in returning to per-CPU callback queueing.
Reported-by: Martin Lau <kafai@fb.com>
Cc: Neeraj Upadhyay <neeraj.iitr10@gmail.com>
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
2021-11-29 17:52:31 -07:00
|
|
|
raw_spin_unlock_irqrestore(&rtp->cbs_gbl_lock, flags);
|
|
|
|
|
}
|
|
|
|
|
|
2021-11-09 14:37:34 -07:00
|
|
|
return needgpcb;
|
|
|
|
|
}
|
|
|
|
|
|
2021-11-09 16:34:56 -07:00
|
|
|
// Advance callbacks and invoke any that are ready.
|
2021-11-10 16:56:40 -07:00
|
|
|
static void rcu_tasks_invoke_cbs(struct rcu_tasks *rtp, struct rcu_tasks_percpu *rtpcp)
|
2020-03-02 12:59:20 -07:00
|
|
|
{
|
rcu-tasks: Stop rcu_tasks_invoke_cbs() from using never-onlined CPUs
The rcu_tasks_invoke_cbs() function relies on queue_work_on() to silently
fall back to WORK_CPU_UNBOUND when the specified CPU is offline. However,
the queue_work_on() function's silent fallback mechanism relies on that
CPU having been online at some time in the past. When queue_work_on()
is passed a CPU that has never been online, workqueue lockups ensue,
which can be bad for your kernel's general health and well-being.
This commit therefore checks whether a given CPU has ever been online,
and, if not substitutes WORK_CPU_UNBOUND in the subsequent call to
queue_work_on(). Why not simply omit the queue_work_on() call entirely?
Because this function is flooding callback-invocation notifications
to all CPUs, and must deal with possibilities that include a sparse
cpu_possible_mask.
This commit also moves the setting of the rcu_data structure's
->beenonline field to rcu_cpu_starting(), which executes on the
incoming CPU before that CPU has ever enabled interrupts. This ensures
that the required workqueues are present. In addition, because the
incoming CPU has not yet enabled its interrupts, there cannot yet have
been any softirq handlers running on this CPU, which means that the
WARN_ON_ONCE(!rdp->beenonline) within the RCU_SOFTIRQ handler cannot
have triggered yet.
Fixes: d363f833c6d88 ("rcu-tasks: Use workqueues for multiple rcu_tasks_invoke_cbs() invocations")
Reported-by: Tejun Heo <tj@kernel.org>
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
2023-04-26 11:11:29 -07:00
|
|
|
int cpuwq;
|
2020-03-02 12:59:20 -07:00
|
|
|
unsigned long flags;
|
2021-11-08 17:18:57 -07:00
|
|
|
int len;
|
2024-07-09 21:45:42 -07:00
|
|
|
int index;
|
2021-11-08 17:18:57 -07:00
|
|
|
struct rcu_head *rhp;
|
2021-11-10 16:56:40 -07:00
|
|
|
struct rcu_cblist rcl = RCU_CBLIST_INITIALIZER(rcl);
|
|
|
|
|
struct rcu_tasks_percpu *rtpcp_next;
|
|
|
|
|
|
2024-07-09 21:45:42 -07:00
|
|
|
index = rtpcp->index * 2 + 1;
|
|
|
|
|
if (index < num_possible_cpus()) {
|
|
|
|
|
rtpcp_next = rtp->rtpcp_array[index];
|
|
|
|
|
if (rtpcp_next->cpu < smp_load_acquire(&rtp->percpu_dequeue_lim)) {
|
|
|
|
|
cpuwq = rcu_cpu_beenfullyonline(rtpcp_next->cpu) ? rtpcp_next->cpu : WORK_CPU_UNBOUND;
|
rcu-tasks: Stop rcu_tasks_invoke_cbs() from using never-onlined CPUs
The rcu_tasks_invoke_cbs() function relies on queue_work_on() to silently
fall back to WORK_CPU_UNBOUND when the specified CPU is offline. However,
the queue_work_on() function's silent fallback mechanism relies on that
CPU having been online at some time in the past. When queue_work_on()
is passed a CPU that has never been online, workqueue lockups ensue,
which can be bad for your kernel's general health and well-being.
This commit therefore checks whether a given CPU has ever been online,
and, if not substitutes WORK_CPU_UNBOUND in the subsequent call to
queue_work_on(). Why not simply omit the queue_work_on() call entirely?
Because this function is flooding callback-invocation notifications
to all CPUs, and must deal with possibilities that include a sparse
cpu_possible_mask.
This commit also moves the setting of the rcu_data structure's
->beenonline field to rcu_cpu_starting(), which executes on the
incoming CPU before that CPU has ever enabled interrupts. This ensures
that the required workqueues are present. In addition, because the
incoming CPU has not yet enabled its interrupts, there cannot yet have
been any softirq handlers running on this CPU, which means that the
WARN_ON_ONCE(!rdp->beenonline) within the RCU_SOFTIRQ handler cannot
have triggered yet.
Fixes: d363f833c6d88 ("rcu-tasks: Use workqueues for multiple rcu_tasks_invoke_cbs() invocations")
Reported-by: Tejun Heo <tj@kernel.org>
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
2023-04-26 11:11:29 -07:00
|
|
|
queue_work_on(cpuwq, system_wq, &rtpcp_next->rtp_work);
|
2024-07-09 21:45:42 -07:00
|
|
|
index++;
|
|
|
|
|
if (index < num_possible_cpus()) {
|
|
|
|
|
rtpcp_next = rtp->rtpcp_array[index];
|
|
|
|
|
if (rtpcp_next->cpu < smp_load_acquire(&rtp->percpu_dequeue_lim)) {
|
|
|
|
|
cpuwq = rcu_cpu_beenfullyonline(rtpcp_next->cpu) ? rtpcp_next->cpu : WORK_CPU_UNBOUND;
|
|
|
|
|
queue_work_on(cpuwq, system_wq, &rtpcp_next->rtp_work);
|
|
|
|
|
}
|
|
|
|
|
}
|
2021-11-09 16:34:56 -07:00
|
|
|
}
|
|
|
|
|
}
|
2021-11-10 16:56:40 -07:00
|
|
|
|
2024-07-09 21:45:42 -07:00
|
|
|
if (rcu_segcblist_empty(&rtpcp->cblist))
|
2021-11-10 16:56:40 -07:00
|
|
|
return;
|
|
|
|
|
raw_spin_lock_irqsave_rcu_node(rtpcp, flags);
|
|
|
|
|
rcu_segcblist_advance(&rtpcp->cblist, rcu_seq_current(&rtp->tasks_gp_seq));
|
|
|
|
|
rcu_segcblist_extract_done_cbs(&rtpcp->cblist, &rcl);
|
|
|
|
|
raw_spin_unlock_irqrestore_rcu_node(rtpcp, flags);
|
|
|
|
|
len = rcl.len;
|
|
|
|
|
for (rhp = rcu_cblist_dequeue(&rcl); rhp; rhp = rcu_cblist_dequeue(&rcl)) {
|
rcu: Dump memory object info if callback function is invalid
When a structure containing an RCU callback rhp is (incorrectly) freed
and reallocated after rhp is passed to call_rcu(), it is not unusual for
rhp->func to be set to NULL. This defeats the debugging prints used by
__call_rcu_common() in kernels built with CONFIG_DEBUG_OBJECTS_RCU_HEAD=y,
which expect to identify the offending code using the identity of this
function.
And in kernels build without CONFIG_DEBUG_OBJECTS_RCU_HEAD=y, things
are even worse, as can be seen from this splat:
Unable to handle kernel NULL pointer dereference at virtual address 0
... ...
PC is at 0x0
LR is at rcu_do_batch+0x1c0/0x3b8
... ...
(rcu_do_batch) from (rcu_core+0x1d4/0x284)
(rcu_core) from (__do_softirq+0x24c/0x344)
(__do_softirq) from (__irq_exit_rcu+0x64/0x108)
(__irq_exit_rcu) from (irq_exit+0x8/0x10)
(irq_exit) from (__handle_domain_irq+0x74/0x9c)
(__handle_domain_irq) from (gic_handle_irq+0x8c/0x98)
(gic_handle_irq) from (__irq_svc+0x5c/0x94)
(__irq_svc) from (arch_cpu_idle+0x20/0x3c)
(arch_cpu_idle) from (default_idle_call+0x4c/0x78)
(default_idle_call) from (do_idle+0xf8/0x150)
(do_idle) from (cpu_startup_entry+0x18/0x20)
(cpu_startup_entry) from (0xc01530)
This commit therefore adds calls to mem_dump_obj(rhp) to output some
information, for example:
slab kmalloc-256 start ffff410c45019900 pointer offset 0 size 256
This provides the rough size of the memory block and the offset of the
rcu_head structure, which as least provides at least a few clues to help
locate the problem. If the problem is reproducible, additional slab
debugging can be enabled, for example, CONFIG_DEBUG_SLAB=y, which can
provide significantly more information.
Signed-off-by: Zhen Lei <thunder.leizhen@huawei.com>
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
Signed-off-by: Frederic Weisbecker <frederic@kernel.org>
2023-08-04 20:17:26 -07:00
|
|
|
debug_rcu_head_callback(rhp);
|
2021-11-10 16:56:40 -07:00
|
|
|
local_bh_disable();
|
|
|
|
|
rhp->func(rhp);
|
|
|
|
|
local_bh_enable();
|
|
|
|
|
cond_resched();
|
|
|
|
|
}
|
|
|
|
|
raw_spin_lock_irqsave_rcu_node(rtpcp, flags);
|
|
|
|
|
rcu_segcblist_add_len(&rtpcp->cblist, -len);
|
|
|
|
|
(void)rcu_segcblist_accelerate(&rtpcp->cblist, rcu_seq_snap(&rtp->tasks_gp_seq));
|
|
|
|
|
raw_spin_unlock_irqrestore_rcu_node(rtpcp, flags);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Workqueue flood to advance callbacks and invoke any that are ready.
|
|
|
|
|
static void rcu_tasks_invoke_cbs_wq(struct work_struct *wp)
|
|
|
|
|
{
|
|
|
|
|
struct rcu_tasks *rtp;
|
|
|
|
|
struct rcu_tasks_percpu *rtpcp = container_of(wp, struct rcu_tasks_percpu, rtp_work);
|
|
|
|
|
|
|
|
|
|
rtp = rtpcp->rtpp;
|
|
|
|
|
rcu_tasks_invoke_cbs(rtp, rtpcp);
|
2021-11-09 16:34:56 -07:00
|
|
|
}
|
|
|
|
|
|
2022-04-19 10:47:28 -07:00
|
|
|
// Wait for one grace period.
|
2022-04-19 15:41:38 -07:00
|
|
|
static void rcu_tasks_one_gp(struct rcu_tasks *rtp, bool midboot)
|
2021-11-09 16:34:56 -07:00
|
|
|
{
|
|
|
|
|
int needgpcb;
|
2022-04-19 10:47:28 -07:00
|
|
|
|
|
|
|
|
mutex_lock(&rtp->tasks_gp_mutex);
|
|
|
|
|
|
|
|
|
|
// If there were none, wait a bit and start over.
|
2022-04-19 15:41:38 -07:00
|
|
|
if (unlikely(midboot)) {
|
|
|
|
|
needgpcb = 0x2;
|
|
|
|
|
} else {
|
rcu-tasks: Fix boot-time RCU tasks debug-only deadlock
In kernels built with CONFIG_PROVE_RCU=y (for example, lockdep kernels),
the following sequence of events can occur:
o rcu_init_tasks_generic() is invoked just before init is spawned.
It invokes rcu_spawn_tasks_kthread() and friends.
o rcu_spawn_tasks_kthread() invokes rcu_spawn_tasks_kthread_generic(),
which uses kthread_run() to create the needed kthread.
o Control returns to rcu_init_tasks_generic(), which, because this
is a CONFIG_PROVE_RCU=y kernel, invokes the version of the
rcu_tasks_initiate_self_tests() function that actually does
something, including invoking synchronize_rcu_tasks(), which
in turn invokes synchronize_rcu_tasks_generic().
o synchronize_rcu_tasks_generic() sees that the ->kthread_ptr is
still NULL, because the newly spawned kthread has not yet
started.
o The new kthread starts, preempting synchronize_rcu_tasks_generic()
just after its check. This kthread invokes rcu_tasks_one_gp(),
which acquires ->tasks_gp_mutex, and, seeing no work, blocks
in rcuwait_wait_event(). Note that this step requires either
a preemptible kernel or a fault-injection-style sleep at the
beginning of mutex_lock().
o synchronize_rcu_tasks_generic() resumes and invokes rcu_tasks_one_gp().
o rcu_tasks_one_gp() attempts to acquire ->tasks_gp_mutex, which
is still held by the newly spawned kthread's rcu_tasks_one_gp()
function. Deadlock.
Because the only reason for ->tasks_gp_mutex is to handle pre-kthread
synchronous grace periods, this commit avoids this deadlock by having
rcu_tasks_one_gp() momentarily release ->tasks_gp_mutex while invoking
rcuwait_wait_event(). This allows the call to rcu_tasks_one_gp() from
synchronize_rcu_tasks_generic() proceed.
Note that it is not necessary to release the mutex anywhere else in
rcu_tasks_one_gp() because rcuwait_wait_event() is the only function
that can block indefinitely.
Reported-by: Guenter Roeck <linux@roeck-us.net>
Reported-by: Roy Hopkins <rhopkins@suse.de>
Reported-by: Peter Zijlstra <peterz@infradead.org>
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
Tested-by: Roy Hopkins <rhopkins@suse.de>
2023-08-01 12:11:18 -07:00
|
|
|
mutex_unlock(&rtp->tasks_gp_mutex);
|
2022-04-19 15:41:38 -07:00
|
|
|
set_tasks_gp_state(rtp, RTGS_WAIT_CBS);
|
|
|
|
|
rcuwait_wait_event(&rtp->cbs_wait,
|
|
|
|
|
(needgpcb = rcu_tasks_need_gpcb(rtp)),
|
|
|
|
|
TASK_IDLE);
|
rcu-tasks: Fix boot-time RCU tasks debug-only deadlock
In kernels built with CONFIG_PROVE_RCU=y (for example, lockdep kernels),
the following sequence of events can occur:
o rcu_init_tasks_generic() is invoked just before init is spawned.
It invokes rcu_spawn_tasks_kthread() and friends.
o rcu_spawn_tasks_kthread() invokes rcu_spawn_tasks_kthread_generic(),
which uses kthread_run() to create the needed kthread.
o Control returns to rcu_init_tasks_generic(), which, because this
is a CONFIG_PROVE_RCU=y kernel, invokes the version of the
rcu_tasks_initiate_self_tests() function that actually does
something, including invoking synchronize_rcu_tasks(), which
in turn invokes synchronize_rcu_tasks_generic().
o synchronize_rcu_tasks_generic() sees that the ->kthread_ptr is
still NULL, because the newly spawned kthread has not yet
started.
o The new kthread starts, preempting synchronize_rcu_tasks_generic()
just after its check. This kthread invokes rcu_tasks_one_gp(),
which acquires ->tasks_gp_mutex, and, seeing no work, blocks
in rcuwait_wait_event(). Note that this step requires either
a preemptible kernel or a fault-injection-style sleep at the
beginning of mutex_lock().
o synchronize_rcu_tasks_generic() resumes and invokes rcu_tasks_one_gp().
o rcu_tasks_one_gp() attempts to acquire ->tasks_gp_mutex, which
is still held by the newly spawned kthread's rcu_tasks_one_gp()
function. Deadlock.
Because the only reason for ->tasks_gp_mutex is to handle pre-kthread
synchronous grace periods, this commit avoids this deadlock by having
rcu_tasks_one_gp() momentarily release ->tasks_gp_mutex while invoking
rcuwait_wait_event(). This allows the call to rcu_tasks_one_gp() from
synchronize_rcu_tasks_generic() proceed.
Note that it is not necessary to release the mutex anywhere else in
rcu_tasks_one_gp() because rcuwait_wait_event() is the only function
that can block indefinitely.
Reported-by: Guenter Roeck <linux@roeck-us.net>
Reported-by: Roy Hopkins <rhopkins@suse.de>
Reported-by: Peter Zijlstra <peterz@infradead.org>
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
Tested-by: Roy Hopkins <rhopkins@suse.de>
2023-08-01 12:11:18 -07:00
|
|
|
mutex_lock(&rtp->tasks_gp_mutex);
|
2022-04-19 15:41:38 -07:00
|
|
|
}
|
2022-04-19 10:47:28 -07:00
|
|
|
|
|
|
|
|
if (needgpcb & 0x2) {
|
|
|
|
|
// Wait for one grace period.
|
|
|
|
|
set_tasks_gp_state(rtp, RTGS_WAIT_GP);
|
|
|
|
|
rtp->gp_start = jiffies;
|
|
|
|
|
rcu_seq_start(&rtp->tasks_gp_seq);
|
|
|
|
|
rtp->gp_func(rtp);
|
|
|
|
|
rcu_seq_end(&rtp->tasks_gp_seq);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Invoke callbacks.
|
|
|
|
|
set_tasks_gp_state(rtp, RTGS_INVOKE_CBS);
|
|
|
|
|
rcu_tasks_invoke_cbs(rtp, per_cpu_ptr(rtp->rtpcpu, 0));
|
|
|
|
|
mutex_unlock(&rtp->tasks_gp_mutex);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// RCU-tasks kthread that detects grace periods and invokes callbacks.
|
|
|
|
|
static int __noreturn rcu_tasks_kthread(void *arg)
|
|
|
|
|
{
|
rcu-tasks: Treat only synchronous grace periods urgently
The performance requirements on RCU Tasks, and in particular on RCU
Tasks Trace, have evolved over time as the workloads have evolved.
The current implementation is designed to provide low grace-period
latencies, and also to accommodate short-duration floods of callbacks.
However, current workloads can also provide a constant background
callback-queuing rate of a few hundred call_rcu_tasks_trace() invocations
per second. This results in continuous back-to-back RCU Tasks Trace
grace periods, which in turn can consume the better part of 10% of a CPU.
One could take the attitude that there are several tens of other CPUs on
the systems running such workloads, but energy efficiency is a thing.
On these systems, although asynchronous grace-period requests happen
every few milliseconds, synchronous grace-period requests are quite rare.
This commit therefore arrranges for grace periods to be initiated
immediately in response to calls to synchronize_rcu_tasks*() and
also to calls to synchronize_rcu_mult() that are passed one of the
call_rcu_tasks*() functions. These are recognized by the tell-tale
wakeme_after_rcu callback function.
In other cases, callbacks are gathered up for up to about 250 milliseconds
before a grace period is initiated. This results in more than an order of
magnitude reduction in RCU Tasks Trace grace periods, with corresponding
reduction in consumption of CPU time.
Reported-by: Alexei Starovoitov <ast@kernel.org>
Reported-by: Martin KaFai Lau <kafai@fb.com>
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
2023-05-14 18:06:12 -07:00
|
|
|
int cpu;
|
2020-03-02 16:16:57 -07:00
|
|
|
struct rcu_tasks *rtp = arg;
|
2020-03-02 12:59:20 -07:00
|
|
|
|
rcu-tasks: Treat only synchronous grace periods urgently
The performance requirements on RCU Tasks, and in particular on RCU
Tasks Trace, have evolved over time as the workloads have evolved.
The current implementation is designed to provide low grace-period
latencies, and also to accommodate short-duration floods of callbacks.
However, current workloads can also provide a constant background
callback-queuing rate of a few hundred call_rcu_tasks_trace() invocations
per second. This results in continuous back-to-back RCU Tasks Trace
grace periods, which in turn can consume the better part of 10% of a CPU.
One could take the attitude that there are several tens of other CPUs on
the systems running such workloads, but energy efficiency is a thing.
On these systems, although asynchronous grace-period requests happen
every few milliseconds, synchronous grace-period requests are quite rare.
This commit therefore arrranges for grace periods to be initiated
immediately in response to calls to synchronize_rcu_tasks*() and
also to calls to synchronize_rcu_mult() that are passed one of the
call_rcu_tasks*() functions. These are recognized by the tell-tale
wakeme_after_rcu callback function.
In other cases, callbacks are gathered up for up to about 250 milliseconds
before a grace period is initiated. This results in more than an order of
magnitude reduction in RCU Tasks Trace grace periods, with corresponding
reduction in consumption of CPU time.
Reported-by: Alexei Starovoitov <ast@kernel.org>
Reported-by: Martin KaFai Lau <kafai@fb.com>
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
2023-05-14 18:06:12 -07:00
|
|
|
for_each_possible_cpu(cpu) {
|
|
|
|
|
struct rcu_tasks_percpu *rtpcp = per_cpu_ptr(rtp->rtpcpu, cpu);
|
|
|
|
|
|
|
|
|
|
timer_setup(&rtpcp->lazy_timer, call_rcu_tasks_generic_timer, 0);
|
|
|
|
|
rtpcp->urgent_gp = 1;
|
|
|
|
|
}
|
|
|
|
|
|
2020-03-02 12:59:20 -07:00
|
|
|
/* Run on housekeeping CPUs by default. Sysadm can move if desired. */
|
2022-02-07 08:59:06 -07:00
|
|
|
housekeeping_affine(current, HK_TYPE_RCU);
|
rcu-tasks: Treat only synchronous grace periods urgently
The performance requirements on RCU Tasks, and in particular on RCU
Tasks Trace, have evolved over time as the workloads have evolved.
The current implementation is designed to provide low grace-period
latencies, and also to accommodate short-duration floods of callbacks.
However, current workloads can also provide a constant background
callback-queuing rate of a few hundred call_rcu_tasks_trace() invocations
per second. This results in continuous back-to-back RCU Tasks Trace
grace periods, which in turn can consume the better part of 10% of a CPU.
One could take the attitude that there are several tens of other CPUs on
the systems running such workloads, but energy efficiency is a thing.
On these systems, although asynchronous grace-period requests happen
every few milliseconds, synchronous grace-period requests are quite rare.
This commit therefore arrranges for grace periods to be initiated
immediately in response to calls to synchronize_rcu_tasks*() and
also to calls to synchronize_rcu_mult() that are passed one of the
call_rcu_tasks*() functions. These are recognized by the tell-tale
wakeme_after_rcu callback function.
In other cases, callbacks are gathered up for up to about 250 milliseconds
before a grace period is initiated. This results in more than an order of
magnitude reduction in RCU Tasks Trace grace periods, with corresponding
reduction in consumption of CPU time.
Reported-by: Alexei Starovoitov <ast@kernel.org>
Reported-by: Martin KaFai Lau <kafai@fb.com>
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
2023-05-14 18:06:12 -07:00
|
|
|
smp_store_release(&rtp->kthread_ptr, current); // Let GPs start!
|
2020-03-02 12:59:20 -07:00
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* Each pass through the following loop makes one check for
|
|
|
|
|
* newly arrived callbacks, and, if there are some, waits for
|
|
|
|
|
* one RCU-tasks grace period and then invokes the callbacks.
|
|
|
|
|
* This loop is terminated by the system going down. ;-)
|
|
|
|
|
*/
|
|
|
|
|
for (;;) {
|
2022-04-19 10:47:28 -07:00
|
|
|
// Wait for one grace period and invoke any callbacks
|
|
|
|
|
// that are ready.
|
2022-04-19 15:41:38 -07:00
|
|
|
rcu_tasks_one_gp(rtp, false);
|
2021-11-09 16:34:56 -07:00
|
|
|
|
2022-04-19 10:47:28 -07:00
|
|
|
// Paranoid sleep to keep this from entering a tight loop.
|
2020-09-09 22:05:41 -07:00
|
|
|
schedule_timeout_idle(rtp->gp_sleep);
|
2020-03-02 12:59:20 -07:00
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2022-04-19 11:06:03 -07:00
|
|
|
// Wait for a grace period for the specified flavor of Tasks RCU.
|
|
|
|
|
static void synchronize_rcu_tasks_generic(struct rcu_tasks *rtp)
|
|
|
|
|
{
|
|
|
|
|
/* Complain if the scheduler has not started. */
|
rcu-tasks: Make rude RCU-Tasks work well with CPU hotplug
The synchronize_rcu_tasks_rude() function invokes rcu_tasks_rude_wait_gp()
to wait one rude RCU-tasks grace period. The rcu_tasks_rude_wait_gp()
function in turn checks if there is only a single online CPU. If so, it
will immediately return, because a call to synchronize_rcu_tasks_rude()
is by definition a grace period on a single-CPU system. (We could
have blocked!)
Unfortunately, this check uses num_online_cpus() without synchronization,
which can result in too-short grace periods. To see this, consider the
following scenario:
CPU0 CPU1 (going offline)
migration/1 task:
cpu_stopper_thread
-> take_cpu_down
-> _cpu_disable
(dec __num_online_cpus)
->cpuhp_invoke_callback
preempt_disable
access old_data0
task1
del old_data0 .....
synchronize_rcu_tasks_rude()
task1 schedule out
....
task2 schedule in
rcu_tasks_rude_wait_gp()
->__num_online_cpus == 1
->return
....
task1 schedule in
->free old_data0
preempt_enable
When CPU1 decrements __num_online_cpus, its value becomes 1. However,
CPU1 has not finished going offline, and will take one last trip through
the scheduler and the idle loop before it actually stops executing
instructions. Because synchronize_rcu_tasks_rude() is mostly used for
tracing, and because both the scheduler and the idle loop can be traced,
this means that CPU0's prematurely ended grace period might disrupt the
tracing on CPU1. Given that this disruption might include CPU1 executing
instructions in memory that was just now freed (and maybe reallocated),
this is a matter of some concern.
This commit therefore removes that problematic single-CPU check from the
rcu_tasks_rude_wait_gp() function. This dispenses with the single-CPU
optimization, but there is no evidence indicating that this optimization
is important. In addition, synchronize_rcu_tasks_generic() contains a
similar optimization (albeit only for early boot), which also splats.
(As in exactly why are you invoking synchronize_rcu_tasks_rude() so
early in boot, anyway???)
It is OK for the synchronize_rcu_tasks_rude() function's check to be
unsynchronized because the only times that this check can evaluate to
true is when there is only a single CPU running with preemption
disabled.
While in the area, this commit also fixes a minor bug in which a
call to synchronize_rcu_tasks_rude() would instead be attributed to
synchronize_rcu_tasks().
[ paulmck: Add "synchronize_" prefix and "()" suffix. ]
Signed-off-by: Zqiang <qiang1.zhang@intel.com>
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
2022-11-30 16:45:33 -07:00
|
|
|
if (WARN_ONCE(rcu_scheduler_active == RCU_SCHEDULER_INACTIVE,
|
|
|
|
|
"synchronize_%s() called too soon", rtp->name))
|
|
|
|
|
return;
|
2022-04-19 11:06:03 -07:00
|
|
|
|
2022-04-19 15:41:38 -07:00
|
|
|
// If the grace-period kthread is running, use it.
|
|
|
|
|
if (READ_ONCE(rtp->kthread_ptr)) {
|
2024-02-23 16:16:20 -07:00
|
|
|
wait_rcu_gp_state(rtp->wait_state, rtp->call_func);
|
2022-04-19 15:41:38 -07:00
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
rcu_tasks_one_gp(rtp, true);
|
2022-04-19 11:06:03 -07:00
|
|
|
}
|
|
|
|
|
|
2020-12-09 13:27:31 -07:00
|
|
|
/* Spawn RCU-tasks grace-period kthread. */
|
2020-03-03 12:49:21 -07:00
|
|
|
static void __init rcu_spawn_tasks_kthread_generic(struct rcu_tasks *rtp)
|
2020-03-02 12:59:20 -07:00
|
|
|
{
|
|
|
|
|
struct task_struct *t;
|
|
|
|
|
|
2020-03-03 16:50:31 -07:00
|
|
|
t = kthread_run(rcu_tasks_kthread, rtp, "%s_kthread", rtp->kname);
|
|
|
|
|
if (WARN_ONCE(IS_ERR(t), "%s: Could not start %s grace-period kthread, OOM is now expected behavior\n", __func__, rtp->name))
|
2020-03-03 12:49:21 -07:00
|
|
|
return;
|
2020-03-02 12:59:20 -07:00
|
|
|
smp_mb(); /* Ensure others see full kthread. */
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
#ifndef CONFIG_TINY_RCU
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* Print any non-default Tasks RCU settings.
|
|
|
|
|
*/
|
|
|
|
|
static void __init rcu_tasks_bootup_oddness(void)
|
|
|
|
|
{
|
rcu-tasks: Add an RCU Tasks Trace to simplify protection of tracing hooks
Because RCU does not watch exception early-entry/late-exit, idle-loop,
or CPU-hotplug execution, protection of tracing and BPF operations is
needlessly complicated. This commit therefore adds a variant of
Tasks RCU that:
o Has explicit read-side markers to allow finite grace periods in
the face of in-kernel loops for PREEMPT=n builds. These markers
are rcu_read_lock_trace() and rcu_read_unlock_trace().
o Protects code in the idle loop, exception entry/exit, and
CPU-hotplug code paths. In this respect, RCU-tasks trace is
similar to SRCU, but with lighter-weight readers.
o Avoids expensive read-side instruction, having overhead similar
to that of Preemptible RCU.
There are of course downsides:
o The grace-period code can send IPIs to CPUs, even when those
CPUs are in the idle loop or in nohz_full userspace. This is
mitigated by later commits.
o It is necessary to scan the full tasklist, much as for Tasks RCU.
o There is a single callback queue guarded by a single lock,
again, much as for Tasks RCU. However, those early use cases
that request multiple grace periods in quick succession are
expected to do so from a single task, which makes the single
lock almost irrelevant. If needed, multiple callback queues
can be provided using any number of schemes.
Perhaps most important, this variant of RCU does not affect the vanilla
flavors, rcu_preempt and rcu_sched. The fact that RCU Tasks Trace
readers can operate from idle, offline, and exception entry/exit in no
way enables rcu_preempt and rcu_sched readers to do so.
The memory ordering was outlined here:
https://lore.kernel.org/lkml/20200319034030.GX3199@paulmck-ThinkPad-P72/
This effort benefited greatly from off-list discussions of BPF
requirements with Alexei Starovoitov and Andrii Nakryiko. At least
some of the on-list discussions are captured in the Link: tags below.
In addition, KCSAN was quite helpful in finding some early bugs.
Link: https://lore.kernel.org/lkml/20200219150744.428764577@infradead.org/
Link: https://lore.kernel.org/lkml/87mu8p797b.fsf@nanos.tec.linutronix.de/
Link: https://lore.kernel.org/lkml/20200225221305.605144982@linutronix.de/
Cc: Alexei Starovoitov <alexei.starovoitov@gmail.com>
Cc: Andrii Nakryiko <andriin@fb.com>
[ paulmck: Apply feedback from Steve Rostedt and Joel Fernandes. ]
[ paulmck: Decrement trc_n_readers_need_end upon IPI failure. ]
[ paulmck: Fix locking issue reported by rcutorture. ]
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
2020-03-09 19:56:53 -07:00
|
|
|
#if defined(CONFIG_TASKS_RCU) || defined(CONFIG_TASKS_TRACE_RCU)
|
rcu-tasks: Print pre-stall-warning informational messages
RCU-tasks stall-warning messages are printed after the grace period is ten
minutes old. Unfortunately, most of us will have rebooted the system in
response to an apparently-hung command long before the ten minutes is up,
and will thus see what looks to be a silent hang.
This commit therefore adds pr_info() messages that are printed earlier.
These should avoid being classified as errors, but should give impatient
users a hint. These are controlled by new rcupdate.rcu_task_stall_info
and rcupdate.rcu_task_stall_info_mult kernel-boot parameters. The former
defines the initial delay in jiffies (defaulting to 10 seconds) and the
latter defines the multiplier (defaulting to 3). Thus, by default, the
first message will appear 10 seconds into the RCU-tasks grace period,
the second 40 seconds in, and the third 160 seconds in. There would be
a fourth at 640 seconds in, but the stall warning message appears 600
seconds in, and once a stall warning is printed for a given grace period,
no further informational messages are printed.
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
2022-02-25 17:01:12 -07:00
|
|
|
int rtsimc;
|
|
|
|
|
|
2020-03-02 12:59:20 -07:00
|
|
|
if (rcu_task_stall_timeout != RCU_TASK_STALL_TIMEOUT)
|
|
|
|
|
pr_info("\tTasks-RCU CPU stall warnings timeout set to %d (rcu_task_stall_timeout).\n", rcu_task_stall_timeout);
|
rcu-tasks: Print pre-stall-warning informational messages
RCU-tasks stall-warning messages are printed after the grace period is ten
minutes old. Unfortunately, most of us will have rebooted the system in
response to an apparently-hung command long before the ten minutes is up,
and will thus see what looks to be a silent hang.
This commit therefore adds pr_info() messages that are printed earlier.
These should avoid being classified as errors, but should give impatient
users a hint. These are controlled by new rcupdate.rcu_task_stall_info
and rcupdate.rcu_task_stall_info_mult kernel-boot parameters. The former
defines the initial delay in jiffies (defaulting to 10 seconds) and the
latter defines the multiplier (defaulting to 3). Thus, by default, the
first message will appear 10 seconds into the RCU-tasks grace period,
the second 40 seconds in, and the third 160 seconds in. There would be
a fourth at 640 seconds in, but the stall warning message appears 600
seconds in, and once a stall warning is printed for a given grace period,
no further informational messages are printed.
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
2022-02-25 17:01:12 -07:00
|
|
|
rtsimc = clamp(rcu_task_stall_info_mult, 1, 10);
|
|
|
|
|
if (rtsimc != rcu_task_stall_info_mult) {
|
|
|
|
|
pr_info("\tTasks-RCU CPU stall info multiplier clamped to %d (rcu_task_stall_info_mult).\n", rtsimc);
|
|
|
|
|
rcu_task_stall_info_mult = rtsimc;
|
|
|
|
|
}
|
rcu-tasks: Add an RCU Tasks Trace to simplify protection of tracing hooks
Because RCU does not watch exception early-entry/late-exit, idle-loop,
or CPU-hotplug execution, protection of tracing and BPF operations is
needlessly complicated. This commit therefore adds a variant of
Tasks RCU that:
o Has explicit read-side markers to allow finite grace periods in
the face of in-kernel loops for PREEMPT=n builds. These markers
are rcu_read_lock_trace() and rcu_read_unlock_trace().
o Protects code in the idle loop, exception entry/exit, and
CPU-hotplug code paths. In this respect, RCU-tasks trace is
similar to SRCU, but with lighter-weight readers.
o Avoids expensive read-side instruction, having overhead similar
to that of Preemptible RCU.
There are of course downsides:
o The grace-period code can send IPIs to CPUs, even when those
CPUs are in the idle loop or in nohz_full userspace. This is
mitigated by later commits.
o It is necessary to scan the full tasklist, much as for Tasks RCU.
o There is a single callback queue guarded by a single lock,
again, much as for Tasks RCU. However, those early use cases
that request multiple grace periods in quick succession are
expected to do so from a single task, which makes the single
lock almost irrelevant. If needed, multiple callback queues
can be provided using any number of schemes.
Perhaps most important, this variant of RCU does not affect the vanilla
flavors, rcu_preempt and rcu_sched. The fact that RCU Tasks Trace
readers can operate from idle, offline, and exception entry/exit in no
way enables rcu_preempt and rcu_sched readers to do so.
The memory ordering was outlined here:
https://lore.kernel.org/lkml/20200319034030.GX3199@paulmck-ThinkPad-P72/
This effort benefited greatly from off-list discussions of BPF
requirements with Alexei Starovoitov and Andrii Nakryiko. At least
some of the on-list discussions are captured in the Link: tags below.
In addition, KCSAN was quite helpful in finding some early bugs.
Link: https://lore.kernel.org/lkml/20200219150744.428764577@infradead.org/
Link: https://lore.kernel.org/lkml/87mu8p797b.fsf@nanos.tec.linutronix.de/
Link: https://lore.kernel.org/lkml/20200225221305.605144982@linutronix.de/
Cc: Alexei Starovoitov <alexei.starovoitov@gmail.com>
Cc: Andrii Nakryiko <andriin@fb.com>
[ paulmck: Apply feedback from Steve Rostedt and Joel Fernandes. ]
[ paulmck: Decrement trc_n_readers_need_end upon IPI failure. ]
[ paulmck: Fix locking issue reported by rcutorture. ]
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
2020-03-09 19:56:53 -07:00
|
|
|
#endif /* #ifdef CONFIG_TASKS_RCU */
|
|
|
|
|
#ifdef CONFIG_TASKS_RCU
|
|
|
|
|
pr_info("\tTrampoline variant of Tasks RCU enabled.\n");
|
2020-03-02 12:59:20 -07:00
|
|
|
#endif /* #ifdef CONFIG_TASKS_RCU */
|
2020-03-02 22:06:43 -07:00
|
|
|
#ifdef CONFIG_TASKS_RUDE_RCU
|
|
|
|
|
pr_info("\tRude variant of Tasks RCU enabled.\n");
|
|
|
|
|
#endif /* #ifdef CONFIG_TASKS_RUDE_RCU */
|
rcu-tasks: Add an RCU Tasks Trace to simplify protection of tracing hooks
Because RCU does not watch exception early-entry/late-exit, idle-loop,
or CPU-hotplug execution, protection of tracing and BPF operations is
needlessly complicated. This commit therefore adds a variant of
Tasks RCU that:
o Has explicit read-side markers to allow finite grace periods in
the face of in-kernel loops for PREEMPT=n builds. These markers
are rcu_read_lock_trace() and rcu_read_unlock_trace().
o Protects code in the idle loop, exception entry/exit, and
CPU-hotplug code paths. In this respect, RCU-tasks trace is
similar to SRCU, but with lighter-weight readers.
o Avoids expensive read-side instruction, having overhead similar
to that of Preemptible RCU.
There are of course downsides:
o The grace-period code can send IPIs to CPUs, even when those
CPUs are in the idle loop or in nohz_full userspace. This is
mitigated by later commits.
o It is necessary to scan the full tasklist, much as for Tasks RCU.
o There is a single callback queue guarded by a single lock,
again, much as for Tasks RCU. However, those early use cases
that request multiple grace periods in quick succession are
expected to do so from a single task, which makes the single
lock almost irrelevant. If needed, multiple callback queues
can be provided using any number of schemes.
Perhaps most important, this variant of RCU does not affect the vanilla
flavors, rcu_preempt and rcu_sched. The fact that RCU Tasks Trace
readers can operate from idle, offline, and exception entry/exit in no
way enables rcu_preempt and rcu_sched readers to do so.
The memory ordering was outlined here:
https://lore.kernel.org/lkml/20200319034030.GX3199@paulmck-ThinkPad-P72/
This effort benefited greatly from off-list discussions of BPF
requirements with Alexei Starovoitov and Andrii Nakryiko. At least
some of the on-list discussions are captured in the Link: tags below.
In addition, KCSAN was quite helpful in finding some early bugs.
Link: https://lore.kernel.org/lkml/20200219150744.428764577@infradead.org/
Link: https://lore.kernel.org/lkml/87mu8p797b.fsf@nanos.tec.linutronix.de/
Link: https://lore.kernel.org/lkml/20200225221305.605144982@linutronix.de/
Cc: Alexei Starovoitov <alexei.starovoitov@gmail.com>
Cc: Andrii Nakryiko <andriin@fb.com>
[ paulmck: Apply feedback from Steve Rostedt and Joel Fernandes. ]
[ paulmck: Decrement trc_n_readers_need_end upon IPI failure. ]
[ paulmck: Fix locking issue reported by rcutorture. ]
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
2020-03-09 19:56:53 -07:00
|
|
|
#ifdef CONFIG_TASKS_TRACE_RCU
|
|
|
|
|
pr_info("\tTracing variant of Tasks RCU enabled.\n");
|
|
|
|
|
#endif /* #ifdef CONFIG_TASKS_TRACE_RCU */
|
2020-03-02 12:59:20 -07:00
|
|
|
}
|
|
|
|
|
|
2020-03-03 12:49:21 -07:00
|
|
|
|
2020-03-16 11:01:55 -07:00
|
|
|
/* Dump out rcutorture-relevant state common to all RCU-tasks flavors. */
|
|
|
|
|
static void show_rcu_tasks_generic_gp_kthread(struct rcu_tasks *rtp, char *s)
|
|
|
|
|
{
|
2022-03-28 16:00:05 -07:00
|
|
|
int cpu;
|
|
|
|
|
bool havecbs = false;
|
rcu-tasks: Treat only synchronous grace periods urgently
The performance requirements on RCU Tasks, and in particular on RCU
Tasks Trace, have evolved over time as the workloads have evolved.
The current implementation is designed to provide low grace-period
latencies, and also to accommodate short-duration floods of callbacks.
However, current workloads can also provide a constant background
callback-queuing rate of a few hundred call_rcu_tasks_trace() invocations
per second. This results in continuous back-to-back RCU Tasks Trace
grace periods, which in turn can consume the better part of 10% of a CPU.
One could take the attitude that there are several tens of other CPUs on
the systems running such workloads, but energy efficiency is a thing.
On these systems, although asynchronous grace-period requests happen
every few milliseconds, synchronous grace-period requests are quite rare.
This commit therefore arrranges for grace periods to be initiated
immediately in response to calls to synchronize_rcu_tasks*() and
also to calls to synchronize_rcu_mult() that are passed one of the
call_rcu_tasks*() functions. These are recognized by the tell-tale
wakeme_after_rcu callback function.
In other cases, callbacks are gathered up for up to about 250 milliseconds
before a grace period is initiated. This results in more than an order of
magnitude reduction in RCU Tasks Trace grace periods, with corresponding
reduction in consumption of CPU time.
Reported-by: Alexei Starovoitov <ast@kernel.org>
Reported-by: Martin KaFai Lau <kafai@fb.com>
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
2023-05-14 18:06:12 -07:00
|
|
|
bool haveurgent = false;
|
|
|
|
|
bool haveurgentcbs = false;
|
2022-03-28 16:00:05 -07:00
|
|
|
|
|
|
|
|
for_each_possible_cpu(cpu) {
|
|
|
|
|
struct rcu_tasks_percpu *rtpcp = per_cpu_ptr(rtp->rtpcpu, cpu);
|
|
|
|
|
|
rcu-tasks: Treat only synchronous grace periods urgently
The performance requirements on RCU Tasks, and in particular on RCU
Tasks Trace, have evolved over time as the workloads have evolved.
The current implementation is designed to provide low grace-period
latencies, and also to accommodate short-duration floods of callbacks.
However, current workloads can also provide a constant background
callback-queuing rate of a few hundred call_rcu_tasks_trace() invocations
per second. This results in continuous back-to-back RCU Tasks Trace
grace periods, which in turn can consume the better part of 10% of a CPU.
One could take the attitude that there are several tens of other CPUs on
the systems running such workloads, but energy efficiency is a thing.
On these systems, although asynchronous grace-period requests happen
every few milliseconds, synchronous grace-period requests are quite rare.
This commit therefore arrranges for grace periods to be initiated
immediately in response to calls to synchronize_rcu_tasks*() and
also to calls to synchronize_rcu_mult() that are passed one of the
call_rcu_tasks*() functions. These are recognized by the tell-tale
wakeme_after_rcu callback function.
In other cases, callbacks are gathered up for up to about 250 milliseconds
before a grace period is initiated. This results in more than an order of
magnitude reduction in RCU Tasks Trace grace periods, with corresponding
reduction in consumption of CPU time.
Reported-by: Alexei Starovoitov <ast@kernel.org>
Reported-by: Martin KaFai Lau <kafai@fb.com>
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
2023-05-14 18:06:12 -07:00
|
|
|
if (!data_race(rcu_segcblist_empty(&rtpcp->cblist)))
|
2022-03-28 16:00:05 -07:00
|
|
|
havecbs = true;
|
rcu-tasks: Treat only synchronous grace periods urgently
The performance requirements on RCU Tasks, and in particular on RCU
Tasks Trace, have evolved over time as the workloads have evolved.
The current implementation is designed to provide low grace-period
latencies, and also to accommodate short-duration floods of callbacks.
However, current workloads can also provide a constant background
callback-queuing rate of a few hundred call_rcu_tasks_trace() invocations
per second. This results in continuous back-to-back RCU Tasks Trace
grace periods, which in turn can consume the better part of 10% of a CPU.
One could take the attitude that there are several tens of other CPUs on
the systems running such workloads, but energy efficiency is a thing.
On these systems, although asynchronous grace-period requests happen
every few milliseconds, synchronous grace-period requests are quite rare.
This commit therefore arrranges for grace periods to be initiated
immediately in response to calls to synchronize_rcu_tasks*() and
also to calls to synchronize_rcu_mult() that are passed one of the
call_rcu_tasks*() functions. These are recognized by the tell-tale
wakeme_after_rcu callback function.
In other cases, callbacks are gathered up for up to about 250 milliseconds
before a grace period is initiated. This results in more than an order of
magnitude reduction in RCU Tasks Trace grace periods, with corresponding
reduction in consumption of CPU time.
Reported-by: Alexei Starovoitov <ast@kernel.org>
Reported-by: Martin KaFai Lau <kafai@fb.com>
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
2023-05-14 18:06:12 -07:00
|
|
|
if (data_race(rtpcp->urgent_gp))
|
|
|
|
|
haveurgent = true;
|
|
|
|
|
if (!data_race(rcu_segcblist_empty(&rtpcp->cblist)) && data_race(rtpcp->urgent_gp))
|
|
|
|
|
haveurgentcbs = true;
|
|
|
|
|
if (havecbs && haveurgent && haveurgentcbs)
|
2022-03-28 16:00:05 -07:00
|
|
|
break;
|
|
|
|
|
}
|
rcu-tasks: Treat only synchronous grace periods urgently
The performance requirements on RCU Tasks, and in particular on RCU
Tasks Trace, have evolved over time as the workloads have evolved.
The current implementation is designed to provide low grace-period
latencies, and also to accommodate short-duration floods of callbacks.
However, current workloads can also provide a constant background
callback-queuing rate of a few hundred call_rcu_tasks_trace() invocations
per second. This results in continuous back-to-back RCU Tasks Trace
grace periods, which in turn can consume the better part of 10% of a CPU.
One could take the attitude that there are several tens of other CPUs on
the systems running such workloads, but energy efficiency is a thing.
On these systems, although asynchronous grace-period requests happen
every few milliseconds, synchronous grace-period requests are quite rare.
This commit therefore arrranges for grace periods to be initiated
immediately in response to calls to synchronize_rcu_tasks*() and
also to calls to synchronize_rcu_mult() that are passed one of the
call_rcu_tasks*() functions. These are recognized by the tell-tale
wakeme_after_rcu callback function.
In other cases, callbacks are gathered up for up to about 250 milliseconds
before a grace period is initiated. This results in more than an order of
magnitude reduction in RCU Tasks Trace grace periods, with corresponding
reduction in consumption of CPU time.
Reported-by: Alexei Starovoitov <ast@kernel.org>
Reported-by: Martin KaFai Lau <kafai@fb.com>
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
2023-05-14 18:06:12 -07:00
|
|
|
pr_info("%s: %s(%d) since %lu g:%lu i:%lu/%lu %c%c%c%c l:%lu %s\n",
|
2020-03-16 11:01:55 -07:00
|
|
|
rtp->kname,
|
2020-03-25 14:36:05 -07:00
|
|
|
tasks_gp_state_getname(rtp), data_race(rtp->gp_state),
|
2020-03-16 12:13:33 -07:00
|
|
|
jiffies - data_race(rtp->gp_jiffies),
|
2021-11-08 15:14:43 -07:00
|
|
|
data_race(rcu_seq_current(&rtp->tasks_gp_seq)),
|
2020-03-25 14:36:05 -07:00
|
|
|
data_race(rtp->n_ipis_fails), data_race(rtp->n_ipis),
|
2020-03-16 11:01:55 -07:00
|
|
|
".k"[!!data_race(rtp->kthread_ptr)],
|
2022-03-28 16:00:05 -07:00
|
|
|
".C"[havecbs],
|
rcu-tasks: Treat only synchronous grace periods urgently
The performance requirements on RCU Tasks, and in particular on RCU
Tasks Trace, have evolved over time as the workloads have evolved.
The current implementation is designed to provide low grace-period
latencies, and also to accommodate short-duration floods of callbacks.
However, current workloads can also provide a constant background
callback-queuing rate of a few hundred call_rcu_tasks_trace() invocations
per second. This results in continuous back-to-back RCU Tasks Trace
grace periods, which in turn can consume the better part of 10% of a CPU.
One could take the attitude that there are several tens of other CPUs on
the systems running such workloads, but energy efficiency is a thing.
On these systems, although asynchronous grace-period requests happen
every few milliseconds, synchronous grace-period requests are quite rare.
This commit therefore arrranges for grace periods to be initiated
immediately in response to calls to synchronize_rcu_tasks*() and
also to calls to synchronize_rcu_mult() that are passed one of the
call_rcu_tasks*() functions. These are recognized by the tell-tale
wakeme_after_rcu callback function.
In other cases, callbacks are gathered up for up to about 250 milliseconds
before a grace period is initiated. This results in more than an order of
magnitude reduction in RCU Tasks Trace grace periods, with corresponding
reduction in consumption of CPU time.
Reported-by: Alexei Starovoitov <ast@kernel.org>
Reported-by: Martin KaFai Lau <kafai@fb.com>
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
2023-05-14 18:06:12 -07:00
|
|
|
".u"[haveurgent],
|
|
|
|
|
".U"[haveurgentcbs],
|
|
|
|
|
rtp->lazy_jiffies,
|
2020-03-16 11:01:55 -07:00
|
|
|
s);
|
|
|
|
|
}
|
2024-08-01 17:34:25 -07:00
|
|
|
|
|
|
|
|
/* Dump out more rcutorture-relevant state common to all RCU-tasks flavors. */
|
|
|
|
|
static void rcu_tasks_torture_stats_print_generic(struct rcu_tasks *rtp, char *tt,
|
|
|
|
|
char *tf, char *tst)
|
|
|
|
|
{
|
|
|
|
|
cpumask_var_t cm;
|
|
|
|
|
int cpu;
|
|
|
|
|
bool gotcb = false;
|
|
|
|
|
unsigned long j = jiffies;
|
|
|
|
|
|
|
|
|
|
pr_alert("%s%s Tasks%s RCU g%ld gp_start %lu gp_jiffies %lu gp_state %d (%s).\n",
|
|
|
|
|
tt, tf, tst, data_race(rtp->tasks_gp_seq),
|
|
|
|
|
j - data_race(rtp->gp_start), j - data_race(rtp->gp_jiffies),
|
|
|
|
|
data_race(rtp->gp_state), tasks_gp_state_getname(rtp));
|
|
|
|
|
pr_alert("\tEnqueue shift %d limit %d Dequeue limit %d gpseq %lu.\n",
|
|
|
|
|
data_race(rtp->percpu_enqueue_shift),
|
|
|
|
|
data_race(rtp->percpu_enqueue_lim),
|
|
|
|
|
data_race(rtp->percpu_dequeue_lim),
|
|
|
|
|
data_race(rtp->percpu_dequeue_gpseq));
|
|
|
|
|
(void)zalloc_cpumask_var(&cm, GFP_KERNEL);
|
|
|
|
|
pr_alert("\tCallback counts:");
|
|
|
|
|
for_each_possible_cpu(cpu) {
|
|
|
|
|
long n;
|
|
|
|
|
struct rcu_tasks_percpu *rtpcp = per_cpu_ptr(rtp->rtpcpu, cpu);
|
|
|
|
|
|
|
|
|
|
if (cpumask_available(cm) && !rcu_barrier_cb_is_done(&rtpcp->barrier_q_head))
|
|
|
|
|
cpumask_set_cpu(cpu, cm);
|
|
|
|
|
n = rcu_segcblist_n_cbs(&rtpcp->cblist);
|
|
|
|
|
if (!n)
|
|
|
|
|
continue;
|
|
|
|
|
pr_cont(" %d:%ld", cpu, n);
|
|
|
|
|
gotcb = true;
|
|
|
|
|
}
|
|
|
|
|
if (gotcb)
|
|
|
|
|
pr_cont(".\n");
|
|
|
|
|
else
|
|
|
|
|
pr_cont(" (none).\n");
|
2024-08-01 17:34:26 -07:00
|
|
|
pr_alert("\tBarrier seq %lu start %lu count %d holdout CPUs ",
|
|
|
|
|
data_race(rtp->barrier_q_seq), j - data_race(rtp->barrier_q_start),
|
|
|
|
|
atomic_read(&rtp->barrier_q_count));
|
2024-08-01 17:34:25 -07:00
|
|
|
if (cpumask_available(cm) && !cpumask_empty(cm))
|
|
|
|
|
pr_cont(" %*pbl.\n", cpumask_pr_args(cm));
|
|
|
|
|
else
|
|
|
|
|
pr_cont("(none).\n");
|
|
|
|
|
free_cpumask_var(cm);
|
|
|
|
|
}
|
|
|
|
|
|
2020-09-15 17:08:03 -07:00
|
|
|
#endif // #ifndef CONFIG_TINY_RCU
|
2020-03-16 11:01:55 -07:00
|
|
|
|
2020-04-05 20:49:13 -07:00
|
|
|
static void exit_tasks_rcu_finish_trace(struct task_struct *t);
|
|
|
|
|
|
|
|
|
|
#if defined(CONFIG_TASKS_RCU) || defined(CONFIG_TASKS_TRACE_RCU)
|
2020-03-03 12:49:21 -07:00
|
|
|
|
2020-03-05 18:07:07 -07:00
|
|
|
////////////////////////////////////////////////////////////////////////
|
|
|
|
|
//
|
|
|
|
|
// Shared code between task-list-scanning variants of Tasks RCU.
|
|
|
|
|
|
|
|
|
|
/* Wait for one RCU-tasks grace period. */
|
|
|
|
|
static void rcu_tasks_wait_gp(struct rcu_tasks *rtp)
|
|
|
|
|
{
|
rcu-tasks: Print pre-stall-warning informational messages
RCU-tasks stall-warning messages are printed after the grace period is ten
minutes old. Unfortunately, most of us will have rebooted the system in
response to an apparently-hung command long before the ten minutes is up,
and will thus see what looks to be a silent hang.
This commit therefore adds pr_info() messages that are printed earlier.
These should avoid being classified as errors, but should give impatient
users a hint. These are controlled by new rcupdate.rcu_task_stall_info
and rcupdate.rcu_task_stall_info_mult kernel-boot parameters. The former
defines the initial delay in jiffies (defaulting to 10 seconds) and the
latter defines the multiplier (defaulting to 3). Thus, by default, the
first message will appear 10 seconds into the RCU-tasks grace period,
the second 40 seconds in, and the third 160 seconds in. There would be
a fourth at 640 seconds in, but the stall warning message appears 600
seconds in, and once a stall warning is printed for a given grace period,
no further informational messages are printed.
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
2022-02-25 17:01:12 -07:00
|
|
|
struct task_struct *g;
|
2020-03-05 18:07:07 -07:00
|
|
|
int fract;
|
rcu-tasks: Print pre-stall-warning informational messages
RCU-tasks stall-warning messages are printed after the grace period is ten
minutes old. Unfortunately, most of us will have rebooted the system in
response to an apparently-hung command long before the ten minutes is up,
and will thus see what looks to be a silent hang.
This commit therefore adds pr_info() messages that are printed earlier.
These should avoid being classified as errors, but should give impatient
users a hint. These are controlled by new rcupdate.rcu_task_stall_info
and rcupdate.rcu_task_stall_info_mult kernel-boot parameters. The former
defines the initial delay in jiffies (defaulting to 10 seconds) and the
latter defines the multiplier (defaulting to 3). Thus, by default, the
first message will appear 10 seconds into the RCU-tasks grace period,
the second 40 seconds in, and the third 160 seconds in. There would be
a fourth at 640 seconds in, but the stall warning message appears 600
seconds in, and once a stall warning is printed for a given grace period,
no further informational messages are printed.
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
2022-02-25 17:01:12 -07:00
|
|
|
LIST_HEAD(holdouts);
|
|
|
|
|
unsigned long j;
|
|
|
|
|
unsigned long lastinfo;
|
|
|
|
|
unsigned long lastreport;
|
|
|
|
|
bool reported = false;
|
|
|
|
|
int rtsi;
|
|
|
|
|
struct task_struct *t;
|
2020-03-05 18:07:07 -07:00
|
|
|
|
2020-03-16 12:13:33 -07:00
|
|
|
set_tasks_gp_state(rtp, RTGS_PRE_WAIT_GP);
|
2022-05-18 16:06:55 -07:00
|
|
|
rtp->pregp_func(&holdouts);
|
2020-03-05 18:07:07 -07:00
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* There were callbacks, so we need to wait for an RCU-tasks
|
|
|
|
|
* grace period. Start off by scanning the task list for tasks
|
|
|
|
|
* that are not already voluntarily blocked. Mark these tasks
|
|
|
|
|
* and make a list of them in holdouts.
|
|
|
|
|
*/
|
2020-03-16 12:13:33 -07:00
|
|
|
set_tasks_gp_state(rtp, RTGS_SCAN_TASKLIST);
|
rcu-tasks: Stop RCU Tasks Trace from scanning full tasks list
This commit takes off the training wheels and relies only on scanning
currently running tasks and tasks that have blocked or been preempted
within their current RCU Tasks Trace read-side critical section.
Before this commit, the time complexity of an RCU Tasks Trace grace
period is O(T), where T is the number of tasks. After this commit,
this time complexity is O(C+B), where C is the number of CPUs and B
is the number of tasks that have blocked (or been preempted) at least
once during their current RCU Tasks Trace read-side critical sections.
Of course, if all tasks have blocked (or been preempted) at least once
during their current RCU Tasks Trace read-side critical sections, this is
still O(T), but current expectations are that RCU Tasks Trace read-side
critical section will be short and that there will normally not be large
numbers of tasks blocked within such a critical section.
Dave Marchevsky kindly measured the effects of this commit on the RCU
Tasks Trace grace-period latency and the rcu_tasks_trace_kthread task's
CPU consumption per RCU Tasks Trace grace period over the course of a
fixed test, all in milliseconds:
Before After
GP latency 22.3 ms stddev > 0.1 17.0 ms stddev < 0.1
GP CPU 2.3 ms stddev 0.3 1.1 ms stddev 0.2
This was on a system with 15,000 tasks, so it is reasonable to expect
much larger savings on the systems on which this issue was first noted,
given that they sport well in excess of 100,000 tasks. CPU consumption
was measured using profiling techniques.
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
Cc: Neeraj Upadhyay <quic_neeraju@quicinc.com>
Cc: Eric Dumazet <edumazet@google.com>
Cc: Alexei Starovoitov <ast@kernel.org>
Cc: Andrii Nakryiko <andrii@kernel.org>
Cc: Martin KaFai Lau <kafai@fb.com>
Cc: KP Singh <kpsingh@kernel.org>
Tested-by: Dave Marchevsky <davemarchevsky@fb.com>
2022-05-20 10:21:00 -07:00
|
|
|
if (rtp->pertask_func) {
|
|
|
|
|
rcu_read_lock();
|
|
|
|
|
for_each_process_thread(g, t)
|
|
|
|
|
rtp->pertask_func(t, &holdouts);
|
|
|
|
|
rcu_read_unlock();
|
|
|
|
|
}
|
2020-03-05 18:07:07 -07:00
|
|
|
|
2020-03-16 12:13:33 -07:00
|
|
|
set_tasks_gp_state(rtp, RTGS_POST_SCAN_TASKLIST);
|
2020-03-22 13:18:54 -07:00
|
|
|
rtp->postscan_func(&holdouts);
|
2020-03-05 18:07:07 -07:00
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* Each pass through the following loop scans the list of holdout
|
|
|
|
|
* tasks, removing any that are no longer holdouts. When the list
|
|
|
|
|
* is empty, we are done.
|
|
|
|
|
*/
|
|
|
|
|
lastreport = jiffies;
|
rcu-tasks: Print pre-stall-warning informational messages
RCU-tasks stall-warning messages are printed after the grace period is ten
minutes old. Unfortunately, most of us will have rebooted the system in
response to an apparently-hung command long before the ten minutes is up,
and will thus see what looks to be a silent hang.
This commit therefore adds pr_info() messages that are printed earlier.
These should avoid being classified as errors, but should give impatient
users a hint. These are controlled by new rcupdate.rcu_task_stall_info
and rcupdate.rcu_task_stall_info_mult kernel-boot parameters. The former
defines the initial delay in jiffies (defaulting to 10 seconds) and the
latter defines the multiplier (defaulting to 3). Thus, by default, the
first message will appear 10 seconds into the RCU-tasks grace period,
the second 40 seconds in, and the third 160 seconds in. There would be
a fourth at 640 seconds in, but the stall warning message appears 600
seconds in, and once a stall warning is printed for a given grace period,
no further informational messages are printed.
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
2022-02-25 17:01:12 -07:00
|
|
|
lastinfo = lastreport;
|
|
|
|
|
rtsi = READ_ONCE(rcu_task_stall_info);
|
2020-03-05 18:07:07 -07:00
|
|
|
|
2020-09-09 21:36:34 -07:00
|
|
|
// Start off with initial wait and slowly back off to 1 HZ wait.
|
|
|
|
|
fract = rtp->init_fract;
|
2020-03-05 18:07:07 -07:00
|
|
|
|
2020-09-15 15:41:50 -07:00
|
|
|
while (!list_empty(&holdouts)) {
|
2022-03-08 10:54:13 -07:00
|
|
|
ktime_t exp;
|
2020-03-05 18:07:07 -07:00
|
|
|
bool firstreport;
|
|
|
|
|
bool needreport;
|
|
|
|
|
int rtst;
|
|
|
|
|
|
rcu-tasks: Print pre-stall-warning informational messages
RCU-tasks stall-warning messages are printed after the grace period is ten
minutes old. Unfortunately, most of us will have rebooted the system in
response to an apparently-hung command long before the ten minutes is up,
and will thus see what looks to be a silent hang.
This commit therefore adds pr_info() messages that are printed earlier.
These should avoid being classified as errors, but should give impatient
users a hint. These are controlled by new rcupdate.rcu_task_stall_info
and rcupdate.rcu_task_stall_info_mult kernel-boot parameters. The former
defines the initial delay in jiffies (defaulting to 10 seconds) and the
latter defines the multiplier (defaulting to 3). Thus, by default, the
first message will appear 10 seconds into the RCU-tasks grace period,
the second 40 seconds in, and the third 160 seconds in. There would be
a fourth at 640 seconds in, but the stall warning message appears 600
seconds in, and once a stall warning is printed for a given grace period,
no further informational messages are printed.
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
2022-02-25 17:01:12 -07:00
|
|
|
// Slowly back off waiting for holdouts
|
2020-03-16 12:13:33 -07:00
|
|
|
set_tasks_gp_state(rtp, RTGS_WAIT_SCAN_HOLDOUTS);
|
2022-03-18 08:08:14 -07:00
|
|
|
if (!IS_ENABLED(CONFIG_PREEMPT_RT)) {
|
|
|
|
|
schedule_timeout_idle(fract);
|
|
|
|
|
} else {
|
|
|
|
|
exp = jiffies_to_nsecs(fract);
|
|
|
|
|
__set_current_state(TASK_IDLE);
|
|
|
|
|
schedule_hrtimeout_range(&exp, jiffies_to_nsecs(HZ / 2), HRTIMER_MODE_REL_HARD);
|
|
|
|
|
}
|
2020-03-05 18:07:07 -07:00
|
|
|
|
2020-09-17 16:17:17 -07:00
|
|
|
if (fract < HZ)
|
|
|
|
|
fract++;
|
2020-03-05 18:07:07 -07:00
|
|
|
|
|
|
|
|
rtst = READ_ONCE(rcu_task_stall_timeout);
|
|
|
|
|
needreport = rtst > 0 && time_after(jiffies, lastreport + rtst);
|
rcu-tasks: Print pre-stall-warning informational messages
RCU-tasks stall-warning messages are printed after the grace period is ten
minutes old. Unfortunately, most of us will have rebooted the system in
response to an apparently-hung command long before the ten minutes is up,
and will thus see what looks to be a silent hang.
This commit therefore adds pr_info() messages that are printed earlier.
These should avoid being classified as errors, but should give impatient
users a hint. These are controlled by new rcupdate.rcu_task_stall_info
and rcupdate.rcu_task_stall_info_mult kernel-boot parameters. The former
defines the initial delay in jiffies (defaulting to 10 seconds) and the
latter defines the multiplier (defaulting to 3). Thus, by default, the
first message will appear 10 seconds into the RCU-tasks grace period,
the second 40 seconds in, and the third 160 seconds in. There would be
a fourth at 640 seconds in, but the stall warning message appears 600
seconds in, and once a stall warning is printed for a given grace period,
no further informational messages are printed.
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
2022-02-25 17:01:12 -07:00
|
|
|
if (needreport) {
|
2020-03-05 18:07:07 -07:00
|
|
|
lastreport = jiffies;
|
rcu-tasks: Print pre-stall-warning informational messages
RCU-tasks stall-warning messages are printed after the grace period is ten
minutes old. Unfortunately, most of us will have rebooted the system in
response to an apparently-hung command long before the ten minutes is up,
and will thus see what looks to be a silent hang.
This commit therefore adds pr_info() messages that are printed earlier.
These should avoid being classified as errors, but should give impatient
users a hint. These are controlled by new rcupdate.rcu_task_stall_info
and rcupdate.rcu_task_stall_info_mult kernel-boot parameters. The former
defines the initial delay in jiffies (defaulting to 10 seconds) and the
latter defines the multiplier (defaulting to 3). Thus, by default, the
first message will appear 10 seconds into the RCU-tasks grace period,
the second 40 seconds in, and the third 160 seconds in. There would be
a fourth at 640 seconds in, but the stall warning message appears 600
seconds in, and once a stall warning is printed for a given grace period,
no further informational messages are printed.
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
2022-02-25 17:01:12 -07:00
|
|
|
reported = true;
|
|
|
|
|
}
|
2020-03-05 18:07:07 -07:00
|
|
|
firstreport = true;
|
|
|
|
|
WARN_ON(signal_pending(current));
|
2020-03-16 12:13:33 -07:00
|
|
|
set_tasks_gp_state(rtp, RTGS_SCAN_HOLDOUTS);
|
2020-03-05 18:07:07 -07:00
|
|
|
rtp->holdouts_func(&holdouts, needreport, &firstreport);
|
rcu-tasks: Print pre-stall-warning informational messages
RCU-tasks stall-warning messages are printed after the grace period is ten
minutes old. Unfortunately, most of us will have rebooted the system in
response to an apparently-hung command long before the ten minutes is up,
and will thus see what looks to be a silent hang.
This commit therefore adds pr_info() messages that are printed earlier.
These should avoid being classified as errors, but should give impatient
users a hint. These are controlled by new rcupdate.rcu_task_stall_info
and rcupdate.rcu_task_stall_info_mult kernel-boot parameters. The former
defines the initial delay in jiffies (defaulting to 10 seconds) and the
latter defines the multiplier (defaulting to 3). Thus, by default, the
first message will appear 10 seconds into the RCU-tasks grace period,
the second 40 seconds in, and the third 160 seconds in. There would be
a fourth at 640 seconds in, but the stall warning message appears 600
seconds in, and once a stall warning is printed for a given grace period,
no further informational messages are printed.
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
2022-02-25 17:01:12 -07:00
|
|
|
|
|
|
|
|
// Print pre-stall informational messages if needed.
|
|
|
|
|
j = jiffies;
|
|
|
|
|
if (rtsi > 0 && !reported && time_after(j, lastinfo + rtsi)) {
|
|
|
|
|
lastinfo = j;
|
|
|
|
|
rtsi = rtsi * rcu_task_stall_info_mult;
|
2022-09-29 11:58:10 -07:00
|
|
|
pr_info("%s: %s grace period number %lu (since boot) is %lu jiffies old.\n",
|
rcu-tasks: Print pre-stall-warning informational messages
RCU-tasks stall-warning messages are printed after the grace period is ten
minutes old. Unfortunately, most of us will have rebooted the system in
response to an apparently-hung command long before the ten minutes is up,
and will thus see what looks to be a silent hang.
This commit therefore adds pr_info() messages that are printed earlier.
These should avoid being classified as errors, but should give impatient
users a hint. These are controlled by new rcupdate.rcu_task_stall_info
and rcupdate.rcu_task_stall_info_mult kernel-boot parameters. The former
defines the initial delay in jiffies (defaulting to 10 seconds) and the
latter defines the multiplier (defaulting to 3). Thus, by default, the
first message will appear 10 seconds into the RCU-tasks grace period,
the second 40 seconds in, and the third 160 seconds in. There would be
a fourth at 640 seconds in, but the stall warning message appears 600
seconds in, and once a stall warning is printed for a given grace period,
no further informational messages are printed.
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
2022-02-25 17:01:12 -07:00
|
|
|
__func__, rtp->kname, rtp->tasks_gp_seq, j - rtp->gp_start);
|
|
|
|
|
}
|
2020-03-05 18:07:07 -07:00
|
|
|
}
|
|
|
|
|
|
2020-03-16 12:13:33 -07:00
|
|
|
set_tasks_gp_state(rtp, RTGS_POST_GP);
|
|
|
|
|
rtp->postgp_func(rtp);
|
2020-03-05 18:07:07 -07:00
|
|
|
}
|
|
|
|
|
|
2020-04-05 20:49:13 -07:00
|
|
|
#endif /* #if defined(CONFIG_TASKS_RCU) || defined(CONFIG_TASKS_TRACE_RCU) */
|
|
|
|
|
|
|
|
|
|
#ifdef CONFIG_TASKS_RCU
|
|
|
|
|
|
2020-03-03 12:49:21 -07:00
|
|
|
////////////////////////////////////////////////////////////////////////
|
|
|
|
|
//
|
|
|
|
|
// Simple variant of RCU whose quiescent states are voluntary context
|
2021-09-15 09:24:18 -07:00
|
|
|
// switch, cond_resched_tasks_rcu_qs(), user-space execution, and idle.
|
2020-03-03 12:49:21 -07:00
|
|
|
// As such, grace periods can take one good long time. There are no
|
|
|
|
|
// read-side primitives similar to rcu_read_lock() and rcu_read_unlock()
|
|
|
|
|
// because this implementation is intended to get the system into a safe
|
|
|
|
|
// state for some of the manipulations involved in tracing and the like.
|
|
|
|
|
// Finally, this implementation does not support high call_rcu_tasks()
|
|
|
|
|
// rates from multiple CPUs. If this is required, per-CPU callback lists
|
|
|
|
|
// will be needed.
|
2021-03-04 15:41:47 -07:00
|
|
|
//
|
|
|
|
|
// The implementation uses rcu_tasks_wait_gp(), which relies on function
|
|
|
|
|
// pointers in the rcu_tasks structure. The rcu_spawn_tasks_kthread()
|
|
|
|
|
// function sets these function pointers up so that rcu_tasks_wait_gp()
|
|
|
|
|
// invokes these functions in this order:
|
|
|
|
|
//
|
|
|
|
|
// rcu_tasks_pregp_step():
|
|
|
|
|
// Invokes synchronize_rcu() in order to wait for all in-flight
|
|
|
|
|
// t->on_rq and t->nvcsw transitions to complete. This works because
|
|
|
|
|
// all such transitions are carried out with interrupts disabled.
|
|
|
|
|
// rcu_tasks_pertask(), invoked on every non-idle task:
|
|
|
|
|
// For every runnable non-idle task other than the current one, use
|
|
|
|
|
// get_task_struct() to pin down that task, snapshot that task's
|
|
|
|
|
// number of voluntary context switches, and add that task to the
|
|
|
|
|
// holdout list.
|
|
|
|
|
// rcu_tasks_postscan():
|
rcu-tasks: Eliminate deadlocks involving do_exit() and RCU tasks
Holding a mutex across synchronize_rcu_tasks() and acquiring
that same mutex in code called from do_exit() after its call to
exit_tasks_rcu_start() but before its call to exit_tasks_rcu_stop()
results in deadlock. This is by design, because tasks that are far
enough into do_exit() are no longer present on the tasks list, making
it a bit difficult for RCU Tasks to find them, let alone wait on them
to do a voluntary context switch. However, such deadlocks are becoming
more frequent. In addition, lockdep currently does not detect such
deadlocks and they can be difficult to reproduce.
In addition, if a task voluntarily context switches during that time
(for example, if it blocks acquiring a mutex), then this task is in an
RCU Tasks quiescent state. And with some adjustments, RCU Tasks could
just as well take advantage of that fact.
This commit therefore eliminates these deadlock by replacing the
SRCU-based wait for do_exit() completion with per-CPU lists of tasks
currently exiting. A given task will be on one of these per-CPU lists for
the same period of time that this task would previously have been in the
previous SRCU read-side critical section. These lists enable RCU Tasks
to find the tasks that have already been removed from the tasks list,
but that must nevertheless be waited upon.
The RCU Tasks grace period gathers any of these do_exit() tasks that it
must wait on, and adds them to the list of holdouts. Per-CPU locking
and get_task_struct() are used to synchronize addition to and removal
from these lists.
Link: https://lore.kernel.org/all/20240118021842.290665-1-chenzhongjin@huawei.com/
Reported-by: Chen Zhongjin <chenzhongjin@huawei.com>
Reported-by: Yang Jihong <yangjihong1@huawei.com>
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
Tested-by: Yang Jihong <yangjihong1@huawei.com>
Tested-by: Chen Zhongjin <chenzhongjin@huawei.com>
Reviewed-by: Frederic Weisbecker <frederic@kernel.org>
Signed-off-by: Boqun Feng <boqun.feng@gmail.com>
2024-02-02 12:49:06 -07:00
|
|
|
// Gather per-CPU lists of tasks in do_exit() to ensure that all
|
|
|
|
|
// tasks that were in the process of exiting (and which thus might
|
|
|
|
|
// not know to synchronize with this RCU Tasks grace period) have
|
|
|
|
|
// completed exiting. The synchronize_rcu() in rcu_tasks_postgp()
|
|
|
|
|
// will take care of any tasks stuck in the non-preemptible region
|
2024-04-25 07:24:04 -07:00
|
|
|
// of do_exit() following its call to exit_tasks_rcu_finish().
|
2021-03-04 15:41:47 -07:00
|
|
|
// check_all_holdout_tasks(), repeatedly until holdout list is empty:
|
|
|
|
|
// Scans the holdout list, attempting to identify a quiescent state
|
|
|
|
|
// for each task on the list. If there is a quiescent state, the
|
|
|
|
|
// corresponding task is removed from the holdout list.
|
|
|
|
|
// rcu_tasks_postgp():
|
|
|
|
|
// Invokes synchronize_rcu() in order to ensure that all prior
|
|
|
|
|
// t->on_rq and t->nvcsw transitions are seen by all CPUs and tasks
|
|
|
|
|
// to have happened before the end of this RCU Tasks grace period.
|
|
|
|
|
// Again, this works because all such transitions are carried out
|
|
|
|
|
// with interrupts disabled.
|
|
|
|
|
//
|
|
|
|
|
// For each exiting task, the exit_tasks_rcu_start() and
|
rcu-tasks: Eliminate deadlocks involving do_exit() and RCU tasks
Holding a mutex across synchronize_rcu_tasks() and acquiring
that same mutex in code called from do_exit() after its call to
exit_tasks_rcu_start() but before its call to exit_tasks_rcu_stop()
results in deadlock. This is by design, because tasks that are far
enough into do_exit() are no longer present on the tasks list, making
it a bit difficult for RCU Tasks to find them, let alone wait on them
to do a voluntary context switch. However, such deadlocks are becoming
more frequent. In addition, lockdep currently does not detect such
deadlocks and they can be difficult to reproduce.
In addition, if a task voluntarily context switches during that time
(for example, if it blocks acquiring a mutex), then this task is in an
RCU Tasks quiescent state. And with some adjustments, RCU Tasks could
just as well take advantage of that fact.
This commit therefore eliminates these deadlock by replacing the
SRCU-based wait for do_exit() completion with per-CPU lists of tasks
currently exiting. A given task will be on one of these per-CPU lists for
the same period of time that this task would previously have been in the
previous SRCU read-side critical section. These lists enable RCU Tasks
to find the tasks that have already been removed from the tasks list,
but that must nevertheless be waited upon.
The RCU Tasks grace period gathers any of these do_exit() tasks that it
must wait on, and adds them to the list of holdouts. Per-CPU locking
and get_task_struct() are used to synchronize addition to and removal
from these lists.
Link: https://lore.kernel.org/all/20240118021842.290665-1-chenzhongjin@huawei.com/
Reported-by: Chen Zhongjin <chenzhongjin@huawei.com>
Reported-by: Yang Jihong <yangjihong1@huawei.com>
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
Tested-by: Yang Jihong <yangjihong1@huawei.com>
Tested-by: Chen Zhongjin <chenzhongjin@huawei.com>
Reviewed-by: Frederic Weisbecker <frederic@kernel.org>
Signed-off-by: Boqun Feng <boqun.feng@gmail.com>
2024-02-02 12:49:06 -07:00
|
|
|
// exit_tasks_rcu_finish() functions add and remove, respectively, the
|
|
|
|
|
// current task to a per-CPU list of tasks that rcu_tasks_postscan() must
|
|
|
|
|
// wait on. This is necessary because rcu_tasks_postscan() must wait on
|
|
|
|
|
// tasks that have already been removed from the global list of tasks.
|
2021-03-04 15:41:47 -07:00
|
|
|
//
|
2021-11-08 17:52:02 -07:00
|
|
|
// Pre-grace-period update-side code is ordered before the grace
|
|
|
|
|
// via the raw_spin_lock.*rcu_node(). Pre-grace-period read-side code
|
|
|
|
|
// is ordered before the grace period via synchronize_rcu() call in
|
|
|
|
|
// rcu_tasks_pregp_step() and by the scheduler's locks and interrupt
|
2021-03-04 15:41:47 -07:00
|
|
|
// disabling.
|
2020-03-03 12:49:21 -07:00
|
|
|
|
2020-03-04 18:31:43 -07:00
|
|
|
/* Pre-grace-period preparation. */
|
2022-05-18 16:06:55 -07:00
|
|
|
static void rcu_tasks_pregp_step(struct list_head *hop)
|
2020-03-04 18:31:43 -07:00
|
|
|
{
|
|
|
|
|
/*
|
|
|
|
|
* Wait for all pre-existing t->on_rq and t->nvcsw transitions
|
|
|
|
|
* to complete. Invoking synchronize_rcu() suffices because all
|
|
|
|
|
* these transitions occur with interrupts disabled. Without this
|
|
|
|
|
* synchronize_rcu(), a read-side critical section that started
|
|
|
|
|
* before the grace period might be incorrectly seen as having
|
|
|
|
|
* started after the grace period.
|
|
|
|
|
*
|
|
|
|
|
* This synchronize_rcu() also dispenses with the need for a
|
|
|
|
|
* memory barrier on the first store to t->rcu_tasks_holdout,
|
|
|
|
|
* as it forces the store to happen after the beginning of the
|
|
|
|
|
* grace period.
|
|
|
|
|
*/
|
|
|
|
|
synchronize_rcu();
|
|
|
|
|
}
|
|
|
|
|
|
2023-10-27 07:40:48 -07:00
|
|
|
/* Check for quiescent states since the pregp's synchronize_rcu() */
|
|
|
|
|
static bool rcu_tasks_is_holdout(struct task_struct *t)
|
|
|
|
|
{
|
|
|
|
|
int cpu;
|
|
|
|
|
|
|
|
|
|
/* Has the task been seen voluntarily sleeping? */
|
|
|
|
|
if (!READ_ONCE(t->on_rq))
|
|
|
|
|
return false;
|
|
|
|
|
|
2024-10-10 02:38:10 -07:00
|
|
|
/*
|
|
|
|
|
* t->on_rq && !t->se.sched_delayed *could* be considered sleeping but
|
|
|
|
|
* since it is a spurious state (it will transition into the
|
|
|
|
|
* traditional blocked state or get woken up without outside
|
|
|
|
|
* dependencies), not considering it such should only affect timing.
|
|
|
|
|
*
|
|
|
|
|
* Be conservative for now and not include it.
|
|
|
|
|
*/
|
|
|
|
|
|
2023-10-27 07:40:48 -07:00
|
|
|
/*
|
|
|
|
|
* Idle tasks (or idle injection) within the idle loop are RCU-tasks
|
|
|
|
|
* quiescent states. But CPU boot code performed by the idle task
|
|
|
|
|
* isn't a quiescent state.
|
|
|
|
|
*/
|
|
|
|
|
if (is_idle_task(t))
|
|
|
|
|
return false;
|
|
|
|
|
|
|
|
|
|
cpu = task_cpu(t);
|
|
|
|
|
|
|
|
|
|
/* Idle tasks on offline CPUs are RCU-tasks quiescent states. */
|
|
|
|
|
if (t == idle_task(cpu) && !rcu_cpu_online(cpu))
|
|
|
|
|
return false;
|
|
|
|
|
|
|
|
|
|
return true;
|
|
|
|
|
}
|
|
|
|
|
|
2020-03-04 18:31:43 -07:00
|
|
|
/* Per-task initial processing. */
|
|
|
|
|
static void rcu_tasks_pertask(struct task_struct *t, struct list_head *hop)
|
|
|
|
|
{
|
2023-10-27 07:40:48 -07:00
|
|
|
if (t != current && rcu_tasks_is_holdout(t)) {
|
2020-03-04 18:31:43 -07:00
|
|
|
get_task_struct(t);
|
|
|
|
|
t->rcu_tasks_nvcsw = READ_ONCE(t->nvcsw);
|
|
|
|
|
WRITE_ONCE(t->rcu_tasks_holdout, true);
|
|
|
|
|
list_add(&t->rcu_tasks_holdout_list, hop);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
rcu-tasks: Eliminate deadlocks involving do_exit() and RCU tasks
Holding a mutex across synchronize_rcu_tasks() and acquiring
that same mutex in code called from do_exit() after its call to
exit_tasks_rcu_start() but before its call to exit_tasks_rcu_stop()
results in deadlock. This is by design, because tasks that are far
enough into do_exit() are no longer present on the tasks list, making
it a bit difficult for RCU Tasks to find them, let alone wait on them
to do a voluntary context switch. However, such deadlocks are becoming
more frequent. In addition, lockdep currently does not detect such
deadlocks and they can be difficult to reproduce.
In addition, if a task voluntarily context switches during that time
(for example, if it blocks acquiring a mutex), then this task is in an
RCU Tasks quiescent state. And with some adjustments, RCU Tasks could
just as well take advantage of that fact.
This commit therefore eliminates these deadlock by replacing the
SRCU-based wait for do_exit() completion with per-CPU lists of tasks
currently exiting. A given task will be on one of these per-CPU lists for
the same period of time that this task would previously have been in the
previous SRCU read-side critical section. These lists enable RCU Tasks
to find the tasks that have already been removed from the tasks list,
but that must nevertheless be waited upon.
The RCU Tasks grace period gathers any of these do_exit() tasks that it
must wait on, and adds them to the list of holdouts. Per-CPU locking
and get_task_struct() are used to synchronize addition to and removal
from these lists.
Link: https://lore.kernel.org/all/20240118021842.290665-1-chenzhongjin@huawei.com/
Reported-by: Chen Zhongjin <chenzhongjin@huawei.com>
Reported-by: Yang Jihong <yangjihong1@huawei.com>
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
Tested-by: Yang Jihong <yangjihong1@huawei.com>
Tested-by: Chen Zhongjin <chenzhongjin@huawei.com>
Reviewed-by: Frederic Weisbecker <frederic@kernel.org>
Signed-off-by: Boqun Feng <boqun.feng@gmail.com>
2024-02-02 12:49:06 -07:00
|
|
|
void call_rcu_tasks(struct rcu_head *rhp, rcu_callback_t func);
|
|
|
|
|
DEFINE_RCU_TASKS(rcu_tasks, rcu_tasks_wait_gp, call_rcu_tasks, "RCU Tasks");
|
|
|
|
|
|
2020-03-04 18:31:43 -07:00
|
|
|
/* Processing between scanning taskslist and draining the holdout list. */
|
2020-05-28 19:27:06 -07:00
|
|
|
static void rcu_tasks_postscan(struct list_head *hop)
|
2020-03-04 18:31:43 -07:00
|
|
|
{
|
rcu-tasks: Eliminate deadlocks involving do_exit() and RCU tasks
Holding a mutex across synchronize_rcu_tasks() and acquiring
that same mutex in code called from do_exit() after its call to
exit_tasks_rcu_start() but before its call to exit_tasks_rcu_stop()
results in deadlock. This is by design, because tasks that are far
enough into do_exit() are no longer present on the tasks list, making
it a bit difficult for RCU Tasks to find them, let alone wait on them
to do a voluntary context switch. However, such deadlocks are becoming
more frequent. In addition, lockdep currently does not detect such
deadlocks and they can be difficult to reproduce.
In addition, if a task voluntarily context switches during that time
(for example, if it blocks acquiring a mutex), then this task is in an
RCU Tasks quiescent state. And with some adjustments, RCU Tasks could
just as well take advantage of that fact.
This commit therefore eliminates these deadlock by replacing the
SRCU-based wait for do_exit() completion with per-CPU lists of tasks
currently exiting. A given task will be on one of these per-CPU lists for
the same period of time that this task would previously have been in the
previous SRCU read-side critical section. These lists enable RCU Tasks
to find the tasks that have already been removed from the tasks list,
but that must nevertheless be waited upon.
The RCU Tasks grace period gathers any of these do_exit() tasks that it
must wait on, and adds them to the list of holdouts. Per-CPU locking
and get_task_struct() are used to synchronize addition to and removal
from these lists.
Link: https://lore.kernel.org/all/20240118021842.290665-1-chenzhongjin@huawei.com/
Reported-by: Chen Zhongjin <chenzhongjin@huawei.com>
Reported-by: Yang Jihong <yangjihong1@huawei.com>
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
Tested-by: Yang Jihong <yangjihong1@huawei.com>
Tested-by: Chen Zhongjin <chenzhongjin@huawei.com>
Reviewed-by: Frederic Weisbecker <frederic@kernel.org>
Signed-off-by: Boqun Feng <boqun.feng@gmail.com>
2024-02-02 12:49:06 -07:00
|
|
|
int cpu;
|
2023-01-11 00:45:00 -07:00
|
|
|
int rtsi = READ_ONCE(rcu_task_stall_info);
|
|
|
|
|
|
|
|
|
|
if (!IS_ENABLED(CONFIG_TINY_RCU)) {
|
|
|
|
|
tasks_rcu_exit_srcu_stall_timer.expires = jiffies + rtsi;
|
|
|
|
|
add_timer(&tasks_rcu_exit_srcu_stall_timer);
|
|
|
|
|
}
|
|
|
|
|
|
2020-03-04 18:31:43 -07:00
|
|
|
/*
|
2022-11-25 06:54:58 -07:00
|
|
|
* Exiting tasks may escape the tasklist scan. Those are vulnerable
|
|
|
|
|
* until their final schedule() with TASK_DEAD state. To cope with
|
|
|
|
|
* this, divide the fragile exit path part in two intersecting
|
|
|
|
|
* read side critical sections:
|
|
|
|
|
*
|
rcu-tasks: Eliminate deadlocks involving do_exit() and RCU tasks
Holding a mutex across synchronize_rcu_tasks() and acquiring
that same mutex in code called from do_exit() after its call to
exit_tasks_rcu_start() but before its call to exit_tasks_rcu_stop()
results in deadlock. This is by design, because tasks that are far
enough into do_exit() are no longer present on the tasks list, making
it a bit difficult for RCU Tasks to find them, let alone wait on them
to do a voluntary context switch. However, such deadlocks are becoming
more frequent. In addition, lockdep currently does not detect such
deadlocks and they can be difficult to reproduce.
In addition, if a task voluntarily context switches during that time
(for example, if it blocks acquiring a mutex), then this task is in an
RCU Tasks quiescent state. And with some adjustments, RCU Tasks could
just as well take advantage of that fact.
This commit therefore eliminates these deadlock by replacing the
SRCU-based wait for do_exit() completion with per-CPU lists of tasks
currently exiting. A given task will be on one of these per-CPU lists for
the same period of time that this task would previously have been in the
previous SRCU read-side critical section. These lists enable RCU Tasks
to find the tasks that have already been removed from the tasks list,
but that must nevertheless be waited upon.
The RCU Tasks grace period gathers any of these do_exit() tasks that it
must wait on, and adds them to the list of holdouts. Per-CPU locking
and get_task_struct() are used to synchronize addition to and removal
from these lists.
Link: https://lore.kernel.org/all/20240118021842.290665-1-chenzhongjin@huawei.com/
Reported-by: Chen Zhongjin <chenzhongjin@huawei.com>
Reported-by: Yang Jihong <yangjihong1@huawei.com>
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
Tested-by: Yang Jihong <yangjihong1@huawei.com>
Tested-by: Chen Zhongjin <chenzhongjin@huawei.com>
Reviewed-by: Frederic Weisbecker <frederic@kernel.org>
Signed-off-by: Boqun Feng <boqun.feng@gmail.com>
2024-02-02 12:49:06 -07:00
|
|
|
* 1) A task_struct list addition before calling exit_notify(),
|
|
|
|
|
* which may remove the task from the tasklist, with the
|
|
|
|
|
* removal after the final preempt_disable() call in do_exit().
|
2022-11-25 06:54:58 -07:00
|
|
|
*
|
|
|
|
|
* 2) An _RCU_ read side starting with the final preempt_disable()
|
|
|
|
|
* call in do_exit() and ending with the final call to schedule()
|
|
|
|
|
* with TASK_DEAD state.
|
|
|
|
|
*
|
|
|
|
|
* This handles the part 1). And postgp will handle part 2) with a
|
|
|
|
|
* call to synchronize_rcu().
|
2020-03-04 18:31:43 -07:00
|
|
|
*/
|
rcu-tasks: Eliminate deadlocks involving do_exit() and RCU tasks
Holding a mutex across synchronize_rcu_tasks() and acquiring
that same mutex in code called from do_exit() after its call to
exit_tasks_rcu_start() but before its call to exit_tasks_rcu_stop()
results in deadlock. This is by design, because tasks that are far
enough into do_exit() are no longer present on the tasks list, making
it a bit difficult for RCU Tasks to find them, let alone wait on them
to do a voluntary context switch. However, such deadlocks are becoming
more frequent. In addition, lockdep currently does not detect such
deadlocks and they can be difficult to reproduce.
In addition, if a task voluntarily context switches during that time
(for example, if it blocks acquiring a mutex), then this task is in an
RCU Tasks quiescent state. And with some adjustments, RCU Tasks could
just as well take advantage of that fact.
This commit therefore eliminates these deadlock by replacing the
SRCU-based wait for do_exit() completion with per-CPU lists of tasks
currently exiting. A given task will be on one of these per-CPU lists for
the same period of time that this task would previously have been in the
previous SRCU read-side critical section. These lists enable RCU Tasks
to find the tasks that have already been removed from the tasks list,
but that must nevertheless be waited upon.
The RCU Tasks grace period gathers any of these do_exit() tasks that it
must wait on, and adds them to the list of holdouts. Per-CPU locking
and get_task_struct() are used to synchronize addition to and removal
from these lists.
Link: https://lore.kernel.org/all/20240118021842.290665-1-chenzhongjin@huawei.com/
Reported-by: Chen Zhongjin <chenzhongjin@huawei.com>
Reported-by: Yang Jihong <yangjihong1@huawei.com>
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
Tested-by: Yang Jihong <yangjihong1@huawei.com>
Tested-by: Chen Zhongjin <chenzhongjin@huawei.com>
Reviewed-by: Frederic Weisbecker <frederic@kernel.org>
Signed-off-by: Boqun Feng <boqun.feng@gmail.com>
2024-02-02 12:49:06 -07:00
|
|
|
|
|
|
|
|
for_each_possible_cpu(cpu) {
|
2024-02-01 07:10:26 -07:00
|
|
|
unsigned long j = jiffies + 1;
|
rcu-tasks: Eliminate deadlocks involving do_exit() and RCU tasks
Holding a mutex across synchronize_rcu_tasks() and acquiring
that same mutex in code called from do_exit() after its call to
exit_tasks_rcu_start() but before its call to exit_tasks_rcu_stop()
results in deadlock. This is by design, because tasks that are far
enough into do_exit() are no longer present on the tasks list, making
it a bit difficult for RCU Tasks to find them, let alone wait on them
to do a voluntary context switch. However, such deadlocks are becoming
more frequent. In addition, lockdep currently does not detect such
deadlocks and they can be difficult to reproduce.
In addition, if a task voluntarily context switches during that time
(for example, if it blocks acquiring a mutex), then this task is in an
RCU Tasks quiescent state. And with some adjustments, RCU Tasks could
just as well take advantage of that fact.
This commit therefore eliminates these deadlock by replacing the
SRCU-based wait for do_exit() completion with per-CPU lists of tasks
currently exiting. A given task will be on one of these per-CPU lists for
the same period of time that this task would previously have been in the
previous SRCU read-side critical section. These lists enable RCU Tasks
to find the tasks that have already been removed from the tasks list,
but that must nevertheless be waited upon.
The RCU Tasks grace period gathers any of these do_exit() tasks that it
must wait on, and adds them to the list of holdouts. Per-CPU locking
and get_task_struct() are used to synchronize addition to and removal
from these lists.
Link: https://lore.kernel.org/all/20240118021842.290665-1-chenzhongjin@huawei.com/
Reported-by: Chen Zhongjin <chenzhongjin@huawei.com>
Reported-by: Yang Jihong <yangjihong1@huawei.com>
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
Tested-by: Yang Jihong <yangjihong1@huawei.com>
Tested-by: Chen Zhongjin <chenzhongjin@huawei.com>
Reviewed-by: Frederic Weisbecker <frederic@kernel.org>
Signed-off-by: Boqun Feng <boqun.feng@gmail.com>
2024-02-02 12:49:06 -07:00
|
|
|
struct rcu_tasks_percpu *rtpcp = per_cpu_ptr(rcu_tasks.rtpcpu, cpu);
|
|
|
|
|
struct task_struct *t;
|
2024-02-01 07:10:26 -07:00
|
|
|
struct task_struct *t1;
|
|
|
|
|
struct list_head tmp;
|
rcu-tasks: Eliminate deadlocks involving do_exit() and RCU tasks
Holding a mutex across synchronize_rcu_tasks() and acquiring
that same mutex in code called from do_exit() after its call to
exit_tasks_rcu_start() but before its call to exit_tasks_rcu_stop()
results in deadlock. This is by design, because tasks that are far
enough into do_exit() are no longer present on the tasks list, making
it a bit difficult for RCU Tasks to find them, let alone wait on them
to do a voluntary context switch. However, such deadlocks are becoming
more frequent. In addition, lockdep currently does not detect such
deadlocks and they can be difficult to reproduce.
In addition, if a task voluntarily context switches during that time
(for example, if it blocks acquiring a mutex), then this task is in an
RCU Tasks quiescent state. And with some adjustments, RCU Tasks could
just as well take advantage of that fact.
This commit therefore eliminates these deadlock by replacing the
SRCU-based wait for do_exit() completion with per-CPU lists of tasks
currently exiting. A given task will be on one of these per-CPU lists for
the same period of time that this task would previously have been in the
previous SRCU read-side critical section. These lists enable RCU Tasks
to find the tasks that have already been removed from the tasks list,
but that must nevertheless be waited upon.
The RCU Tasks grace period gathers any of these do_exit() tasks that it
must wait on, and adds them to the list of holdouts. Per-CPU locking
and get_task_struct() are used to synchronize addition to and removal
from these lists.
Link: https://lore.kernel.org/all/20240118021842.290665-1-chenzhongjin@huawei.com/
Reported-by: Chen Zhongjin <chenzhongjin@huawei.com>
Reported-by: Yang Jihong <yangjihong1@huawei.com>
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
Tested-by: Yang Jihong <yangjihong1@huawei.com>
Tested-by: Chen Zhongjin <chenzhongjin@huawei.com>
Reviewed-by: Frederic Weisbecker <frederic@kernel.org>
Signed-off-by: Boqun Feng <boqun.feng@gmail.com>
2024-02-02 12:49:06 -07:00
|
|
|
|
|
|
|
|
raw_spin_lock_irq_rcu_node(rtpcp);
|
2024-02-01 07:10:26 -07:00
|
|
|
list_for_each_entry_safe(t, t1, &rtpcp->rtp_exit_list, rcu_tasks_exit_list) {
|
rcu-tasks: Eliminate deadlocks involving do_exit() and RCU tasks
Holding a mutex across synchronize_rcu_tasks() and acquiring
that same mutex in code called from do_exit() after its call to
exit_tasks_rcu_start() but before its call to exit_tasks_rcu_stop()
results in deadlock. This is by design, because tasks that are far
enough into do_exit() are no longer present on the tasks list, making
it a bit difficult for RCU Tasks to find them, let alone wait on them
to do a voluntary context switch. However, such deadlocks are becoming
more frequent. In addition, lockdep currently does not detect such
deadlocks and they can be difficult to reproduce.
In addition, if a task voluntarily context switches during that time
(for example, if it blocks acquiring a mutex), then this task is in an
RCU Tasks quiescent state. And with some adjustments, RCU Tasks could
just as well take advantage of that fact.
This commit therefore eliminates these deadlock by replacing the
SRCU-based wait for do_exit() completion with per-CPU lists of tasks
currently exiting. A given task will be on one of these per-CPU lists for
the same period of time that this task would previously have been in the
previous SRCU read-side critical section. These lists enable RCU Tasks
to find the tasks that have already been removed from the tasks list,
but that must nevertheless be waited upon.
The RCU Tasks grace period gathers any of these do_exit() tasks that it
must wait on, and adds them to the list of holdouts. Per-CPU locking
and get_task_struct() are used to synchronize addition to and removal
from these lists.
Link: https://lore.kernel.org/all/20240118021842.290665-1-chenzhongjin@huawei.com/
Reported-by: Chen Zhongjin <chenzhongjin@huawei.com>
Reported-by: Yang Jihong <yangjihong1@huawei.com>
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
Tested-by: Yang Jihong <yangjihong1@huawei.com>
Tested-by: Chen Zhongjin <chenzhongjin@huawei.com>
Reviewed-by: Frederic Weisbecker <frederic@kernel.org>
Signed-off-by: Boqun Feng <boqun.feng@gmail.com>
2024-02-02 12:49:06 -07:00
|
|
|
if (list_empty(&t->rcu_tasks_holdout_list))
|
|
|
|
|
rcu_tasks_pertask(t, hop);
|
2024-02-01 07:10:26 -07:00
|
|
|
|
|
|
|
|
// RT kernels need frequent pauses, otherwise
|
|
|
|
|
// pause at least once per pair of jiffies.
|
|
|
|
|
if (!IS_ENABLED(CONFIG_PREEMPT_RT) && time_before(jiffies, j))
|
|
|
|
|
continue;
|
|
|
|
|
|
|
|
|
|
// Keep our place in the list while pausing.
|
|
|
|
|
// Nothing else traverses this list, so adding a
|
|
|
|
|
// bare list_head is OK.
|
|
|
|
|
list_add(&tmp, &t->rcu_tasks_exit_list);
|
|
|
|
|
raw_spin_unlock_irq_rcu_node(rtpcp);
|
|
|
|
|
cond_resched(); // For CONFIG_PREEMPT=n kernels
|
|
|
|
|
raw_spin_lock_irq_rcu_node(rtpcp);
|
|
|
|
|
t1 = list_entry(tmp.next, struct task_struct, rcu_tasks_exit_list);
|
|
|
|
|
list_del(&tmp);
|
|
|
|
|
j = jiffies + 1;
|
|
|
|
|
}
|
rcu-tasks: Eliminate deadlocks involving do_exit() and RCU tasks
Holding a mutex across synchronize_rcu_tasks() and acquiring
that same mutex in code called from do_exit() after its call to
exit_tasks_rcu_start() but before its call to exit_tasks_rcu_stop()
results in deadlock. This is by design, because tasks that are far
enough into do_exit() are no longer present on the tasks list, making
it a bit difficult for RCU Tasks to find them, let alone wait on them
to do a voluntary context switch. However, such deadlocks are becoming
more frequent. In addition, lockdep currently does not detect such
deadlocks and they can be difficult to reproduce.
In addition, if a task voluntarily context switches during that time
(for example, if it blocks acquiring a mutex), then this task is in an
RCU Tasks quiescent state. And with some adjustments, RCU Tasks could
just as well take advantage of that fact.
This commit therefore eliminates these deadlock by replacing the
SRCU-based wait for do_exit() completion with per-CPU lists of tasks
currently exiting. A given task will be on one of these per-CPU lists for
the same period of time that this task would previously have been in the
previous SRCU read-side critical section. These lists enable RCU Tasks
to find the tasks that have already been removed from the tasks list,
but that must nevertheless be waited upon.
The RCU Tasks grace period gathers any of these do_exit() tasks that it
must wait on, and adds them to the list of holdouts. Per-CPU locking
and get_task_struct() are used to synchronize addition to and removal
from these lists.
Link: https://lore.kernel.org/all/20240118021842.290665-1-chenzhongjin@huawei.com/
Reported-by: Chen Zhongjin <chenzhongjin@huawei.com>
Reported-by: Yang Jihong <yangjihong1@huawei.com>
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
Tested-by: Yang Jihong <yangjihong1@huawei.com>
Tested-by: Chen Zhongjin <chenzhongjin@huawei.com>
Reviewed-by: Frederic Weisbecker <frederic@kernel.org>
Signed-off-by: Boqun Feng <boqun.feng@gmail.com>
2024-02-02 12:49:06 -07:00
|
|
|
raw_spin_unlock_irq_rcu_node(rtpcp);
|
|
|
|
|
}
|
2023-01-11 00:45:00 -07:00
|
|
|
|
|
|
|
|
if (!IS_ENABLED(CONFIG_TINY_RCU))
|
|
|
|
|
del_timer_sync(&tasks_rcu_exit_srcu_stall_timer);
|
2020-03-04 18:31:43 -07:00
|
|
|
}
|
|
|
|
|
|
2020-03-03 12:49:21 -07:00
|
|
|
/* See if tasks are still holding out, complain if so. */
|
|
|
|
|
static void check_holdout_task(struct task_struct *t,
|
|
|
|
|
bool needreport, bool *firstreport)
|
|
|
|
|
{
|
|
|
|
|
int cpu;
|
|
|
|
|
|
|
|
|
|
if (!READ_ONCE(t->rcu_tasks_holdout) ||
|
|
|
|
|
t->rcu_tasks_nvcsw != READ_ONCE(t->nvcsw) ||
|
2023-10-27 07:40:48 -07:00
|
|
|
!rcu_tasks_is_holdout(t) ||
|
2020-03-03 12:49:21 -07:00
|
|
|
(IS_ENABLED(CONFIG_NO_HZ_FULL) &&
|
2023-10-11 09:45:54 -07:00
|
|
|
!is_idle_task(t) && READ_ONCE(t->rcu_tasks_idle_cpu) >= 0)) {
|
2020-03-03 12:49:21 -07:00
|
|
|
WRITE_ONCE(t->rcu_tasks_holdout, false);
|
|
|
|
|
list_del_init(&t->rcu_tasks_holdout_list);
|
|
|
|
|
put_task_struct(t);
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
rcu_request_urgent_qs_task(t);
|
|
|
|
|
if (!needreport)
|
|
|
|
|
return;
|
|
|
|
|
if (*firstreport) {
|
|
|
|
|
pr_err("INFO: rcu_tasks detected stalls on tasks:\n");
|
|
|
|
|
*firstreport = false;
|
|
|
|
|
}
|
|
|
|
|
cpu = task_cpu(t);
|
|
|
|
|
pr_alert("%p: %c%c nvcsw: %lu/%lu holdout: %d idle_cpu: %d/%d\n",
|
|
|
|
|
t, ".I"[is_idle_task(t)],
|
|
|
|
|
"N."[cpu < 0 || !tick_nohz_full_cpu(cpu)],
|
|
|
|
|
t->rcu_tasks_nvcsw, t->nvcsw, t->rcu_tasks_holdout,
|
2023-10-11 09:45:54 -07:00
|
|
|
data_race(t->rcu_tasks_idle_cpu), cpu);
|
2020-03-03 12:49:21 -07:00
|
|
|
sched_show_task(t);
|
|
|
|
|
}
|
|
|
|
|
|
2020-03-04 18:31:43 -07:00
|
|
|
/* Scan the holdout lists for tasks no longer holding out. */
|
|
|
|
|
static void check_all_holdout_tasks(struct list_head *hop,
|
|
|
|
|
bool needreport, bool *firstreport)
|
|
|
|
|
{
|
|
|
|
|
struct task_struct *t, *t1;
|
|
|
|
|
|
|
|
|
|
list_for_each_entry_safe(t, t1, hop, rcu_tasks_holdout_list) {
|
|
|
|
|
check_holdout_task(t, needreport, firstreport);
|
|
|
|
|
cond_resched();
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/* Finish off the Tasks-RCU grace period. */
|
2020-03-16 12:13:33 -07:00
|
|
|
static void rcu_tasks_postgp(struct rcu_tasks *rtp)
|
2020-03-04 18:31:43 -07:00
|
|
|
{
|
|
|
|
|
/*
|
|
|
|
|
* Because ->on_rq and ->nvcsw are not guaranteed to have a full
|
|
|
|
|
* memory barriers prior to them in the schedule() path, memory
|
|
|
|
|
* reordering on other CPUs could cause their RCU-tasks read-side
|
|
|
|
|
* critical sections to extend past the end of the grace period.
|
|
|
|
|
* However, because these ->nvcsw updates are carried out with
|
|
|
|
|
* interrupts disabled, we can use synchronize_rcu() to force the
|
|
|
|
|
* needed ordering on all such CPUs.
|
|
|
|
|
*
|
|
|
|
|
* This synchronize_rcu() also confines all ->rcu_tasks_holdout
|
|
|
|
|
* accesses to be within the grace period, avoiding the need for
|
|
|
|
|
* memory barriers for ->rcu_tasks_holdout accesses.
|
|
|
|
|
*
|
|
|
|
|
* In addition, this synchronize_rcu() waits for exiting tasks
|
|
|
|
|
* to complete their final preempt_disable() region of execution,
|
2022-11-25 06:54:58 -07:00
|
|
|
* enforcing the whole region before tasklist removal until
|
|
|
|
|
* the final schedule() with TASK_DEAD state to be an RCU TASKS
|
|
|
|
|
* read side critical section.
|
2020-03-04 18:31:43 -07:00
|
|
|
*/
|
|
|
|
|
synchronize_rcu();
|
|
|
|
|
}
|
|
|
|
|
|
2023-01-11 00:45:00 -07:00
|
|
|
static void tasks_rcu_exit_srcu_stall(struct timer_list *unused)
|
|
|
|
|
{
|
|
|
|
|
#ifndef CONFIG_TINY_RCU
|
|
|
|
|
int rtsi;
|
|
|
|
|
|
|
|
|
|
rtsi = READ_ONCE(rcu_task_stall_info);
|
|
|
|
|
pr_info("%s: %s grace period number %lu (since boot) gp_state: %s is %lu jiffies old.\n",
|
|
|
|
|
__func__, rcu_tasks.kname, rcu_tasks.tasks_gp_seq,
|
|
|
|
|
tasks_gp_state_getname(&rcu_tasks), jiffies - rcu_tasks.gp_jiffies);
|
|
|
|
|
pr_info("Please check any exiting tasks stuck between calls to exit_tasks_rcu_start() and exit_tasks_rcu_finish()\n");
|
|
|
|
|
tasks_rcu_exit_srcu_stall_timer.expires = jiffies + rtsi;
|
|
|
|
|
add_timer(&tasks_rcu_exit_srcu_stall_timer);
|
|
|
|
|
#endif // #ifndef CONFIG_TINY_RCU
|
|
|
|
|
}
|
|
|
|
|
|
2020-03-03 12:49:21 -07:00
|
|
|
/**
|
|
|
|
|
* call_rcu_tasks() - Queue an RCU for invocation task-based grace period
|
|
|
|
|
* @rhp: structure to be used for queueing the RCU updates.
|
|
|
|
|
* @func: actual callback function to be invoked after the grace period
|
|
|
|
|
*
|
|
|
|
|
* The callback function will be invoked some time after a full grace
|
|
|
|
|
* period elapses, in other words after all currently executing RCU
|
|
|
|
|
* read-side critical sections have completed. call_rcu_tasks() assumes
|
|
|
|
|
* that the read-side critical sections end at a voluntary context
|
2021-09-15 09:24:18 -07:00
|
|
|
* switch (not a preemption!), cond_resched_tasks_rcu_qs(), entry into idle,
|
2020-03-03 12:49:21 -07:00
|
|
|
* or transition to usermode execution. As such, there are no read-side
|
|
|
|
|
* primitives analogous to rcu_read_lock() and rcu_read_unlock() because
|
|
|
|
|
* this primitive is intended to determine that all tasks have passed
|
2021-03-22 22:29:10 -07:00
|
|
|
* through a safe state, not so much for data-structure synchronization.
|
2020-03-03 12:49:21 -07:00
|
|
|
*
|
|
|
|
|
* See the description of call_rcu() for more detailed information on
|
|
|
|
|
* memory ordering guarantees.
|
|
|
|
|
*/
|
|
|
|
|
void call_rcu_tasks(struct rcu_head *rhp, rcu_callback_t func)
|
|
|
|
|
{
|
|
|
|
|
call_rcu_tasks_generic(rhp, func, &rcu_tasks);
|
|
|
|
|
}
|
|
|
|
|
EXPORT_SYMBOL_GPL(call_rcu_tasks);
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* synchronize_rcu_tasks - wait until an rcu-tasks grace period has elapsed.
|
|
|
|
|
*
|
|
|
|
|
* Control will return to the caller some time after a full rcu-tasks
|
|
|
|
|
* grace period has elapsed, in other words after all currently
|
|
|
|
|
* executing rcu-tasks read-side critical sections have elapsed. These
|
|
|
|
|
* read-side critical sections are delimited by calls to schedule(),
|
|
|
|
|
* cond_resched_tasks_rcu_qs(), idle execution, userspace execution, calls
|
|
|
|
|
* to synchronize_rcu_tasks(), and (in theory, anyway) cond_resched().
|
|
|
|
|
*
|
|
|
|
|
* This is a very specialized primitive, intended only for a few uses in
|
|
|
|
|
* tracing and other situations requiring manipulation of function
|
|
|
|
|
* preambles and profiling hooks. The synchronize_rcu_tasks() function
|
|
|
|
|
* is not (yet) intended for heavy use from multiple CPUs.
|
|
|
|
|
*
|
|
|
|
|
* See the description of synchronize_rcu() for more detailed information
|
|
|
|
|
* on memory ordering guarantees.
|
|
|
|
|
*/
|
|
|
|
|
void synchronize_rcu_tasks(void)
|
|
|
|
|
{
|
|
|
|
|
synchronize_rcu_tasks_generic(&rcu_tasks);
|
|
|
|
|
}
|
|
|
|
|
EXPORT_SYMBOL_GPL(synchronize_rcu_tasks);
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* rcu_barrier_tasks - Wait for in-flight call_rcu_tasks() callbacks.
|
|
|
|
|
*
|
|
|
|
|
* Although the current implementation is guaranteed to wait, it is not
|
|
|
|
|
* obligated to, for example, if there are no pending callbacks.
|
|
|
|
|
*/
|
|
|
|
|
void rcu_barrier_tasks(void)
|
|
|
|
|
{
|
2021-11-11 15:53:43 -07:00
|
|
|
rcu_barrier_tasks_generic(&rcu_tasks);
|
2020-03-03 12:49:21 -07:00
|
|
|
}
|
|
|
|
|
EXPORT_SYMBOL_GPL(rcu_barrier_tasks);
|
|
|
|
|
|
2023-08-03 01:06:59 -07:00
|
|
|
static int rcu_tasks_lazy_ms = -1;
|
2023-05-17 06:40:23 -07:00
|
|
|
module_param(rcu_tasks_lazy_ms, int, 0444);
|
|
|
|
|
|
2020-03-03 12:49:21 -07:00
|
|
|
static int __init rcu_spawn_tasks_kthread(void)
|
|
|
|
|
{
|
2020-09-09 22:05:41 -07:00
|
|
|
rcu_tasks.gp_sleep = HZ / 10;
|
2020-09-17 16:17:17 -07:00
|
|
|
rcu_tasks.init_fract = HZ / 10;
|
2023-05-17 06:40:23 -07:00
|
|
|
if (rcu_tasks_lazy_ms >= 0)
|
|
|
|
|
rcu_tasks.lazy_jiffies = msecs_to_jiffies(rcu_tasks_lazy_ms);
|
2020-03-04 18:31:43 -07:00
|
|
|
rcu_tasks.pregp_func = rcu_tasks_pregp_step;
|
|
|
|
|
rcu_tasks.pertask_func = rcu_tasks_pertask;
|
|
|
|
|
rcu_tasks.postscan_func = rcu_tasks_postscan;
|
|
|
|
|
rcu_tasks.holdouts_func = check_all_holdout_tasks;
|
|
|
|
|
rcu_tasks.postgp_func = rcu_tasks_postgp;
|
2024-02-23 16:16:20 -07:00
|
|
|
rcu_tasks.wait_state = TASK_IDLE;
|
2020-03-03 12:49:21 -07:00
|
|
|
rcu_spawn_tasks_kthread_generic(&rcu_tasks);
|
|
|
|
|
return 0;
|
|
|
|
|
}
|
|
|
|
|
|
2020-09-15 17:08:03 -07:00
|
|
|
#if !defined(CONFIG_TINY_RCU)
|
|
|
|
|
void show_rcu_tasks_classic_gp_kthread(void)
|
2020-03-16 11:01:55 -07:00
|
|
|
{
|
|
|
|
|
show_rcu_tasks_generic_gp_kthread(&rcu_tasks, "");
|
|
|
|
|
}
|
2020-09-15 17:08:03 -07:00
|
|
|
EXPORT_SYMBOL_GPL(show_rcu_tasks_classic_gp_kthread);
|
2024-08-01 17:34:25 -07:00
|
|
|
|
|
|
|
|
void rcu_tasks_torture_stats_print(char *tt, char *tf)
|
|
|
|
|
{
|
|
|
|
|
rcu_tasks_torture_stats_print_generic(&rcu_tasks, tt, tf, "");
|
|
|
|
|
}
|
|
|
|
|
EXPORT_SYMBOL_GPL(rcu_tasks_torture_stats_print);
|
2020-09-15 17:08:03 -07:00
|
|
|
#endif // !defined(CONFIG_TINY_RCU)
|
2020-03-16 11:01:55 -07:00
|
|
|
|
2023-06-02 14:38:44 -07:00
|
|
|
struct task_struct *get_rcu_tasks_gp_kthread(void)
|
|
|
|
|
{
|
|
|
|
|
return rcu_tasks.kthread_ptr;
|
|
|
|
|
}
|
|
|
|
|
EXPORT_SYMBOL_GPL(get_rcu_tasks_gp_kthread);
|
|
|
|
|
|
2024-03-18 02:34:11 -07:00
|
|
|
void rcu_tasks_get_gp_data(int *flags, unsigned long *gp_seq)
|
|
|
|
|
{
|
|
|
|
|
*flags = 0;
|
|
|
|
|
*gp_seq = rcu_seq_current(&rcu_tasks.tasks_gp_seq);
|
|
|
|
|
}
|
|
|
|
|
EXPORT_SYMBOL_GPL(rcu_tasks_get_gp_data);
|
|
|
|
|
|
2022-11-25 06:54:58 -07:00
|
|
|
/*
|
2024-02-02 12:28:45 -07:00
|
|
|
* Protect against tasklist scan blind spot while the task is exiting and
|
|
|
|
|
* may be removed from the tasklist. Do this by adding the task to yet
|
|
|
|
|
* another list.
|
|
|
|
|
*
|
|
|
|
|
* Note that the task will remove itself from this list, so there is no
|
|
|
|
|
* need for get_task_struct(), except in the case where rcu_tasks_pertask()
|
|
|
|
|
* adds it to the holdout list, in which case rcu_tasks_pertask() supplies
|
|
|
|
|
* the needed get_task_struct().
|
2022-11-25 06:54:58 -07:00
|
|
|
*/
|
2024-02-02 12:28:45 -07:00
|
|
|
void exit_tasks_rcu_start(void)
|
2020-04-05 20:49:13 -07:00
|
|
|
{
|
2024-02-02 12:28:45 -07:00
|
|
|
unsigned long flags;
|
|
|
|
|
struct rcu_tasks_percpu *rtpcp;
|
|
|
|
|
struct task_struct *t = current;
|
|
|
|
|
|
|
|
|
|
WARN_ON_ONCE(!list_empty(&t->rcu_tasks_exit_list));
|
|
|
|
|
preempt_disable();
|
|
|
|
|
rtpcp = this_cpu_ptr(rcu_tasks.rtpcpu);
|
|
|
|
|
t->rcu_tasks_exit_cpu = smp_processor_id();
|
|
|
|
|
raw_spin_lock_irqsave_rcu_node(rtpcp, flags);
|
2024-02-23 18:07:24 -07:00
|
|
|
WARN_ON_ONCE(!rtpcp->rtp_exit_list.next);
|
2024-02-02 12:28:45 -07:00
|
|
|
list_add(&t->rcu_tasks_exit_list, &rtpcp->rtp_exit_list);
|
|
|
|
|
raw_spin_unlock_irqrestore_rcu_node(rtpcp, flags);
|
|
|
|
|
preempt_enable();
|
2020-04-05 20:49:13 -07:00
|
|
|
}
|
|
|
|
|
|
2022-11-25 06:54:58 -07:00
|
|
|
/*
|
2024-02-02 12:28:45 -07:00
|
|
|
* Remove the task from the "yet another list" because do_exit() is now
|
|
|
|
|
* non-preemptible, allowing synchronize_rcu() to wait beyond this point.
|
2022-11-25 06:54:58 -07:00
|
|
|
*/
|
2024-04-25 07:24:04 -07:00
|
|
|
void exit_tasks_rcu_finish(void)
|
2020-04-05 20:49:13 -07:00
|
|
|
{
|
2024-02-02 12:28:45 -07:00
|
|
|
unsigned long flags;
|
|
|
|
|
struct rcu_tasks_percpu *rtpcp;
|
2020-04-05 20:49:13 -07:00
|
|
|
struct task_struct *t = current;
|
|
|
|
|
|
2024-02-02 12:28:45 -07:00
|
|
|
WARN_ON_ONCE(list_empty(&t->rcu_tasks_exit_list));
|
|
|
|
|
rtpcp = per_cpu_ptr(rcu_tasks.rtpcpu, t->rcu_tasks_exit_cpu);
|
|
|
|
|
raw_spin_lock_irqsave_rcu_node(rtpcp, flags);
|
|
|
|
|
list_del_init(&t->rcu_tasks_exit_list);
|
|
|
|
|
raw_spin_unlock_irqrestore_rcu_node(rtpcp, flags);
|
2022-11-25 06:55:00 -07:00
|
|
|
|
2024-04-25 07:24:04 -07:00
|
|
|
exit_tasks_rcu_finish_trace(t);
|
2020-04-05 20:49:13 -07:00
|
|
|
}
|
|
|
|
|
|
2020-03-16 11:01:55 -07:00
|
|
|
#else /* #ifdef CONFIG_TASKS_RCU */
|
2020-04-05 20:49:13 -07:00
|
|
|
void exit_tasks_rcu_start(void) { }
|
|
|
|
|
void exit_tasks_rcu_finish(void) { exit_tasks_rcu_finish_trace(current); }
|
2020-03-16 11:01:55 -07:00
|
|
|
#endif /* #else #ifdef CONFIG_TASKS_RCU */
|
2020-03-02 22:06:43 -07:00
|
|
|
|
|
|
|
|
#ifdef CONFIG_TASKS_RUDE_RCU
|
|
|
|
|
|
|
|
|
|
////////////////////////////////////////////////////////////////////////
|
|
|
|
|
//
|
2024-07-03 19:47:47 -07:00
|
|
|
// "Rude" variant of Tasks RCU, inspired by Steve Rostedt's
|
|
|
|
|
// trick of passing an empty function to schedule_on_each_cpu().
|
|
|
|
|
// This approach provides batching of concurrent calls to the synchronous
|
|
|
|
|
// synchronize_rcu_tasks_rude() API. This invokes schedule_on_each_cpu()
|
|
|
|
|
// in order to send IPIs far and wide and induces otherwise unnecessary
|
|
|
|
|
// context switches on all online CPUs, whether idle or not.
|
2021-03-04 15:46:59 -07:00
|
|
|
//
|
|
|
|
|
// Callback handling is provided by the rcu_tasks_kthread() function.
|
|
|
|
|
//
|
|
|
|
|
// Ordering is provided by the scheduler's context-switch code.
|
2020-03-02 22:06:43 -07:00
|
|
|
|
|
|
|
|
// Empty function to allow workqueues to force a context switch.
|
|
|
|
|
static void rcu_tasks_be_rude(struct work_struct *work)
|
|
|
|
|
{
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Wait for one rude RCU-tasks grace period.
|
|
|
|
|
static void rcu_tasks_rude_wait_gp(struct rcu_tasks *rtp)
|
|
|
|
|
{
|
2020-03-18 10:54:05 -07:00
|
|
|
rtp->n_ipis += cpumask_weight(cpu_online_mask);
|
2020-03-02 22:06:43 -07:00
|
|
|
schedule_on_each_cpu(rcu_tasks_be_rude);
|
|
|
|
|
}
|
|
|
|
|
|
2024-07-03 19:47:47 -07:00
|
|
|
static void call_rcu_tasks_rude(struct rcu_head *rhp, rcu_callback_t func);
|
2020-03-03 16:50:31 -07:00
|
|
|
DEFINE_RCU_TASKS(rcu_tasks_rude, rcu_tasks_rude_wait_gp, call_rcu_tasks_rude,
|
|
|
|
|
"RCU Tasks Rude");
|
2020-03-02 22:06:43 -07:00
|
|
|
|
2024-07-03 19:47:47 -07:00
|
|
|
/*
|
2020-03-02 22:06:43 -07:00
|
|
|
* call_rcu_tasks_rude() - Queue a callback rude task-based grace period
|
|
|
|
|
* @rhp: structure to be used for queueing the RCU updates.
|
|
|
|
|
* @func: actual callback function to be invoked after the grace period
|
|
|
|
|
*
|
|
|
|
|
* The callback function will be invoked some time after a full grace
|
|
|
|
|
* period elapses, in other words after all currently executing RCU
|
|
|
|
|
* read-side critical sections have completed. call_rcu_tasks_rude()
|
|
|
|
|
* assumes that the read-side critical sections end at context switch,
|
2021-09-15 09:24:18 -07:00
|
|
|
* cond_resched_tasks_rcu_qs(), or transition to usermode execution (as
|
2021-08-18 00:28:43 -07:00
|
|
|
* usermode execution is schedulable). As such, there are no read-side
|
|
|
|
|
* primitives analogous to rcu_read_lock() and rcu_read_unlock() because
|
|
|
|
|
* this primitive is intended to determine that all tasks have passed
|
|
|
|
|
* through a safe state, not so much for data-structure synchronization.
|
2020-03-02 22:06:43 -07:00
|
|
|
*
|
|
|
|
|
* See the description of call_rcu() for more detailed information on
|
|
|
|
|
* memory ordering guarantees.
|
2024-07-03 19:47:47 -07:00
|
|
|
*
|
|
|
|
|
* This is no longer exported, and is instead reserved for use by
|
|
|
|
|
* synchronize_rcu_tasks_rude().
|
2020-03-02 22:06:43 -07:00
|
|
|
*/
|
2024-07-03 19:47:47 -07:00
|
|
|
static void call_rcu_tasks_rude(struct rcu_head *rhp, rcu_callback_t func)
|
2020-03-02 22:06:43 -07:00
|
|
|
{
|
|
|
|
|
call_rcu_tasks_generic(rhp, func, &rcu_tasks_rude);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* synchronize_rcu_tasks_rude - wait for a rude rcu-tasks grace period
|
|
|
|
|
*
|
|
|
|
|
* Control will return to the caller some time after a rude rcu-tasks
|
|
|
|
|
* grace period has elapsed, in other words after all currently
|
|
|
|
|
* executing rcu-tasks read-side critical sections have elapsed. These
|
|
|
|
|
* read-side critical sections are delimited by calls to schedule(),
|
2021-08-18 00:28:43 -07:00
|
|
|
* cond_resched_tasks_rcu_qs(), userspace execution (which is a schedulable
|
|
|
|
|
* context), and (in theory, anyway) cond_resched().
|
2020-03-02 22:06:43 -07:00
|
|
|
*
|
|
|
|
|
* This is a very specialized primitive, intended only for a few uses in
|
|
|
|
|
* tracing and other situations requiring manipulation of function preambles
|
|
|
|
|
* and profiling hooks. The synchronize_rcu_tasks_rude() function is not
|
|
|
|
|
* (yet) intended for heavy use from multiple CPUs.
|
|
|
|
|
*
|
|
|
|
|
* See the description of synchronize_rcu() for more detailed information
|
|
|
|
|
* on memory ordering guarantees.
|
|
|
|
|
*/
|
|
|
|
|
void synchronize_rcu_tasks_rude(void)
|
|
|
|
|
{
|
|
|
|
|
synchronize_rcu_tasks_generic(&rcu_tasks_rude);
|
|
|
|
|
}
|
|
|
|
|
EXPORT_SYMBOL_GPL(synchronize_rcu_tasks_rude);
|
|
|
|
|
|
|
|
|
|
static int __init rcu_spawn_tasks_rude_kthread(void)
|
|
|
|
|
{
|
2020-09-09 22:05:41 -07:00
|
|
|
rcu_tasks_rude.gp_sleep = HZ / 10;
|
2020-03-02 22:06:43 -07:00
|
|
|
rcu_spawn_tasks_kthread_generic(&rcu_tasks_rude);
|
|
|
|
|
return 0;
|
|
|
|
|
}
|
|
|
|
|
|
2020-09-15 17:08:03 -07:00
|
|
|
#if !defined(CONFIG_TINY_RCU)
|
|
|
|
|
void show_rcu_tasks_rude_gp_kthread(void)
|
2020-03-16 11:01:55 -07:00
|
|
|
{
|
|
|
|
|
show_rcu_tasks_generic_gp_kthread(&rcu_tasks_rude, "");
|
|
|
|
|
}
|
2020-09-15 17:08:03 -07:00
|
|
|
EXPORT_SYMBOL_GPL(show_rcu_tasks_rude_gp_kthread);
|
2024-08-01 17:34:25 -07:00
|
|
|
|
|
|
|
|
void rcu_tasks_rude_torture_stats_print(char *tt, char *tf)
|
|
|
|
|
{
|
|
|
|
|
rcu_tasks_torture_stats_print_generic(&rcu_tasks_rude, tt, tf, "");
|
|
|
|
|
}
|
|
|
|
|
EXPORT_SYMBOL_GPL(rcu_tasks_rude_torture_stats_print);
|
2020-09-15 17:08:03 -07:00
|
|
|
#endif // !defined(CONFIG_TINY_RCU)
|
2023-06-03 09:07:44 -07:00
|
|
|
|
|
|
|
|
struct task_struct *get_rcu_tasks_rude_gp_kthread(void)
|
|
|
|
|
{
|
|
|
|
|
return rcu_tasks_rude.kthread_ptr;
|
|
|
|
|
}
|
|
|
|
|
EXPORT_SYMBOL_GPL(get_rcu_tasks_rude_gp_kthread);
|
|
|
|
|
|
2024-03-18 02:34:11 -07:00
|
|
|
void rcu_tasks_rude_get_gp_data(int *flags, unsigned long *gp_seq)
|
|
|
|
|
{
|
|
|
|
|
*flags = 0;
|
|
|
|
|
*gp_seq = rcu_seq_current(&rcu_tasks_rude.tasks_gp_seq);
|
|
|
|
|
}
|
|
|
|
|
EXPORT_SYMBOL_GPL(rcu_tasks_rude_get_gp_data);
|
|
|
|
|
|
2020-09-15 17:08:03 -07:00
|
|
|
#endif /* #ifdef CONFIG_TASKS_RUDE_RCU */
|
rcu-tasks: Add an RCU Tasks Trace to simplify protection of tracing hooks
Because RCU does not watch exception early-entry/late-exit, idle-loop,
or CPU-hotplug execution, protection of tracing and BPF operations is
needlessly complicated. This commit therefore adds a variant of
Tasks RCU that:
o Has explicit read-side markers to allow finite grace periods in
the face of in-kernel loops for PREEMPT=n builds. These markers
are rcu_read_lock_trace() and rcu_read_unlock_trace().
o Protects code in the idle loop, exception entry/exit, and
CPU-hotplug code paths. In this respect, RCU-tasks trace is
similar to SRCU, but with lighter-weight readers.
o Avoids expensive read-side instruction, having overhead similar
to that of Preemptible RCU.
There are of course downsides:
o The grace-period code can send IPIs to CPUs, even when those
CPUs are in the idle loop or in nohz_full userspace. This is
mitigated by later commits.
o It is necessary to scan the full tasklist, much as for Tasks RCU.
o There is a single callback queue guarded by a single lock,
again, much as for Tasks RCU. However, those early use cases
that request multiple grace periods in quick succession are
expected to do so from a single task, which makes the single
lock almost irrelevant. If needed, multiple callback queues
can be provided using any number of schemes.
Perhaps most important, this variant of RCU does not affect the vanilla
flavors, rcu_preempt and rcu_sched. The fact that RCU Tasks Trace
readers can operate from idle, offline, and exception entry/exit in no
way enables rcu_preempt and rcu_sched readers to do so.
The memory ordering was outlined here:
https://lore.kernel.org/lkml/20200319034030.GX3199@paulmck-ThinkPad-P72/
This effort benefited greatly from off-list discussions of BPF
requirements with Alexei Starovoitov and Andrii Nakryiko. At least
some of the on-list discussions are captured in the Link: tags below.
In addition, KCSAN was quite helpful in finding some early bugs.
Link: https://lore.kernel.org/lkml/20200219150744.428764577@infradead.org/
Link: https://lore.kernel.org/lkml/87mu8p797b.fsf@nanos.tec.linutronix.de/
Link: https://lore.kernel.org/lkml/20200225221305.605144982@linutronix.de/
Cc: Alexei Starovoitov <alexei.starovoitov@gmail.com>
Cc: Andrii Nakryiko <andriin@fb.com>
[ paulmck: Apply feedback from Steve Rostedt and Joel Fernandes. ]
[ paulmck: Decrement trc_n_readers_need_end upon IPI failure. ]
[ paulmck: Fix locking issue reported by rcutorture. ]
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
2020-03-09 19:56:53 -07:00
|
|
|
|
|
|
|
|
////////////////////////////////////////////////////////////////////////
|
|
|
|
|
//
|
|
|
|
|
// Tracing variant of Tasks RCU. This variant is designed to be used
|
|
|
|
|
// to protect tracing hooks, including those of BPF. This variant
|
|
|
|
|
// therefore:
|
|
|
|
|
//
|
|
|
|
|
// 1. Has explicit read-side markers to allow finite grace periods
|
|
|
|
|
// in the face of in-kernel loops for PREEMPT=n builds.
|
|
|
|
|
//
|
|
|
|
|
// 2. Protects code in the idle loop, exception entry/exit, and
|
|
|
|
|
// CPU-hotplug code paths, similar to the capabilities of SRCU.
|
|
|
|
|
//
|
2021-08-05 09:54:45 -07:00
|
|
|
// 3. Avoids expensive read-side instructions, having overhead similar
|
rcu-tasks: Add an RCU Tasks Trace to simplify protection of tracing hooks
Because RCU does not watch exception early-entry/late-exit, idle-loop,
or CPU-hotplug execution, protection of tracing and BPF operations is
needlessly complicated. This commit therefore adds a variant of
Tasks RCU that:
o Has explicit read-side markers to allow finite grace periods in
the face of in-kernel loops for PREEMPT=n builds. These markers
are rcu_read_lock_trace() and rcu_read_unlock_trace().
o Protects code in the idle loop, exception entry/exit, and
CPU-hotplug code paths. In this respect, RCU-tasks trace is
similar to SRCU, but with lighter-weight readers.
o Avoids expensive read-side instruction, having overhead similar
to that of Preemptible RCU.
There are of course downsides:
o The grace-period code can send IPIs to CPUs, even when those
CPUs are in the idle loop or in nohz_full userspace. This is
mitigated by later commits.
o It is necessary to scan the full tasklist, much as for Tasks RCU.
o There is a single callback queue guarded by a single lock,
again, much as for Tasks RCU. However, those early use cases
that request multiple grace periods in quick succession are
expected to do so from a single task, which makes the single
lock almost irrelevant. If needed, multiple callback queues
can be provided using any number of schemes.
Perhaps most important, this variant of RCU does not affect the vanilla
flavors, rcu_preempt and rcu_sched. The fact that RCU Tasks Trace
readers can operate from idle, offline, and exception entry/exit in no
way enables rcu_preempt and rcu_sched readers to do so.
The memory ordering was outlined here:
https://lore.kernel.org/lkml/20200319034030.GX3199@paulmck-ThinkPad-P72/
This effort benefited greatly from off-list discussions of BPF
requirements with Alexei Starovoitov and Andrii Nakryiko. At least
some of the on-list discussions are captured in the Link: tags below.
In addition, KCSAN was quite helpful in finding some early bugs.
Link: https://lore.kernel.org/lkml/20200219150744.428764577@infradead.org/
Link: https://lore.kernel.org/lkml/87mu8p797b.fsf@nanos.tec.linutronix.de/
Link: https://lore.kernel.org/lkml/20200225221305.605144982@linutronix.de/
Cc: Alexei Starovoitov <alexei.starovoitov@gmail.com>
Cc: Andrii Nakryiko <andriin@fb.com>
[ paulmck: Apply feedback from Steve Rostedt and Joel Fernandes. ]
[ paulmck: Decrement trc_n_readers_need_end upon IPI failure. ]
[ paulmck: Fix locking issue reported by rcutorture. ]
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
2020-03-09 19:56:53 -07:00
|
|
|
// to that of Preemptible RCU.
|
|
|
|
|
//
|
2022-06-06 21:30:38 -07:00
|
|
|
// There are of course downsides. For example, the grace-period code
|
|
|
|
|
// can send IPIs to CPUs, even when those CPUs are in the idle loop or
|
|
|
|
|
// in nohz_full userspace. If needed, these downsides can be at least
|
|
|
|
|
// partially remedied.
|
rcu-tasks: Add an RCU Tasks Trace to simplify protection of tracing hooks
Because RCU does not watch exception early-entry/late-exit, idle-loop,
or CPU-hotplug execution, protection of tracing and BPF operations is
needlessly complicated. This commit therefore adds a variant of
Tasks RCU that:
o Has explicit read-side markers to allow finite grace periods in
the face of in-kernel loops for PREEMPT=n builds. These markers
are rcu_read_lock_trace() and rcu_read_unlock_trace().
o Protects code in the idle loop, exception entry/exit, and
CPU-hotplug code paths. In this respect, RCU-tasks trace is
similar to SRCU, but with lighter-weight readers.
o Avoids expensive read-side instruction, having overhead similar
to that of Preemptible RCU.
There are of course downsides:
o The grace-period code can send IPIs to CPUs, even when those
CPUs are in the idle loop or in nohz_full userspace. This is
mitigated by later commits.
o It is necessary to scan the full tasklist, much as for Tasks RCU.
o There is a single callback queue guarded by a single lock,
again, much as for Tasks RCU. However, those early use cases
that request multiple grace periods in quick succession are
expected to do so from a single task, which makes the single
lock almost irrelevant. If needed, multiple callback queues
can be provided using any number of schemes.
Perhaps most important, this variant of RCU does not affect the vanilla
flavors, rcu_preempt and rcu_sched. The fact that RCU Tasks Trace
readers can operate from idle, offline, and exception entry/exit in no
way enables rcu_preempt and rcu_sched readers to do so.
The memory ordering was outlined here:
https://lore.kernel.org/lkml/20200319034030.GX3199@paulmck-ThinkPad-P72/
This effort benefited greatly from off-list discussions of BPF
requirements with Alexei Starovoitov and Andrii Nakryiko. At least
some of the on-list discussions are captured in the Link: tags below.
In addition, KCSAN was quite helpful in finding some early bugs.
Link: https://lore.kernel.org/lkml/20200219150744.428764577@infradead.org/
Link: https://lore.kernel.org/lkml/87mu8p797b.fsf@nanos.tec.linutronix.de/
Link: https://lore.kernel.org/lkml/20200225221305.605144982@linutronix.de/
Cc: Alexei Starovoitov <alexei.starovoitov@gmail.com>
Cc: Andrii Nakryiko <andriin@fb.com>
[ paulmck: Apply feedback from Steve Rostedt and Joel Fernandes. ]
[ paulmck: Decrement trc_n_readers_need_end upon IPI failure. ]
[ paulmck: Fix locking issue reported by rcutorture. ]
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
2020-03-09 19:56:53 -07:00
|
|
|
//
|
|
|
|
|
// Perhaps most important, this variant of RCU does not affect the vanilla
|
|
|
|
|
// flavors, rcu_preempt and rcu_sched. The fact that RCU Tasks Trace
|
|
|
|
|
// readers can operate from idle, offline, and exception entry/exit in no
|
|
|
|
|
// way allows rcu_preempt and rcu_sched readers to also do so.
|
2021-02-25 11:26:00 -07:00
|
|
|
//
|
|
|
|
|
// The implementation uses rcu_tasks_wait_gp(), which relies on function
|
|
|
|
|
// pointers in the rcu_tasks structure. The rcu_spawn_tasks_trace_kthread()
|
|
|
|
|
// function sets these function pointers up so that rcu_tasks_wait_gp()
|
|
|
|
|
// invokes these functions in this order:
|
|
|
|
|
//
|
|
|
|
|
// rcu_tasks_trace_pregp_step():
|
2022-06-06 21:30:38 -07:00
|
|
|
// Disables CPU hotplug, adds all currently executing tasks to the
|
|
|
|
|
// holdout list, then checks the state of all tasks that blocked
|
|
|
|
|
// or were preempted within their current RCU Tasks Trace read-side
|
|
|
|
|
// critical section, adding them to the holdout list if appropriate.
|
|
|
|
|
// Finally, this function re-enables CPU hotplug.
|
|
|
|
|
// The ->pertask_func() pointer is NULL, so there is no per-task processing.
|
2021-02-25 11:26:00 -07:00
|
|
|
// rcu_tasks_trace_postscan():
|
2022-06-06 21:30:38 -07:00
|
|
|
// Invokes synchronize_rcu() to wait for late-stage exiting tasks
|
|
|
|
|
// to finish exiting.
|
2021-02-25 11:26:00 -07:00
|
|
|
// check_all_holdout_tasks_trace(), repeatedly until holdout list is empty:
|
|
|
|
|
// Scans the holdout list, attempting to identify a quiescent state
|
|
|
|
|
// for each task on the list. If there is a quiescent state, the
|
2022-06-06 21:30:38 -07:00
|
|
|
// corresponding task is removed from the holdout list. Once this
|
|
|
|
|
// list is empty, the grace period has completed.
|
2021-02-25 11:26:00 -07:00
|
|
|
// rcu_tasks_trace_postgp():
|
2022-06-06 21:30:38 -07:00
|
|
|
// Provides the needed full memory barrier and does debug checks.
|
2021-02-25 11:26:00 -07:00
|
|
|
//
|
|
|
|
|
// The exit_tasks_rcu_finish_trace() synchronizes with exiting tasks.
|
|
|
|
|
//
|
2022-06-06 21:30:38 -07:00
|
|
|
// Pre-grace-period update-side code is ordered before the grace period
|
|
|
|
|
// via the ->cbs_lock and barriers in rcu_tasks_kthread(). Pre-grace-period
|
|
|
|
|
// read-side code is ordered before the grace period by atomic operations
|
|
|
|
|
// on .b.need_qs flag of each task involved in this process, or by scheduler
|
|
|
|
|
// context-switch ordering (for locked-down non-running readers).
|
rcu-tasks: Add an RCU Tasks Trace to simplify protection of tracing hooks
Because RCU does not watch exception early-entry/late-exit, idle-loop,
or CPU-hotplug execution, protection of tracing and BPF operations is
needlessly complicated. This commit therefore adds a variant of
Tasks RCU that:
o Has explicit read-side markers to allow finite grace periods in
the face of in-kernel loops for PREEMPT=n builds. These markers
are rcu_read_lock_trace() and rcu_read_unlock_trace().
o Protects code in the idle loop, exception entry/exit, and
CPU-hotplug code paths. In this respect, RCU-tasks trace is
similar to SRCU, but with lighter-weight readers.
o Avoids expensive read-side instruction, having overhead similar
to that of Preemptible RCU.
There are of course downsides:
o The grace-period code can send IPIs to CPUs, even when those
CPUs are in the idle loop or in nohz_full userspace. This is
mitigated by later commits.
o It is necessary to scan the full tasklist, much as for Tasks RCU.
o There is a single callback queue guarded by a single lock,
again, much as for Tasks RCU. However, those early use cases
that request multiple grace periods in quick succession are
expected to do so from a single task, which makes the single
lock almost irrelevant. If needed, multiple callback queues
can be provided using any number of schemes.
Perhaps most important, this variant of RCU does not affect the vanilla
flavors, rcu_preempt and rcu_sched. The fact that RCU Tasks Trace
readers can operate from idle, offline, and exception entry/exit in no
way enables rcu_preempt and rcu_sched readers to do so.
The memory ordering was outlined here:
https://lore.kernel.org/lkml/20200319034030.GX3199@paulmck-ThinkPad-P72/
This effort benefited greatly from off-list discussions of BPF
requirements with Alexei Starovoitov and Andrii Nakryiko. At least
some of the on-list discussions are captured in the Link: tags below.
In addition, KCSAN was quite helpful in finding some early bugs.
Link: https://lore.kernel.org/lkml/20200219150744.428764577@infradead.org/
Link: https://lore.kernel.org/lkml/87mu8p797b.fsf@nanos.tec.linutronix.de/
Link: https://lore.kernel.org/lkml/20200225221305.605144982@linutronix.de/
Cc: Alexei Starovoitov <alexei.starovoitov@gmail.com>
Cc: Andrii Nakryiko <andriin@fb.com>
[ paulmck: Apply feedback from Steve Rostedt and Joel Fernandes. ]
[ paulmck: Decrement trc_n_readers_need_end upon IPI failure. ]
[ paulmck: Fix locking issue reported by rcutorture. ]
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
2020-03-09 19:56:53 -07:00
|
|
|
|
|
|
|
|
// The lockdep state must be outside of #ifdef to be useful.
|
|
|
|
|
#ifdef CONFIG_DEBUG_LOCK_ALLOC
|
|
|
|
|
static struct lock_class_key rcu_lock_trace_key;
|
|
|
|
|
struct lockdep_map rcu_trace_lock_map =
|
|
|
|
|
STATIC_LOCKDEP_MAP_INIT("rcu_read_lock_trace", &rcu_lock_trace_key);
|
|
|
|
|
EXPORT_SYMBOL_GPL(rcu_trace_lock_map);
|
|
|
|
|
#endif /* #ifdef CONFIG_DEBUG_LOCK_ALLOC */
|
|
|
|
|
|
|
|
|
|
#ifdef CONFIG_TASKS_TRACE_RCU
|
|
|
|
|
|
|
|
|
|
// Record outstanding IPIs to each CPU. No point in sending two...
|
|
|
|
|
static DEFINE_PER_CPU(bool, trc_ipi_to_cpu);
|
|
|
|
|
|
2020-03-22 13:34:34 -07:00
|
|
|
// The number of detections of task quiescent state relying on
|
|
|
|
|
// heavyweight readers executing explicit memory barriers.
|
2020-09-09 14:14:34 -07:00
|
|
|
static unsigned long n_heavy_reader_attempts;
|
|
|
|
|
static unsigned long n_heavy_reader_updates;
|
|
|
|
|
static unsigned long n_heavy_reader_ofl_updates;
|
2022-06-01 21:26:57 -07:00
|
|
|
static unsigned long n_trc_holdouts;
|
2020-03-22 13:34:34 -07:00
|
|
|
|
2020-03-17 11:39:26 -07:00
|
|
|
void call_rcu_tasks_trace(struct rcu_head *rhp, rcu_callback_t func);
|
|
|
|
|
DEFINE_RCU_TASKS(rcu_tasks_trace, rcu_tasks_wait_gp, call_rcu_tasks_trace,
|
|
|
|
|
"RCU Tasks Trace");
|
|
|
|
|
|
2022-05-23 20:50:11 -07:00
|
|
|
/* Load from ->trc_reader_special.b.need_qs with proper ordering. */
|
|
|
|
|
static u8 rcu_ld_need_qs(struct task_struct *t)
|
|
|
|
|
{
|
|
|
|
|
smp_mb(); // Enforce full grace-period ordering.
|
|
|
|
|
return smp_load_acquire(&t->trc_reader_special.b.need_qs);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/* Store to ->trc_reader_special.b.need_qs with proper ordering. */
|
|
|
|
|
static void rcu_st_need_qs(struct task_struct *t, u8 v)
|
|
|
|
|
{
|
|
|
|
|
smp_store_release(&t->trc_reader_special.b.need_qs, v);
|
|
|
|
|
smp_mb(); // Enforce full grace-period ordering.
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* Do a cmpxchg() on ->trc_reader_special.b.need_qs, allowing for
|
|
|
|
|
* the four-byte operand-size restriction of some platforms.
|
2024-03-08 14:26:26 -07:00
|
|
|
*
|
2022-05-23 20:50:11 -07:00
|
|
|
* Returns the old value, which is often ignored.
|
|
|
|
|
*/
|
|
|
|
|
u8 rcu_trc_cmpxchg_need_qs(struct task_struct *t, u8 old, u8 new)
|
|
|
|
|
{
|
|
|
|
|
union rcu_special ret;
|
|
|
|
|
union rcu_special trs_old = READ_ONCE(t->trc_reader_special);
|
|
|
|
|
union rcu_special trs_new = trs_old;
|
|
|
|
|
|
|
|
|
|
if (trs_old.b.need_qs != old)
|
|
|
|
|
return trs_old.b.need_qs;
|
|
|
|
|
trs_new.b.need_qs = new;
|
2024-03-08 14:26:26 -07:00
|
|
|
|
|
|
|
|
// Although cmpxchg() appears to KCSAN to update all four bytes,
|
|
|
|
|
// only the .b.need_qs byte actually changes.
|
|
|
|
|
instrument_atomic_read_write(&t->trc_reader_special.b.need_qs,
|
|
|
|
|
sizeof(t->trc_reader_special.b.need_qs));
|
|
|
|
|
// Avoid false-positive KCSAN failures.
|
|
|
|
|
ret.s = data_race(cmpxchg(&t->trc_reader_special.s, trs_old.s, trs_new.s));
|
|
|
|
|
|
2022-05-23 20:50:11 -07:00
|
|
|
return ret.b.need_qs;
|
|
|
|
|
}
|
|
|
|
|
EXPORT_SYMBOL_GPL(rcu_trc_cmpxchg_need_qs);
|
|
|
|
|
|
2022-06-06 21:30:38 -07:00
|
|
|
/*
|
|
|
|
|
* If we are the last reader, signal the grace-period kthread.
|
|
|
|
|
* Also remove from the per-CPU list of blocked tasks.
|
|
|
|
|
*/
|
2021-07-28 12:28:27 -07:00
|
|
|
void rcu_read_unlock_trace_special(struct task_struct *t)
|
rcu-tasks: Add an RCU Tasks Trace to simplify protection of tracing hooks
Because RCU does not watch exception early-entry/late-exit, idle-loop,
or CPU-hotplug execution, protection of tracing and BPF operations is
needlessly complicated. This commit therefore adds a variant of
Tasks RCU that:
o Has explicit read-side markers to allow finite grace periods in
the face of in-kernel loops for PREEMPT=n builds. These markers
are rcu_read_lock_trace() and rcu_read_unlock_trace().
o Protects code in the idle loop, exception entry/exit, and
CPU-hotplug code paths. In this respect, RCU-tasks trace is
similar to SRCU, but with lighter-weight readers.
o Avoids expensive read-side instruction, having overhead similar
to that of Preemptible RCU.
There are of course downsides:
o The grace-period code can send IPIs to CPUs, even when those
CPUs are in the idle loop or in nohz_full userspace. This is
mitigated by later commits.
o It is necessary to scan the full tasklist, much as for Tasks RCU.
o There is a single callback queue guarded by a single lock,
again, much as for Tasks RCU. However, those early use cases
that request multiple grace periods in quick succession are
expected to do so from a single task, which makes the single
lock almost irrelevant. If needed, multiple callback queues
can be provided using any number of schemes.
Perhaps most important, this variant of RCU does not affect the vanilla
flavors, rcu_preempt and rcu_sched. The fact that RCU Tasks Trace
readers can operate from idle, offline, and exception entry/exit in no
way enables rcu_preempt and rcu_sched readers to do so.
The memory ordering was outlined here:
https://lore.kernel.org/lkml/20200319034030.GX3199@paulmck-ThinkPad-P72/
This effort benefited greatly from off-list discussions of BPF
requirements with Alexei Starovoitov and Andrii Nakryiko. At least
some of the on-list discussions are captured in the Link: tags below.
In addition, KCSAN was quite helpful in finding some early bugs.
Link: https://lore.kernel.org/lkml/20200219150744.428764577@infradead.org/
Link: https://lore.kernel.org/lkml/87mu8p797b.fsf@nanos.tec.linutronix.de/
Link: https://lore.kernel.org/lkml/20200225221305.605144982@linutronix.de/
Cc: Alexei Starovoitov <alexei.starovoitov@gmail.com>
Cc: Andrii Nakryiko <andriin@fb.com>
[ paulmck: Apply feedback from Steve Rostedt and Joel Fernandes. ]
[ paulmck: Decrement trc_n_readers_need_end upon IPI failure. ]
[ paulmck: Fix locking issue reported by rcutorture. ]
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
2020-03-09 19:56:53 -07:00
|
|
|
{
|
2022-05-17 15:01:14 -07:00
|
|
|
unsigned long flags;
|
|
|
|
|
struct rcu_tasks_percpu *rtpcp;
|
|
|
|
|
union rcu_special trs;
|
|
|
|
|
|
|
|
|
|
// Open-coded full-word version of rcu_ld_need_qs().
|
|
|
|
|
smp_mb(); // Enforce full grace-period ordering.
|
|
|
|
|
trs = smp_load_acquire(&t->trc_reader_special);
|
2020-03-17 16:02:06 -07:00
|
|
|
|
2022-05-23 20:50:11 -07:00
|
|
|
if (IS_ENABLED(CONFIG_TASKS_TRACE_RCU_READ_MB) && t->trc_reader_special.b.need_mb)
|
2020-03-17 16:02:06 -07:00
|
|
|
smp_mb(); // Pairs with update-side barriers.
|
|
|
|
|
// Update .need_qs before ->trc_reader_nesting for irq/NMI handlers.
|
2022-05-17 15:01:14 -07:00
|
|
|
if (trs.b.need_qs == (TRC_NEED_QS_CHECKED | TRC_NEED_QS)) {
|
2022-05-23 20:50:11 -07:00
|
|
|
u8 result = rcu_trc_cmpxchg_need_qs(t, TRC_NEED_QS_CHECKED | TRC_NEED_QS,
|
|
|
|
|
TRC_NEED_QS_CHECKED);
|
|
|
|
|
|
2022-05-17 15:01:14 -07:00
|
|
|
WARN_ONCE(result != trs.b.need_qs, "%s: result = %d", __func__, result);
|
|
|
|
|
}
|
|
|
|
|
if (trs.b.blocked) {
|
|
|
|
|
rtpcp = per_cpu_ptr(rcu_tasks_trace.rtpcpu, t->trc_blkd_cpu);
|
|
|
|
|
raw_spin_lock_irqsave_rcu_node(rtpcp, flags);
|
|
|
|
|
list_del_init(&t->trc_blkd_node);
|
|
|
|
|
WRITE_ONCE(t->trc_reader_special.b.blocked, false);
|
|
|
|
|
raw_spin_unlock_irqrestore_rcu_node(rtpcp, flags);
|
2022-05-23 20:50:11 -07:00
|
|
|
}
|
2021-07-28 12:28:27 -07:00
|
|
|
WRITE_ONCE(t->trc_reader_nesting, 0);
|
rcu-tasks: Add an RCU Tasks Trace to simplify protection of tracing hooks
Because RCU does not watch exception early-entry/late-exit, idle-loop,
or CPU-hotplug execution, protection of tracing and BPF operations is
needlessly complicated. This commit therefore adds a variant of
Tasks RCU that:
o Has explicit read-side markers to allow finite grace periods in
the face of in-kernel loops for PREEMPT=n builds. These markers
are rcu_read_lock_trace() and rcu_read_unlock_trace().
o Protects code in the idle loop, exception entry/exit, and
CPU-hotplug code paths. In this respect, RCU-tasks trace is
similar to SRCU, but with lighter-weight readers.
o Avoids expensive read-side instruction, having overhead similar
to that of Preemptible RCU.
There are of course downsides:
o The grace-period code can send IPIs to CPUs, even when those
CPUs are in the idle loop or in nohz_full userspace. This is
mitigated by later commits.
o It is necessary to scan the full tasklist, much as for Tasks RCU.
o There is a single callback queue guarded by a single lock,
again, much as for Tasks RCU. However, those early use cases
that request multiple grace periods in quick succession are
expected to do so from a single task, which makes the single
lock almost irrelevant. If needed, multiple callback queues
can be provided using any number of schemes.
Perhaps most important, this variant of RCU does not affect the vanilla
flavors, rcu_preempt and rcu_sched. The fact that RCU Tasks Trace
readers can operate from idle, offline, and exception entry/exit in no
way enables rcu_preempt and rcu_sched readers to do so.
The memory ordering was outlined here:
https://lore.kernel.org/lkml/20200319034030.GX3199@paulmck-ThinkPad-P72/
This effort benefited greatly from off-list discussions of BPF
requirements with Alexei Starovoitov and Andrii Nakryiko. At least
some of the on-list discussions are captured in the Link: tags below.
In addition, KCSAN was quite helpful in finding some early bugs.
Link: https://lore.kernel.org/lkml/20200219150744.428764577@infradead.org/
Link: https://lore.kernel.org/lkml/87mu8p797b.fsf@nanos.tec.linutronix.de/
Link: https://lore.kernel.org/lkml/20200225221305.605144982@linutronix.de/
Cc: Alexei Starovoitov <alexei.starovoitov@gmail.com>
Cc: Andrii Nakryiko <andriin@fb.com>
[ paulmck: Apply feedback from Steve Rostedt and Joel Fernandes. ]
[ paulmck: Decrement trc_n_readers_need_end upon IPI failure. ]
[ paulmck: Fix locking issue reported by rcutorture. ]
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
2020-03-09 19:56:53 -07:00
|
|
|
}
|
|
|
|
|
EXPORT_SYMBOL_GPL(rcu_read_unlock_trace_special);
|
|
|
|
|
|
2022-05-17 11:30:32 -07:00
|
|
|
/* Add a newly blocked reader task to its CPU's list. */
|
|
|
|
|
void rcu_tasks_trace_qs_blkd(struct task_struct *t)
|
|
|
|
|
{
|
|
|
|
|
unsigned long flags;
|
|
|
|
|
struct rcu_tasks_percpu *rtpcp;
|
|
|
|
|
|
|
|
|
|
local_irq_save(flags);
|
|
|
|
|
rtpcp = this_cpu_ptr(rcu_tasks_trace.rtpcpu);
|
|
|
|
|
raw_spin_lock_rcu_node(rtpcp); // irqs already disabled
|
|
|
|
|
t->trc_blkd_cpu = smp_processor_id();
|
|
|
|
|
if (!rtpcp->rtp_blkd_tasks.next)
|
|
|
|
|
INIT_LIST_HEAD(&rtpcp->rtp_blkd_tasks);
|
|
|
|
|
list_add(&t->trc_blkd_node, &rtpcp->rtp_blkd_tasks);
|
2022-05-17 15:01:14 -07:00
|
|
|
WRITE_ONCE(t->trc_reader_special.b.blocked, true);
|
2022-05-17 11:30:32 -07:00
|
|
|
raw_spin_unlock_irqrestore_rcu_node(rtpcp, flags);
|
|
|
|
|
}
|
|
|
|
|
EXPORT_SYMBOL_GPL(rcu_tasks_trace_qs_blkd);
|
|
|
|
|
|
rcu-tasks: Add an RCU Tasks Trace to simplify protection of tracing hooks
Because RCU does not watch exception early-entry/late-exit, idle-loop,
or CPU-hotplug execution, protection of tracing and BPF operations is
needlessly complicated. This commit therefore adds a variant of
Tasks RCU that:
o Has explicit read-side markers to allow finite grace periods in
the face of in-kernel loops for PREEMPT=n builds. These markers
are rcu_read_lock_trace() and rcu_read_unlock_trace().
o Protects code in the idle loop, exception entry/exit, and
CPU-hotplug code paths. In this respect, RCU-tasks trace is
similar to SRCU, but with lighter-weight readers.
o Avoids expensive read-side instruction, having overhead similar
to that of Preemptible RCU.
There are of course downsides:
o The grace-period code can send IPIs to CPUs, even when those
CPUs are in the idle loop or in nohz_full userspace. This is
mitigated by later commits.
o It is necessary to scan the full tasklist, much as for Tasks RCU.
o There is a single callback queue guarded by a single lock,
again, much as for Tasks RCU. However, those early use cases
that request multiple grace periods in quick succession are
expected to do so from a single task, which makes the single
lock almost irrelevant. If needed, multiple callback queues
can be provided using any number of schemes.
Perhaps most important, this variant of RCU does not affect the vanilla
flavors, rcu_preempt and rcu_sched. The fact that RCU Tasks Trace
readers can operate from idle, offline, and exception entry/exit in no
way enables rcu_preempt and rcu_sched readers to do so.
The memory ordering was outlined here:
https://lore.kernel.org/lkml/20200319034030.GX3199@paulmck-ThinkPad-P72/
This effort benefited greatly from off-list discussions of BPF
requirements with Alexei Starovoitov and Andrii Nakryiko. At least
some of the on-list discussions are captured in the Link: tags below.
In addition, KCSAN was quite helpful in finding some early bugs.
Link: https://lore.kernel.org/lkml/20200219150744.428764577@infradead.org/
Link: https://lore.kernel.org/lkml/87mu8p797b.fsf@nanos.tec.linutronix.de/
Link: https://lore.kernel.org/lkml/20200225221305.605144982@linutronix.de/
Cc: Alexei Starovoitov <alexei.starovoitov@gmail.com>
Cc: Andrii Nakryiko <andriin@fb.com>
[ paulmck: Apply feedback from Steve Rostedt and Joel Fernandes. ]
[ paulmck: Decrement trc_n_readers_need_end upon IPI failure. ]
[ paulmck: Fix locking issue reported by rcutorture. ]
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
2020-03-09 19:56:53 -07:00
|
|
|
/* Add a task to the holdout list, if it is not already on the list. */
|
|
|
|
|
static void trc_add_holdout(struct task_struct *t, struct list_head *bhp)
|
|
|
|
|
{
|
|
|
|
|
if (list_empty(&t->trc_holdout_list)) {
|
|
|
|
|
get_task_struct(t);
|
|
|
|
|
list_add(&t->trc_holdout_list, bhp);
|
2022-06-01 21:26:57 -07:00
|
|
|
n_trc_holdouts++;
|
rcu-tasks: Add an RCU Tasks Trace to simplify protection of tracing hooks
Because RCU does not watch exception early-entry/late-exit, idle-loop,
or CPU-hotplug execution, protection of tracing and BPF operations is
needlessly complicated. This commit therefore adds a variant of
Tasks RCU that:
o Has explicit read-side markers to allow finite grace periods in
the face of in-kernel loops for PREEMPT=n builds. These markers
are rcu_read_lock_trace() and rcu_read_unlock_trace().
o Protects code in the idle loop, exception entry/exit, and
CPU-hotplug code paths. In this respect, RCU-tasks trace is
similar to SRCU, but with lighter-weight readers.
o Avoids expensive read-side instruction, having overhead similar
to that of Preemptible RCU.
There are of course downsides:
o The grace-period code can send IPIs to CPUs, even when those
CPUs are in the idle loop or in nohz_full userspace. This is
mitigated by later commits.
o It is necessary to scan the full tasklist, much as for Tasks RCU.
o There is a single callback queue guarded by a single lock,
again, much as for Tasks RCU. However, those early use cases
that request multiple grace periods in quick succession are
expected to do so from a single task, which makes the single
lock almost irrelevant. If needed, multiple callback queues
can be provided using any number of schemes.
Perhaps most important, this variant of RCU does not affect the vanilla
flavors, rcu_preempt and rcu_sched. The fact that RCU Tasks Trace
readers can operate from idle, offline, and exception entry/exit in no
way enables rcu_preempt and rcu_sched readers to do so.
The memory ordering was outlined here:
https://lore.kernel.org/lkml/20200319034030.GX3199@paulmck-ThinkPad-P72/
This effort benefited greatly from off-list discussions of BPF
requirements with Alexei Starovoitov and Andrii Nakryiko. At least
some of the on-list discussions are captured in the Link: tags below.
In addition, KCSAN was quite helpful in finding some early bugs.
Link: https://lore.kernel.org/lkml/20200219150744.428764577@infradead.org/
Link: https://lore.kernel.org/lkml/87mu8p797b.fsf@nanos.tec.linutronix.de/
Link: https://lore.kernel.org/lkml/20200225221305.605144982@linutronix.de/
Cc: Alexei Starovoitov <alexei.starovoitov@gmail.com>
Cc: Andrii Nakryiko <andriin@fb.com>
[ paulmck: Apply feedback from Steve Rostedt and Joel Fernandes. ]
[ paulmck: Decrement trc_n_readers_need_end upon IPI failure. ]
[ paulmck: Fix locking issue reported by rcutorture. ]
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
2020-03-09 19:56:53 -07:00
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/* Remove a task from the holdout list, if it is in fact present. */
|
|
|
|
|
static void trc_del_holdout(struct task_struct *t)
|
|
|
|
|
{
|
|
|
|
|
if (!list_empty(&t->trc_holdout_list)) {
|
|
|
|
|
list_del_init(&t->trc_holdout_list);
|
|
|
|
|
put_task_struct(t);
|
2022-06-01 21:26:57 -07:00
|
|
|
n_trc_holdouts--;
|
rcu-tasks: Add an RCU Tasks Trace to simplify protection of tracing hooks
Because RCU does not watch exception early-entry/late-exit, idle-loop,
or CPU-hotplug execution, protection of tracing and BPF operations is
needlessly complicated. This commit therefore adds a variant of
Tasks RCU that:
o Has explicit read-side markers to allow finite grace periods in
the face of in-kernel loops for PREEMPT=n builds. These markers
are rcu_read_lock_trace() and rcu_read_unlock_trace().
o Protects code in the idle loop, exception entry/exit, and
CPU-hotplug code paths. In this respect, RCU-tasks trace is
similar to SRCU, but with lighter-weight readers.
o Avoids expensive read-side instruction, having overhead similar
to that of Preemptible RCU.
There are of course downsides:
o The grace-period code can send IPIs to CPUs, even when those
CPUs are in the idle loop or in nohz_full userspace. This is
mitigated by later commits.
o It is necessary to scan the full tasklist, much as for Tasks RCU.
o There is a single callback queue guarded by a single lock,
again, much as for Tasks RCU. However, those early use cases
that request multiple grace periods in quick succession are
expected to do so from a single task, which makes the single
lock almost irrelevant. If needed, multiple callback queues
can be provided using any number of schemes.
Perhaps most important, this variant of RCU does not affect the vanilla
flavors, rcu_preempt and rcu_sched. The fact that RCU Tasks Trace
readers can operate from idle, offline, and exception entry/exit in no
way enables rcu_preempt and rcu_sched readers to do so.
The memory ordering was outlined here:
https://lore.kernel.org/lkml/20200319034030.GX3199@paulmck-ThinkPad-P72/
This effort benefited greatly from off-list discussions of BPF
requirements with Alexei Starovoitov and Andrii Nakryiko. At least
some of the on-list discussions are captured in the Link: tags below.
In addition, KCSAN was quite helpful in finding some early bugs.
Link: https://lore.kernel.org/lkml/20200219150744.428764577@infradead.org/
Link: https://lore.kernel.org/lkml/87mu8p797b.fsf@nanos.tec.linutronix.de/
Link: https://lore.kernel.org/lkml/20200225221305.605144982@linutronix.de/
Cc: Alexei Starovoitov <alexei.starovoitov@gmail.com>
Cc: Andrii Nakryiko <andriin@fb.com>
[ paulmck: Apply feedback from Steve Rostedt and Joel Fernandes. ]
[ paulmck: Decrement trc_n_readers_need_end upon IPI failure. ]
[ paulmck: Fix locking issue reported by rcutorture. ]
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
2020-03-09 19:56:53 -07:00
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/* IPI handler to check task state. */
|
|
|
|
|
static void trc_read_check_handler(void *t_in)
|
|
|
|
|
{
|
2022-05-26 16:12:51 -07:00
|
|
|
int nesting;
|
rcu-tasks: Add an RCU Tasks Trace to simplify protection of tracing hooks
Because RCU does not watch exception early-entry/late-exit, idle-loop,
or CPU-hotplug execution, protection of tracing and BPF operations is
needlessly complicated. This commit therefore adds a variant of
Tasks RCU that:
o Has explicit read-side markers to allow finite grace periods in
the face of in-kernel loops for PREEMPT=n builds. These markers
are rcu_read_lock_trace() and rcu_read_unlock_trace().
o Protects code in the idle loop, exception entry/exit, and
CPU-hotplug code paths. In this respect, RCU-tasks trace is
similar to SRCU, but with lighter-weight readers.
o Avoids expensive read-side instruction, having overhead similar
to that of Preemptible RCU.
There are of course downsides:
o The grace-period code can send IPIs to CPUs, even when those
CPUs are in the idle loop or in nohz_full userspace. This is
mitigated by later commits.
o It is necessary to scan the full tasklist, much as for Tasks RCU.
o There is a single callback queue guarded by a single lock,
again, much as for Tasks RCU. However, those early use cases
that request multiple grace periods in quick succession are
expected to do so from a single task, which makes the single
lock almost irrelevant. If needed, multiple callback queues
can be provided using any number of schemes.
Perhaps most important, this variant of RCU does not affect the vanilla
flavors, rcu_preempt and rcu_sched. The fact that RCU Tasks Trace
readers can operate from idle, offline, and exception entry/exit in no
way enables rcu_preempt and rcu_sched readers to do so.
The memory ordering was outlined here:
https://lore.kernel.org/lkml/20200319034030.GX3199@paulmck-ThinkPad-P72/
This effort benefited greatly from off-list discussions of BPF
requirements with Alexei Starovoitov and Andrii Nakryiko. At least
some of the on-list discussions are captured in the Link: tags below.
In addition, KCSAN was quite helpful in finding some early bugs.
Link: https://lore.kernel.org/lkml/20200219150744.428764577@infradead.org/
Link: https://lore.kernel.org/lkml/87mu8p797b.fsf@nanos.tec.linutronix.de/
Link: https://lore.kernel.org/lkml/20200225221305.605144982@linutronix.de/
Cc: Alexei Starovoitov <alexei.starovoitov@gmail.com>
Cc: Andrii Nakryiko <andriin@fb.com>
[ paulmck: Apply feedback from Steve Rostedt and Joel Fernandes. ]
[ paulmck: Decrement trc_n_readers_need_end upon IPI failure. ]
[ paulmck: Fix locking issue reported by rcutorture. ]
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
2020-03-09 19:56:53 -07:00
|
|
|
struct task_struct *t = current;
|
|
|
|
|
struct task_struct *texp = t_in;
|
|
|
|
|
|
|
|
|
|
// If the task is no longer running on this CPU, leave.
|
2022-05-23 20:50:11 -07:00
|
|
|
if (unlikely(texp != t))
|
rcu-tasks: Add an RCU Tasks Trace to simplify protection of tracing hooks
Because RCU does not watch exception early-entry/late-exit, idle-loop,
or CPU-hotplug execution, protection of tracing and BPF operations is
needlessly complicated. This commit therefore adds a variant of
Tasks RCU that:
o Has explicit read-side markers to allow finite grace periods in
the face of in-kernel loops for PREEMPT=n builds. These markers
are rcu_read_lock_trace() and rcu_read_unlock_trace().
o Protects code in the idle loop, exception entry/exit, and
CPU-hotplug code paths. In this respect, RCU-tasks trace is
similar to SRCU, but with lighter-weight readers.
o Avoids expensive read-side instruction, having overhead similar
to that of Preemptible RCU.
There are of course downsides:
o The grace-period code can send IPIs to CPUs, even when those
CPUs are in the idle loop or in nohz_full userspace. This is
mitigated by later commits.
o It is necessary to scan the full tasklist, much as for Tasks RCU.
o There is a single callback queue guarded by a single lock,
again, much as for Tasks RCU. However, those early use cases
that request multiple grace periods in quick succession are
expected to do so from a single task, which makes the single
lock almost irrelevant. If needed, multiple callback queues
can be provided using any number of schemes.
Perhaps most important, this variant of RCU does not affect the vanilla
flavors, rcu_preempt and rcu_sched. The fact that RCU Tasks Trace
readers can operate from idle, offline, and exception entry/exit in no
way enables rcu_preempt and rcu_sched readers to do so.
The memory ordering was outlined here:
https://lore.kernel.org/lkml/20200319034030.GX3199@paulmck-ThinkPad-P72/
This effort benefited greatly from off-list discussions of BPF
requirements with Alexei Starovoitov and Andrii Nakryiko. At least
some of the on-list discussions are captured in the Link: tags below.
In addition, KCSAN was quite helpful in finding some early bugs.
Link: https://lore.kernel.org/lkml/20200219150744.428764577@infradead.org/
Link: https://lore.kernel.org/lkml/87mu8p797b.fsf@nanos.tec.linutronix.de/
Link: https://lore.kernel.org/lkml/20200225221305.605144982@linutronix.de/
Cc: Alexei Starovoitov <alexei.starovoitov@gmail.com>
Cc: Andrii Nakryiko <andriin@fb.com>
[ paulmck: Apply feedback from Steve Rostedt and Joel Fernandes. ]
[ paulmck: Decrement trc_n_readers_need_end upon IPI failure. ]
[ paulmck: Fix locking issue reported by rcutorture. ]
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
2020-03-09 19:56:53 -07:00
|
|
|
goto reset_ipi; // Already on holdout list, so will check later.
|
|
|
|
|
|
|
|
|
|
// If the task is not in a read-side critical section, and
|
|
|
|
|
// if this is the last reader, awaken the grace-period kthread.
|
2022-05-26 16:12:51 -07:00
|
|
|
nesting = READ_ONCE(t->trc_reader_nesting);
|
|
|
|
|
if (likely(!nesting)) {
|
2022-05-23 20:50:11 -07:00
|
|
|
rcu_trc_cmpxchg_need_qs(t, 0, TRC_NEED_QS_CHECKED);
|
rcu-tasks: Add an RCU Tasks Trace to simplify protection of tracing hooks
Because RCU does not watch exception early-entry/late-exit, idle-loop,
or CPU-hotplug execution, protection of tracing and BPF operations is
needlessly complicated. This commit therefore adds a variant of
Tasks RCU that:
o Has explicit read-side markers to allow finite grace periods in
the face of in-kernel loops for PREEMPT=n builds. These markers
are rcu_read_lock_trace() and rcu_read_unlock_trace().
o Protects code in the idle loop, exception entry/exit, and
CPU-hotplug code paths. In this respect, RCU-tasks trace is
similar to SRCU, but with lighter-weight readers.
o Avoids expensive read-side instruction, having overhead similar
to that of Preemptible RCU.
There are of course downsides:
o The grace-period code can send IPIs to CPUs, even when those
CPUs are in the idle loop or in nohz_full userspace. This is
mitigated by later commits.
o It is necessary to scan the full tasklist, much as for Tasks RCU.
o There is a single callback queue guarded by a single lock,
again, much as for Tasks RCU. However, those early use cases
that request multiple grace periods in quick succession are
expected to do so from a single task, which makes the single
lock almost irrelevant. If needed, multiple callback queues
can be provided using any number of schemes.
Perhaps most important, this variant of RCU does not affect the vanilla
flavors, rcu_preempt and rcu_sched. The fact that RCU Tasks Trace
readers can operate from idle, offline, and exception entry/exit in no
way enables rcu_preempt and rcu_sched readers to do so.
The memory ordering was outlined here:
https://lore.kernel.org/lkml/20200319034030.GX3199@paulmck-ThinkPad-P72/
This effort benefited greatly from off-list discussions of BPF
requirements with Alexei Starovoitov and Andrii Nakryiko. At least
some of the on-list discussions are captured in the Link: tags below.
In addition, KCSAN was quite helpful in finding some early bugs.
Link: https://lore.kernel.org/lkml/20200219150744.428764577@infradead.org/
Link: https://lore.kernel.org/lkml/87mu8p797b.fsf@nanos.tec.linutronix.de/
Link: https://lore.kernel.org/lkml/20200225221305.605144982@linutronix.de/
Cc: Alexei Starovoitov <alexei.starovoitov@gmail.com>
Cc: Andrii Nakryiko <andriin@fb.com>
[ paulmck: Apply feedback from Steve Rostedt and Joel Fernandes. ]
[ paulmck: Decrement trc_n_readers_need_end upon IPI failure. ]
[ paulmck: Fix locking issue reported by rcutorture. ]
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
2020-03-09 19:56:53 -07:00
|
|
|
goto reset_ipi;
|
|
|
|
|
}
|
rcu-tasks: Fix grace-period/unlock race in RCU Tasks Trace
The more intense grace-period processing resulting from the 50x RCU
Tasks Trace grace-period speedups exposed the following race condition:
o Task A running on CPU 0 executes rcu_read_lock_trace(),
entering a read-side critical section.
o When Task A eventually invokes rcu_read_unlock_trace()
to exit its read-side critical section, this function
notes that the ->trc_reader_special.s flag is zero and
and therefore invoke wil set ->trc_reader_nesting to zero
using WRITE_ONCE(). But before that happens...
o The RCU Tasks Trace grace-period kthread running on some other
CPU interrogates Task A, but this fails because this task is
currently running. This kthread therefore sends an IPI to CPU 0.
o CPU 0 receives the IPI, and thus invokes trc_read_check_handler().
Because Task A has not yet cleared its ->trc_reader_nesting
counter, this function sees that Task A is still within its
read-side critical section. This function therefore sets the
->trc_reader_nesting.b.need_qs flag, AKA the .need_qs flag.
Except that Task A has already checked the .need_qs flag, which
is part of the ->trc_reader_special.s flag. The .need_qs flag
therefore remains set until Task A's next rcu_read_unlock_trace().
o Task A now invokes synchronize_rcu_tasks_trace(), which cannot
start a new grace period until the current grace period completes.
And thus cannot return until after that time.
But Task A's .need_qs flag is still set, which prevents the current
grace period from completing. And because Task A is blocked, it
will never execute rcu_read_unlock_trace() until its call to
synchronize_rcu_tasks_trace() returns.
We are therefore deadlocked.
This race is improbable, but 80 hours of rcutorture made it happen twice.
The race was possible before the grace-period speedup, but roughly 50x
less probable. Several thousand hours of rcutorture would have been
necessary to have a reasonable chance of making this happen before this
50x speedup.
This commit therefore eliminates this deadlock by setting
->trc_reader_nesting to a large negative number before checking the
.need_qs and zeroing (or decrementing with respect to its initial
value) ->trc_reader_nesting. For its part, the IPI handler's
trc_read_check_handler() function adds a check for negative values,
deferring evaluation of the task in this case. Taken together, these
changes avoid this deadlock scenario.
Fixes: 276c410448db ("rcu-tasks: Split ->trc_reader_need_end")
Cc: Alexei Starovoitov <alexei.starovoitov@gmail.com>
Cc: Daniel Borkmann <daniel@iogearbox.net>
Cc: Jiri Olsa <jolsa@redhat.com>
Cc: <bpf@vger.kernel.org>
Cc: <stable@vger.kernel.org> # 5.7.x
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
2020-09-14 15:44:37 -07:00
|
|
|
// If we are racing with an rcu_read_unlock_trace(), try again later.
|
2022-05-26 16:12:51 -07:00
|
|
|
if (unlikely(nesting < 0))
|
rcu-tasks: Fix grace-period/unlock race in RCU Tasks Trace
The more intense grace-period processing resulting from the 50x RCU
Tasks Trace grace-period speedups exposed the following race condition:
o Task A running on CPU 0 executes rcu_read_lock_trace(),
entering a read-side critical section.
o When Task A eventually invokes rcu_read_unlock_trace()
to exit its read-side critical section, this function
notes that the ->trc_reader_special.s flag is zero and
and therefore invoke wil set ->trc_reader_nesting to zero
using WRITE_ONCE(). But before that happens...
o The RCU Tasks Trace grace-period kthread running on some other
CPU interrogates Task A, but this fails because this task is
currently running. This kthread therefore sends an IPI to CPU 0.
o CPU 0 receives the IPI, and thus invokes trc_read_check_handler().
Because Task A has not yet cleared its ->trc_reader_nesting
counter, this function sees that Task A is still within its
read-side critical section. This function therefore sets the
->trc_reader_nesting.b.need_qs flag, AKA the .need_qs flag.
Except that Task A has already checked the .need_qs flag, which
is part of the ->trc_reader_special.s flag. The .need_qs flag
therefore remains set until Task A's next rcu_read_unlock_trace().
o Task A now invokes synchronize_rcu_tasks_trace(), which cannot
start a new grace period until the current grace period completes.
And thus cannot return until after that time.
But Task A's .need_qs flag is still set, which prevents the current
grace period from completing. And because Task A is blocked, it
will never execute rcu_read_unlock_trace() until its call to
synchronize_rcu_tasks_trace() returns.
We are therefore deadlocked.
This race is improbable, but 80 hours of rcutorture made it happen twice.
The race was possible before the grace-period speedup, but roughly 50x
less probable. Several thousand hours of rcutorture would have been
necessary to have a reasonable chance of making this happen before this
50x speedup.
This commit therefore eliminates this deadlock by setting
->trc_reader_nesting to a large negative number before checking the
.need_qs and zeroing (or decrementing with respect to its initial
value) ->trc_reader_nesting. For its part, the IPI handler's
trc_read_check_handler() function adds a check for negative values,
deferring evaluation of the task in this case. Taken together, these
changes avoid this deadlock scenario.
Fixes: 276c410448db ("rcu-tasks: Split ->trc_reader_need_end")
Cc: Alexei Starovoitov <alexei.starovoitov@gmail.com>
Cc: Daniel Borkmann <daniel@iogearbox.net>
Cc: Jiri Olsa <jolsa@redhat.com>
Cc: <bpf@vger.kernel.org>
Cc: <stable@vger.kernel.org> # 5.7.x
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
2020-09-14 15:44:37 -07:00
|
|
|
goto reset_ipi;
|
rcu-tasks: Add an RCU Tasks Trace to simplify protection of tracing hooks
Because RCU does not watch exception early-entry/late-exit, idle-loop,
or CPU-hotplug execution, protection of tracing and BPF operations is
needlessly complicated. This commit therefore adds a variant of
Tasks RCU that:
o Has explicit read-side markers to allow finite grace periods in
the face of in-kernel loops for PREEMPT=n builds. These markers
are rcu_read_lock_trace() and rcu_read_unlock_trace().
o Protects code in the idle loop, exception entry/exit, and
CPU-hotplug code paths. In this respect, RCU-tasks trace is
similar to SRCU, but with lighter-weight readers.
o Avoids expensive read-side instruction, having overhead similar
to that of Preemptible RCU.
There are of course downsides:
o The grace-period code can send IPIs to CPUs, even when those
CPUs are in the idle loop or in nohz_full userspace. This is
mitigated by later commits.
o It is necessary to scan the full tasklist, much as for Tasks RCU.
o There is a single callback queue guarded by a single lock,
again, much as for Tasks RCU. However, those early use cases
that request multiple grace periods in quick succession are
expected to do so from a single task, which makes the single
lock almost irrelevant. If needed, multiple callback queues
can be provided using any number of schemes.
Perhaps most important, this variant of RCU does not affect the vanilla
flavors, rcu_preempt and rcu_sched. The fact that RCU Tasks Trace
readers can operate from idle, offline, and exception entry/exit in no
way enables rcu_preempt and rcu_sched readers to do so.
The memory ordering was outlined here:
https://lore.kernel.org/lkml/20200319034030.GX3199@paulmck-ThinkPad-P72/
This effort benefited greatly from off-list discussions of BPF
requirements with Alexei Starovoitov and Andrii Nakryiko. At least
some of the on-list discussions are captured in the Link: tags below.
In addition, KCSAN was quite helpful in finding some early bugs.
Link: https://lore.kernel.org/lkml/20200219150744.428764577@infradead.org/
Link: https://lore.kernel.org/lkml/87mu8p797b.fsf@nanos.tec.linutronix.de/
Link: https://lore.kernel.org/lkml/20200225221305.605144982@linutronix.de/
Cc: Alexei Starovoitov <alexei.starovoitov@gmail.com>
Cc: Andrii Nakryiko <andriin@fb.com>
[ paulmck: Apply feedback from Steve Rostedt and Joel Fernandes. ]
[ paulmck: Decrement trc_n_readers_need_end upon IPI failure. ]
[ paulmck: Fix locking issue reported by rcutorture. ]
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
2020-03-09 19:56:53 -07:00
|
|
|
|
2022-06-06 21:30:38 -07:00
|
|
|
// Get here if the task is in a read-side critical section.
|
|
|
|
|
// Set its state so that it will update state for the grace-period
|
|
|
|
|
// kthread upon exit from that critical section.
|
2022-05-25 13:37:36 -07:00
|
|
|
rcu_trc_cmpxchg_need_qs(t, 0, TRC_NEED_QS | TRC_NEED_QS_CHECKED);
|
rcu-tasks: Add an RCU Tasks Trace to simplify protection of tracing hooks
Because RCU does not watch exception early-entry/late-exit, idle-loop,
or CPU-hotplug execution, protection of tracing and BPF operations is
needlessly complicated. This commit therefore adds a variant of
Tasks RCU that:
o Has explicit read-side markers to allow finite grace periods in
the face of in-kernel loops for PREEMPT=n builds. These markers
are rcu_read_lock_trace() and rcu_read_unlock_trace().
o Protects code in the idle loop, exception entry/exit, and
CPU-hotplug code paths. In this respect, RCU-tasks trace is
similar to SRCU, but with lighter-weight readers.
o Avoids expensive read-side instruction, having overhead similar
to that of Preemptible RCU.
There are of course downsides:
o The grace-period code can send IPIs to CPUs, even when those
CPUs are in the idle loop or in nohz_full userspace. This is
mitigated by later commits.
o It is necessary to scan the full tasklist, much as for Tasks RCU.
o There is a single callback queue guarded by a single lock,
again, much as for Tasks RCU. However, those early use cases
that request multiple grace periods in quick succession are
expected to do so from a single task, which makes the single
lock almost irrelevant. If needed, multiple callback queues
can be provided using any number of schemes.
Perhaps most important, this variant of RCU does not affect the vanilla
flavors, rcu_preempt and rcu_sched. The fact that RCU Tasks Trace
readers can operate from idle, offline, and exception entry/exit in no
way enables rcu_preempt and rcu_sched readers to do so.
The memory ordering was outlined here:
https://lore.kernel.org/lkml/20200319034030.GX3199@paulmck-ThinkPad-P72/
This effort benefited greatly from off-list discussions of BPF
requirements with Alexei Starovoitov and Andrii Nakryiko. At least
some of the on-list discussions are captured in the Link: tags below.
In addition, KCSAN was quite helpful in finding some early bugs.
Link: https://lore.kernel.org/lkml/20200219150744.428764577@infradead.org/
Link: https://lore.kernel.org/lkml/87mu8p797b.fsf@nanos.tec.linutronix.de/
Link: https://lore.kernel.org/lkml/20200225221305.605144982@linutronix.de/
Cc: Alexei Starovoitov <alexei.starovoitov@gmail.com>
Cc: Andrii Nakryiko <andriin@fb.com>
[ paulmck: Apply feedback from Steve Rostedt and Joel Fernandes. ]
[ paulmck: Decrement trc_n_readers_need_end upon IPI failure. ]
[ paulmck: Fix locking issue reported by rcutorture. ]
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
2020-03-09 19:56:53 -07:00
|
|
|
|
|
|
|
|
reset_ipi:
|
|
|
|
|
// Allow future IPIs to be sent on CPU and for task.
|
|
|
|
|
// Also order this IPI handler against any later manipulations of
|
|
|
|
|
// the intended task.
|
2021-06-30 07:08:02 -07:00
|
|
|
smp_store_release(per_cpu_ptr(&trc_ipi_to_cpu, smp_processor_id()), false); // ^^^
|
rcu-tasks: Add an RCU Tasks Trace to simplify protection of tracing hooks
Because RCU does not watch exception early-entry/late-exit, idle-loop,
or CPU-hotplug execution, protection of tracing and BPF operations is
needlessly complicated. This commit therefore adds a variant of
Tasks RCU that:
o Has explicit read-side markers to allow finite grace periods in
the face of in-kernel loops for PREEMPT=n builds. These markers
are rcu_read_lock_trace() and rcu_read_unlock_trace().
o Protects code in the idle loop, exception entry/exit, and
CPU-hotplug code paths. In this respect, RCU-tasks trace is
similar to SRCU, but with lighter-weight readers.
o Avoids expensive read-side instruction, having overhead similar
to that of Preemptible RCU.
There are of course downsides:
o The grace-period code can send IPIs to CPUs, even when those
CPUs are in the idle loop or in nohz_full userspace. This is
mitigated by later commits.
o It is necessary to scan the full tasklist, much as for Tasks RCU.
o There is a single callback queue guarded by a single lock,
again, much as for Tasks RCU. However, those early use cases
that request multiple grace periods in quick succession are
expected to do so from a single task, which makes the single
lock almost irrelevant. If needed, multiple callback queues
can be provided using any number of schemes.
Perhaps most important, this variant of RCU does not affect the vanilla
flavors, rcu_preempt and rcu_sched. The fact that RCU Tasks Trace
readers can operate from idle, offline, and exception entry/exit in no
way enables rcu_preempt and rcu_sched readers to do so.
The memory ordering was outlined here:
https://lore.kernel.org/lkml/20200319034030.GX3199@paulmck-ThinkPad-P72/
This effort benefited greatly from off-list discussions of BPF
requirements with Alexei Starovoitov and Andrii Nakryiko. At least
some of the on-list discussions are captured in the Link: tags below.
In addition, KCSAN was quite helpful in finding some early bugs.
Link: https://lore.kernel.org/lkml/20200219150744.428764577@infradead.org/
Link: https://lore.kernel.org/lkml/87mu8p797b.fsf@nanos.tec.linutronix.de/
Link: https://lore.kernel.org/lkml/20200225221305.605144982@linutronix.de/
Cc: Alexei Starovoitov <alexei.starovoitov@gmail.com>
Cc: Andrii Nakryiko <andriin@fb.com>
[ paulmck: Apply feedback from Steve Rostedt and Joel Fernandes. ]
[ paulmck: Decrement trc_n_readers_need_end upon IPI failure. ]
[ paulmck: Fix locking issue reported by rcutorture. ]
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
2020-03-09 19:56:53 -07:00
|
|
|
smp_store_release(&texp->trc_ipi_to_cpu, -1); // ^^^
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/* Callback function for scheduler to check locked-down task. */
|
2022-05-23 20:50:11 -07:00
|
|
|
static int trc_inspect_reader(struct task_struct *t, void *bhp_in)
|
rcu-tasks: Add an RCU Tasks Trace to simplify protection of tracing hooks
Because RCU does not watch exception early-entry/late-exit, idle-loop,
or CPU-hotplug execution, protection of tracing and BPF operations is
needlessly complicated. This commit therefore adds a variant of
Tasks RCU that:
o Has explicit read-side markers to allow finite grace periods in
the face of in-kernel loops for PREEMPT=n builds. These markers
are rcu_read_lock_trace() and rcu_read_unlock_trace().
o Protects code in the idle loop, exception entry/exit, and
CPU-hotplug code paths. In this respect, RCU-tasks trace is
similar to SRCU, but with lighter-weight readers.
o Avoids expensive read-side instruction, having overhead similar
to that of Preemptible RCU.
There are of course downsides:
o The grace-period code can send IPIs to CPUs, even when those
CPUs are in the idle loop or in nohz_full userspace. This is
mitigated by later commits.
o It is necessary to scan the full tasklist, much as for Tasks RCU.
o There is a single callback queue guarded by a single lock,
again, much as for Tasks RCU. However, those early use cases
that request multiple grace periods in quick succession are
expected to do so from a single task, which makes the single
lock almost irrelevant. If needed, multiple callback queues
can be provided using any number of schemes.
Perhaps most important, this variant of RCU does not affect the vanilla
flavors, rcu_preempt and rcu_sched. The fact that RCU Tasks Trace
readers can operate from idle, offline, and exception entry/exit in no
way enables rcu_preempt and rcu_sched readers to do so.
The memory ordering was outlined here:
https://lore.kernel.org/lkml/20200319034030.GX3199@paulmck-ThinkPad-P72/
This effort benefited greatly from off-list discussions of BPF
requirements with Alexei Starovoitov and Andrii Nakryiko. At least
some of the on-list discussions are captured in the Link: tags below.
In addition, KCSAN was quite helpful in finding some early bugs.
Link: https://lore.kernel.org/lkml/20200219150744.428764577@infradead.org/
Link: https://lore.kernel.org/lkml/87mu8p797b.fsf@nanos.tec.linutronix.de/
Link: https://lore.kernel.org/lkml/20200225221305.605144982@linutronix.de/
Cc: Alexei Starovoitov <alexei.starovoitov@gmail.com>
Cc: Andrii Nakryiko <andriin@fb.com>
[ paulmck: Apply feedback from Steve Rostedt and Joel Fernandes. ]
[ paulmck: Decrement trc_n_readers_need_end upon IPI failure. ]
[ paulmck: Fix locking issue reported by rcutorture. ]
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
2020-03-09 19:56:53 -07:00
|
|
|
{
|
2022-05-23 20:50:11 -07:00
|
|
|
struct list_head *bhp = bhp_in;
|
rcu-tasks: Avoid IPIing userspace/idle tasks if kernel is so built
Systems running CPU-bound real-time task do not want IPIs sent to CPUs
executing nohz_full userspace tasks. Battery-powered systems don't
want IPIs sent to idle CPUs in low-power mode. Unfortunately, RCU tasks
trace can and will send such IPIs in some cases.
Both of these situations occur only when the target CPU is in RCU
dyntick-idle mode, in other words, when RCU is not watching the
target CPU. This suggests that CPUs in dyntick-idle mode should use
memory barriers in outermost invocations of rcu_read_lock_trace()
and rcu_read_unlock_trace(), which would allow the RCU tasks trace
grace period to directly read out the target CPU's read-side state.
One challenge is that RCU tasks trace is not targeting a specific
CPU, but rather a task. And that task could switch from one CPU to
another at any time.
This commit therefore uses try_invoke_on_locked_down_task()
and checks for task_curr() in trc_inspect_reader_notrunning().
When this condition holds, the target task is running and cannot move.
If CONFIG_TASKS_TRACE_RCU_READ_MB=y, the new rcu_dynticks_zero_in_eqs()
function can be used to check if the specified integer (in this case,
t->trc_reader_nesting) is zero while the target CPU remains in that same
dyntick-idle sojourn. If so, the target task is in a quiescent state.
If not, trc_read_check_handler() must indicate failure so that the
grace-period kthread can take appropriate action or retry after an
appropriate delay, as the case may be.
With this change, given CONFIG_TASKS_TRACE_RCU_READ_MB=y, if a given
CPU remains idle or a given task continues executing in nohz_full mode,
the RCU tasks trace grace-period kthread will detect this without the
need to send an IPI.
Suggested-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
2020-03-19 15:33:12 -07:00
|
|
|
int cpu = task_cpu(t);
|
2021-07-28 11:32:28 -07:00
|
|
|
int nesting;
|
2020-03-22 11:24:58 -07:00
|
|
|
bool ofl = cpu_is_offline(cpu);
|
rcu-tasks: Avoid IPIing userspace/idle tasks if kernel is so built
Systems running CPU-bound real-time task do not want IPIs sent to CPUs
executing nohz_full userspace tasks. Battery-powered systems don't
want IPIs sent to idle CPUs in low-power mode. Unfortunately, RCU tasks
trace can and will send such IPIs in some cases.
Both of these situations occur only when the target CPU is in RCU
dyntick-idle mode, in other words, when RCU is not watching the
target CPU. This suggests that CPUs in dyntick-idle mode should use
memory barriers in outermost invocations of rcu_read_lock_trace()
and rcu_read_unlock_trace(), which would allow the RCU tasks trace
grace period to directly read out the target CPU's read-side state.
One challenge is that RCU tasks trace is not targeting a specific
CPU, but rather a task. And that task could switch from one CPU to
another at any time.
This commit therefore uses try_invoke_on_locked_down_task()
and checks for task_curr() in trc_inspect_reader_notrunning().
When this condition holds, the target task is running and cannot move.
If CONFIG_TASKS_TRACE_RCU_READ_MB=y, the new rcu_dynticks_zero_in_eqs()
function can be used to check if the specified integer (in this case,
t->trc_reader_nesting) is zero while the target CPU remains in that same
dyntick-idle sojourn. If so, the target task is in a quiescent state.
If not, trc_read_check_handler() must indicate failure so that the
grace-period kthread can take appropriate action or retry after an
appropriate delay, as the case may be.
With this change, given CONFIG_TASKS_TRACE_RCU_READ_MB=y, if a given
CPU remains idle or a given task continues executing in nohz_full mode,
the RCU tasks trace grace-period kthread will detect this without the
need to send an IPI.
Suggested-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
2020-03-19 15:33:12 -07:00
|
|
|
|
2022-05-24 18:02:40 -07:00
|
|
|
if (task_curr(t) && !ofl) {
|
rcu-tasks: Avoid IPIing userspace/idle tasks if kernel is so built
Systems running CPU-bound real-time task do not want IPIs sent to CPUs
executing nohz_full userspace tasks. Battery-powered systems don't
want IPIs sent to idle CPUs in low-power mode. Unfortunately, RCU tasks
trace can and will send such IPIs in some cases.
Both of these situations occur only when the target CPU is in RCU
dyntick-idle mode, in other words, when RCU is not watching the
target CPU. This suggests that CPUs in dyntick-idle mode should use
memory barriers in outermost invocations of rcu_read_lock_trace()
and rcu_read_unlock_trace(), which would allow the RCU tasks trace
grace period to directly read out the target CPU's read-side state.
One challenge is that RCU tasks trace is not targeting a specific
CPU, but rather a task. And that task could switch from one CPU to
another at any time.
This commit therefore uses try_invoke_on_locked_down_task()
and checks for task_curr() in trc_inspect_reader_notrunning().
When this condition holds, the target task is running and cannot move.
If CONFIG_TASKS_TRACE_RCU_READ_MB=y, the new rcu_dynticks_zero_in_eqs()
function can be used to check if the specified integer (in this case,
t->trc_reader_nesting) is zero while the target CPU remains in that same
dyntick-idle sojourn. If so, the target task is in a quiescent state.
If not, trc_read_check_handler() must indicate failure so that the
grace-period kthread can take appropriate action or retry after an
appropriate delay, as the case may be.
With this change, given CONFIG_TASKS_TRACE_RCU_READ_MB=y, if a given
CPU remains idle or a given task continues executing in nohz_full mode,
the RCU tasks trace grace-period kthread will detect this without the
need to send an IPI.
Suggested-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
2020-03-19 15:33:12 -07:00
|
|
|
// If no chance of heavyweight readers, do it the hard way.
|
2022-05-24 18:02:40 -07:00
|
|
|
if (!IS_ENABLED(CONFIG_TASKS_TRACE_RCU_READ_MB))
|
2021-09-21 12:54:32 -07:00
|
|
|
return -EINVAL;
|
rcu-tasks: Avoid IPIing userspace/idle tasks if kernel is so built
Systems running CPU-bound real-time task do not want IPIs sent to CPUs
executing nohz_full userspace tasks. Battery-powered systems don't
want IPIs sent to idle CPUs in low-power mode. Unfortunately, RCU tasks
trace can and will send such IPIs in some cases.
Both of these situations occur only when the target CPU is in RCU
dyntick-idle mode, in other words, when RCU is not watching the
target CPU. This suggests that CPUs in dyntick-idle mode should use
memory barriers in outermost invocations of rcu_read_lock_trace()
and rcu_read_unlock_trace(), which would allow the RCU tasks trace
grace period to directly read out the target CPU's read-side state.
One challenge is that RCU tasks trace is not targeting a specific
CPU, but rather a task. And that task could switch from one CPU to
another at any time.
This commit therefore uses try_invoke_on_locked_down_task()
and checks for task_curr() in trc_inspect_reader_notrunning().
When this condition holds, the target task is running and cannot move.
If CONFIG_TASKS_TRACE_RCU_READ_MB=y, the new rcu_dynticks_zero_in_eqs()
function can be used to check if the specified integer (in this case,
t->trc_reader_nesting) is zero while the target CPU remains in that same
dyntick-idle sojourn. If so, the target task is in a quiescent state.
If not, trc_read_check_handler() must indicate failure so that the
grace-period kthread can take appropriate action or retry after an
appropriate delay, as the case may be.
With this change, given CONFIG_TASKS_TRACE_RCU_READ_MB=y, if a given
CPU remains idle or a given task continues executing in nohz_full mode,
the RCU tasks trace grace-period kthread will detect this without the
need to send an IPI.
Suggested-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
2020-03-19 15:33:12 -07:00
|
|
|
|
|
|
|
|
// If heavyweight readers are enabled on the remote task,
|
|
|
|
|
// we can inspect its state despite its currently running.
|
|
|
|
|
// However, we cannot safely change its state.
|
2020-03-22 13:34:34 -07:00
|
|
|
n_heavy_reader_attempts++;
|
2022-05-24 18:02:40 -07:00
|
|
|
// Check for "running" idle tasks on offline CPUs.
|
2024-04-29 04:11:34 -07:00
|
|
|
if (!rcu_watching_zero_in_eqs(cpu, &t->trc_reader_nesting))
|
2021-09-21 12:54:32 -07:00
|
|
|
return -EINVAL; // No quiescent state, do it the hard way.
|
2020-03-22 13:34:34 -07:00
|
|
|
n_heavy_reader_updates++;
|
2021-07-28 11:32:28 -07:00
|
|
|
nesting = 0;
|
rcu-tasks: Avoid IPIing userspace/idle tasks if kernel is so built
Systems running CPU-bound real-time task do not want IPIs sent to CPUs
executing nohz_full userspace tasks. Battery-powered systems don't
want IPIs sent to idle CPUs in low-power mode. Unfortunately, RCU tasks
trace can and will send such IPIs in some cases.
Both of these situations occur only when the target CPU is in RCU
dyntick-idle mode, in other words, when RCU is not watching the
target CPU. This suggests that CPUs in dyntick-idle mode should use
memory barriers in outermost invocations of rcu_read_lock_trace()
and rcu_read_unlock_trace(), which would allow the RCU tasks trace
grace period to directly read out the target CPU's read-side state.
One challenge is that RCU tasks trace is not targeting a specific
CPU, but rather a task. And that task could switch from one CPU to
another at any time.
This commit therefore uses try_invoke_on_locked_down_task()
and checks for task_curr() in trc_inspect_reader_notrunning().
When this condition holds, the target task is running and cannot move.
If CONFIG_TASKS_TRACE_RCU_READ_MB=y, the new rcu_dynticks_zero_in_eqs()
function can be used to check if the specified integer (in this case,
t->trc_reader_nesting) is zero while the target CPU remains in that same
dyntick-idle sojourn. If so, the target task is in a quiescent state.
If not, trc_read_check_handler() must indicate failure so that the
grace-period kthread can take appropriate action or retry after an
appropriate delay, as the case may be.
With this change, given CONFIG_TASKS_TRACE_RCU_READ_MB=y, if a given
CPU remains idle or a given task continues executing in nohz_full mode,
the RCU tasks trace grace-period kthread will detect this without the
need to send an IPI.
Suggested-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
2020-03-19 15:33:12 -07:00
|
|
|
} else {
|
2021-05-24 12:48:18 -07:00
|
|
|
// The task is not running, so C-language access is safe.
|
2021-07-28 11:32:28 -07:00
|
|
|
nesting = t->trc_reader_nesting;
|
2023-10-27 07:40:49 -07:00
|
|
|
WARN_ON_ONCE(ofl && task_curr(t) && (t != idle_task(task_cpu(t))));
|
2022-05-24 18:02:40 -07:00
|
|
|
if (IS_ENABLED(CONFIG_TASKS_TRACE_RCU_READ_MB) && ofl)
|
|
|
|
|
n_heavy_reader_ofl_updates++;
|
rcu-tasks: Avoid IPIing userspace/idle tasks if kernel is so built
Systems running CPU-bound real-time task do not want IPIs sent to CPUs
executing nohz_full userspace tasks. Battery-powered systems don't
want IPIs sent to idle CPUs in low-power mode. Unfortunately, RCU tasks
trace can and will send such IPIs in some cases.
Both of these situations occur only when the target CPU is in RCU
dyntick-idle mode, in other words, when RCU is not watching the
target CPU. This suggests that CPUs in dyntick-idle mode should use
memory barriers in outermost invocations of rcu_read_lock_trace()
and rcu_read_unlock_trace(), which would allow the RCU tasks trace
grace period to directly read out the target CPU's read-side state.
One challenge is that RCU tasks trace is not targeting a specific
CPU, but rather a task. And that task could switch from one CPU to
another at any time.
This commit therefore uses try_invoke_on_locked_down_task()
and checks for task_curr() in trc_inspect_reader_notrunning().
When this condition holds, the target task is running and cannot move.
If CONFIG_TASKS_TRACE_RCU_READ_MB=y, the new rcu_dynticks_zero_in_eqs()
function can be used to check if the specified integer (in this case,
t->trc_reader_nesting) is zero while the target CPU remains in that same
dyntick-idle sojourn. If so, the target task is in a quiescent state.
If not, trc_read_check_handler() must indicate failure so that the
grace-period kthread can take appropriate action or retry after an
appropriate delay, as the case may be.
With this change, given CONFIG_TASKS_TRACE_RCU_READ_MB=y, if a given
CPU remains idle or a given task continues executing in nohz_full mode,
the RCU tasks trace grace-period kthread will detect this without the
need to send an IPI.
Suggested-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
2020-03-19 15:33:12 -07:00
|
|
|
}
|
rcu-tasks: Add an RCU Tasks Trace to simplify protection of tracing hooks
Because RCU does not watch exception early-entry/late-exit, idle-loop,
or CPU-hotplug execution, protection of tracing and BPF operations is
needlessly complicated. This commit therefore adds a variant of
Tasks RCU that:
o Has explicit read-side markers to allow finite grace periods in
the face of in-kernel loops for PREEMPT=n builds. These markers
are rcu_read_lock_trace() and rcu_read_unlock_trace().
o Protects code in the idle loop, exception entry/exit, and
CPU-hotplug code paths. In this respect, RCU-tasks trace is
similar to SRCU, but with lighter-weight readers.
o Avoids expensive read-side instruction, having overhead similar
to that of Preemptible RCU.
There are of course downsides:
o The grace-period code can send IPIs to CPUs, even when those
CPUs are in the idle loop or in nohz_full userspace. This is
mitigated by later commits.
o It is necessary to scan the full tasklist, much as for Tasks RCU.
o There is a single callback queue guarded by a single lock,
again, much as for Tasks RCU. However, those early use cases
that request multiple grace periods in quick succession are
expected to do so from a single task, which makes the single
lock almost irrelevant. If needed, multiple callback queues
can be provided using any number of schemes.
Perhaps most important, this variant of RCU does not affect the vanilla
flavors, rcu_preempt and rcu_sched. The fact that RCU Tasks Trace
readers can operate from idle, offline, and exception entry/exit in no
way enables rcu_preempt and rcu_sched readers to do so.
The memory ordering was outlined here:
https://lore.kernel.org/lkml/20200319034030.GX3199@paulmck-ThinkPad-P72/
This effort benefited greatly from off-list discussions of BPF
requirements with Alexei Starovoitov and Andrii Nakryiko. At least
some of the on-list discussions are captured in the Link: tags below.
In addition, KCSAN was quite helpful in finding some early bugs.
Link: https://lore.kernel.org/lkml/20200219150744.428764577@infradead.org/
Link: https://lore.kernel.org/lkml/87mu8p797b.fsf@nanos.tec.linutronix.de/
Link: https://lore.kernel.org/lkml/20200225221305.605144982@linutronix.de/
Cc: Alexei Starovoitov <alexei.starovoitov@gmail.com>
Cc: Andrii Nakryiko <andriin@fb.com>
[ paulmck: Apply feedback from Steve Rostedt and Joel Fernandes. ]
[ paulmck: Decrement trc_n_readers_need_end upon IPI failure. ]
[ paulmck: Fix locking issue reported by rcutorture. ]
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
2020-03-09 19:56:53 -07:00
|
|
|
|
2021-07-28 11:32:28 -07:00
|
|
|
// If not exiting a read-side critical section, mark as checked
|
|
|
|
|
// so that the grace-period kthread will remove it from the
|
|
|
|
|
// holdout list.
|
2022-05-31 21:38:15 -07:00
|
|
|
if (!nesting) {
|
|
|
|
|
rcu_trc_cmpxchg_need_qs(t, 0, TRC_NEED_QS_CHECKED);
|
|
|
|
|
return 0; // In QS, so done.
|
2022-05-23 20:50:11 -07:00
|
|
|
}
|
2022-05-31 21:38:15 -07:00
|
|
|
if (nesting < 0)
|
2022-06-06 21:30:38 -07:00
|
|
|
return -EINVAL; // Reader transitioning, try again later.
|
rcu-tasks: Avoid IPIing userspace/idle tasks if kernel is so built
Systems running CPU-bound real-time task do not want IPIs sent to CPUs
executing nohz_full userspace tasks. Battery-powered systems don't
want IPIs sent to idle CPUs in low-power mode. Unfortunately, RCU tasks
trace can and will send such IPIs in some cases.
Both of these situations occur only when the target CPU is in RCU
dyntick-idle mode, in other words, when RCU is not watching the
target CPU. This suggests that CPUs in dyntick-idle mode should use
memory barriers in outermost invocations of rcu_read_lock_trace()
and rcu_read_unlock_trace(), which would allow the RCU tasks trace
grace period to directly read out the target CPU's read-side state.
One challenge is that RCU tasks trace is not targeting a specific
CPU, but rather a task. And that task could switch from one CPU to
another at any time.
This commit therefore uses try_invoke_on_locked_down_task()
and checks for task_curr() in trc_inspect_reader_notrunning().
When this condition holds, the target task is running and cannot move.
If CONFIG_TASKS_TRACE_RCU_READ_MB=y, the new rcu_dynticks_zero_in_eqs()
function can be used to check if the specified integer (in this case,
t->trc_reader_nesting) is zero while the target CPU remains in that same
dyntick-idle sojourn. If so, the target task is in a quiescent state.
If not, trc_read_check_handler() must indicate failure so that the
grace-period kthread can take appropriate action or retry after an
appropriate delay, as the case may be.
With this change, given CONFIG_TASKS_TRACE_RCU_READ_MB=y, if a given
CPU remains idle or a given task continues executing in nohz_full mode,
the RCU tasks trace grace-period kthread will detect this without the
need to send an IPI.
Suggested-by: Mathieu Desnoyers <mathieu.desnoyers@efficios.com>
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
2020-03-19 15:33:12 -07:00
|
|
|
|
|
|
|
|
// The task is in a read-side critical section, so set up its
|
2022-05-31 21:38:15 -07:00
|
|
|
// state so that it will update state upon exit from that critical
|
|
|
|
|
// section.
|
2022-05-25 13:37:36 -07:00
|
|
|
if (!rcu_trc_cmpxchg_need_qs(t, 0, TRC_NEED_QS | TRC_NEED_QS_CHECKED))
|
2022-05-23 20:50:11 -07:00
|
|
|
trc_add_holdout(t, bhp);
|
2021-09-21 12:54:32 -07:00
|
|
|
return 0;
|
rcu-tasks: Add an RCU Tasks Trace to simplify protection of tracing hooks
Because RCU does not watch exception early-entry/late-exit, idle-loop,
or CPU-hotplug execution, protection of tracing and BPF operations is
needlessly complicated. This commit therefore adds a variant of
Tasks RCU that:
o Has explicit read-side markers to allow finite grace periods in
the face of in-kernel loops for PREEMPT=n builds. These markers
are rcu_read_lock_trace() and rcu_read_unlock_trace().
o Protects code in the idle loop, exception entry/exit, and
CPU-hotplug code paths. In this respect, RCU-tasks trace is
similar to SRCU, but with lighter-weight readers.
o Avoids expensive read-side instruction, having overhead similar
to that of Preemptible RCU.
There are of course downsides:
o The grace-period code can send IPIs to CPUs, even when those
CPUs are in the idle loop or in nohz_full userspace. This is
mitigated by later commits.
o It is necessary to scan the full tasklist, much as for Tasks RCU.
o There is a single callback queue guarded by a single lock,
again, much as for Tasks RCU. However, those early use cases
that request multiple grace periods in quick succession are
expected to do so from a single task, which makes the single
lock almost irrelevant. If needed, multiple callback queues
can be provided using any number of schemes.
Perhaps most important, this variant of RCU does not affect the vanilla
flavors, rcu_preempt and rcu_sched. The fact that RCU Tasks Trace
readers can operate from idle, offline, and exception entry/exit in no
way enables rcu_preempt and rcu_sched readers to do so.
The memory ordering was outlined here:
https://lore.kernel.org/lkml/20200319034030.GX3199@paulmck-ThinkPad-P72/
This effort benefited greatly from off-list discussions of BPF
requirements with Alexei Starovoitov and Andrii Nakryiko. At least
some of the on-list discussions are captured in the Link: tags below.
In addition, KCSAN was quite helpful in finding some early bugs.
Link: https://lore.kernel.org/lkml/20200219150744.428764577@infradead.org/
Link: https://lore.kernel.org/lkml/87mu8p797b.fsf@nanos.tec.linutronix.de/
Link: https://lore.kernel.org/lkml/20200225221305.605144982@linutronix.de/
Cc: Alexei Starovoitov <alexei.starovoitov@gmail.com>
Cc: Andrii Nakryiko <andriin@fb.com>
[ paulmck: Apply feedback from Steve Rostedt and Joel Fernandes. ]
[ paulmck: Decrement trc_n_readers_need_end upon IPI failure. ]
[ paulmck: Fix locking issue reported by rcutorture. ]
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
2020-03-09 19:56:53 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/* Attempt to extract the state for the specified task. */
|
|
|
|
|
static void trc_wait_for_one_reader(struct task_struct *t,
|
|
|
|
|
struct list_head *bhp)
|
|
|
|
|
{
|
|
|
|
|
int cpu;
|
|
|
|
|
|
|
|
|
|
// If a previous IPI is still in flight, let it complete.
|
|
|
|
|
if (smp_load_acquire(&t->trc_ipi_to_cpu) != -1) // Order IPI
|
|
|
|
|
return;
|
|
|
|
|
|
|
|
|
|
// The current task had better be in a quiescent state.
|
|
|
|
|
if (t == current) {
|
2022-05-23 20:50:11 -07:00
|
|
|
rcu_trc_cmpxchg_need_qs(t, 0, TRC_NEED_QS_CHECKED);
|
2021-05-24 12:48:18 -07:00
|
|
|
WARN_ON_ONCE(READ_ONCE(t->trc_reader_nesting));
|
rcu-tasks: Add an RCU Tasks Trace to simplify protection of tracing hooks
Because RCU does not watch exception early-entry/late-exit, idle-loop,
or CPU-hotplug execution, protection of tracing and BPF operations is
needlessly complicated. This commit therefore adds a variant of
Tasks RCU that:
o Has explicit read-side markers to allow finite grace periods in
the face of in-kernel loops for PREEMPT=n builds. These markers
are rcu_read_lock_trace() and rcu_read_unlock_trace().
o Protects code in the idle loop, exception entry/exit, and
CPU-hotplug code paths. In this respect, RCU-tasks trace is
similar to SRCU, but with lighter-weight readers.
o Avoids expensive read-side instruction, having overhead similar
to that of Preemptible RCU.
There are of course downsides:
o The grace-period code can send IPIs to CPUs, even when those
CPUs are in the idle loop or in nohz_full userspace. This is
mitigated by later commits.
o It is necessary to scan the full tasklist, much as for Tasks RCU.
o There is a single callback queue guarded by a single lock,
again, much as for Tasks RCU. However, those early use cases
that request multiple grace periods in quick succession are
expected to do so from a single task, which makes the single
lock almost irrelevant. If needed, multiple callback queues
can be provided using any number of schemes.
Perhaps most important, this variant of RCU does not affect the vanilla
flavors, rcu_preempt and rcu_sched. The fact that RCU Tasks Trace
readers can operate from idle, offline, and exception entry/exit in no
way enables rcu_preempt and rcu_sched readers to do so.
The memory ordering was outlined here:
https://lore.kernel.org/lkml/20200319034030.GX3199@paulmck-ThinkPad-P72/
This effort benefited greatly from off-list discussions of BPF
requirements with Alexei Starovoitov and Andrii Nakryiko. At least
some of the on-list discussions are captured in the Link: tags below.
In addition, KCSAN was quite helpful in finding some early bugs.
Link: https://lore.kernel.org/lkml/20200219150744.428764577@infradead.org/
Link: https://lore.kernel.org/lkml/87mu8p797b.fsf@nanos.tec.linutronix.de/
Link: https://lore.kernel.org/lkml/20200225221305.605144982@linutronix.de/
Cc: Alexei Starovoitov <alexei.starovoitov@gmail.com>
Cc: Andrii Nakryiko <andriin@fb.com>
[ paulmck: Apply feedback from Steve Rostedt and Joel Fernandes. ]
[ paulmck: Decrement trc_n_readers_need_end upon IPI failure. ]
[ paulmck: Fix locking issue reported by rcutorture. ]
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
2020-03-09 19:56:53 -07:00
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
// Attempt to nail down the task for inspection.
|
|
|
|
|
get_task_struct(t);
|
2022-05-23 20:50:11 -07:00
|
|
|
if (!task_call_func(t, trc_inspect_reader, bhp)) {
|
rcu-tasks: Add an RCU Tasks Trace to simplify protection of tracing hooks
Because RCU does not watch exception early-entry/late-exit, idle-loop,
or CPU-hotplug execution, protection of tracing and BPF operations is
needlessly complicated. This commit therefore adds a variant of
Tasks RCU that:
o Has explicit read-side markers to allow finite grace periods in
the face of in-kernel loops for PREEMPT=n builds. These markers
are rcu_read_lock_trace() and rcu_read_unlock_trace().
o Protects code in the idle loop, exception entry/exit, and
CPU-hotplug code paths. In this respect, RCU-tasks trace is
similar to SRCU, but with lighter-weight readers.
o Avoids expensive read-side instruction, having overhead similar
to that of Preemptible RCU.
There are of course downsides:
o The grace-period code can send IPIs to CPUs, even when those
CPUs are in the idle loop or in nohz_full userspace. This is
mitigated by later commits.
o It is necessary to scan the full tasklist, much as for Tasks RCU.
o There is a single callback queue guarded by a single lock,
again, much as for Tasks RCU. However, those early use cases
that request multiple grace periods in quick succession are
expected to do so from a single task, which makes the single
lock almost irrelevant. If needed, multiple callback queues
can be provided using any number of schemes.
Perhaps most important, this variant of RCU does not affect the vanilla
flavors, rcu_preempt and rcu_sched. The fact that RCU Tasks Trace
readers can operate from idle, offline, and exception entry/exit in no
way enables rcu_preempt and rcu_sched readers to do so.
The memory ordering was outlined here:
https://lore.kernel.org/lkml/20200319034030.GX3199@paulmck-ThinkPad-P72/
This effort benefited greatly from off-list discussions of BPF
requirements with Alexei Starovoitov and Andrii Nakryiko. At least
some of the on-list discussions are captured in the Link: tags below.
In addition, KCSAN was quite helpful in finding some early bugs.
Link: https://lore.kernel.org/lkml/20200219150744.428764577@infradead.org/
Link: https://lore.kernel.org/lkml/87mu8p797b.fsf@nanos.tec.linutronix.de/
Link: https://lore.kernel.org/lkml/20200225221305.605144982@linutronix.de/
Cc: Alexei Starovoitov <alexei.starovoitov@gmail.com>
Cc: Andrii Nakryiko <andriin@fb.com>
[ paulmck: Apply feedback from Steve Rostedt and Joel Fernandes. ]
[ paulmck: Decrement trc_n_readers_need_end upon IPI failure. ]
[ paulmck: Fix locking issue reported by rcutorture. ]
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
2020-03-09 19:56:53 -07:00
|
|
|
put_task_struct(t);
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
put_task_struct(t);
|
|
|
|
|
|
2021-05-24 11:26:53 -07:00
|
|
|
// If this task is not yet on the holdout list, then we are in
|
|
|
|
|
// an RCU read-side critical section. Otherwise, the invocation of
|
2021-08-18 00:28:39 -07:00
|
|
|
// trc_add_holdout() that added it to the list did the necessary
|
2021-05-24 11:26:53 -07:00
|
|
|
// get_task_struct(). Either way, the task cannot be freed out
|
|
|
|
|
// from under this code.
|
|
|
|
|
|
rcu-tasks: Add an RCU Tasks Trace to simplify protection of tracing hooks
Because RCU does not watch exception early-entry/late-exit, idle-loop,
or CPU-hotplug execution, protection of tracing and BPF operations is
needlessly complicated. This commit therefore adds a variant of
Tasks RCU that:
o Has explicit read-side markers to allow finite grace periods in
the face of in-kernel loops for PREEMPT=n builds. These markers
are rcu_read_lock_trace() and rcu_read_unlock_trace().
o Protects code in the idle loop, exception entry/exit, and
CPU-hotplug code paths. In this respect, RCU-tasks trace is
similar to SRCU, but with lighter-weight readers.
o Avoids expensive read-side instruction, having overhead similar
to that of Preemptible RCU.
There are of course downsides:
o The grace-period code can send IPIs to CPUs, even when those
CPUs are in the idle loop or in nohz_full userspace. This is
mitigated by later commits.
o It is necessary to scan the full tasklist, much as for Tasks RCU.
o There is a single callback queue guarded by a single lock,
again, much as for Tasks RCU. However, those early use cases
that request multiple grace periods in quick succession are
expected to do so from a single task, which makes the single
lock almost irrelevant. If needed, multiple callback queues
can be provided using any number of schemes.
Perhaps most important, this variant of RCU does not affect the vanilla
flavors, rcu_preempt and rcu_sched. The fact that RCU Tasks Trace
readers can operate from idle, offline, and exception entry/exit in no
way enables rcu_preempt and rcu_sched readers to do so.
The memory ordering was outlined here:
https://lore.kernel.org/lkml/20200319034030.GX3199@paulmck-ThinkPad-P72/
This effort benefited greatly from off-list discussions of BPF
requirements with Alexei Starovoitov and Andrii Nakryiko. At least
some of the on-list discussions are captured in the Link: tags below.
In addition, KCSAN was quite helpful in finding some early bugs.
Link: https://lore.kernel.org/lkml/20200219150744.428764577@infradead.org/
Link: https://lore.kernel.org/lkml/87mu8p797b.fsf@nanos.tec.linutronix.de/
Link: https://lore.kernel.org/lkml/20200225221305.605144982@linutronix.de/
Cc: Alexei Starovoitov <alexei.starovoitov@gmail.com>
Cc: Andrii Nakryiko <andriin@fb.com>
[ paulmck: Apply feedback from Steve Rostedt and Joel Fernandes. ]
[ paulmck: Decrement trc_n_readers_need_end upon IPI failure. ]
[ paulmck: Fix locking issue reported by rcutorture. ]
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
2020-03-09 19:56:53 -07:00
|
|
|
// If currently running, send an IPI, either way, add to list.
|
|
|
|
|
trc_add_holdout(t, bhp);
|
2020-09-09 21:51:09 -07:00
|
|
|
if (task_curr(t) &&
|
|
|
|
|
time_after(jiffies + 1, rcu_tasks_trace.gp_start + rcu_task_ipi_delay)) {
|
rcu-tasks: Add an RCU Tasks Trace to simplify protection of tracing hooks
Because RCU does not watch exception early-entry/late-exit, idle-loop,
or CPU-hotplug execution, protection of tracing and BPF operations is
needlessly complicated. This commit therefore adds a variant of
Tasks RCU that:
o Has explicit read-side markers to allow finite grace periods in
the face of in-kernel loops for PREEMPT=n builds. These markers
are rcu_read_lock_trace() and rcu_read_unlock_trace().
o Protects code in the idle loop, exception entry/exit, and
CPU-hotplug code paths. In this respect, RCU-tasks trace is
similar to SRCU, but with lighter-weight readers.
o Avoids expensive read-side instruction, having overhead similar
to that of Preemptible RCU.
There are of course downsides:
o The grace-period code can send IPIs to CPUs, even when those
CPUs are in the idle loop or in nohz_full userspace. This is
mitigated by later commits.
o It is necessary to scan the full tasklist, much as for Tasks RCU.
o There is a single callback queue guarded by a single lock,
again, much as for Tasks RCU. However, those early use cases
that request multiple grace periods in quick succession are
expected to do so from a single task, which makes the single
lock almost irrelevant. If needed, multiple callback queues
can be provided using any number of schemes.
Perhaps most important, this variant of RCU does not affect the vanilla
flavors, rcu_preempt and rcu_sched. The fact that RCU Tasks Trace
readers can operate from idle, offline, and exception entry/exit in no
way enables rcu_preempt and rcu_sched readers to do so.
The memory ordering was outlined here:
https://lore.kernel.org/lkml/20200319034030.GX3199@paulmck-ThinkPad-P72/
This effort benefited greatly from off-list discussions of BPF
requirements with Alexei Starovoitov and Andrii Nakryiko. At least
some of the on-list discussions are captured in the Link: tags below.
In addition, KCSAN was quite helpful in finding some early bugs.
Link: https://lore.kernel.org/lkml/20200219150744.428764577@infradead.org/
Link: https://lore.kernel.org/lkml/87mu8p797b.fsf@nanos.tec.linutronix.de/
Link: https://lore.kernel.org/lkml/20200225221305.605144982@linutronix.de/
Cc: Alexei Starovoitov <alexei.starovoitov@gmail.com>
Cc: Andrii Nakryiko <andriin@fb.com>
[ paulmck: Apply feedback from Steve Rostedt and Joel Fernandes. ]
[ paulmck: Decrement trc_n_readers_need_end upon IPI failure. ]
[ paulmck: Fix locking issue reported by rcutorture. ]
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
2020-03-09 19:56:53 -07:00
|
|
|
// The task is currently running, so try IPIing it.
|
|
|
|
|
cpu = task_cpu(t);
|
|
|
|
|
|
|
|
|
|
// If there is already an IPI outstanding, let it happen.
|
|
|
|
|
if (per_cpu(trc_ipi_to_cpu, cpu) || t->trc_ipi_to_cpu >= 0)
|
|
|
|
|
return;
|
|
|
|
|
|
|
|
|
|
per_cpu(trc_ipi_to_cpu, cpu) = true;
|
|
|
|
|
t->trc_ipi_to_cpu = cpu;
|
2020-03-18 10:54:05 -07:00
|
|
|
rcu_tasks_trace.n_ipis++;
|
2021-07-28 10:53:41 -07:00
|
|
|
if (smp_call_function_single(cpu, trc_read_check_handler, t, 0)) {
|
rcu-tasks: Add an RCU Tasks Trace to simplify protection of tracing hooks
Because RCU does not watch exception early-entry/late-exit, idle-loop,
or CPU-hotplug execution, protection of tracing and BPF operations is
needlessly complicated. This commit therefore adds a variant of
Tasks RCU that:
o Has explicit read-side markers to allow finite grace periods in
the face of in-kernel loops for PREEMPT=n builds. These markers
are rcu_read_lock_trace() and rcu_read_unlock_trace().
o Protects code in the idle loop, exception entry/exit, and
CPU-hotplug code paths. In this respect, RCU-tasks trace is
similar to SRCU, but with lighter-weight readers.
o Avoids expensive read-side instruction, having overhead similar
to that of Preemptible RCU.
There are of course downsides:
o The grace-period code can send IPIs to CPUs, even when those
CPUs are in the idle loop or in nohz_full userspace. This is
mitigated by later commits.
o It is necessary to scan the full tasklist, much as for Tasks RCU.
o There is a single callback queue guarded by a single lock,
again, much as for Tasks RCU. However, those early use cases
that request multiple grace periods in quick succession are
expected to do so from a single task, which makes the single
lock almost irrelevant. If needed, multiple callback queues
can be provided using any number of schemes.
Perhaps most important, this variant of RCU does not affect the vanilla
flavors, rcu_preempt and rcu_sched. The fact that RCU Tasks Trace
readers can operate from idle, offline, and exception entry/exit in no
way enables rcu_preempt and rcu_sched readers to do so.
The memory ordering was outlined here:
https://lore.kernel.org/lkml/20200319034030.GX3199@paulmck-ThinkPad-P72/
This effort benefited greatly from off-list discussions of BPF
requirements with Alexei Starovoitov and Andrii Nakryiko. At least
some of the on-list discussions are captured in the Link: tags below.
In addition, KCSAN was quite helpful in finding some early bugs.
Link: https://lore.kernel.org/lkml/20200219150744.428764577@infradead.org/
Link: https://lore.kernel.org/lkml/87mu8p797b.fsf@nanos.tec.linutronix.de/
Link: https://lore.kernel.org/lkml/20200225221305.605144982@linutronix.de/
Cc: Alexei Starovoitov <alexei.starovoitov@gmail.com>
Cc: Andrii Nakryiko <andriin@fb.com>
[ paulmck: Apply feedback from Steve Rostedt and Joel Fernandes. ]
[ paulmck: Decrement trc_n_readers_need_end upon IPI failure. ]
[ paulmck: Fix locking issue reported by rcutorture. ]
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
2020-03-09 19:56:53 -07:00
|
|
|
// Just in case there is some other reason for
|
|
|
|
|
// failure than the target CPU being offline.
|
rcu-tasks: Fix IPI failure handling in trc_wait_for_one_reader
The trc_wait_for_one_reader() function is called at multiple stages
of trace rcu-tasks GP function, rcu_tasks_wait_gp():
- First, it is called as part of per task function -
rcu_tasks_trace_pertask(), for all non-idle tasks. As part of per task
processing, this function add the task in the holdout list and if the
task is currently running on a CPU, it sends IPI to the task's CPU.
The IPI handler takes action depending on whether task is in trace
rcu-tasks read side critical section or not:
- a. If the task is in trace rcu-tasks read side critical section
(t->trc_reader_nesting != 0), the IPI handler sets the task's
->trc_reader_special.b.need_qs, so that this task notifies exit
from its outermost read side critical section (by decrementing
trc_n_readers_need_end) to the GP handling function.
trc_wait_for_one_reader() also increments trc_n_readers_need_end,
so that the trace rcu-tasks GP handler function waits for this
task's read side exit notification. The IPI handler also sets
t->trc_reader_checked to true, and no further IPIs are sent for
this task, for this trace rcu-tasks grace period and this
task can be removed from holdout list.
- b. If the task is in the process of exiting its trace rcu-tasks
read side critical section, (t->trc_reader_nesting < 0), defer
this task's processing to future calls to trc_wait_for_one_reader().
- c. If task is not in rcu-task read side critical section,
t->trc_reader_nesting == 0, ->trc_reader_checked is set for this
task, so that this task is removed from holdout list.
- Second, trc_wait_for_one_reader() is called as part of post scan, in
function rcu_tasks_trace_postscan(), for all idle tasks.
- Third, in function check_all_holdout_tasks_trace(), this function is
called for each task in the holdout list, but only if there isn't
a pending IPI for the task (->trc_ipi_to_cpu == -1). This function
removed the task from holdout list, if IPI handler has completed the
required work, to ensure that the current trace rcu-tasks grace period
either waits for this task, or this task is not in a trace rcu-tasks
read side critical section.
Now, considering the scenario where smp_call_function_single() fails in
first case, inside rcu_tasks_trace_pertask(). In this case,
->trc_ipi_to_cpu is set to the current CPU for that task. This will
result in trc_wait_for_one_reader() getting skipped in third case,
inside check_all_holdout_tasks_trace(), for this task. This further
results in ->trc_reader_checked never getting set for this task,
and the task not getting removed from holdout list. This can cause
the current trace rcu-tasks grace period to stall.
Fix the above problem, by resetting ->trc_ipi_to_cpu to -1, on
smp_call_function_single() failure, so that future IPI calls can
be send for this task.
Note that all three of the trc_wait_for_one_reader() function's
callers (rcu_tasks_trace_pertask(), rcu_tasks_trace_postscan(),
check_all_holdout_tasks_trace()) hold cpu_read_lock(). This means
that smp_call_function_single() cannot race with CPU hotplug, and thus
should never fail. Therefore, also add a warning in order to report
any such failure in case smp_call_function_single() grows some other
reason for failure.
Signed-off-by: Neeraj Upadhyay <neeraju@codeaurora.org>
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
2021-08-27 01:13:35 -07:00
|
|
|
WARN_ONCE(1, "%s(): smp_call_function_single() failed for CPU: %d\n",
|
|
|
|
|
__func__, cpu);
|
2020-03-25 14:36:05 -07:00
|
|
|
rcu_tasks_trace.n_ipis_fails++;
|
rcu-tasks: Add an RCU Tasks Trace to simplify protection of tracing hooks
Because RCU does not watch exception early-entry/late-exit, idle-loop,
or CPU-hotplug execution, protection of tracing and BPF operations is
needlessly complicated. This commit therefore adds a variant of
Tasks RCU that:
o Has explicit read-side markers to allow finite grace periods in
the face of in-kernel loops for PREEMPT=n builds. These markers
are rcu_read_lock_trace() and rcu_read_unlock_trace().
o Protects code in the idle loop, exception entry/exit, and
CPU-hotplug code paths. In this respect, RCU-tasks trace is
similar to SRCU, but with lighter-weight readers.
o Avoids expensive read-side instruction, having overhead similar
to that of Preemptible RCU.
There are of course downsides:
o The grace-period code can send IPIs to CPUs, even when those
CPUs are in the idle loop or in nohz_full userspace. This is
mitigated by later commits.
o It is necessary to scan the full tasklist, much as for Tasks RCU.
o There is a single callback queue guarded by a single lock,
again, much as for Tasks RCU. However, those early use cases
that request multiple grace periods in quick succession are
expected to do so from a single task, which makes the single
lock almost irrelevant. If needed, multiple callback queues
can be provided using any number of schemes.
Perhaps most important, this variant of RCU does not affect the vanilla
flavors, rcu_preempt and rcu_sched. The fact that RCU Tasks Trace
readers can operate from idle, offline, and exception entry/exit in no
way enables rcu_preempt and rcu_sched readers to do so.
The memory ordering was outlined here:
https://lore.kernel.org/lkml/20200319034030.GX3199@paulmck-ThinkPad-P72/
This effort benefited greatly from off-list discussions of BPF
requirements with Alexei Starovoitov and Andrii Nakryiko. At least
some of the on-list discussions are captured in the Link: tags below.
In addition, KCSAN was quite helpful in finding some early bugs.
Link: https://lore.kernel.org/lkml/20200219150744.428764577@infradead.org/
Link: https://lore.kernel.org/lkml/87mu8p797b.fsf@nanos.tec.linutronix.de/
Link: https://lore.kernel.org/lkml/20200225221305.605144982@linutronix.de/
Cc: Alexei Starovoitov <alexei.starovoitov@gmail.com>
Cc: Andrii Nakryiko <andriin@fb.com>
[ paulmck: Apply feedback from Steve Rostedt and Joel Fernandes. ]
[ paulmck: Decrement trc_n_readers_need_end upon IPI failure. ]
[ paulmck: Fix locking issue reported by rcutorture. ]
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
2020-03-09 19:56:53 -07:00
|
|
|
per_cpu(trc_ipi_to_cpu, cpu) = false;
|
rcu-tasks: Fix IPI failure handling in trc_wait_for_one_reader
The trc_wait_for_one_reader() function is called at multiple stages
of trace rcu-tasks GP function, rcu_tasks_wait_gp():
- First, it is called as part of per task function -
rcu_tasks_trace_pertask(), for all non-idle tasks. As part of per task
processing, this function add the task in the holdout list and if the
task is currently running on a CPU, it sends IPI to the task's CPU.
The IPI handler takes action depending on whether task is in trace
rcu-tasks read side critical section or not:
- a. If the task is in trace rcu-tasks read side critical section
(t->trc_reader_nesting != 0), the IPI handler sets the task's
->trc_reader_special.b.need_qs, so that this task notifies exit
from its outermost read side critical section (by decrementing
trc_n_readers_need_end) to the GP handling function.
trc_wait_for_one_reader() also increments trc_n_readers_need_end,
so that the trace rcu-tasks GP handler function waits for this
task's read side exit notification. The IPI handler also sets
t->trc_reader_checked to true, and no further IPIs are sent for
this task, for this trace rcu-tasks grace period and this
task can be removed from holdout list.
- b. If the task is in the process of exiting its trace rcu-tasks
read side critical section, (t->trc_reader_nesting < 0), defer
this task's processing to future calls to trc_wait_for_one_reader().
- c. If task is not in rcu-task read side critical section,
t->trc_reader_nesting == 0, ->trc_reader_checked is set for this
task, so that this task is removed from holdout list.
- Second, trc_wait_for_one_reader() is called as part of post scan, in
function rcu_tasks_trace_postscan(), for all idle tasks.
- Third, in function check_all_holdout_tasks_trace(), this function is
called for each task in the holdout list, but only if there isn't
a pending IPI for the task (->trc_ipi_to_cpu == -1). This function
removed the task from holdout list, if IPI handler has completed the
required work, to ensure that the current trace rcu-tasks grace period
either waits for this task, or this task is not in a trace rcu-tasks
read side critical section.
Now, considering the scenario where smp_call_function_single() fails in
first case, inside rcu_tasks_trace_pertask(). In this case,
->trc_ipi_to_cpu is set to the current CPU for that task. This will
result in trc_wait_for_one_reader() getting skipped in third case,
inside check_all_holdout_tasks_trace(), for this task. This further
results in ->trc_reader_checked never getting set for this task,
and the task not getting removed from holdout list. This can cause
the current trace rcu-tasks grace period to stall.
Fix the above problem, by resetting ->trc_ipi_to_cpu to -1, on
smp_call_function_single() failure, so that future IPI calls can
be send for this task.
Note that all three of the trc_wait_for_one_reader() function's
callers (rcu_tasks_trace_pertask(), rcu_tasks_trace_postscan(),
check_all_holdout_tasks_trace()) hold cpu_read_lock(). This means
that smp_call_function_single() cannot race with CPU hotplug, and thus
should never fail. Therefore, also add a warning in order to report
any such failure in case smp_call_function_single() grows some other
reason for failure.
Signed-off-by: Neeraj Upadhyay <neeraju@codeaurora.org>
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
2021-08-27 01:13:35 -07:00
|
|
|
t->trc_ipi_to_cpu = -1;
|
rcu-tasks: Add an RCU Tasks Trace to simplify protection of tracing hooks
Because RCU does not watch exception early-entry/late-exit, idle-loop,
or CPU-hotplug execution, protection of tracing and BPF operations is
needlessly complicated. This commit therefore adds a variant of
Tasks RCU that:
o Has explicit read-side markers to allow finite grace periods in
the face of in-kernel loops for PREEMPT=n builds. These markers
are rcu_read_lock_trace() and rcu_read_unlock_trace().
o Protects code in the idle loop, exception entry/exit, and
CPU-hotplug code paths. In this respect, RCU-tasks trace is
similar to SRCU, but with lighter-weight readers.
o Avoids expensive read-side instruction, having overhead similar
to that of Preemptible RCU.
There are of course downsides:
o The grace-period code can send IPIs to CPUs, even when those
CPUs are in the idle loop or in nohz_full userspace. This is
mitigated by later commits.
o It is necessary to scan the full tasklist, much as for Tasks RCU.
o There is a single callback queue guarded by a single lock,
again, much as for Tasks RCU. However, those early use cases
that request multiple grace periods in quick succession are
expected to do so from a single task, which makes the single
lock almost irrelevant. If needed, multiple callback queues
can be provided using any number of schemes.
Perhaps most important, this variant of RCU does not affect the vanilla
flavors, rcu_preempt and rcu_sched. The fact that RCU Tasks Trace
readers can operate from idle, offline, and exception entry/exit in no
way enables rcu_preempt and rcu_sched readers to do so.
The memory ordering was outlined here:
https://lore.kernel.org/lkml/20200319034030.GX3199@paulmck-ThinkPad-P72/
This effort benefited greatly from off-list discussions of BPF
requirements with Alexei Starovoitov and Andrii Nakryiko. At least
some of the on-list discussions are captured in the Link: tags below.
In addition, KCSAN was quite helpful in finding some early bugs.
Link: https://lore.kernel.org/lkml/20200219150744.428764577@infradead.org/
Link: https://lore.kernel.org/lkml/87mu8p797b.fsf@nanos.tec.linutronix.de/
Link: https://lore.kernel.org/lkml/20200225221305.605144982@linutronix.de/
Cc: Alexei Starovoitov <alexei.starovoitov@gmail.com>
Cc: Andrii Nakryiko <andriin@fb.com>
[ paulmck: Apply feedback from Steve Rostedt and Joel Fernandes. ]
[ paulmck: Decrement trc_n_readers_need_end upon IPI failure. ]
[ paulmck: Fix locking issue reported by rcutorture. ]
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
2020-03-09 19:56:53 -07:00
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2022-05-18 16:06:55 -07:00
|
|
|
/*
|
|
|
|
|
* Initialize for first-round processing for the specified task.
|
|
|
|
|
* Return false if task is NULL or already taken care of, true otherwise.
|
|
|
|
|
*/
|
|
|
|
|
static bool rcu_tasks_trace_pertask_prep(struct task_struct *t, bool notself)
|
rcu-tasks: Add an RCU Tasks Trace to simplify protection of tracing hooks
Because RCU does not watch exception early-entry/late-exit, idle-loop,
or CPU-hotplug execution, protection of tracing and BPF operations is
needlessly complicated. This commit therefore adds a variant of
Tasks RCU that:
o Has explicit read-side markers to allow finite grace periods in
the face of in-kernel loops for PREEMPT=n builds. These markers
are rcu_read_lock_trace() and rcu_read_unlock_trace().
o Protects code in the idle loop, exception entry/exit, and
CPU-hotplug code paths. In this respect, RCU-tasks trace is
similar to SRCU, but with lighter-weight readers.
o Avoids expensive read-side instruction, having overhead similar
to that of Preemptible RCU.
There are of course downsides:
o The grace-period code can send IPIs to CPUs, even when those
CPUs are in the idle loop or in nohz_full userspace. This is
mitigated by later commits.
o It is necessary to scan the full tasklist, much as for Tasks RCU.
o There is a single callback queue guarded by a single lock,
again, much as for Tasks RCU. However, those early use cases
that request multiple grace periods in quick succession are
expected to do so from a single task, which makes the single
lock almost irrelevant. If needed, multiple callback queues
can be provided using any number of schemes.
Perhaps most important, this variant of RCU does not affect the vanilla
flavors, rcu_preempt and rcu_sched. The fact that RCU Tasks Trace
readers can operate from idle, offline, and exception entry/exit in no
way enables rcu_preempt and rcu_sched readers to do so.
The memory ordering was outlined here:
https://lore.kernel.org/lkml/20200319034030.GX3199@paulmck-ThinkPad-P72/
This effort benefited greatly from off-list discussions of BPF
requirements with Alexei Starovoitov and Andrii Nakryiko. At least
some of the on-list discussions are captured in the Link: tags below.
In addition, KCSAN was quite helpful in finding some early bugs.
Link: https://lore.kernel.org/lkml/20200219150744.428764577@infradead.org/
Link: https://lore.kernel.org/lkml/87mu8p797b.fsf@nanos.tec.linutronix.de/
Link: https://lore.kernel.org/lkml/20200225221305.605144982@linutronix.de/
Cc: Alexei Starovoitov <alexei.starovoitov@gmail.com>
Cc: Andrii Nakryiko <andriin@fb.com>
[ paulmck: Apply feedback from Steve Rostedt and Joel Fernandes. ]
[ paulmck: Decrement trc_n_readers_need_end upon IPI failure. ]
[ paulmck: Fix locking issue reported by rcutorture. ]
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
2020-03-09 19:56:53 -07:00
|
|
|
{
|
2020-12-09 13:27:31 -07:00
|
|
|
// During early boot when there is only the one boot CPU, there
|
2022-05-17 17:38:02 -07:00
|
|
|
// is no idle task for the other CPUs. Also, the grace-period
|
|
|
|
|
// kthread is always in a quiescent state. In addition, just return
|
|
|
|
|
// if this task is already on the list.
|
2022-05-18 16:06:55 -07:00
|
|
|
if (unlikely(t == NULL) || (t == current && notself) || !list_empty(&t->trc_holdout_list))
|
|
|
|
|
return false;
|
2020-12-09 13:27:31 -07:00
|
|
|
|
2022-05-23 20:50:11 -07:00
|
|
|
rcu_st_need_qs(t, 0);
|
rcu-tasks: Add an RCU Tasks Trace to simplify protection of tracing hooks
Because RCU does not watch exception early-entry/late-exit, idle-loop,
or CPU-hotplug execution, protection of tracing and BPF operations is
needlessly complicated. This commit therefore adds a variant of
Tasks RCU that:
o Has explicit read-side markers to allow finite grace periods in
the face of in-kernel loops for PREEMPT=n builds. These markers
are rcu_read_lock_trace() and rcu_read_unlock_trace().
o Protects code in the idle loop, exception entry/exit, and
CPU-hotplug code paths. In this respect, RCU-tasks trace is
similar to SRCU, but with lighter-weight readers.
o Avoids expensive read-side instruction, having overhead similar
to that of Preemptible RCU.
There are of course downsides:
o The grace-period code can send IPIs to CPUs, even when those
CPUs are in the idle loop or in nohz_full userspace. This is
mitigated by later commits.
o It is necessary to scan the full tasklist, much as for Tasks RCU.
o There is a single callback queue guarded by a single lock,
again, much as for Tasks RCU. However, those early use cases
that request multiple grace periods in quick succession are
expected to do so from a single task, which makes the single
lock almost irrelevant. If needed, multiple callback queues
can be provided using any number of schemes.
Perhaps most important, this variant of RCU does not affect the vanilla
flavors, rcu_preempt and rcu_sched. The fact that RCU Tasks Trace
readers can operate from idle, offline, and exception entry/exit in no
way enables rcu_preempt and rcu_sched readers to do so.
The memory ordering was outlined here:
https://lore.kernel.org/lkml/20200319034030.GX3199@paulmck-ThinkPad-P72/
This effort benefited greatly from off-list discussions of BPF
requirements with Alexei Starovoitov and Andrii Nakryiko. At least
some of the on-list discussions are captured in the Link: tags below.
In addition, KCSAN was quite helpful in finding some early bugs.
Link: https://lore.kernel.org/lkml/20200219150744.428764577@infradead.org/
Link: https://lore.kernel.org/lkml/87mu8p797b.fsf@nanos.tec.linutronix.de/
Link: https://lore.kernel.org/lkml/20200225221305.605144982@linutronix.de/
Cc: Alexei Starovoitov <alexei.starovoitov@gmail.com>
Cc: Andrii Nakryiko <andriin@fb.com>
[ paulmck: Apply feedback from Steve Rostedt and Joel Fernandes. ]
[ paulmck: Decrement trc_n_readers_need_end upon IPI failure. ]
[ paulmck: Fix locking issue reported by rcutorture. ]
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
2020-03-09 19:56:53 -07:00
|
|
|
t->trc_ipi_to_cpu = -1;
|
2022-05-18 16:06:55 -07:00
|
|
|
return true;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/* Do first-round processing for the specified task. */
|
|
|
|
|
static void rcu_tasks_trace_pertask(struct task_struct *t, struct list_head *hop)
|
|
|
|
|
{
|
|
|
|
|
if (rcu_tasks_trace_pertask_prep(t, true))
|
|
|
|
|
trc_wait_for_one_reader(t, hop);
|
|
|
|
|
}
|
|
|
|
|
|
2022-05-17 17:11:37 -07:00
|
|
|
/* Initialize for a new RCU-tasks-trace grace period. */
|
2022-05-18 16:06:55 -07:00
|
|
|
static void rcu_tasks_trace_pregp_step(struct list_head *hop)
|
2022-05-17 17:11:37 -07:00
|
|
|
{
|
2022-05-18 17:19:27 -07:00
|
|
|
LIST_HEAD(blkd_tasks);
|
2022-05-17 17:11:37 -07:00
|
|
|
int cpu;
|
2022-05-18 17:19:27 -07:00
|
|
|
unsigned long flags;
|
|
|
|
|
struct rcu_tasks_percpu *rtpcp;
|
|
|
|
|
struct task_struct *t;
|
2022-05-17 17:11:37 -07:00
|
|
|
|
|
|
|
|
// There shouldn't be any old IPIs, but...
|
|
|
|
|
for_each_possible_cpu(cpu)
|
|
|
|
|
WARN_ON_ONCE(per_cpu(trc_ipi_to_cpu, cpu));
|
|
|
|
|
|
2022-06-06 21:30:38 -07:00
|
|
|
// Disable CPU hotplug across the CPU scan for the benefit of
|
|
|
|
|
// any IPIs that might be needed. This also waits for all readers
|
|
|
|
|
// in CPU-hotplug code paths.
|
2022-05-17 17:11:37 -07:00
|
|
|
cpus_read_lock();
|
2022-05-18 16:06:55 -07:00
|
|
|
|
2022-06-06 21:30:38 -07:00
|
|
|
// These rcu_tasks_trace_pertask_prep() calls are serialized to
|
2022-05-18 16:06:55 -07:00
|
|
|
// allow safe access to the hop list.
|
rcu-tasks: Eliminate RCU Tasks Trace IPIs to online CPUs
Currently, the RCU Tasks Trace grace-period kthread IPIs each online CPU
using smp_call_function_single() in order to track any tasks currently in
RCU Tasks Trace read-side critical sections during which the corresponding
task has neither blocked nor been preempted. These IPIs are annoying
and are also not strictly necessary because any task that blocks or is
preempted within its current RCU Tasks Trace read-side critical section
will be tracked on one of the per-CPU rcu_tasks_percpu structure's
->rtp_blkd_tasks list. So the only time that this is a problem is if
one of the CPUs runs through a long-duration RCU Tasks Trace read-side
critical section without a context switch.
Note that the task_call_func() function cannot help here because there is
no safe way to identify the target task. Of course, the task_call_func()
function will be very useful later, when processing the list of tasks,
but it needs to know the task.
This commit therefore creates a cpu_curr_snapshot() function that returns
a pointer the task_struct structure of some task that happened to be
running on the specified CPU more or less during the time that the
cpu_curr_snapshot() function was executing. If there was no context
switch during this time, this function will return a pointer to the
task_struct structure of the task that was running throughout. If there
was a context switch, then the outgoing task will be taken care of by
RCU's context-switch hook, and the incoming task was either already taken
care during some previous context switch, or it is not currently within an
RCU Tasks Trace read-side critical section. And in this latter case, the
grace period already started, so there is no need to wait on this task.
This new cpu_curr_snapshot() function is invoked on each CPU early in
the RCU Tasks Trace grace-period processing, and the resulting tasks
are queued for later quiescent-state inspection.
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Neeraj Upadhyay <quic_neeraju@quicinc.com>
Cc: Eric Dumazet <edumazet@google.com>
Cc: Alexei Starovoitov <ast@kernel.org>
Cc: Andrii Nakryiko <andrii@kernel.org>
Cc: Martin KaFai Lau <kafai@fb.com>
Cc: KP Singh <kpsingh@kernel.org>
2022-06-02 17:30:01 -07:00
|
|
|
for_each_online_cpu(cpu) {
|
|
|
|
|
rcu_read_lock();
|
2024-05-17 08:23:02 -07:00
|
|
|
// Note that cpu_curr_snapshot() picks up the target
|
|
|
|
|
// CPU's current task while its runqueue is locked with
|
|
|
|
|
// an smp_mb__after_spinlock(). This ensures that either
|
|
|
|
|
// the grace-period kthread will see that task's read-side
|
|
|
|
|
// critical section or the task will see the updater's pre-GP
|
|
|
|
|
// accesses. The trailing smp_mb() in cpu_curr_snapshot()
|
|
|
|
|
// does not currently play a role other than simplify
|
|
|
|
|
// that function's ordering semantics. If these simplified
|
|
|
|
|
// ordering semantics continue to be redundant, that smp_mb()
|
|
|
|
|
// might be removed.
|
rcu-tasks: Eliminate RCU Tasks Trace IPIs to online CPUs
Currently, the RCU Tasks Trace grace-period kthread IPIs each online CPU
using smp_call_function_single() in order to track any tasks currently in
RCU Tasks Trace read-side critical sections during which the corresponding
task has neither blocked nor been preempted. These IPIs are annoying
and are also not strictly necessary because any task that blocks or is
preempted within its current RCU Tasks Trace read-side critical section
will be tracked on one of the per-CPU rcu_tasks_percpu structure's
->rtp_blkd_tasks list. So the only time that this is a problem is if
one of the CPUs runs through a long-duration RCU Tasks Trace read-side
critical section without a context switch.
Note that the task_call_func() function cannot help here because there is
no safe way to identify the target task. Of course, the task_call_func()
function will be very useful later, when processing the list of tasks,
but it needs to know the task.
This commit therefore creates a cpu_curr_snapshot() function that returns
a pointer the task_struct structure of some task that happened to be
running on the specified CPU more or less during the time that the
cpu_curr_snapshot() function was executing. If there was no context
switch during this time, this function will return a pointer to the
task_struct structure of the task that was running throughout. If there
was a context switch, then the outgoing task will be taken care of by
RCU's context-switch hook, and the incoming task was either already taken
care during some previous context switch, or it is not currently within an
RCU Tasks Trace read-side critical section. And in this latter case, the
grace period already started, so there is no need to wait on this task.
This new cpu_curr_snapshot() function is invoked on each CPU early in
the RCU Tasks Trace grace-period processing, and the resulting tasks
are queued for later quiescent-state inspection.
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Neeraj Upadhyay <quic_neeraju@quicinc.com>
Cc: Eric Dumazet <edumazet@google.com>
Cc: Alexei Starovoitov <ast@kernel.org>
Cc: Andrii Nakryiko <andrii@kernel.org>
Cc: Martin KaFai Lau <kafai@fb.com>
Cc: KP Singh <kpsingh@kernel.org>
2022-06-02 17:30:01 -07:00
|
|
|
t = cpu_curr_snapshot(cpu);
|
|
|
|
|
if (rcu_tasks_trace_pertask_prep(t, true))
|
|
|
|
|
trc_add_holdout(t, hop);
|
|
|
|
|
rcu_read_unlock();
|
2022-07-18 10:57:26 -07:00
|
|
|
cond_resched_tasks_rcu_qs();
|
rcu-tasks: Eliminate RCU Tasks Trace IPIs to online CPUs
Currently, the RCU Tasks Trace grace-period kthread IPIs each online CPU
using smp_call_function_single() in order to track any tasks currently in
RCU Tasks Trace read-side critical sections during which the corresponding
task has neither blocked nor been preempted. These IPIs are annoying
and are also not strictly necessary because any task that blocks or is
preempted within its current RCU Tasks Trace read-side critical section
will be tracked on one of the per-CPU rcu_tasks_percpu structure's
->rtp_blkd_tasks list. So the only time that this is a problem is if
one of the CPUs runs through a long-duration RCU Tasks Trace read-side
critical section without a context switch.
Note that the task_call_func() function cannot help here because there is
no safe way to identify the target task. Of course, the task_call_func()
function will be very useful later, when processing the list of tasks,
but it needs to know the task.
This commit therefore creates a cpu_curr_snapshot() function that returns
a pointer the task_struct structure of some task that happened to be
running on the specified CPU more or less during the time that the
cpu_curr_snapshot() function was executing. If there was no context
switch during this time, this function will return a pointer to the
task_struct structure of the task that was running throughout. If there
was a context switch, then the outgoing task will be taken care of by
RCU's context-switch hook, and the incoming task was either already taken
care during some previous context switch, or it is not currently within an
RCU Tasks Trace read-side critical section. And in this latter case, the
grace period already started, so there is no need to wait on this task.
This new cpu_curr_snapshot() function is invoked on each CPU early in
the RCU Tasks Trace grace-period processing, and the resulting tasks
are queued for later quiescent-state inspection.
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Neeraj Upadhyay <quic_neeraju@quicinc.com>
Cc: Eric Dumazet <edumazet@google.com>
Cc: Alexei Starovoitov <ast@kernel.org>
Cc: Andrii Nakryiko <andrii@kernel.org>
Cc: Martin KaFai Lau <kafai@fb.com>
Cc: KP Singh <kpsingh@kernel.org>
2022-06-02 17:30:01 -07:00
|
|
|
}
|
2022-05-18 17:19:27 -07:00
|
|
|
|
|
|
|
|
// Only after all running tasks have been accounted for is it
|
|
|
|
|
// safe to take care of the tasks that have blocked within their
|
|
|
|
|
// current RCU tasks trace read-side critical section.
|
|
|
|
|
for_each_possible_cpu(cpu) {
|
|
|
|
|
rtpcp = per_cpu_ptr(rcu_tasks_trace.rtpcpu, cpu);
|
|
|
|
|
raw_spin_lock_irqsave_rcu_node(rtpcp, flags);
|
|
|
|
|
list_splice_init(&rtpcp->rtp_blkd_tasks, &blkd_tasks);
|
|
|
|
|
while (!list_empty(&blkd_tasks)) {
|
|
|
|
|
rcu_read_lock();
|
|
|
|
|
t = list_first_entry(&blkd_tasks, struct task_struct, trc_blkd_node);
|
|
|
|
|
list_del_init(&t->trc_blkd_node);
|
|
|
|
|
list_add(&t->trc_blkd_node, &rtpcp->rtp_blkd_tasks);
|
|
|
|
|
raw_spin_unlock_irqrestore_rcu_node(rtpcp, flags);
|
|
|
|
|
rcu_tasks_trace_pertask(t, hop);
|
|
|
|
|
rcu_read_unlock();
|
|
|
|
|
raw_spin_lock_irqsave_rcu_node(rtpcp, flags);
|
|
|
|
|
}
|
|
|
|
|
raw_spin_unlock_irqrestore_rcu_node(rtpcp, flags);
|
2022-07-18 10:57:26 -07:00
|
|
|
cond_resched_tasks_rcu_qs();
|
2022-05-18 17:19:27 -07:00
|
|
|
}
|
2022-06-07 17:25:03 -07:00
|
|
|
|
|
|
|
|
// Re-enable CPU hotplug now that the holdout list is populated.
|
|
|
|
|
cpus_read_unlock();
|
2022-05-17 17:11:37 -07:00
|
|
|
}
|
|
|
|
|
|
2020-03-22 13:18:54 -07:00
|
|
|
/*
|
2022-05-18 17:19:27 -07:00
|
|
|
* Do intermediate processing between task and holdout scans.
|
2020-03-22 13:18:54 -07:00
|
|
|
*/
|
|
|
|
|
static void rcu_tasks_trace_postscan(struct list_head *hop)
|
rcu-tasks: Add an RCU Tasks Trace to simplify protection of tracing hooks
Because RCU does not watch exception early-entry/late-exit, idle-loop,
or CPU-hotplug execution, protection of tracing and BPF operations is
needlessly complicated. This commit therefore adds a variant of
Tasks RCU that:
o Has explicit read-side markers to allow finite grace periods in
the face of in-kernel loops for PREEMPT=n builds. These markers
are rcu_read_lock_trace() and rcu_read_unlock_trace().
o Protects code in the idle loop, exception entry/exit, and
CPU-hotplug code paths. In this respect, RCU-tasks trace is
similar to SRCU, but with lighter-weight readers.
o Avoids expensive read-side instruction, having overhead similar
to that of Preemptible RCU.
There are of course downsides:
o The grace-period code can send IPIs to CPUs, even when those
CPUs are in the idle loop or in nohz_full userspace. This is
mitigated by later commits.
o It is necessary to scan the full tasklist, much as for Tasks RCU.
o There is a single callback queue guarded by a single lock,
again, much as for Tasks RCU. However, those early use cases
that request multiple grace periods in quick succession are
expected to do so from a single task, which makes the single
lock almost irrelevant. If needed, multiple callback queues
can be provided using any number of schemes.
Perhaps most important, this variant of RCU does not affect the vanilla
flavors, rcu_preempt and rcu_sched. The fact that RCU Tasks Trace
readers can operate from idle, offline, and exception entry/exit in no
way enables rcu_preempt and rcu_sched readers to do so.
The memory ordering was outlined here:
https://lore.kernel.org/lkml/20200319034030.GX3199@paulmck-ThinkPad-P72/
This effort benefited greatly from off-list discussions of BPF
requirements with Alexei Starovoitov and Andrii Nakryiko. At least
some of the on-list discussions are captured in the Link: tags below.
In addition, KCSAN was quite helpful in finding some early bugs.
Link: https://lore.kernel.org/lkml/20200219150744.428764577@infradead.org/
Link: https://lore.kernel.org/lkml/87mu8p797b.fsf@nanos.tec.linutronix.de/
Link: https://lore.kernel.org/lkml/20200225221305.605144982@linutronix.de/
Cc: Alexei Starovoitov <alexei.starovoitov@gmail.com>
Cc: Andrii Nakryiko <andriin@fb.com>
[ paulmck: Apply feedback from Steve Rostedt and Joel Fernandes. ]
[ paulmck: Decrement trc_n_readers_need_end upon IPI failure. ]
[ paulmck: Fix locking issue reported by rcutorture. ]
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
2020-03-09 19:56:53 -07:00
|
|
|
{
|
|
|
|
|
// Wait for late-stage exiting tasks to finish exiting.
|
|
|
|
|
// These might have passed the call to exit_tasks_rcu_finish().
|
2022-10-14 04:39:43 -07:00
|
|
|
|
|
|
|
|
// If you remove the following line, update rcu_trace_implies_rcu_gp()!!!
|
rcu-tasks: Add an RCU Tasks Trace to simplify protection of tracing hooks
Because RCU does not watch exception early-entry/late-exit, idle-loop,
or CPU-hotplug execution, protection of tracing and BPF operations is
needlessly complicated. This commit therefore adds a variant of
Tasks RCU that:
o Has explicit read-side markers to allow finite grace periods in
the face of in-kernel loops for PREEMPT=n builds. These markers
are rcu_read_lock_trace() and rcu_read_unlock_trace().
o Protects code in the idle loop, exception entry/exit, and
CPU-hotplug code paths. In this respect, RCU-tasks trace is
similar to SRCU, but with lighter-weight readers.
o Avoids expensive read-side instruction, having overhead similar
to that of Preemptible RCU.
There are of course downsides:
o The grace-period code can send IPIs to CPUs, even when those
CPUs are in the idle loop or in nohz_full userspace. This is
mitigated by later commits.
o It is necessary to scan the full tasklist, much as for Tasks RCU.
o There is a single callback queue guarded by a single lock,
again, much as for Tasks RCU. However, those early use cases
that request multiple grace periods in quick succession are
expected to do so from a single task, which makes the single
lock almost irrelevant. If needed, multiple callback queues
can be provided using any number of schemes.
Perhaps most important, this variant of RCU does not affect the vanilla
flavors, rcu_preempt and rcu_sched. The fact that RCU Tasks Trace
readers can operate from idle, offline, and exception entry/exit in no
way enables rcu_preempt and rcu_sched readers to do so.
The memory ordering was outlined here:
https://lore.kernel.org/lkml/20200319034030.GX3199@paulmck-ThinkPad-P72/
This effort benefited greatly from off-list discussions of BPF
requirements with Alexei Starovoitov and Andrii Nakryiko. At least
some of the on-list discussions are captured in the Link: tags below.
In addition, KCSAN was quite helpful in finding some early bugs.
Link: https://lore.kernel.org/lkml/20200219150744.428764577@infradead.org/
Link: https://lore.kernel.org/lkml/87mu8p797b.fsf@nanos.tec.linutronix.de/
Link: https://lore.kernel.org/lkml/20200225221305.605144982@linutronix.de/
Cc: Alexei Starovoitov <alexei.starovoitov@gmail.com>
Cc: Andrii Nakryiko <andriin@fb.com>
[ paulmck: Apply feedback from Steve Rostedt and Joel Fernandes. ]
[ paulmck: Decrement trc_n_readers_need_end upon IPI failure. ]
[ paulmck: Fix locking issue reported by rcutorture. ]
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
2020-03-09 19:56:53 -07:00
|
|
|
synchronize_rcu();
|
2022-05-23 20:50:11 -07:00
|
|
|
// Any tasks that exit after this point will set
|
|
|
|
|
// TRC_NEED_QS_CHECKED in ->trc_reader_special.b.need_qs.
|
rcu-tasks: Add an RCU Tasks Trace to simplify protection of tracing hooks
Because RCU does not watch exception early-entry/late-exit, idle-loop,
or CPU-hotplug execution, protection of tracing and BPF operations is
needlessly complicated. This commit therefore adds a variant of
Tasks RCU that:
o Has explicit read-side markers to allow finite grace periods in
the face of in-kernel loops for PREEMPT=n builds. These markers
are rcu_read_lock_trace() and rcu_read_unlock_trace().
o Protects code in the idle loop, exception entry/exit, and
CPU-hotplug code paths. In this respect, RCU-tasks trace is
similar to SRCU, but with lighter-weight readers.
o Avoids expensive read-side instruction, having overhead similar
to that of Preemptible RCU.
There are of course downsides:
o The grace-period code can send IPIs to CPUs, even when those
CPUs are in the idle loop or in nohz_full userspace. This is
mitigated by later commits.
o It is necessary to scan the full tasklist, much as for Tasks RCU.
o There is a single callback queue guarded by a single lock,
again, much as for Tasks RCU. However, those early use cases
that request multiple grace periods in quick succession are
expected to do so from a single task, which makes the single
lock almost irrelevant. If needed, multiple callback queues
can be provided using any number of schemes.
Perhaps most important, this variant of RCU does not affect the vanilla
flavors, rcu_preempt and rcu_sched. The fact that RCU Tasks Trace
readers can operate from idle, offline, and exception entry/exit in no
way enables rcu_preempt and rcu_sched readers to do so.
The memory ordering was outlined here:
https://lore.kernel.org/lkml/20200319034030.GX3199@paulmck-ThinkPad-P72/
This effort benefited greatly from off-list discussions of BPF
requirements with Alexei Starovoitov and Andrii Nakryiko. At least
some of the on-list discussions are captured in the Link: tags below.
In addition, KCSAN was quite helpful in finding some early bugs.
Link: https://lore.kernel.org/lkml/20200219150744.428764577@infradead.org/
Link: https://lore.kernel.org/lkml/87mu8p797b.fsf@nanos.tec.linutronix.de/
Link: https://lore.kernel.org/lkml/20200225221305.605144982@linutronix.de/
Cc: Alexei Starovoitov <alexei.starovoitov@gmail.com>
Cc: Andrii Nakryiko <andriin@fb.com>
[ paulmck: Apply feedback from Steve Rostedt and Joel Fernandes. ]
[ paulmck: Decrement trc_n_readers_need_end upon IPI failure. ]
[ paulmck: Fix locking issue reported by rcutorture. ]
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
2020-03-09 19:56:53 -07:00
|
|
|
}
|
|
|
|
|
|
2021-11-09 04:22:14 -07:00
|
|
|
/* Communicate task state back to the RCU tasks trace stall warning request. */
|
|
|
|
|
struct trc_stall_chk_rdr {
|
|
|
|
|
int nesting;
|
|
|
|
|
int ipi_to_cpu;
|
|
|
|
|
u8 needqs;
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
|
static int trc_check_slow_task(struct task_struct *t, void *arg)
|
|
|
|
|
{
|
|
|
|
|
struct trc_stall_chk_rdr *trc_rdrp = arg;
|
|
|
|
|
|
2022-05-25 09:49:26 -07:00
|
|
|
if (task_curr(t) && cpu_online(task_cpu(t)))
|
2021-11-09 04:22:14 -07:00
|
|
|
return false; // It is running, so decline to inspect it.
|
|
|
|
|
trc_rdrp->nesting = READ_ONCE(t->trc_reader_nesting);
|
|
|
|
|
trc_rdrp->ipi_to_cpu = READ_ONCE(t->trc_ipi_to_cpu);
|
2022-05-23 20:50:11 -07:00
|
|
|
trc_rdrp->needqs = rcu_ld_need_qs(t);
|
2021-11-09 04:22:14 -07:00
|
|
|
return true;
|
|
|
|
|
}
|
|
|
|
|
|
2020-03-10 12:13:53 -07:00
|
|
|
/* Show the state of a task stalling the current RCU tasks trace GP. */
|
|
|
|
|
static void show_stalled_task_trace(struct task_struct *t, bool *firstreport)
|
|
|
|
|
{
|
|
|
|
|
int cpu;
|
2021-11-09 04:22:14 -07:00
|
|
|
struct trc_stall_chk_rdr trc_rdr;
|
|
|
|
|
bool is_idle_tsk = is_idle_task(t);
|
2020-03-10 12:13:53 -07:00
|
|
|
|
|
|
|
|
if (*firstreport) {
|
|
|
|
|
pr_err("INFO: rcu_tasks_trace detected stalls on tasks:\n");
|
|
|
|
|
*firstreport = false;
|
|
|
|
|
}
|
|
|
|
|
cpu = task_cpu(t);
|
2021-11-09 04:22:14 -07:00
|
|
|
if (!task_call_func(t, trc_check_slow_task, &trc_rdr))
|
2022-05-24 16:05:15 -07:00
|
|
|
pr_alert("P%d: %c%c\n",
|
2021-11-09 04:22:14 -07:00
|
|
|
t->pid,
|
2022-05-24 16:05:15 -07:00
|
|
|
".I"[t->trc_ipi_to_cpu >= 0],
|
2021-11-09 04:22:14 -07:00
|
|
|
".i"[is_idle_tsk]);
|
|
|
|
|
else
|
2022-05-17 16:47:40 -07:00
|
|
|
pr_alert("P%d: %c%c%c%c nesting: %d%c%c cpu: %d%s\n",
|
2021-11-09 04:22:14 -07:00
|
|
|
t->pid,
|
|
|
|
|
".I"[trc_rdr.ipi_to_cpu >= 0],
|
|
|
|
|
".i"[is_idle_tsk],
|
|
|
|
|
".N"[cpu >= 0 && tick_nohz_full_cpu(cpu)],
|
2022-05-17 16:47:40 -07:00
|
|
|
".B"[!!data_race(t->trc_reader_special.b.blocked)],
|
2021-11-09 04:22:14 -07:00
|
|
|
trc_rdr.nesting,
|
2022-05-25 09:20:59 -07:00
|
|
|
" !CN"[trc_rdr.needqs & 0x3],
|
|
|
|
|
" ?"[trc_rdr.needqs > 0x3],
|
2022-05-24 20:36:08 -07:00
|
|
|
cpu, cpu_online(cpu) ? "" : "(offline)");
|
2020-03-10 12:13:53 -07:00
|
|
|
sched_show_task(t);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/* List stalled IPIs for RCU tasks trace. */
|
|
|
|
|
static void show_stalled_ipi_trace(void)
|
|
|
|
|
{
|
|
|
|
|
int cpu;
|
|
|
|
|
|
|
|
|
|
for_each_possible_cpu(cpu)
|
|
|
|
|
if (per_cpu(trc_ipi_to_cpu, cpu))
|
|
|
|
|
pr_alert("\tIPI outstanding to CPU %d\n", cpu);
|
|
|
|
|
}
|
|
|
|
|
|
rcu-tasks: Add an RCU Tasks Trace to simplify protection of tracing hooks
Because RCU does not watch exception early-entry/late-exit, idle-loop,
or CPU-hotplug execution, protection of tracing and BPF operations is
needlessly complicated. This commit therefore adds a variant of
Tasks RCU that:
o Has explicit read-side markers to allow finite grace periods in
the face of in-kernel loops for PREEMPT=n builds. These markers
are rcu_read_lock_trace() and rcu_read_unlock_trace().
o Protects code in the idle loop, exception entry/exit, and
CPU-hotplug code paths. In this respect, RCU-tasks trace is
similar to SRCU, but with lighter-weight readers.
o Avoids expensive read-side instruction, having overhead similar
to that of Preemptible RCU.
There are of course downsides:
o The grace-period code can send IPIs to CPUs, even when those
CPUs are in the idle loop or in nohz_full userspace. This is
mitigated by later commits.
o It is necessary to scan the full tasklist, much as for Tasks RCU.
o There is a single callback queue guarded by a single lock,
again, much as for Tasks RCU. However, those early use cases
that request multiple grace periods in quick succession are
expected to do so from a single task, which makes the single
lock almost irrelevant. If needed, multiple callback queues
can be provided using any number of schemes.
Perhaps most important, this variant of RCU does not affect the vanilla
flavors, rcu_preempt and rcu_sched. The fact that RCU Tasks Trace
readers can operate from idle, offline, and exception entry/exit in no
way enables rcu_preempt and rcu_sched readers to do so.
The memory ordering was outlined here:
https://lore.kernel.org/lkml/20200319034030.GX3199@paulmck-ThinkPad-P72/
This effort benefited greatly from off-list discussions of BPF
requirements with Alexei Starovoitov and Andrii Nakryiko. At least
some of the on-list discussions are captured in the Link: tags below.
In addition, KCSAN was quite helpful in finding some early bugs.
Link: https://lore.kernel.org/lkml/20200219150744.428764577@infradead.org/
Link: https://lore.kernel.org/lkml/87mu8p797b.fsf@nanos.tec.linutronix.de/
Link: https://lore.kernel.org/lkml/20200225221305.605144982@linutronix.de/
Cc: Alexei Starovoitov <alexei.starovoitov@gmail.com>
Cc: Andrii Nakryiko <andriin@fb.com>
[ paulmck: Apply feedback from Steve Rostedt and Joel Fernandes. ]
[ paulmck: Decrement trc_n_readers_need_end upon IPI failure. ]
[ paulmck: Fix locking issue reported by rcutorture. ]
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
2020-03-09 19:56:53 -07:00
|
|
|
/* Do one scan of the holdout list. */
|
|
|
|
|
static void check_all_holdout_tasks_trace(struct list_head *hop,
|
2020-03-10 12:13:53 -07:00
|
|
|
bool needreport, bool *firstreport)
|
rcu-tasks: Add an RCU Tasks Trace to simplify protection of tracing hooks
Because RCU does not watch exception early-entry/late-exit, idle-loop,
or CPU-hotplug execution, protection of tracing and BPF operations is
needlessly complicated. This commit therefore adds a variant of
Tasks RCU that:
o Has explicit read-side markers to allow finite grace periods in
the face of in-kernel loops for PREEMPT=n builds. These markers
are rcu_read_lock_trace() and rcu_read_unlock_trace().
o Protects code in the idle loop, exception entry/exit, and
CPU-hotplug code paths. In this respect, RCU-tasks trace is
similar to SRCU, but with lighter-weight readers.
o Avoids expensive read-side instruction, having overhead similar
to that of Preemptible RCU.
There are of course downsides:
o The grace-period code can send IPIs to CPUs, even when those
CPUs are in the idle loop or in nohz_full userspace. This is
mitigated by later commits.
o It is necessary to scan the full tasklist, much as for Tasks RCU.
o There is a single callback queue guarded by a single lock,
again, much as for Tasks RCU. However, those early use cases
that request multiple grace periods in quick succession are
expected to do so from a single task, which makes the single
lock almost irrelevant. If needed, multiple callback queues
can be provided using any number of schemes.
Perhaps most important, this variant of RCU does not affect the vanilla
flavors, rcu_preempt and rcu_sched. The fact that RCU Tasks Trace
readers can operate from idle, offline, and exception entry/exit in no
way enables rcu_preempt and rcu_sched readers to do so.
The memory ordering was outlined here:
https://lore.kernel.org/lkml/20200319034030.GX3199@paulmck-ThinkPad-P72/
This effort benefited greatly from off-list discussions of BPF
requirements with Alexei Starovoitov and Andrii Nakryiko. At least
some of the on-list discussions are captured in the Link: tags below.
In addition, KCSAN was quite helpful in finding some early bugs.
Link: https://lore.kernel.org/lkml/20200219150744.428764577@infradead.org/
Link: https://lore.kernel.org/lkml/87mu8p797b.fsf@nanos.tec.linutronix.de/
Link: https://lore.kernel.org/lkml/20200225221305.605144982@linutronix.de/
Cc: Alexei Starovoitov <alexei.starovoitov@gmail.com>
Cc: Andrii Nakryiko <andriin@fb.com>
[ paulmck: Apply feedback from Steve Rostedt and Joel Fernandes. ]
[ paulmck: Decrement trc_n_readers_need_end upon IPI failure. ]
[ paulmck: Fix locking issue reported by rcutorture. ]
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
2020-03-09 19:56:53 -07:00
|
|
|
{
|
|
|
|
|
struct task_struct *g, *t;
|
|
|
|
|
|
2022-06-06 21:30:38 -07:00
|
|
|
// Disable CPU hotplug across the holdout list scan for IPIs.
|
2020-03-22 10:10:07 -07:00
|
|
|
cpus_read_lock();
|
|
|
|
|
|
rcu-tasks: Add an RCU Tasks Trace to simplify protection of tracing hooks
Because RCU does not watch exception early-entry/late-exit, idle-loop,
or CPU-hotplug execution, protection of tracing and BPF operations is
needlessly complicated. This commit therefore adds a variant of
Tasks RCU that:
o Has explicit read-side markers to allow finite grace periods in
the face of in-kernel loops for PREEMPT=n builds. These markers
are rcu_read_lock_trace() and rcu_read_unlock_trace().
o Protects code in the idle loop, exception entry/exit, and
CPU-hotplug code paths. In this respect, RCU-tasks trace is
similar to SRCU, but with lighter-weight readers.
o Avoids expensive read-side instruction, having overhead similar
to that of Preemptible RCU.
There are of course downsides:
o The grace-period code can send IPIs to CPUs, even when those
CPUs are in the idle loop or in nohz_full userspace. This is
mitigated by later commits.
o It is necessary to scan the full tasklist, much as for Tasks RCU.
o There is a single callback queue guarded by a single lock,
again, much as for Tasks RCU. However, those early use cases
that request multiple grace periods in quick succession are
expected to do so from a single task, which makes the single
lock almost irrelevant. If needed, multiple callback queues
can be provided using any number of schemes.
Perhaps most important, this variant of RCU does not affect the vanilla
flavors, rcu_preempt and rcu_sched. The fact that RCU Tasks Trace
readers can operate from idle, offline, and exception entry/exit in no
way enables rcu_preempt and rcu_sched readers to do so.
The memory ordering was outlined here:
https://lore.kernel.org/lkml/20200319034030.GX3199@paulmck-ThinkPad-P72/
This effort benefited greatly from off-list discussions of BPF
requirements with Alexei Starovoitov and Andrii Nakryiko. At least
some of the on-list discussions are captured in the Link: tags below.
In addition, KCSAN was quite helpful in finding some early bugs.
Link: https://lore.kernel.org/lkml/20200219150744.428764577@infradead.org/
Link: https://lore.kernel.org/lkml/87mu8p797b.fsf@nanos.tec.linutronix.de/
Link: https://lore.kernel.org/lkml/20200225221305.605144982@linutronix.de/
Cc: Alexei Starovoitov <alexei.starovoitov@gmail.com>
Cc: Andrii Nakryiko <andriin@fb.com>
[ paulmck: Apply feedback from Steve Rostedt and Joel Fernandes. ]
[ paulmck: Decrement trc_n_readers_need_end upon IPI failure. ]
[ paulmck: Fix locking issue reported by rcutorture. ]
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
2020-03-09 19:56:53 -07:00
|
|
|
list_for_each_entry_safe(t, g, hop, trc_holdout_list) {
|
|
|
|
|
// If safe and needed, try to check the current task.
|
|
|
|
|
if (READ_ONCE(t->trc_ipi_to_cpu) == -1 &&
|
2022-05-23 20:50:11 -07:00
|
|
|
!(rcu_ld_need_qs(t) & TRC_NEED_QS_CHECKED))
|
rcu-tasks: Add an RCU Tasks Trace to simplify protection of tracing hooks
Because RCU does not watch exception early-entry/late-exit, idle-loop,
or CPU-hotplug execution, protection of tracing and BPF operations is
needlessly complicated. This commit therefore adds a variant of
Tasks RCU that:
o Has explicit read-side markers to allow finite grace periods in
the face of in-kernel loops for PREEMPT=n builds. These markers
are rcu_read_lock_trace() and rcu_read_unlock_trace().
o Protects code in the idle loop, exception entry/exit, and
CPU-hotplug code paths. In this respect, RCU-tasks trace is
similar to SRCU, but with lighter-weight readers.
o Avoids expensive read-side instruction, having overhead similar
to that of Preemptible RCU.
There are of course downsides:
o The grace-period code can send IPIs to CPUs, even when those
CPUs are in the idle loop or in nohz_full userspace. This is
mitigated by later commits.
o It is necessary to scan the full tasklist, much as for Tasks RCU.
o There is a single callback queue guarded by a single lock,
again, much as for Tasks RCU. However, those early use cases
that request multiple grace periods in quick succession are
expected to do so from a single task, which makes the single
lock almost irrelevant. If needed, multiple callback queues
can be provided using any number of schemes.
Perhaps most important, this variant of RCU does not affect the vanilla
flavors, rcu_preempt and rcu_sched. The fact that RCU Tasks Trace
readers can operate from idle, offline, and exception entry/exit in no
way enables rcu_preempt and rcu_sched readers to do so.
The memory ordering was outlined here:
https://lore.kernel.org/lkml/20200319034030.GX3199@paulmck-ThinkPad-P72/
This effort benefited greatly from off-list discussions of BPF
requirements with Alexei Starovoitov and Andrii Nakryiko. At least
some of the on-list discussions are captured in the Link: tags below.
In addition, KCSAN was quite helpful in finding some early bugs.
Link: https://lore.kernel.org/lkml/20200219150744.428764577@infradead.org/
Link: https://lore.kernel.org/lkml/87mu8p797b.fsf@nanos.tec.linutronix.de/
Link: https://lore.kernel.org/lkml/20200225221305.605144982@linutronix.de/
Cc: Alexei Starovoitov <alexei.starovoitov@gmail.com>
Cc: Andrii Nakryiko <andriin@fb.com>
[ paulmck: Apply feedback from Steve Rostedt and Joel Fernandes. ]
[ paulmck: Decrement trc_n_readers_need_end upon IPI failure. ]
[ paulmck: Fix locking issue reported by rcutorture. ]
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
2020-03-09 19:56:53 -07:00
|
|
|
trc_wait_for_one_reader(t, hop);
|
|
|
|
|
|
|
|
|
|
// If check succeeded, remove this task from the list.
|
2021-09-18 20:40:48 -07:00
|
|
|
if (smp_load_acquire(&t->trc_ipi_to_cpu) == -1 &&
|
2022-05-23 20:50:11 -07:00
|
|
|
rcu_ld_need_qs(t) == TRC_NEED_QS_CHECKED)
|
rcu-tasks: Add an RCU Tasks Trace to simplify protection of tracing hooks
Because RCU does not watch exception early-entry/late-exit, idle-loop,
or CPU-hotplug execution, protection of tracing and BPF operations is
needlessly complicated. This commit therefore adds a variant of
Tasks RCU that:
o Has explicit read-side markers to allow finite grace periods in
the face of in-kernel loops for PREEMPT=n builds. These markers
are rcu_read_lock_trace() and rcu_read_unlock_trace().
o Protects code in the idle loop, exception entry/exit, and
CPU-hotplug code paths. In this respect, RCU-tasks trace is
similar to SRCU, but with lighter-weight readers.
o Avoids expensive read-side instruction, having overhead similar
to that of Preemptible RCU.
There are of course downsides:
o The grace-period code can send IPIs to CPUs, even when those
CPUs are in the idle loop or in nohz_full userspace. This is
mitigated by later commits.
o It is necessary to scan the full tasklist, much as for Tasks RCU.
o There is a single callback queue guarded by a single lock,
again, much as for Tasks RCU. However, those early use cases
that request multiple grace periods in quick succession are
expected to do so from a single task, which makes the single
lock almost irrelevant. If needed, multiple callback queues
can be provided using any number of schemes.
Perhaps most important, this variant of RCU does not affect the vanilla
flavors, rcu_preempt and rcu_sched. The fact that RCU Tasks Trace
readers can operate from idle, offline, and exception entry/exit in no
way enables rcu_preempt and rcu_sched readers to do so.
The memory ordering was outlined here:
https://lore.kernel.org/lkml/20200319034030.GX3199@paulmck-ThinkPad-P72/
This effort benefited greatly from off-list discussions of BPF
requirements with Alexei Starovoitov and Andrii Nakryiko. At least
some of the on-list discussions are captured in the Link: tags below.
In addition, KCSAN was quite helpful in finding some early bugs.
Link: https://lore.kernel.org/lkml/20200219150744.428764577@infradead.org/
Link: https://lore.kernel.org/lkml/87mu8p797b.fsf@nanos.tec.linutronix.de/
Link: https://lore.kernel.org/lkml/20200225221305.605144982@linutronix.de/
Cc: Alexei Starovoitov <alexei.starovoitov@gmail.com>
Cc: Andrii Nakryiko <andriin@fb.com>
[ paulmck: Apply feedback from Steve Rostedt and Joel Fernandes. ]
[ paulmck: Decrement trc_n_readers_need_end upon IPI failure. ]
[ paulmck: Fix locking issue reported by rcutorture. ]
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
2020-03-09 19:56:53 -07:00
|
|
|
trc_del_holdout(t);
|
2020-03-10 12:13:53 -07:00
|
|
|
else if (needreport)
|
|
|
|
|
show_stalled_task_trace(t, firstreport);
|
2022-07-18 10:57:26 -07:00
|
|
|
cond_resched_tasks_rcu_qs();
|
2020-03-10 12:13:53 -07:00
|
|
|
}
|
2020-03-22 10:10:07 -07:00
|
|
|
|
|
|
|
|
// Re-enable CPU hotplug now that the holdout list scan has completed.
|
|
|
|
|
cpus_read_unlock();
|
|
|
|
|
|
2020-03-10 12:13:53 -07:00
|
|
|
if (needreport) {
|
2021-08-18 00:28:40 -07:00
|
|
|
if (*firstreport)
|
2020-03-10 12:13:53 -07:00
|
|
|
pr_err("INFO: rcu_tasks_trace detected stalls? (Late IPI?)\n");
|
|
|
|
|
show_stalled_ipi_trace();
|
rcu-tasks: Add an RCU Tasks Trace to simplify protection of tracing hooks
Because RCU does not watch exception early-entry/late-exit, idle-loop,
or CPU-hotplug execution, protection of tracing and BPF operations is
needlessly complicated. This commit therefore adds a variant of
Tasks RCU that:
o Has explicit read-side markers to allow finite grace periods in
the face of in-kernel loops for PREEMPT=n builds. These markers
are rcu_read_lock_trace() and rcu_read_unlock_trace().
o Protects code in the idle loop, exception entry/exit, and
CPU-hotplug code paths. In this respect, RCU-tasks trace is
similar to SRCU, but with lighter-weight readers.
o Avoids expensive read-side instruction, having overhead similar
to that of Preemptible RCU.
There are of course downsides:
o The grace-period code can send IPIs to CPUs, even when those
CPUs are in the idle loop or in nohz_full userspace. This is
mitigated by later commits.
o It is necessary to scan the full tasklist, much as for Tasks RCU.
o There is a single callback queue guarded by a single lock,
again, much as for Tasks RCU. However, those early use cases
that request multiple grace periods in quick succession are
expected to do so from a single task, which makes the single
lock almost irrelevant. If needed, multiple callback queues
can be provided using any number of schemes.
Perhaps most important, this variant of RCU does not affect the vanilla
flavors, rcu_preempt and rcu_sched. The fact that RCU Tasks Trace
readers can operate from idle, offline, and exception entry/exit in no
way enables rcu_preempt and rcu_sched readers to do so.
The memory ordering was outlined here:
https://lore.kernel.org/lkml/20200319034030.GX3199@paulmck-ThinkPad-P72/
This effort benefited greatly from off-list discussions of BPF
requirements with Alexei Starovoitov and Andrii Nakryiko. At least
some of the on-list discussions are captured in the Link: tags below.
In addition, KCSAN was quite helpful in finding some early bugs.
Link: https://lore.kernel.org/lkml/20200219150744.428764577@infradead.org/
Link: https://lore.kernel.org/lkml/87mu8p797b.fsf@nanos.tec.linutronix.de/
Link: https://lore.kernel.org/lkml/20200225221305.605144982@linutronix.de/
Cc: Alexei Starovoitov <alexei.starovoitov@gmail.com>
Cc: Andrii Nakryiko <andriin@fb.com>
[ paulmck: Apply feedback from Steve Rostedt and Joel Fernandes. ]
[ paulmck: Decrement trc_n_readers_need_end upon IPI failure. ]
[ paulmck: Fix locking issue reported by rcutorture. ]
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
2020-03-09 19:56:53 -07:00
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2021-07-30 12:17:59 -07:00
|
|
|
static void rcu_tasks_trace_empty_fn(void *unused)
|
|
|
|
|
{
|
|
|
|
|
}
|
|
|
|
|
|
rcu-tasks: Add an RCU Tasks Trace to simplify protection of tracing hooks
Because RCU does not watch exception early-entry/late-exit, idle-loop,
or CPU-hotplug execution, protection of tracing and BPF operations is
needlessly complicated. This commit therefore adds a variant of
Tasks RCU that:
o Has explicit read-side markers to allow finite grace periods in
the face of in-kernel loops for PREEMPT=n builds. These markers
are rcu_read_lock_trace() and rcu_read_unlock_trace().
o Protects code in the idle loop, exception entry/exit, and
CPU-hotplug code paths. In this respect, RCU-tasks trace is
similar to SRCU, but with lighter-weight readers.
o Avoids expensive read-side instruction, having overhead similar
to that of Preemptible RCU.
There are of course downsides:
o The grace-period code can send IPIs to CPUs, even when those
CPUs are in the idle loop or in nohz_full userspace. This is
mitigated by later commits.
o It is necessary to scan the full tasklist, much as for Tasks RCU.
o There is a single callback queue guarded by a single lock,
again, much as for Tasks RCU. However, those early use cases
that request multiple grace periods in quick succession are
expected to do so from a single task, which makes the single
lock almost irrelevant. If needed, multiple callback queues
can be provided using any number of schemes.
Perhaps most important, this variant of RCU does not affect the vanilla
flavors, rcu_preempt and rcu_sched. The fact that RCU Tasks Trace
readers can operate from idle, offline, and exception entry/exit in no
way enables rcu_preempt and rcu_sched readers to do so.
The memory ordering was outlined here:
https://lore.kernel.org/lkml/20200319034030.GX3199@paulmck-ThinkPad-P72/
This effort benefited greatly from off-list discussions of BPF
requirements with Alexei Starovoitov and Andrii Nakryiko. At least
some of the on-list discussions are captured in the Link: tags below.
In addition, KCSAN was quite helpful in finding some early bugs.
Link: https://lore.kernel.org/lkml/20200219150744.428764577@infradead.org/
Link: https://lore.kernel.org/lkml/87mu8p797b.fsf@nanos.tec.linutronix.de/
Link: https://lore.kernel.org/lkml/20200225221305.605144982@linutronix.de/
Cc: Alexei Starovoitov <alexei.starovoitov@gmail.com>
Cc: Andrii Nakryiko <andriin@fb.com>
[ paulmck: Apply feedback from Steve Rostedt and Joel Fernandes. ]
[ paulmck: Decrement trc_n_readers_need_end upon IPI failure. ]
[ paulmck: Fix locking issue reported by rcutorture. ]
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
2020-03-09 19:56:53 -07:00
|
|
|
/* Wait for grace period to complete and provide ordering. */
|
2020-03-16 12:13:33 -07:00
|
|
|
static void rcu_tasks_trace_postgp(struct rcu_tasks *rtp)
|
rcu-tasks: Add an RCU Tasks Trace to simplify protection of tracing hooks
Because RCU does not watch exception early-entry/late-exit, idle-loop,
or CPU-hotplug execution, protection of tracing and BPF operations is
needlessly complicated. This commit therefore adds a variant of
Tasks RCU that:
o Has explicit read-side markers to allow finite grace periods in
the face of in-kernel loops for PREEMPT=n builds. These markers
are rcu_read_lock_trace() and rcu_read_unlock_trace().
o Protects code in the idle loop, exception entry/exit, and
CPU-hotplug code paths. In this respect, RCU-tasks trace is
similar to SRCU, but with lighter-weight readers.
o Avoids expensive read-side instruction, having overhead similar
to that of Preemptible RCU.
There are of course downsides:
o The grace-period code can send IPIs to CPUs, even when those
CPUs are in the idle loop or in nohz_full userspace. This is
mitigated by later commits.
o It is necessary to scan the full tasklist, much as for Tasks RCU.
o There is a single callback queue guarded by a single lock,
again, much as for Tasks RCU. However, those early use cases
that request multiple grace periods in quick succession are
expected to do so from a single task, which makes the single
lock almost irrelevant. If needed, multiple callback queues
can be provided using any number of schemes.
Perhaps most important, this variant of RCU does not affect the vanilla
flavors, rcu_preempt and rcu_sched. The fact that RCU Tasks Trace
readers can operate from idle, offline, and exception entry/exit in no
way enables rcu_preempt and rcu_sched readers to do so.
The memory ordering was outlined here:
https://lore.kernel.org/lkml/20200319034030.GX3199@paulmck-ThinkPad-P72/
This effort benefited greatly from off-list discussions of BPF
requirements with Alexei Starovoitov and Andrii Nakryiko. At least
some of the on-list discussions are captured in the Link: tags below.
In addition, KCSAN was quite helpful in finding some early bugs.
Link: https://lore.kernel.org/lkml/20200219150744.428764577@infradead.org/
Link: https://lore.kernel.org/lkml/87mu8p797b.fsf@nanos.tec.linutronix.de/
Link: https://lore.kernel.org/lkml/20200225221305.605144982@linutronix.de/
Cc: Alexei Starovoitov <alexei.starovoitov@gmail.com>
Cc: Andrii Nakryiko <andriin@fb.com>
[ paulmck: Apply feedback from Steve Rostedt and Joel Fernandes. ]
[ paulmck: Decrement trc_n_readers_need_end upon IPI failure. ]
[ paulmck: Fix locking issue reported by rcutorture. ]
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
2020-03-09 19:56:53 -07:00
|
|
|
{
|
2021-07-30 12:17:59 -07:00
|
|
|
int cpu;
|
2020-03-10 12:13:53 -07:00
|
|
|
|
2021-07-30 12:17:59 -07:00
|
|
|
// Wait for any lingering IPI handlers to complete. Note that
|
|
|
|
|
// if a CPU has gone offline or transitioned to userspace in the
|
|
|
|
|
// meantime, all IPI handlers should have been drained beforehand.
|
|
|
|
|
// Yes, this assumes that CPUs process IPIs in order. If that ever
|
|
|
|
|
// changes, there will need to be a recheck and/or timed wait.
|
|
|
|
|
for_each_online_cpu(cpu)
|
2021-09-18 20:40:48 -07:00
|
|
|
if (WARN_ON_ONCE(smp_load_acquire(per_cpu_ptr(&trc_ipi_to_cpu, cpu))))
|
2021-07-30 12:17:59 -07:00
|
|
|
smp_call_function_single(cpu, rcu_tasks_trace_empty_fn, NULL, 1);
|
|
|
|
|
|
rcu-tasks: Add an RCU Tasks Trace to simplify protection of tracing hooks
Because RCU does not watch exception early-entry/late-exit, idle-loop,
or CPU-hotplug execution, protection of tracing and BPF operations is
needlessly complicated. This commit therefore adds a variant of
Tasks RCU that:
o Has explicit read-side markers to allow finite grace periods in
the face of in-kernel loops for PREEMPT=n builds. These markers
are rcu_read_lock_trace() and rcu_read_unlock_trace().
o Protects code in the idle loop, exception entry/exit, and
CPU-hotplug code paths. In this respect, RCU-tasks trace is
similar to SRCU, but with lighter-weight readers.
o Avoids expensive read-side instruction, having overhead similar
to that of Preemptible RCU.
There are of course downsides:
o The grace-period code can send IPIs to CPUs, even when those
CPUs are in the idle loop or in nohz_full userspace. This is
mitigated by later commits.
o It is necessary to scan the full tasklist, much as for Tasks RCU.
o There is a single callback queue guarded by a single lock,
again, much as for Tasks RCU. However, those early use cases
that request multiple grace periods in quick succession are
expected to do so from a single task, which makes the single
lock almost irrelevant. If needed, multiple callback queues
can be provided using any number of schemes.
Perhaps most important, this variant of RCU does not affect the vanilla
flavors, rcu_preempt and rcu_sched. The fact that RCU Tasks Trace
readers can operate from idle, offline, and exception entry/exit in no
way enables rcu_preempt and rcu_sched readers to do so.
The memory ordering was outlined here:
https://lore.kernel.org/lkml/20200319034030.GX3199@paulmck-ThinkPad-P72/
This effort benefited greatly from off-list discussions of BPF
requirements with Alexei Starovoitov and Andrii Nakryiko. At least
some of the on-list discussions are captured in the Link: tags below.
In addition, KCSAN was quite helpful in finding some early bugs.
Link: https://lore.kernel.org/lkml/20200219150744.428764577@infradead.org/
Link: https://lore.kernel.org/lkml/87mu8p797b.fsf@nanos.tec.linutronix.de/
Link: https://lore.kernel.org/lkml/20200225221305.605144982@linutronix.de/
Cc: Alexei Starovoitov <alexei.starovoitov@gmail.com>
Cc: Andrii Nakryiko <andriin@fb.com>
[ paulmck: Apply feedback from Steve Rostedt and Joel Fernandes. ]
[ paulmck: Decrement trc_n_readers_need_end upon IPI failure. ]
[ paulmck: Fix locking issue reported by rcutorture. ]
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
2020-03-09 19:56:53 -07:00
|
|
|
smp_mb(); // Caller's code must be ordered after wakeup.
|
2020-03-16 20:38:29 -07:00
|
|
|
// Pairs with pretty much every ordering primitive.
|
rcu-tasks: Add an RCU Tasks Trace to simplify protection of tracing hooks
Because RCU does not watch exception early-entry/late-exit, idle-loop,
or CPU-hotplug execution, protection of tracing and BPF operations is
needlessly complicated. This commit therefore adds a variant of
Tasks RCU that:
o Has explicit read-side markers to allow finite grace periods in
the face of in-kernel loops for PREEMPT=n builds. These markers
are rcu_read_lock_trace() and rcu_read_unlock_trace().
o Protects code in the idle loop, exception entry/exit, and
CPU-hotplug code paths. In this respect, RCU-tasks trace is
similar to SRCU, but with lighter-weight readers.
o Avoids expensive read-side instruction, having overhead similar
to that of Preemptible RCU.
There are of course downsides:
o The grace-period code can send IPIs to CPUs, even when those
CPUs are in the idle loop or in nohz_full userspace. This is
mitigated by later commits.
o It is necessary to scan the full tasklist, much as for Tasks RCU.
o There is a single callback queue guarded by a single lock,
again, much as for Tasks RCU. However, those early use cases
that request multiple grace periods in quick succession are
expected to do so from a single task, which makes the single
lock almost irrelevant. If needed, multiple callback queues
can be provided using any number of schemes.
Perhaps most important, this variant of RCU does not affect the vanilla
flavors, rcu_preempt and rcu_sched. The fact that RCU Tasks Trace
readers can operate from idle, offline, and exception entry/exit in no
way enables rcu_preempt and rcu_sched readers to do so.
The memory ordering was outlined here:
https://lore.kernel.org/lkml/20200319034030.GX3199@paulmck-ThinkPad-P72/
This effort benefited greatly from off-list discussions of BPF
requirements with Alexei Starovoitov and Andrii Nakryiko. At least
some of the on-list discussions are captured in the Link: tags below.
In addition, KCSAN was quite helpful in finding some early bugs.
Link: https://lore.kernel.org/lkml/20200219150744.428764577@infradead.org/
Link: https://lore.kernel.org/lkml/87mu8p797b.fsf@nanos.tec.linutronix.de/
Link: https://lore.kernel.org/lkml/20200225221305.605144982@linutronix.de/
Cc: Alexei Starovoitov <alexei.starovoitov@gmail.com>
Cc: Andrii Nakryiko <andriin@fb.com>
[ paulmck: Apply feedback from Steve Rostedt and Joel Fernandes. ]
[ paulmck: Decrement trc_n_readers_need_end upon IPI failure. ]
[ paulmck: Fix locking issue reported by rcutorture. ]
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
2020-03-09 19:56:53 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/* Report any needed quiescent state for this exiting task. */
|
2020-04-05 20:49:13 -07:00
|
|
|
static void exit_tasks_rcu_finish_trace(struct task_struct *t)
|
rcu-tasks: Add an RCU Tasks Trace to simplify protection of tracing hooks
Because RCU does not watch exception early-entry/late-exit, idle-loop,
or CPU-hotplug execution, protection of tracing and BPF operations is
needlessly complicated. This commit therefore adds a variant of
Tasks RCU that:
o Has explicit read-side markers to allow finite grace periods in
the face of in-kernel loops for PREEMPT=n builds. These markers
are rcu_read_lock_trace() and rcu_read_unlock_trace().
o Protects code in the idle loop, exception entry/exit, and
CPU-hotplug code paths. In this respect, RCU-tasks trace is
similar to SRCU, but with lighter-weight readers.
o Avoids expensive read-side instruction, having overhead similar
to that of Preemptible RCU.
There are of course downsides:
o The grace-period code can send IPIs to CPUs, even when those
CPUs are in the idle loop or in nohz_full userspace. This is
mitigated by later commits.
o It is necessary to scan the full tasklist, much as for Tasks RCU.
o There is a single callback queue guarded by a single lock,
again, much as for Tasks RCU. However, those early use cases
that request multiple grace periods in quick succession are
expected to do so from a single task, which makes the single
lock almost irrelevant. If needed, multiple callback queues
can be provided using any number of schemes.
Perhaps most important, this variant of RCU does not affect the vanilla
flavors, rcu_preempt and rcu_sched. The fact that RCU Tasks Trace
readers can operate from idle, offline, and exception entry/exit in no
way enables rcu_preempt and rcu_sched readers to do so.
The memory ordering was outlined here:
https://lore.kernel.org/lkml/20200319034030.GX3199@paulmck-ThinkPad-P72/
This effort benefited greatly from off-list discussions of BPF
requirements with Alexei Starovoitov and Andrii Nakryiko. At least
some of the on-list discussions are captured in the Link: tags below.
In addition, KCSAN was quite helpful in finding some early bugs.
Link: https://lore.kernel.org/lkml/20200219150744.428764577@infradead.org/
Link: https://lore.kernel.org/lkml/87mu8p797b.fsf@nanos.tec.linutronix.de/
Link: https://lore.kernel.org/lkml/20200225221305.605144982@linutronix.de/
Cc: Alexei Starovoitov <alexei.starovoitov@gmail.com>
Cc: Andrii Nakryiko <andriin@fb.com>
[ paulmck: Apply feedback from Steve Rostedt and Joel Fernandes. ]
[ paulmck: Decrement trc_n_readers_need_end upon IPI failure. ]
[ paulmck: Fix locking issue reported by rcutorture. ]
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
2020-03-09 19:56:53 -07:00
|
|
|
{
|
2022-05-17 11:30:32 -07:00
|
|
|
union rcu_special trs = READ_ONCE(t->trc_reader_special);
|
|
|
|
|
|
2022-05-23 20:50:11 -07:00
|
|
|
rcu_trc_cmpxchg_need_qs(t, 0, TRC_NEED_QS_CHECKED);
|
2021-05-24 12:48:18 -07:00
|
|
|
WARN_ON_ONCE(READ_ONCE(t->trc_reader_nesting));
|
2022-05-17 15:01:14 -07:00
|
|
|
if (WARN_ON_ONCE(rcu_ld_need_qs(t) & TRC_NEED_QS || trs.b.blocked))
|
2021-07-28 12:28:27 -07:00
|
|
|
rcu_read_unlock_trace_special(t);
|
2022-05-23 20:50:11 -07:00
|
|
|
else
|
|
|
|
|
WRITE_ONCE(t->trc_reader_nesting, 0);
|
rcu-tasks: Add an RCU Tasks Trace to simplify protection of tracing hooks
Because RCU does not watch exception early-entry/late-exit, idle-loop,
or CPU-hotplug execution, protection of tracing and BPF operations is
needlessly complicated. This commit therefore adds a variant of
Tasks RCU that:
o Has explicit read-side markers to allow finite grace periods in
the face of in-kernel loops for PREEMPT=n builds. These markers
are rcu_read_lock_trace() and rcu_read_unlock_trace().
o Protects code in the idle loop, exception entry/exit, and
CPU-hotplug code paths. In this respect, RCU-tasks trace is
similar to SRCU, but with lighter-weight readers.
o Avoids expensive read-side instruction, having overhead similar
to that of Preemptible RCU.
There are of course downsides:
o The grace-period code can send IPIs to CPUs, even when those
CPUs are in the idle loop or in nohz_full userspace. This is
mitigated by later commits.
o It is necessary to scan the full tasklist, much as for Tasks RCU.
o There is a single callback queue guarded by a single lock,
again, much as for Tasks RCU. However, those early use cases
that request multiple grace periods in quick succession are
expected to do so from a single task, which makes the single
lock almost irrelevant. If needed, multiple callback queues
can be provided using any number of schemes.
Perhaps most important, this variant of RCU does not affect the vanilla
flavors, rcu_preempt and rcu_sched. The fact that RCU Tasks Trace
readers can operate from idle, offline, and exception entry/exit in no
way enables rcu_preempt and rcu_sched readers to do so.
The memory ordering was outlined here:
https://lore.kernel.org/lkml/20200319034030.GX3199@paulmck-ThinkPad-P72/
This effort benefited greatly from off-list discussions of BPF
requirements with Alexei Starovoitov and Andrii Nakryiko. At least
some of the on-list discussions are captured in the Link: tags below.
In addition, KCSAN was quite helpful in finding some early bugs.
Link: https://lore.kernel.org/lkml/20200219150744.428764577@infradead.org/
Link: https://lore.kernel.org/lkml/87mu8p797b.fsf@nanos.tec.linutronix.de/
Link: https://lore.kernel.org/lkml/20200225221305.605144982@linutronix.de/
Cc: Alexei Starovoitov <alexei.starovoitov@gmail.com>
Cc: Andrii Nakryiko <andriin@fb.com>
[ paulmck: Apply feedback from Steve Rostedt and Joel Fernandes. ]
[ paulmck: Decrement trc_n_readers_need_end upon IPI failure. ]
[ paulmck: Fix locking issue reported by rcutorture. ]
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
2020-03-09 19:56:53 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* call_rcu_tasks_trace() - Queue a callback trace task-based grace period
|
|
|
|
|
* @rhp: structure to be used for queueing the RCU updates.
|
|
|
|
|
* @func: actual callback function to be invoked after the grace period
|
|
|
|
|
*
|
2021-08-25 00:10:50 -07:00
|
|
|
* The callback function will be invoked some time after a trace rcu-tasks
|
|
|
|
|
* grace period elapses, in other words after all currently executing
|
|
|
|
|
* trace rcu-tasks read-side critical sections have completed. These
|
|
|
|
|
* read-side critical sections are delimited by calls to rcu_read_lock_trace()
|
|
|
|
|
* and rcu_read_unlock_trace().
|
rcu-tasks: Add an RCU Tasks Trace to simplify protection of tracing hooks
Because RCU does not watch exception early-entry/late-exit, idle-loop,
or CPU-hotplug execution, protection of tracing and BPF operations is
needlessly complicated. This commit therefore adds a variant of
Tasks RCU that:
o Has explicit read-side markers to allow finite grace periods in
the face of in-kernel loops for PREEMPT=n builds. These markers
are rcu_read_lock_trace() and rcu_read_unlock_trace().
o Protects code in the idle loop, exception entry/exit, and
CPU-hotplug code paths. In this respect, RCU-tasks trace is
similar to SRCU, but with lighter-weight readers.
o Avoids expensive read-side instruction, having overhead similar
to that of Preemptible RCU.
There are of course downsides:
o The grace-period code can send IPIs to CPUs, even when those
CPUs are in the idle loop or in nohz_full userspace. This is
mitigated by later commits.
o It is necessary to scan the full tasklist, much as for Tasks RCU.
o There is a single callback queue guarded by a single lock,
again, much as for Tasks RCU. However, those early use cases
that request multiple grace periods in quick succession are
expected to do so from a single task, which makes the single
lock almost irrelevant. If needed, multiple callback queues
can be provided using any number of schemes.
Perhaps most important, this variant of RCU does not affect the vanilla
flavors, rcu_preempt and rcu_sched. The fact that RCU Tasks Trace
readers can operate from idle, offline, and exception entry/exit in no
way enables rcu_preempt and rcu_sched readers to do so.
The memory ordering was outlined here:
https://lore.kernel.org/lkml/20200319034030.GX3199@paulmck-ThinkPad-P72/
This effort benefited greatly from off-list discussions of BPF
requirements with Alexei Starovoitov and Andrii Nakryiko. At least
some of the on-list discussions are captured in the Link: tags below.
In addition, KCSAN was quite helpful in finding some early bugs.
Link: https://lore.kernel.org/lkml/20200219150744.428764577@infradead.org/
Link: https://lore.kernel.org/lkml/87mu8p797b.fsf@nanos.tec.linutronix.de/
Link: https://lore.kernel.org/lkml/20200225221305.605144982@linutronix.de/
Cc: Alexei Starovoitov <alexei.starovoitov@gmail.com>
Cc: Andrii Nakryiko <andriin@fb.com>
[ paulmck: Apply feedback from Steve Rostedt and Joel Fernandes. ]
[ paulmck: Decrement trc_n_readers_need_end upon IPI failure. ]
[ paulmck: Fix locking issue reported by rcutorture. ]
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
2020-03-09 19:56:53 -07:00
|
|
|
*
|
|
|
|
|
* See the description of call_rcu() for more detailed information on
|
|
|
|
|
* memory ordering guarantees.
|
|
|
|
|
*/
|
|
|
|
|
void call_rcu_tasks_trace(struct rcu_head *rhp, rcu_callback_t func)
|
|
|
|
|
{
|
|
|
|
|
call_rcu_tasks_generic(rhp, func, &rcu_tasks_trace);
|
|
|
|
|
}
|
|
|
|
|
EXPORT_SYMBOL_GPL(call_rcu_tasks_trace);
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* synchronize_rcu_tasks_trace - wait for a trace rcu-tasks grace period
|
|
|
|
|
*
|
|
|
|
|
* Control will return to the caller some time after a trace rcu-tasks
|
2020-06-12 13:11:29 -07:00
|
|
|
* grace period has elapsed, in other words after all currently executing
|
2021-08-25 00:10:50 -07:00
|
|
|
* trace rcu-tasks read-side critical sections have elapsed. These read-side
|
2020-06-12 13:11:29 -07:00
|
|
|
* critical sections are delimited by calls to rcu_read_lock_trace()
|
|
|
|
|
* and rcu_read_unlock_trace().
|
rcu-tasks: Add an RCU Tasks Trace to simplify protection of tracing hooks
Because RCU does not watch exception early-entry/late-exit, idle-loop,
or CPU-hotplug execution, protection of tracing and BPF operations is
needlessly complicated. This commit therefore adds a variant of
Tasks RCU that:
o Has explicit read-side markers to allow finite grace periods in
the face of in-kernel loops for PREEMPT=n builds. These markers
are rcu_read_lock_trace() and rcu_read_unlock_trace().
o Protects code in the idle loop, exception entry/exit, and
CPU-hotplug code paths. In this respect, RCU-tasks trace is
similar to SRCU, but with lighter-weight readers.
o Avoids expensive read-side instruction, having overhead similar
to that of Preemptible RCU.
There are of course downsides:
o The grace-period code can send IPIs to CPUs, even when those
CPUs are in the idle loop or in nohz_full userspace. This is
mitigated by later commits.
o It is necessary to scan the full tasklist, much as for Tasks RCU.
o There is a single callback queue guarded by a single lock,
again, much as for Tasks RCU. However, those early use cases
that request multiple grace periods in quick succession are
expected to do so from a single task, which makes the single
lock almost irrelevant. If needed, multiple callback queues
can be provided using any number of schemes.
Perhaps most important, this variant of RCU does not affect the vanilla
flavors, rcu_preempt and rcu_sched. The fact that RCU Tasks Trace
readers can operate from idle, offline, and exception entry/exit in no
way enables rcu_preempt and rcu_sched readers to do so.
The memory ordering was outlined here:
https://lore.kernel.org/lkml/20200319034030.GX3199@paulmck-ThinkPad-P72/
This effort benefited greatly from off-list discussions of BPF
requirements with Alexei Starovoitov and Andrii Nakryiko. At least
some of the on-list discussions are captured in the Link: tags below.
In addition, KCSAN was quite helpful in finding some early bugs.
Link: https://lore.kernel.org/lkml/20200219150744.428764577@infradead.org/
Link: https://lore.kernel.org/lkml/87mu8p797b.fsf@nanos.tec.linutronix.de/
Link: https://lore.kernel.org/lkml/20200225221305.605144982@linutronix.de/
Cc: Alexei Starovoitov <alexei.starovoitov@gmail.com>
Cc: Andrii Nakryiko <andriin@fb.com>
[ paulmck: Apply feedback from Steve Rostedt and Joel Fernandes. ]
[ paulmck: Decrement trc_n_readers_need_end upon IPI failure. ]
[ paulmck: Fix locking issue reported by rcutorture. ]
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
2020-03-09 19:56:53 -07:00
|
|
|
*
|
|
|
|
|
* This is a very specialized primitive, intended only for a few uses in
|
|
|
|
|
* tracing and other situations requiring manipulation of function preambles
|
|
|
|
|
* and profiling hooks. The synchronize_rcu_tasks_trace() function is not
|
|
|
|
|
* (yet) intended for heavy use from multiple CPUs.
|
|
|
|
|
*
|
|
|
|
|
* See the description of synchronize_rcu() for more detailed information
|
|
|
|
|
* on memory ordering guarantees.
|
|
|
|
|
*/
|
|
|
|
|
void synchronize_rcu_tasks_trace(void)
|
|
|
|
|
{
|
|
|
|
|
RCU_LOCKDEP_WARN(lock_is_held(&rcu_trace_lock_map), "Illegal synchronize_rcu_tasks_trace() in RCU Tasks Trace read-side critical section");
|
|
|
|
|
synchronize_rcu_tasks_generic(&rcu_tasks_trace);
|
|
|
|
|
}
|
|
|
|
|
EXPORT_SYMBOL_GPL(synchronize_rcu_tasks_trace);
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* rcu_barrier_tasks_trace - Wait for in-flight call_rcu_tasks_trace() callbacks.
|
|
|
|
|
*
|
|
|
|
|
* Although the current implementation is guaranteed to wait, it is not
|
|
|
|
|
* obligated to, for example, if there are no pending callbacks.
|
|
|
|
|
*/
|
|
|
|
|
void rcu_barrier_tasks_trace(void)
|
|
|
|
|
{
|
2021-11-11 15:53:43 -07:00
|
|
|
rcu_barrier_tasks_generic(&rcu_tasks_trace);
|
rcu-tasks: Add an RCU Tasks Trace to simplify protection of tracing hooks
Because RCU does not watch exception early-entry/late-exit, idle-loop,
or CPU-hotplug execution, protection of tracing and BPF operations is
needlessly complicated. This commit therefore adds a variant of
Tasks RCU that:
o Has explicit read-side markers to allow finite grace periods in
the face of in-kernel loops for PREEMPT=n builds. These markers
are rcu_read_lock_trace() and rcu_read_unlock_trace().
o Protects code in the idle loop, exception entry/exit, and
CPU-hotplug code paths. In this respect, RCU-tasks trace is
similar to SRCU, but with lighter-weight readers.
o Avoids expensive read-side instruction, having overhead similar
to that of Preemptible RCU.
There are of course downsides:
o The grace-period code can send IPIs to CPUs, even when those
CPUs are in the idle loop or in nohz_full userspace. This is
mitigated by later commits.
o It is necessary to scan the full tasklist, much as for Tasks RCU.
o There is a single callback queue guarded by a single lock,
again, much as for Tasks RCU. However, those early use cases
that request multiple grace periods in quick succession are
expected to do so from a single task, which makes the single
lock almost irrelevant. If needed, multiple callback queues
can be provided using any number of schemes.
Perhaps most important, this variant of RCU does not affect the vanilla
flavors, rcu_preempt and rcu_sched. The fact that RCU Tasks Trace
readers can operate from idle, offline, and exception entry/exit in no
way enables rcu_preempt and rcu_sched readers to do so.
The memory ordering was outlined here:
https://lore.kernel.org/lkml/20200319034030.GX3199@paulmck-ThinkPad-P72/
This effort benefited greatly from off-list discussions of BPF
requirements with Alexei Starovoitov and Andrii Nakryiko. At least
some of the on-list discussions are captured in the Link: tags below.
In addition, KCSAN was quite helpful in finding some early bugs.
Link: https://lore.kernel.org/lkml/20200219150744.428764577@infradead.org/
Link: https://lore.kernel.org/lkml/87mu8p797b.fsf@nanos.tec.linutronix.de/
Link: https://lore.kernel.org/lkml/20200225221305.605144982@linutronix.de/
Cc: Alexei Starovoitov <alexei.starovoitov@gmail.com>
Cc: Andrii Nakryiko <andriin@fb.com>
[ paulmck: Apply feedback from Steve Rostedt and Joel Fernandes. ]
[ paulmck: Decrement trc_n_readers_need_end upon IPI failure. ]
[ paulmck: Fix locking issue reported by rcutorture. ]
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
2020-03-09 19:56:53 -07:00
|
|
|
}
|
|
|
|
|
EXPORT_SYMBOL_GPL(rcu_barrier_tasks_trace);
|
|
|
|
|
|
2023-05-17 06:40:23 -07:00
|
|
|
int rcu_tasks_trace_lazy_ms = -1;
|
|
|
|
|
module_param(rcu_tasks_trace_lazy_ms, int, 0444);
|
|
|
|
|
|
rcu-tasks: Add an RCU Tasks Trace to simplify protection of tracing hooks
Because RCU does not watch exception early-entry/late-exit, idle-loop,
or CPU-hotplug execution, protection of tracing and BPF operations is
needlessly complicated. This commit therefore adds a variant of
Tasks RCU that:
o Has explicit read-side markers to allow finite grace periods in
the face of in-kernel loops for PREEMPT=n builds. These markers
are rcu_read_lock_trace() and rcu_read_unlock_trace().
o Protects code in the idle loop, exception entry/exit, and
CPU-hotplug code paths. In this respect, RCU-tasks trace is
similar to SRCU, but with lighter-weight readers.
o Avoids expensive read-side instruction, having overhead similar
to that of Preemptible RCU.
There are of course downsides:
o The grace-period code can send IPIs to CPUs, even when those
CPUs are in the idle loop or in nohz_full userspace. This is
mitigated by later commits.
o It is necessary to scan the full tasklist, much as for Tasks RCU.
o There is a single callback queue guarded by a single lock,
again, much as for Tasks RCU. However, those early use cases
that request multiple grace periods in quick succession are
expected to do so from a single task, which makes the single
lock almost irrelevant. If needed, multiple callback queues
can be provided using any number of schemes.
Perhaps most important, this variant of RCU does not affect the vanilla
flavors, rcu_preempt and rcu_sched. The fact that RCU Tasks Trace
readers can operate from idle, offline, and exception entry/exit in no
way enables rcu_preempt and rcu_sched readers to do so.
The memory ordering was outlined here:
https://lore.kernel.org/lkml/20200319034030.GX3199@paulmck-ThinkPad-P72/
This effort benefited greatly from off-list discussions of BPF
requirements with Alexei Starovoitov and Andrii Nakryiko. At least
some of the on-list discussions are captured in the Link: tags below.
In addition, KCSAN was quite helpful in finding some early bugs.
Link: https://lore.kernel.org/lkml/20200219150744.428764577@infradead.org/
Link: https://lore.kernel.org/lkml/87mu8p797b.fsf@nanos.tec.linutronix.de/
Link: https://lore.kernel.org/lkml/20200225221305.605144982@linutronix.de/
Cc: Alexei Starovoitov <alexei.starovoitov@gmail.com>
Cc: Andrii Nakryiko <andriin@fb.com>
[ paulmck: Apply feedback from Steve Rostedt and Joel Fernandes. ]
[ paulmck: Decrement trc_n_readers_need_end upon IPI failure. ]
[ paulmck: Fix locking issue reported by rcutorture. ]
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
2020-03-09 19:56:53 -07:00
|
|
|
static int __init rcu_spawn_tasks_trace_kthread(void)
|
|
|
|
|
{
|
2020-09-09 21:36:34 -07:00
|
|
|
if (IS_ENABLED(CONFIG_TASKS_TRACE_RCU_READ_MB)) {
|
2020-09-09 22:05:41 -07:00
|
|
|
rcu_tasks_trace.gp_sleep = HZ / 10;
|
2020-09-17 16:17:17 -07:00
|
|
|
rcu_tasks_trace.init_fract = HZ / 10;
|
2020-09-09 21:36:34 -07:00
|
|
|
} else {
|
2020-09-09 22:05:41 -07:00
|
|
|
rcu_tasks_trace.gp_sleep = HZ / 200;
|
|
|
|
|
if (rcu_tasks_trace.gp_sleep <= 0)
|
|
|
|
|
rcu_tasks_trace.gp_sleep = 1;
|
2020-09-17 16:17:17 -07:00
|
|
|
rcu_tasks_trace.init_fract = HZ / 200;
|
2020-09-09 21:36:34 -07:00
|
|
|
if (rcu_tasks_trace.init_fract <= 0)
|
|
|
|
|
rcu_tasks_trace.init_fract = 1;
|
|
|
|
|
}
|
2023-05-17 06:40:23 -07:00
|
|
|
if (rcu_tasks_trace_lazy_ms >= 0)
|
|
|
|
|
rcu_tasks_trace.lazy_jiffies = msecs_to_jiffies(rcu_tasks_trace_lazy_ms);
|
rcu-tasks: Add an RCU Tasks Trace to simplify protection of tracing hooks
Because RCU does not watch exception early-entry/late-exit, idle-loop,
or CPU-hotplug execution, protection of tracing and BPF operations is
needlessly complicated. This commit therefore adds a variant of
Tasks RCU that:
o Has explicit read-side markers to allow finite grace periods in
the face of in-kernel loops for PREEMPT=n builds. These markers
are rcu_read_lock_trace() and rcu_read_unlock_trace().
o Protects code in the idle loop, exception entry/exit, and
CPU-hotplug code paths. In this respect, RCU-tasks trace is
similar to SRCU, but with lighter-weight readers.
o Avoids expensive read-side instruction, having overhead similar
to that of Preemptible RCU.
There are of course downsides:
o The grace-period code can send IPIs to CPUs, even when those
CPUs are in the idle loop or in nohz_full userspace. This is
mitigated by later commits.
o It is necessary to scan the full tasklist, much as for Tasks RCU.
o There is a single callback queue guarded by a single lock,
again, much as for Tasks RCU. However, those early use cases
that request multiple grace periods in quick succession are
expected to do so from a single task, which makes the single
lock almost irrelevant. If needed, multiple callback queues
can be provided using any number of schemes.
Perhaps most important, this variant of RCU does not affect the vanilla
flavors, rcu_preempt and rcu_sched. The fact that RCU Tasks Trace
readers can operate from idle, offline, and exception entry/exit in no
way enables rcu_preempt and rcu_sched readers to do so.
The memory ordering was outlined here:
https://lore.kernel.org/lkml/20200319034030.GX3199@paulmck-ThinkPad-P72/
This effort benefited greatly from off-list discussions of BPF
requirements with Alexei Starovoitov and Andrii Nakryiko. At least
some of the on-list discussions are captured in the Link: tags below.
In addition, KCSAN was quite helpful in finding some early bugs.
Link: https://lore.kernel.org/lkml/20200219150744.428764577@infradead.org/
Link: https://lore.kernel.org/lkml/87mu8p797b.fsf@nanos.tec.linutronix.de/
Link: https://lore.kernel.org/lkml/20200225221305.605144982@linutronix.de/
Cc: Alexei Starovoitov <alexei.starovoitov@gmail.com>
Cc: Andrii Nakryiko <andriin@fb.com>
[ paulmck: Apply feedback from Steve Rostedt and Joel Fernandes. ]
[ paulmck: Decrement trc_n_readers_need_end upon IPI failure. ]
[ paulmck: Fix locking issue reported by rcutorture. ]
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
2020-03-09 19:56:53 -07:00
|
|
|
rcu_tasks_trace.pregp_func = rcu_tasks_trace_pregp_step;
|
|
|
|
|
rcu_tasks_trace.postscan_func = rcu_tasks_trace_postscan;
|
|
|
|
|
rcu_tasks_trace.holdouts_func = check_all_holdout_tasks_trace;
|
|
|
|
|
rcu_tasks_trace.postgp_func = rcu_tasks_trace_postgp;
|
|
|
|
|
rcu_spawn_tasks_kthread_generic(&rcu_tasks_trace);
|
|
|
|
|
return 0;
|
|
|
|
|
}
|
|
|
|
|
|
2020-09-15 17:08:03 -07:00
|
|
|
#if !defined(CONFIG_TINY_RCU)
|
|
|
|
|
void show_rcu_tasks_trace_gp_kthread(void)
|
2020-03-16 11:01:55 -07:00
|
|
|
{
|
2020-03-22 13:34:34 -07:00
|
|
|
char buf[64];
|
2020-03-16 11:01:55 -07:00
|
|
|
|
2024-03-27 10:47:47 -07:00
|
|
|
snprintf(buf, sizeof(buf), "N%lu h:%lu/%lu/%lu",
|
2022-06-01 21:26:57 -07:00
|
|
|
data_race(n_trc_holdouts),
|
2020-03-22 14:09:45 -07:00
|
|
|
data_race(n_heavy_reader_ofl_updates),
|
2020-03-22 13:34:34 -07:00
|
|
|
data_race(n_heavy_reader_updates),
|
|
|
|
|
data_race(n_heavy_reader_attempts));
|
2020-03-16 11:01:55 -07:00
|
|
|
show_rcu_tasks_generic_gp_kthread(&rcu_tasks_trace, buf);
|
|
|
|
|
}
|
2020-09-15 17:08:03 -07:00
|
|
|
EXPORT_SYMBOL_GPL(show_rcu_tasks_trace_gp_kthread);
|
2024-08-01 17:34:25 -07:00
|
|
|
|
|
|
|
|
void rcu_tasks_trace_torture_stats_print(char *tt, char *tf)
|
|
|
|
|
{
|
|
|
|
|
rcu_tasks_torture_stats_print_generic(&rcu_tasks_trace, tt, tf, "");
|
|
|
|
|
}
|
|
|
|
|
EXPORT_SYMBOL_GPL(rcu_tasks_trace_torture_stats_print);
|
2020-09-15 17:08:03 -07:00
|
|
|
#endif // !defined(CONFIG_TINY_RCU)
|
2020-03-16 11:01:55 -07:00
|
|
|
|
2023-05-17 05:01:28 -07:00
|
|
|
struct task_struct *get_rcu_tasks_trace_gp_kthread(void)
|
|
|
|
|
{
|
|
|
|
|
return rcu_tasks_trace.kthread_ptr;
|
|
|
|
|
}
|
|
|
|
|
EXPORT_SYMBOL_GPL(get_rcu_tasks_trace_gp_kthread);
|
|
|
|
|
|
2024-03-18 02:34:11 -07:00
|
|
|
void rcu_tasks_trace_get_gp_data(int *flags, unsigned long *gp_seq)
|
|
|
|
|
{
|
|
|
|
|
*flags = 0;
|
|
|
|
|
*gp_seq = rcu_seq_current(&rcu_tasks_trace.tasks_gp_seq);
|
|
|
|
|
}
|
|
|
|
|
EXPORT_SYMBOL_GPL(rcu_tasks_trace_get_gp_data);
|
|
|
|
|
|
rcu-tasks: Add an RCU Tasks Trace to simplify protection of tracing hooks
Because RCU does not watch exception early-entry/late-exit, idle-loop,
or CPU-hotplug execution, protection of tracing and BPF operations is
needlessly complicated. This commit therefore adds a variant of
Tasks RCU that:
o Has explicit read-side markers to allow finite grace periods in
the face of in-kernel loops for PREEMPT=n builds. These markers
are rcu_read_lock_trace() and rcu_read_unlock_trace().
o Protects code in the idle loop, exception entry/exit, and
CPU-hotplug code paths. In this respect, RCU-tasks trace is
similar to SRCU, but with lighter-weight readers.
o Avoids expensive read-side instruction, having overhead similar
to that of Preemptible RCU.
There are of course downsides:
o The grace-period code can send IPIs to CPUs, even when those
CPUs are in the idle loop or in nohz_full userspace. This is
mitigated by later commits.
o It is necessary to scan the full tasklist, much as for Tasks RCU.
o There is a single callback queue guarded by a single lock,
again, much as for Tasks RCU. However, those early use cases
that request multiple grace periods in quick succession are
expected to do so from a single task, which makes the single
lock almost irrelevant. If needed, multiple callback queues
can be provided using any number of schemes.
Perhaps most important, this variant of RCU does not affect the vanilla
flavors, rcu_preempt and rcu_sched. The fact that RCU Tasks Trace
readers can operate from idle, offline, and exception entry/exit in no
way enables rcu_preempt and rcu_sched readers to do so.
The memory ordering was outlined here:
https://lore.kernel.org/lkml/20200319034030.GX3199@paulmck-ThinkPad-P72/
This effort benefited greatly from off-list discussions of BPF
requirements with Alexei Starovoitov and Andrii Nakryiko. At least
some of the on-list discussions are captured in the Link: tags below.
In addition, KCSAN was quite helpful in finding some early bugs.
Link: https://lore.kernel.org/lkml/20200219150744.428764577@infradead.org/
Link: https://lore.kernel.org/lkml/87mu8p797b.fsf@nanos.tec.linutronix.de/
Link: https://lore.kernel.org/lkml/20200225221305.605144982@linutronix.de/
Cc: Alexei Starovoitov <alexei.starovoitov@gmail.com>
Cc: Andrii Nakryiko <andriin@fb.com>
[ paulmck: Apply feedback from Steve Rostedt and Joel Fernandes. ]
[ paulmck: Decrement trc_n_readers_need_end upon IPI failure. ]
[ paulmck: Fix locking issue reported by rcutorture. ]
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
2020-03-09 19:56:53 -07:00
|
|
|
#else /* #ifdef CONFIG_TASKS_TRACE_RCU */
|
2020-04-05 20:49:13 -07:00
|
|
|
static void exit_tasks_rcu_finish_trace(struct task_struct *t) { }
|
rcu-tasks: Add an RCU Tasks Trace to simplify protection of tracing hooks
Because RCU does not watch exception early-entry/late-exit, idle-loop,
or CPU-hotplug execution, protection of tracing and BPF operations is
needlessly complicated. This commit therefore adds a variant of
Tasks RCU that:
o Has explicit read-side markers to allow finite grace periods in
the face of in-kernel loops for PREEMPT=n builds. These markers
are rcu_read_lock_trace() and rcu_read_unlock_trace().
o Protects code in the idle loop, exception entry/exit, and
CPU-hotplug code paths. In this respect, RCU-tasks trace is
similar to SRCU, but with lighter-weight readers.
o Avoids expensive read-side instruction, having overhead similar
to that of Preemptible RCU.
There are of course downsides:
o The grace-period code can send IPIs to CPUs, even when those
CPUs are in the idle loop or in nohz_full userspace. This is
mitigated by later commits.
o It is necessary to scan the full tasklist, much as for Tasks RCU.
o There is a single callback queue guarded by a single lock,
again, much as for Tasks RCU. However, those early use cases
that request multiple grace periods in quick succession are
expected to do so from a single task, which makes the single
lock almost irrelevant. If needed, multiple callback queues
can be provided using any number of schemes.
Perhaps most important, this variant of RCU does not affect the vanilla
flavors, rcu_preempt and rcu_sched. The fact that RCU Tasks Trace
readers can operate from idle, offline, and exception entry/exit in no
way enables rcu_preempt and rcu_sched readers to do so.
The memory ordering was outlined here:
https://lore.kernel.org/lkml/20200319034030.GX3199@paulmck-ThinkPad-P72/
This effort benefited greatly from off-list discussions of BPF
requirements with Alexei Starovoitov and Andrii Nakryiko. At least
some of the on-list discussions are captured in the Link: tags below.
In addition, KCSAN was quite helpful in finding some early bugs.
Link: https://lore.kernel.org/lkml/20200219150744.428764577@infradead.org/
Link: https://lore.kernel.org/lkml/87mu8p797b.fsf@nanos.tec.linutronix.de/
Link: https://lore.kernel.org/lkml/20200225221305.605144982@linutronix.de/
Cc: Alexei Starovoitov <alexei.starovoitov@gmail.com>
Cc: Andrii Nakryiko <andriin@fb.com>
[ paulmck: Apply feedback from Steve Rostedt and Joel Fernandes. ]
[ paulmck: Decrement trc_n_readers_need_end upon IPI failure. ]
[ paulmck: Fix locking issue reported by rcutorture. ]
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
2020-03-09 19:56:53 -07:00
|
|
|
#endif /* #else #ifdef CONFIG_TASKS_TRACE_RCU */
|
2020-03-15 14:51:20 -07:00
|
|
|
|
2020-05-28 20:03:48 -07:00
|
|
|
#ifndef CONFIG_TINY_RCU
|
2020-03-16 11:01:55 -07:00
|
|
|
void show_rcu_tasks_gp_kthreads(void)
|
|
|
|
|
{
|
|
|
|
|
show_rcu_tasks_classic_gp_kthread();
|
|
|
|
|
show_rcu_tasks_rude_gp_kthread();
|
|
|
|
|
show_rcu_tasks_trace_gp_kthread();
|
|
|
|
|
}
|
2020-05-28 20:03:48 -07:00
|
|
|
#endif /* #ifndef CONFIG_TINY_RCU */
|
2020-03-16 11:01:55 -07:00
|
|
|
|
2020-12-09 13:27:32 -07:00
|
|
|
#ifdef CONFIG_PROVE_RCU
|
|
|
|
|
struct rcu_tasks_test_desc {
|
|
|
|
|
struct rcu_head rh;
|
|
|
|
|
const char *name;
|
|
|
|
|
bool notrun;
|
2022-06-07 15:23:52 -07:00
|
|
|
unsigned long runstart;
|
2020-12-09 13:27:32 -07:00
|
|
|
};
|
|
|
|
|
|
|
|
|
|
static struct rcu_tasks_test_desc tests[] = {
|
|
|
|
|
{
|
|
|
|
|
.name = "call_rcu_tasks()",
|
|
|
|
|
/* If not defined, the test is skipped. */
|
2022-06-07 15:23:52 -07:00
|
|
|
.notrun = IS_ENABLED(CONFIG_TASKS_RCU),
|
2020-12-09 13:27:32 -07:00
|
|
|
},
|
|
|
|
|
{
|
|
|
|
|
.name = "call_rcu_tasks_trace()",
|
|
|
|
|
/* If not defined, the test is skipped. */
|
2022-06-07 15:23:52 -07:00
|
|
|
.notrun = IS_ENABLED(CONFIG_TASKS_TRACE_RCU)
|
2020-12-09 13:27:32 -07:00
|
|
|
}
|
|
|
|
|
};
|
|
|
|
|
|
2024-07-03 19:47:47 -07:00
|
|
|
#if defined(CONFIG_TASKS_RCU) || defined(CONFIG_TASKS_TRACE_RCU)
|
2020-12-09 13:27:32 -07:00
|
|
|
static void test_rcu_tasks_callback(struct rcu_head *rhp)
|
|
|
|
|
{
|
|
|
|
|
struct rcu_tasks_test_desc *rttd =
|
|
|
|
|
container_of(rhp, struct rcu_tasks_test_desc, rh);
|
|
|
|
|
|
|
|
|
|
pr_info("Callback from %s invoked.\n", rttd->name);
|
|
|
|
|
|
2022-06-07 15:23:52 -07:00
|
|
|
rttd->notrun = false;
|
2020-12-09 13:27:32 -07:00
|
|
|
}
|
2024-07-03 19:47:47 -07:00
|
|
|
#endif // #if defined(CONFIG_TASKS_RCU) || defined(CONFIG_TASKS_TRACE_RCU)
|
2020-12-09 13:27:32 -07:00
|
|
|
|
|
|
|
|
static void rcu_tasks_initiate_self_tests(void)
|
|
|
|
|
{
|
|
|
|
|
#ifdef CONFIG_TASKS_RCU
|
2023-07-27 13:13:46 -07:00
|
|
|
pr_info("Running RCU Tasks wait API self tests\n");
|
2022-11-21 08:01:50 -07:00
|
|
|
tests[0].runstart = jiffies;
|
2020-12-09 13:27:32 -07:00
|
|
|
synchronize_rcu_tasks();
|
|
|
|
|
call_rcu_tasks(&tests[0].rh, test_rcu_tasks_callback);
|
|
|
|
|
#endif
|
|
|
|
|
|
|
|
|
|
#ifdef CONFIG_TASKS_RUDE_RCU
|
2023-07-27 13:13:46 -07:00
|
|
|
pr_info("Running RCU Tasks Rude wait API self tests\n");
|
2020-12-09 13:27:32 -07:00
|
|
|
synchronize_rcu_tasks_rude();
|
|
|
|
|
#endif
|
|
|
|
|
|
|
|
|
|
#ifdef CONFIG_TASKS_TRACE_RCU
|
2023-07-27 13:13:46 -07:00
|
|
|
pr_info("Running RCU Tasks Trace wait API self tests\n");
|
2024-07-03 19:47:47 -07:00
|
|
|
tests[1].runstart = jiffies;
|
2020-12-09 13:27:32 -07:00
|
|
|
synchronize_rcu_tasks_trace();
|
2024-07-03 19:47:47 -07:00
|
|
|
call_rcu_tasks_trace(&tests[1].rh, test_rcu_tasks_callback);
|
2020-12-09 13:27:32 -07:00
|
|
|
#endif
|
|
|
|
|
}
|
|
|
|
|
|
2022-06-14 05:06:20 -07:00
|
|
|
/*
|
|
|
|
|
* Return: 0 - test passed
|
|
|
|
|
* 1 - test failed, but have not timed out yet
|
|
|
|
|
* -1 - test failed and timed out
|
|
|
|
|
*/
|
2020-12-09 13:27:32 -07:00
|
|
|
static int rcu_tasks_verify_self_tests(void)
|
|
|
|
|
{
|
|
|
|
|
int ret = 0;
|
|
|
|
|
int i;
|
2022-06-07 15:23:52 -07:00
|
|
|
unsigned long bst = rcu_task_stall_timeout;
|
2020-12-09 13:27:32 -07:00
|
|
|
|
2022-06-07 15:23:52 -07:00
|
|
|
if (bst <= 0 || bst > RCU_TASK_BOOT_STALL_TIMEOUT)
|
|
|
|
|
bst = RCU_TASK_BOOT_STALL_TIMEOUT;
|
2020-12-09 13:27:32 -07:00
|
|
|
for (i = 0; i < ARRAY_SIZE(tests); i++) {
|
2022-06-07 15:23:52 -07:00
|
|
|
while (tests[i].notrun) { // still hanging.
|
|
|
|
|
if (time_after(jiffies, tests[i].runstart + bst)) {
|
|
|
|
|
pr_err("%s has failed boot-time tests.\n", tests[i].name);
|
|
|
|
|
ret = -1;
|
|
|
|
|
break;
|
|
|
|
|
}
|
2022-06-14 05:06:20 -07:00
|
|
|
ret = 1;
|
|
|
|
|
break;
|
2020-12-09 13:27:32 -07:00
|
|
|
}
|
|
|
|
|
}
|
2022-06-14 05:06:20 -07:00
|
|
|
WARN_ON(ret < 0);
|
2020-12-09 13:27:32 -07:00
|
|
|
|
|
|
|
|
return ret;
|
|
|
|
|
}
|
2022-06-14 05:06:20 -07:00
|
|
|
|
|
|
|
|
/*
|
|
|
|
|
* Repeat the rcu_tasks_verify_self_tests() call once every second until the
|
|
|
|
|
* test passes or has timed out.
|
|
|
|
|
*/
|
|
|
|
|
static struct delayed_work rcu_tasks_verify_work;
|
|
|
|
|
static void rcu_tasks_verify_work_fn(struct work_struct *work __maybe_unused)
|
|
|
|
|
{
|
|
|
|
|
int ret = rcu_tasks_verify_self_tests();
|
|
|
|
|
|
|
|
|
|
if (ret <= 0)
|
|
|
|
|
return;
|
|
|
|
|
|
|
|
|
|
/* Test fails but not timed out yet, reschedule another check */
|
|
|
|
|
schedule_delayed_work(&rcu_tasks_verify_work, HZ);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static int rcu_tasks_verify_schedule_work(void)
|
|
|
|
|
{
|
|
|
|
|
INIT_DELAYED_WORK(&rcu_tasks_verify_work, rcu_tasks_verify_work_fn);
|
|
|
|
|
rcu_tasks_verify_work_fn(NULL);
|
|
|
|
|
return 0;
|
|
|
|
|
}
|
|
|
|
|
late_initcall(rcu_tasks_verify_schedule_work);
|
2020-12-09 13:27:32 -07:00
|
|
|
#else /* #ifdef CONFIG_PROVE_RCU */
|
|
|
|
|
static void rcu_tasks_initiate_self_tests(void) { }
|
|
|
|
|
#endif /* #else #ifdef CONFIG_PROVE_RCU */
|
|
|
|
|
|
2024-02-22 13:29:54 -07:00
|
|
|
void __init tasks_cblist_init_generic(void)
|
|
|
|
|
{
|
|
|
|
|
lockdep_assert_irqs_disabled();
|
|
|
|
|
WARN_ON(num_online_cpus() > 1);
|
|
|
|
|
|
|
|
|
|
#ifdef CONFIG_TASKS_RCU
|
|
|
|
|
cblist_init_generic(&rcu_tasks);
|
|
|
|
|
#endif
|
|
|
|
|
|
|
|
|
|
#ifdef CONFIG_TASKS_RUDE_RCU
|
|
|
|
|
cblist_init_generic(&rcu_tasks_rude);
|
|
|
|
|
#endif
|
|
|
|
|
|
|
|
|
|
#ifdef CONFIG_TASKS_TRACE_RCU
|
|
|
|
|
cblist_init_generic(&rcu_tasks_trace);
|
|
|
|
|
#endif
|
|
|
|
|
}
|
|
|
|
|
|
2020-12-09 13:27:31 -07:00
|
|
|
void __init rcu_init_tasks_generic(void)
|
|
|
|
|
{
|
|
|
|
|
#ifdef CONFIG_TASKS_RCU
|
|
|
|
|
rcu_spawn_tasks_kthread();
|
|
|
|
|
#endif
|
|
|
|
|
|
|
|
|
|
#ifdef CONFIG_TASKS_RUDE_RCU
|
|
|
|
|
rcu_spawn_tasks_rude_kthread();
|
|
|
|
|
#endif
|
|
|
|
|
|
|
|
|
|
#ifdef CONFIG_TASKS_TRACE_RCU
|
|
|
|
|
rcu_spawn_tasks_trace_kthread();
|
|
|
|
|
#endif
|
2020-12-09 13:27:32 -07:00
|
|
|
|
|
|
|
|
// Run the self-tests.
|
|
|
|
|
rcu_tasks_initiate_self_tests();
|
2020-12-09 13:27:31 -07:00
|
|
|
}
|
|
|
|
|
|
2020-03-15 14:51:20 -07:00
|
|
|
#else /* #ifdef CONFIG_TASKS_RCU_GENERIC */
|
|
|
|
|
static inline void rcu_tasks_bootup_oddness(void) {}
|
|
|
|
|
#endif /* #else #ifdef CONFIG_TASKS_RCU_GENERIC */
|