614e40faf5
LKMM has long provided only approximate handling of SRCU read-side
critical sections. This has not been a pressing problem because LKMM's
traditional handling is correct for the common cases of non-overlapping
and properly nested critical sections. However, LKMM's traditional
handling of partially overlapping critical sections incorrectly fuses
them into one large critical section.
For example, consider the following litmus test:
------------------------------------------------------------------------
C C-srcu-nest-5
(*
* Result: Sometimes
*
* This demonstrates non-nested overlapping of SRCU read-side critical
* sections. Unlike RCU, SRCU critical sections do not unconditionally
* nest.
*)
{}
P0(int *x, int *y, struct srcu_struct *s1)
{
int r1;
int r2;
int r3;
int r4;
r3 = srcu_read_lock(s1);
r2 = READ_ONCE(*y);
r4 = srcu_read_lock(s1);
srcu_read_unlock(s1, r3);
r1 = READ_ONCE(*x);
srcu_read_unlock(s1, r4);
}
P1(int *x, int *y, struct srcu_struct *s1)
{
WRITE_ONCE(*y, 1);
synchronize_srcu(s1);
WRITE_ONCE(*x, 1);
}
locations [0:r1]
exists (0:r1=1 /\ 0:r2=0)
------------------------------------------------------------------------
Current mainline incorrectly flattens the two critical sections into
one larger critical section, giving "Never" instead of the correct
"Sometimes":
------------------------------------------------------------------------
$ herd7 -conf linux-kernel.cfg C-srcu-nest-5.litmus
Test C-srcu-nest-5 Allowed
States 3
0:r1=0; 0:r2=0;
0:r1=0; 0:r2=1;
0:r1=1; 0:r2=1;
No
Witnesses
Positive: 0 Negative: 3
Flag srcu-bad-nesting
Condition exists (0:r1=1 /\ 0:r2=0)
Observation C-srcu-nest-5 Never 0 3
Time C-srcu-nest-5 0.01
Hash=e692c106cf3e84e20f12991dc438ff1b
------------------------------------------------------------------------
To its credit, it does complain about bad nesting. But with this
commit we get the following result, which has the virtue of being
correct:
------------------------------------------------------------------------
$ herd7 -conf linux-kernel.cfg C-srcu-nest-5.litmus
Test C-srcu-nest-5 Allowed
States 4
0:r1=0; 0:r2=0;
0:r1=0; 0:r2=1;
0:r1=1; 0:r2=0;
0:r1=1; 0:r2=1;
Ok
Witnesses
Positive: 1 Negative: 3
Condition exists (0:r1=1 /\ 0:r2=0)
Observation C-srcu-nest-5 Sometimes 1 3
Time C-srcu-nest-5 0.05
Hash=e692c106cf3e84e20f12991dc438ff1b
------------------------------------------------------------------------
In addition, there are new srcu_down_read() and srcu_up_read()
functions on their way to mainline. Roughly speaking, these are to
srcu_read_lock() and srcu_read_unlock() as down() and up() are to
mutex_lock() and mutex_unlock(). The key point is that
srcu_down_read() can execute in one process and the matching
srcu_up_read() in another, as shown in this litmus test:
------------------------------------------------------------------------
C C-srcu-nest-6
(*
* Result: Never
*
* This would be valid for srcu_down_read() and srcu_up_read().
*)
{}
P0(int *x, int *y, struct srcu_struct *s1, int *idx, int *f)
{
int r2;
int r3;
r3 = srcu_down_read(s1);
WRITE_ONCE(*idx, r3);
r2 = READ_ONCE(*y);
smp_store_release(f, 1);
}
P1(int *x, int *y, struct srcu_struct *s1, int *idx, int *f)
{
int r1;
int r3;
int r4;
r4 = smp_load_acquire(f);
r1 = READ_ONCE(*x);
r3 = READ_ONCE(*idx);
srcu_up_read(s1, r3);
}
P2(int *x, int *y, struct srcu_struct *s1)
{
WRITE_ONCE(*y, 1);
synchronize_srcu(s1);
WRITE_ONCE(*x, 1);
}
locations [0:r1]
filter (1:r4=1)
exists (1:r1=1 /\ 0:r2=0)
------------------------------------------------------------------------
When run on current mainline, this litmus test gets a complaint about
an unknown macro srcu_down_read(). With this commit:
------------------------------------------------------------------------
herd7 -conf linux-kernel.cfg C-srcu-nest-6.litmus
Test C-srcu-nest-6 Allowed
States 3
0:r1=0; 0:r2=0; 1:r1=0;
0:r1=0; 0:r2=1; 1:r1=0;
0:r1=0; 0:r2=1; 1:r1=1;
No
Witnesses
Positive: 0 Negative: 3
Condition exists (1:r1=1 /\ 0:r2=0)
Observation C-srcu-nest-6 Never 0 3
Time C-srcu-nest-6 0.02
Hash=c1f20257d052ca5e899be508bedcb2a1
------------------------------------------------------------------------
Note that the user must supply the flag "f" and the "filter" clause,
similar to what must be done to emulate call_rcu().
The commit works by treating srcu_read_lock()/srcu_down_read() as
loads and srcu_read_unlock()/srcu_up_read() as stores. This allows us
to determine which unlock matches which lock by looking for a data
dependency between them. In order for this to work properly, the data
dependencies have to be tracked through stores to intermediate
variables such as "idx" in the litmus test above; this is handled by
the new carry-srcu-data relation. But it's important here (and in the
existing carry-dep relation) to avoid tracking the dependencies
through SRCU unlock stores. Otherwise, in situations resembling:
A: r1 = srcu_read_lock(s);
B: srcu_read_unlock(s, r1);
C: r2 = srcu_read_lock(s);
D: srcu_read_unlock(s, r2);
it would look as if D was dependent on both A and C, because "s" would
appear to be an intermediate variable written by B and read by C.
This explains the complications in the definitions of carry-srcu-dep
and carry-dep.
As a debugging aid, the commit adds a check for errors in which the
value returned by one call to srcu_read_lock()/srcu_down_read() is
passed to more than one instance of srcu_read_unlock()/srcu_up_read().
Finally, since these SRCU-related primitives are now treated as
ordinary reads and writes, we have to add them into the lists of
marked accesses (i.e., not subject to data races) and lock-related
accesses (i.e., one shouldn't try to access an srcu_struct with a
non-lock-related primitive such as READ_ONCE() or a plain write).
Portions of this approach were suggested by Boqun Feng and Jonas
Oberhauser.
[ paulmck: Fix space-before-tab whitespace nit. ]
Reported-by: Paul E. McKenney <paulmck@kernel.org>
Signed-off-by: Alan Stern <stern@rowland.harvard.edu>
Reviewed-by: Jonas Oberhauser <jonas.oberhauser@huaweicloud.com>
Signed-off-by: Paul E. McKenney <paulmck@kernel.org>
85 lines
3.1 KiB
Plaintext
85 lines
3.1 KiB
Plaintext
// SPDX-License-Identifier: GPL-2.0+
|
|
(*
|
|
* Copyright (C) 2015 Jade Alglave <j.alglave@ucl.ac.uk>,
|
|
* Copyright (C) 2016 Luc Maranget <luc.maranget@inria.fr> for Inria
|
|
* Copyright (C) 2017 Alan Stern <stern@rowland.harvard.edu>,
|
|
* Andrea Parri <parri.andrea@gmail.com>
|
|
*
|
|
* An earlier version of this file appeared in the companion webpage for
|
|
* "Frightening small children and disconcerting grown-ups: Concurrency
|
|
* in the Linux kernel" by Alglave, Maranget, McKenney, Parri, and Stern,
|
|
* which appeared in ASPLOS 2018.
|
|
*)
|
|
|
|
"Linux-kernel memory consistency model"
|
|
|
|
enum Accesses = 'once (*READ_ONCE,WRITE_ONCE*) ||
|
|
'release (*smp_store_release*) ||
|
|
'acquire (*smp_load_acquire*) ||
|
|
'noreturn (* R of non-return RMW *)
|
|
instructions R[{'once,'acquire,'noreturn}]
|
|
instructions W[{'once,'release}]
|
|
instructions RMW[{'once,'acquire,'release}]
|
|
|
|
enum Barriers = 'wmb (*smp_wmb*) ||
|
|
'rmb (*smp_rmb*) ||
|
|
'mb (*smp_mb*) ||
|
|
'barrier (*barrier*) ||
|
|
'rcu-lock (*rcu_read_lock*) ||
|
|
'rcu-unlock (*rcu_read_unlock*) ||
|
|
'sync-rcu (*synchronize_rcu*) ||
|
|
'before-atomic (*smp_mb__before_atomic*) ||
|
|
'after-atomic (*smp_mb__after_atomic*) ||
|
|
'after-spinlock (*smp_mb__after_spinlock*) ||
|
|
'after-unlock-lock (*smp_mb__after_unlock_lock*) ||
|
|
'after-srcu-read-unlock (*smp_mb__after_srcu_read_unlock*)
|
|
instructions F[Barriers]
|
|
|
|
(* SRCU *)
|
|
enum SRCU = 'srcu-lock || 'srcu-unlock || 'sync-srcu
|
|
instructions SRCU[SRCU]
|
|
(* All srcu events *)
|
|
let Srcu = Srcu-lock | Srcu-unlock | Sync-srcu
|
|
|
|
(* Compute matching pairs of nested Rcu-lock and Rcu-unlock *)
|
|
let rcu-rscs = let rec
|
|
unmatched-locks = Rcu-lock \ domain(matched)
|
|
and unmatched-unlocks = Rcu-unlock \ range(matched)
|
|
and unmatched = unmatched-locks | unmatched-unlocks
|
|
and unmatched-po = [unmatched] ; po ; [unmatched]
|
|
and unmatched-locks-to-unlocks =
|
|
[unmatched-locks] ; po ; [unmatched-unlocks]
|
|
and matched = matched | (unmatched-locks-to-unlocks \
|
|
(unmatched-po ; unmatched-po))
|
|
in matched
|
|
|
|
(* Validate nesting *)
|
|
flag ~empty Rcu-lock \ domain(rcu-rscs) as unmatched-rcu-lock
|
|
flag ~empty Rcu-unlock \ range(rcu-rscs) as unmatched-rcu-unlock
|
|
|
|
(* Compute matching pairs of nested Srcu-lock and Srcu-unlock *)
|
|
let carry-srcu-data = (data ; [~ Srcu-unlock] ; rf)*
|
|
let srcu-rscs = ([Srcu-lock] ; carry-srcu-data ; data ; [Srcu-unlock]) & loc
|
|
|
|
(* Validate nesting *)
|
|
flag ~empty Srcu-lock \ domain(srcu-rscs) as unmatched-srcu-lock
|
|
flag ~empty Srcu-unlock \ range(srcu-rscs) as unmatched-srcu-unlock
|
|
flag ~empty (srcu-rscs^-1 ; srcu-rscs) \ id as multiple-srcu-matches
|
|
|
|
(* Check for use of synchronize_srcu() inside an RCU critical section *)
|
|
flag ~empty rcu-rscs & (po ; [Sync-srcu] ; po) as invalid-sleep
|
|
|
|
(* Validate SRCU dynamic match *)
|
|
flag ~empty different-values(srcu-rscs) as srcu-bad-value-match
|
|
|
|
(* Compute marked and plain memory accesses *)
|
|
let Marked = (~M) | IW | Once | Release | Acquire | domain(rmw) | range(rmw) |
|
|
LKR | LKW | UL | LF | RL | RU | Srcu-lock | Srcu-unlock
|
|
let Plain = M \ Marked
|
|
|
|
(* Redefine dependencies to include those carried through plain accesses *)
|
|
let carry-dep = (data ; [~ Srcu-unlock] ; rfi)*
|
|
let addr = carry-dep ; addr
|
|
let ctrl = carry-dep ; ctrl
|
|
let data = carry-dep ; data
|