kevin/linux
1
Fork 0
Code

net: dev: Makes sure netif_rx() can be invoked in any context.

Browse Source 
Dave suggested a while ago (eleven years by now) "Let's make netif_rx()
work in all contexts and get rid of netif_rx_ni()". Eric agreed and
pointed out that modern devices should use netif_receive_skb() to avoid
the overhead.
In the meantime someone added another variant, netif_rx_any_context(),
which behaves as suggested.

netif_rx() must be invoked with disabled bottom halves to ensure that
pending softirqs, which were raised within the function, are handled.
netif_rx_ni() can be invoked only from process context (bottom halves
must be enabled) because the function handles pending softirqs without
checking if bottom halves were disabled or not.
netif_rx_any_context() invokes on the former functions by checking
in_interrupts().

netif_rx() could be taught to handle both cases (disabled and enabled
bottom halves) by simply disabling bottom halves while invoking
netif_rx_internal(). The local_bh_enable() invocation will then invoke
pending softirqs only if the BH-disable counter drops to zero.

Eric is concerned about the overhead of BH-disable+enable especially in
regard to the loopback driver. As critical as this driver is, it will
receive a shortcut to avoid the additional overhead which is not needed.

Add a local_bh_disable() section in netif_rx() to ensure softirqs are
handled if needed.
Provide __netif_rx() which does not disable BH and has a lockdep assert
to ensure that interrupts are disabled. Use this shortcut in the
loopback driver and in drivers/net/*.c.
Make netif_rx_ni() and netif_rx_any_context() invoke netif_rx() so they
can be removed once they are no more users left.

Link: https://lkml.kernel.org/r/20100415.020246.218622820.davem@davemloft.net
Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Reviewed-by: Eric Dumazet <edumazet@google.com>
Reviewed-by: Toke Høiland-Jørgensen <toke@redhat.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
This commit is contained in:
Sebastian Andrzej Siewior 2022-02-12 00:38:38 +01:00 committed by David S. Miller
parent f234ae2947
commit baebdf48c3
16 changed files with 60 additions and 73 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
drivers/net/amt.c
View File

@ -2373,7 +2373,7 @@ static bool amt_membership_query_handler(struct amt_dev *amt,
skb->pkt_type = PACKET_MULTICAST;
skb->ip_summed = CHECKSUM_NONE;
len = skb->len;
if (netif_rx(skb) == NET_RX_SUCCESS) {
if (__netif_rx(skb) == NET_RX_SUCCESS) {
amt_update_gw_status(amt, AMT_STATUS_RECEIVED_QUERY, true);
dev_sw_netstats_rx_add(amt->dev, len);
} else {
@ -2470,7 +2470,7 @@ report:
skb->pkt_type = PACKET_MULTICAST;
skb->ip_summed = CHECKSUM_NONE;
len = skb->len;
if (netif_rx(skb) == NET_RX_SUCCESS) {
if (__netif_rx(skb) == NET_RX_SUCCESS) {
amt_update_relay_status(tunnel, AMT_STATUS_RECEIVED_UPDATE,
true);
dev_sw_netstats_rx_add(amt->dev, len);

4
drivers/net/geneve.c
View File

@ -925,7 +925,7 @@ static int geneve_xmit_skb(struct sk_buff *skb, struct net_device *dev,
}
skb->protocol = eth_type_trans(skb, geneve->dev);
netif_rx(skb);
__netif_rx(skb);
dst_release(&rt->dst);
return -EMSGSIZE;
}
@ -1021,7 +1021,7 @@ static int geneve6_xmit_skb(struct sk_buff *skb, struct net_device *dev,
}
skb->protocol = eth_type_trans(skb, geneve->dev);
netif_rx(skb);
__netif_rx(skb);
dst_release(dst);
return -EMSGSIZE;
}

2
drivers/net/gtp.c
View File

@ -207,7 +207,7 @@ static int gtp_rx(struct pdp_ctx *pctx, struct sk_buff *skb,
dev_sw_netstats_rx_add(pctx->dev, skb->len);
netif_rx(skb);
__netif_rx(skb);
return 0;
err:

4
drivers/net/loopback.c
View File

@ -78,7 +78,7 @@ static netdev_tx_t loopback_xmit(struct sk_buff *skb,
skb_orphan(skb);
/* Before queueing this packet to netif_rx(),
/* Before queueing this packet to __netif_rx(),
* make sure dst is refcounted.
*/
skb_dst_force(skb);
@ -86,7 +86,7 @@ static netdev_tx_t loopback_xmit(struct sk_buff *skb,
skb->protocol = eth_type_trans(skb, dev);
len = skb->len;
if (likely(netif_rx(skb) == NET_RX_SUCCESS))
if (likely(__netif_rx(skb) == NET_RX_SUCCESS))
dev_lstats_add(dev, len);
return NETDEV_TX_OK;

6
drivers/net/macsec.c
View File

@ -1033,7 +1033,7 @@ static enum rx_handler_result handle_not_macsec(struct sk_buff *skb)
else
nskb->pkt_type = PACKET_MULTICAST;
netif_rx(nskb);
__netif_rx(nskb);
}
continue;
}
@ -1056,7 +1056,7 @@ static enum rx_handler_result handle_not_macsec(struct sk_buff *skb)
nskb->dev = ndev;
if (netif_rx(nskb) == NET_RX_SUCCESS) {
if (__netif_rx(nskb) == NET_RX_SUCCESS) {
u64_stats_update_begin(&secy_stats->syncp);
secy_stats->stats.InPktsUntagged++;
u64_stats_update_end(&secy_stats->syncp);
@ -1288,7 +1288,7 @@ nosci:
macsec_reset_skb(nskb, macsec->secy.netdev);
ret = netif_rx(nskb);
ret = __netif_rx(nskb);
if (ret == NET_RX_SUCCESS) {
u64_stats_update_begin(&secy_stats->syncp);
secy_stats->stats.InPktsUnknownSCI++;

4
drivers/net/macvlan.c
View File

@ -410,7 +410,7 @@ static void macvlan_forward_source_one(struct sk_buff *skb,
if (ether_addr_equal_64bits(eth_hdr(skb)->h_dest, dev->dev_addr))
nskb->pkt_type = PACKET_HOST;
ret = netif_rx(nskb);
ret = __netif_rx(nskb);
macvlan_count_rx(vlan, len, ret == NET_RX_SUCCESS, false);
}
@ -468,7 +468,7 @@ static rx_handler_result_t macvlan_handle_frame(struct sk_buff **pskb)
/* forward to original port. */
vlan = src;
ret = macvlan_broadcast_one(skb, vlan, eth, 0) ?:
netif_rx(skb);
__netif_rx(skb);
handle_res = RX_HANDLER_CONSUMED;
goto out;
}

2
drivers/net/mhi_net.c
View File

@ -225,7 +225,7 @@ static void mhi_net_dl_callback(struct mhi_device *mhi_dev,
u64_stats_inc(&mhi_netdev->stats.rx_packets);
u64_stats_add(&mhi_netdev->stats.rx_bytes, skb->len);
u64_stats_update_end(&mhi_netdev->stats.rx_syncp);
netif_rx(skb);
__netif_rx(skb);
}
/* Refill if RX buffers queue becomes low */

2
drivers/net/ntb_netdev.c
View File

@ -119,7 +119,7 @@ static void ntb_netdev_rx_handler(struct ntb_transport_qp *qp, void *qp_data,
skb->protocol = eth_type_trans(skb, ndev);
skb->ip_summed = CHECKSUM_NONE;
if (netif_rx(skb) == NET_RX_DROP) {
if (__netif_rx(skb) == NET_RX_DROP) {
ndev->stats.rx_errors++;
ndev->stats.rx_dropped++;
} else {

2
drivers/net/rionet.c
View File

@ -109,7 +109,7 @@ static int rionet_rx_clean(struct net_device *ndev)
skb_put(rnet->rx_skb[i], RIO_MAX_MSG_SIZE);
rnet->rx_skb[i]->protocol =
eth_type_trans(rnet->rx_skb[i], ndev);
error = netif_rx(rnet->rx_skb[i]);
error = __netif_rx(rnet->rx_skb[i]);
if (error == NET_RX_DROP) {
ndev->stats.rx_dropped++;

2
drivers/net/sb1000.c
View File

@ -872,7 +872,7 @@ printk("cm0: IP identification: %02x%02x fragment offset: %02x%02x\n", buffer[3
/* datagram completed: send to upper level */
skb_trim(skb, dlen);
netif_rx(skb);
__netif_rx(skb);
stats->rx_bytes+=dlen;
stats->rx_packets++;
lp->rx_skb[ns] = NULL;

2
drivers/net/veth.c
View File

@ -287,7 +287,7 @@ static int veth_forward_skb(struct net_device *dev, struct sk_buff *skb,
{
return __dev_forward_skb(dev, skb) ?: xdp ?
veth_xdp_rx(rq, skb) :
netif_rx(skb);
__netif_rx(skb);
}
/* return true if the specified skb has chances of GRO aggregation

2
drivers/net/vrf.c
View File

@ -418,7 +418,7 @@ static int vrf_local_xmit(struct sk_buff *skb, struct net_device *dev,
skb->protocol = eth_type_trans(skb, dev);
if (likely(netif_rx(skb) == NET_RX_SUCCESS))
if (likely(__netif_rx(skb) == NET_RX_SUCCESS))
vrf_rx_stats(dev, len);
else
this_cpu_inc(dev->dstats->rx_drps);

2
drivers/net/vxlan.c
View File

@ -2541,7 +2541,7 @@ static void vxlan_encap_bypass(struct sk_buff *skb, struct vxlan_dev *src_vxlan,
tx_stats->tx_bytes += len;
u64_stats_update_end(&tx_stats->syncp);
if (netif_rx(skb) == NET_RX_SUCCESS) {
if (__netif_rx(skb) == NET_RX_SUCCESS) {
u64_stats_update_begin(&rx_stats->syncp);
rx_stats->rx_packets++;
rx_stats->rx_bytes += len;

14
include/linux/netdevice.h
View File

@ -3672,8 +3672,18 @@ u32 bpf_prog_run_generic_xdp(struct sk_buff *skb, struct xdp_buff *xdp,
void generic_xdp_tx(struct sk_buff *skb, struct bpf_prog *xdp_prog);
int do_xdp_generic(struct bpf_prog *xdp_prog, struct sk_buff *skb);
int netif_rx(struct sk_buff *skb);
int netif_rx_ni(struct sk_buff *skb);
int netif_rx_any_context(struct sk_buff *skb);
int __netif_rx(struct sk_buff *skb);
static inline int netif_rx_ni(struct sk_buff *skb)
{
return netif_rx(skb);
}
static inline int netif_rx_any_context(struct sk_buff *skb)
{
return netif_rx(skb);
}
int netif_receive_skb(struct sk_buff *skb);
int netif_receive_skb_core(struct sk_buff *skb);
void netif_receive_skb_list_internal(struct list_head *head);

14
include/trace/events/net.h
View File

@ -260,13 +260,6 @@ DEFINE_EVENT(net_dev_rx_verbose_template, netif_rx_entry,
TP_ARGS(skb)
);
DEFINE_EVENT(net_dev_rx_verbose_template, netif_rx_ni_entry,
TP_PROTO(const struct sk_buff *skb),
TP_ARGS(skb)
);
DECLARE_EVENT_CLASS(net_dev_rx_exit_template,
TP_PROTO(int ret),
@ -312,13 +305,6 @@ DEFINE_EVENT(net_dev_rx_exit_template, netif_rx_exit,
TP_ARGS(ret)
);
DEFINE_EVENT(net_dev_rx_exit_template, netif_rx_ni_exit,
TP_PROTO(int ret),
TP_ARGS(ret)
);
DEFINE_EVENT(net_dev_rx_exit_template, netif_receive_skb_list_exit,
TP_PROTO(int ret),

67
net/core/dev.c
View File

@ -4815,66 +4815,57 @@ static int netif_rx_internal(struct sk_buff *skb)
return ret;
}
/**
* __netif_rx - Slightly optimized version of netif_rx
* @skb: buffer to post
*
* This behaves as netif_rx except that it does not disable bottom halves.
* As a result this function may only be invoked from the interrupt context
* (either hard or soft interrupt).
*/
int __netif_rx(struct sk_buff *skb)
{
int ret;
lockdep_assert_once(hardirq_count() | softirq_count());
trace_netif_rx_entry(skb);
ret = netif_rx_internal(skb);
trace_netif_rx_exit(ret);
return ret;
}
EXPORT_SYMBOL(__netif_rx);
/**
* netif_rx - post buffer to the network code
* @skb: buffer to post
*
* This function receives a packet from a device driver and queues it for
* the upper (protocol) levels to process. It always succeeds. The buffer
* may be dropped during processing for congestion control or by the
* protocol layers.
* the upper (protocol) levels to process via the backlog NAPI device. It
* always succeeds. The buffer may be dropped during processing for
* congestion control or by the protocol layers.
* The network buffer is passed via the backlog NAPI device. Modern NIC
* driver should use NAPI and GRO.
* This function can used from any context.
*
* return values:
* NET_RX_SUCCESS (no congestion)
* NET_RX_DROP (packet was dropped)
*
*/
int netif_rx(struct sk_buff *skb)
{
int ret;
local_bh_disable();
trace_netif_rx_entry(skb);
ret = netif_rx_internal(skb);
trace_netif_rx_exit(ret);
local_bh_enable();
return ret;
}
EXPORT_SYMBOL(netif_rx);
int netif_rx_ni(struct sk_buff *skb)
{
int err;
trace_netif_rx_ni_entry(skb);
preempt_disable();
err = netif_rx_internal(skb);
if (local_softirq_pending())
do_softirq();
preempt_enable();
trace_netif_rx_ni_exit(err);
return err;
}
EXPORT_SYMBOL(netif_rx_ni);
int netif_rx_any_context(struct sk_buff *skb)
{
/*
* If invoked from contexts which do not invoke bottom half
* processing either at return from interrupt or when softrqs are
* reenabled, use netif_rx_ni() which invokes bottomhalf processing
* directly.
*/
if (in_interrupt())
return netif_rx(skb);
else
return netif_rx_ni(skb);
}
EXPORT_SYMBOL(netif_rx_any_context);
static __latent_entropy void net_tx_action(struct softirq_action *h)
{
struct softnet_data *sd = this_cpu_ptr(&softnet_data);