16e5726269
Since commit 7361c36c52 (af_unix: Allow credentials to work across
user and pid namespaces) af_unix performance dropped a lot.
This is because we now take a reference on pid and cred in each write(),
and release them in read(), usually done from another process,
eventually from another cpu. This triggers false sharing.
# Events: 154K cycles
#
# Overhead Command Shared Object Symbol
# ........ ....... .................. .........................
#
10.40% hackbench [kernel.kallsyms] [k] put_pid
8.60% hackbench [kernel.kallsyms] [k] unix_stream_recvmsg
7.87% hackbench [kernel.kallsyms] [k] unix_stream_sendmsg
6.11% hackbench [kernel.kallsyms] [k] do_raw_spin_lock
4.95% hackbench [kernel.kallsyms] [k] unix_scm_to_skb
4.87% hackbench [kernel.kallsyms] [k] pid_nr_ns
4.34% hackbench [kernel.kallsyms] [k] cred_to_ucred
2.39% hackbench [kernel.kallsyms] [k] unix_destruct_scm
2.24% hackbench [kernel.kallsyms] [k] sub_preempt_count
1.75% hackbench [kernel.kallsyms] [k] fget_light
1.51% hackbench [kernel.kallsyms] [k]
__mutex_lock_interruptible_slowpath
1.42% hackbench [kernel.kallsyms] [k] sock_alloc_send_pskb
This patch includes SCM_CREDENTIALS information in a af_unix message/skb
only if requested by the sender, [man 7 unix for details how to include
ancillary data using sendmsg() system call]
Note: This might break buggy applications that expected SCM_CREDENTIAL
from an unaware write() system call, and receiver not using SO_PASSCRED
socket option.
If SOCK_PASSCRED is set on source or destination socket, we still
include credentials for mere write() syscalls.
Performance boost in hackbench : more than 50% gain on a 16 thread
machine (2 quad-core cpus, 2 threads per core)
hackbench 20 thread 2000
4.228 sec instead of 9.102 sec
Signed-off-by: Eric Dumazet <eric.dumazet@gmail.com>
Acked-by: Tim Chen <tim.c.chen@linux.intel.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
130 lines
3.1 KiB
C
130 lines
3.1 KiB
C
#ifndef __LINUX_NET_SCM_H
|
|
#define __LINUX_NET_SCM_H
|
|
|
|
#include <linux/limits.h>
|
|
#include <linux/net.h>
|
|
#include <linux/security.h>
|
|
#include <linux/pid.h>
|
|
#include <linux/nsproxy.h>
|
|
|
|
/* Well, we should have at least one descriptor open
|
|
* to accept passed FDs 8)
|
|
*/
|
|
#define SCM_MAX_FD 253
|
|
|
|
struct scm_fp_list {
|
|
struct list_head list;
|
|
short count;
|
|
short max;
|
|
struct file *fp[SCM_MAX_FD];
|
|
};
|
|
|
|
struct scm_cookie {
|
|
struct pid *pid; /* Skb credentials */
|
|
const struct cred *cred;
|
|
struct scm_fp_list *fp; /* Passed files */
|
|
struct ucred creds; /* Skb credentials */
|
|
#ifdef CONFIG_SECURITY_NETWORK
|
|
u32 secid; /* Passed security ID */
|
|
#endif
|
|
};
|
|
|
|
extern void scm_detach_fds(struct msghdr *msg, struct scm_cookie *scm);
|
|
extern void scm_detach_fds_compat(struct msghdr *msg, struct scm_cookie *scm);
|
|
extern int __scm_send(struct socket *sock, struct msghdr *msg, struct scm_cookie *scm);
|
|
extern void __scm_destroy(struct scm_cookie *scm);
|
|
extern struct scm_fp_list * scm_fp_dup(struct scm_fp_list *fpl);
|
|
|
|
#ifdef CONFIG_SECURITY_NETWORK
|
|
static __inline__ void unix_get_peersec_dgram(struct socket *sock, struct scm_cookie *scm)
|
|
{
|
|
security_socket_getpeersec_dgram(sock, NULL, &scm->secid);
|
|
}
|
|
#else
|
|
static __inline__ void unix_get_peersec_dgram(struct socket *sock, struct scm_cookie *scm)
|
|
{ }
|
|
#endif /* CONFIG_SECURITY_NETWORK */
|
|
|
|
static __inline__ void scm_set_cred(struct scm_cookie *scm,
|
|
struct pid *pid, const struct cred *cred)
|
|
{
|
|
scm->pid = get_pid(pid);
|
|
scm->cred = cred ? get_cred(cred) : NULL;
|
|
cred_to_ucred(pid, cred, &scm->creds);
|
|
}
|
|
|
|
static __inline__ void scm_destroy_cred(struct scm_cookie *scm)
|
|
{
|
|
put_pid(scm->pid);
|
|
scm->pid = NULL;
|
|
|
|
if (scm->cred)
|
|
put_cred(scm->cred);
|
|
scm->cred = NULL;
|
|
}
|
|
|
|
static __inline__ void scm_destroy(struct scm_cookie *scm)
|
|
{
|
|
scm_destroy_cred(scm);
|
|
if (scm && scm->fp)
|
|
__scm_destroy(scm);
|
|
}
|
|
|
|
static __inline__ int scm_send(struct socket *sock, struct msghdr *msg,
|
|
struct scm_cookie *scm)
|
|
{
|
|
memset(scm, 0, sizeof(*scm));
|
|
unix_get_peersec_dgram(sock, scm);
|
|
if (msg->msg_controllen <= 0)
|
|
return 0;
|
|
return __scm_send(sock, msg, scm);
|
|
}
|
|
|
|
#ifdef CONFIG_SECURITY_NETWORK
|
|
static inline void scm_passec(struct socket *sock, struct msghdr *msg, struct scm_cookie *scm)
|
|
{
|
|
char *secdata;
|
|
u32 seclen;
|
|
int err;
|
|
|
|
if (test_bit(SOCK_PASSSEC, &sock->flags)) {
|
|
err = security_secid_to_secctx(scm->secid, &secdata, &seclen);
|
|
|
|
if (!err) {
|
|
put_cmsg(msg, SOL_SOCKET, SCM_SECURITY, seclen, secdata);
|
|
security_release_secctx(secdata, seclen);
|
|
}
|
|
}
|
|
}
|
|
#else
|
|
static inline void scm_passec(struct socket *sock, struct msghdr *msg, struct scm_cookie *scm)
|
|
{ }
|
|
#endif /* CONFIG_SECURITY_NETWORK */
|
|
|
|
static __inline__ void scm_recv(struct socket *sock, struct msghdr *msg,
|
|
struct scm_cookie *scm, int flags)
|
|
{
|
|
if (!msg->msg_control) {
|
|
if (test_bit(SOCK_PASSCRED, &sock->flags) || scm->fp)
|
|
msg->msg_flags |= MSG_CTRUNC;
|
|
scm_destroy(scm);
|
|
return;
|
|
}
|
|
|
|
if (test_bit(SOCK_PASSCRED, &sock->flags))
|
|
put_cmsg(msg, SOL_SOCKET, SCM_CREDENTIALS, sizeof(scm->creds), &scm->creds);
|
|
|
|
scm_destroy_cred(scm);
|
|
|
|
scm_passec(sock, msg, scm);
|
|
|
|
if (!scm->fp)
|
|
return;
|
|
|
|
scm_detach_fds(msg, scm);
|
|
}
|
|
|
|
|
|
#endif /* __LINUX_NET_SCM_H */
|
|
|