2020-08-25 11:29:17 -07:00
|
|
|
// SPDX-License-Identifier: GPL-2.0
|
|
|
|
/*
|
|
|
|
* Copyright (c) 2019 Facebook
|
|
|
|
* Copyright 2020 Google LLC.
|
|
|
|
*/
|
|
|
|
|
|
|
|
#include <linux/rculist.h>
|
|
|
|
#include <linux/list.h>
|
|
|
|
#include <linux/hash.h>
|
|
|
|
#include <linux/types.h>
|
|
|
|
#include <linux/spinlock.h>
|
|
|
|
#include <linux/bpf.h>
|
|
|
|
#include <linux/bpf_local_storage.h>
|
|
|
|
#include <net/sock.h>
|
|
|
|
#include <uapi/linux/sock_diag.h>
|
|
|
|
#include <uapi/linux/btf.h>
|
|
|
|
#include <linux/bpf_lsm.h>
|
|
|
|
#include <linux/btf_ids.h>
|
|
|
|
#include <linux/fdtable.h>
|
2021-12-24 08:29:15 -07:00
|
|
|
#include <linux/rcupdate_trace.h>
|
2020-08-25 11:29:17 -07:00
|
|
|
|
|
|
|
DEFINE_BPF_STORAGE_CACHE(inode_cache);
|
|
|
|
|
|
|
|
static struct bpf_local_storage __rcu **
|
|
|
|
inode_storage_ptr(void *owner)
|
|
|
|
{
|
|
|
|
struct inode *inode = owner;
|
|
|
|
struct bpf_storage_blob *bsb;
|
|
|
|
|
|
|
|
bsb = bpf_inode(inode);
|
|
|
|
if (!bsb)
|
|
|
|
return NULL;
|
|
|
|
return &bsb->storage;
|
|
|
|
}
|
|
|
|
|
|
|
|
static struct bpf_local_storage_data *inode_storage_lookup(struct inode *inode,
|
|
|
|
struct bpf_map *map,
|
|
|
|
bool cacheit_lockit)
|
|
|
|
{
|
|
|
|
struct bpf_local_storage *inode_storage;
|
|
|
|
struct bpf_local_storage_map *smap;
|
|
|
|
struct bpf_storage_blob *bsb;
|
|
|
|
|
|
|
|
bsb = bpf_inode(inode);
|
|
|
|
if (!bsb)
|
|
|
|
return NULL;
|
|
|
|
|
2021-12-24 08:29:15 -07:00
|
|
|
inode_storage =
|
|
|
|
rcu_dereference_check(bsb->storage, bpf_rcu_lock_held());
|
2020-08-25 11:29:17 -07:00
|
|
|
if (!inode_storage)
|
|
|
|
return NULL;
|
|
|
|
|
|
|
|
smap = (struct bpf_local_storage_map *)map;
|
|
|
|
return bpf_local_storage_lookup(inode_storage, smap, cacheit_lockit);
|
|
|
|
}
|
|
|
|
|
|
|
|
void bpf_inode_storage_free(struct inode *inode)
|
|
|
|
{
|
|
|
|
struct bpf_local_storage *local_storage;
|
|
|
|
struct bpf_storage_blob *bsb;
|
|
|
|
|
|
|
|
bsb = bpf_inode(inode);
|
|
|
|
if (!bsb)
|
|
|
|
return;
|
|
|
|
|
|
|
|
rcu_read_lock();
|
|
|
|
|
|
|
|
local_storage = rcu_dereference(bsb->storage);
|
|
|
|
if (!local_storage) {
|
|
|
|
rcu_read_unlock();
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
2023-03-07 23:59:21 -07:00
|
|
|
bpf_local_storage_destroy(local_storage);
|
2020-08-25 11:29:17 -07:00
|
|
|
rcu_read_unlock();
|
|
|
|
}
|
|
|
|
|
|
|
|
static void *bpf_fd_inode_storage_lookup_elem(struct bpf_map *map, void *key)
|
|
|
|
{
|
|
|
|
struct bpf_local_storage_data *sdata;
|
2024-08-13 14:18:38 -07:00
|
|
|
CLASS(fd_raw, f)(*(int *)key);
|
2020-08-25 11:29:17 -07:00
|
|
|
|
2024-08-13 14:18:38 -07:00
|
|
|
if (fd_empty(f))
|
2021-03-07 05:09:48 -07:00
|
|
|
return ERR_PTR(-EBADF);
|
2020-08-25 11:29:17 -07:00
|
|
|
|
2024-05-31 11:12:01 -07:00
|
|
|
sdata = inode_storage_lookup(file_inode(fd_file(f)), map, true);
|
2020-08-25 11:29:17 -07:00
|
|
|
return sdata ? sdata->data : NULL;
|
|
|
|
}
|
|
|
|
|
bpf: return long from bpf_map_ops funcs
This patch changes the return types of bpf_map_ops functions to long, where
previously int was returned. Using long allows for bpf programs to maintain
the sign bit in the absence of sign extension during situations where
inlined bpf helper funcs make calls to the bpf_map_ops funcs and a negative
error is returned.
The definitions of the helper funcs are generated from comments in the bpf
uapi header at `include/uapi/linux/bpf.h`. The return type of these
helpers was previously changed from int to long in commit bdb7b79b4ce8. For
any case where one of the map helpers call the bpf_map_ops funcs that are
still returning 32-bit int, a compiler might not include sign extension
instructions to properly convert the 32-bit negative value a 64-bit
negative value.
For example:
bpf assembly excerpt of an inlined helper calling a kernel function and
checking for a specific error:
; err = bpf_map_update_elem(&mymap, &key, &val, BPF_NOEXIST);
...
46: call 0xffffffffe103291c ; htab_map_update_elem
; if (err && err != -EEXIST) {
4b: cmp $0xffffffffffffffef,%rax ; cmp -EEXIST,%rax
kernel function assembly excerpt of return value from
`htab_map_update_elem` returning 32-bit int:
movl $0xffffffef, %r9d
...
movl %r9d, %eax
...results in the comparison:
cmp $0xffffffffffffffef, $0x00000000ffffffef
Fixes: bdb7b79b4ce8 ("bpf: Switch most helper return values from 32-bit int to 64-bit long")
Tested-by: Eduard Zingerman <eddyz87@gmail.com>
Signed-off-by: JP Kobryn <inwardvessel@gmail.com>
Link: https://lore.kernel.org/r/20230322194754.185781-3-inwardvessel@gmail.com
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
2023-03-22 12:47:54 -07:00
|
|
|
static long bpf_fd_inode_storage_update_elem(struct bpf_map *map, void *key,
|
|
|
|
void *value, u64 map_flags)
|
2020-08-25 11:29:17 -07:00
|
|
|
{
|
|
|
|
struct bpf_local_storage_data *sdata;
|
2024-08-13 14:18:38 -07:00
|
|
|
CLASS(fd_raw, f)(*(int *)key);
|
2020-08-25 11:29:17 -07:00
|
|
|
|
2024-08-13 14:18:38 -07:00
|
|
|
if (fd_empty(f))
|
2021-01-20 19:08:56 -07:00
|
|
|
return -EBADF;
|
2024-08-13 14:18:38 -07:00
|
|
|
if (!inode_storage_ptr(file_inode(fd_file(f))))
|
2020-08-25 11:29:17 -07:00
|
|
|
return -EBADF;
|
|
|
|
|
2024-05-31 11:12:01 -07:00
|
|
|
sdata = bpf_local_storage_update(file_inode(fd_file(f)),
|
2020-08-25 11:29:17 -07:00
|
|
|
(struct bpf_local_storage_map *)map,
|
2022-03-17 21:55:52 -07:00
|
|
|
value, map_flags, GFP_ATOMIC);
|
2020-08-25 11:29:17 -07:00
|
|
|
return PTR_ERR_OR_ZERO(sdata);
|
|
|
|
}
|
|
|
|
|
|
|
|
static int inode_storage_delete(struct inode *inode, struct bpf_map *map)
|
|
|
|
{
|
|
|
|
struct bpf_local_storage_data *sdata;
|
|
|
|
|
|
|
|
sdata = inode_storage_lookup(inode, map, false);
|
|
|
|
if (!sdata)
|
|
|
|
return -ENOENT;
|
|
|
|
|
2023-03-07 23:59:25 -07:00
|
|
|
bpf_selem_unlink(SELEM(sdata), false);
|
2020-08-25 11:29:17 -07:00
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
bpf: return long from bpf_map_ops funcs
This patch changes the return types of bpf_map_ops functions to long, where
previously int was returned. Using long allows for bpf programs to maintain
the sign bit in the absence of sign extension during situations where
inlined bpf helper funcs make calls to the bpf_map_ops funcs and a negative
error is returned.
The definitions of the helper funcs are generated from comments in the bpf
uapi header at `include/uapi/linux/bpf.h`. The return type of these
helpers was previously changed from int to long in commit bdb7b79b4ce8. For
any case where one of the map helpers call the bpf_map_ops funcs that are
still returning 32-bit int, a compiler might not include sign extension
instructions to properly convert the 32-bit negative value a 64-bit
negative value.
For example:
bpf assembly excerpt of an inlined helper calling a kernel function and
checking for a specific error:
; err = bpf_map_update_elem(&mymap, &key, &val, BPF_NOEXIST);
...
46: call 0xffffffffe103291c ; htab_map_update_elem
; if (err && err != -EEXIST) {
4b: cmp $0xffffffffffffffef,%rax ; cmp -EEXIST,%rax
kernel function assembly excerpt of return value from
`htab_map_update_elem` returning 32-bit int:
movl $0xffffffef, %r9d
...
movl %r9d, %eax
...results in the comparison:
cmp $0xffffffffffffffef, $0x00000000ffffffef
Fixes: bdb7b79b4ce8 ("bpf: Switch most helper return values from 32-bit int to 64-bit long")
Tested-by: Eduard Zingerman <eddyz87@gmail.com>
Signed-off-by: JP Kobryn <inwardvessel@gmail.com>
Link: https://lore.kernel.org/r/20230322194754.185781-3-inwardvessel@gmail.com
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
2023-03-22 12:47:54 -07:00
|
|
|
static long bpf_fd_inode_storage_delete_elem(struct bpf_map *map, void *key)
|
2020-08-25 11:29:17 -07:00
|
|
|
{
|
2024-08-13 14:18:38 -07:00
|
|
|
CLASS(fd_raw, f)(*(int *)key);
|
2020-08-25 11:29:17 -07:00
|
|
|
|
2024-08-13 14:18:38 -07:00
|
|
|
if (fd_empty(f))
|
2020-08-25 11:29:17 -07:00
|
|
|
return -EBADF;
|
2024-08-13 14:18:38 -07:00
|
|
|
return inode_storage_delete(file_inode(fd_file(f)), map);
|
2020-08-25 11:29:17 -07:00
|
|
|
}
|
|
|
|
|
2022-03-17 21:55:52 -07:00
|
|
|
/* *gfp_flags* is a hidden argument provided by the verifier */
|
|
|
|
BPF_CALL_5(bpf_inode_storage_get, struct bpf_map *, map, struct inode *, inode,
|
|
|
|
void *, value, u64, flags, gfp_t, gfp_flags)
|
2020-08-25 11:29:17 -07:00
|
|
|
{
|
|
|
|
struct bpf_local_storage_data *sdata;
|
|
|
|
|
2021-12-24 08:29:15 -07:00
|
|
|
WARN_ON_ONCE(!bpf_rcu_lock_held());
|
2020-08-25 11:29:17 -07:00
|
|
|
if (flags & ~(BPF_LOCAL_STORAGE_GET_F_CREATE))
|
|
|
|
return (unsigned long)NULL;
|
|
|
|
|
|
|
|
/* explicitly check that the inode_storage_ptr is not
|
|
|
|
* NULL as inode_storage_lookup returns NULL in this case and
|
|
|
|
* bpf_local_storage_update expects the owner to have a
|
|
|
|
* valid storage pointer.
|
|
|
|
*/
|
2021-01-12 00:55:24 -07:00
|
|
|
if (!inode || !inode_storage_ptr(inode))
|
2020-08-25 11:29:17 -07:00
|
|
|
return (unsigned long)NULL;
|
|
|
|
|
|
|
|
sdata = inode_storage_lookup(inode, map, true);
|
|
|
|
if (sdata)
|
|
|
|
return (unsigned long)sdata->data;
|
|
|
|
|
2021-01-12 00:55:25 -07:00
|
|
|
/* This helper must only called from where the inode is guaranteed
|
2020-08-25 11:29:17 -07:00
|
|
|
* to have a refcount and cannot be freed.
|
|
|
|
*/
|
|
|
|
if (flags & BPF_LOCAL_STORAGE_GET_F_CREATE) {
|
|
|
|
sdata = bpf_local_storage_update(
|
|
|
|
inode, (struct bpf_local_storage_map *)map, value,
|
2022-03-17 21:55:52 -07:00
|
|
|
BPF_NOEXIST, gfp_flags);
|
2020-08-25 11:29:17 -07:00
|
|
|
return IS_ERR(sdata) ? (unsigned long)NULL :
|
|
|
|
(unsigned long)sdata->data;
|
|
|
|
}
|
|
|
|
|
|
|
|
return (unsigned long)NULL;
|
|
|
|
}
|
|
|
|
|
|
|
|
BPF_CALL_2(bpf_inode_storage_delete,
|
|
|
|
struct bpf_map *, map, struct inode *, inode)
|
|
|
|
{
|
2021-12-24 08:29:15 -07:00
|
|
|
WARN_ON_ONCE(!bpf_rcu_lock_held());
|
2021-01-12 00:55:24 -07:00
|
|
|
if (!inode)
|
|
|
|
return -EINVAL;
|
|
|
|
|
2021-01-12 00:55:25 -07:00
|
|
|
/* This helper must only called from where the inode is guaranteed
|
2020-08-25 11:29:17 -07:00
|
|
|
* to have a refcount and cannot be freed.
|
|
|
|
*/
|
|
|
|
return inode_storage_delete(inode, map);
|
|
|
|
}
|
|
|
|
|
|
|
|
static int notsupp_get_next_key(struct bpf_map *map, void *key,
|
|
|
|
void *next_key)
|
|
|
|
{
|
|
|
|
return -ENOTSUPP;
|
|
|
|
}
|
|
|
|
|
|
|
|
static struct bpf_map *inode_storage_map_alloc(union bpf_attr *attr)
|
|
|
|
{
|
bpf: Use bpf_mem_cache_alloc/free in bpf_local_storage_elem
This patch uses bpf_mem_alloc for the task and cgroup local storage that
the bpf prog can easily get a hold of the storage owner's PTR_TO_BTF_ID.
eg. bpf_get_current_task_btf() can be used in some of the kmalloc code
path which will cause deadlock/recursion. bpf_mem_cache_alloc is
deadlock free and will solve a legit use case in [1].
For sk storage, its batch creation benchmark shows a few percent
regression when the sk create/destroy batch size is larger than 32.
The sk creation/destruction happens much more often and
depends on external traffic. Considering it is hypothetical
to be able to cause deadlock with sk storage, it can cross
the bridge to use bpf_mem_alloc till a legit (ie. useful)
use case comes up.
For inode storage, bpf_local_storage_destroy() is called before
waiting for a rcu gp and its memory cannot be reused immediately.
inode stays with kmalloc/kfree after the rcu [or tasks_trace] gp.
A 'bool bpf_ma' argument is added to bpf_local_storage_map_alloc().
Only task and cgroup storage have 'bpf_ma == true' which
means to use bpf_mem_cache_alloc/free(). This patch only changes
selem to use bpf_mem_alloc for task and cgroup. The next patch
will change the local_storage to use bpf_mem_alloc also for
task and cgroup.
Here is some more details on the changes:
* memory allocation:
After bpf_mem_cache_alloc(), the SDATA(selem)->data is zero-ed because
bpf_mem_cache_alloc() could return a reused selem. It is to keep
the existing bpf_map_kzalloc() behavior. Only SDATA(selem)->data
is zero-ed. SDATA(selem)->data is the visible part to the bpf prog.
No need to use zero_map_value() to do the zeroing because
bpf_selem_free(..., reuse_now = true) ensures no bpf prog is using
the selem before returning the selem through bpf_mem_cache_free().
For the internal fields of selem, they will be initialized when
linking to the new smap and the new local_storage.
When 'bpf_ma == false', nothing changes in this patch. It will
stay with the bpf_map_kzalloc().
* memory free:
The bpf_selem_free() and bpf_selem_free_rcu() are modified to handle
the bpf_ma == true case.
For the common selem free path where its owner is also being destroyed,
the mem is freed in bpf_local_storage_destroy(), the owner (task
and cgroup) has gone through a rcu gp. The memory can be reused
immediately, so bpf_local_storage_destroy() will call
bpf_selem_free(..., reuse_now = true) which will do
bpf_mem_cache_free() for immediate reuse consideration.
An exception is the delete elem code path. The delete elem code path
is called from the helper bpf_*_storage_delete() and the syscall
bpf_map_delete_elem(). This path is an unusual case for local
storage because the common use case is to have the local storage
staying with its owner life time so that the bpf prog and the user
space does not have to monitor the owner's destruction. For the delete
elem path, the selem cannot be reused immediately because there could
be bpf prog using it. It will call bpf_selem_free(..., reuse_now = false)
and it will wait for a rcu tasks trace gp before freeing the elem. The
rcu callback is changed to do bpf_mem_cache_raw_free() instead of kfree().
When 'bpf_ma == false', it should be the same as before.
__bpf_selem_free() is added to do the kfree_rcu and call_tasks_trace_rcu().
A few words on the 'reuse_now == true'. When 'reuse_now == true',
it is still racing with bpf_local_storage_map_free which is under rcu
protection, so it still needs to wait for a rcu gp instead of kfree().
Otherwise, the selem may be reused by slab for a totally different struct
while the bpf_local_storage_map_free() is still using it (as a
rcu reader). For the inode case, there may be other rcu readers also.
In short, when bpf_ma == false and reuse_now == true => vanilla rcu.
[1]: https://lore.kernel.org/bpf/20221118190109.1512674-1-namhyung@kernel.org/
Cc: Namhyung Kim <namhyung@kernel.org>
Signed-off-by: Martin KaFai Lau <martin.lau@kernel.org>
Link: https://lore.kernel.org/r/20230322215246.1675516-3-martin.lau@linux.dev
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
2023-03-22 14:52:43 -07:00
|
|
|
return bpf_local_storage_map_alloc(attr, &inode_cache, false);
|
2020-08-25 11:29:17 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
static void inode_storage_map_free(struct bpf_map *map)
|
|
|
|
{
|
2022-10-25 21:28:45 -07:00
|
|
|
bpf_local_storage_map_free(map, &inode_cache, NULL);
|
2020-08-25 11:29:17 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
const struct bpf_map_ops inode_storage_map_ops = {
|
2020-08-27 18:18:06 -07:00
|
|
|
.map_meta_equal = bpf_map_meta_equal,
|
2020-08-25 11:29:17 -07:00
|
|
|
.map_alloc_check = bpf_local_storage_map_alloc_check,
|
|
|
|
.map_alloc = inode_storage_map_alloc,
|
|
|
|
.map_free = inode_storage_map_free,
|
|
|
|
.map_get_next_key = notsupp_get_next_key,
|
|
|
|
.map_lookup_elem = bpf_fd_inode_storage_lookup_elem,
|
|
|
|
.map_update_elem = bpf_fd_inode_storage_update_elem,
|
|
|
|
.map_delete_elem = bpf_fd_inode_storage_delete_elem,
|
|
|
|
.map_check_btf = bpf_local_storage_map_check_btf,
|
2023-03-05 05:46:11 -07:00
|
|
|
.map_mem_usage = bpf_local_storage_map_mem_usage,
|
2022-11-29 22:21:47 -07:00
|
|
|
.map_btf_id = &bpf_local_storage_map_btf_id[0],
|
2020-08-25 11:29:17 -07:00
|
|
|
.map_owner_storage_ptr = inode_storage_ptr,
|
|
|
|
};
|
|
|
|
|
2020-09-21 05:12:20 -07:00
|
|
|
BTF_ID_LIST_SINGLE(bpf_inode_storage_btf_ids, struct, inode)
|
2020-08-25 11:29:17 -07:00
|
|
|
|
|
|
|
const struct bpf_func_proto bpf_inode_storage_get_proto = {
|
|
|
|
.func = bpf_inode_storage_get,
|
|
|
|
.gpl_only = false,
|
|
|
|
.ret_type = RET_PTR_TO_MAP_VALUE_OR_NULL,
|
|
|
|
.arg1_type = ARG_CONST_MAP_PTR,
|
2023-04-03 21:50:25 -07:00
|
|
|
.arg2_type = ARG_PTR_TO_BTF_ID_OR_NULL,
|
2020-09-21 05:12:20 -07:00
|
|
|
.arg2_btf_id = &bpf_inode_storage_btf_ids[0],
|
2020-08-25 11:29:17 -07:00
|
|
|
.arg3_type = ARG_PTR_TO_MAP_VALUE_OR_NULL,
|
|
|
|
.arg4_type = ARG_ANYTHING,
|
|
|
|
};
|
|
|
|
|
|
|
|
const struct bpf_func_proto bpf_inode_storage_delete_proto = {
|
|
|
|
.func = bpf_inode_storage_delete,
|
|
|
|
.gpl_only = false,
|
|
|
|
.ret_type = RET_INTEGER,
|
|
|
|
.arg1_type = ARG_CONST_MAP_PTR,
|
2023-04-03 21:50:25 -07:00
|
|
|
.arg2_type = ARG_PTR_TO_BTF_ID_OR_NULL,
|
2020-09-21 05:12:20 -07:00
|
|
|
.arg2_btf_id = &bpf_inode_storage_btf_ids[0],
|
2020-08-25 11:29:17 -07:00
|
|
|
};
|