2865baf540
The Spectre-v1 mitigations made "access_ok()" much more expensive, since it has to serialize execution with the test for a valid user address. All the normal user copy routines avoid this by just masking the user address with a data-dependent mask instead, but the fast "unsafe_user_read()" kind of patterms that were supposed to be a fast case got slowed down. This introduces a notion of using src = masked_user_access_begin(src); to do the user address sanity using a data-dependent mask instead of the more traditional conditional if (user_read_access_begin(src, len)) { model. This model only works for dense accesses that start at 'src' and on architectures that have a guard region that is guaranteed to fault in between the user space and the kernel space area. With this, the user access doesn't need to be manually checked, because a bad address is guaranteed to fault (by some architecture masking trick: on x86-64 this involves just turning an invalid user address into all ones, since we don't map the top of address space). This only converts a couple of examples for now. Example x86-64 code generation for loading two words from user space: stac mov %rax,%rcx sar $0x3f,%rcx or %rax,%rcx mov (%rcx),%r13 mov 0x8(%rcx),%r14 clac where all the error handling and -EFAULT is now purely handled out of line by the exception path. Of course, if the micro-architecture does badly at 'clac' and 'stac', the above is still pitifully slow. But at least we did as well as we could. Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
130 lines
3.5 KiB
C
130 lines
3.5 KiB
C
// SPDX-License-Identifier: GPL-2.0
|
|
#include <linux/kernel.h>
|
|
#include <linux/export.h>
|
|
#include <linux/uaccess.h>
|
|
#include <linux/mm.h>
|
|
#include <linux/bitops.h>
|
|
|
|
#include <asm/word-at-a-time.h>
|
|
|
|
/*
|
|
* Do a strnlen, return length of string *with* final '\0'.
|
|
* 'count' is the user-supplied count, while 'max' is the
|
|
* address space maximum.
|
|
*
|
|
* Return 0 for exceptions (which includes hitting the address
|
|
* space maximum), or 'count+1' if hitting the user-supplied
|
|
* maximum count.
|
|
*
|
|
* NOTE! We can sometimes overshoot the user-supplied maximum
|
|
* if it fits in a aligned 'long'. The caller needs to check
|
|
* the return value against "> max".
|
|
*/
|
|
static __always_inline long do_strnlen_user(const char __user *src, unsigned long count, unsigned long max)
|
|
{
|
|
const struct word_at_a_time constants = WORD_AT_A_TIME_CONSTANTS;
|
|
unsigned long align, res = 0;
|
|
unsigned long c;
|
|
|
|
/*
|
|
* Do everything aligned. But that means that we
|
|
* need to also expand the maximum..
|
|
*/
|
|
align = (sizeof(unsigned long) - 1) & (unsigned long)src;
|
|
src -= align;
|
|
max += align;
|
|
|
|
unsafe_get_user(c, (unsigned long __user *)src, efault);
|
|
c |= aligned_byte_mask(align);
|
|
|
|
for (;;) {
|
|
unsigned long data;
|
|
if (has_zero(c, &data, &constants)) {
|
|
data = prep_zero_mask(c, data, &constants);
|
|
data = create_zero_mask(data);
|
|
return res + find_zero(data) + 1 - align;
|
|
}
|
|
res += sizeof(unsigned long);
|
|
/* We already handled 'unsigned long' bytes. Did we do it all ? */
|
|
if (unlikely(max <= sizeof(unsigned long)))
|
|
break;
|
|
max -= sizeof(unsigned long);
|
|
unsafe_get_user(c, (unsigned long __user *)(src+res), efault);
|
|
}
|
|
res -= align;
|
|
|
|
/*
|
|
* Uhhuh. We hit 'max'. But was that the user-specified maximum
|
|
* too? If so, return the marker for "too long".
|
|
*/
|
|
if (res >= count)
|
|
return count+1;
|
|
|
|
/*
|
|
* Nope: we hit the address space limit, and we still had more
|
|
* characters the caller would have wanted. That's 0.
|
|
*/
|
|
efault:
|
|
return 0;
|
|
}
|
|
|
|
/**
|
|
* strnlen_user: - Get the size of a user string INCLUDING final NUL.
|
|
* @str: The string to measure.
|
|
* @count: Maximum count (including NUL character)
|
|
*
|
|
* Context: User context only. This function may sleep if pagefaults are
|
|
* enabled.
|
|
*
|
|
* Get the size of a NUL-terminated string in user space.
|
|
*
|
|
* Returns the size of the string INCLUDING the terminating NUL.
|
|
* If the string is too long, returns a number larger than @count. User
|
|
* has to check the return value against "> count".
|
|
* On exception (or invalid count), returns 0.
|
|
*
|
|
* NOTE! You should basically never use this function. There is
|
|
* almost never any valid case for using the length of a user space
|
|
* string, since the string can be changed at any time by other
|
|
* threads. Use "strncpy_from_user()" instead to get a stable copy
|
|
* of the string.
|
|
*/
|
|
long strnlen_user(const char __user *str, long count)
|
|
{
|
|
unsigned long max_addr, src_addr;
|
|
|
|
if (unlikely(count <= 0))
|
|
return 0;
|
|
|
|
if (can_do_masked_user_access()) {
|
|
long retval;
|
|
|
|
str = masked_user_access_begin(str);
|
|
retval = do_strnlen_user(str, count, count);
|
|
user_read_access_end();
|
|
return retval;
|
|
}
|
|
|
|
max_addr = TASK_SIZE_MAX;
|
|
src_addr = (unsigned long)untagged_addr(str);
|
|
if (likely(src_addr < max_addr)) {
|
|
unsigned long max = max_addr - src_addr;
|
|
long retval;
|
|
|
|
/*
|
|
* Truncate 'max' to the user-specified limit, so that
|
|
* we only have one limit we need to check in the loop
|
|
*/
|
|
if (max > count)
|
|
max = count;
|
|
|
|
if (user_read_access_begin(str, max)) {
|
|
retval = do_strnlen_user(str, count, max);
|
|
user_read_access_end();
|
|
return retval;
|
|
}
|
|
}
|
|
return 0;
|
|
}
|
|
EXPORT_SYMBOL(strnlen_user);
|