bitmap-for-6.12
- switch all bitmamp APIs from inline to __always_inline from Brian Norris; - introduce GENMASK_U128() macro from Anshuman Khandual; -----BEGIN PGP SIGNATURE----- iQGzBAABCgAdFiEEi8GdvG6xMhdgpu/4sUSA/TofvsgFAmb22isACgkQsUSA/Tof vsie2gwAl3l5vye90xnD6N8wFmKBKAWXMn8Iby7JyM9gAn6j1QuE5AppS+3JtIpZ rPRSgFZIVPOgBtiKjb6zAWj7KbtCmaSW+L5ZVaLQ+vtwBVNpWIWHsHKu0uIpuugT 3wp/IeaE92bc/mioqb27pj2Gnv+lzYBmbK7Mu08a3q1Adwv0I7BJ4GvqxN1lLAEW xrFB86xztqdV7QC45J7Q5nIyUw7UBYK078elQ8iKSj5BR8MeaEJiavETwx9DHgAO Z8cG94ek3IpvLpiexNcgG+FTezZj9PnTVHxry9o7CIctafiqjYqXAJ9gks1Q4QUu q1IjPAdueLTAMPkpK67sI3fwC6zPyX5d8DVDUTuA6qhCsMyHW687gTRy4LPR14LL gd1Tzg+J9DQ5KBoG4TYN/g5VoP1hkKQqpetaJhdPqmYocfmqZuzyItb+gBjhyvSp 3YOgLg/4lULy3sZ6Qd/q8CWglWlaNYXXzf13H8f2qUpVx4NLTDOwjj/CVjZR/D0C wje/8XU3 =8jNc -----END PGP SIGNATURE----- Merge tag 'bitmap-for-6.12' of https://github.com/norov/linux Pull bitmap updates from Yury Norov: - switch all bitmamp APIs from inline to __always_inline (Brian Norris) The __always_inline series improves on code generation, and now with the latest compiler versions is required to avoid compilation warnings. It spent enough in my backlog, and I'm thankful to Brian Norris for taking over and moving it forward. - introduce GENMASK_U128() macro (Anshuman Khandual) GENMASK_U128() is a prerequisite needed for arm64 development * tag 'bitmap-for-6.12' of https://github.com/norov/linux: lib/test_bits.c: Add tests for GENMASK_U128() uapi: Define GENMASK_U128 nodemask: Switch from inline to __always_inline cpumask: Switch from inline to __always_inline bitmap: Switch from inline to __always_inline find: Switch from inline to __always_inline
This commit is contained in:
Linus Torvalds
commit
9c44575c78
@ -203,12 +203,12 @@ unsigned long bitmap_find_next_zero_area_off(unsigned long *map,
|
||||
* the bit offset of all zero areas this function finds is multiples of that
|
||||
* power of 2. A @align_mask of 0 means no alignment is required.
|
||||
*/
|
||||
static inline unsigned long
|
||||
bitmap_find_next_zero_area(unsigned long *map,
|
||||
unsigned long size,
|
||||
unsigned long start,
|
||||
unsigned int nr,
|
||||
unsigned long align_mask)
|
||||
static __always_inline
|
||||
unsigned long bitmap_find_next_zero_area(unsigned long *map,
|
||||
unsigned long size,
|
||||
unsigned long start,
|
||||
unsigned int nr,
|
||||
unsigned long align_mask)
|
||||
{
|
||||
return bitmap_find_next_zero_area_off(map, size, start, nr,
|
||||
align_mask, 0);
|
||||
@ -228,7 +228,7 @@ void bitmap_fold(unsigned long *dst, const unsigned long *orig,
|
||||
|
||||
#define bitmap_size(nbits) (ALIGN(nbits, BITS_PER_LONG) / BITS_PER_BYTE)
|
||||
|
||||
static inline void bitmap_zero(unsigned long *dst, unsigned int nbits)
|
||||
static __always_inline void bitmap_zero(unsigned long *dst, unsigned int nbits)
|
||||
{
|
||||
unsigned int len = bitmap_size(nbits);
|
||||
|
||||
@ -238,7 +238,7 @@ static inline void bitmap_zero(unsigned long *dst, unsigned int nbits)
|
||||
memset(dst, 0, len);
|
||||
}
|
||||
|
||||
static inline void bitmap_fill(unsigned long *dst, unsigned int nbits)
|
||||
static __always_inline void bitmap_fill(unsigned long *dst, unsigned int nbits)
|
||||
{
|
||||
unsigned int len = bitmap_size(nbits);
|
||||
|
||||
@ -248,8 +248,8 @@ static inline void bitmap_fill(unsigned long *dst, unsigned int nbits)
|
||||
memset(dst, 0xff, len);
|
||||
}
|
||||
|
||||
static inline void bitmap_copy(unsigned long *dst, const unsigned long *src,
|
||||
unsigned int nbits)
|
||||
static __always_inline
|
||||
void bitmap_copy(unsigned long *dst, const unsigned long *src, unsigned int nbits)
|
||||
{
|
||||
unsigned int len = bitmap_size(nbits);
|
||||
|
||||
@ -262,8 +262,8 @@ static inline void bitmap_copy(unsigned long *dst, const unsigned long *src,
|
||||
/*
|
||||
* Copy bitmap and clear tail bits in last word.
|
||||
*/
|
||||
static inline void bitmap_copy_clear_tail(unsigned long *dst,
|
||||
const unsigned long *src, unsigned int nbits)
|
||||
static __always_inline
|
||||
void bitmap_copy_clear_tail(unsigned long *dst, const unsigned long *src, unsigned int nbits)
|
||||
{
|
||||
bitmap_copy(dst, src, nbits);
|
||||
if (nbits % BITS_PER_LONG)
|
||||
@ -318,16 +318,18 @@ void bitmap_to_arr64(u64 *buf, const unsigned long *bitmap, unsigned int nbits);
|
||||
bitmap_copy_clear_tail((unsigned long *)(buf), (const unsigned long *)(bitmap), (nbits))
|
||||
#endif
|
||||
|
||||
static inline bool bitmap_and(unsigned long *dst, const unsigned long *src1,
|
||||
const unsigned long *src2, unsigned int nbits)
|
||||
static __always_inline
|
||||
bool bitmap_and(unsigned long *dst, const unsigned long *src1,
|
||||
const unsigned long *src2, unsigned int nbits)
|
||||
{
|
||||
if (small_const_nbits(nbits))
|
||||
return (*dst = *src1 & *src2 & BITMAP_LAST_WORD_MASK(nbits)) != 0;
|
||||
return __bitmap_and(dst, src1, src2, nbits);
|
||||
}
|
||||
|
||||
static inline void bitmap_or(unsigned long *dst, const unsigned long *src1,
|
||||
const unsigned long *src2, unsigned int nbits)
|
||||
static __always_inline
|
||||
void bitmap_or(unsigned long *dst, const unsigned long *src1,
|
||||
const unsigned long *src2, unsigned int nbits)
|
||||
{
|
||||
if (small_const_nbits(nbits))
|
||||
*dst = *src1 | *src2;
|
||||
@ -335,8 +337,9 @@ static inline void bitmap_or(unsigned long *dst, const unsigned long *src1,
|
||||
__bitmap_or(dst, src1, src2, nbits);
|
||||
}
|
||||
|
||||
static inline void bitmap_xor(unsigned long *dst, const unsigned long *src1,
|
||||
const unsigned long *src2, unsigned int nbits)
|
||||
static __always_inline
|
||||
void bitmap_xor(unsigned long *dst, const unsigned long *src1,
|
||||
const unsigned long *src2, unsigned int nbits)
|
||||
{
|
||||
if (small_const_nbits(nbits))
|
||||
*dst = *src1 ^ *src2;
|
||||
@ -344,16 +347,17 @@ static inline void bitmap_xor(unsigned long *dst, const unsigned long *src1,
|
||||
__bitmap_xor(dst, src1, src2, nbits);
|
||||
}
|
||||
|
||||
static inline bool bitmap_andnot(unsigned long *dst, const unsigned long *src1,
|
||||
const unsigned long *src2, unsigned int nbits)
|
||||
static __always_inline
|
||||
bool bitmap_andnot(unsigned long *dst, const unsigned long *src1,
|
||||
const unsigned long *src2, unsigned int nbits)
|
||||
{
|
||||
if (small_const_nbits(nbits))
|
||||
return (*dst = *src1 & ~(*src2) & BITMAP_LAST_WORD_MASK(nbits)) != 0;
|
||||
return __bitmap_andnot(dst, src1, src2, nbits);
|
||||
}
|
||||
|
||||
static inline void bitmap_complement(unsigned long *dst, const unsigned long *src,
|
||||
unsigned int nbits)
|
||||
static __always_inline
|
||||
void bitmap_complement(unsigned long *dst, const unsigned long *src, unsigned int nbits)
|
||||
{
|
||||
if (small_const_nbits(nbits))
|
||||
*dst = ~(*src);
|
||||
@ -368,8 +372,8 @@ static inline void bitmap_complement(unsigned long *dst, const unsigned long *sr
|
||||
#endif
|
||||
#define BITMAP_MEM_MASK (BITMAP_MEM_ALIGNMENT - 1)
|
||||
|
||||
static inline bool bitmap_equal(const unsigned long *src1,
|
||||
const unsigned long *src2, unsigned int nbits)
|
||||
static __always_inline
|
||||
bool bitmap_equal(const unsigned long *src1, const unsigned long *src2, unsigned int nbits)
|
||||
{
|
||||
if (small_const_nbits(nbits))
|
||||
return !((*src1 ^ *src2) & BITMAP_LAST_WORD_MASK(nbits));
|
||||
@ -388,10 +392,9 @@ static inline bool bitmap_equal(const unsigned long *src1,
|
||||
*
|
||||
* Returns: True if (*@src1 | *@src2) == *@src3, false otherwise
|
||||
*/
|
||||
static inline bool bitmap_or_equal(const unsigned long *src1,
|
||||
const unsigned long *src2,
|
||||
const unsigned long *src3,
|
||||
unsigned int nbits)
|
||||
static __always_inline
|
||||
bool bitmap_or_equal(const unsigned long *src1, const unsigned long *src2,
|
||||
const unsigned long *src3, unsigned int nbits)
|
||||
{
|
||||
if (!small_const_nbits(nbits))
|
||||
return __bitmap_or_equal(src1, src2, src3, nbits);
|
||||
@ -399,9 +402,8 @@ static inline bool bitmap_or_equal(const unsigned long *src1,
|
||||
return !(((*src1 | *src2) ^ *src3) & BITMAP_LAST_WORD_MASK(nbits));
|
||||
}
|
||||
|
||||
static inline bool bitmap_intersects(const unsigned long *src1,
|
||||
const unsigned long *src2,
|
||||
unsigned int nbits)
|
||||
static __always_inline
|
||||
bool bitmap_intersects(const unsigned long *src1, const unsigned long *src2, unsigned int nbits)
|
||||
{
|
||||
if (small_const_nbits(nbits))
|
||||
return ((*src1 & *src2) & BITMAP_LAST_WORD_MASK(nbits)) != 0;
|
||||
@ -409,8 +411,8 @@ static inline bool bitmap_intersects(const unsigned long *src1,
|
||||
return __bitmap_intersects(src1, src2, nbits);
|
||||
}
|
||||
|
||||
static inline bool bitmap_subset(const unsigned long *src1,
|
||||
const unsigned long *src2, unsigned int nbits)
|
||||
static __always_inline
|
||||
bool bitmap_subset(const unsigned long *src1, const unsigned long *src2, unsigned int nbits)
|
||||
{
|
||||
if (small_const_nbits(nbits))
|
||||
return ! ((*src1 & ~(*src2)) & BITMAP_LAST_WORD_MASK(nbits));
|
||||
@ -418,7 +420,8 @@ static inline bool bitmap_subset(const unsigned long *src1,
|
||||
return __bitmap_subset(src1, src2, nbits);
|
||||
}
|
||||
|
||||
static inline bool bitmap_empty(const unsigned long *src, unsigned nbits)
|
||||
static __always_inline
|
||||
bool bitmap_empty(const unsigned long *src, unsigned nbits)
|
||||
{
|
||||
if (small_const_nbits(nbits))
|
||||
return ! (*src & BITMAP_LAST_WORD_MASK(nbits));
|
||||
@ -426,7 +429,8 @@ static inline bool bitmap_empty(const unsigned long *src, unsigned nbits)
|
||||
return find_first_bit(src, nbits) == nbits;
|
||||
}
|
||||
|
||||
static inline bool bitmap_full(const unsigned long *src, unsigned int nbits)
|
||||
static __always_inline
|
||||
bool bitmap_full(const unsigned long *src, unsigned int nbits)
|
||||
{
|
||||
if (small_const_nbits(nbits))
|
||||
return ! (~(*src) & BITMAP_LAST_WORD_MASK(nbits));
|
||||
@ -460,8 +464,8 @@ unsigned long bitmap_weight_andnot(const unsigned long *src1,
|
||||
return __bitmap_weight_andnot(src1, src2, nbits);
|
||||
}
|
||||
|
||||
static __always_inline void bitmap_set(unsigned long *map, unsigned int start,
|
||||
unsigned int nbits)
|
||||
static __always_inline
|
||||
void bitmap_set(unsigned long *map, unsigned int start, unsigned int nbits)
|
||||
{
|
||||
if (__builtin_constant_p(nbits) && nbits == 1)
|
||||
__set_bit(start, map);
|
||||
@ -476,8 +480,8 @@ static __always_inline void bitmap_set(unsigned long *map, unsigned int start,
|
||||
__bitmap_set(map, start, nbits);
|
||||
}
|
||||
|
||||
static __always_inline void bitmap_clear(unsigned long *map, unsigned int start,
|
||||
unsigned int nbits)
|
||||
static __always_inline
|
||||
void bitmap_clear(unsigned long *map, unsigned int start, unsigned int nbits)
|
||||
{
|
||||
if (__builtin_constant_p(nbits) && nbits == 1)
|
||||
__clear_bit(start, map);
|
||||
@ -492,8 +496,9 @@ static __always_inline void bitmap_clear(unsigned long *map, unsigned int start,
|
||||
__bitmap_clear(map, start, nbits);
|
||||
}
|
||||
|
||||
static inline void bitmap_shift_right(unsigned long *dst, const unsigned long *src,
|
||||
unsigned int shift, unsigned int nbits)
|
||||
static __always_inline
|
||||
void bitmap_shift_right(unsigned long *dst, const unsigned long *src,
|
||||
unsigned int shift, unsigned int nbits)
|
||||
{
|
||||
if (small_const_nbits(nbits))
|
||||
*dst = (*src & BITMAP_LAST_WORD_MASK(nbits)) >> shift;
|
||||
@ -501,8 +506,9 @@ static inline void bitmap_shift_right(unsigned long *dst, const unsigned long *s
|
||||
__bitmap_shift_right(dst, src, shift, nbits);
|
||||
}
|
||||
|
||||
static inline void bitmap_shift_left(unsigned long *dst, const unsigned long *src,
|
||||
unsigned int shift, unsigned int nbits)
|
||||
static __always_inline
|
||||
void bitmap_shift_left(unsigned long *dst, const unsigned long *src,
|
||||
unsigned int shift, unsigned int nbits)
|
||||
{
|
||||
if (small_const_nbits(nbits))
|
||||
*dst = (*src << shift) & BITMAP_LAST_WORD_MASK(nbits);
|
||||
@ -510,11 +516,12 @@ static inline void bitmap_shift_left(unsigned long *dst, const unsigned long *sr
|
||||
__bitmap_shift_left(dst, src, shift, nbits);
|
||||
}
|
||||
|
||||
static inline void bitmap_replace(unsigned long *dst,
|
||||
const unsigned long *old,
|
||||
const unsigned long *new,
|
||||
const unsigned long *mask,
|
||||
unsigned int nbits)
|
||||
static __always_inline
|
||||
void bitmap_replace(unsigned long *dst,
|
||||
const unsigned long *old,
|
||||
const unsigned long *new,
|
||||
const unsigned long *mask,
|
||||
unsigned int nbits)
|
||||
{
|
||||
if (small_const_nbits(nbits))
|
||||
*dst = (*old & ~(*mask)) | (*new & *mask);
|
||||
@ -557,8 +564,9 @@ static inline void bitmap_replace(unsigned long *dst,
|
||||
* bitmap_gather() can be seen as the 'reverse' bitmap_scatter() operation.
|
||||
* See bitmap_scatter() for details related to this relationship.
|
||||
*/
|
||||
static inline void bitmap_scatter(unsigned long *dst, const unsigned long *src,
|
||||
const unsigned long *mask, unsigned int nbits)
|
||||
static __always_inline
|
||||
void bitmap_scatter(unsigned long *dst, const unsigned long *src,
|
||||
const unsigned long *mask, unsigned int nbits)
|
||||
{
|
||||
unsigned int n = 0;
|
||||
unsigned int bit;
|
||||
@ -611,8 +619,9 @@ static inline void bitmap_scatter(unsigned long *dst, const unsigned long *src,
|
||||
* bitmap_scatter(res, src, mask, n) and a call to
|
||||
* bitmap_scatter(res, result, mask, n) will lead to the same res value.
|
||||
*/
|
||||
static inline void bitmap_gather(unsigned long *dst, const unsigned long *src,
|
||||
const unsigned long *mask, unsigned int nbits)
|
||||
static __always_inline
|
||||
void bitmap_gather(unsigned long *dst, const unsigned long *src,
|
||||
const unsigned long *mask, unsigned int nbits)
|
||||
{
|
||||
unsigned int n = 0;
|
||||
unsigned int bit;
|
||||
@ -623,9 +632,9 @@ static inline void bitmap_gather(unsigned long *dst, const unsigned long *src,
|
||||
__assign_bit(n++, dst, test_bit(bit, src));
|
||||
}
|
||||
|
||||
static inline void bitmap_next_set_region(unsigned long *bitmap,
|
||||
unsigned int *rs, unsigned int *re,
|
||||
unsigned int end)
|
||||
static __always_inline
|
||||
void bitmap_next_set_region(unsigned long *bitmap, unsigned int *rs,
|
||||
unsigned int *re, unsigned int end)
|
||||
{
|
||||
*rs = find_next_bit(bitmap, end, *rs);
|
||||
*re = find_next_zero_bit(bitmap, end, *rs + 1);
|
||||
@ -640,7 +649,8 @@ static inline void bitmap_next_set_region(unsigned long *bitmap,
|
||||
* This is the complement to __bitmap_find_free_region() and releases
|
||||
* the found region (by clearing it in the bitmap).
|
||||
*/
|
||||
static inline void bitmap_release_region(unsigned long *bitmap, unsigned int pos, int order)
|
||||
static __always_inline
|
||||
void bitmap_release_region(unsigned long *bitmap, unsigned int pos, int order)
|
||||
{
|
||||
bitmap_clear(bitmap, pos, BIT(order));
|
||||
}
|
||||
@ -656,7 +666,8 @@ static inline void bitmap_release_region(unsigned long *bitmap, unsigned int pos
|
||||
* Returns: 0 on success, or %-EBUSY if specified region wasn't
|
||||
* free (not all bits were zero).
|
||||
*/
|
||||
static inline int bitmap_allocate_region(unsigned long *bitmap, unsigned int pos, int order)
|
||||
static __always_inline
|
||||
int bitmap_allocate_region(unsigned long *bitmap, unsigned int pos, int order)
|
||||
{
|
||||
unsigned int len = BIT(order);
|
||||
|
||||
@ -680,7 +691,8 @@ static inline int bitmap_allocate_region(unsigned long *bitmap, unsigned int pos
|
||||
* Returns: the bit offset in bitmap of the allocated region,
|
||||
* or -errno on failure.
|
||||
*/
|
||||
static inline int bitmap_find_free_region(unsigned long *bitmap, unsigned int bits, int order)
|
||||
static __always_inline
|
||||
int bitmap_find_free_region(unsigned long *bitmap, unsigned int bits, int order)
|
||||
{
|
||||
unsigned int pos, end; /* scans bitmap by regions of size order */
|
||||
|
||||
@ -734,7 +746,7 @@ static inline int bitmap_find_free_region(unsigned long *bitmap, unsigned int bi
|
||||
* That is ``(u32 *)(&val)[0]`` gets the upper 32 bits,
|
||||
* but we expect the lower 32-bits of u64.
|
||||
*/
|
||||
static inline void bitmap_from_u64(unsigned long *dst, u64 mask)
|
||||
static __always_inline void bitmap_from_u64(unsigned long *dst, u64 mask)
|
||||
{
|
||||
bitmap_from_arr64(dst, &mask, 64);
|
||||
}
|
||||
@ -749,9 +761,8 @@ static inline void bitmap_from_u64(unsigned long *dst, u64 mask)
|
||||
* @map memory region. For @nbits = 0 and @nbits > BITS_PER_LONG the return
|
||||
* value is undefined.
|
||||
*/
|
||||
static inline unsigned long bitmap_read(const unsigned long *map,
|
||||
unsigned long start,
|
||||
unsigned long nbits)
|
||||
static __always_inline
|
||||
unsigned long bitmap_read(const unsigned long *map, unsigned long start, unsigned long nbits)
|
||||
{
|
||||
size_t index = BIT_WORD(start);
|
||||
unsigned long offset = start % BITS_PER_LONG;
|
||||
@ -784,8 +795,9 @@ static inline unsigned long bitmap_read(const unsigned long *map,
|
||||
*
|
||||
* For @nbits == 0 and @nbits > BITS_PER_LONG no writes are performed.
|
||||
*/
|
||||
static inline void bitmap_write(unsigned long *map, unsigned long value,
|
||||
unsigned long start, unsigned long nbits)
|
||||
static __always_inline
|
||||
void bitmap_write(unsigned long *map, unsigned long value,
|
||||
unsigned long start, unsigned long nbits)
|
||||
{
|
||||
size_t index;
|
||||
unsigned long offset;
|
||||
|
||||
@ -36,4 +36,19 @@
|
||||
#define GENMASK_ULL(h, l) \
|
||||
(GENMASK_INPUT_CHECK(h, l) + __GENMASK_ULL(h, l))
|
||||
|
||||
#if !defined(__ASSEMBLY__)
|
||||
/*
|
||||
* Missing asm support
|
||||
*
|
||||
* __GENMASK_U128() depends on _BIT128() which would not work
|
||||
* in the asm code, as it shifts an 'unsigned __init128' data
|
||||
* type instead of direct representation of 128 bit constants
|
||||
* such as long and unsigned long. The fundamental problem is
|
||||
* that a 128 bit constant will get silently truncated by the
|
||||
* gcc compiler.
|
||||
*/
|
||||
#define GENMASK_U128(h, l) \
|
||||
(GENMASK_INPUT_CHECK(h, l) + __GENMASK_U128(h, l))
|
||||
#endif
|
||||
|
||||
#endif /* __LINUX_BITS_H */
|
||||
|
||||
@ -30,7 +30,7 @@
|
||||
extern unsigned int nr_cpu_ids;
|
||||
#endif
|
||||
|
||||
static inline void set_nr_cpu_ids(unsigned int nr)
|
||||
static __always_inline void set_nr_cpu_ids(unsigned int nr)
|
||||
{
|
||||
#if (NR_CPUS == 1) || defined(CONFIG_FORCE_NR_CPUS)
|
||||
WARN_ON(nr != nr_cpu_ids);
|
||||
@ -149,7 +149,7 @@ static __always_inline unsigned int cpumask_check(unsigned int cpu)
|
||||
*
|
||||
* Return: >= nr_cpu_ids if no cpus set.
|
||||
*/
|
||||
static inline unsigned int cpumask_first(const struct cpumask *srcp)
|
||||
static __always_inline unsigned int cpumask_first(const struct cpumask *srcp)
|
||||
{
|
||||
return find_first_bit(cpumask_bits(srcp), small_cpumask_bits);
|
||||
}
|
||||
@ -160,7 +160,7 @@ static inline unsigned int cpumask_first(const struct cpumask *srcp)
|
||||
*
|
||||
* Return: >= nr_cpu_ids if all cpus are set.
|
||||
*/
|
||||
static inline unsigned int cpumask_first_zero(const struct cpumask *srcp)
|
||||
static __always_inline unsigned int cpumask_first_zero(const struct cpumask *srcp)
|
||||
{
|
||||
return find_first_zero_bit(cpumask_bits(srcp), small_cpumask_bits);
|
||||
}
|
||||
@ -172,7 +172,7 @@ static inline unsigned int cpumask_first_zero(const struct cpumask *srcp)
|
||||
*
|
||||
* Return: >= nr_cpu_ids if no cpus set in both. See also cpumask_next_and().
|
||||
*/
|
||||
static inline
|
||||
static __always_inline
|
||||
unsigned int cpumask_first_and(const struct cpumask *srcp1, const struct cpumask *srcp2)
|
||||
{
|
||||
return find_first_and_bit(cpumask_bits(srcp1), cpumask_bits(srcp2), small_cpumask_bits);
|
||||
@ -186,7 +186,7 @@ unsigned int cpumask_first_and(const struct cpumask *srcp1, const struct cpumask
|
||||
*
|
||||
* Return: >= nr_cpu_ids if no cpus set in all.
|
||||
*/
|
||||
static inline
|
||||
static __always_inline
|
||||
unsigned int cpumask_first_and_and(const struct cpumask *srcp1,
|
||||
const struct cpumask *srcp2,
|
||||
const struct cpumask *srcp3)
|
||||
@ -201,7 +201,7 @@ unsigned int cpumask_first_and_and(const struct cpumask *srcp1,
|
||||
*
|
||||
* Return: >= nr_cpumask_bits if no CPUs set.
|
||||
*/
|
||||
static inline unsigned int cpumask_last(const struct cpumask *srcp)
|
||||
static __always_inline unsigned int cpumask_last(const struct cpumask *srcp)
|
||||
{
|
||||
return find_last_bit(cpumask_bits(srcp), small_cpumask_bits);
|
||||
}
|
||||
@ -213,7 +213,7 @@ static inline unsigned int cpumask_last(const struct cpumask *srcp)
|
||||
*
|
||||
* Return: >= nr_cpu_ids if no further cpus set.
|
||||
*/
|
||||
static inline
|
||||
static __always_inline
|
||||
unsigned int cpumask_next(int n, const struct cpumask *srcp)
|
||||
{
|
||||
/* -1 is a legal arg here. */
|
||||
@ -229,7 +229,8 @@ unsigned int cpumask_next(int n, const struct cpumask *srcp)
|
||||
*
|
||||
* Return: >= nr_cpu_ids if no further cpus unset.
|
||||
*/
|
||||
static inline unsigned int cpumask_next_zero(int n, const struct cpumask *srcp)
|
||||
static __always_inline
|
||||
unsigned int cpumask_next_zero(int n, const struct cpumask *srcp)
|
||||
{
|
||||
/* -1 is a legal arg here. */
|
||||
if (n != -1)
|
||||
@ -239,18 +240,21 @@ static inline unsigned int cpumask_next_zero(int n, const struct cpumask *srcp)
|
||||
|
||||
#if NR_CPUS == 1
|
||||
/* Uniprocessor: there is only one valid CPU */
|
||||
static inline unsigned int cpumask_local_spread(unsigned int i, int node)
|
||||
static __always_inline
|
||||
unsigned int cpumask_local_spread(unsigned int i, int node)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
static inline unsigned int cpumask_any_and_distribute(const struct cpumask *src1p,
|
||||
const struct cpumask *src2p)
|
||||
static __always_inline
|
||||
unsigned int cpumask_any_and_distribute(const struct cpumask *src1p,
|
||||
const struct cpumask *src2p)
|
||||
{
|
||||
return cpumask_first_and(src1p, src2p);
|
||||
}
|
||||
|
||||
static inline unsigned int cpumask_any_distribute(const struct cpumask *srcp)
|
||||
static __always_inline
|
||||
unsigned int cpumask_any_distribute(const struct cpumask *srcp)
|
||||
{
|
||||
return cpumask_first(srcp);
|
||||
}
|
||||
@ -269,9 +273,9 @@ unsigned int cpumask_any_distribute(const struct cpumask *srcp);
|
||||
*
|
||||
* Return: >= nr_cpu_ids if no further cpus set in both.
|
||||
*/
|
||||
static inline
|
||||
static __always_inline
|
||||
unsigned int cpumask_next_and(int n, const struct cpumask *src1p,
|
||||
const struct cpumask *src2p)
|
||||
const struct cpumask *src2p)
|
||||
{
|
||||
/* -1 is a legal arg here. */
|
||||
if (n != -1)
|
||||
@ -291,7 +295,7 @@ unsigned int cpumask_next_and(int n, const struct cpumask *src1p,
|
||||
for_each_set_bit(cpu, cpumask_bits(mask), small_cpumask_bits)
|
||||
|
||||
#if NR_CPUS == 1
|
||||
static inline
|
||||
static __always_inline
|
||||
unsigned int cpumask_next_wrap(int n, const struct cpumask *mask, int start, bool wrap)
|
||||
{
|
||||
cpumask_check(start);
|
||||
@ -394,7 +398,7 @@ unsigned int __pure cpumask_next_wrap(int n, const struct cpumask *mask, int sta
|
||||
* Often used to find any cpu but smp_processor_id() in a mask.
|
||||
* Return: >= nr_cpu_ids if no cpus set.
|
||||
*/
|
||||
static inline
|
||||
static __always_inline
|
||||
unsigned int cpumask_any_but(const struct cpumask *mask, unsigned int cpu)
|
||||
{
|
||||
unsigned int i;
|
||||
@ -414,7 +418,7 @@ unsigned int cpumask_any_but(const struct cpumask *mask, unsigned int cpu)
|
||||
*
|
||||
* Returns >= nr_cpu_ids if no cpus set.
|
||||
*/
|
||||
static inline
|
||||
static __always_inline
|
||||
unsigned int cpumask_any_and_but(const struct cpumask *mask1,
|
||||
const struct cpumask *mask2,
|
||||
unsigned int cpu)
|
||||
@ -436,7 +440,8 @@ unsigned int cpumask_any_and_but(const struct cpumask *mask1,
|
||||
*
|
||||
* Return: >= nr_cpu_ids if such cpu doesn't exist.
|
||||
*/
|
||||
static inline unsigned int cpumask_nth(unsigned int cpu, const struct cpumask *srcp)
|
||||
static __always_inline
|
||||
unsigned int cpumask_nth(unsigned int cpu, const struct cpumask *srcp)
|
||||
{
|
||||
return find_nth_bit(cpumask_bits(srcp), small_cpumask_bits, cpumask_check(cpu));
|
||||
}
|
||||
@ -449,7 +454,7 @@ static inline unsigned int cpumask_nth(unsigned int cpu, const struct cpumask *s
|
||||
*
|
||||
* Return: >= nr_cpu_ids if such cpu doesn't exist.
|
||||
*/
|
||||
static inline
|
||||
static __always_inline
|
||||
unsigned int cpumask_nth_and(unsigned int cpu, const struct cpumask *srcp1,
|
||||
const struct cpumask *srcp2)
|
||||
{
|
||||
@ -465,7 +470,7 @@ unsigned int cpumask_nth_and(unsigned int cpu, const struct cpumask *srcp1,
|
||||
*
|
||||
* Return: >= nr_cpu_ids if such cpu doesn't exist.
|
||||
*/
|
||||
static inline
|
||||
static __always_inline
|
||||
unsigned int cpumask_nth_andnot(unsigned int cpu, const struct cpumask *srcp1,
|
||||
const struct cpumask *srcp2)
|
||||
{
|
||||
@ -508,12 +513,14 @@ unsigned int cpumask_nth_and_andnot(unsigned int cpu, const struct cpumask *srcp
|
||||
* @cpu: cpu number (< nr_cpu_ids)
|
||||
* @dstp: the cpumask pointer
|
||||
*/
|
||||
static __always_inline void cpumask_set_cpu(unsigned int cpu, struct cpumask *dstp)
|
||||
static __always_inline
|
||||
void cpumask_set_cpu(unsigned int cpu, struct cpumask *dstp)
|
||||
{
|
||||
set_bit(cpumask_check(cpu), cpumask_bits(dstp));
|
||||
}
|
||||
|
||||
static __always_inline void __cpumask_set_cpu(unsigned int cpu, struct cpumask *dstp)
|
||||
static __always_inline
|
||||
void __cpumask_set_cpu(unsigned int cpu, struct cpumask *dstp)
|
||||
{
|
||||
__set_bit(cpumask_check(cpu), cpumask_bits(dstp));
|
||||
}
|
||||
@ -557,7 +564,8 @@ static __always_inline void __cpumask_assign_cpu(int cpu, struct cpumask *dstp,
|
||||
*
|
||||
* Return: true if @cpu is set in @cpumask, else returns false
|
||||
*/
|
||||
static __always_inline bool cpumask_test_cpu(int cpu, const struct cpumask *cpumask)
|
||||
static __always_inline
|
||||
bool cpumask_test_cpu(int cpu, const struct cpumask *cpumask)
|
||||
{
|
||||
return test_bit(cpumask_check(cpu), cpumask_bits((cpumask)));
|
||||
}
|
||||
@ -571,7 +579,8 @@ static __always_inline bool cpumask_test_cpu(int cpu, const struct cpumask *cpum
|
||||
*
|
||||
* Return: true if @cpu is set in old bitmap of @cpumask, else returns false
|
||||
*/
|
||||
static __always_inline bool cpumask_test_and_set_cpu(int cpu, struct cpumask *cpumask)
|
||||
static __always_inline
|
||||
bool cpumask_test_and_set_cpu(int cpu, struct cpumask *cpumask)
|
||||
{
|
||||
return test_and_set_bit(cpumask_check(cpu), cpumask_bits(cpumask));
|
||||
}
|
||||
@ -585,7 +594,8 @@ static __always_inline bool cpumask_test_and_set_cpu(int cpu, struct cpumask *cp
|
||||
*
|
||||
* Return: true if @cpu is set in old bitmap of @cpumask, else returns false
|
||||
*/
|
||||
static __always_inline bool cpumask_test_and_clear_cpu(int cpu, struct cpumask *cpumask)
|
||||
static __always_inline
|
||||
bool cpumask_test_and_clear_cpu(int cpu, struct cpumask *cpumask)
|
||||
{
|
||||
return test_and_clear_bit(cpumask_check(cpu), cpumask_bits(cpumask));
|
||||
}
|
||||
@ -594,7 +604,7 @@ static __always_inline bool cpumask_test_and_clear_cpu(int cpu, struct cpumask *
|
||||
* cpumask_setall - set all cpus (< nr_cpu_ids) in a cpumask
|
||||
* @dstp: the cpumask pointer
|
||||
*/
|
||||
static inline void cpumask_setall(struct cpumask *dstp)
|
||||
static __always_inline void cpumask_setall(struct cpumask *dstp)
|
||||
{
|
||||
if (small_const_nbits(small_cpumask_bits)) {
|
||||
cpumask_bits(dstp)[0] = BITMAP_LAST_WORD_MASK(nr_cpumask_bits);
|
||||
@ -607,7 +617,7 @@ static inline void cpumask_setall(struct cpumask *dstp)
|
||||
* cpumask_clear - clear all cpus (< nr_cpu_ids) in a cpumask
|
||||
* @dstp: the cpumask pointer
|
||||
*/
|
||||
static inline void cpumask_clear(struct cpumask *dstp)
|
||||
static __always_inline void cpumask_clear(struct cpumask *dstp)
|
||||
{
|
||||
bitmap_zero(cpumask_bits(dstp), large_cpumask_bits);
|
||||
}
|
||||
@ -620,9 +630,9 @@ static inline void cpumask_clear(struct cpumask *dstp)
|
||||
*
|
||||
* Return: false if *@dstp is empty, else returns true
|
||||
*/
|
||||
static inline bool cpumask_and(struct cpumask *dstp,
|
||||
const struct cpumask *src1p,
|
||||
const struct cpumask *src2p)
|
||||
static __always_inline
|
||||
bool cpumask_and(struct cpumask *dstp, const struct cpumask *src1p,
|
||||
const struct cpumask *src2p)
|
||||
{
|
||||
return bitmap_and(cpumask_bits(dstp), cpumask_bits(src1p),
|
||||
cpumask_bits(src2p), small_cpumask_bits);
|
||||
@ -634,8 +644,9 @@ static inline bool cpumask_and(struct cpumask *dstp,
|
||||
* @src1p: the first input
|
||||
* @src2p: the second input
|
||||
*/
|
||||
static inline void cpumask_or(struct cpumask *dstp, const struct cpumask *src1p,
|
||||
const struct cpumask *src2p)
|
||||
static __always_inline
|
||||
void cpumask_or(struct cpumask *dstp, const struct cpumask *src1p,
|
||||
const struct cpumask *src2p)
|
||||
{
|
||||
bitmap_or(cpumask_bits(dstp), cpumask_bits(src1p),
|
||||
cpumask_bits(src2p), small_cpumask_bits);
|
||||
@ -647,9 +658,9 @@ static inline void cpumask_or(struct cpumask *dstp, const struct cpumask *src1p,
|
||||
* @src1p: the first input
|
||||
* @src2p: the second input
|
||||
*/
|
||||
static inline void cpumask_xor(struct cpumask *dstp,
|
||||
const struct cpumask *src1p,
|
||||
const struct cpumask *src2p)
|
||||
static __always_inline
|
||||
void cpumask_xor(struct cpumask *dstp, const struct cpumask *src1p,
|
||||
const struct cpumask *src2p)
|
||||
{
|
||||
bitmap_xor(cpumask_bits(dstp), cpumask_bits(src1p),
|
||||
cpumask_bits(src2p), small_cpumask_bits);
|
||||
@ -663,9 +674,9 @@ static inline void cpumask_xor(struct cpumask *dstp,
|
||||
*
|
||||
* Return: false if *@dstp is empty, else returns true
|
||||
*/
|
||||
static inline bool cpumask_andnot(struct cpumask *dstp,
|
||||
const struct cpumask *src1p,
|
||||
const struct cpumask *src2p)
|
||||
static __always_inline
|
||||
bool cpumask_andnot(struct cpumask *dstp, const struct cpumask *src1p,
|
||||
const struct cpumask *src2p)
|
||||
{
|
||||
return bitmap_andnot(cpumask_bits(dstp), cpumask_bits(src1p),
|
||||
cpumask_bits(src2p), small_cpumask_bits);
|
||||
@ -678,8 +689,8 @@ static inline bool cpumask_andnot(struct cpumask *dstp,
|
||||
*
|
||||
* Return: true if the cpumasks are equal, false if not
|
||||
*/
|
||||
static inline bool cpumask_equal(const struct cpumask *src1p,
|
||||
const struct cpumask *src2p)
|
||||
static __always_inline
|
||||
bool cpumask_equal(const struct cpumask *src1p, const struct cpumask *src2p)
|
||||
{
|
||||
return bitmap_equal(cpumask_bits(src1p), cpumask_bits(src2p),
|
||||
small_cpumask_bits);
|
||||
@ -694,9 +705,9 @@ static inline bool cpumask_equal(const struct cpumask *src1p,
|
||||
* Return: true if first cpumask ORed with second cpumask == third cpumask,
|
||||
* otherwise false
|
||||
*/
|
||||
static inline bool cpumask_or_equal(const struct cpumask *src1p,
|
||||
const struct cpumask *src2p,
|
||||
const struct cpumask *src3p)
|
||||
static __always_inline
|
||||
bool cpumask_or_equal(const struct cpumask *src1p, const struct cpumask *src2p,
|
||||
const struct cpumask *src3p)
|
||||
{
|
||||
return bitmap_or_equal(cpumask_bits(src1p), cpumask_bits(src2p),
|
||||
cpumask_bits(src3p), small_cpumask_bits);
|
||||
@ -710,8 +721,8 @@ static inline bool cpumask_or_equal(const struct cpumask *src1p,
|
||||
* Return: true if first cpumask ANDed with second cpumask is non-empty,
|
||||
* otherwise false
|
||||
*/
|
||||
static inline bool cpumask_intersects(const struct cpumask *src1p,
|
||||
const struct cpumask *src2p)
|
||||
static __always_inline
|
||||
bool cpumask_intersects(const struct cpumask *src1p, const struct cpumask *src2p)
|
||||
{
|
||||
return bitmap_intersects(cpumask_bits(src1p), cpumask_bits(src2p),
|
||||
small_cpumask_bits);
|
||||
@ -724,8 +735,8 @@ static inline bool cpumask_intersects(const struct cpumask *src1p,
|
||||
*
|
||||
* Return: true if *@src1p is a subset of *@src2p, else returns false
|
||||
*/
|
||||
static inline bool cpumask_subset(const struct cpumask *src1p,
|
||||
const struct cpumask *src2p)
|
||||
static __always_inline
|
||||
bool cpumask_subset(const struct cpumask *src1p, const struct cpumask *src2p)
|
||||
{
|
||||
return bitmap_subset(cpumask_bits(src1p), cpumask_bits(src2p),
|
||||
small_cpumask_bits);
|
||||
@ -737,7 +748,7 @@ static inline bool cpumask_subset(const struct cpumask *src1p,
|
||||
*
|
||||
* Return: true if srcp is empty (has no bits set), else false
|
||||
*/
|
||||
static inline bool cpumask_empty(const struct cpumask *srcp)
|
||||
static __always_inline bool cpumask_empty(const struct cpumask *srcp)
|
||||
{
|
||||
return bitmap_empty(cpumask_bits(srcp), small_cpumask_bits);
|
||||
}
|
||||
@ -748,7 +759,7 @@ static inline bool cpumask_empty(const struct cpumask *srcp)
|
||||
*
|
||||
* Return: true if srcp is full (has all bits set), else false
|
||||
*/
|
||||
static inline bool cpumask_full(const struct cpumask *srcp)
|
||||
static __always_inline bool cpumask_full(const struct cpumask *srcp)
|
||||
{
|
||||
return bitmap_full(cpumask_bits(srcp), nr_cpumask_bits);
|
||||
}
|
||||
@ -759,7 +770,7 @@ static inline bool cpumask_full(const struct cpumask *srcp)
|
||||
*
|
||||
* Return: count of bits set in *srcp
|
||||
*/
|
||||
static inline unsigned int cpumask_weight(const struct cpumask *srcp)
|
||||
static __always_inline unsigned int cpumask_weight(const struct cpumask *srcp)
|
||||
{
|
||||
return bitmap_weight(cpumask_bits(srcp), small_cpumask_bits);
|
||||
}
|
||||
@ -771,8 +782,8 @@ static inline unsigned int cpumask_weight(const struct cpumask *srcp)
|
||||
*
|
||||
* Return: count of bits set in both *srcp1 and *srcp2
|
||||
*/
|
||||
static inline unsigned int cpumask_weight_and(const struct cpumask *srcp1,
|
||||
const struct cpumask *srcp2)
|
||||
static __always_inline
|
||||
unsigned int cpumask_weight_and(const struct cpumask *srcp1, const struct cpumask *srcp2)
|
||||
{
|
||||
return bitmap_weight_and(cpumask_bits(srcp1), cpumask_bits(srcp2), small_cpumask_bits);
|
||||
}
|
||||
@ -784,8 +795,9 @@ static inline unsigned int cpumask_weight_and(const struct cpumask *srcp1,
|
||||
*
|
||||
* Return: count of bits set in both *srcp1 and *srcp2
|
||||
*/
|
||||
static inline unsigned int cpumask_weight_andnot(const struct cpumask *srcp1,
|
||||
const struct cpumask *srcp2)
|
||||
static __always_inline
|
||||
unsigned int cpumask_weight_andnot(const struct cpumask *srcp1,
|
||||
const struct cpumask *srcp2)
|
||||
{
|
||||
return bitmap_weight_andnot(cpumask_bits(srcp1), cpumask_bits(srcp2), small_cpumask_bits);
|
||||
}
|
||||
@ -796,8 +808,8 @@ static inline unsigned int cpumask_weight_andnot(const struct cpumask *srcp1,
|
||||
* @srcp: the input to shift
|
||||
* @n: the number of bits to shift by
|
||||
*/
|
||||
static inline void cpumask_shift_right(struct cpumask *dstp,
|
||||
const struct cpumask *srcp, int n)
|
||||
static __always_inline
|
||||
void cpumask_shift_right(struct cpumask *dstp, const struct cpumask *srcp, int n)
|
||||
{
|
||||
bitmap_shift_right(cpumask_bits(dstp), cpumask_bits(srcp), n,
|
||||
small_cpumask_bits);
|
||||
@ -809,8 +821,8 @@ static inline void cpumask_shift_right(struct cpumask *dstp,
|
||||
* @srcp: the input to shift
|
||||
* @n: the number of bits to shift by
|
||||
*/
|
||||
static inline void cpumask_shift_left(struct cpumask *dstp,
|
||||
const struct cpumask *srcp, int n)
|
||||
static __always_inline
|
||||
void cpumask_shift_left(struct cpumask *dstp, const struct cpumask *srcp, int n)
|
||||
{
|
||||
bitmap_shift_left(cpumask_bits(dstp), cpumask_bits(srcp), n,
|
||||
nr_cpumask_bits);
|
||||
@ -821,8 +833,8 @@ static inline void cpumask_shift_left(struct cpumask *dstp,
|
||||
* @dstp: the result
|
||||
* @srcp: the input cpumask
|
||||
*/
|
||||
static inline void cpumask_copy(struct cpumask *dstp,
|
||||
const struct cpumask *srcp)
|
||||
static __always_inline
|
||||
void cpumask_copy(struct cpumask *dstp, const struct cpumask *srcp)
|
||||
{
|
||||
bitmap_copy(cpumask_bits(dstp), cpumask_bits(srcp), large_cpumask_bits);
|
||||
}
|
||||
@ -858,8 +870,8 @@ static inline void cpumask_copy(struct cpumask *dstp,
|
||||
*
|
||||
* Return: -errno, or 0 for success.
|
||||
*/
|
||||
static inline int cpumask_parse_user(const char __user *buf, int len,
|
||||
struct cpumask *dstp)
|
||||
static __always_inline
|
||||
int cpumask_parse_user(const char __user *buf, int len, struct cpumask *dstp)
|
||||
{
|
||||
return bitmap_parse_user(buf, len, cpumask_bits(dstp), nr_cpumask_bits);
|
||||
}
|
||||
@ -872,8 +884,8 @@ static inline int cpumask_parse_user(const char __user *buf, int len,
|
||||
*
|
||||
* Return: -errno, or 0 for success.
|
||||
*/
|
||||
static inline int cpumask_parselist_user(const char __user *buf, int len,
|
||||
struct cpumask *dstp)
|
||||
static __always_inline
|
||||
int cpumask_parselist_user(const char __user *buf, int len, struct cpumask *dstp)
|
||||
{
|
||||
return bitmap_parselist_user(buf, len, cpumask_bits(dstp),
|
||||
nr_cpumask_bits);
|
||||
@ -886,7 +898,7 @@ static inline int cpumask_parselist_user(const char __user *buf, int len,
|
||||
*
|
||||
* Return: -errno, or 0 for success.
|
||||
*/
|
||||
static inline int cpumask_parse(const char *buf, struct cpumask *dstp)
|
||||
static __always_inline int cpumask_parse(const char *buf, struct cpumask *dstp)
|
||||
{
|
||||
return bitmap_parse(buf, UINT_MAX, cpumask_bits(dstp), nr_cpumask_bits);
|
||||
}
|
||||
@ -898,7 +910,7 @@ static inline int cpumask_parse(const char *buf, struct cpumask *dstp)
|
||||
*
|
||||
* Return: -errno, or 0 for success.
|
||||
*/
|
||||
static inline int cpulist_parse(const char *buf, struct cpumask *dstp)
|
||||
static __always_inline int cpulist_parse(const char *buf, struct cpumask *dstp)
|
||||
{
|
||||
return bitmap_parselist(buf, cpumask_bits(dstp), nr_cpumask_bits);
|
||||
}
|
||||
@ -908,7 +920,7 @@ static inline int cpulist_parse(const char *buf, struct cpumask *dstp)
|
||||
*
|
||||
* Return: size to allocate for a &struct cpumask in bytes
|
||||
*/
|
||||
static inline unsigned int cpumask_size(void)
|
||||
static __always_inline unsigned int cpumask_size(void)
|
||||
{
|
||||
return bitmap_size(large_cpumask_bits);
|
||||
}
|
||||
@ -920,7 +932,7 @@ static inline unsigned int cpumask_size(void)
|
||||
|
||||
bool alloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags, int node);
|
||||
|
||||
static inline
|
||||
static __always_inline
|
||||
bool zalloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags, int node)
|
||||
{
|
||||
return alloc_cpumask_var_node(mask, flags | __GFP_ZERO, node);
|
||||
@ -938,13 +950,13 @@ bool zalloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags, int node)
|
||||
*
|
||||
* Return: %true if allocation succeeded, %false if not
|
||||
*/
|
||||
static inline
|
||||
static __always_inline
|
||||
bool alloc_cpumask_var(cpumask_var_t *mask, gfp_t flags)
|
||||
{
|
||||
return alloc_cpumask_var_node(mask, flags, NUMA_NO_NODE);
|
||||
}
|
||||
|
||||
static inline
|
||||
static __always_inline
|
||||
bool zalloc_cpumask_var(cpumask_var_t *mask, gfp_t flags)
|
||||
{
|
||||
return alloc_cpumask_var(mask, flags | __GFP_ZERO);
|
||||
@ -954,7 +966,7 @@ void alloc_bootmem_cpumask_var(cpumask_var_t *mask);
|
||||
void free_cpumask_var(cpumask_var_t mask);
|
||||
void free_bootmem_cpumask_var(cpumask_var_t mask);
|
||||
|
||||
static inline bool cpumask_available(cpumask_var_t mask)
|
||||
static __always_inline bool cpumask_available(cpumask_var_t mask)
|
||||
{
|
||||
return mask != NULL;
|
||||
}
|
||||
@ -964,43 +976,43 @@ static inline bool cpumask_available(cpumask_var_t mask)
|
||||
#define this_cpu_cpumask_var_ptr(x) this_cpu_ptr(x)
|
||||
#define __cpumask_var_read_mostly
|
||||
|
||||
static inline bool alloc_cpumask_var(cpumask_var_t *mask, gfp_t flags)
|
||||
static __always_inline bool alloc_cpumask_var(cpumask_var_t *mask, gfp_t flags)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
static inline bool alloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags,
|
||||
static __always_inline bool alloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags,
|
||||
int node)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
static inline bool zalloc_cpumask_var(cpumask_var_t *mask, gfp_t flags)
|
||||
static __always_inline bool zalloc_cpumask_var(cpumask_var_t *mask, gfp_t flags)
|
||||
{
|
||||
cpumask_clear(*mask);
|
||||
return true;
|
||||
}
|
||||
|
||||
static inline bool zalloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags,
|
||||
static __always_inline bool zalloc_cpumask_var_node(cpumask_var_t *mask, gfp_t flags,
|
||||
int node)
|
||||
{
|
||||
cpumask_clear(*mask);
|
||||
return true;
|
||||
}
|
||||
|
||||
static inline void alloc_bootmem_cpumask_var(cpumask_var_t *mask)
|
||||
static __always_inline void alloc_bootmem_cpumask_var(cpumask_var_t *mask)
|
||||
{
|
||||
}
|
||||
|
||||
static inline void free_cpumask_var(cpumask_var_t mask)
|
||||
static __always_inline void free_cpumask_var(cpumask_var_t mask)
|
||||
{
|
||||
}
|
||||
|
||||
static inline void free_bootmem_cpumask_var(cpumask_var_t mask)
|
||||
static __always_inline void free_bootmem_cpumask_var(cpumask_var_t mask)
|
||||
{
|
||||
}
|
||||
|
||||
static inline bool cpumask_available(cpumask_var_t mask)
|
||||
static __always_inline bool cpumask_available(cpumask_var_t mask)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
@ -1058,7 +1070,7 @@ void set_cpu_online(unsigned int cpu, bool online);
|
||||
((struct cpumask *)(1 ? (bitmap) \
|
||||
: (void *)sizeof(__check_is_bitmap(bitmap))))
|
||||
|
||||
static inline int __check_is_bitmap(const unsigned long *bitmap)
|
||||
static __always_inline int __check_is_bitmap(const unsigned long *bitmap)
|
||||
{
|
||||
return 1;
|
||||
}
|
||||
@ -1073,7 +1085,7 @@ static inline int __check_is_bitmap(const unsigned long *bitmap)
|
||||
extern const unsigned long
|
||||
cpu_bit_bitmap[BITS_PER_LONG+1][BITS_TO_LONGS(NR_CPUS)];
|
||||
|
||||
static inline const struct cpumask *get_cpu_mask(unsigned int cpu)
|
||||
static __always_inline const struct cpumask *get_cpu_mask(unsigned int cpu)
|
||||
{
|
||||
const unsigned long *p = cpu_bit_bitmap[1 + cpu % BITS_PER_LONG];
|
||||
p -= cpu / BITS_PER_LONG;
|
||||
@ -1100,32 +1112,32 @@ static __always_inline unsigned int num_online_cpus(void)
|
||||
#define num_present_cpus() cpumask_weight(cpu_present_mask)
|
||||
#define num_active_cpus() cpumask_weight(cpu_active_mask)
|
||||
|
||||
static inline bool cpu_online(unsigned int cpu)
|
||||
static __always_inline bool cpu_online(unsigned int cpu)
|
||||
{
|
||||
return cpumask_test_cpu(cpu, cpu_online_mask);
|
||||
}
|
||||
|
||||
static inline bool cpu_enabled(unsigned int cpu)
|
||||
static __always_inline bool cpu_enabled(unsigned int cpu)
|
||||
{
|
||||
return cpumask_test_cpu(cpu, cpu_enabled_mask);
|
||||
}
|
||||
|
||||
static inline bool cpu_possible(unsigned int cpu)
|
||||
static __always_inline bool cpu_possible(unsigned int cpu)
|
||||
{
|
||||
return cpumask_test_cpu(cpu, cpu_possible_mask);
|
||||
}
|
||||
|
||||
static inline bool cpu_present(unsigned int cpu)
|
||||
static __always_inline bool cpu_present(unsigned int cpu)
|
||||
{
|
||||
return cpumask_test_cpu(cpu, cpu_present_mask);
|
||||
}
|
||||
|
||||
static inline bool cpu_active(unsigned int cpu)
|
||||
static __always_inline bool cpu_active(unsigned int cpu)
|
||||
{
|
||||
return cpumask_test_cpu(cpu, cpu_active_mask);
|
||||
}
|
||||
|
||||
static inline bool cpu_dying(unsigned int cpu)
|
||||
static __always_inline bool cpu_dying(unsigned int cpu)
|
||||
{
|
||||
return cpumask_test_cpu(cpu, cpu_dying_mask);
|
||||
}
|
||||
@ -1138,32 +1150,32 @@ static inline bool cpu_dying(unsigned int cpu)
|
||||
#define num_present_cpus() 1U
|
||||
#define num_active_cpus() 1U
|
||||
|
||||
static inline bool cpu_online(unsigned int cpu)
|
||||
static __always_inline bool cpu_online(unsigned int cpu)
|
||||
{
|
||||
return cpu == 0;
|
||||
}
|
||||
|
||||
static inline bool cpu_possible(unsigned int cpu)
|
||||
static __always_inline bool cpu_possible(unsigned int cpu)
|
||||
{
|
||||
return cpu == 0;
|
||||
}
|
||||
|
||||
static inline bool cpu_enabled(unsigned int cpu)
|
||||
static __always_inline bool cpu_enabled(unsigned int cpu)
|
||||
{
|
||||
return cpu == 0;
|
||||
}
|
||||
|
||||
static inline bool cpu_present(unsigned int cpu)
|
||||
static __always_inline bool cpu_present(unsigned int cpu)
|
||||
{
|
||||
return cpu == 0;
|
||||
}
|
||||
|
||||
static inline bool cpu_active(unsigned int cpu)
|
||||
static __always_inline bool cpu_active(unsigned int cpu)
|
||||
{
|
||||
return cpu == 0;
|
||||
}
|
||||
|
||||
static inline bool cpu_dying(unsigned int cpu)
|
||||
static __always_inline bool cpu_dying(unsigned int cpu)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
@ -1197,7 +1209,7 @@ static inline bool cpu_dying(unsigned int cpu)
|
||||
* Return: the length of the (null-terminated) @buf string, zero if
|
||||
* nothing is copied.
|
||||
*/
|
||||
static inline ssize_t
|
||||
static __always_inline ssize_t
|
||||
cpumap_print_to_pagebuf(bool list, char *buf, const struct cpumask *mask)
|
||||
{
|
||||
return bitmap_print_to_pagebuf(list, buf, cpumask_bits(mask),
|
||||
@ -1220,9 +1232,9 @@ cpumap_print_to_pagebuf(bool list, char *buf, const struct cpumask *mask)
|
||||
* Return: the length of how many bytes have been copied, excluding
|
||||
* terminating '\0'.
|
||||
*/
|
||||
static inline ssize_t
|
||||
cpumap_print_bitmask_to_buf(char *buf, const struct cpumask *mask,
|
||||
loff_t off, size_t count)
|
||||
static __always_inline
|
||||
ssize_t cpumap_print_bitmask_to_buf(char *buf, const struct cpumask *mask,
|
||||
loff_t off, size_t count)
|
||||
{
|
||||
return bitmap_print_bitmask_to_buf(buf, cpumask_bits(mask),
|
||||
nr_cpu_ids, off, count) - 1;
|
||||
@ -1242,9 +1254,9 @@ cpumap_print_bitmask_to_buf(char *buf, const struct cpumask *mask,
|
||||
* Return: the length of how many bytes have been copied, excluding
|
||||
* terminating '\0'.
|
||||
*/
|
||||
static inline ssize_t
|
||||
cpumap_print_list_to_buf(char *buf, const struct cpumask *mask,
|
||||
loff_t off, size_t count)
|
||||
static __always_inline
|
||||
ssize_t cpumap_print_list_to_buf(char *buf, const struct cpumask *mask,
|
||||
loff_t off, size_t count)
|
||||
{
|
||||
return bitmap_print_list_to_buf(buf, cpumask_bits(mask),
|
||||
nr_cpu_ids, off, count) - 1;
|
||||
|
||||
@ -52,7 +52,7 @@ unsigned long _find_next_bit_le(const unsigned long *addr, unsigned
|
||||
* Returns the bit number for the next set bit
|
||||
* If no bits are set, returns @size.
|
||||
*/
|
||||
static inline
|
||||
static __always_inline
|
||||
unsigned long find_next_bit(const unsigned long *addr, unsigned long size,
|
||||
unsigned long offset)
|
||||
{
|
||||
@ -81,7 +81,7 @@ unsigned long find_next_bit(const unsigned long *addr, unsigned long size,
|
||||
* Returns the bit number for the next set bit
|
||||
* If no bits are set, returns @size.
|
||||
*/
|
||||
static inline
|
||||
static __always_inline
|
||||
unsigned long find_next_and_bit(const unsigned long *addr1,
|
||||
const unsigned long *addr2, unsigned long size,
|
||||
unsigned long offset)
|
||||
@ -112,7 +112,7 @@ unsigned long find_next_and_bit(const unsigned long *addr1,
|
||||
* Returns the bit number for the next set bit
|
||||
* If no bits are set, returns @size.
|
||||
*/
|
||||
static inline
|
||||
static __always_inline
|
||||
unsigned long find_next_andnot_bit(const unsigned long *addr1,
|
||||
const unsigned long *addr2, unsigned long size,
|
||||
unsigned long offset)
|
||||
@ -142,7 +142,7 @@ unsigned long find_next_andnot_bit(const unsigned long *addr1,
|
||||
* Returns the bit number for the next set bit
|
||||
* If no bits are set, returns @size.
|
||||
*/
|
||||
static inline
|
||||
static __always_inline
|
||||
unsigned long find_next_or_bit(const unsigned long *addr1,
|
||||
const unsigned long *addr2, unsigned long size,
|
||||
unsigned long offset)
|
||||
@ -171,7 +171,7 @@ unsigned long find_next_or_bit(const unsigned long *addr1,
|
||||
* Returns the bit number of the next zero bit
|
||||
* If no bits are zero, returns @size.
|
||||
*/
|
||||
static inline
|
||||
static __always_inline
|
||||
unsigned long find_next_zero_bit(const unsigned long *addr, unsigned long size,
|
||||
unsigned long offset)
|
||||
{
|
||||
@ -198,7 +198,7 @@ unsigned long find_next_zero_bit(const unsigned long *addr, unsigned long size,
|
||||
* Returns the bit number of the first set bit.
|
||||
* If no bits are set, returns @size.
|
||||
*/
|
||||
static inline
|
||||
static __always_inline
|
||||
unsigned long find_first_bit(const unsigned long *addr, unsigned long size)
|
||||
{
|
||||
if (small_const_nbits(size)) {
|
||||
@ -224,7 +224,7 @@ unsigned long find_first_bit(const unsigned long *addr, unsigned long size)
|
||||
* Returns the bit number of the N'th set bit.
|
||||
* If no such, returns >= @size.
|
||||
*/
|
||||
static inline
|
||||
static __always_inline
|
||||
unsigned long find_nth_bit(const unsigned long *addr, unsigned long size, unsigned long n)
|
||||
{
|
||||
if (n >= size)
|
||||
@ -249,7 +249,7 @@ unsigned long find_nth_bit(const unsigned long *addr, unsigned long size, unsign
|
||||
* Returns the bit number of the N'th set bit.
|
||||
* If no such, returns @size.
|
||||
*/
|
||||
static inline
|
||||
static __always_inline
|
||||
unsigned long find_nth_and_bit(const unsigned long *addr1, const unsigned long *addr2,
|
||||
unsigned long size, unsigned long n)
|
||||
{
|
||||
@ -276,7 +276,7 @@ unsigned long find_nth_and_bit(const unsigned long *addr1, const unsigned long *
|
||||
* Returns the bit number of the N'th set bit.
|
||||
* If no such, returns @size.
|
||||
*/
|
||||
static inline
|
||||
static __always_inline
|
||||
unsigned long find_nth_andnot_bit(const unsigned long *addr1, const unsigned long *addr2,
|
||||
unsigned long size, unsigned long n)
|
||||
{
|
||||
@ -332,7 +332,7 @@ unsigned long find_nth_and_andnot_bit(const unsigned long *addr1,
|
||||
* Returns the bit number for the next set bit
|
||||
* If no bits are set, returns @size.
|
||||
*/
|
||||
static inline
|
||||
static __always_inline
|
||||
unsigned long find_first_and_bit(const unsigned long *addr1,
|
||||
const unsigned long *addr2,
|
||||
unsigned long size)
|
||||
@ -357,7 +357,7 @@ unsigned long find_first_and_bit(const unsigned long *addr1,
|
||||
* Returns the bit number for the first set bit
|
||||
* If no bits are set, returns @size.
|
||||
*/
|
||||
static inline
|
||||
static __always_inline
|
||||
unsigned long find_first_and_and_bit(const unsigned long *addr1,
|
||||
const unsigned long *addr2,
|
||||
const unsigned long *addr3,
|
||||
@ -381,7 +381,7 @@ unsigned long find_first_and_and_bit(const unsigned long *addr1,
|
||||
* Returns the bit number of the first cleared bit.
|
||||
* If no bits are zero, returns @size.
|
||||
*/
|
||||
static inline
|
||||
static __always_inline
|
||||
unsigned long find_first_zero_bit(const unsigned long *addr, unsigned long size)
|
||||
{
|
||||
if (small_const_nbits(size)) {
|
||||
@ -402,7 +402,7 @@ unsigned long find_first_zero_bit(const unsigned long *addr, unsigned long size)
|
||||
*
|
||||
* Returns the bit number of the last set bit, or size.
|
||||
*/
|
||||
static inline
|
||||
static __always_inline
|
||||
unsigned long find_last_bit(const unsigned long *addr, unsigned long size)
|
||||
{
|
||||
if (small_const_nbits(size)) {
|
||||
@ -425,7 +425,7 @@ unsigned long find_last_bit(const unsigned long *addr, unsigned long size)
|
||||
* Returns the bit number for the next set bit, or first set bit up to @offset
|
||||
* If no bits are set, returns @size.
|
||||
*/
|
||||
static inline
|
||||
static __always_inline
|
||||
unsigned long find_next_and_bit_wrap(const unsigned long *addr1,
|
||||
const unsigned long *addr2,
|
||||
unsigned long size, unsigned long offset)
|
||||
@ -448,7 +448,7 @@ unsigned long find_next_and_bit_wrap(const unsigned long *addr1,
|
||||
* Returns the bit number for the next set bit, or first set bit up to @offset
|
||||
* If no bits are set, returns @size.
|
||||
*/
|
||||
static inline
|
||||
static __always_inline
|
||||
unsigned long find_next_bit_wrap(const unsigned long *addr,
|
||||
unsigned long size, unsigned long offset)
|
||||
{
|
||||
@ -465,7 +465,7 @@ unsigned long find_next_bit_wrap(const unsigned long *addr,
|
||||
* Helper for for_each_set_bit_wrap(). Make sure you're doing right thing
|
||||
* before using it alone.
|
||||
*/
|
||||
static inline
|
||||
static __always_inline
|
||||
unsigned long __for_each_wrap(const unsigned long *bitmap, unsigned long size,
|
||||
unsigned long start, unsigned long n)
|
||||
{
|
||||
@ -506,20 +506,20 @@ extern unsigned long find_next_clump8(unsigned long *clump,
|
||||
|
||||
#if defined(__LITTLE_ENDIAN)
|
||||
|
||||
static inline unsigned long find_next_zero_bit_le(const void *addr,
|
||||
unsigned long size, unsigned long offset)
|
||||
static __always_inline
|
||||
unsigned long find_next_zero_bit_le(const void *addr, unsigned long size, unsigned long offset)
|
||||
{
|
||||
return find_next_zero_bit(addr, size, offset);
|
||||
}
|
||||
|
||||
static inline unsigned long find_next_bit_le(const void *addr,
|
||||
unsigned long size, unsigned long offset)
|
||||
static __always_inline
|
||||
unsigned long find_next_bit_le(const void *addr, unsigned long size, unsigned long offset)
|
||||
{
|
||||
return find_next_bit(addr, size, offset);
|
||||
}
|
||||
|
||||
static inline unsigned long find_first_zero_bit_le(const void *addr,
|
||||
unsigned long size)
|
||||
static __always_inline
|
||||
unsigned long find_first_zero_bit_le(const void *addr, unsigned long size)
|
||||
{
|
||||
return find_first_zero_bit(addr, size);
|
||||
}
|
||||
@ -527,7 +527,7 @@ static inline unsigned long find_first_zero_bit_le(const void *addr,
|
||||
#elif defined(__BIG_ENDIAN)
|
||||
|
||||
#ifndef find_next_zero_bit_le
|
||||
static inline
|
||||
static __always_inline
|
||||
unsigned long find_next_zero_bit_le(const void *addr, unsigned
|
||||
long size, unsigned long offset)
|
||||
{
|
||||
@ -546,7 +546,7 @@ unsigned long find_next_zero_bit_le(const void *addr, unsigned
|
||||
#endif
|
||||
|
||||
#ifndef find_first_zero_bit_le
|
||||
static inline
|
||||
static __always_inline
|
||||
unsigned long find_first_zero_bit_le(const void *addr, unsigned long size)
|
||||
{
|
||||
if (small_const_nbits(size)) {
|
||||
@ -560,7 +560,7 @@ unsigned long find_first_zero_bit_le(const void *addr, unsigned long size)
|
||||
#endif
|
||||
|
||||
#ifndef find_next_bit_le
|
||||
static inline
|
||||
static __always_inline
|
||||
unsigned long find_next_bit_le(const void *addr, unsigned
|
||||
long size, unsigned long offset)
|
||||
{
|
||||
|
||||
@ -107,11 +107,11 @@ extern nodemask_t _unused_nodemask_arg_;
|
||||
*/
|
||||
#define nodemask_pr_args(maskp) __nodemask_pr_numnodes(maskp), \
|
||||
__nodemask_pr_bits(maskp)
|
||||
static inline unsigned int __nodemask_pr_numnodes(const nodemask_t *m)
|
||||
static __always_inline unsigned int __nodemask_pr_numnodes(const nodemask_t *m)
|
||||
{
|
||||
return m ? MAX_NUMNODES : 0;
|
||||
}
|
||||
static inline const unsigned long *__nodemask_pr_bits(const nodemask_t *m)
|
||||
static __always_inline const unsigned long *__nodemask_pr_bits(const nodemask_t *m)
|
||||
{
|
||||
return m ? m->bits : NULL;
|
||||
}
|
||||
@ -132,19 +132,19 @@ static __always_inline void __node_set(int node, volatile nodemask_t *dstp)
|
||||
}
|
||||
|
||||
#define node_clear(node, dst) __node_clear((node), &(dst))
|
||||
static inline void __node_clear(int node, volatile nodemask_t *dstp)
|
||||
static __always_inline void __node_clear(int node, volatile nodemask_t *dstp)
|
||||
{
|
||||
clear_bit(node, dstp->bits);
|
||||
}
|
||||
|
||||
#define nodes_setall(dst) __nodes_setall(&(dst), MAX_NUMNODES)
|
||||
static inline void __nodes_setall(nodemask_t *dstp, unsigned int nbits)
|
||||
static __always_inline void __nodes_setall(nodemask_t *dstp, unsigned int nbits)
|
||||
{
|
||||
bitmap_fill(dstp->bits, nbits);
|
||||
}
|
||||
|
||||
#define nodes_clear(dst) __nodes_clear(&(dst), MAX_NUMNODES)
|
||||
static inline void __nodes_clear(nodemask_t *dstp, unsigned int nbits)
|
||||
static __always_inline void __nodes_clear(nodemask_t *dstp, unsigned int nbits)
|
||||
{
|
||||
bitmap_zero(dstp->bits, nbits);
|
||||
}
|
||||
@ -154,14 +154,14 @@ static inline void __nodes_clear(nodemask_t *dstp, unsigned int nbits)
|
||||
|
||||
#define node_test_and_set(node, nodemask) \
|
||||
__node_test_and_set((node), &(nodemask))
|
||||
static inline bool __node_test_and_set(int node, nodemask_t *addr)
|
||||
static __always_inline bool __node_test_and_set(int node, nodemask_t *addr)
|
||||
{
|
||||
return test_and_set_bit(node, addr->bits);
|
||||
}
|
||||
|
||||
#define nodes_and(dst, src1, src2) \
|
||||
__nodes_and(&(dst), &(src1), &(src2), MAX_NUMNODES)
|
||||
static inline void __nodes_and(nodemask_t *dstp, const nodemask_t *src1p,
|
||||
static __always_inline void __nodes_and(nodemask_t *dstp, const nodemask_t *src1p,
|
||||
const nodemask_t *src2p, unsigned int nbits)
|
||||
{
|
||||
bitmap_and(dstp->bits, src1p->bits, src2p->bits, nbits);
|
||||
@ -169,7 +169,7 @@ static inline void __nodes_and(nodemask_t *dstp, const nodemask_t *src1p,
|
||||
|
||||
#define nodes_or(dst, src1, src2) \
|
||||
__nodes_or(&(dst), &(src1), &(src2), MAX_NUMNODES)
|
||||
static inline void __nodes_or(nodemask_t *dstp, const nodemask_t *src1p,
|
||||
static __always_inline void __nodes_or(nodemask_t *dstp, const nodemask_t *src1p,
|
||||
const nodemask_t *src2p, unsigned int nbits)
|
||||
{
|
||||
bitmap_or(dstp->bits, src1p->bits, src2p->bits, nbits);
|
||||
@ -177,7 +177,7 @@ static inline void __nodes_or(nodemask_t *dstp, const nodemask_t *src1p,
|
||||
|
||||
#define nodes_xor(dst, src1, src2) \
|
||||
__nodes_xor(&(dst), &(src1), &(src2), MAX_NUMNODES)
|
||||
static inline void __nodes_xor(nodemask_t *dstp, const nodemask_t *src1p,
|
||||
static __always_inline void __nodes_xor(nodemask_t *dstp, const nodemask_t *src1p,
|
||||
const nodemask_t *src2p, unsigned int nbits)
|
||||
{
|
||||
bitmap_xor(dstp->bits, src1p->bits, src2p->bits, nbits);
|
||||
@ -185,7 +185,7 @@ static inline void __nodes_xor(nodemask_t *dstp, const nodemask_t *src1p,
|
||||
|
||||
#define nodes_andnot(dst, src1, src2) \
|
||||
__nodes_andnot(&(dst), &(src1), &(src2), MAX_NUMNODES)
|
||||
static inline void __nodes_andnot(nodemask_t *dstp, const nodemask_t *src1p,
|
||||
static __always_inline void __nodes_andnot(nodemask_t *dstp, const nodemask_t *src1p,
|
||||
const nodemask_t *src2p, unsigned int nbits)
|
||||
{
|
||||
bitmap_andnot(dstp->bits, src1p->bits, src2p->bits, nbits);
|
||||
@ -193,7 +193,7 @@ static inline void __nodes_andnot(nodemask_t *dstp, const nodemask_t *src1p,
|
||||
|
||||
#define nodes_complement(dst, src) \
|
||||
__nodes_complement(&(dst), &(src), MAX_NUMNODES)
|
||||
static inline void __nodes_complement(nodemask_t *dstp,
|
||||
static __always_inline void __nodes_complement(nodemask_t *dstp,
|
||||
const nodemask_t *srcp, unsigned int nbits)
|
||||
{
|
||||
bitmap_complement(dstp->bits, srcp->bits, nbits);
|
||||
@ -201,7 +201,7 @@ static inline void __nodes_complement(nodemask_t *dstp,
|
||||
|
||||
#define nodes_equal(src1, src2) \
|
||||
__nodes_equal(&(src1), &(src2), MAX_NUMNODES)
|
||||
static inline bool __nodes_equal(const nodemask_t *src1p,
|
||||
static __always_inline bool __nodes_equal(const nodemask_t *src1p,
|
||||
const nodemask_t *src2p, unsigned int nbits)
|
||||
{
|
||||
return bitmap_equal(src1p->bits, src2p->bits, nbits);
|
||||
@ -209,7 +209,7 @@ static inline bool __nodes_equal(const nodemask_t *src1p,
|
||||
|
||||
#define nodes_intersects(src1, src2) \
|
||||
__nodes_intersects(&(src1), &(src2), MAX_NUMNODES)
|
||||
static inline bool __nodes_intersects(const nodemask_t *src1p,
|
||||
static __always_inline bool __nodes_intersects(const nodemask_t *src1p,
|
||||
const nodemask_t *src2p, unsigned int nbits)
|
||||
{
|
||||
return bitmap_intersects(src1p->bits, src2p->bits, nbits);
|
||||
@ -217,33 +217,33 @@ static inline bool __nodes_intersects(const nodemask_t *src1p,
|
||||
|
||||
#define nodes_subset(src1, src2) \
|
||||
__nodes_subset(&(src1), &(src2), MAX_NUMNODES)
|
||||
static inline bool __nodes_subset(const nodemask_t *src1p,
|
||||
static __always_inline bool __nodes_subset(const nodemask_t *src1p,
|
||||
const nodemask_t *src2p, unsigned int nbits)
|
||||
{
|
||||
return bitmap_subset(src1p->bits, src2p->bits, nbits);
|
||||
}
|
||||
|
||||
#define nodes_empty(src) __nodes_empty(&(src), MAX_NUMNODES)
|
||||
static inline bool __nodes_empty(const nodemask_t *srcp, unsigned int nbits)
|
||||
static __always_inline bool __nodes_empty(const nodemask_t *srcp, unsigned int nbits)
|
||||
{
|
||||
return bitmap_empty(srcp->bits, nbits);
|
||||
}
|
||||
|
||||
#define nodes_full(nodemask) __nodes_full(&(nodemask), MAX_NUMNODES)
|
||||
static inline bool __nodes_full(const nodemask_t *srcp, unsigned int nbits)
|
||||
static __always_inline bool __nodes_full(const nodemask_t *srcp, unsigned int nbits)
|
||||
{
|
||||
return bitmap_full(srcp->bits, nbits);
|
||||
}
|
||||
|
||||
#define nodes_weight(nodemask) __nodes_weight(&(nodemask), MAX_NUMNODES)
|
||||
static inline int __nodes_weight(const nodemask_t *srcp, unsigned int nbits)
|
||||
static __always_inline int __nodes_weight(const nodemask_t *srcp, unsigned int nbits)
|
||||
{
|
||||
return bitmap_weight(srcp->bits, nbits);
|
||||
}
|
||||
|
||||
#define nodes_shift_right(dst, src, n) \
|
||||
__nodes_shift_right(&(dst), &(src), (n), MAX_NUMNODES)
|
||||
static inline void __nodes_shift_right(nodemask_t *dstp,
|
||||
static __always_inline void __nodes_shift_right(nodemask_t *dstp,
|
||||
const nodemask_t *srcp, int n, int nbits)
|
||||
{
|
||||
bitmap_shift_right(dstp->bits, srcp->bits, n, nbits);
|
||||
@ -251,7 +251,7 @@ static inline void __nodes_shift_right(nodemask_t *dstp,
|
||||
|
||||
#define nodes_shift_left(dst, src, n) \
|
||||
__nodes_shift_left(&(dst), &(src), (n), MAX_NUMNODES)
|
||||
static inline void __nodes_shift_left(nodemask_t *dstp,
|
||||
static __always_inline void __nodes_shift_left(nodemask_t *dstp,
|
||||
const nodemask_t *srcp, int n, int nbits)
|
||||
{
|
||||
bitmap_shift_left(dstp->bits, srcp->bits, n, nbits);
|
||||
@ -261,13 +261,13 @@ static inline void __nodes_shift_left(nodemask_t *dstp,
|
||||
> MAX_NUMNODES, then the silly min_ts could be dropped. */
|
||||
|
||||
#define first_node(src) __first_node(&(src))
|
||||
static inline unsigned int __first_node(const nodemask_t *srcp)
|
||||
static __always_inline unsigned int __first_node(const nodemask_t *srcp)
|
||||
{
|
||||
return min_t(unsigned int, MAX_NUMNODES, find_first_bit(srcp->bits, MAX_NUMNODES));
|
||||
}
|
||||
|
||||
#define next_node(n, src) __next_node((n), &(src))
|
||||
static inline unsigned int __next_node(int n, const nodemask_t *srcp)
|
||||
static __always_inline unsigned int __next_node(int n, const nodemask_t *srcp)
|
||||
{
|
||||
return min_t(unsigned int, MAX_NUMNODES, find_next_bit(srcp->bits, MAX_NUMNODES, n+1));
|
||||
}
|
||||
@ -277,7 +277,7 @@ static inline unsigned int __next_node(int n, const nodemask_t *srcp)
|
||||
* the first node in src if needed. Returns MAX_NUMNODES if src is empty.
|
||||
*/
|
||||
#define next_node_in(n, src) __next_node_in((n), &(src))
|
||||
static inline unsigned int __next_node_in(int node, const nodemask_t *srcp)
|
||||
static __always_inline unsigned int __next_node_in(int node, const nodemask_t *srcp)
|
||||
{
|
||||
unsigned int ret = __next_node(node, srcp);
|
||||
|
||||
@ -286,7 +286,7 @@ static inline unsigned int __next_node_in(int node, const nodemask_t *srcp)
|
||||
return ret;
|
||||
}
|
||||
|
||||
static inline void init_nodemask_of_node(nodemask_t *mask, int node)
|
||||
static __always_inline void init_nodemask_of_node(nodemask_t *mask, int node)
|
||||
{
|
||||
nodes_clear(*mask);
|
||||
node_set(node, *mask);
|
||||
@ -304,7 +304,7 @@ static inline void init_nodemask_of_node(nodemask_t *mask, int node)
|
||||
})
|
||||
|
||||
#define first_unset_node(mask) __first_unset_node(&(mask))
|
||||
static inline unsigned int __first_unset_node(const nodemask_t *maskp)
|
||||
static __always_inline unsigned int __first_unset_node(const nodemask_t *maskp)
|
||||
{
|
||||
return min_t(unsigned int, MAX_NUMNODES,
|
||||
find_first_zero_bit(maskp->bits, MAX_NUMNODES));
|
||||
@ -338,21 +338,21 @@ static inline unsigned int __first_unset_node(const nodemask_t *maskp)
|
||||
|
||||
#define nodemask_parse_user(ubuf, ulen, dst) \
|
||||
__nodemask_parse_user((ubuf), (ulen), &(dst), MAX_NUMNODES)
|
||||
static inline int __nodemask_parse_user(const char __user *buf, int len,
|
||||
static __always_inline int __nodemask_parse_user(const char __user *buf, int len,
|
||||
nodemask_t *dstp, int nbits)
|
||||
{
|
||||
return bitmap_parse_user(buf, len, dstp->bits, nbits);
|
||||
}
|
||||
|
||||
#define nodelist_parse(buf, dst) __nodelist_parse((buf), &(dst), MAX_NUMNODES)
|
||||
static inline int __nodelist_parse(const char *buf, nodemask_t *dstp, int nbits)
|
||||
static __always_inline int __nodelist_parse(const char *buf, nodemask_t *dstp, int nbits)
|
||||
{
|
||||
return bitmap_parselist(buf, dstp->bits, nbits);
|
||||
}
|
||||
|
||||
#define node_remap(oldbit, old, new) \
|
||||
__node_remap((oldbit), &(old), &(new), MAX_NUMNODES)
|
||||
static inline int __node_remap(int oldbit,
|
||||
static __always_inline int __node_remap(int oldbit,
|
||||
const nodemask_t *oldp, const nodemask_t *newp, int nbits)
|
||||
{
|
||||
return bitmap_bitremap(oldbit, oldp->bits, newp->bits, nbits);
|
||||
@ -360,7 +360,7 @@ static inline int __node_remap(int oldbit,
|
||||
|
||||
#define nodes_remap(dst, src, old, new) \
|
||||
__nodes_remap(&(dst), &(src), &(old), &(new), MAX_NUMNODES)
|
||||
static inline void __nodes_remap(nodemask_t *dstp, const nodemask_t *srcp,
|
||||
static __always_inline void __nodes_remap(nodemask_t *dstp, const nodemask_t *srcp,
|
||||
const nodemask_t *oldp, const nodemask_t *newp, int nbits)
|
||||
{
|
||||
bitmap_remap(dstp->bits, srcp->bits, oldp->bits, newp->bits, nbits);
|
||||
@ -368,7 +368,7 @@ static inline void __nodes_remap(nodemask_t *dstp, const nodemask_t *srcp,
|
||||
|
||||
#define nodes_onto(dst, orig, relmap) \
|
||||
__nodes_onto(&(dst), &(orig), &(relmap), MAX_NUMNODES)
|
||||
static inline void __nodes_onto(nodemask_t *dstp, const nodemask_t *origp,
|
||||
static __always_inline void __nodes_onto(nodemask_t *dstp, const nodemask_t *origp,
|
||||
const nodemask_t *relmapp, int nbits)
|
||||
{
|
||||
bitmap_onto(dstp->bits, origp->bits, relmapp->bits, nbits);
|
||||
@ -376,7 +376,7 @@ static inline void __nodes_onto(nodemask_t *dstp, const nodemask_t *origp,
|
||||
|
||||
#define nodes_fold(dst, orig, sz) \
|
||||
__nodes_fold(&(dst), &(orig), sz, MAX_NUMNODES)
|
||||
static inline void __nodes_fold(nodemask_t *dstp, const nodemask_t *origp,
|
||||
static __always_inline void __nodes_fold(nodemask_t *dstp, const nodemask_t *origp,
|
||||
int sz, int nbits)
|
||||
{
|
||||
bitmap_fold(dstp->bits, origp->bits, sz, nbits);
|
||||
@ -418,22 +418,22 @@ enum node_states {
|
||||
extern nodemask_t node_states[NR_NODE_STATES];
|
||||
|
||||
#if MAX_NUMNODES > 1
|
||||
static inline int node_state(int node, enum node_states state)
|
||||
static __always_inline int node_state(int node, enum node_states state)
|
||||
{
|
||||
return node_isset(node, node_states[state]);
|
||||
}
|
||||
|
||||
static inline void node_set_state(int node, enum node_states state)
|
||||
static __always_inline void node_set_state(int node, enum node_states state)
|
||||
{
|
||||
__node_set(node, &node_states[state]);
|
||||
}
|
||||
|
||||
static inline void node_clear_state(int node, enum node_states state)
|
||||
static __always_inline void node_clear_state(int node, enum node_states state)
|
||||
{
|
||||
__node_clear(node, &node_states[state]);
|
||||
}
|
||||
|
||||
static inline int num_node_state(enum node_states state)
|
||||
static __always_inline int num_node_state(enum node_states state)
|
||||
{
|
||||
return nodes_weight(node_states[state]);
|
||||
}
|
||||
@ -443,11 +443,11 @@ static inline int num_node_state(enum node_states state)
|
||||
|
||||
#define first_online_node first_node(node_states[N_ONLINE])
|
||||
#define first_memory_node first_node(node_states[N_MEMORY])
|
||||
static inline unsigned int next_online_node(int nid)
|
||||
static __always_inline unsigned int next_online_node(int nid)
|
||||
{
|
||||
return next_node(nid, node_states[N_ONLINE]);
|
||||
}
|
||||
static inline unsigned int next_memory_node(int nid)
|
||||
static __always_inline unsigned int next_memory_node(int nid)
|
||||
{
|
||||
return next_node(nid, node_states[N_MEMORY]);
|
||||
}
|
||||
@ -455,13 +455,13 @@ static inline unsigned int next_memory_node(int nid)
|
||||
extern unsigned int nr_node_ids;
|
||||
extern unsigned int nr_online_nodes;
|
||||
|
||||
static inline void node_set_online(int nid)
|
||||
static __always_inline void node_set_online(int nid)
|
||||
{
|
||||
node_set_state(nid, N_ONLINE);
|
||||
nr_online_nodes = num_node_state(N_ONLINE);
|
||||
}
|
||||
|
||||
static inline void node_set_offline(int nid)
|
||||
static __always_inline void node_set_offline(int nid)
|
||||
{
|
||||
node_clear_state(nid, N_ONLINE);
|
||||
nr_online_nodes = num_node_state(N_ONLINE);
|
||||
@ -469,20 +469,20 @@ static inline void node_set_offline(int nid)
|
||||
|
||||
#else
|
||||
|
||||
static inline int node_state(int node, enum node_states state)
|
||||
static __always_inline int node_state(int node, enum node_states state)
|
||||
{
|
||||
return node == 0;
|
||||
}
|
||||
|
||||
static inline void node_set_state(int node, enum node_states state)
|
||||
static __always_inline void node_set_state(int node, enum node_states state)
|
||||
{
|
||||
}
|
||||
|
||||
static inline void node_clear_state(int node, enum node_states state)
|
||||
static __always_inline void node_clear_state(int node, enum node_states state)
|
||||
{
|
||||
}
|
||||
|
||||
static inline int num_node_state(enum node_states state)
|
||||
static __always_inline int num_node_state(enum node_states state)
|
||||
{
|
||||
return 1;
|
||||
}
|
||||
@ -502,7 +502,7 @@ static inline int num_node_state(enum node_states state)
|
||||
|
||||
#endif
|
||||
|
||||
static inline int node_random(const nodemask_t *maskp)
|
||||
static __always_inline int node_random(const nodemask_t *maskp)
|
||||
{
|
||||
#if defined(CONFIG_NUMA) && (MAX_NUMNODES > 1)
|
||||
int w, bit;
|
||||
|
||||
@ -12,4 +12,7 @@
|
||||
(((~_ULL(0)) - (_ULL(1) << (l)) + 1) & \
|
||||
(~_ULL(0) >> (__BITS_PER_LONG_LONG - 1 - (h))))
|
||||
|
||||
#define __GENMASK_U128(h, l) \
|
||||
((_BIT128((h)) << 1) - (_BIT128(l)))
|
||||
|
||||
#endif /* _UAPI_LINUX_BITS_H */
|
||||
|
||||
@ -28,6 +28,23 @@
|
||||
#define _BITUL(x) (_UL(1) << (x))
|
||||
#define _BITULL(x) (_ULL(1) << (x))
|
||||
|
||||
#if !defined(__ASSEMBLY__)
|
||||
/*
|
||||
* Missing asm support
|
||||
*
|
||||
* __BIT128() would not work in the asm code, as it shifts an
|
||||
* 'unsigned __init128' data type as direct representation of
|
||||
* 128 bit constants is not supported in the gcc compiler, as
|
||||
* they get silently truncated.
|
||||
*
|
||||
* TODO: Please revisit this implementation when gcc compiler
|
||||
* starts representing 128 bit constants directly like long
|
||||
* and unsigned long etc. Subsequently drop the comment for
|
||||
* GENMASK_U128() which would then start supporting asm code.
|
||||
*/
|
||||
#define _BIT128(x) ((unsigned __int128)(1) << (x))
|
||||
#endif
|
||||
|
||||
#define __ALIGN_KERNEL(x, a) __ALIGN_KERNEL_MASK(x, (__typeof__(x))(a) - 1)
|
||||
#define __ALIGN_KERNEL_MASK(x, mask) (((x) + (mask)) & ~(mask))
|
||||
|
||||
|
||||
@ -39,6 +39,36 @@ static void genmask_ull_test(struct kunit *test)
|
||||
#endif
|
||||
}
|
||||
|
||||
static void genmask_u128_test(struct kunit *test)
|
||||
{
|
||||
#ifdef CONFIG_ARCH_SUPPORTS_INT128
|
||||
/* Below 64 bit masks */
|
||||
KUNIT_EXPECT_EQ(test, 0x0000000000000001ull, GENMASK_U128(0, 0));
|
||||
KUNIT_EXPECT_EQ(test, 0x0000000000000003ull, GENMASK_U128(1, 0));
|
||||
KUNIT_EXPECT_EQ(test, 0x0000000000000006ull, GENMASK_U128(2, 1));
|
||||
KUNIT_EXPECT_EQ(test, 0x00000000ffffffffull, GENMASK_U128(31, 0));
|
||||
KUNIT_EXPECT_EQ(test, 0x000000ffffe00000ull, GENMASK_U128(39, 21));
|
||||
KUNIT_EXPECT_EQ(test, 0xffffffffffffffffull, GENMASK_U128(63, 0));
|
||||
|
||||
/* Above 64 bit masks - only 64 bit portion can be validated once */
|
||||
KUNIT_EXPECT_EQ(test, 0xffffffffffffffffull, GENMASK_U128(64, 0) >> 1);
|
||||
KUNIT_EXPECT_EQ(test, 0x00000000ffffffffull, GENMASK_U128(81, 50) >> 50);
|
||||
KUNIT_EXPECT_EQ(test, 0x0000000000ffffffull, GENMASK_U128(87, 64) >> 64);
|
||||
KUNIT_EXPECT_EQ(test, 0x0000000000ff0000ull, GENMASK_U128(87, 80) >> 64);
|
||||
|
||||
KUNIT_EXPECT_EQ(test, 0xffffffffffffffffull, GENMASK_U128(127, 0) >> 64);
|
||||
KUNIT_EXPECT_EQ(test, 0xffffffffffffffffull, (u64)GENMASK_U128(127, 0));
|
||||
KUNIT_EXPECT_EQ(test, 0x0000000000000003ull, GENMASK_U128(127, 126) >> 126);
|
||||
KUNIT_EXPECT_EQ(test, 0x0000000000000001ull, GENMASK_U128(127, 127) >> 127);
|
||||
#ifdef TEST_GENMASK_FAILURES
|
||||
/* these should fail compilation */
|
||||
GENMASK_U128(0, 1);
|
||||
GENMASK_U128(0, 10);
|
||||
GENMASK_U128(9, 10);
|
||||
#endif /* TEST_GENMASK_FAILURES */
|
||||
#endif /* CONFIG_ARCH_SUPPORTS_INT128 */
|
||||
}
|
||||
|
||||
static void genmask_input_check_test(struct kunit *test)
|
||||
{
|
||||
unsigned int x, y;
|
||||
@ -56,12 +86,16 @@ static void genmask_input_check_test(struct kunit *test)
|
||||
/* Valid input */
|
||||
KUNIT_EXPECT_EQ(test, 0, GENMASK_INPUT_CHECK(1, 1));
|
||||
KUNIT_EXPECT_EQ(test, 0, GENMASK_INPUT_CHECK(39, 21));
|
||||
KUNIT_EXPECT_EQ(test, 0, GENMASK_INPUT_CHECK(100, 80));
|
||||
KUNIT_EXPECT_EQ(test, 0, GENMASK_INPUT_CHECK(110, 65));
|
||||
KUNIT_EXPECT_EQ(test, 0, GENMASK_INPUT_CHECK(127, 0));
|
||||
}
|
||||
|
||||
|
||||
static struct kunit_case bits_test_cases[] = {
|
||||
KUNIT_CASE(genmask_test),
|
||||
KUNIT_CASE(genmask_ull_test),
|
||||
KUNIT_CASE(genmask_u128_test),
|
||||
KUNIT_CASE(genmask_input_check_test),
|
||||
{}
|
||||
};
|
||||
|
||||
Loading…
Reference in New Issue
Block a user