0069455bcb
Patch series "Memory allocation profiling", v6. Overview: Low overhead [1] per-callsite memory allocation profiling. Not just for debug kernels, overhead low enough to be deployed in production. Example output: root@moria-kvm:~# sort -rn /proc/allocinfo 127664128 31168 mm/page_ext.c:270 func:alloc_page_ext 56373248 4737 mm/slub.c:2259 func:alloc_slab_page 14880768 3633 mm/readahead.c:247 func:page_cache_ra_unbounded 14417920 3520 mm/mm_init.c:2530 func:alloc_large_system_hash 13377536 234 block/blk-mq.c:3421 func:blk_mq_alloc_rqs 11718656 2861 mm/filemap.c:1919 func:__filemap_get_folio 9192960 2800 kernel/fork.c:307 func:alloc_thread_stack_node 4206592 4 net/netfilter/nf_conntrack_core.c:2567 func:nf_ct_alloc_hashtable 4136960 1010 drivers/staging/ctagmod/ctagmod.c:20 [ctagmod] func:ctagmod_start 3940352 962 mm/memory.c:4214 func:alloc_anon_folio 2894464 22613 fs/kernfs/dir.c:615 func:__kernfs_new_node ... Usage: kconfig options: - CONFIG_MEM_ALLOC_PROFILING - CONFIG_MEM_ALLOC_PROFILING_ENABLED_BY_DEFAULT - CONFIG_MEM_ALLOC_PROFILING_DEBUG adds warnings for allocations that weren't accounted because of a missing annotation sysctl: /proc/sys/vm/mem_profiling Runtime info: /proc/allocinfo Notes: [1]: Overhead To measure the overhead we are comparing the following configurations: (1) Baseline with CONFIG_MEMCG_KMEM=n (2) Disabled by default (CONFIG_MEM_ALLOC_PROFILING=y && CONFIG_MEM_ALLOC_PROFILING_BY_DEFAULT=n) (3) Enabled by default (CONFIG_MEM_ALLOC_PROFILING=y && CONFIG_MEM_ALLOC_PROFILING_BY_DEFAULT=y) (4) Enabled at runtime (CONFIG_MEM_ALLOC_PROFILING=y && CONFIG_MEM_ALLOC_PROFILING_BY_DEFAULT=n && /proc/sys/vm/mem_profiling=1) (5) Baseline with CONFIG_MEMCG_KMEM=y && allocating with __GFP_ACCOUNT (6) Disabled by default (CONFIG_MEM_ALLOC_PROFILING=y && CONFIG_MEM_ALLOC_PROFILING_BY_DEFAULT=n) && CONFIG_MEMCG_KMEM=y (7) Enabled by default (CONFIG_MEM_ALLOC_PROFILING=y && CONFIG_MEM_ALLOC_PROFILING_BY_DEFAULT=y) && CONFIG_MEMCG_KMEM=y Performance overhead: To evaluate performance we implemented an in-kernel test executing multiple get_free_page/free_page and kmalloc/kfree calls with allocation sizes growing from 8 to 240 bytes with CPU frequency set to max and CPU affinity set to a specific CPU to minimize the noise. Below are results from running the test on Ubuntu 22.04.2 LTS with 6.8.0-rc1 kernel on 56 core Intel Xeon: kmalloc pgalloc (1 baseline) 6.764s 16.902s (2 default disabled) 6.793s (+0.43%) 17.007s (+0.62%) (3 default enabled) 7.197s (+6.40%) 23.666s (+40.02%) (4 runtime enabled) 7.405s (+9.48%) 23.901s (+41.41%) (5 memcg) 13.388s (+97.94%) 48.460s (+186.71%) (6 def disabled+memcg) 13.332s (+97.10%) 48.105s (+184.61%) (7 def enabled+memcg) 13.446s (+98.78%) 54.963s (+225.18%) Memory overhead: Kernel size: text data bss dec diff (1) 26515311 18890222 17018880 62424413 (2) 26524728 19423818 16740352 62688898 264485 (3) 26524724 19423818 16740352 62688894 264481 (4) 26524728 19423818 16740352 62688898 264485 (5) 26541782 18964374 16957440 62463596 39183 Memory consumption on a 56 core Intel CPU with 125GB of memory: Code tags: 192 kB PageExts: 262144 kB (256MB) SlabExts: 9876 kB (9.6MB) PcpuExts: 512 kB (0.5MB) Total overhead is 0.2% of total memory. Benchmarks: Hackbench tests run 100 times: hackbench -s 512 -l 200 -g 15 -f 25 -P baseline disabled profiling enabled profiling avg 0.3543 0.3559 (+0.0016) 0.3566 (+0.0023) stdev 0.0137 0.0188 0.0077 hackbench -l 10000 baseline disabled profiling enabled profiling avg 6.4218 6.4306 (+0.0088) 6.5077 (+0.0859) stdev 0.0933 0.0286 0.0489 stress-ng tests: stress-ng --class memory --seq 4 -t 60 stress-ng --class cpu --seq 4 -t 60 Results posted at: https://evilpiepirate.org/~kent/memalloc_prof_v4_stress-ng/ [2] https://lore.kernel.org/all/20240306182440.2003814-1-surenb@google.com/ This patch (of 37): The next patch drops vmalloc.h from a system header in order to fix a circular dependency; this adds it to all the files that were pulling it in implicitly. [kent.overstreet@linux.dev: fix arch/alpha/lib/memcpy.c] Link: https://lkml.kernel.org/r/20240327002152.3339937-1-kent.overstreet@linux.dev [surenb@google.com: fix arch/x86/mm/numa_32.c] Link: https://lkml.kernel.org/r/20240402180933.1663992-1-surenb@google.com [kent.overstreet@linux.dev: a few places were depending on sizes.h] Link: https://lkml.kernel.org/r/20240404034744.1664840-1-kent.overstreet@linux.dev [arnd@arndb.de: fix mm/kasan/hw_tags.c] Link: https://lkml.kernel.org/r/20240404124435.3121534-1-arnd@kernel.org [surenb@google.com: fix arc build] Link: https://lkml.kernel.org/r/20240405225115.431056-1-surenb@google.com Link: https://lkml.kernel.org/r/20240321163705.3067592-1-surenb@google.com Link: https://lkml.kernel.org/r/20240321163705.3067592-2-surenb@google.com Signed-off-by: Kent Overstreet <kent.overstreet@linux.dev> Signed-off-by: Suren Baghdasaryan <surenb@google.com> Signed-off-by: Arnd Bergmann <arnd@arndb.de> Reviewed-by: Pasha Tatashin <pasha.tatashin@soleen.com> Tested-by: Kees Cook <keescook@chromium.org> Cc: Alexander Viro <viro@zeniv.linux.org.uk> Cc: Alex Gaynor <alex.gaynor@gmail.com> Cc: Alice Ryhl <aliceryhl@google.com> Cc: Andreas Hindborg <a.hindborg@samsung.com> Cc: Benno Lossin <benno.lossin@proton.me> Cc: "Björn Roy Baron" <bjorn3_gh@protonmail.com> Cc: Boqun Feng <boqun.feng@gmail.com> Cc: Christoph Lameter <cl@linux.com> Cc: Dennis Zhou <dennis@kernel.org> Cc: Gary Guo <gary@garyguo.net> Cc: Miguel Ojeda <ojeda@kernel.org> Cc: Peter Zijlstra <peterz@infradead.org> Cc: Tejun Heo <tj@kernel.org> Cc: Vlastimil Babka <vbabka@suse.cz> Cc: Wedson Almeida Filho <wedsonaf@gmail.com> Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
158 lines
3.8 KiB
C
158 lines
3.8 KiB
C
// SPDX-License-Identifier: GPL-2.0-only
|
|
/*
|
|
* linux/arch/arm/kernel/irq.c
|
|
*
|
|
* Copyright (C) 1992 Linus Torvalds
|
|
* Modifications for ARM processor Copyright (C) 1995-2000 Russell King.
|
|
*
|
|
* Support for Dynamic Tick Timer Copyright (C) 2004-2005 Nokia Corporation.
|
|
* Dynamic Tick Timer written by Tony Lindgren <tony@atomide.com> and
|
|
* Tuukka Tikkanen <tuukka.tikkanen@elektrobit.com>.
|
|
*
|
|
* This file contains the code used by various IRQ handling routines:
|
|
* asking for different IRQ's should be done through these routines
|
|
* instead of just grabbing them. Thus setups with different IRQ numbers
|
|
* shouldn't result in any weird surprises, and installing new handlers
|
|
* should be easier.
|
|
*
|
|
* IRQ's are in fact implemented a bit like signal handlers for the kernel.
|
|
* Naturally it's not a 1:1 relation, but there are similarities.
|
|
*/
|
|
#include <linux/signal.h>
|
|
#include <linux/ioport.h>
|
|
#include <linux/interrupt.h>
|
|
#include <linux/irq.h>
|
|
#include <linux/irqchip.h>
|
|
#include <linux/random.h>
|
|
#include <linux/smp.h>
|
|
#include <linux/init.h>
|
|
#include <linux/seq_file.h>
|
|
#include <linux/errno.h>
|
|
#include <linux/list.h>
|
|
#include <linux/kallsyms.h>
|
|
#include <linux/proc_fs.h>
|
|
#include <linux/export.h>
|
|
#include <linux/vmalloc.h>
|
|
|
|
#include <asm/hardware/cache-l2x0.h>
|
|
#include <asm/hardware/cache-uniphier.h>
|
|
#include <asm/outercache.h>
|
|
#include <asm/softirq_stack.h>
|
|
#include <asm/exception.h>
|
|
#include <asm/mach/arch.h>
|
|
#include <asm/mach/irq.h>
|
|
#include <asm/mach/time.h>
|
|
|
|
#include "reboot.h"
|
|
|
|
unsigned long irq_err_count;
|
|
|
|
#ifdef CONFIG_IRQSTACKS
|
|
|
|
asmlinkage DEFINE_PER_CPU_READ_MOSTLY(u8 *, irq_stack_ptr);
|
|
|
|
static void __init init_irq_stacks(void)
|
|
{
|
|
u8 *stack;
|
|
int cpu;
|
|
|
|
for_each_possible_cpu(cpu) {
|
|
if (!IS_ENABLED(CONFIG_VMAP_STACK))
|
|
stack = (u8 *)__get_free_pages(GFP_KERNEL,
|
|
THREAD_SIZE_ORDER);
|
|
else
|
|
stack = __vmalloc_node(THREAD_SIZE, THREAD_ALIGN,
|
|
THREADINFO_GFP, NUMA_NO_NODE,
|
|
__builtin_return_address(0));
|
|
|
|
if (WARN_ON(!stack))
|
|
break;
|
|
per_cpu(irq_stack_ptr, cpu) = &stack[THREAD_SIZE];
|
|
}
|
|
}
|
|
|
|
#ifdef CONFIG_SOFTIRQ_ON_OWN_STACK
|
|
static void ____do_softirq(void *arg)
|
|
{
|
|
__do_softirq();
|
|
}
|
|
|
|
void do_softirq_own_stack(void)
|
|
{
|
|
call_with_stack(____do_softirq, NULL,
|
|
__this_cpu_read(irq_stack_ptr));
|
|
}
|
|
#endif
|
|
#endif
|
|
|
|
int arch_show_interrupts(struct seq_file *p, int prec)
|
|
{
|
|
#ifdef CONFIG_FIQ
|
|
show_fiq_list(p, prec);
|
|
#endif
|
|
#ifdef CONFIG_SMP
|
|
show_ipi_list(p, prec);
|
|
#endif
|
|
seq_printf(p, "%*s: %10lu\n", prec, "Err", irq_err_count);
|
|
return 0;
|
|
}
|
|
|
|
/*
|
|
* handle_IRQ handles all hardware IRQ's. Decoded IRQs should
|
|
* not come via this function. Instead, they should provide their
|
|
* own 'handler'. Used by platform code implementing C-based 1st
|
|
* level decoding.
|
|
*/
|
|
void handle_IRQ(unsigned int irq, struct pt_regs *regs)
|
|
{
|
|
struct irq_desc *desc;
|
|
|
|
/*
|
|
* Some hardware gives randomly wrong interrupts. Rather
|
|
* than crashing, do something sensible.
|
|
*/
|
|
if (unlikely(!irq || irq >= nr_irqs))
|
|
desc = NULL;
|
|
else
|
|
desc = irq_to_desc(irq);
|
|
|
|
if (likely(desc))
|
|
handle_irq_desc(desc);
|
|
else
|
|
ack_bad_irq(irq);
|
|
}
|
|
|
|
void __init init_IRQ(void)
|
|
{
|
|
int ret;
|
|
|
|
#ifdef CONFIG_IRQSTACKS
|
|
init_irq_stacks();
|
|
#endif
|
|
|
|
if (IS_ENABLED(CONFIG_OF) && !machine_desc->init_irq)
|
|
irqchip_init();
|
|
else
|
|
machine_desc->init_irq();
|
|
|
|
if (IS_ENABLED(CONFIG_OF) && IS_ENABLED(CONFIG_CACHE_L2X0) &&
|
|
(machine_desc->l2c_aux_mask || machine_desc->l2c_aux_val)) {
|
|
if (!outer_cache.write_sec)
|
|
outer_cache.write_sec = machine_desc->l2c_write_sec;
|
|
ret = l2x0_of_init(machine_desc->l2c_aux_val,
|
|
machine_desc->l2c_aux_mask);
|
|
if (ret && ret != -ENODEV)
|
|
pr_err("L2C: failed to init: %d\n", ret);
|
|
}
|
|
|
|
uniphier_cache_init();
|
|
}
|
|
|
|
#ifdef CONFIG_SPARSE_IRQ
|
|
int __init arch_probe_nr_irqs(void)
|
|
{
|
|
nr_irqs = machine_desc->nr_irqs ? machine_desc->nr_irqs : NR_IRQS;
|
|
return nr_irqs;
|
|
}
|
|
#endif
|