function_graph: Convert ret_stack to a series of longs
In order to make it possible to have multiple callbacks registered with the function_graph tracer, the retstack needs to be converted from an array of ftrace_ret_stack structures to an array of longs. This will allow to store the list of callbacks on the stack for the return side of the functions. Link: https://lore.kernel.org/linux-trace-kernel/171509092742.162236.4427737821399314856.stgit@devnote2 Link: https://lore.kernel.org/linux-trace-kernel/20240603190821.073111754@goodmis.org Cc: Mark Rutland <mark.rutland@arm.com> Cc: Mathieu Desnoyers <mathieu.desnoyers@efficios.com> Cc: Andrew Morton <akpm@linux-foundation.org> Cc: Alexei Starovoitov <alexei.starovoitov@gmail.com> Cc: Florent Revest <revest@chromium.org> Cc: Martin KaFai Lau <martin.lau@linux.dev> Cc: bpf <bpf@vger.kernel.org> Cc: Sven Schnelle <svens@linux.ibm.com> Cc: Alexei Starovoitov <ast@kernel.org> Cc: Jiri Olsa <jolsa@kernel.org> Cc: Arnaldo Carvalho de Melo <acme@kernel.org> Cc: Daniel Borkmann <daniel@iogearbox.net> Cc: Alan Maguire <alan.maguire@oracle.com> Cc: Peter Zijlstra <peterz@infradead.org> Cc: Thomas Gleixner <tglx@linutronix.de> Cc: Guo Ren <guoren@kernel.org> Reviewed-by: Masami Hiramatsu (Google) <mhiramat@kernel.org> Signed-off-by: Steven Rostedt (VMware) <rostedt@goodmis.org> Signed-off-by: Masami Hiramatsu (Google) <mhiramat@kernel.org> Signed-off-by: Steven Rostedt (Google) <rostedt@goodmis.org>
This commit is contained in:
parent
c3f38fa61a
commit
42675b723b
@ -1402,7 +1402,7 @@ struct task_struct {
|
||||
int curr_ret_depth;
|
||||
|
||||
/* Stack of return addresses for return function tracing: */
|
||||
struct ftrace_ret_stack *ret_stack;
|
||||
unsigned long *ret_stack;
|
||||
|
||||
/* Timestamp for last schedule: */
|
||||
unsigned long long ftrace_timestamp;
|
||||
|
@ -25,6 +25,30 @@
|
||||
#define ASSIGN_OPS_HASH(opsname, val)
|
||||
#endif
|
||||
|
||||
/*
|
||||
* FGRAPH_FRAME_SIZE: Size in bytes of the meta data on the shadow stack
|
||||
* FGRAPH_FRAME_OFFSET: Size in long words of the meta data frame
|
||||
* SHADOW_STACK_SIZE: The size in bytes of the entire shadow stack
|
||||
* SHADOW_STACK_OFFSET: The size in long words of the shadow stack
|
||||
* SHADOW_STACK_MAX_OFFSET: The max offset of the stack for a new frame to be added
|
||||
*/
|
||||
#define FGRAPH_FRAME_SIZE sizeof(struct ftrace_ret_stack)
|
||||
#define FGRAPH_FRAME_OFFSET (ALIGN(FGRAPH_FRAME_SIZE, sizeof(long)) / sizeof(long))
|
||||
#define SHADOW_STACK_SIZE (PAGE_SIZE)
|
||||
#define SHADOW_STACK_OFFSET \
|
||||
(ALIGN(SHADOW_STACK_SIZE, sizeof(long)) / sizeof(long))
|
||||
/* Leave on a buffer at the end */
|
||||
#define SHADOW_STACK_MAX_INDEX (SHADOW_STACK_OFFSET - FGRAPH_FRAME_OFFSET)
|
||||
|
||||
/*
|
||||
* RET_STACK(): Return the frame from a given @offset from task @t
|
||||
* RET_STACK_INC(): Reserve one frame size on the stack.
|
||||
* RET_STACK_DEC(): Remove one frame size from the stack.
|
||||
*/
|
||||
#define RET_STACK(t, index) ((struct ftrace_ret_stack *)(&(t)->ret_stack[index]))
|
||||
#define RET_STACK_INC(c) ({ c += FGRAPH_FRAME_OFFSET; })
|
||||
#define RET_STACK_DEC(c) ({ c -= FGRAPH_FRAME_OFFSET; })
|
||||
|
||||
DEFINE_STATIC_KEY_FALSE(kill_ftrace_graph);
|
||||
int ftrace_graph_active;
|
||||
|
||||
@ -69,6 +93,7 @@ static int
|
||||
ftrace_push_return_trace(unsigned long ret, unsigned long func,
|
||||
unsigned long frame_pointer, unsigned long *retp)
|
||||
{
|
||||
struct ftrace_ret_stack *ret_stack;
|
||||
unsigned long long calltime;
|
||||
int index;
|
||||
|
||||
@ -85,23 +110,25 @@ ftrace_push_return_trace(unsigned long ret, unsigned long func,
|
||||
smp_rmb();
|
||||
|
||||
/* The return trace stack is full */
|
||||
if (current->curr_ret_stack == FTRACE_RETFUNC_DEPTH - 1) {
|
||||
if (current->curr_ret_stack >= SHADOW_STACK_MAX_INDEX) {
|
||||
atomic_inc(¤t->trace_overrun);
|
||||
return -EBUSY;
|
||||
}
|
||||
|
||||
calltime = trace_clock_local();
|
||||
|
||||
index = ++current->curr_ret_stack;
|
||||
index = current->curr_ret_stack;
|
||||
RET_STACK_INC(current->curr_ret_stack);
|
||||
ret_stack = RET_STACK(current, index);
|
||||
barrier();
|
||||
current->ret_stack[index].ret = ret;
|
||||
current->ret_stack[index].func = func;
|
||||
current->ret_stack[index].calltime = calltime;
|
||||
ret_stack->ret = ret;
|
||||
ret_stack->func = func;
|
||||
ret_stack->calltime = calltime;
|
||||
#ifdef HAVE_FUNCTION_GRAPH_FP_TEST
|
||||
current->ret_stack[index].fp = frame_pointer;
|
||||
ret_stack->fp = frame_pointer;
|
||||
#endif
|
||||
#ifdef HAVE_FUNCTION_GRAPH_RET_ADDR_PTR
|
||||
current->ret_stack[index].retp = retp;
|
||||
ret_stack->retp = retp;
|
||||
#endif
|
||||
return 0;
|
||||
}
|
||||
@ -137,7 +164,7 @@ int function_graph_enter(unsigned long ret, unsigned long func,
|
||||
|
||||
return 0;
|
||||
out_ret:
|
||||
current->curr_ret_stack--;
|
||||
RET_STACK_DEC(current->curr_ret_stack);
|
||||
out:
|
||||
current->curr_ret_depth--;
|
||||
return -EBUSY;
|
||||
@ -148,11 +175,13 @@ static void
|
||||
ftrace_pop_return_trace(struct ftrace_graph_ret *trace, unsigned long *ret,
|
||||
unsigned long frame_pointer)
|
||||
{
|
||||
struct ftrace_ret_stack *ret_stack;
|
||||
int index;
|
||||
|
||||
index = current->curr_ret_stack;
|
||||
RET_STACK_DEC(index);
|
||||
|
||||
if (unlikely(index < 0 || index >= FTRACE_RETFUNC_DEPTH)) {
|
||||
if (unlikely(index < 0 || index > SHADOW_STACK_MAX_INDEX)) {
|
||||
ftrace_graph_stop();
|
||||
WARN_ON(1);
|
||||
/* Might as well panic, otherwise we have no where to go */
|
||||
@ -160,6 +189,7 @@ ftrace_pop_return_trace(struct ftrace_graph_ret *trace, unsigned long *ret,
|
||||
return;
|
||||
}
|
||||
|
||||
ret_stack = RET_STACK(current, index);
|
||||
#ifdef HAVE_FUNCTION_GRAPH_FP_TEST
|
||||
/*
|
||||
* The arch may choose to record the frame pointer used
|
||||
@ -175,22 +205,22 @@ ftrace_pop_return_trace(struct ftrace_graph_ret *trace, unsigned long *ret,
|
||||
* Note, -mfentry does not use frame pointers, and this test
|
||||
* is not needed if CC_USING_FENTRY is set.
|
||||
*/
|
||||
if (unlikely(current->ret_stack[index].fp != frame_pointer)) {
|
||||
if (unlikely(ret_stack->fp != frame_pointer)) {
|
||||
ftrace_graph_stop();
|
||||
WARN(1, "Bad frame pointer: expected %lx, received %lx\n"
|
||||
" from func %ps return to %lx\n",
|
||||
current->ret_stack[index].fp,
|
||||
frame_pointer,
|
||||
(void *)current->ret_stack[index].func,
|
||||
current->ret_stack[index].ret);
|
||||
(void *)ret_stack->func,
|
||||
ret_stack->ret);
|
||||
*ret = (unsigned long)panic;
|
||||
return;
|
||||
}
|
||||
#endif
|
||||
|
||||
*ret = current->ret_stack[index].ret;
|
||||
trace->func = current->ret_stack[index].func;
|
||||
trace->calltime = current->ret_stack[index].calltime;
|
||||
*ret = ret_stack->ret;
|
||||
trace->func = ret_stack->func;
|
||||
trace->calltime = ret_stack->calltime;
|
||||
trace->overrun = atomic_read(¤t->trace_overrun);
|
||||
trace->depth = current->curr_ret_depth--;
|
||||
/*
|
||||
@ -251,7 +281,7 @@ static unsigned long __ftrace_return_to_handler(struct fgraph_ret_regs *ret_regs
|
||||
* curr_ret_stack is after that.
|
||||
*/
|
||||
barrier();
|
||||
current->curr_ret_stack--;
|
||||
RET_STACK_DEC(current->curr_ret_stack);
|
||||
|
||||
if (unlikely(!ret)) {
|
||||
ftrace_graph_stop();
|
||||
@ -294,12 +324,13 @@ unsigned long ftrace_return_to_handler(unsigned long frame_pointer)
|
||||
struct ftrace_ret_stack *
|
||||
ftrace_graph_get_ret_stack(struct task_struct *task, int idx)
|
||||
{
|
||||
idx = task->curr_ret_stack - idx;
|
||||
int index = task->curr_ret_stack;
|
||||
|
||||
if (idx >= 0 && idx <= task->curr_ret_stack)
|
||||
return &task->ret_stack[idx];
|
||||
index -= FGRAPH_FRAME_OFFSET * (idx + 1);
|
||||
if (index < 0)
|
||||
return NULL;
|
||||
|
||||
return NULL;
|
||||
return RET_STACK(task, index);
|
||||
}
|
||||
|
||||
/**
|
||||
@ -321,18 +352,20 @@ ftrace_graph_get_ret_stack(struct task_struct *task, int idx)
|
||||
unsigned long ftrace_graph_ret_addr(struct task_struct *task, int *idx,
|
||||
unsigned long ret, unsigned long *retp)
|
||||
{
|
||||
struct ftrace_ret_stack *ret_stack;
|
||||
int index = task->curr_ret_stack;
|
||||
int i;
|
||||
|
||||
if (ret != (unsigned long)dereference_kernel_function_descriptor(return_to_handler))
|
||||
return ret;
|
||||
|
||||
if (index < 0)
|
||||
return ret;
|
||||
RET_STACK_DEC(index);
|
||||
|
||||
for (i = 0; i <= index; i++)
|
||||
if (task->ret_stack[i].retp == retp)
|
||||
return task->ret_stack[i].ret;
|
||||
for (i = index; i >= 0; RET_STACK_DEC(i)) {
|
||||
ret_stack = RET_STACK(task, i);
|
||||
if (ret_stack->retp == retp)
|
||||
return ret_stack->ret;
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
@ -346,14 +379,15 @@ unsigned long ftrace_graph_ret_addr(struct task_struct *task, int *idx,
|
||||
return ret;
|
||||
|
||||
task_idx = task->curr_ret_stack;
|
||||
RET_STACK_DEC(task_idx);
|
||||
|
||||
if (!task->ret_stack || task_idx < *idx)
|
||||
return ret;
|
||||
|
||||
task_idx -= *idx;
|
||||
(*idx)++;
|
||||
RET_STACK_INC(*idx);
|
||||
|
||||
return task->ret_stack[task_idx].ret;
|
||||
return RET_STACK(task, task_idx);
|
||||
}
|
||||
#endif /* HAVE_FUNCTION_GRAPH_RET_ADDR_PTR */
|
||||
|
||||
@ -391,7 +425,7 @@ trace_func_graph_ent_t ftrace_graph_entry = ftrace_graph_entry_stub;
|
||||
static trace_func_graph_ent_t __ftrace_graph_entry = ftrace_graph_entry_stub;
|
||||
|
||||
/* Try to assign a return stack array on FTRACE_RETSTACK_ALLOC_SIZE tasks. */
|
||||
static int alloc_retstack_tasklist(struct ftrace_ret_stack **ret_stack_list)
|
||||
static int alloc_retstack_tasklist(unsigned long **ret_stack_list)
|
||||
{
|
||||
int i;
|
||||
int ret = 0;
|
||||
@ -399,10 +433,7 @@ static int alloc_retstack_tasklist(struct ftrace_ret_stack **ret_stack_list)
|
||||
struct task_struct *g, *t;
|
||||
|
||||
for (i = 0; i < FTRACE_RETSTACK_ALLOC_SIZE; i++) {
|
||||
ret_stack_list[i] =
|
||||
kmalloc_array(FTRACE_RETFUNC_DEPTH,
|
||||
sizeof(struct ftrace_ret_stack),
|
||||
GFP_KERNEL);
|
||||
ret_stack_list[i] = kmalloc(SHADOW_STACK_SIZE, GFP_KERNEL);
|
||||
if (!ret_stack_list[i]) {
|
||||
start = 0;
|
||||
end = i;
|
||||
@ -420,9 +451,9 @@ static int alloc_retstack_tasklist(struct ftrace_ret_stack **ret_stack_list)
|
||||
|
||||
if (t->ret_stack == NULL) {
|
||||
atomic_set(&t->trace_overrun, 0);
|
||||
t->curr_ret_stack = -1;
|
||||
t->curr_ret_stack = 0;
|
||||
t->curr_ret_depth = -1;
|
||||
/* Make sure the tasks see the -1 first: */
|
||||
/* Make sure the tasks see the 0 first: */
|
||||
smp_wmb();
|
||||
t->ret_stack = ret_stack_list[start++];
|
||||
}
|
||||
@ -442,6 +473,7 @@ ftrace_graph_probe_sched_switch(void *ignore, bool preempt,
|
||||
struct task_struct *next,
|
||||
unsigned int prev_state)
|
||||
{
|
||||
struct ftrace_ret_stack *ret_stack;
|
||||
unsigned long long timestamp;
|
||||
int index;
|
||||
|
||||
@ -466,8 +498,11 @@ ftrace_graph_probe_sched_switch(void *ignore, bool preempt,
|
||||
*/
|
||||
timestamp -= next->ftrace_timestamp;
|
||||
|
||||
for (index = next->curr_ret_stack; index >= 0; index--)
|
||||
next->ret_stack[index].calltime += timestamp;
|
||||
for (index = next->curr_ret_stack - FGRAPH_FRAME_OFFSET; index >= 0; ) {
|
||||
ret_stack = RET_STACK(next, index);
|
||||
ret_stack->calltime += timestamp;
|
||||
index -= FGRAPH_FRAME_OFFSET;
|
||||
}
|
||||
}
|
||||
|
||||
static int ftrace_graph_entry_test(struct ftrace_graph_ent *trace)
|
||||
@ -510,10 +545,10 @@ void update_function_graph_func(void)
|
||||
ftrace_graph_entry = __ftrace_graph_entry;
|
||||
}
|
||||
|
||||
static DEFINE_PER_CPU(struct ftrace_ret_stack *, idle_ret_stack);
|
||||
static DEFINE_PER_CPU(unsigned long *, idle_ret_stack);
|
||||
|
||||
static void
|
||||
graph_init_task(struct task_struct *t, struct ftrace_ret_stack *ret_stack)
|
||||
graph_init_task(struct task_struct *t, unsigned long *ret_stack)
|
||||
{
|
||||
atomic_set(&t->trace_overrun, 0);
|
||||
t->ftrace_timestamp = 0;
|
||||
@ -528,7 +563,7 @@ graph_init_task(struct task_struct *t, struct ftrace_ret_stack *ret_stack)
|
||||
*/
|
||||
void ftrace_graph_init_idle_task(struct task_struct *t, int cpu)
|
||||
{
|
||||
t->curr_ret_stack = -1;
|
||||
t->curr_ret_stack = 0;
|
||||
t->curr_ret_depth = -1;
|
||||
/*
|
||||
* The idle task has no parent, it either has its own
|
||||
@ -538,14 +573,11 @@ void ftrace_graph_init_idle_task(struct task_struct *t, int cpu)
|
||||
WARN_ON(t->ret_stack != per_cpu(idle_ret_stack, cpu));
|
||||
|
||||
if (ftrace_graph_active) {
|
||||
struct ftrace_ret_stack *ret_stack;
|
||||
unsigned long *ret_stack;
|
||||
|
||||
ret_stack = per_cpu(idle_ret_stack, cpu);
|
||||
if (!ret_stack) {
|
||||
ret_stack =
|
||||
kmalloc_array(FTRACE_RETFUNC_DEPTH,
|
||||
sizeof(struct ftrace_ret_stack),
|
||||
GFP_KERNEL);
|
||||
ret_stack = kmalloc(SHADOW_STACK_SIZE, GFP_KERNEL);
|
||||
if (!ret_stack)
|
||||
return;
|
||||
per_cpu(idle_ret_stack, cpu) = ret_stack;
|
||||
@ -559,15 +591,13 @@ void ftrace_graph_init_task(struct task_struct *t)
|
||||
{
|
||||
/* Make sure we do not use the parent ret_stack */
|
||||
t->ret_stack = NULL;
|
||||
t->curr_ret_stack = -1;
|
||||
t->curr_ret_stack = 0;
|
||||
t->curr_ret_depth = -1;
|
||||
|
||||
if (ftrace_graph_active) {
|
||||
struct ftrace_ret_stack *ret_stack;
|
||||
unsigned long *ret_stack;
|
||||
|
||||
ret_stack = kmalloc_array(FTRACE_RETFUNC_DEPTH,
|
||||
sizeof(struct ftrace_ret_stack),
|
||||
GFP_KERNEL);
|
||||
ret_stack = kmalloc(SHADOW_STACK_SIZE, GFP_KERNEL);
|
||||
if (!ret_stack)
|
||||
return;
|
||||
graph_init_task(t, ret_stack);
|
||||
@ -576,7 +606,7 @@ void ftrace_graph_init_task(struct task_struct *t)
|
||||
|
||||
void ftrace_graph_exit_task(struct task_struct *t)
|
||||
{
|
||||
struct ftrace_ret_stack *ret_stack = t->ret_stack;
|
||||
unsigned long *ret_stack = t->ret_stack;
|
||||
|
||||
t->ret_stack = NULL;
|
||||
/* NULL must become visible to IRQs before we free it: */
|
||||
@ -588,12 +618,10 @@ void ftrace_graph_exit_task(struct task_struct *t)
|
||||
/* Allocate a return stack for each task */
|
||||
static int start_graph_tracing(void)
|
||||
{
|
||||
struct ftrace_ret_stack **ret_stack_list;
|
||||
unsigned long **ret_stack_list;
|
||||
int ret, cpu;
|
||||
|
||||
ret_stack_list = kmalloc_array(FTRACE_RETSTACK_ALLOC_SIZE,
|
||||
sizeof(struct ftrace_ret_stack *),
|
||||
GFP_KERNEL);
|
||||
ret_stack_list = kmalloc(SHADOW_STACK_SIZE, GFP_KERNEL);
|
||||
|
||||
if (!ret_stack_list)
|
||||
return -ENOMEM;
|
||||
|
Loading…
Reference in New Issue
Block a user