518049d9d3
When DYNAMIC_FTRACE is enabled in the kernel, all the functions that can be traced by the function tracer have a "nop" placeholder at the start of the function. When function tracing is enabled, the nop is converted into a call to the tracing infrastructure where the functions get traced. This also allows for specifying specific functions to trace, and a lot of infrastructure is built on top of this. When DYNAMIC_FTRACE is not enabled, all the functions have a call to the ftrace trampoline. A check is made to see if a function pointer is the ftrace_stub or not, and if it is not, it calls the function pointer to trace the code. This adds over 10% overhead to the kernel even when tracing is disabled. When an architecture supports DYNAMIC_FTRACE there really is no reason to use the static tracing. I have kept non DYNAMIC_FTRACE available in x86 so that the generic code for non DYNAMIC_FTRACE can be tested. There is no reason to support non DYNAMIC_FTRACE for both x86_64 and x86_32. As the non DYNAMIC_FTRACE for x86_32 does not even support fentry, and we want to remove mcount completely, there's no reason to keep non DYNAMIC_FTRACE around for x86_32. Signed-off-by: Steven Rostedt (VMware) <rostedt@goodmis.org>
209 lines
4.3 KiB
ArmAsm
209 lines
4.3 KiB
ArmAsm
/* SPDX-License-Identifier: GPL-2.0 */
|
|
/*
|
|
* Copyright (C) 2017 Steven Rostedt, VMware Inc.
|
|
*/
|
|
|
|
#include <linux/linkage.h>
|
|
#include <asm/page_types.h>
|
|
#include <asm/segment.h>
|
|
#include <asm/export.h>
|
|
#include <asm/ftrace.h>
|
|
#include <asm/nospec-branch.h>
|
|
|
|
#ifdef CC_USING_FENTRY
|
|
# define function_hook __fentry__
|
|
EXPORT_SYMBOL(__fentry__)
|
|
#else
|
|
# define function_hook mcount
|
|
EXPORT_SYMBOL(mcount)
|
|
#endif
|
|
|
|
/* mcount uses a frame pointer even if CONFIG_FRAME_POINTER is not set */
|
|
#if !defined(CC_USING_FENTRY) || defined(CONFIG_FRAME_POINTER)
|
|
# define USING_FRAME_POINTER
|
|
#endif
|
|
|
|
#ifdef USING_FRAME_POINTER
|
|
# define MCOUNT_FRAME 1 /* using frame = true */
|
|
#else
|
|
# define MCOUNT_FRAME 0 /* using frame = false */
|
|
#endif
|
|
|
|
ENTRY(function_hook)
|
|
ret
|
|
END(function_hook)
|
|
|
|
ENTRY(ftrace_caller)
|
|
|
|
#ifdef USING_FRAME_POINTER
|
|
# ifdef CC_USING_FENTRY
|
|
/*
|
|
* Frame pointers are of ip followed by bp.
|
|
* Since fentry is an immediate jump, we are left with
|
|
* parent-ip, function-ip. We need to add a frame with
|
|
* parent-ip followed by ebp.
|
|
*/
|
|
pushl 4(%esp) /* parent ip */
|
|
pushl %ebp
|
|
movl %esp, %ebp
|
|
pushl 2*4(%esp) /* function ip */
|
|
# endif
|
|
/* For mcount, the function ip is directly above */
|
|
pushl %ebp
|
|
movl %esp, %ebp
|
|
#endif
|
|
pushl %eax
|
|
pushl %ecx
|
|
pushl %edx
|
|
pushl $0 /* Pass NULL as regs pointer */
|
|
|
|
#ifdef USING_FRAME_POINTER
|
|
/* Load parent ebp into edx */
|
|
movl 4*4(%esp), %edx
|
|
#else
|
|
/* There's no frame pointer, load the appropriate stack addr instead */
|
|
lea 4*4(%esp), %edx
|
|
#endif
|
|
|
|
movl (MCOUNT_FRAME+4)*4(%esp), %eax /* load the rip */
|
|
/* Get the parent ip */
|
|
movl 4(%edx), %edx /* edx has ebp */
|
|
|
|
movl function_trace_op, %ecx
|
|
subl $MCOUNT_INSN_SIZE, %eax
|
|
|
|
.globl ftrace_call
|
|
ftrace_call:
|
|
call ftrace_stub
|
|
|
|
addl $4, %esp /* skip NULL pointer */
|
|
popl %edx
|
|
popl %ecx
|
|
popl %eax
|
|
#ifdef USING_FRAME_POINTER
|
|
popl %ebp
|
|
# ifdef CC_USING_FENTRY
|
|
addl $4,%esp /* skip function ip */
|
|
popl %ebp /* this is the orig bp */
|
|
addl $4, %esp /* skip parent ip */
|
|
# endif
|
|
#endif
|
|
.Lftrace_ret:
|
|
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
|
|
.globl ftrace_graph_call
|
|
ftrace_graph_call:
|
|
jmp ftrace_stub
|
|
#endif
|
|
|
|
/* This is weak to keep gas from relaxing the jumps */
|
|
WEAK(ftrace_stub)
|
|
ret
|
|
END(ftrace_caller)
|
|
|
|
ENTRY(ftrace_regs_caller)
|
|
/*
|
|
* i386 does not save SS and ESP when coming from kernel.
|
|
* Instead, to get sp, ®s->sp is used (see ptrace.h).
|
|
* Unfortunately, that means eflags must be at the same location
|
|
* as the current return ip is. We move the return ip into the
|
|
* regs->ip location, and move flags into the return ip location.
|
|
*/
|
|
pushl $__KERNEL_CS
|
|
pushl 4(%esp) /* Save the return ip */
|
|
pushl $0 /* Load 0 into orig_ax */
|
|
pushl %gs
|
|
pushl %fs
|
|
pushl %es
|
|
pushl %ds
|
|
pushl %eax
|
|
|
|
/* Get flags and place them into the return ip slot */
|
|
pushf
|
|
popl %eax
|
|
movl %eax, 8*4(%esp)
|
|
|
|
pushl %ebp
|
|
pushl %edi
|
|
pushl %esi
|
|
pushl %edx
|
|
pushl %ecx
|
|
pushl %ebx
|
|
|
|
movl 12*4(%esp), %eax /* Load ip (1st parameter) */
|
|
subl $MCOUNT_INSN_SIZE, %eax /* Adjust ip */
|
|
#ifdef CC_USING_FENTRY
|
|
movl 15*4(%esp), %edx /* Load parent ip (2nd parameter) */
|
|
#else
|
|
movl 0x4(%ebp), %edx /* Load parent ip (2nd parameter) */
|
|
#endif
|
|
movl function_trace_op, %ecx /* Save ftrace_pos in 3rd parameter */
|
|
pushl %esp /* Save pt_regs as 4th parameter */
|
|
|
|
GLOBAL(ftrace_regs_call)
|
|
call ftrace_stub
|
|
|
|
addl $4, %esp /* Skip pt_regs */
|
|
|
|
/* restore flags */
|
|
push 14*4(%esp)
|
|
popf
|
|
|
|
/* Move return ip back to its original location */
|
|
movl 12*4(%esp), %eax
|
|
movl %eax, 14*4(%esp)
|
|
|
|
popl %ebx
|
|
popl %ecx
|
|
popl %edx
|
|
popl %esi
|
|
popl %edi
|
|
popl %ebp
|
|
popl %eax
|
|
popl %ds
|
|
popl %es
|
|
popl %fs
|
|
popl %gs
|
|
|
|
/* use lea to not affect flags */
|
|
lea 3*4(%esp), %esp /* Skip orig_ax, ip and cs */
|
|
|
|
jmp .Lftrace_ret
|
|
|
|
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
|
|
ENTRY(ftrace_graph_caller)
|
|
pushl %eax
|
|
pushl %ecx
|
|
pushl %edx
|
|
movl 3*4(%esp), %eax
|
|
/* Even with frame pointers, fentry doesn't have one here */
|
|
#ifdef CC_USING_FENTRY
|
|
lea 4*4(%esp), %edx
|
|
movl $0, %ecx
|
|
#else
|
|
lea 0x4(%ebp), %edx
|
|
movl (%ebp), %ecx
|
|
#endif
|
|
subl $MCOUNT_INSN_SIZE, %eax
|
|
call prepare_ftrace_return
|
|
popl %edx
|
|
popl %ecx
|
|
popl %eax
|
|
ret
|
|
END(ftrace_graph_caller)
|
|
|
|
.globl return_to_handler
|
|
return_to_handler:
|
|
pushl %eax
|
|
pushl %edx
|
|
#ifdef CC_USING_FENTRY
|
|
movl $0, %eax
|
|
#else
|
|
movl %ebp, %eax
|
|
#endif
|
|
call ftrace_return_to_handler
|
|
movl %eax, %ecx
|
|
popl %edx
|
|
popl %eax
|
|
JMP_NOSPEC %ecx
|
|
#endif
|