Skip to content

Commit 44eace6

Browse files
author
CKI KWF Bot
committed
Merge: Add ftrace direct call for arm64
MR: https://gitlab.com/redhat/centos-stream/src/kernel/centos-stream-9/-/merge_requests/7333 JIRA: https://issues.redhat.com/browse/RHEL-101598 This series backport the ftrace direct call on arm64 [1] that is needed for LSM BPF. It depends on the "Refactor ftrace direct call APIs" [2] and "arm64/ftrace: Add support for DYNAMIC_FTRACE_WITH_CALL_OPS" [3] series. The "arm64/ftrace: move to DYNAMIC_FTRACE_WITH_ARGS" [4] series was also backported. That one might not have been strictly necessary, but it keeps the backport closer to upstream. [1] https://lore.kernel.org/all/20230405180250.2046566-1-revest@chromium.org/ [2] https://lore.kernel.org/all/20230321140424.345218-1-revest@chromium.org/ [3] https://lore.kernel.org/all/20230123134603.1064407-1-mark.rutland@arm.com/ [4] https://lore.kernel.org/all/20221103170520.931305-1-mark.rutland@arm.com/ Ommited-fix: 68a63a4 ("arm64: Fix build with CC=clang, CONFIG_FTRACE=y and CONFIG_STACK_TRACER=y") This commit has been squashed upstream by the merge of 2aa6ac0 ("arm64: ftrace: Add direct call support") and is basically folded in the backport of said patch. The final fix is brought by commit aa67524 ("arm64: fix selection of HAVE_DYNAMIC_FTRACE_WITH_ARGS"). Signed-off-by: Jerome Marchand <jmarchan@redhat.com> Approved-by: Viktor Malik <vmalik@redhat.com> Approved-by: Joe Lawrence <joe.lawrence@redhat.com> Approved-by: Lenny Szubowicz <lszubowi@redhat.com> Approved-by: CKI KWF Bot <cki-ci-bot+kwf-gitlab-com@redhat.com> Merged-by: CKI GitLab Kmaint Pipeline Bot <26919896-cki-kmaint-pipeline-bot@users.noreply.gitlab.com>
2 parents 9772360 + aa67524 commit 44eace6

28 files changed

+1047
-754
lines changed

‎arch/arm64/Kconfig‎

Lines changed: 25 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -123,10 +123,13 @@ config ARM64
123123
select CPU_PM if (SUSPEND || CPU_IDLE)
124124
select CRC32
125125
select DCACHE_WORD_ACCESS
126+
select DYNAMIC_FTRACE if FUNCTION_TRACER
126127
select DMA_BOUNCE_UNALIGNED_KMALLOC
127128
select DMA_DIRECT_REMAP
128129
select EDAC_SUPPORT
129130
select FRAME_POINTER
131+
select FUNCTION_ALIGNMENT_4B
132+
select FUNCTION_ALIGNMENT_8B if DYNAMIC_FTRACE_WITH_CALL_OPS
130133
select GENERIC_ALLOCATOR
131134
select GENERIC_ARCH_TOPOLOGY
132135
select GENERIC_CLOCKEVENTS_BROADCAST
@@ -189,10 +192,17 @@ config ARM64
189192
select HAVE_DEBUG_KMEMLEAK
190193
select HAVE_DMA_CONTIGUOUS
191194
select HAVE_DYNAMIC_FTRACE
192-
select HAVE_DYNAMIC_FTRACE_WITH_REGS \
193-
if $(cc-option,-fpatchable-function-entry=2)
195+
select HAVE_DYNAMIC_FTRACE_WITH_ARGS \
196+
if (GCC_SUPPORTS_DYNAMIC_FTRACE_WITH_ARGS || \
197+
CLANG_SUPPORTS_DYNAMIC_FTRACE_WITH_ARGS)
198+
select HAVE_DYNAMIC_FTRACE_WITH_DIRECT_CALLS \
199+
if DYNAMIC_FTRACE_WITH_ARGS && DYNAMIC_FTRACE_WITH_CALL_OPS
200+
select HAVE_DYNAMIC_FTRACE_WITH_CALL_OPS \
201+
if (DYNAMIC_FTRACE_WITH_ARGS && !CFI_CLANG)
194202
select FTRACE_MCOUNT_USE_PATCHABLE_FUNCTION_ENTRY \
195-
if DYNAMIC_FTRACE_WITH_REGS
203+
if DYNAMIC_FTRACE_WITH_ARGS
204+
select HAVE_SAMPLE_FTRACE_DIRECT
205+
select HAVE_SAMPLE_FTRACE_DIRECT_MULTI
196206
select HAVE_EFFICIENT_UNALIGNED_ACCESS
197207
select HAVE_FAST_GUP
198208
select HAVE_FTRACE_MCOUNT_RECORD
@@ -247,6 +257,15 @@ config ARM64
247257
help
248258
ARM 64-bit (AArch64) Linux support.
249259

260+
config CLANG_SUPPORTS_DYNAMIC_FTRACE_WITH_ARGS
261+
def_bool CC_IS_CLANG
262+
# https://github.com/ClangBuiltLinux/linux/issues/1507
263+
depends on AS_IS_GNU || (AS_IS_LLVM && (LD_IS_LLD || LD_VERSION >= 23600))
264+
265+
config GCC_SUPPORTS_DYNAMIC_FTRACE_WITH_ARGS
266+
def_bool CC_IS_GCC
267+
depends on $(cc-option,-fpatchable-function-entry=2)
268+
250269
config 64BIT
251270
def_bool y
252271

@@ -1927,7 +1946,7 @@ config ARM64_PTR_AUTH_KERNEL
19271946
# which is only understood by binutils starting with version 2.33.1.
19281947
depends on LD_IS_LLD || LD_VERSION >= 23301 || (CC_IS_GCC && GCC_VERSION < 90100)
19291948
depends on !CC_IS_CLANG || AS_HAS_CFI_NEGATE_RA_STATE
1930-
depends on (!FUNCTION_GRAPH_TRACER || DYNAMIC_FTRACE_WITH_REGS)
1949+
depends on (!FUNCTION_GRAPH_TRACER || DYNAMIC_FTRACE_WITH_ARGS)
19311950
help
19321951
If the compiler supports the -mbranch-protection or
19331952
-msign-return-address flag (e.g. GCC 7 or later), then this option
@@ -1937,7 +1956,7 @@ config ARM64_PTR_AUTH_KERNEL
19371956
disabled with minimal loss of protection.
19381957

19391958
This feature works with FUNCTION_GRAPH_TRACER option only if
1940-
DYNAMIC_FTRACE_WITH_REGS is enabled.
1959+
DYNAMIC_FTRACE_WITH_ARGS is enabled.
19411960

19421961
config CC_HAS_BRANCH_PROT_PAC_RET
19431962
# GCC 9 or later, clang 8 or later
@@ -2038,7 +2057,7 @@ config ARM64_BTI_KERNEL
20382057
depends on !CC_IS_GCC
20392058
# https://github.com/llvm/llvm-project/commit/a88c722e687e6780dcd6a58718350dc76fcc4cc9
20402059
depends on !CC_IS_CLANG || CLANG_VERSION >= 120000
2041-
depends on (!FUNCTION_GRAPH_TRACER || DYNAMIC_FTRACE_WITH_REGS)
2060+
depends on (!FUNCTION_GRAPH_TRACER || DYNAMIC_FTRACE_WITH_ARGS)
20422061
help
20432062
Build the kernel with Branch Target Identification annotations
20442063
and enable enforcement of this for kernel code. When this option

‎arch/arm64/Makefile‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -127,7 +127,10 @@ endif
127127

128128
CHECKFLAGS += -D__aarch64__
129129

130-
ifeq ($(CONFIG_DYNAMIC_FTRACE_WITH_REGS),y)
130+
ifeq ($(CONFIG_DYNAMIC_FTRACE_WITH_CALL_OPS),y)
131+
KBUILD_CPPFLAGS += -DCC_USING_PATCHABLE_FUNCTION_ENTRY
132+
CC_FLAGS_FTRACE := -fpatchable-function-entry=4,2
133+
else ifeq ($(CONFIG_DYNAMIC_FTRACE_WITH_ARGS),y)
131134
KBUILD_CPPFLAGS += -DCC_USING_PATCHABLE_FUNCTION_ENTRY
132135
CC_FLAGS_FTRACE := -fpatchable-function-entry=2
133136
endif

‎arch/arm64/include/asm/ftrace.h‎

Lines changed: 93 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@
2323
*/
2424
#define HAVE_FUNCTION_GRAPH_RET_ADDR_PTR
2525

26-
#ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
26+
#ifdef CONFIG_DYNAMIC_FTRACE_WITH_ARGS
2727
#define ARCH_SUPPORTS_FTRACE_OPS 1
2828
#else
2929
#define MCOUNT_ADDR ((unsigned long)function_nocfi(_mcount))
@@ -33,8 +33,7 @@
3333
#define MCOUNT_INSN_SIZE AARCH64_INSN_SIZE
3434

3535
#define FTRACE_PLT_IDX 0
36-
#define FTRACE_REGS_PLT_IDX 1
37-
#define NR_FTRACE_PLTS 2
36+
#define NR_FTRACE_PLTS 1
3837

3938
/*
4039
* Currently, gcc tends to save the link register after the local variables
@@ -63,25 +62,102 @@ extern unsigned long ftrace_graph_call;
6362

6463
extern void return_to_handler(void);
6564

66-
static inline unsigned long ftrace_call_adjust(unsigned long addr)
65+
unsigned long ftrace_call_adjust(unsigned long addr);
66+
67+
#ifdef CONFIG_DYNAMIC_FTRACE_WITH_ARGS
68+
struct dyn_ftrace;
69+
struct ftrace_ops;
70+
71+
#define arch_ftrace_get_regs(regs) NULL
72+
73+
/*
74+
* Note: sizeof(struct ftrace_regs) must be a multiple of 16 to ensure correct
75+
* stack alignment
76+
*/
77+
struct ftrace_regs {
78+
/* x0 - x8 */
79+
unsigned long regs[9];
80+
81+
#ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
82+
unsigned long direct_tramp;
83+
#else
84+
unsigned long __unused;
85+
#endif
86+
87+
unsigned long fp;
88+
unsigned long lr;
89+
90+
unsigned long sp;
91+
unsigned long pc;
92+
};
93+
94+
static __always_inline unsigned long
95+
ftrace_regs_get_instruction_pointer(const struct ftrace_regs *fregs)
6796
{
68-
/*
69-
* Adjust addr to point at the BL in the callsite.
70-
* See ftrace_init_nop() for the callsite sequence.
71-
*/
72-
if (IS_ENABLED(CONFIG_DYNAMIC_FTRACE_WITH_REGS))
73-
return addr + AARCH64_INSN_SIZE;
74-
/*
75-
* addr is the address of the mcount call instruction.
76-
* recordmcount does the necessary offset calculation.
77-
*/
78-
return addr;
97+
return fregs->pc;
7998
}
8099

81-
#ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
82-
struct dyn_ftrace;
100+
static __always_inline void
101+
ftrace_regs_set_instruction_pointer(struct ftrace_regs *fregs,
102+
unsigned long pc)
103+
{
104+
fregs->pc = pc;
105+
}
106+
107+
static __always_inline unsigned long
108+
ftrace_regs_get_stack_pointer(const struct ftrace_regs *fregs)
109+
{
110+
return fregs->sp;
111+
}
112+
113+
static __always_inline unsigned long
114+
ftrace_regs_get_argument(struct ftrace_regs *fregs, unsigned int n)
115+
{
116+
if (n < 8)
117+
return fregs->regs[n];
118+
return 0;
119+
}
120+
121+
static __always_inline unsigned long
122+
ftrace_regs_get_return_value(const struct ftrace_regs *fregs)
123+
{
124+
return fregs->regs[0];
125+
}
126+
127+
static __always_inline void
128+
ftrace_regs_set_return_value(struct ftrace_regs *fregs,
129+
unsigned long ret)
130+
{
131+
fregs->regs[0] = ret;
132+
}
133+
134+
static __always_inline void
135+
ftrace_override_function_with_return(struct ftrace_regs *fregs)
136+
{
137+
fregs->pc = fregs->lr;
138+
}
139+
140+
int ftrace_regs_query_register_offset(const char *name);
141+
83142
int ftrace_init_nop(struct module *mod, struct dyn_ftrace *rec);
84143
#define ftrace_init_nop ftrace_init_nop
144+
145+
void ftrace_graph_func(unsigned long ip, unsigned long parent_ip,
146+
struct ftrace_ops *op, struct ftrace_regs *fregs);
147+
#define ftrace_graph_func ftrace_graph_func
148+
149+
#ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
150+
static inline void arch_ftrace_set_direct_caller(struct ftrace_regs *fregs,
151+
unsigned long addr)
152+
{
153+
/*
154+
* The ftrace trampoline will return to this address instead of the
155+
* instrumented function.
156+
*/
157+
fregs->direct_tramp = addr;
158+
}
159+
#endif /* CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS */
160+
85161
#endif
86162

87163
#define ftrace_return_address(n) return_address(n)

‎arch/arm64/include/asm/linkage.h‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,8 @@
55
#include <asm/assembler.h>
66
#endif
77

8-
#define __ALIGN .align 2
9-
#define __ALIGN_STR ".align 2"
8+
#define __ALIGN .balign CONFIG_FUNCTION_ALIGNMENT
9+
#define __ALIGN_STR ".balign " #CONFIG_FUNCTION_ALIGNMENT
1010

1111
/*
1212
* When using in-kernel BTI we need to ensure that PCS-conformant

‎arch/arm64/kernel/asm-offsets.c‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99

1010
#include <linux/arm_sdei.h>
1111
#include <linux/sched.h>
12+
#include <linux/ftrace.h>
1213
#include <linux/kexec.h>
1314
#include <linux/mm.h>
1415
#include <linux/dma-mapping.h>
@@ -83,6 +84,22 @@ int main(void)
8384
DEFINE(S_STACKFRAME, offsetof(struct pt_regs, stackframe));
8485
DEFINE(PT_REGS_SIZE, sizeof(struct pt_regs));
8586
BLANK();
87+
#ifdef CONFIG_DYNAMIC_FTRACE_WITH_ARGS
88+
DEFINE(FREGS_X0, offsetof(struct ftrace_regs, regs[0]));
89+
DEFINE(FREGS_X2, offsetof(struct ftrace_regs, regs[2]));
90+
DEFINE(FREGS_X4, offsetof(struct ftrace_regs, regs[4]));
91+
DEFINE(FREGS_X6, offsetof(struct ftrace_regs, regs[6]));
92+
DEFINE(FREGS_X8, offsetof(struct ftrace_regs, regs[8]));
93+
DEFINE(FREGS_FP, offsetof(struct ftrace_regs, fp));
94+
DEFINE(FREGS_LR, offsetof(struct ftrace_regs, lr));
95+
DEFINE(FREGS_SP, offsetof(struct ftrace_regs, sp));
96+
DEFINE(FREGS_PC, offsetof(struct ftrace_regs, pc));
97+
#ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
98+
DEFINE(FREGS_DIRECT_TRAMP, offsetof(struct ftrace_regs, direct_tramp));
99+
#endif
100+
DEFINE(FREGS_SIZE, sizeof(struct ftrace_regs));
101+
BLANK();
102+
#endif
86103
#ifdef CONFIG_COMPAT
87104
DEFINE(COMPAT_SIGFRAME_REGS_OFFSET, offsetof(struct compat_sigframe, uc.uc_mcontext.arm_r0));
88105
DEFINE(COMPAT_RT_SIGFRAME_REGS_OFFSET, offsetof(struct compat_rt_sigframe, sig.uc.uc_mcontext.arm_r0));
@@ -181,6 +198,12 @@ int main(void)
181198
DEFINE(KIMAGE_HEAD, offsetof(struct kimage, head));
182199
DEFINE(KIMAGE_START, offsetof(struct kimage, start));
183200
BLANK();
201+
#endif
202+
#ifdef CONFIG_FUNCTION_TRACER
203+
DEFINE(FTRACE_OPS_FUNC, offsetof(struct ftrace_ops, func));
204+
#ifdef CONFIG_DYNAMIC_FTRACE_WITH_DIRECT_CALLS
205+
DEFINE(FTRACE_OPS_DIRECT_CALL, offsetof(struct ftrace_ops, direct_call));
206+
#endif
184207
#endif
185208
return 0;
186209
}

0 commit comments

Comments
 (0)