Merge branch 'bpf-arm64-stack-argument-fixes'

Puranjay Mohan says:

====================
bpf, arm64: Stack argument fixes

Patch 1 fixes a redundant MOV in the arm64 JIT's
emit_stack_arg_store_imm() and clarifies the stack layout comments. This
is not a bug fix but an improvement.

Patch 2 bumps the stack argument tests from 6-8 args to at least 10 so
they actually exercise the native stack on arm64, where x0-x7 cover the
first 8 arguments.
====================

Link: https://patch.msgid.link/20260528161750.1900674-1-puranjay@kernel.org
Signed-off-by: Alexei Starovoitov <ast@kernel.org>

Author: Alexei Starovoitov

arch/arm64/net/bpf_jit_comp.c

@@ -546,7 +546,8 @@ static int build_prologue(struct jit_ctx *ctx, bool ebpf_from_cbpf)
 	 *                          low
 	 *
 	 * Stack args 6-8 are passed in x5-x7, args 9+ at [SP].
-	 * Incoming args 9+ are at [FP + 16], [FP + 24], ...
+	 * Incoming args 9+ are at [A64_FP + 16], [A64_FP + 24], ...
+	 * (above the saved FP/LR pair pushed in the callee prologue).
 	 */
 
 	emit_kcfi(is_main_prog ? cfi_bpf_hash : cfi_bpf_subprog_hash, ctx);
@@ -1235,11 +1236,12 @@ static void emit_stack_arg_store_imm(s32 imm, s16 bpf_off, const u8 tmp, struct
 {
 	int idx = -bpf_off / sizeof(u64) - 1;
 
-	emit_a64_mov_i(1, tmp, imm, ctx);
-	if (idx < NR_STACK_ARG_REGS)
-		emit(A64_MOV(1, stack_arg_reg[idx], tmp), ctx);
-	else
+	if (idx < NR_STACK_ARG_REGS) {
+		emit_a64_mov_i(1, stack_arg_reg[idx], imm, ctx);
+	} else {
+		emit_a64_mov_i(1, tmp, imm, ctx);
 		emit(A64_STR64I(tmp, A64_SP, (idx - NR_STACK_ARG_REGS) * sizeof(u64)), ctx);
+	}
 }
 
 /* JITs an eBPF instruction.

tools/testing/selftests/bpf/prog_tests/stack_arg.c

@@ -37,7 +37,7 @@ static void test_global_many(void)
 	if (!ASSERT_OK(stack_arg__load(skel), "load"))
 		goto out;
 
-	run_subtest(skel->progs.test_global_many_args, 36);
+	run_subtest(skel->progs.test_global_many_args, 55);
 
 out:
 	stack_arg__destroy(skel);
@@ -62,10 +62,10 @@ static void test_async_cb_many(void)
 	run_subtest(skel->progs.test_async_cb_many_args, 0);
 
 	/* Wait for the timer callback to fire and verify the result.
-	 * 10+20+30+40+50+60+70+80 = 360
+	 * 10+20+30+40+50+60+70+80+90+100 = 550
 	 */
 	usleep(50);
-	ASSERT_EQ(skel->bss->timer_result, 360, "timer_result");
+	ASSERT_EQ(skel->bss->timer_result, 550, "timer_result");
 
 out:
 	stack_arg__destroy(skel);
@@ -87,11 +87,11 @@ static void test_bpf2bpf(void)
 	if (!ASSERT_OK(stack_arg__load(skel), "load"))
 		goto out;
 
-	run_subtest(skel->progs.test_bpf2bpf_ptr_stack_arg, 45);
-	run_subtest(skel->progs.test_bpf2bpf_mix_stack_args, 51);
-	run_subtest(skel->progs.test_bpf2bpf_nesting_stack_arg, 50);
-	run_subtest(skel->progs.test_bpf2bpf_dynptr_stack_arg, 69);
-	run_subtest(skel->progs.test_two_callees, 91);
+	run_subtest(skel->progs.test_bpf2bpf_ptr_stack_arg, 75);
+	run_subtest(skel->progs.test_bpf2bpf_mix_stack_args, 66);
+	run_subtest(skel->progs.test_bpf2bpf_nesting_stack_arg, 84);
+	run_subtest(skel->progs.test_bpf2bpf_dynptr_stack_arg, 99);
+	run_subtest(skel->progs.test_two_callees, 133);
 
 out:
 	stack_arg__destroy(skel);
@@ -113,14 +113,14 @@ static void test_kfunc(void)
 	if (!ASSERT_OK(stack_arg_kfunc__load(skel), "load"))
 		goto out;
 
-	run_subtest(skel->progs.test_stack_arg_scalar, 36);
-	run_subtest(skel->progs.test_stack_arg_ptr, 45);
-	run_subtest(skel->progs.test_stack_arg_mix, 51);
-	run_subtest(skel->progs.test_stack_arg_dynptr, 69);
+	run_subtest(skel->progs.test_stack_arg_scalar, 55);
+	run_subtest(skel->progs.test_stack_arg_ptr, 75);
+	run_subtest(skel->progs.test_stack_arg_mix, 66);
+	run_subtest(skel->progs.test_stack_arg_dynptr, 99);
 	run_subtest(skel->progs.test_stack_arg_mem, 151);
-	run_subtest(skel->progs.test_stack_arg_iter, 115);
-	run_subtest(skel->progs.test_stack_arg_const_str, 15);
-	run_subtest(skel->progs.test_stack_arg_timer, 15);
+	run_subtest(skel->progs.test_stack_arg_iter, 145);
+	run_subtest(skel->progs.test_stack_arg_const_str, 45);
+	run_subtest(skel->progs.test_stack_arg_timer, 45);
 
 out:
 	stack_arg_kfunc__destroy(skel);

tools/testing/selftests/bpf/progs/stack_arg.c

@@ -27,14 +27,16 @@ int timer_result;
 const volatile bool has_stack_arg = true;
 
 __noinline static int static_func_many_args(int a, int b, int c, int d,
-					    int e, int f, int g, int h)
+					    int e, int f, int g, int h,
+					    int i, int j)
 {
-	return a + b + c + d + e + f + g + h;
+	return a + b + c + d + e + f + g + h + i + j;
 }
 
 __noinline int global_calls_many_args(int a, int b, int c)
 {
-	return static_func_many_args(a, b, c, 4, 5, 6, 7, 8);
+	return static_func_many_args(a, b, c, a + 3, a + 4, a + 5, a + 6,
+				     a + 7, a + 8, a + 9);
 }
 
 SEC("tc")
@@ -48,18 +50,20 @@ struct test_data {
 	long y;
 };
 
-/* 1 + 2 + 3 + 4 + 5 + 10 + 20 = 45 */
+/* 1+2+3+4+5+6+7+8+9+10+20 = 75 */
 __noinline static long func_with_ptr_stack_arg(long a, long b, long c, long d,
-					       long e, struct test_data *p)
+					       long e, long f, long g, long h,
+					       long i, struct test_data *p)
 {
-	return a + b + c + d + e + p->x + p->y;
+	return a + b + c + d + e + f + g + h + i + p->x + p->y;
 }
 
 __noinline long global_ptr_stack_arg(long a, long b, long c, long d, long e)
 {
 	struct test_data data = { .x = 10, .y = 20 };
 
-	return func_with_ptr_stack_arg(a, b, c, d, e, &data);
+	return func_with_ptr_stack_arg(a, b, c, d, e, a + 5, a + 6, a + 7,
+				      a + 8, &data);
 }
 
 SEC("tc")
@@ -68,20 +72,22 @@ int test_bpf2bpf_ptr_stack_arg(void)
 	return global_ptr_stack_arg(1, 2, 3, 4, 5);
 }
 
-/* 1 + 2 + 3 + 4 + 5 + 10 + 6 + 20 = 51 */
+/* 1+2+3+4+5+6+7+10+8+20 = 66 */
 __noinline static long func_with_mix_stack_args(long a, long b, long c, long d,
-						long e, struct test_data *p,
-						long f, struct test_data *q)
+						long e, long f, long g,
+						struct test_data *p,
+						long h, struct test_data *q)
 {
-	return a + b + c + d + e + p->x + f + q->y;
+	return a + b + c + d + e + f + g + p->x + h + q->y;
 }
 
 __noinline long global_mix_stack_args(long a, long b, long c, long d, long e)
 {
 	struct test_data p = { .x = 10 };
 	struct test_data q = { .y = 20 };
 
-	return func_with_mix_stack_args(a, b, c, d, e, &p, e + 1, &q);
+	return func_with_mix_stack_args(a, b, c, d, e, e + 1, e + 2, &p,
+					e + 3, &q);
 }
 
 SEC("tc")
@@ -94,26 +100,30 @@ int test_bpf2bpf_mix_stack_args(void)
  * Nesting test: func_outer calls func_inner, both with struct pointer
  * as stack arg.
  *
- * func_inner: (a+1) + (b+1) + (c+1) + (d+1) + (e+1) + p->x + p->y
- *           = 2 + 3 + 4 + 5 + 6 + 10 + 20 = 50
+ * func_inner: (a+1)+...+(i+1) + p->x + p->y
+ *           = 2+3+4+5+6+7+8+9+10+10+20 = 84
  */
 __noinline static long func_inner_ptr(long a, long b, long c, long d,
-				      long e, struct test_data *p)
+				      long e, long f, long g, long h,
+				      long i, struct test_data *p)
 {
-	return a + b + c + d + e + p->x + p->y;
+	return a + b + c + d + e + f + g + h + i + p->x + p->y;
 }
 
 __noinline static long func_outer_ptr(long a, long b, long c, long d,
-				      long e, struct test_data *p)
+				      long e, long f, long g, long h,
+				      long i, struct test_data *p)
 {
-	return func_inner_ptr(a + 1, b + 1, c + 1, d + 1, e + 1, p);
+	return func_inner_ptr(a + 1, b + 1, c + 1, d + 1, e + 1,
+			      f + 1, g + 1, h + 1, i + 1, p);
 }
 
 __noinline long global_nesting_ptr(long a, long b, long c, long d, long e)
 {
 	struct test_data data = { .x = 10, .y = 20 };
 
-	return func_outer_ptr(a, b, c, d, e, &data);
+	return func_outer_ptr(a, b, c, d, e, a + 5, a + 6, a + 7, a + 8,
+			      &data);
 }
 
 SEC("tc")
@@ -122,11 +132,12 @@ int test_bpf2bpf_nesting_stack_arg(void)
 	return global_nesting_ptr(1, 2, 3, 4, 5);
 }
 
-/* 1 + 2 + 3 + 4 + 5 + sizeof(pkt_v4) = 15 + 54 = 69 */
+/* 1+2+3+4+5+6+7+8+9+sizeof(pkt_v4) = 45+54 = 99 */
 __noinline static long func_with_dynptr(long a, long b, long c, long d,
-					long e, struct bpf_dynptr *ptr)
+					long e, long f, long g, long h,
+					long i, struct bpf_dynptr *ptr)
 {
-	return a + b + c + d + e + bpf_dynptr_size(ptr);
+	return a + b + c + d + e + f + g + h + i + bpf_dynptr_size(ptr);
 }
 
 __noinline long global_dynptr_stack_arg(void *ctx __arg_ctx, long a, long b,
@@ -135,7 +146,8 @@ __noinline long global_dynptr_stack_arg(void *ctx __arg_ctx, long a, long b,
 	struct bpf_dynptr ptr;
 
 	bpf_dynptr_from_skb(ctx, 0, &ptr);
-	return func_with_dynptr(a, b, c, d, d + 1, &ptr);
+	return func_with_dynptr(a, b, c, d, d + 1, d + 2, d + 3, d + 4,
+				d + 5, &ptr);
 }
 
 SEC("tc")
@@ -144,33 +156,35 @@ int test_bpf2bpf_dynptr_stack_arg(struct __sk_buff *skb)
 	return global_dynptr_stack_arg(skb, 1, 2, 3, 4);
 }
 
-/* foo1: a+b+c+d+e+f+g+h */
-__noinline static int foo1(int a, int b, int c, int d,
-			   int e, int f, int g, int h)
+/* foo1: a+b+c+d+e+f+g+h+i+j */
+__noinline static int foo1(int a, int b, int c, int d, int e,
+			   int f, int g, int h, int i, int j)
 {
-	return a + b + c + d + e + f + g + h;
+	return a + b + c + d + e + f + g + h + i + j;
 }
 
-/* foo2: a+b+c+d+e+f+g+h+i+j */
+/* foo2: a+b+c+d+e+f+g+h+i+j+k+l */
 __noinline static int foo2(int a, int b, int c, int d, int e,
-			   int f, int g, int h, int i, int j)
+			   int f, int g, int h, int i, int j,
+			   int k, int l)
 {
-	return a + b + c + d + e + f + g + h + i + j;
+	return a + b + c + d + e + f + g + h + i + j + k + l;
 }
 
-/* global_two_callees calls foo1 (3 stack args) and foo2 (5 stack args).
+/* global_two_callees calls foo1 (5 stack args) and foo2 (7 stack args).
  * The outgoing stack arg area is sized for foo2 (the larger callee).
  * Stores for foo1 are a subset of the area used by foo2.
- * Result: foo1(1,2,3,4,5,6,7,8) + foo2(1,2,3,4,5,6,7,8,9,10) = 36 + 55 = 91
+ * Result: foo1(1..10) + foo2(1..12) = 55 + 78 = 133
  *
  * Pass a-e through so the compiler can't constant-fold the stack args away.
  */
 __noinline int global_two_callees(int a, int b, int c, int d, int e)
 {
 	int ret;
 
-	ret = foo1(a, b, c, d, e, a + 5, a + 6, a + 7);
-	ret += foo2(a, b, c, d, e, a + 5, a + 6, a + 7, a + 8, a + 9);
+	ret = foo1(a, b, c, d, e, a + 5, a + 6, a + 7, a + 8, a + 9);
+	ret += foo2(a, b, c, d, e, a + 5, a + 6, a + 7, a + 8, a + 9,
+		    a + 10, a + 11);
 	return ret;
 }
 
@@ -180,9 +194,15 @@ int test_two_callees(void)
 	return global_two_callees(1, 2, 3, 4, 5);
 }
 
+const volatile int timer_base = 10;
+
 static int timer_cb_many_args(void *map, int *key, struct bpf_timer *timer)
 {
-	timer_result = static_func_many_args(10, 20, 30, 40, 50, 60, 70, 80);
+	int v = timer_base;
+
+	timer_result = static_func_many_args(v, v * 2, v * 3, v * 4, v * 5,
+					     v * 6, v * 7, v * 8, v * 9,
+					     v * 10);
 	return 0;
 }
 

tools/testing/selftests/bpf/progs/stack_arg_kfunc.c

@@ -33,15 +33,15 @@ struct {
 SEC("tc")
 int test_stack_arg_scalar(struct __sk_buff *skb)
 {
-	return bpf_kfunc_call_stack_arg(1, 2, 3, 4, 5, 6, 7, 8);
+	return bpf_kfunc_call_stack_arg(1, 2, 3, 4, 5, 6, 7, 8, 9, 10);
 }
 
 SEC("tc")
 int test_stack_arg_ptr(struct __sk_buff *skb)
 {
 	struct prog_test_pass1 p = { .x0 = 10, .x1 = 20 };
 
-	return bpf_kfunc_call_stack_arg_ptr(1, 2, 3, 4, 5, &p);
+	return bpf_kfunc_call_stack_arg_ptr(1, 2, 3, 4, 5, 6, 7, 8, 9, &p);
 }
 
 SEC("tc")
@@ -50,17 +50,17 @@ int test_stack_arg_mix(struct __sk_buff *skb)
 	struct prog_test_pass1 p = { .x0 = 10 };
 	struct prog_test_pass1 q = { .x1 = 20 };
 
-	return bpf_kfunc_call_stack_arg_mix(1, 2, 3, 4, 5, &p, 6, &q);
+	return bpf_kfunc_call_stack_arg_mix(1, 2, 3, 4, 5, 6, 7, &p, 8, &q);
 }
 
-/* 1 + 2 + 3 + 4 + 5 + sizeof(pkt_v4) = 15 + 54 = 69 */
+/* 1+2+3+4+5+6+7+8+9+sizeof(pkt_v4) = 45+54 = 99 */
 SEC("tc")
 int test_stack_arg_dynptr(struct __sk_buff *skb)
 {
 	struct bpf_dynptr ptr;
 
 	bpf_dynptr_from_skb(skb, 0, &ptr);
-	return bpf_kfunc_call_stack_arg_dynptr(1, 2, 3, 4, 5, &ptr);
+	return bpf_kfunc_call_stack_arg_dynptr(1, 2, 3, 4, 5, 6, 7, 8, 9, &ptr);
 }
 
 /* 1 + 2 + 3 + 4 + 5 + (1 + 2 + ... + 16) = 15 + 136 = 151 */
@@ -72,29 +72,30 @@ int test_stack_arg_mem(struct __sk_buff *skb)
 	return bpf_kfunc_call_stack_arg_mem(1, 2, 3, 4, 5, buf, sizeof(buf));
 }
 
-/* 1 + 2 + 3 + 4 + 5 + 100 = 115 */
+/* 1+2+3+4+5+6+7+8+9+100 = 145 */
 SEC("tc")
 int test_stack_arg_iter(struct __sk_buff *skb)
 {
 	struct bpf_iter_testmod_seq it;
 	u64 ret;
 
 	bpf_iter_testmod_seq_new(&it, 100, 10);
-	ret = bpf_kfunc_call_stack_arg_iter(1, 2, 3, 4, 5, &it);
+	ret = bpf_kfunc_call_stack_arg_iter(1, 2, 3, 4, 5, 6, 7, 8, 9, &it);
 	bpf_iter_testmod_seq_destroy(&it);
 	return ret;
 }
 
 const char cstr[] = "hello";
 
-/* 1 + 2 + 3 + 4 + 5 = 15 */
+/* 1+2+3+4+5+6+7+8+9 = 45 */
 SEC("tc")
 int test_stack_arg_const_str(struct __sk_buff *skb)
 {
-	return bpf_kfunc_call_stack_arg_const_str(1, 2, 3, 4, 5, cstr);
+	return bpf_kfunc_call_stack_arg_const_str(1, 2, 3, 4, 5, 6, 7, 8, 9,
+						  cstr);
 }
 
-/* 1 + 2 + 3 + 4 + 5 = 15 */
+/* 1+2+3+4+5+6+7+8+9 = 45 */
 SEC("tc")
 int test_stack_arg_timer(struct __sk_buff *skb)
 {
@@ -104,7 +105,8 @@ int test_stack_arg_timer(struct __sk_buff *skb)
 	val = bpf_map_lookup_elem(&kfunc_timer_map, &key);
 	if (!val)
 		return 0;
-	return bpf_kfunc_call_stack_arg_timer(1, 2, 3, 4, 5, &val->timer);
+	return bpf_kfunc_call_stack_arg_timer(1, 2, 3, 4, 5, 6, 7, 8, 9,
+					      &val->timer);
 }
 
 #else