diff --git a/Documentation/bpf/kfuncs.rst b/Documentation/bpf/kfuncs.rst index 85f73e0bbd0ff..89dea6b0b0241 100644 --- a/Documentation/bpf/kfuncs.rst +++ b/Documentation/bpf/kfuncs.rst @@ -575,6 +575,71 @@ is also covered by this recovery. A kfunc handed an arena pointer may therefore access up to ``GUARD_SZ / 2`` past it without bounds-checking against the arena. Larger accesses must verify the range explicitly. +2.9 kfunc Return Values +----------------------- + +A kfunc may return a scalar, a pointer, or a small struct or union by +value. A scalar or pointer of up to 8 bytes is returned in R0, as usual. + +A struct or union returned by value must be composed only of scalars +(recursively), where a scalar is an integer or an enum; arrays of scalars are +allowed as members. Its bytes are handed back to the program as the raw +contents of R0 (and R2), so a pointer field would be laundered into a scalar +and escape the verifier's pointer provenance and reference tracking. A struct +or union with a pointer member is therefore rejected at load time, and so is +one with a floating-point member, which the ABI may not return in R0:R2 at +all. + +A kfunc may also return a value larger than 8 bytes and up to 16 bytes -- a +scalar-only struct or union, or an ``__int128``. Such a value is returned +in the register pair R0:R2, matching the convention LLVM uses for the BPF +target: the first 8 bytes in R0 and the second 8 bytes in R2. A struct or +union of 8 bytes or less is returned in R0 alone. + +:: + + struct bpf_pair { __u64 a, b; }; /* 16 bytes */ + + __bpf_kfunc struct bpf_pair bpf_kfunc_get_pair(void) + { + struct bpf_pair p = { .a = 1, .b = 2 }; + + return p; /* p.a in R0, p.b in R2 */ + } + +Returning a value in the R0:R2 pair requires the JIT to place the second +half of the return value into R2, which not every architecture supports +right now. A kfunc with a return value larger than 8 bytes is therefore +rejected at load time on a JIT that does not advertise this capability (see +``bpf_jit_supports_kfunc_ret_reg_pair()``), and such a program is never run +by the interpreter. A return value larger than 16 bytes is not supported. + +The same R0:R2 convention applies to a BPF subprogram, global or static, that +returns an ``__int128`` or a struct or union larger than 8 bytes. It is only +used when the program is JITed, since the interpreter propagates only R0 out of +a subprogram: without a JIT the return value stays in R0 alone, and a caller +reading R2 is rejected for reading an uninitialized register. A global +subprogram is verified in isolation, so its by-value struct or union return is +restricted to scalars just like a kfunc's; a static subprogram is verified +inline and has no such restriction. The main program is not covered: its return +value is the program's exit code, read out of R0 alone, so a declared upper +half is never looked at. + +A global subprogram must leave a scalar in *every* register of the pair, so +both halves of the returned value have to be assigned. Leaving the upper half +uninitialized is not merely untidy: the compiler is then free to leave R2 +holding whatever it happened to hold, which for a subprogram taking a pointer +argument is typically that pointer. Handing the caller an unknown scalar built +from a pointer is a leak, so the verifier rejects it with:: + + At subprogram exit the register R2 is not a scalar value (...) + +Initialize the whole return value, for example ``struct pair p = {};``, to +avoid this. A static subprogram is exempt from the scalar-only rule: it is +verified inline, so an unassigned R2 is simply passed back to the caller as +uninitialized and only a caller that reads it fails. A stack pointer left in +R2 is still rejected there, just as one in R0 is. + .. _BPF_kfunc_lifecycle_expectations: 3. kfunc lifecycle expectations diff --git a/arch/arm64/net/bpf_jit_comp.c b/arch/arm64/net/bpf_jit_comp.c index c18e005a41dbe..3aa3ea0bc30bb 100644 --- a/arch/arm64/net/bpf_jit_comp.c +++ b/arch/arm64/net/bpf_jit_comp.c @@ -2388,6 +2388,11 @@ bool bpf_jit_supports_kfunc_call(void) return true; } +bool bpf_jit_supports_kfunc_ret_reg_pair(void) +{ + return true; +} + bool bpf_jit_supports_stack_args(void) { return true; diff --git a/arch/riscv/net/bpf_jit_comp64.c b/arch/riscv/net/bpf_jit_comp64.c index 74efe4b138d2a..47c7bf431ba85 100644 --- a/arch/riscv/net/bpf_jit_comp64.c +++ b/arch/riscv/net/bpf_jit_comp64.c @@ -2121,6 +2121,11 @@ bool bpf_jit_supports_kfunc_call(void) return true; } +bool bpf_jit_supports_kfunc_ret_reg_pair(void) +{ + return true; +} + bool bpf_jit_supports_ptr_xchg(void) { return true; diff --git a/arch/x86/net/bpf_jit_comp.c b/arch/x86/net/bpf_jit_comp.c index 1a9fb530adc3c..48429fae06410 100644 --- a/arch/x86/net/bpf_jit_comp.c +++ b/arch/x86/net/bpf_jit_comp.c @@ -1689,17 +1689,12 @@ static int emit_spectre_bhb_barrier(u8 **pprog, u8 *ip, * arena NULL is offset 0. Return the number of emitted bytes. */ static int emit_kfunc_arena_args(struct bpf_prog *bpf_prog, - const struct bpf_insn *insn, u8 **pprog) + const struct btf_func_model *fm, u8 **pprog) { - const struct btf_func_model *fm; u8 *prog = *pprog; u8 *start = prog; int i; - fm = bpf_jit_find_kfunc_model(bpf_prog, insn); - if (!fm) - return -EINVAL; - for (i = 0; i < min_t(int, fm->nr_args, MAX_BPF_FUNC_REG_ARGS); i++) { u8 flags = fm->arg_flags[i]; u32 reg = BPF_REG_1 + i; @@ -2644,6 +2639,8 @@ st: insn_off = insn->off; /* call */ case BPF_JMP | BPF_CALL: { + const struct btf_func_model *fm = NULL; + func = (u8 *) __bpf_call_base + imm32; if (src_reg == BPF_PSEUDO_CALL && tail_call_reachable) { LOAD_TAIL_CALL_CNT_PTR(stack_depth); @@ -2652,7 +2649,10 @@ st: insn_off = insn->off; if (!imm32) return -EINVAL; if (src_reg == BPF_PSEUDO_KFUNC_CALL) { - err = emit_kfunc_arena_args(bpf_prog, insn, &prog); + fm = bpf_jit_find_kfunc_model(bpf_prog, insn); + if (!fm) + return -EINVAL; + err = emit_kfunc_arena_args(bpf_prog, fm, &prog); if (err < 0) return err; ip += err; @@ -2666,6 +2666,14 @@ st: insn_off = insn->off; return -EINVAL; if (priv_frame_ptr) pop_r9(&prog); + /* + * A kfunc returning more than 8 bytes hands the second + * half back in RDX (the native ABI's second return reg), + * but BPF expects it in R0:R2. BPF R0 is RAX (no move + * needed), while BPF R2 is RSI, so copy RDX into RSI. + */ + if (fm && fm->ret_size > 8) + emit_mov_reg(&prog, true, BPF_REG_2, BPF_REG_3); break; } @@ -4156,6 +4164,11 @@ bool bpf_jit_supports_kfunc_call(void) return true; } +bool bpf_jit_supports_kfunc_ret_reg_pair(void) +{ + return true; +} + bool bpf_jit_supports_stack_args(void) { return true; diff --git a/include/linux/bpf_verifier.h b/include/linux/bpf_verifier.h index bc2af02547fe5..938c9a9eb9d21 100644 --- a/include/linux/bpf_verifier.h +++ b/include/linux/bpf_verifier.h @@ -819,6 +819,8 @@ struct bpf_subprog_info { bool is_async_cb: 1; bool is_exception_cb: 1; bool args_cached: 1; + /* true if the return value is passed in the R0:R2 register pair */ + bool ret_reg_pair: 1; /* true if bpf_fastcall stack region is used by functions that can't be inlined */ bool keep_fastcall_stack: 1; bool changes_pkt_data: 1; @@ -1055,6 +1057,11 @@ static inline struct bpf_subprog_info *subprog_info(struct bpf_verifier_env *env return &env->subprog_info[subprog]; } +static inline bool bpf_ret_reg_pair(struct bpf_verifier_env *env, int subprog) +{ + return subprog_info(env, subprog)->ret_reg_pair; +} + struct bpf_call_summary { u8 num_params; bool is_void; @@ -1460,6 +1467,8 @@ int bpf_jmp_offset(struct bpf_insn *insn); struct bpf_iarray *bpf_insn_successors(struct bpf_verifier_env *env, u32 idx); void bpf_fmt_stack_mask(char *buf, ssize_t buf_sz, u64 stack_mask); bool bpf_subprog_is_global(const struct bpf_verifier_env *env, int subprog); +bool btf_type_is_scalar_struct(struct bpf_verifier_env *env, const struct btf *btf, + const struct btf_type *t, int rec); int bpf_find_subprog(struct bpf_verifier_env *env, int off); bool bpf_is_throw_kfunc(struct bpf_insn *insn); diff --git a/include/linux/filter.h b/include/linux/filter.h index 4a9bc6a848f2e..6e746b0a09306 100644 --- a/include/linux/filter.h +++ b/include/linux/filter.h @@ -1237,6 +1237,7 @@ bool bpf_jit_inlines_helper_call(s32 imm); bool bpf_jit_supports_subprog_tailcalls(void); bool bpf_jit_supports_percpu_insn(void); bool bpf_jit_supports_kfunc_call(void); +bool bpf_jit_supports_kfunc_ret_reg_pair(void); bool bpf_jit_supports_stack_args(void); bool bpf_jit_supports_arena_args(void); bool bpf_jit_supports_far_kfunc_call(void); diff --git a/kernel/bpf/backtrack.c b/kernel/bpf/backtrack.c index a2b18a9f1694c..653db80bcc47b 100644 --- a/kernel/bpf/backtrack.c +++ b/kernel/bpf/backtrack.c @@ -423,6 +423,10 @@ static int backtrack_insn(struct bpf_verifier_env *env, int idx, int subseq_idx, */ verifier_bug_if(idx + 1 != subseq_idx, env, "extra insn from subprog"); + /* global subprog always sets R0 */ + bt_clear_reg(bt, BPF_REG_0); + /* and if it does not set R2, main pass would catch it */ + bt_clear_reg(bt, BPF_REG_2); /* r1-r5 are invalidated after subprog call, * so for global func call it shouldn't be set * anymore @@ -432,8 +436,6 @@ static int backtrack_insn(struct bpf_verifier_env *env, int idx, int subseq_idx, bt_reg_mask(bt)); return -EFAULT; } - /* global subprog always sets R0 */ - bt_clear_reg(bt, BPF_REG_0); return 0; } else { /* static subprog call instruction, which @@ -506,6 +508,8 @@ static int backtrack_insn(struct bpf_verifier_env *env, int idx, int subseq_idx, return -ENOTSUPP; /* regular helper call sets R0 */ bt_clear_reg(bt, BPF_REG_0); + /* kfunc might also set R2 */ + bt_clear_reg(bt, BPF_REG_2); if (bt_reg_mask(bt) & BPF_REGMASK_ARGS) { /* if backtracking was looking for registers R1-R5 * they should have been found already. @@ -520,7 +524,25 @@ static int backtrack_insn(struct bpf_verifier_env *env, int idx, int subseq_idx, return -EFAULT; } } else if (opcode == BPF_EXIT) { - bool r0_precise; + bool from_subprog_call, r0_precise, r2_precise; + + /* BPF_EXIT in subprog or callback always returns + * right after the call instruction, so by checking + * whether the instruction at subseq_idx-1 is subprog + * call or not we can distinguish actual exit from + * *subprog* from exit from *callback*. In the former + * case, we need to propagate the precision of the + * return registers, if necessary. In the latter we + * never do that. + */ + from_subprog_call = subseq_idx - 1 >= 0 && + bpf_pseudo_call(&env->prog->insnsi[subseq_idx - 1]); + + /* Sample the return registers before the callback + * handling below clears R1-R5. + */ + r0_precise = from_subprog_call && bt_is_reg_set(bt, BPF_REG_0); + r2_precise = from_subprog_call && bt_is_reg_set(bt, BPF_REG_2); /* Backtracking to a nested function call, 'idx' is a part of * the inner frame 'subseq_idx' is a part of the outer frame. @@ -533,30 +555,22 @@ static int backtrack_insn(struct bpf_verifier_env *env, int idx, int subseq_idx, if (subseq_idx >= 0 && bpf_calls_callback(env, subseq_idx)) for (i = BPF_REG_1; i <= BPF_REG_5; i++) bt_clear_reg(bt, i); + + bt_clear_reg(bt, BPF_REG_0); + bt_clear_reg(bt, BPF_REG_2); if (bt_reg_mask(bt) & BPF_REGMASK_ARGS) { verifier_bug(env, "backtracking exit unexpected regs %x", bt_reg_mask(bt)); return -EFAULT; } - /* BPF_EXIT in subprog or callback always returns - * right after the call instruction, so by checking - * whether the instruction at subseq_idx-1 is subprog - * call or not we can distinguish actual exit from - * *subprog* from exit from *callback*. In the former - * case, we need to propagate r0 precision, if - * necessary. In the former we never do that. - */ - r0_precise = subseq_idx - 1 >= 0 && - bpf_pseudo_call(&env->prog->insnsi[subseq_idx - 1]) && - bt_is_reg_set(bt, BPF_REG_0); - - bt_clear_reg(bt, BPF_REG_0); if (bt_subprog_enter(bt)) return -EFAULT; if (r0_precise) bt_set_reg(bt, BPF_REG_0); + if (r2_precise) + bt_set_reg(bt, BPF_REG_2); /* r6-r9 and stack slots will stay set in caller frame * bitmasks until we return back from callee(s) */ diff --git a/kernel/bpf/btf.c b/kernel/bpf/btf.c index 5b9d767895c91..ba83fa1d52a38 100644 --- a/kernel/bpf/btf.c +++ b/kernel/bpf/btf.c @@ -7591,7 +7591,7 @@ int btf_distill_func_proto(struct bpf_verifier_log *log, return -EINVAL; } ret = __get_type_size(btf, func->type, &t); - if (ret < 0 || btf_type_is_struct(t)) { + if (ret < 0 || ret > 16) { bpf_log(log, "The function %s return type %s is unsupported.\n", tname, btf_type_str(t)); @@ -7684,6 +7684,12 @@ static int btf_check_func_type_match(struct bpf_verifier_log *log, btf_type_str(t2), fn2); return -EINVAL; } + if (btf_type_has_size(t1) && (t1->size > 8 || t2->size > 8)) { + bpf_log(log, + "Return type of %s() has size %u while %s() has size %u, and a size above 8 bytes cannot be replaced\n", + fn1, t1->size, fn2, t2->size); + return -EINVAL; + } for (i = 0; i < nargs1; i++) { t1 = btf_type_skip_modifiers(btf1, args1[i].type, NULL); @@ -7964,7 +7970,7 @@ static int btf_scan_type_tags(struct bpf_verifier_env *env, /* Check whether the type is a valid return type. */ static int btf_validate_return_type(struct bpf_verifier_env *env, struct btf *btf, - const struct btf_type *t, int subprog) + const struct btf_type *t, int subprog, bool is_global) { u32 tags = 0; int err; @@ -7987,6 +7993,19 @@ static int btf_validate_return_type(struct bpf_verifier_env *env, struct btf *bt if (btf_type_is_void(t) || btf_type_is_int(t) || btf_is_any_enum(t)) return 0; + if (btf_type_is_struct(t) && t->size <= 16) { + /* + * A global function's caller models the return as an opaque + * scalar pair, so it may only return scalars by value. A local + * function is verified inline, so a pointer field stays tracked + * and needs no such restriction. + */ + bool local_func = subprog && !is_global; + + if (local_func || btf_type_is_scalar_struct(env, btf, t, 0)) + return 0; + } + return -EOPNOTSUPP; } @@ -8074,12 +8093,12 @@ int btf_prepare_func_args(struct bpf_verifier_env *env, int subprog) return -EINVAL; } - err = btf_validate_return_type(env, btf, t, subprog); + err = btf_validate_return_type(env, btf, t, subprog, is_global); if (err) { if (is_global) { bpf_log(log, - "Global function %s() return value not void or scalar. " - "Only those are supported.\n", + "Global function %s() has unsupported return type. " + "Only void, scalar, or a scalar-only struct/union up to 16 bytes is supported.\n", tname); } return err; diff --git a/kernel/bpf/core.c b/kernel/bpf/core.c index d55e737ed75ad..5db77d7915dfc 100644 --- a/kernel/bpf/core.c +++ b/kernel/bpf/core.c @@ -3287,6 +3287,11 @@ bool __weak bpf_jit_supports_kfunc_call(void) return false; } +bool __weak bpf_jit_supports_kfunc_ret_reg_pair(void) +{ + return false; +} + bool __weak bpf_jit_supports_stack_args(void) { return false; diff --git a/kernel/bpf/liveness.c b/kernel/bpf/liveness.c index 74fc4b3f80d6e..71f998c6eb888 100644 --- a/kernel/bpf/liveness.c +++ b/kernel/bpf/liveness.c @@ -2060,7 +2060,8 @@ static inline u16 mask_hi(u32 m) { return (u16)(m >> 16); } /* Compute info->{use,def} fields for the instruction */ static void compute_insn_live_regs(struct bpf_verifier_env *env, struct bpf_insn *insn, - struct insn_live_regs *info) + struct insn_live_regs *info, + bool ret_reg_pair) { struct bpf_call_summary cs; const u8 class = BPF_CLASS(insn->code); @@ -2072,6 +2073,7 @@ static void compute_insn_live_regs(struct bpf_verifier_env *env, const u32 src32 = mask_lo(src); const u32 dst32 = mask_lo(dst); const u32 r0 = reg64_mask(0); + const u32 r2 = reg64_mask(BPF_REG_2); u32 def = 0; u32 use = U32_MAX; @@ -2191,7 +2193,7 @@ static void compute_insn_live_regs(struct bpf_verifier_env *env, break; case BPF_EXIT: def = 0; - use = r0; + use = ret_reg_pair ? (r0 | r2) : r0; break; case BPF_CALL: def = ALL_CALLER_SAVED_REGS; @@ -2228,8 +2230,8 @@ int bpf_compute_live_registers(struct bpf_verifier_env *env) struct insn_live_regs *state; int insn_cnt = env->prog->len; u64 pos, insn_pos; - int err = 0, i, j; - bool changed; + int err = 0, i, j, subprog, start, end; + bool changed, ret_reg_pair; /* Use the following algorithm: * - define the following: @@ -2256,8 +2258,14 @@ int bpf_compute_live_registers(struct bpf_verifier_env *env) goto out; } - for (i = 0; i < insn_cnt; ++i) - compute_insn_live_regs(env, &insns[i], &state[i]); + for (subprog = 0; subprog < env->subprog_cnt; subprog++) { + start = env->subprog_info[subprog].start; + end = env->subprog_info[subprog + 1].start; + ret_reg_pair = bpf_ret_reg_pair(env, subprog); + + for (i = start; i < end; ++i) + compute_insn_live_regs(env, &insns[i], &state[i], ret_reg_pair); + } /* Forward pass: resolve stack access through FP-derived pointers */ err = bpf_compute_subprog_arg_access(env); diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index d17f14b35b79f..edc1b4a19f64b 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -385,27 +385,75 @@ bool bpf_subprog_is_global(const struct bpf_verifier_env *env, int subprog) return aux && aux[subprog].linkage == BTF_FUNC_GLOBAL; } -static bool subprog_returns_void(struct bpf_verifier_env *env, int subprog) +static const struct btf_type *subprog_ret_type(struct bpf_verifier_env *env, int subprog) { - const struct btf_type *type, *func, *func_proto; + const struct btf_type *func, *func_proto; const struct btf *btf = env->prog->aux->btf; u32 btf_id; + if (!btf || !env->prog->aux->func_info) + return NULL; + btf_id = env->prog->aux->func_info[subprog].type_id; + /* Both already validated by prepare_btf_func() at prog load. */ func = btf_type_by_id(btf, btf_id); - if (verifier_bug_if(!func, env, "btf_id %u not found", btf_id)) - return false; - func_proto = btf_type_by_id(btf, func->type); - if (!func_proto) - return false; - type = btf_type_skip_modifiers(btf, func_proto->type, NULL); - if (!type) - return false; + return btf_type_skip_modifiers(btf, func_proto->type, NULL); +} + +static bool subprog_returns_void(struct bpf_verifier_env *env, int subprog) +{ + const struct btf_type *type = subprog_ret_type(env, subprog); + + return type && btf_type_is_void(type); +} + +static u32 ret_regs_cnt(u32 size) +{ + return size > 8 && size <= 16 ? 2 : 1; +} + +/* Registers holding a function return value, in order. See ret_regs_cnt(). */ +static const int ret_regs[] = { BPF_REG_0, BPF_REG_2 }; + +static int bpf_compute_subprog_ret_regs(struct bpf_verifier_env *env) +{ + const struct btf *btf = env->prog->aux->btf; + const struct btf_type *type; + int subprog; + u32 size; - return btf_type_is_void(type); + if (!env->prog->jit_requested || bpf_prog_is_offloaded(env->prog->aux)) + return 0; + + /* + * Skip the main program: its return value is the program's exit code, + * read out of R0, so it never uses the register pair. An extension does + * have a real prototype for subprog 0, but bpf_check_attach_target() + * refuses to replace a function returning more than 8 bytes. + */ + for (subprog = 1; subprog < env->subprog_cnt; subprog++) { + type = subprog_ret_type(env, subprog); + /* + * This runs before btf_validate_return_type(), so apply the same + * type filter here. BTF that no compiler would emit can declare + * a return type that validation rejects, an array or a float + * say, which btf_resolve_size() would happily size at 16 bytes. + */ + if (!type || !(btf_type_is_struct(type) || btf_type_is_scalar(type))) + continue; + if (verifier_bug_if(IS_ERR(btf_resolve_size(btf, type, &size)), env, + "cannot size return type of subprog %d", subprog)) + return -EFAULT; + if (ret_regs_cnt(size) > 1) { + subprog_info(env, subprog)->ret_reg_pair = true; + env->prog->jit_required = 1; + } + } + + return 0; } const char *bpf_subprog_name(const struct bpf_verifier_env *env, int subprog) @@ -2877,6 +2925,18 @@ int bpf_add_kfunc_call(struct bpf_verifier_env *env, u32 func_id, u16 offset) err = btf_distill_func_proto(&env->log, kfunc.btf, kfunc.proto, kfunc.name, &func_model); if (err) return err; + if (func_model.ret_size > 8) { + if (kfunc.flags && (*kfunc.flags & KF_FASTCALL)) { + verbose(env, "kfunc %s with >8-byte return is not supported with KF_FASTCALL\n", + kfunc.name); + return -EOPNOTSUPP; + } + if (!bpf_jit_supports_kfunc_ret_reg_pair()) { + verbose(env, "kfunc %s with >8-byte return is not supported by JIT\n", + kfunc.name); + return -EOPNOTSUPP; + } + } memset(&meta, 0, sizeof(meta)); meta.btf = kfunc.btf; @@ -9855,6 +9915,7 @@ static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn, u16 callee_incoming, stack_arg_cnt; struct bpf_func_state *caller; int err, subprog, target_insn; + u32 i, nregs; target_insn = *insn_idx + insn->imm + 1; subprog = bpf_find_subprog(env, target_insn); @@ -9910,9 +9971,14 @@ static int check_func_call(struct bpf_verifier_env *env, struct bpf_insn *insn, clear_caller_saved_regs(env, caller->regs); invalidate_outgoing_stack_args(env, cur_func(env)); - /* All non-void global functions return a 64-bit SCALAR_VALUE. */ + /* + * A non-void global function returns a 64-bit SCALAR_VALUE in + * R0, or a >8 byte SCALAR_VALUE in the R0:R2 register pair. + */ if (!returns_void) { - mark_reg_unknown(env, caller->regs, BPF_REG_0); + nregs = bpf_ret_reg_pair(env, subprog) ? 2 : 1; + for (i = 0; i < nregs; i++) + mark_reg_unknown(env, caller->regs, ret_regs[i]); bpf_diag_mod_end(env); } @@ -10272,11 +10338,15 @@ static int prepare_func_exit(struct bpf_verifier_env *env, int *insn_idx) struct bpf_func_state *caller, *callee; struct bpf_reg_state *r0; bool in_callback_fn; + u32 i, nregs; int err; callee = state->frame[state->curframe]; r0 = &callee->regs[BPF_REG_0]; - if (r0->type == PTR_TO_STACK) { + nregs = bpf_ret_reg_pair(env, callee->subprogno) ? 2 : 1; + for (i = 0; i < nregs; i++) { + if (callee->regs[ret_regs[i]].type != PTR_TO_STACK) + continue; /* technically it's ok to return caller's stack pointer * (or caller's caller's pointer) back to the caller, * since these pointers are valid. Only current stack @@ -10311,9 +10381,13 @@ static int prepare_func_exit(struct bpf_verifier_env *env, int *insn_idx) return -EFAULT; } } else { - /* return to the caller whatever r0 had in the callee */ + /* + * return to the caller whatever the callee had in the + * return register(s) + */ bpf_diag_mod_begin(env, &caller->regs[BPF_REG_0], r0, BPF_DIAG_MOD_WRITE); - caller->regs[BPF_REG_0] = *r0; + for (i = 0; i < nregs; i++) + caller->regs[ret_regs[i]] = callee->regs[ret_regs[i]]; bpf_diag_mod_end(env); } @@ -11248,6 +11322,19 @@ static int check_helper_call(struct bpf_verifier_env *env, struct bpf_insn *insn return 0; } +/* + * Mark the register(s) holding a @size byte kfunc return value as unknown + * scalars. Both halves of a register pair are treated the same way. + */ +static void mark_kfunc_ret_regs(struct bpf_verifier_env *env, + struct bpf_reg_state *regs, u32 size) +{ + u32 i, nregs = ret_regs_cnt(size); + + for (i = 0; i < nregs; i++) + mark_reg_unknown(env, regs, ret_regs[i]); +} + static bool is_kfunc_acquire(struct bpf_call_arg_meta *meta) { return meta->kfunc_flags & KF_ACQUIRE; @@ -11515,9 +11602,9 @@ static bool is_kfunc_arg_implicit(const struct bpf_call_arg_meta *meta, u32 arg_ } /* Returns true if struct is composed of scalars, 4 levels of nesting allowed */ -static bool __btf_type_is_scalar_struct(struct bpf_verifier_env *env, - const struct btf *btf, - const struct btf_type *t, int rec) +bool btf_type_is_scalar_struct(struct bpf_verifier_env *env, + const struct btf *btf, + const struct btf_type *t, int rec) { const struct btf_type *member_type; const struct btf_member *member; @@ -11535,7 +11622,7 @@ static bool __btf_type_is_scalar_struct(struct bpf_verifier_env *env, verbose(env, "max struct nesting depth exceeded\n"); return false; } - if (!__btf_type_is_scalar_struct(env, btf, member_type, rec + 1)) + if (!btf_type_is_scalar_struct(env, btf, member_type, rec + 1)) return false; continue; } @@ -11934,7 +12021,7 @@ get_kfunc_arg_type(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, (is_kfunc_arg_mem_size(meta->btf, &args[arg + 1]) || is_kfunc_arg_const_mem_size(meta->btf, &args[arg + 1]))) { if (!btf_type_is_void(ref_t) && !btf_type_is_scalar(ref_t) && - !__btf_type_is_scalar_struct(env, meta->btf, ref_t, 0)) { + !btf_type_is_scalar_struct(env, meta->btf, ref_t, 0)) { verbose(env, "%s pointer type %s %s must point to void, scalar, or struct with scalar\n", reg_arg_name(env, argno), btf_type_str(ref_t), ref_tname); return -EINVAL; @@ -11950,7 +12037,7 @@ get_kfunc_arg_type(struct bpf_verifier_env *env, struct bpf_call_arg_meta *meta, * scalars. The access size is derived from the pointed-to BTF type. */ if (!btf_type_is_scalar(ref_t) && - !__btf_type_is_scalar_struct(env, meta->btf, ref_t, 0)) { + !btf_type_is_scalar_struct(env, meta->btf, ref_t, 0)) { verbose(env, "%s pointer type %s %s must point to scalar, or struct with scalar\n", reg_arg_name(env, argno), btf_type_str(ref_t), ref_tname); return -EINVAL; @@ -13006,7 +13093,7 @@ static int check_kfunc_args(struct bpf_verifier_env *env, struct bpf_call_arg_me break; } - if (!__btf_type_is_scalar_struct(env, meta->btf, ref_t, 0)) { + if (!btf_type_is_scalar_struct(env, meta->btf, ref_t, 0)) { enum bpf_reg_type reg2btf_type = lookup_reg2btf_ids(ref_id); const char *expected_type; @@ -13544,7 +13631,7 @@ static int check_special_kfunc(struct bpf_verifier_env *env, struct bpf_call_arg struct_meta = btf_find_struct_meta(ret_btf, ret_btf_id); if (is_bpf_percpu_obj_new_kfunc(meta->func_id)) { - if (!__btf_type_is_scalar_struct(env, ret_btf, ret_t, 0)) { + if (!btf_type_is_scalar_struct(env, ret_btf, ret_t, 0)) { verbose(env, "bpf_percpu_obj_new type ID argument must be of a struct of scalars\n"); return -EINVAL; } @@ -13912,10 +13999,25 @@ static int check_kfunc_call(struct bpf_verifier_env *env, struct bpf_insn *insn, } if (btf_type_is_scalar(t)) { - mark_reg_unknown(env, regs, BPF_REG_0); + mark_kfunc_ret_regs(env, regs, t->size); if (meta.btf == btf_vmlinux && (meta.func_id == special_kfunc_list[KF_bpf_res_spin_lock] || meta.func_id == special_kfunc_list[KF_bpf_res_spin_lock_irqsave])) __mark_reg_const_zero(env, ®s[BPF_REG_0]); + } else if (btf_type_is_struct(t)) { + /* + * The returned struct comes back as raw register bits modeled + * as an unknown scalar, so it must contain only scalars: + * otherwise a pointer field would be laundered into a scalar + * and escape provenance and reference tracking. + */ + if (!btf_type_is_scalar_struct(env, desc_btf, t, 0)) { + verbose(env, + "kernel function %s returns %s %s that is not composed of scalars\n", + func_name, btf_type_str(t), + btf_name_by_offset(desc_btf, t->name_off)); + return -EINVAL; + } + mark_kfunc_ret_regs(env, regs, t->size); } else if (btf_type_is_ptr(t)) { ptr_type = btf_type_skip_modifiers(desc_btf, t->type, &ptr_type_id); err = check_special_kfunc(env, &meta, regs, insn_aux, ptr_type, desc_btf); @@ -17451,37 +17553,54 @@ static int check_return_code(struct bpf_verifier_env *env, int regno, const char return 0; } -static int check_global_subprog_return_code(struct bpf_verifier_env *env) +static int check_global_ret_scalar_reg(struct bpf_verifier_env *env, u32 regno) { - struct bpf_reg_state *reg = reg_state(env, BPF_REG_0); - struct bpf_func_state *cur_frame = cur_func(env); + struct bpf_reg_state *reg; int err; - if (subprog_returns_void(env, cur_frame->subprogno)) - return 0; - - err = check_reg_arg(env, BPF_REG_0, SRC_OP); + err = check_reg_arg(env, regno, SRC_OP); if (err) return err; /* Pointers to arena are safe to pass between subprograms. */ - if (is_arena_reg(env, BPF_REG_0)) + if (is_arena_reg(env, regno)) return 0; - if (is_pointer_value(env, BPF_REG_0)) { - verbose(env, "R%d leaks addr as return value\n", BPF_REG_0); + if (is_pointer_value(env, regno)) { + verbose(env, "R%d leaks addr as return value\n", regno); return -EACCES; } + reg = reg_state(env, regno); if (reg->type != SCALAR_VALUE) { - verbose(env, "At subprogram exit the register R0 is not a scalar value (%s)\n", - reg_type_str(env, reg->type)); + verbose(env, "At subprogram exit the register R%d is not a scalar value (%s)\n", + regno, reg_type_str(env, reg->type)); return -EINVAL; } return 0; } +static int check_global_subprog_return_code(struct bpf_verifier_env *env) +{ + struct bpf_func_state *cur_frame = cur_func(env); + u32 subprog = cur_frame->subprogno; + u32 i, nregs; + int err; + + if (subprog_returns_void(env, subprog)) + return 0; + + nregs = bpf_ret_reg_pair(env, subprog) ? 2 : 1; + for (i = 0; i < nregs; i++) { + err = check_global_ret_scalar_reg(env, ret_regs[i]); + if (err) + return err; + } + + return 0; +} + /* Bitmask with 1s for all caller saved registers */ #define ALL_CALLER_SAVED_REGS ((1u << CALLER_SAVED_REGS) - 1) @@ -21153,6 +21272,11 @@ int bpf_check(struct bpf_prog **prog, union bpf_attr *attr, bpfptr_t uattr, if (ret < 0) goto skip_full_check; + /* must precede the first bpf_ret_reg_pair() user below */ + ret = bpf_compute_subprog_ret_regs(env); + if (ret < 0) + goto skip_full_check; + ret = bpf_compute_live_registers(env); if (ret < 0) goto skip_full_check; diff --git a/tools/testing/selftests/bpf/prog_tests/aggregate_ret.c b/tools/testing/selftests/bpf/prog_tests/aggregate_ret.c new file mode 100644 index 0000000000000..e0b94ed10f941 --- /dev/null +++ b/tools/testing/selftests/bpf/prog_tests/aggregate_ret.c @@ -0,0 +1,11 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ +#include +#include "aggregate_ret_func.skel.h" +#include "aggregate_ret_kfunc.skel.h" + +void test_aggregate_ret(void) +{ + RUN_TESTS(aggregate_ret_func); + RUN_TESTS(aggregate_ret_kfunc); +} diff --git a/tools/testing/selftests/bpf/prog_tests/fexit_bpf2bpf.c b/tools/testing/selftests/bpf/prog_tests/fexit_bpf2bpf.c index 2523c07a16c65..6c438df380bb1 100644 --- a/tools/testing/selftests/bpf/prog_tests/fexit_bpf2bpf.c +++ b/tools/testing/selftests/bpf/prog_tests/fexit_bpf2bpf.c @@ -441,6 +441,20 @@ static void test_func_replace_int_with_void(void) " doesn't match type INT of global_func2()"); } +static void test_func_replace_ret_pair(void) +{ + const char *msg = "Return type of new_agg_ret_target_func() has size 8 " + "while agg_ret_target_func() has size 16"; + + /* + * An extension cannot replace a function whose return value comes back + * in the R0:R2 pair: only R0 is checked at the extension's exit, so it + * would leave R2 stale for the target's callers. + */ + test_obj_load_failure_common("freplace_ret_pair.bpf.o", + "./aggregate_ret_target.bpf.o", msg); +} + static int find_prog_btf_id(const char *name, __u32 attach_prog_fd) { struct bpf_prog_info info = {}; @@ -660,6 +674,8 @@ void serial_test_fexit_bpf2bpf(void) test_func_replace_progmap(); if (test__start_subtest("freplace_int_with_void")) test_func_replace_int_with_void(); + if (test__start_subtest("freplace_ret_pair")) + test_func_replace_ret_pair(); if (test__start_subtest("freplace_void")) test_func_replace_void(); if (test__start_subtest("sleepable_fentry_to_xdp")) diff --git a/tools/testing/selftests/bpf/prog_tests/verifier.c b/tools/testing/selftests/bpf/prog_tests/verifier.c index 64ac49ad67e63..f7f94ccebce27 100644 --- a/tools/testing/selftests/bpf/prog_tests/verifier.c +++ b/tools/testing/selftests/bpf/prog_tests/verifier.c @@ -5,6 +5,7 @@ #include "arena_kfunc.skel.h" #include "arena_kfunc_jit.skel.h" #include "cap_helpers.h" +#include "verifier_aggregate_ret.skel.h" #include "verifier_align.skel.h" #include "verifier_and.skel.h" #include "verifier_arena.skel.h" @@ -170,6 +171,7 @@ void test_arena_kfunc(void) { RUN_TESTS(arena_kfunc); } void test_arena_kfunc_jit(void) { RUN_TESTS(arena_kfunc_jit); } +void test_verifier_aggregate_ret(void) { RUN_TESTS(verifier_aggregate_ret); } void test_verifier_align(void) { RUN(verifier_align); } void test_verifier_and(void) { RUN(verifier_and); } void test_verifier_arena(void) { RUN(verifier_arena); } diff --git a/tools/testing/selftests/bpf/progs/aggregate_ret_func.c b/tools/testing/selftests/bpf/progs/aggregate_ret_func.c new file mode 100644 index 0000000000000..e35baaa10fea2 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/aggregate_ret_func.c @@ -0,0 +1,235 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ +#include +#include +#include "bpf_misc.h" + +typedef unsigned __int128 u128; + +__naked u128 global_agg_good(void) +{ + asm volatile ( + "r0 = 0x1234;" /* low 64 bits */ + "r2 = 0x5678;" /* high 64 bits */ + "exit;" + ); +} + +__naked u128 global_agg_bad(void) +{ + asm volatile ( + "r0 = 0;" + "exit;" + ); +} + +__naked u128 global_agg_bad_ptr(void) +{ + asm volatile ( + "r0 = 0;" + "r2 = r10;" + "exit;" + ); +} + +SEC("tc") +__failure __msg("R2 !read_ok") +__naked int aggregate_ret_global_fail(void) +{ + asm volatile ( + "call %[global_agg_bad];" + "r0 = r2;" + "exit;" + : + : __imm(global_agg_bad) + : __clobber_all); +} + +SEC("tc") +__failure __msg("At subprogram exit the register R2 is not a scalar value") +__naked int aggregate_ret_global_ptr_fail(void) +{ + asm volatile ( + "call %[global_agg_bad_ptr];" + "r0 = r2;" + "exit;" + : + : __imm(global_agg_bad_ptr) + : __clobber_all); +} + +static __naked __noinline u128 static_agg_bad_ptr(void) +{ + asm volatile ( + "r0 = 0;" + "r2 = r10;" /* stack pointer placed in the second return register */ + "exit;" + ); +} + +/* + * R2 is a return register once the subprogram returns a pair, so a stack + * pointer left in it is rejected at the callee's exit exactly as one in R0 + * is: the callee frame is gone by the time the caller could use it. + */ +SEC("tc") +__failure __msg("cannot return stack pointer to the caller") +__naked int aggregate_ret_static_ptr_fail(void) +{ + asm volatile ( + "call %[static_agg_bad_ptr];" + "r0 = 0;" + "exit;" + : + : __imm(static_agg_bad_ptr) + : __clobber_all); +} + +static __naked __noinline u128 static_agg_no_r2(void) +{ + asm volatile ( + "r0 = 0;" + "exit;" + ); +} + +SEC("tc") +__failure __msg("R2 !read_ok") +__naked int aggregate_ret_static_uninit_fail(void) +{ + asm volatile ( + "call %[static_agg_no_r2];" + "r0 = r2;" + "exit;" + : + : __imm(static_agg_no_r2) + : __clobber_all); +} + +static __naked __noinline u128 static_agg_precise(void) +{ + asm volatile ( + "r0 = 0;" + "r2 = 4;" /* second half; its value is made precise below */ + "exit;" + ); +} + +SEC("tc") +__load_if_JITed() +__success __retval(0) +__log_level(2) +__msg("mark_precise: frame0: last_idx 5 first_idx 0 subseq_idx -1") +__msg("mark_precise: frame0: regs=r6 stack= before 4: (07) r1 += -8") +__msg("mark_precise: frame0: regs=r6 stack= before 3: (bf) r1 = r10") +__msg("mark_precise: frame0: regs=r6 stack= before 2: (57) r6 &= 7") +__msg("mark_precise: frame0: regs=r6 stack= before 1: (bf) r6 = r2") +__msg("mark_precise: frame0: regs=r2 stack= before 12: (95) exit") +__msg("mark_precise: frame1: regs=r2 stack= before 11: (b7) r2 = 4") +__naked int aggregate_ret_static_precise(void) +{ + asm volatile ( + "call %[static_agg_precise];" + "r6 = r2;" /* derived from the aggregate's second half */ + "r6 &= 7;" /* keep it in [0, 7] to index the stack */ + "r1 = r10;" + "r1 += -8;" + "r1 += r6;" /* ptr += scalar marks r6 (hence R2) precise */ + "r0 = 0;" + "*(u8 *)(r1 + 0) = r0;" + "r0 = 0;" + "exit;" + : + : __imm(static_agg_precise) + : __clobber_all); +} + +SEC("tc") +__load_if_JITed() +__success __retval(0) +__log_level(2) +__msg("mark_precise: frame0: last_idx 5 first_idx 0 subseq_idx -1") +__msg("mark_precise: frame0: regs=r6 stack= before 4: (07) r1 += -8") +__msg("mark_precise: frame0: regs=r6 stack= before 3: (bf) r1 = r10") +__msg("mark_precise: frame0: regs=r6 stack= before 2: (57) r6 &= 7") +__msg("mark_precise: frame0: regs=r6 stack= before 1: (bf) r6 = r2") +__msg("mark_precise: frame0: regs=r2 stack= before 0: (85) call pc+9") +__naked int aggregate_ret_global_precise(void) +{ + asm volatile ( + "call %[global_agg_good];" + "r6 = r2;" /* derived from the aggregate's second half */ + "r6 &= 7;" /* keep it in [0, 7] to index the stack */ + "r1 = r10;" + "r1 += -8;" + "r1 += r6;" /* ptr += scalar marks r6 (hence R2) precise */ + "r0 = 0;" + "*(u8 *)(r1 + 0) = r0;" + "r0 = 0;" + "exit;" + : + : __imm(global_agg_good) + : __clobber_all); +} + +#if defined(__clang_major__) && __clang_major__ >= 23 + +/* A by-value struct that smuggles a pointer, which must be rejected. */ +struct with_ptr { + void *p; + __u64 x; +}; + +/* A by-value union that smuggles a pointer, which must be rejected too. */ +union upair_with_ptr { + void *p; + __u64 halves[2]; +}; + +__naked struct with_ptr global_ret_struct_ptr(void) +{ + asm volatile ( + "r0 = 0;" + "r2 = 0;" + "exit;" + ); +} + +SEC("tc") +__failure __msg("Global function global_ret_struct_ptr() has unsupported return type") +__naked int aggregate_ret_global_struct_ptr_fail(void) +{ + asm volatile ( + "call %[global_ret_struct_ptr];" + "r0 = 0;" + "exit;" + : + : __imm(global_ret_struct_ptr) + : __clobber_all); +} + +__naked union upair_with_ptr global_ret_union_ptr(void) +{ + asm volatile ( + "r0 = 0;" + "r2 = 0;" + "exit;" + ); +} + +SEC("tc") +__failure __msg("Global function global_ret_union_ptr() has unsupported return type") +__naked int aggregate_ret_global_union_ptr_fail(void) +{ + asm volatile ( + "call %[global_ret_union_ptr];" + "r0 = 0;" + "exit;" + : + : __imm(global_ret_union_ptr) + : __clobber_all); +} + +#endif + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/aggregate_ret_kfunc.c b/tools/testing/selftests/bpf/progs/aggregate_ret_kfunc.c new file mode 100644 index 0000000000000..c23b4beb1773c --- /dev/null +++ b/tools/testing/selftests/bpf/progs/aggregate_ret_kfunc.c @@ -0,0 +1,122 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ +#include +#include +#include "bpf_misc.h" +#include "../test_kmods/bpf_testmod_kfunc.h" + +/* + * Reference kfunc addresses to force those BTF to be emitted. Taking the address + * (rather than calling) avoids any dependence on the compiler lowering an + * __int128 or struct return value, which the BPF backend only supports from + * LLVM 23 on. + */ +void __kfunc_btf_root(void) +{ + asm volatile ("" + : + : "r"(&bpf_kfunc_call_test_i128), + "r"(&bpf_kfunc_call_test_ret_fastcall), + "r"(&bpf_kfunc_call_test_ret_ptr), + "r"(&bpf_kfunc_call_test_ret_ii), + "r"(&bpf_kfunc_call_test_ret_big)); +} + +SEC("tc") +__arch_x86_64 __arch_arm64 +__load_if_JITed() +__success __retval(0) +__log_level(2) +__msg("mark_precise: frame0: last_idx 7 first_idx 0 subseq_idx -1") +__msg("mark_precise: frame0: regs=r6 stack= before 6: (07) r1 += -8") +__msg("mark_precise: frame0: regs=r6 stack= before 5: (bf) r1 = r10") +__msg("mark_precise: frame0: regs=r6 stack= before 4: (57) r6 &= 7") +__msg("mark_precise: frame0: regs=r6 stack= before 3: (bf) r6 = r2") +__msg("mark_precise: frame0: regs=r2 stack= before 2: (85) call bpf_kfunc_call_test_i128") +__naked int aggregate_ret_kfunc_precise(void) +{ + asm volatile ( + "r1 = 1;" + "r2 = 2;" + "call %[bpf_kfunc_call_test_i128];" + "r6 = r2;" /* second return half */ + "r6 &= 7;" /* keep it in [0, 7] to index the stack */ + "r1 = r10;" + "r1 += -8;" + "r1 += r6;" /* ptr += scalar marks r6 (hence R2) precise */ + "r0 = 0;" + "*(u8 *)(r1 + 0) = r0;" + "r0 = 0;" + "exit;" + : + : __imm(bpf_kfunc_call_test_i128) + : __clobber_all); +} + +SEC("tc") +__arch_x86_64 __arch_arm64 +__failure __msg("kfunc bpf_kfunc_call_test_ret_fastcall with >8-byte return is not supported with KF_FASTCALL") +__naked int aggregate_ret_kfunc_fastcall_fail(void) +{ + asm volatile ( + "r1 = 1;" + "r2 = 2;" + "call %[bpf_kfunc_call_test_ret_fastcall];" + "r0 = 0;" + "exit;" + : + : __imm(bpf_kfunc_call_test_ret_fastcall) + : __clobber_all); +} + +SEC("tc") +__arch_x86_64 __arch_arm64 +__failure __msg("is not composed of scalars") +__naked int aggregate_ret_kfunc_ptr_fail(void) +{ + asm volatile ( + "r1 = 0;" + "call %[bpf_kfunc_call_test_ret_ptr];" + "r0 = 0;" + "exit;" + : + : __imm(bpf_kfunc_call_test_ret_ptr) + : __clobber_all); +} + +SEC("tc") +__arch_x86_64 __arch_arm64 +__failure __msg("R2 !read_ok") +__naked int aggregate_ret_kfunc_small_no_r2(void) +{ + asm volatile ( + "r1 = 0;" + "r2 = 0;" + "call %[bpf_kfunc_call_test_ret_ii];" + "r0 = r2;" /* R2 is not a return register for a <=8 byte struct */ + "exit;" + : + : __imm(bpf_kfunc_call_test_ret_ii) + : __clobber_all); +} + +/* + * A return value larger than 16 bytes does not fit in R0:R2 and is rejected by + * btf_distill_func_proto(), before the KF_FASTCALL and JIT-capability checks, + * so this behaves the same on every architecture. + */ +SEC("tc") +__arch_x86_64 __arch_arm64 +__failure __msg("The function bpf_kfunc_call_test_ret_big return type STRUCT is unsupported") +__naked int aggregate_ret_kfunc_too_big_fail(void) +{ + asm volatile ( + "call %[bpf_kfunc_call_test_ret_big];" + "r0 = 0;" + "exit;" + : + : __imm(bpf_kfunc_call_test_ret_big) + : __clobber_all); +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/aggregate_ret_target.c b/tools/testing/selftests/bpf/progs/aggregate_ret_target.c new file mode 100644 index 0000000000000..cffd8d7d3241a --- /dev/null +++ b/tools/testing/selftests/bpf/progs/aggregate_ret_target.c @@ -0,0 +1,29 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ +#include +#include +#include "bpf_misc.h" + +/* freplace target: a global subprogram returning 16 bytes in R0:R2. */ +__naked unsigned __int128 agg_ret_target_func(void) +{ + asm volatile ( + "r0 = 0x1234;" + "r2 = 0x5678;" + "exit;" + ); +} + +SEC("tc") +__naked int agg_ret_target(void) +{ + asm volatile ( + "call %[agg_ret_target_func];" + "r0 = 0;" + "exit;" + : + : __imm(agg_ret_target_func) + : __clobber_all); +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/compute_live_registers.c b/tools/testing/selftests/bpf/progs/compute_live_registers.c index d055fc7b3b95d..0be9441ec273b 100644 --- a/tools/testing/selftests/bpf/progs/compute_live_registers.c +++ b/tools/testing/selftests/bpf/progs/compute_live_registers.c @@ -431,6 +431,36 @@ __naked void subprog1(void) ::: __clobber_all); } +static __used __naked unsigned __int128 aux2(void) +{ + asm volatile ( + "r0 = 1;" + "r2 = 2;" + "exit;" + ::: __clobber_all); +} + +SEC("socket") +/* A program observing the pair needs the JIT; see bpf_compute_subprog_ret_regs(). */ +__load_if_JITed() +__log_level(2) +__msg("0: .12345.... (85) call pc+2") +__msg("1: ..2....... (bf) r0 = r2") +/* R2 is not read at the exit of this program, which returns an int, ... */ +__msg("2: 0......... (95) exit") +__msg("3: .......... (b7) r0 = 1") +__msg("4: 0......... (b7) r2 = 2") +/* ... but it is at the exit of aux2(), which returns a register pair. */ +__msg("5: 0.2....... (95) exit") +__naked void subprog_ret_reg_pair(void) +{ + asm volatile ( + "call aux2;" + "r0 = r2;" + "exit;" + ::: __clobber_all); +} + #if defined(__TARGET_ARCH_x86) || defined(__TARGET_ARCH_arm64) SEC("socket") diff --git a/tools/testing/selftests/bpf/progs/exceptions_fail.c b/tools/testing/selftests/bpf/progs/exceptions_fail.c index ac44d60e50666..9708efb93683b 100644 --- a/tools/testing/selftests/bpf/progs/exceptions_fail.c +++ b/tools/testing/selftests/bpf/progs/exceptions_fail.c @@ -60,7 +60,7 @@ __noinline int exception_cb_ok_arg_small(int a) SEC("?tc") __exception_cb(exception_cb_bad_ret_type1) -__failure __msg("Global function exception_cb_bad_ret_type1() return value not void or scalar.") +__failure __msg("Only void, scalar, or a scalar-only struct/union up to 16 bytes is supported.") int reject_exception_cb_type_1(struct __sk_buff *ctx) { bpf_throw(0); diff --git a/tools/testing/selftests/bpf/progs/freplace_ret_pair.c b/tools/testing/selftests/bpf/progs/freplace_ret_pair.c new file mode 100644 index 0000000000000..12c15d293bd79 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/freplace_ret_pair.c @@ -0,0 +1,12 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ +#include +#include + +SEC("freplace/agg_ret_target_func") +__u64 new_agg_ret_target_func(void) +{ + return 0; +} + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/verifier_aggregate_ret.c b/tools/testing/selftests/bpf/progs/verifier_aggregate_ret.c new file mode 100644 index 0000000000000..7851bade2b406 --- /dev/null +++ b/tools/testing/selftests/bpf/progs/verifier_aggregate_ret.c @@ -0,0 +1,178 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2026 Meta Platforms, Inc. and affiliates. */ +#include +#include +#include "../test_kmods/bpf_testmod_kfunc.h" +#include "bpf_misc.h" + +#if defined(__clang_major__) && __clang_major__ >= 23 + +#define MIX_A 0xdeadbeefcafef00dULL +#define MIX_B 0x0123456789abcdefULL + +typedef unsigned __int128 u128; + +struct pair { + __u64 lo; /* R0 */ + __u64 hi; /* R2 */ +}; + +union upair { + __u64 halves[2]; + struct { + __u64 lo; /* R0 */ + __u64 hi; /* R2 */ + } parts; +}; + +static __noinline u128 make_i128(__u64 a, __u64 b) +{ + return ((u128)(a + b) << 64) | (a - b); +} + +SEC("tc") +__load_if_JITed() +__success __retval(0) +int aggregate_ret_int128_c_test(struct __sk_buff *skb) +{ + __u64 a = skb->len ^ MIX_A; + __u64 b = skb->len ^ MIX_B; + u128 v; + + v = make_i128(a, b); + if ((__u64)(v >> 64) != a + b) + return 1; + if ((__u64)v != a - b) + return 2; + + return 0; +} + +static __noinline struct pair make_pair(__u64 a, __u64 b) +{ + struct pair p = { .lo = a + b, .hi = a - b }; + + return p; +} + +SEC("tc") +__load_if_JITed() +__success __retval(0) +int aggregate_ret_struct_c_test(struct __sk_buff *skb) +{ + __u64 a = skb->len ^ MIX_A; + __u64 b = skb->len ^ MIX_B; + struct pair p; + + p = make_pair(a, b); + if (p.lo != a + b) + return 1; + if (p.hi != a - b) + return 2; + + return 0; +} + +__noinline struct pair make_pair_global(__u64 a, __u64 b) +{ + struct pair p = { .lo = a + b, .hi = a - b }; + + return p; +} + +SEC("tc") +__load_if_JITed() +__success __retval(0) +int aggregate_ret_global_struct_c_test(struct __sk_buff *skb) +{ + __u64 a = skb->len ^ MIX_A; + __u64 b = skb->len ^ MIX_B; + struct pair p; + + p = make_pair_global(a, b); + if (p.lo != a + b) + return 1; + if (p.hi != a - b) + return 2; + + return 0; +} + +static __noinline union upair make_upair(__u64 a, __u64 b) +{ + union upair p; + + p.halves[0] = a + b; + p.halves[1] = a - b; + return p; +} + +SEC("tc") +__load_if_JITed() +__success __retval(0) +int aggregate_ret_union_c_test(struct __sk_buff *skb) +{ + __u64 a = skb->len ^ MIX_A; + __u64 b = skb->len ^ MIX_B; + union upair p; + + p = make_upair(a, b); + if (p.parts.lo != a + b) + return 1; + if (p.parts.hi != a - b) + return 2; + + return 0; +} + +SEC("tc") +__arch_x86_64 __arch_arm64 +__load_if_JITed() +__success __retval(0) +int aggregate_ret_kfunc_int128_c_test(struct __sk_buff *skb) +{ + __u64 a = skb->len ^ MIX_A; + __u64 b = skb->len ^ MIX_B; + u128 v; + + v = bpf_kfunc_call_test_i128(a, b); + if ((__u64)(v >> 64) != a + b) + return 1; + if ((__u64)v != a - b) + return 2; + + return 0; +} + +SEC("tc") +__arch_x86_64 __arch_arm64 +__load_if_JITed() +__success __retval(0) +int aggregate_ret_kfunc_struct_c_test(struct __sk_buff *skb) +{ + __u64 a = skb->len ^ MIX_A; + __u64 b = skb->len ^ MIX_B; + struct prog_test_ret_pair p; + + p = bpf_kfunc_call_test_ret_pair(a, b); + if (p.lo != a + b) + return 1; + if (p.hi != a - b) + return 2; + + return 0; +} + +#else + +SEC("socket") +__description("verifier_aggregate_ret: needs LLVM 23, dummy test") +__success +int dummy_test(void) +{ + return 0; +} + +#endif + +char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c b/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c index 9366a3c578f13..20a9b9f20e96f 100644 --- a/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c +++ b/tools/testing/selftests/bpf/test_kmods/bpf_testmod.c @@ -954,6 +954,53 @@ __bpf_kfunc int bpf_kfunc_call_test5(u8 a, u16 b, u32 c) return 0; } +#if defined(__x86_64__) || defined(__aarch64__) +__bpf_kfunc __int128 bpf_kfunc_call_test_i128(u64 a, u64 b) +{ + return (__int128)(((unsigned __int128)(a + b) << 64) | (a - b)); +} + +__bpf_kfunc struct prog_test_ret_pair bpf_kfunc_call_test_ret_pair(u64 a, u64 b) +{ + struct prog_test_ret_pair r = { .lo = a + b, .hi = a - b }; + + return r; +} + +__bpf_kfunc struct prog_test_ret_pair bpf_kfunc_call_test_ret_fastcall(u64 a, u64 b) +{ + struct prog_test_ret_pair r = { .lo = a + b, .hi = a - b }; + + return r; +} + +__bpf_kfunc struct prog_test_ret_ptr bpf_kfunc_call_test_ret_ptr(u64 tag) +{ + struct prog_test_ret_ptr r = { .p = NULL, .tag = tag }; + + return r; +} + +__bpf_kfunc struct prog_test_ret_ii bpf_kfunc_call_test_ret_ii(int a, int b) +{ + struct prog_test_ret_ii r = { .a = a, .b = b }; + + return r; +} +#endif /* __x86_64__ || __aarch64__ */ + +/* + * Takes no argument on purpose: with no arguments there is nothing for the sret + * pointer to displace, so this needs no architecture guard even though it + * returns 24 bytes. See the comment on bpf_kfunc_call_test_i128() above. + */ +__bpf_kfunc struct prog_test_ret_big bpf_kfunc_call_test_ret_big(void) +{ + struct prog_test_ret_big r = { .a = 1, .b = 2, .c = 3 }; + + return r; +} + __bpf_kfunc u64 bpf_kfunc_call_stack_arg(u64 a, u64 b, u64 c, u64 d, u64 e, u64 f, u64 g, u64 h, u64 i, u64 j) @@ -1487,6 +1534,14 @@ BTF_ID_FLAGS(func, bpf_kfunc_call_test2) BTF_ID_FLAGS(func, bpf_kfunc_call_test3) BTF_ID_FLAGS(func, bpf_kfunc_call_test4) BTF_ID_FLAGS(func, bpf_kfunc_call_test5) +#if defined(__x86_64__) || defined(__aarch64__) +BTF_ID_FLAGS(func, bpf_kfunc_call_test_i128) +BTF_ID_FLAGS(func, bpf_kfunc_call_test_ret_pair) +BTF_ID_FLAGS(func, bpf_kfunc_call_test_ret_fastcall, KF_FASTCALL) +BTF_ID_FLAGS(func, bpf_kfunc_call_test_ret_ptr) +BTF_ID_FLAGS(func, bpf_kfunc_call_test_ret_ii) +#endif +BTF_ID_FLAGS(func, bpf_kfunc_call_test_ret_big) BTF_ID_FLAGS(func, bpf_kfunc_call_stack_arg) BTF_ID_FLAGS(func, bpf_kfunc_call_stack_arg_ptr) BTF_ID_FLAGS(func, bpf_kfunc_call_stack_arg_mix) diff --git a/tools/testing/selftests/bpf/test_kmods/bpf_testmod_kfunc.h b/tools/testing/selftests/bpf/test_kmods/bpf_testmod_kfunc.h index 7d81070eefe7f..65e693ada7364 100644 --- a/tools/testing/selftests/bpf/test_kmods/bpf_testmod_kfunc.h +++ b/tools/testing/selftests/bpf/test_kmods/bpf_testmod_kfunc.h @@ -55,6 +55,27 @@ struct prog_test_big_arg { __u64 b; }; +struct prog_test_ret_pair { /* 16 bytes: R0:R2 */ + __u64 lo; + __u64 hi; +}; + +struct prog_test_ret_ii { /* 8 bytes: R0 only */ + int a; + int b; +}; + +struct prog_test_ret_ptr { /* 16 bytes: contains a pointer */ + void *p; + __u64 tag; +}; + +struct prog_test_ret_big { /* 24 bytes: too large for R0:R2 */ + __u64 a; + __u64 b; + __u64 c; +}; + struct prog_test_fail1 { void *p; int x; @@ -131,6 +152,14 @@ int bpf_kfunc_call_test2(struct sock *sk, __u32 a, __u32 b) __ksym; struct sock *bpf_kfunc_call_test3(struct sock *sk) __ksym; long bpf_kfunc_call_test4(signed char a, short b, int c, long d) __ksym; int bpf_kfunc_call_test5(__u8 a, __u16 b, __u32 c) __ksym; +#ifdef __SIZEOF_INT128__ +__int128 bpf_kfunc_call_test_i128(__u64 a, __u64 b) __ksym; +#endif +struct prog_test_ret_pair bpf_kfunc_call_test_ret_pair(__u64 a, __u64 b) __ksym; +struct prog_test_ret_pair bpf_kfunc_call_test_ret_fastcall(__u64 a, __u64 b) __ksym; +struct prog_test_ret_ii bpf_kfunc_call_test_ret_ii(int a, int b) __ksym; +struct prog_test_ret_ptr bpf_kfunc_call_test_ret_ptr(__u64 tag) __ksym; +struct prog_test_ret_big bpf_kfunc_call_test_ret_big(void) __ksym; __u64 bpf_kfunc_call_stack_arg(__u64 a, __u64 b, __u64 c, __u64 d, __u64 e, __u64 f, __u64 g, __u64 h, __u64 i, __u64 j) __ksym;