diff --git a/IR/.vscode/settings.json b/IR/.vscode/settings.json new file mode 100644 index 00000000..6a57310e --- /dev/null +++ b/IR/.vscode/settings.json @@ -0,0 +1,38 @@ +{ + "cSpell.words": [ + "addrval", + "ALLOC", + "ALLOCARRAY", + "cgir", + "CSSA", + "DBGASSERT", + "dsts", + "elemptr", + "FUNCTIONARG", + "GETELEMPTR", + "insn", + "ipos", + "jbin", + "LOADELEM", + "loadimm", + "LOADRAW", + "loadrawextra", + "newbb", + "nonvr", + "notag", + "preds", + "printk", + "RAWOFF", + "repr", + "SIZET", + "STACKOFF", + "stackptr", + "STOREELEM", + "STORERAW", + "struct", + "succs", + "tdst", + "TSSA", + "vpos" + ] +} diff --git a/IR/CMakeLists.txt b/IR/CMakeLists.txt index 1e748dee..38d182cb 100644 --- a/IR/CMakeLists.txt +++ b/IR/CMakeLists.txt @@ -1,6 +1,6 @@ cmake_minimum_required(VERSION 3.20) project(bpf_ir) -set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Werror -Wstrict-prototypes -Wunused-variable -O3") +set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Werror -Wstrict-prototypes -Wenum-compare -Wunused-variable -Wunused-but-set-variable -Wsign-compare -O3") add_library(bpf_ir STATIC bpf_ir.c @@ -15,6 +15,7 @@ add_library(bpf_ir STATIC passes/cut_bb_pass.c aux/prog_check.c aux/disasm.c + aux/optimization.c ir_code_gen.c lii.c ) diff --git a/IR/Makefile b/IR/Makefile index 38e26533..65c717be 100644 --- a/IR/Makefile +++ b/IR/Makefile @@ -1,9 +1,15 @@ +build: format + cmake --build build + +buildall: genctor + cmake --build build format: ./scripts/format.sh -build: format - cmake --build build +genctor: + ./scripts/gen_insn_ctor.py + ./scripts/format.sh kernel: build ./scripts/gen_kernel.sh diff --git a/IR/Readme.md b/IR/Readme.md index 1e8c1120..9e8d4b67 100644 --- a/IR/Readme.md +++ b/IR/Readme.md @@ -19,13 +19,12 @@ One opinion, one benefit of designing the raw constraint from is that our runtim - [x] Env - [x] If ALU ops (including cond jmp) use 64 bits const, load it to register - [x] Switch back for ALU spill -- [ ] CGIR-I and CGIR-II formalization - [ ] Test adding counter & print some result +- [x] CGIR-I and CGIR-II formalization ## Bugs -- `loop2`: Loop BB detected -- `STACK_PTR` Change to R10 insn +- `erase_insn_cg` Not working in the normalization. Should spilling use `safe` list iterations? # TODO diff --git a/IR/array.c b/IR/array.c index 63419156..46ac1103 100644 --- a/IR/array.c +++ b/IR/array.c @@ -87,6 +87,18 @@ void bpf_ir_array_clone(struct bpf_ir_env *env, struct array *res, memcpy(res->data, arr->data, arr->num_elem * arr->elem_size); } +// Merge b into a +void bpf_ir_array_merge(struct bpf_ir_env *env, struct array *a, + struct array *b) +{ + struct ir_insn **pos; + array_for(pos, (*b)) + { + struct ir_insn *insn = *pos; + bpf_ir_array_push_unique(env, a, &insn); + CHECK_ERR(); + } +} void bpf_ir_array_free(struct array *arr) { if (arr->data) { diff --git a/IR/aux/disasm.c b/IR/aux/disasm.c index 403eafea..0b072ef4 100644 --- a/IR/aux/disasm.c +++ b/IR/aux/disasm.c @@ -1,4 +1,4 @@ -#include "linux/bpf_ir.h" +#include #include // #include "disasm.h" diff --git a/IR/aux/optimization.c b/IR/aux/optimization.c new file mode 100644 index 00000000..31af180b --- /dev/null +++ b/IR/aux/optimization.c @@ -0,0 +1,82 @@ +#include + +static void remove_no_user_insn(struct bpf_ir_env *env, struct ir_function *fun) +{ + // Remove all instructions that have no users, except for void instructions & calls + + struct ir_basic_block **pos; + array_for(pos, fun->reachable_bbs) + { + struct ir_basic_block *bb = *pos; + struct ir_insn *insn, *tmp; + list_for_each_entry_safe(insn, tmp, &bb->ir_insn_head, + list_ptr) { + if (bpf_ir_is_void(insn) || insn->op == IR_INSN_CALL) { + continue; + } + if (insn->users.num_elem == 0) { + bpf_ir_erase_insn(env, insn); + CHECK_ERR(); + } + } + } +} + +static void remove_unused_alloc(struct bpf_ir_env *env, struct ir_function *fun) +{ + // Remove all alloc instructions that have no users + struct array alloc_insns; + INIT_ARRAY(&alloc_insns, struct ir_insn *); + + struct ir_basic_block **pos; + array_for(pos, fun->reachable_bbs) + { + struct ir_basic_block *bb = *pos; + struct ir_insn *insn, *tmp; + list_for_each_entry_safe(insn, tmp, &bb->ir_insn_head, + list_ptr) { + if (insn->op == IR_INSN_ALLOC) { + bpf_ir_array_push(env, &alloc_insns, &insn); + } + } + } + + struct ir_insn **pos2; + array_for(pos2, alloc_insns) + { + bool has_load = false; + struct ir_insn *insn = *pos2; + struct ir_insn **pos3; + array_for(pos3, insn->users) + { + struct ir_insn *user = *pos3; + if (user->op == IR_INSN_LOAD) { + has_load = true; + break; + } + } + if (!has_load) { + // Remove all its store + array_for(pos3, insn->users) + { + struct ir_insn *user = *pos3; + bpf_ir_erase_insn(env, user); + CHECK_ERR(); + } + // Remove itself + bpf_ir_erase_insn(env, insn); + CHECK_ERR(); + } + } + + bpf_ir_array_free(&alloc_insns); +} + +void bpf_ir_optimize_ir(struct bpf_ir_env *env, struct ir_function *fun) +{ + remove_no_user_insn(env, fun); + CHECK_ERR(); + + remove_unused_alloc(env, fun); + CHECK_ERR(); +} diff --git a/IR/aux/prog_check.c b/IR/aux/prog_check.c index 022f6124..cac93423 100644 --- a/IR/aux/prog_check.c +++ b/IR/aux/prog_check.c @@ -44,15 +44,19 @@ static void check_insn(struct bpf_ir_env *env, struct ir_function *fun) struct ir_basic_block *bb = *pos; struct ir_insn *insn; list_for_each_entry(insn, &bb->ir_insn_head, list_ptr) { - struct array operands = bpf_ir_get_operands(env, insn); - struct ir_value **vpos; + if (insn->parent_bb != bb) { + print_ir_insn_err( + env, insn, + "Instruction's parent BB wrong"); + RAISE_ERROR("Parent BB error"); + } if (insn->op == IR_INSN_LOADRAW || insn->op == IR_INSN_ALLOC || - insn->op == IR_INSN_JA || insn->op == IR_INSN_PHI) { + insn->op == IR_INSN_JA || insn->op == IR_INSN_PHI || + insn->op == IR_INSN_ALLOCARRAY) { if (!(insn->value_num == 0)) { print_ir_insn_err(env, insn, NULL); RAISE_ERROR( - "Instruction should have no value"); } } @@ -62,13 +66,14 @@ static void check_insn(struct bpf_ir_env *env, struct ir_function *fun) if (!(insn->value_num == 1)) { print_ir_insn_err(env, insn, NULL); RAISE_ERROR( - "Instruction should have 1 values"); } } - if (insn->op == IR_INSN_STORE || (is_alu(insn)) || - (is_cond_jmp(insn))) { + if (insn->op == IR_INSN_STORE || + (bpf_ir_is_alu(insn)) || + (bpf_ir_is_cond_jmp(insn)) || + insn->op == IR_INSN_GETELEMPTR) { if (!(insn->value_num == 2)) { print_ir_insn_err(env, insn, NULL); RAISE_ERROR( @@ -89,9 +94,19 @@ static void check_insn(struct bpf_ir_env *env, struct ir_function *fun) } } + if (insn->op == IR_INSN_GETELEMPTR) { + if (!(insn->values[1].type == IR_VALUE_INSN && + insn->values[1].data.insn_d->op == + IR_INSN_ALLOCARRAY)) { + print_ir_insn_err(env, insn, NULL); + RAISE_ERROR( + "Value should be an allocarray instruction"); + } + } + // TODO: Check: users of alloc instructions must be STORE/LOAD - if (is_alu(insn) || is_cond_jmp(insn)) { + if (bpf_ir_is_alu(insn) || bpf_ir_is_cond_jmp(insn)) { // Binary ALU if (!bpf_ir_valid_alu_type(insn->alu_op)) { print_ir_insn_err(env, insn, NULL); @@ -99,6 +114,8 @@ static void check_insn(struct bpf_ir_env *env, struct ir_function *fun) } } + struct array operands = bpf_ir_get_operands(env, insn); + struct ir_value **vpos; if (insn->op == IR_INSN_ALLOC || insn->op == IR_INSN_LOADRAW || insn->op == IR_INSN_STORERAW) { @@ -107,6 +124,8 @@ static void check_insn(struct bpf_ir_env *env, struct ir_function *fun) RAISE_ERROR("Invalid VR type"); } } + + // Checking operands array_for(vpos, operands) { struct ir_value *val = *vpos; @@ -115,6 +134,12 @@ static void check_insn(struct bpf_ir_env *env, struct ir_function *fun) val->const_type)) { print_ir_insn_err(env, insn, NULL); + + PRINT_LOG( + env, + "Constant type: %d, operand number: %d\n", + val->const_type, + operands.num_elem); RAISE_ERROR( "Invalid Constant type"); } @@ -168,6 +193,7 @@ static void check_users(struct bpf_ir_env *env, struct ir_function *fun) struct ir_insn *insn = fun->function_arg[i]; check_insn_users_use_insn(env, insn); } + check_insn_users_use_insn(env, fun->sp); struct ir_basic_block **pos; array_for(pos, fun->reachable_bbs) { @@ -237,7 +263,7 @@ static void check_jumping(struct bpf_ir_env *env, struct ir_function *fun) struct ir_insn *insn; int jmp_exists = 0; list_for_each_entry(insn, &bb->ir_insn_head, list_ptr) { - if (is_jmp(insn)) { + if (bpf_ir_is_jmp(insn)) { jmp_exists = 1; if (!bpf_ir_is_last_insn(insn)) { // Error @@ -259,7 +285,7 @@ static void check_jumping(struct bpf_ir_env *env, struct ir_function *fun) continue; } // For conditional jumps, both BB1 and BB2 should be successors - if (is_cond_jmp(insn)) { + if (bpf_ir_is_cond_jmp(insn)) { // Get the two basic blocks that the conditional jump statement jumps to struct ir_basic_block *bb1 = insn->bb1; @@ -359,7 +385,7 @@ static void bpf_ir_fix_bb_succ(struct ir_function *fun) { struct ir_basic_block *bb = *pos; struct ir_insn *insn = bpf_ir_get_last_insn(bb); - if (insn && is_cond_jmp(insn)) { + if (insn && bpf_ir_is_cond_jmp(insn)) { // Conditional jmp if (bb->succs.num_elem != 2) { CRITICAL( @@ -385,11 +411,11 @@ static void add_reach(struct bpf_ir_env *env, struct ir_function *fun, bpf_ir_array_push(env, &fun->reachable_bbs, &bb); struct ir_basic_block **succ; - u8 i = 0; + bool first = false; array_for(succ, bb->succs) { - if (i == 0) { - i = 1; + if (!first && bb->succs.num_elem > 1) { + first = true; // Check if visited if ((*succ)->_visited) { RAISE_ERROR("Loop BB detected"); diff --git a/IR/bpf_ir.c b/IR/bpf_ir.c index 38809fe8..9b9a16f1 100644 --- a/IR/bpf_ir.c +++ b/IR/bpf_ir.c @@ -11,7 +11,7 @@ static const u32 helper_func_arg_num[100] = { // All function passes. static const struct function_pass passes[] = { - DEF_FUNC_PASS(remove_trivial_phi, "Remove the trival Phi"), + DEF_FUNC_PASS(remove_trivial_phi, "Removing the trival Phi"), DEF_FUNC_PASS(add_counter, "Adding counter"), }; @@ -27,8 +27,8 @@ static struct ir_insn *add_phi_operands(struct bpf_ir_env *env, struct ssa_transform_env *tenv, u8 reg, struct ir_insn *insn); -static void add_user(struct bpf_ir_env *env, struct ssa_transform_env *tenv, - struct ir_insn *user, struct ir_value val); +static void add_user(struct bpf_ir_env *env, struct ir_insn *user, + struct ir_value val); static int compare_num(const void *a, const void *b) { @@ -231,8 +231,8 @@ static void gen_bb(struct bpf_ir_env *env, struct bb_info *ret, // Allocate instructions for (size_t i = 0; i < bb_entrance.num_elem; ++i) { struct pre_ir_basic_block *real_bb = all_bbs[i].bb; - PRINT_LOG(env, "BB Alloc, %zu %zu\n", real_bb->end_pos, - real_bb->start_pos); + PRINT_LOG(env, "BB Alloc: [%zu, %zu)\n", real_bb->start_pos, + real_bb->end_pos); SAFE_MALLOC(real_bb->pre_insns, sizeof(struct pre_ir_insn) * (real_bb->end_pos - real_bb->start_pos)); @@ -317,23 +317,32 @@ static void print_pre_ir_cfg(struct bpf_ir_env *env, } } -static void init_env(struct bpf_ir_env *env, struct ssa_transform_env *tenv, - struct bb_info info) +static void init_tenv(struct bpf_ir_env *env, struct ssa_transform_env *tenv, + struct bb_info info) { for (size_t i = 0; i < MAX_BPF_REG; ++i) { INIT_ARRAY(&tenv->currentDef[i], struct bb_val); } tenv->info = info; - INIT_ARRAY(&tenv->sp_users, struct ir_insn *); + // Initialize SP + SAFE_MALLOC(tenv->sp, sizeof(struct ir_insn)); + INIT_ARRAY(&tenv->sp->users, struct ir_insn *); + tenv->sp->op = IR_INSN_REG; + tenv->sp->value_num = 0; + tenv->sp->user_data = NULL; + tenv->sp->parent_bb = NULL; + tenv->sp->reg_id = BPF_REG_10; + write_variable(env, tenv, BPF_REG_10, NULL, + bpf_ir_value_insn(tenv->sp)); // Initialize function argument for (u8 i = 0; i < MAX_FUNC_ARG; ++i) { SAFE_MALLOC(tenv->function_arg[i], sizeof(struct ir_insn)); INIT_ARRAY(&tenv->function_arg[i]->users, struct ir_insn *); tenv->function_arg[i]->op = IR_INSN_FUNCTIONARG; - tenv->function_arg[i]->fid = i; - tenv->function_arg[i]->value_num = 0; + tenv->function_arg[i]->fun_arg_id = i; tenv->function_arg[i]->user_data = NULL; + tenv->function_arg[i]->value_num = 0; struct ir_value val; val.type = IR_VALUE_INSN; val.data.insn_d = tenv->function_arg[i]; @@ -357,10 +366,10 @@ static void write_variable(struct bpf_ir_env *env, struct ssa_transform_env *tenv, u8 reg, struct pre_ir_basic_block *bb, struct ir_value val) { - if (reg >= MAX_BPF_REG - 1) { - // Stack pointer is read-only - CRITICAL("Error"); - } + // if (reg >= MAX_BPF_REG - 1) { + // // Stack pointer is read-only + // CRITICAL("Error"); + // } // Write a variable to a BB struct array *currentDef = &tenv->currentDef[reg]; // Traverse the array to find if there exists a value in the same BB @@ -396,7 +405,7 @@ static struct ir_insn *add_phi_operands(struct bpf_ir_env *env, phi.value = read_variable( env, tenv, reg, (struct pre_ir_basic_block *)pred->user_data); - add_user(env, tenv, insn, phi.value); + add_user(env, insn, phi.value); bpf_ir_array_push(env, &pred->users, &insn); bpf_ir_array_push(env, &insn->phi, &phi); } @@ -487,9 +496,7 @@ static struct ir_value read_variable(struct bpf_ir_env *env, // Read a variable from a BB if (reg == BPF_REG_10) { // Stack pointer - struct ir_value val; - val.type = IR_VALUE_STACK_PTR; - return val; + return bpf_ir_value_insn(tenv->sp); } struct array *currentDef = &tenv->currentDef[reg]; for (size_t i = 0; i < currentDef->num_elem; ++i) { @@ -530,15 +537,12 @@ static enum ir_vr_type to_ir_ld_u(u8 size) } // User uses val -static void add_user(struct bpf_ir_env *env, struct ssa_transform_env *tenv, - struct ir_insn *user, struct ir_value val) +static void add_user(struct bpf_ir_env *env, struct ir_insn *user, + struct ir_value val) { if (val.type == IR_VALUE_INSN) { bpf_ir_array_push_unique(env, &val.data.insn_d->users, &user); } - if (val.type == IR_VALUE_STACK_PTR) { - bpf_ir_array_push_unique(env, &tenv->sp_users, &user); - } } /** @@ -610,7 +614,7 @@ static struct ir_value get_src_value(struct bpf_ir_env *env, static struct ir_insn * create_alu_bin(struct bpf_ir_env *env, struct ir_basic_block *bb, struct ir_value val1, struct ir_value val2, enum ir_insn_type ty, - struct ssa_transform_env *tenv, enum ir_alu_op_type alu_ty) + enum ir_alu_op_type alu_ty) { struct ir_insn *new_insn = create_insn_back(bb); new_insn->op = ty; @@ -618,8 +622,8 @@ create_alu_bin(struct bpf_ir_env *env, struct ir_basic_block *bb, new_insn->values[1] = val2; new_insn->value_num = 2; new_insn->alu_op = alu_ty; - add_user(env, tenv, new_insn, new_insn->values[0]); - add_user(env, tenv, new_insn, new_insn->values[1]); + add_user(env, new_insn, new_insn->values[0]); + add_user(env, new_insn, new_insn->values[1]); return new_insn; } @@ -629,7 +633,7 @@ static void alu_write(struct bpf_ir_env *env, struct ssa_transform_env *tenv, { struct ir_insn *new_insn = create_alu_bin( env, bb->ir_bb, read_variable(env, tenv, insn.dst_reg, bb), - get_src_value(env, tenv, bb, insn), ty, tenv, alu_ty); + get_src_value(env, tenv, bb, insn), ty, alu_ty); struct ir_value new_val; new_val.type = IR_VALUE_INSN; new_val.data.insn_d = new_insn; @@ -648,8 +652,8 @@ static void create_cond_jmp(struct bpf_ir_env *env, new_insn->values[1] = get_src_value(env, tenv, bb, insn); new_insn->value_num = 2; new_insn->alu_op = alu_ty; - add_user(env, tenv, new_insn, new_insn->values[0]); - add_user(env, tenv, new_insn, new_insn->values[1]); + add_user(env, new_insn, new_insn->values[0]); + add_user(env, new_insn, new_insn->values[1]); size_t pos = insn.pos + insn.off + 1; new_insn->bb1 = get_ir_bb_from_position(tenv, insn.pos + 1); new_insn->bb2 = get_ir_bb_from_position(tenv, pos); @@ -764,7 +768,7 @@ static void transform_bb(struct bpf_ir_env *env, struct ssa_transform_env *tenv, struct ir_address_value addr_val; addr_val.value = read_variable(env, tenv, insn.src_reg, bb); - add_user(env, tenv, new_insn, addr_val.value); + add_user(env, new_insn, addr_val.value); addr_val.offset = insn.off; new_insn->vr_type = to_ir_ld_u(BPF_SIZE(code)); new_insn->addr_val = addr_val; @@ -782,7 +786,7 @@ static void transform_bb(struct bpf_ir_env *env, struct ssa_transform_env *tenv, struct ir_address_value addr_val; addr_val.value = read_variable(env, tenv, insn.src_reg, bb); - add_user(env, tenv, new_insn, addr_val.value); + add_user(env, new_insn, addr_val.value); addr_val.offset = insn.off; new_insn->vr_type = to_ir_ld_u(BPF_SIZE(code)); new_insn->addr_val = addr_val; @@ -797,7 +801,7 @@ static void transform_bb(struct bpf_ir_env *env, struct ssa_transform_env *tenv, struct ir_address_value addr_val; addr_val.value = read_variable(env, tenv, insn.dst_reg, bb); - add_user(env, tenv, new_insn, addr_val.value); + add_user(env, new_insn, addr_val.value); addr_val.offset = insn.off; new_insn->vr_type = to_ir_ld_u(BPF_SIZE(code)); new_insn->addr_val = addr_val; @@ -812,14 +816,14 @@ static void transform_bb(struct bpf_ir_env *env, struct ssa_transform_env *tenv, struct ir_address_value addr_val; addr_val.value = read_variable(env, tenv, insn.dst_reg, bb); - add_user(env, tenv, new_insn, addr_val.value); + add_user(env, new_insn, addr_val.value); addr_val.offset = insn.off; new_insn->vr_type = to_ir_ld_u(BPF_SIZE(code)); new_insn->addr_val = addr_val; new_insn->values[0] = read_variable(env, tenv, insn.src_reg, bb); new_insn->value_num = 1; - add_user(env, tenv, new_insn, new_insn->values[0]); + add_user(env, new_insn, new_insn->values[0]); } else if (BPF_CLASS(code) == BPF_JMP || BPF_CLASS(code) == BPF_JMP32) { enum ir_alu_op_type alu_ty = IR_ALU_UNKNOWN; @@ -917,7 +921,7 @@ static void transform_bb(struct bpf_ir_env *env, struct ssa_transform_env *tenv, env, tenv, BPF_REG_1 + j, bb); - add_user(env, tenv, new_insn, + add_user(env, new_insn, new_insn->values[j]); } } @@ -950,7 +954,6 @@ static void transform_bb(struct bpf_ir_env *env, struct ssa_transform_env *tenv, static void free_function(struct ir_function *fun) { - bpf_ir_array_free(&fun->sp_users); for (size_t i = 0; i < fun->all_bbs.num_elem; ++i) { struct ir_basic_block *bb = ((struct ir_basic_block **)(fun->all_bbs.data))[i]; @@ -974,18 +977,27 @@ static void free_function(struct ir_function *fun) bpf_ir_array_free(&fun->function_arg[i]->users); free_proto(fun->function_arg[i]); } + if (fun->sp) { + bpf_ir_array_free(&fun->sp->users); + free_proto(fun->sp); + } + for (u8 i = 0; i < BPF_REG_10; ++i) { + struct ir_insn *insn = fun->cg_info.regs[i]; + bpf_ir_array_free(&insn->users); + free_proto(insn); + } bpf_ir_array_free(&fun->all_bbs); bpf_ir_array_free(&fun->reachable_bbs); bpf_ir_array_free(&fun->end_bbs); bpf_ir_array_free(&fun->cg_info.all_var); } -static void gen_function(struct bpf_ir_env *env, struct ir_function *fun, - struct ssa_transform_env *tenv) +static void init_function(struct bpf_ir_env *env, struct ir_function *fun, + struct ssa_transform_env *tenv) { fun->arg_num = 1; fun->entry = tenv->info.entry->ir_bb; - fun->sp_users = tenv->sp_users; + fun->sp = tenv->sp; for (u8 i = 0; i < MAX_FUNC_ARG; ++i) { fun->function_arg[i] = tenv->function_arg[i]; } @@ -1008,6 +1020,17 @@ static void gen_function(struct bpf_ir_env *env, struct ir_function *fun, bpf_ir_array_push(env, &fun->all_bbs, &bb->ir_bb); free_proto(bb); } + for (u8 i = 0; i < BPF_REG_10; ++i) { + struct ir_insn *insn; + SAFE_MALLOC(fun->cg_info.regs[i], sizeof(struct ir_insn)); + // Those should be read-only + insn = fun->cg_info.regs[i]; + insn->op = IR_INSN_REG; + insn->parent_bb = NULL; + INIT_ARRAY(&insn->users, struct ir_insn *); + insn->value_num = 0; + insn->reg_id = i; + } } static void run_passes(struct bpf_ir_env *env, struct ir_function *fun) @@ -1031,27 +1054,40 @@ static void run_passes(struct bpf_ir_env *env, struct ir_function *fun) } } -// static void print_bpf_insn_simple(struct bpf_ir_env *env, struct bpf_insn insn) -// { -// if (insn.off < 0) { -// PRINT_LOG(env, "%4x %x %x %8x -%8x\n", insn.code, -// insn.src_reg, insn.dst_reg, insn.imm, -insn.off); -// } else { -// PRINT_LOG(env, "%4x %x %x %8x %8x\n", insn.code, -// insn.src_reg, insn.dst_reg, insn.imm, insn.off); -// } -// } +static void print_bpf_insn_simple(struct bpf_ir_env *env, + const struct bpf_insn *insn) +{ + if (insn->off < 0) { + PRINT_LOG(env, "%4x %x %x %8x -%8x\n", insn->code, + insn->src_reg, insn->dst_reg, insn->imm, -insn->off); + } else { + PRINT_LOG(env, "%4x %x %x %8x %8x\n", insn->code, + insn->src_reg, insn->dst_reg, insn->imm, insn->off); + } +} static void print_bpf_prog(struct bpf_ir_env *env, const struct bpf_insn *insns, size_t len) { + if (env->opts.print_mode == BPF_IR_PRINT_DETAIL) { + PRINT_LOG(env, " op src dst imm off\n"); + } else if (env->opts.print_mode == BPF_IR_PRINT_BOTH) { + PRINT_LOG(env, " op src dst imm off\n"); + } for (size_t i = 0; i < len; ++i) { const struct bpf_insn *insn = &insns[i]; if (insn->code == 0) { continue; } PRINT_LOG(env, "[%zu] ", i); - bpf_ir_print_bpf_insn(env, insn); + if (env->opts.print_mode == BPF_IR_PRINT_BPF || + env->opts.print_mode == BPF_IR_PRINT_BOTH) { + bpf_ir_print_bpf_insn(env, insn); + } + if (env->opts.print_mode == BPF_IR_PRINT_DETAIL || + env->opts.print_mode == BPF_IR_PRINT_BOTH) { + print_bpf_insn_simple(env, insn); + } } } @@ -1066,7 +1102,7 @@ void bpf_ir_run(struct bpf_ir_env *env, const struct bpf_insn *insns, print_pre_ir_cfg(env, info.entry); struct ssa_transform_env trans_env; - init_env(env, &trans_env, info); + init_tenv(env, &trans_env, info); CHECK_ERR(); init_ir_bbs(env, &trans_env); @@ -1076,7 +1112,7 @@ void bpf_ir_run(struct bpf_ir_env *env, const struct bpf_insn *insns, CHECK_ERR(); struct ir_function fun; - gen_function(env, &fun, &trans_env); + init_function(env, &fun, &trans_env); // Drop env @@ -1108,13 +1144,14 @@ void bpf_ir_run(struct bpf_ir_env *env, const struct bpf_insn *insns, free_function(&fun); } -struct bpf_ir_env *bpf_ir_init_env(void) +struct bpf_ir_env *bpf_ir_init_env(struct ir_opts opts) { struct bpf_ir_env *bpf_ir_env = malloc_proto(sizeof(struct bpf_ir_env)); bpf_ir_env->insn_cnt = 0; bpf_ir_env->insns = NULL; bpf_ir_env->log_pos = 0; bpf_ir_env->err = 0; + bpf_ir_env->opts = opts; return bpf_ir_env; } diff --git a/IR/docs/IR.md b/IR/docs/IR.md index 38201c01..ae95dab6 100644 --- a/IR/docs/IR.md +++ b/IR/docs/IR.md @@ -78,6 +78,22 @@ store %1 200 (Currently other vr type is not working. All alloc uses 64 bits reg/stack) +### `allocarray` + +Syntax: `allocarray ` + +Allocate a continuous array on stack with size `sizeof(vr_type)*array_num`. + +Users could load or store values to it by using `storeraw` and `loadraw`. + +### `getelemptr` + +Syntax: `getelemptr ` + +Get a pointer to the element in an array. + +`values[1]` should point to a `allocarray` instruction. + ### `store` Syntax: `store ` diff --git a/IR/include/linux/bpf_ir.h b/IR/include/linux/bpf_ir.h index df0bd6ea..fe6bfc7e 100644 --- a/IR/include/linux/bpf_ir.h +++ b/IR/include/linux/bpf_ir.h @@ -44,6 +44,15 @@ typedef __u64 u64; #define BPF_IR_LOG_SIZE 100000 +struct ir_opts { + u8 debug; + enum { + BPF_IR_PRINT_BPF, + BPF_IR_PRINT_DETAIL, + BPF_IR_PRINT_BOTH, + } print_mode; +}; + struct bpf_ir_env { int err; @@ -55,6 +64,8 @@ struct bpf_ir_env { char log[BPF_IR_LOG_SIZE]; size_t log_pos; + + struct ir_opts opts; }; void bpf_ir_print_to_log(struct bpf_ir_env *env, char *fmt, ...); @@ -102,6 +113,9 @@ void bpf_ir_array_free(struct array *); struct array bpf_ir_array_null(void); +void bpf_ir_array_merge(struct bpf_ir_env *env, struct array *a, + struct array *b); + void bpf_ir_array_erase(struct array *arr, size_t idx); void *bpf_ir_array_get_void(struct array *arr, size_t idx); @@ -143,17 +157,30 @@ void bpf_ir_array_clone(struct bpf_ir_env *env, struct array *res, #define RAISE_ERROR(str) \ { \ - PRINT_LOG(env, "\n--> %s:%d <%s> %s <--\n", __FILE__, \ - __LINE__, __FUNCTION__, str); \ + PRINT_LOG(env, "\e[1;31mError: %s:%d <%s> %s\e[0m\n", \ + __FILE__, __LINE__, __FUNCTION__, str); \ env->err = -ENOSYS; \ return; \ } +#define RAISE_ERROR_RET(str, ret) \ + { \ + PRINT_LOG(env, "\e[1;31mError: %s:%d <%s> %s\e[0m\n", \ + __FILE__, __LINE__, __FUNCTION__, str); \ + env->err = -ENOSYS; \ + return ret; \ + } + #define DBGASSERT(cond) \ if (!(cond)) { \ CRITICAL("Assertion failed"); \ } +#define ASSERT_DUMP(cond, ret) \ + if (!(cond)) { \ + RAISE_ERROR_RET("Assertion failed", ret); \ + } + #define CRITICAL_DUMP(env, str) \ { \ bpf_ir_print_log_dbg(env); \ @@ -214,10 +241,9 @@ enum ir_alu_op_type { enum ir_value_type { IR_VALUE_CONSTANT, - IR_VALUE_CONSTANT_RAWOFF, // A constant value in raw operations to be added during code - // generation + // A constant value in raw operations to be added during code generation + IR_VALUE_CONSTANT_RAWOFF, IR_VALUE_INSN, - IR_VALUE_STACK_PTR, IR_VALUE_UNDEF, }; @@ -279,6 +305,8 @@ int bpf_ir_valid_vr_type(enum ir_vr_type type); enum ir_insn_type { IR_INSN_ALLOC, + IR_INSN_ALLOCARRAY, + IR_INSN_GETELEMPTR, IR_INSN_STORE, IR_INSN_LOAD, IR_INSN_LOADIMM_EXTRA, @@ -312,11 +340,13 @@ enum ir_insn_type { /** INSN = - ALLOC - | STORE , + ALLOC + | STORE | LOAD - | STORERAW , - | LOADRAW + | ALLOCARRAY + | GETELEMPTR + | STORERAW + | LOADRAW | ADD , | SUB , @@ -362,7 +392,12 @@ struct ir_insn { // Array of phi_value struct array phi; - s32 fid; // Function ID + union { + s32 fid; // Function ID + s32 fun_arg_id; // Function argument ID + s32 reg_id; // Register ID + u32 array_num; // Array number + }; enum ir_loadimm_extra_type imm_extra_type; // For 64 imm load s64 imm64; // H (next_imm:32)(imm:32) L @@ -471,8 +506,9 @@ struct ssa_transform_env { struct array currentDef[MAX_BPF_REG]; struct bb_info info; - // Stack pointer (r10) users - struct array sp_users; + // Stack Pointer + struct ir_insn *sp; + // Function argument struct ir_insn *function_arg[MAX_FUNC_ARG]; }; @@ -489,7 +525,7 @@ void bpf_ir_print_bpf_insn(struct bpf_ir_env *env, const struct bpf_insn *insn); void bpf_ir_free_env(struct bpf_ir_env *env); -struct bpf_ir_env *bpf_ir_init_env(void); +struct bpf_ir_env *bpf_ir_init_env(struct ir_opts); /* Fun Start */ @@ -499,11 +535,12 @@ struct code_gen_info { struct array all_var; // BPF Register Virtual Instruction (used as dst) - struct ir_insn *regs[MAX_BPF_REG]; + struct ir_insn *regs[BPF_REG_10]; // Only use R0-R9 size_t callee_num; - s16 stack_offset; + // The stack offset + s32 stack_offset; // Whether to spill callee saved registers u8 spill_callee; @@ -524,8 +561,8 @@ struct ir_function { // BBs who has no successors struct array end_bbs; - // Stack pointer (r10) users. Should be readonly. No more manual stack access should be allowed. - struct array sp_users; + // Stack pointer + struct ir_insn *sp; // Function argument struct ir_insn *function_arg[MAX_FUNC_ARG]; @@ -613,6 +650,10 @@ void print_ir_value(struct bpf_ir_env *env, struct ir_value v); void print_raw_ir_insn(struct bpf_ir_env *env, struct ir_insn *insn); +void print_raw_ir_insn_full(struct bpf_ir_env *env, struct ir_insn *insn, + void (*print_ir)(struct bpf_ir_env *env, + struct ir_insn *)); + void print_raw_ir_bb(struct bpf_ir_env *env, struct ir_basic_block *bb); void print_insn_ptr_base(struct bpf_ir_env *env, struct ir_insn *insn); @@ -643,142 +684,266 @@ enum insert_position { // Return an array of struct ir_value* struct array bpf_ir_get_operands(struct bpf_ir_env *env, struct ir_insn *insn); +// Return an array of struct ir_value* +struct array bpf_ir_get_operands_and_dst(struct bpf_ir_env *env, + struct ir_insn *insn); + void bpf_ir_replace_all_usage(struct bpf_ir_env *env, struct ir_insn *insn, struct ir_value rep); +void bpf_ir_replace_all_usage_cg(struct bpf_ir_env *env, struct ir_insn *insn, + struct ir_value rep); + void bpf_ir_replace_all_usage_except(struct bpf_ir_env *env, struct ir_insn *insn, struct ir_value rep, struct ir_insn *except); void bpf_ir_erase_insn(struct bpf_ir_env *env, struct ir_insn *insn); +void bpf_ir_erase_insn_cg(struct bpf_ir_env *env, struct ir_insn *insn); + int bpf_ir_is_last_insn(struct ir_insn *insn); -int is_void(struct ir_insn *insn); +int bpf_ir_is_void(struct ir_insn *insn); -int is_jmp(struct ir_insn *insn); +int bpf_ir_is_jmp(struct ir_insn *insn); -int is_cond_jmp(struct ir_insn *insn); +int bpf_ir_is_cond_jmp(struct ir_insn *insn); -int is_alu(struct ir_insn *insn); +int bpf_ir_is_alu(struct ir_insn *insn); -struct ir_insn *prev_insn(struct ir_insn *insn); +struct ir_insn *bpf_ir_prev_insn(struct ir_insn *insn); -struct ir_insn *next_insn(struct ir_insn *insn); +struct ir_insn *bpf_ir_next_insn(struct ir_insn *insn); -struct ir_insn *create_alloc_insn(struct ir_insn *insn, enum ir_vr_type type, - enum insert_position pos); +/* Instruction Constructors */ -struct ir_insn *create_alloc_insn_bb(struct ir_basic_block *bb, - enum ir_vr_type type, - enum insert_position pos); +struct ir_insn *bpf_ir_create_alloc_insn(struct bpf_ir_env *env, + struct ir_insn *pos_insn, + enum ir_vr_type type, + enum insert_position pos); -struct ir_insn *create_store_insn(struct bpf_ir_env *env, struct ir_insn *insn, - struct ir_insn *st_insn, struct ir_value val, - enum insert_position pos); +struct ir_insn *bpf_ir_create_alloc_insn_bb(struct bpf_ir_env *env, + struct ir_basic_block *pos_bb, + enum ir_vr_type type, + enum insert_position pos); + +struct ir_insn *bpf_ir_create_allocarray_insn(struct bpf_ir_env *env, + struct ir_insn *pos_insn, + enum ir_vr_type type, u32 num, + enum insert_position pos); + +struct ir_insn *bpf_ir_create_allocarray_insn_bb(struct bpf_ir_env *env, + struct ir_basic_block *pos_bb, + enum ir_vr_type type, u32 num, + enum insert_position pos); + +struct ir_insn *bpf_ir_create_getelemptr_insn(struct bpf_ir_env *env, + struct ir_insn *pos_insn, + struct ir_insn *alloca_insn, + struct ir_value offset, + enum insert_position pos); + +struct ir_insn *bpf_ir_create_getelemptr_insn_bb(struct bpf_ir_env *env, + struct ir_basic_block *pos_bb, + struct ir_insn *alloca_insn, + struct ir_value offset, + enum insert_position pos); + +struct ir_insn *bpf_ir_create_store_insn(struct bpf_ir_env *env, + struct ir_insn *pos_insn, + struct ir_insn *insn, + struct ir_value val, + enum insert_position pos); -struct ir_insn *create_store_insn_bb(struct bpf_ir_env *env, - struct ir_basic_block *bb, - struct ir_insn *st_insn, - struct ir_value val, - enum insert_position pos); +struct ir_insn *bpf_ir_create_store_insn_bb(struct bpf_ir_env *env, + struct ir_basic_block *pos_bb, + struct ir_insn *insn, + struct ir_value val, + enum insert_position pos); -struct ir_insn *create_load_insn(struct bpf_ir_env *env, struct ir_insn *insn, - struct ir_value val, enum insert_position pos); +struct ir_insn *bpf_ir_create_load_insn(struct bpf_ir_env *env, + struct ir_insn *pos_insn, + struct ir_value val, + enum insert_position pos); -struct ir_insn *create_load_insn_bb(struct bpf_ir_env *env, - struct ir_basic_block *bb, - struct ir_value val, - enum insert_position pos); +struct ir_insn *bpf_ir_create_load_insn_bb(struct bpf_ir_env *env, + struct ir_basic_block *pos_bb, + struct ir_value val, + enum insert_position pos); -struct ir_insn *create_bin_insn(struct bpf_ir_env *env, struct ir_insn *insn, - struct ir_value val1, struct ir_value val2, - enum ir_insn_type ty, enum ir_alu_op_type aluty, - enum insert_position pos); +struct ir_insn * +bpf_ir_create_bin_insn(struct bpf_ir_env *env, struct ir_insn *pos_insn, + struct ir_value val1, struct ir_value val2, + enum ir_insn_type ty, enum ir_alu_op_type alu_type, + enum insert_position pos); -struct ir_insn *create_bin_insn_bb(struct bpf_ir_env *env, - struct ir_basic_block *bb, - struct ir_value val1, struct ir_value val2, - enum ir_insn_type ty, - enum ir_alu_op_type aluty, - enum insert_position pos); +struct ir_insn * +bpf_ir_create_bin_insn_bb(struct bpf_ir_env *env, struct ir_basic_block *pos_bb, + struct ir_value val1, struct ir_value val2, + enum ir_insn_type ty, enum ir_alu_op_type alu_type, + enum insert_position pos); -struct ir_insn *create_ja_insn(struct bpf_ir_env *env, struct ir_insn *insn, - struct ir_basic_block *to_bb, - enum insert_position pos); +struct ir_insn * +bpf_ir_create_bin_insn_cg(struct bpf_ir_env *env, struct ir_insn *pos_insn, + struct ir_value val1, struct ir_value val2, + enum ir_insn_type ty, enum ir_alu_op_type alu_type, + enum insert_position pos); + +struct ir_insn *bpf_ir_create_bin_insn_bb_cg( + struct bpf_ir_env *env, struct ir_basic_block *pos_bb, + struct ir_value val1, struct ir_value val2, enum ir_insn_type ty, + enum ir_alu_op_type alu_type, enum insert_position pos); + +struct ir_insn *bpf_ir_create_ja_insn(struct bpf_ir_env *env, + struct ir_insn *pos_insn, + struct ir_basic_block *to_bb, + enum insert_position pos); -struct ir_insn *create_ja_insn_bb(struct bpf_ir_env *env, - struct ir_basic_block *bb, - struct ir_basic_block *to_bb, - enum insert_position pos); +struct ir_insn *bpf_ir_create_ja_insn_bb(struct bpf_ir_env *env, + struct ir_basic_block *pos_bb, + struct ir_basic_block *to_bb, + enum insert_position pos); -struct ir_insn *create_jbin_insn(struct bpf_ir_env *env, struct ir_insn *insn, - struct ir_value val1, struct ir_value val2, - struct ir_basic_block *to_bb1, - struct ir_basic_block *to_bb2, - enum ir_insn_type ty, - enum ir_alu_op_type aluty, - enum insert_position pos); +struct ir_insn * +bpf_ir_create_jbin_insn(struct bpf_ir_env *env, struct ir_insn *pos_insn, + struct ir_value val1, struct ir_value val2, + struct ir_basic_block *to_bb1, + struct ir_basic_block *to_bb2, enum ir_insn_type ty, + enum ir_alu_op_type alu_type, enum insert_position pos); struct ir_insn * -create_jbin_insn_bb(struct bpf_ir_env *env, struct ir_basic_block *bb, - struct ir_value val1, struct ir_value val2, - struct ir_basic_block *to_bb1, - struct ir_basic_block *to_bb2, enum ir_insn_type ty, - enum ir_alu_op_type aluty, enum insert_position pos); - -struct ir_insn *create_ret_insn(struct bpf_ir_env *env, struct ir_insn *insn, - struct ir_value val, enum insert_position pos); - -struct ir_insn *create_ret_insn_bb(struct bpf_ir_env *env, - struct ir_basic_block *bb, - struct ir_value val, - enum insert_position pos); - -struct ir_insn *create_assign_insn(struct bpf_ir_env *env, struct ir_insn *insn, - struct ir_value val, - enum insert_position pos); - -struct ir_insn *create_assign_insn_bb(struct bpf_ir_env *env, - struct ir_basic_block *bb, - struct ir_value val, - enum insert_position pos); +bpf_ir_create_jbin_insn_bb(struct bpf_ir_env *env, + struct ir_basic_block *pos_bb, struct ir_value val1, + struct ir_value val2, struct ir_basic_block *to_bb1, + struct ir_basic_block *to_bb2, enum ir_insn_type ty, + enum ir_alu_op_type alu_type, + enum insert_position pos); + +struct ir_insn *bpf_ir_create_ret_insn(struct bpf_ir_env *env, + struct ir_insn *pos_insn, + struct ir_value val, + enum insert_position pos); + +struct ir_insn *bpf_ir_create_ret_insn_bb(struct bpf_ir_env *env, + struct ir_basic_block *pos_bb, + struct ir_value val, + enum insert_position pos); + +struct ir_insn *bpf_ir_create_call_insn(struct bpf_ir_env *env, + struct ir_insn *pos_insn, s32 fid, + enum insert_position pos); + +struct ir_insn *bpf_ir_create_call_insn_bb(struct bpf_ir_env *env, + struct ir_basic_block *pos_bb, + s32 fid, enum insert_position pos); + +struct ir_insn *bpf_ir_create_loadraw_insn(struct bpf_ir_env *env, + struct ir_insn *pos_insn, + enum ir_vr_type type, + struct ir_address_value val, + enum insert_position pos); + +struct ir_insn *bpf_ir_create_loadraw_insn_bb(struct bpf_ir_env *env, + struct ir_basic_block *pos_bb, + enum ir_vr_type type, + struct ir_address_value val, + enum insert_position pos); + +struct ir_insn *bpf_ir_create_loadraw_insn_cg(struct bpf_ir_env *env, + struct ir_insn *pos_insn, + enum ir_vr_type type, + struct ir_address_value val, + enum insert_position pos); + +struct ir_insn *bpf_ir_create_loadraw_insn_bb_cg(struct bpf_ir_env *env, + struct ir_basic_block *pos_bb, + enum ir_vr_type type, + struct ir_address_value val, + enum insert_position pos); -struct ir_insn *create_phi_insn(struct ir_insn *insn, enum insert_position pos); +struct ir_insn * +bpf_ir_create_storeraw_insn(struct bpf_ir_env *env, struct ir_insn *pos_insn, + enum ir_vr_type type, struct ir_address_value val, + struct ir_value to_store, enum insert_position pos); -struct ir_insn *create_phi_insn_bb(struct ir_basic_block *bb, - enum insert_position pos); +struct ir_insn *bpf_ir_create_storeraw_insn_bb(struct bpf_ir_env *env, + struct ir_basic_block *pos_bb, + enum ir_vr_type type, + struct ir_address_value val, + struct ir_value to_store, + enum insert_position pos); -void phi_add_operand(struct bpf_ir_env *env, struct ir_insn *insn, - struct ir_basic_block *bb, struct ir_value val); +struct ir_insn *bpf_ir_create_storeraw_insn_cg(struct bpf_ir_env *env, + struct ir_insn *pos_insn, + enum ir_vr_type type, + struct ir_address_value val, + struct ir_value to_store, + enum insert_position pos); -void val_add_user(struct bpf_ir_env *env, struct ir_value val, - struct ir_insn *user); +struct ir_insn *bpf_ir_create_storeraw_insn_bb_cg(struct bpf_ir_env *env, + struct ir_basic_block *pos_bb, + enum ir_vr_type type, + struct ir_address_value val, + struct ir_value to_store, + enum insert_position pos); -void val_remove_user(struct ir_value val, struct ir_insn *user); +struct ir_insn *bpf_ir_create_assign_insn(struct bpf_ir_env *env, + struct ir_insn *pos_insn, + struct ir_value val, + enum insert_position pos); -struct ir_insn *create_assign_insn_cg(struct bpf_ir_env *env, - struct ir_insn *insn, struct ir_value val, - enum insert_position pos); +struct ir_insn *bpf_ir_create_assign_insn_bb(struct bpf_ir_env *env, + struct ir_basic_block *pos_bb, + struct ir_value val, + enum insert_position pos); -struct ir_insn *create_assign_insn_bb_cg(struct bpf_ir_env *env, - struct ir_basic_block *bb, - struct ir_value val, - enum insert_position pos); +struct ir_insn *bpf_ir_create_assign_insn_cg(struct bpf_ir_env *env, + struct ir_insn *pos_insn, + struct ir_value val, + enum insert_position pos); + +struct ir_insn *bpf_ir_create_assign_insn_bb_cg(struct bpf_ir_env *env, + struct ir_basic_block *pos_bb, + struct ir_value val, + enum insert_position pos); + +struct ir_insn *bpf_ir_create_phi_insn(struct bpf_ir_env *env, + struct ir_insn *pos_insn, + enum insert_position pos); + +struct ir_insn *bpf_ir_create_phi_insn_bb(struct bpf_ir_env *env, + struct ir_basic_block *pos_bb, + enum insert_position pos); + +/* Instruction Constructors */ + +void bpf_ir_phi_add_operand(struct bpf_ir_env *env, struct ir_insn *insn, + struct ir_basic_block *bb, struct ir_value val); + +void bpf_ir_phi_add_call_arg(struct bpf_ir_env *env, struct ir_insn *insn, + struct ir_value val); -void replace_operand(struct bpf_ir_env *env, struct ir_insn *insn, - struct ir_value v1, struct ir_value v2); +void bpf_ir_val_add_user(struct bpf_ir_env *env, struct ir_value val, + struct ir_insn *user); -struct ir_insn *create_insn_base_cg(struct bpf_ir_env *env, - struct ir_basic_block *bb); +void bpf_ir_val_remove_user(struct ir_value val, struct ir_insn *user); -struct ir_insn *create_insn_base(struct ir_basic_block *bb); +void bpf_ir_replace_operand(struct bpf_ir_env *env, struct ir_insn *insn, + struct ir_value v1, struct ir_value v2); -void insert_at(struct ir_insn *new_insn, struct ir_insn *insn, - enum insert_position pos); +struct ir_insn *bpf_ir_create_insn_base_cg(struct bpf_ir_env *env, + struct ir_basic_block *bb, + enum ir_insn_type insn_type); -void insert_at_bb(struct ir_insn *new_insn, struct ir_basic_block *bb, - enum insert_position pos); +struct ir_insn *bpf_ir_create_insn_base(struct bpf_ir_env *env, + struct ir_basic_block *bb); + +void bpf_ir_insert_at(struct ir_insn *new_insn, struct ir_insn *insn, + enum insert_position pos); + +void bpf_ir_insert_at_bb(struct ir_insn *new_insn, struct ir_basic_block *bb, + enum insert_position pos); /* IR Instructions End */ @@ -807,6 +972,8 @@ void bpf_ir_init_insn_cg(struct bpf_ir_env *env, struct ir_insn *insn); void bpf_ir_code_gen(struct bpf_ir_env *env, struct ir_function *fun); +void bpf_ir_free_insn_cg(struct ir_insn *insn); + // Extra information needed for code gen struct ir_bb_cg_extra { // Position of the first instruction @@ -815,7 +982,7 @@ struct ir_bb_cg_extra { struct ir_insn_cg_extra { // Destination (Not in SSA form anymore) - struct ir_insn *dst; + struct ir_value dst; // Liveness analysis struct array in; @@ -835,13 +1002,16 @@ struct ir_insn_cg_extra { // Whether the VR is allocated with a real register // If it's a pre-colored register, it's also 1 - u8 allocated; + bool allocated; // When allocating register, whether dst will be spilled // 0: Not spilled - // 1: Spilled on stack position 1 + // -8: Spilled on SP-8 // etc. - size_t spilled; + s32 spilled; + + // The size of the spilled register + u32 spilled_size; // Valid if spilled == 0 && allocated == 1 // Valid number: 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 @@ -856,11 +1026,12 @@ enum val_type { REG, CONST, STACK, + STACKOFF, }; #define insn_cg(insn) ((struct ir_insn_cg_extra *)(insn)->user_data) -#define insn_dst(insn) insn_cg(insn)->dst +#define insn_dst(insn) insn_cg(insn)->dst.data.insn_d /* Code Gen End */ @@ -890,12 +1061,29 @@ struct ir_constraint { /* IR Value Start */ -u8 bpf_ir_value_equal(struct ir_value a, struct ir_value b); +bool bpf_ir_value_equal(struct ir_value a, struct ir_value b); struct ir_value bpf_ir_value_insn(struct ir_insn *); -struct ir_value bpf_ir_value_stack_ptr(void); +struct ir_value bpf_ir_value_const32(s32 val); + +struct ir_value bpf_ir_value_const64(s64 val); + +struct ir_value bpf_ir_value_undef(void); + +struct ir_address_value bpf_ir_addr_val(struct ir_value value, s16 offset); + +struct ir_value bpf_ir_value_stack_ptr(struct ir_function *fun); + +void bpf_ir_change_value(struct bpf_ir_env *env, struct ir_insn *insn, + struct ir_value *old, struct ir_value new); /* IR Value End */ +/* IR Optimization Start */ + +void bpf_ir_optimize_ir(struct bpf_ir_env *env, struct ir_function *fun); + +/* IR Optimization End */ + #endif diff --git a/IR/ir_code_gen.c b/IR/ir_code_gen.c index 63205413..872a8d3f 100644 --- a/IR/ir_code_gen.c +++ b/IR/ir_code_gen.c @@ -1,5 +1,20 @@ +#include #include +static void set_insn_dst(struct bpf_ir_env *env, struct ir_insn *insn, + struct ir_insn *dst) +{ + struct ir_value v = dst ? bpf_ir_value_insn(dst) : bpf_ir_value_undef(); + if (insn_cg(insn)->dst.type == IR_VALUE_INSN) { + // Remove previous user + // Change all users to new dst (used in coalescing) + bpf_ir_replace_all_usage_cg(env, insn, v); + } else { + bpf_ir_val_add_user(env, v, insn); + } + insn_cg(insn)->dst = v; +} + static void init_cg(struct bpf_ir_env *env, struct ir_function *fun) { struct ir_basic_block **pos = NULL; @@ -18,28 +33,29 @@ static void init_cg(struct bpf_ir_env *env, struct ir_function *fun) } } - for (u8 i = 0; i < MAX_BPF_REG; ++i) { - SAFE_MALLOC(fun->cg_info.regs[i], sizeof(struct ir_insn)); - // Those should be read-only + for (u8 i = 0; i < BPF_REG_10; ++i) { struct ir_insn *insn = fun->cg_info.regs[i]; - insn->op = IR_INSN_REG; - insn->parent_bb = NULL; - INIT_ARRAY(&insn->users, struct ir_insn *); - insn->value_num = 0; bpf_ir_init_insn_cg(env, insn); CHECK_ERR(); struct ir_insn_cg_extra *extra = insn_cg(insn); extra->alloc_reg = i; - extra->dst = insn; // Pre-colored registers are allocated - extra->allocated = 1; + extra->allocated = true; extra->spilled = 0; + extra->spilled_size = 0; extra->nonvr = true; } + bpf_ir_init_insn_cg(env, fun->sp); + struct ir_insn_cg_extra *extra = insn_cg(fun->sp); + extra->alloc_reg = 10; + extra->allocated = true; + extra->spilled = 0; + extra->spilled_size = 0; + extra->nonvr = true; } -static void free_insn_cg(struct ir_insn *insn) +void bpf_ir_free_insn_cg(struct ir_insn *insn) { struct ir_insn_cg_extra *extra = insn_cg(insn); bpf_ir_array_free(&extra->adj); @@ -62,16 +78,15 @@ static void free_cg_res(struct ir_function *fun) bb->user_data = NULL; struct ir_insn *insn = NULL; list_for_each_entry(insn, &bb->ir_insn_head, list_ptr) { - free_insn_cg(insn); + bpf_ir_free_insn_cg(insn); } } - for (u8 i = 0; i < MAX_BPF_REG; ++i) { + for (u8 i = 0; i < BPF_REG_10; ++i) { struct ir_insn *insn = fun->cg_info.regs[i]; - bpf_ir_array_free(&insn->users); - free_insn_cg(insn); - free_proto(insn); + bpf_ir_free_insn_cg(insn); } + bpf_ir_free_insn_cg(fun->sp); } static void clean_insn_cg(struct bpf_ir_env *env, struct ir_insn *insn) @@ -94,16 +109,20 @@ static void clean_cg(struct bpf_ir_env *env, struct ir_function *fun) list_for_each_entry(insn, &bb->ir_insn_head, list_ptr) { clean_insn_cg(env, insn); struct ir_insn_cg_extra *extra = insn_cg(insn); - extra->allocated = 0; - extra->spilled = 0; - extra->alloc_reg = 0; + if (insn->op != IR_INSN_ALLOCARRAY) { + extra->allocated = false; + extra->spilled = 0; + extra->spilled_size = 0; + extra->alloc_reg = 0; + } } } - for (u8 i = 0; i < MAX_BPF_REG; ++i) { + for (u8 i = 0; i < BPF_REG_10; ++i) { struct ir_insn *insn = fun->cg_info.regs[i]; clean_insn_cg(env, insn); } + clean_insn_cg(env, fun->sp); // But should have no effect I guess? bpf_ir_array_clear(env, &fun->cg_info.all_var); } @@ -193,17 +212,18 @@ static void to_cssa(struct bpf_ir_env *env, struct ir_function *fun) { struct ir_insn *insn = *pos2; // Create the moved PHI insn - struct ir_insn *new_phi = create_phi_insn(insn, INSERT_FRONT); + struct ir_insn *new_phi = + bpf_ir_create_phi_insn(env, insn, INSERT_FRONT); struct phi_value *pos3; array_for(pos3, insn->phi) { - struct ir_insn *new_insn = create_assign_insn_bb( + struct ir_insn *new_insn = bpf_ir_create_assign_insn_bb( env, pos3->bb, pos3->value, INSERT_BACK_BEFORE_JMP); // Remove use - val_remove_user(pos3->value, insn); - phi_add_operand(env, new_phi, pos3->bb, - bpf_ir_value_insn(new_insn)); + bpf_ir_val_remove_user(pos3->value, insn); + bpf_ir_phi_add_operand(env, new_phi, pos3->bb, + bpf_ir_value_insn(new_insn)); } bpf_ir_array_free(&insn->phi); @@ -211,7 +231,7 @@ static void to_cssa(struct bpf_ir_env *env, struct ir_function *fun) struct ir_value val = bpf_ir_value_insn(new_phi); insn->values[0] = val; insn->value_num = 1; - val_add_user(env, val, insn); + bpf_ir_val_add_user(env, val, insn); } bpf_ir_array_free(&phi_insns); @@ -249,7 +269,8 @@ static void remove_phi(struct bpf_ir_env *env, struct ir_function *fun) if (!repr) { repr = pos3->value.data.insn_d; } else { - insn_cg(pos3->value.data.insn_d)->dst = repr; + set_insn_dst(env, pos3->value.data.insn_d, + repr); } } if (!repr) { @@ -258,48 +279,13 @@ static void remove_phi(struct bpf_ir_env *env, struct ir_function *fun) DBGASSERT(repr == insn_dst(repr)); - bpf_ir_replace_all_usage(env, insn, bpf_ir_value_insn(repr)); - bpf_ir_erase_insn(env, insn); + bpf_ir_replace_all_usage_cg(env, insn, bpf_ir_value_insn(repr)); + bpf_ir_erase_insn_cg(env, insn); } bpf_ir_array_free(&phi_insns); } -// Erase an instruction without checking the users -static void bpf_ir_erase_insn_raw(struct ir_insn *insn) -{ - list_del(&insn->list_ptr); - free_proto(insn); -} - -static void coaleasing(struct ir_function *fun) -{ - struct ir_basic_block **pos; - // For each BB - array_for(pos, fun->reachable_bbs) - { - struct ir_basic_block *bb = *pos; - struct ir_insn *pos2, *tmp; - // For each operation - list_for_each_entry_safe(pos2, tmp, &bb->ir_insn_head, - list_ptr) { - struct ir_insn *insn_dst = insn_dst(pos2); - if (pos2->op == IR_INSN_ASSIGN) { - if (pos2->values[0].type == IR_VALUE_INSN) { - struct ir_insn *src = - pos2->values[0].data.insn_d; - DBGASSERT(src == insn_dst(src)); - if (insn_cg(src)->alloc_reg == - insn_cg(insn_dst)->alloc_reg) { - // Remove - bpf_ir_erase_insn_raw(pos2); - } - } - } - } - } -} - static bool is_insn_final(struct ir_insn *v1) { return v1 == insn_dst(v1); @@ -331,16 +317,18 @@ static void bpf_ir_print_interference_graph(struct bpf_ir_env *env, CRITICAL( "Pre-colored register should not be in all_var"); } - if (!is_insn_final(insn)) { + struct ir_insn_cg_extra *extra = insn_cg(insn); + if (!is_insn_final(extra->dst.data.insn_d)) { // Not final value, give up - CRITICAL("Not Final Value!"); + print_ir_insn_err_full(env, insn, "Instruction", + print_ir_dst); + RAISE_ERROR("Not Final Value!"); } - struct ir_insn_cg_extra *extra = insn_cg(insn); if (extra->allocated) { // Allocated VR PRINT_LOG(env, "%%%zu(", insn->_insn_id); if (extra->spilled) { - PRINT_LOG(env, "sp-%zu", extra->spilled * 8); + PRINT_LOG(env, "sp+%d", extra->spilled); } else { PRINT_LOG(env, "r%u", extra->alloc_reg); } @@ -432,90 +420,214 @@ static void conflict_analysis(struct bpf_ir_env *env, struct ir_function *fun) } } -static enum ir_vr_type alu_to_vr_type(enum ir_alu_op_type ty) +static bool has_conflict(struct ir_insn *v1, struct ir_insn *v2) { - if (ty == IR_ALU_32) { - return IR_VR_TYPE_32; - } else if (ty == IR_ALU_64) { - return IR_VR_TYPE_64; - } else { - CRITICAL("Error"); + if (!is_insn_final(v1) || !is_insn_final(v2)) { + CRITICAL("Can only test conflict on final values"); + } + if (v1 == v2) { + return false; } + struct array adj = insn_cg(v1)->adj; + struct ir_insn **pos; + array_for(pos, adj) + { + if (*pos == v2) { + return true; + } + } + return false; } -// Make register usage explicit -// Example: -// %x = add %y, %arg1 -// arg1 is r0 at the beginning of the function -// We then add a new instruction to the beginning of the function. - -static void explicit_reg(struct bpf_ir_env *env, struct ir_function *fun) +/* Optimization: Coalescing */ +static void coalescing(struct bpf_ir_env *env, struct ir_function *fun) { - // fun is still in IR form - // Before this step, users are correct - // In this step we change some dsts - // We need carefully handle the users - // dsts are NOT users - // Invariant: All operands are final values - // Final value: v == dst(v) struct ir_basic_block **pos; - // Maximum number of functions: MAX_FUNC_ARG + // For each BB array_for(pos, fun->reachable_bbs) { struct ir_basic_block *bb = *pos; - struct ir_insn *insn; - list_for_each_entry(insn, &bb->ir_insn_head, list_ptr) { - if (insn->op == IR_INSN_CALL) { - for (u8 i = 0; i < insn->value_num; ++i) { - struct ir_value val = insn->values[i]; - struct ir_insn *new_insn = - create_assign_insn_cg( - env, insn, val, - INSERT_FRONT); - insn_cg(new_insn)->dst = - fun->cg_info.regs[i + 1]; - val_remove_user(val, insn); - } - insn->value_num = 0; // Remove all operands - struct ir_insn_cg_extra *extra = insn_cg(insn); - extra->dst = NULL; - if (insn->users.num_elem == 0) { - continue; + struct ir_insn *insn, *tmp; + // For each operation + list_for_each_entry_safe(insn, tmp, &bb->ir_insn_head, + list_ptr) { + struct ir_insn *insn_dst = insn_dst(insn); + if (insn->op == IR_INSN_ASSIGN) { + if (insn->values[0].type == IR_VALUE_INSN) { + struct ir_insn *src = + insn->values[0].data.insn_d; + DBGASSERT(src == insn_dst(src)); + // a = a + if (insn_cg(src)->alloc_reg == + insn_cg(insn_dst)->alloc_reg) { + // Remove + // bpf_ir_erase_insn_cg(env, insn); + continue; + } + // R = r + // r = R + // Able to coalesce + if (insn_cg(src)->nonvr && + insn_cg(insn)->nonvr) { + continue; + } + + if (!has_conflict(insn_dst, src)) { + // No Conflict, could coalesce + PRINT_LOG( + env, + "Coalescing %u and %u\n", + insn_cg(src)->alloc_reg, + insn_cg(insn_dst) + ->alloc_reg); + // CRITICAL( + // "Coalescing not implemented"); + // Check if coalescing is beneficial using Briggs' conservative coalescing + u32 count = 0; + struct array merged; + bpf_ir_array_clone( + env, &merged, + &insn_cg(src)->adj); + bpf_ir_array_merge( + env, &merged, + &insn_cg(insn_dst)->adj); + struct ir_insn **pos2; + array_for(pos2, merged) + { + if (*pos2 == insn_dst || + *pos2 == src) { + continue; + } + if (insn_cg((*pos2)) + ->nonvr && + insn_cg((*pos2)) + ->spilled) { + // Pre-colored stack + continue; + } + count++; + } + + // PRINT_LOG(env, "Count: %u\n", count); + if (count < BPF_REG_10) { + // Coalesce + if (insn_cg(src)->nonvr) { + // r = R + set_insn_dst( + env, + insn_dst, + src); + } else { + // R = r or r = r + set_insn_dst( + env, + src, + insn_dst); + } + // bpf_ir_erase_insn_cg( + // env, insn); + } + + bpf_ir_array_free(&merged); + } } - struct ir_insn *new_insn = create_assign_insn_cg( - env, insn, - bpf_ir_value_insn(fun->cg_info.regs[0]), - INSERT_BACK); - bpf_ir_replace_all_usage( - env, insn, bpf_ir_value_insn(new_insn)); } + } + } +} + +// Pre CG +static void change_fun_arg(struct bpf_ir_env *env, struct ir_function *fun) +{ + // Change function call args + for (u8 i = 0; i < MAX_FUNC_ARG; ++i) { + if (fun->function_arg[i]->users.num_elem > 0) { + // Insert ASSIGN arg[i] at the beginning of the function + struct ir_insn *new_insn = bpf_ir_create_assign_insn_bb( + env, fun->entry, + bpf_ir_value_insn(fun->cg_info.regs[i + 1]), + INSERT_FRONT_AFTER_PHI); + bpf_ir_replace_all_usage(env, fun->function_arg[i], + bpf_ir_value_insn(new_insn)); + } + } +} +// CG: After init +static void change_ret(struct bpf_ir_env *env, struct ir_function *fun) +{ + struct ir_basic_block **pos; + array_for(pos, fun->reachable_bbs) + { + struct ir_basic_block *bb = *pos; + struct ir_insn *insn; + list_for_each_entry(insn, &bb->ir_insn_head, list_ptr) { if (insn->op == IR_INSN_RET) { // ret x // ==> // R0 = x // ret struct ir_insn *new_insn = - create_assign_insn_cg(env, insn, - insn->values[0], - INSERT_FRONT); + bpf_ir_create_assign_insn_cg( + env, insn, insn->values[0], + INSERT_FRONT); new_insn->alu_op = IR_ALU_64; - val_remove_user(insn->values[0], insn); - insn_cg(new_insn)->dst = fun->cg_info.regs[0]; + set_insn_dst(env, new_insn, + fun->cg_info.regs[0]); insn->value_num = 0; } } } - // Arg - for (u8 i = 0; i < MAX_FUNC_ARG; ++i) { - if (fun->function_arg[i]->users.num_elem > 0) { - // Insert ASSIGN arg[i] at the beginning of the function - struct ir_insn *new_insn = create_assign_insn_bb_cg( - env, fun->entry, - bpf_ir_value_insn(fun->cg_info.regs[i + 1]), - INSERT_FRONT_AFTER_PHI); - bpf_ir_replace_all_usage(env, fun->function_arg[i], - bpf_ir_value_insn(new_insn)); +} + +static void change_call_pre_cg(struct bpf_ir_env *env, struct ir_function *fun) +{ + struct ir_basic_block **pos; + array_for(pos, fun->reachable_bbs) + { + struct ir_basic_block *bb = *pos; + struct ir_insn *insn; + list_for_each_entry(insn, &bb->ir_insn_head, list_ptr) { + if (insn->op == IR_INSN_CALL) { + if (insn->users.num_elem == 0) { + continue; + } + struct ir_insn *new_insn = + bpf_ir_create_assign_insn( + env, insn, + bpf_ir_value_insn( + fun->cg_info.regs[0]), + INSERT_BACK); + bpf_ir_replace_all_usage( + env, insn, bpf_ir_value_insn(new_insn)); + } + } + } +} + +// After init +static void change_call(struct bpf_ir_env *env, struct ir_function *fun) +{ + struct ir_basic_block **pos; + array_for(pos, fun->reachable_bbs) + { + struct ir_basic_block *bb = *pos; + struct ir_insn *insn; + list_for_each_entry(insn, &bb->ir_insn_head, list_ptr) { + if (insn->op == IR_INSN_CALL) { + for (u8 i = 0; i < insn->value_num; ++i) { + struct ir_value val = insn->values[i]; + bpf_ir_val_remove_user(val, insn); + struct ir_insn *new_insn = + bpf_ir_create_assign_insn_cg( + env, insn, val, + INSERT_FRONT); + set_insn_dst(env, new_insn, + fun->cg_info.regs[i + 1]); + } + insn->value_num = 0; // Remove all operands + set_insn_dst(env, insn, NULL); + } } } } @@ -546,52 +658,68 @@ static void graph_coloring(struct bpf_ir_env *env, struct ir_function *fun) "Pre-colored register should not be in all_var"); } struct ir_insn_cg_extra *extra = insn_cg(insn); + if (extra->allocated) { + // Already allocated + continue; + } struct ir_insn **pos2; int used_reg[MAX_BPF_REG] = { 0 }; struct array used_spill; - INIT_ARRAY(&used_spill, size_t); + INIT_ARRAY(&used_spill, s32); array_for(pos2, extra->adj) { struct ir_insn *insn2 = *pos2; // Adj instruction struct ir_insn_cg_extra *extra2 = insn_cg(insn2); if (extra2->allocated) { if (extra2->spilled) { - bpf_ir_array_push_unique( - env, &used_spill, - &extra2->spilled); + if (extra2->spilled_size == 0) { + RAISE_ERROR( + "Found a spilling a register that has 0 size"); + } + u32 spill_number = + (extra2->spilled_size - 1) / 8 + + 1; + for (u32 i = 0; i < spill_number; i++) { + bpf_ir_array_push_unique( + env, &used_spill, + &extra2->spilled - + i * 8); + } } else { used_reg[extra2->alloc_reg] = 1; } } } - u8 need_spill = 1; + bool need_spill = true; for (u8 i = 0; i < MAX_BPF_REG; i++) { if (!used_reg[i]) { - extra->allocated = 1; + extra->allocated = true; PRINT_LOG(env, "Allocate r%u for %%%zu\n", i, insn->_insn_id); extra->alloc_reg = i; - need_spill = 0; + need_spill = false; break; } } if (need_spill) { - size_t sp = 1; + s32 sp = -8; while (1) { - u8 found = 1; - size_t *pos3; + bool found = true; + s32 *pos3; array_for(pos3, used_spill) { if (*pos3 == sp) { - sp++; - found = 0; + sp -= 8; + found = false; break; } } if (found) { - extra->allocated = 1; + extra->allocated = true; extra->spilled = sp; + extra->spilled_size = + 8; // Default size for VR break; } } @@ -614,7 +742,7 @@ static void gen_kill(struct bpf_ir_env *env, struct ir_function *fun) list_for_each_entry(pos2, &bb->ir_insn_head, list_ptr) { struct ir_insn *insn_dst = insn_dst(pos2); struct ir_insn_cg_extra *insn_cg = pos2->user_data; - if (!is_void(pos2) && insn_dst) { + if (!bpf_ir_is_void(pos2) && insn_dst) { bpf_ir_array_push_unique(env, &insn_cg->kill, &insn_dst); } @@ -665,18 +793,6 @@ static struct array array_delta(struct bpf_ir_env *env, struct array *a, return res; } -static void merge_array(struct bpf_ir_env *env, struct array *a, - struct array *b) -{ - struct ir_insn **pos; - array_for(pos, (*b)) - { - struct ir_insn *insn = *pos; - bpf_ir_array_push_unique(env, a, &insn); - CHECK_ERR(); - } -} - static bool equal_set(struct array *a, struct array *b) { if (a->num_elem != b->num_elem) { @@ -729,8 +845,9 @@ static void in_out(struct bpf_ir_env *env, struct ir_function *fun) struct ir_insn_cg_extra *insn2_cg = first->user_data; - merge_array(env, &insn_cg->out, - &insn2_cg->in); + bpf_ir_array_merge( + env, &insn_cg->out, + &insn2_cg->in); CHECK_ERR(); } } else { @@ -740,8 +857,8 @@ static void in_out(struct bpf_ir_env *env, struct ir_function *fun) struct ir_insn, list_ptr); struct ir_insn_cg_extra *next_insn_cg = next_insn->user_data; - merge_array(env, &insn_cg->out, - &next_insn_cg->in); + bpf_ir_array_merge(env, &insn_cg->out, + &next_insn_cg->in); CHECK_ERR(); } struct array out_kill_delta = array_delta( @@ -750,13 +867,14 @@ static void in_out(struct bpf_ir_env *env, struct ir_function *fun) bpf_ir_array_clone(env, &insn_cg->in, &insn_cg->gen); CHECK_ERR(); - merge_array(env, &insn_cg->in, &out_kill_delta); + bpf_ir_array_merge(env, &insn_cg->in, + &out_kill_delta); CHECK_ERR(); // Check for change if (!equal_set(&insn_cg->in, &old_in)) { change = 1; } - // Collect grabage + // Collect garbage bpf_ir_array_free(&out_kill_delta); bpf_ir_array_free(&old_in); } @@ -821,7 +939,11 @@ static enum val_type vtype_insn(struct ir_insn *insn) } struct ir_insn_cg_extra *extra = insn_cg(insn); if (extra->spilled) { - return STACK; + if (insn->op == IR_INSN_ALLOCARRAY) { + return STACKOFF; + } else { + return STACK; + } } else { return REG; } @@ -834,31 +956,30 @@ static enum val_type vtype(struct ir_value val) } else if (val.type == IR_VALUE_CONSTANT || val.type == IR_VALUE_CONSTANT_RAWOFF) { return CONST; - } else if (val.type == IR_VALUE_STACK_PTR) { - return REG; } else { CRITICAL("No such value type for dst"); } } -static bool is_vr_insn(struct ir_insn *insn) -{ - if (insn == NULL || insn->user_data == NULL) { - // Void - return false; - } - return !insn_cg(insn)->nonvr; -} +/* Test whether an instruction is a VR instruction */ +// static bool is_vr_insn(struct ir_insn *insn) +// { +// if (insn == NULL || insn->user_data == NULL) { +// // Void +// return false; +// } +// return !insn_cg(insn)->nonvr; +// } /* Test whether a value is a VR instruction */ -static bool is_vr(struct ir_value val) -{ - if (val.type == IR_VALUE_INSN) { - return is_vr_insn(val.data.insn_d); - } else { - return false; - } -} +// static bool is_vr(struct ir_value val) +// { +// if (val.type == IR_VALUE_INSN) { +// return is_vr_insn(val.data.insn_d); +// } else { +// return false; +// } +// } // Relocate BB static void calc_pos(struct bpf_ir_env *env, struct ir_function *fun) @@ -907,7 +1028,7 @@ static void relocate(struct bpf_ir_env *env, struct ir_function *fun) target - insn_extra->translated[0].pos - 1; } - if (is_cond_jmp(insn)) { + if (bpf_ir_is_cond_jmp(insn)) { DBGASSERT(insn_extra->translated_num == 1); size_t target = bpf_ir_bb_cg(insn->bb2)->pos; insn_extra->translated[0].off = @@ -924,11 +1045,12 @@ static void cgir_load_const_to_reg(struct bpf_ir_env *env, u8 reg) { struct ir_insn *new_insn = - create_assign_insn_cg(env, insn, *val, INSERT_FRONT); + bpf_ir_create_assign_insn_cg(env, insn, *val, INSERT_FRONT); new_insn->alu_op = IR_ALU_64; - insn_cg(new_insn)->dst = fun->cg_info.regs[reg]; + set_insn_dst(env, new_insn, fun->cg_info.regs[reg]); val->type = IR_VALUE_INSN; val->data.insn_d = fun->cg_info.regs[reg]; + bpf_ir_val_add_user(env, *val, fun->cg_info.regs[reg]); } static void cgir_load_reg_to_reg(struct bpf_ir_env *env, @@ -936,11 +1058,12 @@ static void cgir_load_reg_to_reg(struct bpf_ir_env *env, struct ir_value *val, u8 reg) { struct ir_insn *new_insn = - create_assign_insn_cg(env, insn, *val, INSERT_FRONT); + bpf_ir_create_assign_insn_cg(env, insn, *val, INSERT_FRONT); new_insn->alu_op = IR_ALU_64; - insn_cg(new_insn)->dst = fun->cg_info.regs[reg]; + set_insn_dst(env, new_insn, fun->cg_info.regs[reg]); val->type = IR_VALUE_INSN; val->data.insn_d = fun->cg_info.regs[reg]; + bpf_ir_val_add_user(env, *val, fun->cg_info.regs[reg]); } static void cgir_load_stack_to_reg(struct bpf_ir_env *env, @@ -949,12 +1072,13 @@ static void cgir_load_stack_to_reg(struct bpf_ir_env *env, enum ir_vr_type vtype, u8 reg) { struct ir_insn *tmp = - create_assign_insn_cg(env, insn, *val, INSERT_FRONT); + bpf_ir_create_assign_insn_cg(env, insn, *val, INSERT_FRONT); tmp->vr_type = vtype; - insn_cg(tmp)->dst = fun->cg_info.regs[reg]; + set_insn_dst(env, tmp, fun->cg_info.regs[reg]); val->type = IR_VALUE_INSN; val->data.insn_d = fun->cg_info.regs[reg]; + bpf_ir_val_add_user(env, *val, fun->cg_info.regs[reg]); } static void add_stack_offset_vr(struct ir_function *fun, size_t num) @@ -963,8 +1087,8 @@ static void add_stack_offset_vr(struct ir_function *fun, size_t num) array_for(pos, fun->cg_info.all_var) { struct ir_insn_cg_extra *extra = insn_cg(*pos); - if (extra->spilled > 0) { - extra->spilled += num; + if (extra->spilled) { + extra->spilled -= num * 8; } } } @@ -980,7 +1104,10 @@ static u8 allocated_reg(struct ir_value val) return allocated_reg_insn(val.data.insn_d); } -/* Spilling callee */ +/* Spilling callee + + NOT TESTED YET + */ static void spill_callee(struct bpf_ir_env *env, struct ir_function *fun) { // Spill Callee saved registers if used @@ -990,7 +1117,10 @@ static void spill_callee(struct bpf_ir_env *env, struct ir_function *fun) array_for(pos, fun->cg_info.all_var) { struct ir_insn_cg_extra *extra = insn_cg(*pos); - reg_used[extra->alloc_reg] = 1; + DBGASSERT(extra->allocated); + if (extra->spilled == 0) { + reg_used[extra->alloc_reg] = 1; + } } size_t off = 0; for (u8 i = BPF_REG_6; i < BPF_REG_10; ++i) { @@ -1008,37 +1138,35 @@ static void spill_callee(struct bpf_ir_env *env, struct ir_function *fun) // Spill at sp-off // struct ir_insn *st = create_assign_insn_bb_cg(env, // fun->entry, ir_value_insn(fun->cg_info.regs[i]), INSERT_FRONT); - struct ir_insn *st = - create_insn_base_cg(env, fun->entry); - insert_at_bb(st, fun->entry, INSERT_FRONT); - st->op = IR_INSN_STORERAW; - st->values[0] = bpf_ir_value_insn(fun->cg_info.regs[i]); + struct ir_insn *st = bpf_ir_create_insn_base_cg( + env, fun->entry, IR_INSN_STORERAW); + bpf_ir_insert_at_bb(st, fun->entry, INSERT_FRONT); + // st->values[0] = bpf_ir_value_insn(fun->cg_info.regs[i]); + bpf_ir_val_add_user(env, st->values[0], + fun->cg_info.regs[i]); st->value_num = 1; st->vr_type = IR_VR_TYPE_64; - struct ir_value val; - val.type = IR_VALUE_STACK_PTR; - st->addr_val.value = val; + st->addr_val.value = bpf_ir_value_stack_ptr(fun); st->addr_val.offset = -off * 8; - struct ir_insn_cg_extra *extra = insn_cg(st); - extra->dst = NULL; + set_insn_dst(env, st, NULL); struct ir_basic_block **pos2; array_for(pos2, fun->end_bbs) { struct ir_basic_block *bb = *pos2; - struct ir_insn *ld = - create_insn_base_cg(env, bb); - insert_at_bb(ld, bb, INSERT_BACK_BEFORE_JMP); - ld->op = IR_INSN_LOADRAW; + struct ir_insn *ld = bpf_ir_create_insn_base_cg( + env, bb, IR_INSN_LOADRAW); + bpf_ir_insert_at_bb(ld, bb, + INSERT_BACK_BEFORE_JMP); ld->value_num = 0; ld->vr_type = IR_VR_TYPE_64; - struct ir_value val; - val.type = IR_VALUE_STACK_PTR; - ld->addr_val.value = val; + // ld->addr_val.value = + // bpf_ir_value_stack_ptr(fun); + bpf_ir_val_add_user(env, ld->addr_val.value, + fun->sp); ld->addr_val.offset = -off * 8; - extra = insn_cg(ld); - extra->dst = fun->cg_info.regs[i]; + set_insn_dst(env, ld, fun->cg_info.regs[i]); } } } @@ -1052,16 +1180,15 @@ static struct ir_insn *normalize_load_const(struct bpf_ir_env *env, struct ir_value *val) { if (val->const_type == IR_ALU_32) { - struct ir_insn *new_insn = - create_assign_insn_cg(env, insn, *val, INSERT_FRONT); + struct ir_insn *new_insn = bpf_ir_create_assign_insn_cg( + env, insn, *val, INSERT_FRONT); new_insn->alu_op = IR_ALU_64; val->type = IR_VALUE_INSN; val->data.insn_d = new_insn; return new_insn; } else { - struct ir_insn *new_insn = - create_insn_base_cg(env, insn->parent_bb); - new_insn->op = IR_INSN_LOADIMM_EXTRA; + struct ir_insn *new_insn = bpf_ir_create_insn_base_cg( + env, insn->parent_bb, IR_INSN_LOADIMM_EXTRA); new_insn->imm_extra_type = IR_LOADIMM_IMM64; new_insn->imm64 = val->data.constant_d; new_insn->vr_type = IR_VR_TYPE_64; @@ -1098,23 +1225,20 @@ static void normalize_alu(struct bpf_ir_env *env, struct ir_function *fun, // ==> // reg1 = stack // reg1 = add reg1 const - struct ir_insn *new_insn = - create_assign_insn_cg(env, insn, *v0, INSERT_FRONT); - insn_cg(new_insn)->dst = dst_insn; - // new_insn->vr_type = alu_to_vr_type(insn->alu_op); - v0->type = IR_VALUE_INSN; - v0->data.insn_d = dst_insn; + struct ir_insn *new_insn = bpf_ir_create_assign_insn_cg( + env, insn, *v0, INSERT_FRONT); + set_insn_dst(env, new_insn, dst_insn); + bpf_ir_change_value(env, insn, v0, bpf_ir_value_insn(dst_insn)); } else if (t0 == STACK && t1 == REG) { // reg1 = add stack reg2 // ==> // reg1 = stack // reg1 = add reg1 reg2 - struct ir_insn *new_insn = - create_assign_insn_cg(env, insn, *v0, INSERT_FRONT); - insn_cg(new_insn)->dst = dst_insn; + struct ir_insn *new_insn = bpf_ir_create_assign_insn_cg( + env, insn, *v0, INSERT_FRONT); + set_insn_dst(env, new_insn, dst_insn); - v0->type = IR_VALUE_INSN; - v0->data.insn_d = dst_insn; + bpf_ir_change_value(env, insn, v0, bpf_ir_value_insn(dst_insn)); } else if (t0 == REG && t1 == REG) { // reg1 = add reg2 reg3 u8 reg1 = insn_cg(dst_insn)->alloc_reg; @@ -1124,12 +1248,12 @@ static void normalize_alu(struct bpf_ir_env *env, struct ir_function *fun, // ==> // reg1 = reg2 // reg1 = add reg1 reg3 - struct ir_insn *new_insn = create_assign_insn_cg( + struct ir_insn *new_insn = bpf_ir_create_assign_insn_cg( env, insn, *v0, INSERT_FRONT); DBGASSERT(dst_insn == fun->cg_info.regs[reg1]); - insn_cg(new_insn)->dst = dst_insn; - v0->type = IR_VALUE_INSN; - v0->data.insn_d = dst_insn; + set_insn_dst(env, new_insn, dst_insn); + bpf_ir_change_value(env, insn, v0, + bpf_ir_value_insn(dst_insn)); } } else if (t0 == REG && t1 == CONST) { if (allocated_reg(*v0) != allocated_reg_insn(dst_insn)) { @@ -1137,36 +1261,18 @@ static void normalize_alu(struct bpf_ir_env *env, struct ir_function *fun, // ==> // reg1 = reg2 // reg1 = add reg1 const - struct ir_insn *new_insn = create_assign_insn_cg( + struct ir_insn *new_insn = bpf_ir_create_assign_insn_cg( env, insn, *v0, INSERT_FRONT); - insn_cg(new_insn)->dst = dst_insn; - v0->type = IR_VALUE_INSN; - v0->data.insn_d = dst_insn; + set_insn_dst(env, new_insn, dst_insn); + bpf_ir_change_value(env, insn, v0, + bpf_ir_value_insn(dst_insn)); } } else if (t0 == CONST && t1 == CONST) { DBGASSERT(v1->const_type == IR_ALU_32); - // if (v0->const_type == IR_ALU_32) { - // // Load to reg - // struct ir_insn *new_insn = create_assign_insn_cg( - // env, insn, *v0, INSERT_FRONT); - // new_insn->alu_op = IR_ALU_64; - // insn_cg(new_insn)->dst = dst_insn; - // v0->type = IR_VALUE_INSN; - // v0->data.insn_d = dst_insn; - // } else { - // // ALU64 - // struct ir_insn *new_insn = create_insn_base_cg(env, bb); - // new_insn->op = IR_INSN_LOADIMM_EXTRA; - // new_insn->imm_extra_type = IR_LOADIMM_IMM64; - // new_insn->imm64 = v0->data.constant_d; - // insn_cg(new_insn)->dst = dst_insn; - // v0->type = IR_VALUE_INSN; - // v0->data.insn_d = dst_insn; - // } struct ir_insn *load_const_insn = normalize_load_const(env, insn, v0); - insn_cg(load_const_insn)->dst = dst_insn; - v0->data.insn_d = dst_insn; + set_insn_dst(env, load_const_insn, dst_insn); + // bpf_ir_change_value(env, insn, v0, bpf_ir_value_insn(dst_insn)); } else if (t0 == CONST && t1 == REG) { // reg1 = add const reg2 // ==> @@ -1174,15 +1280,83 @@ static void normalize_alu(struct bpf_ir_env *env, struct ir_function *fun, // reg1 = add reg1 reg2 struct ir_insn *load_const_insn = normalize_load_const(env, insn, v0); - insn_cg(load_const_insn)->dst = dst_insn; - v0->data.insn_d = dst_insn; + set_insn_dst(env, load_const_insn, dst_insn); + // bpf_ir_change_value(env, insn, v0, bpf_ir_value_insn(dst_insn)); } else { CRITICAL_DUMP(env, "Error"); } } -static void normalize_assign(struct ir_insn *insn) +static void normalize_getelemptr(struct bpf_ir_env *env, + struct ir_function *fun, struct ir_insn *insn) +{ + struct ir_value *v0 = &insn->values[0]; + struct ir_value *v1 = &insn->values[1]; + struct ir_value new_v1; + enum val_type t0 = insn->value_num >= 1 ? vtype(*v0) : UNDEF; + enum val_type t1 = insn->value_num >= 2 ? vtype(*v1) : UNDEF; + enum val_type tdst = vtype_insn(insn); + struct ir_insn *dst_insn = insn_dst(insn); + DBGASSERT(tdst == REG); + DBGASSERT(t1 == STACKOFF); + DBGASSERT(v1->type == IR_VALUE_INSN && + v1->data.insn_d->op == IR_INSN_ALLOCARRAY); + struct ir_insn_cg_extra *v1_extra = insn_cg(v1->data.insn_d); + s32 spill_pos = v1_extra->spilled; + insn->op = IR_INSN_ADD; + new_v1.type = IR_VALUE_CONSTANT; + new_v1.const_type = IR_ALU_32; + insn->alu_op = IR_ALU_64; + if (t0 == CONST) { + // reg = getelemptr const ptr + // ==> + // reg = r10 + (const + spill_pos) + DBGASSERT(v0->const_type == IR_ALU_32); + new_v1.data.constant_d = + v0->data.constant_d + spill_pos; // Assume no overflow + bpf_ir_change_value(env, insn, v0, bpf_ir_value_insn(fun->sp)); + bpf_ir_change_value(env, insn, v1, new_v1); + normalize_alu(env, fun, insn); + } + if (t0 == REG) { + new_v1.data.constant_d = spill_pos; + bpf_ir_change_value(env, insn, v1, new_v1); + if (allocated_reg(*v0) == allocated_reg_insn(dst_insn)) { + // reg = getelemptr reg ptr + // ==> + // reg += r10 + // reg += spill_pos + bpf_ir_change_value(env, insn, v0, + bpf_ir_value_insn(dst_insn)); + struct ir_insn *new_insn = bpf_ir_create_bin_insn_cg( + env, insn, bpf_ir_value_insn(dst_insn), + bpf_ir_value_insn(fun->sp), IR_INSN_ADD, + IR_ALU_64, INSERT_FRONT); + set_insn_dst(env, new_insn, dst_insn); + } else { + // reg1 = getelemptr reg2 ptr + // ==> + // reg1 = reg2 + // reg1 += r10 + // reg1 += spill_pos + struct ir_insn *assign_insn = + bpf_ir_create_assign_insn_cg(env, insn, *v0, + INSERT_FRONT); + set_insn_dst(env, assign_insn, dst_insn); + struct ir_insn *alu_insn = bpf_ir_create_bin_insn_cg( + env, insn, bpf_ir_value_insn(dst_insn), + bpf_ir_value_insn(fun->sp), IR_INSN_ADD, + IR_ALU_64, INSERT_FRONT); + set_insn_dst(env, alu_insn, dst_insn); + bpf_ir_change_value(env, insn, v0, + bpf_ir_value_insn(dst_insn)); + } + } +} + +static void normalize_assign(struct bpf_ir_env *env, struct ir_function *fun, + struct ir_insn *insn) { struct ir_value *v0 = &insn->values[0]; enum val_type t0 = insn->value_num >= 1 ? vtype(*v0) : UNDEF; @@ -1198,15 +1372,18 @@ static void normalize_assign(struct ir_insn *insn) DBGASSERT(t0 != STACK); // Change to STORERAW insn->op = IR_INSN_STORERAW; - insn->addr_val.value = bpf_ir_value_stack_ptr(); - insn->addr_val.offset = -insn_cg(dst_insn)->spilled * 8; + + bpf_ir_change_value(env, insn, &insn->addr_val.value, + bpf_ir_value_stack_ptr(fun)); + insn->addr_val.offset = insn_cg(dst_insn)->spilled; } else { if (t0 == STACK) { // Change to LOADRAW insn->op = IR_INSN_LOADRAW; - insn->addr_val.value = bpf_ir_value_stack_ptr(); + bpf_ir_change_value(env, insn, &insn->addr_val.value, + bpf_ir_value_stack_ptr(fun)); insn->addr_val.offset = - -insn_cg(v0->data.insn_d)->spilled * 8; + insn_cg(v0->data.insn_d)->spilled; } if (t0 == CONST && v0->const_type == IR_ALU_64) { // 64 imm load @@ -1217,44 +1394,65 @@ static void normalize_assign(struct ir_insn *insn) } } +static void normalize_stackoff(struct bpf_ir_env *env, struct ir_function *fun, + struct ir_insn *insn) +{ + // Stack already shifted + struct ir_value addrval = insn->addr_val.value; + enum val_type addr_ty = vtype(addrval); + // storeraw STACKOFF ? + // ==> + // storeraw r10 ? + if (addr_ty == STACKOFF) { + insn->addr_val.offset += insn_cg(addrval.data.insn_d)->spilled; + bpf_ir_change_value(env, insn, &insn->addr_val.value, + bpf_ir_value_stack_ptr(fun)); + } +} + static void normalize(struct bpf_ir_env *env, struct ir_function *fun) { struct ir_basic_block **pos; array_for(pos, fun->reachable_bbs) { struct ir_basic_block *bb = *pos; - struct ir_insn *insn; - list_for_each_entry(insn, &bb->ir_insn_head, list_ptr) { + struct ir_insn *insn, *tmp; + list_for_each_entry_safe(insn, tmp, &bb->ir_insn_head, + list_ptr) { if (insn->op == IR_INSN_ALLOC) { // OK + } else if (insn->op == IR_INSN_ALLOCARRAY) { + // OK + } else if (insn->op == IR_INSN_GETELEMPTR) { + normalize_getelemptr(env, fun, insn); } else if (insn->op == IR_INSN_STORE) { // Should be converted to ASSIGN CRITICAL("Error"); } else if (insn->op == IR_INSN_LOAD) { CRITICAL("Error"); } else if (insn->op == IR_INSN_LOADRAW) { - // OK + normalize_stackoff(env, fun, insn); } else if (insn->op == IR_INSN_LOADIMM_EXTRA) { // OK } else if (insn->op == IR_INSN_STORERAW) { - // OK - } else if (is_alu(insn)) { + normalize_stackoff(env, fun, insn); + } else if (bpf_ir_is_alu(insn)) { normalize_alu(env, fun, insn); } else if (insn->op == IR_INSN_ASSIGN) { - normalize_assign(insn); + normalize_assign(env, fun, insn); } else if (insn->op == IR_INSN_RET) { // OK } else if (insn->op == IR_INSN_CALL) { // OK } else if (insn->op == IR_INSN_JA) { // OK - } else if (is_cond_jmp(insn)) { + } else if (bpf_ir_is_cond_jmp(insn)) { // jmp reg const/reg // or // jmp const/reg reg // OK } else { - CRITICAL("No such instruction"); + RAISE_ERROR("No such instruction"); } } } @@ -1309,14 +1507,15 @@ static bool spill_store(struct bpf_ir_env *env, struct ir_function *fun, struct ir_value *v0 = &insn->values[0]; struct ir_value *v1 = &insn->values[1]; // store v0(dst) v1 - // Eequivalent to `v0 = v1` - struct ir_insn_cg_extra *extra = insn_cg(insn); + // Equivalent to `v0 = v1` insn->op = IR_INSN_ASSIGN; DBGASSERT(v0->type == IR_VALUE_INSN); // Should be guaranteed by prog_check DBGASSERT(v0->data.insn_d->op == IR_INSN_ALLOC); insn->vr_type = v0->data.insn_d->vr_type; - extra->dst = v0->data.insn_d; + DBGASSERT(insn_cg(insn)->dst.type == IR_VALUE_UNDEF); + bpf_ir_val_remove_user(*v0, insn); + set_insn_dst(env, insn, v0->data.insn_d); insn->value_num = 1; *v0 = *v1; return spill_assign(env, fun, insn); @@ -1343,7 +1542,6 @@ static bool spill_loadraw(struct bpf_ir_env *env, struct ir_function *fun, { enum val_type tdst = vtype_insn(insn); enum val_type t0 = vtype(insn->addr_val.value); - struct ir_insn_cg_extra *extra = insn_cg(insn); struct ir_insn *dst_insn = insn_dst(insn); // Load from memory // reg = loadraw reg ==> OK @@ -1354,22 +1552,22 @@ static bool spill_loadraw(struct bpf_ir_env *env, struct ir_function *fun, if (tdst == STACK) { if (t0 == REG) { - extra->dst = fun->cg_info.regs[0]; - struct ir_insn *tmp = create_assign_insn_cg( + set_insn_dst(env, insn, fun->cg_info.regs[0]); + struct ir_insn *tmp = bpf_ir_create_assign_insn_cg( env, insn, bpf_ir_value_insn(fun->cg_info.regs[0]), INSERT_BACK); - insn_cg(tmp)->dst = dst_insn; + set_insn_dst(env, tmp, dst_insn); tmp->vr_type = insn->vr_type; return true; } if (t0 == STACK) { - extra->dst = fun->cg_info.regs[0]; - struct ir_insn *tmp = create_assign_insn_cg( + set_insn_dst(env, insn, fun->cg_info.regs[0]); + struct ir_insn *tmp = bpf_ir_create_assign_insn_cg( env, insn, bpf_ir_value_insn(fun->cg_info.regs[0]), INSERT_BACK); - insn_cg(tmp)->dst = dst_insn; + set_insn_dst(env, tmp, dst_insn); tmp->vr_type = insn->vr_type; cgir_load_stack_to_reg(env, fun, insn, &insn->addr_val.value, @@ -1389,7 +1587,6 @@ static bool spill_loadrawextra(struct bpf_ir_env *env, struct ir_function *fun, struct ir_insn *insn) { enum val_type tdst = vtype_insn(insn); - struct ir_insn_cg_extra *extra = insn_cg(insn); struct ir_insn *dst_insn = insn_dst(insn); // IMM64 Map instructions, must load to register if (tdst == STACK) { @@ -1397,11 +1594,11 @@ static bool spill_loadrawextra(struct bpf_ir_env *env, struct ir_function *fun, // ==> // R0 = loadimm // stack = R0 - struct ir_insn *new_insn = create_assign_insn_cg( + struct ir_insn *new_insn = bpf_ir_create_assign_insn_cg( env, insn, bpf_ir_value_insn(fun->cg_info.regs[0]), INSERT_BACK); - insn_cg(new_insn)->dst = dst_insn; - extra->dst = fun->cg_info.regs[0]; + set_insn_dst(env, new_insn, dst_insn); + set_insn_dst(env, insn, fun->cg_info.regs[0]); return true; } return false; @@ -1422,7 +1619,7 @@ static bool spill_storeraw(struct bpf_ir_env *env, struct ir_function *fun, if (t0 == STACK && addr_ty == STACK) { CRITICAL("TODO!"); } - if (t0 == CONST && v0->const_type == IR_VR_TYPE_64) { + if (t0 == CONST && v0->const_type == IR_ALU_64) { CRITICAL("Not supported"); } // Question: are all memory address 64 bits? @@ -1433,7 +1630,7 @@ static bool spill_storeraw(struct bpf_ir_env *env, struct ir_function *fun, // Make sure the new register is not the same as the other register reg = 1; } - cgir_load_stack_to_reg(env, fun, insn, v0, IR_VR_TYPE_64, 0); + cgir_load_stack_to_reg(env, fun, insn, v0, IR_VR_TYPE_64, reg); return true; } if (addr_ty == CONST) { @@ -1459,7 +1656,6 @@ static bool spill_alu(struct bpf_ir_env *env, struct ir_function *fun, enum val_type t0 = insn->value_num >= 1 ? vtype(*v0) : UNDEF; enum val_type t1 = insn->value_num >= 2 ? vtype(*v1) : UNDEF; enum val_type tdst = vtype_insn(insn); - struct ir_insn_cg_extra *extra = insn_cg(insn); struct ir_insn *dst_insn = insn_dst(insn); // Binary ALU // reg = ALU reg reg @@ -1470,12 +1666,11 @@ static bool spill_alu(struct bpf_ir_env *env, struct ir_function *fun, // ==> // R0 = ALU ? ? // stack = R0 - extra->dst = fun->cg_info.regs[0]; - struct ir_insn *tmp = create_assign_insn_cg( + set_insn_dst(env, insn, fun->cg_info.regs[0]); + struct ir_insn *tmp = bpf_ir_create_assign_insn_cg( env, insn, bpf_ir_value_insn(fun->cg_info.regs[0]), INSERT_BACK); - // tmp->vr_type = alu_to_vr_type(insn->alu_op); - insn_cg(tmp)->dst = dst_insn; + set_insn_dst(env, tmp, dst_insn); spill_alu(env, fun, insn); return true; } @@ -1628,69 +1823,181 @@ static bool spill_cond_jump(struct bpf_ir_env *env, struct ir_function *fun, return false; } -static void check_cgir(struct bpf_ir_env *env, struct ir_function *fun) +static bool spill_getelemptr(struct bpf_ir_env *env, struct ir_function *fun, + struct ir_insn *insn) { - // Sanity check of CGIR (the IR after `check_need_spill`) - print_ir_err_init(fun); - struct ir_basic_block **pos; - array_for(pos, fun->reachable_bbs) + struct ir_value *v0 = &insn->values[0]; + struct ir_value *v1 = &insn->values[1]; + enum val_type t0 = insn->value_num >= 1 ? vtype(*v0) : UNDEF; + enum val_type t1 = insn->value_num >= 2 ? vtype(*v1) : UNDEF; + enum val_type tdst = vtype_insn(insn); + struct ir_insn *dst_insn = insn_dst(insn); + ASSERT_DUMP(v1->type == IR_VALUE_INSN, false); + ASSERT_DUMP(v1->data.insn_d->op == IR_INSN_ALLOCARRAY, false); + ASSERT_DUMP(t1 == STACKOFF, false); + if (tdst == STACK) { + set_insn_dst(env, insn, fun->cg_info.regs[0]); + struct ir_insn *tmp = bpf_ir_create_assign_insn_cg( + env, insn, bpf_ir_value_insn(fun->cg_info.regs[0]), + INSERT_BACK); + set_insn_dst(env, tmp, dst_insn); + spill_getelemptr(env, fun, insn); + return true; + } + if (t0 == STACK) { + cgir_load_stack_to_reg(env, fun, insn, v0, IR_VR_TYPE_64, 0); + return true; + } + if (t0 == CONST && v0->const_type == IR_ALU_64) { + cgir_load_const_to_reg(env, fun, insn, v0, 0); + return true; + } + return false; +} + +static void check_insn_users_use_insn_cg(struct bpf_ir_env *env, + struct ir_insn *insn) +{ + struct ir_insn **pos; + array_for(pos, insn->users) { - struct ir_basic_block *bb = *pos; - struct ir_insn *insn; - list_for_each_entry(insn, &bb->ir_insn_head, list_ptr) { - // First check insn - if (insn_dst(insn) == NULL) { - continue; + struct ir_insn *user = *pos; + // Check if the user actually uses this instruction + struct array operands = bpf_ir_get_operands_and_dst(env, user); + struct ir_value **val; + int found = 0; + array_for(val, operands) + { + struct ir_value *v = *val; + if (v->type == IR_VALUE_INSN && + v->data.insn_d == insn) { + // Found the user + found = 1; + break; + } + } + bpf_ir_array_free(&operands); + if (!found) { + // Error! + if (!insn_cg(insn)->nonvr) { + print_ir_insn_err_full(env, insn, + "The instruction", + print_ir_dst); + } else { + PRINT_LOG(env, "The instruction is non-vr.\n"); } - if (insn_dst(insn) != insn_dst(insn_dst(insn))) { - // dst(insn) != dst(dst(insn)) - // dst is not final! - print_ir_insn_err_full(env, insn_dst(insn), - "Instruction's dst", + print_ir_insn_err_full(env, user, + "The user of that instruction", + print_ir_dst); + RAISE_ERROR("User does not use the instruction"); + } + } +} + +static void check_insn_operand_cg(struct bpf_ir_env *env, struct ir_insn *insn) +{ + struct array operands = bpf_ir_get_operands_and_dst(env, insn); + struct ir_value **val; + array_for(val, operands) + { + struct ir_value *v = *val; + if (v->type == IR_VALUE_INSN) { + // Check if the operand actually is used by this instruction + struct ir_insn **pos2; + int found = 0; + array_for(pos2, v->data.insn_d->users) + { + struct ir_insn *user = *pos2; + if (user == insn) { + // Found the user + found = 1; + break; + } + } + if (!found) { + // Error! + print_ir_insn_err_full(env, v->data.insn_d, + "Operand defined here", print_ir_dst); print_ir_insn_err_full( env, insn, - "This instruction's dst is not final!", + "Instruction that uses the operand", print_ir_dst); - CRITICAL_DUMP(env, "Dst is not final!"); + RAISE_ERROR( + "Instruction not found in the operand's users"); } - struct array operands = bpf_ir_get_operands(env, insn); - struct ir_value **vpos; - array_for(vpos, operands) - { - struct ir_value *val = *vpos; - if (val->type == IR_VALUE_INSN) { - struct ir_insn *dst_insn = - val->data.insn_d; - if (insn_dst(dst_insn) == NULL) { - print_ir_insn_err_full( - env, dst_insn, - "Operand's dst is NULL", - print_ir_dst); - print_ir_insn_err_full( - env, insn, - "This instruction's operand's dst is NULL", - print_ir_dst); - CRITICAL_DUMP(env, "NULL dst"); - } - if (insn_dst(dst_insn) != dst_insn) { - print_ir_insn_err_full( - env, insn_dst(dst_insn), - "Operand's dst", - print_ir_dst); - print_ir_insn_err_full( - env, dst_insn, - "Operand", - print_ir_dst); - print_ir_insn_err_full( - env, insn, - "This instruction's operand's dst is NULL", - print_ir_dst); - CRITICAL_DUMP(env, "NULL dst"); - } + + // Check dst + + struct ir_insn *dst_insn = v->data.insn_d; + if (insn_dst(dst_insn) == NULL) { + print_ir_insn_err_full(env, dst_insn, + "Operand's dst is NULL", + print_ir_dst); + print_ir_insn_err_full( + env, insn, + "This instruction's operand's dst is NULL", + print_ir_dst); + RAISE_ERROR("NULL dst"); + } + if (insn_dst(dst_insn) != dst_insn) { + print_ir_insn_err_full(env, insn_dst(dst_insn), + "Operand's dst", + print_ir_dst); + print_ir_insn_err_full(env, dst_insn, "Operand", + print_ir_dst); + print_ir_insn_err_full( + env, insn, + "This instruction's operand's dst is NULL", + print_ir_dst); + RAISE_ERROR("NULL dst"); + } + } + } + bpf_ir_array_free(&operands); +} + +static void prog_check_cg(struct bpf_ir_env *env, struct ir_function *fun) +{ + // CG IR check + // Available to run while dst is maintained + + print_ir_err_init(fun); + + check_insn_users_use_insn_cg(env, fun->sp); + for (u8 i = 0; i < BPF_REG_10; ++i) { + check_insn_users_use_insn_cg(env, fun->cg_info.regs[i]); + } + struct ir_basic_block **pos; + array_for(pos, fun->reachable_bbs) + { + struct ir_basic_block *bb = *pos; + struct ir_insn *insn; + list_for_each_entry(insn, &bb->ir_insn_head, list_ptr) { + // Check dst + if (insn_cg(insn)->dst.type == IR_VALUE_INSN) { + // Check users of this instruction + check_insn_users_use_insn_cg(env, insn); + } else { + if (insn_cg(insn)->dst.type != IR_VALUE_UNDEF) { + print_ir_insn_err_full(env, insn, + "Instruction", + print_ir_dst); + RAISE_ERROR( + "Instruction's dst is incorrect value"); + } + // dst == NULL + // There should be no users! + if (insn->users.num_elem > 0) { + print_ir_insn_err_full(env, insn, + "Instruction", + print_ir_dst); + RAISE_ERROR( + "NULL dst Instruction has users"); } } - bpf_ir_array_free(&operands); + // Check operands of this instruction + check_insn_operand_cg(env, insn); } } } @@ -1703,26 +2010,31 @@ static bool check_need_spill(struct bpf_ir_env *env, struct ir_function *fun) array_for(pos, fun->reachable_bbs) { struct ir_basic_block *bb = *pos; - struct ir_insn *insn; - list_for_each_entry(insn, &bb->ir_insn_head, list_ptr) { + struct ir_insn *insn, *tmp; + list_for_each_entry_safe(insn, tmp, &bb->ir_insn_head, + list_ptr) { if (insn->op == IR_INSN_ALLOC) { // dst = alloc // Nothing to do + } else if (insn->op == IR_INSN_ALLOCARRAY) { + // Nothing to do + } else if (insn->op == IR_INSN_GETELEMPTR) { + need_modify |= spill_getelemptr(env, fun, insn); } else if (insn->op == IR_INSN_STORE) { - need_modify = spill_store(env, fun, insn); + need_modify |= spill_store(env, fun, insn); } else if (insn->op == IR_INSN_LOAD) { - need_modify = spill_load(env, fun, insn); + need_modify |= spill_load(env, fun, insn); } else if (insn->op == IR_INSN_LOADRAW) { - need_modify = spill_loadraw(env, fun, insn); + need_modify |= spill_loadraw(env, fun, insn); } else if (insn->op == IR_INSN_LOADIMM_EXTRA) { - need_modify = + need_modify |= spill_loadrawextra(env, fun, insn); } else if (insn->op == IR_INSN_STORERAW) { - need_modify = spill_storeraw(env, fun, insn); - } else if (is_alu(insn)) { - need_modify = spill_alu(env, fun, insn); + need_modify |= spill_storeraw(env, fun, insn); + } else if (bpf_ir_is_alu(insn)) { + need_modify |= spill_alu(env, fun, insn); } else if (insn->op == IR_INSN_ASSIGN) { - need_modify = spill_assign(env, fun, insn); + need_modify |= spill_assign(env, fun, insn); } else if (insn->op == IR_INSN_RET) { // ret const/reg // Done in explicit_reg pass @@ -1733,11 +2045,12 @@ static bool check_need_spill(struct bpf_ir_env *env, struct ir_function *fun) DBGASSERT(insn->value_num == 0); } else if (insn->op == IR_INSN_JA) { // OK - } else if (is_cond_jmp(insn)) { - need_modify = spill_cond_jump(env, fun, insn); + } else if (bpf_ir_is_cond_jmp(insn)) { + need_modify |= spill_cond_jump(env, fun, insn); } else { - CRITICAL("No such instruction"); + RAISE_ERROR_RET("No such instruction", false); } + CHECK_ERR(false); } } return need_modify; @@ -1765,32 +2078,32 @@ static void calc_callee_num(struct ir_function *fun) static void calc_stack_size(struct ir_function *fun) { // Check callee - size_t off = 0; + s32 off = 0; if (fun->cg_info.spill_callee) { - off += fun->cg_info.callee_num * 8; + off -= fun->cg_info.callee_num * 8; } // Check all VR - size_t max = 0; + s32 max = 0; struct ir_insn **pos; array_for(pos, fun->cg_info.all_var) { struct ir_insn_cg_extra *extra = insn_cg(*pos); - if (extra->spilled > 0) { + if (extra->spilled) { // Spilled! - if (extra->spilled > max) { + if (extra->spilled < max) { max = extra->spilled; } } } - fun->cg_info.stack_offset = -(off + max * 8); + fun->cg_info.stack_offset = off + max; PRINT_DBG("Stack size: %d\n", fun->cg_info.stack_offset); } +// Pre CG static void add_stack_offset_pre_cg(struct bpf_ir_env *env, struct ir_function *fun) { - // Pre CG - struct array users = fun->sp_users; + struct array users = fun->sp->users; struct ir_insn **pos; array_for(pos, users) { @@ -1807,18 +2120,24 @@ static void add_stack_offset_pre_cg(struct bpf_ir_env *env, array_for(pos2, value_uses) { struct ir_value *val = *pos2; - if (val->type == IR_VALUE_STACK_PTR) { + if (val->type == IR_VALUE_INSN && + val->data.insn_d == fun->sp) { // Stack pointer as value struct ir_value new_val; new_val.type = IR_VALUE_CONSTANT_RAWOFF; new_val.const_type = IR_ALU_32; - struct ir_insn *new_insn = create_bin_insn( - env, insn, *val, new_val, IR_INSN_ADD, - IR_ALU_64, INSERT_FRONT); - new_val.type = IR_VALUE_INSN; - new_val.data.insn_d = new_insn; - val_add_user(env, new_val, insn); - *val = new_val; + new_val.data.constant_d = 0; + // tmp = SP + hole + // ... val ==> tmp + struct ir_insn *new_insn = + bpf_ir_create_bin_insn(env, insn, *val, + new_val, + IR_INSN_ADD, + IR_ALU_64, + INSERT_FRONT); + bpf_ir_change_value( + env, insn, val, + bpf_ir_value_insn(new_insn)); } } bpf_ir_array_free(&value_uses); @@ -1828,41 +2147,49 @@ static void add_stack_offset_pre_cg(struct bpf_ir_env *env, static void add_stack_offset(struct bpf_ir_env *env, struct ir_function *fun, s16 offset) { - struct array users = fun->sp_users; - struct ir_insn **pos; - array_for(pos, users) + struct ir_basic_block **pos; + // For each BB + array_for(pos, fun->reachable_bbs) { - struct ir_insn *insn = *pos; - - if (insn->op == IR_INSN_LOADRAW || - insn->op == IR_INSN_STORERAW) { - if (insn->addr_val.value.type == IR_VALUE_STACK_PTR) { - insn->addr_val.offset += offset; - continue; + struct ir_basic_block *bb = *pos; + struct ir_insn *insn; + // For each operation + list_for_each_entry(insn, &bb->ir_insn_head, list_ptr) { + if (insn->op == IR_INSN_LOADRAW || + insn->op == IR_INSN_STORERAW) { + if (insn->addr_val.value.type == + IR_VALUE_INSN && + insn->addr_val.value.data.insn_d == + fun->sp) { + insn->addr_val.offset += offset; + continue; + } } - } - struct array value_uses = bpf_ir_get_operands(env, insn); - struct ir_value **pos2; - array_for(pos2, value_uses) - { - struct ir_value *val = *pos2; - DBGASSERT(val->type != IR_VALUE_STACK_PTR); - if (val->type == IR_VALUE_CONSTANT_RAWOFF) { - // Stack pointer as value - val->data.constant_d = offset; + struct array value_uses = + bpf_ir_get_operands(env, insn); + struct ir_value **pos2; + array_for(pos2, value_uses) + { + struct ir_value *val = *pos2; + if (val->type == IR_VALUE_CONSTANT_RAWOFF) { + // Stack pointer as value + val->data.constant_d = offset; + val->type = IR_VALUE_CONSTANT; + } } + bpf_ir_array_free(&value_uses); } - bpf_ir_array_free(&value_uses); } } static struct pre_ir_insn translate_reg_to_reg(u8 dst, u8 src) { // MOV dst src - struct pre_ir_insn insn; + struct pre_ir_insn insn = { 0 }; insn.opcode = BPF_MOV | BPF_X | BPF_ALU64; insn.dst_reg = dst; insn.src_reg = src; + insn.imm = 0; return insn; } @@ -1870,7 +2197,7 @@ static struct pre_ir_insn translate_const_to_reg(u8 dst, s64 data, enum ir_alu_op_type type) { // MOV dst imm - struct pre_ir_insn insn; + struct pre_ir_insn insn = { 0 }; insn.dst_reg = dst; if (type == IR_ALU_32) { insn.opcode = BPF_MOV | BPF_K | BPF_ALU; @@ -1878,7 +2205,6 @@ static struct pre_ir_insn translate_const_to_reg(u8 dst, s64 data, // Default is imm64 insn.opcode = BPF_MOV | BPF_K | BPF_ALU64; } - insn.it = IMM; insn.imm = data; return insn; } @@ -1903,14 +2229,11 @@ static struct pre_ir_insn load_addr_to_reg(u8 dst, struct ir_address_value addr, enum ir_vr_type type) { // MOV dst src - struct pre_ir_insn insn; + struct pre_ir_insn insn = { 0 }; insn.dst_reg = dst; insn.off = addr.offset; int size = vr_type_to_size(type); - if (addr.value.type == IR_VALUE_STACK_PTR) { - insn.src_reg = BPF_REG_10; - insn.opcode = BPF_LDX | size | BPF_MEM; - } else if (addr.value.type == IR_VALUE_INSN) { + if (addr.value.type == IR_VALUE_INSN) { // Must be REG DBGASSERT(vtype(addr.value) == REG); // Load reg (addr) to reg @@ -1920,7 +2243,10 @@ static struct pre_ir_insn load_addr_to_reg(u8 dst, struct ir_address_value addr, // Must be U64 insn.it = IMM64; insn.imm64 = addr.value.data.constant_d; - insn.opcode = BPF_IMM | size | BPF_LD; + insn.opcode = size; + // Simplify the opcode to reduce compiler warning, the real opcode is as follows + // (but BPF_MM and BPF_LD are all 0) + // insn.opcode = BPF_IMM | size | BPF_LD; } else { CRITICAL("Error"); } @@ -1930,7 +2256,7 @@ static struct pre_ir_insn load_addr_to_reg(u8 dst, struct ir_address_value addr, static struct pre_ir_insn store_reg_to_reg_mem(u8 dst, u8 src, s16 offset, enum ir_vr_type type) { - struct pre_ir_insn insn; + struct pre_ir_insn insn = { 0 }; int size = vr_type_to_size(type); insn.src_reg = src; insn.off = offset; @@ -1942,7 +2268,7 @@ static struct pre_ir_insn store_reg_to_reg_mem(u8 dst, u8 src, s16 offset, static struct pre_ir_insn store_const_to_reg_mem(u8 dst, s64 val, s16 offset, enum ir_vr_type type) { - struct pre_ir_insn insn; + struct pre_ir_insn insn = { 0 }; int size = vr_type_to_size(type); insn.it = IMM; insn.imm = val; @@ -1995,7 +2321,7 @@ static int jmp_code(enum ir_insn_type insn) static struct pre_ir_insn alu_reg(u8 dst, u8 src, enum ir_alu_op_type type, int opcode) { - struct pre_ir_insn insn; + struct pre_ir_insn insn = { 0 }; insn.dst_reg = dst; insn.src_reg = src; int alu_class = type == IR_ALU_64 ? BPF_ALU64 : BPF_ALU; @@ -2006,9 +2332,8 @@ static struct pre_ir_insn alu_reg(u8 dst, u8 src, enum ir_alu_op_type type, static struct pre_ir_insn alu_imm(u8 dst, s64 src, enum ir_alu_op_type type, int opcode) { - struct pre_ir_insn insn; + struct pre_ir_insn insn = { 0 }; insn.dst_reg = dst; - insn.src_reg = src; int alu_class = type == IR_ALU_64 ? BPF_ALU64 : BPF_ALU; insn.it = IMM; insn.imm = src; @@ -2019,7 +2344,7 @@ static struct pre_ir_insn alu_imm(u8 dst, s64 src, enum ir_alu_op_type type, static struct pre_ir_insn cond_jmp_reg(u8 dst, u8 src, enum ir_alu_op_type type, int opcode) { - struct pre_ir_insn insn; + struct pre_ir_insn insn = { 0 }; insn.dst_reg = dst; insn.src_reg = src; int alu_class = type == IR_ALU_64 ? BPF_JMP : BPF_JMP32; @@ -2030,9 +2355,8 @@ static struct pre_ir_insn cond_jmp_reg(u8 dst, u8 src, enum ir_alu_op_type type, static struct pre_ir_insn cond_jmp_imm(u8 dst, s64 src, enum ir_alu_op_type type, int opcode) { - struct pre_ir_insn insn; + struct pre_ir_insn insn = { 0 }; insn.dst_reg = dst; - insn.src_reg = src; int alu_class = type == IR_ALU_64 ? BPF_JMP : BPF_JMP32; insn.it = IMM; insn.imm = src; @@ -2083,20 +2407,7 @@ static void translate_storeraw(struct ir_insn *insn) enum val_type t0 = insn->value_num >= 1 ? vtype(v0) : UNDEF; struct ir_insn_cg_extra *extra = insn_cg(insn); // storeraw - if (insn->addr_val.value.type == IR_VALUE_STACK_PTR) { - // Store value in the stack - if (t0 == REG) { - extra->translated[0] = store_reg_to_reg_mem( - BPF_REG_10, get_alloc_reg(v0.data.insn_d), - insn->addr_val.offset, insn->vr_type); - } else if (t0 == CONST) { - extra->translated[0] = store_const_to_reg_mem( - BPF_REG_10, v0.data.constant_d, - insn->addr_val.offset, insn->vr_type); - } else { - CRITICAL("Error"); - } - } else if (insn->addr_val.value.type == IR_VALUE_INSN) { + if (insn->addr_val.value.type == IR_VALUE_INSN) { // Store value in (address in the value) DBGASSERT(vtype(insn->addr_val.value) == REG); // Store value in the stack @@ -2136,6 +2447,11 @@ static void translate_alu(struct ir_insn *insn) insn->alu_op, alu_code(insn->op)); } else if (t1 == CONST) { + // Remove the instruction in some special cases + if (insn->op == IR_INSN_ADD && v1.data.constant_d == 0) { + extra->translated_num = 0; + return; + } extra->translated[0] = alu_imm(get_alloc_reg(dst_insn), v1.data.constant_d, insn->alu_op, alu_code(insn->op)); @@ -2159,6 +2475,11 @@ static void translate_assign(struct ir_insn *insn) get_alloc_reg(dst_insn), v0.data.constant_d, insn->alu_op); } else if (tdst == REG && t0 == REG) { + if (get_alloc_reg(dst_insn) == get_alloc_reg(v0.data.insn_d)) { + // Remove the instruction + extra->translated_num = 0; + return; + } extra->translated[0] = translate_reg_to_reg( get_alloc_reg(dst_insn), get_alloc_reg(v0.data.insn_d)); } else { @@ -2223,30 +2544,36 @@ static void translate_cond_jmp(struct ir_insn *insn) } } -static void translate(struct ir_function *fun) +static void translate(struct bpf_ir_env *env, struct ir_function *fun) { struct ir_basic_block **pos; array_for(pos, fun->reachable_bbs) { struct ir_basic_block *bb = *pos; - struct ir_insn *insn; - list_for_each_entry(insn, &bb->ir_insn_head, list_ptr) { + struct ir_insn *insn, *tmp; + list_for_each_entry_safe(insn, tmp, &bb->ir_insn_head, + list_ptr) { struct ir_insn_cg_extra *extra = insn_cg(insn); extra->translated_num = 1; // Default: 1 instruction if (insn->op == IR_INSN_ALLOC) { // Nothing to do extra->translated_num = 0; + } else if (insn->op == IR_INSN_ALLOCARRAY) { + // Nothing to do + extra->translated_num = 0; } else if (insn->op == IR_INSN_STORE) { CRITICAL("Error"); } else if (insn->op == IR_INSN_LOAD) { CRITICAL("Error"); + } else if (insn->op == IR_INSN_GETELEMPTR) { + CRITICAL("Error"); } else if (insn->op == IR_INSN_LOADRAW) { translate_loadraw(insn); } else if (insn->op == IR_INSN_LOADIMM_EXTRA) { translate_loadimm_extra(insn); } else if (insn->op == IR_INSN_STORERAW) { translate_storeraw(insn); - } else if (is_alu(insn)) { + } else if (bpf_ir_is_alu(insn)) { translate_alu(insn); } else if (insn->op == IR_INSN_ASSIGN) { translate_assign(insn); @@ -2256,10 +2583,57 @@ static void translate(struct ir_function *fun) translate_call(insn); } else if (insn->op == IR_INSN_JA) { translate_ja(insn); - } else if (is_cond_jmp(insn)) { + } else if (bpf_ir_is_cond_jmp(insn)) { translate_cond_jmp(insn); } else { - CRITICAL("No such instruction"); + RAISE_ERROR("No such instruction"); + } + } + } +} + +static u32 sizeof_vr_type(enum ir_vr_type type) +{ + switch (type) { + case IR_VR_TYPE_32: + return 4; + case IR_VR_TYPE_16: + return 2; + case IR_VR_TYPE_8: + return 1; + case IR_VR_TYPE_64: + return 8; + default: + CRITICAL("Error"); + } +} + +// Spill all `allocarray` instructions +static void spill_array(struct bpf_ir_env *env, struct ir_function *fun) +{ + u32 offset = 0; + struct ir_basic_block **pos; + array_for(pos, fun->reachable_bbs) + { + struct ir_basic_block *bb = *pos; + struct ir_insn *insn, *tmp; + list_for_each_entry_safe(insn, tmp, &bb->ir_insn_head, + list_ptr) { + if (insn->op == IR_INSN_ALLOCARRAY) { + struct ir_insn_cg_extra *extra = insn_cg(insn); + DBGASSERT(extra->dst.data.insn_d == + insn); // Ensure the dst is correct + extra->allocated = true; + // Calculate the offset + u32 size = insn->array_num * + sizeof_vr_type(insn->vr_type); + if (size == 0) { + RAISE_ERROR("Array size is 0"); + } + offset -= (((size - 1) / 8) + 1) * 8; + extra->spilled = offset; + extra->spilled_size = size; + extra->nonvr = true; // Array is not a VR } } } @@ -2271,6 +2645,23 @@ void bpf_ir_code_gen(struct bpf_ir_env *env, struct ir_function *fun) { // Preparation + bpf_ir_optimize_ir(env, fun); + CHECK_ERR(); + bpf_ir_prog_check(env, fun); + CHECK_ERR(); + print_ir_prog_pre_cg(env, fun, "Optimization"); + CHECK_ERR(); + + change_fun_arg(env, fun); + CHECK_ERR(); + bpf_ir_prog_check(env, fun); + CHECK_ERR(); + + change_call_pre_cg(env, fun); + CHECK_ERR(); + bpf_ir_prog_check(env, fun); + CHECK_ERR(); + // Step 1: Flag all raw stack access add_stack_offset_pre_cg(env, fun); CHECK_ERR(); @@ -2284,24 +2675,45 @@ void bpf_ir_code_gen(struct bpf_ir_env *env, struct ir_function *fun) CHECK_ERR(); print_ir_prog_pre_cg(env, fun, "To CSSA"); + CHECK_ERR(); // Init CG, start real code generation init_cg(env, fun); CHECK_ERR(); + prog_check_cg(env, fun); + CHECK_ERR(); + // Debugging settings fun->cg_info.spill_callee = 0; - // Step 3: Use explicit real registers - explicit_reg(env, fun); // Still in SSA form, users are available - CHECK_ERR(); - print_ir_prog_cg_dst(env, fun, "Explicit REG"); - // Step 4: SSA Destruction - // users not available from now on remove_phi(env, fun); CHECK_ERR(); print_ir_prog_cg_dst(env, fun, "PHI Removal"); + prog_check_cg(env, fun); + CHECK_ERR(); + + // No more users, SSA structure is destroyed + + change_ret(env, fun); + CHECK_ERR(); + prog_check_cg(env, fun); + CHECK_ERR(); + + change_call(env, fun); + CHECK_ERR(); + print_ir_prog_cg_dst(env, fun, "Changing calls"); + CHECK_ERR(); + prog_check_cg(env, fun); + CHECK_ERR(); + + spill_array(env, fun); + CHECK_ERR(); + print_ir_prog_cg_dst(env, fun, "Spilling Arrays"); + CHECK_ERR(); + prog_check_cg(env, fun); + CHECK_ERR(); // print_ir_prog_reachable(fun); @@ -2323,17 +2735,25 @@ void bpf_ir_code_gen(struct bpf_ir_env *env, struct ir_function *fun) // Step 7: Graph coloring graph_coloring(env, fun); CHECK_ERR(); - coaleasing(fun); - CHECK_ERR(); PRINT_LOG(env, "Conflicting graph (after coloring):\n"); bpf_ir_print_interference_graph(env, fun); + CHECK_ERR(); print_ir_prog_cg_alloc(env, fun, "After RA"); + coalescing(env, fun); + CHECK_ERR(); + prog_check_cg(env, fun); + CHECK_ERR(); + print_ir_prog_cg_dst(env, fun, "After Coalescing (dst)"); + print_ir_prog_cg_alloc(env, fun, "After Coalescing (reg)"); + // RAISE_ERROR("success"); + // Step 8: Check if need to spill and spill need_spill = check_need_spill(env, fun); CHECK_ERR(); - check_cgir(env, fun); + prog_check_cg(env, fun); CHECK_ERR(); + // print_ir_prog_cg_dst(env, fun, "After Spilling"); if (need_spill) { // Still need to spill @@ -2344,7 +2764,7 @@ void bpf_ir_code_gen(struct bpf_ir_env *env, struct ir_function *fun) } // Register allocation finished (All registers are fixed) - PRINT_LOG(env, "Register allocation finished in %d iteratinos\n", + PRINT_LOG(env, "Register allocation finished in %d iterations\n", iterations); print_ir_prog_cg_alloc(env, fun, "After RA & Spilling"); @@ -2358,23 +2778,28 @@ void bpf_ir_code_gen(struct bpf_ir_env *env, struct ir_function *fun) add_stack_offset(env, fun, fun->cg_info.stack_offset); CHECK_ERR(); print_ir_prog_cg_alloc(env, fun, "Shifting stack access"); + prog_check_cg(env, fun); + CHECK_ERR(); // Step 11: Spill callee saved registers if (fun->cg_info.spill_callee) { spill_callee(env, fun); CHECK_ERR(); print_ir_prog_cg_alloc(env, fun, "Spilling callee-saved regs"); + prog_check_cg(env, fun); + CHECK_ERR(); } // Step 12: Normalize normalize(env, fun); CHECK_ERR(); print_ir_prog_cg_alloc(env, fun, "Normalization"); - check_cgir(env, fun); + prog_check_cg(env, fun); CHECK_ERR(); // Step 13: Direct Translation - translate(fun); + translate(env, fun); + CHECK_ERR(); // Step 14: Relocation relocate(env, fun); @@ -2392,14 +2817,15 @@ void bpf_ir_init_insn_cg(struct bpf_ir_env *env, struct ir_insn *insn) { struct ir_insn_cg_extra *extra = NULL; SAFE_MALLOC(extra, sizeof(struct ir_insn_cg_extra)); + insn->user_data = extra; // When init, the destination is itself - if (is_void(insn)) { - extra->dst = NULL; - } else { - extra->dst = insn; + extra->dst = bpf_ir_value_undef(); + if (!bpf_ir_is_void(insn)) { + set_insn_dst(env, insn, insn); } + INIT_ARRAY(&extra->adj, struct ir_insn *); - extra->allocated = 0; + extra->allocated = false; extra->spilled = 0; extra->alloc_reg = 0; INIT_ARRAY(&extra->gen, struct ir_insn *); @@ -2408,5 +2834,4 @@ void bpf_ir_init_insn_cg(struct bpf_ir_env *env, struct ir_insn *insn) INIT_ARRAY(&extra->out, struct ir_insn *); extra->translated_num = 0; extra->nonvr = false; - insn->user_data = extra; } diff --git a/IR/ir_helper.c b/IR/ir_helper.c index 6155fe0d..567dc37f 100644 --- a/IR/ir_helper.c +++ b/IR/ir_helper.c @@ -64,11 +64,11 @@ void bpf_ir_clean_id(struct ir_function *fun) void print_insn_ptr_base(struct bpf_ir_env *env, struct ir_insn *insn) { if (insn->op == IR_INSN_REG) { - PRINT_LOG(env, "R%u", insn_cg(insn)->alloc_reg); + PRINT_LOG(env, "R%u", insn->reg_id); return; } if (insn->op == IR_INSN_FUNCTIONARG) { - PRINT_LOG(env, "arg%u", insn->fid); + PRINT_LOG(env, "arg%u", insn->fun_arg_id); return; } if (insn->_insn_id == SIZET_MAX) { @@ -121,9 +121,6 @@ static void print_ir_value_full(struct bpf_ir_env *env, struct ir_value v, case IR_VALUE_INSN: print_insn_ptr(env, v.data.insn_d, print_ir); break; - case IR_VALUE_STACK_PTR: - PRINT_LOG(env, "SP"); - break; case IR_VALUE_CONSTANT: print_const(env, v); break; @@ -250,6 +247,11 @@ void print_ir_insn_full(struct bpf_ir_env *env, struct ir_insn *insn, PRINT_LOG(env, "alloc "); print_vr_type(env, insn->vr_type); break; + case IR_INSN_ALLOCARRAY: + PRINT_LOG(env, "allocarray <"); + print_vr_type(env, insn->vr_type); + PRINT_LOG(env, " x %u>", insn->array_num); + break; case IR_INSN_STORE: PRINT_LOG(env, "store "); print_ir_value_full(env, insn->values[0], print_ir); @@ -279,6 +281,12 @@ void print_ir_insn_full(struct bpf_ir_env *env, struct ir_insn *insn, PRINT_LOG(env, " "); print_ir_value_full(env, insn->values[0], print_ir); break; + case IR_INSN_GETELEMPTR: + PRINT_LOG(env, "getelemptr "); + print_ir_value_full(env, insn->values[1], print_ir); + PRINT_LOG(env, "+"); + print_ir_value_full(env, insn->values[0], print_ir); + break; case IR_INSN_ADD: print_alu(env, insn->alu_op, "add"); print_ir_value_full(env, insn->values[0], print_ir); @@ -436,7 +444,7 @@ void print_ir_bb_no_rec( struct list_head *p = NULL; list_for_each(p, &bb->ir_insn_head) { struct ir_insn *insn = list_entry(p, struct ir_insn, list_ptr); - if (is_void(insn)) { + if (bpf_ir_is_void(insn)) { PRINT_LOG(env, " "); } else { PRINT_LOG(env, " "); @@ -516,7 +524,7 @@ void assign_id(struct ir_basic_block *bb, size_t *cnt, size_t *bb_cnt) struct list_head *p = NULL; list_for_each(p, &bb->ir_insn_head) { struct ir_insn *insn = list_entry(p, struct ir_insn, list_ptr); - if (!is_void(insn)) { + if (!bpf_ir_is_void(insn)) { insn->_insn_id = (*cnt)++; } } @@ -562,6 +570,10 @@ void print_ir_prog_notag(struct bpf_ir_env *env, struct ir_function *fun) void print_ir_dst(struct bpf_ir_env *env, struct ir_insn *insn) { + if (!insn_cg(insn)) { + PRINT_LOG(env, "(?)"); + RAISE_ERROR("NULL userdata found"); + } insn = insn_dst(insn); if (insn) { print_insn_ptr_base(env, insn); @@ -577,12 +589,12 @@ void print_ir_alloc(struct bpf_ir_env *env, struct ir_insn *insn) struct ir_insn_cg_extra *extra = insn_cg(insn); if (extra->allocated) { if (extra->spilled) { - PRINT_LOG(env, "sp-%zu", extra->spilled * 8); + PRINT_LOG(env, "sp+%d", extra->spilled); } else { PRINT_LOG(env, "r%u", extra->alloc_reg); } } else { - CRITICAL_DUMP(env, "Not allocated"); + RAISE_ERROR("Not allocated"); } } else { PRINT_LOG(env, "(NULL)"); @@ -605,11 +617,11 @@ void print_ir_insn_err_full(struct bpf_ir_env *env, struct ir_insn *insn, struct ir_insn *)) { PRINT_LOG(env, "In BB %zu,\n", insn->parent_bb->_id); - struct ir_insn *prev = prev_insn(insn); - struct ir_insn *next = next_insn(insn); + struct ir_insn *prev = bpf_ir_prev_insn(insn); + struct ir_insn *next = bpf_ir_next_insn(insn); if (prev) { PRINT_LOG(env, " "); - if (!is_void(prev)) { + if (!bpf_ir_is_void(prev)) { PRINT_LOG(env, "%%%zu", prev->_insn_id); PRINT_LOG(env, " = "); } @@ -619,7 +631,7 @@ void print_ir_insn_err_full(struct bpf_ir_env *env, struct ir_insn *insn, PRINT_LOG(env, " (No instruction)\n"); } PRINT_LOG(env, " "); - if (!is_void(insn)) { + if (!bpf_ir_is_void(insn)) { PRINT_LOG(env, "%%%zu", insn->_insn_id); PRINT_LOG(env, " = "); } @@ -632,7 +644,7 @@ void print_ir_insn_err_full(struct bpf_ir_env *env, struct ir_insn *insn, } if (next) { PRINT_LOG(env, " "); - if (!is_void(next)) { + if (!bpf_ir_is_void(next)) { PRINT_LOG(env, "%%%zu", next->_insn_id); PRINT_LOG(env, " = "); } @@ -674,6 +686,7 @@ void bpf_ir_print_to_log(struct bpf_ir_env *env, char *fmt, ...) va_end(args); } +/* Dump env->log */ void bpf_ir_print_log_dbg(struct bpf_ir_env *env) { if (env->log_pos == 0) { @@ -683,9 +696,9 @@ void bpf_ir_print_log_dbg(struct bpf_ir_env *env) PRINT_DBG("----- Begin of Log -----\n"); // PRINT_DBG("%s", env->log); char line[1000]; - int i = 0; // Global ptr + size_t i = 0; // Global ptr while (i < env->log_pos) { - int j = 0; // Line ptr + size_t j = 0; // Line ptr while (i < env->log_pos && j < 1000) { line[j++] = env->log[i++]; if (env->log[i - 1] == '\n') { diff --git a/IR/ir_insn.c b/IR/ir_insn.c index 7467b2c0..71cc9418 100644 --- a/IR/ir_insn.c +++ b/IR/ir_insn.c @@ -1,10 +1,12 @@ #include -// May have exception -struct ir_insn *create_insn_base(struct ir_basic_block *bb) +struct ir_insn *bpf_ir_create_insn_base(struct bpf_ir_env *env, + struct ir_basic_block *bb) { struct ir_insn *new_insn = malloc_proto(sizeof(struct ir_insn)); if (!new_insn) { + env->err = -ENOMEM; + PRINT_LOG(env, "Failed to allocate memory for ir_insn\n"); return NULL; } new_insn->parent_bb = bb; @@ -13,31 +15,32 @@ struct ir_insn *create_insn_base(struct ir_basic_block *bb) return new_insn; } -// May have exception -struct ir_insn *create_insn_base_cg(struct bpf_ir_env *env, - struct ir_basic_block *bb) +struct ir_insn *bpf_ir_create_insn_base_cg(struct bpf_ir_env *env, + struct ir_basic_block *bb, + enum ir_insn_type insn_type) { - struct ir_insn *new_insn = create_insn_base(bb); + struct ir_insn *new_insn = bpf_ir_create_insn_base(env, bb); if (!new_insn) { + env->err = -ENOMEM; + PRINT_LOG(env, "Failed to allocate memory for ir_insn\n"); return NULL; } - + new_insn->op = insn_type; bpf_ir_init_insn_cg(env, new_insn); CHECK_ERR(NULL); - insn_cg(new_insn)->dst = new_insn; return new_insn; } -void replace_operand(struct bpf_ir_env *env, struct ir_insn *insn, - struct ir_value v1, struct ir_value v2) +void bpf_ir_replace_operand(struct bpf_ir_env *env, struct ir_insn *insn, + struct ir_value v1, struct ir_value v2) { // Replace v1 with v2 in insn if (v1.type == IR_VALUE_INSN) { // Remove user from v1 - val_remove_user(v1, insn); + bpf_ir_val_remove_user(v1, insn); } if (v2.type == IR_VALUE_INSN) { - val_add_user(env, v2, insn); + bpf_ir_val_add_user(env, v2, insn); } } @@ -58,7 +61,32 @@ void bpf_ir_replace_all_usage(struct bpf_ir_env *env, struct ir_insn *insn, (*pos2)->data.insn_d == insn) { // Match, replace **pos2 = rep; - val_add_user(env, rep, user); + bpf_ir_val_add_user(env, rep, user); + } + } + bpf_ir_array_free(&operands); + } + bpf_ir_array_free(&users); +} + +void bpf_ir_replace_all_usage_cg(struct bpf_ir_env *env, struct ir_insn *insn, + struct ir_value rep) +{ + struct ir_insn **pos; + struct array users = insn->users; + INIT_ARRAY(&insn->users, struct ir_insn *); + array_for(pos, users) + { + struct ir_insn *user = *pos; + struct array operands = bpf_ir_get_operands_and_dst(env, user); + struct ir_value **pos2; + array_for(pos2, operands) + { + if ((*pos2)->type == IR_VALUE_INSN && + (*pos2)->data.insn_d == insn) { + // Match, replace + **pos2 = rep; + bpf_ir_val_add_user(env, rep, user); } } bpf_ir_array_free(&operands); @@ -88,7 +116,7 @@ void bpf_ir_replace_all_usage_except(struct bpf_ir_env *env, (*pos2)->data.insn_d == insn) { // Match, replace **pos2 = rep; - val_add_user(env, rep, user); + bpf_ir_val_add_user(env, rep, user); } } bpf_ir_array_free(&operands); @@ -106,6 +134,7 @@ struct array bpf_ir_get_operands(struct bpf_ir_env *env, struct ir_insn *insn) pos = &insn->values[j]; bpf_ir_array_push(env, &uses, &pos); } + // Phi value if (insn->op == IR_INSN_PHI) { struct phi_value *pv_pos2; array_for(pv_pos2, insn->phi) @@ -114,6 +143,7 @@ struct array bpf_ir_get_operands(struct bpf_ir_env *env, struct ir_insn *insn) bpf_ir_array_push(env, &uses, &pos); } } + // Address value if (insn->op == IR_INSN_LOADRAW || insn->op == IR_INSN_STORERAW) { pos = &insn->addr_val.value; bpf_ir_array_push(env, &uses, &pos); @@ -121,28 +151,95 @@ struct array bpf_ir_get_operands(struct bpf_ir_env *env, struct ir_insn *insn) return uses; } +struct array bpf_ir_get_operands_and_dst(struct bpf_ir_env *env, + struct ir_insn *insn) +{ + struct array uses = bpf_ir_get_operands(env, insn); + struct ir_value *val = &insn_cg(insn)->dst; + bpf_ir_array_push(env, &uses, &val); + return uses; +} + int bpf_ir_is_last_insn(struct ir_insn *insn) { return insn->parent_bb->ir_insn_head.prev == &insn->list_ptr; } +void bpf_ir_erase_insn_cg(struct bpf_ir_env *env, struct ir_insn *insn) +{ + if (insn->users.num_elem > 0) { + struct ir_insn **pos; + bool fail = false; + array_for(pos, insn->users) + { + if (*pos != insn) { + fail = true; + break; + } + } + if (fail) { + array_for(pos, insn->users) + { + print_ir_insn_err_full(env, *pos, "User", + print_ir_dst); + } + print_ir_insn_err_full(env, insn, "Has users", + print_ir_dst); + RAISE_ERROR( + "Cannot erase a instruction that has (non-self) users"); + } + } + struct array operands = bpf_ir_get_operands_and_dst(env, insn); + CHECK_ERR(); + struct ir_value **pos2; + array_for(pos2, operands) + { + bpf_ir_val_remove_user((**pos2), insn); + } + bpf_ir_array_free(&operands); + bpf_ir_free_insn_cg(insn); + list_del(&insn->list_ptr); + bpf_ir_array_free(&insn->users); + free_proto(insn); +} + void bpf_ir_erase_insn(struct bpf_ir_env *env, struct ir_insn *insn) { - // TODO: remove users + if (insn->users.num_elem > 0) { + struct ir_insn **pos; + bool fail = false; + array_for(pos, insn->users) + { + if (*pos != insn) { + fail = true; + break; + } + } + if (fail) { + array_for(pos, insn->users) + { + print_ir_insn_err(env, *pos, "User"); + } + print_ir_insn_err(env, insn, "Has users"); + RAISE_ERROR( + "Cannot erase a instruction that has (non-self) users"); + } + } struct array operands = bpf_ir_get_operands(env, insn); CHECK_ERR(); struct ir_value **pos2; array_for(pos2, operands) { - val_remove_user((**pos2), insn); + bpf_ir_val_remove_user((**pos2), insn); } bpf_ir_array_free(&operands); list_del(&insn->list_ptr); + bpf_ir_array_free(&insn->users); free_proto(insn); } -void insert_at(struct ir_insn *new_insn, struct ir_insn *insn, - enum insert_position pos) +void bpf_ir_insert_at(struct ir_insn *new_insn, struct ir_insn *insn, + enum insert_position pos) { if (pos == INSERT_BACK) { list_add(&new_insn->list_ptr, &insn->list_ptr); @@ -153,8 +250,8 @@ void insert_at(struct ir_insn *new_insn, struct ir_insn *insn, } } -void insert_at_bb(struct ir_insn *new_insn, struct ir_basic_block *bb, - enum insert_position pos) +void bpf_ir_insert_at_bb(struct ir_insn *new_insn, struct ir_basic_block *bb, + enum insert_position pos) { if (pos == INSERT_BACK) { list_add_tail(&new_insn->list_ptr, &bb->ir_insn_head); @@ -165,7 +262,7 @@ void insert_at_bb(struct ir_insn *new_insn, struct ir_basic_block *bb, // 2. If there is a JMP at the end, insert before it struct ir_insn *last_insn = bpf_ir_get_last_insn(bb); if (last_insn) { - if (is_jmp(last_insn)) { + if (bpf_ir_is_jmp(last_insn)) { // Insert before this insn list_add_tail(&new_insn->list_ptr, &last_insn->list_ptr); @@ -196,34 +293,7 @@ void insert_at_bb(struct ir_insn *new_insn, struct ir_basic_block *bb, } } -struct ir_insn *create_alloc_insn_base(struct ir_basic_block *bb, - enum ir_vr_type type) -{ - struct ir_insn *new_insn = create_insn_base(bb); - new_insn->op = IR_INSN_ALLOC; - new_insn->vr_type = type; - return new_insn; -} - -struct ir_insn *create_alloc_insn(struct ir_insn *insn, enum ir_vr_type type, - enum insert_position pos) -{ - struct ir_insn *new_insn = - create_alloc_insn_base(insn->parent_bb, type); - insert_at(new_insn, insn, pos); - return new_insn; -} - -struct ir_insn *create_alloc_insn_bb(struct ir_basic_block *bb, - enum ir_vr_type type, - enum insert_position pos) -{ - struct ir_insn *new_insn = create_alloc_insn_base(bb, type); - insert_at_bb(new_insn, bb, pos); - return new_insn; -} - -void val_remove_user(struct ir_value val, struct ir_insn *user) +void bpf_ir_val_remove_user(struct ir_value val, struct ir_insn *user) { if (val.type != IR_VALUE_INSN) { return; @@ -239,373 +309,738 @@ void val_remove_user(struct ir_value val, struct ir_insn *user) PRINT_DBG("Warning: User not found in the users\n"); } -void val_add_user(struct bpf_ir_env *env, struct ir_value val, - struct ir_insn *user) +/** + Add user to the users of val + */ +void bpf_ir_val_add_user(struct bpf_ir_env *env, struct ir_value val, + struct ir_insn *user) { if (val.type != IR_VALUE_INSN) { return; } - bpf_ir_array_push_unique(env, &val.data.insn_d->users, &user); + // May push many same users + bpf_ir_array_push(env, &val.data.insn_d->users, &user); +} + +struct ir_insn *bpf_ir_prev_insn(struct ir_insn *insn) +{ + struct list_head *prev = insn->list_ptr.prev; + if (prev == &insn->parent_bb->ir_insn_head) { + return NULL; + } + return list_entry(prev, struct ir_insn, list_ptr); } -struct ir_insn *create_store_insn_base(struct bpf_ir_env *env, - struct ir_basic_block *bb, - struct ir_insn *insn, - struct ir_value val) +struct ir_insn *bpf_ir_next_insn(struct ir_insn *insn) { - struct ir_insn *new_insn = create_insn_base(bb); + struct list_head *next = insn->list_ptr.next; + if (next == &insn->parent_bb->ir_insn_head) { + return NULL; + } + return list_entry(next, struct ir_insn, list_ptr); +} + +// Note. This includes ret instruction +int bpf_ir_is_jmp(struct ir_insn *insn) +{ + return (insn->op >= IR_INSN_JA && insn->op <= IR_INSN_JNE) || + insn->op == IR_INSN_RET; +} + +int bpf_ir_is_cond_jmp(struct ir_insn *insn) +{ + return (insn->op >= IR_INSN_JEQ && insn->op < IR_INSN_PHI); +} + +int bpf_ir_is_alu(struct ir_insn *insn) +{ + return insn->op >= IR_INSN_ADD && insn->op < IR_INSN_CALL; +} + +int bpf_ir_is_void(struct ir_insn *insn) +{ + return bpf_ir_is_jmp(insn) || insn->op == IR_INSN_STORERAW || + insn->op == IR_INSN_STORE; +} + +void bpf_ir_phi_add_operand(struct bpf_ir_env *env, struct ir_insn *insn, + struct ir_basic_block *bb, struct ir_value val) +{ + // Make sure that bb is a pred of insn parent BB + struct phi_value pv; + pv.value = val; + pv.bb = bb; + bpf_ir_array_push(env, &insn->phi, &pv); + bpf_ir_val_add_user(env, val, insn); +} + +void bpf_ir_phi_add_call_arg(struct bpf_ir_env *env, struct ir_insn *insn, + struct ir_value val) +{ + insn->values[insn->value_num++] = val; + bpf_ir_val_add_user(env, val, insn); +} + +/* Instruction Constructor Protos */ + +static struct ir_insn *create_alloc_insn_base(struct bpf_ir_env *env, + struct ir_basic_block *bb, + enum ir_vr_type type) +{ + struct ir_insn *new_insn = bpf_ir_create_insn_base(env, bb); + new_insn->op = IR_INSN_ALLOC; + new_insn->vr_type = type; + new_insn->value_num = 0; + return new_insn; +} + +static struct ir_insn *create_allocarray_insn_base(struct bpf_ir_env *env, + struct ir_basic_block *bb, + enum ir_vr_type type, + u32 num) +{ + struct ir_insn *new_insn = bpf_ir_create_insn_base(env, bb); + new_insn->op = IR_INSN_ALLOCARRAY; + new_insn->vr_type = type; + new_insn->array_num = num; + new_insn->value_num = 0; + return new_insn; +} + +static struct ir_insn *create_getelemptr_insn_base(struct bpf_ir_env *env, + struct ir_basic_block *bb, + struct ir_insn *alloca_insn, + struct ir_value offset) +{ + struct ir_insn *new_insn = bpf_ir_create_insn_base(env, bb); + new_insn->op = IR_INSN_GETELEMPTR; + new_insn->values[0] = offset; + new_insn->values[1] = bpf_ir_value_insn(alloca_insn); + new_insn->value_num = 2; + bpf_ir_val_add_user(env, new_insn->values[0], new_insn); + bpf_ir_val_add_user(env, new_insn->values[1], new_insn); + return new_insn; +} + +static struct ir_insn *create_store_insn_base(struct bpf_ir_env *env, + struct ir_basic_block *bb, + struct ir_insn *insn, + struct ir_value val) +{ + struct ir_insn *new_insn = bpf_ir_create_insn_base(env, bb); new_insn->op = IR_INSN_STORE; struct ir_value nv = bpf_ir_value_insn(insn); new_insn->values[0] = nv; new_insn->values[1] = val; new_insn->value_num = 2; - val_add_user(env, nv, new_insn); - val_add_user(env, val, new_insn); + bpf_ir_val_add_user(env, nv, new_insn); + bpf_ir_val_add_user(env, val, new_insn); + return new_insn; +} + +static struct ir_insn *create_load_insn_base(struct bpf_ir_env *env, + struct ir_basic_block *bb, + struct ir_value val) +{ + struct ir_insn *new_insn = bpf_ir_create_insn_base(env, bb); + new_insn->op = IR_INSN_LOAD; + new_insn->values[0] = val; + bpf_ir_val_add_user(env, val, new_insn); + new_insn->value_num = 1; + return new_insn; +} + +static struct ir_insn * +create_bin_insn_base(struct bpf_ir_env *env, struct ir_basic_block *bb, + struct ir_value val1, struct ir_value val2, + enum ir_insn_type ty, enum ir_alu_op_type alu_type) +{ + struct ir_insn *new_insn = bpf_ir_create_insn_base(env, bb); + new_insn->op = ty; + new_insn->values[0] = val1; + new_insn->values[1] = val2; + new_insn->alu_op = alu_type; + bpf_ir_val_add_user(env, val1, new_insn); + bpf_ir_val_add_user(env, val2, new_insn); + new_insn->value_num = 2; + return new_insn; +} + +static struct ir_insn * +create_bin_insn_base_cg(struct bpf_ir_env *env, struct ir_basic_block *bb, + struct ir_value val1, struct ir_value val2, + enum ir_insn_type ty, enum ir_alu_op_type alu_type) +{ + struct ir_insn *new_insn = bpf_ir_create_insn_base_cg(env, bb, ty); + new_insn->op = ty; + new_insn->values[0] = val1; + new_insn->values[1] = val2; + new_insn->alu_op = alu_type; + bpf_ir_val_add_user(env, val1, new_insn); + bpf_ir_val_add_user(env, val2, new_insn); + new_insn->value_num = 2; + return new_insn; +} + +static struct ir_insn *create_ja_insn_base(struct bpf_ir_env *env, + struct ir_basic_block *bb, + struct ir_basic_block *to_bb) +{ + struct ir_insn *new_insn = bpf_ir_create_insn_base(env, bb); + new_insn->op = IR_INSN_JA; + new_insn->bb1 = to_bb; + bpf_ir_array_push(env, &to_bb->users, &new_insn); + return new_insn; +} + +static struct ir_insn * +create_jbin_insn_base(struct bpf_ir_env *env, struct ir_basic_block *bb, + struct ir_value val1, struct ir_value val2, + struct ir_basic_block *to_bb1, + struct ir_basic_block *to_bb2, enum ir_insn_type ty, + enum ir_alu_op_type alu_type) +{ + struct ir_insn *new_insn = bpf_ir_create_insn_base(env, bb); + new_insn->op = ty; + new_insn->values[0] = val1; + new_insn->values[1] = val2; + new_insn->bb1 = to_bb1; + new_insn->bb2 = to_bb2; + new_insn->alu_op = alu_type; + bpf_ir_val_add_user(env, val1, new_insn); + bpf_ir_val_add_user(env, val2, new_insn); + bpf_ir_array_push(env, &to_bb1->users, &new_insn); + bpf_ir_array_push(env, &to_bb2->users, &new_insn); + new_insn->value_num = 2; + return new_insn; +} + +static struct ir_insn *create_ret_insn_base(struct bpf_ir_env *env, + struct ir_basic_block *bb, + struct ir_value val) +{ + struct ir_insn *new_insn = bpf_ir_create_insn_base(env, bb); + new_insn->op = IR_INSN_RET; + new_insn->values[0] = val; + new_insn->value_num = 1; + bpf_ir_val_add_user(env, val, new_insn); return new_insn; } -struct ir_insn *create_store_insn(struct bpf_ir_env *env, struct ir_insn *insn, - struct ir_insn *st_insn, struct ir_value val, - enum insert_position pos) +static struct ir_insn *create_call_insn_base(struct bpf_ir_env *env, + struct ir_basic_block *bb, s32 fid) +{ + struct ir_insn *new_insn = bpf_ir_create_insn_base(env, bb); + new_insn->op = IR_INSN_CALL; + new_insn->value_num = 0; + new_insn->fid = fid; + return new_insn; +} + +static struct ir_insn *create_loadraw_insn_base(struct bpf_ir_env *env, + struct ir_basic_block *bb, + enum ir_vr_type type, + struct ir_address_value val) +{ + struct ir_insn *new_insn = bpf_ir_create_insn_base(env, bb); + new_insn->op = IR_INSN_LOADRAW; + new_insn->addr_val = val; + new_insn->value_num = 0; + new_insn->vr_type = type; + bpf_ir_val_add_user(env, val.value, new_insn); + return new_insn; +} + +static struct ir_insn *create_loadraw_insn_base_cg(struct bpf_ir_env *env, + struct ir_basic_block *bb, + enum ir_vr_type type, + struct ir_address_value val) { struct ir_insn *new_insn = - create_store_insn_base(env, insn->parent_bb, st_insn, val); - insert_at(new_insn, insn, pos); + bpf_ir_create_insn_base_cg(env, bb, IR_INSN_LOADRAW); + new_insn->addr_val = val; + new_insn->value_num = 0; + new_insn->vr_type = type; + bpf_ir_val_add_user(env, val.value, new_insn); return new_insn; } -struct ir_insn *create_store_insn_bb(struct bpf_ir_env *env, - struct ir_basic_block *bb, - struct ir_insn *st_insn, - struct ir_value val, - enum insert_position pos) +static struct ir_insn *create_storeraw_insn_base(struct bpf_ir_env *env, + struct ir_basic_block *bb, + enum ir_vr_type type, + struct ir_address_value val, + struct ir_value to_store) +{ + struct ir_insn *new_insn = bpf_ir_create_insn_base(env, bb); + new_insn->op = IR_INSN_STORERAW; + new_insn->addr_val = val; + new_insn->values[0] = to_store; + new_insn->value_num = 1; + new_insn->vr_type = type; + bpf_ir_val_add_user(env, val.value, new_insn); + bpf_ir_val_add_user(env, to_store, new_insn); + return new_insn; +} + +static struct ir_insn *create_storeraw_insn_base_cg(struct bpf_ir_env *env, + struct ir_basic_block *bb, + enum ir_vr_type type, + struct ir_address_value val, + struct ir_value to_store) { struct ir_insn *new_insn = - create_store_insn_base(env, bb, st_insn, val); - insert_at_bb(new_insn, bb, pos); + bpf_ir_create_insn_base_cg(env, bb, IR_INSN_STORERAW); + new_insn->addr_val = val; + new_insn->values[0] = to_store; + new_insn->value_num = 1; + new_insn->vr_type = type; + bpf_ir_val_add_user(env, val.value, new_insn); + bpf_ir_val_add_user(env, to_store, new_insn); return new_insn; } -struct ir_insn *create_load_insn_base(struct bpf_ir_env *env, - struct ir_basic_block *bb, - struct ir_value val) +static struct ir_insn *create_assign_insn_base(struct bpf_ir_env *env, + struct ir_basic_block *bb, + struct ir_value val) { - struct ir_insn *new_insn = create_insn_base(bb); - new_insn->op = IR_INSN_LOAD; + struct ir_insn *new_insn = bpf_ir_create_insn_base(env, bb); + new_insn->op = IR_INSN_ASSIGN; new_insn->values[0] = val; - val_add_user(env, val, new_insn); new_insn->value_num = 1; + bpf_ir_val_add_user(env, val, new_insn); return new_insn; } -struct ir_insn *create_load_insn(struct bpf_ir_env *env, struct ir_insn *insn, - struct ir_value val, enum insert_position pos) +static struct ir_insn *create_assign_insn_base_cg(struct bpf_ir_env *env, + struct ir_basic_block *bb, + struct ir_value val) { struct ir_insn *new_insn = - create_load_insn_base(env, insn->parent_bb, val); - insert_at(new_insn, insn, pos); + bpf_ir_create_insn_base_cg(env, bb, IR_INSN_ASSIGN); + new_insn->values[0] = val; + new_insn->value_num = 1; + new_insn->vr_type = IR_VR_TYPE_UNKNOWN; + new_insn->alu_op = IR_ALU_UNKNOWN; + bpf_ir_val_add_user(env, val, new_insn); return new_insn; } -struct ir_insn *create_load_insn_bb(struct bpf_ir_env *env, - struct ir_basic_block *bb, - struct ir_value val, - enum insert_position pos) +static struct ir_insn *create_phi_insn_base(struct bpf_ir_env *env, + struct ir_basic_block *bb) { - struct ir_insn *new_insn = create_load_insn_base(env, bb, val); - insert_at_bb(new_insn, bb, pos); + struct ir_insn *new_insn = bpf_ir_create_insn_base(env, bb); + new_insn->op = IR_INSN_PHI; + INIT_ARRAY(&new_insn->phi, struct phi_value); return new_insn; } -struct ir_insn *create_bin_insn_base(struct bpf_ir_env *env, - struct ir_basic_block *bb, - struct ir_value val1, struct ir_value val2, - enum ir_insn_type ty, - enum ir_alu_op_type aluty) +/* Generated Constructors */ + +struct ir_insn *bpf_ir_create_alloc_insn(struct bpf_ir_env *env, + struct ir_insn *pos_insn, + enum ir_vr_type type, + enum insert_position pos) { - struct ir_insn *new_insn = create_insn_base(bb); - new_insn->op = ty; - new_insn->values[0] = val1; - new_insn->values[1] = val2; - new_insn->alu_op = aluty; - val_add_user(env, val1, new_insn); - val_add_user(env, val2, new_insn); - new_insn->value_num = 2; + struct ir_insn *new_insn = + create_alloc_insn_base(env, pos_insn->parent_bb, type); + bpf_ir_insert_at(new_insn, pos_insn, pos); return new_insn; } -struct ir_insn *create_bin_insn(struct bpf_ir_env *env, struct ir_insn *insn, - struct ir_value val1, struct ir_value val2, - enum ir_insn_type ty, enum ir_alu_op_type aluty, - enum insert_position pos) +struct ir_insn *bpf_ir_create_alloc_insn_bb(struct bpf_ir_env *env, + struct ir_basic_block *pos_bb, + enum ir_vr_type type, + enum insert_position pos) { - struct ir_insn *new_insn = create_bin_insn_base(env, insn->parent_bb, - val1, val2, ty, aluty); - insert_at(new_insn, insn, pos); + struct ir_insn *new_insn = create_alloc_insn_base(env, pos_bb, type); + bpf_ir_insert_at_bb(new_insn, pos_bb, pos); return new_insn; } -struct ir_insn *create_bin_insn_bb(struct bpf_ir_env *env, - struct ir_basic_block *bb, - struct ir_value val1, struct ir_value val2, - enum ir_insn_type ty, - enum ir_alu_op_type aluty, - enum insert_position pos) +struct ir_insn *bpf_ir_create_allocarray_insn(struct bpf_ir_env *env, + struct ir_insn *pos_insn, + enum ir_vr_type type, u32 num, + enum insert_position pos) +{ + struct ir_insn *new_insn = create_allocarray_insn_base( + env, pos_insn->parent_bb, type, num); + bpf_ir_insert_at(new_insn, pos_insn, pos); + return new_insn; +} + +struct ir_insn *bpf_ir_create_allocarray_insn_bb(struct bpf_ir_env *env, + struct ir_basic_block *pos_bb, + enum ir_vr_type type, u32 num, + enum insert_position pos) { struct ir_insn *new_insn = - create_bin_insn_base(env, bb, val1, val2, ty, aluty); - insert_at_bb(new_insn, bb, pos); + create_allocarray_insn_base(env, pos_bb, type, num); + bpf_ir_insert_at_bb(new_insn, pos_bb, pos); return new_insn; } -struct ir_insn *prev_insn(struct ir_insn *insn) +struct ir_insn *bpf_ir_create_getelemptr_insn(struct bpf_ir_env *env, + struct ir_insn *pos_insn, + struct ir_insn *alloca_insn, + struct ir_value offset, + enum insert_position pos) { - struct list_head *prev = insn->list_ptr.prev; - if (prev == &insn->parent_bb->ir_insn_head) { - return NULL; - } - return list_entry(prev, struct ir_insn, list_ptr); + struct ir_insn *new_insn = create_getelemptr_insn_base( + env, pos_insn->parent_bb, alloca_insn, offset); + bpf_ir_insert_at(new_insn, pos_insn, pos); + return new_insn; } -struct ir_insn *next_insn(struct ir_insn *insn) +struct ir_insn *bpf_ir_create_getelemptr_insn_bb(struct bpf_ir_env *env, + struct ir_basic_block *pos_bb, + struct ir_insn *alloca_insn, + struct ir_value offset, + enum insert_position pos) { - struct list_head *next = insn->list_ptr.next; - if (next == &insn->parent_bb->ir_insn_head) { - return NULL; - } - return list_entry(next, struct ir_insn, list_ptr); + struct ir_insn *new_insn = + create_getelemptr_insn_base(env, pos_bb, alloca_insn, offset); + bpf_ir_insert_at_bb(new_insn, pos_bb, pos); + return new_insn; } -struct ir_insn *create_ja_insn_base(struct bpf_ir_env *env, - struct ir_basic_block *bb, - struct ir_basic_block *to_bb) +struct ir_insn *bpf_ir_create_store_insn(struct bpf_ir_env *env, + struct ir_insn *pos_insn, + struct ir_insn *insn, + struct ir_value val, + enum insert_position pos) { - struct ir_insn *new_insn = create_insn_base(bb); - new_insn->op = IR_INSN_JA; - new_insn->bb1 = to_bb; - bpf_ir_array_push(env, &to_bb->users, &new_insn); + struct ir_insn *new_insn = + create_store_insn_base(env, pos_insn->parent_bb, insn, val); + bpf_ir_insert_at(new_insn, pos_insn, pos); return new_insn; } -struct ir_insn *create_ja_insn(struct bpf_ir_env *env, struct ir_insn *insn, - struct ir_basic_block *to_bb, - enum insert_position pos) +struct ir_insn *bpf_ir_create_store_insn_bb(struct bpf_ir_env *env, + struct ir_basic_block *pos_bb, + struct ir_insn *insn, + struct ir_value val, + enum insert_position pos) { struct ir_insn *new_insn = - create_ja_insn_base(env, insn->parent_bb, to_bb); - insert_at(new_insn, insn, pos); + create_store_insn_base(env, pos_bb, insn, val); + bpf_ir_insert_at_bb(new_insn, pos_bb, pos); return new_insn; } -struct ir_insn *create_ja_insn_bb(struct bpf_ir_env *env, - struct ir_basic_block *bb, - struct ir_basic_block *to_bb, - enum insert_position pos) +struct ir_insn *bpf_ir_create_load_insn(struct bpf_ir_env *env, + struct ir_insn *pos_insn, + struct ir_value val, + enum insert_position pos) { - struct ir_insn *new_insn = create_ja_insn_base(env, bb, to_bb); - insert_at_bb(new_insn, bb, pos); + struct ir_insn *new_insn = + create_load_insn_base(env, pos_insn->parent_bb, val); + bpf_ir_insert_at(new_insn, pos_insn, pos); + return new_insn; +} + +struct ir_insn *bpf_ir_create_load_insn_bb(struct bpf_ir_env *env, + struct ir_basic_block *pos_bb, + struct ir_value val, + enum insert_position pos) +{ + struct ir_insn *new_insn = create_load_insn_base(env, pos_bb, val); + bpf_ir_insert_at_bb(new_insn, pos_bb, pos); return new_insn; } struct ir_insn * -create_jbin_insn_base(struct bpf_ir_env *env, struct ir_basic_block *bb, - struct ir_value val1, struct ir_value val2, - struct ir_basic_block *to_bb1, - struct ir_basic_block *to_bb2, enum ir_insn_type ty, - enum ir_alu_op_type aluty) +bpf_ir_create_bin_insn(struct bpf_ir_env *env, struct ir_insn *pos_insn, + struct ir_value val1, struct ir_value val2, + enum ir_insn_type ty, enum ir_alu_op_type alu_type, + enum insert_position pos) { - struct ir_insn *new_insn = create_insn_base(bb); - new_insn->op = ty; - new_insn->values[0] = val1; - new_insn->values[1] = val2; - new_insn->bb1 = to_bb1; - new_insn->bb2 = to_bb2; - new_insn->alu_op = aluty; - val_add_user(env, val1, new_insn); - val_add_user(env, val2, new_insn); - bpf_ir_array_push(env, &to_bb1->users, &new_insn); - bpf_ir_array_push(env, &to_bb2->users, &new_insn); - new_insn->value_num = 2; + struct ir_insn *new_insn = create_bin_insn_base( + env, pos_insn->parent_bb, val1, val2, ty, alu_type); + bpf_ir_insert_at(new_insn, pos_insn, pos); return new_insn; } -struct ir_insn *create_jbin_insn(struct bpf_ir_env *env, struct ir_insn *insn, - struct ir_value val1, struct ir_value val2, - struct ir_basic_block *to_bb1, - struct ir_basic_block *to_bb2, - enum ir_insn_type ty, - enum ir_alu_op_type aluty, - enum insert_position pos) +struct ir_insn * +bpf_ir_create_bin_insn_bb(struct bpf_ir_env *env, struct ir_basic_block *pos_bb, + struct ir_value val1, struct ir_value val2, + enum ir_insn_type ty, enum ir_alu_op_type alu_type, + enum insert_position pos) { - struct ir_insn *new_insn = create_jbin_insn_base( - env, insn->parent_bb, val1, val2, to_bb1, to_bb2, ty, aluty); - insert_at(new_insn, insn, pos); + struct ir_insn *new_insn = + create_bin_insn_base(env, pos_bb, val1, val2, ty, alu_type); + bpf_ir_insert_at_bb(new_insn, pos_bb, pos); + return new_insn; +} + +struct ir_insn * +bpf_ir_create_bin_insn_cg(struct bpf_ir_env *env, struct ir_insn *pos_insn, + struct ir_value val1, struct ir_value val2, + enum ir_insn_type ty, enum ir_alu_op_type alu_type, + enum insert_position pos) +{ + struct ir_insn *new_insn = create_bin_insn_base_cg( + env, pos_insn->parent_bb, val1, val2, ty, alu_type); + bpf_ir_insert_at(new_insn, pos_insn, pos); + return new_insn; +} + +struct ir_insn *bpf_ir_create_bin_insn_bb_cg( + struct bpf_ir_env *env, struct ir_basic_block *pos_bb, + struct ir_value val1, struct ir_value val2, enum ir_insn_type ty, + enum ir_alu_op_type alu_type, enum insert_position pos) +{ + struct ir_insn *new_insn = + create_bin_insn_base_cg(env, pos_bb, val1, val2, ty, alu_type); + bpf_ir_insert_at_bb(new_insn, pos_bb, pos); + return new_insn; +} + +struct ir_insn *bpf_ir_create_ja_insn(struct bpf_ir_env *env, + struct ir_insn *pos_insn, + struct ir_basic_block *to_bb, + enum insert_position pos) +{ + struct ir_insn *new_insn = + create_ja_insn_base(env, pos_insn->parent_bb, to_bb); + bpf_ir_insert_at(new_insn, pos_insn, pos); + return new_insn; +} + +struct ir_insn *bpf_ir_create_ja_insn_bb(struct bpf_ir_env *env, + struct ir_basic_block *pos_bb, + struct ir_basic_block *to_bb, + enum insert_position pos) +{ + struct ir_insn *new_insn = create_ja_insn_base(env, pos_bb, to_bb); + bpf_ir_insert_at_bb(new_insn, pos_bb, pos); return new_insn; } struct ir_insn * -create_jbin_insn_bb(struct bpf_ir_env *env, struct ir_basic_block *bb, - struct ir_value val1, struct ir_value val2, - struct ir_basic_block *to_bb1, - struct ir_basic_block *to_bb2, enum ir_insn_type ty, - enum ir_alu_op_type aluty, enum insert_position pos) +bpf_ir_create_jbin_insn(struct bpf_ir_env *env, struct ir_insn *pos_insn, + struct ir_value val1, struct ir_value val2, + struct ir_basic_block *to_bb1, + struct ir_basic_block *to_bb2, enum ir_insn_type ty, + enum ir_alu_op_type alu_type, enum insert_position pos) +{ + struct ir_insn *new_insn = + create_jbin_insn_base(env, pos_insn->parent_bb, val1, val2, + to_bb1, to_bb2, ty, alu_type); + bpf_ir_insert_at(new_insn, pos_insn, pos); + return new_insn; +} + +struct ir_insn * +bpf_ir_create_jbin_insn_bb(struct bpf_ir_env *env, + struct ir_basic_block *pos_bb, struct ir_value val1, + struct ir_value val2, struct ir_basic_block *to_bb1, + struct ir_basic_block *to_bb2, enum ir_insn_type ty, + enum ir_alu_op_type alu_type, + enum insert_position pos) { struct ir_insn *new_insn = create_jbin_insn_base( - env, bb, val1, val2, to_bb1, to_bb2, ty, aluty); - insert_at_bb(new_insn, bb, pos); + env, pos_bb, val1, val2, to_bb1, to_bb2, ty, alu_type); + bpf_ir_insert_at_bb(new_insn, pos_bb, pos); return new_insn; } -struct ir_insn *create_ret_insn_base(struct bpf_ir_env *env, - struct ir_basic_block *bb, - struct ir_value val) +struct ir_insn *bpf_ir_create_ret_insn(struct bpf_ir_env *env, + struct ir_insn *pos_insn, + struct ir_value val, + enum insert_position pos) { - struct ir_insn *new_insn = create_insn_base(bb); - new_insn->op = IR_INSN_RET; - new_insn->values[0] = val; - new_insn->value_num = 1; - val_add_user(env, val, new_insn); + struct ir_insn *new_insn = + create_ret_insn_base(env, pos_insn->parent_bb, val); + bpf_ir_insert_at(new_insn, pos_insn, pos); + return new_insn; +} + +struct ir_insn *bpf_ir_create_ret_insn_bb(struct bpf_ir_env *env, + struct ir_basic_block *pos_bb, + struct ir_value val, + enum insert_position pos) +{ + struct ir_insn *new_insn = create_ret_insn_base(env, pos_bb, val); + bpf_ir_insert_at_bb(new_insn, pos_bb, pos); return new_insn; } -struct ir_insn *create_ret_insn(struct bpf_ir_env *env, struct ir_insn *insn, - struct ir_value val, enum insert_position pos) +struct ir_insn *bpf_ir_create_call_insn(struct bpf_ir_env *env, + struct ir_insn *pos_insn, s32 fid, + enum insert_position pos) { struct ir_insn *new_insn = - create_ret_insn_base(env, insn->parent_bb, val); - insert_at(new_insn, insn, pos); + create_call_insn_base(env, pos_insn->parent_bb, fid); + bpf_ir_insert_at(new_insn, pos_insn, pos); return new_insn; } -struct ir_insn *create_ret_insn_bb(struct bpf_ir_env *env, - struct ir_basic_block *bb, - struct ir_value val, - enum insert_position pos) +struct ir_insn *bpf_ir_create_call_insn_bb(struct bpf_ir_env *env, + struct ir_basic_block *pos_bb, + s32 fid, enum insert_position pos) { - struct ir_insn *new_insn = create_ret_insn_base(env, bb, val); - insert_at_bb(new_insn, bb, pos); + struct ir_insn *new_insn = create_call_insn_base(env, pos_bb, fid); + bpf_ir_insert_at_bb(new_insn, pos_bb, pos); return new_insn; } -// Note. This includes ret instruction -int is_jmp(struct ir_insn *insn) +struct ir_insn *bpf_ir_create_loadraw_insn(struct bpf_ir_env *env, + struct ir_insn *pos_insn, + enum ir_vr_type type, + struct ir_address_value val, + enum insert_position pos) { - return (insn->op >= IR_INSN_JA && insn->op <= IR_INSN_JNE) || - insn->op == IR_INSN_RET; + struct ir_insn *new_insn = + create_loadraw_insn_base(env, pos_insn->parent_bb, type, val); + bpf_ir_insert_at(new_insn, pos_insn, pos); + return new_insn; } -int is_cond_jmp(struct ir_insn *insn) +struct ir_insn *bpf_ir_create_loadraw_insn_bb(struct bpf_ir_env *env, + struct ir_basic_block *pos_bb, + enum ir_vr_type type, + struct ir_address_value val, + enum insert_position pos) { - return (insn->op >= IR_INSN_JEQ && insn->op < IR_INSN_PHI); + struct ir_insn *new_insn = + create_loadraw_insn_base(env, pos_bb, type, val); + bpf_ir_insert_at_bb(new_insn, pos_bb, pos); + return new_insn; } -int is_alu(struct ir_insn *insn) +struct ir_insn *bpf_ir_create_loadraw_insn_cg(struct bpf_ir_env *env, + struct ir_insn *pos_insn, + enum ir_vr_type type, + struct ir_address_value val, + enum insert_position pos) { - return insn->op >= IR_INSN_ADD && insn->op < IR_INSN_CALL; + struct ir_insn *new_insn = create_loadraw_insn_base_cg( + env, pos_insn->parent_bb, type, val); + bpf_ir_insert_at(new_insn, pos_insn, pos); + return new_insn; } -int is_void(struct ir_insn *insn) +struct ir_insn *bpf_ir_create_loadraw_insn_bb_cg(struct bpf_ir_env *env, + struct ir_basic_block *pos_bb, + enum ir_vr_type type, + struct ir_address_value val, + enum insert_position pos) { - return is_jmp(insn) || insn->op == IR_INSN_STORERAW || - insn->op == IR_INSN_STORE; + struct ir_insn *new_insn = + create_loadraw_insn_base_cg(env, pos_bb, type, val); + bpf_ir_insert_at_bb(new_insn, pos_bb, pos); + return new_insn; } -struct ir_insn *create_assign_insn_base(struct bpf_ir_env *env, - struct ir_basic_block *bb, - struct ir_value val) +struct ir_insn * +bpf_ir_create_storeraw_insn(struct bpf_ir_env *env, struct ir_insn *pos_insn, + enum ir_vr_type type, struct ir_address_value val, + struct ir_value to_store, enum insert_position pos) { - struct ir_insn *new_insn = create_insn_base(bb); - new_insn->op = IR_INSN_ASSIGN; - new_insn->values[0] = val; - new_insn->value_num = 1; - val_add_user(env, val, new_insn); + struct ir_insn *new_insn = create_storeraw_insn_base( + env, pos_insn->parent_bb, type, val, to_store); + bpf_ir_insert_at(new_insn, pos_insn, pos); return new_insn; } -struct ir_insn *create_assign_insn(struct bpf_ir_env *env, struct ir_insn *insn, - struct ir_value val, - enum insert_position pos) +struct ir_insn *bpf_ir_create_storeraw_insn_bb(struct bpf_ir_env *env, + struct ir_basic_block *pos_bb, + enum ir_vr_type type, + struct ir_address_value val, + struct ir_value to_store, + enum insert_position pos) { struct ir_insn *new_insn = - create_assign_insn_base(env, insn->parent_bb, val); - insert_at(new_insn, insn, pos); + create_storeraw_insn_base(env, pos_bb, type, val, to_store); + bpf_ir_insert_at_bb(new_insn, pos_bb, pos); return new_insn; } -struct ir_insn *create_assign_insn_bb(struct bpf_ir_env *env, - struct ir_basic_block *bb, - struct ir_value val, - enum insert_position pos) +struct ir_insn *bpf_ir_create_storeraw_insn_cg(struct bpf_ir_env *env, + struct ir_insn *pos_insn, + enum ir_vr_type type, + struct ir_address_value val, + struct ir_value to_store, + enum insert_position pos) { - struct ir_insn *new_insn = create_assign_insn_base(env, bb, val); - insert_at_bb(new_insn, bb, pos); + struct ir_insn *new_insn = create_storeraw_insn_base_cg( + env, pos_insn->parent_bb, type, val, to_store); + bpf_ir_insert_at(new_insn, pos_insn, pos); return new_insn; } -struct ir_insn *create_assign_insn_base_cg(struct bpf_ir_env *env, - struct ir_basic_block *bb, - struct ir_value val) +struct ir_insn *bpf_ir_create_storeraw_insn_bb_cg(struct bpf_ir_env *env, + struct ir_basic_block *pos_bb, + enum ir_vr_type type, + struct ir_address_value val, + struct ir_value to_store, + enum insert_position pos) { - struct ir_insn *new_insn = create_insn_base_cg(env, bb); - new_insn->op = IR_INSN_ASSIGN; - new_insn->values[0] = val; - new_insn->value_num = 1; - new_insn->vr_type = IR_VR_TYPE_UNKNOWN; - new_insn->alu_op = IR_ALU_UNKNOWN; - val_add_user(env, val, new_insn); + struct ir_insn *new_insn = + create_storeraw_insn_base_cg(env, pos_bb, type, val, to_store); + bpf_ir_insert_at_bb(new_insn, pos_bb, pos); return new_insn; } -struct ir_insn *create_assign_insn_cg(struct bpf_ir_env *env, - struct ir_insn *insn, struct ir_value val, - enum insert_position pos) +struct ir_insn *bpf_ir_create_assign_insn(struct bpf_ir_env *env, + struct ir_insn *pos_insn, + struct ir_value val, + enum insert_position pos) { struct ir_insn *new_insn = - create_assign_insn_base_cg(env, insn->parent_bb, val); - insert_at(new_insn, insn, pos); + create_assign_insn_base(env, pos_insn->parent_bb, val); + bpf_ir_insert_at(new_insn, pos_insn, pos); return new_insn; } -struct ir_insn *create_assign_insn_bb_cg(struct bpf_ir_env *env, - struct ir_basic_block *bb, - struct ir_value val, - enum insert_position pos) +struct ir_insn *bpf_ir_create_assign_insn_bb(struct bpf_ir_env *env, + struct ir_basic_block *pos_bb, + struct ir_value val, + enum insert_position pos) { - struct ir_insn *new_insn = create_assign_insn_base_cg(env, bb, val); - insert_at_bb(new_insn, bb, pos); + struct ir_insn *new_insn = create_assign_insn_base(env, pos_bb, val); + bpf_ir_insert_at_bb(new_insn, pos_bb, pos); return new_insn; } -struct ir_insn *create_phi_insn_base(struct ir_basic_block *bb) +struct ir_insn *bpf_ir_create_assign_insn_cg(struct bpf_ir_env *env, + struct ir_insn *pos_insn, + struct ir_value val, + enum insert_position pos) { - struct ir_insn *new_insn = create_insn_base(bb); - new_insn->op = IR_INSN_PHI; - INIT_ARRAY(&new_insn->phi, struct phi_value); + struct ir_insn *new_insn = + create_assign_insn_base_cg(env, pos_insn->parent_bb, val); + bpf_ir_insert_at(new_insn, pos_insn, pos); return new_insn; } -struct ir_insn *create_phi_insn(struct ir_insn *insn, enum insert_position pos) +struct ir_insn *bpf_ir_create_assign_insn_bb_cg(struct bpf_ir_env *env, + struct ir_basic_block *pos_bb, + struct ir_value val, + enum insert_position pos) { - struct ir_insn *new_insn = create_phi_insn_base(insn->parent_bb); - insert_at(new_insn, insn, pos); + struct ir_insn *new_insn = create_assign_insn_base_cg(env, pos_bb, val); + bpf_ir_insert_at_bb(new_insn, pos_bb, pos); return new_insn; } -struct ir_insn *create_phi_insn_bb(struct ir_basic_block *bb, - enum insert_position pos) +struct ir_insn *bpf_ir_create_phi_insn(struct bpf_ir_env *env, + struct ir_insn *pos_insn, + enum insert_position pos) { - struct ir_insn *new_insn = create_phi_insn_base(bb); - insert_at_bb(new_insn, bb, pos); + struct ir_insn *new_insn = + create_phi_insn_base(env, pos_insn->parent_bb); + bpf_ir_insert_at(new_insn, pos_insn, pos); return new_insn; } -void phi_add_operand(struct bpf_ir_env *env, struct ir_insn *insn, - struct ir_basic_block *bb, struct ir_value val) +struct ir_insn *bpf_ir_create_phi_insn_bb(struct bpf_ir_env *env, + struct ir_basic_block *pos_bb, + enum insert_position pos) { - // Make sure that bb is a pred of insn parent BB - struct phi_value pv; - pv.value = val; - pv.bb = bb; - bpf_ir_array_push(env, &insn->phi, &pv); - val_add_user(env, val, insn); + struct ir_insn *new_insn = create_phi_insn_base(env, pos_bb); + bpf_ir_insert_at_bb(new_insn, pos_bb, pos); + return new_insn; } + +/* Generated Constructors */ diff --git a/IR/ir_value.c b/IR/ir_value.c index 46d9e691..7f8c5b32 100644 --- a/IR/ir_value.c +++ b/IR/ir_value.c @@ -1,9 +1,9 @@ #include -u8 bpf_ir_value_equal(struct ir_value a, struct ir_value b) +bool bpf_ir_value_equal(struct ir_value a, struct ir_value b) { if (a.type != b.type) { - return 0; + return false; } if (a.type == IR_VALUE_CONSTANT) { return a.data.constant_d == b.data.constant_d; @@ -11,9 +11,6 @@ u8 bpf_ir_value_equal(struct ir_value a, struct ir_value b) if (a.type == IR_VALUE_INSN) { return a.data.insn_d == b.data.insn_d; } - if (a.type == IR_VALUE_STACK_PTR) { - return 1; - } CRITICAL("Error"); } @@ -22,7 +19,39 @@ struct ir_value bpf_ir_value_insn(struct ir_insn *insn) return (struct ir_value){ .type = IR_VALUE_INSN, .data.insn_d = insn }; } -struct ir_value bpf_ir_value_stack_ptr(void) +struct ir_value bpf_ir_value_undef(void) +{ + return (struct ir_value){ .type = IR_VALUE_UNDEF }; +} + +struct ir_value bpf_ir_value_const32(s32 val) +{ + return (struct ir_value){ .type = IR_VALUE_CONSTANT, + .data.constant_d = val, + .const_type = IR_ALU_32 }; +} + +struct ir_value bpf_ir_value_const64(s64 val) +{ + return (struct ir_value){ .type = IR_VALUE_CONSTANT, + .data.constant_d = val, + .const_type = IR_ALU_64 }; +} + +struct ir_address_value bpf_ir_addr_val(struct ir_value value, s16 offset) +{ + return (struct ir_address_value){ .value = value, .offset = offset }; +} + +struct ir_value bpf_ir_value_stack_ptr(struct ir_function *fun) +{ + return bpf_ir_value_insn(fun->sp); +} + +void bpf_ir_change_value(struct bpf_ir_env *env, struct ir_insn *insn, + struct ir_value *old, struct ir_value new) { - return (struct ir_value){ .type = IR_VALUE_STACK_PTR }; + bpf_ir_val_remove_user(*old, insn); + *old = new; + bpf_ir_val_add_user(env, new, insn); } diff --git a/IR/passes/add_constraint_pass.c b/IR/passes/add_constraint_pass.c index 4de32830..feef77b2 100644 --- a/IR/passes/add_constraint_pass.c +++ b/IR/passes/add_constraint_pass.c @@ -14,7 +14,7 @@ void add_constraint(struct bpf_ir_env *env, struct ir_function *fun) struct ir_value val; val.type = IR_VALUE_CONSTANT; val.data.constant_d = 1; - create_ret_insn_bb(env, err_bb, val, INSERT_BACK); + bpf_ir_create_ret_insn_bb(env, err_bb, val, INSERT_BACK); struct ir_constraint *pos; array_for(pos, fun->value_constraints) @@ -23,22 +23,22 @@ void add_constraint(struct bpf_ir_env *env, struct ir_function *fun) if (c.type == CONSTRAINT_TYPE_VALUE_EQUAL) { struct ir_basic_block *newbb = bpf_ir_split_bb(env, fun, c.pos); - create_jbin_insn(env, c.pos, c.val, c.cval, newbb, - err_bb, IR_INSN_JNE, IR_ALU_64, - INSERT_FRONT); + bpf_ir_create_jbin_insn(env, c.pos, c.val, c.cval, + newbb, err_bb, IR_INSN_JNE, + IR_ALU_64, INSERT_FRONT); bpf_ir_connect_bb(env, c.pos->parent_bb, err_bb); } else if (c.type == CONSTRAINT_TYPE_VALUE_RANGE) { struct ir_basic_block *newbb = bpf_ir_split_bb(env, fun, c.pos); - create_jbin_insn(env, c.pos, c.val, c.start, newbb, - err_bb, IR_INSN_JLT, IR_ALU_64, - INSERT_FRONT); + bpf_ir_create_jbin_insn(env, c.pos, c.val, c.start, + newbb, err_bb, IR_INSN_JLT, + IR_ALU_64, INSERT_FRONT); bpf_ir_connect_bb(env, c.pos->parent_bb, err_bb); struct ir_basic_block *newbb2 = bpf_ir_split_bb(env, fun, c.pos); - create_jbin_insn(env, c.pos, c.val, c.end, newbb2, - err_bb, IR_INSN_JGE, IR_ALU_64, - INSERT_FRONT); + bpf_ir_create_jbin_insn(env, c.pos, c.val, c.end, + newbb2, err_bb, IR_INSN_JGE, + IR_ALU_64, INSERT_FRONT); bpf_ir_connect_bb(env, c.pos->parent_bb, err_bb); } else { CRITICAL("Error"); diff --git a/IR/passes/add_counter_pass.c b/IR/passes/add_counter_pass.c index a588a5c0..bbe427c6 100644 --- a/IR/passes/add_counter_pass.c +++ b/IR/passes/add_counter_pass.c @@ -1,21 +1,44 @@ #include +#define MAX_RUN_INSN 1000 + void add_counter(struct bpf_ir_env *env, struct ir_function *fun) { struct ir_basic_block *entry = fun->entry; - struct ir_insn *alloc_insn = - create_alloc_insn_bb(entry, IR_VR_TYPE_64, INSERT_FRONT); - struct ir_value val; - val.type = IR_VALUE_CONSTANT; - val.data.constant_d = 0; - val.const_type = IR_ALU_64; - create_store_insn(env, alloc_insn, alloc_insn, val, INSERT_BACK); + struct ir_insn *alloc_insn = bpf_ir_create_alloc_insn_bb( + env, entry, IR_VR_TYPE_32, INSERT_FRONT); + bpf_ir_create_store_insn(env, alloc_insn, alloc_insn, + bpf_ir_value_const32(0), INSERT_BACK); struct ir_basic_block **pos; struct ir_basic_block *err_bb = bpf_ir_create_bb(env, fun); - val.data.constant_d = 1; - val.const_type = IR_ALU_32; - create_ret_insn_bb(env, err_bb, val, INSERT_BACK); + bpf_ir_create_ret_insn_bb(env, err_bb, bpf_ir_value_const32(1), + INSERT_BACK); + + // Create an 8 bytes array to store the error message "exit" + struct ir_insn *alloc_array = bpf_ir_create_allocarray_insn_bb( + env, err_bb, IR_VR_TYPE_64, 1, INSERT_FRONT); + + struct ir_insn *straw1 = bpf_ir_create_storeraw_insn( + env, alloc_array, IR_VR_TYPE_8, + bpf_ir_addr_val(bpf_ir_value_insn(alloc_array), 0x4), + bpf_ir_value_const32(0), INSERT_BACK); + + struct ir_insn *straw2 = bpf_ir_create_storeraw_insn( + env, straw1, IR_VR_TYPE_32, + bpf_ir_addr_val(bpf_ir_value_insn(alloc_array), 0), + bpf_ir_value_const32(0x74697865), INSERT_BACK); + + struct ir_insn *elemptr = bpf_ir_create_getelemptr_insn( + env, straw2, alloc_array, bpf_ir_value_const32(0), INSERT_BACK); + + struct ir_insn *call_insn = + bpf_ir_create_call_insn(env, elemptr, 6, + INSERT_BACK); // A printk call + + bpf_ir_phi_add_call_arg(env, call_insn, bpf_ir_value_insn(elemptr)); + + bpf_ir_phi_add_call_arg(env, call_insn, bpf_ir_value_const32(5)); array_for(pos, fun->reachable_bbs) { @@ -30,28 +53,22 @@ void add_counter(struct bpf_ir_env *env, struct ir_function *fun) // No insn in the bb continue; } - struct ir_insn *load_insn = create_load_insn( + struct ir_insn *load_insn = bpf_ir_create_load_insn( env, last, bpf_ir_value_insn(alloc_insn), INSERT_FRONT); - struct ir_value val1; - val1.type = IR_VALUE_CONSTANT; - val1.data.constant_d = len; - val1.const_type = IR_ALU_32; - struct ir_value val2; - val2.type = IR_VALUE_INSN; - val2.data.insn_d = load_insn; - struct ir_insn *added = create_bin_insn(env, load_insn, val1, - val2, IR_INSN_ADD, - IR_ALU_64, INSERT_BACK); - val.data.insn_d = added; - val.type = IR_VALUE_INSN; - struct ir_insn *store_back = create_store_insn( - env, added, alloc_insn, val, INSERT_BACK); + struct ir_insn *added = bpf_ir_create_bin_insn( + env, load_insn, bpf_ir_value_insn(load_insn), + bpf_ir_value_const32(len), IR_INSN_ADD, IR_ALU_64, + INSERT_BACK); + struct ir_insn *store_back = bpf_ir_create_store_insn( + env, added, alloc_insn, bpf_ir_value_insn(added), + INSERT_BACK); struct ir_basic_block *new_bb = bpf_ir_split_bb(env, fun, store_back); - val2.data.insn_d = added; - val1.data.constant_d = 0x10000; - create_jbin_insn(env, store_back, val2, val1, new_bb, err_bb, - IR_INSN_JGT, IR_ALU_64, INSERT_BACK); + bpf_ir_create_jbin_insn(env, store_back, + bpf_ir_value_insn(added), + bpf_ir_value_const32(MAX_RUN_INSN), + new_bb, err_bb, IR_INSN_JGT, IR_ALU_64, + INSERT_BACK); // Manually connect BBs bpf_ir_connect_bb(env, bb, err_bb); } diff --git a/IR/passes/phi_pass.c b/IR/passes/phi_pass.c index 134c15b0..1c3ca6ae 100644 --- a/IR/passes/phi_pass.c +++ b/IR/passes/phi_pass.c @@ -32,6 +32,7 @@ void try_remove_trivial_phi(struct bpf_ir_env *env, struct ir_insn *phi) bpf_ir_replace_all_usage_except(env, phi, same, phi); bpf_ir_erase_insn(env, phi); + CHECK_ERR(); } void remove_trivial_phi(struct bpf_ir_env *env, struct ir_function *fun) @@ -44,6 +45,7 @@ void remove_trivial_phi(struct bpf_ir_env *env, struct ir_function *fun) list_for_each_entry_safe(pos, tmp, &bb->ir_insn_head, list_ptr) { try_remove_trivial_phi(env, pos); + CHECK_ERR(); } } } diff --git a/IR/scripts/format.sh b/IR/scripts/format.sh index e9ec00b4..dff29e4f 100755 --- a/IR/scripts/format.sh +++ b/IR/scripts/format.sh @@ -3,10 +3,12 @@ files=$(find . -iname '*.h' -o -iname '*.c' -not -path "./build/*") for file in $files; do - echo "Formatting $file" + # echo "Formatting $file" clang-format -i $file & done for job in `jobs -p`; do wait $job done + +echo "Formatting done" diff --git a/IR/scripts/gen_insn_ctor.py b/IR/scripts/gen_insn_ctor.py new file mode 100755 index 00000000..de9fa011 --- /dev/null +++ b/IR/scripts/gen_insn_ctor.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python3 + +# This script generates the constructor for the instruction class from base constructor + +import regex as re + + +def handle_ir(matches, header, src): + # print(matches) + for insn,extra, args in matches: + args = args.split(",") + args = [arg.strip() for arg in args] + if len(args) < 2: + print("Error: wrong arguments") + continue + if args[0] != "struct bpf_ir_env *env": + print("Error: First argument is not struct bpf_ir_env *env") + continue + if args[1] != "struct ir_basic_block *bb": + print("Error: Second argument is not struct ir_basic_block *bb") + continue + del args[0] + del args[0] + # print(args) + cargs = [] + rec = r".*[ \*](.*)$" + rec = re.compile(rec) + for arg in args: + res = rec.findall(arg) + cargs.append(res[0]) + + pargs = ", ".join(args) + cargs = ", ".join(cargs) + if len(pargs) > 0: + pargs = pargs + "," + if len(cargs) > 0: + cargs = "," +cargs + ir_fun = f""" +struct ir_insn *bpf_ir_{insn}{extra}(struct bpf_ir_env *env, struct ir_insn *pos_insn, {pargs} enum insert_position pos) +{{ + struct ir_insn *new_insn = + {insn}_base{extra}(env, pos_insn->parent_bb {cargs}); + bpf_ir_insert_at(new_insn, pos_insn, pos); + return new_insn; +}} + +struct ir_insn *bpf_ir_{insn}_bb{extra}(struct bpf_ir_env *env, struct ir_basic_block *pos_bb, {pargs} enum insert_position pos) +{{ + struct ir_insn *new_insn = + {insn}_base{extra}(env, pos_bb {cargs}); + bpf_ir_insert_at_bb(new_insn, pos_bb, pos); + return new_insn; +}} +""" + src.append(ir_fun) + + ir_fun_h = f""" +struct ir_insn *bpf_ir_{insn}{extra}(struct bpf_ir_env *env, struct ir_insn *pos_insn, {pargs} enum insert_position pos); + +struct ir_insn *bpf_ir_{insn}_bb{extra}(struct bpf_ir_env *env, struct ir_basic_block *pos_bb, {pargs} enum insert_position pos); +""" + header.append(ir_fun_h) + +def insert(header, src): + srcfile = "" + with open("ir_insn.c") as f: + srcfile = f.read().split("/* Generated Constructors */") + with open("ir_insn.c", "w") as f: + f.write(srcfile[0]) + f.write("/* Generated Constructors */\n") + f.write(src) + f.write("\n/* Generated Constructors */") + f.write(srcfile[2]) + headerfile = "" + with open("include/linux/bpf_ir.h") as f: + headerfile = f.read().split("/* Instruction Constructors */") + with open("include/linux/bpf_ir.h", "w") as f: + f.write(headerfile[0]) + f.write("/* Instruction Constructors */\n") + f.write(header) + f.write("\n/* Instruction Constructors */") + f.write(headerfile[2]) + + +def main(): + header = [] + src = [] + proto = "" + with open("ir_insn.c") as f: + proto = f.read() + regc = r"static struct ir_insn[\*\s]*?(create_.*?_insn)_base(.*?)\(([\s\S]*?)\)" + regc = re.compile(regc) + all_matches = regc.findall(proto) + handle_ir(all_matches, header, src) + header = "".join(header) + src = "".join(src) + # print(src) + insert(header, src) + + +main() diff --git a/IR/tests/Makefile b/IR/tests/Makefile index 6e46f2c2..18dc9b75 100644 --- a/IR/tests/Makefile +++ b/IR/tests/Makefile @@ -1,8 +1,8 @@ $(NAME).o: $(NAME).c - clang -O2 -I/usr/include/$(shell uname -m)-linux-gnu -g -target bpf -c $< -o $@ + clang -O2 -I/usr/include/$(shell uname -m)-linux-gnu -target bpf -c $< -o $@ $(NAME).nop.o: $(NAME).c - clang -O0 -I/usr/include/$(shell uname -m)-linux-gnu -g -target bpf -c $< -o $@ + clang -O0 -I/usr/include/$(shell uname -m)-linux-gnu -target bpf -c $< -o $@ $(NAME).nop.s: $(NAME).nop.o llvm-objdump -S $< > $@ diff --git a/IR/tests/loop2.c b/IR/tests/loop2.c index 436cba61..fb0e3c7e 100644 --- a/IR/tests/loop2.c +++ b/IR/tests/loop2.c @@ -4,11 +4,18 @@ SEC("xdp") int prog(void *ctx) { - bpf_ktime_get_ns(); - for (__u64 i = 0; i < 1000; ++i) { - bpf_ktime_get_ns(); + __u64 i = 0; + while (i < 10000) { + __u64 a = bpf_ktime_get_ns(); + __u64 b = bpf_ktime_get_ns(); + if (a > b) { + break; + } + ++i; } - return 0; + static char msg[] = "finished: %d"; + bpf_trace_printk(msg, sizeof(msg), i); + return XDP_PASS; } char _license[] SEC("license") = "GPL"; diff --git a/IR/tests/loop3.c b/IR/tests/loop3.c new file mode 100644 index 00000000..31c0d008 --- /dev/null +++ b/IR/tests/loop3.c @@ -0,0 +1,13 @@ +#include +#include + +SEC("xdp") +int prog(void *ctx) +{ + for (__u64 i = 0; i < 1000; ++i) { + bpf_ktime_get_ns(); + } + return XDP_PASS; +} + +char _license[] SEC("license") = "GPL"; diff --git a/IR/userspace/print.c b/IR/userspace/print.c index db5e018f..b1b064fa 100644 --- a/IR/userspace/print.c +++ b/IR/userspace/print.c @@ -3,14 +3,14 @@ int print(const struct bpf_insn *insns, size_t len) { - for (int i = 0; i < len; ++i) { + for (size_t i = 0; i < len; ++i) { const struct bpf_insn *insn = &insns[i]; // printf("insn[%d]: code=%x, dst_reg=%x, src_reg=%x, off=%x, imm=%x\n", // i, insn->code, insn->dst_reg, insn->src_reg, insn->off, // insn->imm); __u64 data; memcpy(&data, insn, sizeof(struct bpf_insn)); - printf("insn[%d]: %llu\n", i, data); + printf("insn[%zu]: %llu\n", i, data); } return 0; } diff --git a/IR/userspace/printlog.c b/IR/userspace/printlog.c index e1532d04..14f718b7 100644 --- a/IR/userspace/printlog.c +++ b/IR/userspace/printlog.c @@ -5,9 +5,9 @@ int print(const struct bpf_insn *insns, size_t len) { - for (int i = 0; i < len; ++i) { + for (size_t i = 0; i < len; ++i) { const struct bpf_insn *insn = &insns[i]; - printf("insn[%d]: code=%x, dst_reg=%x, src_reg=%x, off=%x, imm=%x\n", + printf("insn[%zu]: code=%x, dst_reg=%x, src_reg=%x, off=%x, imm=%x\n", i, insn->code, insn->dst_reg, insn->src_reg, insn->off, insn->imm); // u64 data; diff --git a/IR/userspace/read.c b/IR/userspace/read.c index f640faca..0398bdb9 100644 --- a/IR/userspace/read.c +++ b/IR/userspace/read.c @@ -15,7 +15,11 @@ int main(int argn, char **argv) } size_t sz = bpf_program__insn_cnt(prog); const struct bpf_insn *insn = bpf_program__insns(prog); - struct bpf_ir_env *env = bpf_ir_init_env(); + struct ir_opts opts = { + .debug = 1, + .print_mode = BPF_IR_PRINT_BPF, + }; + struct bpf_ir_env *env = bpf_ir_init_env(opts); if (!env) { return 1; } diff --git a/IR/userspace/readlog.c b/IR/userspace/readlog.c index 8b4556c4..f27ee897 100644 --- a/IR/userspace/readlog.c +++ b/IR/userspace/readlog.c @@ -37,7 +37,11 @@ int main(int argc, char **argv) } printf("Loaded program of size %zu\n", index); - struct bpf_ir_env *env = bpf_ir_init_env(); + struct ir_opts opts = { + .debug = 1, + .print_mode = BPF_IR_PRINT_BPF, + }; + struct bpf_ir_env *env = bpf_ir_init_env(opts); if (!env) { return 1; } diff --git a/cli/.gitignore b/cli/.gitignore new file mode 100644 index 00000000..9f970225 --- /dev/null +++ b/cli/.gitignore @@ -0,0 +1 @@ +target/ \ No newline at end of file diff --git a/cli/Cargo.lock b/cli/Cargo.lock new file mode 100644 index 00000000..cf63013f --- /dev/null +++ b/cli/Cargo.lock @@ -0,0 +1,101 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "bitflags" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de" + +[[package]] +name = "cc" +version = "1.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b62ac837cdb5cb22e10a256099b4fc502b1dfe560cb282963a974d7abd80e476" +dependencies = [ + "shlex", +] + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "cfg_aliases" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" + +[[package]] +name = "cli" +version = "0.1.0" +dependencies = [ + "cc", + "libbpf-rs", +] + +[[package]] +name = "libbpf-rs" +version = "0.24.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d51b943363864472bf306570a7076cbfa84571a403b98f59d18af4c4135271ca" +dependencies = [ + "bitflags", + "libbpf-sys", + "libc", + "vsprintf", +] + +[[package]] +name = "libbpf-sys" +version = "1.4.5+v1.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5cabee52b6f7e73308d6fd4f8e6bbbdcb97670f49f6e581c5897e4d2410b6019" +dependencies = [ + "cc", + "nix", + "pkg-config", +] + +[[package]] +name = "libc" +version = "0.2.158" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8adc4bb1803a324070e64a98ae98f38934d91957a99cfb3a43dcbc01bc56439" + +[[package]] +name = "nix" +version = "0.29.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46" +dependencies = [ + "bitflags", + "cfg-if", + "cfg_aliases", + "libc", +] + +[[package]] +name = "pkg-config" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec" + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "vsprintf" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aec2f81b75ca063294776b4f7e8da71d1d5ae81c2b1b149c8d89969230265d63" +dependencies = [ + "cc", + "libc", +] diff --git a/cli/Cargo.toml b/cli/Cargo.toml new file mode 100644 index 00000000..3d5ead27 --- /dev/null +++ b/cli/Cargo.toml @@ -0,0 +1,10 @@ +[package] +name = "cli" +version = "0.1.0" +edition = "2021" + +[dependencies] +libbpf-rs = "0.24" + +[build-dependencies] +cc = "1.1.18" diff --git a/cli/build.rs b/cli/build.rs new file mode 100644 index 00000000..0d659070 --- /dev/null +++ b/cli/build.rs @@ -0,0 +1,35 @@ +const SRC_FILES: &[&str] = &[ + "bpf_ir.c", + "array.c", + "ir_helper.c", + "ir_value.c", + "ir_bb.c", + "ir_insn.c", + "passes/phi_pass.c", + "passes/add_counter_pass.c", + "passes/add_constraint_pass.c", + "passes/cut_bb_pass.c", + "aux/prog_check.c", + "aux/disasm.c", + "ir_code_gen.c", + "lii.c", +]; + +const SRC_DIR: &str = "../IR/"; + +const SRC_INCLUDE: &str = "../IR/include/"; + +fn main() { + // println!("cargo:rustc-link-search=../IR/build/"); + for src_file in SRC_FILES { + println!("cargo:rerun-if-changed={}", format!("{}{}", SRC_DIR, src_file)); + } + println!("cargo:rerun-if-changed={}", format!("{}{}", SRC_INCLUDE, "linux/bpf_ir.h")); + + let mut builder = cc::Build::new(); + builder.include(SRC_INCLUDE); + for src_file in SRC_FILES { + builder.file(format!("{}{}", SRC_DIR, src_file)); + } + builder.compile("hello"); +} diff --git a/cli/src/main.rs b/cli/src/main.rs new file mode 100644 index 00000000..6c50012f --- /dev/null +++ b/cli/src/main.rs @@ -0,0 +1,35 @@ +use libbpf_rs::{libbpf_sys::bpf_insn, ObjectBuilder}; +use std::ffi::{c_char, c_int, c_ulong}; + +#[repr(C)] +pub struct BpfIrEnv { + err: c_int, + insn_cnt: c_ulong, + log: *mut c_char, + insns: *mut bpf_insn, + log_pos: c_ulong, +} + +extern "C" { + pub fn bpf_ir_print_log_dbg(env: *mut BpfIrEnv); + pub fn bpf_ir_init_env() -> *mut BpfIrEnv; + pub fn bpf_ir_free_env(env: *mut BpfIrEnv); + pub fn bpf_ir_run(env: *mut BpfIrEnv, insns: *const bpf_insn, insn_cnt: c_ulong); +} + +fn main() { + let mut builder = ObjectBuilder::default(); + let obj = builder + .open_file("../IR/tests/loop1.o") + .expect("Failed to open object file"); + let prog = obj + .progs() + .find(|map| map.name() == "prog") + .expect("Failed to find program"); + unsafe { + let env = bpf_ir_init_env(); + bpf_ir_run(env, prog.insns().as_ptr(), prog.insns().len() as c_ulong); + bpf_ir_print_log_dbg(env); + bpf_ir_free_env(env); + } +}