diff --git a/core/.vscode/launch.json b/core/.vscode/launch.json index c9d4d084..8f211ad4 100644 --- a/core/.vscode/launch.json +++ b/core/.vscode/launch.json @@ -11,7 +11,9 @@ "program": "${workspaceFolder}/build/epasstool/epass", "args": [ "read", - "bpftests/output/empty.o" + "bpftests/output/loop1.o", + "--gopt", + "verbose=3,cgv2" ], "cwd": "${workspaceFolder}" } diff --git a/core/CMakeLists.txt b/core/CMakeLists.txt index 14cc2625..52fadc2e 100644 --- a/core/CMakeLists.txt +++ b/core/CMakeLists.txt @@ -30,7 +30,9 @@ add_library( aux/disasm.c aux/kern_utils.c ir_cg.c + ir_cg_v2.c ir_cg_norm.c + ir_cg_norm_v2.c lli.c include/linux/bpf_ir.h) diff --git a/core/aux/kern_utils.c b/core/aux/kern_utils.c index 1dbeae08..4d594783 100644 --- a/core/aux/kern_utils.c +++ b/core/aux/kern_utils.c @@ -109,6 +109,8 @@ static int apply_global_opt(struct bpf_ir_env *env, const char *opt) env->opts.enable_printk_log = true; } else if (strcmp(opt, "throw_msg") == 0) { env->opts.enable_throw_msg = true; + } else if (strcmp(opt, "cgv2") == 0) { + env->opts.cg_v2 = true; } else if (strncmp(opt, "verbose=", 8) == 0) { int res = 0; int err = parse_int(opt + 8, &res); diff --git a/core/bpf_ir.c b/core/bpf_ir.c index 058a1e5a..468ebeb5 100644 --- a/core/bpf_ir.c +++ b/core/bpf_ir.c @@ -1411,6 +1411,7 @@ void bpf_ir_free_function(struct ir_function *fun) bpf_ir_array_free(&fun->reachable_bbs); bpf_ir_array_free(&fun->end_bbs); bpf_ir_array_free(&fun->cg_info.all_var); + bpf_ir_ptrset_free(&fun->cg_info.all_var_v2); } static void init_function(struct bpf_ir_env *env, struct ir_function *fun, @@ -1426,6 +1427,7 @@ static void init_function(struct bpf_ir_env *env, struct ir_function *fun, INIT_ARRAY(&fun->reachable_bbs, struct ir_basic_block *); INIT_ARRAY(&fun->end_bbs, struct ir_basic_block *); INIT_ARRAY(&fun->cg_info.all_var, struct ir_insn *); + INIT_PTRSET_DEF(&fun->cg_info.all_var_v2); for (size_t i = 0; i < MAX_BPF_REG; ++i) { struct array *currentDef = &tenv->currentDef[i]; bpf_ir_array_free(currentDef); @@ -1614,19 +1616,19 @@ static void run_single_pass(struct bpf_ir_env *env, struct ir_function *fun, CHECK_ERR(); } -void bpf_ir_run(struct bpf_ir_env *env, struct ir_function *fun) +void bpf_ir_run_passes(struct bpf_ir_env *env, struct ir_function *fun, + const struct function_pass *passes, const size_t cnt) { - u64 starttime = get_cur_time_ns(); - for (size_t i = 0; i < pre_passes_cnt; ++i) { + for (size_t i = 0; i < cnt; ++i) { bool has_override = false; for (size_t j = 0; j < env->opts.builtin_pass_cfg_num; ++j) { if (strcmp(env->opts.builtin_pass_cfg[j].name, - pre_passes[i].name) == 0) { + passes[i].name) == 0) { has_override = true; - if (pre_passes[i].force_enable || + if (passes[i].force_enable || env->opts.builtin_pass_cfg[j].enable) { run_single_pass( - env, fun, &pre_passes[i], + env, fun, &passes[i], env->opts.builtin_pass_cfg[j] .param); } @@ -1634,13 +1636,19 @@ void bpf_ir_run(struct bpf_ir_env *env, struct ir_function *fun) } } if (!has_override) { - if (pre_passes[i].enabled) { - run_single_pass(env, fun, &pre_passes[i], NULL); + if (passes[i].enabled) { + run_single_pass(env, fun, &passes[i], NULL); } } CHECK_ERR(); } +} + +void bpf_ir_run(struct bpf_ir_env *env, struct ir_function *fun) +{ + u64 starttime = get_cur_time_ns(); + bpf_ir_run_passes(env, fun, pre_passes, pre_passes_cnt); for (size_t i = 0; i < env->opts.custom_pass_num; ++i) { if (env->opts.custom_passes[i].pass.enabled) { if (env->opts.custom_passes[i].check_apply) { @@ -1662,30 +1670,7 @@ void bpf_ir_run(struct bpf_ir_env *env, struct ir_function *fun) CHECK_ERR(); } } - for (size_t i = 0; i < post_passes_cnt; ++i) { - bool has_override = false; - for (size_t j = 0; j < env->opts.builtin_pass_cfg_num; ++j) { - if (strcmp(env->opts.builtin_pass_cfg[j].name, - post_passes[i].name) == 0) { - has_override = true; - if (post_passes[i].force_enable || - env->opts.builtin_pass_cfg[j].enable) { - run_single_pass( - env, fun, &post_passes[i], - env->opts.builtin_pass_cfg[j] - .param); - } - break; - } - } - if (!has_override) { - if (post_passes[i].enabled) { - run_single_pass(env, fun, &post_passes[i], - NULL); - } - } - CHECK_ERR(); - } + bpf_ir_run_passes(env, fun, post_passes, post_passes_cnt); env->run_time += get_cur_time_ns() - starttime; } @@ -1825,6 +1810,7 @@ struct bpf_ir_opts bpf_ir_default_opts(void) opts.enable_coalesce = false; opts.force = false; opts.verbose = 1; + opts.cg_v2 = false; opts.max_iteration = 10; opts.disable_prog_check = false; opts.enable_throw_msg = false; diff --git a/core/epasstool/epasstool.c b/core/epasstool/epasstool.c index 0119083d..fb3e50f9 100644 --- a/core/epasstool/epasstool.c +++ b/core/epasstool/epasstool.c @@ -15,16 +15,6 @@ static struct function_pass post_passes_def[] = { DEF_FUNC_PASS(bpf_ir_div_by_zero, "div_by_zero", false), DEF_FUNC_PASS(msan, "msan", false), DEF_FUNC_PASS(insn_counter, "insn_counter", false), - /* CG Preparation Passes */ - DEF_NON_OVERRIDE_FUNC_PASS(translate_throw, "translate_throw"), - DEF_FUNC_PASS(bpf_ir_optimize_code_compaction, "optimize_compaction", - false), - DEF_NON_OVERRIDE_FUNC_PASS(bpf_ir_optimize_ir, "optimize_ir"), - DEF_NON_OVERRIDE_FUNC_PASS(bpf_ir_cg_change_fun_arg, "change_fun_arg"), - DEF_NON_OVERRIDE_FUNC_PASS(bpf_ir_cg_change_call_pre_cg, "change_call"), - DEF_NON_OVERRIDE_FUNC_PASS(bpf_ir_cg_add_stack_offset_pre_cg, - "add_stack_offset"), - DEF_NON_OVERRIDE_FUNC_PASS(bpr_ir_cg_to_cssa, "to_cssa"), }; const struct function_pass *pre_passes = pre_passes_def; diff --git a/core/include/ir_cg.h b/core/include/ir_cg.h index 5be9c85b..ae27e3a3 100644 --- a/core/include/ir_cg.h +++ b/core/include/ir_cg.h @@ -3,11 +3,18 @@ #include +// Number of colors available (r0 - r9) +#define RA_COLORS 10 + void bpf_ir_init_insn_cg(struct bpf_ir_env *env, struct ir_insn *insn); void bpf_ir_init_insn_norm(struct bpf_ir_env *env, struct ir_insn *insn, struct ir_vr_pos pos); +void bpf_ir_cg_norm_v2(struct bpf_ir_env *env, struct ir_function *fun); + +void bpf_ir_init_insn_cg_v2(struct bpf_ir_env *env, struct ir_insn *insn); + void bpf_ir_free_insn_cg(struct ir_insn *insn); // Extra information needed for code gen @@ -32,6 +39,7 @@ struct ir_insn_cg_extra { struct ir_value dst; // Liveness analysis + // Array of struct ir_insn* struct array in; struct array out; struct array gen; @@ -64,6 +72,25 @@ struct ir_insn_cg_extra { bool nonvr; }; +struct ir_insn_cg_extra_v2 { + struct ir_insn *dst; + + // Liveness analysis + struct ptrset in; + struct ptrset out; + + // Adj list in interference graph + struct ptrset adj; + + u32 lambda; // used in MCS + u32 w; // number of maximalCl that has this vertex. used in pre-spill + + struct ir_vr_pos vr_pos; + + // Whether this instruction is a non-VR instruction, like a pre-colored register + bool nonvr; +}; + enum val_type { UNDEF, REG, @@ -74,6 +101,8 @@ enum val_type { #define insn_cg(insn) ((struct ir_insn_cg_extra *)(insn)->user_data) +#define insn_cg_v2(insn) ((struct ir_insn_cg_extra_v2 *)(insn)->user_data) + /* Dst of a instruction Note. This could be only applied to an instruction with return value. diff --git a/core/include/linux/bpf_ir.h b/core/include/linux/bpf_ir.h index 6e3f1230..b55cf539 100644 --- a/core/include/linux/bpf_ir.h +++ b/core/include/linux/bpf_ir.h @@ -65,6 +65,9 @@ struct bpf_ir_opts { // Write an error message to trace when throwing an error bool enable_throw_msg; + // Use new CG pipeline; + bool cg_v2; + // Verbose level int verbose; @@ -200,6 +203,8 @@ void bpf_ir_array_clone(struct bpf_ir_env *env, struct array *res, #define INIT_ARRAY(arr, type) bpf_ir_array_init(arr, sizeof(type)) +#define INIT_PTRSET_DEF(set) bpf_ir_ptrset_init(env, set, 8) + /* Array End */ /* Hashtable Start */ @@ -281,6 +286,8 @@ void bpf_ir_ptrset_clean(struct ptrset *set); void bpf_ir_ptrset_free(struct ptrset *set); +void **bpf_ir_ptrset_next(struct ptrset *set, void **keyd); + struct ptrset bpf_ir_ptrset_union(struct bpf_ir_env *env, struct ptrset *set1, struct ptrset *set2); @@ -297,6 +304,10 @@ void bpf_ir_ptrset_add(struct bpf_ir_env *env, struct ptrset *set1, void bpf_ir_ptrset_minus(struct ptrset *set1, struct ptrset *set2); +#define ptrset_for(pos, set) \ + for (pos = (typeof(pos))bpf_ir_ptrset_next(&(set), NULL); pos; \ + pos = (typeof(pos))bpf_ir_ptrset_next(&(set), (void **)pos)) + /* Ptrset End */ /* DBG Macro Start */ @@ -368,6 +379,8 @@ int parse_int(const char *str, int *val); u64 get_cur_time_ns(void); +#ifdef DEBUG_ALLOC + #define SAFE_MALLOC(dst, size) \ { \ if (size > 10000000) { \ @@ -392,6 +405,28 @@ u64 get_cur_time_ns(void); } \ } +#else + +#define SAFE_MALLOC(dst, size) \ + { \ + dst = malloc_proto(size); \ + if (!dst) { \ + env->err = -ENOMEM; \ + return; \ + } \ + } + +#define SAFE_MALLOC_RET_NULL(dst, size) \ + { \ + dst = malloc_proto(size); \ + if (!dst) { \ + env->err = -ENOMEM; \ + return NULL; \ + } \ + } + +#endif + /* LLI End */ #define MAX_FUNC_ARG 5 @@ -778,6 +813,8 @@ struct code_gen_info { // Array of struct ir_insn* struct array all_var; + struct ptrset all_var_v2; + // BPF Register Virtual Instruction (used as dst) struct ir_insn *regs[BPF_REG_10]; // Only use R0-R9 @@ -1193,6 +1230,15 @@ struct ir_insn *bpf_ir_create_assign_insn_bb_norm(struct bpf_ir_env *env, struct ir_value val, enum insert_position pos); +struct ir_insn *bpf_ir_create_assign_insn_cg_v2(struct bpf_ir_env *env, + struct ir_insn *pos_insn, + struct ir_value val, + enum insert_position pos); + +struct ir_insn *bpf_ir_create_assign_insn_bb_cg_v2( + struct bpf_ir_env *env, struct ir_basic_block *pos_bb, + struct ir_value val, enum insert_position pos); + struct ir_insn *bpf_ir_create_phi_insn(struct bpf_ir_env *env, struct ir_insn *pos_insn, enum insert_position pos); @@ -1352,12 +1398,6 @@ void bpf_ir_div_by_zero(struct bpf_ir_env *env, struct ir_function *fun, void bpf_ir_optimize_code_compaction(struct bpf_ir_env *env, struct ir_function *fun, void *param); -extern const struct function_pass *pre_passes; -extern const size_t pre_passes_cnt; - -extern const struct function_pass *post_passes; -extern const size_t post_passes_cnt; - void translate_throw(struct bpf_ir_env *env, struct ir_function *fun, void *param); @@ -1369,6 +1409,15 @@ struct function_pass { char name[BPF_IR_MAX_PASS_NAME_SIZE]; }; +extern const struct function_pass *pre_passes; +extern const size_t pre_passes_cnt; + +extern const struct function_pass *post_passes; +extern const size_t post_passes_cnt; + +void bpf_ir_run_passes(struct bpf_ir_env *env, struct ir_function *fun, + const struct function_pass *passes, const size_t cnt); + struct custom_pass_cfg { struct function_pass pass; void *param; @@ -1430,6 +1479,8 @@ struct builtin_pass_cfg { void bpf_ir_compile(struct bpf_ir_env *env, struct ir_function *fun); +void bpf_ir_compile_v2(struct bpf_ir_env *env, struct ir_function *fun); + /* Code Gen End */ /* IR Value Start */ diff --git a/core/ir_cg.c b/core/ir_cg.c index 1dce8331..e8ccb8d2 100644 --- a/core/ir_cg.c +++ b/core/ir_cg.c @@ -2,6 +2,19 @@ #include #include "ir_cg.h" +/* CG Preparation Passes */ +static struct function_pass cg_init_passes[] = { + DEF_NON_OVERRIDE_FUNC_PASS(translate_throw, "translate_throw"), + DEF_FUNC_PASS(bpf_ir_optimize_code_compaction, "optimize_compaction", + false), + DEF_NON_OVERRIDE_FUNC_PASS(bpf_ir_optimize_ir, "optimize_ir"), + DEF_NON_OVERRIDE_FUNC_PASS(bpf_ir_cg_change_fun_arg, "change_fun_arg"), + DEF_NON_OVERRIDE_FUNC_PASS(bpf_ir_cg_change_call_pre_cg, "change_call"), + DEF_NON_OVERRIDE_FUNC_PASS(bpf_ir_cg_add_stack_offset_pre_cg, + "add_stack_offset"), + DEF_NON_OVERRIDE_FUNC_PASS(bpr_ir_cg_to_cssa, "to_cssa"), +}; + static void set_insn_dst(struct bpf_ir_env *env, struct ir_insn *insn, struct ir_insn *dst) { @@ -1893,7 +1906,15 @@ static void spill_array(struct bpf_ir_env *env, struct ir_function *fun) void bpf_ir_compile(struct bpf_ir_env *env, struct ir_function *fun) { + if (env->opts.cg_v2) { + bpf_ir_compile_v2(env, fun); + return; + } u64 starttime = get_cur_time_ns(); + + bpf_ir_run_passes(env, fun, cg_init_passes, + sizeof(cg_init_passes) / sizeof(cg_init_passes[0])); + // Init CG, start code generation init_cg(env, fun); CHECK_ERR(); diff --git a/core/ir_cg_norm_v2.c b/core/ir_cg_norm_v2.c new file mode 100644 index 00000000..af8674ce --- /dev/null +++ b/core/ir_cg_norm_v2.c @@ -0,0 +1,1236 @@ +// SPDX-License-Identifier: GPL-2.0-only +#include +#include "ir_cg.h" + +// Normalization + +static void bpf_ir_free_insn_cg_v2(struct ir_insn *insn) +{ + struct ir_insn_cg_extra_v2 *extra = insn_cg_v2(insn); + bpf_ir_ptrset_free(&extra->adj); + bpf_ir_ptrset_free(&extra->in); + bpf_ir_ptrset_free(&extra->out); + free_proto(extra); + insn->user_data = NULL; +} + +static enum val_type vtype(struct ir_value val) +{ + if (val.type == IR_VALUE_FLATTEN_DST) { + if (val.data.vr_pos.allocated) { + if (val.data.vr_pos.spilled) { + // WARNING: cannot determine whether it's a stackoff + return STACK; + } else { + return REG; + } + } else { + return UNDEF; + } + } else if (val.type == IR_VALUE_CONSTANT || + val.type == IR_VALUE_CONSTANT_RAWOFF || + val.type == IR_VALUE_CONSTANT_RAWOFF_REV) { + return CONST; + } else { + CRITICAL("No such value type for norm!"); + } +} + +static enum val_type vtype_insn_norm(struct ir_insn *insn) +{ + struct ir_insn_norm_extra *extra = insn_norm(insn); + if (extra->pos.allocated) { + if (extra->pos.spilled) { + // WARNING: cannot determine whether it's a stackoff + return STACK; + } else { + return REG; + } + } else { + return UNDEF; + } +} + +static void remove_all_users(struct ir_function *fun) +{ + struct ir_basic_block **pos; + array_for(pos, fun->reachable_bbs) + { + struct ir_basic_block *bb = *pos; + struct ir_insn *insn; + list_for_each_entry(insn, &bb->ir_insn_head, list_ptr) { + bpf_ir_array_free(&insn->users); + } + } + for (u8 i = 0; i < MAX_FUNC_ARG; ++i) { + bpf_ir_array_free(&fun->function_arg[i]->users); + } + if (fun->sp) { + bpf_ir_array_free(&fun->sp->users); + } + for (u8 i = 0; i < BPF_REG_10; ++i) { + struct ir_insn *insn = fun->cg_info.regs[i]; + bpf_ir_array_free(&insn->users); + } +} + +// To flatten IR, we first need to change all the values to ir_pos +static void change_all_value_to_ir_pos(struct bpf_ir_env *env, + struct ir_function *fun) +{ + struct ir_basic_block **pos; + array_for(pos, fun->reachable_bbs) + { + struct ir_basic_block *bb = *pos; + struct ir_insn *insn; + list_for_each_entry(insn, &bb->ir_insn_head, list_ptr) { + struct array operands = bpf_ir_get_operands(env, insn); + struct ir_value **pos2; + array_for(pos2, operands) + { + struct ir_value *v = *pos2; + if (v->type == IR_VALUE_INSN) { + struct ir_insn *insn_d = v->data.insn_d; + struct ir_insn *dst = + insn_cg_v2(insn_d)->dst; + struct ir_insn_cg_extra_v2 *extra = + insn_cg_v2(dst); + v->type = IR_VALUE_FLATTEN_DST; + v->data.vr_pos = extra->vr_pos; + } + } + } + } +} + +// Free CG resources, create a new extra data for flattening +static void cg_to_flatten(struct bpf_ir_env *env, struct ir_function *fun) +{ + struct ir_basic_block **pos = NULL; + array_for(pos, fun->reachable_bbs) + { + struct ir_basic_block *bb = *pos; + struct ir_insn *insn = NULL; + list_for_each_entry(insn, &bb->ir_insn_head, list_ptr) { + struct ir_vr_pos pos; + struct ir_insn_cg_extra_v2 *insn_extra = + insn_cg_v2(insn); + if (!insn_extra->dst) { + pos.allocated = false; + } else { + struct ir_insn_cg_extra_v2 *dst_extra = + insn_cg_v2(insn_extra->dst); + pos = dst_extra->vr_pos; + } + insn_cg_v2(insn)->vr_pos = pos; + } + } + + array_for(pos, fun->reachable_bbs) + { + struct ir_basic_block *bb = *pos; + struct ir_insn *insn = NULL; + list_for_each_entry(insn, &bb->ir_insn_head, list_ptr) { + struct ir_insn_cg_extra_v2 *extra = insn_cg_v2(insn); + struct ir_vr_pos pos = extra->vr_pos; + bpf_ir_free_insn_cg_v2(insn); + SAFE_MALLOC(insn->user_data, + sizeof(struct ir_insn_norm_extra)); + insn_norm(insn)->pos = pos; + } + } + + for (u8 i = 0; i < BPF_REG_10; ++i) { + struct ir_insn *insn = fun->cg_info.regs[i]; + bpf_ir_free_insn_cg_v2(insn); + } + bpf_ir_free_insn_cg_v2(fun->sp); +} + +static void cgir_load_stack_to_reg_norm(struct bpf_ir_env *env, + struct ir_insn *insn, + struct ir_value *val, + enum ir_vr_type vtype, + struct ir_vr_pos reg) +{ + struct ir_insn *tmp = bpf_ir_create_assign_insn_norm( + env, insn, reg, *val, INSERT_FRONT); + tmp->vr_type = vtype; + + *val = bpf_ir_value_vrpos(reg); +} + +/* Flatten IR */ +static void flatten_ir(struct bpf_ir_env *env, struct ir_function *fun) +{ + // Make sure no users + remove_all_users(fun); + change_all_value_to_ir_pos(env, fun); + CHECK_ERR(); + cg_to_flatten(env, fun); + CHECK_ERR(); +} + +/* Loading constant used in normalization */ +static struct ir_insn *normalize_load_const(struct bpf_ir_env *env, + struct ir_insn *insn, + struct ir_value *val, + struct ir_vr_pos dst) +{ + struct ir_insn *new_insn = NULL; + if (val->const_type == IR_ALU_32) { + new_insn = bpf_ir_create_assign_insn_norm(env, insn, dst, *val, + INSERT_FRONT); + new_insn->alu_op = IR_ALU_64; + } else { + new_insn = bpf_ir_create_loadimmextra_insn_norm( + env, insn, dst, IR_LOADIMM_IMM64, val->data.constant_d, + INSERT_FRONT); + new_insn->vr_type = IR_VR_TYPE_64; + } + *val = bpf_ir_value_vrpos(dst); + return new_insn; +} + +static void normalize_assign(struct ir_insn *insn) +{ + struct ir_value *v0 = &insn->values[0]; + enum val_type t0 = insn->value_num >= 1 ? vtype(*v0) : UNDEF; + enum val_type tdst = vtype_insn_norm(insn); + struct ir_vr_pos dst_pos = insn_norm(insn)->pos; + // stack = reg + // stack = const32 + // reg = const32 + // reg = const64 + // reg = stack + // reg = reg + if (tdst == STACK) { + DBGASSERT(t0 != STACK); + // Change to STORERAW + insn->op = IR_INSN_STORERAW; + + insn->addr_val.value = bpf_ir_value_norm_stack_ptr(); + insn->addr_val.offset = dst_pos.spilled; + insn->vr_type = + IR_VR_TYPE_64; // TODO: Should be 64 before normalize + } else { + if (t0 == STACK) { + // Change to LOADRAW + insn->op = IR_INSN_LOADRAW; + insn->addr_val.value = bpf_ir_value_norm_stack_ptr(); + insn->addr_val.offset = v0->data.vr_pos.spilled; + } + if (t0 == CONST && v0->const_type == IR_ALU_64) { + // 64 imm load + insn->op = IR_INSN_LOADIMM_EXTRA; + insn->imm_extra_type = IR_LOADIMM_IMM64; + insn->imm64 = v0->data.constant_d; + } + } + if (tdst == REG && t0 == REG) { + if (dst_pos.alloc_reg == v0->data.vr_pos.alloc_reg) { + // The same, erase this instruction + bpf_ir_erase_insn_norm(insn); + } + } +} + +/* Normalize ALU */ +static void normalize_alu(struct bpf_ir_env *env, struct ir_insn *insn) +{ + struct ir_value *v0 = &insn->values[0]; + struct ir_value *v1 = &insn->values[1]; + enum val_type t0 = insn->value_num >= 1 ? vtype(*v0) : UNDEF; + enum val_type t1 = insn->value_num >= 2 ? vtype(*v1) : UNDEF; + enum val_type tdst = vtype_insn_norm(insn); + DBGASSERT(tdst == REG); + struct ir_vr_pos dst_pos = insn_norm(insn)->pos; + if (t1 == REG) { + // tdst != t1 + DBGASSERT(dst_pos.alloc_reg != v1->data.vr_pos.alloc_reg); + } + if (t0 == CONST) { + DBGASSERT(v0->const_type == IR_ALU_32); + } + if (t1 == CONST) { + DBGASSERT(v1->const_type == IR_ALU_32); + } + // Binary ALU + if (t0 == STACK && t1 == CONST) { + // reg1 = add stack const + // ==> + // reg1 = stack + // reg1 = add reg1 const + struct ir_insn *new_insn = bpf_ir_create_assign_insn_norm( + env, insn, dst_pos, *v0, INSERT_FRONT); + new_insn->vr_type = IR_VR_TYPE_64; + *v0 = bpf_ir_value_vrpos(dst_pos); + normalize_assign(new_insn); + + } else if (t0 == STACK && t1 == REG) { + // reg1 = add stack reg2 + // ==> + // reg1 = stack + // reg1 = add reg1 reg2 + struct ir_insn *new_insn = bpf_ir_create_assign_insn_norm( + env, insn, dst_pos, *v0, INSERT_FRONT); + new_insn->vr_type = IR_VR_TYPE_64; + *v0 = bpf_ir_value_vrpos(dst_pos); + normalize_assign(new_insn); + } else if (t0 == REG && t1 == REG) { + // reg1 = add reg2 reg3 + u8 reg1 = dst_pos.alloc_reg; + u8 reg2 = v0->data.vr_pos.alloc_reg; + if (reg1 != reg2) { + // reg1 = add reg2 reg3 + // ==> + // reg1 = reg2 + // reg1 = add reg1 reg3 + bpf_ir_create_assign_insn_norm(env, insn, dst_pos, *v0, + INSERT_FRONT); + // DBGASSERT(dst_insn == + // fun->cg_info.regs[reg1]); // Fixed reg? + // TODO: Investigate here, why did I write this check? + *v0 = bpf_ir_value_vrpos(dst_pos); + } + } else if (t0 == REG && t1 == CONST) { + if (v0->data.vr_pos.alloc_reg != dst_pos.alloc_reg) { + // reg1 = add reg2 const + // ==> + // reg1 = reg2 + // reg1 = add reg1 const + bpf_ir_create_assign_insn_norm(env, insn, dst_pos, *v0, + INSERT_FRONT); + *v0 = bpf_ir_value_vrpos(dst_pos); + } + } else if (t0 == CONST && t1 == CONST) { + DBGASSERT(v1->const_type == IR_ALU_32); + normalize_load_const(env, insn, v0, dst_pos); + } else if (t0 == CONST && t1 == REG) { + // reg1 = add const reg2 + // ==> + // reg1 = const + // reg1 = add reg1 reg2 + normalize_load_const(env, insn, v0, dst_pos); + + } else { + CRITICAL_DUMP(env, "Error"); + } +} + +static void normalize_getelemptr(struct bpf_ir_env *env, struct ir_insn *insn) +{ + struct ir_value *v0 = &insn->values[0]; + struct ir_value *v1 = &insn->values[1]; + enum val_type t0 = insn->value_num >= 1 ? vtype(*v0) : UNDEF; + enum val_type t1 = insn->value_num >= 2 ? vtype(*v1) : UNDEF; + + DBGASSERT(t1 == STACK); + struct ir_vr_pos dstpos = insn_norm(insn)->pos; + DBGASSERT(dstpos.allocated && dstpos.spilled == 0); // dst must be reg + u8 dstreg = dstpos.alloc_reg; + struct ir_vr_pos v1pos = v1->data.vr_pos; + s32 spill_pos = v1pos.spilled; + insn->op = IR_INSN_ADD; + insn->alu_op = IR_ALU_64; + if (t0 == CONST) { + // reg = getelemptr const ptr + // ==> + // reg = r10 + (const + spill_pos) + DBGASSERT(v0->const_type == IR_ALU_32); + *v0 = bpf_ir_value_norm_stack_ptr(); + s64 tmp = v0->data.constant_d + spill_pos; // Assume no overflow + *v1 = bpf_ir_value_const32(tmp); + normalize_alu(env, insn); + } + if (t0 == REG) { + u8 v0reg = v0->data.vr_pos.alloc_reg; + *v1 = bpf_ir_value_const32(spill_pos); + if (v0reg == dstreg) { + // reg = getelemptr reg ptr + // ==> + // reg += r10 + // reg += spill_pos + *v0 = bpf_ir_value_vrpos(dstpos); + bpf_ir_create_bin_insn_norm( + env, insn, dstpos, bpf_ir_value_vrpos(dstpos), + bpf_ir_value_norm_stack_ptr(), IR_INSN_ADD, + IR_ALU_64, INSERT_FRONT); + } else { + // reg1 = getelemptr reg2 ptr + // ==> + // reg1 = reg2 + // reg1 += r10 + // reg1 += spill_pos + bpf_ir_create_assign_insn_norm(env, insn, dstpos, *v0, + INSERT_FRONT); + bpf_ir_create_bin_insn_norm( + env, insn, dstpos, bpf_ir_value_vrpos(dstpos), + bpf_ir_value_norm_stack_ptr(), IR_INSN_ADD, + IR_ALU_64, INSERT_FRONT); + *v0 = bpf_ir_value_vrpos(dstpos); + } + } +} + +static void normalize_stackoff(struct ir_insn *insn) +{ + // Could be storeraw or loadraw + // Stack already shifted + struct ir_value addrval = insn->addr_val.value; + enum val_type addr_ty = vtype(addrval); + // storeraw STACKOFF ? + // ==> + // storeraw r10 ? + if (addr_ty == STACK) { + insn->addr_val.offset += addrval.data.vr_pos.spilled; + insn->addr_val.value = bpf_ir_value_norm_stack_ptr(); + } +} + +static void normalize_neg(struct bpf_ir_env *env, struct ir_insn *insn) +{ + struct ir_value *v0 = &insn->values[0]; + enum val_type t0 = insn->value_num >= 1 ? vtype(*v0) : UNDEF; + enum val_type tdst = vtype_insn_norm(insn); + DBGASSERT(tdst == REG); + struct ir_vr_pos dst_pos = insn_norm(insn)->pos; + // reg = neg reg ==> OK! + if (t0 == REG && v0->data.vr_pos.alloc_reg != dst_pos.alloc_reg) { + // reg1 = neg reg2 + // ==> + // reg1 = reg2 + // reg1 = neg reg1 + bpf_ir_create_assign_insn_norm(env, insn, dst_pos, *v0, + INSERT_FRONT); + *v0 = bpf_ir_value_vrpos(dst_pos); + } + if (t0 == CONST) { + // reg = neg const + RAISE_ERROR("Not supported"); + } else if (t0 == STACK) { + // reg = neg stack + // ==> + // reg = stack + // reg = neg reg + cgir_load_stack_to_reg_norm(env, insn, v0, IR_VR_TYPE_64, + dst_pos); + } +} + +static void normalize_end(struct bpf_ir_env *env, struct ir_insn *insn) +{ + struct ir_value *v0 = &insn->values[0]; + enum val_type t0 = insn->value_num >= 1 ? vtype(*v0) : UNDEF; + enum val_type tdst = vtype_insn_norm(insn); + DBGASSERT(tdst == REG); + struct ir_vr_pos dst_pos = insn_norm(insn)->pos; + // reg = end reg + if (t0 == REG && v0->data.vr_pos.alloc_reg != dst_pos.alloc_reg) { + // reg1 = end reg2 + // ==> + // reg1 = reg2 + // reg1 = end reg1 + bpf_ir_create_assign_insn_norm(env, insn, dst_pos, *v0, + INSERT_FRONT); + *v0 = bpf_ir_value_vrpos(dst_pos); + } + // reg = neg const ==> Not supported + if (t0 == CONST) { + RAISE_ERROR("Not supported"); + } else if (t0 == STACK) { + // reg = end stack + // ==> + // reg = stack + // reg = end reg + cgir_load_stack_to_reg_norm(env, insn, v0, IR_VR_TYPE_64, + dst_pos); + } +} + +static void normalize_ret(struct bpf_ir_env *env, struct ir_insn *insn) +{ + if (insn->value_num == 0) { + return; + } + struct ir_value *v0 = &insn->values[0]; + enum val_type t0 = insn->value_num >= 1 ? vtype(*v0) : UNDEF; + // ret REG + // ==> + // R0 = REG + // ret + DBGASSERT(t0 == REG || t0 == CONST); + struct ir_vr_pos pos = (struct ir_vr_pos){ .allocated = true, + .alloc_reg = BPF_REG_0, + .spilled = 0 }; + struct ir_insn *new_insn = bpf_ir_create_assign_insn_norm( + env, insn, pos, *v0, INSERT_FRONT); + new_insn->vr_type = IR_VR_TYPE_64; + insn->value_num = 0; +} + +static void normalize(struct bpf_ir_env *env, struct ir_function *fun) +{ + struct ir_basic_block **pos; + array_for(pos, fun->reachable_bbs) + { + struct ir_basic_block *bb = *pos; + struct ir_insn *insn, *tmp; + list_for_each_entry_safe(insn, tmp, &bb->ir_insn_head, + list_ptr) { + if (insn->op == IR_INSN_ALLOC) { + // OK + } else if (insn->op == IR_INSN_ALLOCARRAY) { + // OK + } else if (insn->op == IR_INSN_GETELEMPTR) { + normalize_getelemptr(env, insn); + } else if (insn->op == IR_INSN_STORE) { + // Should be converted to ASSIGN + CRITICAL("Error"); + } else if (insn->op == IR_INSN_LOAD) { + CRITICAL("Error"); + } else if (insn->op == IR_INSN_LOADRAW) { + normalize_stackoff(insn); + } else if (insn->op == IR_INSN_LOADIMM_EXTRA) { + // OK + } else if (insn->op == IR_INSN_STORERAW) { + normalize_stackoff(insn); + } else if (insn->op == IR_INSN_NEG) { + normalize_neg(env, insn); + } else if (insn->op == IR_INSN_HTOBE || + insn->op == IR_INSN_HTOLE) { + normalize_end(env, insn); + } else if (bpf_ir_is_bin_alu(insn)) { + normalize_alu(env, insn); + } else if (insn->op == IR_INSN_ASSIGN) { + normalize_assign(insn); + } else if (insn->op == IR_INSN_RET) { + normalize_ret(env, insn); + } else if (insn->op == IR_INSN_CALL) { + // OK + } else if (insn->op == IR_INSN_JA) { + // OK + } else if (bpf_ir_is_cond_jmp(insn)) { + // jmp reg const/reg + // or + // jmp const/reg reg + // OK + } else { + RAISE_ERROR("No such instruction"); + } + } + } +} + +static void print_ir_prog_cg_flatten(struct bpf_ir_env *env, + struct ir_function *fun, char *msg) +{ + PRINT_LOG_DEBUG(env, "\x1B[32m----- CG: %s -----\x1B[0m\n", msg); + print_ir_prog_advanced(env, fun, NULL, NULL, print_ir_flatten); +} + +static struct pre_ir_insn translate_reg_to_reg(u8 dst, u8 src) +{ + // MOV dst src + struct pre_ir_insn insn = { 0 }; + insn.opcode = BPF_MOV | BPF_X | BPF_ALU64; + insn.dst_reg = dst; + insn.src_reg = src; + insn.imm = 0; + return insn; +} + +static struct pre_ir_insn translate_const_to_reg(u8 dst, s64 data, + enum ir_alu_op_type type) +{ + // MOV dst imm + struct pre_ir_insn insn = { 0 }; + insn.dst_reg = dst; + if (type == IR_ALU_32) { + insn.opcode = BPF_MOV | BPF_K | BPF_ALU; + } else { + // Default is imm64 + insn.opcode = BPF_MOV | BPF_K | BPF_ALU64; + } + insn.imm = data; + return insn; +} + +static int vr_type_to_size(enum ir_vr_type type) +{ + switch (type) { + case IR_VR_TYPE_32: + return BPF_W; + case IR_VR_TYPE_16: + return BPF_H; + case IR_VR_TYPE_8: + return BPF_B; + case IR_VR_TYPE_64: + return BPF_DW; + default: + CRITICAL("Error"); + } +} + +static struct pre_ir_insn load_addr_to_reg(u8 dst, struct ir_address_value addr, + enum ir_vr_type type) +{ + // MOV dst src + struct pre_ir_insn insn = { 0 }; + insn.dst_reg = dst; + insn.off = addr.offset; + int size = vr_type_to_size(type); + if (addr.value.type == IR_VALUE_FLATTEN_DST) { + // Must be REG + DBGASSERT(vtype(addr.value) == REG); + // Load reg (addr) to reg + insn.src_reg = addr.value.data.vr_pos.alloc_reg; + insn.opcode = BPF_LDX | size | BPF_MEM; + } else if (addr.value.type == IR_VALUE_CONSTANT) { + // Must be U64 + insn.it = IMM64; + insn.imm64 = addr.value.data.constant_d; + insn.opcode = size; + // Simplify the opcode to reduce compiler warning, the real opcode is as follows + // (but BPF_MM and BPF_LD are all 0) + // insn.opcode = BPF_IMM | size | BPF_LD; + } else { + CRITICAL("Error"); + } + return insn; +} + +static struct pre_ir_insn store_reg_to_reg_mem(u8 dst, u8 src, s16 offset, + enum ir_vr_type type) +{ + struct pre_ir_insn insn = { 0 }; + int size = vr_type_to_size(type); + insn.src_reg = src; + insn.off = offset; + insn.opcode = BPF_STX | size | BPF_MEM; + insn.dst_reg = dst; + return insn; +} + +static struct pre_ir_insn store_const_to_reg_mem(u8 dst, s64 val, s16 offset, + enum ir_vr_type type) +{ + struct pre_ir_insn insn = { 0 }; + int size = vr_type_to_size(type); + insn.it = IMM; + insn.imm = val; + insn.off = offset; + insn.opcode = BPF_ST | size | BPF_MEM; + insn.dst_reg = dst; + return insn; +} + +static int end_code(enum ir_insn_type insn) +{ + if (insn == IR_INSN_HTOBE) { + return BPF_TO_BE; + } else if (insn == IR_INSN_HTOLE) { + return BPF_TO_LE; + } else { + CRITICAL("Error"); + } +} + +static int alu_code(enum ir_insn_type insn) +{ + switch (insn) { + case IR_INSN_NEG: + return BPF_NEG; + case IR_INSN_ADD: + return BPF_ADD; + case IR_INSN_SUB: + return BPF_SUB; + case IR_INSN_MUL: + return BPF_MUL; + case IR_INSN_DIV: + return BPF_DIV; + case IR_INSN_OR: + return BPF_OR; + case IR_INSN_AND: + return BPF_AND; + case IR_INSN_MOD: + return BPF_MOD; + case IR_INSN_XOR: + return BPF_XOR; + case IR_INSN_LSH: + return BPF_LSH; + case IR_INSN_ARSH: + return BPF_ARSH; + case IR_INSN_RSH: + return BPF_RSH; + default: + CRITICAL("Error"); + } +} + +static int jmp_code(enum ir_insn_type insn) +{ + switch (insn) { + case IR_INSN_JA: + return BPF_JA; + case IR_INSN_JEQ: + return BPF_JEQ; + case IR_INSN_JNE: + return BPF_JNE; + case IR_INSN_JLT: + return BPF_JLT; + case IR_INSN_JLE: + return BPF_JLE; + case IR_INSN_JGT: + return BPF_JGT; + case IR_INSN_JGE: + return BPF_JGE; + case IR_INSN_JSGT: + return BPF_JSGT; + case IR_INSN_JSLT: + return BPF_JSLT; + default: + CRITICAL("Error"); + } +} + +static struct pre_ir_insn alu_reg(u8 dst, u8 src, enum ir_alu_op_type type, + int opcode) +{ + struct pre_ir_insn insn = { 0 }; + insn.dst_reg = dst; + insn.src_reg = src; + int alu_class = type == IR_ALU_64 ? BPF_ALU64 : BPF_ALU; + insn.opcode = opcode | BPF_X | alu_class; + return insn; +} + +static struct pre_ir_insn alu_neg(u8 dst, enum ir_alu_op_type type) +{ + struct pre_ir_insn insn = { 0 }; + insn.dst_reg = dst; + int alu_class = type == IR_ALU_64 ? BPF_ALU64 : BPF_ALU; + insn.opcode = BPF_NEG | BPF_K | alu_class; + return insn; +} + +static struct pre_ir_insn alu_end(u8 dst, s32 swap_width, int enty) +{ + struct pre_ir_insn insn = { 0 }; + insn.dst_reg = dst; + insn.opcode = enty | BPF_END | BPF_ALU; + insn.imm = swap_width; + return insn; +} + +static struct pre_ir_insn alu_imm(u8 dst, s64 src, enum ir_alu_op_type type, + int opcode) +{ + struct pre_ir_insn insn = { 0 }; + insn.dst_reg = dst; + int alu_class = type == IR_ALU_64 ? BPF_ALU64 : BPF_ALU; + insn.it = IMM; + insn.imm = src; + insn.opcode = opcode | BPF_K | alu_class; + return insn; +} + +static struct pre_ir_insn cond_jmp_reg(u8 dst, u8 src, enum ir_alu_op_type type, + int opcode) +{ + struct pre_ir_insn insn = { 0 }; + insn.dst_reg = dst; + insn.src_reg = src; + int alu_class = type == IR_ALU_64 ? BPF_JMP : BPF_JMP32; + insn.opcode = opcode | alu_class | BPF_X; + return insn; +} + +static struct pre_ir_insn cond_jmp_imm(u8 dst, s64 src, + enum ir_alu_op_type type, int opcode) +{ + struct pre_ir_insn insn = { 0 }; + insn.dst_reg = dst; + int alu_class = type == IR_ALU_64 ? BPF_JMP : BPF_JMP32; + insn.it = IMM; + insn.imm = src; + insn.opcode = opcode | alu_class | BPF_K; + return insn; +} + +static u8 get_alloc_reg(struct ir_insn *insn) +{ + return insn_norm(insn)->pos.alloc_reg; +} + +static void translate_loadraw(struct ir_insn *insn) +{ + enum val_type tdst = vtype_insn_norm(insn); + struct ir_insn_norm_extra *extra = insn_norm(insn); + DBGASSERT(tdst == REG); + extra->translated[0] = load_addr_to_reg(get_alloc_reg(insn), + insn->addr_val, insn->vr_type); +} + +static void translate_loadimm_extra(struct ir_insn *insn) +{ + enum val_type tdst = vtype_insn_norm(insn); + struct ir_insn_norm_extra *extra = insn_norm(insn); + DBGASSERT(tdst == REG); + extra->translated[0].opcode = BPF_IMM | BPF_LD | BPF_DW; + DBGASSERT(insn->imm_extra_type <= 0x6); + extra->translated[0].src_reg = insn->imm_extra_type; + extra->translated[0].dst_reg = get_alloc_reg(insn); + // 0 2 6 needs next + extra->translated[0].it = IMM64; + extra->translated[0].imm64 = insn->imm64; +} + +static void translate_storeraw(struct ir_insn *insn) +{ + struct ir_value v0 = insn->values[0]; + enum val_type t0 = insn->value_num >= 1 ? vtype(v0) : UNDEF; + struct ir_insn_norm_extra *extra = insn_norm(insn); + // storeraw + if (insn->addr_val.value.type == IR_VALUE_FLATTEN_DST) { + // Store value in (address in the value) + DBGASSERT(vtype(insn->addr_val.value) == REG); + // Store value in the stack + if (t0 == REG) { + extra->translated[0] = store_reg_to_reg_mem( + insn->addr_val.value.data.vr_pos.alloc_reg, + v0.data.vr_pos.alloc_reg, insn->addr_val.offset, + insn->vr_type); + } else if (t0 == CONST) { + extra->translated[0] = store_const_to_reg_mem( + insn->addr_val.value.data.vr_pos.alloc_reg, + v0.data.constant_d, insn->addr_val.offset, + insn->vr_type); + } else { + CRITICAL("Error"); + } + } else { + CRITICAL("Error"); + } +} + +static void translate_alu(struct ir_insn *insn) +{ + struct ir_value v0 = insn->values[0]; + struct ir_value v1 = insn->values[1]; + enum val_type t0 = insn->value_num >= 1 ? vtype(v0) : UNDEF; + enum val_type t1 = insn->value_num >= 2 ? vtype(v1) : UNDEF; + enum val_type tdst = vtype_insn_norm(insn); + struct ir_insn_norm_extra *extra = insn_norm(insn); + DBGASSERT(tdst == REG); + DBGASSERT(t0 == REG); + DBGASSERT(get_alloc_reg(insn) == v0.data.vr_pos.alloc_reg); + if (t1 == REG) { + extra->translated[0] = + alu_reg(get_alloc_reg(insn), v1.data.vr_pos.alloc_reg, + insn->alu_op, alu_code(insn->op)); + } else if (t1 == CONST) { + // Remove the instruction in some special cases + if (insn->op == IR_INSN_ADD && v1.data.constant_d == 0) { + extra->translated_num = 0; + return; + } + extra->translated[0] = alu_imm(get_alloc_reg(insn), + v1.data.constant_d, insn->alu_op, + alu_code(insn->op)); + } else { + CRITICAL("Error"); + } +} + +static void translate_assign(struct ir_insn *insn) +{ + struct ir_value v0 = insn->values[0]; + enum val_type t0 = insn->value_num >= 1 ? vtype(v0) : UNDEF; + enum val_type tdst = vtype_insn_norm(insn); + struct ir_insn_norm_extra *extra = insn_norm(insn); + + // reg = const (alu) + // reg = reg + if (tdst == REG && t0 == CONST) { + extra->translated[0] = translate_const_to_reg( + get_alloc_reg(insn), v0.data.constant_d, insn->alu_op); + } else if (tdst == REG && t0 == REG) { + if (get_alloc_reg(insn) == v0.data.vr_pos.alloc_reg) { + // Remove the instruction + extra->translated_num = 0; + return; + } + extra->translated[0] = translate_reg_to_reg( + get_alloc_reg(insn), v0.data.vr_pos.alloc_reg); + } else { + CRITICAL("Error"); + } +} + +static void translate_ret(struct ir_insn *insn) +{ + struct ir_insn_norm_extra *extra = insn_norm(insn); + extra->translated[0].opcode = BPF_EXIT | BPF_JMP; +} + +static void translate_call(struct ir_insn *insn) +{ + struct ir_insn_norm_extra *extra = insn_norm(insn); + // Currently only support local helper functions + extra->translated[0].opcode = BPF_CALL | BPF_JMP; + extra->translated[0].it = IMM; + extra->translated[0].imm = insn->fid; +} + +static void translate_ja(struct ir_insn *insn) +{ + struct ir_insn_norm_extra *extra = insn_norm(insn); + extra->translated[0].opcode = BPF_JMP | BPF_JA; +} + +static void translate_neg(struct ir_insn *insn) +{ + struct ir_value v0 = insn->values[0]; + enum val_type t0 = insn->value_num >= 1 ? vtype(v0) : UNDEF; + enum val_type tdst = vtype_insn_norm(insn); + struct ir_insn_norm_extra *extra = insn_norm(insn); + DBGASSERT(tdst == REG && t0 == REG); + DBGASSERT(get_alloc_reg(insn) == v0.data.vr_pos.alloc_reg); + extra->translated[0] = alu_neg(get_alloc_reg(insn), insn->alu_op); +} + +static void translate_end(struct ir_insn *insn) +{ + struct ir_value v0 = insn->values[0]; + enum val_type t0 = insn->value_num >= 1 ? vtype(v0) : UNDEF; + enum val_type tdst = vtype_insn_norm(insn); + struct ir_insn_norm_extra *extra = insn_norm(insn); + DBGASSERT(tdst == REG); + DBGASSERT(t0 == REG); + DBGASSERT(get_alloc_reg(insn) == v0.data.vr_pos.alloc_reg); + extra->translated[0] = alu_end(get_alloc_reg(insn), insn->swap_width, + end_code(insn->op)); +} + +static void translate_cond_jmp(struct ir_insn *insn) +{ + struct ir_value v0 = insn->values[0]; + struct ir_value v1 = insn->values[1]; + enum val_type t0 = insn->value_num >= 1 ? vtype(v0) : UNDEF; + enum val_type t1 = insn->value_num >= 2 ? vtype(v1) : UNDEF; + struct ir_insn_norm_extra *extra = insn_norm(insn); + DBGASSERT(t0 == REG || t1 == REG); + if (t0 == REG) { + if (t1 == REG) { + extra->translated[0] = + cond_jmp_reg(v0.data.vr_pos.alloc_reg, + v1.data.vr_pos.alloc_reg, + insn->alu_op, jmp_code(insn->op)); + } else if (t1 == CONST) { + if (v1.const_type == IR_ALU_64) { + CRITICAL("TODO"); + } + extra->translated[0] = cond_jmp_imm( + v0.data.vr_pos.alloc_reg, v1.data.constant_d, + insn->alu_op, jmp_code(insn->op)); + } else { + CRITICAL("Error"); + } + } else { + DBGASSERT(t0 == CONST); + DBGASSERT(t1 == REG); + CRITICAL("TODO"); + // Probably we could switch? + extra->translated[0] = cond_jmp_imm(v1.data.vr_pos.alloc_reg, + v0.data.constant_d, + insn->alu_op, + jmp_code(insn->op)); + } +} + +static u32 bb_insn_cnt(struct ir_basic_block *bb) +{ + u32 cnt = 0; + struct ir_insn *insn, *tmp; + list_for_each_entry_safe(insn, tmp, &bb->ir_insn_head, list_ptr) { + if (insn->op == IR_INSN_ALLOC || + insn->op == IR_INSN_ALLOCARRAY) { + continue; + } else { + cnt++; + } + } + return cnt; +} + +static u32 bb_insn_critical_cnt(struct ir_basic_block *bb) +{ + u32 cnt = bb_insn_cnt(bb); + while (bb->preds.num_elem <= 1) { + if (bb->preds.num_elem == 0) { + break; + } + struct ir_basic_block **tmp = + bpf_ir_array_get_void(&bb->preds, 0); + bb = *tmp; + if (bb->flag & IR_BB_HAS_COUNTER) { + break; + } + cnt += bb_insn_cnt(bb); + } + return cnt; +} + +static void replace_builtin_const(struct bpf_ir_env *env, + struct ir_function *fun) +{ + struct ir_basic_block **pos; + array_for(pos, fun->reachable_bbs) + { + struct ir_basic_block *bb = *pos; + struct ir_insn *insn, *tmp; + list_for_each_entry_safe(insn, tmp, &bb->ir_insn_head, + list_ptr) { + struct array operands = bpf_ir_get_operands(env, insn); + struct ir_value **val; + array_for(val, operands) + { + struct ir_value *v = *val; + if (v->type == IR_VALUE_CONSTANT) { + if (v->builtin_const == + IR_BUILTIN_BB_INSN_CNT) { + v->data.constant_d = + bb_insn_cnt(bb); + } + if (v->builtin_const == + IR_BUILTIN_BB_INSN_CRITICAL_CNT) { + v->data.constant_d = + bb_insn_critical_cnt( + bb); + } + } + } + bpf_ir_array_free(&operands); + } + } +} + +static void check_total_insn(struct bpf_ir_env *env, struct ir_function *fun) +{ + u32 cnt = 0; + struct ir_basic_block **pos; + array_for(pos, fun->reachable_bbs) + { + struct ir_basic_block *bb = *pos; + struct ir_insn *insn, *tmp; + list_for_each_entry_safe(insn, tmp, &bb->ir_insn_head, + list_ptr) { + struct ir_insn_norm_extra *extra = insn_norm(insn); + cnt += extra->translated_num; + } + } + if (cnt >= 1000000) { + RAISE_ERROR("Too many instructions"); + } +} + +static void translate(struct bpf_ir_env *env, struct ir_function *fun) +{ + struct ir_basic_block **pos; + array_for(pos, fun->reachable_bbs) + { + struct ir_basic_block *bb = *pos; + struct ir_insn *insn, *tmp; + list_for_each_entry_safe(insn, tmp, &bb->ir_insn_head, + list_ptr) { + struct ir_insn_norm_extra *extra = insn_norm(insn); + extra->translated_num = 1; // Default: 1 instruction + if (insn->op == IR_INSN_ALLOC) { + // Nothing to do + extra->translated_num = 0; + } else if (insn->op == IR_INSN_ALLOCARRAY) { + // Nothing to do + extra->translated_num = 0; + } else if (insn->op == IR_INSN_STORE) { + CRITICAL("Error"); + } else if (insn->op == IR_INSN_LOAD) { + CRITICAL("Error"); + } else if (insn->op == IR_INSN_GETELEMPTR) { + CRITICAL("Error"); + } else if (insn->op == IR_INSN_LOADRAW) { + translate_loadraw(insn); + } else if (insn->op == IR_INSN_LOADIMM_EXTRA) { + translate_loadimm_extra(insn); + } else if (insn->op == IR_INSN_STORERAW) { + translate_storeraw(insn); + } else if (insn->op == IR_INSN_NEG) { + translate_neg(insn); + } else if (insn->op == IR_INSN_HTOBE || + insn->op == IR_INSN_HTOLE) { + translate_end(insn); + } else if (bpf_ir_is_bin_alu(insn)) { + translate_alu(insn); + } else if (insn->op == IR_INSN_ASSIGN) { + translate_assign(insn); + } else if (insn->op == IR_INSN_RET) { + translate_ret(insn); + } else if (insn->op == IR_INSN_CALL) { + translate_call(insn); + } else if (insn->op == IR_INSN_JA) { + translate_ja(insn); + } else if (bpf_ir_is_cond_jmp(insn)) { + translate_cond_jmp(insn); + } else { + RAISE_ERROR("No such instruction"); + } + } + } +} + +// Relocate BB +static void calc_pos(struct bpf_ir_env *env, struct ir_function *fun) +{ + // Calculate the position of each instruction & BB + size_t ipos = 0; // Instruction position + struct ir_basic_block **pos; + array_for(pos, fun->reachable_bbs) + { + struct ir_basic_block *bb = *pos; + struct ir_bb_cg_extra *bb_extra = bb->user_data; + bb_extra->pos = ipos; + struct ir_insn *insn; + list_for_each_entry(insn, &bb->ir_insn_head, list_ptr) { + struct ir_insn_norm_extra *insn_extra = insn_norm(insn); + for (u8 i = 0; i < insn_extra->translated_num; ++i) { + struct pre_ir_insn *translated_insn = + &insn_extra->translated[i]; + // Pos + translated_insn->pos = ipos; + if (translated_insn->it == IMM) { + ipos += 1; + } else { + ipos += 2; + } + } + } + } + env->insn_cnt = ipos; +} + +static void relocate(struct bpf_ir_env *env, struct ir_function *fun) +{ + calc_pos(env, fun); + struct ir_basic_block **pos; + array_for(pos, fun->reachable_bbs) + { + struct ir_basic_block *bb = *pos; + struct ir_insn *insn; + list_for_each_entry(insn, &bb->ir_insn_head, list_ptr) { + struct ir_insn_norm_extra *insn_extra = insn_norm(insn); + if (insn->op == IR_INSN_JA) { + DBGASSERT(insn_extra->translated_num == 1); + size_t target = bpf_ir_bb_cg(insn->bb1)->pos; + insn_extra->translated[0].off = + target - insn_extra->translated[0].pos - + 1; + } + if (bpf_ir_is_cond_jmp(insn)) { + DBGASSERT(insn_extra->translated_num == 1); + size_t target = bpf_ir_bb_cg(insn->bb2)->pos; + insn_extra->translated[0].off = + target - insn_extra->translated[0].pos - + 1; + } + } + } +} + +static void synthesize(struct bpf_ir_env *env, struct ir_function *fun) +{ + // The last step, synthesizes the program + SAFE_MALLOC(env->insns, env->insn_cnt * sizeof(struct bpf_insn)); + struct ir_basic_block **pos = NULL; + array_for(pos, fun->reachable_bbs) + { + struct ir_basic_block *bb = *pos; + struct ir_insn *insn = NULL; + list_for_each_entry(insn, &bb->ir_insn_head, list_ptr) { + struct ir_insn_norm_extra *extra = insn_norm(insn); + for (u8 i = 0; i < extra->translated_num; ++i) { + struct pre_ir_insn translated_insn = + extra->translated[i]; + // PRINT_DBG("Writing to insn %zu\n", + // translated_insn.pos); + struct bpf_insn *real_insn = + &env->insns[translated_insn.pos]; + real_insn->code = translated_insn.opcode; + real_insn->dst_reg = translated_insn.dst_reg; + real_insn->src_reg = translated_insn.src_reg; + real_insn->off = translated_insn.off; + if (translated_insn.it == IMM) { + real_insn->imm = translated_insn.imm; + } else { + // Wide instruction + struct bpf_insn *real_insn2 = + &env->insns[translated_insn.pos + + 1]; + real_insn->imm = translated_insn.imm64 & + 0xffffffff; + real_insn2->imm = + translated_insn.imm64 >> 32; + } + } + } + } +} + +static void free_cg_final(struct ir_function *fun) +{ + // Free CG resources (after flattening) + + struct ir_basic_block **pos = NULL; + array_for(pos, fun->reachable_bbs) + { + struct ir_basic_block *bb = *pos; + struct ir_bb_cg_extra *bb_cg = bb->user_data; + free_proto(bb_cg); + bb->user_data = NULL; + + struct ir_insn *insn = NULL; + list_for_each_entry(insn, &bb->ir_insn_head, list_ptr) { + free_proto(insn->user_data); + insn->user_data = NULL; + } + } +} + +void bpf_ir_cg_norm_v2(struct bpf_ir_env *env, struct ir_function *fun) +{ + flatten_ir(env, fun); + CHECK_ERR(); + + print_ir_prog_cg_flatten(env, fun, "Flattening"); + + normalize(env, fun); + CHECK_ERR(); + print_ir_prog_cg_flatten(env, fun, "Normalization"); + + replace_builtin_const(env, fun); + CHECK_ERR(); + + translate(env, fun); + CHECK_ERR(); + + check_total_insn(env, fun); + CHECK_ERR(); + + relocate(env, fun); + CHECK_ERR(); + + synthesize(env, fun); + CHECK_ERR(); + + // Free CG resources + free_cg_final(fun); +} diff --git a/core/ir_cg_v2.c b/core/ir_cg_v2.c new file mode 100644 index 00000000..294b2e10 --- /dev/null +++ b/core/ir_cg_v2.c @@ -0,0 +1,912 @@ +// SPDX-License-Identifier: GPL-2.0-only +#include +#include "ir_cg.h" + +/* + +Using SSA-based RA and graph coloring algorithm. + +Algorithms are based on the following paper: + +Pereira, F., and Palsberg, J., "Register Allocation via the Coloring of Chordal Graphs", APLAS, pp 315-329 (2005) + +*/ + +// Erase an instruction. +// Only used in SSA Out process. +// Do not use it within RA (it doesn not maintain adj and all_var stuff properly) +static void erase_insn_cg_v2(struct bpf_ir_env *env, struct ir_function *fun, + struct ir_insn *insn) +{ + if (insn->users.num_elem > 0) { + struct ir_insn **pos; + bool fail = false; + array_for(pos, insn->users) + { + if (*pos != insn) { + fail = true; + break; + } + } + if (fail) { + tag_ir(fun); + array_for(pos, insn->users) + { + print_ir_insn_err(env, *pos, "User"); + } + print_ir_insn_err(env, insn, "Has users"); + RAISE_ERROR( + "Cannot erase a instruction that has (non-self) users"); + } + } + struct array operands = bpf_ir_get_operands(env, insn); + CHECK_ERR(); + struct ir_value **pos2; + array_for(pos2, operands) + { + bpf_ir_val_remove_user((**pos2), insn); + } + bpf_ir_array_free(&operands); + list_del(&insn->list_ptr); + bpf_ir_array_free(&insn->users); + + struct ir_insn_cg_extra_v2 *extra = insn->user_data; + bpf_ir_ptrset_free(&extra->adj); + bpf_ir_ptrset_free(&extra->in); + bpf_ir_ptrset_free(&extra->out); + + free_proto(insn); +} + +static void set_insn_dst(struct ir_insn *insn, struct ir_insn *dst) +{ + insn_cg_v2(insn)->dst = dst; +} + +static void pre_color(struct ir_function *fun, struct ir_insn *insn, u8 reg) +{ + set_insn_dst(insn, fun->cg_info.regs[reg]); + insn_cg_v2(insn)->vr_pos.allocated = true; + insn_cg_v2(insn)->vr_pos.alloc_reg = reg; + insn_cg_v2(insn)->vr_pos.spilled = 0; +} + +void bpf_ir_init_insn_cg_v2(struct bpf_ir_env *env, struct ir_insn *insn) +{ + struct ir_insn_cg_extra_v2 *extra = NULL; + SAFE_MALLOC(extra, sizeof(struct ir_insn_cg_extra_v2)); + insn->user_data = extra; + + extra->dst = bpf_ir_is_void(insn) ? NULL : insn; + extra->vr_pos.allocated = false; + extra->vr_pos.spilled = 0; + extra->vr_pos.spilled_size = 0; + extra->vr_pos.alloc_reg = 0; + extra->lambda = 0; + extra->w = 0; + + INIT_PTRSET_DEF(&extra->adj); + + INIT_PTRSET_DEF(&extra->in); + INIT_PTRSET_DEF(&extra->out); + extra->nonvr = false; +} + +static void init_cg(struct bpf_ir_env *env, struct ir_function *fun) +{ + struct ir_basic_block **pos = NULL; + array_for(pos, fun->reachable_bbs) + { + struct ir_basic_block *bb = *pos; + struct ir_bb_cg_extra *bb_cg = NULL; + SAFE_MALLOC(bb_cg, sizeof(struct ir_bb_cg_extra)); + // Empty bb cg + bb->user_data = bb_cg; + + struct ir_insn *insn = NULL; + list_for_each_entry(insn, &bb->ir_insn_head, list_ptr) { + bpf_ir_init_insn_cg_v2(env, insn); + CHECK_ERR(); + } + } + + for (u8 i = 0; i < BPF_REG_10; ++i) { + struct ir_insn *insn = fun->cg_info.regs[i]; + bpf_ir_init_insn_cg_v2(env, insn); + CHECK_ERR(); + + struct ir_insn_cg_extra_v2 *extra = insn_cg_v2(insn); + // Pre-colored registers are allocated + extra->vr_pos.alloc_reg = i; + extra->vr_pos.allocated = true; + extra->nonvr = true; + } + bpf_ir_init_insn_cg_v2(env, fun->sp); + struct ir_insn_cg_extra_v2 *extra = insn_cg_v2(fun->sp); + extra->vr_pos.alloc_reg = 10; + extra->vr_pos.allocated = true; + extra->nonvr = true; +} + +/* +Pre RA +*/ + +static void change_fun_arg(struct bpf_ir_env *env, struct ir_function *fun) +{ + for (u8 i = 0; i < MAX_FUNC_ARG; ++i) { + if (fun->function_arg[i]->users.num_elem > 0) { + // Insert ASSIGN arg[i] at the beginning of the function + struct ir_insn *new_insn = + bpf_ir_create_assign_insn_bb_cg_v2( + env, fun->entry, + bpf_ir_value_insn( + fun->cg_info.regs[i + 1]), + INSERT_FRONT_AFTER_PHI); + bpf_ir_replace_all_usage(env, fun->function_arg[i], + bpf_ir_value_insn(new_insn)); + } + } +} + +static void change_call(struct bpf_ir_env *env, struct ir_function *fun) +{ + struct ir_basic_block **pos; + array_for(pos, fun->reachable_bbs) + { + struct ir_basic_block *bb = *pos; + struct ir_insn *insn; + list_for_each_entry(insn, &bb->ir_insn_head, list_ptr) { + if (insn->op == IR_INSN_CALL) { + // Change function call args + for (u8 i = 0; i < insn->value_num; ++i) { + struct ir_value val = insn->values[i]; + bpf_ir_val_remove_user(val, insn); + struct ir_insn *new_insn = + bpf_ir_create_assign_insn_cg_v2( + env, insn, val, + INSERT_FRONT); + pre_color(fun, new_insn, i + 1); + } + insn->value_num = 0; // Remove all operands + + // Change function call dst + insn_cg_v2(insn)->dst = NULL; + if (insn->users.num_elem == 0) { + continue; + } + struct ir_insn *new_insn = + bpf_ir_create_assign_insn_cg_v2( + env, insn, + bpf_ir_value_insn( + fun->cg_info.regs[0]), + INSERT_BACK); + bpf_ir_replace_all_usage( + env, insn, bpf_ir_value_insn(new_insn)); + } + } + } +} + +static void spill_array(struct bpf_ir_env *env, struct ir_function *fun) +{ + u32 offset = 0; + struct ir_basic_block **pos; + array_for(pos, fun->reachable_bbs) + { + struct ir_basic_block *bb = *pos; + struct ir_insn *insn, *tmp; + list_for_each_entry_safe(insn, tmp, &bb->ir_insn_head, + list_ptr) { + if (insn->op == IR_INSN_ALLOCARRAY) { + struct ir_insn_cg_extra_v2 *extra = + insn_cg_v2(insn); + extra->vr_pos.allocated = true; + // Calculate the offset + u32 size = insn->array_num * + bpf_ir_sizeof_vr_type(insn->vr_type); + if (size == 0) { + RAISE_ERROR("Array size is 0"); + } + offset -= (((size - 1) / 8) + 1) * 8; + extra->vr_pos.spilled = offset; + extra->vr_pos.spilled_size = size; + extra->nonvr = true; // Array is not a VR + extra->dst = NULL; + } + } + } +} + +/* +Print utils +*/ + +static void print_ir_dst_v2(struct bpf_ir_env *env, struct ir_insn *insn) +{ + if (!insn->user_data) { + PRINT_LOG_DEBUG(env, "(?)"); + RAISE_ERROR("NULL userdata found"); + } + print_insn_ptr_base(env, insn); + insn = insn_cg_v2(insn)->dst; + if (insn) { + PRINT_LOG_DEBUG(env, "("); + print_insn_ptr_base(env, insn); + PRINT_LOG_DEBUG(env, ")"); + } else { + PRINT_LOG_DEBUG(env, "(NULL)"); + } +} + +static void print_ir_alloc_v2(struct bpf_ir_env *env, struct ir_insn *insn) +{ + if (!insn->user_data) { + PRINT_LOG_DEBUG(env, "(?)"); + RAISE_ERROR("NULL userdata found"); + } + if (insn_cg_v2(insn)->dst == NULL) { + PRINT_LOG_DEBUG(env, "(NULL)"); + return; + } + struct ir_vr_pos pos = insn_cg_v2(insn)->vr_pos; + DBGASSERT(pos.allocated); + if (pos.spilled) { + PRINT_LOG_DEBUG(env, "sp+%u", pos.spilled); + } else { + PRINT_LOG_DEBUG(env, "r%u", pos.alloc_reg); + } +} + +static void print_insn_extra(struct bpf_ir_env *env, struct ir_insn *insn) +{ + struct ir_insn_cg_extra_v2 *insn_cg = insn->user_data; + if (insn_cg == NULL) { + CRITICAL("NULL user data"); + } + struct ir_insn **pos; + + PRINT_LOG_DEBUG(env, "\nIn:"); + ptrset_for(pos, insn_cg->in) + { + struct ir_insn *insn = *pos; + PRINT_LOG_DEBUG(env, " "); + print_insn_ptr_base(env, insn); + } + PRINT_LOG_DEBUG(env, "\nOut:"); + ptrset_for(pos, insn_cg->out) + { + struct ir_insn *insn = *pos; + PRINT_LOG_DEBUG(env, " "); + print_insn_ptr_base(env, insn); + } + PRINT_LOG_DEBUG(env, "\n-------------\n"); +} + +/* +SSA liveness analysis. +*/ + +static void live_in_at_statement(struct bpf_ir_env *env, struct ptrset *M, + struct ir_insn *s, struct ir_insn *v); + +static void live_out_at_statement(struct bpf_ir_env *env, struct ptrset *M, + struct ir_insn *s, struct ir_insn *v); + +static void make_conflict(struct bpf_ir_env *env, struct ir_insn *v1, + struct ir_insn *v2) +{ + struct ir_insn_cg_extra_v2 *v1e = insn_cg_v2(v1); + struct ir_insn_cg_extra_v2 *v2e = insn_cg_v2(v2); + bpf_ir_ptrset_insert(env, &v1e->adj, v2); + bpf_ir_ptrset_insert(env, &v2e->adj, v1); +} + +static void live_out_at_block(struct bpf_ir_env *env, struct ptrset *M, + struct ir_basic_block *n, struct ir_insn *v) +{ + if (!bpf_ir_ptrset_exists(M, n)) { + bpf_ir_ptrset_insert(env, M, n); + struct ir_insn *last = bpf_ir_get_last_insn(n); + if (last) { + live_out_at_statement(env, M, last, v); + } else { + // Empty BB + struct array preds = n->preds; + struct ir_basic_block **pos; + array_for(pos, preds) + { + live_out_at_block(env, M, *pos, v); + } + } + } +} + +static void live_out_at_statement(struct bpf_ir_env *env, struct ptrset *M, + struct ir_insn *s, struct ir_insn *v) +{ + struct ir_insn_cg_extra_v2 *se = insn_cg_v2(s); + bpf_ir_ptrset_insert(env, &se->out, v); + if (se->dst) { + if (se->dst != v) { + make_conflict(env, v, se->dst); + live_in_at_statement(env, M, s, v); + } + } else { + // s has no dst (no KILL) + live_in_at_statement(env, M, s, v); + } +} + +static void live_in_at_statement(struct bpf_ir_env *env, struct ptrset *M, + struct ir_insn *s, struct ir_insn *v) +{ + bpf_ir_ptrset_insert(env, &(insn_cg_v2(s))->in, v); + struct ir_insn *prev = bpf_ir_prev_insn(s); + if (prev == NULL) { + // First instruction + struct ir_basic_block **pos; + array_for(pos, s->parent_bb->preds) + { + live_out_at_block(env, M, *pos, v); + } + } else { + live_out_at_statement(env, M, prev, v); + } +} + +static void print_ir_prog_cg_dst_liveness(struct bpf_ir_env *env, + struct ir_function *fun, char *msg) +{ + PRINT_LOG_DEBUG(env, "\x1B[32m----- CG: %s -----\x1B[0m\n", msg); + print_ir_prog_advanced(env, fun, NULL, print_insn_extra, + print_ir_dst_v2); +} + +static void print_ir_prog_cg_dst(struct bpf_ir_env *env, + struct ir_function *fun, char *msg) +{ + PRINT_LOG_DEBUG(env, "\x1B[32m----- CG: %s -----\x1B[0m\n", msg); + print_ir_prog_advanced(env, fun, NULL, NULL, print_ir_dst_v2); +} + +static void print_ir_prog_cg_alloc(struct bpf_ir_env *env, + struct ir_function *fun, char *msg) +{ + PRINT_LOG_DEBUG(env, "\x1B[32m----- CG: %s -----\x1B[0m\n", msg); + print_ir_prog_advanced(env, fun, NULL, NULL, print_ir_alloc_v2); +} + +static void print_interference_graph(struct bpf_ir_env *env, + struct ir_function *fun) +{ + PRINT_LOG_DEBUG(env, + "\x1B[32m----- CG: Interference Graph -----\x1B[0m\n"); + tag_ir(fun); + struct ir_insn **pos2; + ptrset_for(pos2, fun->cg_info.all_var_v2) + { + struct ir_insn *v = *pos2; + print_insn_ptr_base(env, v); + PRINT_LOG_DEBUG(env, ": "); + struct ir_insn **pos3; + ptrset_for(pos3, insn_cg_v2(v)->adj) + { + struct ir_insn *c = *pos3; // conflict vr + print_insn_ptr_base(env, c); + PRINT_LOG_DEBUG(env, " "); + } + PRINT_LOG_DEBUG(env, "\n"); + } +} + +static void liveness_analysis(struct bpf_ir_env *env, struct ir_function *fun) +{ + bpf_ir_ptrset_clean(&fun->cg_info.all_var_v2); + // Add all real registers to the graph + for (int i = 0; i < RA_COLORS; ++i) { + bpf_ir_ptrset_insert(env, &fun->cg_info.all_var_v2, + fun->cg_info.regs[i]); + } + + struct ptrset M; + INIT_PTRSET_DEF(&M); + struct ir_basic_block **pos; + array_for(pos, fun->reachable_bbs) + { + struct ir_basic_block *bb = *pos; + struct ir_insn *v; + list_for_each_entry(v, &bb->ir_insn_head, list_ptr) { + struct ir_insn_cg_extra_v2 *extra = insn_cg_v2(v); + // Clean + extra->lambda = 0; + extra->w = 0; + + if (extra->dst) { + if (extra->dst == v) { + // dst is a VR + bpf_ir_ptrset_insert( + env, &fun->cg_info.all_var_v2, + v); + } + + bpf_ir_ptrset_clean(&M); + struct ir_insn **pos; + array_for(pos, v->users) + { + struct ir_insn *s = *pos; + if (s->op == IR_INSN_PHI) { + struct phi_value *pos2; + bool found = false; + array_for(pos2, s->phi) + { + if (pos2->value.type == + IR_VALUE_INSN && + pos2->value.data.insn_d == + v) { + found = true; + live_out_at_block( + env, &M, + pos2->bb, + v); + break; + } + } + if (!found) { + CRITICAL( + "Not found user!"); + } + } else { + live_in_at_statement(env, &M, s, + v); + } + } + + if (v->op == IR_INSN_PHI) { + // v is considered LIVE OUT for all preds + struct phi_value *pos2; + array_for(pos2, v->phi) + { + live_out_at_block(env, &M, + pos2->bb, v); + } + } + } + } + } + bpf_ir_ptrset_free(&M); + + print_ir_prog_cg_dst_liveness(env, fun, "Liveness"); +} + +static void caller_constraint(struct bpf_ir_env *env, struct ir_function *fun, + struct ir_insn *insn) +{ + for (u8 i = BPF_REG_0; i < BPF_REG_6; ++i) { + // R0-R5 are caller saved register + make_conflict(env, fun->cg_info.regs[i], insn); + } +} + +static void conflict_analysis(struct bpf_ir_env *env, struct ir_function *fun) +{ + // Add constraints to the graph + + struct ir_basic_block **pos; + // For each BB + array_for(pos, fun->reachable_bbs) + { + struct ir_basic_block *bb = *pos; + struct ir_insn *insn; + // For each operation + list_for_each_entry(insn, &bb->ir_insn_head, list_ptr) { + struct ir_insn_cg_extra_v2 *insn_cg = insn->user_data; + if (insn->op == IR_INSN_CALL) { + // Add caller saved register constraints + struct ir_insn **pos2; + ptrset_for(pos2, insn_cg->in) + { + struct ir_insn **pos3; + ptrset_for(pos3, insn_cg->out) + { + if (*pos2 == *pos3) { + // Live across CALL! + caller_constraint( + env, fun, + *pos2); + } + } + } + } + } + } +} + +// Maximum cardinality search +static struct array mcs(struct bpf_ir_env *env, struct ir_function *fun) +{ + struct array sigma; + INIT_ARRAY(&sigma, struct ir_insn *); + struct ptrset allvar; + bpf_ir_ptrset_clone(env, &allvar, &fun->cg_info.all_var_v2); + for (size_t i = 0; i < fun->cg_info.all_var_v2.cnt; ++i) { + u32 max_l = 0; + struct ir_insn *max_i = NULL; + struct ir_insn **pos; + ptrset_for(pos, allvar) + { + struct ir_insn_cg_extra_v2 *ex = insn_cg_v2(*pos); + if (ex->lambda >= max_l) { + max_l = ex->lambda; + max_i = *pos; + } + } + DBGASSERT(max_i != NULL); + bpf_ir_array_push(env, &sigma, &max_i); + + struct ir_insn_cg_extra_v2 *max_iex = insn_cg_v2(max_i); + ptrset_for(pos, max_iex->adj) + { + if (bpf_ir_ptrset_exists(&allvar, *pos)) { + // *pos in allvar /\ N(max_i) + insn_cg_v2(*pos)->lambda++; + } + } + + bpf_ir_ptrset_delete(&allvar, max_i); + } + + bpf_ir_ptrset_free(&allvar); + return sigma; +} + +static struct ptrset *maxcl_need_spill(struct array *eps) +{ + struct ptrset *pos; + array_for(pos, (*eps)) + { + if (pos->cnt > RA_COLORS) { + return pos; + } + } + return NULL; +} + +struct array pre_spill(struct bpf_ir_env *env, struct ir_function *fun) +{ + // First run maximalCl + struct array sigma = mcs(env, fun); + struct array eps; + INIT_ARRAY(&eps, struct ptrset); + for (size_t i = 0; i < sigma.num_elem; ++i) { + struct ir_insn *v = *array_get(&sigma, i, struct ir_insn *); + struct ir_insn_cg_extra_v2 *vex = insn_cg_v2(v); + struct ptrset q; + INIT_PTRSET_DEF(&q); + bpf_ir_ptrset_insert(env, &q, v); + vex->w++; + struct ir_insn **pos; + ptrset_for(pos, vex->adj) + { + struct ir_insn *u = *pos; + + for (size_t j = 0; j < i; ++j) { + struct ir_insn *v2 = + *array_get(&sigma, j, struct ir_insn *); + if (v2 == u) { + bpf_ir_ptrset_insert(env, &q, u); + insn_cg_v2(u)->w++; + break; + } + } + } + bpf_ir_array_push(env, &eps, &q); + } + + struct ptrset *cur; + struct array to_spill; + INIT_ARRAY(&to_spill, struct ir_insn *); + while ((cur = maxcl_need_spill(&eps))) { + // cur has more than RA_COLORS nodes + u32 max_w = 0; + struct ir_insn *max_i = NULL; + + struct ir_insn **pos; + ptrset_for(pos, (*cur)) + { + struct ir_insn *v = *pos; + struct ir_insn_cg_extra_v2 *vex = insn_cg_v2(v); + if (vex->w >= max_w && !vex->nonvr) { + // Must be a vr to be spilled + max_w = vex->w; + max_i = v; + } + } + DBGASSERT(max_i != NULL); + // Spill max_i + bpf_ir_array_push(env, &to_spill, &max_i); + + struct ptrset *pos2; + array_for(pos2, eps) + { + bpf_ir_ptrset_delete(pos2, max_i); + } + } + + struct ptrset *pos; + array_for(pos, eps) + { + bpf_ir_ptrset_free(pos); + } + bpf_ir_array_free(&eps); + bpf_ir_array_free(&sigma); + return to_spill; +} + +static void spill(struct bpf_ir_env *env, struct ir_function *fun, + struct array *to_spill) +{ +} + +static void coloring(struct bpf_ir_env *env, struct ir_function *fun) +{ + struct array sigma = mcs(env, fun); + struct ir_insn **pos; + + array_for(pos, sigma) + { + struct ir_insn *v = *pos; + struct ir_insn_cg_extra_v2 *vex = insn_cg_v2(v); + if (vex->vr_pos.allocated) { + continue; + } + + bool used_reg[RA_COLORS] = { 0 }; + struct ir_insn **pos2; + ptrset_for(pos2, vex->adj) + { + struct ir_insn *insn2 = *pos2; // Adj instruction + struct ir_insn_cg_extra_v2 *extra2 = insn_cg_v2(insn2); + if (extra2->vr_pos.allocated && + extra2->vr_pos.spilled == 0) { + used_reg[extra2->vr_pos.alloc_reg] = true; + } + } + + for (u8 i = 0; i < RA_COLORS; i++) { + if (!used_reg[i]) { + vex->vr_pos.allocated = true; + vex->vr_pos.alloc_reg = i; + break; + } + } + if (!vex->vr_pos.allocated) { + RAISE_ERROR("No register available"); + } + } + bpf_ir_array_free(&sigma); +} + +// Best effort coalescing +static void coalescing(struct bpf_ir_env *env, struct ir_function *fun) +{ + struct ir_basic_block **pos; + array_for(pos, fun->reachable_bbs) + { + struct ir_basic_block *bb = *pos; + struct ir_insn *v; + list_for_each_entry(v, &bb->ir_insn_head, list_ptr) { + struct ir_insn_cg_extra_v2 *extra = insn_cg_v2(v); + if (v->op == IR_INSN_ASSIGN) { + DBGASSERT(extra->dst); + struct ir_insn *v2 = v->values[0].data.insn_d; + struct ir_insn *v0 = extra->dst; + struct ir_insn_cg_extra_v2 *extra_v0 = + insn_cg_v2(v0); + // v0 = v2 + if (extra_v0->vr_pos.spilled == 0 && + v->values[0].type == IR_VALUE_INSN && + insn_cg_v2(v2)->vr_pos.spilled == 0 && + insn_cg_v2(v2)->vr_pos.alloc_reg != + extra_v0->vr_pos.alloc_reg) { + // Coalesce + u8 used_colors[RA_COLORS] = { 0 }; + struct ir_insn **pos2; + ptrset_for(pos2, + extra_v0->adj) // v0's adj + { + struct ir_insn *c = *pos2; + struct ir_insn_cg_extra_v2 *cex = + insn_cg_v2(c); + DBGASSERT( + cex->vr_pos.allocated); + if (cex->vr_pos.spilled == 0) { + used_colors + [cex->vr_pos + .alloc_reg] = + true; + } + } + + ptrset_for( + pos2, + insn_cg_v2(v2)->adj) // v2's adj + { + struct ir_insn *c = *pos2; + struct ir_insn_cg_extra_v2 *cex = + insn_cg_v2(c); + DBGASSERT( + cex->vr_pos.allocated); + if (cex->vr_pos.spilled == 0) { + used_colors + [cex->vr_pos + .alloc_reg] = + true; + } + } + + // There are three cases + // 1. Rx = %y + // 2. %x = Ry + // 3. %x = %y + + if (extra_v0->nonvr) { + if (!used_colors + [extra_v0->vr_pos + .alloc_reg]) { + // Able to merge + insn_cg_v2(v2) + ->vr_pos + .alloc_reg = + extra_v0->vr_pos + .alloc_reg; + } + } else if (insn_cg_v2(v2)->nonvr) { + if (!used_colors + [insn_cg_v2(v2) + ->vr_pos + .alloc_reg]) { + extra_v0->vr_pos + .alloc_reg = + insn_cg_v2(v2) + ->vr_pos + .alloc_reg; + } + } else { + bool has_unused_color = false; + u8 ureg = 0; + for (u8 i = 0; i < RA_COLORS; + ++i) { + if (!used_colors[i]) { + has_unused_color = + true; + ureg = i; + break; + } + } + if (has_unused_color) { + extra_v0->vr_pos + .alloc_reg = + ureg; + insn_cg_v2(v2) + ->vr_pos + .alloc_reg = + ureg; + } + } + } + } + } + } +} + +// Remove PHI insn +// Move out from SSA form +static void remove_phi(struct bpf_ir_env *env, struct ir_function *fun) +{ + struct array phi_insns; + INIT_ARRAY(&phi_insns, struct ir_insn *); + + struct ir_basic_block **pos; + array_for(pos, fun->reachable_bbs) + { + struct ir_basic_block *bb = *pos; + struct ir_insn *insn; + list_for_each_entry(insn, &bb->ir_insn_head, list_ptr) { + if (insn->op == IR_INSN_PHI) { + DBGASSERT(insn_cg_v2(insn)->dst); + // Phi cannot be spilled + DBGASSERT(insn_cg_v2(insn_cg_v2(insn)->dst) + ->vr_pos.spilled == 0); + bpf_ir_array_push(env, &phi_insns, &insn); + } else { + break; + } + } + } + + struct ir_insn **pos2; + array_for(pos2, phi_insns) + { + struct ir_insn *insn = *pos2; + + struct ir_vr_pos vrpos = insn_cg_v2(insn)->vr_pos; + + struct phi_value *pos3; + array_for(pos3, insn->phi) + { + struct ir_insn *new_insn = + bpf_ir_create_assign_insn_bb_cg_v2( + env, pos3->bb, pos3->value, + INSERT_BACK_BEFORE_JMP); + + insn_cg_v2(new_insn)->vr_pos = vrpos; + + // Remove use + bpf_ir_val_remove_user(pos3->value, insn); + } + + bpf_ir_array_free(&insn->phi); + + bpf_ir_replace_all_usage_cg( + env, insn, + bpf_ir_value_insn(fun->cg_info.regs[vrpos.alloc_reg])); + erase_insn_cg_v2(env, fun, insn); + } + + bpf_ir_array_free(&phi_insns); +} + +void bpf_ir_compile_v2(struct bpf_ir_env *env, struct ir_function *fun) +{ + u64 starttime = get_cur_time_ns(); + init_cg(env, fun); + CHECK_ERR(); + + // Debugging settings + fun->cg_info.spill_callee = 0; + + change_call(env, fun); + change_fun_arg(env, fun); + spill_array(env, fun); + + bool done = false; + while (!done) { + liveness_analysis(env, fun); + print_interference_graph(env, fun); + + print_ir_prog_cg_dst(env, fun, "After liveness"); + + conflict_analysis(env, fun); + print_interference_graph(env, fun); + + struct array to_spill = pre_spill(env, fun); + if (to_spill.num_elem == 0) { + // No need to spill + done = true; + } else { + // spill + CRITICAL("todo"); + } + bpf_ir_array_free(&to_spill); + } + + // Graph coloring + coloring(env, fun); + CHECK_ERR(); + print_ir_prog_cg_alloc(env, fun, "After Coloring"); + + // Coalesce + coalescing(env, fun); + CHECK_ERR(); + print_ir_prog_cg_alloc(env, fun, "After Coalescing"); + + // SSA Out + remove_phi(env, fun); + CHECK_ERR(); + print_ir_prog_cg_alloc(env, fun, "SSA Out"); + + bpf_ir_cg_norm_v2(env, fun); + CHECK_ERR(); + env->cg_time += get_cur_time_ns() - starttime; +} diff --git a/core/ir_helper.c b/core/ir_helper.c index c3a0d2c5..cf1b3442 100644 --- a/core/ir_helper.c +++ b/core/ir_helper.c @@ -162,7 +162,7 @@ static void print_vr_pos(struct bpf_ir_env *env, struct ir_vr_pos *pos) PRINT_LOG_DEBUG(env, "r%u", pos->alloc_reg); } } else { - RAISE_ERROR("Not allocated"); + PRINT_LOG_DEBUG(env, "(NULL)"); } } diff --git a/core/ir_insn.c b/core/ir_insn.c index 00ee4a27..1fb773a3 100644 --- a/core/ir_insn.c +++ b/core/ir_insn.c @@ -33,6 +33,22 @@ struct ir_insn *bpf_ir_create_insn_base_cg(struct bpf_ir_env *env, return new_insn; } +struct ir_insn *bpf_ir_create_insn_base_cg_v2(struct bpf_ir_env *env, + struct ir_basic_block *bb, + enum ir_insn_type insn_type) +{ + struct ir_insn *new_insn = bpf_ir_create_insn_base(env, bb); + if (!new_insn) { + env->err = -ENOMEM; + PRINT_LOG_DEBUG(env, "Failed to allocate memory for ir_insn\n"); + return NULL; + } + new_insn->op = insn_type; + bpf_ir_init_insn_cg_v2(env, new_insn); + CHECK_ERR(NULL); + return new_insn; +} + struct ir_insn *bpf_ir_create_insn_base_norm(struct bpf_ir_env *env, struct ir_basic_block *bb, struct ir_vr_pos dstpos) @@ -842,6 +858,20 @@ static struct ir_insn *create_assign_insn_base_norm(struct bpf_ir_env *env, return new_insn; } +static struct ir_insn *create_assign_insn_base_cg_v2(struct bpf_ir_env *env, + struct ir_basic_block *bb, + struct ir_value val) +{ + struct ir_insn *new_insn = + bpf_ir_create_insn_base_cg_v2(env, bb, IR_INSN_ASSIGN); + new_insn->values[0] = val; + new_insn->value_num = 1; + new_insn->vr_type = IR_VR_TYPE_UNKNOWN; + new_insn->alu_op = IR_ALU_UNKNOWN; + bpf_ir_val_add_user(env, val, new_insn); + return new_insn; +} + static struct ir_insn *create_phi_insn_base(struct bpf_ir_env *env, struct ir_basic_block *bb) { @@ -1491,6 +1521,27 @@ struct ir_insn *bpf_ir_create_assign_insn_bb_norm(struct bpf_ir_env *env, return new_insn; } +struct ir_insn *bpf_ir_create_assign_insn_cg_v2(struct bpf_ir_env *env, + struct ir_insn *pos_insn, + struct ir_value val, + enum insert_position pos) +{ + struct ir_insn *new_insn = + create_assign_insn_base_cg_v2(env, pos_insn->parent_bb, val); + bpf_ir_insert_at(new_insn, pos_insn, pos); + return new_insn; +} + +struct ir_insn *bpf_ir_create_assign_insn_bb_cg_v2( + struct bpf_ir_env *env, struct ir_basic_block *pos_bb, + struct ir_value val, enum insert_position pos) +{ + struct ir_insn *new_insn = + create_assign_insn_base_cg_v2(env, pos_bb, val); + bpf_ir_insert_at_bb(new_insn, pos_bb, pos); + return new_insn; +} + struct ir_insn *bpf_ir_create_phi_insn(struct bpf_ir_env *env, struct ir_insn *pos_insn, enum insert_position pos) diff --git a/core/ptrset.c b/core/ptrset.c index dbdf75c6..df42bf82 100644 --- a/core/ptrset.c +++ b/core/ptrset.c @@ -116,11 +116,30 @@ void bpf_ir_ptrset_clean(struct ptrset *set) void bpf_ir_ptrset_free(struct ptrset *set) { bpf_ir_ptrset_clean(set); - free_proto(set->set); + if (set->set) { + free_proto(set->set); + } set->size = 0; set->set = NULL; } +void **bpf_ir_ptrset_next(struct ptrset *set, void **keyd) +{ + struct ptrset_entry *cc; + if (keyd == NULL) { + cc = set->set; + } else { + cc = container_of(keyd, struct ptrset_entry, key) + 1; + } + while ((size_t)(cc - set->set) < set->size) { + if (cc->occupy == 1) { + return &cc->key; + } + cc++; + } + return NULL; +} + struct ptrset bpf_ir_ptrset_union(struct bpf_ir_env *env, struct ptrset *set1, struct ptrset *set2) { @@ -164,10 +183,10 @@ void bpf_ir_ptrset_move(struct ptrset *set1, struct ptrset *set2) } // Clone set2 to set1 +// Make sure set1 is empty (no data) void bpf_ir_ptrset_clone(struct bpf_ir_env *env, struct ptrset *set1, struct ptrset *set2) { - bpf_ir_ptrset_free(set1); bpf_ir_ptrset_init(env, set1, set2->size); for (size_t i = 0; i < set2->size; ++i) { if (set2->set[i].occupy > 0) { diff --git a/core/tests/test_ptrset.c b/core/tests/test_ptrset.c index 6e9accd5..766983ec 100644 --- a/core/tests/test_ptrset.c +++ b/core/tests/test_ptrset.c @@ -93,7 +93,13 @@ void test(int initsize) CRITICAL_ASSERT(env, set5.cnt == 2); - // bpf_ir_ptrset_print_dbg(env, &set3, print_key); + bpf_ir_ptrset_print_dbg(env, &set3, print_key); + + char **pos; + ptrset_for(pos, set3) + { + printf("Key: %s\n", *pos); + } bpf_ir_ptrset_free(&set); bpf_ir_ptrset_free(&set2); @@ -104,10 +110,35 @@ void test(int initsize) bpf_ir_free_env(env); } +void test2(void) +{ + struct bpf_ir_opts opts = bpf_ir_default_opts(); + opts.verbose = 5; + struct bpf_ir_env *env = bpf_ir_init_env(opts, NULL, 0); + + struct ptrset set; + INIT_PTRSET_DEF(&set); + + char ss[10] = "123312"; + + bpf_ir_ptrset_insert(env, &set, ss); + + char **pos; + + ptrset_for(pos, set) + { + printf("Key: %s\n", *pos); + } + + bpf_ir_ptrset_free(&set); + bpf_ir_free_env(env); +} + int main(void) { for (int i = 1; i < 10; i++) { test(i); } + test2(); return 0; }