diff options
Diffstat (limited to 'tools/testing/selftests/bpf/benchs/bench_trigger.c')
| -rw-r--r-- | tools/testing/selftests/bpf/benchs/bench_trigger.c | 224 |
1 files changed, 182 insertions, 42 deletions
diff --git a/tools/testing/selftests/bpf/benchs/bench_trigger.c b/tools/testing/selftests/bpf/benchs/bench_trigger.c index dbd362771d6a..d66eddacd642 100644 --- a/tools/testing/selftests/bpf/benchs/bench_trigger.c +++ b/tools/testing/selftests/bpf/benchs/bench_trigger.c @@ -1,15 +1,47 @@ // SPDX-License-Identifier: GPL-2.0 /* Copyright (c) 2020 Facebook */ +#define _GNU_SOURCE +#include <unistd.h> #include "bench.h" #include "trigger_bench.skel.h" #include "trace_helpers.h" +/* adjust slot shift in inc_hits() if changing */ +#define MAX_BUCKETS 256 + +#pragma GCC diagnostic ignored "-Wattributes" + /* BPF triggering benchmarks */ static struct trigger_ctx { struct trigger_bench *skel; } ctx; -static struct counter base_hits; +static struct counter base_hits[MAX_BUCKETS]; + +static __always_inline void inc_counter(struct counter *counters) +{ + static __thread int tid = 0; + unsigned slot; + + if (unlikely(tid == 0)) + tid = syscall(SYS_gettid); + + /* multiplicative hashing, it's fast */ + slot = 2654435769U * tid; + slot >>= 24; + + atomic_inc(&base_hits[slot].value); /* use highest byte as an index */ +} + +static long sum_and_reset_counters(struct counter *counters) +{ + int i; + long sum = 0; + + for (i = 0; i < MAX_BUCKETS; i++) + sum += atomic_swap(&counters[i].value, 0); + return sum; +} static void trigger_validate(void) { @@ -23,14 +55,14 @@ static void *trigger_base_producer(void *input) { while (true) { (void)syscall(__NR_getpgid); - atomic_inc(&base_hits.value); + inc_counter(base_hits); } return NULL; } static void trigger_base_measure(struct bench_res *res) { - res->hits = atomic_swap(&base_hits.value, 0); + res->hits = sum_and_reset_counters(base_hits); } static void *trigger_producer(void *input) @@ -42,7 +74,7 @@ static void *trigger_producer(void *input) static void trigger_measure(struct bench_res *res) { - res->hits = atomic_swap(&ctx.skel->bss->hits, 0); + res->hits = sum_and_reset_counters(ctx.skel->bss->hits); } static void setup_ctx(void) @@ -85,12 +117,36 @@ static void trigger_kprobe_setup(void) attach_bpf(ctx.skel->progs.bench_trigger_kprobe); } +static void trigger_kretprobe_setup(void) +{ + setup_ctx(); + attach_bpf(ctx.skel->progs.bench_trigger_kretprobe); +} + +static void trigger_kprobe_multi_setup(void) +{ + setup_ctx(); + attach_bpf(ctx.skel->progs.bench_trigger_kprobe_multi); +} + +static void trigger_kretprobe_multi_setup(void) +{ + setup_ctx(); + attach_bpf(ctx.skel->progs.bench_trigger_kretprobe_multi); +} + static void trigger_fentry_setup(void) { setup_ctx(); attach_bpf(ctx.skel->progs.bench_trigger_fentry); } +static void trigger_fexit_setup(void) +{ + setup_ctx(); + attach_bpf(ctx.skel->progs.bench_trigger_fexit); +} + static void trigger_fentry_sleep_setup(void) { setup_ctx(); @@ -113,12 +169,25 @@ static void trigger_fmodret_setup(void) * GCC doesn't generate stack setup preample for these functions due to them * having no input arguments and doing nothing in the body. */ -__weak void uprobe_target_with_nop(void) +__nocf_check __weak void uprobe_target_nop(void) { asm volatile ("nop"); } -__weak void uprobe_target_without_nop(void) +__weak void opaque_noop_func(void) +{ +} + +__nocf_check __weak int uprobe_target_push(void) +{ + /* overhead of function call is negligible compared to uprobe + * triggering, so this shouldn't affect benchmark results much + */ + opaque_noop_func(); + return 1; +} + +__nocf_check __weak void uprobe_target_ret(void) { asm volatile (""); } @@ -126,27 +195,34 @@ __weak void uprobe_target_without_nop(void) static void *uprobe_base_producer(void *input) { while (true) { - uprobe_target_with_nop(); - atomic_inc(&base_hits.value); + uprobe_target_nop(); + inc_counter(base_hits); } return NULL; } -static void *uprobe_producer_with_nop(void *input) +static void *uprobe_producer_nop(void *input) +{ + while (true) + uprobe_target_nop(); + return NULL; +} + +static void *uprobe_producer_push(void *input) { while (true) - uprobe_target_with_nop(); + uprobe_target_push(); return NULL; } -static void *uprobe_producer_without_nop(void *input) +static void *uprobe_producer_ret(void *input) { while (true) - uprobe_target_without_nop(); + uprobe_target_ret(); return NULL; } -static void usetup(bool use_retprobe, bool use_nop) +static void usetup(bool use_retprobe, void *target_addr) { size_t uprobe_offset; struct bpf_link *link; @@ -159,11 +235,7 @@ static void usetup(bool use_retprobe, bool use_nop) exit(1); } - if (use_nop) - uprobe_offset = get_uprobe_offset(&uprobe_target_with_nop); - else - uprobe_offset = get_uprobe_offset(&uprobe_target_without_nop); - + uprobe_offset = get_uprobe_offset(target_addr); link = bpf_program__attach_uprobe(ctx.skel->progs.bench_trigger_uprobe, use_retprobe, -1 /* all PIDs */, @@ -176,24 +248,34 @@ static void usetup(bool use_retprobe, bool use_nop) ctx.skel->links.bench_trigger_uprobe = link; } -static void uprobe_setup_with_nop(void) +static void uprobe_setup_nop(void) +{ + usetup(false, &uprobe_target_nop); +} + +static void uretprobe_setup_nop(void) +{ + usetup(true, &uprobe_target_nop); +} + +static void uprobe_setup_push(void) { - usetup(false, true); + usetup(false, &uprobe_target_push); } -static void uretprobe_setup_with_nop(void) +static void uretprobe_setup_push(void) { - usetup(true, true); + usetup(true, &uprobe_target_push); } -static void uprobe_setup_without_nop(void) +static void uprobe_setup_ret(void) { - usetup(false, false); + usetup(false, &uprobe_target_ret); } -static void uretprobe_setup_without_nop(void) +static void uretprobe_setup_ret(void) { - usetup(true, false); + usetup(true, &uprobe_target_ret); } const struct bench bench_trig_base = { @@ -235,6 +317,36 @@ const struct bench bench_trig_kprobe = { .report_final = hits_drops_report_final, }; +const struct bench bench_trig_kretprobe = { + .name = "trig-kretprobe", + .validate = trigger_validate, + .setup = trigger_kretprobe_setup, + .producer_thread = trigger_producer, + .measure = trigger_measure, + .report_progress = hits_drops_report_progress, + .report_final = hits_drops_report_final, +}; + +const struct bench bench_trig_kprobe_multi = { + .name = "trig-kprobe-multi", + .validate = trigger_validate, + .setup = trigger_kprobe_multi_setup, + .producer_thread = trigger_producer, + .measure = trigger_measure, + .report_progress = hits_drops_report_progress, + .report_final = hits_drops_report_final, +}; + +const struct bench bench_trig_kretprobe_multi = { + .name = "trig-kretprobe-multi", + .validate = trigger_validate, + .setup = trigger_kretprobe_multi_setup, + .producer_thread = trigger_producer, + .measure = trigger_measure, + .report_progress = hits_drops_report_progress, + .report_final = hits_drops_report_final, +}; + const struct bench bench_trig_fentry = { .name = "trig-fentry", .validate = trigger_validate, @@ -245,6 +357,16 @@ const struct bench bench_trig_fentry = { .report_final = hits_drops_report_final, }; +const struct bench bench_trig_fexit = { + .name = "trig-fexit", + .validate = trigger_validate, + .setup = trigger_fexit_setup, + .producer_thread = trigger_producer, + .measure = trigger_measure, + .report_progress = hits_drops_report_progress, + .report_final = hits_drops_report_final, +}; + const struct bench bench_trig_fentry_sleep = { .name = "trig-fentry-sleep", .validate = trigger_validate, @@ -274,37 +396,55 @@ const struct bench bench_trig_uprobe_base = { .report_final = hits_drops_report_final, }; -const struct bench bench_trig_uprobe_with_nop = { - .name = "trig-uprobe-with-nop", - .setup = uprobe_setup_with_nop, - .producer_thread = uprobe_producer_with_nop, +const struct bench bench_trig_uprobe_nop = { + .name = "trig-uprobe-nop", + .setup = uprobe_setup_nop, + .producer_thread = uprobe_producer_nop, + .measure = trigger_measure, + .report_progress = hits_drops_report_progress, + .report_final = hits_drops_report_final, +}; + +const struct bench bench_trig_uretprobe_nop = { + .name = "trig-uretprobe-nop", + .setup = uretprobe_setup_nop, + .producer_thread = uprobe_producer_nop, + .measure = trigger_measure, + .report_progress = hits_drops_report_progress, + .report_final = hits_drops_report_final, +}; + +const struct bench bench_trig_uprobe_push = { + .name = "trig-uprobe-push", + .setup = uprobe_setup_push, + .producer_thread = uprobe_producer_push, .measure = trigger_measure, .report_progress = hits_drops_report_progress, .report_final = hits_drops_report_final, }; -const struct bench bench_trig_uretprobe_with_nop = { - .name = "trig-uretprobe-with-nop", - .setup = uretprobe_setup_with_nop, - .producer_thread = uprobe_producer_with_nop, +const struct bench bench_trig_uretprobe_push = { + .name = "trig-uretprobe-push", + .setup = uretprobe_setup_push, + .producer_thread = uprobe_producer_push, .measure = trigger_measure, .report_progress = hits_drops_report_progress, .report_final = hits_drops_report_final, }; -const struct bench bench_trig_uprobe_without_nop = { - .name = "trig-uprobe-without-nop", - .setup = uprobe_setup_without_nop, - .producer_thread = uprobe_producer_without_nop, +const struct bench bench_trig_uprobe_ret = { + .name = "trig-uprobe-ret", + .setup = uprobe_setup_ret, + .producer_thread = uprobe_producer_ret, .measure = trigger_measure, .report_progress = hits_drops_report_progress, .report_final = hits_drops_report_final, }; -const struct bench bench_trig_uretprobe_without_nop = { - .name = "trig-uretprobe-without-nop", - .setup = uretprobe_setup_without_nop, - .producer_thread = uprobe_producer_without_nop, +const struct bench bench_trig_uretprobe_ret = { + .name = "trig-uretprobe-ret", + .setup = uretprobe_setup_ret, + .producer_thread = uprobe_producer_ret, .measure = trigger_measure, .report_progress = hits_drops_report_progress, .report_final = hits_drops_report_final, |