diff options
Diffstat (limited to 'tools/perf/util')
74 files changed, 4331 insertions, 606 deletions
diff --git a/tools/perf/util/Build b/tools/perf/util/Build index b64bdc1a7026..2d4fa1304178 100644 --- a/tools/perf/util/Build +++ b/tools/perf/util/Build @@ -126,6 +126,7 @@ perf-y += parse-regs-options.o perf-y += parse-sublevel-options.o perf-y += term.o perf-y += help-unknown-cmd.o +perf-y += dlfilter.o perf-y += mem-events.o perf-y += vsprintf.o perf-y += units.o @@ -140,6 +141,7 @@ perf-y += clockid.o perf-$(CONFIG_LIBBPF) += bpf-loader.o perf-$(CONFIG_LIBBPF) += bpf_map.o perf-$(CONFIG_PERF_BPF_SKEL) += bpf_counter.o +perf-$(CONFIG_PERF_BPF_SKEL) += bpf_counter_cgroup.o perf-$(CONFIG_BPF_PROLOGUE) += bpf-prologue.o perf-$(CONFIG_LIBELF) += symbol-elf.o perf-$(CONFIG_LIBELF) += probe-file.o @@ -216,7 +218,7 @@ $(OUTPUT)util/parse-events-flex.c $(OUTPUT)util/parse-events-flex.h: util/parse- $(OUTPUT)util/parse-events-bison.c $(OUTPUT)util/parse-events-bison.h: util/parse-events.y $(call rule_mkdir) - $(Q)$(call echo-cmd,bison)$(BISON) -v $< -d $(PARSER_DEBUG_BISON) \ + $(Q)$(call echo-cmd,bison)$(BISON) -v $< -d $(PARSER_DEBUG_BISON) $(BISON_FILE_PREFIX_MAP) \ -o $(OUTPUT)util/parse-events-bison.c -p parse_events_ $(OUTPUT)util/expr-flex.c $(OUTPUT)util/expr-flex.h: util/expr.l $(OUTPUT)util/expr-bison.c @@ -226,7 +228,7 @@ $(OUTPUT)util/expr-flex.c $(OUTPUT)util/expr-flex.h: util/expr.l $(OUTPUT)util/e $(OUTPUT)util/expr-bison.c $(OUTPUT)util/expr-bison.h: util/expr.y $(call rule_mkdir) - $(Q)$(call echo-cmd,bison)$(BISON) -v $< -d $(PARSER_DEBUG_BISON) \ + $(Q)$(call echo-cmd,bison)$(BISON) -v $< -d $(PARSER_DEBUG_BISON) $(BISON_FILE_PREFIX_MAP) \ -o $(OUTPUT)util/expr-bison.c -p expr_ $(OUTPUT)util/pmu-flex.c $(OUTPUT)util/pmu-flex.h: util/pmu.l $(OUTPUT)util/pmu-bison.c @@ -236,7 +238,7 @@ $(OUTPUT)util/pmu-flex.c $(OUTPUT)util/pmu-flex.h: util/pmu.l $(OUTPUT)util/pmu- $(OUTPUT)util/pmu-bison.c $(OUTPUT)util/pmu-bison.h: util/pmu.y $(call rule_mkdir) - $(Q)$(call echo-cmd,bison)$(BISON) -v $< -d $(PARSER_DEBUG_BISON) \ + $(Q)$(call echo-cmd,bison)$(BISON) -v $< -d $(PARSER_DEBUG_BISON) $(BISON_FILE_PREFIX_MAP) \ -o $(OUTPUT)util/pmu-bison.c -p perf_pmu_ FLEX_GE_26 := $(shell expr $(shell $(FLEX) --version | sed -e 's/flex \([0-9]\+\).\([0-9]\+\)/\1\2/g') \>\= 26) diff --git a/tools/perf/util/annotate.c b/tools/perf/util/annotate.c index abe1499a9164..aa04a3655236 100644 --- a/tools/perf/util/annotate.c +++ b/tools/perf/util/annotate.c @@ -961,7 +961,7 @@ static int symbol__inc_addr_samples(struct map_symbol *ms, if (sym == NULL) return 0; src = symbol__hists(sym, evsel->evlist->core.nr_entries); - return src ? __symbol__inc_addr_samples(ms, src, evsel->idx, addr, sample) : 0; + return src ? __symbol__inc_addr_samples(ms, src, evsel->core.idx, addr, sample) : 0; } static int symbol__account_cycles(u64 addr, u64 start, @@ -2159,7 +2159,7 @@ static void annotation__calc_percent(struct annotation *notes, BUG_ON(i >= al->data_nr); - sym_hist = annotation__histogram(notes, evsel->idx); + sym_hist = annotation__histogram(notes, evsel->core.idx); data = &al->data[i++]; calc_percent(sym_hist, hists, data, al->offset, end); @@ -2340,7 +2340,7 @@ static void print_summary(struct rb_root *root, const char *filename) static void symbol__annotate_hits(struct symbol *sym, struct evsel *evsel) { struct annotation *notes = symbol__annotation(sym); - struct sym_hist *h = annotation__histogram(notes, evsel->idx); + struct sym_hist *h = annotation__histogram(notes, evsel->core.idx); u64 len = symbol__size(sym), offset; for (offset = 0; offset < len; ++offset) @@ -2373,7 +2373,7 @@ int symbol__annotate_printf(struct map_symbol *ms, struct evsel *evsel, const char *d_filename; const char *evsel_name = evsel__name(evsel); struct annotation *notes = symbol__annotation(sym); - struct sym_hist *h = annotation__histogram(notes, evsel->idx); + struct sym_hist *h = annotation__histogram(notes, evsel->core.idx); struct annotation_line *pos, *queue = NULL; u64 start = map__rip_2objdump(map, sym->start); int printed = 2, queue_len = 0, addr_fmt_width; diff --git a/tools/perf/util/arm-spe.c b/tools/perf/util/arm-spe.c index 2539d4baec44..58b7069c5a5f 100644 --- a/tools/perf/util/arm-spe.c +++ b/tools/perf/util/arm-spe.c @@ -26,6 +26,7 @@ #include "symbol.h" #include "thread.h" #include "thread-stack.h" +#include "tsc.h" #include "tool.h" #include "util/synthetic-events.h" @@ -45,6 +46,8 @@ struct arm_spe { struct machine *machine; u32 pmu_type; + struct perf_tsc_conversion tc; + u8 timeless_decoding; u8 data_queued; @@ -231,7 +234,7 @@ static void arm_spe_prep_sample(struct arm_spe *spe, struct arm_spe_record *record = &speq->decoder->record; if (!spe->timeless_decoding) - sample->time = speq->timestamp; + sample->time = tsc_to_perf_time(record->timestamp, &spe->tc); sample->ip = record->from_ip; sample->cpumode = arm_spe_cpumode(spe, sample->ip); @@ -431,12 +434,36 @@ static int arm_spe_sample(struct arm_spe_queue *speq) static int arm_spe_run_decoder(struct arm_spe_queue *speq, u64 *timestamp) { struct arm_spe *spe = speq->spe; + struct arm_spe_record *record; int ret; if (!spe->kernel_start) spe->kernel_start = machine__kernel_start(spe->machine); while (1) { + /* + * The usual logic is firstly to decode the packets, and then + * based the record to synthesize sample; but here the flow is + * reversed: it calls arm_spe_sample() for synthesizing samples + * prior to arm_spe_decode(). + * + * Two reasons for this code logic: + * 1. Firstly, when setup queue in arm_spe__setup_queue(), it + * has decoded trace data and generated a record, but the record + * is left to generate sample until run to here, so it's correct + * to synthesize sample for the left record. + * 2. After decoding trace data, it needs to compare the record + * timestamp with the coming perf event, if the record timestamp + * is later than the perf event, it needs bail out and pushs the + * record into auxtrace heap, thus the record can be deferred to + * synthesize sample until run to here at the next time; so this + * can correlate samples between Arm SPE trace data and other + * perf events with correct time ordering. + */ + ret = arm_spe_sample(speq); + if (ret) + return ret; + ret = arm_spe_decode(speq->decoder); if (!ret) { pr_debug("No data or all data has been processed.\n"); @@ -450,10 +477,17 @@ static int arm_spe_run_decoder(struct arm_spe_queue *speq, u64 *timestamp) if (ret < 0) continue; - ret = arm_spe_sample(speq); - if (ret) - return ret; + record = &speq->decoder->record; + /* Update timestamp for the last record */ + if (record->timestamp > speq->timestamp) + speq->timestamp = record->timestamp; + + /* + * If the timestamp of the queue is later than timestamp of the + * coming perf event, bail out so can allow the perf event to + * be processed ahead. + */ if (!spe->timeless_decoding && speq->timestamp >= *timestamp) { *timestamp = speq->timestamp; return 0; @@ -666,7 +700,7 @@ static int arm_spe_process_event(struct perf_session *session, } if (sample->time && (sample->time != (u64) -1)) - timestamp = sample->time; + timestamp = perf_time_to_tsc(sample->time, &spe->tc); else timestamp = 0; @@ -683,11 +717,7 @@ static int arm_spe_process_event(struct perf_session *session, sample->time); } } else if (timestamp) { - if (event->header.type == PERF_RECORD_EXIT) { - err = arm_spe_process_queues(spe, timestamp); - if (err) - return err; - } + err = arm_spe_process_queues(spe, timestamp); } return err; @@ -1006,6 +1036,7 @@ int arm_spe_process_auxtrace_info(union perf_event *event, { struct perf_record_auxtrace_info *auxtrace_info = &event->auxtrace_info; size_t min_sz = sizeof(u64) * ARM_SPE_AUXTRACE_PRIV_MAX; + struct perf_record_time_conv *tc = &session->time_conv; struct arm_spe *spe; int err; @@ -1027,6 +1058,28 @@ int arm_spe_process_auxtrace_info(union perf_event *event, spe->pmu_type = auxtrace_info->priv[ARM_SPE_PMU_TYPE]; spe->timeless_decoding = arm_spe__is_timeless_decoding(spe); + + /* + * The synthesized event PERF_RECORD_TIME_CONV has been handled ahead + * and the parameters for hardware clock are stored in the session + * context. Passes these parameters to the struct perf_tsc_conversion + * in "spe->tc", which is used for later conversion between clock + * counter and timestamp. + * + * For backward compatibility, copies the fields starting from + * "time_cycles" only if they are contained in the event. + */ + spe->tc.time_shift = tc->time_shift; + spe->tc.time_mult = tc->time_mult; + spe->tc.time_zero = tc->time_zero; + + if (event_contains(*tc, time_cycles)) { + spe->tc.time_cycles = tc->time_cycles; + spe->tc.time_mask = tc->time_mask; + spe->tc.cap_user_time_zero = tc->cap_user_time_zero; + spe->tc.cap_user_time_short = tc->cap_user_time_short; + } + spe->auxtrace.process_event = arm_spe_process_event; spe->auxtrace.process_auxtrace_event = arm_spe_process_auxtrace_event; spe->auxtrace.flush_events = arm_spe_flush; diff --git a/tools/perf/util/auxtrace.c b/tools/perf/util/auxtrace.c index 1b4091a3b508..cb19669d2a5b 100644 --- a/tools/perf/util/auxtrace.c +++ b/tools/perf/util/auxtrace.c @@ -73,8 +73,8 @@ static int evlist__regroup(struct evlist *evlist, struct evsel *leader, struct e grp = false; evlist__for_each_entry(evlist, evsel) { if (grp) { - if (!(evsel->leader == leader || - (evsel->leader == evsel && + if (!(evsel__leader(evsel) == leader || + (evsel__leader(evsel) == evsel && evsel->core.nr_members <= 1))) return -EINVAL; } else if (evsel == leader) { @@ -87,8 +87,8 @@ static int evlist__regroup(struct evlist *evlist, struct evsel *leader, struct e grp = false; evlist__for_each_entry(evlist, evsel) { if (grp) { - if (evsel->leader != leader) { - evsel->leader = leader; + if (!evsel__has_leader(evsel, leader)) { + evsel__set_leader(evsel, leader); if (leader->core.nr_members < 1) leader->core.nr_members = 1; leader->core.nr_members += 1; @@ -1120,8 +1120,9 @@ int auxtrace_queue_data(struct perf_session *session, bool samples, bool events) auxtrace_queue_data_cb, &qd); } -void *auxtrace_buffer__get_data(struct auxtrace_buffer *buffer, int fd) +void *auxtrace_buffer__get_data_rw(struct auxtrace_buffer *buffer, int fd, bool rw) { + int prot = rw ? PROT_READ | PROT_WRITE : PROT_READ; size_t adj = buffer->data_offset & (page_size - 1); size_t size = buffer->size + adj; off_t file_offset = buffer->data_offset - adj; @@ -1130,7 +1131,7 @@ void *auxtrace_buffer__get_data(struct auxtrace_buffer *buffer, int fd) if (buffer->data) return buffer->data; - addr = mmap(NULL, size, PROT_READ, MAP_SHARED, fd, file_offset); + addr = mmap(NULL, size, prot, MAP_SHARED, fd, file_offset); if (addr == MAP_FAILED) return NULL; @@ -1230,11 +1231,11 @@ static void unleader_evsel(struct evlist *evlist, struct evsel *leader) /* Find new leader for the group */ evlist__for_each_entry(evlist, evsel) { - if (evsel->leader != leader || evsel == leader) + if (!evsel__has_leader(evsel, leader) || evsel == leader) continue; if (!new_leader) new_leader = evsel; - evsel->leader = new_leader; + evsel__set_leader(evsel, new_leader); } /* Update group information */ @@ -1404,10 +1405,9 @@ static int get_flags(const char **ptr, unsigned int *plus_flags, unsigned int *m * about the options parsed here, which is introduced after this cset, * when support in 'perf script' for these options is introduced. */ -int itrace_parse_synth_opts(const struct option *opt, const char *str, - int unset) +int itrace_do_parse_synth_opts(struct itrace_synth_opts *synth_opts, + const char *str, int unset) { - struct itrace_synth_opts *synth_opts = opt->value; const char *p; char *endptr; bool period_type_set = false; @@ -1569,6 +1569,9 @@ int itrace_parse_synth_opts(const struct option *opt, const char *str, case 'q': synth_opts->quick += 1; break; + case 'Z': + synth_opts->timeless_decoding = true; + break; case ' ': case ',': break; @@ -1592,6 +1595,11 @@ out_err: return -EINVAL; } +int itrace_parse_synth_opts(const struct option *opt, const char *str, int unset) +{ + return itrace_do_parse_synth_opts(opt->value, str, unset); +} + static const char * const auxtrace_error_type_name[] = { [PERF_AUXTRACE_ERROR_ITRACE] = "instruction trace", }; diff --git a/tools/perf/util/auxtrace.h b/tools/perf/util/auxtrace.h index a4fbb33b7245..cc1c1b9cec9c 100644 --- a/tools/perf/util/auxtrace.h +++ b/tools/perf/util/auxtrace.h @@ -89,6 +89,10 @@ enum itrace_period_type { * @tlb: whether to synthesize TLB events * @remote_access: whether to synthesize remote access events * @mem: whether to synthesize memory events + * @timeless_decoding: prefer "timeless" decoding i.e. ignore timestamps + * @vm_time_correlation: perform VM Time Correlation + * @vm_tm_corr_dry_run: VM Time Correlation dry-run + * @vm_tm_corr_args: VM Time Correlation implementation-specific arguments * @callchain_sz: maximum callchain size * @last_branch_sz: branch context size * @period: 'instructions' events period @@ -128,6 +132,10 @@ struct itrace_synth_opts { bool tlb; bool remote_access; bool mem; + bool timeless_decoding; + bool vm_time_correlation; + bool vm_tm_corr_dry_run; + char *vm_tm_corr_args; unsigned int callchain_sz; unsigned int last_branch_sz; unsigned long long period; @@ -444,7 +452,7 @@ static inline u64 auxtrace_mmap__read_snapshot_head(struct auxtrace_mmap *mm) u64 head = READ_ONCE(pc->aux_head); /* Ensure all reads are done after we read the head */ - rmb(); + smp_rmb(); return head; } @@ -458,7 +466,7 @@ static inline u64 auxtrace_mmap__read_head(struct auxtrace_mmap *mm) #endif /* Ensure all reads are done after we read the head */ - rmb(); + smp_rmb(); return head; } @@ -470,7 +478,7 @@ static inline void auxtrace_mmap__write_tail(struct auxtrace_mmap *mm, u64 tail) #endif /* Ensure all reads are done before we write the tail out */ - mb(); + smp_mb(); #if BITS_PER_LONG == 64 || !defined(HAVE_SYNC_COMPARE_AND_SWAP_SUPPORT) pc->aux_tail = tail; #else @@ -525,7 +533,11 @@ int auxtrace_queue_data(struct perf_session *session, bool samples, bool events); struct auxtrace_buffer *auxtrace_buffer__next(struct auxtrace_queue *queue, struct auxtrace_buffer *buffer); -void *auxtrace_buffer__get_data(struct auxtrace_buffer *buffer, int fd); +void *auxtrace_buffer__get_data_rw(struct auxtrace_buffer *buffer, int fd, bool rw); +static inline void *auxtrace_buffer__get_data(struct auxtrace_buffer *buffer, int fd) +{ + return auxtrace_buffer__get_data_rw(buffer, fd, false); +} void auxtrace_buffer__put_data(struct auxtrace_buffer *buffer); void auxtrace_buffer__drop_data(struct auxtrace_buffer *buffer); void auxtrace_buffer__free(struct auxtrace_buffer *buffer); @@ -595,6 +607,8 @@ s64 perf_event__process_auxtrace(struct perf_session *session, union perf_event *event); int perf_event__process_auxtrace_error(struct perf_session *session, union perf_event *event); +int itrace_do_parse_synth_opts(struct itrace_synth_opts *synth_opts, + const char *str, int unset); int itrace_parse_synth_opts(const struct option *opt, const char *str, int unset); void itrace_synth_opts__set_default(struct itrace_synth_opts *synth_opts, @@ -691,9 +705,26 @@ int auxtrace_record__options(struct auxtrace_record *itr __maybe_unused, return 0; } -#define perf_event__process_auxtrace_info 0 -#define perf_event__process_auxtrace 0 -#define perf_event__process_auxtrace_error 0 +static inline +int perf_event__process_auxtrace_info(struct perf_session *session __maybe_unused, + union perf_event *event __maybe_unused) +{ + return 0; +} + +static inline +s64 perf_event__process_auxtrace(struct perf_session *session __maybe_unused, + union perf_event *event __maybe_unused) +{ + return 0; +} + +static inline +int perf_event__process_auxtrace_error(struct perf_session *session __maybe_unused, + union perf_event *event __maybe_unused) +{ + return 0; +} static inline void perf_session__auxtrace_error_inc(struct perf_session *session @@ -710,6 +741,14 @@ void events_stats__auxtrace_error_warn(const struct events_stats *stats } static inline +int itrace_do_parse_synth_opts(struct itrace_synth_opts *synth_opts __maybe_unused, + const char *str __maybe_unused, int unset __maybe_unused) +{ + pr_err("AUX area tracing not supported\n"); + return -EINVAL; +} + +static inline int itrace_parse_synth_opts(const struct option *opt __maybe_unused, const char *str __maybe_unused, int unset __maybe_unused) diff --git a/tools/perf/util/bpf_counter.c b/tools/perf/util/bpf_counter.c index ddb52f748c8e..8150e03367bb 100644 --- a/tools/perf/util/bpf_counter.c +++ b/tools/perf/util/bpf_counter.c @@ -7,12 +7,8 @@ #include <unistd.h> #include <sys/file.h> #include <sys/time.h> -#include <sys/resource.h> #include <linux/err.h> #include <linux/zalloc.h> -#include <bpf/bpf.h> -#include <bpf/btf.h> -#include <bpf/libbpf.h> #include <api/fs/fs.h> #include <perf/bpf_perf.h> @@ -22,6 +18,7 @@ #include "evsel.h" #include "evlist.h" #include "target.h" +#include "cgroup.h" #include "cpumap.h" #include "thread_map.h" @@ -37,13 +34,6 @@ static inline void *u64_to_ptr(__u64 ptr) return (void *)(unsigned long)ptr; } -static void set_max_rlimit(void) -{ - struct rlimit rinf = { RLIM_INFINITY, RLIM_INFINITY }; - - setrlimit(RLIMIT_MEMLOCK, &rinf); -} - static struct bpf_counter *bpf_counter_alloc(void) { struct bpf_counter *counter; @@ -297,33 +287,6 @@ struct bpf_counter_ops bpf_program_profiler_ops = { .install_pe = bpf_program_profiler__install_pe, }; -static __u32 bpf_link_get_id(int fd) -{ - struct bpf_link_info link_info = {0}; - __u32 link_info_len = sizeof(link_info); - - bpf_obj_get_info_by_fd(fd, &link_info, &link_info_len); - return link_info.id; -} - -static __u32 bpf_link_get_prog_id(int fd) -{ - struct bpf_link_info link_info = {0}; - __u32 link_info_len = sizeof(link_info); - - bpf_obj_get_info_by_fd(fd, &link_info, &link_info_len); - return link_info.prog_id; -} - -static __u32 bpf_map_get_id(int fd) -{ - struct bpf_map_info map_info = {0}; - __u32 map_info_len = sizeof(map_info); - - bpf_obj_get_info_by_fd(fd, &map_info, &map_info_len); - return map_info.id; -} - static bool bperf_attr_map_compatible(int attr_map_fd) { struct bpf_map_info map_info = {0}; @@ -385,26 +348,12 @@ static int bperf_lock_attr_map(struct target *target) return map_fd; } -/* trigger the leader program on a cpu */ -static int bperf_trigger_reading(int prog_fd, int cpu) -{ - DECLARE_LIBBPF_OPTS(bpf_test_run_opts, opts, - .ctx_in = NULL, - .ctx_size_in = 0, - .flags = BPF_F_TEST_RUN_ON_CPU, - .cpu = cpu, - .retval = 0, - ); - - return bpf_prog_test_run_opts(prog_fd, &opts); -} - static int bperf_check_target(struct evsel *evsel, struct target *target, enum bperf_filter_type *filter_type, __u32 *filter_entry_cnt) { - if (evsel->leader->core.nr_members > 1) { + if (evsel->core.leader->nr_members > 1) { pr_err("bpf managed perf events do not yet support groups.\n"); return -1; } @@ -451,10 +400,10 @@ static int bperf_reload_leader_program(struct evsel *evsel, int attr_map_fd, goto out; } - err = -1; link = bpf_program__attach(skel->progs.on_switch); - if (!link) { + if (IS_ERR(link)) { pr_err("Failed to attach leader program\n"); + err = PTR_ERR(link); goto out; } @@ -521,9 +470,10 @@ static int bperf__load(struct evsel *evsel, struct target *target) evsel->bperf_leader_link_fd = bpf_link_get_fd_by_id(entry.link_id); if (evsel->bperf_leader_link_fd < 0 && - bperf_reload_leader_program(evsel, attr_map_fd, &entry)) + bperf_reload_leader_program(evsel, attr_map_fd, &entry)) { + err = -1; goto out; - + } /* * The bpf_link holds reference to the leader program, and the * leader program holds reference to the maps. Therefore, if @@ -550,6 +500,7 @@ static int bperf__load(struct evsel *evsel, struct target *target) /* Step 2: load the follower skeleton */ evsel->follower_skel = bperf_follower_bpf__open(); if (!evsel->follower_skel) { + err = -1; pr_err("Failed to open follower skeleton\n"); goto out; } @@ -792,6 +743,8 @@ struct bpf_counter_ops bperf_ops = { .destroy = bperf__destroy, }; +extern struct bpf_counter_ops bperf_cgrp_ops; + static inline bool bpf_counter_skip(struct evsel *evsel) { return list_empty(&evsel->bpf_counter_list) && @@ -809,6 +762,8 @@ int bpf_counter__load(struct evsel *evsel, struct target *target) { if (target->bpf_str) evsel->bpf_counter_ops = &bpf_program_profiler_ops; + else if (cgrp_event_expanded && target->use_bpf) + evsel->bpf_counter_ops = &bperf_cgrp_ops; else if (target->use_bpf || evsel->bpf_counter || evsel__match_bpf_counter_events(evsel->name)) evsel->bpf_counter_ops = &bperf_ops; diff --git a/tools/perf/util/bpf_counter.h b/tools/perf/util/bpf_counter.h index d6d907c3dcf9..65ebaa6694fb 100644 --- a/tools/perf/util/bpf_counter.h +++ b/tools/perf/util/bpf_counter.h @@ -3,6 +3,10 @@ #define __PERF_BPF_COUNTER_H 1 #include <linux/list.h> +#include <sys/resource.h> +#include <bpf/bpf.h> +#include <bpf/btf.h> +#include <bpf/libbpf.h> struct evsel; struct target; @@ -76,4 +80,52 @@ static inline int bpf_counter__install_pe(struct evsel *evsel __maybe_unused, #endif /* HAVE_BPF_SKEL */ +static inline void set_max_rlimit(void) +{ + struct rlimit rinf = { RLIM_INFINITY, RLIM_INFINITY }; + + setrlimit(RLIMIT_MEMLOCK, &rinf); +} + +static inline __u32 bpf_link_get_id(int fd) +{ + struct bpf_link_info link_info = { .id = 0, }; + __u32 link_info_len = sizeof(link_info); + + bpf_obj_get_info_by_fd(fd, &link_info, &link_info_len); + return link_info.id; +} + +static inline __u32 bpf_link_get_prog_id(int fd) +{ + struct bpf_link_info link_info = { .id = 0, }; + __u32 link_info_len = sizeof(link_info); + + bpf_obj_get_info_by_fd(fd, &link_info, &link_info_len); + return link_info.prog_id; +} + +static inline __u32 bpf_map_get_id(int fd) +{ + struct bpf_map_info map_info = { .id = 0, }; + __u32 map_info_len = sizeof(map_info); + + bpf_obj_get_info_by_fd(fd, &map_info, &map_info_len); + return map_info.id; +} + +/* trigger the leader program on a cpu */ +static inline int bperf_trigger_reading(int prog_fd, int cpu) +{ + DECLARE_LIBBPF_OPTS(bpf_test_run_opts, opts, + .ctx_in = NULL, + .ctx_size_in = 0, + .flags = BPF_F_TEST_RUN_ON_CPU, + .cpu = cpu, + .retval = 0, + ); + + return bpf_prog_test_run_opts(prog_fd, &opts); +} + #endif /* __PERF_BPF_COUNTER_H */ diff --git a/tools/perf/util/bpf_counter_cgroup.c b/tools/perf/util/bpf_counter_cgroup.c new file mode 100644 index 000000000000..89aa5e71db1a --- /dev/null +++ b/tools/perf/util/bpf_counter_cgroup.c @@ -0,0 +1,307 @@ +// SPDX-License-Identifier: GPL-2.0 + +/* Copyright (c) 2021 Facebook */ +/* Copyright (c) 2021 Google */ + +#include <assert.h> +#include <limits.h> +#include <unistd.h> +#include <sys/file.h> +#include <sys/time.h> +#include <sys/resource.h> +#include <linux/err.h> +#include <linux/zalloc.h> +#include <linux/perf_event.h> +#include <api/fs/fs.h> +#include <perf/bpf_perf.h> + +#include "affinity.h" +#include "bpf_counter.h" +#include "cgroup.h" +#include "counts.h" +#include "debug.h" +#include "evsel.h" +#include "evlist.h" +#include "target.h" +#include "cpumap.h" +#include "thread_map.h" + +#include "bpf_skel/bperf_cgroup.skel.h" + +static struct perf_event_attr cgrp_switch_attr = { + .type = PERF_TYPE_SOFTWARE, + .config = PERF_COUNT_SW_CGROUP_SWITCHES, + .size = sizeof(cgrp_switch_attr), + .sample_period = 1, + .disabled = 1, +}; + +static struct evsel *cgrp_switch; +static struct bperf_cgroup_bpf *skel; + +#define FD(evt, cpu) (*(int *)xyarray__entry(evt->core.fd, cpu, 0)) + +static int bperf_load_program(struct evlist *evlist) +{ + struct bpf_link *link; + struct evsel *evsel; + struct cgroup *cgrp, *leader_cgrp; + __u32 i, cpu; + __u32 nr_cpus = evlist->core.all_cpus->nr; + int total_cpus = cpu__max_cpu(); + int map_size, map_fd; + int prog_fd, err; + + skel = bperf_cgroup_bpf__open(); + if (!skel) { + pr_err("Failed to open cgroup skeleton\n"); + return -1; + } + + skel->rodata->num_cpus = total_cpus; + skel->rodata->num_events = evlist->core.nr_entries / nr_cgroups; + + BUG_ON(evlist->core.nr_entries % nr_cgroups != 0); + + /* we need one copy of events per cpu for reading */ + map_size = total_cpus * evlist->core.nr_entries / nr_cgroups; + bpf_map__resize(skel->maps.events, map_size); + bpf_map__resize(skel->maps.cgrp_idx, nr_cgroups); + /* previous result is saved in a per-cpu array */ + map_size = evlist->core.nr_entries / nr_cgroups; + bpf_map__resize(skel->maps.prev_readings, map_size); + /* cgroup result needs all events (per-cpu) */ + map_size = evlist->core.nr_entries; + bpf_map__resize(skel->maps.cgrp_readings, map_size); + + set_max_rlimit(); + + err = bperf_cgroup_bpf__load(skel); + if (err) { + pr_err("Failed to load cgroup skeleton\n"); + goto out; + } + + if (cgroup_is_v2("perf_event") > 0) + skel->bss->use_cgroup_v2 = 1; + + err = -1; + + cgrp_switch = evsel__new(&cgrp_switch_attr); + if (evsel__open_per_cpu(cgrp_switch, evlist->core.all_cpus, -1) < 0) { + pr_err("Failed to open cgroup switches event\n"); + goto out; + } + + for (i = 0; i < nr_cpus; i++) { + link = bpf_program__attach_perf_event(skel->progs.on_cgrp_switch, + FD(cgrp_switch, i)); + if (IS_ERR(link)) { + pr_err("Failed to attach cgroup program\n"); + err = PTR_ERR(link); + goto out; + } + } + + /* + * Update cgrp_idx map from cgroup-id to event index. + */ + cgrp = NULL; + i = 0; + + evlist__for_each_entry(evlist, evsel) { + if (cgrp == NULL || evsel->cgrp == leader_cgrp) { + leader_cgrp = evsel->cgrp; + evsel->cgrp = NULL; + + /* open single copy of the events w/o cgroup */ + err = evsel__open_per_cpu(evsel, evlist->core.all_cpus, -1); + if (err) { + pr_err("Failed to open first cgroup events\n"); + goto out; + } + + map_fd = bpf_map__fd(skel->maps.events); + for (cpu = 0; cpu < nr_cpus; cpu++) { + int fd = FD(evsel, cpu); + __u32 idx = evsel->core.idx * total_cpus + + evlist->core.all_cpus->map[cpu]; + + err = bpf_map_update_elem(map_fd, &idx, &fd, + BPF_ANY); + if (err < 0) { + pr_err("Failed to update perf_event fd\n"); + goto out; + } + } + + evsel->cgrp = leader_cgrp; + } + evsel->supported = true; + + if (evsel->cgrp == cgrp) + continue; + + cgrp = evsel->cgrp; + + if (read_cgroup_id(cgrp) < 0) { + pr_err("Failed to get cgroup id\n"); + err = -1; + goto out; + } + + map_fd = bpf_map__fd(skel->maps.cgrp_idx); + err = bpf_map_update_elem(map_fd, &cgrp->id, &i, BPF_ANY); + if (err < 0) { + pr_err("Failed to update cgroup index map\n"); + goto out; + } + + i++; + } + + /* + * bperf uses BPF_PROG_TEST_RUN to get accurate reading. Check + * whether the kernel support it + */ + prog_fd = bpf_program__fd(skel->progs.trigger_read); + err = bperf_trigger_reading(prog_fd, 0); + if (err) { + pr_warning("The kernel does not support test_run for raw_tp BPF programs.\n" + "Therefore, --for-each-cgroup might show inaccurate readings\n"); + err = 0; + } + +out: + return err; +} + +static int bperf_cgrp__load(struct evsel *evsel, + struct target *target __maybe_unused) +{ + static bool bperf_loaded = false; + + evsel->bperf_leader_prog_fd = -1; + evsel->bperf_leader_link_fd = -1; + + if (!bperf_loaded && bperf_load_program(evsel->evlist)) + return -1; + + bperf_loaded = true; + /* just to bypass bpf_counter_skip() */ + evsel->follower_skel = (struct bperf_follower_bpf *)skel; + + return 0; +} + +static int bperf_cgrp__install_pe(struct evsel *evsel __maybe_unused, + int cpu __maybe_unused, int fd __maybe_unused) +{ + /* nothing to do */ + return 0; +} + +/* + * trigger the leader prog on each cpu, so the cgrp_reading map could get + * the latest results. + */ +static int bperf_cgrp__sync_counters(struct evlist *evlist) +{ + int i, cpu; + int nr_cpus = evlist->core.all_cpus->nr; + int prog_fd = bpf_program__fd(skel->progs.trigger_read); + + for (i = 0; i < nr_cpus; i++) { + cpu = evlist->core.all_cpus->map[i]; + bperf_trigger_reading(prog_fd, cpu); + } + + return 0; +} + +static int bperf_cgrp__enable(struct evsel *evsel) +{ + if (evsel->core.idx) + return 0; + + bperf_cgrp__sync_counters(evsel->evlist); + + skel->bss->enabled = 1; + return 0; +} + +static int bperf_cgrp__disable(struct evsel *evsel) +{ + if (evsel->core.idx) + return 0; + + bperf_cgrp__sync_counters(evsel->evlist); + + skel->bss->enabled = 0; + return 0; +} + +static int bperf_cgrp__read(struct evsel *evsel) +{ + struct evlist *evlist = evsel->evlist; + int i, cpu, nr_cpus = evlist->core.all_cpus->nr; + int total_cpus = cpu__max_cpu(); + struct perf_counts_values *counts; + struct bpf_perf_event_value *values; + int reading_map_fd, err = 0; + __u32 idx; + + if (evsel->core.idx) + return 0; + + bperf_cgrp__sync_counters(evsel->evlist); + + values = calloc(total_cpus, sizeof(*values)); + if (values == NULL) + return -ENOMEM; + + reading_map_fd = bpf_map__fd(skel->maps.cgrp_readings); + + evlist__for_each_entry(evlist, evsel) { + idx = evsel->core.idx; + err = bpf_map_lookup_elem(reading_map_fd, &idx, values); + if (err) { + pr_err("bpf map lookup falied: idx=%u, event=%s, cgrp=%s\n", + idx, evsel__name(evsel), evsel->cgrp->name); + goto out; + } + + for (i = 0; i < nr_cpus; i++) { + cpu = evlist->core.all_cpus->map[i]; + + counts = perf_counts(evsel->counts, i, 0); + counts->val = values[cpu].counter; + counts->ena = values[cpu].enabled; + counts->run = values[cpu].running; + } + } + +out: + free(values); + return err; +} + +static int bperf_cgrp__destroy(struct evsel *evsel) +{ + if (evsel->core.idx) + return 0; + + bperf_cgroup_bpf__destroy(skel); + evsel__delete(cgrp_switch); // it'll destroy on_switch progs too + + return 0; +} + +struct bpf_counter_ops bperf_cgrp_ops = { + .load = bperf_cgrp__load, + .enable = bperf_cgrp__enable, + .disable = bperf_cgrp__disable, + .read = bperf_cgrp__read, + .install_pe = bperf_cgrp__install_pe, + .destroy = bperf_cgrp__destroy, +}; diff --git a/tools/perf/util/bpf_skel/bperf_cgroup.bpf.c b/tools/perf/util/bpf_skel/bperf_cgroup.bpf.c new file mode 100644 index 000000000000..292c430768b5 --- /dev/null +++ b/tools/perf/util/bpf_skel/bperf_cgroup.bpf.c @@ -0,0 +1,191 @@ +// SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +// Copyright (c) 2021 Facebook +// Copyright (c) 2021 Google +#include "vmlinux.h" +#include <bpf/bpf_helpers.h> +#include <bpf/bpf_tracing.h> +#include <bpf/bpf_core_read.h> + +#define MAX_LEVELS 10 // max cgroup hierarchy level: arbitrary +#define MAX_EVENTS 32 // max events per cgroup: arbitrary + +// NOTE: many of map and global data will be modified before loading +// from the userspace (perf tool) using the skeleton helpers. + +// single set of global perf events to measure +struct { + __uint(type, BPF_MAP_TYPE_PERF_EVENT_ARRAY); + __uint(key_size, sizeof(__u32)); + __uint(value_size, sizeof(int)); + __uint(max_entries, 1); +} events SEC(".maps"); + +// from cgroup id to event index +struct { + __uint(type, BPF_MAP_TYPE_HASH); + __uint(key_size, sizeof(__u64)); + __uint(value_size, sizeof(__u32)); + __uint(max_entries, 1); +} cgrp_idx SEC(".maps"); + +// per-cpu event snapshots to calculate delta +struct { + __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); + __uint(key_size, sizeof(__u32)); + __uint(value_size, sizeof(struct bpf_perf_event_value)); +} prev_readings SEC(".maps"); + +// aggregated event values for each cgroup (per-cpu) +// will be read from the user-space +struct { + __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY); + __uint(key_size, sizeof(__u32)); + __uint(value_size, sizeof(struct bpf_perf_event_value)); +} cgrp_readings SEC(".maps"); + +const volatile __u32 num_events = 1; +const volatile __u32 num_cpus = 1; + +int enabled = 0; +int use_cgroup_v2 = 0; + +static inline int get_cgroup_v1_idx(__u32 *cgrps, int size) +{ + struct task_struct *p = (void *)bpf_get_current_task(); + struct cgroup *cgrp; + register int i = 0; + __u32 *elem; + int level; + int cnt; + + cgrp = BPF_CORE_READ(p, cgroups, subsys[perf_event_cgrp_id], cgroup); + level = BPF_CORE_READ(cgrp, level); + + for (cnt = 0; i < MAX_LEVELS; i++) { + __u64 cgrp_id; + + if (i > level) + break; + + // convert cgroup-id to a map index + cgrp_id = BPF_CORE_READ(cgrp, ancestor_ids[i]); + elem = bpf_map_lookup_elem(&cgrp_idx, &cgrp_id); + if (!elem) + continue; + + cgrps[cnt++] = *elem; + if (cnt == size) + break; + } + + return cnt; +} + +static inline int get_cgroup_v2_idx(__u32 *cgrps, int size) +{ + register int i = 0; + __u32 *elem; + int cnt; + + for (cnt = 0; i < MAX_LEVELS; i++) { + __u64 cgrp_id = bpf_get_current_ancestor_cgroup_id(i); + + if (cgrp_id == 0) + break; + + // convert cgroup-id to a map index + elem = bpf_map_lookup_elem(&cgrp_idx, &cgrp_id); + if (!elem) + continue; + + cgrps[cnt++] = *elem; + if (cnt == size) + break; + } + + return cnt; +} + +static int bperf_cgroup_count(void) +{ + register __u32 idx = 0; // to have it in a register to pass BPF verifier + register int c = 0; + struct bpf_perf_event_value val, delta, *prev_val, *cgrp_val; + __u32 cpu = bpf_get_smp_processor_id(); + __u32 cgrp_idx[MAX_LEVELS]; + int cgrp_cnt; + __u32 key, cgrp; + long err; + + if (use_cgroup_v2) + cgrp_cnt = get_cgroup_v2_idx(cgrp_idx, MAX_LEVELS); + else + cgrp_cnt = get_cgroup_v1_idx(cgrp_idx, MAX_LEVELS); + + for ( ; idx < MAX_EVENTS; idx++) { + if (idx == num_events) + break; + + // XXX: do not pass idx directly (for verifier) + key = idx; + // this is per-cpu array for diff + prev_val = bpf_map_lookup_elem(&prev_readings, &key); + if (!prev_val) { + val.counter = val.enabled = val.running = 0; + bpf_map_update_elem(&prev_readings, &key, &val, BPF_ANY); + + prev_val = bpf_map_lookup_elem(&prev_readings, &key); + if (!prev_val) + continue; + } + + // read from global perf_event array + key = idx * num_cpus + cpu; + err = bpf_perf_event_read_value(&events, key, &val, sizeof(val)); + if (err) + continue; + + if (enabled) { + delta.counter = val.counter - prev_val->counter; + delta.enabled = val.enabled - prev_val->enabled; + delta.running = val.running - prev_val->running; + + for (c = 0; c < MAX_LEVELS; c++) { + if (c == cgrp_cnt) + break; + + cgrp = cgrp_idx[c]; + + // aggregate the result by cgroup + key = cgrp * num_events + idx; + cgrp_val = bpf_map_lookup_elem(&cgrp_readings, &key); + if (cgrp_val) { + cgrp_val->counter += delta.counter; + cgrp_val->enabled += delta.enabled; + cgrp_val->running += delta.running; + } else { + bpf_map_update_elem(&cgrp_readings, &key, + &delta, BPF_ANY); + } + } + } + + *prev_val = val; + } + return 0; +} + +// This will be attached to cgroup-switches event for each cpu +SEC("perf_events") +int BPF_PROG(on_cgrp_switch) +{ + return bperf_cgroup_count(); +} + +SEC("raw_tp/sched_switch") +int BPF_PROG(trigger_read) +{ + return bperf_cgroup_count(); +} + +char LICENSE[] SEC("license") = "Dual BSD/GPL"; diff --git a/tools/perf/util/cgroup.c b/tools/perf/util/cgroup.c index f24ab4585553..e99b41f9be45 100644 --- a/tools/perf/util/cgroup.c +++ b/tools/perf/util/cgroup.c @@ -9,6 +9,7 @@ #include <linux/zalloc.h> #include <sys/types.h> #include <sys/stat.h> +#include <sys/statfs.h> #include <fcntl.h> #include <stdlib.h> #include <string.h> @@ -17,6 +18,7 @@ #include <regex.h> int nr_cgroups; +bool cgrp_event_expanded; /* used to match cgroup name with patterns */ struct cgroup_name { @@ -45,6 +47,49 @@ static int open_cgroup(const char *name) return fd; } +#ifdef HAVE_FILE_HANDLE +int read_cgroup_id(struct cgroup *cgrp) +{ + char path[PATH_MAX + 1]; + char mnt[PATH_MAX + 1]; + struct { + struct file_handle fh; + uint64_t cgroup_id; + } handle; + int mount_id; + + if (cgroupfs_find_mountpoint(mnt, PATH_MAX + 1, "perf_event")) + return -1; + + scnprintf(path, PATH_MAX, "%s/%s", mnt, cgrp->name); + + handle.fh.handle_bytes = sizeof(handle.cgroup_id); + if (name_to_handle_at(AT_FDCWD, path, &handle.fh, &mount_id, 0) < 0) + return -1; + + cgrp->id = handle.cgroup_id; + return 0; +} +#endif /* HAVE_FILE_HANDLE */ + +#ifndef CGROUP2_SUPER_MAGIC +#define CGROUP2_SUPER_MAGIC 0x63677270 +#endif + +int cgroup_is_v2(const char *subsys) +{ + char mnt[PATH_MAX + 1]; + struct statfs stbuf; + + if (cgroupfs_find_mountpoint(mnt, PATH_MAX + 1, subsys)) + return -1; + + if (statfs(mnt, &stbuf) < 0) + return -1; + + return (stbuf.f_type == CGROUP2_SUPER_MAGIC); +} + static struct cgroup *evlist__find_cgroup(struct evlist *evlist, const char *str) { struct evsel *counter; @@ -414,7 +459,7 @@ int evlist__expand_cgroup(struct evlist *evlist, const char *str, if (evsel__is_group_leader(pos)) leader = evsel; - evsel->leader = leader; + evsel__set_leader(evsel, leader); evlist__add(tmp_list, evsel); } @@ -440,6 +485,7 @@ int evlist__expand_cgroup(struct evlist *evlist, const char *str, } ret = 0; + cgrp_event_expanded = true; out_err: evlist__delete(orig_list); diff --git a/tools/perf/util/cgroup.h b/tools/perf/util/cgroup.h index 162906f3412a..12256b78608c 100644 --- a/tools/perf/util/cgroup.h +++ b/tools/perf/util/cgroup.h @@ -2,6 +2,7 @@ #ifndef __CGROUP_H__ #define __CGROUP_H__ +#include <linux/compiler.h> #include <linux/refcount.h> #include <linux/rbtree.h> #include "util/env.h" @@ -17,6 +18,7 @@ struct cgroup { }; extern int nr_cgroups; /* number of explicit cgroups defined */ +extern bool cgrp_event_expanded; struct cgroup *cgroup__get(struct cgroup *cgroup); void cgroup__put(struct cgroup *cgroup); @@ -38,4 +40,15 @@ struct cgroup *cgroup__find(struct perf_env *env, uint64_t id); void perf_env__purge_cgroups(struct perf_env *env); +#ifdef HAVE_FILE_HANDLE +int read_cgroup_id(struct cgroup *cgrp); +#else +static inline int read_cgroup_id(struct cgroup *cgrp __maybe_unused) +{ + return -1; +} +#endif /* HAVE_FILE_HANDLE */ + +int cgroup_is_v2(const char *subsys); + #endif /* __CGROUP_H__ */ diff --git a/tools/perf/util/cputopo.c b/tools/perf/util/cputopo.c index 1b52402a8923..ec77e2a7b3ca 100644 --- a/tools/perf/util/cputopo.c +++ b/tools/perf/util/cputopo.c @@ -12,6 +12,7 @@ #include "cpumap.h" #include "debug.h" #include "env.h" +#include "pmu-hybrid.h" #define CORE_SIB_FMT \ "%s/devices/system/cpu/cpu%d/topology/core_siblings_list" @@ -351,3 +352,82 @@ void numa_topology__delete(struct numa_topology *tp) free(tp); } + +static int load_hybrid_node(struct hybrid_topology_node *node, + struct perf_pmu *pmu) +{ + const char *sysfs; + char path[PATH_MAX]; + char *buf = NULL, *p; + FILE *fp; + size_t len = 0; + + node->pmu_name = strdup(pmu->name); + if (!node->pmu_name) + return -1; + + sysfs = sysfs__mountpoint(); + if (!sysfs) + goto err; + + snprintf(path, PATH_MAX, CPUS_TEMPLATE_CPU, sysfs, pmu->name); + fp = fopen(path, "r"); + if (!fp) + goto err; + + if (getline(&buf, &len, fp) <= 0) { + fclose(fp); + goto err; + } + + p = strchr(buf, '\n'); + if (p) + *p = '\0'; + + fclose(fp); + node->cpus = buf; + return 0; + +err: + zfree(&node->pmu_name); + free(buf); + return -1; +} + +struct hybrid_topology *hybrid_topology__new(void) +{ + struct perf_pmu *pmu; + struct hybrid_topology *tp = NULL; + u32 nr, i = 0; + + nr = perf_pmu__hybrid_pmu_num(); + if (nr == 0) + return NULL; + + tp = zalloc(sizeof(*tp) + sizeof(tp->nodes[0]) * nr); + if (!tp) + return NULL; + + tp->nr = nr; + perf_pmu__for_each_hybrid_pmu(pmu) { + if (load_hybrid_node(&tp->nodes[i], pmu)) { + hybrid_topology__delete(tp); + return NULL; + } + i++; + } + + return tp; +} + +void hybrid_topology__delete(struct hybrid_topology *tp) +{ + u32 i; + + for (i = 0; i < tp->nr; i++) { + zfree(&tp->nodes[i].pmu_name); + zfree(&tp->nodes[i].cpus); + } + + free(tp); +} diff --git a/tools/perf/util/cputopo.h b/tools/perf/util/cputopo.h index 6201c3790d86..d9af97177068 100644 --- a/tools/perf/util/cputopo.h +++ b/tools/perf/util/cputopo.h @@ -25,10 +25,23 @@ struct numa_topology { struct numa_topology_node nodes[]; }; +struct hybrid_topology_node { + char *pmu_name; + char *cpus; +}; + +struct hybrid_topology { + u32 nr; + struct hybrid_topology_node nodes[]; +}; + struct cpu_topology *cpu_topology__new(void); void cpu_topology__delete(struct cpu_topology *tp); struct numa_topology *numa_topology__new(void); void numa_topology__delete(struct numa_topology *tp); +struct hybrid_topology *hybrid_topology__new(void); +void hybrid_topology__delete(struct hybrid_topology *tp); + #endif /* __PERF_CPUTOPO_H */ diff --git a/tools/perf/util/cs-etm-decoder/cs-etm-decoder.c b/tools/perf/util/cs-etm-decoder/cs-etm-decoder.c index 059bcec3f651..3e1a05bc82cc 100644 --- a/tools/perf/util/cs-etm-decoder/cs-etm-decoder.c +++ b/tools/perf/util/cs-etm-decoder/cs-etm-decoder.c @@ -6,6 +6,7 @@ * Author: Mathieu Poirier <mathieu.poirier@linaro.org> */ +#include <asm/bug.h> #include <linux/coresight-pmu.h> #include <linux/err.h> #include <linux/list.h> @@ -17,6 +18,7 @@ #include "cs-etm.h" #include "cs-etm-decoder.h" +#include "debug.h" #include "intlist.h" /* use raw logging */ @@ -276,13 +278,13 @@ cs_etm_decoder__do_soft_timestamp(struct cs_etm_queue *etmq, const uint8_t trace_chan_id) { /* No timestamp packet has been received, nothing to do */ - if (!packet_queue->timestamp) + if (!packet_queue->cs_timestamp) return OCSD_RESP_CONT; - packet_queue->timestamp = packet_queue->next_timestamp; + packet_queue->cs_timestamp = packet_queue->next_cs_timestamp; /* Estimate the timestamp for the next range packet */ - packet_queue->next_timestamp += packet_queue->instr_count; + packet_queue->next_cs_timestamp += packet_queue->instr_count; packet_queue->instr_count = 0; /* Tell the front end which traceid_queue needs attention */ @@ -294,7 +296,8 @@ cs_etm_decoder__do_soft_timestamp(struct cs_etm_queue *etmq, static ocsd_datapath_resp_t cs_etm_decoder__do_hard_timestamp(struct cs_etm_queue *etmq, const ocsd_generic_trace_elem *elem, - const uint8_t trace_chan_id) + const uint8_t trace_chan_id, + const ocsd_trc_index_t indx) { struct cs_etm_packet_queue *packet_queue; @@ -308,20 +311,39 @@ cs_etm_decoder__do_hard_timestamp(struct cs_etm_queue *etmq, * Function do_soft_timestamp() will report the value to the front end, * hence asking the decoder to keep decoding rather than stopping. */ - if (packet_queue->timestamp) { - packet_queue->next_timestamp = elem->timestamp; + if (packet_queue->cs_timestamp) { + packet_queue->next_cs_timestamp = elem->timestamp; return OCSD_RESP_CONT; } - /* - * This is the first timestamp we've seen since the beginning of traces - * or a discontinuity. Since timestamps packets are generated *after* - * range packets have been generated, we need to estimate the time at - * which instructions started by subtracting the number of instructions - * executed to the timestamp. - */ - packet_queue->timestamp = elem->timestamp - packet_queue->instr_count; - packet_queue->next_timestamp = elem->timestamp; + + if (!elem->timestamp) { + /* + * Zero timestamps can be seen due to misconfiguration or hardware bugs. + * Warn once, and don't try to subtract instr_count as it would result in an + * underflow. + */ + packet_queue->cs_timestamp = 0; + WARN_ONCE(true, "Zero Coresight timestamp found at Idx:%" OCSD_TRC_IDX_STR + ". Decoding may be improved with --itrace=Z...\n", indx); + } else if (packet_queue->instr_count > elem->timestamp) { + /* + * Sanity check that the elem->timestamp - packet_queue->instr_count would not + * result in an underflow. Warn and clamp at 0 if it would. + */ + packet_queue->cs_timestamp = 0; + pr_err("Timestamp calculation underflow at Idx:%" OCSD_TRC_IDX_STR "\n", indx); + } else { + /* + * This is the first timestamp we've seen since the beginning of traces + * or a discontinuity. Since timestamps packets are generated *after* + * range packets have been generated, we need to estimate the time at + * which instructions started by subtracting the number of instructions + * executed to the timestamp. + */ + packet_queue->cs_timestamp = elem->timestamp - packet_queue->instr_count; + } + packet_queue->next_cs_timestamp = elem->timestamp; packet_queue->instr_count = 0; /* Tell the front end which traceid_queue needs attention */ @@ -334,8 +356,8 @@ cs_etm_decoder__do_hard_timestamp(struct cs_etm_queue *etmq, static void cs_etm_decoder__reset_timestamp(struct cs_etm_packet_queue *packet_queue) { - packet_queue->timestamp = 0; - packet_queue->next_timestamp = 0; + packet_queue->cs_timestamp = 0; + packet_queue->next_cs_timestamp = 0; packet_queue->instr_count = 0; } @@ -542,7 +564,7 @@ cs_etm_decoder__set_tid(struct cs_etm_queue *etmq, static ocsd_datapath_resp_t cs_etm_decoder__gen_trace_elem_printer( const void *context, - const ocsd_trc_index_t indx __maybe_unused, + const ocsd_trc_index_t indx, const u8 trace_chan_id __maybe_unused, const ocsd_generic_trace_elem *elem) { @@ -579,7 +601,8 @@ static ocsd_datapath_resp_t cs_etm_decoder__gen_trace_elem_printer( break; case OCSD_GEN_TRC_ELEM_TIMESTAMP: resp = cs_etm_decoder__do_hard_timestamp(etmq, elem, - trace_chan_id); + trace_chan_id, + indx); break; case OCSD_GEN_TRC_ELEM_PE_CONTEXT: resp = cs_etm_decoder__set_tid(etmq, packet_queue, diff --git a/tools/perf/util/cs-etm.c b/tools/perf/util/cs-etm.c index 7e63e7dedc33..bc1f64873c8f 100644 --- a/tools/perf/util/cs-etm.c +++ b/tools/perf/util/cs-etm.c @@ -38,8 +38,6 @@ #include <tools/libc_compat.h> #include "util/synthetic-events.h" -#define MAX_TIMESTAMP (~0ULL) - struct cs_etm_auxtrace { struct auxtrace auxtrace; struct auxtrace_queues queues; @@ -56,6 +54,7 @@ struct cs_etm_auxtrace { u8 sample_instructions; int num_cpu; + u64 latest_kernel_timestamp; u32 auxtrace_type; u64 branches_sample_type; u64 branches_id; @@ -86,7 +85,7 @@ struct cs_etm_queue { struct cs_etm_decoder *decoder; struct auxtrace_buffer *buffer; unsigned int queue_nr; - u8 pending_timestamp; + u8 pending_timestamp_chan_id; u64 offset; const unsigned char *buf; size_t buf_len, buf_used; @@ -208,7 +207,7 @@ void cs_etm__etmq_set_traceid_queue_timestamp(struct cs_etm_queue *etmq, * be more than one channel per cs_etm_queue, we need to specify * what traceID queue needs servicing. */ - etmq->pending_timestamp = trace_chan_id; + etmq->pending_timestamp_chan_id = trace_chan_id; } static u64 cs_etm__etmq_get_timestamp(struct cs_etm_queue *etmq, @@ -216,22 +215,22 @@ static u64 cs_etm__etmq_get_timestamp(struct cs_etm_queue *etmq, { struct cs_etm_packet_queue *packet_queue; - if (!etmq->pending_timestamp) + if (!etmq->pending_timestamp_chan_id) return 0; if (trace_chan_id) - *trace_chan_id = etmq->pending_timestamp; + *trace_chan_id = etmq->pending_timestamp_chan_id; packet_queue = cs_etm__etmq_get_packet_queue(etmq, - etmq->pending_timestamp); + etmq->pending_timestamp_chan_id); if (!packet_queue) return 0; /* Acknowledge pending status */ - etmq->pending_timestamp = 0; + etmq->pending_timestamp_chan_id = 0; /* See function cs_etm_decoder__do_{hard|soft}_timestamp() */ - return packet_queue->timestamp; + return packet_queue->cs_timestamp; } static void cs_etm__clear_packet_queue(struct cs_etm_packet_queue *queue) @@ -814,7 +813,7 @@ static int cs_etm__setup_queue(struct cs_etm_auxtrace *etm, int ret = 0; unsigned int cs_queue_nr; u8 trace_chan_id; - u64 timestamp; + u64 cs_timestamp; struct cs_etm_queue *etmq = queue->priv; if (list_empty(&queue->head) || etmq) @@ -854,7 +853,7 @@ static int cs_etm__setup_queue(struct cs_etm_auxtrace *etm, /* * Run decoder on the trace block. The decoder will stop when - * encountering a timestamp, a full packet queue or the end of + * encountering a CS timestamp, a full packet queue or the end of * trace for that block. */ ret = cs_etm__decode_data_block(etmq); @@ -865,10 +864,10 @@ static int cs_etm__setup_queue(struct cs_etm_auxtrace *etm, * Function cs_etm_decoder__do_{hard|soft}_timestamp() does all * the timestamp calculation for us. */ - timestamp = cs_etm__etmq_get_timestamp(etmq, &trace_chan_id); + cs_timestamp = cs_etm__etmq_get_timestamp(etmq, &trace_chan_id); /* We found a timestamp, no need to continue. */ - if (timestamp) + if (cs_timestamp) break; /* @@ -892,7 +891,7 @@ static int cs_etm__setup_queue(struct cs_etm_auxtrace *etm, * queue and will be processed in cs_etm__process_queues(). */ cs_queue_nr = TO_CS_QUEUE_NR(queue_nr, trace_chan_id); - ret = auxtrace_heap__add(&etm->heap, cs_queue_nr, timestamp); + ret = auxtrace_heap__add(&etm->heap, cs_queue_nr, cs_timestamp); out: return ret; } @@ -1194,6 +1193,8 @@ static int cs_etm__synth_instruction_sample(struct cs_etm_queue *etmq, event->sample.header.misc = cs_etm__cpu_mode(etmq, addr); event->sample.header.size = sizeof(struct perf_event_header); + if (!etm->timeless_decoding) + sample.time = etm->latest_kernel_timestamp; sample.ip = addr; sample.pid = tidq->pid; sample.tid = tidq->tid; @@ -1250,6 +1251,8 @@ static int cs_etm__synth_branch_sample(struct cs_etm_queue *etmq, event->sample.header.misc = cs_etm__cpu_mode(etmq, ip); event->sample.header.size = sizeof(struct perf_event_header); + if (!etm->timeless_decoding) + sample.time = etm->latest_kernel_timestamp; sample.ip = ip; sample.pid = tidq->pid; sample.tid = tidq->tid; @@ -2221,7 +2224,7 @@ static int cs_etm__process_queues(struct cs_etm_auxtrace *etm) int ret = 0; unsigned int cs_queue_nr, queue_nr; u8 trace_chan_id; - u64 timestamp; + u64 cs_timestamp; struct auxtrace_queue *queue; struct cs_etm_queue *etmq; struct cs_etm_traceid_queue *tidq; @@ -2283,9 +2286,9 @@ refetch: if (ret) goto out; - timestamp = cs_etm__etmq_get_timestamp(etmq, &trace_chan_id); + cs_timestamp = cs_etm__etmq_get_timestamp(etmq, &trace_chan_id); - if (!timestamp) { + if (!cs_timestamp) { /* * Function cs_etm__decode_data_block() returns when * there is no more traces to decode in the current @@ -2308,7 +2311,7 @@ refetch: * this queue/traceID. */ cs_queue_nr = TO_CS_QUEUE_NR(queue_nr, trace_chan_id); - ret = auxtrace_heap__add(&etm->heap, cs_queue_nr, timestamp); + ret = auxtrace_heap__add(&etm->heap, cs_queue_nr, cs_timestamp); } out: @@ -2380,7 +2383,7 @@ static int cs_etm__process_event(struct perf_session *session, struct perf_tool *tool) { int err = 0; - u64 timestamp; + u64 sample_kernel_timestamp; struct cs_etm_auxtrace *etm = container_of(session->auxtrace, struct cs_etm_auxtrace, auxtrace); @@ -2394,16 +2397,21 @@ static int cs_etm__process_event(struct perf_session *session, } if (sample->time && (sample->time != (u64) -1)) - timestamp = sample->time; + sample_kernel_timestamp = sample->time; else - timestamp = 0; + sample_kernel_timestamp = 0; - if (timestamp || etm->timeless_decoding) { + if (sample_kernel_timestamp || etm->timeless_decoding) { err = cs_etm__update_queues(etm); if (err) return err; } + /* + * Don't wait for cs_etm__flush_events() in per-thread/timeless mode to start the decode. We + * need the tid of the PERF_RECORD_EXIT event to assign to the synthesised samples because + * ETM_OPT_CTXTID is not enabled. + */ if (etm->timeless_decoding && event->header.type == PERF_RECORD_EXIT) return cs_etm__process_timeless_queues(etm, @@ -2414,13 +2422,34 @@ static int cs_etm__process_event(struct perf_session *session, else if (event->header.type == PERF_RECORD_SWITCH_CPU_WIDE) return cs_etm__process_switch_cpu_wide(etm, event); - if (!etm->timeless_decoding && - event->header.type == PERF_RECORD_AUX) - return cs_etm__process_queues(etm); + if (!etm->timeless_decoding && event->header.type == PERF_RECORD_AUX) { + /* + * Record the latest kernel timestamp available in the header + * for samples so that synthesised samples occur from this point + * onwards. + */ + etm->latest_kernel_timestamp = sample_kernel_timestamp; + } return 0; } +static void dump_queued_data(struct cs_etm_auxtrace *etm, + struct perf_record_auxtrace *event) +{ + struct auxtrace_buffer *buf; + unsigned int i; + /* + * Find all buffers with same reference in the queues and dump them. + * This is because the queues can contain multiple entries of the same + * buffer that were split on aux records. + */ + for (i = 0; i < etm->queues.nr_queues; ++i) + list_for_each_entry(buf, &etm->queues.queue_array[i].head, list) + if (buf->reference == event->reference) + cs_etm__dump_event(etm, buf); +} + static int cs_etm__process_auxtrace_event(struct perf_session *session, union perf_event *event, struct perf_tool *tool __maybe_unused) @@ -2453,7 +2482,8 @@ static int cs_etm__process_auxtrace_event(struct perf_session *session, cs_etm__dump_event(etm, buffer); auxtrace_buffer__put_data(buffer); } - } + } else if (dump_trace) + dump_queued_data(etm, &event->auxtrace); return 0; } @@ -2464,6 +2494,10 @@ static bool cs_etm__is_timeless_decoding(struct cs_etm_auxtrace *etm) struct evlist *evlist = etm->session->evlist; bool timeless_decoding = true; + /* Override timeless mode with user input from --itrace=Z */ + if (etm->synth_opts.timeless_decoding) + return true; + /* * Circle through the list of event and complain if we find one * with the time bit set. @@ -2666,6 +2700,172 @@ static u64 *cs_etm__create_meta_blk(u64 *buff_in, int *buff_in_offset, return metadata; } +/** + * Puts a fragment of an auxtrace buffer into the auxtrace queues based + * on the bounds of aux_event, if it matches with the buffer that's at + * file_offset. + * + * Normally, whole auxtrace buffers would be added to the queue. But we + * want to reset the decoder for every PERF_RECORD_AUX event, and the decoder + * is reset across each buffer, so splitting the buffers up in advance has + * the same effect. + */ +static int cs_etm__queue_aux_fragment(struct perf_session *session, off_t file_offset, size_t sz, + struct perf_record_aux *aux_event, struct perf_sample *sample) +{ + int err; + char buf[PERF_SAMPLE_MAX_SIZE]; + union perf_event *auxtrace_event_union; + struct perf_record_auxtrace *auxtrace_event; + union perf_event auxtrace_fragment; + __u64 aux_offset, aux_size; + + struct cs_etm_auxtrace *etm = container_of(session->auxtrace, + struct cs_etm_auxtrace, + auxtrace); + + /* + * There should be a PERF_RECORD_AUXTRACE event at the file_offset that we got + * from looping through the auxtrace index. + */ + err = perf_session__peek_event(session, file_offset, buf, + PERF_SAMPLE_MAX_SIZE, &auxtrace_event_union, NULL); + if (err) + return err; + auxtrace_event = &auxtrace_event_union->auxtrace; + if (auxtrace_event->header.type != PERF_RECORD_AUXTRACE) + return -EINVAL; + + if (auxtrace_event->header.size < sizeof(struct perf_record_auxtrace) || + auxtrace_event->header.size != sz) { + return -EINVAL; + } + + /* + * In per-thread mode, CPU is set to -1, but TID will be set instead. See + * auxtrace_mmap_params__set_idx(). Return 'not found' if neither CPU nor TID match. + */ + if ((auxtrace_event->cpu == (__u32) -1 && auxtrace_event->tid != sample->tid) || + auxtrace_event->cpu != sample->cpu) + return 1; + + if (aux_event->flags & PERF_AUX_FLAG_OVERWRITE) { + /* + * Clamp size in snapshot mode. The buffer size is clamped in + * __auxtrace_mmap__read() for snapshots, so the aux record size doesn't reflect + * the buffer size. + */ + aux_size = min(aux_event->aux_size, auxtrace_event->size); + + /* + * In this mode, the head also points to the end of the buffer so aux_offset + * needs to have the size subtracted so it points to the beginning as in normal mode + */ + aux_offset = aux_event->aux_offset - aux_size; + } else { + aux_size = aux_event->aux_size; + aux_offset = aux_event->aux_offset; + } + + if (aux_offset >= auxtrace_event->offset && + aux_offset + aux_size <= auxtrace_event->offset + auxtrace_event->size) { + /* + * If this AUX event was inside this buffer somewhere, create a new auxtrace event + * based on the sizes of the aux event, and queue that fragment. + */ + auxtrace_fragment.auxtrace = *auxtrace_event; + auxtrace_fragment.auxtrace.size = aux_size; + auxtrace_fragment.auxtrace.offset = aux_offset; + file_offset += aux_offset - auxtrace_event->offset + auxtrace_event->header.size; + + pr_debug3("CS ETM: Queue buffer size: %#"PRI_lx64" offset: %#"PRI_lx64 + " tid: %d cpu: %d\n", aux_size, aux_offset, sample->tid, sample->cpu); + return auxtrace_queues__add_event(&etm->queues, session, &auxtrace_fragment, + file_offset, NULL); + } + + /* Wasn't inside this buffer, but there were no parse errors. 1 == 'not found' */ + return 1; +} + +static int cs_etm__queue_aux_records_cb(struct perf_session *session, union perf_event *event, + u64 offset __maybe_unused, void *data __maybe_unused) +{ + struct perf_sample sample; + int ret; + struct auxtrace_index_entry *ent; + struct auxtrace_index *auxtrace_index; + struct evsel *evsel; + size_t i; + + /* Don't care about any other events, we're only queuing buffers for AUX events */ + if (event->header.type != PERF_RECORD_AUX) + return 0; + + if (event->header.size < sizeof(struct perf_record_aux)) + return -EINVAL; + + /* Truncated Aux records can have 0 size and shouldn't result in anything being queued. */ + if (!event->aux.aux_size) + return 0; + + /* + * Parse the sample, we need the sample_id_all data that comes after the event so that the + * CPU or PID can be matched to an AUXTRACE buffer's CPU or PID. + */ + evsel = evlist__event2evsel(session->evlist, event); + if (!evsel) + return -EINVAL; + ret = evsel__parse_sample(evsel, event, &sample); + if (ret) + return ret; + + /* + * Loop through the auxtrace index to find the buffer that matches up with this aux event. + */ + list_for_each_entry(auxtrace_index, &session->auxtrace_index, list) { + for (i = 0; i < auxtrace_index->nr; i++) { + ent = &auxtrace_index->entries[i]; + ret = cs_etm__queue_aux_fragment(session, ent->file_offset, + ent->sz, &event->aux, &sample); + /* + * Stop search on error or successful values. Continue search on + * 1 ('not found') + */ + if (ret != 1) + return ret; + } + } + + /* + * Couldn't find the buffer corresponding to this aux record, something went wrong. Warn but + * don't exit with an error because it will still be possible to decode other aux records. + */ + pr_err("CS ETM: Couldn't find auxtrace buffer for aux_offset: %#"PRI_lx64 + " tid: %d cpu: %d\n", event->aux.aux_offset, sample.tid, sample.cpu); + return 0; +} + +static int cs_etm__queue_aux_records(struct perf_session *session) +{ + struct auxtrace_index *index = list_first_entry_or_null(&session->auxtrace_index, + struct auxtrace_index, list); + if (index && index->nr > 0) + return perf_session__peek_events(session, session->header.data_offset, + session->header.data_size, + cs_etm__queue_aux_records_cb, NULL); + + /* + * We would get here if there are no entries in the index (either no auxtrace + * buffers or no index at all). Fail silently as there is the possibility of + * queueing them in cs_etm__process_auxtrace_event() if etm->data_queued is still + * false. + * + * In that scenario, buffers will not be split by AUX records. + */ + return 0; +} + int cs_etm__process_auxtrace_info(union perf_event *event, struct perf_session *session) { @@ -2810,6 +3010,14 @@ int cs_etm__process_auxtrace_info(union perf_event *event, if (err) goto err_free_etm; + if (session->itrace_synth_opts->set) { + etm->synth_opts = *session->itrace_synth_opts; + } else { + itrace_synth_opts__set_default(&etm->synth_opts, + session->itrace_synth_opts->default_no_sample); + etm->synth_opts.callchain = false; + } + etm->session = session; etm->machine = &session->machines.host; @@ -2851,22 +3059,13 @@ int cs_etm__process_auxtrace_info(union perf_event *event, if (dump_trace) { cs_etm__print_auxtrace_info(auxtrace_info->priv, num_cpu); - return 0; - } - - if (session->itrace_synth_opts->set) { - etm->synth_opts = *session->itrace_synth_opts; - } else { - itrace_synth_opts__set_default(&etm->synth_opts, - session->itrace_synth_opts->default_no_sample); - etm->synth_opts.callchain = false; } err = cs_etm__synth_events(etm, session); if (err) goto err_delete_thread; - err = auxtrace_queues__process_index(&etm->queues, session); + err = cs_etm__queue_aux_records(session); if (err) goto err_delete_thread; diff --git a/tools/perf/util/cs-etm.h b/tools/perf/util/cs-etm.h index 36428918411e..d65c7b19407d 100644 --- a/tools/perf/util/cs-etm.h +++ b/tools/perf/util/cs-etm.h @@ -171,8 +171,8 @@ struct cs_etm_packet_queue { u32 head; u32 tail; u32 instr_count; - u64 timestamp; - u64 next_timestamp; + u64 cs_timestamp; + u64 next_cs_timestamp; struct cs_etm_packet packet_buffer[CS_ETM_PACKET_MAX_BUFFER]; }; diff --git a/tools/perf/util/data.c b/tools/perf/util/data.c index 8fca4779ae6a..f5d260b1df4d 100644 --- a/tools/perf/util/data.c +++ b/tools/perf/util/data.c @@ -20,7 +20,7 @@ static void close_dir(struct perf_data_file *files, int nr) { - while (--nr >= 1) { + while (--nr >= 0) { close(files[nr].fd); zfree(&files[nr].path); } @@ -240,11 +240,12 @@ static bool is_dir(struct perf_data *data) static int open_file_read(struct perf_data *data) { + int flags = data->in_place_update ? O_RDWR : O_RDONLY; struct stat st; int fd; char sbuf[STRERR_BUFSIZE]; - fd = open(data->file.path, O_RDONLY); + fd = open(data->file.path, flags); if (fd < 0) { int err = errno; diff --git a/tools/perf/util/data.h b/tools/perf/util/data.h index 62a3e66fbee8..c9de82af5584 100644 --- a/tools/perf/util/data.h +++ b/tools/perf/util/data.h @@ -31,6 +31,7 @@ struct perf_data { bool is_dir; bool force; bool use_stdio; + bool in_place_update; enum perf_data_mode mode; struct { diff --git a/tools/perf/util/db-export.c b/tools/perf/util/db-export.c index 5cd189172525..e0d4f08839fb 100644 --- a/tools/perf/util/db-export.c +++ b/tools/perf/util/db-export.c @@ -343,7 +343,7 @@ static int db_export__threads(struct db_export *dbe, struct thread *thread, int db_export__sample(struct db_export *dbe, union perf_event *event, struct perf_sample *sample, struct evsel *evsel, - struct addr_location *al) + struct addr_location *al, struct addr_location *addr_al) { struct thread *thread = al->thread; struct export_sample es = { @@ -389,18 +389,14 @@ int db_export__sample(struct db_export *dbe, union perf_event *event, } } - if ((evsel->core.attr.sample_type & PERF_SAMPLE_ADDR) && - sample_addr_correlates_sym(&evsel->core.attr)) { - struct addr_location addr_al; - - thread__resolve(thread, &addr_al, sample); - err = db_ids_from_al(dbe, &addr_al, &es.addr_dso_db_id, + if (addr_al) { + err = db_ids_from_al(dbe, addr_al, &es.addr_dso_db_id, &es.addr_sym_db_id, &es.addr_offset); if (err) goto out_put; if (dbe->crp) { err = thread_stack__process(thread, comm, sample, al, - &addr_al, es.db_id, + addr_al, es.db_id, dbe->crp); if (err) goto out_put; diff --git a/tools/perf/util/db-export.h b/tools/perf/util/db-export.h index 9c3d38f5a40d..23983cb35706 100644 --- a/tools/perf/util/db-export.h +++ b/tools/perf/util/db-export.h @@ -97,7 +97,7 @@ int db_export__branch_type(struct db_export *dbe, u32 branch_type, const char *name); int db_export__sample(struct db_export *dbe, union perf_event *event, struct perf_sample *sample, struct evsel *evsel, - struct addr_location *al); + struct addr_location *al, struct addr_location *addr_al); int db_export__branch_types(struct db_export *dbe); diff --git a/tools/perf/util/dlfilter.c b/tools/perf/util/dlfilter.c new file mode 100644 index 000000000000..ca33fbc5efde --- /dev/null +++ b/tools/perf/util/dlfilter.c @@ -0,0 +1,615 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * dlfilter.c: Interface to perf script --dlfilter shared object + * Copyright (c) 2021, Intel Corporation. + */ +#include <dlfcn.h> +#include <stdlib.h> +#include <string.h> +#include <dirent.h> +#include <subcmd/exec-cmd.h> +#include <linux/zalloc.h> +#include <linux/build_bug.h> + +#include "debug.h" +#include "event.h" +#include "evsel.h" +#include "dso.h" +#include "map.h" +#include "thread.h" +#include "trace-event.h" +#include "symbol.h" +#include "srcline.h" +#include "dlfilter.h" +#include "perf_dlfilter.h" + +static void al_to_d_al(struct addr_location *al, struct perf_dlfilter_al *d_al) +{ + struct symbol *sym = al->sym; + + d_al->size = sizeof(*d_al); + if (al->map) { + struct dso *dso = al->map->dso; + + if (symbol_conf.show_kernel_path && dso->long_name) + d_al->dso = dso->long_name; + else + d_al->dso = dso->name; + d_al->is_64_bit = dso->is_64_bit; + d_al->buildid_size = dso->bid.size; + d_al->buildid = dso->bid.data; + } else { + d_al->dso = NULL; + d_al->is_64_bit = 0; + d_al->buildid_size = 0; + d_al->buildid = NULL; + } + if (sym) { + d_al->sym = sym->name; + d_al->sym_start = sym->start; + d_al->sym_end = sym->end; + if (al->addr < sym->end) + d_al->symoff = al->addr - sym->start; + else + d_al->symoff = al->addr - al->map->start - sym->start; + d_al->sym_binding = sym->binding; + } else { + d_al->sym = NULL; + d_al->sym_start = 0; + d_al->sym_end = 0; + d_al->symoff = 0; + d_al->sym_binding = 0; + } + d_al->addr = al->addr; + d_al->comm = NULL; + d_al->filtered = 0; +} + +static struct addr_location *get_al(struct dlfilter *d) +{ + struct addr_location *al = d->al; + + if (!al->thread && machine__resolve(d->machine, al, d->sample) < 0) + return NULL; + return al; +} + +static struct thread *get_thread(struct dlfilter *d) +{ + struct addr_location *al = get_al(d); + + return al ? al->thread : NULL; +} + +static const struct perf_dlfilter_al *dlfilter__resolve_ip(void *ctx) +{ + struct dlfilter *d = (struct dlfilter *)ctx; + struct perf_dlfilter_al *d_al = d->d_ip_al; + struct addr_location *al; + + if (!d->ctx_valid) + return NULL; + + /* 'size' is also used to indicate already initialized */ + if (d_al->size) + return d_al; + + al = get_al(d); + if (!al) + return NULL; + + al_to_d_al(al, d_al); + + d_al->is_kernel_ip = machine__kernel_ip(d->machine, d->sample->ip); + d_al->comm = al->thread ? thread__comm_str(al->thread) : ":-1"; + d_al->filtered = al->filtered; + + return d_al; +} + +static const struct perf_dlfilter_al *dlfilter__resolve_addr(void *ctx) +{ + struct dlfilter *d = (struct dlfilter *)ctx; + struct perf_dlfilter_al *d_addr_al = d->d_addr_al; + struct addr_location *addr_al = d->addr_al; + + if (!d->ctx_valid || !d->d_sample->addr_correlates_sym) + return NULL; + + /* 'size' is also used to indicate already initialized */ + if (d_addr_al->size) + return d_addr_al; + + if (!addr_al->thread) { + struct thread *thread = get_thread(d); + + if (!thread) + return NULL; + thread__resolve(thread, addr_al, d->sample); + } + + al_to_d_al(addr_al, d_addr_al); + + d_addr_al->is_kernel_ip = machine__kernel_ip(d->machine, d->sample->addr); + + return d_addr_al; +} + +static char **dlfilter__args(void *ctx, int *dlargc) +{ + struct dlfilter *d = (struct dlfilter *)ctx; + + if (dlargc) + *dlargc = 0; + else + return NULL; + + if (!d->ctx_valid && !d->in_start && !d->in_stop) + return NULL; + + *dlargc = d->dlargc; + return d->dlargv; +} + +static __s32 dlfilter__resolve_address(void *ctx, __u64 address, struct perf_dlfilter_al *d_al_p) +{ + struct dlfilter *d = (struct dlfilter *)ctx; + struct perf_dlfilter_al d_al; + struct addr_location al; + struct thread *thread; + __u32 sz; + + if (!d->ctx_valid || !d_al_p) + return -1; + + thread = get_thread(d); + if (!thread) + return -1; + + thread__find_symbol_fb(thread, d->sample->cpumode, address, &al); + + al_to_d_al(&al, &d_al); + + d_al.is_kernel_ip = machine__kernel_ip(d->machine, address); + + sz = d_al_p->size; + memcpy(d_al_p, &d_al, min((size_t)sz, sizeof(d_al))); + d_al_p->size = sz; + + return 0; +} + +static const __u8 *dlfilter__insn(void *ctx, __u32 *len) +{ + struct dlfilter *d = (struct dlfilter *)ctx; + + if (!len) + return NULL; + + *len = 0; + + if (!d->ctx_valid) + return NULL; + + if (d->sample->ip && !d->sample->insn_len) { + struct addr_location *al = d->al; + + if (!al->thread && machine__resolve(d->machine, al, d->sample) < 0) + return NULL; + + if (al->thread->maps && al->thread->maps->machine) + script_fetch_insn(d->sample, al->thread, al->thread->maps->machine); + } + + if (!d->sample->insn_len) + return NULL; + + *len = d->sample->insn_len; + + return (__u8 *)d->sample->insn; +} + +static const char *dlfilter__srcline(void *ctx, __u32 *line_no) +{ + struct dlfilter *d = (struct dlfilter *)ctx; + struct addr_location *al; + unsigned int line = 0; + char *srcfile = NULL; + struct map *map; + u64 addr; + + if (!d->ctx_valid || !line_no) + return NULL; + + al = get_al(d); + if (!al) + return NULL; + + map = al->map; + addr = al->addr; + + if (map && map->dso) + srcfile = get_srcline_split(map->dso, map__rip_2objdump(map, addr), &line); + + *line_no = line; + return srcfile; +} + +static struct perf_event_attr *dlfilter__attr(void *ctx) +{ + struct dlfilter *d = (struct dlfilter *)ctx; + + if (!d->ctx_valid) + return NULL; + + return &d->evsel->core.attr; +} + +static __s32 dlfilter__object_code(void *ctx, __u64 ip, void *buf, __u32 len) +{ + struct dlfilter *d = (struct dlfilter *)ctx; + struct addr_location *al; + struct addr_location a; + struct map *map; + u64 offset; + + if (!d->ctx_valid) + return -1; + + al = get_al(d); + if (!al) + return -1; + + map = al->map; + + if (map && ip >= map->start && ip < map->end && + machine__kernel_ip(d->machine, ip) == machine__kernel_ip(d->machine, d->sample->ip)) + goto have_map; + + thread__find_map_fb(al->thread, d->sample->cpumode, ip, &a); + if (!a.map) + return -1; + + map = a.map; +have_map: + offset = map->map_ip(map, ip); + if (ip + len >= map->end) + len = map->end - ip; + return dso__data_read_offset(map->dso, d->machine, offset, buf, len); +} + +static const struct perf_dlfilter_fns perf_dlfilter_fns = { + .resolve_ip = dlfilter__resolve_ip, + .resolve_addr = dlfilter__resolve_addr, + .args = dlfilter__args, + .resolve_address = dlfilter__resolve_address, + .insn = dlfilter__insn, + .srcline = dlfilter__srcline, + .attr = dlfilter__attr, + .object_code = dlfilter__object_code, +}; + +static char *find_dlfilter(const char *file) +{ + char path[PATH_MAX]; + char *exec_path; + + if (strchr(file, '/')) + goto out; + + if (!access(file, R_OK)) { + /* + * Prepend "./" so that dlopen will find the file in the + * current directory. + */ + snprintf(path, sizeof(path), "./%s", file); + file = path; + goto out; + } + + exec_path = get_argv_exec_path(); + if (!exec_path) + goto out; + snprintf(path, sizeof(path), "%s/dlfilters/%s", exec_path, file); + free(exec_path); + if (!access(path, R_OK)) + file = path; +out: + return strdup(file); +} + +#define CHECK_FLAG(x) BUILD_BUG_ON((u64)PERF_DLFILTER_FLAG_ ## x != (u64)PERF_IP_FLAG_ ## x) + +static int dlfilter__init(struct dlfilter *d, const char *file, int dlargc, char **dlargv) +{ + CHECK_FLAG(BRANCH); + CHECK_FLAG(CALL); + CHECK_FLAG(RETURN); + CHECK_FLAG(CONDITIONAL); + CHECK_FLAG(SYSCALLRET); + CHECK_FLAG(ASYNC); + CHECK_FLAG(INTERRUPT); + CHECK_FLAG(TX_ABORT); + CHECK_FLAG(TRACE_BEGIN); + CHECK_FLAG(TRACE_END); + CHECK_FLAG(IN_TX); + CHECK_FLAG(VMENTRY); + CHECK_FLAG(VMEXIT); + + memset(d, 0, sizeof(*d)); + d->file = find_dlfilter(file); + if (!d->file) + return -1; + d->dlargc = dlargc; + d->dlargv = dlargv; + return 0; +} + +static void dlfilter__exit(struct dlfilter *d) +{ + zfree(&d->file); +} + +static int dlfilter__open(struct dlfilter *d) +{ + d->handle = dlopen(d->file, RTLD_NOW); + if (!d->handle) { + pr_err("dlopen failed for: '%s'\n", d->file); + return -1; + } + d->start = dlsym(d->handle, "start"); + d->filter_event = dlsym(d->handle, "filter_event"); + d->filter_event_early = dlsym(d->handle, "filter_event_early"); + d->stop = dlsym(d->handle, "stop"); + d->fns = dlsym(d->handle, "perf_dlfilter_fns"); + if (d->fns) + memcpy(d->fns, &perf_dlfilter_fns, sizeof(struct perf_dlfilter_fns)); + return 0; +} + +static int dlfilter__close(struct dlfilter *d) +{ + return dlclose(d->handle); +} + +struct dlfilter *dlfilter__new(const char *file, int dlargc, char **dlargv) +{ + struct dlfilter *d = malloc(sizeof(*d)); + + if (!d) + return NULL; + + if (dlfilter__init(d, file, dlargc, dlargv)) + goto err_free; + + if (dlfilter__open(d)) + goto err_exit; + + return d; + +err_exit: + dlfilter__exit(d); +err_free: + free(d); + return NULL; +} + +static void dlfilter__free(struct dlfilter *d) +{ + if (d) { + dlfilter__exit(d); + free(d); + } +} + +int dlfilter__start(struct dlfilter *d, struct perf_session *session) +{ + if (d) { + d->session = session; + if (d->start) { + int ret; + + d->in_start = true; + ret = d->start(&d->data, d); + d->in_start = false; + return ret; + } + } + return 0; +} + +static int dlfilter__stop(struct dlfilter *d) +{ + if (d && d->stop) { + int ret; + + d->in_stop = true; + ret = d->stop(d->data, d); + d->in_stop = false; + return ret; + } + return 0; +} + +void dlfilter__cleanup(struct dlfilter *d) +{ + if (d) { + dlfilter__stop(d); + dlfilter__close(d); + dlfilter__free(d); + } +} + +#define ASSIGN(x) d_sample.x = sample->x + +int dlfilter__do_filter_event(struct dlfilter *d, + union perf_event *event, + struct perf_sample *sample, + struct evsel *evsel, + struct machine *machine, + struct addr_location *al, + struct addr_location *addr_al, + bool early) +{ + struct perf_dlfilter_sample d_sample; + struct perf_dlfilter_al d_ip_al; + struct perf_dlfilter_al d_addr_al; + int ret; + + d->event = event; + d->sample = sample; + d->evsel = evsel; + d->machine = machine; + d->al = al; + d->addr_al = addr_al; + d->d_sample = &d_sample; + d->d_ip_al = &d_ip_al; + d->d_addr_al = &d_addr_al; + + d_sample.size = sizeof(d_sample); + d_ip_al.size = 0; /* To indicate d_ip_al is not initialized */ + d_addr_al.size = 0; /* To indicate d_addr_al is not initialized */ + + ASSIGN(ip); + ASSIGN(pid); + ASSIGN(tid); + ASSIGN(time); + ASSIGN(addr); + ASSIGN(id); + ASSIGN(stream_id); + ASSIGN(period); + ASSIGN(weight); + ASSIGN(ins_lat); + ASSIGN(p_stage_cyc); + ASSIGN(transaction); + ASSIGN(insn_cnt); + ASSIGN(cyc_cnt); + ASSIGN(cpu); + ASSIGN(flags); + ASSIGN(data_src); + ASSIGN(phys_addr); + ASSIGN(data_page_size); + ASSIGN(code_page_size); + ASSIGN(cgroup); + ASSIGN(cpumode); + ASSIGN(misc); + ASSIGN(raw_size); + ASSIGN(raw_data); + + if (sample->branch_stack) { + d_sample.brstack_nr = sample->branch_stack->nr; + d_sample.brstack = (struct perf_branch_entry *)perf_sample__branch_entries(sample); + } else { + d_sample.brstack_nr = 0; + d_sample.brstack = NULL; + } + + if (sample->callchain) { + d_sample.raw_callchain_nr = sample->callchain->nr; + d_sample.raw_callchain = (__u64 *)sample->callchain->ips; + } else { + d_sample.raw_callchain_nr = 0; + d_sample.raw_callchain = NULL; + } + + d_sample.addr_correlates_sym = + (evsel->core.attr.sample_type & PERF_SAMPLE_ADDR) && + sample_addr_correlates_sym(&evsel->core.attr); + + d_sample.event = evsel__name(evsel); + + d->ctx_valid = true; + + if (early) + ret = d->filter_event_early(d->data, &d_sample, d); + else + ret = d->filter_event(d->data, &d_sample, d); + + d->ctx_valid = false; + + return ret; +} + +static bool get_filter_desc(const char *dirname, const char *name, + char **desc, char **long_desc) +{ + char path[PATH_MAX]; + void *handle; + const char *(*desc_fn)(const char **long_description); + + snprintf(path, sizeof(path), "%s/%s", dirname, name); + handle = dlopen(path, RTLD_NOW); + if (!handle || !(dlsym(handle, "filter_event") || dlsym(handle, "filter_event_early"))) + return false; + desc_fn = dlsym(handle, "filter_description"); + if (desc_fn) { + const char *dsc; + const char *long_dsc; + + dsc = desc_fn(&long_dsc); + if (dsc) + *desc = strdup(dsc); + if (long_dsc) + *long_desc = strdup(long_dsc); + } + dlclose(handle); + return true; +} + +static void list_filters(const char *dirname) +{ + struct dirent *entry; + DIR *dir; + + dir = opendir(dirname); + if (!dir) + return; + + while ((entry = readdir(dir)) != NULL) + { + size_t n = strlen(entry->d_name); + char *long_desc = NULL; + char *desc = NULL; + + if (entry->d_type == DT_DIR || n < 4 || + strcmp(".so", entry->d_name + n - 3)) + continue; + if (!get_filter_desc(dirname, entry->d_name, &desc, &long_desc)) + continue; + printf(" %-36s %s\n", entry->d_name, desc ? desc : ""); + if (verbose) { + char *p = long_desc; + char *line; + + while ((line = strsep(&p, "\n")) != NULL) + printf("%39s%s\n", "", line); + } + free(long_desc); + free(desc); + } + + closedir(dir); +} + +int list_available_dlfilters(const struct option *opt __maybe_unused, + const char *s __maybe_unused, + int unset __maybe_unused) +{ + char path[PATH_MAX]; + char *exec_path; + + printf("List of available dlfilters:\n"); + + list_filters("."); + + exec_path = get_argv_exec_path(); + if (!exec_path) + goto out; + snprintf(path, sizeof(path), "%s/dlfilters", exec_path); + + list_filters(path); + + free(exec_path); +out: + exit(0); +} diff --git a/tools/perf/util/dlfilter.h b/tools/perf/util/dlfilter.h new file mode 100644 index 000000000000..505980442360 --- /dev/null +++ b/tools/perf/util/dlfilter.h @@ -0,0 +1,97 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * dlfilter.h: Interface to perf script --dlfilter shared object + * Copyright (c) 2021, Intel Corporation. + */ + +#ifndef PERF_UTIL_DLFILTER_H +#define PERF_UTIL_DLFILTER_H + +struct perf_session; +union perf_event; +struct perf_sample; +struct evsel; +struct machine; +struct addr_location; +struct perf_dlfilter_fns; +struct perf_dlfilter_sample; +struct perf_dlfilter_al; + +struct dlfilter { + char *file; + void *handle; + void *data; + struct perf_session *session; + bool ctx_valid; + bool in_start; + bool in_stop; + int dlargc; + char **dlargv; + + union perf_event *event; + struct perf_sample *sample; + struct evsel *evsel; + struct machine *machine; + struct addr_location *al; + struct addr_location *addr_al; + struct perf_dlfilter_sample *d_sample; + struct perf_dlfilter_al *d_ip_al; + struct perf_dlfilter_al *d_addr_al; + + int (*start)(void **data, void *ctx); + int (*stop)(void *data, void *ctx); + + int (*filter_event)(void *data, + const struct perf_dlfilter_sample *sample, + void *ctx); + int (*filter_event_early)(void *data, + const struct perf_dlfilter_sample *sample, + void *ctx); + + struct perf_dlfilter_fns *fns; +}; + +struct dlfilter *dlfilter__new(const char *file, int dlargc, char **dlargv); + +int dlfilter__start(struct dlfilter *d, struct perf_session *session); + +int dlfilter__do_filter_event(struct dlfilter *d, + union perf_event *event, + struct perf_sample *sample, + struct evsel *evsel, + struct machine *machine, + struct addr_location *al, + struct addr_location *addr_al, + bool early); + +void dlfilter__cleanup(struct dlfilter *d); + +static inline int dlfilter__filter_event(struct dlfilter *d, + union perf_event *event, + struct perf_sample *sample, + struct evsel *evsel, + struct machine *machine, + struct addr_location *al, + struct addr_location *addr_al) +{ + if (!d || !d->filter_event) + return 0; + return dlfilter__do_filter_event(d, event, sample, evsel, machine, al, addr_al, false); +} + +static inline int dlfilter__filter_event_early(struct dlfilter *d, + union perf_event *event, + struct perf_sample *sample, + struct evsel *evsel, + struct machine *machine, + struct addr_location *al, + struct addr_location *addr_al) +{ + if (!d || !d->filter_event_early) + return 0; + return dlfilter__do_filter_event(d, event, sample, evsel, machine, al, addr_al, true); +} + +int list_available_dlfilters(const struct option *opt, const char *s, int unset); + +#endif diff --git a/tools/perf/util/dso.c b/tools/perf/util/dso.c index d786cf6b0cfa..ee15db2be2f4 100644 --- a/tools/perf/util/dso.c +++ b/tools/perf/util/dso.c @@ -1154,8 +1154,10 @@ struct map *dso__new_map(const char *name) struct map *map = NULL; struct dso *dso = dso__new(name); - if (dso) + if (dso) { map = map__new2(0, dso); + dso__put(dso); + } return map; } diff --git a/tools/perf/util/dwarf-aux.c b/tools/perf/util/dwarf-aux.c index b2f4920e19a6..609ca1671501 100644 --- a/tools/perf/util/dwarf-aux.c +++ b/tools/perf/util/dwarf-aux.c @@ -113,14 +113,14 @@ static Dwarf_Line *cu_getsrc_die(Dwarf_Die *cu_die, Dwarf_Addr addr) * * Find a line number and file name for @addr in @cu_die. */ -int cu_find_lineinfo(Dwarf_Die *cu_die, unsigned long addr, - const char **fname, int *lineno) +int cu_find_lineinfo(Dwarf_Die *cu_die, Dwarf_Addr addr, + const char **fname, int *lineno) { Dwarf_Line *line; Dwarf_Die die_mem; Dwarf_Addr faddr; - if (die_find_realfunc(cu_die, (Dwarf_Addr)addr, &die_mem) + if (die_find_realfunc(cu_die, addr, &die_mem) && die_entrypc(&die_mem, &faddr) == 0 && faddr == addr) { *fname = dwarf_decl_file(&die_mem); @@ -128,7 +128,7 @@ int cu_find_lineinfo(Dwarf_Die *cu_die, unsigned long addr, goto out; } - line = cu_getsrc_die(cu_die, (Dwarf_Addr)addr); + line = cu_getsrc_die(cu_die, addr); if (line && dwarf_lineno(line, lineno) == 0) { *fname = dwarf_linesrc(line, NULL, NULL); if (!*fname) @@ -975,9 +975,13 @@ static int __die_find_variable_cb(Dwarf_Die *die_mem, void *data) if ((tag == DW_TAG_formal_parameter || tag == DW_TAG_variable) && die_compare_name(die_mem, fvp->name) && - /* Does the DIE have location information or external instance? */ + /* + * Does the DIE have location information or const value + * or external instance? + */ (dwarf_attr(die_mem, DW_AT_external, &attr) || - dwarf_attr(die_mem, DW_AT_location, &attr))) + dwarf_attr(die_mem, DW_AT_location, &attr) || + dwarf_attr(die_mem, DW_AT_const_value, &attr))) return DIE_FIND_CB_END; if (dwarf_haspc(die_mem, fvp->addr)) return DIE_FIND_CB_CONTINUE; diff --git a/tools/perf/util/dwarf-aux.h b/tools/perf/util/dwarf-aux.h index cb99646843a9..7ee0fa19b5c4 100644 --- a/tools/perf/util/dwarf-aux.h +++ b/tools/perf/util/dwarf-aux.h @@ -19,7 +19,7 @@ const char *cu_find_realpath(Dwarf_Die *cu_die, const char *fname); const char *cu_get_comp_dir(Dwarf_Die *cu_die); /* Get a line number and file name for given address */ -int cu_find_lineinfo(Dwarf_Die *cudie, unsigned long addr, +int cu_find_lineinfo(Dwarf_Die *cudie, Dwarf_Addr addr, const char **fname, int *lineno); /* Walk on functions at given address */ diff --git a/tools/perf/util/env.c b/tools/perf/util/env.c index 9130f6fad8d5..cec2e6cad8aa 100644 --- a/tools/perf/util/env.c +++ b/tools/perf/util/env.c @@ -144,6 +144,7 @@ static void perf_env__purge_bpf(struct perf_env *env) node = rb_entry(next, struct bpf_prog_info_node, rb_node); next = rb_next(&node->rb_node); rb_erase(&node->rb_node, root); + free(node->info_linear); free(node); } @@ -185,10 +186,12 @@ void perf_env__exit(struct perf_env *env) zfree(&env->cpuid); zfree(&env->cmdline); zfree(&env->cmdline_argv); + zfree(&env->sibling_dies); zfree(&env->sibling_cores); zfree(&env->sibling_threads); zfree(&env->pmu_mappings); zfree(&env->cpu); + zfree(&env->cpu_pmu_caps); zfree(&env->numa_map); for (i = 0; i < env->nr_numa_nodes; i++) @@ -202,6 +205,18 @@ void perf_env__exit(struct perf_env *env) for (i = 0; i < env->nr_memory_nodes; i++) zfree(&env->memory_nodes[i].set); zfree(&env->memory_nodes); + + for (i = 0; i < env->nr_hybrid_nodes; i++) { + zfree(&env->hybrid_nodes[i].pmu_name); + zfree(&env->hybrid_nodes[i].cpus); + } + zfree(&env->hybrid_nodes); + + for (i = 0; i < env->nr_hybrid_cpc_nodes; i++) { + zfree(&env->hybrid_cpc_nodes[i].cpu_pmu_caps); + zfree(&env->hybrid_cpc_nodes[i].pmu_name); + } + zfree(&env->hybrid_cpc_nodes); } void perf_env__init(struct perf_env *env __maybe_unused) diff --git a/tools/perf/util/env.h b/tools/perf/util/env.h index ca249bf5e984..6824a7423a2d 100644 --- a/tools/perf/util/env.h +++ b/tools/perf/util/env.h @@ -37,6 +37,18 @@ struct memory_node { unsigned long *set; }; +struct hybrid_node { + char *pmu_name; + char *cpus; +}; + +struct hybrid_cpc_node { + int nr_cpu_pmu_caps; + unsigned int max_branches; + char *cpu_pmu_caps; + char *pmu_name; +}; + struct perf_env { char *hostname; char *os_release; @@ -59,6 +71,8 @@ struct perf_env { int nr_pmu_mappings; int nr_groups; int nr_cpu_pmu_caps; + int nr_hybrid_nodes; + int nr_hybrid_cpc_nodes; char *cmdline; const char **cmdline_argv; char *sibling_cores; @@ -77,6 +91,8 @@ struct perf_env { struct numa_node *numa_nodes; struct memory_node *memory_nodes; unsigned long long memory_bsize; + struct hybrid_node *hybrid_nodes; + struct hybrid_cpc_node *hybrid_cpc_nodes; #ifdef HAVE_LIBBPF_SUPPORT /* * bpf_info_lock protects bpf rbtrees. This is needed because the diff --git a/tools/perf/util/event.h b/tools/perf/util/event.h index 8a62fb39e365..19ad64f2bd83 100644 --- a/tools/perf/util/event.h +++ b/tools/perf/util/event.h @@ -100,7 +100,7 @@ enum { PERF_IP_FLAG_VMEXIT = 1ULL << 12, }; -#define PERF_IP_FLAG_CHARS "bcrosyiABEx" +#define PERF_IP_FLAG_CHARS "bcrosyiABExgh" #define PERF_BRANCH_MASK (\ PERF_IP_FLAG_BRANCH |\ diff --git a/tools/perf/util/evlist.c b/tools/perf/util/evlist.c index 6e5c41528c7d..47581a237c7a 100644 --- a/tools/perf/util/evlist.c +++ b/tools/perf/util/evlist.c @@ -165,11 +165,9 @@ void evlist__delete(struct evlist *evlist) void evlist__add(struct evlist *evlist, struct evsel *entry) { - entry->evlist = evlist; - entry->idx = evlist->core.nr_entries; - entry->tracking = !entry->idx; - perf_evlist__add(&evlist->core, &entry->core); + entry->evlist = evlist; + entry->tracking = !entry->core.idx; if (evlist->core.nr_entries == 1) evlist__set_id_pos(evlist); @@ -194,7 +192,7 @@ void evlist__splice_list_tail(struct evlist *evlist, struct list_head *list) } __evlist__for_each_entry_safe(list, temp, evsel) { - if (evsel->leader == leader) { + if (evsel__has_leader(evsel, leader)) { list_del_init(&evsel->core.node); evlist__add(evlist, evsel); } @@ -225,26 +223,9 @@ out: return err; } -void __evlist__set_leader(struct list_head *list) -{ - struct evsel *evsel, *leader; - - leader = list_entry(list->next, struct evsel, core.node); - evsel = list_entry(list->prev, struct evsel, core.node); - - leader->core.nr_members = evsel->idx - leader->idx + 1; - - __evlist__for_each_entry(list, evsel) { - evsel->leader = leader; - } -} - void evlist__set_leader(struct evlist *evlist) { - if (evlist->core.nr_entries) { - evlist->nr_groups = evlist->core.nr_entries > 1 ? 1 : 0; - __evlist__set_leader(&evlist->core.entries); - } + perf_evlist__set_leader(&evlist->core); } int __evlist__add_default(struct evlist *evlist, bool precise) @@ -425,9 +406,6 @@ static void __evlist__disable(struct evlist *evlist, char *evsel_name) if (affinity__setup(&affinity) < 0) return; - evlist__for_each_entry(evlist, pos) - bpf_counter__disable(pos); - /* Disable 'immediate' events last */ for (imm = 0; imm <= 1; imm++) { evlist__for_each_cpu(evlist, i, cpu) { @@ -1629,7 +1607,7 @@ void evlist__to_front(struct evlist *evlist, struct evsel *move_evsel) return; evlist__for_each_entry_safe(evlist, n, evsel) { - if (evsel->leader == move_evsel->leader) + if (evsel__leader(evsel) == evsel__leader(move_evsel)) list_move_tail(&evsel->core.node, &move); } @@ -1753,7 +1731,7 @@ bool evlist__exclude_kernel(struct evlist *evlist) */ void evlist__force_leader(struct evlist *evlist) { - if (!evlist->nr_groups) { + if (!evlist->core.nr_groups) { struct evsel *leader = evlist__first(evlist); evlist__set_leader(evlist); @@ -1766,7 +1744,8 @@ struct evsel *evlist__reset_weak_group(struct evlist *evsel_list, struct evsel * struct evsel *c2, *leader; bool is_open = true; - leader = evsel->leader; + leader = evsel__leader(evsel); + pr_debug("Weak group for %s/%d failed\n", leader->name, leader->core.nr_members); @@ -1777,10 +1756,10 @@ struct evsel *evlist__reset_weak_group(struct evlist *evsel_list, struct evsel * evlist__for_each_entry(evsel_list, c2) { if (c2 == evsel) is_open = false; - if (c2->leader == leader) { + if (evsel__has_leader(c2, leader)) { if (is_open && close) perf_evsel__close(&c2->core); - c2->leader = c2; + evsel__set_leader(c2, c2); c2->core.nr_members = 0; /* * Set this for all former members of the group @@ -2140,7 +2119,7 @@ struct evsel *evlist__find_evsel(struct evlist *evlist, int idx) struct evsel *evsel; evlist__for_each_entry(evlist, evsel) { - if (evsel->idx == idx) + if (evsel->core.idx == idx) return evsel; } return NULL; @@ -2164,3 +2143,28 @@ int evlist__scnprintf_evsels(struct evlist *evlist, size_t size, char *bf) return printed; } + +void evlist__check_mem_load_aux(struct evlist *evlist) +{ + struct evsel *leader, *evsel, *pos; + + /* + * For some platforms, the 'mem-loads' event is required to use + * together with 'mem-loads-aux' within a group and 'mem-loads-aux' + * must be the group leader. Now we disable this group before reporting + * because 'mem-loads-aux' is just an auxiliary event. It doesn't carry + * any valid memory load information. + */ + evlist__for_each_entry(evlist, evsel) { + leader = evsel__leader(evsel); + if (leader == evsel) + continue; + + if (leader->name && strstr(leader->name, "mem-loads-aux")) { + for_each_group_evsel(pos, leader) { + evsel__set_leader(pos, pos); + pos->core.nr_members = 0; + } + } + } +} diff --git a/tools/perf/util/evlist.h b/tools/perf/util/evlist.h index a8b97b50cceb..5c22383489ae 100644 --- a/tools/perf/util/evlist.h +++ b/tools/perf/util/evlist.h @@ -50,7 +50,6 @@ enum bkw_mmap_state { struct evlist { struct perf_evlist core; - int nr_groups; bool enabled; int id_pos; int is_pos; @@ -202,7 +201,6 @@ void evlist__set_selected(struct evlist *evlist, struct evsel *evsel); int evlist__create_maps(struct evlist *evlist, struct target *target); int evlist__apply_filters(struct evlist *evlist, struct evsel **err_evsel); -void __evlist__set_leader(struct list_head *list); void evlist__set_leader(struct evlist *evlist); u64 __evlist__combined_sample_type(struct evlist *evlist); @@ -367,4 +365,5 @@ int evlist__ctlfd_ack(struct evlist *evlist); struct evsel *evlist__find_evsel(struct evlist *evlist, int idx); int evlist__scnprintf_evsels(struct evlist *evlist, size_t size, char *bf); +void evlist__check_mem_load_aux(struct evlist *evlist); #endif /* __PERF_EVLIST_H */ diff --git a/tools/perf/util/evsel.c b/tools/perf/util/evsel.c index 4a3cd1b5bb33..f61e5dd53f5d 100644 --- a/tools/perf/util/evsel.c +++ b/tools/perf/util/evsel.c @@ -239,10 +239,8 @@ bool evsel__is_function_event(struct evsel *evsel) void evsel__init(struct evsel *evsel, struct perf_event_attr *attr, int idx) { - perf_evsel__init(&evsel->core, attr); - evsel->idx = idx; + perf_evsel__init(&evsel->core, attr, idx); evsel->tracking = !idx; - evsel->leader = evsel; evsel->unit = ""; evsel->scale = 1.0; evsel->max_events = ULONG_MAX; @@ -410,7 +408,7 @@ struct evsel *evsel__clone(struct evsel *orig) evsel->cgrp = cgroup__get(orig->cgrp); evsel->tp_format = orig->tp_format; evsel->handler = orig->handler; - evsel->leader = orig->leader; + evsel->core.leader = orig->core.leader; evsel->max_events = orig->max_events; evsel->tool_event = orig->tool_event; @@ -428,6 +426,7 @@ struct evsel *evsel__clone(struct evsel *orig) evsel->auto_merge_stats = orig->auto_merge_stats; evsel->collect_stat = orig->collect_stat; evsel->weak_group = orig->weak_group; + evsel->use_config_name = orig->use_config_name; if (evsel__copy_config_terms(evsel, orig) < 0) goto out_err; @@ -1074,7 +1073,7 @@ void __weak arch_evsel__set_sample_weight(struct evsel *evsel) void evsel__config(struct evsel *evsel, struct record_opts *opts, struct callchain_param *callchain) { - struct evsel *leader = evsel->leader; + struct evsel *leader = evsel__leader(evsel); struct perf_event_attr *attr = &evsel->core.attr; int track = evsel->tracking; bool per_cpu = opts->target.default_per_cpu && !opts->target.per_thread; @@ -1581,9 +1580,30 @@ int __evsel__read_on_cpu(struct evsel *evsel, int cpu, int thread, bool scale) return 0; } +static int evsel__match_other_cpu(struct evsel *evsel, struct evsel *other, + int cpu) +{ + int cpuid; + + cpuid = perf_cpu_map__cpu(evsel->core.cpus, cpu); + return perf_cpu_map__idx(other->core.cpus, cpuid); +} + +static int evsel__hybrid_group_cpu(struct evsel *evsel, int cpu) +{ + struct evsel *leader = evsel__leader(evsel); + + if ((evsel__is_hybrid(evsel) && !evsel__is_hybrid(leader)) || + (!evsel__is_hybrid(evsel) && evsel__is_hybrid(leader))) { + return evsel__match_other_cpu(evsel, leader, cpu); + } + + return cpu; +} + static int get_group_fd(struct evsel *evsel, int cpu, int thread) { - struct evsel *leader = evsel->leader; + struct evsel *leader = evsel__leader(evsel); int fd; if (evsel__is_group_leader(evsel)) @@ -1595,6 +1615,10 @@ static int get_group_fd(struct evsel *evsel, int cpu, int thread) */ BUG_ON(!leader->core.fd); + cpu = evsel__hybrid_group_cpu(evsel, cpu); + if (cpu == -1) + return -1; + fd = FD(leader, cpu, thread); BUG_ON(fd == -1); @@ -2825,3 +2849,23 @@ bool evsel__is_hybrid(struct evsel *evsel) { return evsel->pmu_name && perf_pmu__is_hybrid(evsel->pmu_name); } + +struct evsel *evsel__leader(struct evsel *evsel) +{ + return container_of(evsel->core.leader, struct evsel, core); +} + +bool evsel__has_leader(struct evsel *evsel, struct evsel *leader) +{ + return evsel->core.leader == &leader->core; +} + +bool evsel__is_leader(struct evsel *evsel) +{ + return evsel__has_leader(evsel, evsel); +} + +void evsel__set_leader(struct evsel *evsel, struct evsel *leader) +{ + evsel->core.leader = &leader->core; +} diff --git a/tools/perf/util/evsel.h b/tools/perf/util/evsel.h index 75cf5dbfe208..80383096d51c 100644 --- a/tools/perf/util/evsel.h +++ b/tools/perf/util/evsel.h @@ -49,7 +49,6 @@ struct evsel { struct perf_evsel core; struct evlist *evlist; off_t id_offset; - int idx; int id_pos; int is_pos; unsigned int sample_size; @@ -83,8 +82,10 @@ struct evsel { bool collect_stat; bool weak_group; bool bpf_counter; + bool use_config_name; int bpf_fd; struct bpf_object *bpf_obj; + struct list_head config_terms; }; /* @@ -116,10 +117,7 @@ struct evsel { bool merged_stat; bool reset_group; bool errored; - bool use_config_name; struct hashmap *per_pkg_mask; - struct evsel *leader; - struct list_head config_terms; int err; int cpu_iter; struct { @@ -368,7 +366,7 @@ static inline struct evsel *evsel__prev(struct evsel *evsel) */ static inline bool evsel__is_group_leader(const struct evsel *evsel) { - return evsel->leader == evsel; + return evsel->core.leader == &evsel->core; } /** @@ -406,19 +404,19 @@ int evsel__open_strerror(struct evsel *evsel, struct target *target, static inline int evsel__group_idx(struct evsel *evsel) { - return evsel->idx - evsel->leader->idx; + return evsel->core.idx - evsel->core.leader->idx; } /* Iterates group WITHOUT the leader. */ #define for_each_group_member(_evsel, _leader) \ for ((_evsel) = list_entry((_leader)->core.node.next, struct evsel, core.node); \ - (_evsel) && (_evsel)->leader == (_leader); \ + (_evsel) && (_evsel)->core.leader == (&_leader->core); \ (_evsel) = list_entry((_evsel)->core.node.next, struct evsel, core.node)) /* Iterates group WITH the leader. */ #define for_each_group_evsel(_evsel, _leader) \ for ((_evsel) = _leader; \ - (_evsel) && (_evsel)->leader == (_leader); \ + (_evsel) && (_evsel)->core.leader == (&_leader->core); \ (_evsel) = list_entry((_evsel)->core.node.next, struct evsel, core.node)) static inline bool evsel__has_branch_callstack(const struct evsel *evsel) @@ -463,4 +461,8 @@ int evsel__store_ids(struct evsel *evsel, struct evlist *evlist); void evsel__zero_per_pkg(struct evsel *evsel); bool evsel__is_hybrid(struct evsel *evsel); +struct evsel *evsel__leader(struct evsel *evsel); +bool evsel__has_leader(struct evsel *evsel, struct evsel *leader); +bool evsel__is_leader(struct evsel *evsel); +void evsel__set_leader(struct evsel *evsel, struct evsel *leader); #endif /* __PERF_EVSEL_H */ diff --git a/tools/perf/util/header.c b/tools/perf/util/header.c index aa1e42518d37..44249027507a 100644 --- a/tools/perf/util/header.c +++ b/tools/perf/util/header.c @@ -49,6 +49,7 @@ #include "cputopo.h" #include "bpf-event.h" #include "clockid.h" +#include "pmu-hybrid.h" #include <linux/ctype.h> #include <internal/lib.h> @@ -777,7 +778,7 @@ static int write_pmu_mappings(struct feat_fd *ff, static int write_group_desc(struct feat_fd *ff, struct evlist *evlist) { - u32 nr_groups = evlist->nr_groups; + u32 nr_groups = evlist->core.nr_groups; struct evsel *evsel; int ret; @@ -788,7 +789,7 @@ static int write_group_desc(struct feat_fd *ff, evlist__for_each_entry(evlist, evsel) { if (evsel__is_group_leader(evsel) && evsel->core.nr_members > 1) { const char *name = evsel->group_name ?: "{anon_group}"; - u32 leader_idx = evsel->idx; + u32 leader_idx = evsel->core.idx; u32 nr_members = evsel->core.nr_members; ret = do_write_string(ff, name); @@ -932,6 +933,40 @@ static int write_clock_data(struct feat_fd *ff, return do_write(ff, data64, sizeof(*data64)); } +static int write_hybrid_topology(struct feat_fd *ff, + struct evlist *evlist __maybe_unused) +{ + struct hybrid_topology *tp; + int ret; + u32 i; + + tp = hybrid_topology__new(); + if (!tp) + return -ENOENT; + + ret = do_write(ff, &tp->nr, sizeof(u32)); + if (ret < 0) + goto err; + + for (i = 0; i < tp->nr; i++) { + struct hybrid_topology_node *n = &tp->nodes[i]; + + ret = do_write_string(ff, n->pmu_name); + if (ret < 0) + goto err; + + ret = do_write_string(ff, n->cpus); + if (ret < 0) + goto err; + } + + ret = 0; + +err: + hybrid_topology__delete(tp); + return ret; +} + static int write_dir_format(struct feat_fd *ff, struct evlist *evlist __maybe_unused) { @@ -1425,18 +1460,14 @@ static int write_compressed(struct feat_fd *ff __maybe_unused, return do_write(ff, &(ff->ph->env.comp_mmap_len), sizeof(ff->ph->env.comp_mmap_len)); } -static int write_cpu_pmu_caps(struct feat_fd *ff, - struct evlist *evlist __maybe_unused) +static int write_per_cpu_pmu_caps(struct feat_fd *ff, struct perf_pmu *pmu, + bool write_pmu) { - struct perf_pmu *cpu_pmu = perf_pmu__find("cpu"); struct perf_pmu_caps *caps = NULL; int nr_caps; int ret; - if (!cpu_pmu) - return -ENOENT; - - nr_caps = perf_pmu__caps_parse(cpu_pmu); + nr_caps = perf_pmu__caps_parse(pmu); if (nr_caps < 0) return nr_caps; @@ -1444,7 +1475,7 @@ static int write_cpu_pmu_caps(struct feat_fd *ff, if (ret < 0) return ret; - list_for_each_entry(caps, &cpu_pmu->caps, list) { + list_for_each_entry(caps, &pmu->caps, list) { ret = do_write_string(ff, caps->name); if (ret < 0) return ret; @@ -1454,9 +1485,49 @@ static int write_cpu_pmu_caps(struct feat_fd *ff, return ret; } + if (write_pmu) { + ret = do_write_string(ff, pmu->name); + if (ret < 0) + return ret; + } + return ret; } +static int write_cpu_pmu_caps(struct feat_fd *ff, + struct evlist *evlist __maybe_unused) +{ + struct perf_pmu *cpu_pmu = perf_pmu__find("cpu"); + + if (!cpu_pmu) + return -ENOENT; + + return write_per_cpu_pmu_caps(ff, cpu_pmu, false); +} + +static int write_hybrid_cpu_pmu_caps(struct feat_fd *ff, + struct evlist *evlist __maybe_unused) +{ + struct perf_pmu *pmu; + u32 nr_pmu = perf_pmu__hybrid_pmu_num(); + int ret; + + if (nr_pmu == 0) + return -ENOENT; + + ret = do_write(ff, &nr_pmu, sizeof(nr_pmu)); + if (ret < 0) + return ret; + + perf_pmu__for_each_hybrid_pmu(pmu) { + ret = write_per_cpu_pmu_caps(ff, pmu, true); + if (ret < 0) + return ret; + } + + return 0; +} + static void print_hostname(struct feat_fd *ff, FILE *fp) { fprintf(fp, "# hostname : %s\n", ff->ph->env.hostname); @@ -1623,6 +1694,18 @@ static void print_clock_data(struct feat_fd *ff, FILE *fp) clockid_name(clockid)); } +static void print_hybrid_topology(struct feat_fd *ff, FILE *fp) +{ + int i; + struct hybrid_node *n; + + fprintf(fp, "# hybrid cpu system:\n"); + for (i = 0; i < ff->ph->env.nr_hybrid_nodes; i++) { + n = &ff->ph->env.hybrid_nodes[i]; + fprintf(fp, "# %s cpu list : %s\n", n->pmu_name, n->cpus); + } +} + static void print_dir_format(struct feat_fd *ff, FILE *fp) { struct perf_session *session; @@ -1761,7 +1844,7 @@ static struct evsel *read_event_desc(struct feat_fd *ff) msz = sz; for (i = 0, evsel = events; i < nre; evsel++, i++) { - evsel->idx = i; + evsel->core.idx = i; /* * must read entire on-file attr struct to @@ -1916,18 +1999,28 @@ static void print_compressed(struct feat_fd *ff, FILE *fp) ff->ph->env.comp_level, ff->ph->env.comp_ratio); } -static void print_cpu_pmu_caps(struct feat_fd *ff, FILE *fp) +static void print_per_cpu_pmu_caps(FILE *fp, int nr_caps, char *cpu_pmu_caps, + char *pmu_name) { - const char *delimiter = "# cpu pmu capabilities: "; - u32 nr_caps = ff->ph->env.nr_cpu_pmu_caps; - char *str; + const char *delimiter; + char *str, buf[128]; if (!nr_caps) { - fprintf(fp, "# cpu pmu capabilities: not available\n"); + if (!pmu_name) + fprintf(fp, "# cpu pmu capabilities: not available\n"); + else + fprintf(fp, "# %s pmu capabilities: not available\n", pmu_name); return; } - str = ff->ph->env.cpu_pmu_caps; + if (!pmu_name) + scnprintf(buf, sizeof(buf), "# cpu pmu capabilities: "); + else + scnprintf(buf, sizeof(buf), "# %s pmu capabilities: ", pmu_name); + + delimiter = buf; + + str = cpu_pmu_caps; while (nr_caps--) { fprintf(fp, "%s%s", delimiter, str); delimiter = ", "; @@ -1937,6 +2030,24 @@ static void print_cpu_pmu_caps(struct feat_fd *ff, FILE *fp) fprintf(fp, "\n"); } +static void print_cpu_pmu_caps(struct feat_fd *ff, FILE *fp) +{ + print_per_cpu_pmu_caps(fp, ff->ph->env.nr_cpu_pmu_caps, + ff->ph->env.cpu_pmu_caps, NULL); +} + +static void print_hybrid_cpu_pmu_caps(struct feat_fd *ff, FILE *fp) +{ + struct hybrid_cpc_node *n; + + for (int i = 0; i < ff->ph->env.nr_hybrid_cpc_nodes; i++) { + n = &ff->ph->env.hybrid_cpc_nodes[i]; + print_per_cpu_pmu_caps(fp, n->nr_cpu_pmu_caps, + n->cpu_pmu_caps, + n->pmu_name); + } +} + static void print_pmu_mappings(struct feat_fd *ff, FILE *fp) { const char *delimiter = "# pmu mappings: "; @@ -2268,7 +2379,7 @@ static struct evsel *evlist__find_by_index(struct evlist *evlist, int idx) struct evsel *evsel; evlist__for_each_entry(evlist, evsel) { - if (evsel->idx == idx) + if (evsel->core.idx == idx) return evsel; } @@ -2282,7 +2393,7 @@ static void evlist__set_event_name(struct evlist *evlist, struct evsel *event) if (!event->name) return; - evsel = evlist__find_by_index(evlist, event->idx); + evsel = evlist__find_by_index(evlist, event->core.idx); if (!evsel) return; @@ -2624,12 +2735,12 @@ static int process_group_desc(struct feat_fd *ff, void *data __maybe_unused) * Rebuild group relationship based on the group_desc */ session = container_of(ff->ph, struct perf_session, header); - session->evlist->nr_groups = nr_groups; + session->evlist->core.nr_groups = nr_groups; i = nr = 0; evlist__for_each_entry(session->evlist, evsel) { - if (evsel->idx == (int) desc[i].leader_idx) { - evsel->leader = evsel; + if (evsel->core.idx == (int) desc[i].leader_idx) { + evsel__set_leader(evsel, evsel); /* {anon_group} is a dummy name */ if (strcmp(desc[i].name, "{anon_group}")) { evsel->group_name = desc[i].name; @@ -2647,7 +2758,7 @@ static int process_group_desc(struct feat_fd *ff, void *data __maybe_unused) i++; } else if (nr) { /* This is a group member */ - evsel->leader = leader; + evsel__set_leader(evsel, leader); nr--; } @@ -2849,6 +2960,46 @@ static int process_clock_data(struct feat_fd *ff, return 0; } +static int process_hybrid_topology(struct feat_fd *ff, + void *data __maybe_unused) +{ + struct hybrid_node *nodes, *n; + u32 nr, i; + + /* nr nodes */ + if (do_read_u32(ff, &nr)) + return -1; + + nodes = zalloc(sizeof(*nodes) * nr); + if (!nodes) + return -ENOMEM; + + for (i = 0; i < nr; i++) { + n = &nodes[i]; + + n->pmu_name = do_read_string(ff); + if (!n->pmu_name) + goto error; + + n->cpus = do_read_string(ff); + if (!n->cpus) + goto error; + } + + ff->ph->env.nr_hybrid_nodes = nr; + ff->ph->env.hybrid_nodes = nodes; + return 0; + +error: + for (i = 0; i < nr; i++) { + free(nodes[i].pmu_name); + free(nodes[i].cpus); + } + + free(nodes); + return -1; +} + static int process_dir_format(struct feat_fd *ff, void *_data __maybe_unused) { @@ -3002,8 +3153,9 @@ static int process_compressed(struct feat_fd *ff, return 0; } -static int process_cpu_pmu_caps(struct feat_fd *ff, - void *data __maybe_unused) +static int process_per_cpu_pmu_caps(struct feat_fd *ff, int *nr_cpu_pmu_caps, + char **cpu_pmu_caps, + unsigned int *max_branches) { char *name, *value; struct strbuf sb; @@ -3017,7 +3169,7 @@ static int process_cpu_pmu_caps(struct feat_fd *ff, return 0; } - ff->ph->env.nr_cpu_pmu_caps = nr_caps; + *nr_cpu_pmu_caps = nr_caps; if (strbuf_init(&sb, 128) < 0) return -1; @@ -3039,12 +3191,12 @@ static int process_cpu_pmu_caps(struct feat_fd *ff, goto free_value; if (!strcmp(name, "branches")) - ff->ph->env.max_branches = atoi(value); + *max_branches = atoi(value); free(value); free(name); } - ff->ph->env.cpu_pmu_caps = strbuf_detach(&sb, NULL); + *cpu_pmu_caps = strbuf_detach(&sb, NULL); return 0; free_value: @@ -3056,6 +3208,63 @@ error: return -1; } +static int process_cpu_pmu_caps(struct feat_fd *ff, + void *data __maybe_unused) +{ + return process_per_cpu_pmu_caps(ff, &ff->ph->env.nr_cpu_pmu_caps, + &ff->ph->env.cpu_pmu_caps, + &ff->ph->env.max_branches); +} + +static int process_hybrid_cpu_pmu_caps(struct feat_fd *ff, + void *data __maybe_unused) +{ + struct hybrid_cpc_node *nodes; + u32 nr_pmu, i; + int ret; + + if (do_read_u32(ff, &nr_pmu)) + return -1; + + if (!nr_pmu) { + pr_debug("hybrid cpu pmu capabilities not available\n"); + return 0; + } + + nodes = zalloc(sizeof(*nodes) * nr_pmu); + if (!nodes) + return -ENOMEM; + + for (i = 0; i < nr_pmu; i++) { + struct hybrid_cpc_node *n = &nodes[i]; + + ret = process_per_cpu_pmu_caps(ff, &n->nr_cpu_pmu_caps, + &n->cpu_pmu_caps, + &n->max_branches); + if (ret) + goto err; + + n->pmu_name = do_read_string(ff); + if (!n->pmu_name) { + ret = -1; + goto err; + } + } + + ff->ph->env.nr_hybrid_cpc_nodes = nr_pmu; + ff->ph->env.hybrid_cpc_nodes = nodes; + return 0; + +err: + for (i = 0; i < nr_pmu; i++) { + free(nodes[i].cpu_pmu_caps); + free(nodes[i].pmu_name); + } + + free(nodes); + return ret; +} + #define FEAT_OPR(n, func, __full_only) \ [HEADER_##n] = { \ .name = __stringify(n), \ @@ -3117,6 +3326,8 @@ const struct perf_header_feature_ops feat_ops[HEADER_LAST_FEATURE] = { FEAT_OPR(COMPRESSED, compressed, false), FEAT_OPR(CPU_PMU_CAPS, cpu_pmu_caps, false), FEAT_OPR(CLOCK_DATA, clock_data, false), + FEAT_OPN(HYBRID_TOPOLOGY, hybrid_topology, true), + FEAT_OPR(HYBRID_CPU_PMU_CAPS, hybrid_cpu_pmu_caps, false), }; struct header_print_data { @@ -3814,6 +4025,11 @@ int perf_session__read_header(struct perf_session *session) if (perf_file_header__read(&f_header, header, fd) < 0) return -EINVAL; + if (header->needs_swap && data->in_place_update) { + pr_err("In-place update not supported when byte-swapping is required\n"); + return -EINVAL; + } + /* * Sanity check that perf.data was written cleanly; data size is * initialized to 0 and updated only if the on_exit function is run. diff --git a/tools/perf/util/header.h b/tools/perf/util/header.h index 2aca71763ecf..ae6b1cf19a7d 100644 --- a/tools/perf/util/header.h +++ b/tools/perf/util/header.h @@ -45,6 +45,8 @@ enum { HEADER_COMPRESSED, HEADER_CPU_PMU_CAPS, HEADER_CLOCK_DATA, + HEADER_HYBRID_TOPOLOGY, + HEADER_HYBRID_CPU_PMU_CAPS, HEADER_LAST_FEATURE, HEADER_FEAT_BITS = 256, }; diff --git a/tools/perf/util/intel-pt-decoder/intel-pt-decoder.c b/tools/perf/util/intel-pt-decoder/intel-pt-decoder.c index 8c59677bee13..5ab631702769 100644 --- a/tools/perf/util/intel-pt-decoder/intel-pt-decoder.c +++ b/tools/perf/util/intel-pt-decoder/intel-pt-decoder.c @@ -35,10 +35,17 @@ #define BIT63 (((uint64_t)1 << 63)) +#define SEVEN_BYTES 0xffffffffffffffULL + +#define NO_VMCS 0xffffffffffULL + #define INTEL_PT_RETURN 1 -/* Maximum number of loops with no packets consumed i.e. stuck in a loop */ -#define INTEL_PT_MAX_LOOPS 10000 +/* + * Default maximum number of loops with no packets consumed i.e. stuck in a + * loop. + */ +#define INTEL_PT_MAX_LOOPS 100000 struct intel_pt_blk { struct intel_pt_blk *prev; @@ -51,6 +58,11 @@ struct intel_pt_stack { int pos; }; +enum intel_pt_p_once { + INTEL_PT_PRT_ONCE_UNK_VMCS, + INTEL_PT_PRT_ONCE_ERANGE, +}; + enum intel_pt_pkt_state { INTEL_PT_STATE_NO_PSB, INTEL_PT_STATE_NO_IP, @@ -64,6 +76,7 @@ enum intel_pt_pkt_state { INTEL_PT_STATE_FUP_NO_TIP, INTEL_PT_STATE_FUP_IN_PSB, INTEL_PT_STATE_RESAMPLE, + INTEL_PT_STATE_VM_TIME_CORRELATION, }; static inline bool intel_pt_sample_time(enum intel_pt_pkt_state pkt_state) @@ -75,6 +88,7 @@ static inline bool intel_pt_sample_time(enum intel_pt_pkt_state pkt_state) case INTEL_PT_STATE_IN_SYNC: case INTEL_PT_STATE_TNT_CONT: case INTEL_PT_STATE_RESAMPLE: + case INTEL_PT_STATE_VM_TIME_CORRELATION: return true; case INTEL_PT_STATE_TNT: case INTEL_PT_STATE_TIP: @@ -107,6 +121,7 @@ struct intel_pt_decoder { uint64_t max_insn_cnt, void *data); bool (*pgd_ip)(uint64_t ip, void *data); int (*lookahead)(void *data, intel_pt_lookahead_cb_t cb, void *cb_data); + struct intel_pt_vmcs_info *(*findnew_vmcs_info)(void *data, uint64_t vmcs); void *data; struct intel_pt_state state; const unsigned char *buf; @@ -122,6 +137,11 @@ struct intel_pt_decoder { bool in_psb; bool hop; bool leap; + bool vm_time_correlation; + bool vm_tm_corr_dry_run; + bool vm_tm_corr_reliable; + bool vm_tm_corr_same_buf; + bool vm_tm_corr_continuous; bool nr; bool next_nr; enum intel_pt_param_flags flags; @@ -139,6 +159,11 @@ struct intel_pt_decoder { uint64_t ctc_delta; uint64_t cycle_cnt; uint64_t cyc_ref_timestamp; + uint64_t first_timestamp; + uint64_t last_reliable_timestamp; + uint64_t vmcs; + uint64_t print_once; + uint64_t last_ctc; uint32_t last_mtc; uint32_t tsc_ctc_ratio_n; uint32_t tsc_ctc_ratio_d; @@ -198,6 +223,7 @@ struct intel_pt_decoder { uint64_t timestamp_insn_cnt; uint64_t sample_insn_cnt; uint64_t stuck_ip; + int max_loops; int no_progress; int stuck_ip_prd; int stuck_ip_cnt; @@ -217,6 +243,31 @@ static uint64_t intel_pt_lower_power_of_2(uint64_t x) return x << i; } +__printf(1, 2) +static void p_log(const char *fmt, ...) +{ + char buf[512]; + va_list args; + + va_start(args, fmt); + vsnprintf(buf, sizeof(buf), fmt, args); + va_end(args); + + fprintf(stderr, "%s\n", buf); + intel_pt_log("%s\n", buf); +} + +static bool intel_pt_print_once(struct intel_pt_decoder *decoder, + enum intel_pt_p_once id) +{ + uint64_t bit = 1ULL << id; + + if (decoder->print_once & bit) + return false; + decoder->print_once |= bit; + return true; +} + static uint64_t intel_pt_cyc_threshold(uint64_t ctl) { if (!(ctl & INTEL_PT_CYC_ENABLE)) @@ -258,11 +309,17 @@ struct intel_pt_decoder *intel_pt_decoder_new(struct intel_pt_params *params) decoder->walk_insn = params->walk_insn; decoder->pgd_ip = params->pgd_ip; decoder->lookahead = params->lookahead; + decoder->findnew_vmcs_info = params->findnew_vmcs_info; decoder->data = params->data; decoder->return_compression = params->return_compression; decoder->branch_enable = params->branch_enable; decoder->hop = params->quick >= 1; decoder->leap = params->quick >= 2; + decoder->vm_time_correlation = params->vm_time_correlation; + decoder->vm_tm_corr_dry_run = params->vm_tm_corr_dry_run; + decoder->first_timestamp = params->first_timestamp; + decoder->last_reliable_timestamp = params->first_timestamp; + decoder->max_loops = params->max_loops ? params->max_loops : INTEL_PT_MAX_LOOPS; decoder->flags = params->flags; @@ -312,6 +369,12 @@ struct intel_pt_decoder *intel_pt_decoder_new(struct intel_pt_params *params) return decoder; } +void intel_pt_set_first_timestamp(struct intel_pt_decoder *decoder, + uint64_t first_timestamp) +{ + decoder->first_timestamp = first_timestamp; +} + static void intel_pt_pop_blk(struct intel_pt_stack *stack) { struct intel_pt_blk *blk = stack->blk; @@ -425,7 +488,7 @@ static const char *intel_pt_err_msgs[] = { [INTEL_PT_ERR_OVR] = "Overflow packet", [INTEL_PT_ERR_LOST] = "Lost trace data", [INTEL_PT_ERR_UNK] = "Unknown error!", - [INTEL_PT_ERR_NELOOP] = "Never-ending loop", + [INTEL_PT_ERR_NELOOP] = "Never-ending loop (refer perf config intel-pt.max-loops)", }; int intel_pt__strerror(int code, char *buf, size_t buflen) @@ -577,6 +640,7 @@ static int intel_pt_get_data(struct intel_pt_decoder *decoder, bool reposition) intel_pt_reposition(decoder); decoder->ref_timestamp = buffer.ref_timestamp; decoder->state.trace_nr = buffer.trace_nr; + decoder->vm_tm_corr_same_buf = false; intel_pt_log("Reference timestamp 0x%" PRIx64 "\n", decoder->ref_timestamp); return -ENOLINK; @@ -1109,7 +1173,7 @@ static int intel_pt_walk_insn(struct intel_pt_decoder *decoder, decoder->stuck_ip = decoder->state.to_ip; decoder->stuck_ip_prd = 1; decoder->stuck_ip_cnt = 1; - } else if (cnt > INTEL_PT_MAX_LOOPS || + } else if (cnt > decoder->max_loops || decoder->state.to_ip == decoder->stuck_ip) { intel_pt_log_at("ERROR: Never-ending loop", decoder->state.to_ip); @@ -1146,6 +1210,8 @@ static bool intel_pt_fup_event(struct intel_pt_decoder *decoder) decoder->set_fup_tx_flags = false; decoder->tx_flags = decoder->fup_tx_flags; decoder->state.type = INTEL_PT_TRANSACTION; + if (decoder->fup_tx_flags & INTEL_PT_ABORT_TX) + decoder->state.type |= INTEL_PT_BRANCH; decoder->state.from_ip = decoder->ip; decoder->state.to_ip = 0; decoder->state.flags = decoder->fup_tx_flags; @@ -1220,8 +1286,10 @@ static int intel_pt_walk_fup(struct intel_pt_decoder *decoder) return 0; if (err == -EAGAIN || intel_pt_fup_with_nlip(decoder, &intel_pt_insn, ip, err)) { + bool no_tip = decoder->pkt_state != INTEL_PT_STATE_FUP; + decoder->pkt_state = INTEL_PT_STATE_IN_SYNC; - if (intel_pt_fup_event(decoder)) + if (intel_pt_fup_event(decoder) && no_tip) return 0; return -EAGAIN; } @@ -1465,9 +1533,24 @@ static uint64_t intel_pt_8b_tsc(uint64_t timestamp, uint64_t ref_timestamp) return timestamp; } +/* For use only when decoder->vm_time_correlation is true */ +static bool intel_pt_time_in_range(struct intel_pt_decoder *decoder, + uint64_t timestamp) +{ + uint64_t max_timestamp = decoder->buf_timestamp; + + if (!max_timestamp) { + max_timestamp = decoder->last_reliable_timestamp + + 0x400000000ULL; + } + return timestamp >= decoder->last_reliable_timestamp && + timestamp < decoder->buf_timestamp; +} + static void intel_pt_calc_tsc_timestamp(struct intel_pt_decoder *decoder) { uint64_t timestamp; + bool bad = false; decoder->have_tma = false; @@ -1489,10 +1572,21 @@ static void intel_pt_calc_tsc_timestamp(struct intel_pt_decoder *decoder) timestamp = decoder->timestamp; } if (timestamp < decoder->timestamp) { - intel_pt_log_to("Wraparound timestamp", timestamp); - timestamp += (1ULL << 56); - decoder->tsc_timestamp = timestamp; + if (!decoder->buf_timestamp || + (timestamp + (1ULL << 56) < decoder->buf_timestamp)) { + intel_pt_log_to("Wraparound timestamp", timestamp); + timestamp += (1ULL << 56); + decoder->tsc_timestamp = timestamp; + } else { + intel_pt_log_to("Suppressing bad timestamp", timestamp); + timestamp = decoder->timestamp; + bad = true; + } } + if (decoder->vm_time_correlation && + (bad || !intel_pt_time_in_range(decoder, timestamp)) && + intel_pt_print_once(decoder, INTEL_PT_PRT_ONCE_ERANGE)) + p_log("Timestamp out of range"); decoder->timestamp = timestamp; decoder->timestamp_insn_cnt = 0; } @@ -1569,6 +1663,7 @@ static void intel_pt_calc_tma(struct intel_pt_decoder *decoder) intel_pt_mtc_cyc_cnt_upd(decoder); decoder->last_mtc = (ctc >> decoder->mtc_shift) & 0xff; + decoder->last_ctc = ctc - ctc_rem; decoder->ctc_timestamp = decoder->tsc_timestamp - fc; if (decoder->tsc_ctc_mult) { decoder->ctc_timestamp -= ctc_rem * decoder->tsc_ctc_mult; @@ -1953,6 +2048,613 @@ static int intel_pt_resample(struct intel_pt_decoder *decoder) return 0; } +struct intel_pt_vm_tsc_info { + struct intel_pt_pkt pip_packet; + struct intel_pt_pkt vmcs_packet; + struct intel_pt_pkt tma_packet; + bool tsc, pip, vmcs, tma, psbend; + uint64_t ctc_delta; + uint64_t last_ctc; + int max_lookahead; +}; + +/* Lookahead and get the PIP, VMCS and TMA packets from PSB+ */ +static int intel_pt_vm_psb_lookahead_cb(struct intel_pt_pkt_info *pkt_info) +{ + struct intel_pt_vm_tsc_info *data = pkt_info->data; + + switch (pkt_info->packet.type) { + case INTEL_PT_PAD: + case INTEL_PT_MNT: + case INTEL_PT_MODE_EXEC: + case INTEL_PT_MODE_TSX: + case INTEL_PT_MTC: + case INTEL_PT_FUP: + case INTEL_PT_CYC: + case INTEL_PT_CBR: + break; + + case INTEL_PT_TSC: + data->tsc = true; + break; + + case INTEL_PT_TMA: + data->tma_packet = pkt_info->packet; + data->tma = true; + break; + + case INTEL_PT_PIP: + data->pip_packet = pkt_info->packet; + data->pip = true; + break; + + case INTEL_PT_VMCS: + data->vmcs_packet = pkt_info->packet; + data->vmcs = true; + break; + + case INTEL_PT_PSBEND: + data->psbend = true; + return 1; + + case INTEL_PT_TIP_PGE: + case INTEL_PT_PTWRITE: + case INTEL_PT_PTWRITE_IP: + case INTEL_PT_EXSTOP: + case INTEL_PT_EXSTOP_IP: + case INTEL_PT_MWAIT: + case INTEL_PT_PWRE: + case INTEL_PT_PWRX: + case INTEL_PT_BBP: + case INTEL_PT_BIP: + case INTEL_PT_BEP: + case INTEL_PT_BEP_IP: + case INTEL_PT_OVF: + case INTEL_PT_BAD: + case INTEL_PT_TNT: + case INTEL_PT_TIP_PGD: + case INTEL_PT_TIP: + case INTEL_PT_PSB: + case INTEL_PT_TRACESTOP: + default: + return 1; + } + + return 0; +} + +struct intel_pt_ovf_fup_info { + int max_lookahead; + bool found; +}; + +/* Lookahead to detect a FUP packet after OVF */ +static int intel_pt_ovf_fup_lookahead_cb(struct intel_pt_pkt_info *pkt_info) +{ + struct intel_pt_ovf_fup_info *data = pkt_info->data; + + if (pkt_info->packet.type == INTEL_PT_CYC || + pkt_info->packet.type == INTEL_PT_MTC || + pkt_info->packet.type == INTEL_PT_TSC) + return !--(data->max_lookahead); + data->found = pkt_info->packet.type == INTEL_PT_FUP; + return 1; +} + +static bool intel_pt_ovf_fup_lookahead(struct intel_pt_decoder *decoder) +{ + struct intel_pt_ovf_fup_info data = { + .max_lookahead = 16, + .found = false, + }; + + intel_pt_pkt_lookahead(decoder, intel_pt_ovf_fup_lookahead_cb, &data); + return data.found; +} + +/* Lookahead and get the TMA packet after TSC */ +static int intel_pt_tma_lookahead_cb(struct intel_pt_pkt_info *pkt_info) +{ + struct intel_pt_vm_tsc_info *data = pkt_info->data; + + if (pkt_info->packet.type == INTEL_PT_CYC || + pkt_info->packet.type == INTEL_PT_MTC) + return !--(data->max_lookahead); + + if (pkt_info->packet.type == INTEL_PT_TMA) { + data->tma_packet = pkt_info->packet; + data->tma = true; + } + return 1; +} + +static uint64_t intel_pt_ctc_to_tsc(struct intel_pt_decoder *decoder, uint64_t ctc) +{ + if (decoder->tsc_ctc_mult) + return ctc * decoder->tsc_ctc_mult; + else + return multdiv(ctc, decoder->tsc_ctc_ratio_n, decoder->tsc_ctc_ratio_d); +} + +static uint64_t intel_pt_calc_expected_tsc(struct intel_pt_decoder *decoder, + uint32_t ctc, + uint32_t fc, + uint64_t last_ctc_timestamp, + uint64_t ctc_delta, + uint32_t last_ctc) +{ + /* Number of CTC ticks from last_ctc_timestamp to last_mtc */ + uint64_t last_mtc_ctc = last_ctc + ctc_delta; + /* + * Number of CTC ticks from there until current TMA packet. We would + * expect last_mtc_ctc to be before ctc, but the TSC packet can slip + * past an MTC, so a sign-extended value is used. + */ + uint64_t delta = (int16_t)((uint16_t)ctc - (uint16_t)last_mtc_ctc); + /* Total CTC ticks from last_ctc_timestamp to current TMA packet */ + uint64_t new_ctc_delta = ctc_delta + delta; + uint64_t expected_tsc; + + /* + * Convert CTC ticks to TSC ticks, add the starting point + * (last_ctc_timestamp) and the fast counter from the TMA packet. + */ + expected_tsc = last_ctc_timestamp + intel_pt_ctc_to_tsc(decoder, new_ctc_delta) + fc; + + if (intel_pt_enable_logging) { + intel_pt_log_x64(last_mtc_ctc); + intel_pt_log_x32(last_ctc); + intel_pt_log_x64(ctc_delta); + intel_pt_log_x64(delta); + intel_pt_log_x32(ctc); + intel_pt_log_x64(new_ctc_delta); + intel_pt_log_x64(last_ctc_timestamp); + intel_pt_log_x32(fc); + intel_pt_log_x64(intel_pt_ctc_to_tsc(decoder, new_ctc_delta)); + intel_pt_log_x64(expected_tsc); + } + + return expected_tsc; +} + +static uint64_t intel_pt_expected_tsc(struct intel_pt_decoder *decoder, + struct intel_pt_vm_tsc_info *data) +{ + uint32_t ctc = data->tma_packet.payload; + uint32_t fc = data->tma_packet.count; + + return intel_pt_calc_expected_tsc(decoder, ctc, fc, + decoder->ctc_timestamp, + data->ctc_delta, data->last_ctc); +} + +static void intel_pt_translate_vm_tsc(struct intel_pt_decoder *decoder, + struct intel_pt_vmcs_info *vmcs_info) +{ + uint64_t payload = decoder->packet.payload; + + /* VMX adds the TSC Offset, so subtract to get host TSC */ + decoder->packet.payload -= vmcs_info->tsc_offset; + /* TSC packet has only 7 bytes */ + decoder->packet.payload &= SEVEN_BYTES; + + /* + * The buffer is mmapped from the data file, so this also updates the + * data file. + */ + if (!decoder->vm_tm_corr_dry_run) + memcpy((void *)decoder->buf + 1, &decoder->packet.payload, 7); + + intel_pt_log("Translated VM TSC %#" PRIx64 " -> %#" PRIx64 + " VMCS %#" PRIx64 " TSC Offset %#" PRIx64 "\n", + payload, decoder->packet.payload, vmcs_info->vmcs, + vmcs_info->tsc_offset); +} + +static void intel_pt_translate_vm_tsc_offset(struct intel_pt_decoder *decoder, + uint64_t tsc_offset) +{ + struct intel_pt_vmcs_info vmcs_info = { + .vmcs = NO_VMCS, + .tsc_offset = tsc_offset + }; + + intel_pt_translate_vm_tsc(decoder, &vmcs_info); +} + +static inline bool in_vm(uint64_t pip_payload) +{ + return pip_payload & 1; +} + +static inline bool pip_in_vm(struct intel_pt_pkt *pip_packet) +{ + return pip_packet->payload & 1; +} + +static void intel_pt_print_vmcs_info(struct intel_pt_vmcs_info *vmcs_info) +{ + p_log("VMCS: %#" PRIx64 " TSC Offset %#" PRIx64, + vmcs_info->vmcs, vmcs_info->tsc_offset); +} + +static void intel_pt_vm_tm_corr_psb(struct intel_pt_decoder *decoder, + struct intel_pt_vm_tsc_info *data) +{ + memset(data, 0, sizeof(*data)); + data->ctc_delta = decoder->ctc_delta; + data->last_ctc = decoder->last_ctc; + intel_pt_pkt_lookahead(decoder, intel_pt_vm_psb_lookahead_cb, data); + if (data->tsc && !data->psbend) + p_log("ERROR: PSB without PSBEND"); + decoder->in_psb = data->psbend; +} + +static void intel_pt_vm_tm_corr_first_tsc(struct intel_pt_decoder *decoder, + struct intel_pt_vm_tsc_info *data, + struct intel_pt_vmcs_info *vmcs_info, + uint64_t host_tsc) +{ + if (!decoder->in_psb) { + /* Can't happen */ + p_log("ERROR: First TSC is not in PSB+"); + } + + if (data->pip) { + if (pip_in_vm(&data->pip_packet)) { /* Guest */ + if (vmcs_info && vmcs_info->tsc_offset) { + intel_pt_translate_vm_tsc(decoder, vmcs_info); + decoder->vm_tm_corr_reliable = true; + } else { + p_log("ERROR: First TSC, unknown TSC Offset"); + } + } else { /* Host */ + decoder->vm_tm_corr_reliable = true; + } + } else { /* Host or Guest */ + decoder->vm_tm_corr_reliable = false; + if (intel_pt_time_in_range(decoder, host_tsc)) { + /* Assume Host */ + } else { + /* Assume Guest */ + if (vmcs_info && vmcs_info->tsc_offset) + intel_pt_translate_vm_tsc(decoder, vmcs_info); + else + p_log("ERROR: First TSC, no PIP, unknown TSC Offset"); + } + } +} + +static void intel_pt_vm_tm_corr_tsc(struct intel_pt_decoder *decoder, + struct intel_pt_vm_tsc_info *data) +{ + struct intel_pt_vmcs_info *vmcs_info; + uint64_t tsc_offset = 0; + uint64_t vmcs; + bool reliable = true; + uint64_t expected_tsc; + uint64_t host_tsc; + uint64_t ref_timestamp; + + bool assign = false; + bool assign_reliable = false; + + /* Already have 'data' for the in_psb case */ + if (!decoder->in_psb) { + memset(data, 0, sizeof(*data)); + data->ctc_delta = decoder->ctc_delta; + data->last_ctc = decoder->last_ctc; + data->max_lookahead = 16; + intel_pt_pkt_lookahead(decoder, intel_pt_tma_lookahead_cb, data); + if (decoder->pge) { + data->pip = true; + data->pip_packet.payload = decoder->pip_payload; + } + } + + /* Calculations depend on having TMA packets */ + if (!data->tma) { + p_log("ERROR: TSC without TMA"); + return; + } + + vmcs = data->vmcs ? data->vmcs_packet.payload : decoder->vmcs; + if (vmcs == NO_VMCS) + vmcs = 0; + + vmcs_info = decoder->findnew_vmcs_info(decoder->data, vmcs); + + ref_timestamp = decoder->timestamp ? decoder->timestamp : decoder->buf_timestamp; + host_tsc = intel_pt_8b_tsc(decoder->packet.payload, ref_timestamp); + + if (!decoder->ctc_timestamp) { + intel_pt_vm_tm_corr_first_tsc(decoder, data, vmcs_info, host_tsc); + return; + } + + expected_tsc = intel_pt_expected_tsc(decoder, data); + + tsc_offset = host_tsc - expected_tsc; + + /* Determine if TSC is from Host or Guest */ + if (data->pip) { + if (pip_in_vm(&data->pip_packet)) { /* Guest */ + if (!vmcs_info) { + /* PIP NR=1 without VMCS cannot happen */ + p_log("ERROR: Missing VMCS"); + intel_pt_translate_vm_tsc_offset(decoder, tsc_offset); + decoder->vm_tm_corr_reliable = false; + return; + } + } else { /* Host */ + decoder->last_reliable_timestamp = host_tsc; + decoder->vm_tm_corr_reliable = true; + return; + } + } else { /* Host or Guest */ + reliable = false; /* Host/Guest is a guess, so not reliable */ + if (decoder->in_psb) { + if (!tsc_offset) + return; /* Zero TSC Offset, assume Host */ + /* + * TSC packet has only 7 bytes of TSC. We have no + * information about the Guest's 8th byte, but it + * doesn't matter because we only need 7 bytes. + * Here, since the 8th byte is unreliable and + * irrelevant, compare only 7 byes. + */ + if (vmcs_info && + (tsc_offset & SEVEN_BYTES) == + (vmcs_info->tsc_offset & SEVEN_BYTES)) { + /* Same TSC Offset as last VMCS, assume Guest */ + goto guest; + } + } + /* + * Check if the host_tsc is within the expected range. + * Note, we could narrow the range more by looking ahead for + * the next host TSC in the same buffer, but we don't bother to + * do that because this is probably good enough. + */ + if (host_tsc >= expected_tsc && intel_pt_time_in_range(decoder, host_tsc)) { + /* Within expected range for Host TSC, assume Host */ + decoder->vm_tm_corr_reliable = false; + return; + } + } + +guest: /* Assuming Guest */ + + /* Determine whether to assign TSC Offset */ + if (vmcs_info && vmcs_info->vmcs) { + if (vmcs_info->tsc_offset && vmcs_info->reliable) { + assign = false; + } else if (decoder->in_psb && data->pip && decoder->vm_tm_corr_reliable && + decoder->vm_tm_corr_continuous && decoder->vm_tm_corr_same_buf) { + /* Continuous tracing, TSC in a PSB is not a time loss */ + assign = true; + assign_reliable = true; + } else if (decoder->in_psb && data->pip && decoder->vm_tm_corr_same_buf) { + /* + * Unlikely to be a time loss TSC in a PSB which is not + * at the start of a buffer. + */ + assign = true; + assign_reliable = false; + } + } + + /* Record VMCS TSC Offset */ + if (assign && (vmcs_info->tsc_offset != tsc_offset || + vmcs_info->reliable != assign_reliable)) { + bool print = vmcs_info->tsc_offset != tsc_offset; + + vmcs_info->tsc_offset = tsc_offset; + vmcs_info->reliable = assign_reliable; + if (print) + intel_pt_print_vmcs_info(vmcs_info); + } + + /* Determine what TSC Offset to use */ + if (vmcs_info && vmcs_info->tsc_offset) { + if (!vmcs_info->reliable) + reliable = false; + intel_pt_translate_vm_tsc(decoder, vmcs_info); + } else { + reliable = false; + if (vmcs_info) { + if (!vmcs_info->error_printed) { + p_log("ERROR: Unknown TSC Offset for VMCS %#" PRIx64, + vmcs_info->vmcs); + vmcs_info->error_printed = true; + } + } else { + if (intel_pt_print_once(decoder, INTEL_PT_PRT_ONCE_UNK_VMCS)) + p_log("ERROR: Unknown VMCS"); + } + intel_pt_translate_vm_tsc_offset(decoder, tsc_offset); + } + + decoder->vm_tm_corr_reliable = reliable; +} + +static void intel_pt_vm_tm_corr_pebs_tsc(struct intel_pt_decoder *decoder) +{ + uint64_t host_tsc = decoder->packet.payload; + uint64_t guest_tsc = decoder->packet.payload; + struct intel_pt_vmcs_info *vmcs_info; + uint64_t vmcs; + + vmcs = decoder->vmcs; + if (vmcs == NO_VMCS) + vmcs = 0; + + vmcs_info = decoder->findnew_vmcs_info(decoder->data, vmcs); + + if (decoder->pge) { + if (in_vm(decoder->pip_payload)) { /* Guest */ + if (!vmcs_info) { + /* PIP NR=1 without VMCS cannot happen */ + p_log("ERROR: Missing VMCS"); + } + } else { /* Host */ + return; + } + } else { /* Host or Guest */ + if (intel_pt_time_in_range(decoder, host_tsc)) { + /* Within expected range for Host TSC, assume Host */ + return; + } + } + + if (vmcs_info) { + /* Translate Guest TSC to Host TSC */ + host_tsc = ((guest_tsc & SEVEN_BYTES) - vmcs_info->tsc_offset) & SEVEN_BYTES; + host_tsc = intel_pt_8b_tsc(host_tsc, decoder->timestamp); + intel_pt_log("Translated VM TSC %#" PRIx64 " -> %#" PRIx64 + " VMCS %#" PRIx64 " TSC Offset %#" PRIx64 "\n", + guest_tsc, host_tsc, vmcs_info->vmcs, + vmcs_info->tsc_offset); + if (!intel_pt_time_in_range(decoder, host_tsc) && + intel_pt_print_once(decoder, INTEL_PT_PRT_ONCE_ERANGE)) + p_log("Timestamp out of range"); + } else { + if (intel_pt_print_once(decoder, INTEL_PT_PRT_ONCE_UNK_VMCS)) + p_log("ERROR: Unknown VMCS"); + host_tsc = decoder->timestamp; + } + + decoder->packet.payload = host_tsc; + + if (!decoder->vm_tm_corr_dry_run) + memcpy((void *)decoder->buf + 1, &host_tsc, 8); +} + +static int intel_pt_vm_time_correlation(struct intel_pt_decoder *decoder) +{ + struct intel_pt_vm_tsc_info data = { .psbend = false }; + bool pge; + int err; + + if (decoder->in_psb) + intel_pt_vm_tm_corr_psb(decoder, &data); + + while (1) { + err = intel_pt_get_next_packet(decoder); + if (err == -ENOLINK) + continue; + if (err) + break; + + switch (decoder->packet.type) { + case INTEL_PT_TIP_PGD: + decoder->pge = false; + decoder->vm_tm_corr_continuous = false; + break; + + case INTEL_PT_TNT: + case INTEL_PT_TIP: + case INTEL_PT_TIP_PGE: + decoder->pge = true; + break; + + case INTEL_PT_OVF: + decoder->in_psb = false; + pge = decoder->pge; + decoder->pge = intel_pt_ovf_fup_lookahead(decoder); + if (pge != decoder->pge) + intel_pt_log("Surprising PGE change in OVF!"); + if (!decoder->pge) + decoder->vm_tm_corr_continuous = false; + break; + + case INTEL_PT_FUP: + if (decoder->in_psb) + decoder->pge = true; + break; + + case INTEL_PT_TRACESTOP: + decoder->pge = false; + decoder->vm_tm_corr_continuous = false; + decoder->have_tma = false; + break; + + case INTEL_PT_PSB: + intel_pt_vm_tm_corr_psb(decoder, &data); + break; + + case INTEL_PT_PIP: + decoder->pip_payload = decoder->packet.payload; + break; + + case INTEL_PT_MTC: + intel_pt_calc_mtc_timestamp(decoder); + break; + + case INTEL_PT_TSC: + intel_pt_vm_tm_corr_tsc(decoder, &data); + intel_pt_calc_tsc_timestamp(decoder); + decoder->vm_tm_corr_same_buf = true; + decoder->vm_tm_corr_continuous = decoder->pge; + break; + + case INTEL_PT_TMA: + intel_pt_calc_tma(decoder); + break; + + case INTEL_PT_CYC: + intel_pt_calc_cyc_timestamp(decoder); + break; + + case INTEL_PT_CBR: + intel_pt_calc_cbr(decoder); + break; + + case INTEL_PT_PSBEND: + decoder->in_psb = false; + data.psbend = false; + break; + + case INTEL_PT_VMCS: + if (decoder->packet.payload != NO_VMCS) + decoder->vmcs = decoder->packet.payload; + break; + + case INTEL_PT_BBP: + decoder->blk_type = decoder->packet.payload; + break; + + case INTEL_PT_BIP: + if (decoder->blk_type == INTEL_PT_PEBS_BASIC && + decoder->packet.count == 2) + intel_pt_vm_tm_corr_pebs_tsc(decoder); + break; + + case INTEL_PT_BEP: + case INTEL_PT_BEP_IP: + decoder->blk_type = 0; + break; + + case INTEL_PT_MODE_EXEC: + case INTEL_PT_MODE_TSX: + case INTEL_PT_MNT: + case INTEL_PT_PAD: + case INTEL_PT_PTWRITE_IP: + case INTEL_PT_PTWRITE: + case INTEL_PT_MWAIT: + case INTEL_PT_PWRE: + case INTEL_PT_EXSTOP_IP: + case INTEL_PT_EXSTOP: + case INTEL_PT_PWRX: + case INTEL_PT_BAD: /* Does not happen */ + default: + break; + } + } + + return err; +} + #define HOP_PROCESS 0 #define HOP_IGNORE 1 #define HOP_RETURN 2 @@ -2894,6 +3596,15 @@ static int intel_pt_sync(struct intel_pt_decoder *decoder) if (err) return err; + if (decoder->vm_time_correlation) { + decoder->in_psb = true; + if (!decoder->timestamp) + decoder->timestamp = 1; + decoder->state.type = 0; + decoder->pkt_state = INTEL_PT_STATE_VM_TIME_CORRELATION; + return 0; + } + decoder->have_last_ip = true; decoder->pkt_state = INTEL_PT_STATE_NO_IP; @@ -2981,6 +3692,9 @@ const struct intel_pt_state *intel_pt_decode(struct intel_pt_decoder *decoder) case INTEL_PT_STATE_RESAMPLE: err = intel_pt_resample(decoder); break; + case INTEL_PT_STATE_VM_TIME_CORRELATION: + err = intel_pt_vm_time_correlation(decoder); + break; default: err = intel_pt_bug(decoder); break; @@ -3227,6 +3941,7 @@ static unsigned char *adj_for_padding(unsigned char *buf_b, * @len_b: size of second buffer * @consecutive: returns true if there is data in buf_b that is consecutive * to buf_a + * @ooo_tsc: out-of-order TSC due to VM TSC offset / scaling * * If the trace contains TSC we can look at the last TSC of @buf_a and the * first TSC of @buf_b in order to determine if the buffers overlap, and then @@ -3239,7 +3954,8 @@ static unsigned char *adj_for_padding(unsigned char *buf_b, static unsigned char *intel_pt_find_overlap_tsc(unsigned char *buf_a, size_t len_a, unsigned char *buf_b, - size_t len_b, bool *consecutive) + size_t len_b, bool *consecutive, + bool ooo_tsc) { uint64_t tsc_a, tsc_b; unsigned char *p; @@ -3274,7 +3990,7 @@ static unsigned char *intel_pt_find_overlap_tsc(unsigned char *buf_a, start = buf_b + len_b - (rem_b - rem_a); return adj_for_padding(start, buf_a, len_a); } - if (cmp < 0) + if (cmp < 0 && !ooo_tsc) return buf_b; /* tsc_a < tsc_b => no overlap */ } @@ -3292,6 +4008,7 @@ static unsigned char *intel_pt_find_overlap_tsc(unsigned char *buf_a, * @have_tsc: can use TSC packets to detect overlap * @consecutive: returns true if there is data in buf_b that is consecutive * to buf_a + * @ooo_tsc: out-of-order TSC due to VM TSC offset / scaling * * When trace samples or snapshots are recorded there is the possibility that * the data overlaps. Note that, for the purposes of decoding, data is only @@ -3302,7 +4019,8 @@ static unsigned char *intel_pt_find_overlap_tsc(unsigned char *buf_a, */ unsigned char *intel_pt_find_overlap(unsigned char *buf_a, size_t len_a, unsigned char *buf_b, size_t len_b, - bool have_tsc, bool *consecutive) + bool have_tsc, bool *consecutive, + bool ooo_tsc) { unsigned char *found; @@ -3315,7 +4033,7 @@ unsigned char *intel_pt_find_overlap(unsigned char *buf_a, size_t len_a, if (have_tsc) { found = intel_pt_find_overlap_tsc(buf_a, len_a, buf_b, len_b, - consecutive); + consecutive, ooo_tsc); if (found) return found; } diff --git a/tools/perf/util/intel-pt-decoder/intel-pt-decoder.h b/tools/perf/util/intel-pt-decoder/intel-pt-decoder.h index d9e62a7f6f0e..4b5e79fcf557 100644 --- a/tools/perf/util/intel-pt-decoder/intel-pt-decoder.h +++ b/tools/perf/util/intel-pt-decoder/intel-pt-decoder.h @@ -11,6 +11,8 @@ #include <stddef.h> #include <stdbool.h> +#include <linux/rbtree.h> + #include "intel-pt-insn-decoder.h" #define INTEL_PT_IN_TX (1 << 0) @@ -199,6 +201,14 @@ struct intel_pt_blk_items { bool is_32_bit; }; +struct intel_pt_vmcs_info { + struct rb_node rb_node; + uint64_t vmcs; + uint64_t tsc_offset; + bool reliable; + bool error_printed; +}; + struct intel_pt_state { enum intel_pt_sample_type type; bool from_nr; @@ -244,9 +254,13 @@ struct intel_pt_params { uint64_t max_insn_cnt, void *data); bool (*pgd_ip)(uint64_t ip, void *data); int (*lookahead)(void *data, intel_pt_lookahead_cb_t cb, void *cb_data); + struct intel_pt_vmcs_info *(*findnew_vmcs_info)(void *data, uint64_t vmcs); void *data; bool return_compression; bool branch_enable; + bool vm_time_correlation; + bool vm_tm_corr_dry_run; + uint64_t first_timestamp; uint64_t ctl; uint64_t period; enum intel_pt_period_type period_type; @@ -256,6 +270,7 @@ struct intel_pt_params { uint32_t tsc_ctc_ratio_d; enum intel_pt_param_flags flags; unsigned int quick; + int max_loops; }; struct intel_pt_decoder; @@ -269,8 +284,12 @@ int intel_pt_fast_forward(struct intel_pt_decoder *decoder, uint64_t timestamp); unsigned char *intel_pt_find_overlap(unsigned char *buf_a, size_t len_a, unsigned char *buf_b, size_t len_b, - bool have_tsc, bool *consecutive); + bool have_tsc, bool *consecutive, + bool ooo_tsc); int intel_pt__strerror(int code, char *buf, size_t buflen); +void intel_pt_set_first_timestamp(struct intel_pt_decoder *decoder, + uint64_t first_timestamp); + #endif diff --git a/tools/perf/util/intel-pt-decoder/intel-pt-log.h b/tools/perf/util/intel-pt-decoder/intel-pt-log.h index 388661f89c44..d900aab24b21 100644 --- a/tools/perf/util/intel-pt-decoder/intel-pt-log.h +++ b/tools/perf/util/intel-pt-decoder/intel-pt-log.h @@ -67,4 +67,9 @@ static inline void intel_pt_log_to(const char *msg, uint64_t u) intel_pt_log("%s to " x64_fmt "\n", msg, u); } +#define intel_pt_log_var(var, fmt) intel_pt_log("%s: " #var " " fmt "\n", __func__, var) + +#define intel_pt_log_x32(var) intel_pt_log_var(var, "%#x") +#define intel_pt_log_x64(var) intel_pt_log_var(var, "%#" PRIx64) + #endif diff --git a/tools/perf/util/intel-pt.c b/tools/perf/util/intel-pt.c index 8658d42ce57a..6f852b305e92 100644 --- a/tools/perf/util/intel-pt.c +++ b/tools/perf/util/intel-pt.c @@ -78,6 +78,7 @@ struct intel_pt { u64 kernel_start; u64 switch_ip; u64 ptss_ip; + u64 first_timestamp; struct perf_tsc_conversion tc; bool cap_user_time_zero; @@ -122,6 +123,7 @@ struct intel_pt { u64 noretcomp_bit; unsigned max_non_turbo_ratio; unsigned cbr2khz; + int max_loops; unsigned long num_events; @@ -133,6 +135,9 @@ struct intel_pt { struct ip_callchain *chain; struct branch_stack *br_stack; + + u64 dflt_tsc_offset; + struct rb_root vmcs_info; }; enum switch_state { @@ -271,6 +276,65 @@ static bool intel_pt_log_events(struct intel_pt *pt, u64 tm) return !n || !perf_time__ranges_skip_sample(range, n, tm); } +static struct intel_pt_vmcs_info *intel_pt_findnew_vmcs(struct rb_root *rb_root, + u64 vmcs, + u64 dflt_tsc_offset) +{ + struct rb_node **p = &rb_root->rb_node; + struct rb_node *parent = NULL; + struct intel_pt_vmcs_info *v; + + while (*p) { + parent = *p; + v = rb_entry(parent, struct intel_pt_vmcs_info, rb_node); + + if (v->vmcs == vmcs) + return v; + + if (vmcs < v->vmcs) + p = &(*p)->rb_left; + else + p = &(*p)->rb_right; + } + + v = zalloc(sizeof(*v)); + if (v) { + v->vmcs = vmcs; + v->tsc_offset = dflt_tsc_offset; + v->reliable = dflt_tsc_offset; + + rb_link_node(&v->rb_node, parent, p); + rb_insert_color(&v->rb_node, rb_root); + } + + return v; +} + +static struct intel_pt_vmcs_info *intel_pt_findnew_vmcs_info(void *data, uint64_t vmcs) +{ + struct intel_pt_queue *ptq = data; + struct intel_pt *pt = ptq->pt; + + if (!vmcs && !pt->dflt_tsc_offset) + return NULL; + + return intel_pt_findnew_vmcs(&pt->vmcs_info, vmcs, pt->dflt_tsc_offset); +} + +static void intel_pt_free_vmcs_info(struct intel_pt *pt) +{ + struct intel_pt_vmcs_info *v; + struct rb_node *n; + + n = rb_first(&pt->vmcs_info); + while (n) { + v = rb_entry(n, struct intel_pt_vmcs_info, rb_node); + n = rb_next(n); + rb_erase(&v->rb_node, &pt->vmcs_info); + free(v); + } +} + static int intel_pt_do_fix_overlap(struct intel_pt *pt, struct auxtrace_buffer *a, struct auxtrace_buffer *b) { @@ -278,9 +342,17 @@ static int intel_pt_do_fix_overlap(struct intel_pt *pt, struct auxtrace_buffer * void *start; start = intel_pt_find_overlap(a->data, a->size, b->data, b->size, - pt->have_tsc, &consecutive); + pt->have_tsc, &consecutive, + pt->synth_opts.vm_time_correlation); if (!start) return -EINVAL; + /* + * In the case of vm_time_correlation, the overlap might contain TSC + * packets that will not be fixed, and that will then no longer work for + * overlap detection. Avoid that by zeroing out the overlap. + */ + if (pt->synth_opts.vm_time_correlation) + memset(b->data, 0, start - b->data); b->use_size = b->data + b->size - start; b->use_data = start; if (b->use_size && consecutive) @@ -707,8 +779,10 @@ static int intel_pt_walk_next_insn(struct intel_pt_insn *intel_pt_insn, *ip += intel_pt_insn->length; - if (to_ip && *ip == to_ip) + if (to_ip && *ip == to_ip) { + intel_pt_insn->length = 0; goto out_no_cache; + } if (*ip >= al.map->end) break; @@ -899,7 +973,7 @@ static bool intel_pt_timeless_decoding(struct intel_pt *pt) bool timeless_decoding = true; u64 config; - if (!pt->tsc_bit || !pt->cap_user_time_zero) + if (!pt->tsc_bit || !pt->cap_user_time_zero || pt->synth_opts.timeless_decoding) return true; evlist__for_each_entry(pt->session->evlist, evsel) { @@ -947,6 +1021,19 @@ static bool intel_pt_have_tsc(struct intel_pt *pt) return have_tsc; } +static bool intel_pt_have_mtc(struct intel_pt *pt) +{ + struct evsel *evsel; + u64 config; + + evlist__for_each_entry(pt->session->evlist, evsel) { + if (intel_pt_get_config(pt, &evsel->core.attr, &config) && + (config & pt->mtc_bit)) + return true; + } + return false; +} + static bool intel_pt_sampling_mode(struct intel_pt *pt) { struct evsel *evsel; @@ -1101,6 +1188,7 @@ static struct intel_pt_queue *intel_pt_alloc_queue(struct intel_pt *pt, params.get_trace = intel_pt_get_trace; params.walk_insn = intel_pt_walk_next_insn; params.lookahead = intel_pt_lookahead; + params.findnew_vmcs_info = intel_pt_findnew_vmcs_info; params.data = ptq; params.return_compression = intel_pt_return_compression(pt); params.branch_enable = intel_pt_branch_enable(pt); @@ -1110,6 +1198,10 @@ static struct intel_pt_queue *intel_pt_alloc_queue(struct intel_pt *pt, params.tsc_ctc_ratio_n = pt->tsc_ctc_ratio_n; params.tsc_ctc_ratio_d = pt->tsc_ctc_ratio_d; params.quick = pt->synth_opts.quick; + params.vm_time_correlation = pt->synth_opts.vm_time_correlation; + params.vm_tm_corr_dry_run = pt->synth_opts.vm_tm_corr_dry_run; + params.first_timestamp = pt->first_timestamp; + params.max_loops = pt->max_loops; if (pt->filts.cnt > 0) params.pgd_ip = intel_pt_pgd_ip; @@ -1174,6 +1266,21 @@ static void intel_pt_free_queue(void *priv) free(ptq); } +static void intel_pt_first_timestamp(struct intel_pt *pt, u64 timestamp) +{ + unsigned int i; + + pt->first_timestamp = timestamp; + + for (i = 0; i < pt->queues.nr_queues; i++) { + struct auxtrace_queue *queue = &pt->queues.queue_array[i]; + struct intel_pt_queue *ptq = queue->priv; + + if (ptq && ptq->decoder) + intel_pt_set_first_timestamp(ptq->decoder, timestamp); + } +} + static void intel_pt_set_pid_tid_cpu(struct intel_pt *pt, struct auxtrace_queue *queue) { @@ -1198,6 +1305,7 @@ static void intel_pt_set_pid_tid_cpu(struct intel_pt *pt, static void intel_pt_sample_flags(struct intel_pt_queue *ptq) { + ptq->insn_len = 0; if (ptq->state->flags & INTEL_PT_ABORT_TX) { ptq->flags = PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_TX_ABORT; } else if (ptq->state->flags & INTEL_PT_ASYNC) { @@ -1211,7 +1319,6 @@ static void intel_pt_sample_flags(struct intel_pt_queue *ptq) ptq->flags = PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_CALL | PERF_IP_FLAG_ASYNC | PERF_IP_FLAG_INTERRUPT; - ptq->insn_len = 0; } else { if (ptq->state->from_ip) ptq->flags = intel_pt_insn_type(ptq->state->insn_op); @@ -2377,7 +2484,7 @@ static int intel_pt_run_decoder(struct intel_pt_queue *ptq, u64 *timestamp) if (pt->per_cpu_mmaps && (pt->have_sched_switch == 1 || pt->have_sched_switch == 3) && !pt->timeless_decoding && intel_pt_tracing_kernel(pt) && - !pt->sampling_mode) { + !pt->sampling_mode && !pt->synth_opts.vm_time_correlation) { pt->switch_ip = intel_pt_switch_ip(pt, &pt->ptss_ip); if (pt->switch_ip) { intel_pt_log("switch_ip: %"PRIx64" ptss_ip: %"PRIx64"\n", @@ -2876,6 +2983,8 @@ static int intel_pt_process_event(struct perf_session *session, sample->time); } } else if (timestamp) { + if (!pt->first_timestamp) + intel_pt_first_timestamp(pt, timestamp); err = intel_pt_process_queues(pt, timestamp); } if (err) @@ -2962,6 +3071,7 @@ static void intel_pt_free(struct perf_session *session) auxtrace_heap__free(&pt->heap); intel_pt_free_events(session); session->auxtrace = NULL; + intel_pt_free_vmcs_info(pt); thread__put(pt->unknown_thread); addr_filters__exit(&pt->filts); zfree(&pt->chain); @@ -3323,6 +3433,9 @@ static int intel_pt_perf_config(const char *var, const char *value, void *data) if (!strcmp(var, "intel-pt.mispred-all")) pt->mispred_all = perf_config_bool(var, value); + if (!strcmp(var, "intel-pt.max-loops")) + perf_config_int(&pt->max_loops, var, value); + return 0; } @@ -3405,6 +3518,65 @@ static int intel_pt_setup_time_ranges(struct intel_pt *pt, return 0; } +static int intel_pt_parse_vm_tm_corr_arg(struct intel_pt *pt, char **args) +{ + struct intel_pt_vmcs_info *vmcs_info; + u64 tsc_offset, vmcs; + char *p = *args; + + errno = 0; + + p = skip_spaces(p); + if (!*p) + return 1; + + tsc_offset = strtoull(p, &p, 0); + if (errno) + return -errno; + p = skip_spaces(p); + if (*p != ':') { + pt->dflt_tsc_offset = tsc_offset; + *args = p; + return 0; + } + while (1) { + vmcs = strtoull(p, &p, 0); + if (errno) + return -errno; + if (!vmcs) + return -EINVAL; + vmcs_info = intel_pt_findnew_vmcs(&pt->vmcs_info, vmcs, tsc_offset); + if (!vmcs_info) + return -ENOMEM; + p = skip_spaces(p); + if (*p != ',') + break; + p += 1; + } + *args = p; + return 0; +} + +static int intel_pt_parse_vm_tm_corr_args(struct intel_pt *pt) +{ + char *args = pt->synth_opts.vm_tm_corr_args; + int ret; + + if (!args) + return 0; + + do { + ret = intel_pt_parse_vm_tm_corr_arg(pt, &args); + } while (!ret); + + if (ret < 0) { + pr_err("Failed to parse VM Time Correlation options\n"); + return ret; + } + + return 0; +} + static const char * const intel_pt_info_fmts[] = { [INTEL_PT_PMU_TYPE] = " PMU Type %"PRId64"\n", [INTEL_PT_TIME_SHIFT] = " Time Shift %"PRIu64"\n", @@ -3467,6 +3639,8 @@ int intel_pt_process_auxtrace_info(union perf_event *event, if (!pt) return -ENOMEM; + pt->vmcs_info = RB_ROOT; + addr_filters__init(&pt->filts); err = perf_config(intel_pt_perf_config, pt); @@ -3479,6 +3653,20 @@ int intel_pt_process_auxtrace_info(union perf_event *event, intel_pt_log_set_name(INTEL_PT_PMU_NAME); + if (session->itrace_synth_opts->set) { + pt->synth_opts = *session->itrace_synth_opts; + } else { + struct itrace_synth_opts *opts = session->itrace_synth_opts; + + itrace_synth_opts__set_default(&pt->synth_opts, opts->default_no_sample); + if (!opts->default_no_sample && !opts->inject) { + pt->synth_opts.branches = false; + pt->synth_opts.callchain = true; + pt->synth_opts.add_callchain = true; + } + pt->synth_opts.thread_stack = opts->thread_stack; + } + pt->session = session; pt->machine = &session->machines.host; /* No kvm support */ pt->auxtrace_type = auxtrace_info->type; @@ -3560,6 +3748,28 @@ int intel_pt_process_auxtrace_info(union perf_event *event, pt->sampling_mode = intel_pt_sampling_mode(pt); pt->est_tsc = !pt->timeless_decoding; + if (pt->synth_opts.vm_time_correlation) { + if (pt->timeless_decoding) { + pr_err("Intel PT has no time information for VM Time Correlation\n"); + err = -EINVAL; + goto err_free_queues; + } + if (session->itrace_synth_opts->ptime_range) { + pr_err("Time ranges cannot be specified with VM Time Correlation\n"); + err = -EINVAL; + goto err_free_queues; + } + /* Currently TSC Offset is calculated using MTC packets */ + if (!intel_pt_have_mtc(pt)) { + pr_err("MTC packets must have been enabled for VM Time Correlation\n"); + err = -EINVAL; + goto err_free_queues; + } + err = intel_pt_parse_vm_tm_corr_args(pt); + if (err) + goto err_free_queues; + } + pt->unknown_thread = thread__new(999999999, 999999999); if (!pt->unknown_thread) { err = -ENOMEM; @@ -3609,21 +3819,6 @@ int intel_pt_process_auxtrace_info(union perf_event *event, goto err_delete_thread; } - if (session->itrace_synth_opts->set) { - pt->synth_opts = *session->itrace_synth_opts; - } else { - itrace_synth_opts__set_default(&pt->synth_opts, - session->itrace_synth_opts->default_no_sample); - if (!session->itrace_synth_opts->default_no_sample && - !session->itrace_synth_opts->inject) { - pt->synth_opts.branches = false; - pt->synth_opts.callchain = true; - pt->synth_opts.add_callchain = true; - } - pt->synth_opts.thread_stack = - session->itrace_synth_opts->thread_stack; - } - if (pt->synth_opts.log) intel_pt_log_enable(); diff --git a/tools/perf/util/llvm-utils.c b/tools/perf/util/llvm-utils.c index 3ceaf7ef3301..cbd9b268f168 100644 --- a/tools/perf/util/llvm-utils.c +++ b/tools/perf/util/llvm-utils.c @@ -504,6 +504,7 @@ int llvm__compile_bpf(const char *path, void **p_obj_buf, goto errout; } + err = -ENOMEM; if (asprintf(&pipe_template, "%s -emit-llvm | %s -march=bpf %s -filetype=obj -o -", template, llc_path, opts) < 0) { pr_err("ERROR:\tnot enough memory to setup command line\n"); @@ -524,6 +525,7 @@ int llvm__compile_bpf(const char *path, void **p_obj_buf, pr_debug("llvm compiling command template: %s\n", template); + err = -ENOMEM; if (asprintf(&command_echo, "echo -n \"%s\"", template) < 0) goto errout; diff --git a/tools/perf/util/lzma.c b/tools/perf/util/lzma.c index 39062df02629..51424cdc3b68 100644 --- a/tools/perf/util/lzma.c +++ b/tools/perf/util/lzma.c @@ -69,7 +69,7 @@ int lzma_decompress_to_file(const char *input, int output_fd) if (ferror(infile)) { pr_err("lzma: read error: %s\n", strerror(errno)); - goto err_fclose; + goto err_lzma_end; } if (feof(infile)) @@ -83,7 +83,7 @@ int lzma_decompress_to_file(const char *input, int output_fd) if (writen(output_fd, buf_out, write_size) != write_size) { pr_err("lzma: write error: %s\n", strerror(errno)); - goto err_fclose; + goto err_lzma_end; } strm.next_out = buf_out; @@ -95,11 +95,13 @@ int lzma_decompress_to_file(const char *input, int output_fd) break; pr_err("lzma: failed %s\n", lzma_strerror(ret)); - goto err_fclose; + goto err_lzma_end; } } err = 0; +err_lzma_end: + lzma_end(&strm); err_fclose: fclose(infile); return err; diff --git a/tools/perf/util/machine.c b/tools/perf/util/machine.c index 3ff4936a15a4..da19be7da284 100644 --- a/tools/perf/util/machine.c +++ b/tools/perf/util/machine.c @@ -776,10 +776,10 @@ static int machine__process_ksymbol_register(struct machine *machine, if (dso) { dso->kernel = DSO_SPACE__KERNEL; map = map__new2(0, dso); + dso__put(dso); } if (!dso || !map) { - dso__put(dso); return -ENOMEM; } @@ -792,6 +792,7 @@ static int machine__process_ksymbol_register(struct machine *machine, map->start = event->ksymbol.addr; map->end = map->start + event->ksymbol.len; maps__insert(&machine->kmaps, map); + map__put(map); dso__set_loaded(dso); if (is_bpf_image(event->ksymbol.name)) { diff --git a/tools/perf/util/mem-events.c b/tools/perf/util/mem-events.c index f93a852ad838..f0e75df72b80 100644 --- a/tools/perf/util/mem-events.c +++ b/tools/perf/util/mem-events.c @@ -12,6 +12,8 @@ #include "mem-events.h" #include "debug.h" #include "symbol.h" +#include "pmu.h" +#include "pmu-hybrid.h" unsigned int perf_mem_events__loads_ldlat = 30; @@ -24,8 +26,6 @@ static struct perf_mem_event perf_mem_events[PERF_MEM_EVENTS__MAX] = { }; #undef E -#undef E - static char mem_loads_name[100]; static bool mem_loads_name__init; @@ -37,7 +37,7 @@ struct perf_mem_event * __weak perf_mem_events__ptr(int i) return &perf_mem_events[i]; } -char * __weak perf_mem_events__name(int i) +char * __weak perf_mem_events__name(int i, char *pmu_name __maybe_unused) { struct perf_mem_event *e = perf_mem_events__ptr(i); @@ -100,6 +100,15 @@ int perf_mem_events__parse(const char *str) return -1; } +static bool perf_mem_event__supported(const char *mnt, char *sysfs_name) +{ + char path[PATH_MAX]; + struct stat st; + + scnprintf(path, PATH_MAX, "%s/devices/%s", mnt, sysfs_name); + return !stat(path, &st); +} + int perf_mem_events__init(void) { const char *mnt = sysfs__mount(); @@ -110,9 +119,9 @@ int perf_mem_events__init(void) return -ENOENT; for (j = 0; j < PERF_MEM_EVENTS__MAX; j++) { - char path[PATH_MAX]; struct perf_mem_event *e = perf_mem_events__ptr(j); - struct stat st; + struct perf_pmu *pmu; + char sysfs_name[100]; /* * If the event entry isn't valid, skip initialization @@ -121,11 +130,20 @@ int perf_mem_events__init(void) if (!e->tag) continue; - scnprintf(path, PATH_MAX, "%s/devices/%s", - mnt, e->sysfs_name); + if (!perf_pmu__has_hybrid()) { + scnprintf(sysfs_name, sizeof(sysfs_name), + e->sysfs_name, "cpu"); + e->supported = perf_mem_event__supported(mnt, sysfs_name); + } else { + perf_pmu__for_each_hybrid_pmu(pmu) { + scnprintf(sysfs_name, sizeof(sysfs_name), + e->sysfs_name, pmu->name); + e->supported |= perf_mem_event__supported(mnt, sysfs_name); + } + } - if (!stat(path, &st)) - e->supported = found = true; + if (e->supported) + found = true; } return found ? 0 : -ENOENT; @@ -141,11 +159,76 @@ void perf_mem_events__list(void) fprintf(stderr, "%-13s%-*s%s\n", e->tag ?: "", verbose > 0 ? 25 : 0, - verbose > 0 ? perf_mem_events__name(j) : "", + verbose > 0 ? perf_mem_events__name(j, NULL) : "", e->supported ? ": available" : ""); } } +static void perf_mem_events__print_unsupport_hybrid(struct perf_mem_event *e, + int idx) +{ + const char *mnt = sysfs__mount(); + char sysfs_name[100]; + struct perf_pmu *pmu; + + perf_pmu__for_each_hybrid_pmu(pmu) { + scnprintf(sysfs_name, sizeof(sysfs_name), e->sysfs_name, + pmu->name); + if (!perf_mem_event__supported(mnt, sysfs_name)) { + pr_err("failed: event '%s' not supported\n", + perf_mem_events__name(idx, pmu->name)); + } + } +} + +int perf_mem_events__record_args(const char **rec_argv, int *argv_nr, + char **rec_tmp, int *tmp_nr) +{ + int i = *argv_nr, k = 0; + struct perf_mem_event *e; + struct perf_pmu *pmu; + char *s; + + for (int j = 0; j < PERF_MEM_EVENTS__MAX; j++) { + e = perf_mem_events__ptr(j); + if (!e->record) + continue; + + if (!perf_pmu__has_hybrid()) { + if (!e->supported) { + pr_err("failed: event '%s' not supported\n", + perf_mem_events__name(j, NULL)); + return -1; + } + + rec_argv[i++] = "-e"; + rec_argv[i++] = perf_mem_events__name(j, NULL); + } else { + if (!e->supported) { + perf_mem_events__print_unsupport_hybrid(e, j); + return -1; + } + + perf_pmu__for_each_hybrid_pmu(pmu) { + rec_argv[i++] = "-e"; + s = perf_mem_events__name(j, pmu->name); + if (s) { + s = strdup(s); + if (!s) + return -1; + + rec_argv[i++] = s; + rec_tmp[k++] = s; + } + } + } + } + + *argv_nr = i; + *tmp_nr = k; + return 0; +} + static const char * const tlb_access[] = { "N/A", "HIT", diff --git a/tools/perf/util/mem-events.h b/tools/perf/util/mem-events.h index cacdebd65b8a..916242f8020a 100644 --- a/tools/perf/util/mem-events.h +++ b/tools/perf/util/mem-events.h @@ -38,11 +38,13 @@ extern unsigned int perf_mem_events__loads_ldlat; int perf_mem_events__parse(const char *str); int perf_mem_events__init(void); -char *perf_mem_events__name(int i); +char *perf_mem_events__name(int i, char *pmu_name); struct perf_mem_event *perf_mem_events__ptr(int i); bool is_mem_loads_aux_event(struct evsel *leader); void perf_mem_events__list(void); +int perf_mem_events__record_args(const char **rec_argv, int *argv_nr, + char **rec_tmp, int *tmp_nr); int perf_mem__tlb_scnprintf(char *out, size_t sz, struct mem_info *mem_info); int perf_mem__lvl_scnprintf(char *out, size_t sz, struct mem_info *mem_info); diff --git a/tools/perf/util/metricgroup.c b/tools/perf/util/metricgroup.c index 8336dd8e8098..99d047c5ead0 100644 --- a/tools/perf/util/metricgroup.c +++ b/tools/perf/util/metricgroup.c @@ -162,10 +162,10 @@ static bool contains_event(struct evsel **metric_events, int num_events, return false; } -static bool evsel_same_pmu(struct evsel *ev1, struct evsel *ev2) +static bool evsel_same_pmu_or_none(struct evsel *ev1, struct evsel *ev2) { if (!ev1->pmu_name || !ev2->pmu_name) - return false; + return true; return !strcmp(ev1->pmu_name, ev2->pmu_name); } @@ -219,9 +219,9 @@ static struct evsel *find_evsel_group(struct evlist *perf_evlist, if (has_constraint && ev->weak_group) continue; /* Ignore event if already used and merging is disabled. */ - if (metric_no_merge && test_bit(ev->idx, evlist_used)) + if (metric_no_merge && test_bit(ev->core.idx, evlist_used)) continue; - if (!has_constraint && ev->leader != current_leader) { + if (!has_constraint && !evsel__has_leader(ev, current_leader)) { /* * Start of a new group, discard the whole match and * start again. @@ -229,7 +229,7 @@ static struct evsel *find_evsel_group(struct evlist *perf_evlist, matched_events = 0; memset(metric_events, 0, sizeof(struct evsel *) * idnum); - current_leader = ev->leader; + current_leader = evsel__leader(ev); } /* * Check for duplicate events with the same name. For example, @@ -269,7 +269,7 @@ static struct evsel *find_evsel_group(struct evlist *perf_evlist, for (i = 0; i < idnum; i++) { ev = metric_events[i]; /* Don't free the used events. */ - set_bit(ev->idx, evlist_used); + set_bit(ev->core.idx, evlist_used); /* * The metric leader points to the identically named event in * metric_events. @@ -287,11 +287,11 @@ static struct evsel *find_evsel_group(struct evlist *perf_evlist, * when then group is left. */ if (!has_constraint && - ev->leader != metric_events[i]->leader && - evsel_same_pmu(ev->leader, metric_events[i]->leader)) + ev->core.leader != metric_events[i]->core.leader && + evsel_same_pmu_or_none(evsel__leader(ev), evsel__leader(metric_events[i]))) break; if (!strcmp(metric_events[i]->name, ev->name)) { - set_bit(ev->idx, evlist_used); + set_bit(ev->core.idx, evlist_used); ev->metric_leader = metric_events[i]; } } @@ -391,7 +391,7 @@ static int metricgroup__setup_events(struct list_head *groups, } evlist__for_each_entry_safe(perf_evlist, tmp, evsel) { - if (!test_bit(evsel->idx, evlist_used)) { + if (!test_bit(evsel->core.idx, evlist_used)) { evlist__remove(perf_evlist, evsel); evsel__delete(evsel); } @@ -1073,16 +1073,18 @@ static int metricgroup__add_metric_sys_event_iter(struct pmu_event *pe, ret = add_metric(d->metric_list, pe, d->metric_no_group, &m, NULL, d->ids); if (ret) - return ret; + goto out; ret = resolve_metric(d->metric_no_group, d->metric_list, NULL, d->ids); if (ret) - return ret; + goto out; *(d->has_match) = true; - return *d->ret; +out: + *(d->ret) = ret; + return ret; } static int metricgroup__add_metric(const char *metric, bool metric_no_group, @@ -1310,7 +1312,7 @@ int metricgroup__copy_metric_events(struct evlist *evlist, struct cgroup *cgrp, nd = rblist__entry(old_metric_events, i); old_me = container_of(nd, struct metric_event, nd); - evsel = evlist__find_evsel(evlist, old_me->evsel->idx); + evsel = evlist__find_evsel(evlist, old_me->evsel->core.idx); if (!evsel) return -EINVAL; new_me = metricgroup__lookup(new_metric_events, evsel, true); @@ -1318,7 +1320,7 @@ int metricgroup__copy_metric_events(struct evlist *evlist, struct cgroup *cgrp, return -ENOMEM; pr_debug("copying metric event for cgroup '%s': %s (idx=%d)\n", - cgrp ? cgrp->name : "root", evsel->name, evsel->idx); + cgrp ? cgrp->name : "root", evsel->name, evsel->core.idx); list_for_each_entry(old_expr, &old_me->head, nd) { new_expr = malloc(sizeof(*new_expr)); @@ -1361,7 +1363,7 @@ int metricgroup__copy_metric_events(struct evlist *evlist, struct cgroup *cgrp, /* copy evsel in the same position */ for (idx = 0; idx < nr; idx++) { evsel = old_expr->metric_events[idx]; - evsel = evlist__find_evsel(evlist, evsel->idx); + evsel = evlist__find_evsel(evlist, evsel->core.idx); if (evsel == NULL) { free(new_expr->metric_events); free(new_expr->metric_refs); diff --git a/tools/perf/util/parse-events.c b/tools/perf/util/parse-events.c index 4dad14265b81..e5eae23cfceb 100644 --- a/tools/perf/util/parse-events.c +++ b/tools/perf/util/parse-events.c @@ -150,6 +150,10 @@ struct event_symbol event_symbols_sw[PERF_COUNT_SW_MAX] = { .symbol = "bpf-output", .alias = "", }, + [PERF_COUNT_SW_CGROUP_SWITCHES] = { + .symbol = "cgroup-switches", + .alias = "", + }, }; #define __PERF_EVENT_FIELD(config, name) \ @@ -1736,7 +1740,7 @@ parse_events__set_leader_for_uncore_aliase(char *name, struct list_head *list, leader = list_first_entry(list, struct evsel, core.node); evsel = list_last_entry(list, struct evsel, core.node); - total_members = evsel->idx - leader->idx + 1; + total_members = evsel->core.idx - leader->core.idx + 1; leaders = calloc(total_members, sizeof(uintptr_t)); if (WARN_ON(!leaders)) @@ -1796,7 +1800,7 @@ parse_events__set_leader_for_uncore_aliase(char *name, struct list_head *list, __evlist__for_each_entry(list, evsel) { if (i >= nr_pmu) i = 0; - evsel->leader = (struct evsel *) leaders[i++]; + evsel__set_leader(evsel, (struct evsel *) leaders[i++]); } /* The number of members and group name are same for each group */ @@ -1829,7 +1833,7 @@ void parse_events__set_leader(char *name, struct list_head *list, if (parse_events__set_leader_for_uncore_aliase(name, list, parse_state)) return; - __evlist__set_leader(list); + __perf_evlist__set_leader(list); leader = list_entry(list->next, struct evsel, core.node); leader->group_name = name ? strdup(name) : NULL; } @@ -2281,7 +2285,7 @@ int __parse_events(struct evlist *evlist, const char *str, if (!ret) { struct evsel *last; - evlist->nr_groups += parse_state.nr_groups; + evlist->core.nr_groups += parse_state.nr_groups; last = evlist__last(evlist); last->cmdline_group_boundary = true; @@ -2928,9 +2932,14 @@ restart: } for (i = 0; i < max; i++, syms++) { + /* + * New attr.config still not supported here, the latest + * example was PERF_COUNT_SW_CGROUP_SWITCHES + */ + if (syms->symbol == NULL) + continue; - if (event_glob != NULL && syms->symbol != NULL && - !(strglobmatch(syms->symbol, event_glob) || + if (event_glob != NULL && !(strglobmatch(syms->symbol, event_glob) || (syms->alias && strglobmatch(syms->alias, event_glob)))) continue; diff --git a/tools/perf/util/parse-events.l b/tools/perf/util/parse-events.l index fb8646cc3e83..923849024b15 100644 --- a/tools/perf/util/parse-events.l +++ b/tools/perf/util/parse-events.l @@ -347,6 +347,7 @@ emulation-faults { return sym(yyscanner, PERF_TYPE_SOFTWARE, PERF_COUNT_SW_EM dummy { return sym(yyscanner, PERF_TYPE_SOFTWARE, PERF_COUNT_SW_DUMMY); } duration_time { return tool(yyscanner, PERF_TOOL_DURATION_TIME); } bpf-output { return sym(yyscanner, PERF_TYPE_SOFTWARE, PERF_COUNT_SW_BPF_OUTPUT); } +cgroup-switches { return sym(yyscanner, PERF_TYPE_SOFTWARE, PERF_COUNT_SW_CGROUP_SWITCHES); } /* * We have to handle the kernel PMU event cycles-ct/cycles-t/mem-loads/mem-stores separately. diff --git a/tools/perf/util/parse-events.y b/tools/perf/util/parse-events.y index aba12a4d488e..9321bd0e2f76 100644 --- a/tools/perf/util/parse-events.y +++ b/tools/perf/util/parse-events.y @@ -316,7 +316,7 @@ event_pmu_name opt_pmu_config if (!strncmp(name, "uncore_", 7) && strncmp($1, "uncore_", 7)) name += 7; - if (!fnmatch(pattern, name, 0)) { + if (!perf_pmu__match(pattern, name, $1)) { if (parse_events_copy_term_list(orig_terms, &terms)) CLEANUP_YYABORT; if (!parse_events_add_pmu(_parse_state, list, pmu->name, terms, true, false)) diff --git a/tools/perf/util/perf_api_probe.c b/tools/perf/util/perf_api_probe.c index 829af17a0867..020411682a3c 100644 --- a/tools/perf/util/perf_api_probe.c +++ b/tools/perf/util/perf_api_probe.c @@ -103,6 +103,11 @@ static void perf_probe_build_id(struct evsel *evsel) evsel->core.attr.build_id = 1; } +static void perf_probe_cgroup(struct evsel *evsel) +{ + evsel->core.attr.cgroup = 1; +} + bool perf_can_sample_identifier(void) { return perf_probe_api(perf_probe_sample_identifier); @@ -182,3 +187,8 @@ bool perf_can_record_build_id(void) { return perf_probe_api(perf_probe_build_id); } + +bool perf_can_record_cgroup(void) +{ + return perf_probe_api(perf_probe_cgroup); +} diff --git a/tools/perf/util/perf_api_probe.h b/tools/perf/util/perf_api_probe.h index f12ca55f509a..b104168efb15 100644 --- a/tools/perf/util/perf_api_probe.h +++ b/tools/perf/util/perf_api_probe.h @@ -12,5 +12,6 @@ bool perf_can_record_switch_events(void); bool perf_can_record_text_poke_events(void); bool perf_can_sample_identifier(void); bool perf_can_record_build_id(void); +bool perf_can_record_cgroup(void); #endif // __PERF_API_PROBE_H diff --git a/tools/perf/util/perf_dlfilter.h b/tools/perf/util/perf_dlfilter.h new file mode 100644 index 000000000000..3eef03d661b4 --- /dev/null +++ b/tools/perf/util/perf_dlfilter.h @@ -0,0 +1,150 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * perf_dlfilter.h: API for perf --dlfilter shared object + * Copyright (c) 2021, Intel Corporation. + */ +#ifndef _LINUX_PERF_DLFILTER_H +#define _LINUX_PERF_DLFILTER_H + +#include <linux/perf_event.h> +#include <linux/types.h> + +/* Definitions for perf_dlfilter_sample flags */ +enum { + PERF_DLFILTER_FLAG_BRANCH = 1ULL << 0, + PERF_DLFILTER_FLAG_CALL = 1ULL << 1, + PERF_DLFILTER_FLAG_RETURN = 1ULL << 2, + PERF_DLFILTER_FLAG_CONDITIONAL = 1ULL << 3, + PERF_DLFILTER_FLAG_SYSCALLRET = 1ULL << 4, + PERF_DLFILTER_FLAG_ASYNC = 1ULL << 5, + PERF_DLFILTER_FLAG_INTERRUPT = 1ULL << 6, + PERF_DLFILTER_FLAG_TX_ABORT = 1ULL << 7, + PERF_DLFILTER_FLAG_TRACE_BEGIN = 1ULL << 8, + PERF_DLFILTER_FLAG_TRACE_END = 1ULL << 9, + PERF_DLFILTER_FLAG_IN_TX = 1ULL << 10, + PERF_DLFILTER_FLAG_VMENTRY = 1ULL << 11, + PERF_DLFILTER_FLAG_VMEXIT = 1ULL << 12, +}; + +/* + * perf sample event information (as per perf script and <linux/perf_event.h>) + */ +struct perf_dlfilter_sample { + __u32 size; /* Size of this structure (for compatibility checking) */ + __u16 ins_lat; /* Refer PERF_SAMPLE_WEIGHT_TYPE in <linux/perf_event.h> */ + __u16 p_stage_cyc; /* Refer PERF_SAMPLE_WEIGHT_TYPE in <linux/perf_event.h> */ + __u64 ip; + __s32 pid; + __s32 tid; + __u64 time; + __u64 addr; + __u64 id; + __u64 stream_id; + __u64 period; + __u64 weight; /* Refer PERF_SAMPLE_WEIGHT_TYPE in <linux/perf_event.h> */ + __u64 transaction; /* Refer PERF_SAMPLE_TRANSACTION in <linux/perf_event.h> */ + __u64 insn_cnt; /* For instructions-per-cycle (IPC) */ + __u64 cyc_cnt; /* For instructions-per-cycle (IPC) */ + __s32 cpu; + __u32 flags; /* Refer PERF_DLFILTER_FLAG_* above */ + __u64 data_src; /* Refer PERF_SAMPLE_DATA_SRC in <linux/perf_event.h> */ + __u64 phys_addr; /* Refer PERF_SAMPLE_PHYS_ADDR in <linux/perf_event.h> */ + __u64 data_page_size; /* Refer PERF_SAMPLE_DATA_PAGE_SIZE in <linux/perf_event.h> */ + __u64 code_page_size; /* Refer PERF_SAMPLE_CODE_PAGE_SIZE in <linux/perf_event.h> */ + __u64 cgroup; /* Refer PERF_SAMPLE_CGROUP in <linux/perf_event.h> */ + __u8 cpumode; /* Refer CPUMODE_MASK etc in <linux/perf_event.h> */ + __u8 addr_correlates_sym; /* True => resolve_addr() can be called */ + __u16 misc; /* Refer perf_event_header in <linux/perf_event.h> */ + __u32 raw_size; /* Refer PERF_SAMPLE_RAW in <linux/perf_event.h> */ + const void *raw_data; /* Refer PERF_SAMPLE_RAW in <linux/perf_event.h> */ + __u64 brstack_nr; /* Number of brstack entries */ + const struct perf_branch_entry *brstack; /* Refer <linux/perf_event.h> */ + __u64 raw_callchain_nr; /* Number of raw_callchain entries */ + const __u64 *raw_callchain; /* Refer <linux/perf_event.h> */ + const char *event; +}; + +/* + * Address location (as per perf script) + */ +struct perf_dlfilter_al { + __u32 size; /* Size of this structure (for compatibility checking) */ + __u32 symoff; + const char *sym; + __u64 addr; /* Mapped address (from dso) */ + __u64 sym_start; + __u64 sym_end; + const char *dso; + __u8 sym_binding; /* STB_LOCAL, STB_GLOBAL or STB_WEAK, refer <elf.h> */ + __u8 is_64_bit; /* Only valid if dso is not NULL */ + __u8 is_kernel_ip; /* True if in kernel space */ + __u32 buildid_size; + __u8 *buildid; + /* Below members are only populated by resolve_ip() */ + __u8 filtered; /* True if this sample event will be filtered out */ + const char *comm; +}; + +struct perf_dlfilter_fns { + /* Return information about ip */ + const struct perf_dlfilter_al *(*resolve_ip)(void *ctx); + /* Return information about addr (if addr_correlates_sym) */ + const struct perf_dlfilter_al *(*resolve_addr)(void *ctx); + /* Return arguments from --dlarg option */ + char **(*args)(void *ctx, int *dlargc); + /* + * Return information about address (al->size must be set before + * calling). Returns 0 on success, -1 otherwise. + */ + __s32 (*resolve_address)(void *ctx, __u64 address, struct perf_dlfilter_al *al); + /* Return instruction bytes and length */ + const __u8 *(*insn)(void *ctx, __u32 *length); + /* Return source file name and line number */ + const char *(*srcline)(void *ctx, __u32 *line_number); + /* Return perf_event_attr, refer <linux/perf_event.h> */ + struct perf_event_attr *(*attr)(void *ctx); + /* Read object code, return numbers of bytes read */ + __s32 (*object_code)(void *ctx, __u64 ip, void *buf, __u32 len); + /* Reserved */ + void *(*reserved[120])(void *); +}; + +/* + * If implemented, 'start' will be called at the beginning, + * before any calls to 'filter_event'. Return 0 to indicate success, + * or return a negative error code. '*data' can be assigned for use + * by other functions. 'ctx' is needed for calls to perf_dlfilter_fns, + * but most perf_dlfilter_fns are not valid when called from 'start'. + */ +int start(void **data, void *ctx); + +/* + * If implemented, 'stop' will be called at the end, + * after any calls to 'filter_event'. Return 0 to indicate success, or + * return a negative error code. 'data' is set by start(). 'ctx' is + * needed for calls to perf_dlfilter_fns, but most perf_dlfilter_fns + * are not valid when called from 'stop'. + */ +int stop(void *data, void *ctx); + +/* + * If implemented, 'filter_event' will be called for each sample + * event. Return 0 to keep the sample event, 1 to filter it out, or + * return a negative error code. 'data' is set by start(). 'ctx' is + * needed for calls to perf_dlfilter_fns. + */ +int filter_event(void *data, const struct perf_dlfilter_sample *sample, void *ctx); + +/* + * The same as 'filter_event' except it is called before internal + * filtering. + */ +int filter_event_early(void *data, const struct perf_dlfilter_sample *sample, void *ctx); + +/* + * If implemented, return a one-line description of the filter, and optionally + * a longer description. + */ +const char *filter_description(const char **long_description); + +#endif diff --git a/tools/perf/util/pfm.c b/tools/perf/util/pfm.c index d735acb6c29c..756295dedccc 100644 --- a/tools/perf/util/pfm.c +++ b/tools/perf/util/pfm.c @@ -62,8 +62,16 @@ int parse_libpfm_events_option(const struct option *opt, const char *str, } /* no event */ - if (*q == '\0') + if (*q == '\0') { + if (*sep == '}') { + if (grp_evt < 0) { + ui__error("cannot close a non-existing event group\n"); + goto error; + } + grp_evt--; + } continue; + } memset(&attr, 0, sizeof(attr)); event_attr_init(&attr); @@ -91,7 +99,7 @@ int parse_libpfm_events_option(const struct option *opt, const char *str, grp_leader = evsel; if (grp_evt > -1) { - evsel->leader = grp_leader; + evsel__set_leader(evsel, grp_leader); grp_leader->core.nr_members++; grp_evt++; } @@ -102,11 +110,12 @@ int parse_libpfm_events_option(const struct option *opt, const char *str, "cannot close a non-existing event group\n"); goto error; } - evlist->nr_groups++; + evlist->core.nr_groups++; grp_leader = NULL; grp_evt = -1; } } + free(p_orig); return 0; error: free(p_orig); diff --git a/tools/perf/util/pmu-hybrid.h b/tools/perf/util/pmu-hybrid.h index d0fa7bc50a76..2b186c26a43e 100644 --- a/tools/perf/util/pmu-hybrid.h +++ b/tools/perf/util/pmu-hybrid.h @@ -19,4 +19,15 @@ struct perf_pmu *perf_pmu__find_hybrid_pmu(const char *name); bool perf_pmu__is_hybrid(const char *name); char *perf_pmu__hybrid_type_to_pmu(const char *type); +static inline int perf_pmu__hybrid_pmu_num(void) +{ + struct perf_pmu *pmu; + int num = 0; + + perf_pmu__for_each_hybrid_pmu(pmu) + num++; + + return num; +} + #endif /* __PMU_HYBRID_H */ diff --git a/tools/perf/util/pmu.c b/tools/perf/util/pmu.c index 88c8ecdc60b0..fc683bc41715 100644 --- a/tools/perf/util/pmu.c +++ b/tools/perf/util/pmu.c @@ -3,6 +3,7 @@ #include <linux/compiler.h> #include <linux/string.h> #include <linux/zalloc.h> +#include <linux/ctype.h> #include <subcmd/pager.h> #include <sys/types.h> #include <errno.h> @@ -17,6 +18,7 @@ #include <locale.h> #include <regex.h> #include <perf/cpumap.h> +#include <fnmatch.h> #include "debug.h" #include "evsel.h" #include "pmu.h" @@ -740,6 +742,35 @@ struct pmu_events_map *__weak pmu_events_map__find(void) return perf_pmu__find_map(NULL); } +/* + * Suffix must be in form tok_{digits}, or tok{digits}, or same as pmu_name + * to be valid. + */ +static bool perf_pmu__valid_suffix(const char *pmu_name, char *tok) +{ + const char *p; + + if (strncmp(pmu_name, tok, strlen(tok))) + return false; + + p = pmu_name + strlen(tok); + if (*p == 0) + return true; + + if (*p == '_') + ++p; + + /* Ensure we end in a number */ + while (1) { + if (!isdigit(*p)) + return false; + if (*(++p) == 0) + break; + } + + return true; +} + bool pmu_uncore_alias_match(const char *pmu_name, const char *name) { char *tmp = NULL, *tok, *str; @@ -766,12 +797,19 @@ bool pmu_uncore_alias_match(const char *pmu_name, const char *name) * match "socket" in "socketX_pmunameY" and then "pmuname" in * "pmunameY". */ - for (; tok; name += strlen(tok), tok = strtok_r(NULL, ",", &tmp)) { + while (1) { + char *next_tok = strtok_r(NULL, ",", &tmp); + name = strstr(name, tok); - if (!name) { + if (!name || + (!next_tok && !perf_pmu__valid_suffix(name, tok))) { res = false; goto out; } + if (!next_tok) + break; + tok = next_tok; + name += strlen(tok); } res = true; @@ -927,6 +965,13 @@ static struct perf_pmu *pmu_lookup(const char *name) LIST_HEAD(format); LIST_HEAD(aliases); __u32 type; + bool is_hybrid = perf_pmu__hybrid_mounted(name); + + /* + * Check pmu name for hybrid and the pmu may be invalid in sysfs + */ + if (!strncmp(name, "cpu_", 4) && !is_hybrid) + return NULL; /* * The pmu data we store & need consists of the pmu @@ -955,7 +1000,7 @@ static struct perf_pmu *pmu_lookup(const char *name) pmu->is_uncore = pmu_is_uncore(name); if (pmu->is_uncore) pmu->id = pmu_id(name); - pmu->is_hybrid = perf_pmu__hybrid_mounted(name); + pmu->is_hybrid = is_hybrid; pmu->max_precise = pmu_max_precise(name); pmu_add_cpu_aliases(&aliases, pmu); pmu_add_sys_aliases(&aliases, pmu); @@ -1872,3 +1917,14 @@ bool perf_pmu__has_hybrid(void) return !list_empty(&perf_pmu__hybrid_pmus); } + +int perf_pmu__match(char *pattern, char *name, char *tok) +{ + if (fnmatch(pattern, name, 0)) + return -1; + + if (tok && !perf_pmu__valid_suffix(name, tok)) + return -1; + + return 0; +} diff --git a/tools/perf/util/pmu.h b/tools/perf/util/pmu.h index a790ef758171..926da483a141 100644 --- a/tools/perf/util/pmu.h +++ b/tools/perf/util/pmu.h @@ -133,5 +133,6 @@ void perf_pmu__warn_invalid_config(struct perf_pmu *pmu, __u64 config, char *name); bool perf_pmu__has_hybrid(void); +int perf_pmu__match(char *pattern, char *name, char *tok); #endif /* __PMU_H */ diff --git a/tools/perf/util/probe-event.c b/tools/perf/util/probe-event.c index a78c8d59a555..b2a02c9ab8ea 100644 --- a/tools/perf/util/probe-event.c +++ b/tools/perf/util/probe-event.c @@ -108,7 +108,6 @@ void exit_probe_symbol_maps(void) static struct ref_reloc_sym *kernel_get_ref_reloc_sym(struct map **pmap) { - /* kmap->ref_reloc_sym should be set if host_machine is initialized */ struct kmap *kmap; struct map *map = machine__kernel_map(host_machine); @@ -180,8 +179,10 @@ struct map *get_target_map(const char *target, struct nsinfo *nsi, bool user) struct map *map; map = dso__new_map(target); - if (map && map->dso) + if (map && map->dso) { + nsinfo__put(map->dso->nsinfo); map->dso->nsinfo = nsinfo__get(nsi); + } return map; } else { return kernel_get_module_map(target); @@ -238,8 +239,8 @@ static void clear_probe_trace_events(struct probe_trace_event *tevs, int ntevs) clear_probe_trace_event(tevs + i); } -static bool kprobe_blacklist__listed(unsigned long address); -static bool kprobe_warn_out_range(const char *symbol, unsigned long address) +static bool kprobe_blacklist__listed(u64 address); +static bool kprobe_warn_out_range(const char *symbol, u64 address) { struct map *map; bool ret = false; @@ -399,8 +400,7 @@ static int find_alternative_probe_point(struct debuginfo *dinfo, pr_debug("Symbol %s address found : %" PRIx64 "\n", pp->function, address); - ret = debuginfo__find_probe_point(dinfo, (unsigned long)address, - result); + ret = debuginfo__find_probe_point(dinfo, address, result); if (ret <= 0) ret = (!ret) ? -ENOENT : ret; else { @@ -588,7 +588,7 @@ static void debuginfo_cache__exit(void) } -static int get_text_start_address(const char *exec, unsigned long *address, +static int get_text_start_address(const char *exec, u64 *address, struct nsinfo *nsi) { Elf *elf; @@ -633,7 +633,7 @@ static int find_perf_probe_point_from_dwarf(struct probe_trace_point *tp, bool is_kprobe) { struct debuginfo *dinfo = NULL; - unsigned long stext = 0; + u64 stext = 0; u64 addr = tp->address; int ret = -ENOENT; @@ -661,8 +661,7 @@ static int find_perf_probe_point_from_dwarf(struct probe_trace_point *tp, dinfo = debuginfo_cache__open(tp->module, verbose <= 0); if (dinfo) - ret = debuginfo__find_probe_point(dinfo, - (unsigned long)addr, pp); + ret = debuginfo__find_probe_point(dinfo, addr, pp); else ret = -ENOENT; @@ -677,14 +676,19 @@ error: /* Adjust symbol name and address */ static int post_process_probe_trace_point(struct probe_trace_point *tp, - struct map *map, unsigned long offs) + struct map *map, u64 offs) { struct symbol *sym; u64 addr = tp->address - offs; sym = map__find_symbol(map, addr); - if (!sym) - return -ENOENT; + if (!sym) { + /* + * If the address is in the inittext section, map can not + * find it. Ignore it if we are probing offline kernel. + */ + return (symbol_conf.ignore_vmlinux_buildid) ? 0 : -ENOENT; + } if (strcmp(sym->name, tp->symbol)) { /* If we have no realname, use symbol for it */ @@ -715,7 +719,7 @@ post_process_offline_probe_trace_events(struct probe_trace_event *tevs, int ntevs, const char *pathname) { struct map *map; - unsigned long stext = 0; + u64 stext = 0; int i, ret = 0; /* Prepare a map for offline binary */ @@ -741,7 +745,7 @@ static int add_exec_to_probe_trace_events(struct probe_trace_event *tevs, struct nsinfo *nsi) { int i, ret = 0; - unsigned long stext = 0; + u64 stext = 0; if (!exec) return 0; @@ -786,7 +790,7 @@ post_process_module_probe_trace_events(struct probe_trace_event *tevs, mod_name = find_module_name(module); for (i = 0; i < ntevs; i++) { ret = post_process_probe_trace_point(&tevs[i].point, - map, (unsigned long)text_offs); + map, text_offs); if (ret < 0) break; tevs[i].point.module = @@ -819,7 +823,10 @@ post_process_kernel_probe_trace_events(struct probe_trace_event *tevs, reloc_sym = kernel_get_ref_reloc_sym(&map); if (!reloc_sym) { - pr_warning("Relocated base symbol is not found!\n"); + pr_warning("Relocated base symbol is not found! " + "Check /proc/sys/kernel/kptr_restrict\n" + "and /proc/sys/kernel/perf_event_paranoid. " + "Or run as privileged perf user.\n\n"); return -EINVAL; } @@ -1527,7 +1534,7 @@ static int parse_perf_probe_point(char *arg, struct perf_probe_event *pev) * so tmp[1] should always valid (but could be '\0'). */ if (tmp && !strncmp(tmp, "0x", 2)) { - pp->abs_address = strtoul(pp->function, &tmp, 0); + pp->abs_address = strtoull(pp->function, &tmp, 0); if (*tmp != '\0') { semantic_error("Invalid absolute address.\n"); return -EINVAL; @@ -1902,7 +1909,7 @@ int parse_probe_trace_command(const char *cmd, struct probe_trace_event *tev) argv[i] = NULL; argc -= 1; } else - tp->address = strtoul(fmt1_str, NULL, 0); + tp->address = strtoull(fmt1_str, NULL, 0); } else { /* Only the symbol-based probe has offset */ tp->symbol = strdup(fmt1_str); @@ -2120,69 +2127,81 @@ static int synthesize_probe_trace_arg(struct probe_trace_arg *arg, } static int -synthesize_uprobe_trace_def(struct probe_trace_event *tev, struct strbuf *buf) +synthesize_probe_trace_args(struct probe_trace_event *tev, struct strbuf *buf) { - struct probe_trace_point *tp = &tev->point; - int err; + int i, ret = 0; - err = strbuf_addf(buf, "%s:0x%lx", tp->module, tp->address); + for (i = 0; i < tev->nargs && ret >= 0; i++) + ret = synthesize_probe_trace_arg(&tev->args[i], buf); - if (err >= 0 && tp->ref_ctr_offset) { - if (!uprobe_ref_ctr_is_supported()) - return -1; - err = strbuf_addf(buf, "(0x%lx)", tp->ref_ctr_offset); - } - return err >= 0 ? 0 : -1; + return ret; } -char *synthesize_probe_trace_command(struct probe_trace_event *tev) +static int +synthesize_uprobe_trace_def(struct probe_trace_point *tp, struct strbuf *buf) { - struct probe_trace_point *tp = &tev->point; - struct strbuf buf; - char *ret = NULL; - int i, err; + int err; /* Uprobes must have tp->module */ - if (tev->uprobes && !tp->module) - return NULL; - - if (strbuf_init(&buf, 32) < 0) - return NULL; - - if (strbuf_addf(&buf, "%c:%s/%s ", tp->retprobe ? 'r' : 'p', - tev->group, tev->event) < 0) - goto error; + if (!tp->module) + return -EINVAL; /* * If tp->address == 0, then this point must be a * absolute address uprobe. * try_to_find_absolute_address() should have made * tp->symbol to "0x0". */ - if (tev->uprobes && !tp->address) { - if (!tp->symbol || strcmp(tp->symbol, "0x0")) - goto error; - } + if (!tp->address && (!tp->symbol || strcmp(tp->symbol, "0x0"))) + return -EINVAL; /* Use the tp->address for uprobes */ - if (tev->uprobes) { - err = synthesize_uprobe_trace_def(tev, &buf); - } else if (!strncmp(tp->symbol, "0x", 2)) { + err = strbuf_addf(buf, "%s:0x%" PRIx64, tp->module, tp->address); + + if (err >= 0 && tp->ref_ctr_offset) { + if (!uprobe_ref_ctr_is_supported()) + return -EINVAL; + err = strbuf_addf(buf, "(0x%lx)", tp->ref_ctr_offset); + } + return err >= 0 ? 0 : err; +} + +static int +synthesize_kprobe_trace_def(struct probe_trace_point *tp, struct strbuf *buf) +{ + if (!strncmp(tp->symbol, "0x", 2)) { /* Absolute address. See try_to_find_absolute_address() */ - err = strbuf_addf(&buf, "%s%s0x%lx", tp->module ?: "", + return strbuf_addf(buf, "%s%s0x%" PRIx64, tp->module ?: "", tp->module ? ":" : "", tp->address); } else { - err = strbuf_addf(&buf, "%s%s%s+%lu", tp->module ?: "", + return strbuf_addf(buf, "%s%s%s+%lu", tp->module ?: "", tp->module ? ":" : "", tp->symbol, tp->offset); } +} - if (err) +char *synthesize_probe_trace_command(struct probe_trace_event *tev) +{ + struct probe_trace_point *tp = &tev->point; + struct strbuf buf; + char *ret = NULL; + int err; + + if (strbuf_init(&buf, 32) < 0) + return NULL; + + if (strbuf_addf(&buf, "%c:%s/%s ", tp->retprobe ? 'r' : 'p', + tev->group, tev->event) < 0) goto error; - for (i = 0; i < tev->nargs; i++) - if (synthesize_probe_trace_arg(&tev->args[i], &buf) < 0) - goto error; + if (tev->uprobes) + err = synthesize_uprobe_trace_def(tp, &buf); + else + err = synthesize_kprobe_trace_def(tp, &buf); - ret = strbuf_detach(&buf, NULL); + if (err >= 0) + err = synthesize_probe_trace_args(tev, &buf); + + if (err >= 0) + ret = strbuf_detach(&buf, NULL); error: strbuf_release(&buf); return ret; @@ -2250,7 +2269,7 @@ static int convert_to_perf_probe_point(struct probe_trace_point *tp, pp->function = strdup(tp->symbol); pp->offset = tp->offset; } else { - ret = e_snprintf(buf, 128, "0x%" PRIx64, (u64)tp->address); + ret = e_snprintf(buf, 128, "0x%" PRIx64, tp->address); if (ret < 0) return ret; pp->function = strdup(buf); @@ -2431,8 +2450,8 @@ void clear_probe_trace_event(struct probe_trace_event *tev) struct kprobe_blacklist_node { struct list_head list; - unsigned long start; - unsigned long end; + u64 start; + u64 end; char *symbol; }; @@ -2477,7 +2496,7 @@ static int kprobe_blacklist__load(struct list_head *blacklist) } INIT_LIST_HEAD(&node->list); list_add_tail(&node->list, blacklist); - if (sscanf(buf, "0x%lx-0x%lx", &node->start, &node->end) != 2) { + if (sscanf(buf, "0x%" PRIx64 "-0x%" PRIx64, &node->start, &node->end) != 2) { ret = -EINVAL; break; } @@ -2493,7 +2512,7 @@ static int kprobe_blacklist__load(struct list_head *blacklist) ret = -ENOMEM; break; } - pr_debug2("Blacklist: 0x%lx-0x%lx, %s\n", + pr_debug2("Blacklist: 0x%" PRIx64 "-0x%" PRIx64 ", %s\n", node->start, node->end, node->symbol); ret++; } @@ -2505,8 +2524,7 @@ static int kprobe_blacklist__load(struct list_head *blacklist) } static struct kprobe_blacklist_node * -kprobe_blacklist__find_by_address(struct list_head *blacklist, - unsigned long address) +kprobe_blacklist__find_by_address(struct list_head *blacklist, u64 address) { struct kprobe_blacklist_node *node; @@ -2534,7 +2552,7 @@ static void kprobe_blacklist__release(void) kprobe_blacklist__delete(&kprobe_blacklist); } -static bool kprobe_blacklist__listed(unsigned long address) +static bool kprobe_blacklist__listed(u64 address) { return !!kprobe_blacklist__find_by_address(&kprobe_blacklist, address); } @@ -2934,7 +2952,7 @@ static int find_probe_functions(struct map *map, char *name, bool cut_version = true; if (map__load(map) < 0) - return 0; + return -EACCES; /* Possible permission error to load symbols */ /* If user gives a version, don't cut off the version from symbols */ if (strchr(name, '@')) @@ -2973,6 +2991,17 @@ void __weak arch__fix_tev_from_maps(struct perf_probe_event *pev __maybe_unused, struct map *map __maybe_unused, struct symbol *sym __maybe_unused) { } + +static void pr_kallsyms_access_error(void) +{ + pr_err("Please ensure you can read the /proc/kallsyms symbol addresses.\n" + "If /proc/sys/kernel/kptr_restrict is '2', you can not read\n" + "kernel symbol addresses even if you are a superuser. Please change\n" + "it to '1'. If kptr_restrict is '1', the superuser can read the\n" + "symbol addresses.\n" + "In that case, please run this command again with sudo.\n"); +} + /* * Find probe function addresses from map. * Return an error or the number of found probe_trace_event @@ -3009,8 +3038,16 @@ static int find_probe_trace_events_from_map(struct perf_probe_event *pev, */ num_matched_functions = find_probe_functions(map, pp->function, syms); if (num_matched_functions <= 0) { - pr_err("Failed to find symbol %s in %s\n", pp->function, - pev->target ? : "kernel"); + if (num_matched_functions == -EACCES) { + pr_err("Failed to load symbols from %s\n", + pev->target ?: "/proc/kallsyms"); + if (pev->target) + pr_err("Please ensure the file is not stripped.\n"); + else + pr_kallsyms_access_error(); + } else + pr_err("Failed to find symbol %s in %s\n", pp->function, + pev->target ? : "kernel"); ret = -ENOENT; goto out; } else if (num_matched_functions > probe_conf.max_probes) { @@ -3025,7 +3062,10 @@ static int find_probe_trace_events_from_map(struct perf_probe_event *pev, (!pp->retprobe || kretprobe_offset_is_supported())) { reloc_sym = kernel_get_ref_reloc_sym(NULL); if (!reloc_sym) { - pr_warning("Relocated base symbol is not found!\n"); + pr_warning("Relocated base symbol is not found! " + "Check /proc/sys/kernel/kptr_restrict\n" + "and /proc/sys/kernel/perf_event_paranoid. " + "Or run as privileged perf user.\n\n"); ret = -EINVAL; goto out; } @@ -3180,7 +3220,7 @@ static int try_to_find_absolute_address(struct perf_probe_event *pev, * In __add_probe_trace_events, a NULL symbol is interpreted as * invalid. */ - if (asprintf(&tp->symbol, "0x%lx", tp->address) < 0) + if (asprintf(&tp->symbol, "0x%" PRIx64, tp->address) < 0) goto errout; /* For kprobe, check range */ @@ -3191,7 +3231,7 @@ static int try_to_find_absolute_address(struct perf_probe_event *pev, goto errout; } - if (asprintf(&tp->realname, "abs_%lx", tp->address) < 0) + if (asprintf(&tp->realname, "abs_%" PRIx64, tp->address) < 0) goto errout; if (pev->target) { @@ -3523,6 +3563,78 @@ int show_probe_trace_events(struct perf_probe_event *pevs, int npevs) return ret; } +static int show_bootconfig_event(struct probe_trace_event *tev) +{ + struct probe_trace_point *tp = &tev->point; + struct strbuf buf; + char *ret = NULL; + int err; + + if (strbuf_init(&buf, 32) < 0) + return -ENOMEM; + + err = synthesize_kprobe_trace_def(tp, &buf); + if (err >= 0) + err = synthesize_probe_trace_args(tev, &buf); + if (err >= 0) + ret = strbuf_detach(&buf, NULL); + strbuf_release(&buf); + + if (ret) { + printf("'%s'", ret); + free(ret); + } + + return err; +} + +int show_bootconfig_events(struct perf_probe_event *pevs, int npevs) +{ + struct strlist *namelist = strlist__new(NULL, NULL); + struct probe_trace_event *tev; + struct perf_probe_event *pev; + char *cur_name = NULL; + int i, j, ret = 0; + + if (!namelist) + return -ENOMEM; + + for (j = 0; j < npevs && !ret; j++) { + pev = &pevs[j]; + if (pev->group && strcmp(pev->group, "probe")) + pr_warning("WARN: Group name %s is ignored\n", pev->group); + if (pev->uprobes) { + pr_warning("ERROR: Bootconfig doesn't support uprobes\n"); + ret = -EINVAL; + break; + } + for (i = 0; i < pev->ntevs && !ret; i++) { + tev = &pev->tevs[i]; + /* Skip if the symbol is out of .text or blacklisted */ + if (!tev->point.symbol && !pev->uprobes) + continue; + + /* Set new name for tev (and update namelist) */ + ret = probe_trace_event__set_name(tev, pev, + namelist, true); + if (ret) + break; + + if (!cur_name || strcmp(cur_name, tev->event)) { + printf("%sftrace.event.kprobes.%s.probe = ", + cur_name ? "\n" : "", tev->event); + cur_name = tev->event; + } else + printf(", "); + ret = show_bootconfig_event(tev); + } + } + printf("\n"); + strlist__delete(namelist); + + return ret; +} + int apply_perf_probe_events(struct perf_probe_event *pevs, int npevs) { int i, ret = 0; diff --git a/tools/perf/util/probe-event.h b/tools/perf/util/probe-event.h index 4f0eb3a20c36..8ad5b1579f1d 100644 --- a/tools/perf/util/probe-event.h +++ b/tools/perf/util/probe-event.h @@ -15,6 +15,7 @@ struct probe_conf { bool force_add; bool no_inlines; bool cache; + bool bootconfig; int max_probes; unsigned long magic_num; }; @@ -32,7 +33,7 @@ struct probe_trace_point { char *module; /* Module name */ unsigned long offset; /* Offset from symbol */ unsigned long ref_ctr_offset; /* SDT reference counter offset */ - unsigned long address; /* Actual address of the trace point */ + u64 address; /* Actual address of the trace point */ bool retprobe; /* Return probe flag */ }; @@ -69,7 +70,7 @@ struct perf_probe_point { bool retprobe; /* Return probe flag */ char *lazy_line; /* Lazy matching pattern */ unsigned long offset; /* Offset from function entry */ - unsigned long abs_address; /* Absolute address of the point */ + u64 abs_address; /* Absolute address of the point */ }; /* Perf probe probing argument field chain */ @@ -163,6 +164,7 @@ int add_perf_probe_events(struct perf_probe_event *pevs, int npevs); int convert_perf_probe_events(struct perf_probe_event *pevs, int npevs); int apply_perf_probe_events(struct perf_probe_event *pevs, int npevs); int show_probe_trace_events(struct perf_probe_event *pevs, int npevs); +int show_bootconfig_events(struct perf_probe_event *pevs, int npevs); void cleanup_perf_probe_events(struct perf_probe_event *pevs, int npevs); struct strfilter; diff --git a/tools/perf/util/probe-file.c b/tools/perf/util/probe-file.c index 52273542e6ef..3d50de3217d5 100644 --- a/tools/perf/util/probe-file.c +++ b/tools/perf/util/probe-file.c @@ -22,6 +22,7 @@ #include "symbol.h" #include "strbuf.h" #include <api/fs/tracing_path.h> +#include <api/fs/fs.h> #include "probe-event.h" #include "probe-file.h" #include "session.h" @@ -31,44 +32,78 @@ /* 4096 - 2 ('\n' + '\0') */ #define MAX_CMDLEN 4094 -static void print_open_warning(int err, bool uprobe) +static bool print_common_warning(int err, bool readwrite) { - char sbuf[STRERR_BUFSIZE]; + if (err == -EACCES) + pr_warning("No permission to %s tracefs.\nPlease %s\n", + readwrite ? "write" : "read", + readwrite ? "run this command again with sudo." : + "try 'sudo mount -o remount,mode=755 /sys/kernel/tracing/'"); + else + return false; - if (err == -ENOENT) { - const char *config; + return true; +} - if (uprobe) - config = "CONFIG_UPROBE_EVENTS"; - else - config = "CONFIG_KPROBE_EVENTS"; +static bool print_configure_probe_event(int kerr, int uerr) +{ + const char *config, *file; + + if (kerr == -ENOENT && uerr == -ENOENT) { + file = "{k,u}probe_events"; + config = "CONFIG_KPROBE_EVENTS=y and CONFIG_UPROBE_EVENTS=y"; + } else if (kerr == -ENOENT) { + file = "kprobe_events"; + config = "CONFIG_KPROBE_EVENTS=y"; + } else if (uerr == -ENOENT) { + file = "uprobe_events"; + config = "CONFIG_UPROBE_EVENTS=y"; + } else + return false; - pr_warning("%cprobe_events file does not exist" - " - please rebuild kernel with %s.\n", - uprobe ? 'u' : 'k', config); - } else if (err == -ENOTSUP) - pr_warning("Tracefs or debugfs is not mounted.\n"); + if (!debugfs__configured() && !tracefs__configured()) + pr_warning("Debugfs or tracefs is not mounted\n" + "Please try 'sudo mount -t tracefs nodev /sys/kernel/tracing/'\n"); else - pr_warning("Failed to open %cprobe_events: %s\n", - uprobe ? 'u' : 'k', - str_error_r(-err, sbuf, sizeof(sbuf))); + pr_warning("%s/%s does not exist.\nPlease rebuild kernel with %s.\n", + tracing_path_mount(), file, config); + + return true; +} + +static void print_open_warning(int err, bool uprobe, bool readwrite) +{ + char sbuf[STRERR_BUFSIZE]; + + if (print_common_warning(err, readwrite)) + return; + + if (print_configure_probe_event(uprobe ? 0 : err, uprobe ? err : 0)) + return; + + pr_warning("Failed to open %s/%cprobe_events: %s\n", + tracing_path_mount(), uprobe ? 'u' : 'k', + str_error_r(-err, sbuf, sizeof(sbuf))); } -static void print_both_open_warning(int kerr, int uerr) +static void print_both_open_warning(int kerr, int uerr, bool readwrite) { - /* Both kprobes and uprobes are disabled, warn it. */ - if (kerr == -ENOTSUP && uerr == -ENOTSUP) - pr_warning("Tracefs or debugfs is not mounted.\n"); - else if (kerr == -ENOENT && uerr == -ENOENT) - pr_warning("Please rebuild kernel with CONFIG_KPROBE_EVENTS " - "or/and CONFIG_UPROBE_EVENTS.\n"); - else { - char sbuf[STRERR_BUFSIZE]; - pr_warning("Failed to open kprobe events: %s.\n", + char sbuf[STRERR_BUFSIZE]; + + if (kerr == uerr && print_common_warning(kerr, readwrite)) + return; + + if (print_configure_probe_event(kerr, uerr)) + return; + + if (kerr < 0) + pr_warning("Failed to open %s/kprobe_events: %s.\n", + tracing_path_mount(), str_error_r(-kerr, sbuf, sizeof(sbuf))); - pr_warning("Failed to open uprobe events: %s.\n", + if (uerr < 0) + pr_warning("Failed to open %s/uprobe_events: %s.\n", + tracing_path_mount(), str_error_r(-uerr, sbuf, sizeof(sbuf))); - } } int open_trace_file(const char *trace_file, bool readwrite) @@ -109,7 +144,7 @@ int probe_file__open(int flag) else fd = open_kprobe_events(flag & PF_FL_RW); if (fd < 0) - print_open_warning(fd, flag & PF_FL_UPROBE); + print_open_warning(fd, flag & PF_FL_UPROBE, flag & PF_FL_RW); return fd; } @@ -122,7 +157,7 @@ int probe_file__open_both(int *kfd, int *ufd, int flag) *kfd = open_kprobe_events(flag & PF_FL_RW); *ufd = open_uprobe_events(flag & PF_FL_RW); if (*kfd < 0 && *ufd < 0) { - print_both_open_warning(*kfd, *ufd); + print_both_open_warning(*kfd, *ufd, flag & PF_FL_RW); return *kfd; } @@ -342,11 +377,11 @@ int probe_file__del_events(int fd, struct strfilter *filter) ret = probe_file__get_events(fd, filter, namelist); if (ret < 0) - return ret; + goto out; ret = probe_file__del_strlist(fd, namelist); +out: strlist__delete(namelist); - return ret; } diff --git a/tools/perf/util/probe-finder.c b/tools/perf/util/probe-finder.c index 866f2d514d72..50d861a80f57 100644 --- a/tools/perf/util/probe-finder.c +++ b/tools/perf/util/probe-finder.c @@ -118,12 +118,17 @@ struct debuginfo *debuginfo__new(const char *path) char buf[PATH_MAX], nil = '\0'; struct dso *dso; struct debuginfo *dinfo = NULL; + struct build_id bid; /* Try to open distro debuginfo files */ dso = dso__new(path); if (!dso) goto out; + /* Set the build id for DSO_BINARY_TYPE__BUILDID_DEBUGINFO */ + if (is_regular_file(path) && filename__read_build_id(path, &bid) > 0) + dso__set_build_id(dso, &bid); + for (type = distro_dwarf_types; !dinfo && *type != DSO_BINARY_TYPE__NOT_FOUND; type++) { @@ -190,6 +195,9 @@ static int convert_variable_location(Dwarf_Die *vr_die, Dwarf_Addr addr, immediate_value_is_supported()) { Dwarf_Sword snum; + if (!tvar) + return 0; + dwarf_formsdata(&attr, &snum); ret = asprintf(&tvar->value, "\\%ld", (long)snum); @@ -660,7 +668,7 @@ static int convert_to_trace_point(Dwarf_Die *sp_die, Dwfl_Module *mod, } tp->offset = (unsigned long)(paddr - eaddr); - tp->address = (unsigned long)paddr; + tp->address = paddr; tp->symbol = strdup(symbol); if (!tp->symbol) return -ENOMEM; @@ -1699,7 +1707,7 @@ int debuginfo__get_text_offset(struct debuginfo *dbg, Dwarf_Addr *offs, } /* Reverse search */ -int debuginfo__find_probe_point(struct debuginfo *dbg, unsigned long addr, +int debuginfo__find_probe_point(struct debuginfo *dbg, u64 addr, struct perf_probe_point *ppt) { Dwarf_Die cudie, spdie, indie; @@ -1712,14 +1720,14 @@ int debuginfo__find_probe_point(struct debuginfo *dbg, unsigned long addr, addr += baseaddr; /* Find cu die */ if (!dwarf_addrdie(dbg->dbg, (Dwarf_Addr)addr, &cudie)) { - pr_warning("Failed to find debug information for address %lx\n", + pr_warning("Failed to find debug information for address %" PRIx64 "\n", addr); ret = -EINVAL; goto end; } /* Find a corresponding line (filename and lineno) */ - cu_find_lineinfo(&cudie, addr, &fname, &lineno); + cu_find_lineinfo(&cudie, (Dwarf_Addr)addr, &fname, &lineno); /* Don't care whether it failed or not */ /* Find a corresponding function (name, baseline and baseaddr) */ @@ -1734,7 +1742,7 @@ int debuginfo__find_probe_point(struct debuginfo *dbg, unsigned long addr, } fname = dwarf_decl_file(&spdie); - if (addr == (unsigned long)baseaddr) { + if (addr == baseaddr) { /* Function entry - Relative line number is 0 */ lineno = baseline; goto post; @@ -1780,7 +1788,7 @@ post: if (lineno) ppt->line = lineno - baseline; else if (basefunc) { - ppt->offset = addr - (unsigned long)baseaddr; + ppt->offset = addr - baseaddr; func = basefunc; } @@ -1820,8 +1828,7 @@ static int line_range_add_line(const char *src, unsigned int lineno, } static int line_range_walk_cb(const char *fname, int lineno, - Dwarf_Addr addr __maybe_unused, - void *data) + Dwarf_Addr addr, void *data) { struct line_finder *lf = data; const char *__fname; diff --git a/tools/perf/util/probe-finder.h b/tools/perf/util/probe-finder.h index 2febb5875678..8bc1c80d3c1c 100644 --- a/tools/perf/util/probe-finder.h +++ b/tools/perf/util/probe-finder.h @@ -46,7 +46,7 @@ int debuginfo__find_trace_events(struct debuginfo *dbg, struct probe_trace_event **tevs); /* Find a perf_probe_point from debuginfo */ -int debuginfo__find_probe_point(struct debuginfo *dbg, unsigned long addr, +int debuginfo__find_probe_point(struct debuginfo *dbg, u64 addr, struct perf_probe_point *ppt); int debuginfo__get_text_offset(struct debuginfo *dbg, Dwarf_Addr *offs, diff --git a/tools/perf/util/python.c b/tools/perf/util/python.c index 412f8e79e409..8feef3a05af7 100644 --- a/tools/perf/util/python.c +++ b/tools/perf/util/python.c @@ -1032,7 +1032,7 @@ static PyObject *pyrf_evlist__add(struct pyrf_evlist *pevlist, Py_INCREF(pevsel); evsel = &((struct pyrf_evsel *)pevsel)->evsel; - evsel->idx = evlist->core.nr_entries; + evsel->core.idx = evlist->core.nr_entries; evlist__add(evlist, evsel); return Py_BuildValue("i", evlist->core.nr_entries); diff --git a/tools/perf/util/record.c b/tools/perf/util/record.c index 43e5b563dee8..bff669b615ee 100644 --- a/tools/perf/util/record.c +++ b/tools/perf/util/record.c @@ -25,12 +25,12 @@ */ static struct evsel *evsel__read_sampler(struct evsel *evsel, struct evlist *evlist) { - struct evsel *leader = evsel->leader; + struct evsel *leader = evsel__leader(evsel); if (evsel__is_aux_event(leader) || arch_topdown_sample_read(leader) || is_mem_loads_aux_event(leader)) { evlist__for_each_entry(evlist, evsel) { - if (evsel->leader == leader && evsel != evsel->leader) + if (evsel__leader(evsel) == leader && evsel != evsel__leader(evsel)) return evsel; } } @@ -53,7 +53,7 @@ static u64 evsel__config_term_mask(struct evsel *evsel) static void evsel__config_leader_sampling(struct evsel *evsel, struct evlist *evlist) { struct perf_event_attr *attr = &evsel->core.attr; - struct evsel *leader = evsel->leader; + struct evsel *leader = evsel__leader(evsel); struct evsel *read_sampler; u64 term_types, freq_mask; diff --git a/tools/perf/util/scripting-engines/trace-event-perl.c b/tools/perf/util/scripting-engines/trace-event-perl.c index 0e608a5ef599..32a721b3e9a5 100644 --- a/tools/perf/util/scripting-engines/trace-event-perl.c +++ b/tools/perf/util/scripting-engines/trace-event-perl.c @@ -371,9 +371,6 @@ static void perl_process_tracepoint(struct perf_sample *sample, s = nsecs / NSEC_PER_SEC; ns = nsecs - s * NSEC_PER_SEC; - scripting_context->event_data = data; - scripting_context->pevent = evsel->tp_format->tep; - ENTER; SAVETMPS; PUSHMARK(SP); @@ -456,8 +453,10 @@ static void perl_process_event_generic(union perf_event *event, static void perl_process_event(union perf_event *event, struct perf_sample *sample, struct evsel *evsel, - struct addr_location *al) + struct addr_location *al, + struct addr_location *addr_al) { + scripting_context__update(scripting_context, event, sample, evsel, al, addr_al); perl_process_tracepoint(sample, evsel, al); perl_process_event_generic(event, sample, evsel); } @@ -474,11 +473,14 @@ static void run_start_sub(void) /* * Start trace script */ -static int perl_start_script(const char *script, int argc, const char **argv) +static int perl_start_script(const char *script, int argc, const char **argv, + struct perf_session *session) { const char **command_line; int i, err = 0; + scripting_context->session = session; + command_line = malloc((argc + 2) * sizeof(const char *)); command_line[0] = ""; command_line[1] = script; @@ -750,6 +752,7 @@ sub print_backtrace\n\ struct scripting_ops perl_scripting_ops = { .name = "Perl", + .dirname = "perl", .start_script = perl_start_script, .flush_script = perl_flush_script, .stop_script = perl_stop_script, diff --git a/tools/perf/util/scripting-engines/trace-event-python.c b/tools/perf/util/scripting-engines/trace-event-python.c index 4e4aa4c97ac5..69129e2aa7a1 100644 --- a/tools/perf/util/scripting-engines/trace-event-python.c +++ b/tools/perf/util/scripting-engines/trace-event-python.c @@ -687,7 +687,7 @@ static void set_sample_datasrc_in_dict(PyObject *dict, _PyUnicode_FromString(decode)); } -static int regs_map(struct regs_dump *regs, uint64_t mask, char *bf, int size) +static void regs_map(struct regs_dump *regs, uint64_t mask, char *bf, int size) { unsigned int i = 0, r; int printed = 0; @@ -695,7 +695,7 @@ static int regs_map(struct regs_dump *regs, uint64_t mask, char *bf, int size) bf[0] = 0; if (!regs || !regs->regs) - return 0; + return; for_each_set_bit(r, (unsigned long *) &mask, sizeof(mask) * 8) { u64 val = regs->regs[i++]; @@ -704,8 +704,6 @@ static int regs_map(struct regs_dump *regs, uint64_t mask, char *bf, int size) "%5s:0x%" PRIx64 " ", perf_reg_name(r), val); } - - return printed; } static void set_regs_in_dict(PyObject *dict, @@ -713,7 +711,16 @@ static void set_regs_in_dict(PyObject *dict, struct evsel *evsel) { struct perf_event_attr *attr = &evsel->core.attr; - char bf[512]; + + /* + * Here value 28 is a constant size which can be used to print + * one register value and its corresponds to: + * 16 chars is to specify 64 bit register in hexadecimal. + * 2 chars is for appending "0x" to the hexadecimal value and + * 10 chars is for register name. + */ + int size = __sw_hweight64(attr->sample_regs_intr) * 28; + char bf[size]; regs_map(&sample->intr_regs, attr->sample_regs_intr, bf, sizeof(bf)); @@ -726,9 +733,49 @@ static void set_regs_in_dict(PyObject *dict, _PyUnicode_FromString(bf)); } +static void set_sym_in_dict(PyObject *dict, struct addr_location *al, + const char *dso_field, const char *sym_field, + const char *symoff_field) +{ + if (al->map) { + pydict_set_item_string_decref(dict, dso_field, + _PyUnicode_FromString(al->map->dso->name)); + } + if (al->sym) { + pydict_set_item_string_decref(dict, sym_field, + _PyUnicode_FromString(al->sym->name)); + pydict_set_item_string_decref(dict, symoff_field, + PyLong_FromUnsignedLong(get_offset(al->sym, al))); + } +} + +static void set_sample_flags(PyObject *dict, u32 flags) +{ + const char *ch = PERF_IP_FLAG_CHARS; + char *p, str[33]; + + for (p = str; *ch; ch++, flags >>= 1) { + if (flags & 1) + *p++ = *ch; + } + *p = 0; + pydict_set_item_string_decref(dict, "flags", _PyUnicode_FromString(str)); +} + +static void python_process_sample_flags(struct perf_sample *sample, PyObject *dict_sample) +{ + char flags_disp[SAMPLE_FLAGS_BUF_SIZE]; + + set_sample_flags(dict_sample, sample->flags); + perf_sample__sprintf_flags(sample->flags, flags_disp, sizeof(flags_disp)); + pydict_set_item_string_decref(dict_sample, "flags_disp", + _PyUnicode_FromString(flags_disp)); +} + static PyObject *get_perf_sample_dict(struct perf_sample *sample, struct evsel *evsel, struct addr_location *al, + struct addr_location *addr_al, PyObject *callchain) { PyObject *dict, *dict_sample, *brstack, *brstacksym; @@ -772,14 +819,7 @@ static PyObject *get_perf_sample_dict(struct perf_sample *sample, (const char *)sample->raw_data, sample->raw_size)); pydict_set_item_string_decref(dict, "comm", _PyUnicode_FromString(thread__comm_str(al->thread))); - if (al->map) { - pydict_set_item_string_decref(dict, "dso", - _PyUnicode_FromString(al->map->dso->name)); - } - if (al->sym) { - pydict_set_item_string_decref(dict, "symbol", - _PyUnicode_FromString(al->sym->name)); - } + set_sym_in_dict(dict, al, "dso", "symbol", "symoff"); pydict_set_item_string_decref(dict, "callchain", callchain); @@ -789,6 +829,26 @@ static PyObject *get_perf_sample_dict(struct perf_sample *sample, brstacksym = python_process_brstacksym(sample, al->thread); pydict_set_item_string_decref(dict, "brstacksym", brstacksym); + pydict_set_item_string_decref(dict_sample, "cpumode", + _PyLong_FromLong((unsigned long)sample->cpumode)); + + if (addr_al) { + pydict_set_item_string_decref(dict_sample, "addr_correlates_sym", + PyBool_FromLong(1)); + set_sym_in_dict(dict_sample, addr_al, "addr_dso", "addr_symbol", "addr_symoff"); + } + + if (sample->flags) + python_process_sample_flags(sample, dict_sample); + + /* Instructions per cycle (IPC) */ + if (sample->insn_cnt && sample->cyc_cnt) { + pydict_set_item_string_decref(dict_sample, "insn_cnt", + PyLong_FromUnsignedLongLong(sample->insn_cnt)); + pydict_set_item_string_decref(dict_sample, "cyc_cnt", + PyLong_FromUnsignedLongLong(sample->cyc_cnt)); + } + set_regs_in_dict(dict, sample, evsel); return dict; @@ -796,7 +856,8 @@ static PyObject *get_perf_sample_dict(struct perf_sample *sample, static void python_process_tracepoint(struct perf_sample *sample, struct evsel *evsel, - struct addr_location *al) + struct addr_location *al, + struct addr_location *addr_al) { struct tep_event *event = evsel->tp_format; PyObject *handler, *context, *t, *obj = NULL, *callchain; @@ -843,9 +904,6 @@ static void python_process_tracepoint(struct perf_sample *sample, s = nsecs / NSEC_PER_SEC; ns = nsecs - s * NSEC_PER_SEC; - scripting_context->event_data = data; - scripting_context->pevent = evsel->tp_format->tep; - context = _PyCapsule_New(scripting_context, NULL, NULL); PyTuple_SetItem(t, n++, _PyUnicode_FromString(handler_name)); @@ -906,7 +964,7 @@ static void python_process_tracepoint(struct perf_sample *sample, PyTuple_SetItem(t, n++, dict); if (get_argument_count(handler) == (int) n + 1) { - all_entries_dict = get_perf_sample_dict(sample, evsel, al, + all_entries_dict = get_perf_sample_dict(sample, evsel, al, addr_al, callchain); PyTuple_SetItem(t, n++, all_entries_dict); } else { @@ -934,7 +992,7 @@ static PyObject *tuple_new(unsigned int sz) return t; } -static int tuple_set_u64(PyObject *t, unsigned int pos, u64 val) +static int tuple_set_s64(PyObject *t, unsigned int pos, s64 val) { #if BITS_PER_LONG == 64 return PyTuple_SetItem(t, pos, _PyLong_FromLong(val)); @@ -944,11 +1002,37 @@ static int tuple_set_u64(PyObject *t, unsigned int pos, u64 val) #endif } +/* + * Databases support only signed 64-bit numbers, so even though we are + * exporting a u64, it must be as s64. + */ +#define tuple_set_d64 tuple_set_s64 + +static int tuple_set_u64(PyObject *t, unsigned int pos, u64 val) +{ +#if BITS_PER_LONG == 64 + return PyTuple_SetItem(t, pos, PyLong_FromUnsignedLong(val)); +#endif +#if BITS_PER_LONG == 32 + return PyTuple_SetItem(t, pos, PyLong_FromUnsignedLongLong(val)); +#endif +} + +static int tuple_set_u32(PyObject *t, unsigned int pos, u32 val) +{ + return PyTuple_SetItem(t, pos, PyLong_FromUnsignedLong(val)); +} + static int tuple_set_s32(PyObject *t, unsigned int pos, s32 val) { return PyTuple_SetItem(t, pos, _PyLong_FromLong(val)); } +static int tuple_set_bool(PyObject *t, unsigned int pos, bool val) +{ + return PyTuple_SetItem(t, pos, PyBool_FromLong(val)); +} + static int tuple_set_string(PyObject *t, unsigned int pos, const char *s) { return PyTuple_SetItem(t, pos, _PyUnicode_FromString(s)); @@ -967,7 +1051,7 @@ static int python_export_evsel(struct db_export *dbe, struct evsel *evsel) t = tuple_new(2); - tuple_set_u64(t, 0, evsel->db_id); + tuple_set_d64(t, 0, evsel->db_id); tuple_set_string(t, 1, evsel__name(evsel)); call_object(tables->evsel_handler, t, "evsel_table"); @@ -985,7 +1069,7 @@ static int python_export_machine(struct db_export *dbe, t = tuple_new(3); - tuple_set_u64(t, 0, machine->db_id); + tuple_set_d64(t, 0, machine->db_id); tuple_set_s32(t, 1, machine->pid); tuple_set_string(t, 2, machine->root_dir ? machine->root_dir : ""); @@ -1004,9 +1088,9 @@ static int python_export_thread(struct db_export *dbe, struct thread *thread, t = tuple_new(5); - tuple_set_u64(t, 0, thread->db_id); - tuple_set_u64(t, 1, machine->db_id); - tuple_set_u64(t, 2, main_thread_db_id); + tuple_set_d64(t, 0, thread->db_id); + tuple_set_d64(t, 1, machine->db_id); + tuple_set_d64(t, 2, main_thread_db_id); tuple_set_s32(t, 3, thread->pid_); tuple_set_s32(t, 4, thread->tid); @@ -1025,10 +1109,10 @@ static int python_export_comm(struct db_export *dbe, struct comm *comm, t = tuple_new(5); - tuple_set_u64(t, 0, comm->db_id); + tuple_set_d64(t, 0, comm->db_id); tuple_set_string(t, 1, comm__str(comm)); - tuple_set_u64(t, 2, thread->db_id); - tuple_set_u64(t, 3, comm->start); + tuple_set_d64(t, 2, thread->db_id); + tuple_set_d64(t, 3, comm->start); tuple_set_s32(t, 4, comm->exec); call_object(tables->comm_handler, t, "comm_table"); @@ -1046,9 +1130,9 @@ static int python_export_comm_thread(struct db_export *dbe, u64 db_id, t = tuple_new(3); - tuple_set_u64(t, 0, db_id); - tuple_set_u64(t, 1, comm->db_id); - tuple_set_u64(t, 2, thread->db_id); + tuple_set_d64(t, 0, db_id); + tuple_set_d64(t, 1, comm->db_id); + tuple_set_d64(t, 2, thread->db_id); call_object(tables->comm_thread_handler, t, "comm_thread_table"); @@ -1068,8 +1152,8 @@ static int python_export_dso(struct db_export *dbe, struct dso *dso, t = tuple_new(5); - tuple_set_u64(t, 0, dso->db_id); - tuple_set_u64(t, 1, machine->db_id); + tuple_set_d64(t, 0, dso->db_id); + tuple_set_d64(t, 1, machine->db_id); tuple_set_string(t, 2, dso->short_name); tuple_set_string(t, 3, dso->long_name); tuple_set_string(t, 4, sbuild_id); @@ -1090,10 +1174,10 @@ static int python_export_symbol(struct db_export *dbe, struct symbol *sym, t = tuple_new(6); - tuple_set_u64(t, 0, *sym_db_id); - tuple_set_u64(t, 1, dso->db_id); - tuple_set_u64(t, 2, sym->start); - tuple_set_u64(t, 3, sym->end); + tuple_set_d64(t, 0, *sym_db_id); + tuple_set_d64(t, 1, dso->db_id); + tuple_set_d64(t, 2, sym->start); + tuple_set_d64(t, 3, sym->end); tuple_set_s32(t, 4, sym->binding); tuple_set_string(t, 5, sym->name); @@ -1130,30 +1214,30 @@ static void python_export_sample_table(struct db_export *dbe, t = tuple_new(24); - tuple_set_u64(t, 0, es->db_id); - tuple_set_u64(t, 1, es->evsel->db_id); - tuple_set_u64(t, 2, es->al->maps->machine->db_id); - tuple_set_u64(t, 3, es->al->thread->db_id); - tuple_set_u64(t, 4, es->comm_db_id); - tuple_set_u64(t, 5, es->dso_db_id); - tuple_set_u64(t, 6, es->sym_db_id); - tuple_set_u64(t, 7, es->offset); - tuple_set_u64(t, 8, es->sample->ip); - tuple_set_u64(t, 9, es->sample->time); + tuple_set_d64(t, 0, es->db_id); + tuple_set_d64(t, 1, es->evsel->db_id); + tuple_set_d64(t, 2, es->al->maps->machine->db_id); + tuple_set_d64(t, 3, es->al->thread->db_id); + tuple_set_d64(t, 4, es->comm_db_id); + tuple_set_d64(t, 5, es->dso_db_id); + tuple_set_d64(t, 6, es->sym_db_id); + tuple_set_d64(t, 7, es->offset); + tuple_set_d64(t, 8, es->sample->ip); + tuple_set_d64(t, 9, es->sample->time); tuple_set_s32(t, 10, es->sample->cpu); - tuple_set_u64(t, 11, es->addr_dso_db_id); - tuple_set_u64(t, 12, es->addr_sym_db_id); - tuple_set_u64(t, 13, es->addr_offset); - tuple_set_u64(t, 14, es->sample->addr); - tuple_set_u64(t, 15, es->sample->period); - tuple_set_u64(t, 16, es->sample->weight); - tuple_set_u64(t, 17, es->sample->transaction); - tuple_set_u64(t, 18, es->sample->data_src); + tuple_set_d64(t, 11, es->addr_dso_db_id); + tuple_set_d64(t, 12, es->addr_sym_db_id); + tuple_set_d64(t, 13, es->addr_offset); + tuple_set_d64(t, 14, es->sample->addr); + tuple_set_d64(t, 15, es->sample->period); + tuple_set_d64(t, 16, es->sample->weight); + tuple_set_d64(t, 17, es->sample->transaction); + tuple_set_d64(t, 18, es->sample->data_src); tuple_set_s32(t, 19, es->sample->flags & PERF_BRANCH_MASK); tuple_set_s32(t, 20, !!(es->sample->flags & PERF_IP_FLAG_IN_TX)); - tuple_set_u64(t, 21, es->call_path_id); - tuple_set_u64(t, 22, es->sample->insn_cnt); - tuple_set_u64(t, 23, es->sample->cyc_cnt); + tuple_set_d64(t, 21, es->call_path_id); + tuple_set_d64(t, 22, es->sample->insn_cnt); + tuple_set_d64(t, 23, es->sample->cyc_cnt); call_object(tables->sample_handler, t, "sample_table"); @@ -1167,8 +1251,8 @@ static void python_export_synth(struct db_export *dbe, struct export_sample *es) t = tuple_new(3); - tuple_set_u64(t, 0, es->db_id); - tuple_set_u64(t, 1, es->evsel->core.attr.config); + tuple_set_d64(t, 0, es->db_id); + tuple_set_d64(t, 1, es->evsel->core.attr.config); tuple_set_bytes(t, 2, es->sample->raw_data, es->sample->raw_size); call_object(tables->synth_handler, t, "synth_data"); @@ -1200,10 +1284,10 @@ static int python_export_call_path(struct db_export *dbe, struct call_path *cp) t = tuple_new(4); - tuple_set_u64(t, 0, cp->db_id); - tuple_set_u64(t, 1, parent_db_id); - tuple_set_u64(t, 2, sym_db_id); - tuple_set_u64(t, 3, cp->ip); + tuple_set_d64(t, 0, cp->db_id); + tuple_set_d64(t, 1, parent_db_id); + tuple_set_d64(t, 2, sym_db_id); + tuple_set_d64(t, 3, cp->ip); call_object(tables->call_path_handler, t, "call_path_table"); @@ -1221,20 +1305,20 @@ static int python_export_call_return(struct db_export *dbe, t = tuple_new(14); - tuple_set_u64(t, 0, cr->db_id); - tuple_set_u64(t, 1, cr->thread->db_id); - tuple_set_u64(t, 2, comm_db_id); - tuple_set_u64(t, 3, cr->cp->db_id); - tuple_set_u64(t, 4, cr->call_time); - tuple_set_u64(t, 5, cr->return_time); - tuple_set_u64(t, 6, cr->branch_count); - tuple_set_u64(t, 7, cr->call_ref); - tuple_set_u64(t, 8, cr->return_ref); - tuple_set_u64(t, 9, cr->cp->parent->db_id); + tuple_set_d64(t, 0, cr->db_id); + tuple_set_d64(t, 1, cr->thread->db_id); + tuple_set_d64(t, 2, comm_db_id); + tuple_set_d64(t, 3, cr->cp->db_id); + tuple_set_d64(t, 4, cr->call_time); + tuple_set_d64(t, 5, cr->return_time); + tuple_set_d64(t, 6, cr->branch_count); + tuple_set_d64(t, 7, cr->call_ref); + tuple_set_d64(t, 8, cr->return_ref); + tuple_set_d64(t, 9, cr->cp->parent->db_id); tuple_set_s32(t, 10, cr->flags); - tuple_set_u64(t, 11, cr->parent_db_id); - tuple_set_u64(t, 12, cr->insn_count); - tuple_set_u64(t, 13, cr->cyc_count); + tuple_set_d64(t, 11, cr->parent_db_id); + tuple_set_d64(t, 12, cr->insn_count); + tuple_set_d64(t, 13, cr->cyc_count); call_object(tables->call_return_handler, t, "call_return_table"); @@ -1254,14 +1338,14 @@ static int python_export_context_switch(struct db_export *dbe, u64 db_id, t = tuple_new(9); - tuple_set_u64(t, 0, db_id); - tuple_set_u64(t, 1, machine->db_id); - tuple_set_u64(t, 2, sample->time); + tuple_set_d64(t, 0, db_id); + tuple_set_d64(t, 1, machine->db_id); + tuple_set_d64(t, 2, sample->time); tuple_set_s32(t, 3, sample->cpu); - tuple_set_u64(t, 4, th_out_id); - tuple_set_u64(t, 5, comm_out_id); - tuple_set_u64(t, 6, th_in_id); - tuple_set_u64(t, 7, comm_in_id); + tuple_set_d64(t, 4, th_out_id); + tuple_set_d64(t, 5, comm_out_id); + tuple_set_d64(t, 6, th_in_id); + tuple_set_d64(t, 7, comm_in_id); tuple_set_s32(t, 8, flags); call_object(tables->context_switch_handler, t, "context_switch"); @@ -1281,7 +1365,8 @@ static int python_process_call_return(struct call_return *cr, u64 *parent_db_id, static void python_process_general_event(struct perf_sample *sample, struct evsel *evsel, - struct addr_location *al) + struct addr_location *al, + struct addr_location *addr_al) { PyObject *handler, *t, *dict, *callchain; static char handler_name[64]; @@ -1303,7 +1388,7 @@ static void python_process_general_event(struct perf_sample *sample, /* ip unwinding */ callchain = python_process_callchain(sample, evsel, al); - dict = get_perf_sample_dict(sample, evsel, al, callchain); + dict = get_perf_sample_dict(sample, evsel, al, addr_al, callchain); PyTuple_SetItem(t, n++, dict); if (_PyTuple_Resize(&t, n) == -1) @@ -1317,23 +1402,64 @@ static void python_process_general_event(struct perf_sample *sample, static void python_process_event(union perf_event *event, struct perf_sample *sample, struct evsel *evsel, - struct addr_location *al) + struct addr_location *al, + struct addr_location *addr_al) { struct tables *tables = &tables_global; + scripting_context__update(scripting_context, event, sample, evsel, al, addr_al); + switch (evsel->core.attr.type) { case PERF_TYPE_TRACEPOINT: - python_process_tracepoint(sample, evsel, al); + python_process_tracepoint(sample, evsel, al, addr_al); break; /* Reserve for future process_hw/sw/raw APIs */ default: if (tables->db_export_mode) - db_export__sample(&tables->dbe, event, sample, evsel, al); + db_export__sample(&tables->dbe, event, sample, evsel, al, addr_al); else - python_process_general_event(sample, evsel, al); + python_process_general_event(sample, evsel, al, addr_al); } } +static void python_do_process_switch(union perf_event *event, + struct perf_sample *sample, + struct machine *machine) +{ + const char *handler_name = "context_switch"; + bool out = event->header.misc & PERF_RECORD_MISC_SWITCH_OUT; + bool out_preempt = out && (event->header.misc & PERF_RECORD_MISC_SWITCH_OUT_PREEMPT); + pid_t np_pid = -1, np_tid = -1; + PyObject *handler, *t; + + handler = get_handler(handler_name); + if (!handler) + return; + + if (event->header.type == PERF_RECORD_SWITCH_CPU_WIDE) { + np_pid = event->context_switch.next_prev_pid; + np_tid = event->context_switch.next_prev_tid; + } + + t = tuple_new(9); + if (!t) + return; + + tuple_set_u64(t, 0, sample->time); + tuple_set_s32(t, 1, sample->cpu); + tuple_set_s32(t, 2, sample->pid); + tuple_set_s32(t, 3, sample->tid); + tuple_set_s32(t, 4, np_pid); + tuple_set_s32(t, 5, np_tid); + tuple_set_s32(t, 6, machine->pid); + tuple_set_bool(t, 7, out); + tuple_set_bool(t, 8, out_preempt); + + call_object(handler, t, handler_name); + + Py_DECREF(t); +} + static void python_process_switch(union perf_event *event, struct perf_sample *sample, struct machine *machine) @@ -1342,6 +1468,44 @@ static void python_process_switch(union perf_event *event, if (tables->db_export_mode) db_export__switch(&tables->dbe, event, sample, machine); + else + python_do_process_switch(event, sample, machine); +} + +static void python_process_auxtrace_error(struct perf_session *session __maybe_unused, + union perf_event *event) +{ + struct perf_record_auxtrace_error *e = &event->auxtrace_error; + u8 cpumode = e->header.misc & PERF_RECORD_MISC_CPUMODE_MASK; + const char *handler_name = "auxtrace_error"; + unsigned long long tm = e->time; + const char *msg = e->msg; + PyObject *handler, *t; + + handler = get_handler(handler_name); + if (!handler) + return; + + if (!e->fmt) { + tm = 0; + msg = (const char *)&e->time; + } + + t = tuple_new(9); + + tuple_set_u32(t, 0, e->type); + tuple_set_u32(t, 1, e->code); + tuple_set_s32(t, 2, e->cpu); + tuple_set_s32(t, 3, e->pid); + tuple_set_s32(t, 4, e->tid); + tuple_set_u64(t, 5, e->ip); + tuple_set_u64(t, 6, tm); + tuple_set_string(t, 7, msg); + tuple_set_u32(t, 8, cpumode); + + call_object(handler, t, handler_name); + + Py_DECREF(t); } static void get_handler_name(char *str, size_t size, @@ -1442,6 +1606,31 @@ static void python_process_stat_interval(u64 tstamp) Py_DECREF(t); } +static int perf_script_context_init(void) +{ + PyObject *perf_script_context; + PyObject *perf_trace_context; + PyObject *dict; + int ret; + + perf_trace_context = PyImport_AddModule("perf_trace_context"); + if (!perf_trace_context) + return -1; + dict = PyModule_GetDict(perf_trace_context); + if (!dict) + return -1; + + perf_script_context = _PyCapsule_New(scripting_context, NULL, NULL); + if (!perf_script_context) + return -1; + + ret = PyDict_SetItemString(dict, "perf_script_context", perf_script_context); + if (!ret) + ret = PyDict_SetItemString(main_dict, "perf_script_context", perf_script_context); + Py_DECREF(perf_script_context); + return ret; +} + static int run_start_sub(void) { main_module = PyImport_AddModule("__main__"); @@ -1454,6 +1643,9 @@ static int run_start_sub(void) goto error; Py_INCREF(main_dict); + if (perf_script_context_init()) + goto error; + try_call_object("trace_begin", NULL); return 0; @@ -1589,7 +1781,8 @@ static void _free_command_line(wchar_t **command_line, int num) /* * Start trace script */ -static int python_start_script(const char *script, int argc, const char **argv) +static int python_start_script(const char *script, int argc, const char **argv, + struct perf_session *session) { struct tables *tables = &tables_global; #if PY_MAJOR_VERSION < 3 @@ -1605,6 +1798,7 @@ static int python_start_script(const char *script, int argc, const char **argv) int i, err = 0; FILE *fp; + scripting_context->session = session; #if PY_MAJOR_VERSION < 3 command_line = malloc((argc + 1) * sizeof(const char *)); command_line[0] = script; @@ -1876,11 +2070,13 @@ static int python_generate_script(struct tep_handle *pevent, const char *outfile struct scripting_ops python_scripting_ops = { .name = "Python", + .dirname = "python", .start_script = python_start_script, .flush_script = python_flush_script, .stop_script = python_stop_script, .process_event = python_process_event, .process_switch = python_process_switch, + .process_auxtrace_error = python_process_auxtrace_error, .process_stat = python_process_stat, .process_stat_interval = python_process_stat_interval, .generate_script = python_generate_script, diff --git a/tools/perf/util/session.c b/tools/perf/util/session.c index 106b3d60881a..51f727402912 100644 --- a/tools/perf/util/session.c +++ b/tools/perf/util/session.c @@ -301,8 +301,12 @@ void perf_session__delete(struct perf_session *session) perf_session__release_decomp_events(session); perf_env__exit(&session->header.env); machines__exit(&session->machines); - if (session->data) + if (session->data) { + if (perf_data__is_read(session->data)) + evlist__delete(session->evlist); perf_data__close(session->data); + } + trace_event__cleanup(&session->tevent); free(session); } @@ -1723,6 +1727,7 @@ int perf_session__peek_event(struct perf_session *session, off_t file_offset, if (event->header.size < hdr_sz || event->header.size > buf_sz) return -1; + buf += hdr_sz; rest = event->header.size - hdr_sz; if (readn(fd, buf, rest) != (ssize_t)rest) @@ -2155,6 +2160,7 @@ struct reader { u64 data_size; u64 data_offset; reader_cb_t process; + bool in_place_update; }; static int @@ -2188,7 +2194,9 @@ reader__process_events(struct reader *rd, struct perf_session *session, mmap_prot = PROT_READ; mmap_flags = MAP_SHARED; - if (session->header.needs_swap) { + if (rd->in_place_update) { + mmap_prot |= PROT_WRITE; + } else if (session->header.needs_swap) { mmap_prot |= PROT_WRITE; mmap_flags = MAP_PRIVATE; } @@ -2274,6 +2282,7 @@ static int __perf_session__process_events(struct perf_session *session) .data_size = session->header.data_size, .data_offset = session->header.data_offset, .process = process_simple, + .in_place_update = session->data->in_place_update, }; struct ordered_events *oe = &session->ordered_events; struct perf_tool *tool = session->tool; diff --git a/tools/perf/util/sort.c b/tools/perf/util/sort.c index 88ce47f2547e..568a88c001c6 100644 --- a/tools/perf/util/sort.c +++ b/tools/perf/util/sort.c @@ -3370,7 +3370,7 @@ static void add_hpp_sort_string(struct strbuf *sb, struct hpp_dimension *s, int add_key(sb, s[i].name, llen); } -const char *sort_help(const char *prefix) +char *sort_help(const char *prefix) { struct strbuf sb; char *s; diff --git a/tools/perf/util/sort.h b/tools/perf/util/sort.h index 87a092645aa7..b67c469aba79 100644 --- a/tools/perf/util/sort.h +++ b/tools/perf/util/sort.h @@ -302,7 +302,7 @@ void reset_output_field(void); void sort__setup_elide(FILE *fp); void perf_hpp__set_elide(int idx, bool elide); -const char *sort_help(const char *prefix); +char *sort_help(const char *prefix); int report_parse_ignore_callees_opt(const struct option *opt, const char *arg, int unset); diff --git a/tools/perf/util/srccode.c b/tools/perf/util/srccode.c index c29edaaca863..476e99896d5e 100644 --- a/tools/perf/util/srccode.c +++ b/tools/perf/util/srccode.c @@ -97,8 +97,7 @@ static struct srcfile *find_srcfile(char *fn) hlist_for_each_entry (h, &srcfile_htab[hval], hash_nd) { if (!strcmp(fn, h->fn)) { /* Move to front */ - list_del(&h->nd); - list_add(&h->nd, &srcfile_list); + list_move(&h->nd, &srcfile_list); return h; } } diff --git a/tools/perf/util/stat-display.c b/tools/perf/util/stat-display.c index a76fff5e7d83..588601000f3f 100644 --- a/tools/perf/util/stat-display.c +++ b/tools/perf/util/stat-display.c @@ -19,6 +19,7 @@ #include "util.h" #include "iostat.h" #include "pmu-hybrid.h" +#include "evlist-hybrid.h" #define CNTR_NOT_SUPPORTED "<not supported>" #define CNTR_NOT_COUNTED "<not counted>" @@ -465,9 +466,11 @@ static void printout(struct perf_stat_config *config, struct aggr_cpu_id id, int config->csv_sep); if (counter->supported) { - config->print_free_counters_hint = 1; - if (is_mixed_hw_group(counter)) - config->print_mixed_hw_group_error = 1; + if (!evlist__has_hybrid(counter->evlist)) { + config->print_free_counters_hint = 1; + if (is_mixed_hw_group(counter)) + config->print_mixed_hw_group_error = 1; + } } fprintf(config->output, "%-*s%s", @@ -541,7 +544,7 @@ static void uniquify_event_name(struct evsel *counter) char *config; int ret = 0; - if (counter->uniquified_name || + if (counter->uniquified_name || counter->use_config_name || !counter->pmu_name || !strncmp(counter->name, counter->pmu_name, strlen(counter->pmu_name))) return; @@ -555,10 +558,8 @@ static void uniquify_event_name(struct evsel *counter) } } else { if (perf_pmu__has_hybrid()) { - if (!counter->use_config_name) { - ret = asprintf(&new_name, "%s/%s/", - counter->pmu_name, counter->name); - } + ret = asprintf(&new_name, "%s/%s/", + counter->pmu_name, counter->name); } else { ret = asprintf(&new_name, "%s [%s]", counter->name, counter->pmu_name); @@ -595,6 +596,18 @@ static void collect_all_aliases(struct perf_stat_config *config, struct evsel *c } } +static bool is_uncore(struct evsel *evsel) +{ + struct perf_pmu *pmu = evsel__find_pmu(evsel); + + return pmu && pmu->is_uncore; +} + +static bool hybrid_uniquify(struct evsel *evsel) +{ + return perf_pmu__has_hybrid() && !is_uncore(evsel); +} + static bool collect_data(struct perf_stat_config *config, struct evsel *counter, void (*cb)(struct perf_stat_config *config, struct evsel *counter, void *data, bool first), @@ -603,7 +616,7 @@ static bool collect_data(struct perf_stat_config *config, struct evsel *counter, if (counter->merged_stat) return false; cb(config, counter, data, true); - if (config->no_merge) + if (config->no_merge || hybrid_uniquify(counter)) uniquify_event_name(counter); else if (counter->auto_merge_stats) collect_all_aliases(config, counter, cb, data); @@ -827,11 +840,11 @@ static void counter_aggr_cb(struct perf_stat_config *config __maybe_unused, bool first __maybe_unused) { struct caggr_data *cd = data; - struct perf_stat_evsel *ps = counter->stats; + struct perf_counts_values *aggr = &counter->counts->aggr; - cd->avg += avg_stats(&ps->res_stats[0]); - cd->avg_enabled += avg_stats(&ps->res_stats[1]); - cd->avg_running += avg_stats(&ps->res_stats[2]); + cd->avg += aggr->val; + cd->avg_enabled += aggr->ena; + cd->avg_running += aggr->run; } /* diff --git a/tools/perf/util/stat-shadow.c b/tools/perf/util/stat-shadow.c index 39967a45f55b..34a7f5c1fff7 100644 --- a/tools/perf/util/stat-shadow.c +++ b/tools/perf/util/stat-shadow.c @@ -379,7 +379,7 @@ void perf_stat__collect_metric_expr(struct evlist *evsel_list) evlist__for_each_entry(evsel_list, counter) { bool invalid = false; - leader = counter->leader; + leader = evsel__leader(counter); if (!counter->metric_expr) continue; diff --git a/tools/perf/util/stat.c b/tools/perf/util/stat.c index 2db46b9bebd0..09ea334586f2 100644 --- a/tools/perf/util/stat.c +++ b/tools/perf/util/stat.c @@ -437,18 +437,6 @@ int perf_stat_process_counter(struct perf_stat_config *config, aggr->val = aggr->ena = aggr->run = 0; - /* - * We calculate counter's data every interval, - * and the display code shows ps->res_stats - * avg value. We need to zero the stats for - * interval mode, otherwise overall avg running - * averages will be shown for each interval. - */ - if (config->interval || config->summary) { - for (i = 0; i < 3; i++) - init_stats(&ps->res_stats[i]); - } - if (counter->per_pkg) evsel__zero_per_pkg(counter); @@ -546,7 +534,7 @@ int create_perf_stat_counter(struct evsel *evsel, int cpu) { struct perf_event_attr *attr = &evsel->core.attr; - struct evsel *leader = evsel->leader; + struct evsel *leader = evsel__leader(evsel); attr->read_format = PERF_FORMAT_TOTAL_TIME_ENABLED | PERF_FORMAT_TOTAL_TIME_RUNNING; diff --git a/tools/perf/util/stream.c b/tools/perf/util/stream.c index 4bd5e5a00aa5..545e44981a27 100644 --- a/tools/perf/util/stream.c +++ b/tools/perf/util/stream.c @@ -139,7 +139,7 @@ static int evlist__init_callchain_streams(struct evlist *evlist, hists__output_resort(hists, NULL); init_hot_callchain(hists, &es[i]); - es[i].evsel_idx = pos->idx; + es[i].evsel_idx = pos->core.idx; i++; } diff --git a/tools/perf/util/symbol-elf.c b/tools/perf/util/symbol-elf.c index 4c56aa837434..31cd59a2b66e 100644 --- a/tools/perf/util/symbol-elf.c +++ b/tools/perf/util/symbol-elf.c @@ -1074,14 +1074,15 @@ static int dso__process_kernel_symbol(struct dso *dso, struct map *map, return 0; } -int dso__load_sym(struct dso *dso, struct map *map, struct symsrc *syms_ss, - struct symsrc *runtime_ss, int kmodule) +static int +dso__load_sym_internal(struct dso *dso, struct map *map, struct symsrc *syms_ss, + struct symsrc *runtime_ss, int kmodule, int dynsym) { struct kmap *kmap = dso->kernel ? map__kmap(map) : NULL; struct maps *kmaps = kmap ? map__kmaps(map) : NULL; struct map *curr_map = map; struct dso *curr_dso = dso; - Elf_Data *symstrs, *secstrs; + Elf_Data *symstrs, *secstrs, *secstrs_run, *secstrs_sym; uint32_t nr_syms; int err = -1; uint32_t idx; @@ -1098,34 +1099,15 @@ int dso__load_sym(struct dso *dso, struct map *map, struct symsrc *syms_ss, if (kmap && !kmaps) return -1; - dso->symtab_type = syms_ss->type; - dso->is_64_bit = syms_ss->is_64_bit; - dso->rel = syms_ss->ehdr.e_type == ET_REL; - - /* - * Modules may already have symbols from kallsyms, but those symbols - * have the wrong values for the dso maps, so remove them. - */ - if (kmodule && syms_ss->symtab) - symbols__delete(&dso->symbols); - - if (!syms_ss->symtab) { - /* - * If the vmlinux is stripped, fail so we will fall back - * to using kallsyms. The vmlinux runtime symbols aren't - * of much use. - */ - if (dso->kernel) - goto out_elf_end; - - syms_ss->symtab = syms_ss->dynsym; - syms_ss->symshdr = syms_ss->dynshdr; - } - elf = syms_ss->elf; ehdr = syms_ss->ehdr; - sec = syms_ss->symtab; - shdr = syms_ss->symshdr; + if (dynsym) { + sec = syms_ss->dynsym; + shdr = syms_ss->dynshdr; + } else { + sec = syms_ss->symtab; + shdr = syms_ss->symshdr; + } if (elf_section_by_name(runtime_ss->elf, &runtime_ss->ehdr, &tshdr, ".text", NULL)) @@ -1150,8 +1132,16 @@ int dso__load_sym(struct dso *dso, struct map *map, struct symsrc *syms_ss, if (sec_strndx == NULL) goto out_elf_end; - secstrs = elf_getdata(sec_strndx, NULL); - if (secstrs == NULL) + secstrs_run = elf_getdata(sec_strndx, NULL); + if (secstrs_run == NULL) + goto out_elf_end; + + sec_strndx = elf_getscn(elf, ehdr.e_shstrndx); + if (sec_strndx == NULL) + goto out_elf_end; + + secstrs_sym = elf_getdata(sec_strndx, NULL); + if (secstrs_sym == NULL) goto out_elf_end; nr_syms = shdr.sh_size / shdr.sh_entsize; @@ -1237,6 +1227,8 @@ int dso__load_sym(struct dso *dso, struct map *map, struct symsrc *syms_ss, gelf_getshdr(sec, &shdr); + secstrs = secstrs_sym; + /* * We have to fallback to runtime when syms' section header has * NOBITS set. NOBITS results in file offset (sh_offset) not @@ -1249,6 +1241,7 @@ int dso__load_sym(struct dso *dso, struct map *map, struct symsrc *syms_ss, goto out_elf_end; gelf_getshdr(sec, &shdr); + secstrs = secstrs_run; } if (is_label && !elf_sec__filter(&shdr, secstrs)) @@ -1312,6 +1305,50 @@ out_elf_end: return err; } +int dso__load_sym(struct dso *dso, struct map *map, struct symsrc *syms_ss, + struct symsrc *runtime_ss, int kmodule) +{ + int nr = 0; + int err = -1; + + dso->symtab_type = syms_ss->type; + dso->is_64_bit = syms_ss->is_64_bit; + dso->rel = syms_ss->ehdr.e_type == ET_REL; + + /* + * Modules may already have symbols from kallsyms, but those symbols + * have the wrong values for the dso maps, so remove them. + */ + if (kmodule && syms_ss->symtab) + symbols__delete(&dso->symbols); + + if (!syms_ss->symtab) { + /* + * If the vmlinux is stripped, fail so we will fall back + * to using kallsyms. The vmlinux runtime symbols aren't + * of much use. + */ + if (dso->kernel) + return err; + } else { + err = dso__load_sym_internal(dso, map, syms_ss, runtime_ss, + kmodule, 0); + if (err < 0) + return err; + nr = err; + } + + if (syms_ss->dynsym) { + err = dso__load_sym_internal(dso, map, syms_ss, runtime_ss, + kmodule, 1); + if (err < 0) + return err; + err += nr; + } + + return err; +} + static int elf_read_maps(Elf *elf, bool exe, mapfn_t mapfn, void *data) { GElf_Phdr phdr; @@ -2412,6 +2449,7 @@ int cleanup_sdt_note_list(struct list_head *sdt_notes) list_for_each_entry_safe(pos, tmp, sdt_notes, note_list) { list_del_init(&pos->note_list); + zfree(&pos->args); zfree(&pos->name); zfree(&pos->provider); free(pos); diff --git a/tools/perf/util/trace-event-scripting.c b/tools/perf/util/trace-event-scripting.c index 714581b0de65..7172ca05265f 100644 --- a/tools/perf/util/trace-event-scripting.c +++ b/tools/perf/util/trace-event-scripting.c @@ -12,10 +12,31 @@ #include "debug.h" #include "trace-event.h" +#include "event.h" +#include "evsel.h" #include <linux/zalloc.h> struct scripting_context *scripting_context; +void scripting_context__update(struct scripting_context *c, + union perf_event *event, + struct perf_sample *sample, + struct evsel *evsel, + struct addr_location *al, + struct addr_location *addr_al) +{ + c->event_data = sample->raw_data; + if (evsel->tp_format) + c->pevent = evsel->tp_format->tep; + else + c->pevent = NULL; + c->event = event; + c->sample = sample; + c->evsel = evsel; + c->al = al; + c->addr_al = addr_al; +} + static int flush_script_unsupported(void) { return 0; @@ -29,7 +50,8 @@ static int stop_script_unsupported(void) static void process_event_unsupported(union perf_event *event __maybe_unused, struct perf_sample *sample __maybe_unused, struct evsel *evsel __maybe_unused, - struct addr_location *al __maybe_unused) + struct addr_location *al __maybe_unused, + struct addr_location *addr_al __maybe_unused) { } @@ -44,7 +66,8 @@ static void print_python_unsupported_msg(void) static int python_start_script_unsupported(const char *script __maybe_unused, int argc __maybe_unused, - const char **argv __maybe_unused) + const char **argv __maybe_unused, + struct perf_session *session __maybe_unused) { print_python_unsupported_msg(); @@ -63,6 +86,7 @@ static int python_generate_script_unsupported(struct tep_handle *pevent struct scripting_ops python_scripting_unsupported_ops = { .name = "Python", + .dirname = "python", .start_script = python_start_script_unsupported, .flush_script = flush_script_unsupported, .stop_script = stop_script_unsupported, @@ -108,7 +132,8 @@ static void print_perl_unsupported_msg(void) static int perl_start_script_unsupported(const char *script __maybe_unused, int argc __maybe_unused, - const char **argv __maybe_unused) + const char **argv __maybe_unused, + struct perf_session *session __maybe_unused) { print_perl_unsupported_msg(); @@ -126,6 +151,7 @@ static int perl_generate_script_unsupported(struct tep_handle *pevent struct scripting_ops perl_scripting_unsupported_ops = { .name = "Perl", + .dirname = "perl", .start_script = perl_start_script_unsupported, .flush_script = flush_script_unsupported, .stop_script = stop_script_unsupported, diff --git a/tools/perf/util/trace-event.h b/tools/perf/util/trace-event.h index 72fdf2a3577c..54aadeedf28c 100644 --- a/tools/perf/util/trace-event.h +++ b/tools/perf/util/trace-event.h @@ -11,6 +11,7 @@ union perf_event; struct perf_tool; struct thread; struct tep_plugin_list; +struct evsel; struct trace_event { struct tep_handle *pevent; @@ -71,16 +72,21 @@ struct perf_stat_config; struct scripting_ops { const char *name; - int (*start_script) (const char *script, int argc, const char **argv); + const char *dirname; /* For script path .../scripts/<dirname>/... */ + int (*start_script)(const char *script, int argc, const char **argv, + struct perf_session *session); int (*flush_script) (void); int (*stop_script) (void); void (*process_event) (union perf_event *event, struct perf_sample *sample, struct evsel *evsel, - struct addr_location *al); + struct addr_location *al, + struct addr_location *addr_al); void (*process_switch)(union perf_event *event, struct perf_sample *sample, struct machine *machine); + void (*process_auxtrace_error)(struct perf_session *session, + union perf_event *event); void (*process_stat)(struct perf_stat_config *config, struct evsel *evsel, u64 tstamp); void (*process_stat_interval)(u64 tstamp); @@ -91,16 +97,35 @@ extern unsigned int scripting_max_stack; int script_spec_register(const char *spec, struct scripting_ops *ops); +void script_fetch_insn(struct perf_sample *sample, struct thread *thread, + struct machine *machine); + void setup_perl_scripting(void); void setup_python_scripting(void); struct scripting_context { struct tep_handle *pevent; void *event_data; + union perf_event *event; + struct perf_sample *sample; + struct evsel *evsel; + struct addr_location *al; + struct addr_location *addr_al; + struct perf_session *session; }; +void scripting_context__update(struct scripting_context *scripting_context, + union perf_event *event, + struct perf_sample *sample, + struct evsel *evsel, + struct addr_location *al, + struct addr_location *addr_al); + int common_pc(struct scripting_context *context); int common_flags(struct scripting_context *context); int common_lock_depth(struct scripting_context *context); +#define SAMPLE_FLAGS_BUF_SIZE 64 +int perf_sample__sprintf_flags(u32 flags, char *str, size_t sz); + #endif /* _PERF_UTIL_TRACE_EVENT_H */ |