perf lock contention: Allow to change stack depth and skip

It needs stack traces to find callers of locks.  To minimize the
performance overhead it only collects up to 8 entries for each stack
trace.  And it skips first 3 entries as they came from BPF, tracepoint
and lock functions which are not interested for most users.

But it turned out that those numbers are different in some
configuration.  Using fixed number can result in non meaningful caller
names.  Let's make them adjustable with --stack-depth and --skip-stack
options.

On my setup, the default output is like below:

  # /perf lock con -ab -F contended,wait_total sleep 3
   contended   total wait         type   caller

          28      4.55 ms     rwlock:W   __bpf_trace_contention_begin+0xb
          33      1.67 ms     rwlock:W   __bpf_trace_contention_begin+0xb
          12    580.28 us     spinlock   __bpf_trace_contention_begin+0xb
          60    240.54 us      rwsem:R   __bpf_trace_contention_begin+0xb
          27     64.45 us     spinlock   __bpf_trace_contention_begin+0xb

If I change the stack skip to 5, the result will be like:

  # perf lock con -ab -F contended,wait_total --stack-skip 5 sleep 3
   contended   total wait         type   caller

          32    715.45 us     spinlock   folio_lruvec_lock_irqsave+0x61
          26    550.22 us     spinlock   folio_lruvec_lock_irqsave+0x61
          15    486.93 us      rwsem:R   mmap_read_lock+0x13
          12    139.66 us      rwsem:W   vm_mmap_pgoff+0x93
           1      7.04 us     spinlock   tick_do_update_jiffies64+0x25

Signed-off-by: Namhyung Kim <namhyung@kernel.org>
Cc: Adrian Hunter <adrian.hunter@intel.com>
Cc: Ian Rogers <irogers@google.com>
Cc: Ingo Molnar <mingo@kernel.org>
Cc: Jiri Olsa <jolsa@kernel.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Song Liu <songliubraving@fb.com>
Cc: bpf@vger.kernel.org
Link: https://lore.kernel.org/r/20220912055314.744552-4-namhyung@kernel.org
Signed-off-by: Arnaldo Carvalho de Melo <acme@redhat.com>

diff --git a/tools/perf/Documentation/perf-lock.txt b/tools/perf/Documentation/perf-lock.txt
index 193c5d8..5f2dc63 100644 (file)
--- a/tools/perf/Documentation/perf-lock.txt
+++ b/tools/perf/Documentation/perf-lock.txt
@@ -148,6 +148,12 @@ CONTENTION OPTIONS
 --map-nr-entries::
        Maximum number of BPF map entries (default: 10240).
 
+--max-stack::
+       Maximum stack depth when collecting lock contention (default: 8).
+
+--stack-skip
+       Number of stack depth to skip when finding a lock caller (default: 3).
+
 
 SEE ALSO
 --------

diff --git a/tools/perf/builtin-lock.c b/tools/perf/builtin-lock.c
index 3715390..25d75fa 100644 (file)
--- a/tools/perf/builtin-lock.c
+++ b/tools/perf/builtin-lock.c
@@ -56,6 +56,8 @@ static bool combine_locks;
 static bool show_thread_stats;
 static bool use_bpf;
 static unsigned long bpf_map_entries = 10240;
+static int max_stack_depth = CONTENTION_STACK_DEPTH;
+static int stack_skip = CONTENTION_STACK_SKIP;
 
 static enum {
        LOCK_AGGR_ADDR,
@@ -936,7 +938,7 @@ static int lock_contention_caller(struct evsel *evsel, struct perf_sample *sampl
 
        /* use caller function name from the callchain */
        ret = thread__resolve_callchain(thread, cursor, evsel, sample,
-                                       NULL, NULL, CONTENTION_STACK_DEPTH);
+                                       NULL, NULL, max_stack_depth);
        if (ret != 0) {
                thread__put(thread);
                return -1;
@@ -953,7 +955,7 @@ static int lock_contention_caller(struct evsel *evsel, struct perf_sample *sampl
                        break;
 
                /* skip first few entries - for lock functions */
-               if (++skip <= CONTENTION_STACK_SKIP)
+               if (++skip <= stack_skip)
                        goto next;
 
                sym = node->ms.sym;
@@ -984,7 +986,7 @@ static u64 callchain_id(struct evsel *evsel, struct perf_sample *sample)
 
        /* use caller function name from the callchain */
        ret = thread__resolve_callchain(thread, cursor, evsel, sample,
-                                       NULL, NULL, CONTENTION_STACK_DEPTH);
+                                       NULL, NULL, max_stack_depth);
        thread__put(thread);
 
        if (ret != 0)
@@ -1000,7 +1002,7 @@ static u64 callchain_id(struct evsel *evsel, struct perf_sample *sample)
                        break;
 
                /* skip first few entries - for lock functions */
-               if (++skip <= CONTENTION_STACK_SKIP)
+               if (++skip <= stack_skip)
                        goto next;
 
                if (node->ms.sym && is_lock_function(machine, node->ip))
@@ -1063,7 +1065,7 @@ static int report_lock_contention_begin_event(struct evsel *evsel,
                        return -ENOMEM;
 
                if (aggr_mode == LOCK_AGGR_CALLER && verbose) {
-                       ls->callstack = get_callstack(sample, CONTENTION_STACK_DEPTH);
+                       ls->callstack = get_callstack(sample, max_stack_depth);
                        if (ls->callstack == NULL)
                                return -ENOMEM;
                }
@@ -1515,7 +1517,7 @@ static void print_contention_result(struct lock_contention *con)
                        char buf[128];
                        u64 ip;
 
-                       for (int i = 0; i < CONTENTION_STACK_DEPTH; i++) {
+                       for (int i = 0; i < max_stack_depth; i++) {
                                if (!st->callstack || !st->callstack[i])
                                        break;
 
@@ -1632,6 +1634,8 @@ static int __cmd_contention(int argc, const char **argv)
                .target = &target,
                .result = &lockhash_table[0],
                .map_nr_entries = bpf_map_entries,
+               .max_stack = max_stack_depth,
+               .stack_skip = stack_skip,
        };
 
        session = perf_session__new(use_bpf ? NULL : &data, &eops);
@@ -1895,6 +1899,12 @@ int cmd_lock(int argc, const char **argv)
                   "Trace on existing thread id (exclusive to --pid)"),
        OPT_CALLBACK(0, "map-nr-entries", &bpf_map_entries, "num",
                     "Max number of BPF map entries", parse_map_entry),
+       OPT_INTEGER(0, "max-stack", &max_stack_depth,
+                   "Set the maximum stack depth when collecting lock contention, "
+                   "Default: " __stringify(CONTENTION_STACK_DEPTH)),
+       OPT_INTEGER(0, "stack-skip", &stack_skip,
+                   "Set the number of stack depth to skip when finding a lock caller, "
+                   "Default: " __stringify(CONTENTION_STACK_SKIP)),
        OPT_PARENT(lock_options)
        };
 

diff --git a/tools/perf/util/bpf_lock_contention.c b/tools/perf/util/bpf_lock_contention.c
index 6545bee..ef5323c 100644 (file)
--- a/tools/perf/util/bpf_lock_contention.c
+++ b/tools/perf/util/bpf_lock_contention.c
@@ -41,6 +41,7 @@ int lock_contention_prepare(struct lock_contention *con)
                return -1;
        }
 
+       bpf_map__set_value_size(skel->maps.stacks, con->max_stack * sizeof(u64));
        bpf_map__set_max_entries(skel->maps.stacks, con->map_nr_entries);
        bpf_map__set_max_entries(skel->maps.lock_stat, con->map_nr_entries);
 
@@ -115,7 +116,7 @@ int lock_contention_read(struct lock_contention *con)
        struct lock_contention_data data;
        struct lock_stat *st;
        struct machine *machine = con->machine;
-       u64 stack_trace[CONTENTION_STACK_DEPTH];
+       u64 stack_trace[con->max_stack];
 
        fd = bpf_map__fd(skel->maps.lock_stat);
        stack = bpf_map__fd(skel->maps.stacks);
@@ -146,9 +147,9 @@ int lock_contention_read(struct lock_contention *con)
                bpf_map_lookup_elem(stack, &key, stack_trace);
 
                /* skip BPF + lock internal functions */
-               idx = CONTENTION_STACK_SKIP;
+               idx = con->stack_skip;
                while (is_lock_function(machine, stack_trace[idx]) &&
-                      idx < CONTENTION_STACK_DEPTH - 1)
+                      idx < con->max_stack - 1)
                        idx++;
 
                st->addr = stack_trace[idx];

diff --git a/tools/perf/util/lock-contention.h b/tools/perf/util/lock-contention.h
index bdb6e2a..67db311 100644 (file)
--- a/tools/perf/util/lock-contention.h
+++ b/tools/perf/util/lock-contention.h
@@ -115,6 +115,8 @@ struct lock_contention {
        struct hlist_head *result;
        unsigned long map_nr_entries;
        unsigned long lost;
+       int max_stack;
+       int stack_skip;
 };
 
 #ifdef HAVE_BPF_SKEL