tools/perf/util/bpf_skel/func_latency.bpf.c

   1 // SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause)
   2 // Copyright (c) 2021 Google
   3 #include "vmlinux.h"
   4 #include <bpf/bpf_helpers.h>
   5 #include <bpf/bpf_tracing.h>
   6
   7 // This should be in sync with "util/ftrace.h"
   8 #define NUM_BUCKET  22
   9
  10 struct {
  11         __uint(type, BPF_MAP_TYPE_HASH);
  12         __uint(key_size, sizeof(__u64));
  13         __uint(value_size, sizeof(__u64));
  14         __uint(max_entries, 10000);
  15 } functime SEC(".maps");
  16
  17 struct {
  18         __uint(type, BPF_MAP_TYPE_HASH);
  19         __uint(key_size, sizeof(__u32));
  20         __uint(value_size, sizeof(__u8));
  21         __uint(max_entries, 1);
  22 } cpu_filter SEC(".maps");
  23
  24 struct {
  25         __uint(type, BPF_MAP_TYPE_HASH);
  26         __uint(key_size, sizeof(__u32));
  27         __uint(value_size, sizeof(__u8));
  28         __uint(max_entries, 1);
  29 } task_filter SEC(".maps");
  30
  31 struct {
  32         __uint(type, BPF_MAP_TYPE_PERCPU_ARRAY);
  33         __uint(key_size, sizeof(__u32));
  34         __uint(value_size, sizeof(__u64));
  35         __uint(max_entries, NUM_BUCKET);
  36 } latency SEC(".maps");
  37
  38
  39 int enabled = 0;
  40
  41 // stats
  42 __s64 total;
  43 __s64 count;
  44 __s64 max;
  45 __s64 min;
  46
  47 const volatile int has_cpu = 0;
  48 const volatile int has_task = 0;
  49 const volatile int use_nsec = 0;
  50 const volatile unsigned int bucket_range;
  51 const volatile unsigned int min_latency;
  52 const volatile unsigned int max_latency;
  53
  54 SEC("kprobe/func")
  55 int BPF_PROG(func_begin)
  56 {
  57         __u64 key, now;
  58
  59         if (!enabled)
  60                 return 0;
  61
  62         key = bpf_get_current_pid_tgid();
  63
  64         if (has_cpu) {
  65                 __u32 cpu = bpf_get_smp_processor_id();
  66                 __u8 *ok;
  67
  68                 ok = bpf_map_lookup_elem(&cpu_filter, &cpu);
  69                 if (!ok)
  70                         return 0;
  71         }
  72
  73         if (has_task) {
  74                 __u32 pid = key & 0xffffffff;
  75                 __u8 *ok;
  76
  77                 ok = bpf_map_lookup_elem(&task_filter, &pid);
  78                 if (!ok)
  79                         return 0;
  80         }
  81
  82         now = bpf_ktime_get_ns();
  83
  84         // overwrite timestamp for nested functions
  85         bpf_map_update_elem(&functime, &key, &now, BPF_ANY);
  86         return 0;
  87 }
  88
  89 SEC("kretprobe/func")
  90 int BPF_PROG(func_end)
  91 {
  92         __u64 tid;
  93         __u64 *start;
  94         __u64 cmp_base = use_nsec ? 1 : 1000;
  95
  96         if (!enabled)
  97                 return 0;
  98
  99         tid = bpf_get_current_pid_tgid();
 100
 101         start = bpf_map_lookup_elem(&functime, &tid);
 102         if (start) {
 103                 __s64 delta = bpf_ktime_get_ns() - *start;
 104                 __u32 key = 0;
 105                 __u64 *hist;
 106
 107                 bpf_map_delete_elem(&functime, &tid);
 108
 109                 if (delta < 0)
 110                         return 0;
 111
 112                 if (bucket_range != 0) {
 113                         delta /= cmp_base;
 114
 115                         if (min_latency > 0) {
 116                                 if (delta > min_latency)
 117                                         delta -= min_latency;
 118                                 else
 119                                         goto do_lookup;
 120                         }
 121
 122                         // Less than 1 unit (ms or ns), or, in the future,
 123                         // than the min latency desired.
 124                         if (delta > 0) { // 1st entry: [ 1 unit .. bucket_range units )
 125                                 // clang 12 doesn't like s64 / u32 division
 126                                 key = (__u64)delta / bucket_range + 1;
 127                                 if (key >= NUM_BUCKET ||
 128                                         delta >= max_latency - min_latency)
 129                                         key = NUM_BUCKET - 1;
 130                         }
 131
 132                         delta += min_latency;
 133                         goto do_lookup;
 134                 }
 135                 // calculate index using delta
 136                 for (key = 0; key < (NUM_BUCKET - 1); key++) {
 137                         if (delta < (cmp_base << key))
 138                                 break;
 139                 }
 140
 141 do_lookup:
 142                 hist = bpf_map_lookup_elem(&latency, &key);
 143                 if (!hist)
 144                         return 0;
 145
 146                 *hist += 1;
 147
 148                 if (bucket_range == 0)
 149                         delta /= cmp_base;
 150
 151                 __sync_fetch_and_add(&total, delta);
 152                 __sync_fetch_and_add(&count, 1);
 153
 154                 if (delta > max)
 155                         max = delta;
 156                 if (delta < min)
 157                         min = delta;
 158         }
 159
 160         return 0;
 161 }