tools/testing/selftests/bpf/prog_tests/ringbuf.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 #define _GNU_SOURCE
   3 #include <linux/compiler.h>
   4 #include <asm/barrier.h>
   5 #include <test_progs.h>
   6 #include <sys/mman.h>
   7 #include <sys/epoll.h>
   8 #include <time.h>
   9 #include <sched.h>
  10 #include <signal.h>
  11 #include <pthread.h>
  12 #include <sys/sysinfo.h>
  13 #include <linux/perf_event.h>
  14 #include <linux/ring_buffer.h>
  15
  16 #include "test_ringbuf.lskel.h"
  17 #include "test_ringbuf_n.lskel.h"
  18 #include "test_ringbuf_map_key.lskel.h"
  19 #include "test_ringbuf_write.lskel.h"
  20
  21 #define EDONE 7777
  22
  23 static int duration = 0;
  24
  25 struct sample {
  26         int pid;
  27         int seq;
  28         long value;
  29         char comm[16];
  30 };
  31
  32 static int sample_cnt;
  33
  34 static void atomic_inc(int *cnt)
  35 {
  36         __atomic_add_fetch(cnt, 1, __ATOMIC_SEQ_CST);
  37 }
  38
  39 static int atomic_xchg(int *cnt, int val)
  40 {
  41         return __atomic_exchange_n(cnt, val, __ATOMIC_SEQ_CST);
  42 }
  43
  44 static int process_sample(void *ctx, void *data, size_t len)
  45 {
  46         struct sample *s = data;
  47
  48         atomic_inc(&sample_cnt);
  49
  50         switch (s->seq) {
  51         case 0:
  52                 CHECK(s->value != 333, "sample1_value", "exp %ld, got %ld\n",
  53                       333L, s->value);
  54                 return 0;
  55         case 1:
  56                 CHECK(s->value != 777, "sample2_value", "exp %ld, got %ld\n",
  57                       777L, s->value);
  58                 return -EDONE;
  59         default:
  60                 /* we don't care about the rest */
  61                 return 0;
  62         }
  63 }
  64
  65 static struct test_ringbuf_map_key_lskel *skel_map_key;
  66 static struct test_ringbuf_lskel *skel;
  67 static struct ring_buffer *ringbuf;
  68
  69 static void trigger_samples()
  70 {
  71         skel->bss->dropped = 0;
  72         skel->bss->total = 0;
  73         skel->bss->discarded = 0;
  74
  75         /* trigger exactly two samples */
  76         skel->bss->value = 333;
  77         syscall(__NR_getpgid);
  78         skel->bss->value = 777;
  79         syscall(__NR_getpgid);
  80 }
  81
  82 static void *poll_thread(void *input)
  83 {
  84         long timeout = (long)input;
  85
  86         return (void *)(long)ring_buffer__poll(ringbuf, timeout);
  87 }
  88
  89 static void ringbuf_write_subtest(void)
  90 {
  91         struct test_ringbuf_write_lskel *skel;
  92         int page_size = getpagesize();
  93         size_t *mmap_ptr;
  94         int err, rb_fd;
  95
  96         skel = test_ringbuf_write_lskel__open();
  97         if (!ASSERT_OK_PTR(skel, "skel_open"))
  98                 return;
  99
 100         skel->maps.ringbuf.max_entries = 0x4000;
 101
 102         err = test_ringbuf_write_lskel__load(skel);
 103         if (!ASSERT_OK(err, "skel_load"))
 104                 goto cleanup;
 105
 106         rb_fd = skel->maps.ringbuf.map_fd;
 107
 108         mmap_ptr = mmap(NULL, page_size, PROT_READ | PROT_WRITE, MAP_SHARED, rb_fd, 0);
 109         if (!ASSERT_OK_PTR(mmap_ptr, "rw_cons_pos"))
 110                 goto cleanup;
 111         *mmap_ptr = 0x3000;
 112         ASSERT_OK(munmap(mmap_ptr, page_size), "unmap_rw");
 113
 114         skel->bss->pid = getpid();
 115
 116         ringbuf = ring_buffer__new(rb_fd, process_sample, NULL, NULL);
 117         if (!ASSERT_OK_PTR(ringbuf, "ringbuf_new"))
 118                 goto cleanup;
 119
 120         err = test_ringbuf_write_lskel__attach(skel);
 121         if (!ASSERT_OK(err, "skel_attach"))
 122                 goto cleanup_ringbuf;
 123
 124         skel->bss->discarded = 0;
 125         skel->bss->passed = 0;
 126
 127         /* trigger exactly two samples */
 128         syscall(__NR_getpgid);
 129         syscall(__NR_getpgid);
 130
 131         ASSERT_EQ(skel->bss->discarded, 2, "discarded");
 132         ASSERT_EQ(skel->bss->passed, 0, "passed");
 133
 134         test_ringbuf_write_lskel__detach(skel);
 135 cleanup_ringbuf:
 136         ring_buffer__free(ringbuf);
 137 cleanup:
 138         test_ringbuf_write_lskel__destroy(skel);
 139 }
 140
 141 static void ringbuf_subtest(void)
 142 {
 143         const size_t rec_sz = BPF_RINGBUF_HDR_SZ + sizeof(struct sample);
 144         pthread_t thread;
 145         long bg_ret = -1;
 146         int err, cnt, rb_fd;
 147         int page_size = getpagesize();
 148         void *mmap_ptr, *tmp_ptr;
 149         struct ring *ring;
 150         int map_fd;
 151         unsigned long avail_data, ring_size, cons_pos, prod_pos;
 152
 153         skel = test_ringbuf_lskel__open();
 154         if (CHECK(!skel, "skel_open", "skeleton open failed\n"))
 155                 return;
 156
 157         skel->maps.ringbuf.max_entries = page_size;
 158
 159         err = test_ringbuf_lskel__load(skel);
 160         if (CHECK(err != 0, "skel_load", "skeleton load failed\n"))
 161                 goto cleanup;
 162
 163         rb_fd = skel->maps.ringbuf.map_fd;
 164         /* good read/write cons_pos */
 165         mmap_ptr = mmap(NULL, page_size, PROT_READ | PROT_WRITE, MAP_SHARED, rb_fd, 0);
 166         ASSERT_OK_PTR(mmap_ptr, "rw_cons_pos");
 167         tmp_ptr = mremap(mmap_ptr, page_size, 2 * page_size, MREMAP_MAYMOVE);
 168         if (!ASSERT_ERR_PTR(tmp_ptr, "rw_extend"))
 169                 goto cleanup;
 170         ASSERT_ERR(mprotect(mmap_ptr, page_size, PROT_EXEC), "exec_cons_pos_protect");
 171         ASSERT_OK(munmap(mmap_ptr, page_size), "unmap_rw");
 172
 173         /* bad writeable prod_pos */
 174         mmap_ptr = mmap(NULL, page_size, PROT_WRITE, MAP_SHARED, rb_fd, page_size);
 175         err = -errno;
 176         ASSERT_ERR_PTR(mmap_ptr, "wr_prod_pos");
 177         ASSERT_EQ(err, -EPERM, "wr_prod_pos_err");
 178
 179         /* bad writeable data pages */
 180         mmap_ptr = mmap(NULL, page_size, PROT_WRITE, MAP_SHARED, rb_fd, 2 * page_size);
 181         err = -errno;
 182         ASSERT_ERR_PTR(mmap_ptr, "wr_data_page_one");
 183         ASSERT_EQ(err, -EPERM, "wr_data_page_one_err");
 184         mmap_ptr = mmap(NULL, page_size, PROT_WRITE, MAP_SHARED, rb_fd, 3 * page_size);
 185         ASSERT_ERR_PTR(mmap_ptr, "wr_data_page_two");
 186         mmap_ptr = mmap(NULL, 2 * page_size, PROT_WRITE, MAP_SHARED, rb_fd, 2 * page_size);
 187         ASSERT_ERR_PTR(mmap_ptr, "wr_data_page_all");
 188
 189         /* good read-only pages */
 190         mmap_ptr = mmap(NULL, 4 * page_size, PROT_READ, MAP_SHARED, rb_fd, 0);
 191         if (!ASSERT_OK_PTR(mmap_ptr, "ro_prod_pos"))
 192                 goto cleanup;
 193
 194         ASSERT_ERR(mprotect(mmap_ptr, 4 * page_size, PROT_WRITE), "write_protect");
 195         ASSERT_ERR(mprotect(mmap_ptr, 4 * page_size, PROT_EXEC), "exec_protect");
 196         ASSERT_ERR_PTR(mremap(mmap_ptr, 0, 4 * page_size, MREMAP_MAYMOVE), "ro_remap");
 197         ASSERT_OK(munmap(mmap_ptr, 4 * page_size), "unmap_ro");
 198
 199         /* good read-only pages with initial offset */
 200         mmap_ptr = mmap(NULL, page_size, PROT_READ, MAP_SHARED, rb_fd, page_size);
 201         if (!ASSERT_OK_PTR(mmap_ptr, "ro_prod_pos"))
 202                 goto cleanup;
 203
 204         ASSERT_ERR(mprotect(mmap_ptr, page_size, PROT_WRITE), "write_protect");
 205         ASSERT_ERR(mprotect(mmap_ptr, page_size, PROT_EXEC), "exec_protect");
 206         ASSERT_ERR_PTR(mremap(mmap_ptr, 0, 3 * page_size, MREMAP_MAYMOVE), "ro_remap");
 207         ASSERT_OK(munmap(mmap_ptr, page_size), "unmap_ro");
 208
 209         /* only trigger BPF program for current process */
 210         skel->bss->pid = getpid();
 211
 212         ringbuf = ring_buffer__new(skel->maps.ringbuf.map_fd,
 213                                    process_sample, NULL, NULL);
 214         if (CHECK(!ringbuf, "ringbuf_create", "failed to create ringbuf\n"))
 215                 goto cleanup;
 216
 217         err = test_ringbuf_lskel__attach(skel);
 218         if (CHECK(err, "skel_attach", "skeleton attachment failed: %d\n", err))
 219                 goto cleanup;
 220
 221         trigger_samples();
 222
 223         ring = ring_buffer__ring(ringbuf, 0);
 224         if (!ASSERT_OK_PTR(ring, "ring_buffer__ring_idx_0"))
 225                 goto cleanup;
 226
 227         map_fd = ring__map_fd(ring);
 228         ASSERT_EQ(map_fd, skel->maps.ringbuf.map_fd, "ring_map_fd");
 229
 230         /* 2 submitted + 1 discarded records */
 231         CHECK(skel->bss->avail_data != 3 * rec_sz,
 232               "err_avail_size", "exp %ld, got %ld\n",
 233               3L * rec_sz, skel->bss->avail_data);
 234         CHECK(skel->bss->ring_size != page_size,
 235               "err_ring_size", "exp %ld, got %ld\n",
 236               (long)page_size, skel->bss->ring_size);
 237         CHECK(skel->bss->cons_pos != 0,
 238               "err_cons_pos", "exp %ld, got %ld\n",
 239               0L, skel->bss->cons_pos);
 240         CHECK(skel->bss->prod_pos != 3 * rec_sz,
 241               "err_prod_pos", "exp %ld, got %ld\n",
 242               3L * rec_sz, skel->bss->prod_pos);
 243
 244         /* verify getting this data directly via the ring object yields the same
 245          * results
 246          */
 247         avail_data = ring__avail_data_size(ring);
 248         ASSERT_EQ(avail_data, 3 * rec_sz, "ring_avail_size");
 249         ring_size = ring__size(ring);
 250         ASSERT_EQ(ring_size, page_size, "ring_ring_size");
 251         cons_pos = ring__consumer_pos(ring);
 252         ASSERT_EQ(cons_pos, 0, "ring_cons_pos");
 253         prod_pos = ring__producer_pos(ring);
 254         ASSERT_EQ(prod_pos, 3 * rec_sz, "ring_prod_pos");
 255
 256         /* poll for samples */
 257         err = ring_buffer__poll(ringbuf, -1);
 258
 259         /* -EDONE is used as an indicator that we are done */
 260         if (CHECK(err != -EDONE, "err_done", "done err: %d\n", err))
 261                 goto cleanup;
 262         cnt = atomic_xchg(&sample_cnt, 0);
 263         CHECK(cnt != 2, "cnt", "exp %d samples, got %d\n", 2, cnt);
 264
 265         /* we expect extra polling to return nothing */
 266         err = ring_buffer__poll(ringbuf, 0);
 267         if (CHECK(err != 0, "extra_samples", "poll result: %d\n", err))
 268                 goto cleanup;
 269         cnt = atomic_xchg(&sample_cnt, 0);
 270         CHECK(cnt != 0, "cnt", "exp %d samples, got %d\n", 0, cnt);
 271
 272         CHECK(skel->bss->dropped != 0, "err_dropped", "exp %ld, got %ld\n",
 273               0L, skel->bss->dropped);
 274         CHECK(skel->bss->total != 2, "err_total", "exp %ld, got %ld\n",
 275               2L, skel->bss->total);
 276         CHECK(skel->bss->discarded != 1, "err_discarded", "exp %ld, got %ld\n",
 277               1L, skel->bss->discarded);
 278
 279         /* now validate consumer position is updated and returned */
 280         trigger_samples();
 281         CHECK(skel->bss->cons_pos != 3 * rec_sz,
 282               "err_cons_pos", "exp %ld, got %ld\n",
 283               3L * rec_sz, skel->bss->cons_pos);
 284         err = ring_buffer__poll(ringbuf, -1);
 285         CHECK(err <= 0, "poll_err", "err %d\n", err);
 286         cnt = atomic_xchg(&sample_cnt, 0);
 287         CHECK(cnt != 2, "cnt", "exp %d samples, got %d\n", 2, cnt);
 288
 289         /* start poll in background w/ long timeout */
 290         err = pthread_create(&thread, NULL, poll_thread, (void *)(long)10000);
 291         if (CHECK(err, "bg_poll", "pthread_create failed: %d\n", err))
 292                 goto cleanup;
 293
 294         /* turn off notifications now */
 295         skel->bss->flags = BPF_RB_NO_WAKEUP;
 296
 297         /* give background thread a bit of a time */
 298         usleep(50000);
 299         trigger_samples();
 300         /* sleeping arbitrarily is bad, but no better way to know that
 301          * epoll_wait() **DID NOT** unblock in background thread
 302          */
 303         usleep(50000);
 304         /* background poll should still be blocked */
 305         err = pthread_tryjoin_np(thread, (void **)&bg_ret);
 306         if (CHECK(err != EBUSY, "try_join", "err %d\n", err))
 307                 goto cleanup;
 308
 309         /* BPF side did everything right */
 310         CHECK(skel->bss->dropped != 0, "err_dropped", "exp %ld, got %ld\n",
 311               0L, skel->bss->dropped);
 312         CHECK(skel->bss->total != 2, "err_total", "exp %ld, got %ld\n",
 313               2L, skel->bss->total);
 314         CHECK(skel->bss->discarded != 1, "err_discarded", "exp %ld, got %ld\n",
 315               1L, skel->bss->discarded);
 316         cnt = atomic_xchg(&sample_cnt, 0);
 317         CHECK(cnt != 0, "cnt", "exp %d samples, got %d\n", 0, cnt);
 318
 319         /* clear flags to return to "adaptive" notification mode */
 320         skel->bss->flags = 0;
 321
 322         /* produce new samples, no notification should be triggered, because
 323          * consumer is now behind
 324          */
 325         trigger_samples();
 326
 327         /* background poll should still be blocked */
 328         err = pthread_tryjoin_np(thread, (void **)&bg_ret);
 329         if (CHECK(err != EBUSY, "try_join", "err %d\n", err))
 330                 goto cleanup;
 331
 332         /* still no samples, because consumer is behind */
 333         cnt = atomic_xchg(&sample_cnt, 0);
 334         CHECK(cnt != 0, "cnt", "exp %d samples, got %d\n", 0, cnt);
 335
 336         skel->bss->dropped = 0;
 337         skel->bss->total = 0;
 338         skel->bss->discarded = 0;
 339
 340         skel->bss->value = 333;
 341         syscall(__NR_getpgid);
 342         /* now force notifications */
 343         skel->bss->flags = BPF_RB_FORCE_WAKEUP;
 344         skel->bss->value = 777;
 345         syscall(__NR_getpgid);
 346
 347         /* now we should get a pending notification */
 348         usleep(50000);
 349         err = pthread_tryjoin_np(thread, (void **)&bg_ret);
 350         if (CHECK(err, "join_bg", "err %d\n", err))
 351                 goto cleanup;
 352
 353         if (CHECK(bg_ret <= 0, "bg_ret", "epoll_wait result: %ld", bg_ret))
 354                 goto cleanup;
 355
 356         /* due to timing variations, there could still be non-notified
 357          * samples, so consume them here to collect all the samples
 358          */
 359         err = ring_buffer__consume(ringbuf);
 360         CHECK(err < 0, "rb_consume", "failed: %d\b", err);
 361
 362         /* also consume using ring__consume to make sure it works the same */
 363         err = ring__consume(ring);
 364         ASSERT_GE(err, 0, "ring_consume");
 365
 366         /* 3 rounds, 2 samples each */
 367         cnt = atomic_xchg(&sample_cnt, 0);
 368         CHECK(cnt != 6, "cnt", "exp %d samples, got %d\n", 6, cnt);
 369
 370         /* BPF side did everything right */
 371         CHECK(skel->bss->dropped != 0, "err_dropped", "exp %ld, got %ld\n",
 372               0L, skel->bss->dropped);
 373         CHECK(skel->bss->total != 2, "err_total", "exp %ld, got %ld\n",
 374               2L, skel->bss->total);
 375         CHECK(skel->bss->discarded != 1, "err_discarded", "exp %ld, got %ld\n",
 376               1L, skel->bss->discarded);
 377
 378         test_ringbuf_lskel__detach(skel);
 379 cleanup:
 380         ring_buffer__free(ringbuf);
 381         test_ringbuf_lskel__destroy(skel);
 382 }
 383
 384 /*
 385  * Test ring_buffer__consume_n() by producing N_TOT_SAMPLES samples in the ring
 386  * buffer, via getpid(), and consuming them in chunks of N_SAMPLES.
 387  */
 388 #define N_TOT_SAMPLES   32
 389 #define N_SAMPLES       4
 390
 391 /* Sample value to verify the callback validity */
 392 #define SAMPLE_VALUE    42L
 393
 394 static int process_n_sample(void *ctx, void *data, size_t len)
 395 {
 396         struct sample *s = data;
 397
 398         ASSERT_EQ(s->value, SAMPLE_VALUE, "sample_value");
 399
 400         return 0;
 401 }
 402
 403 static void ringbuf_n_subtest(void)
 404 {
 405         struct test_ringbuf_n_lskel *skel_n;
 406         int err, i;
 407
 408         skel_n = test_ringbuf_n_lskel__open();
 409         if (!ASSERT_OK_PTR(skel_n, "test_ringbuf_n_lskel__open"))
 410                 return;
 411
 412         skel_n->maps.ringbuf.max_entries = getpagesize();
 413         skel_n->bss->pid = getpid();
 414
 415         err = test_ringbuf_n_lskel__load(skel_n);
 416         if (!ASSERT_OK(err, "test_ringbuf_n_lskel__load"))
 417                 goto cleanup;
 418
 419         ringbuf = ring_buffer__new(skel_n->maps.ringbuf.map_fd,
 420                                    process_n_sample, NULL, NULL);
 421         if (!ASSERT_OK_PTR(ringbuf, "ring_buffer__new"))
 422                 goto cleanup;
 423
 424         err = test_ringbuf_n_lskel__attach(skel_n);
 425         if (!ASSERT_OK(err, "test_ringbuf_n_lskel__attach"))
 426                 goto cleanup_ringbuf;
 427
 428         /* Produce N_TOT_SAMPLES samples in the ring buffer by calling getpid() */
 429         skel_n->bss->value = SAMPLE_VALUE;
 430         for (i = 0; i < N_TOT_SAMPLES; i++)
 431                 syscall(__NR_getpgid);
 432
 433         /* Consume all samples from the ring buffer in batches of N_SAMPLES */
 434         for (i = 0; i < N_TOT_SAMPLES; i += err) {
 435                 err = ring_buffer__consume_n(ringbuf, N_SAMPLES);
 436                 if (!ASSERT_EQ(err, N_SAMPLES, "rb_consume"))
 437                         goto cleanup_ringbuf;
 438         }
 439
 440 cleanup_ringbuf:
 441         ring_buffer__free(ringbuf);
 442 cleanup:
 443         test_ringbuf_n_lskel__destroy(skel_n);
 444 }
 445
 446 static int process_map_key_sample(void *ctx, void *data, size_t len)
 447 {
 448         struct sample *s;
 449         int err, val;
 450
 451         s = data;
 452         switch (s->seq) {
 453         case 1:
 454                 ASSERT_EQ(s->value, 42, "sample_value");
 455                 err = bpf_map_lookup_elem(skel_map_key->maps.hash_map.map_fd,
 456                                           s, &val);
 457                 ASSERT_OK(err, "hash_map bpf_map_lookup_elem");
 458                 ASSERT_EQ(val, 1, "hash_map val");
 459                 return -EDONE;
 460         default:
 461                 return 0;
 462         }
 463 }
 464
 465 static void ringbuf_map_key_subtest(void)
 466 {
 467         int err;
 468
 469         skel_map_key = test_ringbuf_map_key_lskel__open();
 470         if (!ASSERT_OK_PTR(skel_map_key, "test_ringbuf_map_key_lskel__open"))
 471                 return;
 472
 473         skel_map_key->maps.ringbuf.max_entries = getpagesize();
 474         skel_map_key->bss->pid = getpid();
 475
 476         err = test_ringbuf_map_key_lskel__load(skel_map_key);
 477         if (!ASSERT_OK(err, "test_ringbuf_map_key_lskel__load"))
 478                 goto cleanup;
 479
 480         ringbuf = ring_buffer__new(skel_map_key->maps.ringbuf.map_fd,
 481                                    process_map_key_sample, NULL, NULL);
 482         if (!ASSERT_OK_PTR(ringbuf, "ring_buffer__new"))
 483                 goto cleanup;
 484
 485         err = test_ringbuf_map_key_lskel__attach(skel_map_key);
 486         if (!ASSERT_OK(err, "test_ringbuf_map_key_lskel__attach"))
 487                 goto cleanup_ringbuf;
 488
 489         syscall(__NR_getpgid);
 490         ASSERT_EQ(skel_map_key->bss->seq, 1, "skel_map_key->bss->seq");
 491         err = ring_buffer__poll(ringbuf, -1);
 492         ASSERT_EQ(err, -EDONE, "ring_buffer__poll");
 493
 494 cleanup_ringbuf:
 495         ring_buffer__free(ringbuf);
 496 cleanup:
 497         test_ringbuf_map_key_lskel__destroy(skel_map_key);
 498 }
 499
 500 void test_ringbuf(void)
 501 {
 502         if (test__start_subtest("ringbuf"))
 503                 ringbuf_subtest();
 504         if (test__start_subtest("ringbuf_n"))
 505                 ringbuf_n_subtest();
 506         if (test__start_subtest("ringbuf_map_key"))
 507                 ringbuf_map_key_subtest();
 508         if (test__start_subtest("ringbuf_write"))
 509                 ringbuf_write_subtest();
 510 }