tools/testing/selftests/bpf/prog_tests/select_reuseport.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /* Copyright (c) 2018 Facebook */
   3
   4 #include <stdlib.h>
   5 #include <unistd.h>
   6 #include <stdbool.h>
   7 #include <string.h>
   8 #include <errno.h>
   9 #include <assert.h>
  10 #include <fcntl.h>
  11 #include <linux/bpf.h>
  12 #include <linux/err.h>
  13 #include <linux/types.h>
  14 #include <linux/if_ether.h>
  15 #include <sys/types.h>
  16 #include <sys/epoll.h>
  17 #include <sys/socket.h>
  18 #include <netinet/in.h>
  19 #include <bpf/bpf.h>
  20 #include <bpf/libbpf.h>
  21 #include "bpf_util.h"
  22
  23 #include "test_progs.h"
  24 #include "test_select_reuseport_common.h"
  25
  26 #define MAX_TEST_NAME 80
  27 #define MIN_TCPHDR_LEN 20
  28 #define UDPHDR_LEN 8
  29
  30 #define TCP_SYNCOOKIE_SYSCTL "/proc/sys/net/ipv4/tcp_syncookies"
  31 #define TCP_FO_SYSCTL "/proc/sys/net/ipv4/tcp_fastopen"
  32 #define REUSEPORT_ARRAY_SIZE 32
  33
  34 static int result_map, tmp_index_ovr_map, linum_map, data_check_map;
  35 static __u32 expected_results[NR_RESULTS];
  36 static int sk_fds[REUSEPORT_ARRAY_SIZE];
  37 static int reuseport_array = -1, outer_map = -1;
  38 static enum bpf_map_type inner_map_type;
  39 static int select_by_skb_data_prog;
  40 static int saved_tcp_syncookie = -1;
  41 static struct bpf_object *obj;
  42 static int saved_tcp_fo = -1;
  43 static __u32 index_zero;
  44 static int epfd;
  45
  46 static union sa46 {
  47         struct sockaddr_in6 v6;
  48         struct sockaddr_in v4;
  49         sa_family_t family;
  50 } srv_sa;
  51
  52 #define RET_IF(condition, tag, format...) ({                            \
  53         if (CHECK_FAIL(condition)) {                                    \
  54                 printf(tag " " format);                                 \
  55                 return;                                                 \
  56         }                                                               \
  57 })
  58
  59 #define RET_ERR(condition, tag, format...) ({                           \
  60         if (CHECK_FAIL(condition)) {                                    \
  61                 printf(tag " " format);                                 \
  62                 return -1;                                              \
  63         }                                                               \
  64 })
  65
  66 static int create_maps(enum bpf_map_type inner_type)
  67 {
  68         LIBBPF_OPTS(bpf_map_create_opts, opts);
  69
  70         inner_map_type = inner_type;
  71
  72         /* Creating reuseport_array */
  73         reuseport_array = bpf_map_create(inner_type, "reuseport_array",
  74                                          sizeof(__u32), sizeof(__u32), REUSEPORT_ARRAY_SIZE, NULL);
  75         RET_ERR(reuseport_array < 0, "creating reuseport_array",
  76                 "reuseport_array:%d errno:%d\n", reuseport_array, errno);
  77
  78         /* Creating outer_map */
  79         opts.inner_map_fd = reuseport_array;
  80         outer_map = bpf_map_create(BPF_MAP_TYPE_ARRAY_OF_MAPS, "outer_map",
  81                                    sizeof(__u32), sizeof(__u32), 1, &opts);
  82         RET_ERR(outer_map < 0, "creating outer_map",
  83                 "outer_map:%d errno:%d\n", outer_map, errno);
  84
  85         return 0;
  86 }
  87
  88 static int prepare_bpf_obj(void)
  89 {
  90         struct bpf_program *prog;
  91         struct bpf_map *map;
  92         int err;
  93
  94         obj = bpf_object__open("test_select_reuseport_kern.bpf.o");
  95         err = libbpf_get_error(obj);
  96         RET_ERR(err, "open test_select_reuseport_kern.bpf.o",
  97                 "obj:%p PTR_ERR(obj):%d\n", obj, err);
  98
  99         map = bpf_object__find_map_by_name(obj, "outer_map");
 100         RET_ERR(!map, "find outer_map", "!map\n");
 101         err = bpf_map__reuse_fd(map, outer_map);
 102         RET_ERR(err, "reuse outer_map", "err:%d\n", err);
 103
 104         err = bpf_object__load(obj);
 105         RET_ERR(err, "load bpf_object", "err:%d\n", err);
 106
 107         prog = bpf_object__next_program(obj, NULL);
 108         RET_ERR(!prog, "get first bpf_program", "!prog\n");
 109         select_by_skb_data_prog = bpf_program__fd(prog);
 110         RET_ERR(select_by_skb_data_prog < 0, "get prog fd",
 111                 "select_by_skb_data_prog:%d\n", select_by_skb_data_prog);
 112
 113         map = bpf_object__find_map_by_name(obj, "result_map");
 114         RET_ERR(!map, "find result_map", "!map\n");
 115         result_map = bpf_map__fd(map);
 116         RET_ERR(result_map < 0, "get result_map fd",
 117                 "result_map:%d\n", result_map);
 118
 119         map = bpf_object__find_map_by_name(obj, "tmp_index_ovr_map");
 120         RET_ERR(!map, "find tmp_index_ovr_map\n", "!map");
 121         tmp_index_ovr_map = bpf_map__fd(map);
 122         RET_ERR(tmp_index_ovr_map < 0, "get tmp_index_ovr_map fd",
 123                 "tmp_index_ovr_map:%d\n", tmp_index_ovr_map);
 124
 125         map = bpf_object__find_map_by_name(obj, "linum_map");
 126         RET_ERR(!map, "find linum_map", "!map\n");
 127         linum_map = bpf_map__fd(map);
 128         RET_ERR(linum_map < 0, "get linum_map fd",
 129                 "linum_map:%d\n", linum_map);
 130
 131         map = bpf_object__find_map_by_name(obj, "data_check_map");
 132         RET_ERR(!map, "find data_check_map", "!map\n");
 133         data_check_map = bpf_map__fd(map);
 134         RET_ERR(data_check_map < 0, "get data_check_map fd",
 135                 "data_check_map:%d\n", data_check_map);
 136
 137         return 0;
 138 }
 139
 140 static void sa46_init_loopback(union sa46 *sa, sa_family_t family)
 141 {
 142         memset(sa, 0, sizeof(*sa));
 143         sa->family = family;
 144         if (sa->family == AF_INET6)
 145                 sa->v6.sin6_addr = in6addr_loopback;
 146         else
 147                 sa->v4.sin_addr.s_addr = htonl(INADDR_LOOPBACK);
 148 }
 149
 150 static void sa46_init_inany(union sa46 *sa, sa_family_t family)
 151 {
 152         memset(sa, 0, sizeof(*sa));
 153         sa->family = family;
 154         if (sa->family == AF_INET6)
 155                 sa->v6.sin6_addr = in6addr_any;
 156         else
 157                 sa->v4.sin_addr.s_addr = INADDR_ANY;
 158 }
 159
 160 static int read_int_sysctl(const char *sysctl)
 161 {
 162         char buf[16];
 163         int fd, ret;
 164
 165         fd = open(sysctl, 0);
 166         RET_ERR(fd == -1, "open(sysctl)",
 167                 "sysctl:%s fd:%d errno:%d\n", sysctl, fd, errno);
 168
 169         ret = read(fd, buf, sizeof(buf));
 170         RET_ERR(ret <= 0, "read(sysctl)",
 171                 "sysctl:%s ret:%d errno:%d\n", sysctl, ret, errno);
 172
 173         close(fd);
 174         return atoi(buf);
 175 }
 176
 177 static int write_int_sysctl(const char *sysctl, int v)
 178 {
 179         int fd, ret, size;
 180         char buf[16];
 181
 182         fd = open(sysctl, O_RDWR);
 183         RET_ERR(fd == -1, "open(sysctl)",
 184                 "sysctl:%s fd:%d errno:%d\n", sysctl, fd, errno);
 185
 186         size = snprintf(buf, sizeof(buf), "%d", v);
 187         ret = write(fd, buf, size);
 188         RET_ERR(ret != size, "write(sysctl)",
 189                 "sysctl:%s ret:%d size:%d errno:%d\n",
 190                 sysctl, ret, size, errno);
 191
 192         close(fd);
 193         return 0;
 194 }
 195
 196 static void restore_sysctls(void)
 197 {
 198         if (saved_tcp_fo != -1)
 199                 write_int_sysctl(TCP_FO_SYSCTL, saved_tcp_fo);
 200         if (saved_tcp_syncookie != -1)
 201                 write_int_sysctl(TCP_SYNCOOKIE_SYSCTL, saved_tcp_syncookie);
 202 }
 203
 204 static int enable_fastopen(void)
 205 {
 206         int fo;
 207
 208         fo = read_int_sysctl(TCP_FO_SYSCTL);
 209         if (fo < 0)
 210                 return -1;
 211
 212         return write_int_sysctl(TCP_FO_SYSCTL, fo | 7);
 213 }
 214
 215 static int enable_syncookie(void)
 216 {
 217         return write_int_sysctl(TCP_SYNCOOKIE_SYSCTL, 2);
 218 }
 219
 220 static int disable_syncookie(void)
 221 {
 222         return write_int_sysctl(TCP_SYNCOOKIE_SYSCTL, 0);
 223 }
 224
 225 static long get_linum(void)
 226 {
 227         __u32 linum;
 228         int err;
 229
 230         err = bpf_map_lookup_elem(linum_map, &index_zero, &linum);
 231         RET_ERR(err < 0, "lookup_elem(linum_map)", "err:%d errno:%d\n",
 232                 err, errno);
 233
 234         return linum;
 235 }
 236
 237 static void check_data(int type, sa_family_t family, const struct cmd *cmd,
 238                        int cli_fd)
 239 {
 240         struct data_check expected = {}, result;
 241         union sa46 cli_sa;
 242         socklen_t addrlen;
 243         int err;
 244
 245         addrlen = sizeof(cli_sa);
 246         err = getsockname(cli_fd, (struct sockaddr *)&cli_sa,
 247                           &addrlen);
 248         RET_IF(err < 0, "getsockname(cli_fd)", "err:%d errno:%d\n",
 249                err, errno);
 250
 251         err = bpf_map_lookup_elem(data_check_map, &index_zero, &result);
 252         RET_IF(err < 0, "lookup_elem(data_check_map)", "err:%d errno:%d\n",
 253                err, errno);
 254
 255         if (type == SOCK_STREAM) {
 256                 expected.len = MIN_TCPHDR_LEN;
 257                 expected.ip_protocol = IPPROTO_TCP;
 258         } else {
 259                 expected.len = UDPHDR_LEN;
 260                 expected.ip_protocol = IPPROTO_UDP;
 261         }
 262
 263         if (family == AF_INET6) {
 264                 expected.eth_protocol = htons(ETH_P_IPV6);
 265                 expected.bind_inany = !srv_sa.v6.sin6_addr.s6_addr32[3] &&
 266                         !srv_sa.v6.sin6_addr.s6_addr32[2] &&
 267                         !srv_sa.v6.sin6_addr.s6_addr32[1] &&
 268                         !srv_sa.v6.sin6_addr.s6_addr32[0];
 269
 270                 memcpy(&expected.skb_addrs[0], cli_sa.v6.sin6_addr.s6_addr32,
 271                        sizeof(cli_sa.v6.sin6_addr));
 272                 memcpy(&expected.skb_addrs[4], &in6addr_loopback,
 273                        sizeof(in6addr_loopback));
 274                 expected.skb_ports[0] = cli_sa.v6.sin6_port;
 275                 expected.skb_ports[1] = srv_sa.v6.sin6_port;
 276         } else {
 277                 expected.eth_protocol = htons(ETH_P_IP);
 278                 expected.bind_inany = !srv_sa.v4.sin_addr.s_addr;
 279
 280                 expected.skb_addrs[0] = cli_sa.v4.sin_addr.s_addr;
 281                 expected.skb_addrs[1] = htonl(INADDR_LOOPBACK);
 282                 expected.skb_ports[0] = cli_sa.v4.sin_port;
 283                 expected.skb_ports[1] = srv_sa.v4.sin_port;
 284         }
 285
 286         if (memcmp(&result, &expected, offsetof(struct data_check,
 287                                                 equal_check_end))) {
 288                 printf("unexpected data_check\n");
 289                 printf("  result: (0x%x, %u, %u)\n",
 290                        result.eth_protocol, result.ip_protocol,
 291                        result.bind_inany);
 292                 printf("expected: (0x%x, %u, %u)\n",
 293                        expected.eth_protocol, expected.ip_protocol,
 294                        expected.bind_inany);
 295                 RET_IF(1, "data_check result != expected",
 296                        "bpf_prog_linum:%ld\n", get_linum());
 297         }
 298
 299         RET_IF(!result.hash, "data_check result.hash empty",
 300                "result.hash:%u", result.hash);
 301
 302         expected.len += cmd ? sizeof(*cmd) : 0;
 303         if (type == SOCK_STREAM)
 304                 RET_IF(expected.len > result.len, "expected.len > result.len",
 305                        "expected.len:%u result.len:%u bpf_prog_linum:%ld\n",
 306                        expected.len, result.len, get_linum());
 307         else
 308                 RET_IF(expected.len != result.len, "expected.len != result.len",
 309                        "expected.len:%u result.len:%u bpf_prog_linum:%ld\n",
 310                        expected.len, result.len, get_linum());
 311 }
 312
 313 static const char *result_to_str(enum result res)
 314 {
 315         switch (res) {
 316         case DROP_ERR_INNER_MAP:
 317                 return "DROP_ERR_INNER_MAP";
 318         case DROP_ERR_SKB_DATA:
 319                 return "DROP_ERR_SKB_DATA";
 320         case DROP_ERR_SK_SELECT_REUSEPORT:
 321                 return "DROP_ERR_SK_SELECT_REUSEPORT";
 322         case DROP_MISC:
 323                 return "DROP_MISC";
 324         case PASS:
 325                 return "PASS";
 326         case PASS_ERR_SK_SELECT_REUSEPORT:
 327                 return "PASS_ERR_SK_SELECT_REUSEPORT";
 328         default:
 329                 return "UNKNOWN";
 330         }
 331 }
 332
 333 static void check_results(void)
 334 {
 335         __u32 results[NR_RESULTS];
 336         __u32 i, broken = 0;
 337         int err;
 338
 339         for (i = 0; i < NR_RESULTS; i++) {
 340                 err = bpf_map_lookup_elem(result_map, &i, &results[i]);
 341                 RET_IF(err < 0, "lookup_elem(result_map)",
 342                        "i:%u err:%d errno:%d\n", i, err, errno);
 343         }
 344
 345         for (i = 0; i < NR_RESULTS; i++) {
 346                 if (results[i] != expected_results[i]) {
 347                         broken = i;
 348                         break;
 349                 }
 350         }
 351
 352         if (i == NR_RESULTS)
 353                 return;
 354
 355         printf("unexpected result\n");
 356         printf(" result: [");
 357         printf("%u", results[0]);
 358         for (i = 1; i < NR_RESULTS; i++)
 359                 printf(", %u", results[i]);
 360         printf("]\n");
 361
 362         printf("expected: [");
 363         printf("%u", expected_results[0]);
 364         for (i = 1; i < NR_RESULTS; i++)
 365                 printf(", %u", expected_results[i]);
 366         printf("]\n");
 367
 368         printf("mismatch on %s (bpf_prog_linum:%ld)\n", result_to_str(broken),
 369                get_linum());
 370
 371         CHECK_FAIL(true);
 372 }
 373
 374 static int send_data(int type, sa_family_t family, void *data, size_t len,
 375                      enum result expected)
 376 {
 377         union sa46 cli_sa;
 378         int fd, err;
 379
 380         fd = socket(family, type, 0);
 381         RET_ERR(fd == -1, "socket()", "fd:%d errno:%d\n", fd, errno);
 382
 383         sa46_init_loopback(&cli_sa, family);
 384         err = bind(fd, (struct sockaddr *)&cli_sa, sizeof(cli_sa));
 385         RET_ERR(fd == -1, "bind(cli_sa)", "err:%d errno:%d\n", err, errno);
 386
 387         err = sendto(fd, data, len, MSG_FASTOPEN, (struct sockaddr *)&srv_sa,
 388                      sizeof(srv_sa));
 389         RET_ERR(err != len && expected >= PASS,
 390                 "sendto()", "family:%u err:%d errno:%d expected:%d\n",
 391                 family, err, errno, expected);
 392
 393         return fd;
 394 }
 395
 396 static void do_test(int type, sa_family_t family, struct cmd *cmd,
 397                     enum result expected)
 398 {
 399         int nev, srv_fd, cli_fd;
 400         struct epoll_event ev;
 401         struct cmd rcv_cmd;
 402         ssize_t nread;
 403
 404         cli_fd = send_data(type, family, cmd, cmd ? sizeof(*cmd) : 0,
 405                            expected);
 406         if (cli_fd < 0)
 407                 return;
 408         nev = epoll_wait(epfd, &ev, 1, expected >= PASS ? 5 : 0);
 409         RET_IF((nev <= 0 && expected >= PASS) ||
 410                (nev > 0 && expected < PASS),
 411                "nev <> expected",
 412                "nev:%d expected:%d type:%d family:%d data:(%d, %d)\n",
 413                nev, expected, type, family,
 414                cmd ? cmd->reuseport_index : -1,
 415                cmd ? cmd->pass_on_failure : -1);
 416         check_results();
 417         check_data(type, family, cmd, cli_fd);
 418
 419         if (expected < PASS)
 420                 return;
 421
 422         RET_IF(expected != PASS_ERR_SK_SELECT_REUSEPORT &&
 423                cmd->reuseport_index != ev.data.u32,
 424                "check cmd->reuseport_index",
 425                "cmd:(%u, %u) ev.data.u32:%u\n",
 426                cmd->pass_on_failure, cmd->reuseport_index, ev.data.u32);
 427
 428         srv_fd = sk_fds[ev.data.u32];
 429         if (type == SOCK_STREAM) {
 430                 int new_fd = accept(srv_fd, NULL, 0);
 431
 432                 RET_IF(new_fd == -1, "accept(srv_fd)",
 433                        "ev.data.u32:%u new_fd:%d errno:%d\n",
 434                        ev.data.u32, new_fd, errno);
 435
 436                 nread = recv(new_fd, &rcv_cmd, sizeof(rcv_cmd), MSG_DONTWAIT);
 437                 RET_IF(nread != sizeof(rcv_cmd),
 438                        "recv(new_fd)",
 439                        "ev.data.u32:%u nread:%zd sizeof(rcv_cmd):%zu errno:%d\n",
 440                        ev.data.u32, nread, sizeof(rcv_cmd), errno);
 441
 442                 close(new_fd);
 443         } else {
 444                 nread = recv(srv_fd, &rcv_cmd, sizeof(rcv_cmd), MSG_DONTWAIT);
 445                 RET_IF(nread != sizeof(rcv_cmd),
 446                        "recv(sk_fds)",
 447                        "ev.data.u32:%u nread:%zd sizeof(rcv_cmd):%zu errno:%d\n",
 448                        ev.data.u32, nread, sizeof(rcv_cmd), errno);
 449         }
 450
 451         close(cli_fd);
 452 }
 453
 454 static void test_err_inner_map(int type, sa_family_t family)
 455 {
 456         struct cmd cmd = {
 457                 .reuseport_index = 0,
 458                 .pass_on_failure = 0,
 459         };
 460
 461         expected_results[DROP_ERR_INNER_MAP]++;
 462         do_test(type, family, &cmd, DROP_ERR_INNER_MAP);
 463 }
 464
 465 static void test_err_skb_data(int type, sa_family_t family)
 466 {
 467         expected_results[DROP_ERR_SKB_DATA]++;
 468         do_test(type, family, NULL, DROP_ERR_SKB_DATA);
 469 }
 470
 471 static void test_err_sk_select_port(int type, sa_family_t family)
 472 {
 473         struct cmd cmd = {
 474                 .reuseport_index = REUSEPORT_ARRAY_SIZE,
 475                 .pass_on_failure = 0,
 476         };
 477
 478         expected_results[DROP_ERR_SK_SELECT_REUSEPORT]++;
 479         do_test(type, family, &cmd, DROP_ERR_SK_SELECT_REUSEPORT);
 480 }
 481
 482 static void test_pass(int type, sa_family_t family)
 483 {
 484         struct cmd cmd;
 485         int i;
 486
 487         cmd.pass_on_failure = 0;
 488         for (i = 0; i < REUSEPORT_ARRAY_SIZE; i++) {
 489                 expected_results[PASS]++;
 490                 cmd.reuseport_index = i;
 491                 do_test(type, family, &cmd, PASS);
 492         }
 493 }
 494
 495 static void test_syncookie(int type, sa_family_t family)
 496 {
 497         int err, tmp_index = 1;
 498         struct cmd cmd = {
 499                 .reuseport_index = 0,
 500                 .pass_on_failure = 0,
 501         };
 502
 503         /*
 504          * +1 for TCP-SYN and
 505          * +1 for the TCP-ACK (ack the syncookie)
 506          */
 507         expected_results[PASS] += 2;
 508         enable_syncookie();
 509         /*
 510          * Simulate TCP-SYN and TCP-ACK are handled by two different sk:
 511          * TCP-SYN: select sk_fds[tmp_index = 1] tmp_index is from the
 512          *          tmp_index_ovr_map
 513          * TCP-ACK: select sk_fds[reuseport_index = 0] reuseport_index
 514          *          is from the cmd.reuseport_index
 515          */
 516         err = bpf_map_update_elem(tmp_index_ovr_map, &index_zero,
 517                                   &tmp_index, BPF_ANY);
 518         RET_IF(err < 0, "update_elem(tmp_index_ovr_map, 0, 1)",
 519                "err:%d errno:%d\n", err, errno);
 520         do_test(type, family, &cmd, PASS);
 521         err = bpf_map_lookup_elem(tmp_index_ovr_map, &index_zero,
 522                                   &tmp_index);
 523         RET_IF(err < 0 || tmp_index >= 0,
 524                "lookup_elem(tmp_index_ovr_map)",
 525                "err:%d errno:%d tmp_index:%d\n",
 526                err, errno, tmp_index);
 527         disable_syncookie();
 528 }
 529
 530 static void test_pass_on_err(int type, sa_family_t family)
 531 {
 532         struct cmd cmd = {
 533                 .reuseport_index = REUSEPORT_ARRAY_SIZE,
 534                 .pass_on_failure = 1,
 535         };
 536
 537         expected_results[PASS_ERR_SK_SELECT_REUSEPORT] += 1;
 538         do_test(type, family, &cmd, PASS_ERR_SK_SELECT_REUSEPORT);
 539 }
 540
 541 static void test_detach_bpf(int type, sa_family_t family)
 542 {
 543 #ifdef SO_DETACH_REUSEPORT_BPF
 544         __u32 nr_run_before = 0, nr_run_after = 0, tmp, i;
 545         struct epoll_event ev;
 546         int cli_fd, err, nev;
 547         struct cmd cmd = {};
 548         int optvalue = 0;
 549
 550         err = setsockopt(sk_fds[0], SOL_SOCKET, SO_DETACH_REUSEPORT_BPF,
 551                          &optvalue, sizeof(optvalue));
 552         RET_IF(err == -1, "setsockopt(SO_DETACH_REUSEPORT_BPF)",
 553                "err:%d errno:%d\n", err, errno);
 554
 555         err = setsockopt(sk_fds[1], SOL_SOCKET, SO_DETACH_REUSEPORT_BPF,
 556                          &optvalue, sizeof(optvalue));
 557         RET_IF(err == 0 || errno != ENOENT,
 558                "setsockopt(SO_DETACH_REUSEPORT_BPF)",
 559                "err:%d errno:%d\n", err, errno);
 560
 561         for (i = 0; i < NR_RESULTS; i++) {
 562                 err = bpf_map_lookup_elem(result_map, &i, &tmp);
 563                 RET_IF(err < 0, "lookup_elem(result_map)",
 564                        "i:%u err:%d errno:%d\n", i, err, errno);
 565                 nr_run_before += tmp;
 566         }
 567
 568         cli_fd = send_data(type, family, &cmd, sizeof(cmd), PASS);
 569         if (cli_fd < 0)
 570                 return;
 571         nev = epoll_wait(epfd, &ev, 1, 5);
 572         RET_IF(nev <= 0, "nev <= 0",
 573                "nev:%d expected:1 type:%d family:%d data:(0, 0)\n",
 574                nev,  type, family);
 575
 576         for (i = 0; i < NR_RESULTS; i++) {
 577                 err = bpf_map_lookup_elem(result_map, &i, &tmp);
 578                 RET_IF(err < 0, "lookup_elem(result_map)",
 579                        "i:%u err:%d errno:%d\n", i, err, errno);
 580                 nr_run_after += tmp;
 581         }
 582
 583         RET_IF(nr_run_before != nr_run_after,
 584                "nr_run_before != nr_run_after",
 585                "nr_run_before:%u nr_run_after:%u\n",
 586                nr_run_before, nr_run_after);
 587
 588         close(cli_fd);
 589 #else
 590         test__skip();
 591 #endif
 592 }
 593
 594 static void prepare_sk_fds(int type, sa_family_t family, bool inany)
 595 {
 596         const int first = REUSEPORT_ARRAY_SIZE - 1;
 597         int i, err, optval = 1;
 598         struct epoll_event ev;
 599         socklen_t addrlen;
 600
 601         if (inany)
 602                 sa46_init_inany(&srv_sa, family);
 603         else
 604                 sa46_init_loopback(&srv_sa, family);
 605         addrlen = sizeof(srv_sa);
 606
 607         /*
 608          * The sk_fds[] is filled from the back such that the order
 609          * is exactly opposite to the (struct sock_reuseport *)reuse->socks[].
 610          */
 611         for (i = first; i >= 0; i--) {
 612                 sk_fds[i] = socket(family, type, 0);
 613                 RET_IF(sk_fds[i] == -1, "socket()", "sk_fds[%d]:%d errno:%d\n",
 614                        i, sk_fds[i], errno);
 615                 err = setsockopt(sk_fds[i], SOL_SOCKET, SO_REUSEPORT,
 616                                  &optval, sizeof(optval));
 617                 RET_IF(err == -1, "setsockopt(SO_REUSEPORT)",
 618                        "sk_fds[%d] err:%d errno:%d\n",
 619                        i, err, errno);
 620
 621                 if (i == first) {
 622                         err = setsockopt(sk_fds[i], SOL_SOCKET,
 623                                          SO_ATTACH_REUSEPORT_EBPF,
 624                                          &select_by_skb_data_prog,
 625                                          sizeof(select_by_skb_data_prog));
 626                         RET_IF(err < 0, "setsockopt(SO_ATTACH_REUEPORT_EBPF)",
 627                                "err:%d errno:%d\n", err, errno);
 628                 }
 629
 630                 err = bind(sk_fds[i], (struct sockaddr *)&srv_sa, addrlen);
 631                 RET_IF(err < 0, "bind()", "sk_fds[%d] err:%d errno:%d\n",
 632                        i, err, errno);
 633
 634                 if (type == SOCK_STREAM) {
 635                         err = listen(sk_fds[i], 10);
 636                         RET_IF(err < 0, "listen()",
 637                                "sk_fds[%d] err:%d errno:%d\n",
 638                                i, err, errno);
 639                 }
 640
 641                 err = bpf_map_update_elem(reuseport_array, &i, &sk_fds[i],
 642                                           BPF_NOEXIST);
 643                 RET_IF(err < 0, "update_elem(reuseport_array)",
 644                        "sk_fds[%d] err:%d errno:%d\n", i, err, errno);
 645
 646                 if (i == first) {
 647                         socklen_t addrlen = sizeof(srv_sa);
 648
 649                         err = getsockname(sk_fds[i], (struct sockaddr *)&srv_sa,
 650                                           &addrlen);
 651                         RET_IF(err == -1, "getsockname()",
 652                                "sk_fds[%d] err:%d errno:%d\n", i, err, errno);
 653                 }
 654         }
 655
 656         epfd = epoll_create(1);
 657         RET_IF(epfd == -1, "epoll_create(1)",
 658                "epfd:%d errno:%d\n", epfd, errno);
 659
 660         ev.events = EPOLLIN;
 661         for (i = 0; i < REUSEPORT_ARRAY_SIZE; i++) {
 662                 ev.data.u32 = i;
 663                 err = epoll_ctl(epfd, EPOLL_CTL_ADD, sk_fds[i], &ev);
 664                 RET_IF(err, "epoll_ctl(EPOLL_CTL_ADD)", "sk_fds[%d]\n", i);
 665         }
 666 }
 667
 668 static void setup_per_test(int type, sa_family_t family, bool inany,
 669                            bool no_inner_map)
 670 {
 671         int ovr = -1, err;
 672
 673         prepare_sk_fds(type, family, inany);
 674         err = bpf_map_update_elem(tmp_index_ovr_map, &index_zero, &ovr,
 675                                   BPF_ANY);
 676         RET_IF(err < 0, "update_elem(tmp_index_ovr_map, 0, -1)",
 677                "err:%d errno:%d\n", err, errno);
 678
 679         /* Install reuseport_array to outer_map? */
 680         if (no_inner_map)
 681                 return;
 682
 683         err = bpf_map_update_elem(outer_map, &index_zero, &reuseport_array,
 684                                   BPF_ANY);
 685         RET_IF(err < 0, "update_elem(outer_map, 0, reuseport_array)",
 686                "err:%d errno:%d\n", err, errno);
 687 }
 688
 689 static void cleanup_per_test(bool no_inner_map)
 690 {
 691         int i, err, zero = 0;
 692
 693         memset(expected_results, 0, sizeof(expected_results));
 694
 695         for (i = 0; i < NR_RESULTS; i++) {
 696                 err = bpf_map_update_elem(result_map, &i, &zero, BPF_ANY);
 697                 RET_IF(err, "reset elem in result_map",
 698                        "i:%u err:%d errno:%d\n", i, err, errno);
 699         }
 700
 701         err = bpf_map_update_elem(linum_map, &zero, &zero, BPF_ANY);
 702         RET_IF(err, "reset line number in linum_map", "err:%d errno:%d\n",
 703                err, errno);
 704
 705         for (i = 0; i < REUSEPORT_ARRAY_SIZE; i++)
 706                 close(sk_fds[i]);
 707         close(epfd);
 708
 709         /* Delete reuseport_array from outer_map? */
 710         if (no_inner_map)
 711                 return;
 712
 713         err = bpf_map_delete_elem(outer_map, &index_zero);
 714         RET_IF(err < 0, "delete_elem(outer_map)",
 715                "err:%d errno:%d\n", err, errno);
 716 }
 717
 718 static void cleanup(void)
 719 {
 720         if (outer_map >= 0) {
 721                 close(outer_map);
 722                 outer_map = -1;
 723         }
 724
 725         if (reuseport_array >= 0) {
 726                 close(reuseport_array);
 727                 reuseport_array = -1;
 728         }
 729
 730         if (obj) {
 731                 bpf_object__close(obj);
 732                 obj = NULL;
 733         }
 734
 735         memset(expected_results, 0, sizeof(expected_results));
 736 }
 737
 738 static const char *maptype_str(enum bpf_map_type type)
 739 {
 740         switch (type) {
 741         case BPF_MAP_TYPE_REUSEPORT_SOCKARRAY:
 742                 return "reuseport_sockarray";
 743         case BPF_MAP_TYPE_SOCKMAP:
 744                 return "sockmap";
 745         case BPF_MAP_TYPE_SOCKHASH:
 746                 return "sockhash";
 747         default:
 748                 return "unknown";
 749         }
 750 }
 751
 752 static const char *family_str(sa_family_t family)
 753 {
 754         switch (family) {
 755         case AF_INET:
 756                 return "IPv4";
 757         case AF_INET6:
 758                 return "IPv6";
 759         default:
 760                 return "unknown";
 761         }
 762 }
 763
 764 static const char *sotype_str(int sotype)
 765 {
 766         switch (sotype) {
 767         case SOCK_STREAM:
 768                 return "TCP";
 769         case SOCK_DGRAM:
 770                 return "UDP";
 771         default:
 772                 return "unknown";
 773         }
 774 }
 775
 776 #define TEST_INIT(fn_, ...) { .fn = fn_, .name = #fn_, __VA_ARGS__ }
 777
 778 static void test_config(int sotype, sa_family_t family, bool inany)
 779 {
 780         const struct test {
 781                 void (*fn)(int sotype, sa_family_t family);
 782                 const char *name;
 783                 bool no_inner_map;
 784                 int need_sotype;
 785         } tests[] = {
 786                 TEST_INIT(test_err_inner_map,
 787                           .no_inner_map = true),
 788                 TEST_INIT(test_err_skb_data),
 789                 TEST_INIT(test_err_sk_select_port),
 790                 TEST_INIT(test_pass),
 791                 TEST_INIT(test_syncookie,
 792                           .need_sotype = SOCK_STREAM),
 793                 TEST_INIT(test_pass_on_err),
 794                 TEST_INIT(test_detach_bpf),
 795         };
 796         char s[MAX_TEST_NAME];
 797         const struct test *t;
 798
 799         for (t = tests; t < tests + ARRAY_SIZE(tests); t++) {
 800                 if (t->need_sotype && t->need_sotype != sotype)
 801                         continue; /* test not compatible with socket type */
 802
 803                 snprintf(s, sizeof(s), "%s %s/%s %s %s",
 804                          maptype_str(inner_map_type),
 805                          family_str(family), sotype_str(sotype),
 806                          inany ? "INANY" : "LOOPBACK", t->name);
 807
 808                 if (!test__start_subtest(s))
 809                         continue;
 810
 811                 setup_per_test(sotype, family, inany, t->no_inner_map);
 812                 t->fn(sotype, family);
 813                 cleanup_per_test(t->no_inner_map);
 814         }
 815 }
 816
 817 #define BIND_INANY true
 818
 819 static void test_all(void)
 820 {
 821         const struct config {
 822                 int sotype;
 823                 sa_family_t family;
 824                 bool inany;
 825         } configs[] = {
 826                 { SOCK_STREAM, AF_INET },
 827                 { SOCK_STREAM, AF_INET, BIND_INANY },
 828                 { SOCK_STREAM, AF_INET6 },
 829                 { SOCK_STREAM, AF_INET6, BIND_INANY },
 830                 { SOCK_DGRAM, AF_INET },
 831                 { SOCK_DGRAM, AF_INET6 },
 832         };
 833         const struct config *c;
 834
 835         for (c = configs; c < configs + ARRAY_SIZE(configs); c++)
 836                 test_config(c->sotype, c->family, c->inany);
 837 }
 838
 839 void test_map_type(enum bpf_map_type mt)
 840 {
 841         if (create_maps(mt))
 842                 goto out;
 843         if (prepare_bpf_obj())
 844                 goto out;
 845
 846         test_all();
 847 out:
 848         cleanup();
 849 }
 850
 851 void serial_test_select_reuseport(void)
 852 {
 853         saved_tcp_fo = read_int_sysctl(TCP_FO_SYSCTL);
 854         if (saved_tcp_fo < 0)
 855                 goto out;
 856         saved_tcp_syncookie = read_int_sysctl(TCP_SYNCOOKIE_SYSCTL);
 857         if (saved_tcp_syncookie < 0)
 858                 goto out;
 859
 860         if (enable_fastopen())
 861                 goto out;
 862         if (disable_syncookie())
 863                 goto out;
 864
 865         test_map_type(BPF_MAP_TYPE_REUSEPORT_SOCKARRAY);
 866         test_map_type(BPF_MAP_TYPE_SOCKMAP);
 867         test_map_type(BPF_MAP_TYPE_SOCKHASH);
 868 out:
 869         restore_sysctls();
 870 }