tools/testing/selftests/bpf/prog_tests/select_reuseport.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /* Copyright (c) 2018 Facebook */
   3
   4 #include <stdlib.h>
   5 #include <unistd.h>
   6 #include <stdbool.h>
   7 #include <string.h>
   8 #include <errno.h>
   9 #include <assert.h>
  10 #include <fcntl.h>
  11 #include <linux/bpf.h>
  12 #include <linux/err.h>
  13 #include <linux/types.h>
  14 #include <linux/if_ether.h>
  15 #include <sys/types.h>
  16 #include <sys/epoll.h>
  17 #include <sys/socket.h>
  18 #include <netinet/in.h>
  19 #include <bpf/bpf.h>
  20 #include <bpf/libbpf.h>
  21 #include "bpf_util.h"
  22
  23 #include "test_progs.h"
  24 #include "test_select_reuseport_common.h"
  25
  26 #define MAX_TEST_NAME 80
  27 #define MIN_TCPHDR_LEN 20
  28 #define UDPHDR_LEN 8
  29
  30 #define TCP_SYNCOOKIE_SYSCTL "/proc/sys/net/ipv4/tcp_syncookies"
  31 #define TCP_FO_SYSCTL "/proc/sys/net/ipv4/tcp_fastopen"
  32 #define REUSEPORT_ARRAY_SIZE 32
  33
  34 static int result_map, tmp_index_ovr_map, linum_map, data_check_map;
  35 static __u32 expected_results[NR_RESULTS];
  36 static int sk_fds[REUSEPORT_ARRAY_SIZE];
  37 static int reuseport_array = -1, outer_map = -1;
  38 static enum bpf_map_type inner_map_type;
  39 static int select_by_skb_data_prog;
  40 static struct bpf_object *obj;
  41 static __u32 index_zero;
  42 static int epfd;
  43
  44 static union sa46 {
  45         struct sockaddr_in6 v6;
  46         struct sockaddr_in v4;
  47         sa_family_t family;
  48 } srv_sa;
  49
  50 #define RET_IF(condition, tag, format...) ({                            \
  51         if (CHECK_FAIL(condition)) {                                    \
  52                 printf(tag " " format);                                 \
  53                 return;                                                 \
  54         }                                                               \
  55 })
  56
  57 #define RET_ERR(condition, tag, format...) ({                           \
  58         if (CHECK_FAIL(condition)) {                                    \
  59                 printf(tag " " format);                                 \
  60                 return -1;                                              \
  61         }                                                               \
  62 })
  63
  64 static int create_maps(enum bpf_map_type inner_type)
  65 {
  66         LIBBPF_OPTS(bpf_map_create_opts, opts);
  67
  68         inner_map_type = inner_type;
  69
  70         /* Creating reuseport_array */
  71         reuseport_array = bpf_map_create(inner_type, "reuseport_array",
  72                                          sizeof(__u32), sizeof(__u32), REUSEPORT_ARRAY_SIZE, NULL);
  73         RET_ERR(reuseport_array < 0, "creating reuseport_array",
  74                 "reuseport_array:%d errno:%d\n", reuseport_array, errno);
  75
  76         /* Creating outer_map */
  77         opts.inner_map_fd = reuseport_array;
  78         outer_map = bpf_map_create(BPF_MAP_TYPE_ARRAY_OF_MAPS, "outer_map",
  79                                    sizeof(__u32), sizeof(__u32), 1, &opts);
  80         RET_ERR(outer_map < 0, "creating outer_map",
  81                 "outer_map:%d errno:%d\n", outer_map, errno);
  82
  83         return 0;
  84 }
  85
  86 static int prepare_bpf_obj(void)
  87 {
  88         struct bpf_program *prog;
  89         struct bpf_map *map;
  90         int err;
  91
  92         obj = bpf_object__open("test_select_reuseport_kern.bpf.o");
  93         err = libbpf_get_error(obj);
  94         RET_ERR(err, "open test_select_reuseport_kern.bpf.o",
  95                 "obj:%p PTR_ERR(obj):%d\n", obj, err);
  96
  97         map = bpf_object__find_map_by_name(obj, "outer_map");
  98         RET_ERR(!map, "find outer_map", "!map\n");
  99         err = bpf_map__reuse_fd(map, outer_map);
 100         RET_ERR(err, "reuse outer_map", "err:%d\n", err);
 101
 102         err = bpf_object__load(obj);
 103         RET_ERR(err, "load bpf_object", "err:%d\n", err);
 104
 105         prog = bpf_object__next_program(obj, NULL);
 106         RET_ERR(!prog, "get first bpf_program", "!prog\n");
 107         select_by_skb_data_prog = bpf_program__fd(prog);
 108         RET_ERR(select_by_skb_data_prog < 0, "get prog fd",
 109                 "select_by_skb_data_prog:%d\n", select_by_skb_data_prog);
 110
 111         map = bpf_object__find_map_by_name(obj, "result_map");
 112         RET_ERR(!map, "find result_map", "!map\n");
 113         result_map = bpf_map__fd(map);
 114         RET_ERR(result_map < 0, "get result_map fd",
 115                 "result_map:%d\n", result_map);
 116
 117         map = bpf_object__find_map_by_name(obj, "tmp_index_ovr_map");
 118         RET_ERR(!map, "find tmp_index_ovr_map\n", "!map");
 119         tmp_index_ovr_map = bpf_map__fd(map);
 120         RET_ERR(tmp_index_ovr_map < 0, "get tmp_index_ovr_map fd",
 121                 "tmp_index_ovr_map:%d\n", tmp_index_ovr_map);
 122
 123         map = bpf_object__find_map_by_name(obj, "linum_map");
 124         RET_ERR(!map, "find linum_map", "!map\n");
 125         linum_map = bpf_map__fd(map);
 126         RET_ERR(linum_map < 0, "get linum_map fd",
 127                 "linum_map:%d\n", linum_map);
 128
 129         map = bpf_object__find_map_by_name(obj, "data_check_map");
 130         RET_ERR(!map, "find data_check_map", "!map\n");
 131         data_check_map = bpf_map__fd(map);
 132         RET_ERR(data_check_map < 0, "get data_check_map fd",
 133                 "data_check_map:%d\n", data_check_map);
 134
 135         return 0;
 136 }
 137
 138 static void sa46_init_loopback(union sa46 *sa, sa_family_t family)
 139 {
 140         memset(sa, 0, sizeof(*sa));
 141         sa->family = family;
 142         if (sa->family == AF_INET6)
 143                 sa->v6.sin6_addr = in6addr_loopback;
 144         else
 145                 sa->v4.sin_addr.s_addr = htonl(INADDR_LOOPBACK);
 146 }
 147
 148 static void sa46_init_inany(union sa46 *sa, sa_family_t family)
 149 {
 150         memset(sa, 0, sizeof(*sa));
 151         sa->family = family;
 152         if (sa->family == AF_INET6)
 153                 sa->v6.sin6_addr = in6addr_any;
 154         else
 155                 sa->v4.sin_addr.s_addr = INADDR_ANY;
 156 }
 157
 158 static int read_int_sysctl(const char *sysctl)
 159 {
 160         char buf[16];
 161         int fd, ret;
 162
 163         fd = open(sysctl, 0);
 164         RET_ERR(fd == -1, "open(sysctl)",
 165                 "sysctl:%s fd:%d errno:%d\n", sysctl, fd, errno);
 166
 167         ret = read(fd, buf, sizeof(buf));
 168         RET_ERR(ret <= 0, "read(sysctl)",
 169                 "sysctl:%s ret:%d errno:%d\n", sysctl, ret, errno);
 170
 171         close(fd);
 172         return atoi(buf);
 173 }
 174
 175 static int write_int_sysctl(const char *sysctl, int v)
 176 {
 177         int fd, ret, size;
 178         char buf[16];
 179
 180         fd = open(sysctl, O_RDWR);
 181         RET_ERR(fd == -1, "open(sysctl)",
 182                 "sysctl:%s fd:%d errno:%d\n", sysctl, fd, errno);
 183
 184         size = snprintf(buf, sizeof(buf), "%d", v);
 185         ret = write(fd, buf, size);
 186         RET_ERR(ret != size, "write(sysctl)",
 187                 "sysctl:%s ret:%d size:%d errno:%d\n",
 188                 sysctl, ret, size, errno);
 189
 190         close(fd);
 191         return 0;
 192 }
 193
 194 static int enable_fastopen(void)
 195 {
 196         int fo;
 197
 198         fo = read_int_sysctl(TCP_FO_SYSCTL);
 199         if (fo < 0)
 200                 return -1;
 201
 202         return write_int_sysctl(TCP_FO_SYSCTL, fo | 7);
 203 }
 204
 205 static int enable_syncookie(void)
 206 {
 207         return write_int_sysctl(TCP_SYNCOOKIE_SYSCTL, 2);
 208 }
 209
 210 static int disable_syncookie(void)
 211 {
 212         return write_int_sysctl(TCP_SYNCOOKIE_SYSCTL, 0);
 213 }
 214
 215 static long get_linum(void)
 216 {
 217         __u32 linum;
 218         int err;
 219
 220         err = bpf_map_lookup_elem(linum_map, &index_zero, &linum);
 221         RET_ERR(err < 0, "lookup_elem(linum_map)", "err:%d errno:%d\n",
 222                 err, errno);
 223
 224         return linum;
 225 }
 226
 227 static void check_data(int type, sa_family_t family, const struct cmd *cmd,
 228                        int cli_fd)
 229 {
 230         struct data_check expected = {}, result;
 231         union sa46 cli_sa;
 232         socklen_t addrlen;
 233         int err;
 234
 235         addrlen = sizeof(cli_sa);
 236         err = getsockname(cli_fd, (struct sockaddr *)&cli_sa,
 237                           &addrlen);
 238         RET_IF(err < 0, "getsockname(cli_fd)", "err:%d errno:%d\n",
 239                err, errno);
 240
 241         err = bpf_map_lookup_elem(data_check_map, &index_zero, &result);
 242         RET_IF(err < 0, "lookup_elem(data_check_map)", "err:%d errno:%d\n",
 243                err, errno);
 244
 245         if (type == SOCK_STREAM) {
 246                 expected.len = MIN_TCPHDR_LEN;
 247                 expected.ip_protocol = IPPROTO_TCP;
 248         } else {
 249                 expected.len = UDPHDR_LEN;
 250                 expected.ip_protocol = IPPROTO_UDP;
 251         }
 252
 253         if (family == AF_INET6) {
 254                 expected.eth_protocol = htons(ETH_P_IPV6);
 255                 expected.bind_inany = !srv_sa.v6.sin6_addr.s6_addr32[3] &&
 256                         !srv_sa.v6.sin6_addr.s6_addr32[2] &&
 257                         !srv_sa.v6.sin6_addr.s6_addr32[1] &&
 258                         !srv_sa.v6.sin6_addr.s6_addr32[0];
 259
 260                 memcpy(&expected.skb_addrs[0], cli_sa.v6.sin6_addr.s6_addr32,
 261                        sizeof(cli_sa.v6.sin6_addr));
 262                 memcpy(&expected.skb_addrs[4], &in6addr_loopback,
 263                        sizeof(in6addr_loopback));
 264                 expected.skb_ports[0] = cli_sa.v6.sin6_port;
 265                 expected.skb_ports[1] = srv_sa.v6.sin6_port;
 266         } else {
 267                 expected.eth_protocol = htons(ETH_P_IP);
 268                 expected.bind_inany = !srv_sa.v4.sin_addr.s_addr;
 269
 270                 expected.skb_addrs[0] = cli_sa.v4.sin_addr.s_addr;
 271                 expected.skb_addrs[1] = htonl(INADDR_LOOPBACK);
 272                 expected.skb_ports[0] = cli_sa.v4.sin_port;
 273                 expected.skb_ports[1] = srv_sa.v4.sin_port;
 274         }
 275
 276         if (memcmp(&result, &expected, offsetof(struct data_check,
 277                                                 equal_check_end))) {
 278                 printf("unexpected data_check\n");
 279                 printf("  result: (0x%x, %u, %u)\n",
 280                        result.eth_protocol, result.ip_protocol,
 281                        result.bind_inany);
 282                 printf("expected: (0x%x, %u, %u)\n",
 283                        expected.eth_protocol, expected.ip_protocol,
 284                        expected.bind_inany);
 285                 RET_IF(1, "data_check result != expected",
 286                        "bpf_prog_linum:%ld\n", get_linum());
 287         }
 288
 289         RET_IF(!result.hash, "data_check result.hash empty",
 290                "result.hash:%u", result.hash);
 291
 292         expected.len += cmd ? sizeof(*cmd) : 0;
 293         if (type == SOCK_STREAM)
 294                 RET_IF(expected.len > result.len, "expected.len > result.len",
 295                        "expected.len:%u result.len:%u bpf_prog_linum:%ld\n",
 296                        expected.len, result.len, get_linum());
 297         else
 298                 RET_IF(expected.len != result.len, "expected.len != result.len",
 299                        "expected.len:%u result.len:%u bpf_prog_linum:%ld\n",
 300                        expected.len, result.len, get_linum());
 301 }
 302
 303 static const char *result_to_str(enum result res)
 304 {
 305         switch (res) {
 306         case DROP_ERR_INNER_MAP:
 307                 return "DROP_ERR_INNER_MAP";
 308         case DROP_ERR_SKB_DATA:
 309                 return "DROP_ERR_SKB_DATA";
 310         case DROP_ERR_SK_SELECT_REUSEPORT:
 311                 return "DROP_ERR_SK_SELECT_REUSEPORT";
 312         case DROP_MISC:
 313                 return "DROP_MISC";
 314         case PASS:
 315                 return "PASS";
 316         case PASS_ERR_SK_SELECT_REUSEPORT:
 317                 return "PASS_ERR_SK_SELECT_REUSEPORT";
 318         default:
 319                 return "UNKNOWN";
 320         }
 321 }
 322
 323 static void check_results(void)
 324 {
 325         __u32 results[NR_RESULTS];
 326         __u32 i, broken = 0;
 327         int err;
 328
 329         for (i = 0; i < NR_RESULTS; i++) {
 330                 err = bpf_map_lookup_elem(result_map, &i, &results[i]);
 331                 RET_IF(err < 0, "lookup_elem(result_map)",
 332                        "i:%u err:%d errno:%d\n", i, err, errno);
 333         }
 334
 335         for (i = 0; i < NR_RESULTS; i++) {
 336                 if (results[i] != expected_results[i]) {
 337                         broken = i;
 338                         break;
 339                 }
 340         }
 341
 342         if (i == NR_RESULTS)
 343                 return;
 344
 345         printf("unexpected result\n");
 346         printf(" result: [");
 347         printf("%u", results[0]);
 348         for (i = 1; i < NR_RESULTS; i++)
 349                 printf(", %u", results[i]);
 350         printf("]\n");
 351
 352         printf("expected: [");
 353         printf("%u", expected_results[0]);
 354         for (i = 1; i < NR_RESULTS; i++)
 355                 printf(", %u", expected_results[i]);
 356         printf("]\n");
 357
 358         printf("mismatch on %s (bpf_prog_linum:%ld)\n", result_to_str(broken),
 359                get_linum());
 360
 361         CHECK_FAIL(true);
 362 }
 363
 364 static int send_data(int type, sa_family_t family, void *data, size_t len,
 365                      enum result expected)
 366 {
 367         union sa46 cli_sa;
 368         int fd, err;
 369
 370         fd = socket(family, type, 0);
 371         RET_ERR(fd == -1, "socket()", "fd:%d errno:%d\n", fd, errno);
 372
 373         sa46_init_loopback(&cli_sa, family);
 374         err = bind(fd, (struct sockaddr *)&cli_sa, sizeof(cli_sa));
 375         RET_ERR(fd == -1, "bind(cli_sa)", "err:%d errno:%d\n", err, errno);
 376
 377         err = sendto(fd, data, len, MSG_FASTOPEN, (struct sockaddr *)&srv_sa,
 378                      sizeof(srv_sa));
 379         RET_ERR(err != len && expected >= PASS,
 380                 "sendto()", "family:%u err:%d errno:%d expected:%d\n",
 381                 family, err, errno, expected);
 382
 383         return fd;
 384 }
 385
 386 static void do_test(int type, sa_family_t family, struct cmd *cmd,
 387                     enum result expected)
 388 {
 389         int nev, srv_fd, cli_fd;
 390         struct epoll_event ev;
 391         struct cmd rcv_cmd;
 392         ssize_t nread;
 393
 394         cli_fd = send_data(type, family, cmd, cmd ? sizeof(*cmd) : 0,
 395                            expected);
 396         if (cli_fd < 0)
 397                 return;
 398         nev = epoll_wait(epfd, &ev, 1, expected >= PASS ? 5 : 0);
 399         RET_IF((nev <= 0 && expected >= PASS) ||
 400                (nev > 0 && expected < PASS),
 401                "nev <> expected",
 402                "nev:%d expected:%d type:%d family:%d data:(%d, %d)\n",
 403                nev, expected, type, family,
 404                cmd ? cmd->reuseport_index : -1,
 405                cmd ? cmd->pass_on_failure : -1);
 406         check_results();
 407         check_data(type, family, cmd, cli_fd);
 408
 409         if (expected < PASS)
 410                 return;
 411
 412         RET_IF(expected != PASS_ERR_SK_SELECT_REUSEPORT &&
 413                cmd->reuseport_index != ev.data.u32,
 414                "check cmd->reuseport_index",
 415                "cmd:(%u, %u) ev.data.u32:%u\n",
 416                cmd->pass_on_failure, cmd->reuseport_index, ev.data.u32);
 417
 418         srv_fd = sk_fds[ev.data.u32];
 419         if (type == SOCK_STREAM) {
 420                 int new_fd = accept(srv_fd, NULL, 0);
 421
 422                 RET_IF(new_fd == -1, "accept(srv_fd)",
 423                        "ev.data.u32:%u new_fd:%d errno:%d\n",
 424                        ev.data.u32, new_fd, errno);
 425
 426                 nread = recv(new_fd, &rcv_cmd, sizeof(rcv_cmd), MSG_DONTWAIT);
 427                 RET_IF(nread != sizeof(rcv_cmd),
 428                        "recv(new_fd)",
 429                        "ev.data.u32:%u nread:%zd sizeof(rcv_cmd):%zu errno:%d\n",
 430                        ev.data.u32, nread, sizeof(rcv_cmd), errno);
 431
 432                 close(new_fd);
 433         } else {
 434                 nread = recv(srv_fd, &rcv_cmd, sizeof(rcv_cmd), MSG_DONTWAIT);
 435                 RET_IF(nread != sizeof(rcv_cmd),
 436                        "recv(sk_fds)",
 437                        "ev.data.u32:%u nread:%zd sizeof(rcv_cmd):%zu errno:%d\n",
 438                        ev.data.u32, nread, sizeof(rcv_cmd), errno);
 439         }
 440
 441         close(cli_fd);
 442 }
 443
 444 static void test_err_inner_map(int type, sa_family_t family)
 445 {
 446         struct cmd cmd = {
 447                 .reuseport_index = 0,
 448                 .pass_on_failure = 0,
 449         };
 450
 451         expected_results[DROP_ERR_INNER_MAP]++;
 452         do_test(type, family, &cmd, DROP_ERR_INNER_MAP);
 453 }
 454
 455 static void test_err_skb_data(int type, sa_family_t family)
 456 {
 457         expected_results[DROP_ERR_SKB_DATA]++;
 458         do_test(type, family, NULL, DROP_ERR_SKB_DATA);
 459 }
 460
 461 static void test_err_sk_select_port(int type, sa_family_t family)
 462 {
 463         struct cmd cmd = {
 464                 .reuseport_index = REUSEPORT_ARRAY_SIZE,
 465                 .pass_on_failure = 0,
 466         };
 467
 468         expected_results[DROP_ERR_SK_SELECT_REUSEPORT]++;
 469         do_test(type, family, &cmd, DROP_ERR_SK_SELECT_REUSEPORT);
 470 }
 471
 472 static void test_pass(int type, sa_family_t family)
 473 {
 474         struct cmd cmd;
 475         int i;
 476
 477         cmd.pass_on_failure = 0;
 478         for (i = 0; i < REUSEPORT_ARRAY_SIZE; i++) {
 479                 expected_results[PASS]++;
 480                 cmd.reuseport_index = i;
 481                 do_test(type, family, &cmd, PASS);
 482         }
 483 }
 484
 485 static void test_syncookie(int type, sa_family_t family)
 486 {
 487         int err, tmp_index = 1;
 488         struct cmd cmd = {
 489                 .reuseport_index = 0,
 490                 .pass_on_failure = 0,
 491         };
 492
 493         /*
 494          * +1 for TCP-SYN and
 495          * +1 for the TCP-ACK (ack the syncookie)
 496          */
 497         expected_results[PASS] += 2;
 498         enable_syncookie();
 499         /*
 500          * Simulate TCP-SYN and TCP-ACK are handled by two different sk:
 501          * TCP-SYN: select sk_fds[tmp_index = 1] tmp_index is from the
 502          *          tmp_index_ovr_map
 503          * TCP-ACK: select sk_fds[reuseport_index = 0] reuseport_index
 504          *          is from the cmd.reuseport_index
 505          */
 506         err = bpf_map_update_elem(tmp_index_ovr_map, &index_zero,
 507                                   &tmp_index, BPF_ANY);
 508         RET_IF(err < 0, "update_elem(tmp_index_ovr_map, 0, 1)",
 509                "err:%d errno:%d\n", err, errno);
 510         do_test(type, family, &cmd, PASS);
 511         err = bpf_map_lookup_elem(tmp_index_ovr_map, &index_zero,
 512                                   &tmp_index);
 513         RET_IF(err < 0 || tmp_index >= 0,
 514                "lookup_elem(tmp_index_ovr_map)",
 515                "err:%d errno:%d tmp_index:%d\n",
 516                err, errno, tmp_index);
 517         disable_syncookie();
 518 }
 519
 520 static void test_pass_on_err(int type, sa_family_t family)
 521 {
 522         struct cmd cmd = {
 523                 .reuseport_index = REUSEPORT_ARRAY_SIZE,
 524                 .pass_on_failure = 1,
 525         };
 526
 527         expected_results[PASS_ERR_SK_SELECT_REUSEPORT] += 1;
 528         do_test(type, family, &cmd, PASS_ERR_SK_SELECT_REUSEPORT);
 529 }
 530
 531 static void test_detach_bpf(int type, sa_family_t family)
 532 {
 533 #ifdef SO_DETACH_REUSEPORT_BPF
 534         __u32 nr_run_before = 0, nr_run_after = 0, tmp, i;
 535         struct epoll_event ev;
 536         int cli_fd, err, nev;
 537         struct cmd cmd = {};
 538         int optvalue = 0;
 539
 540         err = setsockopt(sk_fds[0], SOL_SOCKET, SO_DETACH_REUSEPORT_BPF,
 541                          &optvalue, sizeof(optvalue));
 542         RET_IF(err == -1, "setsockopt(SO_DETACH_REUSEPORT_BPF)",
 543                "err:%d errno:%d\n", err, errno);
 544
 545         err = setsockopt(sk_fds[1], SOL_SOCKET, SO_DETACH_REUSEPORT_BPF,
 546                          &optvalue, sizeof(optvalue));
 547         RET_IF(err == 0 || errno != ENOENT,
 548                "setsockopt(SO_DETACH_REUSEPORT_BPF)",
 549                "err:%d errno:%d\n", err, errno);
 550
 551         for (i = 0; i < NR_RESULTS; i++) {
 552                 err = bpf_map_lookup_elem(result_map, &i, &tmp);
 553                 RET_IF(err < 0, "lookup_elem(result_map)",
 554                        "i:%u err:%d errno:%d\n", i, err, errno);
 555                 nr_run_before += tmp;
 556         }
 557
 558         cli_fd = send_data(type, family, &cmd, sizeof(cmd), PASS);
 559         if (cli_fd < 0)
 560                 return;
 561         nev = epoll_wait(epfd, &ev, 1, 5);
 562         RET_IF(nev <= 0, "nev <= 0",
 563                "nev:%d expected:1 type:%d family:%d data:(0, 0)\n",
 564                nev,  type, family);
 565
 566         for (i = 0; i < NR_RESULTS; i++) {
 567                 err = bpf_map_lookup_elem(result_map, &i, &tmp);
 568                 RET_IF(err < 0, "lookup_elem(result_map)",
 569                        "i:%u err:%d errno:%d\n", i, err, errno);
 570                 nr_run_after += tmp;
 571         }
 572
 573         RET_IF(nr_run_before != nr_run_after,
 574                "nr_run_before != nr_run_after",
 575                "nr_run_before:%u nr_run_after:%u\n",
 576                nr_run_before, nr_run_after);
 577
 578         close(cli_fd);
 579 #else
 580         test__skip();
 581 #endif
 582 }
 583
 584 static void prepare_sk_fds(int type, sa_family_t family, bool inany)
 585 {
 586         const int first = REUSEPORT_ARRAY_SIZE - 1;
 587         int i, err, optval = 1;
 588         struct epoll_event ev;
 589         socklen_t addrlen;
 590
 591         if (inany)
 592                 sa46_init_inany(&srv_sa, family);
 593         else
 594                 sa46_init_loopback(&srv_sa, family);
 595         addrlen = sizeof(srv_sa);
 596
 597         /*
 598          * The sk_fds[] is filled from the back such that the order
 599          * is exactly opposite to the (struct sock_reuseport *)reuse->socks[].
 600          */
 601         for (i = first; i >= 0; i--) {
 602                 sk_fds[i] = socket(family, type, 0);
 603                 RET_IF(sk_fds[i] == -1, "socket()", "sk_fds[%d]:%d errno:%d\n",
 604                        i, sk_fds[i], errno);
 605                 err = setsockopt(sk_fds[i], SOL_SOCKET, SO_REUSEPORT,
 606                                  &optval, sizeof(optval));
 607                 RET_IF(err == -1, "setsockopt(SO_REUSEPORT)",
 608                        "sk_fds[%d] err:%d errno:%d\n",
 609                        i, err, errno);
 610
 611                 if (i == first) {
 612                         err = setsockopt(sk_fds[i], SOL_SOCKET,
 613                                          SO_ATTACH_REUSEPORT_EBPF,
 614                                          &select_by_skb_data_prog,
 615                                          sizeof(select_by_skb_data_prog));
 616                         RET_IF(err < 0, "setsockopt(SO_ATTACH_REUEPORT_EBPF)",
 617                                "err:%d errno:%d\n", err, errno);
 618                 }
 619
 620                 err = bind(sk_fds[i], (struct sockaddr *)&srv_sa, addrlen);
 621                 RET_IF(err < 0, "bind()", "sk_fds[%d] err:%d errno:%d\n",
 622                        i, err, errno);
 623
 624                 if (type == SOCK_STREAM) {
 625                         err = listen(sk_fds[i], 10);
 626                         RET_IF(err < 0, "listen()",
 627                                "sk_fds[%d] err:%d errno:%d\n",
 628                                i, err, errno);
 629                 }
 630
 631                 err = bpf_map_update_elem(reuseport_array, &i, &sk_fds[i],
 632                                           BPF_NOEXIST);
 633                 RET_IF(err < 0, "update_elem(reuseport_array)",
 634                        "sk_fds[%d] err:%d errno:%d\n", i, err, errno);
 635
 636                 if (i == first) {
 637                         socklen_t addrlen = sizeof(srv_sa);
 638
 639                         err = getsockname(sk_fds[i], (struct sockaddr *)&srv_sa,
 640                                           &addrlen);
 641                         RET_IF(err == -1, "getsockname()",
 642                                "sk_fds[%d] err:%d errno:%d\n", i, err, errno);
 643                 }
 644         }
 645
 646         epfd = epoll_create(1);
 647         RET_IF(epfd == -1, "epoll_create(1)",
 648                "epfd:%d errno:%d\n", epfd, errno);
 649
 650         ev.events = EPOLLIN;
 651         for (i = 0; i < REUSEPORT_ARRAY_SIZE; i++) {
 652                 ev.data.u32 = i;
 653                 err = epoll_ctl(epfd, EPOLL_CTL_ADD, sk_fds[i], &ev);
 654                 RET_IF(err, "epoll_ctl(EPOLL_CTL_ADD)", "sk_fds[%d]\n", i);
 655         }
 656 }
 657
 658 static void setup_per_test(int type, sa_family_t family, bool inany,
 659                            bool no_inner_map)
 660 {
 661         int ovr = -1, err;
 662
 663         prepare_sk_fds(type, family, inany);
 664         err = bpf_map_update_elem(tmp_index_ovr_map, &index_zero, &ovr,
 665                                   BPF_ANY);
 666         RET_IF(err < 0, "update_elem(tmp_index_ovr_map, 0, -1)",
 667                "err:%d errno:%d\n", err, errno);
 668
 669         /* Install reuseport_array to outer_map? */
 670         if (no_inner_map)
 671                 return;
 672
 673         err = bpf_map_update_elem(outer_map, &index_zero, &reuseport_array,
 674                                   BPF_ANY);
 675         RET_IF(err < 0, "update_elem(outer_map, 0, reuseport_array)",
 676                "err:%d errno:%d\n", err, errno);
 677 }
 678
 679 static void cleanup_per_test(bool no_inner_map)
 680 {
 681         int i, err, zero = 0;
 682
 683         memset(expected_results, 0, sizeof(expected_results));
 684
 685         for (i = 0; i < NR_RESULTS; i++) {
 686                 err = bpf_map_update_elem(result_map, &i, &zero, BPF_ANY);
 687                 RET_IF(err, "reset elem in result_map",
 688                        "i:%u err:%d errno:%d\n", i, err, errno);
 689         }
 690
 691         err = bpf_map_update_elem(linum_map, &zero, &zero, BPF_ANY);
 692         RET_IF(err, "reset line number in linum_map", "err:%d errno:%d\n",
 693                err, errno);
 694
 695         for (i = 0; i < REUSEPORT_ARRAY_SIZE; i++)
 696                 close(sk_fds[i]);
 697         close(epfd);
 698
 699         /* Delete reuseport_array from outer_map? */
 700         if (no_inner_map)
 701                 return;
 702
 703         err = bpf_map_delete_elem(outer_map, &index_zero);
 704         RET_IF(err < 0, "delete_elem(outer_map)",
 705                "err:%d errno:%d\n", err, errno);
 706 }
 707
 708 static void cleanup(void)
 709 {
 710         if (outer_map >= 0) {
 711                 close(outer_map);
 712                 outer_map = -1;
 713         }
 714
 715         if (reuseport_array >= 0) {
 716                 close(reuseport_array);
 717                 reuseport_array = -1;
 718         }
 719
 720         if (obj) {
 721                 bpf_object__close(obj);
 722                 obj = NULL;
 723         }
 724
 725         memset(expected_results, 0, sizeof(expected_results));
 726 }
 727
 728 static const char *maptype_str(enum bpf_map_type type)
 729 {
 730         switch (type) {
 731         case BPF_MAP_TYPE_REUSEPORT_SOCKARRAY:
 732                 return "reuseport_sockarray";
 733         case BPF_MAP_TYPE_SOCKMAP:
 734                 return "sockmap";
 735         case BPF_MAP_TYPE_SOCKHASH:
 736                 return "sockhash";
 737         default:
 738                 return "unknown";
 739         }
 740 }
 741
 742 static const char *family_str(sa_family_t family)
 743 {
 744         switch (family) {
 745         case AF_INET:
 746                 return "IPv4";
 747         case AF_INET6:
 748                 return "IPv6";
 749         default:
 750                 return "unknown";
 751         }
 752 }
 753
 754 static const char *sotype_str(int sotype)
 755 {
 756         switch (sotype) {
 757         case SOCK_STREAM:
 758                 return "TCP";
 759         case SOCK_DGRAM:
 760                 return "UDP";
 761         default:
 762                 return "unknown";
 763         }
 764 }
 765
 766 #define TEST_INIT(fn_, ...) { .fn = fn_, .name = #fn_, __VA_ARGS__ }
 767
 768 static void test_config(int sotype, sa_family_t family, bool inany)
 769 {
 770         const struct test {
 771                 void (*fn)(int sotype, sa_family_t family);
 772                 const char *name;
 773                 bool no_inner_map;
 774                 int need_sotype;
 775         } tests[] = {
 776                 TEST_INIT(test_err_inner_map,
 777                           .no_inner_map = true),
 778                 TEST_INIT(test_err_skb_data),
 779                 TEST_INIT(test_err_sk_select_port),
 780                 TEST_INIT(test_pass),
 781                 TEST_INIT(test_syncookie,
 782                           .need_sotype = SOCK_STREAM),
 783                 TEST_INIT(test_pass_on_err),
 784                 TEST_INIT(test_detach_bpf),
 785         };
 786         struct netns_obj *netns;
 787         char s[MAX_TEST_NAME];
 788         const struct test *t;
 789
 790         for (t = tests; t < tests + ARRAY_SIZE(tests); t++) {
 791                 if (t->need_sotype && t->need_sotype != sotype)
 792                         continue; /* test not compatible with socket type */
 793
 794                 snprintf(s, sizeof(s), "%s %s/%s %s %s",
 795                          maptype_str(inner_map_type),
 796                          family_str(family), sotype_str(sotype),
 797                          inany ? "INANY" : "LOOPBACK", t->name);
 798
 799                 if (!test__start_subtest(s))
 800                         continue;
 801
 802                 netns = netns_new("select_reuseport", true);
 803                 if (!ASSERT_OK_PTR(netns, "netns_new"))
 804                         continue;
 805
 806                 if (CHECK_FAIL(enable_fastopen()))
 807                         goto out;
 808                 if (CHECK_FAIL(disable_syncookie()))
 809                         goto out;
 810
 811                 setup_per_test(sotype, family, inany, t->no_inner_map);
 812                 t->fn(sotype, family);
 813                 cleanup_per_test(t->no_inner_map);
 814
 815 out:
 816                 netns_free(netns);
 817         }
 818 }
 819
 820 #define BIND_INANY true
 821
 822 static void test_all(void)
 823 {
 824         const struct config {
 825                 int sotype;
 826                 sa_family_t family;
 827                 bool inany;
 828         } configs[] = {
 829                 { SOCK_STREAM, AF_INET },
 830                 { SOCK_STREAM, AF_INET, BIND_INANY },
 831                 { SOCK_STREAM, AF_INET6 },
 832                 { SOCK_STREAM, AF_INET6, BIND_INANY },
 833                 { SOCK_DGRAM, AF_INET },
 834                 { SOCK_DGRAM, AF_INET6 },
 835         };
 836         const struct config *c;
 837
 838         for (c = configs; c < configs + ARRAY_SIZE(configs); c++)
 839                 test_config(c->sotype, c->family, c->inany);
 840 }
 841
 842 void test_map_type(enum bpf_map_type mt)
 843 {
 844         if (create_maps(mt))
 845                 goto out;
 846         if (prepare_bpf_obj())
 847                 goto out;
 848
 849         test_all();
 850 out:
 851         cleanup();
 852 }
 853
 854 void serial_test_select_reuseport(void)
 855 {
 856         test_map_type(BPF_MAP_TYPE_REUSEPORT_SOCKARRAY);
 857         test_map_type(BPF_MAP_TYPE_SOCKMAP);
 858         test_map_type(BPF_MAP_TYPE_SOCKHASH);
 859 }