Merge tag 'bpf-next-6.12' of git://git.kernel.org/pub/scm/linux/kernel/git/bpf/bpf...
[linux.git] / tools / testing / selftests / bpf / xskxceiver.c
CommitLineData
facb7cb2
WJ
1// SPDX-License-Identifier: GPL-2.0
2/* Copyright(c) 2020 Intel Corporation. */
3
4/*
5 * Some functions in this program are taken from
6 * Linux kernel samples/bpf/xdpsock* and modified
7 * for use.
8 *
9 * See test_xsk.sh for detailed information on test topology
10 * and prerequisite network setup.
11 *
12 * This test program contains two threads, each thread is single socket with
13 * a unique UMEM. It validates in-order packet delivery and packet content
14 * by sending packets to each other.
15 *
16 * Tests Information:
17 * ------------------
18 * These selftests test AF_XDP SKB and Native/DRV modes using veth
19 * Virtual Ethernet interfaces.
20 *
d3e3bf5b 21 * For each mode, the following tests are run:
a4ba98dd 22 * a. nopoll - soft-irq processing in run-to-completion mode
facb7cb2 23 * b. poll - using poll() syscall
6674bf66
WJ
24 * c. Socket Teardown
25 * Create a Tx and a Rx socket, Tx from one socket, Rx on another. Destroy
26 * both sockets, then repeat multiple times. Only nopoll mode is used
7d20441e
WJ
27 * d. Bi-directional sockets
28 * Configure sockets as bi-directional tx/rx sockets, sets up fill and
29 * completion rings on each socket, tx/rx in both directions. Only nopoll
30 * mode is used
b267e5a4
CL
31 * e. Statistics
32 * Trigger some error conditions and ensure that the appropriate statistics
33 * are incremented. Within this test, the following statistics are tested:
34 * i. rx dropped
35 * Increase the UMEM frame headroom to a value which results in
36 * insufficient space in the rx buffer for both the packet and the headroom.
37 * ii. tx invalid
38 * Set the 'len' field of tx descriptors to an invalid value (umem frame
39 * size + 1).
40 * iii. rx ring full
41 * Reduce the size of the RX ring to a fraction of the fill ring size.
42 * iv. fill queue empty
43 * Do not populate the fill queue and then try to receive pkts.
27e1ca25
MF
44 * f. bpf_link resource persistence
45 * Configure sockets at indexes 0 and 1, run a traffic on queue ids 0,
46 * then remove xsk sockets from queue 0 on both veth interfaces and
47 * finally run a traffic on queues ids 1
a4ba98dd 48 * g. unaligned mode
0d1b7f3a
MK
49 * h. tests for invalid and corner case Tx descriptors so that the correct ones
50 * are discarded and let through, respectively.
909f0e28 51 * i. 2K frame size tests
f540d44e 52 * j. If multi-buffer is supported, send 9k packets divided into 3 frames
1005a226
MK
53 * k. If multi-buffer and huge pages are supported, send 9k packets in a single frame
54 * using unaligned mode
69760449
MK
55 * l. If multi-buffer is supported, try various nasty combinations of descriptors to
56 * check if they pass the validation or not
facb7cb2
WJ
57 *
58 * Flow:
59 * -----
60 * - Single process spawns two threads: Tx and Rx
64aef77d
MK
61 * - Each of these two threads attach to a veth interface
62 * - Each thread creates one AF_XDP socket connected to a unique umem for each
facb7cb2 63 * veth interface
64aef77d
MK
64 * - Tx thread Transmits a number of packets from veth<xxxx> to veth<yyyy>
65 * - Rx thread verifies if all packets were received and delivered in-order,
facb7cb2
WJ
66 * and have the right content
67 *
d2b0dfd5 68 * Enable/disable packet dump mode:
facb7cb2
WJ
69 * --------------------------
70 * To enable L2 - L4 headers and payload dump of each packet on STDOUT, add
71 * parameter -D to params array in test_xsk.sh, i.e. params=("-S" "-D")
72 */
73
74#define _GNU_SOURCE
c0801598 75#include <assert.h>
facb7cb2
WJ
76#include <fcntl.h>
77#include <errno.h>
78#include <getopt.h>
facb7cb2
WJ
79#include <linux/if_link.h>
80#include <linux/if_ether.h>
2ddade32 81#include <linux/mman.h>
f540d44e 82#include <linux/netdev.h>
8913e653 83#include <linux/bitmap.h>
776021e0 84#include <linux/ethtool.h>
facb7cb2
WJ
85#include <arpa/inet.h>
86#include <net/if.h>
87#include <locale.h>
88#include <poll.h>
89#include <pthread.h>
90#include <signal.h>
facb7cb2
WJ
91#include <stdio.h>
92#include <stdlib.h>
c0247800 93#include <libgen.h>
facb7cb2
WJ
94#include <string.h>
95#include <stddef.h>
96#include <sys/mman.h>
f90062b5 97#include <sys/socket.h>
db1bd7a9 98#include <sys/time.h>
facb7cb2 99#include <sys/types.h>
facb7cb2 100#include <unistd.h>
7d8319a7
MK
101
102#include "xsk_xdp_progs.skel.h"
f3660063 103#include "xsk.h"
018a8e75 104#include "xskxceiver.h"
0d68e6fe
MF
105#include <bpf/bpf.h>
106#include <linux/filter.h>
facb7cb2 107#include "../kselftest.h"
93ba1124 108#include "xsk_xdp_common.h"
facb7cb2 109
776021e0
TV
110#include <network_helpers.h>
111
3956bc34 112static bool opt_verbose;
c53dab7d 113static bool opt_print_tests;
3956bc34 114static enum test_mode opt_mode = TEST_MODE_ALL;
146e3055 115static u32 opt_run_test = RUN_ALL_TESTS;
3956bc34 116
776021e0
TV
117void test__fail(void) { /* for network_helpers.c */ }
118
facb7cb2
WJ
119static void __exit_with_error(int error, const char *file, const char *func, int line)
120{
279bdf6b
MK
121 ksft_test_result_fail("[%s:%s:%i]: ERROR: %d/\"%s\"\n", file, func, line, error,
122 strerror(error));
123 ksft_exit_xfail();
facb7cb2
WJ
124}
125
126#define exit_with_error(error) __exit_with_error(error, __FILE__, __func__, __LINE__)
f90062b5 127#define busy_poll_string(test) (test)->ifobj_tx->busy_poll ? "BUSY-POLL " : ""
fe2ad08e
MF
128static char *mode_string(struct test_spec *test)
129{
130 switch (test->mode) {
131 case TEST_MODE_SKB:
132 return "SKB";
133 case TEST_MODE_DRV:
134 return "DRV";
135 case TEST_MODE_ZC:
136 return "ZC";
137 default:
138 return "BOGUS";
139 }
140}
af6731d1 141
895b62ee
MK
142static void report_failure(struct test_spec *test)
143{
144 if (test->fail)
145 return;
146
147 ksft_test_result_fail("FAIL: %s %s%s\n", mode_string(test), busy_poll_string(test),
148 test->name);
149 test->fail = true;
150}
facb7cb2 151
feb973a9
MK
152/* The payload is a word consisting of a packet sequence number in the upper
153 * 16-bits and a intra packet data sequence number in the lower 16 bits. So the 3rd packet's
154 * 5th word of data will contain the number (2<<16) | 4 as they are numbered from 0.
155 */
2f6eae0d 156static void write_payload(void *dest, u32 pkt_nb, u32 start, u32 size)
facb7cb2 157{
feb973a9 158 u32 *ptr = (u32 *)dest, i;
facb7cb2 159
2f6eae0d
MK
160 start /= sizeof(*ptr);
161 size /= sizeof(*ptr);
162 for (i = 0; i < size; i++)
163 ptr[i] = htonl(pkt_nb << 16 | (i + start));
facb7cb2
WJ
164}
165
985fd214 166static void gen_eth_hdr(struct xsk_socket_info *xsk, struct ethhdr *eth_hdr)
facb7cb2 167{
985fd214
TV
168 memcpy(eth_hdr->h_dest, xsk->dst_mac, ETH_ALEN);
169 memcpy(eth_hdr->h_source, xsk->src_mac, ETH_ALEN);
df82d2e8 170 eth_hdr->h_proto = htons(ETH_P_LOOPBACK);
facb7cb2
WJ
171}
172
3143d10b
SKR
173static bool is_umem_valid(struct ifobject *ifobj)
174{
175 return !!ifobj->umem->umem;
176}
177
aa61d81f
MK
178static u32 mode_to_xdp_flags(enum test_mode mode)
179{
180 return (mode == TEST_MODE_SKB) ? XDP_FLAGS_SKB_MODE : XDP_FLAGS_DRV_MODE;
181}
182
d9f6d970
MK
183static u64 umem_size(struct xsk_umem_info *umem)
184{
185 return umem->num_frames * umem->frame_size;
186}
187
188static int xsk_configure_umem(struct ifobject *ifobj, struct xsk_umem_info *umem, void *buffer,
189 u64 size)
facb7cb2 190{
b267e5a4
CL
191 struct xsk_umem_config cfg = {
192 .fill_size = XSK_RING_PROD__DEFAULT_NUM_DESCS,
193 .comp_size = XSK_RING_CONS__DEFAULT_NUM_DESCS,
c160d7af 194 .frame_size = umem->frame_size,
83f4ae2f 195 .frame_headroom = umem->frame_headroom,
b267e5a4
CL
196 .flags = XSK_UMEM__DEFAULT_FLAGS
197 };
27e1ca25 198 int ret;
facb7cb2 199
e4a195e2
TV
200 if (umem->fill_size)
201 cfg.fill_size = umem->fill_size;
202
203 if (umem->comp_size)
204 cfg.comp_size = umem->comp_size;
205
a4ba98dd
MK
206 if (umem->unaligned_mode)
207 cfg.flags |= XDP_UMEM_UNALIGNED_CHUNK_FLAG;
208
27e1ca25
MF
209 ret = xsk_umem__create(&umem->umem, buffer, size,
210 &umem->fq, &umem->cq, &cfg);
facb7cb2 211 if (ret)
ed7b74dc 212 return ret;
facb7cb2 213
27e1ca25 214 umem->buffer = buffer;
d9f6d970
MK
215 if (ifobj->shared_umem && ifobj->rx_on) {
216 umem->base_addr = umem_size(umem);
217 umem->next_buffer = umem_size(umem);
218 }
219
ed7b74dc 220 return 0;
facb7cb2
WJ
221}
222
d9f6d970
MK
223static u64 umem_alloc_buffer(struct xsk_umem_info *umem)
224{
225 u64 addr;
226
227 addr = umem->next_buffer;
228 umem->next_buffer += umem->frame_size;
229 if (umem->next_buffer >= umem->base_addr + umem_size(umem))
230 umem->next_buffer = umem->base_addr;
231
232 return addr;
233}
234
235static void umem_reset_alloc(struct xsk_umem_info *umem)
236{
237 umem->next_buffer = 0;
238}
239
f90062b5
MK
240static void enable_busy_poll(struct xsk_socket_info *xsk)
241{
242 int sock_opt;
243
244 sock_opt = 1;
245 if (setsockopt(xsk_socket__fd(xsk->xsk), SOL_SOCKET, SO_PREFER_BUSY_POLL,
246 (void *)&sock_opt, sizeof(sock_opt)) < 0)
247 exit_with_error(errno);
248
249 sock_opt = 20;
250 if (setsockopt(xsk_socket__fd(xsk->xsk), SOL_SOCKET, SO_BUSY_POLL,
251 (void *)&sock_opt, sizeof(sock_opt)) < 0)
252 exit_with_error(errno);
253
c3bd0150 254 sock_opt = xsk->batch_size;
f90062b5
MK
255 if (setsockopt(xsk_socket__fd(xsk->xsk), SOL_SOCKET, SO_BUSY_POLL_BUDGET,
256 (void *)&sock_opt, sizeof(sock_opt)) < 0)
257 exit_with_error(errno);
258}
259
a693ff3e
MF
260static int __xsk_configure_socket(struct xsk_socket_info *xsk, struct xsk_umem_info *umem,
261 struct ifobject *ifobject, bool shared)
facb7cb2 262{
3b22523b 263 struct xsk_socket_config cfg = {};
facb7cb2
WJ
264 struct xsk_ring_cons *rxr;
265 struct xsk_ring_prod *txr;
facb7cb2 266
ed7b74dc 267 xsk->umem = umem;
4bf8ee65 268 cfg.rx_size = xsk->rxqsize;
facb7cb2 269 cfg.tx_size = XSK_RING_PROD__DEFAULT_NUM_DESCS;
af6731d1 270 cfg.bind_flags = ifobject->bind_flags;
3b22523b
MK
271 if (shared)
272 cfg.bind_flags |= XDP_SHARED_UMEM;
8367eb95 273 if (ifobject->mtu > MAX_ETH_PKT_SIZE)
f540d44e 274 cfg.bind_flags |= XDP_USE_SG;
e4a195e2
TV
275 if (umem->comp_size)
276 cfg.tx_size = umem->comp_size;
277 if (umem->fill_size)
278 cfg.rx_size = umem->fill_size;
facb7cb2 279
1856c24d
MK
280 txr = ifobject->tx_on ? &xsk->tx : NULL;
281 rxr = ifobject->rx_on ? &xsk->rx : NULL;
aa61d81f 282 return xsk_socket__create(&xsk->xsk, ifobject->ifindex, 0, umem->umem, rxr, txr, &cfg);
facb7cb2
WJ
283}
284
fe2ad08e
MF
285static bool ifobj_zc_avail(struct ifobject *ifobject)
286{
287 size_t umem_sz = DEFAULT_UMEM_BUFFERS * XSK_UMEM__DEFAULT_FRAME_SIZE;
288 int mmap_flags = MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE;
289 struct xsk_socket_info *xsk;
290 struct xsk_umem_info *umem;
291 bool zc_avail = false;
292 void *bufs;
293 int ret;
294
295 bufs = mmap(NULL, umem_sz, PROT_READ | PROT_WRITE, mmap_flags, -1, 0);
296 if (bufs == MAP_FAILED)
297 exit_with_error(errno);
298
299 umem = calloc(1, sizeof(struct xsk_umem_info));
300 if (!umem) {
301 munmap(bufs, umem_sz);
085dcccf 302 exit_with_error(ENOMEM);
fe2ad08e
MF
303 }
304 umem->frame_size = XSK_UMEM__DEFAULT_FRAME_SIZE;
d9f6d970 305 ret = xsk_configure_umem(ifobject, umem, bufs, umem_sz);
fe2ad08e
MF
306 if (ret)
307 exit_with_error(-ret);
308
309 xsk = calloc(1, sizeof(struct xsk_socket_info));
310 if (!xsk)
311 goto out;
fe2ad08e
MF
312 ifobject->bind_flags = XDP_USE_NEED_WAKEUP | XDP_ZEROCOPY;
313 ifobject->rx_on = true;
314 xsk->rxqsize = XSK_RING_CONS__DEFAULT_NUM_DESCS;
315 ret = __xsk_configure_socket(xsk, umem, ifobject, false);
316 if (!ret)
317 zc_avail = true;
318
319 xsk_socket__delete(xsk->xsk);
320 free(xsk);
321out:
322 munmap(umem->buffer, umem_sz);
323 xsk_umem__delete(umem->umem);
324 free(umem);
325 return zc_avail;
326}
327
d41905b3
MF
328#define MAX_SKB_FRAGS_PATH "/proc/sys/net/core/max_skb_frags"
329static unsigned int get_max_skb_frags(void)
330{
331 unsigned int max_skb_frags = 0;
332 FILE *file;
333
334 file = fopen(MAX_SKB_FRAGS_PATH, "r");
335 if (!file) {
336 ksft_print_msg("Error opening %s\n", MAX_SKB_FRAGS_PATH);
337 return 0;
338 }
339
340 if (fscanf(file, "%u", &max_skb_frags) != 1)
341 ksft_print_msg("Error reading %s\n", MAX_SKB_FRAGS_PATH);
342
343 fclose(file);
344 return max_skb_frags;
345}
346
facb7cb2
WJ
347static struct option long_options[] = {
348 {"interface", required_argument, 0, 'i'},
f90062b5 349 {"busy-poll", no_argument, 0, 'b'},
ecde6061 350 {"verbose", no_argument, 0, 'v'},
3956bc34 351 {"mode", required_argument, 0, 'm'},
c53dab7d 352 {"list", no_argument, 0, 'l'},
146e3055 353 {"test", required_argument, 0, 't'},
4a5f0ba5 354 {"help", no_argument, 0, 'h'},
facb7cb2
WJ
355 {0, 0, 0, 0}
356};
357
146e3055 358static void print_usage(char **argv)
facb7cb2
WJ
359{
360 const char *str =
3956bc34 361 " Usage: xskxceiver [OPTIONS]\n"
25c0a305
MK
362 " Options:\n"
363 " -i, --interface Use interface\n"
f90062b5 364 " -v, --verbose Verbose output\n"
3956bc34 365 " -b, --busy-poll Enable busy poll\n"
c53dab7d 366 " -m, --mode Run only mode skb, drv, or zc\n"
146e3055 367 " -l, --list List all available tests\n"
4a5f0ba5
MK
368 " -t, --test Run a specific test. Enter number from -l option.\n"
369 " -h, --help Display this help and exit\n";
25c0a305 370
146e3055
MK
371 ksft_print_msg(str, basename(argv[0]));
372 ksft_exit_xfail();
facb7cb2
WJ
373}
374
ce74acaf 375static bool validate_interface(struct ifobject *ifobj)
facb7cb2 376{
ce74acaf
MK
377 if (!strcmp(ifobj->ifname, ""))
378 return false;
379 return true;
facb7cb2
WJ
380}
381
af6731d1
MK
382static void parse_command_line(struct ifobject *ifobj_tx, struct ifobject *ifobj_rx, int argc,
383 char **argv)
facb7cb2 384{
ce74acaf
MK
385 struct ifobject *ifobj;
386 u32 interface_nb = 0;
387 int option_index, c;
facb7cb2
WJ
388
389 opterr = 0;
390
391 for (;;) {
146e3055 392 c = getopt_long(argc, argv, "i:vbm:lt:", long_options, &option_index);
facb7cb2
WJ
393 if (c == -1)
394 break;
395
396 switch (c) {
397 case 'i':
ce74acaf 398 if (interface_nb == 0)
af6731d1 399 ifobj = ifobj_tx;
ce74acaf 400 else if (interface_nb == 1)
af6731d1 401 ifobj = ifobj_rx;
ce74acaf 402 else
facb7cb2 403 break;
facb7cb2 404
64aef77d
MK
405 memcpy(ifobj->ifname, optarg,
406 min_t(size_t, MAX_INTERFACE_NAME_CHARS, strlen(optarg)));
aa61d81f
MK
407
408 ifobj->ifindex = if_nametoindex(ifobj->ifname);
409 if (!ifobj->ifindex)
410 exit_with_error(errno);
411
ce74acaf 412 interface_nb++;
facb7cb2 413 break;
ecde6061 414 case 'v':
33a6bef8 415 opt_verbose = true;
ecde6061 416 break;
f90062b5
MK
417 case 'b':
418 ifobj_tx->busy_poll = true;
419 ifobj_rx->busy_poll = true;
420 break;
3956bc34 421 case 'm':
146e3055 422 if (!strncmp("skb", optarg, strlen(optarg)))
3956bc34 423 opt_mode = TEST_MODE_SKB;
146e3055 424 else if (!strncmp("drv", optarg, strlen(optarg)))
3956bc34 425 opt_mode = TEST_MODE_DRV;
146e3055 426 else if (!strncmp("zc", optarg, strlen(optarg)))
3956bc34 427 opt_mode = TEST_MODE_ZC;
146e3055
MK
428 else
429 print_usage(argv);
3956bc34 430 break;
c53dab7d
MK
431 case 'l':
432 opt_print_tests = true;
433 break;
146e3055
MK
434 case 't':
435 errno = 0;
436 opt_run_test = strtol(optarg, NULL, 0);
437 if (errno)
438 print_usage(argv);
439 break;
4a5f0ba5 440 case 'h':
facb7cb2 441 default:
146e3055 442 print_usage(argv);
facb7cb2
WJ
443 }
444 }
ce74acaf 445}
facb7cb2 446
776021e0
TV
447static int set_ring_size(struct ifobject *ifobj)
448{
449 int ret;
450 u32 ctr = 0;
451
452 while (ctr++ < SOCK_RECONF_CTR) {
453 ret = set_hw_ring_size(ifobj->ifname, &ifobj->ring);
454 if (!ret)
455 break;
456
457 /* Retry if it fails */
458 if (ctr >= SOCK_RECONF_CTR || errno != EBUSY)
459 return -errno;
460
461 usleep(USLEEP_MAX);
462 }
463
464 return ret;
465}
466
467static int hw_ring_size_reset(struct ifobject *ifobj)
468{
469 ifobj->ring.tx_pending = ifobj->set_ring.default_tx;
470 ifobj->ring.rx_pending = ifobj->set_ring.default_rx;
471 return set_ring_size(ifobj);
472}
473
ce74acaf
MK
474static void __test_spec_init(struct test_spec *test, struct ifobject *ifobj_tx,
475 struct ifobject *ifobj_rx)
476{
477 u32 i, j;
478
479 for (i = 0; i < MAX_INTERFACES; i++) {
480 struct ifobject *ifobj = i ? ifobj_rx : ifobj_tx;
481
ce74acaf 482 ifobj->xsk = &ifobj->xsk_arr[0];
119d4b02 483 ifobj->use_poll = false;
27e934be
MK
484 ifobj->use_fill_ring = true;
485 ifobj->release_rx = true;
76c57663 486 ifobj->validation_func = NULL;
9a321fd3 487 ifobj->use_metadata = false;
ce74acaf 488
1856c24d
MK
489 if (i == 0) {
490 ifobj->rx_on = false;
491 ifobj->tx_on = true;
492 } else {
493 ifobj->rx_on = true;
494 ifobj->tx_on = false;
495 }
ce74acaf 496
3b22523b
MK
497 memset(ifobj->umem, 0, sizeof(*ifobj->umem));
498 ifobj->umem->num_frames = DEFAULT_UMEM_BUFFERS;
499 ifobj->umem->frame_size = XSK_UMEM__DEFAULT_FRAME_SIZE;
500
ce74acaf 501 for (j = 0; j < MAX_SOCKETS; j++) {
ce74acaf 502 memset(&ifobj->xsk_arr[j], 0, sizeof(ifobj->xsk_arr[j]));
4bf8ee65 503 ifobj->xsk_arr[j].rxqsize = XSK_RING_CONS__DEFAULT_NUM_DESCS;
c3bd0150 504 ifobj->xsk_arr[j].batch_size = DEFAULT_BATCH_SIZE;
8367eb95
TV
505 if (i == 0)
506 ifobj->xsk_arr[j].pkt_stream = test->tx_pkt_stream_default;
507 else
508 ifobj->xsk_arr[j].pkt_stream = test->rx_pkt_stream_default;
985fd214
TV
509
510 memcpy(ifobj->xsk_arr[j].src_mac, g_mac, ETH_ALEN);
511 memcpy(ifobj->xsk_arr[j].dst_mac, g_mac, ETH_ALEN);
512 ifobj->xsk_arr[j].src_mac[5] += ((j * 2) + 0);
513 ifobj->xsk_arr[j].dst_mac[5] += ((j * 2) + 1);
ce74acaf 514 }
facb7cb2 515 }
ce74acaf 516
776021e0
TV
517 if (ifobj_tx->hw_ring_size_supp)
518 hw_ring_size_reset(ifobj_tx);
519
ce74acaf
MK
520 test->ifobj_tx = ifobj_tx;
521 test->ifobj_rx = ifobj_rx;
55be575d
MK
522 test->current_step = 0;
523 test->total_steps = 1;
85c6c957 524 test->nb_sockets = 1;
895b62ee 525 test->fail = false;
776021e0 526 test->set_ring = false;
f540d44e 527 test->mtu = MAX_ETH_PKT_SIZE;
7d8319a7
MK
528 test->xdp_prog_rx = ifobj_rx->xdp_progs->progs.xsk_def_prog;
529 test->xskmap_rx = ifobj_rx->xdp_progs->maps.xsk;
530 test->xdp_prog_tx = ifobj_tx->xdp_progs->progs.xsk_def_prog;
531 test->xskmap_tx = ifobj_tx->xdp_progs->maps.xsk;
ce74acaf
MK
532}
533
534static void test_spec_init(struct test_spec *test, struct ifobject *ifobj_tx,
f20fbcd0
MK
535 struct ifobject *ifobj_rx, enum test_mode mode,
536 const struct test_spec *test_to_run)
ce74acaf 537{
1adef064
MF
538 struct pkt_stream *tx_pkt_stream;
539 struct pkt_stream *rx_pkt_stream;
af6731d1
MK
540 u32 i;
541
1adef064
MF
542 tx_pkt_stream = test->tx_pkt_stream_default;
543 rx_pkt_stream = test->rx_pkt_stream_default;
ce74acaf 544 memset(test, 0, sizeof(*test));
1adef064
MF
545 test->tx_pkt_stream_default = tx_pkt_stream;
546 test->rx_pkt_stream_default = rx_pkt_stream;
af6731d1
MK
547
548 for (i = 0; i < MAX_INTERFACES; i++) {
549 struct ifobject *ifobj = i ? ifobj_rx : ifobj_tx;
550
fe2ad08e
MF
551 ifobj->bind_flags = XDP_USE_NEED_WAKEUP;
552 if (mode == TEST_MODE_ZC)
553 ifobj->bind_flags |= XDP_ZEROCOPY;
554 else
555 ifobj->bind_flags |= XDP_COPY;
af6731d1
MK
556 }
557
f20fbcd0
MK
558 strncpy(test->name, test_to_run->name, MAX_TEST_NAME_SIZE);
559 test->test_func = test_to_run->test_func;
fe2ad08e 560 test->mode = mode;
ce74acaf
MK
561 __test_spec_init(test, ifobj_tx, ifobj_rx);
562}
563
564static void test_spec_reset(struct test_spec *test)
565{
566 __test_spec_init(test, test->ifobj_tx, test->ifobj_rx);
facb7cb2
WJ
567}
568
7d8319a7
MK
569static void test_spec_set_xdp_prog(struct test_spec *test, struct bpf_program *xdp_prog_rx,
570 struct bpf_program *xdp_prog_tx, struct bpf_map *xskmap_rx,
571 struct bpf_map *xskmap_tx)
572{
573 test->xdp_prog_rx = xdp_prog_rx;
574 test->xdp_prog_tx = xdp_prog_tx;
575 test->xskmap_rx = xskmap_rx;
576 test->xskmap_tx = xskmap_tx;
577}
578
f540d44e
MK
579static int test_spec_set_mtu(struct test_spec *test, int mtu)
580{
581 int err;
582
583 if (test->ifobj_rx->mtu != mtu) {
584 err = xsk_set_mtu(test->ifobj_rx->ifindex, mtu);
585 if (err)
586 return err;
587 test->ifobj_rx->mtu = mtu;
588 }
589 if (test->ifobj_tx->mtu != mtu) {
590 err = xsk_set_mtu(test->ifobj_tx->ifindex, mtu);
591 if (err)
592 return err;
593 test->ifobj_tx->mtu = mtu;
594 }
595
596 return 0;
597}
598
5b132056
MK
599static void pkt_stream_reset(struct pkt_stream *pkt_stream)
600{
8913e653 601 if (pkt_stream) {
69fc03d2 602 pkt_stream->current_pkt_nb = 0;
8913e653
TV
603 pkt_stream->nb_rx_pkts = 0;
604 }
5b132056
MK
605}
606
69fc03d2 607static struct pkt *pkt_stream_get_next_tx_pkt(struct pkt_stream *pkt_stream)
960b6e01 608{
69fc03d2 609 if (pkt_stream->current_pkt_nb >= pkt_stream->nb_pkts)
29f128b3
MK
610 return NULL;
611
69fc03d2 612 return &pkt_stream->pkts[pkt_stream->current_pkt_nb++];
29f128b3
MK
613}
614
27e934be 615static struct pkt *pkt_stream_get_next_rx_pkt(struct pkt_stream *pkt_stream, u32 *pkts_sent)
0d1b7f3a 616{
69fc03d2 617 while (pkt_stream->current_pkt_nb < pkt_stream->nb_pkts) {
27e934be 618 (*pkts_sent)++;
69fc03d2
MK
619 if (pkt_stream->pkts[pkt_stream->current_pkt_nb].valid)
620 return &pkt_stream->pkts[pkt_stream->current_pkt_nb++];
621 pkt_stream->current_pkt_nb++;
0d1b7f3a
MK
622 }
623 return NULL;
624}
625
605091c5
MK
626static void pkt_stream_delete(struct pkt_stream *pkt_stream)
627{
628 free(pkt_stream->pkts);
629 free(pkt_stream);
630}
631
632static void pkt_stream_restore_default(struct test_spec *test)
633{
8367eb95
TV
634 struct pkt_stream *tx_pkt_stream = test->ifobj_tx->xsk->pkt_stream;
635 struct pkt_stream *rx_pkt_stream = test->ifobj_rx->xsk->pkt_stream;
27e934be 636
1adef064 637 if (tx_pkt_stream != test->tx_pkt_stream_default) {
8367eb95
TV
638 pkt_stream_delete(test->ifobj_tx->xsk->pkt_stream);
639 test->ifobj_tx->xsk->pkt_stream = test->tx_pkt_stream_default;
6ce67b51 640 }
27e934be 641
1adef064 642 if (rx_pkt_stream != test->rx_pkt_stream_default) {
8367eb95
TV
643 pkt_stream_delete(test->ifobj_rx->xsk->pkt_stream);
644 test->ifobj_rx->xsk->pkt_stream = test->rx_pkt_stream_default;
1adef064 645 }
605091c5
MK
646}
647
0d1b7f3a 648static struct pkt_stream *__pkt_stream_alloc(u32 nb_pkts)
29f128b3
MK
649{
650 struct pkt_stream *pkt_stream;
29f128b3 651
a4ba98dd 652 pkt_stream = calloc(1, sizeof(*pkt_stream));
29f128b3 653 if (!pkt_stream)
0d1b7f3a 654 return NULL;
29f128b3
MK
655
656 pkt_stream->pkts = calloc(nb_pkts, sizeof(*pkt_stream->pkts));
0d1b7f3a
MK
657 if (!pkt_stream->pkts) {
658 free(pkt_stream);
659 return NULL;
660 }
661
662 pkt_stream->nb_pkts = nb_pkts;
663 return pkt_stream;
664}
665
69760449 666static bool pkt_continues(u32 options)
17f1034d 667{
69760449 668 return options & XDP_PKT_CONTD;
17f1034d
MK
669}
670
86e41755
MK
671static u32 ceil_u32(u32 a, u32 b)
672{
673 return (a + b - 1) / b;
674}
675
69760449 676static u32 pkt_nb_frags(u32 frame_size, struct pkt_stream *pkt_stream, struct pkt *pkt)
86e41755 677{
69760449
MK
678 u32 nb_frags = 1, next_frag;
679
680 if (!pkt)
86e41755 681 return 1;
69760449
MK
682
683 if (!pkt_stream->verbatim) {
684 if (!pkt->valid || !pkt->len)
685 return 1;
686 return ceil_u32(pkt->len, frame_size);
687 }
688
689 /* Search for the end of the packet in verbatim mode */
690 if (!pkt_continues(pkt->options))
691 return nb_frags;
692
693 next_frag = pkt_stream->current_pkt_nb;
694 pkt++;
695 while (next_frag++ < pkt_stream->nb_pkts) {
696 nb_frags++;
697 if (!pkt_continues(pkt->options) || !pkt->valid)
698 break;
699 pkt++;
700 }
701 return nb_frags;
86e41755
MK
702}
703
2e1d6a04
TV
704static bool set_pkt_valid(int offset, u32 len)
705{
706 return len <= MAX_ETH_JUMBO_SIZE;
707}
708
8913e653 709static void pkt_set(struct pkt_stream *pkt_stream, struct pkt *pkt, int offset, u32 len)
27e934be 710{
d9f6d970 711 pkt->offset = offset;
27e934be 712 pkt->len = len;
2e1d6a04
TV
713 pkt->valid = set_pkt_valid(offset, len);
714}
715
716static void pkt_stream_pkt_set(struct pkt_stream *pkt_stream, struct pkt *pkt, int offset, u32 len)
717{
718 bool prev_pkt_valid = pkt->valid;
719
720 pkt_set(pkt_stream, pkt, offset, len);
721 pkt_stream->nb_valid_entries += pkt->valid - prev_pkt_valid;
27e934be
MK
722}
723
7cd6df4f
MK
724static u32 pkt_get_buffer_len(struct xsk_umem_info *umem, u32 len)
725{
726 return ceil_u32(len, umem->frame_size) * umem->frame_size;
727}
728
6d198a89 729static struct pkt_stream *__pkt_stream_generate(u32 nb_pkts, u32 pkt_len, u32 nb_start, u32 nb_off)
0d1b7f3a
MK
730{
731 struct pkt_stream *pkt_stream;
732 u32 i;
733
734 pkt_stream = __pkt_stream_alloc(nb_pkts);
735 if (!pkt_stream)
29f128b3
MK
736 exit_with_error(ENOMEM);
737
7cd6df4f
MK
738 pkt_stream->nb_pkts = nb_pkts;
739 pkt_stream->max_pkt_len = pkt_len;
29f128b3 740 for (i = 0; i < nb_pkts; i++) {
2f6eae0d
MK
741 struct pkt *pkt = &pkt_stream->pkts[i];
742
2e1d6a04 743 pkt_stream_pkt_set(pkt_stream, pkt, 0, pkt_len);
6d198a89 744 pkt->pkt_nb = nb_start + i * nb_off;
29f128b3
MK
745 }
746
747 return pkt_stream;
748}
749
6d198a89
TV
750static struct pkt_stream *pkt_stream_generate(u32 nb_pkts, u32 pkt_len)
751{
752 return __pkt_stream_generate(nb_pkts, pkt_len, 0, 1);
753}
754
46e43786 755static struct pkt_stream *pkt_stream_clone(struct pkt_stream *pkt_stream)
a4ba98dd 756{
46e43786 757 return pkt_stream_generate(pkt_stream->nb_pkts, pkt_stream->pkts[0].len);
a4ba98dd
MK
758}
759
605091c5
MK
760static void pkt_stream_replace(struct test_spec *test, u32 nb_pkts, u32 pkt_len)
761{
762 struct pkt_stream *pkt_stream;
763
46e43786 764 pkt_stream = pkt_stream_generate(nb_pkts, pkt_len);
8367eb95 765 test->ifobj_tx->xsk->pkt_stream = pkt_stream;
46e43786 766 pkt_stream = pkt_stream_generate(nb_pkts, pkt_len);
8367eb95 767 test->ifobj_rx->xsk->pkt_stream = pkt_stream;
605091c5
MK
768}
769
1adef064
MF
770static void __pkt_stream_replace_half(struct ifobject *ifobj, u32 pkt_len,
771 int offset)
a4ba98dd 772{
a4ba98dd
MK
773 struct pkt_stream *pkt_stream;
774 u32 i;
775
46e43786 776 pkt_stream = pkt_stream_clone(ifobj->xsk->pkt_stream);
8367eb95 777 for (i = 1; i < ifobj->xsk->pkt_stream->nb_pkts; i += 2)
2e1d6a04 778 pkt_stream_pkt_set(pkt_stream, &pkt_stream->pkts[i], offset, pkt_len);
a4ba98dd 779
8367eb95 780 ifobj->xsk->pkt_stream = pkt_stream;
1adef064
MF
781}
782
783static void pkt_stream_replace_half(struct test_spec *test, u32 pkt_len, int offset)
784{
785 __pkt_stream_replace_half(test->ifobj_tx, pkt_len, offset);
786 __pkt_stream_replace_half(test->ifobj_rx, pkt_len, offset);
a4ba98dd
MK
787}
788
27e934be
MK
789static void pkt_stream_receive_half(struct test_spec *test)
790{
8367eb95 791 struct pkt_stream *pkt_stream = test->ifobj_tx->xsk->pkt_stream;
27e934be
MK
792 u32 i;
793
46e43786 794 test->ifobj_rx->xsk->pkt_stream = pkt_stream_generate(pkt_stream->nb_pkts,
8367eb95
TV
795 pkt_stream->pkts[0].len);
796 pkt_stream = test->ifobj_rx->xsk->pkt_stream;
27e934be
MK
797 for (i = 1; i < pkt_stream->nb_pkts; i += 2)
798 pkt_stream->pkts[i].valid = false;
8913e653
TV
799
800 pkt_stream->nb_valid_entries /= 2;
27e934be
MK
801}
802
6d198a89
TV
803static void pkt_stream_even_odd_sequence(struct test_spec *test)
804{
805 struct pkt_stream *pkt_stream;
806 u32 i;
807
808 for (i = 0; i < test->nb_sockets; i++) {
809 pkt_stream = test->ifobj_tx->xsk_arr[i].pkt_stream;
810 pkt_stream = __pkt_stream_generate(pkt_stream->nb_pkts / 2,
811 pkt_stream->pkts[0].len, i, 2);
812 test->ifobj_tx->xsk_arr[i].pkt_stream = pkt_stream;
813
814 pkt_stream = test->ifobj_rx->xsk_arr[i].pkt_stream;
815 pkt_stream = __pkt_stream_generate(pkt_stream->nb_pkts / 2,
816 pkt_stream->pkts[0].len, i, 2);
817 test->ifobj_rx->xsk_arr[i].pkt_stream = pkt_stream;
818 }
819}
820
d9f6d970
MK
821static u64 pkt_get_addr(struct pkt *pkt, struct xsk_umem_info *umem)
822{
823 if (!pkt->valid)
824 return pkt->offset;
825 return pkt->offset + umem_alloc_buffer(umem);
826}
827
17f1034d
MK
828static void pkt_stream_cancel(struct pkt_stream *pkt_stream)
829{
830 pkt_stream->current_pkt_nb--;
831}
832
985fd214
TV
833static void pkt_generate(struct xsk_socket_info *xsk, struct xsk_umem_info *umem, u64 addr, u32 len,
834 u32 pkt_nb, u32 bytes_written)
29f128b3 835{
985fd214 836 void *data = xsk_umem__get_data(umem->buffer, addr);
29f128b3 837
2f6eae0d 838 if (len < MIN_PKT_SIZE)
d9f6d970 839 return;
29f128b3 840
2f6eae0d 841 if (!bytes_written) {
985fd214 842 gen_eth_hdr(xsk, data);
2f6eae0d
MK
843
844 len -= PKT_HDR_SIZE;
845 data += PKT_HDR_SIZE;
846 } else {
847 bytes_written -= PKT_HDR_SIZE;
848 }
960b6e01 849
2f6eae0d 850 write_payload(data, pkt_nb, bytes_written, len);
960b6e01
MK
851}
852
69760449
MK
853static struct pkt_stream *__pkt_stream_generate_custom(struct ifobject *ifobj, struct pkt *frames,
854 u32 nb_frames, bool verbatim)
0d1b7f3a 855{
69760449 856 u32 i, len = 0, pkt_nb = 0, payload = 0;
0d1b7f3a 857 struct pkt_stream *pkt_stream;
0d1b7f3a 858
69760449 859 pkt_stream = __pkt_stream_alloc(nb_frames);
0d1b7f3a
MK
860 if (!pkt_stream)
861 exit_with_error(ENOMEM);
862
69760449
MK
863 for (i = 0; i < nb_frames; i++) {
864 struct pkt *pkt = &pkt_stream->pkts[pkt_nb];
865 struct pkt *frame = &frames[i];
7cd6df4f 866
69760449
MK
867 pkt->offset = frame->offset;
868 if (verbatim) {
869 *pkt = *frame;
870 pkt->pkt_nb = payload;
871 if (!frame->valid || !pkt_continues(frame->options))
872 payload++;
873 } else {
874 if (frame->valid)
875 len += frame->len;
876 if (frame->valid && pkt_continues(frame->options))
877 continue;
878
879 pkt->pkt_nb = pkt_nb;
880 pkt->len = len;
881 pkt->valid = frame->valid;
882 pkt->options = 0;
883
884 len = 0;
885 }
886
2d2712ca
MK
887 print_verbose("offset: %d len: %u valid: %u options: %u pkt_nb: %u\n",
888 pkt->offset, pkt->len, pkt->valid, pkt->options, pkt->pkt_nb);
889
69760449 890 if (pkt->valid && pkt->len > pkt_stream->max_pkt_len)
7cd6df4f 891 pkt_stream->max_pkt_len = pkt->len;
8913e653
TV
892
893 if (pkt->valid)
894 pkt_stream->nb_valid_entries++;
895
69760449 896 pkt_nb++;
0d1b7f3a 897 }
1adef064 898
69760449
MK
899 pkt_stream->nb_pkts = pkt_nb;
900 pkt_stream->verbatim = verbatim;
901 return pkt_stream;
1adef064
MF
902}
903
904static void pkt_stream_generate_custom(struct test_spec *test, struct pkt *pkts, u32 nb_pkts)
905{
69760449
MK
906 struct pkt_stream *pkt_stream;
907
908 pkt_stream = __pkt_stream_generate_custom(test->ifobj_tx, pkts, nb_pkts, true);
8367eb95 909 test->ifobj_tx->xsk->pkt_stream = pkt_stream;
69760449
MK
910
911 pkt_stream = __pkt_stream_generate_custom(test->ifobj_rx, pkts, nb_pkts, false);
8367eb95 912 test->ifobj_rx->xsk->pkt_stream = pkt_stream;
0d1b7f3a
MK
913}
914
feb973a9
MK
915static void pkt_print_data(u32 *data, u32 cnt)
916{
917 u32 i;
918
919 for (i = 0; i < cnt; i++) {
920 u32 seqnum, pkt_nb;
921
922 seqnum = ntohl(*data) & 0xffff;
923 pkt_nb = ntohl(*data) >> 16;
7c3fcf08 924 ksft_print_msg("%u:%u ", pkt_nb, seqnum);
feb973a9
MK
925 data++;
926 }
927}
928
2f6eae0d 929static void pkt_dump(void *pkt, u32 len, bool eth_header)
0d41f59f 930{
df82d2e8 931 struct ethhdr *ethhdr = pkt;
2f6eae0d
MK
932 u32 i, *data;
933
934 if (eth_header) {
935 /*extract L2 frame */
7c3fcf08 936 ksft_print_msg("DEBUG>> L2: dst mac: ");
2f6eae0d 937 for (i = 0; i < ETH_ALEN; i++)
7c3fcf08 938 ksft_print_msg("%02X", ethhdr->h_dest[i]);
0d41f59f 939
7c3fcf08 940 ksft_print_msg("\nDEBUG>> L2: src mac: ");
2f6eae0d 941 for (i = 0; i < ETH_ALEN; i++)
7c3fcf08 942 ksft_print_msg("%02X", ethhdr->h_source[i]);
0d41f59f 943
2f6eae0d
MK
944 data = pkt + PKT_HDR_SIZE;
945 } else {
946 data = pkt;
947 }
0d41f59f 948
0d41f59f 949 /*extract L5 frame */
7c3fcf08 950 ksft_print_msg("\nDEBUG>> L5: seqnum: ");
2f6eae0d 951 pkt_print_data(data, PKT_DUMP_NB_TO_PRINT);
7c3fcf08 952 ksft_print_msg("....");
feb973a9 953 if (len > PKT_DUMP_NB_TO_PRINT * sizeof(u32)) {
7c3fcf08 954 ksft_print_msg("\n.... ");
2f6eae0d 955 pkt_print_data(data + len / sizeof(u32) - PKT_DUMP_NB_TO_PRINT,
feb973a9
MK
956 PKT_DUMP_NB_TO_PRINT);
957 }
7c3fcf08 958 ksft_print_msg("\n---------------------------------------\n");
0d41f59f
MK
959}
960
d9f6d970 961static bool is_offset_correct(struct xsk_umem_info *umem, struct pkt *pkt, u64 addr)
e34087fc
MK
962{
963 u32 headroom = umem->unaligned_mode ? 0 : umem->frame_headroom;
d9f6d970
MK
964 u32 offset = addr % umem->frame_size, expected_offset;
965 int pkt_offset = pkt->valid ? pkt->offset : 0;
e34087fc 966
d9f6d970
MK
967 if (!umem->unaligned_mode)
968 pkt_offset = 0;
e34087fc 969
d9f6d970 970 expected_offset = (pkt_offset + headroom + XDP_PACKET_HEADROOM) % umem->frame_size;
e34087fc
MK
971
972 if (offset == expected_offset)
973 return true;
974
895b62ee 975 ksft_print_msg("[%s] expected [%u], got [%u]\n", __func__, expected_offset, offset);
e34087fc
MK
976 return false;
977}
978
9a321fd3
TV
979static bool is_metadata_correct(struct pkt *pkt, void *buffer, u64 addr)
980{
981 void *data = xsk_umem__get_data(buffer, addr);
982 struct xdp_info *meta = data - sizeof(struct xdp_info);
983
feb973a9 984 if (meta->count != pkt->pkt_nb) {
fe69a1b1
AR
985 ksft_print_msg("[%s] expected meta_count [%d], got meta_count [%llu]\n",
986 __func__, pkt->pkt_nb,
987 (unsigned long long)meta->count);
9a321fd3
TV
988 return false;
989 }
990
991 return true;
992}
993
17f1034d
MK
994static bool is_frag_valid(struct xsk_umem_info *umem, u64 addr, u32 len, u32 expected_pkt_nb,
995 u32 bytes_processed)
0d41f59f 996{
17f1034d
MK
997 u32 seqnum, pkt_nb, *pkt_data, words_to_end, expected_seqnum;
998 void *data = xsk_umem__get_data(umem->buffer, addr);
0d41f59f 999
17f1034d
MK
1000 addr -= umem->base_addr;
1001
1002 if (addr >= umem->num_frames * umem->frame_size ||
1003 addr + len > umem->num_frames * umem->frame_size) {
fe69a1b1
AR
1004 ksft_print_msg("Frag invalid addr: %llx len: %u\n",
1005 (unsigned long long)addr, len);
17f1034d
MK
1006 return false;
1007 }
1008 if (!umem->unaligned_mode && addr % umem->frame_size + len > umem->frame_size) {
fe69a1b1
AR
1009 ksft_print_msg("Frag crosses frame boundary addr: %llx len: %u\n",
1010 (unsigned long long)addr, len);
17f1034d 1011 return false;
29f128b3
MK
1012 }
1013
17f1034d
MK
1014 pkt_data = data;
1015 if (!bytes_processed) {
1016 pkt_data += PKT_HDR_SIZE / sizeof(*pkt_data);
1017 len -= PKT_HDR_SIZE;
1018 } else {
1019 bytes_processed -= PKT_HDR_SIZE;
0d1b7f3a
MK
1020 }
1021
17f1034d
MK
1022 expected_seqnum = bytes_processed / sizeof(*pkt_data);
1023 seqnum = ntohl(*pkt_data) & 0xffff;
1024 pkt_nb = ntohl(*pkt_data) >> 16;
1025
1026 if (expected_pkt_nb != pkt_nb) {
1027 ksft_print_msg("[%s] expected pkt_nb [%u], got pkt_nb [%u]\n",
1028 __func__, expected_pkt_nb, pkt_nb);
1029 goto error;
1030 }
1031 if (expected_seqnum != seqnum) {
1032 ksft_print_msg("[%s] expected seqnum at start [%u], got seqnum [%u]\n",
1033 __func__, expected_seqnum, seqnum);
7a8a6762 1034 goto error;
0d1b7f3a
MK
1035 }
1036
17f1034d
MK
1037 words_to_end = len / sizeof(*pkt_data) - 1;
1038 pkt_data += words_to_end;
1039 seqnum = ntohl(*pkt_data) & 0xffff;
1040 expected_seqnum += words_to_end;
1041 if (expected_seqnum != seqnum) {
1042 ksft_print_msg("[%s] expected seqnum at end [%u], got seqnum [%u]\n",
1043 __func__, expected_seqnum, seqnum);
7a8a6762 1044 goto error;
0d41f59f 1045 }
29f128b3
MK
1046
1047 return true;
7a8a6762
MK
1048
1049error:
17f1034d 1050 pkt_dump(data, len, !bytes_processed);
7a8a6762 1051 return false;
0d41f59f
MK
1052}
1053
17f1034d
MK
1054static bool is_pkt_valid(struct pkt *pkt, void *buffer, u64 addr, u32 len)
1055{
17f1034d
MK
1056 if (pkt->len != len) {
1057 ksft_print_msg("[%s] expected packet length [%d], got length [%d]\n",
1058 __func__, pkt->len, len);
1059 pkt_dump(xsk_umem__get_data(buffer, addr), len, true);
1060 return false;
1061 }
1062
1063 return true;
1064}
1065
5fc494d5 1066static int kick_tx(struct xsk_socket_info *xsk)
facb7cb2
WJ
1067{
1068 int ret;
1069
1070 ret = sendto(xsk_socket__fd(xsk->xsk), NULL, 0, MSG_DONTWAIT, NULL, 0);
f90062b5 1071 if (ret >= 0)
5fc494d5 1072 return TEST_PASS;
f90062b5
MK
1073 if (errno == ENOBUFS || errno == EAGAIN || errno == EBUSY || errno == ENETDOWN) {
1074 usleep(100);
5fc494d5 1075 return TEST_PASS;
f90062b5 1076 }
5fc494d5 1077 return TEST_FAILURE;
facb7cb2
WJ
1078}
1079
5fc494d5 1080static int kick_rx(struct xsk_socket_info *xsk)
f90062b5
MK
1081{
1082 int ret;
1083
1084 ret = recvfrom(xsk_socket__fd(xsk->xsk), NULL, 0, MSG_DONTWAIT, NULL, NULL);
1085 if (ret < 0)
5fc494d5
MK
1086 return TEST_FAILURE;
1087
1088 return TEST_PASS;
f90062b5
MK
1089}
1090
895b62ee 1091static int complete_pkts(struct xsk_socket_info *xsk, int batch_size)
facb7cb2
WJ
1092{
1093 unsigned int rcvd;
1094 u32 idx;
5fc494d5 1095 int ret;
facb7cb2 1096
5fc494d5
MK
1097 if (xsk_ring_prod__needs_wakeup(&xsk->tx)) {
1098 ret = kick_tx(xsk);
1099 if (ret)
1100 return TEST_FAILURE;
1101 }
facb7cb2
WJ
1102
1103 rcvd = xsk_ring_cons__peek(&xsk->umem->cq, batch_size, &idx);
1104 if (rcvd) {
0d1b7f3a
MK
1105 if (rcvd > xsk->outstanding_tx) {
1106 u64 addr = *xsk_ring_cons__comp_addr(&xsk->umem->cq, idx + rcvd - 1);
1107
895b62ee 1108 ksft_print_msg("[%s] Too many packets completed\n", __func__);
fe69a1b1
AR
1109 ksft_print_msg("Last completion address: %llx\n",
1110 (unsigned long long)addr);
895b62ee 1111 return TEST_FAILURE;
0d1b7f3a
MK
1112 }
1113
facb7cb2
WJ
1114 xsk_ring_cons__release(&xsk->umem->cq, rcvd);
1115 xsk->outstanding_tx -= rcvd;
facb7cb2 1116 }
895b62ee
MK
1117
1118 return TEST_PASS;
facb7cb2
WJ
1119}
1120
8913e653 1121static int __receive_pkts(struct test_spec *test, struct xsk_socket_info *xsk)
facb7cb2 1122{
8913e653 1123 u32 frags_processed = 0, nb_frags = 0, pkt_len = 0;
17f1034d 1124 u32 idx_rx = 0, idx_fq = 0, rcvd, pkts_sent = 0;
8913e653 1125 struct pkt_stream *pkt_stream = xsk->pkt_stream;
3143d10b 1126 struct ifobject *ifobj = test->ifobj_rx;
e34087fc 1127 struct xsk_umem_info *umem = xsk->umem;
8913e653 1128 struct pollfd fds = { };
27e934be 1129 struct pkt *pkt;
925a0157 1130 u64 first_addr = 0;
facb7cb2
WJ
1131 int ret;
1132
8913e653
TV
1133 fds.fd = xsk_socket__fd(xsk->xsk);
1134 fds.events = POLLIN;
17f1034d 1135
8913e653
TV
1136 ret = kick_rx(xsk);
1137 if (ret)
1138 return TEST_FAILURE;
db1bd7a9 1139
8913e653
TV
1140 if (ifobj->use_poll) {
1141 ret = poll(&fds, 1, POLL_TMOUT);
1142 if (ret < 0)
5fc494d5
MK
1143 return TEST_FAILURE;
1144
8913e653
TV
1145 if (!ret) {
1146 if (!is_umem_valid(test->ifobj_tx))
1147 return TEST_PASS;
3143d10b 1148
8913e653
TV
1149 ksft_print_msg("ERROR: [%s] Poll timed out\n", __func__);
1150 return TEST_CONTINUE;
facb7cb2 1151 }
facb7cb2 1152
8913e653
TV
1153 if (!(fds.revents & POLLIN))
1154 return TEST_CONTINUE;
1155 }
3143d10b 1156
c3bd0150 1157 rcvd = xsk_ring_cons__peek(&xsk->rx, xsk->batch_size, &idx_rx);
8913e653
TV
1158 if (!rcvd)
1159 return TEST_CONTINUE;
1160
1161 if (ifobj->use_fill_ring) {
1162 ret = xsk_ring_prod__reserve(&umem->fq, rcvd, &idx_fq);
1163 while (ret != rcvd) {
1164 if (xsk_ring_prod__needs_wakeup(&umem->fq)) {
1165 ret = poll(&fds, 1, POLL_TMOUT);
1166 if (ret < 0)
1167 return TEST_FAILURE;
29f128b3 1168 }
8913e653 1169 ret = xsk_ring_prod__reserve(&umem->fq, rcvd, &idx_fq);
facb7cb2 1170 }
8913e653 1171 }
facb7cb2 1172
8913e653
TV
1173 while (frags_processed < rcvd) {
1174 const struct xdp_desc *desc = xsk_ring_cons__rx_desc(&xsk->rx, idx_rx++);
1175 u64 addr = desc->addr, orig;
829725ec 1176
8913e653
TV
1177 orig = xsk_umem__extract_addr(addr);
1178 addr = xsk_umem__add_offset_to_addr(addr);
e34087fc 1179
8913e653
TV
1180 if (!nb_frags) {
1181 pkt = pkt_stream_get_next_rx_pkt(pkt_stream, &pkts_sent);
69760449
MK
1182 if (!pkt) {
1183 ksft_print_msg("[%s] received too many packets addr: %lx len %u\n",
1184 __func__, addr, desc->len);
1185 return TEST_FAILURE;
1186 }
8913e653 1187 }
69760449 1188
8913e653
TV
1189 print_verbose("Rx: addr: %lx len: %u options: %u pkt_nb: %u valid: %u\n",
1190 addr, desc->len, desc->options, pkt->pkt_nb, pkt->valid);
2d2712ca 1191
8913e653
TV
1192 if (!is_frag_valid(umem, addr, desc->len, pkt->pkt_nb, pkt_len) ||
1193 !is_offset_correct(umem, pkt, addr) || (ifobj->use_metadata &&
1194 !is_metadata_correct(pkt, umem->buffer, addr)))
1195 return TEST_FAILURE;
facb7cb2 1196
8913e653
TV
1197 if (!nb_frags++)
1198 first_addr = addr;
1199 frags_processed++;
1200 pkt_len += desc->len;
1201 if (ifobj->use_fill_ring)
1202 *xsk_ring_prod__fill_addr(&umem->fq, idx_fq++) = orig;
17f1034d 1203
8913e653
TV
1204 if (pkt_continues(desc->options))
1205 continue;
17f1034d 1206
8913e653
TV
1207 /* The complete packet has been received */
1208 if (!is_pkt_valid(pkt, umem->buffer, first_addr, pkt_len) ||
1209 !is_offset_correct(umem, pkt, addr))
1210 return TEST_FAILURE;
17f1034d 1211
8913e653
TV
1212 pkt_stream->nb_rx_pkts++;
1213 nb_frags = 0;
1214 pkt_len = 0;
1215 }
17f1034d 1216
8913e653
TV
1217 if (nb_frags) {
1218 /* In the middle of a packet. Start over from beginning of packet. */
1219 idx_rx -= nb_frags;
1220 xsk_ring_cons__cancel(&xsk->rx, nb_frags);
1221 if (ifobj->use_fill_ring) {
1222 idx_fq -= nb_frags;
1223 xsk_ring_prod__cancel(&umem->fq, nb_frags);
29f128b3 1224 }
8913e653
TV
1225 frags_processed -= nb_frags;
1226 }
facb7cb2 1227
8913e653
TV
1228 if (ifobj->use_fill_ring)
1229 xsk_ring_prod__submit(&umem->fq, frags_processed);
1230 if (ifobj->release_rx)
1231 xsk_ring_cons__release(&xsk->rx, frags_processed);
1232
1233 pthread_mutex_lock(&pacing_mutex);
1234 pkts_in_flight -= pkts_sent;
1235 pthread_mutex_unlock(&pacing_mutex);
1236 pkts_sent = 0;
1237
1238return TEST_CONTINUE;
1239}
1240
1241bool all_packets_received(struct test_spec *test, struct xsk_socket_info *xsk, u32 sock_num,
1242 unsigned long *bitmap)
1243{
1244 struct pkt_stream *pkt_stream = xsk->pkt_stream;
1245
1246 if (!pkt_stream) {
1247 __set_bit(sock_num, bitmap);
1248 return false;
1249 }
1250
1251 if (pkt_stream->nb_rx_pkts == pkt_stream->nb_valid_entries) {
1252 __set_bit(sock_num, bitmap);
1253 if (bitmap_full(bitmap, test->nb_sockets))
1254 return true;
1255 }
1256
1257 return false;
1258}
1259
1260static int receive_pkts(struct test_spec *test)
1261{
1262 struct timeval tv_end, tv_now, tv_timeout = {THREAD_TMOUT, 0};
1263 DECLARE_BITMAP(bitmap, test->nb_sockets);
1264 struct xsk_socket_info *xsk;
1265 u32 sock_num = 0;
1266 int res, ret;
1bf36496 1267
8913e653
TV
1268 ret = gettimeofday(&tv_now, NULL);
1269 if (ret)
1270 exit_with_error(errno);
1271
1272 timeradd(&tv_now, &tv_timeout, &tv_end);
1273
1274 while (1) {
1275 xsk = &test->ifobj_rx->xsk_arr[sock_num];
1276
1277 if ((all_packets_received(test, xsk, sock_num, bitmap)))
1278 break;
1279
1280 res = __receive_pkts(test, xsk);
1281 if (!(res == TEST_PASS || res == TEST_CONTINUE))
1282 return res;
1283
1284 ret = gettimeofday(&tv_now, NULL);
1285 if (ret)
1286 exit_with_error(errno);
1287
1288 if (timercmp(&tv_now, &tv_end, >)) {
1289 ksft_print_msg("ERROR: [%s] Receive loop timed out\n", __func__);
1290 return TEST_FAILURE;
1291 }
1292 sock_num = (sock_num + 1) % test->nb_sockets;
facb7cb2 1293 }
895b62ee
MK
1294
1295 return TEST_PASS;
facb7cb2
WJ
1296}
1297
fd0815ae 1298static int __send_pkts(struct ifobject *ifobject, struct xsk_socket_info *xsk, bool timeout)
facb7cb2 1299{
17f1034d 1300 u32 i, idx = 0, valid_pkts = 0, valid_frags = 0, buffer_len;
fd0815ae 1301 struct pkt_stream *pkt_stream = xsk->pkt_stream;
7cd6df4f 1302 struct xsk_umem_info *umem = ifobject->umem;
3143d10b 1303 bool use_poll = ifobject->use_poll;
fd0815ae 1304 struct pollfd fds = { };
e8f50c4f 1305 int ret;
3143d10b 1306
17f1034d 1307 buffer_len = pkt_get_buffer_len(umem, pkt_stream->max_pkt_len);
7cd6df4f 1308 /* pkts_in_flight might be negative if many invalid packets are sent */
c3bd0150
TV
1309 if (pkts_in_flight >= (int)((umem_size(umem) - xsk->batch_size * buffer_len) /
1310 buffer_len)) {
5fc494d5
MK
1311 ret = kick_tx(xsk);
1312 if (ret)
1313 return TEST_FAILURE;
7cd6df4f
MK
1314 return TEST_CONTINUE;
1315 }
1316
fd0815ae
TV
1317 fds.fd = xsk_socket__fd(xsk->xsk);
1318 fds.events = POLLOUT;
1319
c3bd0150 1320 while (xsk_ring_prod__reserve(&xsk->tx, xsk->batch_size, &idx) < xsk->batch_size) {
3143d10b 1321 if (use_poll) {
fd0815ae 1322 ret = poll(&fds, 1, POLL_TMOUT);
3143d10b
SKR
1323 if (timeout) {
1324 if (ret < 0) {
1325 ksft_print_msg("ERROR: [%s] Poll error %d\n",
085dcccf 1326 __func__, errno);
3143d10b
SKR
1327 return TEST_FAILURE;
1328 }
1329 if (ret == 0)
1330 return TEST_PASS;
1331 break;
1332 }
1333 if (ret <= 0) {
1334 ksft_print_msg("ERROR: [%s] Poll error %d\n",
085dcccf 1335 __func__, errno);
3143d10b
SKR
1336 return TEST_FAILURE;
1337 }
1338 }
facb7cb2 1339
c3bd0150 1340 complete_pkts(xsk, xsk->batch_size);
3143d10b 1341 }
facb7cb2 1342
c3bd0150 1343 for (i = 0; i < xsk->batch_size; i++) {
17f1034d 1344 struct pkt *pkt = pkt_stream_get_next_tx_pkt(pkt_stream);
69760449 1345 u32 nb_frags_left, nb_frags, bytes_written = 0;
facb7cb2 1346
29f128b3
MK
1347 if (!pkt)
1348 break;
facb7cb2 1349
69760449 1350 nb_frags = pkt_nb_frags(umem->frame_size, pkt_stream, pkt);
c3bd0150 1351 if (nb_frags > xsk->batch_size - i) {
17f1034d 1352 pkt_stream_cancel(pkt_stream);
c3bd0150 1353 xsk_ring_prod__cancel(&xsk->tx, xsk->batch_size - i);
17f1034d
MK
1354 break;
1355 }
69760449 1356 nb_frags_left = nb_frags;
17f1034d 1357
69760449 1358 while (nb_frags_left--) {
17f1034d
MK
1359 struct xdp_desc *tx_desc = xsk_ring_prod__tx_desc(&xsk->tx, idx + i);
1360
1361 tx_desc->addr = pkt_get_addr(pkt, ifobject->umem);
69760449
MK
1362 if (pkt_stream->verbatim) {
1363 tx_desc->len = pkt->len;
1364 tx_desc->options = pkt->options;
1365 } else if (nb_frags_left) {
17f1034d
MK
1366 tx_desc->len = umem->frame_size;
1367 tx_desc->options = XDP_PKT_CONTD;
17f1034d
MK
1368 } else {
1369 tx_desc->len = pkt->len - bytes_written;
1370 tx_desc->options = 0;
1371 }
1372 if (pkt->valid)
985fd214 1373 pkt_generate(xsk, umem, tx_desc->addr, tx_desc->len, pkt->pkt_nb,
17f1034d
MK
1374 bytes_written);
1375 bytes_written += tx_desc->len;
69760449 1376
2d2712ca
MK
1377 print_verbose("Tx addr: %llx len: %u options: %u pkt_nb: %u\n",
1378 tx_desc->addr, tx_desc->len, tx_desc->options, pkt->pkt_nb);
1379
69760449
MK
1380 if (nb_frags_left) {
1381 i++;
1382 if (pkt_stream->verbatim)
1383 pkt = pkt_stream_get_next_tx_pkt(pkt_stream);
1384 }
1385 }
1386
1387 if (pkt && pkt->valid) {
1388 valid_pkts++;
1389 valid_frags += nb_frags;
2f6eae0d 1390 }
b267e5a4 1391 }
facb7cb2 1392
1bf36496
MK
1393 pthread_mutex_lock(&pacing_mutex);
1394 pkts_in_flight += valid_pkts;
1bf36496
MK
1395 pthread_mutex_unlock(&pacing_mutex);
1396
29f128b3 1397 xsk_ring_prod__submit(&xsk->tx, i);
17f1034d 1398 xsk->outstanding_tx += valid_frags;
facb7cb2 1399
3143d10b 1400 if (use_poll) {
fd0815ae 1401 ret = poll(&fds, 1, POLL_TMOUT);
3143d10b
SKR
1402 if (ret <= 0) {
1403 if (ret == 0 && timeout)
1404 return TEST_PASS;
1405
1406 ksft_print_msg("ERROR: [%s] Poll error %d\n", __func__, ret);
1407 return TEST_FAILURE;
1408 }
1409 }
1410
1411 if (!timeout) {
1412 if (complete_pkts(xsk, i))
1413 return TEST_FAILURE;
1414
1415 usleep(10);
1416 return TEST_PASS;
1417 }
1418
1419 return TEST_CONTINUE;
facb7cb2
WJ
1420}
1421
64370d7c 1422static int wait_for_tx_completion(struct xsk_socket_info *xsk)
facb7cb2 1423{
64370d7c
MK
1424 struct timeval tv_end, tv_now, tv_timeout = {THREAD_TMOUT, 0};
1425 int ret;
1426
1427 ret = gettimeofday(&tv_now, NULL);
1428 if (ret)
1429 exit_with_error(errno);
1430 timeradd(&tv_now, &tv_timeout, &tv_end);
1431
1432 while (xsk->outstanding_tx) {
1433 ret = gettimeofday(&tv_now, NULL);
1434 if (ret)
1435 exit_with_error(errno);
1436 if (timercmp(&tv_now, &tv_end, >)) {
1437 ksft_print_msg("ERROR: [%s] Transmission loop timed out\n", __func__);
1438 return TEST_FAILURE;
1439 }
1440
c3bd0150 1441 complete_pkts(xsk, xsk->batch_size);
64370d7c
MK
1442 }
1443
1444 return TEST_PASS;
facb7cb2
WJ
1445}
1446
fd0815ae
TV
1447bool all_packets_sent(struct test_spec *test, unsigned long *bitmap)
1448{
1449 return bitmap_full(bitmap, test->nb_sockets);
1450}
1451
895b62ee 1452static int send_pkts(struct test_spec *test, struct ifobject *ifobject)
facb7cb2 1453{
3143d10b 1454 bool timeout = !is_umem_valid(test->ifobj_rx);
fd0815ae
TV
1455 DECLARE_BITMAP(bitmap, test->nb_sockets);
1456 u32 i, ret;
facb7cb2 1457
fd0815ae
TV
1458 while (!(all_packets_sent(test, bitmap))) {
1459 for (i = 0; i < test->nb_sockets; i++) {
1460 struct pkt_stream *pkt_stream;
facb7cb2 1461
fd0815ae
TV
1462 pkt_stream = ifobject->xsk_arr[i].pkt_stream;
1463 if (!pkt_stream || pkt_stream->current_pkt_nb >= pkt_stream->nb_pkts) {
1464 __set_bit(i, bitmap);
1465 continue;
1466 }
1467 ret = __send_pkts(ifobject, &ifobject->xsk_arr[i], timeout);
1468 if (ret == TEST_CONTINUE && !test->fail)
1469 continue;
1470
1471 if ((ret || test->fail) && !timeout)
1472 return TEST_FAILURE;
1473
1474 if (ret == TEST_PASS && timeout)
1475 return ret;
1476
1477 ret = wait_for_tx_completion(&ifobject->xsk_arr[i]);
1478 if (ret)
1479 return TEST_FAILURE;
1480 }
facb7cb2
WJ
1481 }
1482
fd0815ae 1483 return TEST_PASS;
facb7cb2
WJ
1484}
1485
76c57663
MK
1486static int get_xsk_stats(struct xsk_socket *xsk, struct xdp_statistics *stats)
1487{
1488 int fd = xsk_socket__fd(xsk), err;
1489 socklen_t optlen, expected_len;
1490
1491 optlen = sizeof(*stats);
1492 err = getsockopt(fd, SOL_XDP, XDP_STATISTICS, stats, &optlen);
1493 if (err) {
1494 ksft_print_msg("[%s] getsockopt(XDP_STATISTICS) error %u %s\n",
1495 __func__, -err, strerror(-err));
1496 return TEST_FAILURE;
1497 }
1498
1499 expected_len = sizeof(struct xdp_statistics);
1500 if (optlen != expected_len) {
1501 ksft_print_msg("[%s] getsockopt optlen error. Expected: %u got: %u\n",
1502 __func__, expected_len, optlen);
1503 return TEST_FAILURE;
1504 }
1505
1506 return TEST_PASS;
1507}
1508
1509static int validate_rx_dropped(struct ifobject *ifobject)
b267e5a4 1510{
b04fdc4c 1511 struct xsk_socket *xsk = ifobject->xsk->xsk;
b267e5a4 1512 struct xdp_statistics stats;
b267e5a4 1513 int err;
b267e5a4 1514
5fc494d5
MK
1515 err = kick_rx(ifobject->xsk);
1516 if (err)
1517 return TEST_FAILURE;
f90062b5 1518
76c57663
MK
1519 err = get_xsk_stats(xsk, &stats);
1520 if (err)
895b62ee 1521 return TEST_FAILURE;
b267e5a4 1522
68e73221
KC
1523 /* The receiver calls getsockopt after receiving the last (valid)
1524 * packet which is not the final packet sent in this test (valid and
1525 * invalid packets are sent in alternating fashion with the final
1526 * packet being invalid). Since the last packet may or may not have
1527 * been dropped already, both outcomes must be allowed.
1528 */
8367eb95
TV
1529 if (stats.rx_dropped == ifobject->xsk->pkt_stream->nb_pkts / 2 ||
1530 stats.rx_dropped == ifobject->xsk->pkt_stream->nb_pkts / 2 - 1)
76c57663 1531 return TEST_PASS;
b267e5a4 1532
27e934be 1533 return TEST_FAILURE;
76c57663
MK
1534}
1535
1536static int validate_rx_full(struct ifobject *ifobject)
1537{
1538 struct xsk_socket *xsk = ifobject->xsk->xsk;
1539 struct xdp_statistics stats;
76c57663
MK
1540 int err;
1541
27e934be 1542 usleep(1000);
5fc494d5
MK
1543 err = kick_rx(ifobject->xsk);
1544 if (err)
1545 return TEST_FAILURE;
76c57663
MK
1546
1547 err = get_xsk_stats(xsk, &stats);
1548 if (err)
1549 return TEST_FAILURE;
1550
27e934be 1551 if (stats.rx_ring_full)
76c57663
MK
1552 return TEST_PASS;
1553
27e934be 1554 return TEST_FAILURE;
76c57663
MK
1555}
1556
1557static int validate_fill_empty(struct ifobject *ifobject)
1558{
1559 struct xsk_socket *xsk = ifobject->xsk->xsk;
1560 struct xdp_statistics stats;
1561 int err;
1562
27e934be 1563 usleep(1000);
5fc494d5
MK
1564 err = kick_rx(ifobject->xsk);
1565 if (err)
1566 return TEST_FAILURE;
76c57663
MK
1567
1568 err = get_xsk_stats(xsk, &stats);
1569 if (err)
1570 return TEST_FAILURE;
1571
27e934be 1572 if (stats.rx_fill_ring_empty_descs)
76c57663 1573 return TEST_PASS;
b04fdc4c 1574
27e934be 1575 return TEST_FAILURE;
b04fdc4c
MK
1576}
1577
76c57663 1578static int validate_tx_invalid_descs(struct ifobject *ifobject)
b04fdc4c
MK
1579{
1580 struct xsk_socket *xsk = ifobject->xsk->xsk;
1581 int fd = xsk_socket__fd(xsk);
1582 struct xdp_statistics stats;
1583 socklen_t optlen;
1584 int err;
1585
1586 optlen = sizeof(stats);
1587 err = getsockopt(fd, SOL_XDP, XDP_STATISTICS, &stats, &optlen);
1588 if (err) {
895b62ee
MK
1589 ksft_print_msg("[%s] getsockopt(XDP_STATISTICS) error %u %s\n",
1590 __func__, -err, strerror(-err));
1591 return TEST_FAILURE;
b04fdc4c
MK
1592 }
1593
8367eb95 1594 if (stats.tx_invalid_descs != ifobject->xsk->pkt_stream->nb_pkts / 2) {
fe69a1b1
AR
1595 ksft_print_msg("[%s] tx_invalid_descs incorrect. Got [%llu] expected [%u]\n",
1596 __func__,
1597 (unsigned long long)stats.tx_invalid_descs,
8367eb95 1598 ifobject->xsk->pkt_stream->nb_pkts);
895b62ee
MK
1599 return TEST_FAILURE;
1600 }
b04fdc4c 1601
895b62ee 1602 return TEST_PASS;
b267e5a4
CL
1603}
1604
a693ff3e
MF
1605static void xsk_configure_socket(struct test_spec *test, struct ifobject *ifobject,
1606 struct xsk_umem_info *umem, bool tx)
1607{
1608 int i, ret;
1609
1610 for (i = 0; i < test->nb_sockets; i++) {
1611 bool shared = (ifobject->shared_umem && tx) ? true : !!i;
1612 u32 ctr = 0;
1613
1614 while (ctr++ < SOCK_RECONF_CTR) {
1615 ret = __xsk_configure_socket(&ifobject->xsk_arr[i], umem,
1616 ifobject, shared);
1617 if (!ret)
1618 break;
1619
1620 /* Retry if it fails as xsk_socket__create() is asynchronous */
1621 if (ctr >= SOCK_RECONF_CTR)
1622 exit_with_error(-ret);
1623 usleep(USLEEP_MAX);
1624 }
1625 if (ifobject->busy_poll)
1626 enable_busy_poll(&ifobject->xsk_arr[i]);
1627 }
1628}
1629
1630static void thread_common_ops_tx(struct test_spec *test, struct ifobject *ifobject)
1631{
1632 xsk_configure_socket(test, ifobject, test->ifobj_rx->umem, true);
1633 ifobject->xsk = &ifobject->xsk_arr[0];
f0a249df 1634 ifobject->xskmap = test->ifobj_rx->xskmap;
a693ff3e 1635 memcpy(ifobject->umem, test->ifobj_rx->umem, sizeof(struct xsk_umem_info));
d9f6d970 1636 ifobject->umem->base_addr = 0;
a693ff3e
MF
1637}
1638
86e41755
MK
1639static void xsk_populate_fill_ring(struct xsk_umem_info *umem, struct pkt_stream *pkt_stream,
1640 bool fill_up)
a693ff3e 1641{
86e41755
MK
1642 u32 rx_frame_size = umem->frame_size - XDP_PACKET_HEADROOM;
1643 u32 idx = 0, filled = 0, buffers_to_fill, nb_pkts;
a693ff3e
MF
1644 int ret;
1645
1646 if (umem->num_frames < XSK_RING_PROD__DEFAULT_NUM_DESCS)
1647 buffers_to_fill = umem->num_frames;
1648 else
e4a195e2 1649 buffers_to_fill = umem->fill_size;
a693ff3e
MF
1650
1651 ret = xsk_ring_prod__reserve(&umem->fq, buffers_to_fill, &idx);
1652 if (ret != buffers_to_fill)
1653 exit_with_error(ENOSPC);
d9f6d970 1654
86e41755 1655 while (filled < buffers_to_fill) {
d9f6d970 1656 struct pkt *pkt = pkt_stream_get_next_rx_pkt(pkt_stream, &nb_pkts);
a693ff3e 1657 u64 addr;
86e41755
MK
1658 u32 i;
1659
69760449 1660 for (i = 0; i < pkt_nb_frags(rx_frame_size, pkt_stream, pkt); i++) {
86e41755
MK
1661 if (!pkt) {
1662 if (!fill_up)
1663 break;
1664 addr = filled * umem->frame_size + umem->base_addr;
1665 } else if (pkt->offset >= 0) {
1666 addr = pkt->offset % umem->frame_size + umem_alloc_buffer(umem);
1667 } else {
1668 addr = pkt->offset + umem_alloc_buffer(umem);
1669 }
a693ff3e 1670
86e41755
MK
1671 *xsk_ring_prod__fill_addr(&umem->fq, idx++) = addr;
1672 if (++filled >= buffers_to_fill)
1673 break;
1674 }
a693ff3e 1675 }
86e41755
MK
1676 xsk_ring_prod__submit(&umem->fq, filled);
1677 xsk_ring_prod__cancel(&umem->fq, buffers_to_fill - filled);
69fc03d2
MK
1678
1679 pkt_stream_reset(pkt_stream);
d9f6d970 1680 umem_reset_alloc(umem);
a693ff3e
MF
1681}
1682
55be575d 1683static void thread_common_ops(struct test_spec *test, struct ifobject *ifobject)
facb7cb2 1684{
3b22523b 1685 u64 umem_sz = ifobject->umem->num_frames * ifobject->umem->frame_size;
a4ba98dd 1686 int mmap_flags = MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE;
6d4c767c 1687 LIBBPF_OPTS(bpf_xdp_query_opts, opts);
3b22523b 1688 void *bufs;
f0a249df 1689 int ret;
6d198a89 1690 u32 i;
facb7cb2 1691
02e93e04
MK
1692 if (ifobject->umem->unaligned_mode)
1693 mmap_flags |= MAP_HUGETLB | MAP_HUGE_2MB;
a4ba98dd 1694
a693ff3e
MF
1695 if (ifobject->shared_umem)
1696 umem_sz *= 2;
1697
02e93e04 1698 bufs = mmap(NULL, umem_sz, PROT_READ | PROT_WRITE, mmap_flags, -1, 0);
3b22523b
MK
1699 if (bufs == MAP_FAILED)
1700 exit_with_error(errno);
9866bcd6 1701
d9f6d970 1702 ret = xsk_configure_umem(ifobject, ifobject->umem, bufs, umem_sz);
3b22523b
MK
1703 if (ret)
1704 exit_with_error(-ret);
ed7b74dc 1705
a693ff3e 1706 xsk_configure_socket(test, ifobject, ifobject->umem, false);
27e1ca25 1707
ed7b74dc 1708 ifobject->xsk = &ifobject->xsk_arr[0];
3b22523b
MK
1709
1710 if (!ifobject->rx_on)
1711 return;
1712
8367eb95 1713 xsk_populate_fill_ring(ifobject->umem, ifobject->xsk->pkt_stream, ifobject->use_fill_ring);
d9f6d970 1714
6d198a89
TV
1715 for (i = 0; i < test->nb_sockets; i++) {
1716 ifobject->xsk = &ifobject->xsk_arr[i];
1717 ret = xsk_update_xskmap(ifobject->xskmap, ifobject->xsk->xsk, i);
1718 if (ret)
1719 exit_with_error(errno);
1720 }
facb7cb2
WJ
1721}
1722
9866bcd6 1723static void *worker_testapp_validate_tx(void *arg)
facb7cb2 1724{
55be575d
MK
1725 struct test_spec *test = (struct test_spec *)arg;
1726 struct ifobject *ifobject = test->ifobj_tx;
895b62ee 1727 int err;
facb7cb2 1728
a693ff3e
MF
1729 if (test->current_step == 1) {
1730 if (!ifobject->shared_umem)
1731 thread_common_ops(test, ifobject);
1732 else
1733 thread_common_ops_tx(test, ifobject);
1734 }
facb7cb2 1735
895b62ee 1736 err = send_pkts(test, ifobject);
facb7cb2 1737
27e934be 1738 if (!err && ifobject->validation_func)
76c57663 1739 err = ifobject->validation_func(ifobject);
27e934be 1740 if (err)
895b62ee 1741 report_failure(test);
b04fdc4c 1742
9866bcd6
MF
1743 pthread_exit(NULL);
1744}
facb7cb2 1745
9866bcd6
MF
1746static void *worker_testapp_validate_rx(void *arg)
1747{
55be575d
MK
1748 struct test_spec *test = (struct test_spec *)arg;
1749 struct ifobject *ifobject = test->ifobj_rx;
f0a249df 1750 int err;
facb7cb2 1751
a693ff3e 1752 if (test->current_step == 1) {
55be575d 1753 thread_common_ops(test, ifobject);
a693ff3e 1754 } else {
f0a249df 1755 xsk_clear_xskmap(ifobject->xskmap);
fc2cb864 1756 err = xsk_update_xskmap(ifobject->xskmap, ifobject->xsk->xsk, 0);
aa61d81f 1757 if (err) {
7c3fcf08
MK
1758 ksft_print_msg("Error: Failed to update xskmap, error %s\n",
1759 strerror(-err));
f0a249df 1760 exit_with_error(-err);
aa61d81f 1761 }
a693ff3e 1762 }
facb7cb2 1763
96539f1c 1764 pthread_barrier_wait(&barr);
facb7cb2 1765
8913e653 1766 err = receive_pkts(test);
895b62ee 1767
27e934be
MK
1768 if (!err && ifobject->validation_func)
1769 err = ifobject->validation_func(ifobject);
7cd6df4f 1770 if (err)
895b62ee 1771 report_failure(test);
6674bf66 1772
facb7cb2
WJ
1773 pthread_exit(NULL);
1774}
1775
2ddade32
MK
1776static u64 ceil_u64(u64 a, u64 b)
1777{
1778 return (a + b - 1) / b;
1779}
1780
a693ff3e
MF
1781static void testapp_clean_xsk_umem(struct ifobject *ifobj)
1782{
1783 u64 umem_sz = ifobj->umem->num_frames * ifobj->umem->frame_size;
1784
1785 if (ifobj->shared_umem)
1786 umem_sz *= 2;
1787
2ddade32 1788 umem_sz = ceil_u64(umem_sz, HUGEPAGE_SIZE) * HUGEPAGE_SIZE;
a693ff3e
MF
1789 xsk_umem__delete(ifobj->umem->umem);
1790 munmap(ifobj->umem->buffer, umem_sz);
1791}
1792
c29fe883
MF
1793static void handler(int signum)
1794{
1795 pthread_exit(NULL);
1796}
1797
d2e54149 1798static bool xdp_prog_changed_rx(struct test_spec *test)
7d8319a7 1799{
d2e54149
MK
1800 struct ifobject *ifobj = test->ifobj_rx;
1801
7d8319a7
MK
1802 return ifobj->xdp_prog != test->xdp_prog_rx || ifobj->mode != test->mode;
1803}
1804
d2e54149
MK
1805static bool xdp_prog_changed_tx(struct test_spec *test)
1806{
1807 struct ifobject *ifobj = test->ifobj_tx;
1808
1809 return ifobj->xdp_prog != test->xdp_prog_tx || ifobj->mode != test->mode;
1810}
1811
7d8319a7
MK
1812static void xsk_reattach_xdp(struct ifobject *ifobj, struct bpf_program *xdp_prog,
1813 struct bpf_map *xskmap, enum test_mode mode)
1814{
1815 int err;
1816
1817 xsk_detach_xdp_program(ifobj->ifindex, mode_to_xdp_flags(ifobj->mode));
1818 err = xsk_attach_xdp_program(xdp_prog, ifobj->ifindex, mode_to_xdp_flags(mode));
1819 if (err) {
7c3fcf08 1820 ksft_print_msg("Error attaching XDP program\n");
7d8319a7
MK
1821 exit_with_error(-err);
1822 }
1823
1824 if (ifobj->mode != mode && (mode == TEST_MODE_DRV || mode == TEST_MODE_ZC))
1825 if (!xsk_is_in_mode(ifobj->ifindex, XDP_FLAGS_DRV_MODE)) {
1826 ksft_print_msg("ERROR: XDP prog not in DRV mode\n");
1827 exit_with_error(EINVAL);
1828 }
1829
1830 ifobj->xdp_prog = xdp_prog;
1831 ifobj->xskmap = xskmap;
1832 ifobj->mode = mode;
1833}
1834
1835static void xsk_attach_xdp_progs(struct test_spec *test, struct ifobject *ifobj_rx,
1836 struct ifobject *ifobj_tx)
1837{
d2e54149 1838 if (xdp_prog_changed_rx(test))
7d8319a7
MK
1839 xsk_reattach_xdp(ifobj_rx, test->xdp_prog_rx, test->xskmap_rx, test->mode);
1840
1841 if (!ifobj_tx || ifobj_tx->shared_umem)
1842 return;
1843
d2e54149 1844 if (xdp_prog_changed_tx(test))
7d8319a7
MK
1845 xsk_reattach_xdp(ifobj_tx, test->xdp_prog_tx, test->xskmap_tx, test->mode);
1846}
1847
7f881984
MK
1848static int __testapp_validate_traffic(struct test_spec *test, struct ifobject *ifobj1,
1849 struct ifobject *ifobj2)
3143d10b 1850{
7f881984 1851 pthread_t t0, t1;
f540d44e
MK
1852 int err;
1853
1854 if (test->mtu > MAX_ETH_PKT_SIZE) {
1855 if (test->mode == TEST_MODE_ZC && (!ifobj1->multi_buff_zc_supp ||
1856 (ifobj2 && !ifobj2->multi_buff_zc_supp))) {
1857 ksft_test_result_skip("Multi buffer for zero-copy not supported.\n");
1858 return TEST_SKIP;
1859 }
1860 if (test->mode != TEST_MODE_ZC && (!ifobj1->multi_buff_supp ||
1861 (ifobj2 && !ifobj2->multi_buff_supp))) {
1862 ksft_test_result_skip("Multi buffer not supported.\n");
1863 return TEST_SKIP;
1864 }
1865 }
1866 err = test_spec_set_mtu(test, test->mtu);
1867 if (err) {
1868 ksft_print_msg("Error, could not set mtu.\n");
1869 exit_with_error(err);
1870 }
3143d10b 1871
69fc03d2 1872 if (ifobj2) {
7f881984
MK
1873 if (pthread_barrier_init(&barr, NULL, 2))
1874 exit_with_error(errno);
8367eb95 1875 pkt_stream_reset(ifobj2->xsk->pkt_stream);
69fc03d2 1876 }
3143d10b
SKR
1877
1878 test->current_step++;
8367eb95 1879 pkt_stream_reset(ifobj1->xsk->pkt_stream);
3143d10b
SKR
1880 pkts_in_flight = 0;
1881
c29fe883 1882 signal(SIGUSR1, handler);
7f881984
MK
1883 /*Spawn RX thread */
1884 pthread_create(&t0, NULL, ifobj1->func_ptr, test);
3143d10b 1885
7f881984 1886 if (ifobj2) {
3143d10b 1887 pthread_barrier_wait(&barr);
7f881984
MK
1888 if (pthread_barrier_destroy(&barr))
1889 exit_with_error(errno);
3143d10b 1890
7f881984
MK
1891 /*Spawn TX thread */
1892 pthread_create(&t1, NULL, ifobj2->func_ptr, test);
3143d10b 1893
7f881984
MK
1894 pthread_join(t1, NULL);
1895 }
1896
1897 if (!ifobj2)
1898 pthread_kill(t0, SIGUSR1);
1899 else
1900 pthread_join(t0, NULL);
3143d10b 1901
a693ff3e 1902 if (test->total_steps == test->current_step || test->fail) {
8913e653
TV
1903 u32 i;
1904
7f881984 1905 if (ifobj2)
8913e653
TV
1906 for (i = 0; i < test->nb_sockets; i++)
1907 xsk_socket__delete(ifobj2->xsk_arr[i].xsk);
1908
1909 for (i = 0; i < test->nb_sockets; i++)
1910 xsk_socket__delete(ifobj1->xsk_arr[i].xsk);
1911
7f881984
MK
1912 testapp_clean_xsk_umem(ifobj1);
1913 if (ifobj2 && !ifobj2->shared_umem)
1914 testapp_clean_xsk_umem(ifobj2);
a693ff3e
MF
1915 }
1916
3143d10b
SKR
1917 return !!test->fail;
1918}
1919
895b62ee 1920static int testapp_validate_traffic(struct test_spec *test)
facb7cb2 1921{
7d8319a7
MK
1922 struct ifobject *ifobj_rx = test->ifobj_rx;
1923 struct ifobject *ifobj_tx = test->ifobj_tx;
1924
041b68f6
MK
1925 if ((ifobj_rx->umem->unaligned_mode && !ifobj_rx->unaligned_supp) ||
1926 (ifobj_tx->umem->unaligned_mode && !ifobj_tx->unaligned_supp)) {
1927 ksft_test_result_skip("No huge pages present.\n");
1928 return TEST_SKIP;
1929 }
1930
776021e0 1931 if (test->set_ring) {
d80d61ab
TV
1932 if (ifobj_tx->hw_ring_size_supp) {
1933 if (set_ring_size(ifobj_tx)) {
1934 ksft_test_result_skip("Failed to change HW ring size.\n");
1935 return TEST_FAILURE;
1936 }
1937 } else {
1938 ksft_test_result_skip("Changing HW ring size not supported.\n");
1939 return TEST_SKIP;
1940 }
776021e0
TV
1941 }
1942
7d8319a7
MK
1943 xsk_attach_xdp_progs(test, ifobj_rx, ifobj_tx);
1944 return __testapp_validate_traffic(test, ifobj_rx, ifobj_tx);
7f881984 1945}
a693ff3e 1946
7f881984
MK
1947static int testapp_validate_traffic_single_thread(struct test_spec *test, struct ifobject *ifobj)
1948{
1949 return __testapp_validate_traffic(test, ifobj, NULL);
6674bf66
WJ
1950}
1951
041b68f6 1952static int testapp_teardown(struct test_spec *test)
9445f8c7
MF
1953{
1954 int i;
1955
1956 for (i = 0; i < MAX_TEARDOWN_ITER; i++) {
895b62ee 1957 if (testapp_validate_traffic(test))
041b68f6 1958 return TEST_FAILURE;
ce74acaf 1959 test_spec_reset(test);
9445f8c7 1960 }
041b68f6
MK
1961
1962 return TEST_PASS;
9445f8c7
MF
1963}
1964
ce74acaf 1965static void swap_directions(struct ifobject **ifobj1, struct ifobject **ifobj2)
9445f8c7 1966{
ce74acaf 1967 thread_func_t tmp_func_ptr = (*ifobj1)->func_ptr;
ce74acaf 1968 struct ifobject *tmp_ifobj = (*ifobj1);
9445f8c7 1969
ce74acaf 1970 (*ifobj1)->func_ptr = (*ifobj2)->func_ptr;
ce74acaf 1971 (*ifobj2)->func_ptr = tmp_func_ptr;
9445f8c7 1972
ce74acaf
MK
1973 *ifobj1 = *ifobj2;
1974 *ifobj2 = tmp_ifobj;
9445f8c7
MF
1975}
1976
f20fbcd0 1977static int testapp_bidirectional(struct test_spec *test)
6674bf66 1978{
041b68f6
MK
1979 int res;
1980
1856c24d
MK
1981 test->ifobj_tx->rx_on = true;
1982 test->ifobj_rx->tx_on = true;
55be575d 1983 test->total_steps = 2;
895b62ee 1984 if (testapp_validate_traffic(test))
041b68f6 1985 return TEST_FAILURE;
55be575d 1986
2d2712ca 1987 print_verbose("Switching Tx/Rx direction\n");
55be575d 1988 swap_directions(&test->ifobj_rx, &test->ifobj_tx);
041b68f6 1989 res = __testapp_validate_traffic(test, test->ifobj_rx, test->ifobj_tx);
6674bf66 1990
ce74acaf 1991 swap_directions(&test->ifobj_rx, &test->ifobj_tx);
041b68f6 1992 return res;
facb7cb2
WJ
1993}
1994
8913e653 1995static int swap_xsk_resources(struct test_spec *test)
27e1ca25 1996{
f0a249df 1997 int ret;
3b22523b 1998
8913e653
TV
1999 test->ifobj_tx->xsk_arr[0].pkt_stream = NULL;
2000 test->ifobj_rx->xsk_arr[0].pkt_stream = NULL;
2001 test->ifobj_tx->xsk_arr[1].pkt_stream = test->tx_pkt_stream_default;
2002 test->ifobj_rx->xsk_arr[1].pkt_stream = test->rx_pkt_stream_default;
2003 test->ifobj_tx->xsk = &test->ifobj_tx->xsk_arr[1];
2004 test->ifobj_rx->xsk = &test->ifobj_rx->xsk_arr[1];
3b22523b 2005
fc2cb864 2006 ret = xsk_update_xskmap(test->ifobj_rx->xskmap, test->ifobj_rx->xsk->xsk, 0);
3b22523b 2007 if (ret)
5fc494d5
MK
2008 return TEST_FAILURE;
2009
2010 return TEST_PASS;
27e1ca25
MF
2011}
2012
f20fbcd0 2013static int testapp_xdp_prog_cleanup(struct test_spec *test)
27e1ca25 2014{
55be575d 2015 test->total_steps = 2;
85c6c957 2016 test->nb_sockets = 2;
895b62ee 2017 if (testapp_validate_traffic(test))
041b68f6 2018 return TEST_FAILURE;
55be575d 2019
8913e653 2020 if (swap_xsk_resources(test))
5fc494d5 2021 return TEST_FAILURE;
041b68f6 2022 return testapp_validate_traffic(test);
27e1ca25
MF
2023}
2024
041b68f6 2025static int testapp_headroom(struct test_spec *test)
e34087fc 2026{
e34087fc 2027 test->ifobj_rx->umem->frame_headroom = UMEM_HEADROOM_TEST_SIZE;
041b68f6 2028 return testapp_validate_traffic(test);
e34087fc
MK
2029}
2030
041b68f6 2031static int testapp_stats_rx_dropped(struct test_spec *test)
b267e5a4 2032{
041b68f6
MK
2033 if (test->mode == TEST_MODE_ZC) {
2034 ksft_test_result_skip("Can not run RX_DROPPED test for ZC mode\n");
2035 return TEST_SKIP;
2036 }
2037
a693ff3e 2038 pkt_stream_replace_half(test, MIN_PKT_SIZE * 4, 0);
4fec7028 2039 test->ifobj_rx->umem->frame_headroom = test->ifobj_rx->umem->frame_size -
27e934be 2040 XDP_PACKET_HEADROOM - MIN_PKT_SIZE * 3;
27e934be 2041 pkt_stream_receive_half(test);
4fec7028 2042 test->ifobj_rx->validation_func = validate_rx_dropped;
041b68f6 2043 return testapp_validate_traffic(test);
4fec7028 2044}
605091c5 2045
041b68f6 2046static int testapp_stats_tx_invalid_descs(struct test_spec *test)
4fec7028 2047{
27e934be 2048 pkt_stream_replace_half(test, XSK_UMEM__INVALID_FRAME_SIZE, 0);
4fec7028 2049 test->ifobj_tx->validation_func = validate_tx_invalid_descs;
041b68f6 2050 return testapp_validate_traffic(test);
4fec7028 2051}
895b62ee 2052
041b68f6 2053static int testapp_stats_rx_full(struct test_spec *test)
4fec7028 2054{
df82d2e8 2055 pkt_stream_replace(test, DEFAULT_UMEM_BUFFERS + DEFAULT_UMEM_BUFFERS / 2, MIN_PKT_SIZE);
46e43786 2056 test->ifobj_rx->xsk->pkt_stream = pkt_stream_generate(DEFAULT_UMEM_BUFFERS, MIN_PKT_SIZE);
27e934be
MK
2057
2058 test->ifobj_rx->xsk->rxqsize = DEFAULT_UMEM_BUFFERS;
2059 test->ifobj_rx->release_rx = false;
4fec7028 2060 test->ifobj_rx->validation_func = validate_rx_full;
041b68f6 2061 return testapp_validate_traffic(test);
4fec7028 2062}
b267e5a4 2063
041b68f6 2064static int testapp_stats_fill_empty(struct test_spec *test)
4fec7028 2065{
df82d2e8 2066 pkt_stream_replace(test, DEFAULT_UMEM_BUFFERS + DEFAULT_UMEM_BUFFERS / 2, MIN_PKT_SIZE);
46e43786 2067 test->ifobj_rx->xsk->pkt_stream = pkt_stream_generate(DEFAULT_UMEM_BUFFERS, MIN_PKT_SIZE);
27e934be
MK
2068
2069 test->ifobj_rx->use_fill_ring = false;
4fec7028 2070 test->ifobj_rx->validation_func = validate_fill_empty;
041b68f6 2071 return testapp_validate_traffic(test);
b267e5a4
CL
2072}
2073
f20fbcd0 2074static int testapp_send_receive_unaligned(struct test_spec *test)
a4ba98dd 2075{
a4ba98dd
MK
2076 test->ifobj_tx->umem->unaligned_mode = true;
2077 test->ifobj_rx->umem->unaligned_mode = true;
d9f6d970 2078 /* Let half of the packets straddle a 4K buffer boundary */
df82d2e8 2079 pkt_stream_replace_half(test, MIN_PKT_SIZE, -MIN_PKT_SIZE / 2);
a4ba98dd 2080
041b68f6 2081 return testapp_validate_traffic(test);
a4ba98dd
MK
2082}
2083
f20fbcd0 2084static int testapp_send_receive_unaligned_mb(struct test_spec *test)
1005a226 2085{
1005a226
MK
2086 test->mtu = MAX_ETH_JUMBO_SIZE;
2087 test->ifobj_tx->umem->unaligned_mode = true;
2088 test->ifobj_rx->umem->unaligned_mode = true;
2089 pkt_stream_replace(test, DEFAULT_PKT_CNT, MAX_ETH_JUMBO_SIZE);
2090 return testapp_validate_traffic(test);
2091}
2092
041b68f6 2093static int testapp_single_pkt(struct test_spec *test)
96a40678 2094{
d9f6d970 2095 struct pkt pkts[] = {{0, MIN_PKT_SIZE, 0, true}};
96a40678
MK
2096
2097 pkt_stream_generate_custom(test, pkts, ARRAY_SIZE(pkts));
041b68f6 2098 return testapp_validate_traffic(test);
96a40678
MK
2099}
2100
f20fbcd0 2101static int testapp_send_receive_mb(struct test_spec *test)
f540d44e 2102{
f540d44e
MK
2103 test->mtu = MAX_ETH_JUMBO_SIZE;
2104 pkt_stream_replace(test, DEFAULT_PKT_CNT, MAX_ETH_JUMBO_SIZE);
2105
2106 return testapp_validate_traffic(test);
2107}
2108
69760449
MK
2109static int testapp_invalid_desc_mb(struct test_spec *test)
2110{
2111 struct xsk_umem_info *umem = test->ifobj_tx->umem;
2112 u64 umem_size = umem->num_frames * umem->frame_size;
2113 struct pkt pkts[] = {
2114 /* Valid packet for synch to start with */
2115 {0, MIN_PKT_SIZE, 0, true, 0},
2116 /* Zero frame len is not legal */
2117 {0, XSK_UMEM__LARGE_FRAME_SIZE, 0, false, XDP_PKT_CONTD},
2118 {0, XSK_UMEM__LARGE_FRAME_SIZE, 0, false, XDP_PKT_CONTD},
2119 {0, 0, 0, false, 0},
2120 /* Invalid address in the second frame */
2121 {0, XSK_UMEM__LARGE_FRAME_SIZE, 0, false, XDP_PKT_CONTD},
2122 {umem_size, XSK_UMEM__LARGE_FRAME_SIZE, 0, false, XDP_PKT_CONTD},
2123 /* Invalid len in the middle */
2124 {0, XSK_UMEM__LARGE_FRAME_SIZE, 0, false, XDP_PKT_CONTD},
2125 {0, XSK_UMEM__INVALID_FRAME_SIZE, 0, false, XDP_PKT_CONTD},
2126 /* Invalid options in the middle */
2127 {0, XSK_UMEM__LARGE_FRAME_SIZE, 0, false, XDP_PKT_CONTD},
2128 {0, XSK_UMEM__LARGE_FRAME_SIZE, 0, false, XSK_DESC__INVALID_OPTION},
2129 /* Transmit 2 frags, receive 3 */
2130 {0, XSK_UMEM__MAX_FRAME_SIZE, 0, true, XDP_PKT_CONTD},
2131 {0, XSK_UMEM__MAX_FRAME_SIZE, 0, true, 0},
2132 /* Middle frame crosses chunk boundary with small length */
2133 {0, XSK_UMEM__LARGE_FRAME_SIZE, 0, false, XDP_PKT_CONTD},
2134 {-MIN_PKT_SIZE / 2, MIN_PKT_SIZE, 0, false, 0},
2135 /* Valid packet for synch so that something is received */
2136 {0, MIN_PKT_SIZE, 0, true, 0}};
2137
2138 if (umem->unaligned_mode) {
2139 /* Crossing a chunk boundary allowed */
2140 pkts[12].valid = true;
2141 pkts[13].valid = true;
2142 }
2143
2144 test->mtu = MAX_ETH_JUMBO_SIZE;
2145 pkt_stream_generate_custom(test, pkts, ARRAY_SIZE(pkts));
2146 return testapp_validate_traffic(test);
2147}
2148
041b68f6 2149static int testapp_invalid_desc(struct test_spec *test)
0d1b7f3a 2150{
d9f6d970
MK
2151 struct xsk_umem_info *umem = test->ifobj_tx->umem;
2152 u64 umem_size = umem->num_frames * umem->frame_size;
0d1b7f3a 2153 struct pkt pkts[] = {
f3e619bb 2154 /* Zero packet address allowed */
df82d2e8 2155 {0, MIN_PKT_SIZE, 0, true},
f3e619bb 2156 /* Allowed packet */
d9f6d970 2157 {0, MIN_PKT_SIZE, 0, true},
0d1b7f3a 2158 /* Straddling the start of umem */
df82d2e8 2159 {-2, MIN_PKT_SIZE, 0, false},
0d1b7f3a 2160 /* Packet too large */
d9f6d970 2161 {0, XSK_UMEM__INVALID_FRAME_SIZE, 0, false},
ccd1b293 2162 /* Up to end of umem allowed */
d9f6d970 2163 {umem_size - MIN_PKT_SIZE - 2 * umem->frame_size, MIN_PKT_SIZE, 0, true},
0d1b7f3a 2164 /* After umem ends */
df82d2e8 2165 {umem_size, MIN_PKT_SIZE, 0, false},
0d1b7f3a 2166 /* Straddle the end of umem */
df82d2e8 2167 {umem_size - MIN_PKT_SIZE / 2, MIN_PKT_SIZE, 0, false},
d9f6d970
MK
2168 /* Straddle a 4K boundary */
2169 {0x1000 - MIN_PKT_SIZE / 2, MIN_PKT_SIZE, 0, false},
2170 /* Straddle a 2K boundary */
2171 {0x800 - MIN_PKT_SIZE / 2, MIN_PKT_SIZE, 0, true},
0d1b7f3a 2172 /* Valid packet for synch so that something is received */
d9f6d970 2173 {0, MIN_PKT_SIZE, 0, true}};
0d1b7f3a 2174
d9f6d970
MK
2175 if (umem->unaligned_mode) {
2176 /* Crossing a page boundary allowed */
ccd1b293 2177 pkts[7].valid = true;
0d1b7f3a 2178 }
d9f6d970
MK
2179 if (umem->frame_size == XSK_UMEM__DEFAULT_FRAME_SIZE / 2) {
2180 /* Crossing a 2K frame size boundary not allowed */
ccd1b293 2181 pkts[8].valid = false;
909f0e28
MK
2182 }
2183
a693ff3e 2184 if (test->ifobj_tx->shared_umem) {
d9f6d970
MK
2185 pkts[4].offset += umem_size;
2186 pkts[5].offset += umem_size;
2187 pkts[6].offset += umem_size;
a693ff3e
MF
2188 }
2189
0d1b7f3a 2190 pkt_stream_generate_custom(test, pkts, ARRAY_SIZE(pkts));
041b68f6 2191 return testapp_validate_traffic(test);
0d1b7f3a
MK
2192}
2193
041b68f6 2194static int testapp_xdp_drop(struct test_spec *test)
80bea9ac 2195{
7d8319a7
MK
2196 struct xsk_xdp_progs *skel_rx = test->ifobj_rx->xdp_progs;
2197 struct xsk_xdp_progs *skel_tx = test->ifobj_tx->xdp_progs;
80bea9ac 2198
7d8319a7
MK
2199 test_spec_set_xdp_prog(test, skel_rx->progs.xsk_xdp_drop, skel_tx->progs.xsk_xdp_drop,
2200 skel_rx->maps.xsk, skel_tx->maps.xsk);
80bea9ac
MK
2201
2202 pkt_stream_receive_half(test);
041b68f6 2203 return testapp_validate_traffic(test);
80bea9ac
MK
2204}
2205
13c341c4 2206static int testapp_xdp_metadata_copy(struct test_spec *test)
9a321fd3
TV
2207{
2208 struct xsk_xdp_progs *skel_rx = test->ifobj_rx->xdp_progs;
2209 struct xsk_xdp_progs *skel_tx = test->ifobj_tx->xdp_progs;
2210 struct bpf_map *data_map;
2211 int count = 0;
2212 int key = 0;
2213
9a321fd3
TV
2214 test_spec_set_xdp_prog(test, skel_rx->progs.xsk_xdp_populate_metadata,
2215 skel_tx->progs.xsk_xdp_populate_metadata,
2216 skel_rx->maps.xsk, skel_tx->maps.xsk);
2217 test->ifobj_rx->use_metadata = true;
2218
2219 data_map = bpf_object__find_map_by_name(skel_rx->obj, "xsk_xdp_.bss");
5fc494d5
MK
2220 if (!data_map || !bpf_map__is_internal(data_map)) {
2221 ksft_print_msg("Error: could not find bss section of XDP program\n");
2222 return TEST_FAILURE;
2223 }
9a321fd3 2224
5fc494d5
MK
2225 if (bpf_map_update_elem(bpf_map__fd(data_map), &key, &count, BPF_ANY)) {
2226 ksft_print_msg("Error: could not update count element\n");
2227 return TEST_FAILURE;
2228 }
9a321fd3 2229
041b68f6 2230 return testapp_validate_traffic(test);
9a321fd3
TV
2231}
2232
6d198a89
TV
2233static int testapp_xdp_shared_umem(struct test_spec *test)
2234{
2235 struct xsk_xdp_progs *skel_rx = test->ifobj_rx->xdp_progs;
2236 struct xsk_xdp_progs *skel_tx = test->ifobj_tx->xdp_progs;
2237
2238 test->total_steps = 1;
2239 test->nb_sockets = 2;
2240
2241 test_spec_set_xdp_prog(test, skel_rx->progs.xsk_xdp_shared_umem,
2242 skel_tx->progs.xsk_xdp_shared_umem,
2243 skel_rx->maps.xsk, skel_tx->maps.xsk);
2244
2245 pkt_stream_even_odd_sequence(test);
2246
2247 return testapp_validate_traffic(test);
2248}
2249
041b68f6 2250static int testapp_poll_txq_tmout(struct test_spec *test)
7f881984 2251{
7f881984
MK
2252 test->ifobj_tx->use_poll = true;
2253 /* create invalid frame by set umem frame_size and pkt length equal to 2048 */
2254 test->ifobj_tx->umem->frame_size = 2048;
2255 pkt_stream_replace(test, 2 * DEFAULT_PKT_CNT, 2048);
041b68f6 2256 return testapp_validate_traffic_single_thread(test, test->ifobj_tx);
7f881984
MK
2257}
2258
041b68f6 2259static int testapp_poll_rxq_tmout(struct test_spec *test)
7f881984 2260{
7f881984 2261 test->ifobj_rx->use_poll = true;
041b68f6 2262 return testapp_validate_traffic_single_thread(test, test->ifobj_rx);
7f881984
MK
2263}
2264
807bf4da
MK
2265static int testapp_too_many_frags(struct test_spec *test)
2266{
d41905b3 2267 struct pkt *pkts;
807bf4da 2268 u32 max_frags, i;
d41905b3 2269 int ret;
807bf4da 2270
d41905b3 2271 if (test->mode == TEST_MODE_ZC) {
807bf4da 2272 max_frags = test->ifobj_tx->xdp_zc_max_segs;
d41905b3
MF
2273 } else {
2274 max_frags = get_max_skb_frags();
2275 if (!max_frags) {
2276 ksft_print_msg("Couldn't retrieve MAX_SKB_FRAGS from system, using default (17) value\n");
2277 max_frags = 17;
2278 }
2279 max_frags += 1;
2280 }
2281
2282 pkts = calloc(2 * max_frags + 2, sizeof(struct pkt));
2283 if (!pkts)
2284 return TEST_FAILURE;
807bf4da
MK
2285
2286 test->mtu = MAX_ETH_JUMBO_SIZE;
2287
2288 /* Valid packet for synch */
2289 pkts[0].len = MIN_PKT_SIZE;
2290 pkts[0].valid = true;
2291
2292 /* One valid packet with the max amount of frags */
2293 for (i = 1; i < max_frags + 1; i++) {
2294 pkts[i].len = MIN_PKT_SIZE;
2295 pkts[i].options = XDP_PKT_CONTD;
2296 pkts[i].valid = true;
2297 }
2298 pkts[max_frags].options = 0;
2299
2300 /* An invalid packet with the max amount of frags but signals packet
2301 * continues on the last frag
2302 */
2303 for (i = max_frags + 1; i < 2 * max_frags + 1; i++) {
2304 pkts[i].len = MIN_PKT_SIZE;
2305 pkts[i].options = XDP_PKT_CONTD;
2306 pkts[i].valid = false;
2307 }
2308
2309 /* Valid packet for synch */
2310 pkts[2 * max_frags + 1].len = MIN_PKT_SIZE;
2311 pkts[2 * max_frags + 1].valid = true;
2312
2313 pkt_stream_generate_custom(test, pkts, 2 * max_frags + 2);
d41905b3
MF
2314 ret = testapp_validate_traffic(test);
2315
2316 free(pkts);
2317 return ret;
807bf4da
MK
2318}
2319
f0a249df
MK
2320static int xsk_load_xdp_programs(struct ifobject *ifobj)
2321{
2322 ifobj->xdp_progs = xsk_xdp_progs__open_and_load();
2323 if (libbpf_get_error(ifobj->xdp_progs))
2324 return libbpf_get_error(ifobj->xdp_progs);
2325
2326 return 0;
2327}
2328
2329static void xsk_unload_xdp_programs(struct ifobject *ifobj)
2330{
2331 xsk_xdp_progs__destroy(ifobj->xdp_progs);
2332}
2333
041b68f6
MK
2334/* Simple test */
2335static bool hugepages_present(void)
2336{
2337 size_t mmap_sz = 2 * DEFAULT_UMEM_BUFFERS * XSK_UMEM__DEFAULT_FRAME_SIZE;
2338 void *bufs;
2339
2340 bufs = mmap(NULL, mmap_sz, PROT_READ | PROT_WRITE,
2341 MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB, -1, MAP_HUGE_2MB);
2342 if (bufs == MAP_FAILED)
2343 return false;
2344
2345 mmap_sz = ceil_u64(mmap_sz, HUGEPAGE_SIZE) * HUGEPAGE_SIZE;
2346 munmap(bufs, mmap_sz);
2347 return true;
2348}
2349
985fd214 2350static void init_iface(struct ifobject *ifobj, thread_func_t func_ptr)
facb7cb2 2351{
f540d44e 2352 LIBBPF_OPTS(bpf_xdp_query_opts, query_opts);
f0a249df 2353 int err;
e623bfde 2354
744eb5c8 2355 ifobj->func_ptr = func_ptr;
aa61d81f 2356
f0a249df 2357 err = xsk_load_xdp_programs(ifobj);
aa61d81f 2358 if (err) {
7c3fcf08 2359 ksft_print_msg("Error loading XDP program\n");
aa61d81f
MK
2360 exit_with_error(err);
2361 }
041b68f6
MK
2362
2363 if (hugepages_present())
2364 ifobj->unaligned_supp = true;
f540d44e
MK
2365
2366 err = bpf_xdp_query(ifobj->ifindex, XDP_FLAGS_DRV_MODE, &query_opts);
2367 if (err) {
13fd5e14 2368 ksft_print_msg("Error querying XDP capabilities\n");
f540d44e
MK
2369 exit_with_error(-err);
2370 }
2371 if (query_opts.feature_flags & NETDEV_XDP_ACT_RX_SG)
2372 ifobj->multi_buff_supp = true;
807bf4da
MK
2373 if (query_opts.feature_flags & NETDEV_XDP_ACT_XSK_ZEROCOPY) {
2374 if (query_opts.xdp_zc_max_segs > 1) {
f540d44e 2375 ifobj->multi_buff_zc_supp = true;
807bf4da
MK
2376 ifobj->xdp_zc_max_segs = query_opts.xdp_zc_max_segs;
2377 } else {
2378 ifobj->xdp_zc_max_segs = 0;
2379 }
2380 }
facb7cb2
WJ
2381}
2382
13c341c4
MK
2383static int testapp_send_receive(struct test_spec *test)
2384{
13c341c4
MK
2385 return testapp_validate_traffic(test);
2386}
2387
2388static int testapp_send_receive_2k_frame(struct test_spec *test)
2389{
13c341c4
MK
2390 test->ifobj_tx->umem->frame_size = 2048;
2391 test->ifobj_rx->umem->frame_size = 2048;
2392 pkt_stream_replace(test, DEFAULT_PKT_CNT, MIN_PKT_SIZE);
2393 return testapp_validate_traffic(test);
2394}
2395
2396static int testapp_poll_rx(struct test_spec *test)
2397{
2398 test->ifobj_rx->use_poll = true;
13c341c4
MK
2399 return testapp_validate_traffic(test);
2400}
2401
2402static int testapp_poll_tx(struct test_spec *test)
2403{
2404 test->ifobj_tx->use_poll = true;
13c341c4
MK
2405 return testapp_validate_traffic(test);
2406}
2407
2408static int testapp_aligned_inv_desc(struct test_spec *test)
2409{
13c341c4
MK
2410 return testapp_invalid_desc(test);
2411}
2412
2413static int testapp_aligned_inv_desc_2k_frame(struct test_spec *test)
2414{
13c341c4
MK
2415 test->ifobj_tx->umem->frame_size = 2048;
2416 test->ifobj_rx->umem->frame_size = 2048;
2417 return testapp_invalid_desc(test);
2418}
2419
2420static int testapp_unaligned_inv_desc(struct test_spec *test)
2421{
13c341c4
MK
2422 test->ifobj_tx->umem->unaligned_mode = true;
2423 test->ifobj_rx->umem->unaligned_mode = true;
2424 return testapp_invalid_desc(test);
2425}
2426
2427static int testapp_unaligned_inv_desc_4001_frame(struct test_spec *test)
2428{
2429 u64 page_size, umem_size;
2430
13c341c4
MK
2431 /* Odd frame size so the UMEM doesn't end near a page boundary. */
2432 test->ifobj_tx->umem->frame_size = 4001;
2433 test->ifobj_rx->umem->frame_size = 4001;
2434 test->ifobj_tx->umem->unaligned_mode = true;
2435 test->ifobj_rx->umem->unaligned_mode = true;
2436 /* This test exists to test descriptors that staddle the end of
2437 * the UMEM but not a page.
2438 */
2439 page_size = sysconf(_SC_PAGESIZE);
2440 umem_size = test->ifobj_tx->umem->num_frames * test->ifobj_tx->umem->frame_size;
2441 assert(umem_size % page_size > MIN_PKT_SIZE);
2442 assert(umem_size % page_size < page_size - MIN_PKT_SIZE);
2443
2444 return testapp_invalid_desc(test);
2445}
2446
2447static int testapp_aligned_inv_desc_mb(struct test_spec *test)
2448{
13c341c4
MK
2449 return testapp_invalid_desc_mb(test);
2450}
2451
2452static int testapp_unaligned_inv_desc_mb(struct test_spec *test)
2453{
13c341c4
MK
2454 test->ifobj_tx->umem->unaligned_mode = true;
2455 test->ifobj_rx->umem->unaligned_mode = true;
2456 return testapp_invalid_desc_mb(test);
2457}
2458
2459static int testapp_xdp_metadata(struct test_spec *test)
2460{
13c341c4
MK
2461 return testapp_xdp_metadata_copy(test);
2462}
2463
2464static int testapp_xdp_metadata_mb(struct test_spec *test)
2465{
13c341c4
MK
2466 test->mtu = MAX_ETH_JUMBO_SIZE;
2467 return testapp_xdp_metadata_copy(test);
2468}
2469
c4f96053
TV
2470static int testapp_hw_sw_min_ring_size(struct test_spec *test)
2471{
2472 int ret;
2473
2474 test->set_ring = true;
2475 test->total_steps = 2;
2476 test->ifobj_tx->ring.tx_pending = DEFAULT_BATCH_SIZE;
2477 test->ifobj_tx->ring.rx_pending = DEFAULT_BATCH_SIZE * 2;
2478 test->ifobj_tx->xsk->batch_size = 1;
2479 test->ifobj_rx->xsk->batch_size = 1;
2480 ret = testapp_validate_traffic(test);
2481 if (ret)
2482 return ret;
2483
2484 /* Set batch size to hw_ring_size - 1 */
2485 test->ifobj_tx->xsk->batch_size = DEFAULT_BATCH_SIZE - 1;
2486 test->ifobj_rx->xsk->batch_size = DEFAULT_BATCH_SIZE - 1;
2487 return testapp_validate_traffic(test);
2488}
2489
c53908b2
TV
2490static int testapp_hw_sw_max_ring_size(struct test_spec *test)
2491{
e4a195e2 2492 u32 max_descs = XSK_RING_PROD__DEFAULT_NUM_DESCS * 4;
c53908b2
TV
2493 int ret;
2494
2495 test->set_ring = true;
2496 test->total_steps = 2;
2497 test->ifobj_tx->ring.tx_pending = test->ifobj_tx->ring.tx_max_pending;
2498 test->ifobj_tx->ring.rx_pending = test->ifobj_tx->ring.rx_max_pending;
2499 test->ifobj_rx->umem->num_frames = max_descs;
e4a195e2
TV
2500 test->ifobj_rx->umem->fill_size = max_descs;
2501 test->ifobj_rx->umem->comp_size = max_descs;
c53908b2
TV
2502 test->ifobj_tx->xsk->batch_size = XSK_RING_PROD__DEFAULT_NUM_DESCS;
2503 test->ifobj_rx->xsk->batch_size = XSK_RING_PROD__DEFAULT_NUM_DESCS;
2504
2505 ret = testapp_validate_traffic(test);
2506 if (ret)
2507 return ret;
2508
e4a195e2
TV
2509 /* Set batch_size to 8152 for testing, as the ice HW ignores the 3 lowest bits when
2510 * updating the Rx HW tail register.
2511 */
2512 test->ifobj_tx->xsk->batch_size = test->ifobj_tx->ring.tx_max_pending - 8;
2513 test->ifobj_rx->xsk->batch_size = test->ifobj_tx->ring.tx_max_pending - 8;
2514 pkt_stream_replace(test, max_descs, MIN_PKT_SIZE);
c53908b2
TV
2515 return testapp_validate_traffic(test);
2516}
2517
f20fbcd0
MK
2518static void run_pkt_test(struct test_spec *test)
2519{
2520 int ret;
2521
2522 ret = test->test_func(test);
53cb3cec 2523
041b68f6 2524 if (ret == TEST_PASS)
895b62ee
MK
2525 ksft_test_result_pass("PASS: %s %s%s\n", mode_string(test), busy_poll_string(test),
2526 test->name);
e67b2554 2527 pkt_stream_restore_default(test);
d3e3bf5b
CL
2528}
2529
1034b03e
MK
2530static struct ifobject *ifobject_create(void)
2531{
2532 struct ifobject *ifobj;
2533
2534 ifobj = calloc(1, sizeof(struct ifobject));
2535 if (!ifobj)
2536 return NULL;
2537
ed7b74dc 2538 ifobj->xsk_arr = calloc(MAX_SOCKETS, sizeof(*ifobj->xsk_arr));
1034b03e
MK
2539 if (!ifobj->xsk_arr)
2540 goto out_xsk_arr;
2541
3b22523b
MK
2542 ifobj->umem = calloc(1, sizeof(*ifobj->umem));
2543 if (!ifobj->umem)
2544 goto out_umem;
1034b03e
MK
2545
2546 return ifobj;
2547
3b22523b 2548out_umem:
1034b03e
MK
2549 free(ifobj->xsk_arr);
2550out_xsk_arr:
2551 free(ifobj);
2552 return NULL;
2553}
2554
2555static void ifobject_delete(struct ifobject *ifobj)
2556{
3b22523b 2557 free(ifobj->umem);
1034b03e
MK
2558 free(ifobj->xsk_arr);
2559 free(ifobj);
2560}
2561
aa61d81f 2562static bool is_xdp_supported(int ifindex)
0d68e6fe
MF
2563{
2564 int flags = XDP_FLAGS_DRV_MODE;
2565
2566 LIBBPF_OPTS(bpf_link_create_opts, opts, .flags = flags);
2567 struct bpf_insn insns[2] = {
2568 BPF_MOV64_IMM(BPF_REG_0, XDP_PASS),
2569 BPF_EXIT_INSN()
2570 };
0d68e6fe
MF
2571 int prog_fd, insn_cnt = ARRAY_SIZE(insns);
2572 int err;
2573
2574 prog_fd = bpf_prog_load(BPF_PROG_TYPE_XDP, NULL, "GPL", insns, insn_cnt, NULL);
2575 if (prog_fd < 0)
2576 return false;
2577
2578 err = bpf_xdp_attach(ifindex, prog_fd, flags, NULL);
2579 if (err) {
2580 close(prog_fd);
2581 return false;
2582 }
2583
2584 bpf_xdp_detach(ifindex, flags, NULL);
2585 close(prog_fd);
2586
2587 return true;
2588}
2589
f20fbcd0
MK
2590static const struct test_spec tests[] = {
2591 {.name = "SEND_RECEIVE", .test_func = testapp_send_receive},
2592 {.name = "SEND_RECEIVE_2K_FRAME", .test_func = testapp_send_receive_2k_frame},
2593 {.name = "SEND_RECEIVE_SINGLE_PKT", .test_func = testapp_single_pkt},
2594 {.name = "POLL_RX", .test_func = testapp_poll_rx},
2595 {.name = "POLL_TX", .test_func = testapp_poll_tx},
2596 {.name = "POLL_RXQ_FULL", .test_func = testapp_poll_rxq_tmout},
2597 {.name = "POLL_TXQ_FULL", .test_func = testapp_poll_txq_tmout},
2598 {.name = "SEND_RECEIVE_UNALIGNED", .test_func = testapp_send_receive_unaligned},
2599 {.name = "ALIGNED_INV_DESC", .test_func = testapp_aligned_inv_desc},
2600 {.name = "ALIGNED_INV_DESC_2K_FRAME_SIZE", .test_func = testapp_aligned_inv_desc_2k_frame},
2601 {.name = "UNALIGNED_INV_DESC", .test_func = testapp_unaligned_inv_desc},
2602 {.name = "UNALIGNED_INV_DESC_4001_FRAME_SIZE",
2603 .test_func = testapp_unaligned_inv_desc_4001_frame},
2604 {.name = "UMEM_HEADROOM", .test_func = testapp_headroom},
2605 {.name = "TEARDOWN", .test_func = testapp_teardown},
2606 {.name = "BIDIRECTIONAL", .test_func = testapp_bidirectional},
2607 {.name = "STAT_RX_DROPPED", .test_func = testapp_stats_rx_dropped},
2608 {.name = "STAT_TX_INVALID", .test_func = testapp_stats_tx_invalid_descs},
2609 {.name = "STAT_RX_FULL", .test_func = testapp_stats_rx_full},
2610 {.name = "STAT_FILL_EMPTY", .test_func = testapp_stats_fill_empty},
2611 {.name = "XDP_PROG_CLEANUP", .test_func = testapp_xdp_prog_cleanup},
2612 {.name = "XDP_DROP_HALF", .test_func = testapp_xdp_drop},
6d198a89 2613 {.name = "XDP_SHARED_UMEM", .test_func = testapp_xdp_shared_umem},
f20fbcd0
MK
2614 {.name = "XDP_METADATA_COPY", .test_func = testapp_xdp_metadata},
2615 {.name = "XDP_METADATA_COPY_MULTI_BUFF", .test_func = testapp_xdp_metadata_mb},
2616 {.name = "SEND_RECEIVE_9K_PACKETS", .test_func = testapp_send_receive_mb},
2617 {.name = "SEND_RECEIVE_UNALIGNED_9K_PACKETS",
2618 .test_func = testapp_send_receive_unaligned_mb},
2619 {.name = "ALIGNED_INV_DESC_MULTI_BUFF", .test_func = testapp_aligned_inv_desc_mb},
2620 {.name = "UNALIGNED_INV_DESC_MULTI_BUFF", .test_func = testapp_unaligned_inv_desc_mb},
2621 {.name = "TOO_MANY_FRAGS", .test_func = testapp_too_many_frags},
c4f96053 2622 {.name = "HW_SW_MIN_RING_SIZE", .test_func = testapp_hw_sw_min_ring_size},
c53908b2 2623 {.name = "HW_SW_MAX_RING_SIZE", .test_func = testapp_hw_sw_max_ring_size},
776021e0 2624 };
f20fbcd0 2625
c53dab7d
MK
2626static void print_tests(void)
2627{
2628 u32 i;
2629
2630 printf("Tests:\n");
2631 for (i = 0; i < ARRAY_SIZE(tests); i++)
2632 printf("%u: %s\n", i, tests[i].name);
2633}
2634
facb7cb2
WJ
2635int main(int argc, char **argv)
2636{
1adef064
MF
2637 struct pkt_stream *rx_pkt_stream_default;
2638 struct pkt_stream *tx_pkt_stream_default;
ce74acaf 2639 struct ifobject *ifobj_tx, *ifobj_rx;
146e3055 2640 u32 i, j, failed_tests = 0, nb_tests;
0d68e6fe 2641 int modes = TEST_MODE_SKB + 1;
ce74acaf 2642 struct test_spec test;
aa61d81f 2643 bool shared_netdev;
776021e0 2644 int ret;
facb7cb2 2645
b858ba8c
YS
2646 /* Use libbpf 1.0 API mode */
2647 libbpf_set_strict_mode(LIBBPF_STRICT_ALL);
facb7cb2 2648
ce74acaf
MK
2649 ifobj_tx = ifobject_create();
2650 if (!ifobj_tx)
2651 exit_with_error(ENOMEM);
2652 ifobj_rx = ifobject_create();
2653 if (!ifobj_rx)
2654 exit_with_error(ENOMEM);
2655
facb7cb2
WJ
2656 setlocale(LC_ALL, "");
2657
af6731d1 2658 parse_command_line(ifobj_tx, ifobj_rx, argc, argv);
a693ff3e 2659
c53dab7d
MK
2660 if (opt_print_tests) {
2661 print_tests();
2662 ksft_exit_xpass();
2663 }
146e3055
MK
2664 if (opt_run_test != RUN_ALL_TESTS && opt_run_test >= ARRAY_SIZE(tests)) {
2665 ksft_print_msg("Error: test %u does not exist.\n", opt_run_test);
2666 ksft_exit_xfail();
2667 }
c53dab7d 2668
aa61d81f
MK
2669 shared_netdev = (ifobj_tx->ifindex == ifobj_rx->ifindex);
2670 ifobj_tx->shared_umem = shared_netdev;
2671 ifobj_rx->shared_umem = shared_netdev;
ce74acaf 2672
146e3055
MK
2673 if (!validate_interface(ifobj_tx) || !validate_interface(ifobj_rx))
2674 print_usage(argv);
facb7cb2 2675
aa61d81f 2676 if (is_xdp_supported(ifobj_tx->ifindex)) {
0d68e6fe 2677 modes++;
fe2ad08e
MF
2678 if (ifobj_zc_avail(ifobj_tx))
2679 modes++;
2680 }
0d68e6fe 2681
776021e0
TV
2682 ret = get_hw_ring_size(ifobj_tx->ifname, &ifobj_tx->ring);
2683 if (!ret) {
2684 ifobj_tx->hw_ring_size_supp = true;
2685 ifobj_tx->set_ring.default_tx = ifobj_tx->ring.tx_pending;
2686 ifobj_tx->set_ring.default_rx = ifobj_tx->ring.rx_pending;
2687 }
2688
985fd214
TV
2689 init_iface(ifobj_rx, worker_testapp_validate_rx);
2690 init_iface(ifobj_tx, worker_testapp_validate_tx);
aa61d81f 2691
f20fbcd0 2692 test_spec_init(&test, ifobj_tx, ifobj_rx, 0, &tests[0]);
46e43786
TV
2693 tx_pkt_stream_default = pkt_stream_generate(DEFAULT_PKT_CNT, MIN_PKT_SIZE);
2694 rx_pkt_stream_default = pkt_stream_generate(DEFAULT_PKT_CNT, MIN_PKT_SIZE);
1adef064 2695 if (!tx_pkt_stream_default || !rx_pkt_stream_default)
605091c5 2696 exit_with_error(ENOMEM);
1adef064
MF
2697 test.tx_pkt_stream_default = tx_pkt_stream_default;
2698 test.rx_pkt_stream_default = rx_pkt_stream_default;
605091c5 2699
146e3055
MK
2700 if (opt_run_test == RUN_ALL_TESTS)
2701 nb_tests = ARRAY_SIZE(tests);
2702 else
2703 nb_tests = 1;
3956bc34 2704 if (opt_mode == TEST_MODE_ALL) {
146e3055 2705 ksft_set_plan(modes * nb_tests);
3956bc34
MK
2706 } else {
2707 if (opt_mode == TEST_MODE_DRV && modes <= TEST_MODE_DRV) {
2708 ksft_print_msg("Error: XDP_DRV mode not supported.\n");
2709 ksft_exit_xfail();
2710 }
2711 if (opt_mode == TEST_MODE_ZC && modes <= TEST_MODE_ZC) {
2712 ksft_print_msg("Error: zero-copy mode not supported.\n");
2713 ksft_exit_xfail();
2714 }
2715
146e3055 2716 ksft_set_plan(nb_tests);
3956bc34 2717 }
facb7cb2 2718
aa61d81f 2719 for (i = 0; i < modes; i++) {
3956bc34
MK
2720 if (opt_mode != TEST_MODE_ALL && i != opt_mode)
2721 continue;
2722
f20fbcd0 2723 for (j = 0; j < ARRAY_SIZE(tests); j++) {
146e3055
MK
2724 if (opt_run_test != RUN_ALL_TESTS && j != opt_run_test)
2725 continue;
2726
f20fbcd0
MK
2727 test_spec_init(&test, ifobj_tx, ifobj_rx, i, &tests[j]);
2728 run_pkt_test(&test);
1314c353 2729 usleep(USLEEP_MAX);
895b62ee
MK
2730
2731 if (test.fail)
2732 failed_tests++;
1314c353 2733 }
aa61d81f 2734 }
facb7cb2 2735
776021e0
TV
2736 if (ifobj_tx->hw_ring_size_supp)
2737 hw_ring_size_reset(ifobj_tx);
2738
1adef064
MF
2739 pkt_stream_delete(tx_pkt_stream_default);
2740 pkt_stream_delete(rx_pkt_stream_default);
f0a249df
MK
2741 xsk_unload_xdp_programs(ifobj_tx);
2742 xsk_unload_xdp_programs(ifobj_rx);
ce74acaf
MK
2743 ifobject_delete(ifobj_tx);
2744 ifobject_delete(ifobj_rx);
9445f8c7 2745
895b62ee
MK
2746 if (failed_tests)
2747 ksft_exit_fail();
2748 else
2749 ksft_exit_pass();
facb7cb2 2750}
This page took 0.916305 seconds and 4 git commands to generate.