Commit | Line | Data |
---|---|---|
facb7cb2 WJ |
1 | // SPDX-License-Identifier: GPL-2.0 |
2 | /* Copyright(c) 2020 Intel Corporation. */ | |
3 | ||
4 | /* | |
5 | * Some functions in this program are taken from | |
6 | * Linux kernel samples/bpf/xdpsock* and modified | |
7 | * for use. | |
8 | * | |
9 | * See test_xsk.sh for detailed information on test topology | |
10 | * and prerequisite network setup. | |
11 | * | |
12 | * This test program contains two threads, each thread is single socket with | |
13 | * a unique UMEM. It validates in-order packet delivery and packet content | |
14 | * by sending packets to each other. | |
15 | * | |
16 | * Tests Information: | |
17 | * ------------------ | |
18 | * These selftests test AF_XDP SKB and Native/DRV modes using veth | |
19 | * Virtual Ethernet interfaces. | |
20 | * | |
d3e3bf5b | 21 | * For each mode, the following tests are run: |
a4ba98dd | 22 | * a. nopoll - soft-irq processing in run-to-completion mode |
facb7cb2 | 23 | * b. poll - using poll() syscall |
6674bf66 WJ |
24 | * c. Socket Teardown |
25 | * Create a Tx and a Rx socket, Tx from one socket, Rx on another. Destroy | |
26 | * both sockets, then repeat multiple times. Only nopoll mode is used | |
7d20441e WJ |
27 | * d. Bi-directional sockets |
28 | * Configure sockets as bi-directional tx/rx sockets, sets up fill and | |
29 | * completion rings on each socket, tx/rx in both directions. Only nopoll | |
30 | * mode is used | |
b267e5a4 CL |
31 | * e. Statistics |
32 | * Trigger some error conditions and ensure that the appropriate statistics | |
33 | * are incremented. Within this test, the following statistics are tested: | |
34 | * i. rx dropped | |
35 | * Increase the UMEM frame headroom to a value which results in | |
36 | * insufficient space in the rx buffer for both the packet and the headroom. | |
37 | * ii. tx invalid | |
38 | * Set the 'len' field of tx descriptors to an invalid value (umem frame | |
39 | * size + 1). | |
40 | * iii. rx ring full | |
41 | * Reduce the size of the RX ring to a fraction of the fill ring size. | |
42 | * iv. fill queue empty | |
43 | * Do not populate the fill queue and then try to receive pkts. | |
27e1ca25 MF |
44 | * f. bpf_link resource persistence |
45 | * Configure sockets at indexes 0 and 1, run a traffic on queue ids 0, | |
46 | * then remove xsk sockets from queue 0 on both veth interfaces and | |
47 | * finally run a traffic on queues ids 1 | |
a4ba98dd | 48 | * g. unaligned mode |
0d1b7f3a MK |
49 | * h. tests for invalid and corner case Tx descriptors so that the correct ones |
50 | * are discarded and let through, respectively. | |
909f0e28 | 51 | * i. 2K frame size tests |
f540d44e | 52 | * j. If multi-buffer is supported, send 9k packets divided into 3 frames |
1005a226 MK |
53 | * k. If multi-buffer and huge pages are supported, send 9k packets in a single frame |
54 | * using unaligned mode | |
69760449 MK |
55 | * l. If multi-buffer is supported, try various nasty combinations of descriptors to |
56 | * check if they pass the validation or not | |
facb7cb2 WJ |
57 | * |
58 | * Flow: | |
59 | * ----- | |
60 | * - Single process spawns two threads: Tx and Rx | |
64aef77d MK |
61 | * - Each of these two threads attach to a veth interface |
62 | * - Each thread creates one AF_XDP socket connected to a unique umem for each | |
facb7cb2 | 63 | * veth interface |
64aef77d MK |
64 | * - Tx thread Transmits a number of packets from veth<xxxx> to veth<yyyy> |
65 | * - Rx thread verifies if all packets were received and delivered in-order, | |
facb7cb2 WJ |
66 | * and have the right content |
67 | * | |
d2b0dfd5 | 68 | * Enable/disable packet dump mode: |
facb7cb2 WJ |
69 | * -------------------------- |
70 | * To enable L2 - L4 headers and payload dump of each packet on STDOUT, add | |
71 | * parameter -D to params array in test_xsk.sh, i.e. params=("-S" "-D") | |
72 | */ | |
73 | ||
74 | #define _GNU_SOURCE | |
c0801598 | 75 | #include <assert.h> |
facb7cb2 WJ |
76 | #include <fcntl.h> |
77 | #include <errno.h> | |
78 | #include <getopt.h> | |
facb7cb2 WJ |
79 | #include <linux/if_link.h> |
80 | #include <linux/if_ether.h> | |
2ddade32 | 81 | #include <linux/mman.h> |
f540d44e | 82 | #include <linux/netdev.h> |
8913e653 | 83 | #include <linux/bitmap.h> |
776021e0 | 84 | #include <linux/ethtool.h> |
facb7cb2 WJ |
85 | #include <arpa/inet.h> |
86 | #include <net/if.h> | |
87 | #include <locale.h> | |
88 | #include <poll.h> | |
89 | #include <pthread.h> | |
90 | #include <signal.h> | |
facb7cb2 WJ |
91 | #include <stdio.h> |
92 | #include <stdlib.h> | |
c0247800 | 93 | #include <libgen.h> |
facb7cb2 WJ |
94 | #include <string.h> |
95 | #include <stddef.h> | |
96 | #include <sys/mman.h> | |
f90062b5 | 97 | #include <sys/socket.h> |
db1bd7a9 | 98 | #include <sys/time.h> |
facb7cb2 | 99 | #include <sys/types.h> |
facb7cb2 | 100 | #include <unistd.h> |
7d8319a7 MK |
101 | |
102 | #include "xsk_xdp_progs.skel.h" | |
f3660063 | 103 | #include "xsk.h" |
018a8e75 | 104 | #include "xskxceiver.h" |
0d68e6fe MF |
105 | #include <bpf/bpf.h> |
106 | #include <linux/filter.h> | |
facb7cb2 | 107 | #include "../kselftest.h" |
93ba1124 | 108 | #include "xsk_xdp_common.h" |
facb7cb2 | 109 | |
776021e0 TV |
110 | #include <network_helpers.h> |
111 | ||
3956bc34 | 112 | static bool opt_verbose; |
c53dab7d | 113 | static bool opt_print_tests; |
3956bc34 | 114 | static enum test_mode opt_mode = TEST_MODE_ALL; |
146e3055 | 115 | static u32 opt_run_test = RUN_ALL_TESTS; |
3956bc34 | 116 | |
776021e0 TV |
117 | void test__fail(void) { /* for network_helpers.c */ } |
118 | ||
facb7cb2 WJ |
119 | static void __exit_with_error(int error, const char *file, const char *func, int line) |
120 | { | |
279bdf6b MK |
121 | ksft_test_result_fail("[%s:%s:%i]: ERROR: %d/\"%s\"\n", file, func, line, error, |
122 | strerror(error)); | |
123 | ksft_exit_xfail(); | |
facb7cb2 WJ |
124 | } |
125 | ||
126 | #define exit_with_error(error) __exit_with_error(error, __FILE__, __func__, __LINE__) | |
f90062b5 | 127 | #define busy_poll_string(test) (test)->ifobj_tx->busy_poll ? "BUSY-POLL " : "" |
fe2ad08e MF |
128 | static char *mode_string(struct test_spec *test) |
129 | { | |
130 | switch (test->mode) { | |
131 | case TEST_MODE_SKB: | |
132 | return "SKB"; | |
133 | case TEST_MODE_DRV: | |
134 | return "DRV"; | |
135 | case TEST_MODE_ZC: | |
136 | return "ZC"; | |
137 | default: | |
138 | return "BOGUS"; | |
139 | } | |
140 | } | |
af6731d1 | 141 | |
895b62ee MK |
142 | static void report_failure(struct test_spec *test) |
143 | { | |
144 | if (test->fail) | |
145 | return; | |
146 | ||
147 | ksft_test_result_fail("FAIL: %s %s%s\n", mode_string(test), busy_poll_string(test), | |
148 | test->name); | |
149 | test->fail = true; | |
150 | } | |
facb7cb2 | 151 | |
feb973a9 MK |
152 | /* The payload is a word consisting of a packet sequence number in the upper |
153 | * 16-bits and a intra packet data sequence number in the lower 16 bits. So the 3rd packet's | |
154 | * 5th word of data will contain the number (2<<16) | 4 as they are numbered from 0. | |
155 | */ | |
2f6eae0d | 156 | static void write_payload(void *dest, u32 pkt_nb, u32 start, u32 size) |
facb7cb2 | 157 | { |
feb973a9 | 158 | u32 *ptr = (u32 *)dest, i; |
facb7cb2 | 159 | |
2f6eae0d MK |
160 | start /= sizeof(*ptr); |
161 | size /= sizeof(*ptr); | |
162 | for (i = 0; i < size; i++) | |
163 | ptr[i] = htonl(pkt_nb << 16 | (i + start)); | |
facb7cb2 WJ |
164 | } |
165 | ||
985fd214 | 166 | static void gen_eth_hdr(struct xsk_socket_info *xsk, struct ethhdr *eth_hdr) |
facb7cb2 | 167 | { |
985fd214 TV |
168 | memcpy(eth_hdr->h_dest, xsk->dst_mac, ETH_ALEN); |
169 | memcpy(eth_hdr->h_source, xsk->src_mac, ETH_ALEN); | |
df82d2e8 | 170 | eth_hdr->h_proto = htons(ETH_P_LOOPBACK); |
facb7cb2 WJ |
171 | } |
172 | ||
3143d10b SKR |
173 | static bool is_umem_valid(struct ifobject *ifobj) |
174 | { | |
175 | return !!ifobj->umem->umem; | |
176 | } | |
177 | ||
aa61d81f MK |
178 | static u32 mode_to_xdp_flags(enum test_mode mode) |
179 | { | |
180 | return (mode == TEST_MODE_SKB) ? XDP_FLAGS_SKB_MODE : XDP_FLAGS_DRV_MODE; | |
181 | } | |
182 | ||
d9f6d970 MK |
183 | static u64 umem_size(struct xsk_umem_info *umem) |
184 | { | |
185 | return umem->num_frames * umem->frame_size; | |
186 | } | |
187 | ||
188 | static int xsk_configure_umem(struct ifobject *ifobj, struct xsk_umem_info *umem, void *buffer, | |
189 | u64 size) | |
facb7cb2 | 190 | { |
b267e5a4 CL |
191 | struct xsk_umem_config cfg = { |
192 | .fill_size = XSK_RING_PROD__DEFAULT_NUM_DESCS, | |
193 | .comp_size = XSK_RING_CONS__DEFAULT_NUM_DESCS, | |
c160d7af | 194 | .frame_size = umem->frame_size, |
83f4ae2f | 195 | .frame_headroom = umem->frame_headroom, |
b267e5a4 CL |
196 | .flags = XSK_UMEM__DEFAULT_FLAGS |
197 | }; | |
27e1ca25 | 198 | int ret; |
facb7cb2 | 199 | |
e4a195e2 TV |
200 | if (umem->fill_size) |
201 | cfg.fill_size = umem->fill_size; | |
202 | ||
203 | if (umem->comp_size) | |
204 | cfg.comp_size = umem->comp_size; | |
205 | ||
a4ba98dd MK |
206 | if (umem->unaligned_mode) |
207 | cfg.flags |= XDP_UMEM_UNALIGNED_CHUNK_FLAG; | |
208 | ||
27e1ca25 MF |
209 | ret = xsk_umem__create(&umem->umem, buffer, size, |
210 | &umem->fq, &umem->cq, &cfg); | |
facb7cb2 | 211 | if (ret) |
ed7b74dc | 212 | return ret; |
facb7cb2 | 213 | |
27e1ca25 | 214 | umem->buffer = buffer; |
d9f6d970 MK |
215 | if (ifobj->shared_umem && ifobj->rx_on) { |
216 | umem->base_addr = umem_size(umem); | |
217 | umem->next_buffer = umem_size(umem); | |
218 | } | |
219 | ||
ed7b74dc | 220 | return 0; |
facb7cb2 WJ |
221 | } |
222 | ||
d9f6d970 MK |
223 | static u64 umem_alloc_buffer(struct xsk_umem_info *umem) |
224 | { | |
225 | u64 addr; | |
226 | ||
227 | addr = umem->next_buffer; | |
228 | umem->next_buffer += umem->frame_size; | |
229 | if (umem->next_buffer >= umem->base_addr + umem_size(umem)) | |
230 | umem->next_buffer = umem->base_addr; | |
231 | ||
232 | return addr; | |
233 | } | |
234 | ||
235 | static void umem_reset_alloc(struct xsk_umem_info *umem) | |
236 | { | |
237 | umem->next_buffer = 0; | |
238 | } | |
239 | ||
f90062b5 MK |
240 | static void enable_busy_poll(struct xsk_socket_info *xsk) |
241 | { | |
242 | int sock_opt; | |
243 | ||
244 | sock_opt = 1; | |
245 | if (setsockopt(xsk_socket__fd(xsk->xsk), SOL_SOCKET, SO_PREFER_BUSY_POLL, | |
246 | (void *)&sock_opt, sizeof(sock_opt)) < 0) | |
247 | exit_with_error(errno); | |
248 | ||
249 | sock_opt = 20; | |
250 | if (setsockopt(xsk_socket__fd(xsk->xsk), SOL_SOCKET, SO_BUSY_POLL, | |
251 | (void *)&sock_opt, sizeof(sock_opt)) < 0) | |
252 | exit_with_error(errno); | |
253 | ||
c3bd0150 | 254 | sock_opt = xsk->batch_size; |
f90062b5 MK |
255 | if (setsockopt(xsk_socket__fd(xsk->xsk), SOL_SOCKET, SO_BUSY_POLL_BUDGET, |
256 | (void *)&sock_opt, sizeof(sock_opt)) < 0) | |
257 | exit_with_error(errno); | |
258 | } | |
259 | ||
a693ff3e MF |
260 | static int __xsk_configure_socket(struct xsk_socket_info *xsk, struct xsk_umem_info *umem, |
261 | struct ifobject *ifobject, bool shared) | |
facb7cb2 | 262 | { |
3b22523b | 263 | struct xsk_socket_config cfg = {}; |
facb7cb2 WJ |
264 | struct xsk_ring_cons *rxr; |
265 | struct xsk_ring_prod *txr; | |
facb7cb2 | 266 | |
ed7b74dc | 267 | xsk->umem = umem; |
4bf8ee65 | 268 | cfg.rx_size = xsk->rxqsize; |
facb7cb2 | 269 | cfg.tx_size = XSK_RING_PROD__DEFAULT_NUM_DESCS; |
af6731d1 | 270 | cfg.bind_flags = ifobject->bind_flags; |
3b22523b MK |
271 | if (shared) |
272 | cfg.bind_flags |= XDP_SHARED_UMEM; | |
8367eb95 | 273 | if (ifobject->mtu > MAX_ETH_PKT_SIZE) |
f540d44e | 274 | cfg.bind_flags |= XDP_USE_SG; |
e4a195e2 TV |
275 | if (umem->comp_size) |
276 | cfg.tx_size = umem->comp_size; | |
277 | if (umem->fill_size) | |
278 | cfg.rx_size = umem->fill_size; | |
facb7cb2 | 279 | |
1856c24d MK |
280 | txr = ifobject->tx_on ? &xsk->tx : NULL; |
281 | rxr = ifobject->rx_on ? &xsk->rx : NULL; | |
aa61d81f | 282 | return xsk_socket__create(&xsk->xsk, ifobject->ifindex, 0, umem->umem, rxr, txr, &cfg); |
facb7cb2 WJ |
283 | } |
284 | ||
fe2ad08e MF |
285 | static bool ifobj_zc_avail(struct ifobject *ifobject) |
286 | { | |
287 | size_t umem_sz = DEFAULT_UMEM_BUFFERS * XSK_UMEM__DEFAULT_FRAME_SIZE; | |
288 | int mmap_flags = MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE; | |
289 | struct xsk_socket_info *xsk; | |
290 | struct xsk_umem_info *umem; | |
291 | bool zc_avail = false; | |
292 | void *bufs; | |
293 | int ret; | |
294 | ||
295 | bufs = mmap(NULL, umem_sz, PROT_READ | PROT_WRITE, mmap_flags, -1, 0); | |
296 | if (bufs == MAP_FAILED) | |
297 | exit_with_error(errno); | |
298 | ||
299 | umem = calloc(1, sizeof(struct xsk_umem_info)); | |
300 | if (!umem) { | |
301 | munmap(bufs, umem_sz); | |
085dcccf | 302 | exit_with_error(ENOMEM); |
fe2ad08e MF |
303 | } |
304 | umem->frame_size = XSK_UMEM__DEFAULT_FRAME_SIZE; | |
d9f6d970 | 305 | ret = xsk_configure_umem(ifobject, umem, bufs, umem_sz); |
fe2ad08e MF |
306 | if (ret) |
307 | exit_with_error(-ret); | |
308 | ||
309 | xsk = calloc(1, sizeof(struct xsk_socket_info)); | |
310 | if (!xsk) | |
311 | goto out; | |
fe2ad08e MF |
312 | ifobject->bind_flags = XDP_USE_NEED_WAKEUP | XDP_ZEROCOPY; |
313 | ifobject->rx_on = true; | |
314 | xsk->rxqsize = XSK_RING_CONS__DEFAULT_NUM_DESCS; | |
315 | ret = __xsk_configure_socket(xsk, umem, ifobject, false); | |
316 | if (!ret) | |
317 | zc_avail = true; | |
318 | ||
319 | xsk_socket__delete(xsk->xsk); | |
320 | free(xsk); | |
321 | out: | |
322 | munmap(umem->buffer, umem_sz); | |
323 | xsk_umem__delete(umem->umem); | |
324 | free(umem); | |
325 | return zc_avail; | |
326 | } | |
327 | ||
d41905b3 MF |
328 | #define MAX_SKB_FRAGS_PATH "/proc/sys/net/core/max_skb_frags" |
329 | static unsigned int get_max_skb_frags(void) | |
330 | { | |
331 | unsigned int max_skb_frags = 0; | |
332 | FILE *file; | |
333 | ||
334 | file = fopen(MAX_SKB_FRAGS_PATH, "r"); | |
335 | if (!file) { | |
336 | ksft_print_msg("Error opening %s\n", MAX_SKB_FRAGS_PATH); | |
337 | return 0; | |
338 | } | |
339 | ||
340 | if (fscanf(file, "%u", &max_skb_frags) != 1) | |
341 | ksft_print_msg("Error reading %s\n", MAX_SKB_FRAGS_PATH); | |
342 | ||
343 | fclose(file); | |
344 | return max_skb_frags; | |
345 | } | |
346 | ||
facb7cb2 WJ |
347 | static struct option long_options[] = { |
348 | {"interface", required_argument, 0, 'i'}, | |
f90062b5 | 349 | {"busy-poll", no_argument, 0, 'b'}, |
ecde6061 | 350 | {"verbose", no_argument, 0, 'v'}, |
3956bc34 | 351 | {"mode", required_argument, 0, 'm'}, |
c53dab7d | 352 | {"list", no_argument, 0, 'l'}, |
146e3055 | 353 | {"test", required_argument, 0, 't'}, |
4a5f0ba5 | 354 | {"help", no_argument, 0, 'h'}, |
facb7cb2 WJ |
355 | {0, 0, 0, 0} |
356 | }; | |
357 | ||
146e3055 | 358 | static void print_usage(char **argv) |
facb7cb2 WJ |
359 | { |
360 | const char *str = | |
3956bc34 | 361 | " Usage: xskxceiver [OPTIONS]\n" |
25c0a305 MK |
362 | " Options:\n" |
363 | " -i, --interface Use interface\n" | |
f90062b5 | 364 | " -v, --verbose Verbose output\n" |
3956bc34 | 365 | " -b, --busy-poll Enable busy poll\n" |
c53dab7d | 366 | " -m, --mode Run only mode skb, drv, or zc\n" |
146e3055 | 367 | " -l, --list List all available tests\n" |
4a5f0ba5 MK |
368 | " -t, --test Run a specific test. Enter number from -l option.\n" |
369 | " -h, --help Display this help and exit\n"; | |
25c0a305 | 370 | |
146e3055 MK |
371 | ksft_print_msg(str, basename(argv[0])); |
372 | ksft_exit_xfail(); | |
facb7cb2 WJ |
373 | } |
374 | ||
ce74acaf | 375 | static bool validate_interface(struct ifobject *ifobj) |
facb7cb2 | 376 | { |
ce74acaf MK |
377 | if (!strcmp(ifobj->ifname, "")) |
378 | return false; | |
379 | return true; | |
facb7cb2 WJ |
380 | } |
381 | ||
af6731d1 MK |
382 | static void parse_command_line(struct ifobject *ifobj_tx, struct ifobject *ifobj_rx, int argc, |
383 | char **argv) | |
facb7cb2 | 384 | { |
ce74acaf MK |
385 | struct ifobject *ifobj; |
386 | u32 interface_nb = 0; | |
387 | int option_index, c; | |
facb7cb2 WJ |
388 | |
389 | opterr = 0; | |
390 | ||
391 | for (;;) { | |
146e3055 | 392 | c = getopt_long(argc, argv, "i:vbm:lt:", long_options, &option_index); |
facb7cb2 WJ |
393 | if (c == -1) |
394 | break; | |
395 | ||
396 | switch (c) { | |
397 | case 'i': | |
ce74acaf | 398 | if (interface_nb == 0) |
af6731d1 | 399 | ifobj = ifobj_tx; |
ce74acaf | 400 | else if (interface_nb == 1) |
af6731d1 | 401 | ifobj = ifobj_rx; |
ce74acaf | 402 | else |
facb7cb2 | 403 | break; |
facb7cb2 | 404 | |
64aef77d MK |
405 | memcpy(ifobj->ifname, optarg, |
406 | min_t(size_t, MAX_INTERFACE_NAME_CHARS, strlen(optarg))); | |
aa61d81f MK |
407 | |
408 | ifobj->ifindex = if_nametoindex(ifobj->ifname); | |
409 | if (!ifobj->ifindex) | |
410 | exit_with_error(errno); | |
411 | ||
ce74acaf | 412 | interface_nb++; |
facb7cb2 | 413 | break; |
ecde6061 | 414 | case 'v': |
33a6bef8 | 415 | opt_verbose = true; |
ecde6061 | 416 | break; |
f90062b5 MK |
417 | case 'b': |
418 | ifobj_tx->busy_poll = true; | |
419 | ifobj_rx->busy_poll = true; | |
420 | break; | |
3956bc34 | 421 | case 'm': |
146e3055 | 422 | if (!strncmp("skb", optarg, strlen(optarg))) |
3956bc34 | 423 | opt_mode = TEST_MODE_SKB; |
146e3055 | 424 | else if (!strncmp("drv", optarg, strlen(optarg))) |
3956bc34 | 425 | opt_mode = TEST_MODE_DRV; |
146e3055 | 426 | else if (!strncmp("zc", optarg, strlen(optarg))) |
3956bc34 | 427 | opt_mode = TEST_MODE_ZC; |
146e3055 MK |
428 | else |
429 | print_usage(argv); | |
3956bc34 | 430 | break; |
c53dab7d MK |
431 | case 'l': |
432 | opt_print_tests = true; | |
433 | break; | |
146e3055 MK |
434 | case 't': |
435 | errno = 0; | |
436 | opt_run_test = strtol(optarg, NULL, 0); | |
437 | if (errno) | |
438 | print_usage(argv); | |
439 | break; | |
4a5f0ba5 | 440 | case 'h': |
facb7cb2 | 441 | default: |
146e3055 | 442 | print_usage(argv); |
facb7cb2 WJ |
443 | } |
444 | } | |
ce74acaf | 445 | } |
facb7cb2 | 446 | |
776021e0 TV |
447 | static int set_ring_size(struct ifobject *ifobj) |
448 | { | |
449 | int ret; | |
450 | u32 ctr = 0; | |
451 | ||
452 | while (ctr++ < SOCK_RECONF_CTR) { | |
453 | ret = set_hw_ring_size(ifobj->ifname, &ifobj->ring); | |
454 | if (!ret) | |
455 | break; | |
456 | ||
457 | /* Retry if it fails */ | |
458 | if (ctr >= SOCK_RECONF_CTR || errno != EBUSY) | |
459 | return -errno; | |
460 | ||
461 | usleep(USLEEP_MAX); | |
462 | } | |
463 | ||
464 | return ret; | |
465 | } | |
466 | ||
467 | static int hw_ring_size_reset(struct ifobject *ifobj) | |
468 | { | |
469 | ifobj->ring.tx_pending = ifobj->set_ring.default_tx; | |
470 | ifobj->ring.rx_pending = ifobj->set_ring.default_rx; | |
471 | return set_ring_size(ifobj); | |
472 | } | |
473 | ||
ce74acaf MK |
474 | static void __test_spec_init(struct test_spec *test, struct ifobject *ifobj_tx, |
475 | struct ifobject *ifobj_rx) | |
476 | { | |
477 | u32 i, j; | |
478 | ||
479 | for (i = 0; i < MAX_INTERFACES; i++) { | |
480 | struct ifobject *ifobj = i ? ifobj_rx : ifobj_tx; | |
481 | ||
ce74acaf | 482 | ifobj->xsk = &ifobj->xsk_arr[0]; |
119d4b02 | 483 | ifobj->use_poll = false; |
27e934be MK |
484 | ifobj->use_fill_ring = true; |
485 | ifobj->release_rx = true; | |
76c57663 | 486 | ifobj->validation_func = NULL; |
9a321fd3 | 487 | ifobj->use_metadata = false; |
ce74acaf | 488 | |
1856c24d MK |
489 | if (i == 0) { |
490 | ifobj->rx_on = false; | |
491 | ifobj->tx_on = true; | |
492 | } else { | |
493 | ifobj->rx_on = true; | |
494 | ifobj->tx_on = false; | |
495 | } | |
ce74acaf | 496 | |
3b22523b MK |
497 | memset(ifobj->umem, 0, sizeof(*ifobj->umem)); |
498 | ifobj->umem->num_frames = DEFAULT_UMEM_BUFFERS; | |
499 | ifobj->umem->frame_size = XSK_UMEM__DEFAULT_FRAME_SIZE; | |
500 | ||
ce74acaf | 501 | for (j = 0; j < MAX_SOCKETS; j++) { |
ce74acaf | 502 | memset(&ifobj->xsk_arr[j], 0, sizeof(ifobj->xsk_arr[j])); |
4bf8ee65 | 503 | ifobj->xsk_arr[j].rxqsize = XSK_RING_CONS__DEFAULT_NUM_DESCS; |
c3bd0150 | 504 | ifobj->xsk_arr[j].batch_size = DEFAULT_BATCH_SIZE; |
8367eb95 TV |
505 | if (i == 0) |
506 | ifobj->xsk_arr[j].pkt_stream = test->tx_pkt_stream_default; | |
507 | else | |
508 | ifobj->xsk_arr[j].pkt_stream = test->rx_pkt_stream_default; | |
985fd214 TV |
509 | |
510 | memcpy(ifobj->xsk_arr[j].src_mac, g_mac, ETH_ALEN); | |
511 | memcpy(ifobj->xsk_arr[j].dst_mac, g_mac, ETH_ALEN); | |
512 | ifobj->xsk_arr[j].src_mac[5] += ((j * 2) + 0); | |
513 | ifobj->xsk_arr[j].dst_mac[5] += ((j * 2) + 1); | |
ce74acaf | 514 | } |
facb7cb2 | 515 | } |
ce74acaf | 516 | |
776021e0 TV |
517 | if (ifobj_tx->hw_ring_size_supp) |
518 | hw_ring_size_reset(ifobj_tx); | |
519 | ||
ce74acaf MK |
520 | test->ifobj_tx = ifobj_tx; |
521 | test->ifobj_rx = ifobj_rx; | |
55be575d MK |
522 | test->current_step = 0; |
523 | test->total_steps = 1; | |
85c6c957 | 524 | test->nb_sockets = 1; |
895b62ee | 525 | test->fail = false; |
776021e0 | 526 | test->set_ring = false; |
f540d44e | 527 | test->mtu = MAX_ETH_PKT_SIZE; |
7d8319a7 MK |
528 | test->xdp_prog_rx = ifobj_rx->xdp_progs->progs.xsk_def_prog; |
529 | test->xskmap_rx = ifobj_rx->xdp_progs->maps.xsk; | |
530 | test->xdp_prog_tx = ifobj_tx->xdp_progs->progs.xsk_def_prog; | |
531 | test->xskmap_tx = ifobj_tx->xdp_progs->maps.xsk; | |
ce74acaf MK |
532 | } |
533 | ||
534 | static void test_spec_init(struct test_spec *test, struct ifobject *ifobj_tx, | |
f20fbcd0 MK |
535 | struct ifobject *ifobj_rx, enum test_mode mode, |
536 | const struct test_spec *test_to_run) | |
ce74acaf | 537 | { |
1adef064 MF |
538 | struct pkt_stream *tx_pkt_stream; |
539 | struct pkt_stream *rx_pkt_stream; | |
af6731d1 MK |
540 | u32 i; |
541 | ||
1adef064 MF |
542 | tx_pkt_stream = test->tx_pkt_stream_default; |
543 | rx_pkt_stream = test->rx_pkt_stream_default; | |
ce74acaf | 544 | memset(test, 0, sizeof(*test)); |
1adef064 MF |
545 | test->tx_pkt_stream_default = tx_pkt_stream; |
546 | test->rx_pkt_stream_default = rx_pkt_stream; | |
af6731d1 MK |
547 | |
548 | for (i = 0; i < MAX_INTERFACES; i++) { | |
549 | struct ifobject *ifobj = i ? ifobj_rx : ifobj_tx; | |
550 | ||
fe2ad08e MF |
551 | ifobj->bind_flags = XDP_USE_NEED_WAKEUP; |
552 | if (mode == TEST_MODE_ZC) | |
553 | ifobj->bind_flags |= XDP_ZEROCOPY; | |
554 | else | |
555 | ifobj->bind_flags |= XDP_COPY; | |
af6731d1 MK |
556 | } |
557 | ||
f20fbcd0 MK |
558 | strncpy(test->name, test_to_run->name, MAX_TEST_NAME_SIZE); |
559 | test->test_func = test_to_run->test_func; | |
fe2ad08e | 560 | test->mode = mode; |
ce74acaf MK |
561 | __test_spec_init(test, ifobj_tx, ifobj_rx); |
562 | } | |
563 | ||
564 | static void test_spec_reset(struct test_spec *test) | |
565 | { | |
566 | __test_spec_init(test, test->ifobj_tx, test->ifobj_rx); | |
facb7cb2 WJ |
567 | } |
568 | ||
7d8319a7 MK |
569 | static void test_spec_set_xdp_prog(struct test_spec *test, struct bpf_program *xdp_prog_rx, |
570 | struct bpf_program *xdp_prog_tx, struct bpf_map *xskmap_rx, | |
571 | struct bpf_map *xskmap_tx) | |
572 | { | |
573 | test->xdp_prog_rx = xdp_prog_rx; | |
574 | test->xdp_prog_tx = xdp_prog_tx; | |
575 | test->xskmap_rx = xskmap_rx; | |
576 | test->xskmap_tx = xskmap_tx; | |
577 | } | |
578 | ||
f540d44e MK |
579 | static int test_spec_set_mtu(struct test_spec *test, int mtu) |
580 | { | |
581 | int err; | |
582 | ||
583 | if (test->ifobj_rx->mtu != mtu) { | |
584 | err = xsk_set_mtu(test->ifobj_rx->ifindex, mtu); | |
585 | if (err) | |
586 | return err; | |
587 | test->ifobj_rx->mtu = mtu; | |
588 | } | |
589 | if (test->ifobj_tx->mtu != mtu) { | |
590 | err = xsk_set_mtu(test->ifobj_tx->ifindex, mtu); | |
591 | if (err) | |
592 | return err; | |
593 | test->ifobj_tx->mtu = mtu; | |
594 | } | |
595 | ||
596 | return 0; | |
597 | } | |
598 | ||
5b132056 MK |
599 | static void pkt_stream_reset(struct pkt_stream *pkt_stream) |
600 | { | |
8913e653 | 601 | if (pkt_stream) { |
69fc03d2 | 602 | pkt_stream->current_pkt_nb = 0; |
8913e653 TV |
603 | pkt_stream->nb_rx_pkts = 0; |
604 | } | |
5b132056 MK |
605 | } |
606 | ||
69fc03d2 | 607 | static struct pkt *pkt_stream_get_next_tx_pkt(struct pkt_stream *pkt_stream) |
960b6e01 | 608 | { |
69fc03d2 | 609 | if (pkt_stream->current_pkt_nb >= pkt_stream->nb_pkts) |
29f128b3 MK |
610 | return NULL; |
611 | ||
69fc03d2 | 612 | return &pkt_stream->pkts[pkt_stream->current_pkt_nb++]; |
29f128b3 MK |
613 | } |
614 | ||
27e934be | 615 | static struct pkt *pkt_stream_get_next_rx_pkt(struct pkt_stream *pkt_stream, u32 *pkts_sent) |
0d1b7f3a | 616 | { |
69fc03d2 | 617 | while (pkt_stream->current_pkt_nb < pkt_stream->nb_pkts) { |
27e934be | 618 | (*pkts_sent)++; |
69fc03d2 MK |
619 | if (pkt_stream->pkts[pkt_stream->current_pkt_nb].valid) |
620 | return &pkt_stream->pkts[pkt_stream->current_pkt_nb++]; | |
621 | pkt_stream->current_pkt_nb++; | |
0d1b7f3a MK |
622 | } |
623 | return NULL; | |
624 | } | |
625 | ||
605091c5 MK |
626 | static void pkt_stream_delete(struct pkt_stream *pkt_stream) |
627 | { | |
628 | free(pkt_stream->pkts); | |
629 | free(pkt_stream); | |
630 | } | |
631 | ||
632 | static void pkt_stream_restore_default(struct test_spec *test) | |
633 | { | |
8367eb95 TV |
634 | struct pkt_stream *tx_pkt_stream = test->ifobj_tx->xsk->pkt_stream; |
635 | struct pkt_stream *rx_pkt_stream = test->ifobj_rx->xsk->pkt_stream; | |
27e934be | 636 | |
1adef064 | 637 | if (tx_pkt_stream != test->tx_pkt_stream_default) { |
8367eb95 TV |
638 | pkt_stream_delete(test->ifobj_tx->xsk->pkt_stream); |
639 | test->ifobj_tx->xsk->pkt_stream = test->tx_pkt_stream_default; | |
6ce67b51 | 640 | } |
27e934be | 641 | |
1adef064 | 642 | if (rx_pkt_stream != test->rx_pkt_stream_default) { |
8367eb95 TV |
643 | pkt_stream_delete(test->ifobj_rx->xsk->pkt_stream); |
644 | test->ifobj_rx->xsk->pkt_stream = test->rx_pkt_stream_default; | |
1adef064 | 645 | } |
605091c5 MK |
646 | } |
647 | ||
0d1b7f3a | 648 | static struct pkt_stream *__pkt_stream_alloc(u32 nb_pkts) |
29f128b3 MK |
649 | { |
650 | struct pkt_stream *pkt_stream; | |
29f128b3 | 651 | |
a4ba98dd | 652 | pkt_stream = calloc(1, sizeof(*pkt_stream)); |
29f128b3 | 653 | if (!pkt_stream) |
0d1b7f3a | 654 | return NULL; |
29f128b3 MK |
655 | |
656 | pkt_stream->pkts = calloc(nb_pkts, sizeof(*pkt_stream->pkts)); | |
0d1b7f3a MK |
657 | if (!pkt_stream->pkts) { |
658 | free(pkt_stream); | |
659 | return NULL; | |
660 | } | |
661 | ||
662 | pkt_stream->nb_pkts = nb_pkts; | |
663 | return pkt_stream; | |
664 | } | |
665 | ||
69760449 | 666 | static bool pkt_continues(u32 options) |
17f1034d | 667 | { |
69760449 | 668 | return options & XDP_PKT_CONTD; |
17f1034d MK |
669 | } |
670 | ||
86e41755 MK |
671 | static u32 ceil_u32(u32 a, u32 b) |
672 | { | |
673 | return (a + b - 1) / b; | |
674 | } | |
675 | ||
69760449 | 676 | static u32 pkt_nb_frags(u32 frame_size, struct pkt_stream *pkt_stream, struct pkt *pkt) |
86e41755 | 677 | { |
69760449 MK |
678 | u32 nb_frags = 1, next_frag; |
679 | ||
680 | if (!pkt) | |
86e41755 | 681 | return 1; |
69760449 MK |
682 | |
683 | if (!pkt_stream->verbatim) { | |
684 | if (!pkt->valid || !pkt->len) | |
685 | return 1; | |
686 | return ceil_u32(pkt->len, frame_size); | |
687 | } | |
688 | ||
689 | /* Search for the end of the packet in verbatim mode */ | |
690 | if (!pkt_continues(pkt->options)) | |
691 | return nb_frags; | |
692 | ||
693 | next_frag = pkt_stream->current_pkt_nb; | |
694 | pkt++; | |
695 | while (next_frag++ < pkt_stream->nb_pkts) { | |
696 | nb_frags++; | |
697 | if (!pkt_continues(pkt->options) || !pkt->valid) | |
698 | break; | |
699 | pkt++; | |
700 | } | |
701 | return nb_frags; | |
86e41755 MK |
702 | } |
703 | ||
2e1d6a04 TV |
704 | static bool set_pkt_valid(int offset, u32 len) |
705 | { | |
706 | return len <= MAX_ETH_JUMBO_SIZE; | |
707 | } | |
708 | ||
8913e653 | 709 | static void pkt_set(struct pkt_stream *pkt_stream, struct pkt *pkt, int offset, u32 len) |
27e934be | 710 | { |
d9f6d970 | 711 | pkt->offset = offset; |
27e934be | 712 | pkt->len = len; |
2e1d6a04 TV |
713 | pkt->valid = set_pkt_valid(offset, len); |
714 | } | |
715 | ||
716 | static void pkt_stream_pkt_set(struct pkt_stream *pkt_stream, struct pkt *pkt, int offset, u32 len) | |
717 | { | |
718 | bool prev_pkt_valid = pkt->valid; | |
719 | ||
720 | pkt_set(pkt_stream, pkt, offset, len); | |
721 | pkt_stream->nb_valid_entries += pkt->valid - prev_pkt_valid; | |
27e934be MK |
722 | } |
723 | ||
7cd6df4f MK |
724 | static u32 pkt_get_buffer_len(struct xsk_umem_info *umem, u32 len) |
725 | { | |
726 | return ceil_u32(len, umem->frame_size) * umem->frame_size; | |
727 | } | |
728 | ||
6d198a89 | 729 | static struct pkt_stream *__pkt_stream_generate(u32 nb_pkts, u32 pkt_len, u32 nb_start, u32 nb_off) |
0d1b7f3a MK |
730 | { |
731 | struct pkt_stream *pkt_stream; | |
732 | u32 i; | |
733 | ||
734 | pkt_stream = __pkt_stream_alloc(nb_pkts); | |
735 | if (!pkt_stream) | |
29f128b3 MK |
736 | exit_with_error(ENOMEM); |
737 | ||
7cd6df4f MK |
738 | pkt_stream->nb_pkts = nb_pkts; |
739 | pkt_stream->max_pkt_len = pkt_len; | |
29f128b3 | 740 | for (i = 0; i < nb_pkts; i++) { |
2f6eae0d MK |
741 | struct pkt *pkt = &pkt_stream->pkts[i]; |
742 | ||
2e1d6a04 | 743 | pkt_stream_pkt_set(pkt_stream, pkt, 0, pkt_len); |
6d198a89 | 744 | pkt->pkt_nb = nb_start + i * nb_off; |
29f128b3 MK |
745 | } |
746 | ||
747 | return pkt_stream; | |
748 | } | |
749 | ||
6d198a89 TV |
750 | static struct pkt_stream *pkt_stream_generate(u32 nb_pkts, u32 pkt_len) |
751 | { | |
752 | return __pkt_stream_generate(nb_pkts, pkt_len, 0, 1); | |
753 | } | |
754 | ||
46e43786 | 755 | static struct pkt_stream *pkt_stream_clone(struct pkt_stream *pkt_stream) |
a4ba98dd | 756 | { |
46e43786 | 757 | return pkt_stream_generate(pkt_stream->nb_pkts, pkt_stream->pkts[0].len); |
a4ba98dd MK |
758 | } |
759 | ||
605091c5 MK |
760 | static void pkt_stream_replace(struct test_spec *test, u32 nb_pkts, u32 pkt_len) |
761 | { | |
762 | struct pkt_stream *pkt_stream; | |
763 | ||
46e43786 | 764 | pkt_stream = pkt_stream_generate(nb_pkts, pkt_len); |
8367eb95 | 765 | test->ifobj_tx->xsk->pkt_stream = pkt_stream; |
46e43786 | 766 | pkt_stream = pkt_stream_generate(nb_pkts, pkt_len); |
8367eb95 | 767 | test->ifobj_rx->xsk->pkt_stream = pkt_stream; |
605091c5 MK |
768 | } |
769 | ||
1adef064 MF |
770 | static void __pkt_stream_replace_half(struct ifobject *ifobj, u32 pkt_len, |
771 | int offset) | |
a4ba98dd | 772 | { |
a4ba98dd MK |
773 | struct pkt_stream *pkt_stream; |
774 | u32 i; | |
775 | ||
46e43786 | 776 | pkt_stream = pkt_stream_clone(ifobj->xsk->pkt_stream); |
8367eb95 | 777 | for (i = 1; i < ifobj->xsk->pkt_stream->nb_pkts; i += 2) |
2e1d6a04 | 778 | pkt_stream_pkt_set(pkt_stream, &pkt_stream->pkts[i], offset, pkt_len); |
a4ba98dd | 779 | |
8367eb95 | 780 | ifobj->xsk->pkt_stream = pkt_stream; |
1adef064 MF |
781 | } |
782 | ||
783 | static void pkt_stream_replace_half(struct test_spec *test, u32 pkt_len, int offset) | |
784 | { | |
785 | __pkt_stream_replace_half(test->ifobj_tx, pkt_len, offset); | |
786 | __pkt_stream_replace_half(test->ifobj_rx, pkt_len, offset); | |
a4ba98dd MK |
787 | } |
788 | ||
27e934be MK |
789 | static void pkt_stream_receive_half(struct test_spec *test) |
790 | { | |
8367eb95 | 791 | struct pkt_stream *pkt_stream = test->ifobj_tx->xsk->pkt_stream; |
27e934be MK |
792 | u32 i; |
793 | ||
46e43786 | 794 | test->ifobj_rx->xsk->pkt_stream = pkt_stream_generate(pkt_stream->nb_pkts, |
8367eb95 TV |
795 | pkt_stream->pkts[0].len); |
796 | pkt_stream = test->ifobj_rx->xsk->pkt_stream; | |
27e934be MK |
797 | for (i = 1; i < pkt_stream->nb_pkts; i += 2) |
798 | pkt_stream->pkts[i].valid = false; | |
8913e653 TV |
799 | |
800 | pkt_stream->nb_valid_entries /= 2; | |
27e934be MK |
801 | } |
802 | ||
6d198a89 TV |
803 | static void pkt_stream_even_odd_sequence(struct test_spec *test) |
804 | { | |
805 | struct pkt_stream *pkt_stream; | |
806 | u32 i; | |
807 | ||
808 | for (i = 0; i < test->nb_sockets; i++) { | |
809 | pkt_stream = test->ifobj_tx->xsk_arr[i].pkt_stream; | |
810 | pkt_stream = __pkt_stream_generate(pkt_stream->nb_pkts / 2, | |
811 | pkt_stream->pkts[0].len, i, 2); | |
812 | test->ifobj_tx->xsk_arr[i].pkt_stream = pkt_stream; | |
813 | ||
814 | pkt_stream = test->ifobj_rx->xsk_arr[i].pkt_stream; | |
815 | pkt_stream = __pkt_stream_generate(pkt_stream->nb_pkts / 2, | |
816 | pkt_stream->pkts[0].len, i, 2); | |
817 | test->ifobj_rx->xsk_arr[i].pkt_stream = pkt_stream; | |
818 | } | |
819 | } | |
820 | ||
d9f6d970 MK |
821 | static u64 pkt_get_addr(struct pkt *pkt, struct xsk_umem_info *umem) |
822 | { | |
823 | if (!pkt->valid) | |
824 | return pkt->offset; | |
825 | return pkt->offset + umem_alloc_buffer(umem); | |
826 | } | |
827 | ||
17f1034d MK |
828 | static void pkt_stream_cancel(struct pkt_stream *pkt_stream) |
829 | { | |
830 | pkt_stream->current_pkt_nb--; | |
831 | } | |
832 | ||
985fd214 TV |
833 | static void pkt_generate(struct xsk_socket_info *xsk, struct xsk_umem_info *umem, u64 addr, u32 len, |
834 | u32 pkt_nb, u32 bytes_written) | |
29f128b3 | 835 | { |
985fd214 | 836 | void *data = xsk_umem__get_data(umem->buffer, addr); |
29f128b3 | 837 | |
2f6eae0d | 838 | if (len < MIN_PKT_SIZE) |
d9f6d970 | 839 | return; |
29f128b3 | 840 | |
2f6eae0d | 841 | if (!bytes_written) { |
985fd214 | 842 | gen_eth_hdr(xsk, data); |
2f6eae0d MK |
843 | |
844 | len -= PKT_HDR_SIZE; | |
845 | data += PKT_HDR_SIZE; | |
846 | } else { | |
847 | bytes_written -= PKT_HDR_SIZE; | |
848 | } | |
960b6e01 | 849 | |
2f6eae0d | 850 | write_payload(data, pkt_nb, bytes_written, len); |
960b6e01 MK |
851 | } |
852 | ||
69760449 MK |
853 | static struct pkt_stream *__pkt_stream_generate_custom(struct ifobject *ifobj, struct pkt *frames, |
854 | u32 nb_frames, bool verbatim) | |
0d1b7f3a | 855 | { |
69760449 | 856 | u32 i, len = 0, pkt_nb = 0, payload = 0; |
0d1b7f3a | 857 | struct pkt_stream *pkt_stream; |
0d1b7f3a | 858 | |
69760449 | 859 | pkt_stream = __pkt_stream_alloc(nb_frames); |
0d1b7f3a MK |
860 | if (!pkt_stream) |
861 | exit_with_error(ENOMEM); | |
862 | ||
69760449 MK |
863 | for (i = 0; i < nb_frames; i++) { |
864 | struct pkt *pkt = &pkt_stream->pkts[pkt_nb]; | |
865 | struct pkt *frame = &frames[i]; | |
7cd6df4f | 866 | |
69760449 MK |
867 | pkt->offset = frame->offset; |
868 | if (verbatim) { | |
869 | *pkt = *frame; | |
870 | pkt->pkt_nb = payload; | |
871 | if (!frame->valid || !pkt_continues(frame->options)) | |
872 | payload++; | |
873 | } else { | |
874 | if (frame->valid) | |
875 | len += frame->len; | |
876 | if (frame->valid && pkt_continues(frame->options)) | |
877 | continue; | |
878 | ||
879 | pkt->pkt_nb = pkt_nb; | |
880 | pkt->len = len; | |
881 | pkt->valid = frame->valid; | |
882 | pkt->options = 0; | |
883 | ||
884 | len = 0; | |
885 | } | |
886 | ||
2d2712ca MK |
887 | print_verbose("offset: %d len: %u valid: %u options: %u pkt_nb: %u\n", |
888 | pkt->offset, pkt->len, pkt->valid, pkt->options, pkt->pkt_nb); | |
889 | ||
69760449 | 890 | if (pkt->valid && pkt->len > pkt_stream->max_pkt_len) |
7cd6df4f | 891 | pkt_stream->max_pkt_len = pkt->len; |
8913e653 TV |
892 | |
893 | if (pkt->valid) | |
894 | pkt_stream->nb_valid_entries++; | |
895 | ||
69760449 | 896 | pkt_nb++; |
0d1b7f3a | 897 | } |
1adef064 | 898 | |
69760449 MK |
899 | pkt_stream->nb_pkts = pkt_nb; |
900 | pkt_stream->verbatim = verbatim; | |
901 | return pkt_stream; | |
1adef064 MF |
902 | } |
903 | ||
904 | static void pkt_stream_generate_custom(struct test_spec *test, struct pkt *pkts, u32 nb_pkts) | |
905 | { | |
69760449 MK |
906 | struct pkt_stream *pkt_stream; |
907 | ||
908 | pkt_stream = __pkt_stream_generate_custom(test->ifobj_tx, pkts, nb_pkts, true); | |
8367eb95 | 909 | test->ifobj_tx->xsk->pkt_stream = pkt_stream; |
69760449 MK |
910 | |
911 | pkt_stream = __pkt_stream_generate_custom(test->ifobj_rx, pkts, nb_pkts, false); | |
8367eb95 | 912 | test->ifobj_rx->xsk->pkt_stream = pkt_stream; |
0d1b7f3a MK |
913 | } |
914 | ||
feb973a9 MK |
915 | static void pkt_print_data(u32 *data, u32 cnt) |
916 | { | |
917 | u32 i; | |
918 | ||
919 | for (i = 0; i < cnt; i++) { | |
920 | u32 seqnum, pkt_nb; | |
921 | ||
922 | seqnum = ntohl(*data) & 0xffff; | |
923 | pkt_nb = ntohl(*data) >> 16; | |
7c3fcf08 | 924 | ksft_print_msg("%u:%u ", pkt_nb, seqnum); |
feb973a9 MK |
925 | data++; |
926 | } | |
927 | } | |
928 | ||
2f6eae0d | 929 | static void pkt_dump(void *pkt, u32 len, bool eth_header) |
0d41f59f | 930 | { |
df82d2e8 | 931 | struct ethhdr *ethhdr = pkt; |
2f6eae0d MK |
932 | u32 i, *data; |
933 | ||
934 | if (eth_header) { | |
935 | /*extract L2 frame */ | |
7c3fcf08 | 936 | ksft_print_msg("DEBUG>> L2: dst mac: "); |
2f6eae0d | 937 | for (i = 0; i < ETH_ALEN; i++) |
7c3fcf08 | 938 | ksft_print_msg("%02X", ethhdr->h_dest[i]); |
0d41f59f | 939 | |
7c3fcf08 | 940 | ksft_print_msg("\nDEBUG>> L2: src mac: "); |
2f6eae0d | 941 | for (i = 0; i < ETH_ALEN; i++) |
7c3fcf08 | 942 | ksft_print_msg("%02X", ethhdr->h_source[i]); |
0d41f59f | 943 | |
2f6eae0d MK |
944 | data = pkt + PKT_HDR_SIZE; |
945 | } else { | |
946 | data = pkt; | |
947 | } | |
0d41f59f | 948 | |
0d41f59f | 949 | /*extract L5 frame */ |
7c3fcf08 | 950 | ksft_print_msg("\nDEBUG>> L5: seqnum: "); |
2f6eae0d | 951 | pkt_print_data(data, PKT_DUMP_NB_TO_PRINT); |
7c3fcf08 | 952 | ksft_print_msg("...."); |
feb973a9 | 953 | if (len > PKT_DUMP_NB_TO_PRINT * sizeof(u32)) { |
7c3fcf08 | 954 | ksft_print_msg("\n.... "); |
2f6eae0d | 955 | pkt_print_data(data + len / sizeof(u32) - PKT_DUMP_NB_TO_PRINT, |
feb973a9 MK |
956 | PKT_DUMP_NB_TO_PRINT); |
957 | } | |
7c3fcf08 | 958 | ksft_print_msg("\n---------------------------------------\n"); |
0d41f59f MK |
959 | } |
960 | ||
d9f6d970 | 961 | static bool is_offset_correct(struct xsk_umem_info *umem, struct pkt *pkt, u64 addr) |
e34087fc MK |
962 | { |
963 | u32 headroom = umem->unaligned_mode ? 0 : umem->frame_headroom; | |
d9f6d970 MK |
964 | u32 offset = addr % umem->frame_size, expected_offset; |
965 | int pkt_offset = pkt->valid ? pkt->offset : 0; | |
e34087fc | 966 | |
d9f6d970 MK |
967 | if (!umem->unaligned_mode) |
968 | pkt_offset = 0; | |
e34087fc | 969 | |
d9f6d970 | 970 | expected_offset = (pkt_offset + headroom + XDP_PACKET_HEADROOM) % umem->frame_size; |
e34087fc MK |
971 | |
972 | if (offset == expected_offset) | |
973 | return true; | |
974 | ||
895b62ee | 975 | ksft_print_msg("[%s] expected [%u], got [%u]\n", __func__, expected_offset, offset); |
e34087fc MK |
976 | return false; |
977 | } | |
978 | ||
9a321fd3 TV |
979 | static bool is_metadata_correct(struct pkt *pkt, void *buffer, u64 addr) |
980 | { | |
981 | void *data = xsk_umem__get_data(buffer, addr); | |
982 | struct xdp_info *meta = data - sizeof(struct xdp_info); | |
983 | ||
feb973a9 | 984 | if (meta->count != pkt->pkt_nb) { |
fe69a1b1 AR |
985 | ksft_print_msg("[%s] expected meta_count [%d], got meta_count [%llu]\n", |
986 | __func__, pkt->pkt_nb, | |
987 | (unsigned long long)meta->count); | |
9a321fd3 TV |
988 | return false; |
989 | } | |
990 | ||
991 | return true; | |
992 | } | |
993 | ||
17f1034d MK |
994 | static bool is_frag_valid(struct xsk_umem_info *umem, u64 addr, u32 len, u32 expected_pkt_nb, |
995 | u32 bytes_processed) | |
0d41f59f | 996 | { |
17f1034d MK |
997 | u32 seqnum, pkt_nb, *pkt_data, words_to_end, expected_seqnum; |
998 | void *data = xsk_umem__get_data(umem->buffer, addr); | |
0d41f59f | 999 | |
17f1034d MK |
1000 | addr -= umem->base_addr; |
1001 | ||
1002 | if (addr >= umem->num_frames * umem->frame_size || | |
1003 | addr + len > umem->num_frames * umem->frame_size) { | |
fe69a1b1 AR |
1004 | ksft_print_msg("Frag invalid addr: %llx len: %u\n", |
1005 | (unsigned long long)addr, len); | |
17f1034d MK |
1006 | return false; |
1007 | } | |
1008 | if (!umem->unaligned_mode && addr % umem->frame_size + len > umem->frame_size) { | |
fe69a1b1 AR |
1009 | ksft_print_msg("Frag crosses frame boundary addr: %llx len: %u\n", |
1010 | (unsigned long long)addr, len); | |
17f1034d | 1011 | return false; |
29f128b3 MK |
1012 | } |
1013 | ||
17f1034d MK |
1014 | pkt_data = data; |
1015 | if (!bytes_processed) { | |
1016 | pkt_data += PKT_HDR_SIZE / sizeof(*pkt_data); | |
1017 | len -= PKT_HDR_SIZE; | |
1018 | } else { | |
1019 | bytes_processed -= PKT_HDR_SIZE; | |
0d1b7f3a MK |
1020 | } |
1021 | ||
17f1034d MK |
1022 | expected_seqnum = bytes_processed / sizeof(*pkt_data); |
1023 | seqnum = ntohl(*pkt_data) & 0xffff; | |
1024 | pkt_nb = ntohl(*pkt_data) >> 16; | |
1025 | ||
1026 | if (expected_pkt_nb != pkt_nb) { | |
1027 | ksft_print_msg("[%s] expected pkt_nb [%u], got pkt_nb [%u]\n", | |
1028 | __func__, expected_pkt_nb, pkt_nb); | |
1029 | goto error; | |
1030 | } | |
1031 | if (expected_seqnum != seqnum) { | |
1032 | ksft_print_msg("[%s] expected seqnum at start [%u], got seqnum [%u]\n", | |
1033 | __func__, expected_seqnum, seqnum); | |
7a8a6762 | 1034 | goto error; |
0d1b7f3a MK |
1035 | } |
1036 | ||
17f1034d MK |
1037 | words_to_end = len / sizeof(*pkt_data) - 1; |
1038 | pkt_data += words_to_end; | |
1039 | seqnum = ntohl(*pkt_data) & 0xffff; | |
1040 | expected_seqnum += words_to_end; | |
1041 | if (expected_seqnum != seqnum) { | |
1042 | ksft_print_msg("[%s] expected seqnum at end [%u], got seqnum [%u]\n", | |
1043 | __func__, expected_seqnum, seqnum); | |
7a8a6762 | 1044 | goto error; |
0d41f59f | 1045 | } |
29f128b3 MK |
1046 | |
1047 | return true; | |
7a8a6762 MK |
1048 | |
1049 | error: | |
17f1034d | 1050 | pkt_dump(data, len, !bytes_processed); |
7a8a6762 | 1051 | return false; |
0d41f59f MK |
1052 | } |
1053 | ||
17f1034d MK |
1054 | static bool is_pkt_valid(struct pkt *pkt, void *buffer, u64 addr, u32 len) |
1055 | { | |
17f1034d MK |
1056 | if (pkt->len != len) { |
1057 | ksft_print_msg("[%s] expected packet length [%d], got length [%d]\n", | |
1058 | __func__, pkt->len, len); | |
1059 | pkt_dump(xsk_umem__get_data(buffer, addr), len, true); | |
1060 | return false; | |
1061 | } | |
1062 | ||
1063 | return true; | |
1064 | } | |
1065 | ||
5fc494d5 | 1066 | static int kick_tx(struct xsk_socket_info *xsk) |
facb7cb2 WJ |
1067 | { |
1068 | int ret; | |
1069 | ||
1070 | ret = sendto(xsk_socket__fd(xsk->xsk), NULL, 0, MSG_DONTWAIT, NULL, 0); | |
f90062b5 | 1071 | if (ret >= 0) |
5fc494d5 | 1072 | return TEST_PASS; |
f90062b5 MK |
1073 | if (errno == ENOBUFS || errno == EAGAIN || errno == EBUSY || errno == ENETDOWN) { |
1074 | usleep(100); | |
5fc494d5 | 1075 | return TEST_PASS; |
f90062b5 | 1076 | } |
5fc494d5 | 1077 | return TEST_FAILURE; |
facb7cb2 WJ |
1078 | } |
1079 | ||
5fc494d5 | 1080 | static int kick_rx(struct xsk_socket_info *xsk) |
f90062b5 MK |
1081 | { |
1082 | int ret; | |
1083 | ||
1084 | ret = recvfrom(xsk_socket__fd(xsk->xsk), NULL, 0, MSG_DONTWAIT, NULL, NULL); | |
1085 | if (ret < 0) | |
5fc494d5 MK |
1086 | return TEST_FAILURE; |
1087 | ||
1088 | return TEST_PASS; | |
f90062b5 MK |
1089 | } |
1090 | ||
895b62ee | 1091 | static int complete_pkts(struct xsk_socket_info *xsk, int batch_size) |
facb7cb2 WJ |
1092 | { |
1093 | unsigned int rcvd; | |
1094 | u32 idx; | |
5fc494d5 | 1095 | int ret; |
facb7cb2 | 1096 | |
5fc494d5 MK |
1097 | if (xsk_ring_prod__needs_wakeup(&xsk->tx)) { |
1098 | ret = kick_tx(xsk); | |
1099 | if (ret) | |
1100 | return TEST_FAILURE; | |
1101 | } | |
facb7cb2 WJ |
1102 | |
1103 | rcvd = xsk_ring_cons__peek(&xsk->umem->cq, batch_size, &idx); | |
1104 | if (rcvd) { | |
0d1b7f3a MK |
1105 | if (rcvd > xsk->outstanding_tx) { |
1106 | u64 addr = *xsk_ring_cons__comp_addr(&xsk->umem->cq, idx + rcvd - 1); | |
1107 | ||
895b62ee | 1108 | ksft_print_msg("[%s] Too many packets completed\n", __func__); |
fe69a1b1 AR |
1109 | ksft_print_msg("Last completion address: %llx\n", |
1110 | (unsigned long long)addr); | |
895b62ee | 1111 | return TEST_FAILURE; |
0d1b7f3a MK |
1112 | } |
1113 | ||
facb7cb2 WJ |
1114 | xsk_ring_cons__release(&xsk->umem->cq, rcvd); |
1115 | xsk->outstanding_tx -= rcvd; | |
facb7cb2 | 1116 | } |
895b62ee MK |
1117 | |
1118 | return TEST_PASS; | |
facb7cb2 WJ |
1119 | } |
1120 | ||
8913e653 | 1121 | static int __receive_pkts(struct test_spec *test, struct xsk_socket_info *xsk) |
facb7cb2 | 1122 | { |
8913e653 | 1123 | u32 frags_processed = 0, nb_frags = 0, pkt_len = 0; |
17f1034d | 1124 | u32 idx_rx = 0, idx_fq = 0, rcvd, pkts_sent = 0; |
8913e653 | 1125 | struct pkt_stream *pkt_stream = xsk->pkt_stream; |
3143d10b | 1126 | struct ifobject *ifobj = test->ifobj_rx; |
e34087fc | 1127 | struct xsk_umem_info *umem = xsk->umem; |
8913e653 | 1128 | struct pollfd fds = { }; |
27e934be | 1129 | struct pkt *pkt; |
925a0157 | 1130 | u64 first_addr = 0; |
facb7cb2 WJ |
1131 | int ret; |
1132 | ||
8913e653 TV |
1133 | fds.fd = xsk_socket__fd(xsk->xsk); |
1134 | fds.events = POLLIN; | |
17f1034d | 1135 | |
8913e653 TV |
1136 | ret = kick_rx(xsk); |
1137 | if (ret) | |
1138 | return TEST_FAILURE; | |
db1bd7a9 | 1139 | |
8913e653 TV |
1140 | if (ifobj->use_poll) { |
1141 | ret = poll(&fds, 1, POLL_TMOUT); | |
1142 | if (ret < 0) | |
5fc494d5 MK |
1143 | return TEST_FAILURE; |
1144 | ||
8913e653 TV |
1145 | if (!ret) { |
1146 | if (!is_umem_valid(test->ifobj_tx)) | |
1147 | return TEST_PASS; | |
3143d10b | 1148 | |
8913e653 TV |
1149 | ksft_print_msg("ERROR: [%s] Poll timed out\n", __func__); |
1150 | return TEST_CONTINUE; | |
facb7cb2 | 1151 | } |
facb7cb2 | 1152 | |
8913e653 TV |
1153 | if (!(fds.revents & POLLIN)) |
1154 | return TEST_CONTINUE; | |
1155 | } | |
3143d10b | 1156 | |
c3bd0150 | 1157 | rcvd = xsk_ring_cons__peek(&xsk->rx, xsk->batch_size, &idx_rx); |
8913e653 TV |
1158 | if (!rcvd) |
1159 | return TEST_CONTINUE; | |
1160 | ||
1161 | if (ifobj->use_fill_ring) { | |
1162 | ret = xsk_ring_prod__reserve(&umem->fq, rcvd, &idx_fq); | |
1163 | while (ret != rcvd) { | |
1164 | if (xsk_ring_prod__needs_wakeup(&umem->fq)) { | |
1165 | ret = poll(&fds, 1, POLL_TMOUT); | |
1166 | if (ret < 0) | |
1167 | return TEST_FAILURE; | |
29f128b3 | 1168 | } |
8913e653 | 1169 | ret = xsk_ring_prod__reserve(&umem->fq, rcvd, &idx_fq); |
facb7cb2 | 1170 | } |
8913e653 | 1171 | } |
facb7cb2 | 1172 | |
8913e653 TV |
1173 | while (frags_processed < rcvd) { |
1174 | const struct xdp_desc *desc = xsk_ring_cons__rx_desc(&xsk->rx, idx_rx++); | |
1175 | u64 addr = desc->addr, orig; | |
829725ec | 1176 | |
8913e653 TV |
1177 | orig = xsk_umem__extract_addr(addr); |
1178 | addr = xsk_umem__add_offset_to_addr(addr); | |
e34087fc | 1179 | |
8913e653 TV |
1180 | if (!nb_frags) { |
1181 | pkt = pkt_stream_get_next_rx_pkt(pkt_stream, &pkts_sent); | |
69760449 MK |
1182 | if (!pkt) { |
1183 | ksft_print_msg("[%s] received too many packets addr: %lx len %u\n", | |
1184 | __func__, addr, desc->len); | |
1185 | return TEST_FAILURE; | |
1186 | } | |
8913e653 | 1187 | } |
69760449 | 1188 | |
8913e653 TV |
1189 | print_verbose("Rx: addr: %lx len: %u options: %u pkt_nb: %u valid: %u\n", |
1190 | addr, desc->len, desc->options, pkt->pkt_nb, pkt->valid); | |
2d2712ca | 1191 | |
8913e653 TV |
1192 | if (!is_frag_valid(umem, addr, desc->len, pkt->pkt_nb, pkt_len) || |
1193 | !is_offset_correct(umem, pkt, addr) || (ifobj->use_metadata && | |
1194 | !is_metadata_correct(pkt, umem->buffer, addr))) | |
1195 | return TEST_FAILURE; | |
facb7cb2 | 1196 | |
8913e653 TV |
1197 | if (!nb_frags++) |
1198 | first_addr = addr; | |
1199 | frags_processed++; | |
1200 | pkt_len += desc->len; | |
1201 | if (ifobj->use_fill_ring) | |
1202 | *xsk_ring_prod__fill_addr(&umem->fq, idx_fq++) = orig; | |
17f1034d | 1203 | |
8913e653 TV |
1204 | if (pkt_continues(desc->options)) |
1205 | continue; | |
17f1034d | 1206 | |
8913e653 TV |
1207 | /* The complete packet has been received */ |
1208 | if (!is_pkt_valid(pkt, umem->buffer, first_addr, pkt_len) || | |
1209 | !is_offset_correct(umem, pkt, addr)) | |
1210 | return TEST_FAILURE; | |
17f1034d | 1211 | |
8913e653 TV |
1212 | pkt_stream->nb_rx_pkts++; |
1213 | nb_frags = 0; | |
1214 | pkt_len = 0; | |
1215 | } | |
17f1034d | 1216 | |
8913e653 TV |
1217 | if (nb_frags) { |
1218 | /* In the middle of a packet. Start over from beginning of packet. */ | |
1219 | idx_rx -= nb_frags; | |
1220 | xsk_ring_cons__cancel(&xsk->rx, nb_frags); | |
1221 | if (ifobj->use_fill_ring) { | |
1222 | idx_fq -= nb_frags; | |
1223 | xsk_ring_prod__cancel(&umem->fq, nb_frags); | |
29f128b3 | 1224 | } |
8913e653 TV |
1225 | frags_processed -= nb_frags; |
1226 | } | |
facb7cb2 | 1227 | |
8913e653 TV |
1228 | if (ifobj->use_fill_ring) |
1229 | xsk_ring_prod__submit(&umem->fq, frags_processed); | |
1230 | if (ifobj->release_rx) | |
1231 | xsk_ring_cons__release(&xsk->rx, frags_processed); | |
1232 | ||
1233 | pthread_mutex_lock(&pacing_mutex); | |
1234 | pkts_in_flight -= pkts_sent; | |
1235 | pthread_mutex_unlock(&pacing_mutex); | |
1236 | pkts_sent = 0; | |
1237 | ||
1238 | return TEST_CONTINUE; | |
1239 | } | |
1240 | ||
1241 | bool all_packets_received(struct test_spec *test, struct xsk_socket_info *xsk, u32 sock_num, | |
1242 | unsigned long *bitmap) | |
1243 | { | |
1244 | struct pkt_stream *pkt_stream = xsk->pkt_stream; | |
1245 | ||
1246 | if (!pkt_stream) { | |
1247 | __set_bit(sock_num, bitmap); | |
1248 | return false; | |
1249 | } | |
1250 | ||
1251 | if (pkt_stream->nb_rx_pkts == pkt_stream->nb_valid_entries) { | |
1252 | __set_bit(sock_num, bitmap); | |
1253 | if (bitmap_full(bitmap, test->nb_sockets)) | |
1254 | return true; | |
1255 | } | |
1256 | ||
1257 | return false; | |
1258 | } | |
1259 | ||
1260 | static int receive_pkts(struct test_spec *test) | |
1261 | { | |
1262 | struct timeval tv_end, tv_now, tv_timeout = {THREAD_TMOUT, 0}; | |
1263 | DECLARE_BITMAP(bitmap, test->nb_sockets); | |
1264 | struct xsk_socket_info *xsk; | |
1265 | u32 sock_num = 0; | |
1266 | int res, ret; | |
1bf36496 | 1267 | |
8913e653 TV |
1268 | ret = gettimeofday(&tv_now, NULL); |
1269 | if (ret) | |
1270 | exit_with_error(errno); | |
1271 | ||
1272 | timeradd(&tv_now, &tv_timeout, &tv_end); | |
1273 | ||
1274 | while (1) { | |
1275 | xsk = &test->ifobj_rx->xsk_arr[sock_num]; | |
1276 | ||
1277 | if ((all_packets_received(test, xsk, sock_num, bitmap))) | |
1278 | break; | |
1279 | ||
1280 | res = __receive_pkts(test, xsk); | |
1281 | if (!(res == TEST_PASS || res == TEST_CONTINUE)) | |
1282 | return res; | |
1283 | ||
1284 | ret = gettimeofday(&tv_now, NULL); | |
1285 | if (ret) | |
1286 | exit_with_error(errno); | |
1287 | ||
1288 | if (timercmp(&tv_now, &tv_end, >)) { | |
1289 | ksft_print_msg("ERROR: [%s] Receive loop timed out\n", __func__); | |
1290 | return TEST_FAILURE; | |
1291 | } | |
1292 | sock_num = (sock_num + 1) % test->nb_sockets; | |
facb7cb2 | 1293 | } |
895b62ee MK |
1294 | |
1295 | return TEST_PASS; | |
facb7cb2 WJ |
1296 | } |
1297 | ||
fd0815ae | 1298 | static int __send_pkts(struct ifobject *ifobject, struct xsk_socket_info *xsk, bool timeout) |
facb7cb2 | 1299 | { |
17f1034d | 1300 | u32 i, idx = 0, valid_pkts = 0, valid_frags = 0, buffer_len; |
fd0815ae | 1301 | struct pkt_stream *pkt_stream = xsk->pkt_stream; |
7cd6df4f | 1302 | struct xsk_umem_info *umem = ifobject->umem; |
3143d10b | 1303 | bool use_poll = ifobject->use_poll; |
fd0815ae | 1304 | struct pollfd fds = { }; |
e8f50c4f | 1305 | int ret; |
3143d10b | 1306 | |
17f1034d | 1307 | buffer_len = pkt_get_buffer_len(umem, pkt_stream->max_pkt_len); |
7cd6df4f | 1308 | /* pkts_in_flight might be negative if many invalid packets are sent */ |
c3bd0150 TV |
1309 | if (pkts_in_flight >= (int)((umem_size(umem) - xsk->batch_size * buffer_len) / |
1310 | buffer_len)) { | |
5fc494d5 MK |
1311 | ret = kick_tx(xsk); |
1312 | if (ret) | |
1313 | return TEST_FAILURE; | |
7cd6df4f MK |
1314 | return TEST_CONTINUE; |
1315 | } | |
1316 | ||
fd0815ae TV |
1317 | fds.fd = xsk_socket__fd(xsk->xsk); |
1318 | fds.events = POLLOUT; | |
1319 | ||
c3bd0150 | 1320 | while (xsk_ring_prod__reserve(&xsk->tx, xsk->batch_size, &idx) < xsk->batch_size) { |
3143d10b | 1321 | if (use_poll) { |
fd0815ae | 1322 | ret = poll(&fds, 1, POLL_TMOUT); |
3143d10b SKR |
1323 | if (timeout) { |
1324 | if (ret < 0) { | |
1325 | ksft_print_msg("ERROR: [%s] Poll error %d\n", | |
085dcccf | 1326 | __func__, errno); |
3143d10b SKR |
1327 | return TEST_FAILURE; |
1328 | } | |
1329 | if (ret == 0) | |
1330 | return TEST_PASS; | |
1331 | break; | |
1332 | } | |
1333 | if (ret <= 0) { | |
1334 | ksft_print_msg("ERROR: [%s] Poll error %d\n", | |
085dcccf | 1335 | __func__, errno); |
3143d10b SKR |
1336 | return TEST_FAILURE; |
1337 | } | |
1338 | } | |
facb7cb2 | 1339 | |
c3bd0150 | 1340 | complete_pkts(xsk, xsk->batch_size); |
3143d10b | 1341 | } |
facb7cb2 | 1342 | |
c3bd0150 | 1343 | for (i = 0; i < xsk->batch_size; i++) { |
17f1034d | 1344 | struct pkt *pkt = pkt_stream_get_next_tx_pkt(pkt_stream); |
69760449 | 1345 | u32 nb_frags_left, nb_frags, bytes_written = 0; |
facb7cb2 | 1346 | |
29f128b3 MK |
1347 | if (!pkt) |
1348 | break; | |
facb7cb2 | 1349 | |
69760449 | 1350 | nb_frags = pkt_nb_frags(umem->frame_size, pkt_stream, pkt); |
c3bd0150 | 1351 | if (nb_frags > xsk->batch_size - i) { |
17f1034d | 1352 | pkt_stream_cancel(pkt_stream); |
c3bd0150 | 1353 | xsk_ring_prod__cancel(&xsk->tx, xsk->batch_size - i); |
17f1034d MK |
1354 | break; |
1355 | } | |
69760449 | 1356 | nb_frags_left = nb_frags; |
17f1034d | 1357 | |
69760449 | 1358 | while (nb_frags_left--) { |
17f1034d MK |
1359 | struct xdp_desc *tx_desc = xsk_ring_prod__tx_desc(&xsk->tx, idx + i); |
1360 | ||
1361 | tx_desc->addr = pkt_get_addr(pkt, ifobject->umem); | |
69760449 MK |
1362 | if (pkt_stream->verbatim) { |
1363 | tx_desc->len = pkt->len; | |
1364 | tx_desc->options = pkt->options; | |
1365 | } else if (nb_frags_left) { | |
17f1034d MK |
1366 | tx_desc->len = umem->frame_size; |
1367 | tx_desc->options = XDP_PKT_CONTD; | |
17f1034d MK |
1368 | } else { |
1369 | tx_desc->len = pkt->len - bytes_written; | |
1370 | tx_desc->options = 0; | |
1371 | } | |
1372 | if (pkt->valid) | |
985fd214 | 1373 | pkt_generate(xsk, umem, tx_desc->addr, tx_desc->len, pkt->pkt_nb, |
17f1034d MK |
1374 | bytes_written); |
1375 | bytes_written += tx_desc->len; | |
69760449 | 1376 | |
2d2712ca MK |
1377 | print_verbose("Tx addr: %llx len: %u options: %u pkt_nb: %u\n", |
1378 | tx_desc->addr, tx_desc->len, tx_desc->options, pkt->pkt_nb); | |
1379 | ||
69760449 MK |
1380 | if (nb_frags_left) { |
1381 | i++; | |
1382 | if (pkt_stream->verbatim) | |
1383 | pkt = pkt_stream_get_next_tx_pkt(pkt_stream); | |
1384 | } | |
1385 | } | |
1386 | ||
1387 | if (pkt && pkt->valid) { | |
1388 | valid_pkts++; | |
1389 | valid_frags += nb_frags; | |
2f6eae0d | 1390 | } |
b267e5a4 | 1391 | } |
facb7cb2 | 1392 | |
1bf36496 MK |
1393 | pthread_mutex_lock(&pacing_mutex); |
1394 | pkts_in_flight += valid_pkts; | |
1bf36496 MK |
1395 | pthread_mutex_unlock(&pacing_mutex); |
1396 | ||
29f128b3 | 1397 | xsk_ring_prod__submit(&xsk->tx, i); |
17f1034d | 1398 | xsk->outstanding_tx += valid_frags; |
facb7cb2 | 1399 | |
3143d10b | 1400 | if (use_poll) { |
fd0815ae | 1401 | ret = poll(&fds, 1, POLL_TMOUT); |
3143d10b SKR |
1402 | if (ret <= 0) { |
1403 | if (ret == 0 && timeout) | |
1404 | return TEST_PASS; | |
1405 | ||
1406 | ksft_print_msg("ERROR: [%s] Poll error %d\n", __func__, ret); | |
1407 | return TEST_FAILURE; | |
1408 | } | |
1409 | } | |
1410 | ||
1411 | if (!timeout) { | |
1412 | if (complete_pkts(xsk, i)) | |
1413 | return TEST_FAILURE; | |
1414 | ||
1415 | usleep(10); | |
1416 | return TEST_PASS; | |
1417 | } | |
1418 | ||
1419 | return TEST_CONTINUE; | |
facb7cb2 WJ |
1420 | } |
1421 | ||
64370d7c | 1422 | static int wait_for_tx_completion(struct xsk_socket_info *xsk) |
facb7cb2 | 1423 | { |
64370d7c MK |
1424 | struct timeval tv_end, tv_now, tv_timeout = {THREAD_TMOUT, 0}; |
1425 | int ret; | |
1426 | ||
1427 | ret = gettimeofday(&tv_now, NULL); | |
1428 | if (ret) | |
1429 | exit_with_error(errno); | |
1430 | timeradd(&tv_now, &tv_timeout, &tv_end); | |
1431 | ||
1432 | while (xsk->outstanding_tx) { | |
1433 | ret = gettimeofday(&tv_now, NULL); | |
1434 | if (ret) | |
1435 | exit_with_error(errno); | |
1436 | if (timercmp(&tv_now, &tv_end, >)) { | |
1437 | ksft_print_msg("ERROR: [%s] Transmission loop timed out\n", __func__); | |
1438 | return TEST_FAILURE; | |
1439 | } | |
1440 | ||
c3bd0150 | 1441 | complete_pkts(xsk, xsk->batch_size); |
64370d7c MK |
1442 | } |
1443 | ||
1444 | return TEST_PASS; | |
facb7cb2 WJ |
1445 | } |
1446 | ||
fd0815ae TV |
1447 | bool all_packets_sent(struct test_spec *test, unsigned long *bitmap) |
1448 | { | |
1449 | return bitmap_full(bitmap, test->nb_sockets); | |
1450 | } | |
1451 | ||
895b62ee | 1452 | static int send_pkts(struct test_spec *test, struct ifobject *ifobject) |
facb7cb2 | 1453 | { |
3143d10b | 1454 | bool timeout = !is_umem_valid(test->ifobj_rx); |
fd0815ae TV |
1455 | DECLARE_BITMAP(bitmap, test->nb_sockets); |
1456 | u32 i, ret; | |
facb7cb2 | 1457 | |
fd0815ae TV |
1458 | while (!(all_packets_sent(test, bitmap))) { |
1459 | for (i = 0; i < test->nb_sockets; i++) { | |
1460 | struct pkt_stream *pkt_stream; | |
facb7cb2 | 1461 | |
fd0815ae TV |
1462 | pkt_stream = ifobject->xsk_arr[i].pkt_stream; |
1463 | if (!pkt_stream || pkt_stream->current_pkt_nb >= pkt_stream->nb_pkts) { | |
1464 | __set_bit(i, bitmap); | |
1465 | continue; | |
1466 | } | |
1467 | ret = __send_pkts(ifobject, &ifobject->xsk_arr[i], timeout); | |
1468 | if (ret == TEST_CONTINUE && !test->fail) | |
1469 | continue; | |
1470 | ||
1471 | if ((ret || test->fail) && !timeout) | |
1472 | return TEST_FAILURE; | |
1473 | ||
1474 | if (ret == TEST_PASS && timeout) | |
1475 | return ret; | |
1476 | ||
1477 | ret = wait_for_tx_completion(&ifobject->xsk_arr[i]); | |
1478 | if (ret) | |
1479 | return TEST_FAILURE; | |
1480 | } | |
facb7cb2 WJ |
1481 | } |
1482 | ||
fd0815ae | 1483 | return TEST_PASS; |
facb7cb2 WJ |
1484 | } |
1485 | ||
76c57663 MK |
1486 | static int get_xsk_stats(struct xsk_socket *xsk, struct xdp_statistics *stats) |
1487 | { | |
1488 | int fd = xsk_socket__fd(xsk), err; | |
1489 | socklen_t optlen, expected_len; | |
1490 | ||
1491 | optlen = sizeof(*stats); | |
1492 | err = getsockopt(fd, SOL_XDP, XDP_STATISTICS, stats, &optlen); | |
1493 | if (err) { | |
1494 | ksft_print_msg("[%s] getsockopt(XDP_STATISTICS) error %u %s\n", | |
1495 | __func__, -err, strerror(-err)); | |
1496 | return TEST_FAILURE; | |
1497 | } | |
1498 | ||
1499 | expected_len = sizeof(struct xdp_statistics); | |
1500 | if (optlen != expected_len) { | |
1501 | ksft_print_msg("[%s] getsockopt optlen error. Expected: %u got: %u\n", | |
1502 | __func__, expected_len, optlen); | |
1503 | return TEST_FAILURE; | |
1504 | } | |
1505 | ||
1506 | return TEST_PASS; | |
1507 | } | |
1508 | ||
1509 | static int validate_rx_dropped(struct ifobject *ifobject) | |
b267e5a4 | 1510 | { |
b04fdc4c | 1511 | struct xsk_socket *xsk = ifobject->xsk->xsk; |
b267e5a4 | 1512 | struct xdp_statistics stats; |
b267e5a4 | 1513 | int err; |
b267e5a4 | 1514 | |
5fc494d5 MK |
1515 | err = kick_rx(ifobject->xsk); |
1516 | if (err) | |
1517 | return TEST_FAILURE; | |
f90062b5 | 1518 | |
76c57663 MK |
1519 | err = get_xsk_stats(xsk, &stats); |
1520 | if (err) | |
895b62ee | 1521 | return TEST_FAILURE; |
b267e5a4 | 1522 | |
68e73221 KC |
1523 | /* The receiver calls getsockopt after receiving the last (valid) |
1524 | * packet which is not the final packet sent in this test (valid and | |
1525 | * invalid packets are sent in alternating fashion with the final | |
1526 | * packet being invalid). Since the last packet may or may not have | |
1527 | * been dropped already, both outcomes must be allowed. | |
1528 | */ | |
8367eb95 TV |
1529 | if (stats.rx_dropped == ifobject->xsk->pkt_stream->nb_pkts / 2 || |
1530 | stats.rx_dropped == ifobject->xsk->pkt_stream->nb_pkts / 2 - 1) | |
76c57663 | 1531 | return TEST_PASS; |
b267e5a4 | 1532 | |
27e934be | 1533 | return TEST_FAILURE; |
76c57663 MK |
1534 | } |
1535 | ||
1536 | static int validate_rx_full(struct ifobject *ifobject) | |
1537 | { | |
1538 | struct xsk_socket *xsk = ifobject->xsk->xsk; | |
1539 | struct xdp_statistics stats; | |
76c57663 MK |
1540 | int err; |
1541 | ||
27e934be | 1542 | usleep(1000); |
5fc494d5 MK |
1543 | err = kick_rx(ifobject->xsk); |
1544 | if (err) | |
1545 | return TEST_FAILURE; | |
76c57663 MK |
1546 | |
1547 | err = get_xsk_stats(xsk, &stats); | |
1548 | if (err) | |
1549 | return TEST_FAILURE; | |
1550 | ||
27e934be | 1551 | if (stats.rx_ring_full) |
76c57663 MK |
1552 | return TEST_PASS; |
1553 | ||
27e934be | 1554 | return TEST_FAILURE; |
76c57663 MK |
1555 | } |
1556 | ||
1557 | static int validate_fill_empty(struct ifobject *ifobject) | |
1558 | { | |
1559 | struct xsk_socket *xsk = ifobject->xsk->xsk; | |
1560 | struct xdp_statistics stats; | |
1561 | int err; | |
1562 | ||
27e934be | 1563 | usleep(1000); |
5fc494d5 MK |
1564 | err = kick_rx(ifobject->xsk); |
1565 | if (err) | |
1566 | return TEST_FAILURE; | |
76c57663 MK |
1567 | |
1568 | err = get_xsk_stats(xsk, &stats); | |
1569 | if (err) | |
1570 | return TEST_FAILURE; | |
1571 | ||
27e934be | 1572 | if (stats.rx_fill_ring_empty_descs) |
76c57663 | 1573 | return TEST_PASS; |
b04fdc4c | 1574 | |
27e934be | 1575 | return TEST_FAILURE; |
b04fdc4c MK |
1576 | } |
1577 | ||
76c57663 | 1578 | static int validate_tx_invalid_descs(struct ifobject *ifobject) |
b04fdc4c MK |
1579 | { |
1580 | struct xsk_socket *xsk = ifobject->xsk->xsk; | |
1581 | int fd = xsk_socket__fd(xsk); | |
1582 | struct xdp_statistics stats; | |
1583 | socklen_t optlen; | |
1584 | int err; | |
1585 | ||
1586 | optlen = sizeof(stats); | |
1587 | err = getsockopt(fd, SOL_XDP, XDP_STATISTICS, &stats, &optlen); | |
1588 | if (err) { | |
895b62ee MK |
1589 | ksft_print_msg("[%s] getsockopt(XDP_STATISTICS) error %u %s\n", |
1590 | __func__, -err, strerror(-err)); | |
1591 | return TEST_FAILURE; | |
b04fdc4c MK |
1592 | } |
1593 | ||
8367eb95 | 1594 | if (stats.tx_invalid_descs != ifobject->xsk->pkt_stream->nb_pkts / 2) { |
fe69a1b1 AR |
1595 | ksft_print_msg("[%s] tx_invalid_descs incorrect. Got [%llu] expected [%u]\n", |
1596 | __func__, | |
1597 | (unsigned long long)stats.tx_invalid_descs, | |
8367eb95 | 1598 | ifobject->xsk->pkt_stream->nb_pkts); |
895b62ee MK |
1599 | return TEST_FAILURE; |
1600 | } | |
b04fdc4c | 1601 | |
895b62ee | 1602 | return TEST_PASS; |
b267e5a4 CL |
1603 | } |
1604 | ||
a693ff3e MF |
1605 | static void xsk_configure_socket(struct test_spec *test, struct ifobject *ifobject, |
1606 | struct xsk_umem_info *umem, bool tx) | |
1607 | { | |
1608 | int i, ret; | |
1609 | ||
1610 | for (i = 0; i < test->nb_sockets; i++) { | |
1611 | bool shared = (ifobject->shared_umem && tx) ? true : !!i; | |
1612 | u32 ctr = 0; | |
1613 | ||
1614 | while (ctr++ < SOCK_RECONF_CTR) { | |
1615 | ret = __xsk_configure_socket(&ifobject->xsk_arr[i], umem, | |
1616 | ifobject, shared); | |
1617 | if (!ret) | |
1618 | break; | |
1619 | ||
1620 | /* Retry if it fails as xsk_socket__create() is asynchronous */ | |
1621 | if (ctr >= SOCK_RECONF_CTR) | |
1622 | exit_with_error(-ret); | |
1623 | usleep(USLEEP_MAX); | |
1624 | } | |
1625 | if (ifobject->busy_poll) | |
1626 | enable_busy_poll(&ifobject->xsk_arr[i]); | |
1627 | } | |
1628 | } | |
1629 | ||
1630 | static void thread_common_ops_tx(struct test_spec *test, struct ifobject *ifobject) | |
1631 | { | |
1632 | xsk_configure_socket(test, ifobject, test->ifobj_rx->umem, true); | |
1633 | ifobject->xsk = &ifobject->xsk_arr[0]; | |
f0a249df | 1634 | ifobject->xskmap = test->ifobj_rx->xskmap; |
a693ff3e | 1635 | memcpy(ifobject->umem, test->ifobj_rx->umem, sizeof(struct xsk_umem_info)); |
d9f6d970 | 1636 | ifobject->umem->base_addr = 0; |
a693ff3e MF |
1637 | } |
1638 | ||
86e41755 MK |
1639 | static void xsk_populate_fill_ring(struct xsk_umem_info *umem, struct pkt_stream *pkt_stream, |
1640 | bool fill_up) | |
a693ff3e | 1641 | { |
86e41755 MK |
1642 | u32 rx_frame_size = umem->frame_size - XDP_PACKET_HEADROOM; |
1643 | u32 idx = 0, filled = 0, buffers_to_fill, nb_pkts; | |
a693ff3e MF |
1644 | int ret; |
1645 | ||
1646 | if (umem->num_frames < XSK_RING_PROD__DEFAULT_NUM_DESCS) | |
1647 | buffers_to_fill = umem->num_frames; | |
1648 | else | |
e4a195e2 | 1649 | buffers_to_fill = umem->fill_size; |
a693ff3e MF |
1650 | |
1651 | ret = xsk_ring_prod__reserve(&umem->fq, buffers_to_fill, &idx); | |
1652 | if (ret != buffers_to_fill) | |
1653 | exit_with_error(ENOSPC); | |
d9f6d970 | 1654 | |
86e41755 | 1655 | while (filled < buffers_to_fill) { |
d9f6d970 | 1656 | struct pkt *pkt = pkt_stream_get_next_rx_pkt(pkt_stream, &nb_pkts); |
a693ff3e | 1657 | u64 addr; |
86e41755 MK |
1658 | u32 i; |
1659 | ||
69760449 | 1660 | for (i = 0; i < pkt_nb_frags(rx_frame_size, pkt_stream, pkt); i++) { |
86e41755 MK |
1661 | if (!pkt) { |
1662 | if (!fill_up) | |
1663 | break; | |
1664 | addr = filled * umem->frame_size + umem->base_addr; | |
1665 | } else if (pkt->offset >= 0) { | |
1666 | addr = pkt->offset % umem->frame_size + umem_alloc_buffer(umem); | |
1667 | } else { | |
1668 | addr = pkt->offset + umem_alloc_buffer(umem); | |
1669 | } | |
a693ff3e | 1670 | |
86e41755 MK |
1671 | *xsk_ring_prod__fill_addr(&umem->fq, idx++) = addr; |
1672 | if (++filled >= buffers_to_fill) | |
1673 | break; | |
1674 | } | |
a693ff3e | 1675 | } |
86e41755 MK |
1676 | xsk_ring_prod__submit(&umem->fq, filled); |
1677 | xsk_ring_prod__cancel(&umem->fq, buffers_to_fill - filled); | |
69fc03d2 MK |
1678 | |
1679 | pkt_stream_reset(pkt_stream); | |
d9f6d970 | 1680 | umem_reset_alloc(umem); |
a693ff3e MF |
1681 | } |
1682 | ||
55be575d | 1683 | static void thread_common_ops(struct test_spec *test, struct ifobject *ifobject) |
facb7cb2 | 1684 | { |
3b22523b | 1685 | u64 umem_sz = ifobject->umem->num_frames * ifobject->umem->frame_size; |
a4ba98dd | 1686 | int mmap_flags = MAP_PRIVATE | MAP_ANONYMOUS | MAP_NORESERVE; |
6d4c767c | 1687 | LIBBPF_OPTS(bpf_xdp_query_opts, opts); |
3b22523b | 1688 | void *bufs; |
f0a249df | 1689 | int ret; |
6d198a89 | 1690 | u32 i; |
facb7cb2 | 1691 | |
02e93e04 MK |
1692 | if (ifobject->umem->unaligned_mode) |
1693 | mmap_flags |= MAP_HUGETLB | MAP_HUGE_2MB; | |
a4ba98dd | 1694 | |
a693ff3e MF |
1695 | if (ifobject->shared_umem) |
1696 | umem_sz *= 2; | |
1697 | ||
02e93e04 | 1698 | bufs = mmap(NULL, umem_sz, PROT_READ | PROT_WRITE, mmap_flags, -1, 0); |
3b22523b MK |
1699 | if (bufs == MAP_FAILED) |
1700 | exit_with_error(errno); | |
9866bcd6 | 1701 | |
d9f6d970 | 1702 | ret = xsk_configure_umem(ifobject, ifobject->umem, bufs, umem_sz); |
3b22523b MK |
1703 | if (ret) |
1704 | exit_with_error(-ret); | |
ed7b74dc | 1705 | |
a693ff3e | 1706 | xsk_configure_socket(test, ifobject, ifobject->umem, false); |
27e1ca25 | 1707 | |
ed7b74dc | 1708 | ifobject->xsk = &ifobject->xsk_arr[0]; |
3b22523b MK |
1709 | |
1710 | if (!ifobject->rx_on) | |
1711 | return; | |
1712 | ||
8367eb95 | 1713 | xsk_populate_fill_ring(ifobject->umem, ifobject->xsk->pkt_stream, ifobject->use_fill_ring); |
d9f6d970 | 1714 | |
6d198a89 TV |
1715 | for (i = 0; i < test->nb_sockets; i++) { |
1716 | ifobject->xsk = &ifobject->xsk_arr[i]; | |
1717 | ret = xsk_update_xskmap(ifobject->xskmap, ifobject->xsk->xsk, i); | |
1718 | if (ret) | |
1719 | exit_with_error(errno); | |
1720 | } | |
facb7cb2 WJ |
1721 | } |
1722 | ||
9866bcd6 | 1723 | static void *worker_testapp_validate_tx(void *arg) |
facb7cb2 | 1724 | { |
55be575d MK |
1725 | struct test_spec *test = (struct test_spec *)arg; |
1726 | struct ifobject *ifobject = test->ifobj_tx; | |
895b62ee | 1727 | int err; |
facb7cb2 | 1728 | |
a693ff3e MF |
1729 | if (test->current_step == 1) { |
1730 | if (!ifobject->shared_umem) | |
1731 | thread_common_ops(test, ifobject); | |
1732 | else | |
1733 | thread_common_ops_tx(test, ifobject); | |
1734 | } | |
facb7cb2 | 1735 | |
895b62ee | 1736 | err = send_pkts(test, ifobject); |
facb7cb2 | 1737 | |
27e934be | 1738 | if (!err && ifobject->validation_func) |
76c57663 | 1739 | err = ifobject->validation_func(ifobject); |
27e934be | 1740 | if (err) |
895b62ee | 1741 | report_failure(test); |
b04fdc4c | 1742 | |
9866bcd6 MF |
1743 | pthread_exit(NULL); |
1744 | } | |
facb7cb2 | 1745 | |
9866bcd6 MF |
1746 | static void *worker_testapp_validate_rx(void *arg) |
1747 | { | |
55be575d MK |
1748 | struct test_spec *test = (struct test_spec *)arg; |
1749 | struct ifobject *ifobject = test->ifobj_rx; | |
f0a249df | 1750 | int err; |
facb7cb2 | 1751 | |
a693ff3e | 1752 | if (test->current_step == 1) { |
55be575d | 1753 | thread_common_ops(test, ifobject); |
a693ff3e | 1754 | } else { |
f0a249df | 1755 | xsk_clear_xskmap(ifobject->xskmap); |
fc2cb864 | 1756 | err = xsk_update_xskmap(ifobject->xskmap, ifobject->xsk->xsk, 0); |
aa61d81f | 1757 | if (err) { |
7c3fcf08 MK |
1758 | ksft_print_msg("Error: Failed to update xskmap, error %s\n", |
1759 | strerror(-err)); | |
f0a249df | 1760 | exit_with_error(-err); |
aa61d81f | 1761 | } |
a693ff3e | 1762 | } |
facb7cb2 | 1763 | |
96539f1c | 1764 | pthread_barrier_wait(&barr); |
facb7cb2 | 1765 | |
8913e653 | 1766 | err = receive_pkts(test); |
895b62ee | 1767 | |
27e934be MK |
1768 | if (!err && ifobject->validation_func) |
1769 | err = ifobject->validation_func(ifobject); | |
7cd6df4f | 1770 | if (err) |
895b62ee | 1771 | report_failure(test); |
6674bf66 | 1772 | |
facb7cb2 WJ |
1773 | pthread_exit(NULL); |
1774 | } | |
1775 | ||
2ddade32 MK |
1776 | static u64 ceil_u64(u64 a, u64 b) |
1777 | { | |
1778 | return (a + b - 1) / b; | |
1779 | } | |
1780 | ||
a693ff3e MF |
1781 | static void testapp_clean_xsk_umem(struct ifobject *ifobj) |
1782 | { | |
1783 | u64 umem_sz = ifobj->umem->num_frames * ifobj->umem->frame_size; | |
1784 | ||
1785 | if (ifobj->shared_umem) | |
1786 | umem_sz *= 2; | |
1787 | ||
2ddade32 | 1788 | umem_sz = ceil_u64(umem_sz, HUGEPAGE_SIZE) * HUGEPAGE_SIZE; |
a693ff3e MF |
1789 | xsk_umem__delete(ifobj->umem->umem); |
1790 | munmap(ifobj->umem->buffer, umem_sz); | |
1791 | } | |
1792 | ||
c29fe883 MF |
1793 | static void handler(int signum) |
1794 | { | |
1795 | pthread_exit(NULL); | |
1796 | } | |
1797 | ||
d2e54149 | 1798 | static bool xdp_prog_changed_rx(struct test_spec *test) |
7d8319a7 | 1799 | { |
d2e54149 MK |
1800 | struct ifobject *ifobj = test->ifobj_rx; |
1801 | ||
7d8319a7 MK |
1802 | return ifobj->xdp_prog != test->xdp_prog_rx || ifobj->mode != test->mode; |
1803 | } | |
1804 | ||
d2e54149 MK |
1805 | static bool xdp_prog_changed_tx(struct test_spec *test) |
1806 | { | |
1807 | struct ifobject *ifobj = test->ifobj_tx; | |
1808 | ||
1809 | return ifobj->xdp_prog != test->xdp_prog_tx || ifobj->mode != test->mode; | |
1810 | } | |
1811 | ||
7d8319a7 MK |
1812 | static void xsk_reattach_xdp(struct ifobject *ifobj, struct bpf_program *xdp_prog, |
1813 | struct bpf_map *xskmap, enum test_mode mode) | |
1814 | { | |
1815 | int err; | |
1816 | ||
1817 | xsk_detach_xdp_program(ifobj->ifindex, mode_to_xdp_flags(ifobj->mode)); | |
1818 | err = xsk_attach_xdp_program(xdp_prog, ifobj->ifindex, mode_to_xdp_flags(mode)); | |
1819 | if (err) { | |
7c3fcf08 | 1820 | ksft_print_msg("Error attaching XDP program\n"); |
7d8319a7 MK |
1821 | exit_with_error(-err); |
1822 | } | |
1823 | ||
1824 | if (ifobj->mode != mode && (mode == TEST_MODE_DRV || mode == TEST_MODE_ZC)) | |
1825 | if (!xsk_is_in_mode(ifobj->ifindex, XDP_FLAGS_DRV_MODE)) { | |
1826 | ksft_print_msg("ERROR: XDP prog not in DRV mode\n"); | |
1827 | exit_with_error(EINVAL); | |
1828 | } | |
1829 | ||
1830 | ifobj->xdp_prog = xdp_prog; | |
1831 | ifobj->xskmap = xskmap; | |
1832 | ifobj->mode = mode; | |
1833 | } | |
1834 | ||
1835 | static void xsk_attach_xdp_progs(struct test_spec *test, struct ifobject *ifobj_rx, | |
1836 | struct ifobject *ifobj_tx) | |
1837 | { | |
d2e54149 | 1838 | if (xdp_prog_changed_rx(test)) |
7d8319a7 MK |
1839 | xsk_reattach_xdp(ifobj_rx, test->xdp_prog_rx, test->xskmap_rx, test->mode); |
1840 | ||
1841 | if (!ifobj_tx || ifobj_tx->shared_umem) | |
1842 | return; | |
1843 | ||
d2e54149 | 1844 | if (xdp_prog_changed_tx(test)) |
7d8319a7 MK |
1845 | xsk_reattach_xdp(ifobj_tx, test->xdp_prog_tx, test->xskmap_tx, test->mode); |
1846 | } | |
1847 | ||
7f881984 MK |
1848 | static int __testapp_validate_traffic(struct test_spec *test, struct ifobject *ifobj1, |
1849 | struct ifobject *ifobj2) | |
3143d10b | 1850 | { |
7f881984 | 1851 | pthread_t t0, t1; |
f540d44e MK |
1852 | int err; |
1853 | ||
1854 | if (test->mtu > MAX_ETH_PKT_SIZE) { | |
1855 | if (test->mode == TEST_MODE_ZC && (!ifobj1->multi_buff_zc_supp || | |
1856 | (ifobj2 && !ifobj2->multi_buff_zc_supp))) { | |
1857 | ksft_test_result_skip("Multi buffer for zero-copy not supported.\n"); | |
1858 | return TEST_SKIP; | |
1859 | } | |
1860 | if (test->mode != TEST_MODE_ZC && (!ifobj1->multi_buff_supp || | |
1861 | (ifobj2 && !ifobj2->multi_buff_supp))) { | |
1862 | ksft_test_result_skip("Multi buffer not supported.\n"); | |
1863 | return TEST_SKIP; | |
1864 | } | |
1865 | } | |
1866 | err = test_spec_set_mtu(test, test->mtu); | |
1867 | if (err) { | |
1868 | ksft_print_msg("Error, could not set mtu.\n"); | |
1869 | exit_with_error(err); | |
1870 | } | |
3143d10b | 1871 | |
69fc03d2 | 1872 | if (ifobj2) { |
7f881984 MK |
1873 | if (pthread_barrier_init(&barr, NULL, 2)) |
1874 | exit_with_error(errno); | |
8367eb95 | 1875 | pkt_stream_reset(ifobj2->xsk->pkt_stream); |
69fc03d2 | 1876 | } |
3143d10b SKR |
1877 | |
1878 | test->current_step++; | |
8367eb95 | 1879 | pkt_stream_reset(ifobj1->xsk->pkt_stream); |
3143d10b SKR |
1880 | pkts_in_flight = 0; |
1881 | ||
c29fe883 | 1882 | signal(SIGUSR1, handler); |
7f881984 MK |
1883 | /*Spawn RX thread */ |
1884 | pthread_create(&t0, NULL, ifobj1->func_ptr, test); | |
3143d10b | 1885 | |
7f881984 | 1886 | if (ifobj2) { |
3143d10b | 1887 | pthread_barrier_wait(&barr); |
7f881984 MK |
1888 | if (pthread_barrier_destroy(&barr)) |
1889 | exit_with_error(errno); | |
3143d10b | 1890 | |
7f881984 MK |
1891 | /*Spawn TX thread */ |
1892 | pthread_create(&t1, NULL, ifobj2->func_ptr, test); | |
3143d10b | 1893 | |
7f881984 MK |
1894 | pthread_join(t1, NULL); |
1895 | } | |
1896 | ||
1897 | if (!ifobj2) | |
1898 | pthread_kill(t0, SIGUSR1); | |
1899 | else | |
1900 | pthread_join(t0, NULL); | |
3143d10b | 1901 | |
a693ff3e | 1902 | if (test->total_steps == test->current_step || test->fail) { |
8913e653 TV |
1903 | u32 i; |
1904 | ||
7f881984 | 1905 | if (ifobj2) |
8913e653 TV |
1906 | for (i = 0; i < test->nb_sockets; i++) |
1907 | xsk_socket__delete(ifobj2->xsk_arr[i].xsk); | |
1908 | ||
1909 | for (i = 0; i < test->nb_sockets; i++) | |
1910 | xsk_socket__delete(ifobj1->xsk_arr[i].xsk); | |
1911 | ||
7f881984 MK |
1912 | testapp_clean_xsk_umem(ifobj1); |
1913 | if (ifobj2 && !ifobj2->shared_umem) | |
1914 | testapp_clean_xsk_umem(ifobj2); | |
a693ff3e MF |
1915 | } |
1916 | ||
3143d10b SKR |
1917 | return !!test->fail; |
1918 | } | |
1919 | ||
895b62ee | 1920 | static int testapp_validate_traffic(struct test_spec *test) |
facb7cb2 | 1921 | { |
7d8319a7 MK |
1922 | struct ifobject *ifobj_rx = test->ifobj_rx; |
1923 | struct ifobject *ifobj_tx = test->ifobj_tx; | |
1924 | ||
041b68f6 MK |
1925 | if ((ifobj_rx->umem->unaligned_mode && !ifobj_rx->unaligned_supp) || |
1926 | (ifobj_tx->umem->unaligned_mode && !ifobj_tx->unaligned_supp)) { | |
1927 | ksft_test_result_skip("No huge pages present.\n"); | |
1928 | return TEST_SKIP; | |
1929 | } | |
1930 | ||
776021e0 | 1931 | if (test->set_ring) { |
d80d61ab TV |
1932 | if (ifobj_tx->hw_ring_size_supp) { |
1933 | if (set_ring_size(ifobj_tx)) { | |
1934 | ksft_test_result_skip("Failed to change HW ring size.\n"); | |
1935 | return TEST_FAILURE; | |
1936 | } | |
1937 | } else { | |
1938 | ksft_test_result_skip("Changing HW ring size not supported.\n"); | |
1939 | return TEST_SKIP; | |
1940 | } | |
776021e0 TV |
1941 | } |
1942 | ||
7d8319a7 MK |
1943 | xsk_attach_xdp_progs(test, ifobj_rx, ifobj_tx); |
1944 | return __testapp_validate_traffic(test, ifobj_rx, ifobj_tx); | |
7f881984 | 1945 | } |
a693ff3e | 1946 | |
7f881984 MK |
1947 | static int testapp_validate_traffic_single_thread(struct test_spec *test, struct ifobject *ifobj) |
1948 | { | |
1949 | return __testapp_validate_traffic(test, ifobj, NULL); | |
6674bf66 WJ |
1950 | } |
1951 | ||
041b68f6 | 1952 | static int testapp_teardown(struct test_spec *test) |
9445f8c7 MF |
1953 | { |
1954 | int i; | |
1955 | ||
1956 | for (i = 0; i < MAX_TEARDOWN_ITER; i++) { | |
895b62ee | 1957 | if (testapp_validate_traffic(test)) |
041b68f6 | 1958 | return TEST_FAILURE; |
ce74acaf | 1959 | test_spec_reset(test); |
9445f8c7 | 1960 | } |
041b68f6 MK |
1961 | |
1962 | return TEST_PASS; | |
9445f8c7 MF |
1963 | } |
1964 | ||
ce74acaf | 1965 | static void swap_directions(struct ifobject **ifobj1, struct ifobject **ifobj2) |
9445f8c7 | 1966 | { |
ce74acaf | 1967 | thread_func_t tmp_func_ptr = (*ifobj1)->func_ptr; |
ce74acaf | 1968 | struct ifobject *tmp_ifobj = (*ifobj1); |
9445f8c7 | 1969 | |
ce74acaf | 1970 | (*ifobj1)->func_ptr = (*ifobj2)->func_ptr; |
ce74acaf | 1971 | (*ifobj2)->func_ptr = tmp_func_ptr; |
9445f8c7 | 1972 | |
ce74acaf MK |
1973 | *ifobj1 = *ifobj2; |
1974 | *ifobj2 = tmp_ifobj; | |
9445f8c7 MF |
1975 | } |
1976 | ||
f20fbcd0 | 1977 | static int testapp_bidirectional(struct test_spec *test) |
6674bf66 | 1978 | { |
041b68f6 MK |
1979 | int res; |
1980 | ||
1856c24d MK |
1981 | test->ifobj_tx->rx_on = true; |
1982 | test->ifobj_rx->tx_on = true; | |
55be575d | 1983 | test->total_steps = 2; |
895b62ee | 1984 | if (testapp_validate_traffic(test)) |
041b68f6 | 1985 | return TEST_FAILURE; |
55be575d | 1986 | |
2d2712ca | 1987 | print_verbose("Switching Tx/Rx direction\n"); |
55be575d | 1988 | swap_directions(&test->ifobj_rx, &test->ifobj_tx); |
041b68f6 | 1989 | res = __testapp_validate_traffic(test, test->ifobj_rx, test->ifobj_tx); |
6674bf66 | 1990 | |
ce74acaf | 1991 | swap_directions(&test->ifobj_rx, &test->ifobj_tx); |
041b68f6 | 1992 | return res; |
facb7cb2 WJ |
1993 | } |
1994 | ||
8913e653 | 1995 | static int swap_xsk_resources(struct test_spec *test) |
27e1ca25 | 1996 | { |
f0a249df | 1997 | int ret; |
3b22523b | 1998 | |
8913e653 TV |
1999 | test->ifobj_tx->xsk_arr[0].pkt_stream = NULL; |
2000 | test->ifobj_rx->xsk_arr[0].pkt_stream = NULL; | |
2001 | test->ifobj_tx->xsk_arr[1].pkt_stream = test->tx_pkt_stream_default; | |
2002 | test->ifobj_rx->xsk_arr[1].pkt_stream = test->rx_pkt_stream_default; | |
2003 | test->ifobj_tx->xsk = &test->ifobj_tx->xsk_arr[1]; | |
2004 | test->ifobj_rx->xsk = &test->ifobj_rx->xsk_arr[1]; | |
3b22523b | 2005 | |
fc2cb864 | 2006 | ret = xsk_update_xskmap(test->ifobj_rx->xskmap, test->ifobj_rx->xsk->xsk, 0); |
3b22523b | 2007 | if (ret) |
5fc494d5 MK |
2008 | return TEST_FAILURE; |
2009 | ||
2010 | return TEST_PASS; | |
27e1ca25 MF |
2011 | } |
2012 | ||
f20fbcd0 | 2013 | static int testapp_xdp_prog_cleanup(struct test_spec *test) |
27e1ca25 | 2014 | { |
55be575d | 2015 | test->total_steps = 2; |
85c6c957 | 2016 | test->nb_sockets = 2; |
895b62ee | 2017 | if (testapp_validate_traffic(test)) |
041b68f6 | 2018 | return TEST_FAILURE; |
55be575d | 2019 | |
8913e653 | 2020 | if (swap_xsk_resources(test)) |
5fc494d5 | 2021 | return TEST_FAILURE; |
041b68f6 | 2022 | return testapp_validate_traffic(test); |
27e1ca25 MF |
2023 | } |
2024 | ||
041b68f6 | 2025 | static int testapp_headroom(struct test_spec *test) |
e34087fc | 2026 | { |
e34087fc | 2027 | test->ifobj_rx->umem->frame_headroom = UMEM_HEADROOM_TEST_SIZE; |
041b68f6 | 2028 | return testapp_validate_traffic(test); |
e34087fc MK |
2029 | } |
2030 | ||
041b68f6 | 2031 | static int testapp_stats_rx_dropped(struct test_spec *test) |
b267e5a4 | 2032 | { |
041b68f6 MK |
2033 | if (test->mode == TEST_MODE_ZC) { |
2034 | ksft_test_result_skip("Can not run RX_DROPPED test for ZC mode\n"); | |
2035 | return TEST_SKIP; | |
2036 | } | |
2037 | ||
a693ff3e | 2038 | pkt_stream_replace_half(test, MIN_PKT_SIZE * 4, 0); |
4fec7028 | 2039 | test->ifobj_rx->umem->frame_headroom = test->ifobj_rx->umem->frame_size - |
27e934be | 2040 | XDP_PACKET_HEADROOM - MIN_PKT_SIZE * 3; |
27e934be | 2041 | pkt_stream_receive_half(test); |
4fec7028 | 2042 | test->ifobj_rx->validation_func = validate_rx_dropped; |
041b68f6 | 2043 | return testapp_validate_traffic(test); |
4fec7028 | 2044 | } |
605091c5 | 2045 | |
041b68f6 | 2046 | static int testapp_stats_tx_invalid_descs(struct test_spec *test) |
4fec7028 | 2047 | { |
27e934be | 2048 | pkt_stream_replace_half(test, XSK_UMEM__INVALID_FRAME_SIZE, 0); |
4fec7028 | 2049 | test->ifobj_tx->validation_func = validate_tx_invalid_descs; |
041b68f6 | 2050 | return testapp_validate_traffic(test); |
4fec7028 | 2051 | } |
895b62ee | 2052 | |
041b68f6 | 2053 | static int testapp_stats_rx_full(struct test_spec *test) |
4fec7028 | 2054 | { |
df82d2e8 | 2055 | pkt_stream_replace(test, DEFAULT_UMEM_BUFFERS + DEFAULT_UMEM_BUFFERS / 2, MIN_PKT_SIZE); |
46e43786 | 2056 | test->ifobj_rx->xsk->pkt_stream = pkt_stream_generate(DEFAULT_UMEM_BUFFERS, MIN_PKT_SIZE); |
27e934be MK |
2057 | |
2058 | test->ifobj_rx->xsk->rxqsize = DEFAULT_UMEM_BUFFERS; | |
2059 | test->ifobj_rx->release_rx = false; | |
4fec7028 | 2060 | test->ifobj_rx->validation_func = validate_rx_full; |
041b68f6 | 2061 | return testapp_validate_traffic(test); |
4fec7028 | 2062 | } |
b267e5a4 | 2063 | |
041b68f6 | 2064 | static int testapp_stats_fill_empty(struct test_spec *test) |
4fec7028 | 2065 | { |
df82d2e8 | 2066 | pkt_stream_replace(test, DEFAULT_UMEM_BUFFERS + DEFAULT_UMEM_BUFFERS / 2, MIN_PKT_SIZE); |
46e43786 | 2067 | test->ifobj_rx->xsk->pkt_stream = pkt_stream_generate(DEFAULT_UMEM_BUFFERS, MIN_PKT_SIZE); |
27e934be MK |
2068 | |
2069 | test->ifobj_rx->use_fill_ring = false; | |
4fec7028 | 2070 | test->ifobj_rx->validation_func = validate_fill_empty; |
041b68f6 | 2071 | return testapp_validate_traffic(test); |
b267e5a4 CL |
2072 | } |
2073 | ||
f20fbcd0 | 2074 | static int testapp_send_receive_unaligned(struct test_spec *test) |
a4ba98dd | 2075 | { |
a4ba98dd MK |
2076 | test->ifobj_tx->umem->unaligned_mode = true; |
2077 | test->ifobj_rx->umem->unaligned_mode = true; | |
d9f6d970 | 2078 | /* Let half of the packets straddle a 4K buffer boundary */ |
df82d2e8 | 2079 | pkt_stream_replace_half(test, MIN_PKT_SIZE, -MIN_PKT_SIZE / 2); |
a4ba98dd | 2080 | |
041b68f6 | 2081 | return testapp_validate_traffic(test); |
a4ba98dd MK |
2082 | } |
2083 | ||
f20fbcd0 | 2084 | static int testapp_send_receive_unaligned_mb(struct test_spec *test) |
1005a226 | 2085 | { |
1005a226 MK |
2086 | test->mtu = MAX_ETH_JUMBO_SIZE; |
2087 | test->ifobj_tx->umem->unaligned_mode = true; | |
2088 | test->ifobj_rx->umem->unaligned_mode = true; | |
2089 | pkt_stream_replace(test, DEFAULT_PKT_CNT, MAX_ETH_JUMBO_SIZE); | |
2090 | return testapp_validate_traffic(test); | |
2091 | } | |
2092 | ||
041b68f6 | 2093 | static int testapp_single_pkt(struct test_spec *test) |
96a40678 | 2094 | { |
d9f6d970 | 2095 | struct pkt pkts[] = {{0, MIN_PKT_SIZE, 0, true}}; |
96a40678 MK |
2096 | |
2097 | pkt_stream_generate_custom(test, pkts, ARRAY_SIZE(pkts)); | |
041b68f6 | 2098 | return testapp_validate_traffic(test); |
96a40678 MK |
2099 | } |
2100 | ||
f20fbcd0 | 2101 | static int testapp_send_receive_mb(struct test_spec *test) |
f540d44e | 2102 | { |
f540d44e MK |
2103 | test->mtu = MAX_ETH_JUMBO_SIZE; |
2104 | pkt_stream_replace(test, DEFAULT_PKT_CNT, MAX_ETH_JUMBO_SIZE); | |
2105 | ||
2106 | return testapp_validate_traffic(test); | |
2107 | } | |
2108 | ||
69760449 MK |
2109 | static int testapp_invalid_desc_mb(struct test_spec *test) |
2110 | { | |
2111 | struct xsk_umem_info *umem = test->ifobj_tx->umem; | |
2112 | u64 umem_size = umem->num_frames * umem->frame_size; | |
2113 | struct pkt pkts[] = { | |
2114 | /* Valid packet for synch to start with */ | |
2115 | {0, MIN_PKT_SIZE, 0, true, 0}, | |
2116 | /* Zero frame len is not legal */ | |
2117 | {0, XSK_UMEM__LARGE_FRAME_SIZE, 0, false, XDP_PKT_CONTD}, | |
2118 | {0, XSK_UMEM__LARGE_FRAME_SIZE, 0, false, XDP_PKT_CONTD}, | |
2119 | {0, 0, 0, false, 0}, | |
2120 | /* Invalid address in the second frame */ | |
2121 | {0, XSK_UMEM__LARGE_FRAME_SIZE, 0, false, XDP_PKT_CONTD}, | |
2122 | {umem_size, XSK_UMEM__LARGE_FRAME_SIZE, 0, false, XDP_PKT_CONTD}, | |
2123 | /* Invalid len in the middle */ | |
2124 | {0, XSK_UMEM__LARGE_FRAME_SIZE, 0, false, XDP_PKT_CONTD}, | |
2125 | {0, XSK_UMEM__INVALID_FRAME_SIZE, 0, false, XDP_PKT_CONTD}, | |
2126 | /* Invalid options in the middle */ | |
2127 | {0, XSK_UMEM__LARGE_FRAME_SIZE, 0, false, XDP_PKT_CONTD}, | |
2128 | {0, XSK_UMEM__LARGE_FRAME_SIZE, 0, false, XSK_DESC__INVALID_OPTION}, | |
2129 | /* Transmit 2 frags, receive 3 */ | |
2130 | {0, XSK_UMEM__MAX_FRAME_SIZE, 0, true, XDP_PKT_CONTD}, | |
2131 | {0, XSK_UMEM__MAX_FRAME_SIZE, 0, true, 0}, | |
2132 | /* Middle frame crosses chunk boundary with small length */ | |
2133 | {0, XSK_UMEM__LARGE_FRAME_SIZE, 0, false, XDP_PKT_CONTD}, | |
2134 | {-MIN_PKT_SIZE / 2, MIN_PKT_SIZE, 0, false, 0}, | |
2135 | /* Valid packet for synch so that something is received */ | |
2136 | {0, MIN_PKT_SIZE, 0, true, 0}}; | |
2137 | ||
2138 | if (umem->unaligned_mode) { | |
2139 | /* Crossing a chunk boundary allowed */ | |
2140 | pkts[12].valid = true; | |
2141 | pkts[13].valid = true; | |
2142 | } | |
2143 | ||
2144 | test->mtu = MAX_ETH_JUMBO_SIZE; | |
2145 | pkt_stream_generate_custom(test, pkts, ARRAY_SIZE(pkts)); | |
2146 | return testapp_validate_traffic(test); | |
2147 | } | |
2148 | ||
041b68f6 | 2149 | static int testapp_invalid_desc(struct test_spec *test) |
0d1b7f3a | 2150 | { |
d9f6d970 MK |
2151 | struct xsk_umem_info *umem = test->ifobj_tx->umem; |
2152 | u64 umem_size = umem->num_frames * umem->frame_size; | |
0d1b7f3a | 2153 | struct pkt pkts[] = { |
f3e619bb | 2154 | /* Zero packet address allowed */ |
df82d2e8 | 2155 | {0, MIN_PKT_SIZE, 0, true}, |
f3e619bb | 2156 | /* Allowed packet */ |
d9f6d970 | 2157 | {0, MIN_PKT_SIZE, 0, true}, |
0d1b7f3a | 2158 | /* Straddling the start of umem */ |
df82d2e8 | 2159 | {-2, MIN_PKT_SIZE, 0, false}, |
0d1b7f3a | 2160 | /* Packet too large */ |
d9f6d970 | 2161 | {0, XSK_UMEM__INVALID_FRAME_SIZE, 0, false}, |
ccd1b293 | 2162 | /* Up to end of umem allowed */ |
d9f6d970 | 2163 | {umem_size - MIN_PKT_SIZE - 2 * umem->frame_size, MIN_PKT_SIZE, 0, true}, |
0d1b7f3a | 2164 | /* After umem ends */ |
df82d2e8 | 2165 | {umem_size, MIN_PKT_SIZE, 0, false}, |
0d1b7f3a | 2166 | /* Straddle the end of umem */ |
df82d2e8 | 2167 | {umem_size - MIN_PKT_SIZE / 2, MIN_PKT_SIZE, 0, false}, |
d9f6d970 MK |
2168 | /* Straddle a 4K boundary */ |
2169 | {0x1000 - MIN_PKT_SIZE / 2, MIN_PKT_SIZE, 0, false}, | |
2170 | /* Straddle a 2K boundary */ | |
2171 | {0x800 - MIN_PKT_SIZE / 2, MIN_PKT_SIZE, 0, true}, | |
0d1b7f3a | 2172 | /* Valid packet for synch so that something is received */ |
d9f6d970 | 2173 | {0, MIN_PKT_SIZE, 0, true}}; |
0d1b7f3a | 2174 | |
d9f6d970 MK |
2175 | if (umem->unaligned_mode) { |
2176 | /* Crossing a page boundary allowed */ | |
ccd1b293 | 2177 | pkts[7].valid = true; |
0d1b7f3a | 2178 | } |
d9f6d970 MK |
2179 | if (umem->frame_size == XSK_UMEM__DEFAULT_FRAME_SIZE / 2) { |
2180 | /* Crossing a 2K frame size boundary not allowed */ | |
ccd1b293 | 2181 | pkts[8].valid = false; |
909f0e28 MK |
2182 | } |
2183 | ||
a693ff3e | 2184 | if (test->ifobj_tx->shared_umem) { |
d9f6d970 MK |
2185 | pkts[4].offset += umem_size; |
2186 | pkts[5].offset += umem_size; | |
2187 | pkts[6].offset += umem_size; | |
a693ff3e MF |
2188 | } |
2189 | ||
0d1b7f3a | 2190 | pkt_stream_generate_custom(test, pkts, ARRAY_SIZE(pkts)); |
041b68f6 | 2191 | return testapp_validate_traffic(test); |
0d1b7f3a MK |
2192 | } |
2193 | ||
041b68f6 | 2194 | static int testapp_xdp_drop(struct test_spec *test) |
80bea9ac | 2195 | { |
7d8319a7 MK |
2196 | struct xsk_xdp_progs *skel_rx = test->ifobj_rx->xdp_progs; |
2197 | struct xsk_xdp_progs *skel_tx = test->ifobj_tx->xdp_progs; | |
80bea9ac | 2198 | |
7d8319a7 MK |
2199 | test_spec_set_xdp_prog(test, skel_rx->progs.xsk_xdp_drop, skel_tx->progs.xsk_xdp_drop, |
2200 | skel_rx->maps.xsk, skel_tx->maps.xsk); | |
80bea9ac MK |
2201 | |
2202 | pkt_stream_receive_half(test); | |
041b68f6 | 2203 | return testapp_validate_traffic(test); |
80bea9ac MK |
2204 | } |
2205 | ||
13c341c4 | 2206 | static int testapp_xdp_metadata_copy(struct test_spec *test) |
9a321fd3 TV |
2207 | { |
2208 | struct xsk_xdp_progs *skel_rx = test->ifobj_rx->xdp_progs; | |
2209 | struct xsk_xdp_progs *skel_tx = test->ifobj_tx->xdp_progs; | |
2210 | struct bpf_map *data_map; | |
2211 | int count = 0; | |
2212 | int key = 0; | |
2213 | ||
9a321fd3 TV |
2214 | test_spec_set_xdp_prog(test, skel_rx->progs.xsk_xdp_populate_metadata, |
2215 | skel_tx->progs.xsk_xdp_populate_metadata, | |
2216 | skel_rx->maps.xsk, skel_tx->maps.xsk); | |
2217 | test->ifobj_rx->use_metadata = true; | |
2218 | ||
2219 | data_map = bpf_object__find_map_by_name(skel_rx->obj, "xsk_xdp_.bss"); | |
5fc494d5 MK |
2220 | if (!data_map || !bpf_map__is_internal(data_map)) { |
2221 | ksft_print_msg("Error: could not find bss section of XDP program\n"); | |
2222 | return TEST_FAILURE; | |
2223 | } | |
9a321fd3 | 2224 | |
5fc494d5 MK |
2225 | if (bpf_map_update_elem(bpf_map__fd(data_map), &key, &count, BPF_ANY)) { |
2226 | ksft_print_msg("Error: could not update count element\n"); | |
2227 | return TEST_FAILURE; | |
2228 | } | |
9a321fd3 | 2229 | |
041b68f6 | 2230 | return testapp_validate_traffic(test); |
9a321fd3 TV |
2231 | } |
2232 | ||
6d198a89 TV |
2233 | static int testapp_xdp_shared_umem(struct test_spec *test) |
2234 | { | |
2235 | struct xsk_xdp_progs *skel_rx = test->ifobj_rx->xdp_progs; | |
2236 | struct xsk_xdp_progs *skel_tx = test->ifobj_tx->xdp_progs; | |
2237 | ||
2238 | test->total_steps = 1; | |
2239 | test->nb_sockets = 2; | |
2240 | ||
2241 | test_spec_set_xdp_prog(test, skel_rx->progs.xsk_xdp_shared_umem, | |
2242 | skel_tx->progs.xsk_xdp_shared_umem, | |
2243 | skel_rx->maps.xsk, skel_tx->maps.xsk); | |
2244 | ||
2245 | pkt_stream_even_odd_sequence(test); | |
2246 | ||
2247 | return testapp_validate_traffic(test); | |
2248 | } | |
2249 | ||
041b68f6 | 2250 | static int testapp_poll_txq_tmout(struct test_spec *test) |
7f881984 | 2251 | { |
7f881984 MK |
2252 | test->ifobj_tx->use_poll = true; |
2253 | /* create invalid frame by set umem frame_size and pkt length equal to 2048 */ | |
2254 | test->ifobj_tx->umem->frame_size = 2048; | |
2255 | pkt_stream_replace(test, 2 * DEFAULT_PKT_CNT, 2048); | |
041b68f6 | 2256 | return testapp_validate_traffic_single_thread(test, test->ifobj_tx); |
7f881984 MK |
2257 | } |
2258 | ||
041b68f6 | 2259 | static int testapp_poll_rxq_tmout(struct test_spec *test) |
7f881984 | 2260 | { |
7f881984 | 2261 | test->ifobj_rx->use_poll = true; |
041b68f6 | 2262 | return testapp_validate_traffic_single_thread(test, test->ifobj_rx); |
7f881984 MK |
2263 | } |
2264 | ||
807bf4da MK |
2265 | static int testapp_too_many_frags(struct test_spec *test) |
2266 | { | |
d41905b3 | 2267 | struct pkt *pkts; |
807bf4da | 2268 | u32 max_frags, i; |
d41905b3 | 2269 | int ret; |
807bf4da | 2270 | |
d41905b3 | 2271 | if (test->mode == TEST_MODE_ZC) { |
807bf4da | 2272 | max_frags = test->ifobj_tx->xdp_zc_max_segs; |
d41905b3 MF |
2273 | } else { |
2274 | max_frags = get_max_skb_frags(); | |
2275 | if (!max_frags) { | |
2276 | ksft_print_msg("Couldn't retrieve MAX_SKB_FRAGS from system, using default (17) value\n"); | |
2277 | max_frags = 17; | |
2278 | } | |
2279 | max_frags += 1; | |
2280 | } | |
2281 | ||
2282 | pkts = calloc(2 * max_frags + 2, sizeof(struct pkt)); | |
2283 | if (!pkts) | |
2284 | return TEST_FAILURE; | |
807bf4da MK |
2285 | |
2286 | test->mtu = MAX_ETH_JUMBO_SIZE; | |
2287 | ||
2288 | /* Valid packet for synch */ | |
2289 | pkts[0].len = MIN_PKT_SIZE; | |
2290 | pkts[0].valid = true; | |
2291 | ||
2292 | /* One valid packet with the max amount of frags */ | |
2293 | for (i = 1; i < max_frags + 1; i++) { | |
2294 | pkts[i].len = MIN_PKT_SIZE; | |
2295 | pkts[i].options = XDP_PKT_CONTD; | |
2296 | pkts[i].valid = true; | |
2297 | } | |
2298 | pkts[max_frags].options = 0; | |
2299 | ||
2300 | /* An invalid packet with the max amount of frags but signals packet | |
2301 | * continues on the last frag | |
2302 | */ | |
2303 | for (i = max_frags + 1; i < 2 * max_frags + 1; i++) { | |
2304 | pkts[i].len = MIN_PKT_SIZE; | |
2305 | pkts[i].options = XDP_PKT_CONTD; | |
2306 | pkts[i].valid = false; | |
2307 | } | |
2308 | ||
2309 | /* Valid packet for synch */ | |
2310 | pkts[2 * max_frags + 1].len = MIN_PKT_SIZE; | |
2311 | pkts[2 * max_frags + 1].valid = true; | |
2312 | ||
2313 | pkt_stream_generate_custom(test, pkts, 2 * max_frags + 2); | |
d41905b3 MF |
2314 | ret = testapp_validate_traffic(test); |
2315 | ||
2316 | free(pkts); | |
2317 | return ret; | |
807bf4da MK |
2318 | } |
2319 | ||
f0a249df MK |
2320 | static int xsk_load_xdp_programs(struct ifobject *ifobj) |
2321 | { | |
2322 | ifobj->xdp_progs = xsk_xdp_progs__open_and_load(); | |
2323 | if (libbpf_get_error(ifobj->xdp_progs)) | |
2324 | return libbpf_get_error(ifobj->xdp_progs); | |
2325 | ||
2326 | return 0; | |
2327 | } | |
2328 | ||
2329 | static void xsk_unload_xdp_programs(struct ifobject *ifobj) | |
2330 | { | |
2331 | xsk_xdp_progs__destroy(ifobj->xdp_progs); | |
2332 | } | |
2333 | ||
041b68f6 MK |
2334 | /* Simple test */ |
2335 | static bool hugepages_present(void) | |
2336 | { | |
2337 | size_t mmap_sz = 2 * DEFAULT_UMEM_BUFFERS * XSK_UMEM__DEFAULT_FRAME_SIZE; | |
2338 | void *bufs; | |
2339 | ||
2340 | bufs = mmap(NULL, mmap_sz, PROT_READ | PROT_WRITE, | |
2341 | MAP_PRIVATE | MAP_ANONYMOUS | MAP_HUGETLB, -1, MAP_HUGE_2MB); | |
2342 | if (bufs == MAP_FAILED) | |
2343 | return false; | |
2344 | ||
2345 | mmap_sz = ceil_u64(mmap_sz, HUGEPAGE_SIZE) * HUGEPAGE_SIZE; | |
2346 | munmap(bufs, mmap_sz); | |
2347 | return true; | |
2348 | } | |
2349 | ||
985fd214 | 2350 | static void init_iface(struct ifobject *ifobj, thread_func_t func_ptr) |
facb7cb2 | 2351 | { |
f540d44e | 2352 | LIBBPF_OPTS(bpf_xdp_query_opts, query_opts); |
f0a249df | 2353 | int err; |
e623bfde | 2354 | |
744eb5c8 | 2355 | ifobj->func_ptr = func_ptr; |
aa61d81f | 2356 | |
f0a249df | 2357 | err = xsk_load_xdp_programs(ifobj); |
aa61d81f | 2358 | if (err) { |
7c3fcf08 | 2359 | ksft_print_msg("Error loading XDP program\n"); |
aa61d81f MK |
2360 | exit_with_error(err); |
2361 | } | |
041b68f6 MK |
2362 | |
2363 | if (hugepages_present()) | |
2364 | ifobj->unaligned_supp = true; | |
f540d44e MK |
2365 | |
2366 | err = bpf_xdp_query(ifobj->ifindex, XDP_FLAGS_DRV_MODE, &query_opts); | |
2367 | if (err) { | |
13fd5e14 | 2368 | ksft_print_msg("Error querying XDP capabilities\n"); |
f540d44e MK |
2369 | exit_with_error(-err); |
2370 | } | |
2371 | if (query_opts.feature_flags & NETDEV_XDP_ACT_RX_SG) | |
2372 | ifobj->multi_buff_supp = true; | |
807bf4da MK |
2373 | if (query_opts.feature_flags & NETDEV_XDP_ACT_XSK_ZEROCOPY) { |
2374 | if (query_opts.xdp_zc_max_segs > 1) { | |
f540d44e | 2375 | ifobj->multi_buff_zc_supp = true; |
807bf4da MK |
2376 | ifobj->xdp_zc_max_segs = query_opts.xdp_zc_max_segs; |
2377 | } else { | |
2378 | ifobj->xdp_zc_max_segs = 0; | |
2379 | } | |
2380 | } | |
facb7cb2 WJ |
2381 | } |
2382 | ||
13c341c4 MK |
2383 | static int testapp_send_receive(struct test_spec *test) |
2384 | { | |
13c341c4 MK |
2385 | return testapp_validate_traffic(test); |
2386 | } | |
2387 | ||
2388 | static int testapp_send_receive_2k_frame(struct test_spec *test) | |
2389 | { | |
13c341c4 MK |
2390 | test->ifobj_tx->umem->frame_size = 2048; |
2391 | test->ifobj_rx->umem->frame_size = 2048; | |
2392 | pkt_stream_replace(test, DEFAULT_PKT_CNT, MIN_PKT_SIZE); | |
2393 | return testapp_validate_traffic(test); | |
2394 | } | |
2395 | ||
2396 | static int testapp_poll_rx(struct test_spec *test) | |
2397 | { | |
2398 | test->ifobj_rx->use_poll = true; | |
13c341c4 MK |
2399 | return testapp_validate_traffic(test); |
2400 | } | |
2401 | ||
2402 | static int testapp_poll_tx(struct test_spec *test) | |
2403 | { | |
2404 | test->ifobj_tx->use_poll = true; | |
13c341c4 MK |
2405 | return testapp_validate_traffic(test); |
2406 | } | |
2407 | ||
2408 | static int testapp_aligned_inv_desc(struct test_spec *test) | |
2409 | { | |
13c341c4 MK |
2410 | return testapp_invalid_desc(test); |
2411 | } | |
2412 | ||
2413 | static int testapp_aligned_inv_desc_2k_frame(struct test_spec *test) | |
2414 | { | |
13c341c4 MK |
2415 | test->ifobj_tx->umem->frame_size = 2048; |
2416 | test->ifobj_rx->umem->frame_size = 2048; | |
2417 | return testapp_invalid_desc(test); | |
2418 | } | |
2419 | ||
2420 | static int testapp_unaligned_inv_desc(struct test_spec *test) | |
2421 | { | |
13c341c4 MK |
2422 | test->ifobj_tx->umem->unaligned_mode = true; |
2423 | test->ifobj_rx->umem->unaligned_mode = true; | |
2424 | return testapp_invalid_desc(test); | |
2425 | } | |
2426 | ||
2427 | static int testapp_unaligned_inv_desc_4001_frame(struct test_spec *test) | |
2428 | { | |
2429 | u64 page_size, umem_size; | |
2430 | ||
13c341c4 MK |
2431 | /* Odd frame size so the UMEM doesn't end near a page boundary. */ |
2432 | test->ifobj_tx->umem->frame_size = 4001; | |
2433 | test->ifobj_rx->umem->frame_size = 4001; | |
2434 | test->ifobj_tx->umem->unaligned_mode = true; | |
2435 | test->ifobj_rx->umem->unaligned_mode = true; | |
2436 | /* This test exists to test descriptors that staddle the end of | |
2437 | * the UMEM but not a page. | |
2438 | */ | |
2439 | page_size = sysconf(_SC_PAGESIZE); | |
2440 | umem_size = test->ifobj_tx->umem->num_frames * test->ifobj_tx->umem->frame_size; | |
2441 | assert(umem_size % page_size > MIN_PKT_SIZE); | |
2442 | assert(umem_size % page_size < page_size - MIN_PKT_SIZE); | |
2443 | ||
2444 | return testapp_invalid_desc(test); | |
2445 | } | |
2446 | ||
2447 | static int testapp_aligned_inv_desc_mb(struct test_spec *test) | |
2448 | { | |
13c341c4 MK |
2449 | return testapp_invalid_desc_mb(test); |
2450 | } | |
2451 | ||
2452 | static int testapp_unaligned_inv_desc_mb(struct test_spec *test) | |
2453 | { | |
13c341c4 MK |
2454 | test->ifobj_tx->umem->unaligned_mode = true; |
2455 | test->ifobj_rx->umem->unaligned_mode = true; | |
2456 | return testapp_invalid_desc_mb(test); | |
2457 | } | |
2458 | ||
2459 | static int testapp_xdp_metadata(struct test_spec *test) | |
2460 | { | |
13c341c4 MK |
2461 | return testapp_xdp_metadata_copy(test); |
2462 | } | |
2463 | ||
2464 | static int testapp_xdp_metadata_mb(struct test_spec *test) | |
2465 | { | |
13c341c4 MK |
2466 | test->mtu = MAX_ETH_JUMBO_SIZE; |
2467 | return testapp_xdp_metadata_copy(test); | |
2468 | } | |
2469 | ||
c4f96053 TV |
2470 | static int testapp_hw_sw_min_ring_size(struct test_spec *test) |
2471 | { | |
2472 | int ret; | |
2473 | ||
2474 | test->set_ring = true; | |
2475 | test->total_steps = 2; | |
2476 | test->ifobj_tx->ring.tx_pending = DEFAULT_BATCH_SIZE; | |
2477 | test->ifobj_tx->ring.rx_pending = DEFAULT_BATCH_SIZE * 2; | |
2478 | test->ifobj_tx->xsk->batch_size = 1; | |
2479 | test->ifobj_rx->xsk->batch_size = 1; | |
2480 | ret = testapp_validate_traffic(test); | |
2481 | if (ret) | |
2482 | return ret; | |
2483 | ||
2484 | /* Set batch size to hw_ring_size - 1 */ | |
2485 | test->ifobj_tx->xsk->batch_size = DEFAULT_BATCH_SIZE - 1; | |
2486 | test->ifobj_rx->xsk->batch_size = DEFAULT_BATCH_SIZE - 1; | |
2487 | return testapp_validate_traffic(test); | |
2488 | } | |
2489 | ||
c53908b2 TV |
2490 | static int testapp_hw_sw_max_ring_size(struct test_spec *test) |
2491 | { | |
e4a195e2 | 2492 | u32 max_descs = XSK_RING_PROD__DEFAULT_NUM_DESCS * 4; |
c53908b2 TV |
2493 | int ret; |
2494 | ||
2495 | test->set_ring = true; | |
2496 | test->total_steps = 2; | |
2497 | test->ifobj_tx->ring.tx_pending = test->ifobj_tx->ring.tx_max_pending; | |
2498 | test->ifobj_tx->ring.rx_pending = test->ifobj_tx->ring.rx_max_pending; | |
2499 | test->ifobj_rx->umem->num_frames = max_descs; | |
e4a195e2 TV |
2500 | test->ifobj_rx->umem->fill_size = max_descs; |
2501 | test->ifobj_rx->umem->comp_size = max_descs; | |
c53908b2 TV |
2502 | test->ifobj_tx->xsk->batch_size = XSK_RING_PROD__DEFAULT_NUM_DESCS; |
2503 | test->ifobj_rx->xsk->batch_size = XSK_RING_PROD__DEFAULT_NUM_DESCS; | |
2504 | ||
2505 | ret = testapp_validate_traffic(test); | |
2506 | if (ret) | |
2507 | return ret; | |
2508 | ||
e4a195e2 TV |
2509 | /* Set batch_size to 8152 for testing, as the ice HW ignores the 3 lowest bits when |
2510 | * updating the Rx HW tail register. | |
2511 | */ | |
2512 | test->ifobj_tx->xsk->batch_size = test->ifobj_tx->ring.tx_max_pending - 8; | |
2513 | test->ifobj_rx->xsk->batch_size = test->ifobj_tx->ring.tx_max_pending - 8; | |
2514 | pkt_stream_replace(test, max_descs, MIN_PKT_SIZE); | |
c53908b2 TV |
2515 | return testapp_validate_traffic(test); |
2516 | } | |
2517 | ||
f20fbcd0 MK |
2518 | static void run_pkt_test(struct test_spec *test) |
2519 | { | |
2520 | int ret; | |
2521 | ||
2522 | ret = test->test_func(test); | |
53cb3cec | 2523 | |
041b68f6 | 2524 | if (ret == TEST_PASS) |
895b62ee MK |
2525 | ksft_test_result_pass("PASS: %s %s%s\n", mode_string(test), busy_poll_string(test), |
2526 | test->name); | |
e67b2554 | 2527 | pkt_stream_restore_default(test); |
d3e3bf5b CL |
2528 | } |
2529 | ||
1034b03e MK |
2530 | static struct ifobject *ifobject_create(void) |
2531 | { | |
2532 | struct ifobject *ifobj; | |
2533 | ||
2534 | ifobj = calloc(1, sizeof(struct ifobject)); | |
2535 | if (!ifobj) | |
2536 | return NULL; | |
2537 | ||
ed7b74dc | 2538 | ifobj->xsk_arr = calloc(MAX_SOCKETS, sizeof(*ifobj->xsk_arr)); |
1034b03e MK |
2539 | if (!ifobj->xsk_arr) |
2540 | goto out_xsk_arr; | |
2541 | ||
3b22523b MK |
2542 | ifobj->umem = calloc(1, sizeof(*ifobj->umem)); |
2543 | if (!ifobj->umem) | |
2544 | goto out_umem; | |
1034b03e MK |
2545 | |
2546 | return ifobj; | |
2547 | ||
3b22523b | 2548 | out_umem: |
1034b03e MK |
2549 | free(ifobj->xsk_arr); |
2550 | out_xsk_arr: | |
2551 | free(ifobj); | |
2552 | return NULL; | |
2553 | } | |
2554 | ||
2555 | static void ifobject_delete(struct ifobject *ifobj) | |
2556 | { | |
3b22523b | 2557 | free(ifobj->umem); |
1034b03e MK |
2558 | free(ifobj->xsk_arr); |
2559 | free(ifobj); | |
2560 | } | |
2561 | ||
aa61d81f | 2562 | static bool is_xdp_supported(int ifindex) |
0d68e6fe MF |
2563 | { |
2564 | int flags = XDP_FLAGS_DRV_MODE; | |
2565 | ||
2566 | LIBBPF_OPTS(bpf_link_create_opts, opts, .flags = flags); | |
2567 | struct bpf_insn insns[2] = { | |
2568 | BPF_MOV64_IMM(BPF_REG_0, XDP_PASS), | |
2569 | BPF_EXIT_INSN() | |
2570 | }; | |
0d68e6fe MF |
2571 | int prog_fd, insn_cnt = ARRAY_SIZE(insns); |
2572 | int err; | |
2573 | ||
2574 | prog_fd = bpf_prog_load(BPF_PROG_TYPE_XDP, NULL, "GPL", insns, insn_cnt, NULL); | |
2575 | if (prog_fd < 0) | |
2576 | return false; | |
2577 | ||
2578 | err = bpf_xdp_attach(ifindex, prog_fd, flags, NULL); | |
2579 | if (err) { | |
2580 | close(prog_fd); | |
2581 | return false; | |
2582 | } | |
2583 | ||
2584 | bpf_xdp_detach(ifindex, flags, NULL); | |
2585 | close(prog_fd); | |
2586 | ||
2587 | return true; | |
2588 | } | |
2589 | ||
f20fbcd0 MK |
2590 | static const struct test_spec tests[] = { |
2591 | {.name = "SEND_RECEIVE", .test_func = testapp_send_receive}, | |
2592 | {.name = "SEND_RECEIVE_2K_FRAME", .test_func = testapp_send_receive_2k_frame}, | |
2593 | {.name = "SEND_RECEIVE_SINGLE_PKT", .test_func = testapp_single_pkt}, | |
2594 | {.name = "POLL_RX", .test_func = testapp_poll_rx}, | |
2595 | {.name = "POLL_TX", .test_func = testapp_poll_tx}, | |
2596 | {.name = "POLL_RXQ_FULL", .test_func = testapp_poll_rxq_tmout}, | |
2597 | {.name = "POLL_TXQ_FULL", .test_func = testapp_poll_txq_tmout}, | |
2598 | {.name = "SEND_RECEIVE_UNALIGNED", .test_func = testapp_send_receive_unaligned}, | |
2599 | {.name = "ALIGNED_INV_DESC", .test_func = testapp_aligned_inv_desc}, | |
2600 | {.name = "ALIGNED_INV_DESC_2K_FRAME_SIZE", .test_func = testapp_aligned_inv_desc_2k_frame}, | |
2601 | {.name = "UNALIGNED_INV_DESC", .test_func = testapp_unaligned_inv_desc}, | |
2602 | {.name = "UNALIGNED_INV_DESC_4001_FRAME_SIZE", | |
2603 | .test_func = testapp_unaligned_inv_desc_4001_frame}, | |
2604 | {.name = "UMEM_HEADROOM", .test_func = testapp_headroom}, | |
2605 | {.name = "TEARDOWN", .test_func = testapp_teardown}, | |
2606 | {.name = "BIDIRECTIONAL", .test_func = testapp_bidirectional}, | |
2607 | {.name = "STAT_RX_DROPPED", .test_func = testapp_stats_rx_dropped}, | |
2608 | {.name = "STAT_TX_INVALID", .test_func = testapp_stats_tx_invalid_descs}, | |
2609 | {.name = "STAT_RX_FULL", .test_func = testapp_stats_rx_full}, | |
2610 | {.name = "STAT_FILL_EMPTY", .test_func = testapp_stats_fill_empty}, | |
2611 | {.name = "XDP_PROG_CLEANUP", .test_func = testapp_xdp_prog_cleanup}, | |
2612 | {.name = "XDP_DROP_HALF", .test_func = testapp_xdp_drop}, | |
6d198a89 | 2613 | {.name = "XDP_SHARED_UMEM", .test_func = testapp_xdp_shared_umem}, |
f20fbcd0 MK |
2614 | {.name = "XDP_METADATA_COPY", .test_func = testapp_xdp_metadata}, |
2615 | {.name = "XDP_METADATA_COPY_MULTI_BUFF", .test_func = testapp_xdp_metadata_mb}, | |
2616 | {.name = "SEND_RECEIVE_9K_PACKETS", .test_func = testapp_send_receive_mb}, | |
2617 | {.name = "SEND_RECEIVE_UNALIGNED_9K_PACKETS", | |
2618 | .test_func = testapp_send_receive_unaligned_mb}, | |
2619 | {.name = "ALIGNED_INV_DESC_MULTI_BUFF", .test_func = testapp_aligned_inv_desc_mb}, | |
2620 | {.name = "UNALIGNED_INV_DESC_MULTI_BUFF", .test_func = testapp_unaligned_inv_desc_mb}, | |
2621 | {.name = "TOO_MANY_FRAGS", .test_func = testapp_too_many_frags}, | |
c4f96053 | 2622 | {.name = "HW_SW_MIN_RING_SIZE", .test_func = testapp_hw_sw_min_ring_size}, |
c53908b2 | 2623 | {.name = "HW_SW_MAX_RING_SIZE", .test_func = testapp_hw_sw_max_ring_size}, |
776021e0 | 2624 | }; |
f20fbcd0 | 2625 | |
c53dab7d MK |
2626 | static void print_tests(void) |
2627 | { | |
2628 | u32 i; | |
2629 | ||
2630 | printf("Tests:\n"); | |
2631 | for (i = 0; i < ARRAY_SIZE(tests); i++) | |
2632 | printf("%u: %s\n", i, tests[i].name); | |
2633 | } | |
2634 | ||
facb7cb2 WJ |
2635 | int main(int argc, char **argv) |
2636 | { | |
1adef064 MF |
2637 | struct pkt_stream *rx_pkt_stream_default; |
2638 | struct pkt_stream *tx_pkt_stream_default; | |
ce74acaf | 2639 | struct ifobject *ifobj_tx, *ifobj_rx; |
146e3055 | 2640 | u32 i, j, failed_tests = 0, nb_tests; |
0d68e6fe | 2641 | int modes = TEST_MODE_SKB + 1; |
ce74acaf | 2642 | struct test_spec test; |
aa61d81f | 2643 | bool shared_netdev; |
776021e0 | 2644 | int ret; |
facb7cb2 | 2645 | |
b858ba8c YS |
2646 | /* Use libbpf 1.0 API mode */ |
2647 | libbpf_set_strict_mode(LIBBPF_STRICT_ALL); | |
facb7cb2 | 2648 | |
ce74acaf MK |
2649 | ifobj_tx = ifobject_create(); |
2650 | if (!ifobj_tx) | |
2651 | exit_with_error(ENOMEM); | |
2652 | ifobj_rx = ifobject_create(); | |
2653 | if (!ifobj_rx) | |
2654 | exit_with_error(ENOMEM); | |
2655 | ||
facb7cb2 WJ |
2656 | setlocale(LC_ALL, ""); |
2657 | ||
af6731d1 | 2658 | parse_command_line(ifobj_tx, ifobj_rx, argc, argv); |
a693ff3e | 2659 | |
c53dab7d MK |
2660 | if (opt_print_tests) { |
2661 | print_tests(); | |
2662 | ksft_exit_xpass(); | |
2663 | } | |
146e3055 MK |
2664 | if (opt_run_test != RUN_ALL_TESTS && opt_run_test >= ARRAY_SIZE(tests)) { |
2665 | ksft_print_msg("Error: test %u does not exist.\n", opt_run_test); | |
2666 | ksft_exit_xfail(); | |
2667 | } | |
c53dab7d | 2668 | |
aa61d81f MK |
2669 | shared_netdev = (ifobj_tx->ifindex == ifobj_rx->ifindex); |
2670 | ifobj_tx->shared_umem = shared_netdev; | |
2671 | ifobj_rx->shared_umem = shared_netdev; | |
ce74acaf | 2672 | |
146e3055 MK |
2673 | if (!validate_interface(ifobj_tx) || !validate_interface(ifobj_rx)) |
2674 | print_usage(argv); | |
facb7cb2 | 2675 | |
aa61d81f | 2676 | if (is_xdp_supported(ifobj_tx->ifindex)) { |
0d68e6fe | 2677 | modes++; |
fe2ad08e MF |
2678 | if (ifobj_zc_avail(ifobj_tx)) |
2679 | modes++; | |
2680 | } | |
0d68e6fe | 2681 | |
776021e0 TV |
2682 | ret = get_hw_ring_size(ifobj_tx->ifname, &ifobj_tx->ring); |
2683 | if (!ret) { | |
2684 | ifobj_tx->hw_ring_size_supp = true; | |
2685 | ifobj_tx->set_ring.default_tx = ifobj_tx->ring.tx_pending; | |
2686 | ifobj_tx->set_ring.default_rx = ifobj_tx->ring.rx_pending; | |
2687 | } | |
2688 | ||
985fd214 TV |
2689 | init_iface(ifobj_rx, worker_testapp_validate_rx); |
2690 | init_iface(ifobj_tx, worker_testapp_validate_tx); | |
aa61d81f | 2691 | |
f20fbcd0 | 2692 | test_spec_init(&test, ifobj_tx, ifobj_rx, 0, &tests[0]); |
46e43786 TV |
2693 | tx_pkt_stream_default = pkt_stream_generate(DEFAULT_PKT_CNT, MIN_PKT_SIZE); |
2694 | rx_pkt_stream_default = pkt_stream_generate(DEFAULT_PKT_CNT, MIN_PKT_SIZE); | |
1adef064 | 2695 | if (!tx_pkt_stream_default || !rx_pkt_stream_default) |
605091c5 | 2696 | exit_with_error(ENOMEM); |
1adef064 MF |
2697 | test.tx_pkt_stream_default = tx_pkt_stream_default; |
2698 | test.rx_pkt_stream_default = rx_pkt_stream_default; | |
605091c5 | 2699 | |
146e3055 MK |
2700 | if (opt_run_test == RUN_ALL_TESTS) |
2701 | nb_tests = ARRAY_SIZE(tests); | |
2702 | else | |
2703 | nb_tests = 1; | |
3956bc34 | 2704 | if (opt_mode == TEST_MODE_ALL) { |
146e3055 | 2705 | ksft_set_plan(modes * nb_tests); |
3956bc34 MK |
2706 | } else { |
2707 | if (opt_mode == TEST_MODE_DRV && modes <= TEST_MODE_DRV) { | |
2708 | ksft_print_msg("Error: XDP_DRV mode not supported.\n"); | |
2709 | ksft_exit_xfail(); | |
2710 | } | |
2711 | if (opt_mode == TEST_MODE_ZC && modes <= TEST_MODE_ZC) { | |
2712 | ksft_print_msg("Error: zero-copy mode not supported.\n"); | |
2713 | ksft_exit_xfail(); | |
2714 | } | |
2715 | ||
146e3055 | 2716 | ksft_set_plan(nb_tests); |
3956bc34 | 2717 | } |
facb7cb2 | 2718 | |
aa61d81f | 2719 | for (i = 0; i < modes; i++) { |
3956bc34 MK |
2720 | if (opt_mode != TEST_MODE_ALL && i != opt_mode) |
2721 | continue; | |
2722 | ||
f20fbcd0 | 2723 | for (j = 0; j < ARRAY_SIZE(tests); j++) { |
146e3055 MK |
2724 | if (opt_run_test != RUN_ALL_TESTS && j != opt_run_test) |
2725 | continue; | |
2726 | ||
f20fbcd0 MK |
2727 | test_spec_init(&test, ifobj_tx, ifobj_rx, i, &tests[j]); |
2728 | run_pkt_test(&test); | |
1314c353 | 2729 | usleep(USLEEP_MAX); |
895b62ee MK |
2730 | |
2731 | if (test.fail) | |
2732 | failed_tests++; | |
1314c353 | 2733 | } |
aa61d81f | 2734 | } |
facb7cb2 | 2735 | |
776021e0 TV |
2736 | if (ifobj_tx->hw_ring_size_supp) |
2737 | hw_ring_size_reset(ifobj_tx); | |
2738 | ||
1adef064 MF |
2739 | pkt_stream_delete(tx_pkt_stream_default); |
2740 | pkt_stream_delete(rx_pkt_stream_default); | |
f0a249df MK |
2741 | xsk_unload_xdp_programs(ifobj_tx); |
2742 | xsk_unload_xdp_programs(ifobj_rx); | |
ce74acaf MK |
2743 | ifobject_delete(ifobj_tx); |
2744 | ifobject_delete(ifobj_rx); | |
9445f8c7 | 2745 | |
895b62ee MK |
2746 | if (failed_tests) |
2747 | ksft_exit_fail(); | |
2748 | else | |
2749 | ksft_exit_pass(); | |
facb7cb2 | 2750 | } |