// SPDX-License-Identifier: GPL-2.0
/* Copyright(c) 2020 Intel Corporation. */
/*
* Some functions in this program are taken from
* Linux kernel samples/bpf/xdpsock* and modified
* for use.
*
* See test_xsk.sh for detailed information on test topology
* and prerequisite network setup.
*
* This test program contains two threads, each thread is single socket with
* a unique UMEM. It validates in-order packet delivery and packet content
* by sending packets to each other.
*
* Tests Information:
* ------------------
* These selftests test AF_XDP SKB and Native/DRV modes using veth
* Virtual Ethernet interfaces.
*
* The following tests are run:
*
* 1. AF_XDP SKB mode
* Generic mode XDP is driver independent, used when the driver does
* not have support for XDP. Works on any netdevice using sockets and
* generic XDP path. XDP hook from netif_receive_skb().
* a. nopoll - soft-irq processing
* b. poll - using poll() syscall
* c. Socket Teardown
* Create a Tx and a Rx socket, Tx from one socket, Rx on another. Destroy
* both sockets, then repeat multiple times. Only nopoll mode is used
* d. Bi-directional sockets
* Configure sockets as bi-directional tx/rx sockets, sets up fill and
* completion rings on each socket, tx/rx in both directions. Only nopoll
* mode is used
*
* 2. AF_XDP DRV/Native mode
* Works on any netdevice with XDP_REDIRECT support, driver dependent. Processes
* packets before SKB allocation. Provides better performance than SKB. Driver
* hook available just after DMA of buffer descriptor.
* a. nopoll
* b. poll
* c. Socket Teardown
* d. Bi-directional sockets
* - Only copy mode is supported because veth does not currently support
* zero-copy mode
*
* Total tests: 8
*
* Flow:
* -----
* - Single process spawns two threads: Tx and Rx
* - Each of these two threads attach to a veth interface within their assigned
* namespaces
* - Each thread Creates one AF_XDP socket connected to a unique umem for each
* veth interface
* - Tx thread Transmits 10k packets from veth<xxxx> to veth<yyyy>
* - Rx thread verifies if all 10k packets were received and delivered in-order,
* and have the right content
*
* Enable/disable debug mode:
* --------------------------
* To enable L2 - L4 headers and payload dump of each packet on STDOUT, add
* parameter -D to params array in test_xsk.sh, i.e. params=("-S" "-D")
*/
#define _GNU_SOURCE
#include <fcntl.h>
#include <errno.h>
#include <getopt.h>
#include <asm/barrier.h>
typedef __u16 __sum16;
#include <linux/if_link.h>
#include <linux/if_ether.h>
#include <linux/ip.h>
#include <linux/udp.h>
#include <arpa/inet.h>
#include <net/if.h>
#include <locale.h>
#include <poll.h>
#include <pthread.h>
#include <signal.h>
#include <stdbool.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <stddef.h>
#include <sys/mman.h>
#include <sys/resource.h>
#include <sys/types.h>
#include <sys/queue.h>
#include <time.h>
#include <unistd.h>
#include <stdatomic.h>
#include <bpf/xsk.h>
#include "xdpxceiver.h"
#include "../kselftest.h"
static void __exit_with_error(int error, const char *file, const char *func, int line)
{
ksft_test_result_fail
("[%s:%s:%i]: ERROR: %d/\"%s\"\n", file, func, line, error, strerror(error));
ksft_exit_xfail();
}
#define exit_with_error(error) __exit_with_error(error, __FILE__, __func__, __LINE__)
#define print_ksft_result(void)\
(ksft_test_result_pass("PASS: %s %s %s%s\n", uut ? "DRV" : "SKB", opt_poll ? "POLL" :\
"NOPOLL", opt_teardown ? "Socket Teardown" : "",\
opt_bidi ? "Bi-directional Sockets" : ""))
static void pthread_init_mutex(void)
{
pthread_mutex_init(&sync_mutex, NULL);
pthread_mutex_init(&sync_mutex_tx, NULL);
pthread_cond_init(&signal_rx_condition, NULL);
pthread_cond_init(&signal_tx_condition, NULL);
}
static void pthread_destroy_mutex(void)
{
pthread_mutex_destroy(&sync_mutex);
pthread_mutex_destroy(&sync_mutex_tx);
pthread_cond_destroy(&signal_rx_condition);
pthread_cond_destroy(&signal_tx_condition);
}
static void *memset32_htonl(void *dest, u32 val, u32 size)
{
u32 *ptr = (u32 *)dest;
int i;
val = htonl(val);
for (i = 0; i < (size & (~0x3)); i += 4)
ptr[i >> 2] = val;
for (; i < size; i++)
((char *)dest)[i] = ((char *)&val)[i & 3];
return dest;
}
/*
* This function code has been taken from
* Linux kernel lib/checksum.c
*/
static inline unsigned short from32to16(unsigned int x)
{
/* add up 16-bit and 16-bit for 16+c bit */
x = (x & 0xffff) + (x >> 16);
/* add up carry.. */
x = (x & 0xffff) + (x >> 16);
return x;
}
/*
* Fold a partial checksum
* This function code has been taken from
* Linux kernel include/asm-generic/checksum.h
*/
static inline __u16 csum_fold(__u32 csum)
{
u32 sum = (__force u32)csum;
sum = (sum & 0xffff) + (sum >> 16);
sum = (sum & 0xffff) + (sum >> 16);
return (__force __u16)~sum;
}
/*
* This function code has been taken from
* Linux kernel lib/checksum.c
*/