/*
* Stress userfaultfd syscall.
*
* Copyright (C) 2015 Red Hat, Inc.
*
* This work is licensed under the terms of the GNU GPL, version 2. See
* the COPYING file in the top-level directory.
*
* This test allocates two virtual areas and bounces the physical
* memory across the two virtual areas (from area_src to area_dst)
* using userfaultfd.
*
* There are three threads running per CPU:
*
* 1) one per-CPU thread takes a per-page pthread_mutex in a random
* page of the area_dst (while the physical page may still be in
* area_src), and increments a per-page counter in the same page,
* and checks its value against a verification region.
*
* 2) another per-CPU thread handles the userfaults generated by
* thread 1 above. userfaultfd blocking reads or poll() modes are
* exercised interleaved.
*
* 3) one last per-CPU thread transfers the memory in the background
* at maximum bandwidth (if not already transferred by thread
* 2). Each cpu thread takes cares of transferring a portion of the
* area.
*
* When all threads of type 3 completed the transfer, one bounce is
* complete. area_src and area_dst are then swapped. All threads are
* respawned and so the bounce is immediately restarted in the
* opposite direction.
*
* per-CPU threads 1 by triggering userfaults inside
* pthread_mutex_lock will also verify the atomicity of the memory
* transfer (UFFDIO_COPY).
*
* The program takes two parameters: the amounts of physical memory in
* megabytes (MiB) of the area and the number of bounces to execute.
*
* # 100MiB 99999 bounces
* ./userfaultfd 100 99999
*
* # 1GiB 99 bounces
* ./userfaultfd 1000 99
*
* # 10MiB-~6GiB 999 bounces, continue forever unless an error triggers
* while ./userfaultfd $[RANDOM % 6000 + 10] 999; do true; done
*/
#define _GNU_SOURCE
#include <stdio.h>
#include <errno.h>
#include <unistd.h>
#include <stdlib.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <time.h>
#include <signal.h>
#include <poll.h>
#include <string.h>
#include <sys/mman.h>
#include <sys/syscall.h>
#include <sys/ioctl.h>
#include <sys/wait.h>
#include <pthread.h>
#include <linux/userfaultfd.h>
#include <setjmp.h>
#ifdef __NR_userfaultfd
static unsigned long nr_cpus, nr_pages, nr_pages_per_cpu, page_size;
#define BOUNCE_RANDOM (1<<0)
#define BOUNCE_RACINGFAULTS (1<<1)
#define BOUNCE_VERIFY (1<<2)
#define BOUNCE_POLL (1<<3)
static int bounces;
#define TEST_ANON 1
#define TEST_HUGETLB 2
#define TEST_SHMEM 3
static int test_type;
static int huge_fd;
static char *huge_fd_off0;
static unsigned long long *count_verify;
static int uffd, uffd_flags, finished, *pipefd;
static char *area_src, *area_dst;
static char *zeropage;
pthread_attr_t attr;
/* pthread_mutex_t starts at page offset 0 */
#define area_mutex(___area, ___nr) \
((pthread_mutex_t *) ((___area) + (___nr)*page_size))
/*
* count is placed in the page after pthread_mutex_t naturally aligned
* to avoid non alignment faults on non-x86 archs.
*/
#define area_count(___area, ___nr) \
((volatile unsigned long long *) ((unsigned long) \
((___area) + (___nr)*page_size + \
sizeof(pthread_mutex_t) + \
sizeof(unsigned long long) - 1) & \
~(unsigned long)(sizeof(unsigned long long) \
- 1)))
static int anon_release_pages(char *rel_area)
{
int ret = 0;
if (madvise(rel_area, nr_pages * page_size, MADV_DONTNEED)) {
perror("madvise");
ret = 1;
}
return ret;
}
static void anon_allocate_area(void **alloc_area)
{
if (posix_memalign(alloc_area, page_size, nr_pages * page_size)) {
fprintf(stderr, "out of memory\n");
*alloc_area = NULL;
}
}
/* HugeTLB memory */
static int hugetlb_release_pages(char *rel_area)
{
int ret = 0;
if (fallocate(huge_fd, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE,
rel_area == huge_fd_off0 ? 0 :
nr_pages * page_size,
nr_pages * page_size)) {
perror("fallocate");
ret = 1;
}
return ret;
}
static void hugetlb_allocate_area(void **alloc_area)
{
*alloc_area = mmap(NULL, nr_pages * page_size, PROT_READ | PROT_WRITE,
MAP_PRIVATE | MAP_HUGETLB, huge_fd,
*alloc_area == area_src ? 0 :
nr_pages * page_size);
if (*alloc_area == MAP_FAILED) {
fprintf(