// SPDX-License-Identifier: GPL-2.0
/*
* This is a module to test the HMM (Heterogeneous Memory Management)
* mirror and zone device private memory migration APIs of the kernel.
* Userspace programs can register with the driver to mirror their own address
* space and can use the device to read/write any valid virtual address.
*/
#include <linux/init.h>
#include <linux/fs.h>
#include <linux/mm.h>
#include <linux/module.h>
#include <linux/kernel.h>
#include <linux/cdev.h>
#include <linux/device.h>
#include <linux/mutex.h>
#include <linux/rwsem.h>
#include <linux/sched.h>
#include <linux/slab.h>
#include <linux/highmem.h>
#include <linux/delay.h>
#include <linux/pagemap.h>
#include <linux/hmm.h>
#include <linux/vmalloc.h>
#include <linux/swap.h>
#include <linux/swapops.h>
#include <linux/sched/mm.h>
#include <linux/platform_device.h>
#include "test_hmm_uapi.h"
#define DMIRROR_NDEVICES 2
#define DMIRROR_RANGE_FAULT_TIMEOUT 1000
#define DEVMEM_CHUNK_SIZE (256 * 1024 * 1024U)
#define DEVMEM_CHUNKS_RESERVE 16
static const struct dev_pagemap_ops dmirror_devmem_ops;
static const struct mmu_interval_notifier_ops dmirror_min_ops;
static dev_t dmirror_dev;
struct dmirror_device;
struct dmirror_bounce {
void *ptr;
unsigned long size;
unsigned long addr;
unsigned long cpages;
};
#define DPT_XA_TAG_WRITE 3UL
/*
* Data structure to track address ranges and register for mmu interval
* notifier updates.
*/
struct dmirror_interval {
struct mmu_interval_notifier notifier;
struct dmirror *dmirror;
};
/*
* Data attached to the open device file.
* Note that it might be shared after a fork().
*/
struct dmirror {
struct dmirror_device *mdevice;
struct xarray pt;
struct mmu_interval_notifier notifier;
struct mutex mutex;
};
/*
* ZONE_DEVICE pages for migration and simulating device memory.
*/
struct dmirror_chunk {
struct dev_pagemap pagemap;
struct dmirror_device *mdevice;
};
/*
* Per device data.
*/
struct dmirror_device {
struct cdev cdevice;
struct hmm_devmem *devmem;
unsigned int devmem_capacity;
unsigned int devmem_count;
struct dmirror_chunk **devmem_chunks;
struct mutex devmem_lock; /* protects the above */
unsigned long calloc;
unsigned long cfree;
struct page *free_pages;
spinlock_t lock; /* protects the above */
};
static struct dmirror_device dmirror_devices[DMIRROR_NDEVICES];
static int dmirror_bounce_init(struct dmirror_bounce *bounce,
unsigned long addr,
unsigned long size)
{
bounce->addr = addr;
bounce->size = size;
bounce->cpages = 0;
bounce->ptr = vmalloc(size);
if (!bounce->ptr)
return -ENOMEM;
return 0;
}
static void dmirror_bounce_fini(struct dmirror_bounce *bounce)
{
vfree(bounce->ptr);
}
static int dmirror_fops_open(struct inode *inode, struct file *filp)
{
struct cdev *cdev = inode->i_cdev;
struct dmirror *dmirror;
int ret;
/* Mirror this process address space */
dmirror = kzalloc(sizeof(*dmirror), GFP_KERNEL);
if (dmirror == NULL)
return -ENOMEM;
dmirror->mdevice = container_of(cdev, struct dmirror_device, cdevice);
mutex_init(&dmirror->mutex);
xa_init(&dmirror->pt);
ret = mmu_interval_notifier_insert(&dmirror->notifier, current->mm,
0, ULONG_MAX & PAGE_MASK, &dmirror_min_ops);
if (ret) {
kfree(dmirror);
return ret;
}
filp->private_data = dmirror;
return