// SPDX-License-Identifier: GPL-2.0
/*
* channel program interfaces
*
* Copyright IBM Corp. 2017
*
* Author(s): Dong Jia Shi <bjsdjshi@linux.vnet.ibm.com>
* Xiao Feng Ren <renxiaof@linux.vnet.ibm.com>
*/
#include <linux/mm.h>
#include <linux/slab.h>
#include <linux/iommu.h>
#include <linux/vfio.h>
#include <asm/idals.h>
#include "vfio_ccw_cp.h"
/*
* Max length for ccw chain.
* XXX: Limit to 256, need to check more?
*/
#define CCWCHAIN_LEN_MAX 256
struct pfn_array {
unsigned long pa_iova;
unsigned long *pa_iova_pfn;
unsigned long *pa_pfn;
int pa_nr;
};
struct pfn_array_table {
struct pfn_array *pat_pa;
int pat_nr;
};
struct ccwchain {
struct list_head next;
struct ccw1 *ch_ccw;
/* Guest physical address of the current chain. */
u64 ch_iova;
/* Count of the valid ccws in chain. */
int ch_len;
/* Pinned PAGEs for the original data. */
struct pfn_array_table *ch_pat;
};
/*
* pfn_array_pin() - pin user pages in memory
* @pa: pfn_array on which to perform the operation
* @mdev: the mediated device to perform pin/unpin operations
*
* Attempt to pin user pages in memory.
*
* Usage of pfn_array:
* @pa->pa_iova starting guest physical I/O address. Assigned by caller.
* @pa->pa_iova_pfn array that stores PFNs of the pages need to pin. Allocated
* by caller.
* @pa->pa_pfn array that receives PFNs of the pages pinned. Allocated by
* caller.
* @pa->pa_nr number of pages from @pa->pa_iova to pin. Assigned by
* caller.
* number of pages pinned. Assigned by callee.
*
* Returns:
* Number of pages pinned on success.
* If @pa->pa_nr is 0 or negative, returns 0.
* If no pages were pinned, returns -errno.
*/
static int pfn_array_pin(struct pfn_array *pa, struct device *mdev)
{
int i, ret;
if (pa->pa_nr <= 0) {
pa->pa_nr = 0;
return 0;
}
pa->pa_iova_pfn[0] = pa->pa_iova >> PAGE_SHIFT;
for (i = 1; i < pa->pa_nr; i++)
pa->pa_iova_pfn[i] = pa->pa_iova_pfn[i - 1] + 1;
ret = vfio_pin_pages(mdev, pa->pa_iova_pfn, pa->pa_nr,
IOMMU_READ | IOMMU_WRITE, pa->pa_pfn);
if (ret > 0 && ret != pa->pa_nr) {
vfio_unpin_pages(mdev, pa->pa_iova_pfn, ret);
pa->pa_nr = 0;
return 0;
}
return ret;
}
/* Unpin the pages before releasing the memory. */
static void pfn_array_unpin_free(struct pfn_array *pa, struct device *mdev)
{
vfio_unpin_pages(mdev, pa->pa_iova_pfn, pa->pa_nr);
pa->pa_nr = 0;
kfree(pa->pa_iova_pfn);
}
/* Alloc memory for PFNs, then pin pages with them. */
static int pfn_array_alloc_pin(struct pfn_array *pa, struct device *mdev,
u64 iova, unsigned int len)
{
int ret = 0;
if (!len)
return 0;
if (pa->pa_nr)
return -EINVAL;
pa->pa_iova = iova;
pa->pa_nr = ((iova & ~PAGE_MASK) + len + (PAGE_SIZE - 1)) >> PAGE_SHIFT;
if (!pa->pa_nr)
return -EINVAL;
pa->pa_iova_pfn = kcalloc(pa->pa_nr,
sizeof(*pa->pa_iova_pfn) +
sizeof(*pa->pa_pfn),
GFP_KERNEL);
if (unlikely(!pa->pa_iova_pfn))
return -ENOMEM;
pa->pa_pfn = pa->pa_iova_pfn + pa->pa_nr;
ret = pfn_array_pin(pa, mdev);
if (ret > 0)
return ret;
else if (!ret)
ret = -EINVAL;
kfree(pa->pa_iova_pfn);
return ret;
}
static int pfn_array_table_init(struct pfn_array_table *pat, int nr)
{
pat->pat_pa = kcalloc(nr, sizeof(*pat->pat_pa), GFP_KERNEL);
if (unlikely(ZERO_OR_NULL_PTR(pat->pat_pa))) {
pat->pat_nr = 0;
return -ENOMEM;
}
pat->pat_nr = nr;
return 0;
}
static void pfn_array_table_unpin_free(struct pfn_array_table *pat,
struct device *mdev)
{
int i;
for (i = 0; i < pat->pat_nr; i++)
pfn_array_unpin_free(pat->pat_pa + i, mdev);
if (pat->pat_nr) {
kfree(pat->pat_pa);
pat->pat_pa = NULL;
pat->pat_nr = 0;
}
}
static bool pfn_array_table_iova_pinned(struct pfn_array_table *pat,
unsigned long iova)
{
struct pfn_array *pa = pat->pat_pa;
unsigned long iova_pfn = iova >> PAGE_SHIFT;
int i, j;
for (i = 0; i < pat->pat_nr; i++, pa++)
for (j = 0; j < pa->pa_nr; j++)
if (pa->pa_iova_pfn[i] == iova_pfn)
return true;
return false