/*
* Copyright(c) 2013-2015 Intel Corporation. All rights reserved.
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of version 2 of the GNU General Public License as
* published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but
* WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* General Public License for more details.
*/
#include <linux/scatterlist.h>
#include <linux/highmem.h>
#include <linux/sched.h>
#include <linux/slab.h>
#include <linux/hash.h>
#include <linux/pmem.h>
#include <linux/sort.h>
#include <linux/io.h>
#include <linux/nd.h>
#include "nd-core.h"
#include "nd.h"
/*
* For readq() and writeq() on 32-bit builds, the hi-lo, lo-hi order is
* irrelevant.
*/
#include <linux/io-64-nonatomic-hi-lo.h>
static DEFINE_IDA(region_ida);
static DEFINE_PER_CPU(int, flush_idx);
static int nvdimm_map_flush(struct device *dev, struct nvdimm *nvdimm, int dimm,
struct nd_region_data *ndrd)
{
int i, j;
dev_dbg(dev, "%s: map %d flush address%s\n", nvdimm_name(nvdimm),
nvdimm->num_flush, nvdimm->num_flush == 1 ? "" : "es");
for (i = 0; i < (1 << ndrd->hints_shift); i++) {
struct resource *res = &nvdimm->flush_wpq[i];
unsigned long pfn = PHYS_PFN(res->start);
void __iomem *flush_page;
/* check if flush hints share a page */
for (j = 0; j < i; j++) {
struct resource *res_j = &nvdimm->flush_wpq[j];
unsigned long pfn_j = PHYS_PFN(res_j->start);
if (pfn == pfn_j)
break;
}
if (j < i)
flush_page = (void __iomem *) ((unsigned long)
ndrd_get_flush_wpq(ndrd, dimm, j)
& PAGE_MASK);
else
flush_page = devm_nvdimm_ioremap(dev,
PFN_PHYS(pfn), PAGE_SIZE);
if (!flush_page)
return -ENXIO;
ndrd_set_flush_wpq(ndrd, dimm, i, flush_page
+ (res->start & ~PAGE_MASK));
}
return 0;
}
int nd_region_activate(struct nd_region *nd_region)
{
int i, j, num_flush = 0;
struct nd_region_data *ndrd;
struct device *dev = &nd_region->dev;
size_t flush_data_size = sizeof(void *);
nvdimm_bus_lock(&nd_region->dev);
for (i = 0; i < nd_region->ndr_mappings; i++) {
struct nd_mapping *nd_mapping = &nd_region->mapping[i];
struct nvdimm *nvdimm = nd_mapping->nvdimm;
/* at least one null hint slot per-dimm for the "no-hint" case */
flush_data_size += sizeof(void *);
num_flush = min_not_zero(num_flush, nvdimm->num_flush);
if (!nvdimm->num_flush)
continue;
flush_data_size += nvdimm->num_flush * sizeof(void *);
}
nvdimm_bus_unlock(&nd_region->dev);
ndrd = devm_kzalloc(dev, sizeof(*ndrd) + flush_data_size, GFP_KERNEL);
if (!ndrd)
return -ENOMEM;
dev_set_drvdata(dev, ndrd);
if (!num_flush)
return 0;
ndrd->hints_shift = ilog2(num_flush);
for (i = 0; i < nd_region->ndr_mappings; i++) {
struct nd_mapping *nd_mapping = &nd_region->mapping[i];
struct nvdimm *nvdimm = nd_mapping->nvdimm;
int rc = nvdimm_map_flush(&nd_region->dev, nvdimm, i, ndrd);
if (rc)
return rc;
}
/*
* Clear out entries that are duplicates. This should prevent the
* extra flushings.
*/
for (i = 0; i < nd_region->ndr_mappings - 1; i++) {
/* ignore if NULL already */
if (!ndrd_get_flush_wpq(ndrd, i, 0))
continue;
for (j = i + 1; j < nd_region->ndr_mappings; j++)
if (ndrd_get_flush_wpq(ndrd, i, 0) ==
ndrd_get_flush_wpq(ndrd, j, 0