diff options
author | Dan Williams <dan.j.williams@intel.com> | 2016-05-18 09:59:34 -0700 |
---|---|---|
committer | Dan Williams <dan.j.williams@intel.com> | 2016-05-18 09:59:34 -0700 |
commit | 594d6d96ea042366878aa7dc7f5711b8c245db5a (patch) | |
tree | 1b7333bf5b5c1147e136f050d686ab4f888ab85f | |
parent | 1b8d2afde54fade94339f573c4e05644f9ae9866 (diff) | |
parent | 45a0dac0451136fa7ae34a6fea53ef6a136287ce (diff) |
Merge branch 'for-4.7/dax' into libnvdimm-for-next
-rw-r--r-- | drivers/nvdimm/Kconfig | 13 | ||||
-rw-r--r-- | drivers/nvdimm/Makefile | 1 | ||||
-rw-r--r-- | drivers/nvdimm/blk.c | 208 | ||||
-rw-r--r-- | drivers/nvdimm/btt.c | 20 | ||||
-rw-r--r-- | drivers/nvdimm/btt_devs.c | 24 | ||||
-rw-r--r-- | drivers/nvdimm/bus.c | 4 | ||||
-rw-r--r-- | drivers/nvdimm/claim.c | 63 | ||||
-rw-r--r-- | drivers/nvdimm/dax_devs.c | 99 | ||||
-rw-r--r-- | drivers/nvdimm/namespace_devs.c | 38 | ||||
-rw-r--r-- | drivers/nvdimm/nd-core.h | 1 | ||||
-rw-r--r-- | drivers/nvdimm/nd.h | 72 | ||||
-rw-r--r-- | drivers/nvdimm/pfn.h | 4 | ||||
-rw-r--r-- | drivers/nvdimm/pfn_devs.c | 319 | ||||
-rw-r--r-- | drivers/nvdimm/pmem.c | 492 | ||||
-rw-r--r-- | drivers/nvdimm/region.c | 2 | ||||
-rw-r--r-- | drivers/nvdimm/region_devs.c | 29 | ||||
-rw-r--r-- | include/linux/nd.h | 11 | ||||
-rw-r--r-- | include/uapi/linux/ndctl.h | 2 | ||||
-rw-r--r-- | tools/testing/nvdimm/Kbuild | 2 | ||||
-rw-r--r-- | tools/testing/nvdimm/test/iomap.c | 27 |
20 files changed, 837 insertions, 594 deletions
diff --git a/drivers/nvdimm/Kconfig b/drivers/nvdimm/Kconfig index 53c11621d5b1..7c8a3bf07884 100644 --- a/drivers/nvdimm/Kconfig +++ b/drivers/nvdimm/Kconfig @@ -88,4 +88,17 @@ config NVDIMM_PFN Select Y if unsure +config NVDIMM_DAX + bool "NVDIMM DAX: Raw access to persistent memory" + default LIBNVDIMM + depends on NVDIMM_PFN + help + Support raw device dax access to a persistent memory + namespace. For environments that want to hard partition + peristent memory, this capability provides a mechanism to + sub-divide a namespace into character devices that can only be + accessed via DAX (mmap(2)). + + Select Y if unsure + endif diff --git a/drivers/nvdimm/Makefile b/drivers/nvdimm/Makefile index ea84d3c4e8e5..909554c3f955 100644 --- a/drivers/nvdimm/Makefile +++ b/drivers/nvdimm/Makefile @@ -23,3 +23,4 @@ libnvdimm-y += label.o libnvdimm-$(CONFIG_ND_CLAIM) += claim.o libnvdimm-$(CONFIG_BTT) += btt_devs.o libnvdimm-$(CONFIG_NVDIMM_PFN) += pfn_devs.o +libnvdimm-$(CONFIG_NVDIMM_DAX) += dax_devs.o diff --git a/drivers/nvdimm/blk.c b/drivers/nvdimm/blk.c index e9ff9229d942..495e06d9f7e7 100644 --- a/drivers/nvdimm/blk.c +++ b/drivers/nvdimm/blk.c @@ -21,19 +21,19 @@ #include <linux/sizes.h> #include "nd.h" -struct nd_blk_device { - struct request_queue *queue; - struct gendisk *disk; - struct nd_namespace_blk *nsblk; - struct nd_blk_region *ndbr; - size_t disk_size; - u32 sector_size; - u32 internal_lbasize; -}; +static u32 nsblk_meta_size(struct nd_namespace_blk *nsblk) +{ + return nsblk->lbasize - ((nsblk->lbasize >= 4096) ? 4096 : 512); +} + +static u32 nsblk_internal_lbasize(struct nd_namespace_blk *nsblk) +{ + return roundup(nsblk->lbasize, INT_LBASIZE_ALIGNMENT); +} -static u32 nd_blk_meta_size(struct nd_blk_device *blk_dev) +static u32 nsblk_sector_size(struct nd_namespace_blk *nsblk) { - return blk_dev->nsblk->lbasize - blk_dev->sector_size; + return nsblk->lbasize - nsblk_meta_size(nsblk); } static resource_size_t to_dev_offset(struct nd_namespace_blk *nsblk, @@ -57,20 +57,29 @@ static resource_size_t to_dev_offset(struct nd_namespace_blk *nsblk, return SIZE_MAX; } +static struct nd_blk_region *to_ndbr(struct nd_namespace_blk *nsblk) +{ + struct nd_region *nd_region; + struct device *parent; + + parent = nsblk->common.dev.parent; + nd_region = container_of(parent, struct nd_region, dev); + return container_of(nd_region, struct nd_blk_region, nd_region); +} + #ifdef CONFIG_BLK_DEV_INTEGRITY -static int nd_blk_rw_integrity(struct nd_blk_device *blk_dev, - struct bio_integrity_payload *bip, u64 lba, - int rw) +static int nd_blk_rw_integrity(struct nd_namespace_blk *nsblk, + struct bio_integrity_payload *bip, u64 lba, int rw) { - unsigned int len = nd_blk_meta_size(blk_dev); + struct nd_blk_region *ndbr = to_ndbr(nsblk); + unsigned int len = nsblk_meta_size(nsblk); resource_size_t dev_offset, ns_offset; - struct nd_namespace_blk *nsblk; - struct nd_blk_region *ndbr; + u32 internal_lbasize, sector_size; int err = 0; - nsblk = blk_dev->nsblk; - ndbr = blk_dev->ndbr; - ns_offset = lba * blk_dev->internal_lbasize + blk_dev->sector_size; + internal_lbasize = nsblk_internal_lbasize(nsblk); + sector_size = nsblk_sector_size(nsblk); + ns_offset = lba * internal_lbasize + sector_size; dev_offset = to_dev_offset(nsblk, ns_offset, len); if (dev_offset == SIZE_MAX) return -EIO; @@ -104,25 +113,26 @@ static int nd_blk_rw_integrity(struct nd_blk_device *blk_dev, } #else /* CONFIG_BLK_DEV_INTEGRITY */ -static int nd_blk_rw_integrity(struct nd_blk_device *blk_dev, - struct bio_integrity_payload *bip, u64 lba, - int rw) +static int nd_blk_rw_integrity(struct nd_namespace_blk *nsblk, + struct bio_integrity_payload *bip, u64 lba, int rw) { return 0; } #endif -static int nd_blk_do_bvec(struct nd_blk_device *blk_dev, - struct bio_integrity_payload *bip, struct page *page, - unsigned int len, unsigned int off, int rw, - sector_t sector) +static int nsblk_do_bvec(struct nd_namespace_blk *nsblk, + struct bio_integrity_payload *bip, struct page *page, + unsigned int len, unsigned int off, int rw, sector_t sector) { - struct nd_blk_region *ndbr = blk_dev->ndbr; + struct nd_blk_region *ndbr = to_ndbr(nsblk); resource_size_t dev_offset, ns_offset; + u32 internal_lbasize, sector_size; int err = 0; void *iobuf; u64 lba; + internal_lbasize = nsblk_internal_lbasize(nsblk); + sector_size = nsblk_sector_size(nsblk); while (len) { unsigned int cur_len; @@ -132,11 +142,11 @@ static int nd_blk_do_bvec(struct nd_blk_device *blk_dev, * Block Window setup/move steps. the do_io routine is capable * of handling len <= PAGE_SIZE. */ - cur_len = bip ? min(len, blk_dev->sector_size) : len; + cur_len = bip ? min(len, sector_size) : len; - lba = div_u64(sector << SECTOR_SHIFT, blk_dev->sector_size); - ns_offset = lba * blk_dev->internal_lbasize; - dev_offset = to_dev_offset(blk_dev->nsblk, ns_offset, cur_len); + lba = div_u64(sector << SECTOR_SHIFT, sector_size); + ns_offset = lba * internal_lbasize; + dev_offset = to_dev_offset(nsblk, ns_offset, cur_len); if (dev_offset == SIZE_MAX) return -EIO; @@ -147,13 +157,13 @@ static int nd_blk_do_bvec(struct nd_blk_device *blk_dev, return err; if (bip) { - err = nd_blk_rw_integrity(blk_dev, bip, lba, rw); + err = nd_blk_rw_integrity(nsblk, bip, lba, rw); if (err) return err; } len -= cur_len; off += cur_len; - sector += blk_dev->sector_size >> SECTOR_SHIFT; + sector += sector_size >> SECTOR_SHIFT; } return err; @@ -161,10 +171,8 @@ static int nd_blk_do_bvec(struct nd_blk_device *blk_dev, static blk_qc_t nd_blk_make_request(struct request_queue *q, struct bio *bio) { - struct block_device *bdev = bio->bi_bdev; - struct gendisk *disk = bdev->bd_disk; struct bio_integrity_payload *bip; - struct nd_blk_device *blk_dev; + struct nd_namespace_blk *nsblk; struct bvec_iter iter; unsigned long start; struct bio_vec bvec; @@ -183,17 +191,17 @@ static blk_qc_t nd_blk_make_request(struct request_queue *q, struct bio *bio) } bip = bio_integrity(bio); - blk_dev = disk->private_data; + nsblk = q->queuedata; rw = bio_data_dir(bio); do_acct = nd_iostat_start(bio, &start); bio_for_each_segment(bvec, bio, iter) { unsigned int len = bvec.bv_len; BUG_ON(len > PAGE_SIZE); - err = nd_blk_do_bvec(blk_dev, bip, bvec.bv_page, len, - bvec.bv_offset, rw, iter.bi_sector); + err = nsblk_do_bvec(nsblk, bip, bvec.bv_page, len, + bvec.bv_offset, rw, iter.bi_sector); if (err) { - dev_info(&blk_dev->nsblk->common.dev, + dev_dbg(&nsblk->common.dev, "io error in %s sector %lld, len %d,\n", (rw == READ) ? "READ" : "WRITE", (unsigned long long) iter.bi_sector, len); @@ -209,17 +217,16 @@ static blk_qc_t nd_blk_make_request(struct request_queue *q, struct bio *bio) return BLK_QC_T_NONE; } -static int nd_blk_rw_bytes(struct nd_namespace_common *ndns, +static int nsblk_rw_bytes(struct nd_namespace_common *ndns, resource_size_t offset, void *iobuf, size_t n, int rw) { - struct nd_blk_device *blk_dev = dev_get_drvdata(ndns->claim); - struct nd_namespace_blk *nsblk = blk_dev->nsblk; - struct nd_blk_region *ndbr = blk_dev->ndbr; + struct nd_namespace_blk *nsblk = to_nd_namespace_blk(&ndns->dev); + struct nd_blk_region *ndbr = to_ndbr(nsblk); resource_size_t dev_offset; dev_offset = to_dev_offset(nsblk, offset, n); - if (unlikely(offset + n > blk_dev->disk_size)) { + if (unlikely(offset + n > nsblk->size)) { dev_WARN_ONCE(&ndns->dev, 1, "request out of range\n"); return -EFAULT; } @@ -235,51 +242,65 @@ static const struct block_device_operations nd_blk_fops = { .revalidate_disk = nvdimm_revalidate_disk, }; -static int nd_blk_attach_disk(struct nd_namespace_common *ndns, - struct nd_blk_device *blk_dev) +static void nd_blk_release_queue(void *q) +{ + blk_cleanup_queue(q); +} + +static void nd_blk_release_disk(void *disk) +{ + del_gendisk(disk); + put_disk(disk); +} + +static int nsblk_attach_disk(struct nd_namespace_blk *nsblk) { + struct device *dev = &nsblk->common.dev; resource_size_t available_disk_size; + struct request_queue *q; struct gendisk *disk; u64 internal_nlba; - internal_nlba = div_u64(blk_dev->disk_size, blk_dev->internal_lbasize); - available_disk_size = internal_nlba * blk_dev->sector_size; + internal_nlba = div_u64(nsblk->size, nsblk_internal_lbasize(nsblk)); + available_disk_size = internal_nlba * nsblk_sector_size(nsblk); - blk_dev->queue = blk_alloc_queue(GFP_KERNEL); - if (!blk_dev->queue) + q = blk_alloc_queue(GFP_KERNEL); + if (!q) return -ENOMEM; + if (devm_add_action(dev, nd_blk_release_queue, q)) { + blk_cleanup_queue(q); + return -ENOMEM; + } - blk_queue_make_request(blk_dev->queue, nd_blk_make_request); - blk_queue_max_hw_sectors(blk_dev->queue, UINT_MAX); - blk_queue_bounce_limit(blk_dev->queue, BLK_BOUNCE_ANY); - blk_queue_logical_block_size(blk_dev->queue, blk_dev->sector_size); - queue_flag_set_unlocked(QUEUE_FLAG_NONROT, blk_dev->queue); + blk_queue_make_request(q, nd_blk_make_request); + blk_queue_max_hw_sectors(q, UINT_MAX); + blk_queue_bounce_limit(q, BLK_BOUNCE_ANY); + blk_queue_logical_block_size(q, nsblk_sector_size(nsblk)); + queue_flag_set_unlocked(QUEUE_FLAG_NONROT, q); + q->queuedata = nsblk; - disk = blk_dev->disk = alloc_disk(0); - if (!disk) { - blk_cleanup_queue(blk_dev->queue); + disk = alloc_disk(0); + if (!disk) + return -ENOMEM; + if (devm_add_action(dev, nd_blk_release_disk, disk)) { + put_disk(disk); return -ENOMEM; } - disk->driverfs_dev = &ndns->dev; + disk->driverfs_dev = dev; disk->first_minor = 0; disk->fops = &nd_blk_fops; - disk->private_data = blk_dev; - disk->queue = blk_dev->queue; + disk->queue = q; disk->flags = GENHD_FL_EXT_DEVT; - nvdimm_namespace_disk_name(ndns, disk->disk_name); + nvdimm_namespace_disk_name(&nsblk->common, disk->disk_name); set_capacity(disk, 0); add_disk(disk); - if (nd_blk_meta_size(blk_dev)) { - int rc = nd_integrity_init(disk, nd_blk_meta_size(blk_dev)); + if (nsblk_meta_size(nsblk)) { + int rc = nd_integrity_init(disk, nsblk_meta_size(nsblk)); - if (rc) { - del_gendisk(disk); - put_disk(disk); - blk_cleanup_queue(blk_dev->queue); + if (rc) return rc; - } } set_capacity(disk, available_disk_size >> SECTOR_SHIFT); @@ -291,56 +312,29 @@ static int nd_blk_probe(struct device *dev) { struct nd_namespace_common *ndns; struct nd_namespace_blk *nsblk; - struct nd_blk_device *blk_dev; - int rc; ndns = nvdimm_namespace_common_probe(dev); if (IS_ERR(ndns)) return PTR_ERR(ndns); - blk_dev = kzalloc(sizeof(*blk_dev), GFP_KERNEL); - if (!blk_dev) - return -ENOMEM; - nsblk = to_nd_namespace_blk(&ndns->dev); - blk_dev->disk_size = nvdimm_namespace_capacity(ndns); - blk_dev->ndbr = to_nd_blk_region(dev->parent); - blk_dev->nsblk = to_nd_namespace_blk(&ndns->dev); - blk_dev->internal_lbasize = roundup(nsblk->lbasize, - INT_LBASIZE_ALIGNMENT); - blk_dev->sector_size = ((nsblk->lbasize >= 4096) ? 4096 : 512); - dev_set_drvdata(dev, blk_dev); - - ndns->rw_bytes = nd_blk_rw_bytes; + nsblk->size = nvdimm_namespace_capacity(ndns); + dev_set_drvdata(dev, nsblk); + + ndns->rw_bytes = nsblk_rw_bytes; if (is_nd_btt(dev)) - rc = nvdimm_namespace_attach_btt(ndns); - else if (nd_btt_probe(ndns, blk_dev) == 0) { + return nvdimm_namespace_attach_btt(ndns); + else if (nd_btt_probe(dev, ndns) == 0) { /* we'll come back as btt-blk */ - rc = -ENXIO; + return -ENXIO; } else - rc = nd_blk_attach_disk(ndns, blk_dev); - if (rc) - kfree(blk_dev); - return rc; -} - -static void nd_blk_detach_disk(struct nd_blk_device *blk_dev) -{ - del_gendisk(blk_dev->disk); - put_disk(blk_dev->disk); - blk_cleanup_queue(blk_dev->queue); + return nsblk_attach_disk(nsblk); } static int nd_blk_remove(struct device *dev) { - struct nd_blk_device *blk_dev = dev_get_drvdata(dev); - if (is_nd_btt(dev)) - nvdimm_namespace_detach_btt(to_nd_btt(dev)->ndns); - else - nd_blk_detach_disk(blk_dev); - kfree(blk_dev); - + nvdimm_namespace_detach_btt(to_nd_btt(dev)); return 0; } diff --git a/drivers/nvdimm/btt.c b/drivers/nvdimm/btt.c index f068b6513cd2..cc9fafed9362 100644 --- a/drivers/nvdimm/btt.c +++ b/drivers/nvdimm/btt.c @@ -1306,7 +1306,7 @@ static struct btt *btt_init(struct nd_btt *nd_btt, unsigned long long rawsize, struct btt *btt; struct device *dev = &nd_btt->dev; - btt = kzalloc(sizeof(struct btt), GFP_KERNEL); + btt = devm_kzalloc(dev, sizeof(struct btt), GFP_KERNEL); if (!btt) return NULL; @@ -1321,13 +1321,13 @@ static struct btt *btt_init(struct nd_btt *nd_btt, unsigned long long rawsize, ret = discover_arenas(btt); if (ret) { dev_err(dev, "init: error in arena_discover: %d\n", ret); - goto out_free; + return NULL; } if (btt->init_state != INIT_READY && nd_region->ro) { dev_info(dev, "%s is read-only, unable to init btt metadata\n", dev_name(&nd_region->dev)); - goto out_free; + return NULL; } else if (btt->init_state != INIT_READY) { btt->num_arenas = (rawsize / ARENA_MAX_SIZE) + ((rawsize % ARENA_MAX_SIZE) ? 1 : 0); @@ -1337,29 +1337,25 @@ static struct btt *btt_init(struct nd_btt *nd_btt, unsigned long long rawsize, ret = create_arenas(btt); if (ret) { dev_info(dev, "init: create_arenas: %d\n", ret); - goto out_free; + return NULL; } ret = btt_meta_init(btt); if (ret) { dev_err(dev, "init: error in meta_init: %d\n", ret); - goto out_free; + return NULL; } } ret = btt_blk_init(btt); if (ret) { dev_err(dev, "init: error in blk_init: %d\n", ret); - goto out_free; + return NULL; } btt_debugfs_init(btt); return btt; - - out_free: - kfree(btt); - return NULL; } /** @@ -1377,7 +1373,6 @@ static void btt_fini(struct btt *btt) btt_blk_cleanup(btt); free_arenas(btt); debugfs_remove_recursive(btt->debugfs_dir); - kfree(btt); } } @@ -1406,9 +1401,8 @@ int nvdimm_namespace_attach_btt(struct nd_namespace_common *ndns) } EXPORT_SYMBOL(nvdimm_namespace_attach_btt); -int nvdimm_namespace_detach_btt(struct nd_namespace_common *ndns) +int nvdimm_namespace_detach_btt(struct nd_btt *nd_btt) { - struct nd_btt *nd_btt = to_nd_btt(ndns->claim); struct btt *btt = nd_btt->btt; btt_fini(btt); diff --git a/drivers/nvdimm/btt_devs.c b/drivers/nvdimm/btt_devs.c index cb477518dd0e..816d0dae6398 100644 --- a/drivers/nvdimm/btt_devs.c +++ b/drivers/nvdimm/btt_devs.c @@ -273,10 +273,10 @@ static int __nd_btt_probe(struct nd_btt *nd_btt, return 0; } -int nd_btt_probe(struct nd_namespace_common *ndns, void *drvdata) +int nd_btt_probe(struct device *dev, struct nd_namespace_common *ndns) { int rc; - struct device *dev; + struct device *btt_dev; struct btt_sb *btt_sb; struct nd_region *nd_region = to_nd_region(ndns->dev.parent); @@ -284,21 +284,19 @@ int nd_btt_probe(struct nd_namespace_common *ndns, void *drvdata) return -ENODEV; nvdimm_bus_lock(&ndns->dev); - dev = __nd_btt_create(nd_region, 0, NULL, ndns); + btt_dev = __nd_btt_create(nd_region, 0, NULL, ndns); nvdimm_bus_unlock(&ndns->dev); - if (!dev) + if (!btt_dev) return -ENOMEM; - dev_set_drvdata(dev, drvdata); - btt_sb = kzalloc(sizeof(*btt_sb), GFP_KERNEL); - rc = __nd_btt_probe(to_nd_btt(dev), ndns, btt_sb); - kfree(btt_sb); - dev_dbg(&ndns->dev, "%s: btt: %s\n", __func__, - rc == 0 ? dev_name(dev) : "<none>"); + btt_sb = devm_kzalloc(dev, sizeof(*btt_sb), GFP_KERNEL); + rc = __nd_btt_probe(to_nd_btt(btt_dev), ndns, btt_sb); + dev_dbg(dev, "%s: btt: %s\n", __func__, + rc == 0 ? dev_name(btt_dev) : "<none>"); if (rc < 0) { - struct nd_btt *nd_btt = to_nd_btt(dev); + struct nd_btt *nd_btt = to_nd_btt(btt_dev); - __nd_detach_ndns(dev, &nd_btt->ndns); - put_device(dev); + __nd_detach_ndns(btt_dev, &nd_btt->ndns); + put_device(btt_dev); } return rc; diff --git a/drivers/nvdimm/bus.c b/drivers/nvdimm/bus.c index 19f822d7f652..97589e3cb852 100644 --- a/drivers/nvdimm/bus.c +++ b/drivers/nvdimm/bus.c @@ -40,6 +40,8 @@ static int to_nd_device_type(struct device *dev) return ND_DEVICE_REGION_PMEM; else if (is_nd_blk(dev)) return ND_DEVICE_REGION_BLK; + else if (is_nd_dax(dev)) + return ND_DEVICE_DAX_PMEM; else if (is_nd_pmem(dev->parent) || is_nd_blk(dev->parent)) return nd_region_to_nstype(to_nd_region(dev->parent)); @@ -246,6 +248,8 @@ static void nd_async_device_unregister(void *d, async_cookie_t cookie) void __nd_device_register(struct device *dev) { + if (!dev) + return; dev->bus = &nvdimm_bus_type; get_device(dev); async_schedule_domain(nd_async_device_register, dev, diff --git a/drivers/nvdimm/claim.c b/drivers/nvdimm/claim.c index e8f03b0e95e4..5f53db59a058 100644 --- a/drivers/nvdimm/claim.c +++ b/drivers/nvdimm/claim.c @@ -12,6 +12,7 @@ */ #include <linux/device.h> #include <linux/sizes.h> +#include <linux/pmem.h> #include "nd-core.h" #include "pfn.h" #include "btt.h" @@ -84,6 +85,8 @@ static bool is_idle(struct device *dev, struct nd_namespace_common *ndns) seed = nd_region->btt_seed; else if (is_nd_pfn(dev)) seed = nd_region->pfn_seed; + else if (is_nd_dax(dev)) + seed = nd_region->dax_seed; if (seed == dev || ndns || dev->driver) return false; @@ -199,3 +202,63 @@ u64 nd_sb_checksum(struct nd_gen_sb *nd_gen_sb) return sum; } EXPORT_SYMBOL(nd_sb_checksum); + +static int nsio_rw_bytes(struct nd_namespace_common *ndns, + resource_size_t offset, void *buf, size_t size, int rw) +{ + struct nd_namespace_io *nsio = to_nd_namespace_io(&ndns->dev); + + if (unlikely(offset + size > nsio->size)) { + dev_WARN_ONCE(&ndns->dev, 1, "request out of range\n"); + return -EFAULT; + } + + if (rw == READ) { + unsigned int sz_align = ALIGN(size + (offset & (512 - 1)), 512); + + if (unlikely(is_bad_pmem(&nsio->bb, offset / 512, sz_align))) + return -EIO; + return memcpy_from_pmem(buf, nsio->addr + offset, size); + } else { + memcpy_to_pmem(nsio->addr + offset, buf, size); + wmb_pmem(); + } + + return 0; +} + +int devm_nsio_enable(struct device *dev, struct nd_namespace_io *nsio) +{ + struct resource *res = &nsio->res; + struct nd_namespace_common *ndns = &nsio->common; + + nsio->size = resource_size(res); + if (!devm_request_mem_region(dev, res->start, resource_size(res), + dev_name(dev))) { + dev_warn(dev, "could not reserve region %pR\n", res); + return -EBUSY; + } + + ndns->rw_bytes = nsio_rw_bytes; + if (devm_init_badblocks(dev, &nsio->bb)) + return -ENOMEM; + nvdimm_badblocks_populate(to_nd_region(ndns->dev.parent), &nsio->bb, + &nsio->res); + + nsio->addr = devm_memremap(dev, res->start, resource_size(res), + ARCH_MEMREMAP_PMEM); + if (IS_ERR(nsio->addr)) + return PTR_ERR(nsio->addr); + return 0; +} +EXPORT_SYMBOL_GPL(devm_nsio_enable); + +void devm_nsio_disable(struct device *dev, struct nd_namespace_io *nsio) +{ + struct resource *res = &nsio->res; + + devm_memunmap(dev, nsio->addr); + devm_exit_badblocks(dev, &nsio->bb); + devm_release_mem_region(dev, res->start, resource_size(res)); +} +EXPORT_SYMBOL_GPL(devm_nsio_disable); diff --git a/drivers/nvdimm/dax_devs.c b/drivers/nvdimm/dax_devs.c new file mode 100644 index 000000000000..f90f7549e7f4 --- /dev/null +++ b/drivers/nvdimm/dax_devs.c @@ -0,0 +1,99 @@ +/* + * Copyright(c) 2013-2016 Intel Corporation. All rights reserved. + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of version 2 of the GNU General Public License as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * General Public License for more details. + */ +#include <linux/device.h> +#include <linux/sizes.h> +#include <linux/slab.h> +#include <linux/mm.h> +#include "nd-core.h" +#include "nd.h" + +static void nd_dax_release(struct device *dev) +{ + struct nd_region *nd_region = to_nd_region(dev->parent); + struct nd_dax *nd_dax = to_nd_dax(dev); + struct nd_pfn *nd_pfn = &nd_dax->nd_pfn; + + dev_dbg(dev, "%s\n", __func__); + nd_detach_ndns(dev, &nd_pfn->ndns); + ida_simple_remove(&nd_region->dax_ida, nd_pfn->id); + kfree(nd_pfn->uuid); + kfree(nd_dax); +} + +static struct device_type nd_dax_device_type = { + .name = "nd_dax", + .release = nd_dax_release, +}; + +bool is_nd_dax(struct device *dev) +{ + return dev ? dev->type == &nd_dax_device_type : false; +} +EXPORT_SYMBOL(is_nd_dax); + +struct nd_dax *to_nd_dax(struct device *dev) +{ + struct nd_dax *nd_dax = container_of(dev, struct nd_dax, nd_pfn.dev); + + WARN_ON(!is_nd_dax(dev)); + return nd_dax; +} +EXPORT_SYMBOL(to_nd_dax); + +static const struct attribute_group *nd_dax_attribute_groups[] = { + &nd_pfn_attribute_group, + &nd_device_attribute_group, + &nd_numa_attribute_group, + NULL, +}; + +static struct nd_dax *nd_dax_alloc(struct nd_region *nd_region) +{ + struct nd_pfn *nd_pfn; + struct nd_dax *nd_dax; + struct device *dev; + + nd_dax = kzalloc(sizeof(*nd_dax), GFP_KERNEL); + if (!nd_dax) + return NULL; + + nd_pfn = &nd_dax->nd_pfn; + nd_pfn->id = ida_simple_get(&nd_region->dax_ida, 0, 0, GFP_KERNEL); + if (nd_pfn->id < 0) { + kfree(nd_dax); + return NULL; + } + + dev = &nd_pfn->dev; + dev_set_name(dev, "dax%d.%d", nd_region->id, nd_pfn->id); + dev->groups = nd_dax_attribute_groups; + dev->type = &nd_dax_device_type; + dev->parent = &nd_region->dev; + + return nd_dax; +} + +struct device *nd_dax_create(struct nd_region *nd_region) +{ + struct device *dev = NULL; + struct nd_dax *nd_dax; + + if (!is_nd_pmem(&nd_region->dev)) + return NULL; + + nd_dax = nd_dax_alloc(nd_region); + if (nd_dax) + dev = nd_pfn_devinit(&nd_dax->nd_pfn, NULL); + __nd_device_register(dev); + return dev; +} diff --git a/drivers/nvdimm/namespace_devs.c b/drivers/nvdimm/namespace_devs.c index f5cb88601359..c5e3196c45b0 100644 --- a/drivers/nvdimm/namespace_devs.c +++ b/drivers/nvdimm/namespace_devs.c @@ -1288,6 +1288,8 @@ static ssize_t mode_show(struct device *dev, mode = "safe"; else if (claim && is_nd_pfn(claim)) mode = "memory"; + else if (claim && is_nd_dax(claim)) + mode = "dax"; else if (!claim && pmem_should_map_pages(dev)) mode = "memory"; else @@ -1379,21 +1381,19 @@ struct nd_namespace_common *nvdimm_namespace_common_probe(struct device *dev) { struct nd_btt *nd_btt = is_nd_btt(dev) ? to_nd_btt(dev) : NULL; struct nd_pfn *nd_pfn = is_nd_pfn(dev) ? to_nd_pfn(dev) : NULL; - struct nd_namespace_common *ndns; + struct nd_dax *nd_dax = is_nd_dax(dev) ? to_nd_dax(dev) : NULL; + struct nd_namespace_common *ndns = NULL; resource_size_t size; - if (nd_btt || nd_pfn) { - struct device *host = NULL; - - if (nd_btt) { - host = &nd_btt->dev; + if (nd_btt || nd_pfn || nd_dax) { + if (nd_btt) ndns = nd_btt->ndns; - } else if (nd_pfn) { - host = &nd_pfn->dev; + else if (nd_pfn) ndns = nd_pfn->ndns; - } + else if (nd_dax) + ndns = nd_dax->nd_pfn.ndns; - if (!ndns || !host) + if (!ndns) return ERR_PTR(-ENODEV); /* @@ -1404,12 +1404,12 @@ struct nd_namespace_common *nvdimm_namespace_common_probe(struct device *dev) device_unlock(&ndns->dev); if (ndns->dev.driver) { dev_dbg(&ndns->dev, "is active, can't bind %s\n", - dev_name(host)); + dev_name(dev)); return ERR_PTR(-EBUSY); } - if (dev_WARN_ONCE(&ndns->dev, ndns->claim != host, + if (dev_WARN_ONCE(&ndns->dev, ndns->claim != dev, "host (%s) vs claim (%s) mismatch\n", - dev_name(host), + dev_name(dev), dev_name(ndns->claim))) return ERR_PTR(-ENXIO); } else { @@ -1784,6 +1784,18 @@ void nd_region_create_blk_seed(struct nd_region *nd_region) nd_device_register(nd_region->ns_seed); } +void nd_region_create_dax_seed(struct nd_region *nd_region) +{ + WARN_ON(!is_nvdimm_bus_locked(&nd_region->dev)); + nd_region->dax_seed = nd_dax_create(nd_region); + /* + * Seed creation failures are not fatal, provisioning is simply + * disabled until memory becomes available + */ + if (!nd_region->dax_seed) + dev_err(&nd_region->dev, "failed to create dax namespace\n"); +} + void nd_region_create_pfn_seed(struct nd_region *nd_region) { WARN_ON(!is_nvdimm_bus_locked(&nd_region->dev)); diff --git a/drivers/nvdimm/nd-core.h b/drivers/nvdimm/nd-core.h index 1d1500f3d8b5..cb65308c0329 100644 --- a/drivers/nvdimm/nd-core.h +++ b/drivers/nvdimm/nd-core.h @@ -54,6 +54,7 @@ struct nd_region; void nd_region_create_blk_seed(struct nd_region *nd_region); void nd_region_create_btt_seed(struct nd_region *nd_region); void nd_region_create_pfn_seed(struct nd_region *nd_region); +void nd_region_create_dax_seed(struct nd_region *nd_region); void nd_region_disable(struct nvdimm_bus *nvdimm_bus, struct device *dev); int nvdimm_bus_create_ndctl(struct nvdimm_bus *nvdimm_bus); void nvdimm_bus_destroy_ndctl(struct nvdimm_bus *nvdimm_bus); diff --git a/drivers/nvdimm/nd.h b/drivers/nvdimm/nd.h index 875c524fafb0..46910b8f32b1 100644 --- a/drivers/nvdimm/nd.h +++ b/drivers/nvdimm/nd.h @@ -13,6 +13,7 @@ #ifndef __ND_H__ #define __ND_H__ #include <linux/libnvdimm.h> +#include <linux/badblocks.h> #include <linux/blkdev.h> #include <linux/device.h> #include <linux/mutex.h> @@ -100,10 +101,12 @@ struct nd_region { struct ida ns_ida; struct ida btt_ida; struct ida pfn_ida; + struct ida dax_ida; unsigned long flags; struct device *ns_seed; struct device *btt_seed; struct device *pfn_seed; + struct device *dax_seed; u16 ndr_mappings; u64 ndr_size; u64 ndr_start; @@ -160,6 +163,10 @@ struct nd_pfn { struct nd_namespace_common *ndns; }; +struct nd_dax { + struct nd_pfn nd_pfn; +}; + enum nd_async_mode { ND_SYNC, ND_ASYNC, @@ -197,11 +204,12 @@ struct nd_gen_sb { u64 nd_sb_checksum(struct nd_gen_sb *sb); #if IS_ENABLED(CONFIG_BTT) -int nd_btt_probe(struct nd_namespace_common *ndns, void *drvdata); +int nd_btt_probe(struct device *dev, struct nd_namespace_common *ndns); bool is_nd_btt(struct device *dev); struct device *nd_btt_create(struct nd_region *nd_region); #else -static inline int nd_btt_probe(struct nd_namespace_common *ndns, void *drvdata) +static inline int nd_btt_probe(struct device *dev, + struct nd_namespace_common *ndns) { return -ENODEV; } @@ -219,12 +227,16 @@ static inline struct device *nd_btt_create(struct nd_region *nd_region) struct nd_pfn *to_nd_pfn(struct device *dev); #if IS_ENABLED(CONFIG_NVDIMM_PFN) -int nd_pfn_probe(struct nd_namespace_common *ndns, void *drvdata); +int nd_pfn_probe(struct device *dev, struct nd_namespace_common *ndns); bool is_nd_pfn(struct device *dev); struct device *nd_pfn_create(struct nd_region *nd_region); +struct device *nd_pfn_devinit(struct nd_pfn *nd_pfn, + struct nd_namespace_common *ndns); int nd_pfn_validate(struct nd_pfn *nd_pfn); +extern struct attribute_group nd_pfn_attribute_group; #else -static inline int nd_pfn_probe(struct nd_namespace_common *ndns, void *drvdata) +static inline int nd_pfn_probe(struct device *dev, + struct nd_namespace_common *ndns) { return -ENODEV; } @@ -245,6 +257,22 @@ static inline int nd_pfn_validate(struct nd_pfn *nd_pfn) } #endif +struct nd_dax *to_nd_dax(struct device *dev); +#if IS_ENABLED(CONFIG_NVDIMM_DAX) +bool is_nd_dax(struct device *dev); +struct device *nd_dax_create(struct nd_region *nd_region); +#else +static inline bool is_nd_dax(struct device *dev) +{ + return false; +} + +static inline struct device *nd_dax_create(struct nd_region *nd_region) +{ + return NULL; +} +#endif + struct nd_region *to_nd_region(struct device *dev); int nd_region_to_nstype(struct nd_region *nd_region); int nd_region_register_namespaces(struct nd_region *nd_region, int *err); @@ -263,11 +291,32 @@ struct resource *nvdimm_allocate_dpa(struct nvdimm_drvdata *ndd, resource_size_t nvdimm_namespace_capacity(struct nd_namespace_common *ndns); struct nd_namespace_common *nvdimm_namespace_common_probe(struct device *dev); int nvdimm_namespace_attach_btt(struct nd_namespace_common *ndns); -int nvdimm_namespace_detach_btt(struct nd_namespace_common *ndns); +int nvdimm_namespace_detach_btt(struct nd_btt *nd_btt); const char *nvdimm_namespace_disk_name(struct nd_namespace_common *ndns, char *name); void nvdimm_badblocks_populate(struct nd_region *nd_region, struct badblocks *bb, const struct resource *res); +#if IS_ENABLED(CONFIG_ND_CLAIM) +struct vmem_altmap *nvdimm_setup_pfn(struct nd_pfn *nd_pfn, + struct resource *res, struct vmem_altmap *altmap); +int devm_nsio_enable(struct device *dev, struct nd_namespace_io *nsio); +void devm_nsio_disable(struct device *dev, struct nd_namespace_io *nsio); +#else +static inline struct vmem_altmap *nvdimm_setup_pfn(struct nd_pfn *nd_pfn, + struct resource *res, struct vmem_altmap *altmap) +{ + return ERR_PTR(-ENXIO); +} +static inline int devm_nsio_enable(struct device *dev, + |