diff options
65 files changed, 1210 insertions, 1172 deletions
diff --git a/Documentation/ABI/testing/sysfs-block b/Documentation/ABI/testing/sysfs-block index 279da08f7541..8df003963d99 100644 --- a/Documentation/ABI/testing/sysfs-block +++ b/Documentation/ABI/testing/sysfs-block @@ -53,6 +53,14 @@ Description: 512 bytes of data. +What: /sys/block/<disk>/integrity/device_is_integrity_capable +Date: July 2014 +Contact: Martin K. Petersen <martin.petersen@oracle.com> +Description: + Indicates whether a storage device is capable of storing + integrity metadata. Set if the device is T10 PI-capable. + + What: /sys/block/<disk>/integrity/write_generate Date: June 2008 Contact: Martin K. Petersen <martin.petersen@oracle.com> diff --git a/Documentation/block/data-integrity.txt b/Documentation/block/data-integrity.txt index 2d735b0ae383..f56ec97f0d14 100644 --- a/Documentation/block/data-integrity.txt +++ b/Documentation/block/data-integrity.txt @@ -129,11 +129,11 @@ interface for this is being worked on. 4.1 BIO The data integrity patches add a new field to struct bio when -CONFIG_BLK_DEV_INTEGRITY is enabled. bio->bi_integrity is a pointer -to a struct bip which contains the bio integrity payload. Essentially -a bip is a trimmed down struct bio which holds a bio_vec containing -the integrity metadata and the required housekeeping information (bvec -pool, vector count, etc.) +CONFIG_BLK_DEV_INTEGRITY is enabled. bio_integrity(bio) returns a +pointer to a struct bip which contains the bio integrity payload. +Essentially a bip is a trimmed down struct bio which holds a bio_vec +containing the integrity metadata and the required housekeeping +information (bvec pool, vector count, etc.) A kernel subsystem can enable data integrity protection on a bio by calling bio_integrity_alloc(bio). This will allocate and attach the @@ -192,16 +192,6 @@ will require extra work due to the application tag. supported by the block device. - int bdev_integrity_enabled(block_device, int rw); - - bdev_integrity_enabled() will return 1 if the block device - supports integrity metadata transfer for the data direction - specified in 'rw'. - - bdev_integrity_enabled() honors the write_generate and - read_verify flags in sysfs and will respond accordingly. - - int bio_integrity_prep(bio); To generate IMD for WRITE and to set up buffers for READ, the @@ -216,36 +206,6 @@ will require extra work due to the application tag. bio_integrity_enabled() returned 1. - int bio_integrity_tag_size(bio); - - If the filesystem wants to use the application tag space it will - first have to find out how much storage space is available. - Because tag space is generally limited (usually 2 bytes per - sector regardless of sector size), the integrity framework - supports interleaving the information between the sectors in an - I/O. - - Filesystems can call bio_integrity_tag_size(bio) to find out how - many bytes of storage are available for that particular bio. - - Another option is bdev_get_tag_size(block_device) which will - return the number of available bytes per hardware sector. - - - int bio_integrity_set_tag(bio, void *tag_buf, len); - - After a successful return from bio_integrity_prep(), - bio_integrity_set_tag() can be used to attach an opaque tag - buffer to a bio. Obviously this only makes sense if the I/O is - a WRITE. - - - int bio_integrity_get_tag(bio, void *tag_buf, len); - - Similarly, at READ I/O completion time the filesystem can - retrieve the tag buffer using bio_integrity_get_tag(). - - 5.3 PASSING EXISTING INTEGRITY METADATA Filesystems that either generate their own integrity metadata or @@ -298,8 +258,6 @@ will require extra work due to the application tag. .name = "STANDARDSBODY-TYPE-VARIANT-CSUM", .generate_fn = my_generate_fn, .verify_fn = my_verify_fn, - .get_tag_fn = my_get_tag_fn, - .set_tag_fn = my_set_tag_fn, .tuple_size = sizeof(struct my_tuple_size), .tag_size = <tag bytes per hw sector>, }; @@ -321,7 +279,5 @@ will require extra work due to the application tag. are available per hardware sector. For DIF this is either 2 or 0 depending on the value of the Control Mode Page ATO bit. - See 6.2 for a description of get_tag_fn and set_tag_fn. - ---------------------------------------------------------------------- 2007-12-24 Martin K. Petersen <martin.petersen@oracle.com> diff --git a/block/Kconfig b/block/Kconfig index 2429515c05c2..161491d0a879 100644 --- a/block/Kconfig +++ b/block/Kconfig @@ -77,6 +77,7 @@ config BLK_DEV_BSGLIB config BLK_DEV_INTEGRITY bool "Block layer data integrity support" + select CRC_T10DIF if BLK_DEV_INTEGRITY ---help--- Some storage devices allow extra information to be stored/retrieved to help protect the data. The block layer diff --git a/block/Makefile b/block/Makefile index a2ce6ac935ec..00ecc97629db 100644 --- a/block/Makefile +++ b/block/Makefile @@ -20,6 +20,6 @@ obj-$(CONFIG_IOSCHED_DEADLINE) += deadline-iosched.o obj-$(CONFIG_IOSCHED_CFQ) += cfq-iosched.o obj-$(CONFIG_BLOCK_COMPAT) += compat_ioctl.o -obj-$(CONFIG_BLK_DEV_INTEGRITY) += blk-integrity.o obj-$(CONFIG_BLK_CMDLINE_PARSER) += cmdline-parser.o -obj-$(CONFIG_BLK_DEV_INTEGRITY) += bio-integrity.o +obj-$(CONFIG_BLK_DEV_INTEGRITY) += bio-integrity.o blk-integrity.o t10-pi.o + diff --git a/block/bio-integrity.c b/block/bio-integrity.c index f14b4abbebd8..0984232e429f 100644 --- a/block/bio-integrity.c +++ b/block/bio-integrity.c @@ -79,6 +79,7 @@ struct bio_integrity_payload *bio_integrity_alloc(struct bio *bio, bip->bip_slab = idx; bip->bip_bio = bio; bio->bi_integrity = bip; + bio->bi_rw |= REQ_INTEGRITY; return bip; err: @@ -96,11 +97,12 @@ EXPORT_SYMBOL(bio_integrity_alloc); */ void bio_integrity_free(struct bio *bio) { - struct bio_integrity_payload *bip = bio->bi_integrity; + struct bio_integrity_payload *bip = bio_integrity(bio); struct bio_set *bs = bio->bi_pool; - if (bip->bip_owns_buf) - kfree(bip->bip_buf); + if (bip->bip_flags & BIP_BLOCK_INTEGRITY) + kfree(page_address(bip->bip_vec->bv_page) + + bip->bip_vec->bv_offset); if (bs) { if (bip->bip_slab != BIO_POOL_NONE) @@ -128,7 +130,7 @@ EXPORT_SYMBOL(bio_integrity_free); int bio_integrity_add_page(struct bio *bio, struct page *page, unsigned int len, unsigned int offset) { - struct bio_integrity_payload *bip = bio->bi_integrity; + struct bio_integrity_payload *bip = bio_integrity(bio); struct bio_vec *iv; if (bip->bip_vcnt >= bip->bip_max_vcnt) { @@ -147,24 +149,6 @@ int bio_integrity_add_page(struct bio *bio, struct page *page, } EXPORT_SYMBOL(bio_integrity_add_page); -static int bdev_integrity_enabled(struct block_device *bdev, int rw) -{ - struct blk_integrity *bi = bdev_get_integrity(bdev); - - if (bi == NULL) - return 0; - - if (rw == READ && bi->verify_fn != NULL && - (bi->flags & INTEGRITY_FLAG_READ)) - return 1; - - if (rw == WRITE && bi->generate_fn != NULL && - (bi->flags & INTEGRITY_FLAG_WRITE)) - return 1; - - return 0; -} - /** * bio_integrity_enabled - Check whether integrity can be passed * @bio: bio to check @@ -174,171 +158,86 @@ static int bdev_integrity_enabled(struct block_device *bdev, int rw) * set prior to calling. The functions honors the write_generate and * read_verify flags in sysfs. */ -int bio_integrity_enabled(struct bio *bio) +bool bio_integrity_enabled(struct bio *bio) { + struct blk_integrity *bi = bdev_get_integrity(bio->bi_bdev); + if (!bio_is_rw(bio)) - return 0; + return false; /* Already protected? */ if (bio_integrity(bio)) - return 0; + return false; + + if (bi == NULL) + return false; + + if (bio_data_dir(bio) == READ && bi->verify_fn != NULL && + (bi->flags & BLK_INTEGRITY_VERIFY)) + return true; + + if (bio_data_dir(bio) == WRITE && bi->generate_fn != NULL && + (bi->flags & BLK_INTEGRITY_GENERATE)) + return true; - return bdev_integrity_enabled(bio->bi_bdev, bio_data_dir(bio)); + return false; } EXPORT_SYMBOL(bio_integrity_enabled); /** - * bio_integrity_hw_sectors - Convert 512b sectors to hardware ditto + * bio_integrity_intervals - Return number of integrity intervals for a bio * @bi: blk_integrity profile for device - * @sectors: Number of 512 sectors to convert + * @sectors: Size of the bio in 512-byte sectors * * Description: The block layer calculates everything in 512 byte - * sectors but integrity metadata is done in terms of the hardware - * sector size of the storage device. Convert the block layer sectors - * to physical sectors. + * sectors but integrity metadata is done in terms of the data integrity + * interval size of the storage device. Convert the block layer sectors + * to the appropriate number of integrity intervals. */ -static inline unsigned int bio_integrity_hw_sectors(struct blk_integrity *bi, - unsigned int sectors) +static inline unsigned int bio_integrity_intervals(struct blk_integrity *bi, + unsigned int sectors) { - /* At this point there are only 512b or 4096b DIF/EPP devices */ - if (bi->sector_size == 4096) - return sectors >>= 3; - - return sectors; + return sectors >> (ilog2(bi->interval) - 9); } static inline unsigned int bio_integrity_bytes(struct blk_integrity *bi, unsigned int sectors) { - return bio_integrity_hw_sectors(bi, sectors) * bi->tuple_size; + return bio_integrity_intervals(bi, sectors) * bi->tuple_size; } /** - * bio_integrity_tag_size - Retrieve integrity tag space - * @bio: bio to inspect - * - * Description: Returns the maximum number of tag bytes that can be - * attached to this bio. Filesystems can use this to determine how - * much metadata to attach to an I/O. - */ -unsigned int bio_integrity_tag_size(struct bio *bio) -{ - struct blk_integrity *bi = bdev_get_integrity(bio->bi_bdev); - - BUG_ON(bio->bi_iter.bi_size == 0); - - return bi->tag_size * (bio->bi_iter.bi_size / bi->sector_size); -} -EXPORT_SYMBOL(bio_integrity_tag_size); - -static int bio_integrity_tag(struct bio *bio, void *tag_buf, unsigned int len, - int set) -{ - struct bio_integrity_payload *bip = bio->bi_integrity; - struct blk_integrity *bi = bdev_get_integrity(bio->bi_bdev); - unsigned int nr_sectors; - - BUG_ON(bip->bip_buf == NULL); - - if (bi->tag_size == 0) - return -1; - - nr_sectors = bio_integrity_hw_sectors(bi, - DIV_ROUND_UP(len, bi->tag_size)); - - if (nr_sectors * bi->tuple_size > bip->bip_iter.bi_size) { - printk(KERN_ERR "%s: tag too big for bio: %u > %u\n", __func__, - nr_sectors * bi->tuple_size, bip->bip_iter.bi_size); - return -1; - } - - if (set) - bi->set_tag_fn(bip->bip_buf, tag_buf, nr_sectors); - else - bi->get_tag_fn(bip->bip_buf, tag_buf, nr_sectors); - - return 0; -} - -/** - * bio_integrity_set_tag - Attach a tag buffer to a bio - * @bio: bio to attach buffer to - * @tag_buf: Pointer to a buffer containing tag data - * @len: Length of the included buffer - * - * Description: Use this function to tag a bio by leveraging the extra - * space provided by devices formatted with integrity protection. The - * size of the integrity buffer must be <= to the size reported by - * bio_integrity_tag_size(). - */ -int bio_integrity_set_tag(struct bio *bio, void *tag_buf, unsigned int len) -{ - BUG_ON(bio_data_dir(bio) != WRITE); - - return bio_integrity_tag(bio, tag_buf, len, 1); -} -EXPORT_SYMBOL(bio_integrity_set_tag); - -/** - * bio_integrity_get_tag - Retrieve a tag buffer from a bio - * @bio: bio to retrieve buffer from - * @tag_buf: Pointer to a buffer for the tag data - * @len: Length of the target buffer - * - * Description: Use this function to retrieve the tag buffer from a - * completed I/O. The size of the integrity buffer must be <= to the - * size reported by bio_integrity_tag_size(). - */ -int bio_integrity_get_tag(struct bio *bio, void *tag_buf, unsigned int len) -{ - BUG_ON(bio_data_dir(bio) != READ); - - return bio_integrity_tag(bio, tag_buf, len, 0); -} -EXPORT_SYMBOL(bio_integrity_get_tag); - -/** - * bio_integrity_generate_verify - Generate/verify integrity metadata for a bio + * bio_integrity_process - Process integrity metadata for a bio * @bio: bio to generate/verify integrity metadata for - * @operate: operate number, 1 for generate, 0 for verify + * @proc_fn: Pointer to the relevant processing function */ -static int bio_integrity_generate_verify(struct bio *bio, int operate) +static int bio_integrity_process(struct bio *bio, + integrity_processing_fn *proc_fn) { struct blk_integrity *bi = bdev_get_integrity(bio->bi_bdev); - struct blk_integrity_exchg bix; + struct blk_integrity_iter iter; struct bio_vec *bv; - sector_t sector; - unsigned int sectors, ret = 0, i; - void *prot_buf = bio->bi_integrity->bip_buf; - - if (operate) - sector = bio->bi_iter.bi_sector; - else - sector = bio->bi_integrity->bip_iter.bi_sector; + struct bio_integrity_payload *bip = bio_integrity(bio); + unsigned int i, ret = 0; + void *prot_buf = page_address(bip->bip_vec->bv_page) + + bip->bip_vec->bv_offset; - bix.disk_name = bio->bi_bdev->bd_disk->disk_name; - bix.sector_size = bi->sector_size; + iter.disk_name = bio->bi_bdev->bd_disk->disk_name; + iter.interval = bi->interval; + iter.seed = bip_get_seed(bip); + iter.prot_buf = prot_buf; bio_for_each_segment_all(bv, bio, i) { void *kaddr = kmap_atomic(bv->bv_page); - bix.data_buf = kaddr + bv->bv_offset; - bix.data_size = bv->bv_len; - bix.prot_buf = prot_buf; - bix.sector = sector; - - if (operate) - bi->generate_fn(&bix); - else { - ret = bi->verify_fn(&bix); - if (ret) { - kunmap_atomic(kaddr); - return ret; - } - } - sectors = bv->bv_len / bi->sector_size; - sector += sectors; - prot_buf += sectors * bi->tuple_size; + iter.data_buf = kaddr + bv->bv_offset; + iter.data_size = bv->bv_len; + + ret = proc_fn(&iter); + if (ret) { + kunmap_atomic(kaddr); + return ret; + } kunmap_atomic(kaddr); } @@ -346,28 +245,6 @@ static int bio_integrity_generate_verify(struct bio *bio, int operate) } /** - * bio_integrity_generate - Generate integrity metadata for a bio - * @bio: bio to generate integrity metadata for - * - * Description: Generates integrity metadata for a bio by calling the - * block device's generation callback function. The bio must have a - * bip attached with enough room to accommodate the generated - * integrity metadata. - */ -static void bio_integrity_generate(struct bio *bio) -{ - bio_integrity_generate_verify(bio, 1); -} - -static inline unsigned short blk_integrity_tuple_size(struct blk_integrity *bi) -{ - if (bi) - return bi->tuple_size; - - return 0; -} - -/** * bio_integrity_prep - Prepare bio for integrity I/O * @bio: bio to prepare * @@ -387,17 +264,17 @@ int bio_integrity_prep(struct bio *bio) unsigned long start, end; unsigned int len, nr_pages; unsigned int bytes, offset, i; - unsigned int sectors; + unsigned int intervals; bi = bdev_get_integrity(bio->bi_bdev); q = bdev_get_queue(bio->bi_bdev); BUG_ON(bi == NULL); BUG_ON(bio_integrity(bio)); - sectors = bio_integrity_hw_sectors(bi, bio_sectors(bio)); + intervals = bio_integrity_intervals(bi, bio_sectors(bio)); /* Allocate kernel buffer for protection data */ - len = sectors * blk_integrity_tuple_size(bi); + len = intervals * bi->tuple_size; buf = kmalloc(len, GFP_NOIO | q->bounce_gfp); if (unlikely(buf == NULL)) { printk(KERN_ERR "could not allocate integrity buffer\n"); @@ -416,10 +293,12 @@ int bio_integrity_prep(struct bio *bio) return -EIO; } - bip->bip_owns_buf = 1; - bip->bip_buf = buf; + bip->bip_flags |= BIP_BLOCK_INTEGRITY; bip->bip_iter.bi_size = len; - bip->bip_iter.bi_sector = bio->bi_iter.bi_sector; + bip_set_seed(bip, bio->bi_iter.bi_sector); + + if (bi->flags & BLK_INTEGRITY_IP_CHECKSUM) + bip->bip_flags |= BIP_IP_CHECKSUM; /* Map it */ offset = offset_in_page(buf); @@ -455,26 +334,13 @@ int bio_integrity_prep(struct bio *bio) /* Auto-generate integrity metadata if this is a write */ if (bio_data_dir(bio) == WRITE) - bio_integrity_generate(bio); + bio_integrity_process(bio, bi->generate_fn); return 0; } EXPORT_SYMBOL(bio_integrity_prep); /** - * bio_integrity_verify - Verify integrity metadata for a bio - * @bio: bio to verify - * - * Description: This function is called to verify the integrity of a - * bio. The data in the bio io_vec is compared to the integrity - * metadata returned by the HBA. - */ -static int bio_integrity_verify(struct bio *bio) -{ - return bio_integrity_generate_verify(bio, 0); -} - -/** * bio_integrity_verify_fn - Integrity I/O completion worker * @work: Work struct stored in bio to be verified * @@ -487,9 +353,10 @@ static void bio_integrity_verify_fn(struct work_struct *work) struct bio_integrity_payload *bip = container_of(work, struct bio_integrity_payload, bip_work); struct bio *bio = bip->bip_bio; + struct blk_integrity *bi = bdev_get_integrity(bio->bi_bdev); int error; - error = bio_integrity_verify(bio); + error = bio_integrity_process(bio, bi->verify_fn); /* Restore original bio completion handler */ bio->bi_end_io = bip->bip_end_io; @@ -510,7 +377,7 @@ static void bio_integrity_verify_fn(struct work_struct *work) */ void bio_integrity_endio(struct bio *bio, int error) { - struct bio_integrity_payload *bip = bio->bi_integrity; + struct bio_integrity_payload *bip = bio_integrity(bio); BUG_ON(bip->bip_bio != bio); @@ -541,7 +408,7 @@ EXPORT_SYMBOL(bio_integrity_endio); */ void bio_integrity_advance(struct bio *bio, unsigned int bytes_done) { - struct bio_integrity_payload *bip = bio->bi_integrity; + struct bio_integrity_payload *bip = bio_integrity(bio); struct blk_integrity *bi = bdev_get_integrity(bio->bi_bdev); unsigned bytes = bio_integrity_bytes(bi, bytes_done >> 9); @@ -563,7 +430,7 @@ EXPORT_SYMBOL(bio_integrity_advance); void bio_integrity_trim(struct bio *bio, unsigned int offset, unsigned int sectors) { - struct bio_integrity_payload *bip = bio->bi_integrity; + struct bio_integrity_payload *bip = bio_integrity(bio); struct blk_integrity *bi = bdev_get_integrity(bio->bi_bdev); bio_integrity_advance(bio, offset << 9); @@ -582,7 +449,7 @@ EXPORT_SYMBOL(bio_integrity_trim); int bio_integrity_clone(struct bio *bio, struct bio *bio_src, gfp_t gfp_mask) { - struct bio_integrity_payload *bip_src = bio_src->bi_integrity; + struct bio_integrity_payload *bip_src = bio_integrity(bio_src); struct bio_integrity_payload *bip; BUG_ON(bip_src == NULL); @@ -646,6 +513,4 @@ void __init bio_integrity_init(void) sizeof(struct bio_integrity_payload) + sizeof(struct bio_vec) * BIP_INLINE_VECS, 0, SLAB_HWCACHE_ALIGN|SLAB_PANIC, NULL); - if (!bip_slab) - panic("Failed to create slab\n"); } diff --git a/block/bio.c b/block/bio.c index 3e6331d25d90..3e6e1986a5b2 100644 --- a/block/bio.c +++ b/block/bio.c @@ -428,6 +428,9 @@ struct bio *bio_alloc_bioset(gfp_t gfp_mask, int nr_iovecs, struct bio_set *bs) front_pad = 0; inline_vecs = nr_iovecs; } else { + /* should not use nobvec bioset for nr_iovecs > 0 */ + if (WARN_ON_ONCE(!bs->bvec_pool && nr_iovecs > 0)) + return NULL; /* * generic_make_request() converts recursion to iteration; this * means if we're running beneath it, any bios we allocate and @@ -1900,20 +1903,9 @@ void bioset_free(struct bio_set *bs) } EXPORT_SYMBOL(bioset_free); -/** - * bioset_create - Create a bio_set - * @pool_size: Number of bio and bio_vecs to cache in the mempool - * @front_pad: Number of bytes to allocate in front of the returned bio - * - * Description: - * Set up a bio_set to be used with @bio_alloc_bioset. Allows the caller - * to ask for a number of bytes to be allocated in front of the bio. - * Front pad allocation is useful for embedding the bio inside - * another structure, to avoid allocating extra data to go with the bio. - * Note that the bio must be embedded at the END of that structure always, - * or things will break badly. - */ -struct bio_set *bioset_create(unsigned int pool_size, unsigned int front_pad) +static struct bio_set *__bioset_create(unsigned int pool_size, + unsigned int front_pad, + bool create_bvec_pool) { unsigned int back_pad = BIO_INLINE_VECS * sizeof(struct bio_vec); struct bio_set |