/*
* Copyright (C) 2001 Jens Axboe <axboe@suse.de>
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License version 2 as
* published by the Free Software Foundation.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public Licens
* along with this program; if not, write to the Free Software
* Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-
*
*/
#include <linux/mm.h>
#include <linux/swap.h>
#include <linux/bio.h>
#include <linux/blkdev.h>
#include <linux/slab.h>
#include <linux/init.h>
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/mempool.h>
#include <linux/workqueue.h>
#define BIO_POOL_SIZE 256
static kmem_cache_t *bio_slab;
#define BIOVEC_NR_POOLS 6
/*
* a small number of entries is fine, not going to be performance critical.
* basically we just need to survive
*/
#define BIO_SPLIT_ENTRIES 8
mempool_t *bio_split_pool;
struct biovec_slab {
int nr_vecs;
char *name;
kmem_cache_t *slab;
};
/*
* if you change this list, also change bvec_alloc or things will
* break badly! cannot be bigger than what you can fit into an
* unsigned short
*/
#define BV(x) { .nr_vecs = x, .name = "biovec-"__stringify(x) }
static struct biovec_slab bvec_slabs[BIOVEC_NR_POOLS] __read_mostly = {
BV(1), BV(4), BV(16), BV(64), BV(128), BV(BIO_MAX_PAGES),
};
#undef BV
/*
* bio_set is used to allow other portions of the IO system to
* allocate their own private memory pools for bio and iovec structures.
* These memory pools in turn all allocate from the bio_slab
* and the bvec_slabs[].
*/
struct bio_set {
mempool_t *bio_pool;
mempool_t *bvec_pools[BIOVEC_NR_POOLS];
};
/*
* fs_bio_set is the bio_set containing bio and iovec memory pools used by
* IO code that does not need private memory pools.
*/
static struct bio_set *fs_bio_set;
static inline struct bio_vec *bvec_alloc_bs(unsigned int __nocast gfp_mask, int nr, unsigned long *idx, struct bio_set *bs)
{
struct bio_vec *bvl;
struct biovec_slab *bp;
/*
* see comment near bvec_array define!
*/
switch (nr) {
case 1 : *idx = 0; break;
case 2 ... 4: *idx = 1; break;
case 5 ... 16: *idx = 2; break;
case 17 ... 64: *idx = 3; break;
case 65 ... 128: *idx = 4; break;
case 129 ... BIO_MAX_PAGES: *idx = 5; break;
default:
return NULL;
}
/*
* idx now points to the pool we want to allocate from
*/
bp = bvec_slabs + *idx;
bvl = mempool_alloc(bs->bvec_pools[*idx], gfp_mask);
if (bvl)
memset(bvl, 0, bp->nr_vecs * sizeof(struct bio_vec));
return bvl;
}
/*
* default destructor for a bio allocated with bio_alloc_bioset()
*/
static void bio_destructor(struct bio *bio)
{
const int pool_idx = BIO_POOL_IDX(bio);
struct bio_set *bs = bio->bi_set;
BIO_BUG_ON(pool_idx >= BIOVEC_NR_POOLS);
mempool_free(bio->bi_io_vec, bs->bvec_pools[pool_idx]);
mempool_free(bio, bs->bio_pool);
}
inline void bio_init(struct bio *bio)
{
bio->bi_next = NULL;
bio->bi_flags = 1 << BIO_UPTODATE;
bio->bi_rw = 0;
bio->bi_vcnt = 0;
bio->bi_idx = 0;
bio->bi_phys_segments = 0;
bio->bi_hw_segments = 0;
bio->bi_hw_front_size = 0;
bio->bi_hw_back_size = 0;
bio->bi_size = 0;
bio->bi_max_vecs = 0;
bio->bi_end_io = NULL;
atomic_set(&bio->bi_cnt, 1);
bio->bi_private = NULL;
}
/**
* bio_alloc_bioset - allocate a bio for I/O
* @gfp_mask: the GFP_ mask given to the slab allocator
* @nr_iovecs: number of iovecs to pre-allocate
* @bs: the bio_set to allocate from
*
* Description:
* bio_alloc_bioset will first try it's on mempool to satisfy the allocation.
* If %__GFP_WAIT is set then we wil