#include <linux/highmem.h>
#include <linux/mempool.h>
+#include <linux/ioprio.h>
+
+#ifdef CONFIG_BLOCK
/* Platforms may set this to teach the BIO layer about IOMMU hardware. */
#include <asm/io.h>
#define BIO_BUG_ON
#endif
-#define BIO_MAX_PAGES (256)
+#define BIO_MAX_PAGES 256
#define BIO_MAX_SIZE (BIO_MAX_PAGES << PAGE_CACHE_SHIFT)
#define BIO_MAX_SECTORS (BIO_MAX_SIZE >> 9)
struct bio_set;
struct bio;
-typedef int (bio_end_io_t) (struct bio *, unsigned int, int);
+struct bio_integrity_payload;
+typedef void (bio_end_io_t) (struct bio *, int);
typedef void (bio_destructor_t) (struct bio *);
/*
* stacking drivers)
*/
struct bio {
- sector_t bi_sector;
+ sector_t bi_sector; /* device address in 512 byte
+ sectors */
struct bio *bi_next; /* request queue link */
struct block_device *bi_bdev;
unsigned long bi_flags; /* status, command, etc */
atomic_t bi_cnt; /* pin count */
void *bi_private;
+#if defined(CONFIG_BLK_DEV_INTEGRITY)
+ struct bio_integrity_payload *bi_integrity; /* data integrity */
+#endif
bio_destructor_t *bi_destructor; /* destructor */
- struct bio_set *bi_set; /* memory pools set */
};
/*
#define BIO_RW_BARRIER 2
#define BIO_RW_FAILFAST 3
#define BIO_RW_SYNC 4
+#define BIO_RW_META 5
+
+/*
+ * upper 16 bits of bi_rw define the io priority of this bio
+ */
+#define BIO_PRIO_SHIFT (8 * sizeof(unsigned long) - IOPRIO_BITS)
+#define bio_prio(bio) ((bio)->bi_rw >> BIO_PRIO_SHIFT)
+#define bio_prio_valid(bio) ioprio_valid(bio_prio(bio))
+
+#define bio_set_prio(bio, prio) do { \
+ WARN_ON(prio >= (1 << IOPRIO_BITS)); \
+ (bio)->bi_rw &= ((1UL << BIO_PRIO_SHIFT) - 1); \
+ (bio)->bi_rw |= ((unsigned long) (prio) << BIO_PRIO_SHIFT); \
+} while (0)
/*
* various member access, note that bio_data should of course not be used
#define bio_offset(bio) bio_iovec((bio))->bv_offset
#define bio_segments(bio) ((bio)->bi_vcnt - (bio)->bi_idx)
#define bio_sectors(bio) ((bio)->bi_size >> 9)
-#define bio_cur_sectors(bio) (bio_iovec(bio)->bv_len >> 9)
-#define bio_data(bio) (page_address(bio_page((bio))) + bio_offset((bio)))
#define bio_barrier(bio) ((bio)->bi_rw & (1 << BIO_RW_BARRIER))
#define bio_sync(bio) ((bio)->bi_rw & (1 << BIO_RW_SYNC))
#define bio_failfast(bio) ((bio)->bi_rw & (1 << BIO_RW_FAILFAST))
#define bio_rw_ahead(bio) ((bio)->bi_rw & (1 << BIO_RW_AHEAD))
+#define bio_rw_meta(bio) ((bio)->bi_rw & (1 << BIO_RW_META))
+#define bio_empty_barrier(bio) (bio_barrier(bio) && !(bio)->bi_size)
+
+static inline unsigned int bio_cur_sectors(struct bio *bio)
+{
+ if (bio->bi_vcnt)
+ return bio_iovec(bio)->bv_len >> 9;
+
+ return 0;
+}
+
+static inline void *bio_data(struct bio *bio)
+{
+ if (bio->bi_vcnt)
+ return page_address(bio_page(bio)) + bio_offset(bio);
+
+ return NULL;
+}
/*
* will die
#define BIO_SEG_BOUNDARY(q, b1, b2) \
BIOVEC_SEG_BOUNDARY((q), __BVEC_END((b1)), __BVEC_START((b2)))
-#define bio_io_error(bio, bytes) bio_endio((bio), (bytes), -EIO)
+#define bio_io_error(bio) bio_endio((bio), -EIO)
/*
* drivers should not use the __ version unless they _really_ want to
*/
#define bio_get(bio) atomic_inc(&(bio)->bi_cnt)
+#if defined(CONFIG_BLK_DEV_INTEGRITY)
+/*
+ * bio integrity payload
+ */
+struct bio_integrity_payload {
+ struct bio *bip_bio; /* parent bio */
+ struct bio_vec *bip_vec; /* integrity data vector */
+
+ sector_t bip_sector; /* virtual start sector */
+
+ void *bip_buf; /* generated integrity data */
+ bio_end_io_t *bip_end_io; /* saved I/O completion fn */
+
+ int bip_error; /* saved I/O error */
+ unsigned int bip_size;
+
+ unsigned short bip_pool; /* pool the ivec came from */
+ unsigned short bip_vcnt; /* # of integrity bio_vecs */
+ unsigned short bip_idx; /* current bip_vec index */
+
+ struct work_struct bip_work; /* I/O completion */
+};
+#endif /* CONFIG_BLK_DEV_INTEGRITY */
/*
* A bio_pair is used when we need to split a bio.
* in bio2.bi_private
*/
struct bio_pair {
- struct bio bio1, bio2;
- struct bio_vec bv1, bv2;
- atomic_t cnt;
- int error;
+ struct bio bio1, bio2;
+ struct bio_vec bv1, bv2;
+#if defined(CONFIG_BLK_DEV_INTEGRITY)
+ struct bio_integrity_payload bip1, bip2;
+ struct bio_vec iv1, iv2;
+#endif
+ atomic_t cnt;
+ int error;
};
extern struct bio_pair *bio_split(struct bio *bi, mempool_t *pool,
int first_sectors);
extern mempool_t *bio_split_pool;
extern void bio_pair_release(struct bio_pair *dbio);
-extern struct bio_set *bioset_create(int, int, int);
+extern struct bio_set *bioset_create(int, int);
extern void bioset_free(struct bio_set *);
-extern struct bio *bio_alloc(unsigned int __nocast, int);
-extern struct bio *bio_alloc_bioset(unsigned int __nocast, int, struct bio_set *);
+extern struct bio *bio_alloc(gfp_t, int);
+extern struct bio *bio_alloc_bioset(gfp_t, int, struct bio_set *);
extern void bio_put(struct bio *);
+extern void bio_free(struct bio *, struct bio_set *);
-extern void bio_endio(struct bio *, unsigned int, int);
+extern void bio_endio(struct bio *, int);
struct request_queue;
extern int bio_phys_segments(struct request_queue *, struct bio *);
extern int bio_hw_segments(struct request_queue *, struct bio *);
extern void __bio_clone(struct bio *, struct bio *);
-extern struct bio *bio_clone(struct bio *, unsigned int __nocast);
+extern struct bio *bio_clone(struct bio *, gfp_t);
extern void bio_init(struct bio *);
extern int bio_add_page(struct bio *, struct page *, unsigned int,unsigned int);
+extern int bio_add_pc_page(struct request_queue *, struct bio *, struct page *,
+ unsigned int, unsigned int);
extern int bio_get_nr_vecs(struct block_device *);
extern struct bio *bio_map_user(struct request_queue *, struct block_device *,
unsigned long, unsigned int, int);
struct sg_iovec *, int, int);
extern void bio_unmap_user(struct bio *);
extern struct bio *bio_map_kern(struct request_queue *, void *, unsigned int,
- unsigned int);
+ gfp_t);
+extern struct bio *bio_copy_kern(struct request_queue *, void *, unsigned int,
+ gfp_t, int);
extern void bio_set_pages_dirty(struct bio *bio);
extern void bio_check_pages_dirty(struct bio *bio);
extern struct bio *bio_copy_user(struct request_queue *, unsigned long, unsigned int, int);
+extern struct bio *bio_copy_user_iov(struct request_queue *, struct sg_iovec *,
+ int, int);
extern int bio_uncopy_user(struct bio *);
void zero_fill_bio(struct bio *bio);
+extern struct bio_vec *bvec_alloc_bs(gfp_t, int, unsigned long *, struct bio_set *);
+extern unsigned int bvec_nr_vecs(unsigned short idx);
+
+/*
+ * bio_set is used to allow other portions of the IO system to
+ * allocate their own private memory pools for bio and iovec structures.
+ * These memory pools in turn all allocate from the bio_slab
+ * and the bvec_slabs[].
+ */
+#define BIO_POOL_SIZE 2
+#define BIOVEC_NR_POOLS 6
+
+struct bio_set {
+ mempool_t *bio_pool;
+#if defined(CONFIG_BLK_DEV_INTEGRITY)
+ mempool_t *bio_integrity_pool;
+#endif
+ mempool_t *bvec_pools[BIOVEC_NR_POOLS];
+};
+
+struct biovec_slab {
+ int nr_vecs;
+ char *name;
+ struct kmem_cache *slab;
+};
+
+extern struct bio_set *fs_bio_set;
+
+/*
+ * a small number of entries is fine, not going to be performance critical.
+ * basically we just need to survive
+ */
+#define BIO_SPLIT_ENTRIES 2
#ifdef CONFIG_HIGHMEM
/*
* bvec_kmap_irq and bvec_kunmap_irq!!
*
* This function MUST be inlined - it plays with the CPU interrupt flags.
- * Hence the `extern inline'.
*/
-extern inline char *bvec_kmap_irq(struct bio_vec *bvec, unsigned long *flags)
+static inline char *bvec_kmap_irq(struct bio_vec *bvec, unsigned long *flags)
{
unsigned long addr;
return (char *) addr + bvec->bv_offset;
}
-extern inline void bvec_kunmap_irq(char *buffer, unsigned long *flags)
+static inline void bvec_kunmap_irq(char *buffer, unsigned long *flags)
{
unsigned long ptr = (unsigned long) buffer & PAGE_MASK;
#define bvec_kunmap_irq(buf, flags) do { *(flags) = 0; } while (0)
#endif
-extern inline char *__bio_kmap_irq(struct bio *bio, unsigned short idx,
+static inline char *__bio_kmap_irq(struct bio *bio, unsigned short idx,
unsigned long *flags)
{
return bvec_kmap_irq(bio_iovec_idx(bio, idx), flags);
__bio_kmap_irq((bio), (bio)->bi_idx, (flags))
#define bio_kunmap_irq(buf,flags) __bio_kunmap_irq(buf, flags)
+#if defined(CONFIG_BLK_DEV_INTEGRITY)
+
+#define bip_vec_idx(bip, idx) (&(bip->bip_vec[(idx)]))
+#define bip_vec(bip) bip_vec_idx(bip, 0)
+
+#define __bip_for_each_vec(bvl, bip, i, start_idx) \
+ for (bvl = bip_vec_idx((bip), (start_idx)), i = (start_idx); \
+ i < (bip)->bip_vcnt; \
+ bvl++, i++)
+
+#define bip_for_each_vec(bvl, bip, i) \
+ __bip_for_each_vec(bvl, bip, i, (bip)->bip_idx)
+
+static inline int bio_integrity(struct bio *bio)
+{
+#if defined(CONFIG_BLK_DEV_INTEGRITY)
+ return bio->bi_integrity != NULL;
+#else
+ return 0;
+#endif
+}
+
+extern struct bio_integrity_payload *bio_integrity_alloc_bioset(struct bio *, gfp_t, unsigned int, struct bio_set *);
+extern struct bio_integrity_payload *bio_integrity_alloc(struct bio *, gfp_t, unsigned int);
+extern void bio_integrity_free(struct bio *, struct bio_set *);
+extern int bio_integrity_add_page(struct bio *, struct page *, unsigned int, unsigned int);
+extern int bio_integrity_enabled(struct bio *bio);
+extern int bio_integrity_set_tag(struct bio *, void *, unsigned int);
+extern int bio_integrity_get_tag(struct bio *, void *, unsigned int);
+extern int bio_integrity_prep(struct bio *);
+extern void bio_integrity_endio(struct bio *, int);
+extern void bio_integrity_advance(struct bio *, unsigned int);
+extern void bio_integrity_trim(struct bio *, unsigned int, unsigned int);
+extern void bio_integrity_split(struct bio *, struct bio_pair *, int);
+extern int bio_integrity_clone(struct bio *, struct bio *, struct bio_set *);
+extern int bioset_integrity_create(struct bio_set *, int);
+extern void bioset_integrity_free(struct bio_set *);
+extern void bio_integrity_init_slab(void);
+
+#else /* CONFIG_BLK_DEV_INTEGRITY */
+
+#define bio_integrity(a) (0)
+#define bioset_integrity_create(a, b) (0)
+#define bio_integrity_prep(a) (0)
+#define bio_integrity_enabled(a) (0)
+#define bio_integrity_clone(a, b, c) (0)
+#define bioset_integrity_free(a) do { } while (0)
+#define bio_integrity_free(a, b) do { } while (0)
+#define bio_integrity_endio(a, b) do { } while (0)
+#define bio_integrity_advance(a, b) do { } while (0)
+#define bio_integrity_trim(a, b, c) do { } while (0)
+#define bio_integrity_split(a, b, c) do { } while (0)
+#define bio_integrity_set_tag(a, b, c) do { } while (0)
+#define bio_integrity_get_tag(a, b, c) do { } while (0)
+#define bio_integrity_init_slab(a) do { } while (0)
+
+#endif /* CONFIG_BLK_DEV_INTEGRITY */
+
+#endif /* CONFIG_BLOCK */
#endif /* __LINUX_BIO_H */