X-Git-Url: http://ftp.safe.ca/?a=blobdiff_plain;f=fs%2Fdirect-io.c;h=05763bbc2050b8ed917d593275b6ba37d1e8282b;hb=6e8341a11eb21826b7192d0bb88cb5b44900a9af;hp=6874785bb65a846bafc6eb4242875bac9439ba3c;hpb=6a648fa72161d1f6468dabd96c5d3c0db04f598a;p=safe%2Fjmp%2Flinux-2.6 diff --git a/fs/direct-io.c b/fs/direct-io.c index 6874785..05763bb 100644 --- a/fs/direct-io.c +++ b/fs/direct-io.c @@ -5,11 +5,11 @@ * * O_DIRECT * - * 04Jul2002 akpm@zip.com.au + * 04Jul2002 Andrew Morton * Initial version * 11Sep2002 janetinc@us.ibm.com * added readv/writev support. - * 29Oct2002 akpm@zip.com.au + * 29Oct2002 Andrew Morton * rewrote bio_add_page() support. * 30Oct2002 pbadari@us.ibm.com * added support for non-aligned IO. @@ -150,20 +150,14 @@ static int dio_refill_pages(struct dio *dio) int nr_pages; nr_pages = min(dio->total_pages - dio->curr_page, DIO_PAGES); - down_read(¤t->mm->mmap_sem); - ret = get_user_pages( - current, /* Task for fault acounting */ - current->mm, /* whose pages? */ + ret = get_user_pages_fast( dio->curr_user_address, /* Where from? */ nr_pages, /* How many pages? */ dio->rw == READ, /* Write to memory? */ - 0, /* force (?) */ - &dio->pages[0], - NULL); /* vmas */ - up_read(¤t->mm->mmap_sem); + &dio->pages[0]); /* Put results here */ if (ret < 0 && dio->blocks_available && (dio->rw & WRITE)) { - struct page *page = ZERO_PAGE(dio->curr_user_address); + struct page *page = ZERO_PAGE(0); /* * A memory fault, but the filesystem has some outstanding * mapped blocks. We need to use those blocks up to avoid @@ -264,15 +258,12 @@ static int dio_bio_complete(struct dio *dio, struct bio *bio); /* * Asynchronous IO callback. */ -static int dio_bio_end_aio(struct bio *bio, unsigned int bytes_done, int error) +static void dio_bio_end_aio(struct bio *bio, int error) { struct dio *dio = bio->bi_private; unsigned long remaining; unsigned long flags; - if (bio->bi_size) - return 1; - /* cleanup the bio */ dio_bio_complete(dio, bio); @@ -287,8 +278,6 @@ static int dio_bio_end_aio(struct bio *bio, unsigned int bytes_done, int error) aio_complete(dio->iocb, ret, 0); kfree(dio); } - - return 0; } /* @@ -298,21 +287,17 @@ static int dio_bio_end_aio(struct bio *bio, unsigned int bytes_done, int error) * During I/O bi_private points at the dio. After I/O, bi_private is used to * implement a singly-linked list of completed BIOs, at dio->bio_list. */ -static int dio_bio_end_io(struct bio *bio, unsigned int bytes_done, int error) +static void dio_bio_end_io(struct bio *bio, int error) { struct dio *dio = bio->bi_private; unsigned long flags; - if (bio->bi_size) - return 1; - spin_lock_irqsave(&dio->bio_lock, flags); bio->bi_private = dio->bio_list; dio->bio_list = bio; if (--dio->refcount == 1 && dio->waiter) wake_up_process(dio->waiter); spin_unlock_irqrestore(&dio->bio_lock, flags); - return 0; } static int @@ -322,8 +307,6 @@ dio_bio_alloc(struct dio *dio, struct block_device *bdev, struct bio *bio; bio = bio_alloc(GFP_KERNEL, nr_vecs); - if (bio == NULL) - return -ENOMEM; bio->bi_bdev = bdev; bio->bi_sector = first_sector; @@ -772,7 +755,7 @@ static void dio_zero_block(struct dio *dio, int end) this_chunk_bytes = this_chunk_blocks << dio->blkbits; - page = ZERO_PAGE(dio->curr_user_address); + page = ZERO_PAGE(0); if (submit_page_section(dio, page, 0, this_chunk_bytes, dio->next_block_for_io)) return; @@ -887,8 +870,8 @@ do_holes: page_cache_release(page); goto out; } - zero_user_page(page, block_in_page << blkbits, - 1 << blkbits, KM_USER0); + zero_user(page, block_in_page << blkbits, + 1 << blkbits); dio->block_in_file++; block_in_page++; goto next_block; @@ -958,36 +941,22 @@ direct_io_worker(int rw, struct kiocb *iocb, struct inode *inode, ssize_t ret2; size_t bytes; - dio->bio = NULL; dio->inode = inode; dio->rw = rw; dio->blkbits = blkbits; dio->blkfactor = inode->i_blkbits - blkbits; - dio->start_zero_done = 0; - dio->size = 0; dio->block_in_file = offset >> blkbits; - dio->blocks_available = 0; - dio->cur_page = NULL; - dio->boundary = 0; - dio->reap_counter = 0; dio->get_block = get_block; dio->end_io = end_io; - dio->map_bh.b_private = NULL; - dio->map_bh.b_state = 0; dio->final_block_in_bio = -1; dio->next_block_for_io = -1; - dio->page_errors = 0; - dio->io_error = 0; - dio->result = 0; dio->iocb = iocb; dio->i_size = i_size_read(inode); spin_lock_init(&dio->bio_lock); dio->refcount = 1; - dio->bio_list = NULL; - dio->waiter = NULL; /* * In case of non-aligned buffers, we may need 2 more @@ -995,8 +964,6 @@ direct_io_worker(int rw, struct kiocb *iocb, struct inode *inode, */ if (unlikely(dio->blkfactor)) dio->pages_in_io = 2; - else - dio->pages_in_io = 0; for (seg = 0; seg < nr_segs; seg++) { user_addr = (unsigned long)iov[seg].iov_base; @@ -1157,7 +1124,7 @@ __blockdev_direct_IO(int rw, struct kiocb *iocb, struct inode *inode, int acquire_i_mutex = 0; if (rw & WRITE) - rw = WRITE_SYNC; + rw = WRITE_ODIRECT; if (bdev) bdev_blkbits = blksize_bits(bdev_hardsect_size(bdev)); @@ -1184,7 +1151,7 @@ __blockdev_direct_IO(int rw, struct kiocb *iocb, struct inode *inode, } } - dio = kmalloc(sizeof(*dio), GFP_KERNEL); + dio = kzalloc(sizeof(*dio), GFP_KERNEL); retval = -ENOMEM; if (!dio) goto out; @@ -1240,6 +1207,19 @@ __blockdev_direct_IO(int rw, struct kiocb *iocb, struct inode *inode, retval = direct_io_worker(rw, iocb, inode, iov, offset, nr_segs, blkbits, get_block, end_io, dio); + /* + * In case of error extending write may have instantiated a few + * blocks outside i_size. Trim these off again for DIO_LOCKING. + * NOTE: DIO_NO_LOCK/DIO_OWN_LOCK callers have to handle this by + * it's own meaner. + */ + if (unlikely(retval < 0 && (rw & WRITE))) { + loff_t isize = i_size_read(inode); + + if (end > isize && dio_lock_type == DIO_LOCKING) + vmtruncate(inode, isize); + } + if (rw == READ && dio_lock_type == DIO_LOCKING) release_i_mutex = 0;