*
* Maximum number of loop devices when compiled-in now selectable by passing
* max_loop=<1-255> to the kernel on boot.
- * Erik I. Bolsø, <eriki@himolde.no>, Oct 31, 1999
+ * Erik I. Bolsø, <eriki@himolde.no>, Oct 31, 1999
*
* Completely rewrite request handling to be make_request_fn style and
* non blocking, pushing work to a helper thread. Lots of fixes from
* Heinz Mauelshagen <mge@sistina.com>, Feb 2002
*
* Support for falling back on the write file operation when the address space
- * operations prepare_write and/or commit_write are not available on the
- * backing filesystem.
+ * operations write_begin is not available on the backing filesystem.
* Anton Altaparmakov, 16 Feb 2005
*
* Still To Fix:
*
*/
-#include <linux/config.h>
#include <linux/module.h>
#include <linux/moduleparam.h>
#include <linux/sched.h>
#include <linux/blkdev.h>
#include <linux/blkpg.h>
#include <linux/init.h>
-#include <linux/devfs_fs_kernel.h>
-#include <linux/smp_lock.h>
#include <linux/swap.h>
#include <linux/slab.h>
#include <linux/loop.h>
+#include <linux/compat.h>
#include <linux/suspend.h>
+#include <linux/freezer.h>
#include <linux/writeback.h>
#include <linux/buffer_head.h> /* for invalidate_bdev() */
#include <linux/completion.h>
#include <linux/highmem.h>
-#include <linux/gfp.h>
+#include <linux/kthread.h>
+#include <linux/splice.h>
#include <asm/uaccess.h>
-static int max_loop = 8;
-static struct loop_device *loop_dev;
-static struct gendisk **disks;
+static LIST_HEAD(loop_devices);
+static DEFINE_MUTEX(loop_devices_mutex);
+
+static int max_part;
+static int part_shift;
/*
* Transfer functions
if (unlikely((loff_t)x != size))
return -EFBIG;
- set_capacity(disks[lo->lo_number], x);
+ set_capacity(lo->lo_disk, x);
return 0;
}
* do_lo_send_aops - helper for writing data to a loop device
*
* This is the fast version for backing filesystems which implement the address
- * space operations prepare_write and commit_write.
+ * space operations write_begin and write_end.
*/
static int do_lo_send_aops(struct loop_device *lo, struct bio_vec *bvec,
- int bsize, loff_t pos, struct page *page)
+ loff_t pos, struct page *unused)
{
struct file *file = lo->lo_backing_file; /* kudos to NFsckingS */
struct address_space *mapping = file->f_mapping;
- struct address_space_operations *aops = mapping->a_ops;
pgoff_t index;
unsigned offset, bv_offs;
- int len, ret = 0;
+ int len, ret;
- down(&mapping->host->i_sem);
+ mutex_lock(&mapping->host->i_mutex);
index = pos >> PAGE_CACHE_SHIFT;
offset = pos & ((pgoff_t)PAGE_CACHE_SIZE - 1);
bv_offs = bvec->bv_offset;
len = bvec->bv_len;
while (len > 0) {
sector_t IV;
- unsigned size;
+ unsigned size, copied;
int transfer_result;
+ struct page *page;
+ void *fsdata;
IV = ((sector_t)index << (PAGE_CACHE_SHIFT - 9))+(offset >> 9);
size = PAGE_CACHE_SIZE - offset;
if (size > len)
size = len;
- page = grab_cache_page(mapping, index);
- if (unlikely(!page))
+
+ ret = pagecache_write_begin(file, mapping, pos, size, 0,
+ &page, &fsdata);
+ if (ret)
goto fail;
- if (unlikely(aops->prepare_write(file, page, offset,
- offset + size)))
- goto unlock;
+
transfer_result = lo_do_transfer(lo, WRITE, page, offset,
bvec->bv_page, bv_offs, size, IV);
- if (unlikely(transfer_result)) {
- char *kaddr;
-
- /*
- * The transfer failed, but we still write the data to
- * keep prepare/commit calls balanced.
- */
- printk(KERN_ERR "loop: transfer error block %llu\n",
- (unsigned long long)index);
- kaddr = kmap_atomic(page, KM_USER0);
- memset(kaddr + offset, 0, size);
- kunmap_atomic(kaddr, KM_USER0);
- }
- flush_dcache_page(page);
- if (unlikely(aops->commit_write(file, page, offset,
- offset + size)))
- goto unlock;
+ copied = size;
+ if (unlikely(transfer_result))
+ copied = 0;
+
+ ret = pagecache_write_end(file, mapping, pos, size, copied,
+ page, fsdata);
+ if (ret < 0 || ret != copied)
+ goto fail;
+
if (unlikely(transfer_result))
- goto unlock;
- bv_offs += size;
- len -= size;
+ goto fail;
+
+ bv_offs += copied;
+ len -= copied;
offset = 0;
index++;
- pos += size;
- unlock_page(page);
- page_cache_release(page);
+ pos += copied;
}
+ ret = 0;
out:
- up(&mapping->host->i_sem);
+ mutex_unlock(&mapping->host->i_mutex);
return ret;
-unlock:
- unlock_page(page);
- page_cache_release(page);
fail:
ret = -1;
goto out;
* This helper just factors out common code between do_lo_send_direct_write()
* and do_lo_send_write().
*/
-static inline int __do_lo_send_write(struct file *file,
- u8 __user *buf, const int len, loff_t pos)
+static int __do_lo_send_write(struct file *file,
+ u8 *buf, const int len, loff_t pos)
{
ssize_t bw;
mm_segment_t old_fs = get_fs();
* do_lo_send_direct_write - helper for writing data to a loop device
*
* This is the fast, non-transforming version for backing filesystems which do
- * not implement the address space operations prepare_write and commit_write.
+ * not implement the address space operations write_begin and write_end.
* It uses the write file operation which should be present on all writeable
* filesystems.
*/
static int do_lo_send_direct_write(struct loop_device *lo,
- struct bio_vec *bvec, int bsize, loff_t pos, struct page *page)
+ struct bio_vec *bvec, loff_t pos, struct page *page)
{
ssize_t bw = __do_lo_send_write(lo->lo_backing_file,
- (u8 __user *)kmap(bvec->bv_page) + bvec->bv_offset,
+ kmap(bvec->bv_page) + bvec->bv_offset,
bvec->bv_len, pos);
kunmap(bvec->bv_page);
cond_resched();
* do_lo_send_write - helper for writing data to a loop device
*
* This is the slow, transforming version for filesystems which do not
- * implement the address space operations prepare_write and commit_write. It
+ * implement the address space operations write_begin and write_end. It
* uses the write file operation which should be present on all writeable
* filesystems.
*
* destination pages of the backing file.
*/
static int do_lo_send_write(struct loop_device *lo, struct bio_vec *bvec,
- int bsize, loff_t pos, struct page *page)
+ loff_t pos, struct page *page)
{
int ret = lo_do_transfer(lo, WRITE, page, 0, bvec->bv_page,
bvec->bv_offset, bvec->bv_len, pos >> 9);
if (likely(!ret))
return __do_lo_send_write(lo->lo_backing_file,
- (u8 __user *)page_address(page), bvec->bv_len,
+ page_address(page), bvec->bv_len,
pos);
printk(KERN_ERR "loop: Transfer error at byte offset %llu, "
"length %i.\n", (unsigned long long)pos, bvec->bv_len);
return ret;
}
-static int lo_send(struct loop_device *lo, struct bio *bio, int bsize,
- loff_t pos)
+static int lo_send(struct loop_device *lo, struct bio *bio, loff_t pos)
{
- int (*do_lo_send)(struct loop_device *, struct bio_vec *, int, loff_t,
+ int (*do_lo_send)(struct loop_device *, struct bio_vec *, loff_t,
struct page *page);
struct bio_vec *bvec;
struct page *page = NULL;
}
}
bio_for_each_segment(bvec, bio, i) {
- ret = do_lo_send(lo, bvec, bsize, pos, page);
+ ret = do_lo_send(lo, bvec, pos, page);
if (ret < 0)
break;
pos += bvec->bv_len;
};
static int
-lo_read_actor(read_descriptor_t *desc, struct page *page,
- unsigned long offset, unsigned long size)
+lo_splice_actor(struct pipe_inode_info *pipe, struct pipe_buffer *buf,
+ struct splice_desc *sd)
{
- unsigned long count = desc->count;
- struct lo_read_data *p = desc->arg.data;
+ struct lo_read_data *p = sd->u.data;
struct loop_device *lo = p->lo;
+ struct page *page = buf->page;
sector_t IV;
+ int size, ret;
- IV = ((sector_t) page->index << (PAGE_CACHE_SHIFT - 9))+(offset >> 9);
+ ret = buf->ops->confirm(pipe, buf);
+ if (unlikely(ret))
+ return ret;
- if (size > count)
- size = count;
+ IV = ((sector_t) page->index << (PAGE_CACHE_SHIFT - 9)) +
+ (buf->offset >> 9);
+ size = sd->len;
+ if (size > p->bsize)
+ size = p->bsize;
- if (lo_do_transfer(lo, READ, page, offset, p->page, p->offset, size, IV)) {
- size = 0;
+ if (lo_do_transfer(lo, READ, page, buf->offset, p->page, p->offset, size, IV)) {
printk(KERN_ERR "loop: transfer error block %ld\n",
page->index);
- desc->error = -EINVAL;
+ size = -EINVAL;
}
flush_dcache_page(p->page);
- desc->count = count - size;
- desc->written += size;
- p->offset += size;
+ if (size > 0)
+ p->offset += size;
+
return size;
}
static int
+lo_direct_splice_actor(struct pipe_inode_info *pipe, struct splice_desc *sd)
+{
+ return __splice_from_pipe(pipe, sd, lo_splice_actor);
+}
+
+static int
do_lo_receive(struct loop_device *lo,
struct bio_vec *bvec, int bsize, loff_t pos)
{
struct lo_read_data cookie;
+ struct splice_desc sd;
struct file *file;
- int retval;
+ long retval;
cookie.lo = lo;
cookie.page = bvec->bv_page;
cookie.offset = bvec->bv_offset;
cookie.bsize = bsize;
+
+ sd.len = 0;
+ sd.total_len = bvec->bv_len;
+ sd.flags = 0;
+ sd.pos = pos;
+ sd.u.data = &cookie;
+
file = lo->lo_backing_file;
- retval = file->f_op->sendfile(file, &pos, bvec->bv_len,
- lo_read_actor, &cookie);
- return (retval < 0)? retval: 0;
+ retval = splice_direct_to_actor(file, &sd, lo_direct_splice_actor);
+
+ if (retval < 0)
+ return retval;
+
+ return 0;
}
static int
int ret;
pos = ((loff_t) bio->bi_sector << 9) + lo->lo_offset;
- if (bio_rw(bio) == WRITE)
- ret = lo_send(lo, bio, lo->lo_blocksize, pos);
- else
+
+ if (bio_rw(bio) == WRITE) {
+ bool barrier = bio_rw_flagged(bio, BIO_RW_BARRIER);
+ struct file *file = lo->lo_backing_file;
+
+ if (barrier) {
+ if (unlikely(!file->f_op->fsync)) {
+ ret = -EOPNOTSUPP;
+ goto out;
+ }
+
+ ret = vfs_fsync(file, file->f_path.dentry, 0);
+ if (unlikely(ret)) {
+ ret = -EIO;
+ goto out;
+ }
+ }
+
+ ret = lo_send(lo, bio, pos);
+
+ if (barrier && !ret) {
+ ret = vfs_fsync(file, file->f_path.dentry, 0);
+ if (unlikely(ret))
+ ret = -EIO;
+ }
+ } else
ret = lo_receive(lo, bio, lo->lo_blocksize, pos);
+
+out:
return ret;
}
*/
static void loop_add_bio(struct loop_device *lo, struct bio *bio)
{
- unsigned long flags;
-
- spin_lock_irqsave(&lo->lo_lock, flags);
- if (lo->lo_biotail) {
- lo->lo_biotail->bi_next = bio;
- lo->lo_biotail = bio;
- } else
- lo->lo_bio = lo->lo_biotail = bio;
- spin_unlock_irqrestore(&lo->lo_lock, flags);
-
- up(&lo->lo_bh_mutex);
+ bio_list_add(&lo->lo_bio_list, bio);
}
/*
*/
static struct bio *loop_get_bio(struct loop_device *lo)
{
- struct bio *bio;
-
- spin_lock_irq(&lo->lo_lock);
- if ((bio = lo->lo_bio)) {
- if (bio == lo->lo_biotail)
- lo->lo_biotail = NULL;
- lo->lo_bio = bio->bi_next;
- bio->bi_next = NULL;
- }
- spin_unlock_irq(&lo->lo_lock);
-
- return bio;
+ return bio_list_pop(&lo->lo_bio_list);
}
-static int loop_make_request(request_queue_t *q, struct bio *old_bio)
+static int loop_make_request(struct request_queue *q, struct bio *old_bio)
{
struct loop_device *lo = q->queuedata;
int rw = bio_rw(old_bio);
- if (!lo)
- goto out;
+ if (rw == READA)
+ rw = READ;
+
+ BUG_ON(!lo || (rw != READ && rw != WRITE));
spin_lock_irq(&lo->lo_lock);
if (lo->lo_state != Lo_bound)
- goto inactive;
- atomic_inc(&lo->lo_pending);
- spin_unlock_irq(&lo->lo_lock);
-
- if (rw == WRITE) {
- if (lo->lo_flags & LO_FLAGS_READ_ONLY)
- goto err;
- } else if (rw == READA) {
- rw = READ;
- } else if (rw != READ) {
- printk(KERN_ERR "loop: unknown command (%x)\n", rw);
- goto err;
- }
+ goto out;
+ if (unlikely(rw == WRITE && (lo->lo_flags & LO_FLAGS_READ_ONLY)))
+ goto out;
loop_add_bio(lo, old_bio);
+ wake_up(&lo->lo_event);
+ spin_unlock_irq(&lo->lo_lock);
return 0;
-err:
- if (atomic_dec_and_test(&lo->lo_pending))
- up(&lo->lo_bh_mutex);
+
out:
- bio_io_error(old_bio, old_bio->bi_size);
- return 0;
-inactive:
spin_unlock_irq(&lo->lo_lock);
- goto out;
+ bio_io_error(old_bio);
+ return 0;
}
/*
* kick off io on the underlying address space
*/
-static void loop_unplug(request_queue_t *q)
+static void loop_unplug(struct request_queue *q)
{
struct loop_device *lo = q->queuedata;
- clear_bit(QUEUE_FLAG_PLUGGED, &q->queue_flags);
+ queue_flag_clear_unlocked(QUEUE_FLAG_PLUGGED, q);
blk_run_address_space(lo->lo_backing_file->f_mapping);
}
static inline void loop_handle_bio(struct loop_device *lo, struct bio *bio)
{
- int ret;
-
if (unlikely(!bio->bi_bdev)) {
do_loop_switch(lo, bio->bi_private);
bio_put(bio);
} else {
- ret = do_bio_filebacked(lo, bio);
- bio_endio(bio, bio->bi_size, ret);
+ int ret = do_bio_filebacked(lo, bio);
+ bio_endio(bio, ret);
}
}
* to avoid blocking in our make_request_fn. it also does loop decrypting
* on reads for block backed loop, as that is too heavy to do from
* b_end_io context where irqs may be disabled.
+ *
+ * Loop explanation: loop_clr_fd() sets lo_state to Lo_rundown before
+ * calling kthread_stop(). Therefore once kthread_should_stop() is
+ * true, make_request will not place any more requests. Therefore
+ * once kthread_should_stop() is true and lo_bio is NULL, we are
+ * done with the loop.
*/
static int loop_thread(void *data)
{
struct loop_device *lo = data;
struct bio *bio;
- daemonize("loop%d", lo->lo_number);
-
- /*
- * loop can be used in an encrypted device,
- * hence, it mustn't be stopped at all
- * because it could be indirectly used during suspension
- */
- current->flags |= PF_NOFREEZE;
-
set_user_nice(current, -20);
- lo->lo_state = Lo_bound;
- atomic_inc(&lo->lo_pending);
-
- /*
- * up sem, we are running
- */
- up(&lo->lo_sem);
+ while (!kthread_should_stop() || !bio_list_empty(&lo->lo_bio_list)) {
- for (;;) {
- down_interruptible(&lo->lo_bh_mutex);
- /*
- * could be upped because of tear-down, not because of
- * pending work
- */
- if (!atomic_read(&lo->lo_pending))
- break;
+ wait_event_interruptible(lo->lo_event,
+ !bio_list_empty(&lo->lo_bio_list) ||
+ kthread_should_stop());
- bio = loop_get_bio(lo);
- if (!bio) {
- printk("loop: missing bio\n");
+ if (bio_list_empty(&lo->lo_bio_list))
continue;
- }
- loop_handle_bio(lo, bio);
+ spin_lock_irq(&lo->lo_lock);
+ bio = loop_get_bio(lo);
+ spin_unlock_irq(&lo->lo_lock);
- /*
- * upped both for pending work and tear-down, lo_pending
- * will hit zero then
- */
- if (atomic_dec_and_test(&lo->lo_pending))
- break;
+ BUG_ON(!bio);
+ loop_handle_bio(lo, bio);
}
- up(&lo->lo_sem);
return 0;
}
static int loop_switch(struct loop_device *lo, struct file *file)
{
struct switch_request w;
- struct bio *bio = bio_alloc(GFP_KERNEL, 1);
+ struct bio *bio = bio_alloc(GFP_KERNEL, 0);
if (!bio)
return -ENOMEM;
init_completion(&w.wait);
}
/*
+ * Helper to flush the IOs in loop, but keeping loop thread running
+ */
+static int loop_flush(struct loop_device *lo)
+{
+ /* loop not yet configured, no running thread, nothing to flush */
+ if (!lo->lo_thread)
+ return 0;
+
+ return loop_switch(lo, NULL);
+}
+
+/*
* Do the actual switch; called from the BIO completion routine
*/
static void do_loop_switch(struct loop_device *lo, struct switch_request *p)
{
struct file *file = p->file;
struct file *old_file = lo->lo_backing_file;
- struct address_space *mapping = file->f_mapping;
+ struct address_space *mapping;
+ /* if no new file, only flush of queued bios requested */
+ if (!file)
+ goto out;
+
+ mapping = file->f_mapping;
mapping_set_gfp_mask(old_file->f_mapping, lo->old_gfp_mask);
lo->lo_backing_file = file;
- lo->lo_blocksize = mapping->host->i_blksize;
+ lo->lo_blocksize = S_ISBLK(mapping->host->i_mode) ?
+ mapping->host->i_bdev->bd_block_size : PAGE_SIZE;
lo->old_gfp_mask = mapping_gfp_mask(mapping);
mapping_set_gfp_mask(mapping, lo->old_gfp_mask & ~(__GFP_IO|__GFP_FS));
+out:
complete(&p->wait);
}
* This can only work if the loop device is used read-only, and if the
* new backing store is the same size and type as the old backing store.
*/
-static int loop_change_fd(struct loop_device *lo, struct file *lo_file,
- struct block_device *bdev, unsigned int arg)
+static int loop_change_fd(struct loop_device *lo, struct block_device *bdev,
+ unsigned int arg)
{
struct file *file, *old_file;
struct inode *inode;
if (!S_ISREG(inode->i_mode) && !S_ISBLK(inode->i_mode))
goto out_putf;
- /* new backing store needs to support loop (eg sendfile) */
- if (!inode->i_fop->sendfile)
- goto out_putf;
-
/* size of the new backing store needs to be the same */
if (get_loop_size(lo, file) != get_loop_size(lo, old_file))
goto out_putf;
goto out_putf;
fput(old_file);
+ if (max_part > 0)
+ ioctl_by_bdev(bdev, BLKRRPART, 0);
return 0;
out_putf:
return i && S_ISBLK(i->i_mode) && MAJOR(i->i_rdev) == LOOP_MAJOR;
}
-static int loop_set_fd(struct loop_device *lo, struct file *lo_file,
+static int loop_set_fd(struct loop_device *lo, fmode_t mode,
struct block_device *bdev, unsigned int arg)
{
struct file *file, *f;
while (is_loop_device(f)) {
struct loop_device *l;
- if (f->f_mapping->host->i_rdev == lo_file->f_mapping->host->i_rdev)
+ if (f->f_mapping->host->i_bdev == bdev)
goto out_putf;
l = f->f_mapping->host->i_bdev->bd_disk->private_data;
error = -EINVAL;
if (S_ISREG(inode->i_mode) || S_ISBLK(inode->i_mode)) {
- struct address_space_operations *aops = mapping->a_ops;
- /*
- * If we can't read - sorry. If we only can't write - well,
- * it's going to be read-only.
- */
- if (!file->f_op->sendfile)
- goto out_putf;
- if (aops->prepare_write && aops->commit_write)
+ const struct address_space_operations *aops = mapping->a_ops;
+
+ if (aops->write_begin)
lo_flags |= LO_FLAGS_USE_AOPS;
if (!(lo_flags & LO_FLAGS_USE_AOPS) && !file->f_op->write)
lo_flags |= LO_FLAGS_READ_ONLY;
- lo_blocksize = inode->i_blksize;
+ lo_blocksize = S_ISBLK(inode->i_mode) ?
+ inode->i_bdev->bd_block_size : PAGE_SIZE;
+
error = 0;
} else {
goto out_putf;
goto out_putf;
}
- if (!(lo_file->f_mode & FMODE_WRITE))
+ if (!(mode & FMODE_WRITE))
lo_flags |= LO_FLAGS_READ_ONLY;
set_device_ro(bdev, (lo_flags & LO_FLAGS_READ_ONLY) != 0);
lo->lo_device = bdev;
lo->lo_flags = lo_flags;
lo->lo_backing_file = file;
- lo->transfer = NULL;
+ lo->transfer = transfer_none;
lo->ioctl = NULL;
lo->lo_sizelimit = 0;
lo->old_gfp_mask = mapping_gfp_mask(mapping);
mapping_set_gfp_mask(mapping, lo->old_gfp_mask & ~(__GFP_IO|__GFP_FS));
- lo->lo_bio = lo->lo_biotail = NULL;
+ bio_list_init(&lo->lo_bio_list);
/*
* set queue make_request_fn, and add limits based on lower level
lo->lo_queue->queuedata = lo;
lo->lo_queue->unplug_fn = loop_unplug;
- set_capacity(disks[lo->lo_number], size);
+ if (!(lo_flags & LO_FLAGS_READ_ONLY) && file->f_op->fsync)
+ blk_queue_ordered(lo->lo_queue, QUEUE_ORDERED_DRAIN, NULL);
+
+ set_capacity(lo->lo_disk, size);
bd_set_size(bdev, size << 9);
set_blocksize(bdev, lo_blocksize);
- kernel_thread(loop_thread, lo, CLONE_KERNEL);
- down(&lo->lo_sem);
+ lo->lo_thread = kthread_create(loop_thread, lo, "loop%d",
+ lo->lo_number);
+ if (IS_ERR(lo->lo_thread)) {
+ error = PTR_ERR(lo->lo_thread);
+ goto out_clr;
+ }
+ lo->lo_state = Lo_bound;
+ wake_up_process(lo->lo_thread);
+ if (max_part > 0)
+ ioctl_by_bdev(bdev, BLKRRPART, 0);
return 0;
+out_clr:
+ lo->lo_thread = NULL;
+ lo->lo_device = NULL;
+ lo->lo_backing_file = NULL;
+ lo->lo_flags = 0;
+ set_capacity(lo->lo_disk, 0);
+ invalidate_bdev(bdev);
+ bd_set_size(bdev, 0);
+ mapping_set_gfp_mask(mapping, lo->old_gfp_mask);
+ lo->lo_state = Lo_unbound;
out_putf:
fput(file);
out:
static int loop_clr_fd(struct loop_device *lo, struct block_device *bdev)
{
struct file *filp = lo->lo_backing_file;
- int gfp = lo->old_gfp_mask;
+ gfp_t gfp = lo->old_gfp_mask;
if (lo->lo_state != Lo_bound)
return -ENXIO;
spin_lock_irq(&lo->lo_lock);
lo->lo_state = Lo_rundown;
- if (atomic_dec_and_test(&lo->lo_pending))
- up(&lo->lo_bh_mutex);
spin_unlock_irq(&lo->lo_lock);
- down(&lo->lo_sem);
+ kthread_stop(lo->lo_thread);
+ lo->lo_queue->unplug_fn = NULL;
lo->lo_backing_file = NULL;
loop_release_xfer(lo);
lo->lo_sizelimit = 0;
lo->lo_encrypt_key_size = 0;
lo->lo_flags = 0;
+ lo->lo_thread = NULL;
memset(lo->lo_encrypt_key, 0, LO_KEY_SIZE);
memset(lo->lo_crypt_name, 0, LO_NAME_SIZE);
memset(lo->lo_file_name, 0, LO_NAME_SIZE);
- invalidate_bdev(bdev, 0);
- set_capacity(disks[lo->lo_number], 0);
- bd_set_size(bdev, 0);
+ if (bdev)
+ invalidate_bdev(bdev);
+ set_capacity(lo->lo_disk, 0);
+ if (bdev)
+ bd_set_size(bdev, 0);
mapping_set_gfp_mask(filp->f_mapping, gfp);
lo->lo_state = Lo_unbound;
- fput(filp);
/* This is safe: open() is still holding a reference. */
module_put(THIS_MODULE);
+ if (max_part > 0 && bdev)
+ ioctl_by_bdev(bdev, BLKRRPART, 0);
+ mutex_unlock(&lo->lo_ctl_mutex);
+ /*
+ * Need not hold lo_ctl_mutex to fput backing file.
+ * Calling fput holding lo_ctl_mutex triggers a circular
+ * lock dependency possibility warning as fput can take
+ * bd_mutex which is usually taken before lo_ctl_mutex.
+ */
+ fput(filp);
return 0;
}
{
int err;
struct loop_func_table *xfer;
+ uid_t uid = current_uid();
- if (lo->lo_encrypt_key_size && lo->lo_key_owner != current->uid &&
+ if (lo->lo_encrypt_key_size &&
+ lo->lo_key_owner != uid &&
!capable(CAP_SYS_ADMIN))
return -EPERM;
if (lo->lo_state != Lo_bound)
lo->transfer = xfer->transfer;
lo->ioctl = xfer->ioctl;
+ if ((lo->lo_flags & LO_FLAGS_AUTOCLEAR) !=
+ (info->lo_flags & LO_FLAGS_AUTOCLEAR))
+ lo->lo_flags ^= LO_FLAGS_AUTOCLEAR;
+
lo->lo_encrypt_key_size = info->lo_encrypt_key_size;
lo->lo_init[0] = info->lo_init[0];
lo->lo_init[1] = info->lo_init[1];
if (info->lo_encrypt_key_size) {
memcpy(lo->lo_encrypt_key, info->lo_encrypt_key,
info->lo_encrypt_key_size);
- lo->lo_key_owner = current->uid;
+ lo->lo_key_owner = uid;
}
return 0;
if (lo->lo_state != Lo_bound)
return -ENXIO;
- error = vfs_getattr(file->f_vfsmnt, file->f_dentry, &stat);
+ error = vfs_getattr(file->f_path.mnt, file->f_path.dentry, &stat);
if (error)
return error;
memset(info, 0, sizeof(*info));
return err;
}
-static int lo_ioctl(struct inode * inode, struct file * file,
+static int loop_set_capacity(struct loop_device *lo, struct block_device *bdev)
+{
+ int err;
+ sector_t sec;
+ loff_t sz;
+
+ err = -ENXIO;
+ if (unlikely(lo->lo_state != Lo_bound))
+ goto out;
+ err = figure_loop_size(lo);
+ if (unlikely(err))
+ goto out;
+ sec = get_capacity(lo->lo_disk);
+ /* the width of sector_t may be narrow for bit-shift */
+ sz = sec;
+ sz <<= 9;
+ mutex_lock(&bdev->bd_mutex);
+ bd_set_size(bdev, sz);
+ mutex_unlock(&bdev->bd_mutex);
+
+ out:
+ return err;
+}
+
+static int lo_ioctl(struct block_device *bdev, fmode_t mode,
unsigned int cmd, unsigned long arg)
{
- struct loop_device *lo = inode->i_bdev->bd_disk->private_data;
+ struct loop_device *lo = bdev->bd_disk->private_data;
int err;
- down(&lo->lo_ctl_mutex);
+ mutex_lock_nested(&lo->lo_ctl_mutex, 1);
switch (cmd) {
case LOOP_SET_FD:
- err = loop_set_fd(lo, file, inode->i_bdev, arg);
+ err = loop_set_fd(lo, mode, bdev, arg);
break;
case LOOP_CHANGE_FD:
- err = loop_change_fd(lo, file, inode->i_bdev, arg);
+ err = loop_change_fd(lo, bdev, arg);
break;
case LOOP_CLR_FD:
- err = loop_clr_fd(lo, inode->i_bdev);
+ /* loop_clr_fd would have unlocked lo_ctl_mutex on success */
+ err = loop_clr_fd(lo, bdev);
+ if (!err)
+ goto out_unlocked;
break;
case LOOP_SET_STATUS:
err = loop_set_status_old(lo, (struct loop_info __user *) arg);
case LOOP_GET_STATUS64:
err = loop_get_status64(lo, (struct loop_info64 __user *) arg);
break;
+ case LOOP_SET_CAPACITY:
+ err = -EPERM;
+ if ((mode & FMODE_WRITE) || capable(CAP_SYS_ADMIN))
+ err = loop_set_capacity(lo, bdev);
+ break;
default:
err = lo->ioctl ? lo->ioctl(lo, cmd, arg) : -EINVAL;
}
- up(&lo->lo_ctl_mutex);
+ mutex_unlock(&lo->lo_ctl_mutex);
+
+out_unlocked:
return err;
}
-static int lo_open(struct inode *inode, struct file *file)
+#ifdef CONFIG_COMPAT
+struct compat_loop_info {
+ compat_int_t lo_number; /* ioctl r/o */
+ compat_dev_t lo_device; /* ioctl r/o */
+ compat_ulong_t lo_inode; /* ioctl r/o */
+ compat_dev_t lo_rdevice; /* ioctl r/o */
+ compat_int_t lo_offset;
+ compat_int_t lo_encrypt_type;
+ compat_int_t lo_encrypt_key_size; /* ioctl w/o */
+ compat_int_t lo_flags; /* ioctl r/o */
+ char lo_name[LO_NAME_SIZE];
+ unsigned char lo_encrypt_key[LO_KEY_SIZE]; /* ioctl w/o */
+ compat_ulong_t lo_init[2];
+ char reserved[4];
+};
+
+/*
+ * Transfer 32-bit compatibility structure in userspace to 64-bit loop info
+ * - noinlined to reduce stack space usage in main part of driver
+ */
+static noinline int
+loop_info64_from_compat(const struct compat_loop_info __user *arg,
+ struct loop_info64 *info64)
+{
+ struct compat_loop_info info;
+
+ if (copy_from_user(&info, arg, sizeof(info)))
+ return -EFAULT;
+
+ memset(info64, 0, sizeof(*info64));
+ info64->lo_number = info.lo_number;
+ info64->lo_device = info.lo_device;
+ info64->lo_inode = info.lo_inode;
+ info64->lo_rdevice = info.lo_rdevice;
+ info64->lo_offset = info.lo_offset;
+ info64->lo_sizelimit = 0;
+ info64->lo_encrypt_type = info.lo_encrypt_type;
+ info64->lo_encrypt_key_size = info.lo_encrypt_key_size;
+ info64->lo_flags = info.lo_flags;
+ info64->lo_init[0] = info.lo_init[0];
+ info64->lo_init[1] = info.lo_init[1];
+ if (info.lo_encrypt_type == LO_CRYPT_CRYPTOAPI)
+ memcpy(info64->lo_crypt_name, info.lo_name, LO_NAME_SIZE);
+ else
+ memcpy(info64->lo_file_name, info.lo_name, LO_NAME_SIZE);
+ memcpy(info64->lo_encrypt_key, info.lo_encrypt_key, LO_KEY_SIZE);
+ return 0;
+}
+
+/*
+ * Transfer 64-bit loop info to 32-bit compatibility structure in userspace
+ * - noinlined to reduce stack space usage in main part of driver
+ */
+static noinline int
+loop_info64_to_compat(const struct loop_info64 *info64,
+ struct compat_loop_info __user *arg)
+{
+ struct compat_loop_info info;
+
+ memset(&info, 0, sizeof(info));
+ info.lo_number = info64->lo_number;
+ info.lo_device = info64->lo_device;
+ info.lo_inode = info64->lo_inode;
+ info.lo_rdevice = info64->lo_rdevice;
+ info.lo_offset = info64->lo_offset;
+ info.lo_encrypt_type = info64->lo_encrypt_type;
+ info.lo_encrypt_key_size = info64->lo_encrypt_key_size;
+ info.lo_flags = info64->lo_flags;
+ info.lo_init[0] = info64->lo_init[0];
+ info.lo_init[1] = info64->lo_init[1];
+ if (info.lo_encrypt_type == LO_CRYPT_CRYPTOAPI)
+ memcpy(info.lo_name, info64->lo_crypt_name, LO_NAME_SIZE);
+ else
+ memcpy(info.lo_name, info64->lo_file_name, LO_NAME_SIZE);
+ memcpy(info.lo_encrypt_key, info64->lo_encrypt_key, LO_KEY_SIZE);
+
+ /* error in case values were truncated */
+ if (info.lo_device != info64->lo_device ||
+ info.lo_rdevice != info64->lo_rdevice ||
+ info.lo_inode != info64->lo_inode ||
+ info.lo_offset != info64->lo_offset ||
+ info.lo_init[0] != info64->lo_init[0] ||
+ info.lo_init[1] != info64->lo_init[1])
+ return -EOVERFLOW;
+
+ if (copy_to_user(arg, &info, sizeof(info)))
+ return -EFAULT;
+ return 0;
+}
+
+static int
+loop_set_status_compat(struct loop_device *lo,
+ const struct compat_loop_info __user *arg)
+{
+ struct loop_info64 info64;
+ int ret;
+
+ ret = loop_info64_from_compat(arg, &info64);
+ if (ret < 0)
+ return ret;
+ return loop_set_status(lo, &info64);
+}
+
+static int
+loop_get_status_compat(struct loop_device *lo,
+ struct compat_loop_info __user *arg)
{
- struct loop_device *lo = inode->i_bdev->bd_disk->private_data;
+ struct loop_info64 info64;
+ int err = 0;
+
+ if (!arg)
+ err = -EINVAL;
+ if (!err)
+ err = loop_get_status(lo, &info64);
+ if (!err)
+ err = loop_info64_to_compat(&info64, arg);
+ return err;
+}
- down(&lo->lo_ctl_mutex);
+static int lo_compat_ioctl(struct block_device *bdev, fmode_t mode,
+ unsigned int cmd, unsigned long arg)
+{
+ struct loop_device *lo = bdev->bd_disk->private_data;
+ int err;
+
+ switch(cmd) {
+ case LOOP_SET_STATUS:
+ mutex_lock(&lo->lo_ctl_mutex);
+ err = loop_set_status_compat(
+ lo, (const struct compat_loop_info __user *) arg);
+ mutex_unlock(&lo->lo_ctl_mutex);
+ break;
+ case LOOP_GET_STATUS:
+ mutex_lock(&lo->lo_ctl_mutex);
+ err = loop_get_status_compat(
+ lo, (struct compat_loop_info __user *) arg);
+ mutex_unlock(&lo->lo_ctl_mutex);
+ break;
+ case LOOP_SET_CAPACITY:
+ case LOOP_CLR_FD:
+ case LOOP_GET_STATUS64:
+ case LOOP_SET_STATUS64:
+ arg = (unsigned long) compat_ptr(arg);
+ case LOOP_SET_FD:
+ case LOOP_CHANGE_FD:
+ err = lo_ioctl(bdev, mode, cmd, arg);
+ break;
+ default:
+ err = -ENOIOCTLCMD;
+ break;
+ }
+ return err;
+}
+#endif
+
+static int lo_open(struct block_device *bdev, fmode_t mode)
+{
+ struct loop_device *lo = bdev->bd_disk->private_data;
+
+ mutex_lock(&lo->lo_ctl_mutex);
lo->lo_refcnt++;
- up(&lo->lo_ctl_mutex);
+ mutex_unlock(&lo->lo_ctl_mutex);
return 0;
}
-static int lo_release(struct inode *inode, struct file *file)
+static int lo_release(struct gendisk *disk, fmode_t mode)
{
- struct loop_device *lo = inode->i_bdev->bd_disk->private_data;
+ struct loop_device *lo = disk->private_data;
+ int err;
+
+ mutex_lock(&lo->lo_ctl_mutex);
+
+ if (--lo->lo_refcnt)
+ goto out;
- down(&lo->lo_ctl_mutex);
- --lo->lo_refcnt;
- up(&lo->lo_ctl_mutex);
+ if (lo->lo_flags & LO_FLAGS_AUTOCLEAR) {
+ /*
+ * In autoclear mode, stop the loop thread
+ * and remove configuration after last close.
+ */
+ err = loop_clr_fd(lo, NULL);
+ if (!err)
+ goto out_unlocked;
+ } else {
+ /*
+ * Otherwise keep thread (if running) and config,
+ * but flush possible ongoing bios in thread.
+ */
+ loop_flush(lo);
+ }
+out:
+ mutex_unlock(&lo->lo_ctl_mutex);
+out_unlocked:
return 0;
}
-static struct block_device_operations lo_fops = {
+static const struct block_device_operations lo_fops = {
.owner = THIS_MODULE,
.open = lo_open,
.release = lo_release,
.ioctl = lo_ioctl,
+#ifdef CONFIG_COMPAT
+ .compat_ioctl = lo_compat_ioctl,
+#endif
};
/*
* And now the modules code and kernel interface.
*/
+static int max_loop;
module_param(max_loop, int, 0);
-MODULE_PARM_DESC(max_loop, "Maximum number of loop devices (1-256)");
+MODULE_PARM_DESC(max_loop, "Maximum number of loop devices");
+module_param(max_part, int, 0);
+MODULE_PARM_DESC(max_part, "Maximum number of partitions per loop device");
MODULE_LICENSE("GPL");
MODULE_ALIAS_BLOCKDEV_MAJOR(LOOP_MAJOR);
xfer_funcs[n] = NULL;
- for (lo = &loop_dev[0]; lo < &loop_dev[max_loop]; lo++) {
- down(&lo->lo_ctl_mutex);
+ list_for_each_entry(lo, &loop_devices, lo_list) {
+ mutex_lock(&lo->lo_ctl_mutex);
if (lo->lo_encryption == xfer)
loop_release_xfer(lo);
- up(&lo->lo_ctl_mutex);
+ mutex_unlock(&lo->lo_ctl_mutex);
}
return 0;
EXPORT_SYMBOL(loop_register_transfer);
EXPORT_SYMBOL(loop_unregister_transfer);
+static struct loop_device *loop_alloc(int i)
+{
+ struct loop_device *lo;
+ struct gendisk *disk;
+
+ lo = kzalloc(sizeof(*lo), GFP_KERNEL);
+ if (!lo)
+ goto out;
+
+ lo->lo_queue = blk_alloc_queue(GFP_KERNEL);
+ if (!lo->lo_queue)
+ goto out_free_dev;
+
+ disk = lo->lo_disk = alloc_disk(1 << part_shift);
+ if (!disk)
+ goto out_free_queue;
+
+ mutex_init(&lo->lo_ctl_mutex);
+ lo->lo_number = i;
+ lo->lo_thread = NULL;
+ init_waitqueue_head(&lo->lo_event);
+ spin_lock_init(&lo->lo_lock);
+ disk->major = LOOP_MAJOR;
+ disk->first_minor = i << part_shift;
+ disk->fops = &lo_fops;
+ disk->private_data = lo;
+ disk->queue = lo->lo_queue;
+ sprintf(disk->disk_name, "loop%d", i);
+ return lo;
+
+out_free_queue:
+ blk_cleanup_queue(lo->lo_queue);
+out_free_dev:
+ kfree(lo);
+out:
+ return NULL;
+}
+
+static void loop_free(struct loop_device *lo)
+{
+ blk_cleanup_queue(lo->lo_queue);
+ put_disk(lo->lo_disk);
+ list_del(&lo->lo_list);
+ kfree(lo);
+}
+
+static struct loop_device *loop_init_one(int i)
+{
+ struct loop_device *lo;
+
+ list_for_each_entry(lo, &loop_devices, lo_list) {
+ if (lo->lo_number == i)
+ return lo;
+ }
+
+ lo = loop_alloc(i);
+ if (lo) {
+ add_disk(lo->lo_disk);
+ list_add_tail(&lo->lo_list, &loop_devices);
+ }
+ return lo;
+}
+
+static void loop_del_one(struct loop_device *lo)
+{
+ del_gendisk(lo->lo_disk);
+ loop_free(lo);
+}
+
+static struct kobject *loop_probe(dev_t dev, int *part, void *data)
+{
+ struct loop_device *lo;
+ struct kobject *kobj;
+
+ mutex_lock(&loop_devices_mutex);
+ lo = loop_init_one(dev & MINORMASK);
+ kobj = lo ? get_disk(lo->lo_disk) : ERR_PTR(-ENOMEM);
+ mutex_unlock(&loop_devices_mutex);
+
+ *part = 0;
+ return kobj;
+}
+
static int __init loop_init(void)
{
- int i;
+ int i, nr;
+ unsigned long range;
+ struct loop_device *lo, *next;
- if (max_loop < 1 || max_loop > 256) {
- printk(KERN_WARNING "loop: invalid max_loop (must be between"
- " 1 and 256), using default (8)\n");
- max_loop = 8;
+ /*
+ * loop module now has a feature to instantiate underlying device
+ * structure on-demand, provided that there is an access dev node.
+ * However, this will not work well with user space tool that doesn't
+ * know about such "feature". In order to not break any existing
+ * tool, we do the following:
+ *
+ * (1) if max_loop is specified, create that many upfront, and this
+ * also becomes a hard limit.
+ * (2) if max_loop is not specified, create 8 loop device on module
+ * load, user can further extend loop device by create dev node
+ * themselves and have kernel automatically instantiate actual
+ * device on-demand.
+ */
+
+ part_shift = 0;
+ if (max_part > 0)
+ part_shift = fls(max_part);
+
+ if (max_loop > 1UL << (MINORBITS - part_shift))
+ return -EINVAL;
+
+ if (max_loop) {
+ nr = max_loop;
+ range = max_loop;
+ } else {
+ nr = 8;
+ range = 1UL << (MINORBITS - part_shift);
}
if (register_blkdev(LOOP_MAJOR, "loop"))
return -EIO;
- loop_dev = kmalloc(max_loop * sizeof(struct loop_device), GFP_KERNEL);
- if (!loop_dev)
- goto out_mem1;
- memset(loop_dev, 0, max_loop * sizeof(struct loop_device));
+ for (i = 0; i < nr; i++) {
+ lo = loop_alloc(i);
+ if (!lo)
+ goto Enomem;
+ list_add_tail(&lo->lo_list, &loop_devices);
+ }
- disks = kmalloc(max_loop * sizeof(struct gendisk *), GFP_KERNEL);
- if (!disks)
- goto out_mem2;
+ /* point of no return */
- for (i = 0; i < max_loop; i++) {
- disks[i] = alloc_disk(1);
- if (!disks[i])
- goto out_mem3;
- }
+ list_for_each_entry(lo, &loop_devices, lo_list)
+ add_disk(lo->lo_disk);
- devfs_mk_dir("loop");
-
- for (i = 0; i < max_loop; i++) {
- struct loop_device *lo = &loop_dev[i];
- struct gendisk *disk = disks[i];
-
- memset(lo, 0, sizeof(*lo));
- lo->lo_queue = blk_alloc_queue(GFP_KERNEL);
- if (!lo->lo_queue)
- goto out_mem4;
- init_MUTEX(&lo->lo_ctl_mutex);
- init_MUTEX_LOCKED(&lo->lo_sem);
- init_MUTEX_LOCKED(&lo->lo_bh_mutex);
- lo->lo_number = i;
- spin_lock_init(&lo->lo_lock);
- disk->major = LOOP_MAJOR;
- disk->first_minor = i;
- disk->fops = &lo_fops;
- sprintf(disk->disk_name, "loop%d", i);
- sprintf(disk->devfs_name, "loop/%d", i);
- disk->private_data = lo;
- disk->queue = lo->lo_queue;
- }
+ blk_register_region(MKDEV(LOOP_MAJOR, 0), range,
+ THIS_MODULE, loop_probe, NULL, NULL);
- /* We cannot fail after we call this, so another loop!*/
- for (i = 0; i < max_loop; i++)
- add_disk(disks[i]);
- printk(KERN_INFO "loop: loaded (max %d devices)\n", max_loop);
+ printk(KERN_INFO "loop: module loaded\n");
return 0;
-out_mem4:
- while (i--)
- blk_put_queue(loop_dev[i].lo_queue);
- devfs_remove("loop");
- i = max_loop;
-out_mem3:
- while (i--)
- put_disk(disks[i]);
- kfree(disks);
-out_mem2:
- kfree(loop_dev);
-out_mem1:
+Enomem:
+ printk(KERN_INFO "loop: out of memory\n");
+
+ list_for_each_entry_safe(lo, next, &loop_devices, lo_list)
+ loop_free(lo);
+
unregister_blkdev(LOOP_MAJOR, "loop");
- printk(KERN_ERR "loop: ran out of memory\n");
return -ENOMEM;
}
-static void loop_exit(void)
+static void __exit loop_exit(void)
{
- int i;
+ unsigned long range;
+ struct loop_device *lo, *next;
- for (i = 0; i < max_loop; i++) {
- del_gendisk(disks[i]);
- blk_put_queue(loop_dev[i].lo_queue);
- put_disk(disks[i]);
- }
- devfs_remove("loop");
- if (unregister_blkdev(LOOP_MAJOR, "loop"))
- printk(KERN_WARNING "loop: cannot unregister blkdev\n");
+ range = max_loop ? max_loop : 1UL << (MINORBITS - part_shift);
+
+ list_for_each_entry_safe(lo, next, &loop_devices, lo_list)
+ loop_del_one(lo);
- kfree(disks);
- kfree(loop_dev);
+ blk_unregister_region(MKDEV(LOOP_MAJOR, 0), range);
+ unregister_blkdev(LOOP_MAJOR, "loop");
}
module_init(loop_init);
git://ftp.safe.ca / safe / jmp / linux-2.6 / blobdiff