Merge branch 'for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/ryusuke...
[safe/jmp/linux-2.6] / drivers / md / dm-mpath.c
index f1cf8f7..826bce7 100644 (file)
@@ -8,7 +8,6 @@
 #include <linux/device-mapper.h>
 
 #include "dm-path-selector.h"
-#include "dm-bio-record.h"
 #include "dm-uevent.h"
 
 #include <linux/ctype.h>
@@ -65,10 +64,12 @@ struct multipath {
        spinlock_t lock;
 
        const char *hw_handler_name;
+       char *hw_handler_params;
        unsigned nr_priority_groups;
        struct list_head priority_groups;
        unsigned pg_init_required;      /* pg_init needs calling? */
        unsigned pg_init_in_progress;   /* Only one pg_init allowed at once */
+       wait_queue_head_t pg_init_wait; /* Wait for pg_init completion */
 
        unsigned nr_valid_paths;        /* Total number of usable paths */
        struct pgpath *current_pgpath;
@@ -83,7 +84,7 @@ struct multipath {
        unsigned pg_init_count;         /* Number of times pg_init called */
 
        struct work_struct process_queued_ios;
-       struct bio_list queued_ios;
+       struct list_head queued_ios;
        unsigned queue_size;
 
        struct work_struct trigger_event;
@@ -93,6 +94,8 @@ struct multipath {
         * can resubmit bios on error.
         */
        mempool_t *mpio_pool;
+
+       struct mutex work_mutex;
 };
 
 /*
@@ -100,7 +103,7 @@ struct multipath {
  */
 struct dm_mpath_io {
        struct pgpath *pgpath;
-       struct dm_bio_details details;
+       size_t nr_bytes;
 };
 
 typedef int (*action_fn) (struct pgpath *pgpath);
@@ -193,10 +196,13 @@ static struct multipath *alloc_multipath(struct dm_target *ti)
        m = kzalloc(sizeof(*m), GFP_KERNEL);
        if (m) {
                INIT_LIST_HEAD(&m->priority_groups);
+               INIT_LIST_HEAD(&m->queued_ios);
                spin_lock_init(&m->lock);
                m->queue_io = 1;
                INIT_WORK(&m->process_queued_ios, process_queued_ios);
                INIT_WORK(&m->trigger_event, trigger_event);
+               init_waitqueue_head(&m->pg_init_wait);
+               mutex_init(&m->work_mutex);
                m->mpio_pool = mempool_create_slab_pool(MIN_IOS, _mpio_cache);
                if (!m->mpio_pool) {
                        kfree(m);
@@ -219,6 +225,7 @@ static void free_multipath(struct multipath *m)
        }
 
        kfree(m->hw_handler_name);
+       kfree(m->hw_handler_params);
        mempool_destroy(m->mpio_pool);
        kfree(m);
 }
@@ -228,6 +235,21 @@ static void free_multipath(struct multipath *m)
  * Path selection
  *-----------------------------------------------*/
 
+static void __pg_init_all_paths(struct multipath *m)
+{
+       struct pgpath *pgpath;
+
+       m->pg_init_count++;
+       m->pg_init_required = 0;
+       list_for_each_entry(pgpath, &m->current_pg->pgpaths, list) {
+               /* Skip failed paths */
+               if (!pgpath->is_active)
+                       continue;
+               if (queue_work(kmpath_handlerd, &pgpath->activate_path))
+                       m->pg_init_in_progress++;
+       }
+}
+
 static void __switch_pg(struct multipath *m, struct pgpath *pgpath)
 {
        m->current_pg = pgpath->pg;
@@ -244,11 +266,12 @@ static void __switch_pg(struct multipath *m, struct pgpath *pgpath)
        m->pg_init_count = 0;
 }
 
-static int __choose_path_in_pg(struct multipath *m, struct priority_group *pg)
+static int __choose_path_in_pg(struct multipath *m, struct priority_group *pg,
+                              size_t nr_bytes)
 {
        struct dm_path *path;
 
-       path = pg->ps.type->select_path(&pg->ps, &m->repeat_count);
+       path = pg->ps.type->select_path(&pg->ps, &m->repeat_count, nr_bytes);
        if (!path)
                return -ENXIO;
 
@@ -260,7 +283,7 @@ static int __choose_path_in_pg(struct multipath *m, struct priority_group *pg)
        return 0;
 }
 
-static void __choose_pgpath(struct multipath *m)
+static void __choose_pgpath(struct multipath *m, size_t nr_bytes)
 {
        struct priority_group *pg;
        unsigned bypassed = 1;
@@ -272,12 +295,12 @@ static void __choose_pgpath(struct multipath *m)
        if (m->next_pg) {
                pg = m->next_pg;
                m->next_pg = NULL;
-               if (!__choose_path_in_pg(m, pg))
+               if (!__choose_path_in_pg(m, pg, nr_bytes))
                        return;
        }
 
        /* Don't change PG until it has no remaining paths */
-       if (m->current_pg && !__choose_path_in_pg(m, m->current_pg))
+       if (m->current_pg && !__choose_path_in_pg(m, m->current_pg, nr_bytes))
                return;
 
        /*
@@ -289,7 +312,7 @@ static void __choose_pgpath(struct multipath *m)
                list_for_each_entry(pg, &m->priority_groups, list) {
                        if (pg->bypassed == bypassed)
                                continue;
-                       if (!__choose_path_in_pg(m, pg))
+                       if (!__choose_path_in_pg(m, pg, nr_bytes))
                                return;
                }
        } while (bypassed--);
@@ -316,19 +339,21 @@ static int __must_push_back(struct multipath *m)
                dm_noflush_suspending(m->ti));
 }
 
-static int map_io(struct multipath *m, struct bio *bio,
+static int map_io(struct multipath *m, struct request *clone,
                  struct dm_mpath_io *mpio, unsigned was_queued)
 {
        int r = DM_MAPIO_REMAPPED;
+       size_t nr_bytes = blk_rq_bytes(clone);
        unsigned long flags;
        struct pgpath *pgpath;
+       struct block_device *bdev;
 
        spin_lock_irqsave(&m->lock, flags);
 
        /* Do we need to select a new pgpath? */
        if (!m->current_pgpath ||
            (!m->queue_io && (m->repeat_count && --m->repeat_count == 0)))
-               __choose_pgpath(m);
+               __choose_pgpath(m, nr_bytes);
 
        pgpath = m->current_pgpath;
 
@@ -338,21 +363,28 @@ static int map_io(struct multipath *m, struct bio *bio,
        if ((pgpath && m->queue_io) ||
            (!pgpath && m->queue_if_no_path)) {
                /* Queue for the daemon to resubmit */
-               bio_list_add(&m->queued_ios, bio);
+               list_add_tail(&clone->queuelist, &m->queued_ios);
                m->queue_size++;
                if ((m->pg_init_required && !m->pg_init_in_progress) ||
                    !m->queue_io)
                        queue_work(kmultipathd, &m->process_queued_ios);
                pgpath = NULL;
                r = DM_MAPIO_SUBMITTED;
-       } else if (pgpath)
-               bio->bi_bdev = pgpath->path.dev->bdev;
-       else if (__must_push_back(m))
+       } else if (pgpath) {
+               bdev = pgpath->path.dev->bdev;
+               clone->q = bdev_get_queue(bdev);
+               clone->rq_disk = bdev->bd_disk;
+       } else if (__must_push_back(m))
                r = DM_MAPIO_REQUEUE;
        else
                r = -EIO;       /* Failed */
 
        mpio->pgpath = pgpath;
+       mpio->nr_bytes = nr_bytes;
+
+       if (r == DM_MAPIO_REMAPPED && pgpath->pg->ps.type->start_io)
+               pgpath->pg->ps.type->start_io(&pgpath->pg->ps, &pgpath->path,
+                                             nr_bytes);
 
        spin_unlock_irqrestore(&m->lock, flags);
 
@@ -390,30 +422,31 @@ static void dispatch_queued_ios(struct multipath *m)
 {
        int r;
        unsigned long flags;
-       struct bio *bio = NULL, *next;
        struct dm_mpath_io *mpio;
        union map_info *info;
+       struct request *clone, *n;
+       LIST_HEAD(cl);
 
        spin_lock_irqsave(&m->lock, flags);
-       bio = bio_list_get(&m->queued_ios);
+       list_splice_init(&m->queued_ios, &cl);
        spin_unlock_irqrestore(&m->lock, flags);
 
-       while (bio) {
-               next = bio->bi_next;
-               bio->bi_next = NULL;
+       list_for_each_entry_safe(clone, n, &cl, queuelist) {
+               list_del_init(&clone->queuelist);
 
-               info = dm_get_mapinfo(bio);
+               info = dm_get_rq_mapinfo(clone);
                mpio = info->ptr;
 
-               r = map_io(m, bio, mpio, 1);
-               if (r < 0)
-                       bio_endio(bio, r);
-               else if (r == DM_MAPIO_REMAPPED)
-                       generic_make_request(bio);
-               else if (r == DM_MAPIO_REQUEUE)
-                       bio_endio(bio, -EIO);
-
-               bio = next;
+               r = map_io(m, clone, mpio, 1);
+               if (r < 0) {
+                       mempool_free(mpio, m->mpio_pool);
+                       dm_kill_unmapped_request(clone, r);
+               } else if (r == DM_MAPIO_REMAPPED)
+                       dm_dispatch_request(clone);
+               else if (r == DM_MAPIO_REQUEUE) {
+                       mempool_free(mpio, m->mpio_pool);
+                       dm_requeue_unmapped_request(clone);
+               }
        }
 }
 
@@ -421,7 +454,7 @@ static void process_queued_ios(struct work_struct *work)
 {
        struct multipath *m =
                container_of(work, struct multipath, process_queued_ios);
-       struct pgpath *pgpath = NULL, *tmp;
+       struct pgpath *pgpath = NULL;
        unsigned must_queue = 1;
        unsigned long flags;
 
@@ -431,7 +464,7 @@ static void process_queued_ios(struct work_struct *work)
                goto out;
 
        if (!m->current_pgpath)
-               __choose_pgpath(m);
+               __choose_pgpath(m, 0);
 
        pgpath = m->current_pgpath;
 
@@ -439,14 +472,9 @@ static void process_queued_ios(struct work_struct *work)
            (!pgpath && !m->queue_if_no_path))
                must_queue = 0;
 
-       if (m->pg_init_required && !m->pg_init_in_progress && pgpath) {
-               m->pg_init_count++;
-               m->pg_init_required = 0;
-               list_for_each_entry(tmp, &pgpath->pg->pgpaths, list) {
-                       if (queue_work(kmpath_handlerd, &tmp->activate_path))
-                               m->pg_init_in_progress++;
-               }
-       }
+       if (m->pg_init_required && !m->pg_init_in_progress && pgpath)
+               __pg_init_all_paths(m);
+
 out:
        spin_unlock_irqrestore(&m->lock, flags);
        if (!must_queue)
@@ -579,8 +607,8 @@ static struct pgpath *parse_path(struct arg_set *as, struct path_selector *ps,
        if (!p)
                return ERR_PTR(-ENOMEM);
 
-       r = dm_get_device(ti, shift(as), ti->begin, ti->len,
-                         dm_table_get_mode(ti->table), &p->path.dev);
+       r = dm_get_device(ti, shift(as), dm_table_get_mode(ti->table),
+                         &p->path.dev);
        if (r) {
                ti->error = "error getting device";
                goto bad;
@@ -604,6 +632,17 @@ static struct pgpath *parse_path(struct arg_set *as, struct path_selector *ps,
                        dm_put_device(ti, p->path.dev);
                        goto bad;
                }
+
+               if (m->hw_handler_params) {
+                       r = scsi_dh_set_params(q, m->hw_handler_params);
+                       if (r < 0) {
+                               ti->error = "unable to set hardware "
+                                                       "handler parameters";
+                               scsi_dh_detach(q);
+                               dm_put_device(ti, p->path.dev);
+                               goto bad;
+                       }
+               }
        }
 
        r = ps->type->add_path(ps, &p->path, as->argc, as->argv, &ti->error);
@@ -694,6 +733,7 @@ static struct priority_group *parse_priority_group(struct arg_set *as,
 static int parse_hw_handler(struct arg_set *as, struct multipath *m)
 {
        unsigned hw_argc;
+       int ret;
        struct dm_target *ti = m->ti;
 
        static struct param _params[] = {
@@ -715,17 +755,33 @@ static int parse_hw_handler(struct arg_set *as, struct multipath *m)
        request_module("scsi_dh_%s", m->hw_handler_name);
        if (scsi_dh_handler_exist(m->hw_handler_name) == 0) {
                ti->error = "unknown hardware handler type";
-               kfree(m->hw_handler_name);
-               m->hw_handler_name = NULL;
-               return -EINVAL;
+               ret = -EINVAL;
+               goto fail;
        }
 
-       if (hw_argc > 1)
-               DMWARN("Ignoring user-specified arguments for "
-                      "hardware handler \"%s\"", m->hw_handler_name);
+       if (hw_argc > 1) {
+               char *p;
+               int i, j, len = 4;
+
+               for (i = 0; i <= hw_argc - 2; i++)
+                       len += strlen(as->argv[i]) + 1;
+               p = m->hw_handler_params = kzalloc(len, GFP_KERNEL);
+               if (!p) {
+                       ti->error = "memory allocation failed";
+                       ret = -ENOMEM;
+                       goto fail;
+               }
+               j = sprintf(p, "%d", hw_argc - 1);
+               for (i = 0, p+=j+1; i <= hw_argc - 2; i++, p+=j+1)
+                       j = sprintf(p, "%s", as->argv[i]);
+       }
        consume(as, hw_argc - 1);
 
        return 0;
+fail:
+       kfree(m->hw_handler_name);
+       m->hw_handler_name = NULL;
+       return ret;
 }
 
 static int parse_features(struct arg_set *as, struct multipath *m)
@@ -844,32 +900,65 @@ static int multipath_ctr(struct dm_target *ti, unsigned int argc,
        return r;
 }
 
-static void multipath_dtr(struct dm_target *ti)
+static void multipath_wait_for_pg_init_completion(struct multipath *m)
 {
-       struct multipath *m = (struct multipath *) ti->private;
+       DECLARE_WAITQUEUE(wait, current);
+       unsigned long flags;
+
+       add_wait_queue(&m->pg_init_wait, &wait);
+
+       while (1) {
+               set_current_state(TASK_UNINTERRUPTIBLE);
+
+               spin_lock_irqsave(&m->lock, flags);
+               if (!m->pg_init_in_progress) {
+                       spin_unlock_irqrestore(&m->lock, flags);
+                       break;
+               }
+               spin_unlock_irqrestore(&m->lock, flags);
+
+               io_schedule();
+       }
+       set_current_state(TASK_RUNNING);
 
+       remove_wait_queue(&m->pg_init_wait, &wait);
+}
+
+static void flush_multipath_work(struct multipath *m)
+{
        flush_workqueue(kmpath_handlerd);
+       multipath_wait_for_pg_init_completion(m);
        flush_workqueue(kmultipathd);
        flush_scheduled_work();
+}
+
+static void multipath_dtr(struct dm_target *ti)
+{
+       struct multipath *m = ti->private;
+
+       flush_multipath_work(m);
        free_multipath(m);
 }
 
 /*
- * Map bios, recording original fields for later in case we have to resubmit
+ * Map cloned requests
  */
-static int multipath_map(struct dm_target *ti, struct bio *bio,
+static int multipath_map(struct dm_target *ti, struct request *clone,
                         union map_info *map_context)
 {
        int r;
        struct dm_mpath_io *mpio;
        struct multipath *m = (struct multipath *) ti->private;
 
-       mpio = mempool_alloc(m->mpio_pool, GFP_NOIO);
-       dm_bio_record(&mpio->details, bio);
+       mpio = mempool_alloc(m->mpio_pool, GFP_ATOMIC);
+       if (!mpio)
+               /* ENOMEM, requeue */
+               return DM_MAPIO_REQUEUE;
+       memset(mpio, 0, sizeof(*mpio));
 
        map_context->ptr = mpio;
-       bio->bi_rw |= (1 << BIO_RW_FAILFAST_TRANSPORT);
-       r = map_io(m, bio, mpio, 0);
+       clone->cmd_flags |= REQ_FAILFAST_TRANSPORT;
+       r = map_io(m, clone, mpio, 0);
        if (r < 0 || r == DM_MAPIO_REQUEUE)
                mempool_free(mpio, m->mpio_pool);
 
@@ -1072,9 +1161,9 @@ static int pg_init_limit_reached(struct multipath *m, struct pgpath *pgpath)
        return limit_reached;
 }
 
-static void pg_init_done(struct dm_path *path, int errors)
+static void pg_init_done(void *data, int errors)
 {
-       struct pgpath *pgpath = path_to_pgpath(path);
+       struct pgpath *pgpath = data;
        struct priority_group *pg = pgpath->pg;
        struct multipath *m = pg->m;
        unsigned long flags;
@@ -1088,8 +1177,8 @@ static void pg_init_done(struct dm_path *path, int errors)
                        errors = 0;
                        break;
                }
-               DMERR("Cannot failover device because scsi_dh_%s was not "
-                     "loaded.", m->hw_handler_name);
+               DMERR("Could not failover the device: Handler scsi_dh_%s "
+                     "Error %d.", m->hw_handler_name, errors);
                /*
                 * Fail path for now, so we do not ping pong
                 */
@@ -1126,77 +1215,74 @@ static void pg_init_done(struct dm_path *path, int errors)
                        m->current_pgpath = NULL;
                        m->current_pg = NULL;
                }
-       } else if (!m->pg_init_required) {
-               m->queue_io = 0;
+       } else if (!m->pg_init_required)
                pg->bypassed = 0;
-       }
 
-       m->pg_init_in_progress--;
-       if (!m->pg_init_in_progress)
-               queue_work(kmultipathd, &m->process_queued_ios);
+       if (--m->pg_init_in_progress)
+               /* Activations of other paths are still on going */
+               goto out;
+
+       if (!m->pg_init_required)
+               m->queue_io = 0;
+
+       queue_work(kmultipathd, &m->process_queued_ios);
+
+       /*
+        * Wake up any thread waiting to suspend.
+        */
+       wake_up(&m->pg_init_wait);
+
+out:
        spin_unlock_irqrestore(&m->lock, flags);
 }
 
 static void activate_path(struct work_struct *work)
 {
-       int ret;
        struct pgpath *pgpath =
                container_of(work, struct pgpath, activate_path);
 
-       ret = scsi_dh_activate(bdev_get_queue(pgpath->path.dev->bdev));
-       pg_init_done(&pgpath->path, ret);
+       scsi_dh_activate(bdev_get_queue(pgpath->path.dev->bdev),
+                               pg_init_done, pgpath);
 }
 
 /*
  * end_io handling
  */
-static int do_end_io(struct multipath *m, struct bio *bio,
+static int do_end_io(struct multipath *m, struct request *clone,
                     int error, struct dm_mpath_io *mpio)
 {
+       /*
+        * We don't queue any clone request inside the multipath target
+        * during end I/O handling, since those clone requests don't have
+        * bio clones.  If we queue them inside the multipath target,
+        * we need to make bio clones, that requires memory allocation.
+        * (See drivers/md/dm.c:end_clone_bio() about why the clone requests
+        *  don't have bio clones.)
+        * Instead of queueing the clone request here, we queue the original
+        * request into dm core, which will remake a clone request and
+        * clone bios for it and resubmit it later.
+        */
+       int r = DM_ENDIO_REQUEUE;
        unsigned long flags;
 
-       if (!error)
+       if (!error && !clone->errors)
                return 0;       /* I/O complete */
 
-       if ((error == -EWOULDBLOCK) && bio_rw_ahead(bio))
-               return error;
-
        if (error == -EOPNOTSUPP)
                return error;
 
-       spin_lock_irqsave(&m->lock, flags);
-       if (!m->nr_valid_paths) {
-               if (__must_push_back(m)) {
-                       spin_unlock_irqrestore(&m->lock, flags);
-                       return DM_ENDIO_REQUEUE;
-               } else if (!m->queue_if_no_path) {
-                       spin_unlock_irqrestore(&m->lock, flags);
-                       return -EIO;
-               } else {
-                       spin_unlock_irqrestore(&m->lock, flags);
-                       goto requeue;
-               }
-       }
-       spin_unlock_irqrestore(&m->lock, flags);
-
        if (mpio->pgpath)
                fail_path(mpio->pgpath);
 
-      requeue:
-       dm_bio_restore(&mpio->details, bio);
-
-       /* queue for the daemon to resubmit or fail */
        spin_lock_irqsave(&m->lock, flags);
-       bio_list_add(&m->queued_ios, bio);
-       m->queue_size++;
-       if (!m->queue_io)
-               queue_work(kmultipathd, &m->process_queued_ios);
+       if (!m->nr_valid_paths && !m->queue_if_no_path && !__must_push_back(m))
+               r = -EIO;
        spin_unlock_irqrestore(&m->lock, flags);
 
-       return DM_ENDIO_INCOMPLETE;     /* io not complete */
+       return r;
 }
 
-static int multipath_end_io(struct dm_target *ti, struct bio *bio,
+static int multipath_end_io(struct dm_target *ti, struct request *clone,
                            int error, union map_info *map_context)
 {
        struct multipath *m = ti->private;
@@ -1205,14 +1291,13 @@ static int multipath_end_io(struct dm_target *ti, struct bio *bio,
        struct path_selector *ps;
        int r;
 
-       r  = do_end_io(m, bio, error, mpio);
+       r  = do_end_io(m, clone, error, mpio);
        if (pgpath) {
                ps = &pgpath->pg->ps;
                if (ps->type->end_io)
-                       ps->type->end_io(ps, &pgpath->path);
+                       ps->type->end_io(ps, &pgpath->path, mpio->nr_bytes);
        }
-       if (r != DM_ENDIO_INCOMPLETE)
-               mempool_free(mpio, m->mpio_pool);
+       mempool_free(mpio, m->mpio_pool);
 
        return r;
 }
@@ -1230,6 +1315,15 @@ static void multipath_presuspend(struct dm_target *ti)
        queue_if_no_path(m, 0, 1);
 }
 
+static void multipath_postsuspend(struct dm_target *ti)
+{
+       struct multipath *m = ti->private;
+
+       mutex_lock(&m->work_mutex);
+       flush_multipath_work(m);
+       mutex_unlock(&m->work_mutex);
+}
+
 /*
  * Restore the queue_if_no_path setting.
  */
@@ -1366,51 +1460,65 @@ static int multipath_status(struct dm_target *ti, status_type_t type,
 
 static int multipath_message(struct dm_target *ti, unsigned argc, char **argv)
 {
-       int r;
+       int r = -EINVAL;
        struct dm_dev *dev;
        struct multipath *m = (struct multipath *) ti->private;
        action_fn action;
 
+       mutex_lock(&m->work_mutex);
+
+       if (dm_suspended(ti)) {
+               r = -EBUSY;
+               goto out;
+       }
+
        if (argc == 1) {
-               if (!strnicmp(argv[0], MESG_STR("queue_if_no_path")))
-                       return queue_if_no_path(m, 1, 0);
-               else if (!strnicmp(argv[0], MESG_STR("fail_if_no_path")))
-                       return queue_if_no_path(m, 0, 0);
+               if (!strnicmp(argv[0], MESG_STR("queue_if_no_path"))) {
+                       r = queue_if_no_path(m, 1, 0);
+                       goto out;
+               } else if (!strnicmp(argv[0], MESG_STR("fail_if_no_path"))) {
+                       r = queue_if_no_path(m, 0, 0);
+                       goto out;
+               }
        }
 
-       if (argc != 2)
-               goto error;
+       if (argc != 2) {
+               DMWARN("Unrecognised multipath message received.");
+               goto out;
+       }
 
-       if (!strnicmp(argv[0], MESG_STR("disable_group")))
-               return bypass_pg_num(m, argv[1], 1);
-       else if (!strnicmp(argv[0], MESG_STR("enable_group")))
-               return bypass_pg_num(m, argv[1], 0);
-       else if (!strnicmp(argv[0], MESG_STR("switch_group")))
-               return switch_pg_num(m, argv[1]);
-       else if (!strnicmp(argv[0], MESG_STR("reinstate_path")))
+       if (!strnicmp(argv[0], MESG_STR("disable_group"))) {
+               r = bypass_pg_num(m, argv[1], 1);
+               goto out;
+       } else if (!strnicmp(argv[0], MESG_STR("enable_group"))) {
+               r = bypass_pg_num(m, argv[1], 0);
+               goto out;
+       } else if (!strnicmp(argv[0], MESG_STR("switch_group"))) {
+               r = switch_pg_num(m, argv[1]);
+               goto out;
+       } else if (!strnicmp(argv[0], MESG_STR("reinstate_path")))
                action = reinstate_path;
        else if (!strnicmp(argv[0], MESG_STR("fail_path")))
                action = fail_path;
-       else
-               goto error;
+       else {
+               DMWARN("Unrecognised multipath message received.");
+               goto out;
+       }
 
-       r = dm_get_device(ti, argv[1], ti->begin, ti->len,
-                         dm_table_get_mode(ti->table), &dev);
+       r = dm_get_device(ti, argv[1], dm_table_get_mode(ti->table), &dev);
        if (r) {
                DMWARN("message: error getting device %s",
                       argv[1]);
-               return -EINVAL;
+               goto out;
        }
 
        r = action_dev(m, dev, action);
 
        dm_put_device(ti, dev);
 
+out:
+       mutex_unlock(&m->work_mutex);
        return r;
-
-error:
-       DMWARN("Unrecognised multipath message received.");
-       return -EINVAL;
 }
 
 static int multipath_ioctl(struct dm_target *ti, unsigned int cmd,
@@ -1425,7 +1533,7 @@ static int multipath_ioctl(struct dm_target *ti, unsigned int cmd,
        spin_lock_irqsave(&m->lock, flags);
 
        if (!m->current_pgpath)
-               __choose_pgpath(m);
+               __choose_pgpath(m, 0);
 
        if (m->current_pgpath) {
                bdev = m->current_pgpath->path.dev->bdev;
@@ -1442,22 +1550,114 @@ static int multipath_ioctl(struct dm_target *ti, unsigned int cmd,
        return r ? : __blkdev_driver_ioctl(bdev, mode, cmd, arg);
 }
 
+static int multipath_iterate_devices(struct dm_target *ti,
+                                    iterate_devices_callout_fn fn, void *data)
+{
+       struct multipath *m = ti->private;
+       struct priority_group *pg;
+       struct pgpath *p;
+       int ret = 0;
+
+       list_for_each_entry(pg, &m->priority_groups, list) {
+               list_for_each_entry(p, &pg->pgpaths, list) {
+                       ret = fn(ti, p->path.dev, ti->begin, ti->len, data);
+                       if (ret)
+                               goto out;
+               }
+       }
+
+out:
+       return ret;
+}
+
+static int __pgpath_busy(struct pgpath *pgpath)
+{
+       struct request_queue *q = bdev_get_queue(pgpath->path.dev->bdev);
+
+       return dm_underlying_device_busy(q);
+}
+
+/*
+ * We return "busy", only when we can map I/Os but underlying devices
+ * are busy (so even if we map I/Os now, the I/Os will wait on
+ * the underlying queue).
+ * In other words, if we want to kill I/Os or queue them inside us
+ * due to map unavailability, we don't return "busy".  Otherwise,
+ * dm core won't give us the I/Os and we can't do what we want.
+ */
+static int multipath_busy(struct dm_target *ti)
+{
+       int busy = 0, has_active = 0;
+       struct multipath *m = ti->private;
+       struct priority_group *pg;
+       struct pgpath *pgpath;
+       unsigned long flags;
+
+       spin_lock_irqsave(&m->lock, flags);
+
+       /* Guess which priority_group will be used at next mapping time */
+       if (unlikely(!m->current_pgpath && m->next_pg))
+               pg = m->next_pg;
+       else if (likely(m->current_pg))
+               pg = m->current_pg;
+       else
+               /*
+                * We don't know which pg will be used at next mapping time.
+                * We don't call __choose_pgpath() here to avoid to trigger
+                * pg_init just by busy checking.
+                * So we don't know whether underlying devices we will be using
+                * at next mapping time are busy or not. Just try mapping.
+                */
+               goto out;
+
+       /*
+        * If there is one non-busy active path at least, the path selector
+        * will be able to select it. So we consider such a pg as not busy.
+        */
+       busy = 1;
+       list_for_each_entry(pgpath, &pg->pgpaths, list)
+               if (pgpath->is_active) {
+                       has_active = 1;
+
+                       if (!__pgpath_busy(pgpath)) {
+                               busy = 0;
+                               break;
+                       }
+               }
+
+       if (!has_active)
+               /*
+                * No active path in this pg, so this pg won't be used and
+                * the current_pg will be changed at next mapping time.
+                * We need to try mapping to determine it.
+                */
+               busy = 0;
+
+out:
+       spin_unlock_irqrestore(&m->lock, flags);
+
+       return busy;
+}
+
 /*-----------------------------------------------------------------
  * Module setup
  *---------------------------------------------------------------*/
 static struct target_type multipath_target = {
        .name = "multipath",
-       .version = {1, 0, 5},
+       .version = {1, 1, 1},
        .module = THIS_MODULE,
        .ctr = multipath_ctr,
        .dtr = multipath_dtr,
-       .map = multipath_map,
-       .end_io = multipath_end_io,
+       .map_rq = multipath_map,
+       .rq_end_io = multipath_end_io,
        .presuspend = multipath_presuspend,
+       .postsuspend = multipath_postsuspend,
        .resume = multipath_resume,
        .status = multipath_status,
        .message = multipath_message,
        .ioctl  = multipath_ioctl,
+       .iterate_devices = multipath_iterate_devices,
+       .busy = multipath_busy,
 };
 
 static int __init dm_multipath_init(void)