+static void requeue_list(struct btrfs_pending_bios *pending_bios,
+ struct bio *head, struct bio *tail)
+{
+
+ struct bio *old_head;
+
+ old_head = pending_bios->head;
+ pending_bios->head = head;
+ if (pending_bios->tail)
+ tail->bi_next = old_head;
+ else
+ pending_bios->tail = tail;
+}
+
+/*
+ * we try to collect pending bios for a device so we don't get a large
+ * number of procs sending bios down to the same device. This greatly
+ * improves the schedulers ability to collect and merge the bios.
+ *
+ * But, it also turns into a long list of bios to process and that is sure
+ * to eventually make the worker thread block. The solution here is to
+ * make some progress and then put this work struct back at the end of
+ * the list if the block device is congested. This way, multiple devices
+ * can make progress from a single worker thread.
+ */
+static noinline int run_scheduled_bios(struct btrfs_device *device)
+{
+ struct bio *pending;
+ struct backing_dev_info *bdi;
+ struct btrfs_fs_info *fs_info;
+ struct btrfs_pending_bios *pending_bios;
+ struct bio *tail;
+ struct bio *cur;
+ int again = 0;
+ unsigned long num_run;
+ unsigned long num_sync_run;
+ unsigned long batch_run = 0;
+ unsigned long limit;
+ unsigned long last_waited = 0;
+ int force_reg = 0;
+
+ bdi = blk_get_backing_dev_info(device->bdev);
+ fs_info = device->dev_root->fs_info;
+ limit = btrfs_async_submit_limit(fs_info);
+ limit = limit * 2 / 3;
+
+ /* we want to make sure that every time we switch from the sync
+ * list to the normal list, we unplug
+ */
+ num_sync_run = 0;
+
+loop:
+ spin_lock(&device->io_lock);
+
+loop_lock:
+ num_run = 0;
+
+ /* take all the bios off the list at once and process them
+ * later on (without the lock held). But, remember the
+ * tail and other pointers so the bios can be properly reinserted
+ * into the list if we hit congestion
+ */
+ if (!force_reg && device->pending_sync_bios.head) {
+ pending_bios = &device->pending_sync_bios;
+ force_reg = 1;
+ } else {
+ pending_bios = &device->pending_bios;
+ force_reg = 0;
+ }
+
+ pending = pending_bios->head;
+ tail = pending_bios->tail;
+ WARN_ON(pending && !tail);
+
+ /*
+ * if pending was null this time around, no bios need processing
+ * at all and we can stop. Otherwise it'll loop back up again
+ * and do an additional check so no bios are missed.
+ *
+ * device->running_pending is used to synchronize with the
+ * schedule_bio code.
+ */
+ if (device->pending_sync_bios.head == NULL &&
+ device->pending_bios.head == NULL) {
+ again = 0;
+ device->running_pending = 0;
+ } else {
+ again = 1;
+ device->running_pending = 1;
+ }
+
+ pending_bios->head = NULL;
+ pending_bios->tail = NULL;
+
+ spin_unlock(&device->io_lock);
+
+ /*
+ * if we're doing the regular priority list, make sure we unplug
+ * for any high prio bios we've sent down
+ */
+ if (pending_bios == &device->pending_bios && num_sync_run > 0) {
+ num_sync_run = 0;
+ blk_run_backing_dev(bdi, NULL);
+ }
+
+ while (pending) {
+
+ rmb();
+ /* we want to work on both lists, but do more bios on the
+ * sync list than the regular list
+ */
+ if ((num_run > 32 &&
+ pending_bios != &device->pending_sync_bios &&
+ device->pending_sync_bios.head) ||
+ (num_run > 64 && pending_bios == &device->pending_sync_bios &&
+ device->pending_bios.head)) {
+ spin_lock(&device->io_lock);
+ requeue_list(pending_bios, pending, tail);
+ goto loop_lock;
+ }
+
+ cur = pending;
+ pending = pending->bi_next;
+ cur->bi_next = NULL;
+ atomic_dec(&fs_info->nr_async_bios);
+
+ if (atomic_read(&fs_info->nr_async_bios) < limit &&
+ waitqueue_active(&fs_info->async_submit_wait))
+ wake_up(&fs_info->async_submit_wait);
+
+ BUG_ON(atomic_read(&cur->bi_cnt) == 0);
+
+ if (bio_rw_flagged(cur, BIO_RW_SYNCIO))
+ num_sync_run++;
+
+ submit_bio(cur->bi_rw, cur);
+ num_run++;
+ batch_run++;
+ if (need_resched()) {
+ if (num_sync_run) {
+ blk_run_backing_dev(bdi, NULL);
+ num_sync_run = 0;
+ }
+ cond_resched();
+ }
+
+ /*
+ * we made progress, there is more work to do and the bdi
+ * is now congested. Back off and let other work structs
+ * run instead
+ */
+ if (pending && bdi_write_congested(bdi) && batch_run > 8 &&
+ fs_info->fs_devices->open_devices > 1) {
+ struct io_context *ioc;
+
+ ioc = current->io_context;
+
+ /*
+ * the main goal here is that we don't want to
+ * block if we're going to be able to submit
+ * more requests without blocking.
+ *
+ * This code does two great things, it pokes into
+ * the elevator code from a filesystem _and_
+ * it makes assumptions about how batching works.
+ */
+ if (ioc && ioc->nr_batch_requests > 0 &&
+ time_before(jiffies, ioc->last_waited + HZ/50UL) &&
+ (last_waited == 0 ||
+ ioc->last_waited == last_waited)) {
+ /*
+ * we want to go through our batch of
+ * requests and stop. So, we copy out
+ * the ioc->last_waited time and test
+ * against it before looping
+ */
+ last_waited = ioc->last_waited;
+ if (need_resched()) {
+ if (num_sync_run) {
+ blk_run_backing_dev(bdi, NULL);
+ num_sync_run = 0;
+ }
+ cond_resched();
+ }
+ continue;
+ }
+ spin_lock(&device->io_lock);
+ requeue_list(pending_bios, pending, tail);
+ device->running_pending = 1;
+
+ spin_unlock(&device->io_lock);
+ btrfs_requeue_work(&device->work);
+ goto done;
+ }
+ }
+
+ if (num_sync_run) {
+ num_sync_run = 0;
+ blk_run_backing_dev(bdi, NULL);
+ }
+ /*
+ * IO has already been through a long path to get here. Checksumming,
+ * async helper threads, perhaps compression. We've done a pretty
+ * good job of collecting a batch of IO and should just unplug
+ * the device right away.
+ *
+ * This will help anyone who is waiting on the IO, they might have
+ * already unplugged, but managed to do so before the bio they
+ * cared about found its way down here.
+ */
+ blk_run_backing_dev(bdi, NULL);
+
+ cond_resched();
+ if (again)
+ goto loop;
+
+ spin_lock(&device->io_lock);
+ if (device->pending_bios.head || device->pending_sync_bios.head)
+ goto loop_lock;
+ spin_unlock(&device->io_lock);
+
+done:
+ return 0;
+}
+
+static void pending_bios_fn(struct btrfs_work *work)
+{
+ struct btrfs_device *device;
+
+ device = container_of(work, struct btrfs_device, work);
+ run_scheduled_bios(device);
+}
+
+static noinline int device_list_add(const char *path,