rq->end_io = NULL;
rq->end_io_data = NULL;
rq->completion_data = NULL;
+ rq->next_rq = NULL;
}
/**
if (rq == &q->post_flush_rq)
return QUEUE_ORDSEQ_POSTFLUSH;
+ /*
+ * !fs requests don't need to follow barrier ordering. Always
+ * put them at the front. This fixes the following deadlock.
+ *
+ * http://thread.gmane.org/gmane.linux.kernel/537473
+ */
+ if (!blk_fs_request(rq))
+ return QUEUE_ORDSEQ_DRAIN;
+
if ((rq->cmd_flags & REQ_ORDERED_COLOR) ==
(q->orig_bar_rq->cmd_flags & REQ_ORDERED_COLOR))
return QUEUE_ORDSEQ_DRAIN;
static int flush_dry_bio_endio(struct bio *bio, unsigned int bytes, int error)
{
request_queue_t *q = bio->bi_private;
- struct bio_vec *bvec;
- int i;
/*
* This is dry run, restore bio_sector and size. We'll finish
if (bio->bi_size)
return 1;
- /* Rewind bvec's */
- bio->bi_idx = 0;
- bio_for_each_segment(bvec, bio, i) {
- bvec->bv_len += bvec->bv_offset;
- bvec->bv_offset = 0;
- }
-
/* Reset bio */
set_bit(BIO_UPTODATE, &bio->bi_flags);
bio->bi_size = q->bi_size;
* considered part of another segment, since that might
* change with the bounce page.
*/
- high = page_to_pfn(bv->bv_page) >= q->bounce_pfn;
+ high = page_to_pfn(bv->bv_page) > q->bounce_pfn;
if (high || highprv)
goto new_hw_segment;
if (cluster) {
bio->bi_hw_segments = nr_hw_segs;
bio->bi_flags |= (1 << BIO_SEG_VALID);
}
-
+EXPORT_SYMBOL(blk_recount_segments);
static int blk_phys_contig_segment(request_queue_t *q, struct bio *bio,
struct bio *nxt)
if (unlikely(!bio_flagged(nxt, BIO_SEG_VALID)))
blk_recount_segments(q, nxt);
if (!BIOVEC_VIRT_MERGEABLE(__BVEC_END(bio), __BVEC_START(nxt)) ||
- BIOVEC_VIRT_OVERSIZE(bio->bi_hw_front_size + bio->bi_hw_back_size))
+ BIOVEC_VIRT_OVERSIZE(bio->bi_hw_back_size + nxt->bi_hw_front_size))
return 0;
- if (bio->bi_size + nxt->bi_size > q->max_segment_size)
+ if (bio->bi_hw_back_size + nxt->bi_hw_front_size > q->max_segment_size)
return 0;
return 1;
* on a queue, such as calling the unplug function after a timeout.
* A block device may call blk_sync_queue to ensure that any
* such activity is cancelled, thus allowing it to release resources
- * the the callbacks might use. The caller must already have made sure
+ * that the callbacks might use. The caller must already have made sure
* that its ->make_request_fn will not re-add plugging prior to calling
* this function.
*
void blk_sync_queue(struct request_queue *q)
{
del_timer_sync(&q->unplug_timer);
- kblockd_flush();
}
EXPORT_SYMBOL(blk_sync_queue);
{
request_queue_t *q;
- q = kmem_cache_alloc_node(requestq_cachep, gfp_mask, node_id);
+ q = kmem_cache_alloc_node(requestq_cachep,
+ gfp_mask | __GFP_ZERO, node_id);
if (!q)
return NULL;
- memset(q, 0, sizeof(*q));
init_timer(&q->unplug_timer);
snprintf(q->kobj.name, KOBJ_NAME_LEN, "%s", "queue");
blk_queue_max_hw_segments(q, MAX_HW_SEGMENTS);
blk_queue_max_phys_segments(q, MAX_PHYS_SEGMENTS);
+ q->sg_reserved_size = INT_MAX;
+
/*
* all done
*/
*/
bio_get(bio);
- /*
- * for most (all? don't know of any) queues we could
- * skip grabbing the queue lock here. only drivers with
- * funky private ->back_merge_fn() function could be
- * problematic.
- */
- spin_lock_irq(q->queue_lock);
if (!rq->bio)
blk_rq_bio_prep(q, rq, bio);
else if (!ll_back_merge_fn(q, rq, bio)) {
ret = -EINVAL;
- spin_unlock_irq(q->queue_lock);
goto unmap_bio;
} else {
rq->biotail->bi_next = bio;
rq->data_len += bio->bi_size;
}
- spin_unlock_irq(q->queue_lock);
return bio->bi_size;
unsigned long len)
{
unsigned long bytes_read = 0;
+ struct bio *bio = NULL;
int ret;
if (len > (q->max_hw_sectors << 9))
ret = __blk_rq_map_user(q, rq, ubuf, map_len);
if (ret < 0)
goto unmap_rq;
+ if (!bio)
+ bio = rq->bio;
bytes_read += ret;
ubuf += ret;
}
rq->buffer = rq->data = NULL;
return 0;
unmap_rq:
- blk_rq_unmap_user(rq);
+ blk_rq_unmap_user(bio);
return ret;
}
* @rq: request to map data to
* @iov: pointer to the iovec
* @iov_count: number of elements in the iovec
+ * @len: I/O byte count
*
* Description:
* Data will be mapped directly for zero copy io, if possible. Otherwise
/**
* blk_rq_unmap_user - unmap a request with user data
- * @rq: rq to be unmapped
+ * @bio: start of bio list
*
* Description:
- * Unmap a rq previously mapped by blk_rq_map_user().
- * rq->bio must be set to the original head of the request.
+ * Unmap a rq previously mapped by blk_rq_map_user(). The caller must
+ * supply the original rq->bio from the blk_rq_map_user() return, since
+ * the io completion may have changed rq->bio.
*/
-int blk_rq_unmap_user(struct request *rq)
+int blk_rq_unmap_user(struct bio *bio)
{
- struct bio *bio, *mapped_bio;
+ struct bio *mapped_bio;
+ int ret = 0, ret2;
- while ((bio = rq->bio)) {
- if (bio_flagged(bio, BIO_BOUNCED))
+ while (bio) {
+ mapped_bio = bio;
+ if (unlikely(bio_flagged(bio, BIO_BOUNCED)))
mapped_bio = bio->bi_private;
- else
- mapped_bio = bio;
- __blk_rq_unmap_user(mapped_bio);
- rq->bio = bio->bi_next;
- bio_put(bio);
+ ret2 = __blk_rq_unmap_user(mapped_bio);
+ if (ret2 && !ret)
+ ret = ret2;
+
+ mapped_bio = bio;
+ bio = bio->bi_next;
+ bio_put(mapped_bio);
}
- return 0;
+
+ return ret;
}
EXPORT_SYMBOL(blk_rq_unmap_user);
bio->bi_rw |= (1 << BIO_RW);
blk_rq_bio_prep(q, rq, bio);
+ blk_queue_bounce(q, &rq->bio);
rq->buffer = rq->data = NULL;
return 0;
}
* bi_sector for remaps as it sees fit. So the values of these fields
* should NOT be depended on after the call to generic_make_request.
*/
-void generic_make_request(struct bio *bio)
+static inline void __generic_make_request(struct bio *bio)
{
request_queue_t *q;
sector_t maxsector;
} while (ret);
}
+/*
+ * We only want one ->make_request_fn to be active at a time,
+ * else stack usage with stacked devices could be a problem.
+ * So use current->bio_{list,tail} to keep a list of requests
+ * submited by a make_request_fn function.
+ * current->bio_tail is also used as a flag to say if
+ * generic_make_request is currently active in this task or not.
+ * If it is NULL, then no make_request is active. If it is non-NULL,
+ * then a make_request is active, and new requests should be added
+ * at the tail
+ */
+void generic_make_request(struct bio *bio)
+{
+ if (current->bio_tail) {
+ /* make_request is active */
+ *(current->bio_tail) = bio;
+ bio->bi_next = NULL;
+ current->bio_tail = &bio->bi_next;
+ return;
+ }
+ /* following loop may be a bit non-obvious, and so deserves some
+ * explanation.
+ * Before entering the loop, bio->bi_next is NULL (as all callers
+ * ensure that) so we have a list with a single bio.
+ * We pretend that we have just taken it off a longer list, so
+ * we assign bio_list to the next (which is NULL) and bio_tail
+ * to &bio_list, thus initialising the bio_list of new bios to be
+ * added. __generic_make_request may indeed add some more bios
+ * through a recursive call to generic_make_request. If it
+ * did, we find a non-NULL value in bio_list and re-enter the loop
+ * from the top. In this case we really did just take the bio
+ * of the top of the list (no pretending) and so fixup bio_list and
+ * bio_tail or bi_next, and call into __generic_make_request again.
+ *
+ * The loop was structured like this to make only one call to
+ * __generic_make_request (which is important as it is large and
+ * inlined) and to keep the structure simple.
+ */
+ BUG_ON(bio->bi_next);
+ do {
+ current->bio_list = bio->bi_next;
+ if (bio->bi_next == NULL)
+ current->bio_tail = ¤t->bio_list;
+ else
+ bio->bi_next = NULL;
+ __generic_make_request(bio);
+ bio = current->bio_list;
+ } while (bio);
+ current->bio_tail = NULL; /* deactivate */
+}
+
EXPORT_SYMBOL(generic_make_request);
/**
* If a CPU goes away, splice its entries to the current CPU
* and trigger a run of the softirq
*/
- if (action == CPU_DEAD) {
+ if (action == CPU_DEAD || action == CPU_DEAD_FROZEN) {
int cpu = (unsigned long) hcpu;
local_irq_disable();
EXPORT_SYMBOL(kblockd_schedule_work);
-void kblockd_flush(void)
+void kblockd_flush_work(struct work_struct *work)
{
- flush_workqueue(kblockd_workqueue);
+ cancel_work_sync(work);
}
-EXPORT_SYMBOL(kblockd_flush);
+EXPORT_SYMBOL(kblockd_flush_work);
int __init blk_dev_init(void)
{
panic("Failed to create kblockd\n");
request_cachep = kmem_cache_create("blkdev_requests",
- sizeof(struct request), 0, SLAB_PANIC, NULL, NULL);
+ sizeof(struct request), 0, SLAB_PANIC, NULL);
requestq_cachep = kmem_cache_create("blkdev_queue",
- sizeof(request_queue_t), 0, SLAB_PANIC, NULL, NULL);
+ sizeof(request_queue_t), 0, SLAB_PANIC, NULL);
iocontext_cachep = kmem_cache_create("blkdev_ioc",
- sizeof(struct io_context), 0, SLAB_PANIC, NULL, NULL);
+ sizeof(struct io_context), 0, SLAB_PANIC, NULL);
for_each_possible_cpu(i)
INIT_LIST_HEAD(&per_cpu(blk_cpu_done, i));
open_softirq(BLOCK_SOFTIRQ, blk_done_softirq, NULL);
register_hotcpu_notifier(&blk_cpu_notifier);
- blk_max_low_pfn = max_low_pfn;
- blk_max_pfn = max_pfn;
+ blk_max_low_pfn = max_low_pfn - 1;
+ blk_max_pfn = max_pfn - 1;
return 0;
}
ret->nr_batch_requests = 0; /* because this is 0 */
ret->aic = NULL;
ret->cic_root.rb_node = NULL;
+ ret->ioc_data = NULL;
/* make sure set_task_ioprio() sees the settings above */
smp_wmb();
tsk->io_context = ret;
return ret;
}
-EXPORT_SYMBOL(current_io_context);
/*
* If the current task has no IO context then create one and initialise it.
git://ftp.safe.ca / safe / jmp / linux-2.6 / blobdiff