Merge branch 'master' into for-2.6.35

[safe/jmp/linux-2.6] / block / cfq-iosched.c

diff --git a/block/cfq-iosched.c b/block/cfq-iosched.c
index 838834b..9e0df2b 100644 (file)
--- a/block/cfq-iosched.c
+++ b/block/cfq-iosched.c
@@ -143,8 +143,6 @@ struct cfq_queue {
        struct cfq_queue *new_cfqq;
        struct cfq_group *cfqg;
        struct cfq_group *orig_cfqg;
-       /* Sectors dispatched in current dispatch round */
-       unsigned long nr_sectors;
 };
 
 /*
@@ -858,7 +856,7 @@ cfq_group_service_tree_del(struct cfq_data *cfqd, struct cfq_group *cfqg)
        if (!RB_EMPTY_NODE(&cfqg->rb_node))
                cfq_rb_erase(&cfqg->rb_node, st);
        cfqg->saved_workload_slice = 0;
-       blkiocg_update_blkio_group_dequeue_stats(&cfqg->blkg, 1);
+       blkiocg_update_dequeue_stats(&cfqg->blkg, 1);
 }
 
 static inline unsigned int cfq_cfqq_slice_usage(struct cfq_queue *cfqq)
@@ -884,13 +882,12 @@ static inline unsigned int cfq_cfqq_slice_usage(struct cfq_queue *cfqq)
                        slice_used = cfqq->allocated_slice;
        }
 
-       cfq_log_cfqq(cfqq->cfqd, cfqq, "sl_used=%u sect=%lu", slice_used,
-                               cfqq->nr_sectors);
+       cfq_log_cfqq(cfqq->cfqd, cfqq, "sl_used=%u", slice_used);
        return slice_used;
 }
 
 static void cfq_group_served(struct cfq_data *cfqd, struct cfq_group *cfqg,
-                               struct cfq_queue *cfqq)
+                               struct cfq_queue *cfqq, bool forced)
 {
        struct cfq_rb_root *st = &cfqd->grp_service_tree;
        unsigned int used_sl, charge_sl;
@@ -919,8 +916,8 @@ static void cfq_group_served(struct cfq_data *cfqd, struct cfq_group *cfqg,
 
        cfq_log_cfqg(cfqd, cfqg, "served: vt=%llu min_vt=%llu", cfqg->vdisktime,
                                        st->min_vdisktime);
-       blkiocg_update_blkio_group_stats(&cfqg->blkg, used_sl,
-                                               cfqq->nr_sectors);
+       blkiocg_update_timeslice_used(&cfqg->blkg, used_sl);
+       blkiocg_set_start_empty_time(&cfqg->blkg, forced);
 }
 
 #ifdef CONFIG_CFQ_GROUP_IOSCHED
@@ -961,10 +958,10 @@ cfq_find_alloc_cfqg(struct cfq_data *cfqd, struct cgroup *cgroup, int create)
        if (!cfqg)
                goto done;
 
-       cfqg->weight = blkcg->weight;
        for_each_cfqg_st(cfqg, i, j, st)
                *st = CFQ_RB_ROOT;
        RB_CLEAR_NODE(&cfqg->rb_node);
+       blkio_group_init(&cfqg->blkg);
 
        /*
         * Take the initial reference that will be released on destroy
@@ -978,6 +975,7 @@ cfq_find_alloc_cfqg(struct cfq_data *cfqd, struct cgroup *cgroup, int create)
        sscanf(dev_name(bdi->dev), "%u:%u", &major, &minor);
        blkiocg_add_blkio_group(blkcg, &cfqg->blkg, (void *)cfqd,
                                        MKDEV(major, minor));
+       cfqg->weight = blkcg_get_weight(blkcg, cfqg->blkg.dev);
 
        /* Add group on cfqd list */
        hlist_add_head(&cfqg->cfqd_node, &cfqd->cfqg_list);
@@ -1389,7 +1387,12 @@ static void cfq_reposition_rq_rb(struct cfq_queue *cfqq, struct request *rq)
 {
        elv_rb_del(&cfqq->sort_list, rq);
        cfqq->queued[rq_is_sync(rq)]--;
+       blkiocg_update_io_remove_stats(&cfqq->cfqg->blkg, rq_data_dir(rq),
+                                               rq_is_sync(rq));
        cfq_add_rq_rb(rq);
+       blkiocg_update_io_add_stats(
+                       &cfqq->cfqg->blkg, &cfqq->cfqd->serving_group->blkg,
+                       rq_data_dir(rq), rq_is_sync(rq));
 }
 
 static struct request *
@@ -1445,6 +1448,8 @@ static void cfq_remove_request(struct request *rq)
        cfq_del_rq_rb(rq);
 
        cfqq->cfqd->rq_queued--;
+       blkiocg_update_io_remove_stats(&cfqq->cfqg->blkg, rq_data_dir(rq),
+                                               rq_is_sync(rq));
        if (rq_is_meta(rq)) {
                WARN_ON(!cfqq->meta_pending);
                cfqq->meta_pending--;
@@ -1476,6 +1481,14 @@ static void cfq_merged_request(struct request_queue *q, struct request *req,
        }
 }
 
+static void cfq_bio_merged(struct request_queue *q, struct request *req,
+                               struct bio *bio)
+{
+       struct cfq_queue *cfqq = RQ_CFQQ(req);
+       blkiocg_update_io_merged_stats(&cfqq->cfqg->blkg, bio_data_dir(bio),
+                                       cfq_bio_sync(bio));
+}
+
 static void
 cfq_merged_requests(struct request_queue *q, struct request *rq,
                    struct request *next)
@@ -1493,6 +1506,8 @@ cfq_merged_requests(struct request_queue *q, struct request *rq,
        if (cfqq->next_rq == next)
                cfqq->next_rq = rq;
        cfq_remove_request(next);
+       blkiocg_update_io_merged_stats(&cfqq->cfqg->blkg, rq_data_dir(next),
+                                       rq_is_sync(next));
 }
 
 static int cfq_allow_merge(struct request_queue *q, struct request *rq,
@@ -1520,18 +1535,24 @@ static int cfq_allow_merge(struct request_queue *q, struct request *rq,
        return cfqq == RQ_CFQQ(rq);
 }
 
+static inline void cfq_del_timer(struct cfq_data *cfqd, struct cfq_queue *cfqq)
+{
+       del_timer(&cfqd->idle_slice_timer);
+       blkiocg_update_idle_time_stats(&cfqq->cfqg->blkg);
+}
+
 static void __cfq_set_active_queue(struct cfq_data *cfqd,
                                   struct cfq_queue *cfqq)
 {
        if (cfqq) {
                cfq_log_cfqq(cfqd, cfqq, "set_active wl_prio:%d wl_type:%d",
                                cfqd->serving_prio, cfqd->serving_type);
+               blkiocg_update_avg_queue_size_stats(&cfqq->cfqg->blkg);
                cfqq->slice_start = 0;
                cfqq->dispatch_start = jiffies;
                cfqq->allocated_slice = 0;
                cfqq->slice_end = 0;
                cfqq->slice_dispatch = 0;
-               cfqq->nr_sectors = 0;
 
                cfq_clear_cfqq_wait_request(cfqq);
                cfq_clear_cfqq_must_dispatch(cfqq);
@@ -1539,7 +1560,7 @@ static void __cfq_set_active_queue(struct cfq_data *cfqd,
                cfq_clear_cfqq_fifo_expire(cfqq);
                cfq_mark_cfqq_slice_new(cfqq);
 
-               del_timer(&cfqd->idle_slice_timer);
+               cfq_del_timer(cfqd, cfqq);
        }
 
        cfqd->active_queue = cfqq;
@@ -1550,12 +1571,12 @@ static void __cfq_set_active_queue(struct cfq_data *cfqd,
  */
 static void
 __cfq_slice_expired(struct cfq_data *cfqd, struct cfq_queue *cfqq,
-                   bool timed_out)
+                   bool timed_out, bool forced)
 {
        cfq_log_cfqq(cfqd, cfqq, "slice expired t=%d", timed_out);
 
        if (cfq_cfqq_wait_request(cfqq))
-               del_timer(&cfqd->idle_slice_timer);
+               cfq_del_timer(cfqd, cfqq);
 
        cfq_clear_cfqq_wait_request(cfqq);
        cfq_clear_cfqq_wait_busy(cfqq);
@@ -1577,7 +1598,7 @@ __cfq_slice_expired(struct cfq_data *cfqd, struct cfq_queue *cfqq,
                cfq_log_cfqq(cfqd, cfqq, "resid=%ld", cfqq->slice_resid);
        }
 
-       cfq_group_served(cfqd, cfqq->cfqg, cfqq);
+       cfq_group_served(cfqd, cfqq->cfqg, cfqq, forced);
 
        if (cfq_cfqq_on_rr(cfqq) && RB_EMPTY_ROOT(&cfqq->sort_list))
                cfq_del_cfqq_rr(cfqd, cfqq);
@@ -1596,12 +1617,13 @@ __cfq_slice_expired(struct cfq_data *cfqd, struct cfq_queue *cfqq,
        }
 }
 
-static inline void cfq_slice_expired(struct cfq_data *cfqd, bool timed_out)
+static inline void cfq_slice_expired(struct cfq_data *cfqd, bool timed_out,
+                                       bool forced)
 {
        struct cfq_queue *cfqq = cfqd->active_queue;
 
        if (cfqq)
-               __cfq_slice_expired(cfqd, cfqq, timed_out);
+               __cfq_slice_expired(cfqd, cfqq, timed_out, forced);
 }
 
 /*
@@ -1857,6 +1879,7 @@ static void cfq_arm_slice_timer(struct cfq_data *cfqd)
        sl = cfqd->cfq_slice_idle;
 
        mod_timer(&cfqd->idle_slice_timer, jiffies + sl);
+       blkiocg_update_set_idle_time_stats(&cfqq->cfqg->blkg);
        cfq_log_cfqq(cfqd, cfqq, "arm_idle: %lu", sl);
 }
 
@@ -1876,7 +1899,8 @@ static void cfq_dispatch_insert(struct request_queue *q, struct request *rq)
        elv_dispatch_sort(q, rq);
 
        cfqd->rq_in_flight[cfq_cfqq_sync(cfqq)]++;
-       cfqq->nr_sectors += blk_rq_sectors(rq);
+       blkiocg_update_dispatch_stats(&cfqq->cfqg->blkg, blk_rq_bytes(rq),
+                                       rq_data_dir(rq), rq_is_sync(rq));
 }
 
 /*
@@ -2167,7 +2191,7 @@ static struct cfq_queue *cfq_select_queue(struct cfq_data *cfqd)
        }
 
 expire:
-       cfq_slice_expired(cfqd, 0);
+       cfq_slice_expired(cfqd, 0, false);
 new_queue:
        /*
         * Current queue expired. Check if we have to switch to a new
@@ -2193,7 +2217,7 @@ static int __cfq_forced_dispatch_cfqq(struct cfq_queue *cfqq)
        BUG_ON(!list_empty(&cfqq->fifo));
 
        /* By default cfqq is not expired if it is empty. Do it explicitly */
-       __cfq_slice_expired(cfqq->cfqd, cfqq, 0);
+       __cfq_slice_expired(cfqq->cfqd, cfqq, 0, true);
        return dispatched;
 }
 
@@ -2376,7 +2400,7 @@ static int cfq_dispatch_requests(struct request_queue *q, int force)
            cfqq->slice_dispatch >= cfq_prio_to_maxrq(cfqd, cfqq)) ||
            cfq_class_idle(cfqq))) {
                cfqq->slice_end = jiffies + 1;
-               cfq_slice_expired(cfqd, 0);
+               cfq_slice_expired(cfqd, 0, false);
        }
 
        cfq_log_cfqq(cfqd, cfqq, "dispatched a request");
@@ -2407,7 +2431,7 @@ static void cfq_put_queue(struct cfq_queue *cfqq)
        orig_cfqg = cfqq->orig_cfqg;
 
        if (unlikely(cfqd->active_queue == cfqq)) {
-               __cfq_slice_expired(cfqd, cfqq, 0);
+               __cfq_slice_expired(cfqd, cfqq, 0, false);
                cfq_schedule_dispatch(cfqd);
        }
 
@@ -2508,7 +2532,7 @@ static void cfq_exit_cfqq(struct cfq_data *cfqd, struct cfq_queue *cfqq)
        struct cfq_queue *__cfqq, *next;
 
        if (unlikely(cfqq == cfqd->active_queue)) {
-               __cfq_slice_expired(cfqd, cfqq, 0);
+               __cfq_slice_expired(cfqd, cfqq, 0, false);
                cfq_schedule_dispatch(cfqd);
        }
 
@@ -3137,7 +3161,7 @@ cfq_should_preempt(struct cfq_data *cfqd, struct cfq_queue *new_cfqq,
 static void cfq_preempt_queue(struct cfq_data *cfqd, struct cfq_queue *cfqq)
 {
        cfq_log_cfqq(cfqd, cfqq, "preempt");
-       cfq_slice_expired(cfqd, 1);
+       cfq_slice_expired(cfqd, 1, false);
 
        /*
         * Put the new queue at the front of the of the current list,
@@ -3185,11 +3209,14 @@ cfq_rq_enqueued(struct cfq_data *cfqd, struct cfq_queue *cfqq,
                if (cfq_cfqq_wait_request(cfqq)) {
                        if (blk_rq_bytes(rq) > PAGE_CACHE_SIZE ||
                            cfqd->busy_queues > 1) {
-                               del_timer(&cfqd->idle_slice_timer);
+                               cfq_del_timer(cfqd, cfqq);
                                cfq_clear_cfqq_wait_request(cfqq);
                                __blk_run_queue(cfqd->queue);
-                       } else
+                       } else {
+                               blkiocg_update_idle_time_stats(
+                                               &cfqq->cfqg->blkg);
                                cfq_mark_cfqq_must_dispatch(cfqq);
+                       }
                }
        } else if (cfq_should_preempt(cfqd, cfqq, rq)) {
                /*
@@ -3215,6 +3242,9 @@ static void cfq_insert_request(struct request_queue *q, struct request *rq)
        list_add_tail(&rq->queuelist, &cfqq->fifo);
        cfq_add_rq_rb(rq);
 
+       blkiocg_update_io_add_stats(&cfqq->cfqg->blkg,
+                       &cfqd->serving_group->blkg, rq_data_dir(rq),
+                       rq_is_sync(rq));
        cfq_rq_enqueued(cfqd, cfqq, rq);
 }
 
@@ -3300,6 +3330,9 @@ static void cfq_completed_request(struct request_queue *q, struct request *rq)
        WARN_ON(!cfqq->dispatched);
        cfqd->rq_in_driver--;
        cfqq->dispatched--;
+       blkiocg_update_completion_stats(&cfqq->cfqg->blkg, rq_start_time_ns(rq),
+                       rq_io_start_time_ns(rq), rq_data_dir(rq),
+                       rq_is_sync(rq));
 
        cfqd->rq_in_flight[cfq_cfqq_sync(cfqq)]--;
 
@@ -3340,7 +3373,7 @@ static void cfq_completed_request(struct request_queue *q, struct request *rq)
                 * - when there is a close cooperator
                 */
                if (cfq_slice_used(cfqq) || cfq_class_idle(cfqq))
-                       cfq_slice_expired(cfqd, 1);
+                       cfq_slice_expired(cfqd, 1, false);
                else if (sync && cfqq_empty &&
                         !cfq_close_cooperator(cfqd, cfqq)) {
                        cfqd->noidle_tree_requires_idle |= !rq_noidle(rq);
@@ -3600,7 +3633,7 @@ static void cfq_idle_slice_timer(unsigned long data)
                cfq_clear_cfqq_deep(cfqq);
        }
 expire:
-       cfq_slice_expired(cfqd, timed_out);
+       cfq_slice_expired(cfqd, timed_out, false);
 out_kick:
        cfq_schedule_dispatch(cfqd);
 out_cont:
@@ -3643,7 +3676,7 @@ static void cfq_exit_queue(struct elevator_queue *e)
        spin_lock_irq(q->queue_lock);
 
        if (cfqd->active_queue)
-               __cfq_slice_expired(cfqd, cfqd->active_queue, 0);
+               __cfq_slice_expired(cfqd, cfqd->active_queue, 0, false);
 
        while (!list_empty(&cfqd->cic_list)) {
                struct cfq_io_context *cic = list_entry(cfqd->cic_list.next,
@@ -3870,6 +3903,7 @@ static struct elevator_type iosched_cfq = {
                .elevator_merged_fn =           cfq_merged_request,
                .elevator_merge_req_fn =        cfq_merged_requests,
                .elevator_allow_merge_fn =      cfq_allow_merge,
+               .elevator_bio_merged_fn =       cfq_bio_merged,
                .elevator_dispatch_fn =         cfq_dispatch_requests,
                .elevator_add_req_fn =          cfq_insert_request,
                .elevator_activate_req_fn =     cfq_activate_request,