cfq-iosched: improve sync vs async workloads

[safe/jmp/linux-2.6] / block / cfq-iosched.c

diff --git a/block/cfq-iosched.c b/block/cfq-iosched.c
index a843704..a8237be 100644 (file)
--- a/block/cfq-iosched.c
+++ b/block/cfq-iosched.c
@@ -96,6 +96,7 @@ struct cfq_data {
        struct hlist_head *cfq_hash;
 
        int rq_in_driver;
+       int sync_flight;
        int hw_tag;
 
        /*
@@ -106,7 +107,6 @@ struct cfq_data {
 
        struct cfq_queue *active_queue;
        struct cfq_io_context *active_cic;
-       unsigned int dispatch_slice;
 
        struct timer_list idle_class_timer;
 
@@ -752,7 +752,7 @@ __cfq_set_active_queue(struct cfq_data *cfqd, struct cfq_queue *cfqq)
  */
 static void
 __cfq_slice_expired(struct cfq_data *cfqd, struct cfq_queue *cfqq,
-                   int preempted, int timed_out)
+                   int timed_out)
 {
        if (cfq_cfqq_wait_request(cfqq))
                del_timer(&cfqd->idle_slice_timer);
@@ -761,8 +761,7 @@ __cfq_slice_expired(struct cfq_data *cfqd, struct cfq_queue *cfqq,
        cfq_clear_cfqq_wait_request(cfqq);
 
        /*
-        * store what was left of this slice, if the queue idled out
-        * or was preempted
+        * store what was left of this slice, if the queue idled/timed out
         */
        if (timed_out && !cfq_cfqq_slice_new(cfqq))
                cfqq->slice_resid = cfqq->slice_end - jiffies;
@@ -776,17 +775,14 @@ __cfq_slice_expired(struct cfq_data *cfqd, struct cfq_queue *cfqq,
                put_io_context(cfqd->active_cic->ioc);
                cfqd->active_cic = NULL;
        }
-
-       cfqd->dispatch_slice = 0;
 }
 
-static inline void cfq_slice_expired(struct cfq_data *cfqd, int preempted,
-                                    int timed_out)
+static inline void cfq_slice_expired(struct cfq_data *cfqd, int timed_out)
 {
        struct cfq_queue *cfqq = cfqd->active_queue;
 
        if (cfqq)
-               __cfq_slice_expired(cfqd, cfqq, preempted, timed_out);
+               __cfq_slice_expired(cfqd, cfqq, timed_out);
 }
 
 /*
@@ -916,11 +912,15 @@ static void cfq_arm_slice_timer(struct cfq_data *cfqd)
  */
 static void cfq_dispatch_insert(request_queue_t *q, struct request *rq)
 {
+       struct cfq_data *cfqd = q->elevator->elevator_data;
        struct cfq_queue *cfqq = RQ_CFQQ(rq);
 
        cfq_remove_request(rq);
        cfqq->dispatched++;
        elv_dispatch_sort(q, rq);
+
+       if (cfq_cfqq_sync(cfqq))
+               cfqd->sync_flight++;
 }
 
 /*
@@ -995,7 +995,7 @@ static struct cfq_queue *cfq_select_queue(struct cfq_data *cfqd)
        }
 
 expire:
-       cfq_slice_expired(cfqd, 0, 0);
+       cfq_slice_expired(cfqd, 0);
 new_queue:
        cfqq = cfq_set_active_queue(cfqd);
 keep_queue:
@@ -1028,7 +1028,6 @@ __cfq_dispatch_requests(struct cfq_data *cfqd, struct cfq_queue *cfqq,
                 */
                cfq_dispatch_insert(cfqd->queue, rq);
 
-               cfqd->dispatch_slice++;
                dispatched++;
 
                if (!cfqd->active_cic) {
@@ -1046,10 +1045,10 @@ __cfq_dispatch_requests(struct cfq_data *cfqd, struct cfq_queue *cfqq,
         * queue always expire after 1 dispatch round.
         */
        if (cfqd->busy_queues > 1 && ((!cfq_cfqq_sync(cfqq) &&
-           cfqd->dispatch_slice >= cfq_prio_to_maxrq(cfqd, cfqq)) ||
+           dispatched >= cfq_prio_to_maxrq(cfqd, cfqq)) ||
            cfq_class_idle(cfqq))) {
                cfqq->slice_end = jiffies + 1;
-               cfq_slice_expired(cfqd, 0, 0);
+               cfq_slice_expired(cfqd, 0);
        }
 
        return dispatched;
@@ -1083,7 +1082,7 @@ static int cfq_forced_dispatch(struct cfq_data *cfqd)
                dispatched += __cfq_forced_dispatch_cfqq(cfqq);
        }
 
-       cfq_slice_expired(cfqd, 0, 0);
+       cfq_slice_expired(cfqd, 0);
 
        BUG_ON(cfqd->busy_queues);
 
@@ -1106,27 +1105,24 @@ static int cfq_dispatch_requests(request_queue_t *q, int force)
        while ((cfqq = cfq_select_queue(cfqd)) != NULL) {
                int max_dispatch;
 
-               if (cfqd->busy_queues > 1) {
-                       /*
-                        * So we have dispatched before in this round, if the
-                        * next queue has idling enabled (must be sync), don't
-                        * allow it service until the previous have completed.
-                        */
-                       if (cfqd->rq_in_driver && cfq_cfqq_idle_window(cfqq) &&
-                           dispatched)
+               max_dispatch = cfqd->cfq_quantum;
+               if (cfq_class_idle(cfqq))
+                       max_dispatch = 1;
+
+               if (cfqq->dispatched >= max_dispatch) {
+                       if (cfqd->busy_queues > 1)
                                break;
-                       if (cfqq->dispatched >= cfqd->cfq_quantum)
+                       if (cfqq->dispatched >= 4 * max_dispatch)
                                break;
                }
 
+               if (cfqd->sync_flight && !cfq_cfqq_sync(cfqq))
+                       break;
+
                cfq_clear_cfqq_must_dispatch(cfqq);
                cfq_clear_cfqq_wait_request(cfqq);
                del_timer(&cfqd->idle_slice_timer);
 
-               max_dispatch = cfqd->cfq_quantum;
-               if (cfq_class_idle(cfqq))
-                       max_dispatch = 1;
-
                dispatched += __cfq_dispatch_requests(cfqd, cfqq, max_dispatch);
        }
 
@@ -1153,7 +1149,7 @@ static void cfq_put_queue(struct cfq_queue *cfqq)
        BUG_ON(cfq_cfqq_on_rr(cfqq));
 
        if (unlikely(cfqd->active_queue == cfqq)) {
-               __cfq_slice_expired(cfqd, cfqq, 0, 0);
+               __cfq_slice_expired(cfqd, cfqq, 0);
                cfq_schedule_dispatch(cfqd);
        }
 
@@ -1210,7 +1206,7 @@ static void cfq_free_io_context(struct io_context *ioc)
 static void cfq_exit_cfqq(struct cfq_data *cfqd, struct cfq_queue *cfqq)
 {
        if (unlikely(cfqq == cfqd->active_queue)) {
-               __cfq_slice_expired(cfqd, cfqq, 0, 0);
+               __cfq_slice_expired(cfqd, cfqq, 0);
                cfq_schedule_dispatch(cfqd);
        }
 
@@ -1609,7 +1605,12 @@ static void
 cfq_update_idle_window(struct cfq_data *cfqd, struct cfq_queue *cfqq,
                       struct cfq_io_context *cic)
 {
-       int enable_idle = cfq_cfqq_idle_window(cfqq);
+       int enable_idle;
+
+       if (!cfq_cfqq_sync(cfqq))
+               return;
+
+       enable_idle = cfq_cfqq_idle_window(cfqq);
 
        if (!cic->ioc->task || !cfqd->cfq_slice_idle ||
            (cfqd->hw_tag && CIC_SEEKY(cic)))
@@ -1683,7 +1684,7 @@ cfq_should_preempt(struct cfq_data *cfqd, struct cfq_queue *new_cfqq,
  */
 static void cfq_preempt_queue(struct cfq_data *cfqd, struct cfq_queue *cfqq)
 {
-       cfq_slice_expired(cfqd, 1, 1);
+       cfq_slice_expired(cfqd, 1);
 
        /*
         * Put the new queue at the front of the of the current list,
@@ -1768,6 +1769,9 @@ static void cfq_completed_request(request_queue_t *q, struct request *rq)
        cfqd->rq_in_driver--;
        cfqq->dispatched--;
 
+       if (cfq_cfqq_sync(cfqq))
+               cfqd->sync_flight--;
+
        if (!cfq_class_idle(cfqq))
                cfqd->last_end_request = now;
 
@@ -1784,7 +1788,7 @@ static void cfq_completed_request(request_queue_t *q, struct request *rq)
                        cfq_clear_cfqq_slice_new(cfqq);
                }
                if (cfq_slice_used(cfqq))
-                       cfq_slice_expired(cfqd, 0, 1);
+                       cfq_slice_expired(cfqd, 1);
                else if (sync && RB_EMPTY_ROOT(&cfqq->sort_list))
                        cfq_arm_slice_timer(cfqd);
        }
@@ -1979,7 +1983,7 @@ static void cfq_idle_slice_timer(unsigned long data)
                }
        }
 expire:
-       cfq_slice_expired(cfqd, 0, timed_out);
+       cfq_slice_expired(cfqd, timed_out);
 out_kick:
        cfq_schedule_dispatch(cfqd);
 out_cont:
@@ -2025,7 +2029,7 @@ static void cfq_exit_queue(elevator_t *e)
        spin_lock_irq(q->queue_lock);
 
        if (cfqd->active_queue)
-               __cfq_slice_expired(cfqd, cfqd->active_queue, 0, 0);
+               __cfq_slice_expired(cfqd, cfqd->active_queue, 0);
 
        while (!list_empty(&cfqd->cic_list)) {
                struct cfq_io_context *cic = list_entry(cfqd->cic_list.next,