Fix up incorrect "unlikely()" on %gs reload in x86 __switch_to

[safe/jmp/linux-2.6] / arch / i386 / kernel / process.c

diff --git a/arch/i386/kernel/process.c b/arch/i386/kernel/process.c
index be3efba..d949205 100644 (file)
--- a/arch/i386/kernel/process.c
+++ b/arch/i386/kernel/process.c
@@ -13,6 +13,7 @@
 
 #include <stdarg.h>
 
+#include <linux/cpu.h>
 #include <linux/errno.h>
 #include <linux/sched.h>
 #include <linux/fs.h>
@@ -37,6 +38,7 @@
 #include <linux/kallsyms.h>
 #include <linux/ptrace.h>
 #include <linux/random.h>
+#include <linux/kprobes.h>
 
 #include <asm/uaccess.h>
 #include <asm/pgtable.h>
@@ -54,6 +56,9 @@
 #include <linux/irq.h>
 #include <linux/err.h>
 
+#include <asm/tlbflush.h>
+#include <asm/cpu.h>
+
 asmlinkage void ret_from_fork(void) __asm__("ret_from_fork");
 
 static int hlt_counter;
@@ -142,14 +147,42 @@ static void poll_idle (void)
        }
 }
 
+#ifdef CONFIG_HOTPLUG_CPU
+#include <asm/nmi.h>
+/* We don't actually take CPU down, just spin without interrupts. */
+static inline void play_dead(void)
+{
+       /* This must be done before dead CPU ack */
+       cpu_exit_clear();
+       wbinvd();
+       mb();
+       /* Ack it */
+       __get_cpu_var(cpu_state) = CPU_DEAD;
+
+       /*
+        * With physical CPU hotplug, we should halt the cpu
+        */
+       local_irq_disable();
+       while (1)
+               __asm__ __volatile__("hlt":::"memory");
+}
+#else
+static inline void play_dead(void)
+{
+       BUG();
+}
+#endif /* CONFIG_HOTPLUG_CPU */
+
 /*
  * The idle thread. There's no useful work to be
  * done, so just try to conserve power and have a
  * low exit latency (ie sit in a loop waiting for
  * somebody to say that they'd like to reschedule)
  */
-void cpu_idle (void)
+void cpu_idle(void)
 {
+       int cpu = raw_smp_processor_id();
+
        /* endless idle loop with no priority at all */
        while (1) {
                while (!need_resched()) {
@@ -164,6 +197,9 @@ void cpu_idle (void)
                        if (!idle)
                                idle = default_idle;
 
+                       if (cpu_is_offline(cpu))
+                               play_dead();
+
                        __get_cpu_var(irq_stat).idle_timestamp = jiffies;
                        idle();
                }
@@ -222,7 +258,7 @@ static void mwait_idle(void)
        }
 }
 
-void __init select_idle_routine(const struct cpuinfo_x86 *c)
+void __devinit select_idle_routine(const struct cpuinfo_x86 *c)
 {
        if (cpu_has(c, X86_FEATURE_MWAIT)) {
                printk("monitor/mwait feature present.\n");
@@ -339,6 +375,13 @@ void exit_thread(void)
        struct task_struct *tsk = current;
        struct thread_struct *t = &tsk->thread;
 
+       /*
+        * Remove function-return probe instances associated with this task
+        * and put them back on the free list. Do not insert an exit probe for
+        * this function, it will be disabled by kprobe_flush_task if you do.
+        */
+       kprobe_flush_task(tsk);
+
        /* The process may have allocated an io port bitmap... nuke it. */
        if (unlikely(NULL != t->io_bitmap_ptr)) {
                int cpu = get_cpu();
@@ -362,6 +405,13 @@ void flush_thread(void)
 {
        struct task_struct *tsk = current;
 
+       /*
+        * Remove function-return probe instances associated with this task
+        * and put them back on the free list. Do not insert an exit probe for
+        * this function, it will be disabled by kprobe_flush_task if you do.
+        */
+       kprobe_flush_task(tsk);
+
        memset(tsk->thread.debugreg, 0, sizeof(unsigned long)*8);
        memset(tsk->thread.tls_array, 0, sizeof(tsk->thread.tls_array));        
        /*
@@ -567,6 +617,33 @@ handle_io_bitmap(struct thread_struct *next, struct tss_struct *tss)
 }
 
 /*
+ * This function selects if the context switch from prev to next
+ * has to tweak the TSC disable bit in the cr4.
+ */
+static inline void disable_tsc(struct task_struct *prev_p,
+                              struct task_struct *next_p)
+{
+       struct thread_info *prev, *next;
+
+       /*
+        * gcc should eliminate the ->thread_info dereference if
+        * has_secure_computing returns 0 at compile time (SECCOMP=n).
+        */
+       prev = prev_p->thread_info;
+       next = next_p->thread_info;
+
+       if (has_secure_computing(prev) || has_secure_computing(next)) {
+               /* slow path here */
+               if (has_secure_computing(prev) &&
+                   !has_secure_computing(next)) {
+                       write_cr4(read_cr4() & ~X86_CR4_TSD);
+               } else if (!has_secure_computing(prev) &&
+                          has_secure_computing(next))
+                       write_cr4(read_cr4() | X86_CR4_TSD);
+       }
+}
+
+/*
  *     switch_to(x,yn) should switch tasks from x to y.
  *
  * We fsave/fwait so that an exception goes off at the right time
@@ -623,28 +700,34 @@ struct task_struct fastcall * __switch_to(struct task_struct *prev_p, struct tas
 
        /*
         * Restore %fs and %gs if needed.
+        *
+        * Glibc normally makes %fs be zero, and %gs is one of
+        * the TLS segments.
         */
-       if (unlikely(prev->fs | prev->gs | next->fs | next->gs)) {
+       if (unlikely(prev->fs | next->fs))
                loadsegment(fs, next->fs);
+
+       if (prev->gs | next->gs)
                loadsegment(gs, next->gs);
-       }
 
        /*
         * Now maybe reload the debug registers
         */
        if (unlikely(next->debugreg[7])) {
-               set_debugreg(current->thread.debugreg[0], 0);
-               set_debugreg(current->thread.debugreg[1], 1);
-               set_debugreg(current->thread.debugreg[2], 2);
-               set_debugreg(current->thread.debugreg[3], 3);
+               set_debugreg(next->debugreg[0], 0);
+               set_debugreg(next->debugreg[1], 1);
+               set_debugreg(next->debugreg[2], 2);
+               set_debugreg(next->debugreg[3], 3);
                /* no 4 and 5 */
-               set_debugreg(current->thread.debugreg[6], 6);
-               set_debugreg(current->thread.debugreg[7], 7);
+               set_debugreg(next->debugreg[6], 6);
+               set_debugreg(next->debugreg[7], 7);
        }
 
        if (unlikely(prev->io_bitmap_ptr || next->io_bitmap_ptr))
                handle_io_bitmap(next, tss);
 
+       disable_tsc(prev_p, next_p);
+
        return prev_p;
 }