X-Git-Url: http://ftp.safe.ca/?a=blobdiff_plain;f=arch%2Fpowerpc%2Fkernel%2Fsmp.c;h=9b86a74d281504cf255ae9a2f2b3e7b5009076fb;hb=c8a79c3da1ee5dcdbf6db829554fe3d9e68a88ab;hp=46c56cfd1b2f9685aa69a2b4cbc879009b48c6e2;hpb=6ab3d5624e172c553004ecc862bfeac16d9d68b7;p=safe%2Fjmp%2Flinux-2.6 diff --git a/arch/powerpc/kernel/smp.c b/arch/powerpc/kernel/smp.c index 46c56cf..9b86a74 100644 --- a/arch/powerpc/kernel/smp.c +++ b/arch/powerpc/kernel/smp.c @@ -41,6 +41,7 @@ #include #include #include +#include #include #include #include @@ -56,50 +57,24 @@ #define DBG(fmt...) #endif -int smp_hw_index[NR_CPUS]; struct thread_info *secondary_ti; -cpumask_t cpu_possible_map = CPU_MASK_NONE; -cpumask_t cpu_online_map = CPU_MASK_NONE; -cpumask_t cpu_sibling_map[NR_CPUS] = { [0 ... NR_CPUS-1] = CPU_MASK_NONE }; +DEFINE_PER_CPU(cpumask_t, cpu_sibling_map) = CPU_MASK_NONE; +DEFINE_PER_CPU(cpumask_t, cpu_core_map) = CPU_MASK_NONE; -EXPORT_SYMBOL(cpu_online_map); -EXPORT_SYMBOL(cpu_possible_map); +EXPORT_PER_CPU_SYMBOL(cpu_sibling_map); +EXPORT_PER_CPU_SYMBOL(cpu_core_map); /* SMP operations for this machine */ struct smp_ops_t *smp_ops; -static volatile unsigned int cpu_callin_map[NR_CPUS]; - -void smp_call_function_interrupt(void); +/* Can't be static due to PowerMac hackery */ +volatile unsigned int cpu_callin_map[NR_CPUS]; int smt_enabled_at_boot = 1; static void (*crash_ipi_function_ptr)(struct pt_regs *) = NULL; -#ifdef CONFIG_MPIC -int __init smp_mpic_probe(void) -{ - int nr_cpus; - - DBG("smp_mpic_probe()...\n"); - - nr_cpus = cpus_weight(cpu_possible_map); - - DBG("nr_cpus: %d\n", nr_cpus); - - if (nr_cpus > 1) - mpic_request_ipis(); - - return nr_cpus; -} - -void __devinit smp_mpic_setup_cpu(int cpu) -{ - mpic_setup_this_cpu(); -} -#endif /* CONFIG_MPIC */ - #ifdef CONFIG_PPC64 void __devinit smp_generic_kick_cpu(int nr) { @@ -115,23 +90,25 @@ void __devinit smp_generic_kick_cpu(int nr) } #endif -void smp_message_recv(int msg, struct pt_regs *regs) +void smp_message_recv(int msg) { switch(msg) { case PPC_MSG_CALL_FUNCTION: - smp_call_function_interrupt(); + generic_smp_call_function_interrupt(); break; case PPC_MSG_RESCHEDULE: - /* XXX Do we have to do this? */ - set_need_resched(); + /* we notice need_resched on exit */ + break; + case PPC_MSG_CALL_FUNC_SINGLE: + generic_smp_call_function_single_interrupt(); break; case PPC_MSG_DEBUGGER_BREAK: if (crash_ipi_function_ptr) { - crash_ipi_function_ptr(regs); + crash_ipi_function_ptr(get_irq_regs()); break; } #ifdef CONFIG_DEBUGGER - debugger_ipi(regs); + debugger_ipi(get_irq_regs()); break; #endif /* CONFIG_DEBUGGER */ /* FALLTHROUGH */ @@ -142,15 +119,89 @@ void smp_message_recv(int msg, struct pt_regs *regs) } } +static irqreturn_t call_function_action(int irq, void *data) +{ + generic_smp_call_function_interrupt(); + return IRQ_HANDLED; +} + +static irqreturn_t reschedule_action(int irq, void *data) +{ + /* we just need the return path side effect of checking need_resched */ + return IRQ_HANDLED; +} + +static irqreturn_t call_function_single_action(int irq, void *data) +{ + generic_smp_call_function_single_interrupt(); + return IRQ_HANDLED; +} + +static irqreturn_t debug_ipi_action(int irq, void *data) +{ + smp_message_recv(PPC_MSG_DEBUGGER_BREAK); + return IRQ_HANDLED; +} + +static irq_handler_t smp_ipi_action[] = { + [PPC_MSG_CALL_FUNCTION] = call_function_action, + [PPC_MSG_RESCHEDULE] = reschedule_action, + [PPC_MSG_CALL_FUNC_SINGLE] = call_function_single_action, + [PPC_MSG_DEBUGGER_BREAK] = debug_ipi_action, +}; + +const char *smp_ipi_name[] = { + [PPC_MSG_CALL_FUNCTION] = "ipi call function", + [PPC_MSG_RESCHEDULE] = "ipi reschedule", + [PPC_MSG_CALL_FUNC_SINGLE] = "ipi call function single", + [PPC_MSG_DEBUGGER_BREAK] = "ipi debugger", +}; + +/* optional function to request ipi, for controllers with >= 4 ipis */ +int smp_request_message_ipi(int virq, int msg) +{ + int err; + + if (msg < 0 || msg > PPC_MSG_DEBUGGER_BREAK) { + return -EINVAL; + } +#if !defined(CONFIG_DEBUGGER) && !defined(CONFIG_KEXEC) + if (msg == PPC_MSG_DEBUGGER_BREAK) { + return 1; + } +#endif + err = request_irq(virq, smp_ipi_action[msg], IRQF_DISABLED|IRQF_PERCPU, + smp_ipi_name[msg], 0); + WARN(err < 0, "unable to request_irq %d for %s (rc %d)\n", + virq, smp_ipi_name[msg], err); + + return err; +} + void smp_send_reschedule(int cpu) { - smp_ops->message_pass(cpu, PPC_MSG_RESCHEDULE); + if (likely(smp_ops)) + smp_ops->message_pass(cpu, PPC_MSG_RESCHEDULE); +} + +void arch_send_call_function_single_ipi(int cpu) +{ + smp_ops->message_pass(cpu, PPC_MSG_CALL_FUNC_SINGLE); +} + +void arch_send_call_function_ipi_mask(const struct cpumask *mask) +{ + unsigned int cpu; + + for_each_cpu(cpu, mask) + smp_ops->message_pass(cpu, PPC_MSG_CALL_FUNCTION); } #ifdef CONFIG_DEBUGGER void smp_send_debugger_break(int cpu) { - smp_ops->message_pass(cpu, PPC_MSG_DEBUGGER_BREAK); + if (likely(smp_ops)) + smp_ops->message_pass(cpu, PPC_MSG_DEBUGGER_BREAK); } #endif @@ -158,7 +209,7 @@ void smp_send_debugger_break(int cpu) void crash_send_ipi(void (*crash_ipi_callback)(struct pt_regs *)) { crash_ipi_function_ptr = crash_ipi_callback; - if (crash_ipi_callback) { + if (crash_ipi_callback && smp_ops) { mb(); smp_ops->message_pass(MSG_ALL_BUT_SELF, PPC_MSG_DEBUGGER_BREAK); } @@ -174,153 +225,11 @@ static void stop_this_cpu(void *dummy) void smp_send_stop(void) { - smp_call_function(stop_this_cpu, NULL, 1, 0); -} - -/* - * Structure and data for smp_call_function(). This is designed to minimise - * static memory requirements. It also looks cleaner. - * Stolen from the i386 version. - */ -static __cacheline_aligned_in_smp DEFINE_SPINLOCK(call_lock); - -static struct call_data_struct { - void (*func) (void *info); - void *info; - atomic_t started; - atomic_t finished; - int wait; -} *call_data; - -/* delay of at least 8 seconds */ -#define SMP_CALL_TIMEOUT 8 - -/* - * This function sends a 'generic call function' IPI to all other CPUs - * in the system. - * - * [SUMMARY] Run a function on all other CPUs. - * The function to run. This must be fast and non-blocking. - * An arbitrary pointer to pass to the function. - * currently unused. - * If true, wait (atomically) until function has completed on other CPUs. - * [RETURNS] 0 on success, else a negative status code. Does not return until - * remote CPUs are nearly ready to execute <> or are or have executed. - * - * You must not call this function with disabled interrupts or from a - * hardware interrupt handler or from a bottom half handler. - */ -int smp_call_function (void (*func) (void *info), void *info, int nonatomic, - int wait) -{ - struct call_data_struct data; - int ret = -1, cpus; - u64 timeout; - - /* Can deadlock when called with interrupts disabled */ - WARN_ON(irqs_disabled()); - - data.func = func; - data.info = info; - atomic_set(&data.started, 0); - data.wait = wait; - if (wait) - atomic_set(&data.finished, 0); - - spin_lock(&call_lock); - /* Must grab online cpu count with preempt disabled, otherwise - * it can change. */ - cpus = num_online_cpus() - 1; - if (!cpus) { - ret = 0; - goto out; - } - - call_data = &data; - smp_wmb(); - /* Send a message to all other CPUs and wait for them to respond */ - smp_ops->message_pass(MSG_ALL_BUT_SELF, PPC_MSG_CALL_FUNCTION); - - timeout = get_tb() + (u64) SMP_CALL_TIMEOUT * tb_ticks_per_sec; - - /* Wait for response */ - while (atomic_read(&data.started) != cpus) { - HMT_low(); - if (get_tb() >= timeout) { - printk("smp_call_function on cpu %d: other cpus not " - "responding (%d)\n", smp_processor_id(), - atomic_read(&data.started)); - debugger(NULL); - goto out; - } - } - - if (wait) { - while (atomic_read(&data.finished) != cpus) { - HMT_low(); - if (get_tb() >= timeout) { - printk("smp_call_function on cpu %d: other " - "cpus not finishing (%d/%d)\n", - smp_processor_id(), - atomic_read(&data.finished), - atomic_read(&data.started)); - debugger(NULL); - goto out; - } - } - } - - ret = 0; - - out: - call_data = NULL; - HMT_medium(); - spin_unlock(&call_lock); - return ret; -} - -EXPORT_SYMBOL(smp_call_function); - -void smp_call_function_interrupt(void) -{ - void (*func) (void *info); - void *info; - int wait; - - /* call_data will be NULL if the sender timed out while - * waiting on us to receive the call. - */ - if (!call_data) - return; - - func = call_data->func; - info = call_data->info; - wait = call_data->wait; - - if (!wait) - smp_mb__before_atomic_inc(); - - /* - * Notify initiating CPU that I've grabbed the data and am - * about to execute the function - */ - atomic_inc(&call_data->started); - /* - * At this point the info structure may be out of scope unless wait==1 - */ - (*func)(info); - if (wait) { - smp_mb__before_atomic_inc(); - atomic_inc(&call_data->finished); - } + smp_call_function(stop_this_cpu, NULL, 0); } -extern struct gettimeofday_struct do_gtod; - struct thread_info *current_set[NR_CPUS]; -DECLARE_PER_CPU(unsigned int, pvr); - static void __devinit smp_store_cpu_info(int id) { per_cpu(pvr, id) = mfspr(SPRN_PVR); @@ -336,6 +245,8 @@ static void __init smp_create_idle(unsigned int cpu) panic("failed fork for CPU %u: %li", cpu, PTR_ERR(p)); #ifdef CONFIG_PPC64 paca[cpu].__current = p; + paca[cpu].kstack = (unsigned long) task_thread_info(p) + + THREAD_SIZE - STACK_FRAME_OVERHEAD; #endif current_set[cpu] = task_thread_info(p); task_thread_info(p)->cpu = cpu; @@ -357,7 +268,13 @@ void __init smp_prepare_cpus(unsigned int max_cpus) smp_store_cpu_info(boot_cpuid); cpu_callin_map[boot_cpuid] = 1; - max_cpus = smp_ops->probe(); + if (smp_ops) + if (smp_ops->probe) + max_cpus = smp_ops->probe(); + else + max_cpus = NR_CPUS; + else + max_cpus = 1; smp_space_timers(max_cpus); @@ -370,7 +287,9 @@ void __devinit smp_prepare_boot_cpu(void) { BUG_ON(smp_processor_id() != boot_cpuid); - cpu_set(boot_cpuid, cpu_online_map); + set_cpu_online(boot_cpuid, true); + cpu_set(boot_cpuid, per_cpu(cpu_sibling_map, boot_cpuid)); + cpu_set(boot_cpuid, per_cpu(cpu_core_map, boot_cpuid)); #ifdef CONFIG_PPC64 paca[boot_cpuid].__current = current; #endif @@ -388,7 +307,7 @@ int generic_cpu_disable(void) if (cpu == boot_cpuid) return -EBUSY; - cpu_clear(cpu, cpu_online_map); + set_cpu_online(cpu, false); #ifdef CONFIG_PPC64 vdso_data->processorCount--; fixup_irqs(cpu_online_map); @@ -442,24 +361,20 @@ void generic_mach_cpu_die(void) smp_wmb(); while (__get_cpu_var(cpu_state) != CPU_UP_PREPARE) cpu_relax(); - -#ifdef CONFIG_PPC64 - flush_tlb_pending(); -#endif - cpu_set(cpu, cpu_online_map); + set_cpu_online(cpu, true); local_irq_enable(); } #endif static int __devinit cpu_enable(unsigned int cpu) { - if (smp_ops->cpu_enable) + if (smp_ops && smp_ops->cpu_enable) return smp_ops->cpu_enable(cpu); return -ENOSYS; } -int __devinit __cpu_up(unsigned int cpu) +int __cpuinit __cpu_up(unsigned int cpu) { int c; @@ -467,7 +382,8 @@ int __devinit __cpu_up(unsigned int cpu) if (!cpu_enable(cpu)) return 0; - if (smp_ops->cpu_bootable && !smp_ops->cpu_bootable(cpu)) + if (smp_ops == NULL || + (smp_ops->cpu_bootable && !smp_ops->cpu_bootable(cpu))) return -EINVAL; /* Make sure callin-map entry is 0 (can be leftover a CPU @@ -499,9 +415,8 @@ int __devinit __cpu_up(unsigned int cpu) * CPUs can take much longer to come up in the * hotplug case. Wait five seconds. */ - for (c = 25; c && !cpu_callin_map[cpu]; c--) { - msleep(200); - } + for (c = 5000; c && !cpu_callin_map[cpu]; c--) + msleep(1); #endif if (!cpu_callin_map[cpu]) { @@ -521,11 +436,57 @@ int __devinit __cpu_up(unsigned int cpu) return 0; } +/* Return the value of the reg property corresponding to the given + * logical cpu. + */ +int cpu_to_core_id(int cpu) +{ + struct device_node *np; + const int *reg; + int id = -1; + + np = of_get_cpu_node(cpu, NULL); + if (!np) + goto out; + + reg = of_get_property(np, "reg", NULL); + if (!reg) + goto out; + + id = *reg; +out: + of_node_put(np); + return id; +} + +/* Must be called when no change can occur to cpu_present_map, + * i.e. during cpu online or offline. + */ +static struct device_node *cpu_to_l2cache(int cpu) +{ + struct device_node *np; + struct device_node *cache; + + if (!cpu_present(cpu)) + return NULL; + + np = of_get_cpu_node(cpu, NULL); + if (np == NULL) + return NULL; + + cache = of_find_next_cache_node(np); + + of_node_put(np); + + return cache; +} /* Activate a secondary processor. */ int __devinit start_secondary(void *unused) { unsigned int cpu = smp_processor_id(); + struct device_node *l2_cache; + int i, base; atomic_inc(&init_mm.mm_count); current->active_mm = &init_mm; @@ -535,16 +496,47 @@ int __devinit start_secondary(void *unused) preempt_disable(); cpu_callin_map[cpu] = 1; - smp_ops->setup_cpu(cpu); + if (smp_ops->setup_cpu) + smp_ops->setup_cpu(cpu); if (smp_ops->take_timebase) smp_ops->take_timebase(); if (system_state > SYSTEM_BOOTING) snapshot_timebase(); - spin_lock(&call_lock); - cpu_set(cpu, cpu_online_map); - spin_unlock(&call_lock); + secondary_cpu_time_init(); + + ipi_call_lock(); + notify_cpu_starting(cpu); + set_cpu_online(cpu, true); + /* Update sibling maps */ + base = cpu_first_thread_in_core(cpu); + for (i = 0; i < threads_per_core; i++) { + if (cpu_is_offline(base + i)) + continue; + cpu_set(cpu, per_cpu(cpu_sibling_map, base + i)); + cpu_set(base + i, per_cpu(cpu_sibling_map, cpu)); + + /* cpu_core_map should be a superset of + * cpu_sibling_map even if we don't have cache + * information, so update the former here, too. + */ + cpu_set(cpu, per_cpu(cpu_core_map, base +i)); + cpu_set(base + i, per_cpu(cpu_core_map, cpu)); + } + l2_cache = cpu_to_l2cache(cpu); + for_each_online_cpu(i) { + struct device_node *np = cpu_to_l2cache(i); + if (!np) + continue; + if (np == l2_cache) { + cpu_set(cpu, per_cpu(cpu_core_map, i)); + cpu_set(i, per_cpu(cpu_core_map, cpu)); + } + of_node_put(np); + } + of_node_put(l2_cache); + ipi_call_unlock(); local_irq_enable(); @@ -568,7 +560,8 @@ void __init smp_cpus_done(unsigned int max_cpus) old_mask = current->cpus_allowed; set_cpus_allowed(current, cpumask_of_cpu(boot_cpuid)); - smp_ops->setup_cpu(boot_cpuid); + if (smp_ops && smp_ops->setup_cpu) + smp_ops->setup_cpu(boot_cpuid); set_cpus_allowed(current, old_mask); @@ -580,10 +573,42 @@ void __init smp_cpus_done(unsigned int max_cpus) #ifdef CONFIG_HOTPLUG_CPU int __cpu_disable(void) { - if (smp_ops->cpu_disable) - return smp_ops->cpu_disable(); + struct device_node *l2_cache; + int cpu = smp_processor_id(); + int base, i; + int err; - return -ENOSYS; + if (!smp_ops->cpu_disable) + return -ENOSYS; + + err = smp_ops->cpu_disable(); + if (err) + return err; + + /* Update sibling maps */ + base = cpu_first_thread_in_core(cpu); + for (i = 0; i < threads_per_core; i++) { + cpu_clear(cpu, per_cpu(cpu_sibling_map, base + i)); + cpu_clear(base + i, per_cpu(cpu_sibling_map, cpu)); + cpu_clear(cpu, per_cpu(cpu_core_map, base +i)); + cpu_clear(base + i, per_cpu(cpu_core_map, cpu)); + } + + l2_cache = cpu_to_l2cache(cpu); + for_each_present_cpu(i) { + struct device_node *np = cpu_to_l2cache(i); + if (!np) + continue; + if (np == l2_cache) { + cpu_clear(cpu, per_cpu(cpu_core_map, i)); + cpu_clear(i, per_cpu(cpu_core_map, cpu)); + } + of_node_put(np); + } + of_node_put(l2_cache); + + + return 0; } void __cpu_die(unsigned int cpu)