#include <linux/jiffies.h>
#include <linux/profile.h>
#include <linux/lmb.h>
+#include <linux/cpu.h>
#include <asm/head.h>
#include <asm/ptrace.h>
i, cpu_data(i).clock_tick);
}
-static __cacheline_aligned_in_smp DEFINE_SPINLOCK(call_lock);
-
extern void setup_sparc64_timer(void);
static volatile unsigned long callin_flag = 0;
atomic_inc(&init_mm.mm_count);
current->active_mm = &init_mm;
+ /* inform the notifiers about the new cpu */
+ notify_cpu_starting(cpuid);
+
while (!cpu_isset(cpuid, smp_commenced_mask))
rmb();
- spin_lock(&call_lock);
+ ipi_call_lock();
cpu_set(cpuid, cpu_online_map);
- spin_unlock(&call_lock);
+ ipi_call_unlock();
/* idle thread is expected to have preempt disabled */
preempt_disable();
for (i = 0; i < NUM_ITERS; i++) {
t0 = tick_ops->get_tick();
go[MASTER] = 1;
- membar_storeload();
+ membar_safe("#StoreLoad");
while (!(tm = go[SLAVE]))
rmb();
go[SLAVE] = 0;
/* now let the client proceed into his loop */
go[MASTER] = 0;
- membar_storeload();
+ membar_safe("#StoreLoad");
spin_lock_irqsave(&itc_sync_lock, flags);
{
go[MASTER] = 0;
wmb();
go[SLAVE] = tick_ops->get_tick();
- membar_storeload();
+ membar_safe("#StoreLoad");
}
}
spin_unlock_irqrestore(&itc_sync_lock, flags);
return kern_base + (val - KERNBASE);
}
-static void ldom_startcpu_cpuid(unsigned int cpu, unsigned long thread_reg)
+static void __cpuinit ldom_startcpu_cpuid(unsigned int cpu, unsigned long thread_reg)
{
extern unsigned long sparc64_ttable_tl0;
extern unsigned long kern_locked_tte_data;
*/
static struct thread_info *cpu_new_thread = NULL;
-static int __devinit smp_boot_one_cpu(unsigned int cpu)
+static int __cpuinit smp_boot_one_cpu(unsigned int cpu)
{
struct trap_per_cpu *tb = &trap_block[cpu];
unsigned long entry =
/* Multi-cpu list version. */
static void hypervisor_xcall_deliver(struct trap_per_cpu *tb, int cnt)
{
- int retries, this_cpu, prev_sent, i;
+ int retries, this_cpu, prev_sent, i, saw_cpu_error;
unsigned long status;
- cpumask_t error_mask;
u16 *cpu_list;
this_cpu = smp_processor_id();
cpu_list = __va(tb->cpu_list_pa);
- cpus_clear(error_mask);
+ saw_cpu_error = 0;
retries = 0;
prev_sent = 0;
do {
continue;
err = sun4v_cpu_state(cpu);
- if (err >= 0 &&
- err == HV_CPU_STATE_ERROR) {
+ if (err == HV_CPU_STATE_ERROR) {
+ saw_cpu_error = (cpu + 1);
cpu_list[i] = 0xffff;
- cpu_set(cpu, error_mask);
}
}
} else if (unlikely(status != HV_EWOULDBLOCK))
}
} while (1);
- if (unlikely(!cpus_empty(error_mask)))
+ if (unlikely(saw_cpu_error))
goto fatal_mondo_cpu_error;
return;
fatal_mondo_cpu_error:
printk(KERN_CRIT "CPU[%d]: SUN4V mondo cpu error, some target cpus "
- "were in error state\n",
- this_cpu);
- printk(KERN_CRIT "CPU[%d]: Error mask [ ", this_cpu);
- for_each_cpu_mask_nr(i, error_mask)
- printk("%d ", i);
- printk("]\n");
+ "(including %d) were in error state\n",
+ this_cpu, saw_cpu_error - 1);
return;
fatal_mondo_timeout:
* except self. Really, there are only two cases currently,
* "&cpu_online_map" and "&mm->cpu_vm_mask".
*/
-static void smp_cross_call_masked(unsigned long *func, u32 ctx, u64 data1, u64 data2, const cpumask_t *mask_p)
+static void smp_cross_call_masked(unsigned long *func, u32 ctx, u64 data1, u64 data2, const cpumask_t *mask)
{
u64 data0 = (((u64)ctx)<<32 | (((u64)func) & 0xffffffff));
- int this_cpu = get_cpu();
- cpumask_t mask;
- mask = *mask_p;
- if (mask_p != &cpu_online_map)
- cpus_and(mask, mask, cpu_online_map);
- cpu_clear(this_cpu, mask);
-
- xcall_deliver(data0, data1, data2, &mask);
- /* NOTE: Caller runs local copy on master. */
+ xcall_deliver(data0, data1, data2, mask);
+}
- put_cpu();
+/* Send cross call to all processors except self. */
+static void smp_cross_call(unsigned long *func, u32 ctx, u64 data1, u64 data2)
+{
+ smp_cross_call_masked(func, ctx, data1, data2, &cpu_online_map);
}
extern unsigned long xcall_sync_tick;
&cpumask_of_cpu(cpu));
}
-/* Send cross call to all processors except self. */
-#define smp_cross_call(func, ctx, data1, data2) \
- smp_cross_call_masked(func, ctx, data1, data2, &cpu_online_map)
-
void smp_call_function_client(int irq, struct pt_regs *regs)
{
clear_softint(1 << irq);
extern unsigned long xcall_flush_tlb_mm;
extern unsigned long xcall_flush_tlb_pending;
extern unsigned long xcall_flush_tlb_kernel_range;
-#ifdef CONFIG_MAGIC_SYSRQ
extern unsigned long xcall_fetch_glob_regs;
-#endif
extern unsigned long xcall_receive_signal;
extern unsigned long xcall_new_mmu_context_version;
#ifdef CONFIG_KGDB
void smp_flush_dcache_page_impl(struct page *page, int cpu)
{
- cpumask_t mask = cpumask_of_cpu(cpu);
int this_cpu;
if (tlb_type == hypervisor)
}
if (data0) {
xcall_deliver(data0, __pa(pg_addr),
- (u64) pg_addr, &mask);
+ (u64) pg_addr, &cpumask_of_cpu(cpu));
#ifdef CONFIG_DEBUG_DCFLUSH
atomic_inc(&dcpage_flushes_xcall);
#endif
void flush_dcache_page_all(struct mm_struct *mm, struct page *page)
{
- cpumask_t mask = cpu_online_map;
void *pg_addr;
int this_cpu;
u64 data0;
this_cpu = get_cpu();
- cpu_clear(this_cpu, mask);
-
#ifdef CONFIG_DEBUG_DCFLUSH
atomic_inc(&dcpage_flushes);
#endif
- if (cpus_empty(mask))
- goto flush_self;
data0 = 0;
pg_addr = page_address(page);
if (tlb_type == spitfire) {
}
if (data0) {
xcall_deliver(data0, __pa(pg_addr),
- (u64) pg_addr, &mask);
+ (u64) pg_addr, &cpu_online_map);
#ifdef CONFIG_DEBUG_DCFLUSH
atomic_inc(&dcpage_flushes_xcall);
#endif
}
- flush_self:
__local_flush_dcache_page(page);
put_cpu();
}
#endif
-#ifdef CONFIG_MAGIC_SYSRQ
void smp_fetch_global_regs(void)
{
smp_cross_call(&xcall_fetch_glob_regs, 0, 0, 0);
}
-#endif
/* We know that the window frames of the user have been flushed
* to the stack before we get here because all callers of us
smp_processor_id());
#endif
penguins_are_doing_time = 1;
- membar_storestore_loadstore();
atomic_inc(&smp_capture_registry);
smp_cross_call(&xcall_capture, 0, 0, 0);
while (atomic_read(&smp_capture_registry) != ncpus)
smp_processor_id());
#endif
penguins_are_doing_time = 0;
- membar_storeload_storestore();
+ membar_safe("#StoreLoad");
atomic_dec(&smp_capture_registry);
}
}
-/* Imprisoned penguins run with %pil == 15, but PSTATE_IE set, so they
- * can service tlb flush xcalls...
+/* Imprisoned penguins run with %pil == PIL_NORMAL_MAX, but PSTATE_IE
+ * set, so they can service tlb flush xcalls...
*/
extern void prom_world(int);
__asm__ __volatile__("flushw");
prom_world(1);
atomic_inc(&smp_capture_registry);
- membar_storeload_storestore();
+ membar_safe("#StoreLoad");
while (penguins_are_doing_time)
rmb();
atomic_dec(&smp_capture_registry);
c->core_id = 0;
c->proc_id = -1;
- spin_lock(&call_lock);
- cpu_clear(cpu, cpu_online_map);
- spin_unlock(&call_lock);
-
smp_wmb();
/* Make sure no interrupts point to this cpu. */
mdelay(1);
local_irq_disable();
+ ipi_call_lock();
+ cpu_clear(cpu, cpu_online_map);
+ ipi_call_unlock();
+
return 0;
}
git://ftp.safe.ca / safe / jmp / linux-2.6 / blobdiff