-/* Copyright (C) 2004 Mips Technologies, Inc */
+/*
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License
+ * as published by the Free Software Foundation; either version 2
+ * of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
+ *
+ * Copyright (C) 2004 Mips Technologies, Inc
+ * Copyright (C) 2008 Kevin D. Kissell
+ */
+#include <linux/clockchips.h>
#include <linux/kernel.h>
#include <linux/sched.h>
+#include <linux/smp.h>
#include <linux/cpumask.h>
#include <linux/interrupt.h>
+#include <linux/kernel_stat.h>
+#include <linux/module.h>
#include <asm/cpu.h>
#include <asm/processor.h>
#include <asm/system.h>
#include <asm/hardirq.h>
#include <asm/hazards.h>
+#include <asm/irq.h>
#include <asm/mmu_context.h>
-#include <asm/smp.h>
#include <asm/mipsregs.h>
#include <asm/cacheflush.h>
#include <asm/time.h>
#include <asm/addrspace.h>
#include <asm/smtc.h>
-#include <asm/smtc_ipi.h>
#include <asm/smtc_proc.h>
/*
- * This file should be built into the kernel only if CONFIG_MIPS_MT_SMTC is set.
- */
-
-/*
- * MIPSCPU_INT_BASE is identically defined in both
- * asm-mips/mips-boards/maltaint.h and asm-mips/mips-boards/simint.h,
- * but as yet there's no properly organized include structure that
- * will ensure that the right *int.h file will be included for a
- * given platform build.
+ * SMTC Kernel needs to manipulate low-level CPU interrupt mask
+ * in do_IRQ. These are passed in setup_irq_smtc() and stored
+ * in this table.
*/
-
-#define MIPSCPU_INT_BASE 16
-
-#define MIPS_CPU_IPI_IRQ 1
+unsigned long irq_hwmask[NR_IRQS];
#define LOCK_MT_PRA() \
local_irq_save(flags); \
asiduse smtc_live_asid[MAX_SMTC_TLBS][MAX_SMTC_ASIDS];
/*
- * Clock interrupt "latch" buffers, per "CPU"
- */
-
-unsigned int ipi_timer_latch[NR_CPUS];
-
-/*
- * Number of InterProcessor Interupt (IPI) message buffers to allocate
+ * Number of InterProcessor Interrupt (IPI) message buffers to allocate
*/
#define IPIBUF_PER_CPU 4
struct smtc_ipi_q IPIQ[NR_CPUS];
-struct smtc_ipi_q freeIPIq;
+static struct smtc_ipi_q freeIPIq;
/* Forward declarations */
-void ipi_decode(struct pt_regs *, struct smtc_ipi *);
-void post_direct_ipi(int cpu, struct smtc_ipi *pipi);
-void setup_cross_vpe_interrupts(void);
+void ipi_decode(struct smtc_ipi *);
+static void post_direct_ipi(int cpu, struct smtc_ipi *pipi);
+static void setup_cross_vpe_interrupts(unsigned int nvpe);
void init_smtc_stats(void);
/* Global SMTC Status */
-unsigned int smtc_status = 0;
+unsigned int smtc_status;
/* Boot command line configuration overrides */
-static int vpelimit = 0;
-static int tclimit = 0;
-static int ipibuffers = 0;
-static int nostlb = 0;
-static int asidmask = 0;
+static int vpe0limit;
+static int ipibuffers;
+static int nostlb;
+static int asidmask;
unsigned long smtc_asid_mask = 0xff;
-static int __init maxvpes(char *str)
+static int __init vpe0tcs(char *str)
{
- get_option(&str, &vpelimit);
- return 1;
-}
+ get_option(&str, &vpe0limit);
-static int __init maxtcs(char *str)
-{
- get_option(&str, &tclimit);
return 1;
}
return 1;
}
-__setup("maxvpes=", maxvpes);
-__setup("maxtcs=", maxtcs);
+__setup("vpe0tcs=", vpe0tcs);
__setup("ipibufs=", ipibufs);
__setup("nostlb", stlb_disable);
__setup("asidmask=", asidmask_set);
-/* Enable additional debug checks before going into CPU idle loop */
-#define SMTC_IDLE_HOOK_DEBUG
-
-#ifdef SMTC_IDLE_HOOK_DEBUG
+#ifdef CONFIG_SMTC_IDLE_HOOK_DEBUG
-static int hang_trig = 0;
+static int hang_trig;
static int __init hangtrig_enable(char *s)
{
__setup("tintq=", tintq);
-int imstuckcount[2][8];
+static int imstuckcount[2][8];
/* vpemask represents IM/IE bits of per-VPE Status registers, low-to-high */
-int vpemask[2][8] = {{0,1,1,0,0,0,0,1},{0,1,0,0,0,0,0,1}};
+static int vpemask[2][8] = {
+ {0, 0, 1, 0, 0, 0, 0, 1},
+ {0, 0, 0, 0, 0, 0, 0, 1}
+};
int tcnoprog[NR_CPUS];
static atomic_t idle_hook_initialized = {0};
static int clock_hang_reported[NR_CPUS];
-#endif /* SMTC_IDLE_HOOK_DEBUG */
-
-/* Initialize shared TLB - the should probably migrate to smtc_setup_cpus() */
-
-void __init sanitize_tlb_entries(void)
-{
- printk("Deprecated sanitize_tlb_entries() invoked\n");
-}
-
+#endif /* CONFIG_SMTC_IDLE_HOOK_DEBUG */
/*
* Configure shared TLB - VPC configuration bit must be set by caller
*/
-void smtc_configure_tlb(void)
+static void smtc_configure_tlb(void)
{
- int i,tlbsiz,vpes;
+ int i, tlbsiz, vpes;
unsigned long mvpconf0;
unsigned long config1val;
}
}
write_c0_mvpcontrol(read_c0_mvpcontrol() | MVPCONTROL_STLB);
+ ehb();
/*
* Setup kernel data structures to use software total,
* of their initialization in smtc_cpu_setup().
*/
- tlbsiz = tlbsiz & 0x3f; /* MIPS32 limits TLB indices to 64 */
- cpu_data[0].tlbsize = tlbsiz;
+ /* MIPS32 limits TLB indices to 64 */
+ if (tlbsiz > 64)
+ tlbsiz = 64;
+ cpu_data[0].tlbsize = current_cpu_data.tlbsize = tlbsiz;
smtc_status |= SMTC_TLB_SHARED;
+ local_flush_tlb_all();
printk("TLB of %d entry pairs shared by %d VPEs\n",
tlbsiz, vpes);
* possibly leave some TCs/VPEs as "slave" processors.
*
* Use c0_MVPConf0 to find out how many TCs are available, setting up
- * phys_cpu_present_map and the logical/physical mappings.
+ * cpu_possible_map and the logical/physical mappings.
*/
-int __init mipsmt_build_cpu_map(int start_cpu_slot)
+int __init smtc_build_cpu_map(int start_cpu_slot)
{
int i, ntcs;
*/
ntcs = ((read_c0_mvpconf0() & MVPCONF0_PTC) >> MVPCONF0_PTC_SHIFT) + 1;
for (i=start_cpu_slot; i<NR_CPUS && i<ntcs; i++) {
- cpu_set(i, phys_cpu_present_map);
+ set_cpu_possible(i, true);
__cpu_number_map[i] = i;
__cpu_logical_map[i] = i;
}
+#ifdef CONFIG_MIPS_MT_FPAFF
/* Initialize map of CPUs with FPUs */
cpus_clear(mt_fpu_cpumask);
+#endif
/* One of those TC's is the one booting, and not a secondary... */
printk("%i available secondary CPU TC(s)\n", i - 1);
write_tc_c0_tcstatus((read_tc_c0_tcstatus()
& ~(TCSTATUS_TKSU | TCSTATUS_DA | TCSTATUS_IXMT))
| TCSTATUS_A);
- write_tc_c0_tccontext(0);
+ /*
+ * TCContext gets an offset from the base of the IPIQ array
+ * to be used in low-level code to detect the presence of
+ * an active IPI queue
+ */
+ write_tc_c0_tccontext((sizeof(struct smtc_ipi_q) * cpu) << 16);
/* Bind tc to vpe */
write_tc_c0_tcbind(vpe);
/* In general, all TCs should have the same cpu_data indications */
memcpy(&cpu_data[cpu], &cpu_data[0], sizeof(struct cpuinfo_mips));
/* For 34Kf, start with TC/CPU 0 as sole owner of single FPU context */
- if (cpu_data[0].cputype == CPU_34K)
+ if (cpu_data[0].cputype == CPU_34K ||
+ cpu_data[0].cputype == CPU_1004K)
cpu_data[cpu].options &= ~MIPS_CPU_FPU;
cpu_data[cpu].vpe_id = vpe;
cpu_data[cpu].tc_id = tc;
+ /* Multi-core SMTC hasn't been tested, but be prepared */
+ cpu_data[cpu].core = (read_vpe_c0_ebase() >> 1) & 0xff;
}
+/*
+ * Tweak to get Count registes in as close a sync as possible.
+ * Value seems good for 34K-class cores.
+ */
+
+#define CP0_SKEW 8
-void mipsmt_prepare_cpus(void)
+void smtc_prepare_cpus(int cpus)
{
- int i, vpe, tc, ntc, nvpe, tcpervpe, slop, cpu;
+ int i, vpe, tc, ntc, nvpe, tcpervpe[NR_CPUS], slop, cpu;
unsigned long flags;
unsigned long val;
int nipi;
IPIQ[i].head = IPIQ[i].tail = NULL;
spin_lock_init(&IPIQ[i].lock);
IPIQ[i].depth = 0;
- ipi_timer_latch[i] = 0;
+ IPIQ[i].resched_flag = 0; /* No reschedules queued initially */
}
/* cpu_data index starts at zero */
cpu = 0;
cpu_data[cpu].vpe_id = 0;
cpu_data[cpu].tc_id = 0;
+ cpu_data[cpu].core = (read_c0_ebase() >> 1) & 0xff;
cpu++;
/* Report on boot-time options */
- mips_mt_set_cpuoptions ();
+ mips_mt_set_cpuoptions();
if (vpelimit > 0)
printk("Limit of %d VPEs set\n", vpelimit);
if (tclimit > 0)
printk("ASID mask value override to 0x%x\n", asidmask);
/* Temporary */
-#ifdef SMTC_IDLE_HOOK_DEBUG
+#ifdef CONFIG_SMTC_IDLE_HOOK_DEBUG
if (hang_trig)
printk("Logic Analyser Trigger on suspected TC hang\n");
-#endif /* SMTC_IDLE_HOOK_DEBUG */
+#endif /* CONFIG_SMTC_IDLE_HOOK_DEBUG */
/* Put MVPE's into 'configuration state' */
write_c0_mvpcontrol( read_c0_mvpcontrol() | MVPCONTROL_VPC );
ntc = NR_CPUS;
if (tclimit > 0 && ntc > tclimit)
ntc = tclimit;
- tcpervpe = ntc / nvpe;
- slop = ntc % nvpe; /* Residual TCs, < NVPE */
+ slop = ntc % nvpe;
+ for (i = 0; i < nvpe; i++) {
+ tcpervpe[i] = ntc / nvpe;
+ if (slop) {
+ if((slop - i) > 0) tcpervpe[i]++;
+ }
+ }
+ /* Handle command line override for VPE0 */
+ if (vpe0limit > ntc) vpe0limit = ntc;
+ if (vpe0limit > 0) {
+ int slopslop;
+ if (vpe0limit < tcpervpe[0]) {
+ /* Reducing TC count - distribute to others */
+ slop = tcpervpe[0] - vpe0limit;
+ slopslop = slop % (nvpe - 1);
+ tcpervpe[0] = vpe0limit;
+ for (i = 1; i < nvpe; i++) {
+ tcpervpe[i] += slop / (nvpe - 1);
+ if(slopslop && ((slopslop - (i - 1) > 0)))
+ tcpervpe[i]++;
+ }
+ } else if (vpe0limit > tcpervpe[0]) {
+ /* Increasing TC count - steal from others */
+ slop = vpe0limit - tcpervpe[0];
+ slopslop = slop % (nvpe - 1);
+ tcpervpe[0] = vpe0limit;
+ for (i = 1; i < nvpe; i++) {
+ tcpervpe[i] -= slop / (nvpe - 1);
+ if(slopslop && ((slopslop - (i - 1) > 0)))
+ tcpervpe[i]--;
+ }
+ }
+ }
/* Set up shared TLB */
smtc_configure_tlb();
for (tc = 0, vpe = 0 ; (vpe < nvpe) && (tc < ntc) ; vpe++) {
- /*
- * Set the MVP bits.
- */
- settc(tc);
- write_vpe_c0_vpeconf0(read_vpe_c0_vpeconf0() | VPECONF0_MVP);
+ if (tcpervpe[vpe] == 0)
+ continue;
if (vpe != 0)
printk(", ");
printk("VPE %d: TC", vpe);
- for (i = 0; i < tcpervpe; i++) {
+ for (i = 0; i < tcpervpe[vpe]; i++) {
/*
* TC 0 is bound to VPE 0 at reset,
* and is presumably executing this
* code. Leave it alone!
*/
if (tc != 0) {
- smtc_tc_setup(vpe,tc, cpu);
- cpu++;
- }
- printk(" %d", tc);
- tc++;
- }
- if (slop) {
- if (tc != 0) {
- smtc_tc_setup(vpe,tc, cpu);
+ smtc_tc_setup(vpe, tc, cpu);
cpu++;
}
printk(" %d", tc);
tc++;
- slop--;
}
if (vpe != 0) {
/*
+ * Allow this VPE to control others.
+ */
+ write_vpe_c0_vpeconf0(read_vpe_c0_vpeconf0() |
+ VPECONF0_MVP);
+
+ /*
* Clear any stale software interrupts from VPE's Cause
*/
write_vpe_c0_cause(0);
write_vpe_c0_compare(0);
/* Propagate Config7 */
write_vpe_c0_config7(read_c0_config7());
+ write_vpe_c0_count(read_c0_count() + CP0_SKEW);
+ ehb();
}
/* enable multi-threading within VPE */
write_vpe_c0_vpecontrol(read_vpe_c0_vpecontrol() | VPECONTROL_TE);
* Pull any physically present but unused TCs out of circulation.
*/
while (tc < (((val & MVPCONF0_PTC) >> MVPCONF0_PTC_SHIFT) + 1)) {
- cpu_clear(tc, phys_cpu_present_map);
- cpu_clear(tc, cpu_present_map);
+ set_cpu_possible(tc, false);
+ set_cpu_present(tc, false);
tc++;
}
/* Set up coprocessor affinity CPU mask(s) */
+#ifdef CONFIG_MIPS_MT_FPAFF
for (tc = 0; tc < ntc; tc++) {
if (cpu_data[tc].options & MIPS_CPU_FPU)
cpu_set(tc, mt_fpu_cpumask);
}
+#endif
/* set up ipi interrupts... */
/* If we have multiple VPEs running, set up the cross-VPE interrupt */
- if (nvpe > 1)
- setup_cross_vpe_interrupts();
+ setup_cross_vpe_interrupts(nvpe);
/* Set up queue of free IPI "messages". */
nipi = NR_CPUS * IPIBUF_PER_CPU;
* (unsigned long)idle->thread_info the gp
*
*/
-void smtc_boot_secondary(int cpu, struct task_struct *idle)
+void __cpuinit smtc_boot_secondary(int cpu, struct task_struct *idle)
{
extern u32 kernelsp[NR_CPUS];
- long flags;
+ unsigned long flags;
int mtflags;
LOCK_MT_PRA();
write_tc_gpr_sp(__KSTK_TOS(idle));
/* global pointer */
- write_tc_gpr_gp((unsigned long)idle->thread_info);
+ write_tc_gpr_gp((unsigned long)task_thread_info(idle));
smtc_status |= SMTC_MTC_ACTIVE;
write_tc_c0_tchalt(0);
void smtc_init_secondary(void)
{
- /*
- * Start timer on secondary VPEs if necessary.
- * mips_timer_setup should already have been invoked by init/main
- * on "boot" TC. Like per_cpu_trap_init() hack, this assumes that
- * SMTC init code assigns TCs consdecutively and in ascending order
- * to across available VPEs.
- */
- if (((read_c0_tcbind() & TCBIND_CURTC) != 0) &&
- ((read_c0_tcbind() & TCBIND_CURVPE)
- != cpu_data[smp_processor_id() - 1].vpe_id)){
- write_c0_compare (read_c0_count() + mips_hpt_frequency/HZ);
- }
-
local_irq_enable();
}
void smtc_smp_finish(void)
{
+ int cpu = smp_processor_id();
+
+ /*
+ * Lowest-numbered CPU per VPE starts a clock tick.
+ * Like per_cpu_trap_init() hack, this assumes that
+ * SMTC init code assigns TCs consdecutively and
+ * in ascending order across available VPEs.
+ */
+ if (cpu > 0 && (cpu_data[cpu].vpe_id != cpu_data[cpu - 1].vpe_id))
+ write_c0_compare(read_c0_count() + mips_hpt_frequency/HZ);
+
printk("TC %d going on-line as CPU %d\n",
cpu_data[smp_processor_id()].tc_id, smp_processor_id());
}
int setup_irq_smtc(unsigned int irq, struct irqaction * new,
unsigned long hwmask)
{
+#ifdef CONFIG_SMTC_IDLE_HOOK_DEBUG
+ unsigned int vpe = current_cpu_data.vpe_id;
+
+ vpemask[vpe][irq - MIPS_CPU_IRQ_BASE] = 1;
+#endif
irq_hwmask[irq] = hwmask;
return setup_irq(irq, new);
}
+#ifdef CONFIG_MIPS_MT_SMTC_IRQAFF
+/*
+ * Support for IRQ affinity to TCs
+ */
+
+void smtc_set_irq_affinity(unsigned int irq, cpumask_t affinity)
+{
+ /*
+ * If a "fast path" cache of quickly decodable affinity state
+ * is maintained, this is where it gets done, on a call up
+ * from the platform affinity code.
+ */
+}
+
+void smtc_forward_irq(unsigned int irq)
+{
+ int target;
+
+ /*
+ * OK wise guy, now figure out how to get the IRQ
+ * to be serviced on an authorized "CPU".
+ *
+ * Ideally, to handle the situation where an IRQ has multiple
+ * eligible CPUS, we would maintain state per IRQ that would
+ * allow a fair distribution of service requests. Since the
+ * expected use model is any-or-only-one, for simplicity
+ * and efficiency, we just pick the easiest one to find.
+ */
+
+ target = cpumask_first(irq_desc[irq].affinity);
+
+ /*
+ * We depend on the platform code to have correctly processed
+ * IRQ affinity change requests to ensure that the IRQ affinity
+ * mask has been purged of bits corresponding to nonexistent and
+ * offline "CPUs", and to TCs bound to VPEs other than the VPE
+ * connected to the physical interrupt input for the interrupt
+ * in question. Otherwise we have a nasty problem with interrupt
+ * mask management. This is best handled in non-performance-critical
+ * platform IRQ affinity setting code, to minimize interrupt-time
+ * checks.
+ */
+
+ /* If no one is eligible, service locally */
+ if (target >= NR_CPUS) {
+ do_IRQ_no_affinity(irq);
+ return;
+ }
+
+ smtc_send_ipi(target, IRQ_AFFINITY_IPI, irq);
+}
+
+#endif /* CONFIG_MIPS_MT_SMTC_IRQAFF */
+
/*
* IPI model for SMTC is tricky, because interrupts aren't TC-specific.
* Within a VPE one TC can interrupt another by different approaches.
* the VPE.
*/
-void smtc_ipi_qdump(void)
+static void smtc_ipi_qdump(void)
{
int i;
+ struct smtc_ipi *temp;
for (i = 0; i < NR_CPUS ;i++) {
- printk("IPIQ[%d]: head = 0x%x, tail = 0x%x, depth = %d\n",
+ pr_info("IPIQ[%d]: head = 0x%x, tail = 0x%x, depth = %d\n",
i, (unsigned)IPIQ[i].head, (unsigned)IPIQ[i].tail,
IPIQ[i].depth);
+ temp = IPIQ[i].head;
+
+ while (temp != IPIQ[i].tail) {
+ pr_debug("%d %d %d: ", temp->type, temp->dest,
+ (int)temp->arg);
+#ifdef SMTC_IPI_DEBUG
+ pr_debug("%u %lu\n", temp->sender, temp->stamp);
+#else
+ pr_debug("\n");
+#endif
+ temp = temp->flink;
+ }
}
}
* be done with the atomic.h primitives). And since this is
* MIPS MT, we can assume that we have LL/SC.
*/
-static __inline__ int atomic_postincrement(unsigned int *pv)
+static inline int atomic_postincrement(atomic_t *v)
{
unsigned long result;
" addu %1, %0, 1 \n"
" sc %1, %2 \n"
" beqz %1, 1b \n"
- " sync \n"
- : "=&r" (result), "=&r" (temp), "=m" (*pv)
- : "m" (*pv)
- : "memory");
-
- return result;
-}
-
-/* No longer used in IPI dispatch, but retained for future recycling */
-
-static __inline__ int atomic_postclear(unsigned int *pv)
-{
- unsigned long result;
-
- unsigned long temp;
-
- __asm__ __volatile__(
- "1: ll %0, %2 \n"
- " or %1, $0, $0 \n"
- " sc %1, %2 \n"
- " beqz %1, 1b \n"
- " sync \n"
- : "=&r" (result), "=&r" (temp), "=m" (*pv)
- : "m" (*pv)
+ __WEAK_LLSC_MB
+ : "=&r" (result), "=&r" (temp), "=m" (v->counter)
+ : "m" (v->counter)
: "memory");
return result;
}
-
void smtc_send_ipi(int cpu, int type, unsigned int action)
{
int tcstatus;
struct smtc_ipi *pipi;
- long flags;
+ unsigned long flags;
int mtflags;
+ unsigned long tcrestart;
+ extern void r4k_wait_irqoff(void), __pastwait(void);
+ int set_resched_flag = (type == LINUX_SMP_IPI &&
+ action == SMP_RESCHEDULE_YOURSELF);
if (cpu == smp_processor_id()) {
printk("Cannot Send IPI to self!\n");
return;
}
+ if (set_resched_flag && IPIQ[cpu].resched_flag != 0)
+ return; /* There is a reschedule queued already */
+
/* Set up a descriptor, to be delivered either promptly or queued */
pipi = smtc_ipi_dq(&freeIPIq);
if (pipi == NULL) {
pipi->arg = (void *)action;
pipi->dest = cpu;
if (cpu_data[cpu].vpe_id != cpu_data[smp_processor_id()].vpe_id) {
- /* If not on same VPE, enqueue and send cross-VPE interupt */
+ /* If not on same VPE, enqueue and send cross-VPE interrupt */
+ IPIQ[cpu].resched_flag |= set_resched_flag;
smtc_ipi_nq(&IPIQ[cpu], pipi);
LOCK_CORE_PRA();
settc(cpu_data[cpu].tc_id);
if ((tcstatus & TCSTATUS_IXMT) != 0) {
/*
- * Spin-waiting here can deadlock,
- * so we queue the message for the target TC.
+ * If we're in the the irq-off version of the wait
+ * loop, we need to force exit from the wait and
+ * do a direct post of the IPI.
+ */
+ if (cpu_wait == r4k_wait_irqoff) {
+ tcrestart = read_tc_c0_tcrestart();
+ if (tcrestart >= (unsigned long)r4k_wait_irqoff
+ && tcrestart < (unsigned long)__pastwait) {
+ write_tc_c0_tcrestart(__pastwait);
+ tcstatus &= ~TCSTATUS_IXMT;
+ write_tc_c0_tcstatus(tcstatus);
+ goto postdirect;
+ }
+ }
+ /*
+ * Otherwise we queue the message for the target TC
+ * to pick up when he does a local_irq_restore()
*/
write_tc_c0_tchalt(0);
UNLOCK_CORE_PRA();
- /* Try to reduce redundant timer interrupt messages */
- if (type == SMTC_CLOCK_TICK) {
- if (atomic_postincrement(&ipi_timer_latch[cpu])!=0){
- smtc_ipi_nq(&freeIPIq, pipi);
- return;
- }
- }
+ IPIQ[cpu].resched_flag |= set_resched_flag;
smtc_ipi_nq(&IPIQ[cpu], pipi);
} else {
+postdirect:
post_direct_ipi(cpu, pipi);
write_tc_c0_tchalt(0);
UNLOCK_CORE_PRA();
/*
* Send IPI message to Halted TC, TargTC/TargVPE already having been set
*/
-void post_direct_ipi(int cpu, struct smtc_ipi *pipi)
+static void post_direct_ipi(int cpu, struct smtc_ipi *pipi)
{
struct pt_regs *kstack;
unsigned long tcstatus;
unsigned long tcrestart;
extern u32 kernelsp[NR_CPUS];
extern void __smtc_ipi_vector(void);
+//printk("%s: on %d for %d\n", __func__, smp_processor_id(), cpu);
/* Extract Status, EPC from halted TC */
tcstatus = read_tc_c0_tcstatus();
write_tc_c0_tcrestart(__smtc_ipi_vector);
}
-void ipi_resched_interrupt(struct pt_regs *regs)
+static void ipi_resched_interrupt(void)
{
/* Return from interrupt should be enough to cause scheduler check */
}
-
-void ipi_call_interrupt(struct pt_regs *regs)
+static void ipi_call_interrupt(void)
{
/* Invoke generic function invocation code in smp.c */
smp_call_function_interrupt();
}
-void ipi_decode(struct pt_regs *regs, struct smtc_ipi *pipi)
+DECLARE_PER_CPU(struct clock_event_device, mips_clockevent_device);
+
+void ipi_decode(struct smtc_ipi *pipi)
{
+ unsigned int cpu = smp_processor_id();
+ struct clock_event_device *cd;
void *arg_copy = pipi->arg;
int type_copy = pipi->type;
- int dest_copy = pipi->dest;
+ int irq = MIPS_CPU_IRQ_BASE + 1;
smtc_ipi_nq(&freeIPIq, pipi);
+
switch (type_copy) {
case SMTC_CLOCK_TICK:
- /* Invoke Clock "Interrupt" */
- ipi_timer_latch[dest_copy] = 0;
-#ifdef SMTC_IDLE_HOOK_DEBUG
- clock_hang_reported[dest_copy] = 0;
-#endif /* SMTC_IDLE_HOOK_DEBUG */
- local_timer_interrupt(0, NULL, regs);
+ irq_enter();
+ kstat_incr_irqs_this_cpu(irq, irq_to_desc(irq));
+ cd = &per_cpu(mips_clockevent_device, cpu);
+ cd->event_handler(cd);
+ irq_exit();
break;
+
case LINUX_SMP_IPI:
switch ((int)arg_copy) {
case SMP_RESCHEDULE_YOURSELF:
- ipi_resched_interrupt(regs);
+ ipi_resched_interrupt();
break;
case SMP_CALL_FUNCTION:
- ipi_call_interrupt(regs);
+ ipi_call_interrupt();
break;
default:
printk("Impossible SMTC IPI Argument 0x%x\n",
break;
}
break;
+#ifdef CONFIG_MIPS_MT_SMTC_IRQAFF
+ case IRQ_AFFINITY_IPI:
+ /*
+ * Accept a "forwarded" interrupt that was initially
+ * taken by a TC who doesn't have affinity for the IRQ.
+ */
+ do_IRQ_no_affinity((int)arg_copy);
+ break;
+#endif /* CONFIG_MIPS_MT_SMTC_IRQAFF */
default:
printk("Impossible SMTC IPI Type 0x%x\n", type_copy);
break;
}
}
-void deferred_smtc_ipi(struct pt_regs *regs)
+/*
+ * Similar to smtc_ipi_replay(), but invoked from context restore,
+ * so it reuses the current exception frame rather than set up a
+ * new one with self_ipi.
+ */
+
+void deferred_smtc_ipi(void)
{
- struct smtc_ipi *pipi;
- unsigned long flags;
-/* DEBUG */
- int q = smp_processor_id();
+ int cpu = smp_processor_id();
/*
* Test is not atomic, but much faster than a dequeue,
* and the vast majority of invocations will have a null queue.
+ * If irq_disabled when this was called, then any IPIs queued
+ * after we test last will be taken on the next irq_enable/restore.
+ * If interrupts were enabled, then any IPIs added after the
+ * last test will be taken directly.
*/
- if (IPIQ[q].head != NULL) {
- while((pipi = smtc_ipi_dq(&IPIQ[q])) != NULL) {
- /* ipi_decode() should be called with interrupts off */
- local_irq_save(flags);
- ipi_decode(regs, pipi);
- local_irq_restore(flags);
- }
- }
-}
-
-/*
- * Send clock tick to all TCs except the one executing the funtion
- */
-void smtc_timer_broadcast(int vpe)
-{
- int cpu;
- int myTC = cpu_data[smp_processor_id()].tc_id;
- int myVPE = cpu_data[smp_processor_id()].vpe_id;
-
- smtc_cpu_stats[smp_processor_id()].timerints++;
+ while (IPIQ[cpu].head != NULL) {
+ struct smtc_ipi_q *q = &IPIQ[cpu];
+ struct smtc_ipi *pipi;
+ unsigned long flags;
- for_each_online_cpu(cpu) {
- if (cpu_data[cpu].vpe_id == myVPE &&
- cpu_data[cpu].tc_id != myTC)
- smtc_send_ipi(cpu, SMTC_CLOCK_TICK, 0);
+ /*
+ * It may be possible we'll come in with interrupts
+ * already enabled.
+ */
+ local_irq_save(flags);
+ spin_lock(&q->lock);
+ pipi = __smtc_ipi_dq(q);
+ spin_unlock(&q->lock);
+ if (pipi != NULL) {
+ if (pipi->type == LINUX_SMP_IPI &&
+ (int)pipi->arg == SMP_RESCHEDULE_YOURSELF)
+ IPIQ[cpu].resched_flag = 0;
+ ipi_decode(pipi);
+ }
+ /*
+ * The use of the __raw_local restore isn't
+ * as obviously necessary here as in smtc_ipi_replay(),
+ * but it's more efficient, given that we're already
+ * running down the IPI queue.
+ */
+ __raw_local_irq_restore(flags);
}
}
* interrupts.
*/
-static int cpu_ipi_irq = MIPSCPU_INT_BASE + MIPS_CPU_IPI_IRQ;
+static int cpu_ipi_irq = MIPS_CPU_IRQ_BASE + MIPS_CPU_IPI_IRQ;
-static irqreturn_t ipi_interrupt(int irq, void *dev_idm, struct pt_regs *regs)
+static irqreturn_t ipi_interrupt(int irq, void *dev_idm)
{
int my_vpe = cpu_data[smp_processor_id()].vpe_id;
int my_tc = cpu_data[smp_processor_id()].tc_id;
struct smtc_ipi *pipi;
unsigned long tcstatus;
int sent;
- long flags;
+ unsigned long flags;
unsigned int mtflags;
unsigned int vpflags;
* with interrupts off
*/
local_irq_save(flags);
- ipi_decode(regs, pipi);
+ if (pipi->type == LINUX_SMP_IPI &&
+ (int)pipi->arg == SMP_RESCHEDULE_YOURSELF)
+ IPIQ[cpu].resched_flag = 0;
+ ipi_decode(pipi);
local_irq_restore(flags);
}
}
return IRQ_HANDLED;
}
-static void ipi_irq_dispatch(struct pt_regs *regs)
+static void ipi_irq_dispatch(void)
{
- do_IRQ(cpu_ipi_irq, regs);
+ do_IRQ(cpu_ipi_irq);
}
-static struct irqaction irq_ipi;
+static struct irqaction irq_ipi = {
+ .handler = ipi_interrupt,
+ .flags = IRQF_DISABLED | IRQF_PERCPU,
+ .name = "SMTC_IPI"
+};
-void setup_cross_vpe_interrupts(void)
+static void setup_cross_vpe_interrupts(unsigned int nvpe)
{
+ if (nvpe < 1)
+ return;
+
if (!cpu_has_vint)
- panic("SMTC Kernel requires Vectored Interupt support");
+ panic("SMTC Kernel requires Vectored Interrupt support");
set_vi_handler(MIPS_CPU_IPI_IRQ, ipi_irq_dispatch);
- irq_ipi.handler = ipi_interrupt;
- irq_ipi.flags = IRQF_DISABLED;
- irq_ipi.name = "SMTC_IPI";
-
setup_irq_smtc(cpu_ipi_irq, &irq_ipi, (0x100 << MIPS_CPU_IPI_IRQ));
- irq_desc[cpu_ipi_irq].status |= IRQ_PER_CPU;
+ set_irq_handler(cpu_ipi_irq, handle_percpu_irq);
}
/*
* SMTC-specific hacks invoked from elsewhere in the kernel.
*/
+ /*
+ * smtc_ipi_replay is called from raw_local_irq_restore
+ */
+
+void smtc_ipi_replay(void)
+{
+ unsigned int cpu = smp_processor_id();
+
+ /*
+ * To the extent that we've ever turned interrupts off,
+ * we may have accumulated deferred IPIs. This is subtle.
+ * we should be OK: If we pick up something and dispatch
+ * it here, that's great. If we see nothing, but concurrent
+ * with this operation, another TC sends us an IPI, IXMT
+ * is clear, and we'll handle it as a real pseudo-interrupt
+ * and not a pseudo-pseudo interrupt. The important thing
+ * is to do the last check for queued message *after* the
+ * re-enabling of interrupts.
+ */
+ while (IPIQ[cpu].head != NULL) {
+ struct smtc_ipi_q *q = &IPIQ[cpu];
+ struct smtc_ipi *pipi;
+ unsigned long flags;
+
+ /*
+ * It's just possible we'll come in with interrupts
+ * already enabled.
+ */
+ local_irq_save(flags);
+
+ spin_lock(&q->lock);
+ pipi = __smtc_ipi_dq(q);
+ spin_unlock(&q->lock);
+ /*
+ ** But use a raw restore here to avoid recursion.
+ */
+ __raw_local_irq_restore(flags);
+
+ if (pipi) {
+ self_ipi(pipi);
+ smtc_cpu_stats[cpu].selfipis++;
+ }
+ }
+}
+
+EXPORT_SYMBOL(smtc_ipi_replay);
+
void smtc_idle_loop_hook(void)
{
-#ifdef SMTC_IDLE_HOOK_DEBUG
+#ifdef CONFIG_SMTC_IDLE_HOOK_DEBUG
int im;
int flags;
int mtflags;
mtflags = dmt();
pdb_msg = &id_ho_db_msg[0];
im = read_c0_status();
- vpe = cpu_data[smp_processor_id()].vpe_id;
+ vpe = current_cpu_data.vpe_id;
for (bit = 0; bit < 8; bit++) {
/*
* In current prototype, I/O interrupts
}
}
- /*
- * Now that we limit outstanding timer IPIs, check for hung TC
- */
- for (tc = 0; tc < NR_CPUS; tc++) {
- /* Don't check ourself - we'll dequeue IPIs just below */
- if ((tc != smp_processor_id()) &&
- ipi_timer_latch[tc] > timerq_limit) {
- if (clock_hang_reported[tc] == 0) {
- pdb_msg += sprintf(pdb_msg,
- "TC %d looks hung with timer latch at %d\n",
- tc, ipi_timer_latch[tc]);
- clock_hang_reported[tc]++;
- }
- }
- }
emt(mtflags);
local_irq_restore(flags);
if (pdb_msg != &id_ho_db_msg[0])
printk("CPU%d: %s", smp_processor_id(), id_ho_db_msg);
-#endif /* SMTC_IDLE_HOOK_DEBUG */
- /*
- * To the extent that we've ever turned interrupts off,
- * we may have accumulated deferred IPIs. This is subtle.
- * If we use the smtc_ipi_qdepth() macro, we'll get an
- * exact number - but we'll also disable interrupts
- * and create a window of failure where a new IPI gets
- * queued after we test the depth but before we re-enable
- * interrupts. So long as IXMT never gets set, however,
- * we should be OK: If we pick up something and dispatch
- * it here, that's great. If we see nothing, but concurrent
- * with this operation, another TC sends us an IPI, IXMT
- * is clear, and we'll handle it as a real pseudo-interrupt
- * and not a pseudo-pseudo interrupt.
- */
- if (IPIQ[smp_processor_id()].depth > 0) {
- struct smtc_ipi *pipi;
- extern void self_ipi(struct smtc_ipi *);
+#endif /* CONFIG_SMTC_IDLE_HOOK_DEBUG */
- if ((pipi = smtc_ipi_dq(&IPIQ[smp_processor_id()])) != NULL) {
- self_ipi(pipi);
- smtc_cpu_stats[smp_processor_id()].selfipis++;
- }
- }
+ smtc_ipi_replay();
}
void smtc_soft_dump(void)
printk("%d: %ld\n", i, smtc_cpu_stats[i].selfipis);
}
smtc_ipi_qdump();
- printk("Timer IPI Backlogs:\n");
- for (i=0; i < NR_CPUS; i++) {
- printk("%d: %d\n", i, ipi_timer_latch[i]);
- }
printk("%d Recoveries of \"stolen\" FPU\n",
atomic_read(&smtc_fpu_recoveries));
}
* It would be nice to be able to use a spinlock here,
* but this is invoked from within TLB flush routines
* that protect themselves with DVPE, so if a lock is
- * held by another TC, it'll never be freed.
+ * held by another TC, it'll never be freed.
*
* DVPE/DMT must not be done with interrupts enabled,
* so even so most callers will already have disabled
if (cpu_has_vtag_icache)
flush_icache_all();
/* Traverse all online CPUs (hack requires contigous range) */
- for (i = 0; i < num_online_cpus(); i++) {
+ for_each_online_cpu(i) {
/*
* We don't need to worry about our own CPU, nor those of
* CPUs who don't share our TLB.
/*
* SMTC shares the TLB within VPEs and possibly across all VPEs.
*/
- for (i = 0; i < num_online_cpus(); i++) {
+ for_each_online_cpu(i) {
if ((smtc_status & SMTC_TLB_SHARED) ||
(cpu_data[i].vpe_id == cpu_data[cpu].vpe_id))
cpu_context(i, mm) = asid_cache(i) = asid;
* Support for single-threading cache flush operations.
*/
-int halt_state_save[NR_CPUS];
+static int halt_state_save[NR_CPUS];
/*
* To really, really be sure that nothing is being done
git://ftp.safe.ca / safe / jmp / linux-2.6 / blobdiff