[POWERPC] Move iSeries_tb_recal into its own late_initcall.
[safe/jmp/linux-2.6] / arch / powerpc / kernel / head_64.S
index 881e18e..8cdd48e 100644 (file)
@@ -1,6 +1,4 @@
 /*
- *  arch/ppc64/kernel/head.S
- *
  *  PowerPC version
  *    Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org)
  *
@@ -23,7 +21,6 @@
  *  2 of the License, or (at your option) any later version.
  */
 
-#include <linux/config.h>
 #include <linux/threads.h>
 #include <asm/reg.h>
 #include <asm/page.h>
 #include <asm/hvcall.h>
 #include <asm/iseries/lpar_map.h>
 #include <asm/thread_info.h>
+#include <asm/firmware.h>
 
-#ifdef CONFIG_PPC_ISERIES
 #define DO_SOFT_DISABLE
-#endif
 
 /*
  * We layout physical memory as follows:
        .text
        .globl  _stext
 _stext:
-#ifdef CONFIG_PPC_MULTIPLATFORM
 _GLOBAL(__start)
        /* NOP this out unconditionally */
 BEGIN_FTR_SECTION
        b       .__start_initialization_multiplatform
 END_FTR_SECTION(0, 1)
-#endif /* CONFIG_PPC_MULTIPLATFORM */
 
        /* Catch branch to 0 in real mode */
        trap
 
-#ifdef CONFIG_PPC_ISERIES
-       /*
-        * At offset 0x20, there is a pointer to iSeries LPAR data.
-        * This is required by the hypervisor
-        */
-       . = 0x20
-       .llong hvReleaseData-KERNELBASE
-
-       /*
-        * At offset 0x28 and 0x30 are offsets to the mschunks_map
-        * array (used by the iSeries LPAR debugger to do translation
-        * between physical addresses and absolute addresses) and
-        * to the pidhash table (also used by the debugger)
-        */
-       .llong mschunks_map-KERNELBASE
-       .llong 0        /* pidhash-KERNELBASE SFRXXX */
-
-       /* Offset 0x38 - Pointer to start of embedded System.map */
-       .globl  embedded_sysmap_start
-embedded_sysmap_start:
-       .llong  0
-       /* Offset 0x40 - Pointer to end of embedded System.map */
-       .globl  embedded_sysmap_end
-embedded_sysmap_end:
-       .llong  0
-
-#endif /* CONFIG_PPC_ISERIES */
-
        /* Secondary processors spin on this value until it goes to 1. */
        .globl  __secondary_hold_spinloop
 __secondary_hold_spinloop:
@@ -126,10 +92,19 @@ __secondary_hold_spinloop:
 __secondary_hold_acknowledge:
        .llong  0x0
 
+#ifdef CONFIG_PPC_ISERIES
+       /*
+        * At offset 0x20, there is a pointer to iSeries LPAR data.
+        * This is required by the hypervisor
+        */
+       . = 0x20
+       .llong hvReleaseData-KERNELBASE
+#endif /* CONFIG_PPC_ISERIES */
+
        . = 0x60
 /*
- * The following code is used on pSeries to hold secondary processors
- * in a spin loop after they have been freed from OpenFirmware, but
+ * The following code is used to hold secondary processors
+ * in a spin loop after they have entered the kernel, but
  * before the bulk of the kernel has been relocated.  This code
  * is relocated to physical address 0x60 before prom_init is run.
  * All of it must fit below the first exception vector at 0x100.
@@ -139,7 +114,7 @@ _GLOBAL(__secondary_hold)
        ori     r24,r24,MSR_RI
        mtmsrd  r24                     /* RI on */
 
-       /* Grab our linux cpu number */
+       /* Grab our physical cpu number */
        mr      r24,r3
 
        /* Tell the master cpu we're here */
@@ -153,16 +128,14 @@ _GLOBAL(__secondary_hold)
        cmpdi   0,r4,1
        bne     100b
 
-#ifdef CONFIG_HMT
-       b       .hmt_init
-#else
-#ifdef CONFIG_SMP
+#if defined(CONFIG_SMP) || defined(CONFIG_KEXEC)
+       LOAD_REG_IMMEDIATE(r4, .generic_secondary_smp_init)
+       mtctr   r4
        mr      r3,r24
-       b       .pSeries_secondary_smp_init
+       bctr
 #else
        BUG_OPCODE
 #endif
-#endif
 
 /* This value is used to mark exception frames on the stack. */
        .section ".toc","aw"
@@ -200,6 +173,52 @@ exception_marker:
 #define EX_R3          64
 #define EX_LR          72
 
+/*
+ * We're short on space and time in the exception prolog, so we can't
+ * use the normal SET_REG_IMMEDIATE macro. Normally we just need the
+ * low halfword of the address, but for Kdump we need the whole low
+ * word.
+ */
+#ifdef CONFIG_CRASH_DUMP
+#define LOAD_HANDLER(reg, label)                                       \
+       oris    reg,reg,(label)@h;      /* virt addr of handler ... */  \
+       ori     reg,reg,(label)@l;      /* .. and the rest */
+#else
+#define LOAD_HANDLER(reg, label)                                       \
+       ori     reg,reg,(label)@l;      /* virt addr of handler ... */
+#endif
+
+/*
+ * Equal to EXCEPTION_PROLOG_PSERIES, except that it forces 64bit mode.
+ * The firmware calls the registered system_reset_fwnmi and
+ * machine_check_fwnmi handlers in 32bit mode if the cpu happens to run
+ * a 32bit application at the time of the event.
+ * This firmware bug is present on POWER4 and JS20.
+ */
+#define EXCEPTION_PROLOG_PSERIES_FORCE_64BIT(area, label)              \
+       mfspr   r13,SPRN_SPRG3;         /* get paca address into r13 */ \
+       std     r9,area+EX_R9(r13);     /* save r9 - r12 */             \
+       std     r10,area+EX_R10(r13);                                   \
+       std     r11,area+EX_R11(r13);                                   \
+       std     r12,area+EX_R12(r13);                                   \
+       mfspr   r9,SPRN_SPRG1;                                          \
+       std     r9,area+EX_R13(r13);                                    \
+       mfcr    r9;                                                     \
+       clrrdi  r12,r13,32;             /* get high part of &label */   \
+       mfmsr   r10;                                                    \
+       /* force 64bit mode */                                          \
+       li      r11,5;                  /* MSR_SF_LG|MSR_ISF_LG */      \
+       rldimi  r10,r11,61,0;           /* insert into top 3 bits */    \
+       /* done 64bit mode */                                           \
+       mfspr   r11,SPRN_SRR0;          /* save SRR0 */                 \
+       LOAD_HANDLER(r12,label)                                         \
+       ori     r10,r10,MSR_IR|MSR_DR|MSR_RI;                           \
+       mtspr   SPRN_SRR0,r12;                                          \
+       mfspr   r12,SPRN_SRR1;          /* and SRR1 */                  \
+       mtspr   SPRN_SRR1,r10;                                          \
+       rfid;                                                           \
+       b       .       /* prevent speculative execution */
+
 #define EXCEPTION_PROLOG_PSERIES(area, label)                          \
        mfspr   r13,SPRN_SPRG3;         /* get paca address into r13 */ \
        std     r9,area+EX_R9(r13);     /* save r9 - r12 */             \
@@ -212,7 +231,7 @@ exception_marker:
        clrrdi  r12,r13,32;             /* get high part of &label */   \
        mfmsr   r10;                                                    \
        mfspr   r11,SPRN_SRR0;          /* save SRR0 */                 \
-       ori     r12,r12,(label)@l;      /* virt addr of handler */      \
+       LOAD_HANDLER(r12,label)                                         \
        ori     r10,r10,MSR_IR|MSR_DR|MSR_RI;                           \
        mtspr   SPRN_SRR0,r12;                                          \
        mfspr   r12,SPRN_SRR1;          /* and SRR1 */                  \
@@ -236,8 +255,9 @@ exception_marker:
 
 #define EXCEPTION_PROLOG_ISERIES_2                                     \
        mfmsr   r10;                                                    \
-       ld      r11,PACALPPACA+LPPACASRR0(r13);                         \
-       ld      r12,PACALPPACA+LPPACASRR1(r13);                         \
+       ld      r12,PACALPPACAPTR(r13);                                 \
+       ld      r11,LPPACASRR0(r12);                                    \
+       ld      r12,LPPACASRR1(r12);                                    \
        ori     r10,r10,MSR_RI;                                         \
        mtmsrd  r10,1
 
@@ -258,13 +278,18 @@ exception_marker:
        beq-    1f;                                                        \
        ld      r1,PACAKSAVE(r13);      /* kernel stack to use          */ \
 1:     cmpdi   cr1,r1,0;               /* check if r1 is in userspace  */ \
-       bge-    cr1,bad_stack;          /* abort if it is               */ \
-       std     r9,_CCR(r1);            /* save CR in stackframe        */ \
+       bge-    cr1,2f;                 /* abort if it is               */ \
+       b       3f;                                                        \
+2:     li      r1,(n);                 /* will be reloaded later       */ \
+       sth     r1,PACA_TRAP_SAVE(r13);                                    \
+       b       bad_stack;                                                 \
+3:     std     r9,_CCR(r1);            /* save CR in stackframe        */ \
        std     r11,_NIP(r1);           /* save SRR0 in stackframe      */ \
        std     r12,_MSR(r1);           /* save SRR1 in stackframe      */ \
        std     r10,0(r1);              /* make stack chain pointer     */ \
        std     r0,GPR0(r1);            /* save r0 in stackframe        */ \
        std     r10,GPR1(r1);           /* save r1 in stackframe        */ \
+       ACCOUNT_CPU_USER_ENTRY(r9, r10);                                   \
        std     r2,GPR2(r1);            /* save r2 in stackframe        */ \
        SAVE_4GPRS(3, r1);              /* save r3 - r6 in stackframe   */ \
        SAVE_2GPRS(7, r1);              /* save r7, r8 in stackframe    */ \
@@ -283,7 +308,9 @@ exception_marker:
        std     r9,_LINK(r1);                                              \
        mfctr   r10;                    /* save CTR in stackframe       */ \
        std     r10,_CTR(r1);                                              \
+       lbz     r10,PACASOFTIRQEN(r13);                            \
        mfspr   r11,SPRN_XER;           /* save XER in stackframe       */ \
+       std     r10,SOFTE(r1);                                             \
        std     r11,_XER(r1);                                              \
        li      r9,(n)+1;                                                  \
        std     r9,_TRAP(r1);           /* set trap number              */ \
@@ -301,15 +328,56 @@ exception_marker:
 label##_pSeries:                                       \
        HMT_MEDIUM;                                     \
        mtspr   SPRN_SPRG1,r13;         /* save r13 */  \
-       RUNLATCH_ON(r13);                               \
        EXCEPTION_PROLOG_PSERIES(PACA_EXGEN, label##_common)
 
+#define HSTD_EXCEPTION_PSERIES(n, label)               \
+       . = n;                                          \
+       .globl label##_pSeries;                         \
+label##_pSeries:                                       \
+       HMT_MEDIUM;                                     \
+       mtspr   SPRN_SPRG1,r20;         /* save r20 */  \
+       mfspr   r20,SPRN_HSRR0;         /* copy HSRR0 to SRR0 */ \
+       mtspr   SPRN_SRR0,r20;                          \
+       mfspr   r20,SPRN_HSRR1;         /* copy HSRR0 to SRR0 */ \
+       mtspr   SPRN_SRR1,r20;                          \
+       mfspr   r20,SPRN_SPRG1;         /* restore r20 */ \
+       mtspr   SPRN_SPRG1,r13;         /* save r13 */  \
+       EXCEPTION_PROLOG_PSERIES(PACA_EXGEN, label##_common)
+
+
+#define MASKABLE_EXCEPTION_PSERIES(n, label)                           \
+       . = n;                                                          \
+       .globl label##_pSeries;                                         \
+label##_pSeries:                                                       \
+       HMT_MEDIUM;                                                     \
+       mtspr   SPRN_SPRG1,r13;         /* save r13 */                  \
+       mfspr   r13,SPRN_SPRG3;         /* get paca address into r13 */ \
+       std     r9,PACA_EXGEN+EX_R9(r13);       /* save r9, r10 */      \
+       std     r10,PACA_EXGEN+EX_R10(r13);                             \
+       lbz     r10,PACASOFTIRQEN(r13);                                 \
+       mfcr    r9;                                                     \
+       cmpwi   r10,0;                                                  \
+       beq     masked_interrupt;                                       \
+       mfspr   r10,SPRN_SPRG1;                                         \
+       std     r10,PACA_EXGEN+EX_R13(r13);                             \
+       std     r11,PACA_EXGEN+EX_R11(r13);                             \
+       std     r12,PACA_EXGEN+EX_R12(r13);                             \
+       clrrdi  r12,r13,32;             /* get high part of &label */   \
+       mfmsr   r10;                                                    \
+       mfspr   r11,SPRN_SRR0;          /* save SRR0 */                 \
+       LOAD_HANDLER(r12,label##_common)                                \
+       ori     r10,r10,MSR_IR|MSR_DR|MSR_RI;                           \
+       mtspr   SPRN_SRR0,r12;                                          \
+       mfspr   r12,SPRN_SRR1;          /* and SRR1 */                  \
+       mtspr   SPRN_SRR1,r10;                                          \
+       rfid;                                                           \
+       b       .       /* prevent speculative execution */
+
 #define STD_EXCEPTION_ISERIES(n, label, area)          \
        .globl label##_iSeries;                         \
 label##_iSeries:                                       \
        HMT_MEDIUM;                                     \
        mtspr   SPRN_SPRG1,r13;         /* save r13 */  \
-       RUNLATCH_ON(r13);                               \
        EXCEPTION_PROLOG_ISERIES_1(area);               \
        EXCEPTION_PROLOG_ISERIES_2;                     \
        b       label##_common
@@ -319,33 +387,33 @@ label##_iSeries:                                  \
 label##_iSeries:                                                       \
        HMT_MEDIUM;                                                     \
        mtspr   SPRN_SPRG1,r13;         /* save r13 */                  \
-       RUNLATCH_ON(r13);                                               \
        EXCEPTION_PROLOG_ISERIES_1(PACA_EXGEN);                         \
-       lbz     r10,PACAPROCENABLED(r13);                               \
+       lbz     r10,PACASOFTIRQEN(r13);                                 \
        cmpwi   0,r10,0;                                                \
        beq-    label##_iSeries_masked;                                 \
        EXCEPTION_PROLOG_ISERIES_2;                                     \
        b       label##_common;                                         \
 
-#ifdef DO_SOFT_DISABLE
+#ifdef CONFIG_PPC_ISERIES
 #define DISABLE_INTS                           \
-       lbz     r10,PACAPROCENABLED(r13);       \
        li      r11,0;                          \
-       std     r10,SOFTE(r1);                  \
+       stb     r11,PACASOFTIRQEN(r13);         \
+BEGIN_FW_FTR_SECTION;                          \
+       stb     r11,PACAHARDIRQEN(r13);         \
+END_FW_FTR_SECTION_IFCLR(FW_FEATURE_ISERIES);  \
+BEGIN_FW_FTR_SECTION;                          \
        mfmsr   r10;                            \
-       stb     r11,PACAPROCENABLED(r13);       \
        ori     r10,r10,MSR_EE;                 \
-       mtmsrd  r10,1
+       mtmsrd  r10,1;                          \
+END_FW_FTR_SECTION_IFSET(FW_FEATURE_ISERIES)
 
-#define ENABLE_INTS                            \
-       lbz     r10,PACAPROCENABLED(r13);       \
-       mfmsr   r11;                            \
-       std     r10,SOFTE(r1);                  \
-       ori     r11,r11,MSR_EE;                 \
-       mtmsrd  r11,1
+#else
+#define DISABLE_INTS                           \
+       li      r11,0;                          \
+       stb     r11,PACASOFTIRQEN(r13);         \
+       stb     r11,PACAHARDIRQEN(r13)
 
-#else  /* hard enable/disable interrupts */
-#define DISABLE_INTS
+#endif /* CONFIG_PPC_ISERIES */
 
 #define ENABLE_INTS                            \
        ld      r12,_MSR(r1);                   \
@@ -353,8 +421,6 @@ label##_iSeries:                                                    \
        rlwimi  r11,r12,0,MSR_EE;               \
        mtmsrd  r11,1
 
-#endif
-
 #define STD_EXCEPTION_COMMON(trap, label, hdlr)                \
        .align  7;                                      \
        .globl label##_common;                          \
@@ -366,17 +432,54 @@ label##_common:                                           \
        bl      hdlr;                                   \
        b       .ret_from_except
 
+/*
+ * Like STD_EXCEPTION_COMMON, but for exceptions that can occur
+ * in the idle task and therefore need the special idle handling.
+ */
+#define STD_EXCEPTION_COMMON_IDLE(trap, label, hdlr)   \
+       .align  7;                                      \
+       .globl label##_common;                          \
+label##_common:                                                \
+       EXCEPTION_PROLOG_COMMON(trap, PACA_EXGEN);      \
+       FINISH_NAP;                                     \
+       DISABLE_INTS;                                   \
+       bl      .save_nvgprs;                           \
+       addi    r3,r1,STACK_FRAME_OVERHEAD;             \
+       bl      hdlr;                                   \
+       b       .ret_from_except
+
 #define STD_EXCEPTION_COMMON_LITE(trap, label, hdlr)   \
        .align  7;                                      \
        .globl label##_common;                          \
 label##_common:                                                \
        EXCEPTION_PROLOG_COMMON(trap, PACA_EXGEN);      \
+       FINISH_NAP;                                     \
        DISABLE_INTS;                                   \
+       bl      .ppc64_runlatch_on;                     \
        addi    r3,r1,STACK_FRAME_OVERHEAD;             \
        bl      hdlr;                                   \
        b       .ret_from_except_lite
 
 /*
+ * When the idle code in power4_idle puts the CPU into NAP mode,
+ * it has to do so in a loop, and relies on the external interrupt
+ * and decrementer interrupt entry code to get it out of the loop.
+ * It sets the _TLF_NAPPING bit in current_thread_info()->local_flags
+ * to signal that it is in the loop and needs help to get out.
+ */
+#ifdef CONFIG_PPC_970_NAP
+#define FINISH_NAP                             \
+BEGIN_FTR_SECTION                              \
+       clrrdi  r11,r1,THREAD_SHIFT;            \
+       ld      r9,TI_LOCAL_FLAGS(r11);         \
+       andi.   r10,r9,_TLF_NAPPING;            \
+       bnel    power4_fixup_nap;               \
+END_FTR_SECTION_IFSET(CPU_FTR_CAN_NAP)
+#else
+#define FINISH_NAP
+#endif
+
+/*
  * Start of pSeries system interrupt routines
  */
        . = 0x100
@@ -389,7 +492,6 @@ __start_interrupts:
 _machine_check_pSeries:
        HMT_MEDIUM
        mtspr   SPRN_SPRG1,r13          /* save r13 */
-       RUNLATCH_ON(r13)
        EXCEPTION_PROLOG_PSERIES(PACA_EXMC, machine_check_common)
 
        . = 0x300
@@ -405,7 +507,7 @@ BEGIN_FTR_SECTION
        rlwimi  r13,r12,16,0x20
        mfcr    r12
        cmpwi   r13,0x2c
-       beq     .do_stab_bolted_pSeries
+       beq     do_stab_bolted_pSeries
        mtcrf   0x80,r12
        mfspr   r12,SPRN_SPRG2
 END_FTR_SECTION_IFCLR(CPU_FTR_SLB)
@@ -416,7 +518,6 @@ END_FTR_SECTION_IFCLR(CPU_FTR_SLB)
 data_access_slb_pSeries:
        HMT_MEDIUM
        mtspr   SPRN_SPRG1,r13
-       RUNLATCH_ON(r13)
        mfspr   r13,SPRN_SPRG3          /* get paca address into r13 */
        std     r3,PACA_EXSLB+EX_R3(r13)
        mfspr   r3,SPRN_DAR
@@ -442,7 +543,6 @@ data_access_slb_pSeries:
 instruction_access_slb_pSeries:
        HMT_MEDIUM
        mtspr   SPRN_SPRG1,r13
-       RUNLATCH_ON(r13)
        mfspr   r13,SPRN_SPRG3          /* get paca address into r13 */
        std     r3,PACA_EXSLB+EX_R3(r13)
        mfspr   r3,SPRN_SRR0            /* SRR0 is faulting address */
@@ -461,11 +561,11 @@ instruction_access_slb_pSeries:
        mfspr   r12,SPRN_SRR1           /* and SRR1 */
        b       .slb_miss_realmode      /* Rel. branch works in real mode */
 
-       STD_EXCEPTION_PSERIES(0x500, hardware_interrupt)
+       MASKABLE_EXCEPTION_PSERIES(0x500, hardware_interrupt)
        STD_EXCEPTION_PSERIES(0x600, alignment)
        STD_EXCEPTION_PSERIES(0x700, program_check)
        STD_EXCEPTION_PSERIES(0x800, fp_unavailable)
-       STD_EXCEPTION_PSERIES(0x900, decrementer)
+       MASKABLE_EXCEPTION_PSERIES(0x900, decrementer)
        STD_EXCEPTION_PSERIES(0xa00, trap_0a)
        STD_EXCEPTION_PSERIES(0xb00, trap_0b)
 
@@ -473,7 +573,6 @@ instruction_access_slb_pSeries:
        .globl  system_call_pSeries
 system_call_pSeries:
        HMT_MEDIUM
-       RUNLATCH_ON(r9)
        mr      r9,r13
        mfmsr   r10
        mfspr   r13,SPRN_SPRG3
@@ -501,8 +600,17 @@ system_call_pSeries:
 
        STD_EXCEPTION_PSERIES(0xf20, altivec_unavailable)
 
+#ifdef CONFIG_CBE_RAS
+       HSTD_EXCEPTION_PSERIES(0x1200, cbe_system_error)
+#endif /* CONFIG_CBE_RAS */
        STD_EXCEPTION_PSERIES(0x1300, instruction_breakpoint)
+#ifdef CONFIG_CBE_RAS
+       HSTD_EXCEPTION_PSERIES(0x1600, cbe_maintenance)
+#endif /* CONFIG_CBE_RAS */
        STD_EXCEPTION_PSERIES(0x1700, altivec_assist)
+#ifdef CONFIG_CBE_RAS
+       HSTD_EXCEPTION_PSERIES(0x1800, cbe_thermal)
+#endif /* CONFIG_CBE_RAS */
 
        . = 0x3000
 
@@ -511,8 +619,25 @@ system_call_pSeries:
        /* moved from 0xf00 */
        STD_EXCEPTION_PSERIES(., performance_monitor)
 
+/*
+ * An interrupt came in while soft-disabled; clear EE in SRR1,
+ * clear paca->hard_enabled and return.
+ */
+masked_interrupt:
+       stb     r10,PACAHARDIRQEN(r13)
+       mtcrf   0x80,r9
+       ld      r9,PACA_EXGEN+EX_R9(r13)
+       mfspr   r10,SPRN_SRR1
+       rldicl  r10,r10,48,1            /* clear MSR_EE */
+       rotldi  r10,r10,16
+       mtspr   SPRN_SRR1,r10
+       ld      r10,PACA_EXGEN+EX_R10(r13)
+       mfspr   r13,SPRN_SPRG1
+       rfid
+       b       .
+
        .align  7
-_GLOBAL(do_stab_bolted_pSeries)
+do_stab_bolted_pSeries:
        mtcrf   0x80,r12
        mfspr   r12,SPRN_SPRG2
        EXCEPTION_PROLOG_PSERIES(PACA_EXSLB, .do_stab_bolted)
@@ -553,18 +678,18 @@ slb_miss_user_pseries:
  * Vectors for the FWNMI option.  Share common code.
  */
        .globl system_reset_fwnmi
+      .align 7
 system_reset_fwnmi:
        HMT_MEDIUM
        mtspr   SPRN_SPRG1,r13          /* save r13 */
-       RUNLATCH_ON(r13)
-       EXCEPTION_PROLOG_PSERIES(PACA_EXGEN, system_reset_common)
+       EXCEPTION_PROLOG_PSERIES_FORCE_64BIT(PACA_EXGEN, system_reset_common)
 
        .globl machine_check_fwnmi
+      .align 7
 machine_check_fwnmi:
        HMT_MEDIUM
        mtspr   SPRN_SPRG1,r13          /* save r13 */
-       RUNLATCH_ON(r13)
-       EXCEPTION_PROLOG_PSERIES(PACA_EXMC, machine_check_common)
+       EXCEPTION_PROLOG_PSERIES_FORCE_64BIT(PACA_EXMC, machine_check_common)
 
 #ifdef CONFIG_PPC_ISERIES
 /***  ISeries-LPAR interrupt handlers ***/
@@ -614,7 +739,8 @@ data_access_slb_iSeries:
        std     r12,PACA_EXSLB+EX_R12(r13)
        mfspr   r10,SPRN_SPRG1
        std     r10,PACA_EXSLB+EX_R13(r13)
-       ld      r12,PACALPPACA+LPPACASRR1(r13);
+       ld      r12,PACALPPACAPTR(r13)
+       ld      r12,LPPACASRR1(r12)
        b       .slb_miss_realmode
 
        STD_EXCEPTION_ISERIES(0x400, instruction_access, PACA_EXGEN)
@@ -624,7 +750,8 @@ instruction_access_slb_iSeries:
        mtspr   SPRN_SPRG1,r13          /* save r13 */
        mfspr   r13,SPRN_SPRG3          /* get paca address into r13 */
        std     r3,PACA_EXSLB+EX_R3(r13)
-       ld      r3,PACALPPACA+LPPACASRR0(r13)   /* get SRR0 value */
+       ld      r3,PACALPPACAPTR(r13)
+       ld      r3,LPPACASRR0(r3)       /* get SRR0 value */
        std     r9,PACA_EXSLB+EX_R9(r13)
        mfcr    r9
 #ifdef __DISABLED__
@@ -636,7 +763,8 @@ instruction_access_slb_iSeries:
        std     r12,PACA_EXSLB+EX_R12(r13)
        mfspr   r10,SPRN_SPRG1
        std     r10,PACA_EXSLB+EX_R13(r13)
-       ld      r12,PACALPPACA+LPPACASRR1(r13);
+       ld      r12,PACALPPACAPTR(r13)
+       ld      r12,LPPACASRR1(r12)
        b       .slb_miss_realmode
 
 #ifdef __DISABLED__
@@ -693,7 +821,7 @@ system_reset_iSeries:
        lbz     r23,PACAPROCSTART(r13)  /* Test if this processor
                                         * should start */
        sync
-       LOADADDR(r3,current_set)
+       LOAD_REG_IMMEDIATE(r3,current_set)
        sldi    r28,r24,3               /* get current_set[cpu#] */
        ldx     r3,r3,r28
        addi    r1,r3,THREAD_SIZE
@@ -701,7 +829,7 @@ system_reset_iSeries:
 
        cmpwi   0,r23,0
        beq     iSeries_secondary_smp_loop      /* Loop until told to go */
-       bne     .__secondary_start              /* Loop until told to go */
+       bne     __secondary_start               /* Loop until told to go */
 iSeries_secondary_smp_loop:
        /* Let the Hypervisor know we are alive */
        /* 8002 is a call to HvCallCfg::getLps, a harmless Hypervisor function */
@@ -722,20 +850,21 @@ iSeries_secondary_smp_loop:
        b       1b                      /* If SMP not configured, secondaries
                                         * loop forever */
 
-       .globl decrementer_iSeries_masked
 decrementer_iSeries_masked:
+       /* We may not have a valid TOC pointer in here. */
        li      r11,1
-       stb     r11,PACALPPACA+LPPACADECRINT(r13)
-       LOADBASE(r12,tb_ticks_per_jiffy)
-       lwz     r12,OFF(tb_ticks_per_jiffy)(r13)
+       ld      r12,PACALPPACAPTR(r13)
+       stb     r11,LPPACADECRINT(r12)
+       LOAD_REG_IMMEDIATE(r12, tb_ticks_per_jiffy)
+       lwz     r12,0(r12)
        mtspr   SPRN_DEC,r12
        /* fall through */
 
-       .globl hardware_interrupt_iSeries_masked
 hardware_interrupt_iSeries_masked:
        mtcrf   0x80,r9         /* Restore regs */
-       ld      r11,PACALPPACA+LPPACASRR0(r13)
-       ld      r12,PACALPPACA+LPPACASRR1(r13)
+       ld      r12,PACALPPACAPTR(r13)
+       ld      r11,LPPACASRR0(r12)
+       ld      r12,LPPACASRR1(r12)
        mtspr   SPRN_SRR0,r11
        mtspr   SPRN_SRR1,r12
        ld      r9,PACA_EXGEN+EX_R9(r13)
@@ -759,6 +888,7 @@ hardware_interrupt_iSeries_masked:
        .globl machine_check_common
 machine_check_common:
        EXCEPTION_PROLOG_COMMON(0x200, PACA_EXMC)
+       FINISH_NAP
        DISABLE_INTS
        bl      .save_nvgprs
        addi    r3,r1,STACK_FRAME_OVERHEAD
@@ -770,13 +900,18 @@ machine_check_common:
        STD_EXCEPTION_COMMON(0xb00, trap_0b, .unknown_exception)
        STD_EXCEPTION_COMMON(0xd00, single_step, .single_step_exception)
        STD_EXCEPTION_COMMON(0xe00, trap_0e, .unknown_exception)
-       STD_EXCEPTION_COMMON(0xf00, performance_monitor, .performance_monitor_exception)
+       STD_EXCEPTION_COMMON_IDLE(0xf00, performance_monitor, .performance_monitor_exception)
        STD_EXCEPTION_COMMON(0x1300, instruction_breakpoint, .instruction_breakpoint_exception)
 #ifdef CONFIG_ALTIVEC
        STD_EXCEPTION_COMMON(0x1700, altivec_assist, .altivec_assist_exception)
 #else
        STD_EXCEPTION_COMMON(0x1700, altivec_assist, .unknown_exception)
 #endif
+#ifdef CONFIG_CBE_RAS
+       STD_EXCEPTION_COMMON(0x1200, cbe_system_error, .cbe_system_error_exception)
+       STD_EXCEPTION_COMMON(0x1600, cbe_maintenance, .cbe_maintenance_exception)
+       STD_EXCEPTION_COMMON(0x1800, cbe_thermal, .cbe_thermal_exception)
+#endif /* CONFIG_CBE_RAS */
 
 /*
  * Here we have detected that the kernel stack pointer is bad.
@@ -809,6 +944,8 @@ bad_stack:
        SAVE_2GPRS(7,r1)
        SAVE_10GPRS(12,r1)
        SAVE_10GPRS(22,r1)
+       lhz     r12,PACA_TRAP_SAVE(r13)
+       std     r12,_TRAP(r1)
        addi    r11,r1,INT_FRAME_SIZE
        std     r11,0(r1)
        li      r12,0
@@ -826,12 +963,28 @@ bad_stack:
  * any task or sent any task a signal, you should use
  * ret_from_except or ret_from_except_lite instead of this.
  */
+fast_exc_return_irq:                   /* restores irq state too */
+       ld      r3,SOFTE(r1)
+       ld      r12,_MSR(r1)
+       stb     r3,PACASOFTIRQEN(r13)   /* restore paca->soft_enabled */
+       rldicl  r4,r12,49,63            /* get MSR_EE to LSB */
+       stb     r4,PACAHARDIRQEN(r13)   /* restore paca->hard_enabled */
+       b       1f
+
        .globl  fast_exception_return
 fast_exception_return:
        ld      r12,_MSR(r1)
-       ld      r11,_NIP(r1)
+1:     ld      r11,_NIP(r1)
        andi.   r3,r12,MSR_RI           /* check if RI is set */
        beq-    unrecov_fer
+
+#ifdef CONFIG_VIRT_CPU_ACCOUNTING
+       andi.   r3,r12,MSR_PR
+       beq     2f
+       ACCOUNT_CPU_USER_EXIT(r3, r4)
+2:
+#endif
+
        ld      r3,_CCR(r1)
        ld      r4,_LINK(r1)
        ld      r5,_CTR(r1)
@@ -844,7 +997,8 @@ fast_exception_return:
        REST_8GPRS(2, r1)
 
        mfmsr   r10
-       clrrdi  r10,r10,2               /* clear RI (LE is 0 already) */
+       rldicl  r10,r10,48,1            /* clear EE */
+       rldicr  r10,r10,16,61           /* clear RI (LE is 0 already) */
        mtmsrd  r10,1
 
        mtspr   SPRN_SRR1,r12
@@ -868,7 +1022,6 @@ unrecov_fer:
        .align  7
        .globl data_access_common
 data_access_common:
-       RUNLATCH_ON(r10)                /* It wont fit in the 0x300 handler */
        mfspr   r10,SPRN_DAR
        std     r10,PACA_EXGEN+EX_DAR(r13)
        mfspr   r10,SPRN_DSISR
@@ -939,7 +1092,7 @@ slb_miss_fault:
        li      r5,0
        std     r4,_DAR(r1)
        std     r5,_DSISR(r1)
-       b       .handle_page_fault
+       b       handle_page_fault
 
 unrecov_user_slb:
        EXCEPTION_PROLOG_COMMON(0x4200, PACA_EXGEN)
@@ -974,7 +1127,10 @@ _GLOBAL(slb_miss_realmode)
        ld      r3,PACA_EXSLB+EX_R3(r13)
        lwz     r9,PACA_EXSLB+EX_CCR(r13)       /* get saved CR */
 #ifdef CONFIG_PPC_ISERIES
-       ld      r11,PACALPPACA+LPPACASRR0(r13)  /* get SRR0 value */
+BEGIN_FW_FTR_SECTION
+       ld      r11,PACALPPACAPTR(r13)
+       ld      r11,LPPACASRR0(r11)             /* get SRR0 value */
+END_FW_FTR_SECTION_IFSET(FW_FEATURE_ISERIES)
 #endif /* CONFIG_PPC_ISERIES */
 
        mtlr    r10
@@ -989,8 +1145,10 @@ _GLOBAL(slb_miss_realmode)
 .machine       pop
 
 #ifdef CONFIG_PPC_ISERIES
+BEGIN_FW_FTR_SECTION
        mtspr   SPRN_SRR0,r11
        mtspr   SPRN_SRR1,r12
+END_FW_FTR_SECTION_IFSET(FW_FEATURE_ISERIES)
 #endif /* CONFIG_PPC_ISERIES */
        ld      r9,PACA_EXSLB+EX_R9(r13)
        ld      r10,PACA_EXSLB+EX_R10(r13)
@@ -1013,12 +1171,23 @@ unrecov_slb:
        .globl hardware_interrupt_entry
 hardware_interrupt_common:
        EXCEPTION_PROLOG_COMMON(0x500, PACA_EXGEN)
+       FINISH_NAP
 hardware_interrupt_entry:
        DISABLE_INTS
+       bl      .ppc64_runlatch_on
        addi    r3,r1,STACK_FRAME_OVERHEAD
        bl      .do_IRQ
        b       .ret_from_except_lite
 
+#ifdef CONFIG_PPC_970_NAP
+power4_fixup_nap:
+       andc    r9,r9,r10
+       std     r9,TI_LOCAL_FLAGS(r11)
+       ld      r10,_LINK(r1)           /* make idle task do the */
+       std     r10,_NIP(r1)            /* equivalent of a blr */
+       blr
+#endif
+
        .align  7
        .globl alignment_common
 alignment_common:
@@ -1051,12 +1220,13 @@ program_check_common:
        .globl fp_unavailable_common
 fp_unavailable_common:
        EXCEPTION_PROLOG_COMMON(0x800, PACA_EXGEN)
-       bne     .load_up_fpu            /* if from user, just load it up */
+       bne     1f                      /* if from user, just load it up */
        bl      .save_nvgprs
        addi    r3,r1,STACK_FRAME_OVERHEAD
        ENABLE_INTS
        bl      .kernel_fp_unavailable_exception
        BUG_OPCODE
+1:     b       .load_up_fpu
 
        .align  7
        .globl altivec_unavailable_common
@@ -1156,10 +1326,10 @@ _GLOBAL(do_hash_page)
        std     r4,_DSISR(r1)
 
        andis.  r0,r4,0xa450            /* weird error? */
-       bne-    .handle_page_fault      /* if not, try to insert a HPTE */
+       bne-    handle_page_fault       /* if not, try to insert a HPTE */
 BEGIN_FTR_SECTION
        andis.  r0,r4,0x0020            /* Is it a segment table fault? */
-       bne-    .do_ste_alloc           /* If so handle it */
+       bne-    do_ste_alloc            /* If so handle it */
 END_FTR_SECTION_IFCLR(CPU_FTR_SLB)
 
        /*
@@ -1192,6 +1362,7 @@ END_FTR_SECTION_IFCLR(CPU_FTR_SLB)
        cmpdi   r3,0                    /* see if hash_page succeeded */
 
 #ifdef DO_SOFT_DISABLE
+BEGIN_FW_FTR_SECTION
        /*
         * If we had interrupts soft-enabled at the point where the
         * DSI/ISI occurred, and an interrupt came in during hash_page,
@@ -1200,7 +1371,17 @@ END_FTR_SECTION_IFCLR(CPU_FTR_SLB)
         * because ret_from_except_lite will check for and handle pending
         * interrupts if necessary.
         */
-       beq     .ret_from_except_lite
+       beq     13f
+END_FW_FTR_SECTION_IFSET(FW_FEATURE_ISERIES)
+#endif
+BEGIN_FW_FTR_SECTION
+       /*
+        * Here we have interrupts hard-disabled, so it is sufficient
+        * to restore paca->{soft,hard}_enable and get out.
+        */
+       beq     fast_exc_return_irq     /* Return from exception on success */
+END_FW_FTR_SECTION_IFCLR(FW_FEATURE_ISERIES)
+
        /* For a hash failure, we don't bother re-enabling interrupts */
        ble-    12f
 
@@ -1212,22 +1393,16 @@ END_FTR_SECTION_IFCLR(CPU_FTR_SLB)
        ld      r3,SOFTE(r1)
        bl      .local_irq_restore
        b       11f
-#else
-       beq     fast_exception_return   /* Return from exception on success */
-       ble-    12f                     /* Failure return from hash_page */
-
-       /* fall through */
-#endif
 
 /* Here we have a page fault that hash_page can't handle. */
-_GLOBAL(handle_page_fault)
+handle_page_fault:
        ENABLE_INTS
 11:    ld      r4,_DAR(r1)
        ld      r5,_DSISR(r1)
        addi    r3,r1,STACK_FRAME_OVERHEAD
        bl      .do_page_fault
        cmpdi   r3,0
-       beq+    .ret_from_except_lite
+       beq+    13f
        bl      .save_nvgprs
        mr      r5,r3
        addi    r3,r1,STACK_FRAME_OVERHEAD
@@ -1235,6 +1410,8 @@ _GLOBAL(handle_page_fault)
        bl      .bad_page_fault
        b       .ret_from_except
 
+13:    b       .ret_from_except_lite
+
 /* We have a page fault that hash_page could handle but HV refused
  * the PTE insertion
  */
@@ -1245,11 +1422,11 @@ _GLOBAL(handle_page_fault)
        b       .ret_from_except
 
        /* here we have a segment miss */
-_GLOBAL(do_ste_alloc)
+do_ste_alloc:
        bl      .ste_allocate           /* try to insert stab entry */
        cmpdi   r3,0
-       beq+    fast_exception_return
-       b       .handle_page_fault
+       bne-    handle_page_fault
+       b       fast_exception_return
 
 /*
  * r13 points to the PACA, r9 contains the saved CR,
@@ -1346,7 +1523,7 @@ _GLOBAL(do_stab_bolted)
  * fixed address (the linker can't compute (u64)&initial_stab >>
  * PAGE_SHIFT).
  */
-       . = STAB0_PHYS_ADDR     /* 0x6000 */
+       . = STAB0_OFFSET        /* 0x6000 */
        .globl initial_stab
 initial_stab:
        .space  4096
@@ -1375,24 +1552,21 @@ fwnmi_data_area:
         . = 0x8000
 
 /*
- * On pSeries, secondary processors spin in the following code.
+ * On pSeries and most other platforms, secondary processors spin
+ * in the following code.
  * At entry, r3 = this processor's number (physical cpu id)
  */
-_GLOBAL(pSeries_secondary_smp_init)
+_GLOBAL(generic_secondary_smp_init)
        mr      r24,r3
        
        /* turn on 64-bit mode */
        bl      .enable_64b_mode
-       isync
-
-       /* Copy some CPU settings from CPU 0 */
-       bl      .__restore_cpu_setup
 
        /* Set up a paca value for this processor. Since we have the
         * physical cpu id in r24, we need to search the pacas to find
         * which logical id maps to our physical one.
         */
-       LOADADDR(r13, paca)             /* Get base vaddr of paca array  */
+       LOAD_REG_IMMEDIATE(r13, paca)   /* Get base vaddr of paca array  */
        li      r5,0                    /* logical cpu id                */
 1:     lhz     r6,PACAHWCPUID(r13)     /* Load HW procid from paca      */
        cmpw    r6,r24                  /* Compare to our id             */
@@ -1413,21 +1587,34 @@ _GLOBAL(pSeries_secondary_smp_init)
                                        /* start.                        */
        sync
 
-       /* Create a temp kernel stack for use before relocation is on.  */
+#ifndef CONFIG_SMP
+       b       3b                      /* Never go on non-SMP           */
+#else
+       cmpwi   0,r23,0
+       beq     3b                      /* Loop until told to go         */
+
+       /* See if we need to call a cpu state restore handler */
+       LOAD_REG_IMMEDIATE(r23, cur_cpu_spec)
+       ld      r23,0(r23)
+       ld      r23,CPU_SPEC_RESTORE(r23)
+       cmpdi   0,r23,0
+       beq     4f
+       ld      r23,0(r23)
+       mtctr   r23
+       bctrl
+
+4:     /* Create a temp kernel stack for use before relocation is on.  */
        ld      r1,PACAEMERGSP(r13)
        subi    r1,r1,STACK_FRAME_OVERHEAD
 
-       cmpwi   0,r23,0
-#ifdef CONFIG_SMP
-       bne     .__secondary_start
+       b       __secondary_start
 #endif
-       b       3b                      /* Loop until told to go         */
 
 #ifdef CONFIG_PPC_ISERIES
 _STATIC(__start_initialization_iSeries)
        /* Clear out the BSS */
-       LOADADDR(r11,__bss_stop)
-       LOADADDR(r8,__bss_start)
+       LOAD_REG_IMMEDIATE(r11,__bss_stop)
+       LOAD_REG_IMMEDIATE(r8,__bss_start)
        sub     r11,r11,r8              /* bss size                     */
        addi    r11,r11,7               /* round up to an even double word */
        rldicl. r11,r11,61,3            /* shift right by 3             */
@@ -1438,17 +1625,12 @@ _STATIC(__start_initialization_iSeries)
 3:     stdu    r0,8(r8)
        bdnz    3b
 4:
-       LOADADDR(r1,init_thread_union)
+       LOAD_REG_IMMEDIATE(r1,init_thread_union)
        addi    r1,r1,THREAD_SIZE
        li      r0,0
        stdu    r0,-STACK_FRAME_OVERHEAD(r1)
 
-       LOADADDR(r3,cpu_specs)
-       LOADADDR(r4,cur_cpu_spec)
-       li      r5,0
-       bl      .identify_cpu
-
-       LOADADDR(r2,__toc_start)
+       LOAD_REG_IMMEDIATE(r2,__toc_start)
        addi    r2,r2,0x4000
        addi    r2,r2,0x4000
 
@@ -1460,7 +1642,6 @@ _STATIC(__start_initialization_iSeries)
        b       .start_here_common
 #endif /* CONFIG_PPC_ISERIES */
 
-#ifdef CONFIG_PPC_MULTIPLATFORM
 
 _STATIC(__mmu_off)
        mfmsr   r3
@@ -1500,13 +1681,21 @@ _GLOBAL(__start_initialization_multiplatform)
        bl      .enable_64b_mode
 
        /* Setup some critical 970 SPRs before switching MMU off */
-       bl      .__970_cpu_preinit
-
-       /* cpu # */
-       li      r24,0
+       mfspr   r0,SPRN_PVR
+       srwi    r0,r0,16
+       cmpwi   r0,0x39         /* 970 */
+       beq     1f
+       cmpwi   r0,0x3c         /* 970FX */
+       beq     1f
+       cmpwi   r0,0x44         /* 970MP */
+       beq     1f
+       cmpwi   r0,0x45         /* 970GX */
+       bne     2f
+1:     bl      .__cpu_preinit_ppc970
+2:
 
        /* Switch off MMU if not already */
-       LOADADDR(r4, .__after_prom_start - KERNELBASE)
+       LOAD_REG_IMMEDIATE(r4, .__after_prom_start - KERNELBASE)
        add     r4,r4,r30
        bl      .__mmu_off
        b       .__after_prom_start
@@ -1519,13 +1708,20 @@ _STATIC(__boot_from_prom)
        mr      r28,r6
        mr      r27,r7
 
+       /*
+        * Align the stack to 16-byte boundary
+        * Depending on the size and layout of the ELF sections in the initial
+        * boot binary, the stack pointer will be unalignet on PowerMac
+        */
+       rldicr  r1,r1,0,59
+
        /* Make sure we are running in 64 bits mode */
        bl      .enable_64b_mode
 
        /* put a relocation offset into r3 */
        bl      .reloc_offset
 
-       LOADADDR(r2,__toc_start)
+       LOAD_REG_IMMEDIATE(r2,__toc_start)
        addi    r2,r2,0x4000
        addi    r2,r2,0x4000
 
@@ -1544,14 +1740,10 @@ _STATIC(__boot_from_prom)
        /* We never return */
        trap
 
-/*
- * At this point, r3 contains the physical address we are running at,
- * returned by prom_init()
- */
 _STATIC(__after_prom_start)
 
 /*
- * We need to run with __start at physical address 0.
+ * We need to run with __start at physical address PHYSICAL_START.
  * This will leave some code in the first 256B of
  * real memory, which are reserved for software use.
  * The remainder of the first page is loaded with the fixed
@@ -1564,9 +1756,9 @@ _STATIC(__after_prom_start)
  */
        bl      .reloc_offset
        mr      r26,r3
-       SET_REG_TO_CONST(r27,KERNELBASE)
+       LOAD_REG_IMMEDIATE(r27, KERNELBASE)
 
-       li      r3,0                    /* target addr */
+       LOAD_REG_IMMEDIATE(r3, PHYSICAL_START)  /* target addr */
 
        // XXX FIXME: Use phys returned by OF (r30)
        add     r4,r27,r26              /* source addr                   */
@@ -1574,7 +1766,10 @@ _STATIC(__after_prom_start)
                                        /*   i.e. where we are running   */
                                        /*      the source addr          */
 
-       LOADADDR(r5,copy_to_here)       /* # bytes of memory to copy     */
+       cmpdi   r4,0                    /* In some cases the loader may  */
+       beq     .start_here_multiplatform /* have already put us at zero */
+                                       /* so we can skip the copy.      */
+       LOAD_REG_IMMEDIATE(r5,copy_to_here) /* # bytes of memory to copy */
        sub     r5,r5,r27
 
        li      r6,0x100                /* Start offset, the first 0x100 */
@@ -1584,19 +1779,17 @@ _STATIC(__after_prom_start)
                                        /* this includes the code being  */
                                        /* executed here.                */
 
-       LOADADDR(r0, 4f)                /* Jump to the copy of this code */
+       LOAD_REG_IMMEDIATE(r0, 4f)      /* Jump to the copy of this code */
        mtctr   r0                      /* that we just made/relocated   */
        bctr
 
-4:     LOADADDR(r5,klimit)
+4:     LOAD_REG_IMMEDIATE(r5,klimit)
        add     r5,r5,r26
        ld      r5,0(r5)                /* get the value of klimit */
        sub     r5,r5,r27
        bl      .copy_and_flush         /* copy the rest */
        b       .start_here_multiplatform
 
-#endif /* CONFIG_PPC_MULTIPLATFORM */
-
 /*
  * Copy routine used to copy the kernel to start at physical address 0
  * and flush and invalidate the caches as needed.
@@ -1608,7 +1801,7 @@ _STATIC(__after_prom_start)
 _GLOBAL(copy_and_flush)
        addi    r5,r5,-8
        addi    r6,r6,-8
-4:     li      r0,16                   /* Use the least common         */
+4:     li      r0,8                    /* Use the smallest common      */
                                        /* denominator cache line       */
                                        /* size.  This results in       */
                                        /* extra cache line flushes     */
@@ -1659,10 +1852,9 @@ __secondary_start_pmac_0:
 _GLOBAL(pmac_secondary_start)
        /* turn on 64-bit mode */
        bl      .enable_64b_mode
-       isync
 
        /* Copy some CPU settings from CPU 0 */
-       bl      .__restore_cpu_setup
+       bl      .__restore_cpu_ppc970
 
        /* pSeries do that early though I don't think we really need it */
        mfmsr   r3
@@ -1670,7 +1862,7 @@ _GLOBAL(pmac_secondary_start)
        mtmsrd  r3                      /* RI on */
 
        /* Set up a paca value for this processor. */
-       LOADADDR(r4, paca)               /* Get base vaddr of paca array        */
+       LOAD_REG_IMMEDIATE(r4, paca)    /* Get base vaddr of paca array */
        mulli   r13,r24,PACA_SIZE        /* Calculate vaddr of right paca */
        add     r13,r13,r4              /* for this processor.          */
        mtspr   SPRN_SPRG3,r13           /* Save vaddr of paca in SPRG3 */
@@ -1679,7 +1871,7 @@ _GLOBAL(pmac_secondary_start)
        ld      r1,PACAEMERGSP(r13)
        subi    r1,r1,STACK_FRAME_OVERHEAD
 
-       b       .__secondary_start
+       b       __secondary_start
 
 #endif /* CONFIG_PPC_PMAC */
 
@@ -1696,7 +1888,7 @@ _GLOBAL(pmac_secondary_start)
  *   r13   = paca virtual address
  *   SPRG3 = paca virtual address
  */
-_GLOBAL(__secondary_start)
+__secondary_start:
        /* Set thread priority to MEDIUM */
        HMT_MEDIUM
 
@@ -1707,7 +1899,7 @@ _GLOBAL(__secondary_start)
        bl      .early_setup_secondary
 
        /* Initialize the kernel stack.  Just a repeat for iSeries.      */
-       LOADADDR(r3,current_set)
+       LOAD_REG_ADDR(r3, current_set)
        sldi    r28,r24,3               /* get current_set[cpu#]         */
        ldx     r1,r3,r28
        addi    r1,r1,THREAD_SIZE-STACK_FRAME_OVERHEAD
@@ -1718,11 +1910,18 @@ _GLOBAL(__secondary_start)
        mtlr    r7
 
        /* enable MMU and jump to start_secondary */
-       LOADADDR(r3,.start_secondary_prolog)
-       SET_REG_TO_CONST(r4, MSR_KERNEL)
-#ifdef DO_SOFT_DISABLE
+       LOAD_REG_ADDR(r3, .start_secondary_prolog)
+       LOAD_REG_IMMEDIATE(r4, MSR_KERNEL)
+#ifdef CONFIG_PPC_ISERIES
+BEGIN_FW_FTR_SECTION
        ori     r4,r4,MSR_EE
+END_FW_FTR_SECTION_IFSET(FW_FEATURE_ISERIES)
 #endif
+BEGIN_FW_FTR_SECTION
+       stb     r7,PACASOFTIRQEN(r13)
+       stb     r7,PACAHARDIRQEN(r13)
+END_FW_FTR_SECTION_IFCLR(FW_FEATURE_ISERIES)
+
        mtspr   SPRN_SRR0,r3
        mtspr   SPRN_SRR1,r4
        rfid
@@ -1754,7 +1953,6 @@ _GLOBAL(enable_64b_mode)
        isync
        blr
 
-#ifdef CONFIG_PPC_MULTIPLATFORM
 /*
  * This is where the main kernel code starts.
  */
@@ -1768,8 +1966,8 @@ _STATIC(start_here_multiplatform)
         * be detached from the kernel completely. Besides, we need
         * to clear it now for kexec-style entry.
         */
-       LOADADDR(r11,__bss_stop)
-       LOADADDR(r8,__bss_start)
+       LOAD_REG_IMMEDIATE(r11,__bss_stop)
+       LOAD_REG_IMMEDIATE(r8,__bss_start)
        sub     r11,r11,r8              /* bss size                     */
        addi    r11,r11,7               /* round up to an even double word */
        rldicl. r11,r11,61,3            /* shift right by 3             */
@@ -1785,29 +1983,13 @@ _STATIC(start_here_multiplatform)
        ori     r6,r6,MSR_RI
        mtmsrd  r6                      /* RI on */
 
-#ifdef CONFIG_HMT
-       /* Start up the second thread on cpu 0 */
-       mfspr   r3,SPRN_PVR
-       srwi    r3,r3,16
-       cmpwi   r3,0x34                 /* Pulsar  */
-       beq     90f
-       cmpwi   r3,0x36                 /* Icestar */
-       beq     90f
-       cmpwi   r3,0x37                 /* SStar   */
-       beq     90f
-       b       91f                     /* HMT not supported */
-90:    li      r3,0
-       bl      .hmt_start_secondary
-91:
-#endif
-
        /* The following gets the stack and TOC set up with the regs */
        /* pointing to the real addr of the kernel stack.  This is   */
        /* all done to support the C function call below which sets  */
        /* up the htab.  This is done because we have relocated the  */
        /* kernel but are still running in real mode. */
 
-       LOADADDR(r3,init_thread_union)
+       LOAD_REG_IMMEDIATE(r3,init_thread_union)
        add     r3,r3,r26
 
        /* set up a stack pointer (physical address) */
@@ -1816,39 +1998,11 @@ _STATIC(start_here_multiplatform)
        stdu    r0,-STACK_FRAME_OVERHEAD(r1)
 
        /* set up the TOC (physical address) */
-       LOADADDR(r2,__toc_start)
+       LOAD_REG_IMMEDIATE(r2,__toc_start)
        addi    r2,r2,0x4000
        addi    r2,r2,0x4000
        add     r2,r2,r26
 
-       LOADADDR(r3,cpu_specs)
-       add     r3,r3,r26
-       LOADADDR(r4,cur_cpu_spec)
-       add     r4,r4,r26
-       mr      r5,r26
-       bl      .identify_cpu
-
-       /* Save some low level config HIDs of CPU0 to be copied to
-        * other CPUs later on, or used for suspend/resume
-        */
-       bl      .__save_cpu_setup
-       sync
-
-       /* Setup a valid physical PACA pointer in SPRG3 for early_setup
-        * note that boot_cpuid can always be 0 nowadays since there is
-        * nowhere it can be initialized differently before we reach this
-        * code
-        */
-       LOADADDR(r27, boot_cpuid)
-       add     r27,r27,r26
-       lwz     r27,0(r27)
-
-       LOADADDR(r24, paca)             /* Get base vaddr of paca array  */
-       mulli   r13,r27,PACA_SIZE       /* Calculate vaddr of right paca */
-       add     r13,r13,r24             /* for this processor.           */
-       add     r13,r13,r26             /* convert to physical addr      */
-       mtspr   SPRN_SPRG3,r13          /* PPPBBB: Temp... -Peter */
-       
        /* Do very early kernel initializations, including initial hash table,
         * stab and slb setup before we turn on relocation.     */
 
@@ -1856,13 +2010,12 @@ _STATIC(start_here_multiplatform)
        mr      r3,r31
        bl      .early_setup
 
-       LOADADDR(r3,.start_here_common)
-       SET_REG_TO_CONST(r4, MSR_KERNEL)
+       LOAD_REG_IMMEDIATE(r3, .start_here_common)
+       LOAD_REG_IMMEDIATE(r4, MSR_KERNEL)
        mtspr   SPRN_SRR0,r3
        mtspr   SPRN_SRR1,r4
        rfid
        b       .       /* prevent speculative execution */
-#endif /* CONFIG_PPC_MULTIPLATFORM */
        
        /* This is where all platforms converge execution */
 _STATIC(start_here_common)
@@ -1871,29 +2024,15 @@ _STATIC(start_here_common)
        /* The following code sets up the SP and TOC now that we are */
        /* running with translation enabled. */
 
-       LOADADDR(r3,init_thread_union)
+       LOAD_REG_IMMEDIATE(r3,init_thread_union)
 
        /* set up the stack */
        addi    r1,r3,THREAD_SIZE
        li      r0,0
        stdu    r0,-STACK_FRAME_OVERHEAD(r1)
 
-       /* Apply the CPUs-specific fixups (nop out sections not relevant
-        * to this CPU
-        */
-       li      r3,0
-       bl      .do_cpu_ftr_fixups
-
-       LOADADDR(r26, boot_cpuid)
-       lwz     r26,0(r26)
-
-       LOADADDR(r24, paca)             /* Get base vaddr of paca array  */
-       mulli   r13,r26,PACA_SIZE       /* Calculate vaddr of right paca */
-       add     r13,r13,r24             /* for this processor.           */
-       mtspr   SPRN_SPRG3,r13
-
        /* ptr to current */
-       LOADADDR(r4,init_task)
+       LOAD_REG_IMMEDIATE(r4, init_task)
        std     r4,PACACURRENT(r13)
 
        /* Load the TOC */
@@ -1904,87 +2043,23 @@ _STATIC(start_here_common)
 
        /* Load up the kernel context */
 5:
-#ifdef DO_SOFT_DISABLE
        li      r5,0
-       stb     r5,PACAPROCENABLED(r13) /* Soft Disabled */
+       stb     r5,PACASOFTIRQEN(r13)   /* Soft Disabled */
+#ifdef CONFIG_PPC_ISERIES
+BEGIN_FW_FTR_SECTION
        mfmsr   r5
        ori     r5,r5,MSR_EE            /* Hard Enabled */
        mtmsrd  r5
+END_FW_FTR_SECTION_IFSET(FW_FEATURE_ISERIES)
 #endif
+BEGIN_FW_FTR_SECTION
+       stb     r5,PACAHARDIRQEN(r13)
+END_FW_FTR_SECTION_IFCLR(FW_FEATURE_ISERIES)
 
        bl .start_kernel
 
-_GLOBAL(hmt_init)
-#ifdef CONFIG_HMT
-       LOADADDR(r5, hmt_thread_data)
-       mfspr   r7,SPRN_PVR
-       srwi    r7,r7,16
-       cmpwi   r7,0x34                 /* Pulsar  */
-       beq     90f
-       cmpwi   r7,0x36                 /* Icestar */
-       beq     91f
-       cmpwi   r7,0x37                 /* SStar   */
-       beq     91f
-       b       101f
-90:    mfspr   r6,SPRN_PIR
-       andi.   r6,r6,0x1f
-       b       92f
-91:    mfspr   r6,SPRN_PIR
-       andi.   r6,r6,0x3ff
-92:    sldi    r4,r24,3
-       stwx    r6,r5,r4
-       bl      .hmt_start_secondary
-       b       101f
-
-__hmt_secondary_hold:
-       LOADADDR(r5, hmt_thread_data)
-       clrldi  r5,r5,4
-       li      r7,0
-       mfspr   r6,SPRN_PIR
-       mfspr   r8,SPRN_PVR
-       srwi    r8,r8,16
-       cmpwi   r8,0x34
-       bne     93f
-       andi.   r6,r6,0x1f
-       b       103f
-93:    andi.   r6,r6,0x3f
-
-103:   lwzx    r8,r5,r7
-       cmpw    r8,r6
-       beq     104f
-       addi    r7,r7,8
-       b       103b
-
-104:   addi    r7,r7,4
-       lwzx    r9,r5,r7
-       mr      r24,r9
-101:
-#endif
-       mr      r3,r24
-       b       .pSeries_secondary_smp_init
-
-#ifdef CONFIG_HMT
-_GLOBAL(hmt_start_secondary)
-       LOADADDR(r4,__hmt_secondary_hold)
-       clrldi  r4,r4,4
-       mtspr   SPRN_NIADORM, r4
-       mfspr   r4, SPRN_MSRDORM
-       li      r5, -65
-       and     r4, r4, r5
-       mtspr   SPRN_MSRDORM, r4
-       lis     r4,0xffef
-       ori     r4,r4,0x7403
-       mtspr   SPRN_TSC, r4
-       li      r4,0x1f4
-       mtspr   SPRN_TST, r4
-       mfspr   r4, SPRN_HID0
-       ori     r4, r4, 0x1
-       mtspr   SPRN_HID0, r4
-       mfspr   r4, SPRN_CTRLF
-       oris    r4, r4, 0x40
-       mtspr   SPRN_CTRLT, r4
-       blr
-#endif
+       /* Not reached */
+       BUG_OPCODE
 
 /*
  * We put a few things here that have to be page-aligned.