SAFE public projects git trees. - safe/jmp/linux-2.6/blob - arch/mips/cavium-octeon/octeon-irq.c

   1 /*
   2  * This file is subject to the terms and conditions of the GNU General Public
   3  * License.  See the file "COPYING" in the main directory of this archive
   4  * for more details.
   5  *
   6  * Copyright (C) 2004-2008 Cavium Networks
   7  */
   8 #include <linux/irq.h>
   9 #include <linux/interrupt.h>
  10 #include <linux/smp.h>
  11
  12 #include <asm/octeon/octeon.h>
  13 #include <asm/octeon/cvmx-pexp-defs.h>
  14 #include <asm/octeon/cvmx-npi-defs.h>
  15
  16 DEFINE_RWLOCK(octeon_irq_ciu0_rwlock);
  17 DEFINE_RWLOCK(octeon_irq_ciu1_rwlock);
  18 DEFINE_SPINLOCK(octeon_irq_msi_lock);
  19
  20 static int octeon_coreid_for_cpu(int cpu)
  21 {
  22 #ifdef CONFIG_SMP
  23         return cpu_logical_map(cpu);
  24 #else
  25         return cvmx_get_core_num();
  26 #endif
  27 }
  28
  29 static void octeon_irq_core_ack(unsigned int irq)
  30 {
  31         unsigned int bit = irq - OCTEON_IRQ_SW0;
  32         /*
  33          * We don't need to disable IRQs to make these atomic since
  34          * they are already disabled earlier in the low level
  35          * interrupt code.
  36          */
  37         clear_c0_status(0x100 << bit);
  38         /* The two user interrupts must be cleared manually. */
  39         if (bit < 2)
  40                 clear_c0_cause(0x100 << bit);
  41 }
  42
  43 static void octeon_irq_core_eoi(unsigned int irq)
  44 {
  45         struct irq_desc *desc = irq_desc + irq;
  46         unsigned int bit = irq - OCTEON_IRQ_SW0;
  47         /*
  48          * If an IRQ is being processed while we are disabling it the
  49          * handler will attempt to unmask the interrupt after it has
  50          * been disabled.
  51          */
  52         if (desc->status & IRQ_DISABLED)
  53                 return;
  54         /*
  55          * We don't need to disable IRQs to make these atomic since
  56          * they are already disabled earlier in the low level
  57          * interrupt code.
  58          */
  59         set_c0_status(0x100 << bit);
  60 }
  61
  62 static void octeon_irq_core_enable(unsigned int irq)
  63 {
  64         unsigned long flags;
  65         unsigned int bit = irq - OCTEON_IRQ_SW0;
  66
  67         /*
  68          * We need to disable interrupts to make sure our updates are
  69          * atomic.
  70          */
  71         local_irq_save(flags);
  72         set_c0_status(0x100 << bit);
  73         local_irq_restore(flags);
  74 }
  75
  76 static void octeon_irq_core_disable_local(unsigned int irq)
  77 {
  78         unsigned long flags;
  79         unsigned int bit = irq - OCTEON_IRQ_SW0;
  80         /*
  81          * We need to disable interrupts to make sure our updates are
  82          * atomic.
  83          */
  84         local_irq_save(flags);
  85         clear_c0_status(0x100 << bit);
  86         local_irq_restore(flags);
  87 }
  88
  89 static void octeon_irq_core_disable(unsigned int irq)
  90 {
  91 #ifdef CONFIG_SMP
  92         on_each_cpu((void (*)(void *)) octeon_irq_core_disable_local,
  93                     (void *) (long) irq, 1);
  94 #else
  95         octeon_irq_core_disable_local(irq);
  96 #endif
  97 }
  98
  99 static struct irq_chip octeon_irq_chip_core = {
 100         .name = "Core",
 101         .enable = octeon_irq_core_enable,
 102         .disable = octeon_irq_core_disable,
 103         .ack = octeon_irq_core_ack,
 104         .eoi = octeon_irq_core_eoi,
 105 };
 106
 107
 108 static void octeon_irq_ciu0_ack(unsigned int irq)
 109 {
 110         /*
 111          * In order to avoid any locking accessing the CIU, we
 112          * acknowledge CIU interrupts by disabling all of them.  This
 113          * way we can use a per core register and avoid any out of
 114          * core locking requirements.  This has the side affect that
 115          * CIU interrupts can't be processed recursively.
 116          *
 117          * We don't need to disable IRQs to make these atomic since
 118          * they are already disabled earlier in the low level
 119          * interrupt code.
 120          */
 121         clear_c0_status(0x100 << 2);
 122 }
 123
 124 static void octeon_irq_ciu0_eoi(unsigned int irq)
 125 {
 126         /*
 127          * Enable all CIU interrupts again.  We don't need to disable
 128          * IRQs to make these atomic since they are already disabled
 129          * earlier in the low level interrupt code.
 130          */
 131         set_c0_status(0x100 << 2);
 132 }
 133
 134 static void octeon_irq_ciu0_enable(unsigned int irq)
 135 {
 136         int coreid = cvmx_get_core_num();
 137         unsigned long flags;
 138         uint64_t en0;
 139         int bit = irq - OCTEON_IRQ_WORKQ0;      /* Bit 0-63 of EN0 */
 140
 141         /*
 142          * A read lock is used here to make sure only one core is ever
 143          * updating the CIU enable bits at a time. During an enable
 144          * the cores don't interfere with each other. During a disable
 145          * the write lock stops any enables that might cause a
 146          * problem.
 147          */
 148         read_lock_irqsave(&octeon_irq_ciu0_rwlock, flags);
 149         en0 = cvmx_read_csr(CVMX_CIU_INTX_EN0(coreid * 2));
 150         en0 |= 1ull << bit;
 151         cvmx_write_csr(CVMX_CIU_INTX_EN0(coreid * 2), en0);
 152         cvmx_read_csr(CVMX_CIU_INTX_EN0(coreid * 2));
 153         read_unlock_irqrestore(&octeon_irq_ciu0_rwlock, flags);
 154 }
 155
 156 static void octeon_irq_ciu0_disable(unsigned int irq)
 157 {
 158         int bit = irq - OCTEON_IRQ_WORKQ0;      /* Bit 0-63 of EN0 */
 159         unsigned long flags;
 160         uint64_t en0;
 161         int cpu;
 162         write_lock_irqsave(&octeon_irq_ciu0_rwlock, flags);
 163         for_each_online_cpu(cpu) {
 164                 int coreid = octeon_coreid_for_cpu(cpu);
 165                 en0 = cvmx_read_csr(CVMX_CIU_INTX_EN0(coreid * 2));
 166                 en0 &= ~(1ull << bit);
 167                 cvmx_write_csr(CVMX_CIU_INTX_EN0(coreid * 2), en0);
 168         }
 169         /*
 170          * We need to do a read after the last update to make sure all
 171          * of them are done.
 172          */
 173         cvmx_read_csr(CVMX_CIU_INTX_EN0(cvmx_get_core_num() * 2));
 174         write_unlock_irqrestore(&octeon_irq_ciu0_rwlock, flags);
 175 }
 176
 177 /*
 178  * Enable the irq on the current core for chips that have the EN*_W1{S,C}
 179  * registers.
 180  */
 181 static void octeon_irq_ciu0_enable_v2(unsigned int irq)
 182 {
 183         int index = cvmx_get_core_num() * 2;
 184         u64 mask = 1ull << (irq - OCTEON_IRQ_WORKQ0);
 185
 186         cvmx_write_csr(CVMX_CIU_INTX_EN0_W1S(index), mask);
 187 }
 188
 189 /*
 190  * Disable the irq on the current core for chips that have the EN*_W1{S,C}
 191  * registers.
 192  */
 193 static void octeon_irq_ciu0_ack_v2(unsigned int irq)
 194 {
 195         int index = cvmx_get_core_num() * 2;
 196         u64 mask = 1ull << (irq - OCTEON_IRQ_WORKQ0);
 197
 198         cvmx_write_csr(CVMX_CIU_INTX_EN0_W1C(index), mask);
 199 }
 200
 201 /*
 202  * CIU timer type interrupts must be acknoleged by writing a '1' bit
 203  * to their sum0 bit.
 204  */
 205 static void octeon_irq_ciu0_timer_ack(unsigned int irq)
 206 {
 207         int index = cvmx_get_core_num() * 2;
 208         uint64_t mask = 1ull << (irq - OCTEON_IRQ_WORKQ0);
 209         cvmx_write_csr(CVMX_CIU_INTX_SUM0(index), mask);
 210 }
 211
 212 static void octeon_irq_ciu0_timer_ack_v1(unsigned int irq)
 213 {
 214         octeon_irq_ciu0_timer_ack(irq);
 215         octeon_irq_ciu0_ack(irq);
 216 }
 217
 218 static void octeon_irq_ciu0_timer_ack_v2(unsigned int irq)
 219 {
 220         octeon_irq_ciu0_timer_ack(irq);
 221         octeon_irq_ciu0_ack_v2(irq);
 222 }
 223
 224 /*
 225  * Enable the irq on the current core for chips that have the EN*_W1{S,C}
 226  * registers.
 227  */
 228 static void octeon_irq_ciu0_eoi_v2(unsigned int irq)
 229 {
 230         struct irq_desc *desc = irq_desc + irq;
 231         int index = cvmx_get_core_num() * 2;
 232         u64 mask = 1ull << (irq - OCTEON_IRQ_WORKQ0);
 233
 234         if ((desc->status & IRQ_DISABLED) == 0)
 235                 cvmx_write_csr(CVMX_CIU_INTX_EN0_W1S(index), mask);
 236 }
 237
 238 /*
 239  * Disable the irq on the all cores for chips that have the EN*_W1{S,C}
 240  * registers.
 241  */
 242 static void octeon_irq_ciu0_disable_all_v2(unsigned int irq)
 243 {
 244         u64 mask = 1ull << (irq - OCTEON_IRQ_WORKQ0);
 245         int index;
 246         int cpu;
 247         for_each_online_cpu(cpu) {
 248                 index = octeon_coreid_for_cpu(cpu) * 2;
 249                 cvmx_write_csr(CVMX_CIU_INTX_EN0_W1C(index), mask);
 250         }
 251 }
 252
 253 #ifdef CONFIG_SMP
 254 static int octeon_irq_ciu0_set_affinity(unsigned int irq, const struct cpumask *dest)
 255 {
 256         int cpu;
 257         unsigned long flags;
 258         int bit = irq - OCTEON_IRQ_WORKQ0;      /* Bit 0-63 of EN0 */
 259
 260         write_lock_irqsave(&octeon_irq_ciu0_rwlock, flags);
 261         for_each_online_cpu(cpu) {
 262                 int coreid = octeon_coreid_for_cpu(cpu);
 263                 uint64_t en0 =
 264                         cvmx_read_csr(CVMX_CIU_INTX_EN0(coreid * 2));
 265                 if (cpumask_test_cpu(cpu, dest))
 266                         en0 |= 1ull << bit;
 267                 else
 268                         en0 &= ~(1ull << bit);
 269                 cvmx_write_csr(CVMX_CIU_INTX_EN0(coreid * 2), en0);
 270         }
 271         /*
 272          * We need to do a read after the last update to make sure all
 273          * of them are done.
 274          */
 275         cvmx_read_csr(CVMX_CIU_INTX_EN0(cvmx_get_core_num() * 2));
 276         write_unlock_irqrestore(&octeon_irq_ciu0_rwlock, flags);
 277
 278         return 0;
 279 }
 280
 281 /*
 282  * Set affinity for the irq for chips that have the EN*_W1{S,C}
 283  * registers.
 284  */
 285 static int octeon_irq_ciu0_set_affinity_v2(unsigned int irq,
 286                                            const struct cpumask *dest)
 287 {
 288         int cpu;
 289         int index;
 290         u64 mask = 1ull << (irq - OCTEON_IRQ_WORKQ0);
 291         for_each_online_cpu(cpu) {
 292                 index = octeon_coreid_for_cpu(cpu) * 2;
 293                 if (cpumask_test_cpu(cpu, dest))
 294                         cvmx_write_csr(CVMX_CIU_INTX_EN0_W1S(index), mask);
 295                 else
 296                         cvmx_write_csr(CVMX_CIU_INTX_EN0_W1C(index), mask);
 297         }
 298         return 0;
 299 }
 300 #endif
 301
 302 /*
 303  * Newer octeon chips have support for lockless CIU operation.
 304  */
 305 static struct irq_chip octeon_irq_chip_ciu0_v2 = {
 306         .name = "CIU0",
 307         .enable = octeon_irq_ciu0_enable_v2,
 308         .disable = octeon_irq_ciu0_disable_all_v2,
 309         .ack = octeon_irq_ciu0_ack_v2,
 310         .eoi = octeon_irq_ciu0_eoi_v2,
 311 #ifdef CONFIG_SMP
 312         .set_affinity = octeon_irq_ciu0_set_affinity_v2,
 313 #endif
 314 };
 315
 316 static struct irq_chip octeon_irq_chip_ciu0 = {
 317         .name = "CIU0",
 318         .enable = octeon_irq_ciu0_enable,
 319         .disable = octeon_irq_ciu0_disable,
 320         .ack = octeon_irq_ciu0_ack,
 321         .eoi = octeon_irq_ciu0_eoi,
 322 #ifdef CONFIG_SMP
 323         .set_affinity = octeon_irq_ciu0_set_affinity,
 324 #endif
 325 };
 326
 327 static struct irq_chip octeon_irq_chip_ciu0_timer_v2 = {
 328         .name = "CIU0-T",
 329         .enable = octeon_irq_ciu0_enable_v2,
 330         .disable = octeon_irq_ciu0_disable_all_v2,
 331         .ack = octeon_irq_ciu0_timer_ack_v2,
 332         .eoi = octeon_irq_ciu0_eoi_v2,
 333 #ifdef CONFIG_SMP
 334         .set_affinity = octeon_irq_ciu0_set_affinity_v2,
 335 #endif
 336 };
 337
 338 static struct irq_chip octeon_irq_chip_ciu0_timer = {
 339         .name = "CIU0-T",
 340         .enable = octeon_irq_ciu0_enable,
 341         .disable = octeon_irq_ciu0_disable,
 342         .ack = octeon_irq_ciu0_timer_ack_v1,
 343         .eoi = octeon_irq_ciu0_eoi,
 344 #ifdef CONFIG_SMP
 345         .set_affinity = octeon_irq_ciu0_set_affinity,
 346 #endif
 347 };
 348
 349
 350 static void octeon_irq_ciu1_ack(unsigned int irq)
 351 {
 352         /*
 353          * In order to avoid any locking accessing the CIU, we
 354          * acknowledge CIU interrupts by disabling all of them.  This
 355          * way we can use a per core register and avoid any out of
 356          * core locking requirements.  This has the side affect that
 357          * CIU interrupts can't be processed recursively.  We don't
 358          * need to disable IRQs to make these atomic since they are
 359          * already disabled earlier in the low level interrupt code.
 360          */
 361         clear_c0_status(0x100 << 3);
 362 }
 363
 364 static void octeon_irq_ciu1_eoi(unsigned int irq)
 365 {
 366         /*
 367          * Enable all CIU interrupts again.  We don't need to disable
 368          * IRQs to make these atomic since they are already disabled
 369          * earlier in the low level interrupt code.
 370          */
 371         set_c0_status(0x100 << 3);
 372 }
 373
 374 static void octeon_irq_ciu1_enable(unsigned int irq)
 375 {
 376         int coreid = cvmx_get_core_num();
 377         unsigned long flags;
 378         uint64_t en1;
 379         int bit = irq - OCTEON_IRQ_WDOG0;       /* Bit 0-63 of EN1 */
 380
 381         /*
 382          * A read lock is used here to make sure only one core is ever
 383          * updating the CIU enable bits at a time.  During an enable
 384          * the cores don't interfere with each other.  During a disable
 385          * the write lock stops any enables that might cause a
 386          * problem.
 387          */
 388         read_lock_irqsave(&octeon_irq_ciu1_rwlock, flags);
 389         en1 = cvmx_read_csr(CVMX_CIU_INTX_EN1(coreid * 2 + 1));
 390         en1 |= 1ull << bit;
 391         cvmx_write_csr(CVMX_CIU_INTX_EN1(coreid * 2 + 1), en1);
 392         cvmx_read_csr(CVMX_CIU_INTX_EN1(coreid * 2 + 1));
 393         read_unlock_irqrestore(&octeon_irq_ciu1_rwlock, flags);
 394 }
 395
 396 static void octeon_irq_ciu1_disable(unsigned int irq)
 397 {
 398         int bit = irq - OCTEON_IRQ_WDOG0;       /* Bit 0-63 of EN1 */
 399         unsigned long flags;
 400         uint64_t en1;
 401         int cpu;
 402         write_lock_irqsave(&octeon_irq_ciu1_rwlock, flags);
 403         for_each_online_cpu(cpu) {
 404                 int coreid = octeon_coreid_for_cpu(cpu);
 405                 en1 = cvmx_read_csr(CVMX_CIU_INTX_EN1(coreid * 2 + 1));
 406                 en1 &= ~(1ull << bit);
 407                 cvmx_write_csr(CVMX_CIU_INTX_EN1(coreid * 2 + 1), en1);
 408         }
 409         /*
 410          * We need to do a read after the last update to make sure all
 411          * of them are done.
 412          */
 413         cvmx_read_csr(CVMX_CIU_INTX_EN1(cvmx_get_core_num() * 2 + 1));
 414         write_unlock_irqrestore(&octeon_irq_ciu1_rwlock, flags);
 415 }
 416
 417 /*
 418  * Enable the irq on the current core for chips that have the EN*_W1{S,C}
 419  * registers.
 420  */
 421 static void octeon_irq_ciu1_enable_v2(unsigned int irq)
 422 {
 423         int index = cvmx_get_core_num() * 2 + 1;
 424         u64 mask = 1ull << (irq - OCTEON_IRQ_WDOG0);
 425
 426         cvmx_write_csr(CVMX_CIU_INTX_EN1_W1S(index), mask);
 427 }
 428
 429 /*
 430  * Disable the irq on the current core for chips that have the EN*_W1{S,C}
 431  * registers.
 432  */
 433 static void octeon_irq_ciu1_ack_v2(unsigned int irq)
 434 {
 435         int index = cvmx_get_core_num() * 2 + 1;
 436         u64 mask = 1ull << (irq - OCTEON_IRQ_WDOG0);
 437
 438         cvmx_write_csr(CVMX_CIU_INTX_EN1_W1C(index), mask);
 439 }
 440
 441 /*
 442  * Enable the irq on the current core for chips that have the EN*_W1{S,C}
 443  * registers.
 444  */
 445 static void octeon_irq_ciu1_eoi_v2(unsigned int irq)
 446 {
 447         struct irq_desc *desc = irq_desc + irq;
 448         int index = cvmx_get_core_num() * 2 + 1;
 449         u64 mask = 1ull << (irq - OCTEON_IRQ_WDOG0);
 450
 451         if ((desc->status & IRQ_DISABLED) == 0)
 452                 cvmx_write_csr(CVMX_CIU_INTX_EN1_W1S(index), mask);
 453 }
 454
 455 /*
 456  * Disable the irq on the all cores for chips that have the EN*_W1{S,C}
 457  * registers.
 458  */
 459 static void octeon_irq_ciu1_disable_all_v2(unsigned int irq)
 460 {
 461         u64 mask = 1ull << (irq - OCTEON_IRQ_WDOG0);
 462         int index;
 463         int cpu;
 464         for_each_online_cpu(cpu) {
 465                 index = octeon_coreid_for_cpu(cpu) * 2 + 1;
 466                 cvmx_write_csr(CVMX_CIU_INTX_EN1_W1C(index), mask);
 467         }
 468 }
 469
 470 #ifdef CONFIG_SMP
 471 static int octeon_irq_ciu1_set_affinity(unsigned int irq,
 472                                         const struct cpumask *dest)
 473 {
 474         int cpu;
 475         unsigned long flags;
 476         int bit = irq - OCTEON_IRQ_WDOG0;       /* Bit 0-63 of EN1 */
 477
 478         write_lock_irqsave(&octeon_irq_ciu1_rwlock, flags);
 479         for_each_online_cpu(cpu) {
 480                 int coreid = octeon_coreid_for_cpu(cpu);
 481                 uint64_t en1 =
 482                         cvmx_read_csr(CVMX_CIU_INTX_EN1
 483                                 (coreid * 2 + 1));
 484                 if (cpumask_test_cpu(cpu, dest))
 485                         en1 |= 1ull << bit;
 486                 else
 487                         en1 &= ~(1ull << bit);
 488                 cvmx_write_csr(CVMX_CIU_INTX_EN1(coreid * 2 + 1), en1);
 489         }
 490         /*
 491          * We need to do a read after the last update to make sure all
 492          * of them are done.
 493          */
 494         cvmx_read_csr(CVMX_CIU_INTX_EN1(cvmx_get_core_num() * 2 + 1));
 495         write_unlock_irqrestore(&octeon_irq_ciu1_rwlock, flags);
 496
 497         return 0;
 498 }
 499
 500 /*
 501  * Set affinity for the irq for chips that have the EN*_W1{S,C}
 502  * registers.
 503  */
 504 static int octeon_irq_ciu1_set_affinity_v2(unsigned int irq,
 505                                            const struct cpumask *dest)
 506 {
 507         int cpu;
 508         int index;
 509         u64 mask = 1ull << (irq - OCTEON_IRQ_WDOG0);
 510         for_each_online_cpu(cpu) {
 511                 index = octeon_coreid_for_cpu(cpu) * 2 + 1;
 512                 if (cpumask_test_cpu(cpu, dest))
 513                         cvmx_write_csr(CVMX_CIU_INTX_EN1_W1S(index), mask);
 514                 else
 515                         cvmx_write_csr(CVMX_CIU_INTX_EN1_W1C(index), mask);
 516         }
 517         return 0;
 518 }
 519 #endif
 520
 521 /*
 522  * Newer octeon chips have support for lockless CIU operation.
 523  */
 524 static struct irq_chip octeon_irq_chip_ciu1_v2 = {
 525         .name = "CIU0",
 526         .enable = octeon_irq_ciu1_enable_v2,
 527         .disable = octeon_irq_ciu1_disable_all_v2,
 528         .ack = octeon_irq_ciu1_ack_v2,
 529         .eoi = octeon_irq_ciu1_eoi_v2,
 530 #ifdef CONFIG_SMP
 531         .set_affinity = octeon_irq_ciu1_set_affinity_v2,
 532 #endif
 533 };
 534
 535 static struct irq_chip octeon_irq_chip_ciu1 = {
 536         .name = "CIU1",
 537         .enable = octeon_irq_ciu1_enable,
 538         .disable = octeon_irq_ciu1_disable,
 539         .ack = octeon_irq_ciu1_ack,
 540         .eoi = octeon_irq_ciu1_eoi,
 541 #ifdef CONFIG_SMP
 542         .set_affinity = octeon_irq_ciu1_set_affinity,
 543 #endif
 544 };
 545
 546 #ifdef CONFIG_PCI_MSI
 547
 548 static void octeon_irq_msi_ack(unsigned int irq)
 549 {
 550         if (!octeon_has_feature(OCTEON_FEATURE_PCIE)) {
 551                 /* These chips have PCI */
 552                 cvmx_write_csr(CVMX_NPI_NPI_MSI_RCV,
 553                                1ull << (irq - OCTEON_IRQ_MSI_BIT0));
 554         } else {
 555                 /*
 556                  * These chips have PCIe. Thankfully the ACK doesn't
 557                  * need any locking.
 558                  */
 559                 cvmx_write_csr(CVMX_PEXP_NPEI_MSI_RCV0,
 560                                1ull << (irq - OCTEON_IRQ_MSI_BIT0));
 561         }
 562 }
 563
 564 static void octeon_irq_msi_eoi(unsigned int irq)
 565 {
 566         /* Nothing needed */
 567 }
 568
 569 static void octeon_irq_msi_enable(unsigned int irq)
 570 {
 571         if (!octeon_has_feature(OCTEON_FEATURE_PCIE)) {
 572                 /*
 573                  * Octeon PCI doesn't have the ability to mask/unmask
 574                  * MSI interrupts individually.  Instead of
 575                  * masking/unmasking them in groups of 16, we simple
 576                  * assume MSI devices are well behaved.  MSI
 577                  * interrupts are always enable and the ACK is assumed
 578                  * to be enough.
 579                  */
 580         } else {
 581                 /* These chips have PCIe.  Note that we only support
 582                  * the first 64 MSI interrupts.  Unfortunately all the
 583                  * MSI enables are in the same register.  We use
 584                  * MSI0's lock to control access to them all.
 585                  */
 586                 uint64_t en;
 587                 unsigned long flags;
 588                 spin_lock_irqsave(&octeon_irq_msi_lock, flags);
 589                 en = cvmx_read_csr(CVMX_PEXP_NPEI_MSI_ENB0);
 590                 en |= 1ull << (irq - OCTEON_IRQ_MSI_BIT0);
 591                 cvmx_write_csr(CVMX_PEXP_NPEI_MSI_ENB0, en);
 592                 cvmx_read_csr(CVMX_PEXP_NPEI_MSI_ENB0);
 593                 spin_unlock_irqrestore(&octeon_irq_msi_lock, flags);
 594         }
 595 }
 596
 597 static void octeon_irq_msi_disable(unsigned int irq)
 598 {
 599         if (!octeon_has_feature(OCTEON_FEATURE_PCIE)) {
 600                 /* See comment in enable */
 601         } else {
 602                 /*
 603                  * These chips have PCIe.  Note that we only support
 604                  * the first 64 MSI interrupts.  Unfortunately all the
 605                  * MSI enables are in the same register.  We use
 606                  * MSI0's lock to control access to them all.
 607                  */
 608                 uint64_t en;
 609                 unsigned long flags;
 610                 spin_lock_irqsave(&octeon_irq_msi_lock, flags);
 611                 en = cvmx_read_csr(CVMX_PEXP_NPEI_MSI_ENB0);
 612                 en &= ~(1ull << (irq - OCTEON_IRQ_MSI_BIT0));
 613                 cvmx_write_csr(CVMX_PEXP_NPEI_MSI_ENB0, en);
 614                 cvmx_read_csr(CVMX_PEXP_NPEI_MSI_ENB0);
 615                 spin_unlock_irqrestore(&octeon_irq_msi_lock, flags);
 616         }
 617 }
 618
 619 static struct irq_chip octeon_irq_chip_msi = {
 620         .name = "MSI",
 621         .enable = octeon_irq_msi_enable,
 622         .disable = octeon_irq_msi_disable,
 623         .ack = octeon_irq_msi_ack,
 624         .eoi = octeon_irq_msi_eoi,
 625 };
 626 #endif
 627
 628 void __init arch_init_irq(void)
 629 {
 630         int irq;
 631         struct irq_chip *chip0;
 632         struct irq_chip *chip0_timer;
 633         struct irq_chip *chip1;
 634
 635 #ifdef CONFIG_SMP
 636         /* Set the default affinity to the boot cpu. */
 637         cpumask_clear(irq_default_affinity);
 638         cpumask_set_cpu(smp_processor_id(), irq_default_affinity);
 639 #endif
 640
 641         if (NR_IRQS < OCTEON_IRQ_LAST)
 642                 pr_err("octeon_irq_init: NR_IRQS is set too low\n");
 643
 644         if (OCTEON_IS_MODEL(OCTEON_CN58XX_PASS2_X) ||
 645             OCTEON_IS_MODEL(OCTEON_CN56XX_PASS2_X) ||
 646             OCTEON_IS_MODEL(OCTEON_CN52XX_PASS2_X)) {
 647                 chip0 = &octeon_irq_chip_ciu0_v2;
 648                 chip0_timer = &octeon_irq_chip_ciu0_timer_v2;
 649                 chip1 = &octeon_irq_chip_ciu1_v2;
 650         } else {
 651                 chip0 = &octeon_irq_chip_ciu0;
 652                 chip0_timer = &octeon_irq_chip_ciu0_timer;
 653                 chip1 = &octeon_irq_chip_ciu1;
 654         }
 655
 656         /* 0 - 15 reserved for i8259 master and slave controller. */
 657
 658         /* 17 - 23 Mips internal */
 659         for (irq = OCTEON_IRQ_SW0; irq <= OCTEON_IRQ_TIMER; irq++) {
 660                 set_irq_chip_and_handler(irq, &octeon_irq_chip_core,
 661                                          handle_percpu_irq);
 662         }
 663
 664         /* 24 - 87 CIU_INT_SUM0 */
 665         for (irq = OCTEON_IRQ_WORKQ0; irq <= OCTEON_IRQ_BOOTDMA; irq++) {
 666                 switch (irq) {
 667                 case OCTEON_IRQ_GMX_DRP0:
 668                 case OCTEON_IRQ_GMX_DRP1:
 669                 case OCTEON_IRQ_IPD_DRP:
 670                 case OCTEON_IRQ_KEY_ZERO:
 671                 case OCTEON_IRQ_TIMER0:
 672                 case OCTEON_IRQ_TIMER1:
 673                 case OCTEON_IRQ_TIMER2:
 674                 case OCTEON_IRQ_TIMER3:
 675                         set_irq_chip_and_handler(irq, chip0_timer, handle_percpu_irq);
 676                         break;
 677                 default:
 678                         set_irq_chip_and_handler(irq, chip0, handle_percpu_irq);
 679                         break;
 680                 }
 681         }
 682
 683         /* 88 - 151 CIU_INT_SUM1 */
 684         for (irq = OCTEON_IRQ_WDOG0; irq <= OCTEON_IRQ_RESERVED151; irq++) {
 685                 set_irq_chip_and_handler(irq, chip1, handle_percpu_irq);
 686         }
 687
 688 #ifdef CONFIG_PCI_MSI
 689         /* 152 - 215 PCI/PCIe MSI interrupts */
 690         for (irq = OCTEON_IRQ_MSI_BIT0; irq <= OCTEON_IRQ_MSI_BIT63; irq++) {
 691                 set_irq_chip_and_handler(irq, &octeon_irq_chip_msi,
 692                                          handle_percpu_irq);
 693         }
 694 #endif
 695         set_c0_status(0x300 << 2);
 696 }
 697
 698 asmlinkage void plat_irq_dispatch(void)
 699 {
 700         const unsigned long core_id = cvmx_get_core_num();
 701         const uint64_t ciu_sum0_address = CVMX_CIU_INTX_SUM0(core_id * 2);
 702         const uint64_t ciu_en0_address = CVMX_CIU_INTX_EN0(core_id * 2);
 703         const uint64_t ciu_sum1_address = CVMX_CIU_INT_SUM1;
 704         const uint64_t ciu_en1_address = CVMX_CIU_INTX_EN1(core_id * 2 + 1);
 705         unsigned long cop0_cause;
 706         unsigned long cop0_status;
 707         uint64_t ciu_en;
 708         uint64_t ciu_sum;
 709
 710         while (1) {
 711                 cop0_cause = read_c0_cause();
 712                 cop0_status = read_c0_status();
 713                 cop0_cause &= cop0_status;
 714                 cop0_cause &= ST0_IM;
 715
 716                 if (unlikely(cop0_cause & STATUSF_IP2)) {
 717                         ciu_sum = cvmx_read_csr(ciu_sum0_address);
 718                         ciu_en = cvmx_read_csr(ciu_en0_address);
 719                         ciu_sum &= ciu_en;
 720                         if (likely(ciu_sum))
 721                                 do_IRQ(fls64(ciu_sum) + OCTEON_IRQ_WORKQ0 - 1);
 722                         else
 723                                 spurious_interrupt();
 724                 } else if (unlikely(cop0_cause & STATUSF_IP3)) {
 725                         ciu_sum = cvmx_read_csr(ciu_sum1_address);
 726                         ciu_en = cvmx_read_csr(ciu_en1_address);
 727                         ciu_sum &= ciu_en;
 728                         if (likely(ciu_sum))
 729                                 do_IRQ(fls64(ciu_sum) + OCTEON_IRQ_WDOG0 - 1);
 730                         else
 731                                 spurious_interrupt();
 732                 } else if (likely(cop0_cause)) {
 733                         do_IRQ(fls(cop0_cause) - 9 + MIPS_CPU_IRQ_BASE);
 734                 } else {
 735                         break;
 736                 }
 737         }
 738 }
 739
 740 #ifdef CONFIG_HOTPLUG_CPU
 741 static int is_irq_enabled_on_cpu(unsigned int irq, unsigned int cpu)
 742 {
 743         unsigned int isset;
 744         int coreid = octeon_coreid_for_cpu(cpu);
 745         int bit = (irq < OCTEON_IRQ_WDOG0) ?
 746                    irq - OCTEON_IRQ_WORKQ0 : irq - OCTEON_IRQ_WDOG0;
 747        if (irq < 64) {
 748                 isset = (cvmx_read_csr(CVMX_CIU_INTX_EN0(coreid * 2)) &
 749                         (1ull << bit)) >> bit;
 750        } else {
 751                isset = (cvmx_read_csr(CVMX_CIU_INTX_EN1(coreid * 2 + 1)) &
 752                         (1ull << bit)) >> bit;
 753        }
 754        return isset;
 755 }
 756
 757 void fixup_irqs(void)
 758 {
 759        int irq;
 760
 761         for (irq = OCTEON_IRQ_SW0; irq <= OCTEON_IRQ_TIMER; irq++)
 762                 octeon_irq_core_disable_local(irq);
 763
 764         for (irq = OCTEON_IRQ_WORKQ0; irq <= OCTEON_IRQ_GPIO15; irq++) {
 765                 if (is_irq_enabled_on_cpu(irq, smp_processor_id())) {
 766                         /* ciu irq migrates to next cpu */
 767                         octeon_irq_chip_ciu0.disable(irq);
 768                         octeon_irq_ciu0_set_affinity(irq, &cpu_online_map);
 769                 }
 770         }
 771
 772 #if 0
 773         for (irq = OCTEON_IRQ_MBOX0; irq <= OCTEON_IRQ_MBOX1; irq++)
 774                 octeon_irq_mailbox_mask(irq);
 775 #endif
 776         for (irq = OCTEON_IRQ_UART0; irq <= OCTEON_IRQ_BOOTDMA; irq++) {
 777                 if (is_irq_enabled_on_cpu(irq, smp_processor_id())) {
 778                         /* ciu irq migrates to next cpu */
 779                         octeon_irq_chip_ciu0.disable(irq);
 780                         octeon_irq_ciu0_set_affinity(irq, &cpu_online_map);
 781                 }
 782         }
 783
 784         for (irq = OCTEON_IRQ_UART2; irq <= OCTEON_IRQ_RESERVED135; irq++) {
 785                 if (is_irq_enabled_on_cpu(irq, smp_processor_id())) {
 786                         /* ciu irq migrates to next cpu */
 787                         octeon_irq_chip_ciu1.disable(irq);
 788                         octeon_irq_ciu1_set_affinity(irq, &cpu_online_map);
 789                 }
 790         }
 791 }
 792
 793 #endif /* CONFIG_HOTPLUG_CPU */