SAFE public projects git trees. - safe/jmp/linux-2.6/blob - arch/sparc/kernel/perf_event.c

   1 /* Performance event support for sparc64.
   2  *
   3  * Copyright (C) 2009 David S. Miller <davem@davemloft.net>
   4  *
   5  * This code is based almost entirely upon the x86 perf event
   6  * code, which is:
   7  *
   8  *  Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de>
   9  *  Copyright (C) 2008-2009 Red Hat, Inc., Ingo Molnar
  10  *  Copyright (C) 2009 Jaswinder Singh Rajput
  11  *  Copyright (C) 2009 Advanced Micro Devices, Inc., Robert Richter
  12  *  Copyright (C) 2008-2009 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com>
  13  */
  14
  15 #include <linux/perf_event.h>
  16 #include <linux/kprobes.h>
  17 #include <linux/kernel.h>
  18 #include <linux/kdebug.h>
  19 #include <linux/mutex.h>
  20
  21 #include <asm/cpudata.h>
  22 #include <asm/atomic.h>
  23 #include <asm/nmi.h>
  24 #include <asm/pcr.h>
  25
  26 /* Sparc64 chips have two performance counters, 32-bits each, with
  27  * overflow interrupts generated on transition from 0xffffffff to 0.
  28  * The counters are accessed in one go using a 64-bit register.
  29  *
  30  * Both counters are controlled using a single control register.  The
  31  * only way to stop all sampling is to clear all of the context (user,
  32  * supervisor, hypervisor) sampling enable bits.  But these bits apply
  33  * to both counters, thus the two counters can't be enabled/disabled
  34  * individually.
  35  *
  36  * The control register has two event fields, one for each of the two
  37  * counters.  It's thus nearly impossible to have one counter going
  38  * while keeping the other one stopped.  Therefore it is possible to
  39  * get overflow interrupts for counters not currently "in use" and
  40  * that condition must be checked in the overflow interrupt handler.
  41  *
  42  * So we use a hack, in that we program inactive counters with the
  43  * "sw_count0" and "sw_count1" events.  These count how many times
  44  * the instruction "sethi %hi(0xfc000), %g0" is executed.  It's an
  45  * unusual way to encode a NOP and therefore will not trigger in
  46  * normal code.
  47  */
  48
  49 #define MAX_HWEVENTS                    2
  50 #define MAX_PERIOD                      ((1UL << 32) - 1)
  51
  52 #define PIC_UPPER_INDEX                 0
  53 #define PIC_LOWER_INDEX                 1
  54
  55 struct cpu_hw_events {
  56         struct perf_event       *events[MAX_HWEVENTS];
  57         unsigned long           used_mask[BITS_TO_LONGS(MAX_HWEVENTS)];
  58         unsigned long           active_mask[BITS_TO_LONGS(MAX_HWEVENTS)];
  59         int enabled;
  60 };
  61 DEFINE_PER_CPU(struct cpu_hw_events, cpu_hw_events) = { .enabled = 1, };
  62
  63 struct perf_event_map {
  64         u16     encoding;
  65         u8      pic_mask;
  66 #define PIC_NONE        0x00
  67 #define PIC_UPPER       0x01
  68 #define PIC_LOWER       0x02
  69 };
  70
  71 #define C(x) PERF_COUNT_HW_CACHE_##x
  72
  73 #define CACHE_OP_UNSUPPORTED    0xfffe
  74 #define CACHE_OP_NONSENSE       0xffff
  75
  76 typedef struct perf_event_map cache_map_t
  77                                 [PERF_COUNT_HW_CACHE_MAX]
  78                                 [PERF_COUNT_HW_CACHE_OP_MAX]
  79                                 [PERF_COUNT_HW_CACHE_RESULT_MAX];
  80
  81 struct sparc_pmu {
  82         const struct perf_event_map     *(*event_map)(int);
  83         const cache_map_t               *cache_map;
  84         int                             max_events;
  85         int                             upper_shift;
  86         int                             lower_shift;
  87         int                             event_mask;
  88         int                             hv_bit;
  89         int                             irq_bit;
  90         int                             upper_nop;
  91         int                             lower_nop;
  92 };
  93
  94 static const struct perf_event_map ultra3_perfmon_event_map[] = {
  95         [PERF_COUNT_HW_CPU_CYCLES] = { 0x0000, PIC_UPPER | PIC_LOWER },
  96         [PERF_COUNT_HW_INSTRUCTIONS] = { 0x0001, PIC_UPPER | PIC_LOWER },
  97         [PERF_COUNT_HW_CACHE_REFERENCES] = { 0x0009, PIC_LOWER },
  98         [PERF_COUNT_HW_CACHE_MISSES] = { 0x0009, PIC_UPPER },
  99 };
 100
 101 static const struct perf_event_map *ultra3_event_map(int event_id)
 102 {
 103         return &ultra3_perfmon_event_map[event_id];
 104 }
 105
 106 static const cache_map_t ultra3_cache_map = {
 107 [C(L1D)] = {
 108         [C(OP_READ)] = {
 109                 [C(RESULT_ACCESS)] = { 0x09, PIC_LOWER, },
 110                 [C(RESULT_MISS)] = { 0x09, PIC_UPPER, },
 111         },
 112         [C(OP_WRITE)] = {
 113                 [C(RESULT_ACCESS)] = { 0x0a, PIC_LOWER },
 114                 [C(RESULT_MISS)] = { 0x0a, PIC_UPPER },
 115         },
 116         [C(OP_PREFETCH)] = {
 117                 [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED },
 118                 [C(RESULT_MISS)] = { CACHE_OP_UNSUPPORTED },
 119         },
 120 },
 121 [C(L1I)] = {
 122         [C(OP_READ)] = {
 123                 [C(RESULT_ACCESS)] = { 0x09, PIC_LOWER, },
 124                 [C(RESULT_MISS)] = { 0x09, PIC_UPPER, },
 125         },
 126         [ C(OP_WRITE) ] = {
 127                 [ C(RESULT_ACCESS) ] = { CACHE_OP_NONSENSE },
 128                 [ C(RESULT_MISS)   ] = { CACHE_OP_NONSENSE },
 129         },
 130         [ C(OP_PREFETCH) ] = {
 131                 [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
 132                 [ C(RESULT_MISS)   ] = { CACHE_OP_UNSUPPORTED },
 133         },
 134 },
 135 [C(LL)] = {
 136         [C(OP_READ)] = {
 137                 [C(RESULT_ACCESS)] = { 0x0c, PIC_LOWER, },
 138                 [C(RESULT_MISS)] = { 0x0c, PIC_UPPER, },
 139         },
 140         [C(OP_WRITE)] = {
 141                 [C(RESULT_ACCESS)] = { 0x0c, PIC_LOWER },
 142                 [C(RESULT_MISS)] = { 0x0c, PIC_UPPER },
 143         },
 144         [C(OP_PREFETCH)] = {
 145                 [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED },
 146                 [C(RESULT_MISS)] = { CACHE_OP_UNSUPPORTED },
 147         },
 148 },
 149 [C(DTLB)] = {
 150         [C(OP_READ)] = {
 151                 [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED },
 152                 [C(RESULT_MISS)] = { 0x12, PIC_UPPER, },
 153         },
 154         [ C(OP_WRITE) ] = {
 155                 [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
 156                 [ C(RESULT_MISS)   ] = { CACHE_OP_UNSUPPORTED },
 157         },
 158         [ C(OP_PREFETCH) ] = {
 159                 [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
 160                 [ C(RESULT_MISS)   ] = { CACHE_OP_UNSUPPORTED },
 161         },
 162 },
 163 [C(ITLB)] = {
 164         [C(OP_READ)] = {
 165                 [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED },
 166                 [C(RESULT_MISS)] = { 0x11, PIC_UPPER, },
 167         },
 168         [ C(OP_WRITE) ] = {
 169                 [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
 170                 [ C(RESULT_MISS)   ] = { CACHE_OP_UNSUPPORTED },
 171         },
 172         [ C(OP_PREFETCH) ] = {
 173                 [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
 174                 [ C(RESULT_MISS)   ] = { CACHE_OP_UNSUPPORTED },
 175         },
 176 },
 177 [C(BPU)] = {
 178         [C(OP_READ)] = {
 179                 [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED },
 180                 [C(RESULT_MISS)] = { CACHE_OP_UNSUPPORTED },
 181         },
 182         [ C(OP_WRITE) ] = {
 183                 [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
 184                 [ C(RESULT_MISS)   ] = { CACHE_OP_UNSUPPORTED },
 185         },
 186         [ C(OP_PREFETCH) ] = {
 187                 [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
 188                 [ C(RESULT_MISS)   ] = { CACHE_OP_UNSUPPORTED },
 189         },
 190 },
 191 };
 192
 193 static const struct sparc_pmu ultra3_pmu = {
 194         .event_map      = ultra3_event_map,
 195         .cache_map      = &ultra3_cache_map,
 196         .max_events     = ARRAY_SIZE(ultra3_perfmon_event_map),
 197         .upper_shift    = 11,
 198         .lower_shift    = 4,
 199         .event_mask     = 0x3f,
 200         .upper_nop      = 0x1c,
 201         .lower_nop      = 0x14,
 202 };
 203
 204 static const struct perf_event_map niagara2_perfmon_event_map[] = {
 205         [PERF_COUNT_HW_CPU_CYCLES] = { 0x02ff, PIC_UPPER | PIC_LOWER },
 206         [PERF_COUNT_HW_INSTRUCTIONS] = { 0x02ff, PIC_UPPER | PIC_LOWER },
 207         [PERF_COUNT_HW_CACHE_REFERENCES] = { 0x0208, PIC_UPPER | PIC_LOWER },
 208         [PERF_COUNT_HW_CACHE_MISSES] = { 0x0302, PIC_UPPER | PIC_LOWER },
 209         [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = { 0x0201, PIC_UPPER | PIC_LOWER },
 210         [PERF_COUNT_HW_BRANCH_MISSES] = { 0x0202, PIC_UPPER | PIC_LOWER },
 211 };
 212
 213 static const struct perf_event_map *niagara2_event_map(int event_id)
 214 {
 215         return &niagara2_perfmon_event_map[event_id];
 216 }
 217
 218 static const cache_map_t niagara2_cache_map = {
 219 [C(L1D)] = {
 220         [C(OP_READ)] = {
 221                 [C(RESULT_ACCESS)] = { 0x0208, PIC_UPPER | PIC_LOWER, },
 222                 [C(RESULT_MISS)] = { 0x0302, PIC_UPPER | PIC_LOWER, },
 223         },
 224         [C(OP_WRITE)] = {
 225                 [C(RESULT_ACCESS)] = { 0x0210, PIC_UPPER | PIC_LOWER, },
 226                 [C(RESULT_MISS)] = { 0x0302, PIC_UPPER | PIC_LOWER, },
 227         },
 228         [C(OP_PREFETCH)] = {
 229                 [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED },
 230                 [C(RESULT_MISS)] = { CACHE_OP_UNSUPPORTED },
 231         },
 232 },
 233 [C(L1I)] = {
 234         [C(OP_READ)] = {
 235                 [C(RESULT_ACCESS)] = { 0x02ff, PIC_UPPER | PIC_LOWER, },
 236                 [C(RESULT_MISS)] = { 0x0301, PIC_UPPER | PIC_LOWER, },
 237         },
 238         [ C(OP_WRITE) ] = {
 239                 [ C(RESULT_ACCESS) ] = { CACHE_OP_NONSENSE },
 240                 [ C(RESULT_MISS)   ] = { CACHE_OP_NONSENSE },
 241         },
 242         [ C(OP_PREFETCH) ] = {
 243                 [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
 244                 [ C(RESULT_MISS)   ] = { CACHE_OP_UNSUPPORTED },
 245         },
 246 },
 247 [C(LL)] = {
 248         [C(OP_READ)] = {
 249                 [C(RESULT_ACCESS)] = { 0x0208, PIC_UPPER | PIC_LOWER, },
 250                 [C(RESULT_MISS)] = { 0x0330, PIC_UPPER | PIC_LOWER, },
 251         },
 252         [C(OP_WRITE)] = {
 253                 [C(RESULT_ACCESS)] = { 0x0210, PIC_UPPER | PIC_LOWER, },
 254                 [C(RESULT_MISS)] = { 0x0320, PIC_UPPER | PIC_LOWER, },
 255         },
 256         [C(OP_PREFETCH)] = {
 257                 [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED },
 258                 [C(RESULT_MISS)] = { CACHE_OP_UNSUPPORTED },
 259         },
 260 },
 261 [C(DTLB)] = {
 262         [C(OP_READ)] = {
 263                 [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED },
 264                 [C(RESULT_MISS)] = { 0x0b08, PIC_UPPER | PIC_LOWER, },
 265         },
 266         [ C(OP_WRITE) ] = {
 267                 [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
 268                 [ C(RESULT_MISS)   ] = { CACHE_OP_UNSUPPORTED },
 269         },
 270         [ C(OP_PREFETCH) ] = {
 271                 [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
 272                 [ C(RESULT_MISS)   ] = { CACHE_OP_UNSUPPORTED },
 273         },
 274 },
 275 [C(ITLB)] = {
 276         [C(OP_READ)] = {
 277                 [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED },
 278                 [C(RESULT_MISS)] = { 0xb04, PIC_UPPER | PIC_LOWER, },
 279         },
 280         [ C(OP_WRITE) ] = {
 281                 [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
 282                 [ C(RESULT_MISS)   ] = { CACHE_OP_UNSUPPORTED },
 283         },
 284         [ C(OP_PREFETCH) ] = {
 285                 [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
 286                 [ C(RESULT_MISS)   ] = { CACHE_OP_UNSUPPORTED },
 287         },
 288 },
 289 [C(BPU)] = {
 290         [C(OP_READ)] = {
 291                 [C(RESULT_ACCESS)] = { CACHE_OP_UNSUPPORTED },
 292                 [C(RESULT_MISS)] = { CACHE_OP_UNSUPPORTED },
 293         },
 294         [ C(OP_WRITE) ] = {
 295                 [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
 296                 [ C(RESULT_MISS)   ] = { CACHE_OP_UNSUPPORTED },
 297         },
 298         [ C(OP_PREFETCH) ] = {
 299                 [ C(RESULT_ACCESS) ] = { CACHE_OP_UNSUPPORTED },
 300                 [ C(RESULT_MISS)   ] = { CACHE_OP_UNSUPPORTED },
 301         },
 302 },
 303 };
 304
 305 static const struct sparc_pmu niagara2_pmu = {
 306         .event_map      = niagara2_event_map,
 307         .cache_map      = &niagara2_cache_map,
 308         .max_events     = ARRAY_SIZE(niagara2_perfmon_event_map),
 309         .upper_shift    = 19,
 310         .lower_shift    = 6,
 311         .event_mask     = 0xfff,
 312         .hv_bit         = 0x8,
 313         .irq_bit        = 0x03,
 314         .upper_nop      = 0x220,
 315         .lower_nop      = 0x220,
 316 };
 317
 318 static const struct sparc_pmu *sparc_pmu __read_mostly;
 319
 320 static u64 event_encoding(u64 event_id, int idx)
 321 {
 322         if (idx == PIC_UPPER_INDEX)
 323                 event_id <<= sparc_pmu->upper_shift;
 324         else
 325                 event_id <<= sparc_pmu->lower_shift;
 326         return event_id;
 327 }
 328
 329 static u64 mask_for_index(int idx)
 330 {
 331         return event_encoding(sparc_pmu->event_mask, idx);
 332 }
 333
 334 static u64 nop_for_index(int idx)
 335 {
 336         return event_encoding(idx == PIC_UPPER_INDEX ?
 337                               sparc_pmu->upper_nop :
 338                               sparc_pmu->lower_nop, idx);
 339 }
 340
 341 static inline void sparc_pmu_enable_event(struct hw_perf_event *hwc,
 342                                             int idx)
 343 {
 344         u64 val, mask = mask_for_index(idx);
 345
 346         val = pcr_ops->read();
 347         pcr_ops->write((val & ~mask) | hwc->config);
 348 }
 349
 350 static inline void sparc_pmu_disable_event(struct hw_perf_event *hwc,
 351                                              int idx)
 352 {
 353         u64 mask = mask_for_index(idx);
 354         u64 nop = nop_for_index(idx);
 355         u64 val = pcr_ops->read();
 356
 357         pcr_ops->write((val & ~mask) | nop);
 358 }
 359
 360 void hw_perf_enable(void)
 361 {
 362         struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
 363         u64 val;
 364         int i;
 365
 366         if (cpuc->enabled)
 367                 return;
 368
 369         cpuc->enabled = 1;
 370         barrier();
 371
 372         val = pcr_ops->read();
 373
 374         for (i = 0; i < MAX_HWEVENTS; i++) {
 375                 struct perf_event *cp = cpuc->events[i];
 376                 struct hw_perf_event *hwc;
 377
 378                 if (!cp)
 379                         continue;
 380                 hwc = &cp->hw;
 381                 val |= hwc->config_base;
 382         }
 383
 384         pcr_ops->write(val);
 385 }
 386
 387 void hw_perf_disable(void)
 388 {
 389         struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
 390         u64 val;
 391
 392         if (!cpuc->enabled)
 393                 return;
 394
 395         cpuc->enabled = 0;
 396
 397         val = pcr_ops->read();
 398         val &= ~(PCR_UTRACE | PCR_STRACE |
 399                  sparc_pmu->hv_bit | sparc_pmu->irq_bit);
 400         pcr_ops->write(val);
 401 }
 402
 403 static u32 read_pmc(int idx)
 404 {
 405         u64 val;
 406
 407         read_pic(val);
 408         if (idx == PIC_UPPER_INDEX)
 409                 val >>= 32;
 410
 411         return val & 0xffffffff;
 412 }
 413
 414 static void write_pmc(int idx, u64 val)
 415 {
 416         u64 shift, mask, pic;
 417
 418         shift = 0;
 419         if (idx == PIC_UPPER_INDEX)
 420                 shift = 32;
 421
 422         mask = ((u64) 0xffffffff) << shift;
 423         val <<= shift;
 424
 425         read_pic(pic);
 426         pic &= ~mask;
 427         pic |= val;
 428         write_pic(pic);
 429 }
 430
 431 static int sparc_perf_event_set_period(struct perf_event *event,
 432                                          struct hw_perf_event *hwc, int idx)
 433 {
 434         s64 left = atomic64_read(&hwc->period_left);
 435         s64 period = hwc->sample_period;
 436         int ret = 0;
 437
 438         if (unlikely(left <= -period)) {
 439                 left = period;
 440                 atomic64_set(&hwc->period_left, left);
 441                 hwc->last_period = period;
 442                 ret = 1;
 443         }
 444
 445         if (unlikely(left <= 0)) {
 446                 left += period;
 447                 atomic64_set(&hwc->period_left, left);
 448                 hwc->last_period = period;
 449                 ret = 1;
 450         }
 451         if (left > MAX_PERIOD)
 452                 left = MAX_PERIOD;
 453
 454         atomic64_set(&hwc->prev_count, (u64)-left);
 455
 456         write_pmc(idx, (u64)(-left) & 0xffffffff);
 457
 458         perf_event_update_userpage(event);
 459
 460         return ret;
 461 }
 462
 463 static int sparc_pmu_enable(struct perf_event *event)
 464 {
 465         struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
 466         struct hw_perf_event *hwc = &event->hw;
 467         int idx = hwc->idx;
 468
 469         if (test_and_set_bit(idx, cpuc->used_mask))
 470                 return -EAGAIN;
 471
 472         sparc_pmu_disable_event(hwc, idx);
 473
 474         cpuc->events[idx] = event;
 475         set_bit(idx, cpuc->active_mask);
 476
 477         sparc_perf_event_set_period(event, hwc, idx);
 478         sparc_pmu_enable_event(hwc, idx);
 479         perf_event_update_userpage(event);
 480         return 0;
 481 }
 482
 483 static u64 sparc_perf_event_update(struct perf_event *event,
 484                                      struct hw_perf_event *hwc, int idx)
 485 {
 486         int shift = 64 - 32;
 487         u64 prev_raw_count, new_raw_count;
 488         s64 delta;
 489
 490 again:
 491         prev_raw_count = atomic64_read(&hwc->prev_count);
 492         new_raw_count = read_pmc(idx);
 493
 494         if (atomic64_cmpxchg(&hwc->prev_count, prev_raw_count,
 495                              new_raw_count) != prev_raw_count)
 496                 goto again;
 497
 498         delta = (new_raw_count << shift) - (prev_raw_count << shift);
 499         delta >>= shift;
 500
 501         atomic64_add(delta, &event->count);
 502         atomic64_sub(delta, &hwc->period_left);
 503
 504         return new_raw_count;
 505 }
 506
 507 static void sparc_pmu_disable(struct perf_event *event)
 508 {
 509         struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
 510         struct hw_perf_event *hwc = &event->hw;
 511         int idx = hwc->idx;
 512
 513         clear_bit(idx, cpuc->active_mask);
 514         sparc_pmu_disable_event(hwc, idx);
 515
 516         barrier();
 517
 518         sparc_perf_event_update(event, hwc, idx);
 519         cpuc->events[idx] = NULL;
 520         clear_bit(idx, cpuc->used_mask);
 521
 522         perf_event_update_userpage(event);
 523 }
 524
 525 static void sparc_pmu_read(struct perf_event *event)
 526 {
 527         struct hw_perf_event *hwc = &event->hw;
 528         sparc_perf_event_update(event, hwc, hwc->idx);
 529 }
 530
 531 static void sparc_pmu_unthrottle(struct perf_event *event)
 532 {
 533         struct hw_perf_event *hwc = &event->hw;
 534         sparc_pmu_enable_event(hwc, hwc->idx);
 535 }
 536
 537 static atomic_t active_events = ATOMIC_INIT(0);
 538 static DEFINE_MUTEX(pmc_grab_mutex);
 539
 540 void perf_event_grab_pmc(void)
 541 {
 542         if (atomic_inc_not_zero(&active_events))
 543                 return;
 544
 545         mutex_lock(&pmc_grab_mutex);
 546         if (atomic_read(&active_events) == 0) {
 547                 if (atomic_read(&nmi_active) > 0) {
 548                         on_each_cpu(stop_nmi_watchdog, NULL, 1);
 549                         BUG_ON(atomic_read(&nmi_active) != 0);
 550                 }
 551                 atomic_inc(&active_events);
 552         }
 553         mutex_unlock(&pmc_grab_mutex);
 554 }
 555
 556 void perf_event_release_pmc(void)
 557 {
 558         if (atomic_dec_and_mutex_lock(&active_events, &pmc_grab_mutex)) {
 559                 if (atomic_read(&nmi_active) == 0)
 560                         on_each_cpu(start_nmi_watchdog, NULL, 1);
 561                 mutex_unlock(&pmc_grab_mutex);
 562         }
 563 }
 564
 565 static const struct perf_event_map *sparc_map_cache_event(u64 config)
 566 {
 567         unsigned int cache_type, cache_op, cache_result;
 568         const struct perf_event_map *pmap;
 569
 570         if (!sparc_pmu->cache_map)
 571                 return ERR_PTR(-ENOENT);
 572
 573         cache_type = (config >>  0) & 0xff;
 574         if (cache_type >= PERF_COUNT_HW_CACHE_MAX)
 575                 return ERR_PTR(-EINVAL);
 576
 577         cache_op = (config >>  8) & 0xff;
 578         if (cache_op >= PERF_COUNT_HW_CACHE_OP_MAX)
 579                 return ERR_PTR(-EINVAL);
 580
 581         cache_result = (config >> 16) & 0xff;
 582         if (cache_result >= PERF_COUNT_HW_CACHE_RESULT_MAX)
 583                 return ERR_PTR(-EINVAL);
 584
 585         pmap = &((*sparc_pmu->cache_map)[cache_type][cache_op][cache_result]);
 586
 587         if (pmap->encoding == CACHE_OP_UNSUPPORTED)
 588                 return ERR_PTR(-ENOENT);
 589
 590         if (pmap->encoding == CACHE_OP_NONSENSE)
 591                 return ERR_PTR(-EINVAL);
 592
 593         return pmap;
 594 }
 595
 596 static void hw_perf_event_destroy(struct perf_event *event)
 597 {
 598         perf_event_release_pmc();
 599 }
 600
 601 static int __hw_perf_event_init(struct perf_event *event)
 602 {
 603         struct perf_event_attr *attr = &event->attr;
 604         struct hw_perf_event *hwc = &event->hw;
 605         const struct perf_event_map *pmap;
 606         u64 enc;
 607
 608         if (atomic_read(&nmi_active) < 0)
 609                 return -ENODEV;
 610
 611         if (attr->type == PERF_TYPE_HARDWARE) {
 612                 if (attr->config >= sparc_pmu->max_events)
 613                         return -EINVAL;
 614                 pmap = sparc_pmu->event_map(attr->config);
 615         } else if (attr->type == PERF_TYPE_HW_CACHE) {
 616                 pmap = sparc_map_cache_event(attr->config);
 617                 if (IS_ERR(pmap))
 618                         return PTR_ERR(pmap);
 619         } else
 620                 return -EOPNOTSUPP;
 621
 622         perf_event_grab_pmc();
 623         event->destroy = hw_perf_event_destroy;
 624
 625         /* We save the enable bits in the config_base.  So to
 626          * turn off sampling just write 'config', and to enable
 627          * things write 'config | config_base'.
 628          */
 629         hwc->config_base = sparc_pmu->irq_bit;
 630         if (!attr->exclude_user)
 631                 hwc->config_base |= PCR_UTRACE;
 632         if (!attr->exclude_kernel)
 633                 hwc->config_base |= PCR_STRACE;
 634         if (!attr->exclude_hv)
 635                 hwc->config_base |= sparc_pmu->hv_bit;
 636
 637         if (!hwc->sample_period) {
 638                 hwc->sample_period = MAX_PERIOD;
 639                 hwc->last_period = hwc->sample_period;
 640                 atomic64_set(&hwc->period_left, hwc->sample_period);
 641         }
 642
 643         enc = pmap->encoding;
 644         if (pmap->pic_mask & PIC_UPPER) {
 645                 hwc->idx = PIC_UPPER_INDEX;
 646                 enc <<= sparc_pmu->upper_shift;
 647         } else {
 648                 hwc->idx = PIC_LOWER_INDEX;
 649                 enc <<= sparc_pmu->lower_shift;
 650         }
 651
 652         hwc->config |= enc;
 653         return 0;
 654 }
 655
 656 static const struct pmu pmu = {
 657         .enable         = sparc_pmu_enable,
 658         .disable        = sparc_pmu_disable,
 659         .read           = sparc_pmu_read,
 660         .unthrottle     = sparc_pmu_unthrottle,
 661 };
 662
 663 const struct pmu *hw_perf_event_init(struct perf_event *event)
 664 {
 665         int err = __hw_perf_event_init(event);
 666
 667         if (err)
 668                 return ERR_PTR(err);
 669         return &pmu;
 670 }
 671
 672 void perf_event_print_debug(void)
 673 {
 674         unsigned long flags;
 675         u64 pcr, pic;
 676         int cpu;
 677
 678         if (!sparc_pmu)
 679                 return;
 680
 681         local_irq_save(flags);
 682
 683         cpu = smp_processor_id();
 684
 685         pcr = pcr_ops->read();
 686         read_pic(pic);
 687
 688         pr_info("\n");
 689         pr_info("CPU#%d: PCR[%016llx] PIC[%016llx]\n",
 690                 cpu, pcr, pic);
 691
 692         local_irq_restore(flags);
 693 }
 694
 695 static int __kprobes perf_event_nmi_handler(struct notifier_block *self,
 696                                               unsigned long cmd, void *__args)
 697 {
 698         struct die_args *args = __args;
 699         struct perf_sample_data data;
 700         struct cpu_hw_events *cpuc;
 701         struct pt_regs *regs;
 702         int idx;
 703
 704         if (!atomic_read(&active_events))
 705                 return NOTIFY_DONE;
 706
 707         switch (cmd) {
 708         case DIE_NMI:
 709                 break;
 710
 711         default:
 712                 return NOTIFY_DONE;
 713         }
 714
 715         regs = args->regs;
 716
 717         data.addr = 0;
 718
 719         cpuc = &__get_cpu_var(cpu_hw_events);
 720         for (idx = 0; idx < MAX_HWEVENTS; idx++) {
 721                 struct perf_event *event = cpuc->events[idx];
 722                 struct hw_perf_event *hwc;
 723                 u64 val;
 724
 725                 if (!test_bit(idx, cpuc->active_mask))
 726                         continue;
 727                 hwc = &event->hw;
 728                 val = sparc_perf_event_update(event, hwc, idx);
 729                 if (val & (1ULL << 31))
 730                         continue;
 731
 732                 data.period = event->hw.last_period;
 733                 if (!sparc_perf_event_set_period(event, hwc, idx))
 734                         continue;
 735
 736                 if (perf_event_overflow(event, 1, &data, regs))
 737                         sparc_pmu_disable_event(hwc, idx);
 738         }
 739
 740         return NOTIFY_STOP;
 741 }
 742
 743 static __read_mostly struct notifier_block perf_event_nmi_notifier = {
 744         .notifier_call          = perf_event_nmi_handler,
 745 };
 746
 747 static bool __init supported_pmu(void)
 748 {
 749         if (!strcmp(sparc_pmu_type, "ultra3") ||
 750             !strcmp(sparc_pmu_type, "ultra3+") ||
 751             !strcmp(sparc_pmu_type, "ultra3i") ||
 752             !strcmp(sparc_pmu_type, "ultra4+")) {
 753                 sparc_pmu = &ultra3_pmu;
 754                 return true;
 755         }
 756         if (!strcmp(sparc_pmu_type, "niagara2")) {
 757                 sparc_pmu = &niagara2_pmu;
 758                 return true;
 759         }
 760         return false;
 761 }
 762
 763 void __init init_hw_perf_events(void)
 764 {
 765         pr_info("Performance events: ");
 766
 767         if (!supported_pmu()) {
 768                 pr_cont("No support for PMU type '%s'\n", sparc_pmu_type);
 769                 return;
 770         }
 771
 772         pr_cont("Supported PMU type is '%s'\n", sparc_pmu_type);
 773
 774         /* All sparc64 PMUs currently have 2 events.  But this simple
 775          * driver only supports one active event at a time.
 776          */
 777         perf_max_events = 1;
 778
 779         register_die_notifier(&perf_event_nmi_notifier);
 780 }