SAFE public projects git trees. - safe/jmp/linux-2.6/blob - arch/x86/kernel/cpu/perf_counter.c

   1 /*
   2  * Performance counter x86 architecture code
   3  *
   4  *  Copyright(C) 2008 Thomas Gleixner <tglx@linutronix.de>
   5  *  Copyright(C) 2008 Red Hat, Inc., Ingo Molnar
   6  *  Copyright(C) 2009 Jaswinder Singh Rajput
   7  *
   8  *  For licencing details see kernel-base/COPYING
   9  */
  10
  11 #include <linux/perf_counter.h>
  12 #include <linux/capability.h>
  13 #include <linux/notifier.h>
  14 #include <linux/hardirq.h>
  15 #include <linux/kprobes.h>
  16 #include <linux/module.h>
  17 #include <linux/kdebug.h>
  18 #include <linux/sched.h>
  19 #include <linux/uaccess.h>
  20
  21 #include <asm/apic.h>
  22 #include <asm/stacktrace.h>
  23 #include <asm/nmi.h>
  24
  25 static bool perf_counters_initialized __read_mostly;
  26
  27 /*
  28  * Number of (generic) HW counters:
  29  */
  30 static int nr_counters_generic __read_mostly;
  31 static u64 perf_counter_mask __read_mostly;
  32 static u64 counter_value_mask __read_mostly;
  33 static int counter_value_bits __read_mostly;
  34
  35 static int nr_counters_fixed __read_mostly;
  36
  37 struct cpu_hw_counters {
  38         struct perf_counter     *counters[X86_PMC_IDX_MAX];
  39         unsigned long           used[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
  40         unsigned long           interrupts;
  41         u64                     throttle_ctrl;
  42         unsigned long           active_mask[BITS_TO_LONGS(X86_PMC_IDX_MAX)];
  43         int                     enabled;
  44 };
  45
  46 /*
  47  * struct pmc_x86_ops - performance counter x86 ops
  48  */
  49 struct pmc_x86_ops {
  50         u64             (*save_disable_all)(void);
  51         void            (*restore_all)(u64);
  52         u64             (*get_status)(u64);
  53         void            (*ack_status)(u64);
  54         void            (*enable)(int, u64);
  55         void            (*disable)(int, u64);
  56         unsigned        eventsel;
  57         unsigned        perfctr;
  58         u64             (*event_map)(int);
  59         u64             (*raw_event)(u64);
  60         int             max_events;
  61 };
  62
  63 static struct pmc_x86_ops *pmc_ops __read_mostly;
  64
  65 static DEFINE_PER_CPU(struct cpu_hw_counters, cpu_hw_counters) = {
  66         .enabled = 1,
  67 };
  68
  69 static __read_mostly int intel_perfmon_version;
  70
  71 /*
  72  * Intel PerfMon v3. Used on Core2 and later.
  73  */
  74 static const u64 intel_perfmon_event_map[] =
  75 {
  76   [PERF_COUNT_CPU_CYCLES]               = 0x003c,
  77   [PERF_COUNT_INSTRUCTIONS]             = 0x00c0,
  78   [PERF_COUNT_CACHE_REFERENCES]         = 0x4f2e,
  79   [PERF_COUNT_CACHE_MISSES]             = 0x412e,
  80   [PERF_COUNT_BRANCH_INSTRUCTIONS]      = 0x00c4,
  81   [PERF_COUNT_BRANCH_MISSES]            = 0x00c5,
  82   [PERF_COUNT_BUS_CYCLES]               = 0x013c,
  83 };
  84
  85 static u64 pmc_intel_event_map(int event)
  86 {
  87         return intel_perfmon_event_map[event];
  88 }
  89
  90 static u64 pmc_intel_raw_event(u64 event)
  91 {
  92 #define CORE_EVNTSEL_EVENT_MASK         0x000000FFULL
  93 #define CORE_EVNTSEL_UNIT_MASK          0x0000FF00ULL
  94 #define CORE_EVNTSEL_COUNTER_MASK       0xFF000000ULL
  95
  96 #define CORE_EVNTSEL_MASK               \
  97         (CORE_EVNTSEL_EVENT_MASK |      \
  98          CORE_EVNTSEL_UNIT_MASK  |      \
  99          CORE_EVNTSEL_COUNTER_MASK)
 100
 101         return event & CORE_EVNTSEL_MASK;
 102 }
 103
 104 /*
 105  * AMD Performance Monitor K7 and later.
 106  */
 107 static const u64 amd_perfmon_event_map[] =
 108 {
 109   [PERF_COUNT_CPU_CYCLES]               = 0x0076,
 110   [PERF_COUNT_INSTRUCTIONS]             = 0x00c0,
 111   [PERF_COUNT_CACHE_REFERENCES]         = 0x0080,
 112   [PERF_COUNT_CACHE_MISSES]             = 0x0081,
 113   [PERF_COUNT_BRANCH_INSTRUCTIONS]      = 0x00c4,
 114   [PERF_COUNT_BRANCH_MISSES]            = 0x00c5,
 115 };
 116
 117 static u64 pmc_amd_event_map(int event)
 118 {
 119         return amd_perfmon_event_map[event];
 120 }
 121
 122 static u64 pmc_amd_raw_event(u64 event)
 123 {
 124 #define K7_EVNTSEL_EVENT_MASK   0x7000000FFULL
 125 #define K7_EVNTSEL_UNIT_MASK    0x00000FF00ULL
 126 #define K7_EVNTSEL_COUNTER_MASK 0x0FF000000ULL
 127
 128 #define K7_EVNTSEL_MASK                 \
 129         (K7_EVNTSEL_EVENT_MASK |        \
 130          K7_EVNTSEL_UNIT_MASK  |        \
 131          K7_EVNTSEL_COUNTER_MASK)
 132
 133         return event & K7_EVNTSEL_MASK;
 134 }
 135
 136 /*
 137  * Propagate counter elapsed time into the generic counter.
 138  * Can only be executed on the CPU where the counter is active.
 139  * Returns the delta events processed.
 140  */
 141 static void
 142 x86_perf_counter_update(struct perf_counter *counter,
 143                         struct hw_perf_counter *hwc, int idx)
 144 {
 145         u64 prev_raw_count, new_raw_count, delta;
 146
 147         /*
 148          * Careful: an NMI might modify the previous counter value.
 149          *
 150          * Our tactic to handle this is to first atomically read and
 151          * exchange a new raw count - then add that new-prev delta
 152          * count to the generic counter atomically:
 153          */
 154 again:
 155         prev_raw_count = atomic64_read(&hwc->prev_count);
 156         rdmsrl(hwc->counter_base + idx, new_raw_count);
 157
 158         if (atomic64_cmpxchg(&hwc->prev_count, prev_raw_count,
 159                                         new_raw_count) != prev_raw_count)
 160                 goto again;
 161
 162         /*
 163          * Now we have the new raw value and have updated the prev
 164          * timestamp already. We can now calculate the elapsed delta
 165          * (counter-)time and add that to the generic counter.
 166          *
 167          * Careful, not all hw sign-extends above the physical width
 168          * of the count, so we do that by clipping the delta to 32 bits:
 169          */
 170         delta = (u64)(u32)((s32)new_raw_count - (s32)prev_raw_count);
 171
 172         atomic64_add(delta, &counter->count);
 173         atomic64_sub(delta, &hwc->period_left);
 174 }
 175
 176 static atomic_t num_counters;
 177 static DEFINE_MUTEX(pmc_reserve_mutex);
 178
 179 static bool reserve_pmc_hardware(void)
 180 {
 181         int i;
 182
 183         if (nmi_watchdog == NMI_LOCAL_APIC)
 184                 disable_lapic_nmi_watchdog();
 185
 186         for (i = 0; i < nr_counters_generic; i++) {
 187                 if (!reserve_perfctr_nmi(pmc_ops->perfctr + i))
 188                         goto perfctr_fail;
 189         }
 190
 191         for (i = 0; i < nr_counters_generic; i++) {
 192                 if (!reserve_evntsel_nmi(pmc_ops->eventsel + i))
 193                         goto eventsel_fail;
 194         }
 195
 196         return true;
 197
 198 eventsel_fail:
 199         for (i--; i >= 0; i--)
 200                 release_evntsel_nmi(pmc_ops->eventsel + i);
 201
 202         i = nr_counters_generic;
 203
 204 perfctr_fail:
 205         for (i--; i >= 0; i--)
 206                 release_perfctr_nmi(pmc_ops->perfctr + i);
 207
 208         if (nmi_watchdog == NMI_LOCAL_APIC)
 209                 enable_lapic_nmi_watchdog();
 210
 211         return false;
 212 }
 213
 214 static void release_pmc_hardware(void)
 215 {
 216         int i;
 217
 218         for (i = 0; i < nr_counters_generic; i++) {
 219                 release_perfctr_nmi(pmc_ops->perfctr + i);
 220                 release_evntsel_nmi(pmc_ops->eventsel + i);
 221         }
 222
 223         if (nmi_watchdog == NMI_LOCAL_APIC)
 224                 enable_lapic_nmi_watchdog();
 225 }
 226
 227 static void hw_perf_counter_destroy(struct perf_counter *counter)
 228 {
 229         if (atomic_dec_and_mutex_lock(&num_counters, &pmc_reserve_mutex)) {
 230                 release_pmc_hardware();
 231                 mutex_unlock(&pmc_reserve_mutex);
 232         }
 233 }
 234
 235 /*
 236  * Setup the hardware configuration for a given hw_event_type
 237  */
 238 static int __hw_perf_counter_init(struct perf_counter *counter)
 239 {
 240         struct perf_counter_hw_event *hw_event = &counter->hw_event;
 241         struct hw_perf_counter *hwc = &counter->hw;
 242         int err;
 243
 244         if (unlikely(!perf_counters_initialized))
 245                 return -EINVAL;
 246
 247         err = 0;
 248         if (atomic_inc_not_zero(&num_counters)) {
 249                 mutex_lock(&pmc_reserve_mutex);
 250                 if (atomic_read(&num_counters) == 0 && !reserve_pmc_hardware())
 251                         err = -EBUSY;
 252                 else
 253                         atomic_inc(&num_counters);
 254                 mutex_unlock(&pmc_reserve_mutex);
 255         }
 256         if (err)
 257                 return err;
 258
 259         /*
 260          * Generate PMC IRQs:
 261          * (keep 'enabled' bit clear for now)
 262          */
 263         hwc->config = ARCH_PERFMON_EVENTSEL_INT;
 264
 265         /*
 266          * Count user and OS events unless requested not to.
 267          */
 268         if (!hw_event->exclude_user)
 269                 hwc->config |= ARCH_PERFMON_EVENTSEL_USR;
 270         if (!hw_event->exclude_kernel)
 271                 hwc->config |= ARCH_PERFMON_EVENTSEL_OS;
 272
 273         /*
 274          * If privileged enough, allow NMI events:
 275          */
 276         hwc->nmi = 0;
 277         if (capable(CAP_SYS_ADMIN) && hw_event->nmi)
 278                 hwc->nmi = 1;
 279
 280         hwc->irq_period         = hw_event->irq_period;
 281         /*
 282          * Intel PMCs cannot be accessed sanely above 32 bit width,
 283          * so we install an artificial 1<<31 period regardless of
 284          * the generic counter period:
 285          */
 286         if (boot_cpu_data.x86_vendor == X86_VENDOR_INTEL)
 287                 if ((s64)hwc->irq_period <= 0 || hwc->irq_period > 0x7FFFFFFF)
 288                         hwc->irq_period = 0x7FFFFFFF;
 289
 290         atomic64_set(&hwc->period_left, hwc->irq_period);
 291
 292         /*
 293          * Raw event type provide the config in the event structure
 294          */
 295         if (perf_event_raw(hw_event)) {
 296                 hwc->config |= pmc_ops->raw_event(perf_event_config(hw_event));
 297         } else {
 298                 if (perf_event_id(hw_event) >= pmc_ops->max_events)
 299                         return -EINVAL;
 300                 /*
 301                  * The generic map:
 302                  */
 303                 hwc->config |= pmc_ops->event_map(perf_event_id(hw_event));
 304         }
 305
 306         counter->destroy = hw_perf_counter_destroy;
 307
 308         return 0;
 309 }
 310
 311 static u64 pmc_intel_save_disable_all(void)
 312 {
 313         u64 ctrl;
 314
 315         rdmsrl(MSR_CORE_PERF_GLOBAL_CTRL, ctrl);
 316         wrmsrl(MSR_CORE_PERF_GLOBAL_CTRL, 0);
 317
 318         return ctrl;
 319 }
 320
 321 static u64 pmc_amd_save_disable_all(void)
 322 {
 323         struct cpu_hw_counters *cpuc = &__get_cpu_var(cpu_hw_counters);
 324         int enabled, idx;
 325
 326         enabled = cpuc->enabled;
 327         cpuc->enabled = 0;
 328         /*
 329          * ensure we write the disable before we start disabling the
 330          * counters proper, so that pcm_amd_enable() does the right thing.
 331          */
 332         barrier();
 333
 334         for (idx = 0; idx < nr_counters_generic; idx++) {
 335                 u64 val;
 336
 337                 rdmsrl(MSR_K7_EVNTSEL0 + idx, val);
 338                 if (val & ARCH_PERFMON_EVENTSEL0_ENABLE) {
 339                         val &= ~ARCH_PERFMON_EVENTSEL0_ENABLE;
 340                         wrmsrl(MSR_K7_EVNTSEL0 + idx, val);
 341                 }
 342         }
 343
 344         return enabled;
 345 }
 346
 347 u64 hw_perf_save_disable(void)
 348 {
 349         if (unlikely(!perf_counters_initialized))
 350                 return 0;
 351
 352         return pmc_ops->save_disable_all();
 353 }
 354 /*
 355  * Exported because of ACPI idle
 356  */
 357 EXPORT_SYMBOL_GPL(hw_perf_save_disable);
 358
 359 static void pmc_intel_restore_all(u64 ctrl)
 360 {
 361         wrmsrl(MSR_CORE_PERF_GLOBAL_CTRL, ctrl);
 362 }
 363
 364 static void pmc_amd_restore_all(u64 ctrl)
 365 {
 366         struct cpu_hw_counters *cpuc = &__get_cpu_var(cpu_hw_counters);
 367         int idx;
 368
 369         cpuc->enabled = ctrl;
 370         barrier();
 371         if (!ctrl)
 372                 return;
 373
 374         for (idx = 0; idx < nr_counters_generic; idx++) {
 375                 if (test_bit(idx, cpuc->active_mask)) {
 376                         u64 val;
 377
 378                         rdmsrl(MSR_K7_EVNTSEL0 + idx, val);
 379                         val |= ARCH_PERFMON_EVENTSEL0_ENABLE;
 380                         wrmsrl(MSR_K7_EVNTSEL0 + idx, val);
 381                 }
 382         }
 383 }
 384
 385 void hw_perf_restore(u64 ctrl)
 386 {
 387         if (unlikely(!perf_counters_initialized))
 388                 return;
 389
 390         pmc_ops->restore_all(ctrl);
 391 }
 392 /*
 393  * Exported because of ACPI idle
 394  */
 395 EXPORT_SYMBOL_GPL(hw_perf_restore);
 396
 397 static u64 pmc_intel_get_status(u64 mask)
 398 {
 399         u64 status;
 400
 401         rdmsrl(MSR_CORE_PERF_GLOBAL_STATUS, status);
 402
 403         return status;
 404 }
 405
 406 static u64 pmc_amd_get_status(u64 mask)
 407 {
 408         u64 status = 0;
 409         int idx;
 410
 411         for (idx = 0; idx < nr_counters_generic; idx++) {
 412                 s64 val;
 413
 414                 if (!(mask & (1 << idx)))
 415                         continue;
 416
 417                 rdmsrl(MSR_K7_PERFCTR0 + idx, val);
 418                 val <<= (64 - counter_value_bits);
 419                 if (val >= 0)
 420                         status |= (1 << idx);
 421         }
 422
 423         return status;
 424 }
 425
 426 static u64 hw_perf_get_status(u64 mask)
 427 {
 428         if (unlikely(!perf_counters_initialized))
 429                 return 0;
 430
 431         return pmc_ops->get_status(mask);
 432 }
 433
 434 static void pmc_intel_ack_status(u64 ack)
 435 {
 436         wrmsrl(MSR_CORE_PERF_GLOBAL_OVF_CTRL, ack);
 437 }
 438
 439 static void pmc_amd_ack_status(u64 ack)
 440 {
 441 }
 442
 443 static void hw_perf_ack_status(u64 ack)
 444 {
 445         if (unlikely(!perf_counters_initialized))
 446                 return;
 447
 448         pmc_ops->ack_status(ack);
 449 }
 450
 451 static void pmc_intel_enable(int idx, u64 config)
 452 {
 453         wrmsrl(MSR_ARCH_PERFMON_EVENTSEL0 + idx,
 454                         config | ARCH_PERFMON_EVENTSEL0_ENABLE);
 455 }
 456
 457 static void pmc_amd_enable(int idx, u64 config)
 458 {
 459         struct cpu_hw_counters *cpuc = &__get_cpu_var(cpu_hw_counters);
 460
 461         set_bit(idx, cpuc->active_mask);
 462         if (cpuc->enabled)
 463                 config |= ARCH_PERFMON_EVENTSEL0_ENABLE;
 464
 465         wrmsrl(MSR_K7_EVNTSEL0 + idx, config);
 466 }
 467
 468 static void hw_perf_enable(int idx, u64 config)
 469 {
 470         if (unlikely(!perf_counters_initialized))
 471                 return;
 472
 473         pmc_ops->enable(idx, config);
 474 }
 475
 476 static void pmc_intel_disable(int idx, u64 config)
 477 {
 478         wrmsrl(MSR_ARCH_PERFMON_EVENTSEL0 + idx, config);
 479 }
 480
 481 static void pmc_amd_disable(int idx, u64 config)
 482 {
 483         struct cpu_hw_counters *cpuc = &__get_cpu_var(cpu_hw_counters);
 484
 485         clear_bit(idx, cpuc->active_mask);
 486         wrmsrl(MSR_K7_EVNTSEL0 + idx, config);
 487
 488 }
 489
 490 static void hw_perf_disable(int idx, u64 config)
 491 {
 492         if (unlikely(!perf_counters_initialized))
 493                 return;
 494
 495         pmc_ops->disable(idx, config);
 496 }
 497
 498 static inline void
 499 __pmc_fixed_disable(struct perf_counter *counter,
 500                     struct hw_perf_counter *hwc, unsigned int __idx)
 501 {
 502         int idx = __idx - X86_PMC_IDX_FIXED;
 503         u64 ctrl_val, mask;
 504         int err;
 505
 506         mask = 0xfULL << (idx * 4);
 507
 508         rdmsrl(hwc->config_base, ctrl_val);
 509         ctrl_val &= ~mask;
 510         err = checking_wrmsrl(hwc->config_base, ctrl_val);
 511 }
 512
 513 static inline void
 514 __pmc_generic_disable(struct perf_counter *counter,
 515                            struct hw_perf_counter *hwc, unsigned int idx)
 516 {
 517         if (unlikely(hwc->config_base == MSR_ARCH_PERFMON_FIXED_CTR_CTRL))
 518                 __pmc_fixed_disable(counter, hwc, idx);
 519         else
 520                 hw_perf_disable(idx, hwc->config);
 521 }
 522
 523 static DEFINE_PER_CPU(u64, prev_left[X86_PMC_IDX_MAX]);
 524
 525 /*
 526  * Set the next IRQ period, based on the hwc->period_left value.
 527  * To be called with the counter disabled in hw:
 528  */
 529 static void
 530 __hw_perf_counter_set_period(struct perf_counter *counter,
 531                              struct hw_perf_counter *hwc, int idx)
 532 {
 533         s64 left = atomic64_read(&hwc->period_left);
 534         s64 period = hwc->irq_period;
 535         int err;
 536
 537         /*
 538          * If we are way outside a reasoable range then just skip forward:
 539          */
 540         if (unlikely(left <= -period)) {
 541                 left = period;
 542                 atomic64_set(&hwc->period_left, left);
 543         }
 544
 545         if (unlikely(left <= 0)) {
 546                 left += period;
 547                 atomic64_set(&hwc->period_left, left);
 548         }
 549
 550         per_cpu(prev_left[idx], smp_processor_id()) = left;
 551
 552         /*
 553          * The hw counter starts counting from this counter offset,
 554          * mark it to be able to extra future deltas:
 555          */
 556         atomic64_set(&hwc->prev_count, (u64)-left);
 557
 558         err = checking_wrmsrl(hwc->counter_base + idx,
 559                              (u64)(-left) & counter_value_mask);
 560 }
 561
 562 static inline void
 563 __pmc_fixed_enable(struct perf_counter *counter,
 564                    struct hw_perf_counter *hwc, unsigned int __idx)
 565 {
 566         int idx = __idx - X86_PMC_IDX_FIXED;
 567         u64 ctrl_val, bits, mask;
 568         int err;
 569
 570         /*
 571          * Enable IRQ generation (0x8),
 572          * and enable ring-3 counting (0x2) and ring-0 counting (0x1)
 573          * if requested:
 574          */
 575         bits = 0x8ULL;
 576         if (hwc->config & ARCH_PERFMON_EVENTSEL_USR)
 577                 bits |= 0x2;
 578         if (hwc->config & ARCH_PERFMON_EVENTSEL_OS)
 579                 bits |= 0x1;
 580         bits <<= (idx * 4);
 581         mask = 0xfULL << (idx * 4);
 582
 583         rdmsrl(hwc->config_base, ctrl_val);
 584         ctrl_val &= ~mask;
 585         ctrl_val |= bits;
 586         err = checking_wrmsrl(hwc->config_base, ctrl_val);
 587 }
 588
 589 static void
 590 __pmc_generic_enable(struct perf_counter *counter,
 591                           struct hw_perf_counter *hwc, int idx)
 592 {
 593         if (unlikely(hwc->config_base == MSR_ARCH_PERFMON_FIXED_CTR_CTRL))
 594                 __pmc_fixed_enable(counter, hwc, idx);
 595         else
 596                 hw_perf_enable(idx, hwc->config);
 597 }
 598
 599 static int
 600 fixed_mode_idx(struct perf_counter *counter, struct hw_perf_counter *hwc)
 601 {
 602         unsigned int event;
 603
 604         if (boot_cpu_data.x86_vendor == X86_VENDOR_AMD)
 605                 return -1;
 606
 607         if (unlikely(hwc->nmi))
 608                 return -1;
 609
 610         event = hwc->config & ARCH_PERFMON_EVENT_MASK;
 611
 612         if (unlikely(event == pmc_ops->event_map(PERF_COUNT_INSTRUCTIONS)))
 613                 return X86_PMC_IDX_FIXED_INSTRUCTIONS;
 614         if (unlikely(event == pmc_ops->event_map(PERF_COUNT_CPU_CYCLES)))
 615                 return X86_PMC_IDX_FIXED_CPU_CYCLES;
 616         if (unlikely(event == pmc_ops->event_map(PERF_COUNT_BUS_CYCLES)))
 617                 return X86_PMC_IDX_FIXED_BUS_CYCLES;
 618
 619         return -1;
 620 }
 621
 622 /*
 623  * Find a PMC slot for the freshly enabled / scheduled in counter:
 624  */
 625 static int pmc_generic_enable(struct perf_counter *counter)
 626 {
 627         struct cpu_hw_counters *cpuc = &__get_cpu_var(cpu_hw_counters);
 628         struct hw_perf_counter *hwc = &counter->hw;
 629         int idx;
 630
 631         idx = fixed_mode_idx(counter, hwc);
 632         if (idx >= 0) {
 633                 /*
 634                  * Try to get the fixed counter, if that is already taken
 635                  * then try to get a generic counter:
 636                  */
 637                 if (test_and_set_bit(idx, cpuc->used))
 638                         goto try_generic;
 639
 640                 hwc->config_base = MSR_ARCH_PERFMON_FIXED_CTR_CTRL;
 641                 /*
 642                  * We set it so that counter_base + idx in wrmsr/rdmsr maps to
 643                  * MSR_ARCH_PERFMON_FIXED_CTR0 ... CTR2:
 644                  */
 645                 hwc->counter_base =
 646                         MSR_ARCH_PERFMON_FIXED_CTR0 - X86_PMC_IDX_FIXED;
 647                 hwc->idx = idx;
 648         } else {
 649                 idx = hwc->idx;
 650                 /* Try to get the previous generic counter again */
 651                 if (test_and_set_bit(idx, cpuc->used)) {
 652 try_generic:
 653                         idx = find_first_zero_bit(cpuc->used, nr_counters_generic);
 654                         if (idx == nr_counters_generic)
 655                                 return -EAGAIN;
 656
 657                         set_bit(idx, cpuc->used);
 658                         hwc->idx = idx;
 659                 }
 660                 hwc->config_base  = pmc_ops->eventsel;
 661                 hwc->counter_base = pmc_ops->perfctr;
 662         }
 663
 664         perf_counters_lapic_init(hwc->nmi);
 665
 666         __pmc_generic_disable(counter, hwc, idx);
 667
 668         cpuc->counters[idx] = counter;
 669         /*
 670          * Make it visible before enabling the hw:
 671          */
 672         smp_wmb();
 673
 674         __hw_perf_counter_set_period(counter, hwc, idx);
 675         __pmc_generic_enable(counter, hwc, idx);
 676
 677         return 0;
 678 }
 679
 680 void perf_counter_print_debug(void)
 681 {
 682         u64 ctrl, status, overflow, pmc_ctrl, pmc_count, prev_left, fixed;
 683         struct cpu_hw_counters *cpuc;
 684         int cpu, idx;
 685
 686         if (!nr_counters_generic)
 687                 return;
 688
 689         local_irq_disable();
 690
 691         cpu = smp_processor_id();
 692         cpuc = &per_cpu(cpu_hw_counters, cpu);
 693
 694         if (intel_perfmon_version >= 2) {
 695                 rdmsrl(MSR_CORE_PERF_GLOBAL_CTRL, ctrl);
 696                 rdmsrl(MSR_CORE_PERF_GLOBAL_STATUS, status);
 697                 rdmsrl(MSR_CORE_PERF_GLOBAL_OVF_CTRL, overflow);
 698                 rdmsrl(MSR_ARCH_PERFMON_FIXED_CTR_CTRL, fixed);
 699
 700                 pr_info("\n");
 701                 pr_info("CPU#%d: ctrl:       %016llx\n", cpu, ctrl);
 702                 pr_info("CPU#%d: status:     %016llx\n", cpu, status);
 703                 pr_info("CPU#%d: overflow:   %016llx\n", cpu, overflow);
 704                 pr_info("CPU#%d: fixed:      %016llx\n", cpu, fixed);
 705         }
 706         pr_info("CPU#%d: used:       %016llx\n", cpu, *(u64 *)cpuc->used);
 707
 708         for (idx = 0; idx < nr_counters_generic; idx++) {
 709                 rdmsrl(pmc_ops->eventsel + idx, pmc_ctrl);
 710                 rdmsrl(pmc_ops->perfctr  + idx, pmc_count);
 711
 712                 prev_left = per_cpu(prev_left[idx], cpu);
 713
 714                 pr_info("CPU#%d:   gen-PMC%d ctrl:  %016llx\n",
 715                         cpu, idx, pmc_ctrl);
 716                 pr_info("CPU#%d:   gen-PMC%d count: %016llx\n",
 717                         cpu, idx, pmc_count);
 718                 pr_info("CPU#%d:   gen-PMC%d left:  %016llx\n",
 719                         cpu, idx, prev_left);
 720         }
 721         for (idx = 0; idx < nr_counters_fixed; idx++) {
 722                 rdmsrl(MSR_ARCH_PERFMON_FIXED_CTR0 + idx, pmc_count);
 723
 724                 pr_info("CPU#%d: fixed-PMC%d count: %016llx\n",
 725                         cpu, idx, pmc_count);
 726         }
 727         local_irq_enable();
 728 }
 729
 730 static void pmc_generic_disable(struct perf_counter *counter)
 731 {
 732         struct cpu_hw_counters *cpuc = &__get_cpu_var(cpu_hw_counters);
 733         struct hw_perf_counter *hwc = &counter->hw;
 734         unsigned int idx = hwc->idx;
 735
 736         __pmc_generic_disable(counter, hwc, idx);
 737
 738         clear_bit(idx, cpuc->used);
 739         cpuc->counters[idx] = NULL;
 740         /*
 741          * Make sure the cleared pointer becomes visible before we
 742          * (potentially) free the counter:
 743          */
 744         smp_wmb();
 745
 746         /*
 747          * Drain the remaining delta count out of a counter
 748          * that we are disabling:
 749          */
 750         x86_perf_counter_update(counter, hwc, idx);
 751 }
 752
 753 /*
 754  * Save and restart an expired counter. Called by NMI contexts,
 755  * so it has to be careful about preempting normal counter ops:
 756  */
 757 static void perf_save_and_restart(struct perf_counter *counter)
 758 {
 759         struct hw_perf_counter *hwc = &counter->hw;
 760         int idx = hwc->idx;
 761
 762         x86_perf_counter_update(counter, hwc, idx);
 763         __hw_perf_counter_set_period(counter, hwc, idx);
 764
 765         if (counter->state == PERF_COUNTER_STATE_ACTIVE)
 766                 __pmc_generic_enable(counter, hwc, idx);
 767 }
 768
 769 /*
 770  * Maximum interrupt frequency of 100KHz per CPU
 771  */
 772 #define PERFMON_MAX_INTERRUPTS (100000/HZ)
 773
 774 /*
 775  * This handler is triggered by the local APIC, so the APIC IRQ handling
 776  * rules apply:
 777  */
 778 static int __smp_perf_counter_interrupt(struct pt_regs *regs, int nmi)
 779 {
 780         int bit, cpu = smp_processor_id();
 781         u64 ack, status;
 782         struct cpu_hw_counters *cpuc = &per_cpu(cpu_hw_counters, cpu);
 783         int ret = 0;
 784
 785         cpuc->throttle_ctrl = hw_perf_save_disable();
 786
 787         status = hw_perf_get_status(cpuc->throttle_ctrl);
 788         if (!status)
 789                 goto out;
 790
 791         ret = 1;
 792 again:
 793         inc_irq_stat(apic_perf_irqs);
 794         ack = status;
 795         for_each_bit(bit, (unsigned long *)&status, X86_PMC_IDX_MAX) {
 796                 struct perf_counter *counter = cpuc->counters[bit];
 797
 798                 clear_bit(bit, (unsigned long *) &status);
 799                 if (!counter)
 800                         continue;
 801
 802                 perf_save_and_restart(counter);
 803                 perf_counter_output(counter, nmi, regs);
 804         }
 805
 806         hw_perf_ack_status(ack);
 807
 808         /*
 809          * Repeat if there is more work to be done:
 810          */
 811         status = hw_perf_get_status(cpuc->throttle_ctrl);
 812         if (status)
 813                 goto again;
 814 out:
 815         /*
 816          * Restore - do not reenable when global enable is off or throttled:
 817          */
 818         if (++cpuc->interrupts < PERFMON_MAX_INTERRUPTS)
 819                 hw_perf_restore(cpuc->throttle_ctrl);
 820
 821         return ret;
 822 }
 823
 824 void perf_counter_unthrottle(void)
 825 {
 826         struct cpu_hw_counters *cpuc;
 827
 828         if (!cpu_has(&boot_cpu_data, X86_FEATURE_ARCH_PERFMON))
 829                 return;
 830
 831         if (unlikely(!perf_counters_initialized))
 832                 return;
 833
 834         cpuc = &__get_cpu_var(cpu_hw_counters);
 835         if (cpuc->interrupts >= PERFMON_MAX_INTERRUPTS) {
 836                 if (printk_ratelimit())
 837                         printk(KERN_WARNING "PERFMON: max interrupts exceeded!\n");
 838                 hw_perf_restore(cpuc->throttle_ctrl);
 839         }
 840         cpuc->interrupts = 0;
 841 }
 842
 843 void smp_perf_counter_interrupt(struct pt_regs *regs)
 844 {
 845         irq_enter();
 846         apic_write(APIC_LVTPC, LOCAL_PERF_VECTOR);
 847         ack_APIC_irq();
 848         __smp_perf_counter_interrupt(regs, 0);
 849         irq_exit();
 850 }
 851
 852 void smp_perf_pending_interrupt(struct pt_regs *regs)
 853 {
 854         irq_enter();
 855         ack_APIC_irq();
 856         inc_irq_stat(apic_pending_irqs);
 857         perf_counter_do_pending();
 858         irq_exit();
 859 }
 860
 861 void set_perf_counter_pending(void)
 862 {
 863         apic->send_IPI_self(LOCAL_PENDING_VECTOR);
 864 }
 865
 866 void perf_counters_lapic_init(int nmi)
 867 {
 868         u32 apic_val;
 869
 870         if (!perf_counters_initialized)
 871                 return;
 872         /*
 873          * Enable the performance counter vector in the APIC LVT:
 874          */
 875         apic_val = apic_read(APIC_LVTERR);
 876
 877         apic_write(APIC_LVTERR, apic_val | APIC_LVT_MASKED);
 878         if (nmi)
 879                 apic_write(APIC_LVTPC, APIC_DM_NMI);
 880         else
 881                 apic_write(APIC_LVTPC, LOCAL_PERF_VECTOR);
 882         apic_write(APIC_LVTERR, apic_val);
 883 }
 884
 885 static int __kprobes
 886 perf_counter_nmi_handler(struct notifier_block *self,
 887                          unsigned long cmd, void *__args)
 888 {
 889         struct die_args *args = __args;
 890         struct pt_regs *regs;
 891         int ret;
 892
 893         switch (cmd) {
 894         case DIE_NMI:
 895         case DIE_NMI_IPI:
 896                 break;
 897
 898         default:
 899                 return NOTIFY_DONE;
 900         }
 901
 902         regs = args->regs;
 903
 904         apic_write(APIC_LVTPC, APIC_DM_NMI);
 905         ret = __smp_perf_counter_interrupt(regs, 1);
 906
 907         return ret ? NOTIFY_STOP : NOTIFY_OK;
 908 }
 909
 910 static __read_mostly struct notifier_block perf_counter_nmi_notifier = {
 911         .notifier_call          = perf_counter_nmi_handler,
 912         .next                   = NULL,
 913         .priority               = 1
 914 };
 915
 916 static struct pmc_x86_ops pmc_intel_ops = {
 917         .save_disable_all       = pmc_intel_save_disable_all,
 918         .restore_all            = pmc_intel_restore_all,
 919         .get_status             = pmc_intel_get_status,
 920         .ack_status             = pmc_intel_ack_status,
 921         .enable                 = pmc_intel_enable,
 922         .disable                = pmc_intel_disable,
 923         .eventsel               = MSR_ARCH_PERFMON_EVENTSEL0,
 924         .perfctr                = MSR_ARCH_PERFMON_PERFCTR0,
 925         .event_map              = pmc_intel_event_map,
 926         .raw_event              = pmc_intel_raw_event,
 927         .max_events             = ARRAY_SIZE(intel_perfmon_event_map),
 928 };
 929
 930 static struct pmc_x86_ops pmc_amd_ops = {
 931         .save_disable_all       = pmc_amd_save_disable_all,
 932         .restore_all            = pmc_amd_restore_all,
 933         .get_status             = pmc_amd_get_status,
 934         .ack_status             = pmc_amd_ack_status,
 935         .enable                 = pmc_amd_enable,
 936         .disable                = pmc_amd_disable,
 937         .eventsel               = MSR_K7_EVNTSEL0,
 938         .perfctr                = MSR_K7_PERFCTR0,
 939         .event_map              = pmc_amd_event_map,
 940         .raw_event              = pmc_amd_raw_event,
 941         .max_events             = ARRAY_SIZE(amd_perfmon_event_map),
 942 };
 943
 944 static struct pmc_x86_ops *pmc_intel_init(void)
 945 {
 946         union cpuid10_edx edx;
 947         union cpuid10_eax eax;
 948         unsigned int unused;
 949         unsigned int ebx;
 950
 951         /*
 952          * Check whether the Architectural PerfMon supports
 953          * Branch Misses Retired Event or not.
 954          */
 955         cpuid(10, &eax.full, &ebx, &unused, &edx.full);
 956         if (eax.split.mask_length <= ARCH_PERFMON_BRANCH_MISSES_RETIRED)
 957                 return NULL;
 958
 959         intel_perfmon_version = eax.split.version_id;
 960         if (intel_perfmon_version < 2)
 961                 return NULL;
 962
 963         pr_info("Intel Performance Monitoring support detected.\n");
 964         pr_info("... version:         %d\n", intel_perfmon_version);
 965         pr_info("... bit width:       %d\n", eax.split.bit_width);
 966         pr_info("... mask length:     %d\n", eax.split.mask_length);
 967
 968         nr_counters_generic = eax.split.num_counters;
 969         nr_counters_fixed = edx.split.num_counters_fixed;
 970         counter_value_mask = (1ULL << eax.split.bit_width) - 1;
 971
 972         return &pmc_intel_ops;
 973 }
 974
 975 static struct pmc_x86_ops *pmc_amd_init(void)
 976 {
 977         nr_counters_generic = 4;
 978         nr_counters_fixed = 0;
 979         counter_value_mask = 0x0000FFFFFFFFFFFFULL;
 980         counter_value_bits = 48;
 981
 982         pr_info("AMD Performance Monitoring support detected.\n");
 983
 984         return &pmc_amd_ops;
 985 }
 986
 987 void __init init_hw_perf_counters(void)
 988 {
 989         if (!cpu_has(&boot_cpu_data, X86_FEATURE_ARCH_PERFMON))
 990                 return;
 991
 992         switch (boot_cpu_data.x86_vendor) {
 993         case X86_VENDOR_INTEL:
 994                 pmc_ops = pmc_intel_init();
 995                 break;
 996         case X86_VENDOR_AMD:
 997                 pmc_ops = pmc_amd_init();
 998                 break;
 999         }
1000         if (!pmc_ops)
1001                 return;
1002
1003         pr_info("... num counters:    %d\n", nr_counters_generic);
1004         if (nr_counters_generic > X86_PMC_MAX_GENERIC) {
1005                 nr_counters_generic = X86_PMC_MAX_GENERIC;
1006                 WARN(1, KERN_ERR "hw perf counters %d > max(%d), clipping!",
1007                         nr_counters_generic, X86_PMC_MAX_GENERIC);
1008         }
1009         perf_counter_mask = (1 << nr_counters_generic) - 1;
1010         perf_max_counters = nr_counters_generic;
1011
1012         pr_info("... value mask:      %016Lx\n", counter_value_mask);
1013
1014         if (nr_counters_fixed > X86_PMC_MAX_FIXED) {
1015                 nr_counters_fixed = X86_PMC_MAX_FIXED;
1016                 WARN(1, KERN_ERR "hw perf counters fixed %d > max(%d), clipping!",
1017                         nr_counters_fixed, X86_PMC_MAX_FIXED);
1018         }
1019         pr_info("... fixed counters:  %d\n", nr_counters_fixed);
1020
1021         perf_counter_mask |= ((1LL << nr_counters_fixed)-1) << X86_PMC_IDX_FIXED;
1022
1023         pr_info("... counter mask:    %016Lx\n", perf_counter_mask);
1024         perf_counters_initialized = true;
1025
1026         perf_counters_lapic_init(0);
1027         register_die_notifier(&perf_counter_nmi_notifier);
1028 }
1029
1030 static void pmc_generic_read(struct perf_counter *counter)
1031 {
1032         x86_perf_counter_update(counter, &counter->hw, counter->hw.idx);
1033 }
1034
1035 static const struct hw_perf_counter_ops x86_perf_counter_ops = {
1036         .enable         = pmc_generic_enable,
1037         .disable        = pmc_generic_disable,
1038         .read           = pmc_generic_read,
1039 };
1040
1041 const struct hw_perf_counter_ops *
1042 hw_perf_counter_init(struct perf_counter *counter)
1043 {
1044         int err;
1045
1046         err = __hw_perf_counter_init(counter);
1047         if (err)
1048                 return ERR_PTR(err);
1049
1050         return &x86_perf_counter_ops;
1051 }
1052
1053 /*
1054  * callchain support
1055  */
1056
1057 static inline
1058 void callchain_store(struct perf_callchain_entry *entry, unsigned long ip)
1059 {
1060         if (entry->nr < MAX_STACK_DEPTH)
1061                 entry->ip[entry->nr++] = ip;
1062 }
1063
1064 static DEFINE_PER_CPU(struct perf_callchain_entry, irq_entry);
1065 static DEFINE_PER_CPU(struct perf_callchain_entry, nmi_entry);
1066
1067
1068 static void
1069 backtrace_warning_symbol(void *data, char *msg, unsigned long symbol)
1070 {
1071         /* Ignore warnings */
1072 }
1073
1074 static void backtrace_warning(void *data, char *msg)
1075 {
1076         /* Ignore warnings */
1077 }
1078
1079 static int backtrace_stack(void *data, char *name)
1080 {
1081         /* Don't bother with IRQ stacks for now */
1082         return -1;
1083 }
1084
1085 static void backtrace_address(void *data, unsigned long addr, int reliable)
1086 {
1087         struct perf_callchain_entry *entry = data;
1088
1089         if (reliable)
1090                 callchain_store(entry, addr);
1091 }
1092
1093 static const struct stacktrace_ops backtrace_ops = {
1094         .warning                = backtrace_warning,
1095         .warning_symbol         = backtrace_warning_symbol,
1096         .stack                  = backtrace_stack,
1097         .address                = backtrace_address,
1098 };
1099
1100 static void
1101 perf_callchain_kernel(struct pt_regs *regs, struct perf_callchain_entry *entry)
1102 {
1103         unsigned long bp;
1104         char *stack;
1105         int nr = entry->nr;
1106
1107         callchain_store(entry, instruction_pointer(regs));
1108
1109         stack = ((char *)regs + sizeof(struct pt_regs));
1110 #ifdef CONFIG_FRAME_POINTER
1111         bp = frame_pointer(regs);
1112 #else
1113         bp = 0;
1114 #endif
1115
1116         dump_trace(NULL, regs, (void *)stack, bp, &backtrace_ops, entry);
1117
1118         entry->kernel = entry->nr - nr;
1119 }
1120
1121
1122 struct stack_frame {
1123         const void __user       *next_fp;
1124         unsigned long           return_address;
1125 };
1126
1127 static int copy_stack_frame(const void __user *fp, struct stack_frame *frame)
1128 {
1129         int ret;
1130
1131         if (!access_ok(VERIFY_READ, fp, sizeof(*frame)))
1132                 return 0;
1133
1134         ret = 1;
1135         pagefault_disable();
1136         if (__copy_from_user_inatomic(frame, fp, sizeof(*frame)))
1137                 ret = 0;
1138         pagefault_enable();
1139
1140         return ret;
1141 }
1142
1143 static void
1144 perf_callchain_user(struct pt_regs *regs, struct perf_callchain_entry *entry)
1145 {
1146         struct stack_frame frame;
1147         const void __user *fp;
1148         int nr = entry->nr;
1149
1150         regs = (struct pt_regs *)current->thread.sp0 - 1;
1151         fp   = (void __user *)regs->bp;
1152
1153         callchain_store(entry, regs->ip);
1154
1155         while (entry->nr < MAX_STACK_DEPTH) {
1156                 frame.next_fp        = NULL;
1157                 frame.return_address = 0;
1158
1159                 if (!copy_stack_frame(fp, &frame))
1160                         break;
1161
1162                 if ((unsigned long)fp < user_stack_pointer(regs))
1163                         break;
1164
1165                 callchain_store(entry, frame.return_address);
1166                 fp = frame.next_fp;
1167         }
1168
1169         entry->user = entry->nr - nr;
1170 }
1171
1172 static void
1173 perf_do_callchain(struct pt_regs *regs, struct perf_callchain_entry *entry)
1174 {
1175         int is_user;
1176
1177         if (!regs)
1178                 return;
1179
1180         is_user = user_mode(regs);
1181
1182         if (!current || current->pid == 0)
1183                 return;
1184
1185         if (is_user && current->state != TASK_RUNNING)
1186                 return;
1187
1188         if (!is_user)
1189                 perf_callchain_kernel(regs, entry);
1190
1191         if (current->mm)
1192                 perf_callchain_user(regs, entry);
1193 }
1194
1195 struct perf_callchain_entry *perf_callchain(struct pt_regs *regs)
1196 {
1197         struct perf_callchain_entry *entry;
1198
1199         if (in_nmi())
1200                 entry = &__get_cpu_var(nmi_entry);
1201         else
1202                 entry = &__get_cpu_var(irq_entry);
1203
1204         entry->nr = 0;
1205         entry->hv = 0;
1206         entry->kernel = 0;
1207         entry->user = 0;
1208
1209         perf_do_callchain(regs, entry);
1210
1211         return entry;
1212 }