ec8b2e12e10452b2a67ffd88bb02067672c285e4
[safe/jmp/linux-2.6] / arch / x86 / kernel / cpu / perf_event_intel_ds.c
1 #ifdef CONFIG_CPU_SUP_INTEL
2
3 /* The maximal number of PEBS events: */
4 #define MAX_PEBS_EVENTS         4
5
6 /* The size of a BTS record in bytes: */
7 #define BTS_RECORD_SIZE         24
8
9 #define BTS_BUFFER_SIZE         (PAGE_SIZE << 4)
10 #define PEBS_BUFFER_SIZE        PAGE_SIZE
11
12 /*
13  * pebs_record_32 for p4 and core not supported
14
15 struct pebs_record_32 {
16         u32 flags, ip;
17         u32 ax, bc, cx, dx;
18         u32 si, di, bp, sp;
19 };
20
21  */
22
23 struct pebs_record_core {
24         u64 flags, ip;
25         u64 ax, bx, cx, dx;
26         u64 si, di, bp, sp;
27         u64 r8,  r9,  r10, r11;
28         u64 r12, r13, r14, r15;
29 };
30
31 struct pebs_record_nhm {
32         u64 flags, ip;
33         u64 ax, bx, cx, dx;
34         u64 si, di, bp, sp;
35         u64 r8,  r9,  r10, r11;
36         u64 r12, r13, r14, r15;
37         u64 status, dla, dse, lat;
38 };
39
40 /*
41  * A debug store configuration.
42  *
43  * We only support architectures that use 64bit fields.
44  */
45 struct debug_store {
46         u64     bts_buffer_base;
47         u64     bts_index;
48         u64     bts_absolute_maximum;
49         u64     bts_interrupt_threshold;
50         u64     pebs_buffer_base;
51         u64     pebs_index;
52         u64     pebs_absolute_maximum;
53         u64     pebs_interrupt_threshold;
54         u64     pebs_event_reset[MAX_PEBS_EVENTS];
55 };
56
57 static void init_debug_store_on_cpu(int cpu)
58 {
59         struct debug_store *ds = per_cpu(cpu_hw_events, cpu).ds;
60
61         if (!ds)
62                 return;
63
64         wrmsr_on_cpu(cpu, MSR_IA32_DS_AREA,
65                      (u32)((u64)(unsigned long)ds),
66                      (u32)((u64)(unsigned long)ds >> 32));
67 }
68
69 static void fini_debug_store_on_cpu(int cpu)
70 {
71         if (!per_cpu(cpu_hw_events, cpu).ds)
72                 return;
73
74         wrmsr_on_cpu(cpu, MSR_IA32_DS_AREA, 0, 0);
75 }
76
77 static void release_ds_buffers(void)
78 {
79         int cpu;
80
81         if (!x86_pmu.bts && !x86_pmu.pebs)
82                 return;
83
84         get_online_cpus();
85
86         for_each_online_cpu(cpu)
87                 fini_debug_store_on_cpu(cpu);
88
89         for_each_possible_cpu(cpu) {
90                 struct debug_store *ds = per_cpu(cpu_hw_events, cpu).ds;
91
92                 if (!ds)
93                         continue;
94
95                 per_cpu(cpu_hw_events, cpu).ds = NULL;
96
97                 kfree((void *)(unsigned long)ds->pebs_buffer_base);
98                 kfree((void *)(unsigned long)ds->bts_buffer_base);
99                 kfree(ds);
100         }
101
102         put_online_cpus();
103 }
104
105 static int reserve_ds_buffers(void)
106 {
107         int cpu, err = 0;
108
109         if (!x86_pmu.bts && !x86_pmu.pebs)
110                 return 0;
111
112         get_online_cpus();
113
114         for_each_possible_cpu(cpu) {
115                 struct debug_store *ds;
116                 void *buffer;
117                 int max, thresh;
118
119                 err = -ENOMEM;
120                 ds = kzalloc(sizeof(*ds), GFP_KERNEL);
121                 if (unlikely(!ds))
122                         break;
123                 per_cpu(cpu_hw_events, cpu).ds = ds;
124
125                 if (x86_pmu.bts) {
126                         buffer = kzalloc(BTS_BUFFER_SIZE, GFP_KERNEL);
127                         if (unlikely(!buffer))
128                                 break;
129
130                         max = BTS_BUFFER_SIZE / BTS_RECORD_SIZE;
131                         thresh = max / 16;
132
133                         ds->bts_buffer_base = (u64)(unsigned long)buffer;
134                         ds->bts_index = ds->bts_buffer_base;
135                         ds->bts_absolute_maximum = ds->bts_buffer_base +
136                                 max * BTS_RECORD_SIZE;
137                         ds->bts_interrupt_threshold = ds->bts_absolute_maximum -
138                                 thresh * BTS_RECORD_SIZE;
139                 }
140
141                 if (x86_pmu.pebs) {
142                         buffer = kzalloc(PEBS_BUFFER_SIZE, GFP_KERNEL);
143                         if (unlikely(!buffer))
144                                 break;
145
146                         max = PEBS_BUFFER_SIZE / x86_pmu.pebs_record_size;
147
148                         ds->pebs_buffer_base = (u64)(unsigned long)buffer;
149                         ds->pebs_index = ds->pebs_buffer_base;
150                         ds->pebs_absolute_maximum = ds->pebs_buffer_base +
151                                 max * x86_pmu.pebs_record_size;
152                         /*
153                          * Always use single record PEBS
154                          */
155                         ds->pebs_interrupt_threshold = ds->pebs_buffer_base +
156                                 x86_pmu.pebs_record_size;
157                 }
158
159                 err = 0;
160         }
161
162         if (err)
163                 release_ds_buffers();
164         else {
165                 for_each_online_cpu(cpu)
166                         init_debug_store_on_cpu(cpu);
167         }
168
169         put_online_cpus();
170
171         return err;
172 }
173
174 /*
175  * BTS
176  */
177
178 static struct event_constraint bts_constraint =
179         EVENT_CONSTRAINT(0, 1ULL << X86_PMC_IDX_FIXED_BTS, 0);
180
181 static void intel_pmu_enable_bts(u64 config)
182 {
183         unsigned long debugctlmsr;
184
185         debugctlmsr = get_debugctlmsr();
186
187         debugctlmsr |= DEBUGCTLMSR_TR;
188         debugctlmsr |= DEBUGCTLMSR_BTS;
189         debugctlmsr |= DEBUGCTLMSR_BTINT;
190
191         if (!(config & ARCH_PERFMON_EVENTSEL_OS))
192                 debugctlmsr |= DEBUGCTLMSR_BTS_OFF_OS;
193
194         if (!(config & ARCH_PERFMON_EVENTSEL_USR))
195                 debugctlmsr |= DEBUGCTLMSR_BTS_OFF_USR;
196
197         update_debugctlmsr(debugctlmsr);
198 }
199
200 static void intel_pmu_disable_bts(void)
201 {
202         struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
203         unsigned long debugctlmsr;
204
205         if (!cpuc->ds)
206                 return;
207
208         debugctlmsr = get_debugctlmsr();
209
210         debugctlmsr &=
211                 ~(DEBUGCTLMSR_TR | DEBUGCTLMSR_BTS | DEBUGCTLMSR_BTINT |
212                   DEBUGCTLMSR_BTS_OFF_OS | DEBUGCTLMSR_BTS_OFF_USR);
213
214         update_debugctlmsr(debugctlmsr);
215 }
216
217 static void intel_pmu_drain_bts_buffer(void)
218 {
219         struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
220         struct debug_store *ds = cpuc->ds;
221         struct bts_record {
222                 u64     from;
223                 u64     to;
224                 u64     flags;
225         };
226         struct perf_event *event = cpuc->events[X86_PMC_IDX_FIXED_BTS];
227         struct bts_record *at, *top;
228         struct perf_output_handle handle;
229         struct perf_event_header header;
230         struct perf_sample_data data;
231         struct pt_regs regs;
232
233         if (!event)
234                 return;
235
236         if (!ds)
237                 return;
238
239         at  = (struct bts_record *)(unsigned long)ds->bts_buffer_base;
240         top = (struct bts_record *)(unsigned long)ds->bts_index;
241
242         if (top <= at)
243                 return;
244
245         ds->bts_index = ds->bts_buffer_base;
246
247         perf_sample_data_init(&data, 0);
248         data.period = event->hw.last_period;
249         regs.ip     = 0;
250
251         /*
252          * Prepare a generic sample, i.e. fill in the invariant fields.
253          * We will overwrite the from and to address before we output
254          * the sample.
255          */
256         perf_prepare_sample(&header, &data, event, &regs);
257
258         if (perf_output_begin(&handle, event, header.size * (top - at), 1, 1))
259                 return;
260
261         for (; at < top; at++) {
262                 data.ip         = at->from;
263                 data.addr       = at->to;
264
265                 perf_output_sample(&handle, &header, &data, event);
266         }
267
268         perf_output_end(&handle);
269
270         /* There's new data available. */
271         event->hw.interrupts++;
272         event->pending_kill = POLL_IN;
273 }
274
275 /*
276  * PEBS
277  */
278
279 static struct event_constraint intel_core_pebs_events[] = {
280         PEBS_EVENT_CONSTRAINT(0x00c0, 0x1), /* INSTR_RETIRED.ANY */
281         PEBS_EVENT_CONSTRAINT(0xfec1, 0x1), /* X87_OPS_RETIRED.ANY */
282         PEBS_EVENT_CONSTRAINT(0x00c5, 0x1), /* BR_INST_RETIRED.MISPRED */
283         PEBS_EVENT_CONSTRAINT(0x1fc7, 0x1), /* SIMD_INST_RETURED.ANY */
284         PEBS_EVENT_CONSTRAINT(0x01cb, 0x1), /* MEM_LOAD_RETIRED.L1D_MISS */
285         PEBS_EVENT_CONSTRAINT(0x02cb, 0x1), /* MEM_LOAD_RETIRED.L1D_LINE_MISS */
286         PEBS_EVENT_CONSTRAINT(0x04cb, 0x1), /* MEM_LOAD_RETIRED.L2_MISS */
287         PEBS_EVENT_CONSTRAINT(0x08cb, 0x1), /* MEM_LOAD_RETIRED.L2_LINE_MISS */
288         PEBS_EVENT_CONSTRAINT(0x10cb, 0x1), /* MEM_LOAD_RETIRED.DTLB_MISS */
289         EVENT_CONSTRAINT_END
290 };
291
292 static struct event_constraint intel_nehalem_pebs_events[] = {
293         PEBS_EVENT_CONSTRAINT(0x00c0, 0xf), /* INSTR_RETIRED.ANY */
294         PEBS_EVENT_CONSTRAINT(0xfec1, 0xf), /* X87_OPS_RETIRED.ANY */
295         PEBS_EVENT_CONSTRAINT(0x00c5, 0xf), /* BR_INST_RETIRED.MISPRED */
296         PEBS_EVENT_CONSTRAINT(0x1fc7, 0xf), /* SIMD_INST_RETURED.ANY */
297         PEBS_EVENT_CONSTRAINT(0x01cb, 0xf), /* MEM_LOAD_RETIRED.L1D_MISS */
298         PEBS_EVENT_CONSTRAINT(0x02cb, 0xf), /* MEM_LOAD_RETIRED.L1D_LINE_MISS */
299         PEBS_EVENT_CONSTRAINT(0x04cb, 0xf), /* MEM_LOAD_RETIRED.L2_MISS */
300         PEBS_EVENT_CONSTRAINT(0x08cb, 0xf), /* MEM_LOAD_RETIRED.L2_LINE_MISS */
301         PEBS_EVENT_CONSTRAINT(0x10cb, 0xf), /* MEM_LOAD_RETIRED.DTLB_MISS */
302         EVENT_CONSTRAINT_END
303 };
304
305 static struct event_constraint *
306 intel_pebs_constraints(struct perf_event *event)
307 {
308         struct event_constraint *c;
309
310         if (!event->attr.precise)
311                 return NULL;
312
313         if (x86_pmu.pebs_constraints) {
314                 for_each_event_constraint(c, x86_pmu.pebs_constraints) {
315                         if ((event->hw.config & c->cmask) == c->code)
316                                 return c;
317                 }
318         }
319
320         return &emptyconstraint;
321 }
322
323 static void intel_pmu_pebs_enable(struct perf_event *event)
324 {
325         struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
326         struct hw_perf_event *hwc = &event->hw;
327
328         hwc->config &= ~ARCH_PERFMON_EVENTSEL_INT;
329
330         cpuc->pebs_enabled |= 1ULL << hwc->idx;
331         WARN_ON_ONCE(cpuc->enabled);
332
333         if (x86_pmu.intel_cap.pebs_trap)
334                 intel_pmu_lbr_enable(event);
335 }
336
337 static void intel_pmu_pebs_disable(struct perf_event *event)
338 {
339         struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
340         struct hw_perf_event *hwc = &event->hw;
341
342         cpuc->pebs_enabled &= ~(1ULL << hwc->idx);
343         if (cpuc->enabled)
344                 wrmsrl(MSR_IA32_PEBS_ENABLE, cpuc->pebs_enabled);
345
346         hwc->config |= ARCH_PERFMON_EVENTSEL_INT;
347
348         if (x86_pmu.intel_cap.pebs_trap)
349                 intel_pmu_lbr_disable(event);
350 }
351
352 static void intel_pmu_pebs_enable_all(void)
353 {
354         struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
355
356         if (cpuc->pebs_enabled)
357                 wrmsrl(MSR_IA32_PEBS_ENABLE, cpuc->pebs_enabled);
358 }
359
360 static void intel_pmu_pebs_disable_all(void)
361 {
362         struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
363
364         if (cpuc->pebs_enabled)
365                 wrmsrl(MSR_IA32_PEBS_ENABLE, 0);
366 }
367
368 #include <asm/insn.h>
369
370 static inline bool kernel_ip(unsigned long ip)
371 {
372 #ifdef CONFIG_X86_32
373         return ip > PAGE_OFFSET;
374 #else
375         return (long)ip < 0;
376 #endif
377 }
378
379 static int intel_pmu_pebs_fixup_ip(struct pt_regs *regs)
380 {
381         struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
382         unsigned long from = cpuc->lbr_entries[0].from;
383         unsigned long old_to, to = cpuc->lbr_entries[0].to;
384         unsigned long ip = regs->ip;
385
386         /*
387          * We don't need to fixup if the PEBS assist is fault like
388          */
389         if (!x86_pmu.intel_cap.pebs_trap)
390                 return 1;
391
392         /*
393          * No LBR entry, no basic block, no rewinding
394          */
395         if (!cpuc->lbr_stack.nr || !from || !to)
396                 return 0;
397
398         /*
399          * Basic blocks should never cross user/kernel boundaries
400          */
401         if (kernel_ip(ip) != kernel_ip(to))
402                 return 0;
403
404         /*
405          * unsigned math, either ip is before the start (impossible) or
406          * the basic block is larger than 1 page (sanity)
407          */
408         if ((ip - to) > PAGE_SIZE)
409                 return 0;
410
411         /*
412          * We sampled a branch insn, rewind using the LBR stack
413          */
414         if (ip == to) {
415                 regs->ip = from;
416                 return 1;
417         }
418
419         do {
420                 struct insn insn;
421                 u8 buf[MAX_INSN_SIZE];
422                 void *kaddr;
423
424                 old_to = to;
425                 if (!kernel_ip(ip)) {
426                         int bytes, size = MAX_INSN_SIZE;
427
428                         bytes = copy_from_user_nmi(buf, (void __user *)to, size);
429                         if (bytes != size)
430                                 return 0;
431
432                         kaddr = buf;
433                 } else
434                         kaddr = (void *)to;
435
436                 kernel_insn_init(&insn, kaddr);
437                 insn_get_length(&insn);
438                 to += insn.length;
439         } while (to < ip);
440
441         if (to == ip) {
442                 regs->ip = old_to;
443                 return 1;
444         }
445
446         /*
447          * Even though we decoded the basic block, the instruction stream
448          * never matched the given IP, either the TO or the IP got corrupted.
449          */
450         return 0;
451 }
452
453 static int intel_pmu_save_and_restart(struct perf_event *event);
454
455 static void intel_pmu_drain_pebs_core(struct pt_regs *iregs)
456 {
457         struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
458         struct debug_store *ds = cpuc->ds;
459         struct perf_event *event = cpuc->events[0]; /* PMC0 only */
460         struct pebs_record_core *at, *top;
461         struct perf_sample_data data;
462         struct perf_raw_record raw;
463         struct pt_regs regs;
464         int n;
465
466         if (!ds || !x86_pmu.pebs)
467                 return;
468
469         at  = (struct pebs_record_core *)(unsigned long)ds->pebs_buffer_base;
470         top = (struct pebs_record_core *)(unsigned long)ds->pebs_index;
471
472         /*
473          * Whatever else happens, drain the thing
474          */
475         ds->pebs_index = ds->pebs_buffer_base;
476
477         if (!test_bit(0, cpuc->active_mask))
478                 return;
479
480         WARN_ON_ONCE(!event);
481
482         if (!event->attr.precise)
483                 return;
484
485         n = top - at;
486         if (n <= 0)
487                 return;
488
489         if (!intel_pmu_save_and_restart(event))
490                 return;
491
492         /*
493          * Should not happen, we program the threshold at 1 and do not
494          * set a reset value.
495          */
496         WARN_ON_ONCE(n > 1);
497         at += n - 1;
498
499         perf_sample_data_init(&data, 0);
500         data.period = event->hw.last_period;
501
502         if (event->attr.sample_type & PERF_SAMPLE_RAW) {
503                 raw.size = x86_pmu.pebs_record_size;
504                 raw.data = at;
505                 data.raw = &raw;
506         }
507
508         /*
509          * We use the interrupt regs as a base because the PEBS record
510          * does not contain a full regs set, specifically it seems to
511          * lack segment descriptors, which get used by things like
512          * user_mode().
513          *
514          * In the simple case fix up only the IP and BP,SP regs, for
515          * PERF_SAMPLE_IP and PERF_SAMPLE_CALLCHAIN to function properly.
516          * A possible PERF_SAMPLE_REGS will have to transfer all regs.
517          */
518         regs = *iregs;
519         regs.ip = at->ip;
520         regs.bp = at->bp;
521         regs.sp = at->sp;
522
523         if (intel_pmu_pebs_fixup_ip(&regs))
524                 regs.flags |= PERF_EFLAGS_EXACT;
525         else
526                 regs.flags &= ~PERF_EFLAGS_EXACT;
527
528         if (perf_event_overflow(event, 1, &data, &regs))
529                 x86_pmu_stop(event);
530 }
531
532 static void intel_pmu_drain_pebs_nhm(struct pt_regs *iregs)
533 {
534         struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events);
535         struct debug_store *ds = cpuc->ds;
536         struct pebs_record_nhm *at, *top;
537         struct perf_sample_data data;
538         struct perf_event *event = NULL;
539         struct perf_raw_record raw;
540         struct pt_regs regs;
541         u64 status = 0;
542         int bit, n;
543
544         if (!ds || !x86_pmu.pebs)
545                 return;
546
547         at  = (struct pebs_record_nhm *)(unsigned long)ds->pebs_buffer_base;
548         top = (struct pebs_record_nhm *)(unsigned long)ds->pebs_index;
549
550         ds->pebs_index = ds->pebs_buffer_base;
551
552         n = top - at;
553         if (n <= 0)
554                 return;
555
556         /*
557          * Should not happen, we program the threshold at 1 and do not
558          * set a reset value.
559          */
560         WARN_ON_ONCE(n > MAX_PEBS_EVENTS);
561
562         for ( ; at < top; at++) {
563                 for_each_set_bit(bit, (unsigned long *)&at->status, MAX_PEBS_EVENTS) {
564                         event = cpuc->events[bit];
565                         if (!test_bit(bit, cpuc->active_mask))
566                                 continue;
567
568                         WARN_ON_ONCE(!event);
569
570                         if (!event->attr.precise)
571                                 continue;
572
573                         if (__test_and_set_bit(bit, (unsigned long *)&status))
574                                 continue;
575
576                         break;
577                 }
578
579                 if (!event || bit >= MAX_PEBS_EVENTS)
580                         continue;
581
582                 if (!intel_pmu_save_and_restart(event))
583                         continue;
584
585                 perf_sample_data_init(&data, 0);
586                 data.period = event->hw.last_period;
587
588                 if (event->attr.sample_type & PERF_SAMPLE_RAW) {
589                         raw.size = x86_pmu.pebs_record_size;
590                         raw.data = at;
591                         data.raw = &raw;
592                 }
593
594                 /*
595                  * See the comment in intel_pmu_drain_pebs_core()
596                  */
597                 regs = *iregs;
598                 regs.ip = at->ip;
599                 regs.bp = at->bp;
600                 regs.sp = at->sp;
601
602                 if (intel_pmu_pebs_fixup_ip(&regs))
603                         regs.flags |= PERF_EFLAGS_EXACT;
604                 else
605                         regs.flags &= ~PERF_EFLAGS_EXACT;
606
607                 if (perf_event_overflow(event, 1, &data, &regs))
608                         x86_pmu_stop(event);
609         }
610 }
611
612 /*
613  * BTS, PEBS probe and setup
614  */
615
616 static void intel_ds_init(void)
617 {
618         /*
619          * No support for 32bit formats
620          */
621         if (!boot_cpu_has(X86_FEATURE_DTES64))
622                 return;
623
624         x86_pmu.bts  = boot_cpu_has(X86_FEATURE_BTS);
625         x86_pmu.pebs = boot_cpu_has(X86_FEATURE_PEBS);
626         if (x86_pmu.pebs) {
627                 char pebs_type = x86_pmu.intel_cap.pebs_trap ?  '+' : '-';
628                 int format = x86_pmu.intel_cap.pebs_format;
629
630                 switch (format) {
631                 case 0:
632                         printk(KERN_CONT "PEBS fmt0%c, ", pebs_type);
633                         x86_pmu.pebs_record_size = sizeof(struct pebs_record_core);
634                         x86_pmu.drain_pebs = intel_pmu_drain_pebs_core;
635                         x86_pmu.pebs_constraints = intel_core_pebs_events;
636                         break;
637
638                 case 1:
639                         printk(KERN_CONT "PEBS fmt1%c, ", pebs_type);
640                         x86_pmu.pebs_record_size = sizeof(struct pebs_record_nhm);
641                         x86_pmu.drain_pebs = intel_pmu_drain_pebs_nhm;
642                         x86_pmu.pebs_constraints = intel_nehalem_pebs_events;
643                         break;
644
645                 default:
646                         printk(KERN_CONT "no PEBS fmt%d%c, ", format, pebs_type);
647                         x86_pmu.pebs = 0;
648                         break;
649                 }
650         }
651 }
652
653 #else /* CONFIG_CPU_SUP_INTEL */
654
655 static int reserve_ds_buffers(void)
656 {
657         return 0;
658 }
659
660 static void release_ds_buffers(void)
661 {
662 }
663
664 #endif /* CONFIG_CPU_SUP_INTEL */