SAFE public projects git trees. - safe/jmp/linux-2.6/blob - kernel/trace/trace_functions_graph.c

   1 /*
   2  *
   3  * Function graph tracer.
   4  * Copyright (c) 2008-2009 Frederic Weisbecker <fweisbec@gmail.com>
   5  * Mostly borrowed from function tracer which
   6  * is Copyright (c) Steven Rostedt <srostedt@redhat.com>
   7  *
   8  */
   9 #include <linux/debugfs.h>
  10 #include <linux/uaccess.h>
  11 #include <linux/ftrace.h>
  12 #include <linux/fs.h>
  13
  14 #include "trace.h"
  15 #include "trace_output.h"
  16
  17 struct fgraph_cpu_data {
  18         pid_t           last_pid;
  19         int             depth;
  20         int             ignore;
  21 };
  22
  23 struct fgraph_data {
  24         struct fgraph_cpu_data          *cpu_data;
  25
  26         /* Place to preserve last processed entry. */
  27         struct ftrace_graph_ent_entry   ent;
  28         struct ftrace_graph_ret_entry   ret;
  29         int                             failed;
  30         int                             cpu;
  31 };
  32
  33 #define TRACE_GRAPH_INDENT      2
  34
  35 /* Flag options */
  36 #define TRACE_GRAPH_PRINT_OVERRUN       0x1
  37 #define TRACE_GRAPH_PRINT_CPU           0x2
  38 #define TRACE_GRAPH_PRINT_OVERHEAD      0x4
  39 #define TRACE_GRAPH_PRINT_PROC          0x8
  40 #define TRACE_GRAPH_PRINT_DURATION      0x10
  41 #define TRACE_GRAPH_PRINT_ABS_TIME      0X20
  42
  43 static struct tracer_opt trace_opts[] = {
  44         /* Display overruns? (for self-debug purpose) */
  45         { TRACER_OPT(funcgraph-overrun, TRACE_GRAPH_PRINT_OVERRUN) },
  46         /* Display CPU ? */
  47         { TRACER_OPT(funcgraph-cpu, TRACE_GRAPH_PRINT_CPU) },
  48         /* Display Overhead ? */
  49         { TRACER_OPT(funcgraph-overhead, TRACE_GRAPH_PRINT_OVERHEAD) },
  50         /* Display proc name/pid */
  51         { TRACER_OPT(funcgraph-proc, TRACE_GRAPH_PRINT_PROC) },
  52         /* Display duration of execution */
  53         { TRACER_OPT(funcgraph-duration, TRACE_GRAPH_PRINT_DURATION) },
  54         /* Display absolute time of an entry */
  55         { TRACER_OPT(funcgraph-abstime, TRACE_GRAPH_PRINT_ABS_TIME) },
  56         { } /* Empty entry */
  57 };
  58
  59 static struct tracer_flags tracer_flags = {
  60         /* Don't display overruns and proc by default */
  61         .val = TRACE_GRAPH_PRINT_CPU | TRACE_GRAPH_PRINT_OVERHEAD |
  62                TRACE_GRAPH_PRINT_DURATION,
  63         .opts = trace_opts
  64 };
  65
  66 static struct trace_array *graph_array;
  67
  68
  69 /* Add a function return address to the trace stack on thread info.*/
  70 int
  71 ftrace_push_return_trace(unsigned long ret, unsigned long func, int *depth,
  72                          unsigned long frame_pointer)
  73 {
  74         unsigned long long calltime;
  75         int index;
  76
  77         if (!current->ret_stack)
  78                 return -EBUSY;
  79
  80         /*
  81          * We must make sure the ret_stack is tested before we read
  82          * anything else.
  83          */
  84         smp_rmb();
  85
  86         /* The return trace stack is full */
  87         if (current->curr_ret_stack == FTRACE_RETFUNC_DEPTH - 1) {
  88                 atomic_inc(&current->trace_overrun);
  89                 return -EBUSY;
  90         }
  91
  92         calltime = trace_clock_local();
  93
  94         index = ++current->curr_ret_stack;
  95         barrier();
  96         current->ret_stack[index].ret = ret;
  97         current->ret_stack[index].func = func;
  98         current->ret_stack[index].calltime = calltime;
  99         current->ret_stack[index].subtime = 0;
 100         current->ret_stack[index].fp = frame_pointer;
 101         *depth = index;
 102
 103         return 0;
 104 }
 105
 106 /* Retrieve a function return address to the trace stack on thread info.*/
 107 static void
 108 ftrace_pop_return_trace(struct ftrace_graph_ret *trace, unsigned long *ret,
 109                         unsigned long frame_pointer)
 110 {
 111         int index;
 112
 113         index = current->curr_ret_stack;
 114
 115         if (unlikely(index < 0)) {
 116                 ftrace_graph_stop();
 117                 WARN_ON(1);
 118                 /* Might as well panic, otherwise we have no where to go */
 119                 *ret = (unsigned long)panic;
 120                 return;
 121         }
 122
 123 #ifdef CONFIG_HAVE_FUNCTION_GRAPH_FP_TEST
 124         /*
 125          * The arch may choose to record the frame pointer used
 126          * and check it here to make sure that it is what we expect it
 127          * to be. If gcc does not set the place holder of the return
 128          * address in the frame pointer, and does a copy instead, then
 129          * the function graph trace will fail. This test detects this
 130          * case.
 131          *
 132          * Currently, x86_32 with optimize for size (-Os) makes the latest
 133          * gcc do the above.
 134          */
 135         if (unlikely(current->ret_stack[index].fp != frame_pointer)) {
 136                 ftrace_graph_stop();
 137                 WARN(1, "Bad frame pointer: expected %lx, received %lx\n"
 138                      "  from func %ps return to %lx\n",
 139                      current->ret_stack[index].fp,
 140                      frame_pointer,
 141                      (void *)current->ret_stack[index].func,
 142                      current->ret_stack[index].ret);
 143                 *ret = (unsigned long)panic;
 144                 return;
 145         }
 146 #endif
 147
 148         *ret = current->ret_stack[index].ret;
 149         trace->func = current->ret_stack[index].func;
 150         trace->calltime = current->ret_stack[index].calltime;
 151         trace->overrun = atomic_read(&current->trace_overrun);
 152         trace->depth = index;
 153 }
 154
 155 /*
 156  * Send the trace to the ring-buffer.
 157  * @return the original return address.
 158  */
 159 unsigned long ftrace_return_to_handler(unsigned long frame_pointer)
 160 {
 161         struct ftrace_graph_ret trace;
 162         unsigned long ret;
 163
 164         ftrace_pop_return_trace(&trace, &ret, frame_pointer);
 165         trace.rettime = trace_clock_local();
 166         ftrace_graph_return(&trace);
 167         barrier();
 168         current->curr_ret_stack--;
 169
 170         if (unlikely(!ret)) {
 171                 ftrace_graph_stop();
 172                 WARN_ON(1);
 173                 /* Might as well panic. What else to do? */
 174                 ret = (unsigned long)panic;
 175         }
 176
 177         return ret;
 178 }
 179
 180 static int __trace_graph_entry(struct trace_array *tr,
 181                                 struct ftrace_graph_ent *trace,
 182                                 unsigned long flags,
 183                                 int pc)
 184 {
 185         struct ftrace_event_call *call = &event_funcgraph_entry;
 186         struct ring_buffer_event *event;
 187         struct ring_buffer *buffer = tr->buffer;
 188         struct ftrace_graph_ent_entry *entry;
 189
 190         if (unlikely(__this_cpu_read(per_cpu_var(ftrace_cpu_disabled))))
 191                 return 0;
 192
 193         event = trace_buffer_lock_reserve(buffer, TRACE_GRAPH_ENT,
 194                                           sizeof(*entry), flags, pc);
 195         if (!event)
 196                 return 0;
 197         entry   = ring_buffer_event_data(event);
 198         entry->graph_ent                        = *trace;
 199         if (!filter_current_check_discard(buffer, call, entry, event))
 200                 ring_buffer_unlock_commit(buffer, event);
 201
 202         return 1;
 203 }
 204
 205 int trace_graph_entry(struct ftrace_graph_ent *trace)
 206 {
 207         struct trace_array *tr = graph_array;
 208         struct trace_array_cpu *data;
 209         unsigned long flags;
 210         long disabled;
 211         int ret;
 212         int cpu;
 213         int pc;
 214
 215         if (!ftrace_trace_task(current))
 216                 return 0;
 217
 218         if (!ftrace_graph_addr(trace->func))
 219                 return 0;
 220
 221         local_irq_save(flags);
 222         cpu = raw_smp_processor_id();
 223         data = tr->data[cpu];
 224         disabled = atomic_inc_return(&data->disabled);
 225         if (likely(disabled == 1)) {
 226                 pc = preempt_count();
 227                 ret = __trace_graph_entry(tr, trace, flags, pc);
 228         } else {
 229                 ret = 0;
 230         }
 231         /* Only do the atomic if it is not already set */
 232         if (!test_tsk_trace_graph(current))
 233                 set_tsk_trace_graph(current);
 234
 235         atomic_dec(&data->disabled);
 236         local_irq_restore(flags);
 237
 238         return ret;
 239 }
 240
 241 static void __trace_graph_return(struct trace_array *tr,
 242                                 struct ftrace_graph_ret *trace,
 243                                 unsigned long flags,
 244                                 int pc)
 245 {
 246         struct ftrace_event_call *call = &event_funcgraph_exit;
 247         struct ring_buffer_event *event;
 248         struct ring_buffer *buffer = tr->buffer;
 249         struct ftrace_graph_ret_entry *entry;
 250
 251         if (unlikely(__this_cpu_read(per_cpu_var(ftrace_cpu_disabled))))
 252                 return;
 253
 254         event = trace_buffer_lock_reserve(buffer, TRACE_GRAPH_RET,
 255                                           sizeof(*entry), flags, pc);
 256         if (!event)
 257                 return;
 258         entry   = ring_buffer_event_data(event);
 259         entry->ret                              = *trace;
 260         if (!filter_current_check_discard(buffer, call, entry, event))
 261                 ring_buffer_unlock_commit(buffer, event);
 262 }
 263
 264 void trace_graph_return(struct ftrace_graph_ret *trace)
 265 {
 266         struct trace_array *tr = graph_array;
 267         struct trace_array_cpu *data;
 268         unsigned long flags;
 269         long disabled;
 270         int cpu;
 271         int pc;
 272
 273         local_irq_save(flags);
 274         cpu = raw_smp_processor_id();
 275         data = tr->data[cpu];
 276         disabled = atomic_inc_return(&data->disabled);
 277         if (likely(disabled == 1)) {
 278                 pc = preempt_count();
 279                 __trace_graph_return(tr, trace, flags, pc);
 280         }
 281         if (!trace->depth)
 282                 clear_tsk_trace_graph(current);
 283         atomic_dec(&data->disabled);
 284         local_irq_restore(flags);
 285 }
 286
 287 void set_graph_array(struct trace_array *tr)
 288 {
 289         graph_array = tr;
 290
 291         /* Make graph_array visible before we start tracing */
 292
 293         smp_mb();
 294 }
 295
 296 static int graph_trace_init(struct trace_array *tr)
 297 {
 298         int ret;
 299
 300         set_graph_array(tr);
 301         ret = register_ftrace_graph(&trace_graph_return,
 302                                     &trace_graph_entry);
 303         if (ret)
 304                 return ret;
 305         tracing_start_cmdline_record();
 306
 307         return 0;
 308 }
 309
 310 static void graph_trace_reset(struct trace_array *tr)
 311 {
 312         tracing_stop_cmdline_record();
 313         unregister_ftrace_graph();
 314 }
 315
 316 static int max_bytes_for_cpu;
 317
 318 static enum print_line_t
 319 print_graph_cpu(struct trace_seq *s, int cpu)
 320 {
 321         int ret;
 322
 323         /*
 324          * Start with a space character - to make it stand out
 325          * to the right a bit when trace output is pasted into
 326          * email:
 327          */
 328         ret = trace_seq_printf(s, " %*d) ", max_bytes_for_cpu, cpu);
 329         if (!ret)
 330                 return TRACE_TYPE_PARTIAL_LINE;
 331
 332         return TRACE_TYPE_HANDLED;
 333 }
 334
 335 #define TRACE_GRAPH_PROCINFO_LENGTH     14
 336
 337 static enum print_line_t
 338 print_graph_proc(struct trace_seq *s, pid_t pid)
 339 {
 340         char comm[TASK_COMM_LEN];
 341         /* sign + log10(MAX_INT) + '\0' */
 342         char pid_str[11];
 343         int spaces = 0;
 344         int ret;
 345         int len;
 346         int i;
 347
 348         trace_find_cmdline(pid, comm);
 349         comm[7] = '\0';
 350         sprintf(pid_str, "%d", pid);
 351
 352         /* 1 stands for the "-" character */
 353         len = strlen(comm) + strlen(pid_str) + 1;
 354
 355         if (len < TRACE_GRAPH_PROCINFO_LENGTH)
 356                 spaces = TRACE_GRAPH_PROCINFO_LENGTH - len;
 357
 358         /* First spaces to align center */
 359         for (i = 0; i < spaces / 2; i++) {
 360                 ret = trace_seq_printf(s, " ");
 361                 if (!ret)
 362                         return TRACE_TYPE_PARTIAL_LINE;
 363         }
 364
 365         ret = trace_seq_printf(s, "%s-%s", comm, pid_str);
 366         if (!ret)
 367                 return TRACE_TYPE_PARTIAL_LINE;
 368
 369         /* Last spaces to align center */
 370         for (i = 0; i < spaces - (spaces / 2); i++) {
 371                 ret = trace_seq_printf(s, " ");
 372                 if (!ret)
 373                         return TRACE_TYPE_PARTIAL_LINE;
 374         }
 375         return TRACE_TYPE_HANDLED;
 376 }
 377
 378
 379 static enum print_line_t
 380 print_graph_lat_fmt(struct trace_seq *s, struct trace_entry *entry)
 381 {
 382         if (!trace_seq_putc(s, ' '))
 383                 return 0;
 384
 385         return trace_print_lat_fmt(s, entry);
 386 }
 387
 388 /* If the pid changed since the last trace, output this event */
 389 static enum print_line_t
 390 verif_pid(struct trace_seq *s, pid_t pid, int cpu, struct fgraph_data *data)
 391 {
 392         pid_t prev_pid;
 393         pid_t *last_pid;
 394         int ret;
 395
 396         if (!data)
 397                 return TRACE_TYPE_HANDLED;
 398
 399         last_pid = &(per_cpu_ptr(data->cpu_data, cpu)->last_pid);
 400
 401         if (*last_pid == pid)
 402                 return TRACE_TYPE_HANDLED;
 403
 404         prev_pid = *last_pid;
 405         *last_pid = pid;
 406
 407         if (prev_pid == -1)
 408                 return TRACE_TYPE_HANDLED;
 409 /*
 410  * Context-switch trace line:
 411
 412  ------------------------------------------
 413  | 1)  migration/0--1  =>  sshd-1755
 414  ------------------------------------------
 415
 416  */
 417         ret = trace_seq_printf(s,
 418                 " ------------------------------------------\n");
 419         if (!ret)
 420                 return TRACE_TYPE_PARTIAL_LINE;
 421
 422         ret = print_graph_cpu(s, cpu);
 423         if (ret == TRACE_TYPE_PARTIAL_LINE)
 424                 return TRACE_TYPE_PARTIAL_LINE;
 425
 426         ret = print_graph_proc(s, prev_pid);
 427         if (ret == TRACE_TYPE_PARTIAL_LINE)
 428                 return TRACE_TYPE_PARTIAL_LINE;
 429
 430         ret = trace_seq_printf(s, " => ");
 431         if (!ret)
 432                 return TRACE_TYPE_PARTIAL_LINE;
 433
 434         ret = print_graph_proc(s, pid);
 435         if (ret == TRACE_TYPE_PARTIAL_LINE)
 436                 return TRACE_TYPE_PARTIAL_LINE;
 437
 438         ret = trace_seq_printf(s,
 439                 "\n ------------------------------------------\n\n");
 440         if (!ret)
 441                 return TRACE_TYPE_PARTIAL_LINE;
 442
 443         return TRACE_TYPE_HANDLED;
 444 }
 445
 446 static struct ftrace_graph_ret_entry *
 447 get_return_for_leaf(struct trace_iterator *iter,
 448                 struct ftrace_graph_ent_entry *curr)
 449 {
 450         struct fgraph_data *data = iter->private;
 451         struct ring_buffer_iter *ring_iter = NULL;
 452         struct ring_buffer_event *event;
 453         struct ftrace_graph_ret_entry *next;
 454
 455         /*
 456          * If the previous output failed to write to the seq buffer,
 457          * then we just reuse the data from before.
 458          */
 459         if (data && data->failed) {
 460                 curr = &data->ent;
 461                 next = &data->ret;
 462         } else {
 463
 464                 ring_iter = iter->buffer_iter[iter->cpu];
 465
 466                 /* First peek to compare current entry and the next one */
 467                 if (ring_iter)
 468                         event = ring_buffer_iter_peek(ring_iter, NULL);
 469                 else {
 470                         /*
 471                          * We need to consume the current entry to see
 472                          * the next one.
 473                          */
 474                         ring_buffer_consume(iter->tr->buffer, iter->cpu, NULL);
 475                         event = ring_buffer_peek(iter->tr->buffer, iter->cpu,
 476                                                  NULL);
 477                 }
 478
 479                 if (!event)
 480                         return NULL;
 481
 482                 next = ring_buffer_event_data(event);
 483
 484                 if (data) {
 485                         /*
 486                          * Save current and next entries for later reference
 487                          * if the output fails.
 488                          */
 489                         data->ent = *curr;
 490                         data->ret = *next;
 491                 }
 492         }
 493
 494         if (next->ent.type != TRACE_GRAPH_RET)
 495                 return NULL;
 496
 497         if (curr->ent.pid != next->ent.pid ||
 498                         curr->graph_ent.func != next->ret.func)
 499                 return NULL;
 500
 501         /* this is a leaf, now advance the iterator */
 502         if (ring_iter)
 503                 ring_buffer_read(ring_iter, NULL);
 504
 505         return next;
 506 }
 507
 508 /* Signal a overhead of time execution to the output */
 509 static int
 510 print_graph_overhead(unsigned long long duration, struct trace_seq *s)
 511 {
 512         /* If duration disappear, we don't need anything */
 513         if (!(tracer_flags.val & TRACE_GRAPH_PRINT_DURATION))
 514                 return 1;
 515
 516         /* Non nested entry or return */
 517         if (duration == -1)
 518                 return trace_seq_printf(s, "  ");
 519
 520         if (tracer_flags.val & TRACE_GRAPH_PRINT_OVERHEAD) {
 521                 /* Duration exceeded 100 msecs */
 522                 if (duration > 100000ULL)
 523                         return trace_seq_printf(s, "! ");
 524
 525                 /* Duration exceeded 10 msecs */
 526                 if (duration > 10000ULL)
 527                         return trace_seq_printf(s, "+ ");
 528         }
 529
 530         return trace_seq_printf(s, "  ");
 531 }
 532
 533 static int print_graph_abs_time(u64 t, struct trace_seq *s)
 534 {
 535         unsigned long usecs_rem;
 536
 537         usecs_rem = do_div(t, NSEC_PER_SEC);
 538         usecs_rem /= 1000;
 539
 540         return trace_seq_printf(s, "%5lu.%06lu |  ",
 541                         (unsigned long)t, usecs_rem);
 542 }
 543
 544 static enum print_line_t
 545 print_graph_irq(struct trace_iterator *iter, unsigned long addr,
 546                 enum trace_type type, int cpu, pid_t pid)
 547 {
 548         int ret;
 549         struct trace_seq *s = &iter->seq;
 550
 551         if (addr < (unsigned long)__irqentry_text_start ||
 552                 addr >= (unsigned long)__irqentry_text_end)
 553                 return TRACE_TYPE_UNHANDLED;
 554
 555         /* Absolute time */
 556         if (tracer_flags.val & TRACE_GRAPH_PRINT_ABS_TIME) {
 557                 ret = print_graph_abs_time(iter->ts, s);
 558                 if (!ret)
 559                         return TRACE_TYPE_PARTIAL_LINE;
 560         }
 561
 562         /* Cpu */
 563         if (tracer_flags.val & TRACE_GRAPH_PRINT_CPU) {
 564                 ret = print_graph_cpu(s, cpu);
 565                 if (ret == TRACE_TYPE_PARTIAL_LINE)
 566                         return TRACE_TYPE_PARTIAL_LINE;
 567         }
 568
 569         /* Proc */
 570         if (tracer_flags.val & TRACE_GRAPH_PRINT_PROC) {
 571                 ret = print_graph_proc(s, pid);
 572                 if (ret == TRACE_TYPE_PARTIAL_LINE)
 573                         return TRACE_TYPE_PARTIAL_LINE;
 574                 ret = trace_seq_printf(s, " | ");
 575                 if (!ret)
 576                         return TRACE_TYPE_PARTIAL_LINE;
 577         }
 578
 579         /* No overhead */
 580         ret = print_graph_overhead(-1, s);
 581         if (!ret)
 582                 return TRACE_TYPE_PARTIAL_LINE;
 583
 584         if (type == TRACE_GRAPH_ENT)
 585                 ret = trace_seq_printf(s, "==========>");
 586         else
 587                 ret = trace_seq_printf(s, "<==========");
 588
 589         if (!ret)
 590                 return TRACE_TYPE_PARTIAL_LINE;
 591
 592         /* Don't close the duration column if haven't one */
 593         if (tracer_flags.val & TRACE_GRAPH_PRINT_DURATION)
 594                 trace_seq_printf(s, " |");
 595         ret = trace_seq_printf(s, "\n");
 596
 597         if (!ret)
 598                 return TRACE_TYPE_PARTIAL_LINE;
 599         return TRACE_TYPE_HANDLED;
 600 }
 601
 602 enum print_line_t
 603 trace_print_graph_duration(unsigned long long duration, struct trace_seq *s)
 604 {
 605         unsigned long nsecs_rem = do_div(duration, 1000);
 606         /* log10(ULONG_MAX) + '\0' */
 607         char msecs_str[21];
 608         char nsecs_str[5];
 609         int ret, len;
 610         int i;
 611
 612         sprintf(msecs_str, "%lu", (unsigned long) duration);
 613
 614         /* Print msecs */
 615         ret = trace_seq_printf(s, "%s", msecs_str);
 616         if (!ret)
 617                 return TRACE_TYPE_PARTIAL_LINE;
 618
 619         len = strlen(msecs_str);
 620
 621         /* Print nsecs (we don't want to exceed 7 numbers) */
 622         if (len < 7) {
 623                 snprintf(nsecs_str, 8 - len, "%03lu", nsecs_rem);
 624                 ret = trace_seq_printf(s, ".%s", nsecs_str);
 625                 if (!ret)
 626                         return TRACE_TYPE_PARTIAL_LINE;
 627                 len += strlen(nsecs_str);
 628         }
 629
 630         ret = trace_seq_printf(s, " us ");
 631         if (!ret)
 632                 return TRACE_TYPE_PARTIAL_LINE;
 633
 634         /* Print remaining spaces to fit the row's width */
 635         for (i = len; i < 7; i++) {
 636                 ret = trace_seq_printf(s, " ");
 637                 if (!ret)
 638                         return TRACE_TYPE_PARTIAL_LINE;
 639         }
 640         return TRACE_TYPE_HANDLED;
 641 }
 642
 643 static enum print_line_t
 644 print_graph_duration(unsigned long long duration, struct trace_seq *s)
 645 {
 646         int ret;
 647
 648         ret = trace_print_graph_duration(duration, s);
 649         if (ret != TRACE_TYPE_HANDLED)
 650                 return ret;
 651
 652         ret = trace_seq_printf(s, "|  ");
 653         if (!ret)
 654                 return TRACE_TYPE_PARTIAL_LINE;
 655
 656         return TRACE_TYPE_HANDLED;
 657 }
 658
 659 /* Case of a leaf function on its call entry */
 660 static enum print_line_t
 661 print_graph_entry_leaf(struct trace_iterator *iter,
 662                 struct ftrace_graph_ent_entry *entry,
 663                 struct ftrace_graph_ret_entry *ret_entry, struct trace_seq *s)
 664 {
 665         struct fgraph_data *data = iter->private;
 666         struct ftrace_graph_ret *graph_ret;
 667         struct ftrace_graph_ent *call;
 668         unsigned long long duration;
 669         int ret;
 670         int i;
 671
 672         graph_ret = &ret_entry->ret;
 673         call = &entry->graph_ent;
 674         duration = graph_ret->rettime - graph_ret->calltime;
 675
 676         if (data) {
 677                 int cpu = iter->cpu;
 678                 int *depth = &(per_cpu_ptr(data->cpu_data, cpu)->depth);
 679
 680                 /*
 681                  * Comments display at + 1 to depth. Since
 682                  * this is a leaf function, keep the comments
 683                  * equal to this depth.
 684                  */
 685                 *depth = call->depth - 1;
 686         }
 687
 688         /* Overhead */
 689         ret = print_graph_overhead(duration, s);
 690         if (!ret)
 691                 return TRACE_TYPE_PARTIAL_LINE;
 692
 693         /* Duration */
 694         if (tracer_flags.val & TRACE_GRAPH_PRINT_DURATION) {
 695                 ret = print_graph_duration(duration, s);
 696                 if (ret == TRACE_TYPE_PARTIAL_LINE)
 697                         return TRACE_TYPE_PARTIAL_LINE;
 698         }
 699
 700         /* Function */
 701         for (i = 0; i < call->depth * TRACE_GRAPH_INDENT; i++) {
 702                 ret = trace_seq_printf(s, " ");
 703                 if (!ret)
 704                         return TRACE_TYPE_PARTIAL_LINE;
 705         }
 706
 707         ret = trace_seq_printf(s, "%ps();\n", (void *)call->func);
 708         if (!ret)
 709                 return TRACE_TYPE_PARTIAL_LINE;
 710
 711         return TRACE_TYPE_HANDLED;
 712 }
 713
 714 static enum print_line_t
 715 print_graph_entry_nested(struct trace_iterator *iter,
 716                          struct ftrace_graph_ent_entry *entry,
 717                          struct trace_seq *s, int cpu)
 718 {
 719         struct ftrace_graph_ent *call = &entry->graph_ent;
 720         struct fgraph_data *data = iter->private;
 721         int ret;
 722         int i;
 723
 724         if (data) {
 725                 int cpu = iter->cpu;
 726                 int *depth = &(per_cpu_ptr(data->cpu_data, cpu)->depth);
 727
 728                 *depth = call->depth;
 729         }
 730
 731         /* No overhead */
 732         ret = print_graph_overhead(-1, s);
 733         if (!ret)
 734                 return TRACE_TYPE_PARTIAL_LINE;
 735
 736         /* No time */
 737         if (tracer_flags.val & TRACE_GRAPH_PRINT_DURATION) {
 738                 ret = trace_seq_printf(s, "            |  ");
 739                 if (!ret)
 740                         return TRACE_TYPE_PARTIAL_LINE;
 741         }
 742
 743         /* Function */
 744         for (i = 0; i < call->depth * TRACE_GRAPH_INDENT; i++) {
 745                 ret = trace_seq_printf(s, " ");
 746                 if (!ret)
 747                         return TRACE_TYPE_PARTIAL_LINE;
 748         }
 749
 750         ret = trace_seq_printf(s, "%ps() {\n", (void *)call->func);
 751         if (!ret)
 752                 return TRACE_TYPE_PARTIAL_LINE;
 753
 754         /*
 755          * we already consumed the current entry to check the next one
 756          * and see if this is a leaf.
 757          */
 758         return TRACE_TYPE_NO_CONSUME;
 759 }
 760
 761 static enum print_line_t
 762 print_graph_prologue(struct trace_iterator *iter, struct trace_seq *s,
 763                      int type, unsigned long addr)
 764 {
 765         struct fgraph_data *data = iter->private;
 766         struct trace_entry *ent = iter->ent;
 767         int cpu = iter->cpu;
 768         int ret;
 769
 770         /* Pid */
 771         if (verif_pid(s, ent->pid, cpu, data) == TRACE_TYPE_PARTIAL_LINE)
 772                 return TRACE_TYPE_PARTIAL_LINE;
 773
 774         if (type) {
 775                 /* Interrupt */
 776                 ret = print_graph_irq(iter, addr, type, cpu, ent->pid);
 777                 if (ret == TRACE_TYPE_PARTIAL_LINE)
 778                         return TRACE_TYPE_PARTIAL_LINE;
 779         }
 780
 781         /* Absolute time */
 782         if (tracer_flags.val & TRACE_GRAPH_PRINT_ABS_TIME) {
 783                 ret = print_graph_abs_time(iter->ts, s);
 784                 if (!ret)
 785                         return TRACE_TYPE_PARTIAL_LINE;
 786         }
 787
 788         /* Cpu */
 789         if (tracer_flags.val & TRACE_GRAPH_PRINT_CPU) {
 790                 ret = print_graph_cpu(s, cpu);
 791                 if (ret == TRACE_TYPE_PARTIAL_LINE)
 792                         return TRACE_TYPE_PARTIAL_LINE;
 793         }
 794
 795         /* Proc */
 796         if (tracer_flags.val & TRACE_GRAPH_PRINT_PROC) {
 797                 ret = print_graph_proc(s, ent->pid);
 798                 if (ret == TRACE_TYPE_PARTIAL_LINE)
 799                         return TRACE_TYPE_PARTIAL_LINE;
 800
 801                 ret = trace_seq_printf(s, " | ");
 802                 if (!ret)
 803                         return TRACE_TYPE_PARTIAL_LINE;
 804         }
 805
 806         /* Latency format */
 807         if (trace_flags & TRACE_ITER_LATENCY_FMT) {
 808                 ret = print_graph_lat_fmt(s, ent);
 809                 if (ret == TRACE_TYPE_PARTIAL_LINE)
 810                         return TRACE_TYPE_PARTIAL_LINE;
 811         }
 812
 813         return 0;
 814 }
 815
 816 static enum print_line_t
 817 print_graph_entry(struct ftrace_graph_ent_entry *field, struct trace_seq *s,
 818                         struct trace_iterator *iter)
 819 {
 820         struct fgraph_data *data = iter->private;
 821         struct ftrace_graph_ent *call = &field->graph_ent;
 822         struct ftrace_graph_ret_entry *leaf_ret;
 823         static enum print_line_t ret;
 824         int cpu = iter->cpu;
 825
 826         if (print_graph_prologue(iter, s, TRACE_GRAPH_ENT, call->func))
 827                 return TRACE_TYPE_PARTIAL_LINE;
 828
 829         leaf_ret = get_return_for_leaf(iter, field);
 830         if (leaf_ret)
 831                 ret = print_graph_entry_leaf(iter, field, leaf_ret, s);
 832         else
 833                 ret = print_graph_entry_nested(iter, field, s, cpu);
 834
 835         if (data) {
 836                 /*
 837                  * If we failed to write our output, then we need to make
 838                  * note of it. Because we already consumed our entry.
 839                  */
 840                 if (s->full) {
 841                         data->failed = 1;
 842                         data->cpu = cpu;
 843                 } else
 844                         data->failed = 0;
 845         }
 846
 847         return ret;
 848 }
 849
 850 static enum print_line_t
 851 print_graph_return(struct ftrace_graph_ret *trace, struct trace_seq *s,
 852                    struct trace_entry *ent, struct trace_iterator *iter)
 853 {
 854         unsigned long long duration = trace->rettime - trace->calltime;
 855         struct fgraph_data *data = iter->private;
 856         pid_t pid = ent->pid;
 857         int cpu = iter->cpu;
 858         int ret;
 859         int i;
 860
 861         if (data) {
 862                 int cpu = iter->cpu;
 863                 int *depth = &(per_cpu_ptr(data->cpu_data, cpu)->depth);
 864
 865                 /*
 866                  * Comments display at + 1 to depth. This is the
 867                  * return from a function, we now want the comments
 868                  * to display at the same level of the bracket.
 869                  */
 870                 *depth = trace->depth - 1;
 871         }
 872
 873         if (print_graph_prologue(iter, s, 0, 0))
 874                 return TRACE_TYPE_PARTIAL_LINE;
 875
 876         /* Overhead */
 877         ret = print_graph_overhead(duration, s);
 878         if (!ret)
 879                 return TRACE_TYPE_PARTIAL_LINE;
 880
 881         /* Duration */
 882         if (tracer_flags.val & TRACE_GRAPH_PRINT_DURATION) {
 883                 ret = print_graph_duration(duration, s);
 884                 if (ret == TRACE_TYPE_PARTIAL_LINE)
 885                         return TRACE_TYPE_PARTIAL_LINE;
 886         }
 887
 888         /* Closing brace */
 889         for (i = 0; i < trace->depth * TRACE_GRAPH_INDENT; i++) {
 890                 ret = trace_seq_printf(s, " ");
 891                 if (!ret)
 892                         return TRACE_TYPE_PARTIAL_LINE;
 893         }
 894
 895         ret = trace_seq_printf(s, "}\n");
 896         if (!ret)
 897                 return TRACE_TYPE_PARTIAL_LINE;
 898
 899         /* Overrun */
 900         if (tracer_flags.val & TRACE_GRAPH_PRINT_OVERRUN) {
 901                 ret = trace_seq_printf(s, " (Overruns: %lu)\n",
 902                                         trace->overrun);
 903                 if (!ret)
 904                         return TRACE_TYPE_PARTIAL_LINE;
 905         }
 906
 907         ret = print_graph_irq(iter, trace->func, TRACE_GRAPH_RET, cpu, pid);
 908         if (ret == TRACE_TYPE_PARTIAL_LINE)
 909                 return TRACE_TYPE_PARTIAL_LINE;
 910
 911         return TRACE_TYPE_HANDLED;
 912 }
 913
 914 static enum print_line_t
 915 print_graph_comment(struct trace_seq *s,  struct trace_entry *ent,
 916                     struct trace_iterator *iter)
 917 {
 918         unsigned long sym_flags = (trace_flags & TRACE_ITER_SYM_MASK);
 919         struct fgraph_data *data = iter->private;
 920         struct trace_event *event;
 921         int depth = 0;
 922         int ret;
 923         int i;
 924
 925         if (data)
 926                 depth = per_cpu_ptr(data->cpu_data, iter->cpu)->depth;
 927
 928         if (print_graph_prologue(iter, s, 0, 0))
 929                 return TRACE_TYPE_PARTIAL_LINE;
 930
 931         /* No overhead */
 932         ret = print_graph_overhead(-1, s);
 933         if (!ret)
 934                 return TRACE_TYPE_PARTIAL_LINE;
 935
 936         /* No time */
 937         if (tracer_flags.val & TRACE_GRAPH_PRINT_DURATION) {
 938                 ret = trace_seq_printf(s, "            |  ");
 939                 if (!ret)
 940                         return TRACE_TYPE_PARTIAL_LINE;
 941         }
 942
 943         /* Indentation */
 944         if (depth > 0)
 945                 for (i = 0; i < (depth + 1) * TRACE_GRAPH_INDENT; i++) {
 946                         ret = trace_seq_printf(s, " ");
 947                         if (!ret)
 948                                 return TRACE_TYPE_PARTIAL_LINE;
 949                 }
 950
 951         /* The comment */
 952         ret = trace_seq_printf(s, "/* ");
 953         if (!ret)
 954                 return TRACE_TYPE_PARTIAL_LINE;
 955
 956         switch (iter->ent->type) {
 957         case TRACE_BPRINT:
 958                 ret = trace_print_bprintk_msg_only(iter);
 959                 if (ret != TRACE_TYPE_HANDLED)
 960                         return ret;
 961                 break;
 962         case TRACE_PRINT:
 963                 ret = trace_print_printk_msg_only(iter);
 964                 if (ret != TRACE_TYPE_HANDLED)
 965                         return ret;
 966                 break;
 967         default:
 968                 event = ftrace_find_event(ent->type);
 969                 if (!event)
 970                         return TRACE_TYPE_UNHANDLED;
 971
 972                 ret = event->trace(iter, sym_flags);
 973                 if (ret != TRACE_TYPE_HANDLED)
 974                         return ret;
 975         }
 976
 977         /* Strip ending newline */
 978         if (s->buffer[s->len - 1] == '\n') {
 979                 s->buffer[s->len - 1] = '\0';
 980                 s->len--;
 981         }
 982
 983         ret = trace_seq_printf(s, " */\n");
 984         if (!ret)
 985                 return TRACE_TYPE_PARTIAL_LINE;
 986
 987         return TRACE_TYPE_HANDLED;
 988 }
 989
 990
 991 enum print_line_t
 992 print_graph_function(struct trace_iterator *iter)
 993 {
 994         struct ftrace_graph_ent_entry *field;
 995         struct fgraph_data *data = iter->private;
 996         struct trace_entry *entry = iter->ent;
 997         struct trace_seq *s = &iter->seq;
 998         int cpu = iter->cpu;
 999         int ret;
1000
1001         if (data && per_cpu_ptr(data->cpu_data, cpu)->ignore) {
1002                 per_cpu_ptr(data->cpu_data, cpu)->ignore = 0;
1003                 return TRACE_TYPE_HANDLED;
1004         }
1005
1006         /*
1007          * If the last output failed, there's a possibility we need
1008          * to print out the missing entry which would never go out.
1009          */
1010         if (data && data->failed) {
1011                 field = &data->ent;
1012                 iter->cpu = data->cpu;
1013                 ret = print_graph_entry(field, s, iter);
1014                 if (ret == TRACE_TYPE_HANDLED && iter->cpu != cpu) {
1015                         per_cpu_ptr(data->cpu_data, iter->cpu)->ignore = 1;
1016                         ret = TRACE_TYPE_NO_CONSUME;
1017                 }
1018                 iter->cpu = cpu;
1019                 return ret;
1020         }
1021
1022         switch (entry->type) {
1023         case TRACE_GRAPH_ENT: {
1024                 /*
1025                  * print_graph_entry() may consume the current event,
1026                  * thus @field may become invalid, so we need to save it.
1027                  * sizeof(struct ftrace_graph_ent_entry) is very small,
1028                  * it can be safely saved at the stack.
1029                  */
1030                 struct ftrace_graph_ent_entry saved;
1031                 trace_assign_type(field, entry);
1032                 saved = *field;
1033                 return print_graph_entry(&saved, s, iter);
1034         }
1035         case TRACE_GRAPH_RET: {
1036                 struct ftrace_graph_ret_entry *field;
1037                 trace_assign_type(field, entry);
1038                 return print_graph_return(&field->ret, s, entry, iter);
1039         }
1040         default:
1041                 return print_graph_comment(s, entry, iter);
1042         }
1043
1044         return TRACE_TYPE_HANDLED;
1045 }
1046
1047 static void print_lat_header(struct seq_file *s)
1048 {
1049         static const char spaces[] = "                " /* 16 spaces */
1050                 "    "                                  /* 4 spaces */
1051                 "                 ";                    /* 17 spaces */
1052         int size = 0;
1053
1054         if (tracer_flags.val & TRACE_GRAPH_PRINT_ABS_TIME)
1055                 size += 16;
1056         if (tracer_flags.val & TRACE_GRAPH_PRINT_CPU)
1057                 size += 4;
1058         if (tracer_flags.val & TRACE_GRAPH_PRINT_PROC)
1059                 size += 17;
1060
1061         seq_printf(s, "#%.*s  _-----=> irqs-off        \n", size, spaces);
1062         seq_printf(s, "#%.*s / _----=> need-resched    \n", size, spaces);
1063         seq_printf(s, "#%.*s| / _---=> hardirq/softirq \n", size, spaces);
1064         seq_printf(s, "#%.*s|| / _--=> preempt-depth   \n", size, spaces);
1065         seq_printf(s, "#%.*s||| / _-=> lock-depth      \n", size, spaces);
1066         seq_printf(s, "#%.*s|||| /                     \n", size, spaces);
1067 }
1068
1069 static void print_graph_headers(struct seq_file *s)
1070 {
1071         int lat = trace_flags & TRACE_ITER_LATENCY_FMT;
1072
1073         if (lat)
1074                 print_lat_header(s);
1075
1076         /* 1st line */
1077         seq_printf(s, "#");
1078         if (tracer_flags.val & TRACE_GRAPH_PRINT_ABS_TIME)
1079                 seq_printf(s, "     TIME       ");
1080         if (tracer_flags.val & TRACE_GRAPH_PRINT_CPU)
1081                 seq_printf(s, " CPU");
1082         if (tracer_flags.val & TRACE_GRAPH_PRINT_PROC)
1083                 seq_printf(s, "  TASK/PID       ");
1084         if (lat)
1085                 seq_printf(s, "|||||");
1086         if (tracer_flags.val & TRACE_GRAPH_PRINT_DURATION)
1087                 seq_printf(s, "  DURATION   ");
1088         seq_printf(s, "               FUNCTION CALLS\n");
1089
1090         /* 2nd line */
1091         seq_printf(s, "#");
1092         if (tracer_flags.val & TRACE_GRAPH_PRINT_ABS_TIME)
1093                 seq_printf(s, "      |         ");
1094         if (tracer_flags.val & TRACE_GRAPH_PRINT_CPU)
1095                 seq_printf(s, " |  ");
1096         if (tracer_flags.val & TRACE_GRAPH_PRINT_PROC)
1097                 seq_printf(s, "   |    |        ");
1098         if (lat)
1099                 seq_printf(s, "|||||");
1100         if (tracer_flags.val & TRACE_GRAPH_PRINT_DURATION)
1101                 seq_printf(s, "   |   |      ");
1102         seq_printf(s, "               |   |   |   |\n");
1103 }
1104
1105 static void graph_trace_open(struct trace_iterator *iter)
1106 {
1107         /* pid and depth on the last trace processed */
1108         struct fgraph_data *data;
1109         int cpu;
1110
1111         iter->private = NULL;
1112
1113         data = kzalloc(sizeof(*data), GFP_KERNEL);
1114         if (!data)
1115                 goto out_err;
1116
1117         data->cpu_data = alloc_percpu(struct fgraph_cpu_data);
1118         if (!data->cpu_data)
1119                 goto out_err_free;
1120
1121         for_each_possible_cpu(cpu) {
1122                 pid_t *pid = &(per_cpu_ptr(data->cpu_data, cpu)->last_pid);
1123                 int *depth = &(per_cpu_ptr(data->cpu_data, cpu)->depth);
1124                 int *ignore = &(per_cpu_ptr(data->cpu_data, cpu)->ignore);
1125                 *pid = -1;
1126                 *depth = 0;
1127                 *ignore = 0;
1128         }
1129
1130         iter->private = data;
1131
1132         return;
1133
1134  out_err_free:
1135         kfree(data);
1136  out_err:
1137         pr_warning("function graph tracer: not enough memory\n");
1138 }
1139
1140 static void graph_trace_close(struct trace_iterator *iter)
1141 {
1142         struct fgraph_data *data = iter->private;
1143
1144         if (data) {
1145                 free_percpu(data->cpu_data);
1146                 kfree(data);
1147         }
1148 }
1149
1150 static struct tracer graph_trace __read_mostly = {
1151         .name           = "function_graph",
1152         .open           = graph_trace_open,
1153         .pipe_open      = graph_trace_open,
1154         .close          = graph_trace_close,
1155         .pipe_close     = graph_trace_close,
1156         .wait_pipe      = poll_wait_pipe,
1157         .init           = graph_trace_init,
1158         .reset          = graph_trace_reset,
1159         .print_line     = print_graph_function,
1160         .print_header   = print_graph_headers,
1161         .flags          = &tracer_flags,
1162 #ifdef CONFIG_FTRACE_SELFTEST
1163         .selftest       = trace_selftest_startup_function_graph,
1164 #endif
1165 };
1166
1167 static __init int init_graph_trace(void)
1168 {
1169         max_bytes_for_cpu = snprintf(NULL, 0, "%d", nr_cpu_ids - 1);
1170
1171         return register_tracer(&graph_trace);
1172 }
1173
1174 device_initcall(init_graph_trace);