SAFE public projects git trees. - safe/jmp/linux-2.6/blob - kernel/trace/ftrace.c

   1 /*
   2  * Infrastructure for profiling code inserted by 'gcc -pg'.
   3  *
   4  * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
   5  * Copyright (C) 2004-2008 Ingo Molnar <mingo@redhat.com>
   6  *
   7  * Originally ported from the -rt patch by:
   8  *   Copyright (C) 2007 Arnaldo Carvalho de Melo <acme@redhat.com>
   9  *
  10  * Based on code in the latency_tracer, that is:
  11  *
  12  *  Copyright (C) 2004-2006 Ingo Molnar
  13  *  Copyright (C) 2004 William Lee Irwin III
  14  */
  15
  16 #include <linux/stop_machine.h>
  17 #include <linux/clocksource.h>
  18 #include <linux/kallsyms.h>
  19 #include <linux/seq_file.h>
  20 #include <linux/suspend.h>
  21 #include <linux/debugfs.h>
  22 #include <linux/hardirq.h>
  23 #include <linux/kthread.h>
  24 #include <linux/uaccess.h>
  25 #include <linux/kprobes.h>
  26 #include <linux/ftrace.h>
  27 #include <linux/sysctl.h>
  28 #include <linux/ctype.h>
  29 #include <linux/list.h>
  30 #include <linux/hash.h>
  31
  32 #include <trace/events/sched.h>
  33
  34 #include <asm/ftrace.h>
  35 #include <asm/setup.h>
  36
  37 #include "trace_output.h"
  38 #include "trace_stat.h"
  39
  40 #define FTRACE_WARN_ON(cond)                    \
  41         do {                                    \
  42                 if (WARN_ON(cond))              \
  43                         ftrace_kill();          \
  44         } while (0)
  45
  46 #define FTRACE_WARN_ON_ONCE(cond)               \
  47         do {                                    \
  48                 if (WARN_ON_ONCE(cond))         \
  49                         ftrace_kill();          \
  50         } while (0)
  51
  52 /* hash bits for specific function selection */
  53 #define FTRACE_HASH_BITS 7
  54 #define FTRACE_FUNC_HASHSIZE (1 << FTRACE_HASH_BITS)
  55
  56 /* ftrace_enabled is a method to turn ftrace on or off */
  57 int ftrace_enabled __read_mostly;
  58 static int last_ftrace_enabled;
  59
  60 /* Quick disabling of function tracer. */
  61 int function_trace_stop;
  62
  63 /* List for set_ftrace_pid's pids. */
  64 LIST_HEAD(ftrace_pids);
  65 struct ftrace_pid {
  66         struct list_head list;
  67         struct pid *pid;
  68 };
  69
  70 /*
  71  * ftrace_disabled is set when an anomaly is discovered.
  72  * ftrace_disabled is much stronger than ftrace_enabled.
  73  */
  74 static int ftrace_disabled __read_mostly;
  75
  76 static DEFINE_MUTEX(ftrace_lock);
  77
  78 static struct ftrace_ops ftrace_list_end __read_mostly =
  79 {
  80         .func           = ftrace_stub,
  81 };
  82
  83 static struct ftrace_ops *ftrace_list __read_mostly = &ftrace_list_end;
  84 ftrace_func_t ftrace_trace_function __read_mostly = ftrace_stub;
  85 ftrace_func_t __ftrace_trace_function __read_mostly = ftrace_stub;
  86 ftrace_func_t ftrace_pid_function __read_mostly = ftrace_stub;
  87
  88 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
  89 static int ftrace_set_func(unsigned long *array, int *idx, char *buffer);
  90 #endif
  91
  92 static void ftrace_list_func(unsigned long ip, unsigned long parent_ip)
  93 {
  94         struct ftrace_ops *op = ftrace_list;
  95
  96         /* in case someone actually ports this to alpha! */
  97         read_barrier_depends();
  98
  99         while (op != &ftrace_list_end) {
 100                 /* silly alpha */
 101                 read_barrier_depends();
 102                 op->func(ip, parent_ip);
 103                 op = op->next;
 104         };
 105 }
 106
 107 static void ftrace_pid_func(unsigned long ip, unsigned long parent_ip)
 108 {
 109         if (!test_tsk_trace_trace(current))
 110                 return;
 111
 112         ftrace_pid_function(ip, parent_ip);
 113 }
 114
 115 static void set_ftrace_pid_function(ftrace_func_t func)
 116 {
 117         /* do not set ftrace_pid_function to itself! */
 118         if (func != ftrace_pid_func)
 119                 ftrace_pid_function = func;
 120 }
 121
 122 /**
 123  * clear_ftrace_function - reset the ftrace function
 124  *
 125  * This NULLs the ftrace function and in essence stops
 126  * tracing.  There may be lag
 127  */
 128 void clear_ftrace_function(void)
 129 {
 130         ftrace_trace_function = ftrace_stub;
 131         __ftrace_trace_function = ftrace_stub;
 132         ftrace_pid_function = ftrace_stub;
 133 }
 134
 135 #ifndef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST
 136 /*
 137  * For those archs that do not test ftrace_trace_stop in their
 138  * mcount call site, we need to do it from C.
 139  */
 140 static void ftrace_test_stop_func(unsigned long ip, unsigned long parent_ip)
 141 {
 142         if (function_trace_stop)
 143                 return;
 144
 145         __ftrace_trace_function(ip, parent_ip);
 146 }
 147 #endif
 148
 149 static int __register_ftrace_function(struct ftrace_ops *ops)
 150 {
 151         ops->next = ftrace_list;
 152         /*
 153          * We are entering ops into the ftrace_list but another
 154          * CPU might be walking that list. We need to make sure
 155          * the ops->next pointer is valid before another CPU sees
 156          * the ops pointer included into the ftrace_list.
 157          */
 158         smp_wmb();
 159         ftrace_list = ops;
 160
 161         if (ftrace_enabled) {
 162                 ftrace_func_t func;
 163
 164                 if (ops->next == &ftrace_list_end)
 165                         func = ops->func;
 166                 else
 167                         func = ftrace_list_func;
 168
 169                 if (!list_empty(&ftrace_pids)) {
 170                         set_ftrace_pid_function(func);
 171                         func = ftrace_pid_func;
 172                 }
 173
 174                 /*
 175                  * For one func, simply call it directly.
 176                  * For more than one func, call the chain.
 177                  */
 178 #ifdef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST
 179                 ftrace_trace_function = func;
 180 #else
 181                 __ftrace_trace_function = func;
 182                 ftrace_trace_function = ftrace_test_stop_func;
 183 #endif
 184         }
 185
 186         return 0;
 187 }
 188
 189 static int __unregister_ftrace_function(struct ftrace_ops *ops)
 190 {
 191         struct ftrace_ops **p;
 192
 193         /*
 194          * If we are removing the last function, then simply point
 195          * to the ftrace_stub.
 196          */
 197         if (ftrace_list == ops && ops->next == &ftrace_list_end) {
 198                 ftrace_trace_function = ftrace_stub;
 199                 ftrace_list = &ftrace_list_end;
 200                 return 0;
 201         }
 202
 203         for (p = &ftrace_list; *p != &ftrace_list_end; p = &(*p)->next)
 204                 if (*p == ops)
 205                         break;
 206
 207         if (*p != ops)
 208                 return -1;
 209
 210         *p = (*p)->next;
 211
 212         if (ftrace_enabled) {
 213                 /* If we only have one func left, then call that directly */
 214                 if (ftrace_list->next == &ftrace_list_end) {
 215                         ftrace_func_t func = ftrace_list->func;
 216
 217                         if (!list_empty(&ftrace_pids)) {
 218                                 set_ftrace_pid_function(func);
 219                                 func = ftrace_pid_func;
 220                         }
 221 #ifdef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST
 222                         ftrace_trace_function = func;
 223 #else
 224                         __ftrace_trace_function = func;
 225 #endif
 226                 }
 227         }
 228
 229         return 0;
 230 }
 231
 232 static void ftrace_update_pid_func(void)
 233 {
 234         ftrace_func_t func;
 235
 236         if (ftrace_trace_function == ftrace_stub)
 237                 return;
 238
 239 #ifdef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST
 240         func = ftrace_trace_function;
 241 #else
 242         func = __ftrace_trace_function;
 243 #endif
 244
 245         if (!list_empty(&ftrace_pids)) {
 246                 set_ftrace_pid_function(func);
 247                 func = ftrace_pid_func;
 248         } else {
 249                 if (func == ftrace_pid_func)
 250                         func = ftrace_pid_function;
 251         }
 252
 253 #ifdef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST
 254         ftrace_trace_function = func;
 255 #else
 256         __ftrace_trace_function = func;
 257 #endif
 258 }
 259
 260 #ifdef CONFIG_FUNCTION_PROFILER
 261 struct ftrace_profile {
 262         struct hlist_node               node;
 263         unsigned long                   ip;
 264         unsigned long                   counter;
 265 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
 266         unsigned long long              time;
 267 #endif
 268 };
 269
 270 struct ftrace_profile_page {
 271         struct ftrace_profile_page      *next;
 272         unsigned long                   index;
 273         struct ftrace_profile           records[];
 274 };
 275
 276 struct ftrace_profile_stat {
 277         atomic_t                        disabled;
 278         struct hlist_head               *hash;
 279         struct ftrace_profile_page      *pages;
 280         struct ftrace_profile_page      *start;
 281         struct tracer_stat              stat;
 282 };
 283
 284 #define PROFILE_RECORDS_SIZE                                            \
 285         (PAGE_SIZE - offsetof(struct ftrace_profile_page, records))
 286
 287 #define PROFILES_PER_PAGE                                       \
 288         (PROFILE_RECORDS_SIZE / sizeof(struct ftrace_profile))
 289
 290 static int ftrace_profile_bits __read_mostly;
 291 static int ftrace_profile_enabled __read_mostly;
 292
 293 /* ftrace_profile_lock - synchronize the enable and disable of the profiler */
 294 static DEFINE_MUTEX(ftrace_profile_lock);
 295
 296 static DEFINE_PER_CPU(struct ftrace_profile_stat, ftrace_profile_stats);
 297
 298 #define FTRACE_PROFILE_HASH_SIZE 1024 /* must be power of 2 */
 299
 300 static void *
 301 function_stat_next(void *v, int idx)
 302 {
 303         struct ftrace_profile *rec = v;
 304         struct ftrace_profile_page *pg;
 305
 306         pg = (struct ftrace_profile_page *)((unsigned long)rec & PAGE_MASK);
 307
 308  again:
 309         if (idx != 0)
 310                 rec++;
 311
 312         if ((void *)rec >= (void *)&pg->records[pg->index]) {
 313                 pg = pg->next;
 314                 if (!pg)
 315                         return NULL;
 316                 rec = &pg->records[0];
 317                 if (!rec->counter)
 318                         goto again;
 319         }
 320
 321         return rec;
 322 }
 323
 324 static void *function_stat_start(struct tracer_stat *trace)
 325 {
 326         struct ftrace_profile_stat *stat =
 327                 container_of(trace, struct ftrace_profile_stat, stat);
 328
 329         if (!stat || !stat->start)
 330                 return NULL;
 331
 332         return function_stat_next(&stat->start->records[0], 0);
 333 }
 334
 335 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
 336 /* function graph compares on total time */
 337 static int function_stat_cmp(void *p1, void *p2)
 338 {
 339         struct ftrace_profile *a = p1;
 340         struct ftrace_profile *b = p2;
 341
 342         if (a->time < b->time)
 343                 return -1;
 344         if (a->time > b->time)
 345                 return 1;
 346         else
 347                 return 0;
 348 }
 349 #else
 350 /* not function graph compares against hits */
 351 static int function_stat_cmp(void *p1, void *p2)
 352 {
 353         struct ftrace_profile *a = p1;
 354         struct ftrace_profile *b = p2;
 355
 356         if (a->counter < b->counter)
 357                 return -1;
 358         if (a->counter > b->counter)
 359                 return 1;
 360         else
 361                 return 0;
 362 }
 363 #endif
 364
 365 static int function_stat_headers(struct seq_file *m)
 366 {
 367 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
 368         seq_printf(m, "  Function                               "
 369                    "Hit    Time            Avg\n"
 370                       "  --------                               "
 371                    "---    ----            ---\n");
 372 #else
 373         seq_printf(m, "  Function                               Hit\n"
 374                       "  --------                               ---\n");
 375 #endif
 376         return 0;
 377 }
 378
 379 static int function_stat_show(struct seq_file *m, void *v)
 380 {
 381         struct ftrace_profile *rec = v;
 382         char str[KSYM_SYMBOL_LEN];
 383 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
 384         static DEFINE_MUTEX(mutex);
 385         static struct trace_seq s;
 386         unsigned long long avg;
 387 #endif
 388
 389         kallsyms_lookup(rec->ip, NULL, NULL, NULL, str);
 390         seq_printf(m, "  %-30.30s  %10lu", str, rec->counter);
 391
 392 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
 393         seq_printf(m, "    ");
 394         avg = rec->time;
 395         do_div(avg, rec->counter);
 396
 397         mutex_lock(&mutex);
 398         trace_seq_init(&s);
 399         trace_print_graph_duration(rec->time, &s);
 400         trace_seq_puts(&s, "    ");
 401         trace_print_graph_duration(avg, &s);
 402         trace_print_seq(m, &s);
 403         mutex_unlock(&mutex);
 404 #endif
 405         seq_putc(m, '\n');
 406
 407         return 0;
 408 }
 409
 410 static void ftrace_profile_reset(struct ftrace_profile_stat *stat)
 411 {
 412         struct ftrace_profile_page *pg;
 413
 414         pg = stat->pages = stat->start;
 415
 416         while (pg) {
 417                 memset(pg->records, 0, PROFILE_RECORDS_SIZE);
 418                 pg->index = 0;
 419                 pg = pg->next;
 420         }
 421
 422         memset(stat->hash, 0,
 423                FTRACE_PROFILE_HASH_SIZE * sizeof(struct hlist_head));
 424 }
 425
 426 int ftrace_profile_pages_init(struct ftrace_profile_stat *stat)
 427 {
 428         struct ftrace_profile_page *pg;
 429         int functions;
 430         int pages;
 431         int i;
 432
 433         /* If we already allocated, do nothing */
 434         if (stat->pages)
 435                 return 0;
 436
 437         stat->pages = (void *)get_zeroed_page(GFP_KERNEL);
 438         if (!stat->pages)
 439                 return -ENOMEM;
 440
 441 #ifdef CONFIG_DYNAMIC_FTRACE
 442         functions = ftrace_update_tot_cnt;
 443 #else
 444         /*
 445          * We do not know the number of functions that exist because
 446          * dynamic tracing is what counts them. With past experience
 447          * we have around 20K functions. That should be more than enough.
 448          * It is highly unlikely we will execute every function in
 449          * the kernel.
 450          */
 451         functions = 20000;
 452 #endif
 453
 454         pg = stat->start = stat->pages;
 455
 456         pages = DIV_ROUND_UP(functions, PROFILES_PER_PAGE);
 457
 458         for (i = 0; i < pages; i++) {
 459                 pg->next = (void *)get_zeroed_page(GFP_KERNEL);
 460                 if (!pg->next)
 461                         goto out_free;
 462                 pg = pg->next;
 463         }
 464
 465         return 0;
 466
 467  out_free:
 468         pg = stat->start;
 469         while (pg) {
 470                 unsigned long tmp = (unsigned long)pg;
 471
 472                 pg = pg->next;
 473                 free_page(tmp);
 474         }
 475
 476         free_page((unsigned long)stat->pages);
 477         stat->pages = NULL;
 478         stat->start = NULL;
 479
 480         return -ENOMEM;
 481 }
 482
 483 static int ftrace_profile_init_cpu(int cpu)
 484 {
 485         struct ftrace_profile_stat *stat;
 486         int size;
 487
 488         stat = &per_cpu(ftrace_profile_stats, cpu);
 489
 490         if (stat->hash) {
 491                 /* If the profile is already created, simply reset it */
 492                 ftrace_profile_reset(stat);
 493                 return 0;
 494         }
 495
 496         /*
 497          * We are profiling all functions, but usually only a few thousand
 498          * functions are hit. We'll make a hash of 1024 items.
 499          */
 500         size = FTRACE_PROFILE_HASH_SIZE;
 501
 502         stat->hash = kzalloc(sizeof(struct hlist_head) * size, GFP_KERNEL);
 503
 504         if (!stat->hash)
 505                 return -ENOMEM;
 506
 507         if (!ftrace_profile_bits) {
 508                 size--;
 509
 510                 for (; size; size >>= 1)
 511                         ftrace_profile_bits++;
 512         }
 513
 514         /* Preallocate the function profiling pages */
 515         if (ftrace_profile_pages_init(stat) < 0) {
 516                 kfree(stat->hash);
 517                 stat->hash = NULL;
 518                 return -ENOMEM;
 519         }
 520
 521         return 0;
 522 }
 523
 524 static int ftrace_profile_init(void)
 525 {
 526         int cpu;
 527         int ret = 0;
 528
 529         for_each_online_cpu(cpu) {
 530                 ret = ftrace_profile_init_cpu(cpu);
 531                 if (ret)
 532                         break;
 533         }
 534
 535         return ret;
 536 }
 537
 538 /* interrupts must be disabled */
 539 static struct ftrace_profile *
 540 ftrace_find_profiled_func(struct ftrace_profile_stat *stat, unsigned long ip)
 541 {
 542         struct ftrace_profile *rec;
 543         struct hlist_head *hhd;
 544         struct hlist_node *n;
 545         unsigned long key;
 546
 547         key = hash_long(ip, ftrace_profile_bits);
 548         hhd = &stat->hash[key];
 549
 550         if (hlist_empty(hhd))
 551                 return NULL;
 552
 553         hlist_for_each_entry_rcu(rec, n, hhd, node) {
 554                 if (rec->ip == ip)
 555                         return rec;
 556         }
 557
 558         return NULL;
 559 }
 560
 561 static void ftrace_add_profile(struct ftrace_profile_stat *stat,
 562                                struct ftrace_profile *rec)
 563 {
 564         unsigned long key;
 565
 566         key = hash_long(rec->ip, ftrace_profile_bits);
 567         hlist_add_head_rcu(&rec->node, &stat->hash[key]);
 568 }
 569
 570 /*
 571  * The memory is already allocated, this simply finds a new record to use.
 572  */
 573 static struct ftrace_profile *
 574 ftrace_profile_alloc(struct ftrace_profile_stat *stat, unsigned long ip)
 575 {
 576         struct ftrace_profile *rec = NULL;
 577
 578         /* prevent recursion (from NMIs) */
 579         if (atomic_inc_return(&stat->disabled) != 1)
 580                 goto out;
 581
 582         /*
 583          * Try to find the function again since an NMI
 584          * could have added it
 585          */
 586         rec = ftrace_find_profiled_func(stat, ip);
 587         if (rec)
 588                 goto out;
 589
 590         if (stat->pages->index == PROFILES_PER_PAGE) {
 591                 if (!stat->pages->next)
 592                         goto out;
 593                 stat->pages = stat->pages->next;
 594         }
 595
 596         rec = &stat->pages->records[stat->pages->index++];
 597         rec->ip = ip;
 598         ftrace_add_profile(stat, rec);
 599
 600  out:
 601         atomic_dec(&stat->disabled);
 602
 603         return rec;
 604 }
 605
 606 static void
 607 function_profile_call(unsigned long ip, unsigned long parent_ip)
 608 {
 609         struct ftrace_profile_stat *stat;
 610         struct ftrace_profile *rec;
 611         unsigned long flags;
 612
 613         if (!ftrace_profile_enabled)
 614                 return;
 615
 616         local_irq_save(flags);
 617
 618         stat = &__get_cpu_var(ftrace_profile_stats);
 619         if (!stat->hash || !ftrace_profile_enabled)
 620                 goto out;
 621
 622         rec = ftrace_find_profiled_func(stat, ip);
 623         if (!rec) {
 624                 rec = ftrace_profile_alloc(stat, ip);
 625                 if (!rec)
 626                         goto out;
 627         }
 628
 629         rec->counter++;
 630  out:
 631         local_irq_restore(flags);
 632 }
 633
 634 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
 635 static int profile_graph_entry(struct ftrace_graph_ent *trace)
 636 {
 637         function_profile_call(trace->func, 0);
 638         return 1;
 639 }
 640
 641 static void profile_graph_return(struct ftrace_graph_ret *trace)
 642 {
 643         struct ftrace_profile_stat *stat;
 644         unsigned long long calltime;
 645         struct ftrace_profile *rec;
 646         unsigned long flags;
 647
 648         local_irq_save(flags);
 649         stat = &__get_cpu_var(ftrace_profile_stats);
 650         if (!stat->hash || !ftrace_profile_enabled)
 651                 goto out;
 652
 653         calltime = trace->rettime - trace->calltime;
 654
 655         if (!(trace_flags & TRACE_ITER_GRAPH_TIME)) {
 656                 int index;
 657
 658                 index = trace->depth;
 659
 660                 /* Append this call time to the parent time to subtract */
 661                 if (index)
 662                         current->ret_stack[index - 1].subtime += calltime;
 663
 664                 if (current->ret_stack[index].subtime < calltime)
 665                         calltime -= current->ret_stack[index].subtime;
 666                 else
 667                         calltime = 0;
 668         }
 669
 670         rec = ftrace_find_profiled_func(stat, trace->func);
 671         if (rec)
 672                 rec->time += calltime;
 673
 674  out:
 675         local_irq_restore(flags);
 676 }
 677
 678 static int register_ftrace_profiler(void)
 679 {
 680         return register_ftrace_graph(&profile_graph_return,
 681                                      &profile_graph_entry);
 682 }
 683
 684 static void unregister_ftrace_profiler(void)
 685 {
 686         unregister_ftrace_graph();
 687 }
 688 #else
 689 static struct ftrace_ops ftrace_profile_ops __read_mostly =
 690 {
 691         .func           = function_profile_call,
 692 };
 693
 694 static int register_ftrace_profiler(void)
 695 {
 696         return register_ftrace_function(&ftrace_profile_ops);
 697 }
 698
 699 static void unregister_ftrace_profiler(void)
 700 {
 701         unregister_ftrace_function(&ftrace_profile_ops);
 702 }
 703 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
 704
 705 static ssize_t
 706 ftrace_profile_write(struct file *filp, const char __user *ubuf,
 707                      size_t cnt, loff_t *ppos)
 708 {
 709         unsigned long val;
 710         char buf[64];           /* big enough to hold a number */
 711         int ret;
 712
 713         if (cnt >= sizeof(buf))
 714                 return -EINVAL;
 715
 716         if (copy_from_user(&buf, ubuf, cnt))
 717                 return -EFAULT;
 718
 719         buf[cnt] = 0;
 720
 721         ret = strict_strtoul(buf, 10, &val);
 722         if (ret < 0)
 723                 return ret;
 724
 725         val = !!val;
 726
 727         mutex_lock(&ftrace_profile_lock);
 728         if (ftrace_profile_enabled ^ val) {
 729                 if (val) {
 730                         ret = ftrace_profile_init();
 731                         if (ret < 0) {
 732                                 cnt = ret;
 733                                 goto out;
 734                         }
 735
 736                         ret = register_ftrace_profiler();
 737                         if (ret < 0) {
 738                                 cnt = ret;
 739                                 goto out;
 740                         }
 741                         ftrace_profile_enabled = 1;
 742                 } else {
 743                         ftrace_profile_enabled = 0;
 744                         /*
 745                          * unregister_ftrace_profiler calls stop_machine
 746                          * so this acts like an synchronize_sched.
 747                          */
 748                         unregister_ftrace_profiler();
 749                 }
 750         }
 751  out:
 752         mutex_unlock(&ftrace_profile_lock);
 753
 754         filp->f_pos += cnt;
 755
 756         return cnt;
 757 }
 758
 759 static ssize_t
 760 ftrace_profile_read(struct file *filp, char __user *ubuf,
 761                      size_t cnt, loff_t *ppos)
 762 {
 763         char buf[64];           /* big enough to hold a number */
 764         int r;
 765
 766         r = sprintf(buf, "%u\n", ftrace_profile_enabled);
 767         return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
 768 }
 769
 770 static const struct file_operations ftrace_profile_fops = {
 771         .open           = tracing_open_generic,
 772         .read           = ftrace_profile_read,
 773         .write          = ftrace_profile_write,
 774 };
 775
 776 /* used to initialize the real stat files */
 777 static struct tracer_stat function_stats __initdata = {
 778         .name           = "functions",
 779         .stat_start     = function_stat_start,
 780         .stat_next      = function_stat_next,
 781         .stat_cmp       = function_stat_cmp,
 782         .stat_headers   = function_stat_headers,
 783         .stat_show      = function_stat_show
 784 };
 785
 786 static __init void ftrace_profile_debugfs(struct dentry *d_tracer)
 787 {
 788         struct ftrace_profile_stat *stat;
 789         struct dentry *entry;
 790         char *name;
 791         int ret;
 792         int cpu;
 793
 794         for_each_possible_cpu(cpu) {
 795                 stat = &per_cpu(ftrace_profile_stats, cpu);
 796
 797                 /* allocate enough for function name + cpu number */
 798                 name = kmalloc(32, GFP_KERNEL);
 799                 if (!name) {
 800                         /*
 801                          * The files created are permanent, if something happens
 802                          * we still do not free memory.
 803                          */
 804                         WARN(1,
 805                              "Could not allocate stat file for cpu %d\n",
 806                              cpu);
 807                         return;
 808                 }
 809                 stat->stat = function_stats;
 810                 snprintf(name, 32, "function%d", cpu);
 811                 stat->stat.name = name;
 812                 ret = register_stat_tracer(&stat->stat);
 813                 if (ret) {
 814                         WARN(1,
 815                              "Could not register function stat for cpu %d\n",
 816                              cpu);
 817                         kfree(name);
 818                         return;
 819                 }
 820         }
 821
 822         entry = debugfs_create_file("function_profile_enabled", 0644,
 823                                     d_tracer, NULL, &ftrace_profile_fops);
 824         if (!entry)
 825                 pr_warning("Could not create debugfs "
 826                            "'function_profile_enabled' entry\n");
 827 }
 828
 829 #else /* CONFIG_FUNCTION_PROFILER */
 830 static __init void ftrace_profile_debugfs(struct dentry *d_tracer)
 831 {
 832 }
 833 #endif /* CONFIG_FUNCTION_PROFILER */
 834
 835 static struct pid * const ftrace_swapper_pid = &init_struct_pid;
 836
 837 #ifdef CONFIG_DYNAMIC_FTRACE
 838
 839 #ifndef CONFIG_FTRACE_MCOUNT_RECORD
 840 # error Dynamic ftrace depends on MCOUNT_RECORD
 841 #endif
 842
 843 static struct hlist_head ftrace_func_hash[FTRACE_FUNC_HASHSIZE] __read_mostly;
 844
 845 struct ftrace_func_probe {
 846         struct hlist_node       node;
 847         struct ftrace_probe_ops *ops;
 848         unsigned long           flags;
 849         unsigned long           ip;
 850         void                    *data;
 851         struct rcu_head         rcu;
 852 };
 853
 854 enum {
 855         FTRACE_ENABLE_CALLS             = (1 << 0),
 856         FTRACE_DISABLE_CALLS            = (1 << 1),
 857         FTRACE_UPDATE_TRACE_FUNC        = (1 << 2),
 858         FTRACE_ENABLE_MCOUNT            = (1 << 3),
 859         FTRACE_DISABLE_MCOUNT           = (1 << 4),
 860         FTRACE_START_FUNC_RET           = (1 << 5),
 861         FTRACE_STOP_FUNC_RET            = (1 << 6),
 862 };
 863
 864 static int ftrace_filtered;
 865
 866 static struct dyn_ftrace *ftrace_new_addrs;
 867
 868 static DEFINE_MUTEX(ftrace_regex_lock);
 869
 870 struct ftrace_page {
 871         struct ftrace_page      *next;
 872         int                     index;
 873         struct dyn_ftrace       records[];
 874 };
 875
 876 #define ENTRIES_PER_PAGE \
 877   ((PAGE_SIZE - sizeof(struct ftrace_page)) / sizeof(struct dyn_ftrace))
 878
 879 /* estimate from running different kernels */
 880 #define NR_TO_INIT              10000
 881
 882 static struct ftrace_page       *ftrace_pages_start;
 883 static struct ftrace_page       *ftrace_pages;
 884
 885 static struct dyn_ftrace *ftrace_free_records;
 886
 887 /*
 888  * This is a double for. Do not use 'break' to break out of the loop,
 889  * you must use a goto.
 890  */
 891 #define do_for_each_ftrace_rec(pg, rec)                                 \
 892         for (pg = ftrace_pages_start; pg; pg = pg->next) {              \
 893                 int _____i;                                             \
 894                 for (_____i = 0; _____i < pg->index; _____i++) {        \
 895                         rec = &pg->records[_____i];
 896
 897 #define while_for_each_ftrace_rec()             \
 898                 }                               \
 899         }
 900
 901 #ifdef CONFIG_KPROBES
 902
 903 static int frozen_record_count;
 904
 905 static inline void freeze_record(struct dyn_ftrace *rec)
 906 {
 907         if (!(rec->flags & FTRACE_FL_FROZEN)) {
 908                 rec->flags |= FTRACE_FL_FROZEN;
 909                 frozen_record_count++;
 910         }
 911 }
 912
 913 static inline void unfreeze_record(struct dyn_ftrace *rec)
 914 {
 915         if (rec->flags & FTRACE_FL_FROZEN) {
 916                 rec->flags &= ~FTRACE_FL_FROZEN;
 917                 frozen_record_count--;
 918         }
 919 }
 920
 921 static inline int record_frozen(struct dyn_ftrace *rec)
 922 {
 923         return rec->flags & FTRACE_FL_FROZEN;
 924 }
 925 #else
 926 # define freeze_record(rec)                     ({ 0; })
 927 # define unfreeze_record(rec)                   ({ 0; })
 928 # define record_frozen(rec)                     ({ 0; })
 929 #endif /* CONFIG_KPROBES */
 930
 931 static void ftrace_free_rec(struct dyn_ftrace *rec)
 932 {
 933         rec->freelist = ftrace_free_records;
 934         ftrace_free_records = rec;
 935         rec->flags |= FTRACE_FL_FREE;
 936 }
 937
 938 static struct dyn_ftrace *ftrace_alloc_dyn_node(unsigned long ip)
 939 {
 940         struct dyn_ftrace *rec;
 941
 942         /* First check for freed records */
 943         if (ftrace_free_records) {
 944                 rec = ftrace_free_records;
 945
 946                 if (unlikely(!(rec->flags & FTRACE_FL_FREE))) {
 947                         FTRACE_WARN_ON_ONCE(1);
 948                         ftrace_free_records = NULL;
 949                         return NULL;
 950                 }
 951
 952                 ftrace_free_records = rec->freelist;
 953                 memset(rec, 0, sizeof(*rec));
 954                 return rec;
 955         }
 956
 957         if (ftrace_pages->index == ENTRIES_PER_PAGE) {
 958                 if (!ftrace_pages->next) {
 959                         /* allocate another page */
 960                         ftrace_pages->next =
 961                                 (void *)get_zeroed_page(GFP_KERNEL);
 962                         if (!ftrace_pages->next)
 963                                 return NULL;
 964                 }
 965                 ftrace_pages = ftrace_pages->next;
 966         }
 967
 968         return &ftrace_pages->records[ftrace_pages->index++];
 969 }
 970
 971 static struct dyn_ftrace *
 972 ftrace_record_ip(unsigned long ip)
 973 {
 974         struct dyn_ftrace *rec;
 975
 976         if (ftrace_disabled)
 977                 return NULL;
 978
 979         rec = ftrace_alloc_dyn_node(ip);
 980         if (!rec)
 981                 return NULL;
 982
 983         rec->ip = ip;
 984         rec->newlist = ftrace_new_addrs;
 985         ftrace_new_addrs = rec;
 986
 987         return rec;
 988 }
 989
 990 static void print_ip_ins(const char *fmt, unsigned char *p)
 991 {
 992         int i;
 993
 994         printk(KERN_CONT "%s", fmt);
 995
 996         for (i = 0; i < MCOUNT_INSN_SIZE; i++)
 997                 printk(KERN_CONT "%s%02x", i ? ":" : "", p[i]);
 998 }
 999
1000 static void ftrace_bug(int failed, unsigned long ip)
1001 {
1002         switch (failed) {
1003         case -EFAULT:
1004                 FTRACE_WARN_ON_ONCE(1);
1005                 pr_info("ftrace faulted on modifying ");
1006                 print_ip_sym(ip);
1007                 break;
1008         case -EINVAL:
1009                 FTRACE_WARN_ON_ONCE(1);
1010                 pr_info("ftrace failed to modify ");
1011                 print_ip_sym(ip);
1012                 print_ip_ins(" actual: ", (unsigned char *)ip);
1013                 printk(KERN_CONT "\n");
1014                 break;
1015         case -EPERM:
1016                 FTRACE_WARN_ON_ONCE(1);
1017                 pr_info("ftrace faulted on writing ");
1018                 print_ip_sym(ip);
1019                 break;
1020         default:
1021                 FTRACE_WARN_ON_ONCE(1);
1022                 pr_info("ftrace faulted on unknown error ");
1023                 print_ip_sym(ip);
1024         }
1025 }
1026
1027
1028 static int
1029 __ftrace_replace_code(struct dyn_ftrace *rec, int enable)
1030 {
1031         unsigned long ftrace_addr;
1032         unsigned long flag = 0UL;
1033
1034         ftrace_addr = (unsigned long)FTRACE_ADDR;
1035
1036         /*
1037          * If this record is not to be traced or we want to disable it,
1038          * then disable it.
1039          *
1040          * If we want to enable it and filtering is off, then enable it.
1041          *
1042          * If we want to enable it and filtering is on, enable it only if
1043          * it's filtered
1044          */
1045         if (enable && !(rec->flags & FTRACE_FL_NOTRACE)) {
1046                 if (!ftrace_filtered || (rec->flags & FTRACE_FL_FILTER))
1047                         flag = FTRACE_FL_ENABLED;
1048         }
1049
1050         /* If the state of this record hasn't changed, then do nothing */
1051         if ((rec->flags & FTRACE_FL_ENABLED) == flag)
1052                 return 0;
1053
1054         if (flag) {
1055                 rec->flags |= FTRACE_FL_ENABLED;
1056                 return ftrace_make_call(rec, ftrace_addr);
1057         }
1058
1059         rec->flags &= ~FTRACE_FL_ENABLED;
1060         return ftrace_make_nop(NULL, rec, ftrace_addr);
1061 }
1062
1063 static void ftrace_replace_code(int enable)
1064 {
1065         struct dyn_ftrace *rec;
1066         struct ftrace_page *pg;
1067         int failed;
1068
1069         do_for_each_ftrace_rec(pg, rec) {
1070                 /*
1071                  * Skip over free records, records that have
1072                  * failed and not converted.
1073                  */
1074                 if (rec->flags & FTRACE_FL_FREE ||
1075                     rec->flags & FTRACE_FL_FAILED ||
1076                     !(rec->flags & FTRACE_FL_CONVERTED))
1077                         continue;
1078
1079                 /* ignore updates to this record's mcount site */
1080                 if (get_kprobe((void *)rec->ip)) {
1081                         freeze_record(rec);
1082                         continue;
1083                 } else {
1084                         unfreeze_record(rec);
1085                 }
1086
1087                 failed = __ftrace_replace_code(rec, enable);
1088                 if (failed) {
1089                         rec->flags |= FTRACE_FL_FAILED;
1090                         ftrace_bug(failed, rec->ip);
1091                         /* Stop processing */
1092                         return;
1093                 }
1094         } while_for_each_ftrace_rec();
1095 }
1096
1097 static int
1098 ftrace_code_disable(struct module *mod, struct dyn_ftrace *rec)
1099 {
1100         unsigned long ip;
1101         int ret;
1102
1103         ip = rec->ip;
1104
1105         ret = ftrace_make_nop(mod, rec, MCOUNT_ADDR);
1106         if (ret) {
1107                 ftrace_bug(ret, ip);
1108                 rec->flags |= FTRACE_FL_FAILED;
1109                 return 0;
1110         }
1111         return 1;
1112 }
1113
1114 /*
1115  * archs can override this function if they must do something
1116  * before the modifying code is performed.
1117  */
1118 int __weak ftrace_arch_code_modify_prepare(void)
1119 {
1120         return 0;
1121 }
1122
1123 /*
1124  * archs can override this function if they must do something
1125  * after the modifying code is performed.
1126  */
1127 int __weak ftrace_arch_code_modify_post_process(void)
1128 {
1129         return 0;
1130 }
1131
1132 static int __ftrace_modify_code(void *data)
1133 {
1134         int *command = data;
1135
1136         if (*command & FTRACE_ENABLE_CALLS)
1137                 ftrace_replace_code(1);
1138         else if (*command & FTRACE_DISABLE_CALLS)
1139                 ftrace_replace_code(0);
1140
1141         if (*command & FTRACE_UPDATE_TRACE_FUNC)
1142                 ftrace_update_ftrace_func(ftrace_trace_function);
1143
1144         if (*command & FTRACE_START_FUNC_RET)
1145                 ftrace_enable_ftrace_graph_caller();
1146         else if (*command & FTRACE_STOP_FUNC_RET)
1147                 ftrace_disable_ftrace_graph_caller();
1148
1149         return 0;
1150 }
1151
1152 static void ftrace_run_update_code(int command)
1153 {
1154         int ret;
1155
1156         ret = ftrace_arch_code_modify_prepare();
1157         FTRACE_WARN_ON(ret);
1158         if (ret)
1159                 return;
1160
1161         stop_machine(__ftrace_modify_code, &command, NULL);
1162
1163         ret = ftrace_arch_code_modify_post_process();
1164         FTRACE_WARN_ON(ret);
1165 }
1166
1167 static ftrace_func_t saved_ftrace_func;
1168 static int ftrace_start_up;
1169
1170 static void ftrace_startup_enable(int command)
1171 {
1172         if (saved_ftrace_func != ftrace_trace_function) {
1173                 saved_ftrace_func = ftrace_trace_function;
1174                 command |= FTRACE_UPDATE_TRACE_FUNC;
1175         }
1176
1177         if (!command || !ftrace_enabled)
1178                 return;
1179
1180         ftrace_run_update_code(command);
1181 }
1182
1183 static void ftrace_startup(int command)
1184 {
1185         if (unlikely(ftrace_disabled))
1186                 return;
1187
1188         ftrace_start_up++;
1189         command |= FTRACE_ENABLE_CALLS;
1190
1191         ftrace_startup_enable(command);
1192 }
1193
1194 static void ftrace_shutdown(int command)
1195 {
1196         if (unlikely(ftrace_disabled))
1197                 return;
1198
1199         ftrace_start_up--;
1200         /*
1201          * Just warn in case of unbalance, no need to kill ftrace, it's not
1202          * critical but the ftrace_call callers may be never nopped again after
1203          * further ftrace uses.
1204          */
1205         WARN_ON_ONCE(ftrace_start_up < 0);
1206
1207         if (!ftrace_start_up)
1208                 command |= FTRACE_DISABLE_CALLS;
1209
1210         if (saved_ftrace_func != ftrace_trace_function) {
1211                 saved_ftrace_func = ftrace_trace_function;
1212                 command |= FTRACE_UPDATE_TRACE_FUNC;
1213         }
1214
1215         if (!command || !ftrace_enabled)
1216                 return;
1217
1218         ftrace_run_update_code(command);
1219 }
1220
1221 static void ftrace_startup_sysctl(void)
1222 {
1223         int command = FTRACE_ENABLE_MCOUNT;
1224
1225         if (unlikely(ftrace_disabled))
1226                 return;
1227
1228         /* Force update next time */
1229         saved_ftrace_func = NULL;
1230         /* ftrace_start_up is true if we want ftrace running */
1231         if (ftrace_start_up)
1232                 command |= FTRACE_ENABLE_CALLS;
1233
1234         ftrace_run_update_code(command);
1235 }
1236
1237 static void ftrace_shutdown_sysctl(void)
1238 {
1239         int command = FTRACE_DISABLE_MCOUNT;
1240
1241         if (unlikely(ftrace_disabled))
1242                 return;
1243
1244         /* ftrace_start_up is true if ftrace is running */
1245         if (ftrace_start_up)
1246                 command |= FTRACE_DISABLE_CALLS;
1247
1248         ftrace_run_update_code(command);
1249 }
1250
1251 static cycle_t          ftrace_update_time;
1252 static unsigned long    ftrace_update_cnt;
1253 unsigned long           ftrace_update_tot_cnt;
1254
1255 static int ftrace_update_code(struct module *mod)
1256 {
1257         struct dyn_ftrace *p;
1258         cycle_t start, stop;
1259
1260         start = ftrace_now(raw_smp_processor_id());
1261         ftrace_update_cnt = 0;
1262
1263         while (ftrace_new_addrs) {
1264
1265                 /* If something went wrong, bail without enabling anything */
1266                 if (unlikely(ftrace_disabled))
1267                         return -1;
1268
1269                 p = ftrace_new_addrs;
1270                 ftrace_new_addrs = p->newlist;
1271                 p->flags = 0L;
1272
1273                 /* convert record (i.e, patch mcount-call with NOP) */
1274                 if (ftrace_code_disable(mod, p)) {
1275                         p->flags |= FTRACE_FL_CONVERTED;
1276                         ftrace_update_cnt++;
1277                 } else
1278                         ftrace_free_rec(p);
1279         }
1280
1281         stop = ftrace_now(raw_smp_processor_id());
1282         ftrace_update_time = stop - start;
1283         ftrace_update_tot_cnt += ftrace_update_cnt;
1284
1285         return 0;
1286 }
1287
1288 static int __init ftrace_dyn_table_alloc(unsigned long num_to_init)
1289 {
1290         struct ftrace_page *pg;
1291         int cnt;
1292         int i;
1293
1294         /* allocate a few pages */
1295         ftrace_pages_start = (void *)get_zeroed_page(GFP_KERNEL);
1296         if (!ftrace_pages_start)
1297                 return -1;
1298
1299         /*
1300          * Allocate a few more pages.
1301          *
1302          * TODO: have some parser search vmlinux before
1303          *   final linking to find all calls to ftrace.
1304          *   Then we can:
1305          *    a) know how many pages to allocate.
1306          *     and/or
1307          *    b) set up the table then.
1308          *
1309          *  The dynamic code is still necessary for
1310          *  modules.
1311          */
1312
1313         pg = ftrace_pages = ftrace_pages_start;
1314
1315         cnt = num_to_init / ENTRIES_PER_PAGE;
1316         pr_info("ftrace: allocating %ld entries in %d pages\n",
1317                 num_to_init, cnt + 1);
1318
1319         for (i = 0; i < cnt; i++) {
1320                 pg->next = (void *)get_zeroed_page(GFP_KERNEL);
1321
1322                 /* If we fail, we'll try later anyway */
1323                 if (!pg->next)
1324                         break;
1325
1326                 pg = pg->next;
1327         }
1328
1329         return 0;
1330 }
1331
1332 enum {
1333         FTRACE_ITER_FILTER      = (1 << 0),
1334         FTRACE_ITER_NOTRACE     = (1 << 1),
1335         FTRACE_ITER_FAILURES    = (1 << 2),
1336         FTRACE_ITER_PRINTALL    = (1 << 3),
1337         FTRACE_ITER_HASH        = (1 << 4),
1338 };
1339
1340 #define FTRACE_BUFF_MAX (KSYM_SYMBOL_LEN+4) /* room for wildcards */
1341
1342 struct ftrace_iterator {
1343         struct ftrace_page      *pg;
1344         int                     hidx;
1345         int                     idx;
1346         unsigned                flags;
1347         struct trace_parser     parser;
1348 };
1349
1350 static void *
1351 t_hash_next(struct seq_file *m, void *v, loff_t *pos)
1352 {
1353         struct ftrace_iterator *iter = m->private;
1354         struct hlist_node *hnd = v;
1355         struct hlist_head *hhd;
1356
1357         WARN_ON(!(iter->flags & FTRACE_ITER_HASH));
1358
1359         (*pos)++;
1360
1361  retry:
1362         if (iter->hidx >= FTRACE_FUNC_HASHSIZE)
1363                 return NULL;
1364
1365         hhd = &ftrace_func_hash[iter->hidx];
1366
1367         if (hlist_empty(hhd)) {
1368                 iter->hidx++;
1369                 hnd = NULL;
1370                 goto retry;
1371         }
1372
1373         if (!hnd)
1374                 hnd = hhd->first;
1375         else {
1376                 hnd = hnd->next;
1377                 if (!hnd) {
1378                         iter->hidx++;
1379                         goto retry;
1380                 }
1381         }
1382
1383         return hnd;
1384 }
1385
1386 static void *t_hash_start(struct seq_file *m, loff_t *pos)
1387 {
1388         struct ftrace_iterator *iter = m->private;
1389         void *p = NULL;
1390         loff_t l;
1391
1392         if (!(iter->flags & FTRACE_ITER_HASH))
1393                 *pos = 0;
1394
1395         iter->flags |= FTRACE_ITER_HASH;
1396
1397         iter->hidx = 0;
1398         for (l = 0; l <= *pos; ) {
1399                 p = t_hash_next(m, p, &l);
1400                 if (!p)
1401                         break;
1402         }
1403         return p;
1404 }
1405
1406 static int t_hash_show(struct seq_file *m, void *v)
1407 {
1408         struct ftrace_func_probe *rec;
1409         struct hlist_node *hnd = v;
1410
1411         rec = hlist_entry(hnd, struct ftrace_func_probe, node);
1412
1413         if (rec->ops->print)
1414                 return rec->ops->print(m, rec->ip, rec->ops, rec->data);
1415
1416         seq_printf(m, "%ps:%ps", (void *)rec->ip, (void *)rec->ops->func);
1417
1418         if (rec->data)
1419                 seq_printf(m, ":%p", rec->data);
1420         seq_putc(m, '\n');
1421
1422         return 0;
1423 }
1424
1425 static void *
1426 t_next(struct seq_file *m, void *v, loff_t *pos)
1427 {
1428         struct ftrace_iterator *iter = m->private;
1429         struct dyn_ftrace *rec = NULL;
1430
1431         if (iter->flags & FTRACE_ITER_HASH)
1432                 return t_hash_next(m, v, pos);
1433
1434         (*pos)++;
1435
1436         if (iter->flags & FTRACE_ITER_PRINTALL)
1437                 return NULL;
1438
1439  retry:
1440         if (iter->idx >= iter->pg->index) {
1441                 if (iter->pg->next) {
1442                         iter->pg = iter->pg->next;
1443                         iter->idx = 0;
1444                         goto retry;
1445                 }
1446         } else {
1447                 rec = &iter->pg->records[iter->idx++];
1448                 if ((rec->flags & FTRACE_FL_FREE) ||
1449
1450                     (!(iter->flags & FTRACE_ITER_FAILURES) &&
1451                      (rec->flags & FTRACE_FL_FAILED)) ||
1452
1453                     ((iter->flags & FTRACE_ITER_FAILURES) &&
1454                      !(rec->flags & FTRACE_FL_FAILED)) ||
1455
1456                     ((iter->flags & FTRACE_ITER_FILTER) &&
1457                      !(rec->flags & FTRACE_FL_FILTER)) ||
1458
1459                     ((iter->flags & FTRACE_ITER_NOTRACE) &&
1460                      !(rec->flags & FTRACE_FL_NOTRACE))) {
1461                         rec = NULL;
1462                         goto retry;
1463                 }
1464         }
1465
1466         return rec;
1467 }
1468
1469 static void *t_start(struct seq_file *m, loff_t *pos)
1470 {
1471         struct ftrace_iterator *iter = m->private;
1472         void *p = NULL;
1473         loff_t l;
1474
1475         mutex_lock(&ftrace_lock);
1476         /*
1477          * For set_ftrace_filter reading, if we have the filter
1478          * off, we can short cut and just print out that all
1479          * functions are enabled.
1480          */
1481         if (iter->flags & FTRACE_ITER_FILTER && !ftrace_filtered) {
1482                 if (*pos > 0)
1483                         return t_hash_start(m, pos);
1484                 iter->flags |= FTRACE_ITER_PRINTALL;
1485                 return iter;
1486         }
1487
1488         if (iter->flags & FTRACE_ITER_HASH)
1489                 return t_hash_start(m, pos);
1490
1491         iter->pg = ftrace_pages_start;
1492         iter->idx = 0;
1493         for (l = 0; l <= *pos; ) {
1494                 p = t_next(m, p, &l);
1495                 if (!p)
1496                         break;
1497         }
1498
1499         if (!p && iter->flags & FTRACE_ITER_FILTER)
1500                 return t_hash_start(m, pos);
1501
1502         return p;
1503 }
1504
1505 static void t_stop(struct seq_file *m, void *p)
1506 {
1507         mutex_unlock(&ftrace_lock);
1508 }
1509
1510 static int t_show(struct seq_file *m, void *v)
1511 {
1512         struct ftrace_iterator *iter = m->private;
1513         struct dyn_ftrace *rec = v;
1514
1515         if (iter->flags & FTRACE_ITER_HASH)
1516                 return t_hash_show(m, v);
1517
1518         if (iter->flags & FTRACE_ITER_PRINTALL) {
1519                 seq_printf(m, "#### all functions enabled ####\n");
1520                 return 0;
1521         }
1522
1523         if (!rec)
1524                 return 0;
1525
1526         seq_printf(m, "%ps\n", (void *)rec->ip);
1527
1528         return 0;
1529 }
1530
1531 static const struct seq_operations show_ftrace_seq_ops = {
1532         .start = t_start,
1533         .next = t_next,
1534         .stop = t_stop,
1535         .show = t_show,
1536 };
1537
1538 static int
1539 ftrace_avail_open(struct inode *inode, struct file *file)
1540 {
1541         struct ftrace_iterator *iter;
1542         int ret;
1543
1544         if (unlikely(ftrace_disabled))
1545                 return -ENODEV;
1546
1547         iter = kzalloc(sizeof(*iter), GFP_KERNEL);
1548         if (!iter)
1549                 return -ENOMEM;
1550
1551         iter->pg = ftrace_pages_start;
1552
1553         ret = seq_open(file, &show_ftrace_seq_ops);
1554         if (!ret) {
1555                 struct seq_file *m = file->private_data;
1556
1557                 m->private = iter;
1558         } else {
1559                 kfree(iter);
1560         }
1561
1562         return ret;
1563 }
1564
1565 static int
1566 ftrace_failures_open(struct inode *inode, struct file *file)
1567 {
1568         int ret;
1569         struct seq_file *m;
1570         struct ftrace_iterator *iter;
1571
1572         ret = ftrace_avail_open(inode, file);
1573         if (!ret) {
1574                 m = (struct seq_file *)file->private_data;
1575                 iter = (struct ftrace_iterator *)m->private;
1576                 iter->flags = FTRACE_ITER_FAILURES;
1577         }
1578
1579         return ret;
1580 }
1581
1582
1583 static void ftrace_filter_reset(int enable)
1584 {
1585         struct ftrace_page *pg;
1586         struct dyn_ftrace *rec;
1587         unsigned long type = enable ? FTRACE_FL_FILTER : FTRACE_FL_NOTRACE;
1588
1589         mutex_lock(&ftrace_lock);
1590         if (enable)
1591                 ftrace_filtered = 0;
1592         do_for_each_ftrace_rec(pg, rec) {
1593                 if (rec->flags & FTRACE_FL_FAILED)
1594                         continue;
1595                 rec->flags &= ~type;
1596         } while_for_each_ftrace_rec();
1597         mutex_unlock(&ftrace_lock);
1598 }
1599
1600 static int
1601 ftrace_regex_open(struct inode *inode, struct file *file, int enable)
1602 {
1603         struct ftrace_iterator *iter;
1604         int ret = 0;
1605
1606         if (unlikely(ftrace_disabled))
1607                 return -ENODEV;
1608
1609         iter = kzalloc(sizeof(*iter), GFP_KERNEL);
1610         if (!iter)
1611                 return -ENOMEM;
1612
1613         if (trace_parser_get_init(&iter->parser, FTRACE_BUFF_MAX)) {
1614                 kfree(iter);
1615                 return -ENOMEM;
1616         }
1617
1618         mutex_lock(&ftrace_regex_lock);
1619         if ((file->f_mode & FMODE_WRITE) &&
1620             (file->f_flags & O_TRUNC))
1621                 ftrace_filter_reset(enable);
1622
1623         if (file->f_mode & FMODE_READ) {
1624                 iter->pg = ftrace_pages_start;
1625                 iter->flags = enable ? FTRACE_ITER_FILTER :
1626                         FTRACE_ITER_NOTRACE;
1627
1628                 ret = seq_open(file, &show_ftrace_seq_ops);
1629                 if (!ret) {
1630                         struct seq_file *m = file->private_data;
1631                         m->private = iter;
1632                 } else {
1633                         trace_parser_put(&iter->parser);
1634                         kfree(iter);
1635                 }
1636         } else
1637                 file->private_data = iter;
1638         mutex_unlock(&ftrace_regex_lock);
1639
1640         return ret;
1641 }
1642
1643 static int
1644 ftrace_filter_open(struct inode *inode, struct file *file)
1645 {
1646         return ftrace_regex_open(inode, file, 1);
1647 }
1648
1649 static int
1650 ftrace_notrace_open(struct inode *inode, struct file *file)
1651 {
1652         return ftrace_regex_open(inode, file, 0);
1653 }
1654
1655 static loff_t
1656 ftrace_regex_lseek(struct file *file, loff_t offset, int origin)
1657 {
1658         loff_t ret;
1659
1660         if (file->f_mode & FMODE_READ)
1661                 ret = seq_lseek(file, offset, origin);
1662         else
1663                 file->f_pos = ret = 1;
1664
1665         return ret;
1666 }
1667
1668 static int ftrace_match(char *str, char *regex, int len, int type)
1669 {
1670         int matched = 0;
1671         char *ptr;
1672
1673         switch (type) {
1674         case MATCH_FULL:
1675                 if (strcmp(str, regex) == 0)
1676                         matched = 1;
1677                 break;
1678         case MATCH_FRONT_ONLY:
1679                 if (strncmp(str, regex, len) == 0)
1680                         matched = 1;
1681                 break;
1682         case MATCH_MIDDLE_ONLY:
1683                 if (strstr(str, regex))
1684                         matched = 1;
1685                 break;
1686         case MATCH_END_ONLY:
1687                 ptr = strstr(str, regex);
1688                 if (ptr && (ptr[len] == 0))
1689                         matched = 1;
1690                 break;
1691         }
1692
1693         return matched;
1694 }
1695
1696 static int
1697 ftrace_match_record(struct dyn_ftrace *rec, char *regex, int len, int type)
1698 {
1699         char str[KSYM_SYMBOL_LEN];
1700
1701         kallsyms_lookup(rec->ip, NULL, NULL, NULL, str);
1702         return ftrace_match(str, regex, len, type);
1703 }
1704
1705 static void ftrace_match_records(char *buff, int len, int enable)
1706 {
1707         unsigned int search_len;
1708         struct ftrace_page *pg;
1709         struct dyn_ftrace *rec;
1710         unsigned long flag;
1711         char *search;
1712         int type;
1713         int not;
1714
1715         flag = enable ? FTRACE_FL_FILTER : FTRACE_FL_NOTRACE;
1716         type = filter_parse_regex(buff, len, &search, &not);
1717
1718         search_len = strlen(search);
1719
1720         mutex_lock(&ftrace_lock);
1721         do_for_each_ftrace_rec(pg, rec) {
1722
1723                 if (rec->flags & FTRACE_FL_FAILED)
1724                         continue;
1725
1726                 if (ftrace_match_record(rec, search, search_len, type)) {
1727                         if (not)
1728                                 rec->flags &= ~flag;
1729                         else
1730                                 rec->flags |= flag;
1731                 }
1732                 /*
1733                  * Only enable filtering if we have a function that
1734                  * is filtered on.
1735                  */
1736                 if (enable && (rec->flags & FTRACE_FL_FILTER))
1737                         ftrace_filtered = 1;
1738         } while_for_each_ftrace_rec();
1739         mutex_unlock(&ftrace_lock);
1740 }
1741
1742 static int
1743 ftrace_match_module_record(struct dyn_ftrace *rec, char *mod,
1744                            char *regex, int len, int type)
1745 {
1746         char str[KSYM_SYMBOL_LEN];
1747         char *modname;
1748
1749         kallsyms_lookup(rec->ip, NULL, NULL, &modname, str);
1750
1751         if (!modname || strcmp(modname, mod))
1752                 return 0;
1753
1754         /* blank search means to match all funcs in the mod */
1755         if (len)
1756                 return ftrace_match(str, regex, len, type);
1757         else
1758                 return 1;
1759 }
1760
1761 static void ftrace_match_module_records(char *buff, char *mod, int enable)
1762 {
1763         unsigned search_len = 0;
1764         struct ftrace_page *pg;
1765         struct dyn_ftrace *rec;
1766         int type = MATCH_FULL;
1767         char *search = buff;
1768         unsigned long flag;
1769         int not = 0;
1770
1771         flag = enable ? FTRACE_FL_FILTER : FTRACE_FL_NOTRACE;
1772
1773         /* blank or '*' mean the same */
1774         if (strcmp(buff, "*") == 0)
1775                 buff[0] = 0;
1776
1777         /* handle the case of 'dont filter this module' */
1778         if (strcmp(buff, "!") == 0 || strcmp(buff, "!*") == 0) {
1779                 buff[0] = 0;
1780                 not = 1;
1781         }
1782
1783         if (strlen(buff)) {
1784                 type = filter_parse_regex(buff, strlen(buff), &search, &not);
1785                 search_len = strlen(search);
1786         }
1787
1788         mutex_lock(&ftrace_lock);
1789         do_for_each_ftrace_rec(pg, rec) {
1790
1791                 if (rec->flags & FTRACE_FL_FAILED)
1792                         continue;
1793
1794                 if (ftrace_match_module_record(rec, mod,
1795                                                search, search_len, type)) {
1796                         if (not)
1797                                 rec->flags &= ~flag;
1798                         else
1799                                 rec->flags |= flag;
1800                 }
1801                 if (enable && (rec->flags & FTRACE_FL_FILTER))
1802                         ftrace_filtered = 1;
1803
1804         } while_for_each_ftrace_rec();
1805         mutex_unlock(&ftrace_lock);
1806 }
1807
1808 /*
1809  * We register the module command as a template to show others how
1810  * to register the a command as well.
1811  */
1812
1813 static int
1814 ftrace_mod_callback(char *func, char *cmd, char *param, int enable)
1815 {
1816         char *mod;
1817
1818         /*
1819          * cmd == 'mod' because we only registered this func
1820          * for the 'mod' ftrace_func_command.
1821          * But if you register one func with multiple commands,
1822          * you can tell which command was used by the cmd
1823          * parameter.
1824          */
1825
1826         /* we must have a module name */
1827         if (!param)
1828                 return -EINVAL;
1829
1830         mod = strsep(&param, ":");
1831         if (!strlen(mod))
1832                 return -EINVAL;
1833
1834         ftrace_match_module_records(func, mod, enable);
1835         return 0;
1836 }
1837
1838 static struct ftrace_func_command ftrace_mod_cmd = {
1839         .name                   = "mod",
1840         .func                   = ftrace_mod_callback,
1841 };
1842
1843 static int __init ftrace_mod_cmd_init(void)
1844 {
1845         return register_ftrace_command(&ftrace_mod_cmd);
1846 }
1847 device_initcall(ftrace_mod_cmd_init);
1848
1849 static void
1850 function_trace_probe_call(unsigned long ip, unsigned long parent_ip)
1851 {
1852         struct ftrace_func_probe *entry;
1853         struct hlist_head *hhd;
1854         struct hlist_node *n;
1855         unsigned long key;
1856         int resched;
1857
1858         key = hash_long(ip, FTRACE_HASH_BITS);
1859
1860         hhd = &ftrace_func_hash[key];
1861
1862         if (hlist_empty(hhd))
1863                 return;
1864
1865         /*
1866          * Disable preemption for these calls to prevent a RCU grace
1867          * period. This syncs the hash iteration and freeing of items
1868          * on the hash. rcu_read_lock is too dangerous here.
1869          */
1870         resched = ftrace_preempt_disable();
1871         hlist_for_each_entry_rcu(entry, n, hhd, node) {
1872                 if (entry->ip == ip)
1873                         entry->ops->func(ip, parent_ip, &entry->data);
1874         }
1875         ftrace_preempt_enable(resched);
1876 }
1877
1878 static struct ftrace_ops trace_probe_ops __read_mostly =
1879 {
1880         .func           = function_trace_probe_call,
1881 };
1882
1883 static int ftrace_probe_registered;
1884
1885 static void __enable_ftrace_function_probe(void)
1886 {
1887         int i;
1888
1889         if (ftrace_probe_registered)
1890                 return;
1891
1892         for (i = 0; i < FTRACE_FUNC_HASHSIZE; i++) {
1893                 struct hlist_head *hhd = &ftrace_func_hash[i];
1894                 if (hhd->first)
1895                         break;
1896         }
1897         /* Nothing registered? */
1898         if (i == FTRACE_FUNC_HASHSIZE)
1899                 return;
1900
1901         __register_ftrace_function(&trace_probe_ops);
1902         ftrace_startup(0);
1903         ftrace_probe_registered = 1;
1904 }
1905
1906 static void __disable_ftrace_function_probe(void)
1907 {
1908         int i;
1909
1910         if (!ftrace_probe_registered)
1911                 return;
1912
1913         for (i = 0; i < FTRACE_FUNC_HASHSIZE; i++) {
1914                 struct hlist_head *hhd = &ftrace_func_hash[i];
1915                 if (hhd->first)
1916                         return;
1917         }
1918
1919         /* no more funcs left */
1920         __unregister_ftrace_function(&trace_probe_ops);
1921         ftrace_shutdown(0);
1922         ftrace_probe_registered = 0;
1923 }
1924
1925
1926 static void ftrace_free_entry_rcu(struct rcu_head *rhp)
1927 {
1928         struct ftrace_func_probe *entry =
1929                 container_of(rhp, struct ftrace_func_probe, rcu);
1930
1931         if (entry->ops->free)
1932                 entry->ops->free(&entry->data);
1933         kfree(entry);
1934 }
1935
1936
1937 int
1938 register_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops,
1939                               void *data)
1940 {
1941         struct ftrace_func_probe *entry;
1942         struct ftrace_page *pg;
1943         struct dyn_ftrace *rec;
1944         int type, len, not;
1945         unsigned long key;
1946         int count = 0;
1947         char *search;
1948
1949         type = filter_parse_regex(glob, strlen(glob), &search, &not);
1950         len = strlen(search);
1951
1952         /* we do not support '!' for function probes */
1953         if (WARN_ON(not))
1954                 return -EINVAL;
1955
1956         mutex_lock(&ftrace_lock);
1957         do_for_each_ftrace_rec(pg, rec) {
1958
1959                 if (rec->flags & FTRACE_FL_FAILED)
1960                         continue;
1961
1962                 if (!ftrace_match_record(rec, search, len, type))
1963                         continue;
1964
1965                 entry = kmalloc(sizeof(*entry), GFP_KERNEL);
1966                 if (!entry) {
1967                         /* If we did not process any, then return error */
1968                         if (!count)
1969                                 count = -ENOMEM;
1970                         goto out_unlock;
1971                 }
1972
1973                 count++;
1974
1975                 entry->data = data;
1976
1977                 /*
1978                  * The caller might want to do something special
1979                  * for each function we find. We call the callback
1980                  * to give the caller an opportunity to do so.
1981                  */
1982                 if (ops->callback) {
1983                         if (ops->callback(rec->ip, &entry->data) < 0) {
1984                                 /* caller does not like this func */
1985                                 kfree(entry);
1986                                 continue;
1987                         }
1988                 }
1989
1990                 entry->ops = ops;
1991                 entry->ip = rec->ip;
1992
1993                 key = hash_long(entry->ip, FTRACE_HASH_BITS);
1994                 hlist_add_head_rcu(&entry->node, &ftrace_func_hash[key]);
1995
1996         } while_for_each_ftrace_rec();
1997         __enable_ftrace_function_probe();
1998
1999  out_unlock:
2000         mutex_unlock(&ftrace_lock);
2001
2002         return count;
2003 }
2004
2005 enum {
2006         PROBE_TEST_FUNC         = 1,
2007         PROBE_TEST_DATA         = 2
2008 };
2009
2010 static void
2011 __unregister_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops,
2012                                   void *data, int flags)
2013 {
2014         struct ftrace_func_probe *entry;
2015         struct hlist_node *n, *tmp;
2016         char str[KSYM_SYMBOL_LEN];
2017         int type = MATCH_FULL;
2018         int i, len = 0;
2019         char *search;
2020
2021         if (glob && (strcmp(glob, "*") == 0 || !strlen(glob)))
2022                 glob = NULL;
2023         else if (glob) {
2024                 int not;
2025
2026                 type = filter_parse_regex(glob, strlen(glob), &search, &not);
2027                 len = strlen(search);
2028
2029                 /* we do not support '!' for function probes */
2030                 if (WARN_ON(not))
2031                         return;
2032         }
2033
2034         mutex_lock(&ftrace_lock);
2035         for (i = 0; i < FTRACE_FUNC_HASHSIZE; i++) {
2036                 struct hlist_head *hhd = &ftrace_func_hash[i];
2037
2038                 hlist_for_each_entry_safe(entry, n, tmp, hhd, node) {
2039
2040                         /* break up if statements for readability */
2041                         if ((flags & PROBE_TEST_FUNC) && entry->ops != ops)
2042                                 continue;
2043
2044                         if ((flags & PROBE_TEST_DATA) && entry->data != data)
2045                                 continue;
2046
2047                         /* do this last, since it is the most expensive */
2048                         if (glob) {
2049                                 kallsyms_lookup(entry->ip, NULL, NULL,
2050                                                 NULL, str);
2051                                 if (!ftrace_match(str, glob, len, type))
2052                                         continue;
2053                         }
2054
2055                         hlist_del(&entry->node);
2056                         call_rcu(&entry->rcu, ftrace_free_entry_rcu);
2057                 }
2058         }
2059         __disable_ftrace_function_probe();
2060         mutex_unlock(&ftrace_lock);
2061 }
2062
2063 void
2064 unregister_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops,
2065                                 void *data)
2066 {
2067         __unregister_ftrace_function_probe(glob, ops, data,
2068                                           PROBE_TEST_FUNC | PROBE_TEST_DATA);
2069 }
2070
2071 void
2072 unregister_ftrace_function_probe_func(char *glob, struct ftrace_probe_ops *ops)
2073 {
2074         __unregister_ftrace_function_probe(glob, ops, NULL, PROBE_TEST_FUNC);
2075 }
2076
2077 void unregister_ftrace_function_probe_all(char *glob)
2078 {
2079         __unregister_ftrace_function_probe(glob, NULL, NULL, 0);
2080 }
2081
2082 static LIST_HEAD(ftrace_commands);
2083 static DEFINE_MUTEX(ftrace_cmd_mutex);
2084
2085 int register_ftrace_command(struct ftrace_func_command *cmd)
2086 {
2087         struct ftrace_func_command *p;
2088         int ret = 0;
2089
2090         mutex_lock(&ftrace_cmd_mutex);
2091         list_for_each_entry(p, &ftrace_commands, list) {
2092                 if (strcmp(cmd->name, p->name) == 0) {
2093                         ret = -EBUSY;
2094                         goto out_unlock;
2095                 }
2096         }
2097         list_add(&cmd->list, &ftrace_commands);
2098  out_unlock:
2099         mutex_unlock(&ftrace_cmd_mutex);
2100
2101         return ret;
2102 }
2103
2104 int unregister_ftrace_command(struct ftrace_func_command *cmd)
2105 {
2106         struct ftrace_func_command *p, *n;
2107         int ret = -ENODEV;
2108
2109         mutex_lock(&ftrace_cmd_mutex);
2110         list_for_each_entry_safe(p, n, &ftrace_commands, list) {
2111                 if (strcmp(cmd->name, p->name) == 0) {
2112                         ret = 0;
2113                         list_del_init(&p->list);
2114                         goto out_unlock;
2115                 }
2116         }
2117  out_unlock:
2118         mutex_unlock(&ftrace_cmd_mutex);
2119
2120         return ret;
2121 }
2122
2123 static int ftrace_process_regex(char *buff, int len, int enable)
2124 {
2125         char *func, *command, *next = buff;
2126         struct ftrace_func_command *p;
2127         int ret = -EINVAL;
2128
2129         func = strsep(&next, ":");
2130
2131         if (!next) {
2132                 ftrace_match_records(func, len, enable);
2133                 return 0;
2134         }
2135
2136         /* command found */
2137
2138         command = strsep(&next, ":");
2139
2140         mutex_lock(&ftrace_cmd_mutex);
2141         list_for_each_entry(p, &ftrace_commands, list) {
2142                 if (strcmp(p->name, command) == 0) {
2143                         ret = p->func(func, command, next, enable);
2144                         goto out_unlock;
2145                 }
2146         }
2147  out_unlock:
2148         mutex_unlock(&ftrace_cmd_mutex);
2149
2150         return ret;
2151 }
2152
2153 static ssize_t
2154 ftrace_regex_write(struct file *file, const char __user *ubuf,
2155                    size_t cnt, loff_t *ppos, int enable)
2156 {
2157         struct ftrace_iterator *iter;
2158         struct trace_parser *parser;
2159         ssize_t ret, read;
2160
2161         if (!cnt)
2162                 return 0;
2163
2164         mutex_lock(&ftrace_regex_lock);
2165
2166         if (file->f_mode & FMODE_READ) {
2167                 struct seq_file *m = file->private_data;
2168                 iter = m->private;
2169         } else
2170                 iter = file->private_data;
2171
2172         parser = &iter->parser;
2173         read = trace_get_user(parser, ubuf, cnt, ppos);
2174
2175         if (read >= 0 && trace_parser_loaded(parser) &&
2176             !trace_parser_cont(parser)) {
2177                 ret = ftrace_process_regex(parser->buffer,
2178                                            parser->idx, enable);
2179                 if (ret)
2180                         goto out;
2181
2182                 trace_parser_clear(parser);
2183         }
2184
2185         ret = read;
2186
2187         mutex_unlock(&ftrace_regex_lock);
2188 out:
2189         return ret;
2190 }
2191
2192 static ssize_t
2193 ftrace_filter_write(struct file *file, const char __user *ubuf,
2194                     size_t cnt, loff_t *ppos)
2195 {
2196         return ftrace_regex_write(file, ubuf, cnt, ppos, 1);
2197 }
2198
2199 static ssize_t
2200 ftrace_notrace_write(struct file *file, const char __user *ubuf,
2201                      size_t cnt, loff_t *ppos)
2202 {
2203         return ftrace_regex_write(file, ubuf, cnt, ppos, 0);
2204 }
2205
2206 static void
2207 ftrace_set_regex(unsigned char *buf, int len, int reset, int enable)
2208 {
2209         if (unlikely(ftrace_disabled))
2210                 return;
2211
2212         mutex_lock(&ftrace_regex_lock);
2213         if (reset)
2214                 ftrace_filter_reset(enable);
2215         if (buf)
2216                 ftrace_match_records(buf, len, enable);
2217         mutex_unlock(&ftrace_regex_lock);
2218 }
2219
2220 /**
2221  * ftrace_set_filter - set a function to filter on in ftrace
2222  * @buf - the string that holds the function filter text.
2223  * @len - the length of the string.
2224  * @reset - non zero to reset all filters before applying this filter.
2225  *
2226  * Filters denote which functions should be enabled when tracing is enabled.
2227  * If @buf is NULL and reset is set, all functions will be enabled for tracing.
2228  */
2229 void ftrace_set_filter(unsigned char *buf, int len, int reset)
2230 {
2231         ftrace_set_regex(buf, len, reset, 1);
2232 }
2233
2234 /**
2235  * ftrace_set_notrace - set a function to not trace in ftrace
2236  * @buf - the string that holds the function notrace text.
2237  * @len - the length of the string.
2238  * @reset - non zero to reset all filters before applying this filter.
2239  *
2240  * Notrace Filters denote which functions should not be enabled when tracing
2241  * is enabled. If @buf is NULL and reset is set, all functions will be enabled
2242  * for tracing.
2243  */
2244 void ftrace_set_notrace(unsigned char *buf, int len, int reset)
2245 {
2246         ftrace_set_regex(buf, len, reset, 0);
2247 }
2248
2249 /*
2250  * command line interface to allow users to set filters on boot up.
2251  */
2252 #define FTRACE_FILTER_SIZE              COMMAND_LINE_SIZE
2253 static char ftrace_notrace_buf[FTRACE_FILTER_SIZE] __initdata;
2254 static char ftrace_filter_buf[FTRACE_FILTER_SIZE] __initdata;
2255 static char ftrace_graph_buf[FTRACE_FILTER_SIZE] __initdata;
2256
2257 static int __init set_ftrace_notrace(char *str)
2258 {
2259         strncpy(ftrace_notrace_buf, str, FTRACE_FILTER_SIZE);
2260         return 1;
2261 }
2262 __setup("ftrace_notrace=", set_ftrace_notrace);
2263
2264 static int __init set_ftrace_filter(char *str)
2265 {
2266         strncpy(ftrace_filter_buf, str, FTRACE_FILTER_SIZE);
2267         return 1;
2268 }
2269 __setup("ftrace_filter=", set_ftrace_filter);
2270
2271 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
2272 static int __init set_graph_function(char *str)
2273 {
2274         strncpy(ftrace_graph_buf, str, FTRACE_FILTER_SIZE);
2275         return 1;
2276 }
2277 __setup("ftrace_graph_filter=", set_graph_function);
2278
2279 static void __init set_ftrace_early_graph(char *buf)
2280 {
2281         int ret;
2282         char *func;
2283
2284         while (buf) {
2285                 func = strsep(&buf, ",");
2286                 /* we allow only one expression at a time */
2287                 ret = ftrace_set_func(ftrace_graph_funcs, &ftrace_graph_count,
2288                                       func);
2289                 if (ret)
2290                         printk(KERN_DEBUG "ftrace: function %s not "
2291                                           "traceable\n", func);
2292         }
2293 }
2294 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
2295
2296 static void __init set_ftrace_early_filter(char *buf, int enable)
2297 {
2298         char *func;
2299
2300         while (buf) {
2301                 func = strsep(&buf, ",");
2302                 ftrace_set_regex(func, strlen(func), 0, enable);
2303         }
2304 }
2305
2306 static void __init set_ftrace_early_filters(void)
2307 {
2308         if (ftrace_filter_buf[0])
2309                 set_ftrace_early_filter(ftrace_filter_buf, 1);
2310         if (ftrace_notrace_buf[0])
2311                 set_ftrace_early_filter(ftrace_notrace_buf, 0);
2312 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
2313         if (ftrace_graph_buf[0])
2314                 set_ftrace_early_graph(ftrace_graph_buf);
2315 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
2316 }
2317
2318 static int
2319 ftrace_regex_release(struct inode *inode, struct file *file, int enable)
2320 {
2321         struct seq_file *m = (struct seq_file *)file->private_data;
2322         struct ftrace_iterator *iter;
2323         struct trace_parser *parser;
2324
2325         mutex_lock(&ftrace_regex_lock);
2326         if (file->f_mode & FMODE_READ) {
2327                 iter = m->private;
2328
2329                 seq_release(inode, file);
2330         } else
2331                 iter = file->private_data;
2332
2333         parser = &iter->parser;
2334         if (trace_parser_loaded(parser)) {
2335                 parser->buffer[parser->idx] = 0;
2336                 ftrace_match_records(parser->buffer, parser->idx, enable);
2337         }
2338
2339         mutex_lock(&ftrace_lock);
2340         if (ftrace_start_up && ftrace_enabled)
2341                 ftrace_run_update_code(FTRACE_ENABLE_CALLS);
2342         mutex_unlock(&ftrace_lock);
2343
2344         trace_parser_put(parser);
2345         kfree(iter);
2346
2347         mutex_unlock(&ftrace_regex_lock);
2348         return 0;
2349 }
2350
2351 static int
2352 ftrace_filter_release(struct inode *inode, struct file *file)
2353 {
2354         return ftrace_regex_release(inode, file, 1);
2355 }
2356
2357 static int
2358 ftrace_notrace_release(struct inode *inode, struct file *file)
2359 {
2360         return ftrace_regex_release(inode, file, 0);
2361 }
2362
2363 static const struct file_operations ftrace_avail_fops = {
2364         .open = ftrace_avail_open,
2365         .read = seq_read,
2366         .llseek = seq_lseek,
2367         .release = seq_release_private,
2368 };
2369
2370 static const struct file_operations ftrace_failures_fops = {
2371         .open = ftrace_failures_open,
2372         .read = seq_read,
2373         .llseek = seq_lseek,
2374         .release = seq_release_private,
2375 };
2376
2377 static const struct file_operations ftrace_filter_fops = {
2378         .open = ftrace_filter_open,
2379         .read = seq_read,
2380         .write = ftrace_filter_write,
2381         .llseek = ftrace_regex_lseek,
2382         .release = ftrace_filter_release,
2383 };
2384
2385 static const struct file_operations ftrace_notrace_fops = {
2386         .open = ftrace_notrace_open,
2387         .read = seq_read,
2388         .write = ftrace_notrace_write,
2389         .llseek = ftrace_regex_lseek,
2390         .release = ftrace_notrace_release,
2391 };
2392
2393 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
2394
2395 static DEFINE_MUTEX(graph_lock);
2396
2397 int ftrace_graph_count;
2398 unsigned long ftrace_graph_funcs[FTRACE_GRAPH_MAX_FUNCS] __read_mostly;
2399
2400 static void *
2401 __g_next(struct seq_file *m, loff_t *pos)
2402 {
2403         if (*pos >= ftrace_graph_count)
2404                 return NULL;
2405         return &ftrace_graph_funcs[*pos];
2406 }
2407
2408 static void *
2409 g_next(struct seq_file *m, void *v, loff_t *pos)
2410 {
2411         (*pos)++;
2412         return __g_next(m, pos);
2413 }
2414
2415 static void *g_start(struct seq_file *m, loff_t *pos)
2416 {
2417         mutex_lock(&graph_lock);
2418
2419         /* Nothing, tell g_show to print all functions are enabled */
2420         if (!ftrace_graph_count && !*pos)
2421                 return (void *)1;
2422
2423         return __g_next(m, pos);
2424 }
2425
2426 static void g_stop(struct seq_file *m, void *p)
2427 {
2428         mutex_unlock(&graph_lock);
2429 }
2430
2431 static int g_show(struct seq_file *m, void *v)
2432 {
2433         unsigned long *ptr = v;
2434
2435         if (!ptr)
2436                 return 0;
2437
2438         if (ptr == (unsigned long *)1) {
2439                 seq_printf(m, "#### all functions enabled ####\n");
2440                 return 0;
2441         }
2442
2443         seq_printf(m, "%ps\n", (void *)*ptr);
2444
2445         return 0;
2446 }
2447
2448 static const struct seq_operations ftrace_graph_seq_ops = {
2449         .start = g_start,
2450         .next = g_next,
2451         .stop = g_stop,
2452         .show = g_show,
2453 };
2454
2455 static int
2456 ftrace_graph_open(struct inode *inode, struct file *file)
2457 {
2458         int ret = 0;
2459
2460         if (unlikely(ftrace_disabled))
2461                 return -ENODEV;
2462
2463         mutex_lock(&graph_lock);
2464         if ((file->f_mode & FMODE_WRITE) &&
2465             (file->f_flags & O_TRUNC)) {
2466                 ftrace_graph_count = 0;
2467                 memset(ftrace_graph_funcs, 0, sizeof(ftrace_graph_funcs));
2468         }
2469         mutex_unlock(&graph_lock);
2470
2471         if (file->f_mode & FMODE_READ)
2472                 ret = seq_open(file, &ftrace_graph_seq_ops);
2473
2474         return ret;
2475 }
2476
2477 static int
2478 ftrace_graph_release(struct inode *inode, struct file *file)
2479 {
2480         if (file->f_mode & FMODE_READ)
2481                 seq_release(inode, file);
2482         return 0;
2483 }
2484
2485 static int
2486 ftrace_set_func(unsigned long *array, int *idx, char *buffer)
2487 {
2488         struct dyn_ftrace *rec;
2489         struct ftrace_page *pg;
2490         int search_len;
2491         int found = 0;
2492         int type, not;
2493         char *search;
2494         bool exists;
2495         int i;
2496
2497         if (ftrace_disabled)
2498                 return -ENODEV;
2499
2500         /* decode regex */
2501         type = filter_parse_regex(buffer, strlen(buffer), &search, &not);
2502         if (not)
2503                 return -EINVAL;
2504
2505         search_len = strlen(search);
2506
2507         mutex_lock(&ftrace_lock);
2508         do_for_each_ftrace_rec(pg, rec) {
2509
2510                 if (*idx >= FTRACE_GRAPH_MAX_FUNCS)
2511                         break;
2512
2513                 if (rec->flags & (FTRACE_FL_FAILED | FTRACE_FL_FREE))
2514                         continue;
2515
2516                 if (ftrace_match_record(rec, search, search_len, type)) {
2517                         /* ensure it is not already in the array */
2518                         exists = false;
2519                         for (i = 0; i < *idx; i++)
2520                                 if (array[i] == rec->ip) {
2521                                         exists = true;
2522                                         break;
2523                                 }
2524                         if (!exists) {
2525                                 array[(*idx)++] = rec->ip;
2526                                 found = 1;
2527                         }
2528                 }
2529         } while_for_each_ftrace_rec();
2530
2531         mutex_unlock(&ftrace_lock);
2532
2533         return found ? 0 : -EINVAL;
2534 }
2535
2536 static ssize_t
2537 ftrace_graph_write(struct file *file, const char __user *ubuf,
2538                    size_t cnt, loff_t *ppos)
2539 {
2540         struct trace_parser parser;
2541         ssize_t read, ret;
2542
2543         if (!cnt || cnt < 0)
2544                 return 0;
2545
2546         mutex_lock(&graph_lock);
2547
2548         if (ftrace_graph_count >= FTRACE_GRAPH_MAX_FUNCS) {
2549                 ret = -EBUSY;
2550                 goto out_unlock;
2551         }
2552
2553         if (trace_parser_get_init(&parser, FTRACE_BUFF_MAX)) {
2554                 ret = -ENOMEM;
2555                 goto out_unlock;
2556         }
2557
2558         read = trace_get_user(&parser, ubuf, cnt, ppos);
2559
2560         if (read >= 0 && trace_parser_loaded((&parser))) {
2561                 parser.buffer[parser.idx] = 0;
2562
2563                 /* we allow only one expression at a time */
2564                 ret = ftrace_set_func(ftrace_graph_funcs, &ftrace_graph_count,
2565                                         parser.buffer);
2566                 if (ret)
2567                         goto out_free;
2568         }
2569
2570         ret = read;
2571
2572 out_free:
2573         trace_parser_put(&parser);
2574 out_unlock:
2575         mutex_unlock(&graph_lock);
2576
2577         return ret;
2578 }
2579
2580 static const struct file_operations ftrace_graph_fops = {
2581         .open           = ftrace_graph_open,
2582         .read           = seq_read,
2583         .write          = ftrace_graph_write,
2584         .release        = ftrace_graph_release,
2585 };
2586 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
2587
2588 static __init int ftrace_init_dyn_debugfs(struct dentry *d_tracer)
2589 {
2590
2591         trace_create_file("available_filter_functions", 0444,
2592                         d_tracer, NULL, &ftrace_avail_fops);
2593
2594         trace_create_file("failures", 0444,
2595                         d_tracer, NULL, &ftrace_failures_fops);
2596
2597         trace_create_file("set_ftrace_filter", 0644, d_tracer,
2598                         NULL, &ftrace_filter_fops);
2599
2600         trace_create_file("set_ftrace_notrace", 0644, d_tracer,
2601                                     NULL, &ftrace_notrace_fops);
2602
2603 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
2604         trace_create_file("set_graph_function", 0444, d_tracer,
2605                                     NULL,
2606                                     &ftrace_graph_fops);
2607 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
2608
2609         return 0;
2610 }
2611
2612 static int ftrace_convert_nops(struct module *mod,
2613                                unsigned long *start,
2614                                unsigned long *end)
2615 {
2616         unsigned long *p;
2617         unsigned long addr;
2618         unsigned long flags;
2619
2620         mutex_lock(&ftrace_lock);
2621         p = start;
2622         while (p < end) {
2623                 addr = ftrace_call_adjust(*p++);
2624                 /*
2625                  * Some architecture linkers will pad between
2626                  * the different mcount_loc sections of different
2627                  * object files to satisfy alignments.
2628                  * Skip any NULL pointers.
2629                  */
2630                 if (!addr)
2631                         continue;
2632                 ftrace_record_ip(addr);
2633         }
2634
2635         /* disable interrupts to prevent kstop machine */
2636         local_irq_save(flags);
2637         ftrace_update_code(mod);
2638         local_irq_restore(flags);
2639         mutex_unlock(&ftrace_lock);
2640
2641         return 0;
2642 }
2643
2644 #ifdef CONFIG_MODULES
2645 void ftrace_release_mod(struct module *mod)
2646 {
2647         struct dyn_ftrace *rec;
2648         struct ftrace_page *pg;
2649
2650         if (ftrace_disabled)
2651                 return;
2652
2653         mutex_lock(&ftrace_lock);
2654         do_for_each_ftrace_rec(pg, rec) {
2655                 if (within_module_core(rec->ip, mod)) {
2656                         /*
2657                          * rec->ip is changed in ftrace_free_rec()
2658                          * It should not between s and e if record was freed.
2659                          */
2660                         FTRACE_WARN_ON(rec->flags & FTRACE_FL_FREE);
2661                         ftrace_free_rec(rec);
2662                 }
2663         } while_for_each_ftrace_rec();
2664         mutex_unlock(&ftrace_lock);
2665 }
2666
2667 static void ftrace_init_module(struct module *mod,
2668                                unsigned long *start, unsigned long *end)
2669 {
2670         if (ftrace_disabled || start == end)
2671                 return;
2672         ftrace_convert_nops(mod, start, end);
2673 }
2674
2675 static int ftrace_module_notify(struct notifier_block *self,
2676                                 unsigned long val, void *data)
2677 {
2678         struct module *mod = data;
2679
2680         switch (val) {
2681         case MODULE_STATE_COMING:
2682                 ftrace_init_module(mod, mod->ftrace_callsites,
2683                                    mod->ftrace_callsites +
2684                                    mod->num_ftrace_callsites);
2685                 break;
2686         case MODULE_STATE_GOING:
2687                 ftrace_release_mod(mod);
2688                 break;
2689         }
2690
2691         return 0;
2692 }
2693 #else
2694 static int ftrace_module_notify(struct notifier_block *self,
2695                                 unsigned long val, void *data)
2696 {
2697         return 0;
2698 }
2699 #endif /* CONFIG_MODULES */
2700
2701 struct notifier_block ftrace_module_nb = {
2702         .notifier_call = ftrace_module_notify,
2703         .priority = 0,
2704 };
2705
2706 extern unsigned long __start_mcount_loc[];
2707 extern unsigned long __stop_mcount_loc[];
2708
2709 void __init ftrace_init(void)
2710 {
2711         unsigned long count, addr, flags;
2712         int ret;
2713
2714         /* Keep the ftrace pointer to the stub */
2715         addr = (unsigned long)ftrace_stub;
2716
2717         local_irq_save(flags);
2718         ftrace_dyn_arch_init(&addr);
2719         local_irq_restore(flags);
2720
2721         /* ftrace_dyn_arch_init places the return code in addr */
2722         if (addr)
2723                 goto failed;
2724
2725         count = __stop_mcount_loc - __start_mcount_loc;
2726
2727         ret = ftrace_dyn_table_alloc(count);
2728         if (ret)
2729                 goto failed;
2730
2731         last_ftrace_enabled = ftrace_enabled = 1;
2732
2733         ret = ftrace_convert_nops(NULL,
2734                                   __start_mcount_loc,
2735                                   __stop_mcount_loc);
2736
2737         ret = register_module_notifier(&ftrace_module_nb);
2738         if (ret)
2739                 pr_warning("Failed to register trace ftrace module notifier\n");
2740
2741         set_ftrace_early_filters();
2742
2743         return;
2744  failed:
2745         ftrace_disabled = 1;
2746 }
2747
2748 #else
2749
2750 static int __init ftrace_nodyn_init(void)
2751 {
2752         ftrace_enabled = 1;
2753         return 0;
2754 }
2755 device_initcall(ftrace_nodyn_init);
2756
2757 static inline int ftrace_init_dyn_debugfs(struct dentry *d_tracer) { return 0; }
2758 static inline void ftrace_startup_enable(int command) { }
2759 /* Keep as macros so we do not need to define the commands */
2760 # define ftrace_startup(command)        do { } while (0)
2761 # define ftrace_shutdown(command)       do { } while (0)
2762 # define ftrace_startup_sysctl()        do { } while (0)
2763 # define ftrace_shutdown_sysctl()       do { } while (0)
2764 #endif /* CONFIG_DYNAMIC_FTRACE */
2765
2766 static void clear_ftrace_swapper(void)
2767 {
2768         struct task_struct *p;
2769         int cpu;
2770
2771         get_online_cpus();
2772         for_each_online_cpu(cpu) {
2773                 p = idle_task(cpu);
2774                 clear_tsk_trace_trace(p);
2775         }
2776         put_online_cpus();
2777 }
2778
2779 static void set_ftrace_swapper(void)
2780 {
2781         struct task_struct *p;
2782         int cpu;
2783
2784         get_online_cpus();
2785         for_each_online_cpu(cpu) {
2786                 p = idle_task(cpu);
2787                 set_tsk_trace_trace(p);
2788         }
2789         put_online_cpus();
2790 }
2791
2792 static void clear_ftrace_pid(struct pid *pid)
2793 {
2794         struct task_struct *p;
2795
2796         rcu_read_lock();
2797         do_each_pid_task(pid, PIDTYPE_PID, p) {
2798                 clear_tsk_trace_trace(p);
2799         } while_each_pid_task(pid, PIDTYPE_PID, p);
2800         rcu_read_unlock();
2801
2802         put_pid(pid);
2803 }
2804
2805 static void set_ftrace_pid(struct pid *pid)
2806 {
2807         struct task_struct *p;
2808
2809         rcu_read_lock();
2810         do_each_pid_task(pid, PIDTYPE_PID, p) {
2811                 set_tsk_trace_trace(p);
2812         } while_each_pid_task(pid, PIDTYPE_PID, p);
2813         rcu_read_unlock();
2814 }
2815
2816 static void clear_ftrace_pid_task(struct pid *pid)
2817 {
2818         if (pid == ftrace_swapper_pid)
2819                 clear_ftrace_swapper();
2820         else
2821                 clear_ftrace_pid(pid);
2822 }
2823
2824 static void set_ftrace_pid_task(struct pid *pid)
2825 {
2826         if (pid == ftrace_swapper_pid)
2827                 set_ftrace_swapper();
2828         else
2829                 set_ftrace_pid(pid);
2830 }
2831
2832 static int ftrace_pid_add(int p)
2833 {
2834         struct pid *pid;
2835         struct ftrace_pid *fpid;
2836         int ret = -EINVAL;
2837
2838         mutex_lock(&ftrace_lock);
2839
2840         if (!p)
2841                 pid = ftrace_swapper_pid;
2842         else
2843                 pid = find_get_pid(p);
2844
2845         if (!pid)
2846                 goto out;
2847
2848         ret = 0;
2849
2850         list_for_each_entry(fpid, &ftrace_pids, list)
2851                 if (fpid->pid == pid)
2852                         goto out_put;
2853
2854         ret = -ENOMEM;
2855
2856         fpid = kmalloc(sizeof(*fpid), GFP_KERNEL);
2857         if (!fpid)
2858                 goto out_put;
2859
2860         list_add(&fpid->list, &ftrace_pids);
2861         fpid->pid = pid;
2862
2863         set_ftrace_pid_task(pid);
2864
2865         ftrace_update_pid_func();
2866         ftrace_startup_enable(0);
2867
2868         mutex_unlock(&ftrace_lock);
2869         return 0;
2870
2871 out_put:
2872         if (pid != ftrace_swapper_pid)
2873                 put_pid(pid);
2874
2875 out:
2876         mutex_unlock(&ftrace_lock);
2877         return ret;
2878 }
2879
2880 static void ftrace_pid_reset(void)
2881 {
2882         struct ftrace_pid *fpid, *safe;
2883
2884         mutex_lock(&ftrace_lock);
2885         list_for_each_entry_safe(fpid, safe, &ftrace_pids, list) {
2886                 struct pid *pid = fpid->pid;
2887
2888                 clear_ftrace_pid_task(pid);
2889
2890                 list_del(&fpid->list);
2891                 kfree(fpid);
2892         }
2893
2894         ftrace_update_pid_func();
2895         ftrace_startup_enable(0);
2896
2897         mutex_unlock(&ftrace_lock);
2898 }
2899
2900 static void *fpid_start(struct seq_file *m, loff_t *pos)
2901 {
2902         mutex_lock(&ftrace_lock);
2903
2904         if (list_empty(&ftrace_pids) && (!*pos))
2905                 return (void *) 1;
2906
2907         return seq_list_start(&ftrace_pids, *pos);
2908 }
2909
2910 static void *fpid_next(struct seq_file *m, void *v, loff_t *pos)
2911 {
2912         if (v == (void *)1)
2913                 return NULL;
2914
2915         return seq_list_next(v, &ftrace_pids, pos);
2916 }
2917
2918 static void fpid_stop(struct seq_file *m, void *p)
2919 {
2920         mutex_unlock(&ftrace_lock);
2921 }
2922
2923 static int fpid_show(struct seq_file *m, void *v)
2924 {
2925         const struct ftrace_pid *fpid = list_entry(v, struct ftrace_pid, list);
2926
2927         if (v == (void *)1) {
2928                 seq_printf(m, "no pid\n");
2929                 return 0;
2930         }
2931
2932         if (fpid->pid == ftrace_swapper_pid)
2933                 seq_printf(m, "swapper tasks\n");
2934         else
2935                 seq_printf(m, "%u\n", pid_vnr(fpid->pid));
2936
2937         return 0;
2938 }
2939
2940 static const struct seq_operations ftrace_pid_sops = {
2941         .start = fpid_start,
2942         .next = fpid_next,
2943         .stop = fpid_stop,
2944         .show = fpid_show,
2945 };
2946
2947 static int
2948 ftrace_pid_open(struct inode *inode, struct file *file)
2949 {
2950         int ret = 0;
2951
2952         if ((file->f_mode & FMODE_WRITE) &&
2953             (file->f_flags & O_TRUNC))
2954                 ftrace_pid_reset();
2955
2956         if (file->f_mode & FMODE_READ)
2957                 ret = seq_open(file, &ftrace_pid_sops);
2958
2959         return ret;
2960 }
2961
2962 static ssize_t
2963 ftrace_pid_write(struct file *filp, const char __user *ubuf,
2964                    size_t cnt, loff_t *ppos)
2965 {
2966         char buf[64];
2967         long val;
2968         int ret;
2969
2970         if (cnt >= sizeof(buf))
2971                 return -EINVAL;
2972
2973         if (copy_from_user(&buf, ubuf, cnt))
2974                 return -EFAULT;
2975
2976         buf[cnt] = 0;
2977
2978         /*
2979          * Allow "echo > set_ftrace_pid" or "echo -n '' > set_ftrace_pid"
2980          * to clean the filter quietly.
2981          */
2982         strstrip(buf);
2983         if (strlen(buf) == 0)
2984                 return 1;
2985
2986         ret = strict_strtol(buf, 10, &val);
2987         if (ret < 0)
2988                 return ret;
2989
2990         ret = ftrace_pid_add(val);
2991
2992         return ret ? ret : cnt;
2993 }
2994
2995 static int
2996 ftrace_pid_release(struct inode *inode, struct file *file)
2997 {
2998         if (file->f_mode & FMODE_READ)
2999                 seq_release(inode, file);
3000
3001         return 0;
3002 }
3003
3004 static const struct file_operations ftrace_pid_fops = {
3005         .open           = ftrace_pid_open,
3006         .write          = ftrace_pid_write,
3007         .read           = seq_read,
3008         .llseek         = seq_lseek,
3009         .release        = ftrace_pid_release,
3010 };
3011
3012 static __init int ftrace_init_debugfs(void)
3013 {
3014         struct dentry *d_tracer;
3015
3016         d_tracer = tracing_init_dentry();
3017         if (!d_tracer)
3018                 return 0;
3019
3020         ftrace_init_dyn_debugfs(d_tracer);
3021
3022         trace_create_file("set_ftrace_pid", 0644, d_tracer,
3023                             NULL, &ftrace_pid_fops);
3024
3025         ftrace_profile_debugfs(d_tracer);
3026
3027         return 0;
3028 }
3029 fs_initcall(ftrace_init_debugfs);
3030
3031 /**
3032  * ftrace_kill - kill ftrace
3033  *
3034  * This function should be used by panic code. It stops ftrace
3035  * but in a not so nice way. If you need to simply kill ftrace
3036  * from a non-atomic section, use ftrace_kill.
3037  */
3038 void ftrace_kill(void)
3039 {
3040         ftrace_disabled = 1;
3041         ftrace_enabled = 0;
3042         clear_ftrace_function();
3043 }
3044
3045 /**
3046  * register_ftrace_function - register a function for profiling
3047  * @ops - ops structure that holds the function for profiling.
3048  *
3049  * Register a function to be called by all functions in the
3050  * kernel.
3051  *
3052  * Note: @ops->func and all the functions it calls must be labeled
3053  *       with "notrace", otherwise it will go into a
3054  *       recursive loop.
3055  */
3056 int register_ftrace_function(struct ftrace_ops *ops)
3057 {
3058         int ret;
3059
3060         if (unlikely(ftrace_disabled))
3061                 return -1;
3062
3063         mutex_lock(&ftrace_lock);
3064
3065         ret = __register_ftrace_function(ops);
3066         ftrace_startup(0);
3067
3068         mutex_unlock(&ftrace_lock);
3069         return ret;
3070 }
3071
3072 /**
3073  * unregister_ftrace_function - unregister a function for profiling.
3074  * @ops - ops structure that holds the function to unregister
3075  *
3076  * Unregister a function that was added to be called by ftrace profiling.
3077  */
3078 int unregister_ftrace_function(struct ftrace_ops *ops)
3079 {
3080         int ret;
3081
3082         mutex_lock(&ftrace_lock);
3083         ret = __unregister_ftrace_function(ops);
3084         ftrace_shutdown(0);
3085         mutex_unlock(&ftrace_lock);
3086
3087         return ret;
3088 }
3089
3090 int
3091 ftrace_enable_sysctl(struct ctl_table *table, int write,
3092                      void __user *buffer, size_t *lenp,
3093                      loff_t *ppos)
3094 {
3095         int ret;
3096
3097         if (unlikely(ftrace_disabled))
3098                 return -ENODEV;
3099
3100         mutex_lock(&ftrace_lock);
3101
3102         ret  = proc_dointvec(table, write, buffer, lenp, ppos);
3103
3104         if (ret || !write || (last_ftrace_enabled == !!ftrace_enabled))
3105                 goto out;
3106
3107         last_ftrace_enabled = !!ftrace_enabled;
3108
3109         if (ftrace_enabled) {
3110
3111                 ftrace_startup_sysctl();
3112
3113                 /* we are starting ftrace again */
3114                 if (ftrace_list != &ftrace_list_end) {
3115                         if (ftrace_list->next == &ftrace_list_end)
3116                                 ftrace_trace_function = ftrace_list->func;
3117                         else
3118                                 ftrace_trace_function = ftrace_list_func;
3119                 }
3120
3121         } else {
3122                 /* stopping ftrace calls (just send to ftrace_stub) */
3123                 ftrace_trace_function = ftrace_stub;
3124
3125                 ftrace_shutdown_sysctl();
3126         }
3127
3128  out:
3129         mutex_unlock(&ftrace_lock);
3130         return ret;
3131 }
3132
3133 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
3134
3135 static int ftrace_graph_active;
3136 static struct notifier_block ftrace_suspend_notifier;
3137
3138 int ftrace_graph_entry_stub(struct ftrace_graph_ent *trace)
3139 {
3140         return 0;
3141 }
3142
3143 /* The callbacks that hook a function */
3144 trace_func_graph_ret_t ftrace_graph_return =
3145                         (trace_func_graph_ret_t)ftrace_stub;
3146 trace_func_graph_ent_t ftrace_graph_entry = ftrace_graph_entry_stub;
3147
3148 /* Try to assign a return stack array on FTRACE_RETSTACK_ALLOC_SIZE tasks. */
3149 static int alloc_retstack_tasklist(struct ftrace_ret_stack **ret_stack_list)
3150 {
3151         int i;
3152         int ret = 0;
3153         unsigned long flags;
3154         int start = 0, end = FTRACE_RETSTACK_ALLOC_SIZE;
3155         struct task_struct *g, *t;
3156
3157         for (i = 0; i < FTRACE_RETSTACK_ALLOC_SIZE; i++) {
3158                 ret_stack_list[i] = kmalloc(FTRACE_RETFUNC_DEPTH
3159                                         * sizeof(struct ftrace_ret_stack),
3160                                         GFP_KERNEL);
3161                 if (!ret_stack_list[i]) {
3162                         start = 0;
3163                         end = i;
3164                         ret = -ENOMEM;
3165                         goto free;
3166                 }
3167         }
3168
3169         read_lock_irqsave(&tasklist_lock, flags);
3170         do_each_thread(g, t) {
3171                 if (start == end) {
3172                         ret = -EAGAIN;
3173                         goto unlock;
3174                 }
3175
3176                 if (t->ret_stack == NULL) {
3177                         atomic_set(&t->tracing_graph_pause, 0);
3178                         atomic_set(&t->trace_overrun, 0);
3179                         t->curr_ret_stack = -1;
3180                         /* Make sure the tasks see the -1 first: */
3181                         smp_wmb();
3182                         t->ret_stack = ret_stack_list[start++];
3183                 }
3184         } while_each_thread(g, t);
3185
3186 unlock:
3187         read_unlock_irqrestore(&tasklist_lock, flags);
3188 free:
3189         for (i = start; i < end; i++)
3190                 kfree(ret_stack_list[i]);
3191         return ret;
3192 }
3193
3194 static void
3195 ftrace_graph_probe_sched_switch(struct rq *__rq, struct task_struct *prev,
3196                                 struct task_struct *next)
3197 {
3198         unsigned long long timestamp;
3199         int index;
3200
3201         /*
3202          * Does the user want to count the time a function was asleep.
3203          * If so, do not update the time stamps.
3204          */
3205         if (trace_flags & TRACE_ITER_SLEEP_TIME)
3206                 return;
3207
3208         timestamp = trace_clock_local();
3209
3210         prev->ftrace_timestamp = timestamp;
3211
3212         /* only process tasks that we timestamped */
3213         if (!next->ftrace_timestamp)
3214                 return;
3215
3216         /*
3217          * Update all the counters in next to make up for the
3218          * time next was sleeping.
3219          */
3220         timestamp -= next->ftrace_timestamp;
3221
3222         for (index = next->curr_ret_stack; index >= 0; index--)
3223                 next->ret_stack[index].calltime += timestamp;
3224 }
3225
3226 /* Allocate a return stack for each task */
3227 static int start_graph_tracing(void)
3228 {
3229         struct ftrace_ret_stack **ret_stack_list;
3230         int ret, cpu;
3231
3232         ret_stack_list = kmalloc(FTRACE_RETSTACK_ALLOC_SIZE *
3233                                 sizeof(struct ftrace_ret_stack *),
3234                                 GFP_KERNEL);
3235
3236         if (!ret_stack_list)
3237                 return -ENOMEM;
3238
3239         /* The cpu_boot init_task->ret_stack will never be freed */
3240         for_each_online_cpu(cpu) {
3241                 if (!idle_task(cpu)->ret_stack)
3242                         ftrace_graph_init_task(idle_task(cpu));
3243         }
3244
3245         do {
3246                 ret = alloc_retstack_tasklist(ret_stack_list);
3247         } while (ret == -EAGAIN);
3248
3249         if (!ret) {
3250                 ret = register_trace_sched_switch(ftrace_graph_probe_sched_switch);
3251                 if (ret)
3252                         pr_info("ftrace_graph: Couldn't activate tracepoint"
3253                                 " probe to kernel_sched_switch\n");
3254         }
3255
3256         kfree(ret_stack_list);
3257         return ret;
3258 }
3259
3260 /*
3261  * Hibernation protection.
3262  * The state of the current task is too much unstable during
3263  * suspend/restore to disk. We want to protect against that.
3264  */
3265 static int
3266 ftrace_suspend_notifier_call(struct notifier_block *bl, unsigned long state,
3267                                                         void *unused)
3268 {
3269         switch (state) {
3270         case PM_HIBERNATION_PREPARE:
3271                 pause_graph_tracing();
3272                 break;
3273
3274         case PM_POST_HIBERNATION:
3275                 unpause_graph_tracing();
3276                 break;
3277         }
3278         return NOTIFY_DONE;
3279 }
3280
3281 int register_ftrace_graph(trace_func_graph_ret_t retfunc,
3282                         trace_func_graph_ent_t entryfunc)
3283 {
3284         int ret = 0;
3285
3286         mutex_lock(&ftrace_lock);
3287
3288         /* we currently allow only one tracer registered at a time */
3289         if (ftrace_graph_active) {
3290                 ret = -EBUSY;
3291                 goto out;
3292         }
3293
3294         ftrace_suspend_notifier.notifier_call = ftrace_suspend_notifier_call;
3295         register_pm_notifier(&ftrace_suspend_notifier);
3296
3297         ftrace_graph_active++;
3298         ret = start_graph_tracing();
3299         if (ret) {
3300                 ftrace_graph_active--;
3301                 goto out;
3302         }
3303
3304         ftrace_graph_return = retfunc;
3305         ftrace_graph_entry = entryfunc;
3306
3307         ftrace_startup(FTRACE_START_FUNC_RET);
3308
3309 out:
3310         mutex_unlock(&ftrace_lock);
3311         return ret;
3312 }
3313
3314 void unregister_ftrace_graph(void)
3315 {
3316         mutex_lock(&ftrace_lock);
3317
3318         if (unlikely(!ftrace_graph_active))
3319                 goto out;
3320
3321         ftrace_graph_active--;
3322         unregister_trace_sched_switch(ftrace_graph_probe_sched_switch);
3323         ftrace_graph_return = (trace_func_graph_ret_t)ftrace_stub;
3324         ftrace_graph_entry = ftrace_graph_entry_stub;
3325         ftrace_shutdown(FTRACE_STOP_FUNC_RET);
3326         unregister_pm_notifier(&ftrace_suspend_notifier);
3327
3328  out:
3329         mutex_unlock(&ftrace_lock);
3330 }
3331
3332 /* Allocate a return stack for newly created task */
3333 void ftrace_graph_init_task(struct task_struct *t)
3334 {
3335         /* Make sure we do not use the parent ret_stack */
3336         t->ret_stack = NULL;
3337
3338         if (ftrace_graph_active) {
3339                 struct ftrace_ret_stack *ret_stack;
3340
3341                 ret_stack = kmalloc(FTRACE_RETFUNC_DEPTH
3342                                 * sizeof(struct ftrace_ret_stack),
3343                                 GFP_KERNEL);
3344                 if (!ret_stack)
3345                         return;
3346                 t->curr_ret_stack = -1;
3347                 atomic_set(&t->tracing_graph_pause, 0);
3348                 atomic_set(&t->trace_overrun, 0);
3349                 t->ftrace_timestamp = 0;
3350                 /* make curr_ret_stack visable before we add the ret_stack */
3351                 smp_wmb();
3352                 t->ret_stack = ret_stack;
3353         }
3354 }
3355
3356 void ftrace_graph_exit_task(struct task_struct *t)
3357 {
3358         struct ftrace_ret_stack *ret_stack = t->ret_stack;
3359
3360         t->ret_stack = NULL;
3361         /* NULL must become visible to IRQs before we free it: */
3362         barrier();
3363
3364         kfree(ret_stack);
3365 }
3366
3367 void ftrace_graph_stop(void)
3368 {
3369         ftrace_stop();
3370 }
3371 #endif
3372