SAFE public projects git trees. - safe/jmp/linux-2.6/blob - kernel/trace/ftrace.c

   1 /*
   2  * Infrastructure for profiling code inserted by 'gcc -pg'.
   3  *
   4  * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
   5  * Copyright (C) 2004-2008 Ingo Molnar <mingo@redhat.com>
   6  *
   7  * Originally ported from the -rt patch by:
   8  *   Copyright (C) 2007 Arnaldo Carvalho de Melo <acme@redhat.com>
   9  *
  10  * Based on code in the latency_tracer, that is:
  11  *
  12  *  Copyright (C) 2004-2006 Ingo Molnar
  13  *  Copyright (C) 2004 William Lee Irwin III
  14  */
  15
  16 #include <linux/stop_machine.h>
  17 #include <linux/clocksource.h>
  18 #include <linux/kallsyms.h>
  19 #include <linux/seq_file.h>
  20 #include <linux/suspend.h>
  21 #include <linux/debugfs.h>
  22 #include <linux/hardirq.h>
  23 #include <linux/kthread.h>
  24 #include <linux/uaccess.h>
  25 #include <linux/ftrace.h>
  26 #include <linux/sysctl.h>
  27 #include <linux/ctype.h>
  28 #include <linux/list.h>
  29 #include <linux/hash.h>
  30 #include <linux/rcupdate.h>
  31
  32 #include <trace/events/sched.h>
  33
  34 #include <asm/ftrace.h>
  35 #include <asm/setup.h>
  36
  37 #include "trace_output.h"
  38 #include "trace_stat.h"
  39
  40 #define FTRACE_WARN_ON(cond)                    \
  41         do {                                    \
  42                 if (WARN_ON(cond))              \
  43                         ftrace_kill();          \
  44         } while (0)
  45
  46 #define FTRACE_WARN_ON_ONCE(cond)               \
  47         do {                                    \
  48                 if (WARN_ON_ONCE(cond))         \
  49                         ftrace_kill();          \
  50         } while (0)
  51
  52 /* hash bits for specific function selection */
  53 #define FTRACE_HASH_BITS 7
  54 #define FTRACE_FUNC_HASHSIZE (1 << FTRACE_HASH_BITS)
  55
  56 /* ftrace_enabled is a method to turn ftrace on or off */
  57 int ftrace_enabled __read_mostly;
  58 static int last_ftrace_enabled;
  59
  60 /* Quick disabling of function tracer. */
  61 int function_trace_stop;
  62
  63 /* List for set_ftrace_pid's pids. */
  64 LIST_HEAD(ftrace_pids);
  65 struct ftrace_pid {
  66         struct list_head list;
  67         struct pid *pid;
  68 };
  69
  70 /*
  71  * ftrace_disabled is set when an anomaly is discovered.
  72  * ftrace_disabled is much stronger than ftrace_enabled.
  73  */
  74 static int ftrace_disabled __read_mostly;
  75
  76 static DEFINE_MUTEX(ftrace_lock);
  77
  78 static struct ftrace_ops ftrace_list_end __read_mostly =
  79 {
  80         .func           = ftrace_stub,
  81 };
  82
  83 static struct ftrace_ops *ftrace_list __read_mostly = &ftrace_list_end;
  84 ftrace_func_t ftrace_trace_function __read_mostly = ftrace_stub;
  85 ftrace_func_t __ftrace_trace_function __read_mostly = ftrace_stub;
  86 ftrace_func_t ftrace_pid_function __read_mostly = ftrace_stub;
  87
  88 /*
  89  * Traverse the ftrace_list, invoking all entries.  The reason that we
  90  * can use rcu_dereference_raw() is that elements removed from this list
  91  * are simply leaked, so there is no need to interact with a grace-period
  92  * mechanism.  The rcu_dereference_raw() calls are needed to handle
  93  * concurrent insertions into the ftrace_list.
  94  *
  95  * Silly Alpha and silly pointer-speculation compiler optimizations!
  96  */
  97 static void ftrace_list_func(unsigned long ip, unsigned long parent_ip)
  98 {
  99         struct ftrace_ops *op = rcu_dereference_raw(ftrace_list); /*see above*/
 100
 101         while (op != &ftrace_list_end) {
 102                 op->func(ip, parent_ip);
 103                 op = rcu_dereference_raw(op->next); /*see above*/
 104         };
 105 }
 106
 107 static void ftrace_pid_func(unsigned long ip, unsigned long parent_ip)
 108 {
 109         if (!test_tsk_trace_trace(current))
 110                 return;
 111
 112         ftrace_pid_function(ip, parent_ip);
 113 }
 114
 115 static void set_ftrace_pid_function(ftrace_func_t func)
 116 {
 117         /* do not set ftrace_pid_function to itself! */
 118         if (func != ftrace_pid_func)
 119                 ftrace_pid_function = func;
 120 }
 121
 122 /**
 123  * clear_ftrace_function - reset the ftrace function
 124  *
 125  * This NULLs the ftrace function and in essence stops
 126  * tracing.  There may be lag
 127  */
 128 void clear_ftrace_function(void)
 129 {
 130         ftrace_trace_function = ftrace_stub;
 131         __ftrace_trace_function = ftrace_stub;
 132         ftrace_pid_function = ftrace_stub;
 133 }
 134
 135 #ifndef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST
 136 /*
 137  * For those archs that do not test ftrace_trace_stop in their
 138  * mcount call site, we need to do it from C.
 139  */
 140 static void ftrace_test_stop_func(unsigned long ip, unsigned long parent_ip)
 141 {
 142         if (function_trace_stop)
 143                 return;
 144
 145         __ftrace_trace_function(ip, parent_ip);
 146 }
 147 #endif
 148
 149 static int __register_ftrace_function(struct ftrace_ops *ops)
 150 {
 151         ops->next = ftrace_list;
 152         /*
 153          * We are entering ops into the ftrace_list but another
 154          * CPU might be walking that list. We need to make sure
 155          * the ops->next pointer is valid before another CPU sees
 156          * the ops pointer included into the ftrace_list.
 157          */
 158         rcu_assign_pointer(ftrace_list, ops);
 159
 160         if (ftrace_enabled) {
 161                 ftrace_func_t func;
 162
 163                 if (ops->next == &ftrace_list_end)
 164                         func = ops->func;
 165                 else
 166                         func = ftrace_list_func;
 167
 168                 if (!list_empty(&ftrace_pids)) {
 169                         set_ftrace_pid_function(func);
 170                         func = ftrace_pid_func;
 171                 }
 172
 173                 /*
 174                  * For one func, simply call it directly.
 175                  * For more than one func, call the chain.
 176                  */
 177 #ifdef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST
 178                 ftrace_trace_function = func;
 179 #else
 180                 __ftrace_trace_function = func;
 181                 ftrace_trace_function = ftrace_test_stop_func;
 182 #endif
 183         }
 184
 185         return 0;
 186 }
 187
 188 static int __unregister_ftrace_function(struct ftrace_ops *ops)
 189 {
 190         struct ftrace_ops **p;
 191
 192         /*
 193          * If we are removing the last function, then simply point
 194          * to the ftrace_stub.
 195          */
 196         if (ftrace_list == ops && ops->next == &ftrace_list_end) {
 197                 ftrace_trace_function = ftrace_stub;
 198                 ftrace_list = &ftrace_list_end;
 199                 return 0;
 200         }
 201
 202         for (p = &ftrace_list; *p != &ftrace_list_end; p = &(*p)->next)
 203                 if (*p == ops)
 204                         break;
 205
 206         if (*p != ops)
 207                 return -1;
 208
 209         *p = (*p)->next;
 210
 211         if (ftrace_enabled) {
 212                 /* If we only have one func left, then call that directly */
 213                 if (ftrace_list->next == &ftrace_list_end) {
 214                         ftrace_func_t func = ftrace_list->func;
 215
 216                         if (!list_empty(&ftrace_pids)) {
 217                                 set_ftrace_pid_function(func);
 218                                 func = ftrace_pid_func;
 219                         }
 220 #ifdef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST
 221                         ftrace_trace_function = func;
 222 #else
 223                         __ftrace_trace_function = func;
 224 #endif
 225                 }
 226         }
 227
 228         return 0;
 229 }
 230
 231 static void ftrace_update_pid_func(void)
 232 {
 233         ftrace_func_t func;
 234
 235         if (ftrace_trace_function == ftrace_stub)
 236                 return;
 237
 238 #ifdef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST
 239         func = ftrace_trace_function;
 240 #else
 241         func = __ftrace_trace_function;
 242 #endif
 243
 244         if (!list_empty(&ftrace_pids)) {
 245                 set_ftrace_pid_function(func);
 246                 func = ftrace_pid_func;
 247         } else {
 248                 if (func == ftrace_pid_func)
 249                         func = ftrace_pid_function;
 250         }
 251
 252 #ifdef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST
 253         ftrace_trace_function = func;
 254 #else
 255         __ftrace_trace_function = func;
 256 #endif
 257 }
 258
 259 #ifdef CONFIG_FUNCTION_PROFILER
 260 struct ftrace_profile {
 261         struct hlist_node               node;
 262         unsigned long                   ip;
 263         unsigned long                   counter;
 264 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
 265         unsigned long long              time;
 266 #endif
 267 };
 268
 269 struct ftrace_profile_page {
 270         struct ftrace_profile_page      *next;
 271         unsigned long                   index;
 272         struct ftrace_profile           records[];
 273 };
 274
 275 struct ftrace_profile_stat {
 276         atomic_t                        disabled;
 277         struct hlist_head               *hash;
 278         struct ftrace_profile_page      *pages;
 279         struct ftrace_profile_page      *start;
 280         struct tracer_stat              stat;
 281 };
 282
 283 #define PROFILE_RECORDS_SIZE                                            \
 284         (PAGE_SIZE - offsetof(struct ftrace_profile_page, records))
 285
 286 #define PROFILES_PER_PAGE                                       \
 287         (PROFILE_RECORDS_SIZE / sizeof(struct ftrace_profile))
 288
 289 static int ftrace_profile_bits __read_mostly;
 290 static int ftrace_profile_enabled __read_mostly;
 291
 292 /* ftrace_profile_lock - synchronize the enable and disable of the profiler */
 293 static DEFINE_MUTEX(ftrace_profile_lock);
 294
 295 static DEFINE_PER_CPU(struct ftrace_profile_stat, ftrace_profile_stats);
 296
 297 #define FTRACE_PROFILE_HASH_SIZE 1024 /* must be power of 2 */
 298
 299 static void *
 300 function_stat_next(void *v, int idx)
 301 {
 302         struct ftrace_profile *rec = v;
 303         struct ftrace_profile_page *pg;
 304
 305         pg = (struct ftrace_profile_page *)((unsigned long)rec & PAGE_MASK);
 306
 307  again:
 308         if (idx != 0)
 309                 rec++;
 310
 311         if ((void *)rec >= (void *)&pg->records[pg->index]) {
 312                 pg = pg->next;
 313                 if (!pg)
 314                         return NULL;
 315                 rec = &pg->records[0];
 316                 if (!rec->counter)
 317                         goto again;
 318         }
 319
 320         return rec;
 321 }
 322
 323 static void *function_stat_start(struct tracer_stat *trace)
 324 {
 325         struct ftrace_profile_stat *stat =
 326                 container_of(trace, struct ftrace_profile_stat, stat);
 327
 328         if (!stat || !stat->start)
 329                 return NULL;
 330
 331         return function_stat_next(&stat->start->records[0], 0);
 332 }
 333
 334 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
 335 /* function graph compares on total time */
 336 static int function_stat_cmp(void *p1, void *p2)
 337 {
 338         struct ftrace_profile *a = p1;
 339         struct ftrace_profile *b = p2;
 340
 341         if (a->time < b->time)
 342                 return -1;
 343         if (a->time > b->time)
 344                 return 1;
 345         else
 346                 return 0;
 347 }
 348 #else
 349 /* not function graph compares against hits */
 350 static int function_stat_cmp(void *p1, void *p2)
 351 {
 352         struct ftrace_profile *a = p1;
 353         struct ftrace_profile *b = p2;
 354
 355         if (a->counter < b->counter)
 356                 return -1;
 357         if (a->counter > b->counter)
 358                 return 1;
 359         else
 360                 return 0;
 361 }
 362 #endif
 363
 364 static int function_stat_headers(struct seq_file *m)
 365 {
 366 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
 367         seq_printf(m, "  Function                               "
 368                    "Hit    Time            Avg\n"
 369                       "  --------                               "
 370                    "---    ----            ---\n");
 371 #else
 372         seq_printf(m, "  Function                               Hit\n"
 373                       "  --------                               ---\n");
 374 #endif
 375         return 0;
 376 }
 377
 378 static int function_stat_show(struct seq_file *m, void *v)
 379 {
 380         struct ftrace_profile *rec = v;
 381         char str[KSYM_SYMBOL_LEN];
 382 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
 383         static DEFINE_MUTEX(mutex);
 384         static struct trace_seq s;
 385         unsigned long long avg;
 386 #endif
 387
 388         kallsyms_lookup(rec->ip, NULL, NULL, NULL, str);
 389         seq_printf(m, "  %-30.30s  %10lu", str, rec->counter);
 390
 391 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
 392         seq_printf(m, "    ");
 393         avg = rec->time;
 394         do_div(avg, rec->counter);
 395
 396         mutex_lock(&mutex);
 397         trace_seq_init(&s);
 398         trace_print_graph_duration(rec->time, &s);
 399         trace_seq_puts(&s, "    ");
 400         trace_print_graph_duration(avg, &s);
 401         trace_print_seq(m, &s);
 402         mutex_unlock(&mutex);
 403 #endif
 404         seq_putc(m, '\n');
 405
 406         return 0;
 407 }
 408
 409 static void ftrace_profile_reset(struct ftrace_profile_stat *stat)
 410 {
 411         struct ftrace_profile_page *pg;
 412
 413         pg = stat->pages = stat->start;
 414
 415         while (pg) {
 416                 memset(pg->records, 0, PROFILE_RECORDS_SIZE);
 417                 pg->index = 0;
 418                 pg = pg->next;
 419         }
 420
 421         memset(stat->hash, 0,
 422                FTRACE_PROFILE_HASH_SIZE * sizeof(struct hlist_head));
 423 }
 424
 425 int ftrace_profile_pages_init(struct ftrace_profile_stat *stat)
 426 {
 427         struct ftrace_profile_page *pg;
 428         int functions;
 429         int pages;
 430         int i;
 431
 432         /* If we already allocated, do nothing */
 433         if (stat->pages)
 434                 return 0;
 435
 436         stat->pages = (void *)get_zeroed_page(GFP_KERNEL);
 437         if (!stat->pages)
 438                 return -ENOMEM;
 439
 440 #ifdef CONFIG_DYNAMIC_FTRACE
 441         functions = ftrace_update_tot_cnt;
 442 #else
 443         /*
 444          * We do not know the number of functions that exist because
 445          * dynamic tracing is what counts them. With past experience
 446          * we have around 20K functions. That should be more than enough.
 447          * It is highly unlikely we will execute every function in
 448          * the kernel.
 449          */
 450         functions = 20000;
 451 #endif
 452
 453         pg = stat->start = stat->pages;
 454
 455         pages = DIV_ROUND_UP(functions, PROFILES_PER_PAGE);
 456
 457         for (i = 0; i < pages; i++) {
 458                 pg->next = (void *)get_zeroed_page(GFP_KERNEL);
 459                 if (!pg->next)
 460                         goto out_free;
 461                 pg = pg->next;
 462         }
 463
 464         return 0;
 465
 466  out_free:
 467         pg = stat->start;
 468         while (pg) {
 469                 unsigned long tmp = (unsigned long)pg;
 470
 471                 pg = pg->next;
 472                 free_page(tmp);
 473         }
 474
 475         free_page((unsigned long)stat->pages);
 476         stat->pages = NULL;
 477         stat->start = NULL;
 478
 479         return -ENOMEM;
 480 }
 481
 482 static int ftrace_profile_init_cpu(int cpu)
 483 {
 484         struct ftrace_profile_stat *stat;
 485         int size;
 486
 487         stat = &per_cpu(ftrace_profile_stats, cpu);
 488
 489         if (stat->hash) {
 490                 /* If the profile is already created, simply reset it */
 491                 ftrace_profile_reset(stat);
 492                 return 0;
 493         }
 494
 495         /*
 496          * We are profiling all functions, but usually only a few thousand
 497          * functions are hit. We'll make a hash of 1024 items.
 498          */
 499         size = FTRACE_PROFILE_HASH_SIZE;
 500
 501         stat->hash = kzalloc(sizeof(struct hlist_head) * size, GFP_KERNEL);
 502
 503         if (!stat->hash)
 504                 return -ENOMEM;
 505
 506         if (!ftrace_profile_bits) {
 507                 size--;
 508
 509                 for (; size; size >>= 1)
 510                         ftrace_profile_bits++;
 511         }
 512
 513         /* Preallocate the function profiling pages */
 514         if (ftrace_profile_pages_init(stat) < 0) {
 515                 kfree(stat->hash);
 516                 stat->hash = NULL;
 517                 return -ENOMEM;
 518         }
 519
 520         return 0;
 521 }
 522
 523 static int ftrace_profile_init(void)
 524 {
 525         int cpu;
 526         int ret = 0;
 527
 528         for_each_online_cpu(cpu) {
 529                 ret = ftrace_profile_init_cpu(cpu);
 530                 if (ret)
 531                         break;
 532         }
 533
 534         return ret;
 535 }
 536
 537 /* interrupts must be disabled */
 538 static struct ftrace_profile *
 539 ftrace_find_profiled_func(struct ftrace_profile_stat *stat, unsigned long ip)
 540 {
 541         struct ftrace_profile *rec;
 542         struct hlist_head *hhd;
 543         struct hlist_node *n;
 544         unsigned long key;
 545
 546         key = hash_long(ip, ftrace_profile_bits);
 547         hhd = &stat->hash[key];
 548
 549         if (hlist_empty(hhd))
 550                 return NULL;
 551
 552         hlist_for_each_entry_rcu(rec, n, hhd, node) {
 553                 if (rec->ip == ip)
 554                         return rec;
 555         }
 556
 557         return NULL;
 558 }
 559
 560 static void ftrace_add_profile(struct ftrace_profile_stat *stat,
 561                                struct ftrace_profile *rec)
 562 {
 563         unsigned long key;
 564
 565         key = hash_long(rec->ip, ftrace_profile_bits);
 566         hlist_add_head_rcu(&rec->node, &stat->hash[key]);
 567 }
 568
 569 /*
 570  * The memory is already allocated, this simply finds a new record to use.
 571  */
 572 static struct ftrace_profile *
 573 ftrace_profile_alloc(struct ftrace_profile_stat *stat, unsigned long ip)
 574 {
 575         struct ftrace_profile *rec = NULL;
 576
 577         /* prevent recursion (from NMIs) */
 578         if (atomic_inc_return(&stat->disabled) != 1)
 579                 goto out;
 580
 581         /*
 582          * Try to find the function again since an NMI
 583          * could have added it
 584          */
 585         rec = ftrace_find_profiled_func(stat, ip);
 586         if (rec)
 587                 goto out;
 588
 589         if (stat->pages->index == PROFILES_PER_PAGE) {
 590                 if (!stat->pages->next)
 591                         goto out;
 592                 stat->pages = stat->pages->next;
 593         }
 594
 595         rec = &stat->pages->records[stat->pages->index++];
 596         rec->ip = ip;
 597         ftrace_add_profile(stat, rec);
 598
 599  out:
 600         atomic_dec(&stat->disabled);
 601
 602         return rec;
 603 }
 604
 605 static void
 606 function_profile_call(unsigned long ip, unsigned long parent_ip)
 607 {
 608         struct ftrace_profile_stat *stat;
 609         struct ftrace_profile *rec;
 610         unsigned long flags;
 611
 612         if (!ftrace_profile_enabled)
 613                 return;
 614
 615         local_irq_save(flags);
 616
 617         stat = &__get_cpu_var(ftrace_profile_stats);
 618         if (!stat->hash || !ftrace_profile_enabled)
 619                 goto out;
 620
 621         rec = ftrace_find_profiled_func(stat, ip);
 622         if (!rec) {
 623                 rec = ftrace_profile_alloc(stat, ip);
 624                 if (!rec)
 625                         goto out;
 626         }
 627
 628         rec->counter++;
 629  out:
 630         local_irq_restore(flags);
 631 }
 632
 633 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
 634 static int profile_graph_entry(struct ftrace_graph_ent *trace)
 635 {
 636         function_profile_call(trace->func, 0);
 637         return 1;
 638 }
 639
 640 static void profile_graph_return(struct ftrace_graph_ret *trace)
 641 {
 642         struct ftrace_profile_stat *stat;
 643         unsigned long long calltime;
 644         struct ftrace_profile *rec;
 645         unsigned long flags;
 646
 647         local_irq_save(flags);
 648         stat = &__get_cpu_var(ftrace_profile_stats);
 649         if (!stat->hash || !ftrace_profile_enabled)
 650                 goto out;
 651
 652         calltime = trace->rettime - trace->calltime;
 653
 654         if (!(trace_flags & TRACE_ITER_GRAPH_TIME)) {
 655                 int index;
 656
 657                 index = trace->depth;
 658
 659                 /* Append this call time to the parent time to subtract */
 660                 if (index)
 661                         current->ret_stack[index - 1].subtime += calltime;
 662
 663                 if (current->ret_stack[index].subtime < calltime)
 664                         calltime -= current->ret_stack[index].subtime;
 665                 else
 666                         calltime = 0;
 667         }
 668
 669         rec = ftrace_find_profiled_func(stat, trace->func);
 670         if (rec)
 671                 rec->time += calltime;
 672
 673  out:
 674         local_irq_restore(flags);
 675 }
 676
 677 static int register_ftrace_profiler(void)
 678 {
 679         return register_ftrace_graph(&profile_graph_return,
 680                                      &profile_graph_entry);
 681 }
 682
 683 static void unregister_ftrace_profiler(void)
 684 {
 685         unregister_ftrace_graph();
 686 }
 687 #else
 688 static struct ftrace_ops ftrace_profile_ops __read_mostly =
 689 {
 690         .func           = function_profile_call,
 691 };
 692
 693 static int register_ftrace_profiler(void)
 694 {
 695         return register_ftrace_function(&ftrace_profile_ops);
 696 }
 697
 698 static void unregister_ftrace_profiler(void)
 699 {
 700         unregister_ftrace_function(&ftrace_profile_ops);
 701 }
 702 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
 703
 704 static ssize_t
 705 ftrace_profile_write(struct file *filp, const char __user *ubuf,
 706                      size_t cnt, loff_t *ppos)
 707 {
 708         unsigned long val;
 709         char buf[64];           /* big enough to hold a number */
 710         int ret;
 711
 712         if (cnt >= sizeof(buf))
 713                 return -EINVAL;
 714
 715         if (copy_from_user(&buf, ubuf, cnt))
 716                 return -EFAULT;
 717
 718         buf[cnt] = 0;
 719
 720         ret = strict_strtoul(buf, 10, &val);
 721         if (ret < 0)
 722                 return ret;
 723
 724         val = !!val;
 725
 726         mutex_lock(&ftrace_profile_lock);
 727         if (ftrace_profile_enabled ^ val) {
 728                 if (val) {
 729                         ret = ftrace_profile_init();
 730                         if (ret < 0) {
 731                                 cnt = ret;
 732                                 goto out;
 733                         }
 734
 735                         ret = register_ftrace_profiler();
 736                         if (ret < 0) {
 737                                 cnt = ret;
 738                                 goto out;
 739                         }
 740                         ftrace_profile_enabled = 1;
 741                 } else {
 742                         ftrace_profile_enabled = 0;
 743                         /*
 744                          * unregister_ftrace_profiler calls stop_machine
 745                          * so this acts like an synchronize_sched.
 746                          */
 747                         unregister_ftrace_profiler();
 748                 }
 749         }
 750  out:
 751         mutex_unlock(&ftrace_profile_lock);
 752
 753         *ppos += cnt;
 754
 755         return cnt;
 756 }
 757
 758 static ssize_t
 759 ftrace_profile_read(struct file *filp, char __user *ubuf,
 760                      size_t cnt, loff_t *ppos)
 761 {
 762         char buf[64];           /* big enough to hold a number */
 763         int r;
 764
 765         r = sprintf(buf, "%u\n", ftrace_profile_enabled);
 766         return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
 767 }
 768
 769 static const struct file_operations ftrace_profile_fops = {
 770         .open           = tracing_open_generic,
 771         .read           = ftrace_profile_read,
 772         .write          = ftrace_profile_write,
 773 };
 774
 775 /* used to initialize the real stat files */
 776 static struct tracer_stat function_stats __initdata = {
 777         .name           = "functions",
 778         .stat_start     = function_stat_start,
 779         .stat_next      = function_stat_next,
 780         .stat_cmp       = function_stat_cmp,
 781         .stat_headers   = function_stat_headers,
 782         .stat_show      = function_stat_show
 783 };
 784
 785 static __init void ftrace_profile_debugfs(struct dentry *d_tracer)
 786 {
 787         struct ftrace_profile_stat *stat;
 788         struct dentry *entry;
 789         char *name;
 790         int ret;
 791         int cpu;
 792
 793         for_each_possible_cpu(cpu) {
 794                 stat = &per_cpu(ftrace_profile_stats, cpu);
 795
 796                 /* allocate enough for function name + cpu number */
 797                 name = kmalloc(32, GFP_KERNEL);
 798                 if (!name) {
 799                         /*
 800                          * The files created are permanent, if something happens
 801                          * we still do not free memory.
 802                          */
 803                         WARN(1,
 804                              "Could not allocate stat file for cpu %d\n",
 805                              cpu);
 806                         return;
 807                 }
 808                 stat->stat = function_stats;
 809                 snprintf(name, 32, "function%d", cpu);
 810                 stat->stat.name = name;
 811                 ret = register_stat_tracer(&stat->stat);
 812                 if (ret) {
 813                         WARN(1,
 814                              "Could not register function stat for cpu %d\n",
 815                              cpu);
 816                         kfree(name);
 817                         return;
 818                 }
 819         }
 820
 821         entry = debugfs_create_file("function_profile_enabled", 0644,
 822                                     d_tracer, NULL, &ftrace_profile_fops);
 823         if (!entry)
 824                 pr_warning("Could not create debugfs "
 825                            "'function_profile_enabled' entry\n");
 826 }
 827
 828 #else /* CONFIG_FUNCTION_PROFILER */
 829 static __init void ftrace_profile_debugfs(struct dentry *d_tracer)
 830 {
 831 }
 832 #endif /* CONFIG_FUNCTION_PROFILER */
 833
 834 static struct pid * const ftrace_swapper_pid = &init_struct_pid;
 835
 836 #ifdef CONFIG_DYNAMIC_FTRACE
 837
 838 #ifndef CONFIG_FTRACE_MCOUNT_RECORD
 839 # error Dynamic ftrace depends on MCOUNT_RECORD
 840 #endif
 841
 842 static struct hlist_head ftrace_func_hash[FTRACE_FUNC_HASHSIZE] __read_mostly;
 843
 844 struct ftrace_func_probe {
 845         struct hlist_node       node;
 846         struct ftrace_probe_ops *ops;
 847         unsigned long           flags;
 848         unsigned long           ip;
 849         void                    *data;
 850         struct rcu_head         rcu;
 851 };
 852
 853 enum {
 854         FTRACE_ENABLE_CALLS             = (1 << 0),
 855         FTRACE_DISABLE_CALLS            = (1 << 1),
 856         FTRACE_UPDATE_TRACE_FUNC        = (1 << 2),
 857         FTRACE_ENABLE_MCOUNT            = (1 << 3),
 858         FTRACE_DISABLE_MCOUNT           = (1 << 4),
 859         FTRACE_START_FUNC_RET           = (1 << 5),
 860         FTRACE_STOP_FUNC_RET            = (1 << 6),
 861 };
 862
 863 static int ftrace_filtered;
 864
 865 static struct dyn_ftrace *ftrace_new_addrs;
 866
 867 static DEFINE_MUTEX(ftrace_regex_lock);
 868
 869 struct ftrace_page {
 870         struct ftrace_page      *next;
 871         int                     index;
 872         struct dyn_ftrace       records[];
 873 };
 874
 875 #define ENTRIES_PER_PAGE \
 876   ((PAGE_SIZE - sizeof(struct ftrace_page)) / sizeof(struct dyn_ftrace))
 877
 878 /* estimate from running different kernels */
 879 #define NR_TO_INIT              10000
 880
 881 static struct ftrace_page       *ftrace_pages_start;
 882 static struct ftrace_page       *ftrace_pages;
 883
 884 static struct dyn_ftrace *ftrace_free_records;
 885
 886 /*
 887  * This is a double for. Do not use 'break' to break out of the loop,
 888  * you must use a goto.
 889  */
 890 #define do_for_each_ftrace_rec(pg, rec)                                 \
 891         for (pg = ftrace_pages_start; pg; pg = pg->next) {              \
 892                 int _____i;                                             \
 893                 for (_____i = 0; _____i < pg->index; _____i++) {        \
 894                         rec = &pg->records[_____i];
 895
 896 #define while_for_each_ftrace_rec()             \
 897                 }                               \
 898         }
 899
 900 static void ftrace_free_rec(struct dyn_ftrace *rec)
 901 {
 902         rec->freelist = ftrace_free_records;
 903         ftrace_free_records = rec;
 904         rec->flags |= FTRACE_FL_FREE;
 905 }
 906
 907 static struct dyn_ftrace *ftrace_alloc_dyn_node(unsigned long ip)
 908 {
 909         struct dyn_ftrace *rec;
 910
 911         /* First check for freed records */
 912         if (ftrace_free_records) {
 913                 rec = ftrace_free_records;
 914
 915                 if (unlikely(!(rec->flags & FTRACE_FL_FREE))) {
 916                         FTRACE_WARN_ON_ONCE(1);
 917                         ftrace_free_records = NULL;
 918                         return NULL;
 919                 }
 920
 921                 ftrace_free_records = rec->freelist;
 922                 memset(rec, 0, sizeof(*rec));
 923                 return rec;
 924         }
 925
 926         if (ftrace_pages->index == ENTRIES_PER_PAGE) {
 927                 if (!ftrace_pages->next) {
 928                         /* allocate another page */
 929                         ftrace_pages->next =
 930                                 (void *)get_zeroed_page(GFP_KERNEL);
 931                         if (!ftrace_pages->next)
 932                                 return NULL;
 933                 }
 934                 ftrace_pages = ftrace_pages->next;
 935         }
 936
 937         return &ftrace_pages->records[ftrace_pages->index++];
 938 }
 939
 940 static struct dyn_ftrace *
 941 ftrace_record_ip(unsigned long ip)
 942 {
 943         struct dyn_ftrace *rec;
 944
 945         if (ftrace_disabled)
 946                 return NULL;
 947
 948         rec = ftrace_alloc_dyn_node(ip);
 949         if (!rec)
 950                 return NULL;
 951
 952         rec->ip = ip;
 953         rec->newlist = ftrace_new_addrs;
 954         ftrace_new_addrs = rec;
 955
 956         return rec;
 957 }
 958
 959 static void print_ip_ins(const char *fmt, unsigned char *p)
 960 {
 961         int i;
 962
 963         printk(KERN_CONT "%s", fmt);
 964
 965         for (i = 0; i < MCOUNT_INSN_SIZE; i++)
 966                 printk(KERN_CONT "%s%02x", i ? ":" : "", p[i]);
 967 }
 968
 969 static void ftrace_bug(int failed, unsigned long ip)
 970 {
 971         switch (failed) {
 972         case -EFAULT:
 973                 FTRACE_WARN_ON_ONCE(1);
 974                 pr_info("ftrace faulted on modifying ");
 975                 print_ip_sym(ip);
 976                 break;
 977         case -EINVAL:
 978                 FTRACE_WARN_ON_ONCE(1);
 979                 pr_info("ftrace failed to modify ");
 980                 print_ip_sym(ip);
 981                 print_ip_ins(" actual: ", (unsigned char *)ip);
 982                 printk(KERN_CONT "\n");
 983                 break;
 984         case -EPERM:
 985                 FTRACE_WARN_ON_ONCE(1);
 986                 pr_info("ftrace faulted on writing ");
 987                 print_ip_sym(ip);
 988                 break;
 989         default:
 990                 FTRACE_WARN_ON_ONCE(1);
 991                 pr_info("ftrace faulted on unknown error ");
 992                 print_ip_sym(ip);
 993         }
 994 }
 995
 996
 997 /* Return 1 if the address range is reserved for ftrace */
 998 int ftrace_text_reserved(void *start, void *end)
 999 {
1000         struct dyn_ftrace *rec;
1001         struct ftrace_page *pg;
1002
1003         do_for_each_ftrace_rec(pg, rec) {
1004                 if (rec->ip <= (unsigned long)end &&
1005                     rec->ip + MCOUNT_INSN_SIZE > (unsigned long)start)
1006                         return 1;
1007         } while_for_each_ftrace_rec();
1008         return 0;
1009 }
1010
1011
1012 static int
1013 __ftrace_replace_code(struct dyn_ftrace *rec, int enable)
1014 {
1015         unsigned long ftrace_addr;
1016         unsigned long flag = 0UL;
1017
1018         ftrace_addr = (unsigned long)FTRACE_ADDR;
1019
1020         /*
1021          * If this record is not to be traced or we want to disable it,
1022          * then disable it.
1023          *
1024          * If we want to enable it and filtering is off, then enable it.
1025          *
1026          * If we want to enable it and filtering is on, enable it only if
1027          * it's filtered
1028          */
1029         if (enable && !(rec->flags & FTRACE_FL_NOTRACE)) {
1030                 if (!ftrace_filtered || (rec->flags & FTRACE_FL_FILTER))
1031                         flag = FTRACE_FL_ENABLED;
1032         }
1033
1034         /* If the state of this record hasn't changed, then do nothing */
1035         if ((rec->flags & FTRACE_FL_ENABLED) == flag)
1036                 return 0;
1037
1038         if (flag) {
1039                 rec->flags |= FTRACE_FL_ENABLED;
1040                 return ftrace_make_call(rec, ftrace_addr);
1041         }
1042
1043         rec->flags &= ~FTRACE_FL_ENABLED;
1044         return ftrace_make_nop(NULL, rec, ftrace_addr);
1045 }
1046
1047 static void ftrace_replace_code(int enable)
1048 {
1049         struct dyn_ftrace *rec;
1050         struct ftrace_page *pg;
1051         int failed;
1052
1053         do_for_each_ftrace_rec(pg, rec) {
1054                 /*
1055                  * Skip over free records, records that have
1056                  * failed and not converted.
1057                  */
1058                 if (rec->flags & FTRACE_FL_FREE ||
1059                     rec->flags & FTRACE_FL_FAILED ||
1060                     !(rec->flags & FTRACE_FL_CONVERTED))
1061                         continue;
1062
1063                 failed = __ftrace_replace_code(rec, enable);
1064                 if (failed) {
1065                         rec->flags |= FTRACE_FL_FAILED;
1066                         ftrace_bug(failed, rec->ip);
1067                         /* Stop processing */
1068                         return;
1069                 }
1070         } while_for_each_ftrace_rec();
1071 }
1072
1073 static int
1074 ftrace_code_disable(struct module *mod, struct dyn_ftrace *rec)
1075 {
1076         unsigned long ip;
1077         int ret;
1078
1079         ip = rec->ip;
1080
1081         ret = ftrace_make_nop(mod, rec, MCOUNT_ADDR);
1082         if (ret) {
1083                 ftrace_bug(ret, ip);
1084                 rec->flags |= FTRACE_FL_FAILED;
1085                 return 0;
1086         }
1087         return 1;
1088 }
1089
1090 /*
1091  * archs can override this function if they must do something
1092  * before the modifying code is performed.
1093  */
1094 int __weak ftrace_arch_code_modify_prepare(void)
1095 {
1096         return 0;
1097 }
1098
1099 /*
1100  * archs can override this function if they must do something
1101  * after the modifying code is performed.
1102  */
1103 int __weak ftrace_arch_code_modify_post_process(void)
1104 {
1105         return 0;
1106 }
1107
1108 static int __ftrace_modify_code(void *data)
1109 {
1110         int *command = data;
1111
1112         if (*command & FTRACE_ENABLE_CALLS)
1113                 ftrace_replace_code(1);
1114         else if (*command & FTRACE_DISABLE_CALLS)
1115                 ftrace_replace_code(0);
1116
1117         if (*command & FTRACE_UPDATE_TRACE_FUNC)
1118                 ftrace_update_ftrace_func(ftrace_trace_function);
1119
1120         if (*command & FTRACE_START_FUNC_RET)
1121                 ftrace_enable_ftrace_graph_caller();
1122         else if (*command & FTRACE_STOP_FUNC_RET)
1123                 ftrace_disable_ftrace_graph_caller();
1124
1125         return 0;
1126 }
1127
1128 static void ftrace_run_update_code(int command)
1129 {
1130         int ret;
1131
1132         ret = ftrace_arch_code_modify_prepare();
1133         FTRACE_WARN_ON(ret);
1134         if (ret)
1135                 return;
1136
1137         stop_machine(__ftrace_modify_code, &command, NULL);
1138
1139         ret = ftrace_arch_code_modify_post_process();
1140         FTRACE_WARN_ON(ret);
1141 }
1142
1143 static ftrace_func_t saved_ftrace_func;
1144 static int ftrace_start_up;
1145
1146 static void ftrace_startup_enable(int command)
1147 {
1148         if (saved_ftrace_func != ftrace_trace_function) {
1149                 saved_ftrace_func = ftrace_trace_function;
1150                 command |= FTRACE_UPDATE_TRACE_FUNC;
1151         }
1152
1153         if (!command || !ftrace_enabled)
1154                 return;
1155
1156         ftrace_run_update_code(command);
1157 }
1158
1159 static void ftrace_startup(int command)
1160 {
1161         if (unlikely(ftrace_disabled))
1162                 return;
1163
1164         ftrace_start_up++;
1165         command |= FTRACE_ENABLE_CALLS;
1166
1167         ftrace_startup_enable(command);
1168 }
1169
1170 static void ftrace_shutdown(int command)
1171 {
1172         if (unlikely(ftrace_disabled))
1173                 return;
1174
1175         ftrace_start_up--;
1176         /*
1177          * Just warn in case of unbalance, no need to kill ftrace, it's not
1178          * critical but the ftrace_call callers may be never nopped again after
1179          * further ftrace uses.
1180          */
1181         WARN_ON_ONCE(ftrace_start_up < 0);
1182
1183         if (!ftrace_start_up)
1184                 command |= FTRACE_DISABLE_CALLS;
1185
1186         if (saved_ftrace_func != ftrace_trace_function) {
1187                 saved_ftrace_func = ftrace_trace_function;
1188                 command |= FTRACE_UPDATE_TRACE_FUNC;
1189         }
1190
1191         if (!command || !ftrace_enabled)
1192                 return;
1193
1194         ftrace_run_update_code(command);
1195 }
1196
1197 static void ftrace_startup_sysctl(void)
1198 {
1199         int command = FTRACE_ENABLE_MCOUNT;
1200
1201         if (unlikely(ftrace_disabled))
1202                 return;
1203
1204         /* Force update next time */
1205         saved_ftrace_func = NULL;
1206         /* ftrace_start_up is true if we want ftrace running */
1207         if (ftrace_start_up)
1208                 command |= FTRACE_ENABLE_CALLS;
1209
1210         ftrace_run_update_code(command);
1211 }
1212
1213 static void ftrace_shutdown_sysctl(void)
1214 {
1215         int command = FTRACE_DISABLE_MCOUNT;
1216
1217         if (unlikely(ftrace_disabled))
1218                 return;
1219
1220         /* ftrace_start_up is true if ftrace is running */
1221         if (ftrace_start_up)
1222                 command |= FTRACE_DISABLE_CALLS;
1223
1224         ftrace_run_update_code(command);
1225 }
1226
1227 static cycle_t          ftrace_update_time;
1228 static unsigned long    ftrace_update_cnt;
1229 unsigned long           ftrace_update_tot_cnt;
1230
1231 static int ftrace_update_code(struct module *mod)
1232 {
1233         struct dyn_ftrace *p;
1234         cycle_t start, stop;
1235
1236         start = ftrace_now(raw_smp_processor_id());
1237         ftrace_update_cnt = 0;
1238
1239         while (ftrace_new_addrs) {
1240
1241                 /* If something went wrong, bail without enabling anything */
1242                 if (unlikely(ftrace_disabled))
1243                         return -1;
1244
1245                 p = ftrace_new_addrs;
1246                 ftrace_new_addrs = p->newlist;
1247                 p->flags = 0L;
1248
1249                 /*
1250                  * Do the initial record convertion from mcount jump
1251                  * to the NOP instructions.
1252                  */
1253                 if (!ftrace_code_disable(mod, p)) {
1254                         ftrace_free_rec(p);
1255                         continue;
1256                 }
1257
1258                 p->flags |= FTRACE_FL_CONVERTED;
1259                 ftrace_update_cnt++;
1260
1261                 /*
1262                  * If the tracing is enabled, go ahead and enable the record.
1263                  *
1264                  * The reason not to enable the record immediatelly is the
1265                  * inherent check of ftrace_make_nop/ftrace_make_call for
1266                  * correct previous instructions.  Making first the NOP
1267                  * conversion puts the module to the correct state, thus
1268                  * passing the ftrace_make_call check.
1269                  */
1270                 if (ftrace_start_up) {
1271                         int failed = __ftrace_replace_code(p, 1);
1272                         if (failed) {
1273                                 ftrace_bug(failed, p->ip);
1274                                 ftrace_free_rec(p);
1275                         }
1276                 }
1277         }
1278
1279         stop = ftrace_now(raw_smp_processor_id());
1280         ftrace_update_time = stop - start;
1281         ftrace_update_tot_cnt += ftrace_update_cnt;
1282
1283         return 0;
1284 }
1285
1286 static int __init ftrace_dyn_table_alloc(unsigned long num_to_init)
1287 {
1288         struct ftrace_page *pg;
1289         int cnt;
1290         int i;
1291
1292         /* allocate a few pages */
1293         ftrace_pages_start = (void *)get_zeroed_page(GFP_KERNEL);
1294         if (!ftrace_pages_start)
1295                 return -1;
1296
1297         /*
1298          * Allocate a few more pages.
1299          *
1300          * TODO: have some parser search vmlinux before
1301          *   final linking to find all calls to ftrace.
1302          *   Then we can:
1303          *    a) know how many pages to allocate.
1304          *     and/or
1305          *    b) set up the table then.
1306          *
1307          *  The dynamic code is still necessary for
1308          *  modules.
1309          */
1310
1311         pg = ftrace_pages = ftrace_pages_start;
1312
1313         cnt = num_to_init / ENTRIES_PER_PAGE;
1314         pr_info("ftrace: allocating %ld entries in %d pages\n",
1315                 num_to_init, cnt + 1);
1316
1317         for (i = 0; i < cnt; i++) {
1318                 pg->next = (void *)get_zeroed_page(GFP_KERNEL);
1319
1320                 /* If we fail, we'll try later anyway */
1321                 if (!pg->next)
1322                         break;
1323
1324                 pg = pg->next;
1325         }
1326
1327         return 0;
1328 }
1329
1330 enum {
1331         FTRACE_ITER_FILTER      = (1 << 0),
1332         FTRACE_ITER_NOTRACE     = (1 << 1),
1333         FTRACE_ITER_FAILURES    = (1 << 2),
1334         FTRACE_ITER_PRINTALL    = (1 << 3),
1335         FTRACE_ITER_HASH        = (1 << 4),
1336 };
1337
1338 #define FTRACE_BUFF_MAX (KSYM_SYMBOL_LEN+4) /* room for wildcards */
1339
1340 struct ftrace_iterator {
1341         struct ftrace_page      *pg;
1342         int                     hidx;
1343         int                     idx;
1344         unsigned                flags;
1345         struct trace_parser     parser;
1346 };
1347
1348 static void *
1349 t_hash_next(struct seq_file *m, void *v, loff_t *pos)
1350 {
1351         struct ftrace_iterator *iter = m->private;
1352         struct hlist_node *hnd = v;
1353         struct hlist_head *hhd;
1354
1355         WARN_ON(!(iter->flags & FTRACE_ITER_HASH));
1356
1357         (*pos)++;
1358
1359  retry:
1360         if (iter->hidx >= FTRACE_FUNC_HASHSIZE)
1361                 return NULL;
1362
1363         hhd = &ftrace_func_hash[iter->hidx];
1364
1365         if (hlist_empty(hhd)) {
1366                 iter->hidx++;
1367                 hnd = NULL;
1368                 goto retry;
1369         }
1370
1371         if (!hnd)
1372                 hnd = hhd->first;
1373         else {
1374                 hnd = hnd->next;
1375                 if (!hnd) {
1376                         iter->hidx++;
1377                         goto retry;
1378                 }
1379         }
1380
1381         return hnd;
1382 }
1383
1384 static void *t_hash_start(struct seq_file *m, loff_t *pos)
1385 {
1386         struct ftrace_iterator *iter = m->private;
1387         void *p = NULL;
1388         loff_t l;
1389
1390         if (!(iter->flags & FTRACE_ITER_HASH))
1391                 *pos = 0;
1392
1393         iter->flags |= FTRACE_ITER_HASH;
1394
1395         iter->hidx = 0;
1396         for (l = 0; l <= *pos; ) {
1397                 p = t_hash_next(m, p, &l);
1398                 if (!p)
1399                         break;
1400         }
1401         return p;
1402 }
1403
1404 static int t_hash_show(struct seq_file *m, void *v)
1405 {
1406         struct ftrace_func_probe *rec;
1407         struct hlist_node *hnd = v;
1408
1409         rec = hlist_entry(hnd, struct ftrace_func_probe, node);
1410
1411         if (rec->ops->print)
1412                 return rec->ops->print(m, rec->ip, rec->ops, rec->data);
1413
1414         seq_printf(m, "%ps:%ps", (void *)rec->ip, (void *)rec->ops->func);
1415
1416         if (rec->data)
1417                 seq_printf(m, ":%p", rec->data);
1418         seq_putc(m, '\n');
1419
1420         return 0;
1421 }
1422
1423 static void *
1424 t_next(struct seq_file *m, void *v, loff_t *pos)
1425 {
1426         struct ftrace_iterator *iter = m->private;
1427         struct dyn_ftrace *rec = NULL;
1428
1429         if (iter->flags & FTRACE_ITER_HASH)
1430                 return t_hash_next(m, v, pos);
1431
1432         (*pos)++;
1433
1434         if (iter->flags & FTRACE_ITER_PRINTALL)
1435                 return NULL;
1436
1437  retry:
1438         if (iter->idx >= iter->pg->index) {
1439                 if (iter->pg->next) {
1440                         iter->pg = iter->pg->next;
1441                         iter->idx = 0;
1442                         goto retry;
1443                 }
1444         } else {
1445                 rec = &iter->pg->records[iter->idx++];
1446                 if ((rec->flags & FTRACE_FL_FREE) ||
1447
1448                     (!(iter->flags & FTRACE_ITER_FAILURES) &&
1449                      (rec->flags & FTRACE_FL_FAILED)) ||
1450
1451                     ((iter->flags & FTRACE_ITER_FAILURES) &&
1452                      !(rec->flags & FTRACE_FL_FAILED)) ||
1453
1454                     ((iter->flags & FTRACE_ITER_FILTER) &&
1455                      !(rec->flags & FTRACE_FL_FILTER)) ||
1456
1457                     ((iter->flags & FTRACE_ITER_NOTRACE) &&
1458                      !(rec->flags & FTRACE_FL_NOTRACE))) {
1459                         rec = NULL;
1460                         goto retry;
1461                 }
1462         }
1463
1464         return rec;
1465 }
1466
1467 static void *t_start(struct seq_file *m, loff_t *pos)
1468 {
1469         struct ftrace_iterator *iter = m->private;
1470         void *p = NULL;
1471         loff_t l;
1472
1473         mutex_lock(&ftrace_lock);
1474         /*
1475          * For set_ftrace_filter reading, if we have the filter
1476          * off, we can short cut and just print out that all
1477          * functions are enabled.
1478          */
1479         if (iter->flags & FTRACE_ITER_FILTER && !ftrace_filtered) {
1480                 if (*pos > 0)
1481                         return t_hash_start(m, pos);
1482                 iter->flags |= FTRACE_ITER_PRINTALL;
1483                 return iter;
1484         }
1485
1486         if (iter->flags & FTRACE_ITER_HASH)
1487                 return t_hash_start(m, pos);
1488
1489         iter->pg = ftrace_pages_start;
1490         iter->idx = 0;
1491         for (l = 0; l <= *pos; ) {
1492                 p = t_next(m, p, &l);
1493                 if (!p)
1494                         break;
1495         }
1496
1497         if (!p && iter->flags & FTRACE_ITER_FILTER)
1498                 return t_hash_start(m, pos);
1499
1500         return p;
1501 }
1502
1503 static void t_stop(struct seq_file *m, void *p)
1504 {
1505         mutex_unlock(&ftrace_lock);
1506 }
1507
1508 static int t_show(struct seq_file *m, void *v)
1509 {
1510         struct ftrace_iterator *iter = m->private;
1511         struct dyn_ftrace *rec = v;
1512
1513         if (iter->flags & FTRACE_ITER_HASH)
1514                 return t_hash_show(m, v);
1515
1516         if (iter->flags & FTRACE_ITER_PRINTALL) {
1517                 seq_printf(m, "#### all functions enabled ####\n");
1518                 return 0;
1519         }
1520
1521         if (!rec)
1522                 return 0;
1523
1524         seq_printf(m, "%ps\n", (void *)rec->ip);
1525
1526         return 0;
1527 }
1528
1529 static const struct seq_operations show_ftrace_seq_ops = {
1530         .start = t_start,
1531         .next = t_next,
1532         .stop = t_stop,
1533         .show = t_show,
1534 };
1535
1536 static int
1537 ftrace_avail_open(struct inode *inode, struct file *file)
1538 {
1539         struct ftrace_iterator *iter;
1540         int ret;
1541
1542         if (unlikely(ftrace_disabled))
1543                 return -ENODEV;
1544
1545         iter = kzalloc(sizeof(*iter), GFP_KERNEL);
1546         if (!iter)
1547                 return -ENOMEM;
1548
1549         iter->pg = ftrace_pages_start;
1550
1551         ret = seq_open(file, &show_ftrace_seq_ops);
1552         if (!ret) {
1553                 struct seq_file *m = file->private_data;
1554
1555                 m->private = iter;
1556         } else {
1557                 kfree(iter);
1558         }
1559
1560         return ret;
1561 }
1562
1563 static int
1564 ftrace_failures_open(struct inode *inode, struct file *file)
1565 {
1566         int ret;
1567         struct seq_file *m;
1568         struct ftrace_iterator *iter;
1569
1570         ret = ftrace_avail_open(inode, file);
1571         if (!ret) {
1572                 m = (struct seq_file *)file->private_data;
1573                 iter = (struct ftrace_iterator *)m->private;
1574                 iter->flags = FTRACE_ITER_FAILURES;
1575         }
1576
1577         return ret;
1578 }
1579
1580
1581 static void ftrace_filter_reset(int enable)
1582 {
1583         struct ftrace_page *pg;
1584         struct dyn_ftrace *rec;
1585         unsigned long type = enable ? FTRACE_FL_FILTER : FTRACE_FL_NOTRACE;
1586
1587         mutex_lock(&ftrace_lock);
1588         if (enable)
1589                 ftrace_filtered = 0;
1590         do_for_each_ftrace_rec(pg, rec) {
1591                 if (rec->flags & FTRACE_FL_FAILED)
1592                         continue;
1593                 rec->flags &= ~type;
1594         } while_for_each_ftrace_rec();
1595         mutex_unlock(&ftrace_lock);
1596 }
1597
1598 static int
1599 ftrace_regex_open(struct inode *inode, struct file *file, int enable)
1600 {
1601         struct ftrace_iterator *iter;
1602         int ret = 0;
1603
1604         if (unlikely(ftrace_disabled))
1605                 return -ENODEV;
1606
1607         iter = kzalloc(sizeof(*iter), GFP_KERNEL);
1608         if (!iter)
1609                 return -ENOMEM;
1610
1611         if (trace_parser_get_init(&iter->parser, FTRACE_BUFF_MAX)) {
1612                 kfree(iter);
1613                 return -ENOMEM;
1614         }
1615
1616         mutex_lock(&ftrace_regex_lock);
1617         if ((file->f_mode & FMODE_WRITE) &&
1618             (file->f_flags & O_TRUNC))
1619                 ftrace_filter_reset(enable);
1620
1621         if (file->f_mode & FMODE_READ) {
1622                 iter->pg = ftrace_pages_start;
1623                 iter->flags = enable ? FTRACE_ITER_FILTER :
1624                         FTRACE_ITER_NOTRACE;
1625
1626                 ret = seq_open(file, &show_ftrace_seq_ops);
1627                 if (!ret) {
1628                         struct seq_file *m = file->private_data;
1629                         m->private = iter;
1630                 } else {
1631                         trace_parser_put(&iter->parser);
1632                         kfree(iter);
1633                 }
1634         } else
1635                 file->private_data = iter;
1636         mutex_unlock(&ftrace_regex_lock);
1637
1638         return ret;
1639 }
1640
1641 static int
1642 ftrace_filter_open(struct inode *inode, struct file *file)
1643 {
1644         return ftrace_regex_open(inode, file, 1);
1645 }
1646
1647 static int
1648 ftrace_notrace_open(struct inode *inode, struct file *file)
1649 {
1650         return ftrace_regex_open(inode, file, 0);
1651 }
1652
1653 static loff_t
1654 ftrace_regex_lseek(struct file *file, loff_t offset, int origin)
1655 {
1656         loff_t ret;
1657
1658         if (file->f_mode & FMODE_READ)
1659                 ret = seq_lseek(file, offset, origin);
1660         else
1661                 file->f_pos = ret = 1;
1662
1663         return ret;
1664 }
1665
1666 static int ftrace_match(char *str, char *regex, int len, int type)
1667 {
1668         int matched = 0;
1669         int slen;
1670
1671         switch (type) {
1672         case MATCH_FULL:
1673                 if (strcmp(str, regex) == 0)
1674                         matched = 1;
1675                 break;
1676         case MATCH_FRONT_ONLY:
1677                 if (strncmp(str, regex, len) == 0)
1678                         matched = 1;
1679                 break;
1680         case MATCH_MIDDLE_ONLY:
1681                 if (strstr(str, regex))
1682                         matched = 1;
1683                 break;
1684         case MATCH_END_ONLY:
1685                 slen = strlen(str);
1686                 if (slen >= len && memcmp(str + slen - len, regex, len) == 0)
1687                         matched = 1;
1688                 break;
1689         }
1690
1691         return matched;
1692 }
1693
1694 static int
1695 ftrace_match_record(struct dyn_ftrace *rec, char *regex, int len, int type)
1696 {
1697         char str[KSYM_SYMBOL_LEN];
1698
1699         kallsyms_lookup(rec->ip, NULL, NULL, NULL, str);
1700         return ftrace_match(str, regex, len, type);
1701 }
1702
1703 static int ftrace_match_records(char *buff, int len, int enable)
1704 {
1705         unsigned int search_len;
1706         struct ftrace_page *pg;
1707         struct dyn_ftrace *rec;
1708         unsigned long flag;
1709         char *search;
1710         int type;
1711         int not;
1712         int found = 0;
1713
1714         flag = enable ? FTRACE_FL_FILTER : FTRACE_FL_NOTRACE;
1715         type = filter_parse_regex(buff, len, &search, &not);
1716
1717         search_len = strlen(search);
1718
1719         mutex_lock(&ftrace_lock);
1720         do_for_each_ftrace_rec(pg, rec) {
1721
1722                 if (rec->flags & FTRACE_FL_FAILED)
1723                         continue;
1724
1725                 if (ftrace_match_record(rec, search, search_len, type)) {
1726                         if (not)
1727                                 rec->flags &= ~flag;
1728                         else
1729                                 rec->flags |= flag;
1730                         found = 1;
1731                 }
1732                 /*
1733                  * Only enable filtering if we have a function that
1734                  * is filtered on.
1735                  */
1736                 if (enable && (rec->flags & FTRACE_FL_FILTER))
1737                         ftrace_filtered = 1;
1738         } while_for_each_ftrace_rec();
1739         mutex_unlock(&ftrace_lock);
1740
1741         return found;
1742 }
1743
1744 static int
1745 ftrace_match_module_record(struct dyn_ftrace *rec, char *mod,
1746                            char *regex, int len, int type)
1747 {
1748         char str[KSYM_SYMBOL_LEN];
1749         char *modname;
1750
1751         kallsyms_lookup(rec->ip, NULL, NULL, &modname, str);
1752
1753         if (!modname || strcmp(modname, mod))
1754                 return 0;
1755
1756         /* blank search means to match all funcs in the mod */
1757         if (len)
1758                 return ftrace_match(str, regex, len, type);
1759         else
1760                 return 1;
1761 }
1762
1763 static int ftrace_match_module_records(char *buff, char *mod, int enable)
1764 {
1765         unsigned search_len = 0;
1766         struct ftrace_page *pg;
1767         struct dyn_ftrace *rec;
1768         int type = MATCH_FULL;
1769         char *search = buff;
1770         unsigned long flag;
1771         int not = 0;
1772         int found = 0;
1773
1774         flag = enable ? FTRACE_FL_FILTER : FTRACE_FL_NOTRACE;
1775
1776         /* blank or '*' mean the same */
1777         if (strcmp(buff, "*") == 0)
1778                 buff[0] = 0;
1779
1780         /* handle the case of 'dont filter this module' */
1781         if (strcmp(buff, "!") == 0 || strcmp(buff, "!*") == 0) {
1782                 buff[0] = 0;
1783                 not = 1;
1784         }
1785
1786         if (strlen(buff)) {
1787                 type = filter_parse_regex(buff, strlen(buff), &search, &not);
1788                 search_len = strlen(search);
1789         }
1790
1791         mutex_lock(&ftrace_lock);
1792         do_for_each_ftrace_rec(pg, rec) {
1793
1794                 if (rec->flags & FTRACE_FL_FAILED)
1795                         continue;
1796
1797                 if (ftrace_match_module_record(rec, mod,
1798                                                search, search_len, type)) {
1799                         if (not)
1800                                 rec->flags &= ~flag;
1801                         else
1802                                 rec->flags |= flag;
1803                         found = 1;
1804                 }
1805                 if (enable && (rec->flags & FTRACE_FL_FILTER))
1806                         ftrace_filtered = 1;
1807
1808         } while_for_each_ftrace_rec();
1809         mutex_unlock(&ftrace_lock);
1810
1811         return found;
1812 }
1813
1814 /*
1815  * We register the module command as a template to show others how
1816  * to register the a command as well.
1817  */
1818
1819 static int
1820 ftrace_mod_callback(char *func, char *cmd, char *param, int enable)
1821 {
1822         char *mod;
1823
1824         /*
1825          * cmd == 'mod' because we only registered this func
1826          * for the 'mod' ftrace_func_command.
1827          * But if you register one func with multiple commands,
1828          * you can tell which command was used by the cmd
1829          * parameter.
1830          */
1831
1832         /* we must have a module name */
1833         if (!param)
1834                 return -EINVAL;
1835
1836         mod = strsep(&param, ":");
1837         if (!strlen(mod))
1838                 return -EINVAL;
1839
1840         if (ftrace_match_module_records(func, mod, enable))
1841                 return 0;
1842         return -EINVAL;
1843 }
1844
1845 static struct ftrace_func_command ftrace_mod_cmd = {
1846         .name                   = "mod",
1847         .func                   = ftrace_mod_callback,
1848 };
1849
1850 static int __init ftrace_mod_cmd_init(void)
1851 {
1852         return register_ftrace_command(&ftrace_mod_cmd);
1853 }
1854 device_initcall(ftrace_mod_cmd_init);
1855
1856 static void
1857 function_trace_probe_call(unsigned long ip, unsigned long parent_ip)
1858 {
1859         struct ftrace_func_probe *entry;
1860         struct hlist_head *hhd;
1861         struct hlist_node *n;
1862         unsigned long key;
1863         int resched;
1864
1865         key = hash_long(ip, FTRACE_HASH_BITS);
1866
1867         hhd = &ftrace_func_hash[key];
1868
1869         if (hlist_empty(hhd))
1870                 return;
1871
1872         /*
1873          * Disable preemption for these calls to prevent a RCU grace
1874          * period. This syncs the hash iteration and freeing of items
1875          * on the hash. rcu_read_lock is too dangerous here.
1876          */
1877         resched = ftrace_preempt_disable();
1878         hlist_for_each_entry_rcu(entry, n, hhd, node) {
1879                 if (entry->ip == ip)
1880                         entry->ops->func(ip, parent_ip, &entry->data);
1881         }
1882         ftrace_preempt_enable(resched);
1883 }
1884
1885 static struct ftrace_ops trace_probe_ops __read_mostly =
1886 {
1887         .func           = function_trace_probe_call,
1888 };
1889
1890 static int ftrace_probe_registered;
1891
1892 static void __enable_ftrace_function_probe(void)
1893 {
1894         int i;
1895
1896         if (ftrace_probe_registered)
1897                 return;
1898
1899         for (i = 0; i < FTRACE_FUNC_HASHSIZE; i++) {
1900                 struct hlist_head *hhd = &ftrace_func_hash[i];
1901                 if (hhd->first)
1902                         break;
1903         }
1904         /* Nothing registered? */
1905         if (i == FTRACE_FUNC_HASHSIZE)
1906                 return;
1907
1908         __register_ftrace_function(&trace_probe_ops);
1909         ftrace_startup(0);
1910         ftrace_probe_registered = 1;
1911 }
1912
1913 static void __disable_ftrace_function_probe(void)
1914 {
1915         int i;
1916
1917         if (!ftrace_probe_registered)
1918                 return;
1919
1920         for (i = 0; i < FTRACE_FUNC_HASHSIZE; i++) {
1921                 struct hlist_head *hhd = &ftrace_func_hash[i];
1922                 if (hhd->first)
1923                         return;
1924         }
1925
1926         /* no more funcs left */
1927         __unregister_ftrace_function(&trace_probe_ops);
1928         ftrace_shutdown(0);
1929         ftrace_probe_registered = 0;
1930 }
1931
1932
1933 static void ftrace_free_entry_rcu(struct rcu_head *rhp)
1934 {
1935         struct ftrace_func_probe *entry =
1936                 container_of(rhp, struct ftrace_func_probe, rcu);
1937
1938         if (entry->ops->free)
1939                 entry->ops->free(&entry->data);
1940         kfree(entry);
1941 }
1942
1943
1944 int
1945 register_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops,
1946                               void *data)
1947 {
1948         struct ftrace_func_probe *entry;
1949         struct ftrace_page *pg;
1950         struct dyn_ftrace *rec;
1951         int type, len, not;
1952         unsigned long key;
1953         int count = 0;
1954         char *search;
1955
1956         type = filter_parse_regex(glob, strlen(glob), &search, &not);
1957         len = strlen(search);
1958
1959         /* we do not support '!' for function probes */
1960         if (WARN_ON(not))
1961                 return -EINVAL;
1962
1963         mutex_lock(&ftrace_lock);
1964         do_for_each_ftrace_rec(pg, rec) {
1965
1966                 if (rec->flags & FTRACE_FL_FAILED)
1967                         continue;
1968
1969                 if (!ftrace_match_record(rec, search, len, type))
1970                         continue;
1971
1972                 entry = kmalloc(sizeof(*entry), GFP_KERNEL);
1973                 if (!entry) {
1974                         /* If we did not process any, then return error */
1975                         if (!count)
1976                                 count = -ENOMEM;
1977                         goto out_unlock;
1978                 }
1979
1980                 count++;
1981
1982                 entry->data = data;
1983
1984                 /*
1985                  * The caller might want to do something special
1986                  * for each function we find. We call the callback
1987                  * to give the caller an opportunity to do so.
1988                  */
1989                 if (ops->callback) {
1990                         if (ops->callback(rec->ip, &entry->data) < 0) {
1991                                 /* caller does not like this func */
1992                                 kfree(entry);
1993                                 continue;
1994                         }
1995                 }
1996
1997                 entry->ops = ops;
1998                 entry->ip = rec->ip;
1999
2000                 key = hash_long(entry->ip, FTRACE_HASH_BITS);
2001                 hlist_add_head_rcu(&entry->node, &ftrace_func_hash[key]);
2002
2003         } while_for_each_ftrace_rec();
2004         __enable_ftrace_function_probe();
2005
2006  out_unlock:
2007         mutex_unlock(&ftrace_lock);
2008
2009         return count;
2010 }
2011
2012 enum {
2013         PROBE_TEST_FUNC         = 1,
2014         PROBE_TEST_DATA         = 2
2015 };
2016
2017 static void
2018 __unregister_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops,
2019                                   void *data, int flags)
2020 {
2021         struct ftrace_func_probe *entry;
2022         struct hlist_node *n, *tmp;
2023         char str[KSYM_SYMBOL_LEN];
2024         int type = MATCH_FULL;
2025         int i, len = 0;
2026         char *search;
2027
2028         if (glob && (strcmp(glob, "*") == 0 || !strlen(glob)))
2029                 glob = NULL;
2030         else if (glob) {
2031                 int not;
2032
2033                 type = filter_parse_regex(glob, strlen(glob), &search, &not);
2034                 len = strlen(search);
2035
2036                 /* we do not support '!' for function probes */
2037                 if (WARN_ON(not))
2038                         return;
2039         }
2040
2041         mutex_lock(&ftrace_lock);
2042         for (i = 0; i < FTRACE_FUNC_HASHSIZE; i++) {
2043                 struct hlist_head *hhd = &ftrace_func_hash[i];
2044
2045                 hlist_for_each_entry_safe(entry, n, tmp, hhd, node) {
2046
2047                         /* break up if statements for readability */
2048                         if ((flags & PROBE_TEST_FUNC) && entry->ops != ops)
2049                                 continue;
2050
2051                         if ((flags & PROBE_TEST_DATA) && entry->data != data)
2052                                 continue;
2053
2054                         /* do this last, since it is the most expensive */
2055                         if (glob) {
2056                                 kallsyms_lookup(entry->ip, NULL, NULL,
2057                                                 NULL, str);
2058                                 if (!ftrace_match(str, glob, len, type))
2059                                         continue;
2060                         }
2061
2062                         hlist_del(&entry->node);
2063                         call_rcu(&entry->rcu, ftrace_free_entry_rcu);
2064                 }
2065         }
2066         __disable_ftrace_function_probe();
2067         mutex_unlock(&ftrace_lock);
2068 }
2069
2070 void
2071 unregister_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops,
2072                                 void *data)
2073 {
2074         __unregister_ftrace_function_probe(glob, ops, data,
2075                                           PROBE_TEST_FUNC | PROBE_TEST_DATA);
2076 }
2077
2078 void
2079 unregister_ftrace_function_probe_func(char *glob, struct ftrace_probe_ops *ops)
2080 {
2081         __unregister_ftrace_function_probe(glob, ops, NULL, PROBE_TEST_FUNC);
2082 }
2083
2084 void unregister_ftrace_function_probe_all(char *glob)
2085 {
2086         __unregister_ftrace_function_probe(glob, NULL, NULL, 0);
2087 }
2088
2089 static LIST_HEAD(ftrace_commands);
2090 static DEFINE_MUTEX(ftrace_cmd_mutex);
2091
2092 int register_ftrace_command(struct ftrace_func_command *cmd)
2093 {
2094         struct ftrace_func_command *p;
2095         int ret = 0;
2096
2097         mutex_lock(&ftrace_cmd_mutex);
2098         list_for_each_entry(p, &ftrace_commands, list) {
2099                 if (strcmp(cmd->name, p->name) == 0) {
2100                         ret = -EBUSY;
2101                         goto out_unlock;
2102                 }
2103         }
2104         list_add(&cmd->list, &ftrace_commands);
2105  out_unlock:
2106         mutex_unlock(&ftrace_cmd_mutex);
2107
2108         return ret;
2109 }
2110
2111 int unregister_ftrace_command(struct ftrace_func_command *cmd)
2112 {
2113         struct ftrace_func_command *p, *n;
2114         int ret = -ENODEV;
2115
2116         mutex_lock(&ftrace_cmd_mutex);
2117         list_for_each_entry_safe(p, n, &ftrace_commands, list) {
2118                 if (strcmp(cmd->name, p->name) == 0) {
2119                         ret = 0;
2120                         list_del_init(&p->list);
2121                         goto out_unlock;
2122                 }
2123         }
2124  out_unlock:
2125         mutex_unlock(&ftrace_cmd_mutex);
2126
2127         return ret;
2128 }
2129
2130 static int ftrace_process_regex(char *buff, int len, int enable)
2131 {
2132         char *func, *command, *next = buff;
2133         struct ftrace_func_command *p;
2134         int ret = -EINVAL;
2135
2136         func = strsep(&next, ":");
2137
2138         if (!next) {
2139                 if (ftrace_match_records(func, len, enable))
2140                         return 0;
2141                 return ret;
2142         }
2143
2144         /* command found */
2145
2146         command = strsep(&next, ":");
2147
2148         mutex_lock(&ftrace_cmd_mutex);
2149         list_for_each_entry(p, &ftrace_commands, list) {
2150                 if (strcmp(p->name, command) == 0) {
2151                         ret = p->func(func, command, next, enable);
2152                         goto out_unlock;
2153                 }
2154         }
2155  out_unlock:
2156         mutex_unlock(&ftrace_cmd_mutex);
2157
2158         return ret;
2159 }
2160
2161 static ssize_t
2162 ftrace_regex_write(struct file *file, const char __user *ubuf,
2163                    size_t cnt, loff_t *ppos, int enable)
2164 {
2165         struct ftrace_iterator *iter;
2166         struct trace_parser *parser;
2167         ssize_t ret, read;
2168
2169         if (!cnt)
2170                 return 0;
2171
2172         mutex_lock(&ftrace_regex_lock);
2173
2174         if (file->f_mode & FMODE_READ) {
2175                 struct seq_file *m = file->private_data;
2176                 iter = m->private;
2177         } else
2178                 iter = file->private_data;
2179
2180         parser = &iter->parser;
2181         read = trace_get_user(parser, ubuf, cnt, ppos);
2182
2183         if (read >= 0 && trace_parser_loaded(parser) &&
2184             !trace_parser_cont(parser)) {
2185                 ret = ftrace_process_regex(parser->buffer,
2186                                            parser->idx, enable);
2187                 trace_parser_clear(parser);
2188                 if (ret)
2189                         goto out_unlock;
2190         }
2191
2192         ret = read;
2193 out_unlock:
2194         mutex_unlock(&ftrace_regex_lock);
2195
2196         return ret;
2197 }
2198
2199 static ssize_t
2200 ftrace_filter_write(struct file *file, const char __user *ubuf,
2201                     size_t cnt, loff_t *ppos)
2202 {
2203         return ftrace_regex_write(file, ubuf, cnt, ppos, 1);
2204 }
2205
2206 static ssize_t
2207 ftrace_notrace_write(struct file *file, const char __user *ubuf,
2208                      size_t cnt, loff_t *ppos)
2209 {
2210         return ftrace_regex_write(file, ubuf, cnt, ppos, 0);
2211 }
2212
2213 static void
2214 ftrace_set_regex(unsigned char *buf, int len, int reset, int enable)
2215 {
2216         if (unlikely(ftrace_disabled))
2217                 return;
2218
2219         mutex_lock(&ftrace_regex_lock);
2220         if (reset)
2221                 ftrace_filter_reset(enable);
2222         if (buf)
2223                 ftrace_match_records(buf, len, enable);
2224         mutex_unlock(&ftrace_regex_lock);
2225 }
2226
2227 /**
2228  * ftrace_set_filter - set a function to filter on in ftrace
2229  * @buf - the string that holds the function filter text.
2230  * @len - the length of the string.
2231  * @reset - non zero to reset all filters before applying this filter.
2232  *
2233  * Filters denote which functions should be enabled when tracing is enabled.
2234  * If @buf is NULL and reset is set, all functions will be enabled for tracing.
2235  */
2236 void ftrace_set_filter(unsigned char *buf, int len, int reset)
2237 {
2238         ftrace_set_regex(buf, len, reset, 1);
2239 }
2240
2241 /**
2242  * ftrace_set_notrace - set a function to not trace in ftrace
2243  * @buf - the string that holds the function notrace text.
2244  * @len - the length of the string.
2245  * @reset - non zero to reset all filters before applying this filter.
2246  *
2247  * Notrace Filters denote which functions should not be enabled when tracing
2248  * is enabled. If @buf is NULL and reset is set, all functions will be enabled
2249  * for tracing.
2250  */
2251 void ftrace_set_notrace(unsigned char *buf, int len, int reset)
2252 {
2253         ftrace_set_regex(buf, len, reset, 0);
2254 }
2255
2256 /*
2257  * command line interface to allow users to set filters on boot up.
2258  */
2259 #define FTRACE_FILTER_SIZE              COMMAND_LINE_SIZE
2260 static char ftrace_notrace_buf[FTRACE_FILTER_SIZE] __initdata;
2261 static char ftrace_filter_buf[FTRACE_FILTER_SIZE] __initdata;
2262
2263 static int __init set_ftrace_notrace(char *str)
2264 {
2265         strncpy(ftrace_notrace_buf, str, FTRACE_FILTER_SIZE);
2266         return 1;
2267 }
2268 __setup("ftrace_notrace=", set_ftrace_notrace);
2269
2270 static int __init set_ftrace_filter(char *str)
2271 {
2272         strncpy(ftrace_filter_buf, str, FTRACE_FILTER_SIZE);
2273         return 1;
2274 }
2275 __setup("ftrace_filter=", set_ftrace_filter);
2276
2277 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
2278 static char ftrace_graph_buf[FTRACE_FILTER_SIZE] __initdata;
2279 static int ftrace_set_func(unsigned long *array, int *idx, char *buffer);
2280
2281 static int __init set_graph_function(char *str)
2282 {
2283         strlcpy(ftrace_graph_buf, str, FTRACE_FILTER_SIZE);
2284         return 1;
2285 }
2286 __setup("ftrace_graph_filter=", set_graph_function);
2287
2288 static void __init set_ftrace_early_graph(char *buf)
2289 {
2290         int ret;
2291         char *func;
2292
2293         while (buf) {
2294                 func = strsep(&buf, ",");
2295                 /* we allow only one expression at a time */
2296                 ret = ftrace_set_func(ftrace_graph_funcs, &ftrace_graph_count,
2297                                       func);
2298                 if (ret)
2299                         printk(KERN_DEBUG "ftrace: function %s not "
2300                                           "traceable\n", func);
2301         }
2302 }
2303 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
2304
2305 static void __init set_ftrace_early_filter(char *buf, int enable)
2306 {
2307         char *func;
2308
2309         while (buf) {
2310                 func = strsep(&buf, ",");
2311                 ftrace_set_regex(func, strlen(func), 0, enable);
2312         }
2313 }
2314
2315 static void __init set_ftrace_early_filters(void)
2316 {
2317         if (ftrace_filter_buf[0])
2318                 set_ftrace_early_filter(ftrace_filter_buf, 1);
2319         if (ftrace_notrace_buf[0])
2320                 set_ftrace_early_filter(ftrace_notrace_buf, 0);
2321 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
2322         if (ftrace_graph_buf[0])
2323                 set_ftrace_early_graph(ftrace_graph_buf);
2324 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
2325 }
2326
2327 static int
2328 ftrace_regex_release(struct inode *inode, struct file *file, int enable)
2329 {
2330         struct seq_file *m = (struct seq_file *)file->private_data;
2331         struct ftrace_iterator *iter;
2332         struct trace_parser *parser;
2333
2334         mutex_lock(&ftrace_regex_lock);
2335         if (file->f_mode & FMODE_READ) {
2336                 iter = m->private;
2337
2338                 seq_release(inode, file);
2339         } else
2340                 iter = file->private_data;
2341
2342         parser = &iter->parser;
2343         if (trace_parser_loaded(parser)) {
2344                 parser->buffer[parser->idx] = 0;
2345                 ftrace_match_records(parser->buffer, parser->idx, enable);
2346         }
2347
2348         mutex_lock(&ftrace_lock);
2349         if (ftrace_start_up && ftrace_enabled)
2350                 ftrace_run_update_code(FTRACE_ENABLE_CALLS);
2351         mutex_unlock(&ftrace_lock);
2352
2353         trace_parser_put(parser);
2354         kfree(iter);
2355
2356         mutex_unlock(&ftrace_regex_lock);
2357         return 0;
2358 }
2359
2360 static int
2361 ftrace_filter_release(struct inode *inode, struct file *file)
2362 {
2363         return ftrace_regex_release(inode, file, 1);
2364 }
2365
2366 static int
2367 ftrace_notrace_release(struct inode *inode, struct file *file)
2368 {
2369         return ftrace_regex_release(inode, file, 0);
2370 }
2371
2372 static const struct file_operations ftrace_avail_fops = {
2373         .open = ftrace_avail_open,
2374         .read = seq_read,
2375         .llseek = seq_lseek,
2376         .release = seq_release_private,
2377 };
2378
2379 static const struct file_operations ftrace_failures_fops = {
2380         .open = ftrace_failures_open,
2381         .read = seq_read,
2382         .llseek = seq_lseek,
2383         .release = seq_release_private,
2384 };
2385
2386 static const struct file_operations ftrace_filter_fops = {
2387         .open = ftrace_filter_open,
2388         .read = seq_read,
2389         .write = ftrace_filter_write,
2390         .llseek = ftrace_regex_lseek,
2391         .release = ftrace_filter_release,
2392 };
2393
2394 static const struct file_operations ftrace_notrace_fops = {
2395         .open = ftrace_notrace_open,
2396         .read = seq_read,
2397         .write = ftrace_notrace_write,
2398         .llseek = ftrace_regex_lseek,
2399         .release = ftrace_notrace_release,
2400 };
2401
2402 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
2403
2404 static DEFINE_MUTEX(graph_lock);
2405
2406 int ftrace_graph_count;
2407 int ftrace_graph_filter_enabled;
2408 unsigned long ftrace_graph_funcs[FTRACE_GRAPH_MAX_FUNCS] __read_mostly;
2409
2410 static void *
2411 __g_next(struct seq_file *m, loff_t *pos)
2412 {
2413         if (*pos >= ftrace_graph_count)
2414                 return NULL;
2415         return &ftrace_graph_funcs[*pos];
2416 }
2417
2418 static void *
2419 g_next(struct seq_file *m, void *v, loff_t *pos)
2420 {
2421         (*pos)++;
2422         return __g_next(m, pos);
2423 }
2424
2425 static void *g_start(struct seq_file *m, loff_t *pos)
2426 {
2427         mutex_lock(&graph_lock);
2428
2429         /* Nothing, tell g_show to print all functions are enabled */
2430         if (!ftrace_graph_filter_enabled && !*pos)
2431                 return (void *)1;
2432
2433         return __g_next(m, pos);
2434 }
2435
2436 static void g_stop(struct seq_file *m, void *p)
2437 {
2438         mutex_unlock(&graph_lock);
2439 }
2440
2441 static int g_show(struct seq_file *m, void *v)
2442 {
2443         unsigned long *ptr = v;
2444
2445         if (!ptr)
2446                 return 0;
2447
2448         if (ptr == (unsigned long *)1) {
2449                 seq_printf(m, "#### all functions enabled ####\n");
2450                 return 0;
2451         }
2452
2453         seq_printf(m, "%ps\n", (void *)*ptr);
2454
2455         return 0;
2456 }
2457
2458 static const struct seq_operations ftrace_graph_seq_ops = {
2459         .start = g_start,
2460         .next = g_next,
2461         .stop = g_stop,
2462         .show = g_show,
2463 };
2464
2465 static int
2466 ftrace_graph_open(struct inode *inode, struct file *file)
2467 {
2468         int ret = 0;
2469
2470         if (unlikely(ftrace_disabled))
2471                 return -ENODEV;
2472
2473         mutex_lock(&graph_lock);
2474         if ((file->f_mode & FMODE_WRITE) &&
2475             (file->f_flags & O_TRUNC)) {
2476                 ftrace_graph_filter_enabled = 0;
2477                 ftrace_graph_count = 0;
2478                 memset(ftrace_graph_funcs, 0, sizeof(ftrace_graph_funcs));
2479         }
2480         mutex_unlock(&graph_lock);
2481
2482         if (file->f_mode & FMODE_READ)
2483                 ret = seq_open(file, &ftrace_graph_seq_ops);
2484
2485         return ret;
2486 }
2487
2488 static int
2489 ftrace_graph_release(struct inode *inode, struct file *file)
2490 {
2491         if (file->f_mode & FMODE_READ)
2492                 seq_release(inode, file);
2493         return 0;
2494 }
2495
2496 static int
2497 ftrace_set_func(unsigned long *array, int *idx, char *buffer)
2498 {
2499         struct dyn_ftrace *rec;
2500         struct ftrace_page *pg;
2501         int search_len;
2502         int fail = 1;
2503         int type, not;
2504         char *search;
2505         bool exists;
2506         int i;
2507
2508         if (ftrace_disabled)
2509                 return -ENODEV;
2510
2511         /* decode regex */
2512         type = filter_parse_regex(buffer, strlen(buffer), &search, &not);
2513         if (!not && *idx >= FTRACE_GRAPH_MAX_FUNCS)
2514                 return -EBUSY;
2515
2516         search_len = strlen(search);
2517
2518         mutex_lock(&ftrace_lock);
2519         do_for_each_ftrace_rec(pg, rec) {
2520
2521                 if (rec->flags & (FTRACE_FL_FAILED | FTRACE_FL_FREE))
2522                         continue;
2523
2524                 if (ftrace_match_record(rec, search, search_len, type)) {
2525                         /* if it is in the array */
2526                         exists = false;
2527                         for (i = 0; i < *idx; i++) {
2528                                 if (array[i] == rec->ip) {
2529                                         exists = true;
2530                                         break;
2531                                 }
2532                         }
2533
2534                         if (!not) {
2535                                 fail = 0;
2536                                 if (!exists) {
2537                                         array[(*idx)++] = rec->ip;
2538                                         if (*idx >= FTRACE_GRAPH_MAX_FUNCS)
2539                                                 goto out;
2540                                 }
2541                         } else {
2542                                 if (exists) {
2543                                         array[i] = array[--(*idx)];
2544                                         array[*idx] = 0;
2545                                         fail = 0;
2546                                 }
2547                         }
2548                 }
2549         } while_for_each_ftrace_rec();
2550 out:
2551         mutex_unlock(&ftrace_lock);
2552
2553         if (fail)
2554                 return -EINVAL;
2555
2556         ftrace_graph_filter_enabled = 1;
2557         return 0;
2558 }
2559
2560 static ssize_t
2561 ftrace_graph_write(struct file *file, const char __user *ubuf,
2562                    size_t cnt, loff_t *ppos)
2563 {
2564         struct trace_parser parser;
2565         ssize_t read, ret;
2566
2567         if (!cnt)
2568                 return 0;
2569
2570         mutex_lock(&graph_lock);
2571
2572         if (trace_parser_get_init(&parser, FTRACE_BUFF_MAX)) {
2573                 ret = -ENOMEM;
2574                 goto out_unlock;
2575         }
2576
2577         read = trace_get_user(&parser, ubuf, cnt, ppos);
2578
2579         if (read >= 0 && trace_parser_loaded((&parser))) {
2580                 parser.buffer[parser.idx] = 0;
2581
2582                 /* we allow only one expression at a time */
2583                 ret = ftrace_set_func(ftrace_graph_funcs, &ftrace_graph_count,
2584                                         parser.buffer);
2585                 if (ret)
2586                         goto out_free;
2587         }
2588
2589         ret = read;
2590
2591 out_free:
2592         trace_parser_put(&parser);
2593 out_unlock:
2594         mutex_unlock(&graph_lock);
2595
2596         return ret;
2597 }
2598
2599 static const struct file_operations ftrace_graph_fops = {
2600         .open           = ftrace_graph_open,
2601         .read           = seq_read,
2602         .write          = ftrace_graph_write,
2603         .release        = ftrace_graph_release,
2604 };
2605 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
2606
2607 static __init int ftrace_init_dyn_debugfs(struct dentry *d_tracer)
2608 {
2609
2610         trace_create_file("available_filter_functions", 0444,
2611                         d_tracer, NULL, &ftrace_avail_fops);
2612
2613         trace_create_file("failures", 0444,
2614                         d_tracer, NULL, &ftrace_failures_fops);
2615
2616         trace_create_file("set_ftrace_filter", 0644, d_tracer,
2617                         NULL, &ftrace_filter_fops);
2618
2619         trace_create_file("set_ftrace_notrace", 0644, d_tracer,
2620                                     NULL, &ftrace_notrace_fops);
2621
2622 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
2623         trace_create_file("set_graph_function", 0444, d_tracer,
2624                                     NULL,
2625                                     &ftrace_graph_fops);
2626 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
2627
2628         return 0;
2629 }
2630
2631 static int ftrace_process_locs(struct module *mod,
2632                                unsigned long *start,
2633                                unsigned long *end)
2634 {
2635         unsigned long *p;
2636         unsigned long addr;
2637         unsigned long flags;
2638
2639         mutex_lock(&ftrace_lock);
2640         p = start;
2641         while (p < end) {
2642                 addr = ftrace_call_adjust(*p++);
2643                 /*
2644                  * Some architecture linkers will pad between
2645                  * the different mcount_loc sections of different
2646                  * object files to satisfy alignments.
2647                  * Skip any NULL pointers.
2648                  */
2649                 if (!addr)
2650                         continue;
2651                 ftrace_record_ip(addr);
2652         }
2653
2654         /* disable interrupts to prevent kstop machine */
2655         local_irq_save(flags);
2656         ftrace_update_code(mod);
2657         local_irq_restore(flags);
2658         mutex_unlock(&ftrace_lock);
2659
2660         return 0;
2661 }
2662
2663 #ifdef CONFIG_MODULES
2664 void ftrace_release_mod(struct module *mod)
2665 {
2666         struct dyn_ftrace *rec;
2667         struct ftrace_page *pg;
2668
2669         if (ftrace_disabled)
2670                 return;
2671
2672         mutex_lock(&ftrace_lock);
2673         do_for_each_ftrace_rec(pg, rec) {
2674                 if (within_module_core(rec->ip, mod)) {
2675                         /*
2676                          * rec->ip is changed in ftrace_free_rec()
2677                          * It should not between s and e if record was freed.
2678                          */
2679                         FTRACE_WARN_ON(rec->flags & FTRACE_FL_FREE);
2680                         ftrace_free_rec(rec);
2681                 }
2682         } while_for_each_ftrace_rec();
2683         mutex_unlock(&ftrace_lock);
2684 }
2685
2686 static void ftrace_init_module(struct module *mod,
2687                                unsigned long *start, unsigned long *end)
2688 {
2689         if (ftrace_disabled || start == end)
2690                 return;
2691         ftrace_process_locs(mod, start, end);
2692 }
2693
2694 static int ftrace_module_notify(struct notifier_block *self,
2695                                 unsigned long val, void *data)
2696 {
2697         struct module *mod = data;
2698
2699         switch (val) {
2700         case MODULE_STATE_COMING:
2701                 ftrace_init_module(mod, mod->ftrace_callsites,
2702                                    mod->ftrace_callsites +
2703                                    mod->num_ftrace_callsites);
2704                 break;
2705         case MODULE_STATE_GOING:
2706                 ftrace_release_mod(mod);
2707                 break;
2708         }
2709
2710         return 0;
2711 }
2712 #else
2713 static int ftrace_module_notify(struct notifier_block *self,
2714                                 unsigned long val, void *data)
2715 {
2716         return 0;
2717 }
2718 #endif /* CONFIG_MODULES */
2719
2720 struct notifier_block ftrace_module_nb = {
2721         .notifier_call = ftrace_module_notify,
2722         .priority = 0,
2723 };
2724
2725 extern unsigned long __start_mcount_loc[];
2726 extern unsigned long __stop_mcount_loc[];
2727
2728 void __init ftrace_init(void)
2729 {
2730         unsigned long count, addr, flags;
2731         int ret;
2732
2733         /* Keep the ftrace pointer to the stub */
2734         addr = (unsigned long)ftrace_stub;
2735
2736         local_irq_save(flags);
2737         ftrace_dyn_arch_init(&addr);
2738         local_irq_restore(flags);
2739
2740         /* ftrace_dyn_arch_init places the return code in addr */
2741         if (addr)
2742                 goto failed;
2743
2744         count = __stop_mcount_loc - __start_mcount_loc;
2745
2746         ret = ftrace_dyn_table_alloc(count);
2747         if (ret)
2748                 goto failed;
2749
2750         last_ftrace_enabled = ftrace_enabled = 1;
2751
2752         ret = ftrace_process_locs(NULL,
2753                                   __start_mcount_loc,
2754                                   __stop_mcount_loc);
2755
2756         ret = register_module_notifier(&ftrace_module_nb);
2757         if (ret)
2758                 pr_warning("Failed to register trace ftrace module notifier\n");
2759
2760         set_ftrace_early_filters();
2761
2762         return;
2763  failed:
2764         ftrace_disabled = 1;
2765 }
2766
2767 #else
2768
2769 static int __init ftrace_nodyn_init(void)
2770 {
2771         ftrace_enabled = 1;
2772         return 0;
2773 }
2774 device_initcall(ftrace_nodyn_init);
2775
2776 static inline int ftrace_init_dyn_debugfs(struct dentry *d_tracer) { return 0; }
2777 static inline void ftrace_startup_enable(int command) { }
2778 /* Keep as macros so we do not need to define the commands */
2779 # define ftrace_startup(command)        do { } while (0)
2780 # define ftrace_shutdown(command)       do { } while (0)
2781 # define ftrace_startup_sysctl()        do { } while (0)
2782 # define ftrace_shutdown_sysctl()       do { } while (0)
2783 #endif /* CONFIG_DYNAMIC_FTRACE */
2784
2785 static void clear_ftrace_swapper(void)
2786 {
2787         struct task_struct *p;
2788         int cpu;
2789
2790         get_online_cpus();
2791         for_each_online_cpu(cpu) {
2792                 p = idle_task(cpu);
2793                 clear_tsk_trace_trace(p);
2794         }
2795         put_online_cpus();
2796 }
2797
2798 static void set_ftrace_swapper(void)
2799 {
2800         struct task_struct *p;
2801         int cpu;
2802
2803         get_online_cpus();
2804         for_each_online_cpu(cpu) {
2805                 p = idle_task(cpu);
2806                 set_tsk_trace_trace(p);
2807         }
2808         put_online_cpus();
2809 }
2810
2811 static void clear_ftrace_pid(struct pid *pid)
2812 {
2813         struct task_struct *p;
2814
2815         rcu_read_lock();
2816         do_each_pid_task(pid, PIDTYPE_PID, p) {
2817                 clear_tsk_trace_trace(p);
2818         } while_each_pid_task(pid, PIDTYPE_PID, p);
2819         rcu_read_unlock();
2820
2821         put_pid(pid);
2822 }
2823
2824 static void set_ftrace_pid(struct pid *pid)
2825 {
2826         struct task_struct *p;
2827
2828         rcu_read_lock();
2829         do_each_pid_task(pid, PIDTYPE_PID, p) {
2830                 set_tsk_trace_trace(p);
2831         } while_each_pid_task(pid, PIDTYPE_PID, p);
2832         rcu_read_unlock();
2833 }
2834
2835 static void clear_ftrace_pid_task(struct pid *pid)
2836 {
2837         if (pid == ftrace_swapper_pid)
2838                 clear_ftrace_swapper();
2839         else
2840                 clear_ftrace_pid(pid);
2841 }
2842
2843 static void set_ftrace_pid_task(struct pid *pid)
2844 {
2845         if (pid == ftrace_swapper_pid)
2846                 set_ftrace_swapper();
2847         else
2848                 set_ftrace_pid(pid);
2849 }
2850
2851 static int ftrace_pid_add(int p)
2852 {
2853         struct pid *pid;
2854         struct ftrace_pid *fpid;
2855         int ret = -EINVAL;
2856
2857         mutex_lock(&ftrace_lock);
2858
2859         if (!p)
2860                 pid = ftrace_swapper_pid;
2861         else
2862                 pid = find_get_pid(p);
2863
2864         if (!pid)
2865                 goto out;
2866
2867         ret = 0;
2868
2869         list_for_each_entry(fpid, &ftrace_pids, list)
2870                 if (fpid->pid == pid)
2871                         goto out_put;
2872
2873         ret = -ENOMEM;
2874
2875         fpid = kmalloc(sizeof(*fpid), GFP_KERNEL);
2876         if (!fpid)
2877                 goto out_put;
2878
2879         list_add(&fpid->list, &ftrace_pids);
2880         fpid->pid = pid;
2881
2882         set_ftrace_pid_task(pid);
2883
2884         ftrace_update_pid_func();
2885         ftrace_startup_enable(0);
2886
2887         mutex_unlock(&ftrace_lock);
2888         return 0;
2889
2890 out_put:
2891         if (pid != ftrace_swapper_pid)
2892                 put_pid(pid);
2893
2894 out:
2895         mutex_unlock(&ftrace_lock);
2896         return ret;
2897 }
2898
2899 static void ftrace_pid_reset(void)
2900 {
2901         struct ftrace_pid *fpid, *safe;
2902
2903         mutex_lock(&ftrace_lock);
2904         list_for_each_entry_safe(fpid, safe, &ftrace_pids, list) {
2905                 struct pid *pid = fpid->pid;
2906
2907                 clear_ftrace_pid_task(pid);
2908
2909                 list_del(&fpid->list);
2910                 kfree(fpid);
2911         }
2912
2913         ftrace_update_pid_func();
2914         ftrace_startup_enable(0);
2915
2916         mutex_unlock(&ftrace_lock);
2917 }
2918
2919 static void *fpid_start(struct seq_file *m, loff_t *pos)
2920 {
2921         mutex_lock(&ftrace_lock);
2922
2923         if (list_empty(&ftrace_pids) && (!*pos))
2924                 return (void *) 1;
2925
2926         return seq_list_start(&ftrace_pids, *pos);
2927 }
2928
2929 static void *fpid_next(struct seq_file *m, void *v, loff_t *pos)
2930 {
2931         if (v == (void *)1)
2932                 return NULL;
2933
2934         return seq_list_next(v, &ftrace_pids, pos);
2935 }
2936
2937 static void fpid_stop(struct seq_file *m, void *p)
2938 {
2939         mutex_unlock(&ftrace_lock);
2940 }
2941
2942 static int fpid_show(struct seq_file *m, void *v)
2943 {
2944         const struct ftrace_pid *fpid = list_entry(v, struct ftrace_pid, list);
2945
2946         if (v == (void *)1) {
2947                 seq_printf(m, "no pid\n");
2948                 return 0;
2949         }
2950
2951         if (fpid->pid == ftrace_swapper_pid)
2952                 seq_printf(m, "swapper tasks\n");
2953         else
2954                 seq_printf(m, "%u\n", pid_vnr(fpid->pid));
2955
2956         return 0;
2957 }
2958
2959 static const struct seq_operations ftrace_pid_sops = {
2960         .start = fpid_start,
2961         .next = fpid_next,
2962         .stop = fpid_stop,
2963         .show = fpid_show,
2964 };
2965
2966 static int
2967 ftrace_pid_open(struct inode *inode, struct file *file)
2968 {
2969         int ret = 0;
2970
2971         if ((file->f_mode & FMODE_WRITE) &&
2972             (file->f_flags & O_TRUNC))
2973                 ftrace_pid_reset();
2974
2975         if (file->f_mode & FMODE_READ)
2976                 ret = seq_open(file, &ftrace_pid_sops);
2977
2978         return ret;
2979 }
2980
2981 static ssize_t
2982 ftrace_pid_write(struct file *filp, const char __user *ubuf,
2983                    size_t cnt, loff_t *ppos)
2984 {
2985         char buf[64], *tmp;
2986         long val;
2987         int ret;
2988
2989         if (cnt >= sizeof(buf))
2990                 return -EINVAL;
2991
2992         if (copy_from_user(&buf, ubuf, cnt))
2993                 return -EFAULT;
2994
2995         buf[cnt] = 0;
2996
2997         /*
2998          * Allow "echo > set_ftrace_pid" or "echo -n '' > set_ftrace_pid"
2999          * to clean the filter quietly.
3000          */
3001         tmp = strstrip(buf);
3002         if (strlen(tmp) == 0)
3003                 return 1;
3004
3005         ret = strict_strtol(tmp, 10, &val);
3006         if (ret < 0)
3007                 return ret;
3008
3009         ret = ftrace_pid_add(val);
3010
3011         return ret ? ret : cnt;
3012 }
3013
3014 static int
3015 ftrace_pid_release(struct inode *inode, struct file *file)
3016 {
3017         if (file->f_mode & FMODE_READ)
3018                 seq_release(inode, file);
3019
3020         return 0;
3021 }
3022
3023 static const struct file_operations ftrace_pid_fops = {
3024         .open           = ftrace_pid_open,
3025         .write          = ftrace_pid_write,
3026         .read           = seq_read,
3027         .llseek         = seq_lseek,
3028         .release        = ftrace_pid_release,
3029 };
3030
3031 static __init int ftrace_init_debugfs(void)
3032 {
3033         struct dentry *d_tracer;
3034
3035         d_tracer = tracing_init_dentry();
3036         if (!d_tracer)
3037                 return 0;
3038
3039         ftrace_init_dyn_debugfs(d_tracer);
3040
3041         trace_create_file("set_ftrace_pid", 0644, d_tracer,
3042                             NULL, &ftrace_pid_fops);
3043
3044         ftrace_profile_debugfs(d_tracer);
3045
3046         return 0;
3047 }
3048 fs_initcall(ftrace_init_debugfs);
3049
3050 /**
3051  * ftrace_kill - kill ftrace
3052  *
3053  * This function should be used by panic code. It stops ftrace
3054  * but in a not so nice way. If you need to simply kill ftrace
3055  * from a non-atomic section, use ftrace_kill.
3056  */
3057 void ftrace_kill(void)
3058 {
3059         ftrace_disabled = 1;
3060         ftrace_enabled = 0;
3061         clear_ftrace_function();
3062 }
3063
3064 /**
3065  * register_ftrace_function - register a function for profiling
3066  * @ops - ops structure that holds the function for profiling.
3067  *
3068  * Register a function to be called by all functions in the
3069  * kernel.
3070  *
3071  * Note: @ops->func and all the functions it calls must be labeled
3072  *       with "notrace", otherwise it will go into a
3073  *       recursive loop.
3074  */
3075 int register_ftrace_function(struct ftrace_ops *ops)
3076 {
3077         int ret;
3078
3079         if (unlikely(ftrace_disabled))
3080                 return -1;
3081
3082         mutex_lock(&ftrace_lock);
3083
3084         ret = __register_ftrace_function(ops);
3085         ftrace_startup(0);
3086
3087         mutex_unlock(&ftrace_lock);
3088         return ret;
3089 }
3090
3091 /**
3092  * unregister_ftrace_function - unregister a function for profiling.
3093  * @ops - ops structure that holds the function to unregister
3094  *
3095  * Unregister a function that was added to be called by ftrace profiling.
3096  */
3097 int unregister_ftrace_function(struct ftrace_ops *ops)
3098 {
3099         int ret;
3100
3101         mutex_lock(&ftrace_lock);
3102         ret = __unregister_ftrace_function(ops);
3103         ftrace_shutdown(0);
3104         mutex_unlock(&ftrace_lock);
3105
3106         return ret;
3107 }
3108
3109 int
3110 ftrace_enable_sysctl(struct ctl_table *table, int write,
3111                      void __user *buffer, size_t *lenp,
3112                      loff_t *ppos)
3113 {
3114         int ret;
3115
3116         if (unlikely(ftrace_disabled))
3117                 return -ENODEV;
3118
3119         mutex_lock(&ftrace_lock);
3120
3121         ret  = proc_dointvec(table, write, buffer, lenp, ppos);
3122
3123         if (ret || !write || (last_ftrace_enabled == !!ftrace_enabled))
3124                 goto out;
3125
3126         last_ftrace_enabled = !!ftrace_enabled;
3127
3128         if (ftrace_enabled) {
3129
3130                 ftrace_startup_sysctl();
3131
3132                 /* we are starting ftrace again */
3133                 if (ftrace_list != &ftrace_list_end) {
3134                         if (ftrace_list->next == &ftrace_list_end)
3135                                 ftrace_trace_function = ftrace_list->func;
3136                         else
3137                                 ftrace_trace_function = ftrace_list_func;
3138                 }
3139
3140         } else {
3141                 /* stopping ftrace calls (just send to ftrace_stub) */
3142                 ftrace_trace_function = ftrace_stub;
3143
3144                 ftrace_shutdown_sysctl();
3145         }
3146
3147  out:
3148         mutex_unlock(&ftrace_lock);
3149         return ret;
3150 }
3151
3152 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
3153
3154 static int ftrace_graph_active;
3155 static struct notifier_block ftrace_suspend_notifier;
3156
3157 int ftrace_graph_entry_stub(struct ftrace_graph_ent *trace)
3158 {
3159         return 0;
3160 }
3161
3162 /* The callbacks that hook a function */
3163 trace_func_graph_ret_t ftrace_graph_return =
3164                         (trace_func_graph_ret_t)ftrace_stub;
3165 trace_func_graph_ent_t ftrace_graph_entry = ftrace_graph_entry_stub;
3166
3167 /* Try to assign a return stack array on FTRACE_RETSTACK_ALLOC_SIZE tasks. */
3168 static int alloc_retstack_tasklist(struct ftrace_ret_stack **ret_stack_list)
3169 {
3170         int i;
3171         int ret = 0;
3172         unsigned long flags;
3173         int start = 0, end = FTRACE_RETSTACK_ALLOC_SIZE;
3174         struct task_struct *g, *t;
3175
3176         for (i = 0; i < FTRACE_RETSTACK_ALLOC_SIZE; i++) {
3177                 ret_stack_list[i] = kmalloc(FTRACE_RETFUNC_DEPTH
3178                                         * sizeof(struct ftrace_ret_stack),
3179                                         GFP_KERNEL);
3180                 if (!ret_stack_list[i]) {
3181                         start = 0;
3182                         end = i;
3183                         ret = -ENOMEM;
3184                         goto free;
3185                 }
3186         }
3187
3188         read_lock_irqsave(&tasklist_lock, flags);
3189         do_each_thread(g, t) {
3190                 if (start == end) {
3191                         ret = -EAGAIN;
3192                         goto unlock;
3193                 }
3194
3195                 if (t->ret_stack == NULL) {
3196                         atomic_set(&t->tracing_graph_pause, 0);
3197                         atomic_set(&t->trace_overrun, 0);
3198                         t->curr_ret_stack = -1;
3199                         /* Make sure the tasks see the -1 first: */
3200                         smp_wmb();
3201                         t->ret_stack = ret_stack_list[start++];
3202                 }
3203         } while_each_thread(g, t);
3204
3205 unlock:
3206         read_unlock_irqrestore(&tasklist_lock, flags);
3207 free:
3208         for (i = start; i < end; i++)
3209                 kfree(ret_stack_list[i]);
3210         return ret;
3211 }
3212
3213 static void
3214 ftrace_graph_probe_sched_switch(struct rq *__rq, struct task_struct *prev,
3215                                 struct task_struct *next)
3216 {
3217         unsigned long long timestamp;
3218         int index;
3219
3220         /*
3221          * Does the user want to count the time a function was asleep.
3222          * If so, do not update the time stamps.
3223          */
3224         if (trace_flags & TRACE_ITER_SLEEP_TIME)
3225                 return;
3226
3227         timestamp = trace_clock_local();
3228
3229         prev->ftrace_timestamp = timestamp;
3230
3231         /* only process tasks that we timestamped */
3232         if (!next->ftrace_timestamp)
3233                 return;
3234
3235         /*
3236          * Update all the counters in next to make up for the
3237          * time next was sleeping.
3238          */
3239         timestamp -= next->ftrace_timestamp;
3240
3241         for (index = next->curr_ret_stack; index >= 0; index--)
3242                 next->ret_stack[index].calltime += timestamp;
3243 }
3244
3245 /* Allocate a return stack for each task */
3246 static int start_graph_tracing(void)
3247 {
3248         struct ftrace_ret_stack **ret_stack_list;
3249         int ret, cpu;
3250
3251         ret_stack_list = kmalloc(FTRACE_RETSTACK_ALLOC_SIZE *
3252                                 sizeof(struct ftrace_ret_stack *),
3253                                 GFP_KERNEL);
3254
3255         if (!ret_stack_list)
3256                 return -ENOMEM;
3257
3258         /* The cpu_boot init_task->ret_stack will never be freed */
3259         for_each_online_cpu(cpu) {
3260                 if (!idle_task(cpu)->ret_stack)
3261                         ftrace_graph_init_task(idle_task(cpu));
3262         }
3263
3264         do {
3265                 ret = alloc_retstack_tasklist(ret_stack_list);
3266         } while (ret == -EAGAIN);
3267
3268         if (!ret) {
3269                 ret = register_trace_sched_switch(ftrace_graph_probe_sched_switch);
3270                 if (ret)
3271                         pr_info("ftrace_graph: Couldn't activate tracepoint"
3272                                 " probe to kernel_sched_switch\n");
3273         }
3274
3275         kfree(ret_stack_list);
3276         return ret;
3277 }
3278
3279 /*
3280  * Hibernation protection.
3281  * The state of the current task is too much unstable during
3282  * suspend/restore to disk. We want to protect against that.
3283  */
3284 static int
3285 ftrace_suspend_notifier_call(struct notifier_block *bl, unsigned long state,
3286                                                         void *unused)
3287 {
3288         switch (state) {
3289         case PM_HIBERNATION_PREPARE:
3290                 pause_graph_tracing();
3291                 break;
3292
3293         case PM_POST_HIBERNATION:
3294                 unpause_graph_tracing();
3295                 break;
3296         }
3297         return NOTIFY_DONE;
3298 }
3299
3300 int register_ftrace_graph(trace_func_graph_ret_t retfunc,
3301                         trace_func_graph_ent_t entryfunc)
3302 {
3303         int ret = 0;
3304
3305         mutex_lock(&ftrace_lock);
3306
3307         /* we currently allow only one tracer registered at a time */
3308         if (ftrace_graph_active) {
3309                 ret = -EBUSY;
3310                 goto out;
3311         }
3312
3313         ftrace_suspend_notifier.notifier_call = ftrace_suspend_notifier_call;
3314         register_pm_notifier(&ftrace_suspend_notifier);
3315
3316         ftrace_graph_active++;
3317         ret = start_graph_tracing();
3318         if (ret) {
3319                 ftrace_graph_active--;
3320                 goto out;
3321         }
3322
3323         ftrace_graph_return = retfunc;
3324         ftrace_graph_entry = entryfunc;
3325
3326         ftrace_startup(FTRACE_START_FUNC_RET);
3327
3328 out:
3329         mutex_unlock(&ftrace_lock);
3330         return ret;
3331 }
3332
3333 void unregister_ftrace_graph(void)
3334 {
3335         mutex_lock(&ftrace_lock);
3336
3337         if (unlikely(!ftrace_graph_active))
3338                 goto out;
3339
3340         ftrace_graph_active--;
3341         unregister_trace_sched_switch(ftrace_graph_probe_sched_switch);
3342         ftrace_graph_return = (trace_func_graph_ret_t)ftrace_stub;
3343         ftrace_graph_entry = ftrace_graph_entry_stub;
3344         ftrace_shutdown(FTRACE_STOP_FUNC_RET);
3345         unregister_pm_notifier(&ftrace_suspend_notifier);
3346
3347  out:
3348         mutex_unlock(&ftrace_lock);
3349 }
3350
3351 /* Allocate a return stack for newly created task */
3352 void ftrace_graph_init_task(struct task_struct *t)
3353 {
3354         /* Make sure we do not use the parent ret_stack */
3355         t->ret_stack = NULL;
3356         t->curr_ret_stack = -1;
3357
3358         if (ftrace_graph_active) {
3359                 struct ftrace_ret_stack *ret_stack;
3360
3361                 ret_stack = kmalloc(FTRACE_RETFUNC_DEPTH
3362                                 * sizeof(struct ftrace_ret_stack),
3363                                 GFP_KERNEL);
3364                 if (!ret_stack)
3365                         return;
3366                 atomic_set(&t->tracing_graph_pause, 0);
3367                 atomic_set(&t->trace_overrun, 0);
3368                 t->ftrace_timestamp = 0;
3369                 /* make curr_ret_stack visable before we add the ret_stack */
3370                 smp_wmb();
3371                 t->ret_stack = ret_stack;
3372         }
3373 }
3374
3375 void ftrace_graph_exit_task(struct task_struct *t)
3376 {
3377         struct ftrace_ret_stack *ret_stack = t->ret_stack;
3378
3379         t->ret_stack = NULL;
3380         /* NULL must become visible to IRQs before we free it: */
3381         barrier();
3382
3383         kfree(ret_stack);
3384 }
3385
3386 void ftrace_graph_stop(void)
3387 {
3388         ftrace_stop();
3389 }
3390 #endif