SAFE public projects git trees. - safe/jmp/linux-2.6/blob - kernel/trace/ftrace.c

   1 /*
   2  * Infrastructure for profiling code inserted by 'gcc -pg'.
   3  *
   4  * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
   5  * Copyright (C) 2004-2008 Ingo Molnar <mingo@redhat.com>
   6  *
   7  * Originally ported from the -rt patch by:
   8  *   Copyright (C) 2007 Arnaldo Carvalho de Melo <acme@redhat.com>
   9  *
  10  * Based on code in the latency_tracer, that is:
  11  *
  12  *  Copyright (C) 2004-2006 Ingo Molnar
  13  *  Copyright (C) 2004 William Lee Irwin III
  14  */
  15
  16 #include <linux/stop_machine.h>
  17 #include <linux/clocksource.h>
  18 #include <linux/kallsyms.h>
  19 #include <linux/seq_file.h>
  20 #include <linux/suspend.h>
  21 #include <linux/debugfs.h>
  22 #include <linux/hardirq.h>
  23 #include <linux/kthread.h>
  24 #include <linux/uaccess.h>
  25 #include <linux/kprobes.h>
  26 #include <linux/ftrace.h>
  27 #include <linux/sysctl.h>
  28 #include <linux/ctype.h>
  29 #include <linux/list.h>
  30 #include <linux/hash.h>
  31
  32 #include <trace/sched.h>
  33
  34 #include <asm/ftrace.h>
  35
  36 #include "trace_output.h"
  37 #include "trace_stat.h"
  38
  39 #define FTRACE_WARN_ON(cond)                    \
  40         do {                                    \
  41                 if (WARN_ON(cond))              \
  42                         ftrace_kill();          \
  43         } while (0)
  44
  45 #define FTRACE_WARN_ON_ONCE(cond)               \
  46         do {                                    \
  47                 if (WARN_ON_ONCE(cond))         \
  48                         ftrace_kill();          \
  49         } while (0)
  50
  51 /* hash bits for specific function selection */
  52 #define FTRACE_HASH_BITS 7
  53 #define FTRACE_FUNC_HASHSIZE (1 << FTRACE_HASH_BITS)
  54
  55 /* ftrace_enabled is a method to turn ftrace on or off */
  56 int ftrace_enabled __read_mostly;
  57 static int last_ftrace_enabled;
  58
  59 /* Quick disabling of function tracer. */
  60 int function_trace_stop;
  61
  62 /*
  63  * ftrace_disabled is set when an anomaly is discovered.
  64  * ftrace_disabled is much stronger than ftrace_enabled.
  65  */
  66 static int ftrace_disabled __read_mostly;
  67
  68 static DEFINE_MUTEX(ftrace_lock);
  69
  70 static struct ftrace_ops ftrace_list_end __read_mostly =
  71 {
  72         .func           = ftrace_stub,
  73 };
  74
  75 static struct ftrace_ops *ftrace_list __read_mostly = &ftrace_list_end;
  76 ftrace_func_t ftrace_trace_function __read_mostly = ftrace_stub;
  77 ftrace_func_t __ftrace_trace_function __read_mostly = ftrace_stub;
  78 ftrace_func_t ftrace_pid_function __read_mostly = ftrace_stub;
  79
  80 static void ftrace_list_func(unsigned long ip, unsigned long parent_ip)
  81 {
  82         struct ftrace_ops *op = ftrace_list;
  83
  84         /* in case someone actually ports this to alpha! */
  85         read_barrier_depends();
  86
  87         while (op != &ftrace_list_end) {
  88                 /* silly alpha */
  89                 read_barrier_depends();
  90                 op->func(ip, parent_ip);
  91                 op = op->next;
  92         };
  93 }
  94
  95 static void ftrace_pid_func(unsigned long ip, unsigned long parent_ip)
  96 {
  97         if (!test_tsk_trace_trace(current))
  98                 return;
  99
 100         ftrace_pid_function(ip, parent_ip);
 101 }
 102
 103 static void set_ftrace_pid_function(ftrace_func_t func)
 104 {
 105         /* do not set ftrace_pid_function to itself! */
 106         if (func != ftrace_pid_func)
 107                 ftrace_pid_function = func;
 108 }
 109
 110 /**
 111  * clear_ftrace_function - reset the ftrace function
 112  *
 113  * This NULLs the ftrace function and in essence stops
 114  * tracing.  There may be lag
 115  */
 116 void clear_ftrace_function(void)
 117 {
 118         ftrace_trace_function = ftrace_stub;
 119         __ftrace_trace_function = ftrace_stub;
 120         ftrace_pid_function = ftrace_stub;
 121 }
 122
 123 #ifndef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST
 124 /*
 125  * For those archs that do not test ftrace_trace_stop in their
 126  * mcount call site, we need to do it from C.
 127  */
 128 static void ftrace_test_stop_func(unsigned long ip, unsigned long parent_ip)
 129 {
 130         if (function_trace_stop)
 131                 return;
 132
 133         __ftrace_trace_function(ip, parent_ip);
 134 }
 135 #endif
 136
 137 static int __register_ftrace_function(struct ftrace_ops *ops)
 138 {
 139         ops->next = ftrace_list;
 140         /*
 141          * We are entering ops into the ftrace_list but another
 142          * CPU might be walking that list. We need to make sure
 143          * the ops->next pointer is valid before another CPU sees
 144          * the ops pointer included into the ftrace_list.
 145          */
 146         smp_wmb();
 147         ftrace_list = ops;
 148
 149         if (ftrace_enabled) {
 150                 ftrace_func_t func;
 151
 152                 if (ops->next == &ftrace_list_end)
 153                         func = ops->func;
 154                 else
 155                         func = ftrace_list_func;
 156
 157                 if (ftrace_pid_trace) {
 158                         set_ftrace_pid_function(func);
 159                         func = ftrace_pid_func;
 160                 }
 161
 162                 /*
 163                  * For one func, simply call it directly.
 164                  * For more than one func, call the chain.
 165                  */
 166 #ifdef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST
 167                 ftrace_trace_function = func;
 168 #else
 169                 __ftrace_trace_function = func;
 170                 ftrace_trace_function = ftrace_test_stop_func;
 171 #endif
 172         }
 173
 174         return 0;
 175 }
 176
 177 static int __unregister_ftrace_function(struct ftrace_ops *ops)
 178 {
 179         struct ftrace_ops **p;
 180
 181         /*
 182          * If we are removing the last function, then simply point
 183          * to the ftrace_stub.
 184          */
 185         if (ftrace_list == ops && ops->next == &ftrace_list_end) {
 186                 ftrace_trace_function = ftrace_stub;
 187                 ftrace_list = &ftrace_list_end;
 188                 return 0;
 189         }
 190
 191         for (p = &ftrace_list; *p != &ftrace_list_end; p = &(*p)->next)
 192                 if (*p == ops)
 193                         break;
 194
 195         if (*p != ops)
 196                 return -1;
 197
 198         *p = (*p)->next;
 199
 200         if (ftrace_enabled) {
 201                 /* If we only have one func left, then call that directly */
 202                 if (ftrace_list->next == &ftrace_list_end) {
 203                         ftrace_func_t func = ftrace_list->func;
 204
 205                         if (ftrace_pid_trace) {
 206                                 set_ftrace_pid_function(func);
 207                                 func = ftrace_pid_func;
 208                         }
 209 #ifdef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST
 210                         ftrace_trace_function = func;
 211 #else
 212                         __ftrace_trace_function = func;
 213 #endif
 214                 }
 215         }
 216
 217         return 0;
 218 }
 219
 220 static void ftrace_update_pid_func(void)
 221 {
 222         ftrace_func_t func;
 223
 224         if (ftrace_trace_function == ftrace_stub)
 225                 return;
 226
 227         func = ftrace_trace_function;
 228
 229         if (ftrace_pid_trace) {
 230                 set_ftrace_pid_function(func);
 231                 func = ftrace_pid_func;
 232         } else {
 233                 if (func == ftrace_pid_func)
 234                         func = ftrace_pid_function;
 235         }
 236
 237 #ifdef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST
 238         ftrace_trace_function = func;
 239 #else
 240         __ftrace_trace_function = func;
 241 #endif
 242 }
 243
 244 #ifdef CONFIG_FUNCTION_PROFILER
 245 struct ftrace_profile {
 246         struct hlist_node               node;
 247         unsigned long                   ip;
 248         unsigned long                   counter;
 249 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
 250         unsigned long long              time;
 251 #endif
 252 };
 253
 254 struct ftrace_profile_page {
 255         struct ftrace_profile_page      *next;
 256         unsigned long                   index;
 257         struct ftrace_profile           records[];
 258 };
 259
 260 struct ftrace_profile_stat {
 261         atomic_t                        disabled;
 262         struct hlist_head               *hash;
 263         struct ftrace_profile_page      *pages;
 264         struct ftrace_profile_page      *start;
 265         struct tracer_stat              stat;
 266 };
 267
 268 #define PROFILE_RECORDS_SIZE                                            \
 269         (PAGE_SIZE - offsetof(struct ftrace_profile_page, records))
 270
 271 #define PROFILES_PER_PAGE                                       \
 272         (PROFILE_RECORDS_SIZE / sizeof(struct ftrace_profile))
 273
 274 static int ftrace_profile_bits __read_mostly;
 275 static int ftrace_profile_enabled __read_mostly;
 276
 277 /* ftrace_profile_lock - synchronize the enable and disable of the profiler */
 278 static DEFINE_MUTEX(ftrace_profile_lock);
 279
 280 static DEFINE_PER_CPU(struct ftrace_profile_stat, ftrace_profile_stats);
 281
 282 #define FTRACE_PROFILE_HASH_SIZE 1024 /* must be power of 2 */
 283
 284 static void *
 285 function_stat_next(void *v, int idx)
 286 {
 287         struct ftrace_profile *rec = v;
 288         struct ftrace_profile_page *pg;
 289
 290         pg = (struct ftrace_profile_page *)((unsigned long)rec & PAGE_MASK);
 291
 292  again:
 293         rec++;
 294         if ((void *)rec >= (void *)&pg->records[pg->index]) {
 295                 pg = pg->next;
 296                 if (!pg)
 297                         return NULL;
 298                 rec = &pg->records[0];
 299                 if (!rec->counter)
 300                         goto again;
 301         }
 302
 303         return rec;
 304 }
 305
 306 static void *function_stat_start(struct tracer_stat *trace)
 307 {
 308         struct ftrace_profile_stat *stat =
 309                 container_of(trace, struct ftrace_profile_stat, stat);
 310
 311         if (!stat || !stat->start)
 312                 return NULL;
 313
 314         return function_stat_next(&stat->start->records[0], 0);
 315 }
 316
 317 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
 318 /* function graph compares on total time */
 319 static int function_stat_cmp(void *p1, void *p2)
 320 {
 321         struct ftrace_profile *a = p1;
 322         struct ftrace_profile *b = p2;
 323
 324         if (a->time < b->time)
 325                 return -1;
 326         if (a->time > b->time)
 327                 return 1;
 328         else
 329                 return 0;
 330 }
 331 #else
 332 /* not function graph compares against hits */
 333 static int function_stat_cmp(void *p1, void *p2)
 334 {
 335         struct ftrace_profile *a = p1;
 336         struct ftrace_profile *b = p2;
 337
 338         if (a->counter < b->counter)
 339                 return -1;
 340         if (a->counter > b->counter)
 341                 return 1;
 342         else
 343                 return 0;
 344 }
 345 #endif
 346
 347 static int function_stat_headers(struct seq_file *m)
 348 {
 349 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
 350         seq_printf(m, "  Function                               "
 351                    "Hit    Time            Avg\n"
 352                       "  --------                               "
 353                    "---    ----            ---\n");
 354 #else
 355         seq_printf(m, "  Function                               Hit\n"
 356                       "  --------                               ---\n");
 357 #endif
 358         return 0;
 359 }
 360
 361 static int function_stat_show(struct seq_file *m, void *v)
 362 {
 363         struct ftrace_profile *rec = v;
 364         char str[KSYM_SYMBOL_LEN];
 365 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
 366         static DEFINE_MUTEX(mutex);
 367         static struct trace_seq s;
 368         unsigned long long avg;
 369 #endif
 370
 371         kallsyms_lookup(rec->ip, NULL, NULL, NULL, str);
 372         seq_printf(m, "  %-30.30s  %10lu", str, rec->counter);
 373
 374 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
 375         seq_printf(m, "    ");
 376         avg = rec->time;
 377         do_div(avg, rec->counter);
 378
 379         mutex_lock(&mutex);
 380         trace_seq_init(&s);
 381         trace_print_graph_duration(rec->time, &s);
 382         trace_seq_puts(&s, "    ");
 383         trace_print_graph_duration(avg, &s);
 384         trace_print_seq(m, &s);
 385         mutex_unlock(&mutex);
 386 #endif
 387         seq_putc(m, '\n');
 388
 389         return 0;
 390 }
 391
 392 static void ftrace_profile_reset(struct ftrace_profile_stat *stat)
 393 {
 394         struct ftrace_profile_page *pg;
 395
 396         pg = stat->pages = stat->start;
 397
 398         while (pg) {
 399                 memset(pg->records, 0, PROFILE_RECORDS_SIZE);
 400                 pg->index = 0;
 401                 pg = pg->next;
 402         }
 403
 404         memset(stat->hash, 0,
 405                FTRACE_PROFILE_HASH_SIZE * sizeof(struct hlist_head));
 406 }
 407
 408 int ftrace_profile_pages_init(struct ftrace_profile_stat *stat)
 409 {
 410         struct ftrace_profile_page *pg;
 411         int functions;
 412         int pages;
 413         int i;
 414
 415         /* If we already allocated, do nothing */
 416         if (stat->pages)
 417                 return 0;
 418
 419         stat->pages = (void *)get_zeroed_page(GFP_KERNEL);
 420         if (!stat->pages)
 421                 return -ENOMEM;
 422
 423 #ifdef CONFIG_DYNAMIC_FTRACE
 424         functions = ftrace_update_tot_cnt;
 425 #else
 426         /*
 427          * We do not know the number of functions that exist because
 428          * dynamic tracing is what counts them. With past experience
 429          * we have around 20K functions. That should be more than enough.
 430          * It is highly unlikely we will execute every function in
 431          * the kernel.
 432          */
 433         functions = 20000;
 434 #endif
 435
 436         pg = stat->start = stat->pages;
 437
 438         pages = DIV_ROUND_UP(functions, PROFILES_PER_PAGE);
 439
 440         for (i = 0; i < pages; i++) {
 441                 pg->next = (void *)get_zeroed_page(GFP_KERNEL);
 442                 if (!pg->next)
 443                         goto out_free;
 444                 pg = pg->next;
 445         }
 446
 447         return 0;
 448
 449  out_free:
 450         pg = stat->start;
 451         while (pg) {
 452                 unsigned long tmp = (unsigned long)pg;
 453
 454                 pg = pg->next;
 455                 free_page(tmp);
 456         }
 457
 458         free_page((unsigned long)stat->pages);
 459         stat->pages = NULL;
 460         stat->start = NULL;
 461
 462         return -ENOMEM;
 463 }
 464
 465 static int ftrace_profile_init_cpu(int cpu)
 466 {
 467         struct ftrace_profile_stat *stat;
 468         int size;
 469
 470         stat = &per_cpu(ftrace_profile_stats, cpu);
 471
 472         if (stat->hash) {
 473                 /* If the profile is already created, simply reset it */
 474                 ftrace_profile_reset(stat);
 475                 return 0;
 476         }
 477
 478         /*
 479          * We are profiling all functions, but usually only a few thousand
 480          * functions are hit. We'll make a hash of 1024 items.
 481          */
 482         size = FTRACE_PROFILE_HASH_SIZE;
 483
 484         stat->hash = kzalloc(sizeof(struct hlist_head) * size, GFP_KERNEL);
 485
 486         if (!stat->hash)
 487                 return -ENOMEM;
 488
 489         if (!ftrace_profile_bits) {
 490                 size--;
 491
 492                 for (; size; size >>= 1)
 493                         ftrace_profile_bits++;
 494         }
 495
 496         /* Preallocate the function profiling pages */
 497         if (ftrace_profile_pages_init(stat) < 0) {
 498                 kfree(stat->hash);
 499                 stat->hash = NULL;
 500                 return -ENOMEM;
 501         }
 502
 503         return 0;
 504 }
 505
 506 static int ftrace_profile_init(void)
 507 {
 508         int cpu;
 509         int ret = 0;
 510
 511         for_each_online_cpu(cpu) {
 512                 ret = ftrace_profile_init_cpu(cpu);
 513                 if (ret)
 514                         break;
 515         }
 516
 517         return ret;
 518 }
 519
 520 /* interrupts must be disabled */
 521 static struct ftrace_profile *
 522 ftrace_find_profiled_func(struct ftrace_profile_stat *stat, unsigned long ip)
 523 {
 524         struct ftrace_profile *rec;
 525         struct hlist_head *hhd;
 526         struct hlist_node *n;
 527         unsigned long key;
 528
 529         key = hash_long(ip, ftrace_profile_bits);
 530         hhd = &stat->hash[key];
 531
 532         if (hlist_empty(hhd))
 533                 return NULL;
 534
 535         hlist_for_each_entry_rcu(rec, n, hhd, node) {
 536                 if (rec->ip == ip)
 537                         return rec;
 538         }
 539
 540         return NULL;
 541 }
 542
 543 static void ftrace_add_profile(struct ftrace_profile_stat *stat,
 544                                struct ftrace_profile *rec)
 545 {
 546         unsigned long key;
 547
 548         key = hash_long(rec->ip, ftrace_profile_bits);
 549         hlist_add_head_rcu(&rec->node, &stat->hash[key]);
 550 }
 551
 552 /*
 553  * The memory is already allocated, this simply finds a new record to use.
 554  */
 555 static struct ftrace_profile *
 556 ftrace_profile_alloc(struct ftrace_profile_stat *stat, unsigned long ip)
 557 {
 558         struct ftrace_profile *rec = NULL;
 559
 560         /* prevent recursion (from NMIs) */
 561         if (atomic_inc_return(&stat->disabled) != 1)
 562                 goto out;
 563
 564         /*
 565          * Try to find the function again since an NMI
 566          * could have added it
 567          */
 568         rec = ftrace_find_profiled_func(stat, ip);
 569         if (rec)
 570                 goto out;
 571
 572         if (stat->pages->index == PROFILES_PER_PAGE) {
 573                 if (!stat->pages->next)
 574                         goto out;
 575                 stat->pages = stat->pages->next;
 576         }
 577
 578         rec = &stat->pages->records[stat->pages->index++];
 579         rec->ip = ip;
 580         ftrace_add_profile(stat, rec);
 581
 582  out:
 583         atomic_dec(&stat->disabled);
 584
 585         return rec;
 586 }
 587
 588 static void
 589 function_profile_call(unsigned long ip, unsigned long parent_ip)
 590 {
 591         struct ftrace_profile_stat *stat;
 592         struct ftrace_profile *rec;
 593         unsigned long flags;
 594
 595         if (!ftrace_profile_enabled)
 596                 return;
 597
 598         local_irq_save(flags);
 599
 600         stat = &__get_cpu_var(ftrace_profile_stats);
 601         if (!stat->hash)
 602                 goto out;
 603
 604         rec = ftrace_find_profiled_func(stat, ip);
 605         if (!rec) {
 606                 rec = ftrace_profile_alloc(stat, ip);
 607                 if (!rec)
 608                         goto out;
 609         }
 610
 611         rec->counter++;
 612  out:
 613         local_irq_restore(flags);
 614 }
 615
 616 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
 617 static int profile_graph_entry(struct ftrace_graph_ent *trace)
 618 {
 619         function_profile_call(trace->func, 0);
 620         return 1;
 621 }
 622
 623 static void profile_graph_return(struct ftrace_graph_ret *trace)
 624 {
 625         struct ftrace_profile_stat *stat;
 626         unsigned long long calltime;
 627         struct ftrace_profile *rec;
 628         unsigned long flags;
 629
 630         local_irq_save(flags);
 631         stat = &__get_cpu_var(ftrace_profile_stats);
 632         if (!stat->hash)
 633                 goto out;
 634
 635         calltime = trace->rettime - trace->calltime;
 636
 637         if (!(trace_flags & TRACE_ITER_GRAPH_TIME)) {
 638                 int index;
 639
 640                 index = trace->depth;
 641
 642                 /* Append this call time to the parent time to subtract */
 643                 if (index)
 644                         current->ret_stack[index - 1].subtime += calltime;
 645
 646                 if (current->ret_stack[index].subtime < calltime)
 647                         calltime -= current->ret_stack[index].subtime;
 648                 else
 649                         calltime = 0;
 650         }
 651
 652         rec = ftrace_find_profiled_func(stat, trace->func);
 653         if (rec)
 654                 rec->time += calltime;
 655
 656  out:
 657         local_irq_restore(flags);
 658 }
 659
 660 static int register_ftrace_profiler(void)
 661 {
 662         return register_ftrace_graph(&profile_graph_return,
 663                                      &profile_graph_entry);
 664 }
 665
 666 static void unregister_ftrace_profiler(void)
 667 {
 668         unregister_ftrace_graph();
 669 }
 670 #else
 671 static struct ftrace_ops ftrace_profile_ops __read_mostly =
 672 {
 673         .func           = function_profile_call,
 674 };
 675
 676 static int register_ftrace_profiler(void)
 677 {
 678         return register_ftrace_function(&ftrace_profile_ops);
 679 }
 680
 681 static void unregister_ftrace_profiler(void)
 682 {
 683         unregister_ftrace_function(&ftrace_profile_ops);
 684 }
 685 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
 686
 687 static ssize_t
 688 ftrace_profile_write(struct file *filp, const char __user *ubuf,
 689                      size_t cnt, loff_t *ppos)
 690 {
 691         unsigned long val;
 692         char buf[64];           /* big enough to hold a number */
 693         int ret;
 694
 695         if (cnt >= sizeof(buf))
 696                 return -EINVAL;
 697
 698         if (copy_from_user(&buf, ubuf, cnt))
 699                 return -EFAULT;
 700
 701         buf[cnt] = 0;
 702
 703         ret = strict_strtoul(buf, 10, &val);
 704         if (ret < 0)
 705                 return ret;
 706
 707         val = !!val;
 708
 709         mutex_lock(&ftrace_profile_lock);
 710         if (ftrace_profile_enabled ^ val) {
 711                 if (val) {
 712                         ret = ftrace_profile_init();
 713                         if (ret < 0) {
 714                                 cnt = ret;
 715                                 goto out;
 716                         }
 717
 718                         ret = register_ftrace_profiler();
 719                         if (ret < 0) {
 720                                 cnt = ret;
 721                                 goto out;
 722                         }
 723                         ftrace_profile_enabled = 1;
 724                 } else {
 725                         ftrace_profile_enabled = 0;
 726                         unregister_ftrace_profiler();
 727                 }
 728         }
 729  out:
 730         mutex_unlock(&ftrace_profile_lock);
 731
 732         filp->f_pos += cnt;
 733
 734         return cnt;
 735 }
 736
 737 static ssize_t
 738 ftrace_profile_read(struct file *filp, char __user *ubuf,
 739                      size_t cnt, loff_t *ppos)
 740 {
 741         char buf[64];           /* big enough to hold a number */
 742         int r;
 743
 744         r = sprintf(buf, "%u\n", ftrace_profile_enabled);
 745         return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
 746 }
 747
 748 static const struct file_operations ftrace_profile_fops = {
 749         .open           = tracing_open_generic,
 750         .read           = ftrace_profile_read,
 751         .write          = ftrace_profile_write,
 752 };
 753
 754 /* used to initialize the real stat files */
 755 static struct tracer_stat function_stats __initdata = {
 756         .name           = "functions",
 757         .stat_start     = function_stat_start,
 758         .stat_next      = function_stat_next,
 759         .stat_cmp       = function_stat_cmp,
 760         .stat_headers   = function_stat_headers,
 761         .stat_show      = function_stat_show
 762 };
 763
 764 static void ftrace_profile_debugfs(struct dentry *d_tracer)
 765 {
 766         struct ftrace_profile_stat *stat;
 767         struct dentry *entry;
 768         char *name;
 769         int ret;
 770         int cpu;
 771
 772         for_each_possible_cpu(cpu) {
 773                 stat = &per_cpu(ftrace_profile_stats, cpu);
 774
 775                 /* allocate enough for function name + cpu number */
 776                 name = kmalloc(32, GFP_KERNEL);
 777                 if (!name) {
 778                         /*
 779                          * The files created are permanent, if something happens
 780                          * we still do not free memory.
 781                          */
 782                         kfree(stat);
 783                         WARN(1,
 784                              "Could not allocate stat file for cpu %d\n",
 785                              cpu);
 786                         return;
 787                 }
 788                 stat->stat = function_stats;
 789                 snprintf(name, 32, "function%d", cpu);
 790                 stat->stat.name = name;
 791                 ret = register_stat_tracer(&stat->stat);
 792                 if (ret) {
 793                         WARN(1,
 794                              "Could not register function stat for cpu %d\n",
 795                              cpu);
 796                         kfree(name);
 797                         return;
 798                 }
 799         }
 800
 801         entry = debugfs_create_file("function_profile_enabled", 0644,
 802                                     d_tracer, NULL, &ftrace_profile_fops);
 803         if (!entry)
 804                 pr_warning("Could not create debugfs "
 805                            "'function_profile_enabled' entry\n");
 806 }
 807
 808 #else /* CONFIG_FUNCTION_PROFILER */
 809 static void ftrace_profile_debugfs(struct dentry *d_tracer)
 810 {
 811 }
 812 #endif /* CONFIG_FUNCTION_PROFILER */
 813
 814 /* set when tracing only a pid */
 815 struct pid *ftrace_pid_trace;
 816 static struct pid * const ftrace_swapper_pid = &init_struct_pid;
 817
 818 #ifdef CONFIG_DYNAMIC_FTRACE
 819
 820 #ifndef CONFIG_FTRACE_MCOUNT_RECORD
 821 # error Dynamic ftrace depends on MCOUNT_RECORD
 822 #endif
 823
 824 static struct hlist_head ftrace_func_hash[FTRACE_FUNC_HASHSIZE] __read_mostly;
 825
 826 struct ftrace_func_probe {
 827         struct hlist_node       node;
 828         struct ftrace_probe_ops *ops;
 829         unsigned long           flags;
 830         unsigned long           ip;
 831         void                    *data;
 832         struct rcu_head         rcu;
 833 };
 834
 835 enum {
 836         FTRACE_ENABLE_CALLS             = (1 << 0),
 837         FTRACE_DISABLE_CALLS            = (1 << 1),
 838         FTRACE_UPDATE_TRACE_FUNC        = (1 << 2),
 839         FTRACE_ENABLE_MCOUNT            = (1 << 3),
 840         FTRACE_DISABLE_MCOUNT           = (1 << 4),
 841         FTRACE_START_FUNC_RET           = (1 << 5),
 842         FTRACE_STOP_FUNC_RET            = (1 << 6),
 843 };
 844
 845 static int ftrace_filtered;
 846
 847 static struct dyn_ftrace *ftrace_new_addrs;
 848
 849 static DEFINE_MUTEX(ftrace_regex_lock);
 850
 851 struct ftrace_page {
 852         struct ftrace_page      *next;
 853         int                     index;
 854         struct dyn_ftrace       records[];
 855 };
 856
 857 #define ENTRIES_PER_PAGE \
 858   ((PAGE_SIZE - sizeof(struct ftrace_page)) / sizeof(struct dyn_ftrace))
 859
 860 /* estimate from running different kernels */
 861 #define NR_TO_INIT              10000
 862
 863 static struct ftrace_page       *ftrace_pages_start;
 864 static struct ftrace_page       *ftrace_pages;
 865
 866 static struct dyn_ftrace *ftrace_free_records;
 867
 868 /*
 869  * This is a double for. Do not use 'break' to break out of the loop,
 870  * you must use a goto.
 871  */
 872 #define do_for_each_ftrace_rec(pg, rec)                                 \
 873         for (pg = ftrace_pages_start; pg; pg = pg->next) {              \
 874                 int _____i;                                             \
 875                 for (_____i = 0; _____i < pg->index; _____i++) {        \
 876                         rec = &pg->records[_____i];
 877
 878 #define while_for_each_ftrace_rec()             \
 879                 }                               \
 880         }
 881
 882 #ifdef CONFIG_KPROBES
 883
 884 static int frozen_record_count;
 885
 886 static inline void freeze_record(struct dyn_ftrace *rec)
 887 {
 888         if (!(rec->flags & FTRACE_FL_FROZEN)) {
 889                 rec->flags |= FTRACE_FL_FROZEN;
 890                 frozen_record_count++;
 891         }
 892 }
 893
 894 static inline void unfreeze_record(struct dyn_ftrace *rec)
 895 {
 896         if (rec->flags & FTRACE_FL_FROZEN) {
 897                 rec->flags &= ~FTRACE_FL_FROZEN;
 898                 frozen_record_count--;
 899         }
 900 }
 901
 902 static inline int record_frozen(struct dyn_ftrace *rec)
 903 {
 904         return rec->flags & FTRACE_FL_FROZEN;
 905 }
 906 #else
 907 # define freeze_record(rec)                     ({ 0; })
 908 # define unfreeze_record(rec)                   ({ 0; })
 909 # define record_frozen(rec)                     ({ 0; })
 910 #endif /* CONFIG_KPROBES */
 911
 912 static void ftrace_free_rec(struct dyn_ftrace *rec)
 913 {
 914         rec->freelist = ftrace_free_records;
 915         ftrace_free_records = rec;
 916         rec->flags |= FTRACE_FL_FREE;
 917 }
 918
 919 void ftrace_release(void *start, unsigned long size)
 920 {
 921         struct dyn_ftrace *rec;
 922         struct ftrace_page *pg;
 923         unsigned long s = (unsigned long)start;
 924         unsigned long e = s + size;
 925
 926         if (ftrace_disabled || !start)
 927                 return;
 928
 929         mutex_lock(&ftrace_lock);
 930         do_for_each_ftrace_rec(pg, rec) {
 931                 if ((rec->ip >= s) && (rec->ip < e) &&
 932                     !(rec->flags & FTRACE_FL_FREE))
 933                         ftrace_free_rec(rec);
 934         } while_for_each_ftrace_rec();
 935         mutex_unlock(&ftrace_lock);
 936 }
 937
 938 static struct dyn_ftrace *ftrace_alloc_dyn_node(unsigned long ip)
 939 {
 940         struct dyn_ftrace *rec;
 941
 942         /* First check for freed records */
 943         if (ftrace_free_records) {
 944                 rec = ftrace_free_records;
 945
 946                 if (unlikely(!(rec->flags & FTRACE_FL_FREE))) {
 947                         FTRACE_WARN_ON_ONCE(1);
 948                         ftrace_free_records = NULL;
 949                         return NULL;
 950                 }
 951
 952                 ftrace_free_records = rec->freelist;
 953                 memset(rec, 0, sizeof(*rec));
 954                 return rec;
 955         }
 956
 957         if (ftrace_pages->index == ENTRIES_PER_PAGE) {
 958                 if (!ftrace_pages->next) {
 959                         /* allocate another page */
 960                         ftrace_pages->next =
 961                                 (void *)get_zeroed_page(GFP_KERNEL);
 962                         if (!ftrace_pages->next)
 963                                 return NULL;
 964                 }
 965                 ftrace_pages = ftrace_pages->next;
 966         }
 967
 968         return &ftrace_pages->records[ftrace_pages->index++];
 969 }
 970
 971 static struct dyn_ftrace *
 972 ftrace_record_ip(unsigned long ip)
 973 {
 974         struct dyn_ftrace *rec;
 975
 976         if (ftrace_disabled)
 977                 return NULL;
 978
 979         rec = ftrace_alloc_dyn_node(ip);
 980         if (!rec)
 981                 return NULL;
 982
 983         rec->ip = ip;
 984         rec->newlist = ftrace_new_addrs;
 985         ftrace_new_addrs = rec;
 986
 987         return rec;
 988 }
 989
 990 static void print_ip_ins(const char *fmt, unsigned char *p)
 991 {
 992         int i;
 993
 994         printk(KERN_CONT "%s", fmt);
 995
 996         for (i = 0; i < MCOUNT_INSN_SIZE; i++)
 997                 printk(KERN_CONT "%s%02x", i ? ":" : "", p[i]);
 998 }
 999
1000 static void ftrace_bug(int failed, unsigned long ip)
1001 {
1002         switch (failed) {
1003         case -EFAULT:
1004                 FTRACE_WARN_ON_ONCE(1);
1005                 pr_info("ftrace faulted on modifying ");
1006                 print_ip_sym(ip);
1007                 break;
1008         case -EINVAL:
1009                 FTRACE_WARN_ON_ONCE(1);
1010                 pr_info("ftrace failed to modify ");
1011                 print_ip_sym(ip);
1012                 print_ip_ins(" actual: ", (unsigned char *)ip);
1013                 printk(KERN_CONT "\n");
1014                 break;
1015         case -EPERM:
1016                 FTRACE_WARN_ON_ONCE(1);
1017                 pr_info("ftrace faulted on writing ");
1018                 print_ip_sym(ip);
1019                 break;
1020         default:
1021                 FTRACE_WARN_ON_ONCE(1);
1022                 pr_info("ftrace faulted on unknown error ");
1023                 print_ip_sym(ip);
1024         }
1025 }
1026
1027
1028 static int
1029 __ftrace_replace_code(struct dyn_ftrace *rec, int enable)
1030 {
1031         unsigned long ftrace_addr;
1032         unsigned long ip, fl;
1033
1034         ftrace_addr = (unsigned long)FTRACE_ADDR;
1035
1036         ip = rec->ip;
1037
1038         /*
1039          * If this record is not to be traced and
1040          * it is not enabled then do nothing.
1041          *
1042          * If this record is not to be traced and
1043          * it is enabled then disable it.
1044          *
1045          */
1046         if (rec->flags & FTRACE_FL_NOTRACE) {
1047                 if (rec->flags & FTRACE_FL_ENABLED)
1048                         rec->flags &= ~FTRACE_FL_ENABLED;
1049                 else
1050                         return 0;
1051
1052         } else if (ftrace_filtered && enable) {
1053                 /*
1054                  * Filtering is on:
1055                  */
1056
1057                 fl = rec->flags & (FTRACE_FL_FILTER | FTRACE_FL_ENABLED);
1058
1059                 /* Record is filtered and enabled, do nothing */
1060                 if (fl == (FTRACE_FL_FILTER | FTRACE_FL_ENABLED))
1061                         return 0;
1062
1063                 /* Record is not filtered or enabled, do nothing */
1064                 if (!fl)
1065                         return 0;
1066
1067                 /* Record is not filtered but enabled, disable it */
1068                 if (fl == FTRACE_FL_ENABLED)
1069                         rec->flags &= ~FTRACE_FL_ENABLED;
1070                 else
1071                 /* Otherwise record is filtered but not enabled, enable it */
1072                         rec->flags |= FTRACE_FL_ENABLED;
1073         } else {
1074                 /* Disable or not filtered */
1075
1076                 if (enable) {
1077                         /* if record is enabled, do nothing */
1078                         if (rec->flags & FTRACE_FL_ENABLED)
1079                                 return 0;
1080
1081                         rec->flags |= FTRACE_FL_ENABLED;
1082
1083                 } else {
1084
1085                         /* if record is not enabled, do nothing */
1086                         if (!(rec->flags & FTRACE_FL_ENABLED))
1087                                 return 0;
1088
1089                         rec->flags &= ~FTRACE_FL_ENABLED;
1090                 }
1091         }
1092
1093         if (rec->flags & FTRACE_FL_ENABLED)
1094                 return ftrace_make_call(rec, ftrace_addr);
1095         else
1096                 return ftrace_make_nop(NULL, rec, ftrace_addr);
1097 }
1098
1099 static void ftrace_replace_code(int enable)
1100 {
1101         struct dyn_ftrace *rec;
1102         struct ftrace_page *pg;
1103         int failed;
1104
1105         do_for_each_ftrace_rec(pg, rec) {
1106                 /*
1107                  * Skip over free records, records that have
1108                  * failed and not converted.
1109                  */
1110                 if (rec->flags & FTRACE_FL_FREE ||
1111                     rec->flags & FTRACE_FL_FAILED ||
1112                     !(rec->flags & FTRACE_FL_CONVERTED))
1113                         continue;
1114
1115                 /* ignore updates to this record's mcount site */
1116                 if (get_kprobe((void *)rec->ip)) {
1117                         freeze_record(rec);
1118                         continue;
1119                 } else {
1120                         unfreeze_record(rec);
1121                 }
1122
1123                 failed = __ftrace_replace_code(rec, enable);
1124                 if (failed) {
1125                         rec->flags |= FTRACE_FL_FAILED;
1126                         if ((system_state == SYSTEM_BOOTING) ||
1127                             !core_kernel_text(rec->ip)) {
1128                                 ftrace_free_rec(rec);
1129                                 } else {
1130                                 ftrace_bug(failed, rec->ip);
1131                                         /* Stop processing */
1132                                         return;
1133                                 }
1134                 }
1135         } while_for_each_ftrace_rec();
1136 }
1137
1138 static int
1139 ftrace_code_disable(struct module *mod, struct dyn_ftrace *rec)
1140 {
1141         unsigned long ip;
1142         int ret;
1143
1144         ip = rec->ip;
1145
1146         ret = ftrace_make_nop(mod, rec, MCOUNT_ADDR);
1147         if (ret) {
1148                 ftrace_bug(ret, ip);
1149                 rec->flags |= FTRACE_FL_FAILED;
1150                 return 0;
1151         }
1152         return 1;
1153 }
1154
1155 /*
1156  * archs can override this function if they must do something
1157  * before the modifying code is performed.
1158  */
1159 int __weak ftrace_arch_code_modify_prepare(void)
1160 {
1161         return 0;
1162 }
1163
1164 /*
1165  * archs can override this function if they must do something
1166  * after the modifying code is performed.
1167  */
1168 int __weak ftrace_arch_code_modify_post_process(void)
1169 {
1170         return 0;
1171 }
1172
1173 static int __ftrace_modify_code(void *data)
1174 {
1175         int *command = data;
1176
1177         if (*command & FTRACE_ENABLE_CALLS)
1178                 ftrace_replace_code(1);
1179         else if (*command & FTRACE_DISABLE_CALLS)
1180                 ftrace_replace_code(0);
1181
1182         if (*command & FTRACE_UPDATE_TRACE_FUNC)
1183                 ftrace_update_ftrace_func(ftrace_trace_function);
1184
1185         if (*command & FTRACE_START_FUNC_RET)
1186                 ftrace_enable_ftrace_graph_caller();
1187         else if (*command & FTRACE_STOP_FUNC_RET)
1188                 ftrace_disable_ftrace_graph_caller();
1189
1190         return 0;
1191 }
1192
1193 static void ftrace_run_update_code(int command)
1194 {
1195         int ret;
1196
1197         ret = ftrace_arch_code_modify_prepare();
1198         FTRACE_WARN_ON(ret);
1199         if (ret)
1200                 return;
1201
1202         stop_machine(__ftrace_modify_code, &command, NULL);
1203
1204         ret = ftrace_arch_code_modify_post_process();
1205         FTRACE_WARN_ON(ret);
1206 }
1207
1208 static ftrace_func_t saved_ftrace_func;
1209 static int ftrace_start_up;
1210
1211 static void ftrace_startup_enable(int command)
1212 {
1213         if (saved_ftrace_func != ftrace_trace_function) {
1214                 saved_ftrace_func = ftrace_trace_function;
1215                 command |= FTRACE_UPDATE_TRACE_FUNC;
1216         }
1217
1218         if (!command || !ftrace_enabled)
1219                 return;
1220
1221         ftrace_run_update_code(command);
1222 }
1223
1224 static void ftrace_startup(int command)
1225 {
1226         if (unlikely(ftrace_disabled))
1227                 return;
1228
1229         ftrace_start_up++;
1230         command |= FTRACE_ENABLE_CALLS;
1231
1232         ftrace_startup_enable(command);
1233 }
1234
1235 static void ftrace_shutdown(int command)
1236 {
1237         if (unlikely(ftrace_disabled))
1238                 return;
1239
1240         ftrace_start_up--;
1241         if (!ftrace_start_up)
1242                 command |= FTRACE_DISABLE_CALLS;
1243
1244         if (saved_ftrace_func != ftrace_trace_function) {
1245                 saved_ftrace_func = ftrace_trace_function;
1246                 command |= FTRACE_UPDATE_TRACE_FUNC;
1247         }
1248
1249         if (!command || !ftrace_enabled)
1250                 return;
1251
1252         ftrace_run_update_code(command);
1253 }
1254
1255 static void ftrace_startup_sysctl(void)
1256 {
1257         int command = FTRACE_ENABLE_MCOUNT;
1258
1259         if (unlikely(ftrace_disabled))
1260                 return;
1261
1262         /* Force update next time */
1263         saved_ftrace_func = NULL;
1264         /* ftrace_start_up is true if we want ftrace running */
1265         if (ftrace_start_up)
1266                 command |= FTRACE_ENABLE_CALLS;
1267
1268         ftrace_run_update_code(command);
1269 }
1270
1271 static void ftrace_shutdown_sysctl(void)
1272 {
1273         int command = FTRACE_DISABLE_MCOUNT;
1274
1275         if (unlikely(ftrace_disabled))
1276                 return;
1277
1278         /* ftrace_start_up is true if ftrace is running */
1279         if (ftrace_start_up)
1280                 command |= FTRACE_DISABLE_CALLS;
1281
1282         ftrace_run_update_code(command);
1283 }
1284
1285 static cycle_t          ftrace_update_time;
1286 static unsigned long    ftrace_update_cnt;
1287 unsigned long           ftrace_update_tot_cnt;
1288
1289 static int ftrace_update_code(struct module *mod)
1290 {
1291         struct dyn_ftrace *p;
1292         cycle_t start, stop;
1293
1294         start = ftrace_now(raw_smp_processor_id());
1295         ftrace_update_cnt = 0;
1296
1297         while (ftrace_new_addrs) {
1298
1299                 /* If something went wrong, bail without enabling anything */
1300                 if (unlikely(ftrace_disabled))
1301                         return -1;
1302
1303                 p = ftrace_new_addrs;
1304                 ftrace_new_addrs = p->newlist;
1305                 p->flags = 0L;
1306
1307                 /* convert record (i.e, patch mcount-call with NOP) */
1308                 if (ftrace_code_disable(mod, p)) {
1309                         p->flags |= FTRACE_FL_CONVERTED;
1310                         ftrace_update_cnt++;
1311                 } else
1312                         ftrace_free_rec(p);
1313         }
1314
1315         stop = ftrace_now(raw_smp_processor_id());
1316         ftrace_update_time = stop - start;
1317         ftrace_update_tot_cnt += ftrace_update_cnt;
1318
1319         return 0;
1320 }
1321
1322 static int __init ftrace_dyn_table_alloc(unsigned long num_to_init)
1323 {
1324         struct ftrace_page *pg;
1325         int cnt;
1326         int i;
1327
1328         /* allocate a few pages */
1329         ftrace_pages_start = (void *)get_zeroed_page(GFP_KERNEL);
1330         if (!ftrace_pages_start)
1331                 return -1;
1332
1333         /*
1334          * Allocate a few more pages.
1335          *
1336          * TODO: have some parser search vmlinux before
1337          *   final linking to find all calls to ftrace.
1338          *   Then we can:
1339          *    a) know how many pages to allocate.
1340          *     and/or
1341          *    b) set up the table then.
1342          *
1343          *  The dynamic code is still necessary for
1344          *  modules.
1345          */
1346
1347         pg = ftrace_pages = ftrace_pages_start;
1348
1349         cnt = num_to_init / ENTRIES_PER_PAGE;
1350         pr_info("ftrace: allocating %ld entries in %d pages\n",
1351                 num_to_init, cnt + 1);
1352
1353         for (i = 0; i < cnt; i++) {
1354                 pg->next = (void *)get_zeroed_page(GFP_KERNEL);
1355
1356                 /* If we fail, we'll try later anyway */
1357                 if (!pg->next)
1358                         break;
1359
1360                 pg = pg->next;
1361         }
1362
1363         return 0;
1364 }
1365
1366 enum {
1367         FTRACE_ITER_FILTER      = (1 << 0),
1368         FTRACE_ITER_CONT        = (1 << 1),
1369         FTRACE_ITER_NOTRACE     = (1 << 2),
1370         FTRACE_ITER_FAILURES    = (1 << 3),
1371         FTRACE_ITER_PRINTALL    = (1 << 4),
1372         FTRACE_ITER_HASH        = (1 << 5),
1373 };
1374
1375 #define FTRACE_BUFF_MAX (KSYM_SYMBOL_LEN+4) /* room for wildcards */
1376
1377 struct ftrace_iterator {
1378         struct ftrace_page      *pg;
1379         int                     hidx;
1380         int                     idx;
1381         unsigned                flags;
1382         unsigned char           buffer[FTRACE_BUFF_MAX+1];
1383         unsigned                buffer_idx;
1384         unsigned                filtered;
1385 };
1386
1387 static void *
1388 t_hash_next(struct seq_file *m, void *v, loff_t *pos)
1389 {
1390         struct ftrace_iterator *iter = m->private;
1391         struct hlist_node *hnd = v;
1392         struct hlist_head *hhd;
1393
1394         WARN_ON(!(iter->flags & FTRACE_ITER_HASH));
1395
1396         (*pos)++;
1397
1398  retry:
1399         if (iter->hidx >= FTRACE_FUNC_HASHSIZE)
1400                 return NULL;
1401
1402         hhd = &ftrace_func_hash[iter->hidx];
1403
1404         if (hlist_empty(hhd)) {
1405                 iter->hidx++;
1406                 hnd = NULL;
1407                 goto retry;
1408         }
1409
1410         if (!hnd)
1411                 hnd = hhd->first;
1412         else {
1413                 hnd = hnd->next;
1414                 if (!hnd) {
1415                         iter->hidx++;
1416                         goto retry;
1417                 }
1418         }
1419
1420         return hnd;
1421 }
1422
1423 static void *t_hash_start(struct seq_file *m, loff_t *pos)
1424 {
1425         struct ftrace_iterator *iter = m->private;
1426         void *p = NULL;
1427
1428         iter->flags |= FTRACE_ITER_HASH;
1429
1430         return t_hash_next(m, p, pos);
1431 }
1432
1433 static int t_hash_show(struct seq_file *m, void *v)
1434 {
1435         struct ftrace_func_probe *rec;
1436         struct hlist_node *hnd = v;
1437         char str[KSYM_SYMBOL_LEN];
1438
1439         rec = hlist_entry(hnd, struct ftrace_func_probe, node);
1440
1441         if (rec->ops->print)
1442                 return rec->ops->print(m, rec->ip, rec->ops, rec->data);
1443
1444         kallsyms_lookup(rec->ip, NULL, NULL, NULL, str);
1445         seq_printf(m, "%s:", str);
1446
1447         kallsyms_lookup((unsigned long)rec->ops->func, NULL, NULL, NULL, str);
1448         seq_printf(m, "%s", str);
1449
1450         if (rec->data)
1451                 seq_printf(m, ":%p", rec->data);
1452         seq_putc(m, '\n');
1453
1454         return 0;
1455 }
1456
1457 static void *
1458 t_next(struct seq_file *m, void *v, loff_t *pos)
1459 {
1460         struct ftrace_iterator *iter = m->private;
1461         struct dyn_ftrace *rec = NULL;
1462
1463         if (iter->flags & FTRACE_ITER_HASH)
1464                 return t_hash_next(m, v, pos);
1465
1466         (*pos)++;
1467
1468         if (iter->flags & FTRACE_ITER_PRINTALL)
1469                 return NULL;
1470
1471  retry:
1472         if (iter->idx >= iter->pg->index) {
1473                 if (iter->pg->next) {
1474                         iter->pg = iter->pg->next;
1475                         iter->idx = 0;
1476                         goto retry;
1477                 } else {
1478                         iter->idx = -1;
1479                 }
1480         } else {
1481                 rec = &iter->pg->records[iter->idx++];
1482                 if ((rec->flags & FTRACE_FL_FREE) ||
1483
1484                     (!(iter->flags & FTRACE_ITER_FAILURES) &&
1485                      (rec->flags & FTRACE_FL_FAILED)) ||
1486
1487                     ((iter->flags & FTRACE_ITER_FAILURES) &&
1488                      !(rec->flags & FTRACE_FL_FAILED)) ||
1489
1490                     ((iter->flags & FTRACE_ITER_FILTER) &&
1491                      !(rec->flags & FTRACE_FL_FILTER)) ||
1492
1493                     ((iter->flags & FTRACE_ITER_NOTRACE) &&
1494                      !(rec->flags & FTRACE_FL_NOTRACE))) {
1495                         rec = NULL;
1496                         goto retry;
1497                 }
1498         }
1499
1500         return rec;
1501 }
1502
1503 static void *t_start(struct seq_file *m, loff_t *pos)
1504 {
1505         struct ftrace_iterator *iter = m->private;
1506         void *p = NULL;
1507
1508         mutex_lock(&ftrace_lock);
1509         /*
1510          * For set_ftrace_filter reading, if we have the filter
1511          * off, we can short cut and just print out that all
1512          * functions are enabled.
1513          */
1514         if (iter->flags & FTRACE_ITER_FILTER && !ftrace_filtered) {
1515                 if (*pos > 0)
1516                         return t_hash_start(m, pos);
1517                 iter->flags |= FTRACE_ITER_PRINTALL;
1518                 (*pos)++;
1519                 return iter;
1520         }
1521
1522         if (iter->flags & FTRACE_ITER_HASH)
1523                 return t_hash_start(m, pos);
1524
1525         if (*pos > 0) {
1526                 if (iter->idx < 0)
1527                         return p;
1528                 (*pos)--;
1529                 iter->idx--;
1530         }
1531
1532         p = t_next(m, p, pos);
1533
1534         if (!p)
1535                 return t_hash_start(m, pos);
1536
1537         return p;
1538 }
1539
1540 static void t_stop(struct seq_file *m, void *p)
1541 {
1542         mutex_unlock(&ftrace_lock);
1543 }
1544
1545 static int t_show(struct seq_file *m, void *v)
1546 {
1547         struct ftrace_iterator *iter = m->private;
1548         struct dyn_ftrace *rec = v;
1549         char str[KSYM_SYMBOL_LEN];
1550
1551         if (iter->flags & FTRACE_ITER_HASH)
1552                 return t_hash_show(m, v);
1553
1554         if (iter->flags & FTRACE_ITER_PRINTALL) {
1555                 seq_printf(m, "#### all functions enabled ####\n");
1556                 return 0;
1557         }
1558
1559         if (!rec)
1560                 return 0;
1561
1562         kallsyms_lookup(rec->ip, NULL, NULL, NULL, str);
1563
1564         seq_printf(m, "%s\n", str);
1565
1566         return 0;
1567 }
1568
1569 static struct seq_operations show_ftrace_seq_ops = {
1570         .start = t_start,
1571         .next = t_next,
1572         .stop = t_stop,
1573         .show = t_show,
1574 };
1575
1576 static int
1577 ftrace_avail_open(struct inode *inode, struct file *file)
1578 {
1579         struct ftrace_iterator *iter;
1580         int ret;
1581
1582         if (unlikely(ftrace_disabled))
1583                 return -ENODEV;
1584
1585         iter = kzalloc(sizeof(*iter), GFP_KERNEL);
1586         if (!iter)
1587                 return -ENOMEM;
1588
1589         iter->pg = ftrace_pages_start;
1590
1591         ret = seq_open(file, &show_ftrace_seq_ops);
1592         if (!ret) {
1593                 struct seq_file *m = file->private_data;
1594
1595                 m->private = iter;
1596         } else {
1597                 kfree(iter);
1598         }
1599
1600         return ret;
1601 }
1602
1603 int ftrace_avail_release(struct inode *inode, struct file *file)
1604 {
1605         struct seq_file *m = (struct seq_file *)file->private_data;
1606         struct ftrace_iterator *iter = m->private;
1607
1608         seq_release(inode, file);
1609         kfree(iter);
1610
1611         return 0;
1612 }
1613
1614 static int
1615 ftrace_failures_open(struct inode *inode, struct file *file)
1616 {
1617         int ret;
1618         struct seq_file *m;
1619         struct ftrace_iterator *iter;
1620
1621         ret = ftrace_avail_open(inode, file);
1622         if (!ret) {
1623                 m = (struct seq_file *)file->private_data;
1624                 iter = (struct ftrace_iterator *)m->private;
1625                 iter->flags = FTRACE_ITER_FAILURES;
1626         }
1627
1628         return ret;
1629 }
1630
1631
1632 static void ftrace_filter_reset(int enable)
1633 {
1634         struct ftrace_page *pg;
1635         struct dyn_ftrace *rec;
1636         unsigned long type = enable ? FTRACE_FL_FILTER : FTRACE_FL_NOTRACE;
1637
1638         mutex_lock(&ftrace_lock);
1639         if (enable)
1640                 ftrace_filtered = 0;
1641         do_for_each_ftrace_rec(pg, rec) {
1642                 if (rec->flags & FTRACE_FL_FAILED)
1643                         continue;
1644                 rec->flags &= ~type;
1645         } while_for_each_ftrace_rec();
1646         mutex_unlock(&ftrace_lock);
1647 }
1648
1649 static int
1650 ftrace_regex_open(struct inode *inode, struct file *file, int enable)
1651 {
1652         struct ftrace_iterator *iter;
1653         int ret = 0;
1654
1655         if (unlikely(ftrace_disabled))
1656                 return -ENODEV;
1657
1658         iter = kzalloc(sizeof(*iter), GFP_KERNEL);
1659         if (!iter)
1660                 return -ENOMEM;
1661
1662         mutex_lock(&ftrace_regex_lock);
1663         if ((file->f_mode & FMODE_WRITE) &&
1664             !(file->f_flags & O_APPEND))
1665                 ftrace_filter_reset(enable);
1666
1667         if (file->f_mode & FMODE_READ) {
1668                 iter->pg = ftrace_pages_start;
1669                 iter->flags = enable ? FTRACE_ITER_FILTER :
1670                         FTRACE_ITER_NOTRACE;
1671
1672                 ret = seq_open(file, &show_ftrace_seq_ops);
1673                 if (!ret) {
1674                         struct seq_file *m = file->private_data;
1675                         m->private = iter;
1676                 } else
1677                         kfree(iter);
1678         } else
1679                 file->private_data = iter;
1680         mutex_unlock(&ftrace_regex_lock);
1681
1682         return ret;
1683 }
1684
1685 static int
1686 ftrace_filter_open(struct inode *inode, struct file *file)
1687 {
1688         return ftrace_regex_open(inode, file, 1);
1689 }
1690
1691 static int
1692 ftrace_notrace_open(struct inode *inode, struct file *file)
1693 {
1694         return ftrace_regex_open(inode, file, 0);
1695 }
1696
1697 static loff_t
1698 ftrace_regex_lseek(struct file *file, loff_t offset, int origin)
1699 {
1700         loff_t ret;
1701
1702         if (file->f_mode & FMODE_READ)
1703                 ret = seq_lseek(file, offset, origin);
1704         else
1705                 file->f_pos = ret = 1;
1706
1707         return ret;
1708 }
1709
1710 enum {
1711         MATCH_FULL,
1712         MATCH_FRONT_ONLY,
1713         MATCH_MIDDLE_ONLY,
1714         MATCH_END_ONLY,
1715 };
1716
1717 /*
1718  * (static function - no need for kernel doc)
1719  *
1720  * Pass in a buffer containing a glob and this function will
1721  * set search to point to the search part of the buffer and
1722  * return the type of search it is (see enum above).
1723  * This does modify buff.
1724  *
1725  * Returns enum type.
1726  *  search returns the pointer to use for comparison.
1727  *  not returns 1 if buff started with a '!'
1728  *     0 otherwise.
1729  */
1730 static int
1731 ftrace_setup_glob(char *buff, int len, char **search, int *not)
1732 {
1733         int type = MATCH_FULL;
1734         int i;
1735
1736         if (buff[0] == '!') {
1737                 *not = 1;
1738                 buff++;
1739                 len--;
1740         } else
1741                 *not = 0;
1742
1743         *search = buff;
1744
1745         for (i = 0; i < len; i++) {
1746                 if (buff[i] == '*') {
1747                         if (!i) {
1748                                 *search = buff + 1;
1749                                 type = MATCH_END_ONLY;
1750                         } else {
1751                                 if (type == MATCH_END_ONLY)
1752                                         type = MATCH_MIDDLE_ONLY;
1753                                 else
1754                                         type = MATCH_FRONT_ONLY;
1755                                 buff[i] = 0;
1756                                 break;
1757                         }
1758                 }
1759         }
1760
1761         return type;
1762 }
1763
1764 static int ftrace_match(char *str, char *regex, int len, int type)
1765 {
1766         int matched = 0;
1767         char *ptr;
1768
1769         switch (type) {
1770         case MATCH_FULL:
1771                 if (strcmp(str, regex) == 0)
1772                         matched = 1;
1773                 break;
1774         case MATCH_FRONT_ONLY:
1775                 if (strncmp(str, regex, len) == 0)
1776                         matched = 1;
1777                 break;
1778         case MATCH_MIDDLE_ONLY:
1779                 if (strstr(str, regex))
1780                         matched = 1;
1781                 break;
1782         case MATCH_END_ONLY:
1783                 ptr = strstr(str, regex);
1784                 if (ptr && (ptr[len] == 0))
1785                         matched = 1;
1786                 break;
1787         }
1788
1789         return matched;
1790 }
1791
1792 static int
1793 ftrace_match_record(struct dyn_ftrace *rec, char *regex, int len, int type)
1794 {
1795         char str[KSYM_SYMBOL_LEN];
1796
1797         kallsyms_lookup(rec->ip, NULL, NULL, NULL, str);
1798         return ftrace_match(str, regex, len, type);
1799 }
1800
1801 static void ftrace_match_records(char *buff, int len, int enable)
1802 {
1803         unsigned int search_len;
1804         struct ftrace_page *pg;
1805         struct dyn_ftrace *rec;
1806         unsigned long flag;
1807         char *search;
1808         int type;
1809         int not;
1810
1811         flag = enable ? FTRACE_FL_FILTER : FTRACE_FL_NOTRACE;
1812         type = ftrace_setup_glob(buff, len, &search, &not);
1813
1814         search_len = strlen(search);
1815
1816         mutex_lock(&ftrace_lock);
1817         do_for_each_ftrace_rec(pg, rec) {
1818
1819                 if (rec->flags & FTRACE_FL_FAILED)
1820                         continue;
1821
1822                 if (ftrace_match_record(rec, search, search_len, type)) {
1823                         if (not)
1824                                 rec->flags &= ~flag;
1825                         else
1826                                 rec->flags |= flag;
1827                 }
1828                 /*
1829                  * Only enable filtering if we have a function that
1830                  * is filtered on.
1831                  */
1832                 if (enable && (rec->flags & FTRACE_FL_FILTER))
1833                         ftrace_filtered = 1;
1834         } while_for_each_ftrace_rec();
1835         mutex_unlock(&ftrace_lock);
1836 }
1837
1838 static int
1839 ftrace_match_module_record(struct dyn_ftrace *rec, char *mod,
1840                            char *regex, int len, int type)
1841 {
1842         char str[KSYM_SYMBOL_LEN];
1843         char *modname;
1844
1845         kallsyms_lookup(rec->ip, NULL, NULL, &modname, str);
1846
1847         if (!modname || strcmp(modname, mod))
1848                 return 0;
1849
1850         /* blank search means to match all funcs in the mod */
1851         if (len)
1852                 return ftrace_match(str, regex, len, type);
1853         else
1854                 return 1;
1855 }
1856
1857 static void ftrace_match_module_records(char *buff, char *mod, int enable)
1858 {
1859         unsigned search_len = 0;
1860         struct ftrace_page *pg;
1861         struct dyn_ftrace *rec;
1862         int type = MATCH_FULL;
1863         char *search = buff;
1864         unsigned long flag;
1865         int not = 0;
1866
1867         flag = enable ? FTRACE_FL_FILTER : FTRACE_FL_NOTRACE;
1868
1869         /* blank or '*' mean the same */
1870         if (strcmp(buff, "*") == 0)
1871                 buff[0] = 0;
1872
1873         /* handle the case of 'dont filter this module' */
1874         if (strcmp(buff, "!") == 0 || strcmp(buff, "!*") == 0) {
1875                 buff[0] = 0;
1876                 not = 1;
1877         }
1878
1879         if (strlen(buff)) {
1880                 type = ftrace_setup_glob(buff, strlen(buff), &search, &not);
1881                 search_len = strlen(search);
1882         }
1883
1884         mutex_lock(&ftrace_lock);
1885         do_for_each_ftrace_rec(pg, rec) {
1886
1887                 if (rec->flags & FTRACE_FL_FAILED)
1888                         continue;
1889
1890                 if (ftrace_match_module_record(rec, mod,
1891                                                search, search_len, type)) {
1892                         if (not)
1893                                 rec->flags &= ~flag;
1894                         else
1895                                 rec->flags |= flag;
1896                 }
1897                 if (enable && (rec->flags & FTRACE_FL_FILTER))
1898                         ftrace_filtered = 1;
1899
1900         } while_for_each_ftrace_rec();
1901         mutex_unlock(&ftrace_lock);
1902 }
1903
1904 /*
1905  * We register the module command as a template to show others how
1906  * to register the a command as well.
1907  */
1908
1909 static int
1910 ftrace_mod_callback(char *func, char *cmd, char *param, int enable)
1911 {
1912         char *mod;
1913
1914         /*
1915          * cmd == 'mod' because we only registered this func
1916          * for the 'mod' ftrace_func_command.
1917          * But if you register one func with multiple commands,
1918          * you can tell which command was used by the cmd
1919          * parameter.
1920          */
1921
1922         /* we must have a module name */
1923         if (!param)
1924                 return -EINVAL;
1925
1926         mod = strsep(&param, ":");
1927         if (!strlen(mod))
1928                 return -EINVAL;
1929
1930         ftrace_match_module_records(func, mod, enable);
1931         return 0;
1932 }
1933
1934 static struct ftrace_func_command ftrace_mod_cmd = {
1935         .name                   = "mod",
1936         .func                   = ftrace_mod_callback,
1937 };
1938
1939 static int __init ftrace_mod_cmd_init(void)
1940 {
1941         return register_ftrace_command(&ftrace_mod_cmd);
1942 }
1943 device_initcall(ftrace_mod_cmd_init);
1944
1945 static void
1946 function_trace_probe_call(unsigned long ip, unsigned long parent_ip)
1947 {
1948         struct ftrace_func_probe *entry;
1949         struct hlist_head *hhd;
1950         struct hlist_node *n;
1951         unsigned long key;
1952         int resched;
1953
1954         key = hash_long(ip, FTRACE_HASH_BITS);
1955
1956         hhd = &ftrace_func_hash[key];
1957
1958         if (hlist_empty(hhd))
1959                 return;
1960
1961         /*
1962          * Disable preemption for these calls to prevent a RCU grace
1963          * period. This syncs the hash iteration and freeing of items
1964          * on the hash. rcu_read_lock is too dangerous here.
1965          */
1966         resched = ftrace_preempt_disable();
1967         hlist_for_each_entry_rcu(entry, n, hhd, node) {
1968                 if (entry->ip == ip)
1969                         entry->ops->func(ip, parent_ip, &entry->data);
1970         }
1971         ftrace_preempt_enable(resched);
1972 }
1973
1974 static struct ftrace_ops trace_probe_ops __read_mostly =
1975 {
1976         .func           = function_trace_probe_call,
1977 };
1978
1979 static int ftrace_probe_registered;
1980
1981 static void __enable_ftrace_function_probe(void)
1982 {
1983         int i;
1984
1985         if (ftrace_probe_registered)
1986                 return;
1987
1988         for (i = 0; i < FTRACE_FUNC_HASHSIZE; i++) {
1989                 struct hlist_head *hhd = &ftrace_func_hash[i];
1990                 if (hhd->first)
1991                         break;
1992         }
1993         /* Nothing registered? */
1994         if (i == FTRACE_FUNC_HASHSIZE)
1995                 return;
1996
1997         __register_ftrace_function(&trace_probe_ops);
1998         ftrace_startup(0);
1999         ftrace_probe_registered = 1;
2000 }
2001
2002 static void __disable_ftrace_function_probe(void)
2003 {
2004         int i;
2005
2006         if (!ftrace_probe_registered)
2007                 return;
2008
2009         for (i = 0; i < FTRACE_FUNC_HASHSIZE; i++) {
2010                 struct hlist_head *hhd = &ftrace_func_hash[i];
2011                 if (hhd->first)
2012                         return;
2013         }
2014
2015         /* no more funcs left */
2016         __unregister_ftrace_function(&trace_probe_ops);
2017         ftrace_shutdown(0);
2018         ftrace_probe_registered = 0;
2019 }
2020
2021
2022 static void ftrace_free_entry_rcu(struct rcu_head *rhp)
2023 {
2024         struct ftrace_func_probe *entry =
2025                 container_of(rhp, struct ftrace_func_probe, rcu);
2026
2027         if (entry->ops->free)
2028                 entry->ops->free(&entry->data);
2029         kfree(entry);
2030 }
2031
2032
2033 int
2034 register_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops,
2035                               void *data)
2036 {
2037         struct ftrace_func_probe *entry;
2038         struct ftrace_page *pg;
2039         struct dyn_ftrace *rec;
2040         int type, len, not;
2041         unsigned long key;
2042         int count = 0;
2043         char *search;
2044
2045         type = ftrace_setup_glob(glob, strlen(glob), &search, &not);
2046         len = strlen(search);
2047
2048         /* we do not support '!' for function probes */
2049         if (WARN_ON(not))
2050                 return -EINVAL;
2051
2052         mutex_lock(&ftrace_lock);
2053         do_for_each_ftrace_rec(pg, rec) {
2054
2055                 if (rec->flags & FTRACE_FL_FAILED)
2056                         continue;
2057
2058                 if (!ftrace_match_record(rec, search, len, type))
2059                         continue;
2060
2061                 entry = kmalloc(sizeof(*entry), GFP_KERNEL);
2062                 if (!entry) {
2063                         /* If we did not process any, then return error */
2064                         if (!count)
2065                                 count = -ENOMEM;
2066                         goto out_unlock;
2067                 }
2068
2069                 count++;
2070
2071                 entry->data = data;
2072
2073                 /*
2074                  * The caller might want to do something special
2075                  * for each function we find. We call the callback
2076                  * to give the caller an opportunity to do so.
2077                  */
2078                 if (ops->callback) {
2079                         if (ops->callback(rec->ip, &entry->data) < 0) {
2080                                 /* caller does not like this func */
2081                                 kfree(entry);
2082                                 continue;
2083                         }
2084                 }
2085
2086                 entry->ops = ops;
2087                 entry->ip = rec->ip;
2088
2089                 key = hash_long(entry->ip, FTRACE_HASH_BITS);
2090                 hlist_add_head_rcu(&entry->node, &ftrace_func_hash[key]);
2091
2092         } while_for_each_ftrace_rec();
2093         __enable_ftrace_function_probe();
2094
2095  out_unlock:
2096         mutex_unlock(&ftrace_lock);
2097
2098         return count;
2099 }
2100
2101 enum {
2102         PROBE_TEST_FUNC         = 1,
2103         PROBE_TEST_DATA         = 2
2104 };
2105
2106 static void
2107 __unregister_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops,
2108                                   void *data, int flags)
2109 {
2110         struct ftrace_func_probe *entry;
2111         struct hlist_node *n, *tmp;
2112         char str[KSYM_SYMBOL_LEN];
2113         int type = MATCH_FULL;
2114         int i, len = 0;
2115         char *search;
2116
2117         if (glob && (strcmp(glob, "*") || !strlen(glob)))
2118                 glob = NULL;
2119         else {
2120                 int not;
2121
2122                 type = ftrace_setup_glob(glob, strlen(glob), &search, &not);
2123                 len = strlen(search);
2124
2125                 /* we do not support '!' for function probes */
2126                 if (WARN_ON(not))
2127                         return;
2128         }
2129
2130         mutex_lock(&ftrace_lock);
2131         for (i = 0; i < FTRACE_FUNC_HASHSIZE; i++) {
2132                 struct hlist_head *hhd = &ftrace_func_hash[i];
2133
2134                 hlist_for_each_entry_safe(entry, n, tmp, hhd, node) {
2135
2136                         /* break up if statements for readability */
2137                         if ((flags & PROBE_TEST_FUNC) && entry->ops != ops)
2138                                 continue;
2139
2140                         if ((flags & PROBE_TEST_DATA) && entry->data != data)
2141                                 continue;
2142
2143                         /* do this last, since it is the most expensive */
2144                         if (glob) {
2145                                 kallsyms_lookup(entry->ip, NULL, NULL,
2146                                                 NULL, str);
2147                                 if (!ftrace_match(str, glob, len, type))
2148                                         continue;
2149                         }
2150
2151                         hlist_del(&entry->node);
2152                         call_rcu(&entry->rcu, ftrace_free_entry_rcu);
2153                 }
2154         }
2155         __disable_ftrace_function_probe();
2156         mutex_unlock(&ftrace_lock);
2157 }
2158
2159 void
2160 unregister_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops,
2161                                 void *data)
2162 {
2163         __unregister_ftrace_function_probe(glob, ops, data,
2164                                           PROBE_TEST_FUNC | PROBE_TEST_DATA);
2165 }
2166
2167 void
2168 unregister_ftrace_function_probe_func(char *glob, struct ftrace_probe_ops *ops)
2169 {
2170         __unregister_ftrace_function_probe(glob, ops, NULL, PROBE_TEST_FUNC);
2171 }
2172
2173 void unregister_ftrace_function_probe_all(char *glob)
2174 {
2175         __unregister_ftrace_function_probe(glob, NULL, NULL, 0);
2176 }
2177
2178 static LIST_HEAD(ftrace_commands);
2179 static DEFINE_MUTEX(ftrace_cmd_mutex);
2180
2181 int register_ftrace_command(struct ftrace_func_command *cmd)
2182 {
2183         struct ftrace_func_command *p;
2184         int ret = 0;
2185
2186         mutex_lock(&ftrace_cmd_mutex);
2187         list_for_each_entry(p, &ftrace_commands, list) {
2188                 if (strcmp(cmd->name, p->name) == 0) {
2189                         ret = -EBUSY;
2190                         goto out_unlock;
2191                 }
2192         }
2193         list_add(&cmd->list, &ftrace_commands);
2194  out_unlock:
2195         mutex_unlock(&ftrace_cmd_mutex);
2196
2197         return ret;
2198 }
2199
2200 int unregister_ftrace_command(struct ftrace_func_command *cmd)
2201 {
2202         struct ftrace_func_command *p, *n;
2203         int ret = -ENODEV;
2204
2205         mutex_lock(&ftrace_cmd_mutex);
2206         list_for_each_entry_safe(p, n, &ftrace_commands, list) {
2207                 if (strcmp(cmd->name, p->name) == 0) {
2208                         ret = 0;
2209                         list_del_init(&p->list);
2210                         goto out_unlock;
2211                 }
2212         }
2213  out_unlock:
2214         mutex_unlock(&ftrace_cmd_mutex);
2215
2216         return ret;
2217 }
2218
2219 static int ftrace_process_regex(char *buff, int len, int enable)
2220 {
2221         char *func, *command, *next = buff;
2222         struct ftrace_func_command *p;
2223         int ret = -EINVAL;
2224
2225         func = strsep(&next, ":");
2226
2227         if (!next) {
2228                 ftrace_match_records(func, len, enable);
2229                 return 0;
2230         }
2231
2232         /* command found */
2233
2234         command = strsep(&next, ":");
2235
2236         mutex_lock(&ftrace_cmd_mutex);
2237         list_for_each_entry(p, &ftrace_commands, list) {
2238                 if (strcmp(p->name, command) == 0) {
2239                         ret = p->func(func, command, next, enable);
2240                         goto out_unlock;
2241                 }
2242         }
2243  out_unlock:
2244         mutex_unlock(&ftrace_cmd_mutex);
2245
2246         return ret;
2247 }
2248
2249 static ssize_t
2250 ftrace_regex_write(struct file *file, const char __user *ubuf,
2251                    size_t cnt, loff_t *ppos, int enable)
2252 {
2253         struct ftrace_iterator *iter;
2254         char ch;
2255         size_t read = 0;
2256         ssize_t ret;
2257
2258         if (!cnt || cnt < 0)
2259                 return 0;
2260
2261         mutex_lock(&ftrace_regex_lock);
2262
2263         if (file->f_mode & FMODE_READ) {
2264                 struct seq_file *m = file->private_data;
2265                 iter = m->private;
2266         } else
2267                 iter = file->private_data;
2268
2269         if (!*ppos) {
2270                 iter->flags &= ~FTRACE_ITER_CONT;
2271                 iter->buffer_idx = 0;
2272         }
2273
2274         ret = get_user(ch, ubuf++);
2275         if (ret)
2276                 goto out;
2277         read++;
2278         cnt--;
2279
2280         if (!(iter->flags & ~FTRACE_ITER_CONT)) {
2281                 /* skip white space */
2282                 while (cnt && isspace(ch)) {
2283                         ret = get_user(ch, ubuf++);
2284                         if (ret)
2285                                 goto out;
2286                         read++;
2287                         cnt--;
2288                 }
2289
2290                 if (isspace(ch)) {
2291                         file->f_pos += read;
2292                         ret = read;
2293                         goto out;
2294                 }
2295
2296                 iter->buffer_idx = 0;
2297         }
2298
2299         while (cnt && !isspace(ch)) {
2300                 if (iter->buffer_idx < FTRACE_BUFF_MAX)
2301                         iter->buffer[iter->buffer_idx++] = ch;
2302                 else {
2303                         ret = -EINVAL;
2304                         goto out;
2305                 }
2306                 ret = get_user(ch, ubuf++);
2307                 if (ret)
2308                         goto out;
2309                 read++;
2310                 cnt--;
2311         }
2312
2313         if (isspace(ch)) {
2314                 iter->filtered++;
2315                 iter->buffer[iter->buffer_idx] = 0;
2316                 ret = ftrace_process_regex(iter->buffer,
2317                                            iter->buffer_idx, enable);
2318                 if (ret)
2319                         goto out;
2320                 iter->buffer_idx = 0;
2321         } else
2322                 iter->flags |= FTRACE_ITER_CONT;
2323
2324
2325         file->f_pos += read;
2326
2327         ret = read;
2328  out:
2329         mutex_unlock(&ftrace_regex_lock);
2330
2331         return ret;
2332 }
2333
2334 static ssize_t
2335 ftrace_filter_write(struct file *file, const char __user *ubuf,
2336                     size_t cnt, loff_t *ppos)
2337 {
2338         return ftrace_regex_write(file, ubuf, cnt, ppos, 1);
2339 }
2340
2341 static ssize_t
2342 ftrace_notrace_write(struct file *file, const char __user *ubuf,
2343                      size_t cnt, loff_t *ppos)
2344 {
2345         return ftrace_regex_write(file, ubuf, cnt, ppos, 0);
2346 }
2347
2348 static void
2349 ftrace_set_regex(unsigned char *buf, int len, int reset, int enable)
2350 {
2351         if (unlikely(ftrace_disabled))
2352                 return;
2353
2354         mutex_lock(&ftrace_regex_lock);
2355         if (reset)
2356                 ftrace_filter_reset(enable);
2357         if (buf)
2358                 ftrace_match_records(buf, len, enable);
2359         mutex_unlock(&ftrace_regex_lock);
2360 }
2361
2362 /**
2363  * ftrace_set_filter - set a function to filter on in ftrace
2364  * @buf - the string that holds the function filter text.
2365  * @len - the length of the string.
2366  * @reset - non zero to reset all filters before applying this filter.
2367  *
2368  * Filters denote which functions should be enabled when tracing is enabled.
2369  * If @buf is NULL and reset is set, all functions will be enabled for tracing.
2370  */
2371 void ftrace_set_filter(unsigned char *buf, int len, int reset)
2372 {
2373         ftrace_set_regex(buf, len, reset, 1);
2374 }
2375
2376 /**
2377  * ftrace_set_notrace - set a function to not trace in ftrace
2378  * @buf - the string that holds the function notrace text.
2379  * @len - the length of the string.
2380  * @reset - non zero to reset all filters before applying this filter.
2381  *
2382  * Notrace Filters denote which functions should not be enabled when tracing
2383  * is enabled. If @buf is NULL and reset is set, all functions will be enabled
2384  * for tracing.
2385  */
2386 void ftrace_set_notrace(unsigned char *buf, int len, int reset)
2387 {
2388         ftrace_set_regex(buf, len, reset, 0);
2389 }
2390
2391 static int
2392 ftrace_regex_release(struct inode *inode, struct file *file, int enable)
2393 {
2394         struct seq_file *m = (struct seq_file *)file->private_data;
2395         struct ftrace_iterator *iter;
2396
2397         mutex_lock(&ftrace_regex_lock);
2398         if (file->f_mode & FMODE_READ) {
2399                 iter = m->private;
2400
2401                 seq_release(inode, file);
2402         } else
2403                 iter = file->private_data;
2404
2405         if (iter->buffer_idx) {
2406                 iter->filtered++;
2407                 iter->buffer[iter->buffer_idx] = 0;
2408                 ftrace_match_records(iter->buffer, iter->buffer_idx, enable);
2409         }
2410
2411         mutex_lock(&ftrace_lock);
2412         if (ftrace_start_up && ftrace_enabled)
2413                 ftrace_run_update_code(FTRACE_ENABLE_CALLS);
2414         mutex_unlock(&ftrace_lock);
2415
2416         kfree(iter);
2417         mutex_unlock(&ftrace_regex_lock);
2418         return 0;
2419 }
2420
2421 static int
2422 ftrace_filter_release(struct inode *inode, struct file *file)
2423 {
2424         return ftrace_regex_release(inode, file, 1);
2425 }
2426
2427 static int
2428 ftrace_notrace_release(struct inode *inode, struct file *file)
2429 {
2430         return ftrace_regex_release(inode, file, 0);
2431 }
2432
2433 static const struct file_operations ftrace_avail_fops = {
2434         .open = ftrace_avail_open,
2435         .read = seq_read,
2436         .llseek = seq_lseek,
2437         .release = ftrace_avail_release,
2438 };
2439
2440 static const struct file_operations ftrace_failures_fops = {
2441         .open = ftrace_failures_open,
2442         .read = seq_read,
2443         .llseek = seq_lseek,
2444         .release = ftrace_avail_release,
2445 };
2446
2447 static const struct file_operations ftrace_filter_fops = {
2448         .open = ftrace_filter_open,
2449         .read = seq_read,
2450         .write = ftrace_filter_write,
2451         .llseek = ftrace_regex_lseek,
2452         .release = ftrace_filter_release,
2453 };
2454
2455 static const struct file_operations ftrace_notrace_fops = {
2456         .open = ftrace_notrace_open,
2457         .read = seq_read,
2458         .write = ftrace_notrace_write,
2459         .llseek = ftrace_regex_lseek,
2460         .release = ftrace_notrace_release,
2461 };
2462
2463 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
2464
2465 static DEFINE_MUTEX(graph_lock);
2466
2467 int ftrace_graph_count;
2468 unsigned long ftrace_graph_funcs[FTRACE_GRAPH_MAX_FUNCS] __read_mostly;
2469
2470 static void *
2471 g_next(struct seq_file *m, void *v, loff_t *pos)
2472 {
2473         unsigned long *array = m->private;
2474         int index = *pos;
2475
2476         (*pos)++;
2477
2478         if (index >= ftrace_graph_count)
2479                 return NULL;
2480
2481         return &array[index];
2482 }
2483
2484 static void *g_start(struct seq_file *m, loff_t *pos)
2485 {
2486         void *p = NULL;
2487
2488         mutex_lock(&graph_lock);
2489
2490         /* Nothing, tell g_show to print all functions are enabled */
2491         if (!ftrace_graph_count && !*pos)
2492                 return (void *)1;
2493
2494         p = g_next(m, p, pos);
2495
2496         return p;
2497 }
2498
2499 static void g_stop(struct seq_file *m, void *p)
2500 {
2501         mutex_unlock(&graph_lock);
2502 }
2503
2504 static int g_show(struct seq_file *m, void *v)
2505 {
2506         unsigned long *ptr = v;
2507         char str[KSYM_SYMBOL_LEN];
2508
2509         if (!ptr)
2510                 return 0;
2511
2512         if (ptr == (unsigned long *)1) {
2513                 seq_printf(m, "#### all functions enabled ####\n");
2514                 return 0;
2515         }
2516
2517         kallsyms_lookup(*ptr, NULL, NULL, NULL, str);
2518
2519         seq_printf(m, "%s\n", str);
2520
2521         return 0;
2522 }
2523
2524 static struct seq_operations ftrace_graph_seq_ops = {
2525         .start = g_start,
2526         .next = g_next,
2527         .stop = g_stop,
2528         .show = g_show,
2529 };
2530
2531 static int
2532 ftrace_graph_open(struct inode *inode, struct file *file)
2533 {
2534         int ret = 0;
2535
2536         if (unlikely(ftrace_disabled))
2537                 return -ENODEV;
2538
2539         mutex_lock(&graph_lock);
2540         if ((file->f_mode & FMODE_WRITE) &&
2541             !(file->f_flags & O_APPEND)) {
2542                 ftrace_graph_count = 0;
2543                 memset(ftrace_graph_funcs, 0, sizeof(ftrace_graph_funcs));
2544         }
2545
2546         if (file->f_mode & FMODE_READ) {
2547                 ret = seq_open(file, &ftrace_graph_seq_ops);
2548                 if (!ret) {
2549                         struct seq_file *m = file->private_data;
2550                         m->private = ftrace_graph_funcs;
2551                 }
2552         } else
2553                 file->private_data = ftrace_graph_funcs;
2554         mutex_unlock(&graph_lock);
2555
2556         return ret;
2557 }
2558
2559 static int
2560 ftrace_set_func(unsigned long *array, int *idx, char *buffer)
2561 {
2562         struct dyn_ftrace *rec;
2563         struct ftrace_page *pg;
2564         int search_len;
2565         int found = 0;
2566         int type, not;
2567         char *search;
2568         bool exists;
2569         int i;
2570
2571         if (ftrace_disabled)
2572                 return -ENODEV;
2573
2574         /* decode regex */
2575         type = ftrace_setup_glob(buffer, strlen(buffer), &search, &not);
2576         if (not)
2577                 return -EINVAL;
2578
2579         search_len = strlen(search);
2580
2581         mutex_lock(&ftrace_lock);
2582         do_for_each_ftrace_rec(pg, rec) {
2583
2584                 if (*idx >= FTRACE_GRAPH_MAX_FUNCS)
2585                         break;
2586
2587                 if (rec->flags & (FTRACE_FL_FAILED | FTRACE_FL_FREE))
2588                         continue;
2589
2590                 if (ftrace_match_record(rec, search, search_len, type)) {
2591                         /* ensure it is not already in the array */
2592                         exists = false;
2593                         for (i = 0; i < *idx; i++)
2594                                 if (array[i] == rec->ip) {
2595                                         exists = true;
2596                                         break;
2597                                 }
2598                         if (!exists) {
2599                                 array[(*idx)++] = rec->ip;
2600                                 found = 1;
2601                         }
2602                 }
2603         } while_for_each_ftrace_rec();
2604
2605         mutex_unlock(&ftrace_lock);
2606
2607         return found ? 0 : -EINVAL;
2608 }
2609
2610 static ssize_t
2611 ftrace_graph_write(struct file *file, const char __user *ubuf,
2612                    size_t cnt, loff_t *ppos)
2613 {
2614         unsigned char buffer[FTRACE_BUFF_MAX+1];
2615         unsigned long *array;
2616         size_t read = 0;
2617         ssize_t ret;
2618         int index = 0;
2619         char ch;
2620
2621         if (!cnt || cnt < 0)
2622                 return 0;
2623
2624         mutex_lock(&graph_lock);
2625
2626         if (ftrace_graph_count >= FTRACE_GRAPH_MAX_FUNCS) {
2627                 ret = -EBUSY;
2628                 goto out;
2629         }
2630
2631         if (file->f_mode & FMODE_READ) {
2632                 struct seq_file *m = file->private_data;
2633                 array = m->private;
2634         } else
2635                 array = file->private_data;
2636
2637         ret = get_user(ch, ubuf++);
2638         if (ret)
2639                 goto out;
2640         read++;
2641         cnt--;
2642
2643         /* skip white space */
2644         while (cnt && isspace(ch)) {
2645                 ret = get_user(ch, ubuf++);
2646                 if (ret)
2647                         goto out;
2648                 read++;
2649                 cnt--;
2650         }
2651
2652         if (isspace(ch)) {
2653                 *ppos += read;
2654                 ret = read;
2655                 goto out;
2656         }
2657
2658         while (cnt && !isspace(ch)) {
2659                 if (index < FTRACE_BUFF_MAX)
2660                         buffer[index++] = ch;
2661                 else {
2662                         ret = -EINVAL;
2663                         goto out;
2664                 }
2665                 ret = get_user(ch, ubuf++);
2666                 if (ret)
2667                         goto out;
2668                 read++;
2669                 cnt--;
2670         }
2671         buffer[index] = 0;
2672
2673         /* we allow only one expression at a time */
2674         ret = ftrace_set_func(array, &ftrace_graph_count, buffer);
2675         if (ret)
2676                 goto out;
2677
2678         file->f_pos += read;
2679
2680         ret = read;
2681  out:
2682         mutex_unlock(&graph_lock);
2683
2684         return ret;
2685 }
2686
2687 static const struct file_operations ftrace_graph_fops = {
2688         .open = ftrace_graph_open,
2689         .read = seq_read,
2690         .write = ftrace_graph_write,
2691 };
2692 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
2693
2694 static __init int ftrace_init_dyn_debugfs(struct dentry *d_tracer)
2695 {
2696         struct dentry *entry;
2697
2698         entry = debugfs_create_file("available_filter_functions", 0444,
2699                                     d_tracer, NULL, &ftrace_avail_fops);
2700         if (!entry)
2701                 pr_warning("Could not create debugfs "
2702                            "'available_filter_functions' entry\n");
2703
2704         entry = debugfs_create_file("failures", 0444,
2705                                     d_tracer, NULL, &ftrace_failures_fops);
2706         if (!entry)
2707                 pr_warning("Could not create debugfs 'failures' entry\n");
2708
2709         entry = debugfs_create_file("set_ftrace_filter", 0644, d_tracer,
2710                                     NULL, &ftrace_filter_fops);
2711         if (!entry)
2712                 pr_warning("Could not create debugfs "
2713                            "'set_ftrace_filter' entry\n");
2714
2715         entry = debugfs_create_file("set_ftrace_notrace", 0644, d_tracer,
2716                                     NULL, &ftrace_notrace_fops);
2717         if (!entry)
2718                 pr_warning("Could not create debugfs "
2719                            "'set_ftrace_notrace' entry\n");
2720
2721 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
2722         entry = debugfs_create_file("set_graph_function", 0444, d_tracer,
2723                                     NULL,
2724                                     &ftrace_graph_fops);
2725         if (!entry)
2726                 pr_warning("Could not create debugfs "
2727                            "'set_graph_function' entry\n");
2728 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
2729
2730         return 0;
2731 }
2732
2733 static int ftrace_convert_nops(struct module *mod,
2734                                unsigned long *start,
2735                                unsigned long *end)
2736 {
2737         unsigned long *p;
2738         unsigned long addr;
2739         unsigned long flags;
2740
2741         mutex_lock(&ftrace_lock);
2742         p = start;
2743         while (p < end) {
2744                 addr = ftrace_call_adjust(*p++);
2745                 /*
2746                  * Some architecture linkers will pad between
2747                  * the different mcount_loc sections of different
2748                  * object files to satisfy alignments.
2749                  * Skip any NULL pointers.
2750                  */
2751                 if (!addr)
2752                         continue;
2753                 ftrace_record_ip(addr);
2754         }
2755
2756         /* disable interrupts to prevent kstop machine */
2757         local_irq_save(flags);
2758         ftrace_update_code(mod);
2759         local_irq_restore(flags);
2760         mutex_unlock(&ftrace_lock);
2761
2762         return 0;
2763 }
2764
2765 void ftrace_init_module(struct module *mod,
2766                         unsigned long *start, unsigned long *end)
2767 {
2768         if (ftrace_disabled || start == end)
2769                 return;
2770         ftrace_convert_nops(mod, start, end);
2771 }
2772
2773 extern unsigned long __start_mcount_loc[];
2774 extern unsigned long __stop_mcount_loc[];
2775
2776 void __init ftrace_init(void)
2777 {
2778         unsigned long count, addr, flags;
2779         int ret;
2780
2781         /* Keep the ftrace pointer to the stub */
2782         addr = (unsigned long)ftrace_stub;
2783
2784         local_irq_save(flags);
2785         ftrace_dyn_arch_init(&addr);
2786         local_irq_restore(flags);
2787
2788         /* ftrace_dyn_arch_init places the return code in addr */
2789         if (addr)
2790                 goto failed;
2791
2792         count = __stop_mcount_loc - __start_mcount_loc;
2793
2794         ret = ftrace_dyn_table_alloc(count);
2795         if (ret)
2796                 goto failed;
2797
2798         last_ftrace_enabled = ftrace_enabled = 1;
2799
2800         ret = ftrace_convert_nops(NULL,
2801                                   __start_mcount_loc,
2802                                   __stop_mcount_loc);
2803
2804         return;
2805  failed:
2806         ftrace_disabled = 1;
2807 }
2808
2809 #else
2810
2811 static int __init ftrace_nodyn_init(void)
2812 {
2813         ftrace_enabled = 1;
2814         return 0;
2815 }
2816 device_initcall(ftrace_nodyn_init);
2817
2818 static inline int ftrace_init_dyn_debugfs(struct dentry *d_tracer) { return 0; }
2819 static inline void ftrace_startup_enable(int command) { }
2820 /* Keep as macros so we do not need to define the commands */
2821 # define ftrace_startup(command)        do { } while (0)
2822 # define ftrace_shutdown(command)       do { } while (0)
2823 # define ftrace_startup_sysctl()        do { } while (0)
2824 # define ftrace_shutdown_sysctl()       do { } while (0)
2825 #endif /* CONFIG_DYNAMIC_FTRACE */
2826
2827 static ssize_t
2828 ftrace_pid_read(struct file *file, char __user *ubuf,
2829                        size_t cnt, loff_t *ppos)
2830 {
2831         char buf[64];
2832         int r;
2833
2834         if (ftrace_pid_trace == ftrace_swapper_pid)
2835                 r = sprintf(buf, "swapper tasks\n");
2836         else if (ftrace_pid_trace)
2837                 r = sprintf(buf, "%u\n", pid_vnr(ftrace_pid_trace));
2838         else
2839                 r = sprintf(buf, "no pid\n");
2840
2841         return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
2842 }
2843
2844 static void clear_ftrace_swapper(void)
2845 {
2846         struct task_struct *p;
2847         int cpu;
2848
2849         get_online_cpus();
2850         for_each_online_cpu(cpu) {
2851                 p = idle_task(cpu);
2852                 clear_tsk_trace_trace(p);
2853         }
2854         put_online_cpus();
2855 }
2856
2857 static void set_ftrace_swapper(void)
2858 {
2859         struct task_struct *p;
2860         int cpu;
2861
2862         get_online_cpus();
2863         for_each_online_cpu(cpu) {
2864                 p = idle_task(cpu);
2865                 set_tsk_trace_trace(p);
2866         }
2867         put_online_cpus();
2868 }
2869
2870 static void clear_ftrace_pid(struct pid *pid)
2871 {
2872         struct task_struct *p;
2873
2874         rcu_read_lock();
2875         do_each_pid_task(pid, PIDTYPE_PID, p) {
2876                 clear_tsk_trace_trace(p);
2877         } while_each_pid_task(pid, PIDTYPE_PID, p);
2878         rcu_read_unlock();
2879
2880         put_pid(pid);
2881 }
2882
2883 static void set_ftrace_pid(struct pid *pid)
2884 {
2885         struct task_struct *p;
2886
2887         rcu_read_lock();
2888         do_each_pid_task(pid, PIDTYPE_PID, p) {
2889                 set_tsk_trace_trace(p);
2890         } while_each_pid_task(pid, PIDTYPE_PID, p);
2891         rcu_read_unlock();
2892 }
2893
2894 static void clear_ftrace_pid_task(struct pid **pid)
2895 {
2896         if (*pid == ftrace_swapper_pid)
2897                 clear_ftrace_swapper();
2898         else
2899                 clear_ftrace_pid(*pid);
2900
2901         *pid = NULL;
2902 }
2903
2904 static void set_ftrace_pid_task(struct pid *pid)
2905 {
2906         if (pid == ftrace_swapper_pid)
2907                 set_ftrace_swapper();
2908         else
2909                 set_ftrace_pid(pid);
2910 }
2911
2912 static ssize_t
2913 ftrace_pid_write(struct file *filp, const char __user *ubuf,
2914                    size_t cnt, loff_t *ppos)
2915 {
2916         struct pid *pid;
2917         char buf[64];
2918         long val;
2919         int ret;
2920
2921         if (cnt >= sizeof(buf))
2922                 return -EINVAL;
2923
2924         if (copy_from_user(&buf, ubuf, cnt))
2925                 return -EFAULT;
2926
2927         buf[cnt] = 0;
2928
2929         ret = strict_strtol(buf, 10, &val);
2930         if (ret < 0)
2931                 return ret;
2932
2933         mutex_lock(&ftrace_lock);
2934         if (val < 0) {
2935                 /* disable pid tracing */
2936                 if (!ftrace_pid_trace)
2937                         goto out;
2938
2939                 clear_ftrace_pid_task(&ftrace_pid_trace);
2940
2941         } else {
2942                 /* swapper task is special */
2943                 if (!val) {
2944                         pid = ftrace_swapper_pid;
2945                         if (pid == ftrace_pid_trace)
2946                                 goto out;
2947                 } else {
2948                         pid = find_get_pid(val);
2949
2950                         if (pid == ftrace_pid_trace) {
2951                                 put_pid(pid);
2952                                 goto out;
2953                         }
2954                 }
2955
2956                 if (ftrace_pid_trace)
2957                         clear_ftrace_pid_task(&ftrace_pid_trace);
2958
2959                 if (!pid)
2960                         goto out;
2961
2962                 ftrace_pid_trace = pid;
2963
2964                 set_ftrace_pid_task(ftrace_pid_trace);
2965         }
2966
2967         /* update the function call */
2968         ftrace_update_pid_func();
2969         ftrace_startup_enable(0);
2970
2971  out:
2972         mutex_unlock(&ftrace_lock);
2973
2974         return cnt;
2975 }
2976
2977 static const struct file_operations ftrace_pid_fops = {
2978         .read = ftrace_pid_read,
2979         .write = ftrace_pid_write,
2980 };
2981
2982 static __init int ftrace_init_debugfs(void)
2983 {
2984         struct dentry *d_tracer;
2985         struct dentry *entry;
2986
2987         d_tracer = tracing_init_dentry();
2988         if (!d_tracer)
2989                 return 0;
2990
2991         ftrace_init_dyn_debugfs(d_tracer);
2992
2993         entry = debugfs_create_file("set_ftrace_pid", 0644, d_tracer,
2994                                     NULL, &ftrace_pid_fops);
2995         if (!entry)
2996                 pr_warning("Could not create debugfs "
2997                            "'set_ftrace_pid' entry\n");
2998
2999         ftrace_profile_debugfs(d_tracer);
3000
3001         return 0;
3002 }
3003 fs_initcall(ftrace_init_debugfs);
3004
3005 /**
3006  * ftrace_kill - kill ftrace
3007  *
3008  * This function should be used by panic code. It stops ftrace
3009  * but in a not so nice way. If you need to simply kill ftrace
3010  * from a non-atomic section, use ftrace_kill.
3011  */
3012 void ftrace_kill(void)
3013 {
3014         ftrace_disabled = 1;
3015         ftrace_enabled = 0;
3016         clear_ftrace_function();
3017 }
3018
3019 /**
3020  * register_ftrace_function - register a function for profiling
3021  * @ops - ops structure that holds the function for profiling.
3022  *
3023  * Register a function to be called by all functions in the
3024  * kernel.
3025  *
3026  * Note: @ops->func and all the functions it calls must be labeled
3027  *       with "notrace", otherwise it will go into a
3028  *       recursive loop.
3029  */
3030 int register_ftrace_function(struct ftrace_ops *ops)
3031 {
3032         int ret;
3033
3034         if (unlikely(ftrace_disabled))
3035                 return -1;
3036
3037         mutex_lock(&ftrace_lock);
3038
3039         ret = __register_ftrace_function(ops);
3040         ftrace_startup(0);
3041
3042         mutex_unlock(&ftrace_lock);
3043         return ret;
3044 }
3045
3046 /**
3047  * unregister_ftrace_function - unregister a function for profiling.
3048  * @ops - ops structure that holds the function to unregister
3049  *
3050  * Unregister a function that was added to be called by ftrace profiling.
3051  */
3052 int unregister_ftrace_function(struct ftrace_ops *ops)
3053 {
3054         int ret;
3055
3056         mutex_lock(&ftrace_lock);
3057         ret = __unregister_ftrace_function(ops);
3058         ftrace_shutdown(0);
3059         mutex_unlock(&ftrace_lock);
3060
3061         return ret;
3062 }
3063
3064 int
3065 ftrace_enable_sysctl(struct ctl_table *table, int write,
3066                      struct file *file, void __user *buffer, size_t *lenp,
3067                      loff_t *ppos)
3068 {
3069         int ret;
3070
3071         if (unlikely(ftrace_disabled))
3072                 return -ENODEV;
3073
3074         mutex_lock(&ftrace_lock);
3075
3076         ret  = proc_dointvec(table, write, file, buffer, lenp, ppos);
3077
3078         if (ret || !write || (last_ftrace_enabled == ftrace_enabled))
3079                 goto out;
3080
3081         last_ftrace_enabled = ftrace_enabled;
3082
3083         if (ftrace_enabled) {
3084
3085                 ftrace_startup_sysctl();
3086
3087                 /* we are starting ftrace again */
3088                 if (ftrace_list != &ftrace_list_end) {
3089                         if (ftrace_list->next == &ftrace_list_end)
3090                                 ftrace_trace_function = ftrace_list->func;
3091                         else
3092                                 ftrace_trace_function = ftrace_list_func;
3093                 }
3094
3095         } else {
3096                 /* stopping ftrace calls (just send to ftrace_stub) */
3097                 ftrace_trace_function = ftrace_stub;
3098
3099                 ftrace_shutdown_sysctl();
3100         }
3101
3102  out:
3103         mutex_unlock(&ftrace_lock);
3104         return ret;
3105 }
3106
3107 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
3108
3109 static atomic_t ftrace_graph_active;
3110 static struct notifier_block ftrace_suspend_notifier;
3111
3112 int ftrace_graph_entry_stub(struct ftrace_graph_ent *trace)
3113 {
3114         return 0;
3115 }
3116
3117 /* The callbacks that hook a function */
3118 trace_func_graph_ret_t ftrace_graph_return =
3119                         (trace_func_graph_ret_t)ftrace_stub;
3120 trace_func_graph_ent_t ftrace_graph_entry = ftrace_graph_entry_stub;
3121
3122 /* Try to assign a return stack array on FTRACE_RETSTACK_ALLOC_SIZE tasks. */
3123 static int alloc_retstack_tasklist(struct ftrace_ret_stack **ret_stack_list)
3124 {
3125         int i;
3126         int ret = 0;
3127         unsigned long flags;
3128         int start = 0, end = FTRACE_RETSTACK_ALLOC_SIZE;
3129         struct task_struct *g, *t;
3130
3131         for (i = 0; i < FTRACE_RETSTACK_ALLOC_SIZE; i++) {
3132                 ret_stack_list[i] = kmalloc(FTRACE_RETFUNC_DEPTH
3133                                         * sizeof(struct ftrace_ret_stack),
3134                                         GFP_KERNEL);
3135                 if (!ret_stack_list[i]) {
3136                         start = 0;
3137                         end = i;
3138                         ret = -ENOMEM;
3139                         goto free;
3140                 }
3141         }
3142
3143         read_lock_irqsave(&tasklist_lock, flags);
3144         do_each_thread(g, t) {
3145                 if (start == end) {
3146                         ret = -EAGAIN;
3147                         goto unlock;
3148                 }
3149
3150                 if (t->ret_stack == NULL) {
3151                         t->curr_ret_stack = -1;
3152                         /* Make sure IRQs see the -1 first: */
3153                         barrier();
3154                         t->ret_stack = ret_stack_list[start++];
3155                         atomic_set(&t->tracing_graph_pause, 0);
3156                         atomic_set(&t->trace_overrun, 0);
3157                 }
3158         } while_each_thread(g, t);
3159
3160 unlock:
3161         read_unlock_irqrestore(&tasklist_lock, flags);
3162 free:
3163         for (i = start; i < end; i++)
3164                 kfree(ret_stack_list[i]);
3165         return ret;
3166 }
3167
3168 static void
3169 ftrace_graph_probe_sched_switch(struct rq *__rq, struct task_struct *prev,
3170                                 struct task_struct *next)
3171 {
3172         unsigned long long timestamp;
3173         int index;
3174
3175         /*
3176          * Does the user want to count the time a function was asleep.
3177          * If so, do not update the time stamps.
3178          */
3179         if (trace_flags & TRACE_ITER_SLEEP_TIME)
3180                 return;
3181
3182         timestamp = trace_clock_local();
3183
3184         prev->ftrace_timestamp = timestamp;
3185
3186         /* only process tasks that we timestamped */
3187         if (!next->ftrace_timestamp)
3188                 return;
3189
3190         /*
3191          * Update all the counters in next to make up for the
3192          * time next was sleeping.
3193          */
3194         timestamp -= next->ftrace_timestamp;
3195
3196         for (index = next->curr_ret_stack; index >= 0; index--)
3197                 next->ret_stack[index].calltime += timestamp;
3198 }
3199
3200 /* Allocate a return stack for each task */
3201 static int start_graph_tracing(void)
3202 {
3203         struct ftrace_ret_stack **ret_stack_list;
3204         int ret, cpu;
3205
3206         ret_stack_list = kmalloc(FTRACE_RETSTACK_ALLOC_SIZE *
3207                                 sizeof(struct ftrace_ret_stack *),
3208                                 GFP_KERNEL);
3209
3210         if (!ret_stack_list)
3211                 return -ENOMEM;
3212
3213         /* The cpu_boot init_task->ret_stack will never be freed */
3214         for_each_online_cpu(cpu)
3215                 ftrace_graph_init_task(idle_task(cpu));
3216
3217         do {
3218                 ret = alloc_retstack_tasklist(ret_stack_list);
3219         } while (ret == -EAGAIN);
3220
3221         if (!ret) {
3222                 ret = register_trace_sched_switch(ftrace_graph_probe_sched_switch);
3223                 if (ret)
3224                         pr_info("ftrace_graph: Couldn't activate tracepoint"
3225                                 " probe to kernel_sched_switch\n");
3226         }
3227
3228         kfree(ret_stack_list);
3229         return ret;
3230 }
3231
3232 /*
3233  * Hibernation protection.
3234  * The state of the current task is too much unstable during
3235  * suspend/restore to disk. We want to protect against that.
3236  */
3237 static int
3238 ftrace_suspend_notifier_call(struct notifier_block *bl, unsigned long state,
3239                                                         void *unused)
3240 {
3241         switch (state) {
3242         case PM_HIBERNATION_PREPARE:
3243                 pause_graph_tracing();
3244                 break;
3245
3246         case PM_POST_HIBERNATION:
3247                 unpause_graph_tracing();
3248                 break;
3249         }
3250         return NOTIFY_DONE;
3251 }
3252
3253 int register_ftrace_graph(trace_func_graph_ret_t retfunc,
3254                         trace_func_graph_ent_t entryfunc)
3255 {
3256         int ret = 0;
3257
3258         mutex_lock(&ftrace_lock);
3259
3260         /* we currently allow only one tracer registered at a time */
3261         if (atomic_read(&ftrace_graph_active)) {
3262                 ret = -EBUSY;
3263                 goto out;
3264         }
3265
3266         ftrace_suspend_notifier.notifier_call = ftrace_suspend_notifier_call;
3267         register_pm_notifier(&ftrace_suspend_notifier);
3268
3269         atomic_inc(&ftrace_graph_active);
3270         ret = start_graph_tracing();
3271         if (ret) {
3272                 atomic_dec(&ftrace_graph_active);
3273                 goto out;
3274         }
3275
3276         ftrace_graph_return = retfunc;
3277         ftrace_graph_entry = entryfunc;
3278
3279         ftrace_startup(FTRACE_START_FUNC_RET);
3280
3281 out:
3282         mutex_unlock(&ftrace_lock);
3283         return ret;
3284 }
3285
3286 void unregister_ftrace_graph(void)
3287 {
3288         mutex_lock(&ftrace_lock);
3289
3290         atomic_dec(&ftrace_graph_active);
3291         unregister_trace_sched_switch(ftrace_graph_probe_sched_switch);
3292         ftrace_graph_return = (trace_func_graph_ret_t)ftrace_stub;
3293         ftrace_graph_entry = ftrace_graph_entry_stub;
3294         ftrace_shutdown(FTRACE_STOP_FUNC_RET);
3295         unregister_pm_notifier(&ftrace_suspend_notifier);
3296
3297         mutex_unlock(&ftrace_lock);
3298 }
3299
3300 /* Allocate a return stack for newly created task */
3301 void ftrace_graph_init_task(struct task_struct *t)
3302 {
3303         if (atomic_read(&ftrace_graph_active)) {
3304                 t->ret_stack = kmalloc(FTRACE_RETFUNC_DEPTH
3305                                 * sizeof(struct ftrace_ret_stack),
3306                                 GFP_KERNEL);
3307                 if (!t->ret_stack)
3308                         return;
3309                 t->curr_ret_stack = -1;
3310                 atomic_set(&t->tracing_graph_pause, 0);
3311                 atomic_set(&t->trace_overrun, 0);
3312                 t->ftrace_timestamp = 0;
3313         } else
3314                 t->ret_stack = NULL;
3315 }
3316
3317 void ftrace_graph_exit_task(struct task_struct *t)
3318 {
3319         struct ftrace_ret_stack *ret_stack = t->ret_stack;
3320
3321         t->ret_stack = NULL;
3322         /* NULL must become visible to IRQs before we free it: */
3323         barrier();
3324
3325         kfree(ret_stack);
3326 }
3327
3328 void ftrace_graph_stop(void)
3329 {
3330         ftrace_stop();
3331 }
3332 #endif
3333