SAFE public projects git trees. - safe/jmp/linux-2.6/blob - kernel/trace/ftrace.c

   1 /*
   2  * Infrastructure for profiling code inserted by 'gcc -pg'.
   3  *
   4  * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
   5  * Copyright (C) 2004-2008 Ingo Molnar <mingo@redhat.com>
   6  *
   7  * Originally ported from the -rt patch by:
   8  *   Copyright (C) 2007 Arnaldo Carvalho de Melo <acme@redhat.com>
   9  *
  10  * Based on code in the latency_tracer, that is:
  11  *
  12  *  Copyright (C) 2004-2006 Ingo Molnar
  13  *  Copyright (C) 2004 William Lee Irwin III
  14  */
  15
  16 #include <linux/stop_machine.h>
  17 #include <linux/clocksource.h>
  18 #include <linux/kallsyms.h>
  19 #include <linux/seq_file.h>
  20 #include <linux/suspend.h>
  21 #include <linux/debugfs.h>
  22 #include <linux/hardirq.h>
  23 #include <linux/kthread.h>
  24 #include <linux/uaccess.h>
  25 #include <linux/kprobes.h>
  26 #include <linux/ftrace.h>
  27 #include <linux/sysctl.h>
  28 #include <linux/ctype.h>
  29 #include <linux/list.h>
  30 #include <linux/hash.h>
  31
  32 #include <trace/events/sched.h>
  33
  34 #include <asm/ftrace.h>
  35
  36 #include "trace_output.h"
  37 #include "trace_stat.h"
  38
  39 #define FTRACE_WARN_ON(cond)                    \
  40         do {                                    \
  41                 if (WARN_ON(cond))              \
  42                         ftrace_kill();          \
  43         } while (0)
  44
  45 #define FTRACE_WARN_ON_ONCE(cond)               \
  46         do {                                    \
  47                 if (WARN_ON_ONCE(cond))         \
  48                         ftrace_kill();          \
  49         } while (0)
  50
  51 /* hash bits for specific function selection */
  52 #define FTRACE_HASH_BITS 7
  53 #define FTRACE_FUNC_HASHSIZE (1 << FTRACE_HASH_BITS)
  54
  55 /* ftrace_enabled is a method to turn ftrace on or off */
  56 int ftrace_enabled __read_mostly;
  57 static int last_ftrace_enabled;
  58
  59 /* Quick disabling of function tracer. */
  60 int function_trace_stop;
  61
  62 /*
  63  * ftrace_disabled is set when an anomaly is discovered.
  64  * ftrace_disabled is much stronger than ftrace_enabled.
  65  */
  66 static int ftrace_disabled __read_mostly;
  67
  68 static DEFINE_MUTEX(ftrace_lock);
  69
  70 static struct ftrace_ops ftrace_list_end __read_mostly =
  71 {
  72         .func           = ftrace_stub,
  73 };
  74
  75 static struct ftrace_ops *ftrace_list __read_mostly = &ftrace_list_end;
  76 ftrace_func_t ftrace_trace_function __read_mostly = ftrace_stub;
  77 ftrace_func_t __ftrace_trace_function __read_mostly = ftrace_stub;
  78 ftrace_func_t ftrace_pid_function __read_mostly = ftrace_stub;
  79
  80 static void ftrace_list_func(unsigned long ip, unsigned long parent_ip)
  81 {
  82         struct ftrace_ops *op = ftrace_list;
  83
  84         /* in case someone actually ports this to alpha! */
  85         read_barrier_depends();
  86
  87         while (op != &ftrace_list_end) {
  88                 /* silly alpha */
  89                 read_barrier_depends();
  90                 op->func(ip, parent_ip);
  91                 op = op->next;
  92         };
  93 }
  94
  95 static void ftrace_pid_func(unsigned long ip, unsigned long parent_ip)
  96 {
  97         if (!test_tsk_trace_trace(current))
  98                 return;
  99
 100         ftrace_pid_function(ip, parent_ip);
 101 }
 102
 103 static void set_ftrace_pid_function(ftrace_func_t func)
 104 {
 105         /* do not set ftrace_pid_function to itself! */
 106         if (func != ftrace_pid_func)
 107                 ftrace_pid_function = func;
 108 }
 109
 110 /**
 111  * clear_ftrace_function - reset the ftrace function
 112  *
 113  * This NULLs the ftrace function and in essence stops
 114  * tracing.  There may be lag
 115  */
 116 void clear_ftrace_function(void)
 117 {
 118         ftrace_trace_function = ftrace_stub;
 119         __ftrace_trace_function = ftrace_stub;
 120         ftrace_pid_function = ftrace_stub;
 121 }
 122
 123 #ifndef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST
 124 /*
 125  * For those archs that do not test ftrace_trace_stop in their
 126  * mcount call site, we need to do it from C.
 127  */
 128 static void ftrace_test_stop_func(unsigned long ip, unsigned long parent_ip)
 129 {
 130         if (function_trace_stop)
 131                 return;
 132
 133         __ftrace_trace_function(ip, parent_ip);
 134 }
 135 #endif
 136
 137 static int __register_ftrace_function(struct ftrace_ops *ops)
 138 {
 139         ops->next = ftrace_list;
 140         /*
 141          * We are entering ops into the ftrace_list but another
 142          * CPU might be walking that list. We need to make sure
 143          * the ops->next pointer is valid before another CPU sees
 144          * the ops pointer included into the ftrace_list.
 145          */
 146         smp_wmb();
 147         ftrace_list = ops;
 148
 149         if (ftrace_enabled) {
 150                 ftrace_func_t func;
 151
 152                 if (ops->next == &ftrace_list_end)
 153                         func = ops->func;
 154                 else
 155                         func = ftrace_list_func;
 156
 157                 if (ftrace_pid_trace) {
 158                         set_ftrace_pid_function(func);
 159                         func = ftrace_pid_func;
 160                 }
 161
 162                 /*
 163                  * For one func, simply call it directly.
 164                  * For more than one func, call the chain.
 165                  */
 166 #ifdef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST
 167                 ftrace_trace_function = func;
 168 #else
 169                 __ftrace_trace_function = func;
 170                 ftrace_trace_function = ftrace_test_stop_func;
 171 #endif
 172         }
 173
 174         return 0;
 175 }
 176
 177 static int __unregister_ftrace_function(struct ftrace_ops *ops)
 178 {
 179         struct ftrace_ops **p;
 180
 181         /*
 182          * If we are removing the last function, then simply point
 183          * to the ftrace_stub.
 184          */
 185         if (ftrace_list == ops && ops->next == &ftrace_list_end) {
 186                 ftrace_trace_function = ftrace_stub;
 187                 ftrace_list = &ftrace_list_end;
 188                 return 0;
 189         }
 190
 191         for (p = &ftrace_list; *p != &ftrace_list_end; p = &(*p)->next)
 192                 if (*p == ops)
 193                         break;
 194
 195         if (*p != ops)
 196                 return -1;
 197
 198         *p = (*p)->next;
 199
 200         if (ftrace_enabled) {
 201                 /* If we only have one func left, then call that directly */
 202                 if (ftrace_list->next == &ftrace_list_end) {
 203                         ftrace_func_t func = ftrace_list->func;
 204
 205                         if (ftrace_pid_trace) {
 206                                 set_ftrace_pid_function(func);
 207                                 func = ftrace_pid_func;
 208                         }
 209 #ifdef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST
 210                         ftrace_trace_function = func;
 211 #else
 212                         __ftrace_trace_function = func;
 213 #endif
 214                 }
 215         }
 216
 217         return 0;
 218 }
 219
 220 static void ftrace_update_pid_func(void)
 221 {
 222         ftrace_func_t func;
 223
 224         if (ftrace_trace_function == ftrace_stub)
 225                 return;
 226
 227         func = ftrace_trace_function;
 228
 229         if (ftrace_pid_trace) {
 230                 set_ftrace_pid_function(func);
 231                 func = ftrace_pid_func;
 232         } else {
 233                 if (func == ftrace_pid_func)
 234                         func = ftrace_pid_function;
 235         }
 236
 237 #ifdef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST
 238         ftrace_trace_function = func;
 239 #else
 240         __ftrace_trace_function = func;
 241 #endif
 242 }
 243
 244 #ifdef CONFIG_FUNCTION_PROFILER
 245 struct ftrace_profile {
 246         struct hlist_node               node;
 247         unsigned long                   ip;
 248         unsigned long                   counter;
 249 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
 250         unsigned long long              time;
 251 #endif
 252 };
 253
 254 struct ftrace_profile_page {
 255         struct ftrace_profile_page      *next;
 256         unsigned long                   index;
 257         struct ftrace_profile           records[];
 258 };
 259
 260 struct ftrace_profile_stat {
 261         atomic_t                        disabled;
 262         struct hlist_head               *hash;
 263         struct ftrace_profile_page      *pages;
 264         struct ftrace_profile_page      *start;
 265         struct tracer_stat              stat;
 266 };
 267
 268 #define PROFILE_RECORDS_SIZE                                            \
 269         (PAGE_SIZE - offsetof(struct ftrace_profile_page, records))
 270
 271 #define PROFILES_PER_PAGE                                       \
 272         (PROFILE_RECORDS_SIZE / sizeof(struct ftrace_profile))
 273
 274 static int ftrace_profile_bits __read_mostly;
 275 static int ftrace_profile_enabled __read_mostly;
 276
 277 /* ftrace_profile_lock - synchronize the enable and disable of the profiler */
 278 static DEFINE_MUTEX(ftrace_profile_lock);
 279
 280 static DEFINE_PER_CPU(struct ftrace_profile_stat, ftrace_profile_stats);
 281
 282 #define FTRACE_PROFILE_HASH_SIZE 1024 /* must be power of 2 */
 283
 284 static void *
 285 function_stat_next(void *v, int idx)
 286 {
 287         struct ftrace_profile *rec = v;
 288         struct ftrace_profile_page *pg;
 289
 290         pg = (struct ftrace_profile_page *)((unsigned long)rec & PAGE_MASK);
 291
 292  again:
 293         rec++;
 294         if ((void *)rec >= (void *)&pg->records[pg->index]) {
 295                 pg = pg->next;
 296                 if (!pg)
 297                         return NULL;
 298                 rec = &pg->records[0];
 299                 if (!rec->counter)
 300                         goto again;
 301         }
 302
 303         return rec;
 304 }
 305
 306 static void *function_stat_start(struct tracer_stat *trace)
 307 {
 308         struct ftrace_profile_stat *stat =
 309                 container_of(trace, struct ftrace_profile_stat, stat);
 310
 311         if (!stat || !stat->start)
 312                 return NULL;
 313
 314         return function_stat_next(&stat->start->records[0], 0);
 315 }
 316
 317 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
 318 /* function graph compares on total time */
 319 static int function_stat_cmp(void *p1, void *p2)
 320 {
 321         struct ftrace_profile *a = p1;
 322         struct ftrace_profile *b = p2;
 323
 324         if (a->time < b->time)
 325                 return -1;
 326         if (a->time > b->time)
 327                 return 1;
 328         else
 329                 return 0;
 330 }
 331 #else
 332 /* not function graph compares against hits */
 333 static int function_stat_cmp(void *p1, void *p2)
 334 {
 335         struct ftrace_profile *a = p1;
 336         struct ftrace_profile *b = p2;
 337
 338         if (a->counter < b->counter)
 339                 return -1;
 340         if (a->counter > b->counter)
 341                 return 1;
 342         else
 343                 return 0;
 344 }
 345 #endif
 346
 347 static int function_stat_headers(struct seq_file *m)
 348 {
 349 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
 350         seq_printf(m, "  Function                               "
 351                    "Hit    Time            Avg\n"
 352                       "  --------                               "
 353                    "---    ----            ---\n");
 354 #else
 355         seq_printf(m, "  Function                               Hit\n"
 356                       "  --------                               ---\n");
 357 #endif
 358         return 0;
 359 }
 360
 361 static int function_stat_show(struct seq_file *m, void *v)
 362 {
 363         struct ftrace_profile *rec = v;
 364         char str[KSYM_SYMBOL_LEN];
 365 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
 366         static DEFINE_MUTEX(mutex);
 367         static struct trace_seq s;
 368         unsigned long long avg;
 369 #endif
 370
 371         kallsyms_lookup(rec->ip, NULL, NULL, NULL, str);
 372         seq_printf(m, "  %-30.30s  %10lu", str, rec->counter);
 373
 374 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
 375         seq_printf(m, "    ");
 376         avg = rec->time;
 377         do_div(avg, rec->counter);
 378
 379         mutex_lock(&mutex);
 380         trace_seq_init(&s);
 381         trace_print_graph_duration(rec->time, &s);
 382         trace_seq_puts(&s, "    ");
 383         trace_print_graph_duration(avg, &s);
 384         trace_print_seq(m, &s);
 385         mutex_unlock(&mutex);
 386 #endif
 387         seq_putc(m, '\n');
 388
 389         return 0;
 390 }
 391
 392 static void ftrace_profile_reset(struct ftrace_profile_stat *stat)
 393 {
 394         struct ftrace_profile_page *pg;
 395
 396         pg = stat->pages = stat->start;
 397
 398         while (pg) {
 399                 memset(pg->records, 0, PROFILE_RECORDS_SIZE);
 400                 pg->index = 0;
 401                 pg = pg->next;
 402         }
 403
 404         memset(stat->hash, 0,
 405                FTRACE_PROFILE_HASH_SIZE * sizeof(struct hlist_head));
 406 }
 407
 408 int ftrace_profile_pages_init(struct ftrace_profile_stat *stat)
 409 {
 410         struct ftrace_profile_page *pg;
 411         int functions;
 412         int pages;
 413         int i;
 414
 415         /* If we already allocated, do nothing */
 416         if (stat->pages)
 417                 return 0;
 418
 419         stat->pages = (void *)get_zeroed_page(GFP_KERNEL);
 420         if (!stat->pages)
 421                 return -ENOMEM;
 422
 423 #ifdef CONFIG_DYNAMIC_FTRACE
 424         functions = ftrace_update_tot_cnt;
 425 #else
 426         /*
 427          * We do not know the number of functions that exist because
 428          * dynamic tracing is what counts them. With past experience
 429          * we have around 20K functions. That should be more than enough.
 430          * It is highly unlikely we will execute every function in
 431          * the kernel.
 432          */
 433         functions = 20000;
 434 #endif
 435
 436         pg = stat->start = stat->pages;
 437
 438         pages = DIV_ROUND_UP(functions, PROFILES_PER_PAGE);
 439
 440         for (i = 0; i < pages; i++) {
 441                 pg->next = (void *)get_zeroed_page(GFP_KERNEL);
 442                 if (!pg->next)
 443                         goto out_free;
 444                 pg = pg->next;
 445         }
 446
 447         return 0;
 448
 449  out_free:
 450         pg = stat->start;
 451         while (pg) {
 452                 unsigned long tmp = (unsigned long)pg;
 453
 454                 pg = pg->next;
 455                 free_page(tmp);
 456         }
 457
 458         free_page((unsigned long)stat->pages);
 459         stat->pages = NULL;
 460         stat->start = NULL;
 461
 462         return -ENOMEM;
 463 }
 464
 465 static int ftrace_profile_init_cpu(int cpu)
 466 {
 467         struct ftrace_profile_stat *stat;
 468         int size;
 469
 470         stat = &per_cpu(ftrace_profile_stats, cpu);
 471
 472         if (stat->hash) {
 473                 /* If the profile is already created, simply reset it */
 474                 ftrace_profile_reset(stat);
 475                 return 0;
 476         }
 477
 478         /*
 479          * We are profiling all functions, but usually only a few thousand
 480          * functions are hit. We'll make a hash of 1024 items.
 481          */
 482         size = FTRACE_PROFILE_HASH_SIZE;
 483
 484         stat->hash = kzalloc(sizeof(struct hlist_head) * size, GFP_KERNEL);
 485
 486         if (!stat->hash)
 487                 return -ENOMEM;
 488
 489         if (!ftrace_profile_bits) {
 490                 size--;
 491
 492                 for (; size; size >>= 1)
 493                         ftrace_profile_bits++;
 494         }
 495
 496         /* Preallocate the function profiling pages */
 497         if (ftrace_profile_pages_init(stat) < 0) {
 498                 kfree(stat->hash);
 499                 stat->hash = NULL;
 500                 return -ENOMEM;
 501         }
 502
 503         return 0;
 504 }
 505
 506 static int ftrace_profile_init(void)
 507 {
 508         int cpu;
 509         int ret = 0;
 510
 511         for_each_online_cpu(cpu) {
 512                 ret = ftrace_profile_init_cpu(cpu);
 513                 if (ret)
 514                         break;
 515         }
 516
 517         return ret;
 518 }
 519
 520 /* interrupts must be disabled */
 521 static struct ftrace_profile *
 522 ftrace_find_profiled_func(struct ftrace_profile_stat *stat, unsigned long ip)
 523 {
 524         struct ftrace_profile *rec;
 525         struct hlist_head *hhd;
 526         struct hlist_node *n;
 527         unsigned long key;
 528
 529         key = hash_long(ip, ftrace_profile_bits);
 530         hhd = &stat->hash[key];
 531
 532         if (hlist_empty(hhd))
 533                 return NULL;
 534
 535         hlist_for_each_entry_rcu(rec, n, hhd, node) {
 536                 if (rec->ip == ip)
 537                         return rec;
 538         }
 539
 540         return NULL;
 541 }
 542
 543 static void ftrace_add_profile(struct ftrace_profile_stat *stat,
 544                                struct ftrace_profile *rec)
 545 {
 546         unsigned long key;
 547
 548         key = hash_long(rec->ip, ftrace_profile_bits);
 549         hlist_add_head_rcu(&rec->node, &stat->hash[key]);
 550 }
 551
 552 /*
 553  * The memory is already allocated, this simply finds a new record to use.
 554  */
 555 static struct ftrace_profile *
 556 ftrace_profile_alloc(struct ftrace_profile_stat *stat, unsigned long ip)
 557 {
 558         struct ftrace_profile *rec = NULL;
 559
 560         /* prevent recursion (from NMIs) */
 561         if (atomic_inc_return(&stat->disabled) != 1)
 562                 goto out;
 563
 564         /*
 565          * Try to find the function again since an NMI
 566          * could have added it
 567          */
 568         rec = ftrace_find_profiled_func(stat, ip);
 569         if (rec)
 570                 goto out;
 571
 572         if (stat->pages->index == PROFILES_PER_PAGE) {
 573                 if (!stat->pages->next)
 574                         goto out;
 575                 stat->pages = stat->pages->next;
 576         }
 577
 578         rec = &stat->pages->records[stat->pages->index++];
 579         rec->ip = ip;
 580         ftrace_add_profile(stat, rec);
 581
 582  out:
 583         atomic_dec(&stat->disabled);
 584
 585         return rec;
 586 }
 587
 588 static void
 589 function_profile_call(unsigned long ip, unsigned long parent_ip)
 590 {
 591         struct ftrace_profile_stat *stat;
 592         struct ftrace_profile *rec;
 593         unsigned long flags;
 594
 595         if (!ftrace_profile_enabled)
 596                 return;
 597
 598         local_irq_save(flags);
 599
 600         stat = &__get_cpu_var(ftrace_profile_stats);
 601         if (!stat->hash)
 602                 goto out;
 603
 604         rec = ftrace_find_profiled_func(stat, ip);
 605         if (!rec) {
 606                 rec = ftrace_profile_alloc(stat, ip);
 607                 if (!rec)
 608                         goto out;
 609         }
 610
 611         rec->counter++;
 612  out:
 613         local_irq_restore(flags);
 614 }
 615
 616 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
 617 static int profile_graph_entry(struct ftrace_graph_ent *trace)
 618 {
 619         function_profile_call(trace->func, 0);
 620         return 1;
 621 }
 622
 623 static void profile_graph_return(struct ftrace_graph_ret *trace)
 624 {
 625         struct ftrace_profile_stat *stat;
 626         unsigned long long calltime;
 627         struct ftrace_profile *rec;
 628         unsigned long flags;
 629
 630         local_irq_save(flags);
 631         stat = &__get_cpu_var(ftrace_profile_stats);
 632         if (!stat->hash)
 633                 goto out;
 634
 635         calltime = trace->rettime - trace->calltime;
 636
 637         if (!(trace_flags & TRACE_ITER_GRAPH_TIME)) {
 638                 int index;
 639
 640                 index = trace->depth;
 641
 642                 /* Append this call time to the parent time to subtract */
 643                 if (index)
 644                         current->ret_stack[index - 1].subtime += calltime;
 645
 646                 if (current->ret_stack[index].subtime < calltime)
 647                         calltime -= current->ret_stack[index].subtime;
 648                 else
 649                         calltime = 0;
 650         }
 651
 652         rec = ftrace_find_profiled_func(stat, trace->func);
 653         if (rec)
 654                 rec->time += calltime;
 655
 656  out:
 657         local_irq_restore(flags);
 658 }
 659
 660 static int register_ftrace_profiler(void)
 661 {
 662         return register_ftrace_graph(&profile_graph_return,
 663                                      &profile_graph_entry);
 664 }
 665
 666 static void unregister_ftrace_profiler(void)
 667 {
 668         unregister_ftrace_graph();
 669 }
 670 #else
 671 static struct ftrace_ops ftrace_profile_ops __read_mostly =
 672 {
 673         .func           = function_profile_call,
 674 };
 675
 676 static int register_ftrace_profiler(void)
 677 {
 678         return register_ftrace_function(&ftrace_profile_ops);
 679 }
 680
 681 static void unregister_ftrace_profiler(void)
 682 {
 683         unregister_ftrace_function(&ftrace_profile_ops);
 684 }
 685 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
 686
 687 static ssize_t
 688 ftrace_profile_write(struct file *filp, const char __user *ubuf,
 689                      size_t cnt, loff_t *ppos)
 690 {
 691         unsigned long val;
 692         char buf[64];           /* big enough to hold a number */
 693         int ret;
 694
 695         if (cnt >= sizeof(buf))
 696                 return -EINVAL;
 697
 698         if (copy_from_user(&buf, ubuf, cnt))
 699                 return -EFAULT;
 700
 701         buf[cnt] = 0;
 702
 703         ret = strict_strtoul(buf, 10, &val);
 704         if (ret < 0)
 705                 return ret;
 706
 707         val = !!val;
 708
 709         mutex_lock(&ftrace_profile_lock);
 710         if (ftrace_profile_enabled ^ val) {
 711                 if (val) {
 712                         ret = ftrace_profile_init();
 713                         if (ret < 0) {
 714                                 cnt = ret;
 715                                 goto out;
 716                         }
 717
 718                         ret = register_ftrace_profiler();
 719                         if (ret < 0) {
 720                                 cnt = ret;
 721                                 goto out;
 722                         }
 723                         ftrace_profile_enabled = 1;
 724                 } else {
 725                         ftrace_profile_enabled = 0;
 726                         unregister_ftrace_profiler();
 727                 }
 728         }
 729  out:
 730         mutex_unlock(&ftrace_profile_lock);
 731
 732         filp->f_pos += cnt;
 733
 734         return cnt;
 735 }
 736
 737 static ssize_t
 738 ftrace_profile_read(struct file *filp, char __user *ubuf,
 739                      size_t cnt, loff_t *ppos)
 740 {
 741         char buf[64];           /* big enough to hold a number */
 742         int r;
 743
 744         r = sprintf(buf, "%u\n", ftrace_profile_enabled);
 745         return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
 746 }
 747
 748 static const struct file_operations ftrace_profile_fops = {
 749         .open           = tracing_open_generic,
 750         .read           = ftrace_profile_read,
 751         .write          = ftrace_profile_write,
 752 };
 753
 754 /* used to initialize the real stat files */
 755 static struct tracer_stat function_stats __initdata = {
 756         .name           = "functions",
 757         .stat_start     = function_stat_start,
 758         .stat_next      = function_stat_next,
 759         .stat_cmp       = function_stat_cmp,
 760         .stat_headers   = function_stat_headers,
 761         .stat_show      = function_stat_show
 762 };
 763
 764 static void ftrace_profile_debugfs(struct dentry *d_tracer)
 765 {
 766         struct ftrace_profile_stat *stat;
 767         struct dentry *entry;
 768         char *name;
 769         int ret;
 770         int cpu;
 771
 772         for_each_possible_cpu(cpu) {
 773                 stat = &per_cpu(ftrace_profile_stats, cpu);
 774
 775                 /* allocate enough for function name + cpu number */
 776                 name = kmalloc(32, GFP_KERNEL);
 777                 if (!name) {
 778                         /*
 779                          * The files created are permanent, if something happens
 780                          * we still do not free memory.
 781                          */
 782                         kfree(stat);
 783                         WARN(1,
 784                              "Could not allocate stat file for cpu %d\n",
 785                              cpu);
 786                         return;
 787                 }
 788                 stat->stat = function_stats;
 789                 snprintf(name, 32, "function%d", cpu);
 790                 stat->stat.name = name;
 791                 ret = register_stat_tracer(&stat->stat);
 792                 if (ret) {
 793                         WARN(1,
 794                              "Could not register function stat for cpu %d\n",
 795                              cpu);
 796                         kfree(name);
 797                         return;
 798                 }
 799         }
 800
 801         entry = debugfs_create_file("function_profile_enabled", 0644,
 802                                     d_tracer, NULL, &ftrace_profile_fops);
 803         if (!entry)
 804                 pr_warning("Could not create debugfs "
 805                            "'function_profile_enabled' entry\n");
 806 }
 807
 808 #else /* CONFIG_FUNCTION_PROFILER */
 809 static void ftrace_profile_debugfs(struct dentry *d_tracer)
 810 {
 811 }
 812 #endif /* CONFIG_FUNCTION_PROFILER */
 813
 814 /* set when tracing only a pid */
 815 struct pid *ftrace_pid_trace;
 816 static struct pid * const ftrace_swapper_pid = &init_struct_pid;
 817
 818 #ifdef CONFIG_DYNAMIC_FTRACE
 819
 820 #ifndef CONFIG_FTRACE_MCOUNT_RECORD
 821 # error Dynamic ftrace depends on MCOUNT_RECORD
 822 #endif
 823
 824 static struct hlist_head ftrace_func_hash[FTRACE_FUNC_HASHSIZE] __read_mostly;
 825
 826 struct ftrace_func_probe {
 827         struct hlist_node       node;
 828         struct ftrace_probe_ops *ops;
 829         unsigned long           flags;
 830         unsigned long           ip;
 831         void                    *data;
 832         struct rcu_head         rcu;
 833 };
 834
 835 enum {
 836         FTRACE_ENABLE_CALLS             = (1 << 0),
 837         FTRACE_DISABLE_CALLS            = (1 << 1),
 838         FTRACE_UPDATE_TRACE_FUNC        = (1 << 2),
 839         FTRACE_ENABLE_MCOUNT            = (1 << 3),
 840         FTRACE_DISABLE_MCOUNT           = (1 << 4),
 841         FTRACE_START_FUNC_RET           = (1 << 5),
 842         FTRACE_STOP_FUNC_RET            = (1 << 6),
 843 };
 844
 845 static int ftrace_filtered;
 846
 847 static struct dyn_ftrace *ftrace_new_addrs;
 848
 849 static DEFINE_MUTEX(ftrace_regex_lock);
 850
 851 struct ftrace_page {
 852         struct ftrace_page      *next;
 853         int                     index;
 854         struct dyn_ftrace       records[];
 855 };
 856
 857 #define ENTRIES_PER_PAGE \
 858   ((PAGE_SIZE - sizeof(struct ftrace_page)) / sizeof(struct dyn_ftrace))
 859
 860 /* estimate from running different kernels */
 861 #define NR_TO_INIT              10000
 862
 863 static struct ftrace_page       *ftrace_pages_start;
 864 static struct ftrace_page       *ftrace_pages;
 865
 866 static struct dyn_ftrace *ftrace_free_records;
 867
 868 /*
 869  * This is a double for. Do not use 'break' to break out of the loop,
 870  * you must use a goto.
 871  */
 872 #define do_for_each_ftrace_rec(pg, rec)                                 \
 873         for (pg = ftrace_pages_start; pg; pg = pg->next) {              \
 874                 int _____i;                                             \
 875                 for (_____i = 0; _____i < pg->index; _____i++) {        \
 876                         rec = &pg->records[_____i];
 877
 878 #define while_for_each_ftrace_rec()             \
 879                 }                               \
 880         }
 881
 882 #ifdef CONFIG_KPROBES
 883
 884 static int frozen_record_count;
 885
 886 static inline void freeze_record(struct dyn_ftrace *rec)
 887 {
 888         if (!(rec->flags & FTRACE_FL_FROZEN)) {
 889                 rec->flags |= FTRACE_FL_FROZEN;
 890                 frozen_record_count++;
 891         }
 892 }
 893
 894 static inline void unfreeze_record(struct dyn_ftrace *rec)
 895 {
 896         if (rec->flags & FTRACE_FL_FROZEN) {
 897                 rec->flags &= ~FTRACE_FL_FROZEN;
 898                 frozen_record_count--;
 899         }
 900 }
 901
 902 static inline int record_frozen(struct dyn_ftrace *rec)
 903 {
 904         return rec->flags & FTRACE_FL_FROZEN;
 905 }
 906 #else
 907 # define freeze_record(rec)                     ({ 0; })
 908 # define unfreeze_record(rec)                   ({ 0; })
 909 # define record_frozen(rec)                     ({ 0; })
 910 #endif /* CONFIG_KPROBES */
 911
 912 static void ftrace_free_rec(struct dyn_ftrace *rec)
 913 {
 914         rec->freelist = ftrace_free_records;
 915         ftrace_free_records = rec;
 916         rec->flags |= FTRACE_FL_FREE;
 917 }
 918
 919 static struct dyn_ftrace *ftrace_alloc_dyn_node(unsigned long ip)
 920 {
 921         struct dyn_ftrace *rec;
 922
 923         /* First check for freed records */
 924         if (ftrace_free_records) {
 925                 rec = ftrace_free_records;
 926
 927                 if (unlikely(!(rec->flags & FTRACE_FL_FREE))) {
 928                         FTRACE_WARN_ON_ONCE(1);
 929                         ftrace_free_records = NULL;
 930                         return NULL;
 931                 }
 932
 933                 ftrace_free_records = rec->freelist;
 934                 memset(rec, 0, sizeof(*rec));
 935                 return rec;
 936         }
 937
 938         if (ftrace_pages->index == ENTRIES_PER_PAGE) {
 939                 if (!ftrace_pages->next) {
 940                         /* allocate another page */
 941                         ftrace_pages->next =
 942                                 (void *)get_zeroed_page(GFP_KERNEL);
 943                         if (!ftrace_pages->next)
 944                                 return NULL;
 945                 }
 946                 ftrace_pages = ftrace_pages->next;
 947         }
 948
 949         return &ftrace_pages->records[ftrace_pages->index++];
 950 }
 951
 952 static struct dyn_ftrace *
 953 ftrace_record_ip(unsigned long ip)
 954 {
 955         struct dyn_ftrace *rec;
 956
 957         if (ftrace_disabled)
 958                 return NULL;
 959
 960         rec = ftrace_alloc_dyn_node(ip);
 961         if (!rec)
 962                 return NULL;
 963
 964         rec->ip = ip;
 965         rec->newlist = ftrace_new_addrs;
 966         ftrace_new_addrs = rec;
 967
 968         return rec;
 969 }
 970
 971 static void print_ip_ins(const char *fmt, unsigned char *p)
 972 {
 973         int i;
 974
 975         printk(KERN_CONT "%s", fmt);
 976
 977         for (i = 0; i < MCOUNT_INSN_SIZE; i++)
 978                 printk(KERN_CONT "%s%02x", i ? ":" : "", p[i]);
 979 }
 980
 981 static void ftrace_bug(int failed, unsigned long ip)
 982 {
 983         switch (failed) {
 984         case -EFAULT:
 985                 FTRACE_WARN_ON_ONCE(1);
 986                 pr_info("ftrace faulted on modifying ");
 987                 print_ip_sym(ip);
 988                 break;
 989         case -EINVAL:
 990                 FTRACE_WARN_ON_ONCE(1);
 991                 pr_info("ftrace failed to modify ");
 992                 print_ip_sym(ip);
 993                 print_ip_ins(" actual: ", (unsigned char *)ip);
 994                 printk(KERN_CONT "\n");
 995                 break;
 996         case -EPERM:
 997                 FTRACE_WARN_ON_ONCE(1);
 998                 pr_info("ftrace faulted on writing ");
 999                 print_ip_sym(ip);
1000                 break;
1001         default:
1002                 FTRACE_WARN_ON_ONCE(1);
1003                 pr_info("ftrace faulted on unknown error ");
1004                 print_ip_sym(ip);
1005         }
1006 }
1007
1008
1009 static int
1010 __ftrace_replace_code(struct dyn_ftrace *rec, int enable)
1011 {
1012         unsigned long ftrace_addr;
1013         unsigned long ip, fl;
1014
1015         ftrace_addr = (unsigned long)FTRACE_ADDR;
1016
1017         ip = rec->ip;
1018
1019         /*
1020          * If this record is not to be traced and
1021          * it is not enabled then do nothing.
1022          *
1023          * If this record is not to be traced and
1024          * it is enabled then disable it.
1025          *
1026          */
1027         if (rec->flags & FTRACE_FL_NOTRACE) {
1028                 if (rec->flags & FTRACE_FL_ENABLED)
1029                         rec->flags &= ~FTRACE_FL_ENABLED;
1030                 else
1031                         return 0;
1032
1033         } else if (ftrace_filtered && enable) {
1034                 /*
1035                  * Filtering is on:
1036                  */
1037
1038                 fl = rec->flags & (FTRACE_FL_FILTER | FTRACE_FL_ENABLED);
1039
1040                 /* Record is filtered and enabled, do nothing */
1041                 if (fl == (FTRACE_FL_FILTER | FTRACE_FL_ENABLED))
1042                         return 0;
1043
1044                 /* Record is not filtered or enabled, do nothing */
1045                 if (!fl)
1046                         return 0;
1047
1048                 /* Record is not filtered but enabled, disable it */
1049                 if (fl == FTRACE_FL_ENABLED)
1050                         rec->flags &= ~FTRACE_FL_ENABLED;
1051                 else
1052                 /* Otherwise record is filtered but not enabled, enable it */
1053                         rec->flags |= FTRACE_FL_ENABLED;
1054         } else {
1055                 /* Disable or not filtered */
1056
1057                 if (enable) {
1058                         /* if record is enabled, do nothing */
1059                         if (rec->flags & FTRACE_FL_ENABLED)
1060                                 return 0;
1061
1062                         rec->flags |= FTRACE_FL_ENABLED;
1063
1064                 } else {
1065
1066                         /* if record is not enabled, do nothing */
1067                         if (!(rec->flags & FTRACE_FL_ENABLED))
1068                                 return 0;
1069
1070                         rec->flags &= ~FTRACE_FL_ENABLED;
1071                 }
1072         }
1073
1074         if (rec->flags & FTRACE_FL_ENABLED)
1075                 return ftrace_make_call(rec, ftrace_addr);
1076         else
1077                 return ftrace_make_nop(NULL, rec, ftrace_addr);
1078 }
1079
1080 static void ftrace_replace_code(int enable)
1081 {
1082         struct dyn_ftrace *rec;
1083         struct ftrace_page *pg;
1084         int failed;
1085
1086         do_for_each_ftrace_rec(pg, rec) {
1087                 /*
1088                  * Skip over free records, records that have
1089                  * failed and not converted.
1090                  */
1091                 if (rec->flags & FTRACE_FL_FREE ||
1092                     rec->flags & FTRACE_FL_FAILED ||
1093                     !(rec->flags & FTRACE_FL_CONVERTED))
1094                         continue;
1095
1096                 /* ignore updates to this record's mcount site */
1097                 if (get_kprobe((void *)rec->ip)) {
1098                         freeze_record(rec);
1099                         continue;
1100                 } else {
1101                         unfreeze_record(rec);
1102                 }
1103
1104                 failed = __ftrace_replace_code(rec, enable);
1105                 if (failed) {
1106                         rec->flags |= FTRACE_FL_FAILED;
1107                         if ((system_state == SYSTEM_BOOTING) ||
1108                             !core_kernel_text(rec->ip)) {
1109                                 ftrace_free_rec(rec);
1110                                 } else {
1111                                 ftrace_bug(failed, rec->ip);
1112                                         /* Stop processing */
1113                                         return;
1114                                 }
1115                 }
1116         } while_for_each_ftrace_rec();
1117 }
1118
1119 static int
1120 ftrace_code_disable(struct module *mod, struct dyn_ftrace *rec)
1121 {
1122         unsigned long ip;
1123         int ret;
1124
1125         ip = rec->ip;
1126
1127         ret = ftrace_make_nop(mod, rec, MCOUNT_ADDR);
1128         if (ret) {
1129                 ftrace_bug(ret, ip);
1130                 rec->flags |= FTRACE_FL_FAILED;
1131                 return 0;
1132         }
1133         return 1;
1134 }
1135
1136 /*
1137  * archs can override this function if they must do something
1138  * before the modifying code is performed.
1139  */
1140 int __weak ftrace_arch_code_modify_prepare(void)
1141 {
1142         return 0;
1143 }
1144
1145 /*
1146  * archs can override this function if they must do something
1147  * after the modifying code is performed.
1148  */
1149 int __weak ftrace_arch_code_modify_post_process(void)
1150 {
1151         return 0;
1152 }
1153
1154 static int __ftrace_modify_code(void *data)
1155 {
1156         int *command = data;
1157
1158         if (*command & FTRACE_ENABLE_CALLS)
1159                 ftrace_replace_code(1);
1160         else if (*command & FTRACE_DISABLE_CALLS)
1161                 ftrace_replace_code(0);
1162
1163         if (*command & FTRACE_UPDATE_TRACE_FUNC)
1164                 ftrace_update_ftrace_func(ftrace_trace_function);
1165
1166         if (*command & FTRACE_START_FUNC_RET)
1167                 ftrace_enable_ftrace_graph_caller();
1168         else if (*command & FTRACE_STOP_FUNC_RET)
1169                 ftrace_disable_ftrace_graph_caller();
1170
1171         return 0;
1172 }
1173
1174 static void ftrace_run_update_code(int command)
1175 {
1176         int ret;
1177
1178         ret = ftrace_arch_code_modify_prepare();
1179         FTRACE_WARN_ON(ret);
1180         if (ret)
1181                 return;
1182
1183         stop_machine(__ftrace_modify_code, &command, NULL);
1184
1185         ret = ftrace_arch_code_modify_post_process();
1186         FTRACE_WARN_ON(ret);
1187 }
1188
1189 static ftrace_func_t saved_ftrace_func;
1190 static int ftrace_start_up;
1191
1192 static void ftrace_startup_enable(int command)
1193 {
1194         if (saved_ftrace_func != ftrace_trace_function) {
1195                 saved_ftrace_func = ftrace_trace_function;
1196                 command |= FTRACE_UPDATE_TRACE_FUNC;
1197         }
1198
1199         if (!command || !ftrace_enabled)
1200                 return;
1201
1202         ftrace_run_update_code(command);
1203 }
1204
1205 static void ftrace_startup(int command)
1206 {
1207         if (unlikely(ftrace_disabled))
1208                 return;
1209
1210         ftrace_start_up++;
1211         command |= FTRACE_ENABLE_CALLS;
1212
1213         ftrace_startup_enable(command);
1214 }
1215
1216 static void ftrace_shutdown(int command)
1217 {
1218         if (unlikely(ftrace_disabled))
1219                 return;
1220
1221         ftrace_start_up--;
1222         if (!ftrace_start_up)
1223                 command |= FTRACE_DISABLE_CALLS;
1224
1225         if (saved_ftrace_func != ftrace_trace_function) {
1226                 saved_ftrace_func = ftrace_trace_function;
1227                 command |= FTRACE_UPDATE_TRACE_FUNC;
1228         }
1229
1230         if (!command || !ftrace_enabled)
1231                 return;
1232
1233         ftrace_run_update_code(command);
1234 }
1235
1236 static void ftrace_startup_sysctl(void)
1237 {
1238         int command = FTRACE_ENABLE_MCOUNT;
1239
1240         if (unlikely(ftrace_disabled))
1241                 return;
1242
1243         /* Force update next time */
1244         saved_ftrace_func = NULL;
1245         /* ftrace_start_up is true if we want ftrace running */
1246         if (ftrace_start_up)
1247                 command |= FTRACE_ENABLE_CALLS;
1248
1249         ftrace_run_update_code(command);
1250 }
1251
1252 static void ftrace_shutdown_sysctl(void)
1253 {
1254         int command = FTRACE_DISABLE_MCOUNT;
1255
1256         if (unlikely(ftrace_disabled))
1257                 return;
1258
1259         /* ftrace_start_up is true if ftrace is running */
1260         if (ftrace_start_up)
1261                 command |= FTRACE_DISABLE_CALLS;
1262
1263         ftrace_run_update_code(command);
1264 }
1265
1266 static cycle_t          ftrace_update_time;
1267 static unsigned long    ftrace_update_cnt;
1268 unsigned long           ftrace_update_tot_cnt;
1269
1270 static int ftrace_update_code(struct module *mod)
1271 {
1272         struct dyn_ftrace *p;
1273         cycle_t start, stop;
1274
1275         start = ftrace_now(raw_smp_processor_id());
1276         ftrace_update_cnt = 0;
1277
1278         while (ftrace_new_addrs) {
1279
1280                 /* If something went wrong, bail without enabling anything */
1281                 if (unlikely(ftrace_disabled))
1282                         return -1;
1283
1284                 p = ftrace_new_addrs;
1285                 ftrace_new_addrs = p->newlist;
1286                 p->flags = 0L;
1287
1288                 /* convert record (i.e, patch mcount-call with NOP) */
1289                 if (ftrace_code_disable(mod, p)) {
1290                         p->flags |= FTRACE_FL_CONVERTED;
1291                         ftrace_update_cnt++;
1292                 } else
1293                         ftrace_free_rec(p);
1294         }
1295
1296         stop = ftrace_now(raw_smp_processor_id());
1297         ftrace_update_time = stop - start;
1298         ftrace_update_tot_cnt += ftrace_update_cnt;
1299
1300         return 0;
1301 }
1302
1303 static int __init ftrace_dyn_table_alloc(unsigned long num_to_init)
1304 {
1305         struct ftrace_page *pg;
1306         int cnt;
1307         int i;
1308
1309         /* allocate a few pages */
1310         ftrace_pages_start = (void *)get_zeroed_page(GFP_KERNEL);
1311         if (!ftrace_pages_start)
1312                 return -1;
1313
1314         /*
1315          * Allocate a few more pages.
1316          *
1317          * TODO: have some parser search vmlinux before
1318          *   final linking to find all calls to ftrace.
1319          *   Then we can:
1320          *    a) know how many pages to allocate.
1321          *     and/or
1322          *    b) set up the table then.
1323          *
1324          *  The dynamic code is still necessary for
1325          *  modules.
1326          */
1327
1328         pg = ftrace_pages = ftrace_pages_start;
1329
1330         cnt = num_to_init / ENTRIES_PER_PAGE;
1331         pr_info("ftrace: allocating %ld entries in %d pages\n",
1332                 num_to_init, cnt + 1);
1333
1334         for (i = 0; i < cnt; i++) {
1335                 pg->next = (void *)get_zeroed_page(GFP_KERNEL);
1336
1337                 /* If we fail, we'll try later anyway */
1338                 if (!pg->next)
1339                         break;
1340
1341                 pg = pg->next;
1342         }
1343
1344         return 0;
1345 }
1346
1347 enum {
1348         FTRACE_ITER_FILTER      = (1 << 0),
1349         FTRACE_ITER_CONT        = (1 << 1),
1350         FTRACE_ITER_NOTRACE     = (1 << 2),
1351         FTRACE_ITER_FAILURES    = (1 << 3),
1352         FTRACE_ITER_PRINTALL    = (1 << 4),
1353         FTRACE_ITER_HASH        = (1 << 5),
1354 };
1355
1356 #define FTRACE_BUFF_MAX (KSYM_SYMBOL_LEN+4) /* room for wildcards */
1357
1358 struct ftrace_iterator {
1359         struct ftrace_page      *pg;
1360         int                     hidx;
1361         int                     idx;
1362         unsigned                flags;
1363         unsigned char           buffer[FTRACE_BUFF_MAX+1];
1364         unsigned                buffer_idx;
1365         unsigned                filtered;
1366 };
1367
1368 static void *
1369 t_hash_next(struct seq_file *m, void *v, loff_t *pos)
1370 {
1371         struct ftrace_iterator *iter = m->private;
1372         struct hlist_node *hnd = v;
1373         struct hlist_head *hhd;
1374
1375         WARN_ON(!(iter->flags & FTRACE_ITER_HASH));
1376
1377         (*pos)++;
1378
1379  retry:
1380         if (iter->hidx >= FTRACE_FUNC_HASHSIZE)
1381                 return NULL;
1382
1383         hhd = &ftrace_func_hash[iter->hidx];
1384
1385         if (hlist_empty(hhd)) {
1386                 iter->hidx++;
1387                 hnd = NULL;
1388                 goto retry;
1389         }
1390
1391         if (!hnd)
1392                 hnd = hhd->first;
1393         else {
1394                 hnd = hnd->next;
1395                 if (!hnd) {
1396                         iter->hidx++;
1397                         goto retry;
1398                 }
1399         }
1400
1401         return hnd;
1402 }
1403
1404 static void *t_hash_start(struct seq_file *m, loff_t *pos)
1405 {
1406         struct ftrace_iterator *iter = m->private;
1407         void *p = NULL;
1408
1409         iter->flags |= FTRACE_ITER_HASH;
1410
1411         return t_hash_next(m, p, pos);
1412 }
1413
1414 static int t_hash_show(struct seq_file *m, void *v)
1415 {
1416         struct ftrace_func_probe *rec;
1417         struct hlist_node *hnd = v;
1418         char str[KSYM_SYMBOL_LEN];
1419
1420         rec = hlist_entry(hnd, struct ftrace_func_probe, node);
1421
1422         if (rec->ops->print)
1423                 return rec->ops->print(m, rec->ip, rec->ops, rec->data);
1424
1425         kallsyms_lookup(rec->ip, NULL, NULL, NULL, str);
1426         seq_printf(m, "%s:", str);
1427
1428         kallsyms_lookup((unsigned long)rec->ops->func, NULL, NULL, NULL, str);
1429         seq_printf(m, "%s", str);
1430
1431         if (rec->data)
1432                 seq_printf(m, ":%p", rec->data);
1433         seq_putc(m, '\n');
1434
1435         return 0;
1436 }
1437
1438 static void *
1439 t_next(struct seq_file *m, void *v, loff_t *pos)
1440 {
1441         struct ftrace_iterator *iter = m->private;
1442         struct dyn_ftrace *rec = NULL;
1443
1444         if (iter->flags & FTRACE_ITER_HASH)
1445                 return t_hash_next(m, v, pos);
1446
1447         (*pos)++;
1448
1449         if (iter->flags & FTRACE_ITER_PRINTALL)
1450                 return NULL;
1451
1452  retry:
1453         if (iter->idx >= iter->pg->index) {
1454                 if (iter->pg->next) {
1455                         iter->pg = iter->pg->next;
1456                         iter->idx = 0;
1457                         goto retry;
1458                 } else {
1459                         iter->idx = -1;
1460                 }
1461         } else {
1462                 rec = &iter->pg->records[iter->idx++];
1463                 if ((rec->flags & FTRACE_FL_FREE) ||
1464
1465                     (!(iter->flags & FTRACE_ITER_FAILURES) &&
1466                      (rec->flags & FTRACE_FL_FAILED)) ||
1467
1468                     ((iter->flags & FTRACE_ITER_FAILURES) &&
1469                      !(rec->flags & FTRACE_FL_FAILED)) ||
1470
1471                     ((iter->flags & FTRACE_ITER_FILTER) &&
1472                      !(rec->flags & FTRACE_FL_FILTER)) ||
1473
1474                     ((iter->flags & FTRACE_ITER_NOTRACE) &&
1475                      !(rec->flags & FTRACE_FL_NOTRACE))) {
1476                         rec = NULL;
1477                         goto retry;
1478                 }
1479         }
1480
1481         return rec;
1482 }
1483
1484 static void *t_start(struct seq_file *m, loff_t *pos)
1485 {
1486         struct ftrace_iterator *iter = m->private;
1487         void *p = NULL;
1488
1489         mutex_lock(&ftrace_lock);
1490         /*
1491          * For set_ftrace_filter reading, if we have the filter
1492          * off, we can short cut and just print out that all
1493          * functions are enabled.
1494          */
1495         if (iter->flags & FTRACE_ITER_FILTER && !ftrace_filtered) {
1496                 if (*pos > 0)
1497                         return t_hash_start(m, pos);
1498                 iter->flags |= FTRACE_ITER_PRINTALL;
1499                 (*pos)++;
1500                 return iter;
1501         }
1502
1503         if (iter->flags & FTRACE_ITER_HASH)
1504                 return t_hash_start(m, pos);
1505
1506         if (*pos > 0) {
1507                 if (iter->idx < 0)
1508                         return p;
1509                 (*pos)--;
1510                 iter->idx--;
1511         }
1512
1513         p = t_next(m, p, pos);
1514
1515         if (!p)
1516                 return t_hash_start(m, pos);
1517
1518         return p;
1519 }
1520
1521 static void t_stop(struct seq_file *m, void *p)
1522 {
1523         mutex_unlock(&ftrace_lock);
1524 }
1525
1526 static int t_show(struct seq_file *m, void *v)
1527 {
1528         struct ftrace_iterator *iter = m->private;
1529         struct dyn_ftrace *rec = v;
1530         char str[KSYM_SYMBOL_LEN];
1531
1532         if (iter->flags & FTRACE_ITER_HASH)
1533                 return t_hash_show(m, v);
1534
1535         if (iter->flags & FTRACE_ITER_PRINTALL) {
1536                 seq_printf(m, "#### all functions enabled ####\n");
1537                 return 0;
1538         }
1539
1540         if (!rec)
1541                 return 0;
1542
1543         kallsyms_lookup(rec->ip, NULL, NULL, NULL, str);
1544
1545         seq_printf(m, "%s\n", str);
1546
1547         return 0;
1548 }
1549
1550 static struct seq_operations show_ftrace_seq_ops = {
1551         .start = t_start,
1552         .next = t_next,
1553         .stop = t_stop,
1554         .show = t_show,
1555 };
1556
1557 static int
1558 ftrace_avail_open(struct inode *inode, struct file *file)
1559 {
1560         struct ftrace_iterator *iter;
1561         int ret;
1562
1563         if (unlikely(ftrace_disabled))
1564                 return -ENODEV;
1565
1566         iter = kzalloc(sizeof(*iter), GFP_KERNEL);
1567         if (!iter)
1568                 return -ENOMEM;
1569
1570         iter->pg = ftrace_pages_start;
1571
1572         ret = seq_open(file, &show_ftrace_seq_ops);
1573         if (!ret) {
1574                 struct seq_file *m = file->private_data;
1575
1576                 m->private = iter;
1577         } else {
1578                 kfree(iter);
1579         }
1580
1581         return ret;
1582 }
1583
1584 int ftrace_avail_release(struct inode *inode, struct file *file)
1585 {
1586         struct seq_file *m = (struct seq_file *)file->private_data;
1587         struct ftrace_iterator *iter = m->private;
1588
1589         seq_release(inode, file);
1590         kfree(iter);
1591
1592         return 0;
1593 }
1594
1595 static int
1596 ftrace_failures_open(struct inode *inode, struct file *file)
1597 {
1598         int ret;
1599         struct seq_file *m;
1600         struct ftrace_iterator *iter;
1601
1602         ret = ftrace_avail_open(inode, file);
1603         if (!ret) {
1604                 m = (struct seq_file *)file->private_data;
1605                 iter = (struct ftrace_iterator *)m->private;
1606                 iter->flags = FTRACE_ITER_FAILURES;
1607         }
1608
1609         return ret;
1610 }
1611
1612
1613 static void ftrace_filter_reset(int enable)
1614 {
1615         struct ftrace_page *pg;
1616         struct dyn_ftrace *rec;
1617         unsigned long type = enable ? FTRACE_FL_FILTER : FTRACE_FL_NOTRACE;
1618
1619         mutex_lock(&ftrace_lock);
1620         if (enable)
1621                 ftrace_filtered = 0;
1622         do_for_each_ftrace_rec(pg, rec) {
1623                 if (rec->flags & FTRACE_FL_FAILED)
1624                         continue;
1625                 rec->flags &= ~type;
1626         } while_for_each_ftrace_rec();
1627         mutex_unlock(&ftrace_lock);
1628 }
1629
1630 static int
1631 ftrace_regex_open(struct inode *inode, struct file *file, int enable)
1632 {
1633         struct ftrace_iterator *iter;
1634         int ret = 0;
1635
1636         if (unlikely(ftrace_disabled))
1637                 return -ENODEV;
1638
1639         iter = kzalloc(sizeof(*iter), GFP_KERNEL);
1640         if (!iter)
1641                 return -ENOMEM;
1642
1643         mutex_lock(&ftrace_regex_lock);
1644         if ((file->f_mode & FMODE_WRITE) &&
1645             !(file->f_flags & O_APPEND))
1646                 ftrace_filter_reset(enable);
1647
1648         if (file->f_mode & FMODE_READ) {
1649                 iter->pg = ftrace_pages_start;
1650                 iter->flags = enable ? FTRACE_ITER_FILTER :
1651                         FTRACE_ITER_NOTRACE;
1652
1653                 ret = seq_open(file, &show_ftrace_seq_ops);
1654                 if (!ret) {
1655                         struct seq_file *m = file->private_data;
1656                         m->private = iter;
1657                 } else
1658                         kfree(iter);
1659         } else
1660                 file->private_data = iter;
1661         mutex_unlock(&ftrace_regex_lock);
1662
1663         return ret;
1664 }
1665
1666 static int
1667 ftrace_filter_open(struct inode *inode, struct file *file)
1668 {
1669         return ftrace_regex_open(inode, file, 1);
1670 }
1671
1672 static int
1673 ftrace_notrace_open(struct inode *inode, struct file *file)
1674 {
1675         return ftrace_regex_open(inode, file, 0);
1676 }
1677
1678 static loff_t
1679 ftrace_regex_lseek(struct file *file, loff_t offset, int origin)
1680 {
1681         loff_t ret;
1682
1683         if (file->f_mode & FMODE_READ)
1684                 ret = seq_lseek(file, offset, origin);
1685         else
1686                 file->f_pos = ret = 1;
1687
1688         return ret;
1689 }
1690
1691 enum {
1692         MATCH_FULL,
1693         MATCH_FRONT_ONLY,
1694         MATCH_MIDDLE_ONLY,
1695         MATCH_END_ONLY,
1696 };
1697
1698 /*
1699  * (static function - no need for kernel doc)
1700  *
1701  * Pass in a buffer containing a glob and this function will
1702  * set search to point to the search part of the buffer and
1703  * return the type of search it is (see enum above).
1704  * This does modify buff.
1705  *
1706  * Returns enum type.
1707  *  search returns the pointer to use for comparison.
1708  *  not returns 1 if buff started with a '!'
1709  *     0 otherwise.
1710  */
1711 static int
1712 ftrace_setup_glob(char *buff, int len, char **search, int *not)
1713 {
1714         int type = MATCH_FULL;
1715         int i;
1716
1717         if (buff[0] == '!') {
1718                 *not = 1;
1719                 buff++;
1720                 len--;
1721         } else
1722                 *not = 0;
1723
1724         *search = buff;
1725
1726         for (i = 0; i < len; i++) {
1727                 if (buff[i] == '*') {
1728                         if (!i) {
1729                                 *search = buff + 1;
1730                                 type = MATCH_END_ONLY;
1731                         } else {
1732                                 if (type == MATCH_END_ONLY)
1733                                         type = MATCH_MIDDLE_ONLY;
1734                                 else
1735                                         type = MATCH_FRONT_ONLY;
1736                                 buff[i] = 0;
1737                                 break;
1738                         }
1739                 }
1740         }
1741
1742         return type;
1743 }
1744
1745 static int ftrace_match(char *str, char *regex, int len, int type)
1746 {
1747         int matched = 0;
1748         char *ptr;
1749
1750         switch (type) {
1751         case MATCH_FULL:
1752                 if (strcmp(str, regex) == 0)
1753                         matched = 1;
1754                 break;
1755         case MATCH_FRONT_ONLY:
1756                 if (strncmp(str, regex, len) == 0)
1757                         matched = 1;
1758                 break;
1759         case MATCH_MIDDLE_ONLY:
1760                 if (strstr(str, regex))
1761                         matched = 1;
1762                 break;
1763         case MATCH_END_ONLY:
1764                 ptr = strstr(str, regex);
1765                 if (ptr && (ptr[len] == 0))
1766                         matched = 1;
1767                 break;
1768         }
1769
1770         return matched;
1771 }
1772
1773 static int
1774 ftrace_match_record(struct dyn_ftrace *rec, char *regex, int len, int type)
1775 {
1776         char str[KSYM_SYMBOL_LEN];
1777
1778         kallsyms_lookup(rec->ip, NULL, NULL, NULL, str);
1779         return ftrace_match(str, regex, len, type);
1780 }
1781
1782 static void ftrace_match_records(char *buff, int len, int enable)
1783 {
1784         unsigned int search_len;
1785         struct ftrace_page *pg;
1786         struct dyn_ftrace *rec;
1787         unsigned long flag;
1788         char *search;
1789         int type;
1790         int not;
1791
1792         flag = enable ? FTRACE_FL_FILTER : FTRACE_FL_NOTRACE;
1793         type = ftrace_setup_glob(buff, len, &search, &not);
1794
1795         search_len = strlen(search);
1796
1797         mutex_lock(&ftrace_lock);
1798         do_for_each_ftrace_rec(pg, rec) {
1799
1800                 if (rec->flags & FTRACE_FL_FAILED)
1801                         continue;
1802
1803                 if (ftrace_match_record(rec, search, search_len, type)) {
1804                         if (not)
1805                                 rec->flags &= ~flag;
1806                         else
1807                                 rec->flags |= flag;
1808                 }
1809                 /*
1810                  * Only enable filtering if we have a function that
1811                  * is filtered on.
1812                  */
1813                 if (enable && (rec->flags & FTRACE_FL_FILTER))
1814                         ftrace_filtered = 1;
1815         } while_for_each_ftrace_rec();
1816         mutex_unlock(&ftrace_lock);
1817 }
1818
1819 static int
1820 ftrace_match_module_record(struct dyn_ftrace *rec, char *mod,
1821                            char *regex, int len, int type)
1822 {
1823         char str[KSYM_SYMBOL_LEN];
1824         char *modname;
1825
1826         kallsyms_lookup(rec->ip, NULL, NULL, &modname, str);
1827
1828         if (!modname || strcmp(modname, mod))
1829                 return 0;
1830
1831         /* blank search means to match all funcs in the mod */
1832         if (len)
1833                 return ftrace_match(str, regex, len, type);
1834         else
1835                 return 1;
1836 }
1837
1838 static void ftrace_match_module_records(char *buff, char *mod, int enable)
1839 {
1840         unsigned search_len = 0;
1841         struct ftrace_page *pg;
1842         struct dyn_ftrace *rec;
1843         int type = MATCH_FULL;
1844         char *search = buff;
1845         unsigned long flag;
1846         int not = 0;
1847
1848         flag = enable ? FTRACE_FL_FILTER : FTRACE_FL_NOTRACE;
1849
1850         /* blank or '*' mean the same */
1851         if (strcmp(buff, "*") == 0)
1852                 buff[0] = 0;
1853
1854         /* handle the case of 'dont filter this module' */
1855         if (strcmp(buff, "!") == 0 || strcmp(buff, "!*") == 0) {
1856                 buff[0] = 0;
1857                 not = 1;
1858         }
1859
1860         if (strlen(buff)) {
1861                 type = ftrace_setup_glob(buff, strlen(buff), &search, &not);
1862                 search_len = strlen(search);
1863         }
1864
1865         mutex_lock(&ftrace_lock);
1866         do_for_each_ftrace_rec(pg, rec) {
1867
1868                 if (rec->flags & FTRACE_FL_FAILED)
1869                         continue;
1870
1871                 if (ftrace_match_module_record(rec, mod,
1872                                                search, search_len, type)) {
1873                         if (not)
1874                                 rec->flags &= ~flag;
1875                         else
1876                                 rec->flags |= flag;
1877                 }
1878                 if (enable && (rec->flags & FTRACE_FL_FILTER))
1879                         ftrace_filtered = 1;
1880
1881         } while_for_each_ftrace_rec();
1882         mutex_unlock(&ftrace_lock);
1883 }
1884
1885 /*
1886  * We register the module command as a template to show others how
1887  * to register the a command as well.
1888  */
1889
1890 static int
1891 ftrace_mod_callback(char *func, char *cmd, char *param, int enable)
1892 {
1893         char *mod;
1894
1895         /*
1896          * cmd == 'mod' because we only registered this func
1897          * for the 'mod' ftrace_func_command.
1898          * But if you register one func with multiple commands,
1899          * you can tell which command was used by the cmd
1900          * parameter.
1901          */
1902
1903         /* we must have a module name */
1904         if (!param)
1905                 return -EINVAL;
1906
1907         mod = strsep(&param, ":");
1908         if (!strlen(mod))
1909                 return -EINVAL;
1910
1911         ftrace_match_module_records(func, mod, enable);
1912         return 0;
1913 }
1914
1915 static struct ftrace_func_command ftrace_mod_cmd = {
1916         .name                   = "mod",
1917         .func                   = ftrace_mod_callback,
1918 };
1919
1920 static int __init ftrace_mod_cmd_init(void)
1921 {
1922         return register_ftrace_command(&ftrace_mod_cmd);
1923 }
1924 device_initcall(ftrace_mod_cmd_init);
1925
1926 static void
1927 function_trace_probe_call(unsigned long ip, unsigned long parent_ip)
1928 {
1929         struct ftrace_func_probe *entry;
1930         struct hlist_head *hhd;
1931         struct hlist_node *n;
1932         unsigned long key;
1933         int resched;
1934
1935         key = hash_long(ip, FTRACE_HASH_BITS);
1936
1937         hhd = &ftrace_func_hash[key];
1938
1939         if (hlist_empty(hhd))
1940                 return;
1941
1942         /*
1943          * Disable preemption for these calls to prevent a RCU grace
1944          * period. This syncs the hash iteration and freeing of items
1945          * on the hash. rcu_read_lock is too dangerous here.
1946          */
1947         resched = ftrace_preempt_disable();
1948         hlist_for_each_entry_rcu(entry, n, hhd, node) {
1949                 if (entry->ip == ip)
1950                         entry->ops->func(ip, parent_ip, &entry->data);
1951         }
1952         ftrace_preempt_enable(resched);
1953 }
1954
1955 static struct ftrace_ops trace_probe_ops __read_mostly =
1956 {
1957         .func           = function_trace_probe_call,
1958 };
1959
1960 static int ftrace_probe_registered;
1961
1962 static void __enable_ftrace_function_probe(void)
1963 {
1964         int i;
1965
1966         if (ftrace_probe_registered)
1967                 return;
1968
1969         for (i = 0; i < FTRACE_FUNC_HASHSIZE; i++) {
1970                 struct hlist_head *hhd = &ftrace_func_hash[i];
1971                 if (hhd->first)
1972                         break;
1973         }
1974         /* Nothing registered? */
1975         if (i == FTRACE_FUNC_HASHSIZE)
1976                 return;
1977
1978         __register_ftrace_function(&trace_probe_ops);
1979         ftrace_startup(0);
1980         ftrace_probe_registered = 1;
1981 }
1982
1983 static void __disable_ftrace_function_probe(void)
1984 {
1985         int i;
1986
1987         if (!ftrace_probe_registered)
1988                 return;
1989
1990         for (i = 0; i < FTRACE_FUNC_HASHSIZE; i++) {
1991                 struct hlist_head *hhd = &ftrace_func_hash[i];
1992                 if (hhd->first)
1993                         return;
1994         }
1995
1996         /* no more funcs left */
1997         __unregister_ftrace_function(&trace_probe_ops);
1998         ftrace_shutdown(0);
1999         ftrace_probe_registered = 0;
2000 }
2001
2002
2003 static void ftrace_free_entry_rcu(struct rcu_head *rhp)
2004 {
2005         struct ftrace_func_probe *entry =
2006                 container_of(rhp, struct ftrace_func_probe, rcu);
2007
2008         if (entry->ops->free)
2009                 entry->ops->free(&entry->data);
2010         kfree(entry);
2011 }
2012
2013
2014 int
2015 register_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops,
2016                               void *data)
2017 {
2018         struct ftrace_func_probe *entry;
2019         struct ftrace_page *pg;
2020         struct dyn_ftrace *rec;
2021         int type, len, not;
2022         unsigned long key;
2023         int count = 0;
2024         char *search;
2025
2026         type = ftrace_setup_glob(glob, strlen(glob), &search, &not);
2027         len = strlen(search);
2028
2029         /* we do not support '!' for function probes */
2030         if (WARN_ON(not))
2031                 return -EINVAL;
2032
2033         mutex_lock(&ftrace_lock);
2034         do_for_each_ftrace_rec(pg, rec) {
2035
2036                 if (rec->flags & FTRACE_FL_FAILED)
2037                         continue;
2038
2039                 if (!ftrace_match_record(rec, search, len, type))
2040                         continue;
2041
2042                 entry = kmalloc(sizeof(*entry), GFP_KERNEL);
2043                 if (!entry) {
2044                         /* If we did not process any, then return error */
2045                         if (!count)
2046                                 count = -ENOMEM;
2047                         goto out_unlock;
2048                 }
2049
2050                 count++;
2051
2052                 entry->data = data;
2053
2054                 /*
2055                  * The caller might want to do something special
2056                  * for each function we find. We call the callback
2057                  * to give the caller an opportunity to do so.
2058                  */
2059                 if (ops->callback) {
2060                         if (ops->callback(rec->ip, &entry->data) < 0) {
2061                                 /* caller does not like this func */
2062                                 kfree(entry);
2063                                 continue;
2064                         }
2065                 }
2066
2067                 entry->ops = ops;
2068                 entry->ip = rec->ip;
2069
2070                 key = hash_long(entry->ip, FTRACE_HASH_BITS);
2071                 hlist_add_head_rcu(&entry->node, &ftrace_func_hash[key]);
2072
2073         } while_for_each_ftrace_rec();
2074         __enable_ftrace_function_probe();
2075
2076  out_unlock:
2077         mutex_unlock(&ftrace_lock);
2078
2079         return count;
2080 }
2081
2082 enum {
2083         PROBE_TEST_FUNC         = 1,
2084         PROBE_TEST_DATA         = 2
2085 };
2086
2087 static void
2088 __unregister_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops,
2089                                   void *data, int flags)
2090 {
2091         struct ftrace_func_probe *entry;
2092         struct hlist_node *n, *tmp;
2093         char str[KSYM_SYMBOL_LEN];
2094         int type = MATCH_FULL;
2095         int i, len = 0;
2096         char *search;
2097
2098         if (glob && (strcmp(glob, "*") || !strlen(glob)))
2099                 glob = NULL;
2100         else {
2101                 int not;
2102
2103                 type = ftrace_setup_glob(glob, strlen(glob), &search, &not);
2104                 len = strlen(search);
2105
2106                 /* we do not support '!' for function probes */
2107                 if (WARN_ON(not))
2108                         return;
2109         }
2110
2111         mutex_lock(&ftrace_lock);
2112         for (i = 0; i < FTRACE_FUNC_HASHSIZE; i++) {
2113                 struct hlist_head *hhd = &ftrace_func_hash[i];
2114
2115                 hlist_for_each_entry_safe(entry, n, tmp, hhd, node) {
2116
2117                         /* break up if statements for readability */
2118                         if ((flags & PROBE_TEST_FUNC) && entry->ops != ops)
2119                                 continue;
2120
2121                         if ((flags & PROBE_TEST_DATA) && entry->data != data)
2122                                 continue;
2123
2124                         /* do this last, since it is the most expensive */
2125                         if (glob) {
2126                                 kallsyms_lookup(entry->ip, NULL, NULL,
2127                                                 NULL, str);
2128                                 if (!ftrace_match(str, glob, len, type))
2129                                         continue;
2130                         }
2131
2132                         hlist_del(&entry->node);
2133                         call_rcu(&entry->rcu, ftrace_free_entry_rcu);
2134                 }
2135         }
2136         __disable_ftrace_function_probe();
2137         mutex_unlock(&ftrace_lock);
2138 }
2139
2140 void
2141 unregister_ftrace_function_probe(char *glob, struct ftrace_probe_ops *ops,
2142                                 void *data)
2143 {
2144         __unregister_ftrace_function_probe(glob, ops, data,
2145                                           PROBE_TEST_FUNC | PROBE_TEST_DATA);
2146 }
2147
2148 void
2149 unregister_ftrace_function_probe_func(char *glob, struct ftrace_probe_ops *ops)
2150 {
2151         __unregister_ftrace_function_probe(glob, ops, NULL, PROBE_TEST_FUNC);
2152 }
2153
2154 void unregister_ftrace_function_probe_all(char *glob)
2155 {
2156         __unregister_ftrace_function_probe(glob, NULL, NULL, 0);
2157 }
2158
2159 static LIST_HEAD(ftrace_commands);
2160 static DEFINE_MUTEX(ftrace_cmd_mutex);
2161
2162 int register_ftrace_command(struct ftrace_func_command *cmd)
2163 {
2164         struct ftrace_func_command *p;
2165         int ret = 0;
2166
2167         mutex_lock(&ftrace_cmd_mutex);
2168         list_for_each_entry(p, &ftrace_commands, list) {
2169                 if (strcmp(cmd->name, p->name) == 0) {
2170                         ret = -EBUSY;
2171                         goto out_unlock;
2172                 }
2173         }
2174         list_add(&cmd->list, &ftrace_commands);
2175  out_unlock:
2176         mutex_unlock(&ftrace_cmd_mutex);
2177
2178         return ret;
2179 }
2180
2181 int unregister_ftrace_command(struct ftrace_func_command *cmd)
2182 {
2183         struct ftrace_func_command *p, *n;
2184         int ret = -ENODEV;
2185
2186         mutex_lock(&ftrace_cmd_mutex);
2187         list_for_each_entry_safe(p, n, &ftrace_commands, list) {
2188                 if (strcmp(cmd->name, p->name) == 0) {
2189                         ret = 0;
2190                         list_del_init(&p->list);
2191                         goto out_unlock;
2192                 }
2193         }
2194  out_unlock:
2195         mutex_unlock(&ftrace_cmd_mutex);
2196
2197         return ret;
2198 }
2199
2200 static int ftrace_process_regex(char *buff, int len, int enable)
2201 {
2202         char *func, *command, *next = buff;
2203         struct ftrace_func_command *p;
2204         int ret = -EINVAL;
2205
2206         func = strsep(&next, ":");
2207
2208         if (!next) {
2209                 ftrace_match_records(func, len, enable);
2210                 return 0;
2211         }
2212
2213         /* command found */
2214
2215         command = strsep(&next, ":");
2216
2217         mutex_lock(&ftrace_cmd_mutex);
2218         list_for_each_entry(p, &ftrace_commands, list) {
2219                 if (strcmp(p->name, command) == 0) {
2220                         ret = p->func(func, command, next, enable);
2221                         goto out_unlock;
2222                 }
2223         }
2224  out_unlock:
2225         mutex_unlock(&ftrace_cmd_mutex);
2226
2227         return ret;
2228 }
2229
2230 static ssize_t
2231 ftrace_regex_write(struct file *file, const char __user *ubuf,
2232                    size_t cnt, loff_t *ppos, int enable)
2233 {
2234         struct ftrace_iterator *iter;
2235         char ch;
2236         size_t read = 0;
2237         ssize_t ret;
2238
2239         if (!cnt || cnt < 0)
2240                 return 0;
2241
2242         mutex_lock(&ftrace_regex_lock);
2243
2244         if (file->f_mode & FMODE_READ) {
2245                 struct seq_file *m = file->private_data;
2246                 iter = m->private;
2247         } else
2248                 iter = file->private_data;
2249
2250         if (!*ppos) {
2251                 iter->flags &= ~FTRACE_ITER_CONT;
2252                 iter->buffer_idx = 0;
2253         }
2254
2255         ret = get_user(ch, ubuf++);
2256         if (ret)
2257                 goto out;
2258         read++;
2259         cnt--;
2260
2261         if (!(iter->flags & ~FTRACE_ITER_CONT)) {
2262                 /* skip white space */
2263                 while (cnt && isspace(ch)) {
2264                         ret = get_user(ch, ubuf++);
2265                         if (ret)
2266                                 goto out;
2267                         read++;
2268                         cnt--;
2269                 }
2270
2271                 if (isspace(ch)) {
2272                         file->f_pos += read;
2273                         ret = read;
2274                         goto out;
2275                 }
2276
2277                 iter->buffer_idx = 0;
2278         }
2279
2280         while (cnt && !isspace(ch)) {
2281                 if (iter->buffer_idx < FTRACE_BUFF_MAX)
2282                         iter->buffer[iter->buffer_idx++] = ch;
2283                 else {
2284                         ret = -EINVAL;
2285                         goto out;
2286                 }
2287                 ret = get_user(ch, ubuf++);
2288                 if (ret)
2289                         goto out;
2290                 read++;
2291                 cnt--;
2292         }
2293
2294         if (isspace(ch)) {
2295                 iter->filtered++;
2296                 iter->buffer[iter->buffer_idx] = 0;
2297                 ret = ftrace_process_regex(iter->buffer,
2298                                            iter->buffer_idx, enable);
2299                 if (ret)
2300                         goto out;
2301                 iter->buffer_idx = 0;
2302         } else
2303                 iter->flags |= FTRACE_ITER_CONT;
2304
2305
2306         file->f_pos += read;
2307
2308         ret = read;
2309  out:
2310         mutex_unlock(&ftrace_regex_lock);
2311
2312         return ret;
2313 }
2314
2315 static ssize_t
2316 ftrace_filter_write(struct file *file, const char __user *ubuf,
2317                     size_t cnt, loff_t *ppos)
2318 {
2319         return ftrace_regex_write(file, ubuf, cnt, ppos, 1);
2320 }
2321
2322 static ssize_t
2323 ftrace_notrace_write(struct file *file, const char __user *ubuf,
2324                      size_t cnt, loff_t *ppos)
2325 {
2326         return ftrace_regex_write(file, ubuf, cnt, ppos, 0);
2327 }
2328
2329 static void
2330 ftrace_set_regex(unsigned char *buf, int len, int reset, int enable)
2331 {
2332         if (unlikely(ftrace_disabled))
2333                 return;
2334
2335         mutex_lock(&ftrace_regex_lock);
2336         if (reset)
2337                 ftrace_filter_reset(enable);
2338         if (buf)
2339                 ftrace_match_records(buf, len, enable);
2340         mutex_unlock(&ftrace_regex_lock);
2341 }
2342
2343 /**
2344  * ftrace_set_filter - set a function to filter on in ftrace
2345  * @buf - the string that holds the function filter text.
2346  * @len - the length of the string.
2347  * @reset - non zero to reset all filters before applying this filter.
2348  *
2349  * Filters denote which functions should be enabled when tracing is enabled.
2350  * If @buf is NULL and reset is set, all functions will be enabled for tracing.
2351  */
2352 void ftrace_set_filter(unsigned char *buf, int len, int reset)
2353 {
2354         ftrace_set_regex(buf, len, reset, 1);
2355 }
2356
2357 /**
2358  * ftrace_set_notrace - set a function to not trace in ftrace
2359  * @buf - the string that holds the function notrace text.
2360  * @len - the length of the string.
2361  * @reset - non zero to reset all filters before applying this filter.
2362  *
2363  * Notrace Filters denote which functions should not be enabled when tracing
2364  * is enabled. If @buf is NULL and reset is set, all functions will be enabled
2365  * for tracing.
2366  */
2367 void ftrace_set_notrace(unsigned char *buf, int len, int reset)
2368 {
2369         ftrace_set_regex(buf, len, reset, 0);
2370 }
2371
2372 static int
2373 ftrace_regex_release(struct inode *inode, struct file *file, int enable)
2374 {
2375         struct seq_file *m = (struct seq_file *)file->private_data;
2376         struct ftrace_iterator *iter;
2377
2378         mutex_lock(&ftrace_regex_lock);
2379         if (file->f_mode & FMODE_READ) {
2380                 iter = m->private;
2381
2382                 seq_release(inode, file);
2383         } else
2384                 iter = file->private_data;
2385
2386         if (iter->buffer_idx) {
2387                 iter->filtered++;
2388                 iter->buffer[iter->buffer_idx] = 0;
2389                 ftrace_match_records(iter->buffer, iter->buffer_idx, enable);
2390         }
2391
2392         mutex_lock(&ftrace_lock);
2393         if (ftrace_start_up && ftrace_enabled)
2394                 ftrace_run_update_code(FTRACE_ENABLE_CALLS);
2395         mutex_unlock(&ftrace_lock);
2396
2397         kfree(iter);
2398         mutex_unlock(&ftrace_regex_lock);
2399         return 0;
2400 }
2401
2402 static int
2403 ftrace_filter_release(struct inode *inode, struct file *file)
2404 {
2405         return ftrace_regex_release(inode, file, 1);
2406 }
2407
2408 static int
2409 ftrace_notrace_release(struct inode *inode, struct file *file)
2410 {
2411         return ftrace_regex_release(inode, file, 0);
2412 }
2413
2414 static const struct file_operations ftrace_avail_fops = {
2415         .open = ftrace_avail_open,
2416         .read = seq_read,
2417         .llseek = seq_lseek,
2418         .release = ftrace_avail_release,
2419 };
2420
2421 static const struct file_operations ftrace_failures_fops = {
2422         .open = ftrace_failures_open,
2423         .read = seq_read,
2424         .llseek = seq_lseek,
2425         .release = ftrace_avail_release,
2426 };
2427
2428 static const struct file_operations ftrace_filter_fops = {
2429         .open = ftrace_filter_open,
2430         .read = seq_read,
2431         .write = ftrace_filter_write,
2432         .llseek = ftrace_regex_lseek,
2433         .release = ftrace_filter_release,
2434 };
2435
2436 static const struct file_operations ftrace_notrace_fops = {
2437         .open = ftrace_notrace_open,
2438         .read = seq_read,
2439         .write = ftrace_notrace_write,
2440         .llseek = ftrace_regex_lseek,
2441         .release = ftrace_notrace_release,
2442 };
2443
2444 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
2445
2446 static DEFINE_MUTEX(graph_lock);
2447
2448 int ftrace_graph_count;
2449 unsigned long ftrace_graph_funcs[FTRACE_GRAPH_MAX_FUNCS] __read_mostly;
2450
2451 static void *
2452 g_next(struct seq_file *m, void *v, loff_t *pos)
2453 {
2454         unsigned long *array = m->private;
2455         int index = *pos;
2456
2457         (*pos)++;
2458
2459         if (index >= ftrace_graph_count)
2460                 return NULL;
2461
2462         return &array[index];
2463 }
2464
2465 static void *g_start(struct seq_file *m, loff_t *pos)
2466 {
2467         void *p = NULL;
2468
2469         mutex_lock(&graph_lock);
2470
2471         /* Nothing, tell g_show to print all functions are enabled */
2472         if (!ftrace_graph_count && !*pos)
2473                 return (void *)1;
2474
2475         p = g_next(m, p, pos);
2476
2477         return p;
2478 }
2479
2480 static void g_stop(struct seq_file *m, void *p)
2481 {
2482         mutex_unlock(&graph_lock);
2483 }
2484
2485 static int g_show(struct seq_file *m, void *v)
2486 {
2487         unsigned long *ptr = v;
2488         char str[KSYM_SYMBOL_LEN];
2489
2490         if (!ptr)
2491                 return 0;
2492
2493         if (ptr == (unsigned long *)1) {
2494                 seq_printf(m, "#### all functions enabled ####\n");
2495                 return 0;
2496         }
2497
2498         kallsyms_lookup(*ptr, NULL, NULL, NULL, str);
2499
2500         seq_printf(m, "%s\n", str);
2501
2502         return 0;
2503 }
2504
2505 static struct seq_operations ftrace_graph_seq_ops = {
2506         .start = g_start,
2507         .next = g_next,
2508         .stop = g_stop,
2509         .show = g_show,
2510 };
2511
2512 static int
2513 ftrace_graph_open(struct inode *inode, struct file *file)
2514 {
2515         int ret = 0;
2516
2517         if (unlikely(ftrace_disabled))
2518                 return -ENODEV;
2519
2520         mutex_lock(&graph_lock);
2521         if ((file->f_mode & FMODE_WRITE) &&
2522             !(file->f_flags & O_APPEND)) {
2523                 ftrace_graph_count = 0;
2524                 memset(ftrace_graph_funcs, 0, sizeof(ftrace_graph_funcs));
2525         }
2526
2527         if (file->f_mode & FMODE_READ) {
2528                 ret = seq_open(file, &ftrace_graph_seq_ops);
2529                 if (!ret) {
2530                         struct seq_file *m = file->private_data;
2531                         m->private = ftrace_graph_funcs;
2532                 }
2533         } else
2534                 file->private_data = ftrace_graph_funcs;
2535         mutex_unlock(&graph_lock);
2536
2537         return ret;
2538 }
2539
2540 static int
2541 ftrace_set_func(unsigned long *array, int *idx, char *buffer)
2542 {
2543         struct dyn_ftrace *rec;
2544         struct ftrace_page *pg;
2545         int search_len;
2546         int found = 0;
2547         int type, not;
2548         char *search;
2549         bool exists;
2550         int i;
2551
2552         if (ftrace_disabled)
2553                 return -ENODEV;
2554
2555         /* decode regex */
2556         type = ftrace_setup_glob(buffer, strlen(buffer), &search, &not);
2557         if (not)
2558                 return -EINVAL;
2559
2560         search_len = strlen(search);
2561
2562         mutex_lock(&ftrace_lock);
2563         do_for_each_ftrace_rec(pg, rec) {
2564
2565                 if (*idx >= FTRACE_GRAPH_MAX_FUNCS)
2566                         break;
2567
2568                 if (rec->flags & (FTRACE_FL_FAILED | FTRACE_FL_FREE))
2569                         continue;
2570
2571                 if (ftrace_match_record(rec, search, search_len, type)) {
2572                         /* ensure it is not already in the array */
2573                         exists = false;
2574                         for (i = 0; i < *idx; i++)
2575                                 if (array[i] == rec->ip) {
2576                                         exists = true;
2577                                         break;
2578                                 }
2579                         if (!exists) {
2580                                 array[(*idx)++] = rec->ip;
2581                                 found = 1;
2582                         }
2583                 }
2584         } while_for_each_ftrace_rec();
2585
2586         mutex_unlock(&ftrace_lock);
2587
2588         return found ? 0 : -EINVAL;
2589 }
2590
2591 static ssize_t
2592 ftrace_graph_write(struct file *file, const char __user *ubuf,
2593                    size_t cnt, loff_t *ppos)
2594 {
2595         unsigned char buffer[FTRACE_BUFF_MAX+1];
2596         unsigned long *array;
2597         size_t read = 0;
2598         ssize_t ret;
2599         int index = 0;
2600         char ch;
2601
2602         if (!cnt || cnt < 0)
2603                 return 0;
2604
2605         mutex_lock(&graph_lock);
2606
2607         if (ftrace_graph_count >= FTRACE_GRAPH_MAX_FUNCS) {
2608                 ret = -EBUSY;
2609                 goto out;
2610         }
2611
2612         if (file->f_mode & FMODE_READ) {
2613                 struct seq_file *m = file->private_data;
2614                 array = m->private;
2615         } else
2616                 array = file->private_data;
2617
2618         ret = get_user(ch, ubuf++);
2619         if (ret)
2620                 goto out;
2621         read++;
2622         cnt--;
2623
2624         /* skip white space */
2625         while (cnt && isspace(ch)) {
2626                 ret = get_user(ch, ubuf++);
2627                 if (ret)
2628                         goto out;
2629                 read++;
2630                 cnt--;
2631         }
2632
2633         if (isspace(ch)) {
2634                 *ppos += read;
2635                 ret = read;
2636                 goto out;
2637         }
2638
2639         while (cnt && !isspace(ch)) {
2640                 if (index < FTRACE_BUFF_MAX)
2641                         buffer[index++] = ch;
2642                 else {
2643                         ret = -EINVAL;
2644                         goto out;
2645                 }
2646                 ret = get_user(ch, ubuf++);
2647                 if (ret)
2648                         goto out;
2649                 read++;
2650                 cnt--;
2651         }
2652         buffer[index] = 0;
2653
2654         /* we allow only one expression at a time */
2655         ret = ftrace_set_func(array, &ftrace_graph_count, buffer);
2656         if (ret)
2657                 goto out;
2658
2659         file->f_pos += read;
2660
2661         ret = read;
2662  out:
2663         mutex_unlock(&graph_lock);
2664
2665         return ret;
2666 }
2667
2668 static const struct file_operations ftrace_graph_fops = {
2669         .open = ftrace_graph_open,
2670         .read = seq_read,
2671         .write = ftrace_graph_write,
2672 };
2673 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
2674
2675 static __init int ftrace_init_dyn_debugfs(struct dentry *d_tracer)
2676 {
2677
2678         trace_create_file("available_filter_functions", 0444,
2679                         d_tracer, NULL, &ftrace_avail_fops);
2680
2681         trace_create_file("failures", 0444,
2682                         d_tracer, NULL, &ftrace_failures_fops);
2683
2684         trace_create_file("set_ftrace_filter", 0644, d_tracer,
2685                         NULL, &ftrace_filter_fops);
2686
2687         trace_create_file("set_ftrace_notrace", 0644, d_tracer,
2688                                     NULL, &ftrace_notrace_fops);
2689
2690 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
2691         trace_create_file("set_graph_function", 0444, d_tracer,
2692                                     NULL,
2693                                     &ftrace_graph_fops);
2694 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
2695
2696         return 0;
2697 }
2698
2699 static int ftrace_convert_nops(struct module *mod,
2700                                unsigned long *start,
2701                                unsigned long *end)
2702 {
2703         unsigned long *p;
2704         unsigned long addr;
2705         unsigned long flags;
2706
2707         mutex_lock(&ftrace_lock);
2708         p = start;
2709         while (p < end) {
2710                 addr = ftrace_call_adjust(*p++);
2711                 /*
2712                  * Some architecture linkers will pad between
2713                  * the different mcount_loc sections of different
2714                  * object files to satisfy alignments.
2715                  * Skip any NULL pointers.
2716                  */
2717                 if (!addr)
2718                         continue;
2719                 ftrace_record_ip(addr);
2720         }
2721
2722         /* disable interrupts to prevent kstop machine */
2723         local_irq_save(flags);
2724         ftrace_update_code(mod);
2725         local_irq_restore(flags);
2726         mutex_unlock(&ftrace_lock);
2727
2728         return 0;
2729 }
2730
2731 #ifdef CONFIG_MODULES
2732 void ftrace_release(void *start, void *end)
2733 {
2734         struct dyn_ftrace *rec;
2735         struct ftrace_page *pg;
2736         unsigned long s = (unsigned long)start;
2737         unsigned long e = (unsigned long)end;
2738
2739         if (ftrace_disabled || !start || start == end)
2740                 return;
2741
2742         mutex_lock(&ftrace_lock);
2743         do_for_each_ftrace_rec(pg, rec) {
2744                 if ((rec->ip >= s) && (rec->ip < e)) {
2745                         /*
2746                          * rec->ip is changed in ftrace_free_rec()
2747                          * It should not between s and e if record was freed.
2748                          */
2749                         FTRACE_WARN_ON(rec->flags & FTRACE_FL_FREE);
2750                         ftrace_free_rec(rec);
2751                 }
2752         } while_for_each_ftrace_rec();
2753         mutex_unlock(&ftrace_lock);
2754 }
2755
2756 static void ftrace_init_module(struct module *mod,
2757                                unsigned long *start, unsigned long *end)
2758 {
2759         if (ftrace_disabled || start == end)
2760                 return;
2761         ftrace_convert_nops(mod, start, end);
2762 }
2763
2764 static int ftrace_module_notify(struct notifier_block *self,
2765                                 unsigned long val, void *data)
2766 {
2767         struct module *mod = data;
2768
2769         switch (val) {
2770         case MODULE_STATE_COMING:
2771                 ftrace_init_module(mod, mod->ftrace_callsites,
2772                                    mod->ftrace_callsites +
2773                                    mod->num_ftrace_callsites);
2774                 break;
2775         case MODULE_STATE_GOING:
2776                 ftrace_release(mod->ftrace_callsites,
2777                                mod->ftrace_callsites +
2778                                mod->num_ftrace_callsites);
2779                 break;
2780         }
2781
2782         return 0;
2783 }
2784 #else
2785 static int ftrace_module_notify(struct notifier_block *self,
2786                                 unsigned long val, void *data)
2787 {
2788         return 0;
2789 }
2790 #endif /* CONFIG_MODULES */
2791
2792 struct notifier_block ftrace_module_nb = {
2793         .notifier_call = ftrace_module_notify,
2794         .priority = 0,
2795 };
2796
2797 extern unsigned long __start_mcount_loc[];
2798 extern unsigned long __stop_mcount_loc[];
2799
2800 void __init ftrace_init(void)
2801 {
2802         unsigned long count, addr, flags;
2803         int ret;
2804
2805         /* Keep the ftrace pointer to the stub */
2806         addr = (unsigned long)ftrace_stub;
2807
2808         local_irq_save(flags);
2809         ftrace_dyn_arch_init(&addr);
2810         local_irq_restore(flags);
2811
2812         /* ftrace_dyn_arch_init places the return code in addr */
2813         if (addr)
2814                 goto failed;
2815
2816         count = __stop_mcount_loc - __start_mcount_loc;
2817
2818         ret = ftrace_dyn_table_alloc(count);
2819         if (ret)
2820                 goto failed;
2821
2822         last_ftrace_enabled = ftrace_enabled = 1;
2823
2824         ret = ftrace_convert_nops(NULL,
2825                                   __start_mcount_loc,
2826                                   __stop_mcount_loc);
2827
2828         ret = register_module_notifier(&ftrace_module_nb);
2829         if (ret)
2830                 pr_warning("Failed to register trace ftrace module notifier\n");
2831
2832         return;
2833  failed:
2834         ftrace_disabled = 1;
2835 }
2836
2837 #else
2838
2839 static int __init ftrace_nodyn_init(void)
2840 {
2841         ftrace_enabled = 1;
2842         return 0;
2843 }
2844 device_initcall(ftrace_nodyn_init);
2845
2846 static inline int ftrace_init_dyn_debugfs(struct dentry *d_tracer) { return 0; }
2847 static inline void ftrace_startup_enable(int command) { }
2848 /* Keep as macros so we do not need to define the commands */
2849 # define ftrace_startup(command)        do { } while (0)
2850 # define ftrace_shutdown(command)       do { } while (0)
2851 # define ftrace_startup_sysctl()        do { } while (0)
2852 # define ftrace_shutdown_sysctl()       do { } while (0)
2853 #endif /* CONFIG_DYNAMIC_FTRACE */
2854
2855 static ssize_t
2856 ftrace_pid_read(struct file *file, char __user *ubuf,
2857                        size_t cnt, loff_t *ppos)
2858 {
2859         char buf[64];
2860         int r;
2861
2862         if (ftrace_pid_trace == ftrace_swapper_pid)
2863                 r = sprintf(buf, "swapper tasks\n");
2864         else if (ftrace_pid_trace)
2865                 r = sprintf(buf, "%u\n", pid_vnr(ftrace_pid_trace));
2866         else
2867                 r = sprintf(buf, "no pid\n");
2868
2869         return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
2870 }
2871
2872 static void clear_ftrace_swapper(void)
2873 {
2874         struct task_struct *p;
2875         int cpu;
2876
2877         get_online_cpus();
2878         for_each_online_cpu(cpu) {
2879                 p = idle_task(cpu);
2880                 clear_tsk_trace_trace(p);
2881         }
2882         put_online_cpus();
2883 }
2884
2885 static void set_ftrace_swapper(void)
2886 {
2887         struct task_struct *p;
2888         int cpu;
2889
2890         get_online_cpus();
2891         for_each_online_cpu(cpu) {
2892                 p = idle_task(cpu);
2893                 set_tsk_trace_trace(p);
2894         }
2895         put_online_cpus();
2896 }
2897
2898 static void clear_ftrace_pid(struct pid *pid)
2899 {
2900         struct task_struct *p;
2901
2902         rcu_read_lock();
2903         do_each_pid_task(pid, PIDTYPE_PID, p) {
2904                 clear_tsk_trace_trace(p);
2905         } while_each_pid_task(pid, PIDTYPE_PID, p);
2906         rcu_read_unlock();
2907
2908         put_pid(pid);
2909 }
2910
2911 static void set_ftrace_pid(struct pid *pid)
2912 {
2913         struct task_struct *p;
2914
2915         rcu_read_lock();
2916         do_each_pid_task(pid, PIDTYPE_PID, p) {
2917                 set_tsk_trace_trace(p);
2918         } while_each_pid_task(pid, PIDTYPE_PID, p);
2919         rcu_read_unlock();
2920 }
2921
2922 static void clear_ftrace_pid_task(struct pid **pid)
2923 {
2924         if (*pid == ftrace_swapper_pid)
2925                 clear_ftrace_swapper();
2926         else
2927                 clear_ftrace_pid(*pid);
2928
2929         *pid = NULL;
2930 }
2931
2932 static void set_ftrace_pid_task(struct pid *pid)
2933 {
2934         if (pid == ftrace_swapper_pid)
2935                 set_ftrace_swapper();
2936         else
2937                 set_ftrace_pid(pid);
2938 }
2939
2940 static ssize_t
2941 ftrace_pid_write(struct file *filp, const char __user *ubuf,
2942                    size_t cnt, loff_t *ppos)
2943 {
2944         struct pid *pid;
2945         char buf[64];
2946         long val;
2947         int ret;
2948
2949         if (cnt >= sizeof(buf))
2950                 return -EINVAL;
2951
2952         if (copy_from_user(&buf, ubuf, cnt))
2953                 return -EFAULT;
2954
2955         buf[cnt] = 0;
2956
2957         ret = strict_strtol(buf, 10, &val);
2958         if (ret < 0)
2959                 return ret;
2960
2961         mutex_lock(&ftrace_lock);
2962         if (val < 0) {
2963                 /* disable pid tracing */
2964                 if (!ftrace_pid_trace)
2965                         goto out;
2966
2967                 clear_ftrace_pid_task(&ftrace_pid_trace);
2968
2969         } else {
2970                 /* swapper task is special */
2971                 if (!val) {
2972                         pid = ftrace_swapper_pid;
2973                         if (pid == ftrace_pid_trace)
2974                                 goto out;
2975                 } else {
2976                         pid = find_get_pid(val);
2977
2978                         if (pid == ftrace_pid_trace) {
2979                                 put_pid(pid);
2980                                 goto out;
2981                         }
2982                 }
2983
2984                 if (ftrace_pid_trace)
2985                         clear_ftrace_pid_task(&ftrace_pid_trace);
2986
2987                 if (!pid)
2988                         goto out;
2989
2990                 ftrace_pid_trace = pid;
2991
2992                 set_ftrace_pid_task(ftrace_pid_trace);
2993         }
2994
2995         /* update the function call */
2996         ftrace_update_pid_func();
2997         ftrace_startup_enable(0);
2998
2999  out:
3000         mutex_unlock(&ftrace_lock);
3001
3002         return cnt;
3003 }
3004
3005 static const struct file_operations ftrace_pid_fops = {
3006         .read = ftrace_pid_read,
3007         .write = ftrace_pid_write,
3008 };
3009
3010 static __init int ftrace_init_debugfs(void)
3011 {
3012         struct dentry *d_tracer;
3013
3014         d_tracer = tracing_init_dentry();
3015         if (!d_tracer)
3016                 return 0;
3017
3018         ftrace_init_dyn_debugfs(d_tracer);
3019
3020         trace_create_file("set_ftrace_pid", 0644, d_tracer,
3021                             NULL, &ftrace_pid_fops);
3022
3023         ftrace_profile_debugfs(d_tracer);
3024
3025         return 0;
3026 }
3027 fs_initcall(ftrace_init_debugfs);
3028
3029 /**
3030  * ftrace_kill - kill ftrace
3031  *
3032  * This function should be used by panic code. It stops ftrace
3033  * but in a not so nice way. If you need to simply kill ftrace
3034  * from a non-atomic section, use ftrace_kill.
3035  */
3036 void ftrace_kill(void)
3037 {
3038         ftrace_disabled = 1;
3039         ftrace_enabled = 0;
3040         clear_ftrace_function();
3041 }
3042
3043 /**
3044  * register_ftrace_function - register a function for profiling
3045  * @ops - ops structure that holds the function for profiling.
3046  *
3047  * Register a function to be called by all functions in the
3048  * kernel.
3049  *
3050  * Note: @ops->func and all the functions it calls must be labeled
3051  *       with "notrace", otherwise it will go into a
3052  *       recursive loop.
3053  */
3054 int register_ftrace_function(struct ftrace_ops *ops)
3055 {
3056         int ret;
3057
3058         if (unlikely(ftrace_disabled))
3059                 return -1;
3060
3061         mutex_lock(&ftrace_lock);
3062
3063         ret = __register_ftrace_function(ops);
3064         ftrace_startup(0);
3065
3066         mutex_unlock(&ftrace_lock);
3067         return ret;
3068 }
3069
3070 /**
3071  * unregister_ftrace_function - unregister a function for profiling.
3072  * @ops - ops structure that holds the function to unregister
3073  *
3074  * Unregister a function that was added to be called by ftrace profiling.
3075  */
3076 int unregister_ftrace_function(struct ftrace_ops *ops)
3077 {
3078         int ret;
3079
3080         mutex_lock(&ftrace_lock);
3081         ret = __unregister_ftrace_function(ops);
3082         ftrace_shutdown(0);
3083         mutex_unlock(&ftrace_lock);
3084
3085         return ret;
3086 }
3087
3088 int
3089 ftrace_enable_sysctl(struct ctl_table *table, int write,
3090                      struct file *file, void __user *buffer, size_t *lenp,
3091                      loff_t *ppos)
3092 {
3093         int ret;
3094
3095         if (unlikely(ftrace_disabled))
3096                 return -ENODEV;
3097
3098         mutex_lock(&ftrace_lock);
3099
3100         ret  = proc_dointvec(table, write, file, buffer, lenp, ppos);
3101
3102         if (ret || !write || (last_ftrace_enabled == ftrace_enabled))
3103                 goto out;
3104
3105         last_ftrace_enabled = ftrace_enabled;
3106
3107         if (ftrace_enabled) {
3108
3109                 ftrace_startup_sysctl();
3110
3111                 /* we are starting ftrace again */
3112                 if (ftrace_list != &ftrace_list_end) {
3113                         if (ftrace_list->next == &ftrace_list_end)
3114                                 ftrace_trace_function = ftrace_list->func;
3115                         else
3116                                 ftrace_trace_function = ftrace_list_func;
3117                 }
3118
3119         } else {
3120                 /* stopping ftrace calls (just send to ftrace_stub) */
3121                 ftrace_trace_function = ftrace_stub;
3122
3123                 ftrace_shutdown_sysctl();
3124         }
3125
3126  out:
3127         mutex_unlock(&ftrace_lock);
3128         return ret;
3129 }
3130
3131 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
3132
3133 static int ftrace_graph_active;
3134 static struct notifier_block ftrace_suspend_notifier;
3135
3136 int ftrace_graph_entry_stub(struct ftrace_graph_ent *trace)
3137 {
3138         return 0;
3139 }
3140
3141 /* The callbacks that hook a function */
3142 trace_func_graph_ret_t ftrace_graph_return =
3143                         (trace_func_graph_ret_t)ftrace_stub;
3144 trace_func_graph_ent_t ftrace_graph_entry = ftrace_graph_entry_stub;
3145
3146 /* Try to assign a return stack array on FTRACE_RETSTACK_ALLOC_SIZE tasks. */
3147 static int alloc_retstack_tasklist(struct ftrace_ret_stack **ret_stack_list)
3148 {
3149         int i;
3150         int ret = 0;
3151         unsigned long flags;
3152         int start = 0, end = FTRACE_RETSTACK_ALLOC_SIZE;
3153         struct task_struct *g, *t;
3154
3155         for (i = 0; i < FTRACE_RETSTACK_ALLOC_SIZE; i++) {
3156                 ret_stack_list[i] = kmalloc(FTRACE_RETFUNC_DEPTH
3157                                         * sizeof(struct ftrace_ret_stack),
3158                                         GFP_KERNEL);
3159                 if (!ret_stack_list[i]) {
3160                         start = 0;
3161                         end = i;
3162                         ret = -ENOMEM;
3163                         goto free;
3164                 }
3165         }
3166
3167         read_lock_irqsave(&tasklist_lock, flags);
3168         do_each_thread(g, t) {
3169                 if (start == end) {
3170                         ret = -EAGAIN;
3171                         goto unlock;
3172                 }
3173
3174                 if (t->ret_stack == NULL) {
3175                         t->curr_ret_stack = -1;
3176                         /* Make sure IRQs see the -1 first: */
3177                         barrier();
3178                         t->ret_stack = ret_stack_list[start++];
3179                         atomic_set(&t->tracing_graph_pause, 0);
3180                         atomic_set(&t->trace_overrun, 0);
3181                 }
3182         } while_each_thread(g, t);
3183
3184 unlock:
3185         read_unlock_irqrestore(&tasklist_lock, flags);
3186 free:
3187         for (i = start; i < end; i++)
3188                 kfree(ret_stack_list[i]);
3189         return ret;
3190 }
3191
3192 static void
3193 ftrace_graph_probe_sched_switch(struct rq *__rq, struct task_struct *prev,
3194                                 struct task_struct *next)
3195 {
3196         unsigned long long timestamp;
3197         int index;
3198
3199         /*
3200          * Does the user want to count the time a function was asleep.
3201          * If so, do not update the time stamps.
3202          */
3203         if (trace_flags & TRACE_ITER_SLEEP_TIME)
3204                 return;
3205
3206         timestamp = trace_clock_local();
3207
3208         prev->ftrace_timestamp = timestamp;
3209
3210         /* only process tasks that we timestamped */
3211         if (!next->ftrace_timestamp)
3212                 return;
3213
3214         /*
3215          * Update all the counters in next to make up for the
3216          * time next was sleeping.
3217          */
3218         timestamp -= next->ftrace_timestamp;
3219
3220         for (index = next->curr_ret_stack; index >= 0; index--)
3221                 next->ret_stack[index].calltime += timestamp;
3222 }
3223
3224 /* Allocate a return stack for each task */
3225 static int start_graph_tracing(void)
3226 {
3227         struct ftrace_ret_stack **ret_stack_list;
3228         int ret, cpu;
3229
3230         ret_stack_list = kmalloc(FTRACE_RETSTACK_ALLOC_SIZE *
3231                                 sizeof(struct ftrace_ret_stack *),
3232                                 GFP_KERNEL);
3233
3234         if (!ret_stack_list)
3235                 return -ENOMEM;
3236
3237         /* The cpu_boot init_task->ret_stack will never be freed */
3238         for_each_online_cpu(cpu)
3239                 ftrace_graph_init_task(idle_task(cpu));
3240
3241         do {
3242                 ret = alloc_retstack_tasklist(ret_stack_list);
3243         } while (ret == -EAGAIN);
3244
3245         if (!ret) {
3246                 ret = register_trace_sched_switch(ftrace_graph_probe_sched_switch);
3247                 if (ret)
3248                         pr_info("ftrace_graph: Couldn't activate tracepoint"
3249                                 " probe to kernel_sched_switch\n");
3250         }
3251
3252         kfree(ret_stack_list);
3253         return ret;
3254 }
3255
3256 /*
3257  * Hibernation protection.
3258  * The state of the current task is too much unstable during
3259  * suspend/restore to disk. We want to protect against that.
3260  */
3261 static int
3262 ftrace_suspend_notifier_call(struct notifier_block *bl, unsigned long state,
3263                                                         void *unused)
3264 {
3265         switch (state) {
3266         case PM_HIBERNATION_PREPARE:
3267                 pause_graph_tracing();
3268                 break;
3269
3270         case PM_POST_HIBERNATION:
3271                 unpause_graph_tracing();
3272                 break;
3273         }
3274         return NOTIFY_DONE;
3275 }
3276
3277 int register_ftrace_graph(trace_func_graph_ret_t retfunc,
3278                         trace_func_graph_ent_t entryfunc)
3279 {
3280         int ret = 0;
3281
3282         mutex_lock(&ftrace_lock);
3283
3284         /* we currently allow only one tracer registered at a time */
3285         if (ftrace_graph_active) {
3286                 ret = -EBUSY;
3287                 goto out;
3288         }
3289
3290         ftrace_suspend_notifier.notifier_call = ftrace_suspend_notifier_call;
3291         register_pm_notifier(&ftrace_suspend_notifier);
3292
3293         ftrace_graph_active++;
3294         ret = start_graph_tracing();
3295         if (ret) {
3296                 ftrace_graph_active--;
3297                 goto out;
3298         }
3299
3300         ftrace_graph_return = retfunc;
3301         ftrace_graph_entry = entryfunc;
3302
3303         ftrace_startup(FTRACE_START_FUNC_RET);
3304
3305 out:
3306         mutex_unlock(&ftrace_lock);
3307         return ret;
3308 }
3309
3310 void unregister_ftrace_graph(void)
3311 {
3312         mutex_lock(&ftrace_lock);
3313
3314         if (unlikely(!ftrace_graph_active))
3315                 goto out;
3316
3317         ftrace_graph_active--;
3318         unregister_trace_sched_switch(ftrace_graph_probe_sched_switch);
3319         ftrace_graph_return = (trace_func_graph_ret_t)ftrace_stub;
3320         ftrace_graph_entry = ftrace_graph_entry_stub;
3321         ftrace_shutdown(FTRACE_STOP_FUNC_RET);
3322         unregister_pm_notifier(&ftrace_suspend_notifier);
3323
3324  out:
3325         mutex_unlock(&ftrace_lock);
3326 }
3327
3328 /* Allocate a return stack for newly created task */
3329 void ftrace_graph_init_task(struct task_struct *t)
3330 {
3331         if (ftrace_graph_active) {
3332                 t->ret_stack = kmalloc(FTRACE_RETFUNC_DEPTH
3333                                 * sizeof(struct ftrace_ret_stack),
3334                                 GFP_KERNEL);
3335                 if (!t->ret_stack)
3336                         return;
3337                 t->curr_ret_stack = -1;
3338                 atomic_set(&t->tracing_graph_pause, 0);
3339                 atomic_set(&t->trace_overrun, 0);
3340                 t->ftrace_timestamp = 0;
3341         } else
3342                 t->ret_stack = NULL;
3343 }
3344
3345 void ftrace_graph_exit_task(struct task_struct *t)
3346 {
3347         struct ftrace_ret_stack *ret_stack = t->ret_stack;
3348
3349         t->ret_stack = NULL;
3350         /* NULL must become visible to IRQs before we free it: */
3351         barrier();
3352
3353         kfree(ret_stack);
3354 }
3355
3356 void ftrace_graph_stop(void)
3357 {
3358         ftrace_stop();
3359 }
3360 #endif
3361