2 * Infrastructure for profiling code inserted by 'gcc -pg'.
4 * Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
5 * Copyright (C) 2004-2008 Ingo Molnar <mingo@redhat.com>
7 * Originally ported from the -rt patch by:
8 * Copyright (C) 2007 Arnaldo Carvalho de Melo <acme@redhat.com>
10 * Based on code in the latency_tracer, that is:
12 * Copyright (C) 2004-2006 Ingo Molnar
13 * Copyright (C) 2004 William Lee Irwin III
16 #include <linux/stop_machine.h>
17 #include <linux/clocksource.h>
18 #include <linux/kallsyms.h>
19 #include <linux/seq_file.h>
20 #include <linux/debugfs.h>
21 #include <linux/hardirq.h>
22 #include <linux/kthread.h>
23 #include <linux/uaccess.h>
24 #include <linux/kprobes.h>
25 #include <linux/ftrace.h>
26 #include <linux/sysctl.h>
27 #include <linux/ctype.h>
28 #include <linux/list.h>
30 #include <asm/ftrace.h>
34 #define FTRACE_WARN_ON(cond) \
40 #define FTRACE_WARN_ON_ONCE(cond) \
42 if (WARN_ON_ONCE(cond)) \
46 /* ftrace_enabled is a method to turn ftrace on or off */
47 int ftrace_enabled __read_mostly;
48 static int last_ftrace_enabled;
50 /* set when tracing only a pid */
51 struct pid *ftrace_pid_trace;
52 static struct pid * const ftrace_swapper_pid = &init_struct_pid;
54 /* Quick disabling of function tracer. */
55 int function_trace_stop;
58 * ftrace_disabled is set when an anomaly is discovered.
59 * ftrace_disabled is much stronger than ftrace_enabled.
61 static int ftrace_disabled __read_mostly;
63 static DEFINE_SPINLOCK(ftrace_lock);
64 static DEFINE_MUTEX(ftrace_sysctl_lock);
65 static DEFINE_MUTEX(ftrace_start_lock);
67 static struct ftrace_ops ftrace_list_end __read_mostly =
72 static struct ftrace_ops *ftrace_list __read_mostly = &ftrace_list_end;
73 ftrace_func_t ftrace_trace_function __read_mostly = ftrace_stub;
74 ftrace_func_t __ftrace_trace_function __read_mostly = ftrace_stub;
75 ftrace_func_t ftrace_pid_function __read_mostly = ftrace_stub;
77 static void ftrace_list_func(unsigned long ip, unsigned long parent_ip)
79 struct ftrace_ops *op = ftrace_list;
81 /* in case someone actually ports this to alpha! */
82 read_barrier_depends();
84 while (op != &ftrace_list_end) {
86 read_barrier_depends();
87 op->func(ip, parent_ip);
92 static void ftrace_pid_func(unsigned long ip, unsigned long parent_ip)
94 if (!test_tsk_trace_trace(current))
97 ftrace_pid_function(ip, parent_ip);
100 static void set_ftrace_pid_function(ftrace_func_t func)
102 /* do not set ftrace_pid_function to itself! */
103 if (func != ftrace_pid_func)
104 ftrace_pid_function = func;
108 * clear_ftrace_function - reset the ftrace function
110 * This NULLs the ftrace function and in essence stops
111 * tracing. There may be lag
113 void clear_ftrace_function(void)
115 ftrace_trace_function = ftrace_stub;
116 __ftrace_trace_function = ftrace_stub;
117 ftrace_pid_function = ftrace_stub;
120 #ifndef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST
122 * For those archs that do not test ftrace_trace_stop in their
123 * mcount call site, we need to do it from C.
125 static void ftrace_test_stop_func(unsigned long ip, unsigned long parent_ip)
127 if (function_trace_stop)
130 __ftrace_trace_function(ip, parent_ip);
134 static int __register_ftrace_function(struct ftrace_ops *ops)
136 /* should not be called from interrupt context */
137 spin_lock(&ftrace_lock);
139 ops->next = ftrace_list;
141 * We are entering ops into the ftrace_list but another
142 * CPU might be walking that list. We need to make sure
143 * the ops->next pointer is valid before another CPU sees
144 * the ops pointer included into the ftrace_list.
149 if (ftrace_enabled) {
152 if (ops->next == &ftrace_list_end)
155 func = ftrace_list_func;
157 if (ftrace_pid_trace) {
158 set_ftrace_pid_function(func);
159 func = ftrace_pid_func;
163 * For one func, simply call it directly.
164 * For more than one func, call the chain.
166 #ifdef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST
167 ftrace_trace_function = func;
169 __ftrace_trace_function = func;
170 ftrace_trace_function = ftrace_test_stop_func;
174 spin_unlock(&ftrace_lock);
179 static int __unregister_ftrace_function(struct ftrace_ops *ops)
181 struct ftrace_ops **p;
184 /* should not be called from interrupt context */
185 spin_lock(&ftrace_lock);
188 * If we are removing the last function, then simply point
189 * to the ftrace_stub.
191 if (ftrace_list == ops && ops->next == &ftrace_list_end) {
192 ftrace_trace_function = ftrace_stub;
193 ftrace_list = &ftrace_list_end;
197 for (p = &ftrace_list; *p != &ftrace_list_end; p = &(*p)->next)
208 if (ftrace_enabled) {
209 /* If we only have one func left, then call that directly */
210 if (ftrace_list->next == &ftrace_list_end) {
211 ftrace_func_t func = ftrace_list->func;
213 if (ftrace_pid_trace) {
214 set_ftrace_pid_function(func);
215 func = ftrace_pid_func;
217 #ifdef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST
218 ftrace_trace_function = func;
220 __ftrace_trace_function = func;
226 spin_unlock(&ftrace_lock);
231 static void ftrace_update_pid_func(void)
235 /* should not be called from interrupt context */
236 spin_lock(&ftrace_lock);
238 if (ftrace_trace_function == ftrace_stub)
241 func = ftrace_trace_function;
243 if (ftrace_pid_trace) {
244 set_ftrace_pid_function(func);
245 func = ftrace_pid_func;
247 if (func == ftrace_pid_func)
248 func = ftrace_pid_function;
251 #ifdef CONFIG_HAVE_FUNCTION_TRACE_MCOUNT_TEST
252 ftrace_trace_function = func;
254 __ftrace_trace_function = func;
258 spin_unlock(&ftrace_lock);
261 #ifdef CONFIG_DYNAMIC_FTRACE
262 #ifndef CONFIG_FTRACE_MCOUNT_RECORD
263 # error Dynamic ftrace depends on MCOUNT_RECORD
267 * Since MCOUNT_ADDR may point to mcount itself, we do not want
268 * to get it confused by reading a reference in the code as we
269 * are parsing on objcopy output of text. Use a variable for
272 static unsigned long mcount_addr = MCOUNT_ADDR;
275 FTRACE_ENABLE_CALLS = (1 << 0),
276 FTRACE_DISABLE_CALLS = (1 << 1),
277 FTRACE_UPDATE_TRACE_FUNC = (1 << 2),
278 FTRACE_ENABLE_MCOUNT = (1 << 3),
279 FTRACE_DISABLE_MCOUNT = (1 << 4),
280 FTRACE_START_FUNC_RET = (1 << 5),
281 FTRACE_STOP_FUNC_RET = (1 << 6),
284 static int ftrace_filtered;
286 static LIST_HEAD(ftrace_new_addrs);
288 static DEFINE_MUTEX(ftrace_regex_lock);
291 struct ftrace_page *next;
293 struct dyn_ftrace records[];
296 #define ENTRIES_PER_PAGE \
297 ((PAGE_SIZE - sizeof(struct ftrace_page)) / sizeof(struct dyn_ftrace))
299 /* estimate from running different kernels */
300 #define NR_TO_INIT 10000
302 static struct ftrace_page *ftrace_pages_start;
303 static struct ftrace_page *ftrace_pages;
305 static struct dyn_ftrace *ftrace_free_records;
308 #ifdef CONFIG_KPROBES
310 static int frozen_record_count;
312 static inline void freeze_record(struct dyn_ftrace *rec)
314 if (!(rec->flags & FTRACE_FL_FROZEN)) {
315 rec->flags |= FTRACE_FL_FROZEN;
316 frozen_record_count++;
320 static inline void unfreeze_record(struct dyn_ftrace *rec)
322 if (rec->flags & FTRACE_FL_FROZEN) {
323 rec->flags &= ~FTRACE_FL_FROZEN;
324 frozen_record_count--;
328 static inline int record_frozen(struct dyn_ftrace *rec)
330 return rec->flags & FTRACE_FL_FROZEN;
333 # define freeze_record(rec) ({ 0; })
334 # define unfreeze_record(rec) ({ 0; })
335 # define record_frozen(rec) ({ 0; })
336 #endif /* CONFIG_KPROBES */
338 static void ftrace_free_rec(struct dyn_ftrace *rec)
340 rec->ip = (unsigned long)ftrace_free_records;
341 ftrace_free_records = rec;
342 rec->flags |= FTRACE_FL_FREE;
345 void ftrace_release(void *start, unsigned long size)
347 struct dyn_ftrace *rec;
348 struct ftrace_page *pg;
349 unsigned long s = (unsigned long)start;
350 unsigned long e = s + size;
353 if (ftrace_disabled || !start)
356 /* should not be called from interrupt context */
357 spin_lock(&ftrace_lock);
359 for (pg = ftrace_pages_start; pg; pg = pg->next) {
360 for (i = 0; i < pg->index; i++) {
361 rec = &pg->records[i];
363 if ((rec->ip >= s) && (rec->ip < e))
364 ftrace_free_rec(rec);
367 spin_unlock(&ftrace_lock);
370 static struct dyn_ftrace *ftrace_alloc_dyn_node(unsigned long ip)
372 struct dyn_ftrace *rec;
374 /* First check for freed records */
375 if (ftrace_free_records) {
376 rec = ftrace_free_records;
378 if (unlikely(!(rec->flags & FTRACE_FL_FREE))) {
379 FTRACE_WARN_ON_ONCE(1);
380 ftrace_free_records = NULL;
384 ftrace_free_records = (void *)rec->ip;
385 memset(rec, 0, sizeof(*rec));
389 if (ftrace_pages->index == ENTRIES_PER_PAGE) {
390 if (!ftrace_pages->next) {
391 /* allocate another page */
393 (void *)get_zeroed_page(GFP_KERNEL);
394 if (!ftrace_pages->next)
397 ftrace_pages = ftrace_pages->next;
400 return &ftrace_pages->records[ftrace_pages->index++];
403 static struct dyn_ftrace *
404 ftrace_record_ip(unsigned long ip)
406 struct dyn_ftrace *rec;
411 rec = ftrace_alloc_dyn_node(ip);
417 list_add(&rec->list, &ftrace_new_addrs);
422 static void print_ip_ins(const char *fmt, unsigned char *p)
426 printk(KERN_CONT "%s", fmt);
428 for (i = 0; i < MCOUNT_INSN_SIZE; i++)
429 printk(KERN_CONT "%s%02x", i ? ":" : "", p[i]);
432 static void ftrace_bug(int failed, unsigned long ip)
436 FTRACE_WARN_ON_ONCE(1);
437 pr_info("ftrace faulted on modifying ");
441 FTRACE_WARN_ON_ONCE(1);
442 pr_info("ftrace failed to modify ");
444 print_ip_ins(" actual: ", (unsigned char *)ip);
445 printk(KERN_CONT "\n");
448 FTRACE_WARN_ON_ONCE(1);
449 pr_info("ftrace faulted on writing ");
453 FTRACE_WARN_ON_ONCE(1);
454 pr_info("ftrace faulted on unknown error ");
461 __ftrace_replace_code(struct dyn_ftrace *rec, int enable)
463 unsigned long ip, fl;
464 unsigned long ftrace_addr;
466 ftrace_addr = (unsigned long)ftrace_caller;
471 * If this record is not to be traced and
472 * it is not enabled then do nothing.
474 * If this record is not to be traced and
475 * it is enabled then disabled it.
478 if (rec->flags & FTRACE_FL_NOTRACE) {
479 if (rec->flags & FTRACE_FL_ENABLED)
480 rec->flags &= ~FTRACE_FL_ENABLED;
484 } else if (ftrace_filtered && enable) {
489 fl = rec->flags & (FTRACE_FL_FILTER | FTRACE_FL_ENABLED);
491 /* Record is filtered and enabled, do nothing */
492 if (fl == (FTRACE_FL_FILTER | FTRACE_FL_ENABLED))
495 /* Record is not filtered and is not enabled do nothing */
499 /* Record is not filtered but enabled, disable it */
500 if (fl == FTRACE_FL_ENABLED)
501 rec->flags &= ~FTRACE_FL_ENABLED;
503 /* Otherwise record is filtered but not enabled, enable it */
504 rec->flags |= FTRACE_FL_ENABLED;
506 /* Disable or not filtered */
509 /* if record is enabled, do nothing */
510 if (rec->flags & FTRACE_FL_ENABLED)
513 rec->flags |= FTRACE_FL_ENABLED;
517 /* if record is not enabled do nothing */
518 if (!(rec->flags & FTRACE_FL_ENABLED))
521 rec->flags &= ~FTRACE_FL_ENABLED;
525 if (rec->flags & FTRACE_FL_ENABLED)
526 return ftrace_make_call(rec, ftrace_addr);
528 return ftrace_make_nop(NULL, rec, ftrace_addr);
531 static void ftrace_replace_code(int enable)
534 struct dyn_ftrace *rec;
535 struct ftrace_page *pg;
537 for (pg = ftrace_pages_start; pg; pg = pg->next) {
538 for (i = 0; i < pg->index; i++) {
539 rec = &pg->records[i];
542 * Skip over free records and records that have
545 if (rec->flags & FTRACE_FL_FREE ||
546 rec->flags & FTRACE_FL_FAILED)
549 /* ignore updates to this record's mcount site */
550 if (get_kprobe((void *)rec->ip)) {
554 unfreeze_record(rec);
557 failed = __ftrace_replace_code(rec, enable);
558 if (failed && (rec->flags & FTRACE_FL_CONVERTED)) {
559 rec->flags |= FTRACE_FL_FAILED;
560 if ((system_state == SYSTEM_BOOTING) ||
561 !core_kernel_text(rec->ip)) {
562 ftrace_free_rec(rec);
564 ftrace_bug(failed, rec->ip);
571 ftrace_code_disable(struct module *mod, struct dyn_ftrace *rec)
578 ret = ftrace_make_nop(mod, rec, mcount_addr);
581 rec->flags |= FTRACE_FL_FAILED;
587 static int __ftrace_modify_code(void *data)
591 if (*command & FTRACE_ENABLE_CALLS)
592 ftrace_replace_code(1);
593 else if (*command & FTRACE_DISABLE_CALLS)
594 ftrace_replace_code(0);
596 if (*command & FTRACE_UPDATE_TRACE_FUNC)
597 ftrace_update_ftrace_func(ftrace_trace_function);
599 if (*command & FTRACE_START_FUNC_RET)
600 ftrace_enable_ftrace_graph_caller();
601 else if (*command & FTRACE_STOP_FUNC_RET)
602 ftrace_disable_ftrace_graph_caller();
607 static void ftrace_run_update_code(int command)
609 stop_machine(__ftrace_modify_code, &command, NULL);
612 static ftrace_func_t saved_ftrace_func;
613 static int ftrace_start_up;
615 static void ftrace_startup_enable(int command)
617 if (saved_ftrace_func != ftrace_trace_function) {
618 saved_ftrace_func = ftrace_trace_function;
619 command |= FTRACE_UPDATE_TRACE_FUNC;
622 if (!command || !ftrace_enabled)
625 ftrace_run_update_code(command);
628 static void ftrace_startup(int command)
630 if (unlikely(ftrace_disabled))
633 mutex_lock(&ftrace_start_lock);
635 command |= FTRACE_ENABLE_CALLS;
637 ftrace_startup_enable(command);
639 mutex_unlock(&ftrace_start_lock);
642 static void ftrace_shutdown(int command)
644 if (unlikely(ftrace_disabled))
647 mutex_lock(&ftrace_start_lock);
649 if (!ftrace_start_up)
650 command |= FTRACE_DISABLE_CALLS;
652 if (saved_ftrace_func != ftrace_trace_function) {
653 saved_ftrace_func = ftrace_trace_function;
654 command |= FTRACE_UPDATE_TRACE_FUNC;
657 if (!command || !ftrace_enabled)
660 ftrace_run_update_code(command);
662 mutex_unlock(&ftrace_start_lock);
665 static void ftrace_startup_sysctl(void)
667 int command = FTRACE_ENABLE_MCOUNT;
669 if (unlikely(ftrace_disabled))
672 mutex_lock(&ftrace_start_lock);
673 /* Force update next time */
674 saved_ftrace_func = NULL;
675 /* ftrace_start_up is true if we want ftrace running */
677 command |= FTRACE_ENABLE_CALLS;
679 ftrace_run_update_code(command);
680 mutex_unlock(&ftrace_start_lock);
683 static void ftrace_shutdown_sysctl(void)
685 int command = FTRACE_DISABLE_MCOUNT;
687 if (unlikely(ftrace_disabled))
690 mutex_lock(&ftrace_start_lock);
691 /* ftrace_start_up is true if ftrace is running */
693 command |= FTRACE_DISABLE_CALLS;
695 ftrace_run_update_code(command);
696 mutex_unlock(&ftrace_start_lock);
699 static cycle_t ftrace_update_time;
700 static unsigned long ftrace_update_cnt;
701 unsigned long ftrace_update_tot_cnt;
703 static int ftrace_update_code(struct module *mod)
705 struct dyn_ftrace *p, *t;
708 start = ftrace_now(raw_smp_processor_id());
709 ftrace_update_cnt = 0;
711 list_for_each_entry_safe(p, t, &ftrace_new_addrs, list) {
713 /* If something went wrong, bail without enabling anything */
714 if (unlikely(ftrace_disabled))
717 list_del_init(&p->list);
719 /* convert record (i.e, patch mcount-call with NOP) */
720 if (ftrace_code_disable(mod, p)) {
721 p->flags |= FTRACE_FL_CONVERTED;
727 stop = ftrace_now(raw_smp_processor_id());
728 ftrace_update_time = stop - start;
729 ftrace_update_tot_cnt += ftrace_update_cnt;
734 static int __init ftrace_dyn_table_alloc(unsigned long num_to_init)
736 struct ftrace_page *pg;
740 /* allocate a few pages */
741 ftrace_pages_start = (void *)get_zeroed_page(GFP_KERNEL);
742 if (!ftrace_pages_start)
746 * Allocate a few more pages.
748 * TODO: have some parser search vmlinux before
749 * final linking to find all calls to ftrace.
751 * a) know how many pages to allocate.
753 * b) set up the table then.
755 * The dynamic code is still necessary for
759 pg = ftrace_pages = ftrace_pages_start;
761 cnt = num_to_init / ENTRIES_PER_PAGE;
762 pr_info("ftrace: allocating %ld entries in %d pages\n",
763 num_to_init, cnt + 1);
765 for (i = 0; i < cnt; i++) {
766 pg->next = (void *)get_zeroed_page(GFP_KERNEL);
768 /* If we fail, we'll try later anyway */
779 FTRACE_ITER_FILTER = (1 << 0),
780 FTRACE_ITER_CONT = (1 << 1),
781 FTRACE_ITER_NOTRACE = (1 << 2),
782 FTRACE_ITER_FAILURES = (1 << 3),
785 #define FTRACE_BUFF_MAX (KSYM_SYMBOL_LEN+4) /* room for wildcards */
787 struct ftrace_iterator {
788 struct ftrace_page *pg;
791 unsigned char buffer[FTRACE_BUFF_MAX+1];
797 t_next(struct seq_file *m, void *v, loff_t *pos)
799 struct ftrace_iterator *iter = m->private;
800 struct dyn_ftrace *rec = NULL;
804 /* should not be called from interrupt context */
805 spin_lock(&ftrace_lock);
807 if (iter->idx >= iter->pg->index) {
808 if (iter->pg->next) {
809 iter->pg = iter->pg->next;
816 rec = &iter->pg->records[iter->idx++];
817 if ((rec->flags & FTRACE_FL_FREE) ||
819 (!(iter->flags & FTRACE_ITER_FAILURES) &&
820 (rec->flags & FTRACE_FL_FAILED)) ||
822 ((iter->flags & FTRACE_ITER_FAILURES) &&
823 !(rec->flags & FTRACE_FL_FAILED)) ||
825 ((iter->flags & FTRACE_ITER_FILTER) &&
826 !(rec->flags & FTRACE_FL_FILTER)) ||
828 ((iter->flags & FTRACE_ITER_NOTRACE) &&
829 !(rec->flags & FTRACE_FL_NOTRACE))) {
834 spin_unlock(&ftrace_lock);
839 static void *t_start(struct seq_file *m, loff_t *pos)
841 struct ftrace_iterator *iter = m->private;
851 p = t_next(m, p, pos);
856 static void t_stop(struct seq_file *m, void *p)
860 static int t_show(struct seq_file *m, void *v)
862 struct dyn_ftrace *rec = v;
863 char str[KSYM_SYMBOL_LEN];
868 kallsyms_lookup(rec->ip, NULL, NULL, NULL, str);
870 seq_printf(m, "%s\n", str);
875 static struct seq_operations show_ftrace_seq_ops = {
883 ftrace_avail_open(struct inode *inode, struct file *file)
885 struct ftrace_iterator *iter;
888 if (unlikely(ftrace_disabled))
891 iter = kzalloc(sizeof(*iter), GFP_KERNEL);
895 iter->pg = ftrace_pages_start;
897 ret = seq_open(file, &show_ftrace_seq_ops);
899 struct seq_file *m = file->private_data;
909 int ftrace_avail_release(struct inode *inode, struct file *file)
911 struct seq_file *m = (struct seq_file *)file->private_data;
912 struct ftrace_iterator *iter = m->private;
914 seq_release(inode, file);
921 ftrace_failures_open(struct inode *inode, struct file *file)
925 struct ftrace_iterator *iter;
927 ret = ftrace_avail_open(inode, file);
929 m = (struct seq_file *)file->private_data;
930 iter = (struct ftrace_iterator *)m->private;
931 iter->flags = FTRACE_ITER_FAILURES;
938 static void ftrace_filter_reset(int enable)
940 struct ftrace_page *pg;
941 struct dyn_ftrace *rec;
942 unsigned long type = enable ? FTRACE_FL_FILTER : FTRACE_FL_NOTRACE;
945 /* should not be called from interrupt context */
946 spin_lock(&ftrace_lock);
949 pg = ftrace_pages_start;
951 for (i = 0; i < pg->index; i++) {
952 rec = &pg->records[i];
953 if (rec->flags & FTRACE_FL_FAILED)
959 spin_unlock(&ftrace_lock);
963 ftrace_regex_open(struct inode *inode, struct file *file, int enable)
965 struct ftrace_iterator *iter;
968 if (unlikely(ftrace_disabled))
971 iter = kzalloc(sizeof(*iter), GFP_KERNEL);
975 mutex_lock(&ftrace_regex_lock);
976 if ((file->f_mode & FMODE_WRITE) &&
977 !(file->f_flags & O_APPEND))
978 ftrace_filter_reset(enable);
980 if (file->f_mode & FMODE_READ) {
981 iter->pg = ftrace_pages_start;
982 iter->flags = enable ? FTRACE_ITER_FILTER :
985 ret = seq_open(file, &show_ftrace_seq_ops);
987 struct seq_file *m = file->private_data;
992 file->private_data = iter;
993 mutex_unlock(&ftrace_regex_lock);
999 ftrace_filter_open(struct inode *inode, struct file *file)
1001 return ftrace_regex_open(inode, file, 1);
1005 ftrace_notrace_open(struct inode *inode, struct file *file)
1007 return ftrace_regex_open(inode, file, 0);
1011 ftrace_regex_read(struct file *file, char __user *ubuf,
1012 size_t cnt, loff_t *ppos)
1014 if (file->f_mode & FMODE_READ)
1015 return seq_read(file, ubuf, cnt, ppos);
1021 ftrace_regex_lseek(struct file *file, loff_t offset, int origin)
1025 if (file->f_mode & FMODE_READ)
1026 ret = seq_lseek(file, offset, origin);
1028 file->f_pos = ret = 1;
1041 ftrace_match(unsigned char *buff, int len, int enable)
1043 char str[KSYM_SYMBOL_LEN];
1044 char *search = NULL;
1045 struct ftrace_page *pg;
1046 struct dyn_ftrace *rec;
1047 int type = MATCH_FULL;
1048 unsigned long flag = enable ? FTRACE_FL_FILTER : FTRACE_FL_NOTRACE;
1049 unsigned i, match = 0, search_len = 0;
1051 for (i = 0; i < len; i++) {
1052 if (buff[i] == '*') {
1054 search = buff + i + 1;
1055 type = MATCH_END_ONLY;
1056 search_len = len - (i + 1);
1058 if (type == MATCH_END_ONLY) {
1059 type = MATCH_MIDDLE_ONLY;
1062 type = MATCH_FRONT_ONLY;
1070 /* should not be called from interrupt context */
1071 spin_lock(&ftrace_lock);
1073 ftrace_filtered = 1;
1074 pg = ftrace_pages_start;
1076 for (i = 0; i < pg->index; i++) {
1080 rec = &pg->records[i];
1081 if (rec->flags & FTRACE_FL_FAILED)
1083 kallsyms_lookup(rec->ip, NULL, NULL, NULL, str);
1086 if (strcmp(str, buff) == 0)
1089 case MATCH_FRONT_ONLY:
1090 if (memcmp(str, buff, match) == 0)
1093 case MATCH_MIDDLE_ONLY:
1094 if (strstr(str, search))
1097 case MATCH_END_ONLY:
1098 ptr = strstr(str, search);
1099 if (ptr && (ptr[search_len] == 0))
1108 spin_unlock(&ftrace_lock);
1112 ftrace_regex_write(struct file *file, const char __user *ubuf,
1113 size_t cnt, loff_t *ppos, int enable)
1115 struct ftrace_iterator *iter;
1120 if (!cnt || cnt < 0)
1123 mutex_lock(&ftrace_regex_lock);
1125 if (file->f_mode & FMODE_READ) {
1126 struct seq_file *m = file->private_data;
1129 iter = file->private_data;
1132 iter->flags &= ~FTRACE_ITER_CONT;
1133 iter->buffer_idx = 0;
1136 ret = get_user(ch, ubuf++);
1142 if (!(iter->flags & ~FTRACE_ITER_CONT)) {
1143 /* skip white space */
1144 while (cnt && isspace(ch)) {
1145 ret = get_user(ch, ubuf++);
1153 file->f_pos += read;
1158 iter->buffer_idx = 0;
1161 while (cnt && !isspace(ch)) {
1162 if (iter->buffer_idx < FTRACE_BUFF_MAX)
1163 iter->buffer[iter->buffer_idx++] = ch;
1168 ret = get_user(ch, ubuf++);
1177 iter->buffer[iter->buffer_idx] = 0;
1178 ftrace_match(iter->buffer, iter->buffer_idx, enable);
1179 iter->buffer_idx = 0;
1181 iter->flags |= FTRACE_ITER_CONT;
1184 file->f_pos += read;
1188 mutex_unlock(&ftrace_regex_lock);
1194 ftrace_filter_write(struct file *file, const char __user *ubuf,
1195 size_t cnt, loff_t *ppos)
1197 return ftrace_regex_write(file, ubuf, cnt, ppos, 1);
1201 ftrace_notrace_write(struct file *file, const char __user *ubuf,
1202 size_t cnt, loff_t *ppos)
1204 return ftrace_regex_write(file, ubuf, cnt, ppos, 0);
1208 ftrace_set_regex(unsigned char *buf, int len, int reset, int enable)
1210 if (unlikely(ftrace_disabled))
1213 mutex_lock(&ftrace_regex_lock);
1215 ftrace_filter_reset(enable);
1217 ftrace_match(buf, len, enable);
1218 mutex_unlock(&ftrace_regex_lock);
1222 * ftrace_set_filter - set a function to filter on in ftrace
1223 * @buf - the string that holds the function filter text.
1224 * @len - the length of the string.
1225 * @reset - non zero to reset all filters before applying this filter.
1227 * Filters denote which functions should be enabled when tracing is enabled.
1228 * If @buf is NULL and reset is set, all functions will be enabled for tracing.
1230 void ftrace_set_filter(unsigned char *buf, int len, int reset)
1232 ftrace_set_regex(buf, len, reset, 1);
1236 * ftrace_set_notrace - set a function to not trace in ftrace
1237 * @buf - the string that holds the function notrace text.
1238 * @len - the length of the string.
1239 * @reset - non zero to reset all filters before applying this filter.
1241 * Notrace Filters denote which functions should not be enabled when tracing
1242 * is enabled. If @buf is NULL and reset is set, all functions will be enabled
1245 void ftrace_set_notrace(unsigned char *buf, int len, int reset)
1247 ftrace_set_regex(buf, len, reset, 0);
1251 ftrace_regex_release(struct inode *inode, struct file *file, int enable)
1253 struct seq_file *m = (struct seq_file *)file->private_data;
1254 struct ftrace_iterator *iter;
1256 mutex_lock(&ftrace_regex_lock);
1257 if (file->f_mode & FMODE_READ) {
1260 seq_release(inode, file);
1262 iter = file->private_data;
1264 if (iter->buffer_idx) {
1266 iter->buffer[iter->buffer_idx] = 0;
1267 ftrace_match(iter->buffer, iter->buffer_idx, enable);
1270 mutex_lock(&ftrace_sysctl_lock);
1271 mutex_lock(&ftrace_start_lock);
1272 if (ftrace_start_up && ftrace_enabled)
1273 ftrace_run_update_code(FTRACE_ENABLE_CALLS);
1274 mutex_unlock(&ftrace_start_lock);
1275 mutex_unlock(&ftrace_sysctl_lock);
1278 mutex_unlock(&ftrace_regex_lock);
1283 ftrace_filter_release(struct inode *inode, struct file *file)
1285 return ftrace_regex_release(inode, file, 1);
1289 ftrace_notrace_release(struct inode *inode, struct file *file)
1291 return ftrace_regex_release(inode, file, 0);
1294 static struct file_operations ftrace_avail_fops = {
1295 .open = ftrace_avail_open,
1297 .llseek = seq_lseek,
1298 .release = ftrace_avail_release,
1301 static struct file_operations ftrace_failures_fops = {
1302 .open = ftrace_failures_open,
1304 .llseek = seq_lseek,
1305 .release = ftrace_avail_release,
1308 static struct file_operations ftrace_filter_fops = {
1309 .open = ftrace_filter_open,
1310 .read = ftrace_regex_read,
1311 .write = ftrace_filter_write,
1312 .llseek = ftrace_regex_lseek,
1313 .release = ftrace_filter_release,
1316 static struct file_operations ftrace_notrace_fops = {
1317 .open = ftrace_notrace_open,
1318 .read = ftrace_regex_read,
1319 .write = ftrace_notrace_write,
1320 .llseek = ftrace_regex_lseek,
1321 .release = ftrace_notrace_release,
1324 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
1326 static DEFINE_MUTEX(graph_lock);
1328 int ftrace_graph_count;
1329 unsigned long ftrace_graph_funcs[FTRACE_GRAPH_MAX_FUNCS] __read_mostly;
1332 g_next(struct seq_file *m, void *v, loff_t *pos)
1334 unsigned long *array = m->private;
1339 if (index >= ftrace_graph_count)
1342 return &array[index];
1345 static void *g_start(struct seq_file *m, loff_t *pos)
1349 mutex_lock(&graph_lock);
1351 p = g_next(m, p, pos);
1356 static void g_stop(struct seq_file *m, void *p)
1358 mutex_unlock(&graph_lock);
1361 static int g_show(struct seq_file *m, void *v)
1363 unsigned long *ptr = v;
1364 char str[KSYM_SYMBOL_LEN];
1369 kallsyms_lookup(*ptr, NULL, NULL, NULL, str);
1371 seq_printf(m, "%s\n", str);
1376 static struct seq_operations ftrace_graph_seq_ops = {
1384 ftrace_graph_open(struct inode *inode, struct file *file)
1388 if (unlikely(ftrace_disabled))
1391 mutex_lock(&graph_lock);
1392 if ((file->f_mode & FMODE_WRITE) &&
1393 !(file->f_flags & O_APPEND)) {
1394 ftrace_graph_count = 0;
1395 memset(ftrace_graph_funcs, 0, sizeof(ftrace_graph_funcs));
1398 if (file->f_mode & FMODE_READ) {
1399 ret = seq_open(file, &ftrace_graph_seq_ops);
1401 struct seq_file *m = file->private_data;
1402 m->private = ftrace_graph_funcs;
1405 file->private_data = ftrace_graph_funcs;
1406 mutex_unlock(&graph_lock);
1412 ftrace_graph_read(struct file *file, char __user *ubuf,
1413 size_t cnt, loff_t *ppos)
1415 if (file->f_mode & FMODE_READ)
1416 return seq_read(file, ubuf, cnt, ppos);
1422 ftrace_set_func(unsigned long *array, int idx, char *buffer)
1424 char str[KSYM_SYMBOL_LEN];
1425 struct dyn_ftrace *rec;
1426 struct ftrace_page *pg;
1430 if (ftrace_disabled)
1433 /* should not be called from interrupt context */
1434 spin_lock(&ftrace_lock);
1436 for (pg = ftrace_pages_start; pg; pg = pg->next) {
1437 for (i = 0; i < pg->index; i++) {
1438 rec = &pg->records[i];
1440 if (rec->flags & (FTRACE_FL_FAILED | FTRACE_FL_FREE))
1443 kallsyms_lookup(rec->ip, NULL, NULL, NULL, str);
1444 if (strcmp(str, buffer) == 0) {
1446 for (j = 0; j < idx; j++)
1447 if (array[j] == rec->ip) {
1452 array[idx] = rec->ip;
1457 spin_unlock(&ftrace_lock);
1459 return found ? 0 : -EINVAL;
1463 ftrace_graph_write(struct file *file, const char __user *ubuf,
1464 size_t cnt, loff_t *ppos)
1466 unsigned char buffer[FTRACE_BUFF_MAX+1];
1467 unsigned long *array;
1473 if (!cnt || cnt < 0)
1476 mutex_lock(&graph_lock);
1478 if (ftrace_graph_count >= FTRACE_GRAPH_MAX_FUNCS) {
1483 if (file->f_mode & FMODE_READ) {
1484 struct seq_file *m = file->private_data;
1487 array = file->private_data;
1489 ret = get_user(ch, ubuf++);
1495 /* skip white space */
1496 while (cnt && isspace(ch)) {
1497 ret = get_user(ch, ubuf++);
1510 while (cnt && !isspace(ch)) {
1511 if (index < FTRACE_BUFF_MAX)
1512 buffer[index++] = ch;
1517 ret = get_user(ch, ubuf++);
1525 /* we allow only one at a time */
1526 ret = ftrace_set_func(array, ftrace_graph_count, buffer);
1530 ftrace_graph_count++;
1532 file->f_pos += read;
1536 mutex_unlock(&graph_lock);
1541 static const struct file_operations ftrace_graph_fops = {
1542 .open = ftrace_graph_open,
1543 .read = ftrace_graph_read,
1544 .write = ftrace_graph_write,
1546 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
1548 static __init int ftrace_init_dyn_debugfs(struct dentry *d_tracer)
1550 struct dentry *entry;
1552 entry = debugfs_create_file("available_filter_functions", 0444,
1553 d_tracer, NULL, &ftrace_avail_fops);
1555 pr_warning("Could not create debugfs "
1556 "'available_filter_functions' entry\n");
1558 entry = debugfs_create_file("failures", 0444,
1559 d_tracer, NULL, &ftrace_failures_fops);
1561 pr_warning("Could not create debugfs 'failures' entry\n");
1563 entry = debugfs_create_file("set_ftrace_filter", 0644, d_tracer,
1564 NULL, &ftrace_filter_fops);
1566 pr_warning("Could not create debugfs "
1567 "'set_ftrace_filter' entry\n");
1569 entry = debugfs_create_file("set_ftrace_notrace", 0644, d_tracer,
1570 NULL, &ftrace_notrace_fops);
1572 pr_warning("Could not create debugfs "
1573 "'set_ftrace_notrace' entry\n");
1575 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
1576 entry = debugfs_create_file("set_graph_function", 0444, d_tracer,
1578 &ftrace_graph_fops);
1580 pr_warning("Could not create debugfs "
1581 "'set_graph_function' entry\n");
1582 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
1587 static int ftrace_convert_nops(struct module *mod,
1588 unsigned long *start,
1593 unsigned long flags;
1595 mutex_lock(&ftrace_start_lock);
1598 addr = ftrace_call_adjust(*p++);
1600 * Some architecture linkers will pad between
1601 * the different mcount_loc sections of different
1602 * object files to satisfy alignments.
1603 * Skip any NULL pointers.
1607 ftrace_record_ip(addr);
1610 /* disable interrupts to prevent kstop machine */
1611 local_irq_save(flags);
1612 ftrace_update_code(mod);
1613 local_irq_restore(flags);
1614 mutex_unlock(&ftrace_start_lock);
1619 void ftrace_init_module(struct module *mod,
1620 unsigned long *start, unsigned long *end)
1622 if (ftrace_disabled || start == end)
1624 ftrace_convert_nops(mod, start, end);
1627 extern unsigned long __start_mcount_loc[];
1628 extern unsigned long __stop_mcount_loc[];
1630 void __init ftrace_init(void)
1632 unsigned long count, addr, flags;
1635 /* Keep the ftrace pointer to the stub */
1636 addr = (unsigned long)ftrace_stub;
1638 local_irq_save(flags);
1639 ftrace_dyn_arch_init(&addr);
1640 local_irq_restore(flags);
1642 /* ftrace_dyn_arch_init places the return code in addr */
1646 count = __stop_mcount_loc - __start_mcount_loc;
1648 ret = ftrace_dyn_table_alloc(count);
1652 last_ftrace_enabled = ftrace_enabled = 1;
1654 ret = ftrace_convert_nops(NULL,
1660 ftrace_disabled = 1;
1665 static int __init ftrace_nodyn_init(void)
1670 device_initcall(ftrace_nodyn_init);
1672 static inline int ftrace_init_dyn_debugfs(struct dentry *d_tracer) { return 0; }
1673 static inline void ftrace_startup_enable(int command) { }
1674 /* Keep as macros so we do not need to define the commands */
1675 # define ftrace_startup(command) do { } while (0)
1676 # define ftrace_shutdown(command) do { } while (0)
1677 # define ftrace_startup_sysctl() do { } while (0)
1678 # define ftrace_shutdown_sysctl() do { } while (0)
1679 #endif /* CONFIG_DYNAMIC_FTRACE */
1682 ftrace_pid_read(struct file *file, char __user *ubuf,
1683 size_t cnt, loff_t *ppos)
1688 if (ftrace_pid_trace == ftrace_swapper_pid)
1689 r = sprintf(buf, "swapper tasks\n");
1690 else if (ftrace_pid_trace)
1691 r = sprintf(buf, "%u\n", pid_nr(ftrace_pid_trace));
1693 r = sprintf(buf, "no pid\n");
1695 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
1698 static void clear_ftrace_swapper(void)
1700 struct task_struct *p;
1704 for_each_online_cpu(cpu) {
1706 clear_tsk_trace_trace(p);
1711 static void set_ftrace_swapper(void)
1713 struct task_struct *p;
1717 for_each_online_cpu(cpu) {
1719 set_tsk_trace_trace(p);
1724 static void clear_ftrace_pid(struct pid *pid)
1726 struct task_struct *p;
1728 do_each_pid_task(pid, PIDTYPE_PID, p) {
1729 clear_tsk_trace_trace(p);
1730 } while_each_pid_task(pid, PIDTYPE_PID, p);
1734 static void set_ftrace_pid(struct pid *pid)
1736 struct task_struct *p;
1738 do_each_pid_task(pid, PIDTYPE_PID, p) {
1739 set_tsk_trace_trace(p);
1740 } while_each_pid_task(pid, PIDTYPE_PID, p);
1743 static void clear_ftrace_pid_task(struct pid **pid)
1745 if (*pid == ftrace_swapper_pid)
1746 clear_ftrace_swapper();
1748 clear_ftrace_pid(*pid);
1753 static void set_ftrace_pid_task(struct pid *pid)
1755 if (pid == ftrace_swapper_pid)
1756 set_ftrace_swapper();
1758 set_ftrace_pid(pid);
1762 ftrace_pid_write(struct file *filp, const char __user *ubuf,
1763 size_t cnt, loff_t *ppos)
1770 if (cnt >= sizeof(buf))
1773 if (copy_from_user(&buf, ubuf, cnt))
1778 ret = strict_strtol(buf, 10, &val);
1782 mutex_lock(&ftrace_start_lock);
1784 /* disable pid tracing */
1785 if (!ftrace_pid_trace)
1788 clear_ftrace_pid_task(&ftrace_pid_trace);
1791 /* swapper task is special */
1793 pid = ftrace_swapper_pid;
1794 if (pid == ftrace_pid_trace)
1797 pid = find_get_pid(val);
1799 if (pid == ftrace_pid_trace) {
1805 if (ftrace_pid_trace)
1806 clear_ftrace_pid_task(&ftrace_pid_trace);
1811 ftrace_pid_trace = pid;
1813 set_ftrace_pid_task(ftrace_pid_trace);
1816 /* update the function call */
1817 ftrace_update_pid_func();
1818 ftrace_startup_enable(0);
1821 mutex_unlock(&ftrace_start_lock);
1826 static struct file_operations ftrace_pid_fops = {
1827 .read = ftrace_pid_read,
1828 .write = ftrace_pid_write,
1831 static __init int ftrace_init_debugfs(void)
1833 struct dentry *d_tracer;
1834 struct dentry *entry;
1836 d_tracer = tracing_init_dentry();
1840 ftrace_init_dyn_debugfs(d_tracer);
1842 entry = debugfs_create_file("set_ftrace_pid", 0644, d_tracer,
1843 NULL, &ftrace_pid_fops);
1845 pr_warning("Could not create debugfs "
1846 "'set_ftrace_pid' entry\n");
1850 fs_initcall(ftrace_init_debugfs);
1853 * ftrace_kill - kill ftrace
1855 * This function should be used by panic code. It stops ftrace
1856 * but in a not so nice way. If you need to simply kill ftrace
1857 * from a non-atomic section, use ftrace_kill.
1859 void ftrace_kill(void)
1861 ftrace_disabled = 1;
1863 clear_ftrace_function();
1867 * register_ftrace_function - register a function for profiling
1868 * @ops - ops structure that holds the function for profiling.
1870 * Register a function to be called by all functions in the
1873 * Note: @ops->func and all the functions it calls must be labeled
1874 * with "notrace", otherwise it will go into a
1877 int register_ftrace_function(struct ftrace_ops *ops)
1881 if (unlikely(ftrace_disabled))
1884 mutex_lock(&ftrace_sysctl_lock);
1886 ret = __register_ftrace_function(ops);
1889 mutex_unlock(&ftrace_sysctl_lock);
1894 * unregister_ftrace_function - unresgister a function for profiling.
1895 * @ops - ops structure that holds the function to unregister
1897 * Unregister a function that was added to be called by ftrace profiling.
1899 int unregister_ftrace_function(struct ftrace_ops *ops)
1903 mutex_lock(&ftrace_sysctl_lock);
1904 ret = __unregister_ftrace_function(ops);
1906 mutex_unlock(&ftrace_sysctl_lock);
1912 ftrace_enable_sysctl(struct ctl_table *table, int write,
1913 struct file *file, void __user *buffer, size_t *lenp,
1918 if (unlikely(ftrace_disabled))
1921 mutex_lock(&ftrace_sysctl_lock);
1923 ret = proc_dointvec(table, write, file, buffer, lenp, ppos);
1925 if (ret || !write || (last_ftrace_enabled == ftrace_enabled))
1928 last_ftrace_enabled = ftrace_enabled;
1930 if (ftrace_enabled) {
1932 ftrace_startup_sysctl();
1934 /* we are starting ftrace again */
1935 if (ftrace_list != &ftrace_list_end) {
1936 if (ftrace_list->next == &ftrace_list_end)
1937 ftrace_trace_function = ftrace_list->func;
1939 ftrace_trace_function = ftrace_list_func;
1943 /* stopping ftrace calls (just send to ftrace_stub) */
1944 ftrace_trace_function = ftrace_stub;
1946 ftrace_shutdown_sysctl();
1950 mutex_unlock(&ftrace_sysctl_lock);
1954 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
1956 static atomic_t ftrace_graph_active;
1958 int ftrace_graph_entry_stub(struct ftrace_graph_ent *trace)
1963 /* The callbacks that hook a function */
1964 trace_func_graph_ret_t ftrace_graph_return =
1965 (trace_func_graph_ret_t)ftrace_stub;
1966 trace_func_graph_ent_t ftrace_graph_entry = ftrace_graph_entry_stub;
1968 /* Try to assign a return stack array on FTRACE_RETSTACK_ALLOC_SIZE tasks. */
1969 static int alloc_retstack_tasklist(struct ftrace_ret_stack **ret_stack_list)
1973 unsigned long flags;
1974 int start = 0, end = FTRACE_RETSTACK_ALLOC_SIZE;
1975 struct task_struct *g, *t;
1977 for (i = 0; i < FTRACE_RETSTACK_ALLOC_SIZE; i++) {
1978 ret_stack_list[i] = kmalloc(FTRACE_RETFUNC_DEPTH
1979 * sizeof(struct ftrace_ret_stack),
1981 if (!ret_stack_list[i]) {
1989 read_lock_irqsave(&tasklist_lock, flags);
1990 do_each_thread(g, t) {
1996 if (t->ret_stack == NULL) {
1997 t->curr_ret_stack = -1;
1998 /* Make sure IRQs see the -1 first: */
2000 t->ret_stack = ret_stack_list[start++];
2001 atomic_set(&t->tracing_graph_pause, 0);
2002 atomic_set(&t->trace_overrun, 0);
2004 } while_each_thread(g, t);
2007 read_unlock_irqrestore(&tasklist_lock, flags);
2009 for (i = start; i < end; i++)
2010 kfree(ret_stack_list[i]);
2014 /* Allocate a return stack for each task */
2015 static int start_graph_tracing(void)
2017 struct ftrace_ret_stack **ret_stack_list;
2020 ret_stack_list = kmalloc(FTRACE_RETSTACK_ALLOC_SIZE *
2021 sizeof(struct ftrace_ret_stack *),
2024 if (!ret_stack_list)
2028 ret = alloc_retstack_tasklist(ret_stack_list);
2029 } while (ret == -EAGAIN);
2031 kfree(ret_stack_list);
2035 int register_ftrace_graph(trace_func_graph_ret_t retfunc,
2036 trace_func_graph_ent_t entryfunc)
2040 mutex_lock(&ftrace_sysctl_lock);
2042 atomic_inc(&ftrace_graph_active);
2043 ret = start_graph_tracing();
2045 atomic_dec(&ftrace_graph_active);
2049 ftrace_graph_return = retfunc;
2050 ftrace_graph_entry = entryfunc;
2052 ftrace_startup(FTRACE_START_FUNC_RET);
2055 mutex_unlock(&ftrace_sysctl_lock);
2059 void unregister_ftrace_graph(void)
2061 mutex_lock(&ftrace_sysctl_lock);
2063 atomic_dec(&ftrace_graph_active);
2064 ftrace_graph_return = (trace_func_graph_ret_t)ftrace_stub;
2065 ftrace_graph_entry = ftrace_graph_entry_stub;
2066 ftrace_shutdown(FTRACE_STOP_FUNC_RET);
2068 mutex_unlock(&ftrace_sysctl_lock);
2071 /* Allocate a return stack for newly created task */
2072 void ftrace_graph_init_task(struct task_struct *t)
2074 if (atomic_read(&ftrace_graph_active)) {
2075 t->ret_stack = kmalloc(FTRACE_RETFUNC_DEPTH
2076 * sizeof(struct ftrace_ret_stack),
2080 t->curr_ret_stack = -1;
2081 atomic_set(&t->tracing_graph_pause, 0);
2082 atomic_set(&t->trace_overrun, 0);
2084 t->ret_stack = NULL;
2087 void ftrace_graph_exit_task(struct task_struct *t)
2089 struct ftrace_ret_stack *ret_stack = t->ret_stack;
2091 t->ret_stack = NULL;
2092 /* NULL must become visible to IRQs before we free it: */
2098 void ftrace_graph_stop(void)