2 * ring buffer based function tracer
4 * Copyright (C) 2007-2012 Steven Rostedt <srostedt@redhat.com>
5 * Copyright (C) 2008 Ingo Molnar <mingo@redhat.com>
7 * Originally taken from the RT patch by:
8 * Arnaldo Carvalho de Melo <acme@redhat.com>
10 * Based on code from the latency_tracer, that is:
11 * Copyright (C) 2004-2006 Ingo Molnar
12 * Copyright (C) 2004 Nadia Yvette Chambers
14 #include <linux/ring_buffer.h>
15 #include <generated/utsrelease.h>
16 #include <linux/stacktrace.h>
17 #include <linux/writeback.h>
18 #include <linux/kallsyms.h>
19 #include <linux/seq_file.h>
20 #include <linux/notifier.h>
21 #include <linux/irqflags.h>
22 #include <linux/debugfs.h>
23 #include <linux/tracefs.h>
24 #include <linux/pagemap.h>
25 #include <linux/hardirq.h>
26 #include <linux/linkage.h>
27 #include <linux/uaccess.h>
28 #include <linux/vmalloc.h>
29 #include <linux/ftrace.h>
30 #include <linux/module.h>
31 #include <linux/percpu.h>
32 #include <linux/splice.h>
33 #include <linux/kdebug.h>
34 #include <linux/string.h>
35 #include <linux/mount.h>
36 #include <linux/rwsem.h>
37 #include <linux/slab.h>
38 #include <linux/ctype.h>
39 #include <linux/init.h>
40 #include <linux/poll.h>
41 #include <linux/nmi.h>
43 #include <linux/sched/rt.h>
46 #include "trace_output.h"
49 * On boot up, the ring buffer is set to the minimum size, so that
50 * we do not waste memory on systems that are not using tracing.
52 bool ring_buffer_expanded;
55 * We need to change this state when a selftest is running.
56 * A selftest will lurk into the ring-buffer to count the
57 * entries inserted during the selftest although some concurrent
58 * insertions into the ring-buffer such as trace_printk could occurred
59 * at the same time, giving false positive or negative results.
61 static bool __read_mostly tracing_selftest_running;
64 * If a tracer is running, we do not want to run SELFTEST.
66 bool __read_mostly tracing_selftest_disabled;
68 /* Pipe tracepoints to printk */
69 struct trace_iterator *tracepoint_print_iter;
70 int tracepoint_printk;
72 /* For tracers that don't implement custom flags */
73 static struct tracer_opt dummy_tracer_opt[] = {
78 dummy_set_flag(struct trace_array *tr, u32 old_flags, u32 bit, int set)
84 * To prevent the comm cache from being overwritten when no
85 * tracing is active, only save the comm when a trace event
88 static DEFINE_PER_CPU(bool, trace_cmdline_save);
91 * Kill all tracing for good (never come back).
92 * It is initialized to 1 but will turn to zero if the initialization
93 * of the tracer is successful. But that is the only place that sets
96 static int tracing_disabled = 1;
98 cpumask_var_t __read_mostly tracing_buffer_mask;
101 * ftrace_dump_on_oops - variable to dump ftrace buffer on oops
103 * If there is an oops (or kernel panic) and the ftrace_dump_on_oops
104 * is set, then ftrace_dump is called. This will output the contents
105 * of the ftrace buffers to the console. This is very useful for
106 * capturing traces that lead to crashes and outputing it to a
109 * It is default off, but you can enable it with either specifying
110 * "ftrace_dump_on_oops" in the kernel command line, or setting
111 * /proc/sys/kernel/ftrace_dump_on_oops
112 * Set 1 if you want to dump buffers of all CPUs
113 * Set 2 if you want to dump the buffer of the CPU that triggered oops
116 enum ftrace_dump_mode ftrace_dump_on_oops;
118 /* When set, tracing will stop when a WARN*() is hit */
119 int __disable_trace_on_warning;
121 #ifdef CONFIG_TRACE_ENUM_MAP_FILE
122 /* Map of enums to their values, for "enum_map" file */
123 struct trace_enum_map_head {
125 unsigned long length;
128 union trace_enum_map_item;
130 struct trace_enum_map_tail {
132 * "end" is first and points to NULL as it must be different
133 * than "mod" or "enum_string"
135 union trace_enum_map_item *next;
136 const char *end; /* points to NULL */
139 static DEFINE_MUTEX(trace_enum_mutex);
142 * The trace_enum_maps are saved in an array with two extra elements,
143 * one at the beginning, and one at the end. The beginning item contains
144 * the count of the saved maps (head.length), and the module they
145 * belong to if not built in (head.mod). The ending item contains a
146 * pointer to the next array of saved enum_map items.
148 union trace_enum_map_item {
149 struct trace_enum_map map;
150 struct trace_enum_map_head head;
151 struct trace_enum_map_tail tail;
154 static union trace_enum_map_item *trace_enum_maps;
155 #endif /* CONFIG_TRACE_ENUM_MAP_FILE */
157 static int tracing_set_tracer(struct trace_array *tr, const char *buf);
159 #define MAX_TRACER_SIZE 100
160 static char bootup_tracer_buf[MAX_TRACER_SIZE] __initdata;
161 static char *default_bootup_tracer;
163 static bool allocate_snapshot;
165 static int __init set_cmdline_ftrace(char *str)
167 strlcpy(bootup_tracer_buf, str, MAX_TRACER_SIZE);
168 default_bootup_tracer = bootup_tracer_buf;
169 /* We are using ftrace early, expand it */
170 ring_buffer_expanded = true;
173 __setup("ftrace=", set_cmdline_ftrace);
175 static int __init set_ftrace_dump_on_oops(char *str)
177 if (*str++ != '=' || !*str) {
178 ftrace_dump_on_oops = DUMP_ALL;
182 if (!strcmp("orig_cpu", str)) {
183 ftrace_dump_on_oops = DUMP_ORIG;
189 __setup("ftrace_dump_on_oops", set_ftrace_dump_on_oops);
191 static int __init stop_trace_on_warning(char *str)
193 if ((strcmp(str, "=0") != 0 && strcmp(str, "=off") != 0))
194 __disable_trace_on_warning = 1;
197 __setup("traceoff_on_warning", stop_trace_on_warning);
199 static int __init boot_alloc_snapshot(char *str)
201 allocate_snapshot = true;
202 /* We also need the main ring buffer expanded */
203 ring_buffer_expanded = true;
206 __setup("alloc_snapshot", boot_alloc_snapshot);
209 static char trace_boot_options_buf[MAX_TRACER_SIZE] __initdata;
211 static int __init set_trace_boot_options(char *str)
213 strlcpy(trace_boot_options_buf, str, MAX_TRACER_SIZE);
216 __setup("trace_options=", set_trace_boot_options);
218 static char trace_boot_clock_buf[MAX_TRACER_SIZE] __initdata;
219 static char *trace_boot_clock __initdata;
221 static int __init set_trace_boot_clock(char *str)
223 strlcpy(trace_boot_clock_buf, str, MAX_TRACER_SIZE);
224 trace_boot_clock = trace_boot_clock_buf;
227 __setup("trace_clock=", set_trace_boot_clock);
229 static int __init set_tracepoint_printk(char *str)
231 if ((strcmp(str, "=0") != 0 && strcmp(str, "=off") != 0))
232 tracepoint_printk = 1;
235 __setup("tp_printk", set_tracepoint_printk);
237 unsigned long long ns2usecs(cycle_t nsec)
244 /* trace_flags holds trace_options default values */
245 #define TRACE_DEFAULT_FLAGS \
246 (FUNCTION_DEFAULT_FLAGS | \
247 TRACE_ITER_PRINT_PARENT | TRACE_ITER_PRINTK | \
248 TRACE_ITER_ANNOTATE | TRACE_ITER_CONTEXT_INFO | \
249 TRACE_ITER_RECORD_CMD | TRACE_ITER_OVERWRITE | \
250 TRACE_ITER_IRQ_INFO | TRACE_ITER_MARKERS)
252 /* trace_options that are only supported by global_trace */
253 #define TOP_LEVEL_TRACE_FLAGS (TRACE_ITER_PRINTK | \
254 TRACE_ITER_PRINTK_MSGONLY | TRACE_ITER_RECORD_CMD)
256 /* trace_flags that are default zero for instances */
257 #define ZEROED_TRACE_FLAGS \
258 TRACE_ITER_EVENT_FORK
261 * The global_trace is the descriptor that holds the tracing
262 * buffers for the live tracing. For each CPU, it contains
263 * a link list of pages that will store trace entries. The
264 * page descriptor of the pages in the memory is used to hold
265 * the link list by linking the lru item in the page descriptor
266 * to each of the pages in the buffer per CPU.
268 * For each active CPU there is a data field that holds the
269 * pages for the buffer for that CPU. Each CPU has the same number
270 * of pages allocated for its buffer.
272 static struct trace_array global_trace = {
273 .trace_flags = TRACE_DEFAULT_FLAGS,
276 LIST_HEAD(ftrace_trace_arrays);
278 int trace_array_get(struct trace_array *this_tr)
280 struct trace_array *tr;
283 mutex_lock(&trace_types_lock);
284 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
291 mutex_unlock(&trace_types_lock);
296 static void __trace_array_put(struct trace_array *this_tr)
298 WARN_ON(!this_tr->ref);
302 void trace_array_put(struct trace_array *this_tr)
304 mutex_lock(&trace_types_lock);
305 __trace_array_put(this_tr);
306 mutex_unlock(&trace_types_lock);
309 int call_filter_check_discard(struct trace_event_call *call, void *rec,
310 struct ring_buffer *buffer,
311 struct ring_buffer_event *event)
313 if (unlikely(call->flags & TRACE_EVENT_FL_FILTERED) &&
314 !filter_match_preds(call->filter, rec)) {
315 __trace_event_discard_commit(buffer, event);
322 void trace_free_pid_list(struct trace_pid_list *pid_list)
324 vfree(pid_list->pids);
329 * trace_find_filtered_pid - check if a pid exists in a filtered_pid list
330 * @filtered_pids: The list of pids to check
331 * @search_pid: The PID to find in @filtered_pids
333 * Returns true if @search_pid is fonud in @filtered_pids, and false otherwis.
336 trace_find_filtered_pid(struct trace_pid_list *filtered_pids, pid_t search_pid)
339 * If pid_max changed after filtered_pids was created, we
340 * by default ignore all pids greater than the previous pid_max.
342 if (search_pid >= filtered_pids->pid_max)
345 return test_bit(search_pid, filtered_pids->pids);
349 * trace_ignore_this_task - should a task be ignored for tracing
350 * @filtered_pids: The list of pids to check
351 * @task: The task that should be ignored if not filtered
353 * Checks if @task should be traced or not from @filtered_pids.
354 * Returns true if @task should *NOT* be traced.
355 * Returns false if @task should be traced.
358 trace_ignore_this_task(struct trace_pid_list *filtered_pids, struct task_struct *task)
361 * Return false, because if filtered_pids does not exist,
362 * all pids are good to trace.
367 return !trace_find_filtered_pid(filtered_pids, task->pid);
371 * trace_pid_filter_add_remove - Add or remove a task from a pid_list
372 * @pid_list: The list to modify
373 * @self: The current task for fork or NULL for exit
374 * @task: The task to add or remove
376 * If adding a task, if @self is defined, the task is only added if @self
377 * is also included in @pid_list. This happens on fork and tasks should
378 * only be added when the parent is listed. If @self is NULL, then the
379 * @task pid will be removed from the list, which would happen on exit
382 void trace_filter_add_remove_task(struct trace_pid_list *pid_list,
383 struct task_struct *self,
384 struct task_struct *task)
389 /* For forks, we only add if the forking task is listed */
391 if (!trace_find_filtered_pid(pid_list, self->pid))
395 /* Sorry, but we don't support pid_max changing after setting */
396 if (task->pid >= pid_list->pid_max)
399 /* "self" is set for forks, and NULL for exits */
401 set_bit(task->pid, pid_list->pids);
403 clear_bit(task->pid, pid_list->pids);
407 * trace_pid_next - Used for seq_file to get to the next pid of a pid_list
408 * @pid_list: The pid list to show
409 * @v: The last pid that was shown (+1 the actual pid to let zero be displayed)
410 * @pos: The position of the file
412 * This is used by the seq_file "next" operation to iterate the pids
413 * listed in a trace_pid_list structure.
415 * Returns the pid+1 as we want to display pid of zero, but NULL would
416 * stop the iteration.
418 void *trace_pid_next(struct trace_pid_list *pid_list, void *v, loff_t *pos)
420 unsigned long pid = (unsigned long)v;
424 /* pid already is +1 of the actual prevous bit */
425 pid = find_next_bit(pid_list->pids, pid_list->pid_max, pid);
427 /* Return pid + 1 to allow zero to be represented */
428 if (pid < pid_list->pid_max)
429 return (void *)(pid + 1);
435 * trace_pid_start - Used for seq_file to start reading pid lists
436 * @pid_list: The pid list to show
437 * @pos: The position of the file
439 * This is used by seq_file "start" operation to start the iteration
442 * Returns the pid+1 as we want to display pid of zero, but NULL would
443 * stop the iteration.
445 void *trace_pid_start(struct trace_pid_list *pid_list, loff_t *pos)
450 pid = find_first_bit(pid_list->pids, pid_list->pid_max);
451 if (pid >= pid_list->pid_max)
454 /* Return pid + 1 so that zero can be the exit value */
455 for (pid++; pid && l < *pos;
456 pid = (unsigned long)trace_pid_next(pid_list, (void *)pid, &l))
462 * trace_pid_show - show the current pid in seq_file processing
463 * @m: The seq_file structure to write into
464 * @v: A void pointer of the pid (+1) value to display
466 * Can be directly used by seq_file operations to display the current
469 int trace_pid_show(struct seq_file *m, void *v)
471 unsigned long pid = (unsigned long)v - 1;
473 seq_printf(m, "%lu\n", pid);
477 /* 128 should be much more than enough */
478 #define PID_BUF_SIZE 127
480 int trace_pid_write(struct trace_pid_list *filtered_pids,
481 struct trace_pid_list **new_pid_list,
482 const char __user *ubuf, size_t cnt)
484 struct trace_pid_list *pid_list;
485 struct trace_parser parser;
493 if (trace_parser_get_init(&parser, PID_BUF_SIZE + 1))
497 * Always recreate a new array. The write is an all or nothing
498 * operation. Always create a new array when adding new pids by
499 * the user. If the operation fails, then the current list is
502 pid_list = kmalloc(sizeof(*pid_list), GFP_KERNEL);
504 trace_parser_put(&parser);
508 pid_list->pid_max = READ_ONCE(pid_max);
510 /* Only truncating will shrink pid_max */
511 if (filtered_pids && filtered_pids->pid_max > pid_list->pid_max)
512 pid_list->pid_max = filtered_pids->pid_max;
514 pid_list->pids = vzalloc((pid_list->pid_max + 7) >> 3);
515 if (!pid_list->pids) {
516 trace_parser_put(&parser);
522 /* copy the current bits to the new max */
523 for_each_set_bit(pid, filtered_pids->pids,
524 filtered_pids->pid_max) {
525 set_bit(pid, pid_list->pids);
534 ret = trace_get_user(&parser, ubuf, cnt, &pos);
535 if (ret < 0 || !trace_parser_loaded(&parser))
542 parser.buffer[parser.idx] = 0;
545 if (kstrtoul(parser.buffer, 0, &val))
547 if (val >= pid_list->pid_max)
552 set_bit(pid, pid_list->pids);
555 trace_parser_clear(&parser);
558 trace_parser_put(&parser);
561 trace_free_pid_list(pid_list);
566 /* Cleared the list of pids */
567 trace_free_pid_list(pid_list);
572 *new_pid_list = pid_list;
577 static cycle_t buffer_ftrace_now(struct trace_buffer *buf, int cpu)
581 /* Early boot up does not have a buffer yet */
583 return trace_clock_local();
585 ts = ring_buffer_time_stamp(buf->buffer, cpu);
586 ring_buffer_normalize_time_stamp(buf->buffer, cpu, &ts);
591 cycle_t ftrace_now(int cpu)
593 return buffer_ftrace_now(&global_trace.trace_buffer, cpu);
597 * tracing_is_enabled - Show if global_trace has been disabled
599 * Shows if the global trace has been enabled or not. It uses the
600 * mirror flag "buffer_disabled" to be used in fast paths such as for
601 * the irqsoff tracer. But it may be inaccurate due to races. If you
602 * need to know the accurate state, use tracing_is_on() which is a little
603 * slower, but accurate.
605 int tracing_is_enabled(void)
608 * For quick access (irqsoff uses this in fast path), just
609 * return the mirror variable of the state of the ring buffer.
610 * It's a little racy, but we don't really care.
613 return !global_trace.buffer_disabled;
617 * trace_buf_size is the size in bytes that is allocated
618 * for a buffer. Note, the number of bytes is always rounded
621 * This number is purposely set to a low number of 16384.
622 * If the dump on oops happens, it will be much appreciated
623 * to not have to wait for all that output. Anyway this can be
624 * boot time and run time configurable.
626 #define TRACE_BUF_SIZE_DEFAULT 1441792UL /* 16384 * 88 (sizeof(entry)) */
628 static unsigned long trace_buf_size = TRACE_BUF_SIZE_DEFAULT;
630 /* trace_types holds a link list of available tracers. */
631 static struct tracer *trace_types __read_mostly;
634 * trace_types_lock is used to protect the trace_types list.
636 DEFINE_MUTEX(trace_types_lock);
639 * serialize the access of the ring buffer
641 * ring buffer serializes readers, but it is low level protection.
642 * The validity of the events (which returns by ring_buffer_peek() ..etc)
643 * are not protected by ring buffer.
645 * The content of events may become garbage if we allow other process consumes
646 * these events concurrently:
647 * A) the page of the consumed events may become a normal page
648 * (not reader page) in ring buffer, and this page will be rewrited
649 * by events producer.
650 * B) The page of the consumed events may become a page for splice_read,
651 * and this page will be returned to system.
653 * These primitives allow multi process access to different cpu ring buffer
656 * These primitives don't distinguish read-only and read-consume access.
657 * Multi read-only access are also serialized.
661 static DECLARE_RWSEM(all_cpu_access_lock);
662 static DEFINE_PER_CPU(struct mutex, cpu_access_lock);
664 static inline void trace_access_lock(int cpu)
666 if (cpu == RING_BUFFER_ALL_CPUS) {
667 /* gain it for accessing the whole ring buffer. */
668 down_write(&all_cpu_access_lock);
670 /* gain it for accessing a cpu ring buffer. */
672 /* Firstly block other trace_access_lock(RING_BUFFER_ALL_CPUS). */
673 down_read(&all_cpu_access_lock);
675 /* Secondly block other access to this @cpu ring buffer. */
676 mutex_lock(&per_cpu(cpu_access_lock, cpu));
680 static inline void trace_access_unlock(int cpu)
682 if (cpu == RING_BUFFER_ALL_CPUS) {
683 up_write(&all_cpu_access_lock);
685 mutex_unlock(&per_cpu(cpu_access_lock, cpu));
686 up_read(&all_cpu_access_lock);
690 static inline void trace_access_lock_init(void)
694 for_each_possible_cpu(cpu)
695 mutex_init(&per_cpu(cpu_access_lock, cpu));
700 static DEFINE_MUTEX(access_lock);
702 static inline void trace_access_lock(int cpu)
705 mutex_lock(&access_lock);
708 static inline void trace_access_unlock(int cpu)
711 mutex_unlock(&access_lock);
714 static inline void trace_access_lock_init(void)
720 #ifdef CONFIG_STACKTRACE
721 static void __ftrace_trace_stack(struct ring_buffer *buffer,
723 int skip, int pc, struct pt_regs *regs);
724 static inline void ftrace_trace_stack(struct trace_array *tr,
725 struct ring_buffer *buffer,
727 int skip, int pc, struct pt_regs *regs);
730 static inline void __ftrace_trace_stack(struct ring_buffer *buffer,
732 int skip, int pc, struct pt_regs *regs)
735 static inline void ftrace_trace_stack(struct trace_array *tr,
736 struct ring_buffer *buffer,
738 int skip, int pc, struct pt_regs *regs)
744 static void tracer_tracing_on(struct trace_array *tr)
746 if (tr->trace_buffer.buffer)
747 ring_buffer_record_on(tr->trace_buffer.buffer);
749 * This flag is looked at when buffers haven't been allocated
750 * yet, or by some tracers (like irqsoff), that just want to
751 * know if the ring buffer has been disabled, but it can handle
752 * races of where it gets disabled but we still do a record.
753 * As the check is in the fast path of the tracers, it is more
754 * important to be fast than accurate.
756 tr->buffer_disabled = 0;
757 /* Make the flag seen by readers */
762 * tracing_on - enable tracing buffers
764 * This function enables tracing buffers that may have been
765 * disabled with tracing_off.
767 void tracing_on(void)
769 tracer_tracing_on(&global_trace);
771 EXPORT_SYMBOL_GPL(tracing_on);
774 * __trace_puts - write a constant string into the trace buffer.
775 * @ip: The address of the caller
776 * @str: The constant string to write
777 * @size: The size of the string.
779 int __trace_puts(unsigned long ip, const char *str, int size)
781 struct ring_buffer_event *event;
782 struct ring_buffer *buffer;
783 struct print_entry *entry;
784 unsigned long irq_flags;
788 if (!(global_trace.trace_flags & TRACE_ITER_PRINTK))
791 pc = preempt_count();
793 if (unlikely(tracing_selftest_running || tracing_disabled))
796 alloc = sizeof(*entry) + size + 2; /* possible \n added */
798 local_save_flags(irq_flags);
799 buffer = global_trace.trace_buffer.buffer;
800 event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, alloc,
805 entry = ring_buffer_event_data(event);
808 memcpy(&entry->buf, str, size);
810 /* Add a newline if necessary */
811 if (entry->buf[size - 1] != '\n') {
812 entry->buf[size] = '\n';
813 entry->buf[size + 1] = '\0';
815 entry->buf[size] = '\0';
817 __buffer_unlock_commit(buffer, event);
818 ftrace_trace_stack(&global_trace, buffer, irq_flags, 4, pc, NULL);
822 EXPORT_SYMBOL_GPL(__trace_puts);
825 * __trace_bputs - write the pointer to a constant string into trace buffer
826 * @ip: The address of the caller
827 * @str: The constant string to write to the buffer to
829 int __trace_bputs(unsigned long ip, const char *str)
831 struct ring_buffer_event *event;
832 struct ring_buffer *buffer;
833 struct bputs_entry *entry;
834 unsigned long irq_flags;
835 int size = sizeof(struct bputs_entry);
838 if (!(global_trace.trace_flags & TRACE_ITER_PRINTK))
841 pc = preempt_count();
843 if (unlikely(tracing_selftest_running || tracing_disabled))
846 local_save_flags(irq_flags);
847 buffer = global_trace.trace_buffer.buffer;
848 event = trace_buffer_lock_reserve(buffer, TRACE_BPUTS, size,
853 entry = ring_buffer_event_data(event);
857 __buffer_unlock_commit(buffer, event);
858 ftrace_trace_stack(&global_trace, buffer, irq_flags, 4, pc, NULL);
862 EXPORT_SYMBOL_GPL(__trace_bputs);
864 #ifdef CONFIG_TRACER_SNAPSHOT
866 * trace_snapshot - take a snapshot of the current buffer.
868 * This causes a swap between the snapshot buffer and the current live
869 * tracing buffer. You can use this to take snapshots of the live
870 * trace when some condition is triggered, but continue to trace.
872 * Note, make sure to allocate the snapshot with either
873 * a tracing_snapshot_alloc(), or by doing it manually
874 * with: echo 1 > /sys/kernel/debug/tracing/snapshot
876 * If the snapshot buffer is not allocated, it will stop tracing.
877 * Basically making a permanent snapshot.
879 void tracing_snapshot(void)
881 struct trace_array *tr = &global_trace;
882 struct tracer *tracer = tr->current_trace;
886 internal_trace_puts("*** SNAPSHOT CALLED FROM NMI CONTEXT ***\n");
887 internal_trace_puts("*** snapshot is being ignored ***\n");
891 if (!tr->allocated_snapshot) {
892 internal_trace_puts("*** SNAPSHOT NOT ALLOCATED ***\n");
893 internal_trace_puts("*** stopping trace here! ***\n");
898 /* Note, snapshot can not be used when the tracer uses it */
899 if (tracer->use_max_tr) {
900 internal_trace_puts("*** LATENCY TRACER ACTIVE ***\n");
901 internal_trace_puts("*** Can not use snapshot (sorry) ***\n");
905 local_irq_save(flags);
906 update_max_tr(tr, current, smp_processor_id());
907 local_irq_restore(flags);
909 EXPORT_SYMBOL_GPL(tracing_snapshot);
911 static int resize_buffer_duplicate_size(struct trace_buffer *trace_buf,
912 struct trace_buffer *size_buf, int cpu_id);
913 static void set_buffer_entries(struct trace_buffer *buf, unsigned long val);
915 static int alloc_snapshot(struct trace_array *tr)
919 if (!tr->allocated_snapshot) {
921 /* allocate spare buffer */
922 ret = resize_buffer_duplicate_size(&tr->max_buffer,
923 &tr->trace_buffer, RING_BUFFER_ALL_CPUS);
927 tr->allocated_snapshot = true;
933 static void free_snapshot(struct trace_array *tr)
936 * We don't free the ring buffer. instead, resize it because
937 * The max_tr ring buffer has some state (e.g. ring->clock) and
938 * we want preserve it.
940 ring_buffer_resize(tr->max_buffer.buffer, 1, RING_BUFFER_ALL_CPUS);
941 set_buffer_entries(&tr->max_buffer, 1);
942 tracing_reset_online_cpus(&tr->max_buffer);
943 tr->allocated_snapshot = false;
947 * tracing_alloc_snapshot - allocate snapshot buffer.
949 * This only allocates the snapshot buffer if it isn't already
950 * allocated - it doesn't also take a snapshot.
952 * This is meant to be used in cases where the snapshot buffer needs
953 * to be set up for events that can't sleep but need to be able to
954 * trigger a snapshot.
956 int tracing_alloc_snapshot(void)
958 struct trace_array *tr = &global_trace;
961 ret = alloc_snapshot(tr);
966 EXPORT_SYMBOL_GPL(tracing_alloc_snapshot);
969 * trace_snapshot_alloc - allocate and take a snapshot of the current buffer.
971 * This is similar to trace_snapshot(), but it will allocate the
972 * snapshot buffer if it isn't already allocated. Use this only
973 * where it is safe to sleep, as the allocation may sleep.
975 * This causes a swap between the snapshot buffer and the current live
976 * tracing buffer. You can use this to take snapshots of the live
977 * trace when some condition is triggered, but continue to trace.
979 void tracing_snapshot_alloc(void)
983 ret = tracing_alloc_snapshot();
989 EXPORT_SYMBOL_GPL(tracing_snapshot_alloc);
991 void tracing_snapshot(void)
993 WARN_ONCE(1, "Snapshot feature not enabled, but internal snapshot used");
995 EXPORT_SYMBOL_GPL(tracing_snapshot);
996 int tracing_alloc_snapshot(void)
998 WARN_ONCE(1, "Snapshot feature not enabled, but snapshot allocation used");
1001 EXPORT_SYMBOL_GPL(tracing_alloc_snapshot);
1002 void tracing_snapshot_alloc(void)
1007 EXPORT_SYMBOL_GPL(tracing_snapshot_alloc);
1008 #endif /* CONFIG_TRACER_SNAPSHOT */
1010 static void tracer_tracing_off(struct trace_array *tr)
1012 if (tr->trace_buffer.buffer)
1013 ring_buffer_record_off(tr->trace_buffer.buffer);
1015 * This flag is looked at when buffers haven't been allocated
1016 * yet, or by some tracers (like irqsoff), that just want to
1017 * know if the ring buffer has been disabled, but it can handle
1018 * races of where it gets disabled but we still do a record.
1019 * As the check is in the fast path of the tracers, it is more
1020 * important to be fast than accurate.
1022 tr->buffer_disabled = 1;
1023 /* Make the flag seen by readers */
1028 * tracing_off - turn off tracing buffers
1030 * This function stops the tracing buffers from recording data.
1031 * It does not disable any overhead the tracers themselves may
1032 * be causing. This function simply causes all recording to
1033 * the ring buffers to fail.
1035 void tracing_off(void)
1037 tracer_tracing_off(&global_trace);
1039 EXPORT_SYMBOL_GPL(tracing_off);
1041 void disable_trace_on_warning(void)
1043 if (__disable_trace_on_warning)
1048 * tracer_tracing_is_on - show real state of ring buffer enabled
1049 * @tr : the trace array to know if ring buffer is enabled
1051 * Shows real state of the ring buffer if it is enabled or not.
1053 int tracer_tracing_is_on(struct trace_array *tr)
1055 if (tr->trace_buffer.buffer)
1056 return ring_buffer_record_is_on(tr->trace_buffer.buffer);
1057 return !tr->buffer_disabled;
1061 * tracing_is_on - show state of ring buffers enabled
1063 int tracing_is_on(void)
1065 return tracer_tracing_is_on(&global_trace);
1067 EXPORT_SYMBOL_GPL(tracing_is_on);
1069 static int __init set_buf_size(char *str)
1071 unsigned long buf_size;
1075 buf_size = memparse(str, &str);
1076 /* nr_entries can not be zero */
1079 trace_buf_size = buf_size;
1082 __setup("trace_buf_size=", set_buf_size);
1084 static int __init set_tracing_thresh(char *str)
1086 unsigned long threshold;
1091 ret = kstrtoul(str, 0, &threshold);
1094 tracing_thresh = threshold * 1000;
1097 __setup("tracing_thresh=", set_tracing_thresh);
1099 unsigned long nsecs_to_usecs(unsigned long nsecs)
1101 return nsecs / 1000;
1105 * TRACE_FLAGS is defined as a tuple matching bit masks with strings.
1106 * It uses C(a, b) where 'a' is the enum name and 'b' is the string that
1107 * matches it. By defining "C(a, b) b", TRACE_FLAGS becomes a list
1108 * of strings in the order that the enums were defined.
1113 /* These must match the bit postions in trace_iterator_flags */
1114 static const char *trace_options[] = {
1122 int in_ns; /* is this clock in nanoseconds? */
1123 } trace_clocks[] = {
1124 { trace_clock_local, "local", 1 },
1125 { trace_clock_global, "global", 1 },
1126 { trace_clock_counter, "counter", 0 },
1127 { trace_clock_jiffies, "uptime", 0 },
1128 { trace_clock, "perf", 1 },
1129 { ktime_get_mono_fast_ns, "mono", 1 },
1130 { ktime_get_raw_fast_ns, "mono_raw", 1 },
1135 * trace_parser_get_init - gets the buffer for trace parser
1137 int trace_parser_get_init(struct trace_parser *parser, int size)
1139 memset(parser, 0, sizeof(*parser));
1141 parser->buffer = kmalloc(size, GFP_KERNEL);
1142 if (!parser->buffer)
1145 parser->size = size;
1150 * trace_parser_put - frees the buffer for trace parser
1152 void trace_parser_put(struct trace_parser *parser)
1154 kfree(parser->buffer);
1158 * trace_get_user - reads the user input string separated by space
1159 * (matched by isspace(ch))
1161 * For each string found the 'struct trace_parser' is updated,
1162 * and the function returns.
1164 * Returns number of bytes read.
1166 * See kernel/trace/trace.h for 'struct trace_parser' details.
1168 int trace_get_user(struct trace_parser *parser, const char __user *ubuf,
1169 size_t cnt, loff_t *ppos)
1176 trace_parser_clear(parser);
1178 ret = get_user(ch, ubuf++);
1186 * The parser is not finished with the last write,
1187 * continue reading the user input without skipping spaces.
1189 if (!parser->cont) {
1190 /* skip white space */
1191 while (cnt && isspace(ch)) {
1192 ret = get_user(ch, ubuf++);
1199 /* only spaces were written */
1209 /* read the non-space input */
1210 while (cnt && !isspace(ch)) {
1211 if (parser->idx < parser->size - 1)
1212 parser->buffer[parser->idx++] = ch;
1217 ret = get_user(ch, ubuf++);
1224 /* We either got finished input or we have to wait for another call. */
1226 parser->buffer[parser->idx] = 0;
1227 parser->cont = false;
1228 } else if (parser->idx < parser->size - 1) {
1229 parser->cont = true;
1230 parser->buffer[parser->idx++] = ch;
1243 /* TODO add a seq_buf_to_buffer() */
1244 static ssize_t trace_seq_to_buffer(struct trace_seq *s, void *buf, size_t cnt)
1248 if (trace_seq_used(s) <= s->seq.readpos)
1251 len = trace_seq_used(s) - s->seq.readpos;
1254 memcpy(buf, s->buffer + s->seq.readpos, cnt);
1256 s->seq.readpos += cnt;
1260 unsigned long __read_mostly tracing_thresh;
1262 #ifdef CONFIG_TRACER_MAX_TRACE
1264 * Copy the new maximum trace into the separate maximum-trace
1265 * structure. (this way the maximum trace is permanently saved,
1266 * for later retrieval via /sys/kernel/debug/tracing/latency_trace)
1269 __update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
1271 struct trace_buffer *trace_buf = &tr->trace_buffer;
1272 struct trace_buffer *max_buf = &tr->max_buffer;
1273 struct trace_array_cpu *data = per_cpu_ptr(trace_buf->data, cpu);
1274 struct trace_array_cpu *max_data = per_cpu_ptr(max_buf->data, cpu);
1277 max_buf->time_start = data->preempt_timestamp;
1279 max_data->saved_latency = tr->max_latency;
1280 max_data->critical_start = data->critical_start;
1281 max_data->critical_end = data->critical_end;
1283 memcpy(max_data->comm, tsk->comm, TASK_COMM_LEN);
1284 max_data->pid = tsk->pid;
1286 * If tsk == current, then use current_uid(), as that does not use
1287 * RCU. The irq tracer can be called out of RCU scope.
1290 max_data->uid = current_uid();
1292 max_data->uid = task_uid(tsk);
1294 max_data->nice = tsk->static_prio - 20 - MAX_RT_PRIO;
1295 max_data->policy = tsk->policy;
1296 max_data->rt_priority = tsk->rt_priority;
1298 /* record this tasks comm */
1299 tracing_record_cmdline(tsk);
1303 * update_max_tr - snapshot all trace buffers from global_trace to max_tr
1305 * @tsk: the task with the latency
1306 * @cpu: The cpu that initiated the trace.
1308 * Flip the buffers between the @tr and the max_tr and record information
1309 * about which task was the cause of this latency.
1312 update_max_tr(struct trace_array *tr, struct task_struct *tsk, int cpu)
1314 struct ring_buffer *buf;
1319 WARN_ON_ONCE(!irqs_disabled());
1321 if (!tr->allocated_snapshot) {
1322 /* Only the nop tracer should hit this when disabling */
1323 WARN_ON_ONCE(tr->current_trace != &nop_trace);
1327 arch_spin_lock(&tr->max_lock);
1329 /* Inherit the recordable setting from trace_buffer */
1330 if (ring_buffer_record_is_set_on(tr->trace_buffer.buffer))
1331 ring_buffer_record_on(tr->max_buffer.buffer);
1333 ring_buffer_record_off(tr->max_buffer.buffer);
1335 buf = tr->trace_buffer.buffer;
1336 tr->trace_buffer.buffer = tr->max_buffer.buffer;
1337 tr->max_buffer.buffer = buf;
1339 __update_max_tr(tr, tsk, cpu);
1340 arch_spin_unlock(&tr->max_lock);
1344 * update_max_tr_single - only copy one trace over, and reset the rest
1346 * @tsk - task with the latency
1347 * @cpu - the cpu of the buffer to copy.
1349 * Flip the trace of a single CPU buffer between the @tr and the max_tr.
1352 update_max_tr_single(struct trace_array *tr, struct task_struct *tsk, int cpu)
1359 WARN_ON_ONCE(!irqs_disabled());
1360 if (!tr->allocated_snapshot) {
1361 /* Only the nop tracer should hit this when disabling */
1362 WARN_ON_ONCE(tr->current_trace != &nop_trace);
1366 arch_spin_lock(&tr->max_lock);
1368 ret = ring_buffer_swap_cpu(tr->max_buffer.buffer, tr->trace_buffer.buffer, cpu);
1370 if (ret == -EBUSY) {
1372 * We failed to swap the buffer due to a commit taking
1373 * place on this CPU. We fail to record, but we reset
1374 * the max trace buffer (no one writes directly to it)
1375 * and flag that it failed.
1377 trace_array_printk_buf(tr->max_buffer.buffer, _THIS_IP_,
1378 "Failed to swap buffers due to commit in progress\n");
1381 WARN_ON_ONCE(ret && ret != -EAGAIN && ret != -EBUSY);
1383 __update_max_tr(tr, tsk, cpu);
1384 arch_spin_unlock(&tr->max_lock);
1386 #endif /* CONFIG_TRACER_MAX_TRACE */
1388 static int wait_on_pipe(struct trace_iterator *iter, bool full)
1390 /* Iterators are static, they should be filled or empty */
1391 if (trace_buffer_iter(iter, iter->cpu_file))
1394 return ring_buffer_wait(iter->trace_buffer->buffer, iter->cpu_file,
1398 #ifdef CONFIG_FTRACE_STARTUP_TEST
1399 static int run_tracer_selftest(struct tracer *type)
1401 struct trace_array *tr = &global_trace;
1402 struct tracer *saved_tracer = tr->current_trace;
1405 if (!type->selftest || tracing_selftest_disabled)
1409 * Run a selftest on this tracer.
1410 * Here we reset the trace buffer, and set the current
1411 * tracer to be this tracer. The tracer can then run some
1412 * internal tracing to verify that everything is in order.
1413 * If we fail, we do not register this tracer.
1415 tracing_reset_online_cpus(&tr->trace_buffer);
1417 tr->current_trace = type;
1419 #ifdef CONFIG_TRACER_MAX_TRACE
1420 if (type->use_max_tr) {
1421 /* If we expanded the buffers, make sure the max is expanded too */
1422 if (ring_buffer_expanded)
1423 ring_buffer_resize(tr->max_buffer.buffer, trace_buf_size,
1424 RING_BUFFER_ALL_CPUS);
1425 tr->allocated_snapshot = true;
1429 /* the test is responsible for initializing and enabling */
1430 pr_info("Testing tracer %s: ", type->name);
1431 ret = type->selftest(type, tr);
1432 /* the test is responsible for resetting too */
1433 tr->current_trace = saved_tracer;
1435 printk(KERN_CONT "FAILED!\n");
1436 /* Add the warning after printing 'FAILED' */
1440 /* Only reset on passing, to avoid touching corrupted buffers */
1441 tracing_reset_online_cpus(&tr->trace_buffer);
1443 #ifdef CONFIG_TRACER_MAX_TRACE
1444 if (type->use_max_tr) {
1445 tr->allocated_snapshot = false;
1447 /* Shrink the max buffer again */
1448 if (ring_buffer_expanded)
1449 ring_buffer_resize(tr->max_buffer.buffer, 1,
1450 RING_BUFFER_ALL_CPUS);
1454 printk(KERN_CONT "PASSED\n");
1458 static inline int run_tracer_selftest(struct tracer *type)
1462 #endif /* CONFIG_FTRACE_STARTUP_TEST */
1464 static void add_tracer_options(struct trace_array *tr, struct tracer *t);
1466 static void __init apply_trace_boot_options(void);
1469 * register_tracer - register a tracer with the ftrace system.
1470 * @type - the plugin for the tracer
1472 * Register a new plugin tracer.
1474 int __init register_tracer(struct tracer *type)
1480 pr_info("Tracer must have a name\n");
1484 if (strlen(type->name) >= MAX_TRACER_SIZE) {
1485 pr_info("Tracer has a name longer than %d\n", MAX_TRACER_SIZE);
1489 mutex_lock(&trace_types_lock);
1491 tracing_selftest_running = true;
1493 for (t = trace_types; t; t = t->next) {
1494 if (strcmp(type->name, t->name) == 0) {
1496 pr_info("Tracer %s already registered\n",
1503 if (!type->set_flag)
1504 type->set_flag = &dummy_set_flag;
1506 /*allocate a dummy tracer_flags*/
1507 type->flags = kmalloc(sizeof(*type->flags), GFP_KERNEL);
1512 type->flags->val = 0;
1513 type->flags->opts = dummy_tracer_opt;
1515 if (!type->flags->opts)
1516 type->flags->opts = dummy_tracer_opt;
1518 /* store the tracer for __set_tracer_option */
1519 type->flags->trace = type;
1521 ret = run_tracer_selftest(type);
1525 type->next = trace_types;
1527 add_tracer_options(&global_trace, type);
1530 tracing_selftest_running = false;
1531 mutex_unlock(&trace_types_lock);
1533 if (ret || !default_bootup_tracer)
1536 if (strncmp(default_bootup_tracer, type->name, MAX_TRACER_SIZE))
1539 printk(KERN_INFO "Starting tracer '%s'\n", type->name);
1540 /* Do we want this tracer to start on bootup? */
1541 tracing_set_tracer(&global_trace, type->name);
1542 default_bootup_tracer = NULL;
1544 apply_trace_boot_options();
1546 /* disable other selftests, since this will break it. */
1547 tracing_selftest_disabled = true;
1548 #ifdef CONFIG_FTRACE_STARTUP_TEST
1549 printk(KERN_INFO "Disabling FTRACE selftests due to running tracer '%s'\n",
1557 void tracing_reset(struct trace_buffer *buf, int cpu)
1559 struct ring_buffer *buffer = buf->buffer;
1564 ring_buffer_record_disable(buffer);
1566 /* Make sure all commits have finished */
1567 synchronize_sched();
1568 ring_buffer_reset_cpu(buffer, cpu);
1570 ring_buffer_record_enable(buffer);
1573 void tracing_reset_online_cpus(struct trace_buffer *buf)
1575 struct ring_buffer *buffer = buf->buffer;
1581 ring_buffer_record_disable(buffer);
1583 /* Make sure all commits have finished */
1584 synchronize_sched();
1586 buf->time_start = buffer_ftrace_now(buf, buf->cpu);
1588 for_each_online_cpu(cpu)
1589 ring_buffer_reset_cpu(buffer, cpu);
1591 ring_buffer_record_enable(buffer);
1594 /* Must have trace_types_lock held */
1595 void tracing_reset_all_online_cpus(void)
1597 struct trace_array *tr;
1599 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
1600 tracing_reset_online_cpus(&tr->trace_buffer);
1601 #ifdef CONFIG_TRACER_MAX_TRACE
1602 tracing_reset_online_cpus(&tr->max_buffer);
1607 #define SAVED_CMDLINES_DEFAULT 128
1608 #define NO_CMDLINE_MAP UINT_MAX
1609 static arch_spinlock_t trace_cmdline_lock = __ARCH_SPIN_LOCK_UNLOCKED;
1610 struct saved_cmdlines_buffer {
1611 unsigned map_pid_to_cmdline[PID_MAX_DEFAULT+1];
1612 unsigned *map_cmdline_to_pid;
1613 unsigned cmdline_num;
1615 char *saved_cmdlines;
1617 static struct saved_cmdlines_buffer *savedcmd;
1619 /* temporary disable recording */
1620 static atomic_t trace_record_cmdline_disabled __read_mostly;
1622 static inline char *get_saved_cmdlines(int idx)
1624 return &savedcmd->saved_cmdlines[idx * TASK_COMM_LEN];
1627 static inline void set_cmdline(int idx, const char *cmdline)
1629 memcpy(get_saved_cmdlines(idx), cmdline, TASK_COMM_LEN);
1632 static int allocate_cmdlines_buffer(unsigned int val,
1633 struct saved_cmdlines_buffer *s)
1635 s->map_cmdline_to_pid = kmalloc(val * sizeof(*s->map_cmdline_to_pid),
1637 if (!s->map_cmdline_to_pid)
1640 s->saved_cmdlines = kmalloc(val * TASK_COMM_LEN, GFP_KERNEL);
1641 if (!s->saved_cmdlines) {
1642 kfree(s->map_cmdline_to_pid);
1647 s->cmdline_num = val;
1648 memset(&s->map_pid_to_cmdline, NO_CMDLINE_MAP,
1649 sizeof(s->map_pid_to_cmdline));
1650 memset(s->map_cmdline_to_pid, NO_CMDLINE_MAP,
1651 val * sizeof(*s->map_cmdline_to_pid));
1656 static int trace_create_savedcmd(void)
1660 savedcmd = kmalloc(sizeof(*savedcmd), GFP_KERNEL);
1664 ret = allocate_cmdlines_buffer(SAVED_CMDLINES_DEFAULT, savedcmd);
1674 int is_tracing_stopped(void)
1676 return global_trace.stop_count;
1680 * tracing_start - quick start of the tracer
1682 * If tracing is enabled but was stopped by tracing_stop,
1683 * this will start the tracer back up.
1685 void tracing_start(void)
1687 struct ring_buffer *buffer;
1688 unsigned long flags;
1690 if (tracing_disabled)
1693 raw_spin_lock_irqsave(&global_trace.start_lock, flags);
1694 if (--global_trace.stop_count) {
1695 if (global_trace.stop_count < 0) {
1696 /* Someone screwed up their debugging */
1698 global_trace.stop_count = 0;
1703 /* Prevent the buffers from switching */
1704 arch_spin_lock(&global_trace.max_lock);
1706 buffer = global_trace.trace_buffer.buffer;
1708 ring_buffer_record_enable(buffer);
1710 #ifdef CONFIG_TRACER_MAX_TRACE
1711 buffer = global_trace.max_buffer.buffer;
1713 ring_buffer_record_enable(buffer);
1716 arch_spin_unlock(&global_trace.max_lock);
1719 raw_spin_unlock_irqrestore(&global_trace.start_lock, flags);
1722 static void tracing_start_tr(struct trace_array *tr)
1724 struct ring_buffer *buffer;
1725 unsigned long flags;
1727 if (tracing_disabled)
1730 /* If global, we need to also start the max tracer */
1731 if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
1732 return tracing_start();
1734 raw_spin_lock_irqsave(&tr->start_lock, flags);
1736 if (--tr->stop_count) {
1737 if (tr->stop_count < 0) {
1738 /* Someone screwed up their debugging */
1745 buffer = tr->trace_buffer.buffer;
1747 ring_buffer_record_enable(buffer);
1750 raw_spin_unlock_irqrestore(&tr->start_lock, flags);
1754 * tracing_stop - quick stop of the tracer
1756 * Light weight way to stop tracing. Use in conjunction with
1759 void tracing_stop(void)
1761 struct ring_buffer *buffer;
1762 unsigned long flags;
1764 raw_spin_lock_irqsave(&global_trace.start_lock, flags);
1765 if (global_trace.stop_count++)
1768 /* Prevent the buffers from switching */
1769 arch_spin_lock(&global_trace.max_lock);
1771 buffer = global_trace.trace_buffer.buffer;
1773 ring_buffer_record_disable(buffer);
1775 #ifdef CONFIG_TRACER_MAX_TRACE
1776 buffer = global_trace.max_buffer.buffer;
1778 ring_buffer_record_disable(buffer);
1781 arch_spin_unlock(&global_trace.max_lock);
1784 raw_spin_unlock_irqrestore(&global_trace.start_lock, flags);
1787 static void tracing_stop_tr(struct trace_array *tr)
1789 struct ring_buffer *buffer;
1790 unsigned long flags;
1792 /* If global, we need to also stop the max tracer */
1793 if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
1794 return tracing_stop();
1796 raw_spin_lock_irqsave(&tr->start_lock, flags);
1797 if (tr->stop_count++)
1800 buffer = tr->trace_buffer.buffer;
1802 ring_buffer_record_disable(buffer);
1805 raw_spin_unlock_irqrestore(&tr->start_lock, flags);
1808 void trace_stop_cmdline_recording(void);
1810 static int trace_save_cmdline(struct task_struct *tsk)
1814 if (!tsk->pid || unlikely(tsk->pid > PID_MAX_DEFAULT))
1818 * It's not the end of the world if we don't get
1819 * the lock, but we also don't want to spin
1820 * nor do we want to disable interrupts,
1821 * so if we miss here, then better luck next time.
1823 if (!arch_spin_trylock(&trace_cmdline_lock))
1826 idx = savedcmd->map_pid_to_cmdline[tsk->pid];
1827 if (idx == NO_CMDLINE_MAP) {
1828 idx = (savedcmd->cmdline_idx + 1) % savedcmd->cmdline_num;
1831 * Check whether the cmdline buffer at idx has a pid
1832 * mapped. We are going to overwrite that entry so we
1833 * need to clear the map_pid_to_cmdline. Otherwise we
1834 * would read the new comm for the old pid.
1836 pid = savedcmd->map_cmdline_to_pid[idx];
1837 if (pid != NO_CMDLINE_MAP)
1838 savedcmd->map_pid_to_cmdline[pid] = NO_CMDLINE_MAP;
1840 savedcmd->map_cmdline_to_pid[idx] = tsk->pid;
1841 savedcmd->map_pid_to_cmdline[tsk->pid] = idx;
1843 savedcmd->cmdline_idx = idx;
1846 set_cmdline(idx, tsk->comm);
1848 arch_spin_unlock(&trace_cmdline_lock);
1853 static void __trace_find_cmdline(int pid, char comm[])
1858 strcpy(comm, "<idle>");
1862 if (WARN_ON_ONCE(pid < 0)) {
1863 strcpy(comm, "<XXX>");
1867 if (pid > PID_MAX_DEFAULT) {
1868 strcpy(comm, "<...>");
1872 map = savedcmd->map_pid_to_cmdline[pid];
1873 if (map != NO_CMDLINE_MAP)
1874 strcpy(comm, get_saved_cmdlines(map));
1876 strcpy(comm, "<...>");
1879 void trace_find_cmdline(int pid, char comm[])
1882 arch_spin_lock(&trace_cmdline_lock);
1884 __trace_find_cmdline(pid, comm);
1886 arch_spin_unlock(&trace_cmdline_lock);
1890 void tracing_record_cmdline(struct task_struct *tsk)
1892 if (atomic_read(&trace_record_cmdline_disabled) || !tracing_is_on())
1895 if (!__this_cpu_read(trace_cmdline_save))
1898 if (trace_save_cmdline(tsk))
1899 __this_cpu_write(trace_cmdline_save, false);
1903 tracing_generic_entry_update(struct trace_entry *entry, unsigned long flags,
1906 struct task_struct *tsk = current;
1908 entry->preempt_count = pc & 0xff;
1909 entry->pid = (tsk) ? tsk->pid : 0;
1911 #ifdef CONFIG_TRACE_IRQFLAGS_SUPPORT
1912 (irqs_disabled_flags(flags) ? TRACE_FLAG_IRQS_OFF : 0) |
1914 TRACE_FLAG_IRQS_NOSUPPORT |
1916 ((pc & NMI_MASK ) ? TRACE_FLAG_NMI : 0) |
1917 ((pc & HARDIRQ_MASK) ? TRACE_FLAG_HARDIRQ : 0) |
1918 ((pc & SOFTIRQ_OFFSET) ? TRACE_FLAG_SOFTIRQ : 0) |
1919 (tif_need_resched() ? TRACE_FLAG_NEED_RESCHED : 0) |
1920 (test_preempt_need_resched() ? TRACE_FLAG_PREEMPT_RESCHED : 0);
1922 EXPORT_SYMBOL_GPL(tracing_generic_entry_update);
1924 static __always_inline void
1925 trace_event_setup(struct ring_buffer_event *event,
1926 int type, unsigned long flags, int pc)
1928 struct trace_entry *ent = ring_buffer_event_data(event);
1930 tracing_generic_entry_update(ent, flags, pc);
1934 struct ring_buffer_event *
1935 trace_buffer_lock_reserve(struct ring_buffer *buffer,
1938 unsigned long flags, int pc)
1940 struct ring_buffer_event *event;
1942 event = ring_buffer_lock_reserve(buffer, len);
1944 trace_event_setup(event, type, flags, pc);
1949 DEFINE_PER_CPU(struct ring_buffer_event *, trace_buffered_event);
1950 DEFINE_PER_CPU(int, trace_buffered_event_cnt);
1951 static int trace_buffered_event_ref;
1954 * trace_buffered_event_enable - enable buffering events
1956 * When events are being filtered, it is quicker to use a temporary
1957 * buffer to write the event data into if there's a likely chance
1958 * that it will not be committed. The discard of the ring buffer
1959 * is not as fast as committing, and is much slower than copying
1962 * When an event is to be filtered, allocate per cpu buffers to
1963 * write the event data into, and if the event is filtered and discarded
1964 * it is simply dropped, otherwise, the entire data is to be committed
1967 void trace_buffered_event_enable(void)
1969 struct ring_buffer_event *event;
1973 WARN_ON_ONCE(!mutex_is_locked(&event_mutex));
1975 if (trace_buffered_event_ref++)
1978 for_each_tracing_cpu(cpu) {
1979 page = alloc_pages_node(cpu_to_node(cpu),
1980 GFP_KERNEL | __GFP_NORETRY, 0);
1984 event = page_address(page);
1985 memset(event, 0, sizeof(*event));
1987 per_cpu(trace_buffered_event, cpu) = event;
1990 if (cpu == smp_processor_id() &&
1991 this_cpu_read(trace_buffered_event) !=
1992 per_cpu(trace_buffered_event, cpu))
1999 trace_buffered_event_disable();
2002 static void enable_trace_buffered_event(void *data)
2004 /* Probably not needed, but do it anyway */
2006 this_cpu_dec(trace_buffered_event_cnt);
2009 static void disable_trace_buffered_event(void *data)
2011 this_cpu_inc(trace_buffered_event_cnt);
2015 * trace_buffered_event_disable - disable buffering events
2017 * When a filter is removed, it is faster to not use the buffered
2018 * events, and to commit directly into the ring buffer. Free up
2019 * the temp buffers when there are no more users. This requires
2020 * special synchronization with current events.
2022 void trace_buffered_event_disable(void)
2026 WARN_ON_ONCE(!mutex_is_locked(&event_mutex));
2028 if (WARN_ON_ONCE(!trace_buffered_event_ref))
2031 if (--trace_buffered_event_ref)
2035 /* For each CPU, set the buffer as used. */
2036 smp_call_function_many(tracing_buffer_mask,
2037 disable_trace_buffered_event, NULL, 1);
2040 /* Wait for all current users to finish */
2041 synchronize_sched();
2043 for_each_tracing_cpu(cpu) {
2044 free_page((unsigned long)per_cpu(trace_buffered_event, cpu));
2045 per_cpu(trace_buffered_event, cpu) = NULL;
2048 * Make sure trace_buffered_event is NULL before clearing
2049 * trace_buffered_event_cnt.
2054 /* Do the work on each cpu */
2055 smp_call_function_many(tracing_buffer_mask,
2056 enable_trace_buffered_event, NULL, 1);
2061 __buffer_unlock_commit(struct ring_buffer *buffer, struct ring_buffer_event *event)
2063 __this_cpu_write(trace_cmdline_save, true);
2065 /* If this is the temp buffer, we need to commit fully */
2066 if (this_cpu_read(trace_buffered_event) == event) {
2067 /* Length is in event->array[0] */
2068 ring_buffer_write(buffer, event->array[0], &event->array[1]);
2069 /* Release the temp buffer */
2070 this_cpu_dec(trace_buffered_event_cnt);
2072 ring_buffer_unlock_commit(buffer, event);
2075 static struct ring_buffer *temp_buffer;
2077 struct ring_buffer_event *
2078 trace_event_buffer_lock_reserve(struct ring_buffer **current_rb,
2079 struct trace_event_file *trace_file,
2080 int type, unsigned long len,
2081 unsigned long flags, int pc)
2083 struct ring_buffer_event *entry;
2086 *current_rb = trace_file->tr->trace_buffer.buffer;
2088 if ((trace_file->flags &
2089 (EVENT_FILE_FL_SOFT_DISABLED | EVENT_FILE_FL_FILTERED)) &&
2090 (entry = this_cpu_read(trace_buffered_event))) {
2091 /* Try to use the per cpu buffer first */
2092 val = this_cpu_inc_return(trace_buffered_event_cnt);
2094 trace_event_setup(entry, type, flags, pc);
2095 entry->array[0] = len;
2098 this_cpu_dec(trace_buffered_event_cnt);
2101 entry = trace_buffer_lock_reserve(*current_rb,
2102 type, len, flags, pc);
2104 * If tracing is off, but we have triggers enabled
2105 * we still need to look at the event data. Use the temp_buffer
2106 * to store the trace event for the tigger to use. It's recusive
2107 * safe and will not be recorded anywhere.
2109 if (!entry && trace_file->flags & EVENT_FILE_FL_TRIGGER_COND) {
2110 *current_rb = temp_buffer;
2111 entry = trace_buffer_lock_reserve(*current_rb,
2112 type, len, flags, pc);
2116 EXPORT_SYMBOL_GPL(trace_event_buffer_lock_reserve);
2118 void trace_buffer_unlock_commit_regs(struct trace_array *tr,
2119 struct ring_buffer *buffer,
2120 struct ring_buffer_event *event,
2121 unsigned long flags, int pc,
2122 struct pt_regs *regs)
2124 __buffer_unlock_commit(buffer, event);
2127 * If regs is not set, then skip the following callers:
2128 * trace_buffer_unlock_commit_regs
2129 * event_trigger_unlock_commit
2130 * trace_event_buffer_commit
2131 * trace_event_raw_event_sched_switch
2132 * Note, we can still get here via blktrace, wakeup tracer
2133 * and mmiotrace, but that's ok if they lose a function or
2134 * two. They are that meaningful.
2136 ftrace_trace_stack(tr, buffer, flags, regs ? 0 : 4, pc, regs);
2137 ftrace_trace_userstack(buffer, flags, pc);
2141 trace_function(struct trace_array *tr,
2142 unsigned long ip, unsigned long parent_ip, unsigned long flags,
2145 struct trace_event_call *call = &event_function;
2146 struct ring_buffer *buffer = tr->trace_buffer.buffer;
2147 struct ring_buffer_event *event;
2148 struct ftrace_entry *entry;
2150 event = trace_buffer_lock_reserve(buffer, TRACE_FN, sizeof(*entry),
2154 entry = ring_buffer_event_data(event);
2156 entry->parent_ip = parent_ip;
2158 if (!call_filter_check_discard(call, entry, buffer, event))
2159 __buffer_unlock_commit(buffer, event);
2162 #ifdef CONFIG_STACKTRACE
2164 #define FTRACE_STACK_MAX_ENTRIES (PAGE_SIZE / sizeof(unsigned long))
2165 struct ftrace_stack {
2166 unsigned long calls[FTRACE_STACK_MAX_ENTRIES];
2169 static DEFINE_PER_CPU(struct ftrace_stack, ftrace_stack);
2170 static DEFINE_PER_CPU(int, ftrace_stack_reserve);
2172 static void __ftrace_trace_stack(struct ring_buffer *buffer,
2173 unsigned long flags,
2174 int skip, int pc, struct pt_regs *regs)
2176 struct trace_event_call *call = &event_kernel_stack;
2177 struct ring_buffer_event *event;
2178 struct stack_entry *entry;
2179 struct stack_trace trace;
2181 int size = FTRACE_STACK_ENTRIES;
2183 trace.nr_entries = 0;
2187 * Add two, for this function and the call to save_stack_trace()
2188 * If regs is set, then these functions will not be in the way.
2194 * Since events can happen in NMIs there's no safe way to
2195 * use the per cpu ftrace_stacks. We reserve it and if an interrupt
2196 * or NMI comes in, it will just have to use the default
2197 * FTRACE_STACK_SIZE.
2199 preempt_disable_notrace();
2201 use_stack = __this_cpu_inc_return(ftrace_stack_reserve);
2203 * We don't need any atomic variables, just a barrier.
2204 * If an interrupt comes in, we don't care, because it would
2205 * have exited and put the counter back to what we want.
2206 * We just need a barrier to keep gcc from moving things
2210 if (use_stack == 1) {
2211 trace.entries = this_cpu_ptr(ftrace_stack.calls);
2212 trace.max_entries = FTRACE_STACK_MAX_ENTRIES;
2215 save_stack_trace_regs(regs, &trace);
2217 save_stack_trace(&trace);
2219 if (trace.nr_entries > size)
2220 size = trace.nr_entries;
2222 /* From now on, use_stack is a boolean */
2225 size *= sizeof(unsigned long);
2227 event = trace_buffer_lock_reserve(buffer, TRACE_STACK,
2228 sizeof(*entry) + size, flags, pc);
2231 entry = ring_buffer_event_data(event);
2233 memset(&entry->caller, 0, size);
2236 memcpy(&entry->caller, trace.entries,
2237 trace.nr_entries * sizeof(unsigned long));
2239 trace.max_entries = FTRACE_STACK_ENTRIES;
2240 trace.entries = entry->caller;
2242 save_stack_trace_regs(regs, &trace);
2244 save_stack_trace(&trace);
2247 entry->size = trace.nr_entries;
2249 if (!call_filter_check_discard(call, entry, buffer, event))
2250 __buffer_unlock_commit(buffer, event);
2253 /* Again, don't let gcc optimize things here */
2255 __this_cpu_dec(ftrace_stack_reserve);
2256 preempt_enable_notrace();
2260 static inline void ftrace_trace_stack(struct trace_array *tr,
2261 struct ring_buffer *buffer,
2262 unsigned long flags,
2263 int skip, int pc, struct pt_regs *regs)
2265 if (!(tr->trace_flags & TRACE_ITER_STACKTRACE))
2268 __ftrace_trace_stack(buffer, flags, skip, pc, regs);
2271 void __trace_stack(struct trace_array *tr, unsigned long flags, int skip,
2274 __ftrace_trace_stack(tr->trace_buffer.buffer, flags, skip, pc, NULL);
2278 * trace_dump_stack - record a stack back trace in the trace buffer
2279 * @skip: Number of functions to skip (helper handlers)
2281 void trace_dump_stack(int skip)
2283 unsigned long flags;
2285 if (tracing_disabled || tracing_selftest_running)
2288 local_save_flags(flags);
2291 * Skip 3 more, seems to get us at the caller of
2295 __ftrace_trace_stack(global_trace.trace_buffer.buffer,
2296 flags, skip, preempt_count(), NULL);
2299 static DEFINE_PER_CPU(int, user_stack_count);
2302 ftrace_trace_userstack(struct ring_buffer *buffer, unsigned long flags, int pc)
2304 struct trace_event_call *call = &event_user_stack;
2305 struct ring_buffer_event *event;
2306 struct userstack_entry *entry;
2307 struct stack_trace trace;
2309 if (!(global_trace.trace_flags & TRACE_ITER_USERSTACKTRACE))
2313 * NMIs can not handle page faults, even with fix ups.
2314 * The save user stack can (and often does) fault.
2316 if (unlikely(in_nmi()))
2320 * prevent recursion, since the user stack tracing may
2321 * trigger other kernel events.
2324 if (__this_cpu_read(user_stack_count))
2327 __this_cpu_inc(user_stack_count);
2329 event = trace_buffer_lock_reserve(buffer, TRACE_USER_STACK,
2330 sizeof(*entry), flags, pc);
2332 goto out_drop_count;
2333 entry = ring_buffer_event_data(event);
2335 entry->tgid = current->tgid;
2336 memset(&entry->caller, 0, sizeof(entry->caller));
2338 trace.nr_entries = 0;
2339 trace.max_entries = FTRACE_STACK_ENTRIES;
2341 trace.entries = entry->caller;
2343 save_stack_trace_user(&trace);
2344 if (!call_filter_check_discard(call, entry, buffer, event))
2345 __buffer_unlock_commit(buffer, event);
2348 __this_cpu_dec(user_stack_count);
2354 static void __trace_userstack(struct trace_array *tr, unsigned long flags)
2356 ftrace_trace_userstack(tr, flags, preempt_count());
2360 #endif /* CONFIG_STACKTRACE */
2362 /* created for use with alloc_percpu */
2363 struct trace_buffer_struct {
2365 char buffer[4][TRACE_BUF_SIZE];
2368 static struct trace_buffer_struct *trace_percpu_buffer;
2371 * Thise allows for lockless recording. If we're nested too deeply, then
2372 * this returns NULL.
2374 static char *get_trace_buf(void)
2376 struct trace_buffer_struct *buffer = this_cpu_ptr(trace_percpu_buffer);
2378 if (!buffer || buffer->nesting >= 4)
2383 /* Interrupts must see nesting incremented before we use the buffer */
2385 return &buffer->buffer[buffer->nesting][0];
2388 static void put_trace_buf(void)
2390 /* Don't let the decrement of nesting leak before this */
2392 this_cpu_dec(trace_percpu_buffer->nesting);
2395 static int alloc_percpu_trace_buffer(void)
2397 struct trace_buffer_struct *buffers;
2399 buffers = alloc_percpu(struct trace_buffer_struct);
2400 if (WARN(!buffers, "Could not allocate percpu trace_printk buffer"))
2403 trace_percpu_buffer = buffers;
2407 static int buffers_allocated;
2409 void trace_printk_init_buffers(void)
2411 if (buffers_allocated)
2414 if (alloc_percpu_trace_buffer())
2417 /* trace_printk() is for debug use only. Don't use it in production. */
2420 pr_warn("**********************************************************\n");
2421 pr_warn("** NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE **\n");
2423 pr_warn("** trace_printk() being used. Allocating extra memory. **\n");
2425 pr_warn("** This means that this is a DEBUG kernel and it is **\n");
2426 pr_warn("** unsafe for production use. **\n");
2428 pr_warn("** If you see this message and you are not debugging **\n");
2429 pr_warn("** the kernel, report this immediately to your vendor! **\n");
2431 pr_warn("** NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE NOTICE **\n");
2432 pr_warn("**********************************************************\n");
2434 /* Expand the buffers to set size */
2435 tracing_update_buffers();
2437 buffers_allocated = 1;
2440 * trace_printk_init_buffers() can be called by modules.
2441 * If that happens, then we need to start cmdline recording
2442 * directly here. If the global_trace.buffer is already
2443 * allocated here, then this was called by module code.
2445 if (global_trace.trace_buffer.buffer)
2446 tracing_start_cmdline_record();
2449 void trace_printk_start_comm(void)
2451 /* Start tracing comms if trace printk is set */
2452 if (!buffers_allocated)
2454 tracing_start_cmdline_record();
2457 static void trace_printk_start_stop_comm(int enabled)
2459 if (!buffers_allocated)
2463 tracing_start_cmdline_record();
2465 tracing_stop_cmdline_record();
2469 * trace_vbprintk - write binary msg to tracing buffer
2472 int trace_vbprintk(unsigned long ip, const char *fmt, va_list args)
2474 struct trace_event_call *call = &event_bprint;
2475 struct ring_buffer_event *event;
2476 struct ring_buffer *buffer;
2477 struct trace_array *tr = &global_trace;
2478 struct bprint_entry *entry;
2479 unsigned long flags;
2481 int len = 0, size, pc;
2483 if (unlikely(tracing_selftest_running || tracing_disabled))
2486 /* Don't pollute graph traces with trace_vprintk internals */
2487 pause_graph_tracing();
2489 pc = preempt_count();
2490 preempt_disable_notrace();
2492 tbuffer = get_trace_buf();
2498 len = vbin_printf((u32 *)tbuffer, TRACE_BUF_SIZE/sizeof(int), fmt, args);
2500 if (len > TRACE_BUF_SIZE/sizeof(int) || len < 0)
2503 local_save_flags(flags);
2504 size = sizeof(*entry) + sizeof(u32) * len;
2505 buffer = tr->trace_buffer.buffer;
2506 event = trace_buffer_lock_reserve(buffer, TRACE_BPRINT, size,
2510 entry = ring_buffer_event_data(event);
2514 memcpy(entry->buf, tbuffer, sizeof(u32) * len);
2515 if (!call_filter_check_discard(call, entry, buffer, event)) {
2516 __buffer_unlock_commit(buffer, event);
2517 ftrace_trace_stack(tr, buffer, flags, 6, pc, NULL);
2524 preempt_enable_notrace();
2525 unpause_graph_tracing();
2529 EXPORT_SYMBOL_GPL(trace_vbprintk);
2533 __trace_array_vprintk(struct ring_buffer *buffer,
2534 unsigned long ip, const char *fmt, va_list args)
2536 struct trace_event_call *call = &event_print;
2537 struct ring_buffer_event *event;
2538 int len = 0, size, pc;
2539 struct print_entry *entry;
2540 unsigned long flags;
2543 if (tracing_disabled || tracing_selftest_running)
2546 /* Don't pollute graph traces with trace_vprintk internals */
2547 pause_graph_tracing();
2549 pc = preempt_count();
2550 preempt_disable_notrace();
2553 tbuffer = get_trace_buf();
2559 len = vscnprintf(tbuffer, TRACE_BUF_SIZE, fmt, args);
2561 local_save_flags(flags);
2562 size = sizeof(*entry) + len + 1;
2563 event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
2567 entry = ring_buffer_event_data(event);
2570 memcpy(&entry->buf, tbuffer, len + 1);
2571 if (!call_filter_check_discard(call, entry, buffer, event)) {
2572 __buffer_unlock_commit(buffer, event);
2573 ftrace_trace_stack(&global_trace, buffer, flags, 6, pc, NULL);
2580 preempt_enable_notrace();
2581 unpause_graph_tracing();
2587 int trace_array_vprintk(struct trace_array *tr,
2588 unsigned long ip, const char *fmt, va_list args)
2590 return __trace_array_vprintk(tr->trace_buffer.buffer, ip, fmt, args);
2594 int trace_array_printk(struct trace_array *tr,
2595 unsigned long ip, const char *fmt, ...)
2600 if (!(global_trace.trace_flags & TRACE_ITER_PRINTK))
2604 ret = trace_array_vprintk(tr, ip, fmt, ap);
2610 int trace_array_printk_buf(struct ring_buffer *buffer,
2611 unsigned long ip, const char *fmt, ...)
2616 if (!(global_trace.trace_flags & TRACE_ITER_PRINTK))
2620 ret = __trace_array_vprintk(buffer, ip, fmt, ap);
2626 int trace_vprintk(unsigned long ip, const char *fmt, va_list args)
2628 return trace_array_vprintk(&global_trace, ip, fmt, args);
2630 EXPORT_SYMBOL_GPL(trace_vprintk);
2632 static void trace_iterator_increment(struct trace_iterator *iter)
2634 struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, iter->cpu);
2638 ring_buffer_read(buf_iter, NULL);
2641 static struct trace_entry *
2642 peek_next_entry(struct trace_iterator *iter, int cpu, u64 *ts,
2643 unsigned long *lost_events)
2645 struct ring_buffer_event *event;
2646 struct ring_buffer_iter *buf_iter = trace_buffer_iter(iter, cpu);
2649 event = ring_buffer_iter_peek(buf_iter, ts);
2651 event = ring_buffer_peek(iter->trace_buffer->buffer, cpu, ts,
2655 iter->ent_size = ring_buffer_event_length(event);
2656 return ring_buffer_event_data(event);
2662 static struct trace_entry *
2663 __find_next_entry(struct trace_iterator *iter, int *ent_cpu,
2664 unsigned long *missing_events, u64 *ent_ts)
2666 struct ring_buffer *buffer = iter->trace_buffer->buffer;
2667 struct trace_entry *ent, *next = NULL;
2668 unsigned long lost_events = 0, next_lost = 0;
2669 int cpu_file = iter->cpu_file;
2670 u64 next_ts = 0, ts;
2676 * If we are in a per_cpu trace file, don't bother by iterating over
2677 * all cpu and peek directly.
2679 if (cpu_file > RING_BUFFER_ALL_CPUS) {
2680 if (ring_buffer_empty_cpu(buffer, cpu_file))
2682 ent = peek_next_entry(iter, cpu_file, ent_ts, missing_events);
2684 *ent_cpu = cpu_file;
2689 for_each_tracing_cpu(cpu) {
2691 if (ring_buffer_empty_cpu(buffer, cpu))
2694 ent = peek_next_entry(iter, cpu, &ts, &lost_events);
2697 * Pick the entry with the smallest timestamp:
2699 if (ent && (!next || ts < next_ts)) {
2703 next_lost = lost_events;
2704 next_size = iter->ent_size;
2708 iter->ent_size = next_size;
2711 *ent_cpu = next_cpu;
2717 *missing_events = next_lost;
2722 /* Find the next real entry, without updating the iterator itself */
2723 struct trace_entry *trace_find_next_entry(struct trace_iterator *iter,
2724 int *ent_cpu, u64 *ent_ts)
2726 return __find_next_entry(iter, ent_cpu, NULL, ent_ts);
2729 /* Find the next real entry, and increment the iterator to the next entry */
2730 void *trace_find_next_entry_inc(struct trace_iterator *iter)
2732 iter->ent = __find_next_entry(iter, &iter->cpu,
2733 &iter->lost_events, &iter->ts);
2736 trace_iterator_increment(iter);
2738 return iter->ent ? iter : NULL;
2741 static void trace_consume(struct trace_iterator *iter)
2743 ring_buffer_consume(iter->trace_buffer->buffer, iter->cpu, &iter->ts,
2744 &iter->lost_events);
2747 static void *s_next(struct seq_file *m, void *v, loff_t *pos)
2749 struct trace_iterator *iter = m->private;
2753 WARN_ON_ONCE(iter->leftover);
2757 /* can't go backwards */
2762 ent = trace_find_next_entry_inc(iter);
2766 while (ent && iter->idx < i)
2767 ent = trace_find_next_entry_inc(iter);
2774 void tracing_iter_reset(struct trace_iterator *iter, int cpu)
2776 struct ring_buffer_event *event;
2777 struct ring_buffer_iter *buf_iter;
2778 unsigned long entries = 0;
2781 per_cpu_ptr(iter->trace_buffer->data, cpu)->skipped_entries = 0;
2783 buf_iter = trace_buffer_iter(iter, cpu);
2787 ring_buffer_iter_reset(buf_iter);
2790 * We could have the case with the max latency tracers
2791 * that a reset never took place on a cpu. This is evident
2792 * by the timestamp being before the start of the buffer.
2794 while ((event = ring_buffer_iter_peek(buf_iter, &ts))) {
2795 if (ts >= iter->trace_buffer->time_start)
2798 ring_buffer_read(buf_iter, NULL);
2801 per_cpu_ptr(iter->trace_buffer->data, cpu)->skipped_entries = entries;
2805 * The current tracer is copied to avoid a global locking
2808 static void *s_start(struct seq_file *m, loff_t *pos)
2810 struct trace_iterator *iter = m->private;
2811 struct trace_array *tr = iter->tr;
2812 int cpu_file = iter->cpu_file;
2818 * copy the tracer to avoid using a global lock all around.
2819 * iter->trace is a copy of current_trace, the pointer to the
2820 * name may be used instead of a strcmp(), as iter->trace->name
2821 * will point to the same string as current_trace->name.
2823 mutex_lock(&trace_types_lock);
2824 if (unlikely(tr->current_trace && iter->trace->name != tr->current_trace->name))
2825 *iter->trace = *tr->current_trace;
2826 mutex_unlock(&trace_types_lock);
2828 #ifdef CONFIG_TRACER_MAX_TRACE
2829 if (iter->snapshot && iter->trace->use_max_tr)
2830 return ERR_PTR(-EBUSY);
2833 if (!iter->snapshot)
2834 atomic_inc(&trace_record_cmdline_disabled);
2836 if (*pos != iter->pos) {
2841 if (cpu_file == RING_BUFFER_ALL_CPUS) {
2842 for_each_tracing_cpu(cpu)
2843 tracing_iter_reset(iter, cpu);
2845 tracing_iter_reset(iter, cpu_file);
2848 for (p = iter; p && l < *pos; p = s_next(m, p, &l))
2853 * If we overflowed the seq_file before, then we want
2854 * to just reuse the trace_seq buffer again.
2860 p = s_next(m, p, &l);
2864 trace_event_read_lock();
2865 trace_access_lock(cpu_file);
2869 static void s_stop(struct seq_file *m, void *p)
2871 struct trace_iterator *iter = m->private;
2873 #ifdef CONFIG_TRACER_MAX_TRACE
2874 if (iter->snapshot && iter->trace->use_max_tr)
2878 if (!iter->snapshot)
2879 atomic_dec(&trace_record_cmdline_disabled);
2881 trace_access_unlock(iter->cpu_file);
2882 trace_event_read_unlock();
2886 get_total_entries(struct trace_buffer *buf,
2887 unsigned long *total, unsigned long *entries)
2889 unsigned long count;
2895 for_each_tracing_cpu(cpu) {
2896 count = ring_buffer_entries_cpu(buf->buffer, cpu);
2898 * If this buffer has skipped entries, then we hold all
2899 * entries for the trace and we need to ignore the
2900 * ones before the time stamp.
2902 if (per_cpu_ptr(buf->data, cpu)->skipped_entries) {
2903 count -= per_cpu_ptr(buf->data, cpu)->skipped_entries;
2904 /* total is the same as the entries */
2908 ring_buffer_overrun_cpu(buf->buffer, cpu);
2913 static void print_lat_help_header(struct seq_file *m)
2915 seq_puts(m, "# _------=> CPU# \n"
2916 "# / _-----=> irqs-off \n"
2917 "# | / _----=> need-resched \n"
2918 "# || / _---=> hardirq/softirq \n"
2919 "# ||| / _--=> preempt-depth \n"
2921 "# cmd pid ||||| time | caller \n"
2922 "# \\ / ||||| \\ | / \n");
2925 static void print_event_info(struct trace_buffer *buf, struct seq_file *m)
2927 unsigned long total;
2928 unsigned long entries;
2930 get_total_entries(buf, &total, &entries);
2931 seq_printf(m, "# entries-in-buffer/entries-written: %lu/%lu #P:%d\n",
2932 entries, total, num_online_cpus());
2936 static void print_func_help_header(struct trace_buffer *buf, struct seq_file *m)
2938 print_event_info(buf, m);
2939 seq_puts(m, "# TASK-PID CPU# TIMESTAMP FUNCTION\n"
2943 static void print_func_help_header_irq(struct trace_buffer *buf, struct seq_file *m)
2945 print_event_info(buf, m);
2946 seq_puts(m, "# _-----=> irqs-off\n"
2947 "# / _----=> need-resched\n"
2948 "# | / _---=> hardirq/softirq\n"
2949 "# || / _--=> preempt-depth\n"
2951 "# TASK-PID CPU# |||| TIMESTAMP FUNCTION\n"
2952 "# | | | |||| | |\n");
2956 print_trace_header(struct seq_file *m, struct trace_iterator *iter)
2958 unsigned long sym_flags = (global_trace.trace_flags & TRACE_ITER_SYM_MASK);
2959 struct trace_buffer *buf = iter->trace_buffer;
2960 struct trace_array_cpu *data = per_cpu_ptr(buf->data, buf->cpu);
2961 struct tracer *type = iter->trace;
2962 unsigned long entries;
2963 unsigned long total;
2964 const char *name = "preemption";
2968 get_total_entries(buf, &total, &entries);
2970 seq_printf(m, "# %s latency trace v1.1.5 on %s\n",
2972 seq_puts(m, "# -----------------------------------"
2973 "---------------------------------\n");
2974 seq_printf(m, "# latency: %lu us, #%lu/%lu, CPU#%d |"
2975 " (M:%s VP:%d, KP:%d, SP:%d HP:%d",
2976 nsecs_to_usecs(data->saved_latency),
2980 #if defined(CONFIG_PREEMPT_NONE)
2982 #elif defined(CONFIG_PREEMPT_VOLUNTARY)
2984 #elif defined(CONFIG_PREEMPT)
2989 /* These are reserved for later use */
2992 seq_printf(m, " #P:%d)\n", num_online_cpus());
2996 seq_puts(m, "# -----------------\n");
2997 seq_printf(m, "# | task: %.16s-%d "
2998 "(uid:%d nice:%ld policy:%ld rt_prio:%ld)\n",
2999 data->comm, data->pid,
3000 from_kuid_munged(seq_user_ns(m), data->uid), data->nice,
3001 data->policy, data->rt_priority);
3002 seq_puts(m, "# -----------------\n");
3004 if (data->critical_start) {
3005 seq_puts(m, "# => started at: ");
3006 seq_print_ip_sym(&iter->seq, data->critical_start, sym_flags);
3007 trace_print_seq(m, &iter->seq);
3008 seq_puts(m, "\n# => ended at: ");
3009 seq_print_ip_sym(&iter->seq, data->critical_end, sym_flags);
3010 trace_print_seq(m, &iter->seq);
3011 seq_puts(m, "\n#\n");
3017 static void test_cpu_buff_start(struct trace_iterator *iter)
3019 struct trace_seq *s = &iter->seq;
3020 struct trace_array *tr = iter->tr;
3022 if (!(tr->trace_flags & TRACE_ITER_ANNOTATE))
3025 if (!(iter->iter_flags & TRACE_FILE_ANNOTATE))
3028 if (cpumask_available(iter->started) &&
3029 cpumask_test_cpu(iter->cpu, iter->started))
3032 if (per_cpu_ptr(iter->trace_buffer->data, iter->cpu)->skipped_entries)
3035 if (cpumask_available(iter->started))
3036 cpumask_set_cpu(iter->cpu, iter->started);
3038 /* Don't print started cpu buffer for the first entry of the trace */
3040 trace_seq_printf(s, "##### CPU %u buffer started ####\n",
3044 static enum print_line_t print_trace_fmt(struct trace_iterator *iter)
3046 struct trace_array *tr = iter->tr;
3047 struct trace_seq *s = &iter->seq;
3048 unsigned long sym_flags = (tr->trace_flags & TRACE_ITER_SYM_MASK);
3049 struct trace_entry *entry;
3050 struct trace_event *event;
3054 test_cpu_buff_start(iter);
3056 event = ftrace_find_event(entry->type);
3058 if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO) {
3059 if (iter->iter_flags & TRACE_FILE_LAT_FMT)
3060 trace_print_lat_context(iter);
3062 trace_print_context(iter);
3065 if (trace_seq_has_overflowed(s))
3066 return TRACE_TYPE_PARTIAL_LINE;
3069 return event->funcs->trace(iter, sym_flags, event);
3071 trace_seq_printf(s, "Unknown type %d\n", entry->type);
3073 return trace_handle_return(s);
3076 static enum print_line_t print_raw_fmt(struct trace_iterator *iter)
3078 struct trace_array *tr = iter->tr;
3079 struct trace_seq *s = &iter->seq;
3080 struct trace_entry *entry;
3081 struct trace_event *event;
3085 if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO)
3086 trace_seq_printf(s, "%d %d %llu ",
3087 entry->pid, iter->cpu, iter->ts);
3089 if (trace_seq_has_overflowed(s))
3090 return TRACE_TYPE_PARTIAL_LINE;
3092 event = ftrace_find_event(entry->type);
3094 return event->funcs->raw(iter, 0, event);
3096 trace_seq_printf(s, "%d ?\n", entry->type);
3098 return trace_handle_return(s);
3101 static enum print_line_t print_hex_fmt(struct trace_iterator *iter)
3103 struct trace_array *tr = iter->tr;
3104 struct trace_seq *s = &iter->seq;
3105 unsigned char newline = '\n';
3106 struct trace_entry *entry;
3107 struct trace_event *event;
3111 if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO) {
3112 SEQ_PUT_HEX_FIELD(s, entry->pid);
3113 SEQ_PUT_HEX_FIELD(s, iter->cpu);
3114 SEQ_PUT_HEX_FIELD(s, iter->ts);
3115 if (trace_seq_has_overflowed(s))
3116 return TRACE_TYPE_PARTIAL_LINE;
3119 event = ftrace_find_event(entry->type);
3121 enum print_line_t ret = event->funcs->hex(iter, 0, event);
3122 if (ret != TRACE_TYPE_HANDLED)
3126 SEQ_PUT_FIELD(s, newline);
3128 return trace_handle_return(s);
3131 static enum print_line_t print_bin_fmt(struct trace_iterator *iter)
3133 struct trace_array *tr = iter->tr;
3134 struct trace_seq *s = &iter->seq;
3135 struct trace_entry *entry;
3136 struct trace_event *event;
3140 if (tr->trace_flags & TRACE_ITER_CONTEXT_INFO) {
3141 SEQ_PUT_FIELD(s, entry->pid);
3142 SEQ_PUT_FIELD(s, iter->cpu);
3143 SEQ_PUT_FIELD(s, iter->ts);
3144 if (trace_seq_has_overflowed(s))
3145 return TRACE_TYPE_PARTIAL_LINE;
3148 event = ftrace_find_event(entry->type);
3149 return event ? event->funcs->binary(iter, 0, event) :
3153 int trace_empty(struct trace_iterator *iter)
3155 struct ring_buffer_iter *buf_iter;
3158 /* If we are looking at one CPU buffer, only check that one */
3159 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
3160 cpu = iter->cpu_file;
3161 buf_iter = trace_buffer_iter(iter, cpu);
3163 if (!ring_buffer_iter_empty(buf_iter))
3166 if (!ring_buffer_empty_cpu(iter->trace_buffer->buffer, cpu))
3172 for_each_tracing_cpu(cpu) {
3173 buf_iter = trace_buffer_iter(iter, cpu);
3175 if (!ring_buffer_iter_empty(buf_iter))
3178 if (!ring_buffer_empty_cpu(iter->trace_buffer->buffer, cpu))
3186 /* Called with trace_event_read_lock() held. */
3187 enum print_line_t print_trace_line(struct trace_iterator *iter)
3189 struct trace_array *tr = iter->tr;
3190 unsigned long trace_flags = tr->trace_flags;
3191 enum print_line_t ret;
3193 if (iter->lost_events) {
3194 trace_seq_printf(&iter->seq, "CPU:%d [LOST %lu EVENTS]\n",
3195 iter->cpu, iter->lost_events);
3196 if (trace_seq_has_overflowed(&iter->seq))
3197 return TRACE_TYPE_PARTIAL_LINE;
3200 if (iter->trace && iter->trace->print_line) {
3201 ret = iter->trace->print_line(iter);
3202 if (ret != TRACE_TYPE_UNHANDLED)
3206 if (iter->ent->type == TRACE_BPUTS &&
3207 trace_flags & TRACE_ITER_PRINTK &&
3208 trace_flags & TRACE_ITER_PRINTK_MSGONLY)
3209 return trace_print_bputs_msg_only(iter);
3211 if (iter->ent->type == TRACE_BPRINT &&
3212 trace_flags & TRACE_ITER_PRINTK &&
3213 trace_flags & TRACE_ITER_PRINTK_MSGONLY)
3214 return trace_print_bprintk_msg_only(iter);
3216 if (iter->ent->type == TRACE_PRINT &&
3217 trace_flags & TRACE_ITER_PRINTK &&
3218 trace_flags & TRACE_ITER_PRINTK_MSGONLY)
3219 return trace_print_printk_msg_only(iter);
3221 if (trace_flags & TRACE_ITER_BIN)
3222 return print_bin_fmt(iter);
3224 if (trace_flags & TRACE_ITER_HEX)
3225 return print_hex_fmt(iter);
3227 if (trace_flags & TRACE_ITER_RAW)
3228 return print_raw_fmt(iter);
3230 return print_trace_fmt(iter);
3233 void trace_latency_header(struct seq_file *m)
3235 struct trace_iterator *iter = m->private;
3236 struct trace_array *tr = iter->tr;
3238 /* print nothing if the buffers are empty */
3239 if (trace_empty(iter))
3242 if (iter->iter_flags & TRACE_FILE_LAT_FMT)
3243 print_trace_header(m, iter);
3245 if (!(tr->trace_flags & TRACE_ITER_VERBOSE))
3246 print_lat_help_header(m);
3249 void trace_default_header(struct seq_file *m)
3251 struct trace_iterator *iter = m->private;
3252 struct trace_array *tr = iter->tr;
3253 unsigned long trace_flags = tr->trace_flags;
3255 if (!(trace_flags & TRACE_ITER_CONTEXT_INFO))
3258 if (iter->iter_flags & TRACE_FILE_LAT_FMT) {
3259 /* print nothing if the buffers are empty */
3260 if (trace_empty(iter))
3262 print_trace_header(m, iter);
3263 if (!(trace_flags & TRACE_ITER_VERBOSE))
3264 print_lat_help_header(m);
3266 if (!(trace_flags & TRACE_ITER_VERBOSE)) {
3267 if (trace_flags & TRACE_ITER_IRQ_INFO)
3268 print_func_help_header_irq(iter->trace_buffer, m);
3270 print_func_help_header(iter->trace_buffer, m);
3275 static void test_ftrace_alive(struct seq_file *m)
3277 if (!ftrace_is_dead())
3279 seq_puts(m, "# WARNING: FUNCTION TRACING IS CORRUPTED\n"
3280 "# MAY BE MISSING FUNCTION EVENTS\n");
3283 #ifdef CONFIG_TRACER_MAX_TRACE
3284 static void show_snapshot_main_help(struct seq_file *m)
3286 seq_puts(m, "# echo 0 > snapshot : Clears and frees snapshot buffer\n"
3287 "# echo 1 > snapshot : Allocates snapshot buffer, if not already allocated.\n"
3288 "# Takes a snapshot of the main buffer.\n"
3289 "# echo 2 > snapshot : Clears snapshot buffer (but does not allocate or free)\n"
3290 "# (Doesn't have to be '2' works with any number that\n"
3291 "# is not a '0' or '1')\n");
3294 static void show_snapshot_percpu_help(struct seq_file *m)
3296 seq_puts(m, "# echo 0 > snapshot : Invalid for per_cpu snapshot file.\n");
3297 #ifdef CONFIG_RING_BUFFER_ALLOW_SWAP
3298 seq_puts(m, "# echo 1 > snapshot : Allocates snapshot buffer, if not already allocated.\n"
3299 "# Takes a snapshot of the main buffer for this cpu.\n");
3301 seq_puts(m, "# echo 1 > snapshot : Not supported with this kernel.\n"
3302 "# Must use main snapshot file to allocate.\n");
3304 seq_puts(m, "# echo 2 > snapshot : Clears this cpu's snapshot buffer (but does not allocate)\n"
3305 "# (Doesn't have to be '2' works with any number that\n"
3306 "# is not a '0' or '1')\n");
3309 static void print_snapshot_help(struct seq_file *m, struct trace_iterator *iter)
3311 if (iter->tr->allocated_snapshot)
3312 seq_puts(m, "#\n# * Snapshot is allocated *\n#\n");
3314 seq_puts(m, "#\n# * Snapshot is freed *\n#\n");
3316 seq_puts(m, "# Snapshot commands:\n");
3317 if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
3318 show_snapshot_main_help(m);
3320 show_snapshot_percpu_help(m);
3323 /* Should never be called */
3324 static inline void print_snapshot_help(struct seq_file *m, struct trace_iterator *iter) { }
3327 static int s_show(struct seq_file *m, void *v)
3329 struct trace_iterator *iter = v;
3332 if (iter->ent == NULL) {
3334 seq_printf(m, "# tracer: %s\n", iter->trace->name);
3336 test_ftrace_alive(m);
3338 if (iter->snapshot && trace_empty(iter))
3339 print_snapshot_help(m, iter);
3340 else if (iter->trace && iter->trace->print_header)
3341 iter->trace->print_header(m);
3343 trace_default_header(m);
3345 } else if (iter->leftover) {
3347 * If we filled the seq_file buffer earlier, we
3348 * want to just show it now.
3350 ret = trace_print_seq(m, &iter->seq);
3352 /* ret should this time be zero, but you never know */
3353 iter->leftover = ret;
3356 print_trace_line(iter);
3357 ret = trace_print_seq(m, &iter->seq);
3359 * If we overflow the seq_file buffer, then it will
3360 * ask us for this data again at start up.
3362 * ret is 0 if seq_file write succeeded.
3365 iter->leftover = ret;
3372 * Should be used after trace_array_get(), trace_types_lock
3373 * ensures that i_cdev was already initialized.
3375 static inline int tracing_get_cpu(struct inode *inode)
3377 if (inode->i_cdev) /* See trace_create_cpu_file() */
3378 return (long)inode->i_cdev - 1;
3379 return RING_BUFFER_ALL_CPUS;
3382 static const struct seq_operations tracer_seq_ops = {
3389 static struct trace_iterator *
3390 __tracing_open(struct inode *inode, struct file *file, bool snapshot)
3392 struct trace_array *tr = inode->i_private;
3393 struct trace_iterator *iter;
3396 if (tracing_disabled)
3397 return ERR_PTR(-ENODEV);
3399 iter = __seq_open_private(file, &tracer_seq_ops, sizeof(*iter));
3401 return ERR_PTR(-ENOMEM);
3403 iter->buffer_iter = kcalloc(nr_cpu_ids, sizeof(*iter->buffer_iter),
3405 if (!iter->buffer_iter)
3409 * We make a copy of the current tracer to avoid concurrent
3410 * changes on it while we are reading.
3412 mutex_lock(&trace_types_lock);
3413 iter->trace = kzalloc(sizeof(*iter->trace), GFP_KERNEL);
3417 *iter->trace = *tr->current_trace;
3419 if (!zalloc_cpumask_var(&iter->started, GFP_KERNEL))
3424 #ifdef CONFIG_TRACER_MAX_TRACE
3425 /* Currently only the top directory has a snapshot */
3426 if (tr->current_trace->print_max || snapshot)
3427 iter->trace_buffer = &tr->max_buffer;
3430 iter->trace_buffer = &tr->trace_buffer;
3431 iter->snapshot = snapshot;
3433 iter->cpu_file = tracing_get_cpu(inode);
3434 mutex_init(&iter->mutex);
3436 /* Notify the tracer early; before we stop tracing. */
3437 if (iter->trace && iter->trace->open)
3438 iter->trace->open(iter);
3440 /* Annotate start of buffers if we had overruns */
3441 if (ring_buffer_overruns(iter->trace_buffer->buffer))
3442 iter->iter_flags |= TRACE_FILE_ANNOTATE;
3444 /* Output in nanoseconds only if we are using a clock in nanoseconds. */
3445 if (trace_clocks[tr->clock_id].in_ns)
3446 iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
3448 /* stop the trace while dumping if we are not opening "snapshot" */
3449 if (!iter->snapshot)
3450 tracing_stop_tr(tr);
3452 if (iter->cpu_file == RING_BUFFER_ALL_CPUS) {
3453 for_each_tracing_cpu(cpu) {
3454 iter->buffer_iter[cpu] =
3455 ring_buffer_read_prepare(iter->trace_buffer->buffer,
3458 ring_buffer_read_prepare_sync();
3459 for_each_tracing_cpu(cpu) {
3460 ring_buffer_read_start(iter->buffer_iter[cpu]);
3461 tracing_iter_reset(iter, cpu);
3464 cpu = iter->cpu_file;
3465 iter->buffer_iter[cpu] =
3466 ring_buffer_read_prepare(iter->trace_buffer->buffer,
3468 ring_buffer_read_prepare_sync();
3469 ring_buffer_read_start(iter->buffer_iter[cpu]);
3470 tracing_iter_reset(iter, cpu);
3473 mutex_unlock(&trace_types_lock);
3478 mutex_unlock(&trace_types_lock);
3480 kfree(iter->buffer_iter);
3482 seq_release_private(inode, file);
3483 return ERR_PTR(-ENOMEM);
3486 int tracing_open_generic(struct inode *inode, struct file *filp)
3488 if (tracing_disabled)
3491 filp->private_data = inode->i_private;
3495 bool tracing_is_disabled(void)
3497 return (tracing_disabled) ? true: false;
3501 * Open and update trace_array ref count.
3502 * Must have the current trace_array passed to it.
3504 static int tracing_open_generic_tr(struct inode *inode, struct file *filp)
3506 struct trace_array *tr = inode->i_private;
3508 if (tracing_disabled)
3511 if (trace_array_get(tr) < 0)
3514 filp->private_data = inode->i_private;
3519 static int tracing_release(struct inode *inode, struct file *file)
3521 struct trace_array *tr = inode->i_private;
3522 struct seq_file *m = file->private_data;
3523 struct trace_iterator *iter;
3526 if (!(file->f_mode & FMODE_READ)) {
3527 trace_array_put(tr);
3531 /* Writes do not use seq_file */
3533 mutex_lock(&trace_types_lock);
3535 for_each_tracing_cpu(cpu) {
3536 if (iter->buffer_iter[cpu])
3537 ring_buffer_read_finish(iter->buffer_iter[cpu]);
3540 if (iter->trace && iter->trace->close)
3541 iter->trace->close(iter);
3543 if (!iter->snapshot)
3544 /* reenable tracing if it was previously enabled */
3545 tracing_start_tr(tr);
3547 __trace_array_put(tr);
3549 mutex_unlock(&trace_types_lock);
3551 mutex_destroy(&iter->mutex);
3552 free_cpumask_var(iter->started);
3554 kfree(iter->buffer_iter);
3555 seq_release_private(inode, file);
3560 static int tracing_release_generic_tr(struct inode *inode, struct file *file)
3562 struct trace_array *tr = inode->i_private;
3564 trace_array_put(tr);
3568 static int tracing_single_release_tr(struct inode *inode, struct file *file)
3570 struct trace_array *tr = inode->i_private;
3572 trace_array_put(tr);
3574 return single_release(inode, file);
3577 static int tracing_open(struct inode *inode, struct file *file)
3579 struct trace_array *tr = inode->i_private;
3580 struct trace_iterator *iter;
3583 if (trace_array_get(tr) < 0)
3586 /* If this file was open for write, then erase contents */
3587 if ((file->f_mode & FMODE_WRITE) && (file->f_flags & O_TRUNC)) {
3588 int cpu = tracing_get_cpu(inode);
3589 struct trace_buffer *trace_buf = &tr->trace_buffer;
3591 #ifdef CONFIG_TRACER_MAX_TRACE
3592 if (tr->current_trace->print_max)
3593 trace_buf = &tr->max_buffer;
3596 if (cpu == RING_BUFFER_ALL_CPUS)
3597 tracing_reset_online_cpus(trace_buf);
3599 tracing_reset(trace_buf, cpu);
3602 if (file->f_mode & FMODE_READ) {
3603 iter = __tracing_open(inode, file, false);
3605 ret = PTR_ERR(iter);
3606 else if (tr->trace_flags & TRACE_ITER_LATENCY_FMT)
3607 iter->iter_flags |= TRACE_FILE_LAT_FMT;
3611 trace_array_put(tr);
3617 * Some tracers are not suitable for instance buffers.
3618 * A tracer is always available for the global array (toplevel)
3619 * or if it explicitly states that it is.
3622 trace_ok_for_array(struct tracer *t, struct trace_array *tr)
3624 return (tr->flags & TRACE_ARRAY_FL_GLOBAL) || t->allow_instances;
3627 /* Find the next tracer that this trace array may use */
3628 static struct tracer *
3629 get_tracer_for_array(struct trace_array *tr, struct tracer *t)
3631 while (t && !trace_ok_for_array(t, tr))
3638 t_next(struct seq_file *m, void *v, loff_t *pos)
3640 struct trace_array *tr = m->private;
3641 struct tracer *t = v;
3646 t = get_tracer_for_array(tr, t->next);
3651 static void *t_start(struct seq_file *m, loff_t *pos)
3653 struct trace_array *tr = m->private;
3657 mutex_lock(&trace_types_lock);
3659 t = get_tracer_for_array(tr, trace_types);
3660 for (; t && l < *pos; t = t_next(m, t, &l))
3666 static void t_stop(struct seq_file *m, void *p)
3668 mutex_unlock(&trace_types_lock);
3671 static int t_show(struct seq_file *m, void *v)
3673 struct tracer *t = v;
3678 seq_puts(m, t->name);
3687 static const struct seq_operations show_traces_seq_ops = {
3694 static int show_traces_open(struct inode *inode, struct file *file)
3696 struct trace_array *tr = inode->i_private;
3700 if (tracing_disabled)
3703 ret = seq_open(file, &show_traces_seq_ops);
3707 m = file->private_data;
3714 tracing_write_stub(struct file *filp, const char __user *ubuf,
3715 size_t count, loff_t *ppos)
3720 loff_t tracing_lseek(struct file *file, loff_t offset, int whence)
3724 if (file->f_mode & FMODE_READ)
3725 ret = seq_lseek(file, offset, whence);
3727 file->f_pos = ret = 0;
3732 static const struct file_operations tracing_fops = {
3733 .open = tracing_open,
3735 .write = tracing_write_stub,
3736 .llseek = tracing_lseek,
3737 .release = tracing_release,
3740 static const struct file_operations show_traces_fops = {
3741 .open = show_traces_open,
3743 .release = seq_release,
3744 .llseek = seq_lseek,
3748 tracing_cpumask_read(struct file *filp, char __user *ubuf,
3749 size_t count, loff_t *ppos)
3751 struct trace_array *tr = file_inode(filp)->i_private;
3755 len = snprintf(NULL, 0, "%*pb\n",
3756 cpumask_pr_args(tr->tracing_cpumask)) + 1;
3757 mask_str = kmalloc(len, GFP_KERNEL);
3761 len = snprintf(mask_str, len, "%*pb\n",
3762 cpumask_pr_args(tr->tracing_cpumask));
3767 count = simple_read_from_buffer(ubuf, count, ppos, mask_str, len);
3776 tracing_cpumask_write(struct file *filp, const char __user *ubuf,
3777 size_t count, loff_t *ppos)
3779 struct trace_array *tr = file_inode(filp)->i_private;
3780 cpumask_var_t tracing_cpumask_new;
3783 if (!alloc_cpumask_var(&tracing_cpumask_new, GFP_KERNEL))
3786 err = cpumask_parse_user(ubuf, count, tracing_cpumask_new);
3790 local_irq_disable();
3791 arch_spin_lock(&tr->max_lock);
3792 for_each_tracing_cpu(cpu) {
3794 * Increase/decrease the disabled counter if we are
3795 * about to flip a bit in the cpumask:
3797 if (cpumask_test_cpu(cpu, tr->tracing_cpumask) &&
3798 !cpumask_test_cpu(cpu, tracing_cpumask_new)) {
3799 atomic_inc(&per_cpu_ptr(tr->trace_buffer.data, cpu)->disabled);
3800 ring_buffer_record_disable_cpu(tr->trace_buffer.buffer, cpu);
3802 if (!cpumask_test_cpu(cpu, tr->tracing_cpumask) &&
3803 cpumask_test_cpu(cpu, tracing_cpumask_new)) {
3804 atomic_dec(&per_cpu_ptr(tr->trace_buffer.data, cpu)->disabled);
3805 ring_buffer_record_enable_cpu(tr->trace_buffer.buffer, cpu);
3808 arch_spin_unlock(&tr->max_lock);
3811 cpumask_copy(tr->tracing_cpumask, tracing_cpumask_new);
3812 free_cpumask_var(tracing_cpumask_new);
3817 free_cpumask_var(tracing_cpumask_new);
3822 static const struct file_operations tracing_cpumask_fops = {
3823 .open = tracing_open_generic_tr,
3824 .read = tracing_cpumask_read,
3825 .write = tracing_cpumask_write,
3826 .release = tracing_release_generic_tr,
3827 .llseek = generic_file_llseek,
3830 static int tracing_trace_options_show(struct seq_file *m, void *v)
3832 struct tracer_opt *trace_opts;
3833 struct trace_array *tr = m->private;
3837 mutex_lock(&trace_types_lock);
3838 tracer_flags = tr->current_trace->flags->val;
3839 trace_opts = tr->current_trace->flags->opts;
3841 for (i = 0; trace_options[i]; i++) {
3842 if (tr->trace_flags & (1 << i))
3843 seq_printf(m, "%s\n", trace_options[i]);
3845 seq_printf(m, "no%s\n", trace_options[i]);
3848 for (i = 0; trace_opts[i].name; i++) {
3849 if (tracer_flags & trace_opts[i].bit)
3850 seq_printf(m, "%s\n", trace_opts[i].name);
3852 seq_printf(m, "no%s\n", trace_opts[i].name);
3854 mutex_unlock(&trace_types_lock);
3859 static int __set_tracer_option(struct trace_array *tr,
3860 struct tracer_flags *tracer_flags,
3861 struct tracer_opt *opts, int neg)
3863 struct tracer *trace = tracer_flags->trace;
3866 ret = trace->set_flag(tr, tracer_flags->val, opts->bit, !neg);
3871 tracer_flags->val &= ~opts->bit;
3873 tracer_flags->val |= opts->bit;
3877 /* Try to assign a tracer specific option */
3878 static int set_tracer_option(struct trace_array *tr, char *cmp, int neg)
3880 struct tracer *trace = tr->current_trace;
3881 struct tracer_flags *tracer_flags = trace->flags;
3882 struct tracer_opt *opts = NULL;
3885 for (i = 0; tracer_flags->opts[i].name; i++) {
3886 opts = &tracer_flags->opts[i];
3888 if (strcmp(cmp, opts->name) == 0)
3889 return __set_tracer_option(tr, trace->flags, opts, neg);
3895 /* Some tracers require overwrite to stay enabled */
3896 int trace_keep_overwrite(struct tracer *tracer, u32 mask, int set)
3898 if (tracer->enabled && (mask & TRACE_ITER_OVERWRITE) && !set)
3904 int set_tracer_flag(struct trace_array *tr, unsigned int mask, int enabled)
3906 /* do nothing if flag is already set */
3907 if (!!(tr->trace_flags & mask) == !!enabled)
3910 /* Give the tracer a chance to approve the change */
3911 if (tr->current_trace->flag_changed)
3912 if (tr->current_trace->flag_changed(tr, mask, !!enabled))
3916 tr->trace_flags |= mask;
3918 tr->trace_flags &= ~mask;
3920 if (mask == TRACE_ITER_RECORD_CMD)
3921 trace_event_enable_cmd_record(enabled);
3923 if (mask == TRACE_ITER_EVENT_FORK)
3924 trace_event_follow_fork(tr, enabled);
3926 if (mask == TRACE_ITER_OVERWRITE) {
3927 ring_buffer_change_overwrite(tr->trace_buffer.buffer, enabled);
3928 #ifdef CONFIG_TRACER_MAX_TRACE
3929 ring_buffer_change_overwrite(tr->max_buffer.buffer, enabled);
3933 if (mask == TRACE_ITER_PRINTK) {
3934 trace_printk_start_stop_comm(enabled);
3935 trace_printk_control(enabled);
3941 static int trace_set_options(struct trace_array *tr, char *option)
3947 size_t orig_len = strlen(option);
3949 cmp = strstrip(option);
3951 if (strncmp(cmp, "no", 2) == 0) {
3956 mutex_lock(&trace_types_lock);
3958 for (i = 0; trace_options[i]; i++) {
3959 if (strcmp(cmp, trace_options[i]) == 0) {
3960 ret = set_tracer_flag(tr, 1 << i, !neg);
3965 /* If no option could be set, test the specific tracer options */
3966 if (!trace_options[i])
3967 ret = set_tracer_option(tr, cmp, neg);
3969 mutex_unlock(&trace_types_lock);
3972 * If the first trailing whitespace is replaced with '\0' by strstrip,
3973 * turn it back into a space.
3975 if (orig_len > strlen(option))
3976 option[strlen(option)] = ' ';
3981 static void __init apply_trace_boot_options(void)
3983 char *buf = trace_boot_options_buf;
3987 option = strsep(&buf, ",");
3993 trace_set_options(&global_trace, option);
3995 /* Put back the comma to allow this to be called again */
4002 tracing_trace_options_write(struct file *filp, const char __user *ubuf,
4003 size_t cnt, loff_t *ppos)
4005 struct seq_file *m = filp->private_data;
4006 struct trace_array *tr = m->private;
4010 if (cnt >= sizeof(buf))
4013 if (copy_from_user(buf, ubuf, cnt))
4018 ret = trace_set_options(tr, buf);
4027 static int tracing_trace_options_open(struct inode *inode, struct file *file)
4029 struct trace_array *tr = inode->i_private;
4032 if (tracing_disabled)
4035 if (trace_array_get(tr) < 0)
4038 ret = single_open(file, tracing_trace_options_show, inode->i_private);
4040 trace_array_put(tr);
4045 static const struct file_operations tracing_iter_fops = {
4046 .open = tracing_trace_options_open,
4048 .llseek = seq_lseek,
4049 .release = tracing_single_release_tr,
4050 .write = tracing_trace_options_write,
4053 static const char readme_msg[] =
4054 "tracing mini-HOWTO:\n\n"
4055 "# echo 0 > tracing_on : quick way to disable tracing\n"
4056 "# echo 1 > tracing_on : quick way to re-enable tracing\n\n"
4057 " Important files:\n"
4058 " trace\t\t\t- The static contents of the buffer\n"
4059 "\t\t\t To clear the buffer write into this file: echo > trace\n"
4060 " trace_pipe\t\t- A consuming read to see the contents of the buffer\n"
4061 " current_tracer\t- function and latency tracers\n"
4062 " available_tracers\t- list of configured tracers for current_tracer\n"
4063 " buffer_size_kb\t- view and modify size of per cpu buffer\n"
4064 " buffer_total_size_kb - view total size of all cpu buffers\n\n"
4065 " trace_clock\t\t-change the clock used to order events\n"
4066 " local: Per cpu clock but may not be synced across CPUs\n"
4067 " global: Synced across CPUs but slows tracing down.\n"
4068 " counter: Not a clock, but just an increment\n"
4069 " uptime: Jiffy counter from time of boot\n"
4070 " perf: Same clock that perf events use\n"
4071 #ifdef CONFIG_X86_64
4072 " x86-tsc: TSC cycle counter\n"
4074 "\n trace_marker\t\t- Writes into this file writes into the kernel buffer\n"
4075 " tracing_cpumask\t- Limit which CPUs to trace\n"
4076 " instances\t\t- Make sub-buffers with: mkdir instances/foo\n"
4077 "\t\t\t Remove sub-buffer with rmdir\n"
4078 " trace_options\t\t- Set format or modify how tracing happens\n"
4079 "\t\t\t Disable an option by adding a suffix 'no' to the\n"
4080 "\t\t\t option name\n"
4081 " saved_cmdlines_size\t- echo command number in here to store comm-pid list\n"
4082 #ifdef CONFIG_DYNAMIC_FTRACE
4083 "\n available_filter_functions - list of functions that can be filtered on\n"
4084 " set_ftrace_filter\t- echo function name in here to only trace these\n"
4085 "\t\t\t functions\n"
4086 "\t accepts: func_full_name, *func_end, func_begin*, *func_middle*\n"
4087 "\t modules: Can select a group via module\n"
4088 "\t Format: :mod:<module-name>\n"
4089 "\t example: echo :mod:ext3 > set_ftrace_filter\n"
4090 "\t triggers: a command to perform when function is hit\n"
4091 "\t Format: <function>:<trigger>[:count]\n"
4092 "\t trigger: traceon, traceoff\n"
4093 "\t\t enable_event:<system>:<event>\n"
4094 "\t\t disable_event:<system>:<event>\n"
4095 #ifdef CONFIG_STACKTRACE
4098 #ifdef CONFIG_TRACER_SNAPSHOT
4103 "\t example: echo do_fault:traceoff > set_ftrace_filter\n"
4104 "\t echo do_trap:traceoff:3 > set_ftrace_filter\n"
4105 "\t The first one will disable tracing every time do_fault is hit\n"
4106 "\t The second will disable tracing at most 3 times when do_trap is hit\n"
4107 "\t The first time do trap is hit and it disables tracing, the\n"
4108 "\t counter will decrement to 2. If tracing is already disabled,\n"
4109 "\t the counter will not decrement. It only decrements when the\n"
4110 "\t trigger did work\n"
4111 "\t To remove trigger without count:\n"
4112 "\t echo '!<function>:<trigger> > set_ftrace_filter\n"
4113 "\t To remove trigger with a count:\n"
4114 "\t echo '!<function>:<trigger>:0 > set_ftrace_filter\n"
4115 " set_ftrace_notrace\t- echo function name in here to never trace.\n"
4116 "\t accepts: func_full_name, *func_end, func_begin*, *func_middle*\n"
4117 "\t modules: Can select a group via module command :mod:\n"
4118 "\t Does not accept triggers\n"
4119 #endif /* CONFIG_DYNAMIC_FTRACE */
4120 #ifdef CONFIG_FUNCTION_TRACER
4121 " set_ftrace_pid\t- Write pid(s) to only function trace those pids\n"
4124 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
4125 " set_graph_function\t- Trace the nested calls of a function (function_graph)\n"
4126 " set_graph_notrace\t- Do not trace the nested calls of a function (function_graph)\n"
4127 " max_graph_depth\t- Trace a limited depth of nested calls (0 is unlimited)\n"
4129 #ifdef CONFIG_TRACER_SNAPSHOT
4130 "\n snapshot\t\t- Like 'trace' but shows the content of the static\n"
4131 "\t\t\t snapshot buffer. Read the contents for more\n"
4132 "\t\t\t information\n"
4134 #ifdef CONFIG_STACK_TRACER
4135 " stack_trace\t\t- Shows the max stack trace when active\n"
4136 " stack_max_size\t- Shows current max stack size that was traced\n"
4137 "\t\t\t Write into this file to reset the max size (trigger a\n"
4138 "\t\t\t new trace)\n"
4139 #ifdef CONFIG_DYNAMIC_FTRACE
4140 " stack_trace_filter\t- Like set_ftrace_filter but limits what stack_trace\n"
4143 #endif /* CONFIG_STACK_TRACER */
4144 #ifdef CONFIG_KPROBE_EVENT
4145 " kprobe_events\t\t- Add/remove/show the kernel dynamic events\n"
4146 "\t\t\t Write into this file to define/undefine new trace events.\n"
4148 #ifdef CONFIG_UPROBE_EVENT
4149 " uprobe_events\t\t- Add/remove/show the userspace dynamic events\n"
4150 "\t\t\t Write into this file to define/undefine new trace events.\n"
4152 #if defined(CONFIG_KPROBE_EVENT) || defined(CONFIG_UPROBE_EVENT)
4153 "\t accepts: event-definitions (one definition per line)\n"
4154 "\t Format: p|r[:[<group>/]<event>] <place> [<args>]\n"
4155 "\t -:[<group>/]<event>\n"
4156 #ifdef CONFIG_KPROBE_EVENT
4157 "\t place: [<module>:]<symbol>[+<offset>]|<memaddr>\n"
4159 #ifdef CONFIG_UPROBE_EVENT
4160 "\t place: <path>:<offset>\n"
4162 "\t args: <name>=fetcharg[:type]\n"
4163 "\t fetcharg: %<register>, @<address>, @<symbol>[+|-<offset>],\n"
4164 "\t $stack<index>, $stack, $retval, $comm\n"
4165 "\t type: s8/16/32/64, u8/16/32/64, x8/16/32/64, string,\n"
4166 "\t b<bit-width>@<bit-offset>/<container-size>\n"
4168 " events/\t\t- Directory containing all trace event subsystems:\n"
4169 " enable\t\t- Write 0/1 to enable/disable tracing of all events\n"
4170 " events/<system>/\t- Directory containing all trace events for <system>:\n"
4171 " enable\t\t- Write 0/1 to enable/disable tracing of all <system>\n"
4173 " filter\t\t- If set, only events passing filter are traced\n"
4174 " events/<system>/<event>/\t- Directory containing control files for\n"
4176 " enable\t\t- Write 0/1 to enable/disable tracing of <event>\n"
4177 " filter\t\t- If set, only events passing filter are traced\n"
4178 " trigger\t\t- If set, a command to perform when event is hit\n"
4179 "\t Format: <trigger>[:count][if <filter>]\n"
4180 "\t trigger: traceon, traceoff\n"
4181 "\t enable_event:<system>:<event>\n"
4182 "\t disable_event:<system>:<event>\n"
4183 #ifdef CONFIG_HIST_TRIGGERS
4184 "\t enable_hist:<system>:<event>\n"
4185 "\t disable_hist:<system>:<event>\n"
4187 #ifdef CONFIG_STACKTRACE
4190 #ifdef CONFIG_TRACER_SNAPSHOT
4193 #ifdef CONFIG_HIST_TRIGGERS
4194 "\t\t hist (see below)\n"
4196 "\t example: echo traceoff > events/block/block_unplug/trigger\n"
4197 "\t echo traceoff:3 > events/block/block_unplug/trigger\n"
4198 "\t echo 'enable_event:kmem:kmalloc:3 if nr_rq > 1' > \\\n"
4199 "\t events/block/block_unplug/trigger\n"
4200 "\t The first disables tracing every time block_unplug is hit.\n"
4201 "\t The second disables tracing the first 3 times block_unplug is hit.\n"
4202 "\t The third enables the kmalloc event the first 3 times block_unplug\n"
4203 "\t is hit and has value of greater than 1 for the 'nr_rq' event field.\n"
4204 "\t Like function triggers, the counter is only decremented if it\n"
4205 "\t enabled or disabled tracing.\n"
4206 "\t To remove a trigger without a count:\n"
4207 "\t echo '!<trigger> > <system>/<event>/trigger\n"
4208 "\t To remove a trigger with a count:\n"
4209 "\t echo '!<trigger>:0 > <system>/<event>/trigger\n"
4210 "\t Filters can be ignored when removing a trigger.\n"
4211 #ifdef CONFIG_HIST_TRIGGERS
4212 " hist trigger\t- If set, event hits are aggregated into a hash table\n"
4213 "\t Format: hist:keys=<field1[,field2,...]>\n"
4214 "\t [:values=<field1[,field2,...]>]\n"
4215 "\t [:sort=<field1[,field2,...]>]\n"
4216 "\t [:size=#entries]\n"
4217 "\t [:pause][:continue][:clear]\n"
4218 "\t [:name=histname1]\n"
4219 "\t [if <filter>]\n\n"
4220 "\t When a matching event is hit, an entry is added to a hash\n"
4221 "\t table using the key(s) and value(s) named, and the value of a\n"
4222 "\t sum called 'hitcount' is incremented. Keys and values\n"
4223 "\t correspond to fields in the event's format description. Keys\n"
4224 "\t can be any field, or the special string 'stacktrace'.\n"
4225 "\t Compound keys consisting of up to two fields can be specified\n"
4226 "\t by the 'keys' keyword. Values must correspond to numeric\n"
4227 "\t fields. Sort keys consisting of up to two fields can be\n"
4228 "\t specified using the 'sort' keyword. The sort direction can\n"
4229 "\t be modified by appending '.descending' or '.ascending' to a\n"
4230 "\t sort field. The 'size' parameter can be used to specify more\n"
4231 "\t or fewer than the default 2048 entries for the hashtable size.\n"
4232 "\t If a hist trigger is given a name using the 'name' parameter,\n"
4233 "\t its histogram data will be shared with other triggers of the\n"
4234 "\t same name, and trigger hits will update this common data.\n\n"
4235 "\t Reading the 'hist' file for the event will dump the hash\n"
4236 "\t table in its entirety to stdout. If there are multiple hist\n"
4237 "\t triggers attached to an event, there will be a table for each\n"
4238 "\t trigger in the output. The table displayed for a named\n"
4239 "\t trigger will be the same as any other instance having the\n"
4240 "\t same name. The default format used to display a given field\n"
4241 "\t can be modified by appending any of the following modifiers\n"
4242 "\t to the field name, as applicable:\n\n"
4243 "\t .hex display a number as a hex value\n"
4244 "\t .sym display an address as a symbol\n"
4245 "\t .sym-offset display an address as a symbol and offset\n"
4246 "\t .execname display a common_pid as a program name\n"
4247 "\t .syscall display a syscall id as a syscall name\n\n"
4248 "\t .log2 display log2 value rather than raw number\n\n"
4249 "\t The 'pause' parameter can be used to pause an existing hist\n"
4250 "\t trigger or to start a hist trigger but not log any events\n"
4251 "\t until told to do so. 'continue' can be used to start or\n"
4252 "\t restart a paused hist trigger.\n\n"
4253 "\t The 'clear' parameter will clear the contents of a running\n"
4254 "\t hist trigger and leave its current paused/active state\n"
4256 "\t The enable_hist and disable_hist triggers can be used to\n"
4257 "\t have one event conditionally start and stop another event's\n"
4258 "\t already-attached hist trigger. The syntax is analagous to\n"
4259 "\t the enable_event and disable_event triggers.\n"
4264 tracing_readme_read(struct file *filp, char __user *ubuf,
4265 size_t cnt, loff_t *ppos)
4267 return simple_read_from_buffer(ubuf, cnt, ppos,
4268 readme_msg, strlen(readme_msg));
4271 static const struct file_operations tracing_readme_fops = {
4272 .open = tracing_open_generic,
4273 .read = tracing_readme_read,
4274 .llseek = generic_file_llseek,
4277 static void *saved_cmdlines_next(struct seq_file *m, void *v, loff_t *pos)
4279 unsigned int *ptr = v;
4281 if (*pos || m->count)
4286 for (; ptr < &savedcmd->map_cmdline_to_pid[savedcmd->cmdline_num];
4288 if (*ptr == -1 || *ptr == NO_CMDLINE_MAP)
4297 static void *saved_cmdlines_start(struct seq_file *m, loff_t *pos)
4303 arch_spin_lock(&trace_cmdline_lock);
4305 v = &savedcmd->map_cmdline_to_pid[0];
4307 v = saved_cmdlines_next(m, v, &l);
4315 static void saved_cmdlines_stop(struct seq_file *m, void *v)
4317 arch_spin_unlock(&trace_cmdline_lock);
4321 static int saved_cmdlines_show(struct seq_file *m, void *v)
4323 char buf[TASK_COMM_LEN];
4324 unsigned int *pid = v;
4326 __trace_find_cmdline(*pid, buf);
4327 seq_printf(m, "%d %s\n", *pid, buf);
4331 static const struct seq_operations tracing_saved_cmdlines_seq_ops = {
4332 .start = saved_cmdlines_start,
4333 .next = saved_cmdlines_next,
4334 .stop = saved_cmdlines_stop,
4335 .show = saved_cmdlines_show,
4338 static int tracing_saved_cmdlines_open(struct inode *inode, struct file *filp)
4340 if (tracing_disabled)
4343 return seq_open(filp, &tracing_saved_cmdlines_seq_ops);
4346 static const struct file_operations tracing_saved_cmdlines_fops = {
4347 .open = tracing_saved_cmdlines_open,
4349 .llseek = seq_lseek,
4350 .release = seq_release,
4354 tracing_saved_cmdlines_size_read(struct file *filp, char __user *ubuf,
4355 size_t cnt, loff_t *ppos)
4360 arch_spin_lock(&trace_cmdline_lock);
4361 r = scnprintf(buf, sizeof(buf), "%u\n", savedcmd->cmdline_num);
4362 arch_spin_unlock(&trace_cmdline_lock);
4364 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
4367 static void free_saved_cmdlines_buffer(struct saved_cmdlines_buffer *s)
4369 kfree(s->saved_cmdlines);
4370 kfree(s->map_cmdline_to_pid);
4374 static int tracing_resize_saved_cmdlines(unsigned int val)
4376 struct saved_cmdlines_buffer *s, *savedcmd_temp;
4378 s = kmalloc(sizeof(*s), GFP_KERNEL);
4382 if (allocate_cmdlines_buffer(val, s) < 0) {
4387 arch_spin_lock(&trace_cmdline_lock);
4388 savedcmd_temp = savedcmd;
4390 arch_spin_unlock(&trace_cmdline_lock);
4391 free_saved_cmdlines_buffer(savedcmd_temp);
4397 tracing_saved_cmdlines_size_write(struct file *filp, const char __user *ubuf,
4398 size_t cnt, loff_t *ppos)
4403 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
4407 /* must have at least 1 entry or less than PID_MAX_DEFAULT */
4408 if (!val || val > PID_MAX_DEFAULT)
4411 ret = tracing_resize_saved_cmdlines((unsigned int)val);
4420 static const struct file_operations tracing_saved_cmdlines_size_fops = {
4421 .open = tracing_open_generic,
4422 .read = tracing_saved_cmdlines_size_read,
4423 .write = tracing_saved_cmdlines_size_write,
4426 #ifdef CONFIG_TRACE_ENUM_MAP_FILE
4427 static union trace_enum_map_item *
4428 update_enum_map(union trace_enum_map_item *ptr)
4430 if (!ptr->map.enum_string) {
4431 if (ptr->tail.next) {
4432 ptr = ptr->tail.next;
4433 /* Set ptr to the next real item (skip head) */
4441 static void *enum_map_next(struct seq_file *m, void *v, loff_t *pos)
4443 union trace_enum_map_item *ptr = v;
4446 * Paranoid! If ptr points to end, we don't want to increment past it.
4447 * This really should never happen.
4449 ptr = update_enum_map(ptr);
4450 if (WARN_ON_ONCE(!ptr))
4457 ptr = update_enum_map(ptr);
4462 static void *enum_map_start(struct seq_file *m, loff_t *pos)
4464 union trace_enum_map_item *v;
4467 mutex_lock(&trace_enum_mutex);
4469 v = trace_enum_maps;
4473 while (v && l < *pos) {
4474 v = enum_map_next(m, v, &l);
4480 static void enum_map_stop(struct seq_file *m, void *v)
4482 mutex_unlock(&trace_enum_mutex);
4485 static int enum_map_show(struct seq_file *m, void *v)
4487 union trace_enum_map_item *ptr = v;
4489 seq_printf(m, "%s %ld (%s)\n",
4490 ptr->map.enum_string, ptr->map.enum_value,
4496 static const struct seq_operations tracing_enum_map_seq_ops = {
4497 .start = enum_map_start,
4498 .next = enum_map_next,
4499 .stop = enum_map_stop,
4500 .show = enum_map_show,
4503 static int tracing_enum_map_open(struct inode *inode, struct file *filp)
4505 if (tracing_disabled)
4508 return seq_open(filp, &tracing_enum_map_seq_ops);
4511 static const struct file_operations tracing_enum_map_fops = {
4512 .open = tracing_enum_map_open,
4514 .llseek = seq_lseek,
4515 .release = seq_release,
4518 static inline union trace_enum_map_item *
4519 trace_enum_jmp_to_tail(union trace_enum_map_item *ptr)
4521 /* Return tail of array given the head */
4522 return ptr + ptr->head.length + 1;
4526 trace_insert_enum_map_file(struct module *mod, struct trace_enum_map **start,
4529 struct trace_enum_map **stop;
4530 struct trace_enum_map **map;
4531 union trace_enum_map_item *map_array;
4532 union trace_enum_map_item *ptr;
4537 * The trace_enum_maps contains the map plus a head and tail item,
4538 * where the head holds the module and length of array, and the
4539 * tail holds a pointer to the next list.
4541 map_array = kmalloc(sizeof(*map_array) * (len + 2), GFP_KERNEL);
4543 pr_warn("Unable to allocate trace enum mapping\n");
4547 mutex_lock(&trace_enum_mutex);
4549 if (!trace_enum_maps)
4550 trace_enum_maps = map_array;
4552 ptr = trace_enum_maps;
4554 ptr = trace_enum_jmp_to_tail(ptr);
4555 if (!ptr->tail.next)
4557 ptr = ptr->tail.next;
4560 ptr->tail.next = map_array;
4562 map_array->head.mod = mod;
4563 map_array->head.length = len;
4566 for (map = start; (unsigned long)map < (unsigned long)stop; map++) {
4567 map_array->map = **map;
4570 memset(map_array, 0, sizeof(*map_array));
4572 mutex_unlock(&trace_enum_mutex);
4575 static void trace_create_enum_file(struct dentry *d_tracer)
4577 trace_create_file("enum_map", 0444, d_tracer,
4578 NULL, &tracing_enum_map_fops);
4581 #else /* CONFIG_TRACE_ENUM_MAP_FILE */
4582 static inline void trace_create_enum_file(struct dentry *d_tracer) { }
4583 static inline void trace_insert_enum_map_file(struct module *mod,
4584 struct trace_enum_map **start, int len) { }
4585 #endif /* !CONFIG_TRACE_ENUM_MAP_FILE */
4587 static void trace_insert_enum_map(struct module *mod,
4588 struct trace_enum_map **start, int len)
4590 struct trace_enum_map **map;
4597 trace_event_enum_update(map, len);
4599 trace_insert_enum_map_file(mod, start, len);
4603 tracing_set_trace_read(struct file *filp, char __user *ubuf,
4604 size_t cnt, loff_t *ppos)
4606 struct trace_array *tr = filp->private_data;
4607 char buf[MAX_TRACER_SIZE+2];
4610 mutex_lock(&trace_types_lock);
4611 r = sprintf(buf, "%s\n", tr->current_trace->name);
4612 mutex_unlock(&trace_types_lock);
4614 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
4617 int tracer_init(struct tracer *t, struct trace_array *tr)
4619 tracing_reset_online_cpus(&tr->trace_buffer);
4623 static void set_buffer_entries(struct trace_buffer *buf, unsigned long val)
4627 for_each_tracing_cpu(cpu)
4628 per_cpu_ptr(buf->data, cpu)->entries = val;
4631 #ifdef CONFIG_TRACER_MAX_TRACE
4632 /* resize @tr's buffer to the size of @size_tr's entries */
4633 static int resize_buffer_duplicate_size(struct trace_buffer *trace_buf,
4634 struct trace_buffer *size_buf, int cpu_id)
4638 if (cpu_id == RING_BUFFER_ALL_CPUS) {
4639 for_each_tracing_cpu(cpu) {
4640 ret = ring_buffer_resize(trace_buf->buffer,
4641 per_cpu_ptr(size_buf->data, cpu)->entries, cpu);
4644 per_cpu_ptr(trace_buf->data, cpu)->entries =
4645 per_cpu_ptr(size_buf->data, cpu)->entries;
4648 ret = ring_buffer_resize(trace_buf->buffer,
4649 per_cpu_ptr(size_buf->data, cpu_id)->entries, cpu_id);
4651 per_cpu_ptr(trace_buf->data, cpu_id)->entries =
4652 per_cpu_ptr(size_buf->data, cpu_id)->entries;
4657 #endif /* CONFIG_TRACER_MAX_TRACE */
4659 static int __tracing_resize_ring_buffer(struct trace_array *tr,
4660 unsigned long size, int cpu)
4665 * If kernel or user changes the size of the ring buffer
4666 * we use the size that was given, and we can forget about
4667 * expanding it later.
4669 ring_buffer_expanded = true;
4671 /* May be called before buffers are initialized */
4672 if (!tr->trace_buffer.buffer)
4675 ret = ring_buffer_resize(tr->trace_buffer.buffer, size, cpu);
4679 #ifdef CONFIG_TRACER_MAX_TRACE
4680 if (!(tr->flags & TRACE_ARRAY_FL_GLOBAL) ||
4681 !tr->current_trace->use_max_tr)
4684 ret = ring_buffer_resize(tr->max_buffer.buffer, size, cpu);
4686 int r = resize_buffer_duplicate_size(&tr->trace_buffer,
4687 &tr->trace_buffer, cpu);
4690 * AARGH! We are left with different
4691 * size max buffer!!!!
4692 * The max buffer is our "snapshot" buffer.
4693 * When a tracer needs a snapshot (one of the
4694 * latency tracers), it swaps the max buffer
4695 * with the saved snap shot. We succeeded to
4696 * update the size of the main buffer, but failed to
4697 * update the size of the max buffer. But when we tried
4698 * to reset the main buffer to the original size, we
4699 * failed there too. This is very unlikely to
4700 * happen, but if it does, warn and kill all
4704 tracing_disabled = 1;
4709 if (cpu == RING_BUFFER_ALL_CPUS)
4710 set_buffer_entries(&tr->max_buffer, size);
4712 per_cpu_ptr(tr->max_buffer.data, cpu)->entries = size;
4715 #endif /* CONFIG_TRACER_MAX_TRACE */
4717 if (cpu == RING_BUFFER_ALL_CPUS)
4718 set_buffer_entries(&tr->trace_buffer, size);
4720 per_cpu_ptr(tr->trace_buffer.data, cpu)->entries = size;
4725 static ssize_t tracing_resize_ring_buffer(struct trace_array *tr,
4726 unsigned long size, int cpu_id)
4730 mutex_lock(&trace_types_lock);
4732 if (cpu_id != RING_BUFFER_ALL_CPUS) {
4733 /* make sure, this cpu is enabled in the mask */
4734 if (!cpumask_test_cpu(cpu_id, tracing_buffer_mask)) {
4740 ret = __tracing_resize_ring_buffer(tr, size, cpu_id);
4745 mutex_unlock(&trace_types_lock);
4752 * tracing_update_buffers - used by tracing facility to expand ring buffers
4754 * To save on memory when the tracing is never used on a system with it
4755 * configured in. The ring buffers are set to a minimum size. But once
4756 * a user starts to use the tracing facility, then they need to grow
4757 * to their default size.
4759 * This function is to be called when a tracer is about to be used.
4761 int tracing_update_buffers(void)
4765 mutex_lock(&trace_types_lock);
4766 if (!ring_buffer_expanded)
4767 ret = __tracing_resize_ring_buffer(&global_trace, trace_buf_size,
4768 RING_BUFFER_ALL_CPUS);
4769 mutex_unlock(&trace_types_lock);
4774 struct trace_option_dentry;
4777 create_trace_option_files(struct trace_array *tr, struct tracer *tracer);
4780 * Used to clear out the tracer before deletion of an instance.
4781 * Must have trace_types_lock held.
4783 static void tracing_set_nop(struct trace_array *tr)
4785 if (tr->current_trace == &nop_trace)
4788 tr->current_trace->enabled--;
4790 if (tr->current_trace->reset)
4791 tr->current_trace->reset(tr);
4793 tr->current_trace = &nop_trace;
4796 static void add_tracer_options(struct trace_array *tr, struct tracer *t)
4798 /* Only enable if the directory has been created already. */
4802 create_trace_option_files(tr, t);
4805 static int tracing_set_tracer(struct trace_array *tr, const char *buf)
4808 #ifdef CONFIG_TRACER_MAX_TRACE
4813 mutex_lock(&trace_types_lock);
4815 if (!ring_buffer_expanded) {
4816 ret = __tracing_resize_ring_buffer(tr, trace_buf_size,
4817 RING_BUFFER_ALL_CPUS);
4823 for (t = trace_types; t; t = t->next) {
4824 if (strcmp(t->name, buf) == 0)
4831 if (t == tr->current_trace)
4834 /* Some tracers are only allowed for the top level buffer */
4835 if (!trace_ok_for_array(t, tr)) {
4840 /* If trace pipe files are being read, we can't change the tracer */
4841 if (tr->current_trace->ref) {
4846 trace_branch_disable();
4848 tr->current_trace->enabled--;
4850 if (tr->current_trace->reset)
4851 tr->current_trace->reset(tr);
4853 /* Current trace needs to be nop_trace before synchronize_sched */
4854 tr->current_trace = &nop_trace;
4856 #ifdef CONFIG_TRACER_MAX_TRACE
4857 had_max_tr = tr->allocated_snapshot;
4859 if (had_max_tr && !t->use_max_tr) {
4861 * We need to make sure that the update_max_tr sees that
4862 * current_trace changed to nop_trace to keep it from
4863 * swapping the buffers after we resize it.
4864 * The update_max_tr is called from interrupts disabled
4865 * so a synchronized_sched() is sufficient.
4867 synchronize_sched();
4872 #ifdef CONFIG_TRACER_MAX_TRACE
4873 if (t->use_max_tr && !had_max_tr) {
4874 ret = alloc_snapshot(tr);
4881 ret = tracer_init(t, tr);
4886 tr->current_trace = t;
4887 tr->current_trace->enabled++;
4888 trace_branch_enable(tr);
4890 mutex_unlock(&trace_types_lock);
4896 tracing_set_trace_write(struct file *filp, const char __user *ubuf,
4897 size_t cnt, loff_t *ppos)
4899 struct trace_array *tr = filp->private_data;
4900 char buf[MAX_TRACER_SIZE+1];
4907 if (cnt > MAX_TRACER_SIZE)
4908 cnt = MAX_TRACER_SIZE;
4910 if (copy_from_user(buf, ubuf, cnt))
4915 /* strip ending whitespace. */
4916 for (i = cnt - 1; i > 0 && isspace(buf[i]); i--)
4919 err = tracing_set_tracer(tr, buf);
4929 tracing_nsecs_read(unsigned long *ptr, char __user *ubuf,
4930 size_t cnt, loff_t *ppos)
4935 r = snprintf(buf, sizeof(buf), "%ld\n",
4936 *ptr == (unsigned long)-1 ? -1 : nsecs_to_usecs(*ptr));
4937 if (r > sizeof(buf))
4939 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
4943 tracing_nsecs_write(unsigned long *ptr, const char __user *ubuf,
4944 size_t cnt, loff_t *ppos)
4949 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
4959 tracing_thresh_read(struct file *filp, char __user *ubuf,
4960 size_t cnt, loff_t *ppos)
4962 return tracing_nsecs_read(&tracing_thresh, ubuf, cnt, ppos);
4966 tracing_thresh_write(struct file *filp, const char __user *ubuf,
4967 size_t cnt, loff_t *ppos)
4969 struct trace_array *tr = filp->private_data;
4972 mutex_lock(&trace_types_lock);
4973 ret = tracing_nsecs_write(&tracing_thresh, ubuf, cnt, ppos);
4977 if (tr->current_trace->update_thresh) {
4978 ret = tr->current_trace->update_thresh(tr);
4985 mutex_unlock(&trace_types_lock);
4990 #if defined(CONFIG_TRACER_MAX_TRACE) || defined(CONFIG_HWLAT_TRACER)
4993 tracing_max_lat_read(struct file *filp, char __user *ubuf,
4994 size_t cnt, loff_t *ppos)
4996 return tracing_nsecs_read(filp->private_data, ubuf, cnt, ppos);
5000 tracing_max_lat_write(struct file *filp, const char __user *ubuf,
5001 size_t cnt, loff_t *ppos)
5003 return tracing_nsecs_write(filp->private_data, ubuf, cnt, ppos);
5008 static int tracing_open_pipe(struct inode *inode, struct file *filp)
5010 struct trace_array *tr = inode->i_private;
5011 struct trace_iterator *iter;
5014 if (tracing_disabled)
5017 if (trace_array_get(tr) < 0)
5020 mutex_lock(&trace_types_lock);
5022 /* create a buffer to store the information to pass to userspace */
5023 iter = kzalloc(sizeof(*iter), GFP_KERNEL);
5026 __trace_array_put(tr);
5030 trace_seq_init(&iter->seq);
5031 iter->trace = tr->current_trace;
5033 if (!alloc_cpumask_var(&iter->started, GFP_KERNEL)) {
5038 /* trace pipe does not show start of buffer */
5039 cpumask_setall(iter->started);
5041 if (tr->trace_flags & TRACE_ITER_LATENCY_FMT)
5042 iter->iter_flags |= TRACE_FILE_LAT_FMT;
5044 /* Output in nanoseconds only if we are using a clock in nanoseconds. */
5045 if (trace_clocks[tr->clock_id].in_ns)
5046 iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
5049 iter->trace_buffer = &tr->trace_buffer;
5050 iter->cpu_file = tracing_get_cpu(inode);
5051 mutex_init(&iter->mutex);
5052 filp->private_data = iter;
5054 if (iter->trace->pipe_open)
5055 iter->trace->pipe_open(iter);
5057 nonseekable_open(inode, filp);
5059 tr->current_trace->ref++;
5061 mutex_unlock(&trace_types_lock);
5066 __trace_array_put(tr);
5067 mutex_unlock(&trace_types_lock);
5071 static int tracing_release_pipe(struct inode *inode, struct file *file)
5073 struct trace_iterator *iter = file->private_data;
5074 struct trace_array *tr = inode->i_private;
5076 mutex_lock(&trace_types_lock);
5078 tr->current_trace->ref--;
5080 if (iter->trace->pipe_close)
5081 iter->trace->pipe_close(iter);
5083 mutex_unlock(&trace_types_lock);
5085 free_cpumask_var(iter->started);
5086 mutex_destroy(&iter->mutex);
5089 trace_array_put(tr);
5095 trace_poll(struct trace_iterator *iter, struct file *filp, poll_table *poll_table)
5097 struct trace_array *tr = iter->tr;
5099 /* Iterators are static, they should be filled or empty */
5100 if (trace_buffer_iter(iter, iter->cpu_file))
5101 return POLLIN | POLLRDNORM;
5103 if (tr->trace_flags & TRACE_ITER_BLOCK)
5105 * Always select as readable when in blocking mode
5107 return POLLIN | POLLRDNORM;
5109 return ring_buffer_poll_wait(iter->trace_buffer->buffer, iter->cpu_file,
5114 tracing_poll_pipe(struct file *filp, poll_table *poll_table)
5116 struct trace_iterator *iter = filp->private_data;
5118 return trace_poll(iter, filp, poll_table);
5121 /* Must be called with iter->mutex held. */
5122 static int tracing_wait_pipe(struct file *filp)
5124 struct trace_iterator *iter = filp->private_data;
5127 while (trace_empty(iter)) {
5129 if ((filp->f_flags & O_NONBLOCK)) {
5134 * We block until we read something and tracing is disabled.
5135 * We still block if tracing is disabled, but we have never
5136 * read anything. This allows a user to cat this file, and
5137 * then enable tracing. But after we have read something,
5138 * we give an EOF when tracing is again disabled.
5140 * iter->pos will be 0 if we haven't read anything.
5142 if (!tracer_tracing_is_on(iter->tr) && iter->pos)
5145 mutex_unlock(&iter->mutex);
5147 ret = wait_on_pipe(iter, false);
5149 mutex_lock(&iter->mutex);
5162 tracing_read_pipe(struct file *filp, char __user *ubuf,
5163 size_t cnt, loff_t *ppos)
5165 struct trace_iterator *iter = filp->private_data;
5169 * Avoid more than one consumer on a single file descriptor
5170 * This is just a matter of traces coherency, the ring buffer itself
5173 mutex_lock(&iter->mutex);
5175 /* return any leftover data */
5176 sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
5180 trace_seq_init(&iter->seq);
5182 if (iter->trace->read) {
5183 sret = iter->trace->read(iter, filp, ubuf, cnt, ppos);
5189 sret = tracing_wait_pipe(filp);
5193 /* stop when tracing is finished */
5194 if (trace_empty(iter)) {
5199 if (cnt >= PAGE_SIZE)
5200 cnt = PAGE_SIZE - 1;
5202 /* reset all but tr, trace, and overruns */
5203 memset(&iter->seq, 0,
5204 sizeof(struct trace_iterator) -
5205 offsetof(struct trace_iterator, seq));
5206 cpumask_clear(iter->started);
5209 trace_event_read_lock();
5210 trace_access_lock(iter->cpu_file);
5211 while (trace_find_next_entry_inc(iter) != NULL) {
5212 enum print_line_t ret;
5213 int save_len = iter->seq.seq.len;
5215 ret = print_trace_line(iter);
5216 if (ret == TRACE_TYPE_PARTIAL_LINE) {
5217 /* don't print partial lines */
5218 iter->seq.seq.len = save_len;
5221 if (ret != TRACE_TYPE_NO_CONSUME)
5222 trace_consume(iter);
5224 if (trace_seq_used(&iter->seq) >= cnt)
5228 * Setting the full flag means we reached the trace_seq buffer
5229 * size and we should leave by partial output condition above.
5230 * One of the trace_seq_* functions is not used properly.
5232 WARN_ONCE(iter->seq.full, "full flag set for trace type %d",
5235 trace_access_unlock(iter->cpu_file);
5236 trace_event_read_unlock();
5238 /* Now copy what we have to the user */
5239 sret = trace_seq_to_user(&iter->seq, ubuf, cnt);
5240 if (iter->seq.seq.readpos >= trace_seq_used(&iter->seq))
5241 trace_seq_init(&iter->seq);
5244 * If there was nothing to send to user, in spite of consuming trace
5245 * entries, go back to wait for more entries.
5251 mutex_unlock(&iter->mutex);
5256 static void tracing_spd_release_pipe(struct splice_pipe_desc *spd,
5259 __free_page(spd->pages[idx]);
5262 static const struct pipe_buf_operations tracing_pipe_buf_ops = {
5264 .confirm = generic_pipe_buf_confirm,
5265 .release = generic_pipe_buf_release,
5266 .steal = generic_pipe_buf_steal,
5267 .get = generic_pipe_buf_get,
5271 tracing_fill_pipe_page(size_t rem, struct trace_iterator *iter)
5277 /* Seq buffer is page-sized, exactly what we need. */
5279 save_len = iter->seq.seq.len;
5280 ret = print_trace_line(iter);
5282 if (trace_seq_has_overflowed(&iter->seq)) {
5283 iter->seq.seq.len = save_len;
5288 * This should not be hit, because it should only
5289 * be set if the iter->seq overflowed. But check it
5290 * anyway to be safe.
5292 if (ret == TRACE_TYPE_PARTIAL_LINE) {
5293 iter->seq.seq.len = save_len;
5297 count = trace_seq_used(&iter->seq) - save_len;
5300 iter->seq.seq.len = save_len;
5304 if (ret != TRACE_TYPE_NO_CONSUME)
5305 trace_consume(iter);
5307 if (!trace_find_next_entry_inc(iter)) {
5317 static ssize_t tracing_splice_read_pipe(struct file *filp,
5319 struct pipe_inode_info *pipe,
5323 struct page *pages_def[PIPE_DEF_BUFFERS];
5324 struct partial_page partial_def[PIPE_DEF_BUFFERS];
5325 struct trace_iterator *iter = filp->private_data;
5326 struct splice_pipe_desc spd = {
5328 .partial = partial_def,
5329 .nr_pages = 0, /* This gets updated below. */
5330 .nr_pages_max = PIPE_DEF_BUFFERS,
5332 .ops = &tracing_pipe_buf_ops,
5333 .spd_release = tracing_spd_release_pipe,
5339 if (splice_grow_spd(pipe, &spd))
5342 mutex_lock(&iter->mutex);
5344 if (iter->trace->splice_read) {
5345 ret = iter->trace->splice_read(iter, filp,
5346 ppos, pipe, len, flags);
5351 ret = tracing_wait_pipe(filp);
5355 if (!iter->ent && !trace_find_next_entry_inc(iter)) {
5360 trace_event_read_lock();
5361 trace_access_lock(iter->cpu_file);
5363 /* Fill as many pages as possible. */
5364 for (i = 0, rem = len; i < spd.nr_pages_max && rem; i++) {
5365 spd.pages[i] = alloc_page(GFP_KERNEL);
5369 rem = tracing_fill_pipe_page(rem, iter);
5371 /* Copy the data into the page, so we can start over. */
5372 ret = trace_seq_to_buffer(&iter->seq,
5373 page_address(spd.pages[i]),
5374 trace_seq_used(&iter->seq));
5376 __free_page(spd.pages[i]);
5379 spd.partial[i].offset = 0;
5380 spd.partial[i].len = trace_seq_used(&iter->seq);
5382 trace_seq_init(&iter->seq);
5385 trace_access_unlock(iter->cpu_file);
5386 trace_event_read_unlock();
5387 mutex_unlock(&iter->mutex);
5392 ret = splice_to_pipe(pipe, &spd);
5396 splice_shrink_spd(&spd);
5400 mutex_unlock(&iter->mutex);
5405 tracing_entries_read(struct file *filp, char __user *ubuf,
5406 size_t cnt, loff_t *ppos)
5408 struct inode *inode = file_inode(filp);
5409 struct trace_array *tr = inode->i_private;
5410 int cpu = tracing_get_cpu(inode);
5415 mutex_lock(&trace_types_lock);
5417 if (cpu == RING_BUFFER_ALL_CPUS) {
5418 int cpu, buf_size_same;
5423 /* check if all cpu sizes are same */
5424 for_each_tracing_cpu(cpu) {
5425 /* fill in the size from first enabled cpu */
5427 size = per_cpu_ptr(tr->trace_buffer.data, cpu)->entries;
5428 if (size != per_cpu_ptr(tr->trace_buffer.data, cpu)->entries) {
5434 if (buf_size_same) {
5435 if (!ring_buffer_expanded)
5436 r = sprintf(buf, "%lu (expanded: %lu)\n",
5438 trace_buf_size >> 10);
5440 r = sprintf(buf, "%lu\n", size >> 10);
5442 r = sprintf(buf, "X\n");
5444 r = sprintf(buf, "%lu\n", per_cpu_ptr(tr->trace_buffer.data, cpu)->entries >> 10);
5446 mutex_unlock(&trace_types_lock);
5448 ret = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
5453 tracing_entries_write(struct file *filp, const char __user *ubuf,
5454 size_t cnt, loff_t *ppos)
5456 struct inode *inode = file_inode(filp);
5457 struct trace_array *tr = inode->i_private;
5461 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
5465 /* must have at least 1 entry */
5469 /* value is in KB */
5471 ret = tracing_resize_ring_buffer(tr, val, tracing_get_cpu(inode));
5481 tracing_total_entries_read(struct file *filp, char __user *ubuf,
5482 size_t cnt, loff_t *ppos)
5484 struct trace_array *tr = filp->private_data;
5487 unsigned long size = 0, expanded_size = 0;
5489 mutex_lock(&trace_types_lock);
5490 for_each_tracing_cpu(cpu) {
5491 size += per_cpu_ptr(tr->trace_buffer.data, cpu)->entries >> 10;
5492 if (!ring_buffer_expanded)
5493 expanded_size += trace_buf_size >> 10;
5495 if (ring_buffer_expanded)
5496 r = sprintf(buf, "%lu\n", size);
5498 r = sprintf(buf, "%lu (expanded: %lu)\n", size, expanded_size);
5499 mutex_unlock(&trace_types_lock);
5501 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
5505 tracing_free_buffer_write(struct file *filp, const char __user *ubuf,
5506 size_t cnt, loff_t *ppos)
5509 * There is no need to read what the user has written, this function
5510 * is just to make sure that there is no error when "echo" is used
5519 tracing_free_buffer_release(struct inode *inode, struct file *filp)
5521 struct trace_array *tr = inode->i_private;
5523 /* disable tracing ? */
5524 if (tr->trace_flags & TRACE_ITER_STOP_ON_FREE)
5525 tracer_tracing_off(tr);
5526 /* resize the ring buffer to 0 */
5527 tracing_resize_ring_buffer(tr, 0, RING_BUFFER_ALL_CPUS);
5529 trace_array_put(tr);
5535 tracing_mark_write(struct file *filp, const char __user *ubuf,
5536 size_t cnt, loff_t *fpos)
5538 unsigned long addr = (unsigned long)ubuf;
5539 struct trace_array *tr = filp->private_data;
5540 struct ring_buffer_event *event;
5541 struct ring_buffer *buffer;
5542 struct print_entry *entry;
5543 unsigned long irq_flags;
5544 struct page *pages[2];
5554 if (tracing_disabled)
5557 if (!(tr->trace_flags & TRACE_ITER_MARKERS))
5560 if (cnt > TRACE_BUF_SIZE)
5561 cnt = TRACE_BUF_SIZE;
5564 * Userspace is injecting traces into the kernel trace buffer.
5565 * We want to be as non intrusive as possible.
5566 * To do so, we do not want to allocate any special buffers
5567 * or take any locks, but instead write the userspace data
5568 * straight into the ring buffer.
5570 * First we need to pin the userspace buffer into memory,
5571 * which, most likely it is, because it just referenced it.
5572 * But there's no guarantee that it is. By using get_user_pages_fast()
5573 * and kmap_atomic/kunmap_atomic() we can get access to the
5574 * pages directly. We then write the data directly into the
5577 BUILD_BUG_ON(TRACE_BUF_SIZE >= PAGE_SIZE);
5579 /* check if we cross pages */
5580 if ((addr & PAGE_MASK) != ((addr + cnt) & PAGE_MASK))
5583 offset = addr & (PAGE_SIZE - 1);
5586 ret = get_user_pages_fast(addr, nr_pages, 0, pages);
5587 if (ret < nr_pages) {
5589 put_page(pages[ret]);
5594 for (i = 0; i < nr_pages; i++)
5595 map_page[i] = kmap_atomic(pages[i]);
5597 local_save_flags(irq_flags);
5598 size = sizeof(*entry) + cnt + 2; /* possible \n added */
5599 buffer = tr->trace_buffer.buffer;
5600 event = trace_buffer_lock_reserve(buffer, TRACE_PRINT, size,
5601 irq_flags, preempt_count());
5603 /* Ring buffer disabled, return as if not open for write */
5608 entry = ring_buffer_event_data(event);
5609 entry->ip = _THIS_IP_;
5611 if (nr_pages == 2) {
5612 len = PAGE_SIZE - offset;
5613 memcpy(&entry->buf, map_page[0] + offset, len);
5614 memcpy(&entry->buf[len], map_page[1], cnt - len);
5616 memcpy(&entry->buf, map_page[0] + offset, cnt);
5618 if (entry->buf[cnt - 1] != '\n') {
5619 entry->buf[cnt] = '\n';
5620 entry->buf[cnt + 1] = '\0';
5622 entry->buf[cnt] = '\0';
5624 __buffer_unlock_commit(buffer, event);
5631 for (i = nr_pages - 1; i >= 0; i--) {
5632 kunmap_atomic(map_page[i]);
5639 static int tracing_clock_show(struct seq_file *m, void *v)
5641 struct trace_array *tr = m->private;
5644 for (i = 0; i < ARRAY_SIZE(trace_clocks); i++)
5646 "%s%s%s%s", i ? " " : "",
5647 i == tr->clock_id ? "[" : "", trace_clocks[i].name,
5648 i == tr->clock_id ? "]" : "");
5654 static int tracing_set_clock(struct trace_array *tr, const char *clockstr)
5658 for (i = 0; i < ARRAY_SIZE(trace_clocks); i++) {
5659 if (strcmp(trace_clocks[i].name, clockstr) == 0)
5662 if (i == ARRAY_SIZE(trace_clocks))
5665 mutex_lock(&trace_types_lock);
5669 ring_buffer_set_clock(tr->trace_buffer.buffer, trace_clocks[i].func);
5672 * New clock may not be consistent with the previous clock.
5673 * Reset the buffer so that it doesn't have incomparable timestamps.
5675 tracing_reset_online_cpus(&tr->trace_buffer);
5677 #ifdef CONFIG_TRACER_MAX_TRACE
5678 if (tr->max_buffer.buffer)
5679 ring_buffer_set_clock(tr->max_buffer.buffer, trace_clocks[i].func);
5680 tracing_reset_online_cpus(&tr->max_buffer);
5683 mutex_unlock(&trace_types_lock);
5688 static ssize_t tracing_clock_write(struct file *filp, const char __user *ubuf,
5689 size_t cnt, loff_t *fpos)
5691 struct seq_file *m = filp->private_data;
5692 struct trace_array *tr = m->private;
5694 const char *clockstr;
5697 if (cnt >= sizeof(buf))
5700 if (copy_from_user(buf, ubuf, cnt))
5705 clockstr = strstrip(buf);
5707 ret = tracing_set_clock(tr, clockstr);
5716 static int tracing_clock_open(struct inode *inode, struct file *file)
5718 struct trace_array *tr = inode->i_private;
5721 if (tracing_disabled)
5724 if (trace_array_get(tr))
5727 ret = single_open(file, tracing_clock_show, inode->i_private);
5729 trace_array_put(tr);
5734 struct ftrace_buffer_info {
5735 struct trace_iterator iter;
5740 #ifdef CONFIG_TRACER_SNAPSHOT
5741 static int tracing_snapshot_open(struct inode *inode, struct file *file)
5743 struct trace_array *tr = inode->i_private;
5744 struct trace_iterator *iter;
5748 if (trace_array_get(tr) < 0)
5751 if (file->f_mode & FMODE_READ) {
5752 iter = __tracing_open(inode, file, true);
5754 ret = PTR_ERR(iter);
5756 /* Writes still need the seq_file to hold the private data */
5758 m = kzalloc(sizeof(*m), GFP_KERNEL);
5761 iter = kzalloc(sizeof(*iter), GFP_KERNEL);
5769 iter->trace_buffer = &tr->max_buffer;
5770 iter->cpu_file = tracing_get_cpu(inode);
5772 file->private_data = m;
5776 trace_array_put(tr);
5782 tracing_snapshot_write(struct file *filp, const char __user *ubuf, size_t cnt,
5785 struct seq_file *m = filp->private_data;
5786 struct trace_iterator *iter = m->private;
5787 struct trace_array *tr = iter->tr;
5791 ret = tracing_update_buffers();
5795 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
5799 mutex_lock(&trace_types_lock);
5801 if (tr->current_trace->use_max_tr) {
5808 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
5812 if (tr->allocated_snapshot)
5816 /* Only allow per-cpu swap if the ring buffer supports it */
5817 #ifndef CONFIG_RING_BUFFER_ALLOW_SWAP
5818 if (iter->cpu_file != RING_BUFFER_ALL_CPUS) {
5823 if (!tr->allocated_snapshot) {
5824 ret = alloc_snapshot(tr);
5828 local_irq_disable();
5829 /* Now, we're going to swap */
5830 if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
5831 update_max_tr(tr, current, smp_processor_id());
5833 update_max_tr_single(tr, current, iter->cpu_file);
5837 if (tr->allocated_snapshot) {
5838 if (iter->cpu_file == RING_BUFFER_ALL_CPUS)
5839 tracing_reset_online_cpus(&tr->max_buffer);
5841 tracing_reset(&tr->max_buffer, iter->cpu_file);
5851 mutex_unlock(&trace_types_lock);
5855 static int tracing_snapshot_release(struct inode *inode, struct file *file)
5857 struct seq_file *m = file->private_data;
5860 ret = tracing_release(inode, file);
5862 if (file->f_mode & FMODE_READ)
5865 /* If write only, the seq_file is just a stub */
5873 static int tracing_buffers_open(struct inode *inode, struct file *filp);
5874 static ssize_t tracing_buffers_read(struct file *filp, char __user *ubuf,
5875 size_t count, loff_t *ppos);
5876 static int tracing_buffers_release(struct inode *inode, struct file *file);
5877 static ssize_t tracing_buffers_splice_read(struct file *file, loff_t *ppos,
5878 struct pipe_inode_info *pipe, size_t len, unsigned int flags);
5880 static int snapshot_raw_open(struct inode *inode, struct file *filp)
5882 struct ftrace_buffer_info *info;
5885 ret = tracing_buffers_open(inode, filp);
5889 info = filp->private_data;
5891 if (info->iter.trace->use_max_tr) {
5892 tracing_buffers_release(inode, filp);
5896 info->iter.snapshot = true;
5897 info->iter.trace_buffer = &info->iter.tr->max_buffer;
5902 #endif /* CONFIG_TRACER_SNAPSHOT */
5905 static const struct file_operations tracing_thresh_fops = {
5906 .open = tracing_open_generic,
5907 .read = tracing_thresh_read,
5908 .write = tracing_thresh_write,
5909 .llseek = generic_file_llseek,
5912 #if defined(CONFIG_TRACER_MAX_TRACE) || defined(CONFIG_HWLAT_TRACER)
5913 static const struct file_operations tracing_max_lat_fops = {
5914 .open = tracing_open_generic,
5915 .read = tracing_max_lat_read,
5916 .write = tracing_max_lat_write,
5917 .llseek = generic_file_llseek,
5921 static const struct file_operations set_tracer_fops = {
5922 .open = tracing_open_generic,
5923 .read = tracing_set_trace_read,
5924 .write = tracing_set_trace_write,
5925 .llseek = generic_file_llseek,
5928 static const struct file_operations tracing_pipe_fops = {
5929 .open = tracing_open_pipe,
5930 .poll = tracing_poll_pipe,
5931 .read = tracing_read_pipe,
5932 .splice_read = tracing_splice_read_pipe,
5933 .release = tracing_release_pipe,
5934 .llseek = no_llseek,
5937 static const struct file_operations tracing_entries_fops = {
5938 .open = tracing_open_generic_tr,
5939 .read = tracing_entries_read,
5940 .write = tracing_entries_write,
5941 .llseek = generic_file_llseek,
5942 .release = tracing_release_generic_tr,
5945 static const struct file_operations tracing_total_entries_fops = {
5946 .open = tracing_open_generic_tr,
5947 .read = tracing_total_entries_read,
5948 .llseek = generic_file_llseek,
5949 .release = tracing_release_generic_tr,
5952 static const struct file_operations tracing_free_buffer_fops = {
5953 .open = tracing_open_generic_tr,
5954 .write = tracing_free_buffer_write,
5955 .release = tracing_free_buffer_release,
5958 static const struct file_operations tracing_mark_fops = {
5959 .open = tracing_open_generic_tr,
5960 .write = tracing_mark_write,
5961 .llseek = generic_file_llseek,
5962 .release = tracing_release_generic_tr,
5965 static const struct file_operations trace_clock_fops = {
5966 .open = tracing_clock_open,
5968 .llseek = seq_lseek,
5969 .release = tracing_single_release_tr,
5970 .write = tracing_clock_write,
5973 #ifdef CONFIG_TRACER_SNAPSHOT
5974 static const struct file_operations snapshot_fops = {
5975 .open = tracing_snapshot_open,
5977 .write = tracing_snapshot_write,
5978 .llseek = tracing_lseek,
5979 .release = tracing_snapshot_release,
5982 static const struct file_operations snapshot_raw_fops = {
5983 .open = snapshot_raw_open,
5984 .read = tracing_buffers_read,
5985 .release = tracing_buffers_release,
5986 .splice_read = tracing_buffers_splice_read,
5987 .llseek = no_llseek,
5990 #endif /* CONFIG_TRACER_SNAPSHOT */
5992 static int tracing_buffers_open(struct inode *inode, struct file *filp)
5994 struct trace_array *tr = inode->i_private;
5995 struct ftrace_buffer_info *info;
5998 if (tracing_disabled)
6001 if (trace_array_get(tr) < 0)
6004 info = kzalloc(sizeof(*info), GFP_KERNEL);
6006 trace_array_put(tr);
6010 mutex_lock(&trace_types_lock);
6013 info->iter.cpu_file = tracing_get_cpu(inode);
6014 info->iter.trace = tr->current_trace;
6015 info->iter.trace_buffer = &tr->trace_buffer;
6017 /* Force reading ring buffer for first read */
6018 info->read = (unsigned int)-1;
6020 filp->private_data = info;
6022 tr->current_trace->ref++;
6024 mutex_unlock(&trace_types_lock);
6026 ret = nonseekable_open(inode, filp);
6028 trace_array_put(tr);
6034 tracing_buffers_poll(struct file *filp, poll_table *poll_table)
6036 struct ftrace_buffer_info *info = filp->private_data;
6037 struct trace_iterator *iter = &info->iter;
6039 return trace_poll(iter, filp, poll_table);
6043 tracing_buffers_read(struct file *filp, char __user *ubuf,
6044 size_t count, loff_t *ppos)
6046 struct ftrace_buffer_info *info = filp->private_data;
6047 struct trace_iterator *iter = &info->iter;
6054 #ifdef CONFIG_TRACER_MAX_TRACE
6055 if (iter->snapshot && iter->tr->current_trace->use_max_tr)
6060 info->spare = ring_buffer_alloc_read_page(iter->trace_buffer->buffer,
6065 /* Do we have previous read data to read? */
6066 if (info->read < PAGE_SIZE)
6070 trace_access_lock(iter->cpu_file);
6071 ret = ring_buffer_read_page(iter->trace_buffer->buffer,
6075 trace_access_unlock(iter->cpu_file);
6078 if (trace_empty(iter)) {
6079 if ((filp->f_flags & O_NONBLOCK))
6082 ret = wait_on_pipe(iter, false);
6093 size = PAGE_SIZE - info->read;
6097 ret = copy_to_user(ubuf, info->spare + info->read, size);
6109 static int tracing_buffers_release(struct inode *inode, struct file *file)
6111 struct ftrace_buffer_info *info = file->private_data;
6112 struct trace_iterator *iter = &info->iter;
6114 mutex_lock(&trace_types_lock);
6116 iter->tr->current_trace->ref--;
6118 __trace_array_put(iter->tr);
6121 ring_buffer_free_read_page(iter->trace_buffer->buffer, info->spare);
6124 mutex_unlock(&trace_types_lock);
6130 struct ring_buffer *buffer;
6135 static void buffer_pipe_buf_release(struct pipe_inode_info *pipe,
6136 struct pipe_buffer *buf)
6138 struct buffer_ref *ref = (struct buffer_ref *)buf->private;
6143 ring_buffer_free_read_page(ref->buffer, ref->page);
6148 static bool buffer_pipe_buf_get(struct pipe_inode_info *pipe,
6149 struct pipe_buffer *buf)
6151 struct buffer_ref *ref = (struct buffer_ref *)buf->private;
6153 if (ref->ref > INT_MAX/2)
6160 /* Pipe buffer operations for a buffer. */
6161 static const struct pipe_buf_operations buffer_pipe_buf_ops = {
6163 .confirm = generic_pipe_buf_confirm,
6164 .release = buffer_pipe_buf_release,
6165 .steal = generic_pipe_buf_steal,
6166 .get = buffer_pipe_buf_get,
6170 * Callback from splice_to_pipe(), if we need to release some pages
6171 * at the end of the spd in case we error'ed out in filling the pipe.
6173 static void buffer_spd_release(struct splice_pipe_desc *spd, unsigned int i)
6175 struct buffer_ref *ref =
6176 (struct buffer_ref *)spd->partial[i].private;
6181 ring_buffer_free_read_page(ref->buffer, ref->page);
6183 spd->partial[i].private = 0;
6187 tracing_buffers_splice_read(struct file *file, loff_t *ppos,
6188 struct pipe_inode_info *pipe, size_t len,
6191 struct ftrace_buffer_info *info = file->private_data;
6192 struct trace_iterator *iter = &info->iter;
6193 struct partial_page partial_def[PIPE_DEF_BUFFERS];
6194 struct page *pages_def[PIPE_DEF_BUFFERS];
6195 struct splice_pipe_desc spd = {
6197 .partial = partial_def,
6198 .nr_pages_max = PIPE_DEF_BUFFERS,
6200 .ops = &buffer_pipe_buf_ops,
6201 .spd_release = buffer_spd_release,
6203 struct buffer_ref *ref;
6207 #ifdef CONFIG_TRACER_MAX_TRACE
6208 if (iter->snapshot && iter->tr->current_trace->use_max_tr)
6212 if (*ppos & (PAGE_SIZE - 1))
6215 if (len & (PAGE_SIZE - 1)) {
6216 if (len < PAGE_SIZE)
6221 if (splice_grow_spd(pipe, &spd))
6225 trace_access_lock(iter->cpu_file);
6226 entries = ring_buffer_entries_cpu(iter->trace_buffer->buffer, iter->cpu_file);
6228 for (i = 0; i < spd.nr_pages_max && len && entries; i++, len -= PAGE_SIZE) {
6232 ref = kzalloc(sizeof(*ref), GFP_KERNEL);
6239 ref->buffer = iter->trace_buffer->buffer;
6240 ref->page = ring_buffer_alloc_read_page(ref->buffer, iter->cpu_file);
6247 r = ring_buffer_read_page(ref->buffer, &ref->page,
6248 len, iter->cpu_file, 1);
6250 ring_buffer_free_read_page(ref->buffer, ref->page);
6255 page = virt_to_page(ref->page);
6257 spd.pages[i] = page;
6258 spd.partial[i].len = PAGE_SIZE;
6259 spd.partial[i].offset = 0;
6260 spd.partial[i].private = (unsigned long)ref;
6264 entries = ring_buffer_entries_cpu(iter->trace_buffer->buffer, iter->cpu_file);
6267 trace_access_unlock(iter->cpu_file);
6270 /* did we read anything? */
6271 if (!spd.nr_pages) {
6276 if ((file->f_flags & O_NONBLOCK) || (flags & SPLICE_F_NONBLOCK))
6279 ret = wait_on_pipe(iter, true);
6286 ret = splice_to_pipe(pipe, &spd);
6288 splice_shrink_spd(&spd);
6293 static const struct file_operations tracing_buffers_fops = {
6294 .open = tracing_buffers_open,
6295 .read = tracing_buffers_read,
6296 .poll = tracing_buffers_poll,
6297 .release = tracing_buffers_release,
6298 .splice_read = tracing_buffers_splice_read,
6299 .llseek = no_llseek,
6303 tracing_stats_read(struct file *filp, char __user *ubuf,
6304 size_t count, loff_t *ppos)
6306 struct inode *inode = file_inode(filp);
6307 struct trace_array *tr = inode->i_private;
6308 struct trace_buffer *trace_buf = &tr->trace_buffer;
6309 int cpu = tracing_get_cpu(inode);
6310 struct trace_seq *s;
6312 unsigned long long t;
6313 unsigned long usec_rem;
6315 s = kmalloc(sizeof(*s), GFP_KERNEL);
6321 cnt = ring_buffer_entries_cpu(trace_buf->buffer, cpu);
6322 trace_seq_printf(s, "entries: %ld\n", cnt);
6324 cnt = ring_buffer_overrun_cpu(trace_buf->buffer, cpu);
6325 trace_seq_printf(s, "overrun: %ld\n", cnt);
6327 cnt = ring_buffer_commit_overrun_cpu(trace_buf->buffer, cpu);
6328 trace_seq_printf(s, "commit overrun: %ld\n", cnt);
6330 cnt = ring_buffer_bytes_cpu(trace_buf->buffer, cpu);
6331 trace_seq_printf(s, "bytes: %ld\n", cnt);
6333 if (trace_clocks[tr->clock_id].in_ns) {
6334 /* local or global for trace_clock */
6335 t = ns2usecs(ring_buffer_oldest_event_ts(trace_buf->buffer, cpu));
6336 usec_rem = do_div(t, USEC_PER_SEC);
6337 trace_seq_printf(s, "oldest event ts: %5llu.%06lu\n",
6340 t = ns2usecs(ring_buffer_time_stamp(trace_buf->buffer, cpu));
6341 usec_rem = do_div(t, USEC_PER_SEC);
6342 trace_seq_printf(s, "now ts: %5llu.%06lu\n", t, usec_rem);
6344 /* counter or tsc mode for trace_clock */
6345 trace_seq_printf(s, "oldest event ts: %llu\n",
6346 ring_buffer_oldest_event_ts(trace_buf->buffer, cpu));
6348 trace_seq_printf(s, "now ts: %llu\n",
6349 ring_buffer_time_stamp(trace_buf->buffer, cpu));
6352 cnt = ring_buffer_dropped_events_cpu(trace_buf->buffer, cpu);
6353 trace_seq_printf(s, "dropped events: %ld\n", cnt);
6355 cnt = ring_buffer_read_events_cpu(trace_buf->buffer, cpu);
6356 trace_seq_printf(s, "read events: %ld\n", cnt);
6358 count = simple_read_from_buffer(ubuf, count, ppos,
6359 s->buffer, trace_seq_used(s));
6366 static const struct file_operations tracing_stats_fops = {
6367 .open = tracing_open_generic_tr,
6368 .read = tracing_stats_read,
6369 .llseek = generic_file_llseek,
6370 .release = tracing_release_generic_tr,
6373 #ifdef CONFIG_DYNAMIC_FTRACE
6375 int __weak ftrace_arch_read_dyn_info(char *buf, int size)
6381 tracing_read_dyn_info(struct file *filp, char __user *ubuf,
6382 size_t cnt, loff_t *ppos)
6384 static char ftrace_dyn_info_buffer[1024];
6385 static DEFINE_MUTEX(dyn_info_mutex);
6386 unsigned long *p = filp->private_data;
6387 char *buf = ftrace_dyn_info_buffer;
6388 int size = ARRAY_SIZE(ftrace_dyn_info_buffer);
6391 mutex_lock(&dyn_info_mutex);
6392 r = sprintf(buf, "%ld ", *p);
6394 r += ftrace_arch_read_dyn_info(buf+r, (size-1)-r);
6397 r = simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
6399 mutex_unlock(&dyn_info_mutex);
6404 static const struct file_operations tracing_dyn_info_fops = {
6405 .open = tracing_open_generic,
6406 .read = tracing_read_dyn_info,
6407 .llseek = generic_file_llseek,
6409 #endif /* CONFIG_DYNAMIC_FTRACE */
6411 #if defined(CONFIG_TRACER_SNAPSHOT) && defined(CONFIG_DYNAMIC_FTRACE)
6413 ftrace_snapshot(unsigned long ip, unsigned long parent_ip, void **data)
6419 ftrace_count_snapshot(unsigned long ip, unsigned long parent_ip, void **data)
6421 unsigned long *count = (long *)data;
6433 ftrace_snapshot_print(struct seq_file *m, unsigned long ip,
6434 struct ftrace_probe_ops *ops, void *data)
6436 long count = (long)data;
6438 seq_printf(m, "%ps:", (void *)ip);
6440 seq_puts(m, "snapshot");
6443 seq_puts(m, ":unlimited\n");
6445 seq_printf(m, ":count=%ld\n", count);
6450 static struct ftrace_probe_ops snapshot_probe_ops = {
6451 .func = ftrace_snapshot,
6452 .print = ftrace_snapshot_print,
6455 static struct ftrace_probe_ops snapshot_count_probe_ops = {
6456 .func = ftrace_count_snapshot,
6457 .print = ftrace_snapshot_print,
6461 ftrace_trace_snapshot_callback(struct ftrace_hash *hash,
6462 char *glob, char *cmd, char *param, int enable)
6464 struct ftrace_probe_ops *ops;
6465 void *count = (void *)-1;
6469 /* hash funcs only work with set_ftrace_filter */
6473 ops = param ? &snapshot_count_probe_ops : &snapshot_probe_ops;
6475 if (glob[0] == '!') {
6476 unregister_ftrace_function_probe_func(glob+1, ops);
6483 number = strsep(¶m, ":");
6485 if (!strlen(number))
6489 * We use the callback data field (which is a pointer)
6492 ret = kstrtoul(number, 0, (unsigned long *)&count);
6497 ret = alloc_snapshot(&global_trace);
6501 ret = register_ftrace_function_probe(glob, ops, count);
6504 return ret < 0 ? ret : 0;
6507 static struct ftrace_func_command ftrace_snapshot_cmd = {
6509 .func = ftrace_trace_snapshot_callback,
6512 static __init int register_snapshot_cmd(void)
6514 return register_ftrace_command(&ftrace_snapshot_cmd);
6517 static inline __init int register_snapshot_cmd(void) { return 0; }
6518 #endif /* defined(CONFIG_TRACER_SNAPSHOT) && defined(CONFIG_DYNAMIC_FTRACE) */
6520 static struct dentry *tracing_get_dentry(struct trace_array *tr)
6522 if (WARN_ON(!tr->dir))
6523 return ERR_PTR(-ENODEV);
6525 /* Top directory uses NULL as the parent */
6526 if (tr->flags & TRACE_ARRAY_FL_GLOBAL)
6529 /* All sub buffers have a descriptor */
6533 static struct dentry *tracing_dentry_percpu(struct trace_array *tr, int cpu)
6535 struct dentry *d_tracer;
6538 return tr->percpu_dir;
6540 d_tracer = tracing_get_dentry(tr);
6541 if (IS_ERR(d_tracer))
6544 tr->percpu_dir = tracefs_create_dir("per_cpu", d_tracer);
6546 WARN_ONCE(!tr->percpu_dir,
6547 "Could not create tracefs directory 'per_cpu/%d'\n", cpu);
6549 return tr->percpu_dir;
6552 static struct dentry *
6553 trace_create_cpu_file(const char *name, umode_t mode, struct dentry *parent,
6554 void *data, long cpu, const struct file_operations *fops)
6556 struct dentry *ret = trace_create_file(name, mode, parent, data, fops);
6558 if (ret) /* See tracing_get_cpu() */
6559 d_inode(ret)->i_cdev = (void *)(cpu + 1);
6564 tracing_init_tracefs_percpu(struct trace_array *tr, long cpu)
6566 struct dentry *d_percpu = tracing_dentry_percpu(tr, cpu);
6567 struct dentry *d_cpu;
6568 char cpu_dir[30]; /* 30 characters should be more than enough */
6573 snprintf(cpu_dir, 30, "cpu%ld", cpu);
6574 d_cpu = tracefs_create_dir(cpu_dir, d_percpu);
6576 pr_warn("Could not create tracefs '%s' entry\n", cpu_dir);
6580 /* per cpu trace_pipe */
6581 trace_create_cpu_file("trace_pipe", 0444, d_cpu,
6582 tr, cpu, &tracing_pipe_fops);
6585 trace_create_cpu_file("trace", 0644, d_cpu,
6586 tr, cpu, &tracing_fops);
6588 trace_create_cpu_file("trace_pipe_raw", 0444, d_cpu,
6589 tr, cpu, &tracing_buffers_fops);
6591 trace_create_cpu_file("stats", 0444, d_cpu,
6592 tr, cpu, &tracing_stats_fops);
6594 trace_create_cpu_file("buffer_size_kb", 0444, d_cpu,
6595 tr, cpu, &tracing_entries_fops);
6597 #ifdef CONFIG_TRACER_SNAPSHOT
6598 trace_create_cpu_file("snapshot", 0644, d_cpu,
6599 tr, cpu, &snapshot_fops);
6601 trace_create_cpu_file("snapshot_raw", 0444, d_cpu,
6602 tr, cpu, &snapshot_raw_fops);
6606 #ifdef CONFIG_FTRACE_SELFTEST
6607 /* Let selftest have access to static functions in this file */
6608 #include "trace_selftest.c"
6612 trace_options_read(struct file *filp, char __user *ubuf, size_t cnt,
6615 struct trace_option_dentry *topt = filp->private_data;
6618 if (topt->flags->val & topt->opt->bit)
6623 return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
6627 trace_options_write(struct file *filp, const char __user *ubuf, size_t cnt,
6630 struct trace_option_dentry *topt = filp->private_data;
6634 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
6638 if (val != 0 && val != 1)
6641 if (!!(topt->flags->val & topt->opt->bit) != val) {
6642 mutex_lock(&trace_types_lock);
6643 ret = __set_tracer_option(topt->tr, topt->flags,
6645 mutex_unlock(&trace_types_lock);
6656 static const struct file_operations trace_options_fops = {
6657 .open = tracing_open_generic,
6658 .read = trace_options_read,
6659 .write = trace_options_write,
6660 .llseek = generic_file_llseek,
6664 * In order to pass in both the trace_array descriptor as well as the index
6665 * to the flag that the trace option file represents, the trace_array
6666 * has a character array of trace_flags_index[], which holds the index
6667 * of the bit for the flag it represents. index[0] == 0, index[1] == 1, etc.
6668 * The address of this character array is passed to the flag option file
6669 * read/write callbacks.
6671 * In order to extract both the index and the trace_array descriptor,
6672 * get_tr_index() uses the following algorithm.
6676 * As the pointer itself contains the address of the index (remember
6679 * Then to get the trace_array descriptor, by subtracting that index
6680 * from the ptr, we get to the start of the index itself.
6682 * ptr - idx == &index[0]
6684 * Then a simple container_of() from that pointer gets us to the
6685 * trace_array descriptor.
6687 static void get_tr_index(void *data, struct trace_array **ptr,
6688 unsigned int *pindex)
6690 *pindex = *(unsigned char *)data;
6692 *ptr = container_of(data - *pindex, struct trace_array,
6697 trace_options_core_read(struct file *filp, char __user *ubuf, size_t cnt,
6700 void *tr_index = filp->private_data;
6701 struct trace_array *tr;
6705 get_tr_index(tr_index, &tr, &index);
6707 if (tr->trace_flags & (1 << index))
6712 return simple_read_from_buffer(ubuf, cnt, ppos, buf, 2);
6716 trace_options_core_write(struct file *filp, const char __user *ubuf, size_t cnt,
6719 void *tr_index = filp->private_data;
6720 struct trace_array *tr;
6725 get_tr_index(tr_index, &tr, &index);
6727 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
6731 if (val != 0 && val != 1)
6734 mutex_lock(&trace_types_lock);
6735 ret = set_tracer_flag(tr, 1 << index, val);
6736 mutex_unlock(&trace_types_lock);
6746 static const struct file_operations trace_options_core_fops = {
6747 .open = tracing_open_generic,
6748 .read = trace_options_core_read,
6749 .write = trace_options_core_write,
6750 .llseek = generic_file_llseek,
6753 struct dentry *trace_create_file(const char *name,
6755 struct dentry *parent,
6757 const struct file_operations *fops)
6761 ret = tracefs_create_file(name, mode, parent, data, fops);
6763 pr_warn("Could not create tracefs '%s' entry\n", name);
6769 static struct dentry *trace_options_init_dentry(struct trace_array *tr)
6771 struct dentry *d_tracer;
6776 d_tracer = tracing_get_dentry(tr);
6777 if (IS_ERR(d_tracer))
6780 tr->options = tracefs_create_dir("options", d_tracer);
6782 pr_warn("Could not create tracefs directory 'options'\n");
6790 create_trace_option_file(struct trace_array *tr,
6791 struct trace_option_dentry *topt,
6792 struct tracer_flags *flags,
6793 struct tracer_opt *opt)
6795 struct dentry *t_options;
6797 t_options = trace_options_init_dentry(tr);
6801 topt->flags = flags;
6805 topt->entry = trace_create_file(opt->name, 0644, t_options, topt,
6806 &trace_options_fops);
6811 create_trace_option_files(struct trace_array *tr, struct tracer *tracer)
6813 struct trace_option_dentry *topts;
6814 struct trace_options *tr_topts;
6815 struct tracer_flags *flags;
6816 struct tracer_opt *opts;
6823 flags = tracer->flags;
6825 if (!flags || !flags->opts)
6829 * If this is an instance, only create flags for tracers
6830 * the instance may have.
6832 if (!trace_ok_for_array(tracer, tr))
6835 for (i = 0; i < tr->nr_topts; i++) {
6836 /* Make sure there's no duplicate flags. */
6837 if (WARN_ON_ONCE(tr->topts[i].tracer->flags == tracer->flags))
6843 for (cnt = 0; opts[cnt].name; cnt++)
6846 topts = kcalloc(cnt + 1, sizeof(*topts), GFP_KERNEL);
6850 tr_topts = krealloc(tr->topts, sizeof(*tr->topts) * (tr->nr_topts + 1),
6857 tr->topts = tr_topts;
6858 tr->topts[tr->nr_topts].tracer = tracer;
6859 tr->topts[tr->nr_topts].topts = topts;
6862 for (cnt = 0; opts[cnt].name; cnt++) {
6863 create_trace_option_file(tr, &topts[cnt], flags,
6865 WARN_ONCE(topts[cnt].entry == NULL,
6866 "Failed to create trace option: %s",
6871 static struct dentry *
6872 create_trace_option_core_file(struct trace_array *tr,
6873 const char *option, long index)
6875 struct dentry *t_options;
6877 t_options = trace_options_init_dentry(tr);
6881 return trace_create_file(option, 0644, t_options,
6882 (void *)&tr->trace_flags_index[index],
6883 &trace_options_core_fops);
6886 static void create_trace_options_dir(struct trace_array *tr)
6888 struct dentry *t_options;
6889 bool top_level = tr == &global_trace;
6892 t_options = trace_options_init_dentry(tr);
6896 for (i = 0; trace_options[i]; i++) {
6898 !((1 << i) & TOP_LEVEL_TRACE_FLAGS))
6899 create_trace_option_core_file(tr, trace_options[i], i);
6904 rb_simple_read(struct file *filp, char __user *ubuf,
6905 size_t cnt, loff_t *ppos)
6907 struct trace_array *tr = filp->private_data;
6911 r = tracer_tracing_is_on(tr);
6912 r = sprintf(buf, "%d\n", r);
6914 return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
6918 rb_simple_write(struct file *filp, const char __user *ubuf,
6919 size_t cnt, loff_t *ppos)
6921 struct trace_array *tr = filp->private_data;
6922 struct ring_buffer *buffer = tr->trace_buffer.buffer;
6926 ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
6931 mutex_lock(&trace_types_lock);
6932 if (!!val == tracer_tracing_is_on(tr)) {
6933 val = 0; /* do nothing */
6935 tracer_tracing_on(tr);
6936 if (tr->current_trace->start)
6937 tr->current_trace->start(tr);
6939 tracer_tracing_off(tr);
6940 if (tr->current_trace->stop)
6941 tr->current_trace->stop(tr);
6943 mutex_unlock(&trace_types_lock);
6951 static const struct file_operations rb_simple_fops = {
6952 .open = tracing_open_generic_tr,
6953 .read = rb_simple_read,
6954 .write = rb_simple_write,
6955 .release = tracing_release_generic_tr,
6956 .llseek = default_llseek,
6959 struct dentry *trace_instance_dir;
6962 init_tracer_tracefs(struct trace_array *tr, struct dentry *d_tracer);
6965 allocate_trace_buffer(struct trace_array *tr, struct trace_buffer *buf, int size)
6967 enum ring_buffer_flags rb_flags;
6969 rb_flags = tr->trace_flags & TRACE_ITER_OVERWRITE ? RB_FL_OVERWRITE : 0;
6973 buf->buffer = ring_buffer_alloc(size, rb_flags);
6977 buf->data = alloc_percpu(struct trace_array_cpu);
6979 ring_buffer_free(buf->buffer);
6984 /* Allocate the first page for all buffers */
6985 set_buffer_entries(&tr->trace_buffer,
6986 ring_buffer_size(tr->trace_buffer.buffer, 0));
6991 static int allocate_trace_buffers(struct trace_array *tr, int size)
6995 ret = allocate_trace_buffer(tr, &tr->trace_buffer, size);
6999 #ifdef CONFIG_TRACER_MAX_TRACE
7000 ret = allocate_trace_buffer(tr, &tr->max_buffer,
7001 allocate_snapshot ? size : 1);
7003 ring_buffer_free(tr->trace_buffer.buffer);
7004 tr->trace_buffer.buffer = NULL;
7005 free_percpu(tr->trace_buffer.data);
7006 tr->trace_buffer.data = NULL;
7009 tr->allocated_snapshot = allocate_snapshot;
7012 * Only the top level trace array gets its snapshot allocated
7013 * from the kernel command line.
7015 allocate_snapshot = false;
7020 static void free_trace_buffer(struct trace_buffer *buf)
7023 ring_buffer_free(buf->buffer);
7025 free_percpu(buf->data);
7030 static void free_trace_buffers(struct trace_array *tr)
7035 free_trace_buffer(&tr->trace_buffer);
7037 #ifdef CONFIG_TRACER_MAX_TRACE
7038 free_trace_buffer(&tr->max_buffer);
7042 static void init_trace_flags_index(struct trace_array *tr)
7046 /* Used by the trace options files */
7047 for (i = 0; i < TRACE_FLAGS_MAX_SIZE; i++)
7048 tr->trace_flags_index[i] = i;
7051 static void __update_tracer_options(struct trace_array *tr)
7055 for (t = trace_types; t; t = t->next)
7056 add_tracer_options(tr, t);
7059 static void update_tracer_options(struct trace_array *tr)
7061 mutex_lock(&trace_types_lock);
7062 __update_tracer_options(tr);
7063 mutex_unlock(&trace_types_lock);
7066 static int instance_mkdir(const char *name)
7068 struct trace_array *tr;
7071 mutex_lock(&trace_types_lock);
7074 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
7075 if (tr->name && strcmp(tr->name, name) == 0)
7080 tr = kzalloc(sizeof(*tr), GFP_KERNEL);
7084 tr->name = kstrdup(name, GFP_KERNEL);
7088 if (!alloc_cpumask_var(&tr->tracing_cpumask, GFP_KERNEL))
7091 tr->trace_flags = global_trace.trace_flags & ~ZEROED_TRACE_FLAGS;
7093 cpumask_copy(tr->tracing_cpumask, cpu_all_mask);
7095 raw_spin_lock_init(&tr->start_lock);
7097 tr->max_lock = (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
7099 tr->current_trace = &nop_trace;
7101 INIT_LIST_HEAD(&tr->systems);
7102 INIT_LIST_HEAD(&tr->events);
7104 if (allocate_trace_buffers(tr, trace_buf_size) < 0)
7107 tr->dir = tracefs_create_dir(name, trace_instance_dir);
7111 ret = event_trace_add_tracer(tr->dir, tr);
7113 tracefs_remove_recursive(tr->dir);
7117 init_tracer_tracefs(tr, tr->dir);
7118 init_trace_flags_index(tr);
7119 __update_tracer_options(tr);
7121 list_add(&tr->list, &ftrace_trace_arrays);
7123 mutex_unlock(&trace_types_lock);
7128 free_trace_buffers(tr);
7129 free_cpumask_var(tr->tracing_cpumask);
7134 mutex_unlock(&trace_types_lock);
7140 static int instance_rmdir(const char *name)
7142 struct trace_array *tr;
7147 mutex_lock(&trace_types_lock);
7150 list_for_each_entry(tr, &ftrace_trace_arrays, list) {
7151 if (tr->name && strcmp(tr->name, name) == 0) {
7160 if (tr->ref || (tr->current_trace && tr->current_trace->ref))
7163 list_del(&tr->list);
7165 /* Disable all the flags that were enabled coming in */
7166 for (i = 0; i < TRACE_FLAGS_MAX_SIZE; i++) {
7167 if ((1 << i) & ZEROED_TRACE_FLAGS)
7168 set_tracer_flag(tr, 1 << i, 0);
7171 tracing_set_nop(tr);
7172 event_trace_del_tracer(tr);
7173 ftrace_clear_pids(tr);
7174 ftrace_destroy_function_files(tr);
7175 tracefs_remove_recursive(tr->dir);
7176 free_trace_buffers(tr);
7178 for (i = 0; i < tr->nr_topts; i++) {
7179 kfree(tr->topts[i].topts);
7183 free_cpumask_var(tr->tracing_cpumask);
7190 mutex_unlock(&trace_types_lock);
7195 static __init void create_trace_instances(struct dentry *d_tracer)
7197 trace_instance_dir = tracefs_create_instance_dir("instances", d_tracer,
7200 if (WARN_ON(!trace_instance_dir))
7205 init_tracer_tracefs(struct trace_array *tr, struct dentry *d_tracer)
7209 trace_create_file("available_tracers", 0444, d_tracer,
7210 tr, &show_traces_fops);
7212 trace_create_file("current_tracer", 0644, d_tracer,
7213 tr, &set_tracer_fops);
7215 trace_create_file("tracing_cpumask", 0644, d_tracer,
7216 tr, &tracing_cpumask_fops);
7218 trace_create_file("trace_options", 0644, d_tracer,
7219 tr, &tracing_iter_fops);
7221 trace_create_file("trace", 0644, d_tracer,
7224 trace_create_file("trace_pipe", 0444, d_tracer,
7225 tr, &tracing_pipe_fops);
7227 trace_create_file("buffer_size_kb", 0644, d_tracer,
7228 tr, &tracing_entries_fops);
7230 trace_create_file("buffer_total_size_kb", 0444, d_tracer,
7231 tr, &tracing_total_entries_fops);
7233 trace_create_file("free_buffer", 0200, d_tracer,
7234 tr, &tracing_free_buffer_fops);
7236 trace_create_file("trace_marker", 0220, d_tracer,
7237 tr, &tracing_mark_fops);
7239 trace_create_file("trace_clock", 0644, d_tracer, tr,
7242 trace_create_file("tracing_on", 0644, d_tracer,
7243 tr, &rb_simple_fops);
7245 create_trace_options_dir(tr);
7247 #if defined(CONFIG_TRACER_MAX_TRACE) || defined(CONFIG_HWLAT_TRACER)
7248 trace_create_file("tracing_max_latency", 0644, d_tracer,
7249 &tr->max_latency, &tracing_max_lat_fops);
7252 if (ftrace_create_function_files(tr, d_tracer))
7253 WARN(1, "Could not allocate function filter files");
7255 #ifdef CONFIG_TRACER_SNAPSHOT
7256 trace_create_file("snapshot", 0644, d_tracer,
7257 tr, &snapshot_fops);
7260 for_each_tracing_cpu(cpu)
7261 tracing_init_tracefs_percpu(tr, cpu);
7263 ftrace_init_tracefs(tr, d_tracer);
7266 static struct vfsmount *trace_automount(struct dentry *mntpt, void *ingore)
7268 struct vfsmount *mnt;
7269 struct file_system_type *type;
7272 * To maintain backward compatibility for tools that mount
7273 * debugfs to get to the tracing facility, tracefs is automatically
7274 * mounted to the debugfs/tracing directory.
7276 type = get_fs_type("tracefs");
7279 mnt = vfs_submount(mntpt, type, "tracefs", NULL);
7280 put_filesystem(type);
7289 * tracing_init_dentry - initialize top level trace array
7291 * This is called when creating files or directories in the tracing
7292 * directory. It is called via fs_initcall() by any of the boot up code
7293 * and expects to return the dentry of the top level tracing directory.
7295 struct dentry *tracing_init_dentry(void)
7297 struct trace_array *tr = &global_trace;
7299 /* The top level trace array uses NULL as parent */
7303 if (WARN_ON(!tracefs_initialized()) ||
7304 (IS_ENABLED(CONFIG_DEBUG_FS) &&
7305 WARN_ON(!debugfs_initialized())))
7306 return ERR_PTR(-ENODEV);
7309 * As there may still be users that expect the tracing
7310 * files to exist in debugfs/tracing, we must automount
7311 * the tracefs file system there, so older tools still
7312 * work with the newer kerenl.
7314 tr->dir = debugfs_create_automount("tracing", NULL,
7315 trace_automount, NULL);
7317 pr_warn_once("Could not create debugfs directory 'tracing'\n");
7318 return ERR_PTR(-ENOMEM);
7324 extern struct trace_enum_map *__start_ftrace_enum_maps[];
7325 extern struct trace_enum_map *__stop_ftrace_enum_maps[];
7327 static void __init trace_enum_init(void)
7331 len = __stop_ftrace_enum_maps - __start_ftrace_enum_maps;
7332 trace_insert_enum_map(NULL, __start_ftrace_enum_maps, len);
7335 #ifdef CONFIG_MODULES
7336 static void trace_module_add_enums(struct module *mod)
7338 if (!mod->num_trace_enums)
7342 * Modules with bad taint do not have events created, do
7343 * not bother with enums either.
7345 if (trace_module_has_bad_taint(mod))
7348 trace_insert_enum_map(mod, mod->trace_enums, mod->num_trace_enums);
7351 #ifdef CONFIG_TRACE_ENUM_MAP_FILE
7352 static void trace_module_remove_enums(struct module *mod)
7354 union trace_enum_map_item *map;
7355 union trace_enum_map_item **last = &trace_enum_maps;
7357 if (!mod->num_trace_enums)
7360 mutex_lock(&trace_enum_mutex);
7362 map = trace_enum_maps;
7365 if (map->head.mod == mod)
7367 map = trace_enum_jmp_to_tail(map);
7368 last = &map->tail.next;
7369 map = map->tail.next;
7374 *last = trace_enum_jmp_to_tail(map)->tail.next;
7377 mutex_unlock(&trace_enum_mutex);
7380 static inline void trace_module_remove_enums(struct module *mod) { }
7381 #endif /* CONFIG_TRACE_ENUM_MAP_FILE */
7383 static int trace_module_notify(struct notifier_block *self,
7384 unsigned long val, void *data)
7386 struct module *mod = data;
7389 case MODULE_STATE_COMING:
7390 trace_module_add_enums(mod);
7392 case MODULE_STATE_GOING:
7393 trace_module_remove_enums(mod);
7400 static struct notifier_block trace_module_nb = {
7401 .notifier_call = trace_module_notify,
7404 #endif /* CONFIG_MODULES */
7406 static __init int tracer_init_tracefs(void)
7408 struct dentry *d_tracer;
7410 trace_access_lock_init();
7412 d_tracer = tracing_init_dentry();
7413 if (IS_ERR(d_tracer))
7416 init_tracer_tracefs(&global_trace, d_tracer);
7417 ftrace_init_tracefs_toplevel(&global_trace, d_tracer);
7419 trace_create_file("tracing_thresh", 0644, d_tracer,
7420 &global_trace, &tracing_thresh_fops);
7422 trace_create_file("README", 0444, d_tracer,
7423 NULL, &tracing_readme_fops);
7425 trace_create_file("saved_cmdlines", 0444, d_tracer,
7426 NULL, &tracing_saved_cmdlines_fops);
7428 trace_create_file("saved_cmdlines_size", 0644, d_tracer,
7429 NULL, &tracing_saved_cmdlines_size_fops);
7433 trace_create_enum_file(d_tracer);
7435 #ifdef CONFIG_MODULES
7436 register_module_notifier(&trace_module_nb);
7439 #ifdef CONFIG_DYNAMIC_FTRACE
7440 trace_create_file("dyn_ftrace_total_info", 0444, d_tracer,
7441 &ftrace_update_tot_cnt, &tracing_dyn_info_fops);
7444 create_trace_instances(d_tracer);
7446 update_tracer_options(&global_trace);
7451 static int trace_panic_handler(struct notifier_block *this,
7452 unsigned long event, void *unused)
7454 if (ftrace_dump_on_oops)
7455 ftrace_dump(ftrace_dump_on_oops);
7459 static struct notifier_block trace_panic_notifier = {
7460 .notifier_call = trace_panic_handler,
7462 .priority = 150 /* priority: INT_MAX >= x >= 0 */
7465 static int trace_die_handler(struct notifier_block *self,
7471 if (ftrace_dump_on_oops)
7472 ftrace_dump(ftrace_dump_on_oops);
7480 static struct notifier_block trace_die_notifier = {
7481 .notifier_call = trace_die_handler,
7486 * printk is set to max of 1024, we really don't need it that big.
7487 * Nothing should be printing 1000 characters anyway.
7489 #define TRACE_MAX_PRINT 1000
7492 * Define here KERN_TRACE so that we have one place to modify
7493 * it if we decide to change what log level the ftrace dump
7496 #define KERN_TRACE KERN_EMERG
7499 trace_printk_seq(struct trace_seq *s)
7501 /* Probably should print a warning here. */
7502 if (s->seq.len >= TRACE_MAX_PRINT)
7503 s->seq.len = TRACE_MAX_PRINT;
7506 * More paranoid code. Although the buffer size is set to
7507 * PAGE_SIZE, and TRACE_MAX_PRINT is 1000, this is just
7508 * an extra layer of protection.
7510 if (WARN_ON_ONCE(s->seq.len >= s->seq.size))
7511 s->seq.len = s->seq.size - 1;
7513 /* should be zero ended, but we are paranoid. */
7514 s->buffer[s->seq.len] = 0;
7516 printk(KERN_TRACE "%s", s->buffer);
7521 void trace_init_global_iter(struct trace_iterator *iter)
7523 iter->tr = &global_trace;
7524 iter->trace = iter->tr->current_trace;
7525 iter->cpu_file = RING_BUFFER_ALL_CPUS;
7526 iter->trace_buffer = &global_trace.trace_buffer;
7528 if (iter->trace && iter->trace->open)
7529 iter->trace->open(iter);
7531 /* Annotate start of buffers if we had overruns */
7532 if (ring_buffer_overruns(iter->trace_buffer->buffer))
7533 iter->iter_flags |= TRACE_FILE_ANNOTATE;
7535 /* Output in nanoseconds only if we are using a clock in nanoseconds. */
7536 if (trace_clocks[iter->tr->clock_id].in_ns)
7537 iter->iter_flags |= TRACE_FILE_TIME_IN_NS;
7540 void ftrace_dump(enum ftrace_dump_mode oops_dump_mode)
7542 /* use static because iter can be a bit big for the stack */
7543 static struct trace_iterator iter;
7544 static atomic_t dump_running;
7545 struct trace_array *tr = &global_trace;
7546 unsigned int old_userobj;
7547 unsigned long flags;
7550 /* Only allow one dump user at a time. */
7551 if (atomic_inc_return(&dump_running) != 1) {
7552 atomic_dec(&dump_running);
7557 * Always turn off tracing when we dump.
7558 * We don't need to show trace output of what happens
7559 * between multiple crashes.
7561 * If the user does a sysrq-z, then they can re-enable
7562 * tracing with echo 1 > tracing_on.
7566 local_irq_save(flags);
7568 /* Simulate the iterator */
7569 trace_init_global_iter(&iter);
7571 for_each_tracing_cpu(cpu) {
7572 atomic_inc(&per_cpu_ptr(iter.trace_buffer->data, cpu)->disabled);
7575 old_userobj = tr->trace_flags & TRACE_ITER_SYM_USEROBJ;
7577 /* don't look at user memory in panic mode */
7578 tr->trace_flags &= ~TRACE_ITER_SYM_USEROBJ;
7580 switch (oops_dump_mode) {
7582 iter.cpu_file = RING_BUFFER_ALL_CPUS;
7585 iter.cpu_file = raw_smp_processor_id();
7590 printk(KERN_TRACE "Bad dumping mode, switching to all CPUs dump\n");
7591 iter.cpu_file = RING_BUFFER_ALL_CPUS;
7594 printk(KERN_TRACE "Dumping ftrace buffer:\n");
7596 /* Did function tracer already get disabled? */
7597 if (ftrace_is_dead()) {
7598 printk("# WARNING: FUNCTION TRACING IS CORRUPTED\n");
7599 printk("# MAY BE MISSING FUNCTION EVENTS\n");
7603 * We need to stop all tracing on all CPUS to read the
7604 * the next buffer. This is a bit expensive, but is
7605 * not done often. We fill all what we can read,
7606 * and then release the locks again.
7609 while (!trace_empty(&iter)) {
7612 printk(KERN_TRACE "---------------------------------\n");
7616 /* reset all but tr, trace, and overruns */
7617 memset(&iter.seq, 0,
7618 sizeof(struct trace_iterator) -
7619 offsetof(struct trace_iterator, seq));
7620 iter.iter_flags |= TRACE_FILE_LAT_FMT;
7623 if (trace_find_next_entry_inc(&iter) != NULL) {
7626 ret = print_trace_line(&iter);
7627 if (ret != TRACE_TYPE_NO_CONSUME)
7628 trace_consume(&iter);
7630 touch_nmi_watchdog();
7632 trace_printk_seq(&iter.seq);
7636 printk(KERN_TRACE " (ftrace buffer empty)\n");
7638 printk(KERN_TRACE "---------------------------------\n");
7641 tr->trace_flags |= old_userobj;
7643 for_each_tracing_cpu(cpu) {
7644 atomic_dec(&per_cpu_ptr(iter.trace_buffer->data, cpu)->disabled);
7646 atomic_dec(&dump_running);
7647 local_irq_restore(flags);
7649 EXPORT_SYMBOL_GPL(ftrace_dump);
7651 __init static int tracer_alloc_buffers(void)
7657 * Make sure we don't accidently add more trace options
7658 * than we have bits for.
7660 BUILD_BUG_ON(TRACE_ITER_LAST_BIT > TRACE_FLAGS_MAX_SIZE);
7662 if (!alloc_cpumask_var(&tracing_buffer_mask, GFP_KERNEL))
7665 if (!alloc_cpumask_var(&global_trace.tracing_cpumask, GFP_KERNEL))
7666 goto out_free_buffer_mask;
7668 /* Only allocate trace_printk buffers if a trace_printk exists */
7669 if (__stop___trace_bprintk_fmt != __start___trace_bprintk_fmt)
7670 /* Must be called before global_trace.buffer is allocated */
7671 trace_printk_init_buffers();
7673 /* To save memory, keep the ring buffer size to its minimum */
7674 if (ring_buffer_expanded)
7675 ring_buf_size = trace_buf_size;
7679 cpumask_copy(tracing_buffer_mask, cpu_possible_mask);
7680 cpumask_copy(global_trace.tracing_cpumask, cpu_all_mask);
7682 raw_spin_lock_init(&global_trace.start_lock);
7684 /* Used for event triggers */
7685 temp_buffer = ring_buffer_alloc(PAGE_SIZE, RB_FL_OVERWRITE);
7687 goto out_free_cpumask;
7689 if (trace_create_savedcmd() < 0)
7690 goto out_free_temp_buffer;
7692 /* TODO: make the number of buffers hot pluggable with CPUS */
7693 if (allocate_trace_buffers(&global_trace, ring_buf_size) < 0) {
7694 printk(KERN_ERR "tracer: failed to allocate ring buffer!\n");
7696 goto out_free_savedcmd;
7699 if (global_trace.buffer_disabled)
7702 if (trace_boot_clock) {
7703 ret = tracing_set_clock(&global_trace, trace_boot_clock);
7705 pr_warn("Trace clock %s not defined, going back to default\n",
7710 * register_tracer() might reference current_trace, so it
7711 * needs to be set before we register anything. This is
7712 * just a bootstrap of current_trace anyway.
7714 global_trace.current_trace = &nop_trace;
7716 global_trace.max_lock = (arch_spinlock_t)__ARCH_SPIN_LOCK_UNLOCKED;
7718 ftrace_init_global_array_ops(&global_trace);
7720 init_trace_flags_index(&global_trace);
7722 register_tracer(&nop_trace);
7724 /* All seems OK, enable tracing */
7725 tracing_disabled = 0;
7727 atomic_notifier_chain_register(&panic_notifier_list,
7728 &trace_panic_notifier);
7730 register_die_notifier(&trace_die_notifier);
7732 global_trace.flags = TRACE_ARRAY_FL_GLOBAL;
7734 INIT_LIST_HEAD(&global_trace.systems);
7735 INIT_LIST_HEAD(&global_trace.events);
7736 list_add(&global_trace.list, &ftrace_trace_arrays);
7738 apply_trace_boot_options();
7740 register_snapshot_cmd();
7745 free_saved_cmdlines_buffer(savedcmd);
7746 out_free_temp_buffer:
7747 ring_buffer_free(temp_buffer);
7749 free_cpumask_var(global_trace.tracing_cpumask);
7750 out_free_buffer_mask:
7751 free_cpumask_var(tracing_buffer_mask);
7756 void __init trace_init(void)
7758 if (tracepoint_printk) {
7759 tracepoint_print_iter =
7760 kmalloc(sizeof(*tracepoint_print_iter), GFP_KERNEL);
7761 if (WARN_ON(!tracepoint_print_iter))
7762 tracepoint_printk = 0;
7764 tracer_alloc_buffers();
7768 __init static int clear_boot_tracer(void)
7771 * The default tracer at boot buffer is an init section.
7772 * This function is called in lateinit. If we did not
7773 * find the boot tracer, then clear it out, to prevent
7774 * later registration from accessing the buffer that is
7775 * about to be freed.
7777 if (!default_bootup_tracer)
7780 printk(KERN_INFO "ftrace bootup tracer '%s' not registered.\n",
7781 default_bootup_tracer);
7782 default_bootup_tracer = NULL;
7787 fs_initcall(tracer_init_tracefs);
7788 late_initcall_sync(clear_boot_tracer);