blob: eb11011b5292add880af7038800560aa29c5a674 [file] [log] [blame]
/*
* Infrastructure for profiling code inserted by 'gcc -pg'.
*
* Copyright (C) 2007-2008 Steven Rostedt <srostedt@redhat.com>
* Copyright (C) 2004-2008 Ingo Molnar <mingo@redhat.com>
*
* Originally ported from the -rt patch by:
* Copyright (C) 2007 Arnaldo Carvalho de Melo <acme@redhat.com>
*
* Based on code in the latency_tracer, that is:
*
* Copyright (C) 2004-2006 Ingo Molnar
* Copyright (C) 2004 Nadia Yvette Chambers
*/
#include <linux/stop_machine.h>
#include <linux/clocksource.h>
#include <linux/kallsyms.h>
#include <linux/seq_file.h>
#include <linux/suspend.h>
#include <linux/tracefs.h>
#include <linux/hardirq.h>
#include <linux/kthread.h>
#include <linux/uaccess.h>
#include <linux/bsearch.h>
#include <linux/module.h>
#include <linux/ftrace.h>
#include <linux/sysctl.h>
#include <linux/slab.h>
#include <linux/ctype.h>
#include <linux/sort.h>
#include <linux/list.h>
#include <linux/hash.h>
#include <linux/rcupdate.h>
#include <trace/events/sched.h>
#include <asm/setup.h>
#include "trace_output.h"
#include "trace_stat.h"
#define FTRACE_WARN_ON(cond) \
({ \
int ___r = cond; \
if (WARN_ON(___r)) \
ftrace_kill(); \
___r; \
})
#define FTRACE_WARN_ON_ONCE(cond) \
({ \
int ___r = cond; \
if (WARN_ON_ONCE(___r)) \
ftrace_kill(); \
___r; \
})
/* hash bits for specific function selection */
#define FTRACE_HASH_BITS 7
#define FTRACE_FUNC_HASHSIZE (1 << FTRACE_HASH_BITS)
#define FTRACE_HASH_DEFAULT_BITS 10
#define FTRACE_HASH_MAX_BITS 12
#define FL_GLOBAL_CONTROL_MASK (FTRACE_OPS_FL_CONTROL)
#ifdef CONFIG_DYNAMIC_FTRACE
#define INIT_OPS_HASH(opsname) \
.func_hash = &opsname.local_hash, \
.local_hash.regex_lock = __MUTEX_INITIALIZER(opsname.local_hash.regex_lock),
#define ASSIGN_OPS_HASH(opsname, val) \
.func_hash = val, \
.local_hash.regex_lock = __MUTEX_INITIALIZER(opsname.local_hash.regex_lock),
#else
#define INIT_OPS_HASH(opsname)
#define ASSIGN_OPS_HASH(opsname, val)
#endif
static struct ftrace_ops ftrace_list_end __read_mostly = {
.func = ftrace_stub,
.flags = FTRACE_OPS_FL_RECURSION_SAFE | FTRACE_OPS_FL_STUB,
INIT_OPS_HASH(ftrace_list_end)
};
/* ftrace_enabled is a method to turn ftrace on or off */
int ftrace_enabled __read_mostly;
static int last_ftrace_enabled;
/* Current function tracing op */
struct ftrace_ops *function_trace_op __read_mostly = &ftrace_list_end;
/* What to set function_trace_op to */
static struct ftrace_ops *set_function_trace_op;
/* List for set_ftrace_pid's pids. */
LIST_HEAD(ftrace_pids);
struct ftrace_pid {
struct list_head list;
struct pid *pid;
};
static bool ftrace_pids_enabled(void)
{
return !list_empty(&ftrace_pids);
}
static void ftrace_update_trampoline(struct ftrace_ops *ops);
/*
* ftrace_disabled is set when an anomaly is discovered.
* ftrace_disabled is much stronger than ftrace_enabled.
*/
static int ftrace_disabled __read_mostly;
static DEFINE_MUTEX(ftrace_lock);
static struct ftrace_ops *ftrace_control_list __read_mostly = &ftrace_list_end;
static struct ftrace_ops *ftrace_ops_list __read_mostly = &ftrace_list_end;
ftrace_func_t ftrace_trace_function __read_mostly = ftrace_stub;
static struct ftrace_ops global_ops;
static struct ftrace_ops control_ops;
static void ftrace_ops_recurs_func(unsigned long ip, unsigned long parent_ip,
struct ftrace_ops *op, struct pt_regs *regs);
#if ARCH_SUPPORTS_FTRACE_OPS
static void ftrace_ops_list_func(unsigned long ip, unsigned long parent_ip,
struct ftrace_ops *op, struct pt_regs *regs);
#else
/* See comment below, where ftrace_ops_list_func is defined */
static void ftrace_ops_no_ops(unsigned long ip, unsigned long parent_ip);
#define ftrace_ops_list_func ((ftrace_func_t)ftrace_ops_no_ops)
#endif
/*
* Traverse the ftrace_global_list, invoking all entries. The reason that we
* can use rcu_dereference_raw_notrace() is that elements removed from this list
* are simply leaked, so there is no need to interact with a grace-period
* mechanism. The rcu_dereference_raw_notrace() calls are needed to handle
* concurrent insertions into the ftrace_global_list.
*
* Silly Alpha and silly pointer-speculation compiler optimizations!
*/
#define do_for_each_ftrace_op(op, list) \
op = rcu_dereference_raw_notrace(list); \
do
/*
* Optimized for just a single item in the list (as that is the normal case).
*/
#define while_for_each_ftrace_op(op) \
while (likely(op = rcu_dereference_raw_notrace((op)->next)) && \
unlikely((op) != &ftrace_list_end))
static inline void ftrace_ops_init(struct ftrace_ops *ops)
{
#ifdef CONFIG_DYNAMIC_FTRACE
if (!(ops->flags & FTRACE_OPS_FL_INITIALIZED)) {
mutex_init(&ops->local_hash.regex_lock);
ops->func_hash = &ops->local_hash;
ops->flags |= FTRACE_OPS_FL_INITIALIZED;
}
#endif
}
/**
* ftrace_nr_registered_ops - return number of ops registered
*
* Returns the number of ftrace_ops registered and tracing functions
*/
int ftrace_nr_registered_ops(void)
{
struct ftrace_ops *ops;
int cnt = 0;
mutex_lock(&ftrace_lock);
for (ops = ftrace_ops_list;
ops != &ftrace_list_end; ops = ops->next)
cnt++;
mutex_unlock(&ftrace_lock);
return cnt;
}
static void ftrace_pid_func(unsigned long ip, unsigned long parent_ip,
struct ftrace_ops *op, struct pt_regs *regs)
{
if (!test_tsk_trace_trace(current))
return;
op->saved_func(ip, parent_ip, op, regs);
}
/**
* clear_ftrace_function - reset the ftrace function
*
* This NULLs the ftrace function and in essence stops
* tracing. There may be lag
*/
void clear_ftrace_function(void)
{
ftrace_trace_function = ftrace_stub;
}
static void control_ops_disable_all(struct ftrace_ops *ops)
{
int cpu;
for_each_possible_cpu(cpu)
*per_cpu_ptr(ops->disabled, cpu) = 1;
}
static int control_ops_alloc(struct ftrace_ops *ops)
{
int __percpu *disabled;
disabled = alloc_percpu(int);
if (!disabled)
return -ENOMEM;
ops->disabled = disabled;
control_ops_disable_all(ops);
return 0;
}
static void ftrace_sync(struct work_struct *work)
{
/*
* This function is just a stub to implement a hard force
* of synchronize_sched(). This requires synchronizing
* tasks even in userspace and idle.
*
* Yes, function tracing is rude.
*/
}
static void ftrace_sync_ipi(void *data)
{
/* Probably not needed, but do it anyway */
smp_rmb();
}
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
static void update_function_graph_func(void);
#else
static inline void update_function_graph_func(void) { }
#endif
static ftrace_func_t ftrace_ops_get_list_func(struct ftrace_ops *ops)
{
/*
* If this is a dynamic ops or we force list func,
* then it needs to call the list anyway.
*/
if (ops->flags & FTRACE_OPS_FL_DYNAMIC || FTRACE_FORCE_LIST_FUNC)
return ftrace_ops_list_func;
return ftrace_ops_get_func(ops);
}
static void update_ftrace_function(void)
{
ftrace_func_t func;
/*
* Prepare the ftrace_ops that the arch callback will use.
* If there's only one ftrace_ops registered, the ftrace_ops_list
* will point to the ops we want.
*/
set_function_trace_op = ftrace_ops_list;
/* If there's no ftrace_ops registered, just call the stub function */
if (ftrace_ops_list == &ftrace_list_end) {
func = ftrace_stub;
/*
* If we are at the end of the list and this ops is
* recursion safe and not dynamic and the arch supports passing ops,
* then have the mcount trampoline call the function directly.
*/
} else if (ftrace_ops_list->next == &ftrace_list_end) {
func = ftrace_ops_get_list_func(ftrace_ops_list);
} else {
/* Just use the default ftrace_ops */
set_function_trace_op = &ftrace_list_end;
func = ftrace_ops_list_func;
}
update_function_graph_func();
/* If there's no change, then do nothing more here */
if (ftrace_trace_function == func)
return;
/*
* If we are using the list function, it doesn't care
* about the function_trace_ops.
*/
if (func == ftrace_ops_list_func) {
ftrace_trace_function = func;
/*
* Don't even bother setting function_trace_ops,
* it would be racy to do so anyway.
*/
return;
}
#ifndef CONFIG_DYNAMIC_FTRACE
/*
* For static tracing, we need to be a bit more careful.
* The function change takes affect immediately. Thus,
* we need to coorditate the setting of the function_trace_ops
* with the setting of the ftrace_trace_function.
*
* Set the function to the list ops, which will call the
* function we want, albeit indirectly, but it handles the
* ftrace_ops and doesn't depend on function_trace_op.
*/
ftrace_trace_function = ftrace_ops_list_func;
/*
* Make sure all CPUs see this. Yes this is slow, but static
* tracing is slow and nasty to have enabled.
*/
schedule_on_each_cpu(ftrace_sync);
/* Now all cpus are using the list ops. */
function_trace_op = set_function_trace_op;
/* Make sure the function_trace_op is visible on all CPUs */
smp_wmb();
/* Nasty way to force a rmb on all cpus */
smp_call_function(ftrace_sync_ipi, NULL, 1);
/* OK, we are all set to update the ftrace_trace_function now! */
#endif /* !CONFIG_DYNAMIC_FTRACE */
ftrace_trace_function = func;
}
int using_ftrace_ops_list_func(void)
{
return ftrace_trace_function == ftrace_ops_list_func;
}
static void add_ftrace_ops(struct ftrace_ops **list, struct ftrace_ops *ops)
{
ops->next = *list;
/*
* We are entering ops into the list but another
* CPU might be walking that list. We need to make sure
* the ops->next pointer is valid before another CPU sees
* the ops pointer included into the list.
*/
rcu_assign_pointer(*list, ops);
}
static int remove_ftrace_ops(struct ftrace_ops **list, struct ftrace_ops *ops)
{
struct ftrace_ops **p;
/*
* If we are removing the last function, then simply point
* to the ftrace_stub.
*/
if (*list == ops && ops->next == &ftrace_list_end) {
*list = &ftrace_list_end;
return 0;
}
for (p = list; *p != &ftrace_list_end; p = &(*p)->next)
if (*p == ops)
break;
if (*p != ops)
return -1;
*p = (*p)->next;
return 0;
}
static void add_ftrace_list_ops(struct ftrace_ops **list,
struct ftrace_ops *main_ops,
struct ftrace_ops *ops)
{
int first = *list == &ftrace_list_end;
add_ftrace_ops(list, ops);
if (first)
add_ftrace_ops(&ftrace_ops_list, main_ops);
}
static int remove_ftrace_list_ops(struct ftrace_ops **list,
struct ftrace_ops *main_ops,
struct ftrace_ops *ops)
{
int ret = remove_ftrace_ops(list, ops);
if (!ret && *list == &ftrace_list_end)
ret = remove_ftrace_ops(&ftrace_ops_list, main_ops);
return ret;
}
static void ftrace_update_trampoline(struct ftrace_ops *ops);
static int __register_ftrace_function(struct ftrace_ops *ops)
{
if (ops->flags & FTRACE_OPS_FL_DELETED)
return -EINVAL;
if (WARN_ON(ops->flags & FTRACE_OPS_FL_ENABLED))
return -EBUSY;
#ifndef CONFIG_DYNAMIC_FTRACE_WITH_REGS
/*
* If the ftrace_ops specifies SAVE_REGS, then it only can be used
* if the arch supports it, or SAVE_REGS_IF_SUPPORTED is also set.
* Setting SAVE_REGS_IF_SUPPORTED makes SAVE_REGS irrelevant.
*/
if (ops->flags & FTRACE_OPS_FL_SAVE_REGS &&
!(ops->flags & FTRACE_OPS_FL_SAVE_REGS_IF_SUPPORTED))
return -EINVAL;
if (ops->flags & FTRACE_OPS_FL_SAVE_REGS_IF_SUPPORTED)
ops->flags |= FTRACE_OPS_FL_SAVE_REGS;
#endif
if (!core_kernel_data((unsigned long)ops))
ops->flags |= FTRACE_OPS_FL_DYNAMIC;
if (ops->flags & FTRACE_OPS_FL_CONTROL) {
if (control_ops_alloc(ops))
return -ENOMEM;
add_ftrace_list_ops(&ftrace_control_list, &control_ops, ops);
/* The control_ops needs the trampoline update */
ops = &control_ops;
} else
add_ftrace_ops(&ftrace_ops_list, ops);
/* Always save the function, and reset at unregistering */
ops->saved_func = ops->func;
if (ops->flags & FTRACE_OPS_FL_PID && ftrace_pids_enabled())
ops->func = ftrace_pid_func;
ftrace_update_trampoline(ops);
if (ftrace_enabled)
update_ftrace_function();
return 0;
}
static int __unregister_ftrace_function(struct ftrace_ops *ops)
{
int ret;
if (WARN_ON(!(ops->flags & FTRACE_OPS_FL_ENABLED)))
return -EBUSY;
if (ops->flags & FTRACE_OPS_FL_CONTROL) {
ret = remove_ftrace_list_ops(&ftrace_control_list,
&control_ops, ops);
} else
ret = remove_ftrace_ops(&ftrace_ops_list, ops);
if (ret < 0)
return ret;
if (ftrace_enabled)
update_ftrace_function();
ops->func = ops->saved_func;
return 0;
}
static void ftrace_update_pid_func(void)
{
bool enabled = ftrace_pids_enabled();
struct ftrace_ops *op;
/* Only do something if we are tracing something */
if (ftrace_trace_function == ftrace_stub)
return;
do_for_each_ftrace_op(op, ftrace_ops_list) {
if (op->flags & FTRACE_OPS_FL_PID) {
op->func = enabled ? ftrace_pid_func :
op->saved_func;
ftrace_update_trampoline(op);
}
} while_for_each_ftrace_op(op);
update_ftrace_function();
}
#ifdef CONFIG_FUNCTION_PROFILER
struct ftrace_profile {
struct hlist_node node;
unsigned long ip;
unsigned long counter;
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
unsigned long long time;
unsigned long long time_squared;
#endif
};
struct ftrace_profile_page {
struct ftrace_profile_page *next;
unsigned long index;
struct ftrace_profile records[];
};
struct ftrace_profile_stat {
atomic_t disabled;
struct hlist_head *hash;
struct ftrace_profile_page *pages;
struct ftrace_profile_page *start;
struct tracer_stat stat;
};
#define PROFILE_RECORDS_SIZE \
(PAGE_SIZE - offsetof(struct ftrace_profile_page, records))
#define PROFILES_PER_PAGE \
(PROFILE_RECORDS_SIZE / sizeof(struct ftrace_profile))
static int ftrace_profile_enabled __read_mostly;
/* ftrace_profile_lock - synchronize the enable and disable of the profiler */
static DEFINE_MUTEX(ftrace_profile_lock);
static DEFINE_PER_CPU(struct ftrace_profile_stat, ftrace_profile_stats);
#define FTRACE_PROFILE_HASH_BITS 10
#define FTRACE_PROFILE_HASH_SIZE (1 << FTRACE_PROFILE_HASH_BITS)
static void *
function_stat_next(void *v, int idx)
{
struct ftrace_profile *rec = v;
struct ftrace_profile_page *pg;
pg = (struct ftrace_profile_page *)((unsigned long)rec & PAGE_MASK);
again:
if (idx != 0)
rec++;
if ((void *)rec >= (void *)&pg->records[pg->index]) {
pg = pg->next;
if (!pg)
return NULL;
rec = &pg->records[0];
if (!rec->counter)
goto again;
}
return rec;
}
static void *function_stat_start(struct tracer_stat *trace)
{
struct ftrace_profile_stat *stat =
container_of(trace, struct ftrace_profile_stat, stat);
if (!stat || !stat->start)
return NULL;
return function_stat_next(&stat->start->records[0], 0);
}
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
/* function graph compares on total time */
static int function_stat_cmp(void *p1, void *p2)
{
struct ftrace_profile *a = p1;
struct ftrace_profile *b = p2;
if (a->time < b->time)
return -1;
if (a->time > b->time)
return 1;
else
return 0;
}
#else
/* not function graph compares against hits */
static int function_stat_cmp(void *p1, void *p2)
{
struct ftrace_profile *a = p1;
struct ftrace_profile *b = p2;
if (a->counter < b->counter)
return -1;
if (a->counter > b->counter)
return 1;
else
return 0;
}
#endif
static int function_stat_headers(struct seq_file *m)
{
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
seq_puts(m, " Function "
"Hit Time Avg s^2\n"
" -------- "
"--- ---- --- ---\n");
#else
seq_puts(m, " Function Hit\n"
" -------- ---\n");
#endif
return 0;
}
static int function_stat_show(struct seq_file *m, void *v)
{
struct ftrace_profile *rec = v;
char str[KSYM_SYMBOL_LEN];
int ret = 0;
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
static struct trace_seq s;
unsigned long long avg;
unsigned long long stddev;
#endif
mutex_lock(&ftrace_profile_lock);
/* we raced with function_profile_reset() */
if (unlikely(rec->counter == 0)) {
ret = -EBUSY;
goto out;
}
kallsyms_lookup(rec->ip, NULL, NULL, NULL, str);
seq_printf(m, " %-30.30s %10lu", str, rec->counter);
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
seq_puts(m, " ");
avg = rec->time;
do_div(avg, rec->counter);
/* Sample standard deviation (s^2) */
if (rec->counter <= 1)
stddev = 0;
else {
/*
* Apply Welford's method:
* s^2 = 1 / (n * (n-1)) * (n * \Sum (x_i)^2 - (\Sum x_i)^2)
*/
stddev = rec->counter * rec->time_squared -
rec->time * rec->time;
/*
* Divide only 1000 for ns^2 -> us^2 conversion.
* trace_print_graph_duration will divide 1000 again.
*/
do_div(stddev, rec->counter * (rec->counter - 1) * 1000);
}
trace_seq_init(&s);
trace_print_graph_duration(rec->time, &s);
trace_seq_puts(&s, " ");
trace_print_graph_duration(avg, &s);
trace_seq_puts(&s, " ");
trace_print_graph_duration(stddev, &s);
trace_print_seq(m, &s);
#endif
seq_putc(m, '\n');
out:
mutex_unlock(&ftrace_profile_lock);
return ret;
}
static void ftrace_profile_reset(struct ftrace_profile_stat *stat)
{
struct ftrace_profile_page *pg;
pg = stat->pages = stat->start;
while (pg) {
memset(pg->records, 0, PROFILE_RECORDS_SIZE);
pg->index = 0;
pg = pg->next;
}
memset(stat->hash, 0,
FTRACE_PROFILE_HASH_SIZE * sizeof(struct hlist_head));
}
int ftrace_profile_pages_init(struct ftrace_profile_stat *stat)
{
struct ftrace_profile_page *pg;
int functions;
int pages;
int i;
/* If we already allocated, do nothing */
if (stat->pages)
return 0;
stat->pages = (void *)get_zeroed_page(GFP_KERNEL);
if (!stat->pages)
return -ENOMEM;
#ifdef CONFIG_DYNAMIC_FTRACE
functions = ftrace_update_tot_cnt;
#else
/*
* We do not know the number of functions that exist because
* dynamic tracing is what counts them. With past experience
* we have around 20K functions. That should be more than enough.
* It is highly unlikely we will execute every function in
* the kernel.
*/
functions = 20000;
#endif
pg = stat->start = stat->pages;
pages = DIV_ROUND_UP(functions, PROFILES_PER_PAGE);
for (i = 1; i < pages; i++) {
pg->next = (void *)get_zeroed_page(GFP_KERNEL);
if (!pg->next)
goto out_free;
pg = pg->next;
}
return 0;
out_free:
pg = stat->start;
while (pg) {
unsigned long tmp = (unsigned long)pg;
pg = pg->next;
free_page(tmp);
}
stat->pages = NULL;
stat->start = NULL;
return -ENOMEM;
}
static int ftrace_profile_init_cpu(int cpu)
{
struct ftrace_profile_stat *stat;
int size;
stat = &per_cpu(ftrace_profile_stats, cpu);
if (stat->hash) {
/* If the profile is already created, simply reset it */
ftrace_profile_reset(stat);
return 0;
}
/*
* We are profiling all functions, but usually only a few thousand
* functions are hit. We'll make a hash of 1024 items.
*/
size = FTRACE_PROFILE_HASH_SIZE;
stat->hash = kzalloc(sizeof(struct hlist_head) * size, GFP_KERNEL);
if (!stat->hash)
return -ENOMEM;
/* Preallocate the function profiling pages */
if (ftrace_profile_pages_init(stat) < 0) {
kfree(stat->hash);
stat->hash = NULL;
return -ENOMEM;
}
return 0;
}
static int ftrace_profile_init(void)
{
int cpu;
int ret = 0;
for_each_possible_cpu(cpu) {
ret = ftrace_profile_init_cpu(cpu);
if (ret)
break;
}
return ret;
}
/* interrupts must be disabled */
static struct ftrace_profile *
ftrace_find_profiled_func(struct ftrace_profile_stat *stat, unsigned long ip)
{
struct ftrace_profile *rec;
struct hlist_head *hhd;
unsigned long key;
key = hash_long(ip, FTRACE_PROFILE_HASH_BITS);
hhd = &stat->hash[key];
if (hlist_empty(hhd))
return NULL;
hlist_for_each_entry_rcu_notrace(rec, hhd, node) {
if (rec->ip == ip)
return rec;
}
return NULL;
}
static void ftrace_add_profile(struct ftrace_profile_stat *stat,
struct ftrace_profile *rec)
{
unsigned long key;
key = hash_long(rec->ip, FTRACE_PROFILE_HASH_BITS);
hlist_add_head_rcu(&rec->node, &stat->hash[key]);
}
/*
* The memory is already allocated, this simply finds a new record to use.
*/
static struct ftrace_profile *
ftrace_profile_alloc(struct ftrace_profile_stat *stat, unsigned long ip)
{
struct ftrace_profile *rec = NULL;
/* prevent recursion (from NMIs) */
if (atomic_inc_return(&stat->disabled) != 1)
goto out;
/*
* Try to find the function again since an NMI
* could have added it
*/
rec = ftrace_find_profiled_func(stat, ip);
if (rec)
goto out;
if (stat->pages->index == PROFILES_PER_PAGE) {
if (!stat->pages->next)
goto out;
stat->pages = stat->pages->next;
}
rec = &stat->pages->records[stat->pages->index++];
rec->ip = ip;
ftrace_add_profile(stat, rec);
out:
atomic_dec(&stat->disabled);
return rec;
}
static void
function_profile_call(unsigned long ip, unsigned long parent_ip,
struct ftrace_ops *ops, struct pt_regs *regs)
{
struct ftrace_profile_stat *stat;
struct ftrace_profile *rec;
unsigned long flags;
if (!ftrace_profile_enabled)
return;
local_irq_save(flags);
stat = this_cpu_ptr(&ftrace_profile_stats);
if (!stat->hash || !ftrace_profile_enabled)
goto out;
rec = ftrace_find_profiled_func(stat, ip);
if (!rec) {
rec = ftrace_profile_alloc(stat, ip);
if (!rec)
goto out;
}
rec->counter++;
out:
local_irq_restore(flags);
}
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
static int profile_graph_entry(struct ftrace_graph_ent *trace)
{
function_profile_call(trace->func, 0, NULL, NULL);
return 1;
}
static void profile_graph_return(struct ftrace_graph_ret *trace)
{
struct ftrace_profile_stat *stat;
unsigned long long calltime;
struct ftrace_profile *rec;
unsigned long flags;
local_irq_save(flags);
stat = this_cpu_ptr(&ftrace_profile_stats);
if (!stat->hash || !ftrace_profile_enabled)
goto out;
/* If the calltime was zero'd ignore it */
if (!trace->calltime)
goto out;
calltime = trace->rettime - trace->calltime;
if (!(trace_flags & TRACE_ITER_GRAPH_TIME)) {
int index;
index = trace->depth;
/* Append this call time to the parent time to subtract */
if (index)
current->ret_stack[index - 1].subtime += calltime;
if (current->ret_stack[index].subtime < calltime)
calltime -= current->ret_stack[index].subtime;
else
calltime = 0;
}
rec = ftrace_find_profiled_func(stat, trace->func);
if (rec) {
rec->time += calltime;
rec->time_squared += calltime * calltime;
}
out:
local_irq_restore(flags);
}
static int register_ftrace_profiler(void)
{
return register_ftrace_graph(&profile_graph_return,
&profile_graph_entry);
}
static void unregister_ftrace_profiler(void)
{
unregister_ftrace_graph();
}
#else
static struct ftrace_ops ftrace_profile_ops __read_mostly = {
.func = function_profile_call,
.flags = FTRACE_OPS_FL_RECURSION_SAFE | FTRACE_OPS_FL_INITIALIZED,
INIT_OPS_HASH(ftrace_profile_ops)
};
static int register_ftrace_profiler(void)
{
return register_ftrace_function(&ftrace_profile_ops);
}
static void unregister_ftrace_profiler(void)
{
unregister_ftrace_function(&ftrace_profile_ops);
}
#endif /* CONFIG_FUNCTION_GRAPH_TRACER */
static ssize_t
ftrace_profile_write(struct file *filp, const char __user *ubuf,
size_t cnt, loff_t *ppos)
{
unsigned long val;
int ret;
ret = kstrtoul_from_user(ubuf, cnt, 10, &val);
if (ret)
return ret;
val = !!val;
mutex_lock(&ftrace_profile_lock);
if (ftrace_profile_enabled ^ val) {
if (val) {
ret = ftrace_profile_init();
if (ret < 0) {
cnt = ret;
goto out;
}
ret = register_ftrace_profiler();
if (ret < 0) {
cnt = ret;
goto out;
}
ftrace_profile_enabled = 1;
} else {
ftrace_profile_enabled = 0;
/*
* unregister_ftrace_profiler calls stop_machine
* so this acts like an synchronize_sched.
*/
unregister_ftrace_profiler();
}
}
out:
mutex_unlock(&ftrace_profile_lock);
*ppos += cnt;
return cnt;
}
static ssize_t
ftrace_profile_read(struct file *filp, char __user *ubuf,
size_t cnt, loff_t *ppos)
{
char buf[64]; /* big enough to hold a number */
int r;
r = sprintf(buf, "%u\n", ftrace_profile_enabled);
return simple_read_from_buffer(ubuf, cnt, ppos, buf, r);
}
static const struct file_operations ftrace_profile_fops = {
.open = tracing_open_generic,
.read = ftrace_profile_read,
.write = ftrace_profile_write,
.llseek = default_llseek,
};
/* used to initialize the real stat files */
static struct tracer_stat function_stats __initdata = {
.name = "functions",
.stat_start = function_stat_start,
.stat_next = function_stat_next,
.stat_cmp = function_stat_cmp,
.stat_headers = function_stat_headers,
.stat_show = function_stat_show
};
static __init void ftrace_profile_tracefs(struct dentry *d_tracer)
{
struct ftrace_profile_stat *stat;
struct dentry *entry;
char *name;
int ret;
int cpu;
for_each_possible_cpu(cpu) {
stat = &per_cpu(ftrace_profile_stats, cpu);
/* allocate enough for function name + cpu number */
name = kmalloc(32, GFP_KERNEL);
if (!name) {
/*
* The files created are permanent, if something happens
* we still do not free memory.
*/
WARN(1,
"Could not allocate stat file for cpu %d\n",
cpu);
return;
}
stat->stat = function_stats;
snprintf(name, 32, "function%d", cpu);
stat->stat.name = name;
ret = register_stat_tracer(&stat->stat);
if (ret) {
WARN(1,
"Could not register function stat for cpu %d\n",
cpu);
kfree(name);
return;
}
}
entry = tracefs_create_file("function_profile_enabled", 0644,
d_tracer, NULL, &ftrace_profile_fops);
if (!entry)
pr_warning("Could not create tracefs "
"'function_profile_enabled' entry\n");
}
#else /* CONFIG_FUNCTION_PROFILER */
static __init void ftrace_profile_tracefs(struct dentry *d_tracer)
{
}
#endif /* CONFIG_FUNCTION_PROFILER */
static struct pid * const ftrace_swapper_pid = &init_struct_pid;
#ifdef CONFIG_FUNCTION_GRAPH_TRACER
static int ftrace_graph_active;
#else
# define ftrace_graph_active 0
#endif
#ifdef CONFIG_DYNAMIC_FTRACE
static struct ftrace_ops *removed_ops;
/*
* Set when doing a global update, like enabling all recs or disabling them.
* It is not set when just updating a single ftrace_ops.
*/
static bool update_all_ops;
#ifndef CONFIG_FTRACE_MCOUNT_RECORD
# error Dynamic ftrace depends on MCOUNT_RECORD
#endif
static struct hlist_head ftrace_func_hash[FTRACE_FUNC_HASHSIZE] __read_mostly;
struct ftrace_func_probe {
struct hlist_node node;
struct ftrace_probe_ops *ops;
unsigned long flags;
unsigned long ip;
void *data;
struct list_head free_list;
};
struct ftrace_func_entry {
struct hlist_node hlist;
unsigned long ip;
};
struct ftrace_hash {
unsigned long size_bits;
struct hlist_head *buckets;
unsigned long count;
struct rcu_head rcu;
};
/*
* We make these constant because no one should touch them,
* but they are used as the default "empty hash", to avoid allocating
* it all the time. These are in a read only section such that if
* anyone does try to modify it, it will cause an exception.
*/
static const struct hlist_head empty_buckets[1];
static const struct ftrace_hash empty_hash = {
.buckets = (struct hlist_head *)empty_buckets,
};
#define EMPTY_HASH ((struct ftrace_hash *)&empty_hash)
static struct ftrace_ops global_ops = {
.func = ftrace_stub,
.local_hash.notrace_hash = EMPTY_HASH,
.local_hash.filter_hash = EMPTY_HASH,
INIT_OPS_HASH(global_ops)
.flags = FTRACE_OPS_FL_RECURSION_SAFE |
FTRACE_OPS_FL_INITIALIZED |
FTRACE_OPS_FL_PID,
};
/*
* This is used by __kernel_text_address() to return true if the
* address is on a dynamically allocated trampoline that would
* not return true for either core_kernel_text() or
* is_module_text_address().
*/
bool is_ftrace_trampoline(unsigned long addr)
{
struct ftrace_ops *op;
bool ret = false;
/*
* Some of the ops may be dynamically allocated,
* they are freed after a synchronize_sched().
*/
preempt_disable_notrace();
do_for_each_ftrace_op(op, ftrace_ops_list) {
/*
* This is to check for dynamically allocated trampolines.
* Trampolines that are in kernel text will have
* core_kernel_text() return true.
*/
if (op->trampoline && op->trampoline_size)
if (addr >= op->trampoline &&
addr < op->trampoline + op->trampoline_size) {
ret = true;
goto out;
}
} while_for_each_ftrace_op(op);
out:
preempt_enable_notrace();
return ret;
}
struct ftrace_page {
struct ftrace_page *next;
struct dyn_ftrace *records;
int index;
int size;
};
#define ENTRY_SIZE sizeof(struct dyn_ftrace)
#define ENTRIES_PER_PAGE (PAGE_SIZE / ENTRY_SIZE)
/* estimate from running different kernels */
#define NR_TO_INIT 10000
static struct ftrace_page *ftrace_pages_start;
static struct ftrace_page *ftrace_pages;
static bool __always_inline ftrace_hash_empty(struct ftrace_hash *hash)
{
return !hash || !hash->count;
}
static struct ftrace_func_entry *
ftrace_lookup_ip(struct ftrace_hash *hash, unsigned long ip)
{
unsigned long key;
struct ftrace_func_entry *entry;
struct hlist_head *hhd;
if (ftrace_hash_empty(hash))
return NULL;
if (hash->size_bits > 0)
key = hash_long(ip, hash->size_bits);
else
key = 0;
hhd = &hash->buckets[key];
hlist_for_each_entry_rcu_notrace(entry, hhd, hlist) {
if (entry->ip == ip)
return entry;
}
return NULL;
}
static void __add_hash_entry(struct ftrace_hash *hash,
struct ftrace_func_entry *entry)
{
struct hlist_head *hhd;
unsigned long key;
if (hash->size_bits)
key = hash_long(entry->ip, hash->size_bits);
else
key = 0;
hhd = &hash->buckets[key];
hlist_add_head(&entry->hlist, hhd);
hash->count++;
}
static int add_hash_entry(struct ftrace_hash *hash, unsigned long ip)
{
struct ftrace_func_entry *entry;
entry = kmalloc(sizeof(*entry), GFP_KERNEL);
if (!entry)
return -ENOMEM;
entry->ip = ip;
__add_hash_entry(hash, entry);
return 0;
}
static void
free_hash_entry(struct ftrace_hash *hash,
struct ftrace_func_entry *entry)
{
hlist_del(&entry->hlist);
kfree(entry);
hash->count--;
}
static void
remove_hash_entry(struct ftrace_hash *hash,
struct ftrace_func_entry *entry)
{
hlist_del(&entry->hlist);
hash->count--;
}
static void ftrace_hash_clear(struct ftrace_hash *hash)
{
struct hlist_head *hhd;
struct hlist_node *tn;
struct ftrace_func_entry *entry;
int size = 1 << hash->size_bits;
int i;
if (!hash->count)
return;
for (i = 0; i < size; i++) {
hhd = &hash->buckets[i];
hlist_for_each_entry_safe(entry, tn, hhd, hlist)
free_hash_entry(hash, entry);
}
FTRACE_WARN_ON(hash->count);
}
static void free_ftrace_hash(struct ftrace_hash *hash)
{
if (!hash || hash == EMPTY_HASH)
return;
ftrace_hash_clear(hash);
kfree(hash->buckets);
kfree(hash);
}
static void __free_ftrace_hash_rcu(struct rcu_head *rcu)
{
struct ftrace_hash *hash;
hash = container_of(rcu, struct ftrace_hash, rcu);
free_ftrace_hash(hash);
}
static void free_ftrace_hash_rcu(struct ftrace_hash *hash)
{
if (!hash || hash == EMPTY_HASH)
return;
call_rcu_sched(&hash->rcu, __free_ftrace_hash_rcu);
}
void ftrace_free_filter(struct ftrace_ops *ops)
{
ftrace_ops_init(ops);
free_ftrace_hash(ops->func_hash->filter_hash);
free_ftrace_hash(ops->func_hash->notrace_hash);
}
static struct ftrace_hash *alloc_ftrace_hash(int size_bits)
{
struct ftrace_hash *hash;
int size;
hash = kzalloc(sizeof(*hash), GFP_KERNEL);
if (!hash)
return NULL;
size = 1 << size_bits;
hash->buckets = kcalloc(size, sizeof(*hash->buckets), GFP_KERNEL);
if (!hash->buckets) {
kfree(hash);
return NULL;
}
hash->size_bits = size_bits;
return hash;
}
static struct ftrace_hash *
alloc_and_copy_ftrace_hash(int size_bits, struct ftrace_hash *hash)
{
struct ftrace_func_entry *entry;
struct ftrace_hash *new_hash;
int size;
int ret;
int i;
new_hash = alloc_ftrace_hash(size_bits);
if (!new_hash)
return NULL;
/* Empty hash? */
if (ftrace_hash_empty(hash))
return new_hash;
size = 1 << hash->size_bits;
for (i = 0; i < size; i++) {
hlist_for_each_entry(entry, &hash->buckets[i], hlist) {
ret = add_hash_entry(new_hash, entry->ip);
if (ret < 0)
goto free_hash;
}
}
FTRACE_WARN_ON(new_hash->count != hash->count);
return new_hash;
free_hash:
free_ftrace_hash(new_hash);
return NULL;
}
static void
ftrace_hash_rec_disable_modify(struct ftrace_ops *ops, int filter_hash);
static void
ftrace_hash_rec_enable_modify(struct ftrace_ops *ops, int filter_hash);
static int ftrace_hash_ipmodify_update(struct ftrace_ops *ops,
struct ftrace_hash *new_hash);
static int
ftrace_hash_move(struct ftrace_ops *ops, int enable,
struct ftrace_hash **dst, struct ftrace_hash *src)
{
struct ftrace_func_entry *entry;
struct hlist_node *tn;
struct hlist_head *hhd;
struct ftrace_hash *new_hash;
int size = src->count;
int bits = 0;
int ret;
int i;
/* Reject setting notrace hash on IPMODIFY ftrace_ops */
if (ops->flags & FTRACE_OPS_FL_IPMODIFY && !enable)
return -EINVAL;
/*
* If the new source is empty, just free dst and assign it
* the empty_hash.
*/
if (!src->count) {
new_hash = EMPTY_HASH;
goto update;
}
/*
* Make the hash size about 1/2 the # found
*/
for (size /= 2; size; size >>= 1)
bits++;
/* Don't allocate too much */
if (bits > FTRACE_HASH_MAX_BITS)
bits = FTRACE_HASH_MAX_BITS;
new_hash = alloc_ftrace_hash(bits);
if (!new_hash)
return -ENOMEM;
size = 1 << src->size_bits;
for (i = 0; i < size; i++) {
hhd = &src->buckets[i];
hlist_for_each_entry_safe(entry, tn, hhd, hlist) {
remove_hash_entry(src, entry);
__add_hash_entry(new_hash, entry);
}
}
update:
/* Make sure this can be applied if it is IPMODIFY ftrace_ops */
if (enable) {
/* IPMODIFY should be updated only when filter_hash updating */
ret = ftrace_hash_ipmodify_update(ops, new_hash);
if (ret < 0) {
free_ftrace_hash(new_hash);
return ret;
}
}
/*
* Remove the current set, update the hash and add
* them back.
*/
ftrace_hash_rec_disable_modify(ops, enable);
rcu_assign_pointer(*dst, new_hash);
ftrace_hash_rec_enable_modify(ops, enable);
return 0;
}
static bool hash_contains_ip(unsigned long ip,
struct ftrace_ops_hash *hash)
{
/*
* The function record is a match if it exists in the filter
* hash and not in the notrace hash. Note, an emty hash is
* considered a match for the filter hash, but an empty
* notrace hash is considered not in the notrace hash.
*/
return (ftrace_hash_empty(hash->filter_hash) ||
ftrace_lookup_ip(hash->filter_hash, ip)) &&
(ftrace_hash_empty(hash->notrace_hash) ||
!ftrace_lookup_ip(hash->notrace_hash, ip));
}
/*
* Test the hashes for this ops to see if we want to call
* the ops->func or not.
*
* It's a match if the ip is in the ops->filter_hash or
* the filter_hash does not exist or is empty,
* AND
* the ip is not in the ops->notrace_hash.
*
* This needs to be called with preemption disabled as
* the hashes are freed with call_rcu_sched().
*/
static int
ftrace_ops_test(struct ftrace_ops *ops, unsigned long ip, void *regs)
{
struct ftrace_ops_hash hash;
int ret;
#ifdef CONFIG_DYNAMIC_FTRACE_WITH_REGS
/*
* There's a small race when adding ops that the ftrace handler
* that wants regs, may be called without them. We can not
* allow that handler to be called if regs is NULL.
*/
if (regs == NULL && (ops->flags & FTRACE_OPS_FL_SAVE_REGS))
return 0;
#endif
hash.filter_hash = rcu_dereference_raw_notrace(ops->func_hash->filter_hash);
hash.notrace_hash = rcu_dereference_raw_notrace(ops->func_hash->notrace_hash);
if (hash_contains_ip(ip, &hash))
ret = 1;
else
ret = 0;
return ret;
}
/*
* This is a double for. Do not use 'break' to break out of the loop,
* you must use a goto.
*/
#define do_for_each_ftrace_rec(pg, rec) \
for (pg = ftrace_pages_start; pg; pg = pg->next) { \
int _____i; \
for (_____i = 0; _____i < pg->index; _____i++) { \
rec = &pg->records[_____i];
#define while_for_each_ftrace_rec() \
} \
}
static int ftrace_cmp_recs(const void *a, const void *b)
{
const struct dyn_ftrace *key = a;
const struct dyn_ftrace *rec = b;
if (key->flags < rec->ip)
return -1;
if (key->ip >= rec->ip + MCOUNT_INSN_SIZE)
return 1;
return 0;
}
static unsigned long ftrace_location_range(unsigned long start, unsigned long end)
{
struct ftrace_page *pg;
struct dyn_ftrace *rec;
struct dyn_ftrace key;
key.ip = start;
key.flags = end; /* overload flags, as it is unsigned long */
for (pg = ftrace_pages_start; pg; pg = pg->next) {
if (end < pg->records[0].ip ||
start >= (pg->records[pg->index - 1].ip + MCOUNT_INSN_SIZE))
continue;
rec = bsearch(&key, pg->records, pg->index,
sizeof(struct dyn_ftrace),
ftrace_cmp_recs);
if (rec)
return rec->ip;
}
return 0;
}
/**
* ftrace_location - return true if the ip giving is a traced location
* @ip: the instruction pointer to check
*
* Returns rec->ip if @ip given is a pointer to a ftrace location.
* That is, the instruction that is either a NOP or call to
* the function tracer. It checks the ftrace internal tables to
* determine if the address belongs or not.
*/
unsigned long ftrace_location(unsigned long ip)
{
return ftrace_location_range(ip, ip);
}
/**
* ftrace_text_reserved - return true if range contains an ftrace location
* @start: start of range to search
* @end: end of range to search (inclusive). @end points to the last byte to check.
*
* Returns 1 if @start and @end contains a ftrace location.
* That is, the instruction that is either a NOP or call to
* the function tracer. It checks the ftrace internal tables to
* determine if the address belongs or not.
*/
int ftrace_text_reserved(const void *start, const void *end)
{
unsigned long ret;
ret = ftrace_location_range((unsigned long)start,
(unsigned long)end);
return (int)!!ret;
}
/* Test if ops registered to this rec needs regs */
static bool test_rec_ops_needs_regs(struct dyn_ftrace *rec)
{
struct ftrace_ops *ops;
bool keep_regs = false;
for (ops = ftrace_ops_list;
ops != &ftrace_list_end; ops = ops->next) {
/* pass rec in as regs to have non-NULL val */
if (ftrace_ops_test(ops, rec->ip, rec)) {
if (ops->flags & FTRACE_OPS_FL_SAVE_REGS) {
keep_regs = true;
break;
}
}
}
return keep_regs;
}
static void __ftrace_hash_rec_update(struct ftrace_ops *ops,
int filter_hash,
bool inc)
{
struct ftrace_hash *hash;
struct ftrace_hash *other_hash;
struct ftrace_page *pg;
struct dyn_ftrace *rec;
int count = 0;
int all = 0;
/* Only update if the ops has been registered */
if (!(ops->flags & FTRACE_OPS_FL_ENABLED))
return;
/*
* In the filter_hash case:
* If the count is zero, we update all records.
* Otherwise we just update the items in the hash.
*
* In the notrace_hash case:
* We enable the update in the hash.
* As disabling notrace means enabling the tracing,
* and enabling notrace means disabling, the inc variable
* gets inversed.
*/
if (filter_hash) {
hash = ops->func_hash->filter_hash;
other_hash = ops->func_hash->notrace_hash;
if (ftrace_hash_empty(hash))
all = 1;
} else {
inc = !inc;
hash = ops->func_hash->notrace_hash;
other_hash = ops->func_hash->filter_hash;
/*
* If the notrace hash has no items,
* then there's nothing to do.
*/
if (ftrace_hash_empty(hash))
return;
}
do_for_each_ftrace_rec(pg, rec) {
int in_other_hash = 0;
int in_hash = 0;
int match = 0;
if (all) {
/*
* Only the filter_hash affects all records.
* Update if the record is not in the notrace hash.
*/
if (!other_hash || !ftrace_lookup_ip(other_hash, rec->ip))
match = 1;
} else {
in_hash = !!ftrace_lookup_ip(hash, rec->ip);
in_other_hash = !!ftrace_lookup_ip(other_hash, rec->ip);
/*
* If filter_hash is set, we want to match all functions
* that are in the hash but not in the other hash.
*
* If filter_hash is not set, then we are decrementing.
* That means we match anything that is in the hash
* and also in the other_hash. That is, we need to turn
* off functions in the other hash because they are disabled
* by this hash.
*/
if (filter_hash && in_hash && !in_other_hash)
match = 1;
else if (!filter_hash && in_hash &&
(in_other_hash || ftrace_hash_empty(other_hash)))
match = 1;
}
if (!match)
continue;
if (inc) {
rec->flags++;
if (FTRACE_WARN_ON(ftrace_rec_count(rec) == FTRACE_REF_MAX))
return;
/*
* If there's only a single callback registered to a
* function, and the ops has a trampoline registered
* for it, then we can call it directly.
*/
if (ftrace_rec_count(rec) == 1 && ops->trampoline)
rec->flags |= FTRACE_FL_TRAMP;
else
/*
* If we are adding another function callback
* to this function, and the previous had a
* custom trampoline in use, then we need to go
* back to the default trampoline.
*/
rec->flags &= ~FTRACE_FL_TRAMP;
/*
* If any ops wants regs saved for this function
* then all ops will get saved regs.
*/
if (ops->flags & FTRACE_OPS_FL_SAVE_REGS)
rec->flags |= FTRACE_FL_REGS;
} else {
if (FTRACE_WARN_ON(ftrace_rec_count(rec) == 0))
return;
rec->flags--;
/*
* If the rec had REGS enabled and the ops that is
* being removed had REGS set, then see if there is
* still any ops for this record that wants regs.
* If not, we can stop recording them.
*/
if (ftrace_rec_count(rec) > 0 &&
rec->flags & FTRACE_FL_REGS &&
ops->flags & FTRACE_OPS_FL_SAVE_REGS) {
if (!test_rec_ops_needs_regs(rec))
rec->flags &= ~FTRACE_FL_REGS;
}
/*
* If the rec had TRAMP enabled, then it needs to
* be cleared. As TRAMP can only be enabled iff
* there is only a single ops attached to it.
* In otherwords, always disable it on decrementing.
* In the future, we may set it if rec count is
* decremented to one, and the ops that is left
* has a trampoline.
*/
rec->flags &= ~FTRACE_FL_TRAMP;
/*
* flags will be cleared in ftrace_check_record()
* if rec count is zero.
*/
}
count++;
/* Shortcut, if we handled all records, we are done. */
if (!all && count == hash->count)
return;
} while_for_each_ftrace_rec();
}
static void ftrace_hash_rec_disable(struct ftrace_ops *ops,
int filter_hash)
{
__ftrace_hash_rec_update(ops, filter_hash, 0);
}
static void ftrace_hash_rec_enable(struct ftrace_ops *ops,
int filter_hash)
{
__ftrace_hash_rec_update(ops, filter_hash, 1);
}
static void ftrace_hash_rec_update_modify(struct ftrace_ops *ops,
int filter_hash, int inc)
{
struct ftrace_ops *op;
__ftrace_hash_rec_update(ops, filter_hash, inc);
if (ops->func_hash != &global_ops.local_hash)
return;
/*
* If the ops shares the global_ops hash, then we need to update
* all ops that are enabled and use this hash.
*/
do_for_each_ftrace_op(op, ftrace_ops_list) {
/* Already done */
if (op == ops)
continue;
if (op->func_hash == &global_ops.local_hash)
__ftrace_hash_rec_update(op, filter_hash, inc);
} while_for_each_ftrace_op(op);
}
static void ftrace_hash_rec_disable_modify(struct ftrace_ops *ops,
int filter_hash)
{
ftrace_hash_rec_update_modify(ops, filter_hash, 0);
}
static void ftrace_hash_rec_enable_modify(struct ftrace_ops *ops,
int filter_hash)
{
ftrace_hash_rec_update_modify(ops, filter_hash, 1);
}
/*
* Try to update IPMODIFY flag on each ftrace_rec. Return 0 if it is OK
* or no-needed to update, -EBUSY if it detects a conflict of the flag
* on a ftrace_rec, and -EINVAL if the new_hash tries to trace all recs.
* Note that old_hash and new_hash has below meanings
* - If the hash is NULL, it hits all recs (if IPMODIFY is set, this is rejected)
* - If the hash is EMPTY_HASH, it hits nothing
* - Anything else hits the recs which match the hash entries.
*/
static int __ftrace_hash_update_ipmodify(struct ftrace_ops *ops,
struct ftrace_hash *old_hash,
struct ftrace_hash *new_hash)
{
struct ftrace_page *pg;
struct dyn_ftrace *rec, *end = NULL;
int in_old, in_new;
/* Only update if the ops has been registered */
if (!(ops->flags & FTRACE_OPS_FL_ENABLED))
return 0;
if (!(ops->flags & FTRACE_OPS_FL_IPMODIFY))
return 0;
/*
* Since the IPMODIFY is a very address sensitive action, we do not
* allow ftrace_ops to set all functions to new hash.
*/
if (!new_hash || !old_hash)
return -EINVAL;
/* Update rec->flags */
do_for_each_ftrace_rec(pg, rec) {
/* We need to update only differences of filter_hash */
in_old = !!ftrace_lookup_ip(old_hash, rec->ip);
in_new = !!ftrace_lookup_ip(new_hash, rec->ip);
if (in_old == in_new)
continue;
if (in_new) {
/* New entries must ensure no others are using it */
if (rec->flags & FTRACE_FL_IPMODIFY)
goto rollback;
rec->flags |= FTRACE_FL_IPMODIFY;
} else /* Removed entry */
rec->flags &= ~FTRACE_FL_IPMODIFY;
} while_for_each_ftrace_rec();
return 0;
rollback:
end = rec;
/* Roll back what we did above */
do_for_each_ftrace_rec(pg, rec) {
if (rec == end)
goto err_out;
in_old = !!ftrace_lookup_ip(old_hash, rec->ip);
in_new = !!ftrace_lookup_ip(new_hash, rec->ip);
if (in_old == in_new)
continue;
if (in_new)
rec->flags &= ~FTRACE_FL_IPMODIFY;
else
rec->flags |= FTRACE_FL_IPMODIFY;
} while_for_each_ftrace_rec();
err_out:
return -EBUSY;
}
static int ftrace_hash_ipmodify_enable(struct ftrace_ops *ops)
{
struct ftrace_hash *hash = ops->func_hash->filter_hash;
if (ftrace_hash_empty(hash))
hash = NULL;
return __ftrace_hash_update_ipmodify(ops, EMPTY_HASH, hash);
}
/* Disabling always succeeds */
static void ftrace_hash_ipmodify_disable(struct ftrace_ops *ops)
{
struct ftrace_hash *hash = ops->func_hash->filter_hash;
if (ftrace_hash_empty(hash))
hash = NULL;
__ftrace_hash_update_ipmodify(ops, hash, EMPTY_HASH);
}
static int ftrace_hash_ipmodify_update(struct ftrace_ops *ops,
struct ftrace_hash *new_hash)
{
struct ftrace_hash *old_hash = ops->func_hash->filter_hash;
if (ftrace_hash_empty(old_hash))
old_hash = NULL;
if (ftrace_hash_empty(new_hash))
new_hash = NULL;
return __ftrace_hash_update_ipmodify(ops, old_hash, new_hash);
}
static void print_ip_ins(const char *fmt, unsigned char *p)
{
int i;
printk(KERN_CONT "%s", fmt);
for (i = 0; i < MCOUNT_INSN_SIZE; i++)
printk(KERN_CONT "%s%02x", i ? ":" : "", p[i]);
}
static struct ftrace_ops *
ftrace_find_tramp_ops_any(struct dyn_ftrace *rec);
/**
* ftrace_bug - report and shutdown function tracer
* @failed: The failed type (EFAULT, EINVAL, EPERM)
* @rec: The record that failed
*
* The arch code that enables or disables the function tracing
* can call ftrace_bug() when it has detected a problem in
* modifying the code. @failed should be one of either:
* EFAULT - if the problem happens on reading the @ip address
* EINVAL - if what is read at @ip is not what was expected
* EPERM - if the problem happens on writting to the @ip address
*/
void ftrace_bug(int failed, struct dyn_ftrace *rec)
{
unsigned long ip = rec ? rec->ip : 0;
switch (failed) {
case -EFAULT:
FTRACE_WARN_ON_ONCE(1);
pr_info("ftrace faulted on modifying ");
print_ip_sym(ip);
break;
case -EINVAL:
FTRACE_WARN_ON_ONCE(1);
pr_info("ftrace failed to modify ");
print_ip_sym(ip);
print_ip_ins(" actual: ", (unsigned char *)ip);
pr_cont("\n");
break;
case -EPERM:
FTRACE_WARN_ON_ONCE(1);
pr_info("ftrace faulted on writing ");
print_ip_sym(ip);
break;
default:
FTRACE_WARN_ON_ONCE(1);
pr_info("ftrace faulted on unknown error ");
print_ip_sym(ip);
}
if (rec) {
struct ftrace_ops *ops = NULL;
pr_info("ftrace record flags: %lx\n", rec->flags);
pr_cont(" (%ld)%s", ftrace_rec_count(rec),
rec->flags & FTRACE_FL_REGS ? " R" : " ");
if (rec->flags & FTRACE_FL_TRAMP_EN) {
ops = ftrace_find_tramp_ops_any(rec);
if (ops)
pr_cont("\ttramp: %pS",
(void *)ops->trampoline);
else
pr_cont("\ttramp: ERROR!");
}
ip = ftrace_get_addr_curr(rec);
pr_cont(" expected tramp: %lx\n", ip);
}
}
static int ftrace_check_record(struct dyn_ftrace *rec, int enable, int update)
{
unsigned long flag = 0UL;
/*
* If we are updating calls:
*
* If the record has a ref count, then we need to enable it
* because someone is using it.
*
* Otherwise we make sure its disabled.
*
* If we are disabling calls, then disable all records that
* are enabled.
*/
if (enable && ftrace_rec_count(rec))
flag = FTRACE_FL_ENABLED;
/*
* If enabling and the REGS flag does not match the REGS_EN, or
* the TRAMP flag doesn't match the TRAMP_EN, then do not ignore
* this record. Set flags to fail the compare against ENABLED.
*/
if (flag) {
if (!(rec->flags & FTRACE_FL_REGS) !=
!(rec->flags & FTRACE_FL_REGS_EN))
flag |= FTRACE_FL_REGS;
if (!(rec->flags & FTRACE_FL_TRAMP) !=
!(rec->flags & FTRACE_FL_TRAMP_EN))
flag |= FTRACE_FL_TRAMP;
}
/* If the state of this record hasn't changed, then do nothing */
if ((rec->flags & FTRACE_FL_ENABLED) == flag)
return FTRACE_UPDATE_IGNORE;
if (flag) {
/* Save off if rec is being enabled (for return value) */
flag ^= rec->flags & FTRACE_FL_ENABLED;
if (update) {
rec->flags |= FTRACE_FL_ENABLED;
if (flag & FTRACE_FL_REGS) {
if (rec->flags & FTRACE_FL_REGS)
rec->flags |= FTRACE_FL_REGS_EN;
else
rec->flags &= ~FTRACE_FL_REGS_EN;
}
if (flag & FTRACE_FL_TRAMP) {
if (rec->flags & FTRACE_FL_TRAMP)
rec->flags |= FTRACE_FL_TRAMP_EN;
else
rec->flags &= ~FTRACE_FL_TRAMP_EN;
}
}
/*
* If this record is being updated from a nop, then
* return UPDATE_MAKE_CALL.
* Otherwise,
* return UPDATE_MODIFY_CALL to tell the caller to convert
* from the save regs, to a non-save regs function or
* vice versa, or from a trampoline call.
*/
if (flag & FTRACE_FL_ENABLED)
return FTRACE_UPDATE_MAKE_CALL;
return FTRACE_UPDATE_MODIFY_CALL;
}
if (update) {
/* If there's no more users, clear all flags */
if (!ftrace_rec_count(rec))
rec->flags = 0;
else
/*
* Just disable the record, but keep the ops TRAMP
* and REGS states. The _EN flags must be disabled though.
*/
rec->flags &= ~(FTRACE_FL_ENABLED | FTRACE_FL_TRAMP_EN |
FTRACE_FL_REGS_EN);
}
return FTRACE_UPDATE_MAKE_NOP;
}
/**
* ftrace_update_record, set a record that now is tracing or not
* @rec: the record to update
* @enable: set to 1 if the record is tracing, zero to force disable
*
* The records that represent all functions that can be traced need
* to be updated when tracing has been enabled.
*/
int ftrace_update_record(struct dyn_ftrace *rec, int enable)
{
return ftrace_check_record(rec, enable, 1);
}
/**
* ftrace_test_record, check if the record has been enabled or not
* @rec: the record to test
* @enable: set to 1 to check if enabled, 0 if it is disabled
*
* The arch code may need to test if a record is already set to
* tracing to determine how to modify the function code that it
* represents.
*/
int ftrace_test_record(struct dyn_ftrace *rec, int enable)
{
return ftrace_check_record(rec, enable, 0);
}
static struct ftrace_ops *
ftrace_find_tramp_ops_any(struct dyn_ftrace *rec)
{
struct ftrace_ops *op;
unsigned long ip = rec->ip;
do_for_each_ftrace_op(op, ftrace_ops_list) {
if (!op->trampoline)
continue;
if (hash_contains_ip(ip, op->func_hash))
return op;
} while_for_each_ftrace_op(op);
return NULL;
}
static struct ftrace_ops *
ftrace_find_tramp_ops_curr(struct dyn_ftrace *rec)
{
struct ftrace_ops *op;
unsigned long ip = rec->ip;
/*
* Need to check removed ops first.
* If they are being removed, and this rec has a tramp,
* and this rec is in the ops list, then it would be the
* one with the tramp.
*/
if (removed_ops) {
if (hash_contains_ip(ip, &removed_ops->old_hash))
return removed_ops;
}
/*
* Need to find the current trampoline for a rec.
* Now, a trampoline is only attached to a rec if there
* was a single 'ops' attached to it. But this can be called
* when we are adding another op to the rec or removing the
* current one. Thus, if the op is being added, we can
* ignore it because it hasn't attached itself to the rec
* yet.
*
* If an ops is being modified (hooking to different functions)
* then we don't care about the new functions that are being
* added, just the old ones (that are probably being removed).
*
* If we are adding an ops to a function that already is using
* a trampoline, it needs to be removed (trampolines are only
* for single ops connected), then an ops that is not being
* modified also needs to be checked.
*/
do_for_each_ftrace_op(op, ftrace_ops_list) {
if (!op->trampoline)
continue;
/*
* If the ops is being added, it hasn't gotten to
* the point to be removed from this tree yet.
*/
if (op->flags & FTRACE_OPS_FL_ADDING)
continue;
/*
* If the ops is being modified and is in the old
* hash, then it is probably being removed from this
* function.
*/
if ((op->flags & FTRACE_OPS_FL_MODIFYING) &&
hash_contains_ip(ip, &op->old_hash))
return op;
/*
* If the ops is not being added or modified, and it's
* in its normal filter hash, then this must be the one
* we want!
*/
if (!(op->flags & FTRACE_OPS_FL_MODIFYING) &&
hash_contains_ip(ip, op->func_hash))
return op;
} while_for_each_ftrace_op(op);
return NULL;
}
static struct ftrace_ops *
ftrace_find_tramp_ops_new(struct dyn_ftrace *rec)
{
struct ftrace_ops *op;
unsigned long ip = rec->ip;
do_for_each_ftrace_op(op, ftrace_ops_list) {
/* pass rec in as regs to have non-NULL val */
if (hash_contains_ip(ip, op->func_hash))
return op;
} while_for_each_ftrace_op(op);
return NULL;
}
/**
* ftrace_get_addr_new - Get the call address to set to
* @rec: The ftrace record descriptor
*
* If the record has the FTRACE_FL_REGS set, that means that it
* wants to convert to a callback that saves all regs. If FTRACE_FL_REGS
* is not not set, then it wants to convert to the normal callback.
*
* Returns the address of the trampoline to set to
*/
unsigned long ftrace_get_addr_new(struct dyn_ftrace *rec)
{
struct ftrace_ops *ops;
/* Trampolines take precedence over regs */
if (rec->flags & FTRACE_FL_TRAMP) {
ops = ftrace_find_tramp_ops_new(rec);
if (FTRACE_WARN_ON(!ops || !ops->trampoline)) {
pr_warn("Bad trampoline accounting at: %p (%pS) (%lx)\n",
(void *)rec->ip, (void *)rec->ip, rec->flags);
/* Ftrace is shutting down, return anything */
return (unsigned long)FTRACE_ADDR;
}
return ops->trampoline;
}
if (rec->flags & FTRACE_FL_REGS)
return (unsigned long)FTRACE_REGS_ADDR;
else
return (unsigned long)FTRACE_ADDR;
}
/**
* ftrace_get_addr_curr - Get the call address that is already there
* @rec: The ftrace record descriptor
*
* The FTRACE_FL_REGS_EN is set when the record already points to
* a function that saves all the regs. Basically the '_EN' version
* represents the current state of the function.
*
* Returns the address of the trampoline that is currently being called
*/
unsigned long ftrace_get_addr_curr(struct dyn_ftrace *rec)
{
struct ftrace_ops *ops;
/* Trampolines take precedence over regs */
if (rec->flags & FTRACE_FL_TRAMP_EN) {
ops = ftrace_find_tramp_ops_curr(rec);
if (FTRACE_WARN_ON(!ops)) {
pr_warning("Bad trampoline accounting at: %p (%pS)\n",
(void *)rec->ip, (void *)rec->ip);
/* Ftrace is shutting down, return anything */
return (unsigned long)FTRACE_ADDR;
}
return ops->trampoline;
}
if (rec->flags & FTRACE_FL_REGS_EN)
return (unsigned long)FTRACE_REGS_ADDR;
else
return (unsigned long)FTRACE_ADDR;
}
static int
__ftrace_replace_code(struct dyn_ftrace *rec, int enable)
{
unsigned long ftrace_old_addr;
unsigned long ftrace_addr;
int ret;
ftrace_addr = ftrace_get_addr_new(rec);
/* This needs to be done before we call ftrace_update_record */
ftrace_old_addr = ftrace_get_addr_curr(rec);
ret = ftrace_update_record(rec, enable);
switch (ret) {
case FTRACE_UPDATE_IGNORE:
return 0;
case FTRACE_UPDATE_MAKE_CALL:
return ftrace_make_call(rec, ftrace_addr);
case FTRACE_UPDATE_MAKE_NOP:
return ftrace_make_nop(NULL, rec, ftrace_old_addr);
case FTRACE_UPDATE_MODIFY_CALL:
return ftrace_modify_call(rec, ftrace_old_addr, ftrace_addr);
}
return -1; /* unknow ftrace bug */
}
void __weak ftrace_replace_code(int enable)
{
struct dyn_ftrace *rec;
struct ftrace_page *pg;
int failed;
if (unlikely(ftrace_disabled))
return;
do_for_each_ftrace_rec(pg, rec) {
failed = __ftrace_replace_code(rec, enable);
if (failed) {
ftrace_bug(failed, rec);
/* Stop processing */
return;
}
} while_for_each_ftrace_rec();
}
struct ftrace_rec_iter {
struct ftrace_page *pg;
int index;
};
/**
* ftrace_rec_iter_start, start up iterating over traced functions
*
* Returns an iterator handle that is used to iterate over all
* the records that represent address locations where functions
* are traced.
*
* May return NULL if no records are available.
*/
struct ftrace_rec_iter *ftrace_rec_iter_start(void)
{
/*
* We only use a single iterator.
* Protected by the ftrace_lock mutex.
*/
static struct ftrace_rec_iter ftrace_rec_iter;
struct ftrace_rec_iter *iter = &ftrace_rec_iter;
iter->pg = ftrace_pages_start;
iter->index = 0;
/* Could have empty pages */
while (iter->pg && !iter->pg->index)
iter->pg = iter->pg->next;
if (!iter->pg)
return NULL;
return iter;
}
/**
* ftrace_rec_iter_next, get the next record to process.
* @iter: The handle to the iterator.
*
* Returns the next iterator after the given iterator @iter.
*/
struct ftrace_rec_iter *ftrace_rec_iter_next(struct ftrace_rec_iter *iter)
{
iter->index++;
if (iter->index >= iter->pg->index) {
iter->pg = iter->pg->next;
iter->index = 0;
/* Could have empty pages */
while (iter->pg && !iter->pg->index)
iter->pg = iter->pg->next;
}
if (!iter->pg)
return NULL;
return iter;
}
/**
* ftrace_rec_iter_record, get the record at the iterator location
* @iter: The current iterator location
*
* Returns the record that the current @iter is at.
*/
struct dyn_ftrace *ftrace_rec_iter_record(struct ftrace_rec_iter *iter)
{
return &iter->pg->records[iter->index];
}
static int
ftrace_code_disable(struct module *mod, struct dyn_ftrace *rec)
{
int ret;
if (unlikely(ftrace_disabled))
return 0;
ret = ftrace_make_nop(mod, rec, MCOUNT_ADDR);
if (ret) {
ftrace_bug(ret, rec);
return 0;
}
return 1;
}
/*
* archs can override this function if they must do something
* before the modifying code is performed.
*/
int __weak ftrace_arch_code_modify_prepare(void)
{
return 0;
}
/*
* archs can override this function if they must do something
* after the modifying code is performed.
*/
int __weak ftrace_arch_code_modify_post_process(void)
{
return 0;
}
void ftrace_modify_all_code(int command)
{
int update = command & FTRACE_UPDATE_TRACE_FUNC;
int err = 0;
/*
* If the ftrace_caller calls a ftrace_ops func directly,
* we need to make sure that it only traces functions it
* expects to trace. When doing the switch of functions,
* we need to update to the ftrace_ops_list_func first
* before the transition between old and new calls are set,
* as the ftrace_ops_list_func will check the ops hashes
* to make sure the ops are having the right functions
* traced.
*/
if (update) {
err = ftrace_update_ftrace_func(ftrace_ops_list_func);
if (FTRACE_WARN_ON(err))
return;
}
if (command & FTRACE_UPDATE_CALLS)
ftrace_replace_code(1);
else if (command & FTRACE_DISABLE_CALLS)
ftrace_replace_code(0);
if (update && ftrace_trace_function != ftrace_ops_list_func) {
function_trace_op = set_function_trace_op;
smp_wmb();
/* If irqs are disabled, we are in stop machine */
if (!irqs_disabled())
smp_call_function(ftrace_sync_ipi, NULL, 1);
err = ftrace_update_ftrace_func(ftrace_trace_function);
if (FTRACE_WARN_ON(err))
return;
}
if (command & FTRACE_START_FUNC_RET)
err = ftrace_enable_ftrace_graph_caller();
else if (command & FTRACE_STOP_FUNC_RET)
err = ftrace_disable_ftrace_graph_caller();
FTRACE_WARN_ON(err);
}
static int __ftrace_modify_code(void *data)
{
int *command = data;
ftrace_modify_all_code(*command);
return 0;
}
/**
* ftrace_run_stop_machine, go back to the stop machine method
* @command: The command to tell ftrace what to do
*
* If an arch needs to fall back to the stop machine method, the
* it can call this function.
*/
void ftrace_run_stop_machine(int command)
{
stop_machine(__ftrace_modify_code, &command, NULL);
}
/**
* arch_ftrace_update_code, modify the code to trace or not trace
* @command: The command that needs to be done
*
* Archs can override this function if it does not need to
* run stop_machine() to modify code.
*/
void __weak arch_ftrace_update_code(int command)
{
ftrace_run_stop_machine(command);
}
static void ftrace_run_update_code(int command)
{
int ret;
ret = ftrace_arch_code_modify_prepare();
FTRACE_WARN_ON(ret);
if (ret)
return;
/*
* By default we use stop_machine() to modify the code.
* But archs can do what ever they want as long as it
* is safe. The stop_machine() is the safest, but also
* produces the most overhead.
*/
arch_ftrace_update_code(command);
ret = ftrace_arch_code_modify_post_process();
FTRACE_WARN_ON(ret);
}
static void ftrace_run_modify_code(struct ftrace_ops *ops, int command,
struct ftrace_ops_hash *old_hash)
{
ops->flags |= FTRACE_OPS_FL_MODIFYING;
ops->old_hash.filter_hash = old_hash->filter_hash;
ops->old_hash.notrace_hash = old_hash->notrace_hash;
ftrace_run_update_code(command);
ops->old_hash.filter_hash = NULL;
ops->old_hash.notrace_hash = NULL;
ops->flags &= ~FTRACE_OPS_FL_MODIFYING;
}
static ftrace_func_t saved_ftrace_func;
static int ftrace_start_up;
void __weak arch_ftrace_trampoline_free(struct ftrace_ops *ops)
{
}
static void control_ops_free(struct ftrace_ops *ops)
{
free_percpu(ops->disabled);
}
static void ftrace_startup_enable(int command)
{
if (saved_ftrace_func != ftrace_trace_function) {
saved_ftrace_func = ftrace_trace_function;
command |= FTRACE_UPDATE_TRACE_FUNC;
}
if (!command || !ftrace_enabled)
return;
ftrace_run_update_code(command);
}
static void ftrace_startup_all(int command)
{
update_all_ops = true;
ftrace_startup_enable(command);
update_all_ops = false;
}
static int ftrace_startup(struct ftrace_ops *ops, int command)
{
int ret;
if (unlikely(ftrace_disabled))
return -ENODEV;
ret = __register_ftrace_function(ops);
if (ret)
return ret;
ftrace_start_up++;
command |= FTRACE_UPDATE_CALLS;
/*
* Note that ftrace probes uses this to start up
* and modify functions it will probe. But we still
* set the ADDING flag for modification, as probes
* do not have trampolines. If they add them in the
* future, then the probes will need to distinguish
* between adding and updating probes.
*/
ops->flags |= FTRACE_OPS_FL_ENABLED | FTRACE_OPS_FL_ADDING;
ret = ftrace_hash_ipmodify_enable(ops);
if (ret < 0) {
/* Rollback registration process */
__unregister_ftrace_function(ops);
ftrace_start_up--;
ops->flags &= ~FTRACE_OPS_FL_ENABLED;
return ret;
}
ftrace_hash_rec_enable(ops, 1);
ftrace_startup_enable(command);
ops->flags &= ~FTRACE_OPS_FL_ADDING;
return 0;
}
static int ftrace_shutdown(struct ftrace_ops *ops, int command)
{
int ret;
if (unlikely(ftrace_disabled))
return -ENODEV;
ret = __unregister_ftrace_function(ops);
if (ret)
return ret;
ftrace_start_up--;
/*
* Just warn in case of unbalance, no need to kill ftrace, it's not
* critical but the ftrace_call callers may be never nopped again after
* further ftrace uses.
*/
WARN_ON_ONCE(ftrace_start_up < 0);
/* Disabling ipmodify never fails */
ftrace_hash_ipmodify_disable(ops);
ftrace_hash_rec_disable(ops, 1);
ops->flags &= ~FTRACE_OPS_FL_ENABLED;
command |= FTRACE_UPDATE_CALLS;
if (saved_ftrace_func != ftrace_trace_function) {
saved_ftrace_func = ftrace_trace_function;
command |= FTRACE_UPDATE_TRACE_FUNC;
}
if (!command || !ftrace_enabled) {
/*
* If these are control ops, they still need their
* per_cpu field freed. Since, function tracing is
* not currently active, we can just free them
* without synchronizing all CPUs.
*/
if (ops->flags & FTRACE_OPS_FL_CONTROL)
control_ops_free(ops);
return 0;
}
/*
* If the ops uses a trampoline, then it needs to be
* tested first on update.
*/
ops->flags |= FTRACE_OPS_FL_REMOVING;
removed_ops = ops;
/* The trampoline logic checks the old hashes */
ops->old_hash.filter_hash = ops->func_hash->filter_hash;
ops->old_hash.notrace_hash = ops->func_hash->notrace_hash;
ftrace_run_update_code(command);
/*
* If there's no more ops registered with ftrace, run a
* sanity check to make sure all rec flags are cleared.
*/
if (ftrace_ops_list == &ftrace_list_end) {
struct ftrace_page *pg;
struct dyn_ftrace *rec;
do_for_each_ftrace_rec(pg, rec) {
if (FTRACE_WARN_ON_ONCE(rec->flags))
pr_warn(" %pS flags:%lx\n",
(void *)rec->ip, rec->flags);
} while_for_each_ftrace_rec();
}
ops->old_hash.filter_hash = NULL;
ops->old_hash.notrace_hash = NULL;
removed_ops = NULL;
ops->flags &= ~FTRACE_OPS_FL_REMOVING;
/*
* Dynamic ops may be freed, we must make sure that all
* callers are done before leaving this function.
* The same goes for freeing the per_cpu data of the control
* ops.
*
* Again, normal synchronize_sched() is not good enough.
* We need to do a hard force of sched synchronization.
* This is because we use preempt_disable() to do RCU, but
* the function tracers can be called where RCU is not watching
* (like before user_exit()). We can not rely on the RCU
* infrastructure to do the synchronization, thus we must do it
* ourselves.
*/
if (ops->flags & (FTRACE_OPS_FL_DYNAMIC | FTRACE_OPS_FL_CONTROL)) {
schedule_on_each_cpu(ftrace_sync);
arch_ftrace_trampoline_free(ops);
if (ops->flags & FTRACE_OPS_FL_CONTROL)
control_ops_free(ops);
}
return 0;
}
static void ftrace_startup_sysctl(void)
{
int command;
if (unlikely(ftrace_disabled))
return;
/* Force update next time */
saved_ftrace_func = NULL;
/* ftrace_start_up is true if we want ftrace running */
if (ftrace_start_up) {
command = FTRACE_UPDATE_CALLS;
if (ftrace_graph_active)
command |= FTRACE_START_FUNC_RET;
ftrace_startup_enable(command);
}
}
static void ftrace_shutdown_sysctl(void)
{
int command;
if (unlikely(ftrace_disabled))
return;
/* ftrace_start_up is true if ftrace is running */
if (ftrace_start_up) {
command = FTRACE_DISABLE_CALLS;
if (ftrace_graph_active)
command |= FTRACE_STOP_FUNC_RET;
ftrace_run_update_code(command);
}
}
static cycle_t ftrace_update_time;
unsigned long ftrace_update_tot_cnt;
static inline int ops_traces_mod(struct ftrace_ops *ops)
{
/*
* Filter_hash being empty will default to trace module.
* But notrace hash requires a test of individual module functions.
*/
return ftrace_hash_empty(ops->func_hash->filter_hash) &&
ftrace_hash_empty(ops->func_hash->notrace_hash);
}
/*
* Check if the current ops references the record.
*
* If the ops traces all functions, then it was already accounted for.
* If the ops does not trace the current record function, skip it.
* If the ops ignores the function via notrace filter, skip it.
*/
static inline bool
ops_references_rec(struct ftrace_ops *ops, struct dyn_ftrace *rec)
{
/* If ops isn't enabled, ignore it */
if (!(ops->flags & FTRACE_OPS_FL_ENABLED))
return 0;
/* If ops traces all mods, we already accounted for it */
if (ops_traces_mod(ops))
return 0;
/* The function must be in the filter */
if (!ftrace_hash_empty(ops->func_hash->filter_hash) &&
!ftrace_lookup_ip(ops->func_hash->filter_hash, rec->ip))
return 0;
/* If in notrace hash, we ignore it too */
if (ftrace_lookup_ip(ops->func_hash->notrace_hash, rec->ip))
return 0;
return 1;
}
static int referenced_filters(struct dyn_ftrace *rec)
{
struct ftrace_ops *ops;
int cnt = 0;
for (ops = ftrace_ops_list; ops != &ftrace_list_end; ops = ops->next) {
if (ops_references_rec(ops, rec))
cnt++;
}
return cnt;
}
static int ftrace_update_code(struct module *mod, struct ftrace_page *new_pgs)
{
struct ftrace_page *pg;
struct dyn_ftrace *p;
cycle_t start, stop;
unsigned long update_cnt = 0;
unsigned long ref = 0;
bool test = false;
int i;
/*
* When adding a module, we need to check if tracers are
* currently enabled and if they are set to trace all functions.
* If they are, we need to enable the module functions as well
* as update the reference counts for those function records.
*/
if (mod) {
struct ftrace_ops *ops;
for (ops = ftrace_ops_list;
ops != &ftrace_list_end; ops = ops->next) {
if (ops->flags & FTRACE_OPS_FL_ENABLED) {
if (ops_traces_mod(ops))
ref++;
else
test = true;
}
}
}
start = ftrace_now(raw_smp_processor_id());
for (pg = new_pgs; pg; pg = pg->next) {
for (i = 0; i < pg->index; i++) {
int cnt = ref;
/* If something went wrong, bail without enabling anything */
if (unlikely(ftrace_disabled))
return -1;
p = &pg->records[i];
if (test)
cnt += referenced_filters(p);
p->flags = cnt;
/*
* Do the initial record conversion from mcount jump
* to the NOP instructions.
*/
if (!ftrace_code_disable(mod, p))
break;
update_cnt++;
/*
* If the tracing is enabled, go ahead and enable the record.
*
* The reason not to enable the record immediatelly is the
* inherent check of ftrace_make_nop/ftrace_make_call for
* correct previous instructions. Making first the NOP
* conversion puts the module to the correct state, thus
* passing the ftrace_make_call check.
*/
if (ftrace_start_up && cnt) {
int failed = __ftrace_replace_code(p, 1);
if (failed)
ftrace_bug(failed, p);
}
}
}
stop = ftrace_now(raw_smp_processor_id());
ftrace_update_time = stop - start;
ftrace_update_tot_cnt += update_cnt;
return 0;
}
static int ftrace_allocate_records(struct ftrace_page *pg, int count)
{
int order;
int cnt;
if (WARN_ON(!count))
return -EINVAL;
order = get_count_order(DIV_ROUND_UP(count, ENTRIES_PER_PAGE));
/*
* We want to fill as much as possible. No more than a page
* may be empty.
*/
while ((PAGE_SIZE << order) / ENTRY_SIZE >= count + ENTRIES_PER_PAGE)
order--;
again:
pg->records = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, order);
if (!pg->records) {
/* if we can't allocate this size, try something smaller */
if (!order)
return -ENOMEM;
order >>= 1;
goto again;
}
cnt = (PAGE_SIZE << order) / ENTRY_SIZE;
pg->size = cnt;
if (cnt > count)
cnt = count;
return cnt;
}
static struct ftrace_page *
ftrace_allocate_pages(unsigned long num_to_init)
{
struct ftrace_page *start_pg;
struct ftrace_page *pg;
int order;
int cnt;
if (!num_to_init)
return 0;
start_pg = pg = kzalloc(sizeof(*pg), GFP_KERNEL);
if (!pg)
return NULL;
/*
* Try to allocate as much as possible in one continues
* location that fills in all of the space. We want to
* waste as little space as possible.
*/
for (;;) {
cnt = ftrace_allocate_records(pg, num_to_init);
if (cnt < 0)
goto free_pages;
num_to_init -= cnt;
if (!num_to_init)
break;
pg->next = kzalloc(sizeof(*pg), GFP_KERNEL);
if (!pg->next)
goto free_pages;
pg = pg->next;
}
return start_pg;
free_pages:
pg = start_pg;
while (pg) {
order = get_count_order(pg->size / ENTRIES_PER_PAGE);
free_pages((unsigned long)pg->records, order);
start_pg = pg->next;
kfree(pg);
pg = start_pg;
}
pr_info("ftrace: FAILED to allocate memory for functions\n");
return NULL;
}
#define FTRACE_BUFF_MAX (KSYM_SYMBOL_LEN+4) /* room for wildcards */
struct ftrace_iterator {
loff_t pos;
loff_t func_pos;
struct ftrace_page *pg;
struct dyn_ftrace *func;
struct ftrace_func_probe *probe;
struct trace_parser parser;
struct ftrace_hash *hash;
struct ftrace_ops *ops;
int hidx;
int idx;
unsigned flags;
};
static void *
t_hash_next(struct seq_file *m, loff_t *pos)
{
struct ftrace_iterator *iter = m->private;
struct hlist_node *hnd = NULL;
struct hlist_head *hhd;
(*pos)++;
iter->pos = *pos;
if (iter->probe)
hnd = &iter->probe->node;
retry:
if (iter->hidx >= FTRACE_FUNC_HASHSIZE)
return NULL;
hhd = &ftrace_func_hash[iter->hidx];
if (hlist_empty(hhd)) {
iter->hidx++;
hnd = NULL;
goto retry;
}
if (!hnd)
hnd = hhd->first;
else {
hnd = hnd->next;
if (!hnd) {
iter->hidx++;
goto retry;
}
}
if (WARN_ON_ONCE(!hnd))
return NULL;
iter->probe = hlist_entry(hnd, struct ftrace_func_probe, node);
return iter;
}
static void *t_hash_start(struct seq_file *m, loff_t *pos)
{
struct ftrace_iterator *iter = m->private;
void *p = NULL;
loff_t l;
if (!(iter->flags & FTRACE_ITER_DO_HASH))
return NULL;
if (iter->func_pos > *pos)
return NULL;
iter->hidx = 0;
for (l = 0; l <= (*pos - iter->func_pos); ) {
p = t_hash_next(m, &l);
if (!p)
break;
}
if (!p)
return NULL;
/* Only set this if we have an item */
iter->flags |= FTRACE_ITER_HASH;
return iter;
}
static int
t_hash_show(struct seq_file *m, struct ftrace_iterator *iter)
{
struct ftrace_func_probe *rec;
rec = iter->probe;
if (WARN_ON_ONCE(!rec))
return -EIO;
if (rec->ops->print)
return rec->ops->print(m, rec->ip, rec->ops, rec->data);
seq_printf(m, "%ps:%ps", (void *)rec->ip, (void *)rec->ops->func);
if (rec->data)
seq_printf(m, ":%p", rec->data);
seq_putc(m, '\n');
return 0;
}
static void *
t_next(struct seq_file *m, void *v, loff_t *pos)
{
struct ftrace_iterator *iter = m->private;
struct ftrace_ops *ops = iter->ops;
struct dyn_ftrace *rec = NULL;
if (unlikely(ftrace_disabled))
return NULL;
if (iter->flags & FTRACE_ITER_HASH)
return t_hash_next(m, pos);
(*pos)++;
iter->pos = iter->func_pos = *pos;
if (iter->flags & FTRACE_ITER_PRINTALL)
return t_hash_start(m, pos);
retry:
if (iter->idx >= iter->pg->index) {
if (iter->pg->next) {
iter->pg = iter->pg->next;
iter->idx = 0;
goto retry;
}
} else {
rec = &iter->pg->records[iter->idx++];
if (((iter->flags & FTRACE_ITER_FILTER) &&
!(ftrace_lookup_ip(ops->func_hash->filter_hash, rec->ip))) ||
((iter->flags & FTRACE_ITER_NOTRACE) &&
!ftrace_lookup_ip(ops->func_hash->notrace_hash, rec->ip)) ||
((iter->flags & FTRACE_ITER_ENABLED) &&
!(rec->flags & FTRACE_FL_ENABLED))) {
rec = NULL;
goto retry;
}
}
if (!rec)
return t_hash_start(m, pos);
iter->func = rec;
return iter;
}
static void reset_iter_read(struct ftrace_iterator *iter)
{
iter->pos = 0;
iter->func_pos = 0;
iter->flags &= ~(FTRACE_ITER_PRINTALL | FTRACE_ITER_HASH);
}
static void *t_start(struct seq_file *m, loff_t *pos)
{
struct ftrace_iterator *iter = m->private;
struct ftrace_ops *ops = iter->ops;
void *p = NULL;
loff_t l;
mutex_lock(&ftrace_lock);
if (unlikely(ftrace_disabled))
return NULL;
/*
* If an lseek was done, then reset and start from beginning.
*/
if (*pos < iter->pos)
reset_iter_read(iter);
/*
* For set_ftrace_filter reading, if we have the filter
* off, we can short cut and just print out that all
* functions are enabled.
*/
if ((iter->flags & FTRACE_ITER_FILTER &&
ftrace_hash_empty(ops->func_hash->filter_hash)) ||
(iter->flags & FTRACE_ITER_NOTRACE &&
ftrace_hash_empty(ops->func_hash->notrace_hash))) {
if (*pos > 0)
return t_hash_start(m, pos);
iter->flags |= FTRACE_ITER_PRINTALL;
/* reset in case of seek/pread */
iter->flags &= ~FTRACE_ITER_HASH;
return iter;
}
if (iter->flags & FTRACE_ITER_HASH)
return t_hash_start(m, pos);
/*
* Unfortunately, we need to restart at ftrace_pages_start
* every time we let go of the ftrace_mutex. This is because
* those pointers can change without the lock.
*/
iter->pg = ftrace_pages_start;
iter->idx = 0;
for (l = 0; l <= *pos; ) {
p = t_next(m, p, &l);
if (!p)
break;
}
if (!p)
return t_hash_start(m, pos);
return iter;
}
static void t_stop(struct seq_file *m, void *p)
{
mutex_unlock(&ftrace_lock);
}
void * __weak
arch_ftrace_trampoline_func(struct ftrace_ops *ops, struct dyn_ftrace *rec)
{
return NULL;
}
static void add_trampoline_func(struct seq_file *m, struct ftrace_ops *ops,
struct dyn_ftrace *rec)
{
void *ptr;
ptr = arch_ftrace_trampoline_func(ops, rec);
if (ptr)
seq_printf(m, " ->%pS", ptr);
}
static int t_show(struct seq_file *m, void *v)
{
struct ftrace_iterator *iter = m->private;
struct dyn_ftrace *rec;
if (iter->flags & FTRACE_ITER_HASH)
return t_hash_show(m, iter);
if (iter->flags & FTRACE_ITER_PRINTALL) {
if (iter->flags & FTRACE_ITER_NOTRACE)
seq_puts(m, "#### no functions disabled ####\n");
else
seq_puts(m, "#### all functions enabled ####\n");
return 0;
}
rec = iter->func;
if (!rec)
return 0;
seq_printf(m, "%ps", (void *)rec->ip);
if (iter->flags & FTRACE_ITER_ENABLED) {
struct ftrace_ops *ops = NULL;
seq_printf(m, " (%ld)%s%s",
ftrace_rec_count(rec),
rec->flags & FTRACE_FL_REGS ? " R" : " ",
rec->flags & FTRACE_FL_IPMODIFY ? " I" : " ");
if (rec->flags & FTRACE_FL_TRAMP_EN) {
ops = ftrace_find_tramp_ops_any(rec);
if (ops)
seq_printf(m, "\ttramp: %pS",
(void *)ops->trampoline);
else
seq_puts(m, "\ttramp: ERROR!");
}
add_trampoline_func(m, ops, rec);
}
seq_putc(m, '\n');
return 0;
}
static const struct seq_operations show_ftrace_seq_ops = {
.start = t_start,
.next = t_next,
.stop = t_stop,
.show = t_show,
};
static int
ftrace_avail_open(struct inode *inode, struct file *file)
{
struct ftrace_iterator *iter;
if (unlikely(ftrace_disabled))
return -ENODEV;
iter = __seq_open_private(file, &show_ftrace_seq_ops, sizeof(*iter));
if (iter) {
iter->pg = ftrace_pages_start;
iter->ops = &global_ops;
}
return iter ? 0 : -ENOMEM;
}
static int
ftrace_enabled_open(struct inode *inode, struct file *file)
{
struct ftrace_iterator *iter;
iter = __seq_open_private(file, &show_ftrace_seq_ops, sizeof(*iter));
if (iter) {
iter->pg = ftrace_pages_start;
iter->flags = FTRACE_ITER_ENABLED;
iter->ops = &global_ops;
}
return iter ? 0 : -ENOMEM;
}
/**
* ftrace_regex_open - initialize function tracer filter files
* @ops: The ftrace_ops that hold the hash filters
* @flag: The type of filter to process
* @inode: The inode, usually passed in to your open routine
* @file: The file, usually passed in to your open routine
*
* ftrace_regex_open() initializes the filter files for the
* @ops. Depending on @flag it may process the filter hash or
* the notrace hash of @ops. With this called from the open
* routine, you can use ftrace_filter_write() for the write
* routine if @flag has FTRACE_ITER_FILTER set, or
* ftrace_notrace_write() if @flag has FTRACE_ITER_NOTRACE set.
* tracing_lseek() should be used as the lseek routine, and
* release must call ftrace_regex_release().
*/
int
ftrace_regex_open(struct ftrace_ops *ops, int flag,
struct inode *inode, struct file *file)
{
struct ftrace_iterator *iter;
struct