1
linux/tools/perf/builtin-diff.c

250 lines
6.7 KiB
C
Raw Normal View History

perf diff: Introduce tool to show performance difference I guess it is enough to show some examples: [root@doppio linux-2.6-tip]# rm -f perf.data* [root@doppio linux-2.6-tip]# ls -la perf.data* ls: cannot access perf.data*: No such file or directory [root@doppio linux-2.6-tip]# perf record -f find / > /dev/null [ perf record: Woken up 1 times to write data ] [ perf record: Captured and wrote 0.062 MB perf.data (~2699 samples) ] [root@doppio linux-2.6-tip]# ls -la perf.data* -rw------- 1 root root 74440 2009-12-14 20:03 perf.data [root@doppio linux-2.6-tip]# perf record -f find / > /dev/null [ perf record: Woken up 1 times to write data ] [ perf record: Captured and wrote 0.062 MB perf.data (~2692 samples) ] [root@doppio linux-2.6-tip]# ls -la perf.data* -rw------- 1 root root 74280 2009-12-14 20:03 perf.data -rw------- 1 root root 74440 2009-12-14 20:03 perf.data.old [root@doppio linux-2.6-tip]# perf diff | head -5 1 -34994580 /lib64/libc-2.10.1.so _IO_vfprintf_internal 2 -15307806 [kernel.kallsyms] __kmalloc 3 +1 +3665941 /lib64/libc-2.10.1.so __GI_memmove 4 +4 +23508995 /lib64/libc-2.10.1.so _int_malloc 5 +7 +38538813 [kernel.kallsyms] __d_lookup [root@doppio linux-2.6-tip]# perf diff -p | head -5 1 +1.00% /lib64/libc-2.10.1.so _IO_vfprintf_internal 2 [kernel.kallsyms] __kmalloc 3 +1 /lib64/libc-2.10.1.so __GI_memmove 4 +4 /lib64/libc-2.10.1.so _int_malloc 5 +7 -1.00% [kernel.kallsyms] __d_lookup [root@doppio linux-2.6-tip]# perf diff -v | head -5 1 361449551 326454971 -34994580 /lib64/libc-2.10.1.so _IO_vfprintf_internal 2 151009241 135701435 -15307806 [kernel.kallsyms] __kmalloc 3 +1 101805328 105471269 +3665941 /lib64/libc-2.10.1.so __GI_memmove 4 +4 78041440 101550435 +23508995 /lib64/libc-2.10.1.so _int_malloc 5 +7 59536172 98074985 +38538813 [kernel.kallsyms] __d_lookup [root@doppio linux-2.6-tip]# perf diff -vp | head -5 1 9.00% 8.00% +1.00% /lib64/libc-2.10.1.so _IO_vfprintf_internal 2 3.00% 3.00% [kernel.kallsyms] __kmalloc 3 +1 2.00% 2.00% /lib64/libc-2.10.1.so __GI_memmove 4 +4 2.00% 2.00% /lib64/libc-2.10.1.so _int_malloc 5 +7 1.00% 2.00% -1.00% [kernel.kallsyms] __d_lookup [root@doppio linux-2.6-tip]# This should be enough for diffs where the system is non volatile, i.e. when one doesn't updates binaries. For volatile environments, stay tuned for the next perf tool feature: a buildid cache populated by 'perf record', managed by 'perf buildid-cache' a-la ccache, and used by all the report tools. Signed-off-by: Arnaldo Carvalho de Melo <acme@redhat.com> Cc: "Paul E. McKenney" <paulmck@linux.vnet.ibm.com> Cc: Stephen Hemminger <shemminger@vyatta.com> Cc: Frédéric Weisbecker <fweisbec@gmail.com> Cc: Mike Galbraith <efault@gmx.de> Cc: Peter Zijlstra <a.p.zijlstra@chello.nl> Cc: Paul Mackerras <paulus@samba.org> Cc: Paul E. McKenney <paulmck@linux.vnet.ibm.com> LKML-Reference: <1260828571-3613-3-git-send-email-acme@infradead.org> Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-12-14 15:09:31 -07:00
/*
* builtin-diff.c
*
* Builtin diff command: Analyze two perf.data input files, look up and read
* DSOs and symbol information, sort them and produce a diff.
*/
#include "builtin.h"
#include "util/debug.h"
#include "util/event.h"
#include "util/hist.h"
#include "util/session.h"
#include "util/sort.h"
#include "util/symbol.h"
#include "util/util.h"
#include <stdlib.h>
perf diff: Use perf_session__fprintf_hists just like 'perf record' That means that almost everything you can do with 'perf report' can be done with 'perf diff', for instance: $ perf record -f find / > /dev/null [ perf record: Woken up 1 times to write data ] [ perf record: Captured and wrote 0.062 MB perf.data (~2699 samples) ] $ perf record -f find / > /dev/null [ perf record: Woken up 1 times to write data ] [ perf record: Captured and wrote 0.062 MB perf.data (~2687 samples) ] perf diff | head -8 9.02% +1.00% find libc-2.10.1.so [.] _IO_vfprintf_internal 2.91% -1.00% find [kernel] [k] __kmalloc 2.85% -1.00% find [kernel] [k] ext4_htree_store_dirent 1.99% -1.00% find [kernel] [k] _atomic_dec_and_lock 2.44% find [kernel] [k] half_md4_transform $ So if you want to zoom into libc: $ perf diff --dsos libc-2.10.1.so | head -8 37.34% find [.] _IO_vfprintf_internal 10.34% find [.] __GI_memmove 8.25% +2.00% find [.] _int_malloc 5.07% -1.00% find [.] __GI_mempcpy 7.62% +2.00% find [.] _int_free $ And if there were multiple commands using libc, it is also possible to aggregate them all by using --sort symbol: $ perf diff --dsos libc-2.10.1.so --sort symbol | head -8 37.34% [.] _IO_vfprintf_internal 10.34% [.] __GI_memmove 8.25% +2.00% [.] _int_malloc 5.07% -1.00% [.] __GI_mempcpy 7.62% +2.00% [.] _int_free $ The displacement column now is off by default, to use it: perf diff -m --dsos libc-2.10.1.so --sort symbol | head -8 37.34% [.] _IO_vfprintf_internal 10.34% [.] __GI_memmove 8.25% +2.00% [.] _int_malloc 5.07% -1.00% +2 [.] __GI_mempcpy 7.62% +2.00% -1 [.] _int_free $ Using -t/--field-separator can be used for scripting: $ perf diff -t, -m --dsos libc-2.10.1.so --sort symbol | head -8 37.34, , ,[.] _IO_vfprintf_internal 10.34, , ,[.] __GI_memmove 8.25,+2.00%, ,[.] _int_malloc 5.07,-1.00%, +2,[.] __GI_mempcpy 7.62,+2.00%, -1,[.] _int_free 6.99,+1.00%, -1,[.] _IO_new_file_xsputn 1.89,-2.00%, +4,[.] __readdir64 $ Signed-off-by: Arnaldo Carvalho de Melo <acme@redhat.com> Cc: Frédéric Weisbecker <fweisbec@gmail.com> Cc: Mike Galbraith <efault@gmx.de> Cc: Peter Zijlstra <a.p.zijlstra@chello.nl> Cc: Paul Mackerras <paulus@samba.org> LKML-Reference: <1260978567-550-1-git-send-email-acme@infradead.org> Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-12-16 08:49:27 -07:00
static char const *input_old = "perf.data.old",
*input_new = "perf.data";
static char diff__default_sort_order[] = "dso,symbol";
perf diff: Use perf_session__fprintf_hists just like 'perf record' That means that almost everything you can do with 'perf report' can be done with 'perf diff', for instance: $ perf record -f find / > /dev/null [ perf record: Woken up 1 times to write data ] [ perf record: Captured and wrote 0.062 MB perf.data (~2699 samples) ] $ perf record -f find / > /dev/null [ perf record: Woken up 1 times to write data ] [ perf record: Captured and wrote 0.062 MB perf.data (~2687 samples) ] perf diff | head -8 9.02% +1.00% find libc-2.10.1.so [.] _IO_vfprintf_internal 2.91% -1.00% find [kernel] [k] __kmalloc 2.85% -1.00% find [kernel] [k] ext4_htree_store_dirent 1.99% -1.00% find [kernel] [k] _atomic_dec_and_lock 2.44% find [kernel] [k] half_md4_transform $ So if you want to zoom into libc: $ perf diff --dsos libc-2.10.1.so | head -8 37.34% find [.] _IO_vfprintf_internal 10.34% find [.] __GI_memmove 8.25% +2.00% find [.] _int_malloc 5.07% -1.00% find [.] __GI_mempcpy 7.62% +2.00% find [.] _int_free $ And if there were multiple commands using libc, it is also possible to aggregate them all by using --sort symbol: $ perf diff --dsos libc-2.10.1.so --sort symbol | head -8 37.34% [.] _IO_vfprintf_internal 10.34% [.] __GI_memmove 8.25% +2.00% [.] _int_malloc 5.07% -1.00% [.] __GI_mempcpy 7.62% +2.00% [.] _int_free $ The displacement column now is off by default, to use it: perf diff -m --dsos libc-2.10.1.so --sort symbol | head -8 37.34% [.] _IO_vfprintf_internal 10.34% [.] __GI_memmove 8.25% +2.00% [.] _int_malloc 5.07% -1.00% +2 [.] __GI_mempcpy 7.62% +2.00% -1 [.] _int_free $ Using -t/--field-separator can be used for scripting: $ perf diff -t, -m --dsos libc-2.10.1.so --sort symbol | head -8 37.34, , ,[.] _IO_vfprintf_internal 10.34, , ,[.] __GI_memmove 8.25,+2.00%, ,[.] _int_malloc 5.07,-1.00%, +2,[.] __GI_mempcpy 7.62,+2.00%, -1,[.] _int_free 6.99,+1.00%, -1,[.] _IO_new_file_xsputn 1.89,-2.00%, +4,[.] __readdir64 $ Signed-off-by: Arnaldo Carvalho de Melo <acme@redhat.com> Cc: Frédéric Weisbecker <fweisbec@gmail.com> Cc: Mike Galbraith <efault@gmx.de> Cc: Peter Zijlstra <a.p.zijlstra@chello.nl> Cc: Paul Mackerras <paulus@samba.org> LKML-Reference: <1260978567-550-1-git-send-email-acme@infradead.org> Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-12-16 08:49:27 -07:00
static int force;
static bool show_displacement;
perf diff: Introduce tool to show performance difference I guess it is enough to show some examples: [root@doppio linux-2.6-tip]# rm -f perf.data* [root@doppio linux-2.6-tip]# ls -la perf.data* ls: cannot access perf.data*: No such file or directory [root@doppio linux-2.6-tip]# perf record -f find / > /dev/null [ perf record: Woken up 1 times to write data ] [ perf record: Captured and wrote 0.062 MB perf.data (~2699 samples) ] [root@doppio linux-2.6-tip]# ls -la perf.data* -rw------- 1 root root 74440 2009-12-14 20:03 perf.data [root@doppio linux-2.6-tip]# perf record -f find / > /dev/null [ perf record: Woken up 1 times to write data ] [ perf record: Captured and wrote 0.062 MB perf.data (~2692 samples) ] [root@doppio linux-2.6-tip]# ls -la perf.data* -rw------- 1 root root 74280 2009-12-14 20:03 perf.data -rw------- 1 root root 74440 2009-12-14 20:03 perf.data.old [root@doppio linux-2.6-tip]# perf diff | head -5 1 -34994580 /lib64/libc-2.10.1.so _IO_vfprintf_internal 2 -15307806 [kernel.kallsyms] __kmalloc 3 +1 +3665941 /lib64/libc-2.10.1.so __GI_memmove 4 +4 +23508995 /lib64/libc-2.10.1.so _int_malloc 5 +7 +38538813 [kernel.kallsyms] __d_lookup [root@doppio linux-2.6-tip]# perf diff -p | head -5 1 +1.00% /lib64/libc-2.10.1.so _IO_vfprintf_internal 2 [kernel.kallsyms] __kmalloc 3 +1 /lib64/libc-2.10.1.so __GI_memmove 4 +4 /lib64/libc-2.10.1.so _int_malloc 5 +7 -1.00% [kernel.kallsyms] __d_lookup [root@doppio linux-2.6-tip]# perf diff -v | head -5 1 361449551 326454971 -34994580 /lib64/libc-2.10.1.so _IO_vfprintf_internal 2 151009241 135701435 -15307806 [kernel.kallsyms] __kmalloc 3 +1 101805328 105471269 +3665941 /lib64/libc-2.10.1.so __GI_memmove 4 +4 78041440 101550435 +23508995 /lib64/libc-2.10.1.so _int_malloc 5 +7 59536172 98074985 +38538813 [kernel.kallsyms] __d_lookup [root@doppio linux-2.6-tip]# perf diff -vp | head -5 1 9.00% 8.00% +1.00% /lib64/libc-2.10.1.so _IO_vfprintf_internal 2 3.00% 3.00% [kernel.kallsyms] __kmalloc 3 +1 2.00% 2.00% /lib64/libc-2.10.1.so __GI_memmove 4 +4 2.00% 2.00% /lib64/libc-2.10.1.so _int_malloc 5 +7 1.00% 2.00% -1.00% [kernel.kallsyms] __d_lookup [root@doppio linux-2.6-tip]# This should be enough for diffs where the system is non volatile, i.e. when one doesn't updates binaries. For volatile environments, stay tuned for the next perf tool feature: a buildid cache populated by 'perf record', managed by 'perf buildid-cache' a-la ccache, and used by all the report tools. Signed-off-by: Arnaldo Carvalho de Melo <acme@redhat.com> Cc: "Paul E. McKenney" <paulmck@linux.vnet.ibm.com> Cc: Stephen Hemminger <shemminger@vyatta.com> Cc: Frédéric Weisbecker <fweisbec@gmail.com> Cc: Mike Galbraith <efault@gmx.de> Cc: Peter Zijlstra <a.p.zijlstra@chello.nl> Cc: Paul Mackerras <paulus@samba.org> Cc: Paul E. McKenney <paulmck@linux.vnet.ibm.com> LKML-Reference: <1260828571-3613-3-git-send-email-acme@infradead.org> Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-12-14 15:09:31 -07:00
static int perf_session__add_hist_entry(struct perf_session *self,
struct addr_location *al, u64 count)
{
bool hit;
struct hist_entry *he = __perf_session__add_hist_entry(self, al, NULL,
count, &hit);
if (he == NULL)
return -ENOMEM;
if (hit)
he->count += count;
return 0;
}
static int diff__process_sample_event(event_t *event, struct perf_session *session)
{
struct addr_location al;
struct sample_data data = { .period = 1, };
dump_printf("(IP, %d): %d: %p\n", event->header.misc,
event->ip.pid, (void *)(long)event->ip.ip);
if (event__preprocess_sample(event, session, &al, NULL) < 0) {
pr_warning("problem processing %d event, skipping it.\n",
event->header.type);
return -1;
}
if (al.filtered)
return 0;
perf diff: Introduce tool to show performance difference I guess it is enough to show some examples: [root@doppio linux-2.6-tip]# rm -f perf.data* [root@doppio linux-2.6-tip]# ls -la perf.data* ls: cannot access perf.data*: No such file or directory [root@doppio linux-2.6-tip]# perf record -f find / > /dev/null [ perf record: Woken up 1 times to write data ] [ perf record: Captured and wrote 0.062 MB perf.data (~2699 samples) ] [root@doppio linux-2.6-tip]# ls -la perf.data* -rw------- 1 root root 74440 2009-12-14 20:03 perf.data [root@doppio linux-2.6-tip]# perf record -f find / > /dev/null [ perf record: Woken up 1 times to write data ] [ perf record: Captured and wrote 0.062 MB perf.data (~2692 samples) ] [root@doppio linux-2.6-tip]# ls -la perf.data* -rw------- 1 root root 74280 2009-12-14 20:03 perf.data -rw------- 1 root root 74440 2009-12-14 20:03 perf.data.old [root@doppio linux-2.6-tip]# perf diff | head -5 1 -34994580 /lib64/libc-2.10.1.so _IO_vfprintf_internal 2 -15307806 [kernel.kallsyms] __kmalloc 3 +1 +3665941 /lib64/libc-2.10.1.so __GI_memmove 4 +4 +23508995 /lib64/libc-2.10.1.so _int_malloc 5 +7 +38538813 [kernel.kallsyms] __d_lookup [root@doppio linux-2.6-tip]# perf diff -p | head -5 1 +1.00% /lib64/libc-2.10.1.so _IO_vfprintf_internal 2 [kernel.kallsyms] __kmalloc 3 +1 /lib64/libc-2.10.1.so __GI_memmove 4 +4 /lib64/libc-2.10.1.so _int_malloc 5 +7 -1.00% [kernel.kallsyms] __d_lookup [root@doppio linux-2.6-tip]# perf diff -v | head -5 1 361449551 326454971 -34994580 /lib64/libc-2.10.1.so _IO_vfprintf_internal 2 151009241 135701435 -15307806 [kernel.kallsyms] __kmalloc 3 +1 101805328 105471269 +3665941 /lib64/libc-2.10.1.so __GI_memmove 4 +4 78041440 101550435 +23508995 /lib64/libc-2.10.1.so _int_malloc 5 +7 59536172 98074985 +38538813 [kernel.kallsyms] __d_lookup [root@doppio linux-2.6-tip]# perf diff -vp | head -5 1 9.00% 8.00% +1.00% /lib64/libc-2.10.1.so _IO_vfprintf_internal 2 3.00% 3.00% [kernel.kallsyms] __kmalloc 3 +1 2.00% 2.00% /lib64/libc-2.10.1.so __GI_memmove 4 +4 2.00% 2.00% /lib64/libc-2.10.1.so _int_malloc 5 +7 1.00% 2.00% -1.00% [kernel.kallsyms] __d_lookup [root@doppio linux-2.6-tip]# This should be enough for diffs where the system is non volatile, i.e. when one doesn't updates binaries. For volatile environments, stay tuned for the next perf tool feature: a buildid cache populated by 'perf record', managed by 'perf buildid-cache' a-la ccache, and used by all the report tools. Signed-off-by: Arnaldo Carvalho de Melo <acme@redhat.com> Cc: "Paul E. McKenney" <paulmck@linux.vnet.ibm.com> Cc: Stephen Hemminger <shemminger@vyatta.com> Cc: Frédéric Weisbecker <fweisbec@gmail.com> Cc: Mike Galbraith <efault@gmx.de> Cc: Peter Zijlstra <a.p.zijlstra@chello.nl> Cc: Paul Mackerras <paulus@samba.org> Cc: Paul E. McKenney <paulmck@linux.vnet.ibm.com> LKML-Reference: <1260828571-3613-3-git-send-email-acme@infradead.org> Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-12-14 15:09:31 -07:00
event__parse_sample(event, session->sample_type, &data);
if (al.sym && perf_session__add_hist_entry(session, &al, data.period)) {
pr_warning("problem incrementing symbol count, skipping event\n");
return -1;
}
session->events_stats.total += data.period;
return 0;
}
static struct perf_event_ops event_ops = {
.process_sample_event = diff__process_sample_event,
.process_mmap_event = event__process_mmap,
.process_comm_event = event__process_comm,
.process_exit_event = event__process_task,
.process_fork_event = event__process_task,
.process_lost_event = event__process_lost,
};
static void perf_session__insert_hist_entry_by_name(struct rb_root *root,
struct hist_entry *he)
{
struct rb_node **p = &root->rb_node;
struct rb_node *parent = NULL;
struct hist_entry *iter;
while (*p != NULL) {
int cmp;
parent = *p;
iter = rb_entry(parent, struct hist_entry, rb_node);
cmp = strcmp(he->map->dso->name, iter->map->dso->name);
if (cmp > 0)
p = &(*p)->rb_left;
else if (cmp < 0)
p = &(*p)->rb_right;
else {
cmp = strcmp(he->sym->name, iter->sym->name);
if (cmp > 0)
p = &(*p)->rb_left;
else
p = &(*p)->rb_right;
}
}
rb_link_node(&he->rb_node, parent, p);
rb_insert_color(&he->rb_node, root);
}
static void perf_session__resort_by_name(struct perf_session *self)
{
unsigned long position = 1;
struct rb_root tmp = RB_ROOT;
struct rb_node *next = rb_first(&self->hists);
while (next != NULL) {
struct hist_entry *n = rb_entry(next, struct hist_entry, rb_node);
next = rb_next(&n->rb_node);
rb_erase(&n->rb_node, &self->hists);
n->position = position++;
perf_session__insert_hist_entry_by_name(&tmp, n);
}
self->hists = tmp;
}
static struct hist_entry *
perf_session__find_hist_entry_by_name(struct perf_session *self,
struct hist_entry *he)
{
struct rb_node *n = self->hists.rb_node;
while (n) {
struct hist_entry *iter = rb_entry(n, struct hist_entry, rb_node);
int cmp = strcmp(he->map->dso->name, iter->map->dso->name);
if (cmp > 0)
n = n->rb_left;
else if (cmp < 0)
n = n->rb_right;
else {
cmp = strcmp(he->sym->name, iter->sym->name);
if (cmp > 0)
n = n->rb_left;
else if (cmp < 0)
n = n->rb_right;
else
return iter;
}
}
return NULL;
}
static void perf_session__match_hists(struct perf_session *old_session,
struct perf_session *new_session)
{
struct rb_node *nd;
perf_session__resort_by_name(old_session);
for (nd = rb_first(&new_session->hists); nd; nd = rb_next(nd)) {
struct hist_entry *pos = rb_entry(nd, struct hist_entry, rb_node);
pos->pair = perf_session__find_hist_entry_by_name(old_session, pos);
}
}
static int __cmd_diff(void)
{
int ret, i;
struct perf_session *session[2];
session[0] = perf_session__new(input_old, O_RDONLY, force);
session[1] = perf_session__new(input_new, O_RDONLY, force);
perf diff: Introduce tool to show performance difference I guess it is enough to show some examples: [root@doppio linux-2.6-tip]# rm -f perf.data* [root@doppio linux-2.6-tip]# ls -la perf.data* ls: cannot access perf.data*: No such file or directory [root@doppio linux-2.6-tip]# perf record -f find / > /dev/null [ perf record: Woken up 1 times to write data ] [ perf record: Captured and wrote 0.062 MB perf.data (~2699 samples) ] [root@doppio linux-2.6-tip]# ls -la perf.data* -rw------- 1 root root 74440 2009-12-14 20:03 perf.data [root@doppio linux-2.6-tip]# perf record -f find / > /dev/null [ perf record: Woken up 1 times to write data ] [ perf record: Captured and wrote 0.062 MB perf.data (~2692 samples) ] [root@doppio linux-2.6-tip]# ls -la perf.data* -rw------- 1 root root 74280 2009-12-14 20:03 perf.data -rw------- 1 root root 74440 2009-12-14 20:03 perf.data.old [root@doppio linux-2.6-tip]# perf diff | head -5 1 -34994580 /lib64/libc-2.10.1.so _IO_vfprintf_internal 2 -15307806 [kernel.kallsyms] __kmalloc 3 +1 +3665941 /lib64/libc-2.10.1.so __GI_memmove 4 +4 +23508995 /lib64/libc-2.10.1.so _int_malloc 5 +7 +38538813 [kernel.kallsyms] __d_lookup [root@doppio linux-2.6-tip]# perf diff -p | head -5 1 +1.00% /lib64/libc-2.10.1.so _IO_vfprintf_internal 2 [kernel.kallsyms] __kmalloc 3 +1 /lib64/libc-2.10.1.so __GI_memmove 4 +4 /lib64/libc-2.10.1.so _int_malloc 5 +7 -1.00% [kernel.kallsyms] __d_lookup [root@doppio linux-2.6-tip]# perf diff -v | head -5 1 361449551 326454971 -34994580 /lib64/libc-2.10.1.so _IO_vfprintf_internal 2 151009241 135701435 -15307806 [kernel.kallsyms] __kmalloc 3 +1 101805328 105471269 +3665941 /lib64/libc-2.10.1.so __GI_memmove 4 +4 78041440 101550435 +23508995 /lib64/libc-2.10.1.so _int_malloc 5 +7 59536172 98074985 +38538813 [kernel.kallsyms] __d_lookup [root@doppio linux-2.6-tip]# perf diff -vp | head -5 1 9.00% 8.00% +1.00% /lib64/libc-2.10.1.so _IO_vfprintf_internal 2 3.00% 3.00% [kernel.kallsyms] __kmalloc 3 +1 2.00% 2.00% /lib64/libc-2.10.1.so __GI_memmove 4 +4 2.00% 2.00% /lib64/libc-2.10.1.so _int_malloc 5 +7 1.00% 2.00% -1.00% [kernel.kallsyms] __d_lookup [root@doppio linux-2.6-tip]# This should be enough for diffs where the system is non volatile, i.e. when one doesn't updates binaries. For volatile environments, stay tuned for the next perf tool feature: a buildid cache populated by 'perf record', managed by 'perf buildid-cache' a-la ccache, and used by all the report tools. Signed-off-by: Arnaldo Carvalho de Melo <acme@redhat.com> Cc: "Paul E. McKenney" <paulmck@linux.vnet.ibm.com> Cc: Stephen Hemminger <shemminger@vyatta.com> Cc: Frédéric Weisbecker <fweisbec@gmail.com> Cc: Mike Galbraith <efault@gmx.de> Cc: Peter Zijlstra <a.p.zijlstra@chello.nl> Cc: Paul Mackerras <paulus@samba.org> Cc: Paul E. McKenney <paulmck@linux.vnet.ibm.com> LKML-Reference: <1260828571-3613-3-git-send-email-acme@infradead.org> Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-12-14 15:09:31 -07:00
if (session[0] == NULL || session[1] == NULL)
return -ENOMEM;
for (i = 0; i < 2; ++i) {
ret = perf_session__process_events(session[i], &event_ops);
if (ret)
goto out_delete;
perf_session__output_resort(session[i], session[i]->events_stats.total);
}
perf_session__match_hists(session[0], session[1]);
perf diff: Use perf_session__fprintf_hists just like 'perf record' That means that almost everything you can do with 'perf report' can be done with 'perf diff', for instance: $ perf record -f find / > /dev/null [ perf record: Woken up 1 times to write data ] [ perf record: Captured and wrote 0.062 MB perf.data (~2699 samples) ] $ perf record -f find / > /dev/null [ perf record: Woken up 1 times to write data ] [ perf record: Captured and wrote 0.062 MB perf.data (~2687 samples) ] perf diff | head -8 9.02% +1.00% find libc-2.10.1.so [.] _IO_vfprintf_internal 2.91% -1.00% find [kernel] [k] __kmalloc 2.85% -1.00% find [kernel] [k] ext4_htree_store_dirent 1.99% -1.00% find [kernel] [k] _atomic_dec_and_lock 2.44% find [kernel] [k] half_md4_transform $ So if you want to zoom into libc: $ perf diff --dsos libc-2.10.1.so | head -8 37.34% find [.] _IO_vfprintf_internal 10.34% find [.] __GI_memmove 8.25% +2.00% find [.] _int_malloc 5.07% -1.00% find [.] __GI_mempcpy 7.62% +2.00% find [.] _int_free $ And if there were multiple commands using libc, it is also possible to aggregate them all by using --sort symbol: $ perf diff --dsos libc-2.10.1.so --sort symbol | head -8 37.34% [.] _IO_vfprintf_internal 10.34% [.] __GI_memmove 8.25% +2.00% [.] _int_malloc 5.07% -1.00% [.] __GI_mempcpy 7.62% +2.00% [.] _int_free $ The displacement column now is off by default, to use it: perf diff -m --dsos libc-2.10.1.so --sort symbol | head -8 37.34% [.] _IO_vfprintf_internal 10.34% [.] __GI_memmove 8.25% +2.00% [.] _int_malloc 5.07% -1.00% +2 [.] __GI_mempcpy 7.62% +2.00% -1 [.] _int_free $ Using -t/--field-separator can be used for scripting: $ perf diff -t, -m --dsos libc-2.10.1.so --sort symbol | head -8 37.34, , ,[.] _IO_vfprintf_internal 10.34, , ,[.] __GI_memmove 8.25,+2.00%, ,[.] _int_malloc 5.07,-1.00%, +2,[.] __GI_mempcpy 7.62,+2.00%, -1,[.] _int_free 6.99,+1.00%, -1,[.] _IO_new_file_xsputn 1.89,-2.00%, +4,[.] __readdir64 $ Signed-off-by: Arnaldo Carvalho de Melo <acme@redhat.com> Cc: Frédéric Weisbecker <fweisbec@gmail.com> Cc: Mike Galbraith <efault@gmx.de> Cc: Peter Zijlstra <a.p.zijlstra@chello.nl> Cc: Paul Mackerras <paulus@samba.org> LKML-Reference: <1260978567-550-1-git-send-email-acme@infradead.org> Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-12-16 08:49:27 -07:00
perf_session__fprintf_hists(session[1], session[0],
show_displacement, stdout);
perf diff: Introduce tool to show performance difference I guess it is enough to show some examples: [root@doppio linux-2.6-tip]# rm -f perf.data* [root@doppio linux-2.6-tip]# ls -la perf.data* ls: cannot access perf.data*: No such file or directory [root@doppio linux-2.6-tip]# perf record -f find / > /dev/null [ perf record: Woken up 1 times to write data ] [ perf record: Captured and wrote 0.062 MB perf.data (~2699 samples) ] [root@doppio linux-2.6-tip]# ls -la perf.data* -rw------- 1 root root 74440 2009-12-14 20:03 perf.data [root@doppio linux-2.6-tip]# perf record -f find / > /dev/null [ perf record: Woken up 1 times to write data ] [ perf record: Captured and wrote 0.062 MB perf.data (~2692 samples) ] [root@doppio linux-2.6-tip]# ls -la perf.data* -rw------- 1 root root 74280 2009-12-14 20:03 perf.data -rw------- 1 root root 74440 2009-12-14 20:03 perf.data.old [root@doppio linux-2.6-tip]# perf diff | head -5 1 -34994580 /lib64/libc-2.10.1.so _IO_vfprintf_internal 2 -15307806 [kernel.kallsyms] __kmalloc 3 +1 +3665941 /lib64/libc-2.10.1.so __GI_memmove 4 +4 +23508995 /lib64/libc-2.10.1.so _int_malloc 5 +7 +38538813 [kernel.kallsyms] __d_lookup [root@doppio linux-2.6-tip]# perf diff -p | head -5 1 +1.00% /lib64/libc-2.10.1.so _IO_vfprintf_internal 2 [kernel.kallsyms] __kmalloc 3 +1 /lib64/libc-2.10.1.so __GI_memmove 4 +4 /lib64/libc-2.10.1.so _int_malloc 5 +7 -1.00% [kernel.kallsyms] __d_lookup [root@doppio linux-2.6-tip]# perf diff -v | head -5 1 361449551 326454971 -34994580 /lib64/libc-2.10.1.so _IO_vfprintf_internal 2 151009241 135701435 -15307806 [kernel.kallsyms] __kmalloc 3 +1 101805328 105471269 +3665941 /lib64/libc-2.10.1.so __GI_memmove 4 +4 78041440 101550435 +23508995 /lib64/libc-2.10.1.so _int_malloc 5 +7 59536172 98074985 +38538813 [kernel.kallsyms] __d_lookup [root@doppio linux-2.6-tip]# perf diff -vp | head -5 1 9.00% 8.00% +1.00% /lib64/libc-2.10.1.so _IO_vfprintf_internal 2 3.00% 3.00% [kernel.kallsyms] __kmalloc 3 +1 2.00% 2.00% /lib64/libc-2.10.1.so __GI_memmove 4 +4 2.00% 2.00% /lib64/libc-2.10.1.so _int_malloc 5 +7 1.00% 2.00% -1.00% [kernel.kallsyms] __d_lookup [root@doppio linux-2.6-tip]# This should be enough for diffs where the system is non volatile, i.e. when one doesn't updates binaries. For volatile environments, stay tuned for the next perf tool feature: a buildid cache populated by 'perf record', managed by 'perf buildid-cache' a-la ccache, and used by all the report tools. Signed-off-by: Arnaldo Carvalho de Melo <acme@redhat.com> Cc: "Paul E. McKenney" <paulmck@linux.vnet.ibm.com> Cc: Stephen Hemminger <shemminger@vyatta.com> Cc: Frédéric Weisbecker <fweisbec@gmail.com> Cc: Mike Galbraith <efault@gmx.de> Cc: Peter Zijlstra <a.p.zijlstra@chello.nl> Cc: Paul Mackerras <paulus@samba.org> Cc: Paul E. McKenney <paulmck@linux.vnet.ibm.com> LKML-Reference: <1260828571-3613-3-git-send-email-acme@infradead.org> Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-12-14 15:09:31 -07:00
out_delete:
for (i = 0; i < 2; ++i)
perf_session__delete(session[i]);
return ret;
}
static const char * const diff_usage[] = {
perf diff: Introduce tool to show performance difference I guess it is enough to show some examples: [root@doppio linux-2.6-tip]# rm -f perf.data* [root@doppio linux-2.6-tip]# ls -la perf.data* ls: cannot access perf.data*: No such file or directory [root@doppio linux-2.6-tip]# perf record -f find / > /dev/null [ perf record: Woken up 1 times to write data ] [ perf record: Captured and wrote 0.062 MB perf.data (~2699 samples) ] [root@doppio linux-2.6-tip]# ls -la perf.data* -rw------- 1 root root 74440 2009-12-14 20:03 perf.data [root@doppio linux-2.6-tip]# perf record -f find / > /dev/null [ perf record: Woken up 1 times to write data ] [ perf record: Captured and wrote 0.062 MB perf.data (~2692 samples) ] [root@doppio linux-2.6-tip]# ls -la perf.data* -rw------- 1 root root 74280 2009-12-14 20:03 perf.data -rw------- 1 root root 74440 2009-12-14 20:03 perf.data.old [root@doppio linux-2.6-tip]# perf diff | head -5 1 -34994580 /lib64/libc-2.10.1.so _IO_vfprintf_internal 2 -15307806 [kernel.kallsyms] __kmalloc 3 +1 +3665941 /lib64/libc-2.10.1.so __GI_memmove 4 +4 +23508995 /lib64/libc-2.10.1.so _int_malloc 5 +7 +38538813 [kernel.kallsyms] __d_lookup [root@doppio linux-2.6-tip]# perf diff -p | head -5 1 +1.00% /lib64/libc-2.10.1.so _IO_vfprintf_internal 2 [kernel.kallsyms] __kmalloc 3 +1 /lib64/libc-2.10.1.so __GI_memmove 4 +4 /lib64/libc-2.10.1.so _int_malloc 5 +7 -1.00% [kernel.kallsyms] __d_lookup [root@doppio linux-2.6-tip]# perf diff -v | head -5 1 361449551 326454971 -34994580 /lib64/libc-2.10.1.so _IO_vfprintf_internal 2 151009241 135701435 -15307806 [kernel.kallsyms] __kmalloc 3 +1 101805328 105471269 +3665941 /lib64/libc-2.10.1.so __GI_memmove 4 +4 78041440 101550435 +23508995 /lib64/libc-2.10.1.so _int_malloc 5 +7 59536172 98074985 +38538813 [kernel.kallsyms] __d_lookup [root@doppio linux-2.6-tip]# perf diff -vp | head -5 1 9.00% 8.00% +1.00% /lib64/libc-2.10.1.so _IO_vfprintf_internal 2 3.00% 3.00% [kernel.kallsyms] __kmalloc 3 +1 2.00% 2.00% /lib64/libc-2.10.1.so __GI_memmove 4 +4 2.00% 2.00% /lib64/libc-2.10.1.so _int_malloc 5 +7 1.00% 2.00% -1.00% [kernel.kallsyms] __d_lookup [root@doppio linux-2.6-tip]# This should be enough for diffs where the system is non volatile, i.e. when one doesn't updates binaries. For volatile environments, stay tuned for the next perf tool feature: a buildid cache populated by 'perf record', managed by 'perf buildid-cache' a-la ccache, and used by all the report tools. Signed-off-by: Arnaldo Carvalho de Melo <acme@redhat.com> Cc: "Paul E. McKenney" <paulmck@linux.vnet.ibm.com> Cc: Stephen Hemminger <shemminger@vyatta.com> Cc: Frédéric Weisbecker <fweisbec@gmail.com> Cc: Mike Galbraith <efault@gmx.de> Cc: Peter Zijlstra <a.p.zijlstra@chello.nl> Cc: Paul Mackerras <paulus@samba.org> Cc: Paul E. McKenney <paulmck@linux.vnet.ibm.com> LKML-Reference: <1260828571-3613-3-git-send-email-acme@infradead.org> Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-12-14 15:09:31 -07:00
"perf diff [<options>] [old_file] [new_file]",
NULL,
perf diff: Introduce tool to show performance difference I guess it is enough to show some examples: [root@doppio linux-2.6-tip]# rm -f perf.data* [root@doppio linux-2.6-tip]# ls -la perf.data* ls: cannot access perf.data*: No such file or directory [root@doppio linux-2.6-tip]# perf record -f find / > /dev/null [ perf record: Woken up 1 times to write data ] [ perf record: Captured and wrote 0.062 MB perf.data (~2699 samples) ] [root@doppio linux-2.6-tip]# ls -la perf.data* -rw------- 1 root root 74440 2009-12-14 20:03 perf.data [root@doppio linux-2.6-tip]# perf record -f find / > /dev/null [ perf record: Woken up 1 times to write data ] [ perf record: Captured and wrote 0.062 MB perf.data (~2692 samples) ] [root@doppio linux-2.6-tip]# ls -la perf.data* -rw------- 1 root root 74280 2009-12-14 20:03 perf.data -rw------- 1 root root 74440 2009-12-14 20:03 perf.data.old [root@doppio linux-2.6-tip]# perf diff | head -5 1 -34994580 /lib64/libc-2.10.1.so _IO_vfprintf_internal 2 -15307806 [kernel.kallsyms] __kmalloc 3 +1 +3665941 /lib64/libc-2.10.1.so __GI_memmove 4 +4 +23508995 /lib64/libc-2.10.1.so _int_malloc 5 +7 +38538813 [kernel.kallsyms] __d_lookup [root@doppio linux-2.6-tip]# perf diff -p | head -5 1 +1.00% /lib64/libc-2.10.1.so _IO_vfprintf_internal 2 [kernel.kallsyms] __kmalloc 3 +1 /lib64/libc-2.10.1.so __GI_memmove 4 +4 /lib64/libc-2.10.1.so _int_malloc 5 +7 -1.00% [kernel.kallsyms] __d_lookup [root@doppio linux-2.6-tip]# perf diff -v | head -5 1 361449551 326454971 -34994580 /lib64/libc-2.10.1.so _IO_vfprintf_internal 2 151009241 135701435 -15307806 [kernel.kallsyms] __kmalloc 3 +1 101805328 105471269 +3665941 /lib64/libc-2.10.1.so __GI_memmove 4 +4 78041440 101550435 +23508995 /lib64/libc-2.10.1.so _int_malloc 5 +7 59536172 98074985 +38538813 [kernel.kallsyms] __d_lookup [root@doppio linux-2.6-tip]# perf diff -vp | head -5 1 9.00% 8.00% +1.00% /lib64/libc-2.10.1.so _IO_vfprintf_internal 2 3.00% 3.00% [kernel.kallsyms] __kmalloc 3 +1 2.00% 2.00% /lib64/libc-2.10.1.so __GI_memmove 4 +4 2.00% 2.00% /lib64/libc-2.10.1.so _int_malloc 5 +7 1.00% 2.00% -1.00% [kernel.kallsyms] __d_lookup [root@doppio linux-2.6-tip]# This should be enough for diffs where the system is non volatile, i.e. when one doesn't updates binaries. For volatile environments, stay tuned for the next perf tool feature: a buildid cache populated by 'perf record', managed by 'perf buildid-cache' a-la ccache, and used by all the report tools. Signed-off-by: Arnaldo Carvalho de Melo <acme@redhat.com> Cc: "Paul E. McKenney" <paulmck@linux.vnet.ibm.com> Cc: Stephen Hemminger <shemminger@vyatta.com> Cc: Frédéric Weisbecker <fweisbec@gmail.com> Cc: Mike Galbraith <efault@gmx.de> Cc: Peter Zijlstra <a.p.zijlstra@chello.nl> Cc: Paul Mackerras <paulus@samba.org> Cc: Paul E. McKenney <paulmck@linux.vnet.ibm.com> LKML-Reference: <1260828571-3613-3-git-send-email-acme@infradead.org> Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-12-14 15:09:31 -07:00
};
static const struct option options[] = {
OPT_BOOLEAN('v', "verbose", &verbose,
"be more verbose (show symbol address, etc)"),
perf diff: Use perf_session__fprintf_hists just like 'perf record' That means that almost everything you can do with 'perf report' can be done with 'perf diff', for instance: $ perf record -f find / > /dev/null [ perf record: Woken up 1 times to write data ] [ perf record: Captured and wrote 0.062 MB perf.data (~2699 samples) ] $ perf record -f find / > /dev/null [ perf record: Woken up 1 times to write data ] [ perf record: Captured and wrote 0.062 MB perf.data (~2687 samples) ] perf diff | head -8 9.02% +1.00% find libc-2.10.1.so [.] _IO_vfprintf_internal 2.91% -1.00% find [kernel] [k] __kmalloc 2.85% -1.00% find [kernel] [k] ext4_htree_store_dirent 1.99% -1.00% find [kernel] [k] _atomic_dec_and_lock 2.44% find [kernel] [k] half_md4_transform $ So if you want to zoom into libc: $ perf diff --dsos libc-2.10.1.so | head -8 37.34% find [.] _IO_vfprintf_internal 10.34% find [.] __GI_memmove 8.25% +2.00% find [.] _int_malloc 5.07% -1.00% find [.] __GI_mempcpy 7.62% +2.00% find [.] _int_free $ And if there were multiple commands using libc, it is also possible to aggregate them all by using --sort symbol: $ perf diff --dsos libc-2.10.1.so --sort symbol | head -8 37.34% [.] _IO_vfprintf_internal 10.34% [.] __GI_memmove 8.25% +2.00% [.] _int_malloc 5.07% -1.00% [.] __GI_mempcpy 7.62% +2.00% [.] _int_free $ The displacement column now is off by default, to use it: perf diff -m --dsos libc-2.10.1.so --sort symbol | head -8 37.34% [.] _IO_vfprintf_internal 10.34% [.] __GI_memmove 8.25% +2.00% [.] _int_malloc 5.07% -1.00% +2 [.] __GI_mempcpy 7.62% +2.00% -1 [.] _int_free $ Using -t/--field-separator can be used for scripting: $ perf diff -t, -m --dsos libc-2.10.1.so --sort symbol | head -8 37.34, , ,[.] _IO_vfprintf_internal 10.34, , ,[.] __GI_memmove 8.25,+2.00%, ,[.] _int_malloc 5.07,-1.00%, +2,[.] __GI_mempcpy 7.62,+2.00%, -1,[.] _int_free 6.99,+1.00%, -1,[.] _IO_new_file_xsputn 1.89,-2.00%, +4,[.] __readdir64 $ Signed-off-by: Arnaldo Carvalho de Melo <acme@redhat.com> Cc: Frédéric Weisbecker <fweisbec@gmail.com> Cc: Mike Galbraith <efault@gmx.de> Cc: Peter Zijlstra <a.p.zijlstra@chello.nl> Cc: Paul Mackerras <paulus@samba.org> LKML-Reference: <1260978567-550-1-git-send-email-acme@infradead.org> Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-12-16 08:49:27 -07:00
OPT_BOOLEAN('m', "displacement", &show_displacement,
"Show position displacement relative to baseline"),
perf diff: Introduce tool to show performance difference I guess it is enough to show some examples: [root@doppio linux-2.6-tip]# rm -f perf.data* [root@doppio linux-2.6-tip]# ls -la perf.data* ls: cannot access perf.data*: No such file or directory [root@doppio linux-2.6-tip]# perf record -f find / > /dev/null [ perf record: Woken up 1 times to write data ] [ perf record: Captured and wrote 0.062 MB perf.data (~2699 samples) ] [root@doppio linux-2.6-tip]# ls -la perf.data* -rw------- 1 root root 74440 2009-12-14 20:03 perf.data [root@doppio linux-2.6-tip]# perf record -f find / > /dev/null [ perf record: Woken up 1 times to write data ] [ perf record: Captured and wrote 0.062 MB perf.data (~2692 samples) ] [root@doppio linux-2.6-tip]# ls -la perf.data* -rw------- 1 root root 74280 2009-12-14 20:03 perf.data -rw------- 1 root root 74440 2009-12-14 20:03 perf.data.old [root@doppio linux-2.6-tip]# perf diff | head -5 1 -34994580 /lib64/libc-2.10.1.so _IO_vfprintf_internal 2 -15307806 [kernel.kallsyms] __kmalloc 3 +1 +3665941 /lib64/libc-2.10.1.so __GI_memmove 4 +4 +23508995 /lib64/libc-2.10.1.so _int_malloc 5 +7 +38538813 [kernel.kallsyms] __d_lookup [root@doppio linux-2.6-tip]# perf diff -p | head -5 1 +1.00% /lib64/libc-2.10.1.so _IO_vfprintf_internal 2 [kernel.kallsyms] __kmalloc 3 +1 /lib64/libc-2.10.1.so __GI_memmove 4 +4 /lib64/libc-2.10.1.so _int_malloc 5 +7 -1.00% [kernel.kallsyms] __d_lookup [root@doppio linux-2.6-tip]# perf diff -v | head -5 1 361449551 326454971 -34994580 /lib64/libc-2.10.1.so _IO_vfprintf_internal 2 151009241 135701435 -15307806 [kernel.kallsyms] __kmalloc 3 +1 101805328 105471269 +3665941 /lib64/libc-2.10.1.so __GI_memmove 4 +4 78041440 101550435 +23508995 /lib64/libc-2.10.1.so _int_malloc 5 +7 59536172 98074985 +38538813 [kernel.kallsyms] __d_lookup [root@doppio linux-2.6-tip]# perf diff -vp | head -5 1 9.00% 8.00% +1.00% /lib64/libc-2.10.1.so _IO_vfprintf_internal 2 3.00% 3.00% [kernel.kallsyms] __kmalloc 3 +1 2.00% 2.00% /lib64/libc-2.10.1.so __GI_memmove 4 +4 2.00% 2.00% /lib64/libc-2.10.1.so _int_malloc 5 +7 1.00% 2.00% -1.00% [kernel.kallsyms] __d_lookup [root@doppio linux-2.6-tip]# This should be enough for diffs where the system is non volatile, i.e. when one doesn't updates binaries. For volatile environments, stay tuned for the next perf tool feature: a buildid cache populated by 'perf record', managed by 'perf buildid-cache' a-la ccache, and used by all the report tools. Signed-off-by: Arnaldo Carvalho de Melo <acme@redhat.com> Cc: "Paul E. McKenney" <paulmck@linux.vnet.ibm.com> Cc: Stephen Hemminger <shemminger@vyatta.com> Cc: Frédéric Weisbecker <fweisbec@gmail.com> Cc: Mike Galbraith <efault@gmx.de> Cc: Peter Zijlstra <a.p.zijlstra@chello.nl> Cc: Paul Mackerras <paulus@samba.org> Cc: Paul E. McKenney <paulmck@linux.vnet.ibm.com> LKML-Reference: <1260828571-3613-3-git-send-email-acme@infradead.org> Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-12-14 15:09:31 -07:00
OPT_BOOLEAN('D', "dump-raw-trace", &dump_trace,
"dump raw trace in ASCII"),
OPT_BOOLEAN('f', "force", &force, "don't complain, do it"),
OPT_BOOLEAN('m', "modules", &symbol_conf.use_modules,
"load module symbols - WARNING: use only with -k and LIVE kernel"),
OPT_BOOLEAN('P', "full-paths", &event_ops.full_paths,
"Don't shorten the pathnames taking into account the cwd"),
OPT_STRING('d', "dsos", &symbol_conf.dso_list_str, "dso[,dso...]",
"only consider symbols in these dsos"),
OPT_STRING('C', "comms", &symbol_conf.comm_list_str, "comm[,comm...]",
"only consider symbols in these comms"),
OPT_STRING('S', "symbols", &symbol_conf.sym_list_str, "symbol[,symbol...]",
"only consider these symbols"),
perf diff: Use perf_session__fprintf_hists just like 'perf record' That means that almost everything you can do with 'perf report' can be done with 'perf diff', for instance: $ perf record -f find / > /dev/null [ perf record: Woken up 1 times to write data ] [ perf record: Captured and wrote 0.062 MB perf.data (~2699 samples) ] $ perf record -f find / > /dev/null [ perf record: Woken up 1 times to write data ] [ perf record: Captured and wrote 0.062 MB perf.data (~2687 samples) ] perf diff | head -8 9.02% +1.00% find libc-2.10.1.so [.] _IO_vfprintf_internal 2.91% -1.00% find [kernel] [k] __kmalloc 2.85% -1.00% find [kernel] [k] ext4_htree_store_dirent 1.99% -1.00% find [kernel] [k] _atomic_dec_and_lock 2.44% find [kernel] [k] half_md4_transform $ So if you want to zoom into libc: $ perf diff --dsos libc-2.10.1.so | head -8 37.34% find [.] _IO_vfprintf_internal 10.34% find [.] __GI_memmove 8.25% +2.00% find [.] _int_malloc 5.07% -1.00% find [.] __GI_mempcpy 7.62% +2.00% find [.] _int_free $ And if there were multiple commands using libc, it is also possible to aggregate them all by using --sort symbol: $ perf diff --dsos libc-2.10.1.so --sort symbol | head -8 37.34% [.] _IO_vfprintf_internal 10.34% [.] __GI_memmove 8.25% +2.00% [.] _int_malloc 5.07% -1.00% [.] __GI_mempcpy 7.62% +2.00% [.] _int_free $ The displacement column now is off by default, to use it: perf diff -m --dsos libc-2.10.1.so --sort symbol | head -8 37.34% [.] _IO_vfprintf_internal 10.34% [.] __GI_memmove 8.25% +2.00% [.] _int_malloc 5.07% -1.00% +2 [.] __GI_mempcpy 7.62% +2.00% -1 [.] _int_free $ Using -t/--field-separator can be used for scripting: $ perf diff -t, -m --dsos libc-2.10.1.so --sort symbol | head -8 37.34, , ,[.] _IO_vfprintf_internal 10.34, , ,[.] __GI_memmove 8.25,+2.00%, ,[.] _int_malloc 5.07,-1.00%, +2,[.] __GI_mempcpy 7.62,+2.00%, -1,[.] _int_free 6.99,+1.00%, -1,[.] _IO_new_file_xsputn 1.89,-2.00%, +4,[.] __readdir64 $ Signed-off-by: Arnaldo Carvalho de Melo <acme@redhat.com> Cc: Frédéric Weisbecker <fweisbec@gmail.com> Cc: Mike Galbraith <efault@gmx.de> Cc: Peter Zijlstra <a.p.zijlstra@chello.nl> Cc: Paul Mackerras <paulus@samba.org> LKML-Reference: <1260978567-550-1-git-send-email-acme@infradead.org> Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-12-16 08:49:27 -07:00
OPT_STRING('s', "sort", &sort_order, "key[,key2...]",
"sort by key(s): pid, comm, dso, symbol, parent"),
OPT_STRING('t', "field-separator", &symbol_conf.field_sep, "separator",
"separator for columns, no spaces will be added between "
"columns '.' is reserved."),
perf diff: Introduce tool to show performance difference I guess it is enough to show some examples: [root@doppio linux-2.6-tip]# rm -f perf.data* [root@doppio linux-2.6-tip]# ls -la perf.data* ls: cannot access perf.data*: No such file or directory [root@doppio linux-2.6-tip]# perf record -f find / > /dev/null [ perf record: Woken up 1 times to write data ] [ perf record: Captured and wrote 0.062 MB perf.data (~2699 samples) ] [root@doppio linux-2.6-tip]# ls -la perf.data* -rw------- 1 root root 74440 2009-12-14 20:03 perf.data [root@doppio linux-2.6-tip]# perf record -f find / > /dev/null [ perf record: Woken up 1 times to write data ] [ perf record: Captured and wrote 0.062 MB perf.data (~2692 samples) ] [root@doppio linux-2.6-tip]# ls -la perf.data* -rw------- 1 root root 74280 2009-12-14 20:03 perf.data -rw------- 1 root root 74440 2009-12-14 20:03 perf.data.old [root@doppio linux-2.6-tip]# perf diff | head -5 1 -34994580 /lib64/libc-2.10.1.so _IO_vfprintf_internal 2 -15307806 [kernel.kallsyms] __kmalloc 3 +1 +3665941 /lib64/libc-2.10.1.so __GI_memmove 4 +4 +23508995 /lib64/libc-2.10.1.so _int_malloc 5 +7 +38538813 [kernel.kallsyms] __d_lookup [root@doppio linux-2.6-tip]# perf diff -p | head -5 1 +1.00% /lib64/libc-2.10.1.so _IO_vfprintf_internal 2 [kernel.kallsyms] __kmalloc 3 +1 /lib64/libc-2.10.1.so __GI_memmove 4 +4 /lib64/libc-2.10.1.so _int_malloc 5 +7 -1.00% [kernel.kallsyms] __d_lookup [root@doppio linux-2.6-tip]# perf diff -v | head -5 1 361449551 326454971 -34994580 /lib64/libc-2.10.1.so _IO_vfprintf_internal 2 151009241 135701435 -15307806 [kernel.kallsyms] __kmalloc 3 +1 101805328 105471269 +3665941 /lib64/libc-2.10.1.so __GI_memmove 4 +4 78041440 101550435 +23508995 /lib64/libc-2.10.1.so _int_malloc 5 +7 59536172 98074985 +38538813 [kernel.kallsyms] __d_lookup [root@doppio linux-2.6-tip]# perf diff -vp | head -5 1 9.00% 8.00% +1.00% /lib64/libc-2.10.1.so _IO_vfprintf_internal 2 3.00% 3.00% [kernel.kallsyms] __kmalloc 3 +1 2.00% 2.00% /lib64/libc-2.10.1.so __GI_memmove 4 +4 2.00% 2.00% /lib64/libc-2.10.1.so _int_malloc 5 +7 1.00% 2.00% -1.00% [kernel.kallsyms] __d_lookup [root@doppio linux-2.6-tip]# This should be enough for diffs where the system is non volatile, i.e. when one doesn't updates binaries. For volatile environments, stay tuned for the next perf tool feature: a buildid cache populated by 'perf record', managed by 'perf buildid-cache' a-la ccache, and used by all the report tools. Signed-off-by: Arnaldo Carvalho de Melo <acme@redhat.com> Cc: "Paul E. McKenney" <paulmck@linux.vnet.ibm.com> Cc: Stephen Hemminger <shemminger@vyatta.com> Cc: Frédéric Weisbecker <fweisbec@gmail.com> Cc: Mike Galbraith <efault@gmx.de> Cc: Peter Zijlstra <a.p.zijlstra@chello.nl> Cc: Paul Mackerras <paulus@samba.org> Cc: Paul E. McKenney <paulmck@linux.vnet.ibm.com> LKML-Reference: <1260828571-3613-3-git-send-email-acme@infradead.org> Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-12-14 15:09:31 -07:00
OPT_END()
};
int cmd_diff(int argc, const char **argv, const char *prefix __used)
{
sort_order = diff__default_sort_order;
perf diff: Introduce tool to show performance difference I guess it is enough to show some examples: [root@doppio linux-2.6-tip]# rm -f perf.data* [root@doppio linux-2.6-tip]# ls -la perf.data* ls: cannot access perf.data*: No such file or directory [root@doppio linux-2.6-tip]# perf record -f find / > /dev/null [ perf record: Woken up 1 times to write data ] [ perf record: Captured and wrote 0.062 MB perf.data (~2699 samples) ] [root@doppio linux-2.6-tip]# ls -la perf.data* -rw------- 1 root root 74440 2009-12-14 20:03 perf.data [root@doppio linux-2.6-tip]# perf record -f find / > /dev/null [ perf record: Woken up 1 times to write data ] [ perf record: Captured and wrote 0.062 MB perf.data (~2692 samples) ] [root@doppio linux-2.6-tip]# ls -la perf.data* -rw------- 1 root root 74280 2009-12-14 20:03 perf.data -rw------- 1 root root 74440 2009-12-14 20:03 perf.data.old [root@doppio linux-2.6-tip]# perf diff | head -5 1 -34994580 /lib64/libc-2.10.1.so _IO_vfprintf_internal 2 -15307806 [kernel.kallsyms] __kmalloc 3 +1 +3665941 /lib64/libc-2.10.1.so __GI_memmove 4 +4 +23508995 /lib64/libc-2.10.1.so _int_malloc 5 +7 +38538813 [kernel.kallsyms] __d_lookup [root@doppio linux-2.6-tip]# perf diff -p | head -5 1 +1.00% /lib64/libc-2.10.1.so _IO_vfprintf_internal 2 [kernel.kallsyms] __kmalloc 3 +1 /lib64/libc-2.10.1.so __GI_memmove 4 +4 /lib64/libc-2.10.1.so _int_malloc 5 +7 -1.00% [kernel.kallsyms] __d_lookup [root@doppio linux-2.6-tip]# perf diff -v | head -5 1 361449551 326454971 -34994580 /lib64/libc-2.10.1.so _IO_vfprintf_internal 2 151009241 135701435 -15307806 [kernel.kallsyms] __kmalloc 3 +1 101805328 105471269 +3665941 /lib64/libc-2.10.1.so __GI_memmove 4 +4 78041440 101550435 +23508995 /lib64/libc-2.10.1.so _int_malloc 5 +7 59536172 98074985 +38538813 [kernel.kallsyms] __d_lookup [root@doppio linux-2.6-tip]# perf diff -vp | head -5 1 9.00% 8.00% +1.00% /lib64/libc-2.10.1.so _IO_vfprintf_internal 2 3.00% 3.00% [kernel.kallsyms] __kmalloc 3 +1 2.00% 2.00% /lib64/libc-2.10.1.so __GI_memmove 4 +4 2.00% 2.00% /lib64/libc-2.10.1.so _int_malloc 5 +7 1.00% 2.00% -1.00% [kernel.kallsyms] __d_lookup [root@doppio linux-2.6-tip]# This should be enough for diffs where the system is non volatile, i.e. when one doesn't updates binaries. For volatile environments, stay tuned for the next perf tool feature: a buildid cache populated by 'perf record', managed by 'perf buildid-cache' a-la ccache, and used by all the report tools. Signed-off-by: Arnaldo Carvalho de Melo <acme@redhat.com> Cc: "Paul E. McKenney" <paulmck@linux.vnet.ibm.com> Cc: Stephen Hemminger <shemminger@vyatta.com> Cc: Frédéric Weisbecker <fweisbec@gmail.com> Cc: Mike Galbraith <efault@gmx.de> Cc: Peter Zijlstra <a.p.zijlstra@chello.nl> Cc: Paul Mackerras <paulus@samba.org> Cc: Paul E. McKenney <paulmck@linux.vnet.ibm.com> LKML-Reference: <1260828571-3613-3-git-send-email-acme@infradead.org> Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-12-14 15:09:31 -07:00
argc = parse_options(argc, argv, options, diff_usage, 0);
if (argc) {
if (argc > 2)
usage_with_options(diff_usage, options);
if (argc == 2) {
input_old = argv[0];
input_new = argv[1];
} else
input_new = argv[0];
}
perf diff: Use perf_session__fprintf_hists just like 'perf record' That means that almost everything you can do with 'perf report' can be done with 'perf diff', for instance: $ perf record -f find / > /dev/null [ perf record: Woken up 1 times to write data ] [ perf record: Captured and wrote 0.062 MB perf.data (~2699 samples) ] $ perf record -f find / > /dev/null [ perf record: Woken up 1 times to write data ] [ perf record: Captured and wrote 0.062 MB perf.data (~2687 samples) ] perf diff | head -8 9.02% +1.00% find libc-2.10.1.so [.] _IO_vfprintf_internal 2.91% -1.00% find [kernel] [k] __kmalloc 2.85% -1.00% find [kernel] [k] ext4_htree_store_dirent 1.99% -1.00% find [kernel] [k] _atomic_dec_and_lock 2.44% find [kernel] [k] half_md4_transform $ So if you want to zoom into libc: $ perf diff --dsos libc-2.10.1.so | head -8 37.34% find [.] _IO_vfprintf_internal 10.34% find [.] __GI_memmove 8.25% +2.00% find [.] _int_malloc 5.07% -1.00% find [.] __GI_mempcpy 7.62% +2.00% find [.] _int_free $ And if there were multiple commands using libc, it is also possible to aggregate them all by using --sort symbol: $ perf diff --dsos libc-2.10.1.so --sort symbol | head -8 37.34% [.] _IO_vfprintf_internal 10.34% [.] __GI_memmove 8.25% +2.00% [.] _int_malloc 5.07% -1.00% [.] __GI_mempcpy 7.62% +2.00% [.] _int_free $ The displacement column now is off by default, to use it: perf diff -m --dsos libc-2.10.1.so --sort symbol | head -8 37.34% [.] _IO_vfprintf_internal 10.34% [.] __GI_memmove 8.25% +2.00% [.] _int_malloc 5.07% -1.00% +2 [.] __GI_mempcpy 7.62% +2.00% -1 [.] _int_free $ Using -t/--field-separator can be used for scripting: $ perf diff -t, -m --dsos libc-2.10.1.so --sort symbol | head -8 37.34, , ,[.] _IO_vfprintf_internal 10.34, , ,[.] __GI_memmove 8.25,+2.00%, ,[.] _int_malloc 5.07,-1.00%, +2,[.] __GI_mempcpy 7.62,+2.00%, -1,[.] _int_free 6.99,+1.00%, -1,[.] _IO_new_file_xsputn 1.89,-2.00%, +4,[.] __readdir64 $ Signed-off-by: Arnaldo Carvalho de Melo <acme@redhat.com> Cc: Frédéric Weisbecker <fweisbec@gmail.com> Cc: Mike Galbraith <efault@gmx.de> Cc: Peter Zijlstra <a.p.zijlstra@chello.nl> Cc: Paul Mackerras <paulus@samba.org> LKML-Reference: <1260978567-550-1-git-send-email-acme@infradead.org> Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-12-16 08:49:27 -07:00
symbol_conf.exclude_other = false;
if (symbol__init() < 0)
return -1;
setup_sorting(diff_usage, options);
perf diff: Introduce tool to show performance difference I guess it is enough to show some examples: [root@doppio linux-2.6-tip]# rm -f perf.data* [root@doppio linux-2.6-tip]# ls -la perf.data* ls: cannot access perf.data*: No such file or directory [root@doppio linux-2.6-tip]# perf record -f find / > /dev/null [ perf record: Woken up 1 times to write data ] [ perf record: Captured and wrote 0.062 MB perf.data (~2699 samples) ] [root@doppio linux-2.6-tip]# ls -la perf.data* -rw------- 1 root root 74440 2009-12-14 20:03 perf.data [root@doppio linux-2.6-tip]# perf record -f find / > /dev/null [ perf record: Woken up 1 times to write data ] [ perf record: Captured and wrote 0.062 MB perf.data (~2692 samples) ] [root@doppio linux-2.6-tip]# ls -la perf.data* -rw------- 1 root root 74280 2009-12-14 20:03 perf.data -rw------- 1 root root 74440 2009-12-14 20:03 perf.data.old [root@doppio linux-2.6-tip]# perf diff | head -5 1 -34994580 /lib64/libc-2.10.1.so _IO_vfprintf_internal 2 -15307806 [kernel.kallsyms] __kmalloc 3 +1 +3665941 /lib64/libc-2.10.1.so __GI_memmove 4 +4 +23508995 /lib64/libc-2.10.1.so _int_malloc 5 +7 +38538813 [kernel.kallsyms] __d_lookup [root@doppio linux-2.6-tip]# perf diff -p | head -5 1 +1.00% /lib64/libc-2.10.1.so _IO_vfprintf_internal 2 [kernel.kallsyms] __kmalloc 3 +1 /lib64/libc-2.10.1.so __GI_memmove 4 +4 /lib64/libc-2.10.1.so _int_malloc 5 +7 -1.00% [kernel.kallsyms] __d_lookup [root@doppio linux-2.6-tip]# perf diff -v | head -5 1 361449551 326454971 -34994580 /lib64/libc-2.10.1.so _IO_vfprintf_internal 2 151009241 135701435 -15307806 [kernel.kallsyms] __kmalloc 3 +1 101805328 105471269 +3665941 /lib64/libc-2.10.1.so __GI_memmove 4 +4 78041440 101550435 +23508995 /lib64/libc-2.10.1.so _int_malloc 5 +7 59536172 98074985 +38538813 [kernel.kallsyms] __d_lookup [root@doppio linux-2.6-tip]# perf diff -vp | head -5 1 9.00% 8.00% +1.00% /lib64/libc-2.10.1.so _IO_vfprintf_internal 2 3.00% 3.00% [kernel.kallsyms] __kmalloc 3 +1 2.00% 2.00% /lib64/libc-2.10.1.so __GI_memmove 4 +4 2.00% 2.00% /lib64/libc-2.10.1.so _int_malloc 5 +7 1.00% 2.00% -1.00% [kernel.kallsyms] __d_lookup [root@doppio linux-2.6-tip]# This should be enough for diffs where the system is non volatile, i.e. when one doesn't updates binaries. For volatile environments, stay tuned for the next perf tool feature: a buildid cache populated by 'perf record', managed by 'perf buildid-cache' a-la ccache, and used by all the report tools. Signed-off-by: Arnaldo Carvalho de Melo <acme@redhat.com> Cc: "Paul E. McKenney" <paulmck@linux.vnet.ibm.com> Cc: Stephen Hemminger <shemminger@vyatta.com> Cc: Frédéric Weisbecker <fweisbec@gmail.com> Cc: Mike Galbraith <efault@gmx.de> Cc: Peter Zijlstra <a.p.zijlstra@chello.nl> Cc: Paul Mackerras <paulus@samba.org> Cc: Paul E. McKenney <paulmck@linux.vnet.ibm.com> LKML-Reference: <1260828571-3613-3-git-send-email-acme@infradead.org> Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-12-14 15:09:31 -07:00
setup_pager();
perf diff: Use perf_session__fprintf_hists just like 'perf record' That means that almost everything you can do with 'perf report' can be done with 'perf diff', for instance: $ perf record -f find / > /dev/null [ perf record: Woken up 1 times to write data ] [ perf record: Captured and wrote 0.062 MB perf.data (~2699 samples) ] $ perf record -f find / > /dev/null [ perf record: Woken up 1 times to write data ] [ perf record: Captured and wrote 0.062 MB perf.data (~2687 samples) ] perf diff | head -8 9.02% +1.00% find libc-2.10.1.so [.] _IO_vfprintf_internal 2.91% -1.00% find [kernel] [k] __kmalloc 2.85% -1.00% find [kernel] [k] ext4_htree_store_dirent 1.99% -1.00% find [kernel] [k] _atomic_dec_and_lock 2.44% find [kernel] [k] half_md4_transform $ So if you want to zoom into libc: $ perf diff --dsos libc-2.10.1.so | head -8 37.34% find [.] _IO_vfprintf_internal 10.34% find [.] __GI_memmove 8.25% +2.00% find [.] _int_malloc 5.07% -1.00% find [.] __GI_mempcpy 7.62% +2.00% find [.] _int_free $ And if there were multiple commands using libc, it is also possible to aggregate them all by using --sort symbol: $ perf diff --dsos libc-2.10.1.so --sort symbol | head -8 37.34% [.] _IO_vfprintf_internal 10.34% [.] __GI_memmove 8.25% +2.00% [.] _int_malloc 5.07% -1.00% [.] __GI_mempcpy 7.62% +2.00% [.] _int_free $ The displacement column now is off by default, to use it: perf diff -m --dsos libc-2.10.1.so --sort symbol | head -8 37.34% [.] _IO_vfprintf_internal 10.34% [.] __GI_memmove 8.25% +2.00% [.] _int_malloc 5.07% -1.00% +2 [.] __GI_mempcpy 7.62% +2.00% -1 [.] _int_free $ Using -t/--field-separator can be used for scripting: $ perf diff -t, -m --dsos libc-2.10.1.so --sort symbol | head -8 37.34, , ,[.] _IO_vfprintf_internal 10.34, , ,[.] __GI_memmove 8.25,+2.00%, ,[.] _int_malloc 5.07,-1.00%, +2,[.] __GI_mempcpy 7.62,+2.00%, -1,[.] _int_free 6.99,+1.00%, -1,[.] _IO_new_file_xsputn 1.89,-2.00%, +4,[.] __readdir64 $ Signed-off-by: Arnaldo Carvalho de Melo <acme@redhat.com> Cc: Frédéric Weisbecker <fweisbec@gmail.com> Cc: Mike Galbraith <efault@gmx.de> Cc: Peter Zijlstra <a.p.zijlstra@chello.nl> Cc: Paul Mackerras <paulus@samba.org> LKML-Reference: <1260978567-550-1-git-send-email-acme@infradead.org> Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-12-16 08:49:27 -07:00
sort_entry__setup_elide(&sort_dso, symbol_conf.dso_list, "dso", NULL);
sort_entry__setup_elide(&sort_comm, symbol_conf.comm_list, "comm", NULL);
sort_entry__setup_elide(&sort_sym, symbol_conf.sym_list, "symbol", NULL);
perf diff: Introduce tool to show performance difference I guess it is enough to show some examples: [root@doppio linux-2.6-tip]# rm -f perf.data* [root@doppio linux-2.6-tip]# ls -la perf.data* ls: cannot access perf.data*: No such file or directory [root@doppio linux-2.6-tip]# perf record -f find / > /dev/null [ perf record: Woken up 1 times to write data ] [ perf record: Captured and wrote 0.062 MB perf.data (~2699 samples) ] [root@doppio linux-2.6-tip]# ls -la perf.data* -rw------- 1 root root 74440 2009-12-14 20:03 perf.data [root@doppio linux-2.6-tip]# perf record -f find / > /dev/null [ perf record: Woken up 1 times to write data ] [ perf record: Captured and wrote 0.062 MB perf.data (~2692 samples) ] [root@doppio linux-2.6-tip]# ls -la perf.data* -rw------- 1 root root 74280 2009-12-14 20:03 perf.data -rw------- 1 root root 74440 2009-12-14 20:03 perf.data.old [root@doppio linux-2.6-tip]# perf diff | head -5 1 -34994580 /lib64/libc-2.10.1.so _IO_vfprintf_internal 2 -15307806 [kernel.kallsyms] __kmalloc 3 +1 +3665941 /lib64/libc-2.10.1.so __GI_memmove 4 +4 +23508995 /lib64/libc-2.10.1.so _int_malloc 5 +7 +38538813 [kernel.kallsyms] __d_lookup [root@doppio linux-2.6-tip]# perf diff -p | head -5 1 +1.00% /lib64/libc-2.10.1.so _IO_vfprintf_internal 2 [kernel.kallsyms] __kmalloc 3 +1 /lib64/libc-2.10.1.so __GI_memmove 4 +4 /lib64/libc-2.10.1.so _int_malloc 5 +7 -1.00% [kernel.kallsyms] __d_lookup [root@doppio linux-2.6-tip]# perf diff -v | head -5 1 361449551 326454971 -34994580 /lib64/libc-2.10.1.so _IO_vfprintf_internal 2 151009241 135701435 -15307806 [kernel.kallsyms] __kmalloc 3 +1 101805328 105471269 +3665941 /lib64/libc-2.10.1.so __GI_memmove 4 +4 78041440 101550435 +23508995 /lib64/libc-2.10.1.so _int_malloc 5 +7 59536172 98074985 +38538813 [kernel.kallsyms] __d_lookup [root@doppio linux-2.6-tip]# perf diff -vp | head -5 1 9.00% 8.00% +1.00% /lib64/libc-2.10.1.so _IO_vfprintf_internal 2 3.00% 3.00% [kernel.kallsyms] __kmalloc 3 +1 2.00% 2.00% /lib64/libc-2.10.1.so __GI_memmove 4 +4 2.00% 2.00% /lib64/libc-2.10.1.so _int_malloc 5 +7 1.00% 2.00% -1.00% [kernel.kallsyms] __d_lookup [root@doppio linux-2.6-tip]# This should be enough for diffs where the system is non volatile, i.e. when one doesn't updates binaries. For volatile environments, stay tuned for the next perf tool feature: a buildid cache populated by 'perf record', managed by 'perf buildid-cache' a-la ccache, and used by all the report tools. Signed-off-by: Arnaldo Carvalho de Melo <acme@redhat.com> Cc: "Paul E. McKenney" <paulmck@linux.vnet.ibm.com> Cc: Stephen Hemminger <shemminger@vyatta.com> Cc: Frédéric Weisbecker <fweisbec@gmail.com> Cc: Mike Galbraith <efault@gmx.de> Cc: Peter Zijlstra <a.p.zijlstra@chello.nl> Cc: Paul Mackerras <paulus@samba.org> Cc: Paul E. McKenney <paulmck@linux.vnet.ibm.com> LKML-Reference: <1260828571-3613-3-git-send-email-acme@infradead.org> Signed-off-by: Ingo Molnar <mingo@elte.hu>
2009-12-14 15:09:31 -07:00
return __cmd_diff();
}