Piteball/android_kernel_google_msm
1
0
Fork 0
mirror of https://github.com/followmsi/android_kernel_google_msm.git synced 2024-09-21 12:10:52 +00:00
Code Issues Projects Releases Wiki Activity

Merge branch 'perf-core-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/linux-2.6-tip

Browse source 
* 'perf-core-for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/tip/linux-2.6-tip: (107 commits)
  perf stat: Add more cache-miss percentage printouts
  perf stat: Add -d -d and -d -d -d options to show more CPU events
  ftrace/kbuild: Add recordmcount files to force full build
  ftrace: Add self-tests for multiple function trace users
  ftrace: Modify ftrace_set_filter/notrace to take ops
  ftrace: Allow dynamically allocated function tracers
  ftrace: Implement separate user function filtering
  ftrace: Free hash with call_rcu_sched()
  ftrace: Have global_ops store the functions that are to be traced
  ftrace: Add ops parameter to ftrace_startup/shutdown functions
  ftrace: Add enabled_functions file
  ftrace: Use counters to enable functions to trace
  ftrace: Separate hash allocation and assignment
  ftrace: Create a global_ops to hold the filter and notrace hashes
  ftrace: Use hash instead for FTRACE_FL_FILTER
  ftrace: Replace FTRACE_FL_NOTRACE flag with a hash of ignored functions
  perf bench, x86: Add alternatives-asm.h wrapper
  x86, 64-bit: Fix copy_[to/from]_user() checks for the userspace address limit
  x86, mem: memset_64.S: Optimize memset by enhanced REP MOVSB/STOSB
  x86, mem: memmove_64.S: Optimize memmove by enhanced REP MOVSB/STOSB
  ...
This commit is contained in:
Linus Torvalds 2011-05-19 17:36:08 -07:00
parent acd30250d7 29510ec3b6
commit df48d8716e
78 changed files with 4086 additions and 1773 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
Makefile
View file

@ -1268,6 +1268,7 @@ help:
@echo ' make C=1 [targets] Check all c source with $$CHECK (sparse by default)'
@echo ' make C=2 [targets] Force check of all c source with $$CHECK'
@echo ' make W=1 [targets] Enable extra gcc checks'
@echo ' make RECORDMCOUNT_WARN=1 [targets] Warn about ignored mcount sections'
@echo ''
@echo 'Execute "make" or "make all" to build all targets marked with [*] '
@echo 'For further info see the ./README file'

22
arch/mips/include/asm/jump_label.h
View file

@ -20,16 +20,18 @@
#define WORD_INSN ".word"
#endif
#define JUMP_LABEL(key, label) \
do { \
asm goto("1:\tnop\n\t" \
"nop\n\t" \
".pushsection __jump_table, \"a\"\n\t" \
WORD_INSN " 1b, %l[" #label "], %0\n\t" \
".popsection\n\t" \
: : "i" (key) : : label); \
} while (0)
static __always_inline bool arch_static_branch(struct jump_label_key *key)
{
asm goto("1:\tnop\n\t"
"nop\n\t"
".pushsection __jump_table, \"aw\"\n\t"
WORD_INSN " 1b, %l[l_yes], %0\n\t"
".popsection\n\t"
: : "i" (key) : : l_yes);
return false;
l_yes:
return true;
}
#endif /* __KERNEL__ */

1
arch/s390/Kconfig
View file

@ -88,6 +88,7 @@ config S390
select HAVE_KERNEL_XZ
select HAVE_GET_USER_PAGES_FAST
select HAVE_ARCH_MUTEX_CPU_RELAX
select HAVE_ARCH_JUMP_LABEL if !MARCH_G5
select ARCH_INLINE_SPIN_TRYLOCK
select ARCH_INLINE_SPIN_TRYLOCK_BH
select ARCH_INLINE_SPIN_LOCK

4
arch/s390/include/asm/ftrace.h
View file

@ -11,15 +11,13 @@ struct dyn_arch_ftrace { };
#ifdef CONFIG_64BIT
#define MCOUNT_INSN_SIZE 12
#define MCOUNT_OFFSET 8
#else
#define MCOUNT_INSN_SIZE 20
#define MCOUNT_OFFSET 4
#endif
static inline unsigned long ftrace_call_adjust(unsigned long addr)
{
return addr - MCOUNT_OFFSET;
return addr;
}
#endif /* __ASSEMBLY__ */

37
arch/s390/include/asm/jump_label.h Normal file
View file

@ -0,0 +1,37 @@
#ifndef _ASM_S390_JUMP_LABEL_H
#define _ASM_S390_JUMP_LABEL_H
#include <linux/types.h>
#define JUMP_LABEL_NOP_SIZE 6
#ifdef CONFIG_64BIT
#define ASM_PTR ".quad"
#define ASM_ALIGN ".balign 8"
#else
#define ASM_PTR ".long"
#define ASM_ALIGN ".balign 4"
#endif
static __always_inline bool arch_static_branch(struct jump_label_key *key)
{
asm goto("0: brcl 0,0\n"
".pushsection __jump_table, \"aw\"\n"
ASM_ALIGN "\n"
ASM_PTR " 0b, %l[label], %0\n"
".popsection\n"
: : "X" (key) : : label);
return false;
label:
return true;
}
typedef unsigned long jump_label_t;
struct jump_entry {
jump_label_t code;
jump_label_t target;
jump_label_t key;
};
#endif

2
arch/s390/kernel/Makefile
View file

@ -23,7 +23,7 @@ CFLAGS_sysinfo.o += -Iinclude/math-emu -Iarch/s390/math-emu -w
obj-y := bitmap.o traps.o time.o process.o base.o early.o setup.o \
processor.o sys_s390.o ptrace.o signal.o cpcmd.o ebcdic.o \
s390_ext.o debug.o irq.o ipl.o dis.o diag.o mem_detect.o \
vdso.o vtime.o sysinfo.o nmi.o sclp.o
vdso.o vtime.o sysinfo.o nmi.o sclp.o jump_label.o
obj-y += $(if $(CONFIG_64BIT),entry64.o,entry.o)
obj-y += $(if $(CONFIG_64BIT),reipl64.o,reipl.o)

59
arch/s390/kernel/jump_label.c Normal file
View file

@ -0,0 +1,59 @@
/*
* Jump label s390 support
*
* Copyright IBM Corp. 2011
* Author(s): Jan Glauber <jang@linux.vnet.ibm.com>
*/
#include <linux/module.h>
#include <linux/uaccess.h>
#include <linux/stop_machine.h>
#include <linux/jump_label.h>
#include <asm/ipl.h>
#ifdef HAVE_JUMP_LABEL
struct insn {
u16 opcode;
s32 offset;
} __packed;
struct insn_args {
unsigned long *target;
struct insn *insn;
ssize_t size;
};
static int __arch_jump_label_transform(void *data)
{
struct insn_args *args = data;
int rc;
rc = probe_kernel_write(args->target, args->insn, args->size);
WARN_ON_ONCE(rc < 0);
return 0;
}
void arch_jump_label_transform(struct jump_entry *entry,
enum jump_label_type type)
{
struct insn_args args;
struct insn insn;
if (type == JUMP_LABEL_ENABLE) {
/* brcl 15,offset */
insn.opcode = 0xc0f4;
insn.offset = (entry->target - entry->code) >> 1;
} else {
/* brcl 0,0 */
insn.opcode = 0xc004;
insn.offset = 0;
}
args.target = (void *) entry->code;
args.insn = &insn;
args.size = JUMP_LABEL_NOP_SIZE;
stop_machine(__arch_jump_label_transform, &args, NULL);
}
#endif

25
arch/sparc/include/asm/jump_label.h
View file

@ -7,17 +7,20 @@
#define JUMP_LABEL_NOP_SIZE 4
#define JUMP_LABEL(key, label) \
do { \
asm goto("1:\n\t" \
"nop\n\t" \
"nop\n\t" \
".pushsection __jump_table, \"a\"\n\t"\
".align 4\n\t" \
".word 1b, %l[" #label "], %c0\n\t" \
".popsection \n\t" \
: : "i" (key) : : label);\
} while (0)
static __always_inline bool arch_static_branch(struct jump_label_key *key)
{
asm goto("1:\n\t"
"nop\n\t"
"nop\n\t"
".pushsection __jump_table, \"aw\"\n\t"
".align 4\n\t"
".word 1b, %l[l_yes], %c0\n\t"
".popsection \n\t"
: : "i" (key) : : l_yes);
return false;
l_yes:
return true;
}
#endif /* __KERNEL__ */

9
arch/x86/include/asm/alternative-asm.h
View file

@ -15,4 +15,13 @@
.endm
#endif
.macro altinstruction_entry orig alt feature orig_len alt_len
.align 8
.quad \orig
.quad \alt
.word \feature
.byte \orig_len
.byte \alt_len
.endm
#endif /* __ASSEMBLY__ */

3
arch/x86/include/asm/alternative.h
View file

@ -4,7 +4,6 @@
#include <linux/types.h>
#include <linux/stddef.h>
#include <linux/stringify.h>
#include <linux/jump_label.h>
#include <asm/asm.h>
/*
@ -191,7 +190,7 @@ extern void *text_poke(void *addr, const void *opcode, size_t len);
extern void *text_poke_smp(void *addr, const void *opcode, size_t len);
extern void text_poke_smp_batch(struct text_poke_param *params, int n);
#if defined(CONFIG_DYNAMIC_FTRACE) || defined(HAVE_JUMP_LABEL)
#if defined(CONFIG_DYNAMIC_FTRACE) || defined(CONFIG_JUMP_LABEL)
#define IDEAL_NOP_SIZE_5 5
extern unsigned char ideal_nop5[IDEAL_NOP_SIZE_5];
extern void arch_init_ideal_nop5(void);

1
arch/x86/include/asm/cpufeature.h
View file

@ -195,6 +195,7 @@
/* Intel-defined CPU features, CPUID level 0x00000007:0 (ebx), word 9 */
#define X86_FEATURE_FSGSBASE (9*32+ 0) /* {RD/WR}{FS/GS}BASE instructions*/
#define X86_FEATURE_ERMS (9*32+ 9) /* Enhanced REP MOVSB/STOSB */
#if defined(__KERNEL__) && !defined(__ASSEMBLY__)

7
arch/x86/include/asm/ftrace.h
View file

@ -38,11 +38,10 @@ extern void mcount(void);
static inline unsigned long ftrace_call_adjust(unsigned long addr)
{
/*
* call mcount is "e8 <4 byte offset>"
* The addr points to the 4 byte offset and the caller of this
* function wants the pointer to e8. Simply subtract one.
* addr is the address of the mcount call instruction.
* recordmcount does the necessary offset calculation.
*/
return addr - 1;
return addr;
}
#ifdef CONFIG_DYNAMIC_FTRACE

25
arch/x86/include/asm/jump_label.h
View file

@ -5,20 +5,25 @@
#include <linux/types.h>
#include <asm/nops.h>
#include <asm/asm.h>
#define JUMP_LABEL_NOP_SIZE 5
# define JUMP_LABEL_INITIAL_NOP ".byte 0xe9 \n\t .long 0\n\t"
#define JUMP_LABEL_INITIAL_NOP ".byte 0xe9 \n\t .long 0\n\t"
# define JUMP_LABEL(key, label) \
do { \
asm goto("1:" \
JUMP_LABEL_INITIAL_NOP \
".pushsection __jump_table, \"aw\" \n\t"\
_ASM_PTR "1b, %l[" #label "], %c0 \n\t" \
".popsection \n\t" \
: : "i" (key) : : label); \
} while (0)
static __always_inline bool arch_static_branch(struct jump_label_key *key)
{
asm goto("1:"
JUMP_LABEL_INITIAL_NOP
".pushsection __jump_table, \"aw\" \n\t"
_ASM_ALIGN "\n\t"
_ASM_PTR "1b, %l[l_yes], %c0 \n\t"
".popsection \n\t"
: : "i" (key) : : l_yes);
return false;
l_yes:
return true;
}
#endif /* __KERNEL__ */

2
arch/x86/include/asm/setup.h
View file

@ -88,7 +88,7 @@ void *extend_brk(size_t size, size_t align);
* executable.)
*/
#define RESERVE_BRK(name,sz) \
static void __section(.discard.text) __used \
static void __section(.discard.text) __used notrace \
__brk_reservation_fn_##name##__(void) { \
asm volatile ( \
".pushsection .brk_reservation,\"aw\",@nobits;" \

3
arch/x86/include/asm/stacktrace.h
View file

@ -37,9 +37,6 @@ print_context_stack_bp(struct thread_info *tinfo,
/* Generic stack tracer with callbacks */
struct stacktrace_ops {
void (*warning)(void *data, char *msg);
/* msg must contain %s for the symbol */
void (*warning_symbol)(void *data, char *msg, unsigned long symbol);
void (*address)(void *data, unsigned long address, int reliable);
/* On negative return stop dumping */
int (*stack)(void *data, char *name);

2
arch/x86/include/asm/uaccess.h
View file

@ -42,7 +42,7 @@
* Returns 0 if the range is valid, nonzero otherwise.
*
* This is equivalent to the following test:
* (u33)addr + (u33)size >= (u33)current->addr_limit.seg (u65 for x86_64)
* (u33)addr + (u33)size > (u33)current->addr_limit.seg (u65 for x86_64)
*
* This needs 33-bit (65-bit for x86_64) arithmetic. We have a carry...
*/

11
arch/x86/kernel/alternative.c
View file

@ -210,6 +210,15 @@ void __init_or_module apply_alternatives(struct alt_instr *start,
u8 insnbuf[MAX_PATCH_LEN];
DPRINTK("%s: alt table %p -> %p\n", __func__, start, end);
/*
* The scan order should be from start to end. A later scanned
* alternative code can overwrite a previous scanned alternative code.
* Some kernel functions (e.g. memcpy, memset, etc) use this order to
* patch code.
*
* So be careful if you want to change the scan order to any other
* order.
*/
for (a = start; a < end; a++) {
u8 *instr = a->instr;
BUG_ON(a->replacementlen > a->instrlen);
@ -679,7 +688,7 @@ void __kprobes text_poke_smp_batch(struct text_poke_param *params, int n)
__stop_machine(stop_machine_text_poke, (void *)&tpp, NULL);
}
#if defined(CONFIG_DYNAMIC_FTRACE) || defined(HAVE_JUMP_LABEL)
#if defined(CONFIG_DYNAMIC_FTRACE) || defined(CONFIG_JUMP_LABEL)
#ifdef CONFIG_X86_64
unsigned char ideal_nop5[5] = { 0x66, 0x66, 0x66, 0x66, 0x90 };

1
arch/x86/kernel/cpu/common.c
View file

@ -565,7 +565,6 @@ void __cpuinit get_cpu_cap(struct cpuinfo_x86 *c)
cpuid_count(0x00000007, 0, &eax, &ebx, &ecx, &edx);
if (eax > 0)
c->x86_capability[9] = ebx;
}

19
arch/x86/kernel/cpu/intel.c
View file

@ -29,10 +29,10 @@
static void __cpuinit early_init_intel(struct cpuinfo_x86 *c)
{
/* Unmask CPUID levels if masked: */
if (c->x86 > 6 || (c->x86 == 6 && c->x86_model >= 0xd)) {
u64 misc_enable;
/* Unmask CPUID levels if masked: */
if (c->x86 > 6 || (c->x86 == 6 && c->x86_model >= 0xd)) {
rdmsrl(MSR_IA32_MISC_ENABLE, misc_enable);
if (misc_enable & MSR_IA32_MISC_ENABLE_LIMIT_CPUID) {
@ -118,8 +118,6 @@ static void __cpuinit early_init_intel(struct cpuinfo_x86 *c)
* (model 2) with the same problem.
*/
if (c->x86 == 15) {
u64 misc_enable;
rdmsrl(MSR_IA32_MISC_ENABLE, misc_enable);
if (misc_enable & MSR_IA32_MISC_ENABLE_FAST_STRING) {
@ -130,6 +128,19 @@ static void __cpuinit early_init_intel(struct cpuinfo_x86 *c)
}
}
#endif
/*
* If fast string is not enabled in IA32_MISC_ENABLE for any reason,
* clear the fast string and enhanced fast string CPU capabilities.
*/
if (c->x86 > 6 || (c->x86 == 6 && c->x86_model >= 0xd)) {
rdmsrl(MSR_IA32_MISC_ENABLE, misc_enable);
if (!(misc_enable & MSR_IA32_MISC_ENABLE_FAST_STRING)) {
printk(KERN_INFO "Disabled fast string operations\n");
setup_clear_cpu_cap(X86_FEATURE_REP_GOOD);
setup_clear_cpu_cap(X86_FEATURE_ERMS);
}
}
}
#ifdef CONFIG_X86_32

28
arch/x86/kernel/cpu/perf_event.c
View file

@ -31,6 +31,7 @@
#include <asm/nmi.h>
#include <asm/compat.h>
#include <asm/smp.h>
#include <asm/alternative.h>
#if 0
#undef wrmsrl
@ -363,12 +364,18 @@ again:
return new_raw_count;
}
/* using X86_FEATURE_PERFCTR_CORE to later implement ALTERNATIVE() here */
static inline int x86_pmu_addr_offset(int index)
{
if (boot_cpu_has(X86_FEATURE_PERFCTR_CORE))
return index << 1;
return index;
int offset;
/* offset = X86_FEATURE_PERFCTR_CORE ? index << 1 : index */
alternative_io(ASM_NOP2,
"shll $1, %%eax",
X86_FEATURE_PERFCTR_CORE,
"=a" (offset),
"a" (index));
return offset;
}
static inline unsigned int x86_pmu_config_addr(int index)
@ -1766,17 +1773,6 @@ static struct pmu pmu = {
* callchain support
*/
static void
backtrace_warning_symbol(void *data, char *msg, unsigned long symbol)
{
/* Ignore warnings */
}
static void backtrace_warning(void *data, char *msg)
{
/* Ignore warnings */
}
static int backtrace_stack(void *data, char *name)
{
return 0;
@ -1790,8 +1786,6 @@ static void backtrace_address(void *data, unsigned long addr, int reliable)
}
static const struct stacktrace_ops backtrace_ops = {
.warning = backtrace_warning,
.warning_symbol = backtrace_warning_symbol,
.stack = backtrace_stack,
.address = backtrace_address,
.walk_stack = print_context_stack_bp,

2
arch/x86/kernel/cpu/perf_event_amd.c
View file

@ -102,6 +102,8 @@ static const u64 amd_perfmon_event_map[] =
[PERF_COUNT_HW_CACHE_MISSES] = 0x0081,
[PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = 0x00c2,
[PERF_COUNT_HW_BRANCH_MISSES] = 0x00c3,
[PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] = 0x00d0, /* "Decoder empty" event */
[PERF_COUNT_HW_STALLED_CYCLES_BACKEND] = 0x00d1, /* "Dispatch stalls" event */
};
static u64 amd_pmu_event_map(int hw_event)

37
arch/x86/kernel/cpu/perf_event_intel.c
View file

@ -36,7 +36,7 @@ static u64 intel_perfmon_event_map[PERF_COUNT_HW_MAX] __read_mostly =
[PERF_COUNT_HW_BUS_CYCLES] = 0x013c,
};
static struct event_constraint intel_core_event_constraints[] =
static struct event_constraint intel_core_event_constraints[] __read_mostly =
{
INTEL_EVENT_CONSTRAINT(0x11, 0x2), /* FP_ASSIST */
INTEL_EVENT_CONSTRAINT(0x12, 0x2), /* MUL */
@ -47,7 +47,7 @@ static struct event_constraint intel_core_event_constraints[] =
EVENT_CONSTRAINT_END
};
static struct event_constraint intel_core2_event_constraints[] =
static struct event_constraint intel_core2_event_constraints[] __read_mostly =
{
FIXED_EVENT_CONSTRAINT(0x00c0, 0), /* INST_RETIRED.ANY */
FIXED_EVENT_CONSTRAINT(0x003c, 1), /* CPU_CLK_UNHALTED.CORE */
@ -70,7 +70,7 @@ static struct event_constraint intel_core2_event_constraints[] =
EVENT_CONSTRAINT_END
};
static struct event_constraint intel_nehalem_event_constraints[] =
static struct event_constraint intel_nehalem_event_constraints[] __read_mostly =
{
FIXED_EVENT_CONSTRAINT(0x00c0, 0), /* INST_RETIRED.ANY */
FIXED_EVENT_CONSTRAINT(0x003c, 1), /* CPU_CLK_UNHALTED.CORE */
@ -86,19 +86,19 @@ static struct event_constraint intel_nehalem_event_constraints[] =
EVENT_CONSTRAINT_END
};
static struct extra_reg intel_nehalem_extra_regs[] =
static struct extra_reg intel_nehalem_extra_regs[] __read_mostly =
{
INTEL_EVENT_EXTRA_REG(0xb7, MSR_OFFCORE_RSP_0, 0xffff),
EVENT_EXTRA_END
};
static struct event_constraint intel_nehalem_percore_constraints[] =
static struct event_constraint intel_nehalem_percore_constraints[] __read_mostly =
{
INTEL_EVENT_CONSTRAINT(0xb7, 0),
EVENT_CONSTRAINT_END
};
static struct event_constraint intel_westmere_event_constraints[] =
static struct event_constraint intel_westmere_event_constraints[] __read_mostly =
{
FIXED_EVENT_CONSTRAINT(0x00c0, 0), /* INST_RETIRED.ANY */
FIXED_EVENT_CONSTRAINT(0x003c, 1), /* CPU_CLK_UNHALTED.CORE */
@ -110,7 +110,7 @@ static struct event_constraint intel_westmere_event_constraints[] =
EVENT_CONSTRAINT_END
};
static struct event_constraint intel_snb_event_constraints[] =
static struct event_constraint intel_snb_event_constraints[] __read_mostly =
{
FIXED_EVENT_CONSTRAINT(0x00c0, 0), /* INST_RETIRED.ANY */
FIXED_EVENT_CONSTRAINT(0x003c, 1), /* CPU_CLK_UNHALTED.CORE */
@ -123,21 +123,21 @@ static struct event_constraint intel_snb_event_constraints[] =
EVENT_CONSTRAINT_END
};
static struct extra_reg intel_westmere_extra_regs[] =
static struct extra_reg intel_westmere_extra_regs[] __read_mostly =
{
INTEL_EVENT_EXTRA_REG(0xb7, MSR_OFFCORE_RSP_0, 0xffff),
INTEL_EVENT_EXTRA_REG(0xbb, MSR_OFFCORE_RSP_1, 0xffff),
EVENT_EXTRA_END
};
static struct event_constraint intel_westmere_percore_constraints[] =
static struct event_constraint intel_westmere_percore_constraints[] __read_mostly =
{
INTEL_EVENT_CONSTRAINT(0xb7, 0),
INTEL_EVENT_CONSTRAINT(0xbb, 0),
EVENT_CONSTRAINT_END
};
static struct event_constraint intel_gen_event_constraints[] =
static struct event_constraint intel_gen_event_constraints[] __read_mostly =
{
FIXED_EVENT_CONSTRAINT(0x00c0, 0), /* INST_RETIRED.ANY */
FIXED_EVENT_CONSTRAINT(0x003c, 1), /* CPU_CLK_UNHALTED.CORE */
@ -1440,6 +1440,11 @@ static __init int intel_pmu_init(void)
x86_pmu.enable_all = intel_pmu_nhm_enable_all;
x86_pmu.extra_regs = intel_nehalem_extra_regs;
/* UOPS_ISSUED.STALLED_CYCLES */
intel_perfmon_event_map[PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] = 0x180010e;
/* UOPS_EXECUTED.CORE_ACTIVE_CYCLES,c=1,i=1 */
intel_perfmon_event_map[PERF_COUNT_HW_STALLED_CYCLES_BACKEND] = 0x1803fb1;
if (ebx & 0x40) {
/*
* Erratum AAJ80 detected, we work it around by using
@ -1480,6 +1485,12 @@ static __init int intel_pmu_init(void)
x86_pmu.enable_all = intel_pmu_nhm_enable_all;
x86_pmu.pebs_constraints = intel_westmere_pebs_event_constraints;
x86_pmu.extra_regs = intel_westmere_extra_regs;
/* UOPS_ISSUED.STALLED_CYCLES */
intel_perfmon_event_map[PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] = 0x180010e;
/* UOPS_EXECUTED.CORE_ACTIVE_CYCLES,c=1,i=1 */
intel_perfmon_event_map[PERF_COUNT_HW_STALLED_CYCLES_BACKEND] = 0x1803fb1;
pr_cont("Westmere events, ");
break;
@ -1491,6 +1502,12 @@ static __init int intel_pmu_init(void)
x86_pmu.event_constraints = intel_snb_event_constraints;
x86_pmu.pebs_constraints = intel_snb_pebs_events;
/* UOPS_ISSUED.ANY,c=1,i=1 to count stall cycles */
intel_perfmon_event_map[PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] = 0x180010e;
/* UOPS_DISPATCHED.THREAD,c=1,i=1 to count stall cycles*/
intel_perfmon_event_map[PERF_COUNT_HW_STALLED_CYCLES_BACKEND] = 0x18001b1;
pr_cont("SandyBridge events, ");
break;

5
arch/x86/kernel/cpu/perf_event_p4.c
View file

@ -912,8 +912,7 @@ static int p4_pmu_handle_irq(struct pt_regs *regs)
int idx, handled = 0;
u64 val;
data.addr = 0;
data.raw = NULL;
perf_sample_data_init(&data, 0);
cpuc = &__get_cpu_var(cpu_hw_events);
@ -1197,7 +1196,7 @@ static __init int p4_pmu_init(void)
{
unsigned int low, high;
/* If we get stripped -- indexig fails */
/* If we get stripped -- indexing fails */
BUILD_BUG_ON(ARCH_P4_MAX_CCCR > X86_PMC_MAX_GENERIC);
rdmsr(MSR_IA32_MISC_ENABLE, low, high);

16
arch/x86/kernel/dumpstack.c
View file

@ -135,20 +135,6 @@ print_context_stack_bp(struct thread_info *tinfo,
}
EXPORT_SYMBOL_GPL(print_context_stack_bp);
static void
print_trace_warning_symbol(void *data, char *msg, unsigned long symbol)
{
printk(data);
print_symbol(msg, symbol);
printk("\n");
}
static void print_trace_warning(void *data, char *msg)
{
printk("%s%s\n", (char *)data, msg);
}
static int print_trace_stack(void *data, char *name)
{
printk("%s <%s> ", (char *)data, name);
@ -166,8 +152,6 @@ static void print_trace_address(void *data, unsigned long addr, int reliable)
}
static const struct stacktrace_ops print_trace_ops = {
.warning = print_trace_warning,
.warning_symbol = print_trace_warning_symbol,
.stack = print_trace_stack,
.address = print_trace_address,
.walk_stack = print_context_stack,

1
arch/x86/kernel/module.c
View file

@ -24,6 +24,7 @@
#include <linux/bug.h>
#include <linux/mm.h>
#include <linux/gfp.h>
#include <linux/jump_label.h>
#include <asm/system.h>
#include <asm/page.h>

13
arch/x86/kernel/stacktrace.c
View file

@ -9,15 +9,6 @@
#include <linux/uaccess.h>
#include <asm/stacktrace.h>
static void save_stack_warning(void *data, char *msg)
{
}
static void
save_stack_warning_symbol(void *data, char *msg, unsigned long symbol)
{
}
static int save_stack_stack(void *data, char *name)
{
return 0;
@ -53,16 +44,12 @@ save_stack_address_nosched(void *data, unsigned long addr, int reliable)
}
static const struct stacktrace_ops save_stack_ops = {
.warning = save_stack_warning,
.warning_symbol = save_stack_warning_symbol,
.stack = save_stack_stack,
.address = save_stack_address,
.walk_stack = print_context_stack,
};
static const struct stacktrace_ops save_stack_ops_nosched = {
.warning = save_stack_warning,
.warning_symbol = save_stack_warning_symbol,
.stack = save_stack_stack,
.address = save_stack_address_nosched,
.walk_stack = print_context_stack,

33
arch/x86/lib/clear_page_64.S
View file

@ -1,5 +1,6 @@
#include <linux/linkage.h>
#include <asm/dwarf2.h>
#include <asm/alternative-asm.h>
/*
* Zero a page.
@ -14,6 +15,15 @@ ENTRY(clear_page_c)
CFI_ENDPROC
ENDPROC(clear_page_c)
ENTRY(clear_page_c_e)
CFI_STARTPROC
movl $4096,%ecx
xorl %eax,%eax
rep stosb
ret
CFI_ENDPROC
ENDPROC(clear_page_c_e)
ENTRY(clear_page)
CFI_STARTPROC
xorl %eax,%eax
@ -38,21 +48,26 @@ ENTRY(clear_page)
.Lclear_page_end:
ENDPROC(clear_page)
/* Some CPUs run faster using the string instructions.
It is also a lot simpler. Use this when possible */
/*
* Some CPUs support enhanced REP MOVSB/STOSB instructions.
* It is recommended to use this when possible.
* If enhanced REP MOVSB/STOSB is not available, try to use fast string.
* Otherwise, use original function.
*
*/
#include <asm/cpufeature.h>
.section .altinstr_replacement,"ax"
1: .byte 0xeb /* jmp <disp8> */
.byte (clear_page_c - clear_page) - (2f - 1b) /* offset */
2:
2: .byte 0xeb /* jmp <disp8> */
.byte (clear_page_c_e - clear_page) - (3f - 2b) /* offset */
3:
.previous
.section .altinstructions,"a"
.align 8
.quad clear_page
.quad 1b
.word X86_FEATURE_REP_GOOD
.byte .Lclear_page_end - clear_page
.byte 2b - 1b
altinstruction_entry clear_page,1b,X86_FEATURE_REP_GOOD,\
.Lclear_page_end-clear_page, 2b-1b
altinstruction_entry clear_page,2b,X86_FEATURE_ERMS, \
.Lclear_page_end-clear_page,3b-2b
.previous

69
arch/x86/lib/copy_user_64.S
View file

@ -15,23 +15,30 @@
#include <asm/asm-offsets.h>
#include <asm/thread_info.h>
#include <asm/cpufeature.h>
#include <asm/alternative-asm.h>
.macro ALTERNATIVE_JUMP feature,orig,alt
/*
* By placing feature2 after feature1 in altinstructions section, we logically
* implement:
* If CPU has feature2, jmp to alt2 is used
* else if CPU has feature1, jmp to alt1 is used
* else jmp to orig is used.
*/
.macro ALTERNATIVE_JUMP feature1,feature2,orig,alt1,alt2
0:
.byte 0xe9 /* 32bit jump */
.long \orig-1f /* by default jump to orig */
1:
.section .altinstr_replacement,"ax"
2: .byte 0xe9 /* near jump with 32bit immediate */
.long \alt-1b /* offset */ /* or alternatively to alt */
.long \alt1-1b /* offset */ /* or alternatively to alt1 */
3: .byte 0xe9 /* near jump with 32bit immediate */
.long \alt2-1b /* offset */ /* or alternatively to alt2 */
.previous
.section .altinstructions,"a"
.align 8
.quad 0b
.quad 2b
.word \feature /* when feature is set */
.byte 5
.byte 5
altinstruction_entry 0b,2b,\feature1,5,5
altinstruction_entry 0b,3b,\feature2,5,5
.previous
.endm
@ -72,8 +79,10 @@ ENTRY(_copy_to_user)
addq %rdx,%rcx
jc bad_to_user
cmpq TI_addr_limit(%rax),%rcx
jae bad_to_user
ALTERNATIVE_JUMP X86_FEATURE_REP_GOOD,copy_user_generic_unrolled,copy_user_generic_string
ja bad_to_user
ALTERNATIVE_JUMP X86_FEATURE_REP_GOOD,X86_FEATURE_ERMS, \
copy_user_generic_unrolled,copy_user_generic_string, \
copy_user_enhanced_fast_string
CFI_ENDPROC
ENDPROC(_copy_to_user)
@ -85,8 +94,10 @@ ENTRY(_copy_from_user)
addq %rdx,%rcx
jc bad_from_user
cmpq TI_addr_limit(%rax),%rcx
jae bad_from_user
ALTERNATIVE_JUMP X86_FEATURE_REP_GOOD,copy_user_generic_unrolled,copy_user_generic_string
ja bad_from_user
ALTERNATIVE_JUMP X86_FEATURE_REP_GOOD,X86_FEATURE_ERMS, \
copy_user_generic_unrolled,copy_user_generic_string, \
copy_user_enhanced_fast_string
CFI_ENDPROC
ENDPROC(_copy_from_user)
@ -255,3 +266,37 @@ ENTRY(copy_user_generic_string)
.previous
CFI_ENDPROC
ENDPROC(copy_user_generic_string)
/*
* Some CPUs are adding enhanced REP MOVSB/STOSB instructions.
* It's recommended to use enhanced REP MOVSB/STOSB if it's enabled.
*
* Input:
* rdi destination
* rsi source
* rdx count
*
* Output:
* eax uncopied bytes or 0 if successful.
*/
ENTRY(copy_user_enhanced_fast_string)
CFI_STARTPROC
andl %edx,%edx
jz 2f
movl %edx,%ecx
1: rep
movsb
2: xorl %eax,%eax
ret
.section .fixup,"ax"
12: movl %ecx,%edx /* ecx is zerorest also */
jmp copy_user_handle_tail
.previous
.section __ex_table,"a"
.align 8
.quad 1b,12b
.previous
CFI_ENDPROC
ENDPROC(copy_user_enhanced_fast_string)

45
arch/x86/lib/memcpy_64.S
View file

@ -4,6 +4,7 @@
#include <asm/cpufeature.h>
#include <asm/dwarf2.h>
#include <asm/alternative-asm.h>
/*
* memcpy - Copy a memory block.
@ -37,6 +38,23 @@
.Lmemcpy_e:
.previous
/*
* memcpy_c_e() - enhanced fast string memcpy. This is faster and simpler than
* memcpy_c. Use memcpy_c_e when possible.
*
* This gets patched over the unrolled variant (below) via the
* alternative instructions framework:
*/
.section .altinstr_replacement, "ax", @progbits
.Lmemcpy_c_e:
movq %rdi, %rax
movl %edx, %ecx
rep movsb
ret
.Lmemcpy_e_e:
.previous
ENTRY(__memcpy)
ENTRY(memcpy)
CFI_STARTPROC
@ -171,21 +189,22 @@ ENDPROC(memcpy)
ENDPROC(__memcpy)
/*
* Some CPUs run faster using the string copy instructions.
* It is also a lot simpler. Use this when possible:
*/
.section .altinstructions, "a"
.align 8
.quad memcpy
.quad .Lmemcpy_c
.word X86_FEATURE_REP_GOOD
/*
* Some CPUs are adding enhanced REP MOVSB/STOSB feature
* If the feature is supported, memcpy_c_e() is the first choice.
* If enhanced rep movsb copy is not available, use fast string copy
* memcpy_c() when possible. This is faster and code is simpler than
* original memcpy().
* Otherwise, original memcpy() is used.
* In .altinstructions section, ERMS feature is placed after REG_GOOD
* feature to implement the right patch order.
*
* Replace only beginning, memcpy is used to apply alternatives,
* so it is silly to overwrite itself with nops - reboot is the
* only outcome...
*/
.byte .Lmemcpy_e - .Lmemcpy_c
.byte .Lmemcpy_e - .Lmemcpy_c
.section .altinstructions, "a"
altinstruction_entry memcpy,.Lmemcpy_c,X86_FEATURE_REP_GOOD,\
.Lmemcpy_e-.Lmemcpy_c,.Lmemcpy_e-.Lmemcpy_c
altinstruction_entry memcpy,.Lmemcpy_c_e,X86_FEATURE_ERMS, \
.Lmemcpy_e_e-.Lmemcpy_c_e,.Lmemcpy_e_e-.Lmemcpy_c_e
.previous

29
arch/x86/lib/memmove_64.S
View file

@ -8,6 +8,7 @@
#define _STRING_C
#include <linux/linkage.h>
#include <asm/dwarf2.h>
#include <asm/cpufeature.h>
#undef memmove
@ -24,6 +25,7 @@
*/
ENTRY(memmove)
CFI_STARTPROC
/* Handle more 32bytes in loop */
mov %rdi, %rax
cmp $0x20, %rdx
@ -31,8 +33,13 @@ ENTRY(memmove)
/* Decide forward/backward copy mode */
cmp %rdi, %rsi
jb 2f
jge .Lmemmove_begin_forward
mov %rsi, %r8
add %rdx, %r8
cmp %rdi, %r8
jg 2f
.Lmemmove_begin_forward:
/*
* movsq instruction have many startup latency
* so we handle small size by general register.
@ -78,6 +85,8 @@ ENTRY(memmove)
rep movsq
movq %r11, (%r10)
jmp 13f
.Lmemmove_end_forward:
/*
* Handle data backward by movsq.
*/
@ -194,4 +203,22 @@ ENTRY(memmove)
13:
retq
CFI_ENDPROC
.section .altinstr_replacement,"ax"
.Lmemmove_begin_forward_efs:
/* Forward moving data. */
movq %rdx, %rcx
rep movsb
retq
.Lmemmove_end_forward_efs:
.previous
.section .altinstructions,"a"
.align 8
.quad .Lmemmove_begin_forward
.quad .Lmemmove_begin_forward_efs
.word X86_FEATURE_ERMS
.byte .Lmemmove_end_forward-.Lmemmove_begin_forward
.byte .Lmemmove_end_forward_efs-.Lmemmove_begin_forward_efs
.previous
ENDPROC(memmove)

54
arch/x86/lib/memset_64.S
View file

@ -2,9 +2,13 @@
#include <linux/linkage.h>
#include <asm/dwarf2.h>
#include <asm/cpufeature.h>
#include <asm/alternative-asm.h>
/*
* ISO C memset - set a memory block to a byte value.
* ISO C memset - set a memory block to a byte value. This function uses fast
* string to get better performance than the original function. The code is
* simpler and shorter than the orignal function as well.
*
* rdi destination
* rsi value (char)
@ -31,6 +35,28 @@
.Lmemset_e:
.previous
/*
* ISO C memset - set a memory block to a byte value. This function uses
* enhanced rep stosb to override the fast string function.
* The code is simpler and shorter than the fast string function as well.
*
* rdi destination
* rsi value (char)
* rdx count (bytes)
*
* rax original destination
*/
.section .altinstr_replacement, "ax", @progbits
.Lmemset_c_e:
movq %rdi,%r9
movb %sil,%al
movl %edx,%ecx
rep stosb
movq %r9,%rax
ret
.Lmemset_e_e:
.previous
ENTRY(memset)
ENTRY(__memset)
CFI_STARTPROC
@ -112,16 +138,20 @@ ENTRY(__memset)
ENDPROC(memset)
ENDPROC(__memset)
/* Some CPUs run faster using the string instructions.
It is also a lot simpler. Use this when possible */
#include <asm/cpufeature.h>
/* Some CPUs support enhanced REP MOVSB/STOSB feature.
* It is recommended to use this when possible.
*
* If enhanced REP MOVSB/STOSB feature is not available, use fast string
* instructions.
*
* Otherwise, use original memset function.
*
* In .altinstructions section, ERMS feature is placed after REG_GOOD
* feature to implement the right patch order.
*/
.section .altinstructions,"a"
.align 8
.quad memset
.quad .Lmemset_c
.word X86_FEATURE_REP_GOOD
.byte .Lfinal - memset
.byte .Lmemset_e - .Lmemset_c
altinstruction_entry memset,.Lmemset_c,X86_FEATURE_REP_GOOD,\
.Lfinal-memset,.Lmemset_e-.Lmemset_c
altinstruction_entry memset,.Lmemset_c_e,X86_FEATURE_ERMS, \
.Lfinal-memset,.Lmemset_e_e-.Lmemset_c_e
.previous

13
arch/x86/oprofile/backtrace.c
View file

@ -16,17 +16,6 @@
#include <asm/stacktrace.h>
#include <linux/compat.h>
static void backtrace_warning_symbol(void *data, char *msg,
unsigned long symbol)
{
/* Ignore warnings */
}
static void backtrace_warning(void *data, char *msg)
{
/* Ignore warnings */
}
static int backtrace_stack(void *data, char *name)
{
/* Yes, we want all stacks */
@ -42,8 +31,6 @@ static void backtrace_address(void *data, unsigned long addr, int reliable)
}
static struct stacktrace_ops backtrace_ops = {
.warning = backtrace_warning,
.warning_symbol = backtrace_warning_symbol,
.stack = backtrace_stack,
.address = backtrace_address,
.walk_stack = print_context_stack,

14
include/asm-generic/vmlinux.lds.h
View file

@ -171,6 +171,10 @@
*(__tracepoints) \
/* implement dynamic printk debug */ \
. = ALIGN(8); \
VMLINUX_SYMBOL(__start___jump_table) = .; \
*(__jump_table) \
VMLINUX_SYMBOL(__stop___jump_table) = .; \
. = ALIGN(8); \
VMLINUX_SYMBOL(__start___verbose) = .; \
*(__verbose) \
VMLINUX_SYMBOL(__stop___verbose) = .; \
@ -228,8 +232,6 @@
\
BUG_TABLE \
\
JUMP_TABLE \
\
/* PCI quirks */ \
.pci_fixup : AT(ADDR(.pci_fixup) - LOAD_OFFSET) { \
VMLINUX_SYMBOL(__start_pci_fixups_early) = .; \
@ -589,14 +591,6 @@
#define BUG_TABLE
#endif
#define JUMP_TABLE \
. = ALIGN(8); \
__jump_table : AT(ADDR(__jump_table) - LOAD_OFFSET) { \
VMLINUX_SYMBOL(__start___jump_table) = .; \
*(__jump_table) \
VMLINUX_SYMBOL(__stop___jump_table) = .; \
}
#ifdef CONFIG_PM_TRACE
#define TRACEDATA \
. = ALIGN(4); \

2
include/linux/dynamic_debug.h
View file

@ -1,8 +1,6 @@
#ifndef _DYNAMIC_DEBUG_H
#define _DYNAMIC_DEBUG_H
#include <linux/jump_label.h>
/* dynamic_printk_enabled, and dynamic_printk_enabled2 are bitmasks in which
* bit n is set to 1 if any modname hashes into the bucket n, 0 otherwise. They
* use independent hash functions, to reduce the chance of false positives.

31
include/linux/ftrace.h
View file

@ -29,9 +29,22 @@ ftrace_enable_sysctl(struct ctl_table *table, int write,
typedef void (*ftrace_func_t)(unsigned long ip, unsigned long parent_ip);
struct ftrace_hash;
enum {
FTRACE_OPS_FL_ENABLED = 1 << 0,
FTRACE_OPS_FL_GLOBAL = 1 << 1,
FTRACE_OPS_FL_DYNAMIC = 1 << 2,
};
struct ftrace_ops {
ftrace_func_t func;
struct ftrace_ops *next;
unsigned long flags;
#ifdef CONFIG_DYNAMIC_FTRACE
struct ftrace_hash *notrace_hash;
struct ftrace_hash *filter_hash;
#endif
};
extern int function_trace_stop;
@ -146,14 +159,13 @@ extern void unregister_ftrace_function_probe_all(char *glob);
extern int ftrace_text_reserved(void *start, void *end);
enum {
FTRACE_FL_FREE = (1 << 0),
FTRACE_FL_FAILED = (1 << 1),
FTRACE_FL_FILTER = (1 << 2),
FTRACE_FL_ENABLED = (1 << 3),
FTRACE_FL_NOTRACE = (1 << 4),
FTRACE_FL_CONVERTED = (1 << 5),
FTRACE_FL_ENABLED = (1 << 30),
FTRACE_FL_FREE = (1 << 31),
};
#define FTRACE_FL_MASK (0x3UL << 30)
#define FTRACE_REF_MAX ((1 << 30) - 1)
struct dyn_ftrace {
union {
unsigned long ip; /* address of mcount call-site */
@ -167,7 +179,12 @@ struct dyn_ftrace {
};
int ftrace_force_update(void);
void ftrace_set_filter(unsigned char *buf, int len, int reset);
void ftrace_set_filter(struct ftrace_ops *ops, unsigned char *buf,
int len, int reset);
void ftrace_set_notrace(struct ftrace_ops *ops, unsigned char *buf,
int len, int reset);
void ftrace_set_global_filter(unsigned char *buf, int len, int reset);
void ftrace_set_global_notrace(unsigned char *buf, int len, int reset);
int register_ftrace_command(struct ftrace_func_command *cmd);
int unregister_ftrace_command(struct ftrace_func_command *cmd);

14
include/linux/init.h
View file

@ -79,29 +79,29 @@
#define __exitused __used
#endif
#define __exit __section(.exit.text) __exitused __cold
#define __exit __section(.exit.text) __exitused __cold notrace
/* Used for HOTPLUG */
#define __devinit __section(.devinit.text) __cold
#define __devinit __section(.devinit.text) __cold notrace
#define __devinitdata __section(.devinit.data)
#define __devinitconst __section(.devinit.rodata)
#define __devexit __section(.devexit.text) __exitused __cold
#define __devexit __section(.devexit.text) __exitused __cold notrace
#define __devexitdata __section(.devexit.data)
#define __devexitconst __section(.devexit.rodata)
/* Used for HOTPLUG_CPU */
#define __cpuinit __section(.cpuinit.text) __cold
#define __cpuinit __section(.cpuinit.text) __cold notrace
#define __cpuinitdata __section(.cpuinit.data)
#define __cpuinitconst __section(.cpuinit.rodata)
#define __cpuexit __section(.cpuexit.text) __exitused __cold
#define __cpuexit __section(.cpuexit.text) __exitused __cold notrace
#define __cpuexitdata __section(.cpuexit.data)
#define __cpuexitconst __section(.cpuexit.rodata)
/* Used for MEMORY_HOTPLUG */
#define __meminit __section(.meminit.text) __cold
#define __meminit __section(.meminit.text) __cold notrace
#define __meminitdata __section(.meminit.data)
#define __meminitconst __section(.meminit.rodata)
#define __memexit __section(.memexit.text) __exitused __cold
#define __memexit __section(.memexit.text) __exitused __cold notrace
#define __memexitdata __section(.memexit.data)
#define __memexitconst __section(.memexit.rodata)

91
include/linux/jump_label.h
View file

@ -1,20 +1,43 @@
#ifndef _LINUX_JUMP_LABEL_H
#define _LINUX_JUMP_LABEL_H
#include <linux/types.h>
#include <linux/compiler.h>
#if defined(CC_HAVE_ASM_GOTO) && defined(CONFIG_JUMP_LABEL)
struct jump_label_key {
atomic_t enabled;
struct jump_entry *entries;
#ifdef CONFIG_MODULES
struct jump_label_mod *next;
#endif
};
# include <asm/jump_label.h>
# define HAVE_JUMP_LABEL
#endif
enum jump_label_type {
JUMP_LABEL_DISABLE = 0,
JUMP_LABEL_ENABLE,
JUMP_LABEL_DISABLE
};
struct module;
#ifdef HAVE_JUMP_LABEL
#ifdef CONFIG_MODULES
#define JUMP_LABEL_INIT {{ 0 }, NULL, NULL}
#else
#define JUMP_LABEL_INIT {{ 0 }, NULL}
#endif
static __always_inline bool static_branch(struct jump_label_key *key)
{
return arch_static_branch(key);
}
extern struct jump_entry __start___jump_table[];
extern struct jump_entry __stop___jump_table[];
@ -23,37 +46,37 @@ extern void jump_label_unlock(void);
extern void arch_jump_label_transform(struct jump_entry *entry,
enum jump_label_type type);
extern void arch_jump_label_text_poke_early(jump_label_t addr);
extern void jump_label_update(unsigned long key, enum jump_label_type type);
extern void jump_label_apply_nops(struct module *mod);
extern int jump_label_text_reserved(void *start, void *end);
#define jump_label_enable(key) \
jump_label_update((unsigned long)key, JUMP_LABEL_ENABLE);
#define jump_label_disable(key) \
jump_label_update((unsigned long)key, JUMP_LABEL_DISABLE);
extern void jump_label_inc(struct jump_label_key *key);
extern void jump_label_dec(struct jump_label_key *key);
extern bool jump_label_enabled(struct jump_label_key *key);
extern void jump_label_apply_nops(struct module *mod);
#else
#define JUMP_LABEL(key, label) \
do { \
if (unlikely(*key)) \
goto label; \
} while (0)
#include <asm/atomic.h>
#define jump_label_enable(cond_var) \
do { \
*(cond_var) = 1; \
} while (0)
#define JUMP_LABEL_INIT {ATOMIC_INIT(0)}
#define jump_label_disable(cond_var) \
do { \
*(cond_var) = 0; \
} while (0)
struct jump_label_key {
atomic_t enabled;
};
static inline int jump_label_apply_nops(struct module *mod)
static __always_inline bool static_branch(struct jump_label_key *key)
{
return 0;
if (unlikely(atomic_read(&key->enabled)))
return true;
return false;
}
static inline void jump_label_inc(struct jump_label_key *key)
{
atomic_inc(&key->enabled);
}
static inline void jump_label_dec(struct jump_label_key *key)
{
atomic_dec(&key->enabled);
}
static inline int jump_label_text_reserved(void *start, void *end)
@ -64,16 +87,16 @@ static inline int jump_label_text_reserved(void *start, void *end)
static inline void jump_label_lock(void) {}
static inline void jump_label_unlock(void) {}
#endif
static inline bool jump_label_enabled(struct jump_label_key *key)
{
return !!atomic_read(&key->enabled);
}
#define COND_STMT(key, stmt) \
do { \
__label__ jl_enabled; \
JUMP_LABEL(key, jl_enabled); \
if (0) { \
jl_enabled: \
stmt; \
} \
} while (0)
static inline int jump_label_apply_nops(struct module *mod)
{
return 0;
}
#endif
#endif

44
include/linux/jump_label_ref.h
View file

@ -1,44 +0,0 @@
#ifndef _LINUX_JUMP_LABEL_REF_H
#define _LINUX_JUMP_LABEL_REF_H
#include <linux/jump_label.h>
#include <asm/atomic.h>
#ifdef HAVE_JUMP_LABEL
static inline void jump_label_inc(atomic_t *key)
{
if (atomic_add_return(1, key) == 1)
jump_label_enable(key);
}
static inline void jump_label_dec(atomic_t *key)
{
if (atomic_dec_and_test(key))
jump_label_disable(key);
}
#else /* !HAVE_JUMP_LABEL */
static inline void jump_label_inc(atomic_t *key)
{
atomic_inc(key);
}
static inline void jump_label_dec(atomic_t *key)
{
atomic_dec(key);
}
#undef JUMP_LABEL
#define JUMP_LABEL(key, label) \
do { \
if (unlikely(__builtin_choose_expr( \
__builtin_types_compatible_p(typeof(key), atomic_t *), \
atomic_read((atomic_t *)(key)), *(key)))) \
goto label; \
} while (0)
#endif /* HAVE_JUMP_LABEL */
#endif /* _LINUX_JUMP_LABEL_REF_H */

1
include/linux/kernel.h
View file

@ -283,6 +283,7 @@ extern char *get_options(const char *str, int nints, int *ints);
extern unsigned long long memparse(const char *ptr, char **retptr);
extern int core_kernel_text(unsigned long addr);
extern int core_kernel_data(unsigned long addr);
extern int __kernel_text_address(unsigned long addr);
extern int kernel_text_address(unsigned long addr);
extern int func_ptr_is_kernel_text(void *ptr);

55
include/linux/perf_event.h
View file

@ -2,8 +2,8 @@
* Performance events:
*
* Copyright (C) 2008-2009, Thomas Gleixner <tglx@linutronix.de>
* Copyright (C) 2008-2009, Red Hat, Inc., Ingo Molnar
* Copyright (C) 2008-2009, Red Hat, Inc., Peter Zijlstra
* Copyright (C) 2008-2011, Red Hat, Inc., Ingo Molnar
* Copyright (C) 2008-2011, Red Hat, Inc., Peter Zijlstra
*
* Data type definitions, declarations, prototypes.
*
@ -52,6 +52,8 @@ enum perf_hw_id {
PERF_COUNT_HW_BRANCH_INSTRUCTIONS = 4,
PERF_COUNT_HW_BRANCH_MISSES = 5,
PERF_COUNT_HW_BUS_CYCLES = 6,
PERF_COUNT_HW_STALLED_CYCLES_FRONTEND = 7,
PERF_COUNT_HW_STALLED_CYCLES_BACKEND = 8,
PERF_COUNT_HW_MAX, /* non-ABI */
};
@ -484,9 +486,9 @@ enum perf_callchain_context {
#endif
struct perf_guest_info_callbacks {
int (*is_in_guest) (void);
int (*is_user_mode) (void);
unsigned long (*get_guest_ip) (void);
int (*is_in_guest)(void);
int (*is_user_mode)(void);
unsigned long (*get_guest_ip)(void);
};
#ifdef CONFIG_HAVE_HW_BREAKPOINT
@ -505,7 +507,7 @@ struct perf_guest_info_callbacks {
#include <linux/ftrace.h>
#include <linux/cpu.h>
#include <linux/irq_work.h>
#include <linux/jump_label_ref.h>
#include <linux/jump_label.h>
#include <asm/atomic.h>
#include <asm/local.h>
@ -1001,8 +1003,7 @@ struct perf_sample_data {
struct perf_raw_record *raw;
};
static inline
void perf_sample_data_init(struct perf_sample_data *data, u64 addr)
static inline void perf_sample_data_init(struct perf_sample_data *data, u64 addr)
{
data->addr = addr;
data->raw = NULL;
@ -1034,13 +1035,12 @@ static inline int is_software_event(struct perf_event *event)
return event->pmu->task_ctx_nr == perf_sw_context;
}
extern atomic_t perf_swevent_enabled[PERF_COUNT_SW_MAX];
extern struct jump_label_key perf_swevent_enabled[PERF_COUNT_SW_MAX];
extern void __perf_sw_event(u32, u64, int, struct pt_regs *, u64);
#ifndef perf_arch_fetch_caller_regs
static inline void
perf_arch_fetch_caller_regs(struct pt_regs *regs, unsigned long ip) { }
static inline void perf_arch_fetch_caller_regs(struct pt_regs *regs, unsigned long ip) { }
#endif
/*
@ -1063,26 +1063,24 @@ perf_sw_event(u32 event_id, u64 nr, int nmi, struct pt_regs *regs, u64 addr)
{
struct pt_regs hot_regs;
JUMP_LABEL(&perf_swevent_enabled[event_id], have_event);
return;
have_event:
if (static_branch(&perf_swevent_enabled[event_id])) {
if (!regs) {
perf_fetch_caller_regs(&hot_regs);
regs = &hot_regs;
}
__perf_sw_event(event_id, nr, nmi, regs, addr);
}
}
extern atomic_t perf_sched_events;
extern struct jump_label_key perf_sched_events;
static inline void perf_event_task_sched_in(struct task_struct *task)
{
COND_STMT(&perf_sched_events, __perf_event_task_sched_in(task));
if (static_branch(&perf_sched_events))
__perf_event_task_sched_in(task);
}
static inline
void perf_event_task_sched_out(struct task_struct *task, struct task_struct *next)
static inline void perf_event_task_sched_out(struct task_struct *task, struct task_struct *next)
{
perf_sw_event(PERF_COUNT_SW_CONTEXT_SWITCHES, 1, 1, NULL, 0);
@ -1100,14 +1098,10 @@ extern void perf_event_fork(struct task_struct *tsk);
/* Callchains */
DECLARE_PER_CPU(struct perf_callchain_entry, perf_callchain_entry);
extern void perf_callchain_user(struct perf_callchain_entry *entry,
struct pt_regs *regs);
extern void perf_callchain_kernel(struct perf_callchain_entry *entry,
struct pt_regs *regs);
extern void perf_callchain_user(struct perf_callchain_entry *entry, struct pt_regs *regs);
extern void perf_callchain_kernel(struct perf_callchain_entry *entry, struct pt_regs *regs);
static inline void
perf_callchain_store(struct perf_callchain_entry *entry, u64 ip)
static inline void perf_callchain_store(struct perf_callchain_entry *entry, u64 ip)
{
if (entry->nr < PERF_MAX_STACK_DEPTH)
entry->ip[entry->nr++] = ip;
@ -1143,9 +1137,9 @@ extern void perf_tp_event(u64 addr, u64 count, void *record,
extern void perf_bp_event(struct perf_event *event, void *data);
#ifndef perf_misc_flags
#define perf_misc_flags(regs) (user_mode(regs) ? PERF_RECORD_MISC_USER : \
PERF_RECORD_MISC_KERNEL)
#define perf_instruction_pointer(regs) instruction_pointer(regs)
# define perf_misc_flags(regs) \
(user_mode(regs) ? PERF_RECORD_MISC_USER : PERF_RECORD_MISC_KERNEL)
# define perf_instruction_pointer(regs) instruction_pointer(regs)
#endif
extern int perf_output_begin(struct perf_output_handle *handle,
@ -1195,8 +1189,7 @@ static inline void perf_event_disable(struct perf_event *event) { }
static inline void perf_event_task_tick(void) { }
#endif
#define perf_output_put(handle, x) \
perf_output_copy((handle), &(x), sizeof(x))
#define perf_output_put(handle, x) perf_output_copy((handle), &(x), sizeof(x))
/*
* This has to have a higher priority than migration_notifier in sched.c.

8
include/linux/tracepoint.h
View file

@ -29,7 +29,7 @@ struct tracepoint_func {
struct tracepoint {
const char *name; /* Tracepoint name */
int state; /* State. */
struct jump_label_key key;
void (*regfunc)(void);
void (*unregfunc)(void);
struct tracepoint_func __rcu *funcs;
@ -146,9 +146,7 @@ void tracepoint_update_probe_range(struct tracepoint * const *begin,
extern struct tracepoint __tracepoint_##name; \
static inline void trace_##name(proto) \
{ \
JUMP_LABEL(&__tracepoint_##name.state, do_trace); \
return; \
do_trace: \
if (static_branch(&__tracepoint_##name.key)) \
__DO_TRACE(&__tracepoint_##name, \
TP_PROTO(data_proto), \
TP_ARGS(data_args), \
@ -181,7 +179,7 @@ do_trace: \
__attribute__((section("__tracepoints_strings"))) = #name; \
struct tracepoint __tracepoint_##name \
__attribute__((section("__tracepoints"))) = \
{ __tpstrtab_##name, 0, reg, unreg, NULL }; \
{ __tpstrtab_##name, JUMP_LABEL_INIT, reg, unreg, NULL };\
static struct tracepoint * const __tracepoint_ptr_##name __used \
__attribute__((section("__tracepoints_ptrs"))) = \
&__tracepoint_##name;

6
kernel/Makefile
View file

@ -21,7 +21,6 @@ CFLAGS_REMOVE_mutex-debug.o = -pg
CFLAGS_REMOVE_rtmutex-debug.o = -pg
CFLAGS_REMOVE_cgroup-debug.o = -pg
CFLAGS_REMOVE_sched_clock.o = -pg
CFLAGS_REMOVE_perf_event.o = -pg
CFLAGS_REMOVE_irq_work.o = -pg
endif
@ -103,8 +102,9 @@ obj-$(CONFIG_RING_BUFFER) += trace/
obj-$(CONFIG_TRACEPOINTS) += trace/
obj-$(CONFIG_SMP) += sched_cpupri.o
obj-$(CONFIG_IRQ_WORK) += irq_work.o
obj-$(CONFIG_PERF_EVENTS) += perf_event.o
obj-$(CONFIG_HAVE_HW_BREAKPOINT) += hw_breakpoint.o
obj-$(CONFIG_PERF_EVENTS) += events/
obj-$(CONFIG_USER_RETURN_NOTIFIER) += user-return-notifier.o
obj-$(CONFIG_PADATA) += padata.o
obj-$(CONFIG_CRASH_DUMP) += crash_dump.o

6
kernel/events/Makefile Normal file
View file

@ -0,0 +1,6 @@
ifdef CONFIG_FUNCTION_TRACER
CFLAGS_REMOVE_core.o = -pg
endif
obj-y := core.o
obj-$(CONFIG_HAVE_HW_BREAKPOINT) += hw_breakpoint.o

8
kernel/perf_event.c → kernel/events/core.c
View file

@ -2,8 +2,8 @@
* Performance events core code:
*
* Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de>
* Copyright (C) 2008-2009 Red Hat, Inc., Ingo Molnar
* Copyright (C) 2008-2009 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com>
* Copyright (C) 2008-2011 Red Hat, Inc., Ingo Molnar
* Copyright (C) 2008-2011 Red Hat, Inc., Peter Zijlstra <pzijlstr@redhat.com>
* Copyright © 2009 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com>
*
* For licensing details see kernel-base/COPYING
@ -125,7 +125,7 @@ enum event_type_t {
* perf_sched_events : >0 events exist
* perf_cgroup_events: >0 per-cpu cgroup events exist on this cpu
*/
atomic_t perf_sched_events __read_mostly;
struct jump_label_key perf_sched_events __read_mostly;
static DEFINE_PER_CPU(atomic_t, perf_cgroup_events);
static atomic_t nr_mmap_events __read_mostly;
@ -5429,7 +5429,7 @@ fail:
return err;
}
atomic_t perf_swevent_enabled[PERF_COUNT_SW_MAX];
struct jump_label_key perf_swevent_enabled[PERF_COUNT_SW_MAX];
static void sw_perf_event_destroy(struct perf_event *event)
{

0
kernel/hw_breakpoint.c → kernel/events/hw_breakpoint.c
View file

8
kernel/extable.c
View file

@ -72,6 +72,14 @@ int core_kernel_text(unsigned long addr)
return 0;
}
int core_kernel_data(unsigned long addr)
{
if (addr >= (unsigned long)_sdata &&
addr < (unsigned long)_edata)
return 1;
return 0;
}
int __kernel_text_address(unsigned long addr)
{
if (core_kernel_text(addr))

661
kernel/jump_label.c
View file

@ -2,43 +2,23 @@
* jump label support
*
* Copyright (C) 2009 Jason Baron <jbaron@redhat.com>
* Copyright (C) 2011 Peter Zijlstra <pzijlstr@redhat.com>
*
*/
#include <linux/jump_label.h>
#include <linux/memory.h>
#include <linux/uaccess.h>
#include <linux/module.h>
#include <linux/list.h>
#include <linux/jhash.h>
#include <linux/slab.h>
#include <linux/sort.h>
#include <linux/err.h>
#include <linux/jump_label.h>
#ifdef HAVE_JUMP_LABEL
#define JUMP_LABEL_HASH_BITS 6
#define JUMP_LABEL_TABLE_SIZE (1 << JUMP_LABEL_HASH_BITS)
static struct hlist_head jump_label_table[JUMP_LABEL_TABLE_SIZE];
/* mutex to protect coming/going of the the jump_label table */
static DEFINE_MUTEX(jump_label_mutex);
struct jump_label_entry {
struct hlist_node hlist;
struct jump_entry *table;
int nr_entries;
/* hang modules off here */
struct hlist_head modules;
unsigned long key;
};
struct jump_label_module_entry {
struct hlist_node hlist;
struct jump_entry *table;
int nr_entries;
struct module *mod;
};
void jump_label_lock(void)
{
mutex_lock(&jump_label_mutex);
@ -49,6 +29,11 @@ void jump_label_unlock(void)
mutex_unlock(&jump_label_mutex);
}
bool jump_label_enabled(struct jump_label_key *key)
{
return !!atomic_read(&key->enabled);
}
static int jump_label_cmp(const void *a, const void *b)
{
const struct jump_entry *jea = a;
@ -64,7 +49,7 @@ static int jump_label_cmp(const void *a, const void *b)
}
static void
sort_jump_label_entries(struct jump_entry *start, struct jump_entry *stop)
jump_label_sort_entries(struct jump_entry *start, struct jump_entry *stop)
{
unsigned long size;
@ -73,118 +58,25 @@ sort_jump_label_entries(struct jump_entry *start, struct jump_entry *stop)
sort(start, size, sizeof(struct jump_entry), jump_label_cmp, NULL);
}
static struct jump_label_entry *get_jump_label_entry(jump_label_t key)
static void jump_label_update(struct jump_label_key *key, int enable);
void jump_label_inc(struct jump_label_key *key)
{
struct hlist_head *head;
struct hlist_node *node;
struct jump_label_entry *e;
u32 hash = jhash((void *)&key, sizeof(jump_label_t), 0);
head = &jump_label_table[hash & (JUMP_LABEL_TABLE_SIZE - 1)];
hlist_for_each_entry(e, node, head, hlist) {
if (key == e->key)
return e;
}
return NULL;
}
static struct jump_label_entry *
add_jump_label_entry(jump_label_t key, int nr_entries, struct jump_entry *table)
{
struct hlist_head *head;
struct jump_label_entry *e;
u32 hash;
e = get_jump_label_entry(key);
if (e)
return ERR_PTR(-EEXIST);
e = kmalloc(sizeof(struct jump_label_entry), GFP_KERNEL);
if (!e)
return ERR_PTR(-ENOMEM);
hash = jhash((void *)&key, sizeof(jump_label_t), 0);
head = &jump_label_table[hash & (JUMP_LABEL_TABLE_SIZE - 1)];
e->key = key;
e->table = table;
e->nr_entries = nr_entries;
INIT_HLIST_HEAD(&(e->modules));
hlist_add_head(&e->hlist, head);
return e;
}
static int
build_jump_label_hashtable(struct jump_entry *start, struct jump_entry *stop)
{
struct jump_entry *iter, *iter_begin;
struct jump_label_entry *entry;
int count;
sort_jump_label_entries(start, stop);
iter = start;
while (iter < stop) {
entry = get_jump_label_entry(iter->key);
if (!entry) {
iter_begin = iter;
count = 0;
while ((iter < stop) &&
(iter->key == iter_begin->key)) {
iter++;
count++;
}
entry = add_jump_label_entry(iter_begin->key,
count, iter_begin);
if (IS_ERR(entry))
return PTR_ERR(entry);
} else {
WARN_ONCE(1, KERN_ERR "build_jump_hashtable: unexpected entry!\n");
return -1;
}
}
return 0;
}
/***
* jump_label_update - update jump label text
* @key - key value associated with a a jump label
* @type - enum set to JUMP_LABEL_ENABLE or JUMP_LABEL_DISABLE
*
* Will enable/disable the jump for jump label @key, depending on the
* value of @type.
*
*/
void jump_label_update(unsigned long key, enum jump_label_type type)
{
struct jump_entry *iter;
struct jump_label_entry *entry;
struct hlist_node *module_node;
struct jump_label_module_entry *e_module;
int count;
if (atomic_inc_not_zero(&key->enabled))
return;
jump_label_lock();
entry = get_jump_label_entry((jump_label_t)key);
if (entry) {
count = entry->nr_entries;
iter = entry->table;
while (count--) {
if (kernel_text_address(iter->code))
arch_jump_label_transform(iter, type);
iter++;
}
/* eanble/disable jump labels in modules */
hlist_for_each_entry(e_module, module_node, &(entry->modules),
hlist) {
count = e_module->nr_entries;
iter = e_module->table;
while (count--) {
if (iter->key &&
kernel_text_address(iter->code))
arch_jump_label_transform(iter, type);
iter++;
}
}
}
if (atomic_add_return(1, &key->enabled) == 1)
jump_label_update(key, JUMP_LABEL_ENABLE);
jump_label_unlock();
}
void jump_label_dec(struct jump_label_key *key)
{
if (!atomic_dec_and_mutex_lock(&key->enabled, &jump_label_mutex))
return;
jump_label_update(key, JUMP_LABEL_DISABLE);
jump_label_unlock();
}
@ -197,41 +89,254 @@ static int addr_conflict(struct jump_entry *entry, void *start, void *end)
return 0;
}
#ifdef CONFIG_MODULES
static int module_conflict(void *start, void *end)
static int __jump_label_text_reserved(struct jump_entry *iter_start,
struct jump_entry *iter_stop, void *start, void *end)
{
struct hlist_head *head;
struct hlist_node *node, *node_next, *module_node, *module_node_next;
struct jump_label_entry *e;
struct jump_label_module_entry *e_module;
struct jump_entry *iter;
int i, count;
int conflict = 0;
for (i = 0; i < JUMP_LABEL_TABLE_SIZE; i++) {
head = &jump_label_table[i];
hlist_for_each_entry_safe(e, node, node_next, head, hlist) {
hlist_for_each_entry_safe(e_module, module_node,
module_node_next,
&(e->modules), hlist) {
count = e_module->nr_entries;
iter = e_module->table;
while (count--) {
if (addr_conflict(iter, start, end)) {
conflict = 1;
goto out;
}
iter = iter_start;
while (iter < iter_stop) {
if (addr_conflict(iter, start, end))
return 1;
iter++;
}
}
}
}
out:
return conflict;
return 0;
}
static void __jump_label_update(struct jump_label_key *key,
struct jump_entry *entry, int enable)
{
for (; entry->key == (jump_label_t)(unsigned long)key; entry++) {
/*
* entry->code set to 0 invalidates module init text sections
* kernel_text_address() verifies we are not in core kernel
* init code, see jump_label_invalidate_module_init().
*/
if (entry->code && kernel_text_address(entry->code))
arch_jump_label_transform(entry, enable);
}
}
/*
* Not all archs need this.
*/
void __weak arch_jump_label_text_poke_early(jump_label_t addr)
{
}
static __init int jump_label_init(void)
{
struct jump_entry *iter_start = __start___jump_table;
struct jump_entry *iter_stop = __stop___jump_table;
struct jump_label_key *key = NULL;
struct jump_entry *iter;
jump_label_lock();
jump_label_sort_entries(iter_start, iter_stop);
for (iter = iter_start; iter < iter_stop; iter++) {
arch_jump_label_text_poke_early(iter->code);
if (iter->key == (jump_label_t)(unsigned long)key)
continue;
key = (struct jump_label_key *)(unsigned long)iter->key;
atomic_set(&key->enabled, 0);
key->entries = iter;
#ifdef CONFIG_MODULES
key->next = NULL;
#endif
}
jump_label_unlock();
return 0;
}
early_initcall(jump_label_init);
#ifdef CONFIG_MODULES
struct jump_label_mod {
struct jump_label_mod *next;
struct jump_entry *entries;
struct module *mod;
};
static int __jump_label_mod_text_reserved(void *start, void *end)
{
struct module *mod;
mod = __module_text_address((unsigned long)start);
if (!mod)
return 0;
WARN_ON_ONCE(__module_text_address((unsigned long)end) != mod);
return __jump_label_text_reserved(mod->jump_entries,
mod->jump_entries + mod->num_jump_entries,
start, end);
}
static void __jump_label_mod_update(struct jump_label_key *key, int enable)
{
struct jump_label_mod *mod = key->next;
while (mod) {
__jump_label_update(key, mod->entries, enable);
mod = mod->next;
}
}
/***
* apply_jump_label_nops - patch module jump labels with arch_get_jump_label_nop()
* @mod: module to patch
*
* Allow for run-time selection of the optimal nops. Before the module
* loads patch these with arch_get_jump_label_nop(), which is specified by
* the arch specific jump label code.
*/
void jump_label_apply_nops(struct module *mod)
{
struct jump_entry *iter_start = mod->jump_entries;
struct jump_entry *iter_stop = iter_start + mod->num_jump_entries;
struct jump_entry *iter;
/* if the module doesn't have jump label entries, just return */
if (iter_start == iter_stop)
return;
for (iter = iter_start; iter < iter_stop; iter++)
arch_jump_label_text_poke_early(iter->code);
}
static int jump_label_add_module(struct module *mod)
{
struct jump_entry *iter_start = mod->jump_entries;
struct jump_entry *iter_stop = iter_start + mod->num_jump_entries;
struct jump_entry *iter;
struct jump_label_key *key = NULL;
struct jump_label_mod *jlm;
/* if the module doesn't have jump label entries, just return */
if (iter_start == iter_stop)
return 0;
jump_label_sort_entries(iter_start, iter_stop);
for (iter = iter_start; iter < iter_stop; iter++) {
if (iter->key == (jump_label_t)(unsigned long)key)
continue;
key = (struct jump_label_key *)(unsigned long)iter->key;
if (__module_address(iter->key) == mod) {
atomic_set(&key->enabled, 0);
key->entries = iter;
key->next = NULL;
continue;
}
jlm = kzalloc(sizeof(struct jump_label_mod), GFP_KERNEL);
if (!jlm)
return -ENOMEM;
jlm->mod = mod;
jlm->entries = iter;
jlm->next = key->next;
key->next = jlm;
if (jump_label_enabled(key))
__jump_label_update(key, iter, JUMP_LABEL_ENABLE);
}
return 0;
}
static void jump_label_del_module(struct module *mod)
{
struct jump_entry *iter_start = mod->jump_entries;
struct jump_entry *iter_stop = iter_start + mod->num_jump_entries;
struct jump_entry *iter;
struct jump_label_key *key = NULL;
struct jump_label_mod *jlm, **prev;
for (iter = iter_start; iter < iter_stop; iter++) {
if (iter->key == (jump_label_t)(unsigned long)key)
continue;
key = (struct jump_label_key *)(unsigned long)iter->key;
if (__module_address(iter->key) == mod)
continue;
prev = &key->next;
jlm = key->next;
while (jlm && jlm->mod != mod) {
prev = &jlm->next;
jlm = jlm->next;
}
if (jlm) {
*prev = jlm->next;
kfree(jlm);
}
}
}
static void jump_label_invalidate_module_init(struct module *mod)
{
struct jump_entry *iter_start = mod->jump_entries;
struct jump_entry *iter_stop = iter_start + mod->num_jump_entries;
struct jump_entry *iter;
for (iter = iter_start; iter < iter_stop; iter++) {
if (within_module_init(iter->code, mod))
iter->code = 0;
}
}
static int
jump_label_module_notify(struct notifier_block *self, unsigned long val,
void *data)
{
struct module *mod = data;
int ret = 0;
switch (val) {
case MODULE_STATE_COMING:
jump_label_lock();
ret = jump_label_add_module(mod);
if (ret)
jump_label_del_module(mod);
jump_label_unlock();
break;
case MODULE_STATE_GOING:
jump_label_lock();
jump_label_del_module(mod);
jump_label_unlock();
break;
case MODULE_STATE_LIVE:
jump_label_lock();
jump_label_invalidate_module_init(mod);
jump_label_unlock();
break;
}
return notifier_from_errno(ret);
}
struct notifier_block jump_label_module_nb = {
.notifier_call = jump_label_module_notify,
.priority = 1, /* higher than tracepoints */
};
static __init int jump_label_init_module(void)
{
return register_module_notifier(&jump_label_module_nb);
}
early_initcall(jump_label_init_module);
#endif /* CONFIG_MODULES */
/***
* jump_label_text_reserved - check if addr range is reserved
@ -248,237 +353,29 @@ out:
*/
int jump_label_text_reserved(void *start, void *end)
{
struct jump_entry *iter;
struct jump_entry *iter_start = __start___jump_table;
struct jump_entry *iter_stop = __start___jump_table;
int conflict = 0;
int ret = __jump_label_text_reserved(__start___jump_table,
__stop___jump_table, start, end);
iter = iter_start;
while (iter < iter_stop) {
if (addr_conflict(iter, start, end)) {
conflict = 1;
goto out;
}
iter++;
}
/* now check modules */
#ifdef CONFIG_MODULES
conflict = module_conflict(start, end);
#endif
out:
return conflict;
}
/*
* Not all archs need this.
*/
void __weak arch_jump_label_text_poke_early(jump_label_t addr)
{
}
static __init int init_jump_label(void)
{
int ret;
struct jump_entry *iter_start = __start___jump_table;
struct jump_entry *iter_stop = __stop___jump_table;
struct jump_entry *iter;
jump_label_lock();
ret = build_jump_label_hashtable(__start___jump_table,
__stop___jump_table);
iter = iter_start;
while (iter < iter_stop) {
arch_jump_label_text_poke_early(iter->code);
iter++;
}
jump_label_unlock();
return ret;
}
early_initcall(init_jump_label);
#ifdef CONFIG_MODULES
static struct jump_label_module_entry *
add_jump_label_module_entry(struct jump_label_entry *entry,
struct jump_entry *iter_begin,
int count, struct module *mod)
{
struct jump_label_module_entry *e;
e = kmalloc(sizeof(struct jump_label_module_entry), GFP_KERNEL);
if (!e)
return ERR_PTR(-ENOMEM);
e->mod = mod;
e->nr_entries = count;
e->table = iter_begin;
hlist_add_head(&e->hlist, &entry->modules);
return e;
}
static int add_jump_label_module(struct module *mod)
{
struct jump_entry *iter, *iter_begin;
struct jump_label_entry *entry;
struct jump_label_module_entry *module_entry;
int count;
/* if the module doesn't have jump label entries, just return */
if (!mod->num_jump_entries)
return 0;
sort_jump_label_entries(mod->jump_entries,
mod->jump_entries + mod->num_jump_entries);
iter = mod->jump_entries;
while (iter < mod->jump_entries + mod->num_jump_entries) {
entry = get_jump_label_entry(iter->key);
iter_begin = iter;
count = 0;
while ((iter < mod->jump_entries + mod->num_jump_entries) &&
(iter->key == iter_begin->key)) {
iter++;
count++;
}
if (!entry) {
entry = add_jump_label_entry(iter_begin->key, 0, NULL);
if (IS_ERR(entry))
return PTR_ERR(entry);
}
module_entry = add_jump_label_module_entry(entry, iter_begin,
count, mod);
if (IS_ERR(module_entry))
return PTR_ERR(module_entry);
}
return 0;
}
static void remove_jump_label_module(struct module *mod)
{
struct hlist_head *head;
struct hlist_node *node, *node_next, *module_node, *module_node_next;
struct jump_label_entry *e;
struct jump_label_module_entry *e_module;
int i;
/* if the module doesn't have jump label entries, just return */
if (!mod->num_jump_entries)
return;
for (i = 0; i < JUMP_LABEL_TABLE_SIZE; i++) {
head = &jump_label_table[i];
hlist_for_each_entry_safe(e, node, node_next, head, hlist) {
hlist_for_each_entry_safe(e_module, module_node,
module_node_next,
&(e->modules), hlist) {
if (e_module->mod == mod) {
hlist_del(&e_module->hlist);
kfree(e_module);
}
}
if (hlist_empty(&e->modules) && (e->nr_entries == 0)) {
hlist_del(&e->hlist);
kfree(e);
}
}
}
}
static void remove_jump_label_module_init(struct module *mod)
{
struct hlist_head *head;
struct hlist_node *node, *node_next, *module_node, *module_node_next;
struct jump_label_entry *e;
struct jump_label_module_entry *e_module;
struct jump_entry *iter;
int i, count;
/* if the module doesn't have jump label entries, just return */
if (!mod->num_jump_entries)
return;
for (i = 0; i < JUMP_LABEL_TABLE_SIZE; i++) {
head = &jump_label_table[i];
hlist_for_each_entry_safe(e, node, node_next, head, hlist) {
hlist_for_each_entry_safe(e_module, module_node,
module_node_next,
&(e->modules), hlist) {
if (e_module->mod != mod)
continue;
count = e_module->nr_entries;
iter = e_module->table;
while (count--) {
if (within_module_init(iter->code, mod))
iter->key = 0;
iter++;
}
}
}
}
}
static int
jump_label_module_notify(struct notifier_block *self, unsigned long val,
void *data)
{
struct module *mod = data;
int ret = 0;
switch (val) {
case MODULE_STATE_COMING:
jump_label_lock();
ret = add_jump_label_module(mod);
if (ret)
remove_jump_label_module(mod);
jump_label_unlock();
break;
case MODULE_STATE_GOING:
jump_label_lock();
remove_jump_label_module(mod);
jump_label_unlock();
break;
case MODULE_STATE_LIVE:
jump_label_lock();
remove_jump_label_module_init(mod);
jump_label_unlock();
break;
}
return ret;
#ifdef CONFIG_MODULES
ret = __jump_label_mod_text_reserved(start, end);
#endif
return ret;
}
/***
* apply_jump_label_nops - patch module jump labels with arch_get_jump_label_nop()
* @mod: module to patch
*
* Allow for run-time selection of the optimal nops. Before the module
* loads patch these with arch_get_jump_label_nop(), which is specified by
* the arch specific jump label code.
*/
void jump_label_apply_nops(struct module *mod)
static void jump_label_update(struct jump_label_key *key, int enable)
{
struct jump_entry *iter;
struct jump_entry *entry = key->entries;
/* if the module doesn't have jump label entries, just return */
if (!mod->num_jump_entries)
return;
/* if there are no users, entry can be NULL */
if (entry)
__jump_label_update(key, entry, enable);
iter = mod->jump_entries;
while (iter < mod->jump_entries + mod->num_jump_entries) {
arch_jump_label_text_poke_early(iter->code);
iter++;
}
#ifdef CONFIG_MODULES
__jump_label_mod_update(key, enable);
#endif
}
struct notifier_block jump_label_module_nb = {
.notifier_call = jump_label_module_notify,
.priority = 0,
};
static __init int init_jump_label_module(void)
{
return register_module_notifier(&jump_label_module_nb);
}
early_initcall(init_jump_label_module);
#endif /* CONFIG_MODULES */
#endif

1241
kernel/trace/ftrace.c
View file

File diff suppressed because it is too large Load diff

15
kernel/trace/trace.c
View file

@ -2014,9 +2014,10 @@ enum print_line_t print_trace_line(struct trace_iterator *iter)
{
enum print_line_t ret;
if (iter->lost_events)
trace_seq_printf(&iter->seq, "CPU:%d [LOST %lu EVENTS]\n",
iter->cpu, iter->lost_events);
if (iter->lost_events &&
!trace_seq_printf(&iter->seq, "CPU:%d [LOST %lu EVENTS]\n",
iter->cpu, iter->lost_events))
return TRACE_TYPE_PARTIAL_LINE;
if (iter->trace && iter->trace->print_line) {
ret = iter->trace->print_line(iter);
@ -3230,6 +3231,14 @@ waitagain:
if (iter->seq.len >= cnt)
break;
/*
* Setting the full flag means we reached the trace_seq buffer
* size and we should leave by partial output condition above.
* One of the trace_seq_* functions is not used properly.
*/
WARN_ONCE(iter->seq.full, "full flag set for trace type %d",
iter->ent->type);
}
trace_access_unlock(iter->cpu_file);
trace_event_read_unlock();

2
kernel/trace/trace.h
View file

@ -419,6 +419,8 @@ extern void trace_find_cmdline(int pid, char comm[]);
extern unsigned long ftrace_update_tot_cnt;
#define DYN_FTRACE_TEST_NAME trace_selftest_dynamic_test_func
extern int DYN_FTRACE_TEST_NAME(void);
#define DYN_FTRACE_TEST_NAME2 trace_selftest_dynamic_test_func2
extern int DYN_FTRACE_TEST_NAME2(void);
#endif
extern int ring_buffer_expanded;

2
kernel/trace/trace_functions.c
View file

@ -149,11 +149,13 @@ function_stack_trace_call(unsigned long ip, unsigned long parent_ip)
static struct ftrace_ops trace_ops __read_mostly =
{
.func = function_trace_call,
.flags = FTRACE_OPS_FL_GLOBAL,
};
static struct ftrace_ops trace_stack_ops __read_mostly =
{
.func = function_stack_trace_call,
.flags = FTRACE_OPS_FL_GLOBAL,
};
/* Our two options */

1
kernel/trace/trace_irqsoff.c
View file

@ -153,6 +153,7 @@ irqsoff_tracer_call(unsigned long ip, unsigned long parent_ip)
static struct ftrace_ops trace_ops __read_mostly =
{
.func = irqsoff_tracer_call,
.flags = FTRACE_OPS_FL_GLOBAL,
};
#endif /* CONFIG_FUNCTION_TRACER */

3
kernel/trace/trace_output.c
View file

@ -830,6 +830,9 @@ EXPORT_SYMBOL_GPL(unregister_ftrace_event);
enum print_line_t trace_nop_print(struct trace_iterator *iter, int flags,
struct trace_event *event)
{
if (!trace_seq_printf(&iter->seq, "type: %d\n", iter->ent->type))
return TRACE_TYPE_PARTIAL_LINE;
return TRACE_TYPE_HANDLED;
}

120
kernel/trace/trace_printk.c
View file

@ -32,7 +32,7 @@ static DEFINE_MUTEX(btrace_mutex);
struct trace_bprintk_fmt {
struct list_head list;
char fmt[0];
const char *fmt;
};
static inline struct trace_bprintk_fmt *lookup_format(const char *fmt)
@ -49,6 +49,7 @@ static
void hold_module_trace_bprintk_format(const char **start, const char **end)
{
const char **iter;
char *fmt;
mutex_lock(&btrace_mutex);
for (iter = start; iter < end; iter++) {
@ -58,15 +59,19 @@ void hold_module_trace_bprintk_format(const char **start, const char **end)
continue;
}
tb_fmt = kmalloc(offsetof(struct trace_bprintk_fmt, fmt)
+ strlen(*iter) + 1, GFP_KERNEL);
if (tb_fmt) {
tb_fmt = kmalloc(sizeof(*tb_fmt), GFP_KERNEL);
if (tb_fmt)
fmt = kmalloc(strlen(*iter) + 1, GFP_KERNEL);
if (tb_fmt && fmt) {
list_add_tail(&tb_fmt->list, &trace_bprintk_fmt_list);
strcpy(tb_fmt->fmt, *iter);
strcpy(fmt, *iter);
tb_fmt->fmt = fmt;
*iter = tb_fmt->fmt;
} else
} else {
kfree(tb_fmt);
*iter = NULL;
}
}
mutex_unlock(&btrace_mutex);
}
@ -84,6 +89,76 @@ static int module_trace_bprintk_format_notify(struct notifier_block *self,
return 0;
}
/*
* The debugfs/tracing/printk_formats file maps the addresses with
* the ASCII formats that are used in the bprintk events in the
* buffer. For userspace tools to be able to decode the events from
* the buffer, they need to be able to map the address with the format.
*
* The addresses of the bprintk formats are in their own section
* __trace_printk_fmt. But for modules we copy them into a link list.
* The code to print the formats and their addresses passes around the
* address of the fmt string. If the fmt address passed into the seq
* functions is within the kernel core __trace_printk_fmt section, then
* it simply uses the next pointer in the list.
*
* When the fmt pointer is outside the kernel core __trace_printk_fmt
* section, then we need to read the link list pointers. The trick is
* we pass the address of the string to the seq function just like
* we do for the kernel core formats. To get back the structure that
* holds the format, we simply use containerof() and then go to the
* next format in the list.
*/
static const char **
find_next_mod_format(int start_index, void *v, const char **fmt, loff_t *pos)
{
struct trace_bprintk_fmt *mod_fmt;
if (list_empty(&trace_bprintk_fmt_list))
return NULL;
/*
* v will point to the address of the fmt record from t_next
* v will be NULL from t_start.
* If this is the first pointer or called from start
* then we need to walk the list.
*/
if (!v || start_index == *pos) {
struct trace_bprintk_fmt *p;
/* search the module list */
list_for_each_entry(p, &trace_bprintk_fmt_list, list) {
if (start_index == *pos)
return &p->fmt;
start_index++;
}
/* pos > index */
return NULL;
}
/*
* v points to the address of the fmt field in the mod list
* structure that holds the module print format.
*/
mod_fmt = container_of(v, typeof(*mod_fmt), fmt);
if (mod_fmt->list.next == &trace_bprintk_fmt_list)
return NULL;
mod_fmt = container_of(mod_fmt->list.next, typeof(*mod_fmt), list);
return &mod_fmt->fmt;
}
static void format_mod_start(void)
{
mutex_lock(&btrace_mutex);
}
static void format_mod_stop(void)
{
mutex_unlock(&btrace_mutex);
}
#else /* !CONFIG_MODULES */
__init static int
module_trace_bprintk_format_notify(struct notifier_block *self,
@ -91,6 +166,13 @@ module_trace_bprintk_format_notify(struct notifier_block *self,
{
return 0;
}
static inline const char **
find_next_mod_format(int start_index, void *v, const char **fmt, loff_t *pos)
{
return NULL;
}
static inline void format_mod_start(void) { }
static inline void format_mod_stop(void) { }
#endif /* CONFIG_MODULES */
@ -153,20 +235,33 @@ int __ftrace_vprintk(unsigned long ip, const char *fmt, va_list ap)
}
EXPORT_SYMBOL_GPL(__ftrace_vprintk);
static const char **find_next(void *v, loff_t *pos)
{
const char **fmt = v;
int start_index;
if (!fmt)
fmt = __start___trace_bprintk_fmt + *pos;
start_index = __stop___trace_bprintk_fmt - __start___trace_bprintk_fmt;
if (*pos < start_index)
return fmt;
return find_next_mod_format(start_index, v, fmt, pos);
}
static void *
t_start(struct seq_file *m, loff_t *pos)
{
const char **fmt = __start___trace_bprintk_fmt + *pos;
if ((unsigned long)fmt >= (unsigned long)__stop___trace_bprintk_fmt)
return NULL;
return fmt;
format_mod_start();
return find_next(NULL, pos);
}
static void *t_next(struct seq_file *m, void * v, loff_t *pos)
{
(*pos)++;
return t_start(m, pos);
return find_next(v, pos);
}
static int t_show(struct seq_file *m, void *v)
@ -205,6 +300,7 @@ static int t_show(struct seq_file *m, void *v)
static void t_stop(struct seq_file *m, void *p)
{
format_mod_stop();
}
static const struct seq_operations show_format_seq_ops = {

1
kernel/trace/trace_sched_wakeup.c
View file

@ -129,6 +129,7 @@ wakeup_tracer_call(unsigned long ip, unsigned long parent_ip)
static struct ftrace_ops trace_ops __read_mostly =
{
.func = wakeup_tracer_call,
.flags = FTRACE_OPS_FL_GLOBAL,
};
#endif /* CONFIG_FUNCTION_TRACER */

214
kernel/trace/trace_selftest.c
View file

@ -101,6 +101,206 @@ static inline void warn_failed_init_tracer(struct tracer *trace, int init_ret)
#ifdef CONFIG_DYNAMIC_FTRACE
static int trace_selftest_test_probe1_cnt;
static void trace_selftest_test_probe1_func(unsigned long ip,
unsigned long pip)
{
trace_selftest_test_probe1_cnt++;
}
static int trace_selftest_test_probe2_cnt;
static void trace_selftest_test_probe2_func(unsigned long ip,
unsigned long pip)
{
trace_selftest_test_probe2_cnt++;
}
static int trace_selftest_test_probe3_cnt;
static void trace_selftest_test_probe3_func(unsigned long ip,
unsigned long pip)
{
trace_selftest_test_probe3_cnt++;
}
static int trace_selftest_test_global_cnt;
static void trace_selftest_test_global_func(unsigned long ip,
unsigned long pip)
{
trace_selftest_test_global_cnt++;
}
static int trace_selftest_test_dyn_cnt;
static void trace_selftest_test_dyn_func(unsigned long ip,
unsigned long pip)
{
trace_selftest_test_dyn_cnt++;
}
static struct ftrace_ops test_probe1 = {
.func = trace_selftest_test_probe1_func,
};
static struct ftrace_ops test_probe2 = {
.func = trace_selftest_test_probe2_func,
};
static struct ftrace_ops test_probe3 = {
.func = trace_selftest_test_probe3_func,
};
static struct ftrace_ops test_global = {
.func = trace_selftest_test_global_func,
.flags = FTRACE_OPS_FL_GLOBAL,
};
static void print_counts(void)
{
printk("(%d %d %d %d %d) ",
trace_selftest_test_probe1_cnt,
trace_selftest_test_probe2_cnt,
trace_selftest_test_probe3_cnt,
trace_selftest_test_global_cnt,
trace_selftest_test_dyn_cnt);
}
static void reset_counts(void)
{
trace_selftest_test_probe1_cnt = 0;
trace_selftest_test_probe2_cnt = 0;
trace_selftest_test_probe3_cnt = 0;
trace_selftest_test_global_cnt = 0;
trace_selftest_test_dyn_cnt = 0;
}
static int trace_selftest_ops(int cnt)
{
int save_ftrace_enabled = ftrace_enabled;
struct ftrace_ops *dyn_ops;
char *func1_name;
char *func2_name;
int len1;
int len2;
int ret = -1;
printk(KERN_CONT "PASSED\n");
pr_info("Testing dynamic ftrace ops #%d: ", cnt);
ftrace_enabled = 1;
reset_counts();
/* Handle PPC64 '.' name */
func1_name = "*" __stringify(DYN_FTRACE_TEST_NAME);
func2_name = "*" __stringify(DYN_FTRACE_TEST_NAME2);
len1 = strlen(func1_name);
len2 = strlen(func2_name);
/*
* Probe 1 will trace function 1.
* Probe 2 will trace function 2.
* Probe 3 will trace functions 1 and 2.
*/
ftrace_set_filter(&test_probe1, func1_name, len1, 1);
ftrace_set_filter(&test_probe2, func2_name, len2, 1);
ftrace_set_filter(&test_probe3, func1_name, len1, 1);
ftrace_set_filter(&test_probe3, func2_name, len2, 0);
register_ftrace_function(&test_probe1);
register_ftrace_function(&test_probe2);
register_ftrace_function(&test_probe3);
register_ftrace_function(&test_global);
DYN_FTRACE_TEST_NAME();
print_counts();
if (trace_selftest_test_probe1_cnt != 1)
goto out;
if (trace_selftest_test_probe2_cnt != 0)
goto out;
if (trace_selftest_test_probe3_cnt != 1)
goto out;
if (trace_selftest_test_global_cnt == 0)
goto out;
DYN_FTRACE_TEST_NAME2();
print_counts();
if (trace_selftest_test_probe1_cnt != 1)
goto out;
if (trace_selftest_test_probe2_cnt != 1)
goto out;
if (trace_selftest_test_probe3_cnt != 2)
goto out;
/* Add a dynamic probe */
dyn_ops = kzalloc(sizeof(*dyn_ops), GFP_KERNEL);
if (!dyn_ops) {
printk("MEMORY ERROR ");
goto out;
}
dyn_ops->func = trace_selftest_test_dyn_func;
register_ftrace_function(dyn_ops);
trace_selftest_test_global_cnt = 0;
DYN_FTRACE_TEST_NAME();
print_counts();
if (trace_selftest_test_probe1_cnt != 2)
goto out_free;
if (trace_selftest_test_probe2_cnt != 1)
goto out_free;
if (trace_selftest_test_probe3_cnt != 3)
goto out_free;
if (trace_selftest_test_global_cnt == 0)
goto out;
if (trace_selftest_test_dyn_cnt == 0)
goto out_free;
DYN_FTRACE_TEST_NAME2();
print_counts();
if (trace_selftest_test_probe1_cnt != 2)
goto out_free;
if (trace_selftest_test_probe2_cnt != 2)
goto out_free;
if (trace_selftest_test_probe3_cnt != 4)
goto out_free;
ret = 0;
out_free:
unregister_ftrace_function(dyn_ops);
kfree(dyn_ops);
out:
/* Purposely unregister in the same order */
unregister_ftrace_function(&test_probe1);
unregister_ftrace_function(&test_probe2);
unregister_ftrace_function(&test_probe3);
unregister_ftrace_function(&test_global);
/* Make sure everything is off */
reset_counts();
DYN_FTRACE_TEST_NAME();
DYN_FTRACE_TEST_NAME();
if (trace_selftest_test_probe1_cnt ||
trace_selftest_test_probe2_cnt ||
trace_selftest_test_probe3_cnt ||
trace_selftest_test_global_cnt ||
trace_selftest_test_dyn_cnt)
ret = -1;
ftrace_enabled = save_ftrace_enabled;
return ret;
}
/* Test dynamic code modification and ftrace filters */
int trace_selftest_startup_dynamic_tracing(struct tracer *trace,
struct trace_array *tr,
@ -131,7 +331,7 @@ int trace_selftest_startup_dynamic_tracing(struct tracer *trace,
func_name = "*" __stringify(DYN_FTRACE_TEST_NAME);
/* filter only on our function */
ftrace_set_filter(func_name, strlen(func_name), 1);
ftrace_set_global_filter(func_name, strlen(func_name), 1);
/* enable tracing */
ret = tracer_init(trace, tr);
@ -166,22 +366,30 @@ int trace_selftest_startup_dynamic_tracing(struct tracer *trace,
/* check the trace buffer */
ret = trace_test_buffer(tr, &count);
trace->reset(tr);
tracing_start();
/* we should only have one item */
if (!ret && count != 1) {
trace->reset(tr);
printk(KERN_CONT ".. filter failed count=%ld ..", count);
ret = -1;
goto out;
}
/* Test the ops with global tracing running */
ret = trace_selftest_ops(1);
trace->reset(tr);
out:
ftrace_enabled = save_ftrace_enabled;
tracer_enabled = save_tracer_enabled;
/* Enable tracing on all functions again */
ftrace_set_filter(NULL, 0, 1);
ftrace_set_global_filter(NULL, 0, 1);
/* Test the ops with global tracing off */
if (!ret)
ret = trace_selftest_ops(2);
return ret;
}

6
kernel/trace/trace_selftest_dynamic.c
View file

@ -5,3 +5,9 @@ int DYN_FTRACE_TEST_NAME(void)
/* used to call mcount */
return 0;
}
int DYN_FTRACE_TEST_NAME2(void)
{
/* used to call mcount */
return 0;
}

1
kernel/trace/trace_stack.c
View file

@ -133,6 +133,7 @@ stack_trace_call(unsigned long ip, unsigned long parent_ip)
static struct ftrace_ops trace_ops __read_mostly =
{
.func = stack_trace_call,
.flags = FTRACE_OPS_FL_GLOBAL,
};
static ssize_t

23
kernel/tracepoint.c
View file

@ -251,9 +251,9 @@ static void set_tracepoint(struct tracepoint_entry **entry,
{
WARN_ON(strcmp((*entry)->name, elem->name) != 0);
if (elem->regfunc && !elem->state && active)
if (elem->regfunc && !jump_label_enabled(&elem->key) && active)
elem->regfunc();
else if (elem->unregfunc && elem->state && !active)
else if (elem->unregfunc && jump_label_enabled(&elem->key) && !active)
elem->unregfunc();
/*
@ -264,13 +264,10 @@ static void set_tracepoint(struct tracepoint_entry **entry,
* is used.
*/
rcu_assign_pointer(elem->funcs, (*entry)->funcs);
if (!elem->state && active) {
jump_label_enable(&elem->state);
elem->state = active;
} else if (elem->state && !active) {
jump_label_disable(&elem->state);
elem->state = active;
}
if (active && !jump_label_enabled(&elem->key))
jump_label_inc(&elem->key);
else if (!active && jump_label_enabled(&elem->key))
jump_label_dec(&elem->key);
}
/*
@ -281,13 +278,11 @@ static void set_tracepoint(struct tracepoint_entry **entry,
*/
static void disable_tracepoint(struct tracepoint *elem)
{
if (elem->unregfunc && elem->state)
if (elem->unregfunc && jump_label_enabled(&elem->key))
elem->unregfunc();
if (elem->state) {
jump_label_disable(&elem->state);
elem->state = 0;
}
if (jump_label_enabled(&elem->key))
jump_label_dec(&elem->key);
rcu_assign_pointer(elem->funcs, NULL);
}

12
scripts/Makefile.build
View file

@ -244,14 +244,19 @@ endif
ifdef CONFIG_FTRACE_MCOUNT_RECORD
ifdef BUILD_C_RECORDMCOUNT
ifeq ("$(origin RECORDMCOUNT_WARN)", "command line")
RECORDMCOUNT_FLAGS = -w
endif
# Due to recursion, we must skip empty.o.
# The empty.o file is created in the make process in order to determine
# the target endianness and word size. It is made before all other C
# files, including recordmcount.
sub_cmd_record_mcount = \
if [ $(@) != "scripts/mod/empty.o" ]; then \
$(objtree)/scripts/recordmcount "$(@)"; \
$(objtree)/scripts/recordmcount $(RECORDMCOUNT_FLAGS) "$(@)"; \
fi;
recordmcount_source := $(srctree)/scripts/recordmcount.c \
$(srctree)/scripts/recordmcount.h
else
sub_cmd_record_mcount = set -e ; perl $(srctree)/scripts/recordmcount.pl "$(ARCH)" \
"$(if $(CONFIG_CPU_BIG_ENDIAN),big,little)" \
@ -259,6 +264,7 @@ sub_cmd_record_mcount = set -e ; perl $(srctree)/scripts/recordmcount.pl "$(ARCH
"$(OBJDUMP)" "$(OBJCOPY)" "$(CC) $(KBUILD_CFLAGS)" \
"$(LD)" "$(NM)" "$(RM)" "$(MV)" \
"$(if $(part-of-module),1,0)" "$(@)";
recordmcount_source := $(srctree)/scripts/recordmcount.pl
endif
cmd_record_mcount = \
if [ "$(findstring -pg,$(_c_flags))" = "-pg" ]; then \
@ -279,13 +285,13 @@ define rule_cc_o_c
endef
# Built-in and composite module parts
$(obj)/%.o: $(src)/%.c FORCE
$(obj)/%.o: $(src)/%.c $(recordmcount_source) FORCE
$(call cmd,force_checksrc)
$(call if_changed_rule,cc_o_c)
# Single-part modules are special since we need to mark them in $(MODVERDIR)
$(single-used-m): $(obj)/%.o: $(src)/%.c FORCE
$(single-used-m): $(obj)/%.o: $(src)/%.c $(recordmcount_source) FORCE
$(call cmd,force_checksrc)
$(call if_changed_rule,cc_o_c)
@{ echo $(@:.o=.ko); echo $@; } > $(MODVERDIR)/$(@F:.o=.mod)

168
scripts/recordmcount.c
View file

@ -24,6 +24,7 @@
#include <sys/types.h>
#include <sys/mman.h>
#include <sys/stat.h>
#include <getopt.h>
#include <elf.h>
#include <fcntl.h>
#include <setjmp.h>
@ -39,6 +40,7 @@ static char gpfx; /* prefix for global symbol name (sometimes '_') */
static struct stat sb; /* Remember .st_size, etc. */
static jmp_buf jmpenv; /* setjmp/longjmp per-file error escape */
static const char *altmcount; /* alternate mcount symbol name */
static int warn_on_notrace_sect; /* warn when section has mcount not being recorded */
/* setjmp() return values */
enum {
@ -78,7 +80,7 @@ static off_t
ulseek(int const fd, off_t const offset, int const whence)
{
off_t const w = lseek(fd, offset, whence);
if ((off_t)-1 == w) {
if (w == (off_t)-1) {
perror("lseek");
fail_file();
}
@ -111,13 +113,41 @@ static void *
umalloc(size_t size)
{
void *const addr = malloc(size);
if (0 == addr) {
if (addr == 0) {
fprintf(stderr, "malloc failed: %zu bytes\n", size);
fail_file();
}
return addr;
}
static unsigned char ideal_nop5_x86_64[5] = { 0x0f, 0x1f, 0x44, 0x00, 0x00 };
static unsigned char ideal_nop5_x86_32[5] = { 0x3e, 0x8d, 0x74, 0x26, 0x00 };
static unsigned char *ideal_nop;
static char rel_type_nop;
static int (*make_nop)(void *map, size_t const offset);
static int make_nop_x86(void *map, size_t const offset)
{
uint32_t *ptr;
unsigned char *op;
/* Confirm we have 0xe8 0x0 0x0 0x0 0x0 */
ptr = map + offset;
if (*ptr != 0)
return -1;
op = map + offset - 1;
if (*op != 0xe8)
return -1;
/* convert to nop */
ulseek(fd_map, offset - 1, SEEK_SET);
uwrite(fd_map, ideal_nop, 5);
return 0;
}
/*
* Get the whole file as a programming convenience in order to avoid
* malloc+lseek+read+free of many pieces. If successful, then mmap
@ -136,7 +166,7 @@ static void *mmap_file(char const *fname)
void *addr;
fd_map = open(fname, O_RDWR);
if (0 > fd_map || 0 > fstat(fd_map, &sb)) {
if (fd_map < 0 || fstat(fd_map, &sb) < 0) {
perror(fname);
fail_file();
}
@ -147,7 +177,7 @@ static void *mmap_file(char const *fname)
addr = mmap(0, sb.st_size, PROT_READ|PROT_WRITE, MAP_PRIVATE,
fd_map, 0);
mmap_failed = 0;
if (MAP_FAILED == addr) {
if (addr == MAP_FAILED) {
mmap_failed = 1;
addr = umalloc(sb.st_size);
uread(fd_map, addr, sb.st_size);
@ -206,12 +236,13 @@ static uint32_t (*w2)(uint16_t);
static int
is_mcounted_section_name(char const *const txtname)
{
return 0 == strcmp(".text", txtname) ||
0 == strcmp(".ref.text", txtname) ||
0 == strcmp(".sched.text", txtname) ||
0 == strcmp(".spinlock.text", txtname) ||
0 == strcmp(".irqentry.text", txtname) ||
0 == strcmp(".text.unlikely", txtname);
return strcmp(".text", txtname) == 0 ||
strcmp(".ref.text", txtname) == 0 ||
strcmp(".sched.text", txtname) == 0 ||
strcmp(".spinlock.text", txtname) == 0 ||
strcmp(".irqentry.text", txtname) == 0 ||
strcmp(".kprobes.text", txtname) == 0 ||
strcmp(".text.unlikely", txtname) == 0;
}
/* 32 bit and 64 bit are very similar */
@ -264,43 +295,48 @@ do_file(char const *const fname)
w8 = w8nat;
switch (ehdr->e_ident[EI_DATA]) {
static unsigned int const endian = 1;
default: {
default:
fprintf(stderr, "unrecognized ELF data encoding %d: %s\n",
ehdr->e_ident[EI_DATA], fname);
fail_file();
} break;
case ELFDATA2LSB: {
if (1 != *(unsigned char const *)&endian) {
break;
case ELFDATA2LSB:
if (*(unsigned char const *)&endian != 1) {
/* main() is big endian, file.o is little endian. */
w = w4rev;
w2 = w2rev;
w8 = w8rev;
}
} break;
case ELFDATA2MSB: {
if (0 != *(unsigned char const *)&endian) {
break;
case ELFDATA2MSB:
if (*(unsigned char const *)&endian != 0) {
/* main() is little endian, file.o is big endian. */
w = w4rev;
w2 = w2rev;
w8 = w8rev;
}
} break;
break;
} /* end switch */
if (0 != memcmp(ELFMAG, ehdr->e_ident, SELFMAG)
|| ET_REL != w2(ehdr->e_type)
|| EV_CURRENT != ehdr->e_ident[EI_VERSION]) {
if (memcmp(ELFMAG, ehdr->e_ident, SELFMAG) != 0
|| w2(ehdr->e_type) != ET_REL
|| ehdr->e_ident[EI_VERSION] != EV_CURRENT) {
fprintf(stderr, "unrecognized ET_REL file %s\n", fname);
fail_file();
}
gpfx = 0;
switch (w2(ehdr->e_machine)) {
default: {
default:
fprintf(stderr, "unrecognized e_machine %d %s\n",
w2(ehdr->e_machine), fname);
fail_file();
} break;
case EM_386: reltype = R_386_32; break;
break;
case EM_386:
reltype = R_386_32;
make_nop = make_nop_x86;
ideal_nop = ideal_nop5_x86_32;
mcount_adjust_32 = -1;
break;
case EM_ARM: reltype = R_ARM_ABS32;
altmcount = "__gnu_mcount_nc";
break;
@ -311,67 +347,91 @@ do_file(char const *const fname)
case EM_S390: /* reltype: e_class */ gpfx = '_'; break;
case EM_SH: reltype = R_SH_DIR32; break;
case EM_SPARCV9: reltype = R_SPARC_64; gpfx = '_'; break;
case EM_X86_64: reltype = R_X86_64_64; break;
case EM_X86_64:
make_nop = make_nop_x86;
ideal_nop = ideal_nop5_x86_64;
reltype = R_X86_64_64;
mcount_adjust_64 = -1;
break;
} /* end switch */
switch (ehdr->e_ident[EI_CLASS]) {
default: {
default:
fprintf(stderr, "unrecognized ELF class %d %s\n",
ehdr->e_ident[EI_CLASS], fname);
fail_file();
} break;
case ELFCLASS32: {
if (sizeof(Elf32_Ehdr) != w2(ehdr->e_ehsize)
|| sizeof(Elf32_Shdr) != w2(ehdr->e_shentsize)) {
break;
case ELFCLASS32:
if (w2(ehdr->e_ehsize) != sizeof(Elf32_Ehdr)
|| w2(ehdr->e_shentsize) != sizeof(Elf32_Shdr)) {
fprintf(stderr,
"unrecognized ET_REL file: %s\n", fname);
fail_file();
}
if (EM_S390 == w2(ehdr->e_machine))
if (w2(ehdr->e_machine) == EM_S390) {
reltype = R_390_32;
if (EM_MIPS == w2(ehdr->e_machine)) {
mcount_adjust_32 = -4;
}
if (w2(ehdr->e_machine) == EM_MIPS) {
reltype = R_MIPS_32;
is_fake_mcount32 = MIPS32_is_fake_mcount;
}
do32(ehdr, fname, reltype);
} break;
break;
case ELFCLASS64: {
Elf64_Ehdr *const ghdr = (Elf64_Ehdr *)ehdr;
if (sizeof(Elf64_Ehdr) != w2(ghdr->e_ehsize)
|| sizeof(Elf64_Shdr) != w2(ghdr->e_shentsize)) {
if (w2(ghdr->e_ehsize) != sizeof(Elf64_Ehdr)
|| w2(ghdr->e_shentsize) != sizeof(Elf64_Shdr)) {
fprintf(stderr,
"unrecognized ET_REL file: %s\n", fname);
fail_file();
}
if (EM_S390 == w2(ghdr->e_machine))
if (w2(ghdr->e_machine) == EM_S390) {
reltype = R_390_64;
if (EM_MIPS == w2(ghdr->e_machine)) {
mcount_adjust_64 = -8;
}
if (w2(ghdr->e_machine) == EM_MIPS) {
reltype = R_MIPS_64;
Elf64_r_sym = MIPS64_r_sym;
Elf64_r_info = MIPS64_r_info;
is_fake_mcount64 = MIPS64_is_fake_mcount;
}
do64(ghdr, fname, reltype);
} break;
break;
}
} /* end switch */
cleanup();
}
int
main(int argc, char const *argv[])
main(int argc, char *argv[])
{
const char ftrace[] = "/ftrace.o";
int ftrace_size = sizeof(ftrace) - 1;
int n_error = 0; /* gcc-4.3.0 false positive complaint */
int c;
int i;
if (argc <= 1) {
fprintf(stderr, "usage: recordmcount file.o...\n");
while ((c = getopt(argc, argv, "w")) >= 0) {
switch (c) {
case 'w':
warn_on_notrace_sect = 1;
break;
default:
fprintf(stderr, "usage: recordmcount [-w] file.o...\n");
return 0;
}
}
if ((argc - optind) < 1) {
fprintf(stderr, "usage: recordmcount [-w] file.o...\n");
return 0;
}
/* Process each file in turn, allowing deep failure. */
for (--argc, ++argv; 0 < argc; --argc, ++argv) {
for (i = optind; i < argc; i++) {
char *file = argv[i];
int const sjval = setjmp(jmpenv);
int len;
@ -380,29 +440,29 @@ main(int argc, char const *argv[])
* function but does not call it. Since ftrace.o should
* not be traced anyway, we just skip it.
*/
len = strlen(argv[0]);
len = strlen(file);
if (len >= ftrace_size &&
strcmp(argv[0] + (len - ftrace_size), ftrace) == 0)
strcmp(file + (len - ftrace_size), ftrace) == 0)
continue;
switch (sjval) {
default: {
fprintf(stderr, "internal error: %s\n", argv[0]);
default:
fprintf(stderr, "internal error: %s\n", file);
exit(1);
} break;
case SJ_SETJMP: { /* normal sequence */
break;
case SJ_SETJMP: /* normal sequence */
/* Avoid problems if early cleanup() */
fd_map = -1;
ehdr_curr = NULL;
mmap_failed = 1;
do_file(argv[0]);
} break;
case SJ_FAIL: { /* error in do_file or below */
do_file(file);
break;
case SJ_FAIL: /* error in do_file or below */
++n_error;
} break;
case SJ_SUCCEED: { /* premature success */
break;
case SJ_SUCCEED: /* premature success */
/* do nothing */
} break;
break;
} /* end switch */
}
return !!n_error;

174
scripts/recordmcount.h
View file

@ -22,11 +22,15 @@
#undef is_fake_mcount
#undef fn_is_fake_mcount
#undef MIPS_is_fake_mcount
#undef mcount_adjust
#undef sift_rel_mcount
#undef nop_mcount
#undef find_secsym_ndx
#undef __has_rel_mcount
#undef has_rel_mcount
#undef tot_relsize
#undef get_mcountsym
#undef get_sym_str_and_relp
#undef do_func
#undef Elf_Addr
#undef Elf_Ehdr
@ -49,14 +53,18 @@
#ifdef RECORD_MCOUNT_64
# define append_func append64
# define sift_rel_mcount sift64_rel_mcount
# define nop_mcount nop_mcount_64
# define find_secsym_ndx find64_secsym_ndx
# define __has_rel_mcount __has64_rel_mcount
# define has_rel_mcount has64_rel_mcount
# define tot_relsize tot64_relsize
# define get_sym_str_and_relp get_sym_str_and_relp_64
# define do_func do64
# define get_mcountsym get_mcountsym_64
# define is_fake_mcount is_fake_mcount64
# define fn_is_fake_mcount fn_is_fake_mcount64
# define MIPS_is_fake_mcount MIPS64_is_fake_mcount
# define mcount_adjust mcount_adjust_64
# define Elf_Addr Elf64_Addr
# define Elf_Ehdr Elf64_Ehdr
# define Elf_Shdr Elf64_Shdr
@ -77,14 +85,18 @@
#else
# define append_func append32
# define sift_rel_mcount sift32_rel_mcount
# define nop_mcount nop_mcount_32
# define find_secsym_ndx find32_secsym_ndx
# define __has_rel_mcount __has32_rel_mcount
# define has_rel_mcount has32_rel_mcount
# define tot_relsize tot32_relsize
# define get_sym_str_and_relp get_sym_str_and_relp_32
# define do_func do32
# define get_mcountsym get_mcountsym_32
# define is_fake_mcount is_fake_mcount32
# define fn_is_fake_mcount fn_is_fake_mcount32
# define MIPS_is_fake_mcount MIPS32_is_fake_mcount
# define mcount_adjust mcount_adjust_32
# define Elf_Addr Elf32_Addr
# define Elf_Ehdr Elf32_Ehdr
# define Elf_Shdr Elf32_Shdr
@ -123,6 +135,8 @@ static void fn_ELF_R_INFO(Elf_Rel *const rp, unsigned sym, unsigned type)
}
static void (*Elf_r_info)(Elf_Rel *const rp, unsigned sym, unsigned type) = fn_ELF_R_INFO;
static int mcount_adjust = 0;
/*
* MIPS mcount long call has 2 _mcount symbols, only the position of the 1st
* _mcount symbol is needed for dynamic function tracer, with it, to disable
@ -234,6 +248,49 @@ static void append_func(Elf_Ehdr *const ehdr,
uwrite(fd_map, ehdr, sizeof(*ehdr));
}
static unsigned get_mcountsym(Elf_Sym const *const sym0,
Elf_Rel const *relp,
char const *const str0)
{
unsigned mcountsym = 0;
Elf_Sym const *const symp =
&sym0[Elf_r_sym(relp)];
char const *symname = &str0[w(symp->st_name)];
char const *mcount = gpfx == '_' ? "_mcount" : "mcount";
if (symname[0] == '.')
++symname; /* ppc64 hack */
if (strcmp(mcount, symname) == 0 ||
(altmcount && strcmp(altmcount, symname) == 0))
mcountsym = Elf_r_sym(relp);
return mcountsym;
}
static void get_sym_str_and_relp(Elf_Shdr const *const relhdr,
Elf_Ehdr const *const ehdr,
Elf_Sym const **sym0,
char const **str0,
Elf_Rel const **relp)
{
Elf_Shdr *const shdr0 = (Elf_Shdr *)(_w(ehdr->e_shoff)
+ (void *)ehdr);
unsigned const symsec_sh_link = w(relhdr->sh_link);
Elf_Shdr const *const symsec = &shdr0[symsec_sh_link];
Elf_Shdr const *const strsec = &shdr0[w(symsec->sh_link)];
Elf_Rel const *const rel0 = (Elf_Rel const *)(_w(relhdr->sh_offset)
+ (void *)ehdr);
*sym0 = (Elf_Sym const *)(_w(symsec->sh_offset)
+ (void *)ehdr);
*str0 = (char const *)(_w(strsec->sh_offset)
+ (void *)ehdr);
*relp = rel0;
}
/*
* Look at the relocations in order to find the calls to mcount.
* Accumulate the section offsets that are found, and their relocation info,
@ -250,47 +307,27 @@ static uint_t *sift_rel_mcount(uint_t *mlocp,
{
uint_t *const mloc0 = mlocp;
Elf_Rel *mrelp = *mrelpp;
Elf_Shdr *const shdr0 = (Elf_Shdr *)(_w(ehdr->e_shoff)
+ (void *)ehdr);
unsigned const symsec_sh_link = w(relhdr->sh_link);
Elf_Shdr const *const symsec = &shdr0[symsec_sh_link];
Elf_Sym const *const sym0 = (Elf_Sym const *)(_w(symsec->sh_offset)
+ (void *)ehdr);
Elf_Shdr const *const strsec = &shdr0[w(symsec->sh_link)];
char const *const str0 = (char const *)(_w(strsec->sh_offset)
+ (void *)ehdr);
Elf_Rel const *const rel0 = (Elf_Rel const *)(_w(relhdr->sh_offset)
+ (void *)ehdr);
Elf_Sym const *sym0;
char const *str0;
Elf_Rel const *relp;
unsigned rel_entsize = _w(relhdr->sh_entsize);
unsigned const nrel = _w(relhdr->sh_size) / rel_entsize;
Elf_Rel const *relp = rel0;
unsigned mcountsym = 0;
unsigned t;
for (t = nrel; t; --t) {
if (!mcountsym) {
Elf_Sym const *const symp =
&sym0[Elf_r_sym(relp)];
char const *symname = &str0[w(symp->st_name)];
char const *mcount = '_' == gpfx ? "_mcount" : "mcount";
get_sym_str_and_relp(relhdr, ehdr, &sym0, &str0, &relp);
if ('.' == symname[0])
++symname; /* ppc64 hack */
if (0 == strcmp(mcount, symname) ||
(altmcount && 0 == strcmp(altmcount, symname)))
mcountsym = Elf_r_sym(relp);
}
for (t = nrel; t; --t) {
if (!mcountsym)
mcountsym = get_mcountsym(sym0, relp, str0);
if (mcountsym == Elf_r_sym(relp) && !is_fake_mcount(relp)) {
uint_t const addend = _w(_w(relp->r_offset) - recval);
uint_t const addend =
_w(_w(relp->r_offset) - recval + mcount_adjust);
mrelp->r_offset = _w(offbase
+ ((void *)mlocp - (void *)mloc0));
Elf_r_info(mrelp, recsym, reltype);
if (sizeof(Elf_Rela) == rel_entsize) {
if (rel_entsize == sizeof(Elf_Rela)) {
((Elf_Rela *)mrelp)->r_addend = addend;
*mlocp++ = 0;
} else
@ -304,6 +341,63 @@ static uint_t *sift_rel_mcount(uint_t *mlocp,
return mlocp;
}
/*
* Read the relocation table again, but this time its called on sections
* that are not going to be traced. The mcount calls here will be converted
* into nops.
*/
static void nop_mcount(Elf_Shdr const *const relhdr,
Elf_Ehdr const *const ehdr,
const char *const txtname)
{
Elf_Shdr *const shdr0 = (Elf_Shdr *)(_w(ehdr->e_shoff)
+ (void *)ehdr);
Elf_Sym const *sym0;
char const *str0;
Elf_Rel const *relp;
Elf_Shdr const *const shdr = &shdr0[w(relhdr->sh_info)];
unsigned rel_entsize = _w(relhdr->sh_entsize);
unsigned const nrel = _w(relhdr->sh_size) / rel_entsize;
unsigned mcountsym = 0;
unsigned t;
int once = 0;
get_sym_str_and_relp(relhdr, ehdr, &sym0, &str0, &relp);
for (t = nrel; t; --t) {
int ret = -1;
if (!mcountsym)
mcountsym = get_mcountsym(sym0, relp, str0);
if (mcountsym == Elf_r_sym(relp) && !is_fake_mcount(relp)) {
if (make_nop)
ret = make_nop((void *)ehdr, shdr->sh_offset + relp->r_offset);
if (warn_on_notrace_sect && !once) {
printf("Section %s has mcount callers being ignored\n",
txtname);
once = 1;
/* just warn? */
if (!make_nop)
return;
}
}
/*
* If we successfully removed the mcount, mark the relocation
* as a nop (don't do anything with it).
*/
if (!ret) {
Elf_Rel rel;
rel = *(Elf_Rel *)relp;
Elf_r_info(&rel, Elf_r_sym(relp), rel_type_nop);
ulseek(fd_map, (void *)relp - (void *)ehdr, SEEK_SET);
uwrite(fd_map, &rel, sizeof(rel));
}
relp = (Elf_Rel const *)(rel_entsize + (void *)relp);
}
}
/*
* Find a symbol in the given section, to be used as the base for relocating
@ -354,13 +448,13 @@ __has_rel_mcount(Elf_Shdr const *const relhdr, /* is SHT_REL or SHT_RELA */
Elf_Shdr const *const txthdr = &shdr0[w(relhdr->sh_info)];
char const *const txtname = &shstrtab[w(txthdr->sh_name)];
if (0 == strcmp("__mcount_loc", txtname)) {
if (strcmp("__mcount_loc", txtname) == 0) {
fprintf(stderr, "warning: __mcount_loc already exists: %s\n",
fname);
succeed_file();
}
if (SHT_PROGBITS != w(txthdr->sh_type) ||
!is_mcounted_section_name(txtname))
if (w(txthdr->sh_type) != SHT_PROGBITS ||
!(w(txthdr->sh_flags) & SHF_EXECINSTR))
return NULL;
return txtname;
}
@ -370,7 +464,7 @@ static char const *has_rel_mcount(Elf_Shdr const *const relhdr,
char const *const shstrtab,
char const *const fname)
{
if (SHT_REL != w(relhdr->sh_type) && SHT_RELA != w(relhdr->sh_type))
if (w(relhdr->sh_type) != SHT_REL && w(relhdr->sh_type) != SHT_RELA)
return NULL;
return __has_rel_mcount(relhdr, shdr0, shstrtab, fname);
}
@ -383,9 +477,11 @@ static unsigned tot_relsize(Elf_Shdr const *const shdr0,
{
unsigned totrelsz = 0;
Elf_Shdr const *shdrp = shdr0;
char const *txtname;
for (; nhdr; --nhdr, ++shdrp) {
if (has_rel_mcount(shdrp, shdr0, shstrtab, fname))
txtname = has_rel_mcount(shdrp, shdr0, shstrtab, fname);
if (txtname && is_mcounted_section_name(txtname))
totrelsz += _w(shdrp->sh_size);
}
return totrelsz;
@ -421,7 +517,7 @@ do_func(Elf_Ehdr *const ehdr, char const *const fname, unsigned const reltype)
for (relhdr = shdr0, k = nhdr; k; --k, ++relhdr) {
char const *const txtname = has_rel_mcount(relhdr, shdr0,
shstrtab, fname);
if (txtname) {
if (txtname && is_mcounted_section_name(txtname)) {
uint_t recval = 0;
unsigned const recsym = find_secsym_ndx(
w(relhdr->sh_info), txtname, &recval,
@ -432,6 +528,12 @@ do_func(Elf_Ehdr *const ehdr, char const *const fname, unsigned const reltype)
mlocp = sift_rel_mcount(mlocp,
(void *)mlocp - (void *)mloc0, &mrelp,
relhdr, ehdr, recsym, recval, reltype);
} else if (txtname && (warn_on_notrace_sect || make_nop)) {
/*
* This section is ignored by ftrace, but still
* has mcount calls. Convert them to nops now.
*/
nop_mcount(relhdr, ehdr, txtname);
}
}
if (mloc0 != mlocp) {

5
scripts/recordmcount.pl
View file

@ -134,6 +134,7 @@ my %text_sections = (
".sched.text" => 1,
".spinlock.text" => 1,
".irqentry.text" => 1,
".kprobes.text" => 1,
".text.unlikely" => 1,
);
@ -222,6 +223,7 @@ if ($arch eq "x86_64") {
$mcount_regex = "^\\s*([0-9a-fA-F]+):.*\\smcount([+-]0x[0-9a-zA-Z]+)?\$";
$type = ".quad";
$alignment = 8;
$mcount_adjust = -1;
# force flags for this arch
$ld .= " -m elf_x86_64";
@ -231,6 +233,7 @@ if ($arch eq "x86_64") {
} elsif ($arch eq "i386") {
$alignment = 4;
$mcount_adjust = -1;
# force flags for this arch
$ld .= " -m elf_i386";
@ -240,12 +243,14 @@ if ($arch eq "x86_64") {
} elsif ($arch eq "s390" && $bits == 32) {
$mcount_regex = "^\\s*([0-9a-fA-F]+):\\s*R_390_32\\s+_mcount\$";
$mcount_adjust = -4;
$alignment = 4;
$ld .= " -m elf_s390";
$cc .= " -m31";
} elsif ($arch eq "s390" && $bits == 64) {
$mcount_regex = "^\\s*([0-9a-fA-F]+):\\s*R_390_(PC|PLT)32DBL\\s+_mcount\\+0x2\$";
$mcount_adjust = -8;
$alignment = 8;
$type = ".quad";
$ld .= " -m elf64_s390";

52
tools/perf/Documentation/perf-script.txt
View file

@ -113,13 +113,61 @@ OPTIONS
Do various checks like samples ordering and lost events.
-f::
--fields
--fields::
Comma separated list of fields to print. Options are:
comm, tid, pid, time, cpu, event, trace, sym. Field
list must be prepended with the type, trace, sw or hw,
list can be prepended with the type, trace, sw or hw,
to indicate to which event type the field list applies.
e.g., -f sw:comm,tid,time,sym and -f trace:time,cpu,trace
perf script -f <fields>
is equivalent to:
perf script -f trace:<fields> -f sw:<fields> -f hw:<fields>
i.e., the specified fields apply to all event types if the type string
is not given.
The arguments are processed in the order received. A later usage can
reset a prior request. e.g.:
-f trace: -f comm,tid,time,sym
The first -f suppresses trace events (field list is ""), but then the
second invocation sets the fields to comm,tid,time,sym. In this case a
warning is given to the user:
"Overriding previous field request for all events."
Alternativey, consider the order:
-f comm,tid,time,sym -f trace:
The first -f sets the fields for all events and the second -f
suppresses trace events. The user is given a warning message about
the override, and the result of the above is that only S/W and H/W
events are displayed with the given fields.
For the 'wildcard' option if a user selected field is invalid for an
event type, a message is displayed to the user that the option is
ignored for that type. For example:
$ perf script -f comm,tid,trace
'trace' not valid for hardware events. Ignoring.
'trace' not valid for software events. Ignoring.
Alternatively, if the type is given an invalid field is specified it
is an error. For example:
perf script -v -f sw:comm,tid,trace
'trace' not valid for software events.
At this point usage is displayed, and perf-script exits.
Finally, a user may not set fields to none for all event types.
i.e., -f "" is not allowed.
-k::
--vmlinux=<file>::
vmlinux pathname

89
tools/perf/Makefile
View file

@ -5,6 +5,8 @@ endif
# The default target of this Makefile is...
all:
include config/utilities.mak
ifneq ($(OUTPUT),)
# check that the output directory actually exists
OUTDIR := $(shell cd $(OUTPUT) && /bin/pwd)
@ -13,6 +15,12 @@ endif
# Define V to have a more verbose compile.
#
# Define PYTHON to point to the python binary if the default
# `python' is not correct; for example: PYTHON=python2
#
# Define PYTHON_CONFIG to point to the python-config binary if
# the default `$(PYTHON)-config' is not correct.
#
# Define ASCIIDOC8 if you want to format documentation with AsciiDoc 8
#
# Define DOCBOOK_XSL_172 if you want to format man pages with DocBook XSL v1.72.
@ -134,7 +142,7 @@ INSTALL = install
# explicitly what architecture to check for. Fix this up for yours..
SPARSE_FLAGS = -D__BIG_ENDIAN__ -D__powerpc__
-include feature-tests.mak
-include config/feature-tests.mak
ifeq ($(call try-cc,$(SOURCE_HELLO),-Werror -fstack-protector-all),y)
CFLAGS := $(CFLAGS) -fstack-protector-all
@ -169,12 +177,10 @@ grep-libs = $(filter -l%,$(1))
strip-libs = $(filter-out -l%,$(1))
$(OUTPUT)python/perf.so: $(PYRF_OBJS)
$(QUIET_GEN)( \
export CFLAGS="$(BASIC_CFLAGS)"; \
python util/setup.py --quiet build_ext --build-lib='$(OUTPUT)python' \
--build-temp='$(OUTPUT)python/temp' \
)
$(QUIET_GEN)CFLAGS='$(BASIC_CFLAGS)' $(PYTHON_WORD) util/setup.py \
--quiet build_ext \
--build-lib='$(OUTPUT)python' \
--build-temp='$(OUTPUT)python/temp'
#
# No Perl scripts right now:
#
@ -479,17 +485,63 @@ else
endif
endif
ifdef NO_LIBPYTHON
disable-python = $(eval $(disable-python_code))
define disable-python_code
BASIC_CFLAGS += -DNO_LIBPYTHON
$(if $(1),$(warning No $(1) was found))
$(warning Python support won't be built)
endef
override PYTHON := \
$(call get-executable-or-default,PYTHON,python)
ifndef PYTHON
$(call disable-python,python interpreter)
python-clean :=
else
PYTHON_EMBED_LDOPTS = $(shell python-config --ldflags 2>/dev/null)
PYTHON_EMBED_LDFLAGS = $(call strip-libs,$(PYTHON_EMBED_LDOPTS))
PYTHON_EMBED_LIBADD = $(call grep-libs,$(PYTHON_EMBED_LDOPTS))
PYTHON_EMBED_CCOPTS = `python-config --cflags 2>/dev/null`
FLAGS_PYTHON_EMBED=$(PYTHON_EMBED_CCOPTS) $(PYTHON_EMBED_LDOPTS)
PYTHON_WORD := $(call shell-wordify,$(PYTHON))
python-clean := $(PYTHON_WORD) util/setup.py clean \
--build-lib='$(OUTPUT)python' \
--build-temp='$(OUTPUT)python/temp'
ifdef NO_LIBPYTHON
$(call disable-python)
else
override PYTHON_CONFIG := \
$(call get-executable-or-default,PYTHON_CONFIG,$(PYTHON)-config)
ifndef PYTHON_CONFIG
$(call disable-python,python-config tool)
else
PYTHON_CONFIG_SQ := $(call shell-sq,$(PYTHON_CONFIG))
PYTHON_EMBED_LDOPTS := $(shell $(PYTHON_CONFIG_SQ) --ldflags 2>/dev/null)
PYTHON_EMBED_LDFLAGS := $(call strip-libs,$(PYTHON_EMBED_LDOPTS))
PYTHON_EMBED_LIBADD := $(call grep-libs,$(PYTHON_EMBED_LDOPTS))
PYTHON_EMBED_CCOPTS := $(shell $(PYTHON_CONFIG_SQ) --cflags 2>/dev/null)
FLAGS_PYTHON_EMBED := $(PYTHON_EMBED_CCOPTS) $(PYTHON_EMBED_LDOPTS)
ifneq ($(call try-cc,$(SOURCE_PYTHON_EMBED),$(FLAGS_PYTHON_EMBED)),y)
msg := $(warning No Python.h found, install python-dev[el] to have python support in 'perf script' and to build the python bindings)
BASIC_CFLAGS += -DNO_LIBPYTHON
$(call disable-python,Python.h (for Python 2.x))
else
ifneq ($(call try-cc,$(SOURCE_PYTHON_VERSION),$(FLAGS_PYTHON_EMBED)),y)
$(warning Python 3 is not yet supported; please set)
$(warning PYTHON and/or PYTHON_CONFIG appropriately.)
$(warning If you also have Python 2 installed, then)
$(warning try something like:)
$(warning $(and ,))
$(warning $(and ,) make PYTHON=python2)
$(warning $(and ,))
$(warning Otherwise, disable Python support entirely:)
$(warning $(and ,))
$(warning $(and ,) make NO_LIBPYTHON=1)
$(warning $(and ,))
$(error $(and ,))
else
ALL_LDFLAGS += $(PYTHON_EMBED_LDFLAGS)
EXTLIBS += $(PYTHON_EMBED_LIBADD)
@ -497,6 +549,10 @@ else
LIB_OBJS += $(OUTPUT)scripts/python/Perf-Trace-Util/Context.o
LANG_BINDINGS += $(OUTPUT)python/perf.so
endif
endif
endif
endif
endif
ifdef NO_DEMANGLE
@ -837,8 +893,7 @@ clean:
$(RM) *.spec *.pyc *.pyo */*.pyc */*.pyo $(OUTPUT)common-cmds.h TAGS tags cscope*
$(MAKE) -C Documentation/ clean
$(RM) $(OUTPUT)PERF-VERSION-FILE $(OUTPUT)PERF-CFLAGS
@python util/setup.py clean --build-lib='$(OUTPUT)python' \
--build-temp='$(OUTPUT)python/temp'
$(python-clean)
.PHONY: all install clean strip
.PHONY: shell_compatibility_test please_set_SHELL_PATH_to_a_more_modern_shell

282
tools/perf/builtin-script.c
View file

@ -49,57 +49,169 @@ struct output_option {
};
/* default set to maintain compatibility with current format */
static u64 output_fields[PERF_TYPE_MAX] = {
[PERF_TYPE_HARDWARE] = PERF_OUTPUT_COMM | PERF_OUTPUT_TID | \
PERF_OUTPUT_CPU | PERF_OUTPUT_TIME | \
static struct {
bool user_set;
bool wildcard_set;
u64 fields;
u64 invalid_fields;
} output[PERF_TYPE_MAX] = {
[PERF_TYPE_HARDWARE] = {
.user_set = false,
.fields = PERF_OUTPUT_COMM | PERF_OUTPUT_TID |
PERF_OUTPUT_CPU | PERF_OUTPUT_TIME |
PERF_OUTPUT_EVNAME | PERF_OUTPUT_SYM,
[PERF_TYPE_SOFTWARE] = PERF_OUTPUT_COMM | PERF_OUTPUT_TID | \
PERF_OUTPUT_CPU | PERF_OUTPUT_TIME | \
.invalid_fields = PERF_OUTPUT_TRACE,
},
[PERF_TYPE_SOFTWARE] = {
.user_set = false,
.fields = PERF_OUTPUT_COMM | PERF_OUTPUT_TID |
PERF_OUTPUT_CPU | PERF_OUTPUT_TIME |
PERF_OUTPUT_EVNAME | PERF_OUTPUT_SYM,
[PERF_TYPE_TRACEPOINT] = PERF_OUTPUT_COMM | PERF_OUTPUT_TID | \
PERF_OUTPUT_CPU | PERF_OUTPUT_TIME | \
.invalid_fields = PERF_OUTPUT_TRACE,
},
[PERF_TYPE_TRACEPOINT] = {
.user_set = false,
.fields = PERF_OUTPUT_COMM | PERF_OUTPUT_TID |
PERF_OUTPUT_CPU | PERF_OUTPUT_TIME |
PERF_OUTPUT_EVNAME | PERF_OUTPUT_TRACE,
},
[PERF_TYPE_RAW] = {
.user_set = false,
.fields = PERF_OUTPUT_COMM | PERF_OUTPUT_TID |
PERF_OUTPUT_CPU | PERF_OUTPUT_TIME |
PERF_OUTPUT_EVNAME | PERF_OUTPUT_SYM,
.invalid_fields = PERF_OUTPUT_TRACE,
},
};
static bool output_set_by_user;
#define PRINT_FIELD(x) (output_fields[attr->type] & PERF_OUTPUT_##x)
static int perf_session__check_attr(struct perf_session *session,
struct perf_event_attr *attr)
static bool output_set_by_user(void)
{
int j;
for (j = 0; j < PERF_TYPE_MAX; ++j) {
if (output[j].user_set)
return true;
}
return false;
}
static const char *output_field2str(enum perf_output_field field)
{
int i, imax = ARRAY_SIZE(all_output_options);
const char *str = "";
for (i = 0; i < imax; ++i) {
if (all_output_options[i].field == field) {
str = all_output_options[i].str;
break;
}
}
return str;
}
#define PRINT_FIELD(x) (output[attr->type].fields & PERF_OUTPUT_##x)
static int perf_event_attr__check_stype(struct perf_event_attr *attr,
u64 sample_type, const char *sample_msg,
enum perf_output_field field)
{
int type = attr->type;
const char *evname;
if (attr->sample_type & sample_type)
return 0;
if (output[type].user_set) {
evname = __event_name(attr->type, attr->config);
pr_err("Samples for '%s' event do not have %s attribute set. "
"Cannot print '%s' field.\n",
evname, sample_msg, output_field2str(field));
return -1;
}
/* user did not ask for it explicitly so remove from the default list */
output[type].fields &= ~field;
evname = __event_name(attr->type, attr->config);
pr_debug("Samples for '%s' event do not have %s attribute set. "
"Skipping '%s' field.\n",
evname, sample_msg, output_field2str(field));
return 0;
}
static int perf_evsel__check_attr(struct perf_evsel *evsel,
struct perf_session *session)
{
struct perf_event_attr *attr = &evsel->attr;
if (PRINT_FIELD(TRACE) &&
!perf_session__has_traces(session, "record -R"))
return -EINVAL;
if (PRINT_FIELD(SYM)) {
if (!(session->sample_type & PERF_SAMPLE_IP)) {
pr_err("Samples do not contain IP data.\n");
if (perf_event_attr__check_stype(attr, PERF_SAMPLE_IP, "IP",
PERF_OUTPUT_SYM))
return -EINVAL;
}
if (!no_callchain &&
!(session->sample_type & PERF_SAMPLE_CALLCHAIN))
!(attr->sample_type & PERF_SAMPLE_CALLCHAIN))
symbol_conf.use_callchain = false;
}
if ((PRINT_FIELD(PID) || PRINT_FIELD(TID)) &&
!(session->sample_type & PERF_SAMPLE_TID)) {
pr_err("Samples do not contain TID/PID data.\n");
perf_event_attr__check_stype(attr, PERF_SAMPLE_TID, "TID",
PERF_OUTPUT_TID|PERF_OUTPUT_PID))
return -EINVAL;
}
if (PRINT_FIELD(TIME) &&
!(session->sample_type & PERF_SAMPLE_TIME)) {
pr_err("Samples do not contain timestamps.\n");
perf_event_attr__check_stype(attr, PERF_SAMPLE_TIME, "TIME",
PERF_OUTPUT_TIME))
return -EINVAL;
}
if (PRINT_FIELD(CPU) &&
!(session->sample_type & PERF_SAMPLE_CPU)) {
pr_err("Samples do not contain cpu.\n");
perf_event_attr__check_stype(attr, PERF_SAMPLE_CPU, "CPU",
PERF_OUTPUT_CPU))
return -EINVAL;
return 0;
}
/*
* verify all user requested events exist and the samples
* have the expected data
*/
static int perf_session__check_output_opt(struct perf_session *session)
{
int j;
struct perf_evsel *evsel;
for (j = 0; j < PERF_TYPE_MAX; ++j) {
evsel = perf_session__find_first_evtype(session, j);
/*
* even if fields is set to 0 (ie., show nothing) event must
* exist if user explicitly includes it on the command line
*/
if (!evsel && output[j].user_set && !output[j].wildcard_set) {
pr_err("%s events do not exist. "
"Remove corresponding -f option to proceed.\n",
event_type(j));
return -1;
}
if (evsel && output[j].fields &&
perf_evsel__check_attr(evsel, session))
return -1;
}
return 0;
@ -168,10 +280,7 @@ static void process_event(union perf_event *event __unused,
{
struct perf_event_attr *attr = &evsel->attr;
if (output_fields[attr->type] == 0)
return;
if (perf_session__check_attr(session, attr) < 0)
if (output[attr->type].fields == 0)
return;
print_sample_start(sample, thread, attr);
@ -451,6 +560,7 @@ static int parse_output_fields(const struct option *opt __used,
{
char *tok;
int i, imax = sizeof(all_output_options) / sizeof(struct output_option);
int j;
int rc = 0;
char *str = strdup(arg);
int type = -1;
@ -458,52 +568,99 @@ static int parse_output_fields(const struct option *opt __used,
if (!str)
return -ENOMEM;
tok = strtok(str, ":");
if (!tok) {
fprintf(stderr,
"Invalid field string - not prepended with type.");
return -EINVAL;
}
/* first word should state which event type user
* is specifying the fields
/* first word can state for which event type the user is specifying
* the fields. If no type exists, the specified fields apply to all
* event types found in the file minus the invalid fields for a type.
*/
if (!strcmp(tok, "hw"))
tok = strchr(str, ':');
if (tok) {
*tok = '\0';
tok++;
if (!strcmp(str, "hw"))
type = PERF_TYPE_HARDWARE;
else if (!strcmp(tok, "sw"))
else if (!strcmp(str, "sw"))
type = PERF_TYPE_SOFTWARE;
else if (!strcmp(tok, "trace"))
else if (!strcmp(str, "trace"))
type = PERF_TYPE_TRACEPOINT;
else if (!strcmp(str, "raw"))
type = PERF_TYPE_RAW;
else {
fprintf(stderr, "Invalid event type in field string.");
fprintf(stderr, "Invalid event type in field string.\n");
return -EINVAL;
}
output_fields[type] = 0;
while (1) {
tok = strtok(NULL, ",");
if (!tok)
break;
for (i = 0; i < imax; ++i) {
if (strcmp(tok, all_output_options[i].str) == 0) {
output_fields[type] |= all_output_options[i].field;
break;
if (output[type].user_set)
pr_warning("Overriding previous field request for %s events.\n",
event_type(type));
output[type].fields = 0;
output[type].user_set = true;
output[type].wildcard_set = false;
} else {
tok = str;
if (strlen(str) == 0) {
fprintf(stderr,
"Cannot set fields to 'none' for all event types.\n");
rc = -EINVAL;
goto out;
}
if (output_set_by_user())
pr_warning("Overriding previous field request for all events.\n");
for (j = 0; j < PERF_TYPE_MAX; ++j) {
output[j].fields = 0;
output[j].user_set = true;
output[j].wildcard_set = true;
}
}
tok = strtok(tok, ",");
while (tok) {
for (i = 0; i < imax; ++i) {
if (strcmp(tok, all_output_options[i].str) == 0)
break;
}
if (i == imax) {
fprintf(stderr, "Invalid field requested.");
fprintf(stderr, "Invalid field requested.\n");
rc = -EINVAL;
break;
}
goto out;
}
if (output_fields[type] == 0) {
if (type == -1) {
/* add user option to all events types for
* which it is valid
*/
for (j = 0; j < PERF_TYPE_MAX; ++j) {
if (output[j].invalid_fields & all_output_options[i].field) {
pr_warning("\'%s\' not valid for %s events. Ignoring.\n",
all_output_options[i].str, event_type(j));
} else
output[j].fields |= all_output_options[i].field;
}
} else {
if (output[type].invalid_fields & all_output_options[i].field) {
fprintf(stderr, "\'%s\' not valid for %s events.\n",
all_output_options[i].str, event_type(type));
rc = -EINVAL;
goto out;
}
output[type].fields |= all_output_options[i].field;
}
tok = strtok(NULL, ",");
}
if (type >= 0) {
if (output[type].fields == 0) {
pr_debug("No fields requested for %s type. "
"Events will not be displayed\n", event_type(type));
"Events will not be displayed.\n", event_type(type));
}
}
output_set_by_user = true;
out:
free(str);
return rc;
}
@ -829,7 +986,7 @@ static const struct option options[] = {
OPT_STRING(0, "symfs", &symbol_conf.symfs, "directory",
"Look for files with symbols relative to this directory"),
OPT_CALLBACK('f', "fields", NULL, "str",
"comma separated output fields prepend with 'type:'. Valid types: hw,sw,trace. Fields: comm,tid,pid,time,cpu,event,trace,sym",
"comma separated output fields prepend with 'type:'. Valid types: hw,sw,trace,raw. Fields: comm,tid,pid,time,cpu,event,trace,sym",
parse_output_fields),
OPT_END()
@ -1020,7 +1177,7 @@ int cmd_script(int argc, const char **argv, const char *prefix __used)
struct stat perf_stat;
int input;
if (output_set_by_user) {
if (output_set_by_user()) {
fprintf(stderr,
"custom fields not supported for generated scripts");
return -1;
@ -1060,6 +1217,11 @@ int cmd_script(int argc, const char **argv, const char *prefix __used)
pr_debug("perf script started with script %s\n\n", script_name);
}
err = perf_session__check_output_opt(session);
if (err < 0)
goto out;
err = __cmd_script(session);
perf_session__delete(session);

567
tools/perf/builtin-stat.c
View file

@ -6,24 +6,28 @@
*
* Sample output:
$ perf stat ~/hackbench 10
Time: 0.104
$ perf stat ./hackbench 10
Performance counter stats for '/home/mingo/hackbench':
Time: 0.118
1255.538611 task clock ticks # 10.143 CPU utilization factor
54011 context switches # 0.043 M/sec
385 CPU migrations # 0.000 M/sec
17755 pagefaults # 0.014 M/sec
3808323185 CPU cycles # 3033.219 M/sec
1575111190 instructions # 1254.530 M/sec
17367895 cache references # 13.833 M/sec
7674421 cache misses # 6.112 M/sec
Performance counter stats for './hackbench 10':
Wall-clock time elapsed: 123.786620 msecs
1708.761321 task-clock # 11.037 CPUs utilized
41,190 context-switches # 0.024 M/sec
6,735 CPU-migrations # 0.004 M/sec
17,318 page-faults # 0.010 M/sec
5,205,202,243 cycles # 3.046 GHz
3,856,436,920 stalled-cycles-frontend # 74.09% frontend cycles idle
1,600,790,871 stalled-cycles-backend # 30.75% backend cycles idle
2,603,501,247 instructions # 0.50 insns per cycle
# 1.48 stalled cycles per insn
484,357,498 branches # 283.455 M/sec
6,388,934 branch-misses # 1.32% of all branches
0.154822978 seconds time elapsed
*
* Copyright (C) 2008, Red Hat Inc, Ingo Molnar <mingo@redhat.com>
* Copyright (C) 2008-2011, Red Hat Inc, Ingo Molnar <mingo@redhat.com>
*
* Improvements and fixes by:
*
@ -46,6 +50,7 @@
#include "util/evlist.h"
#include "util/evsel.h"
#include "util/debug.h"
#include "util/color.h"
#include "util/header.h"
#include "util/cpumap.h"
#include "util/thread.h"
@ -65,14 +70,107 @@ static struct perf_event_attr default_attrs[] = {
{ .type = PERF_TYPE_SOFTWARE, .config = PERF_COUNT_SW_PAGE_FAULTS },
{ .type = PERF_TYPE_HARDWARE, .config = PERF_COUNT_HW_CPU_CYCLES },
{ .type = PERF_TYPE_HARDWARE, .config = PERF_COUNT_HW_STALLED_CYCLES_FRONTEND },
{ .type = PERF_TYPE_HARDWARE, .config = PERF_COUNT_HW_STALLED_CYCLES_BACKEND },
{ .type = PERF_TYPE_HARDWARE, .config = PERF_COUNT_HW_INSTRUCTIONS },
{ .type = PERF_TYPE_HARDWARE, .config = PERF_COUNT_HW_BRANCH_INSTRUCTIONS },
{ .type = PERF_TYPE_HARDWARE, .config = PERF_COUNT_HW_BRANCH_MISSES },
{ .type = PERF_TYPE_HARDWARE, .config = PERF_COUNT_HW_CACHE_REFERENCES },
{ .type = PERF_TYPE_HARDWARE, .config = PERF_COUNT_HW_CACHE_MISSES },
};
/*
* Detailed stats (-d), covering the L1 and last level data caches:
*/
static struct perf_event_attr detailed_attrs[] = {
{ .type = PERF_TYPE_HW_CACHE,
.config =
PERF_COUNT_HW_CACHE_L1D << 0 |
(PERF_COUNT_HW_CACHE_OP_READ << 8) |
(PERF_COUNT_HW_CACHE_RESULT_ACCESS << 16) },
{ .type = PERF_TYPE_HW_CACHE,
.config =
PERF_COUNT_HW_CACHE_L1D << 0 |
(PERF_COUNT_HW_CACHE_OP_READ << 8) |
(PERF_COUNT_HW_CACHE_RESULT_MISS << 16) },
{ .type = PERF_TYPE_HW_CACHE,
.config =
PERF_COUNT_HW_CACHE_LL << 0 |
(PERF_COUNT_HW_CACHE_OP_READ << 8) |
(PERF_COUNT_HW_CACHE_RESULT_ACCESS << 16) },
{ .type = PERF_TYPE_HW_CACHE,
.config =
PERF_COUNT_HW_CACHE_LL << 0 |
(PERF_COUNT_HW_CACHE_OP_READ << 8) |
(PERF_COUNT_HW_CACHE_RESULT_MISS << 16) },
};
/*
* Very detailed stats (-d -d), covering the instruction cache and the TLB caches:
*/
static struct perf_event_attr very_detailed_attrs[] = {
{ .type = PERF_TYPE_HW_CACHE,
.config =
PERF_COUNT_HW_CACHE_L1I << 0 |
(PERF_COUNT_HW_CACHE_OP_READ << 8) |
(PERF_COUNT_HW_CACHE_RESULT_ACCESS << 16) },
{ .type = PERF_TYPE_HW_CACHE,
.config =
PERF_COUNT_HW_CACHE_L1I << 0 |
(PERF_COUNT_HW_CACHE_OP_READ << 8) |
(PERF_COUNT_HW_CACHE_RESULT_MISS << 16) },
{ .type = PERF_TYPE_HW_CACHE,
.config =
PERF_COUNT_HW_CACHE_DTLB << 0 |
(PERF_COUNT_HW_CACHE_OP_READ << 8) |
(PERF_COUNT_HW_CACHE_RESULT_ACCESS << 16) },
{ .type = PERF_TYPE_HW_CACHE,
.config =
PERF_COUNT_HW_CACHE_DTLB << 0 |
(PERF_COUNT_HW_CACHE_OP_READ << 8) |
(PERF_COUNT_HW_CACHE_RESULT_MISS << 16) },
{ .type = PERF_TYPE_HW_CACHE,
.config =
PERF_COUNT_HW_CACHE_ITLB << 0 |
(PERF_COUNT_HW_CACHE_OP_READ << 8) |
(PERF_COUNT_HW_CACHE_RESULT_ACCESS << 16) },
{ .type = PERF_TYPE_HW_CACHE,
.config =
PERF_COUNT_HW_CACHE_ITLB << 0 |
(PERF_COUNT_HW_CACHE_OP_READ << 8) |
(PERF_COUNT_HW_CACHE_RESULT_MISS << 16) },
};
/*
* Very, very detailed stats (-d -d -d), adding prefetch events:
*/
static struct perf_event_attr very_very_detailed_attrs[] = {
{ .type = PERF_TYPE_HW_CACHE,
.config =
PERF_COUNT_HW_CACHE_L1D << 0 |
(PERF_COUNT_HW_CACHE_OP_PREFETCH << 8) |
(PERF_COUNT_HW_CACHE_RESULT_ACCESS << 16) },
{ .type = PERF_TYPE_HW_CACHE,
.config =
PERF_COUNT_HW_CACHE_L1D << 0 |
(PERF_COUNT_HW_CACHE_OP_PREFETCH << 8) |
(PERF_COUNT_HW_CACHE_RESULT_MISS << 16) },
};
struct perf_evlist *evsel_list;
static bool system_wide = false;
@ -86,6 +184,8 @@ static pid_t target_pid = -1;
static pid_t target_tid = -1;
static pid_t child_pid = -1;
static bool null_run = false;
static int detailed_run = 0;
static bool sync_run = false;
static bool big_num = true;
static int big_num_opt = -1;
static const char *cpu_list;
@ -156,7 +256,15 @@ static double stddev_stats(struct stats *stats)
struct stats runtime_nsecs_stats[MAX_NR_CPUS];
struct stats runtime_cycles_stats[MAX_NR_CPUS];
struct stats runtime_stalled_cycles_front_stats[MAX_NR_CPUS];
struct stats runtime_stalled_cycles_back_stats[MAX_NR_CPUS];
struct stats runtime_branches_stats[MAX_NR_CPUS];
struct stats runtime_cacherefs_stats[MAX_NR_CPUS];
struct stats runtime_l1_dcache_stats[MAX_NR_CPUS];
struct stats runtime_l1_icache_stats[MAX_NR_CPUS];
struct stats runtime_ll_cache_stats[MAX_NR_CPUS];
struct stats runtime_itlb_cache_stats[MAX_NR_CPUS];
struct stats runtime_dtlb_cache_stats[MAX_NR_CPUS];
struct stats walltime_nsecs_stats;
static int create_perf_stat_counter(struct perf_evsel *evsel)
@ -192,6 +300,37 @@ static inline int nsec_counter(struct perf_evsel *evsel)
return 0;
}
/*
* Update various tracking values we maintain to print
* more semantic information such as miss/hit ratios,
* instruction rates, etc:
*/
static void update_shadow_stats(struct perf_evsel *counter, u64 *count)
{
if (perf_evsel__match(counter, SOFTWARE, SW_TASK_CLOCK))
update_stats(&runtime_nsecs_stats[0], count[0]);
else if (perf_evsel__match(counter, HARDWARE, HW_CPU_CYCLES))
update_stats(&runtime_cycles_stats[0], count[0]);
else if (perf_evsel__match(counter, HARDWARE, HW_STALLED_CYCLES_FRONTEND))
update_stats(&runtime_stalled_cycles_front_stats[0], count[0]);
else if (perf_evsel__match(counter, HARDWARE, HW_STALLED_CYCLES_BACKEND))
update_stats(&runtime_stalled_cycles_back_stats[0], count[0]);
else if (perf_evsel__match(counter, HARDWARE, HW_BRANCH_INSTRUCTIONS))
update_stats(&runtime_branches_stats[0], count[0]);
else if (perf_evsel__match(counter, HARDWARE, HW_CACHE_REFERENCES))
update_stats(&runtime_cacherefs_stats[0], count[0]);
else if (perf_evsel__match(counter, HW_CACHE, HW_CACHE_L1D))
update_stats(&runtime_l1_dcache_stats[0], count[0]);
else if (perf_evsel__match(counter, HW_CACHE, HW_CACHE_L1I))
update_stats(&runtime_l1_icache_stats[0], count[0]);
else if (perf_evsel__match(counter, HW_CACHE, HW_CACHE_LL))
update_stats(&runtime_ll_cache_stats[0], count[0]);
else if (perf_evsel__match(counter, HW_CACHE, HW_CACHE_DTLB))
update_stats(&runtime_dtlb_cache_stats[0], count[0]);
else if (perf_evsel__match(counter, HW_CACHE, HW_CACHE_ITLB))
update_stats(&runtime_itlb_cache_stats[0], count[0]);
}
/*
* Read out the results of a single counter:
* aggregate counts across CPUs in system-wide mode
@ -217,12 +356,7 @@ static int read_counter_aggr(struct perf_evsel *counter)
/*
* Save the full runtime - to allow normalization during printout:
*/
if (perf_evsel__match(counter, SOFTWARE, SW_TASK_CLOCK))
update_stats(&runtime_nsecs_stats[0], count[0]);
if (perf_evsel__match(counter, HARDWARE, HW_CPU_CYCLES))
update_stats(&runtime_cycles_stats[0], count[0]);
if (perf_evsel__match(counter, HARDWARE, HW_BRANCH_INSTRUCTIONS))
update_stats(&runtime_branches_stats[0], count[0]);
update_shadow_stats(counter, count);
return 0;
}
@ -242,12 +376,7 @@ static int read_counter(struct perf_evsel *counter)
count = counter->counts->cpu[cpu].values;
if (perf_evsel__match(counter, SOFTWARE, SW_TASK_CLOCK))
update_stats(&runtime_nsecs_stats[cpu], count[0]);
if (perf_evsel__match(counter, HARDWARE, HW_CPU_CYCLES))
update_stats(&runtime_cycles_stats[cpu], count[0]);
if (perf_evsel__match(counter, HARDWARE, HW_BRANCH_INSTRUCTIONS))
update_stats(&runtime_branches_stats[cpu], count[0]);
update_shadow_stats(counter, count);
}
return 0;
@ -315,13 +444,18 @@ static int run_perf_stat(int argc __used, const char **argv)
list_for_each_entry(counter, &evsel_list->entries, node) {
if (create_perf_stat_counter(counter) < 0) {
if (errno == -EPERM || errno == -EACCES) {
if (errno == EINVAL || errno == ENOSYS || errno == ENOENT) {
if (verbose)
ui__warning("%s event is not supported by the kernel.\n",
event_name(counter));
continue;
}
if (errno == EPERM || errno == EACCES) {
error("You may not have permission to collect %sstats.\n"
"\t Consider tweaking"
" /proc/sys/kernel/perf_event_paranoid or running as root.",
system_wide ? "system-wide " : "");
} else if (errno == ENOENT) {
error("%s event is not supported. ", event_name(counter));
} else {
error("open_counter returned with %d (%s). "
"/bin/dmesg may provide additional information.\n",
@ -372,6 +506,16 @@ static int run_perf_stat(int argc __used, const char **argv)
return WEXITSTATUS(status);
}
static void print_noise_pct(double total, double avg)
{
double pct = 0.0;
if (avg)
pct = 100.0*total/avg;
fprintf(stderr, " ( +-%6.2f%% )", pct);
}
static void print_noise(struct perf_evsel *evsel, double avg)
{
struct perf_stat *ps;
@ -380,15 +524,14 @@ static void print_noise(struct perf_evsel *evsel, double avg)
return;
ps = evsel->priv;
fprintf(stderr, " ( +- %7.3f%% )",
100 * stddev_stats(&ps->res_stats[0]) / avg);
print_noise_pct(stddev_stats(&ps->res_stats[0]), avg);
}
static void nsec_printout(int cpu, struct perf_evsel *evsel, double avg)
{
double msecs = avg / 1e6;
char cpustr[16] = { '\0', };
const char *fmt = csv_output ? "%s%.6f%s%s" : "%s%18.6f%s%-24s";
const char *fmt = csv_output ? "%s%.6f%s%s" : "%s%18.6f%s%-25s";
if (no_aggr)
sprintf(cpustr, "CPU%*d%s",
@ -404,8 +547,191 @@ static void nsec_printout(int cpu, struct perf_evsel *evsel, double avg)
return;
if (perf_evsel__match(evsel, SOFTWARE, SW_TASK_CLOCK))
fprintf(stderr, " # %10.3f CPUs ",
avg / avg_stats(&walltime_nsecs_stats));
fprintf(stderr, " # %8.3f CPUs utilized ", avg / avg_stats(&walltime_nsecs_stats));
}
static void print_stalled_cycles_frontend(int cpu, struct perf_evsel *evsel __used, double avg)
{
double total, ratio = 0.0;
const char *color;
total = avg_stats(&runtime_cycles_stats[cpu]);
if (total)
ratio = avg / total * 100.0;
color = PERF_COLOR_NORMAL;
if (ratio > 50.0)
color = PERF_COLOR_RED;
else if (ratio > 30.0)
color = PERF_COLOR_MAGENTA;
else if (ratio > 10.0)
color = PERF_COLOR_YELLOW;
fprintf(stderr, " # ");
color_fprintf(stderr, color, "%6.2f%%", ratio);
fprintf(stderr, " frontend cycles idle ");
}
static void print_stalled_cycles_backend(int cpu, struct perf_evsel *evsel __used, double avg)
{
double total, ratio = 0.0;
const char *color;
total = avg_stats(&runtime_cycles_stats[cpu]);
if (total)
ratio = avg / total * 100.0;
color = PERF_COLOR_NORMAL;
if (ratio > 75.0)
color = PERF_COLOR_RED;
else if (ratio > 50.0)
color = PERF_COLOR_MAGENTA;
else if (ratio > 20.0)
color = PERF_COLOR_YELLOW;
fprintf(stderr, " # ");
color_fprintf(stderr, color, "%6.2f%%", ratio);
fprintf(stderr, " backend cycles idle ");
}
static void print_branch_misses(int cpu, struct perf_evsel *evsel __used, double avg)
{
double total, ratio = 0.0;
const char *color;
total = avg_stats(&runtime_branches_stats[cpu]);
if (total)
ratio = avg / total * 100.0;
color = PERF_COLOR_NORMAL;
if (ratio > 20.0)
color = PERF_COLOR_RED;
else if (ratio > 10.0)
color = PERF_COLOR_MAGENTA;
else if (ratio > 5.0)
color = PERF_COLOR_YELLOW;
fprintf(stderr, " # ");
color_fprintf(stderr, color, "%6.2f%%", ratio);
fprintf(stderr, " of all branches ");
}
static void print_l1_dcache_misses(int cpu, struct perf_evsel *evsel __used, double avg)
{
double total, ratio = 0.0;
const char *color;
total = avg_stats(&runtime_l1_dcache_stats[cpu]);
if (total)
ratio = avg / total * 100.0;
color = PERF_COLOR_NORMAL;
if (ratio > 20.0)
color = PERF_COLOR_RED;
else if (ratio > 10.0)
color = PERF_COLOR_MAGENTA;
else if (ratio > 5.0)
color = PERF_COLOR_YELLOW;
fprintf(stderr, " # ");
color_fprintf(stderr, color, "%6.2f%%", ratio);
fprintf(stderr, " of all L1-dcache hits ");
}
static void print_l1_icache_misses(int cpu, struct perf_evsel *evsel __used, double avg)
{
double total, ratio = 0.0;
const char *color;
total = avg_stats(&runtime_l1_icache_stats[cpu]);
if (total)
ratio = avg / total * 100.0;
color = PERF_COLOR_NORMAL;
if (ratio > 20.0)
color = PERF_COLOR_RED;
else if (ratio > 10.0)
color = PERF_COLOR_MAGENTA;
else if (ratio > 5.0)
color = PERF_COLOR_YELLOW;
fprintf(stderr, " # ");
color_fprintf(stderr, color, "%6.2f%%", ratio);
fprintf(stderr, " of all L1-icache hits ");
}
static void print_dtlb_cache_misses(int cpu, struct perf_evsel *evsel __used, double avg)
{
double total, ratio = 0.0;
const char *color;
total = avg_stats(&runtime_dtlb_cache_stats[cpu]);
if (total)
ratio = avg / total * 100.0;
color = PERF_COLOR_NORMAL;
if (ratio > 20.0)
color = PERF_COLOR_RED;
else if (ratio > 10.0)
color = PERF_COLOR_MAGENTA;
else if (ratio > 5.0)
color = PERF_COLOR_YELLOW;
fprintf(stderr, " # ");
color_fprintf(stderr, color, "%6.2f%%", ratio);
fprintf(stderr, " of all dTLB cache hits ");
}
static void print_itlb_cache_misses(int cpu, struct perf_evsel *evsel __used, double avg)
{
double total, ratio = 0.0;
const char *color;
total = avg_stats(&runtime_itlb_cache_stats[cpu]);
if (total)
ratio = avg / total * 100.0;
color = PERF_COLOR_NORMAL;
if (ratio > 20.0)
color = PERF_COLOR_RED;
else if (ratio > 10.0)
color = PERF_COLOR_MAGENTA;
else if (ratio > 5.0)
color = PERF_COLOR_YELLOW;
fprintf(stderr, " # ");
color_fprintf(stderr, color, "%6.2f%%", ratio);
fprintf(stderr, " of all iTLB cache hits ");
}
static void print_ll_cache_misses(int cpu, struct perf_evsel *evsel __used, double avg)
{
double total, ratio = 0.0;
const char *color;
total = avg_stats(&runtime_ll_cache_stats[cpu]);
if (total)
ratio = avg / total * 100.0;
color = PERF_COLOR_NORMAL;
if (ratio > 20.0)
color = PERF_COLOR_RED;
else if (ratio > 10.0)
color = PERF_COLOR_MAGENTA;
else if (ratio > 5.0)
color = PERF_COLOR_YELLOW;
fprintf(stderr, " # ");
color_fprintf(stderr, color, "%6.2f%%", ratio);
fprintf(stderr, " of all LL-cache hits ");
}
static void abs_printout(int cpu, struct perf_evsel *evsel, double avg)
@ -417,9 +743,9 @@ static void abs_printout(int cpu, struct perf_evsel *evsel, double avg)
if (csv_output)
fmt = "%s%.0f%s%s";
else if (big_num)
fmt = "%s%'18.0f%s%-24s";
fmt = "%s%'18.0f%s%-25s";
else
fmt = "%s%18.0f%s%-24s";
fmt = "%s%18.0f%s%-25s";
if (no_aggr)
sprintf(cpustr, "CPU%*d%s",
@ -442,23 +768,83 @@ static void abs_printout(int cpu, struct perf_evsel *evsel, double avg)
if (total)
ratio = avg / total;
fprintf(stderr, " # %10.3f IPC ", ratio);
fprintf(stderr, " # %5.2f insns per cycle ", ratio);
total = avg_stats(&runtime_stalled_cycles_front_stats[cpu]);
total = max(total, avg_stats(&runtime_stalled_cycles_back_stats[cpu]));
if (total && avg) {
ratio = total / avg;
fprintf(stderr, "\n # %5.2f stalled cycles per insn", ratio);
}
} else if (perf_evsel__match(evsel, HARDWARE, HW_BRANCH_MISSES) &&
runtime_branches_stats[cpu].n != 0) {
total = avg_stats(&runtime_branches_stats[cpu]);
print_branch_misses(cpu, evsel, avg);
} else if (
evsel->attr.type == PERF_TYPE_HW_CACHE &&
evsel->attr.config == ( PERF_COUNT_HW_CACHE_L1D |
((PERF_COUNT_HW_CACHE_OP_READ) << 8) |
((PERF_COUNT_HW_CACHE_RESULT_MISS) << 16)) &&
runtime_l1_dcache_stats[cpu].n != 0) {
print_l1_dcache_misses(cpu, evsel, avg);
} else if (
evsel->attr.type == PERF_TYPE_HW_CACHE &&
evsel->attr.config == ( PERF_COUNT_HW_CACHE_L1I |
((PERF_COUNT_HW_CACHE_OP_READ) << 8) |
((PERF_COUNT_HW_CACHE_RESULT_MISS) << 16)) &&
runtime_l1_icache_stats[cpu].n != 0) {
print_l1_icache_misses(cpu, evsel, avg);
} else if (
evsel->attr.type == PERF_TYPE_HW_CACHE &&
evsel->attr.config == ( PERF_COUNT_HW_CACHE_DTLB |
((PERF_COUNT_HW_CACHE_OP_READ) << 8) |
((PERF_COUNT_HW_CACHE_RESULT_MISS) << 16)) &&
runtime_dtlb_cache_stats[cpu].n != 0) {
print_dtlb_cache_misses(cpu, evsel, avg);
} else if (
evsel->attr.type == PERF_TYPE_HW_CACHE &&
evsel->attr.config == ( PERF_COUNT_HW_CACHE_ITLB |
((PERF_COUNT_HW_CACHE_OP_READ) << 8) |
((PERF_COUNT_HW_CACHE_RESULT_MISS) << 16)) &&
runtime_itlb_cache_stats[cpu].n != 0) {
print_itlb_cache_misses(cpu, evsel, avg);
} else if (
evsel->attr.type == PERF_TYPE_HW_CACHE &&
evsel->attr.config == ( PERF_COUNT_HW_CACHE_LL |
((PERF_COUNT_HW_CACHE_OP_READ) << 8) |
((PERF_COUNT_HW_CACHE_RESULT_MISS) << 16)) &&
runtime_ll_cache_stats[cpu].n != 0) {
print_ll_cache_misses(cpu, evsel, avg);
} else if (perf_evsel__match(evsel, HARDWARE, HW_CACHE_MISSES) &&
runtime_cacherefs_stats[cpu].n != 0) {
total = avg_stats(&runtime_cacherefs_stats[cpu]);
if (total)
ratio = avg * 100 / total;
fprintf(stderr, " # %10.3f %% ", ratio);
fprintf(stderr, " # %8.3f %% of all cache refs ", ratio);
} else if (perf_evsel__match(evsel, HARDWARE, HW_STALLED_CYCLES_FRONTEND)) {
print_stalled_cycles_frontend(cpu, evsel, avg);
} else if (perf_evsel__match(evsel, HARDWARE, HW_STALLED_CYCLES_BACKEND)) {
print_stalled_cycles_backend(cpu, evsel, avg);
} else if (perf_evsel__match(evsel, HARDWARE, HW_CPU_CYCLES)) {
total = avg_stats(&runtime_nsecs_stats[cpu]);
if (total)
ratio = 1.0 * avg / total;
fprintf(stderr, " # %8.3f GHz ", ratio);
} else if (runtime_nsecs_stats[cpu].n != 0) {
total = avg_stats(&runtime_nsecs_stats[cpu]);
if (total)
ratio = 1000.0 * avg / total;
fprintf(stderr, " # %10.3f M/sec", ratio);
fprintf(stderr, " # %8.3f M/sec ", ratio);
} else {
fprintf(stderr, " ");
}
}
@ -505,8 +891,7 @@ static void print_counter_aggr(struct perf_evsel *counter)
avg_enabled = avg_stats(&ps->res_stats[1]);
avg_running = avg_stats(&ps->res_stats[2]);
fprintf(stderr, " (scaled from %.2f%%)",
100 * avg_running / avg_enabled);
fprintf(stderr, " [%5.2f%%]", 100 * avg_running / avg_enabled);
}
fprintf(stderr, "\n");
}
@ -548,10 +933,8 @@ static void print_counter(struct perf_evsel *counter)
if (!csv_output) {
print_noise(counter, 1.0);
if (run != ena) {
fprintf(stderr, " (scaled from %.2f%%)",
100.0 * run / ena);
}
if (run != ena)
fprintf(stderr, " (%.2f%%)", 100.0 * run / ena);
}
fputc('\n', stderr);
}
@ -591,12 +974,13 @@ static void print_stat(int argc, const char **argv)
}
if (!csv_output) {
if (!null_run)
fprintf(stderr, "\n");
fprintf(stderr, " %18.9f seconds time elapsed",
fprintf(stderr, " %17.9f seconds time elapsed",
avg_stats(&walltime_nsecs_stats)/1e9);
if (run_count > 1) {
fprintf(stderr, " ( +- %7.3f%% )",
100*stddev_stats(&walltime_nsecs_stats) /
fprintf(stderr, " ");
print_noise_pct(stddev_stats(&walltime_nsecs_stats),
avg_stats(&walltime_nsecs_stats));
}
fprintf(stderr, "\n\n");
@ -659,6 +1043,10 @@ static const struct option options[] = {
"repeat command and print average + stddev (max: 100)"),
OPT_BOOLEAN('n', "null", &null_run,
"null run - dont start any counters"),
OPT_INCR('d', "detailed", &detailed_run,
"detailed run - start a lot of events"),
OPT_BOOLEAN('S', "sync", &sync_run,
"call sync() before starting a run"),
OPT_CALLBACK_NOOPT('B', "big-num", NULL, NULL,
"print large numbers with thousands\' separators",
stat__set_big_num),
@ -674,6 +1062,70 @@ static const struct option options[] = {
OPT_END()
};
/*
* Add default attributes, if there were no attributes specified or
* if -d/--detailed, -d -d or -d -d -d is used:
*/
static int add_default_attributes(void)
{
struct perf_evsel *pos;
size_t attr_nr = 0;
size_t c;
/* Set attrs if no event is selected and !null_run: */
if (null_run)
return 0;
if (!evsel_list->nr_entries) {
for (c = 0; c < ARRAY_SIZE(default_attrs); c++) {
pos = perf_evsel__new(default_attrs + c, c + attr_nr);
if (pos == NULL)
return -1;
perf_evlist__add(evsel_list, pos);
}
attr_nr += c;
}
/* Detailed events get appended to the event list: */
if (detailed_run < 1)
return 0;
/* Append detailed run extra attributes: */
for (c = 0; c < ARRAY_SIZE(detailed_attrs); c++) {
pos = perf_evsel__new(detailed_attrs + c, c + attr_nr);
if (pos == NULL)
return -1;
perf_evlist__add(evsel_list, pos);
}
attr_nr += c;
if (detailed_run < 2)
return 0;
/* Append very detailed run extra attributes: */
for (c = 0; c < ARRAY_SIZE(very_detailed_attrs); c++) {
pos = perf_evsel__new(very_detailed_attrs + c, c + attr_nr);
if (pos == NULL)
return -1;
perf_evlist__add(evsel_list, pos);
}
if (detailed_run < 3)
return 0;
/* Append very, very detailed run extra attributes: */
for (c = 0; c < ARRAY_SIZE(very_very_detailed_attrs); c++) {
pos = perf_evsel__new(very_very_detailed_attrs + c, c + attr_nr);
if (pos == NULL)
return -1;
perf_evlist__add(evsel_list, pos);
}
return 0;
}
int cmd_stat(int argc, const char **argv, const char *prefix __used)
{
struct perf_evsel *pos;
@ -719,17 +1171,8 @@ int cmd_stat(int argc, const char **argv, const char *prefix __used)
usage_with_options(stat_usage, options);
}
/* Set attrs and nr_counters if no event is selected and !null_run */
if (!null_run && !evsel_list->nr_entries) {
size_t c;
for (c = 0; c < ARRAY_SIZE(default_attrs); ++c) {
pos = perf_evsel__new(&default_attrs[c], c);
if (pos == NULL)
if (add_default_attributes())
goto out;
perf_evlist__add(evsel_list, pos);
}
}
if (target_pid != -1)
target_tid = target_pid;
@ -773,6 +1216,10 @@ int cmd_stat(int argc, const char **argv, const char *prefix __used)
for (run_idx = 0; run_idx < run_count; run_idx++) {
if (run_count != 1 && verbose)
fprintf(stderr, "[ perf stat: executing run #%d ... ]\n", run_idx + 1);
if (sync_run)
sync();
status = run_perf_stat(argc, argv);
}

16
tools/perf/feature-tests.mak → tools/perf/config/feature-tests.mak
View file

@ -79,9 +79,15 @@ endef
endif
ifndef NO_LIBPYTHON
define SOURCE_PYTHON_VERSION
#include <Python.h>
#if PY_VERSION_HEX >= 0x03000000
#error
#endif
int main(void){}
endef
define SOURCE_PYTHON_EMBED
#include <Python.h>
int main(void)
{
Py_Initialize();
@ -120,11 +126,3 @@ int main(void)
return 0;
}
endef
# try-cc
# Usage: option = $(call try-cc, source-to-build, cc-options)
try-cc = $(shell sh -c \
'TMP="$(OUTPUT)$(TMPOUT).$$$$"; \
echo "$(1)" | \
$(CC) -x c - $(2) -o "$$TMP" > /dev/null 2>&1 && echo y; \
rm -f "$$TMP"')

188
tools/perf/config/utilities.mak Normal file
View file

@ -0,0 +1,188 @@
# This allows us to work with the newline character:
define newline
endef
newline := $(newline)
# nl-escape
#
# Usage: escape = $(call nl-escape[,escape])
#
# This is used as the common way to specify
# what should replace a newline when escaping
# newlines; the default is a bizarre string.
#
nl-escape = $(or $(1),m822df3020w6a44id34bt574ctac44eb9f4n)
# escape-nl
#
# Usage: escaped-text = $(call escape-nl,text[,escape])
#
# GNU make's $(shell ...) function converts to a
# single space each newline character in the output
# produced during the expansion; this may not be
# desirable.
#
# The only solution is to change each newline into
# something that won't be converted, so that the
# information can be recovered later with
# $(call unescape-nl...)
#
escape-nl = $(subst $(newline),$(call nl-escape,$(2)),$(1))
# unescape-nl
#
# Usage: text = $(call unescape-nl,escaped-text[,escape])
#
# See escape-nl.
#
unescape-nl = $(subst $(call nl-escape,$(2)),$(newline),$(1))
# shell-escape-nl
#
# Usage: $(shell some-command | $(call shell-escape-nl[,escape]))
#
# Use this to escape newlines from within a shell call;
# the default escape is a bizarre string.
#
# NOTE: The escape is used directly as a string constant
# in an `awk' program that is delimited by shell
# single-quotes, so be wary of the characters
# that are chosen.
#
define shell-escape-nl
awk 'NR==1 {t=$$0} NR>1 {t=t "$(nl-escape)" $$0} END {printf t}'
endef
# shell-unescape-nl
#
# Usage: $(shell some-command | $(call shell-unescape-nl[,escape]))
#
# Use this to unescape newlines from within a shell call;
# the default escape is a bizarre string.
#
# NOTE: The escape is used directly as an extended regular
# expression constant in an `awk' program that is
# delimited by shell single-quotes, so be wary
# of the characters that are chosen.
#
# (The bash shell has a bug where `{gsub(...),...}' is
# misinterpreted as a brace expansion; this can be
# overcome by putting a space between `{' and `gsub').
#
define shell-unescape-nl
awk 'NR==1 {t=$$0} NR>1 {t=t "\n" $$0} END { gsub(/$(nl-escape)/,"\n",t); printf t }'
endef
# escape-for-shell-sq
#
# Usage: embeddable-text = $(call escape-for-shell-sq,text)
#
# This function produces text that is suitable for
# embedding in a shell string that is delimited by
# single-quotes.
#
escape-for-shell-sq = $(subst ','\'',$(1))
# shell-sq
#
# Usage: single-quoted-and-escaped-text = $(call shell-sq,text)
#
shell-sq = '$(escape-for-shell-sq)'
# shell-wordify
#
# Usage: wordified-text = $(call shell-wordify,text)
#
# For instance:
#
# |define text
# |hello
# |world
# |endef
# |
# |target:
# | echo $(call shell-wordify,$(text))
#
# At least GNU make gets confused by expanding a newline
# within the context of a command line of a makefile rule
# (this is in constrast to a `$(shell ...)' function call,
# which can handle it just fine).
#
# This function avoids the problem by producing a string
# that works as a shell word, regardless of whether or
# not it contains a newline.
#
# If the text to be wordified contains a newline, then
# an intrictate shell command substitution is constructed
# to render the text as a single line; when the shell
# processes the resulting escaped text, it transforms
# it into the original unescaped text.
#
# If the text does not contain a newline, then this function
# produces the same results as the `$(shell-sq)' function.
#
shell-wordify = $(if $(findstring $(newline),$(1)),$(_sw-esc-nl),$(shell-sq))
define _sw-esc-nl
"$$(echo $(call escape-nl,$(shell-sq),$(2)) | $(call shell-unescape-nl,$(2)))"
endef
# is-absolute
#
# Usage: bool-value = $(call is-absolute,path)
#
is-absolute = $(shell echo $(shell-sq) | grep ^/ -q && echo y)
# lookup
#
# Usage: absolute-executable-path-or-empty = $(call lookup,path)
#
# (It's necessary to use `sh -c' because GNU make messes up by
# trying too hard and getting things wrong).
#
lookup = $(call unescape-nl,$(shell sh -c $(_l-sh)))
_l-sh = $(call shell-sq,command -v $(shell-sq) | $(call shell-escape-nl,))
# is-executable
#
# Usage: bool-value = $(call is-executable,path)
#
# (It's necessary to use `sh -c' because GNU make messes up by
# trying too hard and getting things wrong).
#
is-executable = $(call _is-executable-helper,$(shell-sq))
_is-executable-helper = $(shell sh -c $(_is-executable-sh))
_is-executable-sh = $(call shell-sq,test -f $(1) -a -x $(1) && echo y)
# get-executable
#
# Usage: absolute-executable-path-or-empty = $(call get-executable,path)
#
# The goal is to get an absolute path for an executable;
# the `command -v' is defined by POSIX, but it's not
# necessarily very portable, so it's only used if
# relative path resolution is requested, as determined
# by the presence of a leading `/'.
#
get-executable = $(if $(1),$(if $(is-absolute),$(_ge-abspath),$(lookup)))
_ge-abspath = $(if $(is-executable),$(1))
# get-supplied-or-default-executable
#
# Usage: absolute-executable-path-or-empty = $(call get-executable-or-default,variable,default)
#
define get-executable-or-default
$(if $($(1)),$(call _ge_attempt,$($(1)),$(1)),$(call _ge_attempt,$(2)))
endef
_ge_attempt = $(or $(get-executable),$(_gea_warn),$(call _gea_err,$(2)))
_gea_warn = $(warning The path '$(1)' is not executable.)
_gea_err = $(if $(1),$(error Please set '$(1)' appropriately))
# try-cc
# Usage: option = $(call try-cc, source-to-build, cc-options)
try-cc = $(shell sh -c \
'TMP="$(OUTPUT)$(TMPOUT).$$$$"; \
echo "$(1)" | \
$(CC) -x c - $(2) -o "$$TMP" > /dev/null 2>&1 && echo y; \
rm -f "$$TMP"')

8
tools/perf/util/include/asm/alternative-asm.h Normal file
View file

@ -0,0 +1,8 @@
#ifndef _PERF_ASM_ALTERNATIVE_ASM_H
#define _PERF_ASM_ALTERNATIVE_ASM_H
/* Just disable it so we can build arch/x86/lib/memcpy_64.S for perf bench: */
#define altinstruction_entry #
#endif

75
tools/perf/util/parse-events.c
View file

@ -32,6 +32,8 @@ char debugfs_path[MAXPATHLEN];
static struct event_symbol event_symbols[] = {
{ CHW(CPU_CYCLES), "cpu-cycles", "cycles" },
{ CHW(STALLED_CYCLES_FRONTEND), "stalled-cycles-frontend", "idle-cycles-frontend" },
{ CHW(STALLED_CYCLES_BACKEND), "stalled-cycles-backend", "idle-cycles-backend" },
{ CHW(INSTRUCTIONS), "instructions", "" },
{ CHW(CACHE_REFERENCES), "cache-references", "" },
{ CHW(CACHE_MISSES), "cache-misses", "" },
@ -58,7 +60,7 @@ static struct event_symbol event_symbols[] = {
#define PERF_EVENT_TYPE(config) __PERF_EVENT_FIELD(config, TYPE)
#define PERF_EVENT_ID(config) __PERF_EVENT_FIELD(config, EVENT)
static const char *hw_event_names[] = {
static const char *hw_event_names[PERF_COUNT_HW_MAX] = {
"cycles",
"instructions",
"cache-references",
@ -66,11 +68,13 @@ static const char *hw_event_names[] = {
"branches",
"branch-misses",
"bus-cycles",
"stalled-cycles-frontend",
"stalled-cycles-backend",
};
static const char *sw_event_names[] = {
"cpu-clock-msecs",
"task-clock-msecs",
static const char *sw_event_names[PERF_COUNT_SW_MAX] = {
"cpu-clock",
"task-clock",
"page-faults",
"context-switches",
"CPU-migrations",
@ -307,7 +311,7 @@ const char *__event_name(int type, u64 config)
switch (type) {
case PERF_TYPE_HARDWARE:
if (config < PERF_COUNT_HW_MAX)
if (config < PERF_COUNT_HW_MAX && hw_event_names[config])
return hw_event_names[config];
return "unknown-hardware";
@ -333,7 +337,7 @@ const char *__event_name(int type, u64 config)
}
case PERF_TYPE_SOFTWARE:
if (config < PERF_COUNT_SW_MAX)
if (config < PERF_COUNT_SW_MAX && sw_event_names[config])
return sw_event_names[config];
return "unknown-software";
@ -648,13 +652,15 @@ static int check_events(const char *str, unsigned int i)
int n;
n = strlen(event_symbols[i].symbol);
if (!strncmp(str, event_symbols[i].symbol, n))
if (!strncasecmp(str, event_symbols[i].symbol, n))
return n;
n = strlen(event_symbols[i].alias);
if (n)
if (!strncmp(str, event_symbols[i].alias, n))
if (n) {
if (!strncasecmp(str, event_symbols[i].alias, n))
return n;
}
return 0;
}
@ -718,15 +724,22 @@ parse_numeric_event(const char **strp, struct perf_event_attr *attr)
return EVT_FAILED;
}
static enum event_result
static int
parse_event_modifier(const char **strp, struct perf_event_attr *attr)
{
const char *str = *strp;
int exclude = 0;
int eu = 0, ek = 0, eh = 0, precise = 0;
if (*str++ != ':')
if (!*str)
return 0;
if (*str == ',')
return 0;
if (*str++ != ':')
return -1;
while (*str) {
if (*str == 'u') {
if (!exclude)
@ -747,14 +760,16 @@ parse_event_modifier(const char **strp, struct perf_event_attr *attr)
++str;
}
if (str >= *strp + 2) {
if (str < *strp + 2)
return -1;
*strp = str;
attr->exclude_user = eu;
attr->exclude_kernel = ek;
attr->exclude_hv = eh;
attr->precise_ip = precise;
return 1;
}
return 0;
}
@ -797,7 +812,12 @@ parse_event_symbols(const struct option *opt, const char **str,
return EVT_FAILED;
modifier:
parse_event_modifier(str, attr);
if (parse_event_modifier(str, attr) < 0) {
fprintf(stderr, "invalid event modifier: '%s'\n", *str);
fprintf(stderr, "Run 'perf list' for a list of valid events and modifiers\n");
return EVT_FAILED;
}
return ret;
}
@ -912,7 +932,7 @@ void print_tracepoint_events(const char *subsys_glob, const char *event_glob)
snprintf(evt_path, MAXPATHLEN, "%s:%s",
sys_dirent.d_name, evt_dirent.d_name);
printf(" %-42s [%s]\n", evt_path,
printf(" %-50s [%s]\n", evt_path,
event_type_descriptors[PERF_TYPE_TRACEPOINT]);
}
closedir(evt_dir);
@ -977,7 +997,7 @@ void print_events_type(u8 type)
else
snprintf(name, sizeof(name), "%s", syms->symbol);
printf(" %-42s [%s]\n", name,
printf(" %-50s [%s]\n", name,
event_type_descriptors[type]);
}
}
@ -995,11 +1015,10 @@ int print_hwcache_events(const char *event_glob)
for (i = 0; i < PERF_COUNT_HW_CACHE_RESULT_MAX; i++) {
char *name = event_cache_name(type, op, i);
if (event_glob != NULL &&
!strglobmatch(name, event_glob))
if (event_glob != NULL && !strglobmatch(name, event_glob))
continue;
printf(" %-42s [%s]\n", name,
printf(" %-50s [%s]\n", name,
event_type_descriptors[PERF_TYPE_HW_CACHE]);
++printed;
}
@ -1009,14 +1028,16 @@ int print_hwcache_events(const char *event_glob)
return printed;
}
#define MAX_NAME_LEN 100
/*
* Print the help text for the event symbols:
*/
void print_events(const char *event_glob)
{
struct event_symbol *syms = event_symbols;
unsigned int i, type, prev_type = -1, printed = 0, ntypes_printed = 0;
char name[40];
struct event_symbol *syms = event_symbols;
char name[MAX_NAME_LEN];
printf("\n");
printf("List of pre-defined events (to be used in -e):\n");
@ -1036,10 +1057,10 @@ void print_events(const char *event_glob)
continue;
if (strlen(syms->alias))
sprintf(name, "%s OR %s", syms->symbol, syms->alias);
snprintf(name, MAX_NAME_LEN, "%s OR %s", syms->symbol, syms->alias);
else
strcpy(name, syms->symbol);
printf(" %-42s [%s]\n", name,
strncpy(name, syms->symbol, MAX_NAME_LEN);
printf(" %-50s [%s]\n", name,
event_type_descriptors[type]);
prev_type = type;
@ -1056,12 +1077,12 @@ void print_events(const char *event_glob)
return;
printf("\n");
printf(" %-42s [%s]\n",
printf(" %-50s [%s]\n",
"rNNN (see 'perf list --help' on how to encode it)",
event_type_descriptors[PERF_TYPE_RAW]);
printf("\n");
printf(" %-42s [%s]\n",
printf(" %-50s [%s]\n",
"mem:<addr>[:access]",
event_type_descriptors[PERF_TYPE_BREAKPOINT]);
printf("\n");

73
tools/perf/util/probe-finder.c
View file

@ -1471,6 +1471,38 @@ static int find_probe_point_by_func(struct probe_finder *pf)
return _param.retval;
}
struct pubname_callback_param {
char *function;
char *file;
Dwarf_Die *cu_die;
Dwarf_Die *sp_die;
int found;
};
static int pubname_search_cb(Dwarf *dbg, Dwarf_Global *gl, void *data)
{
struct pubname_callback_param *param = data;
if (dwarf_offdie(dbg, gl->die_offset, param->sp_die)) {
if (dwarf_tag(param->sp_die) != DW_TAG_subprogram)
return DWARF_CB_OK;
if (die_compare_name(param->sp_die, param->function)) {
if (!dwarf_offdie(dbg, gl->cu_offset, param->cu_die))
return DWARF_CB_OK;
if (param->file &&
strtailcmp(param->file, dwarf_decl_file(param->sp_die)))
return DWARF_CB_OK;
param->found = 1;
return DWARF_CB_ABORT;
}
}
return DWARF_CB_OK;
}
/* Find probe points from debuginfo */
static int find_probes(int fd, struct probe_finder *pf)
{
@ -1498,6 +1530,28 @@ static int find_probes(int fd, struct probe_finder *pf)
off = 0;
line_list__init(&pf->lcache);
/* Fastpath: lookup by function name from .debug_pubnames section */
if (pp->function) {
struct pubname_callback_param pubname_param = {
.function = pp->function,
.file = pp->file,
.cu_die = &pf->cu_die,
.sp_die = &pf->sp_die,
.found = 0,
};
struct dwarf_callback_param probe_param = {
.data = pf,
};
dwarf_getpubnames(dbg, pubname_search_cb, &pubname_param, 0);
if (pubname_param.found) {
ret = probe_point_search_cb(&pf->sp_die, &probe_param);
if (ret)
goto found;
}
}
/* Loop on CUs (Compilation Unit) */
while (!dwarf_nextcu(dbg, off, &noff, &cuhl, NULL, NULL, NULL)) {
/* Get the DIE(Debugging Information Entry) of this CU */
@ -1525,6 +1579,8 @@ static int find_probes(int fd, struct probe_finder *pf)
}
off = noff;
}
found:
line_list__free(&pf->lcache);
if (dwfl)
dwfl_end(dwfl);
@ -1946,6 +2002,22 @@ int find_line_range(int fd, struct line_range *lr)
return -EBADF;
}
/* Fastpath: lookup by function name from .debug_pubnames section */
if (lr->function) {
struct pubname_callback_param pubname_param = {
.function = lr->function, .file = lr->file,
.cu_die = &lf.cu_die, .sp_die = &lf.sp_die, .found = 0};
struct dwarf_callback_param line_range_param = {
.data = (void *)&lf, .retval = 0};
dwarf_getpubnames(dbg, pubname_search_cb, &pubname_param, 0);
if (pubname_param.found) {
line_range_search_cb(&lf.sp_die, &line_range_param);
if (lf.found)
goto found;
}
}
/* Loop on CUs (Compilation Unit) */
while (!lf.found && ret >= 0) {
if (dwarf_nextcu(dbg, off, &noff, &cuhl, NULL, NULL, NULL) != 0)
@ -1974,6 +2046,7 @@ int find_line_range(int fd, struct line_range *lr)
off = noff;
}
found:
/* Store comp_dir */
if (lf.found) {
comp_dir = cu_get_comp_dir(&lf.cu_die);

2
tools/perf/util/probe-finder.h
View file

@ -49,6 +49,7 @@ struct probe_finder {
Dwarf_Addr addr; /* Address */
const char *fname; /* Real file name */
Dwarf_Die cu_die; /* Current CU */
Dwarf_Die sp_die;
struct list_head lcache; /* Line cache for lazy match */
/* For variable searching */
@ -83,6 +84,7 @@ struct line_finder {
int lno_s; /* Start line number */
int lno_e; /* End line number */
Dwarf_Die cu_die; /* Current CU */
Dwarf_Die sp_die;
int found;
};

3
tools/perf/util/python.c
View file

@ -810,6 +810,9 @@ static struct {
{ "COUNT_HW_CACHE_RESULT_ACCESS", PERF_COUNT_HW_CACHE_RESULT_ACCESS },
{ "COUNT_HW_CACHE_RESULT_MISS", PERF_COUNT_HW_CACHE_RESULT_MISS },
{ "COUNT_HW_STALLED_CYCLES_FRONTEND", PERF_COUNT_HW_STALLED_CYCLES_FRONTEND },
{ "COUNT_HW_STALLED_CYCLES_BACKEND", PERF_COUNT_HW_STALLED_CYCLES_BACKEND },
{ "COUNT_SW_CPU_CLOCK", PERF_COUNT_SW_CPU_CLOCK },
{ "COUNT_SW_TASK_CLOCK", PERF_COUNT_SW_TASK_CLOCK },
{ "COUNT_SW_PAGE_FAULTS", PERF_COUNT_SW_PAGE_FAULTS },

12
tools/perf/util/session.c
View file

@ -1156,6 +1156,18 @@ size_t perf_session__fprintf_nr_events(struct perf_session *session, FILE *fp)
return ret;
}
struct perf_evsel *perf_session__find_first_evtype(struct perf_session *session,
unsigned int type)
{
struct perf_evsel *pos;
list_for_each_entry(pos, &session->evlist->entries, node) {
if (pos->attr.type == type)
return pos;
}
return NULL;
}
void perf_session__print_symbols(union perf_event *event,
struct perf_sample *sample,
struct perf_session *session)

3
tools/perf/util/session.h
View file

@ -162,6 +162,9 @@ static inline int perf_session__parse_sample(struct perf_session *session,
session->sample_id_all, sample);
}
struct perf_evsel *perf_session__find_first_evtype(struct perf_session *session,
unsigned int type);
void perf_session__print_symbols(union perf_event *event,
struct perf_sample *sample,
struct perf_session *session);

623
tools/perf/util/symbol.c
View file

File diff suppressed because it is too large Load diff

78
tools/perf/util/symbol.h
View file

@ -62,7 +62,7 @@ struct symbol {
char name[0];
};
void symbol__delete(struct symbol *self);
void symbol__delete(struct symbol *sym);
struct strlist;
@ -96,9 +96,9 @@ struct symbol_conf {
extern struct symbol_conf symbol_conf;
static inline void *symbol__priv(struct symbol *self)
static inline void *symbol__priv(struct symbol *sym)
{
return ((void *)self) - symbol_conf.priv_size;
return ((void *)sym) - symbol_conf.priv_size;
}
struct ref_reloc_sym {
@ -155,43 +155,45 @@ struct dso {
struct dso *dso__new(const char *name);
struct dso *dso__new_kernel(const char *name);
void dso__delete(struct dso *self);
void dso__delete(struct dso *dso);
int dso__name_len(const struct dso *self);
int dso__name_len(const struct dso *dso);
bool dso__loaded(const struct dso *self, enum map_type type);
bool dso__sorted_by_name(const struct dso *self, enum map_type type);
bool dso__loaded(const struct dso *dso, enum map_type type);
bool dso__sorted_by_name(const struct dso *dso, enum map_type type);
static inline void dso__set_loaded(struct dso *self, enum map_type type)
static inline void dso__set_loaded(struct dso *dso, enum map_type type)
{
self->loaded |= (1 << type);
dso->loaded |= (1 << type);
}
void dso__sort_by_name(struct dso *self, enum map_type type);
void dso__sort_by_name(struct dso *dso, enum map_type type);
struct dso *__dsos__findnew(struct list_head *head, const char *name);
int dso__load(struct dso *self, struct map *map, symbol_filter_t filter);
int dso__load_vmlinux(struct dso *self, struct map *map,
int dso__load(struct dso *dso, struct map *map, symbol_filter_t filter);
int dso__load_vmlinux(struct dso *dso, struct map *map,
const char *vmlinux, symbol_filter_t filter);
int dso__load_vmlinux_path(struct dso *self, struct map *map,
int dso__load_vmlinux_path(struct dso *dso, struct map *map,
symbol_filter_t filter);
int dso__load_kallsyms(struct dso *self, const char *filename, struct map *map,
int dso__load_kallsyms(struct dso *dso, const char *filename, struct map *map,
symbol_filter_t filter);
int machine__load_kallsyms(struct machine *self, const char *filename,
int machine__load_kallsyms(struct machine *machine, const char *filename,
enum map_type type, symbol_filter_t filter);
int machine__load_vmlinux_path(struct machine *self, enum map_type type,
int machine__load_vmlinux_path(struct machine *machine, enum map_type type,
symbol_filter_t filter);
size_t __dsos__fprintf(struct list_head *head, FILE *fp);
size_t machine__fprintf_dsos_buildid(struct machine *self, FILE *fp, bool with_hits);
size_t machines__fprintf_dsos(struct rb_root *self, FILE *fp);
size_t machines__fprintf_dsos_buildid(struct rb_root *self, FILE *fp, bool with_hits);
size_t dso__fprintf_buildid(struct dso *self, FILE *fp);
size_t dso__fprintf_symbols_by_name(struct dso *self, enum map_type type, FILE *fp);
size_t dso__fprintf(struct dso *self, enum map_type type, FILE *fp);
size_t machine__fprintf_dsos_buildid(struct machine *machine,
FILE *fp, bool with_hits);
size_t machines__fprintf_dsos(struct rb_root *machines, FILE *fp);
size_t machines__fprintf_dsos_buildid(struct rb_root *machines,
FILE *fp, bool with_hits);
size_t dso__fprintf_buildid(struct dso *dso, FILE *fp);
size_t dso__fprintf_symbols_by_name(struct dso *dso,
enum map_type type, FILE *fp);
size_t dso__fprintf(struct dso *dso, enum map_type type, FILE *fp);
enum symtab_type {
SYMTAB__KALLSYMS = 0,
@ -207,34 +209,36 @@ enum symtab_type {
SYMTAB__NOT_FOUND,
};
char dso__symtab_origin(const struct dso *self);
void dso__set_long_name(struct dso *self, char *name);
void dso__set_build_id(struct dso *self, void *build_id);
void dso__read_running_kernel_build_id(struct dso *self, struct machine *machine);
struct symbol *dso__find_symbol(struct dso *self, enum map_type type, u64 addr);
struct symbol *dso__find_symbol_by_name(struct dso *self, enum map_type type,
char dso__symtab_origin(const struct dso *dso);
void dso__set_long_name(struct dso *dso, char *name);
void dso__set_build_id(struct dso *dso, void *build_id);
void dso__read_running_kernel_build_id(struct dso *dso,
struct machine *machine);
struct symbol *dso__find_symbol(struct dso *dso, enum map_type type,
u64 addr);
struct symbol *dso__find_symbol_by_name(struct dso *dso, enum map_type type,
const char *name);
int filename__read_build_id(const char *filename, void *bf, size_t size);
int sysfs__read_build_id(const char *filename, void *bf, size_t size);
bool __dsos__read_build_ids(struct list_head *head, bool with_hits);
int build_id__sprintf(const u8 *self, int len, char *bf);
int build_id__sprintf(const u8 *build_id, int len, char *bf);
int kallsyms__parse(const char *filename, void *arg,
int (*process_symbol)(void *arg, const char *name,
char type, u64 start, u64 end));
void machine__destroy_kernel_maps(struct machine *self);
int __machine__create_kernel_maps(struct machine *self, struct dso *kernel);
int machine__create_kernel_maps(struct machine *self);
void machine__destroy_kernel_maps(struct machine *machine);
int __machine__create_kernel_maps(struct machine *machine, struct dso *kernel);
int machine__create_kernel_maps(struct machine *machine);
int machines__create_kernel_maps(struct rb_root *self, pid_t pid);
int machines__create_guest_kernel_maps(struct rb_root *self);
void machines__destroy_guest_kernel_maps(struct rb_root *self);
int machines__create_kernel_maps(struct rb_root *machines, pid_t pid);
int machines__create_guest_kernel_maps(struct rb_root *machines);
void machines__destroy_guest_kernel_maps(struct rb_root *machines);
int symbol__init(void);
void symbol__exit(void);
bool symbol_type__is_a(char symbol_type, enum map_type map_type);
size_t machine__fprintf_vmlinux_path(struct machine *self, FILE *fp);
size_t machine__fprintf_vmlinux_path(struct machine *machine, FILE *fp);
#endif /* __PERF_SYMBOL */