diff options
| -rw-r--r-- | arch/arm/include/asm/arch_timer.h | 36 | ||||
| -rw-r--r-- | arch/arm/include/uapi/asm/hwcap.h | 1 | ||||
| -rw-r--r-- | arch/arm/kernel/setup.c | 1 | ||||
| -rw-r--r-- | arch/arm64/include/asm/arch_timer.h | 42 | ||||
| -rw-r--r-- | arch/arm64/include/asm/hwcap.h | 11 | ||||
| -rw-r--r-- | arch/arm64/include/uapi/asm/hwcap.h | 1 | ||||
| -rw-r--r-- | arch/arm64/kernel/setup.c | 11 | ||||
| -rw-r--r-- | drivers/clocksource/Kconfig | 15 | ||||
| -rw-r--r-- | drivers/clocksource/arm_arch_timer.c | 49 | ||||
| -rw-r--r-- | drivers/clocksource/tcb_clksrc.c | 61 | ||||
| -rw-r--r-- | include/clocksource/arm_arch_timer.h | 10 |
11 files changed, 211 insertions, 27 deletions
diff --git a/arch/arm/include/asm/arch_timer.h b/arch/arm/include/asm/arch_timer.h index 5665134bfa3e..0704e0cf5571 100644 --- a/arch/arm/include/asm/arch_timer.h +++ b/arch/arm/include/asm/arch_timer.h @@ -87,17 +87,43 @@ static inline u64 arch_counter_get_cntvct(void) return cval; } -static inline void arch_counter_set_user_access(void) +static inline u32 arch_timer_get_cntkctl(void) { u32 cntkctl; - asm volatile("mrc p15, 0, %0, c14, c1, 0" : "=r" (cntkctl)); + return cntkctl; +} - /* disable user access to everything */ - cntkctl &= ~((3 << 8) | (7 << 0)); - +static inline void arch_timer_set_cntkctl(u32 cntkctl) +{ asm volatile("mcr p15, 0, %0, c14, c1, 0" : : "r" (cntkctl)); } + +static inline void arch_counter_set_user_access(void) +{ + u32 cntkctl = arch_timer_get_cntkctl(); + + /* Disable user access to both physical/virtual counters/timers */ + /* Also disable virtual event stream */ + cntkctl &= ~(ARCH_TIMER_USR_PT_ACCESS_EN + | ARCH_TIMER_USR_VT_ACCESS_EN + | ARCH_TIMER_VIRT_EVT_EN + | ARCH_TIMER_USR_VCT_ACCESS_EN + | ARCH_TIMER_USR_PCT_ACCESS_EN); + arch_timer_set_cntkctl(cntkctl); +} + +static inline void arch_timer_evtstrm_enable(int divider) +{ + u32 cntkctl = arch_timer_get_cntkctl(); + cntkctl &= ~ARCH_TIMER_EVT_TRIGGER_MASK; + /* Set the divider and enable virtual event stream */ + cntkctl |= (divider << ARCH_TIMER_EVT_TRIGGER_SHIFT) + | ARCH_TIMER_VIRT_EVT_EN; + arch_timer_set_cntkctl(cntkctl); + elf_hwcap |= HWCAP_EVTSTRM; +} + #endif #endif diff --git a/arch/arm/include/uapi/asm/hwcap.h b/arch/arm/include/uapi/asm/hwcap.h index 6d34d080372a..7dcc10d67253 100644 --- a/arch/arm/include/uapi/asm/hwcap.h +++ b/arch/arm/include/uapi/asm/hwcap.h @@ -26,5 +26,6 @@ #define HWCAP_VFPD32 (1 << 19) /* set if VFP has 32 regs (not 16) */ #define HWCAP_IDIV (HWCAP_IDIVA | HWCAP_IDIVT) #define HWCAP_LPAE (1 << 20) +#define HWCAP_EVTSTRM (1 << 21) #endif /* _UAPI__ASMARM_HWCAP_H */ diff --git a/arch/arm/kernel/setup.c b/arch/arm/kernel/setup.c index 0e1e2b3afa45..5d65438685d8 100644 --- a/arch/arm/kernel/setup.c +++ b/arch/arm/kernel/setup.c @@ -975,6 +975,7 @@ static const char *hwcap_str[] = { "idivt", "vfpd32", "lpae", + "evtstrm", NULL }; diff --git a/arch/arm64/include/asm/arch_timer.h b/arch/arm64/include/asm/arch_timer.h index c9f1d2816c2b..9400596a0f39 100644 --- a/arch/arm64/include/asm/arch_timer.h +++ b/arch/arm64/include/asm/arch_timer.h @@ -92,19 +92,49 @@ static inline u32 arch_timer_get_cntfrq(void) return val; } -static inline void arch_counter_set_user_access(void) +static inline u32 arch_timer_get_cntkctl(void) { u32 cntkctl; - - /* Disable user access to the timers and the physical counter. */ asm volatile("mrs %0, cntkctl_el1" : "=r" (cntkctl)); - cntkctl &= ~((3 << 8) | (1 << 0)); + return cntkctl; +} - /* Enable user access to the virtual counter and frequency. */ - cntkctl |= (1 << 1); +static inline void arch_timer_set_cntkctl(u32 cntkctl) +{ asm volatile("msr cntkctl_el1, %0" : : "r" (cntkctl)); } +static inline void arch_counter_set_user_access(void) +{ + u32 cntkctl = arch_timer_get_cntkctl(); + + /* Disable user access to the timers and the physical counter */ + /* Also disable virtual event stream */ + cntkctl &= ~(ARCH_TIMER_USR_PT_ACCESS_EN + | ARCH_TIMER_USR_VT_ACCESS_EN + | ARCH_TIMER_VIRT_EVT_EN + | ARCH_TIMER_USR_PCT_ACCESS_EN); + + /* Enable user access to the virtual counter */ + cntkctl |= ARCH_TIMER_USR_VCT_ACCESS_EN; + + arch_timer_set_cntkctl(cntkctl); +} + +static inline void arch_timer_evtstrm_enable(int divider) +{ + u32 cntkctl = arch_timer_get_cntkctl(); + cntkctl &= ~ARCH_TIMER_EVT_TRIGGER_MASK; + /* Set the divider and enable virtual event stream */ + cntkctl |= (divider << ARCH_TIMER_EVT_TRIGGER_SHIFT) + | ARCH_TIMER_VIRT_EVT_EN; + arch_timer_set_cntkctl(cntkctl); + elf_hwcap |= HWCAP_EVTSTRM; +#ifdef CONFIG_COMPAT + compat_elf_hwcap |= COMPAT_HWCAP_EVTSTRM; +#endif +} + static inline u64 arch_counter_get_cntvct(void) { u64 cval; diff --git a/arch/arm64/include/asm/hwcap.h b/arch/arm64/include/asm/hwcap.h index e2950b098e76..6cddbb0c9f54 100644 --- a/arch/arm64/include/asm/hwcap.h +++ b/arch/arm64/include/asm/hwcap.h @@ -30,6 +30,7 @@ #define COMPAT_HWCAP_IDIVA (1 << 17) #define COMPAT_HWCAP_IDIVT (1 << 18) #define COMPAT_HWCAP_IDIV (COMPAT_HWCAP_IDIVA|COMPAT_HWCAP_IDIVT) +#define COMPAT_HWCAP_EVTSTRM (1 << 21) #ifndef __ASSEMBLY__ /* @@ -37,11 +38,11 @@ * instruction set this cpu supports. */ #define ELF_HWCAP (elf_hwcap) -#define COMPAT_ELF_HWCAP (COMPAT_HWCAP_HALF|COMPAT_HWCAP_THUMB|\ - COMPAT_HWCAP_FAST_MULT|COMPAT_HWCAP_EDSP|\ - COMPAT_HWCAP_TLS|COMPAT_HWCAP_VFP|\ - COMPAT_HWCAP_VFPv3|COMPAT_HWCAP_VFPv4|\ - COMPAT_HWCAP_NEON|COMPAT_HWCAP_IDIV) + +#ifdef CONFIG_COMPAT +#define COMPAT_ELF_HWCAP (compat_elf_hwcap) +extern unsigned int compat_elf_hwcap; +#endif extern unsigned long elf_hwcap; #endif diff --git a/arch/arm64/include/uapi/asm/hwcap.h b/arch/arm64/include/uapi/asm/hwcap.h index eea497578b87..9b12476e9c85 100644 --- a/arch/arm64/include/uapi/asm/hwcap.h +++ b/arch/arm64/include/uapi/asm/hwcap.h @@ -21,6 +21,7 @@ */ #define HWCAP_FP (1 << 0) #define HWCAP_ASIMD (1 << 1) +#define HWCAP_EVTSTRM (1 << 2) #endif /* _UAPI__ASM_HWCAP_H */ diff --git a/arch/arm64/kernel/setup.c b/arch/arm64/kernel/setup.c index 055cfb80e05c..d355b7b9710b 100644 --- a/arch/arm64/kernel/setup.c +++ b/arch/arm64/kernel/setup.c @@ -60,6 +60,16 @@ EXPORT_SYMBOL(processor_id); unsigned long elf_hwcap __read_mostly; EXPORT_SYMBOL_GPL(elf_hwcap); +#ifdef CONFIG_COMPAT +#define COMPAT_ELF_HWCAP_DEFAULT \ + (COMPAT_HWCAP_HALF|COMPAT_HWCAP_THUMB|\ + COMPAT_HWCAP_FAST_MULT|COMPAT_HWCAP_EDSP|\ + COMPAT_HWCAP_TLS|COMPAT_HWCAP_VFP|\ + COMPAT_HWCAP_VFPv3|COMPAT_HWCAP_VFPv4|\ + COMPAT_HWCAP_NEON|COMPAT_HWCAP_IDIV) +unsigned int compat_elf_hwcap __read_mostly = COMPAT_ELF_HWCAP_DEFAULT; +#endif + static const char *cpu_name; static const char *machine_name; phys_addr_t __fdt_pointer __initdata; @@ -304,6 +314,7 @@ subsys_initcall(topology_init); static const char *hwcap_str[] = { "fp", "asimd", + "evtstrm", NULL }; diff --git a/drivers/clocksource/Kconfig b/drivers/clocksource/Kconfig index 971d796e071d..5e940f839a2d 100644 --- a/drivers/clocksource/Kconfig +++ b/drivers/clocksource/Kconfig @@ -75,6 +75,21 @@ config ARM_ARCH_TIMER bool select CLKSRC_OF if OF +config ARM_ARCH_TIMER_EVTSTREAM + bool "Support for ARM architected timer event stream generation" + default y if ARM_ARCH_TIMER + help + This option enables support for event stream generation based on + the ARM architected timer. It is used for waking up CPUs executing + the wfe instruction at a frequency represented as a power-of-2 + divisor of the clock rate. + The main use of the event stream is wfe-based timeouts of userspace + locking implementations. It might also be useful for imposing timeout + on wfe to safeguard against any programming errors in case an expected + event is not generated. + This must be disabled for hardware validation purposes to detect any + hardware anomalies of missing events. + config ARM_GLOBAL_TIMER bool select CLKSRC_OF if OF diff --git a/drivers/clocksource/arm_arch_timer.c b/drivers/clocksource/arm_arch_timer.c index ce98d5e70927..b94b0d44c158 100644 --- a/drivers/clocksource/arm_arch_timer.c +++ b/drivers/clocksource/arm_arch_timer.c @@ -13,6 +13,7 @@ #include <linux/device.h> #include <linux/smp.h> #include <linux/cpu.h> +#include <linux/cpu_pm.h> #include <linux/clockchips.h> #include <linux/interrupt.h> #include <linux/of_irq.h> @@ -294,6 +295,19 @@ static void __arch_timer_setup(unsigned type, clockevents_config_and_register(clk, arch_timer_rate, 0xf, 0x7fffffff); } +static void arch_timer_configure_evtstream(void) +{ + int evt_stream_div, pos; + + /* Find the closest power of two to the divisor */ + evt_stream_div = arch_timer_rate / ARCH_TIMER_EVT_STREAM_FREQ; + pos = fls(evt_stream_div); + if (pos > 1 && !(evt_stream_div & (1 << (pos - 2)))) + pos--; + /* enable event stream */ + arch_timer_evtstrm_enable(min(pos, 15)); +} + static int arch_timer_setup(struct clock_event_device *clk) { __arch_timer_setup(ARCH_CP15_TIMER, clk); @@ -307,6 +321,8 @@ static int arch_timer_setup(struct clock_event_device *clk) } arch_counter_set_user_access(); + if (IS_ENABLED(CONFIG_ARM_ARCH_TIMER_EVTSTREAM)) + arch_timer_configure_evtstream(); return 0; } @@ -460,6 +476,33 @@ static struct notifier_block arch_timer_cpu_nb = { .notifier_call = arch_timer_cpu_notify, }; +#ifdef CONFIG_CPU_PM +static unsigned int saved_cntkctl; +static int arch_timer_cpu_pm_notify(struct notifier_block *self, + unsigned long action, void *hcpu) +{ + if (action == CPU_PM_ENTER) + saved_cntkctl = arch_timer_get_cntkctl(); + else if (action == CPU_PM_ENTER_FAILED || action == CPU_PM_EXIT) + arch_timer_set_cntkctl(saved_cntkctl); + return NOTIFY_OK; +} + +static struct notifier_block arch_timer_cpu_pm_notifier = { + .notifier_call = arch_timer_cpu_pm_notify, +}; + +static int __init arch_timer_cpu_pm_init(void) +{ + return cpu_pm_register_notifier(&arch_timer_cpu_pm_notifier); +} +#else +static int __init arch_timer_cpu_pm_init(void) +{ + return 0; +} +#endif + static int __init arch_timer_register(void) { int err; @@ -499,11 +542,17 @@ static int __init arch_timer_register(void) if (err) goto out_free_irq; + err = arch_timer_cpu_pm_init(); + if (err) + goto out_unreg_notify; + /* Immediately configure the timer on the boot CPU */ arch_timer_setup(this_cpu_ptr(arch_timer_evt)); return 0; +out_unreg_notify: + unregister_cpu_notifier(&arch_timer_cpu_nb); out_free_irq: if (arch_timer_use_virtual) free_percpu_irq(arch_timer_ppi[VIRT_PPI], arch_timer_evt); diff --git a/drivers/clocksource/tcb_clksrc.c b/drivers/clocksource/tcb_clksrc.c index 8a6187225dd0..00fdd1170284 100644 --- a/drivers/clocksource/tcb_clksrc.c +++ b/drivers/clocksource/tcb_clksrc.c @@ -100,7 +100,7 @@ static void tc_mode(enum clock_event_mode m, struct clock_event_device *d) || tcd->clkevt.mode == CLOCK_EVT_MODE_ONESHOT) { __raw_writel(0xff, regs + ATMEL_TC_REG(2, IDR)); __raw_writel(ATMEL_TC_CLKDIS, regs + ATMEL_TC_REG(2, CCR)); - clk_disable(tcd->clk); + clk_disable_unprepare(tcd->clk); } switch (m) { @@ -109,7 +109,7 @@ static void tc_mode(enum clock_event_mode m, struct clock_event_device *d) * of oneshot, we get lower overhead and improved accuracy. */ case CLOCK_EVT_MODE_PERIODIC: - clk_enable(tcd->clk); + clk_prepare_enable(tcd->clk); /* slow clock, count up to RC, then irq and restart */ __raw_writel(timer_clock @@ -126,7 +126,7 @@ static void tc_mode(enum clock_event_mode m, struct clock_event_device *d) break; case CLOCK_EVT_MODE_ONESHOT: - clk_enable(tcd->clk); + clk_prepare_enable(tcd->clk); /* slow clock, count up to RC, then irq and stop */ __raw_writel(timer_clock | ATMEL_TC_CPCSTOP @@ -180,15 +180,22 @@ static irqreturn_t ch2_irq(int irq, void *handle) static struct irqaction tc_irqaction = { .name = "tc_clkevt", - .flags = IRQF_TIMER | IRQF_DISABLED, + .flags = IRQF_TIMER, .handler = ch2_irq, }; -static void __init setup_clkevents(struct atmel_tc *tc, int clk32k_divisor_idx) +static int __init setup_clkevents(struct atmel_tc *tc, int clk32k_divisor_idx) { + int ret; struct clk *t2_clk = tc->clk[2]; int irq = tc->irq[2]; + /* try to enable t2 clk to avoid future errors in mode change */ + ret = clk_prepare_enable(t2_clk); + if (ret) + return ret; + clk_disable_unprepare(t2_clk); + clkevt.regs = tc->regs; clkevt.clk = t2_clk; tc_irqaction.dev_id = &clkevt; @@ -197,16 +204,21 @@ static void __init setup_clkevents(struct atmel_tc *tc, int clk32k_divisor_idx) clkevt.clkevt.cpumask = cpumask_of(0); + ret = setup_irq(irq, &tc_irqaction); + if (ret) + return ret; + clockevents_config_and_register(&clkevt.clkevt, 32768, 1, 0xffff); - setup_irq(irq, &tc_irqaction); + return ret; } #else /* !CONFIG_GENERIC_CLOCKEVENTS */ -static void __init setup_clkevents(struct atmel_tc *tc, int clk32k_divisor_idx) +static int __init setup_clkevents(struct atmel_tc *tc, int clk32k_divisor_idx) { /* NOTHING */ + return 0; } #endif @@ -265,6 +277,7 @@ static int __init tcb_clksrc_init(void) int best_divisor_idx = -1; int clk32k_divisor_idx = -1; int i; + int ret; tc = atmel_tc_alloc(CONFIG_ATMEL_TCB_CLKSRC_BLOCK, clksrc.name); if (!tc) { @@ -275,7 +288,11 @@ static int __init tcb_clksrc_init(void) pdev = tc->pdev; t0_clk = tc->clk[0]; - clk_enable(t0_clk); + ret = clk_prepare_enable(t0_clk); + if (ret) { + pr_debug("can't enable T0 clk\n"); + goto err_free_tc; + } /* How fast will we be counting? Pick something over 5 MHz. */ rate = (u32) clk_get_rate(t0_clk); @@ -313,17 +330,39 @@ static int __init tcb_clksrc_init(void) /* tclib will give us three clocks no matter what the * underlying platform supports. */ - clk_enable(tc->clk[1]); + ret = clk_prepare_enable(tc->clk[1]); + if (ret) { + pr_debug("can't enable T1 clk\n"); + goto err_disable_t0; + } /* setup both channel 0 & 1 */ tcb_setup_dual_chan(tc, best_divisor_idx); } /* and away we go! */ - clocksource_register_hz(&clksrc, divided_rate); + ret = clocksource_register_hz(&clksrc, divided_rate); + if (ret) + goto err_disable_t1; /* channel 2: periodic and oneshot timer support */ - setup_clkevents(tc, clk32k_divisor_idx); + ret = setup_clkevents(tc, clk32k_divisor_idx); + if (ret) + goto err_unregister_clksrc; return 0; + +err_unregister_clksrc: + clocksource_unregister(&clksrc); + +err_disable_t1: + if (!tc->tcb_config || tc->tcb_config->counter_width != 32) + clk_disable_unprepare(tc->clk[1]); + +err_disable_t0: + clk_disable_unprepare(t0_clk); + +err_free_tc: + atmel_tc_free(tc); + return ret; } arch_initcall(tcb_clksrc_init); diff --git a/include/clocksource/arm_arch_timer.h b/include/clocksource/arm_arch_timer.h index 93b7f96f9c59..6d26b40cbf5d 100644 --- a/include/clocksource/arm_arch_timer.h +++ b/include/clocksource/arm_arch_timer.h @@ -33,6 +33,16 @@ enum arch_timer_reg { #define ARCH_TIMER_MEM_PHYS_ACCESS 2 #define ARCH_TIMER_MEM_VIRT_ACCESS 3 +#define ARCH_TIMER_USR_PCT_ACCESS_EN (1 << 0) /* physical counter */ +#define ARCH_TIMER_USR_VCT_ACCESS_EN (1 << 1) /* virtual counter */ +#define ARCH_TIMER_VIRT_EVT_EN (1 << 2) +#define ARCH_TIMER_EVT_TRIGGER_SHIFT (4) +#define ARCH_TIMER_EVT_TRIGGER_MASK (0xF << ARCH_TIMER_EVT_TRIGGER_SHIFT) +#define ARCH_TIMER_USR_VT_ACCESS_EN (1 << 8) /* virtual timer registers */ +#define ARCH_TIMER_USR_PT_ACCESS_EN (1 << 9) /* physical timer registers */ + +#define ARCH_TIMER_EVT_STREAM_FREQ 10000 /* 100us */ + #ifdef CONFIG_ARM_ARCH_TIMER extern u32 arch_timer_get_rate(void); |