diff options
Diffstat (limited to 'arch/sparc')
105 files changed, 919 insertions, 844 deletions
diff --git a/arch/sparc/Kconfig b/arch/sparc/Kconfig index 546293d9e6c5..b23c76b42d6e 100644 --- a/arch/sparc/Kconfig +++ b/arch/sparc/Kconfig @@ -22,7 +22,6 @@ config SPARC select HAVE_ARCH_TRACEHOOK select HAVE_EXIT_THREAD select SYSCTL_EXCEPTION_TRACE - select ARCH_HAS_ATOMIC64_DEC_IF_POSITIVE select RTC_CLASS select RTC_DRV_M48T59 select RTC_SYSTOHC @@ -43,6 +42,7 @@ config SPARC select OLD_SIGSUSPEND select ARCH_HAS_SG_CHAIN select CPU_NO_EFFICIENT_FFS + select HAVE_ARCH_HARDENED_USERCOPY config SPARC32 def_bool !64BIT @@ -55,7 +55,6 @@ config SPARC64 def_bool 64BIT select HAVE_FUNCTION_TRACER select HAVE_FUNCTION_GRAPH_TRACER - select HAVE_FUNCTION_GRAPH_FP_TEST select HAVE_KRETPROBES select HAVE_KPROBES select HAVE_RCU_TABLE_FREE if SMP diff --git a/arch/sparc/include/asm/Kbuild b/arch/sparc/include/asm/Kbuild index 6024c26c0585..cfc918067f80 100644 --- a/arch/sparc/include/asm/Kbuild +++ b/arch/sparc/include/asm/Kbuild @@ -6,6 +6,7 @@ generic-y += cputime.h generic-y += div64.h generic-y += emergency-restart.h generic-y += exec.h +generic-y += export.h generic-y += irq_regs.h generic-y += irq_work.h generic-y += linkage.h diff --git a/arch/sparc/include/asm/atomic_32.h b/arch/sparc/include/asm/atomic_32.h index 7dcbebbcaec6..ee3f11c43cda 100644 --- a/arch/sparc/include/asm/atomic_32.h +++ b/arch/sparc/include/asm/atomic_32.h @@ -20,9 +20,10 @@ #define ATOMIC_INIT(i) { (i) } int atomic_add_return(int, atomic_t *); -void atomic_and(int, atomic_t *); -void atomic_or(int, atomic_t *); -void atomic_xor(int, atomic_t *); +int atomic_fetch_add(int, atomic_t *); +int atomic_fetch_and(int, atomic_t *); +int atomic_fetch_or(int, atomic_t *); +int atomic_fetch_xor(int, atomic_t *); int atomic_cmpxchg(atomic_t *, int, int); int atomic_xchg(atomic_t *, int); int __atomic_add_unless(atomic_t *, int, int); @@ -35,7 +36,13 @@ void atomic_set(atomic_t *, int); #define atomic_inc(v) ((void)atomic_add_return( 1, (v))) #define atomic_dec(v) ((void)atomic_add_return( -1, (v))) +#define atomic_and(i, v) ((void)atomic_fetch_and((i), (v))) +#define atomic_or(i, v) ((void)atomic_fetch_or((i), (v))) +#define atomic_xor(i, v) ((void)atomic_fetch_xor((i), (v))) + #define atomic_sub_return(i, v) (atomic_add_return(-(int)(i), (v))) +#define atomic_fetch_sub(i, v) (atomic_fetch_add (-(int)(i), (v))) + #define atomic_inc_return(v) (atomic_add_return( 1, (v))) #define atomic_dec_return(v) (atomic_add_return( -1, (v))) diff --git a/arch/sparc/include/asm/atomic_64.h b/arch/sparc/include/asm/atomic_64.h index f2fbf9e16faf..24827a3f733a 100644 --- a/arch/sparc/include/asm/atomic_64.h +++ b/arch/sparc/include/asm/atomic_64.h @@ -28,16 +28,24 @@ void atomic64_##op(long, atomic64_t *); int atomic_##op##_return(int, atomic_t *); \ long atomic64_##op##_return(long, atomic64_t *); -#define ATOMIC_OPS(op) ATOMIC_OP(op) ATOMIC_OP_RETURN(op) +#define ATOMIC_FETCH_OP(op) \ +int atomic_fetch_##op(int, atomic_t *); \ +long atomic64_fetch_##op(long, atomic64_t *); + +#define ATOMIC_OPS(op) ATOMIC_OP(op) ATOMIC_OP_RETURN(op) ATOMIC_FETCH_OP(op) ATOMIC_OPS(add) ATOMIC_OPS(sub) -ATOMIC_OP(and) -ATOMIC_OP(or) -ATOMIC_OP(xor) +#undef ATOMIC_OPS +#define ATOMIC_OPS(op) ATOMIC_OP(op) ATOMIC_FETCH_OP(op) + +ATOMIC_OPS(and) +ATOMIC_OPS(or) +ATOMIC_OPS(xor) #undef ATOMIC_OPS +#undef ATOMIC_FETCH_OP #undef ATOMIC_OP_RETURN #undef ATOMIC_OP diff --git a/arch/sparc/include/asm/elf_64.h b/arch/sparc/include/asm/elf_64.h index 93310837c2df..3f2d403873bd 100644 --- a/arch/sparc/include/asm/elf_64.h +++ b/arch/sparc/include/asm/elf_64.h @@ -7,7 +7,7 @@ #include <asm/ptrace.h> #include <asm/processor.h> -#include <asm/uaccess.h> +#include <asm/extable_64.h> #include <asm/spitfire.h> /* diff --git a/arch/sparc/include/asm/extable_64.h b/arch/sparc/include/asm/extable_64.h new file mode 100644 index 000000000000..1121cb056ffb --- /dev/null +++ b/arch/sparc/include/asm/extable_64.h @@ -0,0 +1,20 @@ +#ifndef __ASM_EXTABLE64_H +#define __ASM_EXTABLE64_H +/* + * The exception table consists of pairs of addresses: the first is the + * address of an instruction that is allowed to fault, and the second is + * the address at which the program should continue. No registers are + * modified, so it is entirely up to the continuation code to figure out + * what to do. + * + * All the routines below use bits of fixup code that are out of line + * with the main instruction path. This means when everything is well, + * we don't even have to jump over them. Further, they do not intrude + * on our cache or tlb entries. + */ + +struct exception_table_entry { + unsigned int insn, fixup; +}; + +#endif diff --git a/arch/sparc/include/asm/ftrace.h b/arch/sparc/include/asm/ftrace.h index 3192a8e42fd6..62755a339a59 100644 --- a/arch/sparc/include/asm/ftrace.h +++ b/arch/sparc/include/asm/ftrace.h @@ -9,6 +9,10 @@ void _mcount(void); #endif +#endif /* CONFIG_MCOUNT */ + +#if defined(CONFIG_SPARC64) && !defined(CC_USE_FENTRY) +#define HAVE_FUNCTION_GRAPH_FP_TEST #endif #ifdef CONFIG_DYNAMIC_FTRACE diff --git a/arch/sparc/include/asm/head_64.h b/arch/sparc/include/asm/head_64.h index 10e9dabc4c41..f0700cfeedd7 100644 --- a/arch/sparc/include/asm/head_64.h +++ b/arch/sparc/include/asm/head_64.h @@ -15,6 +15,10 @@ #define PTREGS_OFF (STACK_BIAS + STACKFRAME_SZ) +#define RTRAP_PSTATE (PSTATE_TSO|PSTATE_PEF|PSTATE_PRIV|PSTATE_IE) +#define RTRAP_PSTATE_IRQOFF (PSTATE_TSO|PSTATE_PEF|PSTATE_PRIV) +#define RTRAP_PSTATE_AG_IRQOFF (PSTATE_TSO|PSTATE_PEF|PSTATE_PRIV|PSTATE_AG) + #define __CHEETAH_ID 0x003e0014 #define __JALAPENO_ID 0x003e0016 #define __SERRANO_ID 0x003e0022 diff --git a/arch/sparc/include/asm/hugetlb.h b/arch/sparc/include/asm/hugetlb.h index 139e711ff80c..dcbf985ab243 100644 --- a/arch/sparc/include/asm/hugetlb.h +++ b/arch/sparc/include/asm/hugetlb.h @@ -31,14 +31,6 @@ static inline int prepare_hugepage_range(struct file *file, return 0; } -static inline void hugetlb_free_pgd_range(struct mmu_gather *tlb, - unsigned long addr, unsigned long end, - unsigned long floor, - unsigned long ceiling) -{ - free_pgd_range(tlb, addr, end, floor, ceiling); -} - static inline void huge_ptep_clear_flush(struct vm_area_struct *vma, unsigned long addr, pte_t *ptep) { @@ -82,4 +74,8 @@ static inline void arch_clear_hugepage_flags(struct page *page) { } +void hugetlb_free_pgd_range(struct mmu_gather *tlb, unsigned long addr, + unsigned long end, unsigned long floor, + unsigned long ceiling); + #endif /* _ASM_SPARC64_HUGETLB_H */ diff --git a/arch/sparc/include/asm/hypervisor.h b/arch/sparc/include/asm/hypervisor.h index f5b6537306f0..666d5ba230d2 100644 --- a/arch/sparc/include/asm/hypervisor.h +++ b/arch/sparc/include/asm/hypervisor.h @@ -1744,6 +1744,7 @@ unsigned long sun4v_vintr_set_target(unsigned long dev_handle, #define HV_PCI_MAP_ATTR_READ 0x01 #define HV_PCI_MAP_ATTR_WRITE 0x02 +#define HV_PCI_MAP_ATTR_RELAXED_ORDER 0x04 #define HV_PCI_DEVICE_BUILD(b,d,f) \ ((((b) & 0xff) << 16) | \ diff --git a/arch/sparc/include/asm/io_32.h b/arch/sparc/include/asm/io_32.h index 57f26c398dc9..4dd268a3a8b0 100644 --- a/arch/sparc/include/asm/io_32.h +++ b/arch/sparc/include/asm/io_32.h @@ -140,16 +140,6 @@ void ioport_unmap(void __iomem *); struct pci_dev; void pci_iounmap(struct pci_dev *dev, void __iomem *); - - -/* - * At the moment, we do not use CMOS_READ anywhere outside of rtc.c, - * so rtc_port is static in it. This should not change unless a new - * hardware pops up. - */ -#define RTC_PORT(x) (rtc_port + (x)) -#define RTC_ALWAYS_BCD 0 - static inline int sbus_can_dma_64bit(void) { return 0; /* actually, sparc_cpu_model==sun4d */ diff --git a/arch/sparc/include/asm/irq_64.h b/arch/sparc/include/asm/irq_64.h index 3f70f900e834..1d51a11fb261 100644 --- a/arch/sparc/include/asm/irq_64.h +++ b/arch/sparc/include/asm/irq_64.h @@ -86,8 +86,9 @@ static inline unsigned long get_softint(void) return retval; } -void arch_trigger_all_cpu_backtrace(bool); -#define arch_trigger_all_cpu_backtrace arch_trigger_all_cpu_backtrace +void arch_trigger_cpumask_backtrace(const struct cpumask *mask, + bool exclude_self); +#define arch_trigger_cpumask_backtrace arch_trigger_cpumask_backtrace extern void *hardirq_stack[NR_CPUS]; extern void *softirq_stack[NR_CPUS]; diff --git a/arch/sparc/include/asm/mmu_64.h b/arch/sparc/include/asm/mmu_64.h index 70067ce184b1..f7de0dbc38af 100644 --- a/arch/sparc/include/asm/mmu_64.h +++ b/arch/sparc/include/asm/mmu_64.h @@ -92,7 +92,8 @@ struct tsb_config { typedef struct { spinlock_t lock; unsigned long sparc64_ctx_val; - unsigned long huge_pte_count; + unsigned long hugetlb_pte_count; + unsigned long thp_pte_count; struct tsb_config tsb_block[MM_NUM_TSBS]; struct hv_tsb_descr tsb_descr[MM_NUM_TSBS]; } mm_context_t; diff --git a/arch/sparc/include/asm/page_64.h b/arch/sparc/include/asm/page_64.h index 8c2a8c937540..c1263fc390db 100644 --- a/arch/sparc/include/asm/page_64.h +++ b/arch/sparc/include/asm/page_64.h @@ -25,6 +25,7 @@ #define HPAGE_MASK (~(HPAGE_SIZE - 1UL)) #define HUGETLB_PAGE_ORDER (HPAGE_SHIFT - PAGE_SHIFT) #define HAVE_ARCH_HUGETLB_UNMAPPED_AREA +#define REAL_HPAGE_PER_HPAGE (_AC(1,UL) << (HPAGE_SHIFT - REAL_HPAGE_SHIFT)) #endif #ifndef __ASSEMBLY__ diff --git a/arch/sparc/include/asm/pci_64.h b/arch/sparc/include/asm/pci_64.h index 022d16008a00..2303635158f5 100644 --- a/arch/sparc/include/asm/pci_64.h +++ b/arch/sparc/include/asm/pci_64.h @@ -55,9 +55,6 @@ static inline int pci_get_legacy_ide_irq(struct pci_dev *dev, int channel) } #define HAVE_ARCH_PCI_RESOURCE_TO_USER -void pci_resource_to_user(const struct pci_dev *dev, int bar, - const struct resource *rsrc, - resource_size_t *start, resource_size_t *end); #endif /* __KERNEL__ */ #endif /* __SPARC64_PCI_H */ diff --git a/arch/sparc/include/asm/pgalloc_64.h b/arch/sparc/include/asm/pgalloc_64.h index 5e3187185b4a..3529f1378cd8 100644 --- a/arch/sparc/include/asm/pgalloc_64.h +++ b/arch/sparc/include/asm/pgalloc_64.h @@ -41,8 +41,7 @@ static inline void __pud_populate(pud_t *pud, pmd_t *pmd) static inline pud_t *pud_alloc_one(struct mm_struct *mm, unsigned long addr) { - return kmem_cache_alloc(pgtable_cache, - GFP_KERNEL|__GFP_REPEAT); + return kmem_cache_alloc(pgtable_cache, GFP_KERNEL); } static inline void pud_free(struct mm_struct *mm, pud_t *pud) @@ -52,8 +51,7 @@ static inline void pud_free(struct mm_struct *mm, pud_t *pud) static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long addr) { - return kmem_cache_alloc(pgtable_cache, - GFP_KERNEL|__GFP_REPEAT); + return kmem_cache_alloc(pgtable_cache, GFP_KERNEL); } static inline void pmd_free(struct mm_struct *mm, pmd_t *pmd) diff --git a/arch/sparc/include/asm/pgtable_64.h b/arch/sparc/include/asm/pgtable_64.h index e7d82803a48f..1fb317fbc0b3 100644 --- a/arch/sparc/include/asm/pgtable_64.h +++ b/arch/sparc/include/asm/pgtable_64.h @@ -395,7 +395,7 @@ static inline unsigned long __pte_huge_mask(void) static inline pte_t pte_mkhuge(pte_t pte) { - return __pte(pte_val(pte) | __pte_huge_mask()); + return __pte(pte_val(pte) | _PAGE_PMD_HUGE | __pte_huge_mask()); } static inline bool is_hugetlb_pte(pte_t pte) @@ -403,6 +403,11 @@ static inline bool is_hugetlb_pte(pte_t pte) return !!(pte_val(pte) & __pte_huge_mask()); } +static inline bool is_hugetlb_pmd(pmd_t pmd) +{ + return !!(pmd_val(pmd) & _PAGE_PMD_HUGE); +} + #ifdef CONFIG_TRANSPARENT_HUGEPAGE static inline pmd_t pmd_mkhuge(pmd_t pmd) { diff --git a/arch/sparc/include/asm/smp_64.h b/arch/sparc/include/asm/smp_64.h index 26d9e7726867..ce2233f7e662 100644 --- a/arch/sparc/include/asm/smp_64.h +++ b/arch/sparc/include/asm/smp_64.h @@ -43,6 +43,7 @@ void arch_send_call_function_ipi_mask(const struct cpumask *mask); int hard_smp_processor_id(void); #define raw_smp_processor_id() (current_thread_info()->cpu) +void smp_fill_in_cpu_possible_map(void); void smp_fill_in_sib_core_maps(void); void cpu_play_dead(void); @@ -72,6 +73,7 @@ void __cpu_die(unsigned int cpu); #define smp_fill_in_sib_core_maps() do { } while (0) #define smp_fetch_global_regs() do { } while (0) #define smp_fetch_global_pmu() do { } while (0) +#define smp_fill_in_cpu_possible_map() do { } while (0) #endif /* !(CONFIG_SMP) */ diff --git a/arch/sparc/include/asm/spinlock_32.h b/arch/sparc/include/asm/spinlock_32.h index bcc98fc35281..d9c5876c6121 100644 --- a/arch/sparc/include/asm/spinlock_32.h +++ b/arch/sparc/include/asm/spinlock_32.h @@ -9,12 +9,15 @@ #ifndef __ASSEMBLY__ #include <asm/psr.h> +#include <asm/barrier.h> #include <asm/processor.h> /* for cpu_relax */ #define arch_spin_is_locked(lock) (*((volatile unsigned char *)(lock)) != 0) -#define arch_spin_unlock_wait(lock) \ - do { while (arch_spin_is_locked(lock)) cpu_relax(); } while (0) +static inline void arch_spin_unlock_wait(arch_spinlock_t *lock) +{ + smp_cond_load_acquire(&lock->lock, !VAL); +} static inline void arch_spin_lock(arch_spinlock_t *lock) { diff --git a/arch/sparc/include/asm/spinlock_64.h b/arch/sparc/include/asm/spinlock_64.h index 968917694978..87990b7c6b0d 100644 --- a/arch/sparc/include/asm/spinlock_64.h +++ b/arch/sparc/include/asm/spinlock_64.h @@ -8,6 +8,9 @@ #ifndef __ASSEMBLY__ +#include <asm/processor.h> +#include <asm/barrier.h> + /* To get debugging spinlocks which detect and catch * deadlock situations, set CONFIG_DEBUG_SPINLOCK * and rebuild your kernel. @@ -23,9 +26,10 @@ #define arch_spin_is_locked(lp) ((lp)->lock != 0) -#define arch_spin_unlock_wait(lp) \ - do { rmb(); \ - } while((lp)->lock) +static inline void arch_spin_unlock_wait(arch_spinlock_t *lock) +{ + smp_cond_load_acquire(&lock->lock, !VAL); +} static inline void arch_spin_lock(arch_spinlock_t *lock) { diff --git a/arch/sparc/include/asm/string.h b/arch/sparc/include/asm/string.h index 98b72a0c8e6e..86f34be14ce0 100644 --- a/arch/sparc/include/asm/string.h +++ b/arch/sparc/include/asm/string.h @@ -5,4 +5,38 @@ #else #include <asm/string_32.h> #endif + +/* First the mem*() things. */ +#define __HAVE_ARCH_MEMMOVE +void *memmove(void *, const void *, __kernel_size_t); + +#define __HAVE_ARCH_MEMCPY +#define memcpy(t, f, n) __builtin_memcpy(t, f, n) + +#define __HAVE_ARCH_MEMSET +#define memset(s, c, count) __builtin_memset(s, c, count) + +#define __HAVE_ARCH_MEMSCAN + +#define memscan(__arg0, __char, __arg2) \ +({ \ + void *__memscan_zero(void *, size_t); \ + void *__memscan_generic(void *, int, size_t); \ + void *__retval, *__addr = (__arg0); \ + size_t __size = (__arg2); \ + \ + if(__builtin_constant_p(__char) && !(__char)) \ + __retval = __memscan_zero(__addr, __size); \ + else \ + __retval = __memscan_generic(__addr, (__char), __size); \ + \ + __retval; \ +}) + +#define __HAVE_ARCH_MEMCMP +int memcmp(const void *,const void *,__kernel_size_t); + +#define __HAVE_ARCH_STRNCMP +int strncmp(const char *, const char *, __kernel_size_t); + #endif diff --git a/arch/sparc/include/asm/string_32.h b/arch/sparc/include/asm/string_32.h index 69974e924611..649412476a69 100644 --- a/arch/sparc/include/asm/string_32.h +++ b/arch/sparc/include/asm/string_32.h @@ -11,60 +11,4 @@ #include <asm/page.h> -/* Really, userland/ksyms should not see any of this stuff. */ - -#ifdef __KERNEL__ - -void __memmove(void *,const void *,__kernel_size_t); - -#ifndef EXPORT_SYMTAB_STROPS - -/* First the mem*() things. */ -#define __HAVE_ARCH_MEMMOVE -#undef memmove -#define memmove(_to, _from, _n) \ -({ \ - void *_t = (_to); \ - __memmove(_t, (_from), (_n)); \ - _t; \ -}) - -#define __HAVE_ARCH_MEMCPY -#define memcpy(t, f, n) __builtin_memcpy(t, f, n) - -#define __HAVE_ARCH_MEMSET -#define memset(s, c, count) __builtin_memset(s, c, count) - -#define __HAVE_ARCH_MEMSCAN - -#undef memscan -#define memscan(__arg0, __char, __arg2) \ -({ \ - void *__memscan_zero(void *, size_t); \ - void *__memscan_generic(void *, int, size_t); \ - void *__retval, *__addr = (__arg0); \ - size_t __size = (__arg2); \ - \ - if(__builtin_constant_p(__char) && !(__char)) \ - __retval = __memscan_zero(__addr, __size); \ - else \ - __retval = __memscan_generic(__addr, (__char), __size); \ - \ - __retval; \ -}) - -#define __HAVE_ARCH_MEMCMP -int memcmp(const void *,const void *,__kernel_size_t); - -/* Now the str*() stuff... */ -#define __HAVE_ARCH_STRLEN -__kernel_size_t strlen(const char *); - -#define __HAVE_ARCH_STRNCMP -int strncmp(const char *, const char *, __kernel_size_t); - -#endif /* !EXPORT_SYMTAB_STROPS */ - -#endif /* __KERNEL__ */ - #endif /* !(__SPARC_STRING_H__) */ diff --git a/arch/sparc/include/asm/string_64.h b/arch/sparc/include/asm/string_64.h index 5936b8ff3c05..6b9ccb308605 100644 --- a/arch/sparc/include/asm/string_64.h +++ b/arch/sparc/include/asm/string_64.h @@ -9,54 +9,10 @@ #ifndef __SPARC64_STRING_H__ #define __SPARC64_STRING_H__ -/* Really, userland/ksyms should not see any of this stuff. */ - -#ifdef __KERNEL__ - #include <asm/asi.h> -#ifndef EXPORT_SYMTAB_STROPS - -/* First the mem*() things. */ -#define __HAVE_ARCH_MEMMOVE -void *memmove(void *, const void *, __kernel_size_t); - -#define __HAVE_ARCH_MEMCPY -#define memcpy(t, f, n) __builtin_memcpy(t, f, n) - -#define __HAVE_ARCH_MEMSET -#define memset(s, c, count) __builtin_memset(s, c, count) - -#define __HAVE_ARCH_MEMSCAN - -#undef memscan -#define memscan(__arg0, __char, __arg2) \ -({ \ - void *__memscan_zero(void *, size_t); \ - void *__memscan_generic(void *, int, size_t); \ - void *__retval, *__addr = (__arg0); \ - size_t __size = (__arg2); \ - \ - if(__builtin_constant_p(__char) && !(__char)) \ - __retval = __memscan_zero(__addr, __size); \ - else \ - __retval = __memscan_generic(__addr, (__char), __size); \ - \ - __retval; \ -}) - -#define __HAVE_ARCH_MEMCMP -int memcmp(const void *,const void *,__kernel_size_t); - /* Now the str*() stuff... */ #define __HAVE_ARCH_STRLEN __kernel_size_t strlen(const char *); -#define __HAVE_ARCH_STRNCMP -int strncmp(const char *, const char *, __kernel_size_t); - -#endif /* !EXPORT_SYMTAB_STROPS */ - -#endif /* __KERNEL__ */ - #endif /* !(__SPARC64_STRING_H__) */ diff --git a/arch/sparc/include/asm/thread_info_64.h b/arch/sparc/include/asm/thread_info_64.h index bde59825d06c..3d7b925f6516 100644 --- a/arch/sparc/include/asm/thread_info_64.h +++ b/arch/sparc/include/asm/thread_info_64.h @@ -222,32 +222,8 @@ register struct thread_info *current_thread_info_reg asm("g6"); * * Note that there are only 8 bits available. */ -#define TS_RESTORE_SIGMASK 0x0001 /* restore signal mask in do_signal() */ #ifndef __ASSEMBLY__ -#define HAVE_SET_RESTORE_SIGMASK 1 -static inline void set_restore_sigmask(void) -{ - struct thread_info *ti = current_thread_info(); - ti->status |= TS_RESTORE_SIGMASK; - WARN_ON(!test_bit(TIF_SIGPENDING, &ti->flags)); -} -static inline void clear_restore_sigmask(void) -{ - current_thread_info()->status &= ~TS_RESTORE_SIGMASK; -} -static inline bool test_restore_sigmask(void) -{ - return current_thread_info()->status & TS_RESTORE_SIGMASK; -} -static inline bool test_and_clear_restore_sigmask(void) -{ - struct thread_info *ti = current_thread_info(); - if (!(ti->status & TS_RESTORE_SIGMASK)) - return false; - ti->status &= ~TS_RESTORE_SIGMASK; - return true; -} #define thread32_stack_is_64bit(__SP) (((__SP) & 0x1) != 0) #define test_thread_64bit_stack(__SP) \ diff --git a/arch/sparc/include/asm/tsb.h b/arch/sparc/include/asm/tsb.h index c6a155c3904e..32258e08da03 100644 --- a/arch/sparc/include/asm/tsb.h +++ b/arch/sparc/include/asm/tsb.h @@ -203,7 +203,7 @@ extern struct tsb_phys_patch_entry __tsb_phys_patch, __tsb_phys_patch_end; * We have to propagate the 4MB bit of the virtual address * because we are fabricating 8MB pages using 4MB hw pages. */ -#ifdef CONFIG_TRANSPARENT_HUGEPAGE +#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE) #define USER_PGTABLE_CHECK_PMD_HUGE(VADDR, REG1, REG2, FAIL_LABEL, PTE_LABEL) \ brz,pn REG1, FAIL_LABEL; \ sethi %uhi(_PAGE_PMD_HUGE), REG2; \ diff --git a/arch/sparc/include/asm/ttable.h b/arch/sparc/include/asm/ttable.h index 71b5a67522ab..781b9f1dbdc2 100644 --- a/arch/sparc/include/asm/ttable.h +++ b/arch/sparc/include/asm/ttable.h @@ -589,8 +589,8 @@ user_rtt_fill_64bit: \ restored; \ nop; nop; nop; nop; nop; nop; \ nop; nop; nop; nop; nop; \ - ba,a,pt %xcc, user_rtt_fill_fixup; \ - ba,a,pt %xcc, user_rtt_fill_fixup; \ + ba,a,pt %xcc, user_rtt_fill_fixup_dax; \ + ba,a,pt %xcc, user_rtt_fill_fixup_mna; \ ba,a,pt %xcc, user_rtt_fill_fixup; @@ -652,8 +652,8 @@ user_rtt_fill_32bit: \ restored; \ nop; nop; nop; nop; nop; \ nop; nop; nop; \ - ba,a,pt %xcc, user_rtt_fill_fixup; \ - ba,a,pt %xcc, user_rtt_fill_fixup; \ + ba,a,pt %xcc, user_rtt_fill_fixup_dax; \ + ba,a,pt %xcc, user_rtt_fill_fixup_mna; \ ba,a,pt %xcc, user_rtt_fill_fixup; diff --git a/arch/sparc/include/asm/uaccess_32.h b/arch/sparc/include/asm/uaccess_32.h index 57aca2792d29..ea55f86d7ccd 100644 --- a/arch/sparc/include/asm/uaccess_32.h +++ b/arch/sparc/include/asm/uaccess_32.h @@ -248,23 +248,28 @@ unsigned long __copy_user(void __user *to, const void __user *from, unsigned lon static inline unsigned long copy_to_user(void __user *to, const void *from, unsigned long n) { - if (n && __access_ok((unsigned long) to, n)) + if (n && __access_ok((unsigned long) to, n)) { + check_object_size(from, n, true); return __copy_user(to, (__force void __user *) from, n); - else + } else return n; } static inline unsigned long __copy_to_user(void __user *to, const void *from, unsigned long n) { + check_object_size(from, n, true); return __copy_user(to, (__force void __user *) from, n); } static inline unsigned long copy_from_user(void *to, const void __user *from, unsigned long n) { - if (n && __access_ok((unsigned long) from, n)) + if (n && __access_ok((unsigned long) from, n)) { + check_object_size(to, n, false); return __copy_user((__force void __user *) to, from, n); - else + } else { + memset(to, 0, n); return n; + } } static inline unsigned long __copy_from_user(void *to, const void __user *from, unsigned long n) diff --git a/arch/sparc/include/asm/uaccess_64.h b/arch/sparc/include/asm/uaccess_64.h index e9a51d64974d..b68acc563235 100644 --- a/arch/sparc/include/asm/uaccess_64.h +++ b/arch/sparc/include/asm/uaccess_64.h @@ -13,6 +13,7 @@ #include <asm/asi.h> #include <asm/spitfire.h> #include <asm-generic/uaccess-unaligned.h> +#include <asm/extable_64.h> #endif #ifndef __ASSEMBLY__ @@ -81,23 +82,6 @@ static inline int access_ok(int type, const void __user * addr, unsigned long si return 1; } -/* - * The exception table consists of pairs of addresses: the first is the - * address of an instruction that is allowed to fault, and the second is - * the address at which the program should continue. No registers are - * modified, so it is entirely up to the continuation code to figure out - * what to do. - * - * All the routines below use bits of fixup code that are out of line - * with the main instruction path. This means when everything is well, - * we don't even have to jump over them. Further, they do not intrude - * on our cache or tlb entries. - */ - -struct exception_table_entry { - unsigned int insn, fixup; -}; - void __ret_efault(void); void __retl_efault(void); @@ -210,8 +194,11 @@ unsigned long copy_from_user_fixup(void *to, const void __user *from, static inline unsigned long __must_check copy_from_user(void *to, const void __user *from, unsigned long size) { - unsigned long ret = ___copy_from_user(to, from, size); + unsigned long ret; + + check_object_size(to, size, false); + ret = ___copy_from_user(to, from, size); if (unlikely(ret)) ret = copy_from_user_fixup(to, from, size); @@ -227,8 +214,11 @@ unsigned long copy_to_user_fixup(void __user *to, const void *from, static inline unsigned long __must_check copy_to_user(void __user *to, const void *from, unsigned long size) { - unsigned long ret = ___copy_to_user(to, from, size); + unsigned long ret; + + check_object_size(from, size, true); + ret = ___copy_to_user(to, from, size); if (unlikely(ret)) ret = copy_to_user_fixup(to, from, size); return ret; diff --git a/arch/sparc/kernel/Makefile b/arch/sparc/kernel/Makefile index 7cf9c6ea3f1f..fa3c02d41138 100644 --- a/arch/sparc/kernel/Makefile +++ b/arch/sparc/kernel/Makefile @@ -21,6 +21,7 @@ CFLAGS_REMOVE_perf_event.o := -pg CFLAGS_REMOVE_pcr.o := -pg endif +obj-$(CONFIG_SPARC64) += urtt_fill.o obj-$(CONFIG_SPARC32) += entry.o wof.o wuf.o obj-$(CONFIG_SPARC32) += etrap_32.o obj-$(CONFIG_SPARC32) += rtrap_32.o @@ -85,7 +86,7 @@ obj-y += auxio_$(BITS).o obj-$(CONFIG_SUN_PM) += apc.o pmc.o obj-$(CONFIG_MODULES) += module.o -obj-$(CONFIG_MODULES) += sparc_ksyms_$(BITS).o +obj-$(CONFIG_MODULES) += sparc_ksyms.o obj-$(CONFIG_SPARC_LED) += led.o obj-$(CONFIG_KGDB) += kgdb_$(BITS).o diff --git a/arch/sparc/kernel/dtlb_prot.S b/arch/sparc/kernel/dtlb_prot.S index d668ca149e64..4087a62f96b0 100644 --- a/arch/sparc/kernel/dtlb_prot.S +++ b/arch/sparc/kernel/dtlb_prot.S @@ -25,13 +25,13 @@ /* PROT ** ICACHE line 2: More real fault processing */ ldxa [%g4] ASI_DMMU, %g5 ! Put tagaccess in %g5 + srlx %g5, PAGE_SHIFT, %g5 + sllx %g5, PAGE_SHIFT, %g5 ! Clear context ID bits bgu,pn %xcc, winfix_trampoline ! Yes, perform winfixup mov FAULT_CODE_DTLB | FAULT_CODE_WRITE, %g4 ba,pt %xcc, sparc64_realfault_common ! Nope, normal fault nop nop - nop - nop /* PROT ** ICACHE line 3: Unused... */ nop diff --git a/arch/sparc/kernel/entry.S b/arch/sparc/kernel/entry.S index 07918ab3062e..d85bdb999819 100644 --- a/arch/sparc/kernel/entry.S +++ b/arch/sparc/kernel/entry.S @@ -29,6 +29,7 @@ #include <asm/unistd.h> #include <asm/asmmacro.h> +#include <asm/export.h> #define curptr g6 @@ -1207,6 +1208,8 @@ delay_continue: ret restore +EXPORT_SYMBOL(__udelay) +EXPORT_SYMBOL(__ndelay) /* Handle a software breakpoint */ /* We have to inform parent that child has stopped */ diff --git a/arch/sparc/kernel/ftrace.c b/arch/sparc/kernel/ftrace.c index 0a2d2ddff543..6bcff698069b 100644 --- a/arch/sparc/kernel/ftrace.c +++ b/arch/sparc/kernel/ftrace.c @@ -131,7 +131,7 @@ unsigned long prepare_ftrace_return(unsigned long parent, return parent + 8UL; if (ftrace_push_return_trace(parent, self_addr, &trace.depth, - frame_pointer) == -EBUSY) + frame_pointer, NULL) == -EBUSY) return parent + 8UL; trace.func = self_addr; diff --git a/arch/sparc/kernel/head_32.S b/arch/sparc/kernel/head_32.S index 3d92c0a8f6c4..7bb317b87dde 100644 --- a/arch/sparc/kernel/head_32.S +++ b/arch/sparc/kernel/head_32.S @@ -24,6 +24,7 @@ #include <asm/thread_info.h> /* TI_UWINMASK */ #include <asm/errno.h> #include <asm/pgtsrmmu.h> /* SRMMU_PGDIR_SHIFT */ +#include <asm/export.h> .data /* The following are used with the prom_vector node-ops to figure out @@ -60,6 +61,7 @@ sun4e_notsup: */ .globl empty_zero_page empty_zero_page: .skip PAGE_SIZE +EXPORT_SYMBOL(empty_zero_page) .global root_flags .global ram_flags @@ -813,3 +815,4 @@ lvl14_save: __ret_efault: ret restore %g0, -EFAULT, %o0 +EXPORT_SYMBOL(__ret_efault) diff --git a/arch/sparc/kernel/head_64.S b/arch/sparc/kernel/head_64.S index a076b4249e62..beba6c11554c 100644 --- a/arch/sparc/kernel/head_64.S +++ b/arch/sparc/kernel/head_64.S @@ -32,7 +32,8 @@ #include <asm/estate.h> #include <asm/sfafsr.h> #include <asm/unistd.h> - +#include <asm/export.h> + /* This section from from _start to sparc64_boot_end should fit into * 0x0000000000404000 to 0x0000000000408000. */ @@ -143,6 +144,7 @@ prom_cpu_compatible: .skip 64 prom_root_node: .word 0 +EXPORT_SYMBOL(prom_root_node) prom_mmu_ihandle_cache: .word 0 prom_boot_mapped_pc: @@ -158,6 +160,7 @@ is_sun4v: .word 0 sun4v_chip_type: .word SUN4V_CHIP_INVALID +EXPORT_SYMBOL(sun4v_chip_type) 1: rd %pc, %l0 @@ -920,6 +923,7 @@ swapper_4m_tsb: .globl prom_tba, tlb_type prom_tba: .xword 0 tlb_type: .word 0 /* Must NOT end up in BSS */ +EXPORT_SYMBOL(tlb_type) .section ".fixup",#alloc,#execinstr .globl __ret_efault, __retl_efault, __ret_one, __retl_one @@ -927,6 +931,7 @@ ENTRY(__ret_efault) ret restore %g0, -EFAULT, %o0 ENDPROC(__ret_efault) +EXPORT_SYMBOL(__ret_efault) ENTRY(__retl_efault) retl diff --git a/arch/sparc/kernel/helpers.S b/arch/sparc/kernel/helpers.S index 314dd0c9fc5b..e4e5b832fcb6 100644 --- a/arch/sparc/kernel/helpers.S +++ b/arch/sparc/kernel/helpers.S @@ -15,6 +15,7 @@ __flushw_user: 2: retl nop .size __flushw_user,.-__flushw_user +EXPORT_SYMBOL(__flushw_user) /* Flush %fp and %i7 to the stack for all register * windows active inside of the cpu. This allows @@ -61,3 +62,4 @@ real_hard_smp_processor_id: .size hard_smp_processor_id,.-hard_smp_processor_id #endif .size real_hard_smp_processor_id,.-real_hard_smp_processor_id +EXPORT_SYMBOL_GPL(real_hard_smp_processor_id) diff --git a/arch/sparc/kernel/hvcalls.S b/arch/sparc/kernel/hvcalls.S index d127130bf424..4116ee5c7791 100644 --- a/arch/sparc/kernel/hvcalls.S +++ b/arch/sparc/kernel/hvcalls.S @@ -343,6 +343,7 @@ ENTRY(sun4v_mach_set_watchdog) 0: retl nop ENDPROC(sun4v_mach_set_watchdog) +EXPORT_SYMBOL(sun4v_mach_set_watchdog) /* No inputs and does not return. */ ENTRY(sun4v_mach_sir) @@ -776,6 +777,7 @@ ENTRY(sun4v_niagara_getperf) retl nop ENDPROC(sun4v_niagara_getperf) +EXPORT_SYMBOL(sun4v_niagara_getperf) ENTRY(sun4v_niagara_setperf) mov HV_FAST_SET_PERFREG, %o5 @@ -783,6 +785,7 @@ ENTRY(sun4v_niagara_setperf) retl nop ENDPROC(sun4v_niagara_setperf) +EXPORT_SYMBOL(sun4v_niagara_setperf) ENTRY(sun4v_niagara2_getperf) mov %o0, %o4 @@ -792,6 +795,7 @@ ENTRY(sun4v_niagara2_getperf) retl nop ENDPROC(sun4v_niagara2_getperf) +EXPORT_SYMBOL(sun4v_niagara2_getperf) ENTRY(sun4v_niagara2_setperf) mov HV_FAST_N2_SET_PERFREG, %o5 @@ -799,6 +803,7 @@ ENTRY(sun4v_niagara2_setperf) retl nop ENDPROC(sun4v_niagara2_setperf) +EXPORT_SYMBOL(sun4v_niagara2_setperf) ENTRY(sun4v_reboot_data_set) mov HV_FAST_REBOOT_DATA_SET, %o5 diff --git a/arch/sparc/kernel/iommu.c b/arch/sparc/kernel/iommu.c index 37686828c3d9..5c615abff030 100644 --- a/arch/sparc/kernel/iommu.c +++ b/arch/sparc/kernel/iommu.c @@ -196,7 +196,7 @@ static inline void iommu_free_ctx(struct iommu *iommu, int ctx) static void *dma_4u_alloc_coherent(struct device *dev, size_t size, dma_addr_t *dma_addrp, gfp_t gfp, - struct dma_attrs *attrs) + unsigned long attrs) { unsigned long order, first_page; struct iommu *iommu; @@ -245,7 +245,7 @@ static void *dma_4u_alloc_coherent(struct device *dev, size_t size, static void dma_4u_free_coherent(struct device *dev, size_t size, void *cpu, dma_addr_t dvma, - struct dma_attrs *attrs) + unsigned long attrs) { struct iommu *iommu; unsigned long order, npages; @@ -263,7 +263,7 @@ static void dma_4u_free_coherent(struct device *dev, size_t size, static dma_addr_t dma_4u_map_page(struct device *dev, struct page *page, unsigned long offset, size_t sz, enum dma_data_direction direction, - struct dma_attrs *attrs) + unsigned long attrs) { struct iommu *iommu; struct strbuf *strbuf; @@ -385,7 +385,7 @@ do_flush_sync: static void dma_4u_unmap_page(struct device *dev, dma_addr_t bus_addr, size_t sz, enum dma_data_direction direction, - struct dma_attrs *attrs) + unsigned long attrs) { struct iommu *iommu; struct strbuf *strbuf; @@ -431,7 +431,7 @@ static void dma_4u_unmap_page(struct device *dev, dma_addr_t bus_addr, static int dma_4u_map_sg(struct device *dev, struct scatterlist *sglist, int nelems, enum dma_data_direction direction, - struct dma_attrs *attrs) + unsigned long attrs) { struct scatterlist *s, *outs, *segstart; unsigned long flags, handle, prot, ctx; @@ -607,7 +607,7 @@ static unsigned long fetch_sg_ctx(struct iommu *iommu, struct scatterlist *sg) static void dma_4u_unmap_sg(struct device *dev, struct scatterlist *sglist, int nelems, enum dma_data_direction direction, - struct dma_attrs *attrs) + unsigned long attrs) { unsigned long flags, ctx; struct scatterlist *sg; diff --git a/arch/sparc/kernel/ioport.c b/arch/sparc/kernel/ioport.c index ffd5ff4678cf..2344103414d1 100644 --- a/arch/sparc/kernel/ioport.c +++ b/arch/sparc/kernel/ioport.c @@ -260,7 +260,7 @@ EXPORT_SYMBOL(sbus_set_sbus64); */ static void *sbus_alloc_coherent(struct device *dev, size_t len, dma_addr_t *dma_addrp, gfp_t gfp, - struct dma_attrs *attrs) + unsigned long attrs) { struct platform_device *op = to_platform_device(dev); unsigned long len_total = PAGE_ALIGN(len); @@ -315,7 +315,7 @@ err_nopages: } static void sbus_free_coherent(struct device *dev, size_t n, void *p, - dma_addr_t ba, struct dma_attrs *attrs) + dma_addr_t ba, unsigned long attrs) { struct resource *res; struct page *pgv; @@ -355,7 +355,7 @@ static void sbus_free_coherent(struct device *dev, size_t n, void *p, static dma_addr_t sbus_map_page(struct device *dev, struct page *page, unsigned long offset, size_t len, enum dma_data_direction dir, - struct dma_attrs *attrs) + unsigned long attrs) { void *va = page_address(page) + offset; @@ -371,20 +371,20 @@ static dma_addr_t sbus_map_page(struct device *dev, struct page *page, } static void sbus_unmap_page(struct device *dev, dma_addr_t ba, size_t n, - enum dma_data_direction dir, struct dma_attrs *attrs) + enum dma_data_direction dir, unsigned long attrs) { mmu_release_scsi_one(dev, ba, n); } static int sbus_map_sg(struct device *dev, struct scatterlist *sg, int n, - enum dma_data_direction dir, struct dma_attrs *attrs) + enum dma_data_direction dir, unsigned long attrs) { mmu_get_scsi_sgl(dev, sg, n); return n; } static void sbus_unmap_sg(struct device *dev, struct scatterlist *sg, int n, - enum dma_data_direction dir, struct dma_attrs *attrs) + enum dma_data_direction dir, unsigned long attrs) { mmu_release_scsi_sgl(dev, sg, n); } @@ -429,7 +429,7 @@ arch_initcall(sparc_register_ioport); */ static void *pci32_alloc_coherent(struct device *dev, size_t len, dma_addr_t *pba, gfp_t gfp, - struct dma_attrs *attrs) + unsigned long attrs) { unsigned long len_total = PAGE_ALIGN(len); void *va; @@ -482,7 +482,7 @@ err_nopages: * past this call are illegal. */ static void pci32_free_coherent(struct device *dev, size_t n, void *p, - dma_addr_t ba, struct dma_attrs *attrs) + dma_addr_t ba, unsigned long attrs) { struct resource *res; @@ -518,14 +518,14 @@ static void pci32_free_coherent(struct device *dev, size_t n, void *p, static dma_addr_t pci32_map_page(struct device *dev, struct page *page, unsigned long offset, size_t size, enum dma_data_direction dir, - struct dma_attrs *attrs) + unsigned long attrs) { /* IIep is write-through, not flushing. */ return page_to_phys(page) + offset; } static void pci32_unmap_page(struct device *dev, dma_addr_t ba, size_t size, - enum dma_data_direction dir, struct dma_attrs *attrs) + enum dma_data_direction dir, unsigned long attrs) { if (dir != PCI_DMA_TODEVICE) dma_make_coherent(ba, PAGE_ALIGN(size)); @@ -548,7 +548,7 @@ static void pci32_unmap_page(struct device *dev, dma_addr_t ba, size_t size, */ static int pci32_map_sg(struct device *device, struct scatterlist *sgl, int nents, enum dma_data_direction dir, - struct dma_attrs *attrs) + unsigned long attrs) { struct scatterlist *sg; int n; @@ -567,7 +567,7 @@ static int pci32_map_sg(struct device *device, struct scatterlist *sgl, */ static void pci32_unmap_sg(struct device *dev, struct scatterlist *sgl, int nents, enum dma_data_direction dir, - struct dma_attrs *attrs) + unsigned long attrs) { struct scatterlist *sg; int n; diff --git a/arch/sparc/kernel/irq_32.c b/arch/sparc/kernel/irq_32.c index a979e99f8751..cac4a5554c0e 100644 --- a/arch/sparc/kernel/irq_32.c +++ b/arch/sparc/kernel/irq_32.c @@ -165,7 +165,7 @@ void irq_link(unsigned int irq) p = &irq_table[irq]; pil = p->pil; - BUG_ON(pil > SUN4D_MAX_IRQ); + BUG_ON(pil >= SUN4D_MAX_IRQ); p->next = irq_map[pil]; irq_map[pil] = p; @@ -182,7 +182,7 @@ void irq_unlink(unsigned int irq) spin_lock_irqsave(&irq_map_lock, flags); p = &irq_table[irq]; - BUG_ON(p->pil > SUN4D_MAX_IRQ); + BUG_ON(p->pil >= SUN4D_MAX_IRQ); pnext = &irq_map[p->pil]; while (*pnext != p) pnext = &(*pnext)->next; diff --git a/arch/sparc/kernel/irq_64.c b/arch/sparc/kernel/irq_64.c index e22416ce56ea..34a7930b76ef 100644 --- a/arch/sparc/kernel/irq_64.c +++ b/arch/sparc/kernel/irq_64.c @@ -242,7 +242,7 @@ unsigned int irq_alloc(unsigned int dev_handle, unsigned int dev_ino) { int irq; - irq = __irq_alloc_descs(-1, 1, 1, numa_node_id(), NULL); + irq = __irq_alloc_descs(-1, 1, 1, numa_node_id(), NULL, NULL); if (irq <= 0) goto out; diff --git a/arch/sparc/kernel/kprobes.c b/arch/sparc/kernel/kprobes.c index cd83be527586..b0377db12d83 100644 --- a/arch/sparc/kernel/kprobes.c +++ b/arch/sparc/kernel/kprobes.c @@ -5,7 +5,7 @@ #include <linux/kernel.h> #include <linux/kprobes.h> -#include <linux/module.h> +#include <linux/extable.h> #include <linux/kdebug.h> #include <linux/slab.h> #include <linux/context_tracking.h> diff --git a/arch/sparc/kernel/ktlb.S b/arch/sparc/kernel/ktlb.S index ef0d8e9e1210..f22bec0db645 100644 --- a/arch/sparc/kernel/ktlb.S +++ b/arch/sparc/kernel/ktlb.S @@ -20,6 +20,10 @@ kvmap_itlb: mov TLB_TAG_ACCESS, %g4 ldxa [%g4] ASI_IMMU, %g4 + /* The kernel executes in context zero, therefore we do not + * need to clear the context ID bits out of %g4 here. + */ + /* sun4v_itlb_miss branches here with the missing virtual * address already loaded into %g4 */ @@ -128,6 +132,10 @@ kvmap_dtlb: mov TLB_TAG_ACCESS, %g4 ldxa [%g4] ASI_DMMU, %g4 + /* The kernel executes in context zero, therefore we do not + * need to clear the context ID bits out of %g4 here. + */ + /* sun4v_dtlb_miss branches here with the missing virtual * address already loaded into %g4 */ @@ -251,6 +259,10 @@ kvmap_dtlb_longpath: nop .previous + /* The kernel executes in context zero, therefore we do not + * need to clear the context ID bits out of %g5 here. + */ + be,pt %xcc, sparc64_realfault_common mov FAULT_CODE_DTLB, %g4 ba,pt %xcc, winfix_trampoline diff --git a/arch/sparc/kernel/pci.c b/arch/sparc/kernel/pci.c index c2b202d763a1..9c1878f4fa9f 100644 --- a/arch/sparc/kernel/pci.c +++ b/arch/sparc/kernel/pci.c @@ -986,16 +986,18 @@ void pci_resource_to_user(const struct pci_dev *pdev, int bar, const struct resource *rp, resource_size_t *start, resource_size_t *end) { - struct pci_pbm_info *pbm = pdev->dev.archdata.host_controller; - unsigned long offset; - - if (rp->flags & IORESOURCE_IO) - offset = pbm->io_space.start; - else - offset = pbm->mem_space.start; + struct pci_bus_region region; - *start = rp->start - offset; - *end = rp->end - offset; + /* + * "User" addresses are shown in /sys/devices/pci.../.../resource + * and /proc/bus/pci/devices and used as mmap offsets for + * /proc/bus/pci/BB/DD.F files (see proc_bus_pci_mmap()). + * + * On sparc, these are PCI bus addresses, i.e., raw BAR values. + */ + pcibios_resource_to_bus(pdev->bus, ®ion, (struct resource *) rp); + *start = region.start; + *end = region.end; } void pcibios_set_master(struct pci_dev *dev) diff --git a/arch/sparc/kernel/pci_sun4v.c b/arch/sparc/kernel/pci_sun4v.c index 836e8cef47e2..db57d8acdc01 100644 --- a/arch/sparc/kernel/pci_sun4v.c +++ b/arch/sparc/kernel/pci_sun4v.c @@ -30,8 +30,19 @@ #define DRIVER_NAME "pci_sun4v" #define PFX DRIVER_NAME ": " -static unsigned long vpci_major = 1; -static unsigned long vpci_minor = 1; +static unsigned long vpci_major; +static unsigned long vpci_minor; + +struct vpci_version { + unsigned long major; + unsigned long minor; +}; + +/* Ordered from largest major to lowest */ +static struct vpci_version vpci_versions[] = { + { .major = 2, .minor = 0 }, + { .major = 1, .minor = 1 }, +}; #define PGLIST_NENTS (PAGE_SIZE / sizeof(u64)) @@ -67,6 +78,10 @@ static long iommu_batch_flush(struct iommu_batch *p) u64 *pglist = p->pglist; unsigned long npages = p->npages; + /* VPCI maj=1, min=[0,1] only supports read and write */ + if (vpci_major < 2) + prot &= (HV_PCI_MAP_ATTR_READ | HV_PCI_MAP_ATTR_WRITE); + while (npages != 0) { long num; @@ -130,9 +145,10 @@ static inline long iommu_batch_end(void) static void *dma_4v_alloc_coherent(struct device *dev, size_t size, dma_addr_t *dma_addrp, gfp_t gfp, - struct dma_attrs *attrs) + unsigned long attrs) { unsigned long flags, order, first_page, npages, n; + unsigned long prot = 0; struct iommu *iommu; struct page *page; void *ret; @@ -146,6 +162,9 @@ static void *dma_4v_alloc_coherent(struct device *dev, size_t size, npages = size >> IO_PAGE_SHIFT; + if (attrs & DMA_ATTR_WEAK_ORDERING) + prot = HV_PCI_MAP_ATTR_RELAXED_ORDER; + nid = dev->archdata.numa_node; page = alloc_pages_node(nid, gfp, order); if (unlikely(!page)) @@ -169,7 +188,7 @@ static void *dma_4v_alloc_coherent(struct device *dev, size_t size, local_irq_save(flags); iommu_batch_start(dev, - (HV_PCI_MAP_ATTR_READ | + (HV_PCI_MAP_ATTR_READ | prot | HV_PCI_MAP_ATTR_WRITE), entry); @@ -213,7 +232,7 @@ static void dma_4v_iommu_demap(void *demap_arg, unsigned long entry, } static void dma_4v_free_coherent(struct device *dev, size_t size, void *cpu, - dma_addr_t dvma, struct dma_attrs *attrs) + dma_addr_t dvma, unsigned long attrs) { struct pci_pbm_info *pbm; struct iommu *iommu; @@ -235,7 +254,7 @@ static void dma_4v_free_coherent(struct device *dev, size_t size, void *cpu, static dma_addr_t dma_4v_map_page(struct device *dev, struct page *page, unsigned long offset, size_t sz, enum dma_data_direction direction, - struct dma_attrs *attrs) + unsigned long attrs) { struct iommu *iommu; unsigned long flags, npages, oaddr; @@ -266,6 +285,9 @@ static dma_addr_t dma_4v_map_page(struct device *dev, struct page *page, if (direction != DMA_TO_DEVICE) prot |= HV_PCI_MAP_ATTR_WRITE; + if (attrs & DMA_ATTR_WEAK_ORDERING) + prot |= HV_PCI_MAP_ATTR_RELAXED_ORDER; + local_irq_save(flags); iommu_batch_start(dev, prot, entry); @@ -294,7 +316,7 @@ iommu_map_fail: static void dma_4v_unmap_page(struct device *dev, dma_addr_t bus_addr, size_t sz, enum dma_data_direction direction, - struct dma_attrs *attrs) + unsigned long attrs) { struct pci_pbm_info *pbm; struct iommu *iommu; @@ -322,7 +344,7 @@ static void dma_4v_unmap_page(struct device *dev, dma_addr_t bus_addr, static int dma_4v_map_sg(struct device *dev, struct scatterlist *sglist, int nelems, enum dma_data_direction direction, - struct dma_attrs *attrs) + unsigned long attrs) { struct scatterlist *s, *outs, *segstart; unsigned long flags, handle, prot; @@ -344,6 +366,9 @@ static int dma_4v_map_sg(struct device *dev, struct scatterlist *sglist, if (direction != DMA_TO_DEVICE) prot |= HV_PCI_MAP_ATTR_WRITE; + if (attrs & DMA_ATTR_WEAK_ORDERING) + prot |= HV_PCI_MAP_ATTR_RELAXED_ORDER; + outs = s = segstart = &sglist[0]; outcount = 1; incount = nelems; @@ -466,7 +491,7 @@ iommu_map_failed: static void dma_4v_unmap_sg(struct device *dev, struct scatterlist *sglist, int nelems, enum dma_data_direction direction, - struct dma_attrs *attrs) + unsigned long attrs) { struct pci_pbm_info *pbm; struct scatterlist *sg; @@ -907,22 +932,27 @@ static int pci_sun4v_probe(struct platform_device *op) struct device_node *dp; struct iommu *iommu; u32 devhandle; - int i, err; + int i, err = -ENODEV; dp = op->dev.of_node; if (!hvapi_negotiated++) { - err = sun4v_hvapi_register(HV_GRP_PCI, - vpci_major, - &vpci_minor); + for (i = 0; i < ARRAY_SIZE(vpci_versions); i++) { + vpci_major = vpci_versions[i].major; + vpci_minor = vpci_versions[i].minor; + + err = sun4v_hvapi_register(HV_GRP_PCI, vpci_major, + &vpci_minor); + if (!err) + break; + } if (err) { - printk(KERN_ERR PFX "Could not register hvapi, " - "err=%d\n", err); + pr_err(PFX "Could not register hvapi, err=%d\n", err); return err; } - printk(KERN_INFO PFX "Registered hvapi major[%lu] minor[%lu]\n", - vpci_major, vpci_minor); + pr_info(PFX "Registered hvapi major[%lu] minor[%lu]\n", + vpci_major, vpci_minor); dma_ops = &sun4v_dma_ops; } diff --git a/arch/sparc/kernel/process_64.c b/arch/sparc/kernel/process_64.c index fa14402b33f9..47ff5588e521 100644 --- a/arch/sparc/kernel/process_64.c +++ b/arch/sparc/kernel/process_64.c @@ -239,7 +239,7 @@ static void __global_reg_poll(struct global_reg_snapshot *gp) } } -void arch_trigger_all_cpu_backtrace(bool include_self) +void arch_trigger_cpumask_backtrace(const cpumask_t *mask, bool exclude_self) { struct thread_info *tp = current_thread_info(); struct pt_regs *regs = get_irq_regs(); @@ -255,15 +255,15 @@ void arch_trigger_all_cpu_backtrace(bool include_self) memset(global_cpu_snapshot, 0, sizeof(global_cpu_snapshot)); - if (include_self) + if (cpumask_test_cpu(this_cpu, mask) && !exclude_self) __global_reg_self(tp, regs, this_cpu); smp_fetch_global_regs(); - for_each_online_cpu(cpu) { + for_each_cpu(cpu, mask) { struct global_reg_snapshot *gp; - if (!include_self && cpu == this_cpu) + if (exclude_self && cpu == this_cpu) continue; gp = &global_cpu_snapshot[cpu].reg; @@ -300,7 +300,7 @@ void arch_trigger_all_cpu_backtrace(bool include_self) static void sysrq_handle_globreg(int key) { - arch_trigger_all_cpu_backtrace(true); + trigger_all_cpu_backtrace(); } static struct sysrq_key_op sparc_globalreg_op = { diff --git a/arch/sparc/kernel/rtrap_64.S b/arch/sparc/kernel/rtrap_64.S index d08bdaffdbfc..216948ca4382 100644 --- a/arch/sparc/kernel/rtrap_64.S +++ b/arch/sparc/kernel/rtrap_64.S @@ -14,10 +14,6 @@ #include <asm/visasm.h> #include <asm/processor.h> -#define RTRAP_PSTATE (PSTATE_TSO|PSTATE_PEF|PSTATE_PRIV|PSTATE_IE) -#define RTRAP_PSTATE_IRQOFF (PSTATE_TSO|PSTATE_PEF|PSTATE_PRIV) -#define RTRAP_PSTATE_AG_IRQOFF (PSTATE_TSO|PSTATE_PEF|PSTATE_PRIV|PSTATE_AG) - #ifdef CONFIG_CONTEXT_TRACKING # define SCHEDULE_USER schedule_user #else @@ -242,52 +238,17 @@ rt_continue: ldx [%sp + PTREGS_OFF + PT_V9_G1], %g1 wrpr %g1, %cwp ba,a,pt %xcc, user_rtt_fill_64bit -user_rtt_fill_fixup: - rdpr %cwp, %g1 - add %g1, 1, %g1 - wrpr %g1, 0x0, %cwp - - rdpr %wstate, %g2 - sll %g2, 3, %g2 - wrpr %g2, 0x0, %wstate - - /* We know %canrestore and %otherwin are both zero. */ - - sethi %hi(sparc64_kern_pri_context), %g2 - ldx [%g2 + %lo(sparc64_kern_pri_context)], %g2 - mov PRIMARY_CONTEXT, %g1 - -661: stxa %g2, [%g1] ASI_DMMU - .section .sun4v_1insn_patch, "ax" - .word 661b - stxa %g2, [%g1] ASI_MMU - .previous - - sethi %hi(KERNBASE), %g1 - flush %g1 +user_rtt_fill_fixup_dax: + ba,pt %xcc, user_rtt_fill_fixup_common + mov 1, %g3 - or %g4, FAULT_CODE_WINFIXUP, %g4 - stb %g4, [%g6 + TI_FAULT_CODE] - stx %g5, [%g6 + TI_FAULT_ADDR] +user_rtt_fill_fixup_mna: + ba,pt %xcc, user_rtt_fill_fixup_common + mov 2, %g3 - mov %g6, %l1 - wrpr %g0, 0x0, %tl - -661: nop - .section .sun4v_1insn_patch, "ax" - .word 661b - SET_GL(0) - .previous - - wrpr %g0, RTRAP_PSTATE, %pstate - - mov %l1, %g6 - ldx [%g6 + TI_TASK], %g4 - LOAD_PER_CPU_BASE(%g5, %g6, %g1, %g2, %g3) - call do_sparc64_fault - add %sp, PTREGS_OFF, %o0 - ba,pt %xcc, rtrap - nop +user_rtt_fill_fixup: + ba,pt %xcc, user_rtt_fill_fixup_common + clr %g3 user_rtt_pre_restore: add %g1, 1, %g1 diff --git a/arch/sparc/kernel/setup_64.c b/arch/sparc/kernel/setup_64.c index 599f1207eed2..6b7331d198e9 100644 --- a/arch/sparc/kernel/setup_64.c +++ b/arch/sparc/kernel/setup_64.c @@ -31,6 +31,7 @@ #include <linux/initrd.h> #include <linux/module.h> #include <linux/start_kernel.h> +#include <linux/bootmem.h> #include <asm/io.h> #include <asm/processor.h> @@ -50,6 +51,8 @@ #include <asm/elf.h> #include <asm/mdesc.h> #include <asm/cacheflush.h> +#include <asm/dma.h> +#include <asm/irq.h> #ifdef CONFIG_IP_PNP #include <net/ipconfig.h> @@ -590,6 +593,22 @@ static void __init init_sparc64_elf_hwcap(void) pause_patch(); } +void __init alloc_irqstack_bootmem(void) +{ + unsigned int i, node; + + for_each_possible_cpu(i) { + node = cpu_to_node(i); + + softirq_stack[i] = __alloc_bootmem_node(NODE_DATA(node), + THREAD_SIZE, + THREAD_SIZE, 0); + hardirq_stack[i] = __alloc_bootmem_node(NODE_DATA(node), + THREAD_SIZE, + THREAD_SIZE, 0); + } +} + void __init setup_arch(char **cmdline_p) { /* Initialize PROM console and command line. */ @@ -650,6 +669,13 @@ void __init setup_arch(char **cmdline_p) paging_init(); init_sparc64_elf_hwcap(); + smp_fill_in_cpu_possible_map(); + /* + * Once the OF device tree and MDESC have been setup and nr_cpus has + * been parsed, we know the list of possible cpus. Therefore we can + * allocate the IRQ stacks. + */ + alloc_irqstack_bootmem(); } extern int stop_a_enabled; diff --git a/arch/sparc/kernel/signal32.c b/arch/sparc/kernel/signal32.c index 3c25241fa5cb..91cc2f4ae4d9 100644 --- a/arch/sparc/kernel/signal32.c +++ b/arch/sparc/kernel/signal32.c @@ -138,12 +138,24 @@ int copy_siginfo_from_user32(siginfo_t *to, compat_siginfo_t __user *from) return 0; } +/* Checks if the fp is valid. We always build signal frames which are + * 16-byte aligned, therefore we can always enforce that the restore + * frame has that property as well. + */ +static bool invalid_frame_pointer(void __user *fp, int fplen) +{ + if ((((unsigned long) fp) & 15) || + ((unsigned long)fp) > 0x100000000ULL - fplen) + return true; + return false; +} + void do_sigreturn32(struct pt_regs *regs) { struct signal_frame32 __user *sf; compat_uptr_t fpu_save; compat_uptr_t rwin_save; - unsigned int psr; + unsigned int psr, ufp; unsigned int pc, npc; sigset_t set; compat_sigset_t seta; @@ -158,11 +170,16 @@ void do_sigreturn32(struct pt_regs *regs) sf = (struct signal_frame32 __user *) regs->u_regs[UREG_FP]; /* 1. Make sure we are not getting garbage from the user */ - if (!access_ok(VERIFY_READ, sf, sizeof(*sf)) || - (((unsigned long) sf) & 3)) + if (invalid_frame_pointer(sf, sizeof(*sf))) + goto segv; + + if (get_user(ufp, &sf->info.si_regs.u_regs[UREG_FP])) + goto segv; + + if (ufp & 0x7) goto segv; - if (get_user(pc, &sf->info.si_regs.pc) || + if (__get_user(pc, &sf->info.si_regs.pc) || __get_user(npc, &sf->info.si_regs.npc)) goto segv; @@ -227,7 +244,7 @@ segv: asmlinkage void do_rt_sigreturn32(struct pt_regs *regs) { struct rt_signal_frame32 __user *sf; - unsigned int psr, pc, npc; + unsigned int psr, pc, npc, ufp; compat_uptr_t fpu_save; compat_uptr_t rwin_save; sigset_t set; @@ -242,11 +259,16 @@ asmlinkage void do_rt_sigreturn32(struct pt_regs *regs) sf = (struct rt_signal_frame32 __user *) regs->u_regs[UREG_FP]; /* 1. Make sure we are not getting garbage from the user */ - if (!access_ok(VERIFY_READ, sf, sizeof(*sf)) || - (((unsigned long) sf) & 3)) + if (invalid_frame_pointer(sf, sizeof(*sf))) goto segv; - if (get_user(pc, &sf->regs.pc) || + if (get_user(ufp, &sf->regs.u_regs[UREG_FP])) + goto segv; + + if (ufp & 0x7) + goto segv; + + if (__get_user(pc, &sf->regs.pc) || __get_user(npc, &sf->regs.npc)) goto segv; @@ -307,14 +329,6 @@ segv: force_sig(SIGSEGV, current); } -/* Checks if the fp is valid */ -static int invalid_frame_pointer(void __user *fp, int fplen) -{ - if ((((unsigned long) fp) & 7) || ((unsigned long)fp) > 0x100000000ULL - fplen) - return 1; - return 0; -} - static void __user *get_sigframe(struct ksignal *ksig, struct pt_regs *regs, unsigned long framesize) { unsigned long sp; diff --git a/arch/sparc/kernel/signal_32.c b/arch/sparc/kernel/signal_32.c index 52aa5e4ce5e7..c3c12efe0bc0 100644 --- a/arch/sparc/kernel/signal_32.c +++ b/arch/sparc/kernel/signal_32.c @@ -60,10 +60,22 @@ struct rt_signal_frame { #define SF_ALIGNEDSZ (((sizeof(struct signal_frame) + 7) & (~7))) #define RT_ALIGNEDSZ (((sizeof(struct rt_signal_frame) + 7) & (~7))) +/* Checks if the fp is valid. We always build signal frames which are + * 16-byte aligned, therefore we can always enforce that the restore + * frame has that property as well. + */ +static inline bool invalid_frame_pointer(void __user *fp, int fplen) +{ + if ((((unsigned long) fp) & 15) || !__access_ok((unsigned long)fp, fplen)) + return true; + + return false; +} + asmlinkage void do_sigreturn(struct pt_regs *regs) { + unsigned long up_psr, pc, npc, ufp; struct signal_frame __user *sf; - unsigned long up_psr, pc, npc; sigset_t set; __siginfo_fpu_t __user *fpu_save; __siginfo_rwin_t __user *rwin_save; @@ -77,10 +89,13 @@ asmlinkage void do_sigreturn(struct pt_regs *regs) sf = (struct signal_frame __user *) regs->u_regs[UREG_FP]; /* 1. Make sure we are not getting garbage from the user */ - if (!access_ok(VERIFY_READ, sf, sizeof(*sf))) + if (!invalid_frame_pointer(sf, sizeof(*sf))) + goto segv_and_exit; + + if (get_user(ufp, &sf->info.si_regs.u_regs[UREG_FP])) goto segv_and_exit; - if (((unsigned long) sf) & 3) + if (ufp & 0x7) goto segv_and_exit; err = __get_user(pc, &sf->info.si_regs.pc); @@ -127,7 +142,7 @@ segv_and_exit: asmlinkage void do_rt_sigreturn(struct pt_regs *regs) { struct rt_signal_frame __user *sf; - unsigned int psr, pc, npc; + unsigned int psr, pc, npc, ufp; __siginfo_fpu_t __user *fpu_save; __siginfo_rwin_t __user *rwin_save; sigset_t set; @@ -135,8 +150,13 @@ asmlinkage void do_rt_sigreturn(struct pt_regs *regs) synchronize_user_stack(); sf = (struct rt_signal_frame __user *) regs->u_regs[UREG_FP]; - if (!access_ok(VERIFY_READ, sf, sizeof(*sf)) || - (((unsigned long) sf) & 0x03)) + if (!invalid_frame_pointer(sf, sizeof(*sf))) + goto segv; + + if (get_user(ufp, &sf->regs.u_regs[UREG_FP])) + goto segv; + + if (ufp & 0x7) goto segv; err = __get_user(pc, &sf->regs.pc); @@ -178,15 +198,6 @@ segv: force_sig(SIGSEGV, current); } -/* Checks if the fp is valid */ -static inline int invalid_frame_pointer(void __user *fp, int fplen) -{ - if ((((unsigned long) fp) & 7) || !__access_ok((unsigned long)fp, fplen)) - return 1; - - return 0; -} - static inline void __user *get_sigframe(struct ksignal *ksig, struct pt_regs *regs, unsigned long framesize) { unsigned long sp = regs->u_regs[UREG_FP]; diff --git a/arch/sparc/kernel/signal_64.c b/arch/sparc/kernel/signal_64.c index 39aaec173f66..5ee930c48f4c 100644 --- a/arch/sparc/kernel/signal_64.c +++ b/arch/sparc/kernel/signal_64.c @@ -234,6 +234,17 @@ do_sigsegv: goto out; } +/* Checks if the fp is valid. We always build rt signal frames which + * are 16-byte aligned, therefore we can always enforce that the + * restore frame has that property as well. + */ +static bool invalid_frame_pointer(void __user *fp) +{ + if (((unsigned long) fp) & 15) + return true; + return false; +} + struct rt_signal_frame { struct sparc_stackf ss; siginfo_t info; @@ -246,8 +257,8 @@ struct rt_signal_frame { void do_rt_sigreturn(struct pt_regs *regs) { + unsigned long tpc, tnpc, tstate, ufp; struct rt_signal_frame __user *sf; - unsigned long tpc, tnpc, tstate; __siginfo_fpu_t __user *fpu_save; __siginfo_rwin_t __user *rwin_save; sigset_t set; @@ -261,10 +272,16 @@ void do_rt_sigreturn(struct pt_regs *regs) (regs->u_regs [UREG_FP] + STACK_BIAS); /* 1. Make sure we are not getting garbage from the user */ - if (((unsigned long) sf) & 3) + if (invalid_frame_pointer(sf)) + goto segv; + + if (get_user(ufp, &sf->regs.u_regs[UREG_FP])) goto segv; - err = get_user(tpc, &sf->regs.tpc); + if ((ufp + STACK_BIAS) & 0x7) + goto segv; + + err = __get_user(tpc, &sf->regs.tpc); err |= __get_user(tnpc, &sf->regs.tnpc); if (test_thread_flag(TIF_32BIT)) { tpc &= 0xffffffff; @@ -308,14 +325,6 @@ segv: force_sig(SIGSEGV, current); } -/* Checks if the fp is valid */ -static int invalid_frame_pointer(void __user *fp) -{ - if (((unsigned long) fp) & 15) - return 1; - return 0; -} - static inline void __user *get_sigframe(struct ksignal *ksig, struct pt_regs *regs, unsigned long framesize) { unsigned long sp = regs->u_regs[UREG_FP] + STACK_BIAS; diff --git a/arch/sparc/kernel/sigutil_32.c b/arch/sparc/kernel/sigutil_32.c index 0f6eebe71e6c..e5fe8cef9a69 100644 --- a/arch/sparc/kernel/sigutil_32.c +++ b/arch/sparc/kernel/sigutil_32.c @@ -48,6 +48,10 @@ int save_fpu_state(struct pt_regs *regs, __siginfo_fpu_t __user *fpu) int restore_fpu_state(struct pt_regs *regs, __siginfo_fpu_t __user *fpu) { int err; + + if (((unsigned long) fpu) & 3) + return -EFAULT; + #ifdef CONFIG_SMP if (test_tsk_thread_flag(current, TIF_USEDFPU)) regs->psr &= ~PSR_EF; @@ -97,7 +101,10 @@ int restore_rwin_state(__siginfo_rwin_t __user *rp) struct thread_info *t = current_thread_info(); int i, wsaved, err; - __get_user(wsaved, &rp->wsaved); + if (((unsigned long) rp) & 3) + return -EFAULT; + + get_user(wsaved, &rp->wsaved); if (wsaved > NSWINS) return -EFAULT; diff --git a/arch/sparc/kernel/sigutil_64.c b/arch/sparc/kernel/sigutil_64.c index 387834a9c56a..36aadcbeac69 100644 --- a/arch/sparc/kernel/sigutil_64.c +++ b/arch/sparc/kernel/sigutil_64.c @@ -37,7 +37,10 @@ int restore_fpu_state(struct pt_regs *regs, __siginfo_fpu_t __user *fpu) unsigned long fprs; int err; - err = __get_user(fprs, &fpu->si_fprs); + if (((unsigned long) fpu) & 7) + return -EFAULT; + + err = get_user(fprs, &fpu->si_fprs); fprs_write(0); regs->tstate &= ~TSTATE_PEF; if (fprs & FPRS_DL) @@ -72,7 +75,10 @@ int restore_rwin_state(__siginfo_rwin_t __user *rp) struct thread_info *t = current_thread_info(); int i, wsaved, err; - __get_user(wsaved, &rp->wsaved); + if (((unsigned long) rp) & 7) + return -EFAULT; + + get_user(wsaved, &rp->wsaved); if (wsaved > NSWINS) return -EFAULT; diff --git a/arch/sparc/kernel/smp_32.c b/arch/sparc/kernel/smp_32.c index fb30e7c6a5b1..e80e6ba3d500 100644 --- a/arch/sparc/kernel/smp_32.c +++ b/arch/sparc/kernel/smp_32.c @@ -352,9 +352,7 @@ static void sparc_start_secondary(void *arg) preempt_disable(); cpu = smp_processor_id(); - /* Invoke the CPU_STARTING notifier callbacks */ notify_cpu_starting(cpu); - arch_cpu_pre_online(arg); /* Set the CPU in the cpu_online_mask */ diff --git a/arch/sparc/kernel/smp_64.c b/arch/sparc/kernel/smp_64.c index 8a6151a628ce..d3035ba6cd31 100644 --- a/arch/sparc/kernel/smp_64.c +++ b/arch/sparc/kernel/smp_64.c @@ -1227,6 +1227,20 @@ void __init smp_setup_processor_id(void) xcall_deliver_impl = hypervisor_xcall_deliver; } +void __init smp_fill_in_cpu_possible_map(void) +{ + int possible_cpus = num_possible_cpus(); + int i; + + if (possible_cpus > nr_cpu_ids) + possible_cpus = nr_cpu_ids; + + for (i = 0; i < possible_cpus; i++) + set_cpu_possible(i, true); + for (; i < NR_CPUS; i++) + set_cpu_possible(i, false); +} + void smp_fill_in_sib_core_maps(void) { unsigned int i; diff --git a/arch/sparc/kernel/sparc_ksyms.c b/arch/sparc/kernel/sparc_ksyms.c new file mode 100644 index 000000000000..09aa69e422e5 --- /dev/null +++ b/arch/sparc/kernel/sparc_ksyms.c @@ -0,0 +1,12 @@ +/* + * arch/sparc/kernel/ksyms.c: Sparc specific ksyms support. + * + * Copyright (C) 1996 David S. Miller (davem@caip.rutgers.edu) + * Copyright (C) 1996 Eddie C. Dost (ecd@skynet.be) + */ + +#include <linux/init.h> +#include <linux/export.h> + +/* This is needed only for drivers/sbus/char/openprom.c */ +EXPORT_SYMBOL(saved_command_line); diff --git a/arch/sparc/kernel/sparc_ksyms_32.c b/arch/sparc/kernel/sparc_ksyms_32.c deleted file mode 100644 index bf4ccb10a78c..000000000000 --- a/arch/sparc/kernel/sparc_ksyms_32.c +++ /dev/null @@ -1,31 +0,0 @@ -/* - * arch/sparc/kernel/ksyms.c: Sparc specific ksyms support. - * - * Copyright (C) 1996 David S. Miller (davem@caip.rutgers.edu) - * Copyright (C) 1996 Eddie C. Dost (ecd@skynet.be) - */ - -#include <linux/module.h> - -#include <asm/pgtable.h> -#include <asm/uaccess.h> -#include <asm/delay.h> -#include <asm/head.h> -#include <asm/dma.h> - -struct poll { - int fd; - short events; - short revents; -}; - -/* from entry.S */ -EXPORT_SYMBOL(__udelay); -EXPORT_SYMBOL(__ndelay); - -/* from head_32.S */ -EXPORT_SYMBOL(__ret_efault); -EXPORT_SYMBOL(empty_zero_page); - -/* Exporting a symbol from /init/main.c */ -EXPORT_SYMBOL(saved_command_line); diff --git a/arch/sparc/kernel/sparc_ksyms_64.c b/arch/sparc/kernel/sparc_ksyms_64.c deleted file mode 100644 index 9e034f29dcc5..000000000000 --- a/arch/sparc/kernel/sparc_ksyms_64.c +++ /dev/null @@ -1,53 +0,0 @@ -/* arch/sparc64/kernel/sparc64_ksyms.c: Sparc64 specific ksyms support. - * - * Copyright (C) 1996, 2007 David S. Miller (davem@davemloft.net) - * Copyright (C) 1996 Eddie C. Dost (ecd@skynet.be) - * Copyright (C) 1999 Jakub Jelinek (jj@ultra.linux.cz) - */ - -#include <linux/export.h> -#include <linux/pci.h> -#include <linux/bitops.h> - -#include <asm/cpudata.h> -#include <asm/uaccess.h> -#include <asm/spitfire.h> -#include <asm/oplib.h> -#include <asm/hypervisor.h> -#include <asm/cacheflush.h> - -struct poll { - int fd; - short events; - short revents; -}; - -/* from helpers.S */ -EXPORT_SYMBOL(__flushw_user); -EXPORT_SYMBOL_GPL(real_hard_smp_processor_id); - -/* from head_64.S */ -EXPORT_SYMBOL(__ret_efault); -EXPORT_SYMBOL(tlb_type); -EXPORT_SYMBOL(sun4v_chip_type); -EXPORT_SYMBOL(prom_root_node); - -/* from hvcalls.S */ -EXPORT_SYMBOL(sun4v_niagara_getperf); -EXPORT_SYMBOL(sun4v_niagara_setperf); -EXPORT_SYMBOL(sun4v_niagara2_getperf); -EXPORT_SYMBOL(sun4v_niagara2_setperf); -EXPORT_SYMBOL(sun4v_mach_set_watchdog); - -/* from hweight.S */ -EXPORT_SYMBOL(__arch_hweight8); -EXPORT_SYMBOL(__arch_hweight16); -EXPORT_SYMBOL(__arch_hweight32); -EXPORT_SYMBOL(__arch_hweight64); - -/* from ffs_ffz.S */ -EXPORT_SYMBOL(ffs); -EXPORT_SYMBOL(__ffs); - -/* Exporting a symbol from /init/main.c */ -EXPORT_SYMBOL(saved_command_line); diff --git a/arch/sparc/kernel/traps_64.c b/arch/sparc/kernel/traps_64.c index d21cd625c0de..4094a51b1970 100644 --- a/arch/sparc/kernel/traps_64.c +++ b/arch/sparc/kernel/traps_64.c @@ -8,7 +8,7 @@ * I like traps on v9, :)))) */ -#include <linux/module.h> +#include <linux/extable.h> #include <linux/sched.h> #include <linux/linkage.h> #include <linux/kernel.h> diff --git a/arch/sparc/kernel/tsb.S b/arch/sparc/kernel/tsb.S index be98685c14c6..d568c8207af7 100644 --- a/arch/sparc/kernel/tsb.S +++ b/arch/sparc/kernel/tsb.S @@ -29,13 +29,17 @@ */ tsb_miss_dtlb: mov TLB_TAG_ACCESS, %g4 + ldxa [%g4] ASI_DMMU, %g4 + srlx %g4, PAGE_SHIFT, %g4 ba,pt %xcc, tsb_miss_page_table_walk - ldxa [%g4] ASI_DMMU, %g4 + sllx %g4, PAGE_SHIFT, %g4 tsb_miss_itlb: mov TLB_TAG_ACCESS, %g4 + ldxa [%g4] ASI_IMMU, %g4 + srlx %g4, PAGE_SHIFT, %g4 ba,pt %xcc, tsb_miss_page_table_walk - ldxa [%g4] ASI_IMMU, %g4 + sllx %g4, PAGE_SHIFT, %g4 /* At this point we have: * %g1 -- PAGE_SIZE TSB entry address @@ -284,6 +288,10 @@ tsb_do_dtlb_fault: nop .previous + /* Clear context ID bits. */ + srlx %g5, PAGE_SHIFT, %g5 + sllx %g5, PAGE_SHIFT, %g5 + be,pt %xcc, sparc64_realfault_common mov FAULT_CODE_DTLB, %g4 ba,pt %xcc, winfix_trampoline diff --git a/arch/sparc/kernel/unaligned_64.c b/arch/sparc/kernel/unaligned_64.c index 9aacb9159262..52c00d90d4b4 100644 --- a/arch/sparc/kernel/unaligned_64.c +++ b/arch/sparc/kernel/unaligned_64.c @@ -11,7 +11,7 @@ #include <linux/kernel.h> #include <linux/sched.h> #include <linux/mm.h> -#include <linux/module.h> +#include <linux/extable.h> #include <asm/asi.h> #include <asm/ptrace.h> #include <asm/pstate.h> diff --git a/arch/sparc/kernel/urtt_fill.S b/arch/sparc/kernel/urtt_fill.S new file mode 100644 index 000000000000..5604a2b051d4 --- /dev/null +++ b/arch/sparc/kernel/urtt_fill.S @@ -0,0 +1,98 @@ +#include <asm/thread_info.h> +#include <asm/trap_block.h> +#include <asm/spitfire.h> +#include <asm/ptrace.h> +#include <asm/head.h> + + .text + .align 8 + .globl user_rtt_fill_fixup_common +user_rtt_fill_fixup_common: + rdpr %cwp, %g1 + add %g1, 1, %g1 + wrpr %g1, 0x0, %cwp + + rdpr %wstate, %g2 + sll %g2, 3, %g2 + wrpr %g2, 0x0, %wstate + + /* We know %canrestore and %otherwin are both zero. */ + + sethi %hi(sparc64_kern_pri_context), %g2 + ldx [%g2 + %lo(sparc64_kern_pri_context)], %g2 + mov PRIMARY_CONTEXT, %g1 + +661: stxa %g2, [%g1] ASI_DMMU + .section .sun4v_1insn_patch, "ax" + .word 661b + stxa %g2, [%g1] ASI_MMU + .previous + + sethi %hi(KERNBASE), %g1 + flush %g1 + + mov %g4, %l4 + mov %g5, %l5 + brnz,pn %g3, 1f + mov %g3, %l3 + + or %g4, FAULT_CODE_WINFIXUP, %g4 + stb %g4, [%g6 + TI_FAULT_CODE] + stx %g5, [%g6 + TI_FAULT_ADDR] +1: + mov %g6, %l1 + wrpr %g0, 0x0, %tl + +661: nop + .section .sun4v_1insn_patch, "ax" + .word 661b + SET_GL(0) + .previous + + wrpr %g0, RTRAP_PSTATE, %pstate + + mov %l1, %g6 + ldx [%g6 + TI_TASK], %g4 + LOAD_PER_CPU_BASE(%g5, %g6, %g1, %g2, %g3) + + brnz,pn %l3, 1f + nop + + call do_sparc64_fault + add %sp, PTREGS_OFF, %o0 + ba,pt %xcc, rtrap + nop + +1: cmp %g3, 2 + bne,pn %xcc, 2f + nop + + sethi %hi(tlb_type), %g1 + lduw [%g1 + %lo(tlb_type)], %g1 + cmp %g1, 3 + bne,pt %icc, 1f + add %sp, PTREGS_OFF, %o0 + mov %l4, %o2 + call sun4v_do_mna + mov %l5, %o1 + ba,a,pt %xcc, rtrap +1: mov %l4, %o1 + mov %l5, %o2 + call mem_address_unaligned + nop + ba,a,pt %xcc, rtrap + +2: sethi %hi(tlb_type), %g1 + mov %l4, %o1 + lduw [%g1 + %lo(tlb_type)], %g1 + mov %l5, %o2 + cmp %g1, 3 + bne,pt %icc, 1f + add %sp, PTREGS_OFF, %o0 + call sun4v_data_access_exception + nop + ba,a,pt %xcc, rtrap + +1: call spitfire_data_access_exception + nop + ba,a,pt %xcc, rtrap diff --git a/arch/sparc/kernel/vmlinux.lds.S b/arch/sparc/kernel/vmlinux.lds.S index 7d02b1fef025..572db686f845 100644 --- a/arch/sparc/kernel/vmlinux.lds.S +++ b/arch/sparc/kernel/vmlinux.lds.S @@ -49,6 +49,7 @@ SECTIONS HEAD_TEXT TEXT_TEXT SCHED_TEXT + CPUIDLE_TEXT LOCK_TEXT KPROBES_TEXT IRQENTRY_TEXT @@ -150,6 +151,13 @@ SECTIONS } PERCPU_SECTION(SMP_CACHE_BYTES) +#ifdef CONFIG_JUMP_LABEL + . = ALIGN(PAGE_SIZE); + .exit.text : { + EXIT_TEXT + } +#endif + . = ALIGN(PAGE_SIZE); __init_end = .; BSS_SECTION(0, 0, 0) diff --git a/arch/sparc/lib/Makefile b/arch/sparc/lib/Makefile index 3269b0234093..885f00e81d1a 100644 --- a/arch/sparc/lib/Makefile +++ b/arch/sparc/lib/Makefile @@ -43,5 +43,4 @@ lib-$(CONFIG_SPARC64) += mcount.o ipcsum.o xor.o hweight.o ffs.o obj-$(CONFIG_SPARC64) += iomap.o obj-$(CONFIG_SPARC32) += atomic32.o ucmpdi2.o -obj-y += ksyms.o obj-$(CONFIG_SPARC64) += PeeCeeI.o diff --git a/arch/sparc/lib/U1memcpy.S b/arch/sparc/lib/U1memcpy.S index 3e6209ebb7d7..97e1b211090c 100644 --- a/arch/sparc/lib/U1memcpy.S +++ b/arch/sparc/lib/U1memcpy.S @@ -7,6 +7,7 @@ #ifdef __KERNEL__ #include <asm/visasm.h> #include <asm/asi.h> +#include <asm/export.h> #define GLOBAL_SPARE g7 #else #define GLOBAL_SPARE g5 @@ -567,3 +568,4 @@ FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ mov EX_RETVAL(%o4), %o0 .size FUNC_NAME, .-FUNC_NAME +EXPORT_SYMBOL(FUNC_NAME) diff --git a/arch/sparc/lib/VISsave.S b/arch/sparc/lib/VISsave.S index 62c2647bd5ce..1c7b6a39b942 100644 --- a/arch/sparc/lib/VISsave.S +++ b/arch/sparc/lib/VISsave.S @@ -13,6 +13,7 @@ #include <asm/ptrace.h> #include <asm/visasm.h> #include <asm/thread_info.h> +#include <asm/export.h> /* On entry: %o5=current FPRS value, %g7 is callers address */ /* May clobber %o5, %g1, %g2, %g3, %g7, %icc, %xcc */ @@ -79,3 +80,4 @@ vis1: ldub [%g6 + TI_FPSAVED], %g3 80: jmpl %g7 + %g0, %g0 nop ENDPROC(VISenter) +EXPORT_SYMBOL(VISenter) diff --git a/arch/sparc/lib/ashldi3.S b/arch/sparc/lib/ashldi3.S index 86f60de07b0a..c8b1cf71bc73 100644 --- a/arch/sparc/lib/ashldi3.S +++ b/arch/sparc/lib/ashldi3.S @@ -6,6 +6,7 @@ */ #include <linux/linkage.h> +#include <asm/export.h> .text ENTRY(__ashldi3) @@ -33,3 +34,4 @@ ENTRY(__ashldi3) retl nop ENDPROC(__ashldi3) +EXPORT_SYMBOL(__ashldi3) diff --git a/arch/sparc/lib/ashrdi3.S b/arch/sparc/lib/ashrdi3.S index 6eb8ba2dd50e..4310256e7964 100644 --- a/arch/sparc/lib/ashrdi3.S +++ b/arch/sparc/lib/ashrdi3.S @@ -6,6 +6,7 @@ */ #include <linux/linkage.h> +#include <asm/export.h> .text ENTRY(__ashrdi3) @@ -35,3 +36,4 @@ ENTRY(__ashrdi3) jmpl %o7 + 8, %g0 nop ENDPROC(__ashrdi3) +EXPORT_SYMBOL(__ashrdi3) diff --git a/arch/sparc/lib/atomic32.c b/arch/sparc/lib/atomic32.c index b9d63c0a7aab..2c373329d5cb 100644 --- a/arch/sparc/lib/atomic32.c +++ b/arch/sparc/lib/atomic32.c @@ -27,39 +27,44 @@ static DEFINE_SPINLOCK(dummy); #endif /* SMP */ -#define ATOMIC_OP_RETURN(op, c_op) \ -int atomic_##op##_return(int i, atomic_t *v) \ +#define ATOMIC_FETCH_OP(op, c_op) \ +int atomic_fetch_##op(int i, atomic_t *v) \ { \ int ret; \ unsigned long flags; \ spin_lock_irqsave(ATOMIC_HASH(v), flags); \ \ - ret = (v->counter c_op i); \ + ret = v->counter; \ + v->counter c_op i; \ \ spin_unlock_irqrestore(ATOMIC_HASH(v), flags); \ return ret; \ } \ -EXPORT_SYMBOL(atomic_##op##_return); +EXPORT_SYMBOL(atomic_fetch_##op); -#define ATOMIC_OP(op, c_op) \ -void atomic_##op(int i, atomic_t *v) \ +#define ATOMIC_OP_RETURN(op, c_op) \ +int atomic_##op##_return(int i, atomic_t *v) \ { \ + int ret; \ unsigned long flags; \ spin_lock_irqsave(ATOMIC_HASH(v), flags); \ \ - v->counter c_op i; \ + ret = (v->counter c_op i); \ \ spin_unlock_irqrestore(ATOMIC_HASH(v), flags); \ + return ret; \ } \ -EXPORT_SYMBOL(atomic_##op); +EXPORT_SYMBOL(atomic_##op##_return); ATOMIC_OP_RETURN(add, +=) -ATOMIC_OP(and, &=) -ATOMIC_OP(or, |=) -ATOMIC_OP(xor, ^=) +ATOMIC_FETCH_OP(add, +=) +ATOMIC_FETCH_OP(and, &=) +ATOMIC_FETCH_OP(or, |=) +ATOMIC_FETCH_OP(xor, ^=) + +#undef ATOMIC_FETCH_OP #undef ATOMIC_OP_RETURN -#undef ATOMIC_OP int atomic_xchg(atomic_t *v, int new) { diff --git a/arch/sparc/lib/atomic_64.S b/arch/sparc/lib/atomic_64.S index d6b0363f345b..1c6a1bde5138 100644 --- a/arch/sparc/lib/atomic_64.S +++ b/arch/sparc/lib/atomic_64.S @@ -6,13 +6,15 @@ #include <linux/linkage.h> #include <asm/asi.h> #include <asm/backoff.h> +#include <asm/export.h> .text - /* Two versions of the atomic routines, one that + /* Three versions of the atomic routines, one that * does not return a value and does not perform - * memory barriers, and a second which returns - * a value and does the barriers. + * memory barriers, and a two which return + * a value, the new and old value resp. and does the + * barriers. */ #define ATOMIC_OP(op) \ @@ -28,6 +30,7 @@ ENTRY(atomic_##op) /* %o0 = increment, %o1 = atomic_ptr */ \ nop; \ 2: BACKOFF_SPIN(%o2, %o3, 1b); \ ENDPROC(atomic_##op); \ +EXPORT_SYMBOL(atomic_##op); #define ATOMIC_OP_RETURN(op) \ ENTRY(atomic_##op##_return) /* %o0 = increment, %o1 = atomic_ptr */ \ @@ -41,17 +44,38 @@ ENTRY(atomic_##op##_return) /* %o0 = increment, %o1 = atomic_ptr */ \ retl; \ sra %g1, 0, %o0; \ 2: BACKOFF_SPIN(%o2, %o3, 1b); \ -ENDPROC(atomic_##op##_return); +ENDPROC(atomic_##op##_return); \ +EXPORT_SYMBOL(atomic_##op##_return); -#define ATOMIC_OPS(op) ATOMIC_OP(op) ATOMIC_OP_RETURN(op) +#define ATOMIC_FETCH_OP(op) \ +ENTRY(atomic_fetch_##op) /* %o0 = increment, %o1 = atomic_ptr */ \ + BACKOFF_SETUP(%o2); \ +1: lduw [%o1], %g1; \ + op %g1, %o0, %g7; \ + cas [%o1], %g1, %g7; \ + cmp %g1, %g7; \ + bne,pn %icc, BACKOFF_LABEL(2f, 1b); \ + nop; \ + retl; \ + sra %g1, 0, %o0; \ +2: BACKOFF_SPIN(%o2, %o3, 1b); \ +ENDPROC(atomic_fetch_##op); \ +EXPORT_SYMBOL(atomic_fetch_##op); + +#define ATOMIC_OPS(op) ATOMIC_OP(op) ATOMIC_OP_RETURN(op) ATOMIC_FETCH_OP(op) ATOMIC_OPS(add) ATOMIC_OPS(sub) -ATOMIC_OP(and) -ATOMIC_OP(or) -ATOMIC_OP(xor) #undef ATOMIC_OPS +#define ATOMIC_OPS(op) ATOMIC_OP(op) ATOMIC_FETCH_OP(op) + +ATOMIC_OPS(and) +ATOMIC_OPS(or) +ATOMIC_OPS(xor) + +#undef ATOMIC_OPS +#undef ATOMIC_FETCH_OP #undef ATOMIC_OP_RETURN #undef ATOMIC_OP @@ -68,6 +92,7 @@ ENTRY(atomic64_##op) /* %o0 = increment, %o1 = atomic_ptr */ \ nop; \ 2: BACKOFF_SPIN(%o2, %o3, 1b); \ ENDPROC(atomic64_##op); \ +EXPORT_SYMBOL(atomic64_##op); #define ATOMIC64_OP_RETURN(op) \ ENTRY(atomic64_##op##_return) /* %o0 = increment, %o1 = atomic_ptr */ \ @@ -81,17 +106,38 @@ ENTRY(atomic64_##op##_return) /* %o0 = increment, %o1 = atomic_ptr */ \ retl; \ op %g1, %o0, %o0; \ 2: BACKOFF_SPIN(%o2, %o3, 1b); \ -ENDPROC(atomic64_##op##_return); +ENDPROC(atomic64_##op##_return); \ +EXPORT_SYMBOL(atomic64_##op##_return); -#define ATOMIC64_OPS(op) ATOMIC64_OP(op) ATOMIC64_OP_RETURN(op) +#define ATOMIC64_FETCH_OP(op) \ +ENTRY(atomic64_fetch_##op) /* %o0 = increment, %o1 = atomic_ptr */ \ + BACKOFF_SETUP(%o2); \ +1: ldx [%o1], %g1; \ + op %g1, %o0, %g7; \ + casx [%o1], %g1, %g7; \ + cmp %g1, %g7; \ + bne,pn %xcc, BACKOFF_LABEL(2f, 1b); \ + nop; \ + retl; \ + mov %g1, %o0; \ +2: BACKOFF_SPIN(%o2, %o3, 1b); \ +ENDPROC(atomic64_fetch_##op); \ +EXPORT_SYMBOL(atomic64_fetch_##op); + +#define ATOMIC64_OPS(op) ATOMIC64_OP(op) ATOMIC64_OP_RETURN(op) ATOMIC64_FETCH_OP(op) ATOMIC64_OPS(add) ATOMIC64_OPS(sub) -ATOMIC64_OP(and) -ATOMIC64_OP(or) -ATOMIC64_OP(xor) #undef ATOMIC64_OPS +#define ATOMIC64_OPS(op) ATOMIC64_OP(op) ATOMIC64_FETCH_OP(op) + +ATOMIC64_OPS(and) +ATOMIC64_OPS(or) +ATOMIC64_OPS(xor) + +#undef ATOMIC64_OPS +#undef ATOMIC64_FETCH_OP #undef ATOMIC64_OP_RETURN #undef ATOMIC64_OP @@ -108,3 +154,4 @@ ENTRY(atomic64_dec_if_positive) /* %o0 = atomic_ptr */ sub %g1, 1, %o0 2: BACKOFF_SPIN(%o2, %o3, 1b) ENDPROC(atomic64_dec_if_positive) +EXPORT_SYMBOL(atomic64_dec_if_positive) diff --git a/arch/sparc/lib/bitops.S b/arch/sparc/lib/bitops.S index 36f72cc0e67e..7031bf1587cb 100644 --- a/arch/sparc/lib/bitops.S +++ b/arch/sparc/lib/bitops.S @@ -6,6 +6,7 @@ #include <linux/linkage.h> #include <asm/asi.h> #include <asm/backoff.h> +#include <asm/export.h> .text @@ -29,6 +30,7 @@ ENTRY(test_and_set_bit) /* %o0=nr, %o1=addr */ nop 2: BACKOFF_SPIN(%o3, %o4, 1b) ENDPROC(test_and_set_bit) +EXPORT_SYMBOL(test_and_set_bit) ENTRY(test_and_clear_bit) /* %o0=nr, %o1=addr */ BACKOFF_SETUP(%o3) @@ -50,6 +52,7 @@ ENTRY(test_and_clear_bit) /* %o0=nr, %o1=addr */ nop 2: BACKOFF_SPIN(%o3, %o4, 1b) ENDPROC(test_and_clear_bit) +EXPORT_SYMBOL(test_and_clear_bit) ENTRY(test_and_change_bit) /* %o0=nr, %o1=addr */ BACKOFF_SETUP(%o3) @@ -71,6 +74,7 @@ ENTRY(test_and_change_bit) /* %o0=nr, %o1=addr */ nop 2: BACKOFF_SPIN(%o3, %o4, 1b) ENDPROC(test_and_change_bit) +EXPORT_SYMBOL(test_and_change_bit) ENTRY(set_bit) /* %o0=nr, %o1=addr */ BACKOFF_SETUP(%o3) @@ -90,6 +94,7 @@ ENTRY(set_bit) /* %o0=nr, %o1=addr */ nop 2: BACKOFF_SPIN(%o3, %o4, 1b) ENDPROC(set_bit) +EXPORT_SYMBOL(set_bit) ENTRY(clear_bit) /* %o0=nr, %o1=addr */ BACKOFF_SETUP(%o3) @@ -109,6 +114,7 @@ ENTRY(clear_bit) /* %o0=nr, %o1=addr */ nop 2: BACKOFF_SPIN(%o3, %o4, 1b) ENDPROC(clear_bit) +EXPORT_SYMBOL(clear_bit) ENTRY(change_bit) /* %o0=nr, %o1=addr */ BACKOFF_SETUP(%o3) @@ -128,3 +134,4 @@ ENTRY(change_bit) /* %o0=nr, %o1=addr */ nop 2: BACKOFF_SPIN(%o3, %o4, 1b) ENDPROC(change_bit) +EXPORT_SYMBOL(change_bit) diff --git a/arch/sparc/lib/blockops.S b/arch/sparc/lib/blockops.S index 3c771011ff4b..1f2692d59d18 100644 --- a/arch/sparc/lib/blockops.S +++ b/arch/sparc/lib/blockops.S @@ -6,6 +6,7 @@ #include <linux/linkage.h> #include <asm/page.h> +#include <asm/export.h> /* Zero out 64 bytes of memory at (buf + offset). * Assumes %g1 contains zero. @@ -64,6 +65,7 @@ ENTRY(bzero_1page) retl nop ENDPROC(bzero_1page) +EXPORT_SYMBOL(bzero_1page) ENTRY(__copy_1page) /* NOTE: If you change the number of insns of this routine, please check @@ -87,3 +89,4 @@ ENTRY(__copy_1page) retl nop ENDPROC(__copy_1page) +EXPORT_SYMBOL(__copy_1page) diff --git a/arch/sparc/lib/bzero.S b/arch/sparc/lib/bzero.S index 8c058114b649..3bb1914c4fa4 100644 --- a/arch/sparc/lib/bzero.S +++ b/arch/sparc/lib/bzero.S @@ -5,6 +5,7 @@ */ #include <linux/linkage.h> +#include <asm/export.h> .text @@ -78,6 +79,8 @@ __bzero_done: mov %o3, %o0 ENDPROC(__bzero) ENDPROC(memset) +EXPORT_SYMBOL(__bzero) +EXPORT_SYMBOL(memset) #define EX_ST(x,y) \ 98: x,y; \ @@ -143,3 +146,4 @@ __clear_user_done: retl clr %o0 ENDPROC(__clear_user) +EXPORT_SYMBOL(__clear_user) diff --git a/arch/sparc/lib/checksum_32.S b/arch/sparc/lib/checksum_32.S index 0084c3361e15..c9d8b6232111 100644 --- a/arch/sparc/lib/checksum_32.S +++ b/arch/sparc/lib/checksum_32.S @@ -14,6 +14,7 @@ */ #include <asm/errno.h> +#include <asm/export.h> #define CSUM_BIGCHUNK(buf, offset, sum, t0, t1, t2, t3, t4, t5) \ ldd [buf + offset + 0x00], t0; \ @@ -104,6 +105,7 @@ csum_partial_fix_alignment: * buffer of size 0x20. Follow the code path for that case. */ .globl csum_partial + EXPORT_SYMBOL(csum_partial) csum_partial: /* %o0=buf, %o1=len, %o2=sum */ andcc %o0, 0x7, %g0 ! alignment problems? bne csum_partial_fix_alignment ! yep, handle it @@ -335,6 +337,7 @@ cc_dword_align: */ .align 8 .globl __csum_partial_copy_sparc_generic + EXPORT_SYMBOL(__csum_partial_copy_sparc_generic) __csum_partial_copy_sparc_generic: /* %o0=src, %o1=dest, %g1=len, %g7=sum */ xor %o0, %o1, %o4 ! get changing bits diff --git a/arch/sparc/lib/checksum_64.S b/arch/sparc/lib/checksum_64.S index 1d230f693dc4..f6732174fe6b 100644 --- a/arch/sparc/lib/checksum_64.S +++ b/arch/sparc/lib/checksum_64.S @@ -13,6 +13,7 @@ * BSD4.4 portable checksum routine */ +#include <asm/export.h> .text csum_partial_fix_alignment: @@ -37,6 +38,7 @@ csum_partial_fix_alignment: .align 32 .globl csum_partial + EXPORT_SYMBOL(csum_partial) csum_partial: /* %o0=buff, %o1=len, %o2=sum */ prefetch [%o0 + 0x000], #n_reads clr %o4 diff --git a/arch/sparc/lib/clear_page.S b/arch/sparc/lib/clear_page.S index 46272dfc26e8..f30d6b78afbd 100644 --- a/arch/sparc/lib/clear_page.S +++ b/arch/sparc/lib/clear_page.S @@ -10,6 +10,7 @@ #include <asm/pgtable.h> #include <asm/spitfire.h> #include <asm/head.h> +#include <asm/export.h> /* What we used to do was lock a TLB entry into a specific * TLB slot, clear the page with interrupts disabled, then @@ -26,6 +27,7 @@ .text .globl _clear_page + EXPORT_SYMBOL(_clear_page) _clear_page: /* %o0=dest */ ba,pt %xcc, clear_page_common clr %o4 @@ -35,6 +37,7 @@ _clear_page: /* %o0=dest */ */ .align 32 .globl clear_user_page + EXPORT_SYMBOL(clear_user_page) clear_user_page: /* %o0=dest, %o1=vaddr */ lduw [%g6 + TI_PRE_COUNT], %o2 sethi %hi(PAGE_OFFSET), %g2 diff --git a/arch/sparc/lib/copy_in_user.S b/arch/sparc/lib/copy_in_user.S index 302c0e60dc2c..482de093bdae 100644 --- a/arch/sparc/lib/copy_in_user.S +++ b/arch/sparc/lib/copy_in_user.S @@ -5,6 +5,7 @@ #include <linux/linkage.h> #include <asm/asi.h> +#include <asm/export.h> #define XCC xcc @@ -90,3 +91,4 @@ ENTRY(___copy_in_user) /* %o0=dst, %o1=src, %o2=len */ retl clr %o0 ENDPROC(___copy_in_user) +EXPORT_SYMBOL(___copy_in_user) diff --git a/arch/sparc/lib/copy_page.S b/arch/sparc/lib/copy_page.S index dd16c61f3263..7197b7250895 100644 --- a/arch/sparc/lib/copy_page.S +++ b/arch/sparc/lib/copy_page.S @@ -10,6 +10,7 @@ #include <asm/pgtable.h> #include <asm/spitfire.h> #include <asm/head.h> +#include <asm/export.h> /* What we used to do was lock a TLB entry into a specific * TLB slot, clear the page with interrupts disabled, then @@ -44,6 +45,7 @@ .align 32 .globl copy_user_page .type copy_user_page,#function + EXPORT_SYMBOL(copy_user_page) copy_user_page: /* %o0=dest, %o1=src, %o2=vaddr */ lduw [%g6 + TI_PRE_COUNT], %o4 sethi %hi(PAGE_OFFSET), %g2 diff --git a/arch/sparc/lib/copy_user.S b/arch/sparc/lib/copy_user.S index ef095b6c43b1..cea644dc67a6 100644 --- a/arch/sparc/lib/copy_user.S +++ b/arch/sparc/lib/copy_user.S @@ -15,6 +15,7 @@ #include <asm/asmmacro.h> #include <asm/page.h> #include <asm/thread_info.h> +#include <asm/export.h> /* Work around cpp -rob */ #define ALLOC #alloc @@ -119,6 +120,7 @@ __copy_user_begin: .globl __copy_user + EXPORT_SYMBOL(__copy_user) dword_align: andcc %o1, 1, %g0 be 4f diff --git a/arch/sparc/lib/csum_copy.S b/arch/sparc/lib/csum_copy.S index e566c770a0f6..0ecbafc30fd0 100644 --- a/arch/sparc/lib/csum_copy.S +++ b/arch/sparc/lib/csum_copy.S @@ -3,6 +3,8 @@ * Copyright (C) 2005 David S. Miller <davem@davemloft.net> */ +#include <asm/export.h> + #ifdef __KERNEL__ #define GLOBAL_SPARE %g7 #else @@ -63,6 +65,7 @@ add %o5, %o4, %o4 .globl FUNC_NAME + EXPORT_SYMBOL(FUNC_NAME) FUNC_NAME: /* %o0=src, %o1=dst, %o2=len, %o3=sum */ LOAD(prefetch, %o0 + 0x000, #n_reads) xor %o0, %o1, %g1 diff --git a/arch/sparc/lib/divdi3.S b/arch/sparc/lib/divdi3.S index 9614b48b6ef8..a2b5a976be33 100644 --- a/arch/sparc/lib/divdi3.S +++ b/arch/sparc/lib/divdi3.S @@ -17,6 +17,7 @@ along with GNU CC; see the file COPYING. If not, write to the Free Software Foundation, 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. */ +#include <asm/export.h> .text .align 4 .globl __divdi3 @@ -279,3 +280,4 @@ __divdi3: .LL81: ret restore +EXPORT_SYMBOL(__divdi3) diff --git a/arch/sparc/lib/ffs.S b/arch/sparc/lib/ffs.S index b39389f69899..23aab144d28e 100644 --- a/arch/sparc/lib/ffs.S +++ b/arch/sparc/lib/ffs.S @@ -1,4 +1,5 @@ #include <linux/linkage.h> +#include <asm/export.h> .register %g2,#scratch @@ -65,6 +66,8 @@ ENTRY(__ffs) add %o2, %g1, %o0 ENDPROC(ffs) ENDPROC(__ffs) +EXPORT_SYMBOL(__ffs) +EXPORT_SYMBOL(ffs) .section .popc_6insn_patch, "ax" .word ffs diff --git a/arch/sparc/lib/hweight.S b/arch/sparc/lib/hweight.S index 95414e0a6808..f9985f129fb6 100644 --- a/arch/sparc/lib/hweight.S +++ b/arch/sparc/lib/hweight.S @@ -1,4 +1,5 @@ #include <linux/linkage.h> +#include <asm/export.h> .text .align 32 @@ -7,6 +8,7 @@ ENTRY(__arch_hweight8) nop nop ENDPROC(__arch_hweight8) +EXPORT_SYMBOL(__arch_hweight8) .section .popc_3insn_patch, "ax" .word __arch_hweight8 sllx %o0, 64-8, %g1 @@ -19,6 +21,7 @@ ENTRY(__arch_hweight16) nop nop ENDPROC(__arch_hweight16) +EXPORT_SYMBOL(__arch_hweight16) .section .popc_3insn_patch, "ax" .word __arch_hweight16 sllx %o0, 64-16, %g1 @@ -31,6 +34,7 @@ ENTRY(__arch_hweight32) nop nop ENDPROC(__arch_hweight32) +EXPORT_SYMBOL(__arch_hweight32) .section .popc_3insn_patch, "ax" .word __arch_hweight32 sllx %o0, 64-32, %g1 @@ -43,6 +47,7 @@ ENTRY(__arch_hweight64) nop nop ENDPROC(__arch_hweight64) +EXPORT_SYMBOL(__arch_hweight64) .section .popc_3insn_patch, "ax" .word __arch_hweight64 retl diff --git a/arch/sparc/lib/ipcsum.S b/arch/sparc/lib/ipcsum.S index 4742d59029ee..5d61648b53dd 100644 --- a/arch/sparc/lib/ipcsum.S +++ b/arch/sparc/lib/ipcsum.S @@ -1,4 +1,5 @@ #include <linux/linkage.h> +#include <asm/export.h> .text ENTRY(ip_fast_csum) /* %o0 = iph, %o1 = ihl */ @@ -31,3 +32,4 @@ ENTRY(ip_fast_csum) /* %o0 = iph, %o1 = ihl */ retl and %o2, %o1, %o0 ENDPROC(ip_fast_csum) +EXPORT_SYMBOL(ip_fast_csum) diff --git a/arch/sparc/lib/ksyms.c b/arch/sparc/lib/ksyms.c deleted file mode 100644 index 8eb454cfe05c..000000000000 --- a/arch/sparc/lib/ksyms.c +++ /dev/null @@ -1,165 +0,0 @@ -/* - * Export of symbols defined in assembler - */ - -/* Tell string.h we don't want memcpy etc. as cpp defines */ -#define EXPORT_SYMTAB_STROPS - -#include <linux/module.h> -#include <linux/string.h> -#include <linux/types.h> - -#include <asm/checksum.h> -#include <asm/uaccess.h> -#include <asm/ftrace.h> - -/* string functions */ -EXPORT_SYMBOL(strlen); -EXPORT_SYMBOL(strncmp); - -/* mem* functions */ -extern void *__memscan_zero(void *, size_t); -extern void *__memscan_generic(void *, int, size_t); -extern void *__bzero(void *, size_t); - -EXPORT_SYMBOL(memscan); -EXPORT_SYMBOL(__memscan_zero); -EXPORT_SYMBOL(__memscan_generic); -EXPORT_SYMBOL(memcmp); -EXPORT_SYMBOL(memcpy); -EXPORT_SYMBOL(memset); -EXPORT_SYMBOL(memmove); -EXPORT_SYMBOL(__bzero); - -/* Networking helper routines. */ -EXPORT_SYMBOL(csum_partial); - -#ifdef CONFIG_MCOUNT -EXPORT_SYMBOL(_mcount); -#endif - -/* - * sparc - */ -#ifdef CONFIG_SPARC32 -extern int __ashrdi3(int, int); -extern int __ashldi3(int, int); -extern int __lshrdi3(int, int); -extern int __muldi3(int, int); -extern int __divdi3(int, int); - -extern void (*__copy_1page)(void *, const void *); -extern void (*bzero_1page)(void *); - -extern void ___rw_read_enter(void); -extern void ___rw_read_try(void); -extern void ___rw_read_exit(void); -extern void ___rw_write_enter(void); - -/* Networking helper routines. */ -EXPORT_SYMBOL(__csum_partial_copy_sparc_generic); - -/* Special internal versions of library functions. */ -EXPORT_SYMBOL(__copy_1page); -EXPORT_SYMBOL(__memmove); -EXPORT_SYMBOL(bzero_1page); - -/* Moving data to/from/in userspace. */ -EXPORT_SYMBOL(__copy_user); - -/* Used by asm/spinlock.h */ -#ifdef CONFIG_SMP -EXPORT_SYMBOL(___rw_read_enter); -EXPORT_SYMBOL(___rw_read_try); -EXPORT_SYMBOL(___rw_read_exit); -EXPORT_SYMBOL(___rw_write_enter); -#endif - -EXPORT_SYMBOL(__ashrdi3); -EXPORT_SYMBOL(__ashldi3); -EXPORT_SYMBOL(__lshrdi3); -EXPORT_SYMBOL(__muldi3); -EXPORT_SYMBOL(__divdi3); -#endif - -/* - * sparc64 - */ -#ifdef CONFIG_SPARC64 -/* Networking helper routines. */ -EXPORT_SYMBOL(csum_partial_copy_nocheck); -EXPORT_SYMBOL(__csum_partial_copy_from_user); -EXPORT_SYMBOL(__csum_partial_copy_to_user); -EXPORT_SYMBOL(ip_fast_csum); - -/* Moving data to/from/in userspace. */ -EXPORT_SYMBOL(___copy_to_user); -EXPORT_SYMBOL(___copy_from_user); -EXPORT_SYMBOL(___copy_in_user); -EXPORT_SYMBOL(__clear_user); - -/* Atomic counter implementation. */ -#define ATOMIC_OP(op) \ -EXPORT_SYMBOL(atomic_##op); \ -EXPORT_SYMBOL(atomic64_##op); - -#define ATOMIC_OP_RETURN(op) \ -EXPORT_SYMBOL(atomic_##op##_return); \ -EXPORT_SYMBOL(atomic64_##op##_return); - -#define ATOMIC_OPS(op) ATOMIC_OP(op) ATOMIC_OP_RETURN(op) - -ATOMIC_OPS(add) -ATOMIC_OPS(sub) -ATOMIC_OP(and) -ATOMIC_OP(or) -ATOMIC_OP(xor) - -#undef ATOMIC_OPS -#undef ATOMIC_OP_RETURN -#undef ATOMIC_OP - -EXPORT_SYMBOL(atomic64_dec_if_positive); - -/* Atomic bit operations. */ -EXPORT_SYMBOL(test_and_set_bit); -EXPORT_SYMBOL(test_and_clear_bit); -EXPORT_SYMBOL(test_and_change_bit); -EXPORT_SYMBOL(set_bit); -EXPORT_SYMBOL(clear_bit); -EXPORT_SYMBOL(change_bit); - -/* Special internal versions of library functions. */ -EXPORT_SYMBOL(_clear_page); -EXPORT_SYMBOL(clear_user_page); -EXPORT_SYMBOL(copy_user_page); - -/* RAID code needs this */ -void VISenter(void); -EXPORT_SYMBOL(VISenter); - -extern void xor_vis_2(unsigned long, unsigned long *, unsigned long *); -extern void xor_vis_3(unsigned long, unsigned long *, unsigned long *, - unsigned long *); -extern void xor_vis_4(unsigned long, unsigned long *, unsigned long *, - unsigned long *, unsigned long *); -extern void xor_vis_5(unsigned long, unsigned long *, unsigned long *, - unsigned long *, unsigned long *, unsigned long *); -EXPORT_SYMBOL(xor_vis_2); -EXPORT_SYMBOL(xor_vis_3); -EXPORT_SYMBOL(xor_vis_4); -EXPORT_SYMBOL(xor_vis_5); - -extern void xor_niagara_2(unsigned long, unsigned long *, unsigned long *); -extern void xor_niagara_3(unsigned long, unsigned long *, unsigned long *, - unsigned long *); -extern void xor_niagara_4(unsigned long, unsigned long *, unsigned long *, - unsigned long *, unsigned long *); -extern void xor_niagara_5(unsigned long, unsigned long *, unsigned long *, - unsigned long *, unsigned long *, unsigned long *); - -EXPORT_SYMBOL(xor_niagara_2); -EXPORT_SYMBOL(xor_niagara_3); -EXPORT_SYMBOL(xor_niagara_4); -EXPORT_SYMBOL(xor_niagara_5); -#endif diff --git a/arch/sparc/lib/locks.S b/arch/sparc/lib/locks.S index 64f53f2b673d..f38c4e59d078 100644 --- a/arch/sparc/lib/locks.S +++ b/arch/sparc/lib/locks.S @@ -10,6 +10,7 @@ #include <asm/psr.h> #include <asm/smp.h> #include <asm/spinlock.h> +#include <asm/export.h> .text .align 4 @@ -48,6 +49,7 @@ ___rw_write_enter_spin_on_wlock: ld [%g1], %g2 .globl ___rw_read_enter +EXPORT_SYMBOL(___rw_read_enter) ___rw_read_enter: orcc %g2, 0x0, %g0 bne,a ___rw_read_enter_spin_on_wlock @@ -59,6 +61,7 @@ ___rw_read_enter: mov %g4, %o7 .globl ___rw_read_exit +EXPORT_SYMBOL(___rw_read_exit) ___rw_read_exit: orcc %g2, 0x0, %g0 bne,a ___rw_read_exit_spin_on_wlock @@ -70,6 +73,7 @@ ___rw_read_exit: mov %g4, %o7 .globl ___rw_read_try +EXPORT_SYMBOL(___rw_read_try) ___rw_read_try: orcc %g2, 0x0, %g0 bne ___rw_read_try_spin_on_wlock @@ -81,6 +85,7 @@ ___rw_read_try: mov %g4, %o7 .globl ___rw_write_enter +EXPORT_SYMBOL(___rw_write_enter) ___rw_write_enter: orcc %g2, 0x0, %g0 bne ___rw_write_enter_spin_on_wlock diff --git a/arch/sparc/lib/lshrdi3.S b/arch/sparc/lib/lshrdi3.S index 60ebc7cdbee0..c9b9373f8d81 100644 --- a/arch/sparc/lib/lshrdi3.S +++ b/arch/sparc/lib/lshrdi3.S @@ -1,4 +1,5 @@ #include <linux/linkage.h> +#include <asm/export.h> ENTRY(__lshrdi3) cmp %o2, 0 @@ -25,3 +26,4 @@ ENTRY(__lshrdi3) retl nop ENDPROC(__lshrdi3) +EXPORT_SYMBOL(__lshrdi3) diff --git a/arch/sparc/lib/mcount.S b/arch/sparc/lib/mcount.S index 0b0ed4d34219..194f383611c0 100644 --- a/arch/sparc/lib/mcount.S +++ b/arch/sparc/lib/mcount.S @@ -6,6 +6,7 @@ */ #include <linux/linkage.h> +#include <asm/export.h> /* * This is the main variant and is called by C code. GCC's -pg option @@ -16,6 +17,7 @@ .align 32 .globl _mcount .type _mcount,#function + EXPORT_SYMBOL(_mcount) .globl mcount .type mcount,#function _mcount: diff --git a/arch/sparc/lib/memcmp.S b/arch/sparc/lib/memcmp.S index efa106c41ed0..cee7f30dbb61 100644 --- a/arch/sparc/lib/memcmp.S +++ b/arch/sparc/lib/memcmp.S @@ -6,6 +6,7 @@ #include <linux/linkage.h> #include <asm/asm.h> +#include <asm/export.h> .text ENTRY(memcmp) @@ -25,3 +26,4 @@ ENTRY(memcmp) 2: retl mov 0, %o0 ENDPROC(memcmp) +EXPORT_SYMBOL(memcmp) diff --git a/arch/sparc/lib/memcpy.S b/arch/sparc/lib/memcpy.S index 4d8c497517bd..8913feaa7ac7 100644 --- a/arch/sparc/lib/memcpy.S +++ b/arch/sparc/lib/memcpy.S @@ -7,6 +7,7 @@ * Copyright (C) 1996 Jakub Jelinek (jj@sunsite.mff.cuni.cz) */ +#include <asm/export.h> #define FUNC(x) \ .globl x; \ .type x,@function; \ @@ -58,93 +59,11 @@ x: stb %t0, [%dst - (offset) - 0x02]; \ stb %t1, [%dst - (offset) - 0x01]; -/* Both these macros have to start with exactly the same insn */ -#define RMOVE_BIGCHUNK(src, dst, offset, t0, t1, t2, t3, t4, t5, t6, t7) \ - ldd [%src - (offset) - 0x20], %t0; \ - ldd [%src - (offset) - 0x18], %t2; \ - ldd [%src - (offset) - 0x10], %t4; \ - ldd [%src - (offset) - 0x08], %t6; \ - st %t0, [%dst - (offset) - 0x20]; \ - st %t1, [%dst - (offset) - 0x1c]; \ - st %t2, [%dst - (offset) - 0x18]; \ - st %t3, [%dst - (offset) - 0x14]; \ - st %t4, [%dst - (offset) - 0x10]; \ - st %t5, [%dst - (offset) - 0x0c]; \ - st %t6, [%dst - (offset) - 0x08]; \ - st %t7, [%dst - (offset) - 0x04]; - -#define RMOVE_BIGALIGNCHUNK(src, dst, offset, t0, t1, t2, t3, t4, t5, t6, t7) \ - ldd [%src - (offset) - 0x20], %t0; \ - ldd [%src - (offset) - 0x18], %t2; \ - ldd [%src - (offset) - 0x10], %t4; \ - ldd [%src - (offset) - 0x08], %t6; \ - std %t0, [%dst - (offset) - 0x20]; \ - std %t2, [%dst - (offset) - 0x18]; \ - std %t4, [%dst - (offset) - 0x10]; \ - std %t6, [%dst - (offset) - 0x08]; - -#define RMOVE_LASTCHUNK(src, dst, offset, t0, t1, t2, t3) \ - ldd [%src + (offset) + 0x00], %t0; \ - ldd [%src + (offset) + 0x08], %t2; \ - st %t0, [%dst + (offset) + 0x00]; \ - st %t1, [%dst + (offset) + 0x04]; \ - st %t2, [%dst + (offset) + 0x08]; \ - st %t3, [%dst + (offset) + 0x0c]; - -#define RMOVE_SHORTCHUNK(src, dst, offset, t0, t1) \ - ldub [%src + (offset) + 0x00], %t0; \ - ldub [%src + (offset) + 0x01], %t1; \ - stb %t0, [%dst + (offset) + 0x00]; \ - stb %t1, [%dst + (offset) + 0x01]; - -#define SMOVE_CHUNK(src, dst, offset, t0, t1, t2, t3, t4, t5, t6, prev, shil, shir, offset2) \ - ldd [%src + (offset) + 0x00], %t0; \ - ldd [%src + (offset) + 0x08], %t2; \ - srl %t0, shir, %t5; \ - srl %t1, shir, %t6; \ - sll %t0, shil, %t0; \ - or %t5, %prev, %t5; \ - sll %t1, shil, %prev; \ - or %t6, %t0, %t0; \ - srl %t2, shir, %t1; \ - srl %t3, shir, %t6; \ - sll %t2, shil, %t2; \ - or %t1, %prev, %t1; \ - std %t4, [%dst + (offset) + (offset2) - 0x04]; \ - std %t0, [%dst + (offset) + (offset2) + 0x04]; \ - sll %t3, shil, %prev; \ - or %t6, %t2, %t4; - -#define SMOVE_ALIGNCHUNK(src, dst, offset, t0, t1, t2, t3, t4, t5, t6, prev, shil, shir, offset2) \ - ldd [%src + (offset) + 0x00], %t0; \ - ldd [%src + (offset) + 0x08], %t2; \ - srl %t0, shir, %t4; \ - srl %t1, shir, %t5; \ - sll %t0, shil, %t6; \ - or %t4, %prev, %t0; \ - sll %t1, shil, %prev; \ - or %t5, %t6, %t1; \ - srl %t2, shir, %t4; \ - srl %t3, shir, %t5; \ - sll %t2, shil, %t6; \ - or %t4, %prev, %t2; \ - sll %t3, shil, %prev; \ - or %t5, %t6, %t3; \ - std %t0, [%dst + (offset) + (offset2) + 0x00]; \ - std %t2, [%dst + (offset) + (offset2) + 0x08]; - .text .align 4 -0: - retl - nop ! Only bcopy returns here and it retuns void... - -#ifdef __KERNEL__ -FUNC(amemmove) -FUNC(__memmove) -#endif FUNC(memmove) +EXPORT_SYMBOL(memmove) cmp %o0, %o1 mov %o0, %g7 bleu 9f @@ -202,6 +121,7 @@ FUNC(memmove) add %o0, 2, %o0 FUNC(memcpy) /* %o0=dst %o1=src %o2=len */ +EXPORT_SYMBOL(memcpy) sub %o0, %o1, %o4 mov %o0, %g7 diff --git a/arch/sparc/lib/memmove.S b/arch/sparc/lib/memmove.S index 857ad4f8905f..012cdb6ca467 100644 --- a/arch/sparc/lib/memmove.S +++ b/arch/sparc/lib/memmove.S @@ -5,6 +5,7 @@ */ #include <linux/linkage.h> +#include <asm/export.h> .text ENTRY(memmove) /* o0=dst o1=src o2=len */ @@ -57,3 +58,4 @@ ENTRY(memmove) /* o0=dst o1=src o2=len */ stb %g7, [%o0 - 0x1] ba,a,pt %xcc, 99b ENDPROC(memmove) +EXPORT_SYMBOL(memmove) diff --git a/arch/sparc/lib/memscan_32.S b/arch/sparc/lib/memscan_32.S index 4ff1657dfc24..51ce690c42a8 100644 --- a/arch/sparc/lib/memscan_32.S +++ b/arch/sparc/lib/memscan_32.S @@ -4,6 +4,8 @@ * Copyright (C) 1996 David S. Miller (davem@caip.rutgers.edu) */ +#include <asm/export.h> + /* In essence, this is just a fancy strlen. */ #define LO_MAGIC 0x01010101 @@ -13,6 +15,8 @@ .align 4 .globl __memscan_zero, __memscan_generic .globl memscan +EXPORT_SYMBOL(__memscan_zero) +EXPORT_SYMBOL(__memscan_generic) __memscan_zero: /* %o0 = addr, %o1 = size */ cmp %o1, 0 diff --git a/arch/sparc/lib/memscan_64.S b/arch/sparc/lib/memscan_64.S index 5686dfa5dc15..daa96f4b03e6 100644 --- a/arch/sparc/lib/memscan_64.S +++ b/arch/sparc/lib/memscan_64.S @@ -5,6 +5,8 @@ * Copyright (C) 1998 David S. Miller (davem@redhat.com) */ + #include <asm/export.h> + #define HI_MAGIC 0x8080808080808080 #define LO_MAGIC 0x0101010101010101 #define ASI_PL 0x88 @@ -13,6 +15,8 @@ .align 32 .globl __memscan_zero, __memscan_generic .globl memscan + EXPORT_SYMBOL(__memscan_zero) + EXPORT_SYMBOL(__memscan_generic) __memscan_zero: /* %o0 = bufp, %o1 = size */ diff --git a/arch/sparc/lib/memset.S b/arch/sparc/lib/memset.S index f75e6906df14..bb539b42b088 100644 --- a/arch/sparc/lib/memset.S +++ b/arch/sparc/lib/memset.S @@ -9,6 +9,7 @@ */ #include <asm/ptrace.h> +#include <asm/export.h> /* Work around cpp -rob */ #define ALLOC #alloc @@ -63,6 +64,8 @@ __bzero_begin: .globl __bzero .globl memset + EXPORT_SYMBOL(__bzero) + EXPORT_SYMBOL(memset) .globl __memset_start, __memset_end __memset_start: memset: diff --git a/arch/sparc/lib/muldi3.S b/arch/sparc/lib/muldi3.S index 9794939d1c12..17a0f49aef3c 100644 --- a/arch/sparc/lib/muldi3.S +++ b/arch/sparc/lib/muldi3.S @@ -17,6 +17,7 @@ along with GNU CC; see the file COPYING. If not, write to the Free Software Foundation, 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. */ +#include <asm/export.h> .text .align 4 .globl __muldi3 @@ -74,3 +75,4 @@ __muldi3: add %l2, %l0, %i0 ret restore %g0, %l3, %o1 +EXPORT_SYMBOL(__muldi3) diff --git a/arch/sparc/lib/strlen.S b/arch/sparc/lib/strlen.S index 536f83507fbf..ca0e7077e871 100644 --- a/arch/sparc/lib/strlen.S +++ b/arch/sparc/lib/strlen.S @@ -7,6 +7,7 @@ #include <linux/linkage.h> #include <asm/asm.h> +#include <asm/export.h> #define LO_MAGIC 0x01010101 #define HI_MAGIC 0x80808080 @@ -78,3 +79,4 @@ ENTRY(strlen) retl mov 2, %o0 ENDPROC(strlen) +EXPORT_SYMBOL(strlen) diff --git a/arch/sparc/lib/strncmp_32.S b/arch/sparc/lib/strncmp_32.S index c0d1b568c1c5..e3fe014813af 100644 --- a/arch/sparc/lib/strncmp_32.S +++ b/arch/sparc/lib/strncmp_32.S @@ -4,6 +4,7 @@ */ #include <linux/linkage.h> +#include <asm/export.h> .text ENTRY(strncmp) @@ -116,3 +117,4 @@ ENTRY(strncmp) retl sub %o3, %o0, %o0 ENDPROC(strncmp) +EXPORT_SYMBOL(strncmp) diff --git a/arch/sparc/lib/strncmp_64.S b/arch/sparc/lib/strncmp_64.S index 0656627166f3..efb5f884330d 100644 --- a/arch/sparc/lib/strncmp_64.S +++ b/arch/sparc/lib/strncmp_64.S @@ -6,6 +6,7 @@ #include <linux/linkage.h> #include <asm/asi.h> +#include <asm/export.h> .text ENTRY(strncmp) @@ -28,3 +29,4 @@ ENTRY(strncmp) retl clr %o0 ENDPROC(strncmp) +EXPORT_SYMBOL(strncmp) diff --git a/arch/sparc/lib/xor.S b/arch/sparc/lib/xor.S index 2c05641c3263..45a49cb618b5 100644 --- a/arch/sparc/lib/xor.S +++ b/arch/sparc/lib/xor.S @@ -13,6 +13,7 @@ #include <asm/asi.h> #include <asm/dcu.h> #include <asm/spitfire.h> +#include <asm/export.h> /* * Requirements: @@ -90,6 +91,7 @@ ENTRY(xor_vis_2) retl wr %g0, 0, %fprs ENDPROC(xor_vis_2) +EXPORT_SYMBOL(xor_vis_2) ENTRY(xor_vis_3) rd %fprs, %o5 @@ -156,6 +158,7 @@ ENTRY(xor_vis_3) retl wr %g0, 0, %fprs ENDPROC(xor_vis_3) +EXPORT_SYMBOL(xor_vis_3) ENTRY(xor_vis_4) rd %fprs, %o5 @@ -241,6 +244,7 @@ ENTRY(xor_vis_4) retl wr %g0, 0, %fprs ENDPROC(xor_vis_4) +EXPORT_SYMBOL(xor_vis_4) ENTRY(xor_vis_5) save %sp, -192, %sp @@ -347,6 +351,7 @@ ENTRY(xor_vis_5) ret restore ENDPROC(xor_vis_5) +EXPORT_SYMBOL(xor_vis_5) /* Niagara versions. */ ENTRY(xor_niagara_2) /* %o0=bytes, %o1=dest, %o2=src */ @@ -393,6 +398,7 @@ ENTRY(xor_niagara_2) /* %o0=bytes, %o1=dest, %o2=src */ ret restore ENDPROC(xor_niagara_2) +EXPORT_SYMBOL(xor_niagara_2) ENTRY(xor_niagara_3) /* %o0=bytes, %o1=dest, %o2=src1, %o3=src2 */ save %sp, -192, %sp @@ -454,6 +460,7 @@ ENTRY(xor_niagara_3) /* %o0=bytes, %o1=dest, %o2=src1, %o3=src2 */ ret restore ENDPROC(xor_niagara_3) +EXPORT_SYMBOL(xor_niagara_3) ENTRY(xor_niagara_4) /* %o0=bytes, %o1=dest, %o2=src1, %o3=src2, %o4=src3 */ save %sp, -192, %sp @@ -536,6 +543,7 @@ ENTRY(xor_niagara_4) /* %o0=bytes, %o1=dest, %o2=src1, %o3=src2, %o4=src3 */ ret restore ENDPROC(xor_niagara_4) +EXPORT_SYMBOL(xor_niagara_4) ENTRY(xor_niagara_5) /* %o0=bytes, %o1=dest, %o2=src1, %o3=src2, %o4=src3, %o5=src4 */ save %sp, -192, %sp @@ -634,3 +642,4 @@ ENTRY(xor_niagara_5) /* %o0=bytes, %o1=dest, %o2=src1, %o3=src2, %o4=src3, %o5=s ret restore ENDPROC(xor_niagara_5) +EXPORT_SYMBOL(xor_niagara_5) diff --git a/arch/sparc/mm/fault_32.c b/arch/sparc/mm/fault_32.c index b6c559cbd64d..4714061d6cd3 100644 --- a/arch/sparc/mm/fault_32.c +++ b/arch/sparc/mm/fault_32.c @@ -241,7 +241,7 @@ good_area: * make sure we exit gracefully rather than endlessly redo * the fault. */ - fault = handle_mm_fault(mm, vma, address, flags); + fault = handle_mm_fault(vma, address, flags); if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current)) return; @@ -411,7 +411,7 @@ good_area: if (!(vma->vm_flags & (VM_READ | VM_EXEC))) goto bad_area; } - switch (handle_mm_fault(mm, vma, address, flags)) { + switch (handle_mm_fault(vma, address, flags)) { case VM_FAULT_SIGBUS: case VM_FAULT_OOM: goto do_sigbus; diff --git a/arch/sparc/mm/fault_64.c b/arch/sparc/mm/fault_64.c index cb841a33da59..643c149a3151 100644 --- a/arch/sparc/mm/fault_64.c +++ b/arch/sparc/mm/fault_64.c @@ -14,7 +14,7 @@ #include <linux/mman.h> #include <linux/signal.h> #include <linux/mm.h> -#include <linux/module.h> +#include <linux/extable.h> #include <linux/init.h> #include <linux/perf_event.h> #include <linux/interrupt.h> @@ -111,8 +111,8 @@ static unsigned int get_user_insn(unsigned long tpc) if (pmd_none(*pmdp) || unlikely(pmd_bad(*pmdp))) goto out_irq_enable; -#ifdef CONFIG_TRANSPARENT_HUGEPAGE - if (pmd_trans_huge(*pmdp)) { +#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE) + if (is_hugetlb_pmd(*pmdp)) { pa = pmd_pfn(*pmdp) << PAGE_SHIFT; pa += tpc & ~HPAGE_MASK; @@ -436,7 +436,7 @@ good_area: goto bad_area; } - fault = handle_mm_fault(mm, vma, address, flags); + fault = handle_mm_fault(vma, address, flags); if ((fault & VM_FAULT_RETRY) && fatal_signal_pending(current)) goto exit_exception; @@ -476,14 +476,15 @@ good_area: up_read(&mm->mmap_sem); mm_rss = get_mm_rss(mm); -#if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE) - mm_rss -= (mm->context.huge_pte_count * (HPAGE_SIZE / PAGE_SIZE)); +#if defined(CONFIG_TRANSPARENT_HUGEPAGE) + mm_rss -= (mm->context.thp_pte_count * (HPAGE_SIZE / PAGE_SIZE)); #endif if (unlikely(mm_rss > mm->context.tsb_block[MM_TSB_BASE].tsb_rss_limit)) tsb_grow(mm, MM_TSB_BASE, mm_rss); #if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE) - mm_rss = mm->context.huge_pte_count; + mm_rss = mm->context.hugetlb_pte_count + mm->context.thp_pte_count; + mm_rss *= REAL_HPAGE_PER_HPAGE; if (unlikely(mm_rss > mm->context.tsb_block[MM_TSB_HUGE].tsb_rss_limit)) { if (mm->context.tsb_block[MM_TSB_HUGE].tsb) diff --git a/arch/sparc/mm/hugetlbpage.c b/arch/sparc/mm/hugetlbpage.c index ba52e6466a82..988acc8b1b80 100644 --- a/arch/sparc/mm/hugetlbpage.c +++ b/arch/sparc/mm/hugetlbpage.c @@ -12,6 +12,7 @@ #include <asm/mman.h> #include <asm/pgalloc.h> +#include <asm/pgtable.h> #include <asm/tlb.h> #include <asm/tlbflush.h> #include <asm/cacheflush.h> @@ -131,23 +132,13 @@ pte_t *huge_pte_alloc(struct mm_struct *mm, { pgd_t *pgd; pud_t *pud; - pmd_t *pmd; pte_t *pte = NULL; - /* We must align the address, because our caller will run - * set_huge_pte_at() on whatever we return, which writes out - * all of the sub-ptes for the hugepage range. So we have - * to give it the first such sub-pte. - */ - addr &= HPAGE_MASK; - pgd = pgd_offset(mm, addr); pud = pud_alloc(mm, pgd, addr); - if (pud) { - pmd = pmd_alloc(mm, pud, addr); - if (pmd) - pte = pte_alloc_map(mm, pmd, addr); - } + if (pud) + pte = (pte_t *)pmd_alloc(mm, pud, addr); + return pte; } @@ -155,19 +146,13 @@ pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr) { pgd_t *pgd; pud_t *pud; - pmd_t *pmd; pte_t *pte = NULL; - addr &= HPAGE_MASK; - pgd = pgd_offset(mm, addr); if (!pgd_none(*pgd)) { pud = pud_offset(pgd, addr); - if (!pud_none(*pud)) { - pmd = pmd_offset(pud, addr); - if (!pmd_none(*pmd)) - pte = pte_offset_map(pmd, addr); - } + if (!pud_none(*pud)) + pte = (pte_t *)pmd_offset(pud, addr); } return pte; } @@ -175,70 +160,143 @@ pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr) void set_huge_pte_at(struct mm_struct *mm, unsigned long addr, pte_t *ptep, pte_t entry) { - int i; - pte_t orig[2]; - unsigned long nptes; + pte_t orig; if (!pte_present(*ptep) && pte_present(entry)) - mm->context.huge_pte_count++; + mm->context.hugetlb_pte_count++; addr &= HPAGE_MASK; - - nptes = 1 << HUGETLB_PAGE_ORDER; - orig[0] = *ptep; - orig[1] = *(ptep + nptes / 2); - for (i = 0; i < nptes; i++) { - *ptep = entry; - ptep++; - addr += PAGE_SIZE; - pte_val(entry) += PAGE_SIZE; - } + orig = *ptep; + *ptep = entry; /* Issue TLB flush at REAL_HPAGE_SIZE boundaries */ - addr -= REAL_HPAGE_SIZE; - ptep -= nptes / 2; - maybe_tlb_batch_add(mm, addr, ptep, orig[1], 0); - addr -= REAL_HPAGE_SIZE; - ptep -= nptes / 2; - maybe_tlb_batch_add(mm, addr, ptep, orig[0], 0); + maybe_tlb_batch_add(mm, addr, ptep, orig, 0); + maybe_tlb_batch_add(mm, addr + REAL_HPAGE_SIZE, ptep, orig, 0); } pte_t huge_ptep_get_and_clear(struct mm_struct *mm, unsigned long addr, pte_t *ptep) { pte_t entry; - int i; - unsigned long nptes; entry = *ptep; if (pte_present(entry)) - mm->context.huge_pte_count--; + mm->context.hugetlb_pte_count--; addr &= HPAGE_MASK; - nptes = 1 << HUGETLB_PAGE_ORDER; - for (i = 0; i < nptes; i++) { - *ptep = __pte(0UL); - addr += PAGE_SIZE; - ptep++; - } + *ptep = __pte(0UL); /* Issue TLB flush at REAL_HPAGE_SIZE boundaries */ - addr -= REAL_HPAGE_SIZE; - ptep -= nptes / 2; - maybe_tlb_batch_add(mm, addr, ptep, entry, 0); - addr -= REAL_HPAGE_SIZE; - ptep -= nptes / 2; maybe_tlb_batch_add(mm, addr, ptep, entry, 0); + maybe_tlb_batch_add(mm, addr + REAL_HPAGE_SIZE, ptep, entry, 0); return entry; } int pmd_huge(pmd_t pmd) { - return 0; + return !pmd_none(pmd) && + (pmd_val(pmd) & (_PAGE_VALID|_PAGE_PMD_HUGE)) != _PAGE_VALID; } int pud_huge(pud_t pud) { return 0; } + +static void hugetlb_free_pte_range(struct mmu_gather *tlb, pmd_t *pmd, + unsigned long addr) +{ + pgtable_t token = pmd_pgtable(*pmd); + + pmd_clear(pmd); + pte_free_tlb(tlb, token, addr); + atomic_long_dec(&tlb->mm->nr_ptes); +} + +static void hugetlb_free_pmd_range(struct mmu_gather *tlb, pud_t *pud, + unsigned long addr, unsigned long end, + unsigned long floor, unsigned long ceiling) +{ + pmd_t *pmd; + unsigned long next; + unsigned long start; + + start = addr; + pmd = pmd_offset(pud, addr); + do { + next = pmd_addr_end(addr, end); + if (pmd_none(*pmd)) + continue; + if (is_hugetlb_pmd(*pmd)) + pmd_clear(pmd); + else + hugetlb_free_pte_range(tlb, pmd, addr); + } while (pmd++, addr = next, addr != end); + + start &= PUD_MASK; + if (start < floor) + return; + if (ceiling) { + ceiling &= PUD_MASK; + if (!ceiling) + return; + } + if (end - 1 > ceiling - 1) + return; + + pmd = pmd_offset(pud, start); + pud_clear(pud); + pmd_free_tlb(tlb, pmd, start); + mm_dec_nr_pmds(tlb->mm); +} + +static void hugetlb_free_pud_range(struct mmu_gather *tlb, pgd_t *pgd, + unsigned long addr, unsigned long end, + unsigned long floor, unsigned long ceiling) +{ + pud_t *pud; + unsigned long next; + unsigned long start; + + start = addr; + pud = pud_offset(pgd, addr); + do { + next = pud_addr_end(addr, end); + if (pud_none_or_clear_bad(pud)) + continue; + hugetlb_free_pmd_range(tlb, pud, addr, next, floor, + ceiling); + } while (pud++, addr = next, addr != end); + + start &= PGDIR_MASK; + if (start < floor) + return; + if (ceiling) { + ceiling &= PGDIR_MASK; + if (!ceiling) + return; + } + if (end - 1 > ceiling - 1) + return; + + pud = pud_offset(pgd, start); + pgd_clear(pgd); + pud_free_tlb(tlb, pud, start); +} + +void hugetlb_free_pgd_range(struct mmu_gather *tlb, + unsigned long addr, unsigned long end, + unsigned long floor, unsigned long ceiling) +{ + pgd_t *pgd; + unsigned long next; + + pgd = pgd_offset(tlb->mm, addr); + do { + next = pgd_addr_end(addr, end); + if (pgd_none_or_clear_bad(pgd)) + continue; + hugetlb_free_pud_range(tlb, pgd, addr, next, floor, ceiling); + } while (pgd++, addr = next, addr != end); +} diff --git a/arch/sparc/mm/init_64.c b/arch/sparc/mm/init_64.c index 652683cb4b4b..439784b7b7ac 100644 --- a/arch/sparc/mm/init_64.c +++ b/arch/sparc/mm/init_64.c @@ -5,7 +5,7 @@ * Copyright (C) 1997-1999 Jakub Jelinek (jj@sunsite.mff.cuni.cz) */ -#include <linux/module.h> +#include <linux/extable.h> #include <linux/kernel.h> #include <linux/sched.h> #include <linux/string.h> @@ -346,10 +346,13 @@ void update_mmu_cache(struct vm_area_struct *vma, unsigned long address, pte_t * spin_lock_irqsave(&mm->context.lock, flags); #if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE) - if (mm->context.huge_pte_count && is_hugetlb_pte(pte)) + if ((mm->context.hugetlb_pte_count || mm->context.thp_pte_count) && + is_hugetlb_pte(pte)) { + /* We are fabricating 8MB pages using 4MB real hw pages. */ + pte_val(pte) |= (address & (1UL << REAL_HPAGE_SHIFT)); __update_mmu_tsb_insert(mm, MM_TSB_HUGE, REAL_HPAGE_SHIFT, address, pte_val(pte)); - else + } else #endif __update_mmu_tsb_insert(mm, MM_TSB_BASE, PAGE_SHIFT, address, pte_val(pte)); @@ -1157,7 +1160,7 @@ int __node_distance(int from, int to) return numa_latency[from][to]; } -static int find_best_numa_node_for_mlgroup(struct mdesc_mlgroup *grp) +static int __init find_best_numa_node_for_mlgroup(struct mdesc_mlgroup *grp) { int i; @@ -1170,8 +1173,8 @@ static int find_best_numa_node_for_mlgroup(struct mdesc_mlgroup *grp) return i; } -static void find_numa_latencies_for_group(struct mdesc_handle *md, u64 grp, - int index) +static void __init find_numa_latencies_for_group(struct mdesc_handle *md, + u64 grp, int index) { u64 arc; @@ -2078,7 +2081,6 @@ void __init paging_init(void) { unsigned long end_pfn, shift, phys_base; unsigned long real_end, i; - int node; setup_page_offset(); @@ -2247,21 +2249,6 @@ void __init paging_init(void) /* Setup bootmem... */ last_valid_pfn = end_pfn = bootmem_init(phys_base); - /* Once the OF device tree and MDESC have been setup, we know - * the list of possible cpus. Therefore we can allocate the - * IRQ stacks. - */ - for_each_possible_cpu(i) { - node = cpu_to_node(i); - - softirq_stack[i] = __alloc_bootmem_node(NODE_DATA(node), - THREAD_SIZE, - THREAD_SIZE, 0); - hardirq_stack[i] = __alloc_bootmem_node(NODE_DATA(node), - THREAD_SIZE, - THREAD_SIZE, 0); - } - kernel_physical_mapping_init(); { @@ -2704,8 +2691,7 @@ void __flush_tlb_all(void) pte_t *pte_alloc_one_kernel(struct mm_struct *mm, unsigned long address) { - struct page *page = alloc_page(GFP_KERNEL | __GFP_NOTRACK | - __GFP_REPEAT | __GFP_ZERO); + struct page *page = alloc_page(GFP_KERNEL | __GFP_NOTRACK | __GFP_ZERO); pte_t *pte = NULL; if (page) @@ -2717,8 +2703,7 @@ pte_t *pte_alloc_one_kernel(struct mm_struct *mm, pgtable_t pte_alloc_one(struct mm_struct *mm, unsigned long address) { - struct page *page = alloc_page(GFP_KERNEL | __GFP_NOTRACK | - __GFP_REPEAT | __GFP_ZERO); + struct page *page = alloc_page(GFP_KERNEL | __GFP_NOTRACK | __GFP_ZERO); if (!page) return NULL; if (!pgtable_page_ctor(page)) { @@ -2824,9 +2809,10 @@ void hugetlb_setup(struct pt_regs *regs) * the Data-TLB for huge pages. */ if (tlb_type == cheetah_plus) { + bool need_context_reload = false; unsigned long ctx; - spin_lock(&ctx_alloc_lock); + spin_lock_irq(&ctx_alloc_lock); ctx = mm->context.sparc64_ctx_val; ctx &= ~CTX_PGSZ_MASK; ctx |= CTX_PGSZ_BASE << CTX_PGSZ0_SHIFT; @@ -2845,9 +2831,12 @@ void hugetlb_setup(struct pt_regs *regs) * also executing in this address space. */ mm->context.sparc64_ctx_val = ctx; - on_each_cpu(context_reload, mm, 0); + need_context_reload = true; } - spin_unlock(&ctx_alloc_lock); + spin_unlock_irq(&ctx_alloc_lock); + + if (need_context_reload) + on_each_cpu(context_reload, mm, 0); } } #endif diff --git a/arch/sparc/mm/tlb.c b/arch/sparc/mm/tlb.c index f81cd9736700..c56a195c9071 100644 --- a/arch/sparc/mm/tlb.c +++ b/arch/sparc/mm/tlb.c @@ -174,10 +174,25 @@ void set_pmd_at(struct mm_struct *mm, unsigned long addr, return; if ((pmd_val(pmd) ^ pmd_val(orig)) & _PAGE_PMD_HUGE) { - if (pmd_val(pmd) & _PAGE_PMD_HUGE) - mm->context.huge_pte_count++; - else - mm->context.huge_pte_count--; + /* + * Note that this routine only sets pmds for THP pages. + * Hugetlb pages are handled elsewhere. We need to check + * for huge zero page. Huge zero pages are like hugetlb + * pages in that there is no RSS, but there is the need + * for TSB entries. So, huge zero page counts go into + * hugetlb_pte_count. + */ + if (pmd_val(pmd) & _PAGE_PMD_HUGE) { + if (is_huge_zero_page(pmd_page(pmd))) + mm->context.hugetlb_pte_count++; + else + mm->context.thp_pte_count++; + } else { + if (is_huge_zero_page(pmd_page(orig))) + mm->context.hugetlb_pte_count--; + else + mm->context.thp_pte_count--; + } /* Do not try to allocate the TSB hash table if we * don't have one already. We have various locks held @@ -204,6 +219,9 @@ void set_pmd_at(struct mm_struct *mm, unsigned long addr, } } +/* + * This routine is only called when splitting a THP + */ void pmdp_invalidate(struct vm_area_struct *vma, unsigned long address, pmd_t *pmdp) { @@ -213,6 +231,15 @@ void pmdp_invalidate(struct vm_area_struct *vma, unsigned long address, set_pmd_at(vma->vm_mm, address, pmdp, entry); flush_tlb_range(vma, address, address + HPAGE_PMD_SIZE); + + /* + * set_pmd_at() will not be called in a way to decrement + * thp_pte_count when splitting a THP, so do it now. + * Sanity check pmd before doing the actual decrement. + */ + if ((pmd_val(entry) & _PAGE_PMD_HUGE) && + !is_huge_zero_page(pmd_page(entry))) + (vma->vm_mm)->context.thp_pte_count--; } void pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp, diff --git a/arch/sparc/mm/tsb.c b/arch/sparc/mm/tsb.c index a0604a493a36..f2b77112e9d8 100644 --- a/arch/sparc/mm/tsb.c +++ b/arch/sparc/mm/tsb.c @@ -469,8 +469,10 @@ retry_tsb_alloc: int init_new_context(struct task_struct *tsk, struct mm_struct *mm) { + unsigned long mm_rss = get_mm_rss(mm); #if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE) - unsigned long huge_pte_count; + unsigned long saved_hugetlb_pte_count; + unsigned long saved_thp_pte_count; #endif unsigned int i; @@ -479,12 +481,16 @@ int init_new_context(struct task_struct *tsk, struct mm_struct *mm) mm->context.sparc64_ctx_val = 0UL; #if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE) - /* We reset it to zero because the fork() page copying + /* We reset them to zero because the fork() page copying * will re-increment the counters as the parent PTEs are * copied into the child address space. */ - huge_pte_count = mm->context.huge_pte_count; - mm->context.huge_pte_count = 0; + saved_hugetlb_pte_count = mm->context.hugetlb_pte_count; + saved_thp_pte_count = mm->context.thp_pte_count; + mm->context.hugetlb_pte_count = 0; + mm->context.thp_pte_count = 0; + + mm_rss -= saved_thp_pte_count * (HPAGE_SIZE / PAGE_SIZE); #endif /* copy_mm() copies over the parent's mm_struct before calling @@ -497,11 +503,13 @@ int init_new_context(struct task_struct *tsk, struct mm_struct *mm) /* If this is fork, inherit the parent's TSB size. We would * grow it to that size on the first page fault anyways. */ - tsb_grow(mm, MM_TSB_BASE, get_mm_rss(mm)); + tsb_grow(mm, MM_TSB_BASE, mm_rss); #if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE) - if (unlikely(huge_pte_count)) - tsb_grow(mm, MM_TSB_HUGE, huge_pte_count); + if (unlikely(saved_hugetlb_pte_count + saved_thp_pte_count)) + tsb_grow(mm, MM_TSB_HUGE, + (saved_hugetlb_pte_count + saved_thp_pte_count) * + REAL_HPAGE_PER_HPAGE); #endif if (unlikely(!mm->context.tsb_block[MM_TSB_BASE].tsb)) diff --git a/arch/sparc/prom/ranges.c b/arch/sparc/prom/ranges.c index ad143c13bdc0..6d8dc2aa94ce 100644 --- a/arch/sparc/prom/ranges.c +++ b/arch/sparc/prom/ranges.c @@ -16,9 +16,8 @@ static struct linux_prom_ranges promlib_obio_ranges[PROMREG_MAX]; static int num_obio_ranges; /* Adjust register values based upon the ranges parameters. */ -static void -prom_adjust_regs(struct linux_prom_registers *regp, int nregs, - struct linux_prom_ranges *rangep, int nranges) +static void prom_adjust_regs(struct linux_prom_registers *regp, int nregs, + struct linux_prom_ranges *rangep, int nranges) { int regc, rngc; @@ -34,33 +33,30 @@ prom_adjust_regs(struct linux_prom_registers *regp, int nregs, } } -static void -prom_adjust_ranges(struct linux_prom_ranges *ranges1, int nranges1, - struct linux_prom_ranges *ranges2, int nranges2) +static void prom_adjust_ranges(struct linux_prom_ranges *ranges1, int nranges1, + struct linux_prom_ranges *ranges2, int nranges2) { int rng1c, rng2c; - for(rng1c=0; rng1c < nranges1; rng1c++) { - for(rng2c=0; rng2c < nranges2; rng2c++) - if(ranges1[rng1c].ot_parent_space == ranges2[rng2c].ot_child_space && + for (rng1c = 0; rng1c < nranges1; rng1c++) { + for (rng2c = 0; rng2c < nranges2; rng2c++) + if (ranges1[rng1c].ot_parent_space == ranges2[rng2c].ot_child_space && ranges1[rng1c].ot_parent_base >= ranges2[rng2c].ot_child_base && ranges2[rng2c].ot_child_base + ranges2[rng2c].or_size - ranges1[rng1c].ot_parent_base > 0U) break; - if(rng2c == nranges2) /* oops */ + if (rng2c == nranges2) /* oops */ prom_printf("adjust_ranges: Could not find matching bus type...\n"); else if (ranges1[rng1c].ot_parent_base + ranges1[rng1c].or_size > ranges2[rng2c].ot_child_base + ranges2[rng2c].or_size) - ranges1[rng1c].or_size = - ranges2[rng2c].ot_child_base + ranges2[rng2c].or_size - ranges1[rng1c].ot_parent_base; + ranges1[rng1c].or_size = ranges2[rng2c].ot_child_base + ranges2[rng2c].or_size - ranges1[rng1c].ot_parent_base; ranges1[rng1c].ot_parent_space = ranges2[rng2c].ot_parent_space; ranges1[rng1c].ot_parent_base += ranges2[rng2c].ot_parent_base; } } /* Apply probed obio ranges to registers passed, if no ranges return. */ -void -prom_apply_obio_ranges(struct linux_prom_registers *regs, int nregs) +void prom_apply_obio_ranges(struct linux_prom_registers *regs, int nregs) { - if(num_obio_ranges) + if (num_obio_ranges) prom_adjust_regs(regs, nregs, promlib_obio_ranges, num_obio_ranges); } EXPORT_SYMBOL(prom_apply_obio_ranges); @@ -76,40 +72,40 @@ void __init prom_ranges_init(void) node = prom_getchild(prom_root_node); obio_node = prom_searchsiblings(node, "obio"); - if(obio_node) { + if (obio_node) { success = prom_getproperty(obio_node, "ranges", (char *) promlib_obio_ranges, sizeof(promlib_obio_ranges)); - if(success != -1) - num_obio_ranges = (success/sizeof(struct linux_prom_ranges)); + if (success != -1) + num_obio_ranges = (success / sizeof(struct linux_prom_ranges)); } - if(num_obio_ranges) + if (num_obio_ranges) prom_printf("PROMLIB: obio_ranges %d\n", num_obio_ranges); } void prom_apply_generic_ranges(phandle node, phandle parent, - struct linux_prom_registers *regs, int nregs) + struct linux_prom_registers *regs, int nregs) { int success; int num_ranges; struct linux_prom_ranges ranges[PROMREG_MAX]; - + success = prom_getproperty(node, "ranges", (char *) ranges, - sizeof (ranges)); + sizeof(ranges)); if (success != -1) { - num_ranges = (success/sizeof(struct linux_prom_ranges)); + num_ranges = (success / sizeof(struct linux_prom_ranges)); if (parent) { struct linux_prom_ranges parent_ranges[PROMREG_MAX]; int num_parent_ranges; - + success = prom_getproperty(parent, "ranges", - (char *) parent_ranges, - sizeof (parent_ranges)); + (char *) parent_ranges, + sizeof(parent_ranges)); if (success != -1) { - num_parent_ranges = (success/sizeof(struct linux_prom_ranges)); - prom_adjust_ranges (ranges, num_ranges, parent_ranges, num_parent_ranges); + num_parent_ranges = (success / sizeof(struct linux_prom_ranges)); + prom_adjust_ranges(ranges, num_ranges, parent_ranges, num_parent_ranges); } } prom_adjust_regs(regs, nregs, ranges, num_ranges); |