From 926edcc747e2efb3c9add7ed4dbc4e7a3a959d02 Mon Sep 17 00:00:00 2001 From: Cyril Chemparathy Date: Sun, 22 Jul 2012 13:40:38 -0400 Subject: ARM: LPAE: use signed arithmetic for mask definitions This patch applies to PAGE_MASK, PMD_MASK, and PGDIR_MASK, where forcing unsigned long math truncates the mask at the 32-bits. This clearly does bad things on PAE systems. This patch fixes this problem by defining these masks as signed quantities. We then rely on sign extension to do the right thing. Signed-off-by: Cyril Chemparathy Signed-off-by: Vitaly Andrianov Reviewed-by: Nicolas Pitre Reviewed-by: Catalin Marinas Tested-by: Santosh Shilimkar Tested-by: Subash Patel Signed-off-by: Will Deacon --- arch/arm/include/asm/page.h | 2 +- arch/arm/include/asm/pgtable-3level.h | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/arch/arm/include/asm/page.h b/arch/arm/include/asm/page.h index 812a4944e783..6363f3d1d505 100644 --- a/arch/arm/include/asm/page.h +++ b/arch/arm/include/asm/page.h @@ -13,7 +13,7 @@ /* PAGE_SHIFT determines the page size */ #define PAGE_SHIFT 12 #define PAGE_SIZE (_AC(1,UL) << PAGE_SHIFT) -#define PAGE_MASK (~(PAGE_SIZE-1)) +#define PAGE_MASK (~((1 << PAGE_SHIFT) - 1)) #ifndef __ASSEMBLY__ diff --git a/arch/arm/include/asm/pgtable-3level.h b/arch/arm/include/asm/pgtable-3level.h index 86b8fe398b95..5b85b218b0f0 100644 --- a/arch/arm/include/asm/pgtable-3level.h +++ b/arch/arm/include/asm/pgtable-3level.h @@ -48,16 +48,16 @@ #define PMD_SHIFT 21 #define PMD_SIZE (1UL << PMD_SHIFT) -#define PMD_MASK (~(PMD_SIZE-1)) +#define PMD_MASK (~((1 << PMD_SHIFT) - 1)) #define PGDIR_SIZE (1UL << PGDIR_SHIFT) -#define PGDIR_MASK (~(PGDIR_SIZE-1)) +#define PGDIR_MASK (~((1 << PGDIR_SHIFT) - 1)) /* * section address mask and size definitions. */ #define SECTION_SHIFT 21 #define SECTION_SIZE (1UL << SECTION_SHIFT) -#define SECTION_MASK (~(SECTION_SIZE-1)) +#define SECTION_MASK (~((1 << SECTION_SHIFT) - 1)) #define USER_PTRS_PER_PGD (PAGE_OFFSET / PGDIR_SIZE) -- cgit From 20d6956d8cd2452cec0889ff040f18afc03c2e6b Mon Sep 17 00:00:00 2001 From: Vitaly Andrianov Date: Tue, 10 Jul 2012 14:41:17 -0400 Subject: ARM: LPAE: use phys_addr_t in alloc_init_pud() This patch fixes the alloc_init_pud() function to use phys_addr_t instead of unsigned long when passing in the phys argument. This is an extension to commit 97092e0c56830457af0639f6bd904537a150ea4a (ARM: pgtable: use phys_addr_t for physical addresses), which applied similar changes elsewhere in the ARM memory management code. Signed-off-by: Vitaly Andrianov Signed-off-by: Cyril Chemparathy Acked-by: Nicolas Pitre Acked-by: Catalin Marinas Tested-by: Santosh Shilimkar Tested-by: Subash Patel Signed-off-by: Will Deacon --- arch/arm/mm/mmu.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/arch/arm/mm/mmu.c b/arch/arm/mm/mmu.c index e0d8565671a6..b7ce65a82371 100644 --- a/arch/arm/mm/mmu.c +++ b/arch/arm/mm/mmu.c @@ -673,7 +673,8 @@ static void __init alloc_init_pmd(pud_t *pud, unsigned long addr, } static void __init alloc_init_pud(pgd_t *pgd, unsigned long addr, - unsigned long end, unsigned long phys, const struct mem_type *type) + unsigned long end, phys_addr_t phys, + const struct mem_type *type) { pud_t *pud = pud_offset(pgd, addr); unsigned long next; -- cgit From 56bc628666b39dc8cb395c7686d8c032efd731f4 Mon Sep 17 00:00:00 2001 From: Vitaly Andrianov Date: Thu, 21 Jun 2012 08:09:05 -0400 Subject: ARM: LPAE: use phys_addr_t in free_memmap() The free_memmap() was mistakenly using unsigned long type to represent physical addresses. This breaks on PAE systems where memory could be placed above the 32-bit addressible limit. This patch fixes this function to properly use phys_addr_t instead. Signed-off-by: Vitaly Andrianov Signed-off-by: Cyril Chemparathy Acked-by: Nicolas Pitre Tested-by: Santosh Shilimkar Tested-by: Subash Patel Signed-off-by: Will Deacon --- arch/arm/mm/init.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/arch/arm/mm/init.c b/arch/arm/mm/init.c index 9a5cdc01fcdf..68c914e8544e 100644 --- a/arch/arm/mm/init.c +++ b/arch/arm/mm/init.c @@ -442,7 +442,7 @@ static inline void free_memmap(unsigned long start_pfn, unsigned long end_pfn) { struct page *start_pg, *end_pg; - unsigned long pg, pgend; + phys_addr_t pg, pgend; /* * Convert start_pfn/end_pfn to a struct page pointer. @@ -454,8 +454,8 @@ free_memmap(unsigned long start_pfn, unsigned long end_pfn) * Convert to physical addresses, and * round start upwards and end downwards. */ - pg = (unsigned long)PAGE_ALIGN(__pa(start_pg)); - pgend = (unsigned long)__pa(end_pg) & PAGE_MASK; + pg = PAGE_ALIGN(__pa(start_pg)); + pgend = __pa(end_pg) & PAGE_MASK; /* * If there are free pages between these, -- cgit From de22cc6e33449d8d6fb339619e32138ea4fcc2a4 Mon Sep 17 00:00:00 2001 From: Vitaly Andrianov Date: Fri, 22 Jun 2012 14:26:04 -0400 Subject: ARM: LPAE: use phys_addr_t for initrd location This patch fixes the initrd setup code to use phys_addr_t instead of assuming 32-bit addressing. Without this we cannot boot on systems where initrd is located above the 4G physical address limit. Signed-off-by: Vitaly Andrianov Signed-off-by: Cyril Chemparathy Acked-by: Nicolas Pitre Acked-by: Catalin Marinas Tested-by: Santosh Shilimkar Tested-by: Subash Patel Signed-off-by: Will Deacon --- arch/arm/mm/init.c | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/arch/arm/mm/init.c b/arch/arm/mm/init.c index 68c914e8544e..2ffee02d1d5c 100644 --- a/arch/arm/mm/init.c +++ b/arch/arm/mm/init.c @@ -36,12 +36,13 @@ #include "mm.h" -static unsigned long phys_initrd_start __initdata = 0; +static phys_addr_t phys_initrd_start __initdata = 0; static unsigned long phys_initrd_size __initdata = 0; static int __init early_initrd(char *p) { - unsigned long start, size; + phys_addr_t start; + unsigned long size; char *endp; start = memparse(p, &endp); @@ -350,14 +351,14 @@ void __init arm_memblock_init(struct meminfo *mi, struct machine_desc *mdesc) #ifdef CONFIG_BLK_DEV_INITRD if (phys_initrd_size && !memblock_is_region_memory(phys_initrd_start, phys_initrd_size)) { - pr_err("INITRD: 0x%08lx+0x%08lx is not a memory region - disabling initrd\n", - phys_initrd_start, phys_initrd_size); + pr_err("INITRD: 0x%08llx+0x%08lx is not a memory region - disabling initrd\n", + (u64)phys_initrd_start, phys_initrd_size); phys_initrd_start = phys_initrd_size = 0; } if (phys_initrd_size && memblock_is_region_reserved(phys_initrd_start, phys_initrd_size)) { - pr_err("INITRD: 0x%08lx+0x%08lx overlaps in-use memory region - disabling initrd\n", - phys_initrd_start, phys_initrd_size); + pr_err("INITRD: 0x%08llx+0x%08lx overlaps in-use memory region - disabling initrd\n", + (u64)phys_initrd_start, phys_initrd_size); phys_initrd_start = phys_initrd_size = 0; } if (phys_initrd_size) { -- cgit From 13f659b0f363114282679d06094337c5efa12fa8 Mon Sep 17 00:00:00 2001 From: Cyril Chemparathy Date: Mon, 16 Jul 2012 15:37:06 -0400 Subject: ARM: LPAE: use phys_addr_t in switch_mm() This patch modifies the switch_mm() processor functions to use phys_addr_t. On LPAE systems, we now honor the upper 32-bits of the physical address that is being passed in, and program these into TTBR as expected. Signed-off-by: Cyril Chemparathy Signed-off-by: Vitaly Andrianov Reviewed-by: Nicolas Pitre Tested-by: Santosh Shilimkar Tested-by: Subash Patel [will: fixed up conflict in 3-level switch_mm with big-endian changes] Signed-off-by: Will Deacon --- arch/arm/include/asm/proc-fns.h | 4 ++-- arch/arm/mm/proc-v7-3level.S | 16 ++++++++++++---- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/arch/arm/include/asm/proc-fns.h b/arch/arm/include/asm/proc-fns.h index f3628fb3d2b3..75b5f14617c3 100644 --- a/arch/arm/include/asm/proc-fns.h +++ b/arch/arm/include/asm/proc-fns.h @@ -60,7 +60,7 @@ extern struct processor { /* * Set the page table */ - void (*switch_mm)(unsigned long pgd_phys, struct mm_struct *mm); + void (*switch_mm)(phys_addr_t pgd_phys, struct mm_struct *mm); /* * Set a possibly extended PTE. Non-extended PTEs should * ignore 'ext'. @@ -82,7 +82,7 @@ extern void cpu_proc_init(void); extern void cpu_proc_fin(void); extern int cpu_do_idle(void); extern void cpu_dcache_clean_area(void *, int); -extern void cpu_do_switch_mm(unsigned long pgd_phys, struct mm_struct *mm); +extern void cpu_do_switch_mm(phys_addr_t pgd_phys, struct mm_struct *mm); #ifdef CONFIG_ARM_LPAE extern void cpu_set_pte_ext(pte_t *ptep, pte_t pte); #else diff --git a/arch/arm/mm/proc-v7-3level.S b/arch/arm/mm/proc-v7-3level.S index 363027e811d6..995857d3b530 100644 --- a/arch/arm/mm/proc-v7-3level.S +++ b/arch/arm/mm/proc-v7-3level.S @@ -39,6 +39,14 @@ #define TTB_FLAGS_SMP (TTB_IRGN_WBWA|TTB_S|TTB_RGN_OC_WBWA) #define PMD_FLAGS_SMP (PMD_SECT_WBWA|PMD_SECT_S) +#ifndef __ARMEB__ +# define rpgdl r0 +# define rpgdh r1 +#else +# define rpgdl r1 +# define rpgdh r0 +#endif + /* * cpu_v7_switch_mm(pgd_phys, tsk) * @@ -47,10 +55,10 @@ */ ENTRY(cpu_v7_switch_mm) #ifdef CONFIG_MMU - mmid r1, r1 @ get mm->context.id - asid r3, r1 - mov r3, r3, lsl #(48 - 32) @ ASID - mcrr p15, 0, r0, r3, c2 @ set TTB 0 + mmid r2, r2 + asid r2, r2 + orr rpgdh, rpgdh, r2, lsl #(48 - 32) @ upper 32-bits of pgd + mcrr p15, 0, rpgdl, rpgdh, c2 @ set TTB 0 isb #endif mov pc, lr -- cgit From 1fc84ae84b5153e32a4b6ace507f9663e10b0cb2 Mon Sep 17 00:00:00 2001 From: Cyril Chemparathy Date: Mon, 16 Jul 2012 17:20:17 -0400 Subject: ARM: LPAE: use 64-bit accessors for TTBR registers This patch adds TTBR accessor macros, and modifies cpu_get_pgd() and the LPAE version of cpu_set_reserved_ttbr0() to use these instead. In the process, we also fix these functions to correctly handle cases where the physical address lies beyond the 4G limit of 32-bit addressing. Signed-off-by: Cyril Chemparathy Signed-off-by: Vitaly Andrianov Acked-by: Nicolas Pitre Reviewed-by: Catalin Marinas Tested-by: Santosh Shilimkar Tested-by: Subash Patel Signed-off-by: Will Deacon --- arch/arm/include/asm/proc-fns.h | 22 +++++++++++++++++----- arch/arm/mm/context.c | 9 ++------- 2 files changed, 19 insertions(+), 12 deletions(-) diff --git a/arch/arm/include/asm/proc-fns.h b/arch/arm/include/asm/proc-fns.h index 75b5f14617c3..1c3cf9407a31 100644 --- a/arch/arm/include/asm/proc-fns.h +++ b/arch/arm/include/asm/proc-fns.h @@ -116,13 +116,25 @@ extern void cpu_resume(void); #define cpu_switch_mm(pgd,mm) cpu_do_switch_mm(virt_to_phys(pgd),mm) #ifdef CONFIG_ARM_LPAE + +#define cpu_get_ttbr(nr) \ + ({ \ + u64 ttbr; \ + __asm__("mrrc p15, " #nr ", %Q0, %R0, c2" \ + : "=r" (ttbr)); \ + ttbr; \ + }) + +#define cpu_set_ttbr(nr, val) \ + do { \ + u64 ttbr = val; \ + __asm__("mcrr p15, " #nr ", %Q0, %R0, c2" \ + : : "r" (ttbr)); \ + } while (0) + #define cpu_get_pgd() \ ({ \ - unsigned long pg, pg2; \ - __asm__("mrrc p15, 0, %0, %1, c2" \ - : "=r" (pg), "=r" (pg2) \ - : \ - : "cc"); \ + u64 pg = cpu_get_ttbr(0); \ pg &= ~(PTRS_PER_PGD*sizeof(pgd_t)-1); \ (pgd_t *)phys_to_virt(pg); \ }) diff --git a/arch/arm/mm/context.c b/arch/arm/mm/context.c index 2ac37372ef52..3675e31473e3 100644 --- a/arch/arm/mm/context.c +++ b/arch/arm/mm/context.c @@ -20,6 +20,7 @@ #include #include #include +#include /* * On ARMv6, we have the following structure in the Context ID: @@ -55,17 +56,11 @@ static cpumask_t tlb_flush_pending; #ifdef CONFIG_ARM_LPAE static void cpu_set_reserved_ttbr0(void) { - unsigned long ttbl = __pa(swapper_pg_dir); - unsigned long ttbh = 0; - /* * Set TTBR0 to swapper_pg_dir which contains only global entries. The * ASID is set to 0. */ - asm volatile( - " mcrr p15, 0, %0, %1, c2 @ set TTBR0\n" - : - : "r" (ttbl), "r" (ttbh)); + cpu_set_ttbr(0, __pa(swapper_pg_dir)); isb(); } #else -- cgit From a7fbc0d62a4d46e642af889e7288fede5078bc46 Mon Sep 17 00:00:00 2001 From: Cyril Chemparathy Date: Sat, 21 Jul 2012 19:47:52 -0400 Subject: ARM: LPAE: factor out T1SZ and TTBR1 computations This patch moves the TTBR1 offset calculation and the T1SZ calculation out of the TTB setup assembly code. This should not affect functionality in any way, but improves code readability as well as readability of subsequent patches in this series. Signed-off-by: Cyril Chemparathy Signed-off-by: Vitaly Andrianov Acked-by: Nicolas Pitre Tested-by: Santosh Shilimkar Tested-by: Subash Patel Signed-off-by: Will Deacon --- arch/arm/include/asm/pgtable-3level-hwdef.h | 20 ++++++++++++++++++++ arch/arm/mm/proc-v7-3level.S | 29 ++++++++--------------------- 2 files changed, 28 insertions(+), 21 deletions(-) diff --git a/arch/arm/include/asm/pgtable-3level-hwdef.h b/arch/arm/include/asm/pgtable-3level-hwdef.h index 18f5cef82ad5..c6c6e6df22f3 100644 --- a/arch/arm/include/asm/pgtable-3level-hwdef.h +++ b/arch/arm/include/asm/pgtable-3level-hwdef.h @@ -79,4 +79,24 @@ #define PHYS_MASK_SHIFT (40) #define PHYS_MASK ((1ULL << PHYS_MASK_SHIFT) - 1) +/* + * TTBR0/TTBR1 split (PAGE_OFFSET): + * 0x40000000: T0SZ = 2, T1SZ = 0 (not used) + * 0x80000000: T0SZ = 0, T1SZ = 1 + * 0xc0000000: T0SZ = 0, T1SZ = 2 + * + * Only use this feature if PHYS_OFFSET <= PAGE_OFFSET, otherwise + * booting secondary CPUs would end up using TTBR1 for the identity + * mapping set up in TTBR0. + */ +#if defined CONFIG_VMSPLIT_2G +#define TTBR1_OFFSET 16 /* skip two L1 entries */ +#elif defined CONFIG_VMSPLIT_3G +#define TTBR1_OFFSET (4096 * (1 + 3)) /* only L2, skip pgd + 3*pmd */ +#else +#define TTBR1_OFFSET 0 +#endif + +#define TTBR1_SIZE (((PAGE_OFFSET >> 30) - 1) << 16) + #endif diff --git a/arch/arm/mm/proc-v7-3level.S b/arch/arm/mm/proc-v7-3level.S index 995857d3b530..58ab7477bb61 100644 --- a/arch/arm/mm/proc-v7-3level.S +++ b/arch/arm/mm/proc-v7-3level.S @@ -114,7 +114,7 @@ ENDPROC(cpu_v7_set_pte_ext) */ .macro v7_ttb_setup, zero, ttbr0, ttbr1, tmp ldr \tmp, =swapper_pg_dir @ swapper_pg_dir virtual address - cmp \ttbr1, \tmp @ PHYS_OFFSET > PAGE_OFFSET? (branch below) + cmp \ttbr1, \tmp @ PHYS_OFFSET > PAGE_OFFSET? mrc p15, 0, \tmp, c2, c0, 2 @ TTB control register orr \tmp, \tmp, #TTB_EAE ALT_SMP(orr \tmp, \tmp, #TTB_FLAGS_SMP) @@ -122,27 +122,14 @@ ENDPROC(cpu_v7_set_pte_ext) ALT_SMP(orr \tmp, \tmp, #TTB_FLAGS_SMP << 16) ALT_UP(orr \tmp, \tmp, #TTB_FLAGS_UP << 16) /* - * TTBR0/TTBR1 split (PAGE_OFFSET): - * 0x40000000: T0SZ = 2, T1SZ = 0 (not used) - * 0x80000000: T0SZ = 0, T1SZ = 1 - * 0xc0000000: T0SZ = 0, T1SZ = 2 - * - * Only use this feature if PHYS_OFFSET <= PAGE_OFFSET, otherwise - * booting secondary CPUs would end up using TTBR1 for the identity - * mapping set up in TTBR0. + * Only use split TTBRs if PHYS_OFFSET <= PAGE_OFFSET (cmp above), + * otherwise booting secondary CPUs would end up using TTBR1 for the + * identity mapping set up in TTBR0. */ - bhi 9001f @ PHYS_OFFSET > PAGE_OFFSET? - orr \tmp, \tmp, #(((PAGE_OFFSET >> 30) - 1) << 16) @ TTBCR.T1SZ -#if defined CONFIG_VMSPLIT_2G - /* PAGE_OFFSET == 0x80000000, T1SZ == 1 */ - add \ttbr1, \ttbr1, #1 << 4 @ skip two L1 entries -#elif defined CONFIG_VMSPLIT_3G - /* PAGE_OFFSET == 0xc0000000, T1SZ == 2 */ - add \ttbr1, \ttbr1, #4096 * (1 + 3) @ only L2 used, skip pgd+3*pmd -#endif - /* CONFIG_VMSPLIT_1G does not need TTBR1 adjustment */ -9001: mcr p15, 0, \tmp, c2, c0, 2 @ TTB control register - mcrr p15, 1, \ttbr1, \zero, c2 @ load TTBR1 + orrls \tmp, \tmp, #TTBR1_SIZE @ TTBCR.T1SZ + mcr p15, 0, \tmp, c2, c0, 2 @ TTBCR + addls \ttbr1, \ttbr1, #TTBR1_OFFSET + mcrr p15, 1, \ttbr1, \zero, c2 @ load TTBR1 .endm __CPUINIT -- cgit From 4756dcbfd37819a8359d3c69a22be2ee41666d0f Mon Sep 17 00:00:00 2001 From: Cyril Chemparathy Date: Sat, 21 Jul 2012 15:55:04 -0400 Subject: ARM: LPAE: accomodate >32-bit addresses for page table base This patch redefines the early boot time use of the R4 register to steal a few low order bits (ARCH_PGD_SHIFT bits) on LPAE systems. This allows for up to 38-bit physical addresses. Signed-off-by: Cyril Chemparathy Signed-off-by: Vitaly Andrianov Acked-by: Nicolas Pitre Tested-by: Santosh Shilimkar Tested-by: Subash Patel Signed-off-by: Will Deacon --- arch/arm/include/asm/memory.h | 16 ++++++++++++++++ arch/arm/kernel/head.S | 10 ++++------ arch/arm/kernel/smp.c | 11 +++++++++-- arch/arm/mm/proc-v7-3level.S | 8 ++++++++ 4 files changed, 37 insertions(+), 8 deletions(-) diff --git a/arch/arm/include/asm/memory.h b/arch/arm/include/asm/memory.h index 57870ab313c5..e506088a7678 100644 --- a/arch/arm/include/asm/memory.h +++ b/arch/arm/include/asm/memory.h @@ -18,6 +18,8 @@ #include #include +#include + #ifdef CONFIG_NEED_MACH_MEMORY_H #include #endif @@ -141,6 +143,20 @@ #define page_to_phys(page) (__pfn_to_phys(page_to_pfn(page))) #define phys_to_page(phys) (pfn_to_page(__phys_to_pfn(phys))) +/* + * Minimum guaranted alignment in pgd_alloc(). The page table pointers passed + * around in head.S and proc-*.S are shifted by this amount, in order to + * leave spare high bits for systems with physical address extension. This + * does not fully accomodate the 40-bit addressing capability of ARM LPAE, but + * gives us about 38-bits or so. + */ +#ifdef CONFIG_ARM_LPAE +#define ARCH_PGD_SHIFT L1_CACHE_SHIFT +#else +#define ARCH_PGD_SHIFT 0 +#endif +#define ARCH_PGD_MASK ((1 << ARCH_PGD_SHIFT) - 1) + #ifndef __ASSEMBLY__ /* diff --git a/arch/arm/kernel/head.S b/arch/arm/kernel/head.S index 8bac553fe213..45e8935cae4e 100644 --- a/arch/arm/kernel/head.S +++ b/arch/arm/kernel/head.S @@ -156,7 +156,7 @@ ENDPROC(stext) * * Returns: * r0, r3, r5-r7 corrupted - * r4 = physical page table address + * r4 = page table (see ARCH_PGD_SHIFT in asm/memory.h) */ __create_page_tables: pgtbl r4, r8 @ page table address @@ -331,6 +331,7 @@ __create_page_tables: #endif #ifdef CONFIG_ARM_LPAE sub r4, r4, #0x1000 @ point to the PGD table + mov r4, r4, lsr #ARCH_PGD_SHIFT #endif mov pc, lr ENDPROC(__create_page_tables) @@ -408,7 +409,7 @@ __secondary_data: * r0 = cp#15 control register * r1 = machine ID * r2 = atags or dtb pointer - * r4 = page table pointer + * r4 = page table (see ARCH_PGD_SHIFT in asm/memory.h) * r9 = processor ID * r13 = *virtual* address to jump to upon completion */ @@ -427,10 +428,7 @@ __enable_mmu: #ifdef CONFIG_CPU_ICACHE_DISABLE bic r0, r0, #CR_I #endif -#ifdef CONFIG_ARM_LPAE - mov r5, #0 - mcrr p15, 0, r4, r5, c2 @ load TTBR0 -#else +#ifndef CONFIG_ARM_LPAE mov r5, #(domain_val(DOMAIN_USER, DOMAIN_MANAGER) | \ domain_val(DOMAIN_KERNEL, DOMAIN_MANAGER) | \ domain_val(DOMAIN_TABLE, DOMAIN_MANAGER) | \ diff --git a/arch/arm/kernel/smp.c b/arch/arm/kernel/smp.c index 550d63cef68e..217b755aadd4 100644 --- a/arch/arm/kernel/smp.c +++ b/arch/arm/kernel/smp.c @@ -78,6 +78,13 @@ void __init smp_set_ops(struct smp_operations *ops) smp_ops = *ops; }; +static unsigned long get_arch_pgd(pgd_t *pgd) +{ + phys_addr_t pgdir = virt_to_phys(pgd); + BUG_ON(pgdir & ARCH_PGD_MASK); + return pgdir >> ARCH_PGD_SHIFT; +} + int __cpuinit __cpu_up(unsigned int cpu, struct task_struct *idle) { int ret; @@ -87,8 +94,8 @@ int __cpuinit __cpu_up(unsigned int cpu, struct task_struct *idle) * its stack and the page tables. */ secondary_data.stack = task_stack_page(idle) + THREAD_START_SP; - secondary_data.pgdir = virt_to_phys(idmap_pgd); - secondary_data.swapper_pg_dir = virt_to_phys(swapper_pg_dir); + secondary_data.pgdir = get_arch_pgd(idmap_pgd); + secondary_data.swapper_pg_dir = get_arch_pgd(swapper_pg_dir); __cpuc_flush_dcache_area(&secondary_data, sizeof(secondary_data)); outer_clean_range(__pa(&secondary_data), __pa(&secondary_data + 1)); diff --git a/arch/arm/mm/proc-v7-3level.S b/arch/arm/mm/proc-v7-3level.S index 58ab7477bb61..5ffe1956c6d9 100644 --- a/arch/arm/mm/proc-v7-3level.S +++ b/arch/arm/mm/proc-v7-3level.S @@ -114,6 +114,7 @@ ENDPROC(cpu_v7_set_pte_ext) */ .macro v7_ttb_setup, zero, ttbr0, ttbr1, tmp ldr \tmp, =swapper_pg_dir @ swapper_pg_dir virtual address + mov \tmp, \tmp, lsr #ARCH_PGD_SHIFT cmp \ttbr1, \tmp @ PHYS_OFFSET > PAGE_OFFSET? mrc p15, 0, \tmp, c2, c0, 2 @ TTB control register orr \tmp, \tmp, #TTB_EAE @@ -128,8 +129,15 @@ ENDPROC(cpu_v7_set_pte_ext) */ orrls \tmp, \tmp, #TTBR1_SIZE @ TTBCR.T1SZ mcr p15, 0, \tmp, c2, c0, 2 @ TTBCR + mov \tmp, \ttbr1, lsr #(32 - ARCH_PGD_SHIFT) @ upper bits + mov \ttbr1, \ttbr1, lsl #ARCH_PGD_SHIFT @ lower bits addls \ttbr1, \ttbr1, #TTBR1_OFFSET mcrr p15, 1, \ttbr1, \zero, c2 @ load TTBR1 + mov \tmp, \ttbr0, lsr #(32 - ARCH_PGD_SHIFT) @ upper bits + mov \ttbr0, \ttbr0, lsl #ARCH_PGD_SHIFT @ lower bits + mcrr p15, 0, \ttbr0, \zero, c2 @ load TTBR0 + mcrr p15, 1, \ttbr1, \zero, c2 @ load TTBR1 + mcrr p15, 0, \ttbr0, \zero, c2 @ load TTBR0 .endm __CPUINIT -- cgit From 82f667046ec895552caf2f7a4c6841c530bfc215 Mon Sep 17 00:00:00 2001 From: Cyril Chemparathy Date: Fri, 20 Jul 2012 12:01:23 -0400 Subject: ARM: mm: use physical addresses in highmem sanity checks This patch modifies the highmem sanity checking code to use physical addresses instead. This change eliminates the wrap-around problems associated with the original virtual address based checks, and this simplifies the code a bit. The one constraint imposed here is that low physical memory must be mapped in a monotonically increasing fashion if there are multiple banks of memory, i.e., x < y must => pa(x) < pa(y). Signed-off-by: Cyril Chemparathy Signed-off-by: Vitaly Andrianov Acked-by: Nicolas Pitre Tested-by: Santosh Shilimkar Tested-by: Subash Patel Signed-off-by: Will Deacon --- arch/arm/mm/mmu.c | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/arch/arm/mm/mmu.c b/arch/arm/mm/mmu.c index b7ce65a82371..fc6ff1a9db50 100644 --- a/arch/arm/mm/mmu.c +++ b/arch/arm/mm/mmu.c @@ -988,6 +988,7 @@ phys_addr_t arm_lowmem_limit __initdata = 0; void __init sanity_check_meminfo(void) { int i, j, highmem = 0; + phys_addr_t vmalloc_limit = __pa(vmalloc_min - 1) + 1; for (i = 0, j = 0; i < meminfo.nr_banks; i++) { struct membank *bank = &meminfo.bank[j]; @@ -997,8 +998,7 @@ void __init sanity_check_meminfo(void) highmem = 1; #ifdef CONFIG_HIGHMEM - if (__va(bank->start) >= vmalloc_min || - __va(bank->start) < (void *)PAGE_OFFSET) + if (bank->start >= vmalloc_limit) highmem = 1; bank->highmem = highmem; @@ -1007,8 +1007,8 @@ void __init sanity_check_meminfo(void) * Split those memory banks which are partially overlapping * the vmalloc area greatly simplifying things later. */ - if (!highmem && __va(bank->start) < vmalloc_min && - bank->size > vmalloc_min - __va(bank->start)) { + if (!highmem && bank->start < vmalloc_limit && + bank->size > vmalloc_limit - bank->start) { if (meminfo.nr_banks >= NR_BANKS) { printk(KERN_CRIT "NR_BANKS too low, " "ignoring high memory\n"); @@ -1017,12 +1017,12 @@ void __init sanity_check_meminfo(void) (meminfo.nr_banks - i) * sizeof(*bank)); meminfo.nr_banks++; i++; - bank[1].size -= vmalloc_min - __va(bank->start); - bank[1].start = __pa(vmalloc_min - 1) + 1; + bank[1].size -= vmalloc_limit - bank->start; + bank[1].start = vmalloc_limit; bank[1].highmem = highmem = 1; j++; } - bank->size = vmalloc_min - __va(bank->start); + bank->size = vmalloc_limit - bank->start; } #else bank->highmem = highmem; @@ -1042,8 +1042,7 @@ void __init sanity_check_meminfo(void) * Check whether this memory bank would entirely overlap * the vmalloc area. */ - if (__va(bank->start) >= vmalloc_min || - __va(bank->start) < (void *)PAGE_OFFSET) { + if (bank->start >= vmalloc_limit) { printk(KERN_NOTICE "Ignoring RAM at %.8llx-%.8llx " "(vmalloc region overlap).\n", (unsigned long long)bank->start, @@ -1055,9 +1054,8 @@ void __init sanity_check_meminfo(void) * Check whether this memory bank would partially overlap * the vmalloc area. */ - if (__va(bank->start + bank->size - 1) >= vmalloc_min || - __va(bank->start + bank->size - 1) <= __va(bank->start)) { - unsigned long newsize = vmalloc_min - __va(bank->start); + if (bank->start + bank->size > vmalloc_limit) + unsigned long newsize = vmalloc_limit - bank->start; printk(KERN_NOTICE "Truncating RAM at %.8llx-%.8llx " "to -%.8llx (vmalloc region overlap).\n", (unsigned long long)bank->start, -- cgit From 5b20c5b2f014ecc0a6310988af69cd7ede9e7c67 Mon Sep 17 00:00:00 2001 From: Cyril Chemparathy Date: Wed, 12 Sep 2012 10:19:05 -0400 Subject: ARM: fix type of PHYS_PFN_OFFSET to unsigned long On LPAE machines, PHYS_OFFSET evaluates to a phys_addr_t and this type is inherited by the PHYS_PFN_OFFSET definition as well. Consequently, the kernel build emits warnings of the form: init/main.c: In function 'start_kernel': init/main.c:588:7: warning: format '%lx' expects argument of type 'long unsigned int', but argument 2 has type 'phys_addr_t' [-Wformat] This patch fixes this warning by pinning down the PFN type to unsigned long. Signed-off-by: Cyril Chemparathy Acked-by: Nicolas Pitre Tested-by: Santosh Shilimkar Tested-by: Subash Patel Signed-off-by: Will Deacon --- arch/arm/include/asm/memory.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/arm/include/asm/memory.h b/arch/arm/include/asm/memory.h index e506088a7678..584786f740f9 100644 --- a/arch/arm/include/asm/memory.h +++ b/arch/arm/include/asm/memory.h @@ -223,7 +223,7 @@ static inline unsigned long __phys_to_virt(unsigned long x) * direct-mapped view. We assume this is the first page * of RAM in the mem_map as well. */ -#define PHYS_PFN_OFFSET (PHYS_OFFSET >> PAGE_SHIFT) +#define PHYS_PFN_OFFSET ((unsigned long)(PHYS_OFFSET >> PAGE_SHIFT)) /* * These are *only* valid on the kernel direct mapped RAM memory. -- cgit From adf2e9fda34c1cfff2ee4e47078b1e142adb2c30 Mon Sep 17 00:00:00 2001 From: Cyril Chemparathy Date: Fri, 20 Jul 2012 12:24:45 -0400 Subject: ARM: mm: cleanup checks for membank overlap with vmalloc area On Keystone platforms, physical memory is entirely outside the 32-bit addressible range. Therefore, the (bank->start > ULONG_MAX) check below marks the entire system memory as highmem, and this causes unpleasentness all over. This patch eliminates the extra bank start check (against ULONG_MAX) by checking bank->start against the physical address corresponding to vmalloc_min instead. In the process, this patch also cleans up parts of the highmem sanity check code by removing what has now become a redundant check for banks that entirely overlap with the vmalloc range. Signed-off-by: Cyril Chemparathy Signed-off-by: Vitaly Andrianov Acked-by: Nicolas Pitre Tested-by: Santosh Shilimkar Tested-by: Subash Patel Signed-off-by: Will Deacon --- arch/arm/mm/mmu.c | 19 +------------------ 1 file changed, 1 insertion(+), 18 deletions(-) diff --git a/arch/arm/mm/mmu.c b/arch/arm/mm/mmu.c index fc6ff1a9db50..ae249d1ab1d3 100644 --- a/arch/arm/mm/mmu.c +++ b/arch/arm/mm/mmu.c @@ -994,15 +994,12 @@ void __init sanity_check_meminfo(void) struct membank *bank = &meminfo.bank[j]; *bank = meminfo.bank[i]; - if (bank->start > ULONG_MAX) - highmem = 1; - -#ifdef CONFIG_HIGHMEM if (bank->start >= vmalloc_limit) highmem = 1; bank->highmem = highmem; +#ifdef CONFIG_HIGHMEM /* * Split those memory banks which are partially overlapping * the vmalloc area greatly simplifying things later. @@ -1025,8 +1022,6 @@ void __init sanity_check_meminfo(void) bank->size = vmalloc_limit - bank->start; } #else - bank->highmem = highmem; - /* * Highmem banks not allowed with !CONFIG_HIGHMEM. */ @@ -1038,18 +1033,6 @@ void __init sanity_check_meminfo(void) continue; } - /* - * Check whether this memory bank would entirely overlap - * the vmalloc area. - */ - if (bank->start >= vmalloc_limit) { - printk(KERN_NOTICE "Ignoring RAM at %.8llx-%.8llx " - "(vmalloc region overlap).\n", - (unsigned long long)bank->start, - (unsigned long long)bank->start + bank->size - 1); - continue; - } - /* * Check whether this memory bank would partially overlap * the vmalloc area. -- cgit From 28d4bf7a2929c5e525171d249e12662e21130ec3 Mon Sep 17 00:00:00 2001 From: Cyril Chemparathy Date: Fri, 20 Jul 2012 13:16:41 -0400 Subject: ARM: mm: clean up membank size limit checks This patch cleans up the highmem sanity check code by simplifying the range checks with a pre-calculated size_limit. This patch should otherwise have no functional impact on behavior. This patch also removes a redundant (bank->start < vmalloc_limit) check, since this is already covered by the !highmem condition. Signed-off-by: Cyril Chemparathy Signed-off-by: Vitaly Andrianov Acked-by: Nicolas Pitre Tested-by: Santosh Shilimkar Tested-by: Subash Patel Signed-off-by: Will Deacon --- arch/arm/mm/mmu.c | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/arch/arm/mm/mmu.c b/arch/arm/mm/mmu.c index ae249d1ab1d3..280f91d02de2 100644 --- a/arch/arm/mm/mmu.c +++ b/arch/arm/mm/mmu.c @@ -992,10 +992,15 @@ void __init sanity_check_meminfo(void) for (i = 0, j = 0; i < meminfo.nr_banks; i++) { struct membank *bank = &meminfo.bank[j]; + phys_addr_t size_limit; + *bank = meminfo.bank[i]; + size_limit = bank->size; if (bank->start >= vmalloc_limit) highmem = 1; + else + size_limit = vmalloc_limit - bank->start; bank->highmem = highmem; @@ -1004,8 +1009,7 @@ void __init sanity_check_meminfo(void) * Split those memory banks which are partially overlapping * the vmalloc area greatly simplifying things later. */ - if (!highmem && bank->start < vmalloc_limit && - bank->size > vmalloc_limit - bank->start) { + if (!highmem && bank->size > size_limit) { if (meminfo.nr_banks >= NR_BANKS) { printk(KERN_CRIT "NR_BANKS too low, " "ignoring high memory\n"); @@ -1014,12 +1018,12 @@ void __init sanity_check_meminfo(void) (meminfo.nr_banks - i) * sizeof(*bank)); meminfo.nr_banks++; i++; - bank[1].size -= vmalloc_limit - bank->start; + bank[1].size -= size_limit; bank[1].start = vmalloc_limit; bank[1].highmem = highmem = 1; j++; } - bank->size = vmalloc_limit - bank->start; + bank->size = size_limit; } #else /* @@ -1037,14 +1041,13 @@ void __init sanity_check_meminfo(void) * Check whether this memory bank would partially overlap * the vmalloc area. */ - if (bank->start + bank->size > vmalloc_limit) - unsigned long newsize = vmalloc_limit - bank->start; + if (bank->size > size_limit) { printk(KERN_NOTICE "Truncating RAM at %.8llx-%.8llx " "to -%.8llx (vmalloc region overlap).\n", (unsigned long long)bank->start, (unsigned long long)bank->start + bank->size - 1, - (unsigned long long)bank->start + newsize - 1); - bank->size = newsize; + (unsigned long long)bank->start + size_limit - 1); + bank->size = size_limit; } #endif if (!bank->highmem && bank->start + bank->size > arm_lowmem_limit) -- cgit From e38a517578d6c0f764b0d0f6e26dcdf9f70c69d7 Mon Sep 17 00:00:00 2001 From: Will Deacon Date: Thu, 2 May 2013 13:52:01 +0100 Subject: ARM: lpae: fix definition of PTE_HWTABLE_PTRS For 2-level page tables, PTE_HWTABLE_PTRS describes the offset between Linux PTEs and hardware PTEs. On LPAE, there is no distinction (since we have 64-bit descriptors with plenty of space) so PTE_HWTABLE_PTRS should be 0. Unfortunately, it is wrongly defined as PTRS_PER_PTE, meaning that current pte table flushing is off by a page. Luckily, all current LPAE implementations are SMP, so the hardware walker can snoop L1. This patch fixes the broken definition. Acked-by: Catalin Marinas Signed-off-by: Will Deacon --- arch/arm/include/asm/pgtable-3level.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/arm/include/asm/pgtable-3level.h b/arch/arm/include/asm/pgtable-3level.h index 5b85b218b0f0..d03c589cf6b5 100644 --- a/arch/arm/include/asm/pgtable-3level.h +++ b/arch/arm/include/asm/pgtable-3level.h @@ -33,7 +33,7 @@ #define PTRS_PER_PMD 512 #define PTRS_PER_PGD 4 -#define PTE_HWTABLE_PTRS (PTRS_PER_PTE) +#define PTE_HWTABLE_PTRS (0) #define PTE_HWTABLE_OFF (0) #define PTE_HWTABLE_SIZE (PTRS_PER_PTE * sizeof(u64)) -- cgit From a469abd0f868c902b75532579bf87553dcf1b360 Mon Sep 17 00:00:00 2001 From: Will Deacon Date: Mon, 8 Apr 2013 17:13:12 +0100 Subject: ARM: elf: add new hwcap for identifying atomic ldrd/strd instructions CPUs implementing LPAE have atomic ldrd/strd instructions, meaning that userspace software can avoid having to use the exclusive variants of these instructions if they wish. This patch advertises the atomicity of these instructions via the hwcaps, so userspace can detect this CPU feature. Reported-by: Vladimir Danushevsky Signed-off-by: Will Deacon --- arch/arm/include/uapi/asm/hwcap.h | 2 +- arch/arm/kernel/setup.c | 8 +++++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/arch/arm/include/uapi/asm/hwcap.h b/arch/arm/include/uapi/asm/hwcap.h index 3688fd15a32d..6d34d080372a 100644 --- a/arch/arm/include/uapi/asm/hwcap.h +++ b/arch/arm/include/uapi/asm/hwcap.h @@ -25,6 +25,6 @@ #define HWCAP_IDIVT (1 << 18) #define HWCAP_VFPD32 (1 << 19) /* set if VFP has 32 regs (not 16) */ #define HWCAP_IDIV (HWCAP_IDIVA | HWCAP_IDIVT) - +#define HWCAP_LPAE (1 << 20) #endif /* _UAPI__ASMARM_HWCAP_H */ diff --git a/arch/arm/kernel/setup.c b/arch/arm/kernel/setup.c index 1522c7ae31b0..bdcd4dd13230 100644 --- a/arch/arm/kernel/setup.c +++ b/arch/arm/kernel/setup.c @@ -355,7 +355,7 @@ void __init early_print(const char *str, ...) static void __init cpuid_init_hwcaps(void) { - unsigned int divide_instrs; + unsigned int divide_instrs, vmsa; if (cpu_architecture() < CPU_ARCH_ARMv7) return; @@ -368,6 +368,11 @@ static void __init cpuid_init_hwcaps(void) case 1: elf_hwcap |= HWCAP_IDIVT; } + + /* LPAE implies atomic ldrd/strd instructions */ + vmsa = (read_cpuid_ext(CPUID_EXT_MMFR0) & 0xf) >> 0; + if (vmsa >= 5) + elf_hwcap |= HWCAP_LPAE; } static void __init feat_v6_fixup(void) @@ -872,6 +877,7 @@ static const char *hwcap_str[] = { "vfpv4", "idiva", "idivt", + "lpae", NULL }; -- cgit