From 585fa72493edd7d5acb308806e7bb609412c6228 Mon Sep 17 00:00:00 2001 From: Ralf Baechle Date: Sat, 12 Aug 2006 16:40:08 +0100 Subject: [MIPS] Retire flush_icache_page from mm use. On the 34K the redundant cache operations were causing excessive stalls resulting in realtime code running on the second VPE missing its deadline. For all other platforms this patch is just a significant performance improvment as illustrated by below benchmark numbers. Processor, Processes - times in microseconds - smaller is better ------------------------------------------------------------------------------ Host OS Mhz null null open slct sig sig fork exec sh call I/O stat clos TCP inst hndl proc proc proc --------- ------------- ---- ---- ---- ---- ---- ---- ---- ---- ---- ---- ---- 25Kf 2.6.18-rc4 533 0.49 1.16 7.57 33.4 30.5 1.34 12.4 5497 17.K 54.K 25Kf 2.6.18-rc4-p 533 0.49 1.16 6.68 23.0 30.7 1.36 8.55 5030 16.K 48.K 4Kc 2.6.18-rc4 80 4.21 15.0 131. 289. 261. 16.5 258. 18.K 70.K 227K 4Kc 2.6.18-rc4-p 80 4.34 13.1 128. 285. 262. 18.2 258. 12.K 52.K 176K 34Kc 2.6.18-rc4 40 5.01 14.0 61.6 90.0 477. 17.9 94.7 29.K 108K 342K 34Kc 2.6.18-rc4-p 40 4.98 13.9 61.2 89.7 475. 17.6 93.7 8758 44.K 158K BCM1480 2.6.18-rc4 700 0.28 0.60 3.68 5.92 16.0 0.78 5.08 931. 3163 15.K BCM1480 2.6.18-rc4-p 700 0.28 0.61 3.65 5.85 16.0 0.79 5.20 395. 1464 8385 TX49-16K 2.6.18-rc3 197 0.73 2.41 19.0 37.8 82.9 2.94 17.5 4438 14.K 56.K TX49-16K 2.6.18-rc3-p 197 0.73 2.40 19.9 36.3 82.9 2.94 23.4 2577 9103 38.K TX49-32K 2.6.18-rc3 396 0.36 1.19 6.80 11.8 41.0 1.46 8.17 2738 8465 32.K TX49-32K 2.6.18-rc3-p 396 0.36 1.19 6.82 10.2 41.0 1.46 8.18 1330 4638 18.K Original patch by me with enhancements by Atsushi Nemoto. Signed-off-by: Ralf Baechle Signed-off-by: Atsushi Nemoto --- arch/mips/mm/c-sb1.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'arch/mips/mm/c-sb1.c') diff --git a/arch/mips/mm/c-sb1.c b/arch/mips/mm/c-sb1.c index 2d71efb82ac5..4bd9ad8a5e07 100644 --- a/arch/mips/mm/c-sb1.c +++ b/arch/mips/mm/c-sb1.c @@ -520,7 +520,7 @@ void sb1_cache_init(void) /* These routines are for Icache coherence with the Dcache */ flush_icache_range = sb1_flush_icache_range; - flush_icache_page = sb1_flush_icache_page; + __flush_icache_page = sb1_flush_icache_page; flush_icache_all = __sb1_flush_icache_all; /* local only */ /* This implies an Icache flush too, so can't be nop'ed */ -- cgit From f6502791d780b22fc147150137704a07a05ba361 Mon Sep 17 00:00:00 2001 From: Atsushi Nemoto Date: Fri, 25 Aug 2006 17:55:31 +0900 Subject: [MIPS] Do not use drop_mmu_context to flusing other task's VIPT I-cache. c-r4k.c and c-sb1.c use drop_mmu_context() to flush virtually tagged I-caches, but this does not work for flushing other task's icache. This is for example triggered by copy_to_user_page() called from ptrace(2). Use indexed flush for such cases. Signed-off-by: Atsushi Nemoto Signed-off-by: Ralf Baechle --- arch/mips/mm/c-r4k.c | 4 ++-- arch/mips/mm/c-sb1.c | 56 +++++++++++++++++++++++++++++----------------------- 2 files changed, 33 insertions(+), 27 deletions(-) (limited to 'arch/mips/mm/c-sb1.c') diff --git a/arch/mips/mm/c-r4k.c b/arch/mips/mm/c-r4k.c index 4e1498246343..2d729f6f6348 100644 --- a/arch/mips/mm/c-r4k.c +++ b/arch/mips/mm/c-r4k.c @@ -475,7 +475,7 @@ static inline void local_r4k_flush_cache_page(void *args) } } if (exec) { - if (cpu_has_vtag_icache) { + if (cpu_has_vtag_icache && mm == current->active_mm) { int cpu = smp_processor_id(); if (cpu_context(cpu, mm) != 0) @@ -599,7 +599,7 @@ static inline void local_r4k_flush_icache_page(void *args) * We're not sure of the virtual address(es) involved here, so * we have to flush the entire I-cache. */ - if (cpu_has_vtag_icache) { + if (cpu_has_vtag_icache && vma->vm_mm == current->active_mm) { int cpu = smp_processor_id(); if (cpu_context(cpu, vma->vm_mm) != 0) diff --git a/arch/mips/mm/c-sb1.c b/arch/mips/mm/c-sb1.c index 4bd9ad8a5e07..16bad7c0a63f 100644 --- a/arch/mips/mm/c-sb1.c +++ b/arch/mips/mm/c-sb1.c @@ -154,6 +154,26 @@ static inline void __sb1_flush_icache_all(void) } } +/* + * Invalidate a range of the icache. The addresses are virtual, and + * the cache is virtually indexed and tagged. However, we don't + * necessarily have the right ASID context, so use index ops instead + * of hit ops. + */ +static inline void __sb1_flush_icache_range(unsigned long start, + unsigned long end) +{ + start &= ~(icache_line_size - 1); + end = (end + icache_line_size - 1) & ~(icache_line_size - 1); + + while (start != end) { + cache_set_op(Index_Invalidate_I, start & icache_index_mask); + start += icache_line_size; + } + mispredict(); + sync(); +} + /* * Flush the icache for a given physical page. Need to writeback the * dcache first, then invalidate the icache. If the page isn't @@ -173,8 +193,11 @@ static void local_sb1_flush_cache_page(struct vm_area_struct *vma, unsigned long /* * Bumping the ASID is probably cheaper than the flush ... */ - if (cpu_context(cpu, vma->vm_mm) != 0) - drop_mmu_context(vma->vm_mm, cpu); + if (vma->vm_mm == current->active_mm) { + if (cpu_context(cpu, vma->vm_mm) != 0) + drop_mmu_context(vma->vm_mm, cpu); + } else + __sb1_flush_icache_range(addr, addr + PAGE_SIZE); } #ifdef CONFIG_SMP @@ -210,26 +233,6 @@ void sb1_flush_cache_page(struct vm_area_struct *vma, unsigned long addr, unsign __attribute__((alias("local_sb1_flush_cache_page"))); #endif -/* - * Invalidate a range of the icache. The addresses are virtual, and - * the cache is virtually indexed and tagged. However, we don't - * necessarily have the right ASID context, so use index ops instead - * of hit ops. - */ -static inline void __sb1_flush_icache_range(unsigned long start, - unsigned long end) -{ - start &= ~(icache_line_size - 1); - end = (end + icache_line_size - 1) & ~(icache_line_size - 1); - - while (start != end) { - cache_set_op(Index_Invalidate_I, start & icache_index_mask); - start += icache_line_size; - } - mispredict(); - sync(); -} - /* * Invalidate all caches on this CPU @@ -326,9 +329,12 @@ static void local_sb1_flush_icache_page(struct vm_area_struct *vma, * If there's a context, bump the ASID (cheaper than a flush, * since we don't know VAs!) */ - if (cpu_context(cpu, vma->vm_mm) != 0) { - drop_mmu_context(vma->vm_mm, cpu); - } + if (vma->vm_mm == current->active_mm) { + if (cpu_context(cpu, vma->vm_mm) != 0) + drop_mmu_context(vma->vm_mm, cpu); + } else + __sb1_flush_icache_range(start, start + PAGE_SIZE); + } #ifdef CONFIG_SMP -- cgit