From: "Andi Kleen" <ak@suse.de>
Always reload CR3 completely when a lazy MM thread drops a MM. This avoids
keeping stale mappings around in the TLB that could be run into by the CPU by
itself (e.g. during prefetches).
Signed-off-by: Andi Kleen <ak@suse.de>
Signed-off-by: Andrew Morton <akpm@osdl.org>
---
25-akpm/arch/x86_64/kernel/smp.c | 3 ++-
25-akpm/include/asm-x86_64/mmu_context.h | 10 ++++++++--
2 files changed, 10 insertions(+), 3 deletions(-)
diff -puN arch/x86_64/kernel/smp.c~x86_64-always-reload-cr3-completely-when-a-lazy-mm arch/x86_64/kernel/smp.c
--- 25/arch/x86_64/kernel/smp.c~x86_64-always-reload-cr3-completely-when-a-lazy-mm Wed Mar 23 15:38:58 2005
+++ 25-akpm/arch/x86_64/kernel/smp.c Wed Mar 23 15:38:58 2005
@@ -25,6 +25,7 @@
#include <asm/pgalloc.h>
#include <asm/tlbflush.h>
#include <asm/mach_apic.h>
+#include <asm/mmu_context.h>
#include <asm/proto.h>
/*
@@ -52,7 +53,7 @@ static inline void leave_mm (unsigned lo
if (read_pda(mmu_state) == TLBSTATE_OK)
BUG();
clear_bit(cpu, &read_pda(active_mm)->cpu_vm_mask);
- __flush_tlb();
+ load_cr3(swapper_pg_dir);
}
/*
diff -puN include/asm-x86_64/mmu_context.h~x86_64-always-reload-cr3-completely-when-a-lazy-mm include/asm-x86_64/mmu_context.h
--- 25/include/asm-x86_64/mmu_context.h~x86_64-always-reload-cr3-completely-when-a-lazy-mm Wed Mar 23 15:38:58 2005
+++ 25-akpm/include/asm-x86_64/mmu_context.h Wed Mar 23 15:38:58 2005
@@ -28,6 +28,11 @@ static inline void enter_lazy_tlb(struct
}
#endif
+static inline void load_cr3(pgd_t *pgd)
+{
+ asm volatile("movq %0,%%cr3" :: "r" (__pa(pgd)) : "memory");
+}
+
static inline void switch_mm(struct mm_struct *prev, struct mm_struct *next,
struct task_struct *tsk)
{
@@ -40,7 +45,8 @@ static inline void switch_mm(struct mm_s
write_pda(active_mm, next);
#endif
set_bit(cpu, &next->cpu_vm_mask);
- asm volatile("movq %0,%%cr3" :: "r" (__pa(next->pgd)) : "memory");
+ load_cr3(next->pgd);
+
if (unlikely(next->context.ldt != prev->context.ldt))
load_LDT_nolock(&next->context, cpu);
}
@@ -54,7 +60,7 @@ static inline void switch_mm(struct mm_s
* tlb flush IPI delivery. We must reload CR3
* to make sure to use no freed page tables.
*/
- asm volatile("movq %0,%%cr3" :: "r" (__pa(next->pgd)) : "memory");
+ load_cr3(next->pgd);
load_LDT_nolock(&next->context, cpu);
}
}
_