Having a semaphore in there causes modest performance regressions on heavily
mmap-intensive workloads on some hardware. Specifically, up to 30% in SDET on
NUMAQ and big PPC64.
So switch it back to being a spinlock. This does mean that unmap_vmas() needs
to be told whether or not it is allowed to schedule away; that's simple to do
via the zap_details structure.
This change means that there will be high scheuling latencies when someone
truncates a large file which is currently mmapped, but nobody does that
anyway. The scheduling points in unmap_vmas() are mainly for munmap() and
exit(), and they still will work OK for that.
---
25-akpm/fs/hugetlbfs/inode.c | 4 ++--
25-akpm/fs/inode.c | 2 +-
25-akpm/include/linux/fs.h | 2 +-
25-akpm/include/linux/mm.h | 1 +
25-akpm/kernel/fork.c | 4 ++--
25-akpm/mm/filemap.c | 6 +++---
25-akpm/mm/madvise.c | 10 ++++------
25-akpm/mm/memory.c | 12 ++++++------
25-akpm/mm/mmap.c | 32 ++++++++++++++++----------------
25-akpm/mm/mremap.c | 4 ++--
25-akpm/mm/rmap.c | 12 ++++++------
11 files changed, 44 insertions(+), 45 deletions(-)
diff -puN fs/hugetlbfs/inode.c~i_shared_lock fs/hugetlbfs/inode.c
--- 25/fs/hugetlbfs/inode.c~i_shared_lock 2004-04-20 19:43:45.085048136 -0700
+++ 25-akpm/fs/hugetlbfs/inode.c 2004-04-20 20:17:19.550802792 -0700
@@ -319,12 +319,12 @@ static int hugetlb_vmtruncate(struct ino
pgoff = offset >> HPAGE_SHIFT;
inode->i_size = offset;
- down(&mapping->i_shared_sem);
+ spin_lock(&mapping->i_shared_lock);
if (!list_empty(&mapping->i_mmap))
hugetlb_vmtruncate_list(&mapping->i_mmap, pgoff);
if (!list_empty(&mapping->i_mmap_shared))
hugetlb_vmtruncate_list(&mapping->i_mmap_shared, pgoff);
- up(&mapping->i_shared_sem);
+ spin_unlock(&mapping->i_shared_lock);
truncate_hugepages(mapping, offset);
return 0;
}
diff -puN fs/inode.c~i_shared_lock fs/inode.c
--- 25/fs/inode.c~i_shared_lock 2004-04-20 19:43:45.086047984 -0700
+++ 25-akpm/fs/inode.c 2004-04-20 20:17:19.545803552 -0700
@@ -184,7 +184,7 @@ void inode_init_once(struct inode *inode
init_rwsem(&inode->i_alloc_sem);
INIT_RADIX_TREE(&inode->i_data.page_tree, GFP_ATOMIC);
spin_lock_init(&inode->i_data.tree_lock);
- init_MUTEX(&inode->i_data.i_shared_sem);
+ spin_lock_init(&inode->i_data.i_shared_lock);
atomic_set(&inode->i_data.truncate_count, 0);
INIT_LIST_HEAD(&inode->i_data.private_list);
spin_lock_init(&inode->i_data.private_lock);
diff -puN include/linux/fs.h~i_shared_lock include/linux/fs.h
--- 25/include/linux/fs.h~i_shared_lock 2004-04-20 19:43:45.088047680 -0700
+++ 25-akpm/include/linux/fs.h 2004-04-20 20:17:19.548803096 -0700
@@ -331,7 +331,7 @@ struct address_space {
struct address_space_operations *a_ops; /* methods */
struct list_head i_mmap; /* list of private mappings */
struct list_head i_mmap_shared; /* list of shared mappings */
- struct semaphore i_shared_sem; /* protect both above lists */
+ spinlock_t i_shared_lock; /* protect both above lists */
atomic_t truncate_count; /* Cover race condition with truncate */
unsigned long flags; /* error bits/gfp mask */
struct backing_dev_info *backing_dev_info; /* device readahead, etc */
diff -puN include/linux/mm.h~i_shared_lock include/linux/mm.h
--- 25/include/linux/mm.h~i_shared_lock 2004-04-20 19:43:45.089047528 -0700
+++ 25-akpm/include/linux/mm.h 2004-04-20 20:17:11.791982312 -0700
@@ -442,6 +442,7 @@ struct zap_details {
struct address_space *check_mapping; /* Check page->mapping if set */
pgoff_t first_index; /* Lowest page->index to unmap */
pgoff_t last_index; /* Highest page->index to unmap */
+ int atomic; /* May not schedule() */
};
void zap_page_range(struct vm_area_struct *vma, unsigned long address,
diff -puN kernel/fork.c~i_shared_lock kernel/fork.c
--- 25/kernel/fork.c~i_shared_lock 2004-04-20 19:43:45.091047224 -0700
+++ 25-akpm/kernel/fork.c 2004-04-20 20:17:16.219309256 -0700
@@ -323,9 +323,9 @@ static inline int dup_mmap(struct mm_str
atomic_dec(&inode->i_writecount);
/* insert tmp into the share list, just after mpnt */
- down(&file->f_mapping->i_shared_sem);
+ spin_lock(&file->f_mapping->i_shared_lock);
list_add(&tmp->shared, &mpnt->shared);
- up(&file->f_mapping->i_shared_sem);
+ spin_unlock(&file->f_mapping->i_shared_lock);
}
/*
diff -puN mm/filemap.c~i_shared_lock mm/filemap.c
--- 25/mm/filemap.c~i_shared_lock 2004-04-20 19:43:45.092047072 -0700
+++ 25-akpm/mm/filemap.c 2004-04-20 19:43:45.105045096 -0700
@@ -55,17 +55,17 @@
/*
* Lock ordering:
*
- * ->i_shared_sem (vmtruncate)
+ * ->i_shared_lock (vmtruncate)
* ->private_lock (__free_pte->__set_page_dirty_buffers)
* ->swap_list_lock
* ->swap_device_lock (exclusive_swap_page, others)
* ->mapping->tree_lock
*
* ->i_sem
- * ->i_shared_sem (truncate->unmap_mapping_range)
+ * ->i_shared_lock (truncate->unmap_mapping_range)
*
* ->mmap_sem
- * ->i_shared_sem (various places)
+ * ->i_shared_lock (various places)
*
* ->mmap_sem
* ->lock_page (access_process_vm)
diff -puN mm/madvise.c~i_shared_lock mm/madvise.c
--- 25/mm/madvise.c~i_shared_lock 2004-04-20 19:43:45.094046768 -0700
+++ 25-akpm/mm/madvise.c 2004-04-20 19:43:45.113043880 -0700
@@ -92,16 +92,14 @@ static long madvise_willneed(struct vm_a
static long madvise_dontneed(struct vm_area_struct * vma,
unsigned long start, unsigned long end)
{
- struct zap_details details;
-
if (vma->vm_flags & VM_LOCKED)
return -EINVAL;
if (unlikely(vma->vm_flags & VM_NONLINEAR)) {
- details.check_mapping = NULL;
- details.nonlinear_vma = vma;
- details.first_index = 0;
- details.last_index = ULONG_MAX;
+ struct zap_details details = {
+ .nonlinear_vma = vma,
+ .last_index = ULONG_MAX,
+ };
zap_page_range(vma, start, end - start, &details);
} else
zap_page_range(vma, start, end - start, NULL);
diff -puN mm/memory.c~i_shared_lock mm/memory.c
--- 25/mm/memory.c~i_shared_lock 2004-04-20 19:43:45.095046616 -0700
+++ 25-akpm/mm/memory.c 2004-04-20 20:17:49.132305720 -0700
@@ -518,6 +518,7 @@ int unmap_vmas(struct mmu_gather **tlbp,
unsigned long tlb_start = 0; /* For tlb_finish_mmu */
int tlb_start_valid = 0;
int ret = 0;
+ int atomic = details && details->atomic;
for ( ; vma && vma->vm_start < end_addr; vma = vma->vm_next) {
unsigned long start;
@@ -555,7 +556,7 @@ int unmap_vmas(struct mmu_gather **tlbp,
zap_bytes -= block;
if ((long)zap_bytes > 0)
continue;
- if (need_resched()) {
+ if (!atomic && need_resched()) {
int fullmm = tlb_is_full_mm(*tlbp);
tlb_finish_mmu(*tlbp, tlb_start, start);
cond_resched_lock(&mm->page_table_lock);
@@ -583,8 +584,6 @@ void zap_page_range(struct vm_area_struc
unsigned long end = address + size;
unsigned long nr_accounted = 0;
- might_sleep();
-
if (is_vm_hugetlb_page(vma)) {
zap_hugepage_range(vma, address, size);
return;
@@ -1155,7 +1154,7 @@ static void unmap_mapping_range_list(str
* but 0 when invalidating pagecache, don't throw away private data.
*/
void unmap_mapping_range(struct address_space *mapping,
- loff_t const holebegin, loff_t const holelen, int even_cows)
+ loff_t const holebegin, loff_t const holelen, int even_cows)
{
struct zap_details details;
pgoff_t hba = holebegin >> PAGE_SHIFT;
@@ -1173,10 +1172,11 @@ void unmap_mapping_range(struct address_
details.nonlinear_vma = NULL;
details.first_index = hba;
details.last_index = hba + hlen - 1;
+ details.atomic = 1; /* A spinlock is held */
if (details.last_index < details.first_index)
details.last_index = ULONG_MAX;
- down(&mapping->i_shared_sem);
+ spin_lock(&mapping->i_shared_lock);
/* Protect against page fault */
atomic_inc(&mapping->truncate_count);
if (unlikely(!list_empty(&mapping->i_mmap)))
@@ -1187,7 +1187,7 @@ void unmap_mapping_range(struct address_
if (unlikely(!list_empty(&mapping->i_mmap_shared)))
unmap_mapping_range_list(&mapping->i_mmap_shared, &details);
- up(&mapping->i_shared_sem);
+ spin_unlock(&mapping->i_shared_lock);
}
EXPORT_SYMBOL(unmap_mapping_range);
diff -puN mm/mmap.c~i_shared_lock mm/mmap.c
--- 25/mm/mmap.c~i_shared_lock 2004-04-20 19:43:45.097046312 -0700
+++ 25-akpm/mm/mmap.c 2004-04-20 20:17:11.646004504 -0700
@@ -63,7 +63,7 @@ EXPORT_SYMBOL(sysctl_max_map_count);
EXPORT_SYMBOL(vm_committed_space);
/*
- * Requires inode->i_mapping->i_shared_sem
+ * Requires inode->i_mapping->i_shared_lock
*/
static inline void
__remove_shared_vm_struct(struct vm_area_struct *vma, struct inode *inode)
@@ -84,9 +84,9 @@ static void remove_shared_vm_struct(stru
if (file) {
struct address_space *mapping = file->f_mapping;
- down(&mapping->i_shared_sem);
+ spin_lock(&mapping->i_shared_lock);
__remove_shared_vm_struct(vma, file->f_dentry->d_inode);
- up(&mapping->i_shared_sem);
+ spin_unlock(&mapping->i_shared_lock);
}
}
@@ -286,12 +286,12 @@ static void vma_link(struct mm_struct *m
mapping = vma->vm_file->f_mapping;
if (mapping)
- down(&mapping->i_shared_sem);
+ spin_lock(&mapping->i_shared_lock);
spin_lock(&mm->page_table_lock);
__vma_link(mm, vma, prev, rb_link, rb_parent);
spin_unlock(&mm->page_table_lock);
if (mapping)
- up(&mapping->i_shared_sem);
+ spin_unlock(&mapping->i_shared_lock);
mark_mm_hugetlb(mm, vma);
mm->map_count++;
@@ -301,7 +301,7 @@ static void vma_link(struct mm_struct *m
/*
* Insert vm structure into process list sorted by address and into the inode's
* i_mmap ring. The caller should hold mm->page_table_lock and
- * ->f_mappping->i_shared_sem if vm_file is non-NULL.
+ * ->f_mappping->i_shared_lock if vm_file is non-NULL.
*/
static void
__insert_vm_struct(struct mm_struct * mm, struct vm_area_struct * vma)
@@ -391,7 +391,7 @@ static struct vm_area_struct *vma_merge(
{
spinlock_t *lock = &mm->page_table_lock;
struct inode *inode = file ? file->f_dentry->d_inode : NULL;
- struct semaphore *i_shared_sem;
+ spinlock_t *i_shared_lock;
/*
* We later require that vma->vm_flags == vm_flags, so this tests
@@ -400,7 +400,7 @@ static struct vm_area_struct *vma_merge(
if (vm_flags & VM_SPECIAL)
return NULL;
- i_shared_sem = file ? &file->f_mapping->i_shared_sem : NULL;
+ i_shared_lock = file ? &file->f_mapping->i_shared_lock : NULL;
if (!prev) {
prev = rb_entry(rb_parent, struct vm_area_struct, vm_rb);
@@ -417,7 +417,7 @@ static struct vm_area_struct *vma_merge(
if (unlikely(file && prev->vm_next &&
prev->vm_next->vm_file == file)) {
- down(i_shared_sem);
+ spin_lock(i_shared_lock);
need_up = 1;
}
spin_lock(lock);
@@ -435,7 +435,7 @@ static struct vm_area_struct *vma_merge(
__remove_shared_vm_struct(next, inode);
spin_unlock(lock);
if (need_up)
- up(i_shared_sem);
+ spin_unlock(i_shared_lock);
if (file)
fput(file);
@@ -445,7 +445,7 @@ static struct vm_area_struct *vma_merge(
}
spin_unlock(lock);
if (need_up)
- up(i_shared_sem);
+ spin_unlock(i_shared_lock);
return prev;
}
@@ -460,13 +460,13 @@ static struct vm_area_struct *vma_merge(
return NULL;
if (end == prev->vm_start) {
if (file)
- down(i_shared_sem);
+ spin_lock(i_shared_lock);
spin_lock(lock);
prev->vm_start = addr;
prev->vm_pgoff -= (end - addr) >> PAGE_SHIFT;
spin_unlock(lock);
if (file)
- up(i_shared_sem);
+ spin_unlock(i_shared_lock);
return prev;
}
}
@@ -1232,7 +1232,7 @@ int split_vma(struct mm_struct * mm, str
mapping = vma->vm_file->f_mapping;
if (mapping)
- down(&mapping->i_shared_sem);
+ spin_lock(&mapping->i_shared_lock);
spin_lock(&mm->page_table_lock);
if (new_below) {
@@ -1245,7 +1245,7 @@ int split_vma(struct mm_struct * mm, str
spin_unlock(&mm->page_table_lock);
if (mapping)
- up(&mapping->i_shared_sem);
+ spin_unlock(&mapping->i_shared_lock);
return 0;
}
@@ -1479,7 +1479,7 @@ void exit_mmap(struct mm_struct *mm)
/* Insert vm structure into process list sorted by address
* and into the inode's i_mmap ring. If vm_file is non-NULL
- * then i_shared_sem is taken here.
+ * then i_shared_lock is taken here.
*/
void insert_vm_struct(struct mm_struct * mm, struct vm_area_struct * vma)
{
diff -puN mm/mremap.c~i_shared_lock mm/mremap.c
--- 25/mm/mremap.c~i_shared_lock 2004-04-20 19:43:45.098046160 -0700
+++ 25-akpm/mm/mremap.c 2004-04-20 20:17:07.037705072 -0700
@@ -208,7 +208,7 @@ static unsigned long move_vma(struct vm_
* and we propagate stale pages into the dst afterward.
*/
mapping = vma->vm_file->f_mapping;
- down(&mapping->i_shared_sem);
+ spin_lock(&mapping->i_shared_lock);
}
moved_len = move_page_tables(vma, new_addr, old_addr, old_len, &cows);
if (moved_len < old_len) {
@@ -227,7 +227,7 @@ static unsigned long move_vma(struct vm_
printk(KERN_WARNING "%s: mremap moved %d cows\n",
current->comm, cows);
if (mapping)
- up(&mapping->i_shared_sem);
+ spin_unlock(&mapping->i_shared_lock);
/* Conceal VM_ACCOUNT so old reservation is not undone */
if (vm_flags & VM_ACCOUNT) {
diff -puN mm/rmap.c~i_shared_lock mm/rmap.c
--- 25/mm/rmap.c~i_shared_lock 2004-04-20 19:43:45.099046008 -0700
+++ 25-akpm/mm/rmap.c 2004-04-20 19:43:45.110044336 -0700
@@ -298,7 +298,7 @@ migrate:
*
* This function is only called from page_referenced for object-based pages.
*
- * The semaphore address_space->i_shared_sem is tried. If it can't be gotten,
+ * The semaphore address_space->i_shared_lock is tried. If it can't be gotten,
* assume a reference count of 0, so try_to_unmap will then have a go.
*/
static inline int page_referenced_file(struct page *page)
@@ -311,7 +311,7 @@ static inline int page_referenced_file(s
int referenced = 0;
int failed = 0;
- if (down_trylock(&mapping->i_shared_sem))
+ if (!spin_trylock(&mapping->i_shared_lock))
return 0;
list_for_each_entry(vma, &mapping->i_mmap, shared) {
@@ -353,7 +353,7 @@ static inline int page_referenced_file(s
WARN_ON(!failed);
out:
- up(&mapping->i_shared_sem);
+ spin_unlock(&mapping->i_shared_lock);
return referenced;
}
@@ -708,7 +708,7 @@ out:
*
* This function is only called from try_to_unmap for object-based pages.
*
- * The semaphore address_space->i_shared_sem is tried. If it can't be gotten,
+ * The semaphore address_space->i_shared_lock is tried. If it can't be gotten,
* return a temporary error.
*/
static inline int try_to_unmap_file(struct page *page)
@@ -723,7 +723,7 @@ static inline int try_to_unmap_file(stru
unsigned long max_nl_cursor = 0;
unsigned long max_nl_size = 0;
- if (down_trylock(&mapping->i_shared_sem))
+ if (!spin_trylock(&mapping->i_shared_lock))
return ret;
list_for_each_entry(vma, &mapping->i_mmap, shared) {
@@ -822,7 +822,7 @@ static inline int try_to_unmap_file(stru
relock:
rmap_lock(page);
out:
- up(&mapping->i_shared_sem);
+ spin_unlock(&mapping->i_shared_lock);
return ret;
}
_