Merge branch 'for-davem' of git://git.kernel.org/pub/scm/linux/kernel/git/linville...

[safe/jmp/linux-2.6] / mm / rmap.c

diff --git a/mm/rmap.c b/mm/rmap.c
index 23ecd0a..526704e 100644 (file)
--- a/mm/rmap.c
+++ b/mm/rmap.c
@@ -182,7 +182,7 @@ int anon_vma_clone(struct vm_area_struct *dst, struct vm_area_struct *src)
 {
        struct anon_vma_chain *avc, *pavc;
 
-       list_for_each_entry(pavc, &src->anon_vma_chain, same_vma) {
+       list_for_each_entry_reverse(pavc, &src->anon_vma_chain, same_vma) {
                avc = anon_vma_chain_alloc();
                if (!avc)
                        goto enomem_failure;
@@ -232,6 +232,7 @@ int anon_vma_fork(struct vm_area_struct *vma, struct vm_area_struct *pvma)
  out_error_free_anon_vma:
        anon_vma_free(anon_vma);
  out_error:
+       unlink_anon_vmas(vma);
        return -ENOMEM;
 }
 
@@ -329,18 +330,6 @@ vma_address(struct page *page, struct vm_area_struct *vma)
                /* page should be within @vma mapping range */
                return -EFAULT;
        }
-       if (unlikely(vma->vm_flags & VM_LOCK_RMAP)) {
-               /*
-                * This VMA is being unlinked or is not yet linked into the
-                * VMA tree.  Do not try to follow this rmap.  This race
-                * condition can result in page_referenced() ignoring a
-                * reference or in try_to_unmap() failing to unmap a page.
-                * The VMA cannot be freed under us because we hold the
-                * anon_vma->lock, which the munmap code takes while
-                * unlinking the anon_vmas from the VMA.
-                */
-               return -EFAULT;
-       }
        return address;
 }
 
@@ -613,9 +602,6 @@ int page_referenced(struct page *page,
        int referenced = 0;
        int we_locked = 0;
 
-       if (TestClearPageReferenced(page))
-               referenced++;
-
        *vm_flags = 0;
        if (page_mapped(page) && page_rmapping(page)) {
                if (!is_locked && (!PageAnon(page) || PageKsm(page))) {
@@ -716,17 +702,57 @@ int page_mkclean(struct page *page)
 EXPORT_SYMBOL_GPL(page_mkclean);
 
 /**
+ * page_move_anon_rmap - move a page to our anon_vma
+ * @page:      the page to move to our anon_vma
+ * @vma:       the vma the page belongs to
+ * @address:   the user virtual address mapped
+ *
+ * When a page belongs exclusively to one process after a COW event,
+ * that page can be moved into the anon_vma that belongs to just that
+ * process, so the rmap code will not search the parent or sibling
+ * processes.
+ */
+void page_move_anon_rmap(struct page *page,
+       struct vm_area_struct *vma, unsigned long address)
+{
+       struct anon_vma *anon_vma = vma->anon_vma;
+
+       VM_BUG_ON(!PageLocked(page));
+       VM_BUG_ON(!anon_vma);
+       VM_BUG_ON(page->index != linear_page_index(vma, address));
+
+       anon_vma = (void *) anon_vma + PAGE_MAPPING_ANON;
+       page->mapping = (struct address_space *) anon_vma;
+}
+
+/**
  * __page_set_anon_rmap - setup new anonymous rmap
  * @page:      the page to add the mapping to
  * @vma:       the vm area in which the mapping is added
  * @address:   the user virtual address mapped
+ * @exclusive: the page is exclusively owned by the current process
  */
 static void __page_set_anon_rmap(struct page *page,
-       struct vm_area_struct *vma, unsigned long address)
+       struct vm_area_struct *vma, unsigned long address, int exclusive)
 {
        struct anon_vma *anon_vma = vma->anon_vma;
 
        BUG_ON(!anon_vma);
+
+       /*
+        * If the page isn't exclusively mapped into this vma,
+        * we must use the _oldest_ possible anon_vma for the
+        * page mapping!
+        *
+        * So take the last AVC chain entry in the vma, which is
+        * the deepest ancestor, and use the anon_vma from that.
+        */
+       if (!exclusive) {
+               struct anon_vma_chain *avc;
+               avc = list_entry(vma->anon_vma_chain.prev, struct anon_vma_chain, same_vma);
+               anon_vma = avc->anon_vma;
+       }
+
        anon_vma = (void *) anon_vma + PAGE_MAPPING_ANON;
        page->mapping = (struct address_space *) anon_vma;
        page->index = linear_page_index(vma, address);
@@ -781,7 +807,7 @@ void page_add_anon_rmap(struct page *page,
        VM_BUG_ON(!PageLocked(page));
        VM_BUG_ON(address < vma->vm_start || address >= vma->vm_end);
        if (first)
-               __page_set_anon_rmap(page, vma, address);
+               __page_set_anon_rmap(page, vma, address, 0);
        else
                __page_check_anon_rmap(page, vma, address);
 }
@@ -803,7 +829,7 @@ void page_add_new_anon_rmap(struct page *page,
        SetPageSwapBacked(page);
        atomic_set(&page->_mapcount, 0); /* increment count (starts at -1) */
        __inc_zone_page_state(page, NR_ANON_PAGES);
-       __page_set_anon_rmap(page, vma, address);
+       __page_set_anon_rmap(page, vma, address, 1);
        if (page_evictable(page, vma))
                lru_cache_add_lru(page, LRU_ACTIVE_ANON);
        else