do_mremap() untangling, part 3

[safe/jmp/linux-2.6] / mm / ksm.c

diff --git a/mm/ksm.c b/mm/ksm.c
index 722e3f2..5575f86 100644 (file)
--- a/mm/ksm.c
+++ b/mm/ksm.c
@@ -30,9 +30,9 @@
 #include <linux/slab.h>
 #include <linux/rbtree.h>
 #include <linux/mmu_notifier.h>
+#include <linux/swap.h>
 #include <linux/ksm.h>
 
-#include <asm/tlb.h>
 #include <asm/tlbflush.h>
 
 /*
@@ -166,15 +166,15 @@ static unsigned long ksm_rmap_items;
 static unsigned long ksm_max_kernel_pages;
 
 /* Number of pages ksmd should scan in one batch */
-static unsigned int ksm_thread_pages_to_scan;
+static unsigned int ksm_thread_pages_to_scan = 100;
 
 /* Milliseconds ksmd should sleep between batches */
-static unsigned int ksm_thread_sleep_millisecs;
+static unsigned int ksm_thread_sleep_millisecs = 20;
 
 #define KSM_RUN_STOP   0
 #define KSM_RUN_MERGE  1
 #define KSM_RUN_UNMERGE        2
-static unsigned int ksm_run;
+static unsigned int ksm_run = KSM_RUN_STOP;
 
 static DECLARE_WAIT_QUEUE_HEAD(ksm_thread_wait);
 static DEFINE_MUTEX(ksm_thread_mutex);
@@ -285,6 +285,19 @@ static inline int in_stable_tree(struct rmap_item *rmap_item)
 }
 
 /*
+ * ksmd, and unmerge_and_remove_all_rmap_items(), must not touch an mm's
+ * page tables after it has passed through ksm_exit() - which, if necessary,
+ * takes mmap_sem briefly to serialize against them.  ksm_exit() does not set
+ * a special flag: they can just back out as soon as mm_users goes to zero.
+ * ksm_test_exit() is used throughout to make this test for exit: in some
+ * places for correctness, in some places just to avoid unnecessary work.
+ */
+static inline bool ksm_test_exit(struct mm_struct *mm)
+{
+       return atomic_read(&mm->mm_users) == 0;
+}
+
+/*
  * We use break_ksm to break COW on a ksm page: it's a stripped down
  *
  *     if (get_user_pages(current, mm, addr, 1, 1, 1, &page, NULL) == 1)
@@ -506,6 +519,10 @@ static int unmerge_ksm_pages(struct vm_area_struct *vma,
        return err;
 }
 
+#ifdef CONFIG_SYSFS
+/*
+ * Only called through the sysfs control interface:
+ */
 static int unmerge_and_remove_all_rmap_items(void)
 {
        struct mm_slot *mm_slot;
@@ -563,6 +580,7 @@ error:
        spin_unlock(&ksm_mmlist_lock);
        return err;
 }
+#endif /* CONFIG_SYSFS */
 
 static u32 calc_checksum(struct page *page)
 {
@@ -994,6 +1012,7 @@ static struct rmap_item *unstable_tree_search_insert(struct page *page,
                struct rmap_item *tree_rmap_item;
                int ret;
 
+               cond_resched();
                tree_rmap_item = rb_entry(*new, struct rmap_item, node);
                page2[0] = get_mergeable_page(tree_rmap_item);
                if (!page2[0])
@@ -1416,8 +1435,7 @@ int __ksm_enter(struct mm_struct *mm)
        return 0;
 }
 
-void __ksm_exit(struct mm_struct *mm,
-               struct mmu_gather **tlbp, unsigned long end)
+void __ksm_exit(struct mm_struct *mm)
 {
        struct mm_slot *mm_slot;
        int easy_to_free = 0;
@@ -1450,13 +1468,16 @@ void __ksm_exit(struct mm_struct *mm,
                clear_bit(MMF_VM_MERGEABLE, &mm->flags);
                mmdrop(mm);
        } else if (mm_slot) {
-               tlb_finish_mmu(*tlbp, 0, end);
                down_write(&mm->mmap_sem);
                up_write(&mm->mmap_sem);
-               *tlbp = tlb_gather_mmu(mm, 1);
        }
 }
 
+#ifdef CONFIG_SYSFS
+/*
+ * This all compiles without CONFIG_SYSFS, but is a waste of space.
+ */
+
 #define KSM_ATTR_RO(_name) \
        static struct kobj_attribute _name##_attr = __ATTR_RO(_name)
 #define KSM_ATTR(_name) \
@@ -1538,7 +1559,9 @@ static ssize_t run_store(struct kobject *kobj, struct kobj_attribute *attr,
        if (ksm_run != flags) {
                ksm_run = flags;
                if (flags & KSM_RUN_UNMERGE) {
+                       current->flags |= PF_OOM_ORIGIN;
                        err = unmerge_and_remove_all_rmap_items();
+                       current->flags &= ~PF_OOM_ORIGIN;
                        if (err) {
                                ksm_run = KSM_RUN_STOP;
                                count = err;
@@ -1639,12 +1662,15 @@ static struct attribute_group ksm_attr_group = {
        .attrs = ksm_attrs,
        .name = "ksm",
 };
+#endif /* CONFIG_SYSFS */
 
 static int __init ksm_init(void)
 {
        struct task_struct *ksm_thread;
        int err;
 
+       ksm_max_kernel_pages = totalram_pages / 4;
+
        err = ksm_slab_init();
        if (err)
                goto out;
@@ -1660,16 +1686,20 @@ static int __init ksm_init(void)
                goto out_free2;
        }
 
+#ifdef CONFIG_SYSFS
        err = sysfs_create_group(mm_kobj, &ksm_attr_group);
        if (err) {
                printk(KERN_ERR "ksm: register sysfs failed\n");
-               goto out_free3;
+               kthread_stop(ksm_thread);
+               goto out_free2;
        }
+#else
+       ksm_run = KSM_RUN_MERGE;        /* no way for user to start it */
+
+#endif /* CONFIG_SYSFS */
 
        return 0;
 
-out_free3:
-       kthread_stop(ksm_thread);
 out_free2:
        mm_slots_hash_free();
 out_free1: