readahead.c: fix comment

[safe/jmp/linux-2.6] / mm / mempolicy.c

diff --git a/mm/mempolicy.c b/mm/mempolicy.c
index 1311dc4..7575101 100644 (file)
--- a/mm/mempolicy.c
+++ b/mm/mempolicy.c
@@ -63,7 +63,6 @@
    grows down?
    make bind policy root only? It can trigger oom much faster and the
    kernel is not always grateful with that.
    grows down?
    make bind policy root only? It can trigger oom much faster and the
    kernel is not always grateful with that.

-   could replace all the switch()es with a mempolicy_ops structure.

 */
 
 #include <linux/mempolicy.h>
 */
 
 #include <linux/mempolicy.h>


@@ -74,7 +73,6 @@
 #include <linux/sched.h>
 #include <linux/nodemask.h>
 #include <linux/cpuset.h>
 #include <linux/sched.h>
 #include <linux/nodemask.h>
 #include <linux/cpuset.h>

-#include <linux/gfp.h>

 #include <linux/slab.h>
 #include <linux/string.h>
 #include <linux/module.h>
 #include <linux/slab.h>
 #include <linux/string.h>
 #include <linux/module.h>


@@ -86,13 +84,18 @@
 #include <linux/seq_file.h>
 #include <linux/proc_fs.h>
 #include <linux/migrate.h>
 #include <linux/seq_file.h>
 #include <linux/proc_fs.h>
 #include <linux/migrate.h>

+#include <linux/ksm.h>

 #include <linux/rmap.h>
 #include <linux/security.h>
 #include <linux/syscalls.h>
 #include <linux/rmap.h>
 #include <linux/security.h>
 #include <linux/syscalls.h>

+#include <linux/ctype.h>
+#include <linux/mm_inline.h>

 
 #include <asm/tlbflush.h>
 #include <asm/uaccess.h>
 
 
 #include <asm/tlbflush.h>
 #include <asm/uaccess.h>
 

+#include "internal.h"
+

 /* Internal flags */
 #define MPOL_MF_DISCONTIG_OK (MPOL_MF_INTERNAL << 0)   /* Skip checks for continuous vmas */
 #define MPOL_MF_INVERT (MPOL_MF_INTERNAL << 1)         /* Invert check for nodemask */
 /* Internal flags */
 #define MPOL_MF_DISCONTIG_OK (MPOL_MF_INTERNAL << 0)   /* Skip checks for continuous vmas */
 #define MPOL_MF_INVERT (MPOL_MF_INTERNAL << 1)         /* Invert check for nodemask */


@@ -105,74 +108,40 @@ static struct kmem_cache *sn_cache;
    policied. */
 enum zone_type policy_zone = 0;
 
    policied. */
 enum zone_type policy_zone = 0;
 

+/*
+ * run-time system-wide default policy => local allocation
+ */

 struct mempolicy default_policy = {
        .refcnt = ATOMIC_INIT(1), /* never free it */
 struct mempolicy default_policy = {
        .refcnt = ATOMIC_INIT(1), /* never free it */

-       .policy = MPOL_DEFAULT,
+       .mode = MPOL_PREFERRED,
+       .flags = MPOL_F_LOCAL,

 };
 
 };
 

-static void mpol_rebind_policy(struct mempolicy *pol,
-                               const nodemask_t *newmask);
-
-/* Do sanity checking on a policy */
-static int mpol_check_policy(unsigned short mode, nodemask_t *nodes)
-{
-       int was_empty, is_empty;
-
-       if (!nodes)
-               return 0;
-
+static const struct mempolicy_operations {
+       int (*create)(struct mempolicy *pol, const nodemask_t *nodes);

        /*
        /*

-        * "Contextualize" the in-coming nodemast for cpusets:
-        * Remember whether in-coming nodemask was empty,  If not,
-        * restrict the nodes to the allowed nodes in the cpuset.
-        * This is guaranteed to be a subset of nodes with memory.
+        * If read-side task has no lock to protect task->mempolicy, write-side
+        * task will rebind the task->mempolicy by two step. The first step is
+        * setting all the newly nodes, and the second step is cleaning all the
+        * disallowed nodes. In this way, we can avoid finding no node to alloc
+        * page.
+        * If we have a lock to protect task->mempolicy in read-side, we do
+        * rebind directly.
+        *
+        * step:
+        *      MPOL_REBIND_ONCE - do rebind work at once
+        *      MPOL_REBIND_STEP1 - set all the newly nodes
+        *      MPOL_REBIND_STEP2 - clean all the disallowed nodes

         */
         */

-       cpuset_update_task_memory_state();
-       is_empty = was_empty = nodes_empty(*nodes);
-       if (!was_empty) {
-               nodes_and(*nodes, *nodes, cpuset_current_mems_allowed);
-               is_empty = nodes_empty(*nodes); /* after "contextualization" */
-       }
-
-       switch (mode) {
-       case MPOL_DEFAULT:
-               /*
-                * require caller to specify an empty nodemask
-                * before "contextualization"
-                */
-               if (!was_empty)
-                       return -EINVAL;
-               break;
-       case MPOL_BIND:
-       case MPOL_INTERLEAVE:
-               /*
-                * require at least 1 valid node after "contextualization"
-                */
-               if (is_empty)
-                       return -EINVAL;
-               break;
-       case MPOL_PREFERRED:
-               /*
-                * Did caller specify invalid nodes?
-                * Don't silently accept this as "local allocation".
-                */
-               if (!was_empty && is_empty)
-                       return -EINVAL;
-               break;
-       default:
-               BUG();
-       }
-       return 0;
-}
+       void (*rebind)(struct mempolicy *pol, const nodemask_t *nodes,
+                       enum mpol_rebind_step step);
+} mpol_ops[MPOL_MAX];

 
 /* Check that the nodemask contains at least one populated zone */
 
 /* Check that the nodemask contains at least one populated zone */

-static int is_valid_nodemask(nodemask_t *nodemask)
+static int is_valid_nodemask(const nodemask_t *nodemask)

 {
        int nd, k;
 
 {
        int nd, k;
 

-       /* Check that there is something useful in this mask */
-       k = policy_zone;
-

        for_each_node_mask(nd, *nodemask) {
                struct zone *z;
 
        for_each_node_mask(nd, *nodemask) {
                struct zone *z;
 


@@ -186,48 +155,308 @@ static int is_valid_nodemask(nodemask_t *nodemask)
        return 0;
 }
 
        return 0;
 }
 

-/* Create a new policy */
-static struct mempolicy *mpol_new(unsigned short mode, nodemask_t *nodes)
+static inline int mpol_store_user_nodemask(const struct mempolicy *pol)
+{
+       return pol->flags & MPOL_MODE_FLAGS;
+}
+
+static void mpol_relative_nodemask(nodemask_t *ret, const nodemask_t *orig,
+                                  const nodemask_t *rel)
+{
+       nodemask_t tmp;
+       nodes_fold(tmp, *orig, nodes_weight(*rel));
+       nodes_onto(*ret, tmp, *rel);
+}
+
+static int mpol_new_interleave(struct mempolicy *pol, const nodemask_t *nodes)
+{
+       if (nodes_empty(*nodes))
+               return -EINVAL;
+       pol->v.nodes = *nodes;
+       return 0;
+}
+
+static int mpol_new_preferred(struct mempolicy *pol, const nodemask_t *nodes)
+{
+       if (!nodes)
+               pol->flags |= MPOL_F_LOCAL;     /* local allocation */
+       else if (nodes_empty(*nodes))
+               return -EINVAL;                 /*  no allowed nodes */
+       else
+               pol->v.preferred_node = first_node(*nodes);
+       return 0;
+}
+
+static int mpol_new_bind(struct mempolicy *pol, const nodemask_t *nodes)
+{
+       if (!is_valid_nodemask(nodes))
+               return -EINVAL;
+       pol->v.nodes = *nodes;
+       return 0;
+}
+
+/*
+ * mpol_set_nodemask is called after mpol_new() to set up the nodemask, if
+ * any, for the new policy.  mpol_new() has already validated the nodes
+ * parameter with respect to the policy mode and flags.  But, we need to
+ * handle an empty nodemask with MPOL_PREFERRED here.
+ *
+ * Must be called holding task's alloc_lock to protect task's mems_allowed
+ * and mempolicy.  May also be called holding the mmap_semaphore for write.
+ */
+static int mpol_set_nodemask(struct mempolicy *pol,
+                    const nodemask_t *nodes, struct nodemask_scratch *nsc)
+{
+       int ret;
+
+       /* if mode is MPOL_DEFAULT, pol is NULL. This is right. */
+       if (pol == NULL)
+               return 0;
+       /* Check N_HIGH_MEMORY */
+       nodes_and(nsc->mask1,
+                 cpuset_current_mems_allowed, node_states[N_HIGH_MEMORY]);
+
+       VM_BUG_ON(!nodes);
+       if (pol->mode == MPOL_PREFERRED && nodes_empty(*nodes))
+               nodes = NULL;   /* explicit local allocation */
+       else {
+               if (pol->flags & MPOL_F_RELATIVE_NODES)
+                       mpol_relative_nodemask(&nsc->mask2, nodes,&nsc->mask1);
+               else
+                       nodes_and(nsc->mask2, *nodes, nsc->mask1);
+
+               if (mpol_store_user_nodemask(pol))
+                       pol->w.user_nodemask = *nodes;
+               else
+                       pol->w.cpuset_mems_allowed =
+                                               cpuset_current_mems_allowed;
+       }
+
+       if (nodes)
+               ret = mpol_ops[pol->mode].create(pol, &nsc->mask2);
+       else
+               ret = mpol_ops[pol->mode].create(pol, NULL);
+       return ret;
+}
+
+/*
+ * This function just creates a new policy, does some check and simple
+ * initialization. You must invoke mpol_set_nodemask() to set nodes.
+ */
+static struct mempolicy *mpol_new(unsigned short mode, unsigned short flags,
+                                 nodemask_t *nodes)

 {
        struct mempolicy *policy;
 
 {
        struct mempolicy *policy;
 

-       pr_debug("setting mode %d nodes[0] %lx\n",
-                mode, nodes ? nodes_addr(*nodes)[0] : -1);
+       pr_debug("setting mode %d flags %d nodes[0] %lx\n",
+                mode, flags, nodes ? nodes_addr(*nodes)[0] : -1);

 
 

-       if (mode == MPOL_DEFAULT)
-               return NULL;
+       if (mode == MPOL_DEFAULT) {
+               if (nodes && !nodes_empty(*nodes))
+                       return ERR_PTR(-EINVAL);
+               return NULL;    /* simply delete any existing policy */
+       }
+       VM_BUG_ON(!nodes);
+
+       /*
+        * MPOL_PREFERRED cannot be used with MPOL_F_STATIC_NODES or
+        * MPOL_F_RELATIVE_NODES if the nodemask is empty (local allocation).
+        * All other modes require a valid pointer to a non-empty nodemask.
+        */
+       if (mode == MPOL_PREFERRED) {
+               if (nodes_empty(*nodes)) {
+                       if (((flags & MPOL_F_STATIC_NODES) ||
+                            (flags & MPOL_F_RELATIVE_NODES)))
+                               return ERR_PTR(-EINVAL);
+               }
+       } else if (nodes_empty(*nodes))
+               return ERR_PTR(-EINVAL);

        policy = kmem_cache_alloc(policy_cache, GFP_KERNEL);
        if (!policy)
                return ERR_PTR(-ENOMEM);
        atomic_set(&policy->refcnt, 1);
        policy = kmem_cache_alloc(policy_cache, GFP_KERNEL);
        if (!policy)
                return ERR_PTR(-ENOMEM);
        atomic_set(&policy->refcnt, 1);

-       switch (mode) {
-       case MPOL_INTERLEAVE:
-               policy->v.nodes = *nodes;
-               if (nodes_weight(policy->v.nodes) == 0) {
-                       kmem_cache_free(policy_cache, policy);
-                       return ERR_PTR(-EINVAL);
-               }
-               break;
-       case MPOL_PREFERRED:
-               policy->v.preferred_node = first_node(*nodes);
-               if (policy->v.preferred_node >= MAX_NUMNODES)
-                       policy->v.preferred_node = -1;
-               break;
-       case MPOL_BIND:
-               if (!is_valid_nodemask(nodes)) {
-                       kmem_cache_free(policy_cache, policy);
-                       return ERR_PTR(-EINVAL);
-               }
-               policy->v.nodes = *nodes;
-               break;
-       default:
+       policy->mode = mode;
+       policy->flags = flags;
+
+       return policy;
+}
+
+/* Slow path of a mpol destructor. */
+void __mpol_put(struct mempolicy *p)
+{
+       if (!atomic_dec_and_test(&p->refcnt))
+               return;
+       kmem_cache_free(policy_cache, p);
+}
+
+static void mpol_rebind_default(struct mempolicy *pol, const nodemask_t *nodes,
+                               enum mpol_rebind_step step)
+{
+}
+
+/*
+ * step:
+ *     MPOL_REBIND_ONCE  - do rebind work at once
+ *     MPOL_REBIND_STEP1 - set all the newly nodes
+ *     MPOL_REBIND_STEP2 - clean all the disallowed nodes
+ */
+static void mpol_rebind_nodemask(struct mempolicy *pol, const nodemask_t *nodes,
+                                enum mpol_rebind_step step)
+{
+       nodemask_t tmp;
+
+       if (pol->flags & MPOL_F_STATIC_NODES)
+               nodes_and(tmp, pol->w.user_nodemask, *nodes);
+       else if (pol->flags & MPOL_F_RELATIVE_NODES)
+               mpol_relative_nodemask(&tmp, &pol->w.user_nodemask, nodes);
+       else {
+               /*
+                * if step == 1, we use ->w.cpuset_mems_allowed to cache the
+                * result
+                */
+               if (step == MPOL_REBIND_ONCE || step == MPOL_REBIND_STEP1) {
+                       nodes_remap(tmp, pol->v.nodes,
+                                       pol->w.cpuset_mems_allowed, *nodes);
+                       pol->w.cpuset_mems_allowed = step ? tmp : *nodes;
+               } else if (step == MPOL_REBIND_STEP2) {
+                       tmp = pol->w.cpuset_mems_allowed;
+                       pol->w.cpuset_mems_allowed = *nodes;
+               } else
+                       BUG();
+       }
+
+       if (nodes_empty(tmp))
+               tmp = *nodes;
+
+       if (step == MPOL_REBIND_STEP1)
+               nodes_or(pol->v.nodes, pol->v.nodes, tmp);
+       else if (step == MPOL_REBIND_ONCE || step == MPOL_REBIND_STEP2)
+               pol->v.nodes = tmp;
+       else

                BUG();
                BUG();

+
+       if (!node_isset(current->il_next, tmp)) {
+               current->il_next = next_node(current->il_next, tmp);
+               if (current->il_next >= MAX_NUMNODES)
+                       current->il_next = first_node(tmp);
+               if (current->il_next >= MAX_NUMNODES)
+                       current->il_next = numa_node_id();

        }
        }

-       policy->policy = mode;
-       policy->cpuset_mems_allowed = cpuset_mems_allowed(current);
-       return policy;

 }
 
 }
 

+static void mpol_rebind_preferred(struct mempolicy *pol,
+                                 const nodemask_t *nodes,
+                                 enum mpol_rebind_step step)
+{
+       nodemask_t tmp;
+
+       if (pol->flags & MPOL_F_STATIC_NODES) {
+               int node = first_node(pol->w.user_nodemask);
+
+               if (node_isset(node, *nodes)) {
+                       pol->v.preferred_node = node;
+                       pol->flags &= ~MPOL_F_LOCAL;
+               } else
+                       pol->flags |= MPOL_F_LOCAL;
+       } else if (pol->flags & MPOL_F_RELATIVE_NODES) {
+               mpol_relative_nodemask(&tmp, &pol->w.user_nodemask, nodes);
+               pol->v.preferred_node = first_node(tmp);
+       } else if (!(pol->flags & MPOL_F_LOCAL)) {
+               pol->v.preferred_node = node_remap(pol->v.preferred_node,
+                                                  pol->w.cpuset_mems_allowed,
+                                                  *nodes);
+               pol->w.cpuset_mems_allowed = *nodes;
+       }
+}
+
+/*
+ * mpol_rebind_policy - Migrate a policy to a different set of nodes
+ *
+ * If read-side task has no lock to protect task->mempolicy, write-side
+ * task will rebind the task->mempolicy by two step. The first step is
+ * setting all the newly nodes, and the second step is cleaning all the
+ * disallowed nodes. In this way, we can avoid finding no node to alloc
+ * page.
+ * If we have a lock to protect task->mempolicy in read-side, we do
+ * rebind directly.
+ *
+ * step:
+ *     MPOL_REBIND_ONCE  - do rebind work at once
+ *     MPOL_REBIND_STEP1 - set all the newly nodes
+ *     MPOL_REBIND_STEP2 - clean all the disallowed nodes
+ */
+static void mpol_rebind_policy(struct mempolicy *pol, const nodemask_t *newmask,
+                               enum mpol_rebind_step step)
+{
+       if (!pol)
+               return;
+       if (!mpol_store_user_nodemask(pol) && step == 0 &&
+           nodes_equal(pol->w.cpuset_mems_allowed, *newmask))
+               return;
+
+       if (step == MPOL_REBIND_STEP1 && (pol->flags & MPOL_F_REBINDING))
+               return;
+
+       if (step == MPOL_REBIND_STEP2 && !(pol->flags & MPOL_F_REBINDING))
+               BUG();
+
+       if (step == MPOL_REBIND_STEP1)
+               pol->flags |= MPOL_F_REBINDING;
+       else if (step == MPOL_REBIND_STEP2)
+               pol->flags &= ~MPOL_F_REBINDING;
+       else if (step >= MPOL_REBIND_NSTEP)
+               BUG();
+
+       mpol_ops[pol->mode].rebind(pol, newmask, step);
+}
+
+/*
+ * Wrapper for mpol_rebind_policy() that just requires task
+ * pointer, and updates task mempolicy.
+ *
+ * Called with task's alloc_lock held.
+ */
+
+void mpol_rebind_task(struct task_struct *tsk, const nodemask_t *new,
+                       enum mpol_rebind_step step)
+{
+       mpol_rebind_policy(tsk->mempolicy, new, step);
+}
+
+/*
+ * Rebind each vma in mm to new nodemask.
+ *
+ * Call holding a reference to mm.  Takes mm->mmap_sem during call.
+ */
+
+void mpol_rebind_mm(struct mm_struct *mm, nodemask_t *new)
+{
+       struct vm_area_struct *vma;
+
+       down_write(&mm->mmap_sem);
+       for (vma = mm->mmap; vma; vma = vma->vm_next)
+               mpol_rebind_policy(vma->vm_policy, new, MPOL_REBIND_ONCE);
+       up_write(&mm->mmap_sem);
+}
+
+static const struct mempolicy_operations mpol_ops[MPOL_MAX] = {
+       [MPOL_DEFAULT] = {
+               .rebind = mpol_rebind_default,
+       },
+       [MPOL_INTERLEAVE] = {
+               .create = mpol_new_interleave,
+               .rebind = mpol_rebind_nodemask,
+       },
+       [MPOL_PREFERRED] = {
+               .create = mpol_new_preferred,
+               .rebind = mpol_rebind_preferred,
+       },
+       [MPOL_BIND] = {
+               .create = mpol_new_bind,
+               .rebind = mpol_rebind_nodemask,
+       },
+};
+

 static void gather_stats(struct page *, void *, int pte_dirty);
 static void migrate_page_add(struct page *page, struct list_head *pagelist,
                                unsigned long flags);
 static void gather_stats(struct page *, void *, int pte_dirty);
 static void migrate_page_add(struct page *page, struct list_head *pagelist,
                                unsigned long flags);


@@ -253,17 +482,11 @@ static int check_pte_range(struct vm_area_struct *vma, pmd_t *pmd,
                if (!page)
                        continue;
                /*
                if (!page)
                        continue;
                /*

-                * The check for PageReserved here is important to avoid
-                * handling zero pages and other pages that may have been
-                * marked special by the system.
-                *
-                * If the PageReserved would not be checked here then f.e.
-                * the location of the zero page could have an influence
-                * on MPOL_MF_STRICT, zero pages would be counted for
-                * the per node stats, and there would be useless attempts
-                * to put zero pages on the migration list.
+                * vm_normal_page() filters out zero pages, but there might
+                * still be PageReserved pages to skip, perhaps in a VDSO.
+                * And we cannot move PageKsm pages sensibly or safely yet.

                 */
                 */

-               if (PageReserved(page))
+               if (PageReserved(page) || PageKsm(page))

                        continue;
                nid = page_to_nid(page);
                if (node_isset(nid, *nodes) == !!(flags & MPOL_MF_INVERT))
                        continue;
                nid = page_to_nid(page);
                if (node_isset(nid, *nodes) == !!(flags & MPOL_MF_INVERT))


@@ -352,12 +575,6 @@ check_range(struct mm_struct *mm, unsigned long start, unsigned long end,
        int err;
        struct vm_area_struct *first, *vma, *prev;
 
        int err;
        struct vm_area_struct *first, *vma, *prev;
 

-       if (flags & (MPOL_MF_MOVE | MPOL_MF_MOVE_ALL)) {
-
-               err = migrate_prep();
-               if (err)
-                       return ERR_PTR(err);
-       }

 
        first = find_vma(mm, start);
        if (!first)
 
        first = find_vma(mm, start);
        if (!first)


@@ -408,30 +625,56 @@ static int policy_vma(struct vm_area_struct *vma, struct mempolicy *new)
        if (!err) {
                mpol_get(new);
                vma->vm_policy = new;
        if (!err) {
                mpol_get(new);
                vma->vm_policy = new;

-               mpol_free(old);
+               mpol_put(old);

        }
        return err;
 }
 
 /* Step 2: apply policy to a range and do splits. */
        }
        return err;
 }
 
 /* Step 2: apply policy to a range and do splits. */

-static int mbind_range(struct vm_area_struct *vma, unsigned long start,
-                      unsigned long end, struct mempolicy *new)
+static int mbind_range(struct mm_struct *mm, unsigned long start,
+                      unsigned long end, struct mempolicy *new_pol)

 {
        struct vm_area_struct *next;
 {
        struct vm_area_struct *next;

-       int err;
+       struct vm_area_struct *prev;
+       struct vm_area_struct *vma;
+       int err = 0;
+       pgoff_t pgoff;
+       unsigned long vmstart;
+       unsigned long vmend;

 
 

-       err = 0;
-       for (; vma && vma->vm_start < end; vma = next) {
+       vma = find_vma_prev(mm, start, &prev);
+       if (!vma || vma->vm_start > start)
+               return -EFAULT;
+
+       for (; vma && vma->vm_start < end; prev = vma, vma = next) {

                next = vma->vm_next;
                next = vma->vm_next;

-               if (vma->vm_start < start)
-                       err = split_vma(vma->vm_mm, vma, start, 1);
-               if (!err && vma->vm_end > end)
-                       err = split_vma(vma->vm_mm, vma, end, 0);
-               if (!err)
-                       err = policy_vma(vma, new);
+               vmstart = max(start, vma->vm_start);
+               vmend   = min(end, vma->vm_end);
+
+               pgoff = vma->vm_pgoff + ((start - vma->vm_start) >> PAGE_SHIFT);
+               prev = vma_merge(mm, prev, vmstart, vmend, vma->vm_flags,
+                                 vma->anon_vma, vma->vm_file, pgoff, new_pol);
+               if (prev) {
+                       vma = prev;
+                       next = vma->vm_next;
+                       continue;
+               }
+               if (vma->vm_start != vmstart) {
+                       err = split_vma(vma->vm_mm, vma, vmstart, 1);
+                       if (err)
+                               goto out;
+               }
+               if (vma->vm_end != vmend) {
+                       err = split_vma(vma->vm_mm, vma, vmend, 0);
+                       if (err)
+                               goto out;
+               }
+               err = policy_vma(vma, new_pol);

                if (err)
                if (err)

-                       break;
+                       goto out;

        }
        }

+
+ out:

        return err;
 }
 
        return err;
 }
 


@@ -466,41 +709,77 @@ static void mpol_set_task_struct_flag(void)
 }
 
 /* Set the process memory policy */
 }
 
 /* Set the process memory policy */

-static long do_set_mempolicy(unsigned short mode, nodemask_t *nodes)
+static long do_set_mempolicy(unsigned short mode, unsigned short flags,
+                            nodemask_t *nodes)

 {
 {

-       struct mempolicy *new;
+       struct mempolicy *new, *old;
+       struct mm_struct *mm = current->mm;
+       NODEMASK_SCRATCH(scratch);
+       int ret;

 
 

-       if (mpol_check_policy(mode, nodes))
-               return -EINVAL;
-       new = mpol_new(mode, nodes);
-       if (IS_ERR(new))
-               return PTR_ERR(new);
-       mpol_free(current->mempolicy);
+       if (!scratch)
+               return -ENOMEM;
+
+       new = mpol_new(mode, flags, nodes);
+       if (IS_ERR(new)) {
+               ret = PTR_ERR(new);
+               goto out;
+       }
+       /*
+        * prevent changing our mempolicy while show_numa_maps()
+        * is using it.
+        * Note:  do_set_mempolicy() can be called at init time
+        * with no 'mm'.
+        */
+       if (mm)
+               down_write(&mm->mmap_sem);
+       task_lock(current);
+       ret = mpol_set_nodemask(new, nodes, scratch);
+       if (ret) {
+               task_unlock(current);
+               if (mm)
+                       up_write(&mm->mmap_sem);
+               mpol_put(new);
+               goto out;
+       }
+       old = current->mempolicy;

        current->mempolicy = new;
        mpol_set_task_struct_flag();
        current->mempolicy = new;
        mpol_set_task_struct_flag();

-       if (new && new->policy == MPOL_INTERLEAVE)
+       if (new && new->mode == MPOL_INTERLEAVE &&
+           nodes_weight(new->v.nodes))

                current->il_next = first_node(new->v.nodes);
                current->il_next = first_node(new->v.nodes);

-       return 0;
+       task_unlock(current);
+       if (mm)
+               up_write(&mm->mmap_sem);
+
+       mpol_put(old);
+       ret = 0;
+out:
+       NODEMASK_SCRATCH_FREE(scratch);
+       return ret;

 }
 
 }
 

-/* Fill a zone bitmap for a policy */
-static void get_zonemask(struct mempolicy *p, nodemask_t *nodes)
+/*
+ * Return nodemask for policy for get_mempolicy() query
+ *
+ * Called with task's alloc_lock held
+ */
+static void get_policy_nodemask(struct mempolicy *p, nodemask_t *nodes)

 {
        nodes_clear(*nodes);
 {
        nodes_clear(*nodes);

-       switch (p->policy) {
-       case MPOL_DEFAULT:
-               break;
+       if (p == &default_policy)
+               return;
+
+       switch (p->mode) {

        case MPOL_BIND:
                /* Fall through */
        case MPOL_INTERLEAVE:
                *nodes = p->v.nodes;
                break;
        case MPOL_PREFERRED:
        case MPOL_BIND:
                /* Fall through */
        case MPOL_INTERLEAVE:
                *nodes = p->v.nodes;
                break;
        case MPOL_PREFERRED:

-               /* or use current node instead of memory_map? */
-               if (p->v.preferred_node < 0)
-                       *nodes = node_states[N_HIGH_MEMORY];
-               else
+               if (!(p->flags & MPOL_F_LOCAL))

                        node_set(p->v.preferred_node, *nodes);
                        node_set(p->v.preferred_node, *nodes);

+               /* else return empty node mask for local allocation */

                break;
        default:
                BUG();
                break;
        default:
                BUG();


@@ -529,7 +808,6 @@ static long do_get_mempolicy(int *policy, nodemask_t *nmask,
        struct vm_area_struct *vma = NULL;
        struct mempolicy *pol = current->mempolicy;
 
        struct vm_area_struct *vma = NULL;
        struct mempolicy *pol = current->mempolicy;
 

-       cpuset_update_task_memory_state();

        if (flags &
                ~(unsigned long)(MPOL_F_NODE|MPOL_F_ADDR|MPOL_F_MEMS_ALLOWED))
                return -EINVAL;
        if (flags &
                ~(unsigned long)(MPOL_F_NODE|MPOL_F_ADDR|MPOL_F_MEMS_ALLOWED))
                return -EINVAL;


@@ -538,11 +816,18 @@ static long do_get_mempolicy(int *policy, nodemask_t *nmask,
                if (flags & (MPOL_F_NODE|MPOL_F_ADDR))
                        return -EINVAL;
                *policy = 0;    /* just so it's initialized */
                if (flags & (MPOL_F_NODE|MPOL_F_ADDR))
                        return -EINVAL;
                *policy = 0;    /* just so it's initialized */

+               task_lock(current);

                *nmask  = cpuset_current_mems_allowed;
                *nmask  = cpuset_current_mems_allowed;

+               task_unlock(current);

                return 0;
        }
 
        if (flags & MPOL_F_ADDR) {
                return 0;
        }
 
        if (flags & MPOL_F_ADDR) {

+               /*
+                * Do NOT fall back to task policy if the
+                * vma/shared policy at addr is NULL.  We
+                * want to return MPOL_DEFAULT in this case.
+                */

                down_read(&mm->mmap_sem);
                vma = find_vma_intersection(mm, addr, addr+1);
                if (!vma) {
                down_read(&mm->mmap_sem);
                vma = find_vma_intersection(mm, addr, addr+1);
                if (!vma) {


@@ -557,7 +842,7 @@ static long do_get_mempolicy(int *policy, nodemask_t *nmask,
                return -EINVAL;
 
        if (!pol)
                return -EINVAL;
 
        if (!pol)

-               pol = &default_policy;
+               pol = &default_policy;  /* indicates default behavior */

 
        if (flags & MPOL_F_NODE) {
                if (flags & MPOL_F_ADDR) {
 
        if (flags & MPOL_F_NODE) {
                if (flags & MPOL_F_ADDR) {


@@ -566,14 +851,21 @@ static long do_get_mempolicy(int *policy, nodemask_t *nmask,
                                goto out;
                        *policy = err;
                } else if (pol == current->mempolicy &&
                                goto out;
                        *policy = err;
                } else if (pol == current->mempolicy &&

-                               pol->policy == MPOL_INTERLEAVE) {
+                               pol->mode == MPOL_INTERLEAVE) {

                        *policy = current->il_next;
                } else {
                        err = -EINVAL;
                        goto out;
                }
                        *policy = current->il_next;
                } else {
                        err = -EINVAL;
                        goto out;
                }

-       } else
-               *policy = pol->policy;
+       } else {
+               *policy = pol == &default_policy ? MPOL_DEFAULT :
+                                               pol->mode;
+               /*
+                * Internal mempolicy flags must be masked off before exposing
+                * the policy to userspace.
+                */
+               *policy |= (pol->flags & MPOL_MODE_FLAGS);
+       }

 
        if (vma) {
                up_read(&current->mm->mmap_sem);
 
        if (vma) {
                up_read(&current->mm->mmap_sem);


@@ -581,10 +873,18 @@ static long do_get_mempolicy(int *policy, nodemask_t *nmask,
        }
 
        err = 0;
        }
 
        err = 0;

-       if (nmask)
-               get_zonemask(pol, nmask);
+       if (nmask) {
+               if (mpol_store_user_nodemask(pol)) {
+                       *nmask = pol->w.user_nodemask;
+               } else {
+                       task_lock(current);
+                       get_policy_nodemask(pol, nmask);
+                       task_unlock(current);
+               }
+       }

 
  out:
 
  out:

+       mpol_cond_put(pol);

        if (vma)
                up_read(&current->mm->mmap_sem);
        return err;
        if (vma)
                up_read(&current->mm->mmap_sem);
        return err;


@@ -600,13 +900,18 @@ static void migrate_page_add(struct page *page, struct list_head *pagelist,
        /*
         * Avoid migrating a page that is shared with others.
         */
        /*
         * Avoid migrating a page that is shared with others.
         */

-       if ((flags & MPOL_MF_MOVE_ALL) || page_mapcount(page) == 1)
-               isolate_lru_page(page, pagelist);
+       if ((flags & MPOL_MF_MOVE_ALL) || page_mapcount(page) == 1) {
+               if (!isolate_lru_page(page)) {
+                       list_add_tail(&page->lru, pagelist);
+                       inc_zone_page_state(page, NR_ISOLATED_ANON +
+                                           page_is_file_cache(page));
+               }
+       }

 }
 
 static struct page *new_node_page(struct page *page, unsigned long node, int **x)
 {
 }
 
 static struct page *new_node_page(struct page *page, unsigned long node, int **x)
 {

-       return alloc_pages_node(node, GFP_HIGHUSER_MOVABLE, 0);
+       return alloc_pages_exact_node(node, GFP_HIGHUSER_MOVABLE, 0);

 }
 
 /*
 }
 
 /*


@@ -623,11 +928,11 @@ static int migrate_to_node(struct mm_struct *mm, int source, int dest,
        nodes_clear(nmask);
        node_set(source, nmask);
 
        nodes_clear(nmask);
        node_set(source, nmask);
 

-       check_range(mm, mm->mmap->vm_start, TASK_SIZE, &nmask,
+       check_range(mm, mm->mmap->vm_start, mm->task_size, &nmask,

                        flags | MPOL_MF_DISCONTIG_OK, &pagelist);
 
        if (!list_empty(&pagelist))
                        flags | MPOL_MF_DISCONTIG_OK, &pagelist);
 
        if (!list_empty(&pagelist))

-               err = migrate_pages(&pagelist, new_node_page, dest);
+               err = migrate_pages(&pagelist, new_node_page, dest, 0);

 
        return err;
 }
 
        return err;
 }


@@ -641,47 +946,50 @@ static int migrate_to_node(struct mm_struct *mm, int source, int dest,
 int do_migrate_pages(struct mm_struct *mm,
        const nodemask_t *from_nodes, const nodemask_t *to_nodes, int flags)
 {
 int do_migrate_pages(struct mm_struct *mm,
        const nodemask_t *from_nodes, const nodemask_t *to_nodes, int flags)
 {

-       LIST_HEAD(pagelist);

        int busy = 0;
        int busy = 0;

-       int err = 0;
+       int err;

        nodemask_t tmp;
 
        nodemask_t tmp;
 

-       down_read(&mm->mmap_sem);
+       err = migrate_prep();
+       if (err)
+               return err;
+
+       down_read(&mm->mmap_sem);

 
        err = migrate_vmas(mm, from_nodes, to_nodes, flags);
        if (err)
                goto out;
 
 
        err = migrate_vmas(mm, from_nodes, to_nodes, flags);
        if (err)
                goto out;
 

-/*
- * Find a 'source' bit set in 'tmp' whose corresponding 'dest'
- * bit in 'to' is not also set in 'tmp'.  Clear the found 'source'
- * bit in 'tmp', and return that <source, dest> pair for migration.
- * The pair of nodemasks 'to' and 'from' define the map.
- *
- * If no pair of bits is found that way, fallback to picking some
- * pair of 'source' and 'dest' bits that are not the same.  If the
- * 'source' and 'dest' bits are the same, this represents a node
- * that will be migrating to itself, so no pages need move.
- *
- * If no bits are left in 'tmp', or if all remaining bits left
- * in 'tmp' correspond to the same bit in 'to', return false
- * (nothing left to migrate).
- *
- * This lets us pick a pair of nodes to migrate between, such that
- * if possible the dest node is not already occupied by some other
- * source node, minimizing the risk of overloading the memory on a
- * node that would happen if we migrated incoming memory to a node
- * before migrating outgoing memory source that same node.
- *
- * A single scan of tmp is sufficient.  As we go, we remember the
- * most recent <s, d> pair that moved (s != d).  If we find a pair
- * that not only moved, but what's better, moved to an empty slot
- * (d is not set in tmp), then we break out then, with that pair.
- * Otherwise when we finish scannng from_tmp, we at least have the
- * most recent <s, d> pair that moved.  If we get all the way through
- * the scan of tmp without finding any node that moved, much less
- * moved to an empty node, then there is nothing left worth migrating.
- */
+       /*
+        * Find a 'source' bit set in 'tmp' whose corresponding 'dest'
+        * bit in 'to' is not also set in 'tmp'.  Clear the found 'source'
+        * bit in 'tmp', and return that <source, dest> pair for migration.
+        * The pair of nodemasks 'to' and 'from' define the map.
+        *
+        * If no pair of bits is found that way, fallback to picking some
+        * pair of 'source' and 'dest' bits that are not the same.  If the
+        * 'source' and 'dest' bits are the same, this represents a node
+        * that will be migrating to itself, so no pages need move.
+        *
+        * If no bits are left in 'tmp', or if all remaining bits left
+        * in 'tmp' correspond to the same bit in 'to', return false
+        * (nothing left to migrate).
+        *
+        * This lets us pick a pair of nodes to migrate between, such that
+        * if possible the dest node is not already occupied by some other
+        * source node, minimizing the risk of overloading the memory on a
+        * node that would happen if we migrated incoming memory to a node
+        * before migrating outgoing memory source that same node.
+        *
+        * A single scan of tmp is sufficient.  As we go, we remember the
+        * most recent <s, d> pair that moved (s != d).  If we find a pair
+        * that not only moved, but what's better, moved to an empty slot
+        * (d is not set in tmp), then we break out then, with that pair.
+        * Otherwise when we finish scannng from_tmp, we at least have the
+        * most recent <s, d> pair that moved.  If we get all the way through
+        * the scan of tmp without finding any node that moved, much less
+        * moved to an empty node, then there is nothing left worth migrating.
+        */

 
        tmp = *from_nodes;
        while (!nodes_empty(tmp)) {
 
        tmp = *from_nodes;
        while (!nodes_empty(tmp)) {


@@ -763,8 +1071,8 @@ static struct page *new_vma_page(struct page *page, unsigned long private, int *
 #endif
 
 static long do_mbind(unsigned long start, unsigned long len,
 #endif
 
 static long do_mbind(unsigned long start, unsigned long len,

-                    unsigned short mode, nodemask_t *nmask,
-                    unsigned long flags)
+                    unsigned short mode, unsigned short mode_flags,
+                    nodemask_t *nmask, unsigned long flags)

 {
        struct vm_area_struct *vma;
        struct mm_struct *mm = current->mm;
 {
        struct vm_area_struct *vma;
        struct mm_struct *mm = current->mm;


@@ -793,10 +1101,7 @@ static long do_mbind(unsigned long start, unsigned long len,
        if (end == start)
                return 0;
 
        if (end == start)
                return 0;
 

-       if (mpol_check_policy(mode, nmask))
-               return -EINVAL;
-
-       new = mpol_new(mode, nmask);
+       new = mpol_new(mode, mode_flags, nmask);

        if (IS_ERR(new))
                return PTR_ERR(new);
 
        if (IS_ERR(new))
                return PTR_ERR(new);
 


@@ -807,10 +1112,32 @@ static long do_mbind(unsigned long start, unsigned long len,
        if (!new)
                flags |= MPOL_MF_DISCONTIG_OK;
 
        if (!new)
                flags |= MPOL_MF_DISCONTIG_OK;
 

-       pr_debug("mbind %lx-%lx mode:%d nodes:%lx\n", start, start + len,
-                mode, nmask ? nodes_addr(*nmask)[0] : -1);
+       pr_debug("mbind %lx-%lx mode:%d flags:%d nodes:%lx\n",
+                start, start + len, mode, mode_flags,
+                nmask ? nodes_addr(*nmask)[0] : -1);
+
+       if (flags & (MPOL_MF_MOVE | MPOL_MF_MOVE_ALL)) {
+
+               err = migrate_prep();
+               if (err)
+                       goto mpol_out;
+       }
+       {
+               NODEMASK_SCRATCH(scratch);
+               if (scratch) {
+                       down_write(&mm->mmap_sem);
+                       task_lock(current);
+                       err = mpol_set_nodemask(new, nmask, scratch);
+                       task_unlock(current);
+                       if (err)
+                               up_write(&mm->mmap_sem);
+               } else
+                       err = -ENOMEM;
+               NODEMASK_SCRATCH_FREE(scratch);
+       }
+       if (err)
+               goto mpol_out;

 
 

-       down_write(&mm->mmap_sem);

        vma = check_range(mm, start, end, nmask,
                          flags | MPOL_MF_INVERT, &pagelist);
 
        vma = check_range(mm, start, end, nmask,
                          flags | MPOL_MF_INVERT, &pagelist);
 


@@ -818,18 +1145,20 @@ static long do_mbind(unsigned long start, unsigned long len,
        if (!IS_ERR(vma)) {
                int nr_failed = 0;
 
        if (!IS_ERR(vma)) {
                int nr_failed = 0;
 

-               err = mbind_range(vma, start, end, new);
+               err = mbind_range(mm, start, end, new);

 
                if (!list_empty(&pagelist))
                        nr_failed = migrate_pages(&pagelist, new_vma_page,
 
                if (!list_empty(&pagelist))
                        nr_failed = migrate_pages(&pagelist, new_vma_page,

-                                               (unsigned long)vma);
+                                               (unsigned long)vma, 0);

 
                if (!err && nr_failed && (flags & MPOL_MF_STRICT))
                        err = -EIO;
 
                if (!err && nr_failed && (flags & MPOL_MF_STRICT))
                        err = -EIO;

-       }
+       } else
+               putback_lru_pages(&pagelist);

 
        up_write(&mm->mmap_sem);
 
        up_write(&mm->mmap_sem);

-       mpol_free(new);
+ mpol_out:
+       mpol_put(new);

        return err;
 }
 
        return err;
 }
 


@@ -900,41 +1229,52 @@ static int copy_nodes_to_user(unsigned long __user *mask, unsigned long maxnode,
        return copy_to_user(mask, nodes_addr(*nodes), copy) ? -EFAULT : 0;
 }
 
        return copy_to_user(mask, nodes_addr(*nodes), copy) ? -EFAULT : 0;
 }
 

-asmlinkage long sys_mbind(unsigned long start, unsigned long len,
-                       unsigned long mode,
-                       unsigned long __user *nmask, unsigned long maxnode,
-                       unsigned flags)
+SYSCALL_DEFINE6(mbind, unsigned long, start, unsigned long, len,
+               unsigned long, mode, unsigned long __user *, nmask,
+               unsigned long, maxnode, unsigned, flags)

 {
        nodemask_t nodes;
        int err;
 {
        nodemask_t nodes;
        int err;

+       unsigned short mode_flags;

 
 

+       mode_flags = mode & MPOL_MODE_FLAGS;
+       mode &= ~MPOL_MODE_FLAGS;

        if (mode >= MPOL_MAX)
                return -EINVAL;
        if (mode >= MPOL_MAX)
                return -EINVAL;

+       if ((mode_flags & MPOL_F_STATIC_NODES) &&
+           (mode_flags & MPOL_F_RELATIVE_NODES))
+               return -EINVAL;

        err = get_nodes(&nodes, nmask, maxnode);
        if (err)
                return err;
        err = get_nodes(&nodes, nmask, maxnode);
        if (err)
                return err;

-       return do_mbind(start, len, mode, &nodes, flags);
+       return do_mbind(start, len, mode, mode_flags, &nodes, flags);

 }
 
 /* Set the process memory policy */
 }
 
 /* Set the process memory policy */

-asmlinkage long sys_set_mempolicy(int mode, unsigned long __user *nmask,
-               unsigned long maxnode)
+SYSCALL_DEFINE3(set_mempolicy, int, mode, unsigned long __user *, nmask,
+               unsigned long, maxnode)

 {
        int err;
        nodemask_t nodes;
 {
        int err;
        nodemask_t nodes;

+       unsigned short flags;

 
 

-       if (mode < 0 || mode >= MPOL_MAX)
+       flags = mode & MPOL_MODE_FLAGS;
+       mode &= ~MPOL_MODE_FLAGS;
+       if ((unsigned int)mode >= MPOL_MAX)
+               return -EINVAL;
+       if ((flags & MPOL_F_STATIC_NODES) && (flags & MPOL_F_RELATIVE_NODES))

                return -EINVAL;
        err = get_nodes(&nodes, nmask, maxnode);
        if (err)
                return err;
                return -EINVAL;
        err = get_nodes(&nodes, nmask, maxnode);
        if (err)
                return err;

-       return do_set_mempolicy(mode, &nodes);
+       return do_set_mempolicy(mode, flags, &nodes);

 }
 
 }
 

-asmlinkage long sys_migrate_pages(pid_t pid, unsigned long maxnode,
-               const unsigned long __user *old_nodes,
-               const unsigned long __user *new_nodes)
+SYSCALL_DEFINE4(migrate_pages, pid_t, pid, unsigned long, maxnode,
+               const unsigned long __user *, old_nodes,
+               const unsigned long __user *, new_nodes)

 {
 {

+       const struct cred *cred = current_cred(), *tcred;

        struct mm_struct *mm;
        struct task_struct *task;
        nodemask_t old;
        struct mm_struct *mm;
        struct task_struct *task;
        nodemask_t old;


@@ -969,12 +1309,16 @@ asmlinkage long sys_migrate_pages(pid_t pid, unsigned long maxnode,
         * capabilities, superuser privileges or the same
         * userid as the target process.
         */
         * capabilities, superuser privileges or the same
         * userid as the target process.
         */

-       if ((current->euid != task->suid) && (current->euid != task->uid) &&
-           (current->uid != task->suid) && (current->uid != task->uid) &&
+       rcu_read_lock();
+       tcred = __task_cred(task);
+       if (cred->euid != tcred->suid && cred->euid != tcred->uid &&
+           cred->uid  != tcred->suid && cred->uid  != tcred->uid &&

            !capable(CAP_SYS_NICE)) {
            !capable(CAP_SYS_NICE)) {

+               rcu_read_unlock();

                err = -EPERM;
                goto out;
        }
                err = -EPERM;
                goto out;
        }

+       rcu_read_unlock();

 
        task_nodes = cpuset_mems_allowed(task);
        /* Is the user allowed to access the target nodes? */
 
        task_nodes = cpuset_mems_allowed(task);
        /* Is the user allowed to access the target nodes? */


@@ -1001,10 +1345,9 @@ out:
 
 
 /* Retrieve NUMA policy */
 
 
 /* Retrieve NUMA policy */

-asmlinkage long sys_get_mempolicy(int __user *policy,
-                               unsigned long __user *nmask,
-                               unsigned long maxnode,
-                               unsigned long addr, unsigned long flags)
+SYSCALL_DEFINE5(get_mempolicy, int __user *, policy,
+               unsigned long __user *, nmask, unsigned long, maxnode,
+               unsigned long, addr, unsigned long, flags)

 {
        int err;
        int uninitialized_var(pval);
 {
        int err;
        int uninitialized_var(pval);


@@ -1114,37 +1457,41 @@ asmlinkage long compat_sys_mbind(compat_ulong_t start, compat_ulong_t len,
  *
  * Returns effective policy for a VMA at specified address.
  * Falls back to @task or system default policy, as necessary.
  *
  * Returns effective policy for a VMA at specified address.
  * Falls back to @task or system default policy, as necessary.

- * Returned policy has extra reference count if shared, vma,
- * or some other task's policy [show_numa_maps() can pass
- * @task != current].  It is the caller's responsibility to
- * free the reference in these cases.
+ * Current or other task's task mempolicy and non-shared vma policies
+ * are protected by the task's mmap_sem, which must be held for read by
+ * the caller.
+ * Shared policies [those marked as MPOL_F_SHARED] require an extra reference
+ * count--added by the get_policy() vm_op, as appropriate--to protect against
+ * freeing by another task.  It is the caller's responsibility to free the
+ * extra reference for shared policies.

  */
  */

-static struct mempolicy * get_vma_policy(struct task_struct *task,
+static struct mempolicy *get_vma_policy(struct task_struct *task,

                struct vm_area_struct *vma, unsigned long addr)
 {
        struct mempolicy *pol = task->mempolicy;
                struct vm_area_struct *vma, unsigned long addr)
 {
        struct mempolicy *pol = task->mempolicy;

-       int shared_pol = 0;

 
        if (vma) {
                if (vma->vm_ops && vma->vm_ops->get_policy) {
 
        if (vma) {
                if (vma->vm_ops && vma->vm_ops->get_policy) {

-                       pol = vma->vm_ops->get_policy(vma, addr);
-                       shared_pol = 1; /* if pol non-NULL, add ref below */
-               } else if (vma->vm_policy &&
-                               vma->vm_policy->policy != MPOL_DEFAULT)
+                       struct mempolicy *vpol = vma->vm_ops->get_policy(vma,
+                                                                       addr);
+                       if (vpol)
+                               pol = vpol;
+               } else if (vma->vm_policy)

                        pol = vma->vm_policy;
        }
        if (!pol)
                pol = &default_policy;
                        pol = vma->vm_policy;
        }
        if (!pol)
                pol = &default_policy;

-       else if (!shared_pol && pol != current->mempolicy)
-               mpol_get(pol);  /* vma or other task's policy */

        return pol;
 }
 
        return pol;
 }
 

-/* Return a nodemask representing a mempolicy */
-static nodemask_t *nodemask_policy(gfp_t gfp, struct mempolicy *policy)
+/*
+ * Return a nodemask representing a mempolicy for filtering nodes for
+ * page allocation
+ */
+static nodemask_t *policy_nodemask(gfp_t gfp, struct mempolicy *policy)

 {
        /* Lower zones don't get a nodemask applied for MPOL_BIND */
 {
        /* Lower zones don't get a nodemask applied for MPOL_BIND */

-       if (unlikely(policy->policy == MPOL_BIND) &&
+       if (unlikely(policy->mode == MPOL_BIND) &&

                        gfp_zone(gfp) >= policy_zone &&
                        cpuset_nodemask_valid_mems_allowed(&policy->v.nodes))
                return &policy->v.nodes;
                        gfp_zone(gfp) >= policy_zone &&
                        cpuset_nodemask_valid_mems_allowed(&policy->v.nodes))
                return &policy->v.nodes;


@@ -1152,35 +1499,28 @@ static nodemask_t *nodemask_policy(gfp_t gfp, struct mempolicy *policy)
        return NULL;
 }
 
        return NULL;
 }
 

-/* Return a zonelist representing a mempolicy */
-static struct zonelist *zonelist_policy(gfp_t gfp, struct mempolicy *policy)
+/* Return a zonelist indicated by gfp for node representing a mempolicy */
+static struct zonelist *policy_zonelist(gfp_t gfp, struct mempolicy *policy)

 {
 {

-       int nd;
+       int nd = numa_node_id();

 
 

-       switch (policy->policy) {
+       switch (policy->mode) {

        case MPOL_PREFERRED:
        case MPOL_PREFERRED:

-               nd = policy->v.preferred_node;
-               if (nd < 0)
-                       nd = numa_node_id();
+               if (!(policy->flags & MPOL_F_LOCAL))
+                       nd = policy->v.preferred_node;

                break;
        case MPOL_BIND:
                /*
                break;
        case MPOL_BIND:
                /*

-                * Normally, MPOL_BIND allocations node-local are node-local
-                * within the allowed nodemask. However, if __GFP_THISNODE is
-                * set and the current node is part of the mask, we use the
-                * the zonelist for the first node in the mask instead.
+                * Normally, MPOL_BIND allocations are node-local within the
+                * allowed nodemask.  However, if __GFP_THISNODE is set and the
+                * current node isn't part of the mask, we use the zonelist for
+                * the first node in the mask instead.

                 */
                 */

-               nd = numa_node_id();

                if (unlikely(gfp & __GFP_THISNODE) &&
                                unlikely(!node_isset(nd, policy->v.nodes)))
                        nd = first_node(policy->v.nodes);
                break;
                if (unlikely(gfp & __GFP_THISNODE) &&
                                unlikely(!node_isset(nd, policy->v.nodes)))
                        nd = first_node(policy->v.nodes);
                break;

-       case MPOL_INTERLEAVE: /* should not happen */
-       case MPOL_DEFAULT:
-               nd = numa_node_id();
-               break;

        default:
        default:

-               nd = 0;

                BUG();
        }
        return node_zonelist(nd, gfp);
                BUG();
        }
        return node_zonelist(nd, gfp);


@@ -1196,19 +1536,31 @@ static unsigned interleave_nodes(struct mempolicy *policy)
        next = next_node(nid, policy->v.nodes);
        if (next >= MAX_NUMNODES)
                next = first_node(policy->v.nodes);
        next = next_node(nid, policy->v.nodes);
        if (next >= MAX_NUMNODES)
                next = first_node(policy->v.nodes);

-       me->il_next = next;
+       if (next < MAX_NUMNODES)
+               me->il_next = next;

        return nid;
 }
 
 /*
  * Depending on the memory policy provide a node from which to allocate the
  * next slab entry.
        return nid;
 }
 
 /*
  * Depending on the memory policy provide a node from which to allocate the
  * next slab entry.

+ * @policy must be protected by freeing by the caller.  If @policy is
+ * the current task's mempolicy, this protection is implicit, as only the
+ * task can change it's policy.  The system default policy requires no
+ * such protection.

  */
 unsigned slab_node(struct mempolicy *policy)
 {
  */
 unsigned slab_node(struct mempolicy *policy)
 {

-       unsigned short pol = policy ? policy->policy : MPOL_DEFAULT;
+       if (!policy || policy->flags & MPOL_F_LOCAL)
+               return numa_node_id();
+
+       switch (policy->mode) {
+       case MPOL_PREFERRED:
+               /*
+                * handled MPOL_F_LOCAL above
+                */
+               return policy->v.preferred_node;

 
 

-       switch (pol) {

        case MPOL_INTERLEAVE:
                return interleave_nodes(policy);
 
        case MPOL_INTERLEAVE:
                return interleave_nodes(policy);
 


@@ -1227,13 +1579,8 @@ unsigned slab_node(struct mempolicy *policy)
                return zone->node;
        }
 
                return zone->node;
        }
 

-       case MPOL_PREFERRED:
-               if (policy->v.preferred_node >= 0)
-                       return policy->v.preferred_node;
-               /* Fall through */
-

        default:
        default:

-               return numa_node_id();
+               BUG();

        }
 }
 
        }
 }
 


@@ -1242,10 +1589,13 @@ static unsigned offset_il_node(struct mempolicy *pol,
                struct vm_area_struct *vma, unsigned long off)
 {
        unsigned nnodes = nodes_weight(pol->v.nodes);
                struct vm_area_struct *vma, unsigned long off)
 {
        unsigned nnodes = nodes_weight(pol->v.nodes);

-       unsigned target = (unsigned)off % nnodes;
+       unsigned target;

        int c;
        int nid = -1;
 
        int c;
        int nid = -1;
 

+       if (!nnodes)
+               return numa_node_id();
+       target = (unsigned int)off % nnodes;

        c = 0;
        do {
                nid = next_node(nid, pol->v.nodes);
        c = 0;
        do {
                nid = next_node(nid, pol->v.nodes);


@@ -1285,45 +1635,80 @@ static inline unsigned interleave_nid(struct mempolicy *pol,
  * @mpol = pointer to mempolicy pointer for reference counted mempolicy
  * @nodemask = pointer to nodemask pointer for MPOL_BIND nodemask
  *
  * @mpol = pointer to mempolicy pointer for reference counted mempolicy
  * @nodemask = pointer to nodemask pointer for MPOL_BIND nodemask
  *

- * Returns a zonelist suitable for a huge page allocation.
- * If the effective policy is 'BIND, returns pointer to local node's zonelist,
- * and a pointer to the mempolicy's @nodemask for filtering the zonelist.
- * If it is also a policy for which get_vma_policy() returns an extra
- * reference, we must hold that reference until after the allocation.
- * In that case, return policy via @mpol so hugetlb allocation can drop
- * the reference. For non-'BIND referenced policies, we can/do drop the
- * reference here, so the caller doesn't need to know about the special case
- * for default and current task policy.
+ * Returns a zonelist suitable for a huge page allocation and a pointer
+ * to the struct mempolicy for conditional unref after allocation.
+ * If the effective policy is 'BIND, returns a pointer to the mempolicy's
+ * @nodemask for filtering the zonelist.
+ *
+ * Must be protected by get_mems_allowed()

  */
 struct zonelist *huge_zonelist(struct vm_area_struct *vma, unsigned long addr,
                                gfp_t gfp_flags, struct mempolicy **mpol,
                                nodemask_t **nodemask)
 {
  */
 struct zonelist *huge_zonelist(struct vm_area_struct *vma, unsigned long addr,
                                gfp_t gfp_flags, struct mempolicy **mpol,
                                nodemask_t **nodemask)
 {

-       struct mempolicy *pol = get_vma_policy(current, vma, addr);

        struct zonelist *zl;
 
        struct zonelist *zl;
 

-       *mpol = NULL;           /* probably no unref needed */
+       *mpol = get_vma_policy(current, vma, addr);

        *nodemask = NULL;       /* assume !MPOL_BIND */
        *nodemask = NULL;       /* assume !MPOL_BIND */

-       if (pol->policy == MPOL_BIND) {
-                       *nodemask = &pol->v.nodes;
-       } else if (pol->policy == MPOL_INTERLEAVE) {
-               unsigned nid;

 
 

-               nid = interleave_nid(pol, vma, addr, HPAGE_SHIFT);
-               if (unlikely(pol != &default_policy &&
-                               pol != current->mempolicy))
-                       __mpol_free(pol);       /* finished with pol */
-               return node_zonelist(nid, gfp_flags);
+       if (unlikely((*mpol)->mode == MPOL_INTERLEAVE)) {
+               zl = node_zonelist(interleave_nid(*mpol, vma, addr,
+                               huge_page_shift(hstate_vma(vma))), gfp_flags);
+       } else {
+               zl = policy_zonelist(gfp_flags, *mpol);
+               if ((*mpol)->mode == MPOL_BIND)
+                       *nodemask = &(*mpol)->v.nodes;

        }
        }

+       return zl;
+}

 
 

-       zl = zonelist_policy(GFP_HIGHUSER, pol);
-       if (unlikely(pol != &default_policy && pol != current->mempolicy)) {
-               if (pol->policy != MPOL_BIND)
-                       __mpol_free(pol);       /* finished with pol */
+/*
+ * init_nodemask_of_mempolicy
+ *
+ * If the current task's mempolicy is "default" [NULL], return 'false'
+ * to indicate default policy.  Otherwise, extract the policy nodemask
+ * for 'bind' or 'interleave' policy into the argument nodemask, or
+ * initialize the argument nodemask to contain the single node for
+ * 'preferred' or 'local' policy and return 'true' to indicate presence
+ * of non-default mempolicy.
+ *
+ * We don't bother with reference counting the mempolicy [mpol_get/put]
+ * because the current task is examining it's own mempolicy and a task's
+ * mempolicy is only ever changed by the task itself.
+ *
+ * N.B., it is the caller's responsibility to free a returned nodemask.
+ */
+bool init_nodemask_of_mempolicy(nodemask_t *mask)
+{
+       struct mempolicy *mempolicy;
+       int nid;
+
+       if (!(mask && current->mempolicy))
+               return false;
+
+       task_lock(current);
+       mempolicy = current->mempolicy;
+       switch (mempolicy->mode) {
+       case MPOL_PREFERRED:
+               if (mempolicy->flags & MPOL_F_LOCAL)
+                       nid = numa_node_id();

                else
                else

-                       *mpol = pol;    /* unref needed after allocation */
+                       nid = mempolicy->v.preferred_node;
+               init_nodemask_of_node(mask, nid);
+               break;
+
+       case MPOL_BIND:
+               /* Fall through */
+       case MPOL_INTERLEAVE:
+               *mask =  mempolicy->v.nodes;
+               break;
+
+       default:
+               BUG();

        }
        }

-       return zl;
+       task_unlock(current);
+
+       return true;

 }
 #endif
 
 }
 #endif
 


@@ -1369,32 +1754,35 @@ alloc_page_vma(gfp_t gfp, struct vm_area_struct *vma, unsigned long addr)
 {
        struct mempolicy *pol = get_vma_policy(current, vma, addr);
        struct zonelist *zl;
 {
        struct mempolicy *pol = get_vma_policy(current, vma, addr);
        struct zonelist *zl;

+       struct page *page;

 
 

-       cpuset_update_task_memory_state();
-
-       if (unlikely(pol->policy == MPOL_INTERLEAVE)) {
+       get_mems_allowed();
+       if (unlikely(pol->mode == MPOL_INTERLEAVE)) {

                unsigned nid;
 
                nid = interleave_nid(pol, vma, addr, PAGE_SHIFT);
                unsigned nid;
 
                nid = interleave_nid(pol, vma, addr, PAGE_SHIFT);

-               if (unlikely(pol != &default_policy &&
-                               pol != current->mempolicy))
-                       __mpol_free(pol);       /* finished with pol */
-               return alloc_page_interleave(gfp, 0, nid);
+               mpol_cond_put(pol);
+               page = alloc_page_interleave(gfp, 0, nid);
+               put_mems_allowed();
+               return page;

        }
        }

-       zl = zonelist_policy(gfp, pol);
-       if (pol != &default_policy && pol != current->mempolicy) {
+       zl = policy_zonelist(gfp, pol);
+       if (unlikely(mpol_needs_cond_ref(pol))) {

                /*
                /*

-                * slow path: ref counted policy -- shared or vma
+                * slow path: ref counted shared policy

                 */
                struct page *page =  __alloc_pages_nodemask(gfp, 0,
                 */
                struct page *page =  __alloc_pages_nodemask(gfp, 0,

-                                               zl, nodemask_policy(gfp, pol));
-               __mpol_free(pol);
+                                               zl, policy_nodemask(gfp, pol));
+               __mpol_put(pol);
+               put_mems_allowed();

                return page;
        }
        /*
         * fast path:  default or task policy
         */
                return page;
        }
        /*
         * fast path:  default or task policy
         */

-       return __alloc_pages_nodemask(gfp, 0, zl, nodemask_policy(gfp, pol));
+       page = __alloc_pages_nodemask(gfp, 0, zl, policy_nodemask(gfp, pol));
+       put_mems_allowed();
+       return page;

 }
 
 /**
 }
 
 /**


@@ -1419,73 +1807,115 @@ alloc_page_vma(gfp_t gfp, struct vm_area_struct *vma, unsigned long addr)
 struct page *alloc_pages_current(gfp_t gfp, unsigned order)
 {
        struct mempolicy *pol = current->mempolicy;
 struct page *alloc_pages_current(gfp_t gfp, unsigned order)
 {
        struct mempolicy *pol = current->mempolicy;

+       struct page *page;

 
 

-       if ((gfp & __GFP_WAIT) && !in_interrupt())
-               cpuset_update_task_memory_state();

        if (!pol || in_interrupt() || (gfp & __GFP_THISNODE))
                pol = &default_policy;
        if (!pol || in_interrupt() || (gfp & __GFP_THISNODE))
                pol = &default_policy;

-       if (pol->policy == MPOL_INTERLEAVE)
-               return alloc_page_interleave(gfp, order, interleave_nodes(pol));
-       return __alloc_pages_nodemask(gfp, order,
-                       zonelist_policy(gfp, pol), nodemask_policy(gfp, pol));
+
+       get_mems_allowed();
+       /*
+        * No reference counting needed for current->mempolicy
+        * nor system default_policy
+        */
+       if (pol->mode == MPOL_INTERLEAVE)
+               page = alloc_page_interleave(gfp, order, interleave_nodes(pol));
+       else
+               page = __alloc_pages_nodemask(gfp, order,
+                       policy_zonelist(gfp, pol), policy_nodemask(gfp, pol));
+       put_mems_allowed();
+       return page;

 }
 EXPORT_SYMBOL(alloc_pages_current);
 
 /*
 }
 EXPORT_SYMBOL(alloc_pages_current);
 
 /*

- * If mpol_copy() sees current->cpuset == cpuset_being_rebound, then it
+ * If mpol_dup() sees current->cpuset == cpuset_being_rebound, then it

  * rebinds the mempolicy its copying by calling mpol_rebind_policy()
  * with the mems_allowed returned by cpuset_mems_allowed().  This
  * keeps mempolicies cpuset relative after its cpuset moves.  See
  * further kernel/cpuset.c update_nodemask().
  * rebinds the mempolicy its copying by calling mpol_rebind_policy()
  * with the mems_allowed returned by cpuset_mems_allowed().  This
  * keeps mempolicies cpuset relative after its cpuset moves.  See
  * further kernel/cpuset.c update_nodemask().

+ *
+ * current's mempolicy may be rebinded by the other task(the task that changes
+ * cpuset's mems), so we needn't do rebind work for current task.

  */
 
  */
 

-/* Slow path of a mempolicy copy */
-struct mempolicy *__mpol_copy(struct mempolicy *old)
+/* Slow path of a mempolicy duplicate */
+struct mempolicy *__mpol_dup(struct mempolicy *old)

 {
        struct mempolicy *new = kmem_cache_alloc(policy_cache, GFP_KERNEL);
 
        if (!new)
                return ERR_PTR(-ENOMEM);
 {
        struct mempolicy *new = kmem_cache_alloc(policy_cache, GFP_KERNEL);
 
        if (!new)
                return ERR_PTR(-ENOMEM);

+
+       /* task's mempolicy is protected by alloc_lock */
+       if (old == current->mempolicy) {
+               task_lock(current);
+               *new = *old;
+               task_unlock(current);
+       } else
+               *new = *old;
+
+       rcu_read_lock();

        if (current_cpuset_is_being_rebound()) {
                nodemask_t mems = cpuset_mems_allowed(current);
        if (current_cpuset_is_being_rebound()) {
                nodemask_t mems = cpuset_mems_allowed(current);

-               mpol_rebind_policy(old, &mems);
+               if (new->flags & MPOL_F_REBINDING)
+                       mpol_rebind_policy(new, &mems, MPOL_REBIND_STEP2);
+               else
+                       mpol_rebind_policy(new, &mems, MPOL_REBIND_ONCE);

        }
        }

-       *new = *old;
+       rcu_read_unlock();

        atomic_set(&new->refcnt, 1);
        return new;
 }
 
        atomic_set(&new->refcnt, 1);
        return new;
 }
 

+/*
+ * If *frompol needs [has] an extra ref, copy *frompol to *tompol ,
+ * eliminate the * MPOL_F_* flags that require conditional ref and
+ * [NOTE!!!] drop the extra ref.  Not safe to reference *frompol directly
+ * after return.  Use the returned value.
+ *
+ * Allows use of a mempolicy for, e.g., multiple allocations with a single
+ * policy lookup, even if the policy needs/has extra ref on lookup.
+ * shmem_readahead needs this.
+ */
+struct mempolicy *__mpol_cond_copy(struct mempolicy *tompol,
+                                               struct mempolicy *frompol)
+{
+       if (!mpol_needs_cond_ref(frompol))
+               return frompol;
+
+       *tompol = *frompol;
+       tompol->flags &= ~MPOL_F_SHARED;        /* copy doesn't need unref */
+       __mpol_put(frompol);
+       return tompol;
+}
+

 /* Slow path of a mempolicy comparison */
 int __mpol_equal(struct mempolicy *a, struct mempolicy *b)
 {
        if (!a || !b)
                return 0;
 /* Slow path of a mempolicy comparison */
 int __mpol_equal(struct mempolicy *a, struct mempolicy *b)
 {
        if (!a || !b)
                return 0;

-       if (a->policy != b->policy)
+       if (a->mode != b->mode)

                return 0;
                return 0;

-       switch (a->policy) {
-       case MPOL_DEFAULT:
-               return 1;
+       if (a->flags != b->flags)
+               return 0;
+       if (mpol_store_user_nodemask(a))
+               if (!nodes_equal(a->w.user_nodemask, b->w.user_nodemask))
+                       return 0;
+
+       switch (a->mode) {

        case MPOL_BIND:
                /* Fall through */
        case MPOL_INTERLEAVE:
                return nodes_equal(a->v.nodes, b->v.nodes);
        case MPOL_PREFERRED:
        case MPOL_BIND:
                /* Fall through */
        case MPOL_INTERLEAVE:
                return nodes_equal(a->v.nodes, b->v.nodes);
        case MPOL_PREFERRED:

-               return a->v.preferred_node == b->v.preferred_node;
+               return a->v.preferred_node == b->v.preferred_node &&
+                       a->flags == b->flags;

        default:
                BUG();
                return 0;
        }
 }
 
        default:
                BUG();
                return 0;
        }
 }
 

-/* Slow path of a mpol destructor. */
-void __mpol_free(struct mempolicy *p)
-{
-       if (!atomic_dec_and_test(&p->refcnt))
-               return;
-       p->policy = MPOL_DEFAULT;
-       kmem_cache_free(policy_cache, p);
-}
-

 /*
  * Shared memory backing store policy support.
  *
 /*
  * Shared memory backing store policy support.
  *


@@ -1548,7 +1978,7 @@ static void sp_insert(struct shared_policy *sp, struct sp_node *new)
        rb_link_node(&new->nd, parent, p);
        rb_insert_color(&new->nd, &sp->root);
        pr_debug("inserting %lx-%lx: %d\n", new->start, new->end,
        rb_link_node(&new->nd, parent, p);
        rb_insert_color(&new->nd, &sp->root);
        pr_debug("inserting %lx-%lx: %d\n", new->start, new->end,

-                new->policy ? new->policy->policy : 0);
+                new->policy ? new->policy->mode : 0);

 }
 
 /* Find shared policy intersecting idx */
 }
 
 /* Find shared policy intersecting idx */


@@ -1574,7 +2004,7 @@ static void sp_delete(struct shared_policy *sp, struct sp_node *n)
 {
        pr_debug("deleting %lx-l%lx\n", n->start, n->end);
        rb_erase(&n->nd, &sp->root);
 {
        pr_debug("deleting %lx-l%lx\n", n->start, n->end);
        rb_erase(&n->nd, &sp->root);

-       mpol_free(n->policy);
+       mpol_put(n->policy);

        kmem_cache_free(sn_cache, n);
 }
 
        kmem_cache_free(sn_cache, n);
 }
 


@@ -1588,6 +2018,7 @@ static struct sp_node *sp_alloc(unsigned long start, unsigned long end,
        n->start = start;
        n->end = end;
        mpol_get(pol);
        n->start = start;
        n->end = end;
        mpol_get(pol);

+       pol->flags |= MPOL_F_SHARED;    /* for unref */

        n->policy = pol;
        return n;
 }
        n->policy = pol;
        return n;
 }


@@ -1634,33 +2065,56 @@ restart:
                sp_insert(sp, new);
        spin_unlock(&sp->lock);
        if (new2) {
                sp_insert(sp, new);
        spin_unlock(&sp->lock);
        if (new2) {

-               mpol_free(new2->policy);
+               mpol_put(new2->policy);

                kmem_cache_free(sn_cache, new2);
        }
        return 0;
 }
 
                kmem_cache_free(sn_cache, new2);
        }
        return 0;
 }
 

-void mpol_shared_policy_init(struct shared_policy *info, unsigned short policy,
-                               nodemask_t *policy_nodes)
-{
-       info->root = RB_ROOT;
-       spin_lock_init(&info->lock);
-
-       if (policy != MPOL_DEFAULT) {
-               struct mempolicy *newpol;
-
-               /* Falls back to MPOL_DEFAULT on any error */
-               newpol = mpol_new(policy, policy_nodes);
-               if (!IS_ERR(newpol)) {
-                       /* Create pseudo-vma that contains just the policy */
-                       struct vm_area_struct pvma;
-
-                       memset(&pvma, 0, sizeof(struct vm_area_struct));
-                       /* Policy covers entire file */
-                       pvma.vm_end = TASK_SIZE;
-                       mpol_set_shared_policy(info, &pvma, newpol);
-                       mpol_free(newpol);
-               }
+/**
+ * mpol_shared_policy_init - initialize shared policy for inode
+ * @sp: pointer to inode shared policy
+ * @mpol:  struct mempolicy to install
+ *
+ * Install non-NULL @mpol in inode's shared policy rb-tree.
+ * On entry, the current task has a reference on a non-NULL @mpol.
+ * This must be released on exit.
+ * This is called at get_inode() calls and we can use GFP_KERNEL.
+ */
+void mpol_shared_policy_init(struct shared_policy *sp, struct mempolicy *mpol)
+{
+       int ret;
+
+       sp->root = RB_ROOT;             /* empty tree == default mempolicy */
+       spin_lock_init(&sp->lock);
+
+       if (mpol) {
+               struct vm_area_struct pvma;
+               struct mempolicy *new;
+               NODEMASK_SCRATCH(scratch);
+
+               if (!scratch)
+                       return;
+               /* contextualize the tmpfs mount point mempolicy */
+               new = mpol_new(mpol->mode, mpol->flags, &mpol->w.user_nodemask);
+               if (IS_ERR(new))
+                       goto put_free; /* no valid nodemask intersection */
+
+               task_lock(current);
+               ret = mpol_set_nodemask(new, &mpol->w.user_nodemask, scratch);
+               task_unlock(current);
+               mpol_put(mpol); /* drop our ref on sb mpol */
+               if (ret)
+                       goto put_free;
+
+               /* Create pseudo-vma that contains just the policy */
+               memset(&pvma, 0, sizeof(struct vm_area_struct));
+               pvma.vm_end = TASK_SIZE;        /* policy covers entire file */
+               mpol_set_shared_policy(sp, &pvma, new); /* adds ref */
+
+put_free:
+               mpol_put(new);                  /* drop initial ref */
+               NODEMASK_SCRATCH_FREE(scratch);

        }
 }
 
        }
 }
 


@@ -1671,9 +2125,10 @@ int mpol_set_shared_policy(struct shared_policy *info,
        struct sp_node *new = NULL;
        unsigned long sz = vma_pages(vma);
 
        struct sp_node *new = NULL;
        unsigned long sz = vma_pages(vma);
 

-       pr_debug("set_shared_policy %lx sz %lu %d %lx\n",
+       pr_debug("set_shared_policy %lx sz %lu %d %d %lx\n",

                 vma->vm_pgoff,
                 vma->vm_pgoff,

-                sz, npol? npol->policy : -1,
+                sz, npol ? npol->mode : -1,
+                npol ? npol->flags : -1,

                 npol ? nodes_addr(npol->v.nodes)[0] : -1);
 
        if (npol) {
                 npol ? nodes_addr(npol->v.nodes)[0] : -1);
 
        if (npol) {


@@ -1701,7 +2156,7 @@ void mpol_free_shared_policy(struct shared_policy *p)
                n = rb_entry(next, struct sp_node, nd);
                next = rb_next(&n->nd);
                rb_erase(&n->nd, &p->root);
                n = rb_entry(next, struct sp_node, nd);
                next = rb_next(&n->nd);
                rb_erase(&n->nd, &p->root);

-               mpol_free(n->policy);
+               mpol_put(n->policy);

                kmem_cache_free(sn_cache, n);
        }
        spin_unlock(&p->lock);
                kmem_cache_free(sn_cache, n);
        }
        spin_unlock(&p->lock);


@@ -1746,96 +2201,206 @@ void __init numa_policy_init(void)
        if (unlikely(nodes_empty(interleave_nodes)))
                node_set(prefer, interleave_nodes);
 
        if (unlikely(nodes_empty(interleave_nodes)))
                node_set(prefer, interleave_nodes);
 

-       if (do_set_mempolicy(MPOL_INTERLEAVE, &interleave_nodes))
+       if (do_set_mempolicy(MPOL_INTERLEAVE, 0, &interleave_nodes))

                printk("numa_policy_init: interleaving failed\n");
 }
 
 /* Reset policy of current process to default */
 void numa_default_policy(void)
 {
                printk("numa_policy_init: interleaving failed\n");
 }
 
 /* Reset policy of current process to default */
 void numa_default_policy(void)
 {

-       do_set_mempolicy(MPOL_DEFAULT, NULL);
+       do_set_mempolicy(MPOL_DEFAULT, 0, NULL);

 }
 
 }
 

-/* Migrate a policy to a different set of nodes */
-static void mpol_rebind_policy(struct mempolicy *pol,
-                              const nodemask_t *newmask)
+/*
+ * Parse and format mempolicy from/to strings
+ */
+
+/*
+ * "local" is pseudo-policy:  MPOL_PREFERRED with MPOL_F_LOCAL flag
+ * Used only for mpol_parse_str() and mpol_to_str()
+ */
+#define MPOL_LOCAL MPOL_MAX
+static const char * const policy_modes[] =
+{
+       [MPOL_DEFAULT]    = "default",
+       [MPOL_PREFERRED]  = "prefer",
+       [MPOL_BIND]       = "bind",
+       [MPOL_INTERLEAVE] = "interleave",
+       [MPOL_LOCAL]      = "local"
+};
+
+
+#ifdef CONFIG_TMPFS
+/**
+ * mpol_parse_str - parse string to mempolicy
+ * @str:  string containing mempolicy to parse
+ * @mpol:  pointer to struct mempolicy pointer, returned on success.
+ * @no_context:  flag whether to "contextualize" the mempolicy
+ *
+ * Format of input:
+ *     <mode>[=<flags>][:<nodelist>]
+ *
+ * if @no_context is true, save the input nodemask in w.user_nodemask in
+ * the returned mempolicy.  This will be used to "clone" the mempolicy in
+ * a specific context [cpuset] at a later time.  Used to parse tmpfs mpol
+ * mount option.  Note that if 'static' or 'relative' mode flags were
+ * specified, the input nodemask will already have been saved.  Saving
+ * it again is redundant, but safe.
+ *
+ * On success, returns 0, else 1
+ */
+int mpol_parse_str(char *str, struct mempolicy **mpol, int no_context)

 {
 {

-       nodemask_t *mpolmask;
-       nodemask_t tmp;
+       struct mempolicy *new = NULL;
+       unsigned short mode;
+       unsigned short uninitialized_var(mode_flags);
+       nodemask_t nodes;
+       char *nodelist = strchr(str, ':');
+       char *flags = strchr(str, '=');
+       int err = 1;
+
+       if (nodelist) {
+               /* NUL-terminate mode or flags string */
+               *nodelist++ = '\0';
+               if (nodelist_parse(nodelist, nodes))
+                       goto out;
+               if (!nodes_subset(nodes, node_states[N_HIGH_MEMORY]))
+                       goto out;
+       } else
+               nodes_clear(nodes);

 
 

-       if (!pol)
-               return;
-       mpolmask = &pol->cpuset_mems_allowed;
-       if (nodes_equal(*mpolmask, *newmask))
-               return;
+       if (flags)
+               *flags++ = '\0';        /* terminate mode string */

 
 

-       switch (pol->policy) {
-       case MPOL_DEFAULT:
+       for (mode = 0; mode <= MPOL_LOCAL; mode++) {
+               if (!strcmp(str, policy_modes[mode])) {
+                       break;
+               }
+       }
+       if (mode > MPOL_LOCAL)
+               goto out;
+
+       switch (mode) {
+       case MPOL_PREFERRED:
+               /*
+                * Insist on a nodelist of one node only
+                */
+               if (nodelist) {
+                       char *rest = nodelist;
+                       while (isdigit(*rest))
+                               rest++;
+                       if (*rest)
+                               goto out;
+               }

                break;
                break;

-       case MPOL_BIND:
-               /* Fall through */

        case MPOL_INTERLEAVE:
        case MPOL_INTERLEAVE:

-               nodes_remap(tmp, pol->v.nodes, *mpolmask, *newmask);
-               pol->v.nodes = tmp;
-               *mpolmask = *newmask;
-               current->il_next = node_remap(current->il_next,
-                                               *mpolmask, *newmask);
-               break;
-       case MPOL_PREFERRED:
-               pol->v.preferred_node = node_remap(pol->v.preferred_node,
-                                               *mpolmask, *newmask);
-               *mpolmask = *newmask;
+               /*
+                * Default to online nodes with memory if no nodelist
+                */
+               if (!nodelist)
+                       nodes = node_states[N_HIGH_MEMORY];

                break;
                break;

-       default:
-               BUG();
+       case MPOL_LOCAL:
+               /*
+                * Don't allow a nodelist;  mpol_new() checks flags
+                */
+               if (nodelist)
+                       goto out;
+               mode = MPOL_PREFERRED;

                break;
                break;

+       case MPOL_DEFAULT:
+               /*
+                * Insist on a empty nodelist
+                */
+               if (!nodelist)
+                       err = 0;
+               goto out;
+       case MPOL_BIND:
+               /*
+                * Insist on a nodelist
+                */
+               if (!nodelist)
+                       goto out;

        }
        }

-}
-
-/*
- * Wrapper for mpol_rebind_policy() that just requires task
- * pointer, and updates task mempolicy.
- */

 
 

-void mpol_rebind_task(struct task_struct *tsk, const nodemask_t *new)
-{
-       mpol_rebind_policy(tsk->mempolicy, new);
-}
+       mode_flags = 0;
+       if (flags) {
+               /*
+                * Currently, we only support two mutually exclusive
+                * mode flags.
+                */
+               if (!strcmp(flags, "static"))
+                       mode_flags |= MPOL_F_STATIC_NODES;
+               else if (!strcmp(flags, "relative"))
+                       mode_flags |= MPOL_F_RELATIVE_NODES;
+               else
+                       goto out;
+       }

 
 

-/*
- * Rebind each vma in mm to new nodemask.
- *
- * Call holding a reference to mm.  Takes mm->mmap_sem during call.
- */
+       new = mpol_new(mode, mode_flags, &nodes);
+       if (IS_ERR(new))
+               goto out;

 
 

-void mpol_rebind_mm(struct mm_struct *mm, nodemask_t *new)
-{
-       struct vm_area_struct *vma;
+       if (no_context) {
+               /* save for contextualization */
+               new->w.user_nodemask = nodes;
+       } else {
+               int ret;
+               NODEMASK_SCRATCH(scratch);
+               if (scratch) {
+                       task_lock(current);
+                       ret = mpol_set_nodemask(new, &nodes, scratch);
+                       task_unlock(current);
+               } else
+                       ret = -ENOMEM;
+               NODEMASK_SCRATCH_FREE(scratch);
+               if (ret) {
+                       mpol_put(new);
+                       goto out;
+               }
+       }
+       err = 0;

 
 

-       down_write(&mm->mmap_sem);
-       for (vma = mm->mmap; vma; vma = vma->vm_next)
-               mpol_rebind_policy(vma->vm_policy, new);
-       up_write(&mm->mmap_sem);
+out:
+       /* Restore string for error message */
+       if (nodelist)
+               *--nodelist = ':';
+       if (flags)
+               *--flags = '=';
+       if (!err)
+               *mpol = new;
+       return err;

 }
 }

+#endif /* CONFIG_TMPFS */

 
 

-/*
- * Display pages allocated per node and memory policy via /proc.
- */
-
-static const char * const policy_types[] =
-       { "default", "prefer", "bind", "interleave" };
-
-/*
+/**
+ * mpol_to_str - format a mempolicy structure for printing
+ * @buffer:  to contain formatted mempolicy string
+ * @maxlen:  length of @buffer
+ * @pol:  pointer to mempolicy to be formatted
+ * @no_context:  "context free" mempolicy - use nodemask in w.user_nodemask
+ *

  * Convert a mempolicy into a string.
  * Returns the number of characters in buffer (if positive)
  * or an error (negative)
  */
  * Convert a mempolicy into a string.
  * Returns the number of characters in buffer (if positive)
  * or an error (negative)
  */

-static inline int mpol_to_str(char *buffer, int maxlen, struct mempolicy *pol)
+int mpol_to_str(char *buffer, int maxlen, struct mempolicy *pol, int no_context)

 {
        char *p = buffer;
        int l;
        nodemask_t nodes;
 {
        char *p = buffer;
        int l;
        nodemask_t nodes;

-       unsigned short mode = pol ? pol->policy : MPOL_DEFAULT;
+       unsigned short mode;
+       unsigned short flags = pol ? pol->flags : 0;
+
+       /*
+        * Sanity check:  room for longest mode, flag and some nodes
+        */
+       VM_BUG_ON(maxlen < strlen("interleave") + strlen("relative") + 16);
+
+       if (!pol || pol == &default_policy)
+               mode = MPOL_DEFAULT;
+       else
+               mode = pol->mode;

 
        switch (mode) {
        case MPOL_DEFAULT:
 
        switch (mode) {
        case MPOL_DEFAULT:


@@ -1844,31 +2409,50 @@ static inline int mpol_to_str(char *buffer, int maxlen, struct mempolicy *pol)
 
        case MPOL_PREFERRED:
                nodes_clear(nodes);
 
        case MPOL_PREFERRED:
                nodes_clear(nodes);

-               node_set(pol->v.preferred_node, nodes);
+               if (flags & MPOL_F_LOCAL)
+                       mode = MPOL_LOCAL;      /* pseudo-policy */
+               else
+                       node_set(pol->v.preferred_node, nodes);

                break;
 
        case MPOL_BIND:
                /* Fall through */
        case MPOL_INTERLEAVE:
                break;
 
        case MPOL_BIND:
                /* Fall through */
        case MPOL_INTERLEAVE:

-               nodes = pol->v.nodes;
+               if (no_context)
+                       nodes = pol->w.user_nodemask;
+               else
+                       nodes = pol->v.nodes;

                break;
 
        default:
                BUG();
                break;
 
        default:
                BUG();

-               return -EFAULT;

        }
 
        }
 

-       l = strlen(policy_types[mode]);
-       if (buffer + maxlen < p + l + 1)
-               return -ENOSPC;
+       l = strlen(policy_modes[mode]);
+       if (buffer + maxlen < p + l + 1)
+               return -ENOSPC;

 
 

-       strcpy(p, policy_types[mode]);
+       strcpy(p, policy_modes[mode]);

        p += l;
 
        p += l;
 

-       if (!nodes_empty(nodes)) {
+       if (flags & MPOL_MODE_FLAGS) {

                if (buffer + maxlen < p + 2)
                        return -ENOSPC;
                *p++ = '=';
                if (buffer + maxlen < p + 2)
                        return -ENOSPC;
                *p++ = '=';

+
+               /*
+                * Currently, the only defined flags are mutually exclusive
+                */
+               if (flags & MPOL_F_STATIC_NODES)
+                       p += snprintf(p, buffer + maxlen - p, "static");
+               else if (flags & MPOL_F_RELATIVE_NODES)
+                       p += snprintf(p, buffer + maxlen - p, "relative");
+       }
+
+       if (!nodes_empty(nodes)) {
+               if (buffer + maxlen < p + 2)
+                       return -ENOSPC;
+               *p++ = ':';

                p += nodelist_scnprintf(p, buffer + maxlen - p, nodes);
        }
        return p - buffer;
                p += nodelist_scnprintf(p, buffer + maxlen - p, nodes);
        }
        return p - buffer;


@@ -1897,7 +2481,7 @@ static void gather_stats(struct page *page, void *private, int pte_dirty)
        if (PageSwapCache(page))
                md->swapcache++;
 
        if (PageSwapCache(page))
                md->swapcache++;
 

-       if (PageActive(page))
+       if (PageActive(page) || PageUnevictable(page))

                md->active++;
 
        if (PageWriteback(page))
                md->active++;
 
        if (PageWriteback(page))


@@ -1919,9 +2503,12 @@ static void check_huge_range(struct vm_area_struct *vma,
 {
        unsigned long addr;
        struct page *page;
 {
        unsigned long addr;
        struct page *page;

+       struct hstate *h = hstate_vma(vma);
+       unsigned long sz = huge_page_size(h);

 
 

-       for (addr = start; addr < end; addr += HPAGE_SIZE) {
-               pte_t *ptep = huge_pte_offset(vma->vm_mm, addr & HPAGE_MASK);
+       for (addr = start; addr < end; addr += sz) {
+               pte_t *ptep = huge_pte_offset(vma->vm_mm,
+                                               addr & huge_page_mask(h));

                pte_t pte;
 
                if (!ptep)
                pte_t pte;
 
                if (!ptep)


@@ -1946,6 +2533,9 @@ static inline void check_huge_range(struct vm_area_struct *vma,
 }
 #endif
 
 }
 #endif
 

+/*
+ * Display pages allocated per node and memory policy via /proc.
+ */

 int show_numa_map(struct seq_file *m, void *v)
 {
        struct proc_maps_private *priv = m->private;
 int show_numa_map(struct seq_file *m, void *v)
 {
        struct proc_maps_private *priv = m->private;


@@ -1965,12 +2555,8 @@ int show_numa_map(struct seq_file *m, void *v)
                return 0;
 
        pol = get_vma_policy(priv->task, vma, vma->vm_start);
                return 0;
 
        pol = get_vma_policy(priv->task, vma, vma->vm_start);

-       mpol_to_str(buffer, sizeof(buffer), pol);
-       /*
-        * unref shared or other task's mempolicy
-        */
-       if (pol != &default_policy && pol != current->mempolicy)
-               __mpol_free(pol);
+       mpol_to_str(buffer, sizeof(buffer), pol, 0);
+       mpol_cond_put(pol);

 
        seq_printf(m, "%08lx %s", vma->vm_start, buffer);
 
 
        seq_printf(m, "%08lx %s", vma->vm_start, buffer);