hugetlb: modular state for hugetlb page size

[safe/jmp/linux-2.6] / mm / mempolicy.c

diff --git a/mm/mempolicy.c b/mm/mempolicy.c
index 3171f88..e550bec 100644 (file)
--- a/mm/mempolicy.c
+++ b/mm/mempolicy.c
@@ -63,7 +63,6 @@
    grows down?
    make bind policy root only? It can trigger oom much faster and the
    kernel is not always grateful with that.
    grows down?
    make bind policy root only? It can trigger oom much faster and the
    kernel is not always grateful with that.

-   could replace all the switch()es with a mempolicy_ops structure.

 */
 
 #include <linux/mempolicy.h>
 */
 
 #include <linux/mempolicy.h>


@@ -72,20 +71,24 @@
 #include <linux/hugetlb.h>
 #include <linux/kernel.h>
 #include <linux/sched.h>
 #include <linux/hugetlb.h>
 #include <linux/kernel.h>
 #include <linux/sched.h>

-#include <linux/mm.h>

 #include <linux/nodemask.h>
 #include <linux/cpuset.h>
 #include <linux/gfp.h>
 #include <linux/slab.h>
 #include <linux/string.h>
 #include <linux/module.h>
 #include <linux/nodemask.h>
 #include <linux/cpuset.h>
 #include <linux/gfp.h>
 #include <linux/slab.h>
 #include <linux/string.h>
 #include <linux/module.h>

+#include <linux/nsproxy.h>

 #include <linux/interrupt.h>
 #include <linux/init.h>
 #include <linux/compat.h>
 #include <linux/interrupt.h>
 #include <linux/init.h>
 #include <linux/compat.h>

-#include <linux/mempolicy.h>

 #include <linux/swap.h>
 #include <linux/seq_file.h>
 #include <linux/proc_fs.h>
 #include <linux/swap.h>
 #include <linux/seq_file.h>
 #include <linux/proc_fs.h>

+#include <linux/migrate.h>
+#include <linux/rmap.h>
+#include <linux/security.h>
+#include <linux/syscalls.h>
+#include <linux/ctype.h>

 
 #include <asm/tlbflush.h>
 #include <asm/uaccess.h>
 
 #include <asm/tlbflush.h>
 #include <asm/uaccess.h>


@@ -95,98 +98,274 @@
 #define MPOL_MF_INVERT (MPOL_MF_INTERNAL << 1)         /* Invert check for nodemask */
 #define MPOL_MF_STATS (MPOL_MF_INTERNAL << 2)          /* Gather statistics */
 
 #define MPOL_MF_INVERT (MPOL_MF_INTERNAL << 1)         /* Invert check for nodemask */
 #define MPOL_MF_STATS (MPOL_MF_INTERNAL << 2)          /* Gather statistics */
 

-static kmem_cache_t *policy_cache;
-static kmem_cache_t *sn_cache;
-
-#define PDprintk(fmt...)
+static struct kmem_cache *policy_cache;
+static struct kmem_cache *sn_cache;

 
 /* Highest zone. An specific allocation for a zone below that is not
    policied. */
 
 /* Highest zone. An specific allocation for a zone below that is not
    policied. */

-int policy_zone = ZONE_DMA;
+enum zone_type policy_zone = 0;

 
 

+/*
+ * run-time system-wide default policy => local allocation
+ */

 struct mempolicy default_policy = {
        .refcnt = ATOMIC_INIT(1), /* never free it */
 struct mempolicy default_policy = {
        .refcnt = ATOMIC_INIT(1), /* never free it */

-       .policy = MPOL_DEFAULT,
+       .mode = MPOL_PREFERRED,
+       .flags = MPOL_F_LOCAL,

 };
 
 };
 

-/* Do sanity checking on a policy */
-static int mpol_check_policy(int mode, nodemask_t *nodes)
+static const struct mempolicy_operations {
+       int (*create)(struct mempolicy *pol, const nodemask_t *nodes);
+       void (*rebind)(struct mempolicy *pol, const nodemask_t *nodes);
+} mpol_ops[MPOL_MAX];
+
+/* Check that the nodemask contains at least one populated zone */
+static int is_valid_nodemask(const nodemask_t *nodemask)

 {
 {

-       int empty = nodes_empty(*nodes);
+       int nd, k;

 
 

-       switch (mode) {
-       case MPOL_DEFAULT:
-               if (!empty)
-                       return -EINVAL;
-               break;
-       case MPOL_BIND:
-       case MPOL_INTERLEAVE:
-               /* Preferred will only use the first bit, but allow
-                  more for now. */
-               if (empty)
-                       return -EINVAL;
-               break;
+       /* Check that there is something useful in this mask */
+       k = policy_zone;
+
+       for_each_node_mask(nd, *nodemask) {
+               struct zone *z;
+
+               for (k = 0; k <= policy_zone; k++) {
+                       z = &NODE_DATA(nd)->node_zones[k];
+                       if (z->present_pages > 0)
+                               return 1;
+               }

        }
        }

-       return nodes_subset(*nodes, node_online_map) ? 0 : -EINVAL;
+
+       return 0;

 }
 }

-/* Generate a custom zonelist for the BIND policy. */
-static struct zonelist *bind_zonelist(nodemask_t *nodes)
+
+static inline int mpol_store_user_nodemask(const struct mempolicy *pol)

 {
 {

-       struct zonelist *zl;
-       int num, max, nd;
+       return pol->flags & (MPOL_F_STATIC_NODES | MPOL_F_RELATIVE_NODES);
+}

 
 

-       max = 1 + MAX_NR_ZONES * nodes_weight(*nodes);
-       zl = kmalloc(sizeof(void *) * max, GFP_KERNEL);
-       if (!zl)
-               return NULL;
-       num = 0;
-       for_each_node_mask(nd, *nodes)
-               zl->zones[num++] = &NODE_DATA(nd)->node_zones[policy_zone];
-       zl->zones[num] = NULL;
-       return zl;
+static void mpol_relative_nodemask(nodemask_t *ret, const nodemask_t *orig,
+                                  const nodemask_t *rel)
+{
+       nodemask_t tmp;
+       nodes_fold(tmp, *orig, nodes_weight(*rel));
+       nodes_onto(*ret, tmp, *rel);
+}
+
+static int mpol_new_interleave(struct mempolicy *pol, const nodemask_t *nodes)
+{
+       if (nodes_empty(*nodes))
+               return -EINVAL;
+       pol->v.nodes = *nodes;
+       return 0;
+}
+
+static int mpol_new_preferred(struct mempolicy *pol, const nodemask_t *nodes)
+{
+       if (!nodes)
+               pol->flags |= MPOL_F_LOCAL;     /* local allocation */
+       else if (nodes_empty(*nodes))
+               return -EINVAL;                 /*  no allowed nodes */
+       else
+               pol->v.preferred_node = first_node(*nodes);
+       return 0;
+}
+
+static int mpol_new_bind(struct mempolicy *pol, const nodemask_t *nodes)
+{
+       if (!is_valid_nodemask(nodes))
+               return -EINVAL;
+       pol->v.nodes = *nodes;
+       return 0;

 }
 
 /* Create a new policy */
 }
 
 /* Create a new policy */

-static struct mempolicy *mpol_new(int mode, nodemask_t *nodes)
+static struct mempolicy *mpol_new(unsigned short mode, unsigned short flags,
+                                 nodemask_t *nodes)

 {
        struct mempolicy *policy;
 {
        struct mempolicy *policy;

+       nodemask_t cpuset_context_nmask;
+       int ret;

 
 

-       PDprintk("setting mode %d nodes[0] %lx\n", mode, nodes_addr(*nodes)[0]);
-       if (mode == MPOL_DEFAULT)
-               return NULL;
+       pr_debug("setting mode %d flags %d nodes[0] %lx\n",
+                mode, flags, nodes ? nodes_addr(*nodes)[0] : -1);
+
+       if (mode == MPOL_DEFAULT) {
+               if (nodes && !nodes_empty(*nodes))
+                       return ERR_PTR(-EINVAL);
+               return NULL;    /* simply delete any existing policy */
+       }
+       VM_BUG_ON(!nodes);
+
+       /*
+        * MPOL_PREFERRED cannot be used with MPOL_F_STATIC_NODES or
+        * MPOL_F_RELATIVE_NODES if the nodemask is empty (local allocation).
+        * All other modes require a valid pointer to a non-empty nodemask.
+        */
+       if (mode == MPOL_PREFERRED) {
+               if (nodes_empty(*nodes)) {
+                       if (((flags & MPOL_F_STATIC_NODES) ||
+                            (flags & MPOL_F_RELATIVE_NODES)))
+                               return ERR_PTR(-EINVAL);
+                       nodes = NULL;   /* flag local alloc */
+               }
+       } else if (nodes_empty(*nodes))
+               return ERR_PTR(-EINVAL);

        policy = kmem_cache_alloc(policy_cache, GFP_KERNEL);
        if (!policy)
                return ERR_PTR(-ENOMEM);
        atomic_set(&policy->refcnt, 1);
        policy = kmem_cache_alloc(policy_cache, GFP_KERNEL);
        if (!policy)
                return ERR_PTR(-ENOMEM);
        atomic_set(&policy->refcnt, 1);

-       switch (mode) {
-       case MPOL_INTERLEAVE:
-               policy->v.nodes = *nodes;
-               if (nodes_weight(*nodes) == 0) {
-                       kmem_cache_free(policy_cache, policy);
-                       return ERR_PTR(-EINVAL);
-               }
-               break;
-       case MPOL_PREFERRED:
-               policy->v.preferred_node = first_node(*nodes);
-               if (policy->v.preferred_node >= MAX_NUMNODES)
-                       policy->v.preferred_node = -1;
-               break;
-       case MPOL_BIND:
-               policy->v.zonelist = bind_zonelist(nodes);
-               if (policy->v.zonelist == NULL) {
-                       kmem_cache_free(policy_cache, policy);
-                       return ERR_PTR(-ENOMEM);
-               }
-               break;
+       policy->mode = mode;
+       policy->flags = flags;
+
+       if (nodes) {
+               /*
+                * cpuset related setup doesn't apply to local allocation
+                */
+               cpuset_update_task_memory_state();
+               if (flags & MPOL_F_RELATIVE_NODES)
+                       mpol_relative_nodemask(&cpuset_context_nmask, nodes,
+                                              &cpuset_current_mems_allowed);
+               else
+                       nodes_and(cpuset_context_nmask, *nodes,
+                                 cpuset_current_mems_allowed);
+               if (mpol_store_user_nodemask(policy))
+                       policy->w.user_nodemask = *nodes;
+               else
+                       policy->w.cpuset_mems_allowed =
+                                               cpuset_mems_allowed(current);
+       }
+
+       ret = mpol_ops[mode].create(policy,
+                               nodes ? &cpuset_context_nmask : NULL);
+       if (ret < 0) {
+               kmem_cache_free(policy_cache, policy);
+               return ERR_PTR(ret);

        }
        }

-       policy->policy = mode;
-       policy->cpuset_mems_allowed = cpuset_mems_allowed(current);

        return policy;
 }
 
        return policy;
 }
 

-static void gather_stats(struct page *, void *);
-static void migrate_page_add(struct vm_area_struct *vma,
-       struct page *page, struct list_head *pagelist, unsigned long flags);
+/* Slow path of a mpol destructor. */
+void __mpol_put(struct mempolicy *p)
+{
+       if (!atomic_dec_and_test(&p->refcnt))
+               return;
+       kmem_cache_free(policy_cache, p);
+}
+
+static void mpol_rebind_default(struct mempolicy *pol, const nodemask_t *nodes)
+{
+}
+
+static void mpol_rebind_nodemask(struct mempolicy *pol,
+                                const nodemask_t *nodes)
+{
+       nodemask_t tmp;
+
+       if (pol->flags & MPOL_F_STATIC_NODES)
+               nodes_and(tmp, pol->w.user_nodemask, *nodes);
+       else if (pol->flags & MPOL_F_RELATIVE_NODES)
+               mpol_relative_nodemask(&tmp, &pol->w.user_nodemask, nodes);
+       else {
+               nodes_remap(tmp, pol->v.nodes, pol->w.cpuset_mems_allowed,
+                           *nodes);
+               pol->w.cpuset_mems_allowed = *nodes;
+       }
+
+       pol->v.nodes = tmp;
+       if (!node_isset(current->il_next, tmp)) {
+               current->il_next = next_node(current->il_next, tmp);
+               if (current->il_next >= MAX_NUMNODES)
+                       current->il_next = first_node(tmp);
+               if (current->il_next >= MAX_NUMNODES)
+                       current->il_next = numa_node_id();
+       }
+}
+
+static void mpol_rebind_preferred(struct mempolicy *pol,
+                                 const nodemask_t *nodes)
+{
+       nodemask_t tmp;
+
+       if (pol->flags & MPOL_F_STATIC_NODES) {
+               int node = first_node(pol->w.user_nodemask);
+
+               if (node_isset(node, *nodes)) {
+                       pol->v.preferred_node = node;
+                       pol->flags &= ~MPOL_F_LOCAL;
+               } else
+                       pol->flags |= MPOL_F_LOCAL;
+       } else if (pol->flags & MPOL_F_RELATIVE_NODES) {
+               mpol_relative_nodemask(&tmp, &pol->w.user_nodemask, nodes);
+               pol->v.preferred_node = first_node(tmp);
+       } else if (!(pol->flags & MPOL_F_LOCAL)) {
+               pol->v.preferred_node = node_remap(pol->v.preferred_node,
+                                                  pol->w.cpuset_mems_allowed,
+                                                  *nodes);
+               pol->w.cpuset_mems_allowed = *nodes;
+       }
+}
+
+/* Migrate a policy to a different set of nodes */
+static void mpol_rebind_policy(struct mempolicy *pol,
+                              const nodemask_t *newmask)
+{
+       if (!pol)
+               return;
+       if (!mpol_store_user_nodemask(pol) &&
+           nodes_equal(pol->w.cpuset_mems_allowed, *newmask))
+               return;
+       mpol_ops[pol->mode].rebind(pol, newmask);
+}
+
+/*
+ * Wrapper for mpol_rebind_policy() that just requires task
+ * pointer, and updates task mempolicy.
+ */
+
+void mpol_rebind_task(struct task_struct *tsk, const nodemask_t *new)
+{
+       mpol_rebind_policy(tsk->mempolicy, new);
+}
+
+/*
+ * Rebind each vma in mm to new nodemask.
+ *
+ * Call holding a reference to mm.  Takes mm->mmap_sem during call.
+ */
+
+void mpol_rebind_mm(struct mm_struct *mm, nodemask_t *new)
+{
+       struct vm_area_struct *vma;
+
+       down_write(&mm->mmap_sem);
+       for (vma = mm->mmap; vma; vma = vma->vm_next)
+               mpol_rebind_policy(vma->vm_policy, new);
+       up_write(&mm->mmap_sem);
+}
+
+static const struct mempolicy_operations mpol_ops[MPOL_MAX] = {
+       [MPOL_DEFAULT] = {
+               .rebind = mpol_rebind_default,
+       },
+       [MPOL_INTERLEAVE] = {
+               .create = mpol_new_interleave,
+               .rebind = mpol_rebind_nodemask,
+       },
+       [MPOL_PREFERRED] = {
+               .create = mpol_new_preferred,
+               .rebind = mpol_rebind_preferred,
+       },
+       [MPOL_BIND] = {
+               .create = mpol_new_bind,
+               .rebind = mpol_rebind_nodemask,
+       },
+};
+
+static void gather_stats(struct page *, void *, int pte_dirty);
+static void migrate_page_add(struct page *page, struct list_head *pagelist,
+                               unsigned long flags);

 
 /* Scan through pages checking if pages follow certain conditions. */
 static int check_pte_range(struct vm_area_struct *vma, pmd_t *pmd,
 
 /* Scan through pages checking if pages follow certain conditions. */
 static int check_pte_range(struct vm_area_struct *vma, pmd_t *pmd,


@@ -201,13 +380,24 @@ static int check_pte_range(struct vm_area_struct *vma, pmd_t *pmd,
        orig_pte = pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
        do {
                struct page *page;
        orig_pte = pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
        do {
                struct page *page;

-               unsigned int nid;
+               int nid;

 
                if (!pte_present(*pte))
                        continue;
                page = vm_normal_page(vma, addr, *pte);
                if (!page)
                        continue;
 
                if (!pte_present(*pte))
                        continue;
                page = vm_normal_page(vma, addr, *pte);
                if (!page)
                        continue;

+               /*
+                * The check for PageReserved here is important to avoid
+                * handling zero pages and other pages that may have been
+                * marked special by the system.
+                *
+                * If the PageReserved would not be checked here then f.e.
+                * the location of the zero page could have an influence
+                * on MPOL_MF_STRICT, zero pages would be counted for
+                * the per node stats, and there would be useless attempts
+                * to put zero pages on the migration list.
+                */

                if (PageReserved(page))
                        continue;
                nid = page_to_nid(page);
                if (PageReserved(page))
                        continue;
                nid = page_to_nid(page);


@@ -215,12 +405,9 @@ static int check_pte_range(struct vm_area_struct *vma, pmd_t *pmd,
                        continue;
 
                if (flags & MPOL_MF_STATS)
                        continue;
 
                if (flags & MPOL_MF_STATS)

-                       gather_stats(page, private);
-               else if (flags & (MPOL_MF_MOVE | MPOL_MF_MOVE_ALL)) {
-                       spin_unlock(ptl);
-                       migrate_page_add(vma, page, private, flags);
-                       spin_lock(ptl);
-               }
+                       gather_stats(page, private, pte_dirty(*pte));
+               else if (flags & (MPOL_MF_MOVE | MPOL_MF_MOVE_ALL))
+                       migrate_page_add(page, private, flags);

                else
                        break;
        } while (pte++, addr += PAGE_SIZE, addr != end);
                else
                        break;
        } while (pte++, addr += PAGE_SIZE, addr != end);


@@ -288,15 +475,6 @@ static inline int check_pgd_range(struct vm_area_struct *vma,
        return 0;
 }
 
        return 0;
 }
 

-/* Check if a vma is migratable */
-static inline int vma_migratable(struct vm_area_struct *vma)
-{
-       if (vma->vm_flags & (
-               VM_LOCKED|VM_IO|VM_HUGETLB|VM_PFNMAP|VM_RESERVED))
-               return 0;
-       return 1;
-}
-

 /*
  * Check if all pages in a range are on a set of nodes.
  * If pagelist != NULL then isolate pages from the LRU and
 /*
  * Check if all pages in a range are on a set of nodes.
  * If pagelist != NULL then isolate pages from the LRU and


@@ -309,6 +487,13 @@ check_range(struct mm_struct *mm, unsigned long start, unsigned long end,
        int err;
        struct vm_area_struct *first, *vma, *prev;
 
        int err;
        struct vm_area_struct *first, *vma, *prev;
 

+       if (flags & (MPOL_MF_MOVE | MPOL_MF_MOVE_ALL)) {
+
+               err = migrate_prep();
+               if (err)
+                       return ERR_PTR(err);
+       }
+

        first = find_vma(mm, start);
        if (!first)
                return ERR_PTR(-EFAULT);
        first = find_vma(mm, start);
        if (!first)
                return ERR_PTR(-EFAULT);


@@ -348,7 +533,7 @@ static int policy_vma(struct vm_area_struct *vma, struct mempolicy *new)
        int err = 0;
        struct mempolicy *old = vma->vm_policy;
 
        int err = 0;
        struct mempolicy *old = vma->vm_policy;
 

-       PDprintk("vma %lx-%lx/%lx vm_ops %p vm_file %p set_policy %p\n",
+       pr_debug("vma %lx-%lx/%lx vm_ops %p vm_file %p set_policy %p\n",

                 vma->vm_start, vma->vm_end, vma->vm_pgoff,
                 vma->vm_ops, vma->vm_file,
                 vma->vm_ops ? vma->vm_ops->set_policy : NULL);
                 vma->vm_start, vma->vm_end, vma->vm_pgoff,
                 vma->vm_ops, vma->vm_file,
                 vma->vm_ops ? vma->vm_ops->set_policy : NULL);


@@ -358,7 +543,7 @@ static int policy_vma(struct vm_area_struct *vma, struct mempolicy *new)
        if (!err) {
                mpol_get(new);
                vma->vm_policy = new;
        if (!err) {
                mpol_get(new);
                vma->vm_policy = new;

-               mpol_free(old);
+               mpol_put(old);

        }
        return err;
 }
        }
        return err;
 }


@@ -385,57 +570,86 @@ static int mbind_range(struct vm_area_struct *vma, unsigned long start,
        return err;
 }
 
        return err;
 }
 

-static int contextualize_policy(int mode, nodemask_t *nodes)
+/*
+ * Update task->flags PF_MEMPOLICY bit: set iff non-default
+ * mempolicy.  Allows more rapid checking of this (combined perhaps
+ * with other PF_* flag bits) on memory allocation hot code paths.
+ *
+ * If called from outside this file, the task 'p' should -only- be
+ * a newly forked child not yet visible on the task list, because
+ * manipulating the task flags of a visible task is not safe.
+ *
+ * The above limitation is why this routine has the funny name
+ * mpol_fix_fork_child_flag().
+ *
+ * It is also safe to call this with a task pointer of current,
+ * which the static wrapper mpol_set_task_struct_flag() does,
+ * for use within this file.
+ */
+
+void mpol_fix_fork_child_flag(struct task_struct *p)

 {
 {

-       if (!nodes)
-               return 0;
+       if (p->mempolicy)
+               p->flags |= PF_MEMPOLICY;
+       else
+               p->flags &= ~PF_MEMPOLICY;
+}

 
 

-       cpuset_update_task_memory_state();
-       if (!cpuset_nodes_subset_current_mems_allowed(*nodes))
-               return -EINVAL;
-       return mpol_check_policy(mode, nodes);
+static void mpol_set_task_struct_flag(void)
+{
+       mpol_fix_fork_child_flag(current);

 }
 
 /* Set the process memory policy */
 }
 
 /* Set the process memory policy */

-long do_set_mempolicy(int mode, nodemask_t *nodes)
+static long do_set_mempolicy(unsigned short mode, unsigned short flags,
+                            nodemask_t *nodes)

 {
        struct mempolicy *new;
 {
        struct mempolicy *new;

+       struct mm_struct *mm = current->mm;

 
 

-       if (contextualize_policy(mode, nodes))
-               return -EINVAL;
-       new = mpol_new(mode, nodes);
+       new = mpol_new(mode, flags, nodes);

        if (IS_ERR(new))
                return PTR_ERR(new);
        if (IS_ERR(new))
                return PTR_ERR(new);

-       mpol_free(current->mempolicy);
+
+       /*
+        * prevent changing our mempolicy while show_numa_maps()
+        * is using it.
+        * Note:  do_set_mempolicy() can be called at init time
+        * with no 'mm'.
+        */
+       if (mm)
+               down_write(&mm->mmap_sem);
+       mpol_put(current->mempolicy);

        current->mempolicy = new;
        current->mempolicy = new;

-       if (new && new->policy == MPOL_INTERLEAVE)
+       mpol_set_task_struct_flag();
+       if (new && new->mode == MPOL_INTERLEAVE &&
+           nodes_weight(new->v.nodes))

                current->il_next = first_node(new->v.nodes);
                current->il_next = first_node(new->v.nodes);

+       if (mm)
+               up_write(&mm->mmap_sem);
+

        return 0;
 }
 
        return 0;
 }
 

-/* Fill a zone bitmap for a policy */
-static void get_zonemask(struct mempolicy *p, nodemask_t *nodes)
+/*
+ * Return nodemask for policy for get_mempolicy() query
+ */
+static void get_policy_nodemask(struct mempolicy *p, nodemask_t *nodes)

 {
 {

-       int i;
-

        nodes_clear(*nodes);
        nodes_clear(*nodes);

-       switch (p->policy) {
+       if (p == &default_policy)
+               return;
+
+       switch (p->mode) {

        case MPOL_BIND:
        case MPOL_BIND:

-               for (i = 0; p->v.zonelist->zones[i]; i++)
-                       node_set(p->v.zonelist->zones[i]->zone_pgdat->node_id,
-                               *nodes);
-               break;
-       case MPOL_DEFAULT:
-               break;
+               /* Fall through */

        case MPOL_INTERLEAVE:
                *nodes = p->v.nodes;
                break;
        case MPOL_PREFERRED:
        case MPOL_INTERLEAVE:
                *nodes = p->v.nodes;
                break;
        case MPOL_PREFERRED:

-               /* or use current node instead of online map? */
-               if (p->v.preferred_node < 0)
-                       *nodes = node_online_map;
-               else
+               if (!(p->flags & MPOL_F_LOCAL))

                        node_set(p->v.preferred_node, *nodes);
                        node_set(p->v.preferred_node, *nodes);

+               /* else return empty node mask for local allocation */

                break;
        default:
                BUG();
                break;
        default:
                BUG();


@@ -456,8 +670,8 @@ static int lookup_node(struct mm_struct *mm, unsigned long addr)
 }
 
 /* Retrieve NUMA policy */
 }
 
 /* Retrieve NUMA policy */

-long do_get_mempolicy(int *policy, nodemask_t *nmask,
-                       unsigned long addr, unsigned long flags)
+static long do_get_mempolicy(int *policy, nodemask_t *nmask,
+                            unsigned long addr, unsigned long flags)

 {
        int err;
        struct mm_struct *mm = current->mm;
 {
        int err;
        struct mm_struct *mm = current->mm;


@@ -465,9 +679,24 @@ long do_get_mempolicy(int *policy, nodemask_t *nmask,
        struct mempolicy *pol = current->mempolicy;
 
        cpuset_update_task_memory_state();
        struct mempolicy *pol = current->mempolicy;
 
        cpuset_update_task_memory_state();

-       if (flags & ~(unsigned long)(MPOL_F_NODE|MPOL_F_ADDR))
+       if (flags &
+               ~(unsigned long)(MPOL_F_NODE|MPOL_F_ADDR|MPOL_F_MEMS_ALLOWED))

                return -EINVAL;
                return -EINVAL;

+
+       if (flags & MPOL_F_MEMS_ALLOWED) {
+               if (flags & (MPOL_F_NODE|MPOL_F_ADDR))
+                       return -EINVAL;
+               *policy = 0;    /* just so it's initialized */
+               *nmask  = cpuset_current_mems_allowed;
+               return 0;
+       }
+

        if (flags & MPOL_F_ADDR) {
        if (flags & MPOL_F_ADDR) {

+               /*
+                * Do NOT fall back to task policy if the
+                * vma/shared policy at addr is NULL.  We
+                * want to return MPOL_DEFAULT in this case.
+                */

                down_read(&mm->mmap_sem);
                vma = find_vma_intersection(mm, addr, addr+1);
                if (!vma) {
                down_read(&mm->mmap_sem);
                vma = find_vma_intersection(mm, addr, addr+1);
                if (!vma) {


@@ -482,7 +711,7 @@ long do_get_mempolicy(int *policy, nodemask_t *nmask,
                return -EINVAL;
 
        if (!pol)
                return -EINVAL;
 
        if (!pol)

-               pol = &default_policy;
+               pol = &default_policy;  /* indicates default behavior */

 
        if (flags & MPOL_F_NODE) {
                if (flags & MPOL_F_ADDR) {
 
        if (flags & MPOL_F_NODE) {
                if (flags & MPOL_F_ADDR) {


@@ -491,14 +720,21 @@ long do_get_mempolicy(int *policy, nodemask_t *nmask,
                                goto out;
                        *policy = err;
                } else if (pol == current->mempolicy &&
                                goto out;
                        *policy = err;
                } else if (pol == current->mempolicy &&

-                               pol->policy == MPOL_INTERLEAVE) {
+                               pol->mode == MPOL_INTERLEAVE) {

                        *policy = current->il_next;
                } else {
                        err = -EINVAL;
                        goto out;
                }
                        *policy = current->il_next;
                } else {
                        err = -EINVAL;
                        goto out;
                }

-       } else
-               *policy = pol->policy;
+       } else {
+               *policy = pol == &default_policy ? MPOL_DEFAULT :
+                                               pol->mode;
+               /*
+                * Internal mempolicy flags must be masked off before exposing
+                * the policy to userspace.
+                */
+               *policy |= (pol->flags & MPOL_MODE_FLAGS);
+       }

 
        if (vma) {
                up_read(&current->mm->mmap_sem);
 
        if (vma) {
                up_read(&current->mm->mmap_sem);


@@ -507,84 +743,60 @@ long do_get_mempolicy(int *policy, nodemask_t *nmask,
 
        err = 0;
        if (nmask)
 
        err = 0;
        if (nmask)

-               get_zonemask(pol, nmask);
+               get_policy_nodemask(pol, nmask);

 
  out:
 
  out:

+       mpol_cond_put(pol);

        if (vma)
                up_read(&current->mm->mmap_sem);
        return err;
 }
 
        if (vma)
                up_read(&current->mm->mmap_sem);
        return err;
 }
 

+#ifdef CONFIG_MIGRATION

 /*
  * page migration
  */
 /*
  * page migration
  */

-
-/* Check if we are the only process mapping the page in question */
-static inline int single_mm_mapping(struct mm_struct *mm,
-                       struct address_space *mapping)
+static void migrate_page_add(struct page *page, struct list_head *pagelist,
+                               unsigned long flags)

 {
 {

-       struct vm_area_struct *vma;
-       struct prio_tree_iter iter;
-       int rc = 1;
+       /*
+        * Avoid migrating a page that is shared with others.
+        */
+       if ((flags & MPOL_MF_MOVE_ALL) || page_mapcount(page) == 1)
+               isolate_lru_page(page, pagelist);
+}

 
 

-       spin_lock(&mapping->i_mmap_lock);
-       vma_prio_tree_foreach(vma, &iter, &mapping->i_mmap, 0, ULONG_MAX)
-               if (mm != vma->vm_mm) {
-                       rc = 0;
-                       goto out;
-               }
-       list_for_each_entry(vma, &mapping->i_mmap_nonlinear, shared.vm_set.list)
-               if (mm != vma->vm_mm) {
-                       rc = 0;
-                       goto out;
-               }
-out:
-       spin_unlock(&mapping->i_mmap_lock);
-       return rc;
+static struct page *new_node_page(struct page *page, unsigned long node, int **x)
+{
+       return alloc_pages_node(node, GFP_HIGHUSER_MOVABLE, 0);

 }
 
 /*
 }
 
 /*

- * Add a page to be migrated to the pagelist
+ * Migrate pages from one node to a target node.
+ * Returns error or the number of pages not migrated.

  */
  */

-static void migrate_page_add(struct vm_area_struct *vma,
-       struct page *page, struct list_head *pagelist, unsigned long flags)
+static int migrate_to_node(struct mm_struct *mm, int source, int dest,
+                          int flags)

 {
 {

-       /*
-        * Avoid migrating a page that is shared by others and not writable.
-        */
-       if ((flags & MPOL_MF_MOVE_ALL) || !page->mapping || PageAnon(page) ||
-           mapping_writably_mapped(page->mapping) ||
-           single_mm_mapping(vma->vm_mm, page->mapping)) {
-               int rc = isolate_lru_page(page);
+       nodemask_t nmask;
+       LIST_HEAD(pagelist);
+       int err = 0;

 
 

-               if (rc == 1)
-                       list_add(&page->lru, pagelist);
-               /*
-                * If the isolate attempt was not successful then we just
-                * encountered an unswappable page. Something must be wrong.
-                */
-               WARN_ON(rc == 0);
-       }
-}
+       nodes_clear(nmask);
+       node_set(source, nmask);

 
 

-static int swap_pages(struct list_head *pagelist)
-{
-       LIST_HEAD(moved);
-       LIST_HEAD(failed);
-       int n;
+       check_range(mm, mm->mmap->vm_start, TASK_SIZE, &nmask,
+                       flags | MPOL_MF_DISCONTIG_OK, &pagelist);

 
 

-       n = migrate_pages(pagelist, NULL, &moved, &failed);
-       putback_lru_pages(&failed);
-       putback_lru_pages(&moved);
+       if (!list_empty(&pagelist))
+               err = migrate_pages(&pagelist, new_node_page, dest);

 
 

-       return n;
+       return err;

 }
 
 /*
 }
 
 /*

- * For now migrate_pages simply swaps out the pages from nodes that are in
- * the source set but not in the target set. In the future, we would
- * want a function that moves pages between the two nodesets in such
- * a way as to preserve the physical layout as much as possible.
+ * Move pages between the two nodesets so as to preserve the physical
+ * layout as much as possible.

  *
  * Returns the number of page that could not be moved.
  */
  *
  * Returns the number of page that could not be moved.
  */


@@ -592,26 +804,129 @@ int do_migrate_pages(struct mm_struct *mm,
        const nodemask_t *from_nodes, const nodemask_t *to_nodes, int flags)
 {
        LIST_HEAD(pagelist);
        const nodemask_t *from_nodes, const nodemask_t *to_nodes, int flags)
 {
        LIST_HEAD(pagelist);

-       int count = 0;
-       nodemask_t nodes;
-
-       nodes_andnot(nodes, *from_nodes, *to_nodes);
+       int busy = 0;
+       int err = 0;
+       nodemask_t tmp;

 
        down_read(&mm->mmap_sem);
 
        down_read(&mm->mmap_sem);

-       check_range(mm, mm->mmap->vm_start, TASK_SIZE, &nodes,
-                       flags | MPOL_MF_DISCONTIG_OK, &pagelist);

 
 

-       if (!list_empty(&pagelist)) {
-               count = swap_pages(&pagelist);
-               putback_lru_pages(&pagelist);
-       }
+       err = migrate_vmas(mm, from_nodes, to_nodes, flags);
+       if (err)
+               goto out;

 
 

+/*
+ * Find a 'source' bit set in 'tmp' whose corresponding 'dest'
+ * bit in 'to' is not also set in 'tmp'.  Clear the found 'source'
+ * bit in 'tmp', and return that <source, dest> pair for migration.
+ * The pair of nodemasks 'to' and 'from' define the map.
+ *
+ * If no pair of bits is found that way, fallback to picking some
+ * pair of 'source' and 'dest' bits that are not the same.  If the
+ * 'source' and 'dest' bits are the same, this represents a node
+ * that will be migrating to itself, so no pages need move.
+ *
+ * If no bits are left in 'tmp', or if all remaining bits left
+ * in 'tmp' correspond to the same bit in 'to', return false
+ * (nothing left to migrate).
+ *
+ * This lets us pick a pair of nodes to migrate between, such that
+ * if possible the dest node is not already occupied by some other
+ * source node, minimizing the risk of overloading the memory on a
+ * node that would happen if we migrated incoming memory to a node
+ * before migrating outgoing memory source that same node.
+ *
+ * A single scan of tmp is sufficient.  As we go, we remember the
+ * most recent <s, d> pair that moved (s != d).  If we find a pair
+ * that not only moved, but what's better, moved to an empty slot
+ * (d is not set in tmp), then we break out then, with that pair.
+ * Otherwise when we finish scannng from_tmp, we at least have the
+ * most recent <s, d> pair that moved.  If we get all the way through
+ * the scan of tmp without finding any node that moved, much less
+ * moved to an empty node, then there is nothing left worth migrating.
+ */
+
+       tmp = *from_nodes;
+       while (!nodes_empty(tmp)) {
+               int s,d;
+               int source = -1;
+               int dest = 0;
+
+               for_each_node_mask(s, tmp) {
+                       d = node_remap(s, *from_nodes, *to_nodes);
+                       if (s == d)
+                               continue;
+
+                       source = s;     /* Node moved. Memorize */
+                       dest = d;
+
+                       /* dest not in remaining from nodes? */
+                       if (!node_isset(dest, tmp))
+                               break;
+               }
+               if (source == -1)
+                       break;
+
+               node_clear(source, tmp);
+               err = migrate_to_node(mm, source, dest, flags);
+               if (err > 0)
+                       busy += err;
+               if (err < 0)
+                       break;
+       }
+out:

        up_read(&mm->mmap_sem);
        up_read(&mm->mmap_sem);

-       return count;
+       if (err < 0)
+               return err;
+       return busy;
+
+}
+
+/*
+ * Allocate a new page for page migration based on vma policy.
+ * Start assuming that page is mapped by vma pointed to by @private.
+ * Search forward from there, if not.  N.B., this assumes that the
+ * list of pages handed to migrate_pages()--which is how we get here--
+ * is in virtual address order.
+ */
+static struct page *new_vma_page(struct page *page, unsigned long private, int **x)
+{
+       struct vm_area_struct *vma = (struct vm_area_struct *)private;
+       unsigned long uninitialized_var(address);
+
+       while (vma) {
+               address = page_address_in_vma(page, vma);
+               if (address != -EFAULT)
+                       break;
+               vma = vma->vm_next;
+       }
+
+       /*
+        * if !vma, alloc_page_vma() will use task or system default policy
+        */
+       return alloc_page_vma(GFP_HIGHUSER_MOVABLE, vma, address);
+}
+#else
+
+static void migrate_page_add(struct page *page, struct list_head *pagelist,
+                               unsigned long flags)
+{
+}
+
+int do_migrate_pages(struct mm_struct *mm,
+       const nodemask_t *from_nodes, const nodemask_t *to_nodes, int flags)
+{
+       return -ENOSYS;

 }
 
 }
 

-long do_mbind(unsigned long start, unsigned long len,
-               unsigned long mode, nodemask_t *nmask, unsigned long flags)
+static struct page *new_vma_page(struct page *page, unsigned long private, int **x)
+{
+       return NULL;
+}
+#endif
+
+static long do_mbind(unsigned long start, unsigned long len,
+                    unsigned short mode, unsigned short mode_flags,
+                    nodemask_t *nmask, unsigned long flags)

 {
        struct vm_area_struct *vma;
        struct mm_struct *mm = current->mm;
 {
        struct vm_area_struct *vma;
        struct mm_struct *mm = current->mm;


@@ -620,11 +935,10 @@ long do_mbind(unsigned long start, unsigned long len,
        int err;
        LIST_HEAD(pagelist);
 
        int err;
        LIST_HEAD(pagelist);
 

-       if ((flags & ~(unsigned long)(MPOL_MF_STRICT |
-                                     MPOL_MF_MOVE | MPOL_MF_MOVE_ALL))
-           || mode > MPOL_MAX)
+       if (flags & ~(unsigned long)(MPOL_MF_STRICT |
+                                    MPOL_MF_MOVE | MPOL_MF_MOVE_ALL))

                return -EINVAL;
                return -EINVAL;

-       if ((flags & MPOL_MF_MOVE_ALL) && !capable(CAP_SYS_RESOURCE))
+       if ((flags & MPOL_MF_MOVE_ALL) && !capable(CAP_SYS_NICE))

                return -EPERM;
 
        if (start & ~PAGE_MASK)
                return -EPERM;
 
        if (start & ~PAGE_MASK)


@@ -641,10 +955,7 @@ long do_mbind(unsigned long start, unsigned long len,
        if (end == start)
                return 0;
 
        if (end == start)
                return 0;
 

-       if (mpol_check_policy(mode, nmask))
-               return -EINVAL;
-
-       new = mpol_new(mode, nmask);
+       new = mpol_new(mode, mode_flags, nmask);

        if (IS_ERR(new))
                return PTR_ERR(new);
 
        if (IS_ERR(new))
                return PTR_ERR(new);
 


@@ -655,8 +966,9 @@ long do_mbind(unsigned long start, unsigned long len,
        if (!new)
                flags |= MPOL_MF_DISCONTIG_OK;
 
        if (!new)
                flags |= MPOL_MF_DISCONTIG_OK;
 

-       PDprintk("mbind %lx-%lx mode:%ld nodes:%lx\n",start,start+len,
-                       mode,nodes_addr(nodes)[0]);
+       pr_debug("mbind %lx-%lx mode:%d flags:%d nodes:%lx\n",
+                start, start + len, mode, mode_flags,
+                nmask ? nodes_addr(*nmask)[0] : -1);

 
        down_write(&mm->mmap_sem);
        vma = check_range(mm, start, end, nmask,
 
        down_write(&mm->mmap_sem);
        vma = check_range(mm, start, end, nmask,


@@ -667,17 +979,17 @@ long do_mbind(unsigned long start, unsigned long len,
                int nr_failed = 0;
 
                err = mbind_range(vma, start, end, new);
                int nr_failed = 0;
 
                err = mbind_range(vma, start, end, new);

+

                if (!list_empty(&pagelist))
                if (!list_empty(&pagelist))

-                       nr_failed = swap_pages(&pagelist);
+                       nr_failed = migrate_pages(&pagelist, new_vma_page,
+                                               (unsigned long)vma);

 
                if (!err && nr_failed && (flags & MPOL_MF_STRICT))
                        err = -EIO;
        }
 
                if (!err && nr_failed && (flags & MPOL_MF_STRICT))
                        err = -EIO;
        }

-       if (!list_empty(&pagelist))
-               putback_lru_pages(&pagelist);

 
        up_write(&mm->mmap_sem);
 
        up_write(&mm->mmap_sem);

-       mpol_free(new);
+       mpol_put(new);

        return err;
 }
 
        return err;
 }
 


@@ -697,6 +1009,8 @@ static int get_nodes(nodemask_t *nodes, const unsigned long __user *nmask,
        nodes_clear(*nodes);
        if (maxnode == 0 || !nmask)
                return 0;
        nodes_clear(*nodes);
        if (maxnode == 0 || !nmask)
                return 0;

+       if (maxnode > PAGE_SIZE*BITS_PER_BYTE)
+               return -EINVAL;

 
        nlongs = BITS_TO_LONGS(maxnode);
        if ((maxnode % BITS_PER_LONG) == 0)
 
        nlongs = BITS_TO_LONGS(maxnode);
        if ((maxnode % BITS_PER_LONG) == 0)


@@ -753,11 +1067,19 @@ asmlinkage long sys_mbind(unsigned long start, unsigned long len,
 {
        nodemask_t nodes;
        int err;
 {
        nodemask_t nodes;
        int err;

+       unsigned short mode_flags;

 
 

+       mode_flags = mode & MPOL_MODE_FLAGS;
+       mode &= ~MPOL_MODE_FLAGS;
+       if (mode >= MPOL_MAX)
+               return -EINVAL;
+       if ((mode_flags & MPOL_F_STATIC_NODES) &&
+           (mode_flags & MPOL_F_RELATIVE_NODES))
+               return -EINVAL;

        err = get_nodes(&nodes, nmask, maxnode);
        if (err)
                return err;
        err = get_nodes(&nodes, nmask, maxnode);
        if (err)
                return err;

-       return do_mbind(start, len, mode, &nodes, flags);
+       return do_mbind(start, len, mode, mode_flags, &nodes, flags);

 }
 
 /* Set the process memory policy */
 }
 
 /* Set the process memory policy */


@@ -766,13 +1088,18 @@ asmlinkage long sys_set_mempolicy(int mode, unsigned long __user *nmask,
 {
        int err;
        nodemask_t nodes;
 {
        int err;
        nodemask_t nodes;

+       unsigned short flags;

 
 

-       if (mode < 0 || mode > MPOL_MAX)
+       flags = mode & MPOL_MODE_FLAGS;
+       mode &= ~MPOL_MODE_FLAGS;
+       if ((unsigned int)mode >= MPOL_MAX)
+               return -EINVAL;
+       if ((flags & MPOL_F_STATIC_NODES) && (flags & MPOL_F_RELATIVE_NODES))

                return -EINVAL;
        err = get_nodes(&nodes, nmask, maxnode);
        if (err)
                return err;
                return -EINVAL;
        err = get_nodes(&nodes, nmask, maxnode);
        if (err)
                return err;

-       return do_set_mempolicy(mode, &nodes);
+       return do_set_mempolicy(mode, flags, &nodes);

 }
 
 asmlinkage long sys_migrate_pages(pid_t pid, unsigned long maxnode,
 }
 
 asmlinkage long sys_migrate_pages(pid_t pid, unsigned long maxnode,


@@ -796,7 +1123,7 @@ asmlinkage long sys_migrate_pages(pid_t pid, unsigned long maxnode,
 
        /* Find the mm_struct */
        read_lock(&tasklist_lock);
 
        /* Find the mm_struct */
        read_lock(&tasklist_lock);

-       task = pid ? find_task_by_pid(pid) : current;
+       task = pid ? find_task_by_vpid(pid) : current;

        if (!task) {
                read_unlock(&tasklist_lock);
                return -ESRCH;
        if (!task) {
                read_unlock(&tasklist_lock);
                return -ESRCH;


@@ -810,24 +1137,34 @@ asmlinkage long sys_migrate_pages(pid_t pid, unsigned long maxnode,
        /*
         * Check if this process has the right to modify the specified
         * process. The right exists if the process has administrative
        /*
         * Check if this process has the right to modify the specified
         * process. The right exists if the process has administrative

-        * capabilities, superuser priviledges or the same
+        * capabilities, superuser privileges or the same

         * userid as the target process.
         */
        if ((current->euid != task->suid) && (current->euid != task->uid) &&
            (current->uid != task->suid) && (current->uid != task->uid) &&
         * userid as the target process.
         */
        if ((current->euid != task->suid) && (current->euid != task->uid) &&
            (current->uid != task->suid) && (current->uid != task->uid) &&

-           !capable(CAP_SYS_ADMIN)) {
+           !capable(CAP_SYS_NICE)) {

                err = -EPERM;
                goto out;
        }
 
        task_nodes = cpuset_mems_allowed(task);
        /* Is the user allowed to access the target nodes? */
                err = -EPERM;
                goto out;
        }
 
        task_nodes = cpuset_mems_allowed(task);
        /* Is the user allowed to access the target nodes? */

-       if (!nodes_subset(new, task_nodes) && !capable(CAP_SYS_ADMIN)) {
+       if (!nodes_subset(new, task_nodes) && !capable(CAP_SYS_NICE)) {

                err = -EPERM;
                goto out;
        }
 
                err = -EPERM;
                goto out;
        }
 

-       err = do_migrate_pages(mm, &old, &new, MPOL_MF_MOVE);
+       if (!nodes_subset(new, node_states[N_HIGH_MEMORY])) {
+               err = -EINVAL;
+               goto out;
+       }
+
+       err = security_task_movememory(task);
+       if (err)
+               goto out;
+
+       err = do_migrate_pages(mm, &old, &new,
+               capable(CAP_SYS_NICE) ? MPOL_MF_MOVE_ALL : MPOL_MF_MOVE);

 out:
        mmput(mm);
        return err;
 out:
        mmput(mm);
        return err;


@@ -840,7 +1177,8 @@ asmlinkage long sys_get_mempolicy(int __user *policy,
                                unsigned long maxnode,
                                unsigned long addr, unsigned long flags)
 {
                                unsigned long maxnode,
                                unsigned long addr, unsigned long flags)
 {

-       int err, pval;
+       int err;
+       int uninitialized_var(pval);

        nodemask_t nodes;
 
        if (nmask != NULL && maxnode < MAX_NUMNODES)
        nodemask_t nodes;
 
        if (nmask != NULL && maxnode < MAX_NUMNODES)


@@ -939,17 +1277,34 @@ asmlinkage long compat_sys_mbind(compat_ulong_t start, compat_ulong_t len,
 
 #endif
 
 
 #endif
 

-/* Return effective policy for a VMA */
-static struct mempolicy * get_vma_policy(struct task_struct *task,
+/*
+ * get_vma_policy(@task, @vma, @addr)
+ * @task - task for fallback if vma policy == default
+ * @vma   - virtual memory area whose policy is sought
+ * @addr  - address in @vma for shared policy lookup
+ *
+ * Returns effective policy for a VMA at specified address.
+ * Falls back to @task or system default policy, as necessary.
+ * Current or other task's task mempolicy and non-shared vma policies
+ * are protected by the task's mmap_sem, which must be held for read by
+ * the caller.
+ * Shared policies [those marked as MPOL_F_SHARED] require an extra reference
+ * count--added by the get_policy() vm_op, as appropriate--to protect against
+ * freeing by another task.  It is the caller's responsibility to free the
+ * extra reference for shared policies.
+ */
+static struct mempolicy *get_vma_policy(struct task_struct *task,

                struct vm_area_struct *vma, unsigned long addr)
 {
        struct mempolicy *pol = task->mempolicy;
 
        if (vma) {
                struct vm_area_struct *vma, unsigned long addr)
 {
        struct mempolicy *pol = task->mempolicy;
 
        if (vma) {

-               if (vma->vm_ops && vma->vm_ops->get_policy)
-                       pol = vma->vm_ops->get_policy(vma, addr);
-               else if (vma->vm_policy &&
-                               vma->vm_policy->policy != MPOL_DEFAULT)
+               if (vma->vm_ops && vma->vm_ops->get_policy) {
+                       struct mempolicy *vpol = vma->vm_ops->get_policy(vma,
+                                                                       addr);
+                       if (vpol)
+                               pol = vpol;
+               } else if (vma->vm_policy)

                        pol = vma->vm_policy;
        }
        if (!pol)
                        pol = vma->vm_policy;
        }
        if (!pol)


@@ -957,33 +1312,48 @@ static struct mempolicy * get_vma_policy(struct task_struct *task,
        return pol;
 }
 
        return pol;
 }
 

-/* Return a zonelist representing a mempolicy */
-static struct zonelist *zonelist_policy(gfp_t gfp, struct mempolicy *policy)
+/*
+ * Return a nodemask representing a mempolicy for filtering nodes for
+ * page allocation
+ */
+static nodemask_t *policy_nodemask(gfp_t gfp, struct mempolicy *policy)
+{
+       /* Lower zones don't get a nodemask applied for MPOL_BIND */
+       if (unlikely(policy->mode == MPOL_BIND) &&
+                       gfp_zone(gfp) >= policy_zone &&
+                       cpuset_nodemask_valid_mems_allowed(&policy->v.nodes))
+               return &policy->v.nodes;
+
+       return NULL;
+}
+
+/* Return a zonelist indicated by gfp for node representing a mempolicy */
+static struct zonelist *policy_zonelist(gfp_t gfp, struct mempolicy *policy)

 {
 {

-       int nd;
+       int nd = numa_node_id();

 
 

-       switch (policy->policy) {
+       switch (policy->mode) {

        case MPOL_PREFERRED:
        case MPOL_PREFERRED:

-               nd = policy->v.preferred_node;
-               if (nd < 0)
-                       nd = numa_node_id();
+               if (!(policy->flags & MPOL_F_LOCAL))
+                       nd = policy->v.preferred_node;

                break;
        case MPOL_BIND:
                break;
        case MPOL_BIND:

-               /* Lower zones don't get a policy applied */
-               /* Careful: current->mems_allowed might have moved */
-               if (gfp_zone(gfp) >= policy_zone)
-                       if (cpuset_zonelist_valid_mems_allowed(policy->v.zonelist))
-                               return policy->v.zonelist;
-               /*FALL THROUGH*/
+               /*
+                * Normally, MPOL_BIND allocations are node-local within the
+                * allowed nodemask.  However, if __GFP_THISNODE is set and the
+                * current node is part of the mask, we use the zonelist for
+                * the first node in the mask instead.
+                */
+               if (unlikely(gfp & __GFP_THISNODE) &&
+                               unlikely(!node_isset(nd, policy->v.nodes)))
+                       nd = first_node(policy->v.nodes);
+               break;

        case MPOL_INTERLEAVE: /* should not happen */
        case MPOL_INTERLEAVE: /* should not happen */

-       case MPOL_DEFAULT:
-               nd = numa_node_id();

                break;
        default:
                break;
        default:

-               nd = 0;

                BUG();
        }
                BUG();
        }

-       return NODE_DATA(nd)->node_zonelists + gfp_zone(gfp);
+       return node_zonelist(nd, gfp);

 }
 
 /* Do dynamic interleaving for a process */
 }
 
 /* Do dynamic interleaving for a process */


@@ -996,19 +1366,66 @@ static unsigned interleave_nodes(struct mempolicy *policy)
        next = next_node(nid, policy->v.nodes);
        if (next >= MAX_NUMNODES)
                next = first_node(policy->v.nodes);
        next = next_node(nid, policy->v.nodes);
        if (next >= MAX_NUMNODES)
                next = first_node(policy->v.nodes);

-       me->il_next = next;
+       if (next < MAX_NUMNODES)
+               me->il_next = next;

        return nid;
 }
 
        return nid;
 }
 

+/*
+ * Depending on the memory policy provide a node from which to allocate the
+ * next slab entry.
+ * @policy must be protected by freeing by the caller.  If @policy is
+ * the current task's mempolicy, this protection is implicit, as only the
+ * task can change it's policy.  The system default policy requires no
+ * such protection.
+ */
+unsigned slab_node(struct mempolicy *policy)
+{
+       if (!policy || policy->flags & MPOL_F_LOCAL)
+               return numa_node_id();
+
+       switch (policy->mode) {
+       case MPOL_PREFERRED:
+               /*
+                * handled MPOL_F_LOCAL above
+                */
+               return policy->v.preferred_node;
+
+       case MPOL_INTERLEAVE:
+               return interleave_nodes(policy);
+
+       case MPOL_BIND: {
+               /*
+                * Follow bind policy behavior and start allocation at the
+                * first node.
+                */
+               struct zonelist *zonelist;
+               struct zone *zone;
+               enum zone_type highest_zoneidx = gfp_zone(GFP_KERNEL);
+               zonelist = &NODE_DATA(numa_node_id())->node_zonelists[0];
+               (void)first_zones_zonelist(zonelist, highest_zoneidx,
+                                                       &policy->v.nodes,
+                                                       &zone);
+               return zone->node;
+       }
+
+       default:
+               BUG();
+       }
+}
+

 /* Do static interleaving for a VMA with known offset. */
 static unsigned offset_il_node(struct mempolicy *pol,
                struct vm_area_struct *vma, unsigned long off)
 {
        unsigned nnodes = nodes_weight(pol->v.nodes);
 /* Do static interleaving for a VMA with known offset. */
 static unsigned offset_il_node(struct mempolicy *pol,
                struct vm_area_struct *vma, unsigned long off)
 {
        unsigned nnodes = nodes_weight(pol->v.nodes);

-       unsigned target = (unsigned)off % nnodes;
+       unsigned target;

        int c;
        int nid = -1;
 
        int c;
        int nid = -1;
 

+       if (!nnodes)
+               return numa_node_id();
+       target = (unsigned int)off % nnodes;

        c = 0;
        do {
                nid = next_node(nid, pol->v.nodes);
        c = 0;
        do {
                nid = next_node(nid, pol->v.nodes);


@@ -1024,26 +1441,55 @@ static inline unsigned interleave_nid(struct mempolicy *pol,
        if (vma) {
                unsigned long off;
 
        if (vma) {
                unsigned long off;
 

-               off = vma->vm_pgoff;
+               /*
+                * for small pages, there is no difference between
+                * shift and PAGE_SHIFT, so the bit-shift is safe.
+                * for huge pages, since vm_pgoff is in units of small
+                * pages, we need to shift off the always 0 bits to get
+                * a useful offset.
+                */
+               BUG_ON(shift < PAGE_SHIFT);
+               off = vma->vm_pgoff >> (shift - PAGE_SHIFT);

                off += (addr - vma->vm_start) >> shift;
                return offset_il_node(pol, vma, off);
        } else
                return interleave_nodes(pol);
 }
 
                off += (addr - vma->vm_start) >> shift;
                return offset_il_node(pol, vma, off);
        } else
                return interleave_nodes(pol);
 }
 

-/* Return a zonelist suitable for a huge page allocation. */
-struct zonelist *huge_zonelist(struct vm_area_struct *vma, unsigned long addr)
+#ifdef CONFIG_HUGETLBFS
+/*
+ * huge_zonelist(@vma, @addr, @gfp_flags, @mpol)
+ * @vma = virtual memory area whose policy is sought
+ * @addr = address in @vma for shared policy lookup and interleave policy
+ * @gfp_flags = for requested zone
+ * @mpol = pointer to mempolicy pointer for reference counted mempolicy
+ * @nodemask = pointer to nodemask pointer for MPOL_BIND nodemask
+ *
+ * Returns a zonelist suitable for a huge page allocation and a pointer
+ * to the struct mempolicy for conditional unref after allocation.
+ * If the effective policy is 'BIND, returns a pointer to the mempolicy's
+ * @nodemask for filtering the zonelist.
+ */
+struct zonelist *huge_zonelist(struct vm_area_struct *vma, unsigned long addr,
+                               gfp_t gfp_flags, struct mempolicy **mpol,
+                               nodemask_t **nodemask)

 {
 {

-       struct mempolicy *pol = get_vma_policy(current, vma, addr);
+       struct zonelist *zl;

 
 

-       if (pol->policy == MPOL_INTERLEAVE) {
-               unsigned nid;
+       *mpol = get_vma_policy(current, vma, addr);
+       *nodemask = NULL;       /* assume !MPOL_BIND */

 
 

-               nid = interleave_nid(pol, vma, addr, HPAGE_SHIFT);
-               return NODE_DATA(nid)->node_zonelists + gfp_zone(GFP_HIGHUSER);
+       if (unlikely((*mpol)->mode == MPOL_INTERLEAVE)) {
+               zl = node_zonelist(interleave_nid(*mpol, vma, addr,
+                               huge_page_shift(hstate_vma(vma))), gfp_flags);
+       } else {
+               zl = policy_zonelist(gfp_flags, *mpol);
+               if ((*mpol)->mode == MPOL_BIND)
+                       *nodemask = &(*mpol)->v.nodes;

        }
        }

-       return zonelist_policy(GFP_HIGHUSER, pol);
+       return zl;

 }
 }

+#endif

 
 /* Allocate a page in interleaved policy.
    Own path because it needs to do special accounting. */
 
 /* Allocate a page in interleaved policy.
    Own path because it needs to do special accounting. */


@@ -1053,12 +1499,10 @@ static struct page *alloc_page_interleave(gfp_t gfp, unsigned order,
        struct zonelist *zl;
        struct page *page;
 
        struct zonelist *zl;
        struct page *page;
 

-       zl = NODE_DATA(nid)->node_zonelists + gfp_zone(gfp);
+       zl = node_zonelist(nid, gfp);

        page = __alloc_pages(gfp, order, zl);
        page = __alloc_pages(gfp, order, zl);

-       if (page && page_zone(page) == zl->zones[0]) {
-               zone_pcp(zl->zones[0],get_cpu())->interleave_hit++;
-               put_cpu();
-       }
+       if (page && page_zone(page) == zonelist_zone(&zl->_zonerefs[0]))
+               inc_zone_page_state(page, NUMA_INTERLEAVE_HIT);

        return page;
 }
 
        return page;
 }
 


@@ -1088,16 +1532,31 @@ struct page *
 alloc_page_vma(gfp_t gfp, struct vm_area_struct *vma, unsigned long addr)
 {
        struct mempolicy *pol = get_vma_policy(current, vma, addr);
 alloc_page_vma(gfp_t gfp, struct vm_area_struct *vma, unsigned long addr)
 {
        struct mempolicy *pol = get_vma_policy(current, vma, addr);

+       struct zonelist *zl;

 
        cpuset_update_task_memory_state();
 
 
        cpuset_update_task_memory_state();
 

-       if (unlikely(pol->policy == MPOL_INTERLEAVE)) {
+       if (unlikely(pol->mode == MPOL_INTERLEAVE)) {

                unsigned nid;
 
                nid = interleave_nid(pol, vma, addr, PAGE_SHIFT);
                unsigned nid;
 
                nid = interleave_nid(pol, vma, addr, PAGE_SHIFT);

+               mpol_cond_put(pol);

                return alloc_page_interleave(gfp, 0, nid);
        }
                return alloc_page_interleave(gfp, 0, nid);
        }

-       return __alloc_pages(gfp, 0, zonelist_policy(gfp, pol));
+       zl = policy_zonelist(gfp, pol);
+       if (unlikely(mpol_needs_cond_ref(pol))) {
+               /*
+                * slow path: ref counted shared policy
+                */
+               struct page *page =  __alloc_pages_nodemask(gfp, 0,
+                                               zl, policy_nodemask(gfp, pol));
+               __mpol_put(pol);
+               return page;
+       }
+       /*
+        * fast path:  default or task policy
+        */
+       return __alloc_pages_nodemask(gfp, 0, zl, policy_nodemask(gfp, pol));

 }
 
 /**
 }
 
 /**


@@ -1125,25 +1584,30 @@ struct page *alloc_pages_current(gfp_t gfp, unsigned order)
 
        if ((gfp & __GFP_WAIT) && !in_interrupt())
                cpuset_update_task_memory_state();
 
        if ((gfp & __GFP_WAIT) && !in_interrupt())
                cpuset_update_task_memory_state();

-       if (!pol || in_interrupt())
+       if (!pol || in_interrupt() || (gfp & __GFP_THISNODE))

                pol = &default_policy;
                pol = &default_policy;

-       if (pol->policy == MPOL_INTERLEAVE)
+
+       /*
+        * No reference counting needed for current->mempolicy
+        * nor system default_policy
+        */
+       if (pol->mode == MPOL_INTERLEAVE)

                return alloc_page_interleave(gfp, order, interleave_nodes(pol));
                return alloc_page_interleave(gfp, order, interleave_nodes(pol));

-       return __alloc_pages(gfp, order, zonelist_policy(gfp, pol));
+       return __alloc_pages_nodemask(gfp, order,
+                       policy_zonelist(gfp, pol), policy_nodemask(gfp, pol));

 }
 EXPORT_SYMBOL(alloc_pages_current);
 
 /*
 }
 EXPORT_SYMBOL(alloc_pages_current);
 
 /*

- * If mpol_copy() sees current->cpuset == cpuset_being_rebound, then it
+ * If mpol_dup() sees current->cpuset == cpuset_being_rebound, then it

  * rebinds the mempolicy its copying by calling mpol_rebind_policy()
  * with the mems_allowed returned by cpuset_mems_allowed().  This
  * keeps mempolicies cpuset relative after its cpuset moves.  See
  * further kernel/cpuset.c update_nodemask().
  */
  * rebinds the mempolicy its copying by calling mpol_rebind_policy()
  * with the mems_allowed returned by cpuset_mems_allowed().  This
  * keeps mempolicies cpuset relative after its cpuset moves.  See
  * further kernel/cpuset.c update_nodemask().
  */

-void *cpuset_being_rebound;

 
 

-/* Slow path of a mempolicy copy */
-struct mempolicy *__mpol_copy(struct mempolicy *old)
+/* Slow path of a mempolicy duplicate */
+struct mempolicy *__mpol_dup(struct mempolicy *old)

 {
        struct mempolicy *new = kmem_cache_alloc(policy_cache, GFP_KERNEL);
 
 {
        struct mempolicy *new = kmem_cache_alloc(policy_cache, GFP_KERNEL);
 


@@ -1155,56 +1619,64 @@ struct mempolicy *__mpol_copy(struct mempolicy *old)
        }
        *new = *old;
        atomic_set(&new->refcnt, 1);
        }
        *new = *old;
        atomic_set(&new->refcnt, 1);

-       if (new->policy == MPOL_BIND) {
-               int sz = ksize(old->v.zonelist);
-               new->v.zonelist = kmalloc(sz, SLAB_KERNEL);
-               if (!new->v.zonelist) {
-                       kmem_cache_free(policy_cache, new);
-                       return ERR_PTR(-ENOMEM);
-               }
-               memcpy(new->v.zonelist, old->v.zonelist, sz);
-       }

        return new;
 }
 
        return new;
 }
 

+/*
+ * If *frompol needs [has] an extra ref, copy *frompol to *tompol ,
+ * eliminate the * MPOL_F_* flags that require conditional ref and
+ * [NOTE!!!] drop the extra ref.  Not safe to reference *frompol directly
+ * after return.  Use the returned value.
+ *
+ * Allows use of a mempolicy for, e.g., multiple allocations with a single
+ * policy lookup, even if the policy needs/has extra ref on lookup.
+ * shmem_readahead needs this.
+ */
+struct mempolicy *__mpol_cond_copy(struct mempolicy *tompol,
+                                               struct mempolicy *frompol)
+{
+       if (!mpol_needs_cond_ref(frompol))
+               return frompol;
+
+       *tompol = *frompol;
+       tompol->flags &= ~MPOL_F_SHARED;        /* copy doesn't need unref */
+       __mpol_put(frompol);
+       return tompol;
+}
+
+static int mpol_match_intent(const struct mempolicy *a,
+                            const struct mempolicy *b)
+{
+       if (a->flags != b->flags)
+               return 0;
+       if (!mpol_store_user_nodemask(a))
+               return 1;
+       return nodes_equal(a->w.user_nodemask, b->w.user_nodemask);
+}
+

 /* Slow path of a mempolicy comparison */
 int __mpol_equal(struct mempolicy *a, struct mempolicy *b)
 {
        if (!a || !b)
                return 0;
 /* Slow path of a mempolicy comparison */
 int __mpol_equal(struct mempolicy *a, struct mempolicy *b)
 {
        if (!a || !b)
                return 0;

-       if (a->policy != b->policy)
+       if (a->mode != b->mode)

                return 0;
                return 0;

-       switch (a->policy) {
-       case MPOL_DEFAULT:
-               return 1;
+       if (a->mode != MPOL_DEFAULT && !mpol_match_intent(a, b))
+               return 0;
+       switch (a->mode) {
+       case MPOL_BIND:
+               /* Fall through */

        case MPOL_INTERLEAVE:
                return nodes_equal(a->v.nodes, b->v.nodes);
        case MPOL_PREFERRED:
        case MPOL_INTERLEAVE:
                return nodes_equal(a->v.nodes, b->v.nodes);
        case MPOL_PREFERRED:

-               return a->v.preferred_node == b->v.preferred_node;
-       case MPOL_BIND: {
-               int i;
-               for (i = 0; a->v.zonelist->zones[i]; i++)
-                       if (a->v.zonelist->zones[i] != b->v.zonelist->zones[i])
-                               return 0;
-               return b->v.zonelist->zones[i] == NULL;
-       }
+               return a->v.preferred_node == b->v.preferred_node &&
+                       a->flags == b->flags;

        default:
                BUG();
                return 0;
        }
 }
 
        default:
                BUG();
                return 0;
        }
 }
 

-/* Slow path of a mpol destructor. */
-void __mpol_free(struct mempolicy *p)
-{
-       if (!atomic_dec_and_test(&p->refcnt))
-               return;
-       if (p->policy == MPOL_BIND)
-               kfree(p->v.zonelist);
-       p->policy = MPOL_DEFAULT;
-       kmem_cache_free(policy_cache, p);
-}
-

 /*
  * Shared memory backing store policy support.
  *
 /*
  * Shared memory backing store policy support.
  *


@@ -1266,8 +1738,8 @@ static void sp_insert(struct shared_policy *sp, struct sp_node *new)
        }
        rb_link_node(&new->nd, parent, p);
        rb_insert_color(&new->nd, &sp->root);
        }
        rb_link_node(&new->nd, parent, p);
        rb_insert_color(&new->nd, &sp->root);

-       PDprintk("inserting %lx-%lx: %d\n", new->start, new->end,
-                new->policy ? new->policy->policy : 0);
+       pr_debug("inserting %lx-%lx: %d\n", new->start, new->end,
+                new->policy ? new->policy->mode : 0);

 }
 
 /* Find shared policy intersecting idx */
 }
 
 /* Find shared policy intersecting idx */


@@ -1291,14 +1763,14 @@ mpol_shared_policy_lookup(struct shared_policy *sp, unsigned long idx)
 
 static void sp_delete(struct shared_policy *sp, struct sp_node *n)
 {
 
 static void sp_delete(struct shared_policy *sp, struct sp_node *n)
 {

-       PDprintk("deleting %lx-l%x\n", n->start, n->end);
+       pr_debug("deleting %lx-l%lx\n", n->start, n->end);

        rb_erase(&n->nd, &sp->root);
        rb_erase(&n->nd, &sp->root);

-       mpol_free(n->policy);
+       mpol_put(n->policy);

        kmem_cache_free(sn_cache, n);
 }
 
        kmem_cache_free(sn_cache, n);
 }
 

-struct sp_node *
-sp_alloc(unsigned long start, unsigned long end, struct mempolicy *pol)
+static struct sp_node *sp_alloc(unsigned long start, unsigned long end,
+                               struct mempolicy *pol)

 {
        struct sp_node *n = kmem_cache_alloc(sn_cache, GFP_KERNEL);
 
 {
        struct sp_node *n = kmem_cache_alloc(sn_cache, GFP_KERNEL);
 


@@ -1307,6 +1779,7 @@ sp_alloc(unsigned long start, unsigned long end, struct mempolicy *pol)
        n->start = start;
        n->end = end;
        mpol_get(pol);
        n->start = start;
        n->end = end;
        mpol_get(pol);

+       pol->flags |= MPOL_F_SHARED;    /* for unref */

        n->policy = pol;
        return n;
 }
        n->policy = pol;
        return n;
 }


@@ -1353,33 +1826,41 @@ restart:
                sp_insert(sp, new);
        spin_unlock(&sp->lock);
        if (new2) {
                sp_insert(sp, new);
        spin_unlock(&sp->lock);
        if (new2) {

-               mpol_free(new2->policy);
+               mpol_put(new2->policy);

                kmem_cache_free(sn_cache, new2);
        }
        return 0;
 }
 
                kmem_cache_free(sn_cache, new2);
        }
        return 0;
 }
 

-void mpol_shared_policy_init(struct shared_policy *info, int policy,
-                               nodemask_t *policy_nodes)
+/**
+ * mpol_shared_policy_init - initialize shared policy for inode
+ * @sp: pointer to inode shared policy
+ * @mpol:  struct mempolicy to install
+ *
+ * Install non-NULL @mpol in inode's shared policy rb-tree.
+ * On entry, the current task has a reference on a non-NULL @mpol.
+ * This must be released on exit.
+ */
+void mpol_shared_policy_init(struct shared_policy *sp, struct mempolicy *mpol)

 {
 {

-       info->root = RB_ROOT;
-       spin_lock_init(&info->lock);
-
-       if (policy != MPOL_DEFAULT) {
-               struct mempolicy *newpol;
-
-               /* Falls back to MPOL_DEFAULT on any error */
-               newpol = mpol_new(policy, policy_nodes);
-               if (!IS_ERR(newpol)) {
-                       /* Create pseudo-vma that contains just the policy */
-                       struct vm_area_struct pvma;
-
-                       memset(&pvma, 0, sizeof(struct vm_area_struct));
-                       /* Policy covers entire file */
-                       pvma.vm_end = TASK_SIZE;
-                       mpol_set_shared_policy(info, &pvma, newpol);
-                       mpol_free(newpol);
-               }
+       sp->root = RB_ROOT;             /* empty tree == default mempolicy */
+       spin_lock_init(&sp->lock);
+
+       if (mpol) {
+               struct vm_area_struct pvma;
+               struct mempolicy *new;
+
+               /* contextualize the tmpfs mount point mempolicy */
+               new = mpol_new(mpol->mode, mpol->flags, &mpol->w.user_nodemask);
+               mpol_put(mpol); /* drop our ref on sb mpol */
+               if (IS_ERR(new))
+                       return;         /* no valid nodemask intersection */
+
+               /* Create pseudo-vma that contains just the policy */
+               memset(&pvma, 0, sizeof(struct vm_area_struct));
+               pvma.vm_end = TASK_SIZE;        /* policy covers entire file */
+               mpol_set_shared_policy(sp, &pvma, new); /* adds ref */
+               mpol_put(new);                  /* drop initial ref */

        }
 }
 
        }
 }
 


@@ -1390,10 +1871,11 @@ int mpol_set_shared_policy(struct shared_policy *info,
        struct sp_node *new = NULL;
        unsigned long sz = vma_pages(vma);
 
        struct sp_node *new = NULL;
        unsigned long sz = vma_pages(vma);
 

-       PDprintk("set_shared_policy %lx sz %lu %d %lx\n",
+       pr_debug("set_shared_policy %lx sz %lu %d %d %lx\n",

                 vma->vm_pgoff,
                 vma->vm_pgoff,

-                sz, npol? npol->policy : -1,
-               npol ? nodes_addr(npol->v.nodes)[0] : -1);
+                sz, npol ? npol->mode : -1,
+                npol ? npol->flags : -1,
+                npol ? nodes_addr(npol->v.nodes)[0] : -1);

 
        if (npol) {
                new = sp_alloc(vma->vm_pgoff, vma->vm_pgoff + sz, npol);
 
        if (npol) {
                new = sp_alloc(vma->vm_pgoff, vma->vm_pgoff + sz, npol);


@@ -1420,7 +1902,7 @@ void mpol_free_shared_policy(struct shared_policy *p)
                n = rb_entry(next, struct sp_node, nd);
                next = rb_next(&n->nd);
                rb_erase(&n->nd, &p->root);
                n = rb_entry(next, struct sp_node, nd);
                next = rb_next(&n->nd);
                rb_erase(&n->nd, &p->root);

-               mpol_free(n->policy);
+               mpol_put(n->policy);

                kmem_cache_free(sn_cache, n);
        }
        spin_unlock(&p->lock);
                kmem_cache_free(sn_cache, n);
        }
        spin_unlock(&p->lock);


@@ -1429,130 +1911,213 @@ void mpol_free_shared_policy(struct shared_policy *p)
 /* assumes fs == KERNEL_DS */
 void __init numa_policy_init(void)
 {
 /* assumes fs == KERNEL_DS */
 void __init numa_policy_init(void)
 {

+       nodemask_t interleave_nodes;
+       unsigned long largest = 0;
+       int nid, prefer = 0;
+

        policy_cache = kmem_cache_create("numa_policy",
                                         sizeof(struct mempolicy),
        policy_cache = kmem_cache_create("numa_policy",
                                         sizeof(struct mempolicy),

-                                        0, SLAB_PANIC, NULL, NULL);
+                                        0, SLAB_PANIC, NULL);

 
        sn_cache = kmem_cache_create("shared_policy_node",
                                     sizeof(struct sp_node),
 
        sn_cache = kmem_cache_create("shared_policy_node",
                                     sizeof(struct sp_node),

-                                    0, SLAB_PANIC, NULL, NULL);
+                                    0, SLAB_PANIC, NULL);
+
+       /*
+        * Set interleaving policy for system init. Interleaving is only
+        * enabled across suitably sized nodes (default is >= 16MB), or
+        * fall back to the largest node if they're all smaller.
+        */
+       nodes_clear(interleave_nodes);
+       for_each_node_state(nid, N_HIGH_MEMORY) {
+               unsigned long total_pages = node_present_pages(nid);
+
+               /* Preserve the largest node */
+               if (largest < total_pages) {
+                       largest = total_pages;
+                       prefer = nid;
+               }
+
+               /* Interleave this node? */
+               if ((total_pages << PAGE_SHIFT) >= (16 << 20))
+                       node_set(nid, interleave_nodes);
+       }

 
 

-       /* Set interleaving policy for system init. This way not all
-          the data structures allocated at system boot end up in node zero. */
+       /* All too small, use the largest */
+       if (unlikely(nodes_empty(interleave_nodes)))
+               node_set(prefer, interleave_nodes);

 
 

-       if (do_set_mempolicy(MPOL_INTERLEAVE, &node_online_map))
+       if (do_set_mempolicy(MPOL_INTERLEAVE, 0, &interleave_nodes))

                printk("numa_policy_init: interleaving failed\n");
 }
 
 /* Reset policy of current process to default */
 void numa_default_policy(void)
 {
                printk("numa_policy_init: interleaving failed\n");
 }
 
 /* Reset policy of current process to default */
 void numa_default_policy(void)
 {

-       do_set_mempolicy(MPOL_DEFAULT, NULL);
+       do_set_mempolicy(MPOL_DEFAULT, 0, NULL);

 }
 
 }
 

-/* Migrate a policy to a different set of nodes */
-void mpol_rebind_policy(struct mempolicy *pol, const nodemask_t *newmask)
-{
-       nodemask_t *mpolmask;
-       nodemask_t tmp;
+/*
+ * Parse and format mempolicy from/to strings
+ */

 
 

-       if (!pol)
-               return;
-       mpolmask = &pol->cpuset_mems_allowed;
-       if (nodes_equal(*mpolmask, *newmask))
-               return;
+/*
+ * "local" is pseudo-policy:  MPOL_PREFERRED with MPOL_F_LOCAL flag
+ * Used only for mpol_parse_str() and mpol_to_str()
+ */
+#define MPOL_LOCAL (MPOL_INTERLEAVE + 1)
+static const char * const policy_types[] =
+       { "default", "prefer", "bind", "interleave", "local" };

 
 

-       switch (pol->policy) {
-       case MPOL_DEFAULT:
-               break;
-       case MPOL_INTERLEAVE:
-               nodes_remap(tmp, pol->v.nodes, *mpolmask, *newmask);
-               pol->v.nodes = tmp;
-               *mpolmask = *newmask;
-               current->il_next = node_remap(current->il_next,
-                                               *mpolmask, *newmask);
-               break;
-       case MPOL_PREFERRED:
-               pol->v.preferred_node = node_remap(pol->v.preferred_node,
-                                               *mpolmask, *newmask);
-               *mpolmask = *newmask;
-               break;
-       case MPOL_BIND: {
-               nodemask_t nodes;
-               struct zone **z;
-               struct zonelist *zonelist;

 
 

+#ifdef CONFIG_TMPFS
+/**
+ * mpol_parse_str - parse string to mempolicy
+ * @str:  string containing mempolicy to parse
+ * @mpol:  pointer to struct mempolicy pointer, returned on success.
+ * @no_context:  flag whether to "contextualize" the mempolicy
+ *
+ * Format of input:
+ *     <mode>[=<flags>][:<nodelist>]
+ *
+ * if @no_context is true, save the input nodemask in w.user_nodemask in
+ * the returned mempolicy.  This will be used to "clone" the mempolicy in
+ * a specific context [cpuset] at a later time.  Used to parse tmpfs mpol
+ * mount option.  Note that if 'static' or 'relative' mode flags were
+ * specified, the input nodemask will already have been saved.  Saving
+ * it again is redundant, but safe.
+ *
+ * On success, returns 0, else 1
+ */
+int mpol_parse_str(char *str, struct mempolicy **mpol, int no_context)
+{
+       struct mempolicy *new = NULL;
+       unsigned short uninitialized_var(mode);
+       unsigned short uninitialized_var(mode_flags);
+       nodemask_t nodes;
+       char *nodelist = strchr(str, ':');
+       char *flags = strchr(str, '=');
+       int i;
+       int err = 1;
+
+       if (nodelist) {
+               /* NUL-terminate mode or flags string */
+               *nodelist++ = '\0';
+               if (nodelist_parse(nodelist, nodes))
+                       goto out;
+               if (!nodes_subset(nodes, node_states[N_HIGH_MEMORY]))
+                       goto out;
+       } else

                nodes_clear(nodes);
                nodes_clear(nodes);

-               for (z = pol->v.zonelist->zones; *z; z++)
-                       node_set((*z)->zone_pgdat->node_id, nodes);
-               nodes_remap(tmp, nodes, *mpolmask, *newmask);
-               nodes = tmp;

 
 

-               zonelist = bind_zonelist(&nodes);
+       if (flags)
+               *flags++ = '\0';        /* terminate mode string */

 
 

-               /* If no mem, then zonelist is NULL and we keep old zonelist.
-                * If that old zonelist has no remaining mems_allowed nodes,
-                * then zonelist_policy() will "FALL THROUGH" to MPOL_DEFAULT.
-                */
+       for (i = 0; i <= MPOL_LOCAL; i++) {
+               if (!strcmp(str, policy_types[i])) {
+                       mode = i;
+                       break;
+               }
+       }
+       if (i > MPOL_LOCAL)
+               goto out;

 
 

-               if (zonelist) {
-                       /* Good - got mem - substitute new zonelist */
-                       kfree(pol->v.zonelist);
-                       pol->v.zonelist = zonelist;
+       switch (mode) {
+       case MPOL_PREFERRED:
+               /*
+                * Insist on a nodelist of one node only
+                */
+               if (nodelist) {
+                       char *rest = nodelist;
+                       while (isdigit(*rest))
+                               rest++;
+                       if (!*rest)
+                               err = 0;

                }
                }

-               *mpolmask = *newmask;

                break;
                break;

-       }
-       default:
-               BUG();
+       case MPOL_INTERLEAVE:
+               /*
+                * Default to online nodes with memory if no nodelist
+                */
+               if (!nodelist)
+                       nodes = node_states[N_HIGH_MEMORY];
+               err = 0;
+               break;
+       case MPOL_LOCAL:
+               /*
+                * Don't allow a nodelist;  mpol_new() checks flags
+                */
+               if (nodelist)
+                       goto out;
+               mode = MPOL_PREFERRED;

                break;
                break;

-       }
-}
-
-/*
- * Wrapper for mpol_rebind_policy() that just requires task
- * pointer, and updates task mempolicy.
- */

 
 

-void mpol_rebind_task(struct task_struct *tsk, const nodemask_t *new)
-{
-       mpol_rebind_policy(tsk->mempolicy, new);
-}
+       /*
+        * case MPOL_BIND:    mpol_new() enforces non-empty nodemask.
+        * case MPOL_DEFAULT: mpol_new() enforces empty nodemask, ignores flags.
+        */
+       }

 
 

-/*
- * Rebind each vma in mm to new nodemask.
- *
- * Call holding a reference to mm.  Takes mm->mmap_sem during call.
- */
+       mode_flags = 0;
+       if (flags) {
+               /*
+                * Currently, we only support two mutually exclusive
+                * mode flags.
+                */
+               if (!strcmp(flags, "static"))
+                       mode_flags |= MPOL_F_STATIC_NODES;
+               else if (!strcmp(flags, "relative"))
+                       mode_flags |= MPOL_F_RELATIVE_NODES;
+               else
+                       err = 1;
+       }

 
 

-void mpol_rebind_mm(struct mm_struct *mm, nodemask_t *new)
-{
-       struct vm_area_struct *vma;
+       new = mpol_new(mode, mode_flags, &nodes);
+       if (IS_ERR(new))
+               err = 1;
+       else if (no_context)
+               new->w.user_nodemask = nodes;   /* save for contextualization */

 
 

-       down_write(&mm->mmap_sem);
-       for (vma = mm->mmap; vma; vma = vma->vm_next)
-               mpol_rebind_policy(vma->vm_policy, new);
-       up_write(&mm->mmap_sem);
+out:
+       /* Restore string for error message */
+       if (nodelist)
+               *--nodelist = ':';
+       if (flags)
+               *--flags = '=';
+       if (!err)
+               *mpol = new;
+       return err;

 }
 }

+#endif /* CONFIG_TMPFS */

 
 

-/*
- * Display pages allocated per node and memory policy via /proc.
- */
-
-static const char *policy_types[] = { "default", "prefer", "bind",
-                                     "interleave" };
-
-/*
+/**
+ * mpol_to_str - format a mempolicy structure for printing
+ * @buffer:  to contain formatted mempolicy string
+ * @maxlen:  length of @buffer
+ * @pol:  pointer to mempolicy to be formatted
+ * @no_context:  "context free" mempolicy - use nodemask in w.user_nodemask
+ *

  * Convert a mempolicy into a string.
  * Returns the number of characters in buffer (if positive)
  * or an error (negative)
  */
  * Convert a mempolicy into a string.
  * Returns the number of characters in buffer (if positive)
  * or an error (negative)
  */

-static inline int mpol_to_str(char *buffer, int maxlen, struct mempolicy *pol)
+int mpol_to_str(char *buffer, int maxlen, struct mempolicy *pol, int no_context)

 {
        char *p = buffer;
        int l;
        nodemask_t nodes;
 {
        char *p = buffer;
        int l;
        nodemask_t nodes;

-       int mode = pol ? pol->policy : MPOL_DEFAULT;
+       unsigned short mode;
+       unsigned short flags = pol ? pol->flags : 0;
+
+       /*
+        * Sanity check:  room for longest mode, flag and some nodes
+        */
+       VM_BUG_ON(maxlen < strlen("interleave") + strlen("relative") + 16);
+
+       if (!pol || pol == &default_policy)
+               mode = MPOL_DEFAULT;
+       else
+               mode = pol->mode;

 
        switch (mode) {
        case MPOL_DEFAULT:
 
        switch (mode) {
        case MPOL_DEFAULT:


@@ -1561,33 +2126,50 @@ static inline int mpol_to_str(char *buffer, int maxlen, struct mempolicy *pol)
 
        case MPOL_PREFERRED:
                nodes_clear(nodes);
 
        case MPOL_PREFERRED:
                nodes_clear(nodes);

-               node_set(pol->v.preferred_node, nodes);
+               if (flags & MPOL_F_LOCAL)
+                       mode = MPOL_LOCAL;      /* pseudo-policy */
+               else
+                       node_set(pol->v.preferred_node, nodes);

                break;
 
        case MPOL_BIND:
                break;
 
        case MPOL_BIND:

-               get_zonemask(pol, &nodes);
-               break;
-
+               /* Fall through */

        case MPOL_INTERLEAVE:
        case MPOL_INTERLEAVE:

-               nodes = pol->v.nodes;
+               if (no_context)
+                       nodes = pol->w.user_nodemask;
+               else
+                       nodes = pol->v.nodes;

                break;
 
        default:
                BUG();
                break;
 
        default:
                BUG();

-               return -EFAULT;

        }
 
        l = strlen(policy_types[mode]);
        }
 
        l = strlen(policy_types[mode]);

-       if (buffer + maxlen < p + l + 1)
-               return -ENOSPC;
+       if (buffer + maxlen < p + l + 1)
+               return -ENOSPC;

 
        strcpy(p, policy_types[mode]);
        p += l;
 
 
        strcpy(p, policy_types[mode]);
        p += l;
 

-       if (!nodes_empty(nodes)) {
+       if (flags & MPOL_MODE_FLAGS) {

                if (buffer + maxlen < p + 2)
                        return -ENOSPC;
                *p++ = '=';
                if (buffer + maxlen < p + 2)
                        return -ENOSPC;
                *p++ = '=';

+
+               /*
+                * Currently, the only defined flags are mutually exclusive
+                */
+               if (flags & MPOL_F_STATIC_NODES)
+                       p += snprintf(p, buffer + maxlen - p, "static");
+               else if (flags & MPOL_F_RELATIVE_NODES)
+                       p += snprintf(p, buffer + maxlen - p, "relative");
+       }
+
+       if (!nodes_empty(nodes)) {
+               if (buffer + maxlen < p + 2)
+                       return -ENOSPC;
+               *p++ = ':';

                p += nodelist_scnprintf(p, buffer + maxlen - p, nodes);
        }
        return p - buffer;
                p += nodelist_scnprintf(p, buffer + maxlen - p, nodes);
        }
        return p - buffer;


@@ -1596,70 +2178,155 @@ static inline int mpol_to_str(char *buffer, int maxlen, struct mempolicy *pol)
 struct numa_maps {
        unsigned long pages;
        unsigned long anon;
 struct numa_maps {
        unsigned long pages;
        unsigned long anon;

-       unsigned long mapped;
+       unsigned long active;
+       unsigned long writeback;

        unsigned long mapcount_max;
        unsigned long mapcount_max;

+       unsigned long dirty;
+       unsigned long swapcache;

        unsigned long node[MAX_NUMNODES];
 };
 
        unsigned long node[MAX_NUMNODES];
 };
 

-static void gather_stats(struct page *page, void *private)
+static void gather_stats(struct page *page, void *private, int pte_dirty)

 {
        struct numa_maps *md = private;
        int count = page_mapcount(page);
 
 {
        struct numa_maps *md = private;
        int count = page_mapcount(page);
 

-       if (count)
-               md->mapped++;
+       md->pages++;
+       if (pte_dirty || PageDirty(page))
+               md->dirty++;

 
 

-       if (count > md->mapcount_max)
-               md->mapcount_max = count;
+       if (PageSwapCache(page))
+               md->swapcache++;

 
 

-       md->pages++;
+       if (PageActive(page))
+               md->active++;
+
+       if (PageWriteback(page))
+               md->writeback++;

 
        if (PageAnon(page))
                md->anon++;
 
 
        if (PageAnon(page))
                md->anon++;
 

+       if (count > md->mapcount_max)
+               md->mapcount_max = count;
+

        md->node[page_to_nid(page)]++;
        md->node[page_to_nid(page)]++;

-       cond_resched();

 }
 
 }
 

+#ifdef CONFIG_HUGETLB_PAGE
+static void check_huge_range(struct vm_area_struct *vma,
+               unsigned long start, unsigned long end,
+               struct numa_maps *md)
+{
+       unsigned long addr;
+       struct page *page;
+       struct hstate *h = hstate_vma(vma);
+       unsigned long sz = huge_page_size(h);
+
+       for (addr = start; addr < end; addr += sz) {
+               pte_t *ptep = huge_pte_offset(vma->vm_mm,
+                                               addr & huge_page_mask(h));
+               pte_t pte;
+
+               if (!ptep)
+                       continue;
+
+               pte = *ptep;
+               if (pte_none(pte))
+                       continue;
+
+               page = pte_page(pte);
+               if (!page)
+                       continue;
+
+               gather_stats(page, md, pte_dirty(*ptep));
+       }
+}
+#else
+static inline void check_huge_range(struct vm_area_struct *vma,
+               unsigned long start, unsigned long end,
+               struct numa_maps *md)
+{
+}
+#endif
+
+/*
+ * Display pages allocated per node and memory policy via /proc.
+ */

 int show_numa_map(struct seq_file *m, void *v)
 {
 int show_numa_map(struct seq_file *m, void *v)
 {

-       struct task_struct *task = m->private;
+       struct proc_maps_private *priv = m->private;

        struct vm_area_struct *vma = v;
        struct numa_maps *md;
        struct vm_area_struct *vma = v;
        struct numa_maps *md;

+       struct file *file = vma->vm_file;
+       struct mm_struct *mm = vma->vm_mm;
+       struct mempolicy *pol;

        int n;
        char buffer[50];
 
        int n;
        char buffer[50];
 

-       if (!vma->vm_mm)
+       if (!mm)

                return 0;
 
        md = kzalloc(sizeof(struct numa_maps), GFP_KERNEL);
        if (!md)
                return 0;
 
                return 0;
 
        md = kzalloc(sizeof(struct numa_maps), GFP_KERNEL);
        if (!md)
                return 0;
 

-       check_pgd_range(vma, vma->vm_start, vma->vm_end,
-                   &node_online_map, MPOL_MF_STATS, md);
+       pol = get_vma_policy(priv->task, vma, vma->vm_start);
+       mpol_to_str(buffer, sizeof(buffer), pol, 0);
+       mpol_cond_put(pol);
+
+       seq_printf(m, "%08lx %s", vma->vm_start, buffer);

 
 

-       if (md->pages) {
-               mpol_to_str(buffer, sizeof(buffer),
-                           get_vma_policy(task, vma, vma->vm_start));
+       if (file) {
+               seq_printf(m, " file=");
+               seq_path(m, &file->f_path, "\n\t= ");
+       } else if (vma->vm_start <= mm->brk && vma->vm_end >= mm->start_brk) {
+               seq_printf(m, " heap");
+       } else if (vma->vm_start <= mm->start_stack &&
+                       vma->vm_end >= mm->start_stack) {
+               seq_printf(m, " stack");
+       }

 
 

-               seq_printf(m, "%08lx %s pages=%lu mapped=%lu maxref=%lu",
-                          vma->vm_start, buffer, md->pages,
-                          md->mapped, md->mapcount_max);
+       if (is_vm_hugetlb_page(vma)) {
+               check_huge_range(vma, vma->vm_start, vma->vm_end, md);
+               seq_printf(m, " huge");
+       } else {
+               check_pgd_range(vma, vma->vm_start, vma->vm_end,
+                       &node_states[N_HIGH_MEMORY], MPOL_MF_STATS, md);
+       }

 
 

-               if (md->anon)
-                       seq_printf(m," anon=%lu",md->anon);
+       if (!md->pages)
+               goto out;

 
 

-               for_each_online_node(n)
-                       if (md->node[n])
-                               seq_printf(m, " N%d=%lu", n, md->node[n]);
+       if (md->anon)
+               seq_printf(m," anon=%lu",md->anon);

 
 

-               seq_putc(m, '\n');
-       }
+       if (md->dirty)
+               seq_printf(m," dirty=%lu",md->dirty);
+
+       if (md->pages != md->anon && md->pages != md->dirty)
+               seq_printf(m, " mapped=%lu", md->pages);
+
+       if (md->mapcount_max > 1)
+               seq_printf(m, " mapmax=%lu", md->mapcount_max);
+
+       if (md->swapcache)
+               seq_printf(m," swapcache=%lu", md->swapcache);
+
+       if (md->active < md->pages && !is_vm_hugetlb_page(vma))
+               seq_printf(m," active=%lu", md->active);
+
+       if (md->writeback)
+               seq_printf(m," writeback=%lu", md->writeback);
+
+       for_each_node_state(n, N_HIGH_MEMORY)
+               if (md->node[n])
+                       seq_printf(m, " N%d=%lu", n, md->node[n]);
+out:
+       seq_putc(m, '\n');

        kfree(md);
 
        if (m->count < m->size)
        kfree(md);
 
        if (m->count < m->size)

-               m->version = (vma != get_gate_vma(task)) ? vma->vm_start : 0;
+               m->version = (vma != priv->tail_vma) ? vma->vm_start : 0;

        return 0;
 }
        return 0;
 }

-