SGI UV: TLB shootdown using broadcast assist unit

TLB shootdown for SGI UV.

v5: 6/12 corrections/improvements per Ingo's second review

Signed-off-by: Cliff Wickman <cpw@sgi.com>
Signed-off-by: Ingo Molnar <mingo@elte.hu>

diff --git a/arch/x86/kernel/tlb_uv.c b/arch/x86/kernel/tlb_uv.c
index f7bc6a6..d8705e9 100644 (file)
--- a/arch/x86/kernel/tlb_uv.c
+++ b/arch/x86/kernel/tlb_uv.c
@@ -25,15 +25,8 @@ static int uv_bau_retry_limit __read_mostly;
 static int uv_nshift __read_mostly; /* position of pnode (which is nasid>>1) */
 static unsigned long uv_mmask __read_mostly;
 
-char *status_table[] = {
-       "IDLE",
-       "ACTIVE",
-       "DESTINATION TIMEOUT",
-       "SOURCE TIMEOUT"
-};
-
-DEFINE_PER_CPU(struct ptc_stats, ptcstats);
-DEFINE_PER_CPU(struct bau_control, bau_control);
+static DEFINE_PER_CPU(struct ptc_stats, ptcstats);
+static DEFINE_PER_CPU(struct bau_control, bau_control);
 
 /*
  * Free a software acknowledge hardware resource by clearing its Pending
@@ -55,7 +48,6 @@ static void uv_reply_to_message(int resource,
        if (msp)
                msp->seen_by.bits = 0;
        uv_write_local_mmr(UVH_LB_BAU_INTD_SOFTWARE_ACKNOWLEDGE_ALIAS, dw);
-       return;
 }
 
 /*
@@ -73,7 +65,7 @@ static void uv_bau_process_message(struct bau_payload_queue_entry *msg,
        cpu = uv_blade_processor_id();
        msg->number_of_cpus =
            uv_blade_nr_online_cpus(uv_node_to_blade_id(numa_node_id()));
-       this_cpu_mask = (unsigned long)1 << cpu;
+       this_cpu_mask = 1UL << cpu;
        if (msp->seen_by.bits & this_cpu_mask)
                return;
        atomic_or_long(&msp->seen_by.bits, this_cpu_mask);
@@ -94,53 +86,60 @@ static void uv_bau_process_message(struct bau_payload_queue_entry *msg,
        atomic_inc_short(&msg->acknowledge_count);
        if (msg->number_of_cpus == msg->acknowledge_count)
                uv_reply_to_message(sw_ack_slot, msg, msp);
-       return;
 }
 
 /*
- * Examine the payload queue on all the distribution nodes to see
+ * Examine the payload queue on one distribution node to see
  * which messages have not been seen, and which cpu(s) have not seen them.
  *
  * Returns the number of cpu's that have not responded.
  */
-static int uv_examine_destinations(struct bau_target_nodemask *distribution)
+static int uv_examine_destination(struct bau_control *bau_tablesp, int sender)
 {
-       int sender;
        int i;
        int j;
-       int k;
        int count = 0;
-       struct bau_control *bau_tablesp;
        struct bau_payload_queue_entry *msg;
        struct bau_msg_status *msp;
 
+       for (msg = bau_tablesp->va_queue_first, i = 0; i < DEST_Q_SIZE;
+            msg++, i++) {
+               if ((msg->sending_cpu == sender) && (!msg->replied_to)) {
+                       msp = bau_tablesp->msg_statuses + i;
+                       printk(KERN_DEBUG
+                              "blade %d: address:%#lx %d of %d, not cpu(s): ",
+                              i, msg->address, msg->acknowledge_count,
+                              msg->number_of_cpus);
+                       for (j = 0; j < msg->number_of_cpus; j++) {
+                               if (!((long)1 << j & msp-> seen_by.bits)) {
+                                       count++;
+                                       printk("%d ", j);
+                               }
+                       }
+                       printk("\n");
+               }
+       }
+       return count;
+}
+
+/*
+ * Examine the payload queue on all the distribution nodes to see
+ * which messages have not been seen, and which cpu(s) have not seen them.
+ *
+ * Returns the number of cpu's that have not responded.
+ */
+static int uv_examine_destinations(struct bau_target_nodemask *distribution)
+{
+       int sender;
+       int i;
+       int count = 0;
+
        sender = smp_processor_id();
        for (i = 0; i < (sizeof(struct bau_target_nodemask) * BITSPERBYTE);
             i++) {
                if (!bau_node_isset(i, distribution))
                        continue;
-               bau_tablesp = uv_bau_table_bases[i];
-               for (msg = bau_tablesp->va_queue_first, j = 0;
-                    j < DESTINATION_PAYLOAD_QUEUE_SIZE; msg++, j++) {
-                       if ((msg->sending_cpu == sender) &&
-                           (!msg->replied_to)) {
-                               msp = bau_tablesp->msg_statuses + j;
-                               printk(KERN_DEBUG
-                               "blade %d: address:%#lx %d of %d, not cpu(s): ",
-                                      i, msg->address,
-                                      msg->acknowledge_count,
-                                      msg->number_of_cpus);
-                               for (k = 0; k < msg->number_of_cpus;
-                                    k++) {
-                                       if (!((long)1 << k & msp->
-                                             seen_by.bits)) {
-                                               count++;
-                                               printk("%d ", k);
-                                       }
-                               }
-                               printk("\n");
-                       }
-               }
+               count += uv_examine_destination(uv_bau_table_bases[i], sender);
        }
        return count;
 }
@@ -150,7 +149,7 @@ static int uv_examine_destinations(struct bau_target_nodemask *distribution)
  *
  * return COMPLETE, RETRY or GIVEUP
  */
-static int uv_wait_completion(struct bau_activation_descriptor *bau_desc,
+static int uv_wait_completion(struct bau_desc *bau_desc,
                              unsigned long mmr_offset, int right_shift)
 {
        int exams = 0;
@@ -213,8 +212,8 @@ static int uv_wait_completion(struct bau_activation_descriptor *bau_desc,
  * Returns 0 if some remote flushing remains to be done. The mask is left
  * unchanged.
  */
-int uv_flush_send_and_wait(int cpu, int this_blade,
-       struct bau_activation_descriptor *bau_desc, cpumask_t *cpumaskp)
+int uv_flush_send_and_wait(int cpu, int this_blade, struct bau_desc *bau_desc,
+                          cpumask_t *cpumaskp)
 {
        int completion_status = 0;
        int right_shift;
@@ -237,8 +236,8 @@ int uv_flush_send_and_wait(int cpu, int this_blade,
        time1 = get_cycles();
        do {
                tries++;
-               index = ((unsigned long)
-                       1 << UVH_LB_BAU_SB_ACTIVATION_CONTROL_PUSH_SHFT) | cpu;
+               index = (1UL << UVH_LB_BAU_SB_ACTIVATION_CONTROL_PUSH_SHFT) |
+                       cpu;
                uv_write_local_mmr(UVH_LB_BAU_SB_ACTIVATION_CONTROL, index);
                completion_status = uv_wait_completion(bau_desc, mmr_offset,
                                        right_shift);
@@ -303,7 +302,7 @@ int uv_flush_tlb_others(cpumask_t *cpumaskp, struct mm_struct *mm,
        int cpu;
        int this_blade;
        int locals = 0;
-       struct bau_activation_descriptor *bau_desc;
+       struct bau_desc *bau_desc;
 
        cpu = uv_blade_processor_id();
        this_blade = uv_numa_blade_id();
@@ -315,8 +314,7 @@ int uv_flush_tlb_others(cpumask_t *cpumaskp, struct mm_struct *mm,
        i = 0;
        for_each_cpu_mask(bit, *cpumaskp) {
                blade = uv_cpu_to_blade_id(bit);
-               if (blade > (UV_DISTRIBUTION_SIZE - 1))
-                       BUG();
+               BUG_ON(blade > (UV_DISTRIBUTION_SIZE - 1));
                if (blade == this_blade) {
                        locals++;
                        continue;
@@ -360,6 +358,8 @@ void uv_bau_message_interrupt(struct pt_regs *regs)
 {
        struct bau_payload_queue_entry *pqp;
        struct bau_payload_queue_entry *msg;
+       struct bau_payload_queue_entry *va_queue_first;
+       struct bau_payload_queue_entry *va_queue_last;
        struct pt_regs *old_regs = set_irq_regs(regs);
        cycles_t time1, time2;
        int msg_slot;
@@ -376,7 +376,8 @@ void uv_bau_message_interrupt(struct pt_regs *regs)
 
        local_pnode = uv_blade_to_pnode(uv_numa_blade_id());
 
-       pqp = __get_cpu_var(bau_control).va_queue_first;
+       pqp = va_queue_first = __get_cpu_var(bau_control).va_queue_first;
+       va_queue_last = __get_cpu_var(bau_control).va_queue_last;
        msg = __get_cpu_var(bau_control).bau_msg_head;
        while (msg->sw_ack_vector) {
                count++;
@@ -387,8 +388,8 @@ void uv_bau_message_interrupt(struct pt_regs *regs)
                uv_bau_process_message(msg, msg_slot, sw_ack_slot);
 
                msg++;
-               if (msg > __get_cpu_var(bau_control).va_queue_last)
-                       msg = __get_cpu_var(bau_control).va_queue_first;
+               if (msg > va_queue_last)
+                       msg = va_queue_first;
                __get_cpu_var(bau_control).bau_msg_head = msg;
        }
        if (!count)
@@ -401,7 +402,6 @@ void uv_bau_message_interrupt(struct pt_regs *regs)
 
        irq_exit();
        set_irq_regs(old_regs);
-       return;
 }
 
 static void uv_enable_timeouts(void)
@@ -423,7 +423,6 @@ static void uv_enable_timeouts(void)
                pnode = uv_blade_to_pnode(blade);
                cur_cpu += uv_blade_nr_possible_cpus(i);
        }
-       return;
 }
 
 static void *uv_ptc_seq_start(struct seq_file *file, loff_t *offset)
@@ -535,10 +534,10 @@ static ssize_t uv_ptc_proc_write(struct file *file, const char __user *user,
 }
 
 static const struct seq_operations uv_ptc_seq_ops = {
-       .start  = uv_ptc_seq_start,
-       .next   = uv_ptc_seq_next,
-       .stop   = uv_ptc_seq_stop,
-       .show   = uv_ptc_seq_show
+       .start          = uv_ptc_seq_start,
+       .next           = uv_ptc_seq_next,
+       .stop           = uv_ptc_seq_stop,
+       .show           = uv_ptc_seq_show
 };
 
 static int uv_ptc_proc_open(struct inode *inode, struct file *file)
@@ -568,6 +567,7 @@ static int __init uv_ptc_init(void)
        if (!proc_uv_ptc) {
                printk(KERN_ERR "unable to create %s proc entry\n",
                       UV_PTC_BASENAME);
+               remove_proc_entry("sgi_uv", NULL);
                return -EINVAL;
        }
        proc_uv_ptc->proc_fops = &proc_uv_ptc_operations;
@@ -582,33 +582,26 @@ static struct bau_control * __init uv_table_bases_init(int blade, int node)
        int i;
        int *ip;
        struct bau_msg_status *msp;
-       struct bau_control *bau_tablesp;
+       struct bau_control *bau_tabp;
 
-       bau_tablesp =
+       bau_tabp =
            kmalloc_node(sizeof(struct bau_control), GFP_KERNEL, node);
-       if (!bau_tablesp)
-               BUG();
-       bau_tablesp->msg_statuses =
+       BUG_ON(!bau_tabp);
+       bau_tabp->msg_statuses =
            kmalloc_node(sizeof(struct bau_msg_status) *
-                        DESTINATION_PAYLOAD_QUEUE_SIZE, GFP_KERNEL, node);
-       if (!bau_tablesp->msg_statuses)
-               BUG();
-       for (i = 0, msp = bau_tablesp->msg_statuses;
-            i < DESTINATION_PAYLOAD_QUEUE_SIZE; i++, msp++) {
+                        DEST_Q_SIZE, GFP_KERNEL, node);
+       BUG_ON(!bau_tabp->msg_statuses);
+       for (i = 0, msp = bau_tabp->msg_statuses; i < DEST_Q_SIZE; i++, msp++)
                bau_cpubits_clear(&msp->seen_by, (int)
                                  uv_blade_nr_possible_cpus(blade));
-       }
-       bau_tablesp->watching =
-           kmalloc_node(sizeof(int) * DESTINATION_NUM_RESOURCES,
-                        GFP_KERNEL, node);
-       if (!bau_tablesp->watching)
-               BUG();
-       for (i = 0, ip = bau_tablesp->watching;
-            i < DESTINATION_PAYLOAD_QUEUE_SIZE; i++, ip++) {
+       bau_tabp->watching =
+           kmalloc_node(sizeof(int) * DEST_NUM_RESOURCES, GFP_KERNEL, node);
+       BUG_ON(!bau_tabp->watching);
+       for (i = 0, ip = bau_tabp->watching; i < DEST_Q_SIZE; i++, ip++) {
                *ip = 0;
        }
-       uv_bau_table_bases[blade] = bau_tablesp;
-       return bau_tablesp;
+       uv_bau_table_bases[blade] = bau_tabp;
+       return bau_tabsp;
 }
 
 /*
@@ -616,7 +609,7 @@ static struct bau_control * __init uv_table_bases_init(int blade, int node)
  */
 static void __init uv_table_bases_finish(int blade, int node, int cur_cpu,
                                  struct bau_control *bau_tablesp,
-                                 struct bau_activation_descriptor *adp)
+                                 struct bau_desc *adp)
 {
        int i;
        struct bau_control *bcp;
@@ -636,7 +629,7 @@ static void __init uv_table_bases_finish(int blade, int node, int cur_cpu,
 /*
  * initialize the sending side's sending buffers
  */
-static struct bau_activation_descriptor * __init
+static struct bau_desc * __init
 uv_activation_descriptor_init(int node, int pnode)
 {
        int i;
@@ -644,13 +637,12 @@ uv_activation_descriptor_init(int node, int pnode)
        unsigned long m;
        unsigned long n;
        unsigned long mmr_image;
-       struct bau_activation_descriptor *adp;
-       struct bau_activation_descriptor *ad2;
+       struct bau_desc *adp;
+       struct bau_desc *ad2;
 
-       adp = (struct bau_activation_descriptor *)
+       adp = (struct bau_desc *)
            kmalloc_node(16384, GFP_KERNEL, node);
-       if (!adp)
-               BUG();
+       BUG_ON(!adp);
        pa = __pa((unsigned long)adp);
        n = pa >> uv_nshift;
        m = pa & uv_mmask;
@@ -660,7 +652,7 @@ uv_activation_descriptor_init(int node, int pnode)
                                      UVH_LB_BAU_SB_DESCRIPTOR_BASE,
                                      (n << UV_DESC_BASE_PNODE_SHIFT | m));
        for (i = 0, ad2 = adp; i < UV_ACTIVATION_DESCRIPTOR_SIZE; i++, ad2++) {
-               memset(ad2, 0, sizeof(struct bau_activation_descriptor));
+               memset(ad2, 0, sizeof(struct bau_desc));
                ad2->header.sw_ack_flag = 1;
                ad2->header.base_dest_nodeid =
                    uv_blade_to_pnode(uv_cpu_to_blade_id(0));
@@ -683,12 +675,10 @@ static struct bau_payload_queue_entry * __init uv_payload_queue_init(int node,
        char *cp;
        struct bau_payload_queue_entry *pqp;
 
-       pqp = (struct bau_payload_queue_entry *)
-           kmalloc_node((DESTINATION_PAYLOAD_QUEUE_SIZE + 1) *
-                        sizeof(struct bau_payload_queue_entry),
-                        GFP_KERNEL, node);
-       if (!pqp)
-               BUG();
+       pqp = (struct bau_payload_queue_entry *) kmalloc_node(
+               (DEST_Q_SIZE + 1) * sizeof(struct bau_payload_queue_entry),
+               GFP_KERNEL, node);
+       BUG_ON(!pqp);
        cp = (char *)pqp + 31;
        pqp = (struct bau_payload_queue_entry *)(((unsigned long)cp >> 5) << 5);
        bau_tablesp->va_queue_first = pqp;
@@ -699,13 +689,11 @@ static struct bau_payload_queue_entry * __init uv_payload_queue_init(int node,
                              uv_physnodeaddr(pqp));
        uv_write_global_mmr64(pnode, UVH_LB_BAU_INTD_PAYLOAD_QUEUE_TAIL,
                              uv_physnodeaddr(pqp));
-       bau_tablesp->va_queue_last =
-           pqp + (DESTINATION_PAYLOAD_QUEUE_SIZE - 1);
+       bau_tablesp->va_queue_last = pqp + (DEST_Q_SIZE - 1);
        uv_write_global_mmr64(pnode, UVH_LB_BAU_INTD_PAYLOAD_QUEUE_LAST,
                              (unsigned long)
                              uv_physnodeaddr(bau_tablesp->va_queue_last));
-       memset(pqp, 0, sizeof(struct bau_payload_queue_entry) *
-              DESTINATION_PAYLOAD_QUEUE_SIZE);
+       memset(pqp, 0, sizeof(struct bau_payload_queue_entry) * DEST_Q_SIZE);
        return pqp;
 }
 
@@ -717,7 +705,7 @@ static int __init uv_init_blade(int blade, int node, int cur_cpu)
        int pnode;
        unsigned long pa;
        unsigned long apicid;
-       struct bau_activation_descriptor *adp;
+       struct bau_desc *adp;
        struct bau_payload_queue_entry *pqp;
        struct bau_control *bau_tablesp;
 
@@ -755,7 +743,7 @@ static int __init uv_bau_init(void)
 
        uv_bau_retry_limit = 1;
        uv_nshift = uv_hub_info->n_val;
-       uv_mmask = ((unsigned long)1 << uv_hub_info->n_val) - 1;
+       uv_mmask = (1UL << uv_hub_info->n_val) - 1;
        nblades = 0;
        last_blade = -1;
        for_each_online_node(node) {
@@ -767,8 +755,7 @@ static int __init uv_bau_init(void)
        }
        uv_bau_table_bases = (struct bau_control **)
            kmalloc(nblades * sizeof(struct bau_control *), GFP_KERNEL);
-       if (!uv_bau_table_bases)
-               BUG();
+       BUG_ON(!uv_bau_table_bases);
        last_blade = -1;
        for_each_online_node(node) {
                blade = uv_node_to_blade_id(node);

diff --git a/include/asm-x86/uv/uv_bau.h b/include/asm-x86/uv/uv_bau.h
index e52fec8..91ac0df 100644 (file)
--- a/include/asm-x86/uv/uv_bau.h
+++ b/include/asm-x86/uv/uv_bau.h
@@ -54,25 +54,25 @@
 /*
  * bits in UVH_LB_BAU_SB_ACTIVATION_STATUS_0/1
  */
-#define DESC_STATUS_IDLE                0
-#define DESC_STATUS_ACTIVE              1
-#define DESC_STATUS_DESTINATION_TIMEOUT 2
-#define DESC_STATUS_SOURCE_TIMEOUT      3
+#define DESC_STATUS_IDLE               0
+#define DESC_STATUS_ACTIVE             1
+#define DESC_STATUS_DESTINATION_TIMEOUT        2
+#define DESC_STATUS_SOURCE_TIMEOUT     3
 
 /*
  * source side threshholds at which message retries print a warning
  */
-#define SOURCE_TIMEOUT_LIMIT            20
-#define DESTINATION_TIMEOUT_LIMIT       20
+#define SOURCE_TIMEOUT_LIMIT           20
+#define DESTINATION_TIMEOUT_LIMIT      20
 
 /*
  * number of entries in the destination side payload queue
  */
-#define DESTINATION_PAYLOAD_QUEUE_SIZE  17
+#define DEST_Q_SIZE                    17
 /*
  * number of destination side software ack resources
  */
-#define DESTINATION_NUM_RESOURCES       8
+#define DEST_NUM_RESOURCES             8
 #define MAX_CPUS_PER_NODE              32
 /*
  * completion statuses for sending a TLB flush message
@@ -197,10 +197,11 @@ struct bau_msg_header {
 };
 
 /*
+ * The activation descriptor:
  * The format of the message to send, plus all accompanying control
  * Should be 64 bytes
  */
-struct bau_activation_descriptor {
+struct bau_desc {
        struct bau_target_nodemask distribution;
        /*
         * message template, consisting of header and payload:
@@ -281,7 +282,7 @@ struct bau_sw_ack_status {
  * one on every node and per-cpu; to locate the software tables
  */
 struct bau_control {
-       struct bau_activation_descriptor *descriptor_base;
+       struct bau_desc *descriptor_base;
        struct bau_payload_queue_entry *bau_msg_head;
        struct bau_payload_queue_entry *va_queue_first;
        struct bau_payload_queue_entry *va_queue_last;