IB/mad: Allow tuning of QP0 and QP1 sizes

MADs are UD and can be dropped if there are no receives posted, so
allow receive queue size to be set with a module parameter in case the
queue needs to be lengthened.  Send side tuning is done for symmetry
with receive.

Signed-off-by: Hal Rosenstock <hal.rosenstock@gmail.com>
Signed-off-by: Roland Dreier <rolandd@cisco.com>

diff --git a/drivers/infiniband/core/mad.c b/drivers/infiniband/core/mad.c
index bc30c00..0f7da24 100644 (file)
--- a/drivers/infiniband/core/mad.c
+++ b/drivers/infiniband/core/mad.c
@@ -2,6 +2,7 @@
  * Copyright (c) 2004-2007 Voltaire, Inc. All rights reserved.
  * Copyright (c) 2005 Intel Corporation.  All rights reserved.
  * Copyright (c) 2005 Mellanox Technologies Ltd.  All rights reserved.
+ * Copyright (c) 2009 HNR Consulting. All rights reserved.
  *
  * This software is available to you under a choice of one of two
  * licenses.  You may choose to be licensed under the terms of the GNU
@@ -45,6 +46,14 @@ MODULE_DESCRIPTION("kernel IB MAD API");
 MODULE_AUTHOR("Hal Rosenstock");
 MODULE_AUTHOR("Sean Hefty");
 
+int mad_sendq_size = IB_MAD_QP_SEND_SIZE;
+int mad_recvq_size = IB_MAD_QP_RECV_SIZE;
+
+module_param_named(send_queue_size, mad_sendq_size, int, 0444);
+MODULE_PARM_DESC(send_queue_size, "Size of send queue in number of work requests");
+module_param_named(recv_queue_size, mad_recvq_size, int, 0444);
+MODULE_PARM_DESC(recv_queue_size, "Size of receive queue in number of work requests");
+
 static struct kmem_cache *ib_mad_cache;
 
 static struct list_head ib_mad_port_list;
@@ -2736,8 +2745,8 @@ static int create_mad_qp(struct ib_mad_qp_info *qp_info,
        qp_init_attr.send_cq = qp_info->port_priv->cq;
        qp_init_attr.recv_cq = qp_info->port_priv->cq;
        qp_init_attr.sq_sig_type = IB_SIGNAL_ALL_WR;
-       qp_init_attr.cap.max_send_wr = IB_MAD_QP_SEND_SIZE;
-       qp_init_attr.cap.max_recv_wr = IB_MAD_QP_RECV_SIZE;
+       qp_init_attr.cap.max_send_wr = mad_sendq_size;
+       qp_init_attr.cap.max_recv_wr = mad_recvq_size;
        qp_init_attr.cap.max_send_sge = IB_MAD_SEND_REQ_MAX_SG;
        qp_init_attr.cap.max_recv_sge = IB_MAD_RECV_REQ_MAX_SG;
        qp_init_attr.qp_type = qp_type;
@@ -2752,8 +2761,8 @@ static int create_mad_qp(struct ib_mad_qp_info *qp_info,
                goto error;
        }
        /* Use minimum queue sizes unless the CQ is resized */
-       qp_info->send_queue.max_active = IB_MAD_QP_SEND_SIZE;
-       qp_info->recv_queue.max_active = IB_MAD_QP_RECV_SIZE;
+       qp_info->send_queue.max_active = mad_sendq_size;
+       qp_info->recv_queue.max_active = mad_recvq_size;
        return 0;
 
 error:
@@ -2792,7 +2801,7 @@ static int ib_mad_port_open(struct ib_device *device,
        init_mad_qp(port_priv, &port_priv->qp_info[0]);
        init_mad_qp(port_priv, &port_priv->qp_info[1]);
 
-       cq_size = (IB_MAD_QP_SEND_SIZE + IB_MAD_QP_RECV_SIZE) * 2;
+       cq_size = (mad_sendq_size + mad_recvq_size) * 2;
        port_priv->cq = ib_create_cq(port_priv->device,
                                     ib_mad_thread_completion_handler,
                                     NULL, port_priv, cq_size, 0);
@@ -2984,6 +2993,12 @@ static int __init ib_mad_init_module(void)
 {
        int ret;
 
+       mad_recvq_size = min(mad_recvq_size, IB_MAD_QP_MAX_SIZE);
+       mad_recvq_size = max(mad_recvq_size, IB_MAD_QP_MIN_SIZE);
+
+       mad_sendq_size = min(mad_sendq_size, IB_MAD_QP_MAX_SIZE);
+       mad_sendq_size = max(mad_sendq_size, IB_MAD_QP_MIN_SIZE);
+
        spin_lock_init(&ib_mad_port_list_lock);
 
        ib_mad_cache = kmem_cache_create("ib_mad",

diff --git a/drivers/infiniband/core/mad_priv.h b/drivers/infiniband/core/mad_priv.h
index 05ce331..9430ab4 100644 (file)
--- a/drivers/infiniband/core/mad_priv.h
+++ b/drivers/infiniband/core/mad_priv.h
@@ -2,6 +2,7 @@
  * Copyright (c) 2004, 2005, Voltaire, Inc. All rights reserved.
  * Copyright (c) 2005 Intel Corporation. All rights reserved.
  * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved.
+ * Copyright (c) 2009 HNR Consulting. All rights reserved.
  *
  * This software is available to you under a choice of one of two
  * licenses.  You may choose to be licensed under the terms of the GNU
@@ -49,6 +50,8 @@
 /* QP and CQ parameters */
 #define IB_MAD_QP_SEND_SIZE    128
 #define IB_MAD_QP_RECV_SIZE    512
+#define IB_MAD_QP_MIN_SIZE     64
+#define IB_MAD_QP_MAX_SIZE     8192
 #define IB_MAD_SEND_REQ_MAX_SG 2
 #define IB_MAD_RECV_REQ_MAX_SG 1