IB/sa: Require SA registration

Require users to register with SA module, to prevent the sa_query
module text from going away while an SA query callback is still
running.  Update all in-tree users for the new interface.

Signed-off-by: Michael S. Tsirkin <mst@mellanox.co.il>
Signed-off-by: Sean Hefty <sean.hefty@intel.com>
Signed-off-by: Roland Dreier <rolandd@cisco.com>

diff --git a/drivers/infiniband/core/cma.c b/drivers/infiniband/core/cma.c
index 488fa1d..1178bd4 100644 (file)
--- a/drivers/infiniband/core/cma.c
+++ b/drivers/infiniband/core/cma.c
@@ -62,6 +62,7 @@ static struct ib_client cma_client = {
        .remove = cma_remove_one
 };
 
+static struct ib_sa_client sa_client;
 static LIST_HEAD(dev_list);
 static LIST_HEAD(listen_any_list);
 static DEFINE_MUTEX(lock);
@@ -1323,7 +1324,7 @@ static int cma_query_ib_route(struct rdma_id_private *id_priv, int timeout_ms,
        path_rec.pkey = cpu_to_be16(ib_addr_get_pkey(addr));
        path_rec.numb_path = 1;
 
-       id_priv->query_id = ib_sa_path_rec_get(id_priv->id.device,
+       id_priv->query_id = ib_sa_path_rec_get(&sa_client, id_priv->id.device,
                                id_priv->id.port_num, &path_rec,
                                IB_SA_PATH_REC_DGID | IB_SA_PATH_REC_SGID |
                                IB_SA_PATH_REC_PKEY | IB_SA_PATH_REC_NUMB_PATH,
@@ -2199,12 +2200,15 @@ static int cma_init(void)
        if (!cma_wq)
                return -ENOMEM;
 
+       ib_sa_register_client(&sa_client);
+
        ret = ib_register_client(&cma_client);
        if (ret)
                goto err;
        return 0;
 
 err:
+       ib_sa_unregister_client(&sa_client);
        destroy_workqueue(cma_wq);
        return ret;
 }
@@ -2212,6 +2216,7 @@ err:
 static void cma_cleanup(void)
 {
        ib_unregister_client(&cma_client);
+       ib_sa_unregister_client(&sa_client);
        destroy_workqueue(cma_wq);
        idr_destroy(&sdp_ps);
        idr_destroy(&tcp_ps);

diff --git a/drivers/infiniband/core/sa_query.c b/drivers/infiniband/core/sa_query.c
index ca8760a..1706d3c 100644 (file)
--- a/drivers/infiniband/core/sa_query.c
+++ b/drivers/infiniband/core/sa_query.c
@@ -1,6 +1,7 @@
 /*
  * Copyright (c) 2004 Topspin Communications.  All rights reserved.
  * Copyright (c) 2005 Voltaire, Inc.  All rights reserved.
+ * Copyright (c) 2006 Intel Corporation.  All rights reserved.
  *
  * This software is available to you under a choice of one of two
  * licenses.  You may choose to be licensed under the terms of the GNU
@@ -75,6 +76,7 @@ struct ib_sa_device {
 struct ib_sa_query {
        void (*callback)(struct ib_sa_query *, int, struct ib_sa_mad *);
        void (*release)(struct ib_sa_query *);
+       struct ib_sa_client    *client;
        struct ib_sa_port      *port;
        struct ib_mad_send_buf *mad_buf;
        struct ib_sa_sm_ah     *sm_ah;
@@ -415,6 +417,31 @@ static void ib_sa_event(struct ib_event_handler *handler, struct ib_event *event
        }
 }
 
+void ib_sa_register_client(struct ib_sa_client *client)
+{
+       atomic_set(&client->users, 1);
+       init_completion(&client->comp);
+}
+EXPORT_SYMBOL(ib_sa_register_client);
+
+static inline void ib_sa_client_get(struct ib_sa_client *client)
+{
+       atomic_inc(&client->users);
+}
+
+static inline void ib_sa_client_put(struct ib_sa_client *client)
+{
+       if (atomic_dec_and_test(&client->users))
+               complete(&client->comp);
+}
+
+void ib_sa_unregister_client(struct ib_sa_client *client)
+{
+       ib_sa_client_put(client);
+       wait_for_completion(&client->comp);
+}
+EXPORT_SYMBOL(ib_sa_unregister_client);
+
 /**
  * ib_sa_cancel_query - try to cancel an SA query
  * @id:ID of query to cancel
@@ -557,6 +584,7 @@ static void ib_sa_path_rec_release(struct ib_sa_query *sa_query)
 
 /**
  * ib_sa_path_rec_get - Start a Path get query
+ * @client:SA client
  * @device:device to send query on
  * @port_num: port number to send query on
  * @rec:Path Record to send in query
@@ -579,7 +607,8 @@ static void ib_sa_path_rec_release(struct ib_sa_query *sa_query)
  * error code.  Otherwise it is a query ID that can be used to cancel
  * the query.
  */
-int ib_sa_path_rec_get(struct ib_device *device, u8 port_num,
+int ib_sa_path_rec_get(struct ib_sa_client *client,
+                      struct ib_device *device, u8 port_num,
                       struct ib_sa_path_rec *rec,
                       ib_sa_comp_mask comp_mask,
                       int timeout_ms, gfp_t gfp_mask,
@@ -614,8 +643,10 @@ int ib_sa_path_rec_get(struct ib_device *device, u8 port_num,
                goto err1;
        }
 
-       query->callback = callback;
-       query->context  = context;
+       ib_sa_client_get(client);
+       query->sa_query.client = client;
+       query->callback        = callback;
+       query->context         = context;
 
        mad = query->sa_query.mad_buf->mad;
        init_mad(mad, agent);
@@ -639,6 +670,7 @@ int ib_sa_path_rec_get(struct ib_device *device, u8 port_num,
 
 err2:
        *sa_query = NULL;
+       ib_sa_client_put(query->sa_query.client);
        ib_free_send_mad(query->sa_query.mad_buf);
 
 err1:
@@ -671,6 +703,7 @@ static void ib_sa_service_rec_release(struct ib_sa_query *sa_query)
 
 /**
  * ib_sa_service_rec_query - Start Service Record operation
+ * @client:SA client
  * @device:device to send request on
  * @port_num: port number to send request on
  * @method:SA method - should be get, set, or delete
@@ -695,7 +728,8 @@ static void ib_sa_service_rec_release(struct ib_sa_query *sa_query)
  * error code.  Otherwise it is a request ID that can be used to cancel
  * the query.
  */
-int ib_sa_service_rec_query(struct ib_device *device, u8 port_num, u8 method,
+int ib_sa_service_rec_query(struct ib_sa_client *client,
+                           struct ib_device *device, u8 port_num, u8 method,
                            struct ib_sa_service_rec *rec,
                            ib_sa_comp_mask comp_mask,
                            int timeout_ms, gfp_t gfp_mask,
@@ -735,8 +769,10 @@ int ib_sa_service_rec_query(struct ib_device *device, u8 port_num, u8 method,
                goto err1;
        }
 
-       query->callback = callback;
-       query->context  = context;
+       ib_sa_client_get(client);
+       query->sa_query.client = client;
+       query->callback        = callback;
+       query->context         = context;
 
        mad = query->sa_query.mad_buf->mad;
        init_mad(mad, agent);
@@ -761,6 +797,7 @@ int ib_sa_service_rec_query(struct ib_device *device, u8 port_num, u8 method,
 
 err2:
        *sa_query = NULL;
+       ib_sa_client_put(query->sa_query.client);
        ib_free_send_mad(query->sa_query.mad_buf);
 
 err1:
@@ -791,7 +828,8 @@ static void ib_sa_mcmember_rec_release(struct ib_sa_query *sa_query)
        kfree(container_of(sa_query, struct ib_sa_mcmember_query, sa_query));
 }
 
-int ib_sa_mcmember_rec_query(struct ib_device *device, u8 port_num,
+int ib_sa_mcmember_rec_query(struct ib_sa_client *client,
+                            struct ib_device *device, u8 port_num,
                             u8 method,
                             struct ib_sa_mcmember_rec *rec,
                             ib_sa_comp_mask comp_mask,
@@ -827,8 +865,10 @@ int ib_sa_mcmember_rec_query(struct ib_device *device, u8 port_num,
                goto err1;
        }
 
-       query->callback = callback;
-       query->context  = context;
+       ib_sa_client_get(client);
+       query->sa_query.client = client;
+       query->callback        = callback;
+       query->context         = context;
 
        mad = query->sa_query.mad_buf->mad;
        init_mad(mad, agent);
@@ -853,6 +893,7 @@ int ib_sa_mcmember_rec_query(struct ib_device *device, u8 port_num,
 
 err2:
        *sa_query = NULL;
+       ib_sa_client_put(query->sa_query.client);
        ib_free_send_mad(query->sa_query.mad_buf);
 
 err1:
@@ -889,6 +930,7 @@ static void send_handler(struct ib_mad_agent *agent,
 
        ib_free_send_mad(mad_send_wc->send_buf);
        kref_put(&query->sm_ah->ref, free_sm_ah);
+       ib_sa_client_put(query->client);
        query->release(query);
 }
 

diff --git a/drivers/infiniband/ulp/ipoib/ipoib.h b/drivers/infiniband/ulp/ipoib/ipoib.h
index 474aa21..0b8a79d 100644 (file)
--- a/drivers/infiniband/ulp/ipoib/ipoib.h
+++ b/drivers/infiniband/ulp/ipoib/ipoib.h
@@ -336,6 +336,8 @@ static inline void ipoib_unregister_debugfs(void) { }
 extern int ipoib_sendq_size;
 extern int ipoib_recvq_size;
 
+extern struct ib_sa_client ipoib_sa_client;
+
 #ifdef CONFIG_INFINIBAND_IPOIB_DEBUG
 extern int ipoib_debug_level;
 

diff --git a/drivers/infiniband/ulp/ipoib/ipoib_main.c b/drivers/infiniband/ulp/ipoib/ipoib_main.c
index e9a7659..ae3a498 100644 (file)
--- a/drivers/infiniband/ulp/ipoib/ipoib_main.c
+++ b/drivers/infiniband/ulp/ipoib/ipoib_main.c
@@ -82,6 +82,8 @@ static const u8 ipv4_bcast_addr[] = {
 
 struct workqueue_struct *ipoib_workqueue;
 
+struct ib_sa_client ipoib_sa_client;
+
 static void ipoib_add_one(struct ib_device *device);
 static void ipoib_remove_one(struct ib_device *device);
 
@@ -463,7 +465,7 @@ static int path_rec_start(struct net_device *dev,
        init_completion(&path->done);
 
        path->query_id =
-               ib_sa_path_rec_get(priv->ca, priv->port,
+               ib_sa_path_rec_get(&ipoib_sa_client, priv->ca, priv->port,
                                   &path->pathrec,
                                   IB_SA_PATH_REC_DGID          |
                                   IB_SA_PATH_REC_SGID          |
@@ -1191,13 +1193,16 @@ static int __init ipoib_init_module(void)
                goto err_fs;
        }
 
+       ib_sa_register_client(&ipoib_sa_client);
+
        ret = ib_register_client(&ipoib_client);
        if (ret)
-               goto err_wq;
+               goto err_sa;
 
        return 0;
 
-err_wq:
+err_sa:
+       ib_sa_unregister_client(&ipoib_sa_client);
        destroy_workqueue(ipoib_workqueue);
 
 err_fs:
@@ -1209,6 +1214,7 @@ err_fs:
 static void __exit ipoib_cleanup_module(void)
 {
        ib_unregister_client(&ipoib_client);
+       ib_sa_unregister_client(&ipoib_sa_client);
        ipoib_unregister_debugfs();
        destroy_workqueue(ipoib_workqueue);
 }

diff --git a/drivers/infiniband/ulp/ipoib/ipoib_multicast.c b/drivers/infiniband/ulp/ipoib/ipoib_multicast.c
index 60b09f5..fb3e487 100644 (file)
--- a/drivers/infiniband/ulp/ipoib/ipoib_multicast.c
+++ b/drivers/infiniband/ulp/ipoib/ipoib_multicast.c
@@ -361,7 +361,7 @@ static int ipoib_mcast_sendonly_join(struct ipoib_mcast *mcast)
 
        init_completion(&mcast->done);
 
-       ret = ib_sa_mcmember_rec_set(priv->ca, priv->port, &rec,
+       ret = ib_sa_mcmember_rec_set(&ipoib_sa_client, priv->ca, priv->port, &rec,
                                     IB_SA_MCMEMBER_REC_MGID            |
                                     IB_SA_MCMEMBER_REC_PORT_GID        |
                                     IB_SA_MCMEMBER_REC_PKEY            |
@@ -485,9 +485,9 @@ static void ipoib_mcast_join(struct net_device *dev, struct ipoib_mcast *mcast,
 
        init_completion(&mcast->done);
 
-       ret = ib_sa_mcmember_rec_set(priv->ca, priv->port, &rec, comp_mask,
-                                    mcast->backoff * 1000, GFP_ATOMIC,
-                                    ipoib_mcast_join_complete,
+       ret = ib_sa_mcmember_rec_set(&ipoib_sa_client, priv->ca, priv->port,
+                                    &rec, comp_mask, mcast->backoff * 1000,
+                                    GFP_ATOMIC, ipoib_mcast_join_complete,
                                     mcast, &mcast->query);
 
        if (ret < 0) {
@@ -528,7 +528,7 @@ void ipoib_mcast_join_task(void *dev_ptr)
                        priv->local_rate = attr.active_speed *
                                ib_width_enum_to_int(attr.active_width);
                } else
-                       ipoib_warn(priv, "ib_query_port failed\n");
+               ipoib_warn(priv, "ib_query_port failed\n");
        }
 
        if (!priv->broadcast) {
@@ -681,7 +681,7 @@ static int ipoib_mcast_leave(struct net_device *dev, struct ipoib_mcast *mcast)
         * Just make one shot at leaving and don't wait for a reply;
         * if we fail, too bad.
         */
-       ret = ib_sa_mcmember_rec_delete(priv->ca, priv->port, &rec,
+       ret = ib_sa_mcmember_rec_delete(&ipoib_sa_client, priv->ca, priv->port, &rec,
                                        IB_SA_MCMEMBER_REC_MGID         |
                                        IB_SA_MCMEMBER_REC_PORT_GID     |
                                        IB_SA_MCMEMBER_REC_PKEY         |

diff --git a/drivers/infiniband/ulp/srp/ib_srp.c b/drivers/infiniband/ulp/srp/ib_srp.c
index feb1fcd..44b9e5b 100644 (file)
--- a/drivers/infiniband/ulp/srp/ib_srp.c
+++ b/drivers/infiniband/ulp/srp/ib_srp.c
@@ -96,6 +96,8 @@ static struct ib_client srp_client = {
        .remove = srp_remove_one
 };
 
+static struct ib_sa_client srp_sa_client;
+
 static inline struct srp_target_port *host_to_target(struct Scsi_Host *host)
 {
        return (struct srp_target_port *) host->hostdata;
@@ -267,7 +269,8 @@ static int srp_lookup_path(struct srp_target_port *target)
 
        init_completion(&target->done);
 
-       target->path_query_id = ib_sa_path_rec_get(target->srp_host->dev->dev,
+       target->path_query_id = ib_sa_path_rec_get(&srp_sa_client,
+                                                  target->srp_host->dev->dev,
                                                   target->srp_host->port,
                                                   &target->path,
                                                   IB_SA_PATH_REC_DGID          |
@@ -1998,9 +2001,12 @@ static int __init srp_init_module(void)
                return ret;
        }
 
+       ib_sa_register_client(&srp_sa_client);
+
        ret = ib_register_client(&srp_client);
        if (ret) {
                printk(KERN_ERR PFX "couldn't register IB client\n");
+               ib_sa_unregister_client(&srp_sa_client);
                class_unregister(&srp_class);
                return ret;
        }
@@ -2011,6 +2017,7 @@ static int __init srp_init_module(void)
 static void __exit srp_cleanup_module(void)
 {
        ib_unregister_client(&srp_client);
+       ib_sa_unregister_client(&srp_sa_client);
        class_unregister(&srp_class);
 }
 

diff --git a/include/rdma/ib_sa.h b/include/rdma/ib_sa.h
index c99e442..58bb5f7 100644 (file)
--- a/include/rdma/ib_sa.h
+++ b/include/rdma/ib_sa.h
@@ -1,6 +1,7 @@
 /*
  * Copyright (c) 2004 Topspin Communications.  All rights reserved.
  * Copyright (c) 2005 Voltaire, Inc.  All rights reserved.
+ * Copyright (c) 2006 Intel Corporation.  All rights reserved.
  *
  * This software is available to you under a choice of one of two
  * licenses.  You may choose to be licensed under the terms of the GNU
@@ -36,8 +37,11 @@
 #ifndef IB_SA_H
 #define IB_SA_H
 
+#include <linux/completion.h>
 #include <linux/compiler.h>
 
+#include <asm/atomic.h>
+
 #include <rdma/ib_verbs.h>
 #include <rdma/ib_mad.h>
 
@@ -250,11 +254,28 @@ struct ib_sa_service_rec {
        u64             data64[2];
 };
 
+struct ib_sa_client {
+       atomic_t users;
+       struct completion comp;
+};
+
+/**
+ * ib_sa_register_client - Register an SA client.
+ */
+void ib_sa_register_client(struct ib_sa_client *client);
+
+/**
+ * ib_sa_unregister_client - Deregister an SA client.
+ * @client: Client object to deregister.
+ */
+void ib_sa_unregister_client(struct ib_sa_client *client);
+
 struct ib_sa_query;
 
 void ib_sa_cancel_query(int id, struct ib_sa_query *query);
 
-int ib_sa_path_rec_get(struct ib_device *device, u8 port_num,
+int ib_sa_path_rec_get(struct ib_sa_client *client,
+                      struct ib_device *device, u8 port_num,
                       struct ib_sa_path_rec *rec,
                       ib_sa_comp_mask comp_mask,
                       int timeout_ms, gfp_t gfp_mask,
@@ -264,7 +285,8 @@ int ib_sa_path_rec_get(struct ib_device *device, u8 port_num,
                       void *context,
                       struct ib_sa_query **query);
 
-int ib_sa_mcmember_rec_query(struct ib_device *device, u8 port_num,
+int ib_sa_mcmember_rec_query(struct ib_sa_client *client,
+                            struct ib_device *device, u8 port_num,
                             u8 method,
                             struct ib_sa_mcmember_rec *rec,
                             ib_sa_comp_mask comp_mask,
@@ -275,7 +297,8 @@ int ib_sa_mcmember_rec_query(struct ib_device *device, u8 port_num,
                             void *context,
                             struct ib_sa_query **query);
 
-int ib_sa_service_rec_query(struct ib_device *device, u8 port_num,
+int ib_sa_service_rec_query(struct ib_sa_client *client,
+                        struct ib_device *device, u8 port_num,
                         u8 method,
                         struct ib_sa_service_rec *rec,
                         ib_sa_comp_mask comp_mask,
@@ -288,6 +311,7 @@ int ib_sa_service_rec_query(struct ib_device *device, u8 port_num,
 
 /**
  * ib_sa_mcmember_rec_set - Start an MCMember set query
+ * @client:SA client
  * @device:device to send query on
  * @port_num: port number to send query on
  * @rec:MCMember Record to send in query
@@ -311,7 +335,8 @@ int ib_sa_service_rec_query(struct ib_device *device, u8 port_num,
  * cancel the query.
  */
 static inline int
-ib_sa_mcmember_rec_set(struct ib_device *device, u8 port_num,
+ib_sa_mcmember_rec_set(struct ib_sa_client *client,
+                      struct ib_device *device, u8 port_num,
                       struct ib_sa_mcmember_rec *rec,
                       ib_sa_comp_mask comp_mask,
                       int timeout_ms, gfp_t gfp_mask,
@@ -321,7 +346,7 @@ ib_sa_mcmember_rec_set(struct ib_device *device, u8 port_num,
                       void *context,
                       struct ib_sa_query **query)
 {
-       return ib_sa_mcmember_rec_query(device, port_num,
+       return ib_sa_mcmember_rec_query(client, device, port_num,
                                        IB_MGMT_METHOD_SET,
                                        rec, comp_mask,
                                        timeout_ms, gfp_mask, callback,
@@ -330,6 +355,7 @@ ib_sa_mcmember_rec_set(struct ib_device *device, u8 port_num,
 
 /**
  * ib_sa_mcmember_rec_delete - Start an MCMember delete query
+ * @client:SA client
  * @device:device to send query on
  * @port_num: port number to send query on
  * @rec:MCMember Record to send in query
@@ -353,7 +379,8 @@ ib_sa_mcmember_rec_set(struct ib_device *device, u8 port_num,
  * cancel the query.
  */
 static inline int
-ib_sa_mcmember_rec_delete(struct ib_device *device, u8 port_num,
+ib_sa_mcmember_rec_delete(struct ib_sa_client *client,
+                         struct ib_device *device, u8 port_num,
                          struct ib_sa_mcmember_rec *rec,
                          ib_sa_comp_mask comp_mask,
                          int timeout_ms, gfp_t gfp_mask,
@@ -363,7 +390,7 @@ ib_sa_mcmember_rec_delete(struct ib_device *device, u8 port_num,
                          void *context,
                          struct ib_sa_query **query)
 {
-       return ib_sa_mcmember_rec_query(device, port_num,
+       return ib_sa_mcmember_rec_query(client, device, port_num,
                                        IB_SA_METHOD_DELETE,
                                        rec, comp_mask,
                                        timeout_ms, gfp_mask, callback,