SAFE public projects git trees. - safe/jmp/linux-2.6/blob - net/sunrpc/xprtrdma/verbs.c

   1 /*
   2  * Copyright (c) 2003-2007 Network Appliance, Inc. All rights reserved.
   3  *
   4  * This software is available to you under a choice of one of two
   5  * licenses.  You may choose to be licensed under the terms of the GNU
   6  * General Public License (GPL) Version 2, available from the file
   7  * COPYING in the main directory of this source tree, or the BSD-type
   8  * license below:
   9  *
  10  * Redistribution and use in source and binary forms, with or without
  11  * modification, are permitted provided that the following conditions
  12  * are met:
  13  *
  14  *      Redistributions of source code must retain the above copyright
  15  *      notice, this list of conditions and the following disclaimer.
  16  *
  17  *      Redistributions in binary form must reproduce the above
  18  *      copyright notice, this list of conditions and the following
  19  *      disclaimer in the documentation and/or other materials provided
  20  *      with the distribution.
  21  *
  22  *      Neither the name of the Network Appliance, Inc. nor the names of
  23  *      its contributors may be used to endorse or promote products
  24  *      derived from this software without specific prior written
  25  *      permission.
  26  *
  27  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  28  * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  29  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  30  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  31  * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  32  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  33  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  34  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  35  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  36  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  37  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  38  */
  39
  40 /*
  41  * verbs.c
  42  *
  43  * Encapsulates the major functions managing:
  44  *  o adapters
  45  *  o endpoints
  46  *  o connections
  47  *  o buffer memory
  48  */
  49
  50 #include <linux/pci.h>  /* for Tavor hack below */
  51
  52 #include "xprt_rdma.h"
  53
  54 /*
  55  * Globals/Macros
  56  */
  57
  58 #ifdef RPC_DEBUG
  59 # define RPCDBG_FACILITY        RPCDBG_TRANS
  60 #endif
  61
  62 /*
  63  * internal functions
  64  */
  65
  66 /*
  67  * handle replies in tasklet context, using a single, global list
  68  * rdma tasklet function -- just turn around and call the func
  69  * for all replies on the list
  70  */
  71
  72 static DEFINE_SPINLOCK(rpcrdma_tk_lock_g);
  73 static LIST_HEAD(rpcrdma_tasklets_g);
  74
  75 static void
  76 rpcrdma_run_tasklet(unsigned long data)
  77 {
  78         struct rpcrdma_rep *rep;
  79         void (*func)(struct rpcrdma_rep *);
  80         unsigned long flags;
  81
  82         data = data;
  83         spin_lock_irqsave(&rpcrdma_tk_lock_g, flags);
  84         while (!list_empty(&rpcrdma_tasklets_g)) {
  85                 rep = list_entry(rpcrdma_tasklets_g.next,
  86                                  struct rpcrdma_rep, rr_list);
  87                 list_del(&rep->rr_list);
  88                 func = rep->rr_func;
  89                 rep->rr_func = NULL;
  90                 spin_unlock_irqrestore(&rpcrdma_tk_lock_g, flags);
  91
  92                 if (func)
  93                         func(rep);
  94                 else
  95                         rpcrdma_recv_buffer_put(rep);
  96
  97                 spin_lock_irqsave(&rpcrdma_tk_lock_g, flags);
  98         }
  99         spin_unlock_irqrestore(&rpcrdma_tk_lock_g, flags);
 100 }
 101
 102 static DECLARE_TASKLET(rpcrdma_tasklet_g, rpcrdma_run_tasklet, 0UL);
 103
 104 static inline void
 105 rpcrdma_schedule_tasklet(struct rpcrdma_rep *rep)
 106 {
 107         unsigned long flags;
 108
 109         spin_lock_irqsave(&rpcrdma_tk_lock_g, flags);
 110         list_add_tail(&rep->rr_list, &rpcrdma_tasklets_g);
 111         spin_unlock_irqrestore(&rpcrdma_tk_lock_g, flags);
 112         tasklet_schedule(&rpcrdma_tasklet_g);
 113 }
 114
 115 static void
 116 rpcrdma_qp_async_error_upcall(struct ib_event *event, void *context)
 117 {
 118         struct rpcrdma_ep *ep = context;
 119
 120         dprintk("RPC:       %s: QP error %X on device %s ep %p\n",
 121                 __func__, event->event, event->device->name, context);
 122         if (ep->rep_connected == 1) {
 123                 ep->rep_connected = -EIO;
 124                 ep->rep_func(ep);
 125                 wake_up_all(&ep->rep_connect_wait);
 126         }
 127 }
 128
 129 static void
 130 rpcrdma_cq_async_error_upcall(struct ib_event *event, void *context)
 131 {
 132         struct rpcrdma_ep *ep = context;
 133
 134         dprintk("RPC:       %s: CQ error %X on device %s ep %p\n",
 135                 __func__, event->event, event->device->name, context);
 136         if (ep->rep_connected == 1) {
 137                 ep->rep_connected = -EIO;
 138                 ep->rep_func(ep);
 139                 wake_up_all(&ep->rep_connect_wait);
 140         }
 141 }
 142
 143 static inline
 144 void rpcrdma_event_process(struct ib_wc *wc)
 145 {
 146         struct rpcrdma_rep *rep =
 147                         (struct rpcrdma_rep *)(unsigned long) wc->wr_id;
 148
 149         dprintk("RPC:       %s: event rep %p status %X opcode %X length %u\n",
 150                 __func__, rep, wc->status, wc->opcode, wc->byte_len);
 151
 152         if (!rep) /* send or bind completion that we don't care about */
 153                 return;
 154
 155         if (IB_WC_SUCCESS != wc->status) {
 156                 dprintk("RPC:       %s: %s WC status %X, connection lost\n",
 157                         __func__, (wc->opcode & IB_WC_RECV) ? "recv" : "send",
 158                          wc->status);
 159                 rep->rr_len = ~0U;
 160                 rpcrdma_schedule_tasklet(rep);
 161                 return;
 162         }
 163
 164         switch (wc->opcode) {
 165         case IB_WC_RECV:
 166                 rep->rr_len = wc->byte_len;
 167                 ib_dma_sync_single_for_cpu(
 168                         rdmab_to_ia(rep->rr_buffer)->ri_id->device,
 169                         rep->rr_iov.addr, rep->rr_len, DMA_FROM_DEVICE);
 170                 /* Keep (only) the most recent credits, after check validity */
 171                 if (rep->rr_len >= 16) {
 172                         struct rpcrdma_msg *p =
 173                                         (struct rpcrdma_msg *) rep->rr_base;
 174                         unsigned int credits = ntohl(p->rm_credit);
 175                         if (credits == 0) {
 176                                 dprintk("RPC:       %s: server"
 177                                         " dropped credits to 0!\n", __func__);
 178                                 /* don't deadlock */
 179                                 credits = 1;
 180                         } else if (credits > rep->rr_buffer->rb_max_requests) {
 181                                 dprintk("RPC:       %s: server"
 182                                         " over-crediting: %d (%d)\n",
 183                                         __func__, credits,
 184                                         rep->rr_buffer->rb_max_requests);
 185                                 credits = rep->rr_buffer->rb_max_requests;
 186                         }
 187                         atomic_set(&rep->rr_buffer->rb_credits, credits);
 188                 }
 189                 /* fall through */
 190         case IB_WC_BIND_MW:
 191                 rpcrdma_schedule_tasklet(rep);
 192                 break;
 193         default:
 194                 dprintk("RPC:       %s: unexpected WC event %X\n",
 195                         __func__, wc->opcode);
 196                 break;
 197         }
 198 }
 199
 200 static inline int
 201 rpcrdma_cq_poll(struct ib_cq *cq)
 202 {
 203         struct ib_wc wc;
 204         int rc;
 205
 206         for (;;) {
 207                 rc = ib_poll_cq(cq, 1, &wc);
 208                 if (rc < 0) {
 209                         dprintk("RPC:       %s: ib_poll_cq failed %i\n",
 210                                 __func__, rc);
 211                         return rc;
 212                 }
 213                 if (rc == 0)
 214                         break;
 215
 216                 rpcrdma_event_process(&wc);
 217         }
 218
 219         return 0;
 220 }
 221
 222 /*
 223  * rpcrdma_cq_event_upcall
 224  *
 225  * This upcall handles recv, send, bind and unbind events.
 226  * It is reentrant but processes single events in order to maintain
 227  * ordering of receives to keep server credits.
 228  *
 229  * It is the responsibility of the scheduled tasklet to return
 230  * recv buffers to the pool. NOTE: this affects synchronization of
 231  * connection shutdown. That is, the structures required for
 232  * the completion of the reply handler must remain intact until
 233  * all memory has been reclaimed.
 234  *
 235  * Note that send events are suppressed and do not result in an upcall.
 236  */
 237 static void
 238 rpcrdma_cq_event_upcall(struct ib_cq *cq, void *context)
 239 {
 240         int rc;
 241
 242         rc = rpcrdma_cq_poll(cq);
 243         if (rc)
 244                 return;
 245
 246         rc = ib_req_notify_cq(cq, IB_CQ_NEXT_COMP);
 247         if (rc) {
 248                 dprintk("RPC:       %s: ib_req_notify_cq failed %i\n",
 249                         __func__, rc);
 250                 return;
 251         }
 252
 253         rpcrdma_cq_poll(cq);
 254 }
 255
 256 #ifdef RPC_DEBUG
 257 static const char * const conn[] = {
 258         "address resolved",
 259         "address error",
 260         "route resolved",
 261         "route error",
 262         "connect request",
 263         "connect response",
 264         "connect error",
 265         "unreachable",
 266         "rejected",
 267         "established",
 268         "disconnected",
 269         "device removal"
 270 };
 271 #endif
 272
 273 static int
 274 rpcrdma_conn_upcall(struct rdma_cm_id *id, struct rdma_cm_event *event)
 275 {
 276         struct rpcrdma_xprt *xprt = id->context;
 277         struct rpcrdma_ia *ia = &xprt->rx_ia;
 278         struct rpcrdma_ep *ep = &xprt->rx_ep;
 279         struct sockaddr_in *addr = (struct sockaddr_in *) &ep->rep_remote_addr;
 280         struct ib_qp_attr attr;
 281         struct ib_qp_init_attr iattr;
 282         int connstate = 0;
 283
 284         switch (event->event) {
 285         case RDMA_CM_EVENT_ADDR_RESOLVED:
 286         case RDMA_CM_EVENT_ROUTE_RESOLVED:
 287                 ia->ri_async_rc = 0;
 288                 complete(&ia->ri_done);
 289                 break;
 290         case RDMA_CM_EVENT_ADDR_ERROR:
 291                 ia->ri_async_rc = -EHOSTUNREACH;
 292                 dprintk("RPC:       %s: CM address resolution error, ep 0x%p\n",
 293                         __func__, ep);
 294                 complete(&ia->ri_done);
 295                 break;
 296         case RDMA_CM_EVENT_ROUTE_ERROR:
 297                 ia->ri_async_rc = -ENETUNREACH;
 298                 dprintk("RPC:       %s: CM route resolution error, ep 0x%p\n",
 299                         __func__, ep);
 300                 complete(&ia->ri_done);
 301                 break;
 302         case RDMA_CM_EVENT_ESTABLISHED:
 303                 connstate = 1;
 304                 ib_query_qp(ia->ri_id->qp, &attr,
 305                         IB_QP_MAX_QP_RD_ATOMIC | IB_QP_MAX_DEST_RD_ATOMIC,
 306                         &iattr);
 307                 dprintk("RPC:       %s: %d responder resources"
 308                         " (%d initiator)\n",
 309                         __func__, attr.max_dest_rd_atomic, attr.max_rd_atomic);
 310                 goto connected;
 311         case RDMA_CM_EVENT_CONNECT_ERROR:
 312                 connstate = -ENOTCONN;
 313                 goto connected;
 314         case RDMA_CM_EVENT_UNREACHABLE:
 315                 connstate = -ENETDOWN;
 316                 goto connected;
 317         case RDMA_CM_EVENT_REJECTED:
 318                 connstate = -ECONNREFUSED;
 319                 goto connected;
 320         case RDMA_CM_EVENT_DISCONNECTED:
 321                 connstate = -ECONNABORTED;
 322                 goto connected;
 323         case RDMA_CM_EVENT_DEVICE_REMOVAL:
 324                 connstate = -ENODEV;
 325 connected:
 326                 dprintk("RPC:       %s: %s: %u.%u.%u.%u:%u"
 327                         " (ep 0x%p event 0x%x)\n",
 328                         __func__,
 329                         (event->event <= 11) ? conn[event->event] :
 330                                                 "unknown connection error",
 331                         NIPQUAD(addr->sin_addr.s_addr),
 332                         ntohs(addr->sin_port),
 333                         ep, event->event);
 334                 atomic_set(&rpcx_to_rdmax(ep->rep_xprt)->rx_buf.rb_credits, 1);
 335                 dprintk("RPC:       %s: %sconnected\n",
 336                                         __func__, connstate > 0 ? "" : "dis");
 337                 ep->rep_connected = connstate;
 338                 ep->rep_func(ep);
 339                 wake_up_all(&ep->rep_connect_wait);
 340                 break;
 341         default:
 342                 dprintk("RPC:       %s: unexpected CM event %d\n",
 343                         __func__, event->event);
 344                 break;
 345         }
 346
 347 #ifdef RPC_DEBUG
 348         if (connstate == 1) {
 349                 int ird = attr.max_dest_rd_atomic;
 350                 int tird = ep->rep_remote_cma.responder_resources;
 351                 printk(KERN_INFO "rpcrdma: connection to %u.%u.%u.%u:%u "
 352                         "on %s, memreg %d slots %d ird %d%s\n",
 353                         NIPQUAD(addr->sin_addr.s_addr),
 354                         ntohs(addr->sin_port),
 355                         ia->ri_id->device->name,
 356                         ia->ri_memreg_strategy,
 357                         xprt->rx_buf.rb_max_requests,
 358                         ird, ird < 4 && ird < tird / 2 ? " (low!)" : "");
 359         } else if (connstate < 0) {
 360                 printk(KERN_INFO "rpcrdma: connection to %u.%u.%u.%u:%u "
 361                         "closed (%d)\n",
 362                         NIPQUAD(addr->sin_addr.s_addr),
 363                         ntohs(addr->sin_port),
 364                         connstate);
 365         }
 366 #endif
 367
 368         return 0;
 369 }
 370
 371 static struct rdma_cm_id *
 372 rpcrdma_create_id(struct rpcrdma_xprt *xprt,
 373                         struct rpcrdma_ia *ia, struct sockaddr *addr)
 374 {
 375         struct rdma_cm_id *id;
 376         int rc;
 377
 378         init_completion(&ia->ri_done);
 379
 380         id = rdma_create_id(rpcrdma_conn_upcall, xprt, RDMA_PS_TCP);
 381         if (IS_ERR(id)) {
 382                 rc = PTR_ERR(id);
 383                 dprintk("RPC:       %s: rdma_create_id() failed %i\n",
 384                         __func__, rc);
 385                 return id;
 386         }
 387
 388         ia->ri_async_rc = -ETIMEDOUT;
 389         rc = rdma_resolve_addr(id, NULL, addr, RDMA_RESOLVE_TIMEOUT);
 390         if (rc) {
 391                 dprintk("RPC:       %s: rdma_resolve_addr() failed %i\n",
 392                         __func__, rc);
 393                 goto out;
 394         }
 395         wait_for_completion_interruptible_timeout(&ia->ri_done,
 396                                 msecs_to_jiffies(RDMA_RESOLVE_TIMEOUT) + 1);
 397         rc = ia->ri_async_rc;
 398         if (rc)
 399                 goto out;
 400
 401         ia->ri_async_rc = -ETIMEDOUT;
 402         rc = rdma_resolve_route(id, RDMA_RESOLVE_TIMEOUT);
 403         if (rc) {
 404                 dprintk("RPC:       %s: rdma_resolve_route() failed %i\n",
 405                         __func__, rc);
 406                 goto out;
 407         }
 408         wait_for_completion_interruptible_timeout(&ia->ri_done,
 409                                 msecs_to_jiffies(RDMA_RESOLVE_TIMEOUT) + 1);
 410         rc = ia->ri_async_rc;
 411         if (rc)
 412                 goto out;
 413
 414         return id;
 415
 416 out:
 417         rdma_destroy_id(id);
 418         return ERR_PTR(rc);
 419 }
 420
 421 /*
 422  * Drain any cq, prior to teardown.
 423  */
 424 static void
 425 rpcrdma_clean_cq(struct ib_cq *cq)
 426 {
 427         struct ib_wc wc;
 428         int count = 0;
 429
 430         while (1 == ib_poll_cq(cq, 1, &wc))
 431                 ++count;
 432
 433         if (count)
 434                 dprintk("RPC:       %s: flushed %d events (last 0x%x)\n",
 435                         __func__, count, wc.opcode);
 436 }
 437
 438 /*
 439  * Exported functions.
 440  */
 441
 442 /*
 443  * Open and initialize an Interface Adapter.
 444  *  o initializes fields of struct rpcrdma_ia, including
 445  *    interface and provider attributes and protection zone.
 446  */
 447 int
 448 rpcrdma_ia_open(struct rpcrdma_xprt *xprt, struct sockaddr *addr, int memreg)
 449 {
 450         int rc, mem_priv;
 451         struct ib_device_attr devattr;
 452         struct rpcrdma_ia *ia = &xprt->rx_ia;
 453
 454         ia->ri_id = rpcrdma_create_id(xprt, ia, addr);
 455         if (IS_ERR(ia->ri_id)) {
 456                 rc = PTR_ERR(ia->ri_id);
 457                 goto out1;
 458         }
 459
 460         ia->ri_pd = ib_alloc_pd(ia->ri_id->device);
 461         if (IS_ERR(ia->ri_pd)) {
 462                 rc = PTR_ERR(ia->ri_pd);
 463                 dprintk("RPC:       %s: ib_alloc_pd() failed %i\n",
 464                         __func__, rc);
 465                 goto out2;
 466         }
 467
 468         /*
 469          * Query the device to determine if the requested memory
 470          * registration strategy is supported. If it isn't, set the
 471          * strategy to a globally supported model.
 472          */
 473         rc = ib_query_device(ia->ri_id->device, &devattr);
 474         if (rc) {
 475                 dprintk("RPC:       %s: ib_query_device failed %d\n",
 476                         __func__, rc);
 477                 goto out2;
 478         }
 479
 480         if (devattr.device_cap_flags & IB_DEVICE_LOCAL_DMA_LKEY) {
 481                 ia->ri_have_dma_lkey = 1;
 482                 ia->ri_dma_lkey = ia->ri_id->device->local_dma_lkey;
 483         }
 484
 485         switch (memreg) {
 486         case RPCRDMA_MEMWINDOWS:
 487         case RPCRDMA_MEMWINDOWS_ASYNC:
 488                 if (!(devattr.device_cap_flags & IB_DEVICE_MEM_WINDOW)) {
 489                         dprintk("RPC:       %s: MEMWINDOWS registration "
 490                                 "specified but not supported by adapter, "
 491                                 "using slower RPCRDMA_REGISTER\n",
 492                                 __func__);
 493                         memreg = RPCRDMA_REGISTER;
 494                 }
 495                 break;
 496         case RPCRDMA_MTHCAFMR:
 497                 if (!ia->ri_id->device->alloc_fmr) {
 498 #if RPCRDMA_PERSISTENT_REGISTRATION
 499                         dprintk("RPC:       %s: MTHCAFMR registration "
 500                                 "specified but not supported by adapter, "
 501                                 "using riskier RPCRDMA_ALLPHYSICAL\n",
 502                                 __func__);
 503                         memreg = RPCRDMA_ALLPHYSICAL;
 504 #else
 505                         dprintk("RPC:       %s: MTHCAFMR registration "
 506                                 "specified but not supported by adapter, "
 507                                 "using slower RPCRDMA_REGISTER\n",
 508                                 __func__);
 509                         memreg = RPCRDMA_REGISTER;
 510 #endif
 511                 }
 512                 break;
 513         case RPCRDMA_FRMR:
 514                 /* Requires both frmr reg and local dma lkey */
 515                 if ((devattr.device_cap_flags &
 516                      (IB_DEVICE_MEM_MGT_EXTENSIONS|IB_DEVICE_LOCAL_DMA_LKEY)) !=
 517                     (IB_DEVICE_MEM_MGT_EXTENSIONS|IB_DEVICE_LOCAL_DMA_LKEY)) {
 518 #if RPCRDMA_PERSISTENT_REGISTRATION
 519                         dprintk("RPC:       %s: FRMR registration "
 520                                 "specified but not supported by adapter, "
 521                                 "using riskier RPCRDMA_ALLPHYSICAL\n",
 522                                 __func__);
 523                         memreg = RPCRDMA_ALLPHYSICAL;
 524 #else
 525                         dprintk("RPC:       %s: FRMR registration "
 526                                 "specified but not supported by adapter, "
 527                                 "using slower RPCRDMA_REGISTER\n",
 528                                 __func__);
 529                         memreg = RPCRDMA_REGISTER;
 530 #endif
 531                 }
 532                 break;
 533         }
 534
 535         /*
 536          * Optionally obtain an underlying physical identity mapping in
 537          * order to do a memory window-based bind. This base registration
 538          * is protected from remote access - that is enabled only by binding
 539          * for the specific bytes targeted during each RPC operation, and
 540          * revoked after the corresponding completion similar to a storage
 541          * adapter.
 542          */
 543         switch (memreg) {
 544         case RPCRDMA_BOUNCEBUFFERS:
 545         case RPCRDMA_REGISTER:
 546         case RPCRDMA_FRMR:
 547                 break;
 548 #if RPCRDMA_PERSISTENT_REGISTRATION
 549         case RPCRDMA_ALLPHYSICAL:
 550                 mem_priv = IB_ACCESS_LOCAL_WRITE |
 551                                 IB_ACCESS_REMOTE_WRITE |
 552                                 IB_ACCESS_REMOTE_READ;
 553                 goto register_setup;
 554 #endif
 555         case RPCRDMA_MEMWINDOWS_ASYNC:
 556         case RPCRDMA_MEMWINDOWS:
 557                 mem_priv = IB_ACCESS_LOCAL_WRITE |
 558                                 IB_ACCESS_MW_BIND;
 559                 goto register_setup;
 560         case RPCRDMA_MTHCAFMR:
 561                 if (ia->ri_have_dma_lkey)
 562                         break;
 563                 mem_priv = IB_ACCESS_LOCAL_WRITE;
 564         register_setup:
 565                 ia->ri_bind_mem = ib_get_dma_mr(ia->ri_pd, mem_priv);
 566                 if (IS_ERR(ia->ri_bind_mem)) {
 567                         printk(KERN_ALERT "%s: ib_get_dma_mr for "
 568                                 "phys register failed with %lX\n\t"
 569                                 "Will continue with degraded performance\n",
 570                                 __func__, PTR_ERR(ia->ri_bind_mem));
 571                         memreg = RPCRDMA_REGISTER;
 572                         ia->ri_bind_mem = NULL;
 573                 }
 574                 break;
 575         default:
 576                 printk(KERN_ERR "%s: invalid memory registration mode %d\n",
 577                                 __func__, memreg);
 578                 rc = -EINVAL;
 579                 goto out2;
 580         }
 581         dprintk("RPC:       %s: memory registration strategy is %d\n",
 582                 __func__, memreg);
 583
 584         /* Else will do memory reg/dereg for each chunk */
 585         ia->ri_memreg_strategy = memreg;
 586
 587         return 0;
 588 out2:
 589         rdma_destroy_id(ia->ri_id);
 590         ia->ri_id = NULL;
 591 out1:
 592         return rc;
 593 }
 594
 595 /*
 596  * Clean up/close an IA.
 597  *   o if event handles and PD have been initialized, free them.
 598  *   o close the IA
 599  */
 600 void
 601 rpcrdma_ia_close(struct rpcrdma_ia *ia)
 602 {
 603         int rc;
 604
 605         dprintk("RPC:       %s: entering\n", __func__);
 606         if (ia->ri_bind_mem != NULL) {
 607                 rc = ib_dereg_mr(ia->ri_bind_mem);
 608                 dprintk("RPC:       %s: ib_dereg_mr returned %i\n",
 609                         __func__, rc);
 610         }
 611         if (ia->ri_id != NULL && !IS_ERR(ia->ri_id)) {
 612                 if (ia->ri_id->qp)
 613                         rdma_destroy_qp(ia->ri_id);
 614                 rdma_destroy_id(ia->ri_id);
 615                 ia->ri_id = NULL;
 616         }
 617         if (ia->ri_pd != NULL && !IS_ERR(ia->ri_pd)) {
 618                 rc = ib_dealloc_pd(ia->ri_pd);
 619                 dprintk("RPC:       %s: ib_dealloc_pd returned %i\n",
 620                         __func__, rc);
 621         }
 622 }
 623
 624 /*
 625  * Create unconnected endpoint.
 626  */
 627 int
 628 rpcrdma_ep_create(struct rpcrdma_ep *ep, struct rpcrdma_ia *ia,
 629                                 struct rpcrdma_create_data_internal *cdata)
 630 {
 631         struct ib_device_attr devattr;
 632         int rc, err;
 633
 634         rc = ib_query_device(ia->ri_id->device, &devattr);
 635         if (rc) {
 636                 dprintk("RPC:       %s: ib_query_device failed %d\n",
 637                         __func__, rc);
 638                 return rc;
 639         }
 640
 641         /* check provider's send/recv wr limits */
 642         if (cdata->max_requests > devattr.max_qp_wr)
 643                 cdata->max_requests = devattr.max_qp_wr;
 644
 645         ep->rep_attr.event_handler = rpcrdma_qp_async_error_upcall;
 646         ep->rep_attr.qp_context = ep;
 647         /* send_cq and recv_cq initialized below */
 648         ep->rep_attr.srq = NULL;
 649         ep->rep_attr.cap.max_send_wr = cdata->max_requests;
 650         switch (ia->ri_memreg_strategy) {
 651         case RPCRDMA_FRMR:
 652                 /* Add room for frmr register and invalidate WRs */
 653                 ep->rep_attr.cap.max_send_wr *= 3;
 654                 if (ep->rep_attr.cap.max_send_wr > devattr.max_qp_wr)
 655                         return -EINVAL;
 656                 break;
 657         case RPCRDMA_MEMWINDOWS_ASYNC:
 658         case RPCRDMA_MEMWINDOWS:
 659                 /* Add room for mw_binds+unbinds - overkill! */
 660                 ep->rep_attr.cap.max_send_wr++;
 661                 ep->rep_attr.cap.max_send_wr *= (2 * RPCRDMA_MAX_SEGS);
 662                 if (ep->rep_attr.cap.max_send_wr > devattr.max_qp_wr)
 663                         return -EINVAL;
 664                 break;
 665         default:
 666                 break;
 667         }
 668         ep->rep_attr.cap.max_recv_wr = cdata->max_requests;
 669         ep->rep_attr.cap.max_send_sge = (cdata->padding ? 4 : 2);
 670         ep->rep_attr.cap.max_recv_sge = 1;
 671         ep->rep_attr.cap.max_inline_data = 0;
 672         ep->rep_attr.sq_sig_type = IB_SIGNAL_REQ_WR;
 673         ep->rep_attr.qp_type = IB_QPT_RC;
 674         ep->rep_attr.port_num = ~0;
 675
 676         dprintk("RPC:       %s: requested max: dtos: send %d recv %d; "
 677                 "iovs: send %d recv %d\n",
 678                 __func__,
 679                 ep->rep_attr.cap.max_send_wr,
 680                 ep->rep_attr.cap.max_recv_wr,
 681                 ep->rep_attr.cap.max_send_sge,
 682                 ep->rep_attr.cap.max_recv_sge);
 683
 684         /* set trigger for requesting send completion */
 685         ep->rep_cqinit = ep->rep_attr.cap.max_send_wr/2 /*  - 1*/;
 686         switch (ia->ri_memreg_strategy) {
 687         case RPCRDMA_MEMWINDOWS_ASYNC:
 688         case RPCRDMA_MEMWINDOWS:
 689                 ep->rep_cqinit -= RPCRDMA_MAX_SEGS;
 690                 break;
 691         default:
 692                 break;
 693         }
 694         if (ep->rep_cqinit <= 2)
 695                 ep->rep_cqinit = 0;
 696         INIT_CQCOUNT(ep);
 697         ep->rep_ia = ia;
 698         init_waitqueue_head(&ep->rep_connect_wait);
 699
 700         /*
 701          * Create a single cq for receive dto and mw_bind (only ever
 702          * care about unbind, really). Send completions are suppressed.
 703          * Use single threaded tasklet upcalls to maintain ordering.
 704          */
 705         ep->rep_cq = ib_create_cq(ia->ri_id->device, rpcrdma_cq_event_upcall,
 706                                   rpcrdma_cq_async_error_upcall, NULL,
 707                                   ep->rep_attr.cap.max_recv_wr +
 708                                   ep->rep_attr.cap.max_send_wr + 1, 0);
 709         if (IS_ERR(ep->rep_cq)) {
 710                 rc = PTR_ERR(ep->rep_cq);
 711                 dprintk("RPC:       %s: ib_create_cq failed: %i\n",
 712                         __func__, rc);
 713                 goto out1;
 714         }
 715
 716         rc = ib_req_notify_cq(ep->rep_cq, IB_CQ_NEXT_COMP);
 717         if (rc) {
 718                 dprintk("RPC:       %s: ib_req_notify_cq failed: %i\n",
 719                         __func__, rc);
 720                 goto out2;
 721         }
 722
 723         ep->rep_attr.send_cq = ep->rep_cq;
 724         ep->rep_attr.recv_cq = ep->rep_cq;
 725
 726         /* Initialize cma parameters */
 727
 728         /* RPC/RDMA does not use private data */
 729         ep->rep_remote_cma.private_data = NULL;
 730         ep->rep_remote_cma.private_data_len = 0;
 731
 732         /* Client offers RDMA Read but does not initiate */
 733         ep->rep_remote_cma.initiator_depth = 0;
 734         if (ia->ri_memreg_strategy == RPCRDMA_BOUNCEBUFFERS)
 735                 ep->rep_remote_cma.responder_resources = 0;
 736         else if (devattr.max_qp_rd_atom > 32)   /* arbitrary but <= 255 */
 737                 ep->rep_remote_cma.responder_resources = 32;
 738         else
 739                 ep->rep_remote_cma.responder_resources = devattr.max_qp_rd_atom;
 740
 741         ep->rep_remote_cma.retry_count = 7;
 742         ep->rep_remote_cma.flow_control = 0;
 743         ep->rep_remote_cma.rnr_retry_count = 0;
 744
 745         return 0;
 746
 747 out2:
 748         err = ib_destroy_cq(ep->rep_cq);
 749         if (err)
 750                 dprintk("RPC:       %s: ib_destroy_cq returned %i\n",
 751                         __func__, err);
 752 out1:
 753         return rc;
 754 }
 755
 756 /*
 757  * rpcrdma_ep_destroy
 758  *
 759  * Disconnect and destroy endpoint. After this, the only
 760  * valid operations on the ep are to free it (if dynamically
 761  * allocated) or re-create it.
 762  *
 763  * The caller's error handling must be sure to not leak the endpoint
 764  * if this function fails.
 765  */
 766 int
 767 rpcrdma_ep_destroy(struct rpcrdma_ep *ep, struct rpcrdma_ia *ia)
 768 {
 769         int rc;
 770
 771         dprintk("RPC:       %s: entering, connected is %d\n",
 772                 __func__, ep->rep_connected);
 773
 774         if (ia->ri_id->qp) {
 775                 rc = rpcrdma_ep_disconnect(ep, ia);
 776                 if (rc)
 777                         dprintk("RPC:       %s: rpcrdma_ep_disconnect"
 778                                 " returned %i\n", __func__, rc);
 779                 rdma_destroy_qp(ia->ri_id);
 780                 ia->ri_id->qp = NULL;
 781         }
 782
 783         /* padding - could be done in rpcrdma_buffer_destroy... */
 784         if (ep->rep_pad_mr) {
 785                 rpcrdma_deregister_internal(ia, ep->rep_pad_mr, &ep->rep_pad);
 786                 ep->rep_pad_mr = NULL;
 787         }
 788
 789         rpcrdma_clean_cq(ep->rep_cq);
 790         rc = ib_destroy_cq(ep->rep_cq);
 791         if (rc)
 792                 dprintk("RPC:       %s: ib_destroy_cq returned %i\n",
 793                         __func__, rc);
 794
 795         return rc;
 796 }
 797
 798 /*
 799  * Connect unconnected endpoint.
 800  */
 801 int
 802 rpcrdma_ep_connect(struct rpcrdma_ep *ep, struct rpcrdma_ia *ia)
 803 {
 804         struct rdma_cm_id *id;
 805         int rc = 0;
 806         int retry_count = 0;
 807         int reconnect = (ep->rep_connected != 0);
 808
 809         if (reconnect) {
 810                 struct rpcrdma_xprt *xprt;
 811 retry:
 812                 rc = rpcrdma_ep_disconnect(ep, ia);
 813                 if (rc && rc != -ENOTCONN)
 814                         dprintk("RPC:       %s: rpcrdma_ep_disconnect"
 815                                 " status %i\n", __func__, rc);
 816                 rpcrdma_clean_cq(ep->rep_cq);
 817
 818                 xprt = container_of(ia, struct rpcrdma_xprt, rx_ia);
 819                 id = rpcrdma_create_id(xprt, ia,
 820                                 (struct sockaddr *)&xprt->rx_data.addr);
 821                 if (IS_ERR(id)) {
 822                         rc = PTR_ERR(id);
 823                         goto out;
 824                 }
 825                 /* TEMP TEMP TEMP - fail if new device:
 826                  * Deregister/remarshal *all* requests!
 827                  * Close and recreate adapter, pd, etc!
 828                  * Re-determine all attributes still sane!
 829                  * More stuff I haven't thought of!
 830                  * Rrrgh!
 831                  */
 832                 if (ia->ri_id->device != id->device) {
 833                         printk("RPC:       %s: can't reconnect on "
 834                                 "different device!\n", __func__);
 835                         rdma_destroy_id(id);
 836                         rc = -ENETDOWN;
 837                         goto out;
 838                 }
 839                 /* END TEMP */
 840                 rdma_destroy_qp(ia->ri_id);
 841                 rdma_destroy_id(ia->ri_id);
 842                 ia->ri_id = id;
 843         }
 844
 845         rc = rdma_create_qp(ia->ri_id, ia->ri_pd, &ep->rep_attr);
 846         if (rc) {
 847                 dprintk("RPC:       %s: rdma_create_qp failed %i\n",
 848                         __func__, rc);
 849                 goto out;
 850         }
 851
 852 /* XXX Tavor device performs badly with 2K MTU! */
 853 if (strnicmp(ia->ri_id->device->dma_device->bus->name, "pci", 3) == 0) {
 854         struct pci_dev *pcid = to_pci_dev(ia->ri_id->device->dma_device);
 855         if (pcid->device == PCI_DEVICE_ID_MELLANOX_TAVOR &&
 856             (pcid->vendor == PCI_VENDOR_ID_MELLANOX ||
 857              pcid->vendor == PCI_VENDOR_ID_TOPSPIN)) {
 858                 struct ib_qp_attr attr = {
 859                         .path_mtu = IB_MTU_1024
 860                 };
 861                 rc = ib_modify_qp(ia->ri_id->qp, &attr, IB_QP_PATH_MTU);
 862         }
 863 }
 864
 865         ep->rep_connected = 0;
 866
 867         rc = rdma_connect(ia->ri_id, &ep->rep_remote_cma);
 868         if (rc) {
 869                 dprintk("RPC:       %s: rdma_connect() failed with %i\n",
 870                                 __func__, rc);
 871                 goto out;
 872         }
 873
 874         if (reconnect)
 875                 return 0;
 876
 877         wait_event_interruptible(ep->rep_connect_wait, ep->rep_connected != 0);
 878
 879         /*
 880          * Check state. A non-peer reject indicates no listener
 881          * (ECONNREFUSED), which may be a transient state. All
 882          * others indicate a transport condition which has already
 883          * undergone a best-effort.
 884          */
 885         if (ep->rep_connected == -ECONNREFUSED
 886             && ++retry_count <= RDMA_CONNECT_RETRY_MAX) {
 887                 dprintk("RPC:       %s: non-peer_reject, retry\n", __func__);
 888                 goto retry;
 889         }
 890         if (ep->rep_connected <= 0) {
 891                 /* Sometimes, the only way to reliably connect to remote
 892                  * CMs is to use same nonzero values for ORD and IRD. */
 893                 if (retry_count++ <= RDMA_CONNECT_RETRY_MAX + 1 &&
 894                     (ep->rep_remote_cma.responder_resources == 0 ||
 895                      ep->rep_remote_cma.initiator_depth !=
 896                                 ep->rep_remote_cma.responder_resources)) {
 897                         if (ep->rep_remote_cma.responder_resources == 0)
 898                                 ep->rep_remote_cma.responder_resources = 1;
 899                         ep->rep_remote_cma.initiator_depth =
 900                                 ep->rep_remote_cma.responder_resources;
 901                         goto retry;
 902                 }
 903                 rc = ep->rep_connected;
 904         } else {
 905                 dprintk("RPC:       %s: connected\n", __func__);
 906         }
 907
 908 out:
 909         if (rc)
 910                 ep->rep_connected = rc;
 911         return rc;
 912 }
 913
 914 /*
 915  * rpcrdma_ep_disconnect
 916  *
 917  * This is separate from destroy to facilitate the ability
 918  * to reconnect without recreating the endpoint.
 919  *
 920  * This call is not reentrant, and must not be made in parallel
 921  * on the same endpoint.
 922  */
 923 int
 924 rpcrdma_ep_disconnect(struct rpcrdma_ep *ep, struct rpcrdma_ia *ia)
 925 {
 926         int rc;
 927
 928         rpcrdma_clean_cq(ep->rep_cq);
 929         rc = rdma_disconnect(ia->ri_id);
 930         if (!rc) {
 931                 /* returns without wait if not connected */
 932                 wait_event_interruptible(ep->rep_connect_wait,
 933                                                         ep->rep_connected != 1);
 934                 dprintk("RPC:       %s: after wait, %sconnected\n", __func__,
 935                         (ep->rep_connected == 1) ? "still " : "dis");
 936         } else {
 937                 dprintk("RPC:       %s: rdma_disconnect %i\n", __func__, rc);
 938                 ep->rep_connected = rc;
 939         }
 940         return rc;
 941 }
 942
 943 /*
 944  * Initialize buffer memory
 945  */
 946 int
 947 rpcrdma_buffer_create(struct rpcrdma_buffer *buf, struct rpcrdma_ep *ep,
 948         struct rpcrdma_ia *ia, struct rpcrdma_create_data_internal *cdata)
 949 {
 950         char *p;
 951         size_t len;
 952         int i, rc;
 953         struct rpcrdma_mw *r;
 954
 955         buf->rb_max_requests = cdata->max_requests;
 956         spin_lock_init(&buf->rb_lock);
 957         atomic_set(&buf->rb_credits, 1);
 958
 959         /* Need to allocate:
 960          *   1.  arrays for send and recv pointers
 961          *   2.  arrays of struct rpcrdma_req to fill in pointers
 962          *   3.  array of struct rpcrdma_rep for replies
 963          *   4.  padding, if any
 964          *   5.  mw's, fmr's or frmr's, if any
 965          * Send/recv buffers in req/rep need to be registered
 966          */
 967
 968         len = buf->rb_max_requests *
 969                 (sizeof(struct rpcrdma_req *) + sizeof(struct rpcrdma_rep *));
 970         len += cdata->padding;
 971         switch (ia->ri_memreg_strategy) {
 972         case RPCRDMA_FRMR:
 973                 len += buf->rb_max_requests * RPCRDMA_MAX_SEGS *
 974                                 sizeof(struct rpcrdma_mw);
 975                 break;
 976         case RPCRDMA_MTHCAFMR:
 977                 /* TBD we are perhaps overallocating here */
 978                 len += (buf->rb_max_requests + 1) * RPCRDMA_MAX_SEGS *
 979                                 sizeof(struct rpcrdma_mw);
 980                 break;
 981         case RPCRDMA_MEMWINDOWS_ASYNC:
 982         case RPCRDMA_MEMWINDOWS:
 983                 len += (buf->rb_max_requests + 1) * RPCRDMA_MAX_SEGS *
 984                                 sizeof(struct rpcrdma_mw);
 985                 break;
 986         default:
 987                 break;
 988         }
 989
 990         /* allocate 1, 4 and 5 in one shot */
 991         p = kzalloc(len, GFP_KERNEL);
 992         if (p == NULL) {
 993                 dprintk("RPC:       %s: req_t/rep_t/pad kzalloc(%zd) failed\n",
 994                         __func__, len);
 995                 rc = -ENOMEM;
 996                 goto out;
 997         }
 998         buf->rb_pool = p;       /* for freeing it later */
 999
1000         buf->rb_send_bufs = (struct rpcrdma_req **) p;
1001         p = (char *) &buf->rb_send_bufs[buf->rb_max_requests];
1002         buf->rb_recv_bufs = (struct rpcrdma_rep **) p;
1003         p = (char *) &buf->rb_recv_bufs[buf->rb_max_requests];
1004
1005         /*
1006          * Register the zeroed pad buffer, if any.
1007          */
1008         if (cdata->padding) {
1009                 rc = rpcrdma_register_internal(ia, p, cdata->padding,
1010                                             &ep->rep_pad_mr, &ep->rep_pad);
1011                 if (rc)
1012                         goto out;
1013         }
1014         p += cdata->padding;
1015
1016         /*
1017          * Allocate the fmr's, or mw's for mw_bind chunk registration.
1018          * We "cycle" the mw's in order to minimize rkey reuse,
1019          * and also reduce unbind-to-bind collision.
1020          */
1021         INIT_LIST_HEAD(&buf->rb_mws);
1022         r = (struct rpcrdma_mw *)p;
1023         switch (ia->ri_memreg_strategy) {
1024         case RPCRDMA_FRMR:
1025                 for (i = buf->rb_max_requests * RPCRDMA_MAX_SEGS; i; i--) {
1026                         r->r.frmr.fr_mr = ib_alloc_fast_reg_mr(ia->ri_pd,
1027                                                          RPCRDMA_MAX_SEGS);
1028                         if (IS_ERR(r->r.frmr.fr_mr)) {
1029                                 rc = PTR_ERR(r->r.frmr.fr_mr);
1030                                 dprintk("RPC:       %s: ib_alloc_fast_reg_mr"
1031                                         " failed %i\n", __func__, rc);
1032                                 goto out;
1033                         }
1034                         r->r.frmr.fr_pgl =
1035                                 ib_alloc_fast_reg_page_list(ia->ri_id->device,
1036                                                             RPCRDMA_MAX_SEGS);
1037                         if (IS_ERR(r->r.frmr.fr_pgl)) {
1038                                 rc = PTR_ERR(r->r.frmr.fr_pgl);
1039                                 dprintk("RPC:       %s: "
1040                                         "ib_alloc_fast_reg_page_list "
1041                                         "failed %i\n", __func__, rc);
1042                                 goto out;
1043                         }
1044                         list_add(&r->mw_list, &buf->rb_mws);
1045                         ++r;
1046                 }
1047                 break;
1048         case RPCRDMA_MTHCAFMR:
1049                 /* TBD we are perhaps overallocating here */
1050                 for (i = (buf->rb_max_requests+1) * RPCRDMA_MAX_SEGS; i; i--) {
1051                         static struct ib_fmr_attr fa =
1052                                 { RPCRDMA_MAX_DATA_SEGS, 1, PAGE_SHIFT };
1053                         r->r.fmr = ib_alloc_fmr(ia->ri_pd,
1054                                 IB_ACCESS_REMOTE_WRITE | IB_ACCESS_REMOTE_READ,
1055                                 &fa);
1056                         if (IS_ERR(r->r.fmr)) {
1057                                 rc = PTR_ERR(r->r.fmr);
1058                                 dprintk("RPC:       %s: ib_alloc_fmr"
1059                                         " failed %i\n", __func__, rc);
1060                                 goto out;
1061                         }
1062                         list_add(&r->mw_list, &buf->rb_mws);
1063                         ++r;
1064                 }
1065                 break;
1066         case RPCRDMA_MEMWINDOWS_ASYNC:
1067         case RPCRDMA_MEMWINDOWS:
1068                 /* Allocate one extra request's worth, for full cycling */
1069                 for (i = (buf->rb_max_requests+1) * RPCRDMA_MAX_SEGS; i; i--) {
1070                         r->r.mw = ib_alloc_mw(ia->ri_pd);
1071                         if (IS_ERR(r->r.mw)) {
1072                                 rc = PTR_ERR(r->r.mw);
1073                                 dprintk("RPC:       %s: ib_alloc_mw"
1074                                         " failed %i\n", __func__, rc);
1075                                 goto out;
1076                         }
1077                         list_add(&r->mw_list, &buf->rb_mws);
1078                         ++r;
1079                 }
1080                 break;
1081         default:
1082                 break;
1083         }
1084
1085         /*
1086          * Allocate/init the request/reply buffers. Doing this
1087          * using kmalloc for now -- one for each buf.
1088          */
1089         for (i = 0; i < buf->rb_max_requests; i++) {
1090                 struct rpcrdma_req *req;
1091                 struct rpcrdma_rep *rep;
1092
1093                 len = cdata->inline_wsize + sizeof(struct rpcrdma_req);
1094                 /* RPC layer requests *double* size + 1K RPC_SLACK_SPACE! */
1095                 /* Typical ~2400b, so rounding up saves work later */
1096                 if (len < 4096)
1097                         len = 4096;
1098                 req = kmalloc(len, GFP_KERNEL);
1099                 if (req == NULL) {
1100                         dprintk("RPC:       %s: request buffer %d alloc"
1101                                 " failed\n", __func__, i);
1102                         rc = -ENOMEM;
1103                         goto out;
1104                 }
1105                 memset(req, 0, sizeof(struct rpcrdma_req));
1106                 buf->rb_send_bufs[i] = req;
1107                 buf->rb_send_bufs[i]->rl_buffer = buf;
1108
1109                 rc = rpcrdma_register_internal(ia, req->rl_base,
1110                                 len - offsetof(struct rpcrdma_req, rl_base),
1111                                 &buf->rb_send_bufs[i]->rl_handle,
1112                                 &buf->rb_send_bufs[i]->rl_iov);
1113                 if (rc)
1114                         goto out;
1115
1116                 buf->rb_send_bufs[i]->rl_size = len-sizeof(struct rpcrdma_req);
1117
1118                 len = cdata->inline_rsize + sizeof(struct rpcrdma_rep);
1119                 rep = kmalloc(len, GFP_KERNEL);
1120                 if (rep == NULL) {
1121                         dprintk("RPC:       %s: reply buffer %d alloc failed\n",
1122                                 __func__, i);
1123                         rc = -ENOMEM;
1124                         goto out;
1125                 }
1126                 memset(rep, 0, sizeof(struct rpcrdma_rep));
1127                 buf->rb_recv_bufs[i] = rep;
1128                 buf->rb_recv_bufs[i]->rr_buffer = buf;
1129                 init_waitqueue_head(&rep->rr_unbind);
1130
1131                 rc = rpcrdma_register_internal(ia, rep->rr_base,
1132                                 len - offsetof(struct rpcrdma_rep, rr_base),
1133                                 &buf->rb_recv_bufs[i]->rr_handle,
1134                                 &buf->rb_recv_bufs[i]->rr_iov);
1135                 if (rc)
1136                         goto out;
1137
1138         }
1139         dprintk("RPC:       %s: max_requests %d\n",
1140                 __func__, buf->rb_max_requests);
1141         /* done */
1142         return 0;
1143 out:
1144         rpcrdma_buffer_destroy(buf);
1145         return rc;
1146 }
1147
1148 /*
1149  * Unregister and destroy buffer memory. Need to deal with
1150  * partial initialization, so it's callable from failed create.
1151  * Must be called before destroying endpoint, as registrations
1152  * reference it.
1153  */
1154 void
1155 rpcrdma_buffer_destroy(struct rpcrdma_buffer *buf)
1156 {
1157         int rc, i;
1158         struct rpcrdma_ia *ia = rdmab_to_ia(buf);
1159         struct rpcrdma_mw *r;
1160
1161         /* clean up in reverse order from create
1162          *   1.  recv mr memory (mr free, then kfree)
1163          *   1a. bind mw memory
1164          *   2.  send mr memory (mr free, then kfree)
1165          *   3.  padding (if any) [moved to rpcrdma_ep_destroy]
1166          *   4.  arrays
1167          */
1168         dprintk("RPC:       %s: entering\n", __func__);
1169
1170         for (i = 0; i < buf->rb_max_requests; i++) {
1171                 if (buf->rb_recv_bufs && buf->rb_recv_bufs[i]) {
1172                         rpcrdma_deregister_internal(ia,
1173                                         buf->rb_recv_bufs[i]->rr_handle,
1174                                         &buf->rb_recv_bufs[i]->rr_iov);
1175                         kfree(buf->rb_recv_bufs[i]);
1176                 }
1177                 if (buf->rb_send_bufs && buf->rb_send_bufs[i]) {
1178                         while (!list_empty(&buf->rb_mws)) {
1179                                 r = list_entry(buf->rb_mws.next,
1180                                         struct rpcrdma_mw, mw_list);
1181                                 list_del(&r->mw_list);
1182                                 switch (ia->ri_memreg_strategy) {
1183                                 case RPCRDMA_FRMR:
1184                                         rc = ib_dereg_mr(r->r.frmr.fr_mr);
1185                                         if (rc)
1186                                                 dprintk("RPC:       %s:"
1187                                                         " ib_dereg_mr"
1188                                                         " failed %i\n",
1189                                                         __func__, rc);
1190                                         ib_free_fast_reg_page_list(r->r.frmr.fr_pgl);
1191                                         break;
1192                                 case RPCRDMA_MTHCAFMR:
1193                                         rc = ib_dealloc_fmr(r->r.fmr);
1194                                         if (rc)
1195                                                 dprintk("RPC:       %s:"
1196                                                         " ib_dealloc_fmr"
1197                                                         " failed %i\n",
1198                                                         __func__, rc);
1199                                         break;
1200                                 case RPCRDMA_MEMWINDOWS_ASYNC:
1201                                 case RPCRDMA_MEMWINDOWS:
1202                                         rc = ib_dealloc_mw(r->r.mw);
1203                                         if (rc)
1204                                                 dprintk("RPC:       %s:"
1205                                                         " ib_dealloc_mw"
1206                                                         " failed %i\n",
1207                                                         __func__, rc);
1208                                         break;
1209                                 default:
1210                                         break;
1211                                 }
1212                         }
1213                         rpcrdma_deregister_internal(ia,
1214                                         buf->rb_send_bufs[i]->rl_handle,
1215                                         &buf->rb_send_bufs[i]->rl_iov);
1216                         kfree(buf->rb_send_bufs[i]);
1217                 }
1218         }
1219
1220         kfree(buf->rb_pool);
1221 }
1222
1223 /*
1224  * Get a set of request/reply buffers.
1225  *
1226  * Reply buffer (if needed) is attached to send buffer upon return.
1227  * Rule:
1228  *    rb_send_index and rb_recv_index MUST always be pointing to the
1229  *    *next* available buffer (non-NULL). They are incremented after
1230  *    removing buffers, and decremented *before* returning them.
1231  */
1232 struct rpcrdma_req *
1233 rpcrdma_buffer_get(struct rpcrdma_buffer *buffers)
1234 {
1235         struct rpcrdma_req *req;
1236         unsigned long flags;
1237         int i;
1238         struct rpcrdma_mw *r;
1239
1240         spin_lock_irqsave(&buffers->rb_lock, flags);
1241         if (buffers->rb_send_index == buffers->rb_max_requests) {
1242                 spin_unlock_irqrestore(&buffers->rb_lock, flags);
1243                 dprintk("RPC:       %s: out of request buffers\n", __func__);
1244                 return ((struct rpcrdma_req *)NULL);
1245         }
1246
1247         req = buffers->rb_send_bufs[buffers->rb_send_index];
1248         if (buffers->rb_send_index < buffers->rb_recv_index) {
1249                 dprintk("RPC:       %s: %d extra receives outstanding (ok)\n",
1250                         __func__,
1251                         buffers->rb_recv_index - buffers->rb_send_index);
1252                 req->rl_reply = NULL;
1253         } else {
1254                 req->rl_reply = buffers->rb_recv_bufs[buffers->rb_recv_index];
1255                 buffers->rb_recv_bufs[buffers->rb_recv_index++] = NULL;
1256         }
1257         buffers->rb_send_bufs[buffers->rb_send_index++] = NULL;
1258         if (!list_empty(&buffers->rb_mws)) {
1259                 i = RPCRDMA_MAX_SEGS - 1;
1260                 do {
1261                         r = list_entry(buffers->rb_mws.next,
1262                                         struct rpcrdma_mw, mw_list);
1263                         list_del(&r->mw_list);
1264                         req->rl_segments[i].mr_chunk.rl_mw = r;
1265                 } while (--i >= 0);
1266         }
1267         spin_unlock_irqrestore(&buffers->rb_lock, flags);
1268         return req;
1269 }
1270
1271 /*
1272  * Put request/reply buffers back into pool.
1273  * Pre-decrement counter/array index.
1274  */
1275 void
1276 rpcrdma_buffer_put(struct rpcrdma_req *req)
1277 {
1278         struct rpcrdma_buffer *buffers = req->rl_buffer;
1279         struct rpcrdma_ia *ia = rdmab_to_ia(buffers);
1280         int i;
1281         unsigned long flags;
1282
1283         BUG_ON(req->rl_nchunks != 0);
1284         spin_lock_irqsave(&buffers->rb_lock, flags);
1285         buffers->rb_send_bufs[--buffers->rb_send_index] = req;
1286         req->rl_niovs = 0;
1287         if (req->rl_reply) {
1288                 buffers->rb_recv_bufs[--buffers->rb_recv_index] = req->rl_reply;
1289                 init_waitqueue_head(&req->rl_reply->rr_unbind);
1290                 req->rl_reply->rr_func = NULL;
1291                 req->rl_reply = NULL;
1292         }
1293         switch (ia->ri_memreg_strategy) {
1294         case RPCRDMA_FRMR:
1295         case RPCRDMA_MTHCAFMR:
1296         case RPCRDMA_MEMWINDOWS_ASYNC:
1297         case RPCRDMA_MEMWINDOWS:
1298                 /*
1299                  * Cycle mw's back in reverse order, and "spin" them.
1300                  * This delays and scrambles reuse as much as possible.
1301                  */
1302                 i = 1;
1303                 do {
1304                         struct rpcrdma_mw **mw;
1305                         mw = &req->rl_segments[i].mr_chunk.rl_mw;
1306                         list_add_tail(&(*mw)->mw_list, &buffers->rb_mws);
1307                         *mw = NULL;
1308                 } while (++i < RPCRDMA_MAX_SEGS);
1309                 list_add_tail(&req->rl_segments[0].mr_chunk.rl_mw->mw_list,
1310                                         &buffers->rb_mws);
1311                 req->rl_segments[0].mr_chunk.rl_mw = NULL;
1312                 break;
1313         default:
1314                 break;
1315         }
1316         spin_unlock_irqrestore(&buffers->rb_lock, flags);
1317 }
1318
1319 /*
1320  * Recover reply buffers from pool.
1321  * This happens when recovering from error conditions.
1322  * Post-increment counter/array index.
1323  */
1324 void
1325 rpcrdma_recv_buffer_get(struct rpcrdma_req *req)
1326 {
1327         struct rpcrdma_buffer *buffers = req->rl_buffer;
1328         unsigned long flags;
1329
1330         if (req->rl_iov.length == 0)    /* special case xprt_rdma_allocate() */
1331                 buffers = ((struct rpcrdma_req *) buffers)->rl_buffer;
1332         spin_lock_irqsave(&buffers->rb_lock, flags);
1333         if (buffers->rb_recv_index < buffers->rb_max_requests) {
1334                 req->rl_reply = buffers->rb_recv_bufs[buffers->rb_recv_index];
1335                 buffers->rb_recv_bufs[buffers->rb_recv_index++] = NULL;
1336         }
1337         spin_unlock_irqrestore(&buffers->rb_lock, flags);
1338 }
1339
1340 /*
1341  * Put reply buffers back into pool when not attached to
1342  * request. This happens in error conditions, and when
1343  * aborting unbinds. Pre-decrement counter/array index.
1344  */
1345 void
1346 rpcrdma_recv_buffer_put(struct rpcrdma_rep *rep)
1347 {
1348         struct rpcrdma_buffer *buffers = rep->rr_buffer;
1349         unsigned long flags;
1350
1351         rep->rr_func = NULL;
1352         spin_lock_irqsave(&buffers->rb_lock, flags);
1353         buffers->rb_recv_bufs[--buffers->rb_recv_index] = rep;
1354         spin_unlock_irqrestore(&buffers->rb_lock, flags);
1355 }
1356
1357 /*
1358  * Wrappers for internal-use kmalloc memory registration, used by buffer code.
1359  */
1360
1361 int
1362 rpcrdma_register_internal(struct rpcrdma_ia *ia, void *va, int len,
1363                                 struct ib_mr **mrp, struct ib_sge *iov)
1364 {
1365         struct ib_phys_buf ipb;
1366         struct ib_mr *mr;
1367         int rc;
1368
1369         /*
1370          * All memory passed here was kmalloc'ed, therefore phys-contiguous.
1371          */
1372         iov->addr = ib_dma_map_single(ia->ri_id->device,
1373                         va, len, DMA_BIDIRECTIONAL);
1374         iov->length = len;
1375
1376         if (ia->ri_have_dma_lkey) {
1377                 *mrp = NULL;
1378                 iov->lkey = ia->ri_dma_lkey;
1379                 return 0;
1380         } else if (ia->ri_bind_mem != NULL) {
1381                 *mrp = NULL;
1382                 iov->lkey = ia->ri_bind_mem->lkey;
1383                 return 0;
1384         }
1385
1386         ipb.addr = iov->addr;
1387         ipb.size = iov->length;
1388         mr = ib_reg_phys_mr(ia->ri_pd, &ipb, 1,
1389                         IB_ACCESS_LOCAL_WRITE, &iov->addr);
1390
1391         dprintk("RPC:       %s: phys convert: 0x%llx "
1392                         "registered 0x%llx length %d\n",
1393                         __func__, (unsigned long long)ipb.addr,
1394                         (unsigned long long)iov->addr, len);
1395
1396         if (IS_ERR(mr)) {
1397                 *mrp = NULL;
1398                 rc = PTR_ERR(mr);
1399                 dprintk("RPC:       %s: failed with %i\n", __func__, rc);
1400         } else {
1401                 *mrp = mr;
1402                 iov->lkey = mr->lkey;
1403                 rc = 0;
1404         }
1405
1406         return rc;
1407 }
1408
1409 int
1410 rpcrdma_deregister_internal(struct rpcrdma_ia *ia,
1411                                 struct ib_mr *mr, struct ib_sge *iov)
1412 {
1413         int rc;
1414
1415         ib_dma_unmap_single(ia->ri_id->device,
1416                         iov->addr, iov->length, DMA_BIDIRECTIONAL);
1417
1418         if (NULL == mr)
1419                 return 0;
1420
1421         rc = ib_dereg_mr(mr);
1422         if (rc)
1423                 dprintk("RPC:       %s: ib_dereg_mr failed %i\n", __func__, rc);
1424         return rc;
1425 }
1426
1427 /*
1428  * Wrappers for chunk registration, shared by read/write chunk code.
1429  */
1430
1431 static void
1432 rpcrdma_map_one(struct rpcrdma_ia *ia, struct rpcrdma_mr_seg *seg, int writing)
1433 {
1434         seg->mr_dir = writing ? DMA_FROM_DEVICE : DMA_TO_DEVICE;
1435         seg->mr_dmalen = seg->mr_len;
1436         if (seg->mr_page)
1437                 seg->mr_dma = ib_dma_map_page(ia->ri_id->device,
1438                                 seg->mr_page, offset_in_page(seg->mr_offset),
1439                                 seg->mr_dmalen, seg->mr_dir);
1440         else
1441                 seg->mr_dma = ib_dma_map_single(ia->ri_id->device,
1442                                 seg->mr_offset,
1443                                 seg->mr_dmalen, seg->mr_dir);
1444 }
1445
1446 static void
1447 rpcrdma_unmap_one(struct rpcrdma_ia *ia, struct rpcrdma_mr_seg *seg)
1448 {
1449         if (seg->mr_page)
1450                 ib_dma_unmap_page(ia->ri_id->device,
1451                                 seg->mr_dma, seg->mr_dmalen, seg->mr_dir);
1452         else
1453                 ib_dma_unmap_single(ia->ri_id->device,
1454                                 seg->mr_dma, seg->mr_dmalen, seg->mr_dir);
1455 }
1456
1457 static int
1458 rpcrdma_register_frmr_external(struct rpcrdma_mr_seg *seg,
1459                         int *nsegs, int writing, struct rpcrdma_ia *ia,
1460                         struct rpcrdma_xprt *r_xprt)
1461 {
1462         struct rpcrdma_mr_seg *seg1 = seg;
1463         struct ib_send_wr frmr_wr, *bad_wr;
1464         u8 key;
1465         int len, pageoff;
1466         int i, rc;
1467
1468         pageoff = offset_in_page(seg1->mr_offset);
1469         seg1->mr_offset -= pageoff;     /* start of page */
1470         seg1->mr_len += pageoff;
1471         len = -pageoff;
1472         if (*nsegs > RPCRDMA_MAX_DATA_SEGS)
1473                 *nsegs = RPCRDMA_MAX_DATA_SEGS;
1474         for (i = 0; i < *nsegs;) {
1475                 rpcrdma_map_one(ia, seg, writing);
1476                 seg1->mr_chunk.rl_mw->r.frmr.fr_pgl->page_list[i] = seg->mr_dma;
1477                 len += seg->mr_len;
1478                 ++seg;
1479                 ++i;
1480                 /* Check for holes */
1481                 if ((i < *nsegs && offset_in_page(seg->mr_offset)) ||
1482                     offset_in_page((seg-1)->mr_offset + (seg-1)->mr_len))
1483                         break;
1484         }
1485         dprintk("RPC:       %s: Using frmr %p to map %d segments\n",
1486                 __func__, seg1->mr_chunk.rl_mw, i);
1487
1488         /* Bump the key */
1489         key = (u8)(seg1->mr_chunk.rl_mw->r.frmr.fr_mr->rkey & 0x000000FF);
1490         ib_update_fast_reg_key(seg1->mr_chunk.rl_mw->r.frmr.fr_mr, ++key);
1491
1492         /* Prepare FRMR WR */
1493         memset(&frmr_wr, 0, sizeof frmr_wr);
1494         frmr_wr.opcode = IB_WR_FAST_REG_MR;
1495         frmr_wr.send_flags = 0;                 /* unsignaled */
1496         frmr_wr.wr.fast_reg.iova_start = (unsigned long)seg1->mr_dma;
1497         frmr_wr.wr.fast_reg.page_list = seg1->mr_chunk.rl_mw->r.frmr.fr_pgl;
1498         frmr_wr.wr.fast_reg.page_list_len = i;
1499         frmr_wr.wr.fast_reg.page_shift = PAGE_SHIFT;
1500         frmr_wr.wr.fast_reg.length = i << PAGE_SHIFT;
1501         frmr_wr.wr.fast_reg.access_flags = (writing ?
1502                                 IB_ACCESS_REMOTE_WRITE : IB_ACCESS_REMOTE_READ);
1503         frmr_wr.wr.fast_reg.rkey = seg1->mr_chunk.rl_mw->r.frmr.fr_mr->rkey;
1504         DECR_CQCOUNT(&r_xprt->rx_ep);
1505
1506         rc = ib_post_send(ia->ri_id->qp, &frmr_wr, &bad_wr);
1507
1508         if (rc) {
1509                 dprintk("RPC:       %s: failed ib_post_send for register,"
1510                         " status %i\n", __func__, rc);
1511                 while (i--)
1512                         rpcrdma_unmap_one(ia, --seg);
1513         } else {
1514                 seg1->mr_rkey = seg1->mr_chunk.rl_mw->r.frmr.fr_mr->rkey;
1515                 seg1->mr_base = seg1->mr_dma + pageoff;
1516                 seg1->mr_nsegs = i;
1517                 seg1->mr_len = len;
1518         }
1519         *nsegs = i;
1520         return rc;
1521 }
1522
1523 static int
1524 rpcrdma_deregister_frmr_external(struct rpcrdma_mr_seg *seg,
1525                         struct rpcrdma_ia *ia, struct rpcrdma_xprt *r_xprt)
1526 {
1527         struct rpcrdma_mr_seg *seg1 = seg;
1528         struct ib_send_wr invalidate_wr, *bad_wr;
1529         int rc;
1530
1531         while (seg1->mr_nsegs--)
1532                 rpcrdma_unmap_one(ia, seg++);
1533
1534         memset(&invalidate_wr, 0, sizeof invalidate_wr);
1535         invalidate_wr.opcode = IB_WR_LOCAL_INV;
1536         invalidate_wr.send_flags = 0;                   /* unsignaled */
1537         invalidate_wr.ex.invalidate_rkey = seg1->mr_chunk.rl_mw->r.frmr.fr_mr->rkey;
1538         DECR_CQCOUNT(&r_xprt->rx_ep);
1539
1540         rc = ib_post_send(ia->ri_id->qp, &invalidate_wr, &bad_wr);
1541         if (rc)
1542                 dprintk("RPC:       %s: failed ib_post_send for invalidate,"
1543                         " status %i\n", __func__, rc);
1544         return rc;
1545 }
1546
1547 static int
1548 rpcrdma_register_fmr_external(struct rpcrdma_mr_seg *seg,
1549                         int *nsegs, int writing, struct rpcrdma_ia *ia)
1550 {
1551         struct rpcrdma_mr_seg *seg1 = seg;
1552         u64 physaddrs[RPCRDMA_MAX_DATA_SEGS];
1553         int len, pageoff, i, rc;
1554
1555         pageoff = offset_in_page(seg1->mr_offset);
1556         seg1->mr_offset -= pageoff;     /* start of page */
1557         seg1->mr_len += pageoff;
1558         len = -pageoff;
1559         if (*nsegs > RPCRDMA_MAX_DATA_SEGS)
1560                 *nsegs = RPCRDMA_MAX_DATA_SEGS;
1561         for (i = 0; i < *nsegs;) {
1562                 rpcrdma_map_one(ia, seg, writing);
1563                 physaddrs[i] = seg->mr_dma;
1564                 len += seg->mr_len;
1565                 ++seg;
1566                 ++i;
1567                 /* Check for holes */
1568                 if ((i < *nsegs && offset_in_page(seg->mr_offset)) ||
1569                     offset_in_page((seg-1)->mr_offset + (seg-1)->mr_len))
1570                         break;
1571         }
1572         rc = ib_map_phys_fmr(seg1->mr_chunk.rl_mw->r.fmr,
1573                                 physaddrs, i, seg1->mr_dma);
1574         if (rc) {
1575                 dprintk("RPC:       %s: failed ib_map_phys_fmr "
1576                         "%u@0x%llx+%i (%d)... status %i\n", __func__,
1577                         len, (unsigned long long)seg1->mr_dma,
1578                         pageoff, i, rc);
1579                 while (i--)
1580                         rpcrdma_unmap_one(ia, --seg);
1581         } else {
1582                 seg1->mr_rkey = seg1->mr_chunk.rl_mw->r.fmr->rkey;
1583                 seg1->mr_base = seg1->mr_dma + pageoff;
1584                 seg1->mr_nsegs = i;
1585                 seg1->mr_len = len;
1586         }
1587         *nsegs = i;
1588         return rc;
1589 }
1590
1591 static int
1592 rpcrdma_deregister_fmr_external(struct rpcrdma_mr_seg *seg,
1593                         struct rpcrdma_ia *ia)
1594 {
1595         struct rpcrdma_mr_seg *seg1 = seg;
1596         LIST_HEAD(l);
1597         int rc;
1598
1599         list_add(&seg1->mr_chunk.rl_mw->r.fmr->list, &l);
1600         rc = ib_unmap_fmr(&l);
1601         while (seg1->mr_nsegs--)
1602                 rpcrdma_unmap_one(ia, seg++);
1603         if (rc)
1604                 dprintk("RPC:       %s: failed ib_unmap_fmr,"
1605                         " status %i\n", __func__, rc);
1606         return rc;
1607 }
1608
1609 static int
1610 rpcrdma_register_memwin_external(struct rpcrdma_mr_seg *seg,
1611                         int *nsegs, int writing, struct rpcrdma_ia *ia,
1612                         struct rpcrdma_xprt *r_xprt)
1613 {
1614         int mem_priv = (writing ? IB_ACCESS_REMOTE_WRITE :
1615                                   IB_ACCESS_REMOTE_READ);
1616         struct ib_mw_bind param;
1617         int rc;
1618
1619         *nsegs = 1;
1620         rpcrdma_map_one(ia, seg, writing);
1621         param.mr = ia->ri_bind_mem;
1622         param.wr_id = 0ULL;     /* no send cookie */
1623         param.addr = seg->mr_dma;
1624         param.length = seg->mr_len;
1625         param.send_flags = 0;
1626         param.mw_access_flags = mem_priv;
1627
1628         DECR_CQCOUNT(&r_xprt->rx_ep);
1629         rc = ib_bind_mw(ia->ri_id->qp, seg->mr_chunk.rl_mw->r.mw, &param);
1630         if (rc) {
1631                 dprintk("RPC:       %s: failed ib_bind_mw "
1632                         "%u@0x%llx status %i\n",
1633                         __func__, seg->mr_len,
1634                         (unsigned long long)seg->mr_dma, rc);
1635                 rpcrdma_unmap_one(ia, seg);
1636         } else {
1637                 seg->mr_rkey = seg->mr_chunk.rl_mw->r.mw->rkey;
1638                 seg->mr_base = param.addr;
1639                 seg->mr_nsegs = 1;
1640         }
1641         return rc;
1642 }
1643
1644 static int
1645 rpcrdma_deregister_memwin_external(struct rpcrdma_mr_seg *seg,
1646                         struct rpcrdma_ia *ia,
1647                         struct rpcrdma_xprt *r_xprt, void **r)
1648 {
1649         struct ib_mw_bind param;
1650         LIST_HEAD(l);
1651         int rc;
1652
1653         BUG_ON(seg->mr_nsegs != 1);
1654         param.mr = ia->ri_bind_mem;
1655         param.addr = 0ULL;      /* unbind */
1656         param.length = 0;
1657         param.mw_access_flags = 0;
1658         if (*r) {
1659                 param.wr_id = (u64) (unsigned long) *r;
1660                 param.send_flags = IB_SEND_SIGNALED;
1661                 INIT_CQCOUNT(&r_xprt->rx_ep);
1662         } else {
1663                 param.wr_id = 0ULL;
1664                 param.send_flags = 0;
1665                 DECR_CQCOUNT(&r_xprt->rx_ep);
1666         }
1667         rc = ib_bind_mw(ia->ri_id->qp, seg->mr_chunk.rl_mw->r.mw, &param);
1668         rpcrdma_unmap_one(ia, seg);
1669         if (rc)
1670                 dprintk("RPC:       %s: failed ib_(un)bind_mw,"
1671                         " status %i\n", __func__, rc);
1672         else
1673                 *r = NULL;      /* will upcall on completion */
1674         return rc;
1675 }
1676
1677 static int
1678 rpcrdma_register_default_external(struct rpcrdma_mr_seg *seg,
1679                         int *nsegs, int writing, struct rpcrdma_ia *ia)
1680 {
1681         int mem_priv = (writing ? IB_ACCESS_REMOTE_WRITE :
1682                                   IB_ACCESS_REMOTE_READ);
1683         struct rpcrdma_mr_seg *seg1 = seg;
1684         struct ib_phys_buf ipb[RPCRDMA_MAX_DATA_SEGS];
1685         int len, i, rc = 0;
1686
1687         if (*nsegs > RPCRDMA_MAX_DATA_SEGS)
1688                 *nsegs = RPCRDMA_MAX_DATA_SEGS;
1689         for (len = 0, i = 0; i < *nsegs;) {
1690                 rpcrdma_map_one(ia, seg, writing);
1691                 ipb[i].addr = seg->mr_dma;
1692                 ipb[i].size = seg->mr_len;
1693                 len += seg->mr_len;
1694                 ++seg;
1695                 ++i;
1696                 /* Check for holes */
1697                 if ((i < *nsegs && offset_in_page(seg->mr_offset)) ||
1698                     offset_in_page((seg-1)->mr_offset+(seg-1)->mr_len))
1699                         break;
1700         }
1701         seg1->mr_base = seg1->mr_dma;
1702         seg1->mr_chunk.rl_mr = ib_reg_phys_mr(ia->ri_pd,
1703                                 ipb, i, mem_priv, &seg1->mr_base);
1704         if (IS_ERR(seg1->mr_chunk.rl_mr)) {
1705                 rc = PTR_ERR(seg1->mr_chunk.rl_mr);
1706                 dprintk("RPC:       %s: failed ib_reg_phys_mr "
1707                         "%u@0x%llx (%d)... status %i\n",
1708                         __func__, len,
1709                         (unsigned long long)seg1->mr_dma, i, rc);
1710                 while (i--)
1711                         rpcrdma_unmap_one(ia, --seg);
1712         } else {
1713                 seg1->mr_rkey = seg1->mr_chunk.rl_mr->rkey;
1714                 seg1->mr_nsegs = i;
1715                 seg1->mr_len = len;
1716         }
1717         *nsegs = i;
1718         return rc;
1719 }
1720
1721 static int
1722 rpcrdma_deregister_default_external(struct rpcrdma_mr_seg *seg,
1723                         struct rpcrdma_ia *ia)
1724 {
1725         struct rpcrdma_mr_seg *seg1 = seg;
1726         int rc;
1727
1728         rc = ib_dereg_mr(seg1->mr_chunk.rl_mr);
1729         seg1->mr_chunk.rl_mr = NULL;
1730         while (seg1->mr_nsegs--)
1731                 rpcrdma_unmap_one(ia, seg++);
1732         if (rc)
1733                 dprintk("RPC:       %s: failed ib_dereg_mr,"
1734                         " status %i\n", __func__, rc);
1735         return rc;
1736 }
1737
1738 int
1739 rpcrdma_register_external(struct rpcrdma_mr_seg *seg,
1740                         int nsegs, int writing, struct rpcrdma_xprt *r_xprt)
1741 {
1742         struct rpcrdma_ia *ia = &r_xprt->rx_ia;
1743         int rc = 0;
1744
1745         switch (ia->ri_memreg_strategy) {
1746
1747 #if RPCRDMA_PERSISTENT_REGISTRATION
1748         case RPCRDMA_ALLPHYSICAL:
1749                 rpcrdma_map_one(ia, seg, writing);
1750                 seg->mr_rkey = ia->ri_bind_mem->rkey;
1751                 seg->mr_base = seg->mr_dma;
1752                 seg->mr_nsegs = 1;
1753                 nsegs = 1;
1754                 break;
1755 #endif
1756
1757         /* Registration using frmr registration */
1758         case RPCRDMA_FRMR:
1759                 rc = rpcrdma_register_frmr_external(seg, &nsegs, writing, ia, r_xprt);
1760                 break;
1761
1762         /* Registration using fmr memory registration */
1763         case RPCRDMA_MTHCAFMR:
1764                 rc = rpcrdma_register_fmr_external(seg, &nsegs, writing, ia);
1765                 break;
1766
1767         /* Registration using memory windows */
1768         case RPCRDMA_MEMWINDOWS_ASYNC:
1769         case RPCRDMA_MEMWINDOWS:
1770                 rc = rpcrdma_register_memwin_external(seg, &nsegs, writing, ia, r_xprt);
1771                 break;
1772
1773         /* Default registration each time */
1774         default:
1775                 rc = rpcrdma_register_default_external(seg, &nsegs, writing, ia);
1776                 break;
1777         }
1778         if (rc)
1779                 return -1;
1780
1781         return nsegs;
1782 }
1783
1784 int
1785 rpcrdma_deregister_external(struct rpcrdma_mr_seg *seg,
1786                 struct rpcrdma_xprt *r_xprt, void *r)
1787 {
1788         struct rpcrdma_ia *ia = &r_xprt->rx_ia;
1789         int nsegs = seg->mr_nsegs, rc;
1790
1791         switch (ia->ri_memreg_strategy) {
1792
1793 #if RPCRDMA_PERSISTENT_REGISTRATION
1794         case RPCRDMA_ALLPHYSICAL:
1795                 BUG_ON(nsegs != 1);
1796                 rpcrdma_unmap_one(ia, seg);
1797                 rc = 0;
1798                 break;
1799 #endif
1800
1801         case RPCRDMA_FRMR:
1802                 rc = rpcrdma_deregister_frmr_external(seg, ia, r_xprt);
1803                 break;
1804
1805         case RPCRDMA_MTHCAFMR:
1806                 rc = rpcrdma_deregister_fmr_external(seg, ia);
1807                 break;
1808
1809         case RPCRDMA_MEMWINDOWS_ASYNC:
1810         case RPCRDMA_MEMWINDOWS:
1811                 rc = rpcrdma_deregister_memwin_external(seg, ia, r_xprt, &r);
1812                 break;
1813
1814         default:
1815                 rc = rpcrdma_deregister_default_external(seg, ia);
1816                 break;
1817         }
1818         if (r) {
1819                 struct rpcrdma_rep *rep = r;
1820                 void (*func)(struct rpcrdma_rep *) = rep->rr_func;
1821                 rep->rr_func = NULL;
1822                 func(rep);      /* dereg done, callback now */
1823         }
1824         return nsegs;
1825 }
1826
1827 /*
1828  * Prepost any receive buffer, then post send.
1829  *
1830  * Receive buffer is donated to hardware, reclaimed upon recv completion.
1831  */
1832 int
1833 rpcrdma_ep_post(struct rpcrdma_ia *ia,
1834                 struct rpcrdma_ep *ep,
1835                 struct rpcrdma_req *req)
1836 {
1837         struct ib_send_wr send_wr, *send_wr_fail;
1838         struct rpcrdma_rep *rep = req->rl_reply;
1839         int rc;
1840
1841         if (rep) {
1842                 rc = rpcrdma_ep_post_recv(ia, ep, rep);
1843                 if (rc)
1844                         goto out;
1845                 req->rl_reply = NULL;
1846         }
1847
1848         send_wr.next = NULL;
1849         send_wr.wr_id = 0ULL;   /* no send cookie */
1850         send_wr.sg_list = req->rl_send_iov;
1851         send_wr.num_sge = req->rl_niovs;
1852         send_wr.opcode = IB_WR_SEND;
1853         if (send_wr.num_sge == 4)       /* no need to sync any pad (constant) */
1854                 ib_dma_sync_single_for_device(ia->ri_id->device,
1855                         req->rl_send_iov[3].addr, req->rl_send_iov[3].length,
1856                         DMA_TO_DEVICE);
1857         ib_dma_sync_single_for_device(ia->ri_id->device,
1858                 req->rl_send_iov[1].addr, req->rl_send_iov[1].length,
1859                 DMA_TO_DEVICE);
1860         ib_dma_sync_single_for_device(ia->ri_id->device,
1861                 req->rl_send_iov[0].addr, req->rl_send_iov[0].length,
1862                 DMA_TO_DEVICE);
1863
1864         if (DECR_CQCOUNT(ep) > 0)
1865                 send_wr.send_flags = 0;
1866         else { /* Provider must take a send completion every now and then */
1867                 INIT_CQCOUNT(ep);
1868                 send_wr.send_flags = IB_SEND_SIGNALED;
1869         }
1870
1871         rc = ib_post_send(ia->ri_id->qp, &send_wr, &send_wr_fail);
1872         if (rc)
1873                 dprintk("RPC:       %s: ib_post_send returned %i\n", __func__,
1874                         rc);
1875 out:
1876         return rc;
1877 }
1878
1879 /*
1880  * (Re)post a receive buffer.
1881  */
1882 int
1883 rpcrdma_ep_post_recv(struct rpcrdma_ia *ia,
1884                      struct rpcrdma_ep *ep,
1885                      struct rpcrdma_rep *rep)
1886 {
1887         struct ib_recv_wr recv_wr, *recv_wr_fail;
1888         int rc;
1889
1890         recv_wr.next = NULL;
1891         recv_wr.wr_id = (u64) (unsigned long) rep;
1892         recv_wr.sg_list = &rep->rr_iov;
1893         recv_wr.num_sge = 1;
1894
1895         ib_dma_sync_single_for_cpu(ia->ri_id->device,
1896                 rep->rr_iov.addr, rep->rr_iov.length, DMA_BIDIRECTIONAL);
1897
1898         DECR_CQCOUNT(ep);
1899         rc = ib_post_recv(ia->ri_id->qp, &recv_wr, &recv_wr_fail);
1900
1901         if (rc)
1902                 dprintk("RPC:       %s: ib_post_recv returned %i\n", __func__,
1903                         rc);
1904         return rc;
1905 }