SAFE public projects git trees. - safe/jmp/linux-2.6/blob - drivers/infiniband/hw/mthca/mthca_main.c

   1 /*
   2  * Copyright (c) 2004, 2005 Topspin Communications.  All rights reserved.
   3  * Copyright (c) 2005 Sun Microsystems, Inc. All rights reserved.
   4  * Copyright (c) 2005 Mellanox Technologies. All rights reserved.
   5  *
   6  * This software is available to you under a choice of one of two
   7  * licenses.  You may choose to be licensed under the terms of the GNU
   8  * General Public License (GPL) Version 2, available from the file
   9  * COPYING in the main directory of this source tree, or the
  10  * OpenIB.org BSD license below:
  11  *
  12  *     Redistribution and use in source and binary forms, with or
  13  *     without modification, are permitted provided that the following
  14  *     conditions are met:
  15  *
  16  *      - Redistributions of source code must retain the above
  17  *        copyright notice, this list of conditions and the following
  18  *        disclaimer.
  19  *
  20  *      - Redistributions in binary form must reproduce the above
  21  *        copyright notice, this list of conditions and the following
  22  *        disclaimer in the documentation and/or other materials
  23  *        provided with the distribution.
  24  *
  25  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  26  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  27  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  28  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  29  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  30  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  31  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  32  * SOFTWARE.
  33  *
  34  * $Id: mthca_main.c 1396 2004-12-28 04:10:27Z roland $
  35  */
  36
  37 #include <linux/config.h>
  38 #include <linux/module.h>
  39 #include <linux/init.h>
  40 #include <linux/errno.h>
  41 #include <linux/pci.h>
  42 #include <linux/interrupt.h>
  43
  44 #include "mthca_dev.h"
  45 #include "mthca_config_reg.h"
  46 #include "mthca_cmd.h"
  47 #include "mthca_profile.h"
  48 #include "mthca_memfree.h"
  49
  50 MODULE_AUTHOR("Roland Dreier");
  51 MODULE_DESCRIPTION("Mellanox InfiniBand HCA low-level driver");
  52 MODULE_LICENSE("Dual BSD/GPL");
  53 MODULE_VERSION(DRV_VERSION);
  54
  55 #ifdef CONFIG_INFINIBAND_MTHCA_DEBUG
  56
  57 int mthca_debug_level = 0;
  58 module_param_named(debug_level, mthca_debug_level, int, 0644);
  59 MODULE_PARM_DESC(debug_level, "Enable debug tracing if > 0");
  60
  61 #endif /* CONFIG_INFINIBAND_MTHCA_DEBUG */
  62
  63 #ifdef CONFIG_PCI_MSI
  64
  65 static int msi_x = 0;
  66 module_param(msi_x, int, 0444);
  67 MODULE_PARM_DESC(msi_x, "attempt to use MSI-X if nonzero");
  68
  69 static int msi = 0;
  70 module_param(msi, int, 0444);
  71 MODULE_PARM_DESC(msi, "attempt to use MSI if nonzero");
  72
  73 #else /* CONFIG_PCI_MSI */
  74
  75 #define msi_x (0)
  76 #define msi   (0)
  77
  78 #endif /* CONFIG_PCI_MSI */
  79
  80 static const char mthca_version[] __devinitdata =
  81         DRV_NAME ": Mellanox InfiniBand HCA driver v"
  82         DRV_VERSION " (" DRV_RELDATE ")\n";
  83
  84 static struct mthca_profile default_profile = {
  85         .num_qp            = 1 << 16,
  86         .rdb_per_qp        = 4,
  87         .num_cq            = 1 << 16,
  88         .num_mcg           = 1 << 13,
  89         .num_mpt           = 1 << 17,
  90         .num_mtt           = 1 << 20,
  91         .num_udav          = 1 << 15,   /* Tavor only */
  92         .fmr_reserved_mtts = 1 << 18,   /* Tavor only */
  93         .uarc_size         = 1 << 18,   /* Arbel only */
  94 };
  95
  96 static int __devinit mthca_tune_pci(struct mthca_dev *mdev)
  97 {
  98         int cap;
  99         u16 val;
 100
 101         /* First try to max out Read Byte Count */
 102         cap = pci_find_capability(mdev->pdev, PCI_CAP_ID_PCIX);
 103         if (cap) {
 104                 if (pci_read_config_word(mdev->pdev, cap + PCI_X_CMD, &val)) {
 105                         mthca_err(mdev, "Couldn't read PCI-X command register, "
 106                                   "aborting.\n");
 107                         return -ENODEV;
 108                 }
 109                 val = (val & ~PCI_X_CMD_MAX_READ) | (3 << 2);
 110                 if (pci_write_config_word(mdev->pdev, cap + PCI_X_CMD, val)) {
 111                         mthca_err(mdev, "Couldn't write PCI-X command register, "
 112                                   "aborting.\n");
 113                         return -ENODEV;
 114                 }
 115         } else if (!(mdev->mthca_flags & MTHCA_FLAG_PCIE))
 116                 mthca_info(mdev, "No PCI-X capability, not setting RBC.\n");
 117
 118         cap = pci_find_capability(mdev->pdev, PCI_CAP_ID_EXP);
 119         if (cap) {
 120                 if (pci_read_config_word(mdev->pdev, cap + PCI_EXP_DEVCTL, &val)) {
 121                         mthca_err(mdev, "Couldn't read PCI Express device control "
 122                                   "register, aborting.\n");
 123                         return -ENODEV;
 124                 }
 125                 val = (val & ~PCI_EXP_DEVCTL_READRQ) | (5 << 12);
 126                 if (pci_write_config_word(mdev->pdev, cap + PCI_EXP_DEVCTL, val)) {
 127                         mthca_err(mdev, "Couldn't write PCI Express device control "
 128                                   "register, aborting.\n");
 129                         return -ENODEV;
 130                 }
 131         } else if (mdev->mthca_flags & MTHCA_FLAG_PCIE)
 132                 mthca_info(mdev, "No PCI Express capability, "
 133                            "not setting Max Read Request Size.\n");
 134
 135         return 0;
 136 }
 137
 138 static int __devinit mthca_dev_lim(struct mthca_dev *mdev, struct mthca_dev_lim *dev_lim)
 139 {
 140         int err;
 141         u8 status;
 142
 143         err = mthca_QUERY_DEV_LIM(mdev, dev_lim, &status);
 144         if (err) {
 145                 mthca_err(mdev, "QUERY_DEV_LIM command failed, aborting.\n");
 146                 return err;
 147         }
 148         if (status) {
 149                 mthca_err(mdev, "QUERY_DEV_LIM returned status 0x%02x, "
 150                           "aborting.\n", status);
 151                 return -EINVAL;
 152         }
 153         if (dev_lim->min_page_sz > PAGE_SIZE) {
 154                 mthca_err(mdev, "HCA minimum page size of %d bigger than "
 155                           "kernel PAGE_SIZE of %ld, aborting.\n",
 156                           dev_lim->min_page_sz, PAGE_SIZE);
 157                 return -ENODEV;
 158         }
 159         if (dev_lim->num_ports > MTHCA_MAX_PORTS) {
 160                 mthca_err(mdev, "HCA has %d ports, but we only support %d, "
 161                           "aborting.\n",
 162                           dev_lim->num_ports, MTHCA_MAX_PORTS);
 163                 return -ENODEV;
 164         }
 165
 166         if (dev_lim->uar_size > pci_resource_len(mdev->pdev, 2)) {
 167                 mthca_err(mdev, "HCA reported UAR size of 0x%x bigger than "
 168                           "PCI resource 2 size of 0x%lx, aborting.\n",
 169                           dev_lim->uar_size, pci_resource_len(mdev->pdev, 2));
 170                 return -ENODEV;
 171         }
 172
 173         mdev->limits.num_ports          = dev_lim->num_ports;
 174         mdev->limits.vl_cap             = dev_lim->max_vl;
 175         mdev->limits.mtu_cap            = dev_lim->max_mtu;
 176         mdev->limits.gid_table_len      = dev_lim->max_gids;
 177         mdev->limits.pkey_table_len     = dev_lim->max_pkeys;
 178         mdev->limits.local_ca_ack_delay = dev_lim->local_ca_ack_delay;
 179         mdev->limits.max_sg             = dev_lim->max_sg;
 180         mdev->limits.max_wqes           = dev_lim->max_qp_sz;
 181         mdev->limits.max_qp_init_rdma   = dev_lim->max_requester_per_qp;
 182         mdev->limits.reserved_qps       = dev_lim->reserved_qps;
 183         mdev->limits.max_srq_wqes       = dev_lim->max_srq_sz;
 184         mdev->limits.reserved_srqs      = dev_lim->reserved_srqs;
 185         mdev->limits.reserved_eecs      = dev_lim->reserved_eecs;
 186         mdev->limits.max_desc_sz        = dev_lim->max_desc_sz;
 187         /*
 188          * Subtract 1 from the limit because we need to allocate a
 189          * spare CQE so the HCA HW can tell the difference between an
 190          * empty CQ and a full CQ.
 191          */
 192         mdev->limits.max_cqes           = dev_lim->max_cq_sz - 1;
 193         mdev->limits.reserved_cqs       = dev_lim->reserved_cqs;
 194         mdev->limits.reserved_eqs       = dev_lim->reserved_eqs;
 195         mdev->limits.reserved_mtts      = dev_lim->reserved_mtts;
 196         mdev->limits.reserved_mrws      = dev_lim->reserved_mrws;
 197         mdev->limits.reserved_uars      = dev_lim->reserved_uars;
 198         mdev->limits.reserved_pds       = dev_lim->reserved_pds;
 199         mdev->limits.port_width_cap     = dev_lim->max_port_width;
 200         mdev->limits.page_size_cap      = ~(u32) (dev_lim->min_page_sz - 1);
 201         mdev->limits.flags              = dev_lim->flags;
 202
 203         /* IB_DEVICE_RESIZE_MAX_WR not supported by driver.
 204            May be doable since hardware supports it for SRQ.
 205
 206            IB_DEVICE_N_NOTIFY_CQ is supported by hardware but not by driver.
 207
 208            IB_DEVICE_SRQ_RESIZE is supported by hardware but SRQ is not
 209            supported by driver. */
 210         mdev->device_cap_flags = IB_DEVICE_CHANGE_PHY_PORT |
 211                 IB_DEVICE_PORT_ACTIVE_EVENT |
 212                 IB_DEVICE_SYS_IMAGE_GUID |
 213                 IB_DEVICE_RC_RNR_NAK_GEN;
 214
 215         if (dev_lim->flags & DEV_LIM_FLAG_BAD_PKEY_CNTR)
 216                 mdev->device_cap_flags |= IB_DEVICE_BAD_PKEY_CNTR;
 217
 218         if (dev_lim->flags & DEV_LIM_FLAG_BAD_QKEY_CNTR)
 219                 mdev->device_cap_flags |= IB_DEVICE_BAD_QKEY_CNTR;
 220
 221         if (dev_lim->flags & DEV_LIM_FLAG_RAW_MULTI)
 222                 mdev->device_cap_flags |= IB_DEVICE_RAW_MULTI;
 223
 224         if (dev_lim->flags & DEV_LIM_FLAG_AUTO_PATH_MIG)
 225                 mdev->device_cap_flags |= IB_DEVICE_AUTO_PATH_MIG;
 226
 227         if (dev_lim->flags & DEV_LIM_FLAG_UD_AV_PORT_ENFORCE)
 228                 mdev->device_cap_flags |= IB_DEVICE_UD_AV_PORT_ENFORCE;
 229
 230         if (dev_lim->flags & DEV_LIM_FLAG_SRQ)
 231                 mdev->mthca_flags |= MTHCA_FLAG_SRQ;
 232
 233         return 0;
 234 }
 235
 236 static int __devinit mthca_init_tavor(struct mthca_dev *mdev)
 237 {
 238         u8 status;
 239         int err;
 240         struct mthca_dev_lim        dev_lim;
 241         struct mthca_profile        profile;
 242         struct mthca_init_hca_param init_hca;
 243
 244         err = mthca_SYS_EN(mdev, &status);
 245         if (err) {
 246                 mthca_err(mdev, "SYS_EN command failed, aborting.\n");
 247                 return err;
 248         }
 249         if (status) {
 250                 mthca_err(mdev, "SYS_EN returned status 0x%02x, "
 251                           "aborting.\n", status);
 252                 return -EINVAL;
 253         }
 254
 255         err = mthca_QUERY_FW(mdev, &status);
 256         if (err) {
 257                 mthca_err(mdev, "QUERY_FW command failed, aborting.\n");
 258                 goto err_disable;
 259         }
 260         if (status) {
 261                 mthca_err(mdev, "QUERY_FW returned status 0x%02x, "
 262                           "aborting.\n", status);
 263                 err = -EINVAL;
 264                 goto err_disable;
 265         }
 266         err = mthca_QUERY_DDR(mdev, &status);
 267         if (err) {
 268                 mthca_err(mdev, "QUERY_DDR command failed, aborting.\n");
 269                 goto err_disable;
 270         }
 271         if (status) {
 272                 mthca_err(mdev, "QUERY_DDR returned status 0x%02x, "
 273                           "aborting.\n", status);
 274                 err = -EINVAL;
 275                 goto err_disable;
 276         }
 277
 278         err = mthca_dev_lim(mdev, &dev_lim);
 279         if (err) {
 280                 mthca_err(mdev, "QUERY_DEV_LIM command failed, aborting.\n");
 281                 goto err_disable;
 282         }
 283
 284         profile = default_profile;
 285         profile.num_uar   = dev_lim.uar_size / PAGE_SIZE;
 286         profile.uarc_size = 0;
 287         if (mdev->mthca_flags & MTHCA_FLAG_SRQ)
 288                 profile.num_srq = dev_lim.max_srqs;
 289
 290         err = mthca_make_profile(mdev, &profile, &dev_lim, &init_hca);
 291         if (err < 0)
 292                 goto err_disable;
 293
 294         err = mthca_INIT_HCA(mdev, &init_hca, &status);
 295         if (err) {
 296                 mthca_err(mdev, "INIT_HCA command failed, aborting.\n");
 297                 goto err_disable;
 298         }
 299         if (status) {
 300                 mthca_err(mdev, "INIT_HCA returned status 0x%02x, "
 301                           "aborting.\n", status);
 302                 err = -EINVAL;
 303                 goto err_disable;
 304         }
 305
 306         return 0;
 307
 308 err_disable:
 309         mthca_SYS_DIS(mdev, &status);
 310
 311         return err;
 312 }
 313
 314 static int __devinit mthca_load_fw(struct mthca_dev *mdev)
 315 {
 316         u8 status;
 317         int err;
 318
 319         /* FIXME: use HCA-attached memory for FW if present */
 320
 321         mdev->fw.arbel.fw_icm =
 322                 mthca_alloc_icm(mdev, mdev->fw.arbel.fw_pages,
 323                                 GFP_HIGHUSER | __GFP_NOWARN);
 324         if (!mdev->fw.arbel.fw_icm) {
 325                 mthca_err(mdev, "Couldn't allocate FW area, aborting.\n");
 326                 return -ENOMEM;
 327         }
 328
 329         err = mthca_MAP_FA(mdev, mdev->fw.arbel.fw_icm, &status);
 330         if (err) {
 331                 mthca_err(mdev, "MAP_FA command failed, aborting.\n");
 332                 goto err_free;
 333         }
 334         if (status) {
 335                 mthca_err(mdev, "MAP_FA returned status 0x%02x, aborting.\n", status);
 336                 err = -EINVAL;
 337                 goto err_free;
 338         }
 339         err = mthca_RUN_FW(mdev, &status);
 340         if (err) {
 341                 mthca_err(mdev, "RUN_FW command failed, aborting.\n");
 342                 goto err_unmap_fa;
 343         }
 344         if (status) {
 345                 mthca_err(mdev, "RUN_FW returned status 0x%02x, aborting.\n", status);
 346                 err = -EINVAL;
 347                 goto err_unmap_fa;
 348         }
 349
 350         return 0;
 351
 352 err_unmap_fa:
 353         mthca_UNMAP_FA(mdev, &status);
 354
 355 err_free:
 356         mthca_free_icm(mdev, mdev->fw.arbel.fw_icm);
 357         return err;
 358 }
 359
 360 static int __devinit mthca_init_icm(struct mthca_dev *mdev,
 361                                     struct mthca_dev_lim *dev_lim,
 362                                     struct mthca_init_hca_param *init_hca,
 363                                     u64 icm_size)
 364 {
 365         u64 aux_pages;
 366         u8 status;
 367         int err;
 368
 369         err = mthca_SET_ICM_SIZE(mdev, icm_size, &aux_pages, &status);
 370         if (err) {
 371                 mthca_err(mdev, "SET_ICM_SIZE command failed, aborting.\n");
 372                 return err;
 373         }
 374         if (status) {
 375                 mthca_err(mdev, "SET_ICM_SIZE returned status 0x%02x, "
 376                           "aborting.\n", status);
 377                 return -EINVAL;
 378         }
 379
 380         mthca_dbg(mdev, "%lld KB of HCA context requires %lld KB aux memory.\n",
 381                   (unsigned long long) icm_size >> 10,
 382                   (unsigned long long) aux_pages << 2);
 383
 384         mdev->fw.arbel.aux_icm = mthca_alloc_icm(mdev, aux_pages,
 385                                                  GFP_HIGHUSER | __GFP_NOWARN);
 386         if (!mdev->fw.arbel.aux_icm) {
 387                 mthca_err(mdev, "Couldn't allocate aux memory, aborting.\n");
 388                 return -ENOMEM;
 389         }
 390
 391         err = mthca_MAP_ICM_AUX(mdev, mdev->fw.arbel.aux_icm, &status);
 392         if (err) {
 393                 mthca_err(mdev, "MAP_ICM_AUX command failed, aborting.\n");
 394                 goto err_free_aux;
 395         }
 396         if (status) {
 397                 mthca_err(mdev, "MAP_ICM_AUX returned status 0x%02x, aborting.\n", status);
 398                 err = -EINVAL;
 399                 goto err_free_aux;
 400         }
 401
 402         err = mthca_map_eq_icm(mdev, init_hca->eqc_base);
 403         if (err) {
 404                 mthca_err(mdev, "Failed to map EQ context memory, aborting.\n");
 405                 goto err_unmap_aux;
 406         }
 407
 408         mdev->mr_table.mtt_table = mthca_alloc_icm_table(mdev, init_hca->mtt_base,
 409                                                          MTHCA_MTT_SEG_SIZE,
 410                                                          mdev->limits.num_mtt_segs,
 411                                                          mdev->limits.reserved_mtts, 1);
 412         if (!mdev->mr_table.mtt_table) {
 413                 mthca_err(mdev, "Failed to map MTT context memory, aborting.\n");
 414                 err = -ENOMEM;
 415                 goto err_unmap_eq;
 416         }
 417
 418         mdev->mr_table.mpt_table = mthca_alloc_icm_table(mdev, init_hca->mpt_base,
 419                                                          dev_lim->mpt_entry_sz,
 420                                                          mdev->limits.num_mpts,
 421                                                          mdev->limits.reserved_mrws, 1);
 422         if (!mdev->mr_table.mpt_table) {
 423                 mthca_err(mdev, "Failed to map MPT context memory, aborting.\n");
 424                 err = -ENOMEM;
 425                 goto err_unmap_mtt;
 426         }
 427
 428         mdev->qp_table.qp_table = mthca_alloc_icm_table(mdev, init_hca->qpc_base,
 429                                                         dev_lim->qpc_entry_sz,
 430                                                         mdev->limits.num_qps,
 431                                                         mdev->limits.reserved_qps, 0);
 432         if (!mdev->qp_table.qp_table) {
 433                 mthca_err(mdev, "Failed to map QP context memory, aborting.\n");
 434                 err = -ENOMEM;
 435                 goto err_unmap_mpt;
 436         }
 437
 438         mdev->qp_table.eqp_table = mthca_alloc_icm_table(mdev, init_hca->eqpc_base,
 439                                                          dev_lim->eqpc_entry_sz,
 440                                                          mdev->limits.num_qps,
 441                                                          mdev->limits.reserved_qps, 0);
 442         if (!mdev->qp_table.eqp_table) {
 443                 mthca_err(mdev, "Failed to map EQP context memory, aborting.\n");
 444                 err = -ENOMEM;
 445                 goto err_unmap_qp;
 446         }
 447
 448         mdev->qp_table.rdb_table = mthca_alloc_icm_table(mdev, init_hca->rdb_base,
 449                                                          MTHCA_RDB_ENTRY_SIZE,
 450                                                          mdev->limits.num_qps <<
 451                                                          mdev->qp_table.rdb_shift,
 452                                                          0, 0);
 453         if (!mdev->qp_table.rdb_table) {
 454                 mthca_err(mdev, "Failed to map RDB context memory, aborting\n");
 455                 err = -ENOMEM;
 456                 goto err_unmap_eqp;
 457         }
 458
 459        mdev->cq_table.table = mthca_alloc_icm_table(mdev, init_hca->cqc_base,
 460                                                     dev_lim->cqc_entry_sz,
 461                                                     mdev->limits.num_cqs,
 462                                                     mdev->limits.reserved_cqs, 0);
 463         if (!mdev->cq_table.table) {
 464                 mthca_err(mdev, "Failed to map CQ context memory, aborting.\n");
 465                 err = -ENOMEM;
 466                 goto err_unmap_rdb;
 467         }
 468
 469         if (mdev->mthca_flags & MTHCA_FLAG_SRQ) {
 470                 mdev->srq_table.table =
 471                         mthca_alloc_icm_table(mdev, init_hca->srqc_base,
 472                                               dev_lim->srq_entry_sz,
 473                                               mdev->limits.num_srqs,
 474                                               mdev->limits.reserved_srqs, 0);
 475                 if (!mdev->srq_table.table) {
 476                         mthca_err(mdev, "Failed to map SRQ context memory, "
 477                                   "aborting.\n");
 478                         err = -ENOMEM;
 479                         goto err_unmap_cq;
 480                 }
 481         }
 482
 483         /*
 484          * It's not strictly required, but for simplicity just map the
 485          * whole multicast group table now.  The table isn't very big
 486          * and it's a lot easier than trying to track ref counts.
 487          */
 488         mdev->mcg_table.table = mthca_alloc_icm_table(mdev, init_hca->mc_base,
 489                                                       MTHCA_MGM_ENTRY_SIZE,
 490                                                       mdev->limits.num_mgms +
 491                                                       mdev->limits.num_amgms,
 492                                                       mdev->limits.num_mgms +
 493                                                       mdev->limits.num_amgms,
 494                                                       0);
 495         if (!mdev->mcg_table.table) {
 496                 mthca_err(mdev, "Failed to map MCG context memory, aborting.\n");
 497                 err = -ENOMEM;
 498                 goto err_unmap_srq;
 499         }
 500
 501         return 0;
 502
 503 err_unmap_srq:
 504         if (mdev->mthca_flags & MTHCA_FLAG_SRQ)
 505                 mthca_free_icm_table(mdev, mdev->srq_table.table);
 506
 507 err_unmap_cq:
 508         mthca_free_icm_table(mdev, mdev->cq_table.table);
 509
 510 err_unmap_rdb:
 511         mthca_free_icm_table(mdev, mdev->qp_table.rdb_table);
 512
 513 err_unmap_eqp:
 514         mthca_free_icm_table(mdev, mdev->qp_table.eqp_table);
 515
 516 err_unmap_qp:
 517         mthca_free_icm_table(mdev, mdev->qp_table.qp_table);
 518
 519 err_unmap_mpt:
 520         mthca_free_icm_table(mdev, mdev->mr_table.mpt_table);
 521
 522 err_unmap_mtt:
 523         mthca_free_icm_table(mdev, mdev->mr_table.mtt_table);
 524
 525 err_unmap_eq:
 526         mthca_unmap_eq_icm(mdev);
 527
 528 err_unmap_aux:
 529         mthca_UNMAP_ICM_AUX(mdev, &status);
 530
 531 err_free_aux:
 532         mthca_free_icm(mdev, mdev->fw.arbel.aux_icm);
 533
 534         return err;
 535 }
 536
 537 static void mthca_free_icms(struct mthca_dev *mdev)
 538 {
 539         u8 status;
 540
 541         mthca_free_icm_table(mdev, mdev->mcg_table.table);
 542         if (mdev->mthca_flags & MTHCA_FLAG_SRQ)
 543                 mthca_free_icm_table(mdev, mdev->srq_table.table);
 544         mthca_free_icm_table(mdev, mdev->cq_table.table);
 545         mthca_free_icm_table(mdev, mdev->qp_table.rdb_table);
 546         mthca_free_icm_table(mdev, mdev->qp_table.eqp_table);
 547         mthca_free_icm_table(mdev, mdev->qp_table.qp_table);
 548         mthca_free_icm_table(mdev, mdev->mr_table.mpt_table);
 549         mthca_free_icm_table(mdev, mdev->mr_table.mtt_table);
 550         mthca_unmap_eq_icm(mdev);
 551
 552         mthca_UNMAP_ICM_AUX(mdev, &status);
 553         mthca_free_icm(mdev, mdev->fw.arbel.aux_icm);
 554 }
 555
 556 static int __devinit mthca_init_arbel(struct mthca_dev *mdev)
 557 {
 558         struct mthca_dev_lim        dev_lim;
 559         struct mthca_profile        profile;
 560         struct mthca_init_hca_param init_hca;
 561         u64 icm_size;
 562         u8 status;
 563         int err;
 564
 565         err = mthca_QUERY_FW(mdev, &status);
 566         if (err) {
 567                 mthca_err(mdev, "QUERY_FW command failed, aborting.\n");
 568                 return err;
 569         }
 570         if (status) {
 571                 mthca_err(mdev, "QUERY_FW returned status 0x%02x, "
 572                           "aborting.\n", status);
 573                 return -EINVAL;
 574         }
 575
 576         err = mthca_ENABLE_LAM(mdev, &status);
 577         if (err) {
 578                 mthca_err(mdev, "ENABLE_LAM command failed, aborting.\n");
 579                 return err;
 580         }
 581         if (status == MTHCA_CMD_STAT_LAM_NOT_PRE) {
 582                 mthca_dbg(mdev, "No HCA-attached memory (running in MemFree mode)\n");
 583                 mdev->mthca_flags |= MTHCA_FLAG_NO_LAM;
 584         } else if (status) {
 585                 mthca_err(mdev, "ENABLE_LAM returned status 0x%02x, "
 586                           "aborting.\n", status);
 587                 return -EINVAL;
 588         }
 589
 590         err = mthca_load_fw(mdev);
 591         if (err) {
 592                 mthca_err(mdev, "Failed to start FW, aborting.\n");
 593                 goto err_disable;
 594         }
 595
 596         err = mthca_dev_lim(mdev, &dev_lim);
 597         if (err) {
 598                 mthca_err(mdev, "QUERY_DEV_LIM command failed, aborting.\n");
 599                 goto err_stop_fw;
 600         }
 601
 602         profile = default_profile;
 603         profile.num_uar  = dev_lim.uar_size / PAGE_SIZE;
 604         profile.num_udav = 0;
 605         if (mdev->mthca_flags & MTHCA_FLAG_SRQ)
 606                 profile.num_srq = dev_lim.max_srqs;
 607
 608         icm_size = mthca_make_profile(mdev, &profile, &dev_lim, &init_hca);
 609         if ((int) icm_size < 0) {
 610                 err = icm_size;
 611                 goto err_stop_fw;
 612         }
 613
 614         err = mthca_init_icm(mdev, &dev_lim, &init_hca, icm_size);
 615         if (err)
 616                 goto err_stop_fw;
 617
 618         err = mthca_INIT_HCA(mdev, &init_hca, &status);
 619         if (err) {
 620                 mthca_err(mdev, "INIT_HCA command failed, aborting.\n");
 621                 goto err_free_icm;
 622         }
 623         if (status) {
 624                 mthca_err(mdev, "INIT_HCA returned status 0x%02x, "
 625                           "aborting.\n", status);
 626                 err = -EINVAL;
 627                 goto err_free_icm;
 628         }
 629
 630         return 0;
 631
 632 err_free_icm:
 633         mthca_free_icms(mdev);
 634
 635 err_stop_fw:
 636         mthca_UNMAP_FA(mdev, &status);
 637         mthca_free_icm(mdev, mdev->fw.arbel.fw_icm);
 638
 639 err_disable:
 640         if (!(mdev->mthca_flags & MTHCA_FLAG_NO_LAM))
 641                 mthca_DISABLE_LAM(mdev, &status);
 642
 643         return err;
 644 }
 645
 646 static void mthca_close_hca(struct mthca_dev *mdev)
 647 {
 648         u8 status;
 649
 650         mthca_CLOSE_HCA(mdev, 0, &status);
 651
 652         if (mthca_is_memfree(mdev)) {
 653                 mthca_free_icms(mdev);
 654
 655                 mthca_UNMAP_FA(mdev, &status);
 656                 mthca_free_icm(mdev, mdev->fw.arbel.fw_icm);
 657
 658                 if (!(mdev->mthca_flags & MTHCA_FLAG_NO_LAM))
 659                         mthca_DISABLE_LAM(mdev, &status);
 660         } else
 661                 mthca_SYS_DIS(mdev, &status);
 662 }
 663
 664 static int __devinit mthca_init_hca(struct mthca_dev *mdev)
 665 {
 666         u8 status;
 667         int err;
 668         struct mthca_adapter adapter;
 669
 670         if (mthca_is_memfree(mdev))
 671                 err = mthca_init_arbel(mdev);
 672         else
 673                 err = mthca_init_tavor(mdev);
 674
 675         if (err)
 676                 return err;
 677
 678         err = mthca_QUERY_ADAPTER(mdev, &adapter, &status);
 679         if (err) {
 680                 mthca_err(mdev, "QUERY_ADAPTER command failed, aborting.\n");
 681                 goto err_close;
 682         }
 683         if (status) {
 684                 mthca_err(mdev, "QUERY_ADAPTER returned status 0x%02x, "
 685                           "aborting.\n", status);
 686                 err = -EINVAL;
 687                 goto err_close;
 688         }
 689
 690         mdev->eq_table.inta_pin = adapter.inta_pin;
 691         mdev->rev_id            = adapter.revision_id;
 692         memcpy(mdev->board_id, adapter.board_id, sizeof mdev->board_id);
 693
 694         return 0;
 695
 696 err_close:
 697         mthca_close_hca(mdev);
 698         return err;
 699 }
 700
 701 static int __devinit mthca_setup_hca(struct mthca_dev *dev)
 702 {
 703         int err;
 704         u8 status;
 705
 706         MTHCA_INIT_DOORBELL_LOCK(&dev->doorbell_lock);
 707
 708         err = mthca_init_uar_table(dev);
 709         if (err) {
 710                 mthca_err(dev, "Failed to initialize "
 711                           "user access region table, aborting.\n");
 712                 return err;
 713         }
 714
 715         err = mthca_uar_alloc(dev, &dev->driver_uar);
 716         if (err) {
 717                 mthca_err(dev, "Failed to allocate driver access region, "
 718                           "aborting.\n");
 719                 goto err_uar_table_free;
 720         }
 721
 722         dev->kar = ioremap(dev->driver_uar.pfn << PAGE_SHIFT, PAGE_SIZE);
 723         if (!dev->kar) {
 724                 mthca_err(dev, "Couldn't map kernel access region, "
 725                           "aborting.\n");
 726                 err = -ENOMEM;
 727                 goto err_uar_free;
 728         }
 729
 730         err = mthca_init_pd_table(dev);
 731         if (err) {
 732                 mthca_err(dev, "Failed to initialize "
 733                           "protection domain table, aborting.\n");
 734                 goto err_kar_unmap;
 735         }
 736
 737         err = mthca_init_mr_table(dev);
 738         if (err) {
 739                 mthca_err(dev, "Failed to initialize "
 740                           "memory region table, aborting.\n");
 741                 goto err_pd_table_free;
 742         }
 743
 744         err = mthca_pd_alloc(dev, 1, &dev->driver_pd);
 745         if (err) {
 746                 mthca_err(dev, "Failed to create driver PD, "
 747                           "aborting.\n");
 748                 goto err_mr_table_free;
 749         }
 750
 751         err = mthca_init_eq_table(dev);
 752         if (err) {
 753                 mthca_err(dev, "Failed to initialize "
 754                           "event queue table, aborting.\n");
 755                 goto err_pd_free;
 756         }
 757
 758         err = mthca_cmd_use_events(dev);
 759         if (err) {
 760                 mthca_err(dev, "Failed to switch to event-driven "
 761                           "firmware commands, aborting.\n");
 762                 goto err_eq_table_free;
 763         }
 764
 765         err = mthca_NOP(dev, &status);
 766         if (err || status) {
 767                 mthca_err(dev, "NOP command failed to generate interrupt (IRQ %d), aborting.\n",
 768                           dev->mthca_flags & MTHCA_FLAG_MSI_X ?
 769                           dev->eq_table.eq[MTHCA_EQ_CMD].msi_x_vector :
 770                           dev->pdev->irq);
 771                 if (dev->mthca_flags & (MTHCA_FLAG_MSI | MTHCA_FLAG_MSI_X))
 772                         mthca_err(dev, "Try again with MSI/MSI-X disabled.\n");
 773                 else
 774                         mthca_err(dev, "BIOS or ACPI interrupt routing problem?\n");
 775
 776                 goto err_cmd_poll;
 777         }
 778
 779         mthca_dbg(dev, "NOP command IRQ test passed\n");
 780
 781         err = mthca_init_cq_table(dev);
 782         if (err) {
 783                 mthca_err(dev, "Failed to initialize "
 784                           "completion queue table, aborting.\n");
 785                 goto err_cmd_poll;
 786         }
 787
 788         err = mthca_init_srq_table(dev);
 789         if (err) {
 790                 mthca_err(dev, "Failed to initialize "
 791                           "shared receive queue table, aborting.\n");
 792                 goto err_cq_table_free;
 793         }
 794
 795         err = mthca_init_qp_table(dev);
 796         if (err) {
 797                 mthca_err(dev, "Failed to initialize "
 798                           "queue pair table, aborting.\n");
 799                 goto err_srq_table_free;
 800         }
 801
 802         err = mthca_init_av_table(dev);
 803         if (err) {
 804                 mthca_err(dev, "Failed to initialize "
 805                           "address vector table, aborting.\n");
 806                 goto err_qp_table_free;
 807         }
 808
 809         err = mthca_init_mcg_table(dev);
 810         if (err) {
 811                 mthca_err(dev, "Failed to initialize "
 812                           "multicast group table, aborting.\n");
 813                 goto err_av_table_free;
 814         }
 815
 816         return 0;
 817
 818 err_av_table_free:
 819         mthca_cleanup_av_table(dev);
 820
 821 err_qp_table_free:
 822         mthca_cleanup_qp_table(dev);
 823
 824 err_srq_table_free:
 825         mthca_cleanup_srq_table(dev);
 826
 827 err_cq_table_free:
 828         mthca_cleanup_cq_table(dev);
 829
 830 err_cmd_poll:
 831         mthca_cmd_use_polling(dev);
 832
 833 err_eq_table_free:
 834         mthca_cleanup_eq_table(dev);
 835
 836 err_pd_free:
 837         mthca_pd_free(dev, &dev->driver_pd);
 838
 839 err_mr_table_free:
 840         mthca_cleanup_mr_table(dev);
 841
 842 err_pd_table_free:
 843         mthca_cleanup_pd_table(dev);
 844
 845 err_kar_unmap:
 846         iounmap(dev->kar);
 847
 848 err_uar_free:
 849         mthca_uar_free(dev, &dev->driver_uar);
 850
 851 err_uar_table_free:
 852         mthca_cleanup_uar_table(dev);
 853         return err;
 854 }
 855
 856 static int __devinit mthca_request_regions(struct pci_dev *pdev,
 857                                            int ddr_hidden)
 858 {
 859         int err;
 860
 861         /*
 862          * We can't just use pci_request_regions() because the MSI-X
 863          * table is right in the middle of the first BAR.  If we did
 864          * pci_request_region and grab all of the first BAR, then
 865          * setting up MSI-X would fail, since the PCI core wants to do
 866          * request_mem_region on the MSI-X vector table.
 867          *
 868          * So just request what we need right now, and request any
 869          * other regions we need when setting up EQs.
 870          */
 871         if (!request_mem_region(pci_resource_start(pdev, 0) + MTHCA_HCR_BASE,
 872                                 MTHCA_HCR_SIZE, DRV_NAME))
 873                 return -EBUSY;
 874
 875         err = pci_request_region(pdev, 2, DRV_NAME);
 876         if (err)
 877                 goto err_bar2_failed;
 878
 879         if (!ddr_hidden) {
 880                 err = pci_request_region(pdev, 4, DRV_NAME);
 881                 if (err)
 882                         goto err_bar4_failed;
 883         }
 884
 885         return 0;
 886
 887 err_bar4_failed:
 888         pci_release_region(pdev, 2);
 889
 890 err_bar2_failed:
 891         release_mem_region(pci_resource_start(pdev, 0) + MTHCA_HCR_BASE,
 892                            MTHCA_HCR_SIZE);
 893
 894         return err;
 895 }
 896
 897 static void mthca_release_regions(struct pci_dev *pdev,
 898                                   int ddr_hidden)
 899 {
 900         if (!ddr_hidden)
 901                 pci_release_region(pdev, 4);
 902
 903         pci_release_region(pdev, 2);
 904
 905         release_mem_region(pci_resource_start(pdev, 0) + MTHCA_HCR_BASE,
 906                            MTHCA_HCR_SIZE);
 907 }
 908
 909 static int __devinit mthca_enable_msi_x(struct mthca_dev *mdev)
 910 {
 911         struct msix_entry entries[3];
 912         int err;
 913
 914         entries[0].entry = 0;
 915         entries[1].entry = 1;
 916         entries[2].entry = 2;
 917
 918         err = pci_enable_msix(mdev->pdev, entries, ARRAY_SIZE(entries));
 919         if (err) {
 920                 if (err > 0)
 921                         mthca_info(mdev, "Only %d MSI-X vectors available, "
 922                                    "not using MSI-X\n", err);
 923                 return err;
 924         }
 925
 926         mdev->eq_table.eq[MTHCA_EQ_COMP ].msi_x_vector = entries[0].vector;
 927         mdev->eq_table.eq[MTHCA_EQ_ASYNC].msi_x_vector = entries[1].vector;
 928         mdev->eq_table.eq[MTHCA_EQ_CMD  ].msi_x_vector = entries[2].vector;
 929
 930         return 0;
 931 }
 932
 933 /* Types of supported HCA */
 934 enum {
 935         TAVOR,                  /* MT23108                        */
 936         ARBEL_COMPAT,           /* MT25208 in Tavor compat mode   */
 937         ARBEL_NATIVE,           /* MT25208 with extended features */
 938         SINAI                   /* MT25204 */
 939 };
 940
 941 #define MTHCA_FW_VER(major, minor, subminor) \
 942         (((u64) (major) << 32) | ((u64) (minor) << 16) | (u64) (subminor))
 943
 944 static struct {
 945         u64 latest_fw;
 946         u32 flags;
 947 } mthca_hca_table[] = {
 948         [TAVOR]        = { .latest_fw = MTHCA_FW_VER(3, 4, 0),
 949                            .flags     = 0 },
 950         [ARBEL_COMPAT] = { .latest_fw = MTHCA_FW_VER(4, 7, 400),
 951                            .flags     = MTHCA_FLAG_PCIE },
 952         [ARBEL_NATIVE] = { .latest_fw = MTHCA_FW_VER(5, 1, 0),
 953                            .flags     = MTHCA_FLAG_MEMFREE |
 954                                         MTHCA_FLAG_PCIE },
 955         [SINAI]        = { .latest_fw = MTHCA_FW_VER(1, 0, 800),
 956                            .flags     = MTHCA_FLAG_MEMFREE |
 957                                         MTHCA_FLAG_PCIE    |
 958                                         MTHCA_FLAG_SINAI_OPT }
 959 };
 960
 961 static int __devinit mthca_init_one(struct pci_dev *pdev,
 962                                     const struct pci_device_id *id)
 963 {
 964         static int mthca_version_printed = 0;
 965         int ddr_hidden = 0;
 966         int err;
 967         struct mthca_dev *mdev;
 968
 969         if (!mthca_version_printed) {
 970                 printk(KERN_INFO "%s", mthca_version);
 971                 ++mthca_version_printed;
 972         }
 973
 974         printk(KERN_INFO PFX "Initializing %s\n",
 975                pci_name(pdev));
 976
 977         if (id->driver_data >= ARRAY_SIZE(mthca_hca_table)) {
 978                 printk(KERN_ERR PFX "%s has invalid driver data %lx\n",
 979                        pci_name(pdev), id->driver_data);
 980                 return -ENODEV;
 981         }
 982
 983         err = pci_enable_device(pdev);
 984         if (err) {
 985                 dev_err(&pdev->dev, "Cannot enable PCI device, "
 986                         "aborting.\n");
 987                 return err;
 988         }
 989
 990         /*
 991          * Check for BARs.  We expect 0: 1MB, 2: 8MB, 4: DDR (may not
 992          * be present)
 993          */
 994         if (!(pci_resource_flags(pdev, 0) & IORESOURCE_MEM) ||
 995             pci_resource_len(pdev, 0) != 1 << 20) {
 996                 dev_err(&pdev->dev, "Missing DCS, aborting.\n");
 997                 err = -ENODEV;
 998                 goto err_disable_pdev;
 999         }
1000         if (!(pci_resource_flags(pdev, 2) & IORESOURCE_MEM)) {
1001                 dev_err(&pdev->dev, "Missing UAR, aborting.\n");
1002                 err = -ENODEV;
1003                 goto err_disable_pdev;
1004         }
1005         if (!(pci_resource_flags(pdev, 4) & IORESOURCE_MEM))
1006                 ddr_hidden = 1;
1007
1008         err = mthca_request_regions(pdev, ddr_hidden);
1009         if (err) {
1010                 dev_err(&pdev->dev, "Cannot obtain PCI resources, "
1011                         "aborting.\n");
1012                 goto err_disable_pdev;
1013         }
1014
1015         pci_set_master(pdev);
1016
1017         err = pci_set_dma_mask(pdev, DMA_64BIT_MASK);
1018         if (err) {
1019                 dev_warn(&pdev->dev, "Warning: couldn't set 64-bit PCI DMA mask.\n");
1020                 err = pci_set_dma_mask(pdev, DMA_32BIT_MASK);
1021                 if (err) {
1022                         dev_err(&pdev->dev, "Can't set PCI DMA mask, aborting.\n");
1023                         goto err_free_res;
1024                 }
1025         }
1026         err = pci_set_consistent_dma_mask(pdev, DMA_64BIT_MASK);
1027         if (err) {
1028                 dev_warn(&pdev->dev, "Warning: couldn't set 64-bit "
1029                          "consistent PCI DMA mask.\n");
1030                 err = pci_set_consistent_dma_mask(pdev, DMA_32BIT_MASK);
1031                 if (err) {
1032                         dev_err(&pdev->dev, "Can't set consistent PCI DMA mask, "
1033                                 "aborting.\n");
1034                         goto err_free_res;
1035                 }
1036         }
1037
1038         mdev = (struct mthca_dev *) ib_alloc_device(sizeof *mdev);
1039         if (!mdev) {
1040                 dev_err(&pdev->dev, "Device struct alloc failed, "
1041                         "aborting.\n");
1042                 err = -ENOMEM;
1043                 goto err_free_res;
1044         }
1045
1046         mdev->pdev = pdev;
1047
1048         mdev->mthca_flags = mthca_hca_table[id->driver_data].flags;
1049         if (ddr_hidden)
1050                 mdev->mthca_flags |= MTHCA_FLAG_DDR_HIDDEN;
1051
1052         /*
1053          * Now reset the HCA before we touch the PCI capabilities or
1054          * attempt a firmware command, since a boot ROM may have left
1055          * the HCA in an undefined state.
1056          */
1057         err = mthca_reset(mdev);
1058         if (err) {
1059                 mthca_err(mdev, "Failed to reset HCA, aborting.\n");
1060                 goto err_free_dev;
1061         }
1062
1063         if (msi_x && !mthca_enable_msi_x(mdev))
1064                 mdev->mthca_flags |= MTHCA_FLAG_MSI_X;
1065         if (msi && !(mdev->mthca_flags & MTHCA_FLAG_MSI_X) &&
1066             !pci_enable_msi(pdev))
1067                 mdev->mthca_flags |= MTHCA_FLAG_MSI;
1068
1069         if (mthca_cmd_init(mdev)) {
1070                 mthca_err(mdev, "Failed to init command interface, aborting.\n");
1071                 goto err_free_dev;
1072         }
1073
1074         err = mthca_tune_pci(mdev);
1075         if (err)
1076                 goto err_cmd;
1077
1078         err = mthca_init_hca(mdev);
1079         if (err)
1080                 goto err_cmd;
1081
1082         if (mdev->fw_ver < mthca_hca_table[id->driver_data].latest_fw) {
1083                 mthca_warn(mdev, "HCA FW version %d.%d.%d is old (%d.%d.%d is current).\n",
1084                            (int) (mdev->fw_ver >> 32), (int) (mdev->fw_ver >> 16) & 0xffff,
1085                            (int) (mdev->fw_ver & 0xffff),
1086                            (int) (mthca_hca_table[id->driver_data].latest_fw >> 32),
1087                            (int) (mthca_hca_table[id->driver_data].latest_fw >> 16) & 0xffff,
1088                            (int) (mthca_hca_table[id->driver_data].latest_fw & 0xffff));
1089                 mthca_warn(mdev, "If you have problems, try updating your HCA FW.\n");
1090         }
1091
1092         err = mthca_setup_hca(mdev);
1093         if (err)
1094                 goto err_close;
1095
1096         err = mthca_register_device(mdev);
1097         if (err)
1098                 goto err_cleanup;
1099
1100         err = mthca_create_agents(mdev);
1101         if (err)
1102                 goto err_unregister;
1103
1104         pci_set_drvdata(pdev, mdev);
1105
1106         return 0;
1107
1108 err_unregister:
1109         mthca_unregister_device(mdev);
1110
1111 err_cleanup:
1112         mthca_cleanup_mcg_table(mdev);
1113         mthca_cleanup_av_table(mdev);
1114         mthca_cleanup_qp_table(mdev);
1115         mthca_cleanup_srq_table(mdev);
1116         mthca_cleanup_cq_table(mdev);
1117         mthca_cmd_use_polling(mdev);
1118         mthca_cleanup_eq_table(mdev);
1119
1120         mthca_pd_free(mdev, &mdev->driver_pd);
1121
1122         mthca_cleanup_mr_table(mdev);
1123         mthca_cleanup_pd_table(mdev);
1124         mthca_cleanup_uar_table(mdev);
1125
1126 err_close:
1127         mthca_close_hca(mdev);
1128
1129 err_cmd:
1130         mthca_cmd_cleanup(mdev);
1131
1132 err_free_dev:
1133         if (mdev->mthca_flags & MTHCA_FLAG_MSI_X)
1134                 pci_disable_msix(pdev);
1135         if (mdev->mthca_flags & MTHCA_FLAG_MSI)
1136                 pci_disable_msi(pdev);
1137
1138         ib_dealloc_device(&mdev->ib_dev);
1139
1140 err_free_res:
1141         mthca_release_regions(pdev, ddr_hidden);
1142
1143 err_disable_pdev:
1144         pci_disable_device(pdev);
1145         pci_set_drvdata(pdev, NULL);
1146         return err;
1147 }
1148
1149 static void __devexit mthca_remove_one(struct pci_dev *pdev)
1150 {
1151         struct mthca_dev *mdev = pci_get_drvdata(pdev);
1152         u8 status;
1153         int p;
1154
1155         if (mdev) {
1156                 mthca_free_agents(mdev);
1157                 mthca_unregister_device(mdev);
1158
1159                 for (p = 1; p <= mdev->limits.num_ports; ++p)
1160                         mthca_CLOSE_IB(mdev, p, &status);
1161
1162                 mthca_cleanup_mcg_table(mdev);
1163                 mthca_cleanup_av_table(mdev);
1164                 mthca_cleanup_qp_table(mdev);
1165                 mthca_cleanup_srq_table(mdev);
1166                 mthca_cleanup_cq_table(mdev);
1167                 mthca_cmd_use_polling(mdev);
1168                 mthca_cleanup_eq_table(mdev);
1169
1170                 mthca_pd_free(mdev, &mdev->driver_pd);
1171
1172                 mthca_cleanup_mr_table(mdev);
1173                 mthca_cleanup_pd_table(mdev);
1174
1175                 iounmap(mdev->kar);
1176                 mthca_uar_free(mdev, &mdev->driver_uar);
1177                 mthca_cleanup_uar_table(mdev);
1178                 mthca_close_hca(mdev);
1179                 mthca_cmd_cleanup(mdev);
1180
1181                 if (mdev->mthca_flags & MTHCA_FLAG_MSI_X)
1182                         pci_disable_msix(pdev);
1183                 if (mdev->mthca_flags & MTHCA_FLAG_MSI)
1184                         pci_disable_msi(pdev);
1185
1186                 ib_dealloc_device(&mdev->ib_dev);
1187                 mthca_release_regions(pdev, mdev->mthca_flags &
1188                                       MTHCA_FLAG_DDR_HIDDEN);
1189                 pci_disable_device(pdev);
1190                 pci_set_drvdata(pdev, NULL);
1191         }
1192 }
1193
1194 static struct pci_device_id mthca_pci_table[] = {
1195         { PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, PCI_DEVICE_ID_MELLANOX_TAVOR),
1196           .driver_data = TAVOR },
1197         { PCI_DEVICE(PCI_VENDOR_ID_TOPSPIN, PCI_DEVICE_ID_MELLANOX_TAVOR),
1198           .driver_data = TAVOR },
1199         { PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, PCI_DEVICE_ID_MELLANOX_ARBEL_COMPAT),
1200           .driver_data = ARBEL_COMPAT },
1201         { PCI_DEVICE(PCI_VENDOR_ID_TOPSPIN, PCI_DEVICE_ID_MELLANOX_ARBEL_COMPAT),
1202           .driver_data = ARBEL_COMPAT },
1203         { PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, PCI_DEVICE_ID_MELLANOX_ARBEL),
1204           .driver_data = ARBEL_NATIVE },
1205         { PCI_DEVICE(PCI_VENDOR_ID_TOPSPIN, PCI_DEVICE_ID_MELLANOX_ARBEL),
1206           .driver_data = ARBEL_NATIVE },
1207         { PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, PCI_DEVICE_ID_MELLANOX_SINAI),
1208           .driver_data = SINAI },
1209         { PCI_DEVICE(PCI_VENDOR_ID_TOPSPIN, PCI_DEVICE_ID_MELLANOX_SINAI),
1210           .driver_data = SINAI },
1211         { PCI_DEVICE(PCI_VENDOR_ID_MELLANOX, PCI_DEVICE_ID_MELLANOX_SINAI_OLD),
1212           .driver_data = SINAI },
1213         { PCI_DEVICE(PCI_VENDOR_ID_TOPSPIN, PCI_DEVICE_ID_MELLANOX_SINAI_OLD),
1214           .driver_data = SINAI },
1215         { 0, }
1216 };
1217
1218 MODULE_DEVICE_TABLE(pci, mthca_pci_table);
1219
1220 static struct pci_driver mthca_driver = {
1221         .name           = DRV_NAME,
1222         .id_table       = mthca_pci_table,
1223         .probe          = mthca_init_one,
1224         .remove         = __devexit_p(mthca_remove_one)
1225 };
1226
1227 static int __init mthca_init(void)
1228 {
1229         int ret;
1230
1231         ret = pci_register_driver(&mthca_driver);
1232         return ret < 0 ? ret : 0;
1233 }
1234
1235 static void __exit mthca_cleanup(void)
1236 {
1237         pci_unregister_driver(&mthca_driver);
1238 }
1239
1240 module_init(mthca_init);
1241 module_exit(mthca_cleanup);