SAFE public projects git trees. - safe/jmp/linux-2.6/blob - drivers/net/vxge/vxge-config.c

   1 /******************************************************************************
   2  * This software may be used and distributed according to the terms of
   3  * the GNU General Public License (GPL), incorporated herein by reference.
   4  * Drivers based on or derived from this code fall under the GPL and must
   5  * retain the authorship, copyright and license notice.  This file is not
   6  * a complete program and may only be used when the entire operating
   7  * system is licensed under the GPL.
   8  * See the file COPYING in this distribution for more information.
   9  *
  10  * vxge-config.c: Driver for Neterion Inc's X3100 Series 10GbE PCIe I/O
  11  *                Virtualized Server Adapter.
  12  * Copyright(c) 2002-2009 Neterion Inc.
  13  ******************************************************************************/
  14 #include <linux/vmalloc.h>
  15 #include <linux/etherdevice.h>
  16 #include <linux/pci.h>
  17 #include <linux/pci_hotplug.h>
  18
  19 #include "vxge-traffic.h"
  20 #include "vxge-config.h"
  21
  22 /*
  23  * __vxge_hw_channel_allocate - Allocate memory for channel
  24  * This function allocates required memory for the channel and various arrays
  25  * in the channel
  26  */
  27 struct __vxge_hw_channel*
  28 __vxge_hw_channel_allocate(struct __vxge_hw_vpath_handle *vph,
  29                            enum __vxge_hw_channel_type type,
  30         u32 length, u32 per_dtr_space, void *userdata)
  31 {
  32         struct __vxge_hw_channel *channel;
  33         struct __vxge_hw_device *hldev;
  34         int size = 0;
  35         u32 vp_id;
  36
  37         hldev = vph->vpath->hldev;
  38         vp_id = vph->vpath->vp_id;
  39
  40         switch (type) {
  41         case VXGE_HW_CHANNEL_TYPE_FIFO:
  42                 size = sizeof(struct __vxge_hw_fifo);
  43                 break;
  44         case VXGE_HW_CHANNEL_TYPE_RING:
  45                 size = sizeof(struct __vxge_hw_ring);
  46                 break;
  47         default:
  48                 break;
  49         }
  50
  51         channel = kzalloc(size, GFP_KERNEL);
  52         if (channel == NULL)
  53                 goto exit0;
  54         INIT_LIST_HEAD(&channel->item);
  55
  56         channel->common_reg = hldev->common_reg;
  57         channel->first_vp_id = hldev->first_vp_id;
  58         channel->type = type;
  59         channel->devh = hldev;
  60         channel->vph = vph;
  61         channel->userdata = userdata;
  62         channel->per_dtr_space = per_dtr_space;
  63         channel->length = length;
  64         channel->vp_id = vp_id;
  65
  66         channel->work_arr = kzalloc(sizeof(void *)*length, GFP_KERNEL);
  67         if (channel->work_arr == NULL)
  68                 goto exit1;
  69
  70         channel->free_arr = kzalloc(sizeof(void *)*length, GFP_KERNEL);
  71         if (channel->free_arr == NULL)
  72                 goto exit1;
  73         channel->free_ptr = length;
  74
  75         channel->reserve_arr = kzalloc(sizeof(void *)*length, GFP_KERNEL);
  76         if (channel->reserve_arr == NULL)
  77                 goto exit1;
  78         channel->reserve_ptr = length;
  79         channel->reserve_top = 0;
  80
  81         channel->orig_arr = kzalloc(sizeof(void *)*length, GFP_KERNEL);
  82         if (channel->orig_arr == NULL)
  83                 goto exit1;
  84
  85         return channel;
  86 exit1:
  87         __vxge_hw_channel_free(channel);
  88
  89 exit0:
  90         return NULL;
  91 }
  92
  93 /*
  94  * __vxge_hw_channel_free - Free memory allocated for channel
  95  * This function deallocates memory from the channel and various arrays
  96  * in the channel
  97  */
  98 void __vxge_hw_channel_free(struct __vxge_hw_channel *channel)
  99 {
 100         kfree(channel->work_arr);
 101         kfree(channel->free_arr);
 102         kfree(channel->reserve_arr);
 103         kfree(channel->orig_arr);
 104         kfree(channel);
 105 }
 106
 107 /*
 108  * __vxge_hw_channel_initialize - Initialize a channel
 109  * This function initializes a channel by properly setting the
 110  * various references
 111  */
 112 enum vxge_hw_status
 113 __vxge_hw_channel_initialize(struct __vxge_hw_channel *channel)
 114 {
 115         u32 i;
 116         struct __vxge_hw_virtualpath *vpath;
 117
 118         vpath = channel->vph->vpath;
 119
 120         if ((channel->reserve_arr != NULL) && (channel->orig_arr != NULL)) {
 121                 for (i = 0; i < channel->length; i++)
 122                         channel->orig_arr[i] = channel->reserve_arr[i];
 123         }
 124
 125         switch (channel->type) {
 126         case VXGE_HW_CHANNEL_TYPE_FIFO:
 127                 vpath->fifoh = (struct __vxge_hw_fifo *)channel;
 128                 channel->stats = &((struct __vxge_hw_fifo *)
 129                                 channel)->stats->common_stats;
 130                 break;
 131         case VXGE_HW_CHANNEL_TYPE_RING:
 132                 vpath->ringh = (struct __vxge_hw_ring *)channel;
 133                 channel->stats = &((struct __vxge_hw_ring *)
 134                                 channel)->stats->common_stats;
 135                 break;
 136         default:
 137                 break;
 138         }
 139
 140         return VXGE_HW_OK;
 141 }
 142
 143 /*
 144  * __vxge_hw_channel_reset - Resets a channel
 145  * This function resets a channel by properly setting the various references
 146  */
 147 enum vxge_hw_status
 148 __vxge_hw_channel_reset(struct __vxge_hw_channel *channel)
 149 {
 150         u32 i;
 151
 152         for (i = 0; i < channel->length; i++) {
 153                 if (channel->reserve_arr != NULL)
 154                         channel->reserve_arr[i] = channel->orig_arr[i];
 155                 if (channel->free_arr != NULL)
 156                         channel->free_arr[i] = NULL;
 157                 if (channel->work_arr != NULL)
 158                         channel->work_arr[i] = NULL;
 159         }
 160         channel->free_ptr = channel->length;
 161         channel->reserve_ptr = channel->length;
 162         channel->reserve_top = 0;
 163         channel->post_index = 0;
 164         channel->compl_index = 0;
 165
 166         return VXGE_HW_OK;
 167 }
 168
 169 /*
 170  * __vxge_hw_device_pci_e_init
 171  * Initialize certain PCI/PCI-X configuration registers
 172  * with recommended values. Save config space for future hw resets.
 173  */
 174 void
 175 __vxge_hw_device_pci_e_init(struct __vxge_hw_device *hldev)
 176 {
 177         u16 cmd = 0;
 178
 179         /* Set the PErr Repconse bit and SERR in PCI command register. */
 180         pci_read_config_word(hldev->pdev, PCI_COMMAND, &cmd);
 181         cmd |= 0x140;
 182         pci_write_config_word(hldev->pdev, PCI_COMMAND, cmd);
 183
 184         pci_save_state(hldev->pdev);
 185
 186         return;
 187 }
 188
 189 /*
 190  * __vxge_hw_device_register_poll
 191  * Will poll certain register for specified amount of time.
 192  * Will poll until masked bit is not cleared.
 193  */
 194 enum vxge_hw_status
 195 __vxge_hw_device_register_poll(void __iomem *reg, u64 mask, u32 max_millis)
 196 {
 197         u64 val64;
 198         u32 i = 0;
 199         enum vxge_hw_status ret = VXGE_HW_FAIL;
 200
 201         udelay(10);
 202
 203         do {
 204                 val64 = readq(reg);
 205                 if (!(val64 & mask))
 206                         return VXGE_HW_OK;
 207                 udelay(100);
 208         } while (++i <= 9);
 209
 210         i = 0;
 211         do {
 212                 val64 = readq(reg);
 213                 if (!(val64 & mask))
 214                         return VXGE_HW_OK;
 215                 mdelay(1);
 216         } while (++i <= max_millis);
 217
 218         return ret;
 219 }
 220
 221  /* __vxge_hw_device_vpath_reset_in_prog_check - Check if vpath reset
 222  * in progress
 223  * This routine checks the vpath reset in progress register is turned zero
 224  */
 225 enum vxge_hw_status
 226 __vxge_hw_device_vpath_reset_in_prog_check(u64 __iomem *vpath_rst_in_prog)
 227 {
 228         enum vxge_hw_status status;
 229         status = __vxge_hw_device_register_poll(vpath_rst_in_prog,
 230                         VXGE_HW_VPATH_RST_IN_PROG_VPATH_RST_IN_PROG(0x1ffff),
 231                         VXGE_HW_DEF_DEVICE_POLL_MILLIS);
 232         return status;
 233 }
 234
 235 /*
 236  * __vxge_hw_device_toc_get
 237  * This routine sets the swapper and reads the toc pointer and returns the
 238  * memory mapped address of the toc
 239  */
 240 struct vxge_hw_toc_reg __iomem *
 241 __vxge_hw_device_toc_get(void __iomem *bar0)
 242 {
 243         u64 val64;
 244         struct vxge_hw_toc_reg __iomem *toc = NULL;
 245         enum vxge_hw_status status;
 246
 247         struct vxge_hw_legacy_reg __iomem *legacy_reg =
 248                 (struct vxge_hw_legacy_reg __iomem *)bar0;
 249
 250         status = __vxge_hw_legacy_swapper_set(legacy_reg);
 251         if (status != VXGE_HW_OK)
 252                 goto exit;
 253
 254         val64 = readq(&legacy_reg->toc_first_pointer);
 255         toc = (struct vxge_hw_toc_reg __iomem *)(bar0+val64);
 256 exit:
 257         return toc;
 258 }
 259
 260 /*
 261  * __vxge_hw_device_reg_addr_get
 262  * This routine sets the swapper and reads the toc pointer and initializes the
 263  * register location pointers in the device object. It waits until the ric is
 264  * completed initializing registers.
 265  */
 266 enum vxge_hw_status
 267 __vxge_hw_device_reg_addr_get(struct __vxge_hw_device *hldev)
 268 {
 269         u64 val64;
 270         u32 i;
 271         enum vxge_hw_status status = VXGE_HW_OK;
 272
 273         hldev->legacy_reg = (struct vxge_hw_legacy_reg __iomem *)hldev->bar0;
 274
 275         hldev->toc_reg = __vxge_hw_device_toc_get(hldev->bar0);
 276         if (hldev->toc_reg  == NULL) {
 277                 status = VXGE_HW_FAIL;
 278                 goto exit;
 279         }
 280
 281         val64 = readq(&hldev->toc_reg->toc_common_pointer);
 282         hldev->common_reg =
 283         (struct vxge_hw_common_reg __iomem *)(hldev->bar0 + val64);
 284
 285         val64 = readq(&hldev->toc_reg->toc_mrpcim_pointer);
 286         hldev->mrpcim_reg =
 287                 (struct vxge_hw_mrpcim_reg __iomem *)(hldev->bar0 + val64);
 288
 289         for (i = 0; i < VXGE_HW_TITAN_SRPCIM_REG_SPACES; i++) {
 290                 val64 = readq(&hldev->toc_reg->toc_srpcim_pointer[i]);
 291                 hldev->srpcim_reg[i] =
 292                         (struct vxge_hw_srpcim_reg __iomem *)
 293                                 (hldev->bar0 + val64);
 294         }
 295
 296         for (i = 0; i < VXGE_HW_TITAN_VPMGMT_REG_SPACES; i++) {
 297                 val64 = readq(&hldev->toc_reg->toc_vpmgmt_pointer[i]);
 298                 hldev->vpmgmt_reg[i] =
 299                 (struct vxge_hw_vpmgmt_reg __iomem *)(hldev->bar0 + val64);
 300         }
 301
 302         for (i = 0; i < VXGE_HW_TITAN_VPATH_REG_SPACES; i++) {
 303                 val64 = readq(&hldev->toc_reg->toc_vpath_pointer[i]);
 304                 hldev->vpath_reg[i] =
 305                         (struct vxge_hw_vpath_reg __iomem *)
 306                                 (hldev->bar0 + val64);
 307         }
 308
 309         val64 = readq(&hldev->toc_reg->toc_kdfc);
 310
 311         switch (VXGE_HW_TOC_GET_KDFC_INITIAL_BIR(val64)) {
 312         case 0:
 313                 hldev->kdfc = (u8 __iomem *)(hldev->bar0 +
 314                         VXGE_HW_TOC_GET_KDFC_INITIAL_OFFSET(val64));
 315                 break;
 316         default:
 317                 break;
 318         }
 319
 320         status = __vxge_hw_device_vpath_reset_in_prog_check(
 321                         (u64 __iomem *)&hldev->common_reg->vpath_rst_in_prog);
 322 exit:
 323         return status;
 324 }
 325
 326 /*
 327  * __vxge_hw_device_id_get
 328  * This routine returns sets the device id and revision numbers into the device
 329  * structure
 330  */
 331 void __vxge_hw_device_id_get(struct __vxge_hw_device *hldev)
 332 {
 333         u64 val64;
 334
 335         val64 = readq(&hldev->common_reg->titan_asic_id);
 336         hldev->device_id =
 337                 (u16)VXGE_HW_TITAN_ASIC_ID_GET_INITIAL_DEVICE_ID(val64);
 338
 339         hldev->major_revision =
 340                 (u8)VXGE_HW_TITAN_ASIC_ID_GET_INITIAL_MAJOR_REVISION(val64);
 341
 342         hldev->minor_revision =
 343                 (u8)VXGE_HW_TITAN_ASIC_ID_GET_INITIAL_MINOR_REVISION(val64);
 344
 345         return;
 346 }
 347
 348 /*
 349  * __vxge_hw_device_access_rights_get: Get Access Rights of the driver
 350  * This routine returns the Access Rights of the driver
 351  */
 352 static u32
 353 __vxge_hw_device_access_rights_get(u32 host_type, u32 func_id)
 354 {
 355         u32 access_rights = VXGE_HW_DEVICE_ACCESS_RIGHT_VPATH;
 356
 357         switch (host_type) {
 358         case VXGE_HW_NO_MR_NO_SR_NORMAL_FUNCTION:
 359                 access_rights |= VXGE_HW_DEVICE_ACCESS_RIGHT_MRPCIM |
 360                                 VXGE_HW_DEVICE_ACCESS_RIGHT_SRPCIM;
 361                 break;
 362         case VXGE_HW_MR_NO_SR_VH0_BASE_FUNCTION:
 363                 access_rights |= VXGE_HW_DEVICE_ACCESS_RIGHT_MRPCIM |
 364                                 VXGE_HW_DEVICE_ACCESS_RIGHT_SRPCIM;
 365                 break;
 366         case VXGE_HW_NO_MR_SR_VH0_FUNCTION0:
 367                 access_rights |= VXGE_HW_DEVICE_ACCESS_RIGHT_MRPCIM |
 368                                 VXGE_HW_DEVICE_ACCESS_RIGHT_SRPCIM;
 369                 break;
 370         case VXGE_HW_NO_MR_SR_VH0_VIRTUAL_FUNCTION:
 371         case VXGE_HW_SR_VH_VIRTUAL_FUNCTION:
 372         case VXGE_HW_MR_SR_VH0_INVALID_CONFIG:
 373                 break;
 374         case VXGE_HW_SR_VH_FUNCTION0:
 375         case VXGE_HW_VH_NORMAL_FUNCTION:
 376                 access_rights |= VXGE_HW_DEVICE_ACCESS_RIGHT_SRPCIM;
 377                 break;
 378         }
 379
 380         return access_rights;
 381 }
 382 /*
 383  * __vxge_hw_device_is_privilaged
 384  * This routine checks if the device function is privilaged or not
 385  */
 386
 387 enum vxge_hw_status
 388 __vxge_hw_device_is_privilaged(u32 host_type, u32 func_id)
 389 {
 390         if (__vxge_hw_device_access_rights_get(host_type,
 391                 func_id) &
 392                 VXGE_HW_DEVICE_ACCESS_RIGHT_MRPCIM)
 393                 return VXGE_HW_OK;
 394         else
 395                 return VXGE_HW_ERR_PRIVILAGED_OPEARATION;
 396 }
 397
 398 /*
 399  * __vxge_hw_device_host_info_get
 400  * This routine returns the host type assignments
 401  */
 402 void __vxge_hw_device_host_info_get(struct __vxge_hw_device *hldev)
 403 {
 404         u64 val64;
 405         u32 i;
 406
 407         val64 = readq(&hldev->common_reg->host_type_assignments);
 408
 409         hldev->host_type =
 410            (u32)VXGE_HW_HOST_TYPE_ASSIGNMENTS_GET_HOST_TYPE_ASSIGNMENTS(val64);
 411
 412         hldev->vpath_assignments = readq(&hldev->common_reg->vpath_assignments);
 413
 414         for (i = 0; i < VXGE_HW_MAX_VIRTUAL_PATHS; i++) {
 415
 416                 if (!(hldev->vpath_assignments & vxge_mBIT(i)))
 417                         continue;
 418
 419                 hldev->func_id =
 420                         __vxge_hw_vpath_func_id_get(i, hldev->vpmgmt_reg[i]);
 421
 422                 hldev->access_rights = __vxge_hw_device_access_rights_get(
 423                         hldev->host_type, hldev->func_id);
 424
 425                 hldev->first_vp_id = i;
 426                 break;
 427         }
 428
 429         return;
 430 }
 431
 432 /*
 433  * __vxge_hw_verify_pci_e_info - Validate the pci-e link parameters such as
 434  * link width and signalling rate.
 435  */
 436 static enum vxge_hw_status
 437 __vxge_hw_verify_pci_e_info(struct __vxge_hw_device *hldev)
 438 {
 439         int exp_cap;
 440         u16 lnk;
 441
 442         /* Get the negotiated link width and speed from PCI config space */
 443         exp_cap = pci_find_capability(hldev->pdev, PCI_CAP_ID_EXP);
 444         pci_read_config_word(hldev->pdev, exp_cap + PCI_EXP_LNKSTA, &lnk);
 445
 446         if ((lnk & PCI_EXP_LNKSTA_CLS) != 1)
 447                 return VXGE_HW_ERR_INVALID_PCI_INFO;
 448
 449         switch ((lnk & PCI_EXP_LNKSTA_NLW) >> 4) {
 450         case PCIE_LNK_WIDTH_RESRV:
 451         case PCIE_LNK_X1:
 452         case PCIE_LNK_X2:
 453         case PCIE_LNK_X4:
 454         case PCIE_LNK_X8:
 455                 break;
 456         default:
 457                 return VXGE_HW_ERR_INVALID_PCI_INFO;
 458         }
 459
 460         return VXGE_HW_OK;
 461 }
 462
 463 /*
 464  * vxge_hw_wrr_rebalance - Rebalance the RX_WRR and KDFC_WRR calandars.
 465  * Rebalance the RX_WRR and KDFC_WRR calandars.
 466  */
 467 static enum
 468 vxge_hw_status vxge_hw_wrr_rebalance(struct __vxge_hw_device *hldev)
 469 {
 470         u64 val64;
 471         u32 wrr_states[VXGE_HW_WEIGHTED_RR_SERVICE_STATES];
 472         u32 i, j, how_often = 1;
 473         enum vxge_hw_status status = VXGE_HW_OK;
 474
 475         status = __vxge_hw_device_is_privilaged(hldev->host_type,
 476                         hldev->func_id);
 477         if (status != VXGE_HW_OK)
 478                 goto exit;
 479
 480         /* Reset the priorities assigned to the WRR arbitration
 481         phases for the receive traffic */
 482         for (i = 0; i < VXGE_HW_WRR_RING_COUNT; i++)
 483                 writeq(0, ((&hldev->mrpcim_reg->rx_w_round_robin_0) + i));
 484
 485         /* Reset the transmit FIFO servicing calendar for FIFOs */
 486         for (i = 0; i < VXGE_HW_WRR_FIFO_COUNT; i++) {
 487                 writeq(0, ((&hldev->mrpcim_reg->kdfc_w_round_robin_0) + i));
 488                 writeq(0, ((&hldev->mrpcim_reg->kdfc_w_round_robin_20) + i));
 489         }
 490
 491         /* Assign WRR priority  0 for all FIFOs */
 492         for (i = 1; i < VXGE_HW_MAX_VIRTUAL_PATHS; i++) {
 493                 writeq(VXGE_HW_KDFC_FIFO_0_CTRL_WRR_NUMBER(0),
 494                                 ((&hldev->mrpcim_reg->kdfc_fifo_0_ctrl)  + i));
 495
 496                 writeq(VXGE_HW_KDFC_FIFO_17_CTRL_WRR_NUMBER(0),
 497                         ((&hldev->mrpcim_reg->kdfc_fifo_17_ctrl) + i));
 498         }
 499
 500         /* Reset to service non-offload doorbells */
 501         writeq(0, &hldev->mrpcim_reg->kdfc_entry_type_sel_0);
 502         writeq(0, &hldev->mrpcim_reg->kdfc_entry_type_sel_1);
 503
 504         /* Set priority 0 to all receive queues */
 505         writeq(0, &hldev->mrpcim_reg->rx_queue_priority_0);
 506         writeq(0, &hldev->mrpcim_reg->rx_queue_priority_1);
 507         writeq(0, &hldev->mrpcim_reg->rx_queue_priority_2);
 508
 509         /* Initialize all the slots as unused */
 510         for (i = 0; i < VXGE_HW_WEIGHTED_RR_SERVICE_STATES; i++)
 511                 wrr_states[i] = -1;
 512
 513         /* Prepare the Fifo service states */
 514         for (i = 0; i < VXGE_HW_MAX_VIRTUAL_PATHS; i++) {
 515
 516                 if (!hldev->config.vp_config[i].min_bandwidth)
 517                         continue;
 518
 519                 how_often = VXGE_HW_VPATH_BANDWIDTH_MAX /
 520                                 hldev->config.vp_config[i].min_bandwidth;
 521                 if (how_often) {
 522
 523                         for (j = 0; j < VXGE_HW_WRR_FIFO_SERVICE_STATES;) {
 524                                 if (wrr_states[j] == -1) {
 525                                         wrr_states[j] = i;
 526                                         /* Make sure each fifo is serviced
 527                                          * atleast once */
 528                                         if (i == j)
 529                                                 j += VXGE_HW_MAX_VIRTUAL_PATHS;
 530                                         else
 531                                                 j += how_often;
 532                                 } else
 533                                         j++;
 534                         }
 535                 }
 536         }
 537
 538         /* Fill the unused slots with 0 */
 539         for (j = 0; j < VXGE_HW_WEIGHTED_RR_SERVICE_STATES; j++) {
 540                 if (wrr_states[j] == -1)
 541                         wrr_states[j] = 0;
 542         }
 543
 544         /* Assign WRR priority number for FIFOs */
 545         for (i = 0; i < VXGE_HW_MAX_VIRTUAL_PATHS; i++) {
 546                 writeq(VXGE_HW_KDFC_FIFO_0_CTRL_WRR_NUMBER(i),
 547                                 ((&hldev->mrpcim_reg->kdfc_fifo_0_ctrl) + i));
 548
 549                 writeq(VXGE_HW_KDFC_FIFO_17_CTRL_WRR_NUMBER(i),
 550                         ((&hldev->mrpcim_reg->kdfc_fifo_17_ctrl) + i));
 551         }
 552
 553         /* Modify the servicing algorithm applied to the 3 types of doorbells.
 554         i.e, none-offload, message and offload */
 555         writeq(VXGE_HW_KDFC_ENTRY_TYPE_SEL_0_NUMBER_0(0) |
 556                                 VXGE_HW_KDFC_ENTRY_TYPE_SEL_0_NUMBER_1(0) |
 557                                 VXGE_HW_KDFC_ENTRY_TYPE_SEL_0_NUMBER_2(0) |
 558                                 VXGE_HW_KDFC_ENTRY_TYPE_SEL_0_NUMBER_3(0) |
 559                                 VXGE_HW_KDFC_ENTRY_TYPE_SEL_0_NUMBER_4(1) |
 560                                 VXGE_HW_KDFC_ENTRY_TYPE_SEL_0_NUMBER_5(0) |
 561                                 VXGE_HW_KDFC_ENTRY_TYPE_SEL_0_NUMBER_6(0) |
 562                                 VXGE_HW_KDFC_ENTRY_TYPE_SEL_0_NUMBER_7(0),
 563                                 &hldev->mrpcim_reg->kdfc_entry_type_sel_0);
 564
 565         writeq(VXGE_HW_KDFC_ENTRY_TYPE_SEL_1_NUMBER_8(1),
 566                                 &hldev->mrpcim_reg->kdfc_entry_type_sel_1);
 567
 568         for (i = 0, j = 0; i < VXGE_HW_WRR_FIFO_COUNT; i++) {
 569
 570                 val64 = VXGE_HW_KDFC_W_ROUND_ROBIN_0_NUMBER_0(wrr_states[j++]);
 571                 val64 |= VXGE_HW_KDFC_W_ROUND_ROBIN_0_NUMBER_1(wrr_states[j++]);
 572                 val64 |= VXGE_HW_KDFC_W_ROUND_ROBIN_0_NUMBER_2(wrr_states[j++]);
 573                 val64 |= VXGE_HW_KDFC_W_ROUND_ROBIN_0_NUMBER_3(wrr_states[j++]);
 574                 val64 |= VXGE_HW_KDFC_W_ROUND_ROBIN_0_NUMBER_4(wrr_states[j++]);
 575                 val64 |= VXGE_HW_KDFC_W_ROUND_ROBIN_0_NUMBER_5(wrr_states[j++]);
 576                 val64 |= VXGE_HW_KDFC_W_ROUND_ROBIN_0_NUMBER_6(wrr_states[j++]);
 577                 val64 |= VXGE_HW_KDFC_W_ROUND_ROBIN_0_NUMBER_7(wrr_states[j++]);
 578
 579                 writeq(val64, (&hldev->mrpcim_reg->kdfc_w_round_robin_0 + i));
 580                 writeq(val64, (&hldev->mrpcim_reg->kdfc_w_round_robin_20 + i));
 581         }
 582
 583         /* Set up the priorities assigned to receive queues */
 584         writeq(VXGE_HW_RX_QUEUE_PRIORITY_0_RX_Q_NUMBER_0(0) |
 585                         VXGE_HW_RX_QUEUE_PRIORITY_0_RX_Q_NUMBER_1(1) |
 586                         VXGE_HW_RX_QUEUE_PRIORITY_0_RX_Q_NUMBER_2(2) |
 587                         VXGE_HW_RX_QUEUE_PRIORITY_0_RX_Q_NUMBER_3(3) |
 588                         VXGE_HW_RX_QUEUE_PRIORITY_0_RX_Q_NUMBER_4(4) |
 589                         VXGE_HW_RX_QUEUE_PRIORITY_0_RX_Q_NUMBER_5(5) |
 590                         VXGE_HW_RX_QUEUE_PRIORITY_0_RX_Q_NUMBER_6(6) |
 591                         VXGE_HW_RX_QUEUE_PRIORITY_0_RX_Q_NUMBER_7(7),
 592                         &hldev->mrpcim_reg->rx_queue_priority_0);
 593
 594         writeq(VXGE_HW_RX_QUEUE_PRIORITY_1_RX_Q_NUMBER_8(8) |
 595                         VXGE_HW_RX_QUEUE_PRIORITY_1_RX_Q_NUMBER_9(9) |
 596                         VXGE_HW_RX_QUEUE_PRIORITY_1_RX_Q_NUMBER_10(10) |
 597                         VXGE_HW_RX_QUEUE_PRIORITY_1_RX_Q_NUMBER_11(11) |
 598                         VXGE_HW_RX_QUEUE_PRIORITY_1_RX_Q_NUMBER_12(12) |
 599                         VXGE_HW_RX_QUEUE_PRIORITY_1_RX_Q_NUMBER_13(13) |
 600                         VXGE_HW_RX_QUEUE_PRIORITY_1_RX_Q_NUMBER_14(14) |
 601                         VXGE_HW_RX_QUEUE_PRIORITY_1_RX_Q_NUMBER_15(15),
 602                         &hldev->mrpcim_reg->rx_queue_priority_1);
 603
 604         writeq(VXGE_HW_RX_QUEUE_PRIORITY_2_RX_Q_NUMBER_16(16),
 605                                 &hldev->mrpcim_reg->rx_queue_priority_2);
 606
 607         /* Initialize all the slots as unused */
 608         for (i = 0; i < VXGE_HW_WEIGHTED_RR_SERVICE_STATES; i++)
 609                 wrr_states[i] = -1;
 610
 611         /* Prepare the Ring service states */
 612         for (i = 0; i < VXGE_HW_MAX_VIRTUAL_PATHS; i++) {
 613
 614                 if (!hldev->config.vp_config[i].min_bandwidth)
 615                         continue;
 616
 617                 how_often = VXGE_HW_VPATH_BANDWIDTH_MAX /
 618                                 hldev->config.vp_config[i].min_bandwidth;
 619
 620                 if (how_often) {
 621                         for (j = 0; j < VXGE_HW_WRR_RING_SERVICE_STATES;) {
 622                                 if (wrr_states[j] == -1) {
 623                                         wrr_states[j] = i;
 624                                         /* Make sure each ring is
 625                                          * serviced atleast once */
 626                                         if (i == j)
 627                                                 j += VXGE_HW_MAX_VIRTUAL_PATHS;
 628                                         else
 629                                                 j += how_often;
 630                                 } else
 631                                         j++;
 632                         }
 633                 }
 634         }
 635
 636         /* Fill the unused slots with 0 */
 637         for (j = 0; j < VXGE_HW_WEIGHTED_RR_SERVICE_STATES; j++) {
 638                 if (wrr_states[j] == -1)
 639                         wrr_states[j] = 0;
 640         }
 641
 642         for (i = 0, j = 0; i < VXGE_HW_WRR_RING_COUNT; i++) {
 643                 val64 =  VXGE_HW_RX_W_ROUND_ROBIN_0_RX_W_PRIORITY_SS_0(
 644                                 wrr_states[j++]);
 645                 val64 |=  VXGE_HW_RX_W_ROUND_ROBIN_0_RX_W_PRIORITY_SS_1(
 646                                 wrr_states[j++]);
 647                 val64 |=  VXGE_HW_RX_W_ROUND_ROBIN_0_RX_W_PRIORITY_SS_2(
 648                                 wrr_states[j++]);
 649                 val64 |=  VXGE_HW_RX_W_ROUND_ROBIN_0_RX_W_PRIORITY_SS_3(
 650                                 wrr_states[j++]);
 651                 val64 |=  VXGE_HW_RX_W_ROUND_ROBIN_0_RX_W_PRIORITY_SS_4(
 652                                 wrr_states[j++]);
 653                 val64 |=  VXGE_HW_RX_W_ROUND_ROBIN_0_RX_W_PRIORITY_SS_5(
 654                                 wrr_states[j++]);
 655                 val64 |=  VXGE_HW_RX_W_ROUND_ROBIN_0_RX_W_PRIORITY_SS_6(
 656                                 wrr_states[j++]);
 657                 val64 |=  VXGE_HW_RX_W_ROUND_ROBIN_0_RX_W_PRIORITY_SS_7(
 658                                 wrr_states[j++]);
 659
 660                 writeq(val64, ((&hldev->mrpcim_reg->rx_w_round_robin_0) + i));
 661         }
 662 exit:
 663         return status;
 664 }
 665
 666 /*
 667  * __vxge_hw_device_initialize
 668  * Initialize Titan-V hardware.
 669  */
 670 enum vxge_hw_status __vxge_hw_device_initialize(struct __vxge_hw_device *hldev)
 671 {
 672         enum vxge_hw_status status = VXGE_HW_OK;
 673
 674         if (VXGE_HW_OK == __vxge_hw_device_is_privilaged(hldev->host_type,
 675                                 hldev->func_id)) {
 676                 /* Validate the pci-e link width and speed */
 677                 status = __vxge_hw_verify_pci_e_info(hldev);
 678                 if (status != VXGE_HW_OK)
 679                         goto exit;
 680         }
 681
 682         vxge_hw_wrr_rebalance(hldev);
 683 exit:
 684         return status;
 685 }
 686
 687 /**
 688  * vxge_hw_device_hw_info_get - Get the hw information
 689  * Returns the vpath mask that has the bits set for each vpath allocated
 690  * for the driver, FW version information and the first mac addresse for
 691  * each vpath
 692  */
 693 enum vxge_hw_status __devinit
 694 vxge_hw_device_hw_info_get(void __iomem *bar0,
 695                            struct vxge_hw_device_hw_info *hw_info)
 696 {
 697         u32 i;
 698         u64 val64;
 699         struct vxge_hw_toc_reg __iomem *toc;
 700         struct vxge_hw_mrpcim_reg __iomem *mrpcim_reg;
 701         struct vxge_hw_common_reg __iomem *common_reg;
 702         struct vxge_hw_vpath_reg __iomem *vpath_reg;
 703         struct vxge_hw_vpmgmt_reg __iomem *vpmgmt_reg;
 704         enum vxge_hw_status status;
 705
 706         memset(hw_info, 0, sizeof(struct vxge_hw_device_hw_info));
 707
 708         toc = __vxge_hw_device_toc_get(bar0);
 709         if (toc == NULL) {
 710                 status = VXGE_HW_ERR_CRITICAL;
 711                 goto exit;
 712         }
 713
 714         val64 = readq(&toc->toc_common_pointer);
 715         common_reg = (struct vxge_hw_common_reg __iomem *)(bar0 + val64);
 716
 717         status = __vxge_hw_device_vpath_reset_in_prog_check(
 718                 (u64 __iomem *)&common_reg->vpath_rst_in_prog);
 719         if (status != VXGE_HW_OK)
 720                 goto exit;
 721
 722         hw_info->vpath_mask = readq(&common_reg->vpath_assignments);
 723
 724         val64 = readq(&common_reg->host_type_assignments);
 725
 726         hw_info->host_type =
 727            (u32)VXGE_HW_HOST_TYPE_ASSIGNMENTS_GET_HOST_TYPE_ASSIGNMENTS(val64);
 728
 729         for (i = 0; i < VXGE_HW_MAX_VIRTUAL_PATHS; i++) {
 730
 731                 if (!((hw_info->vpath_mask) & vxge_mBIT(i)))
 732                         continue;
 733
 734                 val64 = readq(&toc->toc_vpmgmt_pointer[i]);
 735
 736                 vpmgmt_reg = (struct vxge_hw_vpmgmt_reg __iomem *)
 737                                 (bar0 + val64);
 738
 739                 hw_info->func_id = __vxge_hw_vpath_func_id_get(i, vpmgmt_reg);
 740                 if (__vxge_hw_device_access_rights_get(hw_info->host_type,
 741                         hw_info->func_id) &
 742                         VXGE_HW_DEVICE_ACCESS_RIGHT_MRPCIM) {
 743
 744                         val64 = readq(&toc->toc_mrpcim_pointer);
 745
 746                         mrpcim_reg = (struct vxge_hw_mrpcim_reg __iomem *)
 747                                         (bar0 + val64);
 748
 749                         writeq(0, &mrpcim_reg->xgmac_gen_fw_memo_mask);
 750                         wmb();
 751                 }
 752
 753                 val64 = readq(&toc->toc_vpath_pointer[i]);
 754
 755                 vpath_reg = (struct vxge_hw_vpath_reg __iomem *)(bar0 + val64);
 756
 757                 hw_info->function_mode =
 758                         __vxge_hw_vpath_pci_func_mode_get(i, vpath_reg);
 759
 760                 status = __vxge_hw_vpath_fw_ver_get(i, vpath_reg, hw_info);
 761                 if (status != VXGE_HW_OK)
 762                         goto exit;
 763
 764                 status = __vxge_hw_vpath_card_info_get(i, vpath_reg, hw_info);
 765                 if (status != VXGE_HW_OK)
 766                         goto exit;
 767
 768                 break;
 769         }
 770
 771         for (i = 0; i < VXGE_HW_MAX_VIRTUAL_PATHS; i++) {
 772
 773                 if (!((hw_info->vpath_mask) & vxge_mBIT(i)))
 774                         continue;
 775
 776                 val64 = readq(&toc->toc_vpath_pointer[i]);
 777                 vpath_reg = (struct vxge_hw_vpath_reg __iomem *)(bar0 + val64);
 778
 779                 status =  __vxge_hw_vpath_addr_get(i, vpath_reg,
 780                                 hw_info->mac_addrs[i],
 781                                 hw_info->mac_addr_masks[i]);
 782                 if (status != VXGE_HW_OK)
 783                         goto exit;
 784         }
 785 exit:
 786         return status;
 787 }
 788
 789 /*
 790  * vxge_hw_device_initialize - Initialize Titan device.
 791  * Initialize Titan device. Note that all the arguments of this public API
 792  * are 'IN', including @hldev. Driver cooperates with
 793  * OS to find new Titan device, locate its PCI and memory spaces.
 794  *
 795  * When done, the driver allocates sizeof(struct __vxge_hw_device) bytes for HW
 796  * to enable the latter to perform Titan hardware initialization.
 797  */
 798 enum vxge_hw_status __devinit
 799 vxge_hw_device_initialize(
 800         struct __vxge_hw_device **devh,
 801         struct vxge_hw_device_attr *attr,
 802         struct vxge_hw_device_config *device_config)
 803 {
 804         u32 i;
 805         u32 nblocks = 0;
 806         struct __vxge_hw_device *hldev = NULL;
 807         enum vxge_hw_status status = VXGE_HW_OK;
 808
 809         status = __vxge_hw_device_config_check(device_config);
 810         if (status != VXGE_HW_OK)
 811                 goto exit;
 812
 813         hldev = (struct __vxge_hw_device *)
 814                         vmalloc(sizeof(struct __vxge_hw_device));
 815         if (hldev == NULL) {
 816                 status = VXGE_HW_ERR_OUT_OF_MEMORY;
 817                 goto exit;
 818         }
 819
 820         memset(hldev, 0, sizeof(struct __vxge_hw_device));
 821         hldev->magic = VXGE_HW_DEVICE_MAGIC;
 822
 823         vxge_hw_device_debug_set(hldev, VXGE_ERR, VXGE_COMPONENT_ALL);
 824
 825         /* apply config */
 826         memcpy(&hldev->config, device_config,
 827                 sizeof(struct vxge_hw_device_config));
 828
 829         hldev->bar0 = attr->bar0;
 830         hldev->pdev = attr->pdev;
 831
 832         hldev->uld_callbacks.link_up = attr->uld_callbacks.link_up;
 833         hldev->uld_callbacks.link_down = attr->uld_callbacks.link_down;
 834         hldev->uld_callbacks.crit_err = attr->uld_callbacks.crit_err;
 835
 836         __vxge_hw_device_pci_e_init(hldev);
 837
 838         status = __vxge_hw_device_reg_addr_get(hldev);
 839         if (status != VXGE_HW_OK)
 840                 goto exit;
 841         __vxge_hw_device_id_get(hldev);
 842
 843         __vxge_hw_device_host_info_get(hldev);
 844
 845         /* Incrementing for stats blocks */
 846         nblocks++;
 847
 848         for (i = 0; i < VXGE_HW_MAX_VIRTUAL_PATHS; i++) {
 849
 850                 if (!(hldev->vpath_assignments & vxge_mBIT(i)))
 851                         continue;
 852
 853                 if (device_config->vp_config[i].ring.enable ==
 854                         VXGE_HW_RING_ENABLE)
 855                         nblocks += device_config->vp_config[i].ring.ring_blocks;
 856
 857                 if (device_config->vp_config[i].fifo.enable ==
 858                         VXGE_HW_FIFO_ENABLE)
 859                         nblocks += device_config->vp_config[i].fifo.fifo_blocks;
 860                 nblocks++;
 861         }
 862
 863         if (__vxge_hw_blockpool_create(hldev,
 864                 &hldev->block_pool,
 865                 device_config->dma_blockpool_initial + nblocks,
 866                 device_config->dma_blockpool_max + nblocks) != VXGE_HW_OK) {
 867
 868                 vxge_hw_device_terminate(hldev);
 869                 status = VXGE_HW_ERR_OUT_OF_MEMORY;
 870                 goto exit;
 871         }
 872
 873         status = __vxge_hw_device_initialize(hldev);
 874
 875         if (status != VXGE_HW_OK) {
 876                 vxge_hw_device_terminate(hldev);
 877                 goto exit;
 878         }
 879
 880         *devh = hldev;
 881 exit:
 882         return status;
 883 }
 884
 885 /*
 886  * vxge_hw_device_terminate - Terminate Titan device.
 887  * Terminate HW device.
 888  */
 889 void
 890 vxge_hw_device_terminate(struct __vxge_hw_device *hldev)
 891 {
 892         vxge_assert(hldev->magic == VXGE_HW_DEVICE_MAGIC);
 893
 894         hldev->magic = VXGE_HW_DEVICE_DEAD;
 895         __vxge_hw_blockpool_destroy(&hldev->block_pool);
 896         vfree(hldev);
 897 }
 898
 899 /*
 900  * vxge_hw_device_stats_get - Get the device hw statistics.
 901  * Returns the vpath h/w stats for the device.
 902  */
 903 enum vxge_hw_status
 904 vxge_hw_device_stats_get(struct __vxge_hw_device *hldev,
 905                         struct vxge_hw_device_stats_hw_info *hw_stats)
 906 {
 907         u32 i;
 908         enum vxge_hw_status status = VXGE_HW_OK;
 909
 910         for (i = 0; i < VXGE_HW_MAX_VIRTUAL_PATHS; i++) {
 911
 912                 if (!(hldev->vpaths_deployed & vxge_mBIT(i)) ||
 913                         (hldev->virtual_paths[i].vp_open ==
 914                                 VXGE_HW_VP_NOT_OPEN))
 915                         continue;
 916
 917                 memcpy(hldev->virtual_paths[i].hw_stats_sav,
 918                                 hldev->virtual_paths[i].hw_stats,
 919                                 sizeof(struct vxge_hw_vpath_stats_hw_info));
 920
 921                 status = __vxge_hw_vpath_stats_get(
 922                         &hldev->virtual_paths[i],
 923                         hldev->virtual_paths[i].hw_stats);
 924         }
 925
 926         memcpy(hw_stats, &hldev->stats.hw_dev_info_stats,
 927                         sizeof(struct vxge_hw_device_stats_hw_info));
 928
 929         return status;
 930 }
 931
 932 /*
 933  * vxge_hw_driver_stats_get - Get the device sw statistics.
 934  * Returns the vpath s/w stats for the device.
 935  */
 936 enum vxge_hw_status vxge_hw_driver_stats_get(
 937                         struct __vxge_hw_device *hldev,
 938                         struct vxge_hw_device_stats_sw_info *sw_stats)
 939 {
 940         enum vxge_hw_status status = VXGE_HW_OK;
 941
 942         memcpy(sw_stats, &hldev->stats.sw_dev_info_stats,
 943                 sizeof(struct vxge_hw_device_stats_sw_info));
 944
 945         return status;
 946 }
 947
 948 /*
 949  * vxge_hw_mrpcim_stats_access - Access the statistics from the given location
 950  *                           and offset and perform an operation
 951  * Get the statistics from the given location and offset.
 952  */
 953 enum vxge_hw_status
 954 vxge_hw_mrpcim_stats_access(struct __vxge_hw_device *hldev,
 955                             u32 operation, u32 location, u32 offset, u64 *stat)
 956 {
 957         u64 val64;
 958         enum vxge_hw_status status = VXGE_HW_OK;
 959
 960         status = __vxge_hw_device_is_privilaged(hldev->host_type,
 961                         hldev->func_id);
 962         if (status != VXGE_HW_OK)
 963                 goto exit;
 964
 965         val64 = VXGE_HW_XMAC_STATS_SYS_CMD_OP(operation) |
 966                 VXGE_HW_XMAC_STATS_SYS_CMD_STROBE |
 967                 VXGE_HW_XMAC_STATS_SYS_CMD_LOC_SEL(location) |
 968                 VXGE_HW_XMAC_STATS_SYS_CMD_OFFSET_SEL(offset);
 969
 970         status = __vxge_hw_pio_mem_write64(val64,
 971                                 &hldev->mrpcim_reg->xmac_stats_sys_cmd,
 972                                 VXGE_HW_XMAC_STATS_SYS_CMD_STROBE,
 973                                 hldev->config.device_poll_millis);
 974
 975         if ((status == VXGE_HW_OK) && (operation == VXGE_HW_STATS_OP_READ))
 976                 *stat = readq(&hldev->mrpcim_reg->xmac_stats_sys_data);
 977         else
 978                 *stat = 0;
 979 exit:
 980         return status;
 981 }
 982
 983 /*
 984  * vxge_hw_device_xmac_aggr_stats_get - Get the Statistics on aggregate port
 985  * Get the Statistics on aggregate port
 986  */
 987 enum vxge_hw_status
 988 vxge_hw_device_xmac_aggr_stats_get(struct __vxge_hw_device *hldev, u32 port,
 989                                    struct vxge_hw_xmac_aggr_stats *aggr_stats)
 990 {
 991         u64 *val64;
 992         int i;
 993         u32 offset = VXGE_HW_STATS_AGGRn_OFFSET;
 994         enum vxge_hw_status status = VXGE_HW_OK;
 995
 996         val64 = (u64 *)aggr_stats;
 997
 998         status = __vxge_hw_device_is_privilaged(hldev->host_type,
 999                         hldev->func_id);
1000         if (status != VXGE_HW_OK)
1001                 goto exit;
1002
1003         for (i = 0; i < sizeof(struct vxge_hw_xmac_aggr_stats) / 8; i++) {
1004                 status = vxge_hw_mrpcim_stats_access(hldev,
1005                                         VXGE_HW_STATS_OP_READ,
1006                                         VXGE_HW_STATS_LOC_AGGR,
1007                                         ((offset + (104 * port)) >> 3), val64);
1008                 if (status != VXGE_HW_OK)
1009                         goto exit;
1010
1011                 offset += 8;
1012                 val64++;
1013         }
1014 exit:
1015         return status;
1016 }
1017
1018 /*
1019  * vxge_hw_device_xmac_port_stats_get - Get the Statistics on a port
1020  * Get the Statistics on port
1021  */
1022 enum vxge_hw_status
1023 vxge_hw_device_xmac_port_stats_get(struct __vxge_hw_device *hldev, u32 port,
1024                                    struct vxge_hw_xmac_port_stats *port_stats)
1025 {
1026         u64 *val64;
1027         enum vxge_hw_status status = VXGE_HW_OK;
1028         int i;
1029         u32 offset = 0x0;
1030         val64 = (u64 *) port_stats;
1031
1032         status = __vxge_hw_device_is_privilaged(hldev->host_type,
1033                         hldev->func_id);
1034         if (status != VXGE_HW_OK)
1035                 goto exit;
1036
1037         for (i = 0; i < sizeof(struct vxge_hw_xmac_port_stats) / 8; i++) {
1038                 status = vxge_hw_mrpcim_stats_access(hldev,
1039                                         VXGE_HW_STATS_OP_READ,
1040                                         VXGE_HW_STATS_LOC_AGGR,
1041                                         ((offset + (608 * port)) >> 3), val64);
1042                 if (status != VXGE_HW_OK)
1043                         goto exit;
1044
1045                 offset += 8;
1046                 val64++;
1047         }
1048
1049 exit:
1050         return status;
1051 }
1052
1053 /*
1054  * vxge_hw_device_xmac_stats_get - Get the XMAC Statistics
1055  * Get the XMAC Statistics
1056  */
1057 enum vxge_hw_status
1058 vxge_hw_device_xmac_stats_get(struct __vxge_hw_device *hldev,
1059                               struct vxge_hw_xmac_stats *xmac_stats)
1060 {
1061         enum vxge_hw_status status = VXGE_HW_OK;
1062         u32 i;
1063
1064         status = vxge_hw_device_xmac_aggr_stats_get(hldev,
1065                                         0, &xmac_stats->aggr_stats[0]);
1066
1067         if (status != VXGE_HW_OK)
1068                 goto exit;
1069
1070         status = vxge_hw_device_xmac_aggr_stats_get(hldev,
1071                                 1, &xmac_stats->aggr_stats[1]);
1072         if (status != VXGE_HW_OK)
1073                 goto exit;
1074
1075         for (i = 0; i <= VXGE_HW_MAC_MAX_MAC_PORT_ID; i++) {
1076
1077                 status = vxge_hw_device_xmac_port_stats_get(hldev,
1078                                         i, &xmac_stats->port_stats[i]);
1079                 if (status != VXGE_HW_OK)
1080                         goto exit;
1081         }
1082
1083         for (i = 0; i < VXGE_HW_MAX_VIRTUAL_PATHS; i++) {
1084
1085                 if (!(hldev->vpaths_deployed & vxge_mBIT(i)))
1086                         continue;
1087
1088                 status = __vxge_hw_vpath_xmac_tx_stats_get(
1089                                         &hldev->virtual_paths[i],
1090                                         &xmac_stats->vpath_tx_stats[i]);
1091                 if (status != VXGE_HW_OK)
1092                         goto exit;
1093
1094                 status = __vxge_hw_vpath_xmac_rx_stats_get(
1095                                         &hldev->virtual_paths[i],
1096                                         &xmac_stats->vpath_rx_stats[i]);
1097                 if (status != VXGE_HW_OK)
1098                         goto exit;
1099         }
1100 exit:
1101         return status;
1102 }
1103
1104 /*
1105  * vxge_hw_device_debug_set - Set the debug module, level and timestamp
1106  * This routine is used to dynamically change the debug output
1107  */
1108 void vxge_hw_device_debug_set(struct __vxge_hw_device *hldev,
1109                               enum vxge_debug_level level, u32 mask)
1110 {
1111         if (hldev == NULL)
1112                 return;
1113
1114 #if defined(VXGE_DEBUG_TRACE_MASK) || \
1115         defined(VXGE_DEBUG_ERR_MASK)
1116         hldev->debug_module_mask = mask;
1117         hldev->debug_level = level;
1118 #endif
1119
1120 #if defined(VXGE_DEBUG_ERR_MASK)
1121         hldev->level_err = level & VXGE_ERR;
1122 #endif
1123
1124 #if defined(VXGE_DEBUG_TRACE_MASK)
1125         hldev->level_trace = level & VXGE_TRACE;
1126 #endif
1127 }
1128
1129 /*
1130  * vxge_hw_device_error_level_get - Get the error level
1131  * This routine returns the current error level set
1132  */
1133 u32 vxge_hw_device_error_level_get(struct __vxge_hw_device *hldev)
1134 {
1135 #if defined(VXGE_DEBUG_ERR_MASK)
1136         if (hldev == NULL)
1137                 return VXGE_ERR;
1138         else
1139                 return hldev->level_err;
1140 #else
1141         return 0;
1142 #endif
1143 }
1144
1145 /*
1146  * vxge_hw_device_trace_level_get - Get the trace level
1147  * This routine returns the current trace level set
1148  */
1149 u32 vxge_hw_device_trace_level_get(struct __vxge_hw_device *hldev)
1150 {
1151 #if defined(VXGE_DEBUG_TRACE_MASK)
1152         if (hldev == NULL)
1153                 return VXGE_TRACE;
1154         else
1155                 return hldev->level_trace;
1156 #else
1157         return 0;
1158 #endif
1159 }
1160 /*
1161  * vxge_hw_device_debug_mask_get - Get the debug mask
1162  * This routine returns the current debug mask set
1163  */
1164 u32 vxge_hw_device_debug_mask_get(struct __vxge_hw_device *hldev)
1165 {
1166 #if defined(VXGE_DEBUG_TRACE_MASK) || defined(VXGE_DEBUG_ERR_MASK)
1167         if (hldev == NULL)
1168                 return 0;
1169         return hldev->debug_module_mask;
1170 #else
1171         return 0;
1172 #endif
1173 }
1174
1175 /*
1176  * vxge_hw_getpause_data -Pause frame frame generation and reception.
1177  * Returns the Pause frame generation and reception capability of the NIC.
1178  */
1179 enum vxge_hw_status vxge_hw_device_getpause_data(struct __vxge_hw_device *hldev,
1180                                                  u32 port, u32 *tx, u32 *rx)
1181 {
1182         u64 val64;
1183         enum vxge_hw_status status = VXGE_HW_OK;
1184
1185         if ((hldev == NULL) || (hldev->magic != VXGE_HW_DEVICE_MAGIC)) {
1186                 status = VXGE_HW_ERR_INVALID_DEVICE;
1187                 goto exit;
1188         }
1189
1190         if (port > VXGE_HW_MAC_MAX_MAC_PORT_ID) {
1191                 status = VXGE_HW_ERR_INVALID_PORT;
1192                 goto exit;
1193         }
1194
1195         if (!(hldev->access_rights & VXGE_HW_DEVICE_ACCESS_RIGHT_MRPCIM)) {
1196                 status = VXGE_HW_ERR_PRIVILAGED_OPEARATION;
1197                 goto exit;
1198         }
1199
1200         val64 = readq(&hldev->mrpcim_reg->rxmac_pause_cfg_port[port]);
1201         if (val64 & VXGE_HW_RXMAC_PAUSE_CFG_PORT_GEN_EN)
1202                 *tx = 1;
1203         if (val64 & VXGE_HW_RXMAC_PAUSE_CFG_PORT_RCV_EN)
1204                 *rx = 1;
1205 exit:
1206         return status;
1207 }
1208
1209 /*
1210  * vxge_hw_device_setpause_data -  set/reset pause frame generation.
1211  * It can be used to set or reset Pause frame generation or reception
1212  * support of the NIC.
1213  */
1214
1215 enum vxge_hw_status vxge_hw_device_setpause_data(struct __vxge_hw_device *hldev,
1216                                                  u32 port, u32 tx, u32 rx)
1217 {
1218         u64 val64;
1219         enum vxge_hw_status status = VXGE_HW_OK;
1220
1221         if ((hldev == NULL) || (hldev->magic != VXGE_HW_DEVICE_MAGIC)) {
1222                 status = VXGE_HW_ERR_INVALID_DEVICE;
1223                 goto exit;
1224         }
1225
1226         if (port > VXGE_HW_MAC_MAX_MAC_PORT_ID) {
1227                 status = VXGE_HW_ERR_INVALID_PORT;
1228                 goto exit;
1229         }
1230
1231         status = __vxge_hw_device_is_privilaged(hldev->host_type,
1232                         hldev->func_id);
1233         if (status != VXGE_HW_OK)
1234                 goto exit;
1235
1236         val64 = readq(&hldev->mrpcim_reg->rxmac_pause_cfg_port[port]);
1237         if (tx)
1238                 val64 |= VXGE_HW_RXMAC_PAUSE_CFG_PORT_GEN_EN;
1239         else
1240                 val64 &= ~VXGE_HW_RXMAC_PAUSE_CFG_PORT_GEN_EN;
1241         if (rx)
1242                 val64 |= VXGE_HW_RXMAC_PAUSE_CFG_PORT_RCV_EN;
1243         else
1244                 val64 &= ~VXGE_HW_RXMAC_PAUSE_CFG_PORT_RCV_EN;
1245
1246         writeq(val64, &hldev->mrpcim_reg->rxmac_pause_cfg_port[port]);
1247 exit:
1248         return status;
1249 }
1250
1251 u16 vxge_hw_device_link_width_get(struct __vxge_hw_device *hldev)
1252 {
1253         int link_width, exp_cap;
1254         u16 lnk;
1255
1256         exp_cap = pci_find_capability(hldev->pdev, PCI_CAP_ID_EXP);
1257         pci_read_config_word(hldev->pdev, exp_cap + PCI_EXP_LNKSTA, &lnk);
1258         link_width = (lnk & VXGE_HW_PCI_EXP_LNKCAP_LNK_WIDTH) >> 4;
1259         return link_width;
1260 }
1261
1262 /*
1263  * __vxge_hw_ring_block_memblock_idx - Return the memblock index
1264  * This function returns the index of memory block
1265  */
1266 static inline u32
1267 __vxge_hw_ring_block_memblock_idx(u8 *block)
1268 {
1269         return (u32)*((u64 *)(block + VXGE_HW_RING_MEMBLOCK_IDX_OFFSET));
1270 }
1271
1272 /*
1273  * __vxge_hw_ring_block_memblock_idx_set - Sets the memblock index
1274  * This function sets index to a memory block
1275  */
1276 static inline void
1277 __vxge_hw_ring_block_memblock_idx_set(u8 *block, u32 memblock_idx)
1278 {
1279         *((u64 *)(block + VXGE_HW_RING_MEMBLOCK_IDX_OFFSET)) = memblock_idx;
1280 }
1281
1282 /*
1283  * __vxge_hw_ring_block_next_pointer_set - Sets the next block pointer
1284  * in RxD block
1285  * Sets the next block pointer in RxD block
1286  */
1287 static inline void
1288 __vxge_hw_ring_block_next_pointer_set(u8 *block, dma_addr_t dma_next)
1289 {
1290         *((u64 *)(block + VXGE_HW_RING_NEXT_BLOCK_POINTER_OFFSET)) = dma_next;
1291 }
1292
1293 /*
1294  * __vxge_hw_ring_first_block_address_get - Returns the dma address of the
1295  *             first block
1296  * Returns the dma address of the first RxD block
1297  */
1298 u64 __vxge_hw_ring_first_block_address_get(struct __vxge_hw_ring *ring)
1299 {
1300         struct vxge_hw_mempool_dma *dma_object;
1301
1302         dma_object = ring->mempool->memblocks_dma_arr;
1303         vxge_assert(dma_object != NULL);
1304
1305         return dma_object->addr;
1306 }
1307
1308 /*
1309  * __vxge_hw_ring_item_dma_addr - Return the dma address of an item
1310  * This function returns the dma address of a given item
1311  */
1312 static dma_addr_t __vxge_hw_ring_item_dma_addr(struct vxge_hw_mempool *mempoolh,
1313                                                void *item)
1314 {
1315         u32 memblock_idx;
1316         void *memblock;
1317         struct vxge_hw_mempool_dma *memblock_dma_object;
1318         ptrdiff_t dma_item_offset;
1319
1320         /* get owner memblock index */
1321         memblock_idx = __vxge_hw_ring_block_memblock_idx(item);
1322
1323         /* get owner memblock by memblock index */
1324         memblock = mempoolh->memblocks_arr[memblock_idx];
1325
1326         /* get memblock DMA object by memblock index */
1327         memblock_dma_object = mempoolh->memblocks_dma_arr + memblock_idx;
1328
1329         /* calculate offset in the memblock of this item */
1330         dma_item_offset = (u8 *)item - (u8 *)memblock;
1331
1332         return memblock_dma_object->addr + dma_item_offset;
1333 }
1334
1335 /*
1336  * __vxge_hw_ring_rxdblock_link - Link the RxD blocks
1337  * This function returns the dma address of a given item
1338  */
1339 static void __vxge_hw_ring_rxdblock_link(struct vxge_hw_mempool *mempoolh,
1340                                          struct __vxge_hw_ring *ring, u32 from,
1341                                          u32 to)
1342 {
1343         u8 *to_item , *from_item;
1344         dma_addr_t to_dma;
1345
1346         /* get "from" RxD block */
1347         from_item = mempoolh->items_arr[from];
1348         vxge_assert(from_item);
1349
1350         /* get "to" RxD block */
1351         to_item = mempoolh->items_arr[to];
1352         vxge_assert(to_item);
1353
1354         /* return address of the beginning of previous RxD block */
1355         to_dma = __vxge_hw_ring_item_dma_addr(mempoolh, to_item);
1356
1357         /* set next pointer for this RxD block to point on
1358          * previous item's DMA start address */
1359         __vxge_hw_ring_block_next_pointer_set(from_item, to_dma);
1360 }
1361
1362 /*
1363  * __vxge_hw_ring_mempool_item_alloc - Allocate List blocks for RxD
1364  * block callback
1365  * This function is callback passed to __vxge_hw_mempool_create to create memory
1366  * pool for RxD block
1367  */
1368 static void
1369 __vxge_hw_ring_mempool_item_alloc(struct vxge_hw_mempool *mempoolh,
1370                                   u32 memblock_index,
1371                                   struct vxge_hw_mempool_dma *dma_object,
1372                                   u32 index, u32 is_last)
1373 {
1374         u32 i;
1375         void *item = mempoolh->items_arr[index];
1376         struct __vxge_hw_ring *ring =
1377                 (struct __vxge_hw_ring *)mempoolh->userdata;
1378
1379         /* format rxds array */
1380         for (i = 0; i < ring->rxds_per_block; i++) {
1381                 void *rxdblock_priv;
1382                 void *uld_priv;
1383                 struct vxge_hw_ring_rxd_1 *rxdp;
1384
1385                 u32 reserve_index = ring->channel.reserve_ptr -
1386                                 (index * ring->rxds_per_block + i + 1);
1387                 u32 memblock_item_idx;
1388
1389                 ring->channel.reserve_arr[reserve_index] = ((u8 *)item) +
1390                                                 i * ring->rxd_size;
1391
1392                 /* Note: memblock_item_idx is index of the item within
1393                  *       the memblock. For instance, in case of three RxD-blocks
1394                  *       per memblock this value can be 0, 1 or 2. */
1395                 rxdblock_priv = __vxge_hw_mempool_item_priv(mempoolh,
1396                                         memblock_index, item,
1397                                         &memblock_item_idx);
1398
1399                 rxdp = (struct vxge_hw_ring_rxd_1 *)
1400                                 ring->channel.reserve_arr[reserve_index];
1401
1402                 uld_priv = ((u8 *)rxdblock_priv + ring->rxd_priv_size * i);
1403
1404                 /* pre-format Host_Control */
1405                 rxdp->host_control = (u64)(size_t)uld_priv;
1406         }
1407
1408         __vxge_hw_ring_block_memblock_idx_set(item, memblock_index);
1409
1410         if (is_last) {
1411                 /* link last one with first one */
1412                 __vxge_hw_ring_rxdblock_link(mempoolh, ring, index, 0);
1413         }
1414
1415         if (index > 0) {
1416                 /* link this RxD block with previous one */
1417                 __vxge_hw_ring_rxdblock_link(mempoolh, ring, index - 1, index);
1418         }
1419
1420         return;
1421 }
1422
1423 /*
1424  * __vxge_hw_ring_initial_replenish - Initial replenish of RxDs
1425  * This function replenishes the RxDs from reserve array to work array
1426  */
1427 enum vxge_hw_status
1428 vxge_hw_ring_replenish(struct __vxge_hw_ring *ring, u16 min_flag)
1429 {
1430         void *rxd;
1431         int i = 0;
1432         struct __vxge_hw_channel *channel;
1433         enum vxge_hw_status status = VXGE_HW_OK;
1434
1435         channel = &ring->channel;
1436
1437         while (vxge_hw_channel_dtr_count(channel) > 0) {
1438
1439                 status = vxge_hw_ring_rxd_reserve(ring, &rxd);
1440
1441                 vxge_assert(status == VXGE_HW_OK);
1442
1443                 if (ring->rxd_init) {
1444                         status = ring->rxd_init(rxd, channel->userdata);
1445                         if (status != VXGE_HW_OK) {
1446                                 vxge_hw_ring_rxd_free(ring, rxd);
1447                                 goto exit;
1448                         }
1449                 }
1450
1451                 vxge_hw_ring_rxd_post(ring, rxd);
1452                 if (min_flag) {
1453                         i++;
1454                         if (i == VXGE_HW_RING_MIN_BUFF_ALLOCATION)
1455                                 break;
1456                 }
1457         }
1458         status = VXGE_HW_OK;
1459 exit:
1460         return status;
1461 }
1462
1463 /*
1464  * __vxge_hw_ring_create - Create a Ring
1465  * This function creates Ring and initializes it.
1466  *
1467  */
1468 enum vxge_hw_status
1469 __vxge_hw_ring_create(struct __vxge_hw_vpath_handle *vp,
1470                       struct vxge_hw_ring_attr *attr)
1471 {
1472         enum vxge_hw_status status = VXGE_HW_OK;
1473         struct __vxge_hw_ring *ring;
1474         u32 ring_length;
1475         struct vxge_hw_ring_config *config;
1476         struct __vxge_hw_device *hldev;
1477         u32 vp_id;
1478         struct vxge_hw_mempool_cbs ring_mp_callback;
1479
1480         if ((vp == NULL) || (attr == NULL)) {
1481                 status = VXGE_HW_FAIL;
1482                 goto exit;
1483         }
1484
1485         hldev = vp->vpath->hldev;
1486         vp_id = vp->vpath->vp_id;
1487
1488         config = &hldev->config.vp_config[vp_id].ring;
1489
1490         ring_length = config->ring_blocks *
1491                         vxge_hw_ring_rxds_per_block_get(config->buffer_mode);
1492
1493         ring = (struct __vxge_hw_ring *)__vxge_hw_channel_allocate(vp,
1494                                                 VXGE_HW_CHANNEL_TYPE_RING,
1495                                                 ring_length,
1496                                                 attr->per_rxd_space,
1497                                                 attr->userdata);
1498
1499         if (ring == NULL) {
1500                 status = VXGE_HW_ERR_OUT_OF_MEMORY;
1501                 goto exit;
1502         }
1503
1504         vp->vpath->ringh = ring;
1505         ring->vp_id = vp_id;
1506         ring->vp_reg = vp->vpath->vp_reg;
1507         ring->common_reg = hldev->common_reg;
1508         ring->stats = &vp->vpath->sw_stats->ring_stats;
1509         ring->config = config;
1510         ring->callback = attr->callback;
1511         ring->rxd_init = attr->rxd_init;
1512         ring->rxd_term = attr->rxd_term;
1513         ring->buffer_mode = config->buffer_mode;
1514         ring->rxds_limit = config->rxds_limit;
1515
1516         ring->rxd_size = vxge_hw_ring_rxd_size_get(config->buffer_mode);
1517         ring->rxd_priv_size =
1518                 sizeof(struct __vxge_hw_ring_rxd_priv) + attr->per_rxd_space;
1519         ring->per_rxd_space = attr->per_rxd_space;
1520
1521         ring->rxd_priv_size =
1522                 ((ring->rxd_priv_size + VXGE_CACHE_LINE_SIZE - 1) /
1523                 VXGE_CACHE_LINE_SIZE) * VXGE_CACHE_LINE_SIZE;
1524
1525         /* how many RxDs can fit into one block. Depends on configured
1526          * buffer_mode. */
1527         ring->rxds_per_block =
1528                 vxge_hw_ring_rxds_per_block_get(config->buffer_mode);
1529
1530         /* calculate actual RxD block private size */
1531         ring->rxdblock_priv_size = ring->rxd_priv_size * ring->rxds_per_block;
1532         ring_mp_callback.item_func_alloc = __vxge_hw_ring_mempool_item_alloc;
1533         ring->mempool = __vxge_hw_mempool_create(hldev,
1534                                 VXGE_HW_BLOCK_SIZE,
1535                                 VXGE_HW_BLOCK_SIZE,
1536                                 ring->rxdblock_priv_size,
1537                                 ring->config->ring_blocks,
1538                                 ring->config->ring_blocks,
1539                                 &ring_mp_callback,
1540                                 ring);
1541
1542         if (ring->mempool == NULL) {
1543                 __vxge_hw_ring_delete(vp);
1544                 return VXGE_HW_ERR_OUT_OF_MEMORY;
1545         }
1546
1547         status = __vxge_hw_channel_initialize(&ring->channel);
1548         if (status != VXGE_HW_OK) {
1549                 __vxge_hw_ring_delete(vp);
1550                 goto exit;
1551         }
1552
1553         /* Note:
1554          * Specifying rxd_init callback means two things:
1555          * 1) rxds need to be initialized by driver at channel-open time;
1556          * 2) rxds need to be posted at channel-open time
1557          *    (that's what the initial_replenish() below does)
1558          * Currently we don't have a case when the 1) is done without the 2).
1559          */
1560         if (ring->rxd_init) {
1561                 status = vxge_hw_ring_replenish(ring, 1);
1562                 if (status != VXGE_HW_OK) {
1563                         __vxge_hw_ring_delete(vp);
1564                         goto exit;
1565                 }
1566         }
1567
1568         /* initial replenish will increment the counter in its post() routine,
1569          * we have to reset it */
1570         ring->stats->common_stats.usage_cnt = 0;
1571 exit:
1572         return status;
1573 }
1574
1575 /*
1576  * __vxge_hw_ring_abort - Returns the RxD
1577  * This function terminates the RxDs of ring
1578  */
1579 enum vxge_hw_status __vxge_hw_ring_abort(struct __vxge_hw_ring *ring)
1580 {
1581         void *rxdh;
1582         struct __vxge_hw_channel *channel;
1583
1584         channel = &ring->channel;
1585
1586         for (;;) {
1587                 vxge_hw_channel_dtr_try_complete(channel, &rxdh);
1588
1589                 if (rxdh == NULL)
1590                         break;
1591
1592                 vxge_hw_channel_dtr_complete(channel);
1593
1594                 if (ring->rxd_term)
1595                         ring->rxd_term(rxdh, VXGE_HW_RXD_STATE_POSTED,
1596                                 channel->userdata);
1597
1598                 vxge_hw_channel_dtr_free(channel, rxdh);
1599         }
1600
1601         return VXGE_HW_OK;
1602 }
1603
1604 /*
1605  * __vxge_hw_ring_reset - Resets the ring
1606  * This function resets the ring during vpath reset operation
1607  */
1608 enum vxge_hw_status __vxge_hw_ring_reset(struct __vxge_hw_ring *ring)
1609 {
1610         enum vxge_hw_status status = VXGE_HW_OK;
1611         struct __vxge_hw_channel *channel;
1612
1613         channel = &ring->channel;
1614
1615         __vxge_hw_ring_abort(ring);
1616
1617         status = __vxge_hw_channel_reset(channel);
1618
1619         if (status != VXGE_HW_OK)
1620                 goto exit;
1621
1622         if (ring->rxd_init) {
1623                 status = vxge_hw_ring_replenish(ring, 1);
1624                 if (status != VXGE_HW_OK)
1625                         goto exit;
1626         }
1627 exit:
1628         return status;
1629 }
1630
1631 /*
1632  * __vxge_hw_ring_delete - Removes the ring
1633  * This function freeup the memory pool and removes the ring
1634  */
1635 enum vxge_hw_status __vxge_hw_ring_delete(struct __vxge_hw_vpath_handle *vp)
1636 {
1637         struct __vxge_hw_ring *ring = vp->vpath->ringh;
1638
1639         __vxge_hw_ring_abort(ring);
1640
1641         if (ring->mempool)
1642                 __vxge_hw_mempool_destroy(ring->mempool);
1643
1644         vp->vpath->ringh = NULL;
1645         __vxge_hw_channel_free(&ring->channel);
1646
1647         return VXGE_HW_OK;
1648 }
1649
1650 /*
1651  * __vxge_hw_mempool_grow
1652  * Will resize mempool up to %num_allocate value.
1653  */
1654 enum vxge_hw_status
1655 __vxge_hw_mempool_grow(struct vxge_hw_mempool *mempool, u32 num_allocate,
1656                        u32 *num_allocated)
1657 {
1658         u32 i, first_time = mempool->memblocks_allocated == 0 ? 1 : 0;
1659         u32 n_items = mempool->items_per_memblock;
1660         u32 start_block_idx = mempool->memblocks_allocated;
1661         u32 end_block_idx = mempool->memblocks_allocated + num_allocate;
1662         enum vxge_hw_status status = VXGE_HW_OK;
1663
1664         *num_allocated = 0;
1665
1666         if (end_block_idx > mempool->memblocks_max) {
1667                 status = VXGE_HW_ERR_OUT_OF_MEMORY;
1668                 goto exit;
1669         }
1670
1671         for (i = start_block_idx; i < end_block_idx; i++) {
1672                 u32 j;
1673                 u32 is_last = ((end_block_idx - 1) == i);
1674                 struct vxge_hw_mempool_dma *dma_object =
1675                         mempool->memblocks_dma_arr + i;
1676                 void *the_memblock;
1677
1678                 /* allocate memblock's private part. Each DMA memblock
1679                  * has a space allocated for item's private usage upon
1680                  * mempool's user request. Each time mempool grows, it will
1681                  * allocate new memblock and its private part at once.
1682                  * This helps to minimize memory usage a lot. */
1683                 mempool->memblocks_priv_arr[i] =
1684                                 vmalloc(mempool->items_priv_size * n_items);
1685                 if (mempool->memblocks_priv_arr[i] == NULL) {
1686                         status = VXGE_HW_ERR_OUT_OF_MEMORY;
1687                         goto exit;
1688                 }
1689
1690                 memset(mempool->memblocks_priv_arr[i], 0,
1691                              mempool->items_priv_size * n_items);
1692
1693                 /* allocate DMA-capable memblock */
1694                 mempool->memblocks_arr[i] =
1695                         __vxge_hw_blockpool_malloc(mempool->devh,
1696                                 mempool->memblock_size, dma_object);
1697                 if (mempool->memblocks_arr[i] == NULL) {
1698                         vfree(mempool->memblocks_priv_arr[i]);
1699                         status = VXGE_HW_ERR_OUT_OF_MEMORY;
1700                         goto exit;
1701                 }
1702
1703                 (*num_allocated)++;
1704                 mempool->memblocks_allocated++;
1705
1706                 memset(mempool->memblocks_arr[i], 0, mempool->memblock_size);
1707
1708                 the_memblock = mempool->memblocks_arr[i];
1709
1710                 /* fill the items hash array */
1711                 for (j = 0; j < n_items; j++) {
1712                         u32 index = i * n_items + j;
1713
1714                         if (first_time && index >= mempool->items_initial)
1715                                 break;
1716
1717                         mempool->items_arr[index] =
1718                                 ((char *)the_memblock + j*mempool->item_size);
1719
1720                         /* let caller to do more job on each item */
1721                         if (mempool->item_func_alloc != NULL)
1722                                 mempool->item_func_alloc(mempool, i,
1723                                         dma_object, index, is_last);
1724
1725                         mempool->items_current = index + 1;
1726                 }
1727
1728                 if (first_time && mempool->items_current ==
1729                                         mempool->items_initial)
1730                         break;
1731         }
1732 exit:
1733         return status;
1734 }
1735
1736 /*
1737  * vxge_hw_mempool_create
1738  * This function will create memory pool object. Pool may grow but will
1739  * never shrink. Pool consists of number of dynamically allocated blocks
1740  * with size enough to hold %items_initial number of items. Memory is
1741  * DMA-able but client must map/unmap before interoperating with the device.
1742  */
1743 struct vxge_hw_mempool*
1744 __vxge_hw_mempool_create(
1745         struct __vxge_hw_device *devh,
1746         u32 memblock_size,
1747         u32 item_size,
1748         u32 items_priv_size,
1749         u32 items_initial,
1750         u32 items_max,
1751         struct vxge_hw_mempool_cbs *mp_callback,
1752         void *userdata)
1753 {
1754         enum vxge_hw_status status = VXGE_HW_OK;
1755         u32 memblocks_to_allocate;
1756         struct vxge_hw_mempool *mempool = NULL;
1757         u32 allocated;
1758
1759         if (memblock_size < item_size) {
1760                 status = VXGE_HW_FAIL;
1761                 goto exit;
1762         }
1763
1764         mempool = (struct vxge_hw_mempool *)
1765                         vmalloc(sizeof(struct vxge_hw_mempool));
1766         if (mempool == NULL) {
1767                 status = VXGE_HW_ERR_OUT_OF_MEMORY;
1768                 goto exit;
1769         }
1770         memset(mempool, 0, sizeof(struct vxge_hw_mempool));
1771
1772         mempool->devh                   = devh;
1773         mempool->memblock_size          = memblock_size;
1774         mempool->items_max              = items_max;
1775         mempool->items_initial          = items_initial;
1776         mempool->item_size              = item_size;
1777         mempool->items_priv_size        = items_priv_size;
1778         mempool->item_func_alloc        = mp_callback->item_func_alloc;
1779         mempool->userdata               = userdata;
1780
1781         mempool->memblocks_allocated = 0;
1782
1783         mempool->items_per_memblock = memblock_size / item_size;
1784
1785         mempool->memblocks_max = (items_max + mempool->items_per_memblock - 1) /
1786                                         mempool->items_per_memblock;
1787
1788         /* allocate array of memblocks */
1789         mempool->memblocks_arr =
1790                 (void **) vmalloc(sizeof(void *) * mempool->memblocks_max);
1791         if (mempool->memblocks_arr == NULL) {
1792                 __vxge_hw_mempool_destroy(mempool);
1793                 status = VXGE_HW_ERR_OUT_OF_MEMORY;
1794                 mempool = NULL;
1795                 goto exit;
1796         }
1797         memset(mempool->memblocks_arr, 0,
1798                 sizeof(void *) * mempool->memblocks_max);
1799
1800         /* allocate array of private parts of items per memblocks */
1801         mempool->memblocks_priv_arr =
1802                 (void **) vmalloc(sizeof(void *) * mempool->memblocks_max);
1803         if (mempool->memblocks_priv_arr == NULL) {
1804                 __vxge_hw_mempool_destroy(mempool);
1805                 status = VXGE_HW_ERR_OUT_OF_MEMORY;
1806                 mempool = NULL;
1807                 goto exit;
1808         }
1809         memset(mempool->memblocks_priv_arr, 0,
1810                     sizeof(void *) * mempool->memblocks_max);
1811
1812         /* allocate array of memblocks DMA objects */
1813         mempool->memblocks_dma_arr = (struct vxge_hw_mempool_dma *)
1814                 vmalloc(sizeof(struct vxge_hw_mempool_dma) *
1815                         mempool->memblocks_max);
1816
1817         if (mempool->memblocks_dma_arr == NULL) {
1818                 __vxge_hw_mempool_destroy(mempool);
1819                 status = VXGE_HW_ERR_OUT_OF_MEMORY;
1820                 mempool = NULL;
1821                 goto exit;
1822         }
1823         memset(mempool->memblocks_dma_arr, 0,
1824                         sizeof(struct vxge_hw_mempool_dma) *
1825                         mempool->memblocks_max);
1826
1827         /* allocate hash array of items */
1828         mempool->items_arr =
1829                 (void **) vmalloc(sizeof(void *) * mempool->items_max);
1830         if (mempool->items_arr == NULL) {
1831                 __vxge_hw_mempool_destroy(mempool);
1832                 status = VXGE_HW_ERR_OUT_OF_MEMORY;
1833                 mempool = NULL;
1834                 goto exit;
1835         }
1836         memset(mempool->items_arr, 0, sizeof(void *) * mempool->items_max);
1837
1838         /* calculate initial number of memblocks */
1839         memblocks_to_allocate = (mempool->items_initial +
1840                                  mempool->items_per_memblock - 1) /
1841                                                 mempool->items_per_memblock;
1842
1843         /* pre-allocate the mempool */
1844         status = __vxge_hw_mempool_grow(mempool, memblocks_to_allocate,
1845                                         &allocated);
1846         if (status != VXGE_HW_OK) {
1847                 __vxge_hw_mempool_destroy(mempool);
1848                 status = VXGE_HW_ERR_OUT_OF_MEMORY;
1849                 mempool = NULL;
1850                 goto exit;
1851         }
1852
1853 exit:
1854         return mempool;
1855 }
1856
1857 /*
1858  * vxge_hw_mempool_destroy
1859  */
1860 void __vxge_hw_mempool_destroy(struct vxge_hw_mempool *mempool)
1861 {
1862         u32 i, j;
1863         struct __vxge_hw_device *devh = mempool->devh;
1864
1865         for (i = 0; i < mempool->memblocks_allocated; i++) {
1866                 struct vxge_hw_mempool_dma *dma_object;
1867
1868                 vxge_assert(mempool->memblocks_arr[i]);
1869                 vxge_assert(mempool->memblocks_dma_arr + i);
1870
1871                 dma_object = mempool->memblocks_dma_arr + i;
1872
1873                 for (j = 0; j < mempool->items_per_memblock; j++) {
1874                         u32 index = i * mempool->items_per_memblock + j;
1875
1876                         /* to skip last partially filled(if any) memblock */
1877                         if (index >= mempool->items_current)
1878                                 break;
1879                 }
1880
1881                 vfree(mempool->memblocks_priv_arr[i]);
1882
1883                 __vxge_hw_blockpool_free(devh, mempool->memblocks_arr[i],
1884                                 mempool->memblock_size, dma_object);
1885         }
1886
1887         vfree(mempool->items_arr);
1888
1889         vfree(mempool->memblocks_dma_arr);
1890
1891         vfree(mempool->memblocks_priv_arr);
1892
1893         vfree(mempool->memblocks_arr);
1894
1895         vfree(mempool);
1896 }
1897
1898 /*
1899  * __vxge_hw_device_fifo_config_check - Check fifo configuration.
1900  * Check the fifo configuration
1901  */
1902 enum vxge_hw_status
1903 __vxge_hw_device_fifo_config_check(struct vxge_hw_fifo_config *fifo_config)
1904 {
1905         if ((fifo_config->fifo_blocks < VXGE_HW_MIN_FIFO_BLOCKS) ||
1906              (fifo_config->fifo_blocks > VXGE_HW_MAX_FIFO_BLOCKS))
1907                 return VXGE_HW_BADCFG_FIFO_BLOCKS;
1908
1909         return VXGE_HW_OK;
1910 }
1911
1912 /*
1913  * __vxge_hw_device_vpath_config_check - Check vpath configuration.
1914  * Check the vpath configuration
1915  */
1916 enum vxge_hw_status
1917 __vxge_hw_device_vpath_config_check(struct vxge_hw_vp_config *vp_config)
1918 {
1919         enum vxge_hw_status status;
1920
1921         if ((vp_config->min_bandwidth < VXGE_HW_VPATH_BANDWIDTH_MIN) ||
1922                 (vp_config->min_bandwidth >
1923                                         VXGE_HW_VPATH_BANDWIDTH_MAX))
1924                 return VXGE_HW_BADCFG_VPATH_MIN_BANDWIDTH;
1925
1926         status = __vxge_hw_device_fifo_config_check(&vp_config->fifo);
1927         if (status != VXGE_HW_OK)
1928                 return status;
1929
1930         if ((vp_config->mtu != VXGE_HW_VPATH_USE_FLASH_DEFAULT_INITIAL_MTU) &&
1931                 ((vp_config->mtu < VXGE_HW_VPATH_MIN_INITIAL_MTU) ||
1932                 (vp_config->mtu > VXGE_HW_VPATH_MAX_INITIAL_MTU)))
1933                 return VXGE_HW_BADCFG_VPATH_MTU;
1934
1935         if ((vp_config->rpa_strip_vlan_tag !=
1936                 VXGE_HW_VPATH_RPA_STRIP_VLAN_TAG_USE_FLASH_DEFAULT) &&
1937                 (vp_config->rpa_strip_vlan_tag !=
1938                 VXGE_HW_VPATH_RPA_STRIP_VLAN_TAG_ENABLE) &&
1939                 (vp_config->rpa_strip_vlan_tag !=
1940                 VXGE_HW_VPATH_RPA_STRIP_VLAN_TAG_DISABLE))
1941                 return VXGE_HW_BADCFG_VPATH_RPA_STRIP_VLAN_TAG;
1942
1943         return VXGE_HW_OK;
1944 }
1945
1946 /*
1947  * __vxge_hw_device_config_check - Check device configuration.
1948  * Check the device configuration
1949  */
1950 enum vxge_hw_status
1951 __vxge_hw_device_config_check(struct vxge_hw_device_config *new_config)
1952 {
1953         u32 i;
1954         enum vxge_hw_status status;
1955
1956         if ((new_config->intr_mode != VXGE_HW_INTR_MODE_IRQLINE) &&
1957            (new_config->intr_mode != VXGE_HW_INTR_MODE_MSIX) &&
1958            (new_config->intr_mode != VXGE_HW_INTR_MODE_MSIX_ONE_SHOT) &&
1959            (new_config->intr_mode != VXGE_HW_INTR_MODE_DEF))
1960                 return VXGE_HW_BADCFG_INTR_MODE;
1961
1962         if ((new_config->rts_mac_en != VXGE_HW_RTS_MAC_DISABLE) &&
1963            (new_config->rts_mac_en != VXGE_HW_RTS_MAC_ENABLE))
1964                 return VXGE_HW_BADCFG_RTS_MAC_EN;
1965
1966         for (i = 0; i < VXGE_HW_MAX_VIRTUAL_PATHS; i++) {
1967                 status = __vxge_hw_device_vpath_config_check(
1968                                 &new_config->vp_config[i]);
1969                 if (status != VXGE_HW_OK)
1970                         return status;
1971         }
1972
1973         return VXGE_HW_OK;
1974 }
1975
1976 /*
1977  * vxge_hw_device_config_default_get - Initialize device config with defaults.
1978  * Initialize Titan device config with default values.
1979  */
1980 enum vxge_hw_status __devinit
1981 vxge_hw_device_config_default_get(struct vxge_hw_device_config *device_config)
1982 {
1983         u32 i;
1984
1985         device_config->dma_blockpool_initial =
1986                                         VXGE_HW_INITIAL_DMA_BLOCK_POOL_SIZE;
1987         device_config->dma_blockpool_max = VXGE_HW_MAX_DMA_BLOCK_POOL_SIZE;
1988         device_config->intr_mode = VXGE_HW_INTR_MODE_DEF;
1989         device_config->rth_en = VXGE_HW_RTH_DEFAULT;
1990         device_config->rth_it_type = VXGE_HW_RTH_IT_TYPE_DEFAULT;
1991         device_config->device_poll_millis =  VXGE_HW_DEF_DEVICE_POLL_MILLIS;
1992         device_config->rts_mac_en =  VXGE_HW_RTS_MAC_DEFAULT;
1993
1994         for (i = 0; i < VXGE_HW_MAX_VIRTUAL_PATHS; i++) {
1995
1996                 device_config->vp_config[i].vp_id = i;
1997
1998                 device_config->vp_config[i].min_bandwidth =
1999                                 VXGE_HW_VPATH_BANDWIDTH_DEFAULT;
2000
2001                 device_config->vp_config[i].ring.enable = VXGE_HW_RING_DEFAULT;
2002
2003                 device_config->vp_config[i].ring.ring_blocks =
2004                                 VXGE_HW_DEF_RING_BLOCKS;
2005
2006                 device_config->vp_config[i].ring.buffer_mode =
2007                                 VXGE_HW_RING_RXD_BUFFER_MODE_DEFAULT;
2008
2009                 device_config->vp_config[i].ring.scatter_mode =
2010                                 VXGE_HW_RING_SCATTER_MODE_USE_FLASH_DEFAULT;
2011
2012                 device_config->vp_config[i].ring.rxds_limit =
2013                                 VXGE_HW_DEF_RING_RXDS_LIMIT;
2014
2015                 device_config->vp_config[i].fifo.enable = VXGE_HW_FIFO_ENABLE;
2016
2017                 device_config->vp_config[i].fifo.fifo_blocks =
2018                                 VXGE_HW_MIN_FIFO_BLOCKS;
2019
2020                 device_config->vp_config[i].fifo.max_frags =
2021                                 VXGE_HW_MAX_FIFO_FRAGS;
2022
2023                 device_config->vp_config[i].fifo.memblock_size =
2024                                 VXGE_HW_DEF_FIFO_MEMBLOCK_SIZE;
2025
2026                 device_config->vp_config[i].fifo.alignment_size =
2027                                 VXGE_HW_DEF_FIFO_ALIGNMENT_SIZE;
2028
2029                 device_config->vp_config[i].fifo.intr =
2030                                 VXGE_HW_FIFO_QUEUE_INTR_DEFAULT;
2031
2032                 device_config->vp_config[i].fifo.no_snoop_bits =
2033                                 VXGE_HW_FIFO_NO_SNOOP_DEFAULT;
2034                 device_config->vp_config[i].tti.intr_enable =
2035                                 VXGE_HW_TIM_INTR_DEFAULT;
2036
2037                 device_config->vp_config[i].tti.btimer_val =
2038                                 VXGE_HW_USE_FLASH_DEFAULT;
2039
2040                 device_config->vp_config[i].tti.timer_ac_en =
2041                                 VXGE_HW_USE_FLASH_DEFAULT;
2042
2043                 device_config->vp_config[i].tti.timer_ci_en =
2044                                 VXGE_HW_USE_FLASH_DEFAULT;
2045
2046                 device_config->vp_config[i].tti.timer_ri_en =
2047                                 VXGE_HW_USE_FLASH_DEFAULT;
2048
2049                 device_config->vp_config[i].tti.rtimer_val =
2050                                 VXGE_HW_USE_FLASH_DEFAULT;
2051
2052                 device_config->vp_config[i].tti.util_sel =
2053                                 VXGE_HW_USE_FLASH_DEFAULT;
2054
2055                 device_config->vp_config[i].tti.ltimer_val =
2056                                 VXGE_HW_USE_FLASH_DEFAULT;
2057
2058                 device_config->vp_config[i].tti.urange_a =
2059                                 VXGE_HW_USE_FLASH_DEFAULT;
2060
2061                 device_config->vp_config[i].tti.uec_a =
2062                                 VXGE_HW_USE_FLASH_DEFAULT;
2063
2064                 device_config->vp_config[i].tti.urange_b =
2065                                 VXGE_HW_USE_FLASH_DEFAULT;
2066
2067                 device_config->vp_config[i].tti.uec_b =
2068                                 VXGE_HW_USE_FLASH_DEFAULT;
2069
2070                 device_config->vp_config[i].tti.urange_c =
2071                                 VXGE_HW_USE_FLASH_DEFAULT;
2072
2073                 device_config->vp_config[i].tti.uec_c =
2074                                 VXGE_HW_USE_FLASH_DEFAULT;
2075
2076                 device_config->vp_config[i].tti.uec_d =
2077                                 VXGE_HW_USE_FLASH_DEFAULT;
2078
2079                 device_config->vp_config[i].rti.intr_enable =
2080                                 VXGE_HW_TIM_INTR_DEFAULT;
2081
2082                 device_config->vp_config[i].rti.btimer_val =
2083                                 VXGE_HW_USE_FLASH_DEFAULT;
2084
2085                 device_config->vp_config[i].rti.timer_ac_en =
2086                                 VXGE_HW_USE_FLASH_DEFAULT;
2087
2088                 device_config->vp_config[i].rti.timer_ci_en =
2089                                 VXGE_HW_USE_FLASH_DEFAULT;
2090
2091                 device_config->vp_config[i].rti.timer_ri_en =
2092                                 VXGE_HW_USE_FLASH_DEFAULT;
2093
2094                 device_config->vp_config[i].rti.rtimer_val =
2095                                 VXGE_HW_USE_FLASH_DEFAULT;
2096
2097                 device_config->vp_config[i].rti.util_sel =
2098                                 VXGE_HW_USE_FLASH_DEFAULT;
2099
2100                 device_config->vp_config[i].rti.ltimer_val =
2101                                 VXGE_HW_USE_FLASH_DEFAULT;
2102
2103                 device_config->vp_config[i].rti.urange_a =
2104                                 VXGE_HW_USE_FLASH_DEFAULT;
2105
2106                 device_config->vp_config[i].rti.uec_a =
2107                                 VXGE_HW_USE_FLASH_DEFAULT;
2108
2109                 device_config->vp_config[i].rti.urange_b =
2110                                 VXGE_HW_USE_FLASH_DEFAULT;
2111
2112                 device_config->vp_config[i].rti.uec_b =
2113                                 VXGE_HW_USE_FLASH_DEFAULT;
2114
2115                 device_config->vp_config[i].rti.urange_c =
2116                                 VXGE_HW_USE_FLASH_DEFAULT;
2117
2118                 device_config->vp_config[i].rti.uec_c =
2119                                 VXGE_HW_USE_FLASH_DEFAULT;
2120
2121                 device_config->vp_config[i].rti.uec_d =
2122                                 VXGE_HW_USE_FLASH_DEFAULT;
2123
2124                 device_config->vp_config[i].mtu =
2125                                 VXGE_HW_VPATH_USE_FLASH_DEFAULT_INITIAL_MTU;
2126
2127                 device_config->vp_config[i].rpa_strip_vlan_tag =
2128                         VXGE_HW_VPATH_RPA_STRIP_VLAN_TAG_USE_FLASH_DEFAULT;
2129         }
2130
2131         return VXGE_HW_OK;
2132 }
2133
2134 /*
2135  * _hw_legacy_swapper_set - Set the swapper bits for the legacy secion.
2136  * Set the swapper bits appropriately for the lagacy section.
2137  */
2138 enum vxge_hw_status
2139 __vxge_hw_legacy_swapper_set(struct vxge_hw_legacy_reg __iomem *legacy_reg)
2140 {
2141         u64 val64;
2142         enum vxge_hw_status status = VXGE_HW_OK;
2143
2144         val64 = readq(&legacy_reg->toc_swapper_fb);
2145
2146         wmb();
2147
2148         switch (val64) {
2149
2150         case VXGE_HW_SWAPPER_INITIAL_VALUE:
2151                 return status;
2152
2153         case VXGE_HW_SWAPPER_BYTE_SWAPPED_BIT_FLIPPED:
2154                 writeq(VXGE_HW_SWAPPER_READ_BYTE_SWAP_ENABLE,
2155                         &legacy_reg->pifm_rd_swap_en);
2156                 writeq(VXGE_HW_SWAPPER_READ_BIT_FLAP_ENABLE,
2157                         &legacy_reg->pifm_rd_flip_en);
2158                 writeq(VXGE_HW_SWAPPER_WRITE_BYTE_SWAP_ENABLE,
2159                         &legacy_reg->pifm_wr_swap_en);
2160                 writeq(VXGE_HW_SWAPPER_WRITE_BIT_FLAP_ENABLE,
2161                         &legacy_reg->pifm_wr_flip_en);
2162                 break;
2163
2164         case VXGE_HW_SWAPPER_BYTE_SWAPPED:
2165                 writeq(VXGE_HW_SWAPPER_READ_BYTE_SWAP_ENABLE,
2166                         &legacy_reg->pifm_rd_swap_en);
2167                 writeq(VXGE_HW_SWAPPER_WRITE_BYTE_SWAP_ENABLE,
2168                         &legacy_reg->pifm_wr_swap_en);
2169                 break;
2170
2171         case VXGE_HW_SWAPPER_BIT_FLIPPED:
2172                 writeq(VXGE_HW_SWAPPER_READ_BIT_FLAP_ENABLE,
2173                         &legacy_reg->pifm_rd_flip_en);
2174                 writeq(VXGE_HW_SWAPPER_WRITE_BIT_FLAP_ENABLE,
2175                         &legacy_reg->pifm_wr_flip_en);
2176                 break;
2177         }
2178
2179         wmb();
2180
2181         val64 = readq(&legacy_reg->toc_swapper_fb);
2182
2183         if (val64 != VXGE_HW_SWAPPER_INITIAL_VALUE)
2184                 status = VXGE_HW_ERR_SWAPPER_CTRL;
2185
2186         return status;
2187 }
2188
2189 /*
2190  * __vxge_hw_vpath_swapper_set - Set the swapper bits for the vpath.
2191  * Set the swapper bits appropriately for the vpath.
2192  */
2193 enum vxge_hw_status
2194 __vxge_hw_vpath_swapper_set(struct vxge_hw_vpath_reg __iomem *vpath_reg)
2195 {
2196 #ifndef __BIG_ENDIAN
2197         u64 val64;
2198
2199         val64 = readq(&vpath_reg->vpath_general_cfg1);
2200         wmb();
2201         val64 |= VXGE_HW_VPATH_GENERAL_CFG1_CTL_BYTE_SWAPEN;
2202         writeq(val64, &vpath_reg->vpath_general_cfg1);
2203         wmb();
2204 #endif
2205         return VXGE_HW_OK;
2206 }
2207
2208 /*
2209  * __vxge_hw_kdfc_swapper_set - Set the swapper bits for the kdfc.
2210  * Set the swapper bits appropriately for the vpath.
2211  */
2212 enum vxge_hw_status
2213 __vxge_hw_kdfc_swapper_set(
2214         struct vxge_hw_legacy_reg __iomem *legacy_reg,
2215         struct vxge_hw_vpath_reg __iomem *vpath_reg)
2216 {
2217         u64 val64;
2218
2219         val64 = readq(&legacy_reg->pifm_wr_swap_en);
2220
2221         if (val64 == VXGE_HW_SWAPPER_WRITE_BYTE_SWAP_ENABLE) {
2222                 val64 = readq(&vpath_reg->kdfcctl_cfg0);
2223                 wmb();
2224
2225                 val64 |= VXGE_HW_KDFCCTL_CFG0_BYTE_SWAPEN_FIFO0 |
2226                         VXGE_HW_KDFCCTL_CFG0_BYTE_SWAPEN_FIFO1  |
2227                         VXGE_HW_KDFCCTL_CFG0_BYTE_SWAPEN_FIFO2;
2228
2229                 writeq(val64, &vpath_reg->kdfcctl_cfg0);
2230                 wmb();
2231         }
2232
2233         return VXGE_HW_OK;
2234 }
2235
2236 /*
2237  * vxge_hw_mgmt_device_config - Retrieve device configuration.
2238  * Get device configuration. Permits to retrieve at run-time configuration
2239  * values that were used to initialize and configure the device.
2240  */
2241 enum vxge_hw_status
2242 vxge_hw_mgmt_device_config(struct __vxge_hw_device *hldev,
2243                            struct vxge_hw_device_config *dev_config, int size)
2244 {
2245
2246         if ((hldev == NULL) || (hldev->magic != VXGE_HW_DEVICE_MAGIC))
2247                 return VXGE_HW_ERR_INVALID_DEVICE;
2248
2249         if (size != sizeof(struct vxge_hw_device_config))
2250                 return VXGE_HW_ERR_VERSION_CONFLICT;
2251
2252         memcpy(dev_config, &hldev->config,
2253                 sizeof(struct vxge_hw_device_config));
2254
2255         return VXGE_HW_OK;
2256 }
2257
2258 /*
2259  * vxge_hw_mgmt_reg_read - Read Titan register.
2260  */
2261 enum vxge_hw_status
2262 vxge_hw_mgmt_reg_read(struct __vxge_hw_device *hldev,
2263                       enum vxge_hw_mgmt_reg_type type,
2264                       u32 index, u32 offset, u64 *value)
2265 {
2266         enum vxge_hw_status status = VXGE_HW_OK;
2267
2268         if ((hldev == NULL) || (hldev->magic != VXGE_HW_DEVICE_MAGIC)) {
2269                 status = VXGE_HW_ERR_INVALID_DEVICE;
2270                 goto exit;
2271         }
2272
2273         switch (type) {
2274         case vxge_hw_mgmt_reg_type_legacy:
2275                 if (offset > sizeof(struct vxge_hw_legacy_reg) - 8) {
2276                         status = VXGE_HW_ERR_INVALID_OFFSET;
2277                         break;
2278                 }
2279                 *value = readq((void __iomem *)hldev->legacy_reg + offset);
2280                 break;
2281         case vxge_hw_mgmt_reg_type_toc:
2282                 if (offset > sizeof(struct vxge_hw_toc_reg) - 8) {
2283                         status = VXGE_HW_ERR_INVALID_OFFSET;
2284                         break;
2285                 }
2286                 *value = readq((void __iomem *)hldev->toc_reg + offset);
2287                 break;
2288         case vxge_hw_mgmt_reg_type_common:
2289                 if (offset > sizeof(struct vxge_hw_common_reg) - 8) {
2290                         status = VXGE_HW_ERR_INVALID_OFFSET;
2291                         break;
2292                 }
2293                 *value = readq((void __iomem *)hldev->common_reg + offset);
2294                 break;
2295         case vxge_hw_mgmt_reg_type_mrpcim:
2296                 if (!(hldev->access_rights &
2297                         VXGE_HW_DEVICE_ACCESS_RIGHT_MRPCIM)) {
2298                         status = VXGE_HW_ERR_PRIVILAGED_OPEARATION;
2299                         break;
2300                 }
2301                 if (offset > sizeof(struct vxge_hw_mrpcim_reg) - 8) {
2302                         status = VXGE_HW_ERR_INVALID_OFFSET;
2303                         break;
2304                 }
2305                 *value = readq((void __iomem *)hldev->mrpcim_reg + offset);
2306                 break;
2307         case vxge_hw_mgmt_reg_type_srpcim:
2308                 if (!(hldev->access_rights &
2309                         VXGE_HW_DEVICE_ACCESS_RIGHT_SRPCIM)) {
2310                         status = VXGE_HW_ERR_PRIVILAGED_OPEARATION;
2311                         break;
2312                 }
2313                 if (index > VXGE_HW_TITAN_SRPCIM_REG_SPACES - 1) {
2314                         status = VXGE_HW_ERR_INVALID_INDEX;
2315                         break;
2316                 }
2317                 if (offset > sizeof(struct vxge_hw_srpcim_reg) - 8) {
2318                         status = VXGE_HW_ERR_INVALID_OFFSET;
2319                         break;
2320                 }
2321                 *value = readq((void __iomem *)hldev->srpcim_reg[index] +
2322                                 offset);
2323                 break;
2324         case vxge_hw_mgmt_reg_type_vpmgmt:
2325                 if ((index > VXGE_HW_TITAN_VPMGMT_REG_SPACES - 1) ||
2326                         (!(hldev->vpath_assignments & vxge_mBIT(index)))) {
2327                         status = VXGE_HW_ERR_INVALID_INDEX;
2328                         break;
2329                 }
2330                 if (offset > sizeof(struct vxge_hw_vpmgmt_reg) - 8) {
2331                         status = VXGE_HW_ERR_INVALID_OFFSET;
2332                         break;
2333                 }
2334                 *value = readq((void __iomem *)hldev->vpmgmt_reg[index] +
2335                                 offset);
2336                 break;
2337         case vxge_hw_mgmt_reg_type_vpath:
2338                 if ((index > VXGE_HW_TITAN_VPATH_REG_SPACES - 1) ||
2339                         (!(hldev->vpath_assignments & vxge_mBIT(index)))) {
2340                         status = VXGE_HW_ERR_INVALID_INDEX;
2341                         break;
2342                 }
2343                 if (index > VXGE_HW_TITAN_VPATH_REG_SPACES - 1) {
2344                         status = VXGE_HW_ERR_INVALID_INDEX;
2345                         break;
2346                 }
2347                 if (offset > sizeof(struct vxge_hw_vpath_reg) - 8) {
2348                         status = VXGE_HW_ERR_INVALID_OFFSET;
2349                         break;
2350                 }
2351                 *value = readq((void __iomem *)hldev->vpath_reg[index] +
2352                                 offset);
2353                 break;
2354         default:
2355                 status = VXGE_HW_ERR_INVALID_TYPE;
2356                 break;
2357         }
2358
2359 exit:
2360         return status;
2361 }
2362
2363 /*
2364  * vxge_hw_mgmt_reg_Write - Write Titan register.
2365  */
2366 enum vxge_hw_status
2367 vxge_hw_mgmt_reg_write(struct __vxge_hw_device *hldev,
2368                       enum vxge_hw_mgmt_reg_type type,
2369                       u32 index, u32 offset, u64 value)
2370 {
2371         enum vxge_hw_status status = VXGE_HW_OK;
2372
2373         if ((hldev == NULL) || (hldev->magic != VXGE_HW_DEVICE_MAGIC)) {
2374                 status = VXGE_HW_ERR_INVALID_DEVICE;
2375                 goto exit;
2376         }
2377
2378         switch (type) {
2379         case vxge_hw_mgmt_reg_type_legacy:
2380                 if (offset > sizeof(struct vxge_hw_legacy_reg) - 8) {
2381                         status = VXGE_HW_ERR_INVALID_OFFSET;
2382                         break;
2383                 }
2384                 writeq(value, (void __iomem *)hldev->legacy_reg + offset);
2385                 break;
2386         case vxge_hw_mgmt_reg_type_toc:
2387                 if (offset > sizeof(struct vxge_hw_toc_reg) - 8) {
2388                         status = VXGE_HW_ERR_INVALID_OFFSET;
2389                         break;
2390                 }
2391                 writeq(value, (void __iomem *)hldev->toc_reg + offset);
2392                 break;
2393         case vxge_hw_mgmt_reg_type_common:
2394                 if (offset > sizeof(struct vxge_hw_common_reg) - 8) {
2395                         status = VXGE_HW_ERR_INVALID_OFFSET;
2396                         break;
2397                 }
2398                 writeq(value, (void __iomem *)hldev->common_reg + offset);
2399                 break;
2400         case vxge_hw_mgmt_reg_type_mrpcim:
2401                 if (!(hldev->access_rights &
2402                         VXGE_HW_DEVICE_ACCESS_RIGHT_MRPCIM)) {
2403                         status = VXGE_HW_ERR_PRIVILAGED_OPEARATION;
2404                         break;
2405                 }
2406                 if (offset > sizeof(struct vxge_hw_mrpcim_reg) - 8) {
2407                         status = VXGE_HW_ERR_INVALID_OFFSET;
2408                         break;
2409                 }
2410                 writeq(value, (void __iomem *)hldev->mrpcim_reg + offset);
2411                 break;
2412         case vxge_hw_mgmt_reg_type_srpcim:
2413                 if (!(hldev->access_rights &
2414                         VXGE_HW_DEVICE_ACCESS_RIGHT_SRPCIM)) {
2415                         status = VXGE_HW_ERR_PRIVILAGED_OPEARATION;
2416                         break;
2417                 }
2418                 if (index > VXGE_HW_TITAN_SRPCIM_REG_SPACES - 1) {
2419                         status = VXGE_HW_ERR_INVALID_INDEX;
2420                         break;
2421                 }
2422                 if (offset > sizeof(struct vxge_hw_srpcim_reg) - 8) {
2423                         status = VXGE_HW_ERR_INVALID_OFFSET;
2424                         break;
2425                 }
2426                 writeq(value, (void __iomem *)hldev->srpcim_reg[index] +
2427                         offset);
2428
2429                 break;
2430         case vxge_hw_mgmt_reg_type_vpmgmt:
2431                 if ((index > VXGE_HW_TITAN_VPMGMT_REG_SPACES - 1) ||
2432                         (!(hldev->vpath_assignments & vxge_mBIT(index)))) {
2433                         status = VXGE_HW_ERR_INVALID_INDEX;
2434                         break;
2435                 }
2436                 if (offset > sizeof(struct vxge_hw_vpmgmt_reg) - 8) {
2437                         status = VXGE_HW_ERR_INVALID_OFFSET;
2438                         break;
2439                 }
2440                 writeq(value, (void __iomem *)hldev->vpmgmt_reg[index] +
2441                         offset);
2442                 break;
2443         case vxge_hw_mgmt_reg_type_vpath:
2444                 if ((index > VXGE_HW_TITAN_VPATH_REG_SPACES-1) ||
2445                         (!(hldev->vpath_assignments & vxge_mBIT(index)))) {
2446                         status = VXGE_HW_ERR_INVALID_INDEX;
2447                         break;
2448                 }
2449                 if (offset > sizeof(struct vxge_hw_vpath_reg) - 8) {
2450                         status = VXGE_HW_ERR_INVALID_OFFSET;
2451                         break;
2452                 }
2453                 writeq(value, (void __iomem *)hldev->vpath_reg[index] +
2454                         offset);
2455                 break;
2456         default:
2457                 status = VXGE_HW_ERR_INVALID_TYPE;
2458                 break;
2459         }
2460 exit:
2461         return status;
2462 }
2463
2464 /*
2465  * __vxge_hw_fifo_mempool_item_alloc - Allocate List blocks for TxD
2466  * list callback
2467  * This function is callback passed to __vxge_hw_mempool_create to create memory
2468  * pool for TxD list
2469  */
2470 static void
2471 __vxge_hw_fifo_mempool_item_alloc(
2472         struct vxge_hw_mempool *mempoolh,
2473         u32 memblock_index, struct vxge_hw_mempool_dma *dma_object,
2474         u32 index, u32 is_last)
2475 {
2476         u32 memblock_item_idx;
2477         struct __vxge_hw_fifo_txdl_priv *txdl_priv;
2478         struct vxge_hw_fifo_txd *txdp =
2479                 (struct vxge_hw_fifo_txd *)mempoolh->items_arr[index];
2480         struct __vxge_hw_fifo *fifo =
2481                         (struct __vxge_hw_fifo *)mempoolh->userdata;
2482         void *memblock = mempoolh->memblocks_arr[memblock_index];
2483
2484         vxge_assert(txdp);
2485
2486         txdp->host_control = (u64) (size_t)
2487         __vxge_hw_mempool_item_priv(mempoolh, memblock_index, txdp,
2488                                         &memblock_item_idx);
2489
2490         txdl_priv = __vxge_hw_fifo_txdl_priv(fifo, txdp);
2491
2492         vxge_assert(txdl_priv);
2493
2494         fifo->channel.reserve_arr[fifo->channel.reserve_ptr - 1 - index] = txdp;
2495
2496         /* pre-format HW's TxDL's private */
2497         txdl_priv->dma_offset = (char *)txdp - (char *)memblock;
2498         txdl_priv->dma_addr = dma_object->addr + txdl_priv->dma_offset;
2499         txdl_priv->dma_handle = dma_object->handle;
2500         txdl_priv->memblock   = memblock;
2501         txdl_priv->first_txdp = txdp;
2502         txdl_priv->next_txdl_priv = NULL;
2503         txdl_priv->alloc_frags = 0;
2504
2505         return;
2506 }
2507
2508 /*
2509  * __vxge_hw_fifo_create - Create a FIFO
2510  * This function creates FIFO and initializes it.
2511  */
2512 enum vxge_hw_status
2513 __vxge_hw_fifo_create(struct __vxge_hw_vpath_handle *vp,
2514                       struct vxge_hw_fifo_attr *attr)
2515 {
2516         enum vxge_hw_status status = VXGE_HW_OK;
2517         struct __vxge_hw_fifo *fifo;
2518         struct vxge_hw_fifo_config *config;
2519         u32 txdl_size, txdl_per_memblock;
2520         struct vxge_hw_mempool_cbs fifo_mp_callback;
2521         struct __vxge_hw_virtualpath *vpath;
2522
2523         if ((vp == NULL) || (attr == NULL)) {
2524                 status = VXGE_HW_ERR_INVALID_HANDLE;
2525                 goto exit;
2526         }
2527         vpath = vp->vpath;
2528         config = &vpath->hldev->config.vp_config[vpath->vp_id].fifo;
2529
2530         txdl_size = config->max_frags * sizeof(struct vxge_hw_fifo_txd);
2531
2532         txdl_per_memblock = config->memblock_size / txdl_size;
2533
2534         fifo = (struct __vxge_hw_fifo *)__vxge_hw_channel_allocate(vp,
2535                                         VXGE_HW_CHANNEL_TYPE_FIFO,
2536                                         config->fifo_blocks * txdl_per_memblock,
2537                                         attr->per_txdl_space, attr->userdata);
2538
2539         if (fifo == NULL) {
2540                 status = VXGE_HW_ERR_OUT_OF_MEMORY;
2541                 goto exit;
2542         }
2543
2544         vpath->fifoh = fifo;
2545         fifo->nofl_db = vpath->nofl_db;
2546
2547         fifo->vp_id = vpath->vp_id;
2548         fifo->vp_reg = vpath->vp_reg;
2549         fifo->stats = &vpath->sw_stats->fifo_stats;
2550
2551         fifo->config = config;
2552
2553         /* apply "interrupts per txdl" attribute */
2554         fifo->interrupt_type = VXGE_HW_FIFO_TXD_INT_TYPE_UTILZ;
2555
2556         if (fifo->config->intr)
2557                 fifo->interrupt_type = VXGE_HW_FIFO_TXD_INT_TYPE_PER_LIST;
2558
2559         fifo->no_snoop_bits = config->no_snoop_bits;
2560
2561         /*
2562          * FIFO memory management strategy:
2563          *
2564          * TxDL split into three independent parts:
2565          *      - set of TxD's
2566          *      - TxD HW private part
2567          *      - driver private part
2568          *
2569          * Adaptative memory allocation used. i.e. Memory allocated on
2570          * demand with the size which will fit into one memory block.
2571          * One memory block may contain more than one TxDL.
2572          *
2573          * During "reserve" operations more memory can be allocated on demand
2574          * for example due to FIFO full condition.
2575          *
2576          * Pool of memory memblocks never shrinks except in __vxge_hw_fifo_close
2577          * routine which will essentially stop the channel and free resources.
2578          */
2579
2580         /* TxDL common private size == TxDL private  +  driver private */
2581         fifo->priv_size =
2582                 sizeof(struct __vxge_hw_fifo_txdl_priv) + attr->per_txdl_space;
2583         fifo->priv_size = ((fifo->priv_size  +  VXGE_CACHE_LINE_SIZE - 1) /
2584                         VXGE_CACHE_LINE_SIZE) * VXGE_CACHE_LINE_SIZE;
2585
2586         fifo->per_txdl_space = attr->per_txdl_space;
2587
2588         /* recompute txdl size to be cacheline aligned */
2589         fifo->txdl_size = txdl_size;
2590         fifo->txdl_per_memblock = txdl_per_memblock;
2591
2592         fifo->txdl_term = attr->txdl_term;
2593         fifo->callback = attr->callback;
2594
2595         if (fifo->txdl_per_memblock == 0) {
2596                 __vxge_hw_fifo_delete(vp);
2597                 status = VXGE_HW_ERR_INVALID_BLOCK_SIZE;
2598                 goto exit;
2599         }
2600
2601         fifo_mp_callback.item_func_alloc = __vxge_hw_fifo_mempool_item_alloc;
2602
2603         fifo->mempool =
2604                 __vxge_hw_mempool_create(vpath->hldev,
2605                         fifo->config->memblock_size,
2606                         fifo->txdl_size,
2607                         fifo->priv_size,
2608                         (fifo->config->fifo_blocks * fifo->txdl_per_memblock),
2609                         (fifo->config->fifo_blocks * fifo->txdl_per_memblock),
2610                         &fifo_mp_callback,
2611                         fifo);
2612
2613         if (fifo->mempool == NULL) {
2614                 __vxge_hw_fifo_delete(vp);
2615                 status = VXGE_HW_ERR_OUT_OF_MEMORY;
2616                 goto exit;
2617         }
2618
2619         status = __vxge_hw_channel_initialize(&fifo->channel);
2620         if (status != VXGE_HW_OK) {
2621                 __vxge_hw_fifo_delete(vp);
2622                 goto exit;
2623         }
2624
2625         vxge_assert(fifo->channel.reserve_ptr);
2626 exit:
2627         return status;
2628 }
2629
2630 /*
2631  * __vxge_hw_fifo_abort - Returns the TxD
2632  * This function terminates the TxDs of fifo
2633  */
2634 enum vxge_hw_status __vxge_hw_fifo_abort(struct __vxge_hw_fifo *fifo)
2635 {
2636         void *txdlh;
2637
2638         for (;;) {
2639                 vxge_hw_channel_dtr_try_complete(&fifo->channel, &txdlh);
2640
2641                 if (txdlh == NULL)
2642                         break;
2643
2644                 vxge_hw_channel_dtr_complete(&fifo->channel);
2645
2646                 if (fifo->txdl_term) {
2647                         fifo->txdl_term(txdlh,
2648                         VXGE_HW_TXDL_STATE_POSTED,
2649                         fifo->channel.userdata);
2650                 }
2651
2652                 vxge_hw_channel_dtr_free(&fifo->channel, txdlh);
2653         }
2654
2655         return VXGE_HW_OK;
2656 }
2657
2658 /*
2659  * __vxge_hw_fifo_reset - Resets the fifo
2660  * This function resets the fifo during vpath reset operation
2661  */
2662 enum vxge_hw_status __vxge_hw_fifo_reset(struct __vxge_hw_fifo *fifo)
2663 {
2664         enum vxge_hw_status status = VXGE_HW_OK;
2665
2666         __vxge_hw_fifo_abort(fifo);
2667         status = __vxge_hw_channel_reset(&fifo->channel);
2668
2669         return status;
2670 }
2671
2672 /*
2673  * __vxge_hw_fifo_delete - Removes the FIFO
2674  * This function freeup the memory pool and removes the FIFO
2675  */
2676 enum vxge_hw_status __vxge_hw_fifo_delete(struct __vxge_hw_vpath_handle *vp)
2677 {
2678         struct __vxge_hw_fifo *fifo = vp->vpath->fifoh;
2679
2680         __vxge_hw_fifo_abort(fifo);
2681
2682         if (fifo->mempool)
2683                 __vxge_hw_mempool_destroy(fifo->mempool);
2684
2685         vp->vpath->fifoh = NULL;
2686
2687         __vxge_hw_channel_free(&fifo->channel);
2688
2689         return VXGE_HW_OK;
2690 }
2691
2692 /*
2693  * __vxge_hw_vpath_pci_read - Read the content of given address
2694  *                          in pci config space.
2695  * Read from the vpath pci config space.
2696  */
2697 enum vxge_hw_status
2698 __vxge_hw_vpath_pci_read(struct __vxge_hw_virtualpath *vpath,
2699                          u32 phy_func_0, u32 offset, u32 *val)
2700 {
2701         u64 val64;
2702         enum vxge_hw_status status = VXGE_HW_OK;
2703         struct vxge_hw_vpath_reg __iomem *vp_reg = vpath->vp_reg;
2704
2705         val64 = VXGE_HW_PCI_CONFIG_ACCESS_CFG1_ADDRESS(offset);
2706
2707         if (phy_func_0)
2708                 val64 |= VXGE_HW_PCI_CONFIG_ACCESS_CFG1_SEL_FUNC0;
2709
2710         writeq(val64, &vp_reg->pci_config_access_cfg1);
2711         wmb();
2712         writeq(VXGE_HW_PCI_CONFIG_ACCESS_CFG2_REQ,
2713                         &vp_reg->pci_config_access_cfg2);
2714         wmb();
2715
2716         status = __vxge_hw_device_register_poll(
2717                         &vp_reg->pci_config_access_cfg2,
2718                         VXGE_HW_INTR_MASK_ALL, VXGE_HW_DEF_DEVICE_POLL_MILLIS);
2719
2720         if (status != VXGE_HW_OK)
2721                 goto exit;
2722
2723         val64 = readq(&vp_reg->pci_config_access_status);
2724
2725         if (val64 & VXGE_HW_PCI_CONFIG_ACCESS_STATUS_ACCESS_ERR) {
2726                 status = VXGE_HW_FAIL;
2727                 *val = 0;
2728         } else
2729                 *val = (u32)vxge_bVALn(val64, 32, 32);
2730 exit:
2731         return status;
2732 }
2733
2734 /*
2735  * __vxge_hw_vpath_func_id_get - Get the function id of the vpath.
2736  * Returns the function number of the vpath.
2737  */
2738 u32
2739 __vxge_hw_vpath_func_id_get(u32 vp_id,
2740         struct vxge_hw_vpmgmt_reg __iomem *vpmgmt_reg)
2741 {
2742         u64 val64;
2743
2744         val64 = readq(&vpmgmt_reg->vpath_to_func_map_cfg1);
2745
2746         return
2747          (u32)VXGE_HW_VPATH_TO_FUNC_MAP_CFG1_GET_VPATH_TO_FUNC_MAP_CFG1(val64);
2748 }
2749
2750 /*
2751  * __vxge_hw_read_rts_ds - Program RTS steering critieria
2752  */
2753 static inline void
2754 __vxge_hw_read_rts_ds(struct vxge_hw_vpath_reg __iomem *vpath_reg,
2755                       u64 dta_struct_sel)
2756 {
2757         writeq(0, &vpath_reg->rts_access_steer_ctrl);
2758         wmb();
2759         writeq(dta_struct_sel, &vpath_reg->rts_access_steer_data0);
2760         writeq(0, &vpath_reg->rts_access_steer_data1);
2761         wmb();
2762         return;
2763 }
2764
2765
2766 /*
2767  * __vxge_hw_vpath_card_info_get - Get the serial numbers,
2768  * part number and product description.
2769  */
2770 enum vxge_hw_status
2771 __vxge_hw_vpath_card_info_get(
2772         u32 vp_id,
2773         struct vxge_hw_vpath_reg __iomem *vpath_reg,
2774         struct vxge_hw_device_hw_info *hw_info)
2775 {
2776         u32 i, j;
2777         u64 val64;
2778         u64 data1 = 0ULL;
2779         u64 data2 = 0ULL;
2780         enum vxge_hw_status status = VXGE_HW_OK;
2781         u8 *serial_number = hw_info->serial_number;
2782         u8 *part_number = hw_info->part_number;
2783         u8 *product_desc = hw_info->product_desc;
2784
2785         __vxge_hw_read_rts_ds(vpath_reg,
2786                 VXGE_HW_RTS_ACCESS_STEER_DATA0_MEMO_ITEM_SERIAL_NUMBER);
2787
2788         val64 = VXGE_HW_RTS_ACCESS_STEER_CTRL_ACTION(
2789                         VXGE_HW_RTS_ACCESS_STEER_CTRL_ACTION_READ_MEMO_ENTRY) |
2790                 VXGE_HW_RTS_ACCESS_STEER_CTRL_DATA_STRUCT_SEL(
2791                         VXGE_HW_RTS_ACCESS_STEER_CTRL_DATA_STRUCT_SEL_FW_MEMO) |
2792                 VXGE_HW_RTS_ACCESS_STEER_CTRL_STROBE |
2793                 VXGE_HW_RTS_ACCESS_STEER_CTRL_OFFSET(0);
2794
2795         status = __vxge_hw_pio_mem_write64(val64,
2796                                 &vpath_reg->rts_access_steer_ctrl,
2797                                 VXGE_HW_RTS_ACCESS_STEER_CTRL_STROBE,
2798                                 VXGE_HW_DEF_DEVICE_POLL_MILLIS);
2799
2800         if (status != VXGE_HW_OK)
2801                 return status;
2802
2803         val64 = readq(&vpath_reg->rts_access_steer_ctrl);
2804
2805         if (val64 & VXGE_HW_RTS_ACCESS_STEER_CTRL_RMACJ_STATUS) {
2806                 data1 = readq(&vpath_reg->rts_access_steer_data0);
2807                 ((u64 *)serial_number)[0] = be64_to_cpu(data1);
2808
2809                 data2 = readq(&vpath_reg->rts_access_steer_data1);
2810                 ((u64 *)serial_number)[1] = be64_to_cpu(data2);
2811                 status = VXGE_HW_OK;
2812         } else
2813                 *serial_number = 0;
2814
2815         __vxge_hw_read_rts_ds(vpath_reg,
2816                         VXGE_HW_RTS_ACCESS_STEER_DATA0_MEMO_ITEM_PART_NUMBER);
2817
2818         val64 = VXGE_HW_RTS_ACCESS_STEER_CTRL_ACTION(
2819                         VXGE_HW_RTS_ACCESS_STEER_CTRL_ACTION_READ_MEMO_ENTRY) |
2820                 VXGE_HW_RTS_ACCESS_STEER_CTRL_DATA_STRUCT_SEL(
2821                         VXGE_HW_RTS_ACCESS_STEER_CTRL_DATA_STRUCT_SEL_FW_MEMO) |
2822                 VXGE_HW_RTS_ACCESS_STEER_CTRL_STROBE |
2823                 VXGE_HW_RTS_ACCESS_STEER_CTRL_OFFSET(0);
2824
2825         status = __vxge_hw_pio_mem_write64(val64,
2826                                 &vpath_reg->rts_access_steer_ctrl,
2827                                 VXGE_HW_RTS_ACCESS_STEER_CTRL_STROBE,
2828                                 VXGE_HW_DEF_DEVICE_POLL_MILLIS);
2829
2830         if (status != VXGE_HW_OK)
2831                 return status;
2832
2833         val64 = readq(&vpath_reg->rts_access_steer_ctrl);
2834
2835         if (val64 & VXGE_HW_RTS_ACCESS_STEER_CTRL_RMACJ_STATUS) {
2836
2837                 data1 = readq(&vpath_reg->rts_access_steer_data0);
2838                 ((u64 *)part_number)[0] = be64_to_cpu(data1);
2839
2840                 data2 = readq(&vpath_reg->rts_access_steer_data1);
2841                 ((u64 *)part_number)[1] = be64_to_cpu(data2);
2842
2843                 status = VXGE_HW_OK;
2844
2845         } else
2846                 *part_number = 0;
2847
2848         j = 0;
2849
2850         for (i = VXGE_HW_RTS_ACCESS_STEER_DATA0_MEMO_ITEM_DESC_0;
2851              i <= VXGE_HW_RTS_ACCESS_STEER_DATA0_MEMO_ITEM_DESC_3; i++) {
2852
2853                 __vxge_hw_read_rts_ds(vpath_reg, i);
2854
2855                 val64 = VXGE_HW_RTS_ACCESS_STEER_CTRL_ACTION(
2856                         VXGE_HW_RTS_ACCESS_STEER_CTRL_ACTION_READ_MEMO_ENTRY) |
2857                         VXGE_HW_RTS_ACCESS_STEER_CTRL_DATA_STRUCT_SEL(
2858                         VXGE_HW_RTS_ACCESS_STEER_CTRL_DATA_STRUCT_SEL_FW_MEMO) |
2859                         VXGE_HW_RTS_ACCESS_STEER_CTRL_STROBE |
2860                         VXGE_HW_RTS_ACCESS_STEER_CTRL_OFFSET(0);
2861
2862                 status = __vxge_hw_pio_mem_write64(val64,
2863                                 &vpath_reg->rts_access_steer_ctrl,
2864                                 VXGE_HW_RTS_ACCESS_STEER_CTRL_STROBE,
2865                                 VXGE_HW_DEF_DEVICE_POLL_MILLIS);
2866
2867                 if (status != VXGE_HW_OK)
2868                         return status;
2869
2870                 val64 = readq(&vpath_reg->rts_access_steer_ctrl);
2871
2872                 if (val64 & VXGE_HW_RTS_ACCESS_STEER_CTRL_RMACJ_STATUS) {
2873
2874                         data1 = readq(&vpath_reg->rts_access_steer_data0);
2875                         ((u64 *)product_desc)[j++] = be64_to_cpu(data1);
2876
2877                         data2 = readq(&vpath_reg->rts_access_steer_data1);
2878                         ((u64 *)product_desc)[j++] = be64_to_cpu(data2);
2879
2880                         status = VXGE_HW_OK;
2881                 } else
2882                         *product_desc = 0;
2883         }
2884
2885         return status;
2886 }
2887
2888 /*
2889  * __vxge_hw_vpath_fw_ver_get - Get the fw version
2890  * Returns FW Version
2891  */
2892 enum vxge_hw_status
2893 __vxge_hw_vpath_fw_ver_get(
2894         u32 vp_id,
2895         struct vxge_hw_vpath_reg __iomem *vpath_reg,
2896         struct vxge_hw_device_hw_info *hw_info)
2897 {
2898         u64 val64;
2899         u64 data1 = 0ULL;
2900         u64 data2 = 0ULL;
2901         struct vxge_hw_device_version *fw_version = &hw_info->fw_version;
2902         struct vxge_hw_device_date *fw_date = &hw_info->fw_date;
2903         struct vxge_hw_device_version *flash_version = &hw_info->flash_version;
2904         struct vxge_hw_device_date *flash_date = &hw_info->flash_date;
2905         enum vxge_hw_status status = VXGE_HW_OK;
2906
2907         val64 = VXGE_HW_RTS_ACCESS_STEER_CTRL_ACTION(
2908                 VXGE_HW_RTS_ACCESS_STEER_CTRL_ACTION_READ_ENTRY) |
2909                 VXGE_HW_RTS_ACCESS_STEER_CTRL_DATA_STRUCT_SEL(
2910                 VXGE_HW_RTS_ACCESS_STEER_CTRL_DATA_STRUCT_SEL_FW_MEMO) |
2911                 VXGE_HW_RTS_ACCESS_STEER_CTRL_STROBE |
2912                 VXGE_HW_RTS_ACCESS_STEER_CTRL_OFFSET(0);
2913
2914         status = __vxge_hw_pio_mem_write64(val64,
2915                                 &vpath_reg->rts_access_steer_ctrl,
2916                                 VXGE_HW_RTS_ACCESS_STEER_CTRL_STROBE,
2917                                 VXGE_HW_DEF_DEVICE_POLL_MILLIS);
2918
2919         if (status != VXGE_HW_OK)
2920                 goto exit;
2921
2922         val64 = readq(&vpath_reg->rts_access_steer_ctrl);
2923
2924         if (val64 & VXGE_HW_RTS_ACCESS_STEER_CTRL_RMACJ_STATUS) {
2925
2926                 data1 = readq(&vpath_reg->rts_access_steer_data0);
2927                 data2 = readq(&vpath_reg->rts_access_steer_data1);
2928
2929                 fw_date->day =
2930                         (u32)VXGE_HW_RTS_ACCESS_STEER_DATA0_GET_FW_VER_DAY(
2931                                                 data1);
2932                 fw_date->month =
2933                         (u32)VXGE_HW_RTS_ACCESS_STEER_DATA0_GET_FW_VER_MONTH(
2934                                                 data1);
2935                 fw_date->year =
2936                         (u32)VXGE_HW_RTS_ACCESS_STEER_DATA0_GET_FW_VER_YEAR(
2937                                                 data1);
2938
2939                 snprintf(fw_date->date, VXGE_HW_FW_STRLEN, "%2.2d/%2.2d/%4.4d",
2940                         fw_date->month, fw_date->day, fw_date->year);
2941
2942                 fw_version->major =
2943                     (u32)VXGE_HW_RTS_ACCESS_STEER_DATA0_GET_FW_VER_MAJOR(data1);
2944                 fw_version->minor =
2945                     (u32)VXGE_HW_RTS_ACCESS_STEER_DATA0_GET_FW_VER_MINOR(data1);
2946                 fw_version->build =
2947                     (u32)VXGE_HW_RTS_ACCESS_STEER_DATA0_GET_FW_VER_BUILD(data1);
2948
2949                 snprintf(fw_version->version, VXGE_HW_FW_STRLEN, "%d.%d.%d",
2950                     fw_version->major, fw_version->minor, fw_version->build);
2951
2952                 flash_date->day =
2953                   (u32)VXGE_HW_RTS_ACCESS_STEER_DATA1_GET_FLASH_VER_DAY(data2);
2954                 flash_date->month =
2955                  (u32)VXGE_HW_RTS_ACCESS_STEER_DATA1_GET_FLASH_VER_MONTH(data2);
2956                 flash_date->year =
2957                  (u32)VXGE_HW_RTS_ACCESS_STEER_DATA1_GET_FLASH_VER_YEAR(data2);
2958
2959                 snprintf(flash_date->date, VXGE_HW_FW_STRLEN,
2960                         "%2.2d/%2.2d/%4.4d",
2961                         flash_date->month, flash_date->day, flash_date->year);
2962
2963                 flash_version->major =
2964                  (u32)VXGE_HW_RTS_ACCESS_STEER_DATA1_GET_FLASH_VER_MAJOR(data2);
2965                 flash_version->minor =
2966                  (u32)VXGE_HW_RTS_ACCESS_STEER_DATA1_GET_FLASH_VER_MINOR(data2);
2967                 flash_version->build =
2968                  (u32)VXGE_HW_RTS_ACCESS_STEER_DATA1_GET_FLASH_VER_BUILD(data2);
2969
2970                 snprintf(flash_version->version, VXGE_HW_FW_STRLEN, "%d.%d.%d",
2971                         flash_version->major, flash_version->minor,
2972                         flash_version->build);
2973
2974                 status = VXGE_HW_OK;
2975
2976         } else
2977                 status = VXGE_HW_FAIL;
2978 exit:
2979         return status;
2980 }
2981
2982 /*
2983  * __vxge_hw_vpath_pci_func_mode_get - Get the pci mode
2984  * Returns pci function mode
2985  */
2986 u64
2987 __vxge_hw_vpath_pci_func_mode_get(
2988         u32  vp_id,
2989         struct vxge_hw_vpath_reg __iomem *vpath_reg)
2990 {
2991         u64 val64;
2992         u64 data1 = 0ULL;
2993         enum vxge_hw_status status = VXGE_HW_OK;
2994
2995         __vxge_hw_read_rts_ds(vpath_reg,
2996                 VXGE_HW_RTS_ACCESS_STEER_DATA0_MEMO_ITEM_PCI_MODE);
2997
2998         val64 = VXGE_HW_RTS_ACCESS_STEER_CTRL_ACTION(
2999                         VXGE_HW_RTS_ACCESS_STEER_CTRL_ACTION_READ_MEMO_ENTRY) |
3000                 VXGE_HW_RTS_ACCESS_STEER_CTRL_DATA_STRUCT_SEL(
3001                         VXGE_HW_RTS_ACCESS_STEER_CTRL_DATA_STRUCT_SEL_FW_MEMO) |
3002                 VXGE_HW_RTS_ACCESS_STEER_CTRL_STROBE |
3003                 VXGE_HW_RTS_ACCESS_STEER_CTRL_OFFSET(0);
3004
3005         status = __vxge_hw_pio_mem_write64(val64,
3006                                 &vpath_reg->rts_access_steer_ctrl,
3007                                 VXGE_HW_RTS_ACCESS_STEER_CTRL_STROBE,
3008                                 VXGE_HW_DEF_DEVICE_POLL_MILLIS);
3009
3010         if (status != VXGE_HW_OK)
3011                 goto exit;
3012
3013         val64 = readq(&vpath_reg->rts_access_steer_ctrl);
3014
3015         if (val64 & VXGE_HW_RTS_ACCESS_STEER_CTRL_RMACJ_STATUS) {
3016                 data1 = readq(&vpath_reg->rts_access_steer_data0);
3017                 status = VXGE_HW_OK;
3018         } else {
3019                 data1 = 0;
3020                 status = VXGE_HW_FAIL;
3021         }
3022 exit:
3023         return data1;
3024 }
3025
3026 /**
3027  * vxge_hw_device_flick_link_led - Flick (blink) link LED.
3028  * @hldev: HW device.
3029  * @on_off: TRUE if flickering to be on, FALSE to be off
3030  *
3031  * Flicker the link LED.
3032  */
3033 enum vxge_hw_status
3034 vxge_hw_device_flick_link_led(struct __vxge_hw_device *hldev,
3035                                u64 on_off)
3036 {
3037         u64 val64;
3038         enum vxge_hw_status status = VXGE_HW_OK;
3039         struct vxge_hw_vpath_reg __iomem *vp_reg;
3040
3041         if (hldev == NULL) {
3042                 status = VXGE_HW_ERR_INVALID_DEVICE;
3043                 goto exit;
3044         }
3045
3046         vp_reg = hldev->vpath_reg[hldev->first_vp_id];
3047
3048         writeq(0, &vp_reg->rts_access_steer_ctrl);
3049         wmb();
3050         writeq(on_off, &vp_reg->rts_access_steer_data0);
3051         writeq(0, &vp_reg->rts_access_steer_data1);
3052         wmb();
3053
3054         val64 = VXGE_HW_RTS_ACCESS_STEER_CTRL_ACTION(
3055                         VXGE_HW_RTS_ACCESS_STEER_CTRL_ACTION_LED_CONTROL) |
3056                 VXGE_HW_RTS_ACCESS_STEER_CTRL_DATA_STRUCT_SEL(
3057                         VXGE_HW_RTS_ACCESS_STEER_CTRL_DATA_STRUCT_SEL_FW_MEMO) |
3058                 VXGE_HW_RTS_ACCESS_STEER_CTRL_STROBE |
3059                 VXGE_HW_RTS_ACCESS_STEER_CTRL_OFFSET(0);
3060
3061         status = __vxge_hw_pio_mem_write64(val64,
3062                                 &vp_reg->rts_access_steer_ctrl,
3063                                 VXGE_HW_RTS_ACCESS_STEER_CTRL_STROBE,
3064                                 VXGE_HW_DEF_DEVICE_POLL_MILLIS);
3065 exit:
3066         return status;
3067 }
3068
3069 /*
3070  * __vxge_hw_vpath_rts_table_get - Get the entries from RTS access tables
3071  */
3072 enum vxge_hw_status
3073 __vxge_hw_vpath_rts_table_get(
3074         struct __vxge_hw_vpath_handle *vp,
3075         u32 action, u32 rts_table, u32 offset, u64 *data1, u64 *data2)
3076 {
3077         u64 val64;
3078         struct __vxge_hw_virtualpath *vpath;
3079         struct vxge_hw_vpath_reg __iomem *vp_reg;
3080
3081         enum vxge_hw_status status = VXGE_HW_OK;
3082
3083         if (vp == NULL) {
3084                 status = VXGE_HW_ERR_INVALID_HANDLE;
3085                 goto exit;
3086         }
3087
3088         vpath = vp->vpath;
3089         vp_reg = vpath->vp_reg;
3090
3091         val64 = VXGE_HW_RTS_ACCESS_STEER_CTRL_ACTION(action) |
3092                 VXGE_HW_RTS_ACCESS_STEER_CTRL_DATA_STRUCT_SEL(rts_table) |
3093                 VXGE_HW_RTS_ACCESS_STEER_CTRL_STROBE |
3094                 VXGE_HW_RTS_ACCESS_STEER_CTRL_OFFSET(offset);
3095
3096         if ((rts_table ==
3097                 VXGE_HW_RTS_ACS_STEER_CTRL_DATA_STRUCT_SEL_RTH_SOLO_IT) ||
3098             (rts_table ==
3099                 VXGE_HW_RTS_ACS_STEER_CTRL_DATA_STRUCT_SEL_RTH_MULTI_IT) ||
3100             (rts_table ==
3101                 VXGE_HW_RTS_ACCESS_STEER_CTRL_DATA_STRUCT_SEL_RTH_MASK) ||
3102             (rts_table ==
3103                 VXGE_HW_RTS_ACCESS_STEER_CTRL_DATA_STRUCT_SEL_RTH_KEY)) {
3104                 val64 = val64 | VXGE_HW_RTS_ACCESS_STEER_CTRL_TABLE_SEL;
3105         }
3106
3107         status = __vxge_hw_pio_mem_write64(val64,
3108                                 &vp_reg->rts_access_steer_ctrl,
3109                                 VXGE_HW_RTS_ACCESS_STEER_CTRL_STROBE,
3110                                 vpath->hldev->config.device_poll_millis);
3111
3112         if (status != VXGE_HW_OK)
3113                 goto exit;
3114
3115         val64 = readq(&vp_reg->rts_access_steer_ctrl);
3116
3117         if (val64 & VXGE_HW_RTS_ACCESS_STEER_CTRL_RMACJ_STATUS) {
3118
3119                 *data1 = readq(&vp_reg->rts_access_steer_data0);
3120
3121                 if ((rts_table ==
3122                 VXGE_HW_RTS_ACCESS_STEER_CTRL_DATA_STRUCT_SEL_DA) ||
3123                 (rts_table ==
3124                 VXGE_HW_RTS_ACS_STEER_CTRL_DATA_STRUCT_SEL_RTH_MULTI_IT)) {
3125                         *data2 = readq(&vp_reg->rts_access_steer_data1);
3126                 }
3127                 status = VXGE_HW_OK;
3128         } else
3129                 status = VXGE_HW_FAIL;
3130 exit:
3131         return status;
3132 }
3133
3134 /*
3135  * __vxge_hw_vpath_rts_table_set - Set the entries of RTS access tables
3136  */
3137 enum vxge_hw_status
3138 __vxge_hw_vpath_rts_table_set(
3139         struct __vxge_hw_vpath_handle *vp, u32 action, u32 rts_table,
3140         u32 offset, u64 data1, u64 data2)
3141 {
3142         u64 val64;
3143         struct __vxge_hw_virtualpath *vpath;
3144         enum vxge_hw_status status = VXGE_HW_OK;
3145         struct vxge_hw_vpath_reg __iomem *vp_reg;
3146
3147         if (vp == NULL) {
3148                 status = VXGE_HW_ERR_INVALID_HANDLE;
3149                 goto exit;
3150         }
3151
3152         vpath = vp->vpath;
3153         vp_reg = vpath->vp_reg;
3154
3155         writeq(data1, &vp_reg->rts_access_steer_data0);
3156         wmb();
3157
3158         if ((rts_table == VXGE_HW_RTS_ACCESS_STEER_CTRL_DATA_STRUCT_SEL_DA) ||
3159             (rts_table ==
3160                 VXGE_HW_RTS_ACS_STEER_CTRL_DATA_STRUCT_SEL_RTH_MULTI_IT)) {
3161                 writeq(data2, &vp_reg->rts_access_steer_data1);
3162                 wmb();
3163         }
3164
3165         val64 = VXGE_HW_RTS_ACCESS_STEER_CTRL_ACTION(action) |
3166                 VXGE_HW_RTS_ACCESS_STEER_CTRL_DATA_STRUCT_SEL(rts_table) |
3167                 VXGE_HW_RTS_ACCESS_STEER_CTRL_STROBE |
3168                 VXGE_HW_RTS_ACCESS_STEER_CTRL_OFFSET(offset);
3169
3170         status = __vxge_hw_pio_mem_write64(val64,
3171                                 &vp_reg->rts_access_steer_ctrl,
3172                                 VXGE_HW_RTS_ACCESS_STEER_CTRL_STROBE,
3173                                 vpath->hldev->config.device_poll_millis);
3174
3175         if (status != VXGE_HW_OK)
3176                 goto exit;
3177
3178         val64 = readq(&vp_reg->rts_access_steer_ctrl);
3179
3180         if (val64 & VXGE_HW_RTS_ACCESS_STEER_CTRL_RMACJ_STATUS)
3181                 status = VXGE_HW_OK;
3182         else
3183                 status = VXGE_HW_FAIL;
3184 exit:
3185         return status;
3186 }
3187
3188 /*
3189  * __vxge_hw_vpath_addr_get - Get the hw address entry for this vpath
3190  *               from MAC address table.
3191  */
3192 enum vxge_hw_status
3193 __vxge_hw_vpath_addr_get(
3194         u32 vp_id, struct vxge_hw_vpath_reg __iomem *vpath_reg,
3195         u8 (macaddr)[ETH_ALEN], u8 (macaddr_mask)[ETH_ALEN])
3196 {
3197         u32 i;
3198         u64 val64;
3199         u64 data1 = 0ULL;
3200         u64 data2 = 0ULL;
3201         enum vxge_hw_status status = VXGE_HW_OK;
3202
3203         val64 = VXGE_HW_RTS_ACCESS_STEER_CTRL_ACTION(
3204                 VXGE_HW_RTS_ACCESS_STEER_CTRL_ACTION_LIST_FIRST_ENTRY) |
3205                 VXGE_HW_RTS_ACCESS_STEER_CTRL_DATA_STRUCT_SEL(
3206                 VXGE_HW_RTS_ACCESS_STEER_CTRL_DATA_STRUCT_SEL_DA) |
3207                 VXGE_HW_RTS_ACCESS_STEER_CTRL_STROBE |
3208                 VXGE_HW_RTS_ACCESS_STEER_CTRL_OFFSET(0);
3209
3210         status = __vxge_hw_pio_mem_write64(val64,
3211                                 &vpath_reg->rts_access_steer_ctrl,
3212                                 VXGE_HW_RTS_ACCESS_STEER_CTRL_STROBE,
3213                                 VXGE_HW_DEF_DEVICE_POLL_MILLIS);
3214
3215         if (status != VXGE_HW_OK)
3216                 goto exit;
3217
3218         val64 = readq(&vpath_reg->rts_access_steer_ctrl);
3219
3220         if (val64 & VXGE_HW_RTS_ACCESS_STEER_CTRL_RMACJ_STATUS) {
3221
3222                 data1 = readq(&vpath_reg->rts_access_steer_data0);
3223                 data2 = readq(&vpath_reg->rts_access_steer_data1);
3224
3225                 data1 = VXGE_HW_RTS_ACCESS_STEER_DATA0_GET_DA_MAC_ADDR(data1);
3226                 data2 = VXGE_HW_RTS_ACCESS_STEER_DATA1_GET_DA_MAC_ADDR_MASK(
3227                                                         data2);
3228
3229                 for (i = ETH_ALEN; i > 0; i--) {
3230                         macaddr[i-1] = (u8)(data1 & 0xFF);
3231                         data1 >>= 8;
3232
3233                         macaddr_mask[i-1] = (u8)(data2 & 0xFF);
3234                         data2 >>= 8;
3235                 }
3236                 status = VXGE_HW_OK;
3237         } else
3238                 status = VXGE_HW_FAIL;
3239 exit:
3240         return status;
3241 }
3242
3243 /*
3244  * vxge_hw_vpath_rts_rth_set - Set/configure RTS hashing.
3245  */
3246 enum vxge_hw_status vxge_hw_vpath_rts_rth_set(
3247                         struct __vxge_hw_vpath_handle *vp,
3248                         enum vxge_hw_rth_algoritms algorithm,
3249                         struct vxge_hw_rth_hash_types *hash_type,
3250                         u16 bucket_size)
3251 {
3252         u64 data0, data1;
3253         enum vxge_hw_status status = VXGE_HW_OK;
3254
3255         if (vp == NULL) {
3256                 status = VXGE_HW_ERR_INVALID_HANDLE;
3257                 goto exit;
3258         }
3259
3260         status = __vxge_hw_vpath_rts_table_get(vp,
3261                      VXGE_HW_RTS_ACCESS_STEER_CTRL_ACTION_READ_ENTRY,
3262                      VXGE_HW_RTS_ACCESS_STEER_CTRL_DATA_STRUCT_SEL_RTH_GEN_CFG,
3263                         0, &data0, &data1);
3264
3265         data0 &= ~(VXGE_HW_RTS_ACCESS_STEER_DATA0_RTH_GEN_BUCKET_SIZE(0xf) |
3266                         VXGE_HW_RTS_ACCESS_STEER_DATA0_RTH_GEN_ALG_SEL(0x3));
3267
3268         data0 |= VXGE_HW_RTS_ACCESS_STEER_DATA0_RTH_GEN_RTH_EN |
3269         VXGE_HW_RTS_ACCESS_STEER_DATA0_RTH_GEN_BUCKET_SIZE(bucket_size) |
3270         VXGE_HW_RTS_ACCESS_STEER_DATA0_RTH_GEN_ALG_SEL(algorithm);
3271
3272         if (hash_type->hash_type_tcpipv4_en)
3273                 data0 |= VXGE_HW_RTS_ACCESS_STEER_DATA0_RTH_GEN_RTH_TCP_IPV4_EN;
3274
3275         if (hash_type->hash_type_ipv4_en)
3276                 data0 |= VXGE_HW_RTS_ACCESS_STEER_DATA0_RTH_GEN_RTH_IPV4_EN;
3277
3278         if (hash_type->hash_type_tcpipv6_en)
3279                 data0 |= VXGE_HW_RTS_ACCESS_STEER_DATA0_RTH_GEN_RTH_TCP_IPV6_EN;
3280
3281         if (hash_type->hash_type_ipv6_en)
3282                 data0 |= VXGE_HW_RTS_ACCESS_STEER_DATA0_RTH_GEN_RTH_IPV6_EN;
3283
3284         if (hash_type->hash_type_tcpipv6ex_en)
3285                 data0 |=
3286                 VXGE_HW_RTS_ACCESS_STEER_DATA0_RTH_GEN_RTH_TCP_IPV6_EX_EN;
3287
3288         if (hash_type->hash_type_ipv6ex_en)
3289                 data0 |= VXGE_HW_RTS_ACCESS_STEER_DATA0_RTH_GEN_RTH_IPV6_EX_EN;
3290
3291         if (VXGE_HW_RTS_ACCESS_STEER_DATA0_GET_RTH_GEN_ACTIVE_TABLE(data0))
3292                 data0 &= ~VXGE_HW_RTS_ACCESS_STEER_DATA0_RTH_GEN_ACTIVE_TABLE;
3293         else
3294                 data0 |= VXGE_HW_RTS_ACCESS_STEER_DATA0_RTH_GEN_ACTIVE_TABLE;
3295
3296         status = __vxge_hw_vpath_rts_table_set(vp,
3297                 VXGE_HW_RTS_ACCESS_STEER_CTRL_ACTION_WRITE_ENTRY,
3298                 VXGE_HW_RTS_ACCESS_STEER_CTRL_DATA_STRUCT_SEL_RTH_GEN_CFG,
3299                 0, data0, 0);
3300 exit:
3301         return status;
3302 }
3303
3304 static void
3305 vxge_hw_rts_rth_data0_data1_get(u32 j, u64 *data0, u64 *data1,
3306                                 u16 flag, u8 *itable)
3307 {
3308         switch (flag) {
3309         case 1:
3310                 *data0 = VXGE_HW_RTS_ACCESS_STEER_DATA0_RTH_ITEM0_BUCKET_NUM(j)|
3311                         VXGE_HW_RTS_ACCESS_STEER_DATA0_RTH_ITEM0_ENTRY_EN |
3312                         VXGE_HW_RTS_ACCESS_STEER_DATA0_RTH_ITEM0_BUCKET_DATA(
3313                         itable[j]);
3314         case 2:
3315                 *data0 |=
3316                         VXGE_HW_RTS_ACCESS_STEER_DATA0_RTH_ITEM1_BUCKET_NUM(j)|
3317                         VXGE_HW_RTS_ACCESS_STEER_DATA0_RTH_ITEM1_ENTRY_EN |
3318                         VXGE_HW_RTS_ACCESS_STEER_DATA0_RTH_ITEM1_BUCKET_DATA(
3319                         itable[j]);
3320         case 3:
3321                 *data1 = VXGE_HW_RTS_ACCESS_STEER_DATA1_RTH_ITEM0_BUCKET_NUM(j)|
3322                         VXGE_HW_RTS_ACCESS_STEER_DATA1_RTH_ITEM0_ENTRY_EN |
3323                         VXGE_HW_RTS_ACCESS_STEER_DATA1_RTH_ITEM0_BUCKET_DATA(
3324                         itable[j]);
3325         case 4:
3326                 *data1 |=
3327                         VXGE_HW_RTS_ACCESS_STEER_DATA1_RTH_ITEM1_BUCKET_NUM(j)|
3328                         VXGE_HW_RTS_ACCESS_STEER_DATA1_RTH_ITEM1_ENTRY_EN |
3329                         VXGE_HW_RTS_ACCESS_STEER_DATA1_RTH_ITEM1_BUCKET_DATA(
3330                         itable[j]);
3331         default:
3332                 return;
3333         }
3334 }
3335 /*
3336  * vxge_hw_vpath_rts_rth_itable_set - Set/configure indirection table (IT).
3337  */
3338 enum vxge_hw_status vxge_hw_vpath_rts_rth_itable_set(
3339                         struct __vxge_hw_vpath_handle **vpath_handles,
3340                         u32 vpath_count,
3341                         u8 *mtable,
3342                         u8 *itable,
3343                         u32 itable_size)
3344 {
3345         u32 i, j, action, rts_table;
3346         u64 data0;
3347         u64 data1;
3348         u32 max_entries;
3349         enum vxge_hw_status status = VXGE_HW_OK;
3350         struct __vxge_hw_vpath_handle *vp = vpath_handles[0];
3351
3352         if (vp == NULL) {
3353                 status = VXGE_HW_ERR_INVALID_HANDLE;
3354                 goto exit;
3355         }
3356
3357         max_entries = (((u32)1) << itable_size);
3358
3359         if (vp->vpath->hldev->config.rth_it_type
3360                                 == VXGE_HW_RTH_IT_TYPE_SOLO_IT) {
3361                 action = VXGE_HW_RTS_ACCESS_STEER_CTRL_ACTION_WRITE_ENTRY;
3362                 rts_table =
3363                         VXGE_HW_RTS_ACS_STEER_CTRL_DATA_STRUCT_SEL_RTH_SOLO_IT;
3364
3365                 for (j = 0; j < max_entries; j++) {
3366
3367                         data1 = 0;
3368
3369                         data0 =
3370                         VXGE_HW_RTS_ACCESS_STEER_DATA0_RTH_SOLO_IT_BUCKET_DATA(
3371                                 itable[j]);
3372
3373                         status = __vxge_hw_vpath_rts_table_set(vpath_handles[0],
3374                                 action, rts_table, j, data0, data1);
3375
3376                         if (status != VXGE_HW_OK)
3377                                 goto exit;
3378                 }
3379
3380                 for (j = 0; j < max_entries; j++) {
3381
3382                         data1 = 0;
3383
3384                         data0 =
3385                         VXGE_HW_RTS_ACCESS_STEER_DATA0_RTH_SOLO_IT_ENTRY_EN |
3386                         VXGE_HW_RTS_ACCESS_STEER_DATA0_RTH_SOLO_IT_BUCKET_DATA(
3387                                 itable[j]);
3388
3389                         status = __vxge_hw_vpath_rts_table_set(
3390                                 vpath_handles[mtable[itable[j]]], action,
3391                                 rts_table, j, data0, data1);
3392
3393                         if (status != VXGE_HW_OK)
3394                                 goto exit;
3395                 }
3396         } else {
3397                 action = VXGE_HW_RTS_ACCESS_STEER_CTRL_ACTION_WRITE_ENTRY;
3398                 rts_table =
3399                         VXGE_HW_RTS_ACS_STEER_CTRL_DATA_STRUCT_SEL_RTH_MULTI_IT;
3400                 for (i = 0; i < vpath_count; i++) {
3401
3402                         for (j = 0; j < max_entries;) {
3403
3404                                 data0 = 0;
3405                                 data1 = 0;
3406
3407                                 while (j < max_entries) {
3408                                         if (mtable[itable[j]] != i) {
3409                                                 j++;
3410                                                 continue;
3411                                         }
3412                                         vxge_hw_rts_rth_data0_data1_get(j,
3413                                                 &data0, &data1, 1, itable);
3414                                         j++;
3415                                         break;
3416                                 }
3417
3418                                 while (j < max_entries) {
3419                                         if (mtable[itable[j]] != i) {
3420                                                 j++;
3421                                                 continue;
3422                                         }
3423                                         vxge_hw_rts_rth_data0_data1_get(j,
3424                                                 &data0, &data1, 2, itable);
3425                                         j++;
3426                                         break;
3427                                 }
3428
3429                                 while (j < max_entries) {
3430                                         if (mtable[itable[j]] != i) {
3431                                                 j++;
3432                                                 continue;
3433                                         }
3434                                         vxge_hw_rts_rth_data0_data1_get(j,
3435                                                 &data0, &data1, 3, itable);
3436                                         j++;
3437                                         break;
3438                                 }
3439
3440                                 while (j < max_entries) {
3441                                         if (mtable[itable[j]] != i) {
3442                                                 j++;
3443                                                 continue;
3444                                         }
3445                                         vxge_hw_rts_rth_data0_data1_get(j,
3446                                                 &data0, &data1, 4, itable);
3447                                         j++;
3448                                         break;
3449                                 }
3450
3451                                 if (data0 != 0) {
3452                                         status = __vxge_hw_vpath_rts_table_set(
3453                                                         vpath_handles[i],
3454                                                         action, rts_table,
3455                                                         0, data0, data1);
3456
3457                                         if (status != VXGE_HW_OK)
3458                                                 goto exit;
3459                                 }
3460                         }
3461                 }
3462         }
3463 exit:
3464         return status;
3465 }
3466
3467 /**
3468  * vxge_hw_vpath_check_leak - Check for memory leak
3469  * @ringh: Handle to the ring object used for receive
3470  *
3471  * If PRC_RXD_DOORBELL_VPn.NEW_QW_CNT is larger or equal to
3472  * PRC_CFG6_VPn.RXD_SPAT then a leak has occurred.
3473  * Returns: VXGE_HW_FAIL, if leak has occurred.
3474  *
3475  */
3476 enum vxge_hw_status
3477 vxge_hw_vpath_check_leak(struct __vxge_hw_ring *ring)
3478 {
3479         enum vxge_hw_status status = VXGE_HW_OK;
3480         u64 rxd_new_count, rxd_spat;
3481
3482         if (ring == NULL)
3483                 return status;
3484
3485         rxd_new_count = readl(&ring->vp_reg->prc_rxd_doorbell);
3486         rxd_spat = readq(&ring->vp_reg->prc_cfg6);
3487         rxd_spat = VXGE_HW_PRC_CFG6_RXD_SPAT(rxd_spat);
3488
3489         if (rxd_new_count >= rxd_spat)
3490                 status = VXGE_HW_FAIL;
3491
3492         return status;
3493 }
3494
3495 /*
3496  * __vxge_hw_vpath_mgmt_read
3497  * This routine reads the vpath_mgmt registers
3498  */
3499 static enum vxge_hw_status
3500 __vxge_hw_vpath_mgmt_read(
3501         struct __vxge_hw_device *hldev,
3502         struct __vxge_hw_virtualpath *vpath)
3503 {
3504         u32 i, mtu = 0, max_pyld = 0;
3505         u64 val64;
3506         enum vxge_hw_status status = VXGE_HW_OK;
3507
3508         for (i = 0; i < VXGE_HW_MAC_MAX_MAC_PORT_ID; i++) {
3509
3510                 val64 = readq(&vpath->vpmgmt_reg->
3511                                 rxmac_cfg0_port_vpmgmt_clone[i]);
3512                 max_pyld =
3513                         (u32)
3514                         VXGE_HW_RXMAC_CFG0_PORT_VPMGMT_CLONE_GET_MAX_PYLD_LEN
3515                         (val64);
3516                 if (mtu < max_pyld)
3517                         mtu = max_pyld;
3518         }
3519
3520         vpath->max_mtu = mtu + VXGE_HW_MAC_HEADER_MAX_SIZE;
3521
3522         val64 = readq(&vpath->vpmgmt_reg->xmac_vsport_choices_vp);
3523
3524         for (i = 0; i < VXGE_HW_MAX_VIRTUAL_PATHS; i++) {
3525                 if (val64 & vxge_mBIT(i))
3526                         vpath->vsport_number = i;
3527         }
3528
3529         val64 = readq(&vpath->vpmgmt_reg->xgmac_gen_status_vpmgmt_clone);
3530
3531         if (val64 & VXGE_HW_XGMAC_GEN_STATUS_VPMGMT_CLONE_XMACJ_NTWK_OK)
3532                 VXGE_HW_DEVICE_LINK_STATE_SET(vpath->hldev, VXGE_HW_LINK_UP);
3533         else
3534                 VXGE_HW_DEVICE_LINK_STATE_SET(vpath->hldev, VXGE_HW_LINK_DOWN);
3535
3536         return status;
3537 }
3538
3539 /*
3540  * __vxge_hw_vpath_reset_check - Check if resetting the vpath completed
3541  * This routine checks the vpath_rst_in_prog register to see if
3542  * adapter completed the reset process for the vpath
3543  */
3544 enum vxge_hw_status
3545 __vxge_hw_vpath_reset_check(struct __vxge_hw_virtualpath *vpath)
3546 {
3547         enum vxge_hw_status status;
3548
3549         status = __vxge_hw_device_register_poll(
3550                         &vpath->hldev->common_reg->vpath_rst_in_prog,
3551                         VXGE_HW_VPATH_RST_IN_PROG_VPATH_RST_IN_PROG(
3552                                 1 << (16 - vpath->vp_id)),
3553                         vpath->hldev->config.device_poll_millis);
3554
3555         return status;
3556 }
3557
3558 /*
3559  * __vxge_hw_vpath_reset
3560  * This routine resets the vpath on the device
3561  */
3562 enum vxge_hw_status
3563 __vxge_hw_vpath_reset(struct __vxge_hw_device *hldev, u32 vp_id)
3564 {
3565         u64 val64;
3566         enum vxge_hw_status status = VXGE_HW_OK;
3567
3568         val64 = VXGE_HW_CMN_RSTHDLR_CFG0_SW_RESET_VPATH(1 << (16 - vp_id));
3569
3570         __vxge_hw_pio_mem_write32_upper((u32)vxge_bVALn(val64, 0, 32),
3571                                 &hldev->common_reg->cmn_rsthdlr_cfg0);
3572
3573         return status;
3574 }
3575
3576 /*
3577  * __vxge_hw_vpath_sw_reset
3578  * This routine resets the vpath structures
3579  */
3580 enum vxge_hw_status
3581 __vxge_hw_vpath_sw_reset(struct __vxge_hw_device *hldev, u32 vp_id)
3582 {
3583         enum vxge_hw_status status = VXGE_HW_OK;
3584         struct __vxge_hw_virtualpath *vpath;
3585
3586         vpath = (struct __vxge_hw_virtualpath *)&hldev->virtual_paths[vp_id];
3587
3588         if (vpath->ringh) {
3589                 status = __vxge_hw_ring_reset(vpath->ringh);
3590                 if (status != VXGE_HW_OK)
3591                         goto exit;
3592         }
3593
3594         if (vpath->fifoh)
3595                 status = __vxge_hw_fifo_reset(vpath->fifoh);
3596 exit:
3597         return status;
3598 }
3599
3600 /*
3601  * __vxge_hw_vpath_prc_configure
3602  * This routine configures the prc registers of virtual path using the config
3603  * passed
3604  */
3605 void
3606 __vxge_hw_vpath_prc_configure(struct __vxge_hw_device *hldev, u32 vp_id)
3607 {
3608         u64 val64;
3609         struct __vxge_hw_virtualpath *vpath;
3610         struct vxge_hw_vp_config *vp_config;
3611         struct vxge_hw_vpath_reg __iomem *vp_reg;
3612
3613         vpath = &hldev->virtual_paths[vp_id];
3614         vp_reg = vpath->vp_reg;
3615         vp_config = vpath->vp_config;
3616
3617         if (vp_config->ring.enable == VXGE_HW_RING_DISABLE)
3618                 return;
3619
3620         val64 = readq(&vp_reg->prc_cfg1);
3621         val64 |= VXGE_HW_PRC_CFG1_RTI_TINT_DISABLE;
3622         writeq(val64, &vp_reg->prc_cfg1);
3623
3624         val64 = readq(&vpath->vp_reg->prc_cfg6);
3625         val64 |= VXGE_HW_PRC_CFG6_DOORBELL_MODE_EN;
3626         writeq(val64, &vpath->vp_reg->prc_cfg6);
3627
3628         val64 = readq(&vp_reg->prc_cfg7);
3629
3630         if (vpath->vp_config->ring.scatter_mode !=
3631                 VXGE_HW_RING_SCATTER_MODE_USE_FLASH_DEFAULT) {
3632
3633                 val64 &= ~VXGE_HW_PRC_CFG7_SCATTER_MODE(0x3);
3634
3635                 switch (vpath->vp_config->ring.scatter_mode) {
3636                 case VXGE_HW_RING_SCATTER_MODE_A:
3637                         val64 |= VXGE_HW_PRC_CFG7_SCATTER_MODE(
3638                                         VXGE_HW_PRC_CFG7_SCATTER_MODE_A);
3639                         break;
3640                 case VXGE_HW_RING_SCATTER_MODE_B:
3641                         val64 |= VXGE_HW_PRC_CFG7_SCATTER_MODE(
3642                                         VXGE_HW_PRC_CFG7_SCATTER_MODE_B);
3643                         break;
3644                 case VXGE_HW_RING_SCATTER_MODE_C:
3645                         val64 |= VXGE_HW_PRC_CFG7_SCATTER_MODE(
3646                                         VXGE_HW_PRC_CFG7_SCATTER_MODE_C);
3647                         break;
3648                 }
3649         }
3650
3651         writeq(val64, &vp_reg->prc_cfg7);
3652
3653         writeq(VXGE_HW_PRC_CFG5_RXD0_ADD(
3654                                 __vxge_hw_ring_first_block_address_get(
3655                                         vpath->ringh) >> 3), &vp_reg->prc_cfg5);
3656
3657         val64 = readq(&vp_reg->prc_cfg4);
3658         val64 |= VXGE_HW_PRC_CFG4_IN_SVC;
3659         val64 &= ~VXGE_HW_PRC_CFG4_RING_MODE(0x3);
3660
3661         val64 |= VXGE_HW_PRC_CFG4_RING_MODE(
3662                         VXGE_HW_PRC_CFG4_RING_MODE_ONE_BUFFER);
3663
3664         if (hldev->config.rth_en == VXGE_HW_RTH_DISABLE)
3665                 val64 |= VXGE_HW_PRC_CFG4_RTH_DISABLE;
3666         else
3667                 val64 &= ~VXGE_HW_PRC_CFG4_RTH_DISABLE;
3668
3669         writeq(val64, &vp_reg->prc_cfg4);
3670         return;
3671 }
3672
3673 /*
3674  * __vxge_hw_vpath_kdfc_configure
3675  * This routine configures the kdfc registers of virtual path using the
3676  * config passed
3677  */
3678 enum vxge_hw_status
3679 __vxge_hw_vpath_kdfc_configure(struct __vxge_hw_device *hldev, u32 vp_id)
3680 {
3681         u64 val64;
3682         u64 vpath_stride;
3683         enum vxge_hw_status status = VXGE_HW_OK;
3684         struct __vxge_hw_virtualpath *vpath;
3685         struct vxge_hw_vpath_reg __iomem *vp_reg;
3686
3687         vpath = &hldev->virtual_paths[vp_id];
3688         vp_reg = vpath->vp_reg;
3689         status = __vxge_hw_kdfc_swapper_set(hldev->legacy_reg, vp_reg);
3690
3691         if (status != VXGE_HW_OK)
3692                 goto exit;
3693
3694         val64 = readq(&vp_reg->kdfc_drbl_triplet_total);
3695
3696         vpath->max_kdfc_db =
3697                 (u32)VXGE_HW_KDFC_DRBL_TRIPLET_TOTAL_GET_KDFC_MAX_SIZE(
3698                         val64+1)/2;
3699
3700         if (vpath->vp_config->fifo.enable == VXGE_HW_FIFO_ENABLE) {
3701
3702                 vpath->max_nofl_db = vpath->max_kdfc_db;
3703
3704                 if (vpath->max_nofl_db <
3705                         ((vpath->vp_config->fifo.memblock_size /
3706                         (vpath->vp_config->fifo.max_frags *
3707                         sizeof(struct vxge_hw_fifo_txd))) *
3708                         vpath->vp_config->fifo.fifo_blocks)) {
3709
3710                         return VXGE_HW_BADCFG_FIFO_BLOCKS;
3711                 }
3712                 val64 = VXGE_HW_KDFC_FIFO_TRPL_PARTITION_LENGTH_0(
3713                                 (vpath->max_nofl_db*2)-1);
3714         }
3715
3716         writeq(val64, &vp_reg->kdfc_fifo_trpl_partition);
3717
3718         writeq(VXGE_HW_KDFC_FIFO_TRPL_CTRL_TRIPLET_ENABLE,
3719                 &vp_reg->kdfc_fifo_trpl_ctrl);
3720
3721         val64 = readq(&vp_reg->kdfc_trpl_fifo_0_ctrl);
3722
3723         val64 &= ~(VXGE_HW_KDFC_TRPL_FIFO_0_CTRL_MODE(0x3) |
3724                    VXGE_HW_KDFC_TRPL_FIFO_0_CTRL_SELECT(0xFF));
3725
3726         val64 |= VXGE_HW_KDFC_TRPL_FIFO_0_CTRL_MODE(
3727                  VXGE_HW_KDFC_TRPL_FIFO_0_CTRL_MODE_NON_OFFLOAD_ONLY) |
3728 #ifndef __BIG_ENDIAN
3729                  VXGE_HW_KDFC_TRPL_FIFO_0_CTRL_SWAP_EN |
3730 #endif
3731                  VXGE_HW_KDFC_TRPL_FIFO_0_CTRL_SELECT(0);
3732
3733         writeq(val64, &vp_reg->kdfc_trpl_fifo_0_ctrl);
3734         writeq((u64)0, &vp_reg->kdfc_trpl_fifo_0_wb_address);
3735         wmb();
3736         vpath_stride = readq(&hldev->toc_reg->toc_kdfc_vpath_stride);
3737
3738         vpath->nofl_db =
3739                 (struct __vxge_hw_non_offload_db_wrapper __iomem *)
3740                 (hldev->kdfc + (vp_id *
3741                 VXGE_HW_TOC_KDFC_VPATH_STRIDE_GET_TOC_KDFC_VPATH_STRIDE(
3742                                         vpath_stride)));
3743 exit:
3744         return status;
3745 }
3746
3747 /*
3748  * __vxge_hw_vpath_mac_configure
3749  * This routine configures the mac of virtual path using the config passed
3750  */
3751 enum vxge_hw_status
3752 __vxge_hw_vpath_mac_configure(struct __vxge_hw_device *hldev, u32 vp_id)
3753 {
3754         u64 val64;
3755         enum vxge_hw_status status = VXGE_HW_OK;
3756         struct __vxge_hw_virtualpath *vpath;
3757         struct vxge_hw_vp_config *vp_config;
3758         struct vxge_hw_vpath_reg __iomem *vp_reg;
3759
3760         vpath = &hldev->virtual_paths[vp_id];
3761         vp_reg = vpath->vp_reg;
3762         vp_config = vpath->vp_config;
3763
3764         writeq(VXGE_HW_XMAC_VSPORT_CHOICE_VSPORT_NUMBER(
3765                         vpath->vsport_number), &vp_reg->xmac_vsport_choice);
3766
3767         if (vp_config->ring.enable == VXGE_HW_RING_ENABLE) {
3768
3769                 val64 = readq(&vp_reg->xmac_rpa_vcfg);
3770
3771                 if (vp_config->rpa_strip_vlan_tag !=
3772                         VXGE_HW_VPATH_RPA_STRIP_VLAN_TAG_USE_FLASH_DEFAULT) {
3773                         if (vp_config->rpa_strip_vlan_tag)
3774                                 val64 |= VXGE_HW_XMAC_RPA_VCFG_STRIP_VLAN_TAG;
3775                         else
3776                                 val64 &= ~VXGE_HW_XMAC_RPA_VCFG_STRIP_VLAN_TAG;
3777                 }
3778
3779                 writeq(val64, &vp_reg->xmac_rpa_vcfg);
3780                 val64 = readq(&vp_reg->rxmac_vcfg0);
3781
3782                 if (vp_config->mtu !=
3783                                 VXGE_HW_VPATH_USE_FLASH_DEFAULT_INITIAL_MTU) {
3784                         val64 &= ~VXGE_HW_RXMAC_VCFG0_RTS_MAX_FRM_LEN(0x3fff);
3785                         if ((vp_config->mtu  +
3786                                 VXGE_HW_MAC_HEADER_MAX_SIZE) < vpath->max_mtu)
3787                                 val64 |= VXGE_HW_RXMAC_VCFG0_RTS_MAX_FRM_LEN(
3788                                         vp_config->mtu  +
3789                                         VXGE_HW_MAC_HEADER_MAX_SIZE);
3790                         else
3791                                 val64 |= VXGE_HW_RXMAC_VCFG0_RTS_MAX_FRM_LEN(
3792                                         vpath->max_mtu);
3793                 }
3794
3795                 writeq(val64, &vp_reg->rxmac_vcfg0);
3796
3797                 val64 = readq(&vp_reg->rxmac_vcfg1);
3798
3799                 val64 &= ~(VXGE_HW_RXMAC_VCFG1_RTS_RTH_MULTI_IT_BD_MODE(0x3) |
3800                         VXGE_HW_RXMAC_VCFG1_RTS_RTH_MULTI_IT_EN_MODE);
3801
3802                 if (hldev->config.rth_it_type ==
3803                                 VXGE_HW_RTH_IT_TYPE_MULTI_IT) {
3804                         val64 |= VXGE_HW_RXMAC_VCFG1_RTS_RTH_MULTI_IT_BD_MODE(
3805                                 0x2) |
3806                                 VXGE_HW_RXMAC_VCFG1_RTS_RTH_MULTI_IT_EN_MODE;
3807                 }
3808
3809                 writeq(val64, &vp_reg->rxmac_vcfg1);
3810         }
3811         return status;
3812 }
3813
3814 /*
3815  * __vxge_hw_vpath_tim_configure
3816  * This routine configures the tim registers of virtual path using the config
3817  * passed
3818  */
3819 enum vxge_hw_status
3820 __vxge_hw_vpath_tim_configure(struct __vxge_hw_device *hldev, u32 vp_id)
3821 {
3822         u64 val64;
3823         enum vxge_hw_status status = VXGE_HW_OK;
3824         struct __vxge_hw_virtualpath *vpath;
3825         struct vxge_hw_vpath_reg __iomem *vp_reg;
3826         struct vxge_hw_vp_config *config;
3827
3828         vpath = &hldev->virtual_paths[vp_id];
3829         vp_reg = vpath->vp_reg;
3830         config = vpath->vp_config;
3831
3832         writeq((u64)0, &vp_reg->tim_dest_addr);
3833         writeq((u64)0, &vp_reg->tim_vpath_map);
3834         writeq((u64)0, &vp_reg->tim_bitmap);
3835         writeq((u64)0, &vp_reg->tim_remap);
3836
3837         if (config->ring.enable == VXGE_HW_RING_ENABLE)
3838                 writeq(VXGE_HW_TIM_RING_ASSN_INT_NUM(
3839                         (vp_id * VXGE_HW_MAX_INTR_PER_VP) +
3840                         VXGE_HW_VPATH_INTR_RX), &vp_reg->tim_ring_assn);
3841
3842         val64 = readq(&vp_reg->tim_pci_cfg);
3843         val64 |= VXGE_HW_TIM_PCI_CFG_ADD_PAD;
3844         writeq(val64, &vp_reg->tim_pci_cfg);
3845
3846         if (config->fifo.enable == VXGE_HW_FIFO_ENABLE) {
3847
3848                 val64 = readq(&vp_reg->tim_cfg1_int_num[VXGE_HW_VPATH_INTR_TX]);
3849
3850                 if (config->tti.btimer_val != VXGE_HW_USE_FLASH_DEFAULT) {
3851                         val64 &= ~VXGE_HW_TIM_CFG1_INT_NUM_BTIMER_VAL(
3852                                 0x3ffffff);
3853                         val64 |= VXGE_HW_TIM_CFG1_INT_NUM_BTIMER_VAL(
3854                                         config->tti.btimer_val);
3855                 }
3856
3857                 val64 &= ~VXGE_HW_TIM_CFG1_INT_NUM_BITMP_EN;
3858
3859                 if (config->tti.timer_ac_en != VXGE_HW_USE_FLASH_DEFAULT) {
3860                         if (config->tti.timer_ac_en)
3861                                 val64 |= VXGE_HW_TIM_CFG1_INT_NUM_TIMER_AC;
3862                         else
3863                                 val64 &= ~VXGE_HW_TIM_CFG1_INT_NUM_TIMER_AC;
3864                 }
3865
3866                 if (config->tti.timer_ci_en != VXGE_HW_USE_FLASH_DEFAULT) {
3867                         if (config->tti.timer_ci_en)
3868                                 val64 |= VXGE_HW_TIM_CFG1_INT_NUM_TIMER_CI;
3869                         else
3870                                 val64 &= ~VXGE_HW_TIM_CFG1_INT_NUM_TIMER_CI;
3871                 }
3872
3873                 if (config->tti.urange_a != VXGE_HW_USE_FLASH_DEFAULT) {
3874                         val64 &= ~VXGE_HW_TIM_CFG1_INT_NUM_URNG_A(0x3f);
3875                         val64 |= VXGE_HW_TIM_CFG1_INT_NUM_URNG_A(
3876                                         config->tti.urange_a);
3877                 }
3878
3879                 if (config->tti.urange_b != VXGE_HW_USE_FLASH_DEFAULT) {
3880                         val64 &= ~VXGE_HW_TIM_CFG1_INT_NUM_URNG_B(0x3f);
3881                         val64 |= VXGE_HW_TIM_CFG1_INT_NUM_URNG_B(
3882                                         config->tti.urange_b);
3883                 }
3884
3885                 if (config->tti.urange_c != VXGE_HW_USE_FLASH_DEFAULT) {
3886                         val64 &= ~VXGE_HW_TIM_CFG1_INT_NUM_URNG_C(0x3f);
3887                         val64 |= VXGE_HW_TIM_CFG1_INT_NUM_URNG_C(
3888                                         config->tti.urange_c);
3889                 }
3890
3891                 writeq(val64, &vp_reg->tim_cfg1_int_num[VXGE_HW_VPATH_INTR_TX]);
3892                 val64 = readq(&vp_reg->tim_cfg2_int_num[VXGE_HW_VPATH_INTR_TX]);
3893
3894                 if (config->tti.uec_a != VXGE_HW_USE_FLASH_DEFAULT) {
3895                         val64 &= ~VXGE_HW_TIM_CFG2_INT_NUM_UEC_A(0xffff);
3896                         val64 |= VXGE_HW_TIM_CFG2_INT_NUM_UEC_A(
3897                                                 config->tti.uec_a);
3898                 }
3899
3900                 if (config->tti.uec_b != VXGE_HW_USE_FLASH_DEFAULT) {
3901                         val64 &= ~VXGE_HW_TIM_CFG2_INT_NUM_UEC_B(0xffff);
3902                         val64 |= VXGE_HW_TIM_CFG2_INT_NUM_UEC_B(
3903                                                 config->tti.uec_b);
3904                 }
3905
3906                 if (config->tti.uec_c != VXGE_HW_USE_FLASH_DEFAULT) {
3907                         val64 &= ~VXGE_HW_TIM_CFG2_INT_NUM_UEC_C(0xffff);
3908                         val64 |= VXGE_HW_TIM_CFG2_INT_NUM_UEC_C(
3909                                                 config->tti.uec_c);
3910                 }
3911
3912                 if (config->tti.uec_d != VXGE_HW_USE_FLASH_DEFAULT) {
3913                         val64 &= ~VXGE_HW_TIM_CFG2_INT_NUM_UEC_D(0xffff);
3914                         val64 |= VXGE_HW_TIM_CFG2_INT_NUM_UEC_D(
3915                                                 config->tti.uec_d);
3916                 }
3917
3918                 writeq(val64, &vp_reg->tim_cfg2_int_num[VXGE_HW_VPATH_INTR_TX]);
3919                 val64 = readq(&vp_reg->tim_cfg3_int_num[VXGE_HW_VPATH_INTR_TX]);
3920
3921                 if (config->tti.timer_ri_en != VXGE_HW_USE_FLASH_DEFAULT) {
3922                         if (config->tti.timer_ri_en)
3923                                 val64 |= VXGE_HW_TIM_CFG3_INT_NUM_TIMER_RI;
3924                         else
3925                                 val64 &= ~VXGE_HW_TIM_CFG3_INT_NUM_TIMER_RI;
3926                 }
3927
3928                 if (config->tti.rtimer_val != VXGE_HW_USE_FLASH_DEFAULT) {
3929                         val64 &= ~VXGE_HW_TIM_CFG3_INT_NUM_RTIMER_VAL(
3930                                         0x3ffffff);
3931                         val64 |= VXGE_HW_TIM_CFG3_INT_NUM_RTIMER_VAL(
3932                                         config->tti.rtimer_val);
3933                 }
3934
3935                 if (config->tti.util_sel != VXGE_HW_USE_FLASH_DEFAULT) {
3936                         val64 &= ~VXGE_HW_TIM_CFG3_INT_NUM_UTIL_SEL(0x3f);
3937                         val64 |= VXGE_HW_TIM_CFG3_INT_NUM_UTIL_SEL(
3938                                         config->tti.util_sel);
3939                 }
3940
3941                 if (config->tti.ltimer_val != VXGE_HW_USE_FLASH_DEFAULT) {
3942                         val64 &= ~VXGE_HW_TIM_CFG3_INT_NUM_LTIMER_VAL(
3943                                         0x3ffffff);
3944                         val64 |= VXGE_HW_TIM_CFG3_INT_NUM_LTIMER_VAL(
3945                                         config->tti.ltimer_val);
3946                 }
3947
3948                 writeq(val64, &vp_reg->tim_cfg3_int_num[VXGE_HW_VPATH_INTR_TX]);
3949         }
3950
3951         if (config->ring.enable == VXGE_HW_RING_ENABLE) {
3952
3953                 val64 = readq(&vp_reg->tim_cfg1_int_num[VXGE_HW_VPATH_INTR_RX]);
3954
3955                 if (config->rti.btimer_val != VXGE_HW_USE_FLASH_DEFAULT) {
3956                         val64 &= ~VXGE_HW_TIM_CFG1_INT_NUM_BTIMER_VAL(
3957                                         0x3ffffff);
3958                         val64 |= VXGE_HW_TIM_CFG1_INT_NUM_BTIMER_VAL(
3959                                         config->rti.btimer_val);
3960                 }
3961
3962                 val64 &= ~VXGE_HW_TIM_CFG1_INT_NUM_BITMP_EN;
3963
3964                 if (config->rti.timer_ac_en != VXGE_HW_USE_FLASH_DEFAULT) {
3965                         if (config->rti.timer_ac_en)
3966                                 val64 |= VXGE_HW_TIM_CFG1_INT_NUM_TIMER_AC;
3967                         else
3968                                 val64 &= ~VXGE_HW_TIM_CFG1_INT_NUM_TIMER_AC;
3969                 }
3970
3971                 if (config->rti.timer_ci_en != VXGE_HW_USE_FLASH_DEFAULT) {
3972                         if (config->rti.timer_ci_en)
3973                                 val64 |= VXGE_HW_TIM_CFG1_INT_NUM_TIMER_CI;
3974                         else
3975                                 val64 &= ~VXGE_HW_TIM_CFG1_INT_NUM_TIMER_CI;
3976                 }
3977
3978                 if (config->rti.urange_a != VXGE_HW_USE_FLASH_DEFAULT) {
3979                         val64 &= ~VXGE_HW_TIM_CFG1_INT_NUM_URNG_A(0x3f);
3980                         val64 |= VXGE_HW_TIM_CFG1_INT_NUM_URNG_A(
3981                                         config->rti.urange_a);
3982                 }
3983
3984                 if (config->rti.urange_b != VXGE_HW_USE_FLASH_DEFAULT) {
3985                         val64 &= ~VXGE_HW_TIM_CFG1_INT_NUM_URNG_B(0x3f);
3986                         val64 |= VXGE_HW_TIM_CFG1_INT_NUM_URNG_B(
3987                                         config->rti.urange_b);
3988                 }
3989
3990                 if (config->rti.urange_c != VXGE_HW_USE_FLASH_DEFAULT) {
3991                         val64 &= ~VXGE_HW_TIM_CFG1_INT_NUM_URNG_C(0x3f);
3992                         val64 |= VXGE_HW_TIM_CFG1_INT_NUM_URNG_C(
3993                                         config->rti.urange_c);
3994                 }
3995
3996                 writeq(val64, &vp_reg->tim_cfg1_int_num[VXGE_HW_VPATH_INTR_RX]);
3997                 val64 = readq(&vp_reg->tim_cfg2_int_num[VXGE_HW_VPATH_INTR_RX]);
3998
3999                 if (config->rti.uec_a != VXGE_HW_USE_FLASH_DEFAULT) {
4000                         val64 &= ~VXGE_HW_TIM_CFG2_INT_NUM_UEC_A(0xffff);
4001                         val64 |= VXGE_HW_TIM_CFG2_INT_NUM_UEC_A(
4002                                                 config->rti.uec_a);
4003                 }
4004
4005                 if (config->rti.uec_b != VXGE_HW_USE_FLASH_DEFAULT) {
4006                         val64 &= ~VXGE_HW_TIM_CFG2_INT_NUM_UEC_B(0xffff);
4007                         val64 |= VXGE_HW_TIM_CFG2_INT_NUM_UEC_B(
4008                                                 config->rti.uec_b);
4009                 }
4010
4011                 if (config->rti.uec_c != VXGE_HW_USE_FLASH_DEFAULT) {
4012                         val64 &= ~VXGE_HW_TIM_CFG2_INT_NUM_UEC_C(0xffff);
4013                         val64 |= VXGE_HW_TIM_CFG2_INT_NUM_UEC_C(
4014                                                 config->rti.uec_c);
4015                 }
4016
4017                 if (config->rti.uec_d != VXGE_HW_USE_FLASH_DEFAULT) {
4018                         val64 &= ~VXGE_HW_TIM_CFG2_INT_NUM_UEC_D(0xffff);
4019                         val64 |= VXGE_HW_TIM_CFG2_INT_NUM_UEC_D(
4020                                                 config->rti.uec_d);
4021                 }
4022
4023                 writeq(val64, &vp_reg->tim_cfg2_int_num[VXGE_HW_VPATH_INTR_RX]);
4024                 val64 = readq(&vp_reg->tim_cfg3_int_num[VXGE_HW_VPATH_INTR_RX]);
4025
4026                 if (config->rti.timer_ri_en != VXGE_HW_USE_FLASH_DEFAULT) {
4027                         if (config->rti.timer_ri_en)
4028                                 val64 |= VXGE_HW_TIM_CFG3_INT_NUM_TIMER_RI;
4029                         else
4030                                 val64 &= ~VXGE_HW_TIM_CFG3_INT_NUM_TIMER_RI;
4031                 }
4032
4033                 if (config->rti.rtimer_val != VXGE_HW_USE_FLASH_DEFAULT) {
4034                         val64 &= ~VXGE_HW_TIM_CFG3_INT_NUM_RTIMER_VAL(
4035                                         0x3ffffff);
4036                         val64 |= VXGE_HW_TIM_CFG3_INT_NUM_RTIMER_VAL(
4037                                         config->rti.rtimer_val);
4038                 }
4039
4040                 if (config->rti.util_sel != VXGE_HW_USE_FLASH_DEFAULT) {
4041                         val64 &= ~VXGE_HW_TIM_CFG3_INT_NUM_UTIL_SEL(0x3f);
4042                         val64 |= VXGE_HW_TIM_CFG3_INT_NUM_UTIL_SEL(
4043                                         config->rti.util_sel);
4044                 }
4045
4046                 if (config->rti.ltimer_val != VXGE_HW_USE_FLASH_DEFAULT) {
4047                         val64 &= ~VXGE_HW_TIM_CFG3_INT_NUM_LTIMER_VAL(
4048                                         0x3ffffff);
4049                         val64 |= VXGE_HW_TIM_CFG3_INT_NUM_LTIMER_VAL(
4050                                         config->rti.ltimer_val);
4051                 }
4052
4053                 writeq(val64, &vp_reg->tim_cfg3_int_num[VXGE_HW_VPATH_INTR_RX]);
4054         }
4055
4056         val64 = 0;
4057         writeq(val64, &vp_reg->tim_cfg1_int_num[VXGE_HW_VPATH_INTR_EINTA]);
4058         writeq(val64, &vp_reg->tim_cfg2_int_num[VXGE_HW_VPATH_INTR_EINTA]);
4059         writeq(val64, &vp_reg->tim_cfg3_int_num[VXGE_HW_VPATH_INTR_EINTA]);
4060         writeq(val64, &vp_reg->tim_cfg1_int_num[VXGE_HW_VPATH_INTR_BMAP]);
4061         writeq(val64, &vp_reg->tim_cfg2_int_num[VXGE_HW_VPATH_INTR_BMAP]);
4062         writeq(val64, &vp_reg->tim_cfg3_int_num[VXGE_HW_VPATH_INTR_BMAP]);
4063
4064         return status;
4065 }
4066
4067 /*
4068  * __vxge_hw_vpath_initialize
4069  * This routine is the final phase of init which initializes the
4070  * registers of the vpath using the configuration passed.
4071  */
4072 enum vxge_hw_status
4073 __vxge_hw_vpath_initialize(struct __vxge_hw_device *hldev, u32 vp_id)
4074 {
4075         u64 val64;
4076         u32 val32;
4077         enum vxge_hw_status status = VXGE_HW_OK;
4078         struct __vxge_hw_virtualpath *vpath;
4079         struct vxge_hw_vpath_reg __iomem *vp_reg;
4080
4081         vpath = &hldev->virtual_paths[vp_id];
4082
4083         if (!(hldev->vpath_assignments & vxge_mBIT(vp_id))) {
4084                 status = VXGE_HW_ERR_VPATH_NOT_AVAILABLE;
4085                 goto exit;
4086         }
4087         vp_reg = vpath->vp_reg;
4088
4089         status =  __vxge_hw_vpath_swapper_set(vpath->vp_reg);
4090
4091         if (status != VXGE_HW_OK)
4092                 goto exit;
4093
4094         status =  __vxge_hw_vpath_mac_configure(hldev, vp_id);
4095
4096         if (status != VXGE_HW_OK)
4097                 goto exit;
4098
4099         status =  __vxge_hw_vpath_kdfc_configure(hldev, vp_id);
4100
4101         if (status != VXGE_HW_OK)
4102                 goto exit;
4103
4104         status = __vxge_hw_vpath_tim_configure(hldev, vp_id);
4105
4106         if (status != VXGE_HW_OK)
4107                 goto exit;
4108
4109         val64 = readq(&vp_reg->rtdma_rd_optimization_ctrl);
4110
4111         /* Get MRRS value from device control */
4112         status  = __vxge_hw_vpath_pci_read(vpath, 1, 0x78, &val32);
4113
4114         if (status == VXGE_HW_OK) {
4115                 val32 = (val32 & VXGE_HW_PCI_EXP_DEVCTL_READRQ) >> 12;
4116                 val64 &=
4117                     ~(VXGE_HW_RTDMA_RD_OPTIMIZATION_CTRL_FB_FILL_THRESH(7));
4118                 val64 |=
4119                     VXGE_HW_RTDMA_RD_OPTIMIZATION_CTRL_FB_FILL_THRESH(val32);
4120
4121                 val64 |= VXGE_HW_RTDMA_RD_OPTIMIZATION_CTRL_FB_WAIT_FOR_SPACE;
4122         }
4123
4124         val64 &= ~(VXGE_HW_RTDMA_RD_OPTIMIZATION_CTRL_FB_ADDR_BDRY(7));
4125         val64 |=
4126             VXGE_HW_RTDMA_RD_OPTIMIZATION_CTRL_FB_ADDR_BDRY(
4127                     VXGE_HW_MAX_PAYLOAD_SIZE_512);
4128
4129         val64 |= VXGE_HW_RTDMA_RD_OPTIMIZATION_CTRL_FB_ADDR_BDRY_EN;
4130         writeq(val64, &vp_reg->rtdma_rd_optimization_ctrl);
4131
4132 exit:
4133         return status;
4134 }
4135
4136 /*
4137  * __vxge_hw_vp_initialize - Initialize Virtual Path structure
4138  * This routine is the initial phase of init which resets the vpath and
4139  * initializes the software support structures.
4140  */
4141 enum vxge_hw_status
4142 __vxge_hw_vp_initialize(struct __vxge_hw_device *hldev, u32 vp_id,
4143                         struct vxge_hw_vp_config *config)
4144 {
4145         struct __vxge_hw_virtualpath *vpath;
4146         enum vxge_hw_status status = VXGE_HW_OK;
4147
4148         if (!(hldev->vpath_assignments & vxge_mBIT(vp_id))) {
4149                 status = VXGE_HW_ERR_VPATH_NOT_AVAILABLE;
4150                 goto exit;
4151         }
4152
4153         vpath = &hldev->virtual_paths[vp_id];
4154
4155         vpath->vp_id = vp_id;
4156         vpath->vp_open = VXGE_HW_VP_OPEN;
4157         vpath->hldev = hldev;
4158         vpath->vp_config = config;
4159         vpath->vp_reg = hldev->vpath_reg[vp_id];
4160         vpath->vpmgmt_reg = hldev->vpmgmt_reg[vp_id];
4161
4162         __vxge_hw_vpath_reset(hldev, vp_id);
4163
4164         status = __vxge_hw_vpath_reset_check(vpath);
4165
4166         if (status != VXGE_HW_OK) {
4167                 memset(vpath, 0, sizeof(struct __vxge_hw_virtualpath));
4168                 goto exit;
4169         }
4170
4171         status = __vxge_hw_vpath_mgmt_read(hldev, vpath);
4172
4173         if (status != VXGE_HW_OK) {
4174                 memset(vpath, 0, sizeof(struct __vxge_hw_virtualpath));
4175                 goto exit;
4176         }
4177
4178         INIT_LIST_HEAD(&vpath->vpath_handles);
4179
4180         vpath->sw_stats = &hldev->stats.sw_dev_info_stats.vpath_info[vp_id];
4181
4182         VXGE_HW_DEVICE_TIM_INT_MASK_SET(hldev->tim_int_mask0,
4183                 hldev->tim_int_mask1, vp_id);
4184
4185         status = __vxge_hw_vpath_initialize(hldev, vp_id);
4186
4187         if (status != VXGE_HW_OK)
4188                 __vxge_hw_vp_terminate(hldev, vp_id);
4189 exit:
4190         return status;
4191 }
4192
4193 /*
4194  * __vxge_hw_vp_terminate - Terminate Virtual Path structure
4195  * This routine closes all channels it opened and freeup memory
4196  */
4197 void
4198 __vxge_hw_vp_terminate(struct __vxge_hw_device *hldev, u32 vp_id)
4199 {
4200         struct __vxge_hw_virtualpath *vpath;
4201
4202         vpath = &hldev->virtual_paths[vp_id];
4203
4204         if (vpath->vp_open == VXGE_HW_VP_NOT_OPEN)
4205                 goto exit;
4206
4207         VXGE_HW_DEVICE_TIM_INT_MASK_RESET(vpath->hldev->tim_int_mask0,
4208                 vpath->hldev->tim_int_mask1, vpath->vp_id);
4209         hldev->stats.hw_dev_info_stats.vpath_info[vpath->vp_id] = NULL;
4210
4211         memset(vpath, 0, sizeof(struct __vxge_hw_virtualpath));
4212 exit:
4213         return;
4214 }
4215
4216 /*
4217  * vxge_hw_vpath_mtu_set - Set MTU.
4218  * Set new MTU value. Example, to use jumbo frames:
4219  * vxge_hw_vpath_mtu_set(my_device, 9600);
4220  */
4221 enum vxge_hw_status
4222 vxge_hw_vpath_mtu_set(struct __vxge_hw_vpath_handle *vp, u32 new_mtu)
4223 {
4224         u64 val64;
4225         enum vxge_hw_status status = VXGE_HW_OK;
4226         struct __vxge_hw_virtualpath *vpath;
4227
4228         if (vp == NULL) {
4229                 status = VXGE_HW_ERR_INVALID_HANDLE;
4230                 goto exit;
4231         }
4232         vpath = vp->vpath;
4233
4234         new_mtu += VXGE_HW_MAC_HEADER_MAX_SIZE;
4235
4236         if ((new_mtu < VXGE_HW_MIN_MTU) || (new_mtu > vpath->max_mtu))
4237                 status = VXGE_HW_ERR_INVALID_MTU_SIZE;
4238
4239         val64 = readq(&vpath->vp_reg->rxmac_vcfg0);
4240
4241         val64 &= ~VXGE_HW_RXMAC_VCFG0_RTS_MAX_FRM_LEN(0x3fff);
4242         val64 |= VXGE_HW_RXMAC_VCFG0_RTS_MAX_FRM_LEN(new_mtu);
4243
4244         writeq(val64, &vpath->vp_reg->rxmac_vcfg0);
4245
4246         vpath->vp_config->mtu = new_mtu - VXGE_HW_MAC_HEADER_MAX_SIZE;
4247
4248 exit:
4249         return status;
4250 }
4251
4252 /*
4253  * vxge_hw_vpath_open - Open a virtual path on a given adapter
4254  * This function is used to open access to virtual path of an
4255  * adapter for offload, GRO operations. This function returns
4256  * synchronously.
4257  */
4258 enum vxge_hw_status
4259 vxge_hw_vpath_open(struct __vxge_hw_device *hldev,
4260                    struct vxge_hw_vpath_attr *attr,
4261                    struct __vxge_hw_vpath_handle **vpath_handle)
4262 {
4263         struct __vxge_hw_virtualpath *vpath;
4264         struct __vxge_hw_vpath_handle *vp;
4265         enum vxge_hw_status status;
4266
4267         vpath = &hldev->virtual_paths[attr->vp_id];
4268
4269         if (vpath->vp_open == VXGE_HW_VP_OPEN) {
4270                 status = VXGE_HW_ERR_INVALID_STATE;
4271                 goto vpath_open_exit1;
4272         }
4273
4274         status = __vxge_hw_vp_initialize(hldev, attr->vp_id,
4275                         &hldev->config.vp_config[attr->vp_id]);
4276
4277         if (status != VXGE_HW_OK)
4278                 goto vpath_open_exit1;
4279
4280         vp = (struct __vxge_hw_vpath_handle *)
4281                 vmalloc(sizeof(struct __vxge_hw_vpath_handle));
4282         if (vp == NULL) {
4283                 status = VXGE_HW_ERR_OUT_OF_MEMORY;
4284                 goto vpath_open_exit2;
4285         }
4286
4287         memset(vp, 0, sizeof(struct __vxge_hw_vpath_handle));
4288
4289         vp->vpath = vpath;
4290
4291         if (vpath->vp_config->fifo.enable == VXGE_HW_FIFO_ENABLE) {
4292                 status = __vxge_hw_fifo_create(vp, &attr->fifo_attr);
4293                 if (status != VXGE_HW_OK)
4294                         goto vpath_open_exit6;
4295         }
4296
4297         if (vpath->vp_config->ring.enable == VXGE_HW_RING_ENABLE) {
4298                 status = __vxge_hw_ring_create(vp, &attr->ring_attr);
4299                 if (status != VXGE_HW_OK)
4300                         goto vpath_open_exit7;
4301
4302                 __vxge_hw_vpath_prc_configure(hldev, attr->vp_id);
4303         }
4304
4305         vpath->fifoh->tx_intr_num =
4306                 (attr->vp_id * VXGE_HW_MAX_INTR_PER_VP)  +
4307                         VXGE_HW_VPATH_INTR_TX;
4308
4309         vpath->stats_block = __vxge_hw_blockpool_block_allocate(hldev,
4310                                 VXGE_HW_BLOCK_SIZE);
4311
4312         if (vpath->stats_block == NULL) {
4313                 status = VXGE_HW_ERR_OUT_OF_MEMORY;
4314                 goto vpath_open_exit8;
4315         }
4316
4317         vpath->hw_stats = (struct vxge_hw_vpath_stats_hw_info *)vpath->
4318                         stats_block->memblock;
4319         memset(vpath->hw_stats, 0,
4320                 sizeof(struct vxge_hw_vpath_stats_hw_info));
4321
4322         hldev->stats.hw_dev_info_stats.vpath_info[attr->vp_id] =
4323                                                 vpath->hw_stats;
4324
4325         vpath->hw_stats_sav =
4326                 &hldev->stats.hw_dev_info_stats.vpath_info_sav[attr->vp_id];
4327         memset(vpath->hw_stats_sav, 0,
4328                         sizeof(struct vxge_hw_vpath_stats_hw_info));
4329
4330         writeq(vpath->stats_block->dma_addr, &vpath->vp_reg->stats_cfg);
4331
4332         status = vxge_hw_vpath_stats_enable(vp);
4333         if (status != VXGE_HW_OK)
4334                 goto vpath_open_exit8;
4335
4336         list_add(&vp->item, &vpath->vpath_handles);
4337
4338         hldev->vpaths_deployed |= vxge_mBIT(vpath->vp_id);
4339
4340         *vpath_handle = vp;
4341
4342         attr->fifo_attr.userdata = vpath->fifoh;
4343         attr->ring_attr.userdata = vpath->ringh;
4344
4345         return VXGE_HW_OK;
4346
4347 vpath_open_exit8:
4348         if (vpath->ringh != NULL)
4349                 __vxge_hw_ring_delete(vp);
4350 vpath_open_exit7:
4351         if (vpath->fifoh != NULL)
4352                 __vxge_hw_fifo_delete(vp);
4353 vpath_open_exit6:
4354         vfree(vp);
4355 vpath_open_exit2:
4356         __vxge_hw_vp_terminate(hldev, attr->vp_id);
4357 vpath_open_exit1:
4358
4359         return status;
4360 }
4361
4362 /**
4363  * vxge_hw_vpath_rx_doorbell_post - Close the handle got from previous vpath
4364  * (vpath) open
4365  * @vp: Handle got from previous vpath open
4366  *
4367  * This function is used to close access to virtual path opened
4368  * earlier.
4369  */
4370 void
4371 vxge_hw_vpath_rx_doorbell_init(struct __vxge_hw_vpath_handle *vp)
4372 {
4373         struct __vxge_hw_virtualpath *vpath = NULL;
4374         u64 new_count, val64, val164;
4375         struct __vxge_hw_ring *ring;
4376
4377         vpath = vp->vpath;
4378         ring = vpath->ringh;
4379
4380         new_count = readq(&vpath->vp_reg->rxdmem_size);
4381         new_count &= 0x1fff;
4382         val164 = (VXGE_HW_RXDMEM_SIZE_PRC_RXDMEM_SIZE(new_count));
4383
4384         writeq(VXGE_HW_PRC_RXD_DOORBELL_NEW_QW_CNT(val164),
4385                 &vpath->vp_reg->prc_rxd_doorbell);
4386         readl(&vpath->vp_reg->prc_rxd_doorbell);
4387
4388         val164 /= 2;
4389         val64 = readq(&vpath->vp_reg->prc_cfg6);
4390         val64 = VXGE_HW_PRC_CFG6_RXD_SPAT(val64);
4391         val64 &= 0x1ff;
4392
4393         /*
4394          * Each RxD is of 4 qwords
4395          */
4396         new_count -= (val64 + 1);
4397         val64 = min(val164, new_count) / 4;
4398
4399         ring->rxds_limit = min(ring->rxds_limit, val64);
4400         if (ring->rxds_limit < 4)
4401                 ring->rxds_limit = 4;
4402 }
4403
4404 /*
4405  * vxge_hw_vpath_close - Close the handle got from previous vpath (vpath) open
4406  * This function is used to close access to virtual path opened
4407  * earlier.
4408  */
4409 enum vxge_hw_status vxge_hw_vpath_close(struct __vxge_hw_vpath_handle *vp)
4410 {
4411         struct __vxge_hw_virtualpath *vpath = NULL;
4412         struct __vxge_hw_device *devh = NULL;
4413         u32 vp_id = vp->vpath->vp_id;
4414         u32 is_empty = TRUE;
4415         enum vxge_hw_status status = VXGE_HW_OK;
4416
4417         vpath = vp->vpath;
4418         devh = vpath->hldev;
4419
4420         if (vpath->vp_open == VXGE_HW_VP_NOT_OPEN) {
4421                 status = VXGE_HW_ERR_VPATH_NOT_OPEN;
4422                 goto vpath_close_exit;
4423         }
4424
4425         list_del(&vp->item);
4426
4427         if (!list_empty(&vpath->vpath_handles)) {
4428                 list_add(&vp->item, &vpath->vpath_handles);
4429                 is_empty = FALSE;
4430         }
4431
4432         if (!is_empty) {
4433                 status = VXGE_HW_FAIL;
4434                 goto vpath_close_exit;
4435         }
4436
4437         devh->vpaths_deployed &= ~vxge_mBIT(vp_id);
4438
4439         if (vpath->ringh != NULL)
4440                 __vxge_hw_ring_delete(vp);
4441
4442         if (vpath->fifoh != NULL)
4443                 __vxge_hw_fifo_delete(vp);
4444
4445         if (vpath->stats_block != NULL)
4446                 __vxge_hw_blockpool_block_free(devh, vpath->stats_block);
4447
4448         vfree(vp);
4449
4450         __vxge_hw_vp_terminate(devh, vp_id);
4451
4452         vpath->vp_open = VXGE_HW_VP_NOT_OPEN;
4453
4454 vpath_close_exit:
4455         return status;
4456 }
4457
4458 /*
4459  * vxge_hw_vpath_reset - Resets vpath
4460  * This function is used to request a reset of vpath
4461  */
4462 enum vxge_hw_status vxge_hw_vpath_reset(struct __vxge_hw_vpath_handle *vp)
4463 {
4464         enum vxge_hw_status status;
4465         u32 vp_id;
4466         struct __vxge_hw_virtualpath *vpath = vp->vpath;
4467
4468         vp_id = vpath->vp_id;
4469
4470         if (vpath->vp_open == VXGE_HW_VP_NOT_OPEN) {
4471                 status = VXGE_HW_ERR_VPATH_NOT_OPEN;
4472                 goto exit;
4473         }
4474
4475         status = __vxge_hw_vpath_reset(vpath->hldev, vp_id);
4476         if (status == VXGE_HW_OK)
4477                 vpath->sw_stats->soft_reset_cnt++;
4478 exit:
4479         return status;
4480 }
4481
4482 /*
4483  * vxge_hw_vpath_recover_from_reset - Poll for reset complete and re-initialize.
4484  * This function poll's for the vpath reset completion and re initializes
4485  * the vpath.
4486  */
4487 enum vxge_hw_status
4488 vxge_hw_vpath_recover_from_reset(struct __vxge_hw_vpath_handle *vp)
4489 {
4490         struct __vxge_hw_virtualpath *vpath = NULL;
4491         enum vxge_hw_status status;
4492         struct __vxge_hw_device *hldev;
4493         u32 vp_id;
4494
4495         vp_id = vp->vpath->vp_id;
4496         vpath = vp->vpath;
4497         hldev = vpath->hldev;
4498
4499         if (vpath->vp_open == VXGE_HW_VP_NOT_OPEN) {
4500                 status = VXGE_HW_ERR_VPATH_NOT_OPEN;
4501                 goto exit;
4502         }
4503
4504         status = __vxge_hw_vpath_reset_check(vpath);
4505         if (status != VXGE_HW_OK)
4506                 goto exit;
4507
4508         status = __vxge_hw_vpath_sw_reset(hldev, vp_id);
4509         if (status != VXGE_HW_OK)
4510                 goto exit;
4511
4512         status = __vxge_hw_vpath_initialize(hldev, vp_id);
4513         if (status != VXGE_HW_OK)
4514                 goto exit;
4515
4516         if (vpath->ringh != NULL)
4517                 __vxge_hw_vpath_prc_configure(hldev, vp_id);
4518
4519         memset(vpath->hw_stats, 0,
4520                 sizeof(struct vxge_hw_vpath_stats_hw_info));
4521
4522         memset(vpath->hw_stats_sav, 0,
4523                 sizeof(struct vxge_hw_vpath_stats_hw_info));
4524
4525         writeq(vpath->stats_block->dma_addr,
4526                 &vpath->vp_reg->stats_cfg);
4527
4528         status = vxge_hw_vpath_stats_enable(vp);
4529
4530 exit:
4531         return status;
4532 }
4533
4534 /*
4535  * vxge_hw_vpath_enable - Enable vpath.
4536  * This routine clears the vpath reset thereby enabling a vpath
4537  * to start forwarding frames and generating interrupts.
4538  */
4539 void
4540 vxge_hw_vpath_enable(struct __vxge_hw_vpath_handle *vp)
4541 {
4542         struct __vxge_hw_device *hldev;
4543         u64 val64;
4544
4545         hldev = vp->vpath->hldev;
4546
4547         val64 = VXGE_HW_CMN_RSTHDLR_CFG1_CLR_VPATH_RESET(
4548                 1 << (16 - vp->vpath->vp_id));
4549
4550         __vxge_hw_pio_mem_write32_upper((u32)vxge_bVALn(val64, 0, 32),
4551                 &hldev->common_reg->cmn_rsthdlr_cfg1);
4552 }
4553
4554 /*
4555  * vxge_hw_vpath_stats_enable - Enable vpath h/wstatistics.
4556  * Enable the DMA vpath statistics. The function is to be called to re-enable
4557  * the adapter to update stats into the host memory
4558  */
4559 enum vxge_hw_status
4560 vxge_hw_vpath_stats_enable(struct __vxge_hw_vpath_handle *vp)
4561 {
4562         enum vxge_hw_status status = VXGE_HW_OK;
4563         struct __vxge_hw_virtualpath *vpath;
4564
4565         vpath = vp->vpath;
4566
4567         if (vpath->vp_open == VXGE_HW_VP_NOT_OPEN) {
4568                 status = VXGE_HW_ERR_VPATH_NOT_OPEN;
4569                 goto exit;
4570         }
4571
4572         memcpy(vpath->hw_stats_sav, vpath->hw_stats,
4573                         sizeof(struct vxge_hw_vpath_stats_hw_info));
4574
4575         status = __vxge_hw_vpath_stats_get(vpath, vpath->hw_stats);
4576 exit:
4577         return status;
4578 }
4579
4580 /*
4581  * __vxge_hw_vpath_stats_access - Get the statistics from the given location
4582  *                           and offset and perform an operation
4583  */
4584 enum vxge_hw_status
4585 __vxge_hw_vpath_stats_access(struct __vxge_hw_virtualpath *vpath,
4586                              u32 operation, u32 offset, u64 *stat)
4587 {
4588         u64 val64;
4589         enum vxge_hw_status status = VXGE_HW_OK;
4590         struct vxge_hw_vpath_reg __iomem *vp_reg;
4591
4592         if (vpath->vp_open == VXGE_HW_VP_NOT_OPEN) {
4593                 status = VXGE_HW_ERR_VPATH_NOT_OPEN;
4594                 goto vpath_stats_access_exit;
4595         }
4596
4597         vp_reg = vpath->vp_reg;
4598
4599         val64 =  VXGE_HW_XMAC_STATS_ACCESS_CMD_OP(operation) |
4600                  VXGE_HW_XMAC_STATS_ACCESS_CMD_STROBE |
4601                  VXGE_HW_XMAC_STATS_ACCESS_CMD_OFFSET_SEL(offset);
4602
4603         status = __vxge_hw_pio_mem_write64(val64,
4604                                 &vp_reg->xmac_stats_access_cmd,
4605                                 VXGE_HW_XMAC_STATS_ACCESS_CMD_STROBE,
4606                                 vpath->hldev->config.device_poll_millis);
4607
4608         if ((status == VXGE_HW_OK) && (operation == VXGE_HW_STATS_OP_READ))
4609                 *stat = readq(&vp_reg->xmac_stats_access_data);
4610         else
4611                 *stat = 0;
4612
4613 vpath_stats_access_exit:
4614         return status;
4615 }
4616
4617 /*
4618  * __vxge_hw_vpath_xmac_tx_stats_get - Get the TX Statistics of a vpath
4619  */
4620 enum vxge_hw_status
4621 __vxge_hw_vpath_xmac_tx_stats_get(
4622         struct __vxge_hw_virtualpath *vpath,
4623         struct vxge_hw_xmac_vpath_tx_stats *vpath_tx_stats)
4624 {
4625         u64 *val64;
4626         int i;
4627         u32 offset = VXGE_HW_STATS_VPATH_TX_OFFSET;
4628         enum vxge_hw_status status = VXGE_HW_OK;
4629
4630         val64 = (u64 *) vpath_tx_stats;
4631
4632         if (vpath->vp_open == VXGE_HW_VP_NOT_OPEN) {
4633                 status = VXGE_HW_ERR_VPATH_NOT_OPEN;
4634                 goto exit;
4635         }
4636
4637         for (i = 0; i < sizeof(struct vxge_hw_xmac_vpath_tx_stats) / 8; i++) {
4638                 status = __vxge_hw_vpath_stats_access(vpath,
4639                                         VXGE_HW_STATS_OP_READ,
4640                                         offset, val64);
4641                 if (status != VXGE_HW_OK)
4642                         goto exit;
4643                 offset++;
4644                 val64++;
4645         }
4646 exit:
4647         return status;
4648 }
4649
4650 /*
4651  * __vxge_hw_vpath_xmac_rx_stats_get - Get the RX Statistics of a vpath
4652  */
4653 enum vxge_hw_status
4654 __vxge_hw_vpath_xmac_rx_stats_get(struct __vxge_hw_virtualpath *vpath,
4655                         struct vxge_hw_xmac_vpath_rx_stats *vpath_rx_stats)
4656 {
4657         u64 *val64;
4658         enum vxge_hw_status status = VXGE_HW_OK;
4659         int i;
4660         u32 offset = VXGE_HW_STATS_VPATH_RX_OFFSET;
4661         val64 = (u64 *) vpath_rx_stats;
4662
4663         if (vpath->vp_open == VXGE_HW_VP_NOT_OPEN) {
4664                 status = VXGE_HW_ERR_VPATH_NOT_OPEN;
4665                 goto exit;
4666         }
4667         for (i = 0; i < sizeof(struct vxge_hw_xmac_vpath_rx_stats) / 8; i++) {
4668                 status = __vxge_hw_vpath_stats_access(vpath,
4669                                         VXGE_HW_STATS_OP_READ,
4670                                         offset >> 3, val64);
4671                 if (status != VXGE_HW_OK)
4672                         goto exit;
4673
4674                 offset += 8;
4675                 val64++;
4676         }
4677 exit:
4678         return status;
4679 }
4680
4681 /*
4682  * __vxge_hw_vpath_stats_get - Get the vpath hw statistics.
4683  */
4684 enum vxge_hw_status __vxge_hw_vpath_stats_get(
4685                         struct __vxge_hw_virtualpath *vpath,
4686                         struct vxge_hw_vpath_stats_hw_info *hw_stats)
4687 {
4688         u64 val64;
4689         enum vxge_hw_status status = VXGE_HW_OK;
4690         struct vxge_hw_vpath_reg __iomem *vp_reg;
4691
4692         if (vpath->vp_open == VXGE_HW_VP_NOT_OPEN) {
4693                 status = VXGE_HW_ERR_VPATH_NOT_OPEN;
4694                 goto exit;
4695         }
4696         vp_reg = vpath->vp_reg;
4697
4698         val64 = readq(&vp_reg->vpath_debug_stats0);
4699         hw_stats->ini_num_mwr_sent =
4700                 (u32)VXGE_HW_VPATH_DEBUG_STATS0_GET_INI_NUM_MWR_SENT(val64);
4701
4702         val64 = readq(&vp_reg->vpath_debug_stats1);
4703         hw_stats->ini_num_mrd_sent =
4704                 (u32)VXGE_HW_VPATH_DEBUG_STATS1_GET_INI_NUM_MRD_SENT(val64);
4705
4706         val64 = readq(&vp_reg->vpath_debug_stats2);
4707         hw_stats->ini_num_cpl_rcvd =
4708                 (u32)VXGE_HW_VPATH_DEBUG_STATS2_GET_INI_NUM_CPL_RCVD(val64);
4709
4710         val64 = readq(&vp_reg->vpath_debug_stats3);
4711         hw_stats->ini_num_mwr_byte_sent =
4712                 VXGE_HW_VPATH_DEBUG_STATS3_GET_INI_NUM_MWR_BYTE_SENT(val64);
4713
4714         val64 = readq(&vp_reg->vpath_debug_stats4);
4715         hw_stats->ini_num_cpl_byte_rcvd =
4716                 VXGE_HW_VPATH_DEBUG_STATS4_GET_INI_NUM_CPL_BYTE_RCVD(val64);
4717
4718         val64 = readq(&vp_reg->vpath_debug_stats5);
4719         hw_stats->wrcrdtarb_xoff =
4720                 (u32)VXGE_HW_VPATH_DEBUG_STATS5_GET_WRCRDTARB_XOFF(val64);
4721
4722         val64 = readq(&vp_reg->vpath_debug_stats6);
4723         hw_stats->rdcrdtarb_xoff =
4724                 (u32)VXGE_HW_VPATH_DEBUG_STATS6_GET_RDCRDTARB_XOFF(val64);
4725
4726         val64 = readq(&vp_reg->vpath_genstats_count01);
4727         hw_stats->vpath_genstats_count0 =
4728         (u32)VXGE_HW_VPATH_GENSTATS_COUNT01_GET_PPIF_VPATH_GENSTATS_COUNT0(
4729                 val64);
4730
4731         val64 = readq(&vp_reg->vpath_genstats_count01);
4732         hw_stats->vpath_genstats_count1 =
4733         (u32)VXGE_HW_VPATH_GENSTATS_COUNT01_GET_PPIF_VPATH_GENSTATS_COUNT1(
4734                 val64);
4735
4736         val64 = readq(&vp_reg->vpath_genstats_count23);
4737         hw_stats->vpath_genstats_count2 =
4738         (u32)VXGE_HW_VPATH_GENSTATS_COUNT23_GET_PPIF_VPATH_GENSTATS_COUNT2(
4739                 val64);
4740
4741         val64 = readq(&vp_reg->vpath_genstats_count01);
4742         hw_stats->vpath_genstats_count3 =
4743         (u32)VXGE_HW_VPATH_GENSTATS_COUNT23_GET_PPIF_VPATH_GENSTATS_COUNT3(
4744                 val64);
4745
4746         val64 = readq(&vp_reg->vpath_genstats_count4);
4747         hw_stats->vpath_genstats_count4 =
4748         (u32)VXGE_HW_VPATH_GENSTATS_COUNT4_GET_PPIF_VPATH_GENSTATS_COUNT4(
4749                 val64);
4750
4751         val64 = readq(&vp_reg->vpath_genstats_count5);
4752         hw_stats->vpath_genstats_count5 =
4753         (u32)VXGE_HW_VPATH_GENSTATS_COUNT5_GET_PPIF_VPATH_GENSTATS_COUNT5(
4754                 val64);
4755
4756         status = __vxge_hw_vpath_xmac_tx_stats_get(vpath, &hw_stats->tx_stats);
4757         if (status != VXGE_HW_OK)
4758                 goto exit;
4759
4760         status = __vxge_hw_vpath_xmac_rx_stats_get(vpath, &hw_stats->rx_stats);
4761         if (status != VXGE_HW_OK)
4762                 goto exit;
4763
4764         VXGE_HW_VPATH_STATS_PIO_READ(
4765                 VXGE_HW_STATS_VPATH_PROG_EVENT_VNUM0_OFFSET);
4766
4767         hw_stats->prog_event_vnum0 =
4768                         (u32)VXGE_HW_STATS_GET_VPATH_PROG_EVENT_VNUM0(val64);
4769
4770         hw_stats->prog_event_vnum1 =
4771                         (u32)VXGE_HW_STATS_GET_VPATH_PROG_EVENT_VNUM1(val64);
4772
4773         VXGE_HW_VPATH_STATS_PIO_READ(
4774                 VXGE_HW_STATS_VPATH_PROG_EVENT_VNUM2_OFFSET);
4775
4776         hw_stats->prog_event_vnum2 =
4777                         (u32)VXGE_HW_STATS_GET_VPATH_PROG_EVENT_VNUM2(val64);
4778
4779         hw_stats->prog_event_vnum3 =
4780                         (u32)VXGE_HW_STATS_GET_VPATH_PROG_EVENT_VNUM3(val64);
4781
4782         val64 = readq(&vp_reg->rx_multi_cast_stats);
4783         hw_stats->rx_multi_cast_frame_discard =
4784                 (u16)VXGE_HW_RX_MULTI_CAST_STATS_GET_FRAME_DISCARD(val64);
4785
4786         val64 = readq(&vp_reg->rx_frm_transferred);
4787         hw_stats->rx_frm_transferred =
4788                 (u32)VXGE_HW_RX_FRM_TRANSFERRED_GET_RX_FRM_TRANSFERRED(val64);
4789
4790         val64 = readq(&vp_reg->rxd_returned);
4791         hw_stats->rxd_returned =
4792                 (u16)VXGE_HW_RXD_RETURNED_GET_RXD_RETURNED(val64);
4793
4794         val64 = readq(&vp_reg->dbg_stats_rx_mpa);
4795         hw_stats->rx_mpa_len_fail_frms =
4796                 (u16)VXGE_HW_DBG_STATS_GET_RX_MPA_LEN_FAIL_FRMS(val64);
4797         hw_stats->rx_mpa_mrk_fail_frms =
4798                 (u16)VXGE_HW_DBG_STATS_GET_RX_MPA_MRK_FAIL_FRMS(val64);
4799         hw_stats->rx_mpa_crc_fail_frms =
4800                 (u16)VXGE_HW_DBG_STATS_GET_RX_MPA_CRC_FAIL_FRMS(val64);
4801
4802         val64 = readq(&vp_reg->dbg_stats_rx_fau);
4803         hw_stats->rx_permitted_frms =
4804                 (u16)VXGE_HW_DBG_STATS_GET_RX_FAU_RX_PERMITTED_FRMS(val64);
4805         hw_stats->rx_vp_reset_discarded_frms =
4806         (u16)VXGE_HW_DBG_STATS_GET_RX_FAU_RX_VP_RESET_DISCARDED_FRMS(val64);
4807         hw_stats->rx_wol_frms =
4808                 (u16)VXGE_HW_DBG_STATS_GET_RX_FAU_RX_WOL_FRMS(val64);
4809
4810         val64 = readq(&vp_reg->tx_vp_reset_discarded_frms);
4811         hw_stats->tx_vp_reset_discarded_frms =
4812         (u16)VXGE_HW_TX_VP_RESET_DISCARDED_FRMS_GET_TX_VP_RESET_DISCARDED_FRMS(
4813                 val64);
4814 exit:
4815         return status;
4816 }
4817
4818 /*
4819  * __vxge_hw_blockpool_create - Create block pool
4820  */
4821
4822 enum vxge_hw_status
4823 __vxge_hw_blockpool_create(struct __vxge_hw_device *hldev,
4824                            struct __vxge_hw_blockpool *blockpool,
4825                            u32 pool_size,
4826                            u32 pool_max)
4827 {
4828         u32 i;
4829         struct __vxge_hw_blockpool_entry *entry = NULL;
4830         void *memblock;
4831         dma_addr_t dma_addr;
4832         struct pci_dev *dma_handle;
4833         struct pci_dev *acc_handle;
4834         enum vxge_hw_status status = VXGE_HW_OK;
4835
4836         if (blockpool == NULL) {
4837                 status = VXGE_HW_FAIL;
4838                 goto blockpool_create_exit;
4839         }
4840
4841         blockpool->hldev = hldev;
4842         blockpool->block_size = VXGE_HW_BLOCK_SIZE;
4843         blockpool->pool_size = 0;
4844         blockpool->pool_max = pool_max;
4845         blockpool->req_out = 0;
4846
4847         INIT_LIST_HEAD(&blockpool->free_block_list);
4848         INIT_LIST_HEAD(&blockpool->free_entry_list);
4849
4850         for (i = 0; i < pool_size + pool_max; i++) {
4851                 entry = kzalloc(sizeof(struct __vxge_hw_blockpool_entry),
4852                                 GFP_KERNEL);
4853                 if (entry == NULL) {
4854                         __vxge_hw_blockpool_destroy(blockpool);
4855                         status = VXGE_HW_ERR_OUT_OF_MEMORY;
4856                         goto blockpool_create_exit;
4857                 }
4858                 list_add(&entry->item, &blockpool->free_entry_list);
4859         }
4860
4861         for (i = 0; i < pool_size; i++) {
4862
4863                 memblock = vxge_os_dma_malloc(
4864                                 hldev->pdev,
4865                                 VXGE_HW_BLOCK_SIZE,
4866                                 &dma_handle,
4867                                 &acc_handle);
4868
4869                 if (memblock == NULL) {
4870                         __vxge_hw_blockpool_destroy(blockpool);
4871                         status = VXGE_HW_ERR_OUT_OF_MEMORY;
4872                         goto blockpool_create_exit;
4873                 }
4874
4875                 dma_addr = pci_map_single(hldev->pdev, memblock,
4876                                 VXGE_HW_BLOCK_SIZE, PCI_DMA_BIDIRECTIONAL);
4877
4878                 if (unlikely(pci_dma_mapping_error(hldev->pdev,
4879                                 dma_addr))) {
4880
4881                         vxge_os_dma_free(hldev->pdev, memblock, &acc_handle);
4882                         __vxge_hw_blockpool_destroy(blockpool);
4883                         status = VXGE_HW_ERR_OUT_OF_MEMORY;
4884                         goto blockpool_create_exit;
4885                 }
4886
4887                 if (!list_empty(&blockpool->free_entry_list))
4888                         entry = (struct __vxge_hw_blockpool_entry *)
4889                                 list_first_entry(&blockpool->free_entry_list,
4890                                         struct __vxge_hw_blockpool_entry,
4891                                         item);
4892
4893                 if (entry == NULL)
4894                         entry =
4895                             kzalloc(sizeof(struct __vxge_hw_blockpool_entry),
4896                                         GFP_KERNEL);
4897                 if (entry != NULL) {
4898                         list_del(&entry->item);
4899                         entry->length = VXGE_HW_BLOCK_SIZE;
4900                         entry->memblock = memblock;
4901                         entry->dma_addr = dma_addr;
4902                         entry->acc_handle = acc_handle;
4903                         entry->dma_handle = dma_handle;
4904                         list_add(&entry->item,
4905                                           &blockpool->free_block_list);
4906                         blockpool->pool_size++;
4907                 } else {
4908                         __vxge_hw_blockpool_destroy(blockpool);
4909                         status = VXGE_HW_ERR_OUT_OF_MEMORY;
4910                         goto blockpool_create_exit;
4911                 }
4912         }
4913
4914 blockpool_create_exit:
4915         return status;
4916 }
4917
4918 /*
4919  * __vxge_hw_blockpool_destroy - Deallocates the block pool
4920  */
4921
4922 void __vxge_hw_blockpool_destroy(struct __vxge_hw_blockpool *blockpool)
4923 {
4924
4925         struct __vxge_hw_device *hldev;
4926         struct list_head *p, *n;
4927         u16 ret;
4928
4929         if (blockpool == NULL) {
4930                 ret = 1;
4931                 goto exit;
4932         }
4933
4934         hldev = blockpool->hldev;
4935
4936         list_for_each_safe(p, n, &blockpool->free_block_list) {
4937
4938                 pci_unmap_single(hldev->pdev,
4939                         ((struct __vxge_hw_blockpool_entry *)p)->dma_addr,
4940                         ((struct __vxge_hw_blockpool_entry *)p)->length,
4941                         PCI_DMA_BIDIRECTIONAL);
4942
4943                 vxge_os_dma_free(hldev->pdev,
4944                         ((struct __vxge_hw_blockpool_entry *)p)->memblock,
4945                         &((struct __vxge_hw_blockpool_entry *) p)->acc_handle);
4946
4947                 list_del(
4948                         &((struct __vxge_hw_blockpool_entry *)p)->item);
4949                 kfree(p);
4950                 blockpool->pool_size--;
4951         }
4952
4953         list_for_each_safe(p, n, &blockpool->free_entry_list) {
4954                 list_del(
4955                         &((struct __vxge_hw_blockpool_entry *)p)->item);
4956                 kfree((void *)p);
4957         }
4958         ret = 0;
4959 exit:
4960         return;
4961 }
4962
4963 /*
4964  * __vxge_hw_blockpool_blocks_add - Request additional blocks
4965  */
4966 static
4967 void __vxge_hw_blockpool_blocks_add(struct __vxge_hw_blockpool *blockpool)
4968 {
4969         u32 nreq = 0, i;
4970
4971         if ((blockpool->pool_size  +  blockpool->req_out) <
4972                 VXGE_HW_MIN_DMA_BLOCK_POOL_SIZE) {
4973                 nreq = VXGE_HW_INCR_DMA_BLOCK_POOL_SIZE;
4974                 blockpool->req_out += nreq;
4975         }
4976
4977         for (i = 0; i < nreq; i++)
4978                 vxge_os_dma_malloc_async(
4979                         ((struct __vxge_hw_device *)blockpool->hldev)->pdev,
4980                         blockpool->hldev, VXGE_HW_BLOCK_SIZE);
4981 }
4982
4983 /*
4984  * __vxge_hw_blockpool_blocks_remove - Free additional blocks
4985  */
4986 static
4987 void __vxge_hw_blockpool_blocks_remove(struct __vxge_hw_blockpool *blockpool)
4988 {
4989         struct list_head *p, *n;
4990
4991         list_for_each_safe(p, n, &blockpool->free_block_list) {
4992
4993                 if (blockpool->pool_size < blockpool->pool_max)
4994                         break;
4995
4996                 pci_unmap_single(
4997                         ((struct __vxge_hw_device *)blockpool->hldev)->pdev,
4998                         ((struct __vxge_hw_blockpool_entry *)p)->dma_addr,
4999                         ((struct __vxge_hw_blockpool_entry *)p)->length,
5000                         PCI_DMA_BIDIRECTIONAL);
5001
5002                 vxge_os_dma_free(
5003                         ((struct __vxge_hw_device *)blockpool->hldev)->pdev,
5004                         ((struct __vxge_hw_blockpool_entry *)p)->memblock,
5005                         &((struct __vxge_hw_blockpool_entry *)p)->acc_handle);
5006
5007                 list_del(&((struct __vxge_hw_blockpool_entry *)p)->item);
5008
5009                 list_add(p, &blockpool->free_entry_list);
5010
5011                 blockpool->pool_size--;
5012
5013         }
5014 }
5015
5016 /*
5017  * vxge_hw_blockpool_block_add - callback for vxge_os_dma_malloc_async
5018  * Adds a block to block pool
5019  */
5020 void vxge_hw_blockpool_block_add(
5021                         struct __vxge_hw_device *devh,
5022                         void *block_addr,
5023                         u32 length,
5024                         struct pci_dev *dma_h,
5025                         struct pci_dev *acc_handle)
5026 {
5027         struct __vxge_hw_blockpool  *blockpool;
5028         struct __vxge_hw_blockpool_entry  *entry = NULL;
5029         dma_addr_t dma_addr;
5030         enum vxge_hw_status status = VXGE_HW_OK;
5031         u32 req_out;
5032
5033         blockpool = &devh->block_pool;
5034
5035         if (block_addr == NULL) {
5036                 blockpool->req_out--;
5037                 status = VXGE_HW_FAIL;
5038                 goto exit;
5039         }
5040
5041         dma_addr = pci_map_single(devh->pdev, block_addr, length,
5042                                 PCI_DMA_BIDIRECTIONAL);
5043
5044         if (unlikely(pci_dma_mapping_error(devh->pdev, dma_addr))) {
5045
5046                 vxge_os_dma_free(devh->pdev, block_addr, &acc_handle);
5047                 blockpool->req_out--;
5048                 status = VXGE_HW_FAIL;
5049                 goto exit;
5050         }
5051
5052
5053         if (!list_empty(&blockpool->free_entry_list))
5054                 entry = (struct __vxge_hw_blockpool_entry *)
5055                         list_first_entry(&blockpool->free_entry_list,
5056                                 struct __vxge_hw_blockpool_entry,
5057                                 item);
5058
5059         if (entry == NULL)
5060                 entry = (struct __vxge_hw_blockpool_entry *)
5061                         vmalloc(sizeof(struct __vxge_hw_blockpool_entry));
5062         else
5063                 list_del(&entry->item);
5064
5065         if (entry != NULL) {
5066                 entry->length = length;
5067                 entry->memblock = block_addr;
5068                 entry->dma_addr = dma_addr;
5069                 entry->acc_handle = acc_handle;
5070                 entry->dma_handle = dma_h;
5071                 list_add(&entry->item, &blockpool->free_block_list);
5072                 blockpool->pool_size++;
5073                 status = VXGE_HW_OK;
5074         } else
5075                 status = VXGE_HW_ERR_OUT_OF_MEMORY;
5076
5077         blockpool->req_out--;
5078
5079         req_out = blockpool->req_out;
5080 exit:
5081         return;
5082 }
5083
5084 /*
5085  * __vxge_hw_blockpool_malloc - Allocate a memory block from pool
5086  * Allocates a block of memory of given size, either from block pool
5087  * or by calling vxge_os_dma_malloc()
5088  */
5089 void *
5090 __vxge_hw_blockpool_malloc(struct __vxge_hw_device *devh, u32 size,
5091                                 struct vxge_hw_mempool_dma *dma_object)
5092 {
5093         struct __vxge_hw_blockpool_entry *entry = NULL;
5094         struct __vxge_hw_blockpool  *blockpool;
5095         void *memblock = NULL;
5096         enum vxge_hw_status status = VXGE_HW_OK;
5097
5098         blockpool = &devh->block_pool;
5099
5100         if (size != blockpool->block_size) {
5101
5102                 memblock = vxge_os_dma_malloc(devh->pdev, size,
5103                                                 &dma_object->handle,
5104                                                 &dma_object->acc_handle);
5105
5106                 if (memblock == NULL) {
5107                         status = VXGE_HW_ERR_OUT_OF_MEMORY;
5108                         goto exit;
5109                 }
5110
5111                 dma_object->addr = pci_map_single(devh->pdev, memblock, size,
5112                                         PCI_DMA_BIDIRECTIONAL);
5113
5114                 if (unlikely(pci_dma_mapping_error(devh->pdev,
5115                                 dma_object->addr))) {
5116                         vxge_os_dma_free(devh->pdev, memblock,
5117                                 &dma_object->acc_handle);
5118                         status = VXGE_HW_ERR_OUT_OF_MEMORY;
5119                         goto exit;
5120                 }
5121
5122         } else {
5123
5124                 if (!list_empty(&blockpool->free_block_list))
5125                         entry = (struct __vxge_hw_blockpool_entry *)
5126                                 list_first_entry(&blockpool->free_block_list,
5127                                         struct __vxge_hw_blockpool_entry,
5128                                         item);
5129
5130                 if (entry != NULL) {
5131                         list_del(&entry->item);
5132                         dma_object->addr = entry->dma_addr;
5133                         dma_object->handle = entry->dma_handle;
5134                         dma_object->acc_handle = entry->acc_handle;
5135                         memblock = entry->memblock;
5136
5137                         list_add(&entry->item,
5138                                 &blockpool->free_entry_list);
5139                         blockpool->pool_size--;
5140                 }
5141
5142                 if (memblock != NULL)
5143                         __vxge_hw_blockpool_blocks_add(blockpool);
5144         }
5145 exit:
5146         return memblock;
5147 }
5148
5149 /*
5150  * __vxge_hw_blockpool_free - Frees the memory allcoated with
5151                                 __vxge_hw_blockpool_malloc
5152  */
5153 void
5154 __vxge_hw_blockpool_free(struct __vxge_hw_device *devh,
5155                         void *memblock, u32 size,
5156                         struct vxge_hw_mempool_dma *dma_object)
5157 {
5158         struct __vxge_hw_blockpool_entry *entry = NULL;
5159         struct __vxge_hw_blockpool  *blockpool;
5160         enum vxge_hw_status status = VXGE_HW_OK;
5161
5162         blockpool = &devh->block_pool;
5163
5164         if (size != blockpool->block_size) {
5165                 pci_unmap_single(devh->pdev, dma_object->addr, size,
5166                         PCI_DMA_BIDIRECTIONAL);
5167                 vxge_os_dma_free(devh->pdev, memblock, &dma_object->acc_handle);
5168         } else {
5169
5170                 if (!list_empty(&blockpool->free_entry_list))
5171                         entry = (struct __vxge_hw_blockpool_entry *)
5172                                 list_first_entry(&blockpool->free_entry_list,
5173                                         struct __vxge_hw_blockpool_entry,
5174                                         item);
5175
5176                 if (entry == NULL)
5177                         entry = (struct __vxge_hw_blockpool_entry *)
5178                                 vmalloc(sizeof(
5179                                         struct __vxge_hw_blockpool_entry));
5180                 else
5181                         list_del(&entry->item);
5182
5183                 if (entry != NULL) {
5184                         entry->length = size;
5185                         entry->memblock = memblock;
5186                         entry->dma_addr = dma_object->addr;
5187                         entry->acc_handle = dma_object->acc_handle;
5188                         entry->dma_handle = dma_object->handle;
5189                         list_add(&entry->item,
5190                                         &blockpool->free_block_list);
5191                         blockpool->pool_size++;
5192                         status = VXGE_HW_OK;
5193                 } else
5194                         status = VXGE_HW_ERR_OUT_OF_MEMORY;
5195
5196                 if (status == VXGE_HW_OK)
5197                         __vxge_hw_blockpool_blocks_remove(blockpool);
5198         }
5199
5200         return;
5201 }
5202
5203 /*
5204  * __vxge_hw_blockpool_block_allocate - Allocates a block from block pool
5205  * This function allocates a block from block pool or from the system
5206  */
5207 struct __vxge_hw_blockpool_entry *
5208 __vxge_hw_blockpool_block_allocate(struct __vxge_hw_device *devh, u32 size)
5209 {
5210         struct __vxge_hw_blockpool_entry *entry = NULL;
5211         struct __vxge_hw_blockpool  *blockpool;
5212
5213         blockpool = &devh->block_pool;
5214
5215         if (size == blockpool->block_size) {
5216
5217                 if (!list_empty(&blockpool->free_block_list))
5218                         entry = (struct __vxge_hw_blockpool_entry *)
5219                                 list_first_entry(&blockpool->free_block_list,
5220                                         struct __vxge_hw_blockpool_entry,
5221                                         item);
5222
5223                 if (entry != NULL) {
5224                         list_del(&entry->item);
5225                         blockpool->pool_size--;
5226                 }
5227         }
5228
5229         if (entry != NULL)
5230                 __vxge_hw_blockpool_blocks_add(blockpool);
5231
5232         return entry;
5233 }
5234
5235 /*
5236  * __vxge_hw_blockpool_block_free - Frees a block from block pool
5237  * @devh: Hal device
5238  * @entry: Entry of block to be freed
5239  *
5240  * This function frees a block from block pool
5241  */
5242 void
5243 __vxge_hw_blockpool_block_free(struct __vxge_hw_device *devh,
5244                         struct __vxge_hw_blockpool_entry *entry)
5245 {
5246         struct __vxge_hw_blockpool  *blockpool;
5247
5248         blockpool = &devh->block_pool;
5249
5250         if (entry->length == blockpool->block_size) {
5251                 list_add(&entry->item, &blockpool->free_block_list);
5252                 blockpool->pool_size++;
5253         }
5254
5255         __vxge_hw_blockpool_blocks_remove(blockpool);
5256
5257         return;
5258 }