return get_wqe(qp, qp->sq.offset + (n << qp->sq.wqe_shift));
}
+/*
+ * Stamp a SQ WQE so that it is invalid if prefetched by marking the
+ * first four bytes of every 64 byte chunk with 0xffffffff, except for
+ * the very first chunk of the WQE.
+ */
+static void stamp_send_wqe(struct mlx4_ib_qp *qp, int n)
+{
+ u32 *wqe = get_send_wqe(qp, n);
+ int i;
+
+ for (i = 16; i < 1 << (qp->sq.wqe_shift - 2); i += 16)
+ wqe[i] = 0xffffffff;
+}
+
static void mlx4_ib_qp_event(struct mlx4_qp *qp, enum mlx4_event type)
{
struct ib_event event;
case IB_QPT_GSI:
return sizeof (struct mlx4_wqe_ctrl_seg) +
ALIGN(MLX4_IB_UD_HEADER_SIZE +
+ DIV_ROUND_UP(MLX4_IB_UD_HEADER_SIZE,
+ MLX4_INLINE_ALIGN) *
sizeof (struct mlx4_wqe_inline_seg),
sizeof (struct mlx4_wqe_data_seg)) +
ALIGN(4 +
}
}
-static int set_qp_size(struct mlx4_ib_dev *dev, struct ib_qp_cap *cap,
- enum ib_qp_type type, struct mlx4_ib_qp *qp)
+static int set_rq_size(struct mlx4_ib_dev *dev, struct ib_qp_cap *cap,
+ int is_user, int has_srq, struct mlx4_ib_qp *qp)
{
- /* Sanity check QP size before proceeding */
+ /* Sanity check RQ size before proceeding */
+ if (cap->max_recv_wr > dev->dev->caps.max_wqes ||
+ cap->max_recv_sge > dev->dev->caps.max_rq_sg)
+ return -EINVAL;
+
+ if (has_srq) {
+ /* QPs attached to an SRQ should have no RQ */
+ if (cap->max_recv_wr)
+ return -EINVAL;
+
+ qp->rq.wqe_cnt = qp->rq.max_gs = 0;
+ } else {
+ /* HW requires >= 1 RQ entry with >= 1 gather entry */
+ if (is_user && (!cap->max_recv_wr || !cap->max_recv_sge))
+ return -EINVAL;
+
+ qp->rq.wqe_cnt = roundup_pow_of_two(max(1U, cap->max_recv_wr));
+ qp->rq.max_gs = roundup_pow_of_two(max(1U, cap->max_recv_sge));
+ qp->rq.wqe_shift = ilog2(qp->rq.max_gs * sizeof (struct mlx4_wqe_data_seg));
+ }
+
+ cap->max_recv_wr = qp->rq.max_post = qp->rq.wqe_cnt;
+ cap->max_recv_sge = qp->rq.max_gs;
+
+ return 0;
+}
+
+static int set_kernel_sq_size(struct mlx4_ib_dev *dev, struct ib_qp_cap *cap,
+ enum ib_qp_type type, struct mlx4_ib_qp *qp)
+{
+ /* Sanity check SQ size before proceeding */
if (cap->max_send_wr > dev->dev->caps.max_wqes ||
- cap->max_recv_wr > dev->dev->caps.max_wqes ||
cap->max_send_sge > dev->dev->caps.max_sq_sg ||
- cap->max_recv_sge > dev->dev->caps.max_rq_sg ||
cap->max_inline_data + send_wqe_overhead(type) +
sizeof (struct mlx4_wqe_inline_seg) > dev->dev->caps.max_sq_desc_sz)
return -EINVAL;
cap->max_send_sge + 2 > dev->dev->caps.max_sq_sg)
return -EINVAL;
- qp->rq.max = cap->max_recv_wr ? roundup_pow_of_two(cap->max_recv_wr) : 0;
- qp->sq.max = cap->max_send_wr ? roundup_pow_of_two(cap->max_send_wr) : 0;
-
- qp->rq.wqe_shift = ilog2(roundup_pow_of_two(cap->max_recv_sge *
- sizeof (struct mlx4_wqe_data_seg)));
- qp->rq.max_gs = (1 << qp->rq.wqe_shift) / sizeof (struct mlx4_wqe_data_seg);
-
qp->sq.wqe_shift = ilog2(roundup_pow_of_two(max(cap->max_send_sge *
sizeof (struct mlx4_wqe_data_seg),
cap->max_inline_data +
qp->sq.max_gs = ((1 << qp->sq.wqe_shift) - send_wqe_overhead(type)) /
sizeof (struct mlx4_wqe_data_seg);
- qp->buf_size = (qp->rq.max << qp->rq.wqe_shift) +
- (qp->sq.max << qp->sq.wqe_shift);
+ /*
+ * We need to leave 2 KB + 1 WQE of headroom in the SQ to
+ * allow HW to prefetch.
+ */
+ qp->sq_spare_wqes = (2048 >> qp->sq.wqe_shift) + 1;
+ qp->sq.wqe_cnt = roundup_pow_of_two(cap->max_send_wr + qp->sq_spare_wqes);
+
+ qp->buf_size = (qp->rq.wqe_cnt << qp->rq.wqe_shift) +
+ (qp->sq.wqe_cnt << qp->sq.wqe_shift);
if (qp->rq.wqe_shift > qp->sq.wqe_shift) {
qp->rq.offset = 0;
- qp->sq.offset = qp->rq.max << qp->rq.wqe_shift;
+ qp->sq.offset = qp->rq.wqe_cnt << qp->rq.wqe_shift;
} else {
- qp->rq.offset = qp->sq.max << qp->sq.wqe_shift;
+ qp->rq.offset = qp->sq.wqe_cnt << qp->sq.wqe_shift;
qp->sq.offset = 0;
}
- cap->max_send_wr = qp->sq.max;
- cap->max_recv_wr = qp->rq.max;
+ cap->max_send_wr = qp->sq.max_post = qp->sq.wqe_cnt - qp->sq_spare_wqes;
cap->max_send_sge = qp->sq.max_gs;
- cap->max_recv_sge = qp->rq.max_gs;
- cap->max_inline_data = (1 << qp->sq.wqe_shift) - send_wqe_overhead(type) -
- sizeof (struct mlx4_wqe_inline_seg);
+ /* We don't support inline sends for kernel QPs (yet) */
+ cap->max_inline_data = 0;
+
+ return 0;
+}
+
+static int set_user_sq_size(struct mlx4_ib_qp *qp,
+ struct mlx4_ib_create_qp *ucmd)
+{
+ qp->sq.wqe_cnt = 1 << ucmd->log_sq_bb_count;
+ qp->sq.wqe_shift = ucmd->log_sq_stride;
+
+ qp->buf_size = (qp->rq.wqe_cnt << qp->rq.wqe_shift) +
+ (qp->sq.wqe_cnt << qp->sq.wqe_shift);
return 0;
}
struct ib_qp_init_attr *init_attr,
struct ib_udata *udata, int sqpn, struct mlx4_ib_qp *qp)
{
- struct mlx4_wqe_ctrl_seg *ctrl;
int err;
- int i;
mutex_init(&qp->mutex);
spin_lock_init(&qp->sq.lock);
qp->sq.head = 0;
qp->sq.tail = 0;
- err = set_qp_size(dev, &init_attr->cap, init_attr->qp_type, qp);
+ err = set_rq_size(dev, &init_attr->cap, !!pd->uobject, !!init_attr->srq, qp);
if (err)
goto err;
goto err;
}
+ qp->sq_no_prefetch = ucmd.sq_no_prefetch;
+
+ err = set_user_sq_size(qp, &ucmd);
+ if (err)
+ goto err;
+
qp->umem = ib_umem_get(pd->uobject->context, ucmd.buf_addr,
qp->buf_size, 0);
if (IS_ERR(qp->umem)) {
if (err)
goto err_mtt;
- err = mlx4_ib_db_map_user(to_mucontext(pd->uobject->context),
- ucmd.db_addr, &qp->db);
- if (err)
- goto err_mtt;
+ if (!init_attr->srq) {
+ err = mlx4_ib_db_map_user(to_mucontext(pd->uobject->context),
+ ucmd.db_addr, &qp->db);
+ if (err)
+ goto err_mtt;
+ }
} else {
- err = mlx4_ib_db_alloc(dev, &qp->db, 0);
+ qp->sq_no_prefetch = 0;
+
+ err = set_kernel_sq_size(dev, &init_attr->cap, init_attr->qp_type, qp);
if (err)
goto err;
- *qp->db.db = 0;
+ if (!init_attr->srq) {
+ err = mlx4_ib_db_alloc(dev, &qp->db, 0);
+ if (err)
+ goto err;
+
+ *qp->db.db = 0;
+ }
if (mlx4_buf_alloc(dev->dev, qp->buf_size, PAGE_SIZE * 2, &qp->buf)) {
err = -ENOMEM;
if (err)
goto err_mtt;
- for (i = 0; i < qp->sq.max; ++i) {
- ctrl = get_send_wqe(qp, i);
- ctrl->owner_opcode = cpu_to_be32(1 << 31);
- }
-
- qp->sq.wrid = kmalloc(qp->sq.max * sizeof (u64), GFP_KERNEL);
- qp->rq.wrid = kmalloc(qp->rq.max * sizeof (u64), GFP_KERNEL);
+ qp->sq.wrid = kmalloc(qp->sq.wqe_cnt * sizeof (u64), GFP_KERNEL);
+ qp->rq.wrid = kmalloc(qp->rq.wqe_cnt * sizeof (u64), GFP_KERNEL);
if (!qp->sq.wrid || !qp->rq.wrid) {
err = -ENOMEM;
goto err_wrid;
}
-
- /* We don't support inline sends for kernel QPs (yet) */
- init_attr->cap.max_inline_data = 0;
}
err = mlx4_qp_alloc(dev->dev, sqpn, &qp->mqp);
return 0;
err_wrid:
- if (pd->uobject)
+ if (pd->uobject && !init_attr->srq)
mlx4_ib_db_unmap_user(to_mucontext(pd->uobject->context), &qp->db);
else {
kfree(qp->sq.wrid);
mlx4_buf_free(dev->dev, qp->buf_size, &qp->buf);
err_db:
- if (!pd->uobject)
+ if (!pd->uobject && !init_attr->srq)
mlx4_ib_db_free(dev, &qp->db);
err:
mlx4_mtt_cleanup(dev->dev, &qp->mtt);
if (is_user) {
- mlx4_ib_db_unmap_user(to_mucontext(qp->ibqp.uobject->context),
- &qp->db);
+ if (!qp->ibqp.srq)
+ mlx4_ib_db_unmap_user(to_mucontext(qp->ibqp.uobject->context),
+ &qp->db);
ib_umem_release(qp->umem);
} else {
kfree(qp->sq.wrid);
kfree(qp->rq.wrid);
mlx4_buf_free(dev->dev, qp->buf_size, &qp->buf);
- mlx4_ib_db_free(dev, &qp->db);
+ if (!qp->ibqp.srq)
+ mlx4_ib_db_free(dev, &qp->db);
}
}
return 0;
}
-static void init_port(struct mlx4_ib_dev *dev, int port)
-{
- struct mlx4_init_port_param param;
- int err;
-
- memset(¶m, 0, sizeof param);
-
- param.port_width_cap = dev->dev->caps.port_width_cap;
- param.vl_cap = dev->dev->caps.vl_cap;
- param.mtu = ib_mtu_enum_to_int(dev->dev->caps.mtu_cap);
- param.max_gid = dev->dev->caps.gid_table_len;
- param.max_pkey = dev->dev->caps.pkey_table_len;
-
- err = mlx4_INIT_PORT(dev->dev, ¶m, port);
- if (err)
- printk(KERN_WARNING "INIT_PORT failed, return code %d.\n", err);
-}
-
static int to_mlx4_st(enum ib_qp_type type)
{
switch (type) {
path->counter_index = 0xff;
if (ah->ah_flags & IB_AH_GRH) {
- if (ah->grh.sgid_index >= dev->dev->caps.gid_table_len) {
+ if (ah->grh.sgid_index >= dev->dev->caps.gid_table_len[port]) {
printk(KERN_ERR "sgid_index (%u) too large. max is %d\n",
- ah->grh.sgid_index, dev->dev->caps.gid_table_len - 1);
+ ah->grh.sgid_index, dev->dev->caps.gid_table_len[port] - 1);
return -1;
}
context->mtu_msgmax = (attr->path_mtu << 5) | 31;
}
- if (qp->rq.max)
- context->rq_size_stride = ilog2(qp->rq.max) << 3;
+ if (qp->rq.wqe_cnt)
+ context->rq_size_stride = ilog2(qp->rq.wqe_cnt) << 3;
context->rq_size_stride |= qp->rq.wqe_shift - 4;
- if (qp->sq.max)
- context->sq_size_stride = ilog2(qp->sq.max) << 3;
+ if (qp->sq.wqe_cnt)
+ context->sq_size_stride = ilog2(qp->sq.wqe_cnt) << 3;
context->sq_size_stride |= qp->sq.wqe_shift - 4;
+ if (cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT)
+ context->sq_size_stride |= !!qp->sq_no_prefetch << 7;
+
if (qp->ibqp.uobject)
context->usr_page = cpu_to_be32(to_mucontext(ibqp->uobject->context)->uar.index);
else
optpar |= MLX4_QP_OPTPAR_PKEY_INDEX;
}
- if (attr_mask & IB_QP_RNR_RETRY) {
- context->params1 |= cpu_to_be32(attr->rnr_retry << 13);
- optpar |= MLX4_QP_OPTPAR_RNR_RETRY;
- }
-
if (attr_mask & IB_QP_AV) {
if (mlx4_set_path(dev, &attr->ah_attr, &context->pri_path,
attr_mask & IB_QP_PORT ? attr->port_num : qp->port)) {
}
if (attr_mask & IB_QP_ALT_PATH) {
- if (attr->alt_pkey_index >= dev->dev->caps.pkey_table_len)
- return -EINVAL;
-
if (attr->alt_port_num == 0 ||
attr->alt_port_num > dev->dev->caps.num_ports)
return -EINVAL;
+ if (attr->alt_pkey_index >=
+ dev->dev->caps.pkey_table_len[attr->alt_port_num])
+ return -EINVAL;
+
if (mlx4_set_path(dev, &attr->alt_ah_attr, &context->alt_path,
attr->alt_port_num))
return -EINVAL;
context->pd = cpu_to_be32(to_mpd(ibqp->pd)->pdn);
context->params1 = cpu_to_be32(MLX4_IB_ACK_REQ_FREQ << 28);
+
+ if (attr_mask & IB_QP_RNR_RETRY) {
+ context->params1 |= cpu_to_be32(attr->rnr_retry << 13);
+ optpar |= MLX4_QP_OPTPAR_RNR_RETRY;
+ }
+
if (attr_mask & IB_QP_RETRY_CNT) {
context->params1 |= cpu_to_be32(attr->retry_cnt << 16);
optpar |= MLX4_QP_OPTPAR_RETRY_COUNT;
if (ibqp->srq)
context->srqn = cpu_to_be32(1 << 24 | to_msrq(ibqp->srq)->msrq.srqn);
- if (cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT)
+ if (!ibqp->srq && cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT)
context->db_rec_addr = cpu_to_be64(qp->db.dma);
if (cur_state == IB_QPS_INIT &&
else
sqd_event = 0;
+ /*
+ * Before passing a kernel QP to the HW, make sure that the
+ * ownership bits of the send queue are set and the SQ
+ * headroom is stamped so that the hardware doesn't start
+ * processing stale work requests.
+ */
+ if (!ibqp->uobject && cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT) {
+ struct mlx4_wqe_ctrl_seg *ctrl;
+ int i;
+
+ for (i = 0; i < qp->sq.wqe_cnt; ++i) {
+ ctrl = get_send_wqe(qp, i);
+ ctrl->owner_opcode = cpu_to_be32(1 << 31);
+
+ stamp_send_wqe(qp, i);
+ }
+ }
+
err = mlx4_qp_modify(dev->dev, &qp->mtt, to_mlx4_state(cur_state),
to_mlx4_state(new_state), context, optpar,
sqd_event, &qp->mqp);
*/
if (is_qp0(dev, qp)) {
if (cur_state != IB_QPS_RTR && new_state == IB_QPS_RTR)
- init_port(dev, qp->port);
+ if (mlx4_INIT_PORT(dev->dev, qp->port))
+ printk(KERN_WARNING "INIT_PORT failed for port %d\n",
+ qp->port);
if (cur_state != IB_QPS_RESET && cur_state != IB_QPS_ERR &&
(new_state == IB_QPS_RESET || new_state == IB_QPS_ERR))
qp->rq.tail = 0;
qp->sq.head = 0;
qp->sq.tail = 0;
- *qp->db.db = 0;
+ if (!ibqp->srq)
+ *qp->db.db = 0;
}
out:
if (!ib_modify_qp_is_ok(cur_state, new_state, ibqp->qp_type, attr_mask))
goto out;
- if ((attr_mask & IB_QP_PKEY_INDEX) &&
- attr->pkey_index >= dev->dev->caps.pkey_table_len) {
- goto out;
- }
-
if ((attr_mask & IB_QP_PORT) &&
(attr->port_num == 0 || attr->port_num > dev->dev->caps.num_ports)) {
goto out;
}
+ if (attr_mask & IB_QP_PKEY_INDEX) {
+ int p = attr_mask & IB_QP_PORT ? attr->port_num : qp->port;
+ if (attr->pkey_index >= dev->dev->caps.pkey_table_len[p])
+ goto out;
+ }
+
if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC &&
attr->max_rd_atomic > dev->dev->caps.max_qp_init_rdma) {
goto out;
u16 pkey;
int send_size;
int header_size;
+ int spc;
int i;
send_size = 0;
printk("\n");
}
- inl->byte_count = cpu_to_be32(1 << 31 | header_size);
- memcpy(inl + 1, sqp->header_buf, header_size);
+ /*
+ * Inline data segments may not cross a 64 byte boundary. If
+ * our UD header is bigger than the space available up to the
+ * next 64 byte boundary in the WQE, use two inline data
+ * segments to hold the UD header.
+ */
+ spc = MLX4_INLINE_ALIGN -
+ ((unsigned long) (inl + 1) & (MLX4_INLINE_ALIGN - 1));
+ if (header_size <= spc) {
+ inl->byte_count = cpu_to_be32(1 << 31 | header_size);
+ memcpy(inl + 1, sqp->header_buf, header_size);
+ i = 1;
+ } else {
+ inl->byte_count = cpu_to_be32(1 << 31 | spc);
+ memcpy(inl + 1, sqp->header_buf, spc);
- return ALIGN(sizeof (struct mlx4_wqe_inline_seg) + header_size, 16);
+ inl = (void *) (inl + 1) + spc;
+ memcpy(inl + 1, sqp->header_buf + spc, header_size - spc);
+ /*
+ * Need a barrier here to make sure all the data is
+ * visible before the byte_count field is set.
+ * Otherwise the HCA prefetcher could grab the 64-byte
+ * chunk with this inline segment and get a valid (!=
+ * 0xffffffff) byte count but stale data, and end up
+ * generating a packet with bad headers.
+ *
+ * The first inline segment's byte_count field doesn't
+ * need a barrier, because it comes after a
+ * control/MLX segment and therefore is at an offset
+ * of 16 mod 64.
+ */
+ wmb();
+ inl->byte_count = cpu_to_be32(1 << 31 | (header_size - spc));
+ i = 2;
+ }
+
+ return ALIGN(i * sizeof (struct mlx4_wqe_inline_seg) + header_size, 16);
}
static int mlx4_wq_overflow(struct mlx4_ib_wq *wq, int nreq, struct ib_cq *ib_cq)
struct mlx4_ib_cq *cq;
cur = wq->head - wq->tail;
- if (likely(cur + nreq < wq->max))
+ if (likely(cur + nreq < wq->max_post))
return 0;
cq = to_mcq(ib_cq);
cur = wq->head - wq->tail;
spin_unlock(&cq->lock);
- return cur + nreq >= wq->max;
+ return cur + nreq >= wq->max_post;
}
int mlx4_ib_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
goto out;
}
- ctrl = wqe = get_send_wqe(qp, ind & (qp->sq.max - 1));
- qp->sq.wrid[ind & (qp->sq.max - 1)] = wr->wr_id;
+ ctrl = wqe = get_send_wqe(qp, ind & (qp->sq.wqe_cnt - 1));
+ qp->sq.wrid[ind & (qp->sq.wqe_cnt - 1)] = wr->wr_id;
ctrl->srcrb_flags =
(wr->send_flags & IB_SEND_SIGNALED ?
}
ctrl->owner_opcode = mlx4_ib_opcode[wr->opcode] |
- (ind & qp->sq.max ? cpu_to_be32(1 << 31) : 0);
+ (ind & qp->sq.wqe_cnt ? cpu_to_be32(1 << 31) : 0);
+
+ /*
+ * We can improve latency by not stamping the last
+ * send queue WQE until after ringing the doorbell, so
+ * only stamp here if there are still more WQEs to post.
+ */
+ if (wr->next)
+ stamp_send_wqe(qp, (ind + qp->sq_spare_wqes) &
+ (qp->sq.wqe_cnt - 1));
++ind;
}
* and reach the HCA out of order.
*/
mmiowb();
+
+ stamp_send_wqe(qp, (ind + qp->sq_spare_wqes - 1) &
+ (qp->sq.wqe_cnt - 1));
}
spin_unlock_irqrestore(&qp->rq.lock, flags);
spin_lock_irqsave(&qp->rq.lock, flags);
- ind = qp->rq.head & (qp->rq.max - 1);
+ ind = qp->rq.head & (qp->rq.wqe_cnt - 1);
for (nreq = 0; wr; ++nreq, wr = wr->next) {
if (mlx4_wq_overflow(&qp->rq, nreq, qp->ibqp.send_cq)) {
qp->rq.wrid[ind] = wr->wr_id;
- ind = (ind + 1) & (qp->rq.max - 1);
+ ind = (ind + 1) & (qp->rq.wqe_cnt - 1);
}
out:
git://ftp.safe.ca / safe / jmp / linux-2.6 / blobdiff