210 lines
6.4 KiB
C
210 lines
6.4 KiB
C
|
/*
|
||
|
|
* Copyright(c) 2016 Intel Corporation.
|
||
|
|
*
|
||
|
|
* This file is provided under a dual BSD/GPLv2 license. When using or
|
||
|
|
* redistributing this file, you may do so under either license.
|
||
|
|
*
|
||
|
|
* GPL LICENSE SUMMARY
|
||
|
|
*
|
||
|
|
* This program is free software; you can redistribute it and/or modify
|
||
|
|
* it under the terms of version 2 of the GNU General Public License as
|
||
|
|
* published by the Free Software Foundation.
|
||
|
|
*
|
||
|
|
* This program is distributed in the hope that it will be useful, but
|
||
|
|
* WITHOUT ANY WARRANTY; without even the implied warranty of
|
||
|
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
||
|
|
* General Public License for more details.
|
||
|
|
*
|
||
|
|
* BSD LICENSE
|
||
|
|
*
|
||
|
|
* Redistribution and use in source and binary forms, with or without
|
||
|
|
* modification, are permitted provided that the following conditions
|
||
|
|
* are met:
|
||
|
|
*
|
||
|
|
* - Redistributions of source code must retain the above copyright
|
||
|
|
* notice, this list of conditions and the following disclaimer.
|
||
|
|
* - Redistributions in binary form must reproduce the above copyright
|
||
|
|
* notice, this list of conditions and the following disclaimer in
|
||
|
|
* the documentation and/or other materials provided with the
|
||
|
|
* distribution.
|
||
|
|
* - Neither the name of Intel Corporation nor the names of its
|
||
|
|
* contributors may be used to endorse or promote products derived
|
||
|
|
* from this software without specific prior written permission.
|
||
|
|
*
|
||
|
|
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
||
|
|
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
||
|
|
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
||
|
|
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
||
|
|
* OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
||
|
|
* SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
||
|
|
* LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
||
|
|
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
||
|
|
* THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
||
|
|
* (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||
|
|
* OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||
|
|
*
|
||
|
|
*/
|
||
|
|
|
||
|
|
#include <rdma/rdmavt_qp.h>
|
||
|
|
#include <rdma/ib_hdrs.h>
|
||
|
|
|
||
|
|
/*
|
||
|
|
* Convert the AETH credit code into the number of credits.
|
||
|
|
*/
|
||
|
|
static const u16 credit_table[31] = {
|
||
|
|
0, /* 0 */
|
||
|
|
1, /* 1 */
|
||
|
|
2, /* 2 */
|
||
|
|
3, /* 3 */
|
||
|
|
4, /* 4 */
|
||
|
|
6, /* 5 */
|
||
|
|
8, /* 6 */
|
||
|
|
12, /* 7 */
|
||
|
|
16, /* 8 */
|
||
|
|
24, /* 9 */
|
||
|
|
32, /* A */
|
||
|
|
48, /* B */
|
||
|
|
64, /* C */
|
||
|
|
96, /* D */
|
||
|
|
128, /* E */
|
||
|
|
192, /* F */
|
||
|
|
256, /* 10 */
|
||
|
|
384, /* 11 */
|
||
|
|
512, /* 12 */
|
||
|
|
768, /* 13 */
|
||
|
|
1024, /* 14 */
|
||
|
|
1536, /* 15 */
|
||
|
|
2048, /* 16 */
|
||
|
|
3072, /* 17 */
|
||
|
|
4096, /* 18 */
|
||
|
|
6144, /* 19 */
|
||
|
|
8192, /* 1A */
|
||
|
|
12288, /* 1B */
|
||
|
|
16384, /* 1C */
|
||
|
|
24576, /* 1D */
|
||
|
|
32768 /* 1E */
|
||
|
|
};
|
||
|
|
|
||
|
|
/**
|
||
|
|
* rvt_compute_aeth - compute the AETH (syndrome + MSN)
|
||
|
|
* @qp: the queue pair to compute the AETH for
|
||
|
|
*
|
||
|
|
* Returns the AETH.
|
||
|
|
*/
|
||
|
|
__be32 rvt_compute_aeth(struct rvt_qp *qp)
|
||
|
|
{
|
||
|
|
u32 aeth = qp->r_msn & IB_MSN_MASK;
|
||
|
|
|
||
|
|
if (qp->ibqp.srq) {
|
||
|
|
/*
|
||
|
|
* Shared receive queues don't generate credits.
|
||
|
|
* Set the credit field to the invalid value.
|
||
|
|
*/
|
||
|
|
aeth |= IB_AETH_CREDIT_INVAL << IB_AETH_CREDIT_SHIFT;
|
||
|
|
} else {
|
||
|
|
u32 min, max, x;
|
||
|
|
u32 credits;
|
||
|
|
u32 head;
|
||
|
|
u32 tail;
|
||
|
|
|
||
|
|
credits = READ_ONCE(qp->r_rq.kwq->count);
|
||
|
|
if (credits == 0) {
|
||
|
|
/* sanity check pointers before trusting them */
|
||
|
|
if (qp->ip) {
|
||
|
|
head = RDMA_READ_UAPI_ATOMIC(qp->r_rq.wq->head);
|
||
|
|
tail = RDMA_READ_UAPI_ATOMIC(qp->r_rq.wq->tail);
|
||
|
|
} else {
|
||
|
|
head = READ_ONCE(qp->r_rq.kwq->head);
|
||
|
|
tail = READ_ONCE(qp->r_rq.kwq->tail);
|
||
|
|
}
|
||
|
|
if (head >= qp->r_rq.size)
|
||
|
|
head = 0;
|
||
|
|
if (tail >= qp->r_rq.size)
|
||
|
|
tail = 0;
|
||
|
|
/*
|
||
|
|
* Compute the number of credits available (RWQEs).
|
||
|
|
* There is a small chance that the pair of reads are
|
||
|
|
* not atomic, which is OK, since the fuzziness is
|
||
|
|
* resolved as further ACKs go out.
|
||
|
|
*/
|
||
|
|
credits = head - tail;
|
||
|
|
if ((int)credits < 0)
|
||
|
|
credits += qp->r_rq.size;
|
||
|
|
}
|
||
|
|
/*
|
||
|
|
* Binary search the credit table to find the code to
|
||
|
|
* use.
|
||
|
|
*/
|
||
|
|
min = 0;
|
||
|
|
max = 31;
|
||
|
|
for (;;) {
|
||
|
|
x = (min + max) / 2;
|
||
|
|
if (credit_table[x] == credits)
|
||
|
|
break;
|
||
|
|
if (credit_table[x] > credits) {
|
||
|
|
max = x;
|
||
|
|
} else {
|
||
|
|
if (min == x)
|
||
|
|
break;
|
||
|
|
min = x;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
aeth |= x << IB_AETH_CREDIT_SHIFT;
|
||
|
|
}
|
||
|
|
return cpu_to_be32(aeth);
|
||
|
|
}
|
||
|
|
EXPORT_SYMBOL(rvt_compute_aeth);
|
||
|
|
|
||
|
|
/**
|
||
|
|
* rvt_get_credit - flush the send work queue of a QP
|
||
|
|
* @qp: the qp who's send work queue to flush
|
||
|
|
* @aeth: the Acknowledge Extended Transport Header
|
||
|
|
*
|
||
|
|
* The QP s_lock should be held.
|
||
|
|
*/
|
||
|
|
void rvt_get_credit(struct rvt_qp *qp, u32 aeth)
|
||
|
|
{
|
||
|
|
struct rvt_dev_info *rdi = ib_to_rvt(qp->ibqp.device);
|
||
|
|
u32 credit = (aeth >> IB_AETH_CREDIT_SHIFT) & IB_AETH_CREDIT_MASK;
|
||
|
|
|
||
|
|
lockdep_assert_held(&qp->s_lock);
|
||
|
|
/*
|
||
|
|
* If the credit is invalid, we can send
|
||
|
|
* as many packets as we like. Otherwise, we have to
|
||
|
|
* honor the credit field.
|
||
|
|
*/
|
||
|
|
if (credit == IB_AETH_CREDIT_INVAL) {
|
||
|
|
if (!(qp->s_flags & RVT_S_UNLIMITED_CREDIT)) {
|
||
|
|
qp->s_flags |= RVT_S_UNLIMITED_CREDIT;
|
||
|
|
if (qp->s_flags & RVT_S_WAIT_SSN_CREDIT) {
|
||
|
|
qp->s_flags &= ~RVT_S_WAIT_SSN_CREDIT;
|
||
|
|
rdi->driver_f.schedule_send(qp);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
} else if (!(qp->s_flags & RVT_S_UNLIMITED_CREDIT)) {
|
||
|
|
/* Compute new LSN (i.e., MSN + credit) */
|
||
|
|
credit = (aeth + credit_table[credit]) & IB_MSN_MASK;
|
||
|
|
if (rvt_cmp_msn(credit, qp->s_lsn) > 0) {
|
||
|
|
qp->s_lsn = credit;
|
||
|
|
if (qp->s_flags & RVT_S_WAIT_SSN_CREDIT) {
|
||
|
|
qp->s_flags &= ~RVT_S_WAIT_SSN_CREDIT;
|
||
|
|
rdi->driver_f.schedule_send(qp);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
EXPORT_SYMBOL(rvt_get_credit);
|
||
|
|
|
||
|
|
/* rvt_restart_sge - rewind the sge state for a wqe */
|
||
|
|
u32 rvt_restart_sge(struct rvt_sge_state *ss, struct rvt_swqe *wqe, u32 len)
|
||
|
|
{
|
||
|
|
ss->sge = wqe->sg_list[0];
|
||
|
|
ss->sg_list = wqe->sg_list + 1;
|
||
|
|
ss->num_sge = wqe->wr.num_sge;
|
||
|
|
ss->total_len = wqe->length;
|
||
|
|
rvt_skip_sge(ss, len, false);
|
||
|
|
return wqe->length - len;
|
||
|
|
}
|
||
|
|
EXPORT_SYMBOL(rvt_restart_sge);
|
||
|
|
|