aboutsummaryrefslogtreecommitdiff
path: root/drivers/infiniband/hw/erdma
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/infiniband/hw/erdma')
-rw-r--r--drivers/infiniband/hw/erdma/erdma.h2
-rw-r--r--drivers/infiniband/hw/erdma/erdma_cm.h10
-rw-r--r--drivers/infiniband/hw/erdma/erdma_cmdq.c42
-rw-r--r--drivers/infiniband/hw/erdma/erdma_cq.c4
-rw-r--r--drivers/infiniband/hw/erdma/erdma_eq.c9
-rw-r--r--drivers/infiniband/hw/erdma/erdma_hw.h8
-rw-r--r--drivers/infiniband/hw/erdma/erdma_main.c41
-rw-r--r--drivers/infiniband/hw/erdma/erdma_qp.c4
-rw-r--r--drivers/infiniband/hw/erdma/erdma_verbs.c17
-rw-r--r--drivers/infiniband/hw/erdma/erdma_verbs.h2
10 files changed, 71 insertions, 68 deletions
diff --git a/drivers/infiniband/hw/erdma/erdma.h b/drivers/infiniband/hw/erdma/erdma.h
index 3d8c11aa23a2..e819e4032490 100644
--- a/drivers/infiniband/hw/erdma/erdma.h
+++ b/drivers/infiniband/hw/erdma/erdma.h
@@ -32,7 +32,7 @@ struct erdma_eq {
atomic64_t event_num;
atomic64_t notify_num;
- u64 __iomem *db_addr;
+ void __iomem *db;
u64 *db_record;
};
diff --git a/drivers/infiniband/hw/erdma/erdma_cm.h b/drivers/infiniband/hw/erdma/erdma_cm.h
index 8a3f998fec9b..a26d80770188 100644
--- a/drivers/infiniband/hw/erdma/erdma_cm.h
+++ b/drivers/infiniband/hw/erdma/erdma_cm.h
@@ -33,11 +33,11 @@ struct mpa_rr_params {
* MPA request/response Hdr bits & fields
*/
enum {
- MPA_RR_FLAG_MARKERS = __cpu_to_be16(0x8000),
- MPA_RR_FLAG_CRC = __cpu_to_be16(0x4000),
- MPA_RR_FLAG_REJECT = __cpu_to_be16(0x2000),
- MPA_RR_RESERVED = __cpu_to_be16(0x1f00),
- MPA_RR_MASK_REVISION = __cpu_to_be16(0x00ff)
+ MPA_RR_FLAG_MARKERS = cpu_to_be16(0x8000),
+ MPA_RR_FLAG_CRC = cpu_to_be16(0x4000),
+ MPA_RR_FLAG_REJECT = cpu_to_be16(0x2000),
+ MPA_RR_RESERVED = cpu_to_be16(0x1f00),
+ MPA_RR_MASK_REVISION = cpu_to_be16(0x00ff)
};
/*
diff --git a/drivers/infiniband/hw/erdma/erdma_cmdq.c b/drivers/infiniband/hw/erdma/erdma_cmdq.c
index 6ebfa6989b11..a151a7bdd504 100644
--- a/drivers/infiniband/hw/erdma/erdma_cmdq.c
+++ b/drivers/infiniband/hw/erdma/erdma_cmdq.c
@@ -166,8 +166,7 @@ static int erdma_cmdq_eq_init(struct erdma_dev *dev)
spin_lock_init(&eq->lock);
atomic64_set(&eq->event_num, 0);
- eq->db_addr =
- (u64 __iomem *)(dev->func_bar + ERDMA_REGS_CEQ_DB_BASE_REG);
+ eq->db = dev->func_bar + ERDMA_REGS_CEQ_DB_BASE_REG;
eq->db_record = (u64 *)(eq->qbuf + buf_size);
erdma_reg_write32(dev, ERDMA_REGS_CMDQ_EQ_ADDR_H_REG,
@@ -183,9 +182,8 @@ static int erdma_cmdq_eq_init(struct erdma_dev *dev)
int erdma_cmdq_init(struct erdma_dev *dev)
{
- int err, i;
struct erdma_cmdq *cmdq = &dev->cmdq;
- u32 sts, ctrl;
+ int err;
cmdq->max_outstandings = ERDMA_CMDQ_MAX_OUTSTANDING;
cmdq->use_event = false;
@@ -208,34 +206,10 @@ int erdma_cmdq_init(struct erdma_dev *dev)
if (err)
goto err_destroy_cq;
- ctrl = FIELD_PREP(ERDMA_REG_DEV_CTRL_INIT_MASK, 1);
- erdma_reg_write32(dev, ERDMA_REGS_DEV_CTRL_REG, ctrl);
-
- for (i = 0; i < ERDMA_WAIT_DEV_DONE_CNT; i++) {
- sts = erdma_reg_read32_filed(dev, ERDMA_REGS_DEV_ST_REG,
- ERDMA_REG_DEV_ST_INIT_DONE_MASK);
- if (sts)
- break;
-
- msleep(ERDMA_REG_ACCESS_WAIT_MS);
- }
-
- if (i == ERDMA_WAIT_DEV_DONE_CNT) {
- dev_err(&dev->pdev->dev, "wait init done failed.\n");
- err = -ETIMEDOUT;
- goto err_destroy_eq;
- }
-
set_bit(ERDMA_CMDQ_STATE_OK_BIT, &cmdq->state);
return 0;
-err_destroy_eq:
- dma_free_coherent(&dev->pdev->dev,
- (cmdq->eq.depth << EQE_SHIFT) +
- ERDMA_EXTRA_BUFFER_SIZE,
- cmdq->eq.qbuf, cmdq->eq.qbuf_dma_addr);
-
err_destroy_cq:
dma_free_coherent(&dev->pdev->dev,
(cmdq->cq.depth << CQE_SHIFT) +
@@ -283,7 +257,7 @@ static void *get_next_valid_cmdq_cqe(struct erdma_cmdq *cmdq)
__be32 *cqe = get_queue_entry(cmdq->cq.qbuf, cmdq->cq.ci,
cmdq->cq.depth, CQE_SHIFT);
u32 owner = FIELD_GET(ERDMA_CQE_HDR_OWNER_MASK,
- __be32_to_cpu(READ_ONCE(*cqe)));
+ be32_to_cpu(READ_ONCE(*cqe)));
return owner ^ !!(cmdq->cq.ci & cmdq->cq.depth) ? cqe : NULL;
}
@@ -319,7 +293,6 @@ static int erdma_poll_single_cmd_completion(struct erdma_cmdq *cmdq)
__be32 *cqe;
u16 ctx_id;
u64 *sqe;
- int i;
cqe = get_next_valid_cmdq_cqe(cmdq);
if (!cqe)
@@ -328,8 +301,8 @@ static int erdma_poll_single_cmd_completion(struct erdma_cmdq *cmdq)
cmdq->cq.ci++;
dma_rmb();
- hdr0 = __be32_to_cpu(*cqe);
- sqe_idx = __be32_to_cpu(*(cqe + 1));
+ hdr0 = be32_to_cpu(*cqe);
+ sqe_idx = be32_to_cpu(*(cqe + 1));
sqe = get_queue_entry(cmdq->sq.qbuf, sqe_idx, cmdq->sq.depth,
SQEBB_SHIFT);
@@ -341,9 +314,8 @@ static int erdma_poll_single_cmd_completion(struct erdma_cmdq *cmdq)
comp_wait->cmd_status = ERDMA_CMD_STATUS_FINISHED;
comp_wait->comp_status = FIELD_GET(ERDMA_CQE_HDR_SYNDROME_MASK, hdr0);
cmdq->sq.ci += cmdq->sq.wqebb_cnt;
-
- for (i = 0; i < 4; i++)
- comp_wait->comp_data[i] = __be32_to_cpu(*(cqe + 2 + i));
+ /* Copy 16B comp data after cqe hdr to outer */
+ be32_to_cpu_array(comp_wait->comp_data, cqe + 2, 4);
if (cmdq->use_event)
complete(&comp_wait->wait_event);
diff --git a/drivers/infiniband/hw/erdma/erdma_cq.c b/drivers/infiniband/hw/erdma/erdma_cq.c
index cabd8678b355..c1cb5568eab2 100644
--- a/drivers/infiniband/hw/erdma/erdma_cq.c
+++ b/drivers/infiniband/hw/erdma/erdma_cq.c
@@ -11,7 +11,7 @@ static void *get_next_valid_cqe(struct erdma_cq *cq)
__be32 *cqe = get_queue_entry(cq->kern_cq.qbuf, cq->kern_cq.ci,
cq->depth, CQE_SHIFT);
u32 owner = FIELD_GET(ERDMA_CQE_HDR_OWNER_MASK,
- __be32_to_cpu(READ_ONCE(*cqe)));
+ be32_to_cpu(READ_ONCE(*cqe)));
return owner ^ !!(cq->kern_cq.ci & cq->depth) ? cqe : NULL;
}
@@ -65,7 +65,7 @@ static const enum ib_wc_opcode wc_mapping_table[ERDMA_NUM_OPCODES] = {
[ERDMA_OP_LOCAL_INV] = IB_WC_LOCAL_INV,
[ERDMA_OP_READ_WITH_INV] = IB_WC_RDMA_READ,
[ERDMA_OP_ATOMIC_CAS] = IB_WC_COMP_SWAP,
- [ERDMA_OP_ATOMIC_FAD] = IB_WC_FETCH_ADD,
+ [ERDMA_OP_ATOMIC_FAA] = IB_WC_FETCH_ADD,
};
static const struct {
diff --git a/drivers/infiniband/hw/erdma/erdma_eq.c b/drivers/infiniband/hw/erdma/erdma_eq.c
index ed54130d924b..ea47cb21fdb8 100644
--- a/drivers/infiniband/hw/erdma/erdma_eq.c
+++ b/drivers/infiniband/hw/erdma/erdma_eq.c
@@ -14,7 +14,7 @@ void notify_eq(struct erdma_eq *eq)
FIELD_PREP(ERDMA_EQDB_ARM_MASK, 1);
*eq->db_record = db_data;
- writeq(db_data, eq->db_addr);
+ writeq(db_data, eq->db);
atomic64_inc(&eq->notify_num);
}
@@ -98,7 +98,7 @@ int erdma_aeq_init(struct erdma_dev *dev)
atomic64_set(&eq->event_num, 0);
atomic64_set(&eq->notify_num, 0);
- eq->db_addr = (u64 __iomem *)(dev->func_bar + ERDMA_REGS_AEQ_DB_REG);
+ eq->db = dev->func_bar + ERDMA_REGS_AEQ_DB_REG;
eq->db_record = (u64 *)(eq->qbuf + buf_size);
erdma_reg_write32(dev, ERDMA_REGS_AEQ_ADDR_H_REG,
@@ -243,9 +243,8 @@ static int erdma_ceq_init_one(struct erdma_dev *dev, u16 ceqn)
atomic64_set(&eq->notify_num, 0);
eq->depth = ERDMA_DEFAULT_EQ_DEPTH;
- eq->db_addr =
- (u64 __iomem *)(dev->func_bar + ERDMA_REGS_CEQ_DB_BASE_REG +
- (ceqn + 1) * ERDMA_DB_SIZE);
+ eq->db = dev->func_bar + ERDMA_REGS_CEQ_DB_BASE_REG +
+ (ceqn + 1) * ERDMA_DB_SIZE;
eq->db_record = (u64 *)(eq->qbuf + buf_size);
eq->ci = 0;
dev->ceqs[ceqn].dev = dev;
diff --git a/drivers/infiniband/hw/erdma/erdma_hw.h b/drivers/infiniband/hw/erdma/erdma_hw.h
index 4c38d99c73f1..76ce2856be28 100644
--- a/drivers/infiniband/hw/erdma/erdma_hw.h
+++ b/drivers/infiniband/hw/erdma/erdma_hw.h
@@ -112,6 +112,10 @@
#define ERDMA_PAGE_SIZE_SUPPORT 0x7FFFF000
+/* Hardware page size definition */
+#define ERDMA_HW_PAGE_SHIFT 12
+#define ERDMA_HW_PAGE_SIZE 4096
+
/* WQE related. */
#define EQE_SIZE 16
#define EQE_SHIFT 4
@@ -441,7 +445,7 @@ struct erdma_reg_mr_sqe {
};
/* EQ related. */
-#define ERDMA_DEFAULT_EQ_DEPTH 256
+#define ERDMA_DEFAULT_EQ_DEPTH 4096
/* ceqe */
#define ERDMA_CEQE_HDR_DB_MASK BIT_ULL(63)
@@ -491,7 +495,7 @@ enum erdma_opcode {
ERDMA_OP_LOCAL_INV = 15,
ERDMA_OP_READ_WITH_INV = 16,
ERDMA_OP_ATOMIC_CAS = 17,
- ERDMA_OP_ATOMIC_FAD = 18,
+ ERDMA_OP_ATOMIC_FAA = 18,
ERDMA_NUM_OPCODES = 19,
ERDMA_OP_INVALID = ERDMA_NUM_OPCODES + 1
};
diff --git a/drivers/infiniband/hw/erdma/erdma_main.c b/drivers/infiniband/hw/erdma/erdma_main.c
index 5dc31e5df5cb..7c74abeee864 100644
--- a/drivers/infiniband/hw/erdma/erdma_main.c
+++ b/drivers/infiniband/hw/erdma/erdma_main.c
@@ -56,7 +56,7 @@ done:
static int erdma_enum_and_get_netdev(struct erdma_dev *dev)
{
struct net_device *netdev;
- int ret = -ENODEV;
+ int ret = -EPROBE_DEFER;
/* Already binded to a net_device, so we skip. */
if (dev->netdev)
@@ -211,13 +211,36 @@ static int erdma_device_init(struct erdma_dev *dev, struct pci_dev *pdev)
return 0;
}
-static void erdma_device_uninit(struct erdma_dev *dev)
+static void erdma_hw_reset(struct erdma_dev *dev)
{
u32 ctrl = FIELD_PREP(ERDMA_REG_DEV_CTRL_RESET_MASK, 1);
erdma_reg_write32(dev, ERDMA_REGS_DEV_CTRL_REG, ctrl);
}
+static int erdma_wait_hw_init_done(struct erdma_dev *dev)
+{
+ int i;
+
+ erdma_reg_write32(dev, ERDMA_REGS_DEV_CTRL_REG,
+ FIELD_PREP(ERDMA_REG_DEV_CTRL_INIT_MASK, 1));
+
+ for (i = 0; i < ERDMA_WAIT_DEV_DONE_CNT; i++) {
+ if (erdma_reg_read32_filed(dev, ERDMA_REGS_DEV_ST_REG,
+ ERDMA_REG_DEV_ST_INIT_DONE_MASK))
+ break;
+
+ msleep(ERDMA_REG_ACCESS_WAIT_MS);
+ }
+
+ if (i == ERDMA_WAIT_DEV_DONE_CNT) {
+ dev_err(&dev->pdev->dev, "wait init done failed.\n");
+ return -ETIMEDOUT;
+ }
+
+ return 0;
+}
+
static const struct pci_device_id erdma_pci_tbl[] = {
{ PCI_DEVICE(PCI_VENDOR_ID_ALIBABA, 0x107f) },
{}
@@ -293,16 +316,22 @@ static int erdma_probe_dev(struct pci_dev *pdev)
if (err)
goto err_uninit_aeq;
- err = erdma_ceqs_init(dev);
+ err = erdma_wait_hw_init_done(dev);
if (err)
goto err_uninit_cmdq;
+ err = erdma_ceqs_init(dev);
+ if (err)
+ goto err_reset_hw;
+
erdma_finish_cmdq_init(dev);
return 0;
+err_reset_hw:
+ erdma_hw_reset(dev);
+
err_uninit_cmdq:
- erdma_device_uninit(dev);
erdma_cmdq_destroy(dev);
err_uninit_aeq:
@@ -334,9 +363,7 @@ static void erdma_remove_dev(struct pci_dev *pdev)
struct erdma_dev *dev = pci_get_drvdata(pdev);
erdma_ceqs_uninit(dev);
-
- erdma_device_uninit(dev);
-
+ erdma_hw_reset(dev);
erdma_cmdq_destroy(dev);
erdma_aeq_destroy(dev);
erdma_comm_irq_uninit(dev);
diff --git a/drivers/infiniband/hw/erdma/erdma_qp.c b/drivers/infiniband/hw/erdma/erdma_qp.c
index d088d6bef431..44923c51a01b 100644
--- a/drivers/infiniband/hw/erdma/erdma_qp.c
+++ b/drivers/infiniband/hw/erdma/erdma_qp.c
@@ -405,7 +405,7 @@ static int erdma_push_one_sqe(struct erdma_qp *qp, u16 *pi,
FIELD_PREP(ERDMA_SQE_MR_MTT_CNT_MASK,
mr->mem.mtt_nents);
- if (mr->mem.mtt_nents < ERDMA_MAX_INLINE_MTT_ENTRIES) {
+ if (mr->mem.mtt_nents <= ERDMA_MAX_INLINE_MTT_ENTRIES) {
attrs |= FIELD_PREP(ERDMA_SQE_MR_MTT_TYPE_MASK, 0);
/* Copy SGLs to SQE content to accelerate */
memcpy(get_queue_entry(qp->kern_qp.sq_buf, idx + 1,
@@ -439,7 +439,7 @@ static int erdma_push_one_sqe(struct erdma_qp *qp, u16 *pi,
cpu_to_le64(atomic_wr(send_wr)->compare_add);
} else {
wqe_hdr |= FIELD_PREP(ERDMA_SQE_HDR_OPCODE_MASK,
- ERDMA_OP_ATOMIC_FAD);
+ ERDMA_OP_ATOMIC_FAA);
atomic_sqe->fetchadd_swap_data =
cpu_to_le64(atomic_wr(send_wr)->compare_add);
}
diff --git a/drivers/infiniband/hw/erdma/erdma_verbs.c b/drivers/infiniband/hw/erdma/erdma_verbs.c
index 9c30d78730aa..83e1b0d55977 100644
--- a/drivers/infiniband/hw/erdma/erdma_verbs.c
+++ b/drivers/infiniband/hw/erdma/erdma_verbs.c
@@ -38,7 +38,7 @@ static int create_qp_cmd(struct erdma_dev *dev, struct erdma_qp *qp)
FIELD_PREP(ERDMA_CMD_CREATE_QP_PD_MASK, pd->pdn);
if (rdma_is_kernel_res(&qp->ibqp.res)) {
- u32 pgsz_range = ilog2(SZ_1M) - PAGE_SHIFT;
+ u32 pgsz_range = ilog2(SZ_1M) - ERDMA_HW_PAGE_SHIFT;
req.sq_cqn_mtt_cfg =
FIELD_PREP(ERDMA_CMD_CREATE_QP_PAGE_SIZE_MASK,
@@ -66,13 +66,13 @@ static int create_qp_cmd(struct erdma_dev *dev, struct erdma_qp *qp)
user_qp = &qp->user_qp;
req.sq_cqn_mtt_cfg = FIELD_PREP(
ERDMA_CMD_CREATE_QP_PAGE_SIZE_MASK,
- ilog2(user_qp->sq_mtt.page_size) - PAGE_SHIFT);
+ ilog2(user_qp->sq_mtt.page_size) - ERDMA_HW_PAGE_SHIFT);
req.sq_cqn_mtt_cfg |=
FIELD_PREP(ERDMA_CMD_CREATE_QP_CQN_MASK, qp->scq->cqn);
req.rq_cqn_mtt_cfg = FIELD_PREP(
ERDMA_CMD_CREATE_QP_PAGE_SIZE_MASK,
- ilog2(user_qp->rq_mtt.page_size) - PAGE_SHIFT);
+ ilog2(user_qp->rq_mtt.page_size) - ERDMA_HW_PAGE_SHIFT);
req.rq_cqn_mtt_cfg |=
FIELD_PREP(ERDMA_CMD_CREATE_QP_CQN_MASK, qp->rcq->cqn);
@@ -162,7 +162,7 @@ static int create_cq_cmd(struct erdma_dev *dev, struct erdma_cq *cq)
if (rdma_is_kernel_res(&cq->ibcq.res)) {
page_size = SZ_32M;
req.cfg0 |= FIELD_PREP(ERDMA_CMD_CREATE_CQ_PAGESIZE_MASK,
- ilog2(page_size) - PAGE_SHIFT);
+ ilog2(page_size) - ERDMA_HW_PAGE_SHIFT);
req.qbuf_addr_l = lower_32_bits(cq->kern_cq.qbuf_dma_addr);
req.qbuf_addr_h = upper_32_bits(cq->kern_cq.qbuf_dma_addr);
@@ -175,8 +175,9 @@ static int create_cq_cmd(struct erdma_dev *dev, struct erdma_cq *cq)
cq->kern_cq.qbuf_dma_addr + (cq->depth << CQE_SHIFT);
} else {
mtt = &cq->user_cq.qbuf_mtt;
- req.cfg0 |= FIELD_PREP(ERDMA_CMD_CREATE_CQ_PAGESIZE_MASK,
- ilog2(mtt->page_size) - PAGE_SHIFT);
+ req.cfg0 |=
+ FIELD_PREP(ERDMA_CMD_CREATE_CQ_PAGESIZE_MASK,
+ ilog2(mtt->page_size) - ERDMA_HW_PAGE_SHIFT);
if (mtt->mtt_nents == 1) {
req.qbuf_addr_l = lower_32_bits(*(u64 *)mtt->mtt_buf);
req.qbuf_addr_h = upper_32_bits(*(u64 *)mtt->mtt_buf);
@@ -636,7 +637,7 @@ static int init_user_qp(struct erdma_qp *qp, struct erdma_ucontext *uctx,
u32 rq_offset;
int ret;
- if (len < (PAGE_ALIGN(qp->attrs.sq_size * SQEBB_SIZE) +
+ if (len < (ALIGN(qp->attrs.sq_size * SQEBB_SIZE, ERDMA_HW_PAGE_SIZE) +
qp->attrs.rq_size * RQE_SIZE))
return -EINVAL;
@@ -646,7 +647,7 @@ static int init_user_qp(struct erdma_qp *qp, struct erdma_ucontext *uctx,
if (ret)
return ret;
- rq_offset = PAGE_ALIGN(qp->attrs.sq_size << SQEBB_SHIFT);
+ rq_offset = ALIGN(qp->attrs.sq_size << SQEBB_SHIFT, ERDMA_HW_PAGE_SIZE);
qp->user_qp.rq_offset = rq_offset;
ret = get_mtt_entries(qp->dev, &qp->user_qp.rq_mtt, va + rq_offset,
diff --git a/drivers/infiniband/hw/erdma/erdma_verbs.h b/drivers/infiniband/hw/erdma/erdma_verbs.h
index e0a993bc032a..131cf5f40982 100644
--- a/drivers/infiniband/hw/erdma/erdma_verbs.h
+++ b/drivers/infiniband/hw/erdma/erdma_verbs.h
@@ -11,7 +11,7 @@
/* RDMA Capability. */
#define ERDMA_MAX_PD (128 * 1024)
-#define ERDMA_MAX_SEND_WR 4096
+#define ERDMA_MAX_SEND_WR 8192
#define ERDMA_MAX_ORD 128
#define ERDMA_MAX_IRD 128
#define ERDMA_MAX_SGE_RD 1
Powered by cgit, Gruvboxed by Blaster4385.