diff options
| author | Linus Torvalds <torvalds@linux-foundation.org> | 2019-07-15 20:38:15 -0700 |
|---|---|---|
| committer | Linus Torvalds <torvalds@linux-foundation.org> | 2019-07-15 20:38:15 -0700 |
| commit | 2a3c389a0fde49b241430df806a34276568cfb29 (patch) | |
| tree | 9cf35829317e8cc2aaffc4341fb824dad63fce02 /drivers/infiniband/hw | |
| parent | 8de262531f5fbb7458463224a7587429800c24bf (diff) | |
| parent | 0b043644c0ca601cb19943a81aa1f1455dbe9461 (diff) | |
Merge tag 'for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/rdma/rdma
Pull rdma updates from Jason Gunthorpe:
"A smaller cycle this time. Notably we see another new driver, 'Soft
iWarp', and the deletion of an ancient unused driver for nes.
- Revise and simplify the signature offload RDMA MR APIs
- More progress on hoisting object allocation boiler plate code out
of the drivers
- Driver bug fixes and revisions for hns, hfi1, efa, cxgb4, qib,
i40iw
- Tree wide cleanups: struct_size, put_user_page, xarray, rst doc
conversion
- Removal of obsolete ib_ucm chardev and nes driver
- netlink based discovery of chardevs and autoloading of the modules
providing them
- Move more of the rdamvt/hfi1 uapi to include/uapi/rdma
- New driver 'siw' for software based iWarp running on top of netdev,
much like rxe's software RoCE.
- mlx5 feature to report events in their raw devx format to userspace
- Expose per-object counters through rdma tool
- Adaptive interrupt moderation for RDMA (DIM), sharing the DIM core
from netdev"
* tag 'for-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/rdma/rdma: (194 commits)
RMDA/siw: Require a 64 bit arch
RDMA/siw: Mark expected switch fall-throughs
RDMA/core: Fix -Wunused-const-variable warnings
rdma/siw: Remove set but not used variable 's'
rdma/siw: Add missing dependencies on LIBCRC32C and DMA_VIRT_OPS
RDMA/siw: Add missing rtnl_lock around access to ifa
rdma/siw: Use proper enumerated type in map_cqe_status
RDMA/siw: Remove unnecessary kthread create/destroy printouts
IB/rdmavt: Fix variable shadowing issue in rvt_create_cq
RDMA/core: Fix race when resolving IP address
RDMA/core: Make rdma_counter.h compile stand alone
IB/core: Work on the caller socket net namespace in nldev_newlink()
RDMA/rxe: Fill in wc byte_len with IB_WC_RECV_RDMA_WITH_IMM
RDMA/mlx5: Set RDMA DIM to be enabled by default
RDMA/nldev: Added configuration of RDMA dynamic interrupt moderation to netlink
RDMA/core: Provide RDMA DIM support for ULPs
linux/dim: Implement RDMA adaptive moderation (DIM)
IB/mlx5: Report correctly tag matching rendezvous capability
docs: infiniband: add it to the driver-api bookset
IB/mlx5: Implement VHCA tunnel mechanism in DEVX
...
Diffstat (limited to 'drivers/infiniband/hw')
115 files changed, 4131 insertions, 21746 deletions
diff --git a/drivers/infiniband/hw/Makefile b/drivers/infiniband/hw/Makefile index 77094be1b262..433fca59febd 100644 --- a/drivers/infiniband/hw/Makefile +++ b/drivers/infiniband/hw/Makefile @@ -7,7 +7,6 @@ obj-$(CONFIG_INFINIBAND_EFA) += efa/ obj-$(CONFIG_INFINIBAND_I40IW) += i40iw/ obj-$(CONFIG_MLX4_INFINIBAND) += mlx4/ obj-$(CONFIG_MLX5_INFINIBAND) += mlx5/ -obj-$(CONFIG_INFINIBAND_NES) += nes/ obj-$(CONFIG_INFINIBAND_OCRDMA) += ocrdma/ obj-$(CONFIG_INFINIBAND_VMWARE_PVRDMA) += vmw_pvrdma/ obj-$(CONFIG_INFINIBAND_USNIC) += usnic/ diff --git a/drivers/infiniband/hw/bnxt_re/ib_verbs.c b/drivers/infiniband/hw/bnxt_re/ib_verbs.c index 2c3685faa57a..a91653aabf38 100644 --- a/drivers/infiniband/hw/bnxt_re/ib_verbs.c +++ b/drivers/infiniband/hw/bnxt_re/ib_verbs.c @@ -805,10 +805,8 @@ int bnxt_re_destroy_qp(struct ib_qp *ib_qp, struct ib_udata *udata) rdev->sqp_ah = NULL; } - if (!IS_ERR_OR_NULL(qp->rumem)) - ib_umem_release(qp->rumem); - if (!IS_ERR_OR_NULL(qp->sumem)) - ib_umem_release(qp->sumem); + ib_umem_release(qp->rumem); + ib_umem_release(qp->sumem); mutex_lock(&rdev->qp_lock); list_del(&qp->list); @@ -1201,12 +1199,8 @@ struct ib_qp *bnxt_re_create_qp(struct ib_pd *ib_pd, qp_destroy: bnxt_qplib_destroy_qp(&rdev->qplib_res, &qp->qplib_qp); free_umem: - if (udata) { - if (qp->rumem) - ib_umem_release(qp->rumem); - if (qp->sumem) - ib_umem_release(qp->sumem); - } + ib_umem_release(qp->rumem); + ib_umem_release(qp->sumem); fail: kfree(qp); return ERR_PTR(rc); @@ -1302,8 +1296,7 @@ void bnxt_re_destroy_srq(struct ib_srq *ib_srq, struct ib_udata *udata) if (qplib_srq->cq) nq = qplib_srq->cq->nq; bnxt_qplib_destroy_srq(&rdev->qplib_res, qplib_srq); - if (srq->umem) - ib_umem_release(srq->umem); + ib_umem_release(srq->umem); atomic_dec(&rdev->srq_count); if (nq) nq->budget--; @@ -1412,8 +1405,7 @@ int bnxt_re_create_srq(struct ib_srq *ib_srq, return 0; fail: - if (srq->umem) - ib_umem_release(srq->umem); + ib_umem_release(srq->umem); exit: return rc; } @@ -2517,9 +2509,8 @@ int bnxt_re_post_recv(struct ib_qp *ib_qp, const struct ib_recv_wr *wr, } /* Completion Queues */ -int bnxt_re_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata) +void bnxt_re_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata) { - int rc; struct bnxt_re_cq *cq; struct bnxt_qplib_nq *nq; struct bnxt_re_dev *rdev; @@ -2528,29 +2519,20 @@ int bnxt_re_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata) rdev = cq->rdev; nq = cq->qplib_cq.nq; - rc = bnxt_qplib_destroy_cq(&rdev->qplib_res, &cq->qplib_cq); - if (rc) { - dev_err(rdev_to_dev(rdev), "Failed to destroy HW CQ"); - return rc; - } - if (!IS_ERR_OR_NULL(cq->umem)) - ib_umem_release(cq->umem); + bnxt_qplib_destroy_cq(&rdev->qplib_res, &cq->qplib_cq); + ib_umem_release(cq->umem); atomic_dec(&rdev->cq_count); nq->budget--; kfree(cq->cql); - kfree(cq); - - return 0; } -struct ib_cq *bnxt_re_create_cq(struct ib_device *ibdev, - const struct ib_cq_init_attr *attr, - struct ib_udata *udata) +int bnxt_re_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, + struct ib_udata *udata) { - struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibdev, ibdev); + struct bnxt_re_dev *rdev = to_bnxt_re_dev(ibcq->device, ibdev); struct bnxt_qplib_dev_attr *dev_attr = &rdev->dev_attr; - struct bnxt_re_cq *cq = NULL; + struct bnxt_re_cq *cq = container_of(ibcq, struct bnxt_re_cq, ib_cq); int rc, entries; int cqe = attr->cqe; struct bnxt_qplib_nq *nq = NULL; @@ -2559,11 +2541,8 @@ struct ib_cq *bnxt_re_create_cq(struct ib_device *ibdev, /* Validate CQ fields */ if (cqe < 1 || cqe > dev_attr->max_cq_wqes) { dev_err(rdev_to_dev(rdev), "Failed to create CQ -max exceeded"); - return ERR_PTR(-EINVAL); + return -EINVAL; } - cq = kzalloc(sizeof(*cq), GFP_KERNEL); - if (!cq) - return ERR_PTR(-ENOMEM); cq->rdev = rdev; cq->qplib_cq.cq_handle = (u64)(unsigned long)(&cq->qplib_cq); @@ -2641,15 +2620,13 @@ struct ib_cq *bnxt_re_create_cq(struct ib_device *ibdev, } } - return &cq->ib_cq; + return 0; c2fail: - if (udata) - ib_umem_release(cq->umem); + ib_umem_release(cq->umem); fail: kfree(cq->cql); - kfree(cq); - return ERR_PTR(rc); + return rc; } static u8 __req_to_ib_wc_status(u8 qstatus) @@ -3353,8 +3330,7 @@ int bnxt_re_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata) mr->npages = 0; mr->pages = NULL; } - if (!IS_ERR_OR_NULL(mr->ib_umem)) - ib_umem_release(mr->ib_umem); + ib_umem_release(mr->ib_umem); kfree(mr); atomic_dec(&rdev->mr_count); @@ -3630,10 +3606,10 @@ int bnxt_re_alloc_ucontext(struct ib_ucontext *ctx, struct ib_udata *udata) u32 chip_met_rev_num = 0; int rc; - dev_dbg(rdev_to_dev(rdev), "ABI version requested %d", - ibdev->uverbs_abi_ver); + dev_dbg(rdev_to_dev(rdev), "ABI version requested %u", + ibdev->ops.uverbs_abi_ver); - if (ibdev->uverbs_abi_ver != BNXT_RE_ABI_VERSION) { + if (ibdev->ops.uverbs_abi_ver != BNXT_RE_ABI_VERSION) { dev_dbg(rdev_to_dev(rdev), " is different from the device %d ", BNXT_RE_ABI_VERSION); return -EPERM; diff --git a/drivers/infiniband/hw/bnxt_re/ib_verbs.h b/drivers/infiniband/hw/bnxt_re/ib_verbs.h index 09a33049e42f..31662b1ee35a 100644 --- a/drivers/infiniband/hw/bnxt_re/ib_verbs.h +++ b/drivers/infiniband/hw/bnxt_re/ib_verbs.h @@ -94,11 +94,11 @@ struct bnxt_re_qp { }; struct bnxt_re_cq { + struct ib_cq ib_cq; struct bnxt_re_dev *rdev; spinlock_t cq_lock; /* protect cq */ u16 cq_count; u16 cq_period; - struct ib_cq ib_cq; struct bnxt_qplib_cq qplib_cq; struct bnxt_qplib_cqe *cql; #define MAX_CQL_PER_POLL 1024 @@ -190,10 +190,9 @@ int bnxt_re_post_send(struct ib_qp *qp, const struct ib_send_wr *send_wr, const struct ib_send_wr **bad_send_wr); int bnxt_re_post_recv(struct ib_qp *qp, const struct ib_recv_wr *recv_wr, const struct ib_recv_wr **bad_recv_wr); -struct ib_cq *bnxt_re_create_cq(struct ib_device *ibdev, - const struct ib_cq_init_attr *attr, - struct ib_udata *udata); -int bnxt_re_destroy_cq(struct ib_cq *cq, struct ib_udata *udata); +int bnxt_re_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, + struct ib_udata *udata); +void bnxt_re_destroy_cq(struct ib_cq *cq, struct ib_udata *udata); int bnxt_re_poll_cq(struct ib_cq *cq, int num_entries, struct ib_wc *wc); int bnxt_re_req_notify_cq(struct ib_cq *cq, enum ib_cq_notify_flags flags); struct ib_mr *bnxt_re_get_dma_mr(struct ib_pd *pd, int mr_access_flags); diff --git a/drivers/infiniband/hw/bnxt_re/main.c b/drivers/infiniband/hw/bnxt_re/main.c index 814f959c7db9..029babe713f3 100644 --- a/drivers/infiniband/hw/bnxt_re/main.c +++ b/drivers/infiniband/hw/bnxt_re/main.c @@ -596,6 +596,10 @@ static void bnxt_re_unregister_ib(struct bnxt_re_dev *rdev) } static const struct ib_device_ops bnxt_re_dev_ops = { + .owner = THIS_MODULE, + .driver_id = RDMA_DRIVER_BNXT_RE, + .uverbs_abi_ver = BNXT_RE_ABI_VERSION, + .add_gid = bnxt_re_add_gid, .alloc_hw_stats = bnxt_re_ib_alloc_hw_stats, .alloc_mr = bnxt_re_alloc_mr, @@ -637,6 +641,7 @@ static const struct ib_device_ops bnxt_re_dev_ops = { .reg_user_mr = bnxt_re_reg_user_mr, .req_notify_cq = bnxt_re_req_notify_cq, INIT_RDMA_OBJ_SIZE(ib_ah, bnxt_re_ah, ib_ah), + INIT_RDMA_OBJ_SIZE(ib_cq, bnxt_re_cq, ib_cq), INIT_RDMA_OBJ_SIZE(ib_pd, bnxt_re_pd, ib_pd), INIT_RDMA_OBJ_SIZE(ib_srq, bnxt_re_srq, ib_srq), INIT_RDMA_OBJ_SIZE(ib_ucontext, bnxt_re_ucontext, ib_uctx), @@ -648,7 +653,6 @@ static int bnxt_re_register_ib(struct bnxt_re_dev *rdev) int ret; /* ib device init */ - ibdev->owner = THIS_MODULE; ibdev->node_type = RDMA_NODE_IB_CA; strlcpy(ibdev->node_desc, BNXT_RE_DESC " HCA", strlen(BNXT_RE_DESC) + 5); @@ -661,7 +665,6 @@ static int bnxt_re_register_ib(struct bnxt_re_dev *rdev) ibdev->local_dma_lkey = BNXT_QPLIB_RSVD_LKEY; /* User space */ - ibdev->uverbs_abi_ver = BNXT_RE_ABI_VERSION; ibdev->uverbs_cmd_mask = (1ull << IB_USER_VERBS_CMD_GET_CONTEXT) | (1ull << IB_USER_VERBS_CMD_QUERY_DEVICE) | @@ -691,7 +694,6 @@ static int bnxt_re_register_ib(struct bnxt_re_dev *rdev) rdma_set_device_sysfs_group(ibdev, &bnxt_re_dev_attr_group); - ibdev->driver_id = RDMA_DRIVER_BNXT_RE; ib_set_device_ops(ibdev, &bnxt_re_dev_ops); ret = ib_device_set_netdev(&rdev->ibdev, rdev->netdev, 1); if (ret) diff --git a/drivers/infiniband/hw/cxgb3/cxio_hal.c b/drivers/infiniband/hw/cxgb3/cxio_hal.c index 8ac72ac7cbac..95b22a651673 100644 --- a/drivers/infiniband/hw/cxgb3/cxio_hal.c +++ b/drivers/infiniband/hw/cxgb3/cxio_hal.c @@ -174,7 +174,6 @@ int cxio_create_cq(struct cxio_rdev *rdev_p, struct t3_cq *cq, int kernel) return -ENOMEM; } dma_unmap_addr_set(cq, mapping, cq->dma_addr); - memset(cq->queue, 0, size); setup.id = cq->cqid; setup.base_addr = (u64) (cq->dma_addr); setup.size = 1UL << cq->size_log2; @@ -187,20 +186,6 @@ int cxio_create_cq(struct cxio_rdev *rdev_p, struct t3_cq *cq, int kernel) return (rdev_p->t3cdev_p->ctl(rdev_p->t3cdev_p, RDMA_CQ_SETUP, &setup)); } -#ifdef notyet -int cxio_resize_cq(struct cxio_rdev *rdev_p, struct t3_cq *cq) -{ - struct rdma_cq_setup setup; - setup.id = cq->cqid; - setup.base_addr = (u64) (cq->dma_addr); - setup.size = 1UL << cq->size_log2; - setup.credits = setup.size; - setup.credit_thres = setup.size; /* TBD: overflow recovery */ - setup.ovfl_mode = 1; - return (rdev_p->t3cdev_p->ctl(rdev_p->t3cdev_p, RDMA_CQ_SETUP, &setup)); -} -#endif - static u32 get_qpid(struct cxio_rdev *rdev_p, struct cxio_ucontext *uctx) { struct cxio_qpid_list *entry; @@ -219,7 +204,7 @@ static u32 get_qpid(struct cxio_rdev *rdev_p, struct cxio_ucontext *uctx) if (!qpid) goto out; for (i = qpid+1; i & rdev_p->qpmask; i++) { - entry = kmalloc(sizeof *entry, GFP_KERNEL); + entry = kmalloc(sizeof(*entry), GFP_KERNEL); if (!entry) break; entry->qpid = i; @@ -237,7 +222,7 @@ static void put_qpid(struct cxio_rdev *rdev_p, u32 qpid, { struct cxio_qpid_list *entry; - entry = kmalloc(sizeof *entry, GFP_KERNEL); + entry = kmalloc(sizeof(*entry), GFP_KERNEL); if (!entry) return; pr_debug("%s qpid 0x%x\n", __func__, qpid); @@ -317,17 +302,15 @@ err1: return -ENOMEM; } -int cxio_destroy_cq(struct cxio_rdev *rdev_p, struct t3_cq *cq) +void cxio_destroy_cq(struct cxio_rdev *rdev_p, struct t3_cq *cq) { - int err; - err = cxio_hal_clear_cq_ctx(rdev_p, cq->cqid); + cxio_hal_clear_cq_ctx(rdev_p, cq->cqid); kfree(cq->sw_queue); dma_free_coherent(&(rdev_p->rnic_info.pdev->dev), (1UL << (cq->size_log2)) * sizeof(struct t3_cqe) + 1, cq->queue, dma_unmap_addr(cq, mapping)); cxio_hal_put_cqid(rdev_p->rscp, cq->cqid); - return err; } int cxio_destroy_qp(struct cxio_rdev *rdev_p, struct t3_wq *wq, @@ -538,8 +521,6 @@ static int cxio_hal_init_ctrl_qp(struct cxio_rdev *rdev_p) dma_unmap_addr_set(&rdev_p->ctrl_qp, mapping, rdev_p->ctrl_qp.dma_addr); rdev_p->ctrl_qp.doorbell = (void __iomem *)rdev_p->rnic_info.kdb_addr; - memset(rdev_p->ctrl_qp.workq, 0, - (1 << T3_CTRL_QP_SIZE_LOG2) * sizeof(union t3_wr)); mutex_init(&rdev_p->ctrl_qp.lock); init_waitqueue_head(&rdev_p->ctrl_qp.waitq); @@ -565,9 +546,9 @@ static int cxio_hal_init_ctrl_qp(struct cxio_rdev *rdev_p) wqe->sge_cmd = cpu_to_be64(sge_cmd); wqe->ctx1 = cpu_to_be64(ctx1); wqe->ctx0 = cpu_to_be64(ctx0); - pr_debug("CtrlQP dma_addr 0x%llx workq %p size %d\n", - (unsigned long long)rdev_p->ctrl_qp.dma_addr, - rdev_p->ctrl_qp.workq, 1 << T3_CTRL_QP_SIZE_LOG2); + pr_debug("CtrlQP dma_addr %pad workq %p size %d\n", + &rdev_p->ctrl_qp.dma_addr, rdev_p->ctrl_qp.workq, + 1 << T3_CTRL_QP_SIZE_LOG2); skb->priority = CPL_PRIORITY_CONTROL; return iwch_cxgb3_ofld_send(rdev_p->t3cdev_p, skb); err: diff --git a/drivers/infiniband/hw/cxgb3/cxio_hal.h b/drivers/infiniband/hw/cxgb3/cxio_hal.h index c64e50b5a548..40c029ffa425 100644 --- a/drivers/infiniband/hw/cxgb3/cxio_hal.h +++ b/drivers/infiniband/hw/cxgb3/cxio_hal.h @@ -158,8 +158,7 @@ void cxio_rdev_close(struct cxio_rdev *rdev); int cxio_hal_cq_op(struct cxio_rdev *rdev, struct t3_cq *cq, enum t3_cq_opcode op, u32 credit); int cxio_create_cq(struct cxio_rdev *rdev, struct t3_cq *cq, int kernel); -int cxio_destroy_cq(struct cxio_rdev *rdev, struct t3_cq *cq); -int cxio_resize_cq(struct cxio_rdev *rdev, struct t3_cq *cq); +void cxio_destroy_cq(struct cxio_rdev *rdev, struct t3_cq *cq); void cxio_release_ucontext(struct cxio_rdev *rdev, struct cxio_ucontext *uctx); void cxio_init_ucontext(struct cxio_rdev *rdev, struct cxio_ucontext *uctx); int cxio_create_qp(struct cxio_rdev *rdev, u32 kernel_domain, struct t3_wq *wq, diff --git a/drivers/infiniband/hw/cxgb3/iwch_cm.c b/drivers/infiniband/hw/cxgb3/iwch_cm.c index 1c90c86fc8b8..0bca72cb4d9a 100644 --- a/drivers/infiniband/hw/cxgb3/iwch_cm.c +++ b/drivers/infiniband/hw/cxgb3/iwch_cm.c @@ -170,7 +170,7 @@ static void release_tid(struct t3cdev *tdev, u32 hwtid, struct sk_buff *skb) { struct cpl_tid_release *req; - skb = get_skb(skb, sizeof *req, GFP_KERNEL); + skb = get_skb(skb, sizeof(*req), GFP_KERNEL); if (!skb) return; req = skb_put(skb, sizeof(*req)); diff --git a/drivers/infiniband/hw/cxgb3/iwch_provider.c b/drivers/infiniband/hw/cxgb3/iwch_provider.c index 3a481dfb1607..e775c1a1a450 100644 --- a/drivers/infiniband/hw/cxgb3/iwch_provider.c +++ b/drivers/infiniband/hw/cxgb3/iwch_provider.c @@ -88,7 +88,7 @@ static int iwch_alloc_ucontext(struct ib_ucontext *ucontext, return 0; } -static int iwch_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata) +static void iwch_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata) { struct iwch_cq *chp; @@ -100,17 +100,16 @@ static int iwch_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata) wait_event(chp->wait, !atomic_read(&chp->refcnt)); cxio_destroy_cq(&chp->rhp->rdev, &chp->cq); - kfree(chp); - return 0; } -static struct ib_cq *iwch_create_cq(struct ib_device *ibdev, - const struct ib_cq_init_attr *attr, - struct ib_udata *udata) +static int iwch_create_cq(struct ib_cq *ibcq, + const struct ib_cq_init_attr *attr, + struct ib_udata *udata) { + struct ib_device *ibdev = ibcq->device; int entries = attr->cqe; - struct iwch_dev *rhp; - struct iwch_cq *chp; + struct iwch_dev *rhp = to_iwch_dev(ibcq->device); + struct iwch_cq *chp = to_iwch_cq(ibcq); struct iwch_create_cq_resp uresp; struct iwch_create_cq_req ureq; static int warned; @@ -118,19 +117,13 @@ static struct ib_cq *iwch_create_cq(struct ib_device *ibdev, pr_debug("%s ib_dev %p entries %d\n", __func__, ibdev, entries); if (attr->flags) - return ERR_PTR(-EINVAL); - - rhp = to_iwch_dev(ibdev); - chp = kzalloc(sizeof(*chp), GFP_KERNEL); - if (!chp) - return ERR_PTR(-ENOMEM); + return -EINVAL; if (udata) { if (!t3a_device(rhp)) { - if (ib_copy_from_udata(&ureq, udata, sizeof (ureq))) { - kfree(chp); - return ERR_PTR(-EFAULT); - } + if (ib_copy_from_udata(&ureq, udata, sizeof(ureq))) + return -EFAULT; + chp->user_rptr_addr = (u32 __user *)(unsigned long)ureq.user_rptr_addr; } } @@ -151,10 +144,9 @@ static struct ib_cq *iwch_create_cq(struct ib_device *ibdev, entries = roundup_pow_of_two(entries); chp->cq.size_log2 = ilog2(entries); - if (cxio_create_cq(&rhp->rdev, &chp->cq, !udata)) { - kfree(chp); - return ERR_PTR(-ENOMEM); - } + if (cxio_create_cq(&rhp->rdev, &chp->cq, !udata)) + return -ENOMEM; + chp->rhp = rhp; chp->ibcq.cqe = 1 << chp->cq.size_log2; spin_lock_init(&chp->lock); @@ -163,8 +155,7 @@ static struct ib_cq *iwch_create_cq(struct ib_device *ibdev, init_waitqueue_head(&chp->wait); if (xa_store_irq(&rhp->cqs, chp->cq.cqid, chp, GFP_KERNEL)) { cxio_destroy_cq(&chp->rhp->rdev, &chp->cq); - kfree(chp); - return ERR_PTR(-ENOMEM); + return -ENOMEM; } if (udata) { @@ -172,10 +163,10 @@ static struct ib_cq *iwch_create_cq(struct ib_device *ibdev, struct iwch_ucontext *ucontext = rdma_udata_to_drv_context( udata, struct iwch_ucontext, ibucontext); - mm = kmalloc(sizeof *mm, GFP_KERNEL); + mm = kmalloc(sizeof(*mm), GFP_KERNEL); if (!mm) { iwch_destroy_cq(&chp->ibcq, udata); - return ERR_PTR(-ENOMEM); + return -ENOMEM; } uresp.cqid = chp->cq.cqid; uresp.size_log2 = chp->cq.size_log2; @@ -185,7 +176,7 @@ static struct ib_cq *iwch_create_cq(struct ib_device *ibdev, spin_unlock(&ucontext->mmap_lock); mm->key = uresp.key; mm->addr = virt_to_phys(chp->cq.queue); - if (udata->outlen < sizeof uresp) { + if (udata->outlen < sizeof(uresp)) { if (!warned++) pr_warn("Warning - downlevel libcxgb3 (non-fatal)\n"); mm->len = PAGE_ALIGN((1UL << uresp.size_log2) * @@ -196,86 +187,19 @@ static struct ib_cq *iwch_create_cq(struct ib_device *ibdev, sizeof(struct t3_cqe)); uresp.memsize = mm->len; uresp.reserved = 0; - resplen = sizeof uresp; + resplen = sizeof(uresp); } if (ib_copy_to_udata(udata, &uresp, resplen)) { kfree(mm); iwch_destroy_cq(&chp->ibcq, udata); - return ERR_PTR(-EFAULT); + return -EFAULT; } insert_mmap(ucontext, mm); } - pr_debug("created cqid 0x%0x chp %p size 0x%0x, dma_addr 0x%0llx\n", + pr_debug("created cqid 0x%0x chp %p size 0x%0x, dma_addr %pad\n", chp->cq.cqid, chp, (1 << chp->cq.size_log2), - (unsigned long long)chp->cq.dma_addr); - return &chp->ibcq; -} - -static int iwch_resize_cq(struct ib_cq *cq, int cqe, struct ib_udata *udata) -{ -#ifdef notyet - struct iwch_cq *chp = to_iwch_cq(cq); - struct t3_cq oldcq, newcq; - int ret; - - pr_debug("%s ib_cq %p cqe %d\n", __func__, cq, cqe); - - /* We don't downsize... */ - if (cqe <= cq->cqe) - return 0; - - /* create new t3_cq with new size */ - cqe = roundup_pow_of_two(cqe+1); - newcq.size_log2 = ilog2(cqe); - - /* Dont allow resize to less than the current wce count */ - if (cqe < Q_COUNT(chp->cq.rptr, chp->cq.wptr)) { - return -ENOMEM; - } - - /* Quiesce all QPs using this CQ */ - ret = iwch_quiesce_qps(chp); - if (ret) { - return ret; - } - - ret = cxio_create_cq(&chp->rhp->rdev, &newcq); - if (ret) { - return ret; - } - - /* copy CQEs */ - memcpy(newcq.queue, chp->cq.queue, (1 << chp->cq.size_log2) * - sizeof(struct t3_cqe)); - - /* old iwch_qp gets new t3_cq but keeps old cqid */ - oldcq = chp->cq; - chp->cq = newcq; - chp->cq.cqid = oldcq.cqid; - - /* resize new t3_cq to update the HW context */ - ret = cxio_resize_cq(&chp->rhp->rdev, &chp->cq); - if (ret) { - chp->cq = oldcq; - return ret; - } - chp->ibcq.cqe = (1<<chp->cq.size_log2) - 1; - - /* destroy old t3_cq */ - oldcq.cqid = newcq.cqid; - ret = cxio_destroy_cq(&chp->rhp->rdev, &oldcq); - if (ret) { - pr_err("%s - cxio_destroy_cq failed %d\n", __func__, ret); - } - - /* add user hooks here */ - - /* resume qps */ - ret = iwch_resume_qps(chp); - return ret; -#else - return -ENOSYS; -#endif + &chp->cq.dma_addr); + return 0; } static int iwch_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags) @@ -422,8 +346,7 @@ static int iwch_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata) xa_erase_irq(&rhp->mrs, mmid); if (mhp->kva) kfree((void *) (unsigned long) mhp->kva); - if (mhp->umem) - ib_umem_release(mhp->umem); + ib_umem_release(mhp->umem); pr_debug("%s mmid 0x%x ptr %p\n", __func__, mmid, mhp); kfree(mhp); return 0; @@ -553,7 +476,7 @@ static struct ib_mr *iwch_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, for_each_sg_dma_page(mhp->umem->sg_head.sgl, &sg_iter, mhp->umem->nmap, 0) { pages[i++] = cpu_to_be64(sg_page_iter_dma_address(&sg_iter)); - if (i == PAGE_SIZE / sizeof *pages) { + if (i == PAGE_SIZE / sizeof(*pages)) { err = iwch_write_pbl(mhp, pages, i, n); if (err) goto pbl_done; @@ -587,7 +510,7 @@ pbl_done: pr_debug("%s user resp pbl_addr 0x%x\n", __func__, uresp.pbl_addr); - if (ib_copy_to_udata(udata, &uresp, sizeof (uresp))) { + if (ib_copy_to_udata(udata, &uresp, sizeof(uresp))) { iwch_dereg_mr(&mhp->ibmr, udata); err = -EFAULT; goto err; @@ -880,13 +803,13 @@ static struct ib_qp *iwch_create_qp(struct ib_pd *pd, struct iwch_mm_entry *mm1, *mm2; - mm1 = kmalloc(sizeof *mm1, GFP_KERNEL); + mm1 = kmalloc(sizeof(*mm1), GFP_KERNEL); if (!mm1) { iwch_destroy_qp(&qhp->ibqp, udata); return ERR_PTR(-ENOMEM); } - mm2 = kmalloc(sizeof *mm2, GFP_KERNEL); + mm2 = kmalloc(sizeof(*mm2), GFP_KERNEL); if (!mm2) { kfree(mm1); iwch_destroy_qp(&qhp->ibqp, udata); @@ -903,7 +826,7 @@ static struct ib_qp *iwch_create_qp(struct ib_pd *pd, uresp.db_key = ucontext->key; ucontext->key += PAGE_SIZE; spin_unlock(&ucontext->mmap_lock); - if (ib_copy_to_udata(udata, &uresp, sizeof (uresp))) { + if (ib_copy_to_udata(udata, &uresp, sizeof(uresp))) { kfree(mm1); kfree(mm2); iwch_destroy_qp(&qhp->ibqp, udata); @@ -911,7 +834,7 @@ static struct ib_qp *iwch_create_qp(struct ib_pd *pd, } mm1->key = uresp.key; mm1->addr = virt_to_phys(qhp->wq.queue); - mm1->len = PAGE_ALIGN(wqsize * sizeof (union t3_wr)); + mm1->len = PAGE_ALIGN(wqsize * sizeof(union t3_wr)); insert_mmap(ucontext, mm1); mm2->key = uresp.db_key; mm2->addr = qhp->wq.udb & PAGE_MASK; @@ -919,10 +842,11 @@ static struct ib_qp *iwch_create_qp(struct ib_pd *pd, insert_mmap(ucontext, mm2); } qhp->ibqp.qp_num = qhp->wq.qpid; - pr_debug("%s sq_num_entries %d, rq_num_entries %d qpid 0x%0x qhp %p dma_addr 0x%llx size %d rq_addr 0x%x\n", - __func__, qhp->attr.sq_num_entries, qhp->attr.rq_num_entries, - qhp->wq.qpid, qhp, (unsigned long long)qhp->wq.dma_addr, - 1 << qhp->wq.size_log2, qhp->wq.rq_addr); + pr_debug( + "%s sq_num_entries %d, rq_num_entries %d qpid 0x%0x qhp %p dma_addr %pad size %d rq_addr 0x%x\n", + __func__, qhp->attr.sq_num_entries, qhp->attr.rq_num_entries, + qhp->wq.qpid, qhp, &qhp->wq.dma_addr, 1 << qhp->wq.size_log2, + qhp->wq.rq_addr); return &qhp->ibqp; } @@ -932,7 +856,7 @@ static int iwch_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, struct iwch_dev *rhp; struct iwch_qp *qhp; enum iwch_qp_attr_mask mask = 0; - struct iwch_qp_attributes attrs; + struct iwch_qp_attributes attrs = {}; pr_debug("%s ib_qp %p\n", __func__, ibqp); @@ -944,7 +868,6 @@ static int iwch_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, if (!attr_mask) return 0; - memset(&attrs, 0, sizeof attrs); qhp = to_iwch_qp(ibqp); rhp = qhp->rhp; @@ -1040,7 +963,6 @@ static int iwch_query_device(struct ib_device *ibdev, struct ib_device_attr *pro return -EINVAL; dev = to_iwch_dev(ibdev); - memset(props, 0, sizeof *props); memcpy(&props->sys_image_guid, dev->rdev.t3cdev_p->lldev->dev_addr, 6); props->hw_ver = dev->rdev.t3cdev_p->type; props->fw_ver = fw_vers_string_to_u64(dev); @@ -1304,6 +1226,11 @@ static void get_dev_fw_ver_str(struct ib_device *ibdev, char *str) } static const struct ib_device_ops iwch_dev_ops = { + .owner = THIS_MODULE, + .driver_id = RDMA_DRIVER_CXGB3, + .uverbs_abi_ver = IWCH_UVERBS_ABI_VERSION, + .uverbs_no_driver_id_binding = 1, + .alloc_hw_stats = iwch_alloc_stats, .alloc_mr = iwch_alloc_mr, .alloc_mw = iwch_alloc_mw, @@ -1341,8 +1268,8 @@ static const struct ib_device_ops iwch_dev_ops = { .query_port = iwch_query_port, .reg_user_mr = iwch_reg_user_mr, .req_notify_cq = iwch_arm_cq, - .resize_cq = iwch_resize_cq, INIT_RDMA_OBJ_SIZE(ib_pd, iwch_pd, ibpd), + INIT_RDMA_OBJ_SIZE(ib_cq, iwch_cq, ibcq), INIT_RDMA_OBJ_SIZE(ib_ucontext, iwch_ucontext, ibucontext), }; @@ -1351,7 +1278,6 @@ int iwch_register_device(struct iwch_dev *dev) pr_debug("%s iwch_dev %p\n", __func__, dev); memset(&dev->ibdev.node_guid, 0, sizeof(dev->ibdev.node_guid)); memcpy(&dev->ibdev.node_guid, dev->rdev.t3cdev_p->lldev->dev_addr, 6); - dev->ibdev.owner = THIS_MODULE; dev->device_cap_flags = IB_DEVICE_LOCAL_DMA_LKEY | IB_DEVICE_MEM_WINDOW | IB_DEVICE_MEM_MGT_EXTENSIONS; @@ -1383,12 +1309,10 @@ int iwch_register_device(struct iwch_dev *dev) dev->ibdev.phys_port_cnt = dev->rdev.port_info.nports; dev->ibdev.num_comp_vectors = 1; dev->ibdev.dev.parent = &dev->rdev.rnic_info.pdev->dev; - dev->ibdev.uverbs_abi_ver = IWCH_UVERBS_ABI_VERSION; memcpy(dev->ibdev.iw_ifname, dev->rdev.t3cdev_p->lldev->name, sizeof(dev->ibdev.iw_ifname)); - dev->ibdev.driver_id = RDMA_DRIVER_CXGB3; rdma_set_device_sysfs_group(&dev->ibdev, &iwch_attr_group); ib_set_device_ops(&dev->ibdev, &iwch_dev_ops); return ib_register_device(&dev->ibdev, "cxgb3_%d"); diff --git a/drivers/infiniband/hw/cxgb4/cm.c b/drivers/infiniband/hw/cxgb4/cm.c index 09fcfc9e052d..e87fc0408470 100644 --- a/drivers/infiniband/hw/cxgb4/cm.c +++ b/drivers/infiniband/hw/cxgb4/cm.c @@ -953,7 +953,7 @@ static int send_mpa_req(struct c4iw_ep *ep, struct sk_buff *skb, mpalen = sizeof(*mpa) + ep->plen; if (mpa_rev_to_use == 2) mpalen += sizeof(struct mpa_v2_conn_params); - wrlen = roundup(mpalen + sizeof *req, 16); + wrlen = roundup(mpalen + sizeof(*req), 16); skb = get_skb(skb, wrlen, GFP_KERNEL); if (!skb) { connect_reply_upcall(ep, -ENOMEM); @@ -997,8 +997,9 @@ static int send_mpa_req(struct c4iw_ep *ep, struct sk_buff *skb, } if (mpa_rev_to_use == 2) { - mpa->private_data_size = htons(ntohs(mpa->private_data_size) + - sizeof (struct mpa_v2_conn_params)); + mpa->private_data_size = + htons(ntohs(mpa->private_data_size) + + sizeof(struct mpa_v2_conn_params)); pr_debug("initiator ird %u ord %u\n", ep->ird, ep->ord); mpa_v2_params.ird = htons((u16)ep->ird); @@ -1057,7 +1058,7 @@ static int send_mpa_reject(struct c4iw_ep *ep, const void *pdata, u8 plen) mpalen = sizeof(*mpa) + plen; if (ep->mpa_attr.version == 2 && ep->mpa_attr.enhanced_rdma_conn) mpalen += sizeof(struct mpa_v2_conn_params); - wrlen = roundup(mpalen + sizeof *req, 16); + wrlen = roundup(mpalen + sizeof(*req), 16); skb = get_skb(NULL, wrlen, GFP_KERNEL); if (!skb) { @@ -1088,8 +1089,9 @@ static int send_mpa_reject(struct c4iw_ep *ep, const void *pdata, u8 plen) if (ep->mpa_attr.version == 2 && ep->mpa_attr.enhanced_rdma_conn) { mpa->flags |= MPA_ENHANCED_RDMA_CONN; - mpa->private_data_size = htons(ntohs(mpa->private_data_size) + - sizeof (struct mpa_v2_conn_params)); + mpa->private_data_size = + htons(ntohs(mpa->private_data_size) + + sizeof(struct mpa_v2_conn_params)); mpa_v2_params.ird = htons(((u16)ep->ird) | (peer2peer ? MPA_V2_PEER2PEER_MODEL : 0)); @@ -1136,7 +1138,7 @@ static int send_mpa_reply(struct c4iw_ep *ep, const void *pdata, u8 plen) mpalen = sizeof(*mpa) + plen; if (ep->mpa_attr.version == 2 && ep->mpa_attr.enhanced_rdma_conn) mpalen += sizeof(struct mpa_v2_conn_params); - wrlen = roundup(mpalen + sizeof *req, 16); + wrlen = roundup(mpalen + sizeof(*req), 16); skb = get_skb(NULL, wrlen, GFP_KERNEL); if (!skb) { @@ -1171,8 +1173,9 @@ static int send_mpa_reply(struct c4iw_ep *ep, const void *pdata, u8 plen) if (ep->mpa_attr.version == 2 && ep->mpa_attr.enhanced_rdma_conn) { mpa->flags |= MPA_ENHANCED_RDMA_CONN; - mpa->private_data_size = htons(ntohs(mpa->private_data_size) + - sizeof (struct mpa_v2_conn_params)); + mpa->private_data_size = + htons(ntohs(mpa->private_data_size) + + sizeof(struct mpa_v2_conn_params)); mpa_v2_params.ird = htons((u16)ep->ird); mpa_v2_params.ord = htons((u16)ep->ord); if (peer2peer && (ep->mpa_attr.p2p_type != diff --git a/drivers/infiniband/hw/cxgb4/cq.c b/drivers/infiniband/hw/cxgb4/cq.c index 52ce586621c6..b1bb61c65f4f 100644 --- a/drivers/infiniband/hw/cxgb4/cq.c +++ b/drivers/infiniband/hw/cxgb4/cq.c @@ -34,16 +34,15 @@ #include "iw_cxgb4.h" -static int destroy_cq(struct c4iw_rdev *rdev, struct t4_cq *cq, - struct c4iw_dev_ucontext *uctx, struct sk_buff *skb, - struct c4iw_wr_wait *wr_waitp) +static void destroy_cq(struct c4iw_rdev *rdev, struct t4_cq *cq, + struct c4iw_dev_ucontext *uctx, struct sk_buff *skb, + struct c4iw_wr_wait *wr_waitp) { struct fw_ri_res_wr *res_wr; struct fw_ri_res *res; int wr_len; - int ret; - wr_len = sizeof *res_wr + sizeof *res; + wr_len = sizeof(*res_wr) + sizeof(*res); set_wr_txq(skb, CPL_PRIORITY_CONTROL, 0); res_wr = __skb_put_zero(skb, wr_len); @@ -59,14 +58,13 @@ static int destroy_cq(struct c4iw_rdev *rdev, struct t4_cq *cq, res->u.cq.iqid = cpu_to_be32(cq->cqid); c4iw_init_wr_wait(wr_waitp); - ret = c4iw_ref_send_wait(rdev, skb, wr_waitp, 0, 0, __func__); + c4iw_ref_send_wait(rdev, skb, wr_waitp, 0, 0, __func__); kfree(cq->sw_queue); dma_free_coherent(&(rdev->lldi.pdev->dev), cq->memsize, cq->queue, dma_unmap_addr(cq, mapping)); c4iw_put_cqid(rdev, cq->cqid, uctx); - return ret; } static int create_cq(struct c4iw_rdev *rdev, struct t4_cq *cq, @@ -104,7 +102,6 @@ static int create_cq(struct c4iw_rdev *rdev, struct t4_cq *cq, goto err3; } dma_unmap_addr_set(cq, mapping, cq->dma_addr); - memset(cq->queue, 0, cq->memsize); if (user && ucontext->is_32b_cqe) { cq->qp_errp = &((struct t4_status_page *) @@ -117,7 +114,7 @@ static int create_cq(struct c4iw_rdev *rdev, struct t4_cq *cq, } /* build fw_ri_res_wr */ - wr_len = sizeof *res_wr + sizeof *res; + wr_len = sizeof(*res_wr) + sizeof(*res); skb = alloc_skb(wr_len, GFP_KERNEL); if (!skb) { @@ -970,7 +967,7 @@ int c4iw_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc) return !err || err == -ENODATA ? npolled : err; } -int c4iw_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata) +void c4iw_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata) { struct c4iw_cq *chp; struct c4iw_ucontext *ucontext; @@ -988,18 +985,16 @@ int c4iw_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata) ucontext ? &ucontext->uctx : &chp->cq.rdev->uctx, chp->destroy_skb, chp->wr_waitp); c4iw_put_wr_wait(chp->wr_waitp); - kfree(chp); - return 0; } -struct ib_cq *c4iw_create_cq(struct ib_device *ibdev, - const struct ib_cq_init_attr *attr, - struct ib_udata *udata) +int c4iw_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, + struct ib_udata *udata) { + struct ib_device *ibdev = ibcq->device; int entries = attr->cqe; int vector = attr->comp_vector; - struct c4iw_dev *rhp; - struct c4iw_cq *chp; + struct c4iw_dev *rhp = to_c4iw_dev(ibcq->device); + struct c4iw_cq *chp = to_c4iw_cq(ibcq); struct c4iw_create_cq ucmd; struct c4iw_create_cq_resp uresp; int ret, wr_len; @@ -1010,22 +1005,16 @@ struct ib_cq *c4iw_create_cq(struct ib_device *ibdev, pr_debug("ib_dev %p entries %d\n", ibdev, entries); if (attr->flags) - return ERR_PTR(-EINVAL); - - rhp = to_c4iw_dev(ibdev); + return -EINVAL; if (vector >= rhp->rdev.lldi.nciq) - return ERR_PTR(-EINVAL); + return -EINVAL; if (udata) { if (udata->inlen < sizeof(ucmd)) ucontext->is_32b_cqe = 1; } - chp = kzalloc(sizeof(*chp), GFP_KERNEL); - if (!chp) - return ERR_PTR(-ENOMEM); - chp->wr_waitp = c4iw_alloc_wr_wait(GFP_KERNEL); if (!chp->wr_waitp) { ret = -ENOMEM; @@ -1095,10 +1084,10 @@ struct ib_cq *c4iw_create_cq(struct ib_device *ibdev, if (ucontext) { ret = -ENOMEM; - mm = kmalloc(sizeof *mm, GFP_KERNEL); + mm = kmalloc(sizeof(*mm), GFP_KERNEL); if (!mm) goto err_remove_handle; - mm2 = kmalloc(sizeof *mm2, GFP_KERNEL); + mm2 = kmalloc(sizeof(*mm2), GFP_KERNEL); if (!mm2) goto err_free_mm; @@ -1135,10 +1124,11 @@ struct ib_cq *c4iw_create_cq(struct ib_device *ibdev, mm2->len = PAGE_SIZE; insert_mmap(ucontext, mm2); } - pr_debug("cqid 0x%0x chp %p size %u memsize %zu, dma_addr 0x%0llx\n", - chp->cq.cqid, chp, chp->cq.size, - chp->cq.memsize, (unsigned long long)chp->cq.dma_addr); - return &chp->ibcq; + + pr_debug("cqid 0x%0x chp %p size %u memsize %zu, dma_addr %pad\n", + chp->cq.cqid, chp, chp->cq.size, chp->cq.memsize, + &chp->cq.dma_addr); + return 0; err_free_mm2: kfree(mm2); err_free_mm: @@ -1154,8 +1144,7 @@ err_free_skb: err_free_wr_wait: c4iw_put_wr_wait(chp->wr_waitp); err_free_chp: - kfree(chp); - return ERR_PTR(ret); + return ret; } int c4iw_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags) diff --git a/drivers/infiniband/hw/cxgb4/device.c b/drivers/infiniband/hw/cxgb4/device.c index 4c0d925c5ff5..a8b9548bd1a2 100644 --- a/drivers/infiniband/hw/cxgb4/device.c +++ b/drivers/infiniband/hw/cxgb4/device.c @@ -327,7 +327,7 @@ static int qp_open(struct inode *inode, struct file *file) unsigned long index; int count = 1; - qpd = kmalloc(sizeof *qpd, GFP_KERNEL); + qpd = kmalloc(sizeof(*qpd), GFP_KERNEL); if (!qpd) return -ENOMEM; @@ -421,7 +421,7 @@ static int stag_open(struct inode *inode, struct file *file) int ret = 0; int count = 1; - stagd = kmalloc(sizeof *stagd, GFP_KERNEL); + stagd = kmalloc(sizeof(*stagd), GFP_KERNEL); if (!stagd) { ret = -ENOMEM; goto out; @@ -1075,7 +1075,7 @@ static void *c4iw_uld_add(const struct cxgb4_lld_info *infop) pr_info("Chelsio T4/T5 RDMA Driver - version %s\n", DRV_VERSION); - ctx = kzalloc(sizeof *ctx, GFP_KERNEL); + ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); if (!ctx) { ctx = ERR_PTR(-ENOMEM); goto out; @@ -1243,10 +1243,9 @@ static int c4iw_uld_state_change(void *handle, enum cxgb4_state new_state) case CXGB4_STATE_START_RECOVERY: pr_info("%s: Fatal Error\n", pci_name(ctx->lldi.pdev)); if (ctx->dev) { - struct ib_event event; + struct ib_event event = {}; ctx->dev->rdev.flags |= T4_FATAL_ERROR; - memset(&event, 0, sizeof event); event.event = IB_EVENT_DEVICE_FATAL; event.device = &ctx->dev->ibdev; ib_dispatch_event(&event); diff --git a/drivers/infiniband/hw/cxgb4/iw_cxgb4.h b/drivers/infiniband/hw/cxgb4/iw_cxgb4.h index 916ef982172e..7d06b0f8d49a 100644 --- a/drivers/infiniband/hw/cxgb4/iw_cxgb4.h +++ b/drivers/infiniband/hw/cxgb4/iw_cxgb4.h @@ -490,13 +490,13 @@ struct c4iw_qp { struct t4_wq wq; spinlock_t lock; struct mutex mutex; - struct kref kref; wait_queue_head_t wait; int sq_sig_all; struct c4iw_srq *srq; - struct work_struct free_work; struct c4iw_ucontext *ucontext; struct c4iw_wr_wait *wr_waitp; + struct completion qp_rel_comp; + refcount_t qp_refcnt; }; static inline struct c4iw_qp *to_c4iw_qp(struct ib_qp *ibqp) @@ -992,10 +992,9 @@ struct ib_mr *c4iw_reg_user_mr(struct ib_pd *pd, u64 start, struct ib_udata *udata); struct ib_mr *c4iw_get_dma_mr(struct ib_pd *pd, int acc); int c4iw_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata); -int c4iw_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata); -struct ib_cq *c4iw_create_cq(struct ib_device *ibdev, - const struct ib_cq_init_attr *attr, - struct ib_udata *udata); +void c4iw_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata); +int c4iw_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, + struct ib_udata *udata); int c4iw_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags); int c4iw_modify_srq(struct ib_srq *ib_srq, struct ib_srq_attr *attr, enum ib_srq_attr_mask srq_attr_mask, diff --git a/drivers/infiniband/hw/cxgb4/mem.c b/drivers/infiniband/hw/cxgb4/mem.c index 811c0c8c5b16..aa772ee0706f 100644 --- a/drivers/infiniband/hw/cxgb4/mem.c +++ b/drivers/infiniband/hw/cxgb4/mem.c @@ -130,8 +130,9 @@ static int _c4iw_write_mem_inline(struct c4iw_rdev *rdev, u32 addr, u32 len, copy_len = len > C4IW_MAX_INLINE_SIZE ? C4IW_MAX_INLINE_SIZE : len; - wr_len = roundup(sizeof *req + sizeof *sc + - roundup(copy_len, T4_ULPTX_MIN_IO), 16); + wr_len = roundup(sizeof(*req) + sizeof(*sc) + + roundup(copy_len, T4_ULPTX_MIN_IO), + 16); if (!skb) { skb = alloc_skb(wr_len, GFP_KERNEL | __GFP_NOFAIL); @@ -807,8 +808,7 @@ int c4iw_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata) mhp->attr.pbl_size << 3); if (mhp->kva) kfree((void *) (unsigned long) mhp->kva); - if (mhp->umem) - ib_umem_release(mhp->umem); + ib_umem_release(mhp->umem); pr_debug("mmid 0x%x ptr %p\n", mmid, mhp); c4iw_put_wr_wait(mhp->wr_waitp); kfree(mhp); diff --git a/drivers/infiniband/hw/cxgb4/provider.c b/drivers/infiniband/hw/cxgb4/provider.c index 74b795642fca..5e59c5708729 100644 --- a/drivers/infiniband/hw/cxgb4/provider.c +++ b/drivers/infiniband/hw/cxgb4/provider.c @@ -271,7 +271,6 @@ static int c4iw_query_device(struct ib_device *ibdev, struct ib_device_attr *pro return -EINVAL; dev = to_c4iw_dev(ibdev); - memset(props, 0, sizeof *props); memcpy(&props->sys_image_guid, dev->rdev.lldi.ports[0]->dev_addr, 6); props->hw_ver = CHELSIO_CHIP_RELEASE(dev->rdev.lldi.adapter_type); props->fw_ver = dev->rdev.lldi.fw_vers; @@ -490,6 +489,10 @@ static int fill_res_entry(struct sk_buff *msg, struct rdma_restrack_entry *res) } static const struct ib_device_ops c4iw_dev_ops = { + .owner = THIS_MODULE, + .driver_id = RDMA_DRIVER_CXGB4, + .uverbs_abi_ver = C4IW_UVERBS_ABI_VERSION, + .alloc_hw_stats = c4iw_alloc_stats, .alloc_mr = c4iw_alloc_mr, .alloc_mw = c4iw_alloc_mw, @@ -534,6 +537,7 @@ static const struct ib_device_ops c4iw_dev_ops = { .reg_user_mr = c4iw_reg_user_mr, .req_notify_cq = c4iw_arm_cq, INIT_RDMA_OBJ_SIZE(ib_pd, c4iw_pd, ibpd), + INIT_RDMA_OBJ_SIZE(ib_cq, c4iw_cq, ibcq), INIT_RDMA_OBJ_SIZE(ib_srq, c4iw_srq, ibsrq), INIT_RDMA_OBJ_SIZE(ib_ucontext, c4iw_ucontext, ibucontext), }; @@ -561,7 +565,6 @@ void c4iw_register_device(struct work_struct *work) pr_debug("c4iw_dev %p\n", dev); memset(&dev->ibdev.node_guid, 0, sizeof(dev->ibdev.node_guid)); memcpy(&dev->ibdev.node_guid, dev->rdev.lldi.ports[0]->dev_addr, 6); - dev->ibdev.owner = THIS_MODULE; dev->device_cap_flags = IB_DEVICE_LOCAL_DMA_LKEY | IB_DEVICE_MEM_WINDOW; if (fastreg_support) dev->device_cap_flags |= IB_DEVICE_MEM_MGT_EXTENSIONS; @@ -594,13 +597,11 @@ void c4iw_register_device(struct work_struct *work) dev->ibdev.phys_port_cnt = dev->rdev.lldi.nports; dev->ibdev.num_comp_vectors = dev->rdev.lldi.nciq; dev->ibdev.dev.parent = &dev->rdev.lldi.pdev->dev; - dev->ibdev.uverbs_abi_ver = C4IW_UVERBS_ABI_VERSION; memcpy(dev->ibdev.iw_ifname, dev->rdev.lldi.ports[0]->name, sizeof(dev->ibdev.iw_ifname)); rdma_set_device_sysfs_group(&dev->ibdev, &c4iw_attr_group); - dev->ibdev.driver_id = RDMA_DRIVER_CXGB4; ib_set_device_ops(&dev->ibdev, &c4iw_dev_ops); ret = set_netdevs(&dev->ibdev, &dev->rdev); if (ret) diff --git a/drivers/infiniband/hw/cxgb4/qp.c b/drivers/infiniband/hw/cxgb4/qp.c index e92b9544357a..eb9368be28c1 100644 --- a/drivers/infiniband/hw/cxgb4/qp.c +++ b/drivers/infiniband/hw/cxgb4/qp.c @@ -274,7 +274,6 @@ static int create_qp(struct c4iw_rdev *rdev, struct t4_wq *wq, (unsigned long long)virt_to_phys(wq->sq.queue), wq->rq.queue, (unsigned long long)virt_to_phys(wq->rq.queue)); - memset(wq->rq.queue, 0, wq->rq.memsize); dma_unmap_addr_set(&wq->rq, mapping, wq->rq.dma_addr); } @@ -303,7 +302,7 @@ static int create_qp(struct c4iw_rdev *rdev, struct t4_wq *wq, wq->rq.msn = 1; /* build fw_ri_res_wr */ - wr_len = sizeof *res_wr + 2 * sizeof *res; + wr_len = sizeof(*res_wr) + 2 * sizeof(*res); if (need_rq) wr_len += sizeof(*res); skb = alloc_skb(wr_len, GFP_KERNEL); @@ -439,7 +438,7 @@ static int build_immd(struct t4_sq *sq, struct fw_ri_immd *immdp, rem -= len; } } - len = roundup(plen + sizeof *immdp, 16) - (plen + sizeof *immdp); + len = roundup(plen + sizeof(*immdp), 16) - (plen + sizeof(*immdp)); if (len) memset(dstp, 0, len); immdp->op = FW_RI_DATA_IMMD; @@ -528,7 +527,7 @@ static int build_rdma_send(struct t4_sq *sq, union t4_wr *wqe, T4_MAX_SEND_INLINE, &plen); if (ret) return ret; - size = sizeof wqe->send + sizeof(struct fw_ri_immd) + + size = sizeof(wqe->send) + sizeof(struct fw_ri_immd) + plen; } else { ret = build_isgl((__be64 *)sq->queue, @@ -537,7 +536,7 @@ static int build_rdma_send(struct t4_sq *sq, union t4_wr *wqe, wr->sg_list, wr->num_sge, &plen); if (ret) return ret; - size = sizeof wqe->send + sizeof(struct fw_ri_isgl) + + size = sizeof(wqe->send) + sizeof(struct fw_ri_isgl) + wr->num_sge * sizeof(struct fw_ri_sge); } } else { @@ -545,7 +544,7 @@ static int build_rdma_send(struct t4_sq *sq, union t4_wr *wqe, wqe->send.u.immd_src[0].r1 = 0; wqe->send.u.immd_src[0].r2 = 0; wqe->send.u.immd_src[0].immdlen = 0; - size = sizeof wqe->send + sizeof(struct fw_ri_immd); + size = sizeof(wqe->send) + sizeof(struct fw_ri_immd); plen = 0; } *len16 = DIV_ROUND_UP(size, 16); @@ -579,7 +578,7 @@ static int build_rdma_write(struct t4_sq *sq, union t4_wr *wqe, T4_MAX_WRITE_INLINE, &plen); if (ret) return ret; - size = sizeof wqe->write + sizeof(struct fw_ri_immd) + + size = sizeof(wqe->write) + sizeof(struct fw_ri_immd) + plen; } else { ret = build_isgl((__be64 *)sq->queue, @@ -588,7 +587,7 @@ static int build_rdma_write(struct t4_sq *sq, union t4_wr *wqe, wr->sg_list, wr->num_sge, &plen); if (ret) return ret; - size = sizeof wqe->write + sizeof(struct fw_ri_isgl) + + size = sizeof(wqe->write) + sizeof(struct fw_ri_isgl) + wr->num_sge * sizeof(struct fw_ri_sge); } } else { @@ -596,7 +595,7 @@ static int build_rdma_write(struct t4_sq *sq, union t4_wr *wqe, wqe->write.u.immd_src[0].r1 = 0; wqe->write.u.immd_src[0].r2 = 0; wqe->write.u.immd_src[0].immdlen = 0; - size = sizeof wqe->write + sizeof(struct fw_ri_immd); + size = sizeof(wqe->write) + sizeof(struct fw_ri_immd); plen = 0; } *len16 = DIV_ROUND_UP(size, 16); @@ -683,7 +682,7 @@ static int build_rdma_read(union t4_wr *wqe, const struct ib_send_wr *wr, } wqe->read.r2 = 0; wqe->read.r5 = 0; - *len16 = DIV_ROUND_UP(sizeof wqe->read, 16); + *len16 = DIV_ROUND_UP(sizeof(wqe->read), 16); return 0; } @@ -766,8 +765,8 @@ static int build_rdma_recv(struct c4iw_qp *qhp, union t4_recv_wr *wqe, &wqe->recv.isgl, wr->sg_list, wr->num_sge, NULL); if (ret) return ret; - *len16 = DIV_ROUND_UP(sizeof wqe->recv + - wr->num_sge * sizeof(struct fw_ri_sge), 16); + *len16 = DIV_ROUND_UP( + sizeof(wqe->recv) + wr->num_sge * sizeof(struct fw_ri_sge), 16); return 0; } @@ -886,47 +885,21 @@ static int build_inv_stag(union t4_wr *wqe, const struct ib_send_wr *wr, { wqe->inv.stag_inv = cpu_to_be32(wr->ex.invalidate_rkey); wqe->inv.r2 = 0; - *len16 = DIV_ROUND_UP(sizeof wqe->inv, 16); + *len16 = DIV_ROUND_UP(sizeof(wqe->inv), 16); return 0; } -static void free_qp_work(struct work_struct *work) -{ - struct c4iw_ucontext *ucontext; - struct c4iw_qp *qhp; - struct c4iw_dev *rhp; - - qhp = container_of(work, struct c4iw_qp, free_work); - ucontext = qhp->ucontext; - rhp = qhp->rhp; - - pr_debug("qhp %p ucontext %p\n", qhp, ucontext); - destroy_qp(&rhp->rdev, &qhp->wq, - ucontext ? &ucontext->uctx : &rhp->rdev.uctx, !qhp->srq); - - c4iw_put_wr_wait(qhp->wr_waitp); - kfree(qhp); -} - -static void queue_qp_free(struct kref *kref) -{ - struct c4iw_qp *qhp; - - qhp = container_of(kref, struct c4iw_qp, kref); - pr_debug("qhp %p\n", qhp); - queue_work(qhp->rhp->rdev.free_workq, &qhp->free_work); -} - void c4iw_qp_add_ref(struct ib_qp *qp) { pr_debug("ib_qp %p\n", qp); - kref_get(&to_c4iw_qp(qp)->kref); + refcount_inc(&to_c4iw_qp(qp)->qp_refcnt); } void c4iw_qp_rem_ref(struct ib_qp *qp) { pr_debug("ib_qp %p\n", qp); - kref_put(&to_c4iw_qp(qp)->kref, queue_qp_free); + if (refcount_dec_and_test(&to_c4iw_qp(qp)->qp_refcnt)) + complete(&to_c4iw_qp(qp)->qp_rel_comp); } static void add_to_fc_list(struct list_head *head, struct list_head *entry) @@ -1606,7 +1579,7 @@ static void post_terminate(struct c4iw_qp *qhp, struct t4_cqe *err_cqe, FW_WR_LEN16_V(DIV_ROUND_UP(sizeof(*wqe), 16))); wqe->u.terminate.type = FW_RI_TYPE_TERMINATE; - wqe->u.terminate.immdlen = cpu_to_be32(sizeof *term); + wqe->u.terminate.immdlen = cpu_to_be32(sizeof(*term)); term = (struct terminate_message *)wqe->u.terminate.termmsg; if (qhp->attr.layer_etype == (LAYER_MPA|DDP_LLP)) { term->layer_etype = qhp->attr.layer_etype; @@ -1751,16 +1724,15 @@ static int rdma_fini(struct c4iw_dev *rhp, struct c4iw_qp *qhp, static void build_rtr_msg(u8 p2p_type, struct fw_ri_init *init) { pr_debug("p2p_type = %d\n", p2p_type); - memset(&init->u, 0, sizeof init->u); + memset(&init->u, 0, sizeof(init->u)); switch (p2p_type) { case FW_RI_INIT_P2PTYPE_RDMA_WRITE: init->u.write.opcode = FW_RI_RDMA_WRITE_WR; init->u.write.stag_sink = cpu_to_be32(1); init->u.write.to_sink = cpu_to_be64(1); init->u.write.u.immd_src[0].op = FW_RI_DATA_IMMD; - init->u.write.len16 = DIV_ROUND_UP(sizeof init->u.write + - sizeof(struct fw_ri_immd), - 16); + init->u.write.len16 = DIV_ROUND_UP( + sizeof(init->u.write) + sizeof(struct fw_ri_immd), 16); break; case FW_RI_INIT_P2PTYPE_READ_REQ: init->u.write.opcode = FW_RI_RDMA_READ_WR; @@ -1768,7 +1740,7 @@ static void build_rtr_msg(u8 p2p_type, struct fw_ri_init *init) init->u.read.to_src_lo = cpu_to_be32(1); init->u.read.stag_sink = cpu_to_be32(1); init->u.read.to_sink_lo = cpu_to_be32(1); - init->u.read.len16 = DIV_ROUND_UP(sizeof init->u.read, 16); + init->u.read.len16 = DIV_ROUND_UP(sizeof(init->u.read), 16); break; } } @@ -1782,7 +1754,7 @@ static int rdma_init(struct c4iw_dev *rhp, struct c4iw_qp *qhp) pr_debug("qhp %p qid 0x%x tid %u ird %u ord %u\n", qhp, qhp->wq.sq.qid, qhp->ep->hwtid, qhp->ep->ird, qhp->ep->ord); - skb = alloc_skb(sizeof *wqe, GFP_KERNEL); + skb = alloc_skb(sizeof(*wqe), GFP_KERNEL); if (!skb) { ret = -ENOMEM; goto out; @@ -2099,10 +2071,12 @@ int c4iw_destroy_qp(struct ib_qp *ib_qp, struct ib_udata *udata) { struct c4iw_dev *rhp; struct c4iw_qp *qhp; + struct c4iw_ucontext *ucontext; struct c4iw_qp_attributes attrs; qhp = to_c4iw_qp(ib_qp); rhp = qhp->rhp; + ucontext = qhp->ucontext; attrs.next_state = C4IW_QP_STATE_ERROR; if (qhp->attr.state == C4IW_QP_STATE_TERMINATE) @@ -2120,7 +2094,17 @@ int c4iw_destroy_qp(struct ib_qp *ib_qp, struct ib_udata *udata) c4iw_qp_rem_ref(ib_qp); + wait_for_completion(&qhp->qp_rel_comp); + pr_debug("ib_qp %p qpid 0x%0x\n", ib_qp, qhp->wq.sq.qid); + pr_debug("qhp %p ucontext %p\n", qhp, ucontext); + + destroy_qp(&rhp->rdev, &qhp->wq, + ucontext ? &ucontext->uctx : &rhp->rdev.uctx, !qhp->srq); + + c4iw_put_wr_wait(qhp->wr_waitp); + + kfree(qhp); return 0; } @@ -2230,8 +2214,8 @@ struct ib_qp *c4iw_create_qp(struct ib_pd *pd, struct ib_qp_init_attr *attrs, spin_lock_init(&qhp->lock); mutex_init(&qhp->mutex); init_waitqueue_head(&qhp->wait); - kref_init(&qhp->kref); - INIT_WORK(&qhp->free_work, free_qp_work); + init_completion(&qhp->qp_rel_comp); + refcount_set(&qhp->qp_refcnt, 1); ret = xa_insert_irq(&rhp->qps, qhp->wq.sq.qid, qhp, GFP_KERNEL); if (ret) @@ -2302,7 +2286,7 @@ struct ib_qp *c4iw_create_qp(struct ib_pd *pd, struct ib_qp_init_attr *attrs, ucontext->key += PAGE_SIZE; } spin_unlock(&ucontext->mmap_lock); - ret = ib_copy_to_udata(udata, &uresp, sizeof uresp); + ret = ib_copy_to_udata(udata, &uresp, sizeof(uresp)); if (ret) goto err_free_ma_sync_key; sq_key_mm->key = uresp.sq_key; @@ -2386,7 +2370,7 @@ int c4iw_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, struct c4iw_dev *rhp; struct c4iw_qp *qhp; enum c4iw_qp_attr_mask mask = 0; - struct c4iw_qp_attributes attrs; + struct c4iw_qp_attributes attrs = {}; pr_debug("ib_qp %p\n", ibqp); @@ -2398,7 +2382,6 @@ int c4iw_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, if (!attr_mask) return 0; - memset(&attrs, 0, sizeof attrs); qhp = to_c4iw_qp(ibqp); rhp = qhp->rhp; @@ -2482,8 +2465,8 @@ int c4iw_ib_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, { struct c4iw_qp *qhp = to_c4iw_qp(ibqp); - memset(attr, 0, sizeof *attr); - memset(init_attr, 0, sizeof *init_attr); + memset(attr, 0, sizeof(*attr)); + memset(init_attr, 0, sizeof(*init_attr)); attr->qp_state = to_ib_qp_state(qhp->attr.state); init_attr->cap.max_send_wr = qhp->attr.sq_num_entries; init_attr->cap.max_recv_wr = qhp->attr.rq_num_entries; diff --git a/drivers/infiniband/hw/cxgb4/resource.c b/drivers/infiniband/hw/cxgb4/resource.c index 57ed26b3cc21..5c95c789f302 100644 --- a/drivers/infiniband/hw/cxgb4/resource.c +++ b/drivers/infiniband/hw/cxgb4/resource.c @@ -126,7 +126,7 @@ u32 c4iw_get_cqid(struct c4iw_rdev *rdev, struct c4iw_dev_ucontext *uctx) rdev->stats.qid.cur += rdev->qpmask + 1; mutex_unlock(&rdev->stats.lock); for (i = qid+1; i & rdev->qpmask; i++) { - entry = kmalloc(sizeof *entry, GFP_KERNEL); + entry = kmalloc(sizeof(*entry), GFP_KERNEL); if (!entry) goto out; entry->qid = i; @@ -137,13 +137,13 @@ u32 c4iw_get_cqid(struct c4iw_rdev *rdev, struct c4iw_dev_ucontext *uctx) * now put the same ids on the qp list since they all * map to the same db/gts page. */ - entry = kmalloc(sizeof *entry, GFP_KERNEL); + entry = kmalloc(sizeof(*entry), GFP_KERNEL); if (!entry) goto out; entry->qid = qid; list_add_tail(&entry->entry, &uctx->qpids); for (i = qid+1; i & rdev->qpmask; i++) { - entry = kmalloc(sizeof *entry, GFP_KERNEL); + entry = kmalloc(sizeof(*entry), GFP_KERNEL); if (!entry) goto out; entry->qid = i; @@ -165,7 +165,7 @@ void c4iw_put_cqid(struct c4iw_rdev *rdev, u32 qid, { struct c4iw_qid_list *entry; - entry = kmalloc(sizeof *entry, GFP_KERNEL); + entry = kmalloc(sizeof(*entry), GFP_KERNEL); if (!entry) return; pr_debug("qid 0x%x\n", qid); @@ -200,7 +200,7 @@ u32 c4iw_get_qpid(struct c4iw_rdev *rdev, struct c4iw_dev_ucontext *uctx) rdev->stats.qid.cur += rdev->qpmask + 1; mutex_unlock(&rdev->stats.lock); for (i = qid+1; i & rdev->qpmask; i++) { - entry = kmalloc(sizeof *entry, GFP_KERNEL); + entry = kmalloc(sizeof(*entry), GFP_KERNEL); if (!entry) goto out; entry->qid = i; @@ -211,13 +211,13 @@ u32 c4iw_get_qpid(struct c4iw_rdev *rdev, struct c4iw_dev_ucontext *uctx) * now put the same ids on the cq list since they all * map to the same db/gts page. */ - entry = kmalloc(sizeof *entry, GFP_KERNEL); + entry = kmalloc(sizeof(*entry), GFP_KERNEL); if (!entry) goto out; entry->qid = qid; list_add_tail(&entry->entry, &uctx->cqids); for (i = qid; i & rdev->qpmask; i++) { - entry = kmalloc(sizeof *entry, GFP_KERNEL); + entry = kmalloc(sizeof(*entry), GFP_KERNEL); if (!entry) goto out; entry->qid = i; @@ -239,7 +239,7 @@ void c4iw_put_qpid(struct c4iw_rdev *rdev, u32 qid, { struct c4iw_qid_list *entry; - entry = kmalloc(sizeof *entry, GFP_KERNEL); + entry = kmalloc(sizeof(*entry), GFP_KERNEL); if (!entry) return; pr_debug("qid 0x%x\n", qid); diff --git a/drivers/infiniband/hw/efa/efa.h b/drivers/infiniband/hw/efa/efa.h index 9e3cc3239c13..119f8efec564 100644 --- a/drivers/infiniband/hw/efa/efa.h +++ b/drivers/infiniband/hw/efa/efa.h @@ -7,10 +7,8 @@ #define _EFA_H_ #include <linux/bitops.h> -#include <linux/idr.h> #include <linux/interrupt.h> #include <linux/pci.h> -#include <linux/sched.h> #include <rdma/efa-abi.h> #include <rdma/ib_verbs.h> @@ -136,10 +134,9 @@ int efa_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata); struct ib_qp *efa_create_qp(struct ib_pd *ibpd, struct ib_qp_init_attr *init_attr, struct ib_udata *udata); -int efa_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata); -struct ib_cq *efa_create_cq(struct ib_device *ibdev, - const struct ib_cq_init_attr *attr, - struct ib_udata *udata); +void efa_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata); +int efa_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, + struct ib_udata *udata); struct ib_mr *efa_reg_mr(struct ib_pd *ibpd, u64 start, u64 length, u64 virt_addr, int access_flags, struct ib_udata *udata); diff --git a/drivers/infiniband/hw/efa/efa_com.c b/drivers/infiniband/hw/efa/efa_com.c index a5c788741a04..2cb42484b0f8 100644 --- a/drivers/infiniband/hw/efa/efa_com.c +++ b/drivers/infiniband/hw/efa/efa_com.c @@ -39,8 +39,6 @@ enum efa_cmd_status { EFA_CMD_SUBMITTED, EFA_CMD_COMPLETED, - /* Abort - canceled by the driver */ - EFA_CMD_ABORTED, }; struct efa_comp_ctx { @@ -280,36 +278,34 @@ static void efa_com_dealloc_ctx_id(struct efa_com_admin_queue *aq, static inline void efa_com_put_comp_ctx(struct efa_com_admin_queue *aq, struct efa_comp_ctx *comp_ctx) { - u16 comp_id = comp_ctx->user_cqe->acq_common_descriptor.command & - EFA_ADMIN_ACQ_COMMON_DESC_COMMAND_ID_MASK; + u16 cmd_id = comp_ctx->user_cqe->acq_common_descriptor.command & + EFA_ADMIN_ACQ_COMMON_DESC_COMMAND_ID_MASK; + u16 ctx_id = cmd_id & (aq->depth - 1); - ibdev_dbg(aq->efa_dev, "Putting completion command_id %d\n", comp_id); + ibdev_dbg(aq->efa_dev, "Put completion command_id %#x\n", cmd_id); comp_ctx->occupied = 0; - efa_com_dealloc_ctx_id(aq, comp_id); + efa_com_dealloc_ctx_id(aq, ctx_id); } static struct efa_comp_ctx *efa_com_get_comp_ctx(struct efa_com_admin_queue *aq, - u16 command_id, bool capture) + u16 cmd_id, bool capture) { - if (command_id >= aq->depth) { - ibdev_err(aq->efa_dev, - "command id is larger than the queue size. cmd_id: %u queue size %d\n", - command_id, aq->depth); - return NULL; - } + u16 ctx_id = cmd_id & (aq->depth - 1); - if (aq->comp_ctx[command_id].occupied && capture) { - ibdev_err(aq->efa_dev, "Completion context is occupied\n"); + if (aq->comp_ctx[ctx_id].occupied && capture) { + ibdev_err(aq->efa_dev, + "Completion context for command_id %#x is occupied\n", + cmd_id); return NULL; } if (capture) { - aq->comp_ctx[command_id].occupied = 1; - ibdev_dbg(aq->efa_dev, "Taking completion ctxt command_id %d\n", - command_id); + aq->comp_ctx[ctx_id].occupied = 1; + ibdev_dbg(aq->efa_dev, + "Take completion ctxt for command_id %#x\n", cmd_id); } - return &aq->comp_ctx[command_id]; + return &aq->comp_ctx[ctx_id]; } static struct efa_comp_ctx *__efa_com_submit_admin_cmd(struct efa_com_admin_queue *aq, @@ -320,6 +316,7 @@ static struct efa_comp_ctx *__efa_com_submit_admin_cmd(struct efa_com_admin_queu { struct efa_comp_ctx *comp_ctx; u16 queue_size_mask; + u16 cmd_id; u16 ctx_id; u16 pi; @@ -328,13 +325,16 @@ static struct efa_comp_ctx *__efa_com_submit_admin_cmd(struct efa_com_admin_queu ctx_id = efa_com_alloc_ctx_id(aq); + /* cmd_id LSBs are the ctx_id and MSBs are entropy bits from pc */ + cmd_id = ctx_id & queue_size_mask; + cmd_id |= aq->sq.pc & ~queue_size_mask; + cmd_id &= EFA_ADMIN_AQ_COMMON_DESC_COMMAND_ID_MASK; + + cmd->aq_common_descriptor.command_id = cmd_id; cmd->aq_common_descriptor.flags |= aq->sq.phase & EFA_ADMIN_AQ_COMMON_DESC_PHASE_MASK; - cmd->aq_common_descriptor.command_id |= ctx_id & - EFA_ADMIN_AQ_COMMON_DESC_COMMAND_ID_MASK; - - comp_ctx = efa_com_get_comp_ctx(aq, ctx_id, true); + comp_ctx = efa_com_get_comp_ctx(aq, cmd_id, true); if (!comp_ctx) { efa_com_dealloc_ctx_id(aq, ctx_id); return ERR_PTR(-EINVAL); @@ -532,16 +532,6 @@ static int efa_com_wait_and_process_admin_cq_polling(struct efa_comp_ctx *comp_c msleep(aq->poll_interval); } - if (comp_ctx->status == EFA_CMD_ABORTED) { - ibdev_err(aq->efa_dev, "Command was aborted\n"); - atomic64_inc(&aq->stats.aborted_cmd); - err = -ENODEV; - goto out; - } - - WARN_ONCE(comp_ctx->status != EFA_CMD_COMPLETED, - "Invalid completion status %d\n", comp_ctx->status); - err = efa_com_comp_status_to_errno(comp_ctx->comp_status); out: efa_com_put_comp_ctx(aq, comp_ctx); @@ -666,66 +656,6 @@ int efa_com_cmd_exec(struct efa_com_admin_queue *aq, } /** - * efa_com_abort_admin_commands - Abort all the outstanding admin commands. - * @edev: EFA communication layer struct - * - * This method aborts all the outstanding admin commands. - * The caller should then call efa_com_wait_for_abort_completion to make sure - * all the commands were completed. - */ -static void efa_com_abort_admin_commands(struct efa_com_dev *edev) -{ - struct efa_com_admin_queue *aq = &edev->aq; - struct efa_comp_ctx *comp_ctx; - unsigned long flags; - u16 i; - - spin_lock(&aq->sq.lock); - spin_lock_irqsave(&aq->cq.lock, flags); - for (i = 0; i < aq->depth; i++) { - comp_ctx = efa_com_get_comp_ctx(aq, i, false); - if (!comp_ctx) - break; - - comp_ctx->status = EFA_CMD_ABORTED; - - complete(&comp_ctx->wait_event); - } - spin_unlock_irqrestore(&aq->cq.lock, flags); - spin_unlock(&aq->sq.lock); -} - -/** - * efa_com_wait_for_abort_completion - Wait for admin commands abort. - * @edev: EFA communication layer struct - * - * This method wait until all the outstanding admin commands will be completed. - */ -static void efa_com_wait_for_abort_completion(struct efa_com_dev *edev) -{ - struct efa_com_admin_queue *aq = &edev->aq; - int i; - - /* all mine */ - for (i = 0; i < aq->depth; i++) - down(&aq->avail_cmds); - - /* let it go */ - for (i = 0; i < aq->depth; i++) - up(&aq->avail_cmds); -} - -static void efa_com_admin_flush(struct efa_com_dev *edev) -{ - struct efa_com_admin_queue *aq = &edev->aq; - - clear_bit(EFA_AQ_STATE_RUNNING_BIT, &aq->state); - - efa_com_abort_admin_commands(edev); - efa_com_wait_for_abort_completion(edev); -} - -/** * efa_com_admin_destroy - Destroy the admin and the async events queues. * @edev: EFA communication layer struct */ @@ -737,7 +667,7 @@ void efa_com_admin_destroy(struct efa_com_dev *edev) struct efa_com_admin_sq *sq = &aq->sq; u16 size; - efa_com_admin_flush(edev); + clear_bit(EFA_AQ_STATE_RUNNING_BIT, &aq->state); devm_kfree(edev->dmadev, aq->comp_ctx_pool); devm_kfree(edev->dmadev, aq->comp_ctx); diff --git a/drivers/infiniband/hw/efa/efa_com.h b/drivers/infiniband/hw/efa/efa_com.h index 84d96724a74b..c67dd8109d1c 100644 --- a/drivers/infiniband/hw/efa/efa_com.h +++ b/drivers/infiniband/hw/efa/efa_com.h @@ -45,7 +45,6 @@ struct efa_com_admin_sq { /* Don't use anything other than atomic64 */ struct efa_com_stats_admin { - atomic64_t aborted_cmd; atomic64_t submitted_cmd; atomic64_t completed_cmd; atomic64_t no_completion; diff --git a/drivers/infiniband/hw/efa/efa_com_cmd.c b/drivers/infiniband/hw/efa/efa_com_cmd.c index c0016648804c..62345d8abf3c 100644 --- a/drivers/infiniband/hw/efa/efa_com_cmd.c +++ b/drivers/infiniband/hw/efa/efa_com_cmd.c @@ -3,7 +3,6 @@ * Copyright 2018-2019 Amazon.com, Inc. or its affiliates. All rights reserved. */ -#include "efa.h" #include "efa_com.h" #include "efa_com_cmd.h" @@ -57,7 +56,7 @@ int efa_com_create_qp(struct efa_com_dev *edev, res->send_sub_cq_idx = cmd_completion.send_sub_cq_idx; res->recv_sub_cq_idx = cmd_completion.recv_sub_cq_idx; - return err; + return 0; } int efa_com_modify_qp(struct efa_com_dev *edev, @@ -181,7 +180,7 @@ int efa_com_create_cq(struct efa_com_dev *edev, result->cq_idx = cmd_completion.cq_idx; result->actual_depth = params->cq_depth; - return err; + return 0; } int efa_com_destroy_cq(struct efa_com_dev *edev, @@ -307,7 +306,8 @@ int efa_com_create_ah(struct efa_com_dev *edev, (struct efa_admin_acq_entry *)&cmd_completion, sizeof(cmd_completion)); if (err) { - ibdev_err(edev->efa_dev, "Failed to create ah [%d]\n", err); + ibdev_err(edev->efa_dev, "Failed to create ah for %pI6 [%d]\n", + ah_cmd.dest_addr, err); return err; } diff --git a/drivers/infiniband/hw/efa/efa_main.c b/drivers/infiniband/hw/efa/efa_main.c index db974caf1eb1..dd1c6d49466f 100644 --- a/drivers/infiniband/hw/efa/efa_main.c +++ b/drivers/infiniband/hw/efa/efa_main.c @@ -100,7 +100,7 @@ static int efa_request_mgmnt_irq(struct efa_dev *dev) nr_cpumask_bits, &irq->affinity_hint_mask, irq->vector); irq_set_affinity_hint(irq->vector, &irq->affinity_hint_mask); - return err; + return 0; } static void efa_setup_mgmnt_irq(struct efa_dev *dev) @@ -197,6 +197,10 @@ static void efa_stats_init(struct efa_dev *dev) } static const struct ib_device_ops efa_dev_ops = { + .owner = THIS_MODULE, + .driver_id = RDMA_DRIVER_EFA, + .uverbs_abi_ver = EFA_UVERBS_ABI_VERSION, + .alloc_pd = efa_alloc_pd, .alloc_ucontext = efa_alloc_ucontext, .create_ah = efa_create_ah, @@ -220,6 +224,7 @@ static const struct ib_device_ops efa_dev_ops = { .reg_user_mr = efa_reg_mr, INIT_RDMA_OBJ_SIZE(ib_ah, efa_ah, ibah), + INIT_RDMA_OBJ_SIZE(ib_cq, efa_cq, ibcq), INIT_RDMA_OBJ_SIZE(ib_pd, efa_pd, ibpd), INIT_RDMA_OBJ_SIZE(ib_ucontext, efa_ucontext, ibucontext), }; @@ -259,12 +264,10 @@ static int efa_ib_device_add(struct efa_dev *dev) if (err) goto err_release_doorbell_bar; - dev->ibdev.owner = THIS_MODULE; dev->ibdev.node_type = RDMA_NODE_UNSPECIFIED; dev->ibdev.phys_port_cnt = 1; dev->ibdev.num_comp_vectors = 1; dev->ibdev.dev.parent = &pdev->dev; - dev->ibdev.uverbs_abi_ver = EFA_UVERBS_ABI_VERSION; dev->ibdev.uverbs_cmd_mask = (1ull << IB_USER_VERBS_CMD_GET_CONTEXT) | @@ -287,7 +290,6 @@ static int efa_ib_device_add(struct efa_dev *dev) dev->ibdev.uverbs_ex_cmd_mask = (1ull << IB_USER_VERBS_EX_CMD_QUERY_DEVICE); - dev->ibdev.driver_id = RDMA_DRIVER_EFA; ib_set_device_ops(&dev->ibdev, &efa_dev_ops); err = ib_register_device(&dev->ibdev, "efa_%d"); diff --git a/drivers/infiniband/hw/efa/efa_verbs.c b/drivers/infiniband/hw/efa/efa_verbs.c index fb6115244d4c..df77bc312a25 100644 --- a/drivers/infiniband/hw/efa/efa_verbs.c +++ b/drivers/infiniband/hw/efa/efa_verbs.c @@ -447,12 +447,6 @@ void efa_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) struct efa_dev *dev = to_edev(ibpd->device); struct efa_pd *pd = to_epd(ibpd); - if (udata->inlen && - !ib_is_udata_cleared(udata, 0, udata->inlen)) { - ibdev_dbg(&dev->ibdev, "Incompatible ABI params\n"); - return; - } - ibdev_dbg(&dev->ibdev, "Dealloc pd[%d]\n", pd->pdn); efa_pd_dealloc(dev, pd->pdn); } @@ -470,12 +464,6 @@ int efa_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata) struct efa_qp *qp = to_eqp(ibqp); int err; - if (udata->inlen && - !ib_is_udata_cleared(udata, 0, udata->inlen)) { - ibdev_dbg(&dev->ibdev, "Incompatible ABI params\n"); - return -EINVAL; - } - ibdev_dbg(&dev->ibdev, "Destroy qp[%u]\n", ibqp->qp_num); err = efa_destroy_qp_handle(dev, qp->qp_handle); if (err) @@ -870,31 +858,18 @@ static int efa_destroy_cq_idx(struct efa_dev *dev, int cq_idx) return efa_com_destroy_cq(&dev->edev, ¶ms); } -int efa_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata) +void efa_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata) { struct efa_dev *dev = to_edev(ibcq->device); struct efa_cq *cq = to_ecq(ibcq); - int err; - - if (udata->inlen && - !ib_is_udata_cleared(udata, 0, udata->inlen)) { - ibdev_dbg(&dev->ibdev, "Incompatible ABI params\n"); - return -EINVAL; - } ibdev_dbg(&dev->ibdev, "Destroy cq[%d] virt[0x%p] freed: size[%lu], dma[%pad]\n", cq->cq_idx, cq->cpu_addr, cq->size, &cq->dma_addr); - err = efa_destroy_cq_idx(dev, cq->cq_idx); - if (err) - return err; - + efa_destroy_cq_idx(dev, cq->cq_idx); dma_unmap_single(&dev->pdev->dev, cq->dma_addr, cq->size, DMA_FROM_DEVICE); - - kfree(cq); - return 0; } static int cq_mmap_entries_setup(struct efa_dev *dev, struct efa_cq *cq, @@ -910,17 +885,20 @@ static int cq_mmap_entries_setup(struct efa_dev *dev, struct efa_cq *cq, return 0; } -static struct ib_cq *do_create_cq(struct ib_device *ibdev, int entries, - int vector, struct ib_ucontext *ibucontext, - struct ib_udata *udata) +int efa_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, + struct ib_udata *udata) { + struct efa_ucontext *ucontext = rdma_udata_to_drv_context( + udata, struct efa_ucontext, ibucontext); struct efa_ibv_create_cq_resp resp = {}; struct efa_com_create_cq_params params; struct efa_com_create_cq_result result; + struct ib_device *ibdev = ibcq->device; struct efa_dev *dev = to_edev(ibdev); struct efa_ibv_create_cq cmd = {}; + struct efa_cq *cq = to_ecq(ibcq); bool cq_entry_inserted = false; - struct efa_cq *cq; + int entries = attr->cqe; int err; ibdev_dbg(ibdev, "create_cq entries %d\n", entries); @@ -978,19 +956,13 @@ static struct ib_cq *do_create_cq(struct ib_device *ibdev, int entries, goto err_out; } - cq = kzalloc(sizeof(*cq), GFP_KERNEL); - if (!cq) { - err = -ENOMEM; - goto err_out; - } - - cq->ucontext = to_eucontext(ibucontext); + cq->ucontext = ucontext; cq->size = PAGE_ALIGN(cmd.cq_entry_size * entries * cmd.num_sub_cqs); cq->cpu_addr = efa_zalloc_mapped(dev, &cq->dma_addr, cq->size, DMA_FROM_DEVICE); if (!cq->cpu_addr) { err = -ENOMEM; - goto err_free_cq; + goto err_out; } params.uarn = cq->ucontext->uarn; @@ -1009,8 +981,8 @@ static struct ib_cq *do_create_cq(struct ib_device *ibdev, int entries, err = cq_mmap_entries_setup(dev, cq, &resp); if (err) { - ibdev_dbg(ibdev, - "Could not setup cq[%u] mmap entries\n", cq->cq_idx); + ibdev_dbg(ibdev, "Could not setup cq[%u] mmap entries\n", + cq->cq_idx); goto err_destroy_cq; } @@ -1026,11 +998,10 @@ static struct ib_cq *do_create_cq(struct ib_device *ibdev, int entries, } } - ibdev_dbg(ibdev, - "Created cq[%d], cq depth[%u]. dma[%pad] virt[0x%p]\n", + ibdev_dbg(ibdev, "Created cq[%d], cq depth[%u]. dma[%pad] virt[0x%p]\n", cq->cq_idx, result.actual_depth, &cq->dma_addr, cq->cpu_addr); - return &cq->ibcq; + return 0; err_destroy_cq: efa_destroy_cq_idx(dev, cq->cq_idx); @@ -1039,23 +1010,9 @@ err_free_mapped: DMA_FROM_DEVICE); if (!cq_entry_inserted) free_pages_exact(cq->cpu_addr, cq->size); -err_free_cq: - kfree(cq); err_out: atomic64_inc(&dev->stats.sw_stats.create_cq_err); - return ERR_PTR(err); -} - -struct ib_cq *efa_create_cq(struct ib_device *ibdev, - const struct ib_cq_init_attr *attr, - struct ib_udata *udata) -{ - struct efa_ucontext *ucontext = rdma_udata_to_drv_context(udata, - struct efa_ucontext, - ibucontext); - - return do_create_cq(ibdev, attr->cqe, attr->comp_vector, - &ucontext->ibucontext, udata); + return err; } static int umem_to_page_list(struct efa_dev *dev, @@ -1065,21 +1022,15 @@ static int umem_to_page_list(struct efa_dev *dev, u8 hp_shift) { u32 pages_in_hp = BIT(hp_shift - PAGE_SHIFT); - struct sg_dma_page_iter sg_iter; - unsigned int page_idx = 0; + struct ib_block_iter biter; unsigned int hp_idx = 0; ibdev_dbg(&dev->ibdev, "hp_cnt[%u], pages_in_hp[%u]\n", hp_cnt, pages_in_hp); - for_each_sg_dma_page(umem->sg_head.sgl, &sg_iter, umem->nmap, 0) { - if (page_idx % pages_in_hp == 0) { - page_list[hp_idx] = sg_page_iter_dma_address(&sg_iter); - hp_idx++; - } - - page_idx++; - } + rdma_for_each_block(umem->sg_head.sgl, &biter, umem->nmap, + BIT(hp_shift)) + page_list[hp_idx++] = rdma_block_iter_dma_address(&biter); return 0; } @@ -1114,14 +1065,14 @@ err: */ static int pbl_chunk_list_create(struct efa_dev *dev, struct pbl_context *pbl) { - unsigned int entry, payloads_in_sg, chunk_list_size, chunk_idx, payload_idx; struct pbl_chunk_list *chunk_list = &pbl->phys.indirect.chunk_list; int page_cnt = pbl->phys.indirect.pbl_buf_size_in_pages; struct scatterlist *pages_sgl = pbl->phys.indirect.sgl; + unsigned int chunk_list_size, chunk_idx, payload_idx; int sg_dma_cnt = pbl->phys.indirect.sg_dma_cnt; struct efa_com_ctrl_buff_info *ctrl_buf; u64 *cur_chunk_buf, *prev_chunk_buf; - struct scatterlist *sg; + struct ib_block_iter biter; dma_addr_t dma_addr; int i; @@ -1155,18 +1106,15 @@ static int pbl_chunk_list_create(struct efa_dev *dev, struct pbl_context *pbl) chunk_idx = 0; payload_idx = 0; cur_chunk_buf = chunk_list->chunks[0].buf; - for_each_sg(pages_sgl, sg, sg_dma_cnt, entry) { - payloads_in_sg = sg_dma_len(sg) >> EFA_CHUNK_PAYLOAD_SHIFT; - for (i = 0; i < payloads_in_sg; i++) { - cur_chunk_buf[payload_idx++] = - (sg_dma_address(sg) & ~(EFA_CHUNK_PAYLOAD_SIZE - 1)) + - (EFA_CHUNK_PAYLOAD_SIZE * i); - - if (payload_idx == EFA_PTRS_PER_CHUNK) { - chunk_idx++; - cur_chunk_buf = chunk_list->chunks[chunk_idx].buf; - payload_idx = 0; - } + rdma_for_each_block(pages_sgl, &biter, sg_dma_cnt, + EFA_CHUNK_PAYLOAD_SIZE) { + cur_chunk_buf[payload_idx++] = + rdma_block_iter_dma_address(&biter); + + if (payload_idx == EFA_PTRS_PER_CHUNK) { + chunk_idx++; + cur_chunk_buf = chunk_list->chunks[chunk_idx].buf; + payload_idx = 0; } } @@ -1314,30 +1262,30 @@ static int pbl_create(struct efa_dev *dev, int err; pbl->pbl_buf_size_in_bytes = hp_cnt * EFA_CHUNK_PAYLOAD_PTR_SIZE; - pbl->pbl_buf = kzalloc(pbl->pbl_buf_size_in_bytes, - GFP_KERNEL | __GFP_NOWARN); - if (pbl->pbl_buf) { - pbl->physically_continuous = 1; + pbl->pbl_buf = kvzalloc(pbl->pbl_buf_size_in_bytes, GFP_KERNEL); + if (!pbl->pbl_buf) + return -ENOMEM; + + if (is_vmalloc_addr(pbl->pbl_buf)) { + pbl->physically_continuous = 0; err = umem_to_page_list(dev, umem, pbl->pbl_buf, hp_cnt, hp_shift); if (err) - goto err_continuous; - err = pbl_continuous_initialize(dev, pbl); + goto err_free; + + err = pbl_indirect_initialize(dev, pbl); if (err) - goto err_continuous; + goto err_free; } else { - pbl->physically_continuous = 0; - pbl->pbl_buf = vzalloc(pbl->pbl_buf_size_in_bytes); - if (!pbl->pbl_buf) - return -ENOMEM; - + pbl->physically_continuous = 1; err = umem_to_page_list(dev, umem, pbl->pbl_buf, hp_cnt, hp_shift); if (err) - goto err_indirect; - err = pbl_indirect_initialize(dev, pbl); + goto err_free; + + err = pbl_continuous_initialize(dev, pbl); if (err) - goto err_indirect; + goto err_free; } ibdev_dbg(&dev->ibdev, @@ -1346,24 +1294,20 @@ static int pbl_create(struct efa_dev *dev, return 0; -err_continuous: - kfree(pbl->pbl_buf); - return err; -err_indirect: - vfree(pbl->pbl_buf); +err_free: + kvfree(pbl->pbl_buf); return err; } static void pbl_destroy(struct efa_dev *dev, struct pbl_context *pbl) { - if (pbl->physically_continuous) { + if (pbl->physically_continuous) dma_unmap_single(&dev->pdev->dev, pbl->phys.continuous.dma_addr, pbl->pbl_buf_size_in_bytes, DMA_TO_DEVICE); - kfree(pbl->pbl_buf); - } else { + else pbl_indirect_terminate(dev, pbl); - vfree(pbl->pbl_buf); - } + + kvfree(pbl->pbl_buf); } static int efa_create_inline_pbl(struct efa_dev *dev, struct efa_mr *mr, @@ -1417,56 +1361,6 @@ static int efa_create_pbl(struct efa_dev *dev, return 0; } -static void efa_cont_pages(struct ib_umem *umem, u64 addr, - unsigned long max_page_shift, - int *count, u8 *shift, u32 *ncont) -{ - struct scatterlist *sg; - u64 base = ~0, p = 0; - unsigned long tmp; - unsigned long m; - u64 len, pfn; - int i = 0; - int entry; - - addr = addr >> PAGE_SHIFT; - tmp = (unsigned long)addr; - m = find_first_bit(&tmp, BITS_PER_LONG); - if (max_page_shift) - m = min_t(unsigned long, max_page_shift - PAGE_SHIFT, m); - - for_each_sg(umem->sg_head.sgl, sg, umem->nmap, entry) { - len = DIV_ROUND_UP(sg_dma_len(sg), PAGE_SIZE); - pfn = sg_dma_address(sg) >> PAGE_SHIFT; - if (base + p != pfn) { - /* - * If either the offset or the new - * base are unaligned update m - */ - tmp = (unsigned long)(pfn | p); - if (!IS_ALIGNED(tmp, 1 << m)) - m = find_first_bit(&tmp, BITS_PER_LONG); - - base = pfn; - p = 0; - } - - p += len; - i += len; - } - - if (i) { - m = min_t(unsigned long, ilog2(roundup_pow_of_two(i)), m); - *ncont = DIV_ROUND_UP(i, (1 << m)); - } else { - m = 0; - *ncont = 0; - } - - *shift = PAGE_SHIFT + m; - *count = i; -} - struct ib_mr *efa_reg_mr(struct ib_pd *ibpd, u64 start, u64 length, u64 virt_addr, int access_flags, struct ib_udata *udata) @@ -1474,11 +1368,10 @@ struct ib_mr *efa_reg_mr(struct ib_pd *ibpd, u64 start, u64 length, struct efa_dev *dev = to_edev(ibpd->device); struct efa_com_reg_mr_params params = {}; struct efa_com_reg_mr_result result = {}; - unsigned long max_page_shift; struct pbl_context pbl; + unsigned int pg_sz; struct efa_mr *mr; int inline_size; - int npages; int err; if (udata->inlen && @@ -1515,13 +1408,24 @@ struct ib_mr *efa_reg_mr(struct ib_pd *ibpd, u64 start, u64 length, params.iova = virt_addr; params.mr_length_in_bytes = length; params.permissions = access_flags & 0x1; - max_page_shift = fls64(dev->dev_attr.page_size_cap); - efa_cont_pages(mr->umem, start, max_page_shift, &npages, - ¶ms.page_shift, ¶ms.page_num); + pg_sz = ib_umem_find_best_pgsz(mr->umem, + dev->dev_attr.page_size_cap, + virt_addr); + if (!pg_sz) { + err = -EOPNOTSUPP; + ibdev_dbg(&dev->ibdev, "Failed to find a suitable page size in page_size_cap %#llx\n", + dev->dev_attr.page_size_cap); + goto err_unmap; + } + + params.page_shift = __ffs(pg_sz); + params.page_num = DIV_ROUND_UP(length + (start & (pg_sz - 1)), + pg_sz); + ibdev_dbg(&dev->ibdev, - "start %#llx length %#llx npages %d params.page_shift %u params.page_num %u\n", - start, length, npages, params.page_shift, params.page_num); + "start %#llx length %#llx params.page_shift %u params.page_num %u\n", + start, length, params.page_shift, params.page_num); inline_size = ARRAY_SIZE(params.pbl.inline_pbl_array); if (params.page_num <= inline_size) { @@ -1567,12 +1471,6 @@ int efa_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata) struct efa_mr *mr = to_emr(ibmr); int err; - if (udata->inlen && - !ib_is_udata_cleared(udata, 0, udata->inlen)) { - ibdev_dbg(&dev->ibdev, "Incompatible ABI params\n"); - return -EINVAL; - } - ibdev_dbg(&dev->ibdev, "Deregister mr[%d]\n", ibmr->lkey); if (mr->umem) { @@ -1580,8 +1478,8 @@ int efa_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata) err = efa_com_dereg_mr(&dev->edev, ¶ms); if (err) return err; - ib_umem_release(mr->umem); } + ib_umem_release(mr->umem); kfree(mr); @@ -1707,13 +1605,15 @@ static int __efa_mmap(struct efa_dev *dev, struct efa_ucontext *ucontext, err = -EINVAL; } - if (err) + if (err) { ibdev_dbg( &dev->ibdev, "Couldn't mmap address[%#llx] length[%#llx] mmap_flag[%d] err[%d]\n", entry->address, length, entry->mmap_flag, err); + return err; + } - return err; + return 0; } int efa_mmap(struct ib_ucontext *ibucontext, diff --git a/drivers/infiniband/hw/hfi1/Makefile b/drivers/infiniband/hw/hfi1/Makefile index 4044a8c8dbf4..0405d26d0833 100644 --- a/drivers/infiniband/hw/hfi1/Makefile +++ b/drivers/infiniband/hw/hfi1/Makefile @@ -10,6 +10,7 @@ obj-$(CONFIG_INFINIBAND_HFI1) += hfi1.o hfi1-y := \ affinity.o \ + aspm.o \ chip.o \ device.o \ driver.o \ diff --git a/drivers/infiniband/hw/hfi1/aspm.c b/drivers/infiniband/hw/hfi1/aspm.c new file mode 100644 index 000000000000..a3c53be4072c --- /dev/null +++ b/drivers/infiniband/hw/hfi1/aspm.c @@ -0,0 +1,270 @@ +// SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause) +/* + * Copyright(c) 2019 Intel Corporation. + * + */ + +#include "aspm.h" + +/* Time after which the timer interrupt will re-enable ASPM */ +#define ASPM_TIMER_MS 1000 +/* Time for which interrupts are ignored after a timer has been scheduled */ +#define ASPM_RESCHED_TIMER_MS (ASPM_TIMER_MS / 2) +/* Two interrupts within this time trigger ASPM disable */ +#define ASPM_TRIGGER_MS 1 +#define ASPM_TRIGGER_NS (ASPM_TRIGGER_MS * 1000 * 1000ull) +#define ASPM_L1_SUPPORTED(reg) \ + ((((reg) & PCI_EXP_LNKCAP_ASPMS) >> 10) & 0x2) + +uint aspm_mode = ASPM_MODE_DISABLED; +module_param_named(aspm, aspm_mode, uint, 0444); +MODULE_PARM_DESC(aspm, "PCIe ASPM: 0: disable, 1: enable, 2: dynamic"); + +static bool aspm_hw_l1_supported(struct hfi1_devdata *dd) +{ + struct pci_dev *parent = dd->pcidev->bus->self; + u32 up, dn; + + /* + * If the driver does not have access to the upstream component, + * it cannot support ASPM L1 at all. + */ + if (!parent) + return false; + + pcie_capability_read_dword(dd->pcidev, PCI_EXP_LNKCAP, &dn); + dn = ASPM_L1_SUPPORTED(dn); + + pcie_capability_read_dword(parent, PCI_EXP_LNKCAP, &up); + up = ASPM_L1_SUPPORTED(up); + + /* ASPM works on A-step but is reported as not supported */ + return (!!dn || is_ax(dd)) && !!up; +} + +/* Set L1 entrance latency for slower entry to L1 */ +static void aspm_hw_set_l1_ent_latency(struct hfi1_devdata *dd) +{ + u32 l1_ent_lat = 0x4u; + u32 reg32; + + pci_read_config_dword(dd->pcidev, PCIE_CFG_REG_PL3, ®32); + reg32 &= ~PCIE_CFG_REG_PL3_L1_ENT_LATENCY_SMASK; + reg32 |= l1_ent_lat << PCIE_CFG_REG_PL3_L1_ENT_LATENCY_SHIFT; + pci_write_config_dword(dd->pcidev, PCIE_CFG_REG_PL3, reg32); +} + +static void aspm_hw_enable_l1(struct hfi1_devdata *dd) +{ + struct pci_dev *parent = dd->pcidev->bus->self; + + /* + * If the driver does not have access to the upstream component, + * it cannot support ASPM L1 at all. + */ + if (!parent) + return; + + /* Enable ASPM L1 first in upstream component and then downstream */ + pcie_capability_clear_and_set_word(parent, PCI_EXP_LNKCTL, + PCI_EXP_LNKCTL_ASPMC, + PCI_EXP_LNKCTL_ASPM_L1); + pcie_capability_clear_and_set_word(dd->pcidev, PCI_EXP_LNKCTL, + PCI_EXP_LNKCTL_ASPMC, + PCI_EXP_LNKCTL_ASPM_L1); +} + +void aspm_hw_disable_l1(struct hfi1_devdata *dd) +{ + struct pci_dev *parent = dd->pcidev->bus->self; + + /* Disable ASPM L1 first in downstream component and then upstream */ + pcie_capability_clear_and_set_word(dd->pcidev, PCI_EXP_LNKCTL, + PCI_EXP_LNKCTL_ASPMC, 0x0); + if (parent) + pcie_capability_clear_and_set_word(parent, PCI_EXP_LNKCTL, + PCI_EXP_LNKCTL_ASPMC, 0x0); +} + +static void aspm_enable(struct hfi1_devdata *dd) +{ + if (dd->aspm_enabled || aspm_mode == ASPM_MODE_DISABLED || + !dd->aspm_supported) + return; + + aspm_hw_enable_l1(dd); + dd->aspm_enabled = true; +} + +static void aspm_disable(struct hfi1_devdata *dd) +{ + if (!dd->aspm_enabled || aspm_mode == ASPM_MODE_ENABLED) + return; + + aspm_hw_disable_l1(dd); + dd->aspm_enabled = false; +} + +static void aspm_disable_inc(struct hfi1_devdata *dd) +{ + unsigned long flags; + + spin_lock_irqsave(&dd->aspm_lock, flags); + aspm_disable(dd); + atomic_inc(&dd->aspm_disabled_cnt); + spin_unlock_irqrestore(&dd->aspm_lock, flags); +} + +static void aspm_enable_dec(struct hfi1_devdata *dd) +{ + unsigned long flags; + + spin_lock_irqsave(&dd->aspm_lock, flags); + if (atomic_dec_and_test(&dd->aspm_disabled_cnt)) + aspm_enable(dd); + spin_unlock_irqrestore(&dd->aspm_lock, flags); +} + +/* ASPM processing for each receive context interrupt */ +void __aspm_ctx_disable(struct hfi1_ctxtdata *rcd) +{ + bool restart_timer; + bool close_interrupts; + unsigned long flags; + ktime_t now, prev; + + spin_lock_irqsave(&rcd->aspm_lock, flags); + /* PSM contexts are open */ + if (!rcd->aspm_intr_enable) + goto unlock; + + prev = rcd->aspm_ts_last_intr; + now = ktime_get(); + rcd->aspm_ts_last_intr = now; + + /* An interrupt pair close together in time */ + close_interrupts = ktime_to_ns(ktime_sub(now, prev)) < ASPM_TRIGGER_NS; + + /* Don't push out our timer till this much time has elapsed */ + restart_timer = ktime_to_ns(ktime_sub(now, rcd->aspm_ts_timer_sched)) > + ASPM_RESCHED_TIMER_MS * NSEC_PER_MSEC; + restart_timer = restart_timer && close_interrupts; + + /* Disable ASPM and schedule timer */ + if (rcd->aspm_enabled && close_interrupts) { + aspm_disable_inc(rcd->dd); + rcd->aspm_enabled = false; + restart_timer = true; + } + + if (restart_timer) { + mod_timer(&rcd->aspm_timer, + jiffies + msecs_to_jiffies(ASPM_TIMER_MS)); + rcd->aspm_ts_timer_sched = now; + } +unlock: + spin_unlock_irqrestore(&rcd->aspm_lock, flags); +} + +/* Timer function for re-enabling ASPM in the absence of interrupt activity */ +static void aspm_ctx_timer_function(struct timer_list *t) +{ + struct hfi1_ctxtdata *rcd = from_timer(rcd, t, aspm_timer); + unsigned long flags; + + spin_lock_irqsave(&rcd->aspm_lock, flags); + aspm_enable_dec(rcd->dd); + rcd->aspm_enabled = true; + spin_unlock_irqrestore(&rcd->aspm_lock, flags); +} + +/* + * Disable interrupt processing for verbs contexts when PSM or VNIC contexts + * are open. + */ +void aspm_disable_all(struct hfi1_devdata *dd) +{ + struct hfi1_ctxtdata *rcd; + unsigned long flags; + u16 i; + + for (i = 0; i < dd->first_dyn_alloc_ctxt; i++) { + rcd = hfi1_rcd_get_by_index(dd, i); + if (rcd) { + del_timer_sync(&rcd->aspm_timer); + spin_lock_irqsave(&rcd->aspm_lock, flags); + rcd->aspm_intr_enable = false; + spin_unlock_irqrestore(&rcd->aspm_lock, flags); + hfi1_rcd_put(rcd); + } + } + + aspm_disable(dd); + atomic_set(&dd->aspm_disabled_cnt, 0); +} + +/* Re-enable interrupt processing for verbs contexts */ +void aspm_enable_all(struct hfi1_devdata *dd) +{ + struct hfi1_ctxtdata *rcd; + unsigned long flags; + u16 i; + + aspm_enable(dd); + + if (aspm_mode != ASPM_MODE_DYNAMIC) + return; + + for (i = 0; i < dd->first_dyn_alloc_ctxt; i++) { + rcd = hfi1_rcd_get_by_index(dd, i); + if (rcd) { + spin_lock_irqsave(&rcd->aspm_lock, flags); + rcd->aspm_intr_enable = true; + rcd->aspm_enabled = true; + spin_unlock_irqrestore(&rcd->aspm_lock, flags); + hfi1_rcd_put(rcd); + } + } +} + +static void aspm_ctx_init(struct hfi1_ctxtdata *rcd) +{ + spin_lock_init(&rcd->aspm_lock); + timer_setup(&rcd->aspm_timer, aspm_ctx_timer_function, 0); + rcd->aspm_intr_supported = rcd->dd->aspm_supported && + aspm_mode == ASPM_MODE_DYNAMIC && + rcd->ctxt < rcd->dd->first_dyn_alloc_ctxt; +} + +void aspm_init(struct hfi1_devdata *dd) +{ + struct hfi1_ctxtdata *rcd; + u16 i; + + spin_lock_init(&dd->aspm_lock); + dd->aspm_supported = aspm_hw_l1_supported(dd); + + for (i = 0; i < dd->first_dyn_alloc_ctxt; i++) { + rcd = hfi1_rcd_get_by_index(dd, i); + if (rcd) + aspm_ctx_init(rcd); + hfi1_rcd_put(rcd); + } + + /* Start with ASPM disabled */ + aspm_hw_set_l1_ent_latency(dd); + dd->aspm_enabled = false; + aspm_hw_disable_l1(dd); + + /* Now turn on ASPM if configured */ + aspm_enable_all(dd); +} + +void aspm_exit(struct hfi1_devdata *dd) +{ + aspm_disable_all(dd); + + /* Turn on ASPM on exit to conserve power */ + aspm_enable(dd); +} + diff --git a/drivers/infiniband/hw/hfi1/aspm.h b/drivers/infiniband/hw/hfi1/aspm.h index e8133870ee87..75d5d18da3da 100644 --- a/drivers/infiniband/hw/hfi1/aspm.h +++ b/drivers/infiniband/hw/hfi1/aspm.h @@ -57,266 +57,20 @@ enum aspm_mode { ASPM_MODE_DYNAMIC = 2, /* ASPM enabled/disabled dynamically */ }; -/* Time after which the timer interrupt will re-enable ASPM */ -#define ASPM_TIMER_MS 1000 -/* Time for which interrupts are ignored after a timer has been scheduled */ -#define ASPM_RESCHED_TIMER_MS (ASPM_TIMER_MS / 2) -/* Two interrupts within this time trigger ASPM disable */ -#define ASPM_TRIGGER_MS 1 -#define ASPM_TRIGGER_NS (ASPM_TRIGGER_MS * 1000 * 1000ull) -#define ASPM_L1_SUPPORTED(reg) \ - (((reg & PCI_EXP_LNKCAP_ASPMS) >> 10) & 0x2) +void aspm_init(struct hfi1_devdata *dd); +void aspm_exit(struct hfi1_devdata *dd); +void aspm_hw_disable_l1(struct hfi1_devdata *dd); +void __aspm_ctx_disable(struct hfi1_ctxtdata *rcd); +void aspm_disable_all(struct hfi1_devdata *dd); +void aspm_enable_all(struct hfi1_devdata *dd); -static inline bool aspm_hw_l1_supported(struct hfi1_devdata *dd) -{ - struct pci_dev *parent = dd->pcidev->bus->self; - u32 up, dn; - - /* - * If the driver does not have access to the upstream component, - * it cannot support ASPM L1 at all. - */ - if (!parent) - return false; - - pcie_capability_read_dword(dd->pcidev, PCI_EXP_LNKCAP, &dn); - dn = ASPM_L1_SUPPORTED(dn); - - pcie_capability_read_dword(parent, PCI_EXP_LNKCAP, &up); - up = ASPM_L1_SUPPORTED(up); - - /* ASPM works on A-step but is reported as not supported */ - return (!!dn || is_ax(dd)) && !!up; -} - -/* Set L1 entrance latency for slower entry to L1 */ -static inline void aspm_hw_set_l1_ent_latency(struct hfi1_devdata *dd) -{ - u32 l1_ent_lat = 0x4u; - u32 reg32; - - pci_read_config_dword(dd->pcidev, PCIE_CFG_REG_PL3, ®32); - reg32 &= ~PCIE_CFG_REG_PL3_L1_ENT_LATENCY_SMASK; - reg32 |= l1_ent_lat << PCIE_CFG_REG_PL3_L1_ENT_LATENCY_SHIFT; - pci_write_config_dword(dd->pcidev, PCIE_CFG_REG_PL3, reg32); -} - -static inline void aspm_hw_enable_l1(struct hfi1_devdata *dd) -{ - struct pci_dev *parent = dd->pcidev->bus->self; - - /* - * If the driver does not have access to the upstream component, - * it cannot support ASPM L1 at all. - */ - if (!parent) - return; - - /* Enable ASPM L1 first in upstream component and then downstream */ - pcie_capability_clear_and_set_word(parent, PCI_EXP_LNKCTL, - PCI_EXP_LNKCTL_ASPMC, - PCI_EXP_LNKCTL_ASPM_L1); - pcie_capability_clear_and_set_word(dd->pcidev, PCI_EXP_LNKCTL, - PCI_EXP_LNKCTL_ASPMC, - PCI_EXP_LNKCTL_ASPM_L1); -} - -static inline void aspm_hw_disable_l1(struct hfi1_devdata *dd) -{ - struct pci_dev *parent = dd->pcidev->bus->self; - - /* Disable ASPM L1 first in downstream component and then upstream */ - pcie_capability_clear_and_set_word(dd->pcidev, PCI_EXP_LNKCTL, - PCI_EXP_LNKCTL_ASPMC, 0x0); - if (parent) - pcie_capability_clear_and_set_word(parent, PCI_EXP_LNKCTL, - PCI_EXP_LNKCTL_ASPMC, 0x0); -} - -static inline void aspm_enable(struct hfi1_devdata *dd) -{ - if (dd->aspm_enabled || aspm_mode == ASPM_MODE_DISABLED || - !dd->aspm_supported) - return; - - aspm_hw_enable_l1(dd); - dd->aspm_enabled = true; -} - -static inline void aspm_disable(struct hfi1_devdata *dd) -{ - if (!dd->aspm_enabled || aspm_mode == ASPM_MODE_ENABLED) - return; - - aspm_hw_disable_l1(dd); - dd->aspm_enabled = false; -} - -static inline void aspm_disable_inc(struct hfi1_devdata *dd) -{ - unsigned long flags; - - spin_lock_irqsave(&dd->aspm_lock, flags); - aspm_disable(dd); - atomic_inc(&dd->aspm_disabled_cnt); - spin_unlock_irqrestore(&dd->aspm_lock, flags); -} - -static inline void aspm_enable_dec(struct hfi1_devdata *dd) -{ - unsigned long flags; - - spin_lock_irqsave(&dd->aspm_lock, flags); - if (atomic_dec_and_test(&dd->aspm_disabled_cnt)) - aspm_enable(dd); - spin_unlock_irqrestore(&dd->aspm_lock, flags); -} - -/* ASPM processing for each receive context interrupt */ static inline void aspm_ctx_disable(struct hfi1_ctxtdata *rcd) { - bool restart_timer; - bool close_interrupts; - unsigned long flags; - ktime_t now, prev; - /* Quickest exit for minimum impact */ - if (!rcd->aspm_intr_supported) - return; - - spin_lock_irqsave(&rcd->aspm_lock, flags); - /* PSM contexts are open */ - if (!rcd->aspm_intr_enable) - goto unlock; - - prev = rcd->aspm_ts_last_intr; - now = ktime_get(); - rcd->aspm_ts_last_intr = now; - - /* An interrupt pair close together in time */ - close_interrupts = ktime_to_ns(ktime_sub(now, prev)) < ASPM_TRIGGER_NS; - - /* Don't push out our timer till this much time has elapsed */ - restart_timer = ktime_to_ns(ktime_sub(now, rcd->aspm_ts_timer_sched)) > - ASPM_RESCHED_TIMER_MS * NSEC_PER_MSEC; - restart_timer = restart_timer && close_interrupts; - - /* Disable ASPM and schedule timer */ - if (rcd->aspm_enabled && close_interrupts) { - aspm_disable_inc(rcd->dd); - rcd->aspm_enabled = false; - restart_timer = true; - } - - if (restart_timer) { - mod_timer(&rcd->aspm_timer, - jiffies + msecs_to_jiffies(ASPM_TIMER_MS)); - rcd->aspm_ts_timer_sched = now; - } -unlock: - spin_unlock_irqrestore(&rcd->aspm_lock, flags); -} - -/* Timer function for re-enabling ASPM in the absence of interrupt activity */ -static inline void aspm_ctx_timer_function(struct timer_list *t) -{ - struct hfi1_ctxtdata *rcd = from_timer(rcd, t, aspm_timer); - unsigned long flags; - - spin_lock_irqsave(&rcd->aspm_lock, flags); - aspm_enable_dec(rcd->dd); - rcd->aspm_enabled = true; - spin_unlock_irqrestore(&rcd->aspm_lock, flags); -} - -/* - * Disable interrupt processing for verbs contexts when PSM or VNIC contexts - * are open. - */ -static inline void aspm_disable_all(struct hfi1_devdata *dd) -{ - struct hfi1_ctxtdata *rcd; - unsigned long flags; - u16 i; - - for (i = 0; i < dd->first_dyn_alloc_ctxt; i++) { - rcd = hfi1_rcd_get_by_index(dd, i); - if (rcd) { - del_timer_sync(&rcd->aspm_timer); - spin_lock_irqsave(&rcd->aspm_lock, flags); - rcd->aspm_intr_enable = false; - spin_unlock_irqrestore(&rcd->aspm_lock, flags); - hfi1_rcd_put(rcd); - } - } - - aspm_disable(dd); - atomic_set(&dd->aspm_disabled_cnt, 0); -} - -/* Re-enable interrupt processing for verbs contexts */ -static inline void aspm_enable_all(struct hfi1_devdata *dd) -{ - struct hfi1_ctxtdata *rcd; - unsigned long flags; - u16 i; - - aspm_enable(dd); - - if (aspm_mode != ASPM_MODE_DYNAMIC) + if (likely(!rcd->aspm_intr_supported)) return; - for (i = 0; i < dd->first_dyn_alloc_ctxt; i++) { - rcd = hfi1_rcd_get_by_index(dd, i); - if (rcd) { - spin_lock_irqsave(&rcd->aspm_lock, flags); - rcd->aspm_intr_enable = true; - rcd->aspm_enabled = true; - spin_unlock_irqrestore(&rcd->aspm_lock, flags); - hfi1_rcd_put(rcd); - } - } -} - -static inline void aspm_ctx_init(struct hfi1_ctxtdata *rcd) -{ - spin_lock_init(&rcd->aspm_lock); - timer_setup(&rcd->aspm_timer, aspm_ctx_timer_function, 0); - rcd->aspm_intr_supported = rcd->dd->aspm_supported && - aspm_mode == ASPM_MODE_DYNAMIC && - rcd->ctxt < rcd->dd->first_dyn_alloc_ctxt; -} - -static inline void aspm_init(struct hfi1_devdata *dd) -{ - struct hfi1_ctxtdata *rcd; - u16 i; - - spin_lock_init(&dd->aspm_lock); - dd->aspm_supported = aspm_hw_l1_supported(dd); - - for (i = 0; i < dd->first_dyn_alloc_ctxt; i++) { - rcd = hfi1_rcd_get_by_index(dd, i); - if (rcd) - aspm_ctx_init(rcd); - hfi1_rcd_put(rcd); - } - - /* Start with ASPM disabled */ - aspm_hw_set_l1_ent_latency(dd); - dd->aspm_enabled = false; - aspm_hw_disable_l1(dd); - - /* Now turn on ASPM if configured */ - aspm_enable_all(dd); -} - -static inline void aspm_exit(struct hfi1_devdata *dd) -{ - aspm_disable_all(dd); - - /* Turn on ASPM on exit to conserve power */ - aspm_enable(dd); + __aspm_ctx_disable(rcd); } #endif /* _ASPM_H */ diff --git a/drivers/infiniband/hw/hfi1/debugfs.c b/drivers/infiniband/hw/hfi1/debugfs.c index 15efb4a380b2..d268bf9c42ee 100644 --- a/drivers/infiniband/hw/hfi1/debugfs.c +++ b/drivers/infiniband/hw/hfi1/debugfs.c @@ -987,9 +987,6 @@ static int __i2c_debugfs_open(struct inode *in, struct file *fp, u32 target) struct hfi1_pportdata *ppd; int ret; - if (!try_module_get(THIS_MODULE)) - return -ENODEV; - ppd = private2ppd(fp); ret = acquire_chip_resource(ppd->dd, i2c_target(target), 0); @@ -1155,6 +1152,7 @@ static int exprom_wp_debugfs_release(struct inode *in, struct file *fp) { \ .name = nm, \ .ops = { \ + .owner = THIS_MODULE, \ .read = readroutine, \ .write = writeroutine, \ .llseek = generic_file_llseek, \ @@ -1165,6 +1163,7 @@ static int exprom_wp_debugfs_release(struct inode *in, struct file *fp) { \ .name = nm, \ .ops = { \ + .owner = THIS_MODULE, \ .read = readf, \ .write = writef, \ .llseek = generic_file_llseek, \ diff --git a/drivers/infiniband/hw/hfi1/mad.c b/drivers/infiniband/hw/hfi1/mad.c index 4228393e6c4c..184dba3c2828 100644 --- a/drivers/infiniband/hw/hfi1/mad.c +++ b/drivers/infiniband/hw/hfi1/mad.c @@ -2744,8 +2744,7 @@ static int pma_get_opa_portstatus(struct opa_pma_mad *pmp, u16 link_width; u16 link_speed; - response_data_size = sizeof(struct opa_port_status_rsp) + - num_vls * sizeof(struct _vls_pctrs); + response_data_size = struct_size(rsp, vls, num_vls); if (response_data_size > sizeof(pmp->data)) { pmp->mad_hdr.status |= OPA_PM_STATUS_REQUEST_TOO_LARGE; return reply((struct ib_mad_hdr *)pmp); @@ -3014,8 +3013,7 @@ static int pma_get_opa_datacounters(struct opa_pma_mad *pmp, } /* Sanity check */ - response_data_size = sizeof(struct opa_port_data_counters_msg) + - num_vls * sizeof(struct _vls_dctrs); + response_data_size = struct_size(req, port[0].vls, num_vls); if (response_data_size > sizeof(pmp->data)) { pmp->mad_hdr.status |= IB_SMP_INVALID_FIELD; @@ -3232,8 +3230,7 @@ static int pma_get_opa_porterrors(struct opa_pma_mad *pmp, return reply((struct ib_mad_hdr *)pmp); } - response_data_size = sizeof(struct opa_port_error_counters64_msg) + - num_vls * sizeof(struct _vls_ectrs); + response_data_size = struct_size(req, port[0].vls, num_vls); if (response_data_size > sizeof(pmp->data)) { pmp->mad_hdr.status |= IB_SMP_INVALID_FIELD; diff --git a/drivers/infiniband/hw/hfi1/pcie.c b/drivers/infiniband/hw/hfi1/pcie.c index c96d193bb236..61aa5504d7c3 100644 --- a/drivers/infiniband/hw/hfi1/pcie.c +++ b/drivers/infiniband/hw/hfi1/pcie.c @@ -1,5 +1,5 @@ /* - * Copyright(c) 2015 - 2018 Intel Corporation. + * Copyright(c) 2015 - 2019 Intel Corporation. * * This file is provided under a dual BSD/GPLv2 license. When using or * redistributing this file, you may do so under either license. @@ -450,10 +450,6 @@ static int hfi1_pcie_caps; module_param_named(pcie_caps, hfi1_pcie_caps, int, 0444); MODULE_PARM_DESC(pcie_caps, "Max PCIe tuning: Payload (0..3), ReadReq (4..7)"); -uint aspm_mode = ASPM_MODE_DISABLED; -module_param_named(aspm, aspm_mode, uint, 0444); -MODULE_PARM_DESC(aspm, "PCIe ASPM: 0: disable, 1: enable, 2: dynamic"); - /** * tune_pcie_caps() - Code to adjust PCIe capabilities. * @dd: Valid device data structure diff --git a/drivers/infiniband/hw/hfi1/pio.c b/drivers/infiniband/hw/hfi1/pio.c index 4e5c2d1b8cfa..79126b2b14ab 100644 --- a/drivers/infiniband/hw/hfi1/pio.c +++ b/drivers/infiniband/hw/hfi1/pio.c @@ -1594,9 +1594,8 @@ void hfi1_sc_wantpiobuf_intr(struct send_context *sc, u32 needint) else sc_del_credit_return_intr(sc); trace_hfi1_wantpiointr(sc, needint, sc->credit_ctrl); - if (needint) { + if (needint) sc_return_credits(sc); - } } /** diff --git a/drivers/infiniband/hw/hfi1/qp.c b/drivers/infiniband/hw/hfi1/qp.c index 4e0e9fc0a777..f8e733aa3bb8 100644 --- a/drivers/infiniband/hw/hfi1/qp.c +++ b/drivers/infiniband/hw/hfi1/qp.c @@ -1,5 +1,5 @@ /* - * Copyright(c) 2015 - 2018 Intel Corporation. + * Copyright(c) 2015 - 2019 Intel Corporation. * * This file is provided under a dual BSD/GPLv2 license. When using or * redistributing this file, you may do so under either license. @@ -348,7 +348,7 @@ int hfi1_setup_wqe(struct rvt_qp *qp, struct rvt_swqe *wqe, bool *call_send) break; case IB_QPT_GSI: case IB_QPT_UD: - ah = ibah_to_rvtah(wqe->ud_wr.ah); + ah = rvt_get_swqe_ah(wqe); if (wqe->length > (1 << ah->log_pmtu)) return -EINVAL; if (ibp->sl_to_sc[rdma_ah_get_sl(&ah->attr)] == 0xf) @@ -702,8 +702,8 @@ void qp_iter_print(struct seq_file *s, struct rvt_qp_iter *iter) sde ? sde->this_idx : 0, send_context, send_context ? send_context->sw_index : 0, - ibcq_to_rvtcq(qp->ibqp.send_cq)->queue->head, - ibcq_to_rvtcq(qp->ibqp.send_cq)->queue->tail, + ib_cq_head(qp->ibqp.send_cq), + ib_cq_tail(qp->ibqp.send_cq), qp->pid, qp->s_state, qp->s_ack_state, diff --git a/drivers/infiniband/hw/hfi1/rc.c b/drivers/infiniband/hw/hfi1/rc.c index 7c8cfb149da0..0477c14633ab 100644 --- a/drivers/infiniband/hw/hfi1/rc.c +++ b/drivers/infiniband/hw/hfi1/rc.c @@ -1830,23 +1830,14 @@ void hfi1_rc_send_complete(struct rvt_qp *qp, struct hfi1_opa_header *opah) } while (qp->s_last != qp->s_acked) { - u32 s_last; - wqe = rvt_get_swqe_ptr(qp, qp->s_last); if (cmp_psn(wqe->lpsn, qp->s_sending_psn) >= 0 && cmp_psn(qp->s_sending_psn, qp->s_sending_hpsn) <= 0) break; trdma_clean_swqe(qp, wqe); rvt_qp_wqe_unreserve(qp, wqe); - s_last = qp->s_last; - trace_hfi1_qp_send_completion(qp, wqe, s_last); - if (++s_last >= qp->s_size) - s_last = 0; - qp->s_last = s_last; - /* see post_send() */ - barrier(); - rvt_put_qp_swqe(qp, wqe); - rvt_qp_swqe_complete(qp, + trace_hfi1_qp_send_completion(qp, wqe, qp->s_last); + rvt_qp_complete_swqe(qp, wqe, ib_hfi1_wc_opcode[wqe->wr.opcode], IB_WC_SUCCESS); @@ -1890,19 +1881,10 @@ struct rvt_swqe *do_rc_completion(struct rvt_qp *qp, trace_hfi1_rc_completion(qp, wqe->lpsn); if (cmp_psn(wqe->lpsn, qp->s_sending_psn) < 0 || cmp_psn(qp->s_sending_psn, qp->s_sending_hpsn) > 0) { - u32 s_last; - trdma_clean_swqe(qp, wqe); - rvt_put_qp_swqe(qp, wqe); rvt_qp_wqe_unreserve(qp, wqe); - s_last = qp->s_last; - trace_hfi1_qp_send_completion(qp, wqe, s_last); - if (++s_last >= qp->s_size) - s_last = 0; - qp->s_last = s_last; - /* see post_send() */ - barrier(); - rvt_qp_swqe_complete(qp, + trace_hfi1_qp_send_completion(qp, wqe, qp->s_last); + rvt_qp_complete_swqe(qp, wqe, ib_hfi1_wc_opcode[wqe->wr.opcode], IB_WC_SUCCESS); @@ -3026,8 +3008,7 @@ send_last: wc.dlid_path_bits = 0; wc.port_num = 0; /* Signal completion event if the solicited bit is set. */ - rvt_cq_enter(ibcq_to_rvtcq(qp->ibqp.recv_cq), &wc, - ib_bth_is_solicited(ohdr)); + rvt_recv_cq(qp, &wc, ib_bth_is_solicited(ohdr)); break; case OP(RDMA_WRITE_ONLY): diff --git a/drivers/infiniband/hw/hfi1/tid_rdma.c b/drivers/infiniband/hw/hfi1/tid_rdma.c index aa9c8d3ef87b..92acccaaaa86 100644 --- a/drivers/infiniband/hw/hfi1/tid_rdma.c +++ b/drivers/infiniband/hw/hfi1/tid_rdma.c @@ -475,7 +475,7 @@ static struct rvt_qp *first_qp(struct hfi1_ctxtdata *rcd, * Must hold the qp s_lock and the exp_lock. * * Return: - * false if either of the conditions below are statisfied: + * false if either of the conditions below are satisfied: * 1. The list is empty or * 2. The indicated qp is at the head of the list and the * HFI1_S_WAIT_TID_SPACE bit is set in qp->s_flags. @@ -2024,7 +2024,6 @@ static int tid_rdma_rcv_error(struct hfi1_packet *packet, trace_hfi1_tid_req_rcv_err(qp, 0, e->opcode, e->psn, e->lpsn, req); if (e->opcode == TID_OP(READ_REQ)) { struct ib_reth *reth; - u32 offset; u32 len; u32 rkey; u64 vaddr; @@ -2036,7 +2035,6 @@ static int tid_rdma_rcv_error(struct hfi1_packet *packet, * The requester always restarts from the start of the original * request. */ - offset = delta_psn(psn, e->psn) * qp->pmtu; len = be32_to_cpu(reth->length); if (psn != e->psn || len != req->total_len) goto unlock; @@ -4550,7 +4548,7 @@ void hfi1_rc_rcv_tid_rdma_ack(struct hfi1_packet *packet) struct rvt_swqe *wqe; struct tid_rdma_request *req; struct tid_rdma_flow *flow; - u32 aeth, psn, req_psn, ack_psn, fspsn, resync_psn, ack_kpsn; + u32 aeth, psn, req_psn, ack_psn, resync_psn, ack_kpsn; unsigned long flags; u16 fidx; @@ -4754,7 +4752,6 @@ done: IB_AETH_CREDIT_MASK) { case 0: /* PSN sequence error */ flow = &req->flows[req->acked_tail]; - fspsn = full_flow_psn(flow, flow->flow_state.spsn); trace_hfi1_tid_flow_rcv_tid_ack(qp, req->acked_tail, flow); req->r_ack_psn = mask_psn(be32_to_cpu(ohdr->bth[2])); diff --git a/drivers/infiniband/hw/hfi1/trace_ibhdrs.h b/drivers/infiniband/hw/hfi1/trace_ibhdrs.h index d1372cc66de6..2f84290a88ca 100644 --- a/drivers/infiniband/hw/hfi1/trace_ibhdrs.h +++ b/drivers/infiniband/hw/hfi1/trace_ibhdrs.h @@ -79,6 +79,8 @@ __print_symbolic(opcode, \ ib_opcode_name(RC_ATOMIC_ACKNOWLEDGE), \ ib_opcode_name(RC_COMPARE_SWAP), \ ib_opcode_name(RC_FETCH_ADD), \ + ib_opcode_name(RC_SEND_LAST_WITH_INVALIDATE), \ + ib_opcode_name(RC_SEND_ONLY_WITH_INVALIDATE), \ ib_opcode_name(TID_RDMA_WRITE_REQ), \ ib_opcode_name(TID_RDMA_WRITE_RESP), \ ib_opcode_name(TID_RDMA_WRITE_DATA), \ diff --git a/drivers/infiniband/hw/hfi1/uc.c b/drivers/infiniband/hw/hfi1/uc.c index 4ed4fcfabd6c..0c77f18120ed 100644 --- a/drivers/infiniband/hw/hfi1/uc.c +++ b/drivers/infiniband/hw/hfi1/uc.c @@ -476,8 +476,7 @@ last_imm: wc.dlid_path_bits = 0; wc.port_num = 0; /* Signal completion event if the solicited bit is set. */ - rvt_cq_enter(ibcq_to_rvtcq(qp->ibqp.recv_cq), &wc, - ib_bth_is_solicited(ohdr)); + rvt_recv_cq(qp, &wc, ib_bth_is_solicited(ohdr)); break; case OP(RDMA_WRITE_FIRST): diff --git a/drivers/infiniband/hw/hfi1/ud.c b/drivers/infiniband/hw/hfi1/ud.c index 4cb0fce5c096..e804af71b629 100644 --- a/drivers/infiniband/hw/hfi1/ud.c +++ b/drivers/infiniband/hw/hfi1/ud.c @@ -1,5 +1,5 @@ /* - * Copyright(c) 2015 - 2018 Intel Corporation. + * Copyright(c) 2015 - 2019 Intel Corporation. * * This file is provided under a dual BSD/GPLv2 license. When using or * redistributing this file, you may do so under either license. @@ -87,7 +87,7 @@ static void ud_loopback(struct rvt_qp *sqp, struct rvt_swqe *swqe) rcu_read_lock(); qp = rvt_lookup_qpn(ib_to_rvt(sqp->ibqp.device), &ibp->rvp, - swqe->ud_wr.remote_qpn); + rvt_get_swqe_remote_qpn(swqe)); if (!qp) { ibp->rvp.n_pkt_drops++; rcu_read_unlock(); @@ -105,7 +105,7 @@ static void ud_loopback(struct rvt_qp *sqp, struct rvt_swqe *swqe) goto drop; } - ah_attr = &ibah_to_rvtah(swqe->ud_wr.ah)->attr; + ah_attr = rvt_get_swqe_ah_attr(swqe); ppd = ppd_from_ibp(ibp); if (qp->ibqp.qp_num > 1) { @@ -135,8 +135,8 @@ static void ud_loopback(struct rvt_qp *sqp, struct rvt_swqe *swqe) if (qp->ibqp.qp_num) { u32 qkey; - qkey = (int)swqe->ud_wr.remote_qkey < 0 ? - sqp->qkey : swqe->ud_wr.remote_qkey; + qkey = (int)rvt_get_swqe_remote_qkey(swqe) < 0 ? + sqp->qkey : rvt_get_swqe_remote_qkey(swqe); if (unlikely(qkey != qp->qkey)) goto drop; /* silently drop per IBTA spec */ } @@ -240,7 +240,7 @@ static void ud_loopback(struct rvt_qp *sqp, struct rvt_swqe *swqe) if (qp->ibqp.qp_type == IB_QPT_GSI || qp->ibqp.qp_type == IB_QPT_SMI) { if (sqp->ibqp.qp_type == IB_QPT_GSI || sqp->ibqp.qp_type == IB_QPT_SMI) - wc.pkey_index = swqe->ud_wr.pkey_index; + wc.pkey_index = rvt_get_swqe_pkey_index(swqe); else wc.pkey_index = sqp->s_pkey_index; } else { @@ -255,8 +255,7 @@ static void ud_loopback(struct rvt_qp *sqp, struct rvt_swqe *swqe) wc.dlid_path_bits = rdma_ah_get_dlid(ah_attr) & ((1 << ppd->lmc) - 1); wc.port_num = qp->port_num; /* Signal completion event if the solicited bit is set. */ - rvt_cq_enter(ibcq_to_rvtcq(qp->ibqp.recv_cq), &wc, - swqe->wr.send_flags & IB_SEND_SOLICITED); + rvt_recv_cq(qp, &wc, swqe->wr.send_flags & IB_SEND_SOLICITED); ibp->rvp.n_loop_pkts++; bail_unlock: spin_unlock_irqrestore(&qp->r_lock, flags); @@ -283,20 +282,21 @@ static void hfi1_make_bth_deth(struct rvt_qp *qp, struct rvt_swqe *wqe, bth0 |= IB_BTH_SOLICITED; bth0 |= extra_bytes << 20; if (qp->ibqp.qp_type == IB_QPT_GSI || qp->ibqp.qp_type == IB_QPT_SMI) - *pkey = hfi1_get_pkey(ibp, wqe->ud_wr.pkey_index); + *pkey = hfi1_get_pkey(ibp, rvt_get_swqe_pkey_index(wqe)); else *pkey = hfi1_get_pkey(ibp, qp->s_pkey_index); if (!bypass) bth0 |= *pkey; ohdr->bth[0] = cpu_to_be32(bth0); - ohdr->bth[1] = cpu_to_be32(wqe->ud_wr.remote_qpn); + ohdr->bth[1] = cpu_to_be32(rvt_get_swqe_remote_qpn(wqe)); ohdr->bth[2] = cpu_to_be32(mask_psn(wqe->psn)); /* * Qkeys with the high order bit set mean use the * qkey from the QP context instead of the WR (see 10.2.5). */ - ohdr->u.ud.deth[0] = cpu_to_be32((int)wqe->ud_wr.remote_qkey < 0 ? - qp->qkey : wqe->ud_wr.remote_qkey); + ohdr->u.ud.deth[0] = + cpu_to_be32((int)rvt_get_swqe_remote_qkey(wqe) < 0 ? qp->qkey : + rvt_get_swqe_remote_qkey(wqe)); ohdr->u.ud.deth[1] = cpu_to_be32(qp->ibqp.qp_num); } @@ -316,7 +316,7 @@ void hfi1_make_ud_req_9B(struct rvt_qp *qp, struct hfi1_pkt_state *ps, ibp = to_iport(qp->ibqp.device, qp->port_num); ppd = ppd_from_ibp(ibp); - ah_attr = &ibah_to_rvtah(wqe->ud_wr.ah)->attr; + ah_attr = rvt_get_swqe_ah_attr(wqe); extra_bytes = -wqe->length & 3; nwords = ((wqe->length + extra_bytes) >> 2) + SIZE_OF_CRC; @@ -380,7 +380,7 @@ void hfi1_make_ud_req_16B(struct rvt_qp *qp, struct hfi1_pkt_state *ps, struct hfi1_pportdata *ppd; struct hfi1_ibport *ibp; u32 dlid, slid, nwords, extra_bytes; - u32 dest_qp = wqe->ud_wr.remote_qpn; + u32 dest_qp = rvt_get_swqe_remote_qpn(wqe); u32 src_qp = qp->ibqp.qp_num; u16 len, pkey; u8 l4, sc5; @@ -388,7 +388,7 @@ void hfi1_make_ud_req_16B(struct rvt_qp *qp, struct hfi1_pkt_state *ps, ibp = to_iport(qp->ibqp.device, qp->port_num); ppd = ppd_from_ibp(ibp); - ah_attr = &ibah_to_rvtah(wqe->ud_wr.ah)->attr; + ah_attr = rvt_get_swqe_ah_attr(wqe); /* * Build 16B Management Packet if either the destination @@ -450,7 +450,7 @@ void hfi1_make_ud_req_16B(struct rvt_qp *qp, struct hfi1_pkt_state *ps, if (is_mgmt) { l4 = OPA_16B_L4_FM; - pkey = hfi1_get_pkey(ibp, wqe->ud_wr.pkey_index); + pkey = hfi1_get_pkey(ibp, rvt_get_swqe_pkey_index(wqe)); hfi1_16B_set_qpn(&ps->s_txreq->phdr.hdr.opah.u.mgmt, dest_qp, src_qp); } else { @@ -515,7 +515,7 @@ int hfi1_make_ud_req(struct rvt_qp *qp, struct hfi1_pkt_state *ps) /* Construct the header. */ ibp = to_iport(qp->ibqp.device, qp->port_num); ppd = ppd_from_ibp(ibp); - ah_attr = &ibah_to_rvtah(wqe->ud_wr.ah)->attr; + ah_attr = rvt_get_swqe_ah_attr(wqe); priv->hdr_type = hfi1_get_hdr_type(ppd->lid, ah_attr); if ((!hfi1_check_mcast(rdma_ah_get_dlid(ah_attr))) || (rdma_ah_get_dlid(ah_attr) == be32_to_cpu(OPA_LID_PERMISSIVE))) { @@ -1061,7 +1061,7 @@ void hfi1_ud_rcv(struct hfi1_packet *packet) dlid & ((1 << ppd_from_ibp(ibp)->lmc) - 1); wc.port_num = qp->port_num; /* Signal completion event if the solicited bit is set. */ - rvt_cq_enter(ibcq_to_rvtcq(qp->ibqp.recv_cq), &wc, solicited); + rvt_recv_cq(qp, &wc, solicited); return; drop: diff --git a/drivers/infiniband/hw/hfi1/user_pages.c b/drivers/infiniband/hw/hfi1/user_pages.c index 02eee8eff1db..b89a9b9aef7a 100644 --- a/drivers/infiniband/hw/hfi1/user_pages.c +++ b/drivers/infiniband/hw/hfi1/user_pages.c @@ -118,13 +118,10 @@ int hfi1_acquire_user_pages(struct mm_struct *mm, unsigned long vaddr, size_t np void hfi1_release_user_pages(struct mm_struct *mm, struct page **p, size_t npages, bool dirty) { - size_t i; - - for (i = 0; i < npages; i++) { - if (dirty) - set_page_dirty_lock(p[i]); - put_page(p[i]); - } + if (dirty) + put_user_pages_dirty_lock(p, npages); + else + put_user_pages(p, npages); if (mm) { /* during close after signal, mm can be NULL */ atomic64_sub(npages, &mm->pinned_vm); diff --git a/drivers/infiniband/hw/hfi1/verbs.c b/drivers/infiniband/hw/hfi1/verbs.c index bad3229bad37..c4b243f50c76 100644 --- a/drivers/infiniband/hw/hfi1/verbs.c +++ b/drivers/infiniband/hw/hfi1/verbs.c @@ -1779,6 +1779,9 @@ static int get_hw_stats(struct ib_device *ibdev, struct rdma_hw_stats *stats, } static const struct ib_device_ops hfi1_dev_ops = { + .owner = THIS_MODULE, + .driver_id = RDMA_DRIVER_HFI1, + .alloc_hw_stats = alloc_hw_stats, .alloc_rdma_netdev = hfi1_vnic_alloc_rn, .get_dev_fw_str = hfi1_get_dev_fw_str, @@ -1829,7 +1832,6 @@ int hfi1_register_ib_device(struct hfi1_devdata *dd) */ if (!ib_hfi1_sys_image_guid) ib_hfi1_sys_image_guid = ibdev->node_guid; - ibdev->owner = THIS_MODULE; ibdev->phys_port_cnt = dd->num_pports; ibdev->dev.parent = &dd->pcidev->dev; @@ -1923,7 +1925,7 @@ int hfi1_register_ib_device(struct hfi1_devdata *dd) rdma_set_device_sysfs_group(&dd->verbs_dev.rdi.ibdev, &ib_hfi1_attr_group); - ret = rvt_register_device(&dd->verbs_dev.rdi, RDMA_DRIVER_HFI1); + ret = rvt_register_device(&dd->verbs_dev.rdi); if (ret) goto err_verbs_txreq; diff --git a/drivers/infiniband/hw/hns/Kconfig b/drivers/infiniband/hw/hns/Kconfig index 61cda7d00627..8bf847bcd8d3 100644 --- a/drivers/infiniband/hw/hns/Kconfig +++ b/drivers/infiniband/hw/hns/Kconfig @@ -8,25 +8,24 @@ config INFINIBAND_HNS is used in Hisilicon Hip06 and more further ICT SoC based on platform device. - To compile this driver as a module, choose M here: the module - will be called hns-roce. + To compile HIP06 or HIP08 driver as module, choose M here. config INFINIBAND_HNS_HIP06 - tristate "Hisilicon Hip06 Family RoCE support" + bool "Hisilicon Hip06 Family RoCE support" depends on INFINIBAND_HNS && HNS && HNS_DSAF && HNS_ENET ---help--- RoCE driver support for Hisilicon RoCE engine in Hisilicon Hip06 and Hip07 SoC. These RoCE engines are platform devices. - To compile this driver as a module, choose M here: the module - will be called hns-roce-hw-v1. + To compile this driver, choose Y here: if INFINIBAND_HNS is m, this + module will be called hns-roce-hw-v1 config INFINIBAND_HNS_HIP08 - tristate "Hisilicon Hip08 Family RoCE support" + bool "Hisilicon Hip08 Family RoCE support" depends on INFINIBAND_HNS && PCI && HNS3 ---help--- RoCE driver support for Hisilicon RoCE engine in Hisilicon Hip08 SoC. The RoCE engine is a PCI device. - To compile this driver as a module, choose M here: the module - will be called hns-roce-hw-v2. + To compile this driver, choose Y here: if INFINIBAND_HNS is m, this + module will be called hns-roce-hw-v2. diff --git a/drivers/infiniband/hw/hns/Makefile b/drivers/infiniband/hw/hns/Makefile index f22d9922cfee..e105945b94a1 100644 --- a/drivers/infiniband/hw/hns/Makefile +++ b/drivers/infiniband/hw/hns/Makefile @@ -5,11 +5,16 @@ ccflags-y := -I $(srctree)/drivers/net/ethernet/hisilicon/hns3 -obj-$(CONFIG_INFINIBAND_HNS) += hns-roce.o hns-roce-objs := hns_roce_main.o hns_roce_cmd.o hns_roce_pd.o \ hns_roce_ah.o hns_roce_hem.o hns_roce_mr.o hns_roce_qp.o \ hns_roce_cq.o hns_roce_alloc.o hns_roce_db.o hns_roce_srq.o hns_roce_restrack.o -obj-$(CONFIG_INFINIBAND_HNS_HIP06) += hns-roce-hw-v1.o -hns-roce-hw-v1-objs := hns_roce_hw_v1.o -obj-$(CONFIG_INFINIBAND_HNS_HIP08) += hns-roce-hw-v2.o -hns-roce-hw-v2-objs := hns_roce_hw_v2.o hns_roce_hw_v2_dfx.o + +ifdef CONFIG_INFINIBAND_HNS_HIP06 +hns-roce-hw-v1-objs := hns_roce_hw_v1.o $(hns-roce-objs) +obj-$(CONFIG_INFINIBAND_HNS) += hns-roce-hw-v1.o +endif + +ifdef CONFIG_INFINIBAND_HNS_HIP08 +hns-roce-hw-v2-objs := hns_roce_hw_v2.o hns_roce_hw_v2_dfx.o $(hns-roce-objs) +obj-$(CONFIG_INFINIBAND_HNS) += hns-roce-hw-v2.o +endif diff --git a/drivers/infiniband/hw/hns/hns_roce_alloc.c b/drivers/infiniband/hw/hns/hns_roce_alloc.c index dac058d3df53..8c063c598d2a 100644 --- a/drivers/infiniband/hw/hns/hns_roce_alloc.c +++ b/drivers/infiniband/hw/hns/hns_roce_alloc.c @@ -34,6 +34,7 @@ #include <linux/platform_device.h> #include <linux/vmalloc.h> #include "hns_roce_device.h" +#include <rdma/ib_umem.h> int hns_roce_bitmap_alloc(struct hns_roce_bitmap *bitmap, unsigned long *obj) { @@ -67,7 +68,6 @@ void hns_roce_bitmap_free(struct hns_roce_bitmap *bitmap, unsigned long obj, { hns_roce_bitmap_free_range(bitmap, obj, 1, rr); } -EXPORT_SYMBOL_GPL(hns_roce_bitmap_free); int hns_roce_bitmap_alloc_range(struct hns_roce_bitmap *bitmap, int cnt, int align, unsigned long *obj) @@ -174,7 +174,6 @@ void hns_roce_buf_free(struct hns_roce_dev *hr_dev, u32 size, kfree(buf->page_list); } } -EXPORT_SYMBOL_GPL(hns_roce_buf_free); int hns_roce_buf_alloc(struct hns_roce_dev *hr_dev, u32 size, u32 max_direct, struct hns_roce_buf *buf, u32 page_shift) @@ -238,6 +237,104 @@ err_free: return -ENOMEM; } +int hns_roce_get_kmem_bufs(struct hns_roce_dev *hr_dev, dma_addr_t *bufs, + int buf_cnt, int start, struct hns_roce_buf *buf) +{ + int i, end; + int total; + + end = start + buf_cnt; + if (end > buf->npages) { + dev_err(hr_dev->dev, + "invalid kmem region,offset %d,buf_cnt %d,total %d!\n", + start, buf_cnt, buf->npages); + return -EINVAL; + } + + total = 0; + for (i = start; i < end; i++) + if (buf->nbufs == 1) + bufs[total++] = buf->direct.map + + ((dma_addr_t)i << buf->page_shift); + else + bufs[total++] = buf->page_list[i].map; + + return total; +} + +int hns_roce_get_umem_bufs(struct hns_roce_dev *hr_dev, dma_addr_t *bufs, + int buf_cnt, int start, struct ib_umem *umem, + int page_shift) +{ + struct ib_block_iter biter; + int total = 0; + int idx = 0; + u64 addr; + + if (page_shift < PAGE_SHIFT) { + dev_err(hr_dev->dev, "invalid page shift %d!\n", page_shift); + return -EINVAL; + } + + /* convert system page cnt to hw page cnt */ + rdma_for_each_block(umem->sg_head.sgl, &biter, umem->nmap, + 1 << page_shift) { + addr = rdma_block_iter_dma_address(&biter); + if (idx >= start) { + bufs[total++] = addr; + if (total >= buf_cnt) + goto done; + } + idx++; + } + +done: + return total; +} + +void hns_roce_init_buf_region(struct hns_roce_buf_region *region, int hopnum, + int offset, int buf_cnt) +{ + if (hopnum == HNS_ROCE_HOP_NUM_0) + region->hopnum = 0; + else + region->hopnum = hopnum; + + region->offset = offset; + region->count = buf_cnt; +} + +void hns_roce_free_buf_list(dma_addr_t **bufs, int region_cnt) +{ + int i; + + for (i = 0; i < region_cnt; i++) { + kfree(bufs[i]); + bufs[i] = NULL; + } +} + +int hns_roce_alloc_buf_list(struct hns_roce_buf_region *regions, + dma_addr_t **bufs, int region_cnt) +{ + struct hns_roce_buf_region *r; + int i; + + for (i = 0; i < region_cnt; i++) { + r = ®ions[i]; + bufs[i] = kcalloc(r->count, sizeof(dma_addr_t), GFP_KERNEL); + if (!bufs[i]) + goto err_alloc; + } + + return 0; + +err_alloc: + hns_roce_free_buf_list(bufs, i); + + return -ENOMEM; +} + void hns_roce_cleanup_bitmap(struct hns_roce_dev *hr_dev) { if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_SRQ) diff --git a/drivers/infiniband/hw/hns/hns_roce_cmd.c b/drivers/infiniband/hw/hns/hns_roce_cmd.c index 2acf946d02e5..0cd09bf4d7ea 100644 --- a/drivers/infiniband/hw/hns/hns_roce_cmd.c +++ b/drivers/infiniband/hw/hns/hns_roce_cmd.c @@ -103,7 +103,6 @@ void hns_roce_cmd_event(struct hns_roce_dev *hr_dev, u16 token, u8 status, context->out_param = out_param; complete(&context->done); } -EXPORT_SYMBOL_GPL(hns_roce_cmd_event); /* this should be called with "use_events" */ static int __hns_roce_cmd_mbox_wait(struct hns_roce_dev *hr_dev, u64 in_param, @@ -162,7 +161,7 @@ static int hns_roce_cmd_mbox_wait(struct hns_roce_dev *hr_dev, u64 in_param, u64 out_param, unsigned long in_modifier, u8 op_modifier, u16 op, unsigned long timeout) { - int ret = 0; + int ret; down(&hr_dev->cmd.event_sem); ret = __hns_roce_cmd_mbox_wait(hr_dev, in_param, out_param, @@ -204,7 +203,6 @@ int hns_roce_cmd_mbox(struct hns_roce_dev *hr_dev, u64 in_param, u64 out_param, return ret; } -EXPORT_SYMBOL_GPL(hns_roce_cmd_mbox); int hns_roce_cmd_init(struct hns_roce_dev *hr_dev) { @@ -291,7 +289,6 @@ struct hns_roce_cmd_mailbox return mailbox; } -EXPORT_SYMBOL_GPL(hns_roce_alloc_cmd_mailbox); void hns_roce_free_cmd_mailbox(struct hns_roce_dev *hr_dev, struct hns_roce_cmd_mailbox *mailbox) @@ -302,4 +299,3 @@ void hns_roce_free_cmd_mailbox(struct hns_roce_dev *hr_dev, dma_pool_free(hr_dev->cmd.pool, mailbox->buf, mailbox->dma); kfree(mailbox); } -EXPORT_SYMBOL_GPL(hns_roce_free_cmd_mailbox); diff --git a/drivers/infiniband/hw/hns/hns_roce_cq.c b/drivers/infiniband/hw/hns/hns_roce_cq.c index 9caf35061721..4e50c22a2da4 100644 --- a/drivers/infiniband/hw/hns/hns_roce_cq.c +++ b/drivers/infiniband/hw/hns/hns_roce_cq.c @@ -205,7 +205,6 @@ void hns_roce_free_cq(struct hns_roce_dev *hr_dev, struct hns_roce_cq *hr_cq) hns_roce_table_put(hr_dev, &cq_table->table, hr_cq->cqn); hns_roce_bitmap_free(&cq_table->bitmap, hr_cq->cqn, BITMAP_NO_RR); } -EXPORT_SYMBOL_GPL(hns_roce_free_cq); static int hns_roce_ib_get_cq_umem(struct hns_roce_dev *hr_dev, struct ib_udata *udata, @@ -235,8 +234,7 @@ static int hns_roce_ib_get_cq_umem(struct hns_roce_dev *hr_dev, &buf->hr_mtt); } else { ret = hns_roce_mtt_init(hr_dev, ib_umem_page_count(*umem), - (*umem)->page_shift, - &buf->hr_mtt); + PAGE_SHIFT, &buf->hr_mtt); } if (ret) goto err_buf; @@ -300,15 +298,15 @@ static void hns_roce_ib_free_cq_buf(struct hns_roce_dev *hr_dev, &buf->hr_buf); } -struct ib_cq *hns_roce_ib_create_cq(struct ib_device *ib_dev, - const struct ib_cq_init_attr *attr, - struct ib_udata *udata) +int hns_roce_ib_create_cq(struct ib_cq *ib_cq, + const struct ib_cq_init_attr *attr, + struct ib_udata *udata) { - struct hns_roce_dev *hr_dev = to_hr_dev(ib_dev); + struct hns_roce_dev *hr_dev = to_hr_dev(ib_cq->device); struct device *dev = hr_dev->dev; struct hns_roce_ib_create_cq ucmd; struct hns_roce_ib_create_cq_resp resp = {}; - struct hns_roce_cq *hr_cq = NULL; + struct hns_roce_cq *hr_cq = to_hr_cq(ib_cq); struct hns_roce_uar *uar = NULL; int vector = attr->comp_vector; int cq_entries = attr->cqe; @@ -319,13 +317,9 @@ struct ib_cq *hns_roce_ib_create_cq(struct ib_device *ib_dev, if (cq_entries < 1 || cq_entries > hr_dev->caps.max_cqes) { dev_err(dev, "Creat CQ failed. entries=%d, max=%d\n", cq_entries, hr_dev->caps.max_cqes); - return ERR_PTR(-EINVAL); + return -EINVAL; } - hr_cq = kzalloc(sizeof(*hr_cq), GFP_KERNEL); - if (!hr_cq) - return ERR_PTR(-ENOMEM); - if (hr_dev->caps.min_cqes) cq_entries = max(cq_entries, hr_dev->caps.min_cqes); @@ -416,7 +410,7 @@ struct ib_cq *hns_roce_ib_create_cq(struct ib_device *ib_dev, goto err_cqc; } - return &hr_cq->ib_cq; + return 0; err_cqc: hns_roce_free_cq(hr_dev, hr_cq); @@ -428,9 +422,8 @@ err_dbmap: err_mtt: hns_roce_mtt_cleanup(hr_dev, &hr_cq->hr_buf.hr_mtt); - if (udata) - ib_umem_release(hr_cq->umem); - else + ib_umem_release(hr_cq->umem); + if (!udata) hns_roce_ib_free_cq_buf(hr_dev, &hr_cq->hr_buf, hr_cq->ib_cq.cqe); @@ -439,47 +432,37 @@ err_db: hns_roce_free_db(hr_dev, &hr_cq->db); err_cq: - kfree(hr_cq); - return ERR_PTR(ret); + return ret; } -EXPORT_SYMBOL_GPL(hns_roce_ib_create_cq); -int hns_roce_ib_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata) +void hns_roce_ib_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata) { struct hns_roce_dev *hr_dev = to_hr_dev(ib_cq->device); struct hns_roce_cq *hr_cq = to_hr_cq(ib_cq); - int ret = 0; if (hr_dev->hw->destroy_cq) { - ret = hr_dev->hw->destroy_cq(ib_cq, udata); - } else { - hns_roce_free_cq(hr_dev, hr_cq); - hns_roce_mtt_cleanup(hr_dev, &hr_cq->hr_buf.hr_mtt); - - if (udata) { - ib_umem_release(hr_cq->umem); - - if (hr_cq->db_en == 1) - hns_roce_db_unmap_user( - rdma_udata_to_drv_context( - udata, - struct hns_roce_ucontext, - ibucontext), - &hr_cq->db); - } else { - /* Free the buff of stored cq */ - hns_roce_ib_free_cq_buf(hr_dev, &hr_cq->hr_buf, - ib_cq->cqe); - if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RECORD_DB) - hns_roce_free_db(hr_dev, &hr_cq->db); - } - - kfree(hr_cq); + hr_dev->hw->destroy_cq(ib_cq, udata); + return; } - return ret; + hns_roce_free_cq(hr_dev, hr_cq); + hns_roce_mtt_cleanup(hr_dev, &hr_cq->hr_buf.hr_mtt); + + ib_umem_release(hr_cq->umem); + if (udata) { + if (hr_cq->db_en == 1) + hns_roce_db_unmap_user(rdma_udata_to_drv_context( + udata, + struct hns_roce_ucontext, + ibucontext), + &hr_cq->db); + } else { + /* Free the buff of stored cq */ + hns_roce_ib_free_cq_buf(hr_dev, &hr_cq->hr_buf, ib_cq->cqe); + if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RECORD_DB) + hns_roce_free_db(hr_dev, &hr_cq->db); + } } -EXPORT_SYMBOL_GPL(hns_roce_ib_destroy_cq); void hns_roce_cq_completion(struct hns_roce_dev *hr_dev, u32 cqn) { @@ -495,7 +478,6 @@ void hns_roce_cq_completion(struct hns_roce_dev *hr_dev, u32 cqn) ++cq->arm_sn; cq->comp(cq); } -EXPORT_SYMBOL_GPL(hns_roce_cq_completion); void hns_roce_cq_event(struct hns_roce_dev *hr_dev, u32 cqn, int event_type) { @@ -517,7 +499,6 @@ void hns_roce_cq_event(struct hns_roce_dev *hr_dev, u32 cqn, int event_type) if (atomic_dec_and_test(&cq->refcount)) complete(&cq->free); } -EXPORT_SYMBOL_GPL(hns_roce_cq_event); int hns_roce_init_cq_table(struct hns_roce_dev *hr_dev) { diff --git a/drivers/infiniband/hw/hns/hns_roce_db.c b/drivers/infiniband/hw/hns/hns_roce_db.c index 0c6c1fe87705..627aa46ef683 100644 --- a/drivers/infiniband/hw/hns/hns_roce_db.c +++ b/drivers/infiniband/hw/hns/hns_roce_db.c @@ -51,7 +51,6 @@ out: return ret; } -EXPORT_SYMBOL(hns_roce_db_map_user); void hns_roce_db_unmap_user(struct hns_roce_ucontext *context, struct hns_roce_db *db) @@ -67,7 +66,6 @@ void hns_roce_db_unmap_user(struct hns_roce_ucontext *context, mutex_unlock(&context->page_mutex); } -EXPORT_SYMBOL(hns_roce_db_unmap_user); static struct hns_roce_db_pgdir *hns_roce_alloc_db_pgdir( struct device *dma_device) @@ -78,7 +76,8 @@ static struct hns_roce_db_pgdir *hns_roce_alloc_db_pgdir( if (!pgdir) return NULL; - bitmap_fill(pgdir->order1, HNS_ROCE_DB_PER_PAGE / 2); + bitmap_fill(pgdir->order1, + HNS_ROCE_DB_PER_PAGE / HNS_ROCE_DB_TYPE_COUNT); pgdir->bits[0] = pgdir->order0; pgdir->bits[1] = pgdir->order1; pgdir->page = dma_alloc_coherent(dma_device, PAGE_SIZE, @@ -116,7 +115,7 @@ found: db->u.pgdir = pgdir; db->index = i; db->db_record = pgdir->page + db->index; - db->dma = pgdir->db_dma + db->index * 4; + db->dma = pgdir->db_dma + db->index * HNS_ROCE_DB_UNIT_SIZE; db->order = order; return 0; @@ -150,7 +149,6 @@ out: return ret; } -EXPORT_SYMBOL_GPL(hns_roce_alloc_db); void hns_roce_free_db(struct hns_roce_dev *hr_dev, struct hns_roce_db *db) { @@ -170,7 +168,8 @@ void hns_roce_free_db(struct hns_roce_dev *hr_dev, struct hns_roce_db *db) i >>= o; set_bit(i, db->u.pgdir->bits[o]); - if (bitmap_full(db->u.pgdir->order1, HNS_ROCE_DB_PER_PAGE / 2)) { + if (bitmap_full(db->u.pgdir->order1, + HNS_ROCE_DB_PER_PAGE / HNS_ROCE_DB_TYPE_COUNT)) { dma_free_coherent(hr_dev->dev, PAGE_SIZE, db->u.pgdir->page, db->u.pgdir->db_dma); list_del(&db->u.pgdir->list); @@ -179,4 +178,3 @@ void hns_roce_free_db(struct hns_roce_dev *hr_dev, struct hns_roce_db *db) mutex_unlock(&hr_dev->pgdir_mutex); } -EXPORT_SYMBOL_GPL(hns_roce_free_db); diff --git a/drivers/infiniband/hw/hns/hns_roce_device.h b/drivers/infiniband/hw/hns/hns_roce_device.h index 563cf39df6d5..a548b28aab63 100644 --- a/drivers/infiniband/hw/hns/hns_roce_device.h +++ b/drivers/infiniband/hw/hns/hns_roce_device.h @@ -37,9 +37,12 @@ #define DRV_NAME "hns_roce" +/* hip08 is a pci device, it includes two version according pci version id */ +#define PCI_REVISION_ID_HIP08_A 0x20 +#define PCI_REVISION_ID_HIP08_B 0x21 + #define HNS_ROCE_HW_VER1 ('h' << 24 | 'i' << 16 | '0' << 8 | '6') -#define MAC_ADDR_OCTET_NUM 6 #define HNS_ROCE_MAX_MSG_LEN 0x80000000 #define HNS_ROCE_ALOGN_UP(a, b) ((((a) + (b) - 1) / (b)) * (b)) @@ -48,6 +51,10 @@ #define HNS_ROCE_BA_SIZE (32 * 4096) +#define BA_BYTE_LEN 8 + +#define BITS_PER_BYTE 8 + /* Hardware specification only for v1 engine */ #define HNS_ROCE_MIN_CQE_NUM 0x40 #define HNS_ROCE_MIN_WQE_NUM 0x20 @@ -55,6 +62,7 @@ /* Hardware specification only for v1 engine */ #define HNS_ROCE_MAX_INNER_MTPT_NUM 0x7 #define HNS_ROCE_MAX_MTPT_PBL_NUM 0x100000 +#define HNS_ROCE_MAX_SGE_NUM 2 #define HNS_ROCE_EACH_FREE_CQ_WAIT_MSECS 20 #define HNS_ROCE_MAX_FREE_CQ_WAIT_CNT \ @@ -64,6 +72,9 @@ #define HNS_ROCE_MAX_IRQ_NUM 128 +#define HNS_ROCE_SGE_IN_WQE 2 +#define HNS_ROCE_SGE_SHIFT 4 + #define EQ_ENABLE 1 #define EQ_DISABLE 0 @@ -81,6 +92,7 @@ #define HNS_ROCE_MAX_PORTS 6 #define HNS_ROCE_MAX_GID_NUM 16 #define HNS_ROCE_GID_SIZE 16 +#define HNS_ROCE_SGE_SIZE 16 #define HNS_ROCE_HOP_NUM_0 0xff @@ -111,6 +123,8 @@ #define PAGES_SHIFT_24 24 #define PAGES_SHIFT_32 32 +#define HNS_ROCE_PCI_BAR_NUM 2 + #define HNS_ROCE_IDX_QUE_ENTRY_SZ 4 #define SRQ_DB_REG 0x230 @@ -213,6 +227,9 @@ enum hns_roce_mtt_type { MTT_TYPE_IDX }; +#define HNS_ROCE_DB_TYPE_COUNT 2 +#define HNS_ROCE_DB_UNIT_SIZE 4 + enum { HNS_ROCE_DB_PER_PAGE = PAGE_SIZE / 4 }; @@ -324,6 +341,29 @@ struct hns_roce_mtt { enum hns_roce_mtt_type mtt_type; }; +struct hns_roce_buf_region { + int offset; /* page offset */ + u32 count; /* page count*/ + int hopnum; /* addressing hop num */ +}; + +#define HNS_ROCE_MAX_BT_REGION 3 +#define HNS_ROCE_MAX_BT_LEVEL 3 +struct hns_roce_hem_list { + struct list_head root_bt; + /* link all bt dma mem by hop config */ + struct list_head mid_bt[HNS_ROCE_MAX_BT_REGION][HNS_ROCE_MAX_BT_LEVEL]; + struct list_head btm_bt; /* link all bottom bt in @mid_bt */ + dma_addr_t root_ba; /* pointer to the root ba table */ + int bt_pg_shift; +}; + +/* memory translate region */ +struct hns_roce_mtr { + struct hns_roce_hem_list hem_list; + int buf_pg_shift; +}; + struct hns_roce_mw { struct ib_mw ibmw; u32 pdn; @@ -413,8 +453,8 @@ struct hns_roce_buf { struct hns_roce_db_pgdir { struct list_head list; DECLARE_BITMAP(order0, HNS_ROCE_DB_PER_PAGE); - DECLARE_BITMAP(order1, HNS_ROCE_DB_PER_PAGE / 2); - unsigned long *bits[2]; + DECLARE_BITMAP(order1, HNS_ROCE_DB_PER_PAGE / HNS_ROCE_DB_TYPE_COUNT); + unsigned long *bits[HNS_ROCE_DB_TYPE_COUNT]; u32 *page; dma_addr_t db_dma; }; @@ -472,7 +512,7 @@ struct hns_roce_idx_que { u32 buf_size; struct ib_umem *umem; struct hns_roce_mtt mtt; - u64 *bitmap; + unsigned long *bitmap; }; struct hns_roce_srq { @@ -535,7 +575,7 @@ struct hns_roce_av { u8 hop_limit; __le32 sl_tclass_flowlabel; u8 dgid[HNS_ROCE_GID_SIZE]; - u8 mac[6]; + u8 mac[ETH_ALEN]; __le16 vlan; bool vlan_en; }; @@ -620,6 +660,14 @@ struct hns_roce_qp { struct ib_umem *umem; struct hns_roce_mtt mtt; + struct hns_roce_mtr mtr; + + /* this define must less than HNS_ROCE_MAX_BT_REGION */ +#define HNS_ROCE_WQE_REGION_MAX 3 + struct hns_roce_buf_region regions[HNS_ROCE_WQE_REGION_MAX]; + int region_cnt; + int wqe_bt_pg_shift; + u32 buff_size; struct mutex mutex; u8 port; @@ -830,6 +878,9 @@ struct hns_roce_caps { u32 mtt_ba_pg_sz; u32 mtt_buf_pg_sz; u32 mtt_hop_num; + u32 wqe_sq_hop_num; + u32 wqe_sge_hop_num; + u32 wqe_rq_hop_num; u32 sccc_ba_pg_sz; u32 sccc_buf_pg_sz; u32 sccc_hop_num; @@ -921,7 +972,7 @@ struct hns_roce_hw { int (*poll_cq)(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc); int (*dereg_mr)(struct hns_roce_dev *hr_dev, struct hns_roce_mr *mr, struct ib_udata *udata); - int (*destroy_cq)(struct ib_cq *ibcq, struct ib_udata *udata); + void (*destroy_cq)(struct ib_cq *ibcq, struct ib_udata *udata); int (*modify_cq)(struct ib_cq *cq, u16 cq_count, u16 cq_period); int (*init_eq)(struct hns_roce_dev *hr_dev); void (*cleanup_eq)(struct hns_roce_dev *hr_dev); @@ -940,6 +991,16 @@ struct hns_roce_hw { const struct ib_device_ops *hns_roce_dev_srq_ops; }; +enum hns_phy_state { + HNS_ROCE_PHY_SLEEP = 1, + HNS_ROCE_PHY_POLLING = 2, + HNS_ROCE_PHY_DISABLED = 3, + HNS_ROCE_PHY_TRAINING = 4, + HNS_ROCE_PHY_LINKUP = 5, + HNS_ROCE_PHY_LINKERR = 6, + HNS_ROCE_PHY_TEST = 7 +}; + struct hns_roce_dev { struct ib_device ib_dev; struct platform_device *pdev; @@ -962,7 +1023,7 @@ struct hns_roce_dev { struct hns_roce_caps caps; struct xarray qp_table_xa; - unsigned char dev_addr[HNS_ROCE_MAX_PORTS][MAC_ADDR_OCTET_NUM]; + unsigned char dev_addr[HNS_ROCE_MAX_PORTS][ETH_ALEN]; u64 sys_image_guid; u32 vendor_id; u32 vendor_part_id; @@ -1084,6 +1145,19 @@ void hns_roce_mtt_cleanup(struct hns_roce_dev *hr_dev, int hns_roce_buf_write_mtt(struct hns_roce_dev *hr_dev, struct hns_roce_mtt *mtt, struct hns_roce_buf *buf); +void hns_roce_mtr_init(struct hns_roce_mtr *mtr, int bt_pg_shift, + int buf_pg_shift); +int hns_roce_mtr_attach(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr, + dma_addr_t **bufs, struct hns_roce_buf_region *regions, + int region_cnt); +void hns_roce_mtr_cleanup(struct hns_roce_dev *hr_dev, + struct hns_roce_mtr *mtr); + +/* hns roce hw need current block and next block addr from mtt */ +#define MTT_MIN_COUNT 2 +int hns_roce_mtr_find(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr, + int offset, u64 *mtt_buf, int mtt_max, u64 *base_addr); + int hns_roce_init_pd_table(struct hns_roce_dev *hr_dev); int hns_roce_init_mr_table(struct hns_roce_dev *hr_dev); int hns_roce_init_eq_table(struct hns_roce_dev *hr_dev); @@ -1148,6 +1222,18 @@ int hns_roce_buf_alloc(struct hns_roce_dev *hr_dev, u32 size, u32 max_direct, int hns_roce_ib_umem_write_mtt(struct hns_roce_dev *hr_dev, struct hns_roce_mtt *mtt, struct ib_umem *umem); +void hns_roce_init_buf_region(struct hns_roce_buf_region *region, int hopnum, + int offset, int buf_cnt); +int hns_roce_alloc_buf_list(struct hns_roce_buf_region *regions, + dma_addr_t **bufs, int count); +void hns_roce_free_buf_list(dma_addr_t **bufs, int count); + +int hns_roce_get_kmem_bufs(struct hns_roce_dev *hr_dev, dma_addr_t *bufs, + int buf_cnt, int start, struct hns_roce_buf *buf); +int hns_roce_get_umem_bufs(struct hns_roce_dev *hr_dev, dma_addr_t *bufs, + int buf_cnt, int start, struct ib_umem *umem, + int page_shift); + int hns_roce_create_srq(struct ib_srq *srq, struct ib_srq_init_attr *srq_init_attr, struct ib_udata *udata); @@ -1178,11 +1264,11 @@ void hns_roce_release_range_qp(struct hns_roce_dev *hr_dev, int base_qpn, __be32 send_ieth(const struct ib_send_wr *wr); int to_hr_qp_type(int qp_type); -struct ib_cq *hns_roce_ib_create_cq(struct ib_device *ib_dev, - const struct ib_cq_init_attr *attr, - struct ib_udata *udata); +int hns_roce_ib_create_cq(struct ib_cq *ib_cq, + const struct ib_cq_init_attr *attr, + struct ib_udata *udata); -int hns_roce_ib_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata); +void hns_roce_ib_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata); void hns_roce_free_cq(struct hns_roce_dev *hr_dev, struct hns_roce_cq *hr_cq); int hns_roce_db_map_user(struct hns_roce_ucontext *context, diff --git a/drivers/infiniband/hw/hns/hns_roce_hem.c b/drivers/infiniband/hw/hns/hns_roce_hem.c index 8e29dbb5b5fb..f4da5bd2884f 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hem.c +++ b/drivers/infiniband/hw/hns/hns_roce_hem.c @@ -56,7 +56,6 @@ bool hns_roce_check_whether_mhop(struct hns_roce_dev *hr_dev, u32 type) return false; } -EXPORT_SYMBOL_GPL(hns_roce_check_whether_mhop); static bool hns_roce_check_hem_null(struct hns_roce_hem **hem, u64 start_idx, u32 bt_chunk_num) @@ -165,7 +164,7 @@ int hns_roce_calc_hem_mhop(struct hns_roce_dev *hr_dev, + PAGE_SHIFT); mhop->bt_chunk_size = 1 << (hr_dev->caps.mtt_ba_pg_sz + PAGE_SHIFT); - mhop->ba_l0_num = mhop->bt_chunk_size / 8; + mhop->ba_l0_num = mhop->bt_chunk_size / BA_BYTE_LEN; mhop->hop_num = hr_dev->caps.mtt_hop_num; break; case HEM_TYPE_CQE: @@ -173,7 +172,7 @@ int hns_roce_calc_hem_mhop(struct hns_roce_dev *hr_dev, + PAGE_SHIFT); mhop->bt_chunk_size = 1 << (hr_dev->caps.cqe_ba_pg_sz + PAGE_SHIFT); - mhop->ba_l0_num = mhop->bt_chunk_size / 8; + mhop->ba_l0_num = mhop->bt_chunk_size / BA_BYTE_LEN; mhop->hop_num = hr_dev->caps.cqe_hop_num; break; case HEM_TYPE_SRQWQE: @@ -181,7 +180,7 @@ int hns_roce_calc_hem_mhop(struct hns_roce_dev *hr_dev, + PAGE_SHIFT); mhop->bt_chunk_size = 1 << (hr_dev->caps.srqwqe_ba_pg_sz + PAGE_SHIFT); - mhop->ba_l0_num = mhop->bt_chunk_size / 8; + mhop->ba_l0_num = mhop->bt_chunk_size / BA_BYTE_LEN; mhop->hop_num = hr_dev->caps.srqwqe_hop_num; break; case HEM_TYPE_IDX: @@ -189,7 +188,7 @@ int hns_roce_calc_hem_mhop(struct hns_roce_dev *hr_dev, + PAGE_SHIFT); mhop->bt_chunk_size = 1 << (hr_dev->caps.idx_ba_pg_sz + PAGE_SHIFT); - mhop->ba_l0_num = mhop->bt_chunk_size / 8; + mhop->ba_l0_num = mhop->bt_chunk_size / BA_BYTE_LEN; mhop->hop_num = hr_dev->caps.idx_hop_num; break; default: @@ -206,7 +205,7 @@ int hns_roce_calc_hem_mhop(struct hns_roce_dev *hr_dev, * MTT/CQE alloc hem for bt pages. */ bt_num = hns_roce_get_bt_num(table->type, mhop->hop_num); - chunk_ba_num = mhop->bt_chunk_size / 8; + chunk_ba_num = mhop->bt_chunk_size / BA_BYTE_LEN; chunk_size = table->type < HEM_TYPE_MTT ? mhop->buf_chunk_size : mhop->bt_chunk_size; table_idx = (*obj & (table->num_obj - 1)) / @@ -234,7 +233,6 @@ int hns_roce_calc_hem_mhop(struct hns_roce_dev *hr_dev, return 0; } -EXPORT_SYMBOL_GPL(hns_roce_calc_hem_mhop); static struct hns_roce_hem *hns_roce_alloc_hem(struct hns_roce_dev *hr_dev, int npages, @@ -376,18 +374,19 @@ static int hns_roce_set_hem(struct hns_roce_dev *hr_dev, bt_cmd = hr_dev->reg_base + ROCEE_BT_CMD_H_REG; - end = msecs_to_jiffies(HW_SYNC_TIMEOUT_MSECS) + jiffies; - while (1) { - if (readl(bt_cmd) >> BT_CMD_SYNC_SHIFT) { - if (!(time_before(jiffies, end))) { - dev_err(dev, "Write bt_cmd err,hw_sync is not zero.\n"); - spin_unlock_irqrestore(lock, flags); - return -EBUSY; - } - } else { + end = HW_SYNC_TIMEOUT_MSECS; + while (end) { + if (!(readl(bt_cmd) >> BT_CMD_SYNC_SHIFT)) break; - } + mdelay(HW_SYNC_SLEEP_TIME_INTERVAL); + end -= HW_SYNC_SLEEP_TIME_INTERVAL; + } + + if (end <= 0) { + dev_err(dev, "Write bt_cmd err,hw_sync is not zero.\n"); + spin_unlock_irqrestore(lock, flags); + return -EBUSY; } bt_cmd_l = (u32)bt_ba; @@ -435,7 +434,7 @@ static int hns_roce_table_mhop_get(struct hns_roce_dev *hr_dev, buf_chunk_size = mhop.buf_chunk_size; bt_chunk_size = mhop.bt_chunk_size; hop_num = mhop.hop_num; - chunk_ba_num = bt_chunk_size / 8; + chunk_ba_num = bt_chunk_size / BA_BYTE_LEN; bt_num = hns_roce_get_bt_num(table->type, hop_num); switch (bt_num) { @@ -620,7 +619,6 @@ out: mutex_unlock(&table->mutex); return ret; } -EXPORT_SYMBOL_GPL(hns_roce_table_get); static void hns_roce_table_mhop_put(struct hns_roce_dev *hr_dev, struct hns_roce_hem_table *table, @@ -645,7 +643,7 @@ static void hns_roce_table_mhop_put(struct hns_roce_dev *hr_dev, bt_chunk_size = mhop.bt_chunk_size; hop_num = mhop.hop_num; - chunk_ba_num = bt_chunk_size / 8; + chunk_ba_num = bt_chunk_size / BA_BYTE_LEN; bt_num = hns_roce_get_bt_num(table->type, hop_num); switch (bt_num) { @@ -763,7 +761,6 @@ void hns_roce_table_put(struct hns_roce_dev *hr_dev, mutex_unlock(&table->mutex); } -EXPORT_SYMBOL_GPL(hns_roce_table_put); void *hns_roce_table_find(struct hns_roce_dev *hr_dev, struct hns_roce_hem_table *table, @@ -799,7 +796,7 @@ void *hns_roce_table_find(struct hns_roce_dev *hr_dev, i = mhop.l0_idx; j = mhop.l1_idx; if (mhop.hop_num == 2) - hem_idx = i * (mhop.bt_chunk_size / 8) + j; + hem_idx = i * (mhop.bt_chunk_size / BA_BYTE_LEN) + j; else if (mhop.hop_num == 1 || mhop.hop_num == HNS_ROCE_HOP_NUM_0) hem_idx = i; @@ -836,7 +833,6 @@ out: mutex_unlock(&table->mutex); return addr; } -EXPORT_SYMBOL_GPL(hns_roce_table_find); int hns_roce_table_get_range(struct hns_roce_dev *hr_dev, struct hns_roce_hem_table *table, @@ -999,7 +995,7 @@ int hns_roce_init_hem_table(struct hns_roce_dev *hr_dev, } obj_per_chunk = buf_chunk_size / obj_size; num_hem = (nobj + obj_per_chunk - 1) / obj_per_chunk; - bt_chunk_num = bt_chunk_size / 8; + bt_chunk_num = bt_chunk_size / BA_BYTE_LEN; if (type >= HEM_TYPE_MTT) num_bt_l0 = bt_chunk_num; @@ -1156,3 +1152,463 @@ void hns_roce_cleanup_hem(struct hns_roce_dev *hr_dev) &hr_dev->mr_table.mtt_cqe_table); hns_roce_cleanup_hem_table(hr_dev, &hr_dev->mr_table.mtt_table); } + +struct roce_hem_item { + struct list_head list; /* link all hems in the same bt level */ + struct list_head sibling; /* link all hems in last hop for mtt */ + void *addr; + dma_addr_t dma_addr; + size_t count; /* max ba numbers */ + int start; /* start buf offset in this hem */ + int end; /* end buf offset in this hem */ +}; + +static struct roce_hem_item *hem_list_alloc_item(struct hns_roce_dev *hr_dev, + int start, int end, + int count, bool exist_bt, + int bt_level) +{ + struct roce_hem_item *hem; + + hem = kzalloc(sizeof(*hem), GFP_KERNEL); + if (!hem) + return NULL; + + if (exist_bt) { + hem->addr = dma_alloc_coherent(hr_dev->dev, + count * BA_BYTE_LEN, + &hem->dma_addr, GFP_KERNEL); + if (!hem->addr) { + kfree(hem); + return NULL; + } + } + + hem->count = count; + hem->start = start; + hem->end = end; + INIT_LIST_HEAD(&hem->list); + INIT_LIST_HEAD(&hem->sibling); + + return hem; +} + +static void hem_list_free_item(struct hns_roce_dev *hr_dev, + struct roce_hem_item *hem, bool exist_bt) +{ + if (exist_bt) + dma_free_coherent(hr_dev->dev, hem->count * BA_BYTE_LEN, + hem->addr, hem->dma_addr); + kfree(hem); +} + +static void hem_list_free_all(struct hns_roce_dev *hr_dev, + struct list_head *head, bool exist_bt) +{ + struct roce_hem_item *hem, *temp_hem; + + list_for_each_entry_safe(hem, temp_hem, head, list) { + list_del(&hem->list); + hem_list_free_item(hr_dev, hem, exist_bt); + } +} + +static void hem_list_link_bt(struct hns_roce_dev *hr_dev, void *base_addr, + u64 table_addr) +{ + *(u64 *)(base_addr) = table_addr; +} + +/* assign L0 table address to hem from root bt */ +static void hem_list_assign_bt(struct hns_roce_dev *hr_dev, + struct roce_hem_item *hem, void *cpu_addr, + u64 phy_addr) +{ + hem->addr = cpu_addr; + hem->dma_addr = (dma_addr_t)phy_addr; +} + +static inline bool hem_list_page_is_in_range(struct roce_hem_item *hem, + int offset) +{ + return (hem->start <= offset && offset <= hem->end); +} + +static struct roce_hem_item *hem_list_search_item(struct list_head *ba_list, + int page_offset) +{ + struct roce_hem_item *hem, *temp_hem; + struct roce_hem_item *found = NULL; + + list_for_each_entry_safe(hem, temp_hem, ba_list, list) { + if (hem_list_page_is_in_range(hem, page_offset)) { + found = hem; + break; + } + } + + return found; +} + +static bool hem_list_is_bottom_bt(int hopnum, int bt_level) +{ + /* + * hopnum base address table levels + * 0 L0(buf) + * 1 L0 -> buf + * 2 L0 -> L1 -> buf + * 3 L0 -> L1 -> L2 -> buf + */ + return bt_level >= (hopnum ? hopnum - 1 : hopnum); +} + +/** + * calc base address entries num + * @hopnum: num of mutihop addressing + * @bt_level: base address table level + * @unit: ba entries per bt page + */ +static u32 hem_list_calc_ba_range(int hopnum, int bt_level, int unit) +{ + u32 step; + int max; + int i; + + if (hopnum <= bt_level) + return 0; + /* + * hopnum bt_level range + * 1 0 unit + * ------------ + * 2 0 unit * unit + * 2 1 unit + * ------------ + * 3 0 unit * unit * unit + * 3 1 unit * unit + * 3 2 unit + */ + step = 1; + max = hopnum - bt_level; + for (i = 0; i < max; i++) + step = step * unit; + + return step; +} + +/** + * calc the root ba entries which could cover all regions + * @regions: buf region array + * @region_cnt: array size of @regions + * @unit: ba entries per bt page + */ +int hns_roce_hem_list_calc_root_ba(const struct hns_roce_buf_region *regions, + int region_cnt, int unit) +{ + struct hns_roce_buf_region *r; + int total = 0; + int step; + int i; + + for (i = 0; i < region_cnt; i++) { + r = (struct hns_roce_buf_region *)®ions[i]; + if (r->hopnum > 1) { + step = hem_list_calc_ba_range(r->hopnum, 1, unit); + if (step > 0) + total += (r->count + step - 1) / step; + } else { + total += r->count; + } + } + + return total; +} + +static int hem_list_alloc_mid_bt(struct hns_roce_dev *hr_dev, + const struct hns_roce_buf_region *r, int unit, + int offset, struct list_head *mid_bt, + struct list_head *btm_bt) +{ + struct roce_hem_item *hem_ptrs[HNS_ROCE_MAX_BT_LEVEL] = { NULL }; + struct list_head temp_list[HNS_ROCE_MAX_BT_LEVEL]; + struct roce_hem_item *cur, *pre; + const int hopnum = r->hopnum; + int start_aligned; + int distance; + int ret = 0; + int max_ofs; + int level; + u32 step; + int end; + + if (hopnum <= 1) + return 0; + + if (hopnum > HNS_ROCE_MAX_BT_LEVEL) { + dev_err(hr_dev->dev, "invalid hopnum %d!\n", hopnum); + return -EINVAL; + } + + if (offset < r->offset) { + dev_err(hr_dev->dev, "invalid offset %d,min %d!\n", + offset, r->offset); + return -EINVAL; + } + + distance = offset - r->offset; + max_ofs = r->offset + r->count - 1; + for (level = 0; level < hopnum; level++) + INIT_LIST_HEAD(&temp_list[level]); + + /* config L1 bt to last bt and link them to corresponding parent */ + for (level = 1; level < hopnum; level++) { + cur = hem_list_search_item(&mid_bt[level], offset); + if (cur) { + hem_ptrs[level] = cur; + continue; + } + + step = hem_list_calc_ba_range(hopnum, level, unit); + if (step < 1) { + ret = -EINVAL; + goto err_exit; + } + + start_aligned = (distance / step) * step + r->offset; + end = min_t(int, start_aligned + step - 1, max_ofs); + cur = hem_list_alloc_item(hr_dev, start_aligned, end, unit, + true, level); + if (!cur) { + ret = -ENOMEM; + goto err_exit; + } + hem_ptrs[level] = cur; + list_add(&cur->list, &temp_list[level]); + if (hem_list_is_bottom_bt(hopnum, level)) + list_add(&cur->sibling, &temp_list[0]); + + /* link bt to parent bt */ + if (level > 1) { + pre = hem_ptrs[level - 1]; + step = (cur->start - pre->start) / step * BA_BYTE_LEN; + hem_list_link_bt(hr_dev, pre->addr + step, + cur->dma_addr); + } + } + + list_splice(&temp_list[0], btm_bt); + for (level = 1; level < hopnum; level++) + list_splice(&temp_list[level], &mid_bt[level]); + + return 0; + +err_exit: + for (level = 1; level < hopnum; level++) + hem_list_free_all(hr_dev, &temp_list[level], true); + + return ret; +} + +static int hem_list_alloc_root_bt(struct hns_roce_dev *hr_dev, + struct hns_roce_hem_list *hem_list, int unit, + const struct hns_roce_buf_region *regions, + int region_cnt) +{ + struct roce_hem_item *hem, *temp_hem, *root_hem; + struct list_head temp_list[HNS_ROCE_MAX_BT_REGION]; + const struct hns_roce_buf_region *r; + struct list_head temp_root; + struct list_head temp_btm; + void *cpu_base; + u64 phy_base; + int ret = 0; + int offset; + int total; + int step; + int i; + + r = ®ions[0]; + root_hem = hem_list_search_item(&hem_list->root_bt, r->offset); + if (root_hem) + return 0; + + INIT_LIST_HEAD(&temp_root); + total = r->offset; + /* indicate to last region */ + r = ®ions[region_cnt - 1]; + root_hem = hem_list_alloc_item(hr_dev, total, r->offset + r->count - 1, + unit, true, 0); + if (!root_hem) + return -ENOMEM; + list_add(&root_hem->list, &temp_root); + + hem_list->root_ba = root_hem->dma_addr; + + INIT_LIST_HEAD(&temp_btm); + for (i = 0; i < region_cnt; i++) + INIT_LIST_HEAD(&temp_list[i]); + + total = 0; + for (i = 0; i < region_cnt && total < unit; i++) { + r = ®ions[i]; + if (!r->count) + continue; + + /* all regions's mid[x][0] shared the root_bt's trunk */ + cpu_base = root_hem->addr + total * BA_BYTE_LEN; + phy_base = root_hem->dma_addr + total * BA_BYTE_LEN; + + /* if hopnum is 0 or 1, cut a new fake hem from the root bt + * which's address share to all regions. + */ + if (hem_list_is_bottom_bt(r->hopnum, 0)) { + hem = hem_list_alloc_item(hr_dev, r->offset, + r->offset + r->count - 1, + r->count, false, 0); + if (!hem) { + ret = -ENOMEM; + goto err_exit; + } + hem_list_assign_bt(hr_dev, hem, cpu_base, phy_base); + list_add(&hem->list, &temp_list[i]); + list_add(&hem->sibling, &temp_btm); + total += r->count; + } else { + step = hem_list_calc_ba_range(r->hopnum, 1, unit); + if (step < 1) { + ret = -EINVAL; + goto err_exit; + } + /* if exist mid bt, link L1 to L0 */ + list_for_each_entry_safe(hem, temp_hem, + &hem_list->mid_bt[i][1], list) { + offset = hem->start / step * BA_BYTE_LEN; + hem_list_link_bt(hr_dev, cpu_base + offset, + hem->dma_addr); + total++; + } + } + } + + list_splice(&temp_btm, &hem_list->btm_bt); + list_splice(&temp_root, &hem_list->root_bt); + for (i = 0; i < region_cnt; i++) + list_splice(&temp_list[i], &hem_list->mid_bt[i][0]); + + return 0; + +err_exit: + for (i = 0; i < region_cnt; i++) + hem_list_free_all(hr_dev, &temp_list[i], false); + + hem_list_free_all(hr_dev, &temp_root, true); + + return ret; +} + +/* construct the base address table and link them by address hop config */ +int hns_roce_hem_list_request(struct hns_roce_dev *hr_dev, + struct hns_roce_hem_list *hem_list, + const struct hns_roce_buf_region *regions, + int region_cnt) +{ + const struct hns_roce_buf_region *r; + int ofs, end; + int ret = 0; + int unit; + int i; + + if (region_cnt > HNS_ROCE_MAX_BT_REGION) { + dev_err(hr_dev->dev, "invalid region region_cnt %d!\n", + region_cnt); + return -EINVAL; + } + + unit = (1 << hem_list->bt_pg_shift) / BA_BYTE_LEN; + for (i = 0; i < region_cnt; i++) { + r = ®ions[i]; + if (!r->count) + continue; + + end = r->offset + r->count; + for (ofs = r->offset; ofs < end; ofs += unit) { + ret = hem_list_alloc_mid_bt(hr_dev, r, unit, ofs, + hem_list->mid_bt[i], + &hem_list->btm_bt); + if (ret) { + dev_err(hr_dev->dev, + "alloc hem trunk fail ret=%d!\n", ret); + goto err_alloc; + } + } + } + + ret = hem_list_alloc_root_bt(hr_dev, hem_list, unit, regions, + region_cnt); + if (ret) + dev_err(hr_dev->dev, "alloc hem root fail ret=%d!\n", ret); + else + return 0; + +err_alloc: + hns_roce_hem_list_release(hr_dev, hem_list); + + return ret; +} + +void hns_roce_hem_list_release(struct hns_roce_dev *hr_dev, + struct hns_roce_hem_list *hem_list) +{ + int i, j; + + for (i = 0; i < HNS_ROCE_MAX_BT_REGION; i++) + for (j = 0; j < HNS_ROCE_MAX_BT_LEVEL; j++) + hem_list_free_all(hr_dev, &hem_list->mid_bt[i][j], + j != 0); + + hem_list_free_all(hr_dev, &hem_list->root_bt, true); + INIT_LIST_HEAD(&hem_list->btm_bt); + hem_list->root_ba = 0; +} + +void hns_roce_hem_list_init(struct hns_roce_hem_list *hem_list, + int bt_page_order) +{ + int i, j; + + INIT_LIST_HEAD(&hem_list->root_bt); + INIT_LIST_HEAD(&hem_list->btm_bt); + for (i = 0; i < HNS_ROCE_MAX_BT_REGION; i++) + for (j = 0; j < HNS_ROCE_MAX_BT_LEVEL; j++) + INIT_LIST_HEAD(&hem_list->mid_bt[i][j]); + + hem_list->bt_pg_shift = bt_page_order; +} + +void *hns_roce_hem_list_find_mtt(struct hns_roce_dev *hr_dev, + struct hns_roce_hem_list *hem_list, + int offset, int *mtt_cnt, u64 *phy_addr) +{ + struct list_head *head = &hem_list->btm_bt; + struct roce_hem_item *hem, *temp_hem; + void *cpu_base = NULL; + u64 phy_base = 0; + int nr = 0; + + list_for_each_entry_safe(hem, temp_hem, head, sibling) { + if (hem_list_page_is_in_range(hem, offset)) { + nr = offset - hem->start; + cpu_base = hem->addr + nr * BA_BYTE_LEN; + phy_base = hem->dma_addr + nr * BA_BYTE_LEN; + nr = hem->end + 1 - offset; + break; + } + } + + if (mtt_cnt) + *mtt_cnt = nr; + + if (phy_addr) + *phy_addr = phy_base; + + return cpu_base; +} diff --git a/drivers/infiniband/hw/hns/hns_roce_hem.h b/drivers/infiniband/hw/hns/hns_roce_hem.h index d9d668992e49..f1ccb8f35fe5 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hem.h +++ b/drivers/infiniband/hw/hns/hns_roce_hem.h @@ -34,8 +34,8 @@ #ifndef _HNS_ROCE_HEM_H #define _HNS_ROCE_HEM_H -#define HW_SYNC_TIMEOUT_MSECS 500 #define HW_SYNC_SLEEP_TIME_INTERVAL 20 +#define HW_SYNC_TIMEOUT_MSECS (25 * HW_SYNC_SLEEP_TIME_INTERVAL) #define BT_CMD_SYNC_SHIFT 31 enum { @@ -133,6 +133,20 @@ int hns_roce_calc_hem_mhop(struct hns_roce_dev *hr_dev, struct hns_roce_hem_mhop *mhop); bool hns_roce_check_whether_mhop(struct hns_roce_dev *hr_dev, u32 type); +void hns_roce_hem_list_init(struct hns_roce_hem_list *hem_list, + int bt_page_order); +int hns_roce_hem_list_calc_root_ba(const struct hns_roce_buf_region *regions, + int region_cnt, int unit); +int hns_roce_hem_list_request(struct hns_roce_dev *hr_dev, + struct hns_roce_hem_list *hem_list, + const struct hns_roce_buf_region *regions, + int region_cnt); +void hns_roce_hem_list_release(struct hns_roce_dev *hr_dev, + struct hns_roce_hem_list *hem_list); +void *hns_roce_hem_list_find_mtt(struct hns_roce_dev *hr_dev, + struct hns_roce_hem_list *hem_list, + int offset, int *mtt_cnt, u64 *phy_addr); + static inline void hns_roce_hem_first(struct hns_roce_hem *hem, struct hns_roce_hem_iter *iter) { diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v1.c b/drivers/infiniband/hw/hns/hns_roce_hw_v1.c index 3afd3e9330e7..81e6dedb1e02 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hw_v1.c +++ b/drivers/infiniband/hw/hns/hns_roce_hw_v1.c @@ -717,7 +717,7 @@ static int hns_roce_v1_rsv_lp_qp(struct hns_roce_dev *hr_dev) union ib_gid dgid; u64 subnet_prefix; int attr_mask = 0; - int ret = -ENOMEM; + int ret; int i, j; u8 queue_en[HNS_ROCE_V1_RESV_QP] = { 0 }; u8 phy_port; @@ -730,10 +730,16 @@ static int hns_roce_v1_rsv_lp_qp(struct hns_roce_dev *hr_dev) /* Reserved cq for loop qp */ cq_init_attr.cqe = HNS_ROCE_MIN_WQE_NUM * 2; cq_init_attr.comp_vector = 0; - cq = hns_roce_ib_create_cq(&hr_dev->ib_dev, &cq_init_attr, NULL); - if (IS_ERR(cq)) { - dev_err(dev, "Create cq for reserved loop qp failed!"); + + ibdev = &hr_dev->ib_dev; + cq = rdma_zalloc_drv_obj(ibdev, ib_cq); + if (!cq) return -ENOMEM; + + ret = hns_roce_ib_create_cq(cq, &cq_init_attr, NULL); + if (ret) { + dev_err(dev, "Create cq for reserved loop qp failed!"); + goto alloc_cq_failed; } free_mr->mr_free_cq = to_hr_cq(cq); free_mr->mr_free_cq->ib_cq.device = &hr_dev->ib_dev; @@ -743,7 +749,6 @@ static int hns_roce_v1_rsv_lp_qp(struct hns_roce_dev *hr_dev) free_mr->mr_free_cq->ib_cq.cq_context = NULL; atomic_set(&free_mr->mr_free_cq->ib_cq.usecnt, 0); - ibdev = &hr_dev->ib_dev; pd = rdma_zalloc_drv_obj(ibdev, ib_pd); if (!pd) goto alloc_mem_failed; @@ -818,7 +823,7 @@ static int hns_roce_v1_rsv_lp_qp(struct hns_roce_dev *hr_dev) attr.dest_qp_num = hr_qp->qpn; memcpy(rdma_ah_retrieve_dmac(&attr.ah_attr), hr_dev->dev_addr[port], - MAC_ADDR_OCTET_NUM); + ETH_ALEN); memcpy(&dgid.raw, &subnet_prefix, sizeof(u64)); memcpy(&dgid.raw[8], hr_dev->dev_addr[port], 3); @@ -865,9 +870,9 @@ alloc_pd_failed: kfree(pd); alloc_mem_failed: - if (hns_roce_ib_destroy_cq(cq, NULL)) - dev_err(dev, "Destroy cq for create_lp_qp failed!\n"); - + hns_roce_ib_destroy_cq(cq, NULL); +alloc_cq_failed: + kfree(cq); return ret; } @@ -894,10 +899,8 @@ static void hns_roce_v1_release_lp_qp(struct hns_roce_dev *hr_dev) i, ret); } - ret = hns_roce_ib_destroy_cq(&free_mr->mr_free_cq->ib_cq, NULL); - if (ret) - dev_err(dev, "Destroy cq for mr_free failed(%d)!\n", ret); - + hns_roce_ib_destroy_cq(&free_mr->mr_free_cq->ib_cq, NULL); + kfree(&free_mr->mr_free_cq->ib_cq); hns_roce_dealloc_pd(&free_mr->mr_free_pd->ibpd, NULL); kfree(&free_mr->mr_free_pd->ibpd); } @@ -966,8 +969,7 @@ static int hns_roce_v1_recreate_lp_qp(struct hns_roce_dev *hr_dev) struct hns_roce_free_mr *free_mr; struct hns_roce_v1_priv *priv; struct completion comp; - unsigned long end = - msecs_to_jiffies(HNS_ROCE_V1_RECREATE_LP_QP_TIMEOUT_MSECS) + jiffies; + unsigned long end = HNS_ROCE_V1_RECREATE_LP_QP_TIMEOUT_MSECS; priv = (struct hns_roce_v1_priv *)hr_dev->priv; free_mr = &priv->free_mr; @@ -987,10 +989,11 @@ static int hns_roce_v1_recreate_lp_qp(struct hns_roce_dev *hr_dev) queue_work(free_mr->free_mr_wq, &(lp_qp_work->work)); - while (time_before_eq(jiffies, end)) { + while (end) { if (try_wait_for_completion(&comp)) return 0; msleep(HNS_ROCE_V1_RECREATE_LP_QP_WAIT_VALUE); + end -= HNS_ROCE_V1_RECREATE_LP_QP_WAIT_VALUE; } lp_qp_work->comp_flag = 0; @@ -1104,8 +1107,7 @@ static int hns_roce_v1_dereg_mr(struct hns_roce_dev *hr_dev, struct hns_roce_free_mr *free_mr; struct hns_roce_v1_priv *priv; struct completion comp; - unsigned long end = - msecs_to_jiffies(HNS_ROCE_V1_FREE_MR_TIMEOUT_MSECS) + jiffies; + unsigned long end = HNS_ROCE_V1_FREE_MR_TIMEOUT_MSECS; unsigned long start = jiffies; int npages; int ret = 0; @@ -1135,10 +1137,11 @@ static int hns_roce_v1_dereg_mr(struct hns_roce_dev *hr_dev, queue_work(free_mr->free_mr_wq, &(mr_work->work)); - while (time_before_eq(jiffies, end)) { + while (end) { if (try_wait_for_completion(&comp)) goto free_mr; msleep(HNS_ROCE_V1_FREE_MR_WAIT_VALUE); + end -= HNS_ROCE_V1_FREE_MR_WAIT_VALUE; } mr_work->comp_flag = 0; @@ -1161,8 +1164,7 @@ free_mr: hns_roce_bitmap_free(&hr_dev->mr_table.mtpt_bitmap, key_to_hw_index(mr->key), 0); - if (mr->umem) - ib_umem_release(mr->umem); + ib_umem_release(mr->umem); kfree(mr); @@ -1557,6 +1559,7 @@ static int hns_roce_v1_profile(struct hns_roce_dev *hr_dev) caps->reserved_mrws = 1; caps->reserved_uars = 0; caps->reserved_cqs = 0; + caps->reserved_qps = 12; /* 2 SQP per port, six ports total 12 */ caps->chunk_sz = HNS_ROCE_V1_TABLE_CHUNK_SIZE; for (i = 0; i < caps->num_ports; i++) @@ -1742,11 +1745,14 @@ static int hns_roce_v1_set_gid(struct hns_roce_dev *hr_dev, u8 port, int gid_index, const union ib_gid *gid, const struct ib_gid_attr *attr) { + unsigned long flags; u32 *p = NULL; u8 gid_idx = 0; gid_idx = hns_get_gid_index(hr_dev, port, gid_index); + spin_lock_irqsave(&hr_dev->iboe.lock, flags); + p = (u32 *)&gid->raw[0]; roce_raw_write(*p, hr_dev->reg_base + ROCEE_PORT_GID_L_0_REG + (HNS_ROCE_V1_GID_NUM * gid_idx)); @@ -1763,6 +1769,8 @@ static int hns_roce_v1_set_gid(struct hns_roce_dev *hr_dev, u8 port, roce_raw_write(*p, hr_dev->reg_base + ROCEE_PORT_GID_H_0_REG + (HNS_ROCE_V1_GID_NUM * gid_idx)); + spin_unlock_irqrestore(&hr_dev->iboe.lock, flags); + return 0; } @@ -2458,10 +2466,10 @@ static int hns_roce_v1_clear_hem(struct hns_roce_dev *hr_dev, bt_cmd = hr_dev->reg_base + ROCEE_BT_CMD_H_REG; - end = msecs_to_jiffies(HW_SYNC_TIMEOUT_MSECS) + jiffies; + end = HW_SYNC_TIMEOUT_MSECS; while (1) { if (readl(bt_cmd) >> BT_CMD_SYNC_SHIFT) { - if (!(time_before(jiffies, end))) { + if (!end) { dev_err(dev, "Write bt_cmd err,hw_sync is not zero.\n"); spin_unlock_irqrestore(&hr_dev->bt_cmd_lock, flags); @@ -2470,7 +2478,8 @@ static int hns_roce_v1_clear_hem(struct hns_roce_dev *hr_dev, } else { break; } - msleep(HW_SYNC_SLEEP_TIME_INTERVAL); + mdelay(HW_SYNC_SLEEP_TIME_INTERVAL); + end -= HW_SYNC_SLEEP_TIME_INTERVAL; } bt_cmd_val[0] = (__le32)bt_ba; @@ -3633,9 +3642,8 @@ int hns_roce_v1_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata) hns_roce_mtt_cleanup(hr_dev, &hr_qp->mtt); - if (udata) - ib_umem_release(hr_qp->umem); - else { + ib_umem_release(hr_qp->umem); + if (!udata) { kfree(hr_qp->sq.wrid); kfree(hr_qp->rq.wrid); @@ -3649,7 +3657,7 @@ int hns_roce_v1_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata) return 0; } -static int hns_roce_v1_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata) +static void hns_roce_v1_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata) { struct hns_roce_dev *hr_dev = to_hr_dev(ibcq->device); struct hns_roce_cq *hr_cq = to_hr_cq(ibcq); @@ -3658,7 +3666,6 @@ static int hns_roce_v1_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata) u32 cqe_cnt_cur; u32 cq_buf_size; int wait_time = 0; - int ret = 0; hns_roce_free_cq(hr_dev, hr_cq); @@ -3680,7 +3687,6 @@ static int hns_roce_v1_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata) if (wait_time > HNS_ROCE_MAX_FREE_CQ_WAIT_CNT) { dev_warn(dev, "Destroy cq 0x%lx timeout!\n", hr_cq->cqn); - ret = -ETIMEDOUT; break; } wait_time++; @@ -3688,17 +3694,12 @@ static int hns_roce_v1_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata) hns_roce_mtt_cleanup(hr_dev, &hr_cq->hr_buf.hr_mtt); - if (ibcq->uobject) - ib_umem_release(hr_cq->umem); - else { + ib_umem_release(hr_cq->umem); + if (!udata) { /* Free the buff of stored cq */ cq_buf_size = (ibcq->cqe + 1) * hr_dev->caps.cq_entry_sz; hns_roce_buf_free(hr_dev, cq_buf_size, &hr_cq->hr_buf.hr_buf); } - - kfree(hr_cq); - - return ret; } static void set_eq_cons_index_v1(struct hns_roce_eq *eq, int req_not) @@ -3902,7 +3903,8 @@ static int hns_roce_v1_aeq_int(struct hns_roce_dev *hr_dev, */ dma_rmb(); - dev_dbg(dev, "aeqe = %p, aeqe->asyn.event_type = 0x%lx\n", aeqe, + dev_dbg(dev, "aeqe = %pK, aeqe->asyn.event_type = 0x%lx\n", + aeqe, roce_get_field(aeqe->asyn, HNS_ROCE_AEQE_U32_4_EVENT_TYPE_M, HNS_ROCE_AEQE_U32_4_EVENT_TYPE_S)); @@ -4265,7 +4267,6 @@ static int hns_roce_v1_create_eq(struct hns_roce_dev *hr_dev, } eq->buf_list[i].map = tmp_dma_addr; - memset(eq->buf_list[i].buf, 0, HNS_ROCE_BA_SIZE); } eq->cons_index = 0; roce_set_field(tmp, ROCEE_CAEP_AEQC_AEQE_SHIFT_CAEP_AEQC_STATE_M, diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c index b5392cb5b20f..b76e3beeafb8 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hw_v2.c +++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2.c @@ -1098,7 +1098,7 @@ static int hns_roce_cmq_send(struct hns_roce_dev *hr_dev, if (ret == CMD_RST_PRC_SUCCESS) return 0; if (ret == CMD_RST_PRC_EBUSY) - return ret; + return -EBUSY; ret = __hns_roce_cmq_send(hr_dev, desc, num); if (ret) { @@ -1106,7 +1106,7 @@ static int hns_roce_cmq_send(struct hns_roce_dev *hr_dev, if (retval == CMD_RST_PRC_SUCCESS) return 0; else if (retval == CMD_RST_PRC_EBUSY) - return retval; + return -EBUSY; } return ret; @@ -1130,6 +1130,45 @@ static int hns_roce_cmq_query_hw_info(struct hns_roce_dev *hr_dev) return 0; } +static void hns_roce_function_clear(struct hns_roce_dev *hr_dev) +{ + struct hns_roce_func_clear *resp; + struct hns_roce_cmq_desc desc; + unsigned long end; + int ret; + + hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_FUNC_CLEAR, false); + resp = (struct hns_roce_func_clear *)desc.data; + + ret = hns_roce_cmq_send(hr_dev, &desc, 1); + if (ret) { + dev_err(hr_dev->dev, "Func clear write failed, ret = %d.\n", + ret); + return; + } + + msleep(HNS_ROCE_V2_READ_FUNC_CLEAR_FLAG_INTERVAL); + end = HNS_ROCE_V2_FUNC_CLEAR_TIMEOUT_MSECS; + while (end) { + msleep(HNS_ROCE_V2_READ_FUNC_CLEAR_FLAG_FAIL_WAIT); + end -= HNS_ROCE_V2_READ_FUNC_CLEAR_FLAG_FAIL_WAIT; + + hns_roce_cmq_setup_basic_desc(&desc, HNS_ROCE_OPC_FUNC_CLEAR, + true); + + ret = hns_roce_cmq_send(hr_dev, &desc, 1); + if (ret) + continue; + + if (roce_get_bit(resp->func_done, FUNC_CLEAR_RST_FUN_DONE_S)) { + hr_dev->is_reset = true; + return; + } + } + + dev_err(hr_dev->dev, "Func clear fail.\n"); +} + static int hns_roce_query_fw_ver(struct hns_roce_dev *hr_dev) { struct hns_roce_query_fw_info *resp; @@ -1574,7 +1613,10 @@ static int hns_roce_v2_profile(struct hns_roce_dev *hr_dev) caps->mtt_ba_pg_sz = 0; caps->mtt_buf_pg_sz = 0; caps->mtt_hop_num = HNS_ROCE_MTT_HOP_NUM; - caps->cqe_ba_pg_sz = 0; + caps->wqe_sq_hop_num = 2; + caps->wqe_sge_hop_num = 1; + caps->wqe_rq_hop_num = 2; + caps->cqe_ba_pg_sz = 6; caps->cqe_buf_pg_sz = 0; caps->cqe_hop_num = HNS_ROCE_CQE_HOP_NUM; caps->srqwqe_ba_pg_sz = 0; @@ -1774,7 +1816,6 @@ static int hns_roce_init_link_table(struct hns_roce_dev *hr_dev, goto err_alloc_buf_failed; link_tbl->pg_list[i].map = t; - memset(link_tbl->pg_list[i].buf, 0, buf_chk_sz); entry[i].blk_ba0 = (t >> 12) & 0xffffffff; roce_set_field(entry[i].blk_ba1_nxt_ptr, @@ -1891,6 +1932,9 @@ static void hns_roce_v2_exit(struct hns_roce_dev *hr_dev) { struct hns_roce_v2_priv *priv = hr_dev->priv; + if (hr_dev->pci_dev->revision == 0x21) + hns_roce_function_clear(hr_dev); + hns_roce_free_link_table(hr_dev, &priv->tpq); hns_roce_free_link_table(hr_dev, &priv->tsq); } @@ -1974,7 +2018,7 @@ static int hns_roce_v2_chk_mbox(struct hns_roce_dev *hr_dev, unsigned long timeout) { struct device *dev = hr_dev->dev; - unsigned long end = 0; + unsigned long end; u32 status; end = msecs_to_jiffies(timeout) + jiffies; @@ -2340,15 +2384,10 @@ static void *get_srq_wqe(struct hns_roce_srq *srq, int n) static void hns_roce_free_srq_wqe(struct hns_roce_srq *srq, int wqe_index) { - u32 bitmap_num; - int bit_num; - /* always called with interrupts disabled. */ spin_lock(&srq->lock); - bitmap_num = wqe_index / (sizeof(u64) * 8); - bit_num = wqe_index % (sizeof(u64) * 8); - srq->idx_que.bitmap[bitmap_num] |= (1ULL << bit_num); + bitmap_clear(srq->idx_que.bitmap, wqe_index, 1); srq->tail++; spin_unlock(&srq->lock); @@ -2977,7 +3016,7 @@ static int hns_roce_v2_clear_hem(struct hns_roce_dev *hr_dev, { struct device *dev = hr_dev->dev; struct hns_roce_cmd_mailbox *mailbox; - int ret = 0; + int ret; u16 op = 0xff; if (!hns_roce_check_whether_mhop(hr_dev, table->type)) @@ -3026,7 +3065,6 @@ static int hns_roce_v2_clear_hem(struct hns_roce_dev *hr_dev, } static int hns_roce_v2_qp_modify(struct hns_roce_dev *hr_dev, - struct hns_roce_mtt *mtt, enum ib_qp_state cur_state, enum ib_qp_state new_state, struct hns_roce_v2_qp_context *context, @@ -3426,7 +3464,9 @@ static void modify_qp_init_to_init(struct ib_qp *ibqp, else roce_set_field(context->byte_4_sqpn_tst, V2_QPC_BYTE_4_SGE_SHIFT_M, - V2_QPC_BYTE_4_SGE_SHIFT_S, hr_qp->sq.max_gs > 2 ? + V2_QPC_BYTE_4_SGE_SHIFT_S, + hr_qp->sq.max_gs > + HNS_ROCE_V2_UC_RC_SGE_NUM_IN_WQE ? ilog2((unsigned int)hr_qp->sge.sge_cnt) : 0); roce_set_field(qpc_mask->byte_4_sqpn_tst, V2_QPC_BYTE_4_SGE_SHIFT_M, @@ -3520,6 +3560,31 @@ static void modify_qp_init_to_init(struct ib_qp *ibqp, } } +static bool check_wqe_rq_mtt_count(struct hns_roce_dev *hr_dev, + struct hns_roce_qp *hr_qp, int mtt_cnt, + u32 page_size) +{ + struct device *dev = hr_dev->dev; + + if (hr_qp->rq.wqe_cnt < 1) + return true; + + if (mtt_cnt < 1) { + dev_err(dev, "qp(0x%lx) rqwqe buf ba find failed\n", + hr_qp->qpn); + return false; + } + + if (mtt_cnt < MTT_MIN_COUNT && + (hr_qp->rq.offset + page_size) < hr_qp->buff_size) { + dev_err(dev, "qp(0x%lx) next rqwqe buf ba find failed\n", + hr_qp->qpn); + return false; + } + + return true; +} + static int modify_qp_init_to_rtr(struct ib_qp *ibqp, const struct ib_qp_attr *attr, int attr_mask, struct hns_roce_v2_qp_context *context, @@ -3529,25 +3594,27 @@ static int modify_qp_init_to_rtr(struct ib_qp *ibqp, struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); struct hns_roce_qp *hr_qp = to_hr_qp(ibqp); struct device *dev = hr_dev->dev; + u64 mtts[MTT_MIN_COUNT] = { 0 }; dma_addr_t dma_handle_3; dma_addr_t dma_handle_2; - dma_addr_t dma_handle; + u64 wqe_sge_ba; u32 page_size; u8 port_num; u64 *mtts_3; u64 *mtts_2; - u64 *mtts; + int count; u8 *dmac; u8 *smac; int port; /* Search qp buf's mtts */ - mtts = hns_roce_table_find(hr_dev, &hr_dev->mr_table.mtt_table, - hr_qp->mtt.first_seg, &dma_handle); - if (!mtts) { - dev_err(dev, "qp buf pa find failed\n"); - return -EINVAL; - } + page_size = 1 << (hr_dev->caps.mtt_buf_pg_sz + PAGE_SHIFT); + count = hns_roce_mtr_find(hr_dev, &hr_qp->mtr, + hr_qp->rq.offset / page_size, mtts, + MTT_MIN_COUNT, &wqe_sge_ba); + if (!ibqp->srq) + if (!check_wqe_rq_mtt_count(hr_dev, hr_qp, count, page_size)) + return -EINVAL; /* Search IRRL's mtts */ mtts_2 = hns_roce_table_find(hr_dev, &hr_dev->qp_table.irrl_table, @@ -3571,7 +3638,7 @@ static int modify_qp_init_to_rtr(struct ib_qp *ibqp, } dmac = (u8 *)attr->ah_attr.roce.dmac; - context->wqe_sge_ba = (u32)(dma_handle >> 3); + context->wqe_sge_ba = (u32)(wqe_sge_ba >> 3); qpc_mask->wqe_sge_ba = 0; /* @@ -3581,22 +3648,23 @@ static int modify_qp_init_to_rtr(struct ib_qp *ibqp, * 0 at the same time, else set them to 0x1. */ roce_set_field(context->byte_12_sq_hop, V2_QPC_BYTE_12_WQE_SGE_BA_M, - V2_QPC_BYTE_12_WQE_SGE_BA_S, dma_handle >> (32 + 3)); + V2_QPC_BYTE_12_WQE_SGE_BA_S, wqe_sge_ba >> (32 + 3)); roce_set_field(qpc_mask->byte_12_sq_hop, V2_QPC_BYTE_12_WQE_SGE_BA_M, V2_QPC_BYTE_12_WQE_SGE_BA_S, 0); roce_set_field(context->byte_12_sq_hop, V2_QPC_BYTE_12_SQ_HOP_NUM_M, V2_QPC_BYTE_12_SQ_HOP_NUM_S, - hr_dev->caps.mtt_hop_num == HNS_ROCE_HOP_NUM_0 ? - 0 : hr_dev->caps.mtt_hop_num); + hr_dev->caps.wqe_sq_hop_num == HNS_ROCE_HOP_NUM_0 ? + 0 : hr_dev->caps.wqe_sq_hop_num); roce_set_field(qpc_mask->byte_12_sq_hop, V2_QPC_BYTE_12_SQ_HOP_NUM_M, V2_QPC_BYTE_12_SQ_HOP_NUM_S, 0); roce_set_field(context->byte_20_smac_sgid_idx, V2_QPC_BYTE_20_SGE_HOP_NUM_M, V2_QPC_BYTE_20_SGE_HOP_NUM_S, - ((ibqp->qp_type == IB_QPT_GSI) || hr_qp->sq.max_gs > 2) ? - hr_dev->caps.mtt_hop_num : 0); + ((ibqp->qp_type == IB_QPT_GSI) || + hr_qp->sq.max_gs > HNS_ROCE_V2_UC_RC_SGE_NUM_IN_WQE) ? + hr_dev->caps.wqe_sge_hop_num : 0); roce_set_field(qpc_mask->byte_20_smac_sgid_idx, V2_QPC_BYTE_20_SGE_HOP_NUM_M, V2_QPC_BYTE_20_SGE_HOP_NUM_S, 0); @@ -3604,8 +3672,8 @@ static int modify_qp_init_to_rtr(struct ib_qp *ibqp, roce_set_field(context->byte_20_smac_sgid_idx, V2_QPC_BYTE_20_RQ_HOP_NUM_M, V2_QPC_BYTE_20_RQ_HOP_NUM_S, - hr_dev->caps.mtt_hop_num == HNS_ROCE_HOP_NUM_0 ? - 0 : hr_dev->caps.mtt_hop_num); + hr_dev->caps.wqe_rq_hop_num == HNS_ROCE_HOP_NUM_0 ? + 0 : hr_dev->caps.wqe_rq_hop_num); roce_set_field(qpc_mask->byte_20_smac_sgid_idx, V2_QPC_BYTE_20_RQ_HOP_NUM_M, V2_QPC_BYTE_20_RQ_HOP_NUM_S, 0); @@ -3613,7 +3681,7 @@ static int modify_qp_init_to_rtr(struct ib_qp *ibqp, roce_set_field(context->byte_16_buf_ba_pg_sz, V2_QPC_BYTE_16_WQE_SGE_BA_PG_SZ_M, V2_QPC_BYTE_16_WQE_SGE_BA_PG_SZ_S, - hr_dev->caps.mtt_ba_pg_sz + PG_SHIFT_OFFSET); + hr_qp->wqe_bt_pg_shift + PG_SHIFT_OFFSET); roce_set_field(qpc_mask->byte_16_buf_ba_pg_sz, V2_QPC_BYTE_16_WQE_SGE_BA_PG_SZ_M, V2_QPC_BYTE_16_WQE_SGE_BA_PG_SZ_S, 0); @@ -3626,29 +3694,24 @@ static int modify_qp_init_to_rtr(struct ib_qp *ibqp, V2_QPC_BYTE_16_WQE_SGE_BUF_PG_SZ_M, V2_QPC_BYTE_16_WQE_SGE_BUF_PG_SZ_S, 0); - page_size = 1 << (hr_dev->caps.mtt_buf_pg_sz + PAGE_SHIFT); - context->rq_cur_blk_addr = (u32)(mtts[hr_qp->rq.offset / page_size] - >> PAGE_ADDR_SHIFT); + context->rq_cur_blk_addr = (u32)(mtts[0] >> PAGE_ADDR_SHIFT); qpc_mask->rq_cur_blk_addr = 0; roce_set_field(context->byte_92_srq_info, V2_QPC_BYTE_92_RQ_CUR_BLK_ADDR_M, V2_QPC_BYTE_92_RQ_CUR_BLK_ADDR_S, - mtts[hr_qp->rq.offset / page_size] - >> (32 + PAGE_ADDR_SHIFT)); + mtts[0] >> (32 + PAGE_ADDR_SHIFT)); roce_set_field(qpc_mask->byte_92_srq_info, V2_QPC_BYTE_92_RQ_CUR_BLK_ADDR_M, V2_QPC_BYTE_92_RQ_CUR_BLK_ADDR_S, 0); - context->rq_nxt_blk_addr = (u32)(mtts[hr_qp->rq.offset / page_size + 1] - >> PAGE_ADDR_SHIFT); + context->rq_nxt_blk_addr = (u32)(mtts[1] >> PAGE_ADDR_SHIFT); qpc_mask->rq_nxt_blk_addr = 0; roce_set_field(context->byte_104_rq_sge, V2_QPC_BYTE_104_RQ_NXT_BLK_ADDR_M, V2_QPC_BYTE_104_RQ_NXT_BLK_ADDR_S, - mtts[hr_qp->rq.offset / page_size + 1] - >> (32 + PAGE_ADDR_SHIFT)); + mtts[1] >> (32 + PAGE_ADDR_SHIFT)); roce_set_field(qpc_mask->byte_104_rq_sge, V2_QPC_BYTE_104_RQ_NXT_BLK_ADDR_M, V2_QPC_BYTE_104_RQ_NXT_BLK_ADDR_S, 0); @@ -3708,13 +3771,14 @@ static int modify_qp_init_to_rtr(struct ib_qp *ibqp, roce_set_field(qpc_mask->byte_20_smac_sgid_idx, V2_QPC_BYTE_20_SGID_IDX_M, V2_QPC_BYTE_20_SGID_IDX_S, 0); - memcpy(&(context->dmac), dmac, 4); + memcpy(&(context->dmac), dmac, sizeof(u32)); roce_set_field(context->byte_52_udpspn_dmac, V2_QPC_BYTE_52_DMAC_M, V2_QPC_BYTE_52_DMAC_S, *((u16 *)(&dmac[4]))); qpc_mask->dmac = 0; roce_set_field(qpc_mask->byte_52_udpspn_dmac, V2_QPC_BYTE_52_DMAC_M, V2_QPC_BYTE_52_DMAC_S, 0); + /* mtu*(2^LP_PKTN_INI) should not bigger than 1 message length 64kb */ roce_set_field(context->byte_56_dqpn_err, V2_QPC_BYTE_56_LP_PKTN_INI_M, V2_QPC_BYTE_56_LP_PKTN_INI_S, 4); roce_set_field(qpc_mask->byte_56_dqpn_err, V2_QPC_BYTE_56_LP_PKTN_INI_M, @@ -3756,6 +3820,7 @@ static int modify_qp_init_to_rtr(struct ib_qp *ibqp, roce_set_field(qpc_mask->byte_132_trrl, V2_QPC_BYTE_132_TRRL_TAIL_MAX_M, V2_QPC_BYTE_132_TRRL_TAIL_MAX_S, 0); + /* rocee send 2^lp_sgen_ini segs every time */ roce_set_field(context->byte_168_irrl_idx, V2_QPC_BYTE_168_LP_SGEN_INI_M, V2_QPC_BYTE_168_LP_SGEN_INI_S, 3); @@ -3774,18 +3839,30 @@ static int modify_qp_rtr_to_rts(struct ib_qp *ibqp, struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device); struct hns_roce_qp *hr_qp = to_hr_qp(ibqp); struct device *dev = hr_dev->dev; - dma_addr_t dma_handle; + u64 sge_cur_blk = 0; + u64 sq_cur_blk = 0; u32 page_size; - u64 *mtts; + int count; /* Search qp buf's mtts */ - mtts = hns_roce_table_find(hr_dev, &hr_dev->mr_table.mtt_table, - hr_qp->mtt.first_seg, &dma_handle); - if (!mtts) { - dev_err(dev, "qp buf pa find failed\n"); + count = hns_roce_mtr_find(hr_dev, &hr_qp->mtr, 0, &sq_cur_blk, 1, NULL); + if (count < 1) { + dev_err(dev, "qp(0x%lx) buf pa find failed\n", hr_qp->qpn); return -EINVAL; } + if (hr_qp->sge.offset) { + page_size = 1 << (hr_dev->caps.mtt_buf_pg_sz + PAGE_SHIFT); + count = hns_roce_mtr_find(hr_dev, &hr_qp->mtr, + hr_qp->sge.offset / page_size, + &sge_cur_blk, 1, NULL); + if (count < 1) { + dev_err(dev, "qp(0x%lx) sge pa find failed\n", + hr_qp->qpn); + return -EINVAL; + } + } + /* Not support alternate path and path migration */ if ((attr_mask & IB_QP_ALT_PATH) || (attr_mask & IB_QP_PATH_MIG_STATE)) { @@ -3799,37 +3876,37 @@ static int modify_qp_rtr_to_rts(struct ib_qp *ibqp, * we should set all bits of the relevant fields in context mask to * 0 at the same time, else set them to 0x1. */ - context->sq_cur_blk_addr = (u32)(mtts[0] >> PAGE_ADDR_SHIFT); + context->sq_cur_blk_addr = (u32)(sq_cur_blk >> PAGE_ADDR_SHIFT); roce_set_field(context->byte_168_irrl_idx, V2_QPC_BYTE_168_SQ_CUR_BLK_ADDR_M, V2_QPC_BYTE_168_SQ_CUR_BLK_ADDR_S, - mtts[0] >> (32 + PAGE_ADDR_SHIFT)); + sq_cur_blk >> (32 + PAGE_ADDR_SHIFT)); qpc_mask->sq_cur_blk_addr = 0; roce_set_field(qpc_mask->byte_168_irrl_idx, V2_QPC_BYTE_168_SQ_CUR_BLK_ADDR_M, V2_QPC_BYTE_168_SQ_CUR_BLK_ADDR_S, 0); - page_size = 1 << (hr_dev->caps.mtt_buf_pg_sz + PAGE_SHIFT); - context->sq_cur_sge_blk_addr = - ((ibqp->qp_type == IB_QPT_GSI) || hr_qp->sq.max_gs > 2) ? - ((u32)(mtts[hr_qp->sge.offset / page_size] - >> PAGE_ADDR_SHIFT)) : 0; + context->sq_cur_sge_blk_addr = ((ibqp->qp_type == IB_QPT_GSI) || + hr_qp->sq.max_gs > HNS_ROCE_V2_UC_RC_SGE_NUM_IN_WQE) ? + ((u32)(sge_cur_blk >> + PAGE_ADDR_SHIFT)) : 0; roce_set_field(context->byte_184_irrl_idx, V2_QPC_BYTE_184_SQ_CUR_SGE_BLK_ADDR_M, V2_QPC_BYTE_184_SQ_CUR_SGE_BLK_ADDR_S, - ((ibqp->qp_type == IB_QPT_GSI) || hr_qp->sq.max_gs > 2) ? - (mtts[hr_qp->sge.offset / page_size] >> + ((ibqp->qp_type == IB_QPT_GSI) || hr_qp->sq.max_gs > + HNS_ROCE_V2_UC_RC_SGE_NUM_IN_WQE) ? + (sge_cur_blk >> (32 + PAGE_ADDR_SHIFT)) : 0); qpc_mask->sq_cur_sge_blk_addr = 0; roce_set_field(qpc_mask->byte_184_irrl_idx, V2_QPC_BYTE_184_SQ_CUR_SGE_BLK_ADDR_M, V2_QPC_BYTE_184_SQ_CUR_SGE_BLK_ADDR_S, 0); - context->rx_sq_cur_blk_addr = (u32)(mtts[0] >> PAGE_ADDR_SHIFT); + context->rx_sq_cur_blk_addr = (u32)(sq_cur_blk >> PAGE_ADDR_SHIFT); roce_set_field(context->byte_232_irrl_sge, V2_QPC_BYTE_232_RX_SQ_CUR_BLK_ADDR_M, V2_QPC_BYTE_232_RX_SQ_CUR_BLK_ADDR_S, - mtts[0] >> (32 + PAGE_ADDR_SHIFT)); + sq_cur_blk >> (32 + PAGE_ADDR_SHIFT)); qpc_mask->rx_sq_cur_blk_addr = 0; roce_set_field(qpc_mask->byte_232_irrl_sge, V2_QPC_BYTE_232_RX_SQ_CUR_BLK_ADDR_M, @@ -4144,7 +4221,7 @@ static int hns_roce_v2_modify_qp(struct ib_qp *ibqp, roce_set_field(context->byte_224_retry_msg, V2_QPC_BYTE_224_RETRY_MSG_PSN_M, V2_QPC_BYTE_224_RETRY_MSG_PSN_S, - attr->sq_psn >> 16); + attr->sq_psn >> V2_QPC_BYTE_220_RETRY_MSG_PSN_S); roce_set_field(qpc_mask->byte_224_retry_msg, V2_QPC_BYTE_224_RETRY_MSG_PSN_M, V2_QPC_BYTE_224_RETRY_MSG_PSN_S, 0); @@ -4230,7 +4307,7 @@ static int hns_roce_v2_modify_qp(struct ib_qp *ibqp, V2_QPC_BYTE_60_QP_ST_S, 0); /* SW pass context to HW */ - ret = hns_roce_v2_qp_modify(hr_dev, &hr_qp->mtt, cur_state, new_state, + ret = hns_roce_v2_qp_modify(hr_dev, cur_state, new_state, context, hr_qp); if (ret) { dev_err(dev, "hns_roce_qp_modify failed(%d)\n", ret); @@ -4374,11 +4451,12 @@ static int hns_roce_v2_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr, V2_QPC_BYTE_56_DQPN_M, V2_QPC_BYTE_56_DQPN_S); qp_attr->qp_access_flags = ((roce_get_bit(context->byte_76_srqn_op_en, - V2_QPC_BYTE_76_RRE_S)) << 2) | - ((roce_get_bit(context->byte_76_srqn_op_en, - V2_QPC_BYTE_76_RWE_S)) << 1) | - ((roce_get_bit(context->byte_76_srqn_op_en, - V2_QPC_BYTE_76_ATE_S)) << 3); + V2_QPC_BYTE_76_RRE_S)) << V2_QP_RWE_S) | + ((roce_get_bit(context->byte_76_srqn_op_en, + V2_QPC_BYTE_76_RWE_S)) << V2_QP_RRE_S) | + ((roce_get_bit(context->byte_76_srqn_op_en, + V2_QPC_BYTE_76_ATE_S)) << V2_QP_ATE_S); + if (hr_qp->ibqp.qp_type == IB_QPT_RC || hr_qp->ibqp.qp_type == IB_QPT_UC) { struct ib_global_route *grh = @@ -4487,7 +4565,7 @@ static int hns_roce_v2_destroy_qp_common(struct hns_roce_dev *hr_dev, (hr_qp->ibqp.qp_type == IB_QPT_UD)) hns_roce_release_range_qp(hr_dev, hr_qp->qpn, 1); - hns_roce_mtt_cleanup(hr_dev, &hr_qp->mtt); + hns_roce_mtr_cleanup(hr_dev, &hr_qp->mtr); if (udata) { struct hns_roce_ucontext *context = @@ -4501,7 +4579,6 @@ static int hns_roce_v2_destroy_qp_common(struct hns_roce_dev *hr_dev, if (hr_qp->rq.wqe_cnt && (hr_qp->rdb_en == 1)) hns_roce_db_unmap_user(context, &hr_qp->rdb); - ib_umem_release(hr_qp->umem); } else { kfree(hr_qp->sq.wrid); kfree(hr_qp->rq.wrid); @@ -4509,6 +4586,7 @@ static int hns_roce_v2_destroy_qp_common(struct hns_roce_dev *hr_dev, if (hr_qp->rq.wqe_cnt) hns_roce_free_db(hr_dev, &hr_qp->rdb); } + ib_umem_release(hr_qp->umem); if ((hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RQ_INLINE) && hr_qp->rq.wqe_cnt) { @@ -4682,7 +4760,6 @@ static void hns_roce_irq_work_handle(struct work_struct *work) dev_warn(dev, "Path migration failed.\n"); break; case HNS_ROCE_EVENT_TYPE_COMM_EST: - dev_info(dev, "Communication established.\n"); break; case HNS_ROCE_EVENT_TYPE_SQ_DRAINED: dev_warn(dev, "Send queue drained.\n"); @@ -5151,8 +5228,8 @@ static void hns_roce_mhop_free_eq(struct hns_roce_dev *hr_dev, dma_free_coherent(dev, bt_chk_sz, eq->bt_l1[i], eq->l1_dma[i]); - for (j = 0; j < bt_chk_sz / 8; j++) { - idx = i * (bt_chk_sz / 8) + j; + for (j = 0; j < bt_chk_sz / BA_BYTE_LEN; j++) { + idx = i * (bt_chk_sz / BA_BYTE_LEN) + j; if ((i == eq->l0_last_num - 1) && j == eq->l1_last_num - 1) { eqe_alloc = (buf_chk_sz / eq->eqe_size) @@ -5368,9 +5445,9 @@ static int hns_roce_mhop_alloc_eq(struct hns_roce_dev *hr_dev, buf_chk_sz = 1 << (hr_dev->caps.eqe_buf_pg_sz + PAGE_SHIFT); bt_chk_sz = 1 << (hr_dev->caps.eqe_ba_pg_sz + PAGE_SHIFT); - ba_num = (PAGE_ALIGN(eq->entries * eq->eqe_size) + buf_chk_sz - 1) - / buf_chk_sz; - bt_num = (ba_num + bt_chk_sz / 8 - 1) / (bt_chk_sz / 8); + ba_num = DIV_ROUND_UP(PAGE_ALIGN(eq->entries * eq->eqe_size), + buf_chk_sz); + bt_num = DIV_ROUND_UP(ba_num, bt_chk_sz / BA_BYTE_LEN); /* hop_num = 0 */ if (mhop_num == HNS_ROCE_HOP_NUM_0) { @@ -5387,8 +5464,6 @@ static int hns_roce_mhop_alloc_eq(struct hns_roce_dev *hr_dev, eq->cur_eqe_ba = eq->l0_dma; eq->nxt_eqe_ba = 0; - memset(eq->bt_l0, 0, eq->entries * eq->eqe_size); - return 0; } @@ -5415,12 +5490,12 @@ static int hns_roce_mhop_alloc_eq(struct hns_roce_dev *hr_dev, goto err_dma_alloc_l0; if (mhop_num == 1) { - if (ba_num > (bt_chk_sz / 8)) + if (ba_num > (bt_chk_sz / BA_BYTE_LEN)) dev_err(dev, "ba_num %d is too large for 1 hop\n", ba_num); /* alloc buf */ - for (i = 0; i < bt_chk_sz / 8; i++) { + for (i = 0; i < bt_chk_sz / BA_BYTE_LEN; i++) { if (eq_buf_cnt + 1 < ba_num) { size = buf_chk_sz; } else { @@ -5444,7 +5519,7 @@ static int hns_roce_mhop_alloc_eq(struct hns_roce_dev *hr_dev, } else if (mhop_num == 2) { /* alloc L1 BT and buf */ - for (i = 0; i < bt_chk_sz / 8; i++) { + for (i = 0; i < bt_chk_sz / BA_BYTE_LEN; i++) { eq->bt_l1[i] = dma_alloc_coherent(dev, bt_chk_sz, &(eq->l1_dma[i]), GFP_KERNEL); @@ -5452,8 +5527,8 @@ static int hns_roce_mhop_alloc_eq(struct hns_roce_dev *hr_dev, goto err_dma_alloc_l1; *(eq->bt_l0 + i) = eq->l1_dma[i]; - for (j = 0; j < bt_chk_sz / 8; j++) { - idx = i * bt_chk_sz / 8 + j; + for (j = 0; j < bt_chk_sz / BA_BYTE_LEN; j++) { + idx = i * bt_chk_sz / BA_BYTE_LEN + j; if (eq_buf_cnt + 1 < ba_num) { size = buf_chk_sz; } else { @@ -5498,8 +5573,8 @@ err_dma_alloc_l1: dma_free_coherent(dev, bt_chk_sz, eq->bt_l1[i], eq->l1_dma[i]); - for (j = 0; j < bt_chk_sz / 8; j++) { - idx = i * bt_chk_sz / 8 + j; + for (j = 0; j < bt_chk_sz / BA_BYTE_LEN; j++) { + idx = i * bt_chk_sz / BA_BYTE_LEN + j; dma_free_coherent(dev, buf_chk_sz, eq->buf[idx], eq->buf_dma[idx]); } @@ -5522,11 +5597,11 @@ err_dma_alloc_buf: dma_free_coherent(dev, bt_chk_sz, eq->bt_l1[i], eq->l1_dma[i]); - for (j = 0; j < bt_chk_sz / 8; j++) { + for (j = 0; j < bt_chk_sz / BA_BYTE_LEN; j++) { if (i == record_i && j >= record_j) break; - idx = i * bt_chk_sz / 8 + j; + idx = i * bt_chk_sz / BA_BYTE_LEN + j; dma_free_coherent(dev, buf_chk_sz, eq->buf[idx], eq->buf_dma[idx]); @@ -5972,18 +6047,19 @@ out: return ret; } -static int find_empty_entry(struct hns_roce_idx_que *idx_que) +static int find_empty_entry(struct hns_roce_idx_que *idx_que, + unsigned long size) { - int bit_num; - int i; + int wqe_idx; - /* bitmap[i] is set zero if all bits are allocated */ - for (i = 0; idx_que->bitmap[i] == 0; ++i) - ; - bit_num = ffs(idx_que->bitmap[i]); - idx_que->bitmap[i] &= ~(1ULL << (bit_num - 1)); + if (unlikely(bitmap_full(idx_que->bitmap, size))) + return -ENOSPC; + + wqe_idx = find_first_zero_bit(idx_que->bitmap, size); - return i * sizeof(u64) * 8 + (bit_num - 1); + bitmap_set(idx_que->bitmap, wqe_idx, 1); + + return wqe_idx; } static void fill_idx_queue(struct hns_roce_idx_que *idx_que, @@ -6029,7 +6105,13 @@ static int hns_roce_v2_post_srq_recv(struct ib_srq *ibsrq, break; } - wqe_idx = find_empty_entry(&srq->idx_que); + wqe_idx = find_empty_entry(&srq->idx_que, srq->max); + if (wqe_idx < 0) { + ret = -ENOMEM; + *bad_wr = wr; + break; + } + fill_idx_queue(&srq->idx_que, ind, wqe_idx); wqe = get_srq_wqe(srq, wqe_idx); dseg = (struct hns_roce_v2_wqe_data_seg *)wqe; @@ -6041,9 +6123,9 @@ static int hns_roce_v2_post_srq_recv(struct ib_srq *ibsrq, } if (i < srq->max_gs) { - dseg->len = 0; - dseg->lkey = cpu_to_le32(0x100); - dseg->addr = 0; + dseg[i].len = 0; + dseg[i].lkey = cpu_to_le32(0x100); + dseg[i].addr = 0; } srq->wrid[wqe_idx] = wr->wr_id; @@ -6059,7 +6141,8 @@ static int hns_roce_v2_post_srq_recv(struct ib_srq *ibsrq, */ wmb(); - srq_db.byte_4 = HNS_ROCE_V2_SRQ_DB << 24 | srq->srqn; + srq_db.byte_4 = HNS_ROCE_V2_SRQ_DB << V2_DB_BYTE_4_CMD_S | + (srq->srqn & V2_DB_BYTE_4_TAG_M); srq_db.parameter = srq->head; hns_roce_write64(hr_dev, (__le32 *)&srq_db, srq->db_reg_l); @@ -6301,6 +6384,7 @@ static int hns_roce_hw_v2_reset_notify_down(struct hnae3_handle *handle) if (!hr_dev) return 0; + hr_dev->is_reset = true; hr_dev->active = false; hr_dev->dis_db = true; diff --git a/drivers/infiniband/hw/hns/hns_roce_hw_v2.h b/drivers/infiniband/hw/hns/hns_roce_hw_v2.h index edfdbe2ce0db..478f5a5b7aa1 100644 --- a/drivers/infiniband/hw/hns/hns_roce_hw_v2.h +++ b/drivers/infiniband/hw/hns/hns_roce_hw_v2.h @@ -54,7 +54,7 @@ #define HNS_ROCE_V2_MAX_CQ_NUM 0x100000 #define HNS_ROCE_V2_MAX_CQC_TIMER_NUM 0x100 #define HNS_ROCE_V2_MAX_SRQ_NUM 0x100000 -#define HNS_ROCE_V2_MAX_CQE_NUM 0x10000 +#define HNS_ROCE_V2_MAX_CQE_NUM 0x400000 #define HNS_ROCE_V2_MAX_SRQWQE_NUM 0x8000 #define HNS_ROCE_V2_MAX_RQ_SGE_NUM 0x100 #define HNS_ROCE_V2_MAX_SQ_SGE_NUM 0xff @@ -241,6 +241,7 @@ enum hns_roce_opcode_type { HNS_ROCE_OPC_POST_MB = 0x8504, HNS_ROCE_OPC_QUERY_MB_ST = 0x8505, HNS_ROCE_OPC_CFG_BT_ATTR = 0x8506, + HNS_ROCE_OPC_FUNC_CLEAR = 0x8508, HNS_ROCE_OPC_CLR_SCCC = 0x8509, HNS_ROCE_OPC_QUERY_SCCC = 0x850a, HNS_ROCE_OPC_RESET_SCCC = 0x850b, @@ -886,6 +887,10 @@ struct hns_roce_v2_qp_context { #define V2_QPC_BYTE_256_SQ_FLUSH_IDX_S 16 #define V2_QPC_BYTE_256_SQ_FLUSH_IDX_M GENMASK(31, 16) +#define V2_QP_RWE_S 1 /* rdma write enable */ +#define V2_QP_RRE_S 2 /* rdma read enable */ +#define V2_QP_ATE_S 3 /* rdma atomic enable */ + struct hns_roce_v2_cqe { __le32 byte_4; union { @@ -1226,6 +1231,22 @@ struct hns_roce_query_fw_info { __le32 rsv[5]; }; +struct hns_roce_func_clear { + __le32 rst_funcid_en; + __le32 func_done; + __le32 rsv[4]; +}; + +#define FUNC_CLEAR_RST_FUN_DONE_S 0 +/* Each physical function manages up to 248 virtual functionsï¼› + * it takes up to 100ms for each function to execute clearï¼› + * if an abnormal reset occurs, it is executed twice at most; + * so it takes up to 249 * 2 * 100ms. + */ +#define HNS_ROCE_V2_FUNC_CLEAR_TIMEOUT_MSECS (249 * 2 * 100) +#define HNS_ROCE_V2_READ_FUNC_CLEAR_FLAG_INTERVAL 40 +#define HNS_ROCE_V2_READ_FUNC_CLEAR_FLAG_FAIL_WAIT 20 + struct hns_roce_cfg_llm_a { __le32 base_addr_l; __le32 base_addr_h; diff --git a/drivers/infiniband/hw/hns/hns_roce_main.c b/drivers/infiniband/hw/hns/hns_roce_main.c index 8da5f18bf820..1e4ba48f5613 100644 --- a/drivers/infiniband/hw/hns/hns_roce_main.c +++ b/drivers/infiniband/hw/hns/hns_roce_main.c @@ -57,17 +57,16 @@ int hns_get_gid_index(struct hns_roce_dev *hr_dev, u8 port, int gid_index) { return gid_index * hr_dev->caps.num_ports + port; } -EXPORT_SYMBOL_GPL(hns_get_gid_index); static int hns_roce_set_mac(struct hns_roce_dev *hr_dev, u8 port, u8 *addr) { u8 phy_port; u32 i = 0; - if (!memcmp(hr_dev->dev_addr[port], addr, MAC_ADDR_OCTET_NUM)) + if (!memcmp(hr_dev->dev_addr[port], addr, ETH_ALEN)) return 0; - for (i = 0; i < MAC_ADDR_OCTET_NUM; i++) + for (i = 0; i < ETH_ALEN; i++) hr_dev->dev_addr[port][i] = addr[i]; phy_port = hr_dev->iboe.phy_port[port]; @@ -78,18 +77,13 @@ static int hns_roce_add_gid(const struct ib_gid_attr *attr, void **context) { struct hns_roce_dev *hr_dev = to_hr_dev(attr->device); u8 port = attr->port_num - 1; - unsigned long flags; int ret; if (port >= hr_dev->caps.num_ports) return -EINVAL; - spin_lock_irqsave(&hr_dev->iboe.lock, flags); - ret = hr_dev->hw->set_gid(hr_dev, port, attr->index, &attr->gid, attr); - spin_unlock_irqrestore(&hr_dev->iboe.lock, flags); - return ret; } @@ -98,18 +92,13 @@ static int hns_roce_del_gid(const struct ib_gid_attr *attr, void **context) struct hns_roce_dev *hr_dev = to_hr_dev(attr->device); struct ib_gid_attr zattr = { }; u8 port = attr->port_num - 1; - unsigned long flags; int ret; if (port >= hr_dev->caps.num_ports) return -EINVAL; - spin_lock_irqsave(&hr_dev->iboe.lock, flags); - ret = hr_dev->hw->set_gid(hr_dev, port, attr->index, &zgid, &zattr); - spin_unlock_irqrestore(&hr_dev->iboe.lock, flags); - return ret; } @@ -272,7 +261,8 @@ static int hns_roce_query_port(struct ib_device *ib_dev, u8 port_num, props->active_mtu = mtu ? min(props->max_mtu, mtu) : IB_MTU_256; props->state = (netif_running(net_dev) && netif_carrier_ok(net_dev)) ? IB_PORT_ACTIVE : IB_PORT_DOWN; - props->phys_state = (props->state == IB_PORT_ACTIVE) ? 5 : 3; + props->phys_state = (props->state == IB_PORT_ACTIVE) ? + HNS_ROCE_PHY_LINKUP : HNS_ROCE_PHY_DISABLED; spin_unlock_irqrestore(&hr_dev->iboe.lock, flags); @@ -319,7 +309,7 @@ static int hns_roce_modify_port(struct ib_device *ib_dev, u8 port_num, int mask, static int hns_roce_alloc_ucontext(struct ib_ucontext *uctx, struct ib_udata *udata) { - int ret = 0; + int ret; struct hns_roce_ucontext *context = to_hr_ucontext(uctx); struct hns_roce_ib_alloc_ucontext_resp resp = {}; struct hns_roce_dev *hr_dev = to_hr_dev(uctx->device); @@ -423,6 +413,11 @@ static void hns_roce_unregister_device(struct hns_roce_dev *hr_dev) } static const struct ib_device_ops hns_roce_dev_ops = { + .owner = THIS_MODULE, + .driver_id = RDMA_DRIVER_HNS, + .uverbs_abi_ver = 1, + .uverbs_no_driver_id_binding = 1, + .add_gid = hns_roce_add_gid, .alloc_pd = hns_roce_alloc_pd, .alloc_ucontext = hns_roce_alloc_ucontext, @@ -451,6 +446,7 @@ static const struct ib_device_ops hns_roce_dev_ops = { .reg_user_mr = hns_roce_reg_user_mr, INIT_RDMA_OBJ_SIZE(ib_ah, hns_roce_ah, ibah), + INIT_RDMA_OBJ_SIZE(ib_cq, hns_roce_cq, ib_cq), INIT_RDMA_OBJ_SIZE(ib_pd, hns_roce_pd, ibpd), INIT_RDMA_OBJ_SIZE(ib_ucontext, hns_roce_ucontext, ibucontext), }; @@ -489,14 +485,12 @@ static int hns_roce_register_device(struct hns_roce_dev *hr_dev) ib_dev = &hr_dev->ib_dev; - ib_dev->owner = THIS_MODULE; ib_dev->node_type = RDMA_NODE_IB_CA; ib_dev->dev.parent = dev; ib_dev->phys_port_cnt = hr_dev->caps.num_ports; ib_dev->local_dma_lkey = hr_dev->caps.reserved_lkey; ib_dev->num_comp_vectors = hr_dev->caps.num_comp_vectors; - ib_dev->uverbs_abi_ver = 1; ib_dev->uverbs_cmd_mask = (1ULL << IB_USER_VERBS_CMD_GET_CONTEXT) | (1ULL << IB_USER_VERBS_CMD_QUERY_DEVICE) | @@ -545,7 +539,6 @@ static int hns_roce_register_device(struct hns_roce_dev *hr_dev) ib_set_device_ops(ib_dev, hr_dev->hw->hns_roce_dev_srq_ops); } - ib_dev->driver_id = RDMA_DRIVER_HNS; ib_set_device_ops(ib_dev, hr_dev->hw->hns_roce_dev_ops); ib_set_device_ops(ib_dev, &hns_roce_dev_ops); for (i = 0; i < hr_dev->caps.num_ports; i++) { @@ -980,7 +973,6 @@ error_failed_cmq_init: return ret; } -EXPORT_SYMBOL_GPL(hns_roce_init); void hns_roce_exit(struct hns_roce_dev *hr_dev) { @@ -1001,7 +993,6 @@ void hns_roce_exit(struct hns_roce_dev *hr_dev) if (hr_dev->hw->reset) hr_dev->hw->reset(hr_dev, false); } -EXPORT_SYMBOL_GPL(hns_roce_exit); MODULE_LICENSE("Dual BSD/GPL"); MODULE_AUTHOR("Wei Hu <xavier.huwei@huawei.com>"); diff --git a/drivers/infiniband/hw/hns/hns_roce_mr.c b/drivers/infiniband/hw/hns/hns_roce_mr.c index 6110ec408626..549e1a38dfe0 100644 --- a/drivers/infiniband/hw/hns/hns_roce_mr.c +++ b/drivers/infiniband/hw/hns/hns_roce_mr.c @@ -47,7 +47,6 @@ unsigned long key_to_hw_index(u32 key) { return (key << 24) | (key >> 8); } -EXPORT_SYMBOL_GPL(key_to_hw_index); static int hns_roce_sw2hw_mpt(struct hns_roce_dev *hr_dev, struct hns_roce_cmd_mailbox *mailbox, @@ -66,7 +65,6 @@ int hns_roce_hw2sw_mpt(struct hns_roce_dev *hr_dev, mpt_index, !mailbox, HNS_ROCE_CMD_HW2SW_MPT, HNS_ROCE_CMD_TIMEOUT_MSECS); } -EXPORT_SYMBOL_GPL(hns_roce_hw2sw_mpt); static int hns_roce_buddy_alloc(struct hns_roce_buddy *buddy, int order, unsigned long *seg) @@ -293,7 +291,6 @@ void hns_roce_mtt_cleanup(struct hns_roce_dev *hr_dev, struct hns_roce_mtt *mtt) break; } } -EXPORT_SYMBOL_GPL(hns_roce_mtt_cleanup); static void hns_roce_loop_free(struct hns_roce_dev *hr_dev, struct hns_roce_mr *mr, int err_loop_index, @@ -314,11 +311,11 @@ static void hns_roce_loop_free(struct hns_roce_dev *hr_dev, dma_free_coherent(dev, pbl_bt_sz, mr->pbl_bt_l1[i], mr->pbl_l1_dma_addr[i]); - for (j = 0; j < pbl_bt_sz / 8; j++) { + for (j = 0; j < pbl_bt_sz / BA_BYTE_LEN; j++) { if (i == loop_i && j >= loop_j) break; - bt_idx = i * pbl_bt_sz / 8 + j; + bt_idx = i * pbl_bt_sz / BA_BYTE_LEN + j; dma_free_coherent(dev, pbl_bt_sz, mr->pbl_bt_l2[bt_idx], mr->pbl_l2_dma_addr[bt_idx]); @@ -329,8 +326,8 @@ static void hns_roce_loop_free(struct hns_roce_dev *hr_dev, dma_free_coherent(dev, pbl_bt_sz, mr->pbl_bt_l1[i], mr->pbl_l1_dma_addr[i]); - for (j = 0; j < pbl_bt_sz / 8; j++) { - bt_idx = i * pbl_bt_sz / 8 + j; + for (j = 0; j < pbl_bt_sz / BA_BYTE_LEN; j++) { + bt_idx = i * pbl_bt_sz / BA_BYTE_LEN + j; dma_free_coherent(dev, pbl_bt_sz, mr->pbl_bt_l2[bt_idx], mr->pbl_l2_dma_addr[bt_idx]); @@ -533,7 +530,7 @@ static int hns_roce_mr_alloc(struct hns_roce_dev *hr_dev, u32 pd, u64 iova, { struct device *dev = hr_dev->dev; unsigned long index = 0; - int ret = 0; + int ret; /* Allocate a key for mr from mr_table */ ret = hns_roce_bitmap_alloc(&hr_dev->mr_table.mtpt_bitmap, &index); @@ -559,7 +556,8 @@ static int hns_roce_mr_alloc(struct hns_roce_dev *hr_dev, u32 pd, u64 iova, mr->pbl_l0_dma_addr = 0; } else { if (!hr_dev->caps.pbl_hop_num) { - mr->pbl_buf = dma_alloc_coherent(dev, npages * 8, + mr->pbl_buf = dma_alloc_coherent(dev, + npages * BA_BYTE_LEN, &(mr->pbl_dma_addr), GFP_KERNEL); if (!mr->pbl_buf) @@ -590,9 +588,8 @@ static void hns_roce_mhop_free(struct hns_roce_dev *hr_dev, if (mhop_num == HNS_ROCE_HOP_NUM_0) return; - /* hop_num = 1 */ if (mhop_num == 1) { - dma_free_coherent(dev, (unsigned int)(npages * 8), + dma_free_coherent(dev, (unsigned int)(npages * BA_BYTE_LEN), mr->pbl_buf, mr->pbl_dma_addr); return; } @@ -603,12 +600,13 @@ static void hns_roce_mhop_free(struct hns_roce_dev *hr_dev, if (mhop_num == 2) { for (i = 0; i < mr->l0_chunk_last_num; i++) { if (i == mr->l0_chunk_last_num - 1) { - npages_allocated = i * (pbl_bt_sz / 8); + npages_allocated = + i * (pbl_bt_sz / BA_BYTE_LEN); dma_free_coherent(dev, - (npages - npages_allocated) * 8, - mr->pbl_bt_l1[i], - mr->pbl_l1_dma_addr[i]); + (npages - npages_allocated) * BA_BYTE_LEN, + mr->pbl_bt_l1[i], + mr->pbl_l1_dma_addr[i]); break; } @@ -621,16 +619,17 @@ static void hns_roce_mhop_free(struct hns_roce_dev *hr_dev, dma_free_coherent(dev, pbl_bt_sz, mr->pbl_bt_l1[i], mr->pbl_l1_dma_addr[i]); - for (j = 0; j < pbl_bt_sz / 8; j++) { - bt_idx = i * (pbl_bt_sz / 8) + j; + for (j = 0; j < pbl_bt_sz / BA_BYTE_LEN; j++) { + bt_idx = i * (pbl_bt_sz / BA_BYTE_LEN) + j; if ((i == mr->l0_chunk_last_num - 1) && j == mr->l1_chunk_last_num - 1) { npages_allocated = bt_idx * - (pbl_bt_sz / 8); + (pbl_bt_sz / BA_BYTE_LEN); dma_free_coherent(dev, - (npages - npages_allocated) * 8, + (npages - npages_allocated) * + BA_BYTE_LEN, mr->pbl_bt_l2[bt_idx], mr->pbl_l2_dma_addr[bt_idx]); @@ -675,7 +674,8 @@ static void hns_roce_mr_free(struct hns_roce_dev *hr_dev, npages = ib_umem_page_count(mr->umem); if (!hr_dev->caps.pbl_hop_num) - dma_free_coherent(dev, (unsigned int)(npages * 8), + dma_free_coherent(dev, + (unsigned int)(npages * BA_BYTE_LEN), mr->pbl_buf, mr->pbl_dma_addr); else hns_roce_mhop_free(hr_dev, mr); @@ -1059,6 +1059,7 @@ static int hns_roce_ib_umem_write_mr(struct hns_roce_dev *hr_dev, for_each_sg_dma_page(umem->sg_head.sgl, &sg_iter, umem->nmap, 0) { page_addr = sg_page_iter_dma_address(&sg_iter); if (!hr_dev->caps.pbl_hop_num) { + /* for hip06, page addr is aligned to 4K */ mr->pbl_buf[i++] = page_addr >> 12; } else if (hr_dev->caps.pbl_hop_num == 1) { mr->pbl_buf[i++] = page_addr; @@ -1069,7 +1070,7 @@ static int hns_roce_ib_umem_write_mr(struct hns_roce_dev *hr_dev, mr->pbl_bt_l2[i][j] = page_addr; j++; - if (j >= (pbl_bt_sz / 8)) { + if (j >= (pbl_bt_sz / BA_BYTE_LEN)) { i++; j = 0; } @@ -1117,7 +1118,8 @@ struct ib_mr *hns_roce_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, } else { u64 pbl_size = 1; - bt_size = (1 << (hr_dev->caps.pbl_ba_pg_sz + PAGE_SHIFT)) / 8; + bt_size = (1 << (hr_dev->caps.pbl_ba_pg_sz + PAGE_SHIFT)) / + BA_BYTE_LEN; for (i = 0; i < hr_dev->caps.pbl_hop_num; i++) pbl_size *= bt_size; if (n > pbl_size) { @@ -1293,9 +1295,7 @@ int hns_roce_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata) } else { hns_roce_mr_free(hr_dev, mr); - if (mr->umem) - ib_umem_release(mr->umem); - + ib_umem_release(mr->umem); kfree(mr); } @@ -1491,3 +1491,119 @@ int hns_roce_dealloc_mw(struct ib_mw *ibmw) return 0; } + +void hns_roce_mtr_init(struct hns_roce_mtr *mtr, int bt_pg_shift, + int buf_pg_shift) +{ + hns_roce_hem_list_init(&mtr->hem_list, bt_pg_shift); + mtr->buf_pg_shift = buf_pg_shift; +} + +void hns_roce_mtr_cleanup(struct hns_roce_dev *hr_dev, + struct hns_roce_mtr *mtr) +{ + hns_roce_hem_list_release(hr_dev, &mtr->hem_list); +} + +static int hns_roce_write_mtr(struct hns_roce_dev *hr_dev, + struct hns_roce_mtr *mtr, dma_addr_t *bufs, + struct hns_roce_buf_region *r) +{ + int offset; + int count; + int npage; + u64 *mtts; + int end; + int i; + + offset = r->offset; + end = offset + r->count; + npage = 0; + while (offset < end) { + mtts = hns_roce_hem_list_find_mtt(hr_dev, &mtr->hem_list, + offset, &count, NULL); + if (!mtts) + return -ENOBUFS; + + /* Save page addr, low 12 bits : 0 */ + for (i = 0; i < count; i++) { + if (hr_dev->hw_rev == HNS_ROCE_HW_VER1) + mtts[i] = cpu_to_le64(bufs[npage] >> + PAGE_ADDR_SHIFT); + else + mtts[i] = cpu_to_le64(bufs[npage]); + + npage++; + } + offset += count; + } + + return 0; +} + +int hns_roce_mtr_attach(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr, + dma_addr_t **bufs, struct hns_roce_buf_region *regions, + int region_cnt) +{ + struct hns_roce_buf_region *r; + int ret; + int i; + + ret = hns_roce_hem_list_request(hr_dev, &mtr->hem_list, regions, + region_cnt); + if (ret) + return ret; + + for (i = 0; i < region_cnt; i++) { + r = ®ions[i]; + ret = hns_roce_write_mtr(hr_dev, mtr, bufs[i], r); + if (ret) { + dev_err(hr_dev->dev, + "write mtr[%d/%d] err %d,offset=%d.\n", + i, region_cnt, ret, r->offset); + goto err_write; + } + } + + return 0; + +err_write: + hns_roce_hem_list_release(hr_dev, &mtr->hem_list); + + return ret; +} + +int hns_roce_mtr_find(struct hns_roce_dev *hr_dev, struct hns_roce_mtr *mtr, + int offset, u64 *mtt_buf, int mtt_max, u64 *base_addr) +{ + u64 *mtts = mtt_buf; + int mtt_count; + int total = 0; + u64 *addr; + int npage; + int left; + + if (mtts == NULL || mtt_max < 1) + goto done; + + left = mtt_max; + while (left > 0) { + mtt_count = 0; + addr = hns_roce_hem_list_find_mtt(hr_dev, &mtr->hem_list, + offset + total, + &mtt_count, NULL); + if (!addr || !mtt_count) + goto done; + + npage = min(mtt_count, left); + memcpy(&mtts[total], addr, BA_BYTE_LEN * npage); + left -= npage; + total += npage; + } + +done: + if (base_addr) + *base_addr = mtr->hem_list.root_ba; + + return total; +} diff --git a/drivers/infiniband/hw/hns/hns_roce_pd.c b/drivers/infiniband/hw/hns/hns_roce_pd.c index 813401384d78..912b89b4da34 100644 --- a/drivers/infiniband/hw/hns/hns_roce_pd.c +++ b/drivers/infiniband/hw/hns/hns_roce_pd.c @@ -83,18 +83,16 @@ int hns_roce_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) return 0; } -EXPORT_SYMBOL_GPL(hns_roce_alloc_pd); void hns_roce_dealloc_pd(struct ib_pd *pd, struct ib_udata *udata) { hns_roce_pd_free(to_hr_dev(pd->device), to_hr_pd(pd)->pdn); } -EXPORT_SYMBOL_GPL(hns_roce_dealloc_pd); int hns_roce_uar_alloc(struct hns_roce_dev *hr_dev, struct hns_roce_uar *uar) { struct resource *res; - int ret = 0; + int ret; /* Using bitmap to manager UAR index */ ret = hns_roce_bitmap_alloc(&hr_dev->uar_table.bitmap, &uar->logic_idx); diff --git a/drivers/infiniband/hw/hns/hns_roce_qp.c b/drivers/infiniband/hw/hns/hns_roce_qp.c index 8db2817a249e..e0424029b058 100644 --- a/drivers/infiniband/hw/hns/hns_roce_qp.c +++ b/drivers/infiniband/hw/hns/hns_roce_qp.c @@ -64,7 +64,6 @@ void hns_roce_qp_event(struct hns_roce_dev *hr_dev, u32 qpn, int event_type) if (atomic_dec_and_test(&qp->refcount)) complete(&qp->free); } -EXPORT_SYMBOL_GPL(hns_roce_qp_event); static void hns_roce_ib_qp_event(struct hns_roce_qp *hr_qp, enum hns_roce_event type) @@ -139,7 +138,6 @@ enum hns_roce_qp_state to_hns_roce_state(enum ib_qp_state state) return HNS_ROCE_QP_NUM_STATE; } } -EXPORT_SYMBOL_GPL(to_hns_roce_state); static int hns_roce_gsi_qp_alloc(struct hns_roce_dev *hr_dev, unsigned long qpn, struct hns_roce_qp *hr_qp) @@ -242,7 +240,6 @@ void hns_roce_qp_remove(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp) __xa_erase(xa, hr_qp->qpn & (hr_dev->caps.num_qps - 1)); xa_unlock_irqrestore(xa, flags); } -EXPORT_SYMBOL_GPL(hns_roce_qp_remove); void hns_roce_qp_free(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp) { @@ -257,22 +254,19 @@ void hns_roce_qp_free(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp) hns_roce_table_put(hr_dev, &qp_table->trrl_table, hr_qp->qpn); hns_roce_table_put(hr_dev, &qp_table->irrl_table, hr_qp->qpn); - hns_roce_table_put(hr_dev, &qp_table->qp_table, hr_qp->qpn); } } -EXPORT_SYMBOL_GPL(hns_roce_qp_free); void hns_roce_release_range_qp(struct hns_roce_dev *hr_dev, int base_qpn, int cnt) { struct hns_roce_qp_table *qp_table = &hr_dev->qp_table; - if (base_qpn < SQP_NUM) + if (base_qpn < hr_dev->caps.reserved_qps) return; hns_roce_bitmap_free_range(&qp_table->bitmap, base_qpn, cnt, BITMAP_RR); } -EXPORT_SYMBOL_GPL(hns_roce_release_range_qp); static int hns_roce_set_rq_size(struct hns_roce_dev *hr_dev, struct ib_qp_cap *cap, bool is_user, int has_rq, @@ -392,8 +386,8 @@ static int hns_roce_set_user_sq_size(struct hns_roce_dev *hr_dev, hr_qp->sq.wqe_shift), PAGE_SIZE); } else { page_size = 1 << (hr_dev->caps.mtt_buf_pg_sz + PAGE_SHIFT); - hr_qp->sge.sge_cnt = - max(page_size / (1 << hr_qp->sge.sge_shift), ex_sge_num); + hr_qp->sge.sge_cnt = ex_sge_num ? + max(page_size / (1 << hr_qp->sge.sge_shift), ex_sge_num) : 0; hr_qp->buff_size = HNS_ROCE_ALOGN_UP((hr_qp->rq.wqe_cnt << hr_qp->rq.wqe_shift), page_size) + HNS_ROCE_ALOGN_UP((hr_qp->sge.sge_cnt << @@ -422,6 +416,91 @@ static int hns_roce_set_user_sq_size(struct hns_roce_dev *hr_dev, return 0; } +static int split_wqe_buf_region(struct hns_roce_dev *hr_dev, + struct hns_roce_qp *hr_qp, + struct hns_roce_buf_region *regions, + int region_max, int page_shift) +{ + int page_size = 1 << page_shift; + bool is_extend_sge; + int region_cnt = 0; + int buf_size; + int buf_cnt; + + if (hr_qp->buff_size < 1 || region_max < 1) + return region_cnt; + + if (hr_qp->sge.sge_cnt > 0) + is_extend_sge = true; + else + is_extend_sge = false; + + /* sq region */ + if (is_extend_sge) + buf_size = hr_qp->sge.offset - hr_qp->sq.offset; + else + buf_size = hr_qp->rq.offset - hr_qp->sq.offset; + + if (buf_size > 0 && region_cnt < region_max) { + buf_cnt = DIV_ROUND_UP(buf_size, page_size); + hns_roce_init_buf_region(®ions[region_cnt], + hr_dev->caps.wqe_sq_hop_num, + hr_qp->sq.offset / page_size, + buf_cnt); + region_cnt++; + } + + /* sge region */ + if (is_extend_sge) { + buf_size = hr_qp->rq.offset - hr_qp->sge.offset; + if (buf_size > 0 && region_cnt < region_max) { + buf_cnt = DIV_ROUND_UP(buf_size, page_size); + hns_roce_init_buf_region(®ions[region_cnt], + hr_dev->caps.wqe_sge_hop_num, + hr_qp->sge.offset / page_size, + buf_cnt); + region_cnt++; + } + } + + /* rq region */ + buf_size = hr_qp->buff_size - hr_qp->rq.offset; + if (buf_size > 0) { + buf_cnt = DIV_ROUND_UP(buf_size, page_size); + hns_roce_init_buf_region(®ions[region_cnt], + hr_dev->caps.wqe_rq_hop_num, + hr_qp->rq.offset / page_size, + buf_cnt); + region_cnt++; + } + + return region_cnt; +} + +static int calc_wqe_bt_page_shift(struct hns_roce_dev *hr_dev, + struct hns_roce_buf_region *regions, + int region_cnt) +{ + int bt_pg_shift; + int ba_num; + int ret; + + bt_pg_shift = PAGE_SHIFT + hr_dev->caps.mtt_ba_pg_sz; + + /* all root ba entries must in one bt page */ + do { + ba_num = (1 << bt_pg_shift) / BA_BYTE_LEN; + ret = hns_roce_hem_list_calc_root_ba(regions, region_cnt, + ba_num); + if (ret <= ba_num) + break; + + bt_pg_shift++; + } while (ret > ba_num); + + return bt_pg_shift - PAGE_SHIFT; +} + static int hns_roce_set_kernel_sq_size(struct hns_roce_dev *hr_dev, struct ib_qp_cap *cap, struct hns_roce_qp *hr_qp) @@ -534,15 +613,17 @@ static int hns_roce_create_qp_common(struct hns_roce_dev *hr_dev, struct ib_udata *udata, unsigned long sqpn, struct hns_roce_qp *hr_qp) { + dma_addr_t *buf_list[ARRAY_SIZE(hr_qp->regions)] = { 0 }; struct device *dev = hr_dev->dev; struct hns_roce_ib_create_qp ucmd; struct hns_roce_ib_create_qp_resp resp = {}; struct hns_roce_ucontext *uctx = rdma_udata_to_drv_context( udata, struct hns_roce_ucontext, ibucontext); + struct hns_roce_buf_region *r; unsigned long qpn = 0; - int ret = 0; u32 page_shift; - u32 npages; + int buf_count; + int ret; int i; mutex_init(&hr_qp->mutex); @@ -596,6 +677,7 @@ static int hns_roce_create_qp_common(struct hns_roce_dev *hr_dev, init_attr->cap.max_recv_sge]; } + page_shift = PAGE_SHIFT + hr_dev->caps.mtt_buf_pg_sz; if (udata) { if (ib_copy_from_udata(&ucmd, udata, sizeof(ucmd))) { dev_err(dev, "ib_copy_from_udata error for create qp\n"); @@ -617,32 +699,28 @@ static int hns_roce_create_qp_common(struct hns_roce_dev *hr_dev, ret = PTR_ERR(hr_qp->umem); goto err_rq_sge_list; } - - hr_qp->mtt.mtt_type = MTT_TYPE_WQE; - page_shift = PAGE_SHIFT; - if (hr_dev->caps.mtt_buf_pg_sz) { - npages = (ib_umem_page_count(hr_qp->umem) + - (1 << hr_dev->caps.mtt_buf_pg_sz) - 1) / - (1 << hr_dev->caps.mtt_buf_pg_sz); - page_shift += hr_dev->caps.mtt_buf_pg_sz; - ret = hns_roce_mtt_init(hr_dev, npages, - page_shift, - &hr_qp->mtt); - } else { - ret = hns_roce_mtt_init(hr_dev, - ib_umem_page_count(hr_qp->umem), - page_shift, &hr_qp->mtt); - } + hr_qp->region_cnt = split_wqe_buf_region(hr_dev, hr_qp, + hr_qp->regions, ARRAY_SIZE(hr_qp->regions), + page_shift); + ret = hns_roce_alloc_buf_list(hr_qp->regions, buf_list, + hr_qp->region_cnt); if (ret) { - dev_err(dev, "hns_roce_mtt_init error for create qp\n"); - goto err_buf; + dev_err(dev, "alloc buf_list error for create qp\n"); + goto err_alloc_list; } - ret = hns_roce_ib_umem_write_mtt(hr_dev, &hr_qp->mtt, - hr_qp->umem); - if (ret) { - dev_err(dev, "hns_roce_ib_umem_write_mtt error for create qp\n"); - goto err_mtt; + for (i = 0; i < hr_qp->region_cnt; i++) { + r = &hr_qp->regions[i]; + buf_count = hns_roce_get_umem_bufs(hr_dev, + buf_list[i], r->count, r->offset, + hr_qp->umem, page_shift); + if (buf_count != r->count) { + dev_err(dev, + "get umem buf err, expect %d,ret %d.\n", + r->count, buf_count); + ret = -ENOBUFS; + goto err_get_bufs; + } } if ((hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_SQ_RECORD_DB) && @@ -653,7 +731,7 @@ static int hns_roce_create_qp_common(struct hns_roce_dev *hr_dev, &hr_qp->sdb); if (ret) { dev_err(dev, "sq record doorbell map failed!\n"); - goto err_mtt; + goto err_get_bufs; } /* indicate kernel supports sq record db */ @@ -715,7 +793,6 @@ static int hns_roce_create_qp_common(struct hns_roce_dev *hr_dev, } /* Allocate QP buf */ - page_shift = PAGE_SHIFT + hr_dev->caps.mtt_buf_pg_sz; if (hns_roce_buf_alloc(hr_dev, hr_qp->buff_size, (1 << page_shift) * 2, &hr_qp->hr_buf, page_shift)) { @@ -723,21 +800,28 @@ static int hns_roce_create_qp_common(struct hns_roce_dev *hr_dev, ret = -ENOMEM; goto err_db; } - - hr_qp->mtt.mtt_type = MTT_TYPE_WQE; - /* Write MTT */ - ret = hns_roce_mtt_init(hr_dev, hr_qp->hr_buf.npages, - hr_qp->hr_buf.page_shift, &hr_qp->mtt); + hr_qp->region_cnt = split_wqe_buf_region(hr_dev, hr_qp, + hr_qp->regions, ARRAY_SIZE(hr_qp->regions), + page_shift); + ret = hns_roce_alloc_buf_list(hr_qp->regions, buf_list, + hr_qp->region_cnt); if (ret) { - dev_err(dev, "hns_roce_mtt_init error for kernel create qp\n"); - goto err_buf; + dev_err(dev, "alloc buf_list error for create qp!\n"); + goto err_alloc_list; } - ret = hns_roce_buf_write_mtt(hr_dev, &hr_qp->mtt, - &hr_qp->hr_buf); - if (ret) { - dev_err(dev, "hns_roce_buf_write_mtt error for kernel create qp\n"); - goto err_mtt; + for (i = 0; i < hr_qp->region_cnt; i++) { + r = &hr_qp->regions[i]; + buf_count = hns_roce_get_kmem_bufs(hr_dev, + buf_list[i], r->count, r->offset, + &hr_qp->hr_buf); + if (buf_count != r->count) { + dev_err(dev, + "get kmem buf err, expect %d,ret %d.\n", + r->count, buf_count); + ret = -ENOBUFS; + goto err_get_bufs; + } } hr_qp->sq.wrid = kcalloc(hr_qp->sq.wqe_cnt, sizeof(u64), @@ -761,6 +845,17 @@ static int hns_roce_create_qp_common(struct hns_roce_dev *hr_dev, } } + hr_qp->wqe_bt_pg_shift = calc_wqe_bt_page_shift(hr_dev, hr_qp->regions, + hr_qp->region_cnt); + hns_roce_mtr_init(&hr_qp->mtr, PAGE_SHIFT + hr_qp->wqe_bt_pg_shift, + page_shift); + ret = hns_roce_mtr_attach(hr_dev, &hr_qp->mtr, buf_list, + hr_qp->regions, hr_qp->region_cnt); + if (ret) { + dev_err(dev, "mtr attach error for create qp\n"); + goto err_mtr; + } + if (init_attr->qp_type == IB_QPT_GSI && hr_dev->hw_rev == HNS_ROCE_HW_VER1) { /* In v1 engine, GSI QP context in RoCE engine's register */ @@ -796,6 +891,7 @@ static int hns_roce_create_qp_common(struct hns_roce_dev *hr_dev, } hr_qp->event = hns_roce_ib_qp_event; + hns_roce_free_buf_list(buf_list, hr_qp->region_cnt); return 0; @@ -810,6 +906,9 @@ err_qpn: if (!sqpn) hns_roce_release_range_qp(hr_dev, qpn, 1); +err_mtr: + hns_roce_mtr_cleanup(hr_dev, &hr_qp->mtr); + err_wrid: if (udata) { if ((hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RECORD_DB) && @@ -829,14 +928,13 @@ err_sq_dbmap: hns_roce_qp_has_sq(init_attr)) hns_roce_db_unmap_user(uctx, &hr_qp->sdb); -err_mtt: - hns_roce_mtt_cleanup(hr_dev, &hr_qp->mtt); +err_get_bufs: + hns_roce_free_buf_list(buf_list, hr_qp->region_cnt); -err_buf: - if (hr_qp->umem) - ib_umem_release(hr_qp->umem); - else +err_alloc_list: + if (!hr_qp->umem) hns_roce_buf_free(hr_dev, hr_qp->buff_size, &hr_qp->hr_buf); + ib_umem_release(hr_qp->umem); err_db: if (!udata && hns_roce_qp_has_rq(init_attr) && @@ -923,7 +1021,6 @@ struct ib_qp *hns_roce_create_qp(struct ib_pd *pd, return &hr_qp->ibqp; } -EXPORT_SYMBOL_GPL(hns_roce_create_qp); int to_hr_qp_type(int qp_type) { @@ -942,7 +1039,6 @@ int to_hr_qp_type(int qp_type) return transport_type; } -EXPORT_SYMBOL_GPL(to_hr_qp_type); int hns_roce_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int attr_mask, struct ib_udata *udata) @@ -1062,7 +1158,6 @@ void hns_roce_lock_cqs(struct hns_roce_cq *send_cq, struct hns_roce_cq *recv_cq) spin_lock_nested(&send_cq->lock, SINGLE_DEPTH_NESTING); } } -EXPORT_SYMBOL_GPL(hns_roce_lock_cqs); void hns_roce_unlock_cqs(struct hns_roce_cq *send_cq, struct hns_roce_cq *recv_cq) __releases(&send_cq->lock) @@ -1079,7 +1174,6 @@ void hns_roce_unlock_cqs(struct hns_roce_cq *send_cq, spin_unlock_irq(&recv_cq->lock); } } -EXPORT_SYMBOL_GPL(hns_roce_unlock_cqs); static void *get_wqe(struct hns_roce_qp *hr_qp, int offset) { @@ -1091,20 +1185,17 @@ void *get_recv_wqe(struct hns_roce_qp *hr_qp, int n) { return get_wqe(hr_qp, hr_qp->rq.offset + (n << hr_qp->rq.wqe_shift)); } -EXPORT_SYMBOL_GPL(get_recv_wqe); void *get_send_wqe(struct hns_roce_qp *hr_qp, int n) { return get_wqe(hr_qp, hr_qp->sq.offset + (n << hr_qp->sq.wqe_shift)); } -EXPORT_SYMBOL_GPL(get_send_wqe); void *get_send_extend_sge(struct hns_roce_qp *hr_qp, int n) { return hns_roce_buf_offset(&hr_qp->hr_buf, hr_qp->sge.offset + (n << hr_qp->sge.sge_shift)); } -EXPORT_SYMBOL_GPL(get_send_extend_sge); bool hns_roce_wq_overflow(struct hns_roce_wq *hr_wq, int nreq, struct ib_cq *ib_cq) @@ -1123,7 +1214,6 @@ bool hns_roce_wq_overflow(struct hns_roce_wq *hr_wq, int nreq, return cur + nreq >= hr_wq->max_post; } -EXPORT_SYMBOL_GPL(hns_roce_wq_overflow); int hns_roce_init_qp_table(struct hns_roce_dev *hr_dev) { @@ -1135,11 +1225,7 @@ int hns_roce_init_qp_table(struct hns_roce_dev *hr_dev) mutex_init(&qp_table->scc_mutex); xa_init(&hr_dev->qp_table_xa); - /* In hw v1, a port include two SQP, six ports total 12 */ - if (hr_dev->caps.max_sq_sg <= 2) - reserved_from_bot = SQP_NUM; - else - reserved_from_bot = hr_dev->caps.reserved_qps; + reserved_from_bot = hr_dev->caps.reserved_qps; ret = hns_roce_bitmap_init(&qp_table->bitmap, hr_dev->caps.num_qps, hr_dev->caps.num_qps - 1, reserved_from_bot, diff --git a/drivers/infiniband/hw/hns/hns_roce_srq.c b/drivers/infiniband/hw/hns/hns_roce_srq.c index b3421b1f21e0..38bb548eaa6d 100644 --- a/drivers/infiniband/hw/hns/hns_roce_srq.c +++ b/drivers/infiniband/hw/hns/hns_roce_srq.c @@ -30,7 +30,6 @@ void hns_roce_srq_event(struct hns_roce_dev *hr_dev, u32 srqn, int event_type) if (atomic_dec_and_test(&srq->refcount)) complete(&srq->free); } -EXPORT_SYMBOL_GPL(hns_roce_srq_event); static void hns_roce_ib_srq_event(struct hns_roce_srq *srq, enum hns_roce_event event_type) @@ -181,28 +180,19 @@ static int hns_roce_create_idx_que(struct ib_pd *pd, struct hns_roce_srq *srq, { struct hns_roce_dev *hr_dev = to_hr_dev(pd->device); struct hns_roce_idx_que *idx_que = &srq->idx_que; - u32 bitmap_num; - int i; - bitmap_num = HNS_ROCE_ALOGN_UP(srq->max, 8 * sizeof(u64)); - - idx_que->bitmap = kcalloc(1, bitmap_num / 8, GFP_KERNEL); + idx_que->bitmap = bitmap_zalloc(srq->max, GFP_KERNEL); if (!idx_que->bitmap) return -ENOMEM; - bitmap_num = bitmap_num / (8 * sizeof(u64)); - idx_que->buf_size = srq->idx_que.buf_size; if (hns_roce_buf_alloc(hr_dev, idx_que->buf_size, (1 << page_shift) * 2, &idx_que->idx_buf, page_shift)) { - kfree(idx_que->bitmap); + bitmap_free(idx_que->bitmap); return -ENOMEM; } - for (i = 0; i < bitmap_num; i++) - idx_que->bitmap[i] = ~(0UL); - return 0; } @@ -264,8 +254,7 @@ int hns_roce_create_srq(struct ib_srq *ib_srq, } else ret = hns_roce_mtt_init(hr_dev, ib_umem_page_count(srq->umem), - srq->umem->page_shift, - &srq->mtt); + PAGE_SHIFT, &srq->mtt); if (ret) goto err_buf; @@ -291,10 +280,9 @@ int hns_roce_create_srq(struct ib_srq *ib_srq, ret = hns_roce_mtt_init(hr_dev, npages, page_shift, &srq->idx_que.mtt); } else { - ret = hns_roce_mtt_init(hr_dev, - ib_umem_page_count(srq->idx_que.umem), - srq->idx_que.umem->page_shift, - &srq->idx_que.mtt); + ret = hns_roce_mtt_init( + hr_dev, ib_umem_page_count(srq->idx_que.umem), + PAGE_SHIFT, &srq->idx_que.mtt); } if (ret) { @@ -391,21 +379,19 @@ err_idx_buf: hns_roce_mtt_cleanup(hr_dev, &srq->idx_que.mtt); err_idx_mtt: - if (udata) - ib_umem_release(srq->idx_que.umem); + ib_umem_release(srq->idx_que.umem); err_create_idx: hns_roce_buf_free(hr_dev, srq->idx_que.buf_size, &srq->idx_que.idx_buf); - kfree(srq->idx_que.bitmap); + bitmap_free(srq->idx_que.bitmap); err_srq_mtt: hns_roce_mtt_cleanup(hr_dev, &srq->mtt); err_buf: - if (udata) - ib_umem_release(srq->umem); - else + ib_umem_release(srq->umem); + if (!udata) hns_roce_buf_free(hr_dev, srq_buf_size, &srq->buf); return ret; @@ -419,15 +405,15 @@ void hns_roce_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata) hns_roce_srq_free(hr_dev, srq); hns_roce_mtt_cleanup(hr_dev, &srq->mtt); - if (ibsrq->uobject) { + if (udata) { hns_roce_mtt_cleanup(hr_dev, &srq->idx_que.mtt); - ib_umem_release(srq->idx_que.umem); - ib_umem_release(srq->umem); } else { kvfree(srq->wrid); hns_roce_buf_free(hr_dev, srq->max << srq->wqe_shift, &srq->buf); } + ib_umem_release(srq->idx_que.umem); + ib_umem_release(srq->umem); } int hns_roce_init_srq_table(struct hns_roce_dev *hr_dev) diff --git a/drivers/infiniband/hw/i40iw/i40iw_cm.c b/drivers/infiniband/hw/i40iw/i40iw_cm.c index 700a5d06b60c..2d6a378e8560 100644 --- a/drivers/infiniband/hw/i40iw/i40iw_cm.c +++ b/drivers/infiniband/hw/i40iw/i40iw_cm.c @@ -4279,11 +4279,11 @@ static void i40iw_qhash_ctrl(struct i40iw_device *iwdev, /* if not found then add a child listener if interface is going up */ if (!ifup) return; - child_listen_node = kzalloc(sizeof(*child_listen_node), GFP_ATOMIC); + child_listen_node = kmemdup(parent_listen_node, + sizeof(*child_listen_node), GFP_ATOMIC); if (!child_listen_node) return; node_allocated = true; - memcpy(child_listen_node, parent_listen_node, sizeof(*child_listen_node)); memcpy(child_listen_node->loc_addr, ipaddr, ipv4 ? 4 : 16); diff --git a/drivers/infiniband/hw/i40iw/i40iw_verbs.c b/drivers/infiniband/hw/i40iw/i40iw_verbs.c index 5689d742bafb..d169a8031375 100644 --- a/drivers/infiniband/hw/i40iw/i40iw_verbs.c +++ b/drivers/infiniband/hw/i40iw/i40iw_verbs.c @@ -772,6 +772,8 @@ static int i40iw_query_qp(struct ib_qp *ibqp, struct i40iw_qp *iwqp = to_iwqp(ibqp); struct i40iw_sc_qp *qp = &iwqp->sc_qp; + attr->qp_state = iwqp->ibqp_state; + attr->cur_qp_state = attr->qp_state; attr->qp_access_flags = 0; attr->cap.max_send_wr = qp->qp_uk.sq_size; attr->cap.max_recv_wr = qp->qp_uk.rq_size; @@ -1064,44 +1066,38 @@ void i40iw_cq_wq_destroy(struct i40iw_device *iwdev, struct i40iw_sc_cq *cq) * @ib_cq: cq pointer * @udata: user data or NULL for kernel object */ -static int i40iw_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata) +static void i40iw_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata) { struct i40iw_cq *iwcq; struct i40iw_device *iwdev; struct i40iw_sc_cq *cq; - if (!ib_cq) { - i40iw_pr_err("ib_cq == NULL\n"); - return 0; - } - iwcq = to_iwcq(ib_cq); iwdev = to_iwdev(ib_cq->device); cq = &iwcq->sc_cq; i40iw_cq_wq_destroy(iwdev, cq); cq_free_resources(iwdev, iwcq); - kfree(iwcq); i40iw_rem_devusecount(iwdev); - return 0; } /** * i40iw_create_cq - create cq - * @ibdev: device pointer from stack + * @ibcq: CQ allocated * @attr: attributes for cq * @udata: user data */ -static struct ib_cq *i40iw_create_cq(struct ib_device *ibdev, - const struct ib_cq_init_attr *attr, - struct ib_udata *udata) +static int i40iw_create_cq(struct ib_cq *ibcq, + const struct ib_cq_init_attr *attr, + struct ib_udata *udata) { + struct ib_device *ibdev = ibcq->device; struct i40iw_device *iwdev = to_iwdev(ibdev); - struct i40iw_cq *iwcq; + struct i40iw_cq *iwcq = to_iwcq(ibcq); struct i40iw_pbl *iwpbl; u32 cq_num = 0; struct i40iw_sc_cq *cq; struct i40iw_sc_dev *dev = &iwdev->sc_dev; - struct i40iw_cq_init_info info; + struct i40iw_cq_init_info info = {}; enum i40iw_status_code status; struct i40iw_cqp_request *cqp_request; struct cqp_commands_info *cqp_info; @@ -1111,22 +1107,16 @@ static struct ib_cq *i40iw_create_cq(struct ib_device *ibdev, int entries = attr->cqe; if (iwdev->closing) - return ERR_PTR(-ENODEV); + return -ENODEV; if (entries > iwdev->max_cqe) - return ERR_PTR(-EINVAL); - - iwcq = kzalloc(sizeof(*iwcq), GFP_KERNEL); - if (!iwcq) - return ERR_PTR(-ENOMEM); - - memset(&info, 0, sizeof(info)); + return -EINVAL; err_code = i40iw_alloc_resource(iwdev, iwdev->allocated_cqs, iwdev->max_cq, &cq_num, &iwdev->next_cq); if (err_code) - goto error; + return err_code; cq = &iwcq->sc_cq; cq->back_cq = (void *)iwcq; @@ -1233,15 +1223,13 @@ static struct ib_cq *i40iw_create_cq(struct ib_device *ibdev, } i40iw_add_devusecount(iwdev); - return (struct ib_cq *)iwcq; + return 0; cq_destroy: i40iw_cq_wq_destroy(iwdev, cq); cq_free_resources: cq_free_resources(iwdev, iwcq); -error: - kfree(iwcq); - return ERR_PTR(err_code); + return err_code; } /** @@ -2018,8 +2006,7 @@ static int i40iw_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata) struct cqp_commands_info *cqp_info; u32 stag_idx; - if (iwmr->region) - ib_umem_release(iwmr->region); + ib_umem_release(iwmr->region); if (iwmr->type != IW_MEMREG_TYPE_MEM) { /* region is released. only test for userness. */ @@ -2655,6 +2642,11 @@ static int i40iw_query_pkey(struct ib_device *ibdev, } static const struct ib_device_ops i40iw_dev_ops = { + .owner = THIS_MODULE, + .driver_id = RDMA_DRIVER_I40IW, + /* NOTE: Older kernels wrongly use 0 for the uverbs_abi_ver */ + .uverbs_abi_ver = I40IW_ABI_VER, + .alloc_hw_stats = i40iw_alloc_hw_stats, .alloc_mr = i40iw_alloc_mr, .alloc_pd = i40iw_alloc_pd, @@ -2694,6 +2686,7 @@ static const struct ib_device_ops i40iw_dev_ops = { .reg_user_mr = i40iw_reg_user_mr, .req_notify_cq = i40iw_req_notify_cq, INIT_RDMA_OBJ_SIZE(ib_pd, i40iw_pd, ibpd), + INIT_RDMA_OBJ_SIZE(ib_cq, i40iw_cq, ibcq), INIT_RDMA_OBJ_SIZE(ib_ucontext, i40iw_ucontext, ibucontext), }; @@ -2712,7 +2705,6 @@ static struct i40iw_ib_device *i40iw_init_rdma_device(struct i40iw_device *iwdev i40iw_pr_err("iwdev == NULL\n"); return NULL; } - iwibdev->ibdev.owner = THIS_MODULE; iwdev->iwibdev = iwibdev; iwibdev->iwdev = iwdev; @@ -2771,9 +2763,6 @@ void i40iw_port_ibevent(struct i40iw_device *iwdev) */ void i40iw_destroy_rdma_device(struct i40iw_ib_device *iwibdev) { - if (!iwibdev) - return; - ib_unregister_device(&iwibdev->ibdev); wait_event_timeout(iwibdev->iwdev->close_wq, !atomic64_read(&iwibdev->iwdev->use_count), @@ -2795,7 +2784,6 @@ int i40iw_register_rdma_device(struct i40iw_device *iwdev) return -ENOMEM; iwibdev = iwdev->iwibdev; rdma_set_device_sysfs_group(&iwibdev->ibdev, &i40iw_attr_group); - iwibdev->ibdev.driver_id = RDMA_DRIVER_I40IW; ret = ib_register_device(&iwibdev->ibdev, "i40iw%d"); if (ret) goto error; diff --git a/drivers/infiniband/hw/mlx4/cq.c b/drivers/infiniband/hw/mlx4/cq.c index 022a0b4ea452..a7d238d312f0 100644 --- a/drivers/infiniband/hw/mlx4/cq.c +++ b/drivers/infiniband/hw/mlx4/cq.c @@ -172,14 +172,14 @@ err_buf: } #define CQ_CREATE_FLAGS_SUPPORTED IB_UVERBS_CQ_FLAGS_TIMESTAMP_COMPLETION -struct ib_cq *mlx4_ib_create_cq(struct ib_device *ibdev, - const struct ib_cq_init_attr *attr, - struct ib_udata *udata) +int mlx4_ib_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, + struct ib_udata *udata) { + struct ib_device *ibdev = ibcq->device; int entries = attr->cqe; int vector = attr->comp_vector; struct mlx4_ib_dev *dev = to_mdev(ibdev); - struct mlx4_ib_cq *cq; + struct mlx4_ib_cq *cq = to_mcq(ibcq); struct mlx4_uar *uar; void *buf_addr; int err; @@ -187,14 +187,10 @@ struct ib_cq *mlx4_ib_create_cq(struct ib_device *ibdev, udata, struct mlx4_ib_ucontext, ibucontext); if (entries < 1 || entries > dev->dev->caps.max_cqes) - return ERR_PTR(-EINVAL); + return -EINVAL; if (attr->flags & ~CQ_CREATE_FLAGS_SUPPORTED) - return ERR_PTR(-EINVAL); - - cq = kzalloc(sizeof(*cq), GFP_KERNEL); - if (!cq) - return ERR_PTR(-ENOMEM); + return -EINVAL; entries = roundup_pow_of_two(entries + 1); cq->ibcq.cqe = entries - 1; @@ -269,7 +265,7 @@ struct ib_cq *mlx4_ib_create_cq(struct ib_device *ibdev, goto err_cq_free; } - return &cq->ibcq; + return 0; err_cq_free: mlx4_cq_free(dev->dev, &cq->mcq); @@ -281,19 +277,15 @@ err_dbmap: err_mtt: mlx4_mtt_cleanup(dev->dev, &cq->buf.mtt); - if (udata) - ib_umem_release(cq->umem); - else + ib_umem_release(cq->umem); + if (!udata) mlx4_ib_free_cq_buf(dev, &cq->buf, cq->ibcq.cqe); err_db: if (!udata) mlx4_db_free(dev->dev, &cq->db); - err_cq: - kfree(cq); - - return ERR_PTR(err); + return err; } static int mlx4_alloc_resize_buf(struct mlx4_ib_dev *dev, struct mlx4_ib_cq *cq, @@ -475,18 +467,15 @@ err_buf: kfree(cq->resize_buf); cq->resize_buf = NULL; - if (cq->resize_umem) { - ib_umem_release(cq->resize_umem); - cq->resize_umem = NULL; - } - + ib_umem_release(cq->resize_umem); + cq->resize_umem = NULL; out: mutex_unlock(&cq->resize_mutex); return err; } -int mlx4_ib_destroy_cq(struct ib_cq *cq, struct ib_udata *udata) +void mlx4_ib_destroy_cq(struct ib_cq *cq, struct ib_udata *udata) { struct mlx4_ib_dev *dev = to_mdev(cq->device); struct mlx4_ib_cq *mcq = to_mcq(cq); @@ -501,15 +490,11 @@ int mlx4_ib_destroy_cq(struct ib_cq *cq, struct ib_udata *udata) struct mlx4_ib_ucontext, ibucontext), &mcq->db); - ib_umem_release(mcq->umem); } else { mlx4_ib_free_cq_buf(dev, &mcq->buf, cq->cqe); mlx4_db_free(dev->dev, &mcq->db); } - - kfree(mcq); - - return 0; + ib_umem_release(mcq->umem); } static void dump_cqe(void *cqe) diff --git a/drivers/infiniband/hw/mlx4/main.c b/drivers/infiniband/hw/mlx4/main.c index 25d09d53b51c..8790101facb7 100644 --- a/drivers/infiniband/hw/mlx4/main.c +++ b/drivers/infiniband/hw/mlx4/main.c @@ -1089,7 +1089,8 @@ static int mlx4_ib_alloc_ucontext(struct ib_ucontext *uctx, if (!dev->ib_active) return -EAGAIN; - if (ibdev->uverbs_abi_ver == MLX4_IB_UVERBS_NO_DEV_CAPS_ABI_VERSION) { + if (ibdev->ops.uverbs_abi_ver == + MLX4_IB_UVERBS_NO_DEV_CAPS_ABI_VERSION) { resp_v3.qp_tab_size = dev->dev->caps.num_qps; resp_v3.bf_reg_size = dev->dev->caps.bf_reg_size; resp_v3.bf_regs_per_page = dev->dev->caps.bf_regs_per_page; @@ -1111,7 +1112,7 @@ static int mlx4_ib_alloc_ucontext(struct ib_ucontext *uctx, INIT_LIST_HEAD(&context->wqn_ranges_list); mutex_init(&context->wqn_ranges_mutex); - if (ibdev->uverbs_abi_ver == MLX4_IB_UVERBS_NO_DEV_CAPS_ABI_VERSION) + if (ibdev->ops.uverbs_abi_ver == MLX4_IB_UVERBS_NO_DEV_CAPS_ABI_VERSION) err = ib_copy_to_udata(udata, &resp_v3, sizeof(resp_v3)); else err = ib_copy_to_udata(udata, &resp, sizeof(resp)); @@ -2509,6 +2510,10 @@ static void get_fw_ver_str(struct ib_device *device, char *str) } static const struct ib_device_ops mlx4_ib_dev_ops = { + .owner = THIS_MODULE, + .driver_id = RDMA_DRIVER_MLX4, + .uverbs_abi_ver = MLX4_IB_UVERBS_ABI_VERSION, + .add_gid = mlx4_ib_add_gid, .alloc_mr = mlx4_ib_alloc_mr, .alloc_pd = mlx4_ib_alloc_pd, @@ -2560,6 +2565,7 @@ static const struct ib_device_ops mlx4_ib_dev_ops = { .resize_cq = mlx4_ib_resize_cq, INIT_RDMA_OBJ_SIZE(ib_ah, mlx4_ib_ah, ibah), + INIT_RDMA_OBJ_SIZE(ib_cq, mlx4_ib_cq, ibcq), INIT_RDMA_OBJ_SIZE(ib_pd, mlx4_ib_pd, ibpd), INIT_RDMA_OBJ_SIZE(ib_srq, mlx4_ib_srq, ibsrq), INIT_RDMA_OBJ_SIZE(ib_ucontext, mlx4_ib_ucontext, ibucontext), @@ -2642,7 +2648,6 @@ static void *mlx4_ib_add(struct mlx4_dev *dev) ibdev->dev = dev; ibdev->bond_next_port = 0; - ibdev->ib_dev.owner = THIS_MODULE; ibdev->ib_dev.node_type = RDMA_NODE_IB_CA; ibdev->ib_dev.local_dma_lkey = dev->caps.reserved_lkey; ibdev->num_ports = num_ports; @@ -2651,11 +2656,6 @@ static void *mlx4_ib_add(struct mlx4_dev *dev) ibdev->ib_dev.num_comp_vectors = dev->caps.num_comp_vectors; ibdev->ib_dev.dev.parent = &dev->persist->pdev->dev; - if (dev->caps.userspace_caps) - ibdev->ib_dev.uverbs_abi_ver = MLX4_IB_UVERBS_ABI_VERSION; - else - ibdev->ib_dev.uverbs_abi_ver = MLX4_IB_UVERBS_NO_DEV_CAPS_ABI_VERSION; - ibdev->ib_dev.uverbs_cmd_mask = (1ull << IB_USER_VERBS_CMD_GET_CONTEXT) | (1ull << IB_USER_VERBS_CMD_QUERY_DEVICE) | @@ -2729,6 +2729,10 @@ static void *mlx4_ib_add(struct mlx4_dev *dev) ib_set_device_ops(&ibdev->ib_dev, &mlx4_ib_dev_fs_ops); } + if (!dev->caps.userspace_caps) + ibdev->ib_dev.ops.uverbs_abi_ver = + MLX4_IB_UVERBS_NO_DEV_CAPS_ABI_VERSION; + mlx4_ib_alloc_eqs(dev, ibdev); spin_lock_init(&iboe->lock); @@ -2839,7 +2843,6 @@ static void *mlx4_ib_add(struct mlx4_dev *dev) goto err_steer_free_bitmap; rdma_set_device_sysfs_group(&ibdev->ib_dev, &mlx4_attr_group); - ibdev->ib_dev.driver_id = RDMA_DRIVER_MLX4; if (ib_register_device(&ibdev->ib_dev, "mlx4_%d")) goto err_diag_counters; diff --git a/drivers/infiniband/hw/mlx4/mlx4_ib.h b/drivers/infiniband/hw/mlx4/mlx4_ib.h index 26897102057d..eb53bb4c0c91 100644 --- a/drivers/infiniband/hw/mlx4/mlx4_ib.h +++ b/drivers/infiniband/hw/mlx4/mlx4_ib.h @@ -743,10 +743,9 @@ int mlx4_ib_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents, unsigned int *sg_offset); int mlx4_ib_modify_cq(struct ib_cq *cq, u16 cq_count, u16 cq_period); int mlx4_ib_resize_cq(struct ib_cq *ibcq, int entries, struct ib_udata *udata); -struct ib_cq *mlx4_ib_create_cq(struct ib_device *ibdev, - const struct ib_cq_init_attr *attr, - struct ib_udata *udata); -int mlx4_ib_destroy_cq(struct ib_cq *cq, struct ib_udata *udata); +int mlx4_ib_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, + struct ib_udata *udata); +void mlx4_ib_destroy_cq(struct ib_cq *cq, struct ib_udata *udata); int mlx4_ib_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc); int mlx4_ib_arm_cq(struct ib_cq *cq, enum ib_cq_notify_flags flags); void __mlx4_ib_cq_clean(struct mlx4_ib_cq *cq, u32 qpn, struct mlx4_ib_srq *srq); @@ -907,7 +906,7 @@ void mlx4_ib_sl2vl_update(struct mlx4_ib_dev *mdev, int port); struct ib_wq *mlx4_ib_create_wq(struct ib_pd *pd, struct ib_wq_init_attr *init_attr, struct ib_udata *udata); -int mlx4_ib_destroy_wq(struct ib_wq *wq, struct ib_udata *udata); +void mlx4_ib_destroy_wq(struct ib_wq *wq, struct ib_udata *udata); int mlx4_ib_modify_wq(struct ib_wq *wq, struct ib_wq_attr *wq_attr, u32 wq_attr_mask, struct ib_udata *udata); diff --git a/drivers/infiniband/hw/mlx4/mr.c b/drivers/infiniband/hw/mlx4/mr.c index 355205a28544..753479285ce9 100644 --- a/drivers/infiniband/hw/mlx4/mr.c +++ b/drivers/infiniband/hw/mlx4/mr.c @@ -258,7 +258,7 @@ int mlx4_ib_umem_calc_optimal_mtt_size(struct ib_umem *umem, u64 start_va, int *num_of_mtts) { u64 block_shift = MLX4_MAX_MTT_SHIFT; - u64 min_shift = umem->page_shift; + u64 min_shift = PAGE_SHIFT; u64 last_block_aligned_end = 0; u64 current_block_start = 0; u64 first_block_start = 0; @@ -295,8 +295,8 @@ int mlx4_ib_umem_calc_optimal_mtt_size(struct ib_umem *umem, u64 start_va, * in access to the wrong data. */ misalignment_bits = - (start_va & (~(((u64)(BIT(umem->page_shift))) - 1ULL))) - ^ current_block_start; + (start_va & (~(((u64)(PAGE_SIZE)) - 1ULL))) ^ + current_block_start; block_shift = min(alignment_of(misalignment_bits), block_shift); } @@ -368,8 +368,7 @@ end: } static struct ib_umem *mlx4_get_umem_mr(struct ib_udata *udata, u64 start, - u64 length, u64 virt_addr, - int access_flags) + u64 length, int access_flags) { /* * Force registering the memory as writable if the underlying pages @@ -415,8 +414,7 @@ struct ib_mr *mlx4_ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, if (!mr) return ERR_PTR(-ENOMEM); - mr->umem = - mlx4_get_umem_mr(udata, start, length, virt_addr, access_flags); + mr->umem = mlx4_get_umem_mr(udata, start, length, access_flags); if (IS_ERR(mr->umem)) { err = PTR_ERR(mr->umem); goto err_free; @@ -505,7 +503,7 @@ int mlx4_ib_rereg_user_mr(struct ib_mr *mr, int flags, mlx4_mr_rereg_mem_cleanup(dev->dev, &mmr->mmr); ib_umem_release(mmr->umem); - mmr->umem = mlx4_get_umem_mr(udata, start, length, virt_addr, + mmr->umem = mlx4_get_umem_mr(udata, start, length, mr_access_flags); if (IS_ERR(mmr->umem)) { err = PTR_ERR(mmr->umem); @@ -514,7 +512,7 @@ int mlx4_ib_rereg_user_mr(struct ib_mr *mr, int flags, goto release_mpt_entry; } n = ib_umem_page_count(mmr->umem); - shift = mmr->umem->page_shift; + shift = PAGE_SHIFT; err = mlx4_mr_rereg_mem_write(dev->dev, &mmr->mmr, virt_addr, length, n, shift, diff --git a/drivers/infiniband/hw/mlx4/qp.c b/drivers/infiniband/hw/mlx4/qp.c index 5221c0794d1d..82aff2f2fdc2 100644 --- a/drivers/infiniband/hw/mlx4/qp.c +++ b/drivers/infiniband/hw/mlx4/qp.c @@ -1207,10 +1207,9 @@ err_mtt: mlx4_mtt_cleanup(dev->dev, &qp->mtt); err_buf: - if (qp->umem) - ib_umem_release(qp->umem); - else + if (!qp->umem) mlx4_buf_free(dev->dev, qp->buf_size, &qp->buf); + ib_umem_release(qp->umem); err_db: if (!udata && qp_has_rq(init_attr)) @@ -1421,7 +1420,6 @@ static void destroy_qp_common(struct mlx4_ib_dev *dev, struct mlx4_ib_qp *qp, mlx4_ib_db_unmap_user(mcontext, &qp->db); } - ib_umem_release(qp->umem); } else { kvfree(qp->sq.wrid); kvfree(qp->rq.wrid); @@ -1432,6 +1430,7 @@ static void destroy_qp_common(struct mlx4_ib_dev *dev, struct mlx4_ib_qp *qp, if (qp->rq.wqe_cnt) mlx4_db_free(dev->dev, &qp->db); } + ib_umem_release(qp->umem); del_gid_entries(qp); } @@ -4248,7 +4247,7 @@ int mlx4_ib_modify_wq(struct ib_wq *ibwq, struct ib_wq_attr *wq_attr, return err; } -int mlx4_ib_destroy_wq(struct ib_wq *ibwq, struct ib_udata *udata) +void mlx4_ib_destroy_wq(struct ib_wq *ibwq, struct ib_udata *udata) { struct mlx4_ib_dev *dev = to_mdev(ibwq->device); struct mlx4_ib_qp *qp = to_mqp((struct ib_qp *)ibwq); @@ -4259,8 +4258,6 @@ int mlx4_ib_destroy_wq(struct ib_wq *ibwq, struct ib_udata *udata) destroy_qp_common(dev, qp, MLX4_IB_RWQ_SRC, udata); kfree(qp); - - return 0; } struct ib_rwq_ind_table diff --git a/drivers/infiniband/hw/mlx4/srq.c b/drivers/infiniband/hw/mlx4/srq.c index 4bf2946b9759..848db7264cc9 100644 --- a/drivers/infiniband/hw/mlx4/srq.c +++ b/drivers/infiniband/hw/mlx4/srq.c @@ -115,7 +115,7 @@ int mlx4_ib_create_srq(struct ib_srq *ib_srq, return PTR_ERR(srq->umem); err = mlx4_mtt_init(dev->dev, ib_umem_page_count(srq->umem), - srq->umem->page_shift, &srq->mtt); + PAGE_SHIFT, &srq->mtt); if (err) goto err_buf; @@ -204,10 +204,9 @@ err_mtt: mlx4_mtt_cleanup(dev->dev, &srq->mtt); err_buf: - if (srq->umem) - ib_umem_release(srq->umem); - else + if (!srq->umem) mlx4_buf_free(dev->dev, buf_size, &srq->buf); + ib_umem_release(srq->umem); err_db: if (!udata) @@ -275,13 +274,13 @@ void mlx4_ib_destroy_srq(struct ib_srq *srq, struct ib_udata *udata) struct mlx4_ib_ucontext, ibucontext), &msrq->db); - ib_umem_release(msrq->umem); } else { kvfree(msrq->wrid); mlx4_buf_free(dev->dev, msrq->msrq.max << msrq->msrq.wqe_shift, &msrq->buf); mlx4_db_free(dev->dev, &msrq->db); } + ib_umem_release(msrq->umem); } void mlx4_ib_free_srq_wqe(struct mlx4_ib_srq *srq, int wqe_index) diff --git a/drivers/infiniband/hw/mlx5/cq.c b/drivers/infiniband/hw/mlx5/cq.c index 4efbbd2fce0c..45f48cde6b9d 100644 --- a/drivers/infiniband/hw/mlx5/cq.c +++ b/drivers/infiniband/hw/mlx5/cq.c @@ -884,15 +884,15 @@ static void notify_soft_wc_handler(struct work_struct *work) cq->ibcq.comp_handler(&cq->ibcq, cq->ibcq.cq_context); } -struct ib_cq *mlx5_ib_create_cq(struct ib_device *ibdev, - const struct ib_cq_init_attr *attr, - struct ib_udata *udata) +int mlx5_ib_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, + struct ib_udata *udata) { + struct ib_device *ibdev = ibcq->device; int entries = attr->cqe; int vector = attr->comp_vector; struct mlx5_ib_dev *dev = to_mdev(ibdev); + struct mlx5_ib_cq *cq = to_mcq(ibcq); u32 out[MLX5_ST_SZ_DW(create_cq_out)]; - struct mlx5_ib_cq *cq; int uninitialized_var(index); int uninitialized_var(inlen); u32 *cqb = NULL; @@ -904,18 +904,14 @@ struct ib_cq *mlx5_ib_create_cq(struct ib_device *ibdev, if (entries < 0 || (entries > (1 << MLX5_CAP_GEN(dev->mdev, log_max_cq_sz)))) - return ERR_PTR(-EINVAL); + return -EINVAL; if (check_cq_create_flags(attr->flags)) - return ERR_PTR(-EOPNOTSUPP); + return -EOPNOTSUPP; entries = roundup_pow_of_two(entries + 1); if (entries > (1 << MLX5_CAP_GEN(dev->mdev, log_max_cq_sz))) - return ERR_PTR(-EINVAL); - - cq = kzalloc(sizeof(*cq), GFP_KERNEL); - if (!cq) - return ERR_PTR(-ENOMEM); + return -EINVAL; cq->ibcq.cqe = entries - 1; mutex_init(&cq->resize_mutex); @@ -930,13 +926,13 @@ struct ib_cq *mlx5_ib_create_cq(struct ib_device *ibdev, err = create_cq_user(dev, udata, cq, entries, &cqb, &cqe_size, &index, &inlen); if (err) - goto err_create; + return err; } else { cqe_size = cache_line_size() == 128 ? 128 : 64; err = create_cq_kernel(dev, cq, entries, cqe_size, &cqb, &index, &inlen); if (err) - goto err_create; + return err; INIT_WORK(&cq->notify_work, notify_soft_wc_handler); } @@ -981,7 +977,7 @@ struct ib_cq *mlx5_ib_create_cq(struct ib_device *ibdev, kvfree(cqb); - return &cq->ibcq; + return 0; err_cmd: mlx5_core_destroy_cq(dev->mdev, &cq->mcq); @@ -992,14 +988,10 @@ err_cqb: destroy_cq_user(cq, udata); else destroy_cq_kernel(dev, cq); - -err_create: - kfree(cq); - - return ERR_PTR(err); + return err; } -int mlx5_ib_destroy_cq(struct ib_cq *cq, struct ib_udata *udata) +void mlx5_ib_destroy_cq(struct ib_cq *cq, struct ib_udata *udata) { struct mlx5_ib_dev *dev = to_mdev(cq->device); struct mlx5_ib_cq *mcq = to_mcq(cq); @@ -1009,10 +1001,6 @@ int mlx5_ib_destroy_cq(struct ib_cq *cq, struct ib_udata *udata) destroy_cq_user(mcq, udata); else destroy_cq_kernel(dev, mcq); - - kfree(mcq); - - return 0; } static int is_equal_rsn(struct mlx5_cqe64 *cqe64, u32 rsn) @@ -1138,11 +1126,6 @@ static int resize_user(struct mlx5_ib_dev *dev, struct mlx5_ib_cq *cq, return 0; } -static void un_resize_user(struct mlx5_ib_cq *cq) -{ - ib_umem_release(cq->resize_umem); -} - static int resize_kernel(struct mlx5_ib_dev *dev, struct mlx5_ib_cq *cq, int entries, int cqe_size) { @@ -1165,12 +1148,6 @@ ex: return err; } -static void un_resize_kernel(struct mlx5_ib_dev *dev, struct mlx5_ib_cq *cq) -{ - free_cq_buf(dev, cq->resize_buf); - cq->resize_buf = NULL; -} - static int copy_resize_cqes(struct mlx5_ib_cq *cq) { struct mlx5_ib_dev *dev = to_mdev(cq->ibcq.device); @@ -1351,10 +1328,11 @@ ex_alloc: kvfree(in); ex_resize: - if (udata) - un_resize_user(cq); - else - un_resize_kernel(dev, cq); + ib_umem_release(cq->resize_umem); + if (!udata) { + free_cq_buf(dev, cq->resize_buf); + cq->resize_buf = NULL; + } ex: mutex_unlock(&cq->resize_mutex); return err; diff --git a/drivers/infiniband/hw/mlx5/devx.c b/drivers/infiniband/hw/mlx5/devx.c index 931f587dfb8f..ec4370f99381 100644 --- a/drivers/infiniband/hw/mlx5/devx.c +++ b/drivers/infiniband/hw/mlx5/devx.c @@ -14,13 +14,17 @@ #include <linux/mlx5/driver.h> #include <linux/mlx5/fs.h> #include "mlx5_ib.h" +#include <linux/xarray.h> #define UVERBS_MODULE_NAME mlx5_ib #include <rdma/uverbs_named_ioctl.h> +static void dispatch_event_fd(struct list_head *fd_list, const void *data); + enum devx_obj_flags { DEVX_OBJ_FLAGS_INDIRECT_MKEY = 1 << 0, DEVX_OBJ_FLAGS_DCT = 1 << 1, + DEVX_OBJ_FLAGS_CQ = 1 << 2, }; struct devx_async_data { @@ -33,9 +37,61 @@ struct devx_async_data { struct mlx5_ib_uapi_devx_async_cmd_hdr hdr; }; +struct devx_async_event_data { + struct list_head list; /* headed in ev_file->event_list */ + struct mlx5_ib_uapi_devx_async_event_hdr hdr; +}; + +/* first level XA value data structure */ +struct devx_event { + struct xarray object_ids; /* second XA level, Key = object id */ + struct list_head unaffiliated_list; +}; + +/* second level XA value data structure */ +struct devx_obj_event { + struct rcu_head rcu; + struct list_head obj_sub_list; +}; + +struct devx_event_subscription { + struct list_head file_list; /* headed in ev_file-> + * subscribed_events_list + */ + struct list_head xa_list; /* headed in devx_event->unaffiliated_list or + * devx_obj_event->obj_sub_list + */ + struct list_head obj_list; /* headed in devx_object */ + struct list_head event_list; /* headed in ev_file->event_list or in + * temp list via subscription + */ + + u8 is_cleaned:1; + u32 xa_key_level1; + u32 xa_key_level2; + struct rcu_head rcu; + u64 cookie; + struct devx_async_event_file *ev_file; + struct file *filp; /* Upon hot unplug we need a direct access to */ + struct eventfd_ctx *eventfd; +}; + +struct devx_async_event_file { + struct ib_uobject uobj; + /* Head of events that are subscribed to this FD */ + struct list_head subscribed_events_list; + spinlock_t lock; + wait_queue_head_t poll_wait; + struct list_head event_list; + struct mlx5_ib_dev *dev; + u8 omit_data:1; + u8 is_overflow_err:1; + u8 is_destroyed:1; +}; + #define MLX5_MAX_DESTROY_INBOX_SIZE_DW MLX5_ST_SZ_DW(delete_fte_in) struct devx_obj { - struct mlx5_core_dev *mdev; + struct mlx5_ib_dev *ib_dev; u64 obj_id; u32 dinlen; /* destroy inbox length */ u32 dinbox[MLX5_MAX_DESTROY_INBOX_SIZE_DW]; @@ -43,7 +99,9 @@ struct devx_obj { union { struct mlx5_ib_devx_mr devx_mr; struct mlx5_core_dct core_dct; + struct mlx5_core_cq core_cq; }; + struct list_head event_sub; /* holds devx_event_subscription entries */ }; struct devx_umem { @@ -149,6 +207,127 @@ bool mlx5_ib_devx_is_flow_counter(void *obj, u32 *counter_id) return false; } +static bool is_legacy_unaffiliated_event_num(u16 event_num) +{ + switch (event_num) { + case MLX5_EVENT_TYPE_PORT_CHANGE: + return true; + default: + return false; + } +} + +static bool is_legacy_obj_event_num(u16 event_num) +{ + switch (event_num) { + case MLX5_EVENT_TYPE_PATH_MIG: + case MLX5_EVENT_TYPE_COMM_EST: + case MLX5_EVENT_TYPE_SQ_DRAINED: + case MLX5_EVENT_TYPE_SRQ_LAST_WQE: + case MLX5_EVENT_TYPE_SRQ_RQ_LIMIT: + case MLX5_EVENT_TYPE_CQ_ERROR: + case MLX5_EVENT_TYPE_WQ_CATAS_ERROR: + case MLX5_EVENT_TYPE_PATH_MIG_FAILED: + case MLX5_EVENT_TYPE_WQ_INVAL_REQ_ERROR: + case MLX5_EVENT_TYPE_WQ_ACCESS_ERROR: + case MLX5_EVENT_TYPE_SRQ_CATAS_ERROR: + case MLX5_EVENT_TYPE_DCT_DRAINED: + case MLX5_EVENT_TYPE_COMP: + return true; + default: + return false; + } +} + +static u16 get_legacy_obj_type(u16 opcode) +{ + switch (opcode) { + case MLX5_CMD_OP_CREATE_RQ: + return MLX5_EVENT_QUEUE_TYPE_RQ; + case MLX5_CMD_OP_CREATE_QP: + return MLX5_EVENT_QUEUE_TYPE_QP; + case MLX5_CMD_OP_CREATE_SQ: + return MLX5_EVENT_QUEUE_TYPE_SQ; + case MLX5_CMD_OP_CREATE_DCT: + return MLX5_EVENT_QUEUE_TYPE_DCT; + default: + return 0; + } +} + +static u16 get_dec_obj_type(struct devx_obj *obj, u16 event_num) +{ + u16 opcode; + + opcode = (obj->obj_id >> 32) & 0xffff; + + if (is_legacy_obj_event_num(event_num)) + return get_legacy_obj_type(opcode); + + switch (opcode) { + case MLX5_CMD_OP_CREATE_GENERAL_OBJECT: + return (obj->obj_id >> 48); + case MLX5_CMD_OP_CREATE_RQ: + return MLX5_OBJ_TYPE_RQ; + case MLX5_CMD_OP_CREATE_QP: + return MLX5_OBJ_TYPE_QP; + case MLX5_CMD_OP_CREATE_SQ: + return MLX5_OBJ_TYPE_SQ; + case MLX5_CMD_OP_CREATE_DCT: + return MLX5_OBJ_TYPE_DCT; + case MLX5_CMD_OP_CREATE_TIR: + return MLX5_OBJ_TYPE_TIR; + case MLX5_CMD_OP_CREATE_TIS: + return MLX5_OBJ_TYPE_TIS; + case MLX5_CMD_OP_CREATE_PSV: + return MLX5_OBJ_TYPE_PSV; + case MLX5_OBJ_TYPE_MKEY: + return MLX5_OBJ_TYPE_MKEY; + case MLX5_CMD_OP_CREATE_RMP: + return MLX5_OBJ_TYPE_RMP; + case MLX5_CMD_OP_CREATE_XRC_SRQ: + return MLX5_OBJ_TYPE_XRC_SRQ; + case MLX5_CMD_OP_CREATE_XRQ: + return MLX5_OBJ_TYPE_XRQ; + case MLX5_CMD_OP_CREATE_RQT: + return MLX5_OBJ_TYPE_RQT; + case MLX5_CMD_OP_ALLOC_FLOW_COUNTER: + return MLX5_OBJ_TYPE_FLOW_COUNTER; + case MLX5_CMD_OP_CREATE_CQ: + return MLX5_OBJ_TYPE_CQ; + default: + return 0; + } +} + +static u16 get_event_obj_type(unsigned long event_type, struct mlx5_eqe *eqe) +{ + switch (event_type) { + case MLX5_EVENT_TYPE_WQ_CATAS_ERROR: + case MLX5_EVENT_TYPE_WQ_ACCESS_ERROR: + case MLX5_EVENT_TYPE_WQ_INVAL_REQ_ERROR: + case MLX5_EVENT_TYPE_SRQ_LAST_WQE: + case MLX5_EVENT_TYPE_PATH_MIG: + case MLX5_EVENT_TYPE_PATH_MIG_FAILED: + case MLX5_EVENT_TYPE_COMM_EST: + case MLX5_EVENT_TYPE_SQ_DRAINED: + case MLX5_EVENT_TYPE_SRQ_RQ_LIMIT: + case MLX5_EVENT_TYPE_SRQ_CATAS_ERROR: + return eqe->data.qp_srq.type; + case MLX5_EVENT_TYPE_CQ_ERROR: + return 0; + case MLX5_EVENT_TYPE_DCT_DRAINED: + return MLX5_EVENT_QUEUE_TYPE_DCT; + default: + return MLX5_GET(affiliated_event_header, &eqe->data, obj_type); + } +} + +static u32 get_dec_obj_id(u64 obj_id) +{ + return (obj_id & 0xffffffff); +} + /* * As the obj_id in the firmware is not globally unique the object type * must be considered upon checking for a valid object id. @@ -715,12 +894,16 @@ static int devx_get_uid(struct mlx5_ib_ucontext *c, void *cmd_in) return c->devx_uid; } -static bool devx_is_general_cmd(void *in) + +static bool devx_is_general_cmd(void *in, struct mlx5_ib_dev *dev) { u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, in, opcode); - if (opcode >= MLX5_CMD_OP_GENERAL_START && - opcode < MLX5_CMD_OP_GENERAL_END) + /* Pass all cmds for vhca_tunnel as general, tracking is done in FW */ + if ((MLX5_CAP_GEN_64(dev->mdev, vhca_tunnel_commands) && + MLX5_GET(general_obj_in_cmd_hdr, in, vhca_tunnel_id)) || + (opcode >= MLX5_CMD_OP_GENERAL_START && + opcode < MLX5_CMD_OP_GENERAL_END)) return true; switch (opcode) { @@ -846,7 +1029,7 @@ static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OTHER)( return uid; /* Only white list of some general HCA commands are allowed for this method. */ - if (!devx_is_general_cmd(cmd_in)) + if (!devx_is_general_cmd(cmd_in, dev)) return -EINVAL; cmd_out = uverbs_zalloc(attrs, cmd_out_len); @@ -1111,33 +1294,72 @@ static void devx_free_indirect_mkey(struct rcu_head *rcu) */ static void devx_cleanup_mkey(struct devx_obj *obj) { - xa_erase(&obj->mdev->priv.mkey_table, + xa_erase(&obj->ib_dev->mdev->priv.mkey_table, mlx5_base_mkey(obj->devx_mr.mmkey.key)); } +static void devx_cleanup_subscription(struct mlx5_ib_dev *dev, + struct devx_event_subscription *sub) +{ + struct devx_event *event; + struct devx_obj_event *xa_val_level2; + + if (sub->is_cleaned) + return; + + sub->is_cleaned = 1; + list_del_rcu(&sub->xa_list); + + if (list_empty(&sub->obj_list)) + return; + + list_del_rcu(&sub->obj_list); + /* check whether key level 1 for this obj_sub_list is empty */ + event = xa_load(&dev->devx_event_table.event_xa, + sub->xa_key_level1); + WARN_ON(!event); + + xa_val_level2 = xa_load(&event->object_ids, sub->xa_key_level2); + if (list_empty(&xa_val_level2->obj_sub_list)) { + xa_erase(&event->object_ids, + sub->xa_key_level2); + kfree_rcu(xa_val_level2, rcu); + } +} + static int devx_obj_cleanup(struct ib_uobject *uobject, enum rdma_remove_reason why, struct uverbs_attr_bundle *attrs) { u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)]; + struct mlx5_devx_event_table *devx_event_table; struct devx_obj *obj = uobject->object; + struct devx_event_subscription *sub_entry, *tmp; + struct mlx5_ib_dev *dev; int ret; + dev = mlx5_udata_to_mdev(&attrs->driver_udata); if (obj->flags & DEVX_OBJ_FLAGS_INDIRECT_MKEY) devx_cleanup_mkey(obj); if (obj->flags & DEVX_OBJ_FLAGS_DCT) - ret = mlx5_core_destroy_dct(obj->mdev, &obj->core_dct); + ret = mlx5_core_destroy_dct(obj->ib_dev->mdev, &obj->core_dct); + else if (obj->flags & DEVX_OBJ_FLAGS_CQ) + ret = mlx5_core_destroy_cq(obj->ib_dev->mdev, &obj->core_cq); else - ret = mlx5_cmd_exec(obj->mdev, obj->dinbox, obj->dinlen, out, - sizeof(out)); + ret = mlx5_cmd_exec(obj->ib_dev->mdev, obj->dinbox, + obj->dinlen, out, sizeof(out)); if (ib_is_destroy_retryable(ret, why, uobject)) return ret; - if (obj->flags & DEVX_OBJ_FLAGS_INDIRECT_MKEY) { - struct mlx5_ib_dev *dev = - mlx5_udata_to_mdev(&attrs->driver_udata); + devx_event_table = &dev->devx_event_table; + + mutex_lock(&devx_event_table->event_xa_lock); + list_for_each_entry_safe(sub_entry, tmp, &obj->event_sub, obj_list) + devx_cleanup_subscription(dev, sub_entry); + mutex_unlock(&devx_event_table->event_xa_lock); + if (obj->flags & DEVX_OBJ_FLAGS_INDIRECT_MKEY) { call_srcu(&dev->mr_srcu, &obj->devx_mr.rcu, devx_free_indirect_mkey); return ret; @@ -1147,6 +1369,29 @@ static int devx_obj_cleanup(struct ib_uobject *uobject, return ret; } +static void devx_cq_comp(struct mlx5_core_cq *mcq, struct mlx5_eqe *eqe) +{ + struct devx_obj *obj = container_of(mcq, struct devx_obj, core_cq); + struct mlx5_devx_event_table *table; + struct devx_event *event; + struct devx_obj_event *obj_event; + u32 obj_id = mcq->cqn; + + table = &obj->ib_dev->devx_event_table; + rcu_read_lock(); + event = xa_load(&table->event_xa, MLX5_EVENT_TYPE_COMP); + if (!event) + goto out; + + obj_event = xa_load(&event->object_ids, obj_id); + if (!obj_event) + goto out; + + dispatch_event_fd(&obj_event->obj_sub_list, eqe); +out: + rcu_read_unlock(); +} + static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OBJ_CREATE)( struct uverbs_attr_bundle *attrs) { @@ -1169,6 +1414,9 @@ static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OBJ_CREATE)( u32 obj_id; u16 opcode; + if (MLX5_GET(general_obj_in_cmd_hdr, cmd_in, vhca_tunnel_id)) + return -EINVAL; + uid = devx_get_uid(c, cmd_in); if (uid < 0) return uid; @@ -1198,6 +1446,12 @@ static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OBJ_CREATE)( err = mlx5_core_create_dct(dev->mdev, &obj->core_dct, cmd_in, cmd_in_len, cmd_out, cmd_out_len); + } else if (opcode == MLX5_CMD_OP_CREATE_CQ) { + obj->flags |= DEVX_OBJ_FLAGS_CQ; + obj->core_cq.comp = devx_cq_comp; + err = mlx5_core_create_cq(dev->mdev, &obj->core_cq, + cmd_in, cmd_in_len, cmd_out, + cmd_out_len); } else { err = mlx5_cmd_exec(dev->mdev, cmd_in, cmd_in_len, @@ -1208,7 +1462,8 @@ static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OBJ_CREATE)( goto obj_free; uobj->object = obj; - obj->mdev = dev->mdev; + INIT_LIST_HEAD(&obj->event_sub); + obj->ib_dev = dev; devx_obj_build_destroy_cmd(cmd_in, cmd_out, obj->dinbox, &obj->dinlen, &obj_id); WARN_ON(obj->dinlen > MLX5_MAX_DESTROY_INBOX_SIZE_DW * sizeof(u32)); @@ -1235,9 +1490,11 @@ err_copy: devx_cleanup_mkey(obj); obj_destroy: if (obj->flags & DEVX_OBJ_FLAGS_DCT) - mlx5_core_destroy_dct(obj->mdev, &obj->core_dct); + mlx5_core_destroy_dct(obj->ib_dev->mdev, &obj->core_dct); + else if (obj->flags & DEVX_OBJ_FLAGS_CQ) + mlx5_core_destroy_cq(obj->ib_dev->mdev, &obj->core_cq); else - mlx5_cmd_exec(obj->mdev, obj->dinbox, obj->dinlen, out, + mlx5_cmd_exec(obj->ib_dev->mdev, obj->dinbox, obj->dinlen, out, sizeof(out)); obj_free: kfree(obj); @@ -1259,6 +1516,9 @@ static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OBJ_MODIFY)( int err; int uid; + if (MLX5_GET(general_obj_in_cmd_hdr, cmd_in, vhca_tunnel_id)) + return -EINVAL; + uid = devx_get_uid(c, cmd_in); if (uid < 0) return uid; @@ -1301,6 +1561,9 @@ static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OBJ_QUERY)( int uid; struct mlx5_ib_dev *mdev = to_mdev(c->ibucontext.device); + if (MLX5_GET(general_obj_in_cmd_hdr, cmd_in, vhca_tunnel_id)) + return -EINVAL; + uid = devx_get_uid(c, cmd_in); if (uid < 0) return uid; @@ -1365,6 +1628,38 @@ static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_ASYNC_CMD_FD_ALLOC)( return 0; } +static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_ASYNC_EVENT_FD_ALLOC)( + struct uverbs_attr_bundle *attrs) +{ + struct ib_uobject *uobj = uverbs_attr_get_uobject( + attrs, MLX5_IB_ATTR_DEVX_ASYNC_EVENT_FD_ALLOC_HANDLE); + struct devx_async_event_file *ev_file; + struct mlx5_ib_ucontext *c = rdma_udata_to_drv_context( + &attrs->driver_udata, struct mlx5_ib_ucontext, ibucontext); + struct mlx5_ib_dev *dev = to_mdev(c->ibucontext.device); + u32 flags; + int err; + + err = uverbs_get_flags32(&flags, attrs, + MLX5_IB_ATTR_DEVX_ASYNC_EVENT_FD_ALLOC_FLAGS, + MLX5_IB_UAPI_DEVX_CR_EV_CH_FLAGS_OMIT_DATA); + + if (err) + return err; + + ev_file = container_of(uobj, struct devx_async_event_file, + uobj); + spin_lock_init(&ev_file->lock); + INIT_LIST_HEAD(&ev_file->event_list); + init_waitqueue_head(&ev_file->poll_wait); + if (flags & MLX5_IB_UAPI_DEVX_CR_EV_CH_FLAGS_OMIT_DATA) + ev_file->omit_data = 1; + INIT_LIST_HEAD(&ev_file->subscribed_events_list); + ev_file->dev = dev; + get_device(&dev->ib_dev.dev); + return 0; +} + static void devx_query_callback(int status, struct mlx5_async_work *context) { struct devx_async_data *async_data = @@ -1406,6 +1701,9 @@ static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OBJ_ASYNC_QUERY)( struct devx_async_cmd_event_file *ev_file; struct devx_async_data *async_data; + if (MLX5_GET(general_obj_in_cmd_hdr, cmd_in, vhca_tunnel_id)) + return -EINVAL; + uid = devx_get_uid(c, cmd_in); if (uid < 0) return uid; @@ -1474,6 +1772,331 @@ sub_bytes: return err; } +static void +subscribe_event_xa_dealloc(struct mlx5_devx_event_table *devx_event_table, + u32 key_level1, + bool is_level2, + u32 key_level2) +{ + struct devx_event *event; + struct devx_obj_event *xa_val_level2; + + /* Level 1 is valid for future use, no need to free */ + if (!is_level2) + return; + + event = xa_load(&devx_event_table->event_xa, key_level1); + WARN_ON(!event); + + xa_val_level2 = xa_load(&event->object_ids, + key_level2); + if (list_empty(&xa_val_level2->obj_sub_list)) { + xa_erase(&event->object_ids, + key_level2); + kfree_rcu(xa_val_level2, rcu); + } +} + +static int +subscribe_event_xa_alloc(struct mlx5_devx_event_table *devx_event_table, + u32 key_level1, + bool is_level2, + u32 key_level2) +{ + struct devx_obj_event *obj_event; + struct devx_event *event; + int err; + + event = xa_load(&devx_event_table->event_xa, key_level1); + if (!event) { + event = kzalloc(sizeof(*event), GFP_KERNEL); + if (!event) + return -ENOMEM; + + INIT_LIST_HEAD(&event->unaffiliated_list); + xa_init(&event->object_ids); + + err = xa_insert(&devx_event_table->event_xa, + key_level1, + event, + GFP_KERNEL); + if (err) { + kfree(event); + return err; + } + } + + if (!is_level2) + return 0; + + obj_event = xa_load(&event->object_ids, key_level2); + if (!obj_event) { + obj_event = kzalloc(sizeof(*obj_event), GFP_KERNEL); + if (!obj_event) + /* Level1 is valid for future use, no need to free */ + return -ENOMEM; + + err = xa_insert(&event->object_ids, + key_level2, + obj_event, + GFP_KERNEL); + if (err) + return err; + INIT_LIST_HEAD(&obj_event->obj_sub_list); + } + + return 0; +} + +static bool is_valid_events_legacy(int num_events, u16 *event_type_num_list, + struct devx_obj *obj) +{ + int i; + + for (i = 0; i < num_events; i++) { + if (obj) { + if (!is_legacy_obj_event_num(event_type_num_list[i])) + return false; + } else if (!is_legacy_unaffiliated_event_num( + event_type_num_list[i])) { + return false; + } + } + + return true; +} + +#define MAX_SUPP_EVENT_NUM 255 +static bool is_valid_events(struct mlx5_core_dev *dev, + int num_events, u16 *event_type_num_list, + struct devx_obj *obj) +{ + __be64 *aff_events; + __be64 *unaff_events; + int mask_entry; + int mask_bit; + int i; + + if (MLX5_CAP_GEN(dev, event_cap)) { + aff_events = MLX5_CAP_DEV_EVENT(dev, + user_affiliated_events); + unaff_events = MLX5_CAP_DEV_EVENT(dev, + user_unaffiliated_events); + } else { + return is_valid_events_legacy(num_events, event_type_num_list, + obj); + } + + for (i = 0; i < num_events; i++) { + if (event_type_num_list[i] > MAX_SUPP_EVENT_NUM) + return false; + + mask_entry = event_type_num_list[i] / 64; + mask_bit = event_type_num_list[i] % 64; + + if (obj) { + /* CQ completion */ + if (event_type_num_list[i] == 0) + continue; + + if (!(be64_to_cpu(aff_events[mask_entry]) & + (1ull << mask_bit))) + return false; + + continue; + } + + if (!(be64_to_cpu(unaff_events[mask_entry]) & + (1ull << mask_bit))) + return false; + } + + return true; +} + +#define MAX_NUM_EVENTS 16 +static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_SUBSCRIBE_EVENT)( + struct uverbs_attr_bundle *attrs) +{ + struct ib_uobject *devx_uobj = uverbs_attr_get_uobject( + attrs, + MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_OBJ_HANDLE); + struct mlx5_ib_ucontext *c = rdma_udata_to_drv_context( + &attrs->driver_udata, struct mlx5_ib_ucontext, ibucontext); + struct mlx5_ib_dev *dev = to_mdev(c->ibucontext.device); + struct ib_uobject *fd_uobj; + struct devx_obj *obj = NULL; + struct devx_async_event_file *ev_file; + struct mlx5_devx_event_table *devx_event_table = &dev->devx_event_table; + u16 *event_type_num_list; + struct devx_event_subscription *event_sub, *tmp_sub; + struct list_head sub_list; + int redirect_fd; + bool use_eventfd = false; + int num_events; + int num_alloc_xa_entries = 0; + u16 obj_type = 0; + u64 cookie = 0; + u32 obj_id = 0; + int err; + int i; + + if (!c->devx_uid) + return -EINVAL; + + if (!IS_ERR(devx_uobj)) { + obj = (struct devx_obj *)devx_uobj->object; + if (obj) + obj_id = get_dec_obj_id(obj->obj_id); + } + + fd_uobj = uverbs_attr_get_uobject(attrs, + MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_FD_HANDLE); + if (IS_ERR(fd_uobj)) + return PTR_ERR(fd_uobj); + + ev_file = container_of(fd_uobj, struct devx_async_event_file, + uobj); + + if (uverbs_attr_is_valid(attrs, + MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_FD_NUM)) { + err = uverbs_copy_from(&redirect_fd, attrs, + MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_FD_NUM); + if (err) + return err; + + use_eventfd = true; + } + + if (uverbs_attr_is_valid(attrs, + MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_COOKIE)) { + if (use_eventfd) + return -EINVAL; + + err = uverbs_copy_from(&cookie, attrs, + MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_COOKIE); + if (err) + return err; + } + + num_events = uverbs_attr_ptr_get_array_size( + attrs, MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_TYPE_NUM_LIST, + sizeof(u16)); + + if (num_events < 0) + return num_events; + + if (num_events > MAX_NUM_EVENTS) + return -EINVAL; + + event_type_num_list = uverbs_attr_get_alloced_ptr(attrs, + MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_TYPE_NUM_LIST); + + if (!is_valid_events(dev->mdev, num_events, event_type_num_list, obj)) + return -EINVAL; + + INIT_LIST_HEAD(&sub_list); + + /* Protect from concurrent subscriptions to same XA entries to allow + * both to succeed + */ + mutex_lock(&devx_event_table->event_xa_lock); + for (i = 0; i < num_events; i++) { + u32 key_level1; + + if (obj) + obj_type = get_dec_obj_type(obj, + event_type_num_list[i]); + key_level1 = event_type_num_list[i] | obj_type << 16; + + err = subscribe_event_xa_alloc(devx_event_table, + key_level1, + obj, + obj_id); + if (err) + goto err; + + num_alloc_xa_entries++; + event_sub = kzalloc(sizeof(*event_sub), GFP_KERNEL); + if (!event_sub) + goto err; + + list_add_tail(&event_sub->event_list, &sub_list); + if (use_eventfd) { + event_sub->eventfd = + eventfd_ctx_fdget(redirect_fd); + + if (IS_ERR(event_sub)) { + err = PTR_ERR(event_sub->eventfd); + event_sub->eventfd = NULL; + goto err; + } + } + + event_sub->cookie = cookie; + event_sub->ev_file = ev_file; + event_sub->filp = fd_uobj->object; + /* May be needed upon cleanup the devx object/subscription */ + event_sub->xa_key_level1 = key_level1; + event_sub->xa_key_level2 = obj_id; + INIT_LIST_HEAD(&event_sub->obj_list); + } + + /* Once all the allocations and the XA data insertions were done we + * can go ahead and add all the subscriptions to the relevant lists + * without concern of a failure. + */ + list_for_each_entry_safe(event_sub, tmp_sub, &sub_list, event_list) { + struct devx_event *event; + struct devx_obj_event *obj_event; + + list_del_init(&event_sub->event_list); + + spin_lock_irq(&ev_file->lock); + list_add_tail_rcu(&event_sub->file_list, + &ev_file->subscribed_events_list); + spin_unlock_irq(&ev_file->lock); + + event = xa_load(&devx_event_table->event_xa, + event_sub->xa_key_level1); + WARN_ON(!event); + + if (!obj) { + list_add_tail_rcu(&event_sub->xa_list, + &event->unaffiliated_list); + continue; + } + + obj_event = xa_load(&event->object_ids, obj_id); + WARN_ON(!obj_event); + list_add_tail_rcu(&event_sub->xa_list, + &obj_event->obj_sub_list); + list_add_tail_rcu(&event_sub->obj_list, + &obj->event_sub); + } + + mutex_unlock(&devx_event_table->event_xa_lock); + return 0; + +err: + list_for_each_entry_safe(event_sub, tmp_sub, &sub_list, event_list) { + list_del(&event_sub->event_list); + + subscribe_event_xa_dealloc(devx_event_table, + event_sub->xa_key_level1, + obj, + obj_id); + + if (event_sub->eventfd) + eventfd_ctx_put(event_sub->eventfd); + + kfree(event_sub); + } + + mutex_unlock(&devx_event_table->event_xa_lock); + return err; +} + static int devx_umem_get(struct mlx5_ib_dev *dev, struct ib_ucontext *ucontext, struct uverbs_attr_bundle *attrs, struct devx_umem *obj) @@ -1621,6 +2244,203 @@ static int devx_umem_cleanup(struct ib_uobject *uobject, return 0; } +static bool is_unaffiliated_event(struct mlx5_core_dev *dev, + unsigned long event_type) +{ + __be64 *unaff_events; + int mask_entry; + int mask_bit; + + if (!MLX5_CAP_GEN(dev, event_cap)) + return is_legacy_unaffiliated_event_num(event_type); + + unaff_events = MLX5_CAP_DEV_EVENT(dev, + user_unaffiliated_events); + WARN_ON(event_type > MAX_SUPP_EVENT_NUM); + + mask_entry = event_type / 64; + mask_bit = event_type % 64; + + if (!(be64_to_cpu(unaff_events[mask_entry]) & (1ull << mask_bit))) + return false; + + return true; +} + +static u32 devx_get_obj_id_from_event(unsigned long event_type, void *data) +{ + struct mlx5_eqe *eqe = data; + u32 obj_id = 0; + + switch (event_type) { + case MLX5_EVENT_TYPE_SRQ_CATAS_ERROR: + case MLX5_EVENT_TYPE_SRQ_RQ_LIMIT: + case MLX5_EVENT_TYPE_PATH_MIG: + case MLX5_EVENT_TYPE_COMM_EST: + case MLX5_EVENT_TYPE_SQ_DRAINED: + case MLX5_EVENT_TYPE_SRQ_LAST_WQE: + case MLX5_EVENT_TYPE_WQ_CATAS_ERROR: + case MLX5_EVENT_TYPE_PATH_MIG_FAILED: + case MLX5_EVENT_TYPE_WQ_INVAL_REQ_ERROR: + case MLX5_EVENT_TYPE_WQ_ACCESS_ERROR: + obj_id = be32_to_cpu(eqe->data.qp_srq.qp_srq_n) & 0xffffff; + break; + case MLX5_EVENT_TYPE_DCT_DRAINED: + obj_id = be32_to_cpu(eqe->data.dct.dctn) & 0xffffff; + break; + case MLX5_EVENT_TYPE_CQ_ERROR: + obj_id = be32_to_cpu(eqe->data.cq_err.cqn) & 0xffffff; + break; + default: + obj_id = MLX5_GET(affiliated_event_header, &eqe->data, obj_id); + break; + } + + return obj_id; +} + +static int deliver_event(struct devx_event_subscription *event_sub, + const void *data) +{ + struct devx_async_event_file *ev_file; + struct devx_async_event_data *event_data; + unsigned long flags; + + ev_file = event_sub->ev_file; + + if (ev_file->omit_data) { + spin_lock_irqsave(&ev_file->lock, flags); + if (!list_empty(&event_sub->event_list)) { + spin_unlock_irqrestore(&ev_file->lock, flags); + return 0; + } + + list_add_tail(&event_sub->event_list, &ev_file->event_list); + spin_unlock_irqrestore(&ev_file->lock, flags); + wake_up_interruptible(&ev_file->poll_wait); + return 0; + } + + event_data = kzalloc(sizeof(*event_data) + sizeof(struct mlx5_eqe), + GFP_ATOMIC); + if (!event_data) { + spin_lock_irqsave(&ev_file->lock, flags); + ev_file->is_overflow_err = 1; + spin_unlock_irqrestore(&ev_file->lock, flags); + return -ENOMEM; + } + + event_data->hdr.cookie = event_sub->cookie; + memcpy(event_data->hdr.out_data, data, sizeof(struct mlx5_eqe)); + + spin_lock_irqsave(&ev_file->lock, flags); + list_add_tail(&event_data->list, &ev_file->event_list); + spin_unlock_irqrestore(&ev_file->lock, flags); + wake_up_interruptible(&ev_file->poll_wait); + + return 0; +} + +static void dispatch_event_fd(struct list_head *fd_list, + const void *data) +{ + struct devx_event_subscription *item; + + list_for_each_entry_rcu(item, fd_list, xa_list) { + if (!get_file_rcu(item->filp)) + continue; + + if (item->eventfd) { + eventfd_signal(item->eventfd, 1); + fput(item->filp); + continue; + } + + deliver_event(item, data); + fput(item->filp); + } +} + +static int devx_event_notifier(struct notifier_block *nb, + unsigned long event_type, void *data) +{ + struct mlx5_devx_event_table *table; + struct mlx5_ib_dev *dev; + struct devx_event *event; + struct devx_obj_event *obj_event; + u16 obj_type = 0; + bool is_unaffiliated; + u32 obj_id; + + /* Explicit filtering to kernel events which may occur frequently */ + if (event_type == MLX5_EVENT_TYPE_CMD || + event_type == MLX5_EVENT_TYPE_PAGE_REQUEST) + return NOTIFY_OK; + + table = container_of(nb, struct mlx5_devx_event_table, devx_nb.nb); + dev = container_of(table, struct mlx5_ib_dev, devx_event_table); + is_unaffiliated = is_unaffiliated_event(dev->mdev, event_type); + + if (!is_unaffiliated) + obj_type = get_event_obj_type(event_type, data); + + rcu_read_lock(); + event = xa_load(&table->event_xa, event_type | (obj_type << 16)); + if (!event) { + rcu_read_unlock(); + return NOTIFY_DONE; + } + + if (is_unaffiliated) { + dispatch_event_fd(&event->unaffiliated_list, data); + rcu_read_unlock(); + return NOTIFY_OK; + } + + obj_id = devx_get_obj_id_from_event(event_type, data); + obj_event = xa_load(&event->object_ids, obj_id); + if (!obj_event) { + rcu_read_unlock(); + return NOTIFY_DONE; + } + + dispatch_event_fd(&obj_event->obj_sub_list, data); + + rcu_read_unlock(); + return NOTIFY_OK; +} + +void mlx5_ib_devx_init_event_table(struct mlx5_ib_dev *dev) +{ + struct mlx5_devx_event_table *table = &dev->devx_event_table; + + xa_init(&table->event_xa); + mutex_init(&table->event_xa_lock); + MLX5_NB_INIT(&table->devx_nb, devx_event_notifier, NOTIFY_ANY); + mlx5_eq_notifier_register(dev->mdev, &table->devx_nb); +} + +void mlx5_ib_devx_cleanup_event_table(struct mlx5_ib_dev *dev) +{ + struct mlx5_devx_event_table *table = &dev->devx_event_table; + struct devx_event_subscription *sub, *tmp; + struct devx_event *event; + void *entry; + unsigned long id; + + mlx5_eq_notifier_unregister(dev->mdev, &table->devx_nb); + mutex_lock(&dev->devx_event_table.event_xa_lock); + xa_for_each(&table->event_xa, id, entry) { + event = entry; + list_for_each_entry_safe(sub, tmp, &event->unaffiliated_list, + xa_list) + devx_cleanup_subscription(dev, sub); + kfree(entry); + } + mutex_unlock(&dev->devx_event_table.event_xa_lock); + xa_destroy(&table->event_xa); +} + static ssize_t devx_async_cmd_event_read(struct file *filp, char __user *buf, size_t count, loff_t *pos) { @@ -1719,6 +2539,149 @@ static const struct file_operations devx_async_cmd_event_fops = { .llseek = no_llseek, }; +static ssize_t devx_async_event_read(struct file *filp, char __user *buf, + size_t count, loff_t *pos) +{ + struct devx_async_event_file *ev_file = filp->private_data; + struct devx_event_subscription *event_sub; + struct devx_async_event_data *uninitialized_var(event); + int ret = 0; + size_t eventsz; + bool omit_data; + void *event_data; + + omit_data = ev_file->omit_data; + + spin_lock_irq(&ev_file->lock); + + if (ev_file->is_overflow_err) { + ev_file->is_overflow_err = 0; + spin_unlock_irq(&ev_file->lock); + return -EOVERFLOW; + } + + if (ev_file->is_destroyed) { + spin_unlock_irq(&ev_file->lock); + return -EIO; + } + + while (list_empty(&ev_file->event_list)) { + spin_unlock_irq(&ev_file->lock); + + if (filp->f_flags & O_NONBLOCK) + return -EAGAIN; + + if (wait_event_interruptible(ev_file->poll_wait, + (!list_empty(&ev_file->event_list) || + ev_file->is_destroyed))) { + return -ERESTARTSYS; + } + + spin_lock_irq(&ev_file->lock); + if (ev_file->is_destroyed) { + spin_unlock_irq(&ev_file->lock); + return -EIO; + } + } + + if (omit_data) { + event_sub = list_first_entry(&ev_file->event_list, + struct devx_event_subscription, + event_list); + eventsz = sizeof(event_sub->cookie); + event_data = &event_sub->cookie; + } else { + event = list_first_entry(&ev_file->event_list, + struct devx_async_event_data, list); + eventsz = sizeof(struct mlx5_eqe) + + sizeof(struct mlx5_ib_uapi_devx_async_event_hdr); + event_data = &event->hdr; + } + + if (eventsz > count) { + spin_unlock_irq(&ev_file->lock); + return -EINVAL; + } + + if (omit_data) + list_del_init(&event_sub->event_list); + else + list_del(&event->list); + + spin_unlock_irq(&ev_file->lock); + + if (copy_to_user(buf, event_data, eventsz)) + /* This points to an application issue, not a kernel concern */ + ret = -EFAULT; + else + ret = eventsz; + + if (!omit_data) + kfree(event); + return ret; +} + +static __poll_t devx_async_event_poll(struct file *filp, + struct poll_table_struct *wait) +{ + struct devx_async_event_file *ev_file = filp->private_data; + __poll_t pollflags = 0; + + poll_wait(filp, &ev_file->poll_wait, wait); + + spin_lock_irq(&ev_file->lock); + if (ev_file->is_destroyed) + pollflags = EPOLLIN | EPOLLRDNORM | EPOLLRDHUP; + else if (!list_empty(&ev_file->event_list)) + pollflags = EPOLLIN | EPOLLRDNORM; + spin_unlock_irq(&ev_file->lock); + + return pollflags; +} + +static int devx_async_event_close(struct inode *inode, struct file *filp) +{ + struct devx_async_event_file *ev_file = filp->private_data; + struct devx_event_subscription *event_sub, *event_sub_tmp; + struct devx_async_event_data *entry, *tmp; + + mutex_lock(&ev_file->dev->devx_event_table.event_xa_lock); + /* delete the subscriptions which are related to this FD */ + list_for_each_entry_safe(event_sub, event_sub_tmp, + &ev_file->subscribed_events_list, file_list) { + devx_cleanup_subscription(ev_file->dev, event_sub); + if (event_sub->eventfd) + eventfd_ctx_put(event_sub->eventfd); + + list_del_rcu(&event_sub->file_list); + /* subscription may not be used by the read API any more */ + kfree_rcu(event_sub, rcu); + } + + mutex_unlock(&ev_file->dev->devx_event_table.event_xa_lock); + + /* free the pending events allocation */ + if (!ev_file->omit_data) { + spin_lock_irq(&ev_file->lock); + list_for_each_entry_safe(entry, tmp, + &ev_file->event_list, list) + kfree(entry); /* read can't come any more */ + spin_unlock_irq(&ev_file->lock); + } + + uverbs_close_fd(filp); + put_device(&ev_file->dev->ib_dev.dev); + return 0; +} + +static const struct file_operations devx_async_event_fops = { + .owner = THIS_MODULE, + .read = devx_async_event_read, + .poll = devx_async_event_poll, + .release = devx_async_event_close, + .llseek = no_llseek, +}; + static int devx_hot_unplug_async_cmd_event_file(struct ib_uobject *uobj, enum rdma_remove_reason why) { @@ -1738,6 +2701,21 @@ static int devx_hot_unplug_async_cmd_event_file(struct ib_uobject *uobj, return 0; }; +static int devx_hot_unplug_async_event_file(struct ib_uobject *uobj, + enum rdma_remove_reason why) +{ + struct devx_async_event_file *ev_file = + container_of(uobj, struct devx_async_event_file, + uobj); + + spin_lock_irq(&ev_file->lock); + ev_file->is_destroyed = 1; + spin_unlock_irq(&ev_file->lock); + + wake_up_interruptible(&ev_file->poll_wait); + return 0; +}; + DECLARE_UVERBS_NAMED_METHOD( MLX5_IB_METHOD_DEVX_UMEM_REG, UVERBS_ATTR_IDR(MLX5_IB_ATTR_DEVX_UMEM_REG_HANDLE, @@ -1869,10 +2847,32 @@ DECLARE_UVERBS_NAMED_METHOD( UVERBS_ATTR_TYPE(u64), UA_MANDATORY)); +DECLARE_UVERBS_NAMED_METHOD( + MLX5_IB_METHOD_DEVX_SUBSCRIBE_EVENT, + UVERBS_ATTR_FD(MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_FD_HANDLE, + MLX5_IB_OBJECT_DEVX_ASYNC_EVENT_FD, + UVERBS_ACCESS_READ, + UA_MANDATORY), + UVERBS_ATTR_IDR(MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_OBJ_HANDLE, + MLX5_IB_OBJECT_DEVX_OBJ, + UVERBS_ACCESS_READ, + UA_OPTIONAL), + UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_TYPE_NUM_LIST, + UVERBS_ATTR_MIN_SIZE(sizeof(u16)), + UA_MANDATORY, + UA_ALLOC_AND_COPY), + UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_COOKIE, + UVERBS_ATTR_TYPE(u64), + UA_OPTIONAL), + UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_DEVX_SUBSCRIBE_EVENT_FD_NUM, + UVERBS_ATTR_TYPE(u32), + UA_OPTIONAL)); + DECLARE_UVERBS_GLOBAL_METHODS(MLX5_IB_OBJECT_DEVX, &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_OTHER), &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_QUERY_UAR), - &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_QUERY_EQN)); + &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_QUERY_EQN), + &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_SUBSCRIBE_EVENT)); DECLARE_UVERBS_NAMED_OBJECT(MLX5_IB_OBJECT_DEVX_OBJ, UVERBS_TYPE_ALLOC_IDR(devx_obj_cleanup), @@ -1903,6 +2903,24 @@ DECLARE_UVERBS_NAMED_OBJECT( O_RDONLY), &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_ASYNC_CMD_FD_ALLOC)); +DECLARE_UVERBS_NAMED_METHOD( + MLX5_IB_METHOD_DEVX_ASYNC_EVENT_FD_ALLOC, + UVERBS_ATTR_FD(MLX5_IB_ATTR_DEVX_ASYNC_EVENT_FD_ALLOC_HANDLE, + MLX5_IB_OBJECT_DEVX_ASYNC_EVENT_FD, + UVERBS_ACCESS_NEW, + UA_MANDATORY), + UVERBS_ATTR_FLAGS_IN(MLX5_IB_ATTR_DEVX_ASYNC_EVENT_FD_ALLOC_FLAGS, + enum mlx5_ib_uapi_devx_create_event_channel_flags, + UA_MANDATORY)); + +DECLARE_UVERBS_NAMED_OBJECT( + MLX5_IB_OBJECT_DEVX_ASYNC_EVENT_FD, + UVERBS_TYPE_ALLOC_FD(sizeof(struct devx_async_event_file), + devx_hot_unplug_async_event_file, + &devx_async_event_fops, "[devx_async_event]", + O_RDONLY), + &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_ASYNC_EVENT_FD_ALLOC)); + static bool devx_is_supported(struct ib_device *device) { struct mlx5_ib_dev *dev = to_mdev(device); @@ -1923,5 +2941,8 @@ const struct uapi_definition mlx5_ib_devx_defs[] = { UAPI_DEF_CHAIN_OBJ_TREE_NAMED( MLX5_IB_OBJECT_DEVX_ASYNC_CMD_FD, UAPI_DEF_IS_OBJ_SUPPORTED(devx_is_supported)), + UAPI_DEF_CHAIN_OBJ_TREE_NAMED( + MLX5_IB_OBJECT_DEVX_ASYNC_EVENT_FD, + UAPI_DEF_IS_OBJ_SUPPORTED(devx_is_supported)), {}, }; diff --git a/drivers/infiniband/hw/mlx5/mad.c b/drivers/infiniband/hw/mlx5/mad.c index 6c529e6f3a01..348c1df69cdc 100644 --- a/drivers/infiniband/hw/mlx5/mad.c +++ b/drivers/infiniband/hw/mlx5/mad.c @@ -200,19 +200,33 @@ static void pma_cnt_assign(struct ib_pma_portcounters *pma_cnt, vl_15_dropped); } -static int process_pma_cmd(struct mlx5_core_dev *mdev, u8 port_num, +static int process_pma_cmd(struct mlx5_ib_dev *dev, u8 port_num, const struct ib_mad *in_mad, struct ib_mad *out_mad) { - int err; + struct mlx5_core_dev *mdev; + bool native_port = true; + u8 mdev_port_num; void *out_cnt; + int err; + mdev = mlx5_ib_get_native_port_mdev(dev, port_num, &mdev_port_num); + if (!mdev) { + /* Fail to get the native port, likely due to 2nd port is still + * unaffiliated. In such case default to 1st port and attached + * PF device. + */ + native_port = false; + mdev = dev->mdev; + mdev_port_num = 1; + } /* Declaring support of extended counters */ if (in_mad->mad_hdr.attr_id == IB_PMA_CLASS_PORT_INFO) { struct ib_class_port_info cpi = {}; cpi.capability_mask = IB_PMA_CLASS_CAP_EXT_WIDTH; memcpy((out_mad->data + 40), &cpi, sizeof(cpi)); - return IB_MAD_RESULT_SUCCESS | IB_MAD_RESULT_REPLY; + err = IB_MAD_RESULT_SUCCESS | IB_MAD_RESULT_REPLY; + goto done; } if (in_mad->mad_hdr.attr_id == IB_PMA_PORT_COUNTERS_EXT) { @@ -221,11 +235,13 @@ static int process_pma_cmd(struct mlx5_core_dev *mdev, u8 port_num, int sz = MLX5_ST_SZ_BYTES(query_vport_counter_out); out_cnt = kvzalloc(sz, GFP_KERNEL); - if (!out_cnt) - return IB_MAD_RESULT_FAILURE; + if (!out_cnt) { + err = IB_MAD_RESULT_FAILURE; + goto done; + } err = mlx5_core_query_vport_counter(mdev, 0, 0, - port_num, out_cnt, sz); + mdev_port_num, out_cnt, sz); if (!err) pma_cnt_ext_assign(pma_cnt_ext, out_cnt); } else { @@ -234,20 +250,23 @@ static int process_pma_cmd(struct mlx5_core_dev *mdev, u8 port_num, int sz = MLX5_ST_SZ_BYTES(ppcnt_reg); out_cnt = kvzalloc(sz, GFP_KERNEL); - if (!out_cnt) - return IB_MAD_RESULT_FAILURE; + if (!out_cnt) { + err = IB_MAD_RESULT_FAILURE; + goto done; + } - err = mlx5_core_query_ib_ppcnt(mdev, port_num, + err = mlx5_core_query_ib_ppcnt(mdev, mdev_port_num, out_cnt, sz); if (!err) pma_cnt_assign(pma_cnt, out_cnt); - } - + } kvfree(out_cnt); - if (err) - return IB_MAD_RESULT_FAILURE; - - return IB_MAD_RESULT_SUCCESS | IB_MAD_RESULT_REPLY; + err = err ? IB_MAD_RESULT_FAILURE : + IB_MAD_RESULT_SUCCESS | IB_MAD_RESULT_REPLY; +done: + if (native_port) + mlx5_ib_put_native_port_mdev(dev, port_num); + return err; } int mlx5_ib_process_mad(struct ib_device *ibdev, int mad_flags, u8 port_num, @@ -259,8 +278,6 @@ int mlx5_ib_process_mad(struct ib_device *ibdev, int mad_flags, u8 port_num, struct mlx5_ib_dev *dev = to_mdev(ibdev); const struct ib_mad *in_mad = (const struct ib_mad *)in; struct ib_mad *out_mad = (struct ib_mad *)out; - struct mlx5_core_dev *mdev; - u8 mdev_port_num; int ret; if (WARN_ON_ONCE(in_mad_size != sizeof(*in_mad) || @@ -269,19 +286,14 @@ int mlx5_ib_process_mad(struct ib_device *ibdev, int mad_flags, u8 port_num, memset(out_mad->data, 0, sizeof(out_mad->data)); - mdev = mlx5_ib_get_native_port_mdev(dev, port_num, &mdev_port_num); - if (!mdev) - return IB_MAD_RESULT_FAILURE; - - if (MLX5_CAP_GEN(mdev, vport_counters) && + if (MLX5_CAP_GEN(dev->mdev, vport_counters) && in_mad->mad_hdr.mgmt_class == IB_MGMT_CLASS_PERF_MGMT && in_mad->mad_hdr.method == IB_MGMT_METHOD_GET) { - ret = process_pma_cmd(mdev, mdev_port_num, in_mad, out_mad); + ret = process_pma_cmd(dev, port_num, in_mad, out_mad); } else { ret = process_mad(ibdev, mad_flags, port_num, in_wc, in_grh, in_mad, out_mad); } - mlx5_ib_put_native_port_mdev(dev, port_num); return ret; } diff --git a/drivers/infiniband/hw/mlx5/main.c b/drivers/infiniband/hw/mlx5/main.c index ba312bf59c7a..c2a5780cb394 100644 --- a/drivers/infiniband/hw/mlx5/main.c +++ b/drivers/infiniband/hw/mlx5/main.c @@ -52,6 +52,7 @@ #include <linux/mlx5/port.h> #include <linux/mlx5/vport.h> #include <linux/mlx5/fs.h> +#include <linux/mlx5/eswitch.h> #include <linux/list.h> #include <rdma/ib_smi.h> #include <rdma/ib_umem.h> @@ -888,7 +889,7 @@ static int mlx5_ib_query_device(struct ib_device *ibdev, } props->device_cap_flags |= IB_DEVICE_MEM_MGT_EXTENSIONS; if (MLX5_CAP_GEN(mdev, sho)) { - props->device_cap_flags |= IB_DEVICE_SIGNATURE_HANDOVER; + props->device_cap_flags |= IB_DEVICE_INTEGRITY_HANDOVER; /* At this stage no support for signature handover */ props->sig_prot_cap = IB_PROT_T10DIF_TYPE_1 | IB_PROT_T10DIF_TYPE_2 | @@ -1008,6 +1009,8 @@ static int mlx5_ib_query_device(struct ib_device *ibdev, props->max_srq_sge = max_rq_sg - 1; props->max_fast_reg_page_list_len = 1 << MLX5_CAP_GEN(mdev, log_max_klm_list_size); + props->max_pi_fast_reg_page_list_len = + props->max_fast_reg_page_list_len / 2; get_atomic_caps_qp(dev, props); props->masked_atomic_cap = IB_ATOMIC_NONE; props->max_mcast_grp = 1 << MLX5_CAP_GEN(mdev, log_max_mcg); @@ -1043,15 +1046,19 @@ static int mlx5_ib_query_device(struct ib_device *ibdev, } if (MLX5_CAP_GEN(mdev, tag_matching)) { - props->tm_caps.max_rndv_hdr_size = MLX5_TM_MAX_RNDV_MSG_SIZE; props->tm_caps.max_num_tags = (1 << MLX5_CAP_GEN(mdev, log_tag_matching_list_sz)) - 1; - props->tm_caps.flags = IB_TM_CAP_RC; props->tm_caps.max_ops = 1 << MLX5_CAP_GEN(mdev, log_max_qp_sz); props->tm_caps.max_sge = MLX5_TM_MAX_SGE; } + if (MLX5_CAP_GEN(mdev, tag_matching) && + MLX5_CAP_GEN(mdev, rndv_offload_rc)) { + props->tm_caps.flags = IB_TM_CAP_RNDV_RC; + props->tm_caps.max_rndv_hdr_size = MLX5_TM_MAX_RNDV_MSG_SIZE; + } + if (MLX5_CAP_GEN(dev->mdev, cq_moderation)) { props->cq_caps.max_cq_moderation_count = MLX5_MAX_CQ_COUNT; @@ -3257,11 +3264,14 @@ static struct mlx5_ib_flow_prio *get_flow_table(struct mlx5_ib_dev *dev, int max_table_size; int num_entries; int num_groups; + bool esw_encap; u32 flags = 0; int priority; max_table_size = BIT(MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, log_max_ft_size)); + esw_encap = mlx5_eswitch_get_encap_mode(dev->mdev) != + DEVLINK_ESWITCH_ENCAP_MODE_NONE; if (flow_attr->type == IB_FLOW_ATTR_NORMAL) { enum mlx5_flow_namespace_type fn_type; @@ -3274,10 +3284,10 @@ static struct mlx5_ib_flow_prio *get_flow_table(struct mlx5_ib_dev *dev, if (ft_type == MLX5_IB_FT_RX) { fn_type = MLX5_FLOW_NAMESPACE_BYPASS; prio = &dev->flow_db->prios[priority]; - if (!dev->is_rep && + if (!dev->is_rep && !esw_encap && MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, decap)) flags |= MLX5_FLOW_TABLE_TUNNEL_EN_DECAP; - if (!dev->is_rep && + if (!dev->is_rep && !esw_encap && MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, reformat_l3_tunnel_to_l2)) flags |= MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT; @@ -3287,7 +3297,7 @@ static struct mlx5_ib_flow_prio *get_flow_table(struct mlx5_ib_dev *dev, log_max_ft_size)); fn_type = MLX5_FLOW_NAMESPACE_EGRESS; prio = &dev->flow_db->egress_prios[priority]; - if (!dev->is_rep && + if (!dev->is_rep && !esw_encap && MLX5_CAP_FLOWTABLE_NIC_TX(dev->mdev, reformat)) flags |= MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT; } @@ -3923,6 +3933,7 @@ _get_flow_table(struct mlx5_ib_dev *dev, struct mlx5_flow_namespace *ns = NULL; struct mlx5_ib_flow_prio *prio = NULL; int max_table_size = 0; + bool esw_encap; u32 flags = 0; int priority; @@ -3931,22 +3942,30 @@ _get_flow_table(struct mlx5_ib_dev *dev, else priority = ib_prio_to_core_prio(fs_matcher->priority, false); + esw_encap = mlx5_eswitch_get_encap_mode(dev->mdev) != + DEVLINK_ESWITCH_ENCAP_MODE_NONE; if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_BYPASS) { max_table_size = BIT(MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, log_max_ft_size)); - if (MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, decap)) + if (MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, decap) && !esw_encap) flags |= MLX5_FLOW_TABLE_TUNNEL_EN_DECAP; if (MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, - reformat_l3_tunnel_to_l2)) + reformat_l3_tunnel_to_l2) && + !esw_encap) flags |= MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT; } else if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_EGRESS) { max_table_size = BIT( MLX5_CAP_FLOWTABLE_NIC_TX(dev->mdev, log_max_ft_size)); - if (MLX5_CAP_FLOWTABLE_NIC_TX(dev->mdev, reformat)) + if (MLX5_CAP_FLOWTABLE_NIC_TX(dev->mdev, reformat) && !esw_encap) flags |= MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT; } else if (fs_matcher->ns_type == MLX5_FLOW_NAMESPACE_FDB) { max_table_size = BIT( MLX5_CAP_ESW_FLOWTABLE_FDB(dev->mdev, log_max_ft_size)); + if (MLX5_CAP_ESW_FLOWTABLE_FDB(dev->mdev, decap) && esw_encap) + flags |= MLX5_FLOW_TABLE_TUNNEL_EN_DECAP; + if (MLX5_CAP_ESW_FLOWTABLE_FDB(dev->mdev, reformat_l3_tunnel_to_l2) && + esw_encap) + flags |= MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT; priority = FDB_BYPASS_PATH; } @@ -4711,7 +4730,7 @@ static int __get_port_caps(struct mlx5_ib_dev *dev, u8 port) int err = -ENOMEM; struct ib_udata uhw = {.inlen = 0, .outlen = 0}; - pprops = kmalloc(sizeof(*pprops), GFP_KERNEL); + pprops = kzalloc(sizeof(*pprops), GFP_KERNEL); if (!pprops) goto out; @@ -4725,7 +4744,6 @@ static int __get_port_caps(struct mlx5_ib_dev *dev, u8 port) goto out; } - memset(pprops, 0, sizeof(*pprops)); err = mlx5_ib_query_port(&dev->ib_dev, port, pprops); if (err) { mlx5_ib_warn(dev, "query_port %d failed %d\n", @@ -4926,18 +4944,19 @@ static int create_dev_resources(struct mlx5_ib_resources *devr) if (ret) goto error0; - devr->c0 = mlx5_ib_create_cq(&dev->ib_dev, &cq_attr, NULL); - if (IS_ERR(devr->c0)) { - ret = PTR_ERR(devr->c0); + devr->c0 = rdma_zalloc_drv_obj(ibdev, ib_cq); + if (!devr->c0) { + ret = -ENOMEM; goto error1; } - devr->c0->device = &dev->ib_dev; - devr->c0->uobject = NULL; - devr->c0->comp_handler = NULL; - devr->c0->event_handler = NULL; - devr->c0->cq_context = NULL; + + devr->c0->device = &dev->ib_dev; atomic_set(&devr->c0->usecnt, 0); + ret = mlx5_ib_create_cq(devr->c0, &cq_attr, NULL); + if (ret) + goto err_create_cq; + devr->x0 = mlx5_ib_alloc_xrcd(&dev->ib_dev, NULL); if (IS_ERR(devr->x0)) { ret = PTR_ERR(devr->x0); @@ -5029,6 +5048,8 @@ error3: mlx5_ib_dealloc_xrcd(devr->x0, NULL); error2: mlx5_ib_destroy_cq(devr->c0, NULL); +err_create_cq: + kfree(devr->c0); error1: mlx5_ib_dealloc_pd(devr->p0, NULL); error0: @@ -5047,6 +5068,7 @@ static void destroy_dev_resources(struct mlx5_ib_resources *devr) mlx5_ib_dealloc_xrcd(devr->x0, NULL); mlx5_ib_dealloc_xrcd(devr->x1, NULL); mlx5_ib_destroy_cq(devr->c0, NULL); + kfree(devr->c0); mlx5_ib_dealloc_pd(devr->p0, NULL); kfree(devr->p0); @@ -5459,7 +5481,8 @@ static struct rdma_hw_stats *mlx5_ib_alloc_hw_stats(struct ib_device *ibdev, static int mlx5_ib_query_q_counters(struct mlx5_core_dev *mdev, struct mlx5_ib_port *port, - struct rdma_hw_stats *stats) + struct rdma_hw_stats *stats, + u16 set_id) { int outlen = MLX5_ST_SZ_BYTES(query_q_counter_out); void *out; @@ -5470,9 +5493,7 @@ static int mlx5_ib_query_q_counters(struct mlx5_core_dev *mdev, if (!out) return -ENOMEM; - ret = mlx5_core_query_q_counter(mdev, - port->cnts.set_id, 0, - out, outlen); + ret = mlx5_core_query_q_counter(mdev, set_id, 0, out, outlen); if (ret) goto free; @@ -5532,7 +5553,8 @@ static int mlx5_ib_get_hw_stats(struct ib_device *ibdev, port->cnts.num_ext_ppcnt_counters; /* q_counters are per IB device, query the master mdev */ - ret = mlx5_ib_query_q_counters(dev->mdev, port, stats); + ret = mlx5_ib_query_q_counters(dev->mdev, port, stats, + port->cnts.set_id); if (ret) return ret; @@ -5568,6 +5590,68 @@ done: return num_counters; } +static struct rdma_hw_stats * +mlx5_ib_counter_alloc_stats(struct rdma_counter *counter) +{ + struct mlx5_ib_dev *dev = to_mdev(counter->device); + struct mlx5_ib_port *port = &dev->port[counter->port - 1]; + + /* Q counters are in the beginning of all counters */ + return rdma_alloc_hw_stats_struct(port->cnts.names, + port->cnts.num_q_counters, + RDMA_HW_STATS_DEFAULT_LIFESPAN); +} + +static int mlx5_ib_counter_update_stats(struct rdma_counter *counter) +{ + struct mlx5_ib_dev *dev = to_mdev(counter->device); + struct mlx5_ib_port *port = &dev->port[counter->port - 1]; + + return mlx5_ib_query_q_counters(dev->mdev, port, + counter->stats, counter->id); +} + +static int mlx5_ib_counter_bind_qp(struct rdma_counter *counter, + struct ib_qp *qp) +{ + struct mlx5_ib_dev *dev = to_mdev(qp->device); + u16 cnt_set_id = 0; + int err; + + if (!counter->id) { + err = mlx5_cmd_alloc_q_counter(dev->mdev, + &cnt_set_id, + MLX5_SHARED_RESOURCE_UID); + if (err) + return err; + counter->id = cnt_set_id; + } + + err = mlx5_ib_qp_set_counter(qp, counter); + if (err) + goto fail_set_counter; + + return 0; + +fail_set_counter: + mlx5_core_dealloc_q_counter(dev->mdev, cnt_set_id); + counter->id = 0; + + return err; +} + +static int mlx5_ib_counter_unbind_qp(struct ib_qp *qp) +{ + return mlx5_ib_qp_set_counter(qp, NULL); +} + +static int mlx5_ib_counter_dealloc(struct rdma_counter *counter) +{ + struct mlx5_ib_dev *dev = to_mdev(counter->device); + + return mlx5_core_dealloc_q_counter(dev->mdev, counter->id); +} + static int mlx5_ib_rn_get_params(struct ib_device *device, u8 port_num, enum rdma_netdev_t type, struct rdma_netdev_alloc_params *params) @@ -6079,7 +6163,6 @@ static int mlx5_ib_stage_init_init(struct mlx5_ib_dev *dev) if (mlx5_use_mad_ifc(dev)) get_ext_port_caps(dev); - dev->ib_dev.owner = THIS_MODULE; dev->ib_dev.node_type = RDMA_NODE_IB_CA; dev->ib_dev.local_dma_lkey = 0 /* not supported for now */; dev->ib_dev.phys_port_cnt = dev->num_ports; @@ -6159,8 +6242,13 @@ static void mlx5_ib_stage_flow_db_cleanup(struct mlx5_ib_dev *dev) } static const struct ib_device_ops mlx5_ib_dev_ops = { + .owner = THIS_MODULE, + .driver_id = RDMA_DRIVER_MLX5, + .uverbs_abi_ver = MLX5_IB_UVERBS_ABI_VERSION, + .add_gid = mlx5_ib_add_gid, .alloc_mr = mlx5_ib_alloc_mr, + .alloc_mr_integrity = mlx5_ib_alloc_mr_integrity, .alloc_pd = mlx5_ib_alloc_pd, .alloc_ucontext = mlx5_ib_alloc_ucontext, .attach_mcast = mlx5_ib_mcg_attach, @@ -6190,6 +6278,7 @@ static const struct ib_device_ops mlx5_ib_dev_ops = { .get_dma_mr = mlx5_ib_get_dma_mr, .get_link_layer = mlx5_ib_port_link_layer, .map_mr_sg = mlx5_ib_map_mr_sg, + .map_mr_sg_pi = mlx5_ib_map_mr_sg_pi, .mmap = mlx5_ib_mmap, .modify_cq = mlx5_ib_modify_cq, .modify_device = mlx5_ib_modify_device, @@ -6214,6 +6303,7 @@ static const struct ib_device_ops mlx5_ib_dev_ops = { .resize_cq = mlx5_ib_resize_cq, INIT_RDMA_OBJ_SIZE(ib_ah, mlx5_ib_ah, ibah), + INIT_RDMA_OBJ_SIZE(ib_cq, mlx5_ib_cq, ibcq), INIT_RDMA_OBJ_SIZE(ib_pd, mlx5_ib_pd, ibpd), INIT_RDMA_OBJ_SIZE(ib_srq, mlx5_ib_srq, ibsrq), INIT_RDMA_OBJ_SIZE(ib_ucontext, mlx5_ib_ucontext, ibucontext), @@ -6256,7 +6346,6 @@ static int mlx5_ib_stage_caps_init(struct mlx5_ib_dev *dev) struct mlx5_core_dev *mdev = dev->mdev; int err; - dev->ib_dev.uverbs_abi_ver = MLX5_IB_UVERBS_ABI_VERSION; dev->ib_dev.uverbs_cmd_mask = (1ull << IB_USER_VERBS_CMD_GET_CONTEXT) | (1ull << IB_USER_VERBS_CMD_QUERY_DEVICE) | @@ -6325,7 +6414,6 @@ static int mlx5_ib_stage_caps_init(struct mlx5_ib_dev *dev) if (mlx5_accel_ipsec_device_caps(dev->mdev) & MLX5_ACCEL_IPSEC_CAP_DEVICE) ib_set_device_ops(&dev->ib_dev, &mlx5_ib_dev_flow_ipsec_ops); - dev->ib_dev.driver_id = RDMA_DRIVER_MLX5; ib_set_device_ops(&dev->ib_dev, &mlx5_ib_dev_ops); if (IS_ENABLED(CONFIG_INFINIBAND_USER_ACCESS)) @@ -6340,6 +6428,8 @@ static int mlx5_ib_stage_caps_init(struct mlx5_ib_dev *dev) MLX5_CAP_GEN(dev->mdev, disable_local_lb_mc))) mutex_init(&dev->lb.mutex); + dev->ib_dev.use_cq_dim = true; + return 0; } @@ -6487,6 +6577,11 @@ static void mlx5_ib_stage_odp_cleanup(struct mlx5_ib_dev *dev) static const struct ib_device_ops mlx5_ib_dev_hw_stats_ops = { .alloc_hw_stats = mlx5_ib_alloc_hw_stats, .get_hw_stats = mlx5_ib_get_hw_stats, + .counter_bind_qp = mlx5_ib_counter_bind_qp, + .counter_unbind_qp = mlx5_ib_counter_unbind_qp, + .counter_dealloc = mlx5_ib_counter_dealloc, + .counter_alloc_stats = mlx5_ib_counter_alloc_stats, + .counter_update_stats = mlx5_ib_counter_update_stats, }; static int mlx5_ib_stage_counters_init(struct mlx5_ib_dev *dev) @@ -6607,15 +6702,19 @@ static int mlx5_ib_stage_devx_init(struct mlx5_ib_dev *dev) int uid; uid = mlx5_ib_devx_create(dev, false); - if (uid > 0) + if (uid > 0) { dev->devx_whitelist_uid = uid; + mlx5_ib_devx_init_event_table(dev); + } return 0; } static void mlx5_ib_stage_devx_cleanup(struct mlx5_ib_dev *dev) { - if (dev->devx_whitelist_uid) + if (dev->devx_whitelist_uid) { + mlx5_ib_devx_cleanup_event_table(dev); mlx5_ib_devx_destroy(dev, dev->devx_whitelist_uid); + } } void __mlx5_ib_remove(struct mlx5_ib_dev *dev, diff --git a/drivers/infiniband/hw/mlx5/mem.c b/drivers/infiniband/hw/mlx5/mem.c index 9f90be296ee0..fe1a76d8531c 100644 --- a/drivers/infiniband/hw/mlx5/mem.c +++ b/drivers/infiniband/hw/mlx5/mem.c @@ -55,9 +55,10 @@ void mlx5_ib_cont_pages(struct ib_umem *umem, u64 addr, int i = 0; struct scatterlist *sg; int entry; - unsigned long page_shift = umem->page_shift; if (umem->is_odp) { + unsigned int page_shift = to_ib_umem_odp(umem)->page_shift; + *ncont = ib_umem_page_count(umem); *count = *ncont << (page_shift - PAGE_SHIFT); *shift = page_shift; @@ -67,15 +68,15 @@ void mlx5_ib_cont_pages(struct ib_umem *umem, u64 addr, return; } - addr = addr >> page_shift; + addr = addr >> PAGE_SHIFT; tmp = (unsigned long)addr; m = find_first_bit(&tmp, BITS_PER_LONG); if (max_page_shift) - m = min_t(unsigned long, max_page_shift - page_shift, m); + m = min_t(unsigned long, max_page_shift - PAGE_SHIFT, m); for_each_sg(umem->sg_head.sgl, sg, umem->nmap, entry) { - len = sg_dma_len(sg) >> page_shift; - pfn = sg_dma_address(sg) >> page_shift; + len = sg_dma_len(sg) >> PAGE_SHIFT; + pfn = sg_dma_address(sg) >> PAGE_SHIFT; if (base + p != pfn) { /* If either the offset or the new * base are unaligned update m @@ -107,7 +108,7 @@ void mlx5_ib_cont_pages(struct ib_umem *umem, u64 addr, *ncont = 0; } - *shift = page_shift + m; + *shift = PAGE_SHIFT + m; *count = i; } @@ -140,8 +141,7 @@ void __mlx5_ib_populate_pas(struct mlx5_ib_dev *dev, struct ib_umem *umem, int page_shift, size_t offset, size_t num_pages, __be64 *pas, int access_flags) { - unsigned long umem_page_shift = umem->page_shift; - int shift = page_shift - umem_page_shift; + int shift = page_shift - PAGE_SHIFT; int mask = (1 << shift) - 1; int i, k, idx; u64 cur = 0; @@ -165,7 +165,7 @@ void __mlx5_ib_populate_pas(struct mlx5_ib_dev *dev, struct ib_umem *umem, i = 0; for_each_sg(umem->sg_head.sgl, sg, umem->nmap, entry) { - len = sg_dma_len(sg) >> umem_page_shift; + len = sg_dma_len(sg) >> PAGE_SHIFT; base = sg_dma_address(sg); /* Skip elements below offset */ @@ -184,7 +184,7 @@ void __mlx5_ib_populate_pas(struct mlx5_ib_dev *dev, struct ib_umem *umem, for (; k < len; k++) { if (!(i & mask)) { - cur = base + (k << umem_page_shift); + cur = base + (k << PAGE_SHIFT); cur |= access_flags; idx = (i >> shift) - offset; diff --git a/drivers/infiniband/hw/mlx5/mlx5_ib.h b/drivers/infiniband/hw/mlx5/mlx5_ib.h index ee73dc122d28..c482f19958b3 100644 --- a/drivers/infiniband/hw/mlx5/mlx5_ib.h +++ b/drivers/infiniband/hw/mlx5/mlx5_ib.h @@ -431,9 +431,6 @@ struct mlx5_ib_qp { int create_type; - /* Store signature errors */ - bool signature_en; - struct list_head qps_list; struct list_head cq_recv_list; struct list_head cq_send_list; @@ -442,6 +439,10 @@ struct mlx5_ib_qp { u32 flags_en; /* storage for qp sub type when core qp type is IB_QPT_DRIVER */ enum ib_qp_type qp_sub_type; + /* A flag to indicate if there's a new counter is configured + * but not take effective + */ + u32 counter_pending; }; struct mlx5_ib_cq_buf { @@ -587,6 +588,9 @@ struct mlx5_ib_mr { void *descs; dma_addr_t desc_map; int ndescs; + int data_length; + int meta_ndescs; + int meta_length; int max_descs; int desc_size; int access_mode; @@ -605,6 +609,13 @@ struct mlx5_ib_mr { int access_flags; /* Needed for rereg MR */ struct mlx5_ib_mr *parent; + /* Needed for IB_MR_TYPE_INTEGRITY */ + struct mlx5_ib_mr *pi_mr; + struct mlx5_ib_mr *klm_mr; + struct mlx5_ib_mr *mtt_mr; + u64 data_iova; + u64 pi_iova; + atomic_t num_leaf_free; wait_queue_head_t q_leaf_free; struct mlx5_async_work cb_work; @@ -929,6 +940,13 @@ struct mlx5_ib_pf_eq { mempool_t *pool; }; +struct mlx5_devx_event_table { + struct mlx5_nb devx_nb; + /* serialize updating the event_xa */ + struct mutex event_xa_lock; + struct xarray event_xa; +}; + struct mlx5_ib_dev { struct ib_device ib_dev; struct mlx5_core_dev *mdev; @@ -978,6 +996,7 @@ struct mlx5_ib_dev { u16 devx_whitelist_uid; struct mlx5_srq_table srq_table; struct mlx5_async_ctx async_ctx; + struct mlx5_devx_event_table devx_event_table; }; static inline struct mlx5_ib_cq *to_mibcq(struct mlx5_core_cq *mcq) @@ -1115,10 +1134,9 @@ int mlx5_ib_read_user_wqe_rq(struct mlx5_ib_qp *qp, int wqe_index, void *buffer, int buflen, size_t *bc); int mlx5_ib_read_user_wqe_srq(struct mlx5_ib_srq *srq, int wqe_index, void *buffer, int buflen, size_t *bc); -struct ib_cq *mlx5_ib_create_cq(struct ib_device *ibdev, - const struct ib_cq_init_attr *attr, - struct ib_udata *udata); -int mlx5_ib_destroy_cq(struct ib_cq *cq, struct ib_udata *udata); +int mlx5_ib_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, + struct ib_udata *udata); +void mlx5_ib_destroy_cq(struct ib_cq *cq, struct ib_udata *udata); int mlx5_ib_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc); int mlx5_ib_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags); int mlx5_ib_modify_cq(struct ib_cq *cq, u16 cq_count, u16 cq_period); @@ -1148,8 +1166,15 @@ int mlx5_ib_rereg_user_mr(struct ib_mr *ib_mr, int flags, u64 start, int mlx5_ib_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata); struct ib_mr *mlx5_ib_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type, u32 max_num_sg, struct ib_udata *udata); +struct ib_mr *mlx5_ib_alloc_mr_integrity(struct ib_pd *pd, + u32 max_num_sg, + u32 max_num_meta_sg); int mlx5_ib_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents, unsigned int *sg_offset); +int mlx5_ib_map_mr_sg_pi(struct ib_mr *ibmr, struct scatterlist *data_sg, + int data_sg_nents, unsigned int *data_sg_offset, + struct scatterlist *meta_sg, int meta_sg_nents, + unsigned int *meta_sg_offset); int mlx5_ib_process_mad(struct ib_device *ibdev, int mad_flags, u8 port_num, const struct ib_wc *in_wc, const struct ib_grh *in_grh, const struct ib_mad_hdr *in, size_t in_mad_size, @@ -1201,7 +1226,7 @@ int mlx5_ib_check_mr_status(struct ib_mr *ibmr, u32 check_mask, struct ib_wq *mlx5_ib_create_wq(struct ib_pd *pd, struct ib_wq_init_attr *init_attr, struct ib_udata *udata); -int mlx5_ib_destroy_wq(struct ib_wq *wq, struct ib_udata *udata); +void mlx5_ib_destroy_wq(struct ib_wq *wq, struct ib_udata *udata); int mlx5_ib_modify_wq(struct ib_wq *wq, struct ib_wq_attr *wq_attr, u32 wq_attr_mask, struct ib_udata *udata); struct ib_rwq_ind_table *mlx5_ib_create_rwq_ind_table(struct ib_device *device, @@ -1311,6 +1336,8 @@ void mlx5_ib_put_native_port_mdev(struct mlx5_ib_dev *dev, #if IS_ENABLED(CONFIG_INFINIBAND_USER_ACCESS) int mlx5_ib_devx_create(struct mlx5_ib_dev *dev, bool is_user); void mlx5_ib_devx_destroy(struct mlx5_ib_dev *dev, u16 uid); +void mlx5_ib_devx_init_event_table(struct mlx5_ib_dev *dev); +void mlx5_ib_devx_cleanup_event_table(struct mlx5_ib_dev *dev); const struct uverbs_object_tree_def *mlx5_ib_get_devx_tree(void); extern const struct uapi_definition mlx5_ib_devx_defs[]; extern const struct uapi_definition mlx5_ib_flow_defs[]; @@ -1328,6 +1355,8 @@ static inline int mlx5_ib_devx_create(struct mlx5_ib_dev *dev, bool is_user) { return -EOPNOTSUPP; } static inline void mlx5_ib_devx_destroy(struct mlx5_ib_dev *dev, u16 uid) {} +static inline void mlx5_ib_devx_init_event_table(struct mlx5_ib_dev *dev) {} +static inline void mlx5_ib_devx_cleanup_event_table(struct mlx5_ib_dev *dev) {} static inline bool mlx5_ib_devx_is_flow_dest(void *obj, int *dest_id, int *dest_type) { @@ -1443,4 +1472,6 @@ void mlx5_ib_put_xlt_emergency_page(void); int bfregn_to_uar_index(struct mlx5_ib_dev *dev, struct mlx5_bfreg_info *bfregi, u32 bfregn, bool dyn_bfreg); + +int mlx5_ib_qp_set_counter(struct ib_qp *qp, struct rdma_counter *counter); #endif /* MLX5_IB_H */ diff --git a/drivers/infiniband/hw/mlx5/mr.c b/drivers/infiniband/hw/mlx5/mr.c index 83b452d977d4..20ece6e0b2fc 100644 --- a/drivers/infiniband/hw/mlx5/mr.c +++ b/drivers/infiniband/hw/mlx5/mr.c @@ -1507,10 +1507,9 @@ int mlx5_ib_rereg_user_mr(struct ib_mr *ib_mr, int flags, u64 start, return 0; err: - if (mr->umem) { - ib_umem_release(mr->umem); - mr->umem = NULL; - } + ib_umem_release(mr->umem); + mr->umem = NULL; + clean_mr(dev, mr); return err; } @@ -1606,8 +1605,9 @@ static void dereg_mr(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr) synchronize_srcu(&dev->mr_srcu); /* Destroy all page mappings */ if (umem_odp->page_list) - mlx5_ib_invalidate_range(umem_odp, ib_umem_start(umem), - ib_umem_end(umem)); + mlx5_ib_invalidate_range(umem_odp, + ib_umem_start(umem_odp), + ib_umem_end(umem_odp)); else mlx5_ib_free_implicit_mr(mr); /* @@ -1629,28 +1629,85 @@ static void dereg_mr(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr) * remove the DMA mapping. */ mlx5_mr_cache_free(dev, mr); - if (umem) { - ib_umem_release(umem); + ib_umem_release(umem); + if (umem) atomic_sub(npages, &dev->mdev->priv.reg_pages); - } + if (!mr->allocated_from_cache) kfree(mr); } int mlx5_ib_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata) { - dereg_mr(to_mdev(ibmr->device), to_mmr(ibmr)); + struct mlx5_ib_mr *mmr = to_mmr(ibmr); + + if (ibmr->type == IB_MR_TYPE_INTEGRITY) { + dereg_mr(to_mdev(mmr->mtt_mr->ibmr.device), mmr->mtt_mr); + dereg_mr(to_mdev(mmr->klm_mr->ibmr.device), mmr->klm_mr); + } + + dereg_mr(to_mdev(ibmr->device), mmr); + return 0; } -struct ib_mr *mlx5_ib_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type, - u32 max_num_sg, struct ib_udata *udata) +static void mlx5_set_umr_free_mkey(struct ib_pd *pd, u32 *in, int ndescs, + int access_mode, int page_shift) +{ + void *mkc; + + mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry); + + MLX5_SET(mkc, mkc, free, 1); + MLX5_SET(mkc, mkc, qpn, 0xffffff); + MLX5_SET(mkc, mkc, pd, to_mpd(pd)->pdn); + MLX5_SET(mkc, mkc, translations_octword_size, ndescs); + MLX5_SET(mkc, mkc, access_mode_1_0, access_mode & 0x3); + MLX5_SET(mkc, mkc, access_mode_4_2, (access_mode >> 2) & 0x7); + MLX5_SET(mkc, mkc, umr_en, 1); + MLX5_SET(mkc, mkc, log_page_size, page_shift); +} + +static int _mlx5_alloc_mkey_descs(struct ib_pd *pd, struct mlx5_ib_mr *mr, + int ndescs, int desc_size, int page_shift, + int access_mode, u32 *in, int inlen) { struct mlx5_ib_dev *dev = to_mdev(pd->device); + int err; + + mr->access_mode = access_mode; + mr->desc_size = desc_size; + mr->max_descs = ndescs; + + err = mlx5_alloc_priv_descs(pd->device, mr, ndescs, desc_size); + if (err) + return err; + + mlx5_set_umr_free_mkey(pd, in, ndescs, access_mode, page_shift); + + err = mlx5_core_create_mkey(dev->mdev, &mr->mmkey, in, inlen); + if (err) + goto err_free_descs; + + mr->mmkey.type = MLX5_MKEY_MR; + mr->ibmr.lkey = mr->mmkey.key; + mr->ibmr.rkey = mr->mmkey.key; + + return 0; + +err_free_descs: + mlx5_free_priv_descs(mr); + return err; +} + +static struct mlx5_ib_mr *mlx5_ib_alloc_pi_mr(struct ib_pd *pd, + u32 max_num_sg, u32 max_num_meta_sg, + int desc_size, int access_mode) +{ int inlen = MLX5_ST_SZ_BYTES(create_mkey_in); - int ndescs = ALIGN(max_num_sg, 4); + int ndescs = ALIGN(max_num_sg + max_num_meta_sg, 4); + int page_shift = 0; struct mlx5_ib_mr *mr; - void *mkc; u32 *in; int err; @@ -1658,99 +1715,168 @@ struct ib_mr *mlx5_ib_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type, if (!mr) return ERR_PTR(-ENOMEM); + mr->ibmr.pd = pd; + mr->ibmr.device = pd->device; + in = kzalloc(inlen, GFP_KERNEL); if (!in) { err = -ENOMEM; goto err_free; } + if (access_mode == MLX5_MKC_ACCESS_MODE_MTT) + page_shift = PAGE_SHIFT; + + err = _mlx5_alloc_mkey_descs(pd, mr, ndescs, desc_size, page_shift, + access_mode, in, inlen); + if (err) + goto err_free_in; + + mr->umem = NULL; + kfree(in); + + return mr; + +err_free_in: + kfree(in); +err_free: + kfree(mr); + return ERR_PTR(err); +} + +static int mlx5_alloc_mem_reg_descs(struct ib_pd *pd, struct mlx5_ib_mr *mr, + int ndescs, u32 *in, int inlen) +{ + return _mlx5_alloc_mkey_descs(pd, mr, ndescs, sizeof(struct mlx5_mtt), + PAGE_SHIFT, MLX5_MKC_ACCESS_MODE_MTT, in, + inlen); +} + +static int mlx5_alloc_sg_gaps_descs(struct ib_pd *pd, struct mlx5_ib_mr *mr, + int ndescs, u32 *in, int inlen) +{ + return _mlx5_alloc_mkey_descs(pd, mr, ndescs, sizeof(struct mlx5_klm), + 0, MLX5_MKC_ACCESS_MODE_KLMS, in, inlen); +} + +static int mlx5_alloc_integrity_descs(struct ib_pd *pd, struct mlx5_ib_mr *mr, + int max_num_sg, int max_num_meta_sg, + u32 *in, int inlen) +{ + struct mlx5_ib_dev *dev = to_mdev(pd->device); + u32 psv_index[2]; + void *mkc; + int err; + + mr->sig = kzalloc(sizeof(*mr->sig), GFP_KERNEL); + if (!mr->sig) + return -ENOMEM; + + /* create mem & wire PSVs */ + err = mlx5_core_create_psv(dev->mdev, to_mpd(pd)->pdn, 2, psv_index); + if (err) + goto err_free_sig; + + mr->sig->psv_memory.psv_idx = psv_index[0]; + mr->sig->psv_wire.psv_idx = psv_index[1]; + + mr->sig->sig_status_checked = true; + mr->sig->sig_err_exists = false; + /* Next UMR, Arm SIGERR */ + ++mr->sig->sigerr_count; + mr->klm_mr = mlx5_ib_alloc_pi_mr(pd, max_num_sg, max_num_meta_sg, + sizeof(struct mlx5_klm), + MLX5_MKC_ACCESS_MODE_KLMS); + if (IS_ERR(mr->klm_mr)) { + err = PTR_ERR(mr->klm_mr); + goto err_destroy_psv; + } + mr->mtt_mr = mlx5_ib_alloc_pi_mr(pd, max_num_sg, max_num_meta_sg, + sizeof(struct mlx5_mtt), + MLX5_MKC_ACCESS_MODE_MTT); + if (IS_ERR(mr->mtt_mr)) { + err = PTR_ERR(mr->mtt_mr); + goto err_free_klm_mr; + } + + /* Set bsf descriptors for mkey */ mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry); - MLX5_SET(mkc, mkc, free, 1); - MLX5_SET(mkc, mkc, translations_octword_size, ndescs); - MLX5_SET(mkc, mkc, qpn, 0xffffff); - MLX5_SET(mkc, mkc, pd, to_mpd(pd)->pdn); + MLX5_SET(mkc, mkc, bsf_en, 1); + MLX5_SET(mkc, mkc, bsf_octword_size, MLX5_MKEY_BSF_OCTO_SIZE); - if (mr_type == IB_MR_TYPE_MEM_REG) { - mr->access_mode = MLX5_MKC_ACCESS_MODE_MTT; - MLX5_SET(mkc, mkc, log_page_size, PAGE_SHIFT); - err = mlx5_alloc_priv_descs(pd->device, mr, - ndescs, sizeof(struct mlx5_mtt)); - if (err) - goto err_free_in; + err = _mlx5_alloc_mkey_descs(pd, mr, 4, sizeof(struct mlx5_klm), 0, + MLX5_MKC_ACCESS_MODE_KLMS, in, inlen); + if (err) + goto err_free_mtt_mr; - mr->desc_size = sizeof(struct mlx5_mtt); - mr->max_descs = ndescs; - } else if (mr_type == IB_MR_TYPE_SG_GAPS) { - mr->access_mode = MLX5_MKC_ACCESS_MODE_KLMS; + return 0; - err = mlx5_alloc_priv_descs(pd->device, mr, - ndescs, sizeof(struct mlx5_klm)); - if (err) - goto err_free_in; - mr->desc_size = sizeof(struct mlx5_klm); - mr->max_descs = ndescs; - } else if (mr_type == IB_MR_TYPE_SIGNATURE) { - u32 psv_index[2]; - - MLX5_SET(mkc, mkc, bsf_en, 1); - MLX5_SET(mkc, mkc, bsf_octword_size, MLX5_MKEY_BSF_OCTO_SIZE); - mr->sig = kzalloc(sizeof(*mr->sig), GFP_KERNEL); - if (!mr->sig) { - err = -ENOMEM; - goto err_free_in; - } +err_free_mtt_mr: + dereg_mr(to_mdev(mr->mtt_mr->ibmr.device), mr->mtt_mr); + mr->mtt_mr = NULL; +err_free_klm_mr: + dereg_mr(to_mdev(mr->klm_mr->ibmr.device), mr->klm_mr); + mr->klm_mr = NULL; +err_destroy_psv: + if (mlx5_core_destroy_psv(dev->mdev, mr->sig->psv_memory.psv_idx)) + mlx5_ib_warn(dev, "failed to destroy mem psv %d\n", + mr->sig->psv_memory.psv_idx); + if (mlx5_core_destroy_psv(dev->mdev, mr->sig->psv_wire.psv_idx)) + mlx5_ib_warn(dev, "failed to destroy wire psv %d\n", + mr->sig->psv_wire.psv_idx); +err_free_sig: + kfree(mr->sig); - /* create mem & wire PSVs */ - err = mlx5_core_create_psv(dev->mdev, to_mpd(pd)->pdn, - 2, psv_index); - if (err) - goto err_free_sig; + return err; +} + +static struct ib_mr *__mlx5_ib_alloc_mr(struct ib_pd *pd, + enum ib_mr_type mr_type, u32 max_num_sg, + u32 max_num_meta_sg) +{ + struct mlx5_ib_dev *dev = to_mdev(pd->device); + int inlen = MLX5_ST_SZ_BYTES(create_mkey_in); + int ndescs = ALIGN(max_num_sg, 4); + struct mlx5_ib_mr *mr; + u32 *in; + int err; - mr->access_mode = MLX5_MKC_ACCESS_MODE_KLMS; - mr->sig->psv_memory.psv_idx = psv_index[0]; - mr->sig->psv_wire.psv_idx = psv_index[1]; + mr = kzalloc(sizeof(*mr), GFP_KERNEL); + if (!mr) + return ERR_PTR(-ENOMEM); - mr->sig->sig_status_checked = true; - mr->sig->sig_err_exists = false; - /* Next UMR, Arm SIGERR */ - ++mr->sig->sigerr_count; - } else { + in = kzalloc(inlen, GFP_KERNEL); + if (!in) { + err = -ENOMEM; + goto err_free; + } + + mr->ibmr.device = pd->device; + mr->umem = NULL; + + switch (mr_type) { + case IB_MR_TYPE_MEM_REG: + err = mlx5_alloc_mem_reg_descs(pd, mr, ndescs, in, inlen); + break; + case IB_MR_TYPE_SG_GAPS: + err = mlx5_alloc_sg_gaps_descs(pd, mr, ndescs, in, inlen); + break; + case IB_MR_TYPE_INTEGRITY: + err = mlx5_alloc_integrity_descs(pd, mr, max_num_sg, + max_num_meta_sg, in, inlen); + break; + default: mlx5_ib_warn(dev, "Invalid mr type %d\n", mr_type); err = -EINVAL; - goto err_free_in; } - MLX5_SET(mkc, mkc, access_mode_1_0, mr->access_mode & 0x3); - MLX5_SET(mkc, mkc, access_mode_4_2, (mr->access_mode >> 2) & 0x7); - MLX5_SET(mkc, mkc, umr_en, 1); - - mr->ibmr.device = pd->device; - err = mlx5_core_create_mkey(dev->mdev, &mr->mmkey, in, inlen); if (err) - goto err_destroy_psv; + goto err_free_in; - mr->mmkey.type = MLX5_MKEY_MR; - mr->ibmr.lkey = mr->mmkey.key; - mr->ibmr.rkey = mr->mmkey.key; - mr->umem = NULL; kfree(in); return &mr->ibmr; -err_destroy_psv: - if (mr->sig) { - if (mlx5_core_destroy_psv(dev->mdev, - mr->sig->psv_memory.psv_idx)) - mlx5_ib_warn(dev, "failed to destroy mem psv %d\n", - mr->sig->psv_memory.psv_idx); - if (mlx5_core_destroy_psv(dev->mdev, - mr->sig->psv_wire.psv_idx)) - mlx5_ib_warn(dev, "failed to destroy wire psv %d\n", - mr->sig->psv_wire.psv_idx); - } - mlx5_free_priv_descs(mr); -err_free_sig: - kfree(mr->sig); err_free_in: kfree(in); err_free: @@ -1758,6 +1884,19 @@ err_free: return ERR_PTR(err); } +struct ib_mr *mlx5_ib_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type, + u32 max_num_sg, struct ib_udata *udata) +{ + return __mlx5_ib_alloc_mr(pd, mr_type, max_num_sg, 0); +} + +struct ib_mr *mlx5_ib_alloc_mr_integrity(struct ib_pd *pd, + u32 max_num_sg, u32 max_num_meta_sg) +{ + return __mlx5_ib_alloc_mr(pd, IB_MR_TYPE_INTEGRITY, max_num_sg, + max_num_meta_sg); +} + struct ib_mw *mlx5_ib_alloc_mw(struct ib_pd *pd, enum ib_mw_type type, struct ib_udata *udata) { @@ -1887,16 +2026,53 @@ done: } static int +mlx5_ib_map_pa_mr_sg_pi(struct ib_mr *ibmr, struct scatterlist *data_sg, + int data_sg_nents, unsigned int *data_sg_offset, + struct scatterlist *meta_sg, int meta_sg_nents, + unsigned int *meta_sg_offset) +{ + struct mlx5_ib_mr *mr = to_mmr(ibmr); + unsigned int sg_offset = 0; + int n = 0; + + mr->meta_length = 0; + if (data_sg_nents == 1) { + n++; + mr->ndescs = 1; + if (data_sg_offset) + sg_offset = *data_sg_offset; + mr->data_length = sg_dma_len(data_sg) - sg_offset; + mr->data_iova = sg_dma_address(data_sg) + sg_offset; + if (meta_sg_nents == 1) { + n++; + mr->meta_ndescs = 1; + if (meta_sg_offset) + sg_offset = *meta_sg_offset; + else + sg_offset = 0; + mr->meta_length = sg_dma_len(meta_sg) - sg_offset; + mr->pi_iova = sg_dma_address(meta_sg) + sg_offset; + } + ibmr->length = mr->data_length + mr->meta_length; + } + + return n; +} + +static int mlx5_ib_sg_to_klms(struct mlx5_ib_mr *mr, struct scatterlist *sgl, unsigned short sg_nents, - unsigned int *sg_offset_p) + unsigned int *sg_offset_p, + struct scatterlist *meta_sgl, + unsigned short meta_sg_nents, + unsigned int *meta_sg_offset_p) { struct scatterlist *sg = sgl; struct mlx5_klm *klms = mr->descs; unsigned int sg_offset = sg_offset_p ? *sg_offset_p : 0; u32 lkey = mr->ibmr.pd->local_dma_lkey; - int i; + int i, j = 0; mr->ibmr.iova = sg_dma_address(sg) + sg_offset; mr->ibmr.length = 0; @@ -1911,12 +2087,36 @@ mlx5_ib_sg_to_klms(struct mlx5_ib_mr *mr, sg_offset = 0; } - mr->ndescs = i; if (sg_offset_p) *sg_offset_p = sg_offset; - return i; + mr->ndescs = i; + mr->data_length = mr->ibmr.length; + + if (meta_sg_nents) { + sg = meta_sgl; + sg_offset = meta_sg_offset_p ? *meta_sg_offset_p : 0; + for_each_sg(meta_sgl, sg, meta_sg_nents, j) { + if (unlikely(i + j >= mr->max_descs)) + break; + klms[i + j].va = cpu_to_be64(sg_dma_address(sg) + + sg_offset); + klms[i + j].bcount = cpu_to_be32(sg_dma_len(sg) - + sg_offset); + klms[i + j].key = cpu_to_be32(lkey); + mr->ibmr.length += sg_dma_len(sg) - sg_offset; + + sg_offset = 0; + } + if (meta_sg_offset_p) + *meta_sg_offset_p = sg_offset; + + mr->meta_ndescs = j; + mr->meta_length = mr->ibmr.length - mr->data_length; + } + + return i + j; } static int mlx5_set_page(struct ib_mr *ibmr, u64 addr) @@ -1933,6 +2133,181 @@ static int mlx5_set_page(struct ib_mr *ibmr, u64 addr) return 0; } +static int mlx5_set_page_pi(struct ib_mr *ibmr, u64 addr) +{ + struct mlx5_ib_mr *mr = to_mmr(ibmr); + __be64 *descs; + + if (unlikely(mr->ndescs + mr->meta_ndescs == mr->max_descs)) + return -ENOMEM; + + descs = mr->descs; + descs[mr->ndescs + mr->meta_ndescs++] = + cpu_to_be64(addr | MLX5_EN_RD | MLX5_EN_WR); + + return 0; +} + +static int +mlx5_ib_map_mtt_mr_sg_pi(struct ib_mr *ibmr, struct scatterlist *data_sg, + int data_sg_nents, unsigned int *data_sg_offset, + struct scatterlist *meta_sg, int meta_sg_nents, + unsigned int *meta_sg_offset) +{ + struct mlx5_ib_mr *mr = to_mmr(ibmr); + struct mlx5_ib_mr *pi_mr = mr->mtt_mr; + int n; + + pi_mr->ndescs = 0; + pi_mr->meta_ndescs = 0; + pi_mr->meta_length = 0; + + ib_dma_sync_single_for_cpu(ibmr->device, pi_mr->desc_map, + pi_mr->desc_size * pi_mr->max_descs, + DMA_TO_DEVICE); + + pi_mr->ibmr.page_size = ibmr->page_size; + n = ib_sg_to_pages(&pi_mr->ibmr, data_sg, data_sg_nents, data_sg_offset, + mlx5_set_page); + if (n != data_sg_nents) + return n; + + pi_mr->data_iova = pi_mr->ibmr.iova; + pi_mr->data_length = pi_mr->ibmr.length; + pi_mr->ibmr.length = pi_mr->data_length; + ibmr->length = pi_mr->data_length; + + if (meta_sg_nents) { + u64 page_mask = ~((u64)ibmr->page_size - 1); + u64 iova = pi_mr->data_iova; + + n += ib_sg_to_pages(&pi_mr->ibmr, meta_sg, meta_sg_nents, + meta_sg_offset, mlx5_set_page_pi); + + pi_mr->meta_length = pi_mr->ibmr.length; + /* + * PI address for the HW is the offset of the metadata address + * relative to the first data page address. + * It equals to first data page address + size of data pages + + * metadata offset at the first metadata page + */ + pi_mr->pi_iova = (iova & page_mask) + + pi_mr->ndescs * ibmr->page_size + + (pi_mr->ibmr.iova & ~page_mask); + /* + * In order to use one MTT MR for data and metadata, we register + * also the gaps between the end of the data and the start of + * the metadata (the sig MR will verify that the HW will access + * to right addresses). This mapping is safe because we use + * internal mkey for the registration. + */ + pi_mr->ibmr.length = pi_mr->pi_iova + pi_mr->meta_length - iova; + pi_mr->ibmr.iova = iova; + ibmr->length += pi_mr->meta_length; + } + + ib_dma_sync_single_for_device(ibmr->device, pi_mr->desc_map, + pi_mr->desc_size * pi_mr->max_descs, + DMA_TO_DEVICE); + + return n; +} + +static int +mlx5_ib_map_klm_mr_sg_pi(struct ib_mr *ibmr, struct scatterlist *data_sg, + int data_sg_nents, unsigned int *data_sg_offset, + struct scatterlist *meta_sg, int meta_sg_nents, + unsigned int *meta_sg_offset) +{ + struct mlx5_ib_mr *mr = to_mmr(ibmr); + struct mlx5_ib_mr *pi_mr = mr->klm_mr; + int n; + + pi_mr->ndescs = 0; + pi_mr->meta_ndescs = 0; + pi_mr->meta_length = 0; + + ib_dma_sync_single_for_cpu(ibmr->device, pi_mr->desc_map, + pi_mr->desc_size * pi_mr->max_descs, + DMA_TO_DEVICE); + + n = mlx5_ib_sg_to_klms(pi_mr, data_sg, data_sg_nents, data_sg_offset, + meta_sg, meta_sg_nents, meta_sg_offset); + + ib_dma_sync_single_for_device(ibmr->device, pi_mr->desc_map, + pi_mr->desc_size * pi_mr->max_descs, + DMA_TO_DEVICE); + + /* This is zero-based memory region */ + pi_mr->data_iova = 0; + pi_mr->ibmr.iova = 0; + pi_mr->pi_iova = pi_mr->data_length; + ibmr->length = pi_mr->ibmr.length; + + return n; +} + +int mlx5_ib_map_mr_sg_pi(struct ib_mr *ibmr, struct scatterlist *data_sg, + int data_sg_nents, unsigned int *data_sg_offset, + struct scatterlist *meta_sg, int meta_sg_nents, + unsigned int *meta_sg_offset) +{ + struct mlx5_ib_mr *mr = to_mmr(ibmr); + struct mlx5_ib_mr *pi_mr = NULL; + int n; + + WARN_ON(ibmr->type != IB_MR_TYPE_INTEGRITY); + + mr->ndescs = 0; + mr->data_length = 0; + mr->data_iova = 0; + mr->meta_ndescs = 0; + mr->pi_iova = 0; + /* + * As a performance optimization, if possible, there is no need to + * perform UMR operation to register the data/metadata buffers. + * First try to map the sg lists to PA descriptors with local_dma_lkey. + * Fallback to UMR only in case of a failure. + */ + n = mlx5_ib_map_pa_mr_sg_pi(ibmr, data_sg, data_sg_nents, + data_sg_offset, meta_sg, meta_sg_nents, + meta_sg_offset); + if (n == data_sg_nents + meta_sg_nents) + goto out; + /* + * As a performance optimization, if possible, there is no need to map + * the sg lists to KLM descriptors. First try to map the sg lists to MTT + * descriptors and fallback to KLM only in case of a failure. + * It's more efficient for the HW to work with MTT descriptors + * (especially in high load). + * Use KLM (indirect access) only if it's mandatory. + */ + pi_mr = mr->mtt_mr; + n = mlx5_ib_map_mtt_mr_sg_pi(ibmr, data_sg, data_sg_nents, + data_sg_offset, meta_sg, meta_sg_nents, + meta_sg_offset); + if (n == data_sg_nents + meta_sg_nents) + goto out; + + pi_mr = mr->klm_mr; + n = mlx5_ib_map_klm_mr_sg_pi(ibmr, data_sg, data_sg_nents, + data_sg_offset, meta_sg, meta_sg_nents, + meta_sg_offset); + if (unlikely(n != data_sg_nents + meta_sg_nents)) + return -ENOMEM; + +out: + /* This is zero-based memory region */ + ibmr->iova = 0; + mr->pi_mr = pi_mr; + if (pi_mr) + ibmr->sig_attrs->meta_length = pi_mr->meta_length; + else + ibmr->sig_attrs->meta_length = mr->meta_length; + + return 0; +} + int mlx5_ib_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents, unsigned int *sg_offset) { @@ -1946,7 +2321,8 @@ int mlx5_ib_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents, DMA_TO_DEVICE); if (mr->access_mode == MLX5_MKC_ACCESS_MODE_KLMS) - n = mlx5_ib_sg_to_klms(mr, sg, sg_nents, sg_offset); + n = mlx5_ib_sg_to_klms(mr, sg, sg_nents, sg_offset, NULL, 0, + NULL); else n = ib_sg_to_pages(ibmr, sg, sg_nents, sg_offset, mlx5_set_page); diff --git a/drivers/infiniband/hw/mlx5/odp.c b/drivers/infiniband/hw/mlx5/odp.c index 831c450b271a..5b642d81e617 100644 --- a/drivers/infiniband/hw/mlx5/odp.c +++ b/drivers/infiniband/hw/mlx5/odp.c @@ -150,7 +150,7 @@ static struct ib_umem_odp *odp_lookup(u64 start, u64 length, if (!rb) goto not_found; odp = rb_entry(rb, struct ib_umem_odp, interval_tree.rb); - if (ib_umem_start(&odp->umem) > start + length) + if (ib_umem_start(odp) > start + length) goto not_found; } not_found: @@ -200,7 +200,7 @@ void mlx5_odp_populate_klm(struct mlx5_klm *pklm, size_t offset, static void mr_leaf_free_action(struct work_struct *work) { struct ib_umem_odp *odp = container_of(work, struct ib_umem_odp, work); - int idx = ib_umem_start(&odp->umem) >> MLX5_IMR_MTT_SHIFT; + int idx = ib_umem_start(odp) >> MLX5_IMR_MTT_SHIFT; struct mlx5_ib_mr *mr = odp->private, *imr = mr->parent; mr->parent = NULL; @@ -224,7 +224,6 @@ void mlx5_ib_invalidate_range(struct ib_umem_odp *umem_odp, unsigned long start, const u64 umr_block_mask = (MLX5_UMR_MTT_ALIGNMENT / sizeof(struct mlx5_mtt)) - 1; u64 idx = 0, blk_start_idx = 0; - struct ib_umem *umem; int in_block = 0; u64 addr; @@ -232,15 +231,14 @@ void mlx5_ib_invalidate_range(struct ib_umem_odp *umem_odp, unsigned long start, pr_err("invalidation called on NULL umem or non-ODP umem\n"); return; } - umem = &umem_odp->umem; mr = umem_odp->private; if (!mr || !mr->ibmr.pd) return; - start = max_t(u64, ib_umem_start(umem), start); - end = min_t(u64, ib_umem_end(umem), end); + start = max_t(u64, ib_umem_start(umem_odp), start); + end = min_t(u64, ib_umem_end(umem_odp), end); /* * Iteration one - zap the HW's MTTs. The notifiers_count ensures that @@ -249,8 +247,8 @@ void mlx5_ib_invalidate_range(struct ib_umem_odp *umem_odp, unsigned long start, * but they will write 0s as well, so no difference in the end result. */ - for (addr = start; addr < end; addr += BIT(umem->page_shift)) { - idx = (addr - ib_umem_start(umem)) >> umem->page_shift; + for (addr = start; addr < end; addr += BIT(umem_odp->page_shift)) { + idx = (addr - ib_umem_start(umem_odp)) >> umem_odp->page_shift; /* * Strive to write the MTTs in chunks, but avoid overwriting * non-existing MTTs. The huristic here can be improved to @@ -544,13 +542,12 @@ static int mr_leaf_free(struct ib_umem_odp *umem_odp, u64 start, u64 end, void *cookie) { struct mlx5_ib_mr *mr = umem_odp->private, *imr = cookie; - struct ib_umem *umem = &umem_odp->umem; if (mr->parent != imr) return 0; - ib_umem_odp_unmap_dma_pages(umem_odp, ib_umem_start(umem), - ib_umem_end(umem)); + ib_umem_odp_unmap_dma_pages(umem_odp, ib_umem_start(umem_odp), + ib_umem_end(umem_odp)); if (umem_odp->dying) return 0; @@ -602,9 +599,9 @@ static int pagefault_mr(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr, } next_mr: - size = min_t(size_t, bcnt, ib_umem_end(&odp->umem) - io_virt); + size = min_t(size_t, bcnt, ib_umem_end(odp) - io_virt); - page_shift = mr->umem->page_shift; + page_shift = odp->page_shift; page_mask = ~(BIT(page_shift) - 1); start_idx = (io_virt - (mr->mmkey.iova & page_mask)) >> page_shift; access_mask = ODP_READ_ALLOWED_BIT; diff --git a/drivers/infiniband/hw/mlx5/qp.c b/drivers/infiniband/hw/mlx5/qp.c index 768c7e81f688..2a97619ed603 100644 --- a/drivers/infiniband/hw/mlx5/qp.c +++ b/drivers/infiniband/hw/mlx5/qp.c @@ -34,6 +34,7 @@ #include <rdma/ib_umem.h> #include <rdma/ib_cache.h> #include <rdma/ib_user_verbs.h> +#include <rdma/rdma_counter.h> #include <linux/mlx5/fs.h> #include "mlx5_ib.h" #include "ib_rep.h" @@ -442,9 +443,9 @@ static int calc_send_wqe(struct ib_qp_init_attr *attr) } size += attr->cap.max_send_sge * sizeof(struct mlx5_wqe_data_seg); - if (attr->create_flags & IB_QP_CREATE_SIGNATURE_EN && + if (attr->create_flags & IB_QP_CREATE_INTEGRITY_EN && ALIGN(max_t(int, inl_size, size), MLX5_SEND_WQE_BB) < MLX5_SIG_WQE_SIZE) - return MLX5_SIG_WQE_SIZE; + return MLX5_SIG_WQE_SIZE; else return ALIGN(max_t(int, inl_size, size), MLX5_SEND_WQE_BB); } @@ -496,9 +497,6 @@ static int calc_sq_size(struct mlx5_ib_dev *dev, struct ib_qp_init_attr *attr, sizeof(struct mlx5_wqe_inline_seg); attr->cap.max_inline_data = qp->max_inline_data; - if (attr->create_flags & IB_QP_CREATE_SIGNATURE_EN) - qp->signature_en = true; - wq_size = roundup_pow_of_two(attr->cap.max_send_wr * wqe_size); qp->sq.wqe_cnt = wq_size / MLX5_SEND_WQE_BB; if (qp->sq.wqe_cnt > (1 << MLX5_CAP_GEN(dev->mdev, log_max_qp_sz))) { @@ -790,8 +788,7 @@ static void destroy_user_rq(struct mlx5_ib_dev *dev, struct ib_pd *pd, atomic_dec(&dev->delay_drop.rqs_cnt); mlx5_ib_db_unmap_user(context, &rwq->db); - if (rwq->umem) - ib_umem_release(rwq->umem); + ib_umem_release(rwq->umem); } static int create_user_rq(struct mlx5_ib_dev *dev, struct ib_pd *pd, @@ -977,8 +974,7 @@ err_free: kvfree(*in); err_umem: - if (ubuffer->umem) - ib_umem_release(ubuffer->umem); + ib_umem_release(ubuffer->umem); err_bfreg: if (bfregn != MLX5_IB_INVALID_BFREG) @@ -997,8 +993,7 @@ static void destroy_qp_user(struct mlx5_ib_dev *dev, struct ib_pd *pd, ibucontext); mlx5_ib_db_unmap_user(context, &qp->db); - if (base->ubuffer.umem) - ib_umem_release(base->ubuffer.umem); + ib_umem_release(base->ubuffer.umem); /* * Free only the BFREGs which are handled by the kernel. @@ -1042,7 +1037,7 @@ static int create_kernel_qp(struct mlx5_ib_dev *dev, void *qpc; int err; - if (init_attr->create_flags & ~(IB_QP_CREATE_SIGNATURE_EN | + if (init_attr->create_flags & ~(IB_QP_CREATE_INTEGRITY_EN | IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK | IB_QP_CREATE_IPOIB_UD_LSO | IB_QP_CREATE_NETIF_QP | @@ -3386,6 +3381,35 @@ static unsigned int get_tx_affinity(struct mlx5_ib_dev *dev, return tx_port_affinity; } +static int __mlx5_ib_qp_set_counter(struct ib_qp *qp, + struct rdma_counter *counter) +{ + struct mlx5_ib_dev *dev = to_mdev(qp->device); + struct mlx5_ib_qp *mqp = to_mqp(qp); + struct mlx5_qp_context context = {}; + struct mlx5_ib_port *mibport = NULL; + struct mlx5_ib_qp_base *base; + u32 set_id; + + if (!MLX5_CAP_GEN(dev->mdev, rts2rts_qp_counters_set_id)) + return 0; + + if (counter) { + set_id = counter->id; + } else { + mibport = &dev->port[mqp->port - 1]; + set_id = mibport->cnts.set_id; + } + + base = &mqp->trans_qp.base; + context.qp_counter_set_usr_page &= cpu_to_be32(0xffffff); + context.qp_counter_set_usr_page |= cpu_to_be32(set_id << 24); + return mlx5_core_qp_modify(dev->mdev, + MLX5_CMD_OP_RTS2RTS_QP, + MLX5_QP_OPTPAR_COUNTER_SET_ID, + &context, &base->mqp); +} + static int __mlx5_ib_modify_qp(struct ib_qp *ibqp, const struct ib_qp_attr *attr, int attr_mask, enum ib_qp_state cur_state, @@ -3439,6 +3463,7 @@ static int __mlx5_ib_modify_qp(struct ib_qp *ibqp, struct mlx5_ib_port *mibport = NULL; enum mlx5_qp_state mlx5_cur, mlx5_new; enum mlx5_qp_optpar optpar; + u32 set_id = 0; int mlx5_st; int err; u16 op; @@ -3601,8 +3626,12 @@ static int __mlx5_ib_modify_qp(struct ib_qp *ibqp, port_num = 0; mibport = &dev->port[port_num]; + if (ibqp->counter) + set_id = ibqp->counter->id; + else + set_id = mibport->cnts.set_id; context->qp_counter_set_usr_page |= - cpu_to_be32((u32)(mibport->cnts.set_id) << 24); + cpu_to_be32(set_id << 24); } if (!ibqp->uobject && cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT) @@ -3630,7 +3659,7 @@ static int __mlx5_ib_modify_qp(struct ib_qp *ibqp, raw_qp_param.operation = op; if (cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT) { - raw_qp_param.rq_q_ctr_id = mibport->cnts.set_id; + raw_qp_param.rq_q_ctr_id = set_id; raw_qp_param.set_mask |= MLX5_RAW_QP_MOD_SET_RQ_Q_CTR_ID; } @@ -3707,6 +3736,12 @@ static int __mlx5_ib_modify_qp(struct ib_qp *ibqp, qp->db.db[MLX5_SND_DBR] = 0; } + if ((new_state == IB_QPS_RTS) && qp->counter_pending) { + err = __mlx5_ib_qp_set_counter(ibqp, ibqp->counter); + if (!err) + qp->counter_pending = 0; + } + out: kfree(context); return err; @@ -4170,15 +4205,13 @@ static __be64 sig_mkey_mask(void) } static void set_reg_umr_seg(struct mlx5_wqe_umr_ctrl_seg *umr, - struct mlx5_ib_mr *mr, bool umr_inline) + struct mlx5_ib_mr *mr, u8 flags) { - int size = mr->ndescs * mr->desc_size; + int size = (mr->ndescs + mr->meta_ndescs) * mr->desc_size; memset(umr, 0, sizeof(*umr)); - umr->flags = MLX5_UMR_CHECK_NOT_FREE; - if (umr_inline) - umr->flags |= MLX5_UMR_INLINE; + umr->flags = flags; umr->xlt_octowords = cpu_to_be16(get_xlt_octo(size)); umr->mkey_mask = frwr_mkey_mask(); } @@ -4305,7 +4338,7 @@ static void set_reg_mkey_seg(struct mlx5_mkey_seg *seg, struct mlx5_ib_mr *mr, u32 key, int access) { - int ndescs = ALIGN(mr->ndescs, 8) >> 1; + int ndescs = ALIGN(mr->ndescs + mr->meta_ndescs, 8) >> 1; memset(seg, 0, sizeof(*seg)); @@ -4356,7 +4389,7 @@ static void set_reg_data_seg(struct mlx5_wqe_data_seg *dseg, struct mlx5_ib_mr *mr, struct mlx5_ib_pd *pd) { - int bcount = mr->desc_size * mr->ndescs; + int bcount = mr->desc_size * (mr->ndescs + mr->meta_ndescs); dseg->addr = cpu_to_be64(mr->desc_map); dseg->byte_count = cpu_to_be32(ALIGN(bcount, 64)); @@ -4549,23 +4582,37 @@ static int mlx5_set_bsf(struct ib_mr *sig_mr, return 0; } -static int set_sig_data_segment(const struct ib_sig_handover_wr *wr, - struct mlx5_ib_qp *qp, void **seg, - int *size, void **cur_edge) +static int set_sig_data_segment(const struct ib_send_wr *send_wr, + struct ib_mr *sig_mr, + struct ib_sig_attrs *sig_attrs, + struct mlx5_ib_qp *qp, void **seg, int *size, + void **cur_edge) { - struct ib_sig_attrs *sig_attrs = wr->sig_attrs; - struct ib_mr *sig_mr = wr->sig_mr; struct mlx5_bsf *bsf; - u32 data_len = wr->wr.sg_list->length; - u32 data_key = wr->wr.sg_list->lkey; - u64 data_va = wr->wr.sg_list->addr; + u32 data_len; + u32 data_key; + u64 data_va; + u32 prot_len = 0; + u32 prot_key = 0; + u64 prot_va = 0; + bool prot = false; int ret; int wqe_size; + struct mlx5_ib_mr *mr = to_mmr(sig_mr); + struct mlx5_ib_mr *pi_mr = mr->pi_mr; + + data_len = pi_mr->data_length; + data_key = pi_mr->ibmr.lkey; + data_va = pi_mr->data_iova; + if (pi_mr->meta_ndescs) { + prot_len = pi_mr->meta_length; + prot_key = pi_mr->ibmr.lkey; + prot_va = pi_mr->pi_iova; + prot = true; + } - if (!wr->prot || - (data_key == wr->prot->lkey && - data_va == wr->prot->addr && - data_len == wr->prot->length)) { + if (!prot || (data_key == prot_key && data_va == prot_va && + data_len == prot_len)) { /** * Source domain doesn't contain signature information * or data and protection are interleaved in memory. @@ -4599,8 +4646,6 @@ static int set_sig_data_segment(const struct ib_sig_handover_wr *wr, struct mlx5_stride_block_ctrl_seg *sblock_ctrl; struct mlx5_stride_block_entry *data_sentry; struct mlx5_stride_block_entry *prot_sentry; - u32 prot_key = wr->prot->lkey; - u64 prot_va = wr->prot->addr; u16 block_size = sig_attrs->mem.sig.dif.pi_interval; int prot_size; @@ -4650,17 +4695,15 @@ static int set_sig_data_segment(const struct ib_sig_handover_wr *wr, } static void set_sig_mkey_segment(struct mlx5_mkey_seg *seg, - const struct ib_sig_handover_wr *wr, u32 size, - u32 length, u32 pdn) + struct ib_mr *sig_mr, int access_flags, + u32 size, u32 length, u32 pdn) { - struct ib_mr *sig_mr = wr->sig_mr; u32 sig_key = sig_mr->rkey; u8 sigerr = to_mmr(sig_mr)->sig->sigerr_count & 1; memset(seg, 0, sizeof(*seg)); - seg->flags = get_umr_flags(wr->access_flags) | - MLX5_MKC_ACCESS_MODE_KLMS; + seg->flags = get_umr_flags(access_flags) | MLX5_MKC_ACCESS_MODE_KLMS; seg->qpn_mkey7_0 = cpu_to_be32((sig_key & 0xff) | 0xffffff00); seg->flags_pd = cpu_to_be32(MLX5_MKEY_REMOTE_INVAL | sigerr << 26 | MLX5_MKEY_BSF_EN | pdn); @@ -4680,49 +4723,50 @@ static void set_sig_umr_segment(struct mlx5_wqe_umr_ctrl_seg *umr, umr->mkey_mask = sig_mkey_mask(); } - -static int set_sig_umr_wr(const struct ib_send_wr *send_wr, - struct mlx5_ib_qp *qp, void **seg, int *size, - void **cur_edge) +static int set_pi_umr_wr(const struct ib_send_wr *send_wr, + struct mlx5_ib_qp *qp, void **seg, int *size, + void **cur_edge) { - const struct ib_sig_handover_wr *wr = sig_handover_wr(send_wr); - struct mlx5_ib_mr *sig_mr = to_mmr(wr->sig_mr); + const struct ib_reg_wr *wr = reg_wr(send_wr); + struct mlx5_ib_mr *sig_mr = to_mmr(wr->mr); + struct mlx5_ib_mr *pi_mr = sig_mr->pi_mr; + struct ib_sig_attrs *sig_attrs = sig_mr->ibmr.sig_attrs; u32 pdn = get_pd(qp)->pdn; u32 xlt_size; int region_len, ret; - if (unlikely(wr->wr.num_sge != 1) || - unlikely(wr->access_flags & IB_ACCESS_REMOTE_ATOMIC) || - unlikely(!sig_mr->sig) || unlikely(!qp->signature_en) || + if (unlikely(send_wr->num_sge != 0) || + unlikely(wr->access & IB_ACCESS_REMOTE_ATOMIC) || + unlikely(!sig_mr->sig) || unlikely(!qp->ibqp.integrity_en) || unlikely(!sig_mr->sig->sig_status_checked)) return -EINVAL; /* length of the protected region, data + protection */ - region_len = wr->wr.sg_list->length; - if (wr->prot && - (wr->prot->lkey != wr->wr.sg_list->lkey || - wr->prot->addr != wr->wr.sg_list->addr || - wr->prot->length != wr->wr.sg_list->length)) - region_len += wr->prot->length; + region_len = pi_mr->ibmr.length; /** * KLM octoword size - if protection was provided * then we use strided block format (3 octowords), * else we use single KLM (1 octoword) **/ - xlt_size = wr->prot ? 0x30 : sizeof(struct mlx5_klm); + if (sig_attrs->mem.sig_type != IB_SIG_TYPE_NONE) + xlt_size = 0x30; + else + xlt_size = sizeof(struct mlx5_klm); set_sig_umr_segment(*seg, xlt_size); *seg += sizeof(struct mlx5_wqe_umr_ctrl_seg); *size += sizeof(struct mlx5_wqe_umr_ctrl_seg) / 16; handle_post_send_edge(&qp->sq, seg, *size, cur_edge); - set_sig_mkey_segment(*seg, wr, xlt_size, region_len, pdn); + set_sig_mkey_segment(*seg, wr->mr, wr->access, xlt_size, region_len, + pdn); *seg += sizeof(struct mlx5_mkey_seg); *size += sizeof(struct mlx5_mkey_seg) / 16; handle_post_send_edge(&qp->sq, seg, *size, cur_edge); - ret = set_sig_data_segment(wr, qp, seg, size, cur_edge); + ret = set_sig_data_segment(send_wr, wr->mr, sig_attrs, qp, seg, size, + cur_edge); if (ret) return ret; @@ -4759,12 +4803,14 @@ static int set_psv_wr(struct ib_sig_domain *domain, static int set_reg_wr(struct mlx5_ib_qp *qp, const struct ib_reg_wr *wr, - void **seg, int *size, void **cur_edge) + void **seg, int *size, void **cur_edge, + bool check_not_free) { struct mlx5_ib_mr *mr = to_mmr(wr->mr); struct mlx5_ib_pd *pd = to_mpd(qp->ibqp.pd); - size_t mr_list_size = mr->ndescs * mr->desc_size; + int mr_list_size = (mr->ndescs + mr->meta_ndescs) * mr->desc_size; bool umr_inline = mr_list_size <= MLX5_IB_SQ_UMR_INLINE_THRESHOLD; + u8 flags = 0; if (unlikely(wr->wr.send_flags & IB_SEND_INLINE)) { mlx5_ib_warn(to_mdev(qp->ibqp.device), @@ -4772,7 +4818,12 @@ static int set_reg_wr(struct mlx5_ib_qp *qp, return -EINVAL; } - set_reg_umr_seg(*seg, mr, umr_inline); + if (check_not_free) + flags |= MLX5_UMR_CHECK_NOT_FREE; + if (umr_inline) + flags |= MLX5_UMR_INLINE; + + set_reg_umr_seg(*seg, mr, flags); *seg += sizeof(struct mlx5_wqe_umr_ctrl_seg); *size += sizeof(struct mlx5_wqe_umr_ctrl_seg) / 16; handle_post_send_edge(&qp->sq, seg, *size, cur_edge); @@ -4898,8 +4949,12 @@ static int _mlx5_ib_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr, struct mlx5_wqe_ctrl_seg *ctrl = NULL; /* compiler warning */ struct mlx5_ib_dev *dev = to_mdev(ibqp->device); struct mlx5_core_dev *mdev = dev->mdev; + struct ib_reg_wr reg_pi_wr; struct mlx5_ib_qp *qp; struct mlx5_ib_mr *mr; + struct mlx5_ib_mr *pi_mr; + struct mlx5_ib_mr pa_pi_mr; + struct ib_sig_attrs *sig_attrs; struct mlx5_wqe_xrc_seg *xrc; struct mlx5_bf *bf; void *cur_edge; @@ -4953,7 +5008,8 @@ static int _mlx5_ib_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr, goto out; } - if (wr->opcode == IB_WR_REG_MR) { + if (wr->opcode == IB_WR_REG_MR || + wr->opcode == IB_WR_REG_MR_INTEGRITY) { fence = dev->umr_fence; next_fence = MLX5_FENCE_MODE_INITIATOR_SMALL; } else { @@ -5003,7 +5059,7 @@ static int _mlx5_ib_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr, qp->sq.wr_data[idx] = IB_WR_REG_MR; ctrl->imm = cpu_to_be32(reg_wr(wr)->key); err = set_reg_wr(qp, reg_wr(wr), &seg, &size, - &cur_edge); + &cur_edge, true); if (err) { *bad_wr = wr; goto out; @@ -5011,26 +5067,82 @@ static int _mlx5_ib_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr, num_sge = 0; break; - case IB_WR_REG_SIG_MR: - qp->sq.wr_data[idx] = IB_WR_REG_SIG_MR; - mr = to_mmr(sig_handover_wr(wr)->sig_mr); - + case IB_WR_REG_MR_INTEGRITY: + qp->sq.wr_data[idx] = IB_WR_REG_MR_INTEGRITY; + + mr = to_mmr(reg_wr(wr)->mr); + pi_mr = mr->pi_mr; + + if (pi_mr) { + memset(®_pi_wr, 0, + sizeof(struct ib_reg_wr)); + + reg_pi_wr.mr = &pi_mr->ibmr; + reg_pi_wr.access = reg_wr(wr)->access; + reg_pi_wr.key = pi_mr->ibmr.rkey; + + ctrl->imm = cpu_to_be32(reg_pi_wr.key); + /* UMR for data + prot registration */ + err = set_reg_wr(qp, ®_pi_wr, &seg, + &size, &cur_edge, + false); + if (err) { + *bad_wr = wr; + goto out; + } + finish_wqe(qp, ctrl, seg, size, + cur_edge, idx, wr->wr_id, + nreq, fence, + MLX5_OPCODE_UMR); + + err = begin_wqe(qp, &seg, &ctrl, wr, + &idx, &size, &cur_edge, + nreq); + if (err) { + mlx5_ib_warn(dev, "\n"); + err = -ENOMEM; + *bad_wr = wr; + goto out; + } + } else { + memset(&pa_pi_mr, 0, + sizeof(struct mlx5_ib_mr)); + /* No UMR, use local_dma_lkey */ + pa_pi_mr.ibmr.lkey = + mr->ibmr.pd->local_dma_lkey; + + pa_pi_mr.ndescs = mr->ndescs; + pa_pi_mr.data_length = mr->data_length; + pa_pi_mr.data_iova = mr->data_iova; + if (mr->meta_ndescs) { + pa_pi_mr.meta_ndescs = + mr->meta_ndescs; + pa_pi_mr.meta_length = + mr->meta_length; + pa_pi_mr.pi_iova = mr->pi_iova; + } + + pa_pi_mr.ibmr.length = mr->ibmr.length; + mr->pi_mr = &pa_pi_mr; + } ctrl->imm = cpu_to_be32(mr->ibmr.rkey); - err = set_sig_umr_wr(wr, qp, &seg, &size, - &cur_edge); + /* UMR for sig MR */ + err = set_pi_umr_wr(wr, qp, &seg, &size, + &cur_edge); if (err) { mlx5_ib_warn(dev, "\n"); *bad_wr = wr; goto out; } - finish_wqe(qp, ctrl, seg, size, cur_edge, idx, wr->wr_id, nreq, fence, MLX5_OPCODE_UMR); + /* * SET_PSV WQEs are not signaled and solicited * on error */ + sig_attrs = mr->ibmr.sig_attrs; err = __begin_wqe(qp, &seg, &ctrl, wr, &idx, &size, &cur_edge, nreq, false, true); @@ -5040,19 +5152,18 @@ static int _mlx5_ib_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr, *bad_wr = wr; goto out; } - - err = set_psv_wr(&sig_handover_wr(wr)->sig_attrs->mem, - mr->sig->psv_memory.psv_idx, &seg, - &size); + err = set_psv_wr(&sig_attrs->mem, + mr->sig->psv_memory.psv_idx, + &seg, &size); if (err) { mlx5_ib_warn(dev, "\n"); *bad_wr = wr; goto out; } - finish_wqe(qp, ctrl, seg, size, cur_edge, idx, - wr->wr_id, nreq, fence, + wr->wr_id, nreq, next_fence, MLX5_OPCODE_SET_PSV); + err = __begin_wqe(qp, &seg, &ctrl, wr, &idx, &size, &cur_edge, nreq, false, true); @@ -5062,20 +5173,20 @@ static int _mlx5_ib_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr, *bad_wr = wr; goto out; } - - err = set_psv_wr(&sig_handover_wr(wr)->sig_attrs->wire, - mr->sig->psv_wire.psv_idx, &seg, - &size); + err = set_psv_wr(&sig_attrs->wire, + mr->sig->psv_wire.psv_idx, + &seg, &size); if (err) { mlx5_ib_warn(dev, "\n"); *bad_wr = wr; goto out; } - finish_wqe(qp, ctrl, seg, size, cur_edge, idx, - wr->wr_id, nreq, fence, + wr->wr_id, nreq, next_fence, MLX5_OPCODE_SET_PSV); - qp->next_fence = MLX5_FENCE_MODE_INITIATOR_SMALL; + + qp->next_fence = + MLX5_FENCE_MODE_INITIATOR_SMALL; num_sge = 0; goto skip_psv; @@ -6047,7 +6158,7 @@ err: return ERR_PTR(err); } -int mlx5_ib_destroy_wq(struct ib_wq *wq, struct ib_udata *udata) +void mlx5_ib_destroy_wq(struct ib_wq *wq, struct ib_udata *udata) { struct mlx5_ib_dev *dev = to_mdev(wq->device); struct mlx5_ib_rwq *rwq = to_mrwq(wq); @@ -6055,8 +6166,6 @@ int mlx5_ib_destroy_wq(struct ib_wq *wq, struct ib_udata *udata) mlx5_core_destroy_rq_tracked(dev->mdev, &rwq->core_qp); destroy_user_rq(dev, wq->pd, rwq, udata); kfree(rwq); - - return 0; } struct ib_rwq_ind_table *mlx5_ib_create_rwq_ind_table(struct ib_device *device, @@ -6367,3 +6476,34 @@ void mlx5_ib_drain_rq(struct ib_qp *qp) handle_drain_completion(cq, &rdrain, dev); } + +/** + * Bind a qp to a counter. If @counter is NULL then bind the qp to + * the default counter + */ +int mlx5_ib_qp_set_counter(struct ib_qp *qp, struct rdma_counter *counter) +{ + struct mlx5_ib_qp *mqp = to_mqp(qp); + int err = 0; + + mutex_lock(&mqp->mutex); + if (mqp->state == IB_QPS_RESET) { + qp->counter = counter; + goto out; + } + + if (mqp->state == IB_QPS_RTS) { + err = __mlx5_ib_qp_set_counter(qp, counter); + if (!err) + qp->counter = counter; + + goto out; + } + + mqp->counter_pending = 1; + qp->counter = counter; + +out: + mutex_unlock(&mqp->mutex); + return err; +} diff --git a/drivers/infiniband/hw/mthca/mthca_allocator.c b/drivers/infiniband/hw/mthca/mthca_allocator.c index aaf10dd5364d..aef1d274a14e 100644 --- a/drivers/infiniband/hw/mthca/mthca_allocator.c +++ b/drivers/infiniband/hw/mthca/mthca_allocator.c @@ -214,8 +214,6 @@ int mthca_buf_alloc(struct mthca_dev *dev, int size, int max_direct, dma_unmap_addr_set(&buf->direct, mapping, t); - memset(buf->direct.buf, 0, size); - while (t & ((1 << shift) - 1)) { --shift; npages *= 2; diff --git a/drivers/infiniband/hw/mthca/mthca_memfree.c b/drivers/infiniband/hw/mthca/mthca_memfree.c index 8ff0e90d7564..edccfd6e178f 100644 --- a/drivers/infiniband/hw/mthca/mthca_memfree.c +++ b/drivers/infiniband/hw/mthca/mthca_memfree.c @@ -482,7 +482,7 @@ int mthca_map_user_db(struct mthca_dev *dev, struct mthca_uar *uar, ret = pci_map_sg(dev->pdev, &db_tab->page[i].mem, 1, PCI_DMA_TODEVICE); if (ret < 0) { - put_page(pages[0]); + put_user_page(pages[0]); goto out; } @@ -490,7 +490,7 @@ int mthca_map_user_db(struct mthca_dev *dev, struct mthca_uar *uar, mthca_uarc_virt(dev, uar, i)); if (ret) { pci_unmap_sg(dev->pdev, &db_tab->page[i].mem, 1, PCI_DMA_TODEVICE); - put_page(sg_page(&db_tab->page[i].mem)); + put_user_page(sg_page(&db_tab->page[i].mem)); goto out; } @@ -556,7 +556,7 @@ void mthca_cleanup_user_db_tab(struct mthca_dev *dev, struct mthca_uar *uar, if (db_tab->page[i].uvirt) { mthca_UNMAP_ICM(dev, mthca_uarc_virt(dev, uar, i), 1); pci_unmap_sg(dev->pdev, &db_tab->page[i].mem, 1, PCI_DMA_TODEVICE); - put_page(sg_page(&db_tab->page[i].mem)); + put_user_page(sg_page(&db_tab->page[i].mem)); } } diff --git a/drivers/infiniband/hw/mthca/mthca_provider.c b/drivers/infiniband/hw/mthca/mthca_provider.c index 4f40dfedf920..23554d8bf241 100644 --- a/drivers/infiniband/hw/mthca/mthca_provider.c +++ b/drivers/infiniband/hw/mthca/mthca_provider.c @@ -601,10 +601,11 @@ static int mthca_destroy_qp(struct ib_qp *qp, struct ib_udata *udata) return 0; } -static struct ib_cq *mthca_create_cq(struct ib_device *ibdev, - const struct ib_cq_init_attr *attr, - struct ib_udata *udata) +static int mthca_create_cq(struct ib_cq *ibcq, + const struct ib_cq_init_attr *attr, + struct ib_udata *udata) { + struct ib_device *ibdev = ibcq->device; int entries = attr->cqe; struct mthca_create_cq ucmd; struct mthca_cq *cq; @@ -614,20 +615,20 @@ static struct ib_cq *mthca_create_cq(struct ib_device *ibdev, udata, struct mthca_ucontext, ibucontext); if (attr->flags) - return ERR_PTR(-EINVAL); + return -EINVAL; if (entries < 1 || entries > to_mdev(ibdev)->limits.max_cqes) - return ERR_PTR(-EINVAL); + return -EINVAL; if (udata) { - if (ib_copy_from_udata(&ucmd, udata, sizeof ucmd)) - return ERR_PTR(-EFAULT); + if (ib_copy_from_udata(&ucmd, udata, sizeof(ucmd))) + return -EFAULT; err = mthca_map_user_db(to_mdev(ibdev), &context->uar, context->db_tab, ucmd.set_db_index, ucmd.set_db_page); if (err) - return ERR_PTR(err); + return err; err = mthca_map_user_db(to_mdev(ibdev), &context->uar, context->db_tab, ucmd.arm_db_index, @@ -636,11 +637,7 @@ static struct ib_cq *mthca_create_cq(struct ib_device *ibdev, goto err_unmap_set; } - cq = kzalloc(sizeof(*cq), GFP_KERNEL); - if (!cq) { - err = -ENOMEM; - goto err_unmap_arm; - } + cq = to_mcq(ibcq); if (udata) { cq->buf.mr.ibmr.lkey = ucmd.lkey; @@ -655,20 +652,17 @@ static struct ib_cq *mthca_create_cq(struct ib_device *ibdev, udata ? ucmd.pdn : to_mdev(ibdev)->driver_pd.pd_num, cq); if (err) - goto err_free; + goto err_unmap_arm; if (udata && ib_copy_to_udata(udata, &cq->cqn, sizeof(__u32))) { mthca_free_cq(to_mdev(ibdev), cq); err = -EFAULT; - goto err_free; + goto err_unmap_arm; } cq->resize_buf = NULL; - return &cq->ibcq; - -err_free: - kfree(cq); + return 0; err_unmap_arm: if (udata) @@ -680,7 +674,7 @@ err_unmap_set: mthca_unmap_user_db(to_mdev(ibdev), &context->uar, context->db_tab, ucmd.set_db_index); - return ERR_PTR(err); + return err; } static int mthca_alloc_resize_buf(struct mthca_dev *dev, struct mthca_cq *cq, @@ -804,7 +798,7 @@ out: return ret; } -static int mthca_destroy_cq(struct ib_cq *cq, struct ib_udata *udata) +static void mthca_destroy_cq(struct ib_cq *cq, struct ib_udata *udata) { if (udata) { struct mthca_ucontext *context = @@ -823,9 +817,6 @@ static int mthca_destroy_cq(struct ib_cq *cq, struct ib_udata *udata) to_mcq(cq)->set_ci_db_index); } mthca_free_cq(to_mdev(cq->device), to_mcq(cq)); - kfree(cq); - - return 0; } static inline u32 convert_access(int acc) @@ -962,8 +953,7 @@ static int mthca_dereg_mr(struct ib_mr *mr, struct ib_udata *udata) struct mthca_mr *mmr = to_mmr(mr); mthca_free_mr(to_mdev(mr->device), mmr); - if (mmr->umem) - ib_umem_release(mmr->umem); + ib_umem_release(mmr->umem); kfree(mmr); return 0; @@ -1153,6 +1143,11 @@ static void get_dev_fw_str(struct ib_device *device, char *str) } static const struct ib_device_ops mthca_dev_ops = { + .owner = THIS_MODULE, + .driver_id = RDMA_DRIVER_MTHCA, + .uverbs_abi_ver = MTHCA_UVERBS_ABI_VERSION, + .uverbs_no_driver_id_binding = 1, + .alloc_pd = mthca_alloc_pd, .alloc_ucontext = mthca_alloc_ucontext, .attach_mcast = mthca_multicast_attach, @@ -1185,6 +1180,7 @@ static const struct ib_device_ops mthca_dev_ops = { .resize_cq = mthca_resize_cq, INIT_RDMA_OBJ_SIZE(ib_ah, mthca_ah, ibah), + INIT_RDMA_OBJ_SIZE(ib_cq, mthca_cq, ibcq), INIT_RDMA_OBJ_SIZE(ib_pd, mthca_pd, ibpd), INIT_RDMA_OBJ_SIZE(ib_ucontext, mthca_ucontext, ibucontext), }; @@ -1243,9 +1239,6 @@ int mthca_register_device(struct mthca_dev *dev) if (ret) return ret; - dev->ib_dev.owner = THIS_MODULE; - - dev->ib_dev.uverbs_abi_ver = MTHCA_UVERBS_ABI_VERSION; dev->ib_dev.uverbs_cmd_mask = (1ull << IB_USER_VERBS_CMD_GET_CONTEXT) | (1ull << IB_USER_VERBS_CMD_QUERY_DEVICE) | @@ -1303,7 +1296,6 @@ int mthca_register_device(struct mthca_dev *dev) mutex_init(&dev->cap_mask_mutex); rdma_set_device_sysfs_group(&dev->ib_dev, &mthca_attr_group); - dev->ib_dev.driver_id = RDMA_DRIVER_MTHCA; ret = ib_register_device(&dev->ib_dev, "mthca%d"); if (ret) return ret; diff --git a/drivers/infiniband/hw/nes/Kconfig b/drivers/infiniband/hw/nes/Kconfig deleted file mode 100644 index 8245353e8106..000000000000 --- a/drivers/infiniband/hw/nes/Kconfig +++ /dev/null @@ -1,16 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0-only -config INFINIBAND_NES - tristate "NetEffect RNIC Driver" - depends on PCI && INET - select LIBCRC32C - ---help--- - This is the RDMA Network Interface Card (RNIC) driver for - NetEffect Ethernet Cluster Server Adapters. - -config INFINIBAND_NES_DEBUG - bool "Verbose debugging output" - depends on INFINIBAND_NES - default n - ---help--- - This option enables debug messages from the NetEffect RNIC - driver. Select this if you are diagnosing a problem. diff --git a/drivers/infiniband/hw/nes/Makefile b/drivers/infiniband/hw/nes/Makefile deleted file mode 100644 index 239689ad1f9e..000000000000 --- a/drivers/infiniband/hw/nes/Makefile +++ /dev/null @@ -1,4 +0,0 @@ -# SPDX-License-Identifier: GPL-2.0-only -obj-$(CONFIG_INFINIBAND_NES) += iw_nes.o - -iw_nes-objs := nes.o nes_hw.o nes_nic.o nes_utils.o nes_verbs.o nes_cm.o nes_mgt.o diff --git a/drivers/infiniband/hw/nes/nes.c b/drivers/infiniband/hw/nes/nes.c deleted file mode 100644 index 29b324726ea6..000000000000 --- a/drivers/infiniband/hw/nes/nes.c +++ /dev/null @@ -1,1211 +0,0 @@ -/* - * Copyright (c) 2006 - 2011 Intel Corporation. All rights reserved. - * Copyright (c) 2005 Open Grid Computing, Inc. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -#include <linux/module.h> -#include <linux/moduleparam.h> -#include <linux/netdevice.h> -#include <linux/etherdevice.h> -#include <linux/ethtool.h> -#include <linux/mii.h> -#include <linux/if_vlan.h> -#include <linux/crc32.h> -#include <linux/in.h> -#include <linux/fs.h> -#include <linux/init.h> -#include <linux/if_arp.h> -#include <linux/highmem.h> -#include <linux/slab.h> -#include <asm/io.h> -#include <asm/irq.h> -#include <asm/byteorder.h> -#include <rdma/ib_smi.h> -#include <rdma/ib_verbs.h> -#include <rdma/ib_pack.h> -#include <rdma/iw_cm.h> - -#include "nes.h" - -#include <net/netevent.h> -#include <net/neighbour.h> -#include <linux/route.h> -#include <net/ip_fib.h> - -MODULE_AUTHOR("NetEffect"); -MODULE_DESCRIPTION("NetEffect RNIC Low-level iWARP Driver"); -MODULE_LICENSE("Dual BSD/GPL"); - -int interrupt_mod_interval = 0; - -/* Interoperability */ -int mpa_version = 1; -module_param(mpa_version, int, 0644); -MODULE_PARM_DESC(mpa_version, "MPA version to be used int MPA Req/Resp (0 or 1)"); - -/* Interoperability */ -int disable_mpa_crc = 0; -module_param(disable_mpa_crc, int, 0644); -MODULE_PARM_DESC(disable_mpa_crc, "Disable checking of MPA CRC"); - -unsigned int nes_drv_opt = NES_DRV_OPT_DISABLE_INT_MOD | NES_DRV_OPT_ENABLE_PAU; -module_param(nes_drv_opt, int, 0644); -MODULE_PARM_DESC(nes_drv_opt, "Driver option parameters"); - -unsigned int nes_debug_level = 0; -module_param_named(debug_level, nes_debug_level, uint, 0644); -MODULE_PARM_DESC(debug_level, "Enable debug output level"); - -unsigned int wqm_quanta = 0x10000; -module_param(wqm_quanta, int, 0644); -MODULE_PARM_DESC(wqm_quanta, "WQM quanta"); - -static bool limit_maxrdreqsz; -module_param(limit_maxrdreqsz, bool, 0644); -MODULE_PARM_DESC(limit_maxrdreqsz, "Limit max read request size to 256 Bytes"); - -LIST_HEAD(nes_adapter_list); -static LIST_HEAD(nes_dev_list); - -atomic_t qps_destroyed; - -static unsigned int ee_flsh_adapter; -static unsigned int sysfs_nonidx_addr; -static unsigned int sysfs_idx_addr; - -static const struct pci_device_id nes_pci_table[] = { - { PCI_VDEVICE(NETEFFECT, PCI_DEVICE_ID_NETEFFECT_NE020), }, - { PCI_VDEVICE(NETEFFECT, PCI_DEVICE_ID_NETEFFECT_NE020_KR), }, - {0} -}; - -MODULE_DEVICE_TABLE(pci, nes_pci_table); - -static int nes_inetaddr_event(struct notifier_block *, unsigned long, void *); -static int nes_net_event(struct notifier_block *, unsigned long, void *); -static int nes_notifiers_registered; - - -static struct notifier_block nes_inetaddr_notifier = { - .notifier_call = nes_inetaddr_event -}; - -static struct notifier_block nes_net_notifier = { - .notifier_call = nes_net_event -}; - -/** - * nes_inetaddr_event - */ -static int nes_inetaddr_event(struct notifier_block *notifier, - unsigned long event, void *ptr) -{ - struct in_ifaddr *ifa = ptr; - struct net_device *event_netdev = ifa->ifa_dev->dev; - struct nes_device *nesdev; - struct net_device *netdev; - struct net_device *upper_dev; - struct nes_vnic *nesvnic; - unsigned int is_bonded; - - nes_debug(NES_DBG_NETDEV, "nes_inetaddr_event: ip address %pI4, netmask %pI4.\n", - &ifa->ifa_address, &ifa->ifa_mask); - list_for_each_entry(nesdev, &nes_dev_list, list) { - nes_debug(NES_DBG_NETDEV, "Nesdev list entry = 0x%p. (%s)\n", - nesdev, nesdev->netdev[0]->name); - netdev = nesdev->netdev[0]; - nesvnic = netdev_priv(netdev); - upper_dev = netdev_master_upper_dev_get(netdev); - is_bonded = netif_is_bond_slave(netdev) && - (upper_dev == event_netdev); - if ((netdev == event_netdev) || is_bonded) { - if (nesvnic->rdma_enabled == 0) { - nes_debug(NES_DBG_NETDEV, "Returning without processing event for %s since" - " RDMA is not enabled.\n", - netdev->name); - return NOTIFY_OK; - } - /* we have ifa->ifa_address/mask here if we need it */ - switch (event) { - case NETDEV_DOWN: - nes_debug(NES_DBG_NETDEV, "event:DOWN\n"); - nes_write_indexed(nesdev, - NES_IDX_DST_IP_ADDR+(0x10*PCI_FUNC(nesdev->pcidev->devfn)), 0); - - nes_manage_arp_cache(netdev, netdev->dev_addr, - ntohl(nesvnic->local_ipaddr), NES_ARP_DELETE); - nesvnic->local_ipaddr = 0; - if (is_bonded) - continue; - else - return NOTIFY_OK; - break; - case NETDEV_UP: - nes_debug(NES_DBG_NETDEV, "event:UP\n"); - - if (nesvnic->local_ipaddr != 0) { - nes_debug(NES_DBG_NETDEV, "Interface already has local_ipaddr\n"); - return NOTIFY_OK; - } - /* fall through */ - case NETDEV_CHANGEADDR: - /* Add the address to the IP table */ - if (upper_dev) { - struct in_device *in; - - rcu_read_lock(); - in = __in_dev_get_rcu(upper_dev); - if (in) { - struct in_ifaddr *ifa; - - ifa = rcu_dereference(in->ifa_list); - if (ifa) - nesvnic->local_ipaddr = ifa->ifa_address; - } - rcu_read_unlock(); - } else { - nesvnic->local_ipaddr = ifa->ifa_address; - } - - nes_write_indexed(nesdev, - NES_IDX_DST_IP_ADDR+(0x10*PCI_FUNC(nesdev->pcidev->devfn)), - ntohl(nesvnic->local_ipaddr)); - nes_manage_arp_cache(netdev, netdev->dev_addr, - ntohl(nesvnic->local_ipaddr), NES_ARP_ADD); - if (is_bonded) - continue; - else - return NOTIFY_OK; - break; - default: - break; - } - } - } - - return NOTIFY_DONE; -} - - -/** - * nes_net_event - */ -static int nes_net_event(struct notifier_block *notifier, - unsigned long event, void *ptr) -{ - struct neighbour *neigh = ptr; - struct nes_device *nesdev; - struct net_device *netdev; - struct nes_vnic *nesvnic; - - switch (event) { - case NETEVENT_NEIGH_UPDATE: - list_for_each_entry(nesdev, &nes_dev_list, list) { - /* nes_debug(NES_DBG_NETDEV, "Nesdev list entry = 0x%p.\n", nesdev); */ - netdev = nesdev->netdev[0]; - nesvnic = netdev_priv(netdev); - if (netdev == neigh->dev) { - if (nesvnic->rdma_enabled == 0) { - nes_debug(NES_DBG_NETDEV, "Skipping device %s since no RDMA\n", - netdev->name); - } else { - if (neigh->nud_state & NUD_VALID) { - nes_manage_arp_cache(neigh->dev, neigh->ha, - ntohl(*(__be32 *)neigh->primary_key), NES_ARP_ADD); - } else { - nes_manage_arp_cache(neigh->dev, neigh->ha, - ntohl(*(__be32 *)neigh->primary_key), NES_ARP_DELETE); - } - } - return NOTIFY_OK; - } - } - break; - default: - nes_debug(NES_DBG_NETDEV, "NETEVENT_ %lu undefined\n", event); - break; - } - - return NOTIFY_DONE; -} - - -/** - * nes_add_ref - */ -void nes_add_ref(struct ib_qp *ibqp) -{ - struct nes_qp *nesqp; - - nesqp = to_nesqp(ibqp); - nes_debug(NES_DBG_QP, "Bumping refcount for QP%u. Pre-inc value = %u\n", - ibqp->qp_num, atomic_read(&nesqp->refcount)); - atomic_inc(&nesqp->refcount); -} - -static void nes_cqp_rem_ref_callback(struct nes_device *nesdev, struct nes_cqp_request *cqp_request) -{ - unsigned long flags; - struct nes_qp *nesqp = cqp_request->cqp_callback_pointer; - struct nes_adapter *nesadapter = nesdev->nesadapter; - - atomic_inc(&qps_destroyed); - - /* Free the control structures */ - - if (nesqp->pbl_vbase) { - pci_free_consistent(nesdev->pcidev, nesqp->qp_mem_size, - nesqp->hwqp.q2_vbase, nesqp->hwqp.q2_pbase); - spin_lock_irqsave(&nesadapter->pbl_lock, flags); - nesadapter->free_256pbl++; - spin_unlock_irqrestore(&nesadapter->pbl_lock, flags); - pci_free_consistent(nesdev->pcidev, 256, nesqp->pbl_vbase, nesqp->pbl_pbase); - nesqp->pbl_vbase = NULL; - - } else { - pci_free_consistent(nesdev->pcidev, nesqp->qp_mem_size, - nesqp->hwqp.sq_vbase, nesqp->hwqp.sq_pbase); - } - nes_free_resource(nesadapter, nesadapter->allocated_qps, nesqp->hwqp.qp_id); - - nesadapter->qp_table[nesqp->hwqp.qp_id-NES_FIRST_QPN] = NULL; - kfree(nesqp->allocated_buffer); - -} - -/** - * nes_rem_ref - */ -void nes_rem_ref(struct ib_qp *ibqp) -{ - u64 u64temp; - struct nes_qp *nesqp; - struct nes_vnic *nesvnic = to_nesvnic(ibqp->device); - struct nes_device *nesdev = nesvnic->nesdev; - struct nes_hw_cqp_wqe *cqp_wqe; - struct nes_cqp_request *cqp_request; - u32 opcode; - - nesqp = to_nesqp(ibqp); - - if (atomic_read(&nesqp->refcount) == 0) { - printk(KERN_INFO PFX "%s: Reference count already 0 for QP%d, last aeq = 0x%04X.\n", - __func__, ibqp->qp_num, nesqp->last_aeq); - BUG(); - } - - if (atomic_dec_and_test(&nesqp->refcount)) { - if (nesqp->pau_mode) - nes_destroy_pau_qp(nesdev, nesqp); - - /* Destroy the QP */ - cqp_request = nes_get_cqp_request(nesdev); - if (cqp_request == NULL) { - nes_debug(NES_DBG_QP, "Failed to get a cqp_request.\n"); - return; - } - cqp_request->waiting = 0; - cqp_request->callback = 1; - cqp_request->cqp_callback = nes_cqp_rem_ref_callback; - cqp_request->cqp_callback_pointer = nesqp; - cqp_wqe = &cqp_request->cqp_wqe; - - nes_fill_init_cqp_wqe(cqp_wqe, nesdev); - opcode = NES_CQP_DESTROY_QP | NES_CQP_QP_TYPE_IWARP; - - if (nesqp->hte_added) { - opcode |= NES_CQP_QP_DEL_HTE; - nesqp->hte_added = 0; - } - set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX, opcode); - set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX, nesqp->hwqp.qp_id); - u64temp = (u64)nesqp->nesqp_context_pbase; - set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_QP_WQE_CONTEXT_LOW_IDX, u64temp); - nes_post_cqp_request(nesdev, cqp_request); - } -} - - -/** - * nes_get_qp - */ -struct ib_qp *nes_get_qp(struct ib_device *device, int qpn) -{ - struct nes_vnic *nesvnic = to_nesvnic(device); - struct nes_device *nesdev = nesvnic->nesdev; - struct nes_adapter *nesadapter = nesdev->nesadapter; - - if ((qpn < NES_FIRST_QPN) || (qpn >= (NES_FIRST_QPN + nesadapter->max_qp))) - return NULL; - - return &nesadapter->qp_table[qpn - NES_FIRST_QPN]->ibqp; -} - - -/** - * nes_print_macaddr - */ -static void nes_print_macaddr(struct net_device *netdev) -{ - nes_debug(NES_DBG_INIT, "%s: %pM, IRQ %u\n", - netdev->name, netdev->dev_addr, netdev->irq); -} - -/** - * nes_interrupt - handle interrupts - */ -static irqreturn_t nes_interrupt(int irq, void *dev_id) -{ - struct nes_device *nesdev = (struct nes_device *)dev_id; - int handled = 0; - u32 int_mask; - u32 int_req; - u32 int_stat; - u32 intf_int_stat; - u32 timer_stat; - - if (nesdev->msi_enabled) { - /* No need to read the interrupt pending register if msi is enabled */ - handled = 1; - } else { - if (unlikely(nesdev->nesadapter->hw_rev == NE020_REV)) { - /* Master interrupt enable provides synchronization for kicking off bottom half - when interrupt sharing is going on */ - int_mask = nes_read32(nesdev->regs + NES_INT_MASK); - if (int_mask & 0x80000000) { - /* Check interrupt status to see if this might be ours */ - int_stat = nes_read32(nesdev->regs + NES_INT_STAT); - int_req = nesdev->int_req; - if (int_stat&int_req) { - /* if interesting CEQ or AEQ is pending, claim the interrupt */ - if ((int_stat&int_req) & (~(NES_INT_TIMER|NES_INT_INTF))) { - handled = 1; - } else { - if (((int_stat & int_req) & NES_INT_TIMER) == NES_INT_TIMER) { - /* Timer might be running but might be for another function */ - timer_stat = nes_read32(nesdev->regs + NES_TIMER_STAT); - if ((timer_stat & nesdev->timer_int_req) != 0) { - handled = 1; - } - } - if ((((int_stat & int_req) & NES_INT_INTF) == NES_INT_INTF) && - (handled == 0)) { - intf_int_stat = nes_read32(nesdev->regs+NES_INTF_INT_STAT); - if ((intf_int_stat & nesdev->intf_int_req) != 0) { - handled = 1; - } - } - } - if (handled) { - nes_write32(nesdev->regs+NES_INT_MASK, int_mask & (~0x80000000)); - int_mask = nes_read32(nesdev->regs+NES_INT_MASK); - /* Save off the status to save an additional read */ - nesdev->int_stat = int_stat; - nesdev->napi_isr_ran = 1; - } - } - } - } else { - handled = nes_read32(nesdev->regs+NES_INT_PENDING); - } - } - - if (handled) { - - if (nes_napi_isr(nesdev) == 0) { - tasklet_schedule(&nesdev->dpc_tasklet); - - } - return IRQ_HANDLED; - } else { - return IRQ_NONE; - } -} - - -/** - * nes_probe - Device initialization - */ -static int nes_probe(struct pci_dev *pcidev, const struct pci_device_id *ent) -{ - struct net_device *netdev = NULL; - struct nes_device *nesdev = NULL; - int ret = 0; - void __iomem *mmio_regs = NULL; - u8 hw_rev; - - printk(KERN_INFO PFX "NetEffect RNIC driver v%s loading. (%s)\n", - DRV_VERSION, pci_name(pcidev)); - - ret = pci_enable_device(pcidev); - if (ret) { - printk(KERN_ERR PFX "Unable to enable PCI device. (%s)\n", pci_name(pcidev)); - goto bail0; - } - - nes_debug(NES_DBG_INIT, "BAR0 (@0x%08lX) size = 0x%lX bytes\n", - (long unsigned int)pci_resource_start(pcidev, BAR_0), - (long unsigned int)pci_resource_len(pcidev, BAR_0)); - nes_debug(NES_DBG_INIT, "BAR1 (@0x%08lX) size = 0x%lX bytes\n", - (long unsigned int)pci_resource_start(pcidev, BAR_1), - (long unsigned int)pci_resource_len(pcidev, BAR_1)); - - /* Make sure PCI base addr are MMIO */ - if (!(pci_resource_flags(pcidev, BAR_0) & IORESOURCE_MEM) || - !(pci_resource_flags(pcidev, BAR_1) & IORESOURCE_MEM)) { - printk(KERN_ERR PFX "PCI regions not an MMIO resource\n"); - ret = -ENODEV; - goto bail1; - } - - /* Reserve PCI I/O and memory resources */ - ret = pci_request_regions(pcidev, DRV_NAME); - if (ret) { - printk(KERN_ERR PFX "Unable to request regions. (%s)\n", pci_name(pcidev)); - goto bail1; - } - - if ((sizeof(dma_addr_t) > 4)) { - ret = pci_set_dma_mask(pcidev, DMA_BIT_MASK(64)); - if (ret < 0) { - printk(KERN_ERR PFX "64b DMA mask configuration failed\n"); - goto bail2; - } - ret = pci_set_consistent_dma_mask(pcidev, DMA_BIT_MASK(64)); - if (ret) { - printk(KERN_ERR PFX "64b DMA consistent mask configuration failed\n"); - goto bail2; - } - } else { - ret = pci_set_dma_mask(pcidev, DMA_BIT_MASK(32)); - if (ret < 0) { - printk(KERN_ERR PFX "32b DMA mask configuration failed\n"); - goto bail2; - } - ret = pci_set_consistent_dma_mask(pcidev, DMA_BIT_MASK(32)); - if (ret) { - printk(KERN_ERR PFX "32b DMA consistent mask configuration failed\n"); - goto bail2; - } - } - - pci_set_master(pcidev); - - /* Allocate hardware structure */ - nesdev = kzalloc(sizeof(struct nes_device), GFP_KERNEL); - if (!nesdev) { - ret = -ENOMEM; - goto bail2; - } - - nes_debug(NES_DBG_INIT, "Allocated nes device at %p\n", nesdev); - nesdev->pcidev = pcidev; - pci_set_drvdata(pcidev, nesdev); - - pci_read_config_byte(pcidev, 0x0008, &hw_rev); - nes_debug(NES_DBG_INIT, "hw_rev=%u\n", hw_rev); - - spin_lock_init(&nesdev->indexed_regs_lock); - - /* Remap the PCI registers in adapter BAR0 to kernel VA space */ - mmio_regs = ioremap_nocache(pci_resource_start(pcidev, BAR_0), - pci_resource_len(pcidev, BAR_0)); - if (mmio_regs == NULL) { - printk(KERN_ERR PFX "Unable to remap BAR0\n"); - ret = -EIO; - goto bail3; - } - nesdev->regs = mmio_regs; - nesdev->index_reg = 0x50 + (PCI_FUNC(pcidev->devfn)*8) + mmio_regs; - - /* Ensure interrupts are disabled */ - nes_write32(nesdev->regs+NES_INT_MASK, 0x7fffffff); - - if (nes_drv_opt & NES_DRV_OPT_ENABLE_MSI) { - if (!pci_enable_msi(nesdev->pcidev)) { - nesdev->msi_enabled = 1; - nes_debug(NES_DBG_INIT, "MSI is enabled for device %s\n", - pci_name(pcidev)); - } else { - nes_debug(NES_DBG_INIT, "MSI is disabled by linux for device %s\n", - pci_name(pcidev)); - } - } else { - nes_debug(NES_DBG_INIT, "MSI not requested due to driver options for device %s\n", - pci_name(pcidev)); - } - - nesdev->csr_start = pci_resource_start(nesdev->pcidev, BAR_0); - nesdev->doorbell_region = pci_resource_start(nesdev->pcidev, BAR_1); - - /* Init the adapter */ - nesdev->nesadapter = nes_init_adapter(nesdev, hw_rev); - if (!nesdev->nesadapter) { - printk(KERN_ERR PFX "Unable to initialize adapter.\n"); - ret = -ENOMEM; - goto bail5; - } - nesdev->nesadapter->et_rx_coalesce_usecs_irq = interrupt_mod_interval; - nesdev->nesadapter->wqm_quanta = wqm_quanta; - - /* nesdev->base_doorbell_index = - nesdev->nesadapter->pd_config_base[PCI_FUNC(nesdev->pcidev->devfn)]; */ - nesdev->base_doorbell_index = 1; - nesdev->doorbell_start = nesdev->nesadapter->doorbell_start; - if (nesdev->nesadapter->phy_type[0] == NES_PHY_TYPE_PUMA_1G) { - switch (PCI_FUNC(nesdev->pcidev->devfn) % - nesdev->nesadapter->port_count) { - case 1: - nesdev->mac_index = 2; - break; - case 2: - nesdev->mac_index = 1; - break; - case 3: - nesdev->mac_index = 3; - break; - case 0: - default: - nesdev->mac_index = 0; - } - } else { - nesdev->mac_index = PCI_FUNC(nesdev->pcidev->devfn) % - nesdev->nesadapter->port_count; - } - - if ((limit_maxrdreqsz || - ((nesdev->nesadapter->phy_type[0] == NES_PHY_TYPE_GLADIUS) && - (hw_rev == NE020_REV1))) && - (pcie_get_readrq(pcidev) > 256)) { - if (pcie_set_readrq(pcidev, 256)) - printk(KERN_ERR PFX "Unable to set max read request" - " to 256 bytes\n"); - else - nes_debug(NES_DBG_INIT, "Max read request size set" - " to 256 bytes\n"); - } - - tasklet_init(&nesdev->dpc_tasklet, nes_dpc, (unsigned long)nesdev); - - /* bring up the Control QP */ - if (nes_init_cqp(nesdev)) { - ret = -ENODEV; - goto bail6; - } - - /* Arm the CCQ */ - nes_write32(nesdev->regs+NES_CQE_ALLOC, NES_CQE_ALLOC_NOTIFY_NEXT | - PCI_FUNC(nesdev->pcidev->devfn)); - nes_read32(nesdev->regs+NES_CQE_ALLOC); - - /* Enable the interrupts */ - nesdev->int_req = (0x101 << PCI_FUNC(nesdev->pcidev->devfn)) | - (1 << (PCI_FUNC(nesdev->pcidev->devfn)+16)); - if (PCI_FUNC(nesdev->pcidev->devfn) < 4) { - nesdev->int_req |= (1 << (PCI_FUNC(nesdev->mac_index)+24)); - } - - /* TODO: This really should be the first driver to load, not function 0 */ - if (PCI_FUNC(nesdev->pcidev->devfn) == 0) { - /* pick up PCI and critical errors if the first driver to load */ - nesdev->intf_int_req = NES_INTF_INT_PCIERR | NES_INTF_INT_CRITERR; - nesdev->int_req |= NES_INT_INTF; - } else { - nesdev->intf_int_req = 0; - } - nesdev->intf_int_req |= (1 << (PCI_FUNC(nesdev->pcidev->devfn)+16)); - nes_write_indexed(nesdev, NES_IDX_DEBUG_ERROR_MASKS0, 0); - nes_write_indexed(nesdev, NES_IDX_DEBUG_ERROR_MASKS1, 0); - nes_write_indexed(nesdev, NES_IDX_DEBUG_ERROR_MASKS2, 0x00001265); - nes_write_indexed(nesdev, NES_IDX_DEBUG_ERROR_MASKS4, 0x18021804); - - nes_write_indexed(nesdev, NES_IDX_DEBUG_ERROR_MASKS3, 0x17801790); - - /* deal with both periodic and one_shot */ - nesdev->timer_int_req = 0x101 << PCI_FUNC(nesdev->pcidev->devfn); - nesdev->nesadapter->timer_int_req |= nesdev->timer_int_req; - nes_debug(NES_DBG_INIT, "setting int_req for function %u, nesdev = 0x%04X, adapter = 0x%04X\n", - PCI_FUNC(nesdev->pcidev->devfn), - nesdev->timer_int_req, nesdev->nesadapter->timer_int_req); - - nes_write32(nesdev->regs+NES_INTF_INT_MASK, ~(nesdev->intf_int_req)); - - list_add_tail(&nesdev->list, &nes_dev_list); - - /* Request an interrupt line for the driver */ - ret = request_irq(pcidev->irq, nes_interrupt, IRQF_SHARED, DRV_NAME, nesdev); - if (ret) { - printk(KERN_ERR PFX "%s: requested IRQ %u is busy\n", - pci_name(pcidev), pcidev->irq); - goto bail65; - } - - nes_write32(nesdev->regs+NES_INT_MASK, ~nesdev->int_req); - - if (nes_notifiers_registered == 0) { - register_inetaddr_notifier(&nes_inetaddr_notifier); - register_netevent_notifier(&nes_net_notifier); - } - nes_notifiers_registered++; - - INIT_DELAYED_WORK(&nesdev->work, nes_recheck_link_status); - - /* Initialize network devices */ - netdev = nes_netdev_init(nesdev, mmio_regs); - if (netdev == NULL) { - ret = -ENOMEM; - goto bail7; - } - - /* Register network device */ - ret = register_netdev(netdev); - if (ret) { - printk(KERN_ERR PFX "Unable to register netdev, ret = %d\n", ret); - nes_netdev_destroy(netdev); - goto bail7; - } - - nes_print_macaddr(netdev); - - nesdev->netdev_count++; - nesdev->nesadapter->netdev_count++; - - printk(KERN_INFO PFX "%s: NetEffect RNIC driver successfully loaded.\n", - pci_name(pcidev)); - return 0; - - bail7: - printk(KERN_ERR PFX "bail7\n"); - while (nesdev->netdev_count > 0) { - nesdev->netdev_count--; - nesdev->nesadapter->netdev_count--; - - unregister_netdev(nesdev->netdev[nesdev->netdev_count]); - nes_netdev_destroy(nesdev->netdev[nesdev->netdev_count]); - } - - nes_debug(NES_DBG_INIT, "netdev_count=%d, nesadapter->netdev_count=%d\n", - nesdev->netdev_count, nesdev->nesadapter->netdev_count); - - nes_notifiers_registered--; - if (nes_notifiers_registered == 0) { - unregister_netevent_notifier(&nes_net_notifier); - unregister_inetaddr_notifier(&nes_inetaddr_notifier); - } - - list_del(&nesdev->list); - nes_destroy_cqp(nesdev); - - bail65: - printk(KERN_ERR PFX "bail65\n"); - free_irq(pcidev->irq, nesdev); - if (nesdev->msi_enabled) { - pci_disable_msi(pcidev); - } - bail6: - printk(KERN_ERR PFX "bail6\n"); - tasklet_kill(&nesdev->dpc_tasklet); - /* Deallocate the Adapter Structure */ - nes_destroy_adapter(nesdev->nesadapter); - - bail5: - printk(KERN_ERR PFX "bail5\n"); - iounmap(nesdev->regs); - - bail3: - printk(KERN_ERR PFX "bail3\n"); - kfree(nesdev); - - bail2: - pci_release_regions(pcidev); - - bail1: - pci_disable_device(pcidev); - - bail0: - return ret; -} - - -/** - * nes_remove - unload from kernel - */ -static void nes_remove(struct pci_dev *pcidev) -{ - struct nes_device *nesdev = pci_get_drvdata(pcidev); - struct net_device *netdev; - int netdev_index = 0; - unsigned long flags; - - if (nesdev->netdev_count) { - netdev = nesdev->netdev[netdev_index]; - if (netdev) { - netif_stop_queue(netdev); - unregister_netdev(netdev); - nes_netdev_destroy(netdev); - - nesdev->netdev[netdev_index] = NULL; - nesdev->netdev_count--; - nesdev->nesadapter->netdev_count--; - } - } - - nes_notifiers_registered--; - if (nes_notifiers_registered == 0) { - unregister_netevent_notifier(&nes_net_notifier); - unregister_inetaddr_notifier(&nes_inetaddr_notifier); - } - - list_del(&nesdev->list); - nes_destroy_cqp(nesdev); - - free_irq(pcidev->irq, nesdev); - tasklet_kill(&nesdev->dpc_tasklet); - - spin_lock_irqsave(&nesdev->nesadapter->phy_lock, flags); - if (nesdev->link_recheck) { - spin_unlock_irqrestore(&nesdev->nesadapter->phy_lock, flags); - cancel_delayed_work_sync(&nesdev->work); - } else { - spin_unlock_irqrestore(&nesdev->nesadapter->phy_lock, flags); - } - - /* Deallocate the Adapter Structure */ - nes_destroy_adapter(nesdev->nesadapter); - - if (nesdev->msi_enabled) { - pci_disable_msi(pcidev); - } - - iounmap(nesdev->regs); - kfree(nesdev); - - /* nes_debug(NES_DBG_SHUTDOWN, "calling pci_release_regions.\n"); */ - pci_release_regions(pcidev); - pci_disable_device(pcidev); - pci_set_drvdata(pcidev, NULL); -} - - -static ssize_t adapter_show(struct device_driver *ddp, char *buf) -{ - unsigned int devfn = 0xffffffff; - unsigned char bus_number = 0xff; - unsigned int i = 0; - struct nes_device *nesdev; - - list_for_each_entry(nesdev, &nes_dev_list, list) { - if (i == ee_flsh_adapter) { - devfn = nesdev->pcidev->devfn; - bus_number = nesdev->pcidev->bus->number; - break; - } - i++; - } - - return snprintf(buf, PAGE_SIZE, "%x:%x\n", bus_number, devfn); -} - -static ssize_t adapter_store(struct device_driver *ddp, - const char *buf, size_t count) -{ - char *p = (char *)buf; - - ee_flsh_adapter = simple_strtoul(p, &p, 10); - return strnlen(buf, count); -} - -static ssize_t eeprom_cmd_show(struct device_driver *ddp, char *buf) -{ - u32 eeprom_cmd = 0xdead; - u32 i = 0; - struct nes_device *nesdev; - - list_for_each_entry(nesdev, &nes_dev_list, list) { - if (i == ee_flsh_adapter) { - eeprom_cmd = nes_read32(nesdev->regs + NES_EEPROM_COMMAND); - break; - } - i++; - } - return snprintf(buf, PAGE_SIZE, "0x%x\n", eeprom_cmd); -} - -static ssize_t eeprom_cmd_store(struct device_driver *ddp, - const char *buf, size_t count) -{ - char *p = (char *)buf; - u32 val; - u32 i = 0; - struct nes_device *nesdev; - - if (p[1] == 'x' || p[1] == 'X' || p[0] == 'x' || p[0] == 'X') { - val = simple_strtoul(p, &p, 16); - list_for_each_entry(nesdev, &nes_dev_list, list) { - if (i == ee_flsh_adapter) { - nes_write32(nesdev->regs + NES_EEPROM_COMMAND, val); - break; - } - i++; - } - } - return strnlen(buf, count); -} - -static ssize_t eeprom_data_show(struct device_driver *ddp, char *buf) -{ - u32 eeprom_data = 0xdead; - u32 i = 0; - struct nes_device *nesdev; - - list_for_each_entry(nesdev, &nes_dev_list, list) { - if (i == ee_flsh_adapter) { - eeprom_data = nes_read32(nesdev->regs + NES_EEPROM_DATA); - break; - } - i++; - } - - return snprintf(buf, PAGE_SIZE, "0x%x\n", eeprom_data); -} - -static ssize_t eeprom_data_store(struct device_driver *ddp, - const char *buf, size_t count) -{ - char *p = (char *)buf; - u32 val; - u32 i = 0; - struct nes_device *nesdev; - - if (p[1] == 'x' || p[1] == 'X' || p[0] == 'x' || p[0] == 'X') { - val = simple_strtoul(p, &p, 16); - list_for_each_entry(nesdev, &nes_dev_list, list) { - if (i == ee_flsh_adapter) { - nes_write32(nesdev->regs + NES_EEPROM_DATA, val); - break; - } - i++; - } - } - return strnlen(buf, count); -} - -static ssize_t flash_cmd_show(struct device_driver *ddp, char *buf) -{ - u32 flash_cmd = 0xdead; - u32 i = 0; - struct nes_device *nesdev; - - list_for_each_entry(nesdev, &nes_dev_list, list) { - if (i == ee_flsh_adapter) { - flash_cmd = nes_read32(nesdev->regs + NES_FLASH_COMMAND); - break; - } - i++; - } - - return snprintf(buf, PAGE_SIZE, "0x%x\n", flash_cmd); -} - -static ssize_t flash_cmd_store(struct device_driver *ddp, - const char *buf, size_t count) -{ - char *p = (char *)buf; - u32 val; - u32 i = 0; - struct nes_device *nesdev; - - if (p[1] == 'x' || p[1] == 'X' || p[0] == 'x' || p[0] == 'X') { - val = simple_strtoul(p, &p, 16); - list_for_each_entry(nesdev, &nes_dev_list, list) { - if (i == ee_flsh_adapter) { - nes_write32(nesdev->regs + NES_FLASH_COMMAND, val); - break; - } - i++; - } - } - return strnlen(buf, count); -} - -static ssize_t flash_data_show(struct device_driver *ddp, char *buf) -{ - u32 flash_data = 0xdead; - u32 i = 0; - struct nes_device *nesdev; - - list_for_each_entry(nesdev, &nes_dev_list, list) { - if (i == ee_flsh_adapter) { - flash_data = nes_read32(nesdev->regs + NES_FLASH_DATA); - break; - } - i++; - } - - return snprintf(buf, PAGE_SIZE, "0x%x\n", flash_data); -} - -static ssize_t flash_data_store(struct device_driver *ddp, - const char *buf, size_t count) -{ - char *p = (char *)buf; - u32 val; - u32 i = 0; - struct nes_device *nesdev; - - if (p[1] == 'x' || p[1] == 'X' || p[0] == 'x' || p[0] == 'X') { - val = simple_strtoul(p, &p, 16); - list_for_each_entry(nesdev, &nes_dev_list, list) { - if (i == ee_flsh_adapter) { - nes_write32(nesdev->regs + NES_FLASH_DATA, val); - break; - } - i++; - } - } - return strnlen(buf, count); -} - -static ssize_t nonidx_addr_show(struct device_driver *ddp, char *buf) -{ - return snprintf(buf, PAGE_SIZE, "0x%x\n", sysfs_nonidx_addr); -} - -static ssize_t nonidx_addr_store(struct device_driver *ddp, - const char *buf, size_t count) -{ - char *p = (char *)buf; - - if (p[1] == 'x' || p[1] == 'X' || p[0] == 'x' || p[0] == 'X') - sysfs_nonidx_addr = simple_strtoul(p, &p, 16); - - return strnlen(buf, count); -} - -static ssize_t nonidx_data_show(struct device_driver *ddp, char *buf) -{ - u32 nonidx_data = 0xdead; - u32 i = 0; - struct nes_device *nesdev; - - list_for_each_entry(nesdev, &nes_dev_list, list) { - if (i == ee_flsh_adapter) { - nonidx_data = nes_read32(nesdev->regs + sysfs_nonidx_addr); - break; - } - i++; - } - - return snprintf(buf, PAGE_SIZE, "0x%x\n", nonidx_data); -} - -static ssize_t nonidx_data_store(struct device_driver *ddp, - const char *buf, size_t count) -{ - char *p = (char *)buf; - u32 val; - u32 i = 0; - struct nes_device *nesdev; - - if (p[1] == 'x' || p[1] == 'X' || p[0] == 'x' || p[0] == 'X') { - val = simple_strtoul(p, &p, 16); - list_for_each_entry(nesdev, &nes_dev_list, list) { - if (i == ee_flsh_adapter) { - nes_write32(nesdev->regs + sysfs_nonidx_addr, val); - break; - } - i++; - } - } - return strnlen(buf, count); -} - -static ssize_t idx_addr_show(struct device_driver *ddp, char *buf) -{ - return snprintf(buf, PAGE_SIZE, "0x%x\n", sysfs_idx_addr); -} - -static ssize_t idx_addr_store(struct device_driver *ddp, - const char *buf, size_t count) -{ - char *p = (char *)buf; - - if (p[1] == 'x' || p[1] == 'X' || p[0] == 'x' || p[0] == 'X') - sysfs_idx_addr = simple_strtoul(p, &p, 16); - - return strnlen(buf, count); -} - -static ssize_t idx_data_show(struct device_driver *ddp, char *buf) -{ - u32 idx_data = 0xdead; - u32 i = 0; - struct nes_device *nesdev; - - list_for_each_entry(nesdev, &nes_dev_list, list) { - if (i == ee_flsh_adapter) { - idx_data = nes_read_indexed(nesdev, sysfs_idx_addr); - break; - } - i++; - } - - return snprintf(buf, PAGE_SIZE, "0x%x\n", idx_data); -} - -static ssize_t idx_data_store(struct device_driver *ddp, - const char *buf, size_t count) -{ - char *p = (char *)buf; - u32 val; - u32 i = 0; - struct nes_device *nesdev; - - if (p[1] == 'x' || p[1] == 'X' || p[0] == 'x' || p[0] == 'X') { - val = simple_strtoul(p, &p, 16); - list_for_each_entry(nesdev, &nes_dev_list, list) { - if (i == ee_flsh_adapter) { - nes_write_indexed(nesdev, sysfs_idx_addr, val); - break; - } - i++; - } - } - return strnlen(buf, count); -} - -static ssize_t wqm_quanta_show(struct device_driver *ddp, char *buf) -{ - u32 wqm_quanta_value = 0xdead; - u32 i = 0; - struct nes_device *nesdev; - - list_for_each_entry(nesdev, &nes_dev_list, list) { - if (i == ee_flsh_adapter) { - wqm_quanta_value = nesdev->nesadapter->wqm_quanta; - break; - } - i++; - } - - return snprintf(buf, PAGE_SIZE, "0x%X\n", wqm_quanta_value); -} - -static ssize_t wqm_quanta_store(struct device_driver *ddp, const char *buf, - size_t count) -{ - unsigned long wqm_quanta_value; - u32 wqm_config1; - u32 i = 0; - struct nes_device *nesdev; - - if (kstrtoul(buf, 0, &wqm_quanta_value) < 0) - return -EINVAL; - - list_for_each_entry(nesdev, &nes_dev_list, list) { - if (i == ee_flsh_adapter) { - nesdev->nesadapter->wqm_quanta = wqm_quanta_value; - wqm_config1 = nes_read_indexed(nesdev, - NES_IDX_WQM_CONFIG1); - nes_write_indexed(nesdev, NES_IDX_WQM_CONFIG1, - ((wqm_quanta_value << 1) | - (wqm_config1 & 0x00000001))); - break; - } - i++; - } - return strnlen(buf, count); -} - -static DRIVER_ATTR_RW(adapter); -static DRIVER_ATTR_RW(eeprom_cmd); -static DRIVER_ATTR_RW(eeprom_data); -static DRIVER_ATTR_RW(flash_cmd); -static DRIVER_ATTR_RW(flash_data); -static DRIVER_ATTR_RW(nonidx_addr); -static DRIVER_ATTR_RW(nonidx_data); -static DRIVER_ATTR_RW(idx_addr); -static DRIVER_ATTR_RW(idx_data); -static DRIVER_ATTR_RW(wqm_quanta); - -static struct attribute *nes_attrs[] = { - &driver_attr_adapter.attr, - &driver_attr_eeprom_cmd.attr, - &driver_attr_eeprom_data.attr, - &driver_attr_flash_cmd.attr, - &driver_attr_flash_data.attr, - &driver_attr_nonidx_addr.attr, - &driver_attr_nonidx_data.attr, - &driver_attr_idx_addr.attr, - &driver_attr_idx_data.attr, - &driver_attr_wqm_quanta.attr, - NULL, -}; -ATTRIBUTE_GROUPS(nes); - -static struct pci_driver nes_pci_driver = { - .name = DRV_NAME, - .id_table = nes_pci_table, - .probe = nes_probe, - .remove = nes_remove, - .groups = nes_groups, -}; - - -/** - * nes_init_module - module initialization entry point - */ -static int __init nes_init_module(void) -{ - int retval; - - retval = nes_cm_start(); - if (retval) { - printk(KERN_ERR PFX "Unable to start NetEffect iWARP CM.\n"); - return retval; - } - return pci_register_driver(&nes_pci_driver); -} - - -/** - * nes_exit_module - module unload entry point - */ -static void __exit nes_exit_module(void) -{ - nes_cm_stop(); - - pci_unregister_driver(&nes_pci_driver); -} - - -module_init(nes_init_module); -module_exit(nes_exit_module); diff --git a/drivers/infiniband/hw/nes/nes.h b/drivers/infiniband/hw/nes/nes.h deleted file mode 100644 index a895fe980d10..000000000000 --- a/drivers/infiniband/hw/nes/nes.h +++ /dev/null @@ -1,574 +0,0 @@ -/* - * Copyright (c) 2006 - 2011 Intel Corporation. All rights reserved. - * Copyright (c) 2005 Open Grid Computing, Inc. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -#ifndef __NES_H -#define __NES_H - -#include <linux/netdevice.h> -#include <linux/inetdevice.h> -#include <linux/interrupt.h> -#include <linux/spinlock.h> -#include <linux/kernel.h> -#include <linux/delay.h> -#include <linux/pci.h> -#include <linux/dma-mapping.h> -#include <linux/workqueue.h> -#include <linux/slab.h> -#include <asm/io.h> -#include <linux/crc32c.h> - -#include <rdma/ib_smi.h> -#include <rdma/ib_verbs.h> -#include <rdma/ib_pack.h> -#include <rdma/rdma_cm.h> -#include <rdma/iw_cm.h> -#include <rdma/rdma_netlink.h> -#include <rdma/iw_portmap.h> - -#define NES_SEND_FIRST_WRITE - -#define QUEUE_DISCONNECTS - -#define DRV_NAME "iw_nes" -#define DRV_VERSION "1.5.0.1" -#define PFX DRV_NAME ": " - -/* - * NetEffect PCI vendor id and NE010 PCI device id. - */ -#ifndef PCI_VENDOR_ID_NETEFFECT /* not in pci.ids yet */ -#define PCI_VENDOR_ID_NETEFFECT 0x1678 -#define PCI_DEVICE_ID_NETEFFECT_NE020 0x0100 -#define PCI_DEVICE_ID_NETEFFECT_NE020_KR 0x0110 -#endif - -#define NE020_REV 4 -#define NE020_REV1 5 - -#define BAR_0 0 -#define BAR_1 2 - -#define RX_BUF_SIZE (1536 + 8) -#define NES_REG0_SIZE (4 * 1024) -#define NES_TX_TIMEOUT (6*HZ) -#define NES_FIRST_QPN 64 -#define NES_SW_CONTEXT_ALIGN 1024 - -#define NES_MAX_MTU 9000 - -#define NES_NIC_MAX_NICS 16 -#define NES_MAX_ARP_TABLE_SIZE 4096 - -#define NES_NIC_CEQ_SIZE 8 -/* NICs will be on a separate CQ */ -#define NES_CCEQ_SIZE ((nesadapter->max_cq / nesadapter->port_count) - 32) - -#define NES_MAX_PORT_COUNT 4 - -#define MAX_DPC_ITERATIONS 128 - -#define NES_DRV_OPT_ENABLE_MPA_VER_0 0x00000001 -#define NES_DRV_OPT_DISABLE_MPA_CRC 0x00000002 -#define NES_DRV_OPT_DISABLE_FIRST_WRITE 0x00000004 -#define NES_DRV_OPT_DISABLE_INTF 0x00000008 -#define NES_DRV_OPT_ENABLE_MSI 0x00000010 -#define NES_DRV_OPT_DUAL_LOGICAL_PORT 0x00000020 -#define NES_DRV_OPT_SUPRESS_OPTION_BC 0x00000040 -#define NES_DRV_OPT_NO_INLINE_DATA 0x00000080 -#define NES_DRV_OPT_DISABLE_INT_MOD 0x00000100 -#define NES_DRV_OPT_DISABLE_VIRT_WQ 0x00000200 -#define NES_DRV_OPT_ENABLE_PAU 0x00000400 - -#define NES_AEQ_EVENT_TIMEOUT 2500 -#define NES_DISCONNECT_EVENT_TIMEOUT 2000 - -/* debug levels */ -/* must match userspace */ -#define NES_DBG_HW 0x00000001 -#define NES_DBG_INIT 0x00000002 -#define NES_DBG_ISR 0x00000004 -#define NES_DBG_PHY 0x00000008 -#define NES_DBG_NETDEV 0x00000010 -#define NES_DBG_CM 0x00000020 -#define NES_DBG_CM1 0x00000040 -#define NES_DBG_NIC_RX 0x00000080 -#define NES_DBG_NIC_TX 0x00000100 -#define NES_DBG_CQP 0x00000200 -#define NES_DBG_MMAP 0x00000400 -#define NES_DBG_MR 0x00000800 -#define NES_DBG_PD 0x00001000 -#define NES_DBG_CQ 0x00002000 -#define NES_DBG_QP 0x00004000 -#define NES_DBG_MOD_QP 0x00008000 -#define NES_DBG_AEQ 0x00010000 -#define NES_DBG_IW_RX 0x00020000 -#define NES_DBG_IW_TX 0x00040000 -#define NES_DBG_SHUTDOWN 0x00080000 -#define NES_DBG_PAU 0x00100000 -#define NES_DBG_NLMSG 0x00200000 -#define NES_DBG_RSVD1 0x10000000 -#define NES_DBG_RSVD2 0x20000000 -#define NES_DBG_RSVD3 0x40000000 -#define NES_DBG_RSVD4 0x80000000 -#define NES_DBG_ALL 0xffffffff - -#ifdef CONFIG_INFINIBAND_NES_DEBUG -#define nes_debug(level, fmt, args...) \ -do { \ - if (level & nes_debug_level) \ - printk(KERN_ERR PFX "%s[%u]: " fmt, __func__, __LINE__, ##args); \ -} while (0) - -#define NES_EVENT_TIMEOUT 1200000 -#else -#define nes_debug(level, fmt, args...) no_printk(fmt, ##args) - -#define NES_EVENT_TIMEOUT 100000 -#endif - -#include "nes_hw.h" -#include "nes_verbs.h" -#include "nes_context.h" -#include <rdma/nes-abi.h> -#include "nes_cm.h" -#include "nes_mgt.h" - -extern int interrupt_mod_interval; -extern int nes_if_count; -extern int mpa_version; -extern int disable_mpa_crc; -extern unsigned int nes_drv_opt; -extern unsigned int nes_debug_level; -extern unsigned int wqm_quanta; -extern struct list_head nes_adapter_list; - -extern atomic_t cm_connects; -extern atomic_t cm_accepts; -extern atomic_t cm_disconnects; -extern atomic_t cm_closes; -extern atomic_t cm_connecteds; -extern atomic_t cm_connect_reqs; -extern atomic_t cm_rejects; -extern atomic_t mod_qp_timouts; -extern atomic_t qps_created; -extern atomic_t qps_destroyed; -extern atomic_t sw_qps_destroyed; -extern u32 mh_detected; -extern u32 mh_pauses_sent; -extern u32 cm_packets_sent; -extern u32 cm_packets_bounced; -extern u32 cm_packets_created; -extern u32 cm_packets_received; -extern u32 cm_packets_dropped; -extern u32 cm_packets_retrans; -extern atomic_t cm_listens_created; -extern atomic_t cm_listens_destroyed; -extern u32 cm_backlog_drops; -extern atomic_t cm_loopbacks; -extern atomic_t cm_nodes_created; -extern atomic_t cm_nodes_destroyed; -extern atomic_t cm_accel_dropped_pkts; -extern atomic_t cm_resets_recvd; -extern atomic_t pau_qps_created; -extern atomic_t pau_qps_destroyed; - -extern u32 int_mod_timer_init; -extern u32 int_mod_cq_depth_256; -extern u32 int_mod_cq_depth_128; -extern u32 int_mod_cq_depth_32; -extern u32 int_mod_cq_depth_24; -extern u32 int_mod_cq_depth_16; -extern u32 int_mod_cq_depth_4; -extern u32 int_mod_cq_depth_1; - -struct nes_device { - struct nes_adapter *nesadapter; - void __iomem *regs; - void __iomem *index_reg; - struct pci_dev *pcidev; - struct net_device *netdev[NES_NIC_MAX_NICS]; - u64 link_status_interrupts; - struct tasklet_struct dpc_tasklet; - spinlock_t indexed_regs_lock; - unsigned long csr_start; - unsigned long doorbell_region; - unsigned long doorbell_start; - unsigned long mac_tx_errors; - unsigned long mac_pause_frames_sent; - unsigned long mac_pause_frames_received; - unsigned long mac_rx_errors; - unsigned long mac_rx_crc_errors; - unsigned long mac_rx_symbol_err_frames; - unsigned long mac_rx_jabber_frames; - unsigned long mac_rx_oversized_frames; - unsigned long mac_rx_short_frames; - unsigned long port_rx_discards; - unsigned long port_tx_discards; - unsigned int mac_index; - unsigned int nes_stack_start; - - /* Control Structures */ - void *cqp_vbase; - dma_addr_t cqp_pbase; - u32 cqp_mem_size; - u8 ceq_index; - u8 nic_ceq_index; - struct nes_hw_cqp cqp; - struct nes_hw_cq ccq; - struct list_head cqp_avail_reqs; - struct list_head cqp_pending_reqs; - struct nes_cqp_request *nes_cqp_requests; - - u32 int_req; - u32 int_stat; - u32 timer_int_req; - u32 timer_only_int_count; - u32 intf_int_req; - u32 last_mac_tx_pauses; - u32 last_used_chunks_tx; - struct list_head list; - - u16 base_doorbell_index; - u16 currcq_count; - u16 deepcq_count; - u8 iw_status; - u8 msi_enabled; - u8 netdev_count; - u8 napi_isr_ran; - u8 disable_rx_flow_control; - u8 disable_tx_flow_control; - - struct delayed_work work; - u8 link_recheck; -}; - -/* Receive skb private area - must fit in skb->cb area */ -struct nes_rskb_cb { - u64 busaddr; - u32 maplen; - u32 seqnum; - u8 *data_start; - struct nes_qp *nesqp; -}; - -static inline __le32 get_crc_value(struct nes_v4_quad *nes_quad) -{ - u32 crc_value; - crc_value = crc32c(~0, (void *)nes_quad, sizeof (struct nes_v4_quad)); - - /* - * With commit ef19454b ("[LIB] crc32c: Keep intermediate crc - * state in cpu order"), behavior of crc32c changes on - * big-endian platforms. Our algorithm expects the previous - * behavior; otherwise we have RDMA connection establishment - * issue on big-endian. - */ - return cpu_to_le32(crc_value); -} - -static inline void -set_wqe_64bit_value(__le32 *wqe_words, u32 index, u64 value) -{ - wqe_words[index] = cpu_to_le32((u32) value); - wqe_words[index + 1] = cpu_to_le32(upper_32_bits(value)); -} - -static inline void -set_wqe_32bit_value(__le32 *wqe_words, u32 index, u32 value) -{ - wqe_words[index] = cpu_to_le32(value); -} - -static inline void -nes_fill_init_cqp_wqe(struct nes_hw_cqp_wqe *cqp_wqe, struct nes_device *nesdev) -{ - cqp_wqe->wqe_words[NES_CQP_WQE_COMP_CTX_LOW_IDX] = 0; - cqp_wqe->wqe_words[NES_CQP_WQE_COMP_CTX_HIGH_IDX] = 0; - cqp_wqe->wqe_words[NES_CQP_WQE_COMP_SCRATCH_LOW_IDX] = 0; - cqp_wqe->wqe_words[NES_CQP_WQE_COMP_SCRATCH_HIGH_IDX] = 0; - cqp_wqe->wqe_words[NES_CQP_STAG_WQE_PBL_BLK_COUNT_IDX] = 0; - cqp_wqe->wqe_words[NES_CQP_STAG_WQE_PBL_LEN_IDX] = 0; - cqp_wqe->wqe_words[NES_CQP_STAG_WQE_LEN_LOW_IDX] = 0; - cqp_wqe->wqe_words[NES_CQP_STAG_WQE_PA_LOW_IDX] = 0; - cqp_wqe->wqe_words[NES_CQP_STAG_WQE_PA_HIGH_IDX] = 0; -} - -static inline void -nes_fill_init_qp_wqe(struct nes_hw_qp_wqe *wqe, struct nes_qp *nesqp, u32 head) -{ - u32 value; - value = ((u32)((unsigned long) nesqp)) | head; - set_wqe_32bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_COMP_CTX_HIGH_IDX, - (u32)(upper_32_bits((unsigned long)(nesqp)))); - set_wqe_32bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_COMP_CTX_LOW_IDX, value); -} - -/* Read from memory-mapped device */ -static inline u32 nes_read_indexed(struct nes_device *nesdev, u32 reg_index) -{ - unsigned long flags; - void __iomem *addr = nesdev->index_reg; - u32 value; - - spin_lock_irqsave(&nesdev->indexed_regs_lock, flags); - - writel(reg_index, addr); - value = readl((void __iomem *)addr + 4); - - spin_unlock_irqrestore(&nesdev->indexed_regs_lock, flags); - return value; -} - -static inline u32 nes_read32(const void __iomem *addr) -{ - return readl(addr); -} - -static inline u16 nes_read16(const void __iomem *addr) -{ - return readw(addr); -} - -static inline u8 nes_read8(const void __iomem *addr) -{ - return readb(addr); -} - -/* Write to memory-mapped device */ -static inline void nes_write_indexed(struct nes_device *nesdev, u32 reg_index, u32 val) -{ - unsigned long flags; - void __iomem *addr = nesdev->index_reg; - - spin_lock_irqsave(&nesdev->indexed_regs_lock, flags); - - writel(reg_index, addr); - writel(val, (void __iomem *)addr + 4); - - spin_unlock_irqrestore(&nesdev->indexed_regs_lock, flags); -} - -static inline void nes_write32(void __iomem *addr, u32 val) -{ - writel(val, addr); -} - -static inline void nes_write16(void __iomem *addr, u16 val) -{ - writew(val, addr); -} - -static inline void nes_write8(void __iomem *addr, u8 val) -{ - writeb(val, addr); -} - -enum nes_resource { - NES_RESOURCE_MW = 1, - NES_RESOURCE_FAST_MR, - NES_RESOURCE_PHYS_MR, - NES_RESOURCE_USER_MR, - NES_RESOURCE_PD, - NES_RESOURCE_QP, - NES_RESOURCE_CQ, - NES_RESOURCE_ARP -}; - -static inline int nes_alloc_resource(struct nes_adapter *nesadapter, - unsigned long *resource_array, u32 max_resources, - u32 *req_resource_num, u32 *next, enum nes_resource resource_type) -{ - unsigned long flags; - u32 resource_num; - - spin_lock_irqsave(&nesadapter->resource_lock, flags); - - resource_num = find_next_zero_bit(resource_array, max_resources, *next); - if (resource_num >= max_resources) { - resource_num = find_first_zero_bit(resource_array, max_resources); - if (resource_num >= max_resources) { - printk(KERN_ERR PFX "%s: No available resources [type=%u].\n", __func__, resource_type); - spin_unlock_irqrestore(&nesadapter->resource_lock, flags); - return -EMFILE; - } - } - set_bit(resource_num, resource_array); - *next = resource_num+1; - if (*next == max_resources) { - *next = 0; - } - spin_unlock_irqrestore(&nesadapter->resource_lock, flags); - *req_resource_num = resource_num; - - return 0; -} - -static inline int nes_is_resource_allocated(struct nes_adapter *nesadapter, - unsigned long *resource_array, u32 resource_num) -{ - unsigned long flags; - int bit_is_set; - - spin_lock_irqsave(&nesadapter->resource_lock, flags); - - bit_is_set = test_bit(resource_num, resource_array); - nes_debug(NES_DBG_HW, "resource_num %u is%s allocated.\n", - resource_num, (bit_is_set ? "": " not")); - spin_unlock_irqrestore(&nesadapter->resource_lock, flags); - - return bit_is_set; -} - -static inline void nes_free_resource(struct nes_adapter *nesadapter, - unsigned long *resource_array, u32 resource_num) -{ - unsigned long flags; - - spin_lock_irqsave(&nesadapter->resource_lock, flags); - clear_bit(resource_num, resource_array); - spin_unlock_irqrestore(&nesadapter->resource_lock, flags); -} - -static inline struct nes_vnic *to_nesvnic(struct ib_device *ibdev) -{ - return container_of(ibdev, struct nes_ib_device, ibdev)->nesvnic; -} - -static inline struct nes_pd *to_nespd(struct ib_pd *ibpd) -{ - return container_of(ibpd, struct nes_pd, ibpd); -} - -static inline struct nes_ucontext *to_nesucontext(struct ib_ucontext *ibucontext) -{ - return container_of(ibucontext, struct nes_ucontext, ibucontext); -} - -static inline struct nes_mr *to_nesmr(struct ib_mr *ibmr) -{ - return container_of(ibmr, struct nes_mr, ibmr); -} - -static inline struct nes_mr *to_nesmr_from_ibfmr(struct ib_fmr *ibfmr) -{ - return container_of(ibfmr, struct nes_mr, ibfmr); -} - -static inline struct nes_mr *to_nesmw(struct ib_mw *ibmw) -{ - return container_of(ibmw, struct nes_mr, ibmw); -} - -static inline struct nes_fmr *to_nesfmr(struct nes_mr *nesmr) -{ - return container_of(nesmr, struct nes_fmr, nesmr); -} - -static inline struct nes_cq *to_nescq(struct ib_cq *ibcq) -{ - return container_of(ibcq, struct nes_cq, ibcq); -} - -static inline struct nes_qp *to_nesqp(struct ib_qp *ibqp) -{ - return container_of(ibqp, struct nes_qp, ibqp); -} - - - -/* nes.c */ -void nes_add_ref(struct ib_qp *); -void nes_rem_ref(struct ib_qp *); -struct ib_qp *nes_get_qp(struct ib_device *, int); - - -/* nes_hw.c */ -struct nes_adapter *nes_init_adapter(struct nes_device *, u8); -void nes_nic_init_timer_defaults(struct nes_device *, u8); -void nes_destroy_adapter(struct nes_adapter *); -int nes_init_cqp(struct nes_device *); -int nes_init_phy(struct nes_device *); -int nes_init_nic_qp(struct nes_device *, struct net_device *); -void nes_destroy_nic_qp(struct nes_vnic *); -int nes_napi_isr(struct nes_device *); -void nes_dpc(unsigned long); -void nes_nic_ce_handler(struct nes_device *, struct nes_hw_nic_cq *); -void nes_iwarp_ce_handler(struct nes_device *, struct nes_hw_cq *); -int nes_destroy_cqp(struct nes_device *); -int nes_nic_cm_xmit(struct sk_buff *, struct net_device *); -void nes_recheck_link_status(struct work_struct *work); -void nes_terminate_timeout(struct timer_list *t); - -/* nes_nic.c */ -struct net_device *nes_netdev_init(struct nes_device *, void __iomem *); -void nes_netdev_destroy(struct net_device *); -int nes_nic_cm_xmit(struct sk_buff *, struct net_device *); - -/* nes_cm.c */ -void *nes_cm_create(struct net_device *); -int nes_cm_recv(struct sk_buff *, struct net_device *); -void nes_update_arp(unsigned char *, u32, u32, u16, u16); -void nes_manage_arp_cache(struct net_device *, unsigned char *, u32, u32); -void nes_sock_release(struct nes_qp *, unsigned long *); -void flush_wqes(struct nes_device *nesdev, struct nes_qp *, u32, u32); -int nes_manage_apbvt(struct nes_vnic *, u32, u32, u32); -int nes_cm_disconn(struct nes_qp *); -void nes_cm_disconn_worker(void *); - -/* nes_verbs.c */ -int nes_hw_modify_qp(struct nes_device *, struct nes_qp *, u32, u32, u32); -int nes_modify_qp(struct ib_qp *, struct ib_qp_attr *, int, struct ib_udata *); -struct nes_ib_device *nes_init_ofa_device(struct net_device *); -void nes_port_ibevent(struct nes_vnic *nesvnic); -void nes_destroy_ofa_device(struct nes_ib_device *); -int nes_register_ofa_device(struct nes_ib_device *); - -/* nes_util.c */ -int nes_read_eeprom_values(struct nes_device *, struct nes_adapter *); -void nes_write_1G_phy_reg(struct nes_device *, u8, u8, u16); -void nes_read_1G_phy_reg(struct nes_device *, u8, u8, u16 *); -void nes_write_10G_phy_reg(struct nes_device *, u16, u8, u16, u16); -void nes_read_10G_phy_reg(struct nes_device *, u8, u8, u16); -struct nes_cqp_request *nes_get_cqp_request(struct nes_device *); -void nes_free_cqp_request(struct nes_device *nesdev, - struct nes_cqp_request *cqp_request); -void nes_put_cqp_request(struct nes_device *nesdev, - struct nes_cqp_request *cqp_request); -void nes_post_cqp_request(struct nes_device *, struct nes_cqp_request *); -int nes_arp_table(struct nes_device *, u32, u8 *, u32); -void nes_mh_fix(struct timer_list *t); -void nes_clc(struct timer_list *t); -void nes_dump_mem(unsigned int, void *, int); -u32 nes_crc32(u32, u32, u32, u32, u8 *, u32, u32, u32); - -#endif /* __NES_H */ diff --git a/drivers/infiniband/hw/nes/nes_cm.c b/drivers/infiniband/hw/nes/nes_cm.c deleted file mode 100644 index 62bf986eba67..000000000000 --- a/drivers/infiniband/hw/nes/nes_cm.c +++ /dev/null @@ -1,3992 +0,0 @@ -/* - * Copyright (c) 2006 - 2014 Intel Corporation. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - */ - - -#define TCPOPT_TIMESTAMP 8 - -#include <linux/atomic.h> -#include <linux/skbuff.h> -#include <linux/ip.h> -#include <linux/tcp.h> -#include <linux/init.h> -#include <linux/if_arp.h> -#include <linux/if_vlan.h> -#include <linux/notifier.h> -#include <linux/net.h> -#include <linux/types.h> -#include <linux/timer.h> -#include <linux/time.h> -#include <linux/delay.h> -#include <linux/etherdevice.h> -#include <linux/netdevice.h> -#include <linux/random.h> -#include <linux/list.h> -#include <linux/threads.h> -#include <linux/highmem.h> -#include <linux/slab.h> -#include <net/arp.h> -#include <net/neighbour.h> -#include <net/route.h> -#include <net/ip_fib.h> -#include <net/secure_seq.h> -#include <net/tcp.h> -#include <linux/fcntl.h> - -#include "nes.h" - -u32 cm_packets_sent; -u32 cm_packets_bounced; -u32 cm_packets_dropped; -u32 cm_packets_retrans; -u32 cm_packets_created; -u32 cm_packets_received; -atomic_t cm_listens_created; -atomic_t cm_listens_destroyed; -u32 cm_backlog_drops; -atomic_t cm_loopbacks; -atomic_t cm_nodes_created; -atomic_t cm_nodes_destroyed; -atomic_t cm_accel_dropped_pkts; -atomic_t cm_resets_recvd; - -static inline int mini_cm_accelerated(struct nes_cm_core *, struct nes_cm_node *); -static struct nes_cm_listener *mini_cm_listen(struct nes_cm_core *, struct nes_vnic *, struct nes_cm_info *); -static int mini_cm_del_listen(struct nes_cm_core *, struct nes_cm_listener *); -static struct nes_cm_node *mini_cm_connect(struct nes_cm_core *, struct nes_vnic *, u16, void *, struct nes_cm_info *); -static int mini_cm_close(struct nes_cm_core *, struct nes_cm_node *); -static int mini_cm_accept(struct nes_cm_core *, struct nes_cm_node *); -static int mini_cm_reject(struct nes_cm_core *, struct nes_cm_node *); -static int mini_cm_recv_pkt(struct nes_cm_core *, struct nes_vnic *, struct sk_buff *); -static int mini_cm_dealloc_core(struct nes_cm_core *); -static int mini_cm_get(struct nes_cm_core *); -static int mini_cm_set(struct nes_cm_core *, u32, u32); - -static void form_cm_frame(struct sk_buff *, struct nes_cm_node *, void *, u32, void *, u32, u8); -static int add_ref_cm_node(struct nes_cm_node *); -static int rem_ref_cm_node(struct nes_cm_core *, struct nes_cm_node *); - -static int nes_cm_disconn_true(struct nes_qp *); -static int nes_cm_post_event(struct nes_cm_event *event); -static int nes_disconnect(struct nes_qp *nesqp, int abrupt); -static void nes_disconnect_worker(struct work_struct *work); - -static int send_mpa_request(struct nes_cm_node *, struct sk_buff *); -static int send_mpa_reject(struct nes_cm_node *); -static int send_syn(struct nes_cm_node *, u32, struct sk_buff *); -static int send_reset(struct nes_cm_node *, struct sk_buff *); -static int send_ack(struct nes_cm_node *cm_node, struct sk_buff *skb); -static int send_fin(struct nes_cm_node *cm_node, struct sk_buff *skb); -static void process_packet(struct nes_cm_node *, struct sk_buff *, struct nes_cm_core *); - -static void active_open_err(struct nes_cm_node *, struct sk_buff *, int); -static void passive_open_err(struct nes_cm_node *, struct sk_buff *, int); -static void cleanup_retrans_entry(struct nes_cm_node *); -static void handle_rcv_mpa(struct nes_cm_node *, struct sk_buff *); -static void free_retrans_entry(struct nes_cm_node *cm_node); -static int handle_tcp_options(struct nes_cm_node *cm_node, struct tcphdr *tcph, struct sk_buff *skb, int optionsize, int passive); - -/* CM event handler functions */ -static void cm_event_connected(struct nes_cm_event *); -static void cm_event_connect_error(struct nes_cm_event *); -static void cm_event_reset(struct nes_cm_event *); -static void cm_event_mpa_req(struct nes_cm_event *); -static void cm_event_mpa_reject(struct nes_cm_event *); -static void handle_recv_entry(struct nes_cm_node *cm_node, u32 rem_node); - -/* MPA build functions */ -static int cm_build_mpa_frame(struct nes_cm_node *, u8 **, u16 *, u8 *, u8); -static void build_mpa_v2(struct nes_cm_node *, void *, u8); -static void build_mpa_v1(struct nes_cm_node *, void *, u8); -static void build_rdma0_msg(struct nes_cm_node *, struct nes_qp **); - -static void print_core(struct nes_cm_core *core); -static void record_ird_ord(struct nes_cm_node *, u16, u16); - -/* External CM API Interface */ -/* instance of function pointers for client API */ -/* set address of this instance to cm_core->cm_ops at cm_core alloc */ -static const struct nes_cm_ops nes_cm_api = { - .accelerated = mini_cm_accelerated, - .listen = mini_cm_listen, - .stop_listener = mini_cm_del_listen, - .connect = mini_cm_connect, - .close = mini_cm_close, - .accept = mini_cm_accept, - .reject = mini_cm_reject, - .recv_pkt = mini_cm_recv_pkt, - .destroy_cm_core = mini_cm_dealloc_core, - .get = mini_cm_get, - .set = mini_cm_set -}; - -static struct nes_cm_core *g_cm_core; - -atomic_t cm_connects; -atomic_t cm_accepts; -atomic_t cm_disconnects; -atomic_t cm_closes; -atomic_t cm_connecteds; -atomic_t cm_connect_reqs; -atomic_t cm_rejects; - -int nes_add_ref_cm_node(struct nes_cm_node *cm_node) -{ - return add_ref_cm_node(cm_node); -} - -int nes_rem_ref_cm_node(struct nes_cm_node *cm_node) -{ - return rem_ref_cm_node(cm_node->cm_core, cm_node); -} -/** - * create_event - */ -static struct nes_cm_event *create_event(struct nes_cm_node * cm_node, - enum nes_cm_event_type type) -{ - struct nes_cm_event *event; - - if (!cm_node->cm_id) - return NULL; - - /* allocate an empty event */ - event = kzalloc(sizeof(*event), GFP_ATOMIC); - - if (!event) - return NULL; - - event->type = type; - event->cm_node = cm_node; - event->cm_info.rem_addr = cm_node->rem_addr; - event->cm_info.loc_addr = cm_node->loc_addr; - event->cm_info.rem_port = cm_node->rem_port; - event->cm_info.loc_port = cm_node->loc_port; - event->cm_info.cm_id = cm_node->cm_id; - - nes_debug(NES_DBG_CM, "cm_node=%p Created event=%p, type=%u, " - "dst_addr=%08x[%x], src_addr=%08x[%x]\n", - cm_node, event, type, event->cm_info.loc_addr, - event->cm_info.loc_port, event->cm_info.rem_addr, - event->cm_info.rem_port); - - nes_cm_post_event(event); - return event; -} - - -/** - * send_mpa_request - */ -static int send_mpa_request(struct nes_cm_node *cm_node, struct sk_buff *skb) -{ - u8 start_addr = 0; - u8 *start_ptr = &start_addr; - u8 **start_buff = &start_ptr; - u16 buff_len = 0; - - if (!skb) { - nes_debug(NES_DBG_CM, "skb set to NULL\n"); - return -1; - } - - /* send an MPA Request frame */ - cm_build_mpa_frame(cm_node, start_buff, &buff_len, NULL, MPA_KEY_REQUEST); - form_cm_frame(skb, cm_node, NULL, 0, *start_buff, buff_len, SET_ACK); - - return schedule_nes_timer(cm_node, skb, NES_TIMER_TYPE_SEND, 1, 0); -} - - - -static int send_mpa_reject(struct nes_cm_node *cm_node) -{ - struct sk_buff *skb = NULL; - u8 start_addr = 0; - u8 *start_ptr = &start_addr; - u8 **start_buff = &start_ptr; - u16 buff_len = 0; - struct ietf_mpa_v1 *mpa_frame; - - skb = dev_alloc_skb(MAX_CM_BUFFER); - if (!skb) { - nes_debug(NES_DBG_CM, "Failed to get a Free pkt\n"); - return -ENOMEM; - } - - /* send an MPA reject frame */ - cm_build_mpa_frame(cm_node, start_buff, &buff_len, NULL, MPA_KEY_REPLY); - mpa_frame = (struct ietf_mpa_v1 *)*start_buff; - mpa_frame->flags |= IETF_MPA_FLAGS_REJECT; - form_cm_frame(skb, cm_node, NULL, 0, *start_buff, buff_len, SET_ACK | SET_FIN); - - cm_node->state = NES_CM_STATE_FIN_WAIT1; - return schedule_nes_timer(cm_node, skb, NES_TIMER_TYPE_SEND, 1, 0); -} - - -/** - * recv_mpa - process a received TCP pkt, we are expecting an - * IETF MPA frame - */ -static int parse_mpa(struct nes_cm_node *cm_node, u8 *buffer, u32 *type, - u32 len) -{ - struct ietf_mpa_v1 *mpa_frame; - struct ietf_mpa_v2 *mpa_v2_frame; - struct ietf_rtr_msg *rtr_msg; - int mpa_hdr_len; - int priv_data_len; - - *type = NES_MPA_REQUEST_ACCEPT; - - /* assume req frame is in tcp data payload */ - if (len < sizeof(struct ietf_mpa_v1)) { - nes_debug(NES_DBG_CM, "The received ietf buffer was too small (%x)\n", len); - return -EINVAL; - } - - /* points to the beginning of the frame, which could be MPA V1 or V2 */ - mpa_frame = (struct ietf_mpa_v1 *)buffer; - mpa_hdr_len = sizeof(struct ietf_mpa_v1); - priv_data_len = ntohs(mpa_frame->priv_data_len); - - /* make sure mpa private data len is less than 512 bytes */ - if (priv_data_len > IETF_MAX_PRIV_DATA_LEN) { - nes_debug(NES_DBG_CM, "The received Length of Private" - " Data field exceeds 512 octets\n"); - return -EINVAL; - } - /* - * make sure MPA receiver interoperate with the - * received MPA version and MPA key information - * - */ - if (mpa_frame->rev != IETF_MPA_V1 && mpa_frame->rev != IETF_MPA_V2) { - nes_debug(NES_DBG_CM, "The received mpa version" - " is not supported\n"); - return -EINVAL; - } - /* - * backwards compatibility only - */ - if (mpa_frame->rev > cm_node->mpa_frame_rev) { - nes_debug(NES_DBG_CM, "The received mpa version" - " can not be interoperated\n"); - return -EINVAL; - } else { - cm_node->mpa_frame_rev = mpa_frame->rev; - } - - if (cm_node->state != NES_CM_STATE_MPAREQ_SENT) { - if (memcmp(mpa_frame->key, IEFT_MPA_KEY_REQ, IETF_MPA_KEY_SIZE)) { - nes_debug(NES_DBG_CM, "Unexpected MPA Key received \n"); - return -EINVAL; - } - } else { - if (memcmp(mpa_frame->key, IEFT_MPA_KEY_REP, IETF_MPA_KEY_SIZE)) { - nes_debug(NES_DBG_CM, "Unexpected MPA Key received \n"); - return -EINVAL; - } - } - - if (priv_data_len + mpa_hdr_len != len) { - nes_debug(NES_DBG_CM, "The received ietf buffer was not right" - " complete (%x + %x != %x)\n", - priv_data_len, mpa_hdr_len, len); - return -EINVAL; - } - /* make sure it does not exceed the max size */ - if (len > MAX_CM_BUFFER) { - nes_debug(NES_DBG_CM, "The received ietf buffer was too large" - " (%x + %x != %x)\n", - priv_data_len, mpa_hdr_len, len); - return -EINVAL; - } - - cm_node->mpa_frame_size = priv_data_len; - - switch (mpa_frame->rev) { - case IETF_MPA_V2: { - u16 ird_size; - u16 ord_size; - u16 rtr_ctrl_ird; - u16 rtr_ctrl_ord; - - mpa_v2_frame = (struct ietf_mpa_v2 *)buffer; - mpa_hdr_len += IETF_RTR_MSG_SIZE; - cm_node->mpa_frame_size -= IETF_RTR_MSG_SIZE; - rtr_msg = &mpa_v2_frame->rtr_msg; - - /* parse rtr message */ - rtr_ctrl_ird = ntohs(rtr_msg->ctrl_ird); - rtr_ctrl_ord = ntohs(rtr_msg->ctrl_ord); - ird_size = rtr_ctrl_ird & IETF_NO_IRD_ORD; - ord_size = rtr_ctrl_ord & IETF_NO_IRD_ORD; - - if (!(rtr_ctrl_ird & IETF_PEER_TO_PEER)) { - /* send reset */ - return -EINVAL; - } - if (ird_size == IETF_NO_IRD_ORD || ord_size == IETF_NO_IRD_ORD) - cm_node->mpav2_ird_ord = IETF_NO_IRD_ORD; - - if (cm_node->mpav2_ird_ord != IETF_NO_IRD_ORD) { - /* responder */ - if (cm_node->state != NES_CM_STATE_MPAREQ_SENT) { - /* we are still negotiating */ - if (ord_size > NES_MAX_IRD) { - cm_node->ird_size = NES_MAX_IRD; - } else { - cm_node->ird_size = ord_size; - if (ord_size == 0 && - (rtr_ctrl_ord & IETF_RDMA0_READ)) { - cm_node->ird_size = 1; - nes_debug(NES_DBG_CM, - "%s: Remote peer doesn't support RDMA0_READ (ord=%u)\n", - __func__, ord_size); - } - } - if (ird_size > NES_MAX_ORD) - cm_node->ord_size = NES_MAX_ORD; - else - cm_node->ord_size = ird_size; - } else { /* initiator */ - if (ord_size > NES_MAX_IRD) { - nes_debug(NES_DBG_CM, - "%s: Unable to support the requested (ord =%u)\n", - __func__, ord_size); - return -EINVAL; - } - cm_node->ird_size = ord_size; - - if (ird_size > NES_MAX_ORD) { - cm_node->ord_size = NES_MAX_ORD; - } else { - if (ird_size == 0 && - (rtr_ctrl_ord & IETF_RDMA0_READ)) { - nes_debug(NES_DBG_CM, - "%s: Remote peer doesn't support RDMA0_READ (ird=%u)\n", - __func__, ird_size); - return -EINVAL; - } else { - cm_node->ord_size = ird_size; - } - } - } - } - - if (rtr_ctrl_ord & IETF_RDMA0_READ) { - cm_node->send_rdma0_op = SEND_RDMA_READ_ZERO; - - } else if (rtr_ctrl_ord & IETF_RDMA0_WRITE) { - cm_node->send_rdma0_op = SEND_RDMA_WRITE_ZERO; - } else { /* Not supported RDMA0 operation */ - return -EINVAL; - } - break; - } - case IETF_MPA_V1: - default: - break; - } - - /* copy entire MPA frame to our cm_node's frame */ - memcpy(cm_node->mpa_frame_buf, buffer + mpa_hdr_len, cm_node->mpa_frame_size); - - if (mpa_frame->flags & IETF_MPA_FLAGS_REJECT) - *type = NES_MPA_REQUEST_REJECT; - return 0; -} - - -/** - * form_cm_frame - get a free packet and build empty frame Use - * node info to build. - */ -static void form_cm_frame(struct sk_buff *skb, - struct nes_cm_node *cm_node, void *options, u32 optionsize, - void *data, u32 datasize, u8 flags) -{ - struct tcphdr *tcph; - struct iphdr *iph; - struct ethhdr *ethh; - u8 *buf; - u16 packetsize = sizeof(*iph); - - packetsize += sizeof(*tcph); - packetsize += optionsize + datasize; - - skb_trim(skb, 0); - memset(skb->data, 0x00, ETH_HLEN + sizeof(*iph) + sizeof(*tcph)); - - buf = skb_put(skb, packetsize + ETH_HLEN); - - ethh = (struct ethhdr *)buf; - buf += ETH_HLEN; - - iph = (struct iphdr *)buf; - buf += sizeof(*iph); - tcph = (struct tcphdr *)buf; - skb_reset_mac_header(skb); - skb_set_network_header(skb, ETH_HLEN); - skb_set_transport_header(skb, ETH_HLEN + sizeof(*iph)); - buf += sizeof(*tcph); - - skb->ip_summed = CHECKSUM_PARTIAL; - if (!(cm_node->netdev->features & NETIF_F_IP_CSUM)) - skb->ip_summed = CHECKSUM_NONE; - skb->protocol = htons(0x800); - skb->data_len = 0; - skb->mac_len = ETH_HLEN; - - memcpy(ethh->h_dest, cm_node->rem_mac, ETH_ALEN); - memcpy(ethh->h_source, cm_node->loc_mac, ETH_ALEN); - ethh->h_proto = htons(0x0800); - - iph->version = IPVERSION; - iph->ihl = 5; /* 5 * 4Byte words, IP headr len */ - iph->tos = 0; - iph->tot_len = htons(packetsize); - iph->id = htons(++cm_node->tcp_cntxt.loc_id); - - iph->frag_off = htons(0x4000); - iph->ttl = 0x40; - iph->protocol = 0x06; /* IPPROTO_TCP */ - - iph->saddr = htonl(cm_node->loc_addr); - iph->daddr = htonl(cm_node->rem_addr); - - tcph->source = htons(cm_node->loc_port); - tcph->dest = htons(cm_node->rem_port); - tcph->seq = htonl(cm_node->tcp_cntxt.loc_seq_num); - - if (flags & SET_ACK) { - cm_node->tcp_cntxt.loc_ack_num = cm_node->tcp_cntxt.rcv_nxt; - tcph->ack_seq = htonl(cm_node->tcp_cntxt.loc_ack_num); - tcph->ack = 1; - } else { - tcph->ack_seq = 0; - } - - if (flags & SET_SYN) { - cm_node->tcp_cntxt.loc_seq_num++; - tcph->syn = 1; - } else { - cm_node->tcp_cntxt.loc_seq_num += datasize; - } - - if (flags & SET_FIN) { - cm_node->tcp_cntxt.loc_seq_num++; - tcph->fin = 1; - } - - if (flags & SET_RST) - tcph->rst = 1; - - tcph->doff = (u16)((sizeof(*tcph) + optionsize + 3) >> 2); - tcph->window = htons(cm_node->tcp_cntxt.rcv_wnd); - tcph->urg_ptr = 0; - if (optionsize) - memcpy(buf, options, optionsize); - buf += optionsize; - if (datasize) - memcpy(buf, data, datasize); - - skb_shinfo(skb)->nr_frags = 0; - cm_packets_created++; -} - -/** - * print_core - dump a cm core - */ -static void print_core(struct nes_cm_core *core) -{ - nes_debug(NES_DBG_CM, "---------------------------------------------\n"); - nes_debug(NES_DBG_CM, "CM Core -- (core = %p )\n", core); - if (!core) - return; - nes_debug(NES_DBG_CM, "---------------------------------------------\n"); - - nes_debug(NES_DBG_CM, "State : %u \n", core->state); - - nes_debug(NES_DBG_CM, "Listen Nodes : %u \n", atomic_read(&core->listen_node_cnt)); - nes_debug(NES_DBG_CM, "Active Nodes : %u \n", atomic_read(&core->node_cnt)); - - nes_debug(NES_DBG_CM, "core : %p \n", core); - - nes_debug(NES_DBG_CM, "-------------- end core ---------------\n"); -} - -static void record_ird_ord(struct nes_cm_node *cm_node, - u16 conn_ird, u16 conn_ord) -{ - if (conn_ird > NES_MAX_IRD) - conn_ird = NES_MAX_IRD; - - if (conn_ord > NES_MAX_ORD) - conn_ord = NES_MAX_ORD; - - cm_node->ird_size = conn_ird; - cm_node->ord_size = conn_ord; -} - -/** - * cm_build_mpa_frame - build a MPA V1 frame or MPA V2 frame - */ -static int cm_build_mpa_frame(struct nes_cm_node *cm_node, u8 **start_buff, - u16 *buff_len, u8 *pci_mem, u8 mpa_key) -{ - int ret = 0; - - *start_buff = (pci_mem) ? pci_mem : &cm_node->mpa_frame_buf[0]; - - switch (cm_node->mpa_frame_rev) { - case IETF_MPA_V1: - *start_buff = (u8 *)*start_buff + sizeof(struct ietf_rtr_msg); - *buff_len = sizeof(struct ietf_mpa_v1) + cm_node->mpa_frame_size; - build_mpa_v1(cm_node, *start_buff, mpa_key); - break; - case IETF_MPA_V2: - *buff_len = sizeof(struct ietf_mpa_v2) + cm_node->mpa_frame_size; - build_mpa_v2(cm_node, *start_buff, mpa_key); - break; - default: - ret = -EINVAL; - } - return ret; -} - -/** - * build_mpa_v2 - build a MPA V2 frame - */ -static void build_mpa_v2(struct nes_cm_node *cm_node, - void *start_addr, u8 mpa_key) -{ - struct ietf_mpa_v2 *mpa_frame = (struct ietf_mpa_v2 *)start_addr; - struct ietf_rtr_msg *rtr_msg = &mpa_frame->rtr_msg; - u16 ctrl_ird; - u16 ctrl_ord; - - /* initialize the upper 5 bytes of the frame */ - build_mpa_v1(cm_node, start_addr, mpa_key); - mpa_frame->flags |= IETF_MPA_V2_FLAG; /* set a bit to indicate MPA V2 */ - mpa_frame->priv_data_len += htons(IETF_RTR_MSG_SIZE); - - /* initialize RTR msg */ - if (cm_node->mpav2_ird_ord == IETF_NO_IRD_ORD) { - ctrl_ird = IETF_NO_IRD_ORD; - ctrl_ord = IETF_NO_IRD_ORD; - } else { - ctrl_ird = cm_node->ird_size & IETF_NO_IRD_ORD; - ctrl_ord = cm_node->ord_size & IETF_NO_IRD_ORD; - } - ctrl_ird |= IETF_PEER_TO_PEER; - - switch (mpa_key) { - case MPA_KEY_REQUEST: - ctrl_ord |= IETF_RDMA0_WRITE; - ctrl_ord |= IETF_RDMA0_READ; - break; - case MPA_KEY_REPLY: - switch (cm_node->send_rdma0_op) { - case SEND_RDMA_WRITE_ZERO: - ctrl_ord |= IETF_RDMA0_WRITE; - break; - case SEND_RDMA_READ_ZERO: - ctrl_ord |= IETF_RDMA0_READ; - break; - } - } - rtr_msg->ctrl_ird = htons(ctrl_ird); - rtr_msg->ctrl_ord = htons(ctrl_ord); -} - -/** - * build_mpa_v1 - build a MPA V1 frame - */ -static void build_mpa_v1(struct nes_cm_node *cm_node, void *start_addr, u8 mpa_key) -{ - struct ietf_mpa_v1 *mpa_frame = (struct ietf_mpa_v1 *)start_addr; - - switch (mpa_key) { - case MPA_KEY_REQUEST: - memcpy(mpa_frame->key, IEFT_MPA_KEY_REQ, IETF_MPA_KEY_SIZE); - break; - case MPA_KEY_REPLY: - memcpy(mpa_frame->key, IEFT_MPA_KEY_REP, IETF_MPA_KEY_SIZE); - break; - } - mpa_frame->flags = IETF_MPA_FLAGS_CRC; - mpa_frame->rev = cm_node->mpa_frame_rev; - mpa_frame->priv_data_len = htons(cm_node->mpa_frame_size); -} - -static void build_rdma0_msg(struct nes_cm_node *cm_node, struct nes_qp **nesqp_addr) -{ - u64 u64temp; - struct nes_qp *nesqp = *nesqp_addr; - struct nes_hw_qp_wqe *wqe = &nesqp->hwqp.sq_vbase[0]; - - u64temp = (unsigned long)nesqp->nesuqp_addr; - u64temp |= NES_SW_CONTEXT_ALIGN >> 1; - set_wqe_64bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_COMP_CTX_LOW_IDX, u64temp); - - wqe->wqe_words[NES_IWARP_SQ_WQE_FRAG0_LOW_IDX] = 0; - wqe->wqe_words[NES_IWARP_SQ_WQE_FRAG0_HIGH_IDX] = 0; - - switch (cm_node->send_rdma0_op) { - case SEND_RDMA_WRITE_ZERO: - nes_debug(NES_DBG_CM, "Sending first write.\n"); - wqe->wqe_words[NES_IWARP_SQ_WQE_MISC_IDX] = - cpu_to_le32(NES_IWARP_SQ_OP_RDMAW); - wqe->wqe_words[NES_IWARP_SQ_WQE_TOTAL_PAYLOAD_IDX] = 0; - wqe->wqe_words[NES_IWARP_SQ_WQE_LENGTH0_IDX] = 0; - wqe->wqe_words[NES_IWARP_SQ_WQE_STAG0_IDX] = 0; - break; - - case SEND_RDMA_READ_ZERO: - default: - if (cm_node->send_rdma0_op != SEND_RDMA_READ_ZERO) - WARN(1, "Unsupported RDMA0 len operation=%u\n", - cm_node->send_rdma0_op); - nes_debug(NES_DBG_CM, "Sending first rdma operation.\n"); - wqe->wqe_words[NES_IWARP_SQ_WQE_MISC_IDX] = - cpu_to_le32(NES_IWARP_SQ_OP_RDMAR); - wqe->wqe_words[NES_IWARP_SQ_WQE_RDMA_TO_LOW_IDX] = 1; - wqe->wqe_words[NES_IWARP_SQ_WQE_RDMA_TO_HIGH_IDX] = 0; - wqe->wqe_words[NES_IWARP_SQ_WQE_RDMA_LENGTH_IDX] = 0; - wqe->wqe_words[NES_IWARP_SQ_WQE_RDMA_STAG_IDX] = 1; - wqe->wqe_words[NES_IWARP_SQ_WQE_STAG0_IDX] = 1; - break; - } - - if (nesqp->sq_kmapped) { - nesqp->sq_kmapped = 0; - kunmap(nesqp->page); - } - - /*use the reserved spot on the WQ for the extra first WQE*/ - nesqp->nesqp_context->ird_ord_sizes &= cpu_to_le32(~(NES_QPCONTEXT_ORDIRD_LSMM_PRESENT | - NES_QPCONTEXT_ORDIRD_WRPDU | - NES_QPCONTEXT_ORDIRD_ALSMM)); - nesqp->skip_lsmm = 1; - nesqp->hwqp.sq_tail = 0; -} - -/** - * schedule_nes_timer - * note - cm_node needs to be protected before calling this. Encase in: - * rem_ref_cm_node(cm_core, cm_node);add_ref_cm_node(cm_node); - */ -int schedule_nes_timer(struct nes_cm_node *cm_node, struct sk_buff *skb, - enum nes_timer_type type, int send_retrans, - int close_when_complete) -{ - unsigned long flags; - struct nes_cm_core *cm_core = cm_node->cm_core; - struct nes_timer_entry *new_send; - int ret = 0; - - new_send = kzalloc(sizeof(*new_send), GFP_ATOMIC); - if (!new_send) - return -ENOMEM; - - /* new_send->timetosend = currenttime */ - new_send->retrycount = NES_DEFAULT_RETRYS; - new_send->retranscount = NES_DEFAULT_RETRANS; - new_send->skb = skb; - new_send->timetosend = jiffies; - new_send->type = type; - new_send->netdev = cm_node->netdev; - new_send->send_retrans = send_retrans; - new_send->close_when_complete = close_when_complete; - - if (type == NES_TIMER_TYPE_CLOSE) { - new_send->timetosend += (HZ / 10); - if (cm_node->recv_entry) { - kfree(new_send); - WARN_ON(1); - return -EINVAL; - } - cm_node->recv_entry = new_send; - } - - if (type == NES_TIMER_TYPE_SEND) { - new_send->seq_num = ntohl(tcp_hdr(skb)->seq); - refcount_inc(&new_send->skb->users); - spin_lock_irqsave(&cm_node->retrans_list_lock, flags); - cm_node->send_entry = new_send; - add_ref_cm_node(cm_node); - spin_unlock_irqrestore(&cm_node->retrans_list_lock, flags); - new_send->timetosend = jiffies + NES_RETRY_TIMEOUT; - - ret = nes_nic_cm_xmit(new_send->skb, cm_node->netdev); - if (ret != NETDEV_TX_OK) { - nes_debug(NES_DBG_CM, "Error sending packet %p " - "(jiffies = %lu)\n", new_send, jiffies); - new_send->timetosend = jiffies; - ret = NETDEV_TX_OK; - } else { - cm_packets_sent++; - if (!send_retrans) { - cleanup_retrans_entry(cm_node); - if (close_when_complete) - rem_ref_cm_node(cm_core, cm_node); - return ret; - } - } - } - - if (!timer_pending(&cm_core->tcp_timer)) - mod_timer(&cm_core->tcp_timer, new_send->timetosend); - - return ret; -} - -static void nes_retrans_expired(struct nes_cm_node *cm_node) -{ - struct iw_cm_id *cm_id = cm_node->cm_id; - enum nes_cm_node_state state = cm_node->state; - cm_node->state = NES_CM_STATE_CLOSED; - - switch (state) { - case NES_CM_STATE_SYN_RCVD: - case NES_CM_STATE_CLOSING: - rem_ref_cm_node(cm_node->cm_core, cm_node); - break; - case NES_CM_STATE_LAST_ACK: - case NES_CM_STATE_FIN_WAIT1: - if (cm_node->cm_id) - cm_id->rem_ref(cm_id); - send_reset(cm_node, NULL); - break; - default: - add_ref_cm_node(cm_node); - send_reset(cm_node, NULL); - create_event(cm_node, NES_CM_EVENT_ABORTED); - } -} - -static void handle_recv_entry(struct nes_cm_node *cm_node, u32 rem_node) -{ - struct nes_timer_entry *recv_entry = cm_node->recv_entry; - struct iw_cm_id *cm_id = cm_node->cm_id; - struct nes_qp *nesqp; - unsigned long qplockflags; - - if (!recv_entry) - return; - nesqp = (struct nes_qp *)recv_entry->skb; - if (nesqp) { - spin_lock_irqsave(&nesqp->lock, qplockflags); - if (nesqp->cm_id) { - nes_debug(NES_DBG_CM, "QP%u: cm_id = %p, " - "refcount = %d: HIT A " - "NES_TIMER_TYPE_CLOSE with something " - "to do!!!\n", nesqp->hwqp.qp_id, cm_id, - atomic_read(&nesqp->refcount)); - nesqp->hw_tcp_state = NES_AEQE_TCP_STATE_CLOSED; - nesqp->last_aeq = NES_AEQE_AEID_RESET_SENT; - nesqp->ibqp_state = IB_QPS_ERR; - spin_unlock_irqrestore(&nesqp->lock, qplockflags); - nes_cm_disconn(nesqp); - } else { - spin_unlock_irqrestore(&nesqp->lock, qplockflags); - nes_debug(NES_DBG_CM, "QP%u: cm_id = %p, " - "refcount = %d: HIT A " - "NES_TIMER_TYPE_CLOSE with nothing " - "to do!!!\n", nesqp->hwqp.qp_id, cm_id, - atomic_read(&nesqp->refcount)); - } - } else if (rem_node) { - /* TIME_WAIT state */ - rem_ref_cm_node(cm_node->cm_core, cm_node); - } - if (cm_node->cm_id) - cm_id->rem_ref(cm_id); - kfree(recv_entry); - cm_node->recv_entry = NULL; -} - -/** - * nes_cm_timer_tick - */ -static void nes_cm_timer_tick(struct timer_list *unused) -{ - unsigned long flags; - unsigned long nexttimeout = jiffies + NES_LONG_TIME; - struct nes_cm_node *cm_node; - struct nes_timer_entry *send_entry, *recv_entry; - struct list_head *list_core_temp; - struct list_head *list_node; - struct nes_cm_core *cm_core = g_cm_core; - u32 settimer = 0; - unsigned long timetosend; - int ret = NETDEV_TX_OK; - - struct list_head timer_list; - - INIT_LIST_HEAD(&timer_list); - spin_lock_irqsave(&cm_core->ht_lock, flags); - - list_for_each_safe(list_node, list_core_temp, - &cm_core->connected_nodes) { - cm_node = container_of(list_node, struct nes_cm_node, list); - if ((cm_node->recv_entry) || (cm_node->send_entry)) { - add_ref_cm_node(cm_node); - list_add(&cm_node->timer_entry, &timer_list); - } - } - spin_unlock_irqrestore(&cm_core->ht_lock, flags); - - list_for_each_safe(list_node, list_core_temp, &timer_list) { - cm_node = container_of(list_node, struct nes_cm_node, - timer_entry); - recv_entry = cm_node->recv_entry; - - if (recv_entry) { - if (time_after(recv_entry->timetosend, jiffies)) { - if (nexttimeout > recv_entry->timetosend || - !settimer) { - nexttimeout = recv_entry->timetosend; - settimer = 1; - } - } else { - handle_recv_entry(cm_node, 1); - } - } - - spin_lock_irqsave(&cm_node->retrans_list_lock, flags); - do { - send_entry = cm_node->send_entry; - if (!send_entry) - break; - if (time_after(send_entry->timetosend, jiffies)) { - if (cm_node->state != NES_CM_STATE_TSA) { - if ((nexttimeout > - send_entry->timetosend) || - !settimer) { - nexttimeout = - send_entry->timetosend; - settimer = 1; - } - } else { - free_retrans_entry(cm_node); - } - break; - } - - if ((cm_node->state == NES_CM_STATE_TSA) || - (cm_node->state == NES_CM_STATE_CLOSED)) { - free_retrans_entry(cm_node); - break; - } - - if (!send_entry->retranscount || - !send_entry->retrycount) { - cm_packets_dropped++; - free_retrans_entry(cm_node); - - spin_unlock_irqrestore( - &cm_node->retrans_list_lock, flags); - nes_retrans_expired(cm_node); - cm_node->state = NES_CM_STATE_CLOSED; - spin_lock_irqsave(&cm_node->retrans_list_lock, - flags); - break; - } - refcount_inc(&send_entry->skb->users); - cm_packets_retrans++; - nes_debug(NES_DBG_CM, "Retransmitting send_entry %p " - "for node %p, jiffies = %lu, time to send = " - "%lu, retranscount = %u, send_entry->seq_num = " - "0x%08X, cm_node->tcp_cntxt.rem_ack_num = " - "0x%08X\n", send_entry, cm_node, jiffies, - send_entry->timetosend, - send_entry->retranscount, - send_entry->seq_num, - cm_node->tcp_cntxt.rem_ack_num); - - spin_unlock_irqrestore(&cm_node->retrans_list_lock, - flags); - ret = nes_nic_cm_xmit(send_entry->skb, cm_node->netdev); - spin_lock_irqsave(&cm_node->retrans_list_lock, flags); - if (ret != NETDEV_TX_OK) { - nes_debug(NES_DBG_CM, "rexmit failed for " - "node=%p\n", cm_node); - cm_packets_bounced++; - send_entry->retrycount--; - nexttimeout = jiffies + NES_SHORT_TIME; - settimer = 1; - break; - } else { - cm_packets_sent++; - } - nes_debug(NES_DBG_CM, "Packet Sent: retrans count = " - "%u, retry count = %u.\n", - send_entry->retranscount, - send_entry->retrycount); - if (send_entry->send_retrans) { - send_entry->retranscount--; - timetosend = (NES_RETRY_TIMEOUT << - (NES_DEFAULT_RETRANS - send_entry->retranscount)); - - send_entry->timetosend = jiffies + - min(timetosend, NES_MAX_TIMEOUT); - if (nexttimeout > send_entry->timetosend || - !settimer) { - nexttimeout = send_entry->timetosend; - settimer = 1; - } - } else { - int close_when_complete; - close_when_complete = - send_entry->close_when_complete; - nes_debug(NES_DBG_CM, "cm_node=%p state=%d\n", - cm_node, cm_node->state); - free_retrans_entry(cm_node); - if (close_when_complete) - rem_ref_cm_node(cm_node->cm_core, - cm_node); - } - } while (0); - - spin_unlock_irqrestore(&cm_node->retrans_list_lock, flags); - rem_ref_cm_node(cm_node->cm_core, cm_node); - } - - if (settimer) { - if (!timer_pending(&cm_core->tcp_timer)) - mod_timer(&cm_core->tcp_timer, nexttimeout); - } -} - - -/** - * send_syn - */ -static int send_syn(struct nes_cm_node *cm_node, u32 sendack, - struct sk_buff *skb) -{ - int ret; - int flags = SET_SYN; - char optionsbuffer[sizeof(struct option_mss) + - sizeof(struct option_windowscale) + sizeof(struct option_base) + - TCP_OPTIONS_PADDING]; - - int optionssize = 0; - /* Sending MSS option */ - union all_known_options *options; - - if (!cm_node) - return -EINVAL; - - options = (union all_known_options *)&optionsbuffer[optionssize]; - options->as_mss.optionnum = OPTION_NUMBER_MSS; - options->as_mss.length = sizeof(struct option_mss); - options->as_mss.mss = htons(cm_node->tcp_cntxt.mss); - optionssize += sizeof(struct option_mss); - - options = (union all_known_options *)&optionsbuffer[optionssize]; - options->as_windowscale.optionnum = OPTION_NUMBER_WINDOW_SCALE; - options->as_windowscale.length = sizeof(struct option_windowscale); - options->as_windowscale.shiftcount = cm_node->tcp_cntxt.rcv_wscale; - optionssize += sizeof(struct option_windowscale); - - if (sendack && !(NES_DRV_OPT_SUPRESS_OPTION_BC & nes_drv_opt)) { - options = (union all_known_options *)&optionsbuffer[optionssize]; - options->as_base.optionnum = OPTION_NUMBER_WRITE0; - options->as_base.length = sizeof(struct option_base); - optionssize += sizeof(struct option_base); - /* we need the size to be a multiple of 4 */ - options = (union all_known_options *)&optionsbuffer[optionssize]; - options->as_end = 1; - optionssize += 1; - options = (union all_known_options *)&optionsbuffer[optionssize]; - options->as_end = 1; - optionssize += 1; - } - - options = (union all_known_options *)&optionsbuffer[optionssize]; - options->as_end = OPTION_NUMBER_END; - optionssize += 1; - - if (!skb) - skb = dev_alloc_skb(MAX_CM_BUFFER); - if (!skb) { - nes_debug(NES_DBG_CM, "Failed to get a Free pkt\n"); - return -1; - } - - if (sendack) - flags |= SET_ACK; - - form_cm_frame(skb, cm_node, optionsbuffer, optionssize, NULL, 0, flags); - ret = schedule_nes_timer(cm_node, skb, NES_TIMER_TYPE_SEND, 1, 0); - - return ret; -} - - -/** - * send_reset - */ -static int send_reset(struct nes_cm_node *cm_node, struct sk_buff *skb) -{ - int ret; - int flags = SET_RST | SET_ACK; - - if (!skb) - skb = dev_alloc_skb(MAX_CM_BUFFER); - if (!skb) { - nes_debug(NES_DBG_CM, "Failed to get a Free pkt\n"); - return -ENOMEM; - } - - form_cm_frame(skb, cm_node, NULL, 0, NULL, 0, flags); - ret = schedule_nes_timer(cm_node, skb, NES_TIMER_TYPE_SEND, 0, 1); - - return ret; -} - - -/** - * send_ack - */ -static int send_ack(struct nes_cm_node *cm_node, struct sk_buff *skb) -{ - int ret; - - if (!skb) - skb = dev_alloc_skb(MAX_CM_BUFFER); - - if (!skb) { - nes_debug(NES_DBG_CM, "Failed to get a Free pkt\n"); - return -1; - } - - form_cm_frame(skb, cm_node, NULL, 0, NULL, 0, SET_ACK); - ret = schedule_nes_timer(cm_node, skb, NES_TIMER_TYPE_SEND, 0, 0); - - return ret; -} - - -/** - * send_fin - */ -static int send_fin(struct nes_cm_node *cm_node, struct sk_buff *skb) -{ - int ret; - - /* if we didn't get a frame get one */ - if (!skb) - skb = dev_alloc_skb(MAX_CM_BUFFER); - - if (!skb) { - nes_debug(NES_DBG_CM, "Failed to get a Free pkt\n"); - return -1; - } - - form_cm_frame(skb, cm_node, NULL, 0, NULL, 0, SET_ACK | SET_FIN); - ret = schedule_nes_timer(cm_node, skb, NES_TIMER_TYPE_SEND, 1, 0); - - return ret; -} - - -/** - * find_node - find a cm node that matches the reference cm node - */ -static struct nes_cm_node *find_node(struct nes_cm_core *cm_core, - u16 rem_port, nes_addr_t rem_addr, u16 loc_port, nes_addr_t loc_addr) -{ - unsigned long flags; - struct list_head *hte; - struct nes_cm_node *cm_node; - - /* get a handle on the hte */ - hte = &cm_core->connected_nodes; - - /* walk list and find cm_node associated with this session ID */ - spin_lock_irqsave(&cm_core->ht_lock, flags); - list_for_each_entry(cm_node, hte, list) { - /* compare quad, return node handle if a match */ - nes_debug(NES_DBG_CM, "finding node %x:%x =? %x:%x ^ %x:%x =? %x:%x\n", - cm_node->loc_addr, cm_node->loc_port, - loc_addr, loc_port, - cm_node->rem_addr, cm_node->rem_port, - rem_addr, rem_port); - if ((cm_node->loc_addr == loc_addr) && - (cm_node->loc_port == loc_port) && - (cm_node->rem_addr == rem_addr) && - (cm_node->rem_port == rem_port)) { - add_ref_cm_node(cm_node); - spin_unlock_irqrestore(&cm_core->ht_lock, flags); - return cm_node; - } - } - spin_unlock_irqrestore(&cm_core->ht_lock, flags); - - /* no owner node */ - return NULL; -} - - -/** - * find_listener - find a cm node listening on this addr-port pair - */ -static struct nes_cm_listener *find_listener(struct nes_cm_core *cm_core, - nes_addr_t dst_addr, u16 dst_port, - enum nes_cm_listener_state listener_state) -{ - unsigned long flags; - struct nes_cm_listener *listen_node; - nes_addr_t listen_addr; - u16 listen_port; - - /* walk list and find cm_node associated with this session ID */ - spin_lock_irqsave(&cm_core->listen_list_lock, flags); - list_for_each_entry(listen_node, &cm_core->listen_list.list, list) { - listen_addr = listen_node->loc_addr; - listen_port = listen_node->loc_port; - - /* compare node pair, return node handle if a match */ - if (((listen_addr == dst_addr) || - listen_addr == 0x00000000) && - (listen_port == dst_port) && - (listener_state & listen_node->listener_state)) { - atomic_inc(&listen_node->ref_count); - spin_unlock_irqrestore(&cm_core->listen_list_lock, flags); - return listen_node; - } - } - spin_unlock_irqrestore(&cm_core->listen_list_lock, flags); - - /* no listener */ - return NULL; -} - -/** - * add_hte_node - add a cm node to the hash table - */ -static int add_hte_node(struct nes_cm_core *cm_core, struct nes_cm_node *cm_node) -{ - unsigned long flags; - struct list_head *hte; - - if (!cm_node || !cm_core) - return -EINVAL; - - nes_debug(NES_DBG_CM, "Adding Node %p to Active Connection HT\n", - cm_node); - - spin_lock_irqsave(&cm_core->ht_lock, flags); - - /* get a handle on the hash table element (list head for this slot) */ - hte = &cm_core->connected_nodes; - list_add_tail(&cm_node->list, hte); - atomic_inc(&cm_core->ht_node_cnt); - - spin_unlock_irqrestore(&cm_core->ht_lock, flags); - - return 0; -} - - -/** - * mini_cm_dec_refcnt_listen - */ -static int mini_cm_dec_refcnt_listen(struct nes_cm_core *cm_core, - struct nes_cm_listener *listener, int free_hanging_nodes) -{ - int ret = -EINVAL; - int err = 0; - unsigned long flags; - struct list_head *list_pos = NULL; - struct list_head *list_temp = NULL; - struct nes_cm_node *cm_node = NULL; - struct list_head reset_list; - - nes_debug(NES_DBG_CM, "attempting listener= %p free_nodes= %d, " - "refcnt=%d\n", listener, free_hanging_nodes, - atomic_read(&listener->ref_count)); - /* free non-accelerated child nodes for this listener */ - INIT_LIST_HEAD(&reset_list); - if (free_hanging_nodes) { - spin_lock_irqsave(&cm_core->ht_lock, flags); - list_for_each_safe(list_pos, list_temp, - &g_cm_core->connected_nodes) { - cm_node = container_of(list_pos, struct nes_cm_node, - list); - if ((cm_node->listener == listener) && - (!cm_node->accelerated)) { - add_ref_cm_node(cm_node); - list_add(&cm_node->reset_entry, &reset_list); - } - } - spin_unlock_irqrestore(&cm_core->ht_lock, flags); - } - - list_for_each_safe(list_pos, list_temp, &reset_list) { - cm_node = container_of(list_pos, struct nes_cm_node, - reset_entry); - { - struct nes_cm_node *loopback = cm_node->loopbackpartner; - enum nes_cm_node_state old_state; - if (NES_CM_STATE_FIN_WAIT1 <= cm_node->state) { - rem_ref_cm_node(cm_node->cm_core, cm_node); - } else { - if (!loopback) { - cleanup_retrans_entry(cm_node); - err = send_reset(cm_node, NULL); - if (err) { - cm_node->state = - NES_CM_STATE_CLOSED; - WARN_ON(1); - } else { - old_state = cm_node->state; - cm_node->state = NES_CM_STATE_LISTENER_DESTROYED; - if (old_state != NES_CM_STATE_MPAREQ_RCVD) - rem_ref_cm_node( - cm_node->cm_core, - cm_node); - } - } else { - struct nes_cm_event event; - - event.cm_node = loopback; - event.cm_info.rem_addr = - loopback->rem_addr; - event.cm_info.loc_addr = - loopback->loc_addr; - event.cm_info.rem_port = - loopback->rem_port; - event.cm_info.loc_port = - loopback->loc_port; - event.cm_info.cm_id = loopback->cm_id; - add_ref_cm_node(loopback); - loopback->state = NES_CM_STATE_CLOSED; - cm_event_connect_error(&event); - cm_node->state = NES_CM_STATE_LISTENER_DESTROYED; - - rem_ref_cm_node(cm_node->cm_core, - cm_node); - - } - } - } - } - - spin_lock_irqsave(&cm_core->listen_list_lock, flags); - if (!atomic_dec_return(&listener->ref_count)) { - list_del(&listener->list); - - /* decrement our listen node count */ - atomic_dec(&cm_core->listen_node_cnt); - - spin_unlock_irqrestore(&cm_core->listen_list_lock, flags); - - if (listener->nesvnic) { - nes_manage_apbvt(listener->nesvnic, - listener->loc_port, - PCI_FUNC(listener->nesvnic->nesdev->pcidev->devfn), - NES_MANAGE_APBVT_DEL); - - nes_debug(NES_DBG_NLMSG, - "Delete APBVT loc_port = %04X\n", - listener->loc_port); - } - - nes_debug(NES_DBG_CM, "destroying listener (%p)\n", listener); - - kfree(listener); - listener = NULL; - ret = 0; - atomic_inc(&cm_listens_destroyed); - } else { - spin_unlock_irqrestore(&cm_core->listen_list_lock, flags); - } - if (listener) { - if (atomic_read(&listener->pend_accepts_cnt) > 0) - nes_debug(NES_DBG_CM, "destroying listener (%p)" - " with non-zero pending accepts=%u\n", - listener, atomic_read(&listener->pend_accepts_cnt)); - } - - return ret; -} - - -/** - * mini_cm_del_listen - */ -static int mini_cm_del_listen(struct nes_cm_core *cm_core, - struct nes_cm_listener *listener) -{ - listener->listener_state = NES_CM_LISTENER_PASSIVE_STATE; - listener->cm_id = NULL; /* going to be destroyed pretty soon */ - return mini_cm_dec_refcnt_listen(cm_core, listener, 1); -} - - -/** - * mini_cm_accelerated - */ -static inline int mini_cm_accelerated(struct nes_cm_core *cm_core, - struct nes_cm_node *cm_node) -{ - cm_node->accelerated = true; - - if (cm_node->accept_pend) { - BUG_ON(!cm_node->listener); - atomic_dec(&cm_node->listener->pend_accepts_cnt); - cm_node->accept_pend = 0; - BUG_ON(atomic_read(&cm_node->listener->pend_accepts_cnt) < 0); - } - - if (!timer_pending(&cm_core->tcp_timer)) - mod_timer(&cm_core->tcp_timer, (jiffies + NES_SHORT_TIME)); - - return 0; -} - - -/** - * nes_addr_resolve_neigh - */ -static int nes_addr_resolve_neigh(struct nes_vnic *nesvnic, u32 dst_ip, int arpindex) -{ - struct rtable *rt; - struct neighbour *neigh; - int rc = arpindex; - struct nes_adapter *nesadapter = nesvnic->nesdev->nesadapter; - __be32 dst_ipaddr = htonl(dst_ip); - - rt = ip_route_output(&init_net, dst_ipaddr, nesvnic->local_ipaddr, 0, 0); - if (IS_ERR(rt)) { - printk(KERN_ERR "%s: ip_route_output_key failed for 0x%08X\n", - __func__, dst_ip); - return rc; - } - - neigh = dst_neigh_lookup(&rt->dst, &dst_ipaddr); - - rcu_read_lock(); - if (neigh) { - if (neigh->nud_state & NUD_VALID) { - nes_debug(NES_DBG_CM, "Neighbor MAC address for 0x%08X" - " is %pM, Gateway is 0x%08X \n", dst_ip, - neigh->ha, ntohl(rt->rt_gw4)); - - if (arpindex >= 0) { - if (ether_addr_equal(nesadapter->arp_table[arpindex].mac_addr, neigh->ha)) { - /* Mac address same as in nes_arp_table */ - goto out; - } - - nes_manage_arp_cache(nesvnic->netdev, - nesadapter->arp_table[arpindex].mac_addr, - dst_ip, NES_ARP_DELETE); - } - - nes_manage_arp_cache(nesvnic->netdev, neigh->ha, - dst_ip, NES_ARP_ADD); - rc = nes_arp_table(nesvnic->nesdev, dst_ip, NULL, - NES_ARP_RESOLVE); - } else { - neigh_event_send(neigh, NULL); - } - } -out: - rcu_read_unlock(); - - if (neigh) - neigh_release(neigh); - - ip_rt_put(rt); - return rc; -} - -/** - * make_cm_node - create a new instance of a cm node - */ -static struct nes_cm_node *make_cm_node(struct nes_cm_core *cm_core, - struct nes_vnic *nesvnic, struct nes_cm_info *cm_info, - struct nes_cm_listener *listener) -{ - struct nes_cm_node *cm_node; - int oldarpindex = 0; - int arpindex = 0; - struct nes_device *nesdev; - struct nes_adapter *nesadapter; - - /* create an hte and cm_node for this instance */ - cm_node = kzalloc(sizeof(*cm_node), GFP_ATOMIC); - if (!cm_node) - return NULL; - - /* set our node specific transport info */ - if (listener) { - cm_node->loc_addr = listener->loc_addr; - cm_node->loc_port = listener->loc_port; - } else { - cm_node->loc_addr = cm_info->loc_addr; - cm_node->loc_port = cm_info->loc_port; - } - cm_node->rem_addr = cm_info->rem_addr; - cm_node->rem_port = cm_info->rem_port; - - cm_node->mpa_frame_rev = mpa_version; - cm_node->send_rdma0_op = SEND_RDMA_READ_ZERO; - cm_node->mpav2_ird_ord = 0; - cm_node->ird_size = 0; - cm_node->ord_size = 0; - - nes_debug(NES_DBG_CM, "Make node addresses : loc = %pI4:%x, rem = %pI4:%x\n", - &cm_node->loc_addr, cm_node->loc_port, - &cm_node->rem_addr, cm_node->rem_port); - cm_node->listener = listener; - if (listener) - cm_node->tos = listener->tos; - cm_node->netdev = nesvnic->netdev; - cm_node->cm_id = cm_info->cm_id; - memcpy(cm_node->loc_mac, nesvnic->netdev->dev_addr, ETH_ALEN); - - nes_debug(NES_DBG_CM, "listener=%p, cm_id=%p\n", cm_node->listener, - cm_node->cm_id); - - spin_lock_init(&cm_node->retrans_list_lock); - - cm_node->loopbackpartner = NULL; - atomic_set(&cm_node->ref_count, 1); - /* associate our parent CM core */ - cm_node->cm_core = cm_core; - cm_node->tcp_cntxt.loc_id = NES_CM_DEF_LOCAL_ID; - cm_node->tcp_cntxt.rcv_wscale = NES_CM_DEFAULT_RCV_WND_SCALE; - cm_node->tcp_cntxt.rcv_wnd = NES_CM_DEFAULT_RCV_WND_SCALED >> - NES_CM_DEFAULT_RCV_WND_SCALE; - cm_node->tcp_cntxt.loc_seq_num = secure_tcp_seq(htonl(cm_node->loc_addr), - htonl(cm_node->rem_addr), - htons(cm_node->loc_port), - htons(cm_node->rem_port)); - cm_node->tcp_cntxt.mss = nesvnic->max_frame_size - sizeof(struct iphdr) - - sizeof(struct tcphdr) - ETH_HLEN - VLAN_HLEN; - cm_node->tcp_cntxt.rcv_nxt = 0; - /* get a unique session ID , add thread_id to an upcounter to handle race */ - atomic_inc(&cm_core->node_cnt); - cm_node->conn_type = cm_info->conn_type; - cm_node->apbvt_set = 0; - cm_node->accept_pend = 0; - - cm_node->nesvnic = nesvnic; - /* get some device handles, for arp lookup */ - nesdev = nesvnic->nesdev; - nesadapter = nesdev->nesadapter; - - cm_node->loopbackpartner = NULL; - - /* get the mac addr for the remote node */ - oldarpindex = nes_arp_table(nesdev, cm_node->rem_addr, - NULL, NES_ARP_RESOLVE); - arpindex = nes_addr_resolve_neigh(nesvnic, cm_node->rem_addr, - oldarpindex); - if (arpindex < 0) { - kfree(cm_node); - return NULL; - } - - /* copy the mac addr to node context */ - memcpy(cm_node->rem_mac, nesadapter->arp_table[arpindex].mac_addr, ETH_ALEN); - nes_debug(NES_DBG_CM, "Remote mac addr from arp table: %pM\n", - cm_node->rem_mac); - - add_hte_node(cm_core, cm_node); - atomic_inc(&cm_nodes_created); - - return cm_node; -} - - -/** - * add_ref_cm_node - destroy an instance of a cm node - */ -static int add_ref_cm_node(struct nes_cm_node *cm_node) -{ - atomic_inc(&cm_node->ref_count); - return 0; -} - - -/** - * rem_ref_cm_node - destroy an instance of a cm node - */ -static int rem_ref_cm_node(struct nes_cm_core *cm_core, - struct nes_cm_node *cm_node) -{ - unsigned long flags; - struct nes_qp *nesqp; - - if (!cm_node) - return -EINVAL; - - spin_lock_irqsave(&cm_node->cm_core->ht_lock, flags); - if (atomic_dec_return(&cm_node->ref_count)) { - spin_unlock_irqrestore(&cm_node->cm_core->ht_lock, flags); - return 0; - } - list_del(&cm_node->list); - atomic_dec(&cm_core->ht_node_cnt); - spin_unlock_irqrestore(&cm_node->cm_core->ht_lock, flags); - - /* if the node is destroyed before connection was accelerated */ - if (!cm_node->accelerated && cm_node->accept_pend) { - BUG_ON(!cm_node->listener); - atomic_dec(&cm_node->listener->pend_accepts_cnt); - BUG_ON(atomic_read(&cm_node->listener->pend_accepts_cnt) < 0); - } - WARN_ON(cm_node->send_entry); - if (cm_node->recv_entry) - handle_recv_entry(cm_node, 0); - if (cm_node->listener) { - mini_cm_dec_refcnt_listen(cm_core, cm_node->listener, 0); - } else { - if (cm_node->apbvt_set && cm_node->nesvnic) { - nes_manage_apbvt(cm_node->nesvnic, cm_node->loc_port, - PCI_FUNC(cm_node->nesvnic->nesdev->pcidev->devfn), - NES_MANAGE_APBVT_DEL); - } - nes_debug(NES_DBG_NLMSG, "Delete APBVT loc_port = %04X\n", - cm_node->loc_port); - } - - atomic_dec(&cm_core->node_cnt); - atomic_inc(&cm_nodes_destroyed); - nesqp = cm_node->nesqp; - if (nesqp) { - nesqp->cm_node = NULL; - nes_rem_ref(&nesqp->ibqp); - cm_node->nesqp = NULL; - } - - kfree(cm_node); - return 0; -} - -/** - * process_options - */ -static int process_options(struct nes_cm_node *cm_node, u8 *optionsloc, - u32 optionsize, u32 syn_packet) -{ - u32 tmp; - u32 offset = 0; - union all_known_options *all_options; - char got_mss_option = 0; - - while (offset < optionsize) { - all_options = (union all_known_options *)(optionsloc + offset); - switch (all_options->as_base.optionnum) { - case OPTION_NUMBER_END: - offset = optionsize; - break; - case OPTION_NUMBER_NONE: - offset += 1; - continue; - case OPTION_NUMBER_MSS: - nes_debug(NES_DBG_CM, "%s: MSS Length: %d Offset: %d " - "Size: %d\n", __func__, - all_options->as_mss.length, offset, optionsize); - got_mss_option = 1; - if (all_options->as_mss.length != 4) { - return 1; - } else { - tmp = ntohs(all_options->as_mss.mss); - if (tmp > 0 && tmp < - cm_node->tcp_cntxt.mss) - cm_node->tcp_cntxt.mss = tmp; - } - break; - case OPTION_NUMBER_WINDOW_SCALE: - cm_node->tcp_cntxt.snd_wscale = - all_options->as_windowscale.shiftcount; - break; - default: - nes_debug(NES_DBG_CM, "TCP Option not understood: %x\n", - all_options->as_base.optionnum); - break; - } - offset += all_options->as_base.length; - } - if ((!got_mss_option) && (syn_packet)) - cm_node->tcp_cntxt.mss = NES_CM_DEFAULT_MSS; - return 0; -} - -static void drop_packet(struct sk_buff *skb) -{ - atomic_inc(&cm_accel_dropped_pkts); - dev_kfree_skb_any(skb); -} - -static void handle_fin_pkt(struct nes_cm_node *cm_node) -{ - nes_debug(NES_DBG_CM, "Received FIN, cm_node = %p, state = %u. " - "refcnt=%d\n", cm_node, cm_node->state, - atomic_read(&cm_node->ref_count)); - switch (cm_node->state) { - case NES_CM_STATE_SYN_RCVD: - case NES_CM_STATE_SYN_SENT: - case NES_CM_STATE_ESTABLISHED: - case NES_CM_STATE_MPAREJ_RCVD: - cm_node->tcp_cntxt.rcv_nxt++; - cleanup_retrans_entry(cm_node); - cm_node->state = NES_CM_STATE_LAST_ACK; - send_fin(cm_node, NULL); - break; - case NES_CM_STATE_MPAREQ_SENT: - create_event(cm_node, NES_CM_EVENT_ABORTED); - cm_node->tcp_cntxt.rcv_nxt++; - cleanup_retrans_entry(cm_node); - cm_node->state = NES_CM_STATE_CLOSED; - add_ref_cm_node(cm_node); - send_reset(cm_node, NULL); - break; - case NES_CM_STATE_FIN_WAIT1: - cm_node->tcp_cntxt.rcv_nxt++; - cleanup_retrans_entry(cm_node); - cm_node->state = NES_CM_STATE_CLOSING; - send_ack(cm_node, NULL); - /* Wait for ACK as this is simultaneous close.. - * After we receive ACK, do not send anything.. - * Just rm the node.. Done.. */ - break; - case NES_CM_STATE_FIN_WAIT2: - cm_node->tcp_cntxt.rcv_nxt++; - cleanup_retrans_entry(cm_node); - cm_node->state = NES_CM_STATE_TIME_WAIT; - send_ack(cm_node, NULL); - schedule_nes_timer(cm_node, NULL, NES_TIMER_TYPE_CLOSE, 1, 0); - break; - case NES_CM_STATE_TIME_WAIT: - cm_node->tcp_cntxt.rcv_nxt++; - cleanup_retrans_entry(cm_node); - cm_node->state = NES_CM_STATE_CLOSED; - rem_ref_cm_node(cm_node->cm_core, cm_node); - break; - case NES_CM_STATE_TSA: - default: - nes_debug(NES_DBG_CM, "Error Rcvd FIN for node-%p state = %d\n", - cm_node, cm_node->state); - break; - } -} - - -static void handle_rst_pkt(struct nes_cm_node *cm_node, struct sk_buff *skb, - struct tcphdr *tcph) -{ - - int reset = 0; /* whether to send reset in case of err.. */ - atomic_inc(&cm_resets_recvd); - nes_debug(NES_DBG_CM, "Received Reset, cm_node = %p, state = %u." - " refcnt=%d\n", cm_node, cm_node->state, - atomic_read(&cm_node->ref_count)); - cleanup_retrans_entry(cm_node); - switch (cm_node->state) { - case NES_CM_STATE_SYN_SENT: - case NES_CM_STATE_MPAREQ_SENT: - nes_debug(NES_DBG_CM, "%s[%u] create abort for cm_node=%p " - "listener=%p state=%d\n", __func__, __LINE__, cm_node, - cm_node->listener, cm_node->state); - switch (cm_node->mpa_frame_rev) { - case IETF_MPA_V2: - cm_node->mpa_frame_rev = IETF_MPA_V1; - /* send a syn and goto syn sent state */ - cm_node->state = NES_CM_STATE_SYN_SENT; - if (send_syn(cm_node, 0, NULL)) { - active_open_err(cm_node, skb, reset); - } - break; - case IETF_MPA_V1: - default: - active_open_err(cm_node, skb, reset); - break; - } - break; - case NES_CM_STATE_MPAREQ_RCVD: - atomic_inc(&cm_node->passive_state); - dev_kfree_skb_any(skb); - break; - case NES_CM_STATE_ESTABLISHED: - case NES_CM_STATE_SYN_RCVD: - case NES_CM_STATE_LISTENING: - nes_debug(NES_DBG_CM, "Bad state %s[%u]\n", __func__, __LINE__); - passive_open_err(cm_node, skb, reset); - break; - case NES_CM_STATE_TSA: - active_open_err(cm_node, skb, reset); - break; - case NES_CM_STATE_CLOSED: - drop_packet(skb); - break; - case NES_CM_STATE_FIN_WAIT2: - case NES_CM_STATE_FIN_WAIT1: - case NES_CM_STATE_LAST_ACK: - cm_node->cm_id->rem_ref(cm_node->cm_id); - /* fall through */ - case NES_CM_STATE_TIME_WAIT: - cm_node->state = NES_CM_STATE_CLOSED; - rem_ref_cm_node(cm_node->cm_core, cm_node); - drop_packet(skb); - break; - default: - drop_packet(skb); - break; - } -} - - -static void handle_rcv_mpa(struct nes_cm_node *cm_node, struct sk_buff *skb) -{ - int ret = 0; - int datasize = skb->len; - u8 *dataloc = skb->data; - - enum nes_cm_event_type type = NES_CM_EVENT_UNKNOWN; - u32 res_type; - - ret = parse_mpa(cm_node, dataloc, &res_type, datasize); - if (ret) { - nes_debug(NES_DBG_CM, "didn't like MPA Request\n"); - if (cm_node->state == NES_CM_STATE_MPAREQ_SENT) { - nes_debug(NES_DBG_CM, "%s[%u] create abort for " - "cm_node=%p listener=%p state=%d\n", __func__, - __LINE__, cm_node, cm_node->listener, - cm_node->state); - active_open_err(cm_node, skb, 1); - } else { - passive_open_err(cm_node, skb, 1); - } - return; - } - - switch (cm_node->state) { - case NES_CM_STATE_ESTABLISHED: - if (res_type == NES_MPA_REQUEST_REJECT) - /*BIG problem as we are receiving the MPA.. So should - * not be REJECT.. This is Passive Open.. We can - * only receive it Reject for Active Open...*/ - WARN_ON(1); - cm_node->state = NES_CM_STATE_MPAREQ_RCVD; - type = NES_CM_EVENT_MPA_REQ; - atomic_set(&cm_node->passive_state, - NES_PASSIVE_STATE_INDICATED); - break; - case NES_CM_STATE_MPAREQ_SENT: - cleanup_retrans_entry(cm_node); - if (res_type == NES_MPA_REQUEST_REJECT) { - type = NES_CM_EVENT_MPA_REJECT; - cm_node->state = NES_CM_STATE_MPAREJ_RCVD; - } else { - type = NES_CM_EVENT_CONNECTED; - cm_node->state = NES_CM_STATE_TSA; - } - send_ack(cm_node, NULL); - break; - default: - WARN_ON(1); - break; - } - dev_kfree_skb_any(skb); - create_event(cm_node, type); -} - -static void indicate_pkt_err(struct nes_cm_node *cm_node, struct sk_buff *skb) -{ - switch (cm_node->state) { - case NES_CM_STATE_SYN_SENT: - case NES_CM_STATE_MPAREQ_SENT: - nes_debug(NES_DBG_CM, "%s[%u] create abort for cm_node=%p " - "listener=%p state=%d\n", __func__, __LINE__, cm_node, - cm_node->listener, cm_node->state); - active_open_err(cm_node, skb, 1); - break; - case NES_CM_STATE_ESTABLISHED: - case NES_CM_STATE_SYN_RCVD: - passive_open_err(cm_node, skb, 1); - break; - case NES_CM_STATE_TSA: - default: - drop_packet(skb); - } -} - -static int check_syn(struct nes_cm_node *cm_node, struct tcphdr *tcph, - struct sk_buff *skb) -{ - int err; - - err = ((ntohl(tcph->ack_seq) == cm_node->tcp_cntxt.loc_seq_num)) ? 0 : 1; - if (err) - active_open_err(cm_node, skb, 1); - - return err; -} - -static int check_seq(struct nes_cm_node *cm_node, struct tcphdr *tcph, - struct sk_buff *skb) -{ - int err = 0; - u32 seq; - u32 ack_seq; - u32 loc_seq_num = cm_node->tcp_cntxt.loc_seq_num; - u32 rcv_nxt = cm_node->tcp_cntxt.rcv_nxt; - u32 rcv_wnd; - - seq = ntohl(tcph->seq); - ack_seq = ntohl(tcph->ack_seq); - rcv_wnd = cm_node->tcp_cntxt.rcv_wnd; - if (ack_seq != loc_seq_num) - err = 1; - else if (!between(seq, rcv_nxt, (rcv_nxt + rcv_wnd))) - err = 1; - if (err) { - nes_debug(NES_DBG_CM, "%s[%u] create abort for cm_node=%p " - "listener=%p state=%d\n", __func__, __LINE__, cm_node, - cm_node->listener, cm_node->state); - indicate_pkt_err(cm_node, skb); - nes_debug(NES_DBG_CM, "seq ERROR cm_node =%p seq=0x%08X " - "rcv_nxt=0x%08X rcv_wnd=0x%x\n", cm_node, seq, rcv_nxt, - rcv_wnd); - } - return err; -} - -/* - * handle_syn_pkt() is for Passive node. The syn packet is received when a node - * is created with a listener or it may comein as rexmitted packet which in - * that case will be just dropped. - */ -static void handle_syn_pkt(struct nes_cm_node *cm_node, struct sk_buff *skb, - struct tcphdr *tcph) -{ - int ret; - u32 inc_sequence; - int optionsize; - - optionsize = (tcph->doff << 2) - sizeof(struct tcphdr); - skb_trim(skb, 0); - inc_sequence = ntohl(tcph->seq); - - switch (cm_node->state) { - case NES_CM_STATE_SYN_SENT: - case NES_CM_STATE_MPAREQ_SENT: - /* Rcvd syn on active open connection*/ - active_open_err(cm_node, skb, 1); - break; - case NES_CM_STATE_LISTENING: - /* Passive OPEN */ - if (atomic_read(&cm_node->listener->pend_accepts_cnt) > - cm_node->listener->backlog) { - nes_debug(NES_DBG_CM, "drop syn due to backlog " - "pressure \n"); - cm_backlog_drops++; - passive_open_err(cm_node, skb, 0); - break; - } - ret = handle_tcp_options(cm_node, tcph, skb, optionsize, - 1); - if (ret) { - passive_open_err(cm_node, skb, 0); - /* drop pkt */ - break; - } - cm_node->tcp_cntxt.rcv_nxt = inc_sequence + 1; - BUG_ON(cm_node->send_entry); - cm_node->accept_pend = 1; - atomic_inc(&cm_node->listener->pend_accepts_cnt); - - cm_node->state = NES_CM_STATE_SYN_RCVD; - send_syn(cm_node, 1, skb); - break; - case NES_CM_STATE_CLOSED: - cleanup_retrans_entry(cm_node); - add_ref_cm_node(cm_node); - send_reset(cm_node, skb); - break; - case NES_CM_STATE_TSA: - case NES_CM_STATE_ESTABLISHED: - case NES_CM_STATE_FIN_WAIT1: - case NES_CM_STATE_FIN_WAIT2: - case NES_CM_STATE_MPAREQ_RCVD: - case NES_CM_STATE_LAST_ACK: - case NES_CM_STATE_CLOSING: - case NES_CM_STATE_UNKNOWN: - default: - drop_packet(skb); - break; - } -} - -static void handle_synack_pkt(struct nes_cm_node *cm_node, struct sk_buff *skb, - struct tcphdr *tcph) -{ - int ret; - u32 inc_sequence; - int optionsize; - - optionsize = (tcph->doff << 2) - sizeof(struct tcphdr); - skb_trim(skb, 0); - inc_sequence = ntohl(tcph->seq); - switch (cm_node->state) { - case NES_CM_STATE_SYN_SENT: - cleanup_retrans_entry(cm_node); - /* active open */ - if (check_syn(cm_node, tcph, skb)) - return; - cm_node->tcp_cntxt.rem_ack_num = ntohl(tcph->ack_seq); - /* setup options */ - ret = handle_tcp_options(cm_node, tcph, skb, optionsize, 0); - if (ret) { - nes_debug(NES_DBG_CM, "cm_node=%p tcp_options failed\n", - cm_node); - break; - } - cleanup_retrans_entry(cm_node); - cm_node->tcp_cntxt.rcv_nxt = inc_sequence + 1; - send_mpa_request(cm_node, skb); - cm_node->state = NES_CM_STATE_MPAREQ_SENT; - break; - case NES_CM_STATE_MPAREQ_RCVD: - /* passive open, so should not be here */ - passive_open_err(cm_node, skb, 1); - break; - case NES_CM_STATE_LISTENING: - cm_node->tcp_cntxt.loc_seq_num = ntohl(tcph->ack_seq); - cleanup_retrans_entry(cm_node); - cm_node->state = NES_CM_STATE_CLOSED; - send_reset(cm_node, skb); - break; - case NES_CM_STATE_CLOSED: - cm_node->tcp_cntxt.loc_seq_num = ntohl(tcph->ack_seq); - cleanup_retrans_entry(cm_node); - add_ref_cm_node(cm_node); - send_reset(cm_node, skb); - break; - case NES_CM_STATE_ESTABLISHED: - case NES_CM_STATE_FIN_WAIT1: - case NES_CM_STATE_FIN_WAIT2: - case NES_CM_STATE_LAST_ACK: - case NES_CM_STATE_TSA: - case NES_CM_STATE_CLOSING: - case NES_CM_STATE_UNKNOWN: - case NES_CM_STATE_MPAREQ_SENT: - default: - drop_packet(skb); - break; - } -} - -static int handle_ack_pkt(struct nes_cm_node *cm_node, struct sk_buff *skb, - struct tcphdr *tcph) -{ - int datasize = 0; - u32 inc_sequence; - int ret = 0; - int optionsize; - - optionsize = (tcph->doff << 2) - sizeof(struct tcphdr); - - if (check_seq(cm_node, tcph, skb)) - return -EINVAL; - - skb_pull(skb, tcph->doff << 2); - inc_sequence = ntohl(tcph->seq); - datasize = skb->len; - switch (cm_node->state) { - case NES_CM_STATE_SYN_RCVD: - /* Passive OPEN */ - cleanup_retrans_entry(cm_node); - ret = handle_tcp_options(cm_node, tcph, skb, optionsize, 1); - if (ret) - break; - cm_node->tcp_cntxt.rem_ack_num = ntohl(tcph->ack_seq); - cm_node->state = NES_CM_STATE_ESTABLISHED; - if (datasize) { - cm_node->tcp_cntxt.rcv_nxt = inc_sequence + datasize; - handle_rcv_mpa(cm_node, skb); - } else { /* rcvd ACK only */ - dev_kfree_skb_any(skb); - } - break; - case NES_CM_STATE_ESTABLISHED: - /* Passive OPEN */ - cleanup_retrans_entry(cm_node); - if (datasize) { - cm_node->tcp_cntxt.rcv_nxt = inc_sequence + datasize; - handle_rcv_mpa(cm_node, skb); - } else { - drop_packet(skb); - } - break; - case NES_CM_STATE_MPAREQ_SENT: - cm_node->tcp_cntxt.rem_ack_num = ntohl(tcph->ack_seq); - if (datasize) { - cm_node->tcp_cntxt.rcv_nxt = inc_sequence + datasize; - handle_rcv_mpa(cm_node, skb); - } else { /* Could be just an ack pkt.. */ - dev_kfree_skb_any(skb); - } - break; - case NES_CM_STATE_LISTENING: - cleanup_retrans_entry(cm_node); - cm_node->state = NES_CM_STATE_CLOSED; - send_reset(cm_node, skb); - break; - case NES_CM_STATE_CLOSED: - cleanup_retrans_entry(cm_node); - add_ref_cm_node(cm_node); - send_reset(cm_node, skb); - break; - case NES_CM_STATE_LAST_ACK: - case NES_CM_STATE_CLOSING: - cleanup_retrans_entry(cm_node); - cm_node->state = NES_CM_STATE_CLOSED; - cm_node->cm_id->rem_ref(cm_node->cm_id); - rem_ref_cm_node(cm_node->cm_core, cm_node); - drop_packet(skb); - break; - case NES_CM_STATE_FIN_WAIT1: - cleanup_retrans_entry(cm_node); - drop_packet(skb); - cm_node->state = NES_CM_STATE_FIN_WAIT2; - break; - case NES_CM_STATE_SYN_SENT: - case NES_CM_STATE_FIN_WAIT2: - case NES_CM_STATE_TSA: - case NES_CM_STATE_MPAREQ_RCVD: - case NES_CM_STATE_UNKNOWN: - default: - cleanup_retrans_entry(cm_node); - drop_packet(skb); - break; - } - return ret; -} - - - -static int handle_tcp_options(struct nes_cm_node *cm_node, struct tcphdr *tcph, - struct sk_buff *skb, int optionsize, int passive) -{ - u8 *optionsloc = (u8 *)&tcph[1]; - - if (optionsize) { - if (process_options(cm_node, optionsloc, optionsize, - (u32)tcph->syn)) { - nes_debug(NES_DBG_CM, "%s: Node %p, Sending RESET\n", - __func__, cm_node); - if (passive) - passive_open_err(cm_node, skb, 1); - else - active_open_err(cm_node, skb, 1); - return 1; - } - } - - cm_node->tcp_cntxt.snd_wnd = ntohs(tcph->window) << - cm_node->tcp_cntxt.snd_wscale; - - if (cm_node->tcp_cntxt.snd_wnd > cm_node->tcp_cntxt.max_snd_wnd) - cm_node->tcp_cntxt.max_snd_wnd = cm_node->tcp_cntxt.snd_wnd; - return 0; -} - -/* - * active_open_err() will send reset() if flag set.. - * It will also send ABORT event. - */ -static void active_open_err(struct nes_cm_node *cm_node, struct sk_buff *skb, - int reset) -{ - cleanup_retrans_entry(cm_node); - if (reset) { - nes_debug(NES_DBG_CM, "ERROR active err called for cm_node=%p, " - "state=%d\n", cm_node, cm_node->state); - add_ref_cm_node(cm_node); - send_reset(cm_node, skb); - } else { - dev_kfree_skb_any(skb); - } - - cm_node->state = NES_CM_STATE_CLOSED; - create_event(cm_node, NES_CM_EVENT_ABORTED); -} - -/* - * passive_open_err() will either do a reset() or will free up the skb and - * remove the cm_node. - */ -static void passive_open_err(struct nes_cm_node *cm_node, struct sk_buff *skb, - int reset) -{ - cleanup_retrans_entry(cm_node); - cm_node->state = NES_CM_STATE_CLOSED; - if (reset) { - nes_debug(NES_DBG_CM, "passive_open_err sending RST for " - "cm_node=%p state =%d\n", cm_node, cm_node->state); - send_reset(cm_node, skb); - } else { - dev_kfree_skb_any(skb); - rem_ref_cm_node(cm_node->cm_core, cm_node); - } -} - -/* - * free_retrans_entry() routines assumes that the retrans_list_lock has - * been acquired before calling. - */ -static void free_retrans_entry(struct nes_cm_node *cm_node) -{ - struct nes_timer_entry *send_entry; - - send_entry = cm_node->send_entry; - if (send_entry) { - cm_node->send_entry = NULL; - dev_kfree_skb_any(send_entry->skb); - kfree(send_entry); - rem_ref_cm_node(cm_node->cm_core, cm_node); - } -} - -static void cleanup_retrans_entry(struct nes_cm_node *cm_node) -{ - unsigned long flags; - - spin_lock_irqsave(&cm_node->retrans_list_lock, flags); - free_retrans_entry(cm_node); - spin_unlock_irqrestore(&cm_node->retrans_list_lock, flags); -} - -/** - * process_packet - * Returns skb if to be freed, else it will return NULL if already used.. - */ -static void process_packet(struct nes_cm_node *cm_node, struct sk_buff *skb, - struct nes_cm_core *cm_core) -{ - enum nes_tcpip_pkt_type pkt_type = NES_PKT_TYPE_UNKNOWN; - struct tcphdr *tcph = tcp_hdr(skb); - u32 fin_set = 0; - int ret = 0; - - skb_pull(skb, ip_hdr(skb)->ihl << 2); - - nes_debug(NES_DBG_CM, "process_packet: cm_node=%p state =%d syn=%d " - "ack=%d rst=%d fin=%d\n", cm_node, cm_node->state, tcph->syn, - tcph->ack, tcph->rst, tcph->fin); - - if (tcph->rst) { - pkt_type = NES_PKT_TYPE_RST; - } else if (tcph->syn) { - pkt_type = NES_PKT_TYPE_SYN; - if (tcph->ack) - pkt_type = NES_PKT_TYPE_SYNACK; - } else if (tcph->ack) { - pkt_type = NES_PKT_TYPE_ACK; - } - if (tcph->fin) - fin_set = 1; - - switch (pkt_type) { - case NES_PKT_TYPE_SYN: - handle_syn_pkt(cm_node, skb, tcph); - break; - case NES_PKT_TYPE_SYNACK: - handle_synack_pkt(cm_node, skb, tcph); - break; - case NES_PKT_TYPE_ACK: - ret = handle_ack_pkt(cm_node, skb, tcph); - if (fin_set && !ret) - handle_fin_pkt(cm_node); - break; - case NES_PKT_TYPE_RST: - handle_rst_pkt(cm_node, skb, tcph); - break; - default: - if ((fin_set) && (!check_seq(cm_node, tcph, skb))) - handle_fin_pkt(cm_node); - drop_packet(skb); - break; - } -} - -/** - * mini_cm_listen - create a listen node with params - */ -static struct nes_cm_listener *mini_cm_listen(struct nes_cm_core *cm_core, - struct nes_vnic *nesvnic, struct nes_cm_info *cm_info) -{ - struct nes_cm_listener *listener; - unsigned long flags; - - nes_debug(NES_DBG_CM, "Search for 0x%08x : 0x%04x\n", - cm_info->loc_addr, cm_info->loc_port); - - /* cannot have multiple matching listeners */ - listener = find_listener(cm_core, cm_info->loc_addr, cm_info->loc_port, - NES_CM_LISTENER_EITHER_STATE); - - if (listener && listener->listener_state == NES_CM_LISTENER_ACTIVE_STATE) { - /* find automatically incs ref count ??? */ - atomic_dec(&listener->ref_count); - nes_debug(NES_DBG_CM, "Not creating listener since it already exists\n"); - return NULL; - } - - if (!listener) { - /* create a CM listen node (1/2 node to compare incoming traffic to) */ - listener = kzalloc(sizeof(*listener), GFP_ATOMIC); - if (!listener) - return NULL; - - listener->loc_addr = cm_info->loc_addr; - listener->loc_port = cm_info->loc_port; - listener->reused_node = 0; - - atomic_set(&listener->ref_count, 1); - } - /* pasive case */ - /* find already inc'ed the ref count */ - else { - listener->reused_node = 1; - } - - listener->cm_id = cm_info->cm_id; - atomic_set(&listener->pend_accepts_cnt, 0); - listener->cm_core = cm_core; - listener->nesvnic = nesvnic; - atomic_inc(&cm_core->node_cnt); - - listener->conn_type = cm_info->conn_type; - listener->backlog = cm_info->backlog; - listener->listener_state = NES_CM_LISTENER_ACTIVE_STATE; - - if (!listener->reused_node) { - spin_lock_irqsave(&cm_core->listen_list_lock, flags); - list_add(&listener->list, &cm_core->listen_list.list); - spin_unlock_irqrestore(&cm_core->listen_list_lock, flags); - atomic_inc(&cm_core->listen_node_cnt); - } - - nes_debug(NES_DBG_CM, "Api - listen(): addr=0x%08X, port=0x%04x," - " listener = %p, backlog = %d, cm_id = %p.\n", - cm_info->loc_addr, cm_info->loc_port, - listener, listener->backlog, listener->cm_id); - - return listener; -} - - -/** - * mini_cm_connect - make a connection node with params - */ -static struct nes_cm_node *mini_cm_connect(struct nes_cm_core *cm_core, - struct nes_vnic *nesvnic, u16 private_data_len, - void *private_data, struct nes_cm_info *cm_info) -{ - int ret = 0; - struct nes_cm_node *cm_node; - struct nes_cm_listener *loopbackremotelistener; - struct nes_cm_node *loopbackremotenode; - struct nes_cm_info loopback_cm_info; - u8 *start_buff; - - /* create a CM connection node */ - cm_node = make_cm_node(cm_core, nesvnic, cm_info, NULL); - if (!cm_node) - return NULL; - - /* set our node side to client (active) side */ - cm_node->tcp_cntxt.client = 1; - cm_node->tcp_cntxt.rcv_wscale = NES_CM_DEFAULT_RCV_WND_SCALE; - - if (cm_info->loc_addr == cm_info->rem_addr) { - loopbackremotelistener = find_listener(cm_core, - cm_node->loc_addr, cm_node->rem_port, - NES_CM_LISTENER_ACTIVE_STATE); - if (loopbackremotelistener == NULL) { - create_event(cm_node, NES_CM_EVENT_ABORTED); - } else { - loopback_cm_info = *cm_info; - loopback_cm_info.loc_port = cm_info->rem_port; - loopback_cm_info.rem_port = cm_info->loc_port; - loopback_cm_info.loc_port = - cm_info->rem_port; - loopback_cm_info.rem_port = - cm_info->loc_port; - loopback_cm_info.cm_id = loopbackremotelistener->cm_id; - loopbackremotenode = make_cm_node(cm_core, nesvnic, - &loopback_cm_info, loopbackremotelistener); - if (!loopbackremotenode) { - rem_ref_cm_node(cm_node->cm_core, cm_node); - return NULL; - } - atomic_inc(&cm_loopbacks); - loopbackremotenode->loopbackpartner = cm_node; - loopbackremotenode->tcp_cntxt.rcv_wscale = - NES_CM_DEFAULT_RCV_WND_SCALE; - cm_node->loopbackpartner = loopbackremotenode; - memcpy(loopbackremotenode->mpa_frame_buf, private_data, - private_data_len); - loopbackremotenode->mpa_frame_size = private_data_len; - - /* we are done handling this state. */ - /* set node to a TSA state */ - cm_node->state = NES_CM_STATE_TSA; - cm_node->tcp_cntxt.rcv_nxt = - loopbackremotenode->tcp_cntxt.loc_seq_num; - loopbackremotenode->tcp_cntxt.rcv_nxt = - cm_node->tcp_cntxt.loc_seq_num; - cm_node->tcp_cntxt.max_snd_wnd = - loopbackremotenode->tcp_cntxt.rcv_wnd; - loopbackremotenode->tcp_cntxt.max_snd_wnd = - cm_node->tcp_cntxt.rcv_wnd; - cm_node->tcp_cntxt.snd_wnd = - loopbackremotenode->tcp_cntxt.rcv_wnd; - loopbackremotenode->tcp_cntxt.snd_wnd = - cm_node->tcp_cntxt.rcv_wnd; - cm_node->tcp_cntxt.snd_wscale = - loopbackremotenode->tcp_cntxt.rcv_wscale; - loopbackremotenode->tcp_cntxt.snd_wscale = - cm_node->tcp_cntxt.rcv_wscale; - loopbackremotenode->state = NES_CM_STATE_MPAREQ_RCVD; - create_event(loopbackremotenode, NES_CM_EVENT_MPA_REQ); - } - return cm_node; - } - - start_buff = &cm_node->mpa_frame_buf[0] + sizeof(struct ietf_mpa_v2); - cm_node->mpa_frame_size = private_data_len; - - memcpy(start_buff, private_data, private_data_len); - - /* send a syn and goto syn sent state */ - cm_node->state = NES_CM_STATE_SYN_SENT; - ret = send_syn(cm_node, 0, NULL); - - if (ret) { - /* error in sending the syn free up the cm_node struct */ - nes_debug(NES_DBG_CM, "Api - connect() FAILED: dest " - "addr=0x%08X, port=0x%04x, cm_node=%p, cm_id = %p.\n", - cm_node->rem_addr, cm_node->rem_port, cm_node, - cm_node->cm_id); - rem_ref_cm_node(cm_node->cm_core, cm_node); - cm_node = NULL; - } - - if (cm_node) { - nes_debug(NES_DBG_CM, "Api - connect(): dest addr=0x%08X," - "port=0x%04x, cm_node=%p, cm_id = %p.\n", - cm_node->rem_addr, cm_node->rem_port, cm_node, - cm_node->cm_id); - } - - return cm_node; -} - - -/** - * mini_cm_accept - accept a connection - * This function is never called - */ -static int mini_cm_accept(struct nes_cm_core *cm_core, struct nes_cm_node *cm_node) -{ - return 0; -} - - -/** - * mini_cm_reject - reject and teardown a connection - */ -static int mini_cm_reject(struct nes_cm_core *cm_core, struct nes_cm_node *cm_node) -{ - int ret = 0; - int err = 0; - int passive_state; - struct nes_cm_event event; - struct iw_cm_id *cm_id = cm_node->cm_id; - struct nes_cm_node *loopback = cm_node->loopbackpartner; - - nes_debug(NES_DBG_CM, "%s cm_node=%p type=%d state=%d\n", - __func__, cm_node, cm_node->tcp_cntxt.client, cm_node->state); - - if (cm_node->tcp_cntxt.client) - return ret; - cleanup_retrans_entry(cm_node); - - if (!loopback) { - passive_state = atomic_add_return(1, &cm_node->passive_state); - if (passive_state == NES_SEND_RESET_EVENT) { - cm_node->state = NES_CM_STATE_CLOSED; - rem_ref_cm_node(cm_core, cm_node); - } else { - if (cm_node->state == NES_CM_STATE_LISTENER_DESTROYED) { - rem_ref_cm_node(cm_core, cm_node); - } else { - ret = send_mpa_reject(cm_node); - if (ret) { - cm_node->state = NES_CM_STATE_CLOSED; - err = send_reset(cm_node, NULL); - if (err) - WARN_ON(1); - } else { - cm_id->add_ref(cm_id); - } - } - } - } else { - cm_node->cm_id = NULL; - if (cm_node->state == NES_CM_STATE_LISTENER_DESTROYED) { - rem_ref_cm_node(cm_core, cm_node); - rem_ref_cm_node(cm_core, loopback); - } else { - event.cm_node = loopback; - event.cm_info.rem_addr = loopback->rem_addr; - event.cm_info.loc_addr = loopback->loc_addr; - event.cm_info.rem_port = loopback->rem_port; - event.cm_info.loc_port = loopback->loc_port; - event.cm_info.cm_id = loopback->cm_id; - cm_event_mpa_reject(&event); - rem_ref_cm_node(cm_core, cm_node); - loopback->state = NES_CM_STATE_CLOSING; - - cm_id = loopback->cm_id; - rem_ref_cm_node(cm_core, loopback); - cm_id->rem_ref(cm_id); - } - } - - return ret; -} - - -/** - * mini_cm_close - */ -static int mini_cm_close(struct nes_cm_core *cm_core, struct nes_cm_node *cm_node) -{ - int ret = 0; - - if (!cm_core || !cm_node) - return -EINVAL; - - switch (cm_node->state) { - case NES_CM_STATE_SYN_RCVD: - case NES_CM_STATE_SYN_SENT: - case NES_CM_STATE_ONE_SIDE_ESTABLISHED: - case NES_CM_STATE_ESTABLISHED: - case NES_CM_STATE_ACCEPTING: - case NES_CM_STATE_MPAREQ_SENT: - case NES_CM_STATE_MPAREQ_RCVD: - cleanup_retrans_entry(cm_node); - send_reset(cm_node, NULL); - break; - case NES_CM_STATE_CLOSE_WAIT: - cm_node->state = NES_CM_STATE_LAST_ACK; - send_fin(cm_node, NULL); - break; - case NES_CM_STATE_FIN_WAIT1: - case NES_CM_STATE_FIN_WAIT2: - case NES_CM_STATE_LAST_ACK: - case NES_CM_STATE_TIME_WAIT: - case NES_CM_STATE_CLOSING: - ret = -1; - break; - case NES_CM_STATE_LISTENING: - cleanup_retrans_entry(cm_node); - send_reset(cm_node, NULL); - break; - case NES_CM_STATE_MPAREJ_RCVD: - case NES_CM_STATE_UNKNOWN: - case NES_CM_STATE_INITED: - case NES_CM_STATE_CLOSED: - case NES_CM_STATE_LISTENER_DESTROYED: - ret = rem_ref_cm_node(cm_core, cm_node); - break; - case NES_CM_STATE_TSA: - if (cm_node->send_entry) - printk(KERN_ERR "ERROR Close got called from STATE_TSA " - "send_entry=%p\n", cm_node->send_entry); - ret = rem_ref_cm_node(cm_core, cm_node); - break; - } - return ret; -} - - -/** - * recv_pkt - recv an ETHERNET packet, and process it through CM - * node state machine - */ -static int mini_cm_recv_pkt(struct nes_cm_core *cm_core, - struct nes_vnic *nesvnic, struct sk_buff *skb) -{ - struct nes_cm_node *cm_node = NULL; - struct nes_cm_listener *listener = NULL; - struct iphdr *iph; - struct tcphdr *tcph; - struct nes_cm_info nfo; - int skb_handled = 1; - __be32 tmp_daddr, tmp_saddr; - - if (!skb) - return 0; - if (skb->len < sizeof(struct iphdr) + sizeof(struct tcphdr)) - return 0; - - iph = (struct iphdr *)skb->data; - tcph = (struct tcphdr *)(skb->data + sizeof(struct iphdr)); - - nfo.loc_addr = ntohl(iph->daddr); - nfo.loc_port = ntohs(tcph->dest); - nfo.rem_addr = ntohl(iph->saddr); - nfo.rem_port = ntohs(tcph->source); - - tmp_daddr = cpu_to_be32(iph->daddr); - tmp_saddr = cpu_to_be32(iph->saddr); - - nes_debug(NES_DBG_CM, "Received packet: dest=%pI4:0x%04X src=%pI4:0x%04X\n", - &tmp_daddr, tcph->dest, &tmp_saddr, tcph->source); - - do { - cm_node = find_node(cm_core, - nfo.rem_port, nfo.rem_addr, - nfo.loc_port, nfo.loc_addr); - - if (!cm_node) { - /* Only type of packet accepted are for */ - /* the PASSIVE open (syn only) */ - if ((!tcph->syn) || (tcph->ack)) { - skb_handled = 0; - break; - } - listener = find_listener(cm_core, nfo.loc_addr, - nfo.loc_port, - NES_CM_LISTENER_ACTIVE_STATE); - if (!listener) { - nfo.cm_id = NULL; - nfo.conn_type = 0; - nes_debug(NES_DBG_CM, "Unable to find listener for the pkt\n"); - skb_handled = 0; - break; - } - nfo.cm_id = listener->cm_id; - nfo.conn_type = listener->conn_type; - cm_node = make_cm_node(cm_core, nesvnic, &nfo, - listener); - if (!cm_node) { - nes_debug(NES_DBG_CM, "Unable to allocate " - "node\n"); - cm_packets_dropped++; - atomic_dec(&listener->ref_count); - dev_kfree_skb_any(skb); - break; - } - if (!tcph->rst && !tcph->fin) { - cm_node->state = NES_CM_STATE_LISTENING; - } else { - cm_packets_dropped++; - rem_ref_cm_node(cm_core, cm_node); - dev_kfree_skb_any(skb); - break; - } - add_ref_cm_node(cm_node); - } else if (cm_node->state == NES_CM_STATE_TSA) { - if (cm_node->nesqp->pau_mode) - nes_queue_mgt_skbs(skb, nesvnic, cm_node->nesqp); - else { - rem_ref_cm_node(cm_core, cm_node); - atomic_inc(&cm_accel_dropped_pkts); - dev_kfree_skb_any(skb); - } - break; - } - skb_reset_network_header(skb); - skb_set_transport_header(skb, sizeof(*tcph)); - skb->len = ntohs(iph->tot_len); - process_packet(cm_node, skb, cm_core); - rem_ref_cm_node(cm_core, cm_node); - } while (0); - return skb_handled; -} - - -/** - * nes_cm_alloc_core - allocate a top level instance of a cm core - */ -static struct nes_cm_core *nes_cm_alloc_core(void) -{ - struct nes_cm_core *cm_core; - - /* setup the CM core */ - /* alloc top level core control structure */ - cm_core = kzalloc(sizeof(*cm_core), GFP_KERNEL); - if (!cm_core) - return NULL; - - INIT_LIST_HEAD(&cm_core->connected_nodes); - timer_setup(&cm_core->tcp_timer, nes_cm_timer_tick, 0); - - cm_core->mtu = NES_CM_DEFAULT_MTU; - cm_core->state = NES_CM_STATE_INITED; - cm_core->free_tx_pkt_max = NES_CM_DEFAULT_FREE_PKTS; - - atomic_set(&cm_core->events_posted, 0); - - cm_core->api = &nes_cm_api; - - spin_lock_init(&cm_core->ht_lock); - spin_lock_init(&cm_core->listen_list_lock); - - INIT_LIST_HEAD(&cm_core->listen_list.list); - - nes_debug(NES_DBG_CM, "Init CM Core completed -- cm_core=%p\n", cm_core); - - nes_debug(NES_DBG_CM, "Enable QUEUE EVENTS\n"); - cm_core->event_wq = alloc_ordered_workqueue("nesewq", 0); - if (!cm_core->event_wq) - goto out_free_cmcore; - cm_core->post_event = nes_cm_post_event; - nes_debug(NES_DBG_CM, "Enable QUEUE DISCONNECTS\n"); - cm_core->disconn_wq = alloc_ordered_workqueue("nesdwq", 0); - if (!cm_core->disconn_wq) - goto out_free_wq; - - print_core(cm_core); - return cm_core; - -out_free_wq: - destroy_workqueue(cm_core->event_wq); -out_free_cmcore: - kfree(cm_core); - return NULL; -} - - -/** - * mini_cm_dealloc_core - deallocate a top level instance of a cm core - */ -static int mini_cm_dealloc_core(struct nes_cm_core *cm_core) -{ - nes_debug(NES_DBG_CM, "De-Alloc CM Core (%p)\n", cm_core); - - if (!cm_core) - return -EINVAL; - - barrier(); - - if (timer_pending(&cm_core->tcp_timer)) - del_timer(&cm_core->tcp_timer); - - destroy_workqueue(cm_core->event_wq); - destroy_workqueue(cm_core->disconn_wq); - nes_debug(NES_DBG_CM, "\n"); - kfree(cm_core); - - return 0; -} - - -/** - * mini_cm_get - */ -static int mini_cm_get(struct nes_cm_core *cm_core) -{ - return cm_core->state; -} - - -/** - * mini_cm_set - */ -static int mini_cm_set(struct nes_cm_core *cm_core, u32 type, u32 value) -{ - int ret = 0; - - switch (type) { - case NES_CM_SET_PKT_SIZE: - cm_core->mtu = value; - break; - case NES_CM_SET_FREE_PKT_Q_SIZE: - cm_core->free_tx_pkt_max = value; - break; - default: - /* unknown set option */ - ret = -EINVAL; - } - - return ret; -} - - -/** - * nes_cm_init_tsa_conn setup HW; MPA frames must be - * successfully exchanged when this is called - */ -static int nes_cm_init_tsa_conn(struct nes_qp *nesqp, struct nes_cm_node *cm_node) -{ - int ret = 0; - - if (!nesqp) - return -EINVAL; - - nesqp->nesqp_context->misc |= cpu_to_le32(NES_QPCONTEXT_MISC_IPV4 | - NES_QPCONTEXT_MISC_NO_NAGLE | NES_QPCONTEXT_MISC_DO_NOT_FRAG | - NES_QPCONTEXT_MISC_DROS); - - if (cm_node->tcp_cntxt.snd_wscale || cm_node->tcp_cntxt.rcv_wscale) - nesqp->nesqp_context->misc |= cpu_to_le32(NES_QPCONTEXT_MISC_WSCALE); - - nesqp->nesqp_context->misc2 |= cpu_to_le32(64 << NES_QPCONTEXT_MISC2_TTL_SHIFT); - - nesqp->nesqp_context->misc2 |= cpu_to_le32( - cm_node->tos << NES_QPCONTEXT_MISC2_TOS_SHIFT); - - nesqp->nesqp_context->mss |= cpu_to_le32(((u32)cm_node->tcp_cntxt.mss) << 16); - - nesqp->nesqp_context->tcp_state_flow_label |= cpu_to_le32( - (u32)NES_QPCONTEXT_TCPSTATE_EST << NES_QPCONTEXT_TCPFLOW_TCP_STATE_SHIFT); - - nesqp->nesqp_context->pd_index_wscale |= cpu_to_le32( - (cm_node->tcp_cntxt.snd_wscale << NES_QPCONTEXT_PDWSCALE_SND_WSCALE_SHIFT) & - NES_QPCONTEXT_PDWSCALE_SND_WSCALE_MASK); - - nesqp->nesqp_context->pd_index_wscale |= cpu_to_le32( - (cm_node->tcp_cntxt.rcv_wscale << NES_QPCONTEXT_PDWSCALE_RCV_WSCALE_SHIFT) & - NES_QPCONTEXT_PDWSCALE_RCV_WSCALE_MASK); - - nesqp->nesqp_context->keepalive = cpu_to_le32(0x80); - nesqp->nesqp_context->ts_recent = 0; - nesqp->nesqp_context->ts_age = 0; - nesqp->nesqp_context->snd_nxt = cpu_to_le32(cm_node->tcp_cntxt.loc_seq_num); - nesqp->nesqp_context->snd_wnd = cpu_to_le32(cm_node->tcp_cntxt.snd_wnd); - nesqp->nesqp_context->rcv_nxt = cpu_to_le32(cm_node->tcp_cntxt.rcv_nxt); - nesqp->nesqp_context->rcv_wnd = cpu_to_le32(cm_node->tcp_cntxt.rcv_wnd << - cm_node->tcp_cntxt.rcv_wscale); - nesqp->nesqp_context->snd_max = cpu_to_le32(cm_node->tcp_cntxt.loc_seq_num); - nesqp->nesqp_context->snd_una = cpu_to_le32(cm_node->tcp_cntxt.loc_seq_num); - nesqp->nesqp_context->srtt = 0; - nesqp->nesqp_context->rttvar = cpu_to_le32(0x6); - nesqp->nesqp_context->ssthresh = cpu_to_le32(0x3FFFC000); - nesqp->nesqp_context->cwnd = cpu_to_le32(2 * cm_node->tcp_cntxt.mss); - nesqp->nesqp_context->snd_wl1 = cpu_to_le32(cm_node->tcp_cntxt.rcv_nxt); - nesqp->nesqp_context->snd_wl2 = cpu_to_le32(cm_node->tcp_cntxt.loc_seq_num); - nesqp->nesqp_context->max_snd_wnd = cpu_to_le32(cm_node->tcp_cntxt.max_snd_wnd); - - nes_debug(NES_DBG_CM, "QP%u: rcv_nxt = 0x%08X, snd_nxt = 0x%08X," - " Setting MSS to %u, PDWscale = 0x%08X, rcv_wnd = %u, context misc = 0x%08X.\n", - nesqp->hwqp.qp_id, le32_to_cpu(nesqp->nesqp_context->rcv_nxt), - le32_to_cpu(nesqp->nesqp_context->snd_nxt), - cm_node->tcp_cntxt.mss, le32_to_cpu(nesqp->nesqp_context->pd_index_wscale), - le32_to_cpu(nesqp->nesqp_context->rcv_wnd), - le32_to_cpu(nesqp->nesqp_context->misc)); - nes_debug(NES_DBG_CM, " snd_wnd = 0x%08X.\n", le32_to_cpu(nesqp->nesqp_context->snd_wnd)); - nes_debug(NES_DBG_CM, " snd_cwnd = 0x%08X.\n", le32_to_cpu(nesqp->nesqp_context->cwnd)); - nes_debug(NES_DBG_CM, " max_swnd = 0x%08X.\n", le32_to_cpu(nesqp->nesqp_context->max_snd_wnd)); - - nes_debug(NES_DBG_CM, "Change cm_node state to TSA\n"); - cm_node->state = NES_CM_STATE_TSA; - - return ret; -} - - -/** - * nes_cm_disconn - */ -int nes_cm_disconn(struct nes_qp *nesqp) -{ - struct disconn_work *work; - - work = kzalloc(sizeof *work, GFP_ATOMIC); - if (!work) - return -ENOMEM; /* Timer will clean up */ - - nes_add_ref(&nesqp->ibqp); - work->nesqp = nesqp; - INIT_WORK(&work->work, nes_disconnect_worker); - queue_work(g_cm_core->disconn_wq, &work->work); - return 0; -} - - -/** - * nes_disconnect_worker - */ -static void nes_disconnect_worker(struct work_struct *work) -{ - struct disconn_work *dwork = container_of(work, struct disconn_work, work); - struct nes_qp *nesqp = dwork->nesqp; - - kfree(dwork); - nes_debug(NES_DBG_CM, "processing AEQE id 0x%04X for QP%u.\n", - nesqp->last_aeq, nesqp->hwqp.qp_id); - nes_cm_disconn_true(nesqp); - nes_rem_ref(&nesqp->ibqp); -} - - -/** - * nes_cm_disconn_true - */ -static int nes_cm_disconn_true(struct nes_qp *nesqp) -{ - unsigned long flags; - int ret = 0; - struct iw_cm_id *cm_id; - struct iw_cm_event cm_event; - struct nes_vnic *nesvnic; - u16 last_ae; - u8 original_hw_tcp_state; - u8 original_ibqp_state; - int disconn_status = 0; - int issue_disconn = 0; - int issue_close = 0; - int issue_flush = 0; - u32 flush_q = NES_CQP_FLUSH_RQ; - struct ib_event ibevent; - - if (!nesqp) { - nes_debug(NES_DBG_CM, "disconnect_worker nesqp is NULL\n"); - return -1; - } - - spin_lock_irqsave(&nesqp->lock, flags); - cm_id = nesqp->cm_id; - /* make sure we havent already closed this connection */ - if (!cm_id) { - nes_debug(NES_DBG_CM, "QP%u disconnect_worker cmid is NULL\n", - nesqp->hwqp.qp_id); - spin_unlock_irqrestore(&nesqp->lock, flags); - return -1; - } - - nesvnic = to_nesvnic(nesqp->ibqp.device); - nes_debug(NES_DBG_CM, "Disconnecting QP%u\n", nesqp->hwqp.qp_id); - - original_hw_tcp_state = nesqp->hw_tcp_state; - original_ibqp_state = nesqp->ibqp_state; - last_ae = nesqp->last_aeq; - - if (nesqp->term_flags) { - issue_disconn = 1; - issue_close = 1; - nesqp->cm_id = NULL; - del_timer(&nesqp->terminate_timer); - if (nesqp->flush_issued == 0) { - nesqp->flush_issued = 1; - issue_flush = 1; - } - } else if ((original_hw_tcp_state == NES_AEQE_TCP_STATE_CLOSE_WAIT) || - ((original_ibqp_state == IB_QPS_RTS) && - (last_ae == NES_AEQE_AEID_LLP_CONNECTION_RESET))) { - issue_disconn = 1; - if (last_ae == NES_AEQE_AEID_LLP_CONNECTION_RESET) - disconn_status = -ECONNRESET; - } - - if (((original_hw_tcp_state == NES_AEQE_TCP_STATE_CLOSED) || - (original_hw_tcp_state == NES_AEQE_TCP_STATE_TIME_WAIT) || - (last_ae == NES_AEQE_AEID_RDMAP_ROE_BAD_LLP_CLOSE) || - (last_ae == NES_AEQE_AEID_LLP_CONNECTION_RESET))) { - issue_close = 1; - nesqp->cm_id = NULL; - if (nesqp->flush_issued == 0) { - nesqp->flush_issued = 1; - issue_flush = 1; - } - } - - spin_unlock_irqrestore(&nesqp->lock, flags); - - if ((issue_flush) && (nesqp->destroyed == 0)) { - /* Flush the queue(s) */ - if (nesqp->hw_iwarp_state >= NES_AEQE_IWARP_STATE_TERMINATE) - flush_q |= NES_CQP_FLUSH_SQ; - flush_wqes(nesvnic->nesdev, nesqp, flush_q, 1); - - if (nesqp->term_flags) { - ibevent.device = nesqp->ibqp.device; - ibevent.event = nesqp->terminate_eventtype; - ibevent.element.qp = &nesqp->ibqp; - if (nesqp->ibqp.event_handler) - nesqp->ibqp.event_handler(&ibevent, nesqp->ibqp.qp_context); - } - } - - if ((cm_id) && (cm_id->event_handler)) { - if (issue_disconn) { - atomic_inc(&cm_disconnects); - cm_event.event = IW_CM_EVENT_DISCONNECT; - cm_event.status = disconn_status; - cm_event.local_addr = cm_id->m_local_addr; - cm_event.remote_addr = cm_id->m_remote_addr; - cm_event.private_data = NULL; - cm_event.private_data_len = 0; - - nes_debug(NES_DBG_CM, "Generating a CM Disconnect Event" - " for QP%u, SQ Head = %u, SQ Tail = %u. " - "cm_id = %p, refcount = %u.\n", - nesqp->hwqp.qp_id, nesqp->hwqp.sq_head, - nesqp->hwqp.sq_tail, cm_id, - atomic_read(&nesqp->refcount)); - - ret = cm_id->event_handler(cm_id, &cm_event); - if (ret) - nes_debug(NES_DBG_CM, "OFA CM event_handler " - "returned, ret=%d\n", ret); - } - - if (issue_close) { - atomic_inc(&cm_closes); - nes_disconnect(nesqp, 1); - - cm_id->provider_data = nesqp; - /* Send up the close complete event */ - cm_event.event = IW_CM_EVENT_CLOSE; - cm_event.status = 0; - cm_event.provider_data = cm_id->provider_data; - cm_event.local_addr = cm_id->m_local_addr; - cm_event.remote_addr = cm_id->m_remote_addr; - cm_event.private_data = NULL; - cm_event.private_data_len = 0; - - ret = cm_id->event_handler(cm_id, &cm_event); - if (ret) - nes_debug(NES_DBG_CM, "OFA CM event_handler returned, ret=%d\n", ret); - - cm_id->rem_ref(cm_id); - } - } - - return 0; -} - - -/** - * nes_disconnect - */ -static int nes_disconnect(struct nes_qp *nesqp, int abrupt) -{ - int ret = 0; - struct nes_vnic *nesvnic; - struct nes_device *nesdev; - struct nes_ib_device *nesibdev; - - nesvnic = to_nesvnic(nesqp->ibqp.device); - if (!nesvnic) - return -EINVAL; - - nesdev = nesvnic->nesdev; - nesibdev = nesvnic->nesibdev; - - nes_debug(NES_DBG_CM, "netdev refcnt = %u.\n", - netdev_refcnt_read(nesvnic->netdev)); - - if (nesqp->active_conn) { - - /* indicate this connection is NOT active */ - nesqp->active_conn = 0; - } else { - /* Need to free the Last Streaming Mode Message */ - if (nesqp->ietf_frame) { - if (nesqp->lsmm_mr) - nesibdev->ibdev.ops.dereg_mr(nesqp->lsmm_mr, - NULL); - pci_free_consistent(nesdev->pcidev, - nesqp->private_data_len + nesqp->ietf_frame_size, - nesqp->ietf_frame, nesqp->ietf_frame_pbase); - } - } - - /* close the CM node down if it is still active */ - if (nesqp->cm_node) { - nes_debug(NES_DBG_CM, "Call close API\n"); - - g_cm_core->api->close(g_cm_core, nesqp->cm_node); - } - - return ret; -} - - -/** - * nes_accept - */ -int nes_accept(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param) -{ - u64 u64temp; - struct ib_qp *ibqp; - struct nes_qp *nesqp; - struct nes_vnic *nesvnic; - struct nes_device *nesdev; - struct nes_cm_node *cm_node; - struct nes_adapter *adapter; - struct ib_qp_attr attr; - struct iw_cm_event cm_event; - struct nes_hw_qp_wqe *wqe; - struct nes_v4_quad nes_quad; - u32 crc_value; - int ret; - int passive_state; - struct ib_mr *ibmr = NULL; - struct nes_pd *nespd; - u64 tagged_offset; - u8 mpa_frame_offset = 0; - struct ietf_mpa_v2 *mpa_v2_frame; - u8 start_addr = 0; - u8 *start_ptr = &start_addr; - u8 **start_buff = &start_ptr; - u16 buff_len = 0; - struct sockaddr_in *laddr = (struct sockaddr_in *)&cm_id->m_local_addr; - struct sockaddr_in *raddr = (struct sockaddr_in *)&cm_id->m_remote_addr; - - ibqp = nes_get_qp(cm_id->device, conn_param->qpn); - if (!ibqp) - return -EINVAL; - - /* get all our handles */ - nesqp = to_nesqp(ibqp); - nesvnic = to_nesvnic(nesqp->ibqp.device); - nesdev = nesvnic->nesdev; - adapter = nesdev->nesadapter; - - cm_node = (struct nes_cm_node *)cm_id->provider_data; - nes_debug(NES_DBG_CM, "nes_accept: cm_node= %p nesvnic=%p, netdev=%p," - "%s\n", cm_node, nesvnic, nesvnic->netdev, - nesvnic->netdev->name); - - if (NES_CM_STATE_LISTENER_DESTROYED == cm_node->state) { - if (cm_node->loopbackpartner) - rem_ref_cm_node(cm_node->cm_core, cm_node->loopbackpartner); - rem_ref_cm_node(cm_node->cm_core, cm_node); - return -EINVAL; - } - - passive_state = atomic_add_return(1, &cm_node->passive_state); - if (passive_state == NES_SEND_RESET_EVENT) { - rem_ref_cm_node(cm_node->cm_core, cm_node); - return -ECONNRESET; - } - /* associate the node with the QP */ - nesqp->cm_node = (void *)cm_node; - cm_node->nesqp = nesqp; - - - nes_debug(NES_DBG_CM, "QP%u, cm_node=%p, jiffies = %lu listener = %p\n", - nesqp->hwqp.qp_id, cm_node, jiffies, cm_node->listener); - atomic_inc(&cm_accepts); - - nes_debug(NES_DBG_CM, "netdev refcnt = %u.\n", - netdev_refcnt_read(nesvnic->netdev)); - - nesqp->ietf_frame_size = sizeof(struct ietf_mpa_v2); - /* allocate the ietf frame and space for private data */ - nesqp->ietf_frame = pci_alloc_consistent(nesdev->pcidev, - nesqp->ietf_frame_size + conn_param->private_data_len, - &nesqp->ietf_frame_pbase); - - if (!nesqp->ietf_frame) { - nes_debug(NES_DBG_CM, "Unable to allocate memory for private data\n"); - return -ENOMEM; - } - mpa_v2_frame = (struct ietf_mpa_v2 *)nesqp->ietf_frame; - - if (cm_node->mpa_frame_rev == IETF_MPA_V1) - mpa_frame_offset = 4; - - if (cm_node->mpa_frame_rev == IETF_MPA_V1 || - cm_node->mpav2_ird_ord == IETF_NO_IRD_ORD) { - record_ird_ord(cm_node, (u16)conn_param->ird, (u16)conn_param->ord); - } - - memcpy(mpa_v2_frame->priv_data, conn_param->private_data, - conn_param->private_data_len); - - cm_build_mpa_frame(cm_node, start_buff, &buff_len, nesqp->ietf_frame, MPA_KEY_REPLY); - nesqp->private_data_len = conn_param->private_data_len; - - /* setup our first outgoing iWarp send WQE (the IETF frame response) */ - wqe = &nesqp->hwqp.sq_vbase[0]; - - if (raddr->sin_addr.s_addr != laddr->sin_addr.s_addr) { - u64temp = (unsigned long)nesqp; - nespd = nesqp->nespd; - tagged_offset = (u64)(unsigned long)*start_buff; - ibmr = nes_reg_phys_mr(&nespd->ibpd, - nesqp->ietf_frame_pbase + mpa_frame_offset, - buff_len, IB_ACCESS_LOCAL_WRITE, - &tagged_offset); - if (IS_ERR(ibmr)) { - nes_debug(NES_DBG_CM, "Unable to register memory region" - "for lSMM for cm_node = %p \n", - cm_node); - pci_free_consistent(nesdev->pcidev, - nesqp->private_data_len + nesqp->ietf_frame_size, - nesqp->ietf_frame, nesqp->ietf_frame_pbase); - return PTR_ERR(ibmr); - } - - ibmr->pd = &nespd->ibpd; - ibmr->device = nespd->ibpd.device; - nesqp->lsmm_mr = ibmr; - - u64temp |= NES_SW_CONTEXT_ALIGN >> 1; - set_wqe_64bit_value(wqe->wqe_words, - NES_IWARP_SQ_WQE_COMP_CTX_LOW_IDX, - u64temp); - wqe->wqe_words[NES_IWARP_SQ_WQE_MISC_IDX] = - cpu_to_le32(NES_IWARP_SQ_WQE_STREAMING | - NES_IWARP_SQ_WQE_WRPDU); - wqe->wqe_words[NES_IWARP_SQ_WQE_TOTAL_PAYLOAD_IDX] = - cpu_to_le32(buff_len); - set_wqe_64bit_value(wqe->wqe_words, - NES_IWARP_SQ_WQE_FRAG0_LOW_IDX, - (u64)(unsigned long)(*start_buff)); - wqe->wqe_words[NES_IWARP_SQ_WQE_LENGTH0_IDX] = - cpu_to_le32(buff_len); - wqe->wqe_words[NES_IWARP_SQ_WQE_STAG0_IDX] = ibmr->lkey; - if (nesqp->sq_kmapped) { - nesqp->sq_kmapped = 0; - kunmap(nesqp->page); - } - - nesqp->nesqp_context->ird_ord_sizes |= - cpu_to_le32(NES_QPCONTEXT_ORDIRD_LSMM_PRESENT | - NES_QPCONTEXT_ORDIRD_WRPDU); - } else { - nesqp->nesqp_context->ird_ord_sizes |= - cpu_to_le32(NES_QPCONTEXT_ORDIRD_WRPDU); - } - nesqp->skip_lsmm = 1; - - /* Cache the cm_id in the qp */ - nesqp->cm_id = cm_id; - cm_node->cm_id = cm_id; - - /* nesqp->cm_node = (void *)cm_id->provider_data; */ - cm_id->provider_data = nesqp; - nesqp->active_conn = 0; - - if (cm_node->state == NES_CM_STATE_TSA) - nes_debug(NES_DBG_CM, "Already state = TSA for cm_node=%p\n", - cm_node); - - nes_cm_init_tsa_conn(nesqp, cm_node); - - nesqp->nesqp_context->tcpPorts[0] = - cpu_to_le16(cm_node->loc_port); - nesqp->nesqp_context->tcpPorts[1] = - cpu_to_le16(cm_node->rem_port); - - nesqp->nesqp_context->ip0 = cpu_to_le32(cm_node->rem_addr); - - nesqp->nesqp_context->misc2 |= cpu_to_le32( - (u32)PCI_FUNC(nesdev->pcidev->devfn) << - NES_QPCONTEXT_MISC2_SRC_IP_SHIFT); - - nesqp->nesqp_context->arp_index_vlan |= - cpu_to_le32(nes_arp_table(nesdev, - le32_to_cpu(nesqp->nesqp_context->ip0), NULL, - NES_ARP_RESOLVE) << 16); - - nesqp->nesqp_context->ts_val_delta = cpu_to_le32( - jiffies - nes_read_indexed(nesdev, NES_IDX_TCP_NOW)); - - nesqp->nesqp_context->ird_index = cpu_to_le32(nesqp->hwqp.qp_id); - - nesqp->nesqp_context->ird_ord_sizes |= cpu_to_le32( - ((u32)1 << NES_QPCONTEXT_ORDIRD_IWARP_MODE_SHIFT)); - nesqp->nesqp_context->ird_ord_sizes |= - cpu_to_le32((u32)cm_node->ord_size); - - memset(&nes_quad, 0, sizeof(nes_quad)); - nes_quad.DstIpAdrIndex = - cpu_to_le32((u32)PCI_FUNC(nesdev->pcidev->devfn) << 24); - nes_quad.SrcIpadr = htonl(cm_node->rem_addr); - nes_quad.TcpPorts[0] = htons(cm_node->rem_port); - nes_quad.TcpPorts[1] = htons(cm_node->loc_port); - - /* Produce hash key */ - crc_value = get_crc_value(&nes_quad); - nesqp->hte_index = cpu_to_be32(crc_value ^ 0xffffffff); - nes_debug(NES_DBG_CM, "HTE Index = 0x%08X, CRC = 0x%08X\n", - nesqp->hte_index, nesqp->hte_index & adapter->hte_index_mask); - - nesqp->hte_index &= adapter->hte_index_mask; - nesqp->nesqp_context->hte_index = cpu_to_le32(nesqp->hte_index); - - cm_node->cm_core->api->accelerated(cm_node->cm_core, cm_node); - - nes_debug(NES_DBG_CM, "QP%u, Destination IP = 0x%08X:0x%04X, local = " - "0x%08X:0x%04X, rcv_nxt=0x%08X, snd_nxt=0x%08X, mpa + " - "private data length=%u.\n", nesqp->hwqp.qp_id, - ntohl(raddr->sin_addr.s_addr), ntohs(raddr->sin_port), - ntohl(laddr->sin_addr.s_addr), ntohs(laddr->sin_port), - le32_to_cpu(nesqp->nesqp_context->rcv_nxt), - le32_to_cpu(nesqp->nesqp_context->snd_nxt), - buff_len); - - /* notify OF layer that accept event was successful */ - cm_id->add_ref(cm_id); - nes_add_ref(&nesqp->ibqp); - - cm_event.event = IW_CM_EVENT_ESTABLISHED; - cm_event.status = 0; - cm_event.provider_data = (void *)nesqp; - cm_event.local_addr = cm_id->m_local_addr; - cm_event.remote_addr = cm_id->m_remote_addr; - cm_event.private_data = NULL; - cm_event.private_data_len = 0; - cm_event.ird = cm_node->ird_size; - cm_event.ord = cm_node->ord_size; - - ret = cm_id->event_handler(cm_id, &cm_event); - attr.qp_state = IB_QPS_RTS; - nes_modify_qp(&nesqp->ibqp, &attr, IB_QP_STATE, NULL); - if (cm_node->loopbackpartner) { - cm_node->loopbackpartner->mpa_frame_size = - nesqp->private_data_len; - /* copy entire MPA frame to our cm_node's frame */ - memcpy(cm_node->loopbackpartner->mpa_frame_buf, - conn_param->private_data, conn_param->private_data_len); - create_event(cm_node->loopbackpartner, NES_CM_EVENT_CONNECTED); - } - if (ret) - printk(KERN_ERR "%s[%u] OFA CM event_handler returned, " - "ret=%d\n", __func__, __LINE__, ret); - - return 0; -} - - -/** - * nes_reject - */ -int nes_reject(struct iw_cm_id *cm_id, const void *pdata, u8 pdata_len) -{ - struct nes_cm_node *cm_node; - struct nes_cm_node *loopback; - struct nes_cm_core *cm_core; - u8 *start_buff; - - atomic_inc(&cm_rejects); - cm_node = (struct nes_cm_node *)cm_id->provider_data; - loopback = cm_node->loopbackpartner; - cm_core = cm_node->cm_core; - cm_node->cm_id = cm_id; - - if (pdata_len + sizeof(struct ietf_mpa_v2) > MAX_CM_BUFFER) - return -EINVAL; - - if (loopback) { - memcpy(&loopback->mpa_frame.priv_data, pdata, pdata_len); - loopback->mpa_frame.priv_data_len = pdata_len; - loopback->mpa_frame_size = pdata_len; - } else { - start_buff = &cm_node->mpa_frame_buf[0] + sizeof(struct ietf_mpa_v2); - cm_node->mpa_frame_size = pdata_len; - memcpy(start_buff, pdata, pdata_len); - } - return cm_core->api->reject(cm_core, cm_node); -} - - -/** - * nes_connect - * setup and launch cm connect node - */ -int nes_connect(struct iw_cm_id *cm_id, struct iw_cm_conn_param *conn_param) -{ - struct ib_qp *ibqp; - struct nes_qp *nesqp; - struct nes_vnic *nesvnic; - struct nes_device *nesdev; - struct nes_cm_node *cm_node; - struct nes_cm_info cm_info; - int apbvt_set = 0; - struct sockaddr_in *laddr = (struct sockaddr_in *)&cm_id->m_local_addr; - struct sockaddr_in *raddr = (struct sockaddr_in *)&cm_id->m_remote_addr; - - if (cm_id->remote_addr.ss_family != AF_INET) - return -ENOSYS; - ibqp = nes_get_qp(cm_id->device, conn_param->qpn); - if (!ibqp) - return -EINVAL; - nesqp = to_nesqp(ibqp); - if (!nesqp) - return -EINVAL; - nesvnic = to_nesvnic(nesqp->ibqp.device); - if (!nesvnic) - return -EINVAL; - nesdev = nesvnic->nesdev; - if (!nesdev) - return -EINVAL; - - if (!laddr->sin_port || !raddr->sin_port) - return -EINVAL; - - nes_debug(NES_DBG_CM, "QP%u, current IP = 0x%08X, Destination IP = " - "0x%08X:0x%04X, local = 0x%08X:0x%04X.\n", nesqp->hwqp.qp_id, - ntohl(nesvnic->local_ipaddr), ntohl(raddr->sin_addr.s_addr), - ntohs(raddr->sin_port), ntohl(laddr->sin_addr.s_addr), - ntohs(laddr->sin_port)); - - atomic_inc(&cm_connects); - nesqp->active_conn = 1; - - /* cache the cm_id in the qp */ - nesqp->cm_id = cm_id; - cm_id->provider_data = nesqp; - nesqp->private_data_len = conn_param->private_data_len; - - nes_debug(NES_DBG_CM, "requested ord = 0x%08X.\n", (u32)conn_param->ord); - nes_debug(NES_DBG_CM, "mpa private data len =%u\n", - conn_param->private_data_len); - - /* set up the connection params for the node */ - cm_info.loc_addr = ntohl(laddr->sin_addr.s_addr); - cm_info.loc_port = ntohs(laddr->sin_port); - cm_info.rem_addr = ntohl(raddr->sin_addr.s_addr); - cm_info.rem_port = ntohs(raddr->sin_port); - cm_info.cm_id = cm_id; - cm_info.conn_type = NES_CM_IWARP_CONN_TYPE; - - if (laddr->sin_addr.s_addr != raddr->sin_addr.s_addr) { - nes_manage_apbvt(nesvnic, cm_info.loc_port, - PCI_FUNC(nesdev->pcidev->devfn), - NES_MANAGE_APBVT_ADD); - apbvt_set = 1; - } - - cm_id->add_ref(cm_id); - - /* create a connect CM node connection */ - cm_node = g_cm_core->api->connect(g_cm_core, nesvnic, - conn_param->private_data_len, (void *)conn_param->private_data, - &cm_info); - if (!cm_node) { - if (apbvt_set) - nes_manage_apbvt(nesvnic, cm_info.loc_port, - PCI_FUNC(nesdev->pcidev->devfn), - NES_MANAGE_APBVT_DEL); - - nes_debug(NES_DBG_NLMSG, "Delete loc_port = %04X\n", - cm_info.loc_port); - cm_id->rem_ref(cm_id); - return -ENOMEM; - } - - record_ird_ord(cm_node, (u16)conn_param->ird, (u16)conn_param->ord); - if (cm_node->send_rdma0_op == SEND_RDMA_READ_ZERO && - cm_node->ord_size == 0) - cm_node->ord_size = 1; - - cm_node->apbvt_set = apbvt_set; - cm_node->tos = cm_id->tos; - nesqp->cm_node = cm_node; - cm_node->nesqp = nesqp; - nes_add_ref(&nesqp->ibqp); - - return 0; -} - - -/** - * nes_create_listen - */ -int nes_create_listen(struct iw_cm_id *cm_id, int backlog) -{ - struct nes_vnic *nesvnic; - struct nes_cm_listener *cm_node; - struct nes_cm_info cm_info; - int err; - struct sockaddr_in *laddr = (struct sockaddr_in *)&cm_id->m_local_addr; - - nes_debug(NES_DBG_CM, "cm_id = %p, local port = 0x%04X.\n", - cm_id, ntohs(laddr->sin_port)); - - if (cm_id->m_local_addr.ss_family != AF_INET) - return -ENOSYS; - nesvnic = to_nesvnic(cm_id->device); - if (!nesvnic) - return -EINVAL; - - nes_debug(NES_DBG_CM, "nesvnic=%p, netdev=%p, %s\n", - nesvnic, nesvnic->netdev, nesvnic->netdev->name); - - nes_debug(NES_DBG_CM, "nesvnic->local_ipaddr=0x%08x, sin_addr.s_addr=0x%08x\n", - nesvnic->local_ipaddr, laddr->sin_addr.s_addr); - - /* setup listen params in our api call struct */ - cm_info.loc_addr = ntohl(nesvnic->local_ipaddr); - cm_info.loc_port = ntohs(laddr->sin_port); - cm_info.backlog = backlog; - cm_info.cm_id = cm_id; - - cm_info.conn_type = NES_CM_IWARP_CONN_TYPE; - - cm_node = g_cm_core->api->listen(g_cm_core, nesvnic, &cm_info); - if (!cm_node) { - printk(KERN_ERR "%s[%u] Error returned from listen API call\n", - __func__, __LINE__); - return -ENOMEM; - } - - cm_id->provider_data = cm_node; - cm_node->tos = cm_id->tos; - - if (!cm_node->reused_node) { - err = nes_manage_apbvt(nesvnic, cm_node->loc_port, - PCI_FUNC(nesvnic->nesdev->pcidev->devfn), - NES_MANAGE_APBVT_ADD); - if (err) { - printk(KERN_ERR "nes_manage_apbvt call returned %d.\n", - err); - g_cm_core->api->stop_listener(g_cm_core, (void *)cm_node); - return err; - } - atomic_inc(&cm_listens_created); - } - - cm_id->add_ref(cm_id); - cm_id->provider_data = (void *)cm_node; - - - return 0; -} - - -/** - * nes_destroy_listen - */ -int nes_destroy_listen(struct iw_cm_id *cm_id) -{ - if (cm_id->provider_data) - g_cm_core->api->stop_listener(g_cm_core, cm_id->provider_data); - else - nes_debug(NES_DBG_CM, "cm_id->provider_data was NULL\n"); - - cm_id->rem_ref(cm_id); - - return 0; -} - - -/** - * nes_cm_recv - */ -int nes_cm_recv(struct sk_buff *skb, struct net_device *netdevice) -{ - int rc = 0; - - cm_packets_received++; - if ((g_cm_core) && (g_cm_core->api)) - rc = g_cm_core->api->recv_pkt(g_cm_core, netdev_priv(netdevice), skb); - else - nes_debug(NES_DBG_CM, "Unable to process packet for CM," - " cm is not setup properly.\n"); - - return rc; -} - - -/** - * nes_cm_start - * Start and init a cm core module - */ -int nes_cm_start(void) -{ - nes_debug(NES_DBG_CM, "\n"); - /* create the primary CM core, pass this handle to subsequent core inits */ - g_cm_core = nes_cm_alloc_core(); - if (g_cm_core) - return 0; - else - return -ENOMEM; -} - - -/** - * nes_cm_stop - * stop and dealloc all cm core instances - */ -int nes_cm_stop(void) -{ - g_cm_core->api->destroy_cm_core(g_cm_core); - return 0; -} - - -/** - * cm_event_connected - * handle a connected event, setup QPs and HW - */ -static void cm_event_connected(struct nes_cm_event *event) -{ - struct nes_qp *nesqp; - struct nes_vnic *nesvnic; - struct nes_device *nesdev; - struct nes_cm_node *cm_node; - struct nes_adapter *nesadapter; - struct ib_qp_attr attr; - struct iw_cm_id *cm_id; - struct iw_cm_event cm_event; - struct nes_v4_quad nes_quad; - u32 crc_value; - int ret; - struct sockaddr_in *laddr; - struct sockaddr_in *raddr; - struct sockaddr_in *cm_event_laddr; - - /* get all our handles */ - cm_node = event->cm_node; - cm_id = cm_node->cm_id; - nes_debug(NES_DBG_CM, "cm_event_connected - %p - cm_id = %p\n", cm_node, cm_id); - nesqp = (struct nes_qp *)cm_id->provider_data; - nesvnic = to_nesvnic(nesqp->ibqp.device); - nesdev = nesvnic->nesdev; - nesadapter = nesdev->nesadapter; - laddr = (struct sockaddr_in *)&cm_id->m_local_addr; - raddr = (struct sockaddr_in *)&cm_id->m_remote_addr; - cm_event_laddr = (struct sockaddr_in *)&cm_event.local_addr; - - if (nesqp->destroyed) - return; - atomic_inc(&cm_connecteds); - nes_debug(NES_DBG_CM, "QP%u attempting to connect to 0x%08X:0x%04X on" - " local port 0x%04X. jiffies = %lu.\n", - nesqp->hwqp.qp_id, ntohl(raddr->sin_addr.s_addr), - ntohs(raddr->sin_port), ntohs(laddr->sin_port), jiffies); - - nes_cm_init_tsa_conn(nesqp, cm_node); - - /* set the QP tsa context */ - nesqp->nesqp_context->tcpPorts[0] = - cpu_to_le16(cm_node->loc_port); - nesqp->nesqp_context->tcpPorts[1] = - cpu_to_le16(cm_node->rem_port); - nesqp->nesqp_context->ip0 = cpu_to_le32(cm_node->rem_addr); - - nesqp->nesqp_context->misc2 |= cpu_to_le32( - (u32)PCI_FUNC(nesdev->pcidev->devfn) << - NES_QPCONTEXT_MISC2_SRC_IP_SHIFT); - nesqp->nesqp_context->arp_index_vlan |= cpu_to_le32( - nes_arp_table(nesdev, - le32_to_cpu(nesqp->nesqp_context->ip0), - NULL, NES_ARP_RESOLVE) << 16); - nesqp->nesqp_context->ts_val_delta = cpu_to_le32( - jiffies - nes_read_indexed(nesdev, NES_IDX_TCP_NOW)); - nesqp->nesqp_context->ird_index = cpu_to_le32(nesqp->hwqp.qp_id); - nesqp->nesqp_context->ird_ord_sizes |= - cpu_to_le32((u32)1 << - NES_QPCONTEXT_ORDIRD_IWARP_MODE_SHIFT); - nesqp->nesqp_context->ird_ord_sizes |= - cpu_to_le32((u32)cm_node->ord_size); - - /* Adjust tail for not having a LSMM */ - /*nesqp->hwqp.sq_tail = 1;*/ - - build_rdma0_msg(cm_node, &nesqp); - - nes_write32(nesdev->regs + NES_WQE_ALLOC, - (1 << 24) | 0x00800000 | nesqp->hwqp.qp_id); - - memset(&nes_quad, 0, sizeof(nes_quad)); - - nes_quad.DstIpAdrIndex = - cpu_to_le32((u32)PCI_FUNC(nesdev->pcidev->devfn) << 24); - nes_quad.SrcIpadr = htonl(cm_node->rem_addr); - nes_quad.TcpPorts[0] = htons(cm_node->rem_port); - nes_quad.TcpPorts[1] = htons(cm_node->loc_port); - - /* Produce hash key */ - crc_value = get_crc_value(&nes_quad); - nesqp->hte_index = cpu_to_be32(crc_value ^ 0xffffffff); - nes_debug(NES_DBG_CM, "HTE Index = 0x%08X, After CRC = 0x%08X\n", - nesqp->hte_index, nesqp->hte_index & nesadapter->hte_index_mask); - - nesqp->hte_index &= nesadapter->hte_index_mask; - nesqp->nesqp_context->hte_index = cpu_to_le32(nesqp->hte_index); - - nesqp->ietf_frame = &cm_node->mpa_frame; - nesqp->private_data_len = (u8)cm_node->mpa_frame_size; - cm_node->cm_core->api->accelerated(cm_node->cm_core, cm_node); - - /* notify OF layer we successfully created the requested connection */ - cm_event.event = IW_CM_EVENT_CONNECT_REPLY; - cm_event.status = 0; - cm_event.provider_data = cm_id->provider_data; - cm_event_laddr->sin_family = AF_INET; - cm_event_laddr->sin_port = laddr->sin_port; - cm_event.remote_addr = cm_id->m_remote_addr; - - cm_event.private_data = (void *)event->cm_node->mpa_frame_buf; - cm_event.private_data_len = (u8)event->cm_node->mpa_frame_size; - cm_event.ird = cm_node->ird_size; - cm_event.ord = cm_node->ord_size; - - cm_event_laddr->sin_addr.s_addr = htonl(event->cm_info.loc_addr); - ret = cm_id->event_handler(cm_id, &cm_event); - nes_debug(NES_DBG_CM, "OFA CM event_handler returned, ret=%d\n", ret); - - if (ret) - printk(KERN_ERR "%s[%u] OFA CM event_handler returned, " - "ret=%d\n", __func__, __LINE__, ret); - attr.qp_state = IB_QPS_RTS; - nes_modify_qp(&nesqp->ibqp, &attr, IB_QP_STATE, NULL); - - nes_debug(NES_DBG_CM, "Exiting connect thread for QP%u. jiffies = " - "%lu\n", nesqp->hwqp.qp_id, jiffies); - - return; -} - - -/** - * cm_event_connect_error - */ -static void cm_event_connect_error(struct nes_cm_event *event) -{ - struct nes_qp *nesqp; - struct iw_cm_id *cm_id; - struct iw_cm_event cm_event; - /* struct nes_cm_info cm_info; */ - int ret; - - if (!event->cm_node) - return; - - cm_id = event->cm_node->cm_id; - if (!cm_id) - return; - - nes_debug(NES_DBG_CM, "cm_node=%p, cm_id=%p\n", event->cm_node, cm_id); - nesqp = cm_id->provider_data; - - if (!nesqp) - return; - - /* notify OF layer about this connection error event */ - /* cm_id->rem_ref(cm_id); */ - nesqp->cm_id = NULL; - cm_id->provider_data = NULL; - cm_event.event = IW_CM_EVENT_CONNECT_REPLY; - cm_event.status = -ECONNRESET; - cm_event.provider_data = cm_id->provider_data; - cm_event.local_addr = cm_id->m_local_addr; - cm_event.remote_addr = cm_id->m_remote_addr; - cm_event.private_data = NULL; - cm_event.private_data_len = 0; - -#ifdef CONFIG_INFINIBAND_NES_DEBUG - { - struct sockaddr_in *cm_event_laddr = (struct sockaddr_in *) - &cm_event.local_addr; - struct sockaddr_in *cm_event_raddr = (struct sockaddr_in *) - &cm_event.remote_addr; - nes_debug(NES_DBG_CM, "call CM_EVENT REJECTED, local_addr=%08x, remote_addr=%08x\n", - cm_event_laddr->sin_addr.s_addr, cm_event_raddr->sin_addr.s_addr); - } -#endif - - ret = cm_id->event_handler(cm_id, &cm_event); - nes_debug(NES_DBG_CM, "OFA CM event_handler returned, ret=%d\n", ret); - if (ret) - printk(KERN_ERR "%s[%u] OFA CM event_handler returned, " - "ret=%d\n", __func__, __LINE__, ret); - cm_id->rem_ref(cm_id); - - rem_ref_cm_node(event->cm_node->cm_core, event->cm_node); - return; -} - - -/** - * cm_event_reset - */ -static void cm_event_reset(struct nes_cm_event *event) -{ - struct nes_qp *nesqp; - struct iw_cm_id *cm_id; - struct iw_cm_event cm_event; - /* struct nes_cm_info cm_info; */ - int ret; - - if (!event->cm_node) - return; - - if (!event->cm_node->cm_id) - return; - - cm_id = event->cm_node->cm_id; - - nes_debug(NES_DBG_CM, "%p - cm_id = %p\n", event->cm_node, cm_id); - nesqp = cm_id->provider_data; - if (!nesqp) - return; - - nesqp->cm_id = NULL; - /* cm_id->provider_data = NULL; */ - cm_event.event = IW_CM_EVENT_DISCONNECT; - cm_event.status = -ECONNRESET; - cm_event.provider_data = cm_id->provider_data; - cm_event.local_addr = cm_id->m_local_addr; - cm_event.remote_addr = cm_id->m_remote_addr; - cm_event.private_data = NULL; - cm_event.private_data_len = 0; - - cm_id->add_ref(cm_id); - ret = cm_id->event_handler(cm_id, &cm_event); - atomic_inc(&cm_closes); - cm_event.event = IW_CM_EVENT_CLOSE; - cm_event.status = 0; - cm_event.provider_data = cm_id->provider_data; - cm_event.local_addr = cm_id->m_local_addr; - cm_event.remote_addr = cm_id->m_remote_addr; - cm_event.private_data = NULL; - cm_event.private_data_len = 0; - nes_debug(NES_DBG_CM, "NODE %p Generating CLOSE\n", event->cm_node); - ret = cm_id->event_handler(cm_id, &cm_event); - - nes_debug(NES_DBG_CM, "OFA CM event_handler returned, ret=%d\n", ret); - - - /* notify OF layer about this connection error event */ - cm_id->rem_ref(cm_id); - - return; -} - - -/** - * cm_event_mpa_req - */ -static void cm_event_mpa_req(struct nes_cm_event *event) -{ - struct iw_cm_id *cm_id; - struct iw_cm_event cm_event; - int ret; - struct nes_cm_node *cm_node; - struct sockaddr_in *cm_event_laddr = (struct sockaddr_in *) - &cm_event.local_addr; - struct sockaddr_in *cm_event_raddr = (struct sockaddr_in *) - &cm_event.remote_addr; - - cm_node = event->cm_node; - if (!cm_node) - return; - cm_id = cm_node->cm_id; - - atomic_inc(&cm_connect_reqs); - nes_debug(NES_DBG_CM, "cm_node = %p - cm_id = %p, jiffies = %lu\n", - cm_node, cm_id, jiffies); - - cm_event.event = IW_CM_EVENT_CONNECT_REQUEST; - cm_event.status = 0; - cm_event.provider_data = (void *)cm_node; - - cm_event_laddr->sin_family = AF_INET; - cm_event_laddr->sin_port = htons(event->cm_info.loc_port); - cm_event_laddr->sin_addr.s_addr = htonl(event->cm_info.loc_addr); - - cm_event_raddr->sin_family = AF_INET; - cm_event_raddr->sin_port = htons(event->cm_info.rem_port); - cm_event_raddr->sin_addr.s_addr = htonl(event->cm_info.rem_addr); - cm_event.private_data = cm_node->mpa_frame_buf; - cm_event.private_data_len = (u8)cm_node->mpa_frame_size; - if (cm_node->mpa_frame_rev == IETF_MPA_V1) { - cm_event.ird = NES_MAX_IRD; - cm_event.ord = NES_MAX_ORD; - } else { - cm_event.ird = cm_node->ird_size; - cm_event.ord = cm_node->ord_size; - } - - ret = cm_id->event_handler(cm_id, &cm_event); - if (ret) - printk(KERN_ERR "%s[%u] OFA CM event_handler returned, ret=%d\n", - __func__, __LINE__, ret); - return; -} - - -static void cm_event_mpa_reject(struct nes_cm_event *event) -{ - struct iw_cm_id *cm_id; - struct iw_cm_event cm_event; - struct nes_cm_node *cm_node; - int ret; - struct sockaddr_in *cm_event_laddr = (struct sockaddr_in *) - &cm_event.local_addr; - struct sockaddr_in *cm_event_raddr = (struct sockaddr_in *) - &cm_event.remote_addr; - - cm_node = event->cm_node; - if (!cm_node) - return; - cm_id = cm_node->cm_id; - - atomic_inc(&cm_connect_reqs); - nes_debug(NES_DBG_CM, "cm_node = %p - cm_id = %p, jiffies = %lu\n", - cm_node, cm_id, jiffies); - - cm_event.event = IW_CM_EVENT_CONNECT_REPLY; - cm_event.status = -ECONNREFUSED; - cm_event.provider_data = cm_id->provider_data; - - cm_event_laddr->sin_family = AF_INET; - cm_event_laddr->sin_port = htons(event->cm_info.loc_port); - cm_event_laddr->sin_addr.s_addr = htonl(event->cm_info.loc_addr); - - cm_event_raddr->sin_family = AF_INET; - cm_event_raddr->sin_port = htons(event->cm_info.rem_port); - cm_event_raddr->sin_addr.s_addr = htonl(event->cm_info.rem_addr); - - cm_event.private_data = cm_node->mpa_frame_buf; - cm_event.private_data_len = (u8)cm_node->mpa_frame_size; - - nes_debug(NES_DBG_CM, "call CM_EVENT_MPA_REJECTED, local_addr=%08x, " - "remove_addr=%08x\n", - cm_event_laddr->sin_addr.s_addr, - cm_event_raddr->sin_addr.s_addr); - - ret = cm_id->event_handler(cm_id, &cm_event); - if (ret) - printk(KERN_ERR "%s[%u] OFA CM event_handler returned, ret=%d\n", - __func__, __LINE__, ret); - - return; -} - - -static void nes_cm_event_handler(struct work_struct *); - -/** - * nes_cm_post_event - * post an event to the cm event handler - */ -static int nes_cm_post_event(struct nes_cm_event *event) -{ - atomic_inc(&event->cm_node->cm_core->events_posted); - add_ref_cm_node(event->cm_node); - event->cm_info.cm_id->add_ref(event->cm_info.cm_id); - INIT_WORK(&event->event_work, nes_cm_event_handler); - nes_debug(NES_DBG_CM, "cm_node=%p queue_work, event=%p\n", - event->cm_node, event); - - queue_work(event->cm_node->cm_core->event_wq, &event->event_work); - - nes_debug(NES_DBG_CM, "Exit\n"); - return 0; -} - - -/** - * nes_cm_event_handler - * worker function to handle cm events - * will free instance of nes_cm_event - */ -static void nes_cm_event_handler(struct work_struct *work) -{ - struct nes_cm_event *event = container_of(work, struct nes_cm_event, - event_work); - struct nes_cm_core *cm_core; - - if ((!event) || (!event->cm_node) || (!event->cm_node->cm_core)) - return; - - cm_core = event->cm_node->cm_core; - nes_debug(NES_DBG_CM, "event=%p, event->type=%u, events posted=%u\n", - event, event->type, atomic_read(&cm_core->events_posted)); - - switch (event->type) { - case NES_CM_EVENT_MPA_REQ: - cm_event_mpa_req(event); - nes_debug(NES_DBG_CM, "cm_node=%p CM Event: MPA REQUEST\n", - event->cm_node); - break; - case NES_CM_EVENT_RESET: - nes_debug(NES_DBG_CM, "cm_node = %p CM Event: RESET\n", - event->cm_node); - cm_event_reset(event); - break; - case NES_CM_EVENT_CONNECTED: - if ((!event->cm_node->cm_id) || - (event->cm_node->state != NES_CM_STATE_TSA)) - break; - cm_event_connected(event); - nes_debug(NES_DBG_CM, "CM Event: CONNECTED\n"); - break; - case NES_CM_EVENT_MPA_REJECT: - if ((!event->cm_node->cm_id) || - (event->cm_node->state == NES_CM_STATE_TSA)) - break; - cm_event_mpa_reject(event); - nes_debug(NES_DBG_CM, "CM Event: REJECT\n"); - break; - - case NES_CM_EVENT_ABORTED: - if ((!event->cm_node->cm_id) || - (event->cm_node->state == NES_CM_STATE_TSA)) - break; - cm_event_connect_error(event); - nes_debug(NES_DBG_CM, "CM Event: ABORTED\n"); - break; - case NES_CM_EVENT_DROPPED_PKT: - nes_debug(NES_DBG_CM, "CM Event: DROPPED PKT\n"); - break; - default: - nes_debug(NES_DBG_CM, "CM Event: UNKNOWN EVENT TYPE\n"); - break; - } - - atomic_dec(&cm_core->events_posted); - event->cm_info.cm_id->rem_ref(event->cm_info.cm_id); - rem_ref_cm_node(cm_core, event->cm_node); - kfree(event); - - return; -} diff --git a/drivers/infiniband/hw/nes/nes_cm.h b/drivers/infiniband/hw/nes/nes_cm.h deleted file mode 100644 index b9cc02b4e8d5..000000000000 --- a/drivers/infiniband/hw/nes/nes_cm.h +++ /dev/null @@ -1,470 +0,0 @@ -/* - * Copyright (c) 2006 - 2014 Intel Corporation. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - */ - -#ifndef NES_CM_H -#define NES_CM_H - -#define QUEUE_EVENTS - -#define NES_MANAGE_APBVT_DEL 0 -#define NES_MANAGE_APBVT_ADD 1 - -#define NES_MPA_REQUEST_ACCEPT 1 -#define NES_MPA_REQUEST_REJECT 2 - -/* IETF MPA -- defines, enums, structs */ -#define IEFT_MPA_KEY_REQ "MPA ID Req Frame" -#define IEFT_MPA_KEY_REP "MPA ID Rep Frame" -#define IETF_MPA_KEY_SIZE 16 -#define IETF_MPA_VERSION 1 -#define IETF_MAX_PRIV_DATA_LEN 512 -#define IETF_MPA_FRAME_SIZE 20 -#define IETF_RTR_MSG_SIZE 4 -#define IETF_MPA_V2_FLAG 0x10 - -/* IETF RTR MSG Fields */ -#define IETF_PEER_TO_PEER 0x8000 -#define IETF_FLPDU_ZERO_LEN 0x4000 -#define IETF_RDMA0_WRITE 0x8000 -#define IETF_RDMA0_READ 0x4000 -#define IETF_NO_IRD_ORD 0x3FFF -#define NES_MAX_IRD 0x40 -#define NES_MAX_ORD 0x7F - -enum ietf_mpa_flags { - IETF_MPA_FLAGS_MARKERS = 0x80, /* receive Markers */ - IETF_MPA_FLAGS_CRC = 0x40, /* receive Markers */ - IETF_MPA_FLAGS_REJECT = 0x20, /* Reject */ -}; - -struct ietf_mpa_v1 { - u8 key[IETF_MPA_KEY_SIZE]; - u8 flags; - u8 rev; - __be16 priv_data_len; - u8 priv_data[0]; -}; - -#define ietf_mpa_req_resp_frame ietf_mpa_frame - -struct ietf_rtr_msg { - __be16 ctrl_ird; - __be16 ctrl_ord; -}; - -struct ietf_mpa_v2 { - u8 key[IETF_MPA_KEY_SIZE]; - u8 flags; - u8 rev; - __be16 priv_data_len; - struct ietf_rtr_msg rtr_msg; - u8 priv_data[0]; -}; - -struct nes_v4_quad { - u32 rsvd0; - __le32 DstIpAdrIndex; /* Only most significant 5 bits are valid */ - __be32 SrcIpadr; - __be16 TcpPorts[2]; /* src is low, dest is high */ -}; - -struct nes_cm_node; -enum nes_timer_type { - NES_TIMER_TYPE_SEND, - NES_TIMER_TYPE_RECV, - NES_TIMER_NODE_CLEANUP, - NES_TIMER_TYPE_CLOSE, -}; - -#define NES_PASSIVE_STATE_INDICATED 0 -#define NES_DO_NOT_SEND_RESET_EVENT 1 -#define NES_SEND_RESET_EVENT 2 - -#define MAX_NES_IFS 4 - -#define SET_ACK 1 -#define SET_SYN 2 -#define SET_FIN 4 -#define SET_RST 8 - -#define TCP_OPTIONS_PADDING 3 - -struct option_base { - u8 optionnum; - u8 length; -}; - -enum option_numbers { - OPTION_NUMBER_END, - OPTION_NUMBER_NONE, - OPTION_NUMBER_MSS, - OPTION_NUMBER_WINDOW_SCALE, - OPTION_NUMBER_SACK_PERM, - OPTION_NUMBER_SACK, - OPTION_NUMBER_WRITE0 = 0xbc -}; - -struct option_mss { - u8 optionnum; - u8 length; - __be16 mss; -}; - -struct option_windowscale { - u8 optionnum; - u8 length; - u8 shiftcount; -}; - -union all_known_options { - char as_end; - struct option_base as_base; - struct option_mss as_mss; - struct option_windowscale as_windowscale; -}; - -struct nes_timer_entry { - struct list_head list; - unsigned long timetosend; /* jiffies */ - struct sk_buff *skb; - u32 type; - u32 retrycount; - u32 retranscount; - u32 context; - u32 seq_num; - u32 send_retrans; - int close_when_complete; - struct net_device *netdev; -}; - -#define NES_DEFAULT_RETRYS 64 -#define NES_DEFAULT_RETRANS 8 -#ifdef CONFIG_INFINIBAND_NES_DEBUG -#define NES_RETRY_TIMEOUT (1000*HZ/1000) -#else -#define NES_RETRY_TIMEOUT (3000*HZ/1000) -#endif -#define NES_SHORT_TIME (10) -#define NES_LONG_TIME (2000*HZ/1000) -#define NES_MAX_TIMEOUT ((unsigned long) (12*HZ)) - -#define NES_CM_HASHTABLE_SIZE 1024 -#define NES_CM_TCP_TIMER_INTERVAL 3000 -#define NES_CM_DEFAULT_MTU 1540 -#define NES_CM_DEFAULT_FRAME_CNT 10 -#define NES_CM_THREAD_STACK_SIZE 256 -#define NES_CM_DEFAULT_RCV_WND 64240 // before we know that window scaling is allowed -#define NES_CM_DEFAULT_RCV_WND_SCALED 256960 // after we know that window scaling is allowed -#define NES_CM_DEFAULT_RCV_WND_SCALE 2 -#define NES_CM_DEFAULT_FREE_PKTS 0x000A -#define NES_CM_FREE_PKT_LO_WATERMARK 2 - -#define NES_CM_DEFAULT_MSS 536 - -#define NES_CM_DEF_SEQ 0x159bf75f -#define NES_CM_DEF_LOCAL_ID 0x3b47 - -#define NES_CM_DEF_SEQ2 0x18ed5740 -#define NES_CM_DEF_LOCAL_ID2 0xb807 -#define MAX_CM_BUFFER (IETF_MPA_FRAME_SIZE + IETF_RTR_MSG_SIZE + IETF_MAX_PRIV_DATA_LEN) - -typedef u32 nes_addr_t; - -#define nes_cm_tsa_context nes_qp_context - -struct nes_qp; - -/* cm node transition states */ -enum nes_cm_node_state { - NES_CM_STATE_UNKNOWN, - NES_CM_STATE_INITED, - NES_CM_STATE_LISTENING, - NES_CM_STATE_SYN_RCVD, - NES_CM_STATE_SYN_SENT, - NES_CM_STATE_ONE_SIDE_ESTABLISHED, - NES_CM_STATE_ESTABLISHED, - NES_CM_STATE_ACCEPTING, - NES_CM_STATE_MPAREQ_SENT, - NES_CM_STATE_MPAREQ_RCVD, - NES_CM_STATE_MPAREJ_RCVD, - NES_CM_STATE_TSA, - NES_CM_STATE_FIN_WAIT1, - NES_CM_STATE_FIN_WAIT2, - NES_CM_STATE_CLOSE_WAIT, - NES_CM_STATE_TIME_WAIT, - NES_CM_STATE_LAST_ACK, - NES_CM_STATE_CLOSING, - NES_CM_STATE_LISTENER_DESTROYED, - NES_CM_STATE_CLOSED -}; - -enum mpa_frame_version { - IETF_MPA_V1 = 1, - IETF_MPA_V2 = 2 -}; - -enum mpa_frame_key { - MPA_KEY_REQUEST, - MPA_KEY_REPLY -}; - -enum send_rdma0 { - SEND_RDMA_READ_ZERO = 1, - SEND_RDMA_WRITE_ZERO = 2 -}; - -enum nes_tcpip_pkt_type { - NES_PKT_TYPE_UNKNOWN, - NES_PKT_TYPE_SYN, - NES_PKT_TYPE_SYNACK, - NES_PKT_TYPE_ACK, - NES_PKT_TYPE_FIN, - NES_PKT_TYPE_RST -}; - - -/* type of nes connection */ -enum nes_cm_conn_type { - NES_CM_IWARP_CONN_TYPE, -}; - -/* CM context params */ -struct nes_cm_tcp_context { - u8 client; - - u32 loc_seq_num; - u32 loc_ack_num; - u32 rem_ack_num; - u32 rcv_nxt; - - u32 loc_id; - u32 rem_id; - - u32 snd_wnd; - u32 max_snd_wnd; - - u32 rcv_wnd; - u32 mss; - u8 snd_wscale; - u8 rcv_wscale; - - struct nes_cm_tsa_context tsa_cntxt; -}; - - -enum nes_cm_listener_state { - NES_CM_LISTENER_PASSIVE_STATE = 1, - NES_CM_LISTENER_ACTIVE_STATE = 2, - NES_CM_LISTENER_EITHER_STATE = 3 -}; - -struct nes_cm_listener { - struct list_head list; - struct nes_cm_core *cm_core; - u8 loc_mac[ETH_ALEN]; - nes_addr_t loc_addr; - u16 loc_port; - struct iw_cm_id *cm_id; - enum nes_cm_conn_type conn_type; - atomic_t ref_count; - struct nes_vnic *nesvnic; - atomic_t pend_accepts_cnt; - int backlog; - enum nes_cm_listener_state listener_state; - u32 reused_node; - u8 tos; -}; - -/* per connection node and node state information */ -struct nes_cm_node { - nes_addr_t loc_addr, rem_addr; - u16 loc_port, rem_port; - - u8 loc_mac[ETH_ALEN]; - u8 rem_mac[ETH_ALEN]; - - enum nes_cm_node_state state; - struct nes_cm_tcp_context tcp_cntxt; - struct nes_cm_core *cm_core; - struct sk_buff_head resend_list; - atomic_t ref_count; - struct net_device *netdev; - - struct nes_cm_node *loopbackpartner; - - struct nes_timer_entry *send_entry; - struct nes_timer_entry *recv_entry; - spinlock_t retrans_list_lock; - enum send_rdma0 send_rdma0_op; - - union { - struct ietf_mpa_v1 mpa_frame; - struct ietf_mpa_v2 mpa_v2_frame; - u8 mpa_frame_buf[MAX_CM_BUFFER]; - }; - enum mpa_frame_version mpa_frame_rev; - u16 ird_size; - u16 ord_size; - u16 mpav2_ird_ord; - - u16 mpa_frame_size; - struct iw_cm_id *cm_id; - struct list_head list; - bool accelerated; - struct nes_cm_listener *listener; - enum nes_cm_conn_type conn_type; - struct nes_vnic *nesvnic; - int apbvt_set; - int accept_pend; - struct list_head timer_entry; - struct list_head reset_entry; - struct nes_qp *nesqp; - atomic_t passive_state; - u8 tos; -}; - -/* structure for client or CM to fill when making CM api calls. */ -/* - only need to set relevant data, based on op. */ -struct nes_cm_info { - union { - struct iw_cm_id *cm_id; - struct net_device *netdev; - }; - - u16 loc_port; - u16 rem_port; - nes_addr_t loc_addr; - nes_addr_t rem_addr; - enum nes_cm_conn_type conn_type; - int backlog; -}; - -/* CM event codes */ -enum nes_cm_event_type { - NES_CM_EVENT_UNKNOWN, - NES_CM_EVENT_ESTABLISHED, - NES_CM_EVENT_MPA_REQ, - NES_CM_EVENT_MPA_CONNECT, - NES_CM_EVENT_MPA_ACCEPT, - NES_CM_EVENT_MPA_REJECT, - NES_CM_EVENT_MPA_ESTABLISHED, - NES_CM_EVENT_CONNECTED, - NES_CM_EVENT_CLOSED, - NES_CM_EVENT_RESET, - NES_CM_EVENT_DROPPED_PKT, - NES_CM_EVENT_CLOSE_IMMED, - NES_CM_EVENT_CLOSE_HARD, - NES_CM_EVENT_CLOSE_CLEAN, - NES_CM_EVENT_ABORTED, - NES_CM_EVENT_SEND_FIRST -}; - -/* event to post to CM event handler */ -struct nes_cm_event { - enum nes_cm_event_type type; - - struct nes_cm_info cm_info; - struct work_struct event_work; - struct nes_cm_node *cm_node; -}; - -struct nes_cm_core { - enum nes_cm_node_state state; - - atomic_t listen_node_cnt; - struct nes_cm_node listen_list; - spinlock_t listen_list_lock; - - u32 mtu; - u32 free_tx_pkt_max; - u32 rx_pkt_posted; - atomic_t ht_node_cnt; - struct list_head connected_nodes; - /* struct list_head hashtable[NES_CM_HASHTABLE_SIZE]; */ - spinlock_t ht_lock; - - struct timer_list tcp_timer; - - const struct nes_cm_ops *api; - - int (*post_event)(struct nes_cm_event *event); - atomic_t events_posted; - struct workqueue_struct *event_wq; - struct workqueue_struct *disconn_wq; - - atomic_t node_cnt; - u64 aborted_connects; - u32 options; - - struct nes_cm_node *current_listen_node; -}; - - -#define NES_CM_SET_PKT_SIZE (1 << 1) -#define NES_CM_SET_FREE_PKT_Q_SIZE (1 << 2) - -/* CM ops/API for client interface */ -struct nes_cm_ops { - int (*accelerated)(struct nes_cm_core *, struct nes_cm_node *); - struct nes_cm_listener * (*listen)(struct nes_cm_core *, struct nes_vnic *, - struct nes_cm_info *); - int (*stop_listener)(struct nes_cm_core *, struct nes_cm_listener *); - struct nes_cm_node * (*connect)(struct nes_cm_core *, - struct nes_vnic *, u16, void *, - struct nes_cm_info *); - int (*close)(struct nes_cm_core *, struct nes_cm_node *); - int (*accept)(struct nes_cm_core *, struct nes_cm_node *); - int (*reject)(struct nes_cm_core *, struct nes_cm_node *); - int (*recv_pkt)(struct nes_cm_core *, struct nes_vnic *, - struct sk_buff *); - int (*destroy_cm_core)(struct nes_cm_core *); - int (*get)(struct nes_cm_core *); - int (*set)(struct nes_cm_core *, u32, u32); -}; - -int schedule_nes_timer(struct nes_cm_node *, struct sk_buff *, - enum nes_timer_type, int, int); - -int nes_accept(struct iw_cm_id *, struct iw_cm_conn_param *); -int nes_reject(struct iw_cm_id *, const void *, u8); -int nes_connect(struct iw_cm_id *, struct iw_cm_conn_param *); -int nes_create_listen(struct iw_cm_id *, int); -int nes_destroy_listen(struct iw_cm_id *); - -int nes_cm_recv(struct sk_buff *, struct net_device *); -int nes_cm_start(void); -int nes_cm_stop(void); -int nes_add_ref_cm_node(struct nes_cm_node *cm_node); -int nes_rem_ref_cm_node(struct nes_cm_node *cm_node); - -#endif /* NES_CM_H */ diff --git a/drivers/infiniband/hw/nes/nes_context.h b/drivers/infiniband/hw/nes/nes_context.h deleted file mode 100644 index a69eef16d72d..000000000000 --- a/drivers/infiniband/hw/nes/nes_context.h +++ /dev/null @@ -1,193 +0,0 @@ -/* - * Copyright (c) 2006 - 2011 Intel Corporation. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -#ifndef NES_CONTEXT_H -#define NES_CONTEXT_H - -struct nes_qp_context { - __le32 misc; - __le32 cqs; - __le32 sq_addr_low; - __le32 sq_addr_high; - __le32 rq_addr_low; - __le32 rq_addr_high; - __le32 misc2; - __le16 tcpPorts[2]; - __le32 ip0; - __le32 ip1; - __le32 ip2; - __le32 ip3; - __le32 mss; - __le32 arp_index_vlan; - __le32 tcp_state_flow_label; - __le32 pd_index_wscale; - __le32 keepalive; - u32 ts_recent; - u32 ts_age; - __le32 snd_nxt; - __le32 snd_wnd; - __le32 rcv_nxt; - __le32 rcv_wnd; - __le32 snd_max; - __le32 snd_una; - u32 srtt; - __le32 rttvar; - __le32 ssthresh; - __le32 cwnd; - __le32 snd_wl1; - __le32 snd_wl2; - __le32 max_snd_wnd; - __le32 ts_val_delta; - u32 retransmit; - u32 probe_cnt; - u32 hte_index; - __le32 q2_addr_low; - __le32 q2_addr_high; - __le32 ird_index; - u32 Rsvd3; - __le32 ird_ord_sizes; - u32 mrkr_offset; - __le32 aeq_token_low; - __le32 aeq_token_high; -}; - -/* QP Context Misc Field */ - -#define NES_QPCONTEXT_MISC_IWARP_VER_MASK 0x00000003 -#define NES_QPCONTEXT_MISC_IWARP_VER_SHIFT 0 -#define NES_QPCONTEXT_MISC_EFB_SIZE_MASK 0x000000C0 -#define NES_QPCONTEXT_MISC_EFB_SIZE_SHIFT 6 -#define NES_QPCONTEXT_MISC_RQ_SIZE_MASK 0x00000300 -#define NES_QPCONTEXT_MISC_RQ_SIZE_SHIFT 8 -#define NES_QPCONTEXT_MISC_SQ_SIZE_MASK 0x00000c00 -#define NES_QPCONTEXT_MISC_SQ_SIZE_SHIFT 10 -#define NES_QPCONTEXT_MISC_PCI_FCN_MASK 0x00007000 -#define NES_QPCONTEXT_MISC_PCI_FCN_SHIFT 12 -#define NES_QPCONTEXT_MISC_DUP_ACKS_MASK 0x00070000 -#define NES_QPCONTEXT_MISC_DUP_ACKS_SHIFT 16 - -enum nes_qp_context_misc_bits { - NES_QPCONTEXT_MISC_RX_WQE_SIZE = 0x00000004, - NES_QPCONTEXT_MISC_IPV4 = 0x00000008, - NES_QPCONTEXT_MISC_DO_NOT_FRAG = 0x00000010, - NES_QPCONTEXT_MISC_INSERT_VLAN = 0x00000020, - NES_QPCONTEXT_MISC_DROS = 0x00008000, - NES_QPCONTEXT_MISC_WSCALE = 0x00080000, - NES_QPCONTEXT_MISC_KEEPALIVE = 0x00100000, - NES_QPCONTEXT_MISC_TIMESTAMP = 0x00200000, - NES_QPCONTEXT_MISC_SACK = 0x00400000, - NES_QPCONTEXT_MISC_RDMA_WRITE_EN = 0x00800000, - NES_QPCONTEXT_MISC_RDMA_READ_EN = 0x01000000, - NES_QPCONTEXT_MISC_WBIND_EN = 0x10000000, - NES_QPCONTEXT_MISC_FAST_REGISTER_EN = 0x20000000, - NES_QPCONTEXT_MISC_PRIV_EN = 0x40000000, - NES_QPCONTEXT_MISC_NO_NAGLE = 0x80000000 -}; - -enum nes_qp_acc_wq_sizes { - HCONTEXT_TSA_WQ_SIZE_4 = 0, - HCONTEXT_TSA_WQ_SIZE_32 = 1, - HCONTEXT_TSA_WQ_SIZE_128 = 2, - HCONTEXT_TSA_WQ_SIZE_512 = 3 -}; - -/* QP Context Misc2 Fields */ -#define NES_QPCONTEXT_MISC2_TTL_MASK 0x000000ff -#define NES_QPCONTEXT_MISC2_TTL_SHIFT 0 -#define NES_QPCONTEXT_MISC2_HOP_LIMIT_MASK 0x000000ff -#define NES_QPCONTEXT_MISC2_HOP_LIMIT_SHIFT 0 -#define NES_QPCONTEXT_MISC2_LIMIT_MASK 0x00000300 -#define NES_QPCONTEXT_MISC2_LIMIT_SHIFT 8 -#define NES_QPCONTEXT_MISC2_NIC_INDEX_MASK 0x0000fc00 -#define NES_QPCONTEXT_MISC2_NIC_INDEX_SHIFT 10 -#define NES_QPCONTEXT_MISC2_SRC_IP_MASK 0x001f0000 -#define NES_QPCONTEXT_MISC2_SRC_IP_SHIFT 16 -#define NES_QPCONTEXT_MISC2_TOS_MASK 0xff000000 -#define NES_QPCONTEXT_MISC2_TOS_SHIFT 24 -#define NES_QPCONTEXT_MISC2_TRAFFIC_CLASS_MASK 0xff000000 -#define NES_QPCONTEXT_MISC2_TRAFFIC_CLASS_SHIFT 24 - -/* QP Context Tcp State/Flow Label Fields */ -#define NES_QPCONTEXT_TCPFLOW_FLOW_LABEL_MASK 0x000fffff -#define NES_QPCONTEXT_TCPFLOW_FLOW_LABEL_SHIFT 0 -#define NES_QPCONTEXT_TCPFLOW_TCP_STATE_MASK 0xf0000000 -#define NES_QPCONTEXT_TCPFLOW_TCP_STATE_SHIFT 28 - -enum nes_qp_tcp_state { - NES_QPCONTEXT_TCPSTATE_CLOSED = 1, - NES_QPCONTEXT_TCPSTATE_EST = 5, - NES_QPCONTEXT_TCPSTATE_TIME_WAIT = 11, -}; - -/* QP Context PD Index/wscale Fields */ -#define NES_QPCONTEXT_PDWSCALE_RCV_WSCALE_MASK 0x0000000f -#define NES_QPCONTEXT_PDWSCALE_RCV_WSCALE_SHIFT 0 -#define NES_QPCONTEXT_PDWSCALE_SND_WSCALE_MASK 0x00000f00 -#define NES_QPCONTEXT_PDWSCALE_SND_WSCALE_SHIFT 8 -#define NES_QPCONTEXT_PDWSCALE_PDINDEX_MASK 0xffff0000 -#define NES_QPCONTEXT_PDWSCALE_PDINDEX_SHIFT 16 - -/* QP Context Keepalive Fields */ -#define NES_QPCONTEXT_KEEPALIVE_DELTA_MASK 0x0000ffff -#define NES_QPCONTEXT_KEEPALIVE_DELTA_SHIFT 0 -#define NES_QPCONTEXT_KEEPALIVE_PROBE_CNT_MASK 0x00ff0000 -#define NES_QPCONTEXT_KEEPALIVE_PROBE_CNT_SHIFT 16 -#define NES_QPCONTEXT_KEEPALIVE_INTV_MASK 0xff000000 -#define NES_QPCONTEXT_KEEPALIVE_INTV_SHIFT 24 - -/* QP Context ORD/IRD Fields */ -#define NES_QPCONTEXT_ORDIRD_ORDSIZE_MASK 0x0000007f -#define NES_QPCONTEXT_ORDIRD_ORDSIZE_SHIFT 0 -#define NES_QPCONTEXT_ORDIRD_IRDSIZE_MASK 0x00030000 -#define NES_QPCONTEXT_ORDIRD_IRDSIZE_SHIFT 16 -#define NES_QPCONTEXT_ORDIRD_IWARP_MODE_MASK 0x30000000 -#define NES_QPCONTEXT_ORDIRD_IWARP_MODE_SHIFT 28 - -enum nes_ord_ird_bits { - NES_QPCONTEXT_ORDIRD_WRPDU = 0x02000000, - NES_QPCONTEXT_ORDIRD_LSMM_PRESENT = 0x04000000, - NES_QPCONTEXT_ORDIRD_ALSMM = 0x08000000, - NES_QPCONTEXT_ORDIRD_AAH = 0x40000000, - NES_QPCONTEXT_ORDIRD_RNMC = 0x80000000 -}; - -enum nes_iwarp_qp_state { - NES_QPCONTEXT_IWARP_STATE_NONEXIST = 0, - NES_QPCONTEXT_IWARP_STATE_IDLE = 1, - NES_QPCONTEXT_IWARP_STATE_RTS = 2, - NES_QPCONTEXT_IWARP_STATE_CLOSING = 3, - NES_QPCONTEXT_IWARP_STATE_TERMINATE = 5, - NES_QPCONTEXT_IWARP_STATE_ERROR = 6 -}; - - -#endif /* NES_CONTEXT_H */ diff --git a/drivers/infiniband/hw/nes/nes_hw.c b/drivers/infiniband/hw/nes/nes_hw.c deleted file mode 100644 index 5517e392bc01..000000000000 --- a/drivers/infiniband/hw/nes/nes_hw.c +++ /dev/null @@ -1,3887 +0,0 @@ -/* - * Copyright (c) 2006 - 2011 Intel Corporation. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - */ - -#include <linux/module.h> -#include <linux/moduleparam.h> -#include <linux/netdevice.h> -#include <linux/etherdevice.h> -#include <linux/if_vlan.h> -#include <linux/slab.h> - -#include "nes.h" - -static int wide_ppm_offset; -module_param(wide_ppm_offset, int, 0644); -MODULE_PARM_DESC(wide_ppm_offset, "Increase CX4 interface clock ppm offset, 0=100ppm (default), 1=300ppm"); - -static u32 crit_err_count; -u32 int_mod_timer_init; -u32 int_mod_cq_depth_256; -u32 int_mod_cq_depth_128; -u32 int_mod_cq_depth_32; -u32 int_mod_cq_depth_24; -u32 int_mod_cq_depth_16; -u32 int_mod_cq_depth_4; -u32 int_mod_cq_depth_1; -static const u8 nes_max_critical_error_count = 100; -#include "nes_cm.h" - -static void nes_cqp_ce_handler(struct nes_device *nesdev, struct nes_hw_cq *cq); -static void nes_init_csr_ne020(struct nes_device *nesdev, u8 hw_rev, u8 port_count); -static int nes_init_serdes(struct nes_device *nesdev, u8 hw_rev, u8 port_count, - struct nes_adapter *nesadapter, u8 OneG_Mode); -static void nes_nic_napi_ce_handler(struct nes_device *nesdev, struct nes_hw_nic_cq *cq); -static void nes_process_aeq(struct nes_device *nesdev, struct nes_hw_aeq *aeq); -static void nes_process_ceq(struct nes_device *nesdev, struct nes_hw_ceq *ceq); -static void nes_process_iwarp_aeqe(struct nes_device *nesdev, - struct nes_hw_aeqe *aeqe); -static void process_critical_error(struct nes_device *nesdev); -static void nes_process_mac_intr(struct nes_device *nesdev, u32 mac_number); -static unsigned int nes_reset_adapter_ne020(struct nes_device *nesdev, u8 *OneG_Mode); -static void nes_terminate_start_timer(struct nes_qp *nesqp); - -static const char *const nes_iwarp_state_str[] = { - "Non-Existent", - "Idle", - "RTS", - "Closing", - "RSVD1", - "Terminate", - "Error", - "RSVD2", -}; - -static const char *const nes_tcp_state_str[] = { - "Non-Existent", - "Closed", - "Listen", - "SYN Sent", - "SYN Rcvd", - "Established", - "Close Wait", - "FIN Wait 1", - "Closing", - "Last Ack", - "FIN Wait 2", - "Time Wait", - "RSVD1", - "RSVD2", - "RSVD3", - "RSVD4", -}; - -static inline void print_ip(struct nes_cm_node *cm_node) -{ - unsigned char *rem_addr; - if (cm_node) { - rem_addr = (unsigned char *)&cm_node->rem_addr; - printk(KERN_ERR PFX "Remote IP addr: %pI4\n", rem_addr); - } -} - -/** - * nes_nic_init_timer_defaults - */ -void nes_nic_init_timer_defaults(struct nes_device *nesdev, u8 jumbomode) -{ - unsigned long flags; - struct nes_adapter *nesadapter = nesdev->nesadapter; - struct nes_hw_tune_timer *shared_timer = &nesadapter->tune_timer; - - spin_lock_irqsave(&nesadapter->periodic_timer_lock, flags); - - shared_timer->timer_in_use_min = NES_NIC_FAST_TIMER_LOW; - shared_timer->timer_in_use_max = NES_NIC_FAST_TIMER_HIGH; - if (jumbomode) { - shared_timer->threshold_low = DEFAULT_JUMBO_NES_QL_LOW; - shared_timer->threshold_target = DEFAULT_JUMBO_NES_QL_TARGET; - shared_timer->threshold_high = DEFAULT_JUMBO_NES_QL_HIGH; - } else { - shared_timer->threshold_low = DEFAULT_NES_QL_LOW; - shared_timer->threshold_target = DEFAULT_NES_QL_TARGET; - shared_timer->threshold_high = DEFAULT_NES_QL_HIGH; - } - - /* todo use netdev->mtu to set thresholds */ - spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags); -} - - -/** - * nes_nic_init_timer - */ -static void nes_nic_init_timer(struct nes_device *nesdev) -{ - unsigned long flags; - struct nes_adapter *nesadapter = nesdev->nesadapter; - struct nes_hw_tune_timer *shared_timer = &nesadapter->tune_timer; - - spin_lock_irqsave(&nesadapter->periodic_timer_lock, flags); - - if (shared_timer->timer_in_use_old == 0) { - nesdev->deepcq_count = 0; - shared_timer->timer_direction_upward = 0; - shared_timer->timer_direction_downward = 0; - shared_timer->timer_in_use = NES_NIC_FAST_TIMER; - shared_timer->timer_in_use_old = 0; - - } - if (shared_timer->timer_in_use != shared_timer->timer_in_use_old) { - shared_timer->timer_in_use_old = shared_timer->timer_in_use; - nes_write32(nesdev->regs+NES_PERIODIC_CONTROL, - 0x80000000 | ((u32)(shared_timer->timer_in_use*8))); - } - /* todo use netdev->mtu to set thresholds */ - spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags); -} - - -/** - * nes_nic_tune_timer - */ -static void nes_nic_tune_timer(struct nes_device *nesdev) -{ - unsigned long flags; - struct nes_adapter *nesadapter = nesdev->nesadapter; - struct nes_hw_tune_timer *shared_timer = &nesadapter->tune_timer; - u16 cq_count = nesdev->currcq_count; - - spin_lock_irqsave(&nesadapter->periodic_timer_lock, flags); - - if (shared_timer->cq_count_old <= cq_count) - shared_timer->cq_direction_downward = 0; - else - shared_timer->cq_direction_downward++; - shared_timer->cq_count_old = cq_count; - if (shared_timer->cq_direction_downward > NES_NIC_CQ_DOWNWARD_TREND) { - if (cq_count <= shared_timer->threshold_low && - shared_timer->threshold_low > 4) { - shared_timer->threshold_low = shared_timer->threshold_low/2; - shared_timer->cq_direction_downward=0; - nesdev->currcq_count = 0; - spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags); - return; - } - } - - if (cq_count > 1) { - nesdev->deepcq_count += cq_count; - if (cq_count <= shared_timer->threshold_low) { /* increase timer gently */ - shared_timer->timer_direction_upward++; - shared_timer->timer_direction_downward = 0; - } else if (cq_count <= shared_timer->threshold_target) { /* balanced */ - shared_timer->timer_direction_upward = 0; - shared_timer->timer_direction_downward = 0; - } else if (cq_count <= shared_timer->threshold_high) { /* decrease timer gently */ - shared_timer->timer_direction_downward++; - shared_timer->timer_direction_upward = 0; - } else if (cq_count <= (shared_timer->threshold_high) * 2) { - shared_timer->timer_in_use -= 2; - shared_timer->timer_direction_upward = 0; - shared_timer->timer_direction_downward++; - } else { - shared_timer->timer_in_use -= 4; - shared_timer->timer_direction_upward = 0; - shared_timer->timer_direction_downward++; - } - - if (shared_timer->timer_direction_upward > 3 ) { /* using history */ - shared_timer->timer_in_use += 3; - shared_timer->timer_direction_upward = 0; - shared_timer->timer_direction_downward = 0; - } - if (shared_timer->timer_direction_downward > 5) { /* using history */ - shared_timer->timer_in_use -= 4 ; - shared_timer->timer_direction_downward = 0; - shared_timer->timer_direction_upward = 0; - } - } - - /* boundary checking */ - if (shared_timer->timer_in_use > shared_timer->threshold_high) - shared_timer->timer_in_use = shared_timer->threshold_high; - else if (shared_timer->timer_in_use < shared_timer->threshold_low) - shared_timer->timer_in_use = shared_timer->threshold_low; - - nesdev->currcq_count = 0; - - spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags); -} - - -/** - * nes_init_adapter - initialize adapter - */ -struct nes_adapter *nes_init_adapter(struct nes_device *nesdev, u8 hw_rev) { - struct nes_adapter *nesadapter = NULL; - unsigned long num_pds; - u32 u32temp; - u32 port_count; - u16 max_rq_wrs; - u16 max_sq_wrs; - u32 max_mr; - u32 max_256pbl; - u32 max_4kpbl; - u32 max_qp; - u32 max_irrq; - u32 max_cq; - u32 hte_index_mask; - u32 adapter_size; - u32 arp_table_size; - u16 vendor_id; - u16 device_id; - u8 OneG_Mode; - u8 func_index; - - /* search the list of existing adapters */ - list_for_each_entry(nesadapter, &nes_adapter_list, list) { - nes_debug(NES_DBG_INIT, "Searching Adapter list for PCI devfn = 0x%X," - " adapter PCI slot/bus = %u/%u, pci devices PCI slot/bus = %u/%u, .\n", - nesdev->pcidev->devfn, - PCI_SLOT(nesadapter->devfn), - nesadapter->bus_number, - PCI_SLOT(nesdev->pcidev->devfn), - nesdev->pcidev->bus->number ); - if ((PCI_SLOT(nesadapter->devfn) == PCI_SLOT(nesdev->pcidev->devfn)) && - (nesadapter->bus_number == nesdev->pcidev->bus->number)) { - nesadapter->ref_count++; - return nesadapter; - } - } - - /* no adapter found */ - num_pds = pci_resource_len(nesdev->pcidev, BAR_1) >> PAGE_SHIFT; - if ((hw_rev != NE020_REV) && (hw_rev != NE020_REV1)) { - nes_debug(NES_DBG_INIT, "NE020 driver detected unknown hardware revision 0x%x\n", - hw_rev); - return NULL; - } - - nes_debug(NES_DBG_INIT, "Determine Soft Reset, QP_control=0x%x, CPU0=0x%x, CPU1=0x%x, CPU2=0x%x\n", - nes_read_indexed(nesdev, NES_IDX_QP_CONTROL + PCI_FUNC(nesdev->pcidev->devfn) * 8), - nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS), - nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS + 4), - nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS + 8)); - - nes_debug(NES_DBG_INIT, "Reset and init NE020\n"); - - - if ((port_count = nes_reset_adapter_ne020(nesdev, &OneG_Mode)) == 0) - return NULL; - - max_qp = nes_read_indexed(nesdev, NES_IDX_QP_CTX_SIZE); - nes_debug(NES_DBG_INIT, "QP_CTX_SIZE=%u\n", max_qp); - - u32temp = nes_read_indexed(nesdev, NES_IDX_QUAD_HASH_TABLE_SIZE); - if (max_qp > ((u32)1 << (u32temp & 0x001f))) { - nes_debug(NES_DBG_INIT, "Reducing Max QPs to %u due to hash table size = 0x%08X\n", - max_qp, u32temp); - max_qp = (u32)1 << (u32temp & 0x001f); - } - - hte_index_mask = ((u32)1 << ((u32temp & 0x001f)+1))-1; - nes_debug(NES_DBG_INIT, "Max QP = %u, hte_index_mask = 0x%08X.\n", - max_qp, hte_index_mask); - - u32temp = nes_read_indexed(nesdev, NES_IDX_IRRQ_COUNT); - - max_irrq = 1 << (u32temp & 0x001f); - - if (max_qp > max_irrq) { - max_qp = max_irrq; - nes_debug(NES_DBG_INIT, "Reducing Max QPs to %u due to Available Q1s.\n", - max_qp); - } - - /* there should be no reason to allocate more pds than qps */ - if (num_pds > max_qp) - num_pds = max_qp; - - u32temp = nes_read_indexed(nesdev, NES_IDX_MRT_SIZE); - max_mr = (u32)8192 << (u32temp & 0x7); - - u32temp = nes_read_indexed(nesdev, NES_IDX_PBL_REGION_SIZE); - max_256pbl = (u32)1 << (u32temp & 0x0000001f); - max_4kpbl = (u32)1 << ((u32temp >> 16) & 0x0000001f); - max_cq = nes_read_indexed(nesdev, NES_IDX_CQ_CTX_SIZE); - - u32temp = nes_read_indexed(nesdev, NES_IDX_ARP_CACHE_SIZE); - arp_table_size = 1 << u32temp; - - adapter_size = (sizeof(struct nes_adapter) + - (sizeof(unsigned long)-1)) & (~(sizeof(unsigned long)-1)); - adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(max_qp); - adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(max_mr); - adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(max_cq); - adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(num_pds); - adapter_size += sizeof(unsigned long) * BITS_TO_LONGS(arp_table_size); - adapter_size += sizeof(struct nes_qp **) * max_qp; - - /* allocate a new adapter struct */ - nesadapter = kzalloc(adapter_size, GFP_KERNEL); - if (!nesadapter) - return NULL; - - nes_debug(NES_DBG_INIT, "Allocating new nesadapter @ %p, size = %u (actual size = %u).\n", - nesadapter, (u32)sizeof(struct nes_adapter), adapter_size); - - if (nes_read_eeprom_values(nesdev, nesadapter)) { - printk(KERN_ERR PFX "Unable to read EEPROM data.\n"); - kfree(nesadapter); - return NULL; - } - - nesadapter->vendor_id = (((u32) nesadapter->mac_addr_high) << 8) | - (nesadapter->mac_addr_low >> 24); - - pci_bus_read_config_word(nesdev->pcidev->bus, nesdev->pcidev->devfn, - PCI_DEVICE_ID, &device_id); - nesadapter->vendor_part_id = device_id; - - if (nes_init_serdes(nesdev, hw_rev, port_count, nesadapter, - OneG_Mode)) { - kfree(nesadapter); - return NULL; - } - nes_init_csr_ne020(nesdev, hw_rev, port_count); - - memset(nesadapter->pft_mcast_map, 255, - sizeof nesadapter->pft_mcast_map); - - /* populate the new nesadapter */ - nesadapter->nesdev = nesdev; - nesadapter->devfn = nesdev->pcidev->devfn; - nesadapter->bus_number = nesdev->pcidev->bus->number; - nesadapter->ref_count = 1; - nesadapter->timer_int_req = 0xffff0000; - nesadapter->OneG_Mode = OneG_Mode; - nesadapter->doorbell_start = nesdev->doorbell_region; - - /* nesadapter->tick_delta = clk_divisor; */ - nesadapter->hw_rev = hw_rev; - nesadapter->port_count = port_count; - - nesadapter->max_qp = max_qp; - nesadapter->hte_index_mask = hte_index_mask; - nesadapter->max_irrq = max_irrq; - nesadapter->max_mr = max_mr; - nesadapter->max_256pbl = max_256pbl - 1; - nesadapter->max_4kpbl = max_4kpbl - 1; - nesadapter->max_cq = max_cq; - nesadapter->free_256pbl = max_256pbl - 1; - nesadapter->free_4kpbl = max_4kpbl - 1; - nesadapter->max_pd = num_pds; - nesadapter->arp_table_size = arp_table_size; - - nesadapter->et_pkt_rate_low = NES_TIMER_ENABLE_LIMIT; - if (nes_drv_opt & NES_DRV_OPT_DISABLE_INT_MOD) { - nesadapter->et_use_adaptive_rx_coalesce = 0; - nesadapter->timer_int_limit = NES_TIMER_INT_LIMIT; - nesadapter->et_rx_coalesce_usecs_irq = interrupt_mod_interval; - } else { - nesadapter->et_use_adaptive_rx_coalesce = 1; - nesadapter->timer_int_limit = NES_TIMER_INT_LIMIT_DYNAMIC; - nesadapter->et_rx_coalesce_usecs_irq = 0; - printk(PFX "%s: Using Adaptive Interrupt Moderation\n", __func__); - } - /* Setup and enable the periodic timer */ - if (nesadapter->et_rx_coalesce_usecs_irq) - nes_write32(nesdev->regs+NES_PERIODIC_CONTROL, 0x80000000 | - ((u32)(nesadapter->et_rx_coalesce_usecs_irq * 8))); - else - nes_write32(nesdev->regs+NES_PERIODIC_CONTROL, 0x00000000); - - nesadapter->base_pd = 1; - - nesadapter->device_cap_flags = IB_DEVICE_LOCAL_DMA_LKEY | - IB_DEVICE_MEM_WINDOW | - IB_DEVICE_MEM_MGT_EXTENSIONS; - - nesadapter->allocated_qps = (unsigned long *)&(((unsigned char *)nesadapter) - [(sizeof(struct nes_adapter)+(sizeof(unsigned long)-1))&(~(sizeof(unsigned long)-1))]); - nesadapter->allocated_cqs = &nesadapter->allocated_qps[BITS_TO_LONGS(max_qp)]; - nesadapter->allocated_mrs = &nesadapter->allocated_cqs[BITS_TO_LONGS(max_cq)]; - nesadapter->allocated_pds = &nesadapter->allocated_mrs[BITS_TO_LONGS(max_mr)]; - nesadapter->allocated_arps = &nesadapter->allocated_pds[BITS_TO_LONGS(num_pds)]; - nesadapter->qp_table = (struct nes_qp **)(&nesadapter->allocated_arps[BITS_TO_LONGS(arp_table_size)]); - - - /* mark the usual suspect QPs, MR and CQs as in use */ - for (u32temp = 0; u32temp < NES_FIRST_QPN; u32temp++) { - set_bit(u32temp, nesadapter->allocated_qps); - set_bit(u32temp, nesadapter->allocated_cqs); - } - set_bit(0, nesadapter->allocated_mrs); - - for (u32temp = 0; u32temp < 20; u32temp++) - set_bit(u32temp, nesadapter->allocated_pds); - u32temp = nes_read_indexed(nesdev, NES_IDX_QP_MAX_CFG_SIZES); - - max_rq_wrs = ((u32temp >> 8) & 3); - switch (max_rq_wrs) { - case 0: - max_rq_wrs = 4; - break; - case 1: - max_rq_wrs = 16; - break; - case 2: - max_rq_wrs = 32; - break; - case 3: - max_rq_wrs = 512; - break; - } - - max_sq_wrs = (u32temp & 3); - switch (max_sq_wrs) { - case 0: - max_sq_wrs = 4; - break; - case 1: - max_sq_wrs = 16; - break; - case 2: - max_sq_wrs = 32; - break; - case 3: - max_sq_wrs = 512; - break; - } - nesadapter->max_qp_wr = min(max_rq_wrs, max_sq_wrs); - nesadapter->max_irrq_wr = (u32temp >> 16) & 3; - - nesadapter->max_sge = 4; - nesadapter->max_cqe = 32766; - - if (nes_read_eeprom_values(nesdev, nesadapter)) { - printk(KERN_ERR PFX "Unable to read EEPROM data.\n"); - kfree(nesadapter); - return NULL; - } - - u32temp = nes_read_indexed(nesdev, NES_IDX_TCP_TIMER_CONFIG); - nes_write_indexed(nesdev, NES_IDX_TCP_TIMER_CONFIG, - (u32temp & 0xff000000) | (nesadapter->tcp_timer_core_clk_divisor & 0x00ffffff)); - - /* setup port configuration */ - if (nesadapter->port_count == 1) { - nesadapter->log_port = 0x00000000; - if (nes_drv_opt & NES_DRV_OPT_DUAL_LOGICAL_PORT) - nes_write_indexed(nesdev, NES_IDX_TX_POOL_SIZE, 0x00000002); - else - nes_write_indexed(nesdev, NES_IDX_TX_POOL_SIZE, 0x00000003); - } else { - if (nesadapter->phy_type[0] == NES_PHY_TYPE_PUMA_1G) { - nesadapter->log_port = 0x000000D8; - } else { - if (nesadapter->port_count == 2) - nesadapter->log_port = 0x00000044; - else - nesadapter->log_port = 0x000000e4; - } - nes_write_indexed(nesdev, NES_IDX_TX_POOL_SIZE, 0x00000003); - } - - nes_write_indexed(nesdev, NES_IDX_NIC_LOGPORT_TO_PHYPORT, - nesadapter->log_port); - nes_debug(NES_DBG_INIT, "Probe time, LOG2PHY=%u\n", - nes_read_indexed(nesdev, NES_IDX_NIC_LOGPORT_TO_PHYPORT)); - - spin_lock_init(&nesadapter->resource_lock); - spin_lock_init(&nesadapter->phy_lock); - spin_lock_init(&nesadapter->pbl_lock); - spin_lock_init(&nesadapter->periodic_timer_lock); - - INIT_LIST_HEAD(&nesadapter->nesvnic_list[0]); - INIT_LIST_HEAD(&nesadapter->nesvnic_list[1]); - INIT_LIST_HEAD(&nesadapter->nesvnic_list[2]); - INIT_LIST_HEAD(&nesadapter->nesvnic_list[3]); - - if ((!nesadapter->OneG_Mode) && (nesadapter->port_count == 2)) { - u32 pcs_control_status0, pcs_control_status1; - u32 reset_value; - u32 i = 0; - u32 int_cnt = 0; - u32 ext_cnt = 0; - unsigned long flags; - u32 j = 0; - - pcs_control_status0 = nes_read_indexed(nesdev, - NES_IDX_PHY_PCS_CONTROL_STATUS0); - pcs_control_status1 = nes_read_indexed(nesdev, - NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200); - - for (i = 0; i < NES_MAX_LINK_CHECK; i++) { - pcs_control_status0 = nes_read_indexed(nesdev, - NES_IDX_PHY_PCS_CONTROL_STATUS0); - pcs_control_status1 = nes_read_indexed(nesdev, - NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200); - if ((0x0F000100 == (pcs_control_status0 & 0x0F000100)) - || (0x0F000100 == (pcs_control_status1 & 0x0F000100))) - int_cnt++; - usleep_range(1000, 2000); - } - if (int_cnt > 1) { - spin_lock_irqsave(&nesadapter->phy_lock, flags); - nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F0C8); - mh_detected++; - reset_value = nes_read32(nesdev->regs+NES_SOFTWARE_RESET); - reset_value |= 0x0000003d; - nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value); - - while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET) - & 0x00000040) != 0x00000040) && (j++ < 5000)); - spin_unlock_irqrestore(&nesadapter->phy_lock, flags); - - pcs_control_status0 = nes_read_indexed(nesdev, - NES_IDX_PHY_PCS_CONTROL_STATUS0); - pcs_control_status1 = nes_read_indexed(nesdev, - NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200); - - for (i = 0; i < NES_MAX_LINK_CHECK; i++) { - pcs_control_status0 = nes_read_indexed(nesdev, - NES_IDX_PHY_PCS_CONTROL_STATUS0); - pcs_control_status1 = nes_read_indexed(nesdev, - NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200); - if ((0x0F000100 == (pcs_control_status0 & 0x0F000100)) - || (0x0F000100 == (pcs_control_status1 & 0x0F000100))) { - if (++ext_cnt > int_cnt) { - spin_lock_irqsave(&nesadapter->phy_lock, flags); - nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, - 0x0000F088); - mh_detected++; - reset_value = nes_read32(nesdev->regs+NES_SOFTWARE_RESET); - reset_value |= 0x0000003d; - nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value); - - while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET) - & 0x00000040) != 0x00000040) && (j++ < 5000)); - spin_unlock_irqrestore(&nesadapter->phy_lock, flags); - break; - } - } - usleep_range(1000, 2000); - } - } - } - - if (nesadapter->hw_rev == NE020_REV) { - timer_setup(&nesadapter->mh_timer, nes_mh_fix, 0); - nesadapter->mh_timer.expires = jiffies + (HZ/5); /* 1 second */ - add_timer(&nesadapter->mh_timer); - } else { - nes_write32(nesdev->regs+NES_INTF_INT_STAT, 0x0f000000); - } - - timer_setup(&nesadapter->lc_timer, nes_clc, 0); - nesadapter->lc_timer.expires = jiffies + 3600 * HZ; /* 1 hour */ - add_timer(&nesadapter->lc_timer); - - list_add_tail(&nesadapter->list, &nes_adapter_list); - - for (func_index = 0; func_index < 8; func_index++) { - pci_bus_read_config_word(nesdev->pcidev->bus, - PCI_DEVFN(PCI_SLOT(nesdev->pcidev->devfn), - func_index), 0, &vendor_id); - if (vendor_id == 0xffff) - break; - } - nes_debug(NES_DBG_INIT, "%s %d functions found for %s.\n", __func__, - func_index, pci_name(nesdev->pcidev)); - nesadapter->adapter_fcn_count = func_index; - - return nesadapter; -} - - -/** - * nes_reset_adapter_ne020 - */ -static unsigned int nes_reset_adapter_ne020(struct nes_device *nesdev, u8 *OneG_Mode) -{ - u32 port_count; - u32 u32temp; - u32 i; - - u32temp = nes_read32(nesdev->regs+NES_SOFTWARE_RESET); - port_count = ((u32temp & 0x00000300) >> 8) + 1; - /* TODO: assuming that both SERDES are set the same for now */ - *OneG_Mode = (u32temp & 0x00003c00) ? 0 : 1; - nes_debug(NES_DBG_INIT, "Initial Software Reset = 0x%08X, port_count=%u\n", - u32temp, port_count); - if (*OneG_Mode) - nes_debug(NES_DBG_INIT, "Running in 1G mode.\n"); - u32temp &= 0xff00ffc0; - switch (port_count) { - case 1: - u32temp |= 0x00ee0000; - break; - case 2: - u32temp |= 0x00cc0000; - break; - case 4: - u32temp |= 0x00000000; - break; - default: - return 0; - break; - } - - /* check and do full reset if needed */ - if (nes_read_indexed(nesdev, NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8))) { - nes_debug(NES_DBG_INIT, "Issuing Full Soft reset = 0x%08X\n", u32temp | 0xd); - nes_write32(nesdev->regs+NES_SOFTWARE_RESET, u32temp | 0xd); - - i = 0; - while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET) & 0x00000040) == 0) && i++ < 10000) - mdelay(1); - if (i > 10000) { - nes_debug(NES_DBG_INIT, "Did not see full soft reset done.\n"); - return 0; - } - - i = 0; - while ((nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS) != 0x80) && i++ < 10000) - mdelay(1); - if (i > 10000) { - printk(KERN_ERR PFX "Internal CPU not ready, status = %02X\n", - nes_read_indexed(nesdev, NES_IDX_INT_CPU_STATUS)); - return 0; - } - } - - /* port reset */ - switch (port_count) { - case 1: - u32temp |= 0x00ee0010; - break; - case 2: - u32temp |= 0x00cc0030; - break; - case 4: - u32temp |= 0x00000030; - break; - } - - nes_debug(NES_DBG_INIT, "Issuing Port Soft reset = 0x%08X\n", u32temp | 0xd); - nes_write32(nesdev->regs+NES_SOFTWARE_RESET, u32temp | 0xd); - - i = 0; - while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET) & 0x00000040) == 0) && i++ < 10000) - mdelay(1); - if (i > 10000) { - nes_debug(NES_DBG_INIT, "Did not see port soft reset done.\n"); - return 0; - } - - /* serdes 0 */ - i = 0; - while (((u32temp = (nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0) - & 0x0000000f)) != 0x0000000f) && i++ < 5000) - mdelay(1); - if (i > 5000) { - nes_debug(NES_DBG_INIT, "Serdes 0 not ready, status=%x\n", u32temp); - return 0; - } - - /* serdes 1 */ - if (port_count > 1) { - i = 0; - while (((u32temp = (nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS1) - & 0x0000000f)) != 0x0000000f) && i++ < 5000) - mdelay(1); - if (i > 5000) { - nes_debug(NES_DBG_INIT, "Serdes 1 not ready, status=%x\n", u32temp); - return 0; - } - } - - return port_count; -} - - -/** - * nes_init_serdes - */ -static int nes_init_serdes(struct nes_device *nesdev, u8 hw_rev, u8 port_count, - struct nes_adapter *nesadapter, u8 OneG_Mode) -{ - int i; - u32 u32temp; - u32 sds; - - if (hw_rev != NE020_REV) { - /* init serdes 0 */ - switch (nesadapter->phy_type[0]) { - case NES_PHY_TYPE_CX4: - if (wide_ppm_offset) - nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000FFFAA); - else - nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000000FF); - break; - case NES_PHY_TYPE_KR: - nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000000FF); - nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP0, 0x00000000); - break; - case NES_PHY_TYPE_PUMA_1G: - nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000000FF); - sds = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0); - sds |= 0x00000100; - nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0, sds); - break; - default: - nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000000FF); - break; - } - - if (!OneG_Mode) - nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_HIGHZ_LANE_MODE0, 0x11110000); - - if (port_count < 2) - return 0; - - /* init serdes 1 */ - if (!(OneG_Mode && (nesadapter->phy_type[1] != NES_PHY_TYPE_PUMA_1G))) - nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL1, 0x000000FF); - - switch (nesadapter->phy_type[1]) { - case NES_PHY_TYPE_ARGUS: - case NES_PHY_TYPE_SFP_D: - nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP0, 0x00000000); - nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP1, 0x00000000); - break; - case NES_PHY_TYPE_CX4: - if (wide_ppm_offset) - nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL1, 0x000FFFAA); - break; - case NES_PHY_TYPE_KR: - nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP1, 0x00000000); - break; - case NES_PHY_TYPE_PUMA_1G: - sds = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1); - sds |= 0x000000100; - nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, sds); - } - if (!OneG_Mode) { - nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_HIGHZ_LANE_MODE1, 0x11110000); - sds = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1); - sds &= 0xFFFFFFBF; - nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, sds); - } - } else { - /* init serdes 0 */ - nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0, 0x00000008); - i = 0; - while (((u32temp = (nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0) - & 0x0000000f)) != 0x0000000f) && i++ < 5000) - mdelay(1); - if (i > 5000) { - nes_debug(NES_DBG_PHY, "Init: serdes 0 not ready, status=%x\n", u32temp); - return 1; - } - nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP0, 0x000bdef7); - nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_DRIVE0, 0x9ce73000); - nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_MODE0, 0x0ff00000); - nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_SIGDET0, 0x00000000); - nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_BYPASS0, 0x00000000); - nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_LOOPBACK_CONTROL0, 0x00000000); - if (OneG_Mode) - nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_EQ_CONTROL0, 0xf0182222); - else - nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_EQ_CONTROL0, 0xf0042222); - - nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000000ff); - if (port_count > 1) { - /* init serdes 1 */ - nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x00000048); - i = 0; - while (((u32temp = (nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS1) - & 0x0000000f)) != 0x0000000f) && (i++ < 5000)) - mdelay(1); - if (i > 5000) { - printk("%s: Init: serdes 1 not ready, status=%x\n", __func__, u32temp); - /* return 1; */ - } - nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP1, 0x000bdef7); - nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_DRIVE1, 0x9ce73000); - nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_MODE1, 0x0ff00000); - nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_SIGDET1, 0x00000000); - nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_BYPASS1, 0x00000000); - nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_LOOPBACK_CONTROL1, 0x00000000); - nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_EQ_CONTROL1, 0xf0002222); - nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL1, 0x000000ff); - } - } - return 0; -} - - -/** - * nes_init_csr_ne020 - * Initialize registers for ne020 hardware - */ -static void nes_init_csr_ne020(struct nes_device *nesdev, u8 hw_rev, u8 port_count) -{ - u32 u32temp; - - nes_debug(NES_DBG_INIT, "port_count=%d\n", port_count); - - nes_write_indexed(nesdev, 0x000001E4, 0x00000007); - /* nes_write_indexed(nesdev, 0x000001E8, 0x000208C4); */ - nes_write_indexed(nesdev, 0x000001E8, 0x00020874); - nes_write_indexed(nesdev, 0x000001D8, 0x00048002); - /* nes_write_indexed(nesdev, 0x000001D8, 0x0004B002); */ - nes_write_indexed(nesdev, 0x000001FC, 0x00050005); - nes_write_indexed(nesdev, 0x00000600, 0x55555555); - nes_write_indexed(nesdev, 0x00000604, 0x55555555); - - /* TODO: move these MAC register settings to NIC bringup */ - nes_write_indexed(nesdev, 0x00002000, 0x00000001); - nes_write_indexed(nesdev, 0x00002004, 0x00000001); - nes_write_indexed(nesdev, 0x00002008, 0x0000FFFF); - nes_write_indexed(nesdev, 0x0000200C, 0x00000001); - nes_write_indexed(nesdev, 0x00002010, 0x000003c1); - nes_write_indexed(nesdev, 0x0000201C, 0x75345678); - if (port_count > 1) { - nes_write_indexed(nesdev, 0x00002200, 0x00000001); - nes_write_indexed(nesdev, 0x00002204, 0x00000001); - nes_write_indexed(nesdev, 0x00002208, 0x0000FFFF); - nes_write_indexed(nesdev, 0x0000220C, 0x00000001); - nes_write_indexed(nesdev, 0x00002210, 0x000003c1); - nes_write_indexed(nesdev, 0x0000221C, 0x75345678); - nes_write_indexed(nesdev, 0x00000908, 0x20000001); - } - if (port_count > 2) { - nes_write_indexed(nesdev, 0x00002400, 0x00000001); - nes_write_indexed(nesdev, 0x00002404, 0x00000001); - nes_write_indexed(nesdev, 0x00002408, 0x0000FFFF); - nes_write_indexed(nesdev, 0x0000240C, 0x00000001); - nes_write_indexed(nesdev, 0x00002410, 0x000003c1); - nes_write_indexed(nesdev, 0x0000241C, 0x75345678); - nes_write_indexed(nesdev, 0x00000910, 0x20000001); - - nes_write_indexed(nesdev, 0x00002600, 0x00000001); - nes_write_indexed(nesdev, 0x00002604, 0x00000001); - nes_write_indexed(nesdev, 0x00002608, 0x0000FFFF); - nes_write_indexed(nesdev, 0x0000260C, 0x00000001); - nes_write_indexed(nesdev, 0x00002610, 0x000003c1); - nes_write_indexed(nesdev, 0x0000261C, 0x75345678); - nes_write_indexed(nesdev, 0x00000918, 0x20000001); - } - - nes_write_indexed(nesdev, 0x00005000, 0x00018000); - /* nes_write_indexed(nesdev, 0x00005000, 0x00010000); */ - nes_write_indexed(nesdev, NES_IDX_WQM_CONFIG1, (wqm_quanta << 1) | - 0x00000001); - nes_write_indexed(nesdev, 0x00005008, 0x1F1F1F1F); - nes_write_indexed(nesdev, 0x00005010, 0x1F1F1F1F); - nes_write_indexed(nesdev, 0x00005018, 0x1F1F1F1F); - nes_write_indexed(nesdev, 0x00005020, 0x1F1F1F1F); - nes_write_indexed(nesdev, 0x00006090, 0xFFFFFFFF); - - /* TODO: move this to code, get from EEPROM */ - nes_write_indexed(nesdev, 0x00000900, 0x20000001); - nes_write_indexed(nesdev, 0x000060C0, 0x0000028e); - nes_write_indexed(nesdev, 0x000060C8, 0x00000020); - - nes_write_indexed(nesdev, 0x000001EC, 0x7b2625a0); - /* nes_write_indexed(nesdev, 0x000001EC, 0x5f2625a0); */ - - if (hw_rev != NE020_REV) { - u32temp = nes_read_indexed(nesdev, 0x000008e8); - u32temp |= 0x80000000; - nes_write_indexed(nesdev, 0x000008e8, u32temp); - u32temp = nes_read_indexed(nesdev, 0x000021f8); - u32temp &= 0x7fffffff; - u32temp |= 0x7fff0010; - nes_write_indexed(nesdev, 0x000021f8, u32temp); - if (port_count > 1) { - u32temp = nes_read_indexed(nesdev, 0x000023f8); - u32temp &= 0x7fffffff; - u32temp |= 0x7fff0010; - nes_write_indexed(nesdev, 0x000023f8, u32temp); - } - } -} - - -/** - * nes_destroy_adapter - destroy the adapter structure - */ -void nes_destroy_adapter(struct nes_adapter *nesadapter) -{ - struct nes_adapter *tmp_adapter; - - list_for_each_entry(tmp_adapter, &nes_adapter_list, list) { - nes_debug(NES_DBG_SHUTDOWN, "Nes Adapter list entry = 0x%p.\n", - tmp_adapter); - } - - nesadapter->ref_count--; - if (!nesadapter->ref_count) { - if (nesadapter->hw_rev == NE020_REV) { - del_timer(&nesadapter->mh_timer); - } - del_timer(&nesadapter->lc_timer); - - list_del(&nesadapter->list); - kfree(nesadapter); - } -} - - -/** - * nes_init_cqp - */ -int nes_init_cqp(struct nes_device *nesdev) -{ - struct nes_adapter *nesadapter = nesdev->nesadapter; - struct nes_hw_cqp_qp_context *cqp_qp_context; - struct nes_hw_cqp_wqe *cqp_wqe; - struct nes_hw_ceq *ceq; - struct nes_hw_ceq *nic_ceq; - struct nes_hw_aeq *aeq; - void *vmem; - dma_addr_t pmem; - u32 count=0; - u32 cqp_head; - u64 u64temp; - u32 u32temp; - - /* allocate CQP memory */ - /* Need to add max_cq to the aeq size once cq overflow checking is added back */ - /* SQ is 512 byte aligned, others are 256 byte aligned */ - nesdev->cqp_mem_size = 512 + - (sizeof(struct nes_hw_cqp_wqe) * NES_CQP_SQ_SIZE) + - (sizeof(struct nes_hw_cqe) * NES_CCQ_SIZE) + - max(((u32)sizeof(struct nes_hw_ceqe) * NES_CCEQ_SIZE), (u32)256) + - max(((u32)sizeof(struct nes_hw_ceqe) * NES_NIC_CEQ_SIZE), (u32)256) + - (sizeof(struct nes_hw_aeqe) * nesadapter->max_qp) + - sizeof(struct nes_hw_cqp_qp_context); - - nesdev->cqp_vbase = pci_zalloc_consistent(nesdev->pcidev, - nesdev->cqp_mem_size, - &nesdev->cqp_pbase); - if (!nesdev->cqp_vbase) { - nes_debug(NES_DBG_INIT, "Unable to allocate memory for host descriptor rings\n"); - return -ENOMEM; - } - - /* Allocate a twice the number of CQP requests as the SQ size */ - nesdev->nes_cqp_requests = kzalloc(sizeof(struct nes_cqp_request) * - 2 * NES_CQP_SQ_SIZE, GFP_KERNEL); - if (!nesdev->nes_cqp_requests) { - pci_free_consistent(nesdev->pcidev, nesdev->cqp_mem_size, nesdev->cqp.sq_vbase, - nesdev->cqp.sq_pbase); - return -ENOMEM; - } - - nes_debug(NES_DBG_INIT, "Allocated CQP structures at %p (phys = %016lX), size = %u.\n", - nesdev->cqp_vbase, (unsigned long)nesdev->cqp_pbase, nesdev->cqp_mem_size); - - spin_lock_init(&nesdev->cqp.lock); - init_waitqueue_head(&nesdev->cqp.waitq); - - /* Setup Various Structures */ - vmem = (void *)(((unsigned long)nesdev->cqp_vbase + (512 - 1)) & - ~(unsigned long)(512 - 1)); - pmem = (dma_addr_t)(((unsigned long long)nesdev->cqp_pbase + (512 - 1)) & - ~(unsigned long long)(512 - 1)); - - nesdev->cqp.sq_vbase = vmem; - nesdev->cqp.sq_pbase = pmem; - nesdev->cqp.sq_size = NES_CQP_SQ_SIZE; - nesdev->cqp.sq_head = 0; - nesdev->cqp.sq_tail = 0; - nesdev->cqp.qp_id = PCI_FUNC(nesdev->pcidev->devfn); - - vmem += (sizeof(struct nes_hw_cqp_wqe) * nesdev->cqp.sq_size); - pmem += (sizeof(struct nes_hw_cqp_wqe) * nesdev->cqp.sq_size); - - nesdev->ccq.cq_vbase = vmem; - nesdev->ccq.cq_pbase = pmem; - nesdev->ccq.cq_size = NES_CCQ_SIZE; - nesdev->ccq.cq_head = 0; - nesdev->ccq.ce_handler = nes_cqp_ce_handler; - nesdev->ccq.cq_number = PCI_FUNC(nesdev->pcidev->devfn); - - vmem += (sizeof(struct nes_hw_cqe) * nesdev->ccq.cq_size); - pmem += (sizeof(struct nes_hw_cqe) * nesdev->ccq.cq_size); - - nesdev->ceq_index = PCI_FUNC(nesdev->pcidev->devfn); - ceq = &nesadapter->ceq[nesdev->ceq_index]; - ceq->ceq_vbase = vmem; - ceq->ceq_pbase = pmem; - ceq->ceq_size = NES_CCEQ_SIZE; - ceq->ceq_head = 0; - - vmem += max(((u32)sizeof(struct nes_hw_ceqe) * ceq->ceq_size), (u32)256); - pmem += max(((u32)sizeof(struct nes_hw_ceqe) * ceq->ceq_size), (u32)256); - - nesdev->nic_ceq_index = PCI_FUNC(nesdev->pcidev->devfn) + 8; - nic_ceq = &nesadapter->ceq[nesdev->nic_ceq_index]; - nic_ceq->ceq_vbase = vmem; - nic_ceq->ceq_pbase = pmem; - nic_ceq->ceq_size = NES_NIC_CEQ_SIZE; - nic_ceq->ceq_head = 0; - - vmem += max(((u32)sizeof(struct nes_hw_ceqe) * nic_ceq->ceq_size), (u32)256); - pmem += max(((u32)sizeof(struct nes_hw_ceqe) * nic_ceq->ceq_size), (u32)256); - - aeq = &nesadapter->aeq[PCI_FUNC(nesdev->pcidev->devfn)]; - aeq->aeq_vbase = vmem; - aeq->aeq_pbase = pmem; - aeq->aeq_size = nesadapter->max_qp; - aeq->aeq_head = 0; - - /* Setup QP Context */ - vmem += (sizeof(struct nes_hw_aeqe) * aeq->aeq_size); - pmem += (sizeof(struct nes_hw_aeqe) * aeq->aeq_size); - - cqp_qp_context = vmem; - cqp_qp_context->context_words[0] = - cpu_to_le32((PCI_FUNC(nesdev->pcidev->devfn) << 12) + (2 << 10)); - cqp_qp_context->context_words[1] = 0; - cqp_qp_context->context_words[2] = cpu_to_le32((u32)nesdev->cqp.sq_pbase); - cqp_qp_context->context_words[3] = cpu_to_le32(((u64)nesdev->cqp.sq_pbase) >> 32); - - - /* Write the address to Create CQP */ - if ((sizeof(dma_addr_t) > 4)) { - nes_write_indexed(nesdev, - NES_IDX_CREATE_CQP_HIGH + (PCI_FUNC(nesdev->pcidev->devfn) * 8), - ((u64)pmem) >> 32); - } else { - nes_write_indexed(nesdev, - NES_IDX_CREATE_CQP_HIGH + (PCI_FUNC(nesdev->pcidev->devfn) * 8), 0); - } - nes_write_indexed(nesdev, - NES_IDX_CREATE_CQP_LOW + (PCI_FUNC(nesdev->pcidev->devfn) * 8), - (u32)pmem); - - INIT_LIST_HEAD(&nesdev->cqp_avail_reqs); - INIT_LIST_HEAD(&nesdev->cqp_pending_reqs); - - for (count = 0; count < 2*NES_CQP_SQ_SIZE; count++) { - init_waitqueue_head(&nesdev->nes_cqp_requests[count].waitq); - list_add_tail(&nesdev->nes_cqp_requests[count].list, &nesdev->cqp_avail_reqs); - } - - /* Write Create CCQ WQE */ - cqp_head = nesdev->cqp.sq_head++; - cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head]; - nes_fill_init_cqp_wqe(cqp_wqe, nesdev); - set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX, - (NES_CQP_CREATE_CQ | NES_CQP_CQ_CEQ_VALID | - NES_CQP_CQ_CHK_OVERFLOW | ((u32)nesdev->ccq.cq_size << 16))); - set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX, - (nesdev->ccq.cq_number | - ((u32)nesdev->ceq_index << 16))); - u64temp = (u64)nesdev->ccq.cq_pbase; - set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp); - cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] = 0; - u64temp = (unsigned long)&nesdev->ccq; - cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_LOW_IDX] = - cpu_to_le32((u32)(u64temp >> 1)); - cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] = - cpu_to_le32(((u32)((u64temp) >> 33)) & 0x7FFFFFFF); - cqp_wqe->wqe_words[NES_CQP_CQ_WQE_DOORBELL_INDEX_HIGH_IDX] = 0; - - /* Write Create CEQ WQE */ - cqp_head = nesdev->cqp.sq_head++; - cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head]; - nes_fill_init_cqp_wqe(cqp_wqe, nesdev); - set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX, - (NES_CQP_CREATE_CEQ + ((u32)nesdev->ceq_index << 8))); - set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_CEQ_WQE_ELEMENT_COUNT_IDX, ceq->ceq_size); - u64temp = (u64)ceq->ceq_pbase; - set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp); - - /* Write Create AEQ WQE */ - cqp_head = nesdev->cqp.sq_head++; - cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head]; - nes_fill_init_cqp_wqe(cqp_wqe, nesdev); - set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX, - (NES_CQP_CREATE_AEQ + ((u32)PCI_FUNC(nesdev->pcidev->devfn) << 8))); - set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_AEQ_WQE_ELEMENT_COUNT_IDX, aeq->aeq_size); - u64temp = (u64)aeq->aeq_pbase; - set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp); - - /* Write Create NIC CEQ WQE */ - cqp_head = nesdev->cqp.sq_head++; - cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head]; - nes_fill_init_cqp_wqe(cqp_wqe, nesdev); - set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX, - (NES_CQP_CREATE_CEQ + ((u32)nesdev->nic_ceq_index << 8))); - set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_CEQ_WQE_ELEMENT_COUNT_IDX, nic_ceq->ceq_size); - u64temp = (u64)nic_ceq->ceq_pbase; - set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp); - - /* Poll until CCQP done */ - count = 0; - do { - if (count++ > 1000) { - printk(KERN_ERR PFX "Error creating CQP\n"); - pci_free_consistent(nesdev->pcidev, nesdev->cqp_mem_size, - nesdev->cqp_vbase, nesdev->cqp_pbase); - return -1; - } - udelay(10); - } while (!(nes_read_indexed(nesdev, - NES_IDX_QP_CONTROL + (PCI_FUNC(nesdev->pcidev->devfn) * 8)) & (1 << 8))); - - nes_debug(NES_DBG_INIT, "CQP Status = 0x%08X\n", nes_read_indexed(nesdev, - NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8))); - - u32temp = 0x04800000; - nes_write32(nesdev->regs+NES_WQE_ALLOC, u32temp | nesdev->cqp.qp_id); - - /* wait for the CCQ, CEQ, and AEQ to get created */ - count = 0; - do { - if (count++ > 1000) { - printk(KERN_ERR PFX "Error creating CCQ, CEQ, and AEQ\n"); - pci_free_consistent(nesdev->pcidev, nesdev->cqp_mem_size, - nesdev->cqp_vbase, nesdev->cqp_pbase); - return -1; - } - udelay(10); - } while (((nes_read_indexed(nesdev, - NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8)) & (15<<8)) != (15<<8))); - - /* dump the QP status value */ - nes_debug(NES_DBG_INIT, "QP Status = 0x%08X\n", nes_read_indexed(nesdev, - NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8))); - - nesdev->cqp.sq_tail++; - - return 0; -} - - -/** - * nes_destroy_cqp - */ -int nes_destroy_cqp(struct nes_device *nesdev) -{ - struct nes_hw_cqp_wqe *cqp_wqe; - u32 count = 0; - u32 cqp_head; - unsigned long flags; - - do { - if (count++ > 1000) - break; - udelay(10); - } while (!(nesdev->cqp.sq_head == nesdev->cqp.sq_tail)); - - /* Reset CCQ */ - nes_write32(nesdev->regs+NES_CQE_ALLOC, NES_CQE_ALLOC_RESET | - nesdev->ccq.cq_number); - - /* Disable device interrupts */ - nes_write32(nesdev->regs+NES_INT_MASK, 0x7fffffff); - - spin_lock_irqsave(&nesdev->cqp.lock, flags); - - /* Destroy the AEQ */ - cqp_head = nesdev->cqp.sq_head++; - nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1; - cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head]; - cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_AEQ | - ((u32)PCI_FUNC(nesdev->pcidev->devfn) << 8)); - cqp_wqe->wqe_words[NES_CQP_WQE_COMP_CTX_HIGH_IDX] = 0; - - /* Destroy the NIC CEQ */ - cqp_head = nesdev->cqp.sq_head++; - nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1; - cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head]; - cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_CEQ | - ((u32)nesdev->nic_ceq_index << 8)); - - /* Destroy the CEQ */ - cqp_head = nesdev->cqp.sq_head++; - nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1; - cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head]; - cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_CEQ | - (nesdev->ceq_index << 8)); - - /* Destroy the CCQ */ - cqp_head = nesdev->cqp.sq_head++; - nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1; - cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head]; - cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_CQ); - cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(nesdev->ccq.cq_number | - ((u32)nesdev->ceq_index << 16)); - - /* Destroy CQP */ - cqp_head = nesdev->cqp.sq_head++; - nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1; - cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head]; - cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_DESTROY_QP | - NES_CQP_QP_TYPE_CQP); - cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(nesdev->cqp.qp_id); - - barrier(); - /* Ring doorbell (5 WQEs) */ - nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x05800000 | nesdev->cqp.qp_id); - - spin_unlock_irqrestore(&nesdev->cqp.lock, flags); - - /* wait for the CCQ, CEQ, and AEQ to get destroyed */ - count = 0; - do { - if (count++ > 1000) { - printk(KERN_ERR PFX "Function%d: Error destroying CCQ, CEQ, and AEQ\n", - PCI_FUNC(nesdev->pcidev->devfn)); - break; - } - udelay(10); - } while (((nes_read_indexed(nesdev, - NES_IDX_QP_CONTROL + (PCI_FUNC(nesdev->pcidev->devfn)*8)) & (15 << 8)) != 0)); - - /* dump the QP status value */ - nes_debug(NES_DBG_SHUTDOWN, "Function%d: QP Status = 0x%08X\n", - PCI_FUNC(nesdev->pcidev->devfn), - nes_read_indexed(nesdev, - NES_IDX_QP_CONTROL+(PCI_FUNC(nesdev->pcidev->devfn)*8))); - - kfree(nesdev->nes_cqp_requests); - - /* Free the control structures */ - pci_free_consistent(nesdev->pcidev, nesdev->cqp_mem_size, nesdev->cqp.sq_vbase, - nesdev->cqp.sq_pbase); - - return 0; -} - - -/** - * nes_init_1g_phy - */ -static int nes_init_1g_phy(struct nes_device *nesdev, u8 phy_type, u8 phy_index) -{ - u32 counter = 0; - u16 phy_data; - int ret = 0; - - nes_read_1G_phy_reg(nesdev, 1, phy_index, &phy_data); - nes_write_1G_phy_reg(nesdev, 23, phy_index, 0xb000); - - /* Reset the PHY */ - nes_write_1G_phy_reg(nesdev, 0, phy_index, 0x8000); - udelay(100); - counter = 0; - do { - nes_read_1G_phy_reg(nesdev, 0, phy_index, &phy_data); - if (counter++ > 100) { - ret = -1; - break; - } - } while (phy_data & 0x8000); - - /* Setting no phy loopback */ - phy_data &= 0xbfff; - phy_data |= 0x1140; - nes_write_1G_phy_reg(nesdev, 0, phy_index, phy_data); - nes_read_1G_phy_reg(nesdev, 0, phy_index, &phy_data); - nes_read_1G_phy_reg(nesdev, 0x17, phy_index, &phy_data); - nes_read_1G_phy_reg(nesdev, 0x1e, phy_index, &phy_data); - - /* Setting the interrupt mask */ - nes_read_1G_phy_reg(nesdev, 0x19, phy_index, &phy_data); - nes_write_1G_phy_reg(nesdev, 0x19, phy_index, 0xffee); - nes_read_1G_phy_reg(nesdev, 0x19, phy_index, &phy_data); - - /* turning on flow control */ - nes_read_1G_phy_reg(nesdev, 4, phy_index, &phy_data); - nes_write_1G_phy_reg(nesdev, 4, phy_index, (phy_data & ~(0x03E0)) | 0xc00); - nes_read_1G_phy_reg(nesdev, 4, phy_index, &phy_data); - - /* Clear Half duplex */ - nes_read_1G_phy_reg(nesdev, 9, phy_index, &phy_data); - nes_write_1G_phy_reg(nesdev, 9, phy_index, phy_data & ~(0x0100)); - nes_read_1G_phy_reg(nesdev, 9, phy_index, &phy_data); - - nes_read_1G_phy_reg(nesdev, 0, phy_index, &phy_data); - nes_write_1G_phy_reg(nesdev, 0, phy_index, phy_data | 0x0300); - - return ret; -} - - -/** - * nes_init_2025_phy - */ -static int nes_init_2025_phy(struct nes_device *nesdev, u8 phy_type, u8 phy_index) -{ - u32 temp_phy_data = 0; - u32 temp_phy_data2 = 0; - u32 counter = 0; - u32 sds; - u32 mac_index = nesdev->mac_index; - int ret = 0; - unsigned int first_attempt = 1; - - /* Check firmware heartbeat */ - nes_read_10G_phy_reg(nesdev, phy_index, 0x3, 0xd7ee); - temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL); - udelay(1500); - nes_read_10G_phy_reg(nesdev, phy_index, 0x3, 0xd7ee); - temp_phy_data2 = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL); - - if (temp_phy_data != temp_phy_data2) { - nes_read_10G_phy_reg(nesdev, phy_index, 0x3, 0xd7fd); - temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL); - if ((temp_phy_data & 0xff) > 0x20) - return 0; - printk(PFX "Reinitialize external PHY\n"); - } - - /* no heartbeat, configure the PHY */ - nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0x0000, 0x8000); - nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc300, 0x0000); - nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc316, 0x000A); - nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc318, 0x0052); - - switch (phy_type) { - case NES_PHY_TYPE_ARGUS: - nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc316, 0x000A); - nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc318, 0x0052); - nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc302, 0x000C); - nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc319, 0x0008); - nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0027, 0x0001); - nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc31a, 0x0098); - nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0026, 0x0E00); - - /* setup LEDs */ - nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd006, 0x0007); - nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd007, 0x000A); - nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd008, 0x0009); - break; - - case NES_PHY_TYPE_SFP_D: - nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc316, 0x000A); - nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc318, 0x0052); - nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc302, 0x0004); - nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc319, 0x0038); - nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0027, 0x0013); - nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc31a, 0x0098); - nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0026, 0x0E00); - - /* setup LEDs */ - nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd006, 0x0007); - nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd007, 0x000A); - nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd008, 0x0009); - break; - - case NES_PHY_TYPE_KR: - nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc316, 0x000A); - nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc318, 0x0052); - nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc302, 0x000C); - nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc319, 0x0010); - nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0027, 0x0013); - nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc31a, 0x0080); - nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0026, 0x0E00); - - /* setup LEDs */ - nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd006, 0x000B); - nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd007, 0x0003); - nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd008, 0x0004); - - nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0022, 0x406D); - nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0023, 0x0020); - break; - } - - nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0x0028, 0xA528); - - /* Bring PHY out of reset */ - nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc300, 0x0002); - - /* Check for heartbeat */ - counter = 0; - mdelay(690); - nes_read_10G_phy_reg(nesdev, phy_index, 0x3, 0xd7ee); - temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL); - do { - if (counter++ > 150) { - printk(PFX "No PHY heartbeat\n"); - break; - } - mdelay(1); - nes_read_10G_phy_reg(nesdev, phy_index, 0x3, 0xd7ee); - temp_phy_data2 = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL); - } while (temp_phy_data2 == temp_phy_data); - - /* wait for tracking */ - counter = 0; - do { - nes_read_10G_phy_reg(nesdev, phy_index, 0x3, 0xd7fd); - temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL); - if (counter++ > 300) { - if (((temp_phy_data & 0xff) == 0x0) && first_attempt) { - first_attempt = 0; - counter = 0; - /* reset AMCC PHY and try again */ - nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0xe854, 0x00c0); - nes_write_10G_phy_reg(nesdev, phy_index, 0x3, 0xe854, 0x0040); - continue; - } else { - ret = 1; - break; - } - } - mdelay(10); - } while ((temp_phy_data & 0xff) < 0x30); - - /* setup signal integrity */ - nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xd003, 0x0000); - nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xF00D, 0x00FE); - nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xF00E, 0x0032); - if (phy_type == NES_PHY_TYPE_KR) { - nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xF00F, 0x000C); - } else { - nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xF00F, 0x0002); - nes_write_10G_phy_reg(nesdev, phy_index, 0x1, 0xc314, 0x0063); - } - - /* reset serdes */ - sds = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0 + mac_index * 0x200); - sds |= 0x1; - nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0 + mac_index * 0x200, sds); - sds &= 0xfffffffe; - nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0 + mac_index * 0x200, sds); - - counter = 0; - while (((nes_read32(nesdev->regs + NES_SOFTWARE_RESET) & 0x00000040) != 0x00000040) - && (counter++ < 5000)) - ; - - return ret; -} - - -/** - * nes_init_phy - */ -int nes_init_phy(struct nes_device *nesdev) -{ - struct nes_adapter *nesadapter = nesdev->nesadapter; - u32 mac_index = nesdev->mac_index; - u32 tx_config = 0; - unsigned long flags; - u8 phy_type = nesadapter->phy_type[mac_index]; - u8 phy_index = nesadapter->phy_index[mac_index]; - int ret = 0; - - tx_config = nes_read_indexed(nesdev, NES_IDX_MAC_TX_CONFIG); - if (phy_type == NES_PHY_TYPE_1G) { - /* setup 1G MDIO operation */ - tx_config &= 0xFFFFFFE3; - tx_config |= 0x04; - } else { - /* setup 10G MDIO operation */ - tx_config &= 0xFFFFFFE3; - tx_config |= 0x1D; - } - nes_write_indexed(nesdev, NES_IDX_MAC_TX_CONFIG, tx_config); - - spin_lock_irqsave(&nesdev->nesadapter->phy_lock, flags); - - switch (phy_type) { - case NES_PHY_TYPE_1G: - ret = nes_init_1g_phy(nesdev, phy_type, phy_index); - break; - case NES_PHY_TYPE_ARGUS: - case NES_PHY_TYPE_SFP_D: - case NES_PHY_TYPE_KR: - ret = nes_init_2025_phy(nesdev, phy_type, phy_index); - break; - } - - spin_unlock_irqrestore(&nesdev->nesadapter->phy_lock, flags); - - return ret; -} - - -/** - * nes_replenish_nic_rq - */ -static void nes_replenish_nic_rq(struct nes_vnic *nesvnic) -{ - unsigned long flags; - dma_addr_t bus_address; - struct sk_buff *skb; - struct nes_hw_nic_rq_wqe *nic_rqe; - struct nes_hw_nic *nesnic; - struct nes_device *nesdev; - struct nes_rskb_cb *cb; - u32 rx_wqes_posted = 0; - - nesnic = &nesvnic->nic; - nesdev = nesvnic->nesdev; - spin_lock_irqsave(&nesnic->rq_lock, flags); - if (nesnic->replenishing_rq !=0) { - if (((nesnic->rq_size-1) == atomic_read(&nesvnic->rx_skbs_needed)) && - (atomic_read(&nesvnic->rx_skb_timer_running) == 0)) { - atomic_set(&nesvnic->rx_skb_timer_running, 1); - spin_unlock_irqrestore(&nesnic->rq_lock, flags); - nesvnic->rq_wqes_timer.expires = jiffies + (HZ/2); /* 1/2 second */ - add_timer(&nesvnic->rq_wqes_timer); - } else - spin_unlock_irqrestore(&nesnic->rq_lock, flags); - return; - } - nesnic->replenishing_rq = 1; - spin_unlock_irqrestore(&nesnic->rq_lock, flags); - do { - skb = dev_alloc_skb(nesvnic->max_frame_size); - if (skb) { - skb->dev = nesvnic->netdev; - - bus_address = pci_map_single(nesdev->pcidev, - skb->data, nesvnic->max_frame_size, PCI_DMA_FROMDEVICE); - cb = (struct nes_rskb_cb *)&skb->cb[0]; - cb->busaddr = bus_address; - cb->maplen = nesvnic->max_frame_size; - - nic_rqe = &nesnic->rq_vbase[nesvnic->nic.rq_head]; - nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_1_0_IDX] = - cpu_to_le32(nesvnic->max_frame_size); - nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_3_2_IDX] = 0; - nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_LOW_IDX] = - cpu_to_le32((u32)bus_address); - nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_HIGH_IDX] = - cpu_to_le32((u32)((u64)bus_address >> 32)); - nesnic->rx_skb[nesnic->rq_head] = skb; - nesnic->rq_head++; - nesnic->rq_head &= nesnic->rq_size - 1; - atomic_dec(&nesvnic->rx_skbs_needed); - barrier(); - if (++rx_wqes_posted == 255) { - nes_write32(nesdev->regs+NES_WQE_ALLOC, (rx_wqes_posted << 24) | nesnic->qp_id); - rx_wqes_posted = 0; - } - } else { - spin_lock_irqsave(&nesnic->rq_lock, flags); - if (((nesnic->rq_size-1) == atomic_read(&nesvnic->rx_skbs_needed)) && - (atomic_read(&nesvnic->rx_skb_timer_running) == 0)) { - atomic_set(&nesvnic->rx_skb_timer_running, 1); - spin_unlock_irqrestore(&nesnic->rq_lock, flags); - nesvnic->rq_wqes_timer.expires = jiffies + (HZ/2); /* 1/2 second */ - add_timer(&nesvnic->rq_wqes_timer); - } else - spin_unlock_irqrestore(&nesnic->rq_lock, flags); - break; - } - } while (atomic_read(&nesvnic->rx_skbs_needed)); - barrier(); - if (rx_wqes_posted) - nes_write32(nesdev->regs+NES_WQE_ALLOC, (rx_wqes_posted << 24) | nesnic->qp_id); - nesnic->replenishing_rq = 0; -} - - -/** - * nes_rq_wqes_timeout - */ -static void nes_rq_wqes_timeout(struct timer_list *t) -{ - struct nes_vnic *nesvnic = from_timer(nesvnic, t, rq_wqes_timer); - printk("%s: Timer fired.\n", __func__); - atomic_set(&nesvnic->rx_skb_timer_running, 0); - if (atomic_read(&nesvnic->rx_skbs_needed)) - nes_replenish_nic_rq(nesvnic); -} - - -/** - * nes_init_nic_qp - */ -int nes_init_nic_qp(struct nes_device *nesdev, struct net_device *netdev) -{ - struct nes_hw_cqp_wqe *cqp_wqe; - struct nes_hw_nic_sq_wqe *nic_sqe; - struct nes_hw_nic_qp_context *nic_context; - struct sk_buff *skb; - struct nes_hw_nic_rq_wqe *nic_rqe; - struct nes_vnic *nesvnic = netdev_priv(netdev); - unsigned long flags; - void *vmem; - dma_addr_t pmem; - u64 u64temp; - int ret; - u32 cqp_head; - u32 counter; - u32 wqe_count; - struct nes_rskb_cb *cb; - u8 jumbomode=0; - - /* Allocate fragment, SQ, RQ, and CQ; Reuse CEQ based on the PCI function */ - nesvnic->nic_mem_size = 256 + - (NES_NIC_WQ_SIZE * sizeof(struct nes_first_frag)) + - (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_sq_wqe)) + - (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_rq_wqe)) + - (NES_NIC_WQ_SIZE * 2 * sizeof(struct nes_hw_nic_cqe)) + - sizeof(struct nes_hw_nic_qp_context); - - nesvnic->nic_vbase = pci_zalloc_consistent(nesdev->pcidev, - nesvnic->nic_mem_size, - &nesvnic->nic_pbase); - if (!nesvnic->nic_vbase) { - nes_debug(NES_DBG_INIT, "Unable to allocate memory for NIC host descriptor rings\n"); - return -ENOMEM; - } - nes_debug(NES_DBG_INIT, "Allocated NIC QP structures at %p (phys = %016lX), size = %u.\n", - nesvnic->nic_vbase, (unsigned long)nesvnic->nic_pbase, nesvnic->nic_mem_size); - - vmem = (void *)(((unsigned long)nesvnic->nic_vbase + (256 - 1)) & - ~(unsigned long)(256 - 1)); - pmem = (dma_addr_t)(((unsigned long long)nesvnic->nic_pbase + (256 - 1)) & - ~(unsigned long long)(256 - 1)); - - /* Setup the first Fragment buffers */ - nesvnic->nic.first_frag_vbase = vmem; - - for (counter = 0; counter < NES_NIC_WQ_SIZE; counter++) { - nesvnic->nic.frag_paddr[counter] = pmem; - pmem += sizeof(struct nes_first_frag); - } - - /* setup the SQ */ - vmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_first_frag)); - - nesvnic->nic.sq_vbase = (void *)vmem; - nesvnic->nic.sq_pbase = pmem; - nesvnic->nic.sq_head = 0; - nesvnic->nic.sq_tail = 0; - nesvnic->nic.sq_size = NES_NIC_WQ_SIZE; - for (counter = 0; counter < NES_NIC_WQ_SIZE; counter++) { - nic_sqe = &nesvnic->nic.sq_vbase[counter]; - nic_sqe->wqe_words[NES_NIC_SQ_WQE_MISC_IDX] = - cpu_to_le32(NES_NIC_SQ_WQE_DISABLE_CHKSUM | - NES_NIC_SQ_WQE_COMPLETION); - nic_sqe->wqe_words[NES_NIC_SQ_WQE_LENGTH_0_TAG_IDX] = - cpu_to_le32((u32)NES_FIRST_FRAG_SIZE << 16); - nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_LOW_IDX] = - cpu_to_le32((u32)nesvnic->nic.frag_paddr[counter]); - nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_HIGH_IDX] = - cpu_to_le32((u32)((u64)nesvnic->nic.frag_paddr[counter] >> 32)); - } - - nesvnic->get_cqp_request = nes_get_cqp_request; - nesvnic->post_cqp_request = nes_post_cqp_request; - nesvnic->mcrq_mcast_filter = NULL; - - spin_lock_init(&nesvnic->nic.rq_lock); - - /* setup the RQ */ - vmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_sq_wqe)); - pmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_sq_wqe)); - - - nesvnic->nic.rq_vbase = vmem; - nesvnic->nic.rq_pbase = pmem; - nesvnic->nic.rq_head = 0; - nesvnic->nic.rq_tail = 0; - nesvnic->nic.rq_size = NES_NIC_WQ_SIZE; - - /* setup the CQ */ - vmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_rq_wqe)); - pmem += (NES_NIC_WQ_SIZE * sizeof(struct nes_hw_nic_rq_wqe)); - - if (nesdev->nesadapter->netdev_count > 2) - nesvnic->mcrq_qp_id = nesvnic->nic_index + 32; - else - nesvnic->mcrq_qp_id = nesvnic->nic.qp_id + 4; - - nesvnic->nic_cq.cq_vbase = vmem; - nesvnic->nic_cq.cq_pbase = pmem; - nesvnic->nic_cq.cq_head = 0; - nesvnic->nic_cq.cq_size = NES_NIC_WQ_SIZE * 2; - - nesvnic->nic_cq.ce_handler = nes_nic_napi_ce_handler; - - /* Send CreateCQ request to CQP */ - spin_lock_irqsave(&nesdev->cqp.lock, flags); - cqp_head = nesdev->cqp.sq_head; - - cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head]; - nes_fill_init_cqp_wqe(cqp_wqe, nesdev); - - cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32( - NES_CQP_CREATE_CQ | NES_CQP_CQ_CEQ_VALID | - ((u32)nesvnic->nic_cq.cq_size << 16)); - cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32( - nesvnic->nic_cq.cq_number | ((u32)nesdev->nic_ceq_index << 16)); - u64temp = (u64)nesvnic->nic_cq.cq_pbase; - set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp); - cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] = 0; - u64temp = (unsigned long)&nesvnic->nic_cq; - cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_LOW_IDX] = cpu_to_le32((u32)(u64temp >> 1)); - cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] = - cpu_to_le32(((u32)((u64temp) >> 33)) & 0x7FFFFFFF); - cqp_wqe->wqe_words[NES_CQP_CQ_WQE_DOORBELL_INDEX_HIGH_IDX] = 0; - if (++cqp_head >= nesdev->cqp.sq_size) - cqp_head = 0; - cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head]; - nes_fill_init_cqp_wqe(cqp_wqe, nesdev); - - /* Send CreateQP request to CQP */ - nic_context = (void *)(&nesvnic->nic_cq.cq_vbase[nesvnic->nic_cq.cq_size]); - nic_context->context_words[NES_NIC_CTX_MISC_IDX] = - cpu_to_le32((u32)NES_NIC_CTX_SIZE | - ((u32)PCI_FUNC(nesdev->pcidev->devfn) << 12)); - nes_debug(NES_DBG_INIT, "RX_WINDOW_BUFFER_PAGE_TABLE_SIZE = 0x%08X, RX_WINDOW_BUFFER_SIZE = 0x%08X\n", - nes_read_indexed(nesdev, NES_IDX_RX_WINDOW_BUFFER_PAGE_TABLE_SIZE), - nes_read_indexed(nesdev, NES_IDX_RX_WINDOW_BUFFER_SIZE)); - if (nes_read_indexed(nesdev, NES_IDX_RX_WINDOW_BUFFER_SIZE) != 0) { - nic_context->context_words[NES_NIC_CTX_MISC_IDX] |= cpu_to_le32(NES_NIC_BACK_STORE); - } - - u64temp = (u64)nesvnic->nic.sq_pbase; - nic_context->context_words[NES_NIC_CTX_SQ_LOW_IDX] = cpu_to_le32((u32)u64temp); - nic_context->context_words[NES_NIC_CTX_SQ_HIGH_IDX] = cpu_to_le32((u32)(u64temp >> 32)); - u64temp = (u64)nesvnic->nic.rq_pbase; - nic_context->context_words[NES_NIC_CTX_RQ_LOW_IDX] = cpu_to_le32((u32)u64temp); - nic_context->context_words[NES_NIC_CTX_RQ_HIGH_IDX] = cpu_to_le32((u32)(u64temp >> 32)); - - cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_CREATE_QP | - NES_CQP_QP_TYPE_NIC); - cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(nesvnic->nic.qp_id); - u64temp = (u64)nesvnic->nic_cq.cq_pbase + - (nesvnic->nic_cq.cq_size * sizeof(struct nes_hw_nic_cqe)); - set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_QP_WQE_CONTEXT_LOW_IDX, u64temp); - - if (++cqp_head >= nesdev->cqp.sq_size) - cqp_head = 0; - nesdev->cqp.sq_head = cqp_head; - - barrier(); - - /* Ring doorbell (2 WQEs) */ - nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x02800000 | nesdev->cqp.qp_id); - - spin_unlock_irqrestore(&nesdev->cqp.lock, flags); - nes_debug(NES_DBG_INIT, "Waiting for create NIC QP%u to complete.\n", - nesvnic->nic.qp_id); - - ret = wait_event_timeout(nesdev->cqp.waitq, (nesdev->cqp.sq_tail == cqp_head), - NES_EVENT_TIMEOUT); - nes_debug(NES_DBG_INIT, "Create NIC QP%u completed, wait_event_timeout ret = %u.\n", - nesvnic->nic.qp_id, ret); - if (!ret) { - nes_debug(NES_DBG_INIT, "NIC QP%u create timeout expired\n", nesvnic->nic.qp_id); - pci_free_consistent(nesdev->pcidev, nesvnic->nic_mem_size, nesvnic->nic_vbase, - nesvnic->nic_pbase); - return -EIO; - } - - /* Populate the RQ */ - for (counter = 0; counter < (NES_NIC_WQ_SIZE - 1); counter++) { - skb = dev_alloc_skb(nesvnic->max_frame_size); - if (!skb) { - nes_debug(NES_DBG_INIT, "%s: out of memory for receive skb\n", netdev->name); - - nes_destroy_nic_qp(nesvnic); - return -ENOMEM; - } - - skb->dev = netdev; - - pmem = pci_map_single(nesdev->pcidev, skb->data, - nesvnic->max_frame_size, PCI_DMA_FROMDEVICE); - cb = (struct nes_rskb_cb *)&skb->cb[0]; - cb->busaddr = pmem; - cb->maplen = nesvnic->max_frame_size; - - nic_rqe = &nesvnic->nic.rq_vbase[counter]; - nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_1_0_IDX] = cpu_to_le32(nesvnic->max_frame_size); - nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_3_2_IDX] = 0; - nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_LOW_IDX] = cpu_to_le32((u32)pmem); - nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_HIGH_IDX] = cpu_to_le32((u32)((u64)pmem >> 32)); - nesvnic->nic.rx_skb[counter] = skb; - } - - wqe_count = NES_NIC_WQ_SIZE - 1; - nesvnic->nic.rq_head = wqe_count; - barrier(); - do { - counter = min(wqe_count, ((u32)255)); - wqe_count -= counter; - nes_write32(nesdev->regs+NES_WQE_ALLOC, (counter << 24) | nesvnic->nic.qp_id); - } while (wqe_count); - timer_setup(&nesvnic->rq_wqes_timer, nes_rq_wqes_timeout, 0); - nes_debug(NES_DBG_INIT, "NAPI support Enabled\n"); - if (nesdev->nesadapter->et_use_adaptive_rx_coalesce) - { - nes_nic_init_timer(nesdev); - if (netdev->mtu > 1500) - jumbomode = 1; - nes_nic_init_timer_defaults(nesdev, jumbomode); - } - if ((nesdev->nesadapter->allow_unaligned_fpdus) && - (nes_init_mgt_qp(nesdev, netdev, nesvnic))) { - nes_debug(NES_DBG_INIT, "%s: Out of memory for pau nic\n", - netdev->name); - nes_destroy_nic_qp(nesvnic); - return -ENOMEM; - } - - return 0; -} - - -/** - * nes_destroy_nic_qp - */ -void nes_destroy_nic_qp(struct nes_vnic *nesvnic) -{ - u64 u64temp; - dma_addr_t bus_address; - struct nes_device *nesdev = nesvnic->nesdev; - struct nes_hw_cqp_wqe *cqp_wqe; - struct nes_hw_nic_sq_wqe *nic_sqe; - __le16 *wqe_fragment_length; - u16 wqe_fragment_index; - u32 cqp_head; - u32 wqm_cfg0; - unsigned long flags; - struct sk_buff *rx_skb; - struct nes_rskb_cb *cb; - int ret; - - if (nesdev->nesadapter->allow_unaligned_fpdus) - nes_destroy_mgt(nesvnic); - - /* clear wqe stall before destroying NIC QP */ - wqm_cfg0 = nes_read_indexed(nesdev, NES_IDX_WQM_CONFIG0); - nes_write_indexed(nesdev, NES_IDX_WQM_CONFIG0, wqm_cfg0 & 0xFFFF7FFF); - - /* Free remaining NIC receive buffers */ - while (nesvnic->nic.rq_head != nesvnic->nic.rq_tail) { - rx_skb = nesvnic->nic.rx_skb[nesvnic->nic.rq_tail]; - cb = (struct nes_rskb_cb *)&rx_skb->cb[0]; - pci_unmap_single(nesdev->pcidev, cb->busaddr, cb->maplen, - PCI_DMA_FROMDEVICE); - - dev_kfree_skb(nesvnic->nic.rx_skb[nesvnic->nic.rq_tail++]); - nesvnic->nic.rq_tail &= (nesvnic->nic.rq_size - 1); - } - - /* Free remaining NIC transmit buffers */ - while (nesvnic->nic.sq_head != nesvnic->nic.sq_tail) { - nic_sqe = &nesvnic->nic.sq_vbase[nesvnic->nic.sq_tail]; - wqe_fragment_index = 1; - wqe_fragment_length = (__le16 *) - &nic_sqe->wqe_words[NES_NIC_SQ_WQE_LENGTH_0_TAG_IDX]; - /* bump past the vlan tag */ - wqe_fragment_length++; - if (le16_to_cpu(wqe_fragment_length[wqe_fragment_index]) != 0) { - u64temp = (u64)le32_to_cpu( - nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_LOW_IDX+ - wqe_fragment_index*2]); - u64temp += ((u64)le32_to_cpu( - nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_HIGH_IDX - + wqe_fragment_index*2]))<<32; - bus_address = (dma_addr_t)u64temp; - if (test_and_clear_bit(nesvnic->nic.sq_tail, - nesvnic->nic.first_frag_overflow)) { - pci_unmap_single(nesdev->pcidev, - bus_address, - le16_to_cpu(wqe_fragment_length[ - wqe_fragment_index++]), - PCI_DMA_TODEVICE); - } - for (; wqe_fragment_index < 5; wqe_fragment_index++) { - if (wqe_fragment_length[wqe_fragment_index]) { - u64temp = le32_to_cpu( - nic_sqe->wqe_words[ - NES_NIC_SQ_WQE_FRAG0_LOW_IDX+ - wqe_fragment_index*2]); - u64temp += ((u64)le32_to_cpu( - nic_sqe->wqe_words[ - NES_NIC_SQ_WQE_FRAG0_HIGH_IDX+ - wqe_fragment_index*2]))<<32; - bus_address = (dma_addr_t)u64temp; - pci_unmap_page(nesdev->pcidev, - bus_address, - le16_to_cpu( - wqe_fragment_length[ - wqe_fragment_index]), - PCI_DMA_TODEVICE); - } else - break; - } - } - if (nesvnic->nic.tx_skb[nesvnic->nic.sq_tail]) - dev_kfree_skb( - nesvnic->nic.tx_skb[nesvnic->nic.sq_tail]); - - nesvnic->nic.sq_tail = (nesvnic->nic.sq_tail + 1) - & (nesvnic->nic.sq_size - 1); - } - - spin_lock_irqsave(&nesdev->cqp.lock, flags); - - /* Destroy NIC QP */ - cqp_head = nesdev->cqp.sq_head; - cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head]; - nes_fill_init_cqp_wqe(cqp_wqe, nesdev); - - set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX, - (NES_CQP_DESTROY_QP | NES_CQP_QP_TYPE_NIC)); - set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX, - nesvnic->nic.qp_id); - - if (++cqp_head >= nesdev->cqp.sq_size) - cqp_head = 0; - - cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head]; - - /* Destroy NIC CQ */ - nes_fill_init_cqp_wqe(cqp_wqe, nesdev); - set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX, - (NES_CQP_DESTROY_CQ | ((u32)nesvnic->nic_cq.cq_size << 16))); - set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX, - (nesvnic->nic_cq.cq_number | ((u32)nesdev->nic_ceq_index << 16))); - - if (++cqp_head >= nesdev->cqp.sq_size) - cqp_head = 0; - - nesdev->cqp.sq_head = cqp_head; - barrier(); - - /* Ring doorbell (2 WQEs) */ - nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x02800000 | nesdev->cqp.qp_id); - - spin_unlock_irqrestore(&nesdev->cqp.lock, flags); - nes_debug(NES_DBG_SHUTDOWN, "Waiting for CQP, cqp_head=%u, cqp.sq_head=%u," - " cqp.sq_tail=%u, cqp.sq_size=%u\n", - cqp_head, nesdev->cqp.sq_head, - nesdev->cqp.sq_tail, nesdev->cqp.sq_size); - - ret = wait_event_timeout(nesdev->cqp.waitq, (nesdev->cqp.sq_tail == cqp_head), - NES_EVENT_TIMEOUT); - - nes_debug(NES_DBG_SHUTDOWN, "Destroy NIC QP returned, wait_event_timeout ret = %u, cqp_head=%u," - " cqp.sq_head=%u, cqp.sq_tail=%u\n", - ret, cqp_head, nesdev->cqp.sq_head, nesdev->cqp.sq_tail); - if (!ret) { - nes_debug(NES_DBG_SHUTDOWN, "NIC QP%u destroy timeout expired\n", - nesvnic->nic.qp_id); - } - - pci_free_consistent(nesdev->pcidev, nesvnic->nic_mem_size, nesvnic->nic_vbase, - nesvnic->nic_pbase); - - /* restore old wqm_cfg0 value */ - nes_write_indexed(nesdev, NES_IDX_WQM_CONFIG0, wqm_cfg0); -} - -/** - * nes_napi_isr - */ -int nes_napi_isr(struct nes_device *nesdev) -{ - struct nes_adapter *nesadapter = nesdev->nesadapter; - u32 int_stat; - - if (nesdev->napi_isr_ran) { - /* interrupt status has already been read in ISR */ - int_stat = nesdev->int_stat; - } else { - int_stat = nes_read32(nesdev->regs + NES_INT_STAT); - nesdev->int_stat = int_stat; - nesdev->napi_isr_ran = 1; - } - - int_stat &= nesdev->int_req; - /* iff NIC, process here, else wait for DPC */ - if ((int_stat) && ((int_stat & 0x0000ff00) == int_stat)) { - nesdev->napi_isr_ran = 0; - nes_write32(nesdev->regs + NES_INT_STAT, - (int_stat & - ~(NES_INT_INTF | NES_INT_TIMER | NES_INT_MAC0 | NES_INT_MAC1 | NES_INT_MAC2 | NES_INT_MAC3))); - - /* Process the CEQs */ - nes_process_ceq(nesdev, &nesdev->nesadapter->ceq[nesdev->nic_ceq_index]); - - if (unlikely((((nesadapter->et_rx_coalesce_usecs_irq) && - (!nesadapter->et_use_adaptive_rx_coalesce)) || - ((nesadapter->et_use_adaptive_rx_coalesce) && - (nesdev->deepcq_count > nesadapter->et_pkt_rate_low))))) { - if ((nesdev->int_req & NES_INT_TIMER) == 0) { - /* Enable Periodic timer interrupts */ - nesdev->int_req |= NES_INT_TIMER; - /* ack any pending periodic timer interrupts so we don't get an immediate interrupt */ - /* TODO: need to also ack other unused periodic timer values, get from nesadapter */ - nes_write32(nesdev->regs+NES_TIMER_STAT, - nesdev->timer_int_req | ~(nesdev->nesadapter->timer_int_req)); - nes_write32(nesdev->regs+NES_INTF_INT_MASK, - ~(nesdev->intf_int_req | NES_INTF_PERIODIC_TIMER)); - } - - if (unlikely(nesadapter->et_use_adaptive_rx_coalesce)) - { - nes_nic_init_timer(nesdev); - } - /* Enable interrupts, except CEQs */ - nes_write32(nesdev->regs+NES_INT_MASK, 0x0000ffff | (~nesdev->int_req)); - } else { - /* Enable interrupts, make sure timer is off */ - nesdev->int_req &= ~NES_INT_TIMER; - nes_write32(nesdev->regs+NES_INTF_INT_MASK, ~(nesdev->intf_int_req)); - nes_write32(nesdev->regs+NES_INT_MASK, ~nesdev->int_req); - } - nesdev->deepcq_count = 0; - return 1; - } else { - return 0; - } -} - -static void process_critical_error(struct nes_device *nesdev) -{ - u32 debug_error; - u32 nes_idx_debug_error_masks0 = 0; - u16 error_module = 0; - - debug_error = nes_read_indexed(nesdev, NES_IDX_DEBUG_ERROR_CONTROL_STATUS); - printk(KERN_ERR PFX "Critical Error reported by device!!! 0x%02X\n", - (u16)debug_error); - nes_write_indexed(nesdev, NES_IDX_DEBUG_ERROR_CONTROL_STATUS, - 0x01010000 | (debug_error & 0x0000ffff)); - if (crit_err_count++ > 10) - nes_write_indexed(nesdev, NES_IDX_DEBUG_ERROR_MASKS1, 1 << 0x17); - error_module = (u16) (debug_error & 0x1F00) >> 8; - if (++nesdev->nesadapter->crit_error_count[error_module-1] >= - nes_max_critical_error_count) { - printk(KERN_ERR PFX "Masking off critical error for module " - "0x%02X\n", (u16)error_module); - nes_idx_debug_error_masks0 = nes_read_indexed(nesdev, - NES_IDX_DEBUG_ERROR_MASKS0); - nes_write_indexed(nesdev, NES_IDX_DEBUG_ERROR_MASKS0, - nes_idx_debug_error_masks0 | (1 << error_module)); - } -} -/** - * nes_dpc - */ -void nes_dpc(unsigned long param) -{ - struct nes_device *nesdev = (struct nes_device *)param; - struct nes_adapter *nesadapter = nesdev->nesadapter; - u32 counter; - u32 loop_counter = 0; - u32 int_status_bit; - u32 int_stat; - u32 timer_stat; - u32 temp_int_stat; - u32 intf_int_stat; - u32 processed_intf_int = 0; - u16 processed_timer_int = 0; - u16 completion_ints = 0; - u16 timer_ints = 0; - - /* nes_debug(NES_DBG_ISR, "\n"); */ - - do { - timer_stat = 0; - if (nesdev->napi_isr_ran) { - nesdev->napi_isr_ran = 0; - int_stat = nesdev->int_stat; - } else - int_stat = nes_read32(nesdev->regs+NES_INT_STAT); - if (processed_intf_int != 0) - int_stat &= nesdev->int_req & ~NES_INT_INTF; - else - int_stat &= nesdev->int_req; - if (processed_timer_int == 0) { - processed_timer_int = 1; - if (int_stat & NES_INT_TIMER) { - timer_stat = nes_read32(nesdev->regs + NES_TIMER_STAT); - if ((timer_stat & nesdev->timer_int_req) == 0) { - int_stat &= ~NES_INT_TIMER; - } - } - } else { - int_stat &= ~NES_INT_TIMER; - } - - if (int_stat) { - if (int_stat & ~(NES_INT_INTF | NES_INT_TIMER | NES_INT_MAC0| - NES_INT_MAC1|NES_INT_MAC2 | NES_INT_MAC3)) { - /* Ack the interrupts */ - nes_write32(nesdev->regs+NES_INT_STAT, - (int_stat & ~(NES_INT_INTF | NES_INT_TIMER | NES_INT_MAC0| - NES_INT_MAC1 | NES_INT_MAC2 | NES_INT_MAC3))); - } - - temp_int_stat = int_stat; - for (counter = 0, int_status_bit = 1; counter < 16; counter++) { - if (int_stat & int_status_bit) { - nes_process_ceq(nesdev, &nesadapter->ceq[counter]); - temp_int_stat &= ~int_status_bit; - completion_ints = 1; - } - if (!(temp_int_stat & 0x0000ffff)) - break; - int_status_bit <<= 1; - } - - /* Process the AEQ for this pci function */ - int_status_bit = 1 << (16 + PCI_FUNC(nesdev->pcidev->devfn)); - if (int_stat & int_status_bit) { - nes_process_aeq(nesdev, &nesadapter->aeq[PCI_FUNC(nesdev->pcidev->devfn)]); - } - - /* Process the MAC interrupt for this pci function */ - int_status_bit = 1 << (24 + nesdev->mac_index); - if (int_stat & int_status_bit) { - nes_process_mac_intr(nesdev, nesdev->mac_index); - } - - if (int_stat & NES_INT_TIMER) { - if (timer_stat & nesdev->timer_int_req) { - nes_write32(nesdev->regs + NES_TIMER_STAT, - (timer_stat & nesdev->timer_int_req) | - ~(nesdev->nesadapter->timer_int_req)); - timer_ints = 1; - } - } - - if (int_stat & NES_INT_INTF) { - processed_intf_int = 1; - intf_int_stat = nes_read32(nesdev->regs+NES_INTF_INT_STAT); - intf_int_stat &= nesdev->intf_int_req; - if (NES_INTF_INT_CRITERR & intf_int_stat) { - process_critical_error(nesdev); - } - if (NES_INTF_INT_PCIERR & intf_int_stat) { - printk(KERN_ERR PFX "PCI Error reported by device!!!\n"); - BUG(); - } - if (NES_INTF_INT_AEQ_OFLOW & intf_int_stat) { - printk(KERN_ERR PFX "AEQ Overflow reported by device!!!\n"); - BUG(); - } - nes_write32(nesdev->regs+NES_INTF_INT_STAT, intf_int_stat); - } - - if (int_stat & NES_INT_TSW) { - } - } - /* Don't use the interface interrupt bit stay in loop */ - int_stat &= ~NES_INT_INTF | NES_INT_TIMER | NES_INT_MAC0 | - NES_INT_MAC1 | NES_INT_MAC2 | NES_INT_MAC3; - } while ((int_stat != 0) && (loop_counter++ < MAX_DPC_ITERATIONS)); - - if (timer_ints == 1) { - if ((nesadapter->et_rx_coalesce_usecs_irq) || (nesadapter->et_use_adaptive_rx_coalesce)) { - if (completion_ints == 0) { - nesdev->timer_only_int_count++; - if (nesdev->timer_only_int_count>=nesadapter->timer_int_limit) { - nesdev->timer_only_int_count = 0; - nesdev->int_req &= ~NES_INT_TIMER; - nes_write32(nesdev->regs + NES_INTF_INT_MASK, ~(nesdev->intf_int_req)); - nes_write32(nesdev->regs + NES_INT_MASK, ~nesdev->int_req); - } else { - nes_write32(nesdev->regs+NES_INT_MASK, 0x0000ffff | (~nesdev->int_req)); - } - } else { - if (unlikely(nesadapter->et_use_adaptive_rx_coalesce)) - { - nes_nic_init_timer(nesdev); - } - nesdev->timer_only_int_count = 0; - nes_write32(nesdev->regs+NES_INT_MASK, 0x0000ffff | (~nesdev->int_req)); - } - } else { - nesdev->timer_only_int_count = 0; - nesdev->int_req &= ~NES_INT_TIMER; - nes_write32(nesdev->regs+NES_INTF_INT_MASK, ~(nesdev->intf_int_req)); - nes_write32(nesdev->regs+NES_TIMER_STAT, - nesdev->timer_int_req | ~(nesdev->nesadapter->timer_int_req)); - nes_write32(nesdev->regs+NES_INT_MASK, ~nesdev->int_req); - } - } else { - if ( (completion_ints == 1) && - (((nesadapter->et_rx_coalesce_usecs_irq) && - (!nesadapter->et_use_adaptive_rx_coalesce)) || - ((nesdev->deepcq_count > nesadapter->et_pkt_rate_low) && - (nesadapter->et_use_adaptive_rx_coalesce) )) ) { - /* nes_debug(NES_DBG_ISR, "Enabling periodic timer interrupt.\n" ); */ - nesdev->timer_only_int_count = 0; - nesdev->int_req |= NES_INT_TIMER; - nes_write32(nesdev->regs+NES_TIMER_STAT, - nesdev->timer_int_req | ~(nesdev->nesadapter->timer_int_req)); - nes_write32(nesdev->regs+NES_INTF_INT_MASK, - ~(nesdev->intf_int_req | NES_INTF_PERIODIC_TIMER)); - nes_write32(nesdev->regs+NES_INT_MASK, 0x0000ffff | (~nesdev->int_req)); - } else { - nes_write32(nesdev->regs+NES_INT_MASK, ~nesdev->int_req); - } - } - nesdev->deepcq_count = 0; -} - - -/** - * nes_process_ceq - */ -static void nes_process_ceq(struct nes_device *nesdev, struct nes_hw_ceq *ceq) -{ - u64 u64temp; - struct nes_hw_cq *cq; - u32 head; - u32 ceq_size; - - /* nes_debug(NES_DBG_CQ, "\n"); */ - head = ceq->ceq_head; - ceq_size = ceq->ceq_size; - - do { - if (le32_to_cpu(ceq->ceq_vbase[head].ceqe_words[NES_CEQE_CQ_CTX_HIGH_IDX]) & - NES_CEQE_VALID) { - u64temp = (((u64)(le32_to_cpu(ceq->ceq_vbase[head].ceqe_words[NES_CEQE_CQ_CTX_HIGH_IDX]))) << 32) | - ((u64)(le32_to_cpu(ceq->ceq_vbase[head].ceqe_words[NES_CEQE_CQ_CTX_LOW_IDX]))); - u64temp <<= 1; - cq = *((struct nes_hw_cq **)&u64temp); - /* nes_debug(NES_DBG_CQ, "pCQ = %p\n", cq); */ - barrier(); - ceq->ceq_vbase[head].ceqe_words[NES_CEQE_CQ_CTX_HIGH_IDX] = 0; - - /* call the event handler */ - cq->ce_handler(nesdev, cq); - - if (++head >= ceq_size) - head = 0; - } else { - break; - } - - } while (1); - - ceq->ceq_head = head; -} - - -/** - * nes_process_aeq - */ -static void nes_process_aeq(struct nes_device *nesdev, struct nes_hw_aeq *aeq) -{ - /* u64 u64temp; */ - u32 head; - u32 aeq_size; - u32 aeqe_misc; - u32 aeqe_cq_id; - struct nes_hw_aeqe volatile *aeqe; - - head = aeq->aeq_head; - aeq_size = aeq->aeq_size; - - do { - aeqe = &aeq->aeq_vbase[head]; - if ((le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]) & NES_AEQE_VALID) == 0) - break; - aeqe_misc = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]); - aeqe_cq_id = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]); - if (aeqe_misc & (NES_AEQE_QP|NES_AEQE_CQ)) { - if (aeqe_cq_id >= NES_FIRST_QPN) { - /* dealing with an accelerated QP related AE */ - /* - * u64temp = (((u64)(le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_HIGH_IDX]))) << 32) | - * ((u64)(le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_LOW_IDX]))); - */ - nes_process_iwarp_aeqe(nesdev, (struct nes_hw_aeqe *)aeqe); - } else { - /* TODO: dealing with a CQP related AE */ - nes_debug(NES_DBG_AEQ, "Processing CQP related AE, misc = 0x%04X\n", - (u16)(aeqe_misc >> 16)); - } - } - - aeqe->aeqe_words[NES_AEQE_MISC_IDX] = 0; - - if (++head >= aeq_size) - head = 0; - - nes_write32(nesdev->regs + NES_AEQ_ALLOC, 1 << 16); - } - while (1); - aeq->aeq_head = head; -} - -static void nes_reset_link(struct nes_device *nesdev, u32 mac_index) -{ - struct nes_adapter *nesadapter = nesdev->nesadapter; - u32 reset_value; - u32 i=0; - u32 u32temp; - - if (nesadapter->hw_rev == NE020_REV) { - return; - } - mh_detected++; - - reset_value = nes_read32(nesdev->regs+NES_SOFTWARE_RESET); - - if ((mac_index == 0) || ((mac_index == 1) && (nesadapter->OneG_Mode))) - reset_value |= 0x0000001d; - else - reset_value |= 0x0000002d; - - if (4 <= (nesadapter->link_interrupt_count[mac_index] / ((u16)NES_MAX_LINK_INTERRUPTS))) { - if ((!nesadapter->OneG_Mode) && (nesadapter->port_count == 2)) { - nesadapter->link_interrupt_count[0] = 0; - nesadapter->link_interrupt_count[1] = 0; - u32temp = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1); - if (0x00000040 & u32temp) - nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F088); - else - nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F0C8); - - reset_value |= 0x0000003d; - } - nesadapter->link_interrupt_count[mac_index] = 0; - } - - nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value); - - while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET) - & 0x00000040) != 0x00000040) && (i++ < 5000)); - - if (0x0000003d == (reset_value & 0x0000003d)) { - u32 pcs_control_status0, pcs_control_status1; - - for (i = 0; i < 10; i++) { - pcs_control_status0 = nes_read_indexed(nesdev, NES_IDX_PHY_PCS_CONTROL_STATUS0); - pcs_control_status1 = nes_read_indexed(nesdev, NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200); - if (((0x0F000000 == (pcs_control_status0 & 0x0F000000)) - && (pcs_control_status0 & 0x00100000)) - || ((0x0F000000 == (pcs_control_status1 & 0x0F000000)) - && (pcs_control_status1 & 0x00100000))) - continue; - else - break; - } - if (10 == i) { - u32temp = nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1); - if (0x00000040 & u32temp) - nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F088); - else - nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL1, 0x0000F0C8); - - nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value); - - while (((nes_read32(nesdev->regs + NES_SOFTWARE_RESET) - & 0x00000040) != 0x00000040) && (i++ < 5000)); - } - } -} - -/** - * nes_process_mac_intr - */ -static void nes_process_mac_intr(struct nes_device *nesdev, u32 mac_number) -{ - unsigned long flags; - u32 pcs_control_status; - struct nes_adapter *nesadapter = nesdev->nesadapter; - struct nes_vnic *nesvnic; - u32 mac_status; - u32 mac_index = nesdev->mac_index; - u32 u32temp; - u16 phy_data; - u16 temp_phy_data; - u32 pcs_val = 0x0f0f0000; - u32 pcs_mask = 0x0f1f0000; - u32 cdr_ctrl; - - spin_lock_irqsave(&nesadapter->phy_lock, flags); - if (nesadapter->mac_sw_state[mac_number] != NES_MAC_SW_IDLE) { - spin_unlock_irqrestore(&nesadapter->phy_lock, flags); - return; - } - nesadapter->mac_sw_state[mac_number] = NES_MAC_SW_INTERRUPT; - - /* ack the MAC interrupt */ - mac_status = nes_read_indexed(nesdev, NES_IDX_MAC_INT_STATUS + (mac_index * 0x200)); - /* Clear the interrupt */ - nes_write_indexed(nesdev, NES_IDX_MAC_INT_STATUS + (mac_index * 0x200), mac_status); - - nes_debug(NES_DBG_PHY, "MAC%u interrupt status = 0x%X.\n", mac_number, mac_status); - - if (mac_status & (NES_MAC_INT_LINK_STAT_CHG | NES_MAC_INT_XGMII_EXT)) { - nesdev->link_status_interrupts++; - if (0 == (++nesadapter->link_interrupt_count[mac_index] % ((u16)NES_MAX_LINK_INTERRUPTS))) - nes_reset_link(nesdev, mac_index); - - /* read the PHY interrupt status register */ - if ((nesadapter->OneG_Mode) && - (nesadapter->phy_type[mac_index] != NES_PHY_TYPE_PUMA_1G)) { - do { - nes_read_1G_phy_reg(nesdev, 0x1a, - nesadapter->phy_index[mac_index], &phy_data); - nes_debug(NES_DBG_PHY, "Phy%d data from register 0x1a = 0x%X.\n", - nesadapter->phy_index[mac_index], phy_data); - } while (phy_data&0x8000); - - temp_phy_data = 0; - do { - nes_read_1G_phy_reg(nesdev, 0x11, - nesadapter->phy_index[mac_index], &phy_data); - nes_debug(NES_DBG_PHY, "Phy%d data from register 0x11 = 0x%X.\n", - nesadapter->phy_index[mac_index], phy_data); - if (temp_phy_data == phy_data) - break; - temp_phy_data = phy_data; - } while (1); - - nes_read_1G_phy_reg(nesdev, 0x1e, - nesadapter->phy_index[mac_index], &phy_data); - nes_debug(NES_DBG_PHY, "Phy%d data from register 0x1e = 0x%X.\n", - nesadapter->phy_index[mac_index], phy_data); - - nes_read_1G_phy_reg(nesdev, 1, - nesadapter->phy_index[mac_index], &phy_data); - nes_debug(NES_DBG_PHY, "1G phy%u data from register 1 = 0x%X\n", - nesadapter->phy_index[mac_index], phy_data); - - if (temp_phy_data & 0x1000) { - nes_debug(NES_DBG_PHY, "The Link is up according to the PHY\n"); - phy_data = 4; - } else { - nes_debug(NES_DBG_PHY, "The Link is down according to the PHY\n"); - } - } - nes_debug(NES_DBG_PHY, "Eth SERDES Common Status: 0=0x%08X, 1=0x%08X\n", - nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0), - nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0+0x200)); - - if (nesadapter->phy_type[mac_index] == NES_PHY_TYPE_PUMA_1G) { - switch (mac_index) { - case 1: - case 3: - pcs_control_status = nes_read_indexed(nesdev, - NES_IDX_PHY_PCS_CONTROL_STATUS0 + 0x200); - break; - default: - pcs_control_status = nes_read_indexed(nesdev, - NES_IDX_PHY_PCS_CONTROL_STATUS0); - break; - } - } else { - pcs_control_status = nes_read_indexed(nesdev, - NES_IDX_PHY_PCS_CONTROL_STATUS0 + ((mac_index & 1) * 0x200)); - pcs_control_status = nes_read_indexed(nesdev, - NES_IDX_PHY_PCS_CONTROL_STATUS0 + ((mac_index & 1) * 0x200)); - } - - nes_debug(NES_DBG_PHY, "PCS PHY Control/Status%u: 0x%08X\n", - mac_index, pcs_control_status); - if ((nesadapter->OneG_Mode) && - (nesadapter->phy_type[mac_index] != NES_PHY_TYPE_PUMA_1G)) { - u32temp = 0x01010000; - if (nesadapter->port_count > 2) { - u32temp |= 0x02020000; - } - if ((pcs_control_status & u32temp)!= u32temp) { - phy_data = 0; - nes_debug(NES_DBG_PHY, "PCS says the link is down\n"); - } - } else { - switch (nesadapter->phy_type[mac_index]) { - case NES_PHY_TYPE_ARGUS: - case NES_PHY_TYPE_SFP_D: - case NES_PHY_TYPE_KR: - /* clear the alarms */ - nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0x0008); - nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0xc001); - nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0xc002); - nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0xc005); - nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 4, 0xc006); - nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 0x9003); - nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 0x9004); - nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 0x9005); - /* check link status */ - nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 0x9003); - temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL); - - nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 3, 0x0021); - nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL); - nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 3, 0x0021); - phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL); - - phy_data = (!temp_phy_data && (phy_data == 0x8000)) ? 0x4 : 0x0; - - nes_debug(NES_DBG_PHY, "%s: Phy data = 0x%04X, link was %s.\n", - __func__, phy_data, nesadapter->mac_link_down[mac_index] ? "DOWN" : "UP"); - break; - - case NES_PHY_TYPE_PUMA_1G: - if (mac_index < 2) - pcs_val = pcs_mask = 0x01010000; - else - pcs_val = pcs_mask = 0x02020000; - /* fall through */ - default: - phy_data = (pcs_val == (pcs_control_status & pcs_mask)) ? 0x4 : 0x0; - break; - } - } - - if (phy_data & 0x0004) { - if (wide_ppm_offset && - (nesadapter->phy_type[mac_index] == NES_PHY_TYPE_CX4) && - (nesadapter->hw_rev != NE020_REV)) { - cdr_ctrl = nes_read_indexed(nesdev, - NES_IDX_ETH_SERDES_CDR_CONTROL0 + - mac_index * 0x200); - nes_write_indexed(nesdev, - NES_IDX_ETH_SERDES_CDR_CONTROL0 + - mac_index * 0x200, - cdr_ctrl | 0x000F0000); - } - nesadapter->mac_link_down[mac_index] = 0; - list_for_each_entry(nesvnic, &nesadapter->nesvnic_list[mac_index], list) { - nes_debug(NES_DBG_PHY, "The Link is UP!!. linkup was %d\n", - nesvnic->linkup); - if (nesvnic->linkup == 0) { - printk(PFX "The Link is now up for port %s, netdev %p.\n", - nesvnic->netdev->name, nesvnic->netdev); - if (netif_queue_stopped(nesvnic->netdev)) - netif_start_queue(nesvnic->netdev); - nesvnic->linkup = 1; - netif_carrier_on(nesvnic->netdev); - - spin_lock(&nesvnic->port_ibevent_lock); - if (nesvnic->of_device_registered) { - if (nesdev->iw_status == 0) { - nesdev->iw_status = 1; - nes_port_ibevent(nesvnic); - } - } - spin_unlock(&nesvnic->port_ibevent_lock); - } - } - } else { - if (wide_ppm_offset && - (nesadapter->phy_type[mac_index] == NES_PHY_TYPE_CX4) && - (nesadapter->hw_rev != NE020_REV)) { - cdr_ctrl = nes_read_indexed(nesdev, - NES_IDX_ETH_SERDES_CDR_CONTROL0 + - mac_index * 0x200); - nes_write_indexed(nesdev, - NES_IDX_ETH_SERDES_CDR_CONTROL0 + - mac_index * 0x200, - cdr_ctrl & 0xFFF0FFFF); - } - nesadapter->mac_link_down[mac_index] = 1; - list_for_each_entry(nesvnic, &nesadapter->nesvnic_list[mac_index], list) { - nes_debug(NES_DBG_PHY, "The Link is Down!!. linkup was %d\n", - nesvnic->linkup); - if (nesvnic->linkup == 1) { - printk(PFX "The Link is now down for port %s, netdev %p.\n", - nesvnic->netdev->name, nesvnic->netdev); - if (!(netif_queue_stopped(nesvnic->netdev))) - netif_stop_queue(nesvnic->netdev); - nesvnic->linkup = 0; - netif_carrier_off(nesvnic->netdev); - - spin_lock(&nesvnic->port_ibevent_lock); - if (nesvnic->of_device_registered) { - if (nesdev->iw_status == 1) { - nesdev->iw_status = 0; - nes_port_ibevent(nesvnic); - } - } - spin_unlock(&nesvnic->port_ibevent_lock); - } - } - } - if (nesadapter->phy_type[mac_index] == NES_PHY_TYPE_SFP_D) { - nesdev->link_recheck = 1; - mod_delayed_work(system_wq, &nesdev->work, - NES_LINK_RECHECK_DELAY); - } - } - - spin_unlock_irqrestore(&nesadapter->phy_lock, flags); - - nesadapter->mac_sw_state[mac_number] = NES_MAC_SW_IDLE; -} - -void nes_recheck_link_status(struct work_struct *work) -{ - unsigned long flags; - struct nes_device *nesdev = container_of(work, struct nes_device, work.work); - struct nes_adapter *nesadapter = nesdev->nesadapter; - struct nes_vnic *nesvnic; - u32 mac_index = nesdev->mac_index; - u16 phy_data; - u16 temp_phy_data; - - spin_lock_irqsave(&nesadapter->phy_lock, flags); - - /* check link status */ - nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 1, 0x9003); - temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL); - - nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 3, 0x0021); - nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL); - nes_read_10G_phy_reg(nesdev, nesadapter->phy_index[mac_index], 3, 0x0021); - phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL); - - phy_data = (!temp_phy_data && (phy_data == 0x8000)) ? 0x4 : 0x0; - - nes_debug(NES_DBG_PHY, "%s: Phy data = 0x%04X, link was %s.\n", - __func__, phy_data, - nesadapter->mac_link_down[mac_index] ? "DOWN" : "UP"); - - if (phy_data & 0x0004) { - nesadapter->mac_link_down[mac_index] = 0; - list_for_each_entry(nesvnic, &nesadapter->nesvnic_list[mac_index], list) { - if (nesvnic->linkup == 0) { - printk(PFX "The Link is now up for port %s, netdev %p.\n", - nesvnic->netdev->name, nesvnic->netdev); - if (netif_queue_stopped(nesvnic->netdev)) - netif_start_queue(nesvnic->netdev); - nesvnic->linkup = 1; - netif_carrier_on(nesvnic->netdev); - - spin_lock(&nesvnic->port_ibevent_lock); - if (nesvnic->of_device_registered) { - if (nesdev->iw_status == 0) { - nesdev->iw_status = 1; - nes_port_ibevent(nesvnic); - } - } - spin_unlock(&nesvnic->port_ibevent_lock); - } - } - - } else { - nesadapter->mac_link_down[mac_index] = 1; - list_for_each_entry(nesvnic, &nesadapter->nesvnic_list[mac_index], list) { - if (nesvnic->linkup == 1) { - printk(PFX "The Link is now down for port %s, netdev %p.\n", - nesvnic->netdev->name, nesvnic->netdev); - if (!(netif_queue_stopped(nesvnic->netdev))) - netif_stop_queue(nesvnic->netdev); - nesvnic->linkup = 0; - netif_carrier_off(nesvnic->netdev); - - spin_lock(&nesvnic->port_ibevent_lock); - if (nesvnic->of_device_registered) { - if (nesdev->iw_status == 1) { - nesdev->iw_status = 0; - nes_port_ibevent(nesvnic); - } - } - spin_unlock(&nesvnic->port_ibevent_lock); - } - } - } - if (nesdev->link_recheck++ < NES_LINK_RECHECK_MAX) - schedule_delayed_work(&nesdev->work, NES_LINK_RECHECK_DELAY); - else - nesdev->link_recheck = 0; - - spin_unlock_irqrestore(&nesadapter->phy_lock, flags); -} - - -static void nes_nic_napi_ce_handler(struct nes_device *nesdev, struct nes_hw_nic_cq *cq) -{ - struct nes_vnic *nesvnic = container_of(cq, struct nes_vnic, nic_cq); - - napi_schedule(&nesvnic->napi); -} - - -/* The MAX_RQES_TO_PROCESS defines how many max read requests to complete before -* getting out of nic_ce_handler -*/ -#define MAX_RQES_TO_PROCESS 384 - -/** - * nes_nic_ce_handler - */ -void nes_nic_ce_handler(struct nes_device *nesdev, struct nes_hw_nic_cq *cq) -{ - u64 u64temp; - dma_addr_t bus_address; - struct nes_hw_nic *nesnic; - struct nes_vnic *nesvnic = container_of(cq, struct nes_vnic, nic_cq); - struct nes_adapter *nesadapter = nesdev->nesadapter; - struct nes_hw_nic_rq_wqe *nic_rqe; - struct nes_hw_nic_sq_wqe *nic_sqe; - struct sk_buff *skb; - struct sk_buff *rx_skb; - struct nes_rskb_cb *cb; - __le16 *wqe_fragment_length; - u32 head; - u32 cq_size; - u32 rx_pkt_size; - u32 cqe_count=0; - u32 cqe_errv; - u32 cqe_misc; - u16 wqe_fragment_index = 1; /* first fragment (0) is used by copy buffer */ - u16 vlan_tag; - u16 pkt_type; - u16 rqes_processed = 0; - u8 sq_cqes = 0; - - head = cq->cq_head; - cq_size = cq->cq_size; - cq->cqes_pending = 1; - do { - if (le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_NIC_CQE_MISC_IDX]) & - NES_NIC_CQE_VALID) { - nesnic = &nesvnic->nic; - cqe_misc = le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_NIC_CQE_MISC_IDX]); - if (cqe_misc & NES_NIC_CQE_SQ) { - sq_cqes++; - wqe_fragment_index = 1; - nic_sqe = &nesnic->sq_vbase[nesnic->sq_tail]; - skb = nesnic->tx_skb[nesnic->sq_tail]; - wqe_fragment_length = (__le16 *)&nic_sqe->wqe_words[NES_NIC_SQ_WQE_LENGTH_0_TAG_IDX]; - /* bump past the vlan tag */ - wqe_fragment_length++; - if (le16_to_cpu(wqe_fragment_length[wqe_fragment_index]) != 0) { - u64temp = (u64) le32_to_cpu(nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_LOW_IDX + - wqe_fragment_index * 2]); - u64temp += ((u64)le32_to_cpu(nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_HIGH_IDX + - wqe_fragment_index * 2])) << 32; - bus_address = (dma_addr_t)u64temp; - if (test_and_clear_bit(nesnic->sq_tail, nesnic->first_frag_overflow)) { - pci_unmap_single(nesdev->pcidev, - bus_address, - le16_to_cpu(wqe_fragment_length[wqe_fragment_index++]), - PCI_DMA_TODEVICE); - } - for (; wqe_fragment_index < 5; wqe_fragment_index++) { - if (wqe_fragment_length[wqe_fragment_index]) { - u64temp = le32_to_cpu(nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_LOW_IDX + - wqe_fragment_index * 2]); - u64temp += ((u64)le32_to_cpu(nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_HIGH_IDX - + wqe_fragment_index * 2])) <<32; - bus_address = (dma_addr_t)u64temp; - pci_unmap_page(nesdev->pcidev, - bus_address, - le16_to_cpu(wqe_fragment_length[wqe_fragment_index]), - PCI_DMA_TODEVICE); - } else - break; - } - } - if (skb) - dev_kfree_skb_any(skb); - nesnic->sq_tail++; - nesnic->sq_tail &= nesnic->sq_size-1; - if (sq_cqes > 128) { - barrier(); - /* restart the queue if it had been stopped */ - if (netif_queue_stopped(nesvnic->netdev)) - netif_wake_queue(nesvnic->netdev); - sq_cqes = 0; - } - } else { - rqes_processed ++; - - cq->rx_cqes_completed++; - cq->rx_pkts_indicated++; - rx_pkt_size = cqe_misc & 0x0000ffff; - nic_rqe = &nesnic->rq_vbase[nesnic->rq_tail]; - /* Get the skb */ - rx_skb = nesnic->rx_skb[nesnic->rq_tail]; - nic_rqe = &nesnic->rq_vbase[nesvnic->nic.rq_tail]; - bus_address = (dma_addr_t)le32_to_cpu(nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_LOW_IDX]); - bus_address += ((u64)le32_to_cpu(nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_HIGH_IDX])) << 32; - pci_unmap_single(nesdev->pcidev, bus_address, - nesvnic->max_frame_size, PCI_DMA_FROMDEVICE); - cb = (struct nes_rskb_cb *)&rx_skb->cb[0]; - cb->busaddr = 0; - /* rx_skb->tail = rx_skb->data + rx_pkt_size; */ - /* rx_skb->len = rx_pkt_size; */ - rx_skb->len = 0; /* TODO: see if this is necessary */ - skb_put(rx_skb, rx_pkt_size); - rx_skb->protocol = eth_type_trans(rx_skb, nesvnic->netdev); - nesnic->rq_tail++; - nesnic->rq_tail &= nesnic->rq_size - 1; - - atomic_inc(&nesvnic->rx_skbs_needed); - if (atomic_read(&nesvnic->rx_skbs_needed) > (nesvnic->nic.rq_size>>1)) { - nes_write32(nesdev->regs+NES_CQE_ALLOC, - cq->cq_number | (cqe_count << 16)); - /* nesadapter->tune_timer.cq_count += cqe_count; */ - nesdev->currcq_count += cqe_count; - cqe_count = 0; - nes_replenish_nic_rq(nesvnic); - } - pkt_type = (u16)(le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_NIC_CQE_TAG_PKT_TYPE_IDX])); - cqe_errv = (cqe_misc & NES_NIC_CQE_ERRV_MASK) >> NES_NIC_CQE_ERRV_SHIFT; - rx_skb->ip_summed = CHECKSUM_NONE; - - if ((NES_PKT_TYPE_TCPV4_BITS == (pkt_type & NES_PKT_TYPE_TCPV4_MASK)) || - (NES_PKT_TYPE_UDPV4_BITS == (pkt_type & NES_PKT_TYPE_UDPV4_MASK))) { - if ((cqe_errv & - (NES_NIC_ERRV_BITS_IPV4_CSUM_ERR | NES_NIC_ERRV_BITS_TCPUDP_CSUM_ERR | - NES_NIC_ERRV_BITS_IPH_ERR | NES_NIC_ERRV_BITS_WQE_OVERRUN)) == 0) { - if (nesvnic->netdev->features & NETIF_F_RXCSUM) - rx_skb->ip_summed = CHECKSUM_UNNECESSARY; - } else - nes_debug(NES_DBG_CQ, "%s: unsuccessfully checksummed TCP or UDP packet." - " errv = 0x%X, pkt_type = 0x%X.\n", - nesvnic->netdev->name, cqe_errv, pkt_type); - - } else if ((pkt_type & NES_PKT_TYPE_IPV4_MASK) == NES_PKT_TYPE_IPV4_BITS) { - if ((cqe_errv & - (NES_NIC_ERRV_BITS_IPV4_CSUM_ERR | NES_NIC_ERRV_BITS_IPH_ERR | - NES_NIC_ERRV_BITS_WQE_OVERRUN)) == 0) { - if (nesvnic->netdev->features & NETIF_F_RXCSUM) { - rx_skb->ip_summed = CHECKSUM_UNNECESSARY; - /* nes_debug(NES_DBG_CQ, "%s: Reporting successfully checksummed IPv4 packet.\n", - nesvnic->netdev->name); */ - } - } else - nes_debug(NES_DBG_CQ, "%s: unsuccessfully checksummed TCP or UDP packet." - " errv = 0x%X, pkt_type = 0x%X.\n", - nesvnic->netdev->name, cqe_errv, pkt_type); - } - /* nes_debug(NES_DBG_CQ, "pkt_type=%x, APBVT_MASK=%x\n", - pkt_type, (pkt_type & NES_PKT_TYPE_APBVT_MASK)); */ - - if ((pkt_type & NES_PKT_TYPE_APBVT_MASK) == NES_PKT_TYPE_APBVT_BITS) { - if (nes_cm_recv(rx_skb, nesvnic->netdev)) - rx_skb = NULL; - } - if (rx_skb == NULL) - goto skip_rx_indicate0; - - - if (cqe_misc & NES_NIC_CQE_TAG_VALID) { - vlan_tag = (u16)(le32_to_cpu( - cq->cq_vbase[head].cqe_words[NES_NIC_CQE_TAG_PKT_TYPE_IDX]) - >> 16); - nes_debug(NES_DBG_CQ, "%s: Reporting stripped VLAN packet. Tag = 0x%04X\n", - nesvnic->netdev->name, vlan_tag); - - __vlan_hwaccel_put_tag(rx_skb, htons(ETH_P_8021Q), vlan_tag); - } - napi_gro_receive(&nesvnic->napi, rx_skb); - -skip_rx_indicate0: - ; - /* nesvnic->netstats.rx_packets++; */ - /* nesvnic->netstats.rx_bytes += rx_pkt_size; */ - } - - cq->cq_vbase[head].cqe_words[NES_NIC_CQE_MISC_IDX] = 0; - /* Accounting... */ - cqe_count++; - if (++head >= cq_size) - head = 0; - if (cqe_count == 255) { - /* Replenish Nic CQ */ - nes_write32(nesdev->regs+NES_CQE_ALLOC, - cq->cq_number | (cqe_count << 16)); - /* nesdev->nesadapter->tune_timer.cq_count += cqe_count; */ - nesdev->currcq_count += cqe_count; - cqe_count = 0; - } - - if (cq->rx_cqes_completed >= nesvnic->budget) - break; - } else { - cq->cqes_pending = 0; - break; - } - - } while (1); - - if (sq_cqes) { - barrier(); - /* restart the queue if it had been stopped */ - if (netif_queue_stopped(nesvnic->netdev)) - netif_wake_queue(nesvnic->netdev); - } - cq->cq_head = head; - /* nes_debug(NES_DBG_CQ, "CQ%u Processed = %u cqes, new head = %u.\n", - cq->cq_number, cqe_count, cq->cq_head); */ - cq->cqe_allocs_pending = cqe_count; - if (unlikely(nesadapter->et_use_adaptive_rx_coalesce)) - { - /* nesdev->nesadapter->tune_timer.cq_count += cqe_count; */ - nesdev->currcq_count += cqe_count; - nes_nic_tune_timer(nesdev); - } - if (atomic_read(&nesvnic->rx_skbs_needed)) - nes_replenish_nic_rq(nesvnic); -} - - - -/** - * nes_cqp_ce_handler - */ -static void nes_cqp_ce_handler(struct nes_device *nesdev, struct nes_hw_cq *cq) -{ - u64 u64temp; - unsigned long flags; - struct nes_hw_cqp *cqp = NULL; - struct nes_cqp_request *cqp_request; - struct nes_hw_cqp_wqe *cqp_wqe; - u32 head; - u32 cq_size; - u32 cqe_count=0; - u32 error_code; - u32 opcode; - u32 ctx_index; - /* u32 counter; */ - - head = cq->cq_head; - cq_size = cq->cq_size; - - do { - /* process the CQE */ - /* nes_debug(NES_DBG_CQP, "head=%u cqe_words=%08X\n", head, - le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX])); */ - - opcode = le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX]); - if (opcode & NES_CQE_VALID) { - cqp = &nesdev->cqp; - - error_code = le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_CQE_ERROR_CODE_IDX]); - if (error_code) { - nes_debug(NES_DBG_CQP, "Bad Completion code for opcode 0x%02X from CQP," - " Major/Minor codes = 0x%04X:%04X.\n", - le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX])&0x3f, - (u16)(error_code >> 16), - (u16)error_code); - } - - u64temp = (((u64)(le32_to_cpu(cq->cq_vbase[head]. - cqe_words[NES_CQE_COMP_COMP_CTX_HIGH_IDX]))) << 32) | - ((u64)(le32_to_cpu(cq->cq_vbase[head]. - cqe_words[NES_CQE_COMP_COMP_CTX_LOW_IDX]))); - - cqp_request = (struct nes_cqp_request *)(unsigned long)u64temp; - if (cqp_request) { - if (cqp_request->waiting) { - /* nes_debug(NES_DBG_CQP, "%s: Waking up requestor\n"); */ - cqp_request->major_code = (u16)(error_code >> 16); - cqp_request->minor_code = (u16)error_code; - barrier(); - cqp_request->request_done = 1; - wake_up(&cqp_request->waitq); - nes_put_cqp_request(nesdev, cqp_request); - } else { - if (cqp_request->callback) - cqp_request->cqp_callback(nesdev, cqp_request); - nes_free_cqp_request(nesdev, cqp_request); - } - } else { - wake_up(&nesdev->cqp.waitq); - } - - cq->cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX] = 0; - nes_write32(nesdev->regs + NES_CQE_ALLOC, cq->cq_number | (1 << 16)); - if (++cqp->sq_tail >= cqp->sq_size) - cqp->sq_tail = 0; - - /* Accounting... */ - cqe_count++; - if (++head >= cq_size) - head = 0; - } else { - break; - } - } while (1); - cq->cq_head = head; - - spin_lock_irqsave(&nesdev->cqp.lock, flags); - while ((!list_empty(&nesdev->cqp_pending_reqs)) && - ((((nesdev->cqp.sq_tail+nesdev->cqp.sq_size)-nesdev->cqp.sq_head) & - (nesdev->cqp.sq_size - 1)) != 1)) { - cqp_request = list_entry(nesdev->cqp_pending_reqs.next, - struct nes_cqp_request, list); - list_del_init(&cqp_request->list); - head = nesdev->cqp.sq_head++; - nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1; - cqp_wqe = &nesdev->cqp.sq_vbase[head]; - memcpy(cqp_wqe, &cqp_request->cqp_wqe, sizeof(*cqp_wqe)); - barrier(); - - opcode = le32_to_cpu(cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX]); - if ((opcode & NES_CQP_OPCODE_MASK) == NES_CQP_DOWNLOAD_SEGMENT) - ctx_index = NES_CQP_WQE_DL_COMP_CTX_LOW_IDX; - else - ctx_index = NES_CQP_WQE_COMP_CTX_LOW_IDX; - cqp_wqe->wqe_words[ctx_index] = - cpu_to_le32((u32)((unsigned long)cqp_request)); - cqp_wqe->wqe_words[ctx_index + 1] = - cpu_to_le32((u32)(upper_32_bits((unsigned long)cqp_request))); - nes_debug(NES_DBG_CQP, "CQP request %p (opcode 0x%02X) put on CQPs SQ wqe%u.\n", - cqp_request, le32_to_cpu(cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX])&0x3f, head); - /* Ring doorbell (1 WQEs) */ - barrier(); - nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x01800000 | nesdev->cqp.qp_id); - } - spin_unlock_irqrestore(&nesdev->cqp.lock, flags); - - /* Arm the CCQ */ - nes_write32(nesdev->regs+NES_CQE_ALLOC, NES_CQE_ALLOC_NOTIFY_NEXT | - cq->cq_number); - nes_read32(nesdev->regs+NES_CQE_ALLOC); -} - -static u8 *locate_mpa(u8 *pkt, u32 aeq_info) -{ - if (aeq_info & NES_AEQE_Q2_DATA_ETHERNET) { - /* skip over ethernet header */ - pkt += ETH_HLEN; - - /* Skip over IP and TCP headers */ - pkt += 4 * (pkt[0] & 0x0f); - pkt += 4 * ((pkt[12] >> 4) & 0x0f); - } - return pkt; -} - -/* Determine if incoming error pkt is rdma layer */ -static u32 iwarp_opcode(struct nes_qp *nesqp, u32 aeq_info) -{ - u8 *pkt; - u16 *mpa; - u32 opcode = 0xffffffff; - - if (aeq_info & NES_AEQE_Q2_DATA_WRITTEN) { - pkt = nesqp->hwqp.q2_vbase + BAD_FRAME_OFFSET; - mpa = (u16 *)locate_mpa(pkt, aeq_info); - opcode = be16_to_cpu(mpa[1]) & 0xf; - } - - return opcode; -} - -/* Build iWARP terminate header */ -static int nes_bld_terminate_hdr(struct nes_qp *nesqp, u16 async_event_id, u32 aeq_info) -{ - u8 *pkt = nesqp->hwqp.q2_vbase + BAD_FRAME_OFFSET; - u16 ddp_seg_len; - int copy_len = 0; - u8 is_tagged = 0; - u8 flush_code = 0; - struct nes_terminate_hdr *termhdr; - - termhdr = (struct nes_terminate_hdr *)nesqp->hwqp.q2_vbase; - memset(termhdr, 0, 64); - - if (aeq_info & NES_AEQE_Q2_DATA_WRITTEN) { - - /* Use data from offending packet to fill in ddp & rdma hdrs */ - pkt = locate_mpa(pkt, aeq_info); - ddp_seg_len = be16_to_cpu(*(u16 *)pkt); - if (ddp_seg_len) { - copy_len = 2; - termhdr->hdrct = DDP_LEN_FLAG; - if (pkt[2] & 0x80) { - is_tagged = 1; - if (ddp_seg_len >= TERM_DDP_LEN_TAGGED) { - copy_len += TERM_DDP_LEN_TAGGED; - termhdr->hdrct |= DDP_HDR_FLAG; - } - } else { - if (ddp_seg_len >= TERM_DDP_LEN_UNTAGGED) { - copy_len += TERM_DDP_LEN_UNTAGGED; - termhdr->hdrct |= DDP_HDR_FLAG; - } - - if (ddp_seg_len >= (TERM_DDP_LEN_UNTAGGED + TERM_RDMA_LEN)) { - if ((pkt[3] & RDMA_OPCODE_MASK) == RDMA_READ_REQ_OPCODE) { - copy_len += TERM_RDMA_LEN; - termhdr->hdrct |= RDMA_HDR_FLAG; - } - } - } - } - } - - switch (async_event_id) { - case NES_AEQE_AEID_AMP_UNALLOCATED_STAG: - switch (iwarp_opcode(nesqp, aeq_info)) { - case IWARP_OPCODE_WRITE: - flush_code = IB_WC_LOC_PROT_ERR; - termhdr->layer_etype = (LAYER_DDP << 4) | DDP_TAGGED_BUFFER; - termhdr->error_code = DDP_TAGGED_INV_STAG; - break; - default: - flush_code = IB_WC_REM_ACCESS_ERR; - termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT; - termhdr->error_code = RDMAP_INV_STAG; - } - break; - case NES_AEQE_AEID_AMP_INVALID_STAG: - flush_code = IB_WC_REM_ACCESS_ERR; - termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT; - termhdr->error_code = RDMAP_INV_STAG; - break; - case NES_AEQE_AEID_AMP_BAD_QP: - flush_code = IB_WC_LOC_QP_OP_ERR; - termhdr->layer_etype = (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER; - termhdr->error_code = DDP_UNTAGGED_INV_QN; - break; - case NES_AEQE_AEID_AMP_BAD_STAG_KEY: - case NES_AEQE_AEID_AMP_BAD_STAG_INDEX: - switch (iwarp_opcode(nesqp, aeq_info)) { - case IWARP_OPCODE_SEND_INV: - case IWARP_OPCODE_SEND_SE_INV: - flush_code = IB_WC_REM_OP_ERR; - termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_OP; - termhdr->error_code = RDMAP_CANT_INV_STAG; - break; - default: - flush_code = IB_WC_REM_ACCESS_ERR; - termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT; - termhdr->error_code = RDMAP_INV_STAG; - } - break; - case NES_AEQE_AEID_AMP_BOUNDS_VIOLATION: - if (aeq_info & (NES_AEQE_Q2_DATA_ETHERNET | NES_AEQE_Q2_DATA_MPA)) { - flush_code = IB_WC_LOC_PROT_ERR; - termhdr->layer_etype = (LAYER_DDP << 4) | DDP_TAGGED_BUFFER; - termhdr->error_code = DDP_TAGGED_BOUNDS; - } else { - flush_code = IB_WC_REM_ACCESS_ERR; - termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT; - termhdr->error_code = RDMAP_INV_BOUNDS; - } - break; - case NES_AEQE_AEID_AMP_RIGHTS_VIOLATION: - case NES_AEQE_AEID_AMP_INVALIDATE_NO_REMOTE_ACCESS_RIGHTS: - case NES_AEQE_AEID_PRIV_OPERATION_DENIED: - flush_code = IB_WC_REM_ACCESS_ERR; - termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT; - termhdr->error_code = RDMAP_ACCESS; - break; - case NES_AEQE_AEID_AMP_TO_WRAP: - flush_code = IB_WC_REM_ACCESS_ERR; - termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT; - termhdr->error_code = RDMAP_TO_WRAP; - break; - case NES_AEQE_AEID_AMP_BAD_PD: - switch (iwarp_opcode(nesqp, aeq_info)) { - case IWARP_OPCODE_WRITE: - flush_code = IB_WC_LOC_PROT_ERR; - termhdr->layer_etype = (LAYER_DDP << 4) | DDP_TAGGED_BUFFER; - termhdr->error_code = DDP_TAGGED_UNASSOC_STAG; - break; - case IWARP_OPCODE_SEND_INV: - case IWARP_OPCODE_SEND_SE_INV: - flush_code = IB_WC_REM_ACCESS_ERR; - termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT; - termhdr->error_code = RDMAP_CANT_INV_STAG; - break; - default: - flush_code = IB_WC_REM_ACCESS_ERR; - termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT; - termhdr->error_code = RDMAP_UNASSOC_STAG; - } - break; - case NES_AEQE_AEID_LLP_RECEIVED_MARKER_AND_LENGTH_FIELDS_DONT_MATCH: - flush_code = IB_WC_LOC_LEN_ERR; - termhdr->layer_etype = (LAYER_MPA << 4) | DDP_LLP; - termhdr->error_code = MPA_MARKER; - break; - case NES_AEQE_AEID_LLP_RECEIVED_MPA_CRC_ERROR: - flush_code = IB_WC_GENERAL_ERR; - termhdr->layer_etype = (LAYER_MPA << 4) | DDP_LLP; - termhdr->error_code = MPA_CRC; - break; - case NES_AEQE_AEID_LLP_SEGMENT_TOO_LARGE: - case NES_AEQE_AEID_LLP_SEGMENT_TOO_SMALL: - flush_code = IB_WC_LOC_LEN_ERR; - termhdr->layer_etype = (LAYER_DDP << 4) | DDP_CATASTROPHIC; - termhdr->error_code = DDP_CATASTROPHIC_LOCAL; - break; - case NES_AEQE_AEID_DDP_LCE_LOCAL_CATASTROPHIC: - case NES_AEQE_AEID_DDP_NO_L_BIT: - flush_code = IB_WC_FATAL_ERR; - termhdr->layer_etype = (LAYER_DDP << 4) | DDP_CATASTROPHIC; - termhdr->error_code = DDP_CATASTROPHIC_LOCAL; - break; - case NES_AEQE_AEID_DDP_INVALID_MSN_GAP_IN_MSN: - case NES_AEQE_AEID_DDP_INVALID_MSN_RANGE_IS_NOT_VALID: - flush_code = IB_WC_GENERAL_ERR; - termhdr->layer_etype = (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER; - termhdr->error_code = DDP_UNTAGGED_INV_MSN_RANGE; - break; - case NES_AEQE_AEID_DDP_UBE_DDP_MESSAGE_TOO_LONG_FOR_AVAILABLE_BUFFER: - flush_code = IB_WC_LOC_LEN_ERR; - termhdr->layer_etype = (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER; - termhdr->error_code = DDP_UNTAGGED_INV_TOO_LONG; - break; - case NES_AEQE_AEID_DDP_UBE_INVALID_DDP_VERSION: - flush_code = IB_WC_GENERAL_ERR; - if (is_tagged) { - termhdr->layer_etype = (LAYER_DDP << 4) | DDP_TAGGED_BUFFER; - termhdr->error_code = DDP_TAGGED_INV_DDP_VER; - } else { - termhdr->layer_etype = (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER; - termhdr->error_code = DDP_UNTAGGED_INV_DDP_VER; - } - break; - case NES_AEQE_AEID_DDP_UBE_INVALID_MO: - flush_code = IB_WC_GENERAL_ERR; - termhdr->layer_etype = (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER; - termhdr->error_code = DDP_UNTAGGED_INV_MO; - break; - case NES_AEQE_AEID_DDP_UBE_INVALID_MSN_NO_BUFFER_AVAILABLE: - flush_code = IB_WC_REM_OP_ERR; - termhdr->layer_etype = (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER; - termhdr->error_code = DDP_UNTAGGED_INV_MSN_NO_BUF; - break; - case NES_AEQE_AEID_DDP_UBE_INVALID_QN: - flush_code = IB_WC_GENERAL_ERR; - termhdr->layer_etype = (LAYER_DDP << 4) | DDP_UNTAGGED_BUFFER; - termhdr->error_code = DDP_UNTAGGED_INV_QN; - break; - case NES_AEQE_AEID_RDMAP_ROE_INVALID_RDMAP_VERSION: - flush_code = IB_WC_GENERAL_ERR; - termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_OP; - termhdr->error_code = RDMAP_INV_RDMAP_VER; - break; - case NES_AEQE_AEID_RDMAP_ROE_UNEXPECTED_OPCODE: - flush_code = IB_WC_LOC_QP_OP_ERR; - termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_OP; - termhdr->error_code = RDMAP_UNEXPECTED_OP; - break; - default: - flush_code = IB_WC_FATAL_ERR; - termhdr->layer_etype = (LAYER_RDMA << 4) | RDMAP_REMOTE_OP; - termhdr->error_code = RDMAP_UNSPECIFIED; - break; - } - - if (copy_len) - memcpy(termhdr + 1, pkt, copy_len); - - if ((flush_code) && ((NES_AEQE_INBOUND_RDMA & aeq_info) == 0)) { - if (aeq_info & NES_AEQE_SQ) - nesqp->term_sq_flush_code = flush_code; - else - nesqp->term_rq_flush_code = flush_code; - } - - return sizeof(struct nes_terminate_hdr) + copy_len; -} - -static void nes_terminate_connection(struct nes_device *nesdev, struct nes_qp *nesqp, - struct nes_hw_aeqe *aeqe, enum ib_event_type eventtype) -{ - u64 context; - unsigned long flags; - u32 aeq_info; - u16 async_event_id; - u8 tcp_state; - u8 iwarp_state; - u32 termlen = 0; - u32 mod_qp_flags = NES_CQP_QP_IWARP_STATE_TERMINATE | - NES_CQP_QP_TERM_DONT_SEND_FIN; - struct nes_adapter *nesadapter = nesdev->nesadapter; - - if (nesqp->term_flags & NES_TERM_SENT) - return; /* Sanity check */ - - aeq_info = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]); - tcp_state = (aeq_info & NES_AEQE_TCP_STATE_MASK) >> NES_AEQE_TCP_STATE_SHIFT; - iwarp_state = (aeq_info & NES_AEQE_IWARP_STATE_MASK) >> NES_AEQE_IWARP_STATE_SHIFT; - async_event_id = (u16)aeq_info; - - context = (unsigned long)nesadapter->qp_table[le32_to_cpu( - aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]) - NES_FIRST_QPN]; - if (!context) { - WARN_ON(!context); - return; - } - - nesqp = (struct nes_qp *)(unsigned long)context; - spin_lock_irqsave(&nesqp->lock, flags); - nesqp->hw_iwarp_state = iwarp_state; - nesqp->hw_tcp_state = tcp_state; - nesqp->last_aeq = async_event_id; - nesqp->terminate_eventtype = eventtype; - spin_unlock_irqrestore(&nesqp->lock, flags); - - if (nesadapter->send_term_ok) - termlen = nes_bld_terminate_hdr(nesqp, async_event_id, aeq_info); - else - mod_qp_flags |= NES_CQP_QP_TERM_DONT_SEND_TERM_MSG; - - if (!nesdev->iw_status) { - nesqp->term_flags = NES_TERM_DONE; - nes_hw_modify_qp(nesdev, nesqp, NES_CQP_QP_IWARP_STATE_ERROR, 0, 0); - nes_cm_disconn(nesqp); - } else { - nes_terminate_start_timer(nesqp); - nesqp->term_flags |= NES_TERM_SENT; - nes_hw_modify_qp(nesdev, nesqp, mod_qp_flags, termlen, 0); - } -} - -static void nes_terminate_send_fin(struct nes_device *nesdev, - struct nes_qp *nesqp, struct nes_hw_aeqe *aeqe) -{ - u32 aeq_info; - u16 async_event_id; - u8 tcp_state; - u8 iwarp_state; - unsigned long flags; - - aeq_info = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]); - tcp_state = (aeq_info & NES_AEQE_TCP_STATE_MASK) >> NES_AEQE_TCP_STATE_SHIFT; - iwarp_state = (aeq_info & NES_AEQE_IWARP_STATE_MASK) >> NES_AEQE_IWARP_STATE_SHIFT; - async_event_id = (u16)aeq_info; - - spin_lock_irqsave(&nesqp->lock, flags); - nesqp->hw_iwarp_state = iwarp_state; - nesqp->hw_tcp_state = tcp_state; - nesqp->last_aeq = async_event_id; - spin_unlock_irqrestore(&nesqp->lock, flags); - - /* Send the fin only */ - nes_hw_modify_qp(nesdev, nesqp, NES_CQP_QP_IWARP_STATE_TERMINATE | - NES_CQP_QP_TERM_DONT_SEND_TERM_MSG, 0, 0); -} - -/* Cleanup after a terminate sent or received */ -static void nes_terminate_done(struct nes_qp *nesqp, int timeout_occurred) -{ - u32 next_iwarp_state = NES_CQP_QP_IWARP_STATE_ERROR; - unsigned long flags; - struct nes_vnic *nesvnic = to_nesvnic(nesqp->ibqp.device); - struct nes_device *nesdev = nesvnic->nesdev; - u8 first_time = 0; - - spin_lock_irqsave(&nesqp->lock, flags); - if (nesqp->hte_added) { - nesqp->hte_added = 0; - next_iwarp_state |= NES_CQP_QP_DEL_HTE; - } - - first_time = (nesqp->term_flags & NES_TERM_DONE) == 0; - nesqp->term_flags |= NES_TERM_DONE; - spin_unlock_irqrestore(&nesqp->lock, flags); - - /* Make sure we go through this only once */ - if (first_time) { - if (timeout_occurred == 0) - del_timer(&nesqp->terminate_timer); - else - next_iwarp_state |= NES_CQP_QP_RESET; - - nes_hw_modify_qp(nesdev, nesqp, next_iwarp_state, 0, 0); - nes_cm_disconn(nesqp); - } -} - -static void nes_terminate_received(struct nes_device *nesdev, - struct nes_qp *nesqp, struct nes_hw_aeqe *aeqe) -{ - u32 aeq_info; - u8 *pkt; - u32 *mpa; - u8 ddp_ctl; - u8 rdma_ctl; - u16 aeq_id = 0; - - aeq_info = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]); - if (aeq_info & NES_AEQE_Q2_DATA_WRITTEN) { - /* Terminate is not a performance path so the silicon */ - /* did not validate the frame - do it now */ - pkt = nesqp->hwqp.q2_vbase + BAD_FRAME_OFFSET; - mpa = (u32 *)locate_mpa(pkt, aeq_info); - ddp_ctl = (be32_to_cpu(mpa[0]) >> 8) & 0xff; - rdma_ctl = be32_to_cpu(mpa[0]) & 0xff; - if ((ddp_ctl & 0xc0) != 0x40) - aeq_id = NES_AEQE_AEID_DDP_LCE_LOCAL_CATASTROPHIC; - else if ((ddp_ctl & 0x03) != 1) - aeq_id = NES_AEQE_AEID_DDP_UBE_INVALID_DDP_VERSION; - else if (be32_to_cpu(mpa[2]) != 2) - aeq_id = NES_AEQE_AEID_DDP_UBE_INVALID_QN; - else if (be32_to_cpu(mpa[3]) != 1) - aeq_id = NES_AEQE_AEID_DDP_INVALID_MSN_GAP_IN_MSN; - else if (be32_to_cpu(mpa[4]) != 0) - aeq_id = NES_AEQE_AEID_DDP_UBE_INVALID_MO; - else if ((rdma_ctl & 0xc0) != 0x40) - aeq_id = NES_AEQE_AEID_RDMAP_ROE_INVALID_RDMAP_VERSION; - - if (aeq_id) { - /* Bad terminate recvd - send back a terminate */ - aeq_info = (aeq_info & 0xffff0000) | aeq_id; - aeqe->aeqe_words[NES_AEQE_MISC_IDX] = cpu_to_le32(aeq_info); - nes_terminate_connection(nesdev, nesqp, aeqe, IB_EVENT_QP_FATAL); - return; - } - } - - nesqp->term_flags |= NES_TERM_RCVD; - nesqp->terminate_eventtype = IB_EVENT_QP_FATAL; - nes_terminate_start_timer(nesqp); - nes_terminate_send_fin(nesdev, nesqp, aeqe); -} - -/* Timeout routine in case terminate fails to complete */ -void nes_terminate_timeout(struct timer_list *t) -{ - struct nes_qp *nesqp = from_timer(nesqp, t, terminate_timer); - - nes_terminate_done(nesqp, 1); -} - -/* Set a timer in case hw cannot complete the terminate sequence */ -static void nes_terminate_start_timer(struct nes_qp *nesqp) -{ - mod_timer(&nesqp->terminate_timer, (jiffies + HZ)); -} - -/** - * nes_process_iwarp_aeqe - */ -static void nes_process_iwarp_aeqe(struct nes_device *nesdev, - struct nes_hw_aeqe *aeqe) -{ - u64 context; - unsigned long flags; - struct nes_qp *nesqp; - struct nes_hw_cq *hw_cq; - struct nes_cq *nescq; - int resource_allocated; - struct nes_adapter *nesadapter = nesdev->nesadapter; - u32 aeq_info; - u32 next_iwarp_state = 0; - u32 aeqe_cq_id; - u16 async_event_id; - u8 tcp_state; - u8 iwarp_state; - struct ib_event ibevent; - - nes_debug(NES_DBG_AEQ, "\n"); - aeq_info = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_MISC_IDX]); - if ((NES_AEQE_INBOUND_RDMA & aeq_info) || (!(NES_AEQE_QP & aeq_info))) { - context = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_LOW_IDX]); - context += ((u64)le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_CTXT_HIGH_IDX])) << 32; - } else { - context = (unsigned long)nesadapter->qp_table[le32_to_cpu( - aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]) - NES_FIRST_QPN]; - BUG_ON(!context); - } - - /* context is nesqp unless async_event_id == CQ ERROR */ - nesqp = (struct nes_qp *)(unsigned long)context; - async_event_id = (u16)aeq_info; - tcp_state = (aeq_info & NES_AEQE_TCP_STATE_MASK) >> NES_AEQE_TCP_STATE_SHIFT; - iwarp_state = (aeq_info & NES_AEQE_IWARP_STATE_MASK) >> NES_AEQE_IWARP_STATE_SHIFT; - nes_debug(NES_DBG_AEQ, "aeid = 0x%04X, qp-cq id = %d, aeqe = %p," - " Tcp state = %s, iWARP state = %s\n", - async_event_id, - le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]), aeqe, - nes_tcp_state_str[tcp_state], nes_iwarp_state_str[iwarp_state]); - - aeqe_cq_id = le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]); - if (aeq_info & NES_AEQE_QP) { - if (!nes_is_resource_allocated(nesadapter, - nesadapter->allocated_qps, - aeqe_cq_id)) - return; - } - - switch (async_event_id) { - case NES_AEQE_AEID_LLP_FIN_RECEIVED: - if (nesqp->term_flags) - return; /* Ignore it, wait for close complete */ - - if (atomic_inc_return(&nesqp->close_timer_started) == 1) { - if ((tcp_state == NES_AEQE_TCP_STATE_CLOSE_WAIT) && - (nesqp->ibqp_state == IB_QPS_RTS)) { - spin_lock_irqsave(&nesqp->lock, flags); - nesqp->hw_iwarp_state = iwarp_state; - nesqp->hw_tcp_state = tcp_state; - nesqp->last_aeq = async_event_id; - next_iwarp_state = NES_CQP_QP_IWARP_STATE_CLOSING; - nesqp->hw_iwarp_state = NES_AEQE_IWARP_STATE_CLOSING; - spin_unlock_irqrestore(&nesqp->lock, flags); - nes_hw_modify_qp(nesdev, nesqp, next_iwarp_state, 0, 0); - nes_cm_disconn(nesqp); - } - nesqp->cm_id->add_ref(nesqp->cm_id); - schedule_nes_timer(nesqp->cm_node, (struct sk_buff *)nesqp, - NES_TIMER_TYPE_CLOSE, 1, 0); - nes_debug(NES_DBG_AEQ, "QP%u Not decrementing QP refcount (%d)," - " need ae to finish up, original_last_aeq = 0x%04X." - " last_aeq = 0x%04X, scheduling timer. TCP state = %d\n", - nesqp->hwqp.qp_id, atomic_read(&nesqp->refcount), - async_event_id, nesqp->last_aeq, tcp_state); - } - break; - case NES_AEQE_AEID_LLP_CLOSE_COMPLETE: - spin_lock_irqsave(&nesqp->lock, flags); - nesqp->hw_iwarp_state = iwarp_state; - nesqp->hw_tcp_state = tcp_state; - nesqp->last_aeq = async_event_id; - spin_unlock_irqrestore(&nesqp->lock, flags); - nes_cm_disconn(nesqp); - break; - - case NES_AEQE_AEID_RESET_SENT: - tcp_state = NES_AEQE_TCP_STATE_CLOSED; - spin_lock_irqsave(&nesqp->lock, flags); - nesqp->hw_iwarp_state = iwarp_state; - nesqp->hw_tcp_state = tcp_state; - nesqp->last_aeq = async_event_id; - nesqp->hte_added = 0; - spin_unlock_irqrestore(&nesqp->lock, flags); - next_iwarp_state = NES_CQP_QP_IWARP_STATE_ERROR | NES_CQP_QP_DEL_HTE; - nes_hw_modify_qp(nesdev, nesqp, next_iwarp_state, 0, 0); - nes_cm_disconn(nesqp); - break; - - case NES_AEQE_AEID_LLP_CONNECTION_RESET: - if (atomic_read(&nesqp->close_timer_started)) - return; - spin_lock_irqsave(&nesqp->lock, flags); - nesqp->hw_iwarp_state = iwarp_state; - nesqp->hw_tcp_state = tcp_state; - nesqp->last_aeq = async_event_id; - spin_unlock_irqrestore(&nesqp->lock, flags); - nes_cm_disconn(nesqp); - break; - - case NES_AEQE_AEID_TERMINATE_SENT: - nes_terminate_send_fin(nesdev, nesqp, aeqe); - break; - - case NES_AEQE_AEID_LLP_TERMINATE_RECEIVED: - nes_terminate_received(nesdev, nesqp, aeqe); - break; - - case NES_AEQE_AEID_AMP_BAD_STAG_KEY: - case NES_AEQE_AEID_AMP_BAD_STAG_INDEX: - case NES_AEQE_AEID_AMP_UNALLOCATED_STAG: - case NES_AEQE_AEID_AMP_INVALID_STAG: - case NES_AEQE_AEID_AMP_RIGHTS_VIOLATION: - case NES_AEQE_AEID_AMP_INVALIDATE_NO_REMOTE_ACCESS_RIGHTS: - case NES_AEQE_AEID_PRIV_OPERATION_DENIED: - case NES_AEQE_AEID_DDP_UBE_DDP_MESSAGE_TOO_LONG_FOR_AVAILABLE_BUFFER: - case NES_AEQE_AEID_AMP_BOUNDS_VIOLATION: - case NES_AEQE_AEID_AMP_TO_WRAP: - printk(KERN_ERR PFX "QP[%u] async_event_id=0x%04X IB_EVENT_QP_ACCESS_ERR\n", - nesqp->hwqp.qp_id, async_event_id); - nes_terminate_connection(nesdev, nesqp, aeqe, IB_EVENT_QP_ACCESS_ERR); - break; - - case NES_AEQE_AEID_LLP_SEGMENT_TOO_LARGE: - case NES_AEQE_AEID_LLP_SEGMENT_TOO_SMALL: - case NES_AEQE_AEID_DDP_UBE_INVALID_MO: - case NES_AEQE_AEID_DDP_UBE_INVALID_QN: - if (iwarp_opcode(nesqp, aeq_info) > IWARP_OPCODE_TERM) { - aeq_info &= 0xffff0000; - aeq_info |= NES_AEQE_AEID_RDMAP_ROE_UNEXPECTED_OPCODE; - aeqe->aeqe_words[NES_AEQE_MISC_IDX] = cpu_to_le32(aeq_info); - } - /* fall through */ - case NES_AEQE_AEID_RDMAP_ROE_BAD_LLP_CLOSE: - case NES_AEQE_AEID_LLP_TOO_MANY_RETRIES: - case NES_AEQE_AEID_DDP_UBE_INVALID_MSN_NO_BUFFER_AVAILABLE: - case NES_AEQE_AEID_LLP_RECEIVED_MPA_CRC_ERROR: - case NES_AEQE_AEID_AMP_BAD_QP: - case NES_AEQE_AEID_LLP_RECEIVED_MARKER_AND_LENGTH_FIELDS_DONT_MATCH: - case NES_AEQE_AEID_DDP_LCE_LOCAL_CATASTROPHIC: - case NES_AEQE_AEID_DDP_NO_L_BIT: - case NES_AEQE_AEID_DDP_INVALID_MSN_GAP_IN_MSN: - case NES_AEQE_AEID_DDP_INVALID_MSN_RANGE_IS_NOT_VALID: - case NES_AEQE_AEID_DDP_UBE_INVALID_DDP_VERSION: - case NES_AEQE_AEID_RDMAP_ROE_INVALID_RDMAP_VERSION: - case NES_AEQE_AEID_RDMAP_ROE_UNEXPECTED_OPCODE: - case NES_AEQE_AEID_AMP_BAD_PD: - case NES_AEQE_AEID_AMP_FASTREG_SHARED: - case NES_AEQE_AEID_AMP_FASTREG_VALID_STAG: - case NES_AEQE_AEID_AMP_FASTREG_MW_STAG: - case NES_AEQE_AEID_AMP_FASTREG_INVALID_RIGHTS: - case NES_AEQE_AEID_AMP_FASTREG_PBL_TABLE_OVERFLOW: - case NES_AEQE_AEID_AMP_FASTREG_INVALID_LENGTH: - case NES_AEQE_AEID_AMP_INVALIDATE_SHARED: - case NES_AEQE_AEID_AMP_INVALIDATE_MR_WITH_BOUND_WINDOWS: - case NES_AEQE_AEID_AMP_MWBIND_VALID_STAG: - case NES_AEQE_AEID_AMP_MWBIND_OF_MR_STAG: - case NES_AEQE_AEID_AMP_MWBIND_TO_ZERO_BASED_STAG: - case NES_AEQE_AEID_AMP_MWBIND_TO_MW_STAG: - case NES_AEQE_AEID_AMP_MWBIND_INVALID_RIGHTS: - case NES_AEQE_AEID_AMP_MWBIND_INVALID_BOUNDS: - case NES_AEQE_AEID_AMP_MWBIND_TO_INVALID_PARENT: - case NES_AEQE_AEID_AMP_MWBIND_BIND_DISABLED: - case NES_AEQE_AEID_BAD_CLOSE: - case NES_AEQE_AEID_RDMA_READ_WHILE_ORD_ZERO: - case NES_AEQE_AEID_STAG_ZERO_INVALID: - case NES_AEQE_AEID_ROE_INVALID_RDMA_READ_REQUEST: - case NES_AEQE_AEID_ROE_INVALID_RDMA_WRITE_OR_READ_RESP: - printk(KERN_ERR PFX "QP[%u] async_event_id=0x%04X IB_EVENT_QP_FATAL\n", - nesqp->hwqp.qp_id, async_event_id); - print_ip(nesqp->cm_node); - if (!atomic_read(&nesqp->close_timer_started)) - nes_terminate_connection(nesdev, nesqp, aeqe, IB_EVENT_QP_FATAL); - break; - - case NES_AEQE_AEID_CQ_OPERATION_ERROR: - context <<= 1; - nes_debug(NES_DBG_AEQ, "Processing an NES_AEQE_AEID_CQ_OPERATION_ERROR event on CQ%u, %p\n", - le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX]), (void *)(unsigned long)context); - resource_allocated = nes_is_resource_allocated(nesadapter, nesadapter->allocated_cqs, - le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX])); - if (resource_allocated) { - printk(KERN_ERR PFX "%s: Processing an NES_AEQE_AEID_CQ_OPERATION_ERROR event on CQ%u\n", - __func__, le32_to_cpu(aeqe->aeqe_words[NES_AEQE_COMP_QP_CQ_ID_IDX])); - hw_cq = (struct nes_hw_cq *)(unsigned long)context; - if (hw_cq) { - nescq = container_of(hw_cq, struct nes_cq, hw_cq); - if (nescq->ibcq.event_handler) { - ibevent.device = nescq->ibcq.device; - ibevent.event = IB_EVENT_CQ_ERR; - ibevent.element.cq = &nescq->ibcq; - nescq->ibcq.event_handler(&ibevent, nescq->ibcq.cq_context); - } - } - } - break; - - default: - nes_debug(NES_DBG_AEQ, "Processing an iWARP related AE for QP, misc = 0x%04X\n", - async_event_id); - break; - } - -} - -/** - * nes_iwarp_ce_handler - */ -void nes_iwarp_ce_handler(struct nes_device *nesdev, struct nes_hw_cq *hw_cq) -{ - struct nes_cq *nescq = container_of(hw_cq, struct nes_cq, hw_cq); - - /* nes_debug(NES_DBG_CQ, "Processing completion event for iWARP CQ%u.\n", - nescq->hw_cq.cq_number); */ - nes_write32(nesdev->regs+NES_CQ_ACK, nescq->hw_cq.cq_number); - - if (nescq->ibcq.comp_handler) - nescq->ibcq.comp_handler(&nescq->ibcq, nescq->ibcq.cq_context); - - return; -} - - -/** - * nes_manage_apbvt() - */ -int nes_manage_apbvt(struct nes_vnic *nesvnic, u32 accel_local_port, - u32 nic_index, u32 add_port) -{ - struct nes_device *nesdev = nesvnic->nesdev; - struct nes_hw_cqp_wqe *cqp_wqe; - struct nes_cqp_request *cqp_request; - int ret = 0; - u16 major_code; - - /* Send manage APBVT request to CQP */ - cqp_request = nes_get_cqp_request(nesdev); - if (cqp_request == NULL) { - nes_debug(NES_DBG_QP, "Failed to get a cqp_request.\n"); - return -ENOMEM; - } - cqp_request->waiting = 1; - cqp_wqe = &cqp_request->cqp_wqe; - - nes_debug(NES_DBG_QP, "%s APBV for local port=%u(0x%04x), nic_index=%u\n", - (add_port == NES_MANAGE_APBVT_ADD) ? "ADD" : "DEL", - accel_local_port, accel_local_port, nic_index); - - nes_fill_init_cqp_wqe(cqp_wqe, nesdev); - set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX, (NES_CQP_MANAGE_APBVT | - ((add_port == NES_MANAGE_APBVT_ADD) ? NES_CQP_APBVT_ADD : 0))); - set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX, - ((nic_index << NES_CQP_APBVT_NIC_SHIFT) | accel_local_port)); - - nes_debug(NES_DBG_QP, "Waiting for CQP completion for APBVT.\n"); - - atomic_set(&cqp_request->refcount, 2); - nes_post_cqp_request(nesdev, cqp_request); - - if (add_port == NES_MANAGE_APBVT_ADD) - ret = wait_event_timeout(cqp_request->waitq, (cqp_request->request_done != 0), - NES_EVENT_TIMEOUT); - nes_debug(NES_DBG_QP, "Completed, ret=%u, CQP Major:Minor codes = 0x%04X:0x%04X\n", - ret, cqp_request->major_code, cqp_request->minor_code); - major_code = cqp_request->major_code; - - nes_put_cqp_request(nesdev, cqp_request); - - if (!ret) - return -ETIME; - else if (major_code) - return -EIO; - else - return 0; -} - - -/** - * nes_manage_arp_cache - */ -void nes_manage_arp_cache(struct net_device *netdev, unsigned char *mac_addr, - u32 ip_addr, u32 action) -{ - struct nes_hw_cqp_wqe *cqp_wqe; - struct nes_vnic *nesvnic = netdev_priv(netdev); - struct nes_device *nesdev; - struct nes_cqp_request *cqp_request; - int arp_index; - - nesdev = nesvnic->nesdev; - arp_index = nes_arp_table(nesdev, ip_addr, mac_addr, action); - if (arp_index == -1) { - return; - } - - /* update the ARP entry */ - cqp_request = nes_get_cqp_request(nesdev); - if (cqp_request == NULL) { - nes_debug(NES_DBG_NETDEV, "Failed to get a cqp_request.\n"); - return; - } - cqp_request->waiting = 0; - cqp_wqe = &cqp_request->cqp_wqe; - nes_fill_init_cqp_wqe(cqp_wqe, nesdev); - - cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32( - NES_CQP_MANAGE_ARP_CACHE | NES_CQP_ARP_PERM); - cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] |= cpu_to_le32( - (u32)PCI_FUNC(nesdev->pcidev->devfn) << NES_CQP_ARP_AEQ_INDEX_SHIFT); - cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(arp_index); - - if (action == NES_ARP_ADD) { - cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] |= cpu_to_le32(NES_CQP_ARP_VALID); - cqp_wqe->wqe_words[NES_CQP_ARP_WQE_MAC_ADDR_LOW_IDX] = cpu_to_le32( - (((u32)mac_addr[2]) << 24) | (((u32)mac_addr[3]) << 16) | - (((u32)mac_addr[4]) << 8) | (u32)mac_addr[5]); - cqp_wqe->wqe_words[NES_CQP_ARP_WQE_MAC_HIGH_IDX] = cpu_to_le32( - (((u32)mac_addr[0]) << 8) | (u32)mac_addr[1]); - } else { - cqp_wqe->wqe_words[NES_CQP_ARP_WQE_MAC_ADDR_LOW_IDX] = 0; - cqp_wqe->wqe_words[NES_CQP_ARP_WQE_MAC_HIGH_IDX] = 0; - } - - nes_debug(NES_DBG_NETDEV, "Not waiting for CQP, cqp.sq_head=%u, cqp.sq_tail=%u\n", - nesdev->cqp.sq_head, nesdev->cqp.sq_tail); - - atomic_set(&cqp_request->refcount, 1); - nes_post_cqp_request(nesdev, cqp_request); -} - - -/** - * flush_wqes - */ -void flush_wqes(struct nes_device *nesdev, struct nes_qp *nesqp, - u32 which_wq, u32 wait_completion) -{ - struct nes_cqp_request *cqp_request; - struct nes_hw_cqp_wqe *cqp_wqe; - u32 sq_code = (NES_IWARP_CQE_MAJOR_FLUSH << 16) | NES_IWARP_CQE_MINOR_FLUSH; - u32 rq_code = (NES_IWARP_CQE_MAJOR_FLUSH << 16) | NES_IWARP_CQE_MINOR_FLUSH; - int ret; - - cqp_request = nes_get_cqp_request(nesdev); - if (cqp_request == NULL) { - nes_debug(NES_DBG_QP, "Failed to get a cqp_request.\n"); - return; - } - if (wait_completion) { - cqp_request->waiting = 1; - atomic_set(&cqp_request->refcount, 2); - } else { - cqp_request->waiting = 0; - } - cqp_wqe = &cqp_request->cqp_wqe; - nes_fill_init_cqp_wqe(cqp_wqe, nesdev); - - /* If wqe in error was identified, set code to be put into cqe */ - if ((nesqp->term_sq_flush_code) && (which_wq & NES_CQP_FLUSH_SQ)) { - which_wq |= NES_CQP_FLUSH_MAJ_MIN; - sq_code = (CQE_MAJOR_DRV << 16) | nesqp->term_sq_flush_code; - nesqp->term_sq_flush_code = 0; - } - - if ((nesqp->term_rq_flush_code) && (which_wq & NES_CQP_FLUSH_RQ)) { - which_wq |= NES_CQP_FLUSH_MAJ_MIN; - rq_code = (CQE_MAJOR_DRV << 16) | nesqp->term_rq_flush_code; - nesqp->term_rq_flush_code = 0; - } - - if (which_wq & NES_CQP_FLUSH_MAJ_MIN) { - cqp_wqe->wqe_words[NES_CQP_QP_WQE_FLUSH_SQ_CODE] = cpu_to_le32(sq_code); - cqp_wqe->wqe_words[NES_CQP_QP_WQE_FLUSH_RQ_CODE] = cpu_to_le32(rq_code); - } - - cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = - cpu_to_le32(NES_CQP_FLUSH_WQES | which_wq); - cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(nesqp->hwqp.qp_id); - - nes_post_cqp_request(nesdev, cqp_request); - - if (wait_completion) { - /* Wait for CQP */ - ret = wait_event_timeout(cqp_request->waitq, (cqp_request->request_done != 0), - NES_EVENT_TIMEOUT); - nes_debug(NES_DBG_QP, "Flush SQ QP WQEs completed, ret=%u," - " CQP Major:Minor codes = 0x%04X:0x%04X\n", - ret, cqp_request->major_code, cqp_request->minor_code); - nes_put_cqp_request(nesdev, cqp_request); - } -} diff --git a/drivers/infiniband/hw/nes/nes_hw.h b/drivers/infiniband/hw/nes/nes_hw.h deleted file mode 100644 index 3c56470816a8..000000000000 --- a/drivers/infiniband/hw/nes/nes_hw.h +++ /dev/null @@ -1,1380 +0,0 @@ -/* -* Copyright (c) 2006 - 2011 Intel Corporation. All rights reserved. -* -* This software is available to you under a choice of one of two -* licenses. You may choose to be licensed under the terms of the GNU -* General Public License (GPL) Version 2, available from the file -* COPYING in the main directory of this source tree, or the -* OpenIB.org BSD license below: -* -* Redistribution and use in source and binary forms, with or -* without modification, are permitted provided that the following -* conditions are met: -* -* - Redistributions of source code must retain the above -* copyright notice, this list of conditions and the following -* disclaimer. -* -* - Redistributions in binary form must reproduce the above -* copyright notice, this list of conditions and the following -* disclaimer in the documentation and/or other materials -* provided with the distribution. -* -* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS -* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN -* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN -* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -* SOFTWARE. -*/ - -#ifndef __NES_HW_H -#define __NES_HW_H - -#define NES_PHY_TYPE_CX4 1 -#define NES_PHY_TYPE_1G 2 -#define NES_PHY_TYPE_ARGUS 4 -#define NES_PHY_TYPE_PUMA_1G 5 -#define NES_PHY_TYPE_PUMA_10G 6 -#define NES_PHY_TYPE_GLADIUS 7 -#define NES_PHY_TYPE_SFP_D 8 -#define NES_PHY_TYPE_KR 9 - -#define NES_MULTICAST_PF_MAX 8 -#define NES_A0 3 - -#define NES_ENABLE_PAU 0x07000001 -#define NES_DISABLE_PAU 0x07000000 -#define NES_PAU_COUNTER 10 -#define NES_CQP_OPCODE_MASK 0x3f - -enum pci_regs { - NES_INT_STAT = 0x0000, - NES_INT_MASK = 0x0004, - NES_INT_PENDING = 0x0008, - NES_INTF_INT_STAT = 0x000C, - NES_INTF_INT_MASK = 0x0010, - NES_TIMER_STAT = 0x0014, - NES_PERIODIC_CONTROL = 0x0018, - NES_ONE_SHOT_CONTROL = 0x001C, - NES_EEPROM_COMMAND = 0x0020, - NES_EEPROM_DATA = 0x0024, - NES_FLASH_COMMAND = 0x0028, - NES_FLASH_DATA = 0x002C, - NES_SOFTWARE_RESET = 0x0030, - NES_CQ_ACK = 0x0034, - NES_WQE_ALLOC = 0x0040, - NES_CQE_ALLOC = 0x0044, - NES_AEQ_ALLOC = 0x0048 -}; - -enum indexed_regs { - NES_IDX_CREATE_CQP_LOW = 0x0000, - NES_IDX_CREATE_CQP_HIGH = 0x0004, - NES_IDX_QP_CONTROL = 0x0040, - NES_IDX_FLM_CONTROL = 0x0080, - NES_IDX_INT_CPU_STATUS = 0x00a0, - NES_IDX_GPR_TRIGGER = 0x00bc, - NES_IDX_GPIO_CONTROL = 0x00f0, - NES_IDX_GPIO_DATA = 0x00f4, - NES_IDX_GPR2 = 0x010c, - NES_IDX_TCP_CONFIG0 = 0x01e4, - NES_IDX_TCP_TIMER_CONFIG = 0x01ec, - NES_IDX_TCP_NOW = 0x01f0, - NES_IDX_QP_MAX_CFG_SIZES = 0x0200, - NES_IDX_QP_CTX_SIZE = 0x0218, - NES_IDX_TCP_TIMER_SIZE0 = 0x0238, - NES_IDX_TCP_TIMER_SIZE1 = 0x0240, - NES_IDX_ARP_CACHE_SIZE = 0x0258, - NES_IDX_CQ_CTX_SIZE = 0x0260, - NES_IDX_MRT_SIZE = 0x0278, - NES_IDX_PBL_REGION_SIZE = 0x0280, - NES_IDX_IRRQ_COUNT = 0x02b0, - NES_IDX_RX_WINDOW_BUFFER_PAGE_TABLE_SIZE = 0x02f0, - NES_IDX_RX_WINDOW_BUFFER_SIZE = 0x0300, - NES_IDX_DST_IP_ADDR = 0x0400, - NES_IDX_PCIX_DIAG = 0x08e8, - NES_IDX_MPP_DEBUG = 0x0a00, - NES_IDX_PORT_RX_DISCARDS = 0x0a30, - NES_IDX_PORT_TX_DISCARDS = 0x0a34, - NES_IDX_MPP_LB_DEBUG = 0x0b00, - NES_IDX_DENALI_CTL_22 = 0x1058, - NES_IDX_MAC_TX_CONTROL = 0x2000, - NES_IDX_MAC_TX_CONFIG = 0x2004, - NES_IDX_MAC_TX_PAUSE_QUANTA = 0x2008, - NES_IDX_MAC_RX_CONTROL = 0x200c, - NES_IDX_MAC_RX_CONFIG = 0x2010, - NES_IDX_MAC_EXACT_MATCH_BOTTOM = 0x201c, - NES_IDX_MAC_MDIO_CONTROL = 0x2084, - NES_IDX_MAC_TX_OCTETS_LOW = 0x2100, - NES_IDX_MAC_TX_OCTETS_HIGH = 0x2104, - NES_IDX_MAC_TX_FRAMES_LOW = 0x2108, - NES_IDX_MAC_TX_FRAMES_HIGH = 0x210c, - NES_IDX_MAC_TX_PAUSE_FRAMES = 0x2118, - NES_IDX_MAC_TX_ERRORS = 0x2138, - NES_IDX_MAC_RX_OCTETS_LOW = 0x213c, - NES_IDX_MAC_RX_OCTETS_HIGH = 0x2140, - NES_IDX_MAC_RX_FRAMES_LOW = 0x2144, - NES_IDX_MAC_RX_FRAMES_HIGH = 0x2148, - NES_IDX_MAC_RX_BC_FRAMES_LOW = 0x214c, - NES_IDX_MAC_RX_MC_FRAMES_HIGH = 0x2150, - NES_IDX_MAC_RX_PAUSE_FRAMES = 0x2154, - NES_IDX_MAC_RX_SHORT_FRAMES = 0x2174, - NES_IDX_MAC_RX_OVERSIZED_FRAMES = 0x2178, - NES_IDX_MAC_RX_JABBER_FRAMES = 0x217c, - NES_IDX_MAC_RX_CRC_ERR_FRAMES = 0x2180, - NES_IDX_MAC_RX_LENGTH_ERR_FRAMES = 0x2184, - NES_IDX_MAC_RX_SYMBOL_ERR_FRAMES = 0x2188, - NES_IDX_MAC_INT_STATUS = 0x21f0, - NES_IDX_MAC_INT_MASK = 0x21f4, - NES_IDX_PHY_PCS_CONTROL_STATUS0 = 0x2800, - NES_IDX_PHY_PCS_CONTROL_STATUS1 = 0x2a00, - NES_IDX_ETH_SERDES_COMMON_CONTROL0 = 0x2808, - NES_IDX_ETH_SERDES_COMMON_CONTROL1 = 0x2a08, - NES_IDX_ETH_SERDES_COMMON_STATUS0 = 0x280c, - NES_IDX_ETH_SERDES_COMMON_STATUS1 = 0x2a0c, - NES_IDX_ETH_SERDES_TX_EMP0 = 0x2810, - NES_IDX_ETH_SERDES_TX_EMP1 = 0x2a10, - NES_IDX_ETH_SERDES_TX_DRIVE0 = 0x2814, - NES_IDX_ETH_SERDES_TX_DRIVE1 = 0x2a14, - NES_IDX_ETH_SERDES_RX_MODE0 = 0x2818, - NES_IDX_ETH_SERDES_RX_MODE1 = 0x2a18, - NES_IDX_ETH_SERDES_RX_SIGDET0 = 0x281c, - NES_IDX_ETH_SERDES_RX_SIGDET1 = 0x2a1c, - NES_IDX_ETH_SERDES_BYPASS0 = 0x2820, - NES_IDX_ETH_SERDES_BYPASS1 = 0x2a20, - NES_IDX_ETH_SERDES_LOOPBACK_CONTROL0 = 0x2824, - NES_IDX_ETH_SERDES_LOOPBACK_CONTROL1 = 0x2a24, - NES_IDX_ETH_SERDES_RX_EQ_CONTROL0 = 0x2828, - NES_IDX_ETH_SERDES_RX_EQ_CONTROL1 = 0x2a28, - NES_IDX_ETH_SERDES_RX_EQ_STATUS0 = 0x282c, - NES_IDX_ETH_SERDES_RX_EQ_STATUS1 = 0x2a2c, - NES_IDX_ETH_SERDES_CDR_RESET0 = 0x2830, - NES_IDX_ETH_SERDES_CDR_RESET1 = 0x2a30, - NES_IDX_ETH_SERDES_CDR_CONTROL0 = 0x2834, - NES_IDX_ETH_SERDES_CDR_CONTROL1 = 0x2a34, - NES_IDX_ETH_SERDES_TX_HIGHZ_LANE_MODE0 = 0x2838, - NES_IDX_ETH_SERDES_TX_HIGHZ_LANE_MODE1 = 0x2a38, - NES_IDX_ENDNODE0_NSTAT_RX_DISCARD = 0x3080, - NES_IDX_ENDNODE0_NSTAT_RX_OCTETS_LO = 0x3000, - NES_IDX_ENDNODE0_NSTAT_RX_OCTETS_HI = 0x3004, - NES_IDX_ENDNODE0_NSTAT_RX_FRAMES_LO = 0x3008, - NES_IDX_ENDNODE0_NSTAT_RX_FRAMES_HI = 0x300c, - NES_IDX_ENDNODE0_NSTAT_TX_OCTETS_LO = 0x7000, - NES_IDX_ENDNODE0_NSTAT_TX_OCTETS_HI = 0x7004, - NES_IDX_ENDNODE0_NSTAT_TX_FRAMES_LO = 0x7008, - NES_IDX_ENDNODE0_NSTAT_TX_FRAMES_HI = 0x700c, - NES_IDX_WQM_CONFIG0 = 0x5000, - NES_IDX_WQM_CONFIG1 = 0x5004, - NES_IDX_CM_CONFIG = 0x5100, - NES_IDX_NIC_LOGPORT_TO_PHYPORT = 0x6000, - NES_IDX_NIC_PHYPORT_TO_USW = 0x6008, - NES_IDX_NIC_ACTIVE = 0x6010, - NES_IDX_NIC_UNICAST_ALL = 0x6018, - NES_IDX_NIC_MULTICAST_ALL = 0x6020, - NES_IDX_NIC_MULTICAST_ENABLE = 0x6028, - NES_IDX_NIC_BROADCAST_ON = 0x6030, - NES_IDX_USED_CHUNKS_TX = 0x60b0, - NES_IDX_TX_POOL_SIZE = 0x60b8, - NES_IDX_QUAD_HASH_TABLE_SIZE = 0x6148, - NES_IDX_PERFECT_FILTER_LOW = 0x6200, - NES_IDX_PERFECT_FILTER_HIGH = 0x6204, - NES_IDX_IPV4_TCP_REXMITS = 0x7080, - NES_IDX_DEBUG_ERROR_CONTROL_STATUS = 0x913c, - NES_IDX_DEBUG_ERROR_MASKS0 = 0x9140, - NES_IDX_DEBUG_ERROR_MASKS1 = 0x9144, - NES_IDX_DEBUG_ERROR_MASKS2 = 0x9148, - NES_IDX_DEBUG_ERROR_MASKS3 = 0x914c, - NES_IDX_DEBUG_ERROR_MASKS4 = 0x9150, - NES_IDX_DEBUG_ERROR_MASKS5 = 0x9154, -}; - -#define NES_IDX_MAC_TX_CONFIG_ENABLE_PAUSE 1 -#define NES_IDX_MPP_DEBUG_PORT_DISABLE_PAUSE (1 << 17) - -enum nes_cqp_opcodes { - NES_CQP_CREATE_QP = 0x00, - NES_CQP_MODIFY_QP = 0x01, - NES_CQP_DESTROY_QP = 0x02, - NES_CQP_CREATE_CQ = 0x03, - NES_CQP_MODIFY_CQ = 0x04, - NES_CQP_DESTROY_CQ = 0x05, - NES_CQP_ALLOCATE_STAG = 0x09, - NES_CQP_REGISTER_STAG = 0x0a, - NES_CQP_QUERY_STAG = 0x0b, - NES_CQP_REGISTER_SHARED_STAG = 0x0c, - NES_CQP_DEALLOCATE_STAG = 0x0d, - NES_CQP_MANAGE_ARP_CACHE = 0x0f, - NES_CQP_DOWNLOAD_SEGMENT = 0x10, - NES_CQP_SUSPEND_QPS = 0x11, - NES_CQP_UPLOAD_CONTEXT = 0x13, - NES_CQP_CREATE_CEQ = 0x16, - NES_CQP_DESTROY_CEQ = 0x18, - NES_CQP_CREATE_AEQ = 0x19, - NES_CQP_DESTROY_AEQ = 0x1b, - NES_CQP_LMI_ACCESS = 0x20, - NES_CQP_FLUSH_WQES = 0x22, - NES_CQP_MANAGE_APBVT = 0x23, - NES_CQP_MANAGE_QUAD_HASH = 0x25 -}; - -enum nes_cqp_wqe_word_idx { - NES_CQP_WQE_OPCODE_IDX = 0, - NES_CQP_WQE_ID_IDX = 1, - NES_CQP_WQE_COMP_CTX_LOW_IDX = 2, - NES_CQP_WQE_COMP_CTX_HIGH_IDX = 3, - NES_CQP_WQE_COMP_SCRATCH_LOW_IDX = 4, - NES_CQP_WQE_COMP_SCRATCH_HIGH_IDX = 5, -}; - -enum nes_cqp_wqe_word_download_idx { /* format differs from other cqp ops */ - NES_CQP_WQE_DL_OPCODE_IDX = 0, - NES_CQP_WQE_DL_COMP_CTX_LOW_IDX = 1, - NES_CQP_WQE_DL_COMP_CTX_HIGH_IDX = 2, - NES_CQP_WQE_DL_LENGTH_0_TOTAL_IDX = 3 - /* For index values 4-15 use NES_NIC_SQ_WQE_ values */ -}; - -enum nes_cqp_cq_wqeword_idx { - NES_CQP_CQ_WQE_PBL_LOW_IDX = 6, - NES_CQP_CQ_WQE_PBL_HIGH_IDX = 7, - NES_CQP_CQ_WQE_CQ_CONTEXT_LOW_IDX = 8, - NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX = 9, - NES_CQP_CQ_WQE_DOORBELL_INDEX_HIGH_IDX = 10, -}; - -enum nes_cqp_stag_wqeword_idx { - NES_CQP_STAG_WQE_PBL_BLK_COUNT_IDX = 1, - NES_CQP_STAG_WQE_LEN_HIGH_PD_IDX = 6, - NES_CQP_STAG_WQE_LEN_LOW_IDX = 7, - NES_CQP_STAG_WQE_STAG_IDX = 8, - NES_CQP_STAG_WQE_VA_LOW_IDX = 10, - NES_CQP_STAG_WQE_VA_HIGH_IDX = 11, - NES_CQP_STAG_WQE_PA_LOW_IDX = 12, - NES_CQP_STAG_WQE_PA_HIGH_IDX = 13, - NES_CQP_STAG_WQE_PBL_LEN_IDX = 14 -}; - -#define NES_CQP_OP_LOGICAL_PORT_SHIFT 26 -#define NES_CQP_OP_IWARP_STATE_SHIFT 28 -#define NES_CQP_OP_TERMLEN_SHIFT 28 - -enum nes_cqp_qp_bits { - NES_CQP_QP_ARP_VALID = (1<<8), - NES_CQP_QP_WINBUF_VALID = (1<<9), - NES_CQP_QP_CONTEXT_VALID = (1<<10), - NES_CQP_QP_ORD_VALID = (1<<11), - NES_CQP_QP_WINBUF_DATAIND_EN = (1<<12), - NES_CQP_QP_VIRT_WQS = (1<<13), - NES_CQP_QP_DEL_HTE = (1<<14), - NES_CQP_QP_CQS_VALID = (1<<15), - NES_CQP_QP_TYPE_TSA = 0, - NES_CQP_QP_TYPE_IWARP = (1<<16), - NES_CQP_QP_TYPE_CQP = (4<<16), - NES_CQP_QP_TYPE_NIC = (5<<16), - NES_CQP_QP_MSS_CHG = (1<<20), - NES_CQP_QP_STATIC_RESOURCES = (1<<21), - NES_CQP_QP_IGNORE_MW_BOUND = (1<<22), - NES_CQP_QP_VWQ_USE_LMI = (1<<23), - NES_CQP_QP_IWARP_STATE_IDLE = (1<<NES_CQP_OP_IWARP_STATE_SHIFT), - NES_CQP_QP_IWARP_STATE_RTS = (2<<NES_CQP_OP_IWARP_STATE_SHIFT), - NES_CQP_QP_IWARP_STATE_CLOSING = (3<<NES_CQP_OP_IWARP_STATE_SHIFT), - NES_CQP_QP_IWARP_STATE_TERMINATE = (5<<NES_CQP_OP_IWARP_STATE_SHIFT), - NES_CQP_QP_IWARP_STATE_ERROR = (6<<NES_CQP_OP_IWARP_STATE_SHIFT), - NES_CQP_QP_IWARP_STATE_MASK = (7<<NES_CQP_OP_IWARP_STATE_SHIFT), - NES_CQP_QP_TERM_DONT_SEND_FIN = (1<<24), - NES_CQP_QP_TERM_DONT_SEND_TERM_MSG = (1<<25), - NES_CQP_QP_RESET = (1<<31), -}; - -enum nes_cqp_qp_wqe_word_idx { - NES_CQP_QP_WQE_CONTEXT_LOW_IDX = 6, - NES_CQP_QP_WQE_CONTEXT_HIGH_IDX = 7, - NES_CQP_QP_WQE_FLUSH_SQ_CODE = 8, - NES_CQP_QP_WQE_FLUSH_RQ_CODE = 9, - NES_CQP_QP_WQE_NEW_MSS_IDX = 15, -}; - -enum nes_nic_ctx_bits { - NES_NIC_CTX_RQ_SIZE_32 = (3<<8), - NES_NIC_CTX_RQ_SIZE_512 = (3<<8), - NES_NIC_CTX_SQ_SIZE_32 = (1<<10), - NES_NIC_CTX_SQ_SIZE_512 = (3<<10), -}; - -enum nes_nic_qp_ctx_word_idx { - NES_NIC_CTX_MISC_IDX = 0, - NES_NIC_CTX_SQ_LOW_IDX = 2, - NES_NIC_CTX_SQ_HIGH_IDX = 3, - NES_NIC_CTX_RQ_LOW_IDX = 4, - NES_NIC_CTX_RQ_HIGH_IDX = 5, -}; - -enum nes_cqp_cq_bits { - NES_CQP_CQ_CEQE_MASK = (1<<9), - NES_CQP_CQ_CEQ_VALID = (1<<10), - NES_CQP_CQ_RESIZE = (1<<11), - NES_CQP_CQ_CHK_OVERFLOW = (1<<12), - NES_CQP_CQ_4KB_CHUNK = (1<<14), - NES_CQP_CQ_VIRT = (1<<15), -}; - -enum nes_cqp_stag_bits { - NES_CQP_STAG_VA_TO = (1<<9), - NES_CQP_STAG_DEALLOC_PBLS = (1<<10), - NES_CQP_STAG_PBL_BLK_SIZE = (1<<11), - NES_CQP_STAG_MR = (1<<13), - NES_CQP_STAG_RIGHTS_LOCAL_READ = (1<<16), - NES_CQP_STAG_RIGHTS_LOCAL_WRITE = (1<<17), - NES_CQP_STAG_RIGHTS_REMOTE_READ = (1<<18), - NES_CQP_STAG_RIGHTS_REMOTE_WRITE = (1<<19), - NES_CQP_STAG_RIGHTS_WINDOW_BIND = (1<<20), - NES_CQP_STAG_REM_ACC_EN = (1<<21), - NES_CQP_STAG_LEAVE_PENDING = (1<<31), -}; - -enum nes_cqp_ceq_wqeword_idx { - NES_CQP_CEQ_WQE_ELEMENT_COUNT_IDX = 1, - NES_CQP_CEQ_WQE_PBL_LOW_IDX = 6, - NES_CQP_CEQ_WQE_PBL_HIGH_IDX = 7, -}; - -enum nes_cqp_ceq_bits { - NES_CQP_CEQ_4KB_CHUNK = (1<<14), - NES_CQP_CEQ_VIRT = (1<<15), -}; - -enum nes_cqp_aeq_wqeword_idx { - NES_CQP_AEQ_WQE_ELEMENT_COUNT_IDX = 1, - NES_CQP_AEQ_WQE_PBL_LOW_IDX = 6, - NES_CQP_AEQ_WQE_PBL_HIGH_IDX = 7, -}; - -enum nes_cqp_aeq_bits { - NES_CQP_AEQ_4KB_CHUNK = (1<<14), - NES_CQP_AEQ_VIRT = (1<<15), -}; - -enum nes_cqp_lmi_wqeword_idx { - NES_CQP_LMI_WQE_LMI_OFFSET_IDX = 1, - NES_CQP_LMI_WQE_FRAG_LOW_IDX = 8, - NES_CQP_LMI_WQE_FRAG_HIGH_IDX = 9, - NES_CQP_LMI_WQE_FRAG_LEN_IDX = 10, -}; - -enum nes_cqp_arp_wqeword_idx { - NES_CQP_ARP_WQE_MAC_ADDR_LOW_IDX = 6, - NES_CQP_ARP_WQE_MAC_HIGH_IDX = 7, - NES_CQP_ARP_WQE_REACHABILITY_MAX_IDX = 1, -}; - -enum nes_cqp_upload_wqeword_idx { - NES_CQP_UPLOAD_WQE_CTXT_LOW_IDX = 6, - NES_CQP_UPLOAD_WQE_CTXT_HIGH_IDX = 7, - NES_CQP_UPLOAD_WQE_HTE_IDX = 8, -}; - -enum nes_cqp_arp_bits { - NES_CQP_ARP_VALID = (1<<8), - NES_CQP_ARP_PERM = (1<<9), -}; - -enum nes_cqp_flush_bits { - NES_CQP_FLUSH_SQ = (1<<30), - NES_CQP_FLUSH_RQ = (1<<31), - NES_CQP_FLUSH_MAJ_MIN = (1<<28), -}; - -enum nes_cqe_opcode_bits { - NES_CQE_STAG_VALID = (1<<6), - NES_CQE_ERROR = (1<<7), - NES_CQE_SQ = (1<<8), - NES_CQE_SE = (1<<9), - NES_CQE_PSH = (1<<29), - NES_CQE_FIN = (1<<30), - NES_CQE_VALID = (1<<31), -}; - - -enum nes_cqe_word_idx { - NES_CQE_PAYLOAD_LENGTH_IDX = 0, - NES_CQE_COMP_COMP_CTX_LOW_IDX = 2, - NES_CQE_COMP_COMP_CTX_HIGH_IDX = 3, - NES_CQE_INV_STAG_IDX = 4, - NES_CQE_QP_ID_IDX = 5, - NES_CQE_ERROR_CODE_IDX = 6, - NES_CQE_OPCODE_IDX = 7, -}; - -enum nes_ceqe_word_idx { - NES_CEQE_CQ_CTX_LOW_IDX = 0, - NES_CEQE_CQ_CTX_HIGH_IDX = 1, -}; - -enum nes_ceqe_status_bit { - NES_CEQE_VALID = (1<<31), -}; - -enum nes_int_bits { - NES_INT_CEQ0 = (1<<0), - NES_INT_CEQ1 = (1<<1), - NES_INT_CEQ2 = (1<<2), - NES_INT_CEQ3 = (1<<3), - NES_INT_CEQ4 = (1<<4), - NES_INT_CEQ5 = (1<<5), - NES_INT_CEQ6 = (1<<6), - NES_INT_CEQ7 = (1<<7), - NES_INT_CEQ8 = (1<<8), - NES_INT_CEQ9 = (1<<9), - NES_INT_CEQ10 = (1<<10), - NES_INT_CEQ11 = (1<<11), - NES_INT_CEQ12 = (1<<12), - NES_INT_CEQ13 = (1<<13), - NES_INT_CEQ14 = (1<<14), - NES_INT_CEQ15 = (1<<15), - NES_INT_AEQ0 = (1<<16), - NES_INT_AEQ1 = (1<<17), - NES_INT_AEQ2 = (1<<18), - NES_INT_AEQ3 = (1<<19), - NES_INT_AEQ4 = (1<<20), - NES_INT_AEQ5 = (1<<21), - NES_INT_AEQ6 = (1<<22), - NES_INT_AEQ7 = (1<<23), - NES_INT_MAC0 = (1<<24), - NES_INT_MAC1 = (1<<25), - NES_INT_MAC2 = (1<<26), - NES_INT_MAC3 = (1<<27), - NES_INT_TSW = (1<<28), - NES_INT_TIMER = (1<<29), - NES_INT_INTF = (1<<30), -}; - -enum nes_intf_int_bits { - NES_INTF_INT_PCIERR = (1<<0), - NES_INTF_PERIODIC_TIMER = (1<<2), - NES_INTF_ONE_SHOT_TIMER = (1<<3), - NES_INTF_INT_CRITERR = (1<<14), - NES_INTF_INT_AEQ0_OFLOW = (1<<16), - NES_INTF_INT_AEQ1_OFLOW = (1<<17), - NES_INTF_INT_AEQ2_OFLOW = (1<<18), - NES_INTF_INT_AEQ3_OFLOW = (1<<19), - NES_INTF_INT_AEQ4_OFLOW = (1<<20), - NES_INTF_INT_AEQ5_OFLOW = (1<<21), - NES_INTF_INT_AEQ6_OFLOW = (1<<22), - NES_INTF_INT_AEQ7_OFLOW = (1<<23), - NES_INTF_INT_AEQ_OFLOW = (0xff<<16), -}; - -enum nes_mac_int_bits { - NES_MAC_INT_LINK_STAT_CHG = (1<<1), - NES_MAC_INT_XGMII_EXT = (1<<2), - NES_MAC_INT_TX_UNDERFLOW = (1<<6), - NES_MAC_INT_TX_ERROR = (1<<7), -}; - -enum nes_cqe_allocate_bits { - NES_CQE_ALLOC_INC_SELECT = (1<<28), - NES_CQE_ALLOC_NOTIFY_NEXT = (1<<29), - NES_CQE_ALLOC_NOTIFY_SE = (1<<30), - NES_CQE_ALLOC_RESET = (1<<31), -}; - -enum nes_nic_rq_wqe_word_idx { - NES_NIC_RQ_WQE_LENGTH_1_0_IDX = 0, - NES_NIC_RQ_WQE_LENGTH_3_2_IDX = 1, - NES_NIC_RQ_WQE_FRAG0_LOW_IDX = 2, - NES_NIC_RQ_WQE_FRAG0_HIGH_IDX = 3, - NES_NIC_RQ_WQE_FRAG1_LOW_IDX = 4, - NES_NIC_RQ_WQE_FRAG1_HIGH_IDX = 5, - NES_NIC_RQ_WQE_FRAG2_LOW_IDX = 6, - NES_NIC_RQ_WQE_FRAG2_HIGH_IDX = 7, - NES_NIC_RQ_WQE_FRAG3_LOW_IDX = 8, - NES_NIC_RQ_WQE_FRAG3_HIGH_IDX = 9, -}; - -enum nes_nic_sq_wqe_word_idx { - NES_NIC_SQ_WQE_MISC_IDX = 0, - NES_NIC_SQ_WQE_TOTAL_LENGTH_IDX = 1, - NES_NIC_SQ_WQE_LSO_INFO_IDX = 2, - NES_NIC_SQ_WQE_LENGTH_0_TAG_IDX = 3, - NES_NIC_SQ_WQE_LENGTH_2_1_IDX = 4, - NES_NIC_SQ_WQE_LENGTH_4_3_IDX = 5, - NES_NIC_SQ_WQE_FRAG0_LOW_IDX = 6, - NES_NIC_SQ_WQE_FRAG0_HIGH_IDX = 7, - NES_NIC_SQ_WQE_FRAG1_LOW_IDX = 8, - NES_NIC_SQ_WQE_FRAG1_HIGH_IDX = 9, - NES_NIC_SQ_WQE_FRAG2_LOW_IDX = 10, - NES_NIC_SQ_WQE_FRAG2_HIGH_IDX = 11, - NES_NIC_SQ_WQE_FRAG3_LOW_IDX = 12, - NES_NIC_SQ_WQE_FRAG3_HIGH_IDX = 13, - NES_NIC_SQ_WQE_FRAG4_LOW_IDX = 14, - NES_NIC_SQ_WQE_FRAG4_HIGH_IDX = 15, -}; - -enum nes_iwarp_sq_wqe_word_idx { - NES_IWARP_SQ_WQE_MISC_IDX = 0, - NES_IWARP_SQ_WQE_TOTAL_PAYLOAD_IDX = 1, - NES_IWARP_SQ_WQE_COMP_CTX_LOW_IDX = 2, - NES_IWARP_SQ_WQE_COMP_CTX_HIGH_IDX = 3, - NES_IWARP_SQ_WQE_COMP_SCRATCH_LOW_IDX = 4, - NES_IWARP_SQ_WQE_COMP_SCRATCH_HIGH_IDX = 5, - NES_IWARP_SQ_WQE_INV_STAG_LOW_IDX = 7, - NES_IWARP_SQ_WQE_RDMA_TO_LOW_IDX = 8, - NES_IWARP_SQ_WQE_RDMA_TO_HIGH_IDX = 9, - NES_IWARP_SQ_WQE_RDMA_LENGTH_IDX = 10, - NES_IWARP_SQ_WQE_RDMA_STAG_IDX = 11, - NES_IWARP_SQ_WQE_IMM_DATA_START_IDX = 12, - NES_IWARP_SQ_WQE_FRAG0_LOW_IDX = 16, - NES_IWARP_SQ_WQE_FRAG0_HIGH_IDX = 17, - NES_IWARP_SQ_WQE_LENGTH0_IDX = 18, - NES_IWARP_SQ_WQE_STAG0_IDX = 19, - NES_IWARP_SQ_WQE_FRAG1_LOW_IDX = 20, - NES_IWARP_SQ_WQE_FRAG1_HIGH_IDX = 21, - NES_IWARP_SQ_WQE_LENGTH1_IDX = 22, - NES_IWARP_SQ_WQE_STAG1_IDX = 23, - NES_IWARP_SQ_WQE_FRAG2_LOW_IDX = 24, - NES_IWARP_SQ_WQE_FRAG2_HIGH_IDX = 25, - NES_IWARP_SQ_WQE_LENGTH2_IDX = 26, - NES_IWARP_SQ_WQE_STAG2_IDX = 27, - NES_IWARP_SQ_WQE_FRAG3_LOW_IDX = 28, - NES_IWARP_SQ_WQE_FRAG3_HIGH_IDX = 29, - NES_IWARP_SQ_WQE_LENGTH3_IDX = 30, - NES_IWARP_SQ_WQE_STAG3_IDX = 31, -}; - -enum nes_iwarp_sq_bind_wqe_word_idx { - NES_IWARP_SQ_BIND_WQE_MR_IDX = 6, - NES_IWARP_SQ_BIND_WQE_MW_IDX = 7, - NES_IWARP_SQ_BIND_WQE_LENGTH_LOW_IDX = 8, - NES_IWARP_SQ_BIND_WQE_LENGTH_HIGH_IDX = 9, - NES_IWARP_SQ_BIND_WQE_VA_FBO_LOW_IDX = 10, - NES_IWARP_SQ_BIND_WQE_VA_FBO_HIGH_IDX = 11, -}; - -enum nes_iwarp_sq_fmr_wqe_word_idx { - NES_IWARP_SQ_FMR_WQE_MR_STAG_IDX = 7, - NES_IWARP_SQ_FMR_WQE_LENGTH_LOW_IDX = 8, - NES_IWARP_SQ_FMR_WQE_LENGTH_HIGH_IDX = 9, - NES_IWARP_SQ_FMR_WQE_VA_FBO_LOW_IDX = 10, - NES_IWARP_SQ_FMR_WQE_VA_FBO_HIGH_IDX = 11, - NES_IWARP_SQ_FMR_WQE_PBL_ADDR_LOW_IDX = 12, - NES_IWARP_SQ_FMR_WQE_PBL_ADDR_HIGH_IDX = 13, - NES_IWARP_SQ_FMR_WQE_PBL_LENGTH_IDX = 14, -}; - -enum nes_iwarp_sq_fmr_opcodes { - NES_IWARP_SQ_FMR_WQE_ZERO_BASED = (1<<6), - NES_IWARP_SQ_FMR_WQE_PAGE_SIZE_4K = (0<<7), - NES_IWARP_SQ_FMR_WQE_PAGE_SIZE_2M = (1<<7), - NES_IWARP_SQ_FMR_WQE_RIGHTS_ENABLE_LOCAL_READ = (1<<16), - NES_IWARP_SQ_FMR_WQE_RIGHTS_ENABLE_LOCAL_WRITE = (1<<17), - NES_IWARP_SQ_FMR_WQE_RIGHTS_ENABLE_REMOTE_READ = (1<<18), - NES_IWARP_SQ_FMR_WQE_RIGHTS_ENABLE_REMOTE_WRITE = (1<<19), - NES_IWARP_SQ_FMR_WQE_RIGHTS_ENABLE_WINDOW_BIND = (1<<20), -}; - -#define NES_IWARP_SQ_FMR_WQE_MR_LENGTH_HIGH_MASK 0xFF; - -enum nes_iwarp_sq_locinv_wqe_word_idx { - NES_IWARP_SQ_LOCINV_WQE_INV_STAG_IDX = 6, -}; - -enum nes_iwarp_rq_wqe_word_idx { - NES_IWARP_RQ_WQE_TOTAL_PAYLOAD_IDX = 1, - NES_IWARP_RQ_WQE_COMP_CTX_LOW_IDX = 2, - NES_IWARP_RQ_WQE_COMP_CTX_HIGH_IDX = 3, - NES_IWARP_RQ_WQE_COMP_SCRATCH_LOW_IDX = 4, - NES_IWARP_RQ_WQE_COMP_SCRATCH_HIGH_IDX = 5, - NES_IWARP_RQ_WQE_FRAG0_LOW_IDX = 8, - NES_IWARP_RQ_WQE_FRAG0_HIGH_IDX = 9, - NES_IWARP_RQ_WQE_LENGTH0_IDX = 10, - NES_IWARP_RQ_WQE_STAG0_IDX = 11, - NES_IWARP_RQ_WQE_FRAG1_LOW_IDX = 12, - NES_IWARP_RQ_WQE_FRAG1_HIGH_IDX = 13, - NES_IWARP_RQ_WQE_LENGTH1_IDX = 14, - NES_IWARP_RQ_WQE_STAG1_IDX = 15, - NES_IWARP_RQ_WQE_FRAG2_LOW_IDX = 16, - NES_IWARP_RQ_WQE_FRAG2_HIGH_IDX = 17, - NES_IWARP_RQ_WQE_LENGTH2_IDX = 18, - NES_IWARP_RQ_WQE_STAG2_IDX = 19, - NES_IWARP_RQ_WQE_FRAG3_LOW_IDX = 20, - NES_IWARP_RQ_WQE_FRAG3_HIGH_IDX = 21, - NES_IWARP_RQ_WQE_LENGTH3_IDX = 22, - NES_IWARP_RQ_WQE_STAG3_IDX = 23, -}; - -enum nes_nic_sq_wqe_bits { - NES_NIC_SQ_WQE_PHDR_CS_READY = (1<<21), - NES_NIC_SQ_WQE_LSO_ENABLE = (1<<22), - NES_NIC_SQ_WQE_TAGVALUE_ENABLE = (1<<23), - NES_NIC_SQ_WQE_DISABLE_CHKSUM = (1<<30), - NES_NIC_SQ_WQE_COMPLETION = (1<<31), -}; - -enum nes_nic_cqe_word_idx { - NES_NIC_CQE_ACCQP_ID_IDX = 0, - NES_NIC_CQE_HASH_RCVNXT = 1, - NES_NIC_CQE_TAG_PKT_TYPE_IDX = 2, - NES_NIC_CQE_MISC_IDX = 3, -}; - -#define NES_PKT_TYPE_APBVT_BITS 0xC112 -#define NES_PKT_TYPE_APBVT_MASK 0xff3e - -#define NES_PKT_TYPE_PVALID_BITS 0x10000000 -#define NES_PKT_TYPE_PVALID_MASK 0x30000000 - -#define NES_PKT_TYPE_TCPV4_BITS 0x0110 -#define NES_PKT_TYPE_TCPV4_MASK 0x3f30 - -#define NES_PKT_TYPE_UDPV4_BITS 0x0210 -#define NES_PKT_TYPE_UDPV4_MASK 0x3f30 - -#define NES_PKT_TYPE_IPV4_BITS 0x0010 -#define NES_PKT_TYPE_IPV4_MASK 0x3f30 - -#define NES_PKT_TYPE_OTHER_BITS 0x0000 -#define NES_PKT_TYPE_OTHER_MASK 0x0030 - -#define NES_NIC_CQE_ERRV_SHIFT 16 -enum nes_nic_ev_bits { - NES_NIC_ERRV_BITS_MODE = (1<<0), - NES_NIC_ERRV_BITS_IPV4_CSUM_ERR = (1<<1), - NES_NIC_ERRV_BITS_TCPUDP_CSUM_ERR = (1<<2), - NES_NIC_ERRV_BITS_WQE_OVERRUN = (1<<3), - NES_NIC_ERRV_BITS_IPH_ERR = (1<<4), -}; - -enum nes_nic_cqe_bits { - NES_NIC_CQE_ERRV_MASK = (0xff<<NES_NIC_CQE_ERRV_SHIFT), - NES_NIC_CQE_SQ = (1<<24), - NES_NIC_CQE_ACCQP_PORT = (1<<28), - NES_NIC_CQE_ACCQP_VALID = (1<<29), - NES_NIC_CQE_TAG_VALID = (1<<30), - NES_NIC_CQE_VALID = (1<<31), -}; - -enum nes_aeqe_word_idx { - NES_AEQE_COMP_CTXT_LOW_IDX = 0, - NES_AEQE_COMP_CTXT_HIGH_IDX = 1, - NES_AEQE_COMP_QP_CQ_ID_IDX = 2, - NES_AEQE_MISC_IDX = 3, -}; - -enum nes_aeqe_bits { - NES_AEQE_QP = (1<<16), - NES_AEQE_CQ = (1<<17), - NES_AEQE_SQ = (1<<18), - NES_AEQE_INBOUND_RDMA = (1<<19), - NES_AEQE_IWARP_STATE_MASK = (7<<20), - NES_AEQE_TCP_STATE_MASK = (0xf<<24), - NES_AEQE_Q2_DATA_WRITTEN = (0x3<<28), - NES_AEQE_VALID = (1<<31), -}; - -#define NES_AEQE_IWARP_STATE_SHIFT 20 -#define NES_AEQE_TCP_STATE_SHIFT 24 -#define NES_AEQE_Q2_DATA_ETHERNET (1<<28) -#define NES_AEQE_Q2_DATA_MPA (1<<29) - -enum nes_aeqe_iwarp_state { - NES_AEQE_IWARP_STATE_NON_EXISTANT = 0, - NES_AEQE_IWARP_STATE_IDLE = 1, - NES_AEQE_IWARP_STATE_RTS = 2, - NES_AEQE_IWARP_STATE_CLOSING = 3, - NES_AEQE_IWARP_STATE_TERMINATE = 5, - NES_AEQE_IWARP_STATE_ERROR = 6 -}; - -enum nes_aeqe_tcp_state { - NES_AEQE_TCP_STATE_NON_EXISTANT = 0, - NES_AEQE_TCP_STATE_CLOSED = 1, - NES_AEQE_TCP_STATE_LISTEN = 2, - NES_AEQE_TCP_STATE_SYN_SENT = 3, - NES_AEQE_TCP_STATE_SYN_RCVD = 4, - NES_AEQE_TCP_STATE_ESTABLISHED = 5, - NES_AEQE_TCP_STATE_CLOSE_WAIT = 6, - NES_AEQE_TCP_STATE_FIN_WAIT_1 = 7, - NES_AEQE_TCP_STATE_CLOSING = 8, - NES_AEQE_TCP_STATE_LAST_ACK = 9, - NES_AEQE_TCP_STATE_FIN_WAIT_2 = 10, - NES_AEQE_TCP_STATE_TIME_WAIT = 11 -}; - -enum nes_aeqe_aeid { - NES_AEQE_AEID_AMP_UNALLOCATED_STAG = 0x0102, - NES_AEQE_AEID_AMP_INVALID_STAG = 0x0103, - NES_AEQE_AEID_AMP_BAD_QP = 0x0104, - NES_AEQE_AEID_AMP_BAD_PD = 0x0105, - NES_AEQE_AEID_AMP_BAD_STAG_KEY = 0x0106, - NES_AEQE_AEID_AMP_BAD_STAG_INDEX = 0x0107, - NES_AEQE_AEID_AMP_BOUNDS_VIOLATION = 0x0108, - NES_AEQE_AEID_AMP_RIGHTS_VIOLATION = 0x0109, - NES_AEQE_AEID_AMP_TO_WRAP = 0x010a, - NES_AEQE_AEID_AMP_FASTREG_SHARED = 0x010b, - NES_AEQE_AEID_AMP_FASTREG_VALID_STAG = 0x010c, - NES_AEQE_AEID_AMP_FASTREG_MW_STAG = 0x010d, - NES_AEQE_AEID_AMP_FASTREG_INVALID_RIGHTS = 0x010e, - NES_AEQE_AEID_AMP_FASTREG_PBL_TABLE_OVERFLOW = 0x010f, - NES_AEQE_AEID_AMP_FASTREG_INVALID_LENGTH = 0x0110, - NES_AEQE_AEID_AMP_INVALIDATE_SHARED = 0x0111, - NES_AEQE_AEID_AMP_INVALIDATE_NO_REMOTE_ACCESS_RIGHTS = 0x0112, - NES_AEQE_AEID_AMP_INVALIDATE_MR_WITH_BOUND_WINDOWS = 0x0113, - NES_AEQE_AEID_AMP_MWBIND_VALID_STAG = 0x0114, - NES_AEQE_AEID_AMP_MWBIND_OF_MR_STAG = 0x0115, - NES_AEQE_AEID_AMP_MWBIND_TO_ZERO_BASED_STAG = 0x0116, - NES_AEQE_AEID_AMP_MWBIND_TO_MW_STAG = 0x0117, - NES_AEQE_AEID_AMP_MWBIND_INVALID_RIGHTS = 0x0118, - NES_AEQE_AEID_AMP_MWBIND_INVALID_BOUNDS = 0x0119, - NES_AEQE_AEID_AMP_MWBIND_TO_INVALID_PARENT = 0x011a, - NES_AEQE_AEID_AMP_MWBIND_BIND_DISABLED = 0x011b, - NES_AEQE_AEID_BAD_CLOSE = 0x0201, - NES_AEQE_AEID_RDMAP_ROE_BAD_LLP_CLOSE = 0x0202, - NES_AEQE_AEID_CQ_OPERATION_ERROR = 0x0203, - NES_AEQE_AEID_PRIV_OPERATION_DENIED = 0x0204, - NES_AEQE_AEID_RDMA_READ_WHILE_ORD_ZERO = 0x0205, - NES_AEQE_AEID_STAG_ZERO_INVALID = 0x0206, - NES_AEQE_AEID_DDP_INVALID_MSN_GAP_IN_MSN = 0x0301, - NES_AEQE_AEID_DDP_INVALID_MSN_RANGE_IS_NOT_VALID = 0x0302, - NES_AEQE_AEID_DDP_UBE_DDP_MESSAGE_TOO_LONG_FOR_AVAILABLE_BUFFER = 0x0303, - NES_AEQE_AEID_DDP_UBE_INVALID_DDP_VERSION = 0x0304, - NES_AEQE_AEID_DDP_UBE_INVALID_MO = 0x0305, - NES_AEQE_AEID_DDP_UBE_INVALID_MSN_NO_BUFFER_AVAILABLE = 0x0306, - NES_AEQE_AEID_DDP_UBE_INVALID_QN = 0x0307, - NES_AEQE_AEID_DDP_NO_L_BIT = 0x0308, - NES_AEQE_AEID_RDMAP_ROE_INVALID_RDMAP_VERSION = 0x0311, - NES_AEQE_AEID_RDMAP_ROE_UNEXPECTED_OPCODE = 0x0312, - NES_AEQE_AEID_ROE_INVALID_RDMA_READ_REQUEST = 0x0313, - NES_AEQE_AEID_ROE_INVALID_RDMA_WRITE_OR_READ_RESP = 0x0314, - NES_AEQE_AEID_INVALID_ARP_ENTRY = 0x0401, - NES_AEQE_AEID_INVALID_TCP_OPTION_RCVD = 0x0402, - NES_AEQE_AEID_STALE_ARP_ENTRY = 0x0403, - NES_AEQE_AEID_LLP_CLOSE_COMPLETE = 0x0501, - NES_AEQE_AEID_LLP_CONNECTION_RESET = 0x0502, - NES_AEQE_AEID_LLP_FIN_RECEIVED = 0x0503, - NES_AEQE_AEID_LLP_RECEIVED_MARKER_AND_LENGTH_FIELDS_DONT_MATCH = 0x0504, - NES_AEQE_AEID_LLP_RECEIVED_MPA_CRC_ERROR = 0x0505, - NES_AEQE_AEID_LLP_SEGMENT_TOO_LARGE = 0x0506, - NES_AEQE_AEID_LLP_SEGMENT_TOO_SMALL = 0x0507, - NES_AEQE_AEID_LLP_SYN_RECEIVED = 0x0508, - NES_AEQE_AEID_LLP_TERMINATE_RECEIVED = 0x0509, - NES_AEQE_AEID_LLP_TOO_MANY_RETRIES = 0x050a, - NES_AEQE_AEID_LLP_TOO_MANY_KEEPALIVE_RETRIES = 0x050b, - NES_AEQE_AEID_RESET_SENT = 0x0601, - NES_AEQE_AEID_TERMINATE_SENT = 0x0602, - NES_AEQE_AEID_DDP_LCE_LOCAL_CATASTROPHIC = 0x0700 -}; - -enum nes_iwarp_sq_opcodes { - NES_IWARP_SQ_WQE_WRPDU = (1<<15), - NES_IWARP_SQ_WQE_PSH = (1<<21), - NES_IWARP_SQ_WQE_STREAMING = (1<<23), - NES_IWARP_SQ_WQE_IMM_DATA = (1<<28), - NES_IWARP_SQ_WQE_READ_FENCE = (1<<29), - NES_IWARP_SQ_WQE_LOCAL_FENCE = (1<<30), - NES_IWARP_SQ_WQE_SIGNALED_COMPL = (1<<31), -}; - -enum nes_iwarp_sq_wqe_bits { - NES_IWARP_SQ_OP_RDMAW = 0, - NES_IWARP_SQ_OP_RDMAR = 1, - NES_IWARP_SQ_OP_SEND = 3, - NES_IWARP_SQ_OP_SENDINV = 4, - NES_IWARP_SQ_OP_SENDSE = 5, - NES_IWARP_SQ_OP_SENDSEINV = 6, - NES_IWARP_SQ_OP_BIND = 8, - NES_IWARP_SQ_OP_FAST_REG = 9, - NES_IWARP_SQ_OP_LOCINV = 10, - NES_IWARP_SQ_OP_RDMAR_LOCINV = 11, - NES_IWARP_SQ_OP_NOP = 12, -}; - -enum nes_iwarp_cqe_major_code { - NES_IWARP_CQE_MAJOR_FLUSH = 1, - NES_IWARP_CQE_MAJOR_DRV = 0x8000 -}; - -enum nes_iwarp_cqe_minor_code { - NES_IWARP_CQE_MINOR_FLUSH = 1 -}; - -#define NES_EEPROM_READ_REQUEST (1<<16) -#define NES_MAC_ADDR_VALID (1<<20) - -/* - * NES index registers init values. - */ -struct nes_init_values { - u32 index; - u32 data; - u8 wrt; -}; - -/* - * NES registers in BAR0. - */ -struct nes_pci_regs { - u32 int_status; - u32 int_mask; - u32 int_pending; - u32 intf_int_status; - u32 intf_int_mask; - u32 other_regs[59]; /* pad out to 256 bytes for now */ -}; - -#define NES_CQP_SQ_SIZE 128 -#define NES_CCQ_SIZE 128 -#define NES_NIC_WQ_SIZE 512 -#define NES_NIC_CTX_SIZE ((NES_NIC_CTX_RQ_SIZE_512) | (NES_NIC_CTX_SQ_SIZE_512)) -#define NES_NIC_BACK_STORE 0x00038000 - -struct nes_device; - -struct nes_hw_nic_qp_context { - __le32 context_words[6]; -}; - -struct nes_hw_nic_sq_wqe { - __le32 wqe_words[16]; -}; - -struct nes_hw_nic_rq_wqe { - __le32 wqe_words[16]; -}; - -struct nes_hw_nic_cqe { - __le32 cqe_words[4]; -}; - -struct nes_hw_cqp_qp_context { - __le32 context_words[4]; -}; - -struct nes_hw_cqp_wqe { - __le32 wqe_words[16]; -}; - -struct nes_hw_qp_wqe { - __le32 wqe_words[32]; -}; - -struct nes_hw_cqe { - __le32 cqe_words[8]; -}; - -struct nes_hw_ceqe { - __le32 ceqe_words[2]; -}; - -struct nes_hw_aeqe { - __le32 aeqe_words[4]; -}; - -struct nes_cqp_request { - union { - u64 cqp_callback_context; - void *cqp_callback_pointer; - }; - wait_queue_head_t waitq; - struct nes_hw_cqp_wqe cqp_wqe; - struct list_head list; - atomic_t refcount; - void (*cqp_callback)(struct nes_device *nesdev, struct nes_cqp_request *cqp_request); - u16 major_code; - u16 minor_code; - u8 waiting; - u8 request_done; - u8 dynamic; - u8 callback; -}; - -struct nes_hw_cqp { - struct nes_hw_cqp_wqe *sq_vbase; - dma_addr_t sq_pbase; - spinlock_t lock; - wait_queue_head_t waitq; - u16 qp_id; - u16 sq_head; - u16 sq_tail; - u16 sq_size; -}; - -#define NES_FIRST_FRAG_SIZE 128 -struct nes_first_frag { - u8 buffer[NES_FIRST_FRAG_SIZE]; -}; - -struct nes_hw_nic { - struct nes_first_frag *first_frag_vbase; /* virtual address of first frags */ - struct nes_hw_nic_sq_wqe *sq_vbase; /* virtual address of sq */ - struct nes_hw_nic_rq_wqe *rq_vbase; /* virtual address of rq */ - struct sk_buff *tx_skb[NES_NIC_WQ_SIZE]; - struct sk_buff *rx_skb[NES_NIC_WQ_SIZE]; - dma_addr_t frag_paddr[NES_NIC_WQ_SIZE]; - unsigned long first_frag_overflow[BITS_TO_LONGS(NES_NIC_WQ_SIZE)]; - dma_addr_t sq_pbase; /* PCI memory for host rings */ - dma_addr_t rq_pbase; /* PCI memory for host rings */ - - u16 qp_id; - u16 sq_head; - u16 sq_tail; - u16 sq_size; - u16 rq_head; - u16 rq_tail; - u16 rq_size; - u8 replenishing_rq; - u8 reserved; - - spinlock_t rq_lock; -}; - -struct nes_hw_nic_cq { - struct nes_hw_nic_cqe volatile *cq_vbase; /* PCI memory for host rings */ - void (*ce_handler)(struct nes_device *nesdev, struct nes_hw_nic_cq *cq); - dma_addr_t cq_pbase; /* PCI memory for host rings */ - int rx_cqes_completed; - int cqe_allocs_pending; - int rx_pkts_indicated; - u16 cq_head; - u16 cq_size; - u16 cq_number; - u8 cqes_pending; -}; - -struct nes_hw_qp { - struct nes_hw_qp_wqe *sq_vbase; /* PCI memory for host rings */ - struct nes_hw_qp_wqe *rq_vbase; /* PCI memory for host rings */ - void *q2_vbase; /* PCI memory for host rings */ - dma_addr_t sq_pbase; /* PCI memory for host rings */ - dma_addr_t rq_pbase; /* PCI memory for host rings */ - dma_addr_t q2_pbase; /* PCI memory for host rings */ - u32 qp_id; - u16 sq_head; - u16 sq_tail; - u16 sq_size; - u16 rq_head; - u16 rq_tail; - u16 rq_size; - u8 rq_encoded_size; - u8 sq_encoded_size; -}; - -struct nes_hw_cq { - struct nes_hw_cqe *cq_vbase; /* PCI memory for host rings */ - void (*ce_handler)(struct nes_device *nesdev, struct nes_hw_cq *cq); - dma_addr_t cq_pbase; /* PCI memory for host rings */ - u16 cq_head; - u16 cq_size; - u16 cq_number; -}; - -struct nes_hw_ceq { - struct nes_hw_ceqe volatile *ceq_vbase; /* PCI memory for host rings */ - dma_addr_t ceq_pbase; /* PCI memory for host rings */ - u16 ceq_head; - u16 ceq_size; -}; - -struct nes_hw_aeq { - struct nes_hw_aeqe volatile *aeq_vbase; /* PCI memory for host rings */ - dma_addr_t aeq_pbase; /* PCI memory for host rings */ - u16 aeq_head; - u16 aeq_size; -}; - -struct nic_qp_map { - u8 qpid; - u8 nic_index; - u8 logical_port; - u8 is_hnic; -}; - -#define NES_CQP_ARP_AEQ_INDEX_MASK 0x000f0000 -#define NES_CQP_ARP_AEQ_INDEX_SHIFT 16 - -#define NES_CQP_APBVT_ADD 0x00008000 -#define NES_CQP_APBVT_NIC_SHIFT 16 - -#define NES_ARP_ADD 1 -#define NES_ARP_DELETE 2 -#define NES_ARP_RESOLVE 3 - -#define NES_MAC_SW_IDLE 0 -#define NES_MAC_SW_INTERRUPT 1 -#define NES_MAC_SW_MH 2 - -struct nes_arp_entry { - u32 ip_addr; - u8 mac_addr[ETH_ALEN]; -}; - -#define NES_NIC_FAST_TIMER 96 -#define NES_NIC_FAST_TIMER_LOW 40 -#define NES_NIC_FAST_TIMER_HIGH 1000 -#define DEFAULT_NES_QL_HIGH 256 -#define DEFAULT_NES_QL_LOW 16 -#define DEFAULT_NES_QL_TARGET 64 -#define DEFAULT_JUMBO_NES_QL_LOW 12 -#define DEFAULT_JUMBO_NES_QL_TARGET 40 -#define DEFAULT_JUMBO_NES_QL_HIGH 128 -#define NES_NIC_CQ_DOWNWARD_TREND 16 -#define NES_PFT_SIZE 48 - -#define NES_MGT_WQ_COUNT 32 -#define NES_MGT_CTX_SIZE ((NES_NIC_CTX_RQ_SIZE_32) | (NES_NIC_CTX_SQ_SIZE_32)) -#define NES_MGT_QP_OFFSET 36 -#define NES_MGT_QP_COUNT 4 - -struct nes_hw_tune_timer { - /* u16 cq_count; */ - u16 threshold_low; - u16 threshold_target; - u16 threshold_high; - u16 timer_in_use; - u16 timer_in_use_old; - u16 timer_in_use_min; - u16 timer_in_use_max; - u8 timer_direction_upward; - u8 timer_direction_downward; - u16 cq_count_old; - u8 cq_direction_downward; -}; - -#define NES_TIMER_INT_LIMIT 2 -#define NES_TIMER_INT_LIMIT_DYNAMIC 10 -#define NES_TIMER_ENABLE_LIMIT 4 -#define NES_MAX_LINK_INTERRUPTS 128 -#define NES_MAX_LINK_CHECK 200 - -struct nes_adapter { - u64 fw_ver; - unsigned long *allocated_qps; - unsigned long *allocated_cqs; - unsigned long *allocated_mrs; - unsigned long *allocated_pds; - unsigned long *allocated_arps; - struct nes_qp **qp_table; - struct workqueue_struct *work_q; - - struct list_head list; - struct list_head active_listeners; - /* list of the netdev's associated with each logical port */ - struct list_head nesvnic_list[4]; - - struct timer_list mh_timer; - struct timer_list lc_timer; - struct work_struct work; - spinlock_t resource_lock; - spinlock_t phy_lock; - spinlock_t pbl_lock; - spinlock_t periodic_timer_lock; - - struct nes_arp_entry arp_table[NES_MAX_ARP_TABLE_SIZE]; - - /* Adapter CEQ and AEQs */ - struct nes_hw_ceq ceq[16]; - struct nes_hw_aeq aeq[8]; - - struct nes_hw_tune_timer tune_timer; - - unsigned long doorbell_start; - - u32 hw_rev; - u32 vendor_id; - u32 vendor_part_id; - u32 device_cap_flags; - u32 tick_delta; - u32 timer_int_req; - u32 arp_table_size; - u32 next_arp_index; - - u32 max_mr; - u32 max_256pbl; - u32 max_4kpbl; - u32 free_256pbl; - u32 free_4kpbl; - u32 max_mr_size; - u32 max_qp; - u32 next_qp; - u32 max_irrq; - u32 max_qp_wr; - u32 max_sge; - u32 max_cq; - u32 next_cq; - u32 max_cqe; - u32 max_pd; - u32 base_pd; - u32 next_pd; - u32 hte_index_mask; - - /* EEPROM information */ - u32 rx_pool_size; - u32 tx_pool_size; - u32 rx_threshold; - u32 tcp_timer_core_clk_divisor; - u32 iwarp_config; - u32 cm_config; - u32 sws_timer_config; - u32 tcp_config1; - u32 wqm_wat; - u32 core_clock; - u32 firmware_version; - u32 eeprom_version; - - u32 nic_rx_eth_route_err; - - u32 et_rx_coalesce_usecs; - u32 et_rx_max_coalesced_frames; - u32 et_rx_coalesce_usecs_irq; - u32 et_rx_max_coalesced_frames_irq; - u32 et_pkt_rate_low; - u32 et_rx_coalesce_usecs_low; - u32 et_rx_max_coalesced_frames_low; - u32 et_pkt_rate_high; - u32 et_rx_coalesce_usecs_high; - u32 et_rx_max_coalesced_frames_high; - u32 et_rate_sample_interval; - u32 timer_int_limit; - u32 wqm_quanta; - u8 allow_unaligned_fpdus; - - /* Adapter base MAC address */ - u32 mac_addr_low; - u16 mac_addr_high; - - u16 firmware_eeprom_offset; - u16 software_eeprom_offset; - - u16 max_irrq_wr; - - /* pd config for each port */ - u16 pd_config_size[4]; - u16 pd_config_base[4]; - - u16 link_interrupt_count[4]; - u8 crit_error_count[32]; - - /* the phy index for each port */ - u8 phy_index[4]; - u8 mac_sw_state[4]; - u8 mac_link_down[4]; - u8 phy_type[4]; - u8 log_port; - - /* PCI information */ - struct nes_device *nesdev; - unsigned int devfn; - unsigned char bus_number; - unsigned char OneG_Mode; - - unsigned char ref_count; - u8 netdev_count; - u8 netdev_max; /* from host nic address count in EEPROM */ - u8 port_count; - u8 virtwq; - u8 send_term_ok; - u8 et_use_adaptive_rx_coalesce; - u8 adapter_fcn_count; - u8 pft_mcast_map[NES_PFT_SIZE]; -}; - -struct nes_pbl { - u64 *pbl_vbase; - dma_addr_t pbl_pbase; - struct page *page; - unsigned long user_base; - u32 pbl_size; - struct list_head list; - /* TODO: need to add list for two level tables */ -}; - -#define NES_4K_PBL_CHUNK_SIZE 4096 - -struct nes_fast_mr_wqe_pbl { - u64 *kva; - dma_addr_t paddr; -}; - -struct nes_listener { - struct work_struct work; - struct workqueue_struct *wq; - struct nes_vnic *nesvnic; - struct iw_cm_id *cm_id; - struct list_head list; - unsigned long socket; - u8 accept_failed; -}; - -struct nes_ib_device; - -#define NES_EVENT_DELAY msecs_to_jiffies(100) - -struct nes_vnic { - struct nes_ib_device *nesibdev; - u64 sq_full; - u64 tso_requests; - u64 segmented_tso_requests; - u64 linearized_skbs; - u64 tx_sw_dropped; - u64 endnode_nstat_rx_discard; - u64 endnode_nstat_rx_octets; - u64 endnode_nstat_rx_frames; - u64 endnode_nstat_tx_octets; - u64 endnode_nstat_tx_frames; - u64 endnode_ipv4_tcp_retransmits; - /* void *mem; */ - struct nes_device *nesdev; - struct net_device *netdev; - atomic_t rx_skbs_needed; - atomic_t rx_skb_timer_running; - int budget; - u32 msg_enable; - /* u32 tx_avail; */ - __be32 local_ipaddr; - struct napi_struct napi; - spinlock_t tx_lock; /* could use netdev tx lock? */ - struct timer_list rq_wqes_timer; - u32 nic_mem_size; - void *nic_vbase; - dma_addr_t nic_pbase; - struct nes_hw_nic nic; - struct nes_hw_nic_cq nic_cq; - u32 mcrq_qp_id; - struct nes_ucontext *mcrq_ucontext; - struct nes_cqp_request* (*get_cqp_request)(struct nes_device *nesdev); - void (*post_cqp_request)(struct nes_device*, struct nes_cqp_request *); - int (*mcrq_mcast_filter)( struct nes_vnic* nesvnic, __u8* dmi_addr ); - struct net_device_stats netstats; - /* used to put the netdev on the adapters logical port list */ - struct list_head list; - u16 max_frame_size; - u8 netdev_open; - u8 linkup; - u8 logical_port; - u8 netdev_index; /* might not be needed, indexes nesdev->netdev */ - u8 perfect_filter_index; - u8 nic_index; - u8 qp_nic_index[4]; - u8 next_qp_nic_index; - u8 of_device_registered; - u8 rdma_enabled; - struct timer_list event_timer; - enum ib_event_type delayed_event; - enum ib_event_type last_dispatched_event; - spinlock_t port_ibevent_lock; - u32 mgt_mem_size; - void *mgt_vbase; - dma_addr_t mgt_pbase; - struct nes_vnic_mgt *mgtvnic[NES_MGT_QP_COUNT]; - struct task_struct *mgt_thread; - wait_queue_head_t mgt_wait_queue; - struct sk_buff_head mgt_skb_list; - -}; - -struct nes_ib_device { - struct ib_device ibdev; - struct nes_vnic *nesvnic; - - /* Virtual RNIC Limits */ - u32 max_mr; - u32 max_qp; - u32 max_cq; - u32 max_pd; - u32 num_mr; - u32 num_qp; - u32 num_cq; - u32 num_pd; -}; - -enum nes_hdrct_flags { - DDP_LEN_FLAG = 0x80, - DDP_HDR_FLAG = 0x40, - RDMA_HDR_FLAG = 0x20 -}; - -enum nes_term_layers { - LAYER_RDMA = 0, - LAYER_DDP = 1, - LAYER_MPA = 2 -}; - -enum nes_term_error_types { - RDMAP_CATASTROPHIC = 0, - RDMAP_REMOTE_PROT = 1, - RDMAP_REMOTE_OP = 2, - DDP_CATASTROPHIC = 0, - DDP_TAGGED_BUFFER = 1, - DDP_UNTAGGED_BUFFER = 2, - DDP_LLP = 3 -}; - -enum nes_term_rdma_errors { - RDMAP_INV_STAG = 0x00, - RDMAP_INV_BOUNDS = 0x01, - RDMAP_ACCESS = 0x02, - RDMAP_UNASSOC_STAG = 0x03, - RDMAP_TO_WRAP = 0x04, - RDMAP_INV_RDMAP_VER = 0x05, - RDMAP_UNEXPECTED_OP = 0x06, - RDMAP_CATASTROPHIC_LOCAL = 0x07, - RDMAP_CATASTROPHIC_GLOBAL = 0x08, - RDMAP_CANT_INV_STAG = 0x09, - RDMAP_UNSPECIFIED = 0xff -}; - -enum nes_term_ddp_errors { - DDP_CATASTROPHIC_LOCAL = 0x00, - DDP_TAGGED_INV_STAG = 0x00, - DDP_TAGGED_BOUNDS = 0x01, - DDP_TAGGED_UNASSOC_STAG = 0x02, - DDP_TAGGED_TO_WRAP = 0x03, - DDP_TAGGED_INV_DDP_VER = 0x04, - DDP_UNTAGGED_INV_QN = 0x01, - DDP_UNTAGGED_INV_MSN_NO_BUF = 0x02, - DDP_UNTAGGED_INV_MSN_RANGE = 0x03, - DDP_UNTAGGED_INV_MO = 0x04, - DDP_UNTAGGED_INV_TOO_LONG = 0x05, - DDP_UNTAGGED_INV_DDP_VER = 0x06 -}; - -enum nes_term_mpa_errors { - MPA_CLOSED = 0x01, - MPA_CRC = 0x02, - MPA_MARKER = 0x03, - MPA_REQ_RSP = 0x04, -}; - -struct nes_terminate_hdr { - u8 layer_etype; - u8 error_code; - u8 hdrct; - u8 rsvd; -}; - -/* Used to determine how to fill in terminate error codes */ -#define IWARP_OPCODE_WRITE 0 -#define IWARP_OPCODE_READREQ 1 -#define IWARP_OPCODE_READRSP 2 -#define IWARP_OPCODE_SEND 3 -#define IWARP_OPCODE_SEND_INV 4 -#define IWARP_OPCODE_SEND_SE 5 -#define IWARP_OPCODE_SEND_SE_INV 6 -#define IWARP_OPCODE_TERM 7 - -/* These values are used only during terminate processing */ -#define TERM_DDP_LEN_TAGGED 14 -#define TERM_DDP_LEN_UNTAGGED 18 -#define TERM_RDMA_LEN 28 -#define RDMA_OPCODE_MASK 0x0f -#define RDMA_READ_REQ_OPCODE 1 -#define BAD_FRAME_OFFSET 64 -#define CQE_MAJOR_DRV 0x8000 - -/* Used for link status recheck after interrupt processing */ -#define NES_LINK_RECHECK_DELAY msecs_to_jiffies(50) -#define NES_LINK_RECHECK_MAX 60 - -#endif /* __NES_HW_H */ diff --git a/drivers/infiniband/hw/nes/nes_mgt.c b/drivers/infiniband/hw/nes/nes_mgt.c deleted file mode 100644 index cc4dce5c3e5f..000000000000 --- a/drivers/infiniband/hw/nes/nes_mgt.c +++ /dev/null @@ -1,1155 +0,0 @@ -/* - * Copyright (c) 2006 - 2011 Intel-NE, Inc. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - */ - -#include <linux/skbuff.h> -#include <linux/etherdevice.h> -#include <linux/kthread.h> -#include <linux/ip.h> -#include <linux/tcp.h> -#include <net/tcp.h> -#include "nes.h" -#include "nes_mgt.h" - -atomic_t pau_qps_created; -atomic_t pau_qps_destroyed; - -static void nes_replenish_mgt_rq(struct nes_vnic_mgt *mgtvnic) -{ - unsigned long flags; - dma_addr_t bus_address; - struct sk_buff *skb; - struct nes_hw_nic_rq_wqe *nic_rqe; - struct nes_hw_mgt *nesmgt; - struct nes_device *nesdev; - struct nes_rskb_cb *cb; - u32 rx_wqes_posted = 0; - - nesmgt = &mgtvnic->mgt; - nesdev = mgtvnic->nesvnic->nesdev; - spin_lock_irqsave(&nesmgt->rq_lock, flags); - if (nesmgt->replenishing_rq != 0) { - if (((nesmgt->rq_size - 1) == atomic_read(&mgtvnic->rx_skbs_needed)) && - (atomic_read(&mgtvnic->rx_skb_timer_running) == 0)) { - atomic_set(&mgtvnic->rx_skb_timer_running, 1); - spin_unlock_irqrestore(&nesmgt->rq_lock, flags); - mgtvnic->rq_wqes_timer.expires = jiffies + (HZ / 2); /* 1/2 second */ - add_timer(&mgtvnic->rq_wqes_timer); - } else { - spin_unlock_irqrestore(&nesmgt->rq_lock, flags); - } - return; - } - nesmgt->replenishing_rq = 1; - spin_unlock_irqrestore(&nesmgt->rq_lock, flags); - do { - skb = dev_alloc_skb(mgtvnic->nesvnic->max_frame_size); - if (skb) { - skb->dev = mgtvnic->nesvnic->netdev; - - bus_address = pci_map_single(nesdev->pcidev, - skb->data, mgtvnic->nesvnic->max_frame_size, PCI_DMA_FROMDEVICE); - cb = (struct nes_rskb_cb *)&skb->cb[0]; - cb->busaddr = bus_address; - cb->maplen = mgtvnic->nesvnic->max_frame_size; - - nic_rqe = &nesmgt->rq_vbase[mgtvnic->mgt.rq_head]; - nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_1_0_IDX] = - cpu_to_le32(mgtvnic->nesvnic->max_frame_size); - nic_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_3_2_IDX] = 0; - nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_LOW_IDX] = - cpu_to_le32((u32)bus_address); - nic_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_HIGH_IDX] = - cpu_to_le32((u32)((u64)bus_address >> 32)); - nesmgt->rx_skb[nesmgt->rq_head] = skb; - nesmgt->rq_head++; - nesmgt->rq_head &= nesmgt->rq_size - 1; - atomic_dec(&mgtvnic->rx_skbs_needed); - barrier(); - if (++rx_wqes_posted == 255) { - nes_write32(nesdev->regs + NES_WQE_ALLOC, (rx_wqes_posted << 24) | nesmgt->qp_id); - rx_wqes_posted = 0; - } - } else { - spin_lock_irqsave(&nesmgt->rq_lock, flags); - if (((nesmgt->rq_size - 1) == atomic_read(&mgtvnic->rx_skbs_needed)) && - (atomic_read(&mgtvnic->rx_skb_timer_running) == 0)) { - atomic_set(&mgtvnic->rx_skb_timer_running, 1); - spin_unlock_irqrestore(&nesmgt->rq_lock, flags); - mgtvnic->rq_wqes_timer.expires = jiffies + (HZ / 2); /* 1/2 second */ - add_timer(&mgtvnic->rq_wqes_timer); - } else { - spin_unlock_irqrestore(&nesmgt->rq_lock, flags); - } - break; - } - } while (atomic_read(&mgtvnic->rx_skbs_needed)); - barrier(); - if (rx_wqes_posted) - nes_write32(nesdev->regs + NES_WQE_ALLOC, (rx_wqes_posted << 24) | nesmgt->qp_id); - nesmgt->replenishing_rq = 0; -} - -/** - * nes_mgt_rq_wqes_timeout - */ -static void nes_mgt_rq_wqes_timeout(struct timer_list *t) -{ - struct nes_vnic_mgt *mgtvnic = from_timer(mgtvnic, t, - rq_wqes_timer); - - atomic_set(&mgtvnic->rx_skb_timer_running, 0); - if (atomic_read(&mgtvnic->rx_skbs_needed)) - nes_replenish_mgt_rq(mgtvnic); -} - -/** - * nes_mgt_free_skb - unmap and free skb - */ -static void nes_mgt_free_skb(struct nes_device *nesdev, struct sk_buff *skb, u32 dir) -{ - struct nes_rskb_cb *cb; - - cb = (struct nes_rskb_cb *)&skb->cb[0]; - pci_unmap_single(nesdev->pcidev, cb->busaddr, cb->maplen, dir); - cb->busaddr = 0; - dev_kfree_skb_any(skb); -} - -/** - * nes_download_callback - handle download completions - */ -static void nes_download_callback(struct nes_device *nesdev, struct nes_cqp_request *cqp_request) -{ - struct pau_fpdu_info *fpdu_info = cqp_request->cqp_callback_pointer; - struct nes_qp *nesqp = fpdu_info->nesqp; - struct sk_buff *skb; - int i; - - for (i = 0; i < fpdu_info->frag_cnt; i++) { - skb = fpdu_info->frags[i].skb; - if (fpdu_info->frags[i].cmplt) { - nes_mgt_free_skb(nesdev, skb, PCI_DMA_TODEVICE); - nes_rem_ref_cm_node(nesqp->cm_node); - } - } - - if (fpdu_info->hdr_vbase) - pci_free_consistent(nesdev->pcidev, fpdu_info->hdr_len, - fpdu_info->hdr_vbase, fpdu_info->hdr_pbase); - kfree(fpdu_info); -} - -/** - * nes_get_seq - Get the seq, ack_seq and window from the packet - */ -static u32 nes_get_seq(struct sk_buff *skb, u32 *ack, u16 *wnd, u32 *fin_rcvd, u32 *rst_rcvd) -{ - struct nes_rskb_cb *cb = (struct nes_rskb_cb *)&skb->cb[0]; - struct iphdr *iph = (struct iphdr *)(cb->data_start + ETH_HLEN); - struct tcphdr *tcph = (struct tcphdr *)(((char *)iph) + (4 * iph->ihl)); - - *ack = be32_to_cpu(tcph->ack_seq); - *wnd = be16_to_cpu(tcph->window); - *fin_rcvd = tcph->fin; - *rst_rcvd = tcph->rst; - return be32_to_cpu(tcph->seq); -} - -/** - * nes_get_next_skb - Get the next skb based on where current skb is in the queue - */ -static struct sk_buff *nes_get_next_skb(struct nes_device *nesdev, struct nes_qp *nesqp, - struct sk_buff *skb, u32 nextseq, u32 *ack, - u16 *wnd, u32 *fin_rcvd, u32 *rst_rcvd) -{ - u32 seq; - bool processacks; - struct sk_buff *old_skb; - - if (skb) { - /* Continue processing fpdu */ - skb = skb_peek_next(skb, &nesqp->pau_list); - if (!skb) - goto out; - processacks = false; - } else { - /* Starting a new one */ - if (skb_queue_empty(&nesqp->pau_list)) - goto out; - skb = skb_peek(&nesqp->pau_list); - processacks = true; - } - - while (1) { - if (skb_queue_empty(&nesqp->pau_list)) - goto out; - - seq = nes_get_seq(skb, ack, wnd, fin_rcvd, rst_rcvd); - if (seq == nextseq) { - if (skb->len || processacks) - break; - } else if (after(seq, nextseq)) { - goto out; - } - - old_skb = skb; - skb = skb_peek_next(skb, &nesqp->pau_list); - skb_unlink(old_skb, &nesqp->pau_list); - nes_mgt_free_skb(nesdev, old_skb, PCI_DMA_TODEVICE); - nes_rem_ref_cm_node(nesqp->cm_node); - if (!skb) - goto out; - } - return skb; - -out: - return NULL; -} - -/** - * get_fpdu_info - Find the next complete fpdu and return its fragments. - */ -static int get_fpdu_info(struct nes_device *nesdev, struct nes_qp *nesqp, - struct pau_fpdu_info **pau_fpdu_info) -{ - struct sk_buff *skb; - struct iphdr *iph; - struct tcphdr *tcph; - struct nes_rskb_cb *cb; - struct pau_fpdu_info *fpdu_info = NULL; - struct pau_fpdu_frag frags[MAX_FPDU_FRAGS]; - u32 fpdu_len = 0; - u32 tmp_len; - int frag_cnt = 0; - u32 tot_len; - u32 frag_tot; - u32 ack; - u32 fin_rcvd; - u32 rst_rcvd; - u16 wnd; - int i; - int rc = 0; - - *pau_fpdu_info = NULL; - - skb = nes_get_next_skb(nesdev, nesqp, NULL, nesqp->pau_rcv_nxt, &ack, &wnd, &fin_rcvd, &rst_rcvd); - if (!skb) - goto out; - - cb = (struct nes_rskb_cb *)&skb->cb[0]; - if (skb->len) { - fpdu_len = be16_to_cpu(*(__be16 *) skb->data) + MPA_FRAMING; - fpdu_len = (fpdu_len + 3) & 0xfffffffc; - tmp_len = fpdu_len; - - /* See if we have all of the fpdu */ - frag_tot = 0; - memset(&frags, 0, sizeof frags); - for (i = 0; i < MAX_FPDU_FRAGS; i++) { - frags[i].physaddr = cb->busaddr; - frags[i].physaddr += skb->data - cb->data_start; - frags[i].frag_len = min(tmp_len, skb->len); - frags[i].skb = skb; - frags[i].cmplt = (skb->len == frags[i].frag_len); - frag_tot += frags[i].frag_len; - frag_cnt++; - - tmp_len -= frags[i].frag_len; - if (tmp_len == 0) - break; - - skb = nes_get_next_skb(nesdev, nesqp, skb, - nesqp->pau_rcv_nxt + frag_tot, &ack, &wnd, &fin_rcvd, &rst_rcvd); - if (!skb) - goto out; - if (rst_rcvd) { - /* rst received in the middle of fpdu */ - for (; i >= 0; i--) { - skb_unlink(frags[i].skb, &nesqp->pau_list); - nes_mgt_free_skb(nesdev, frags[i].skb, PCI_DMA_TODEVICE); - } - cb = (struct nes_rskb_cb *)&skb->cb[0]; - frags[0].physaddr = cb->busaddr; - frags[0].physaddr += skb->data - cb->data_start; - frags[0].frag_len = skb->len; - frags[0].skb = skb; - frags[0].cmplt = true; - frag_cnt = 1; - break; - } - - cb = (struct nes_rskb_cb *)&skb->cb[0]; - } - } else { - /* no data */ - frags[0].physaddr = cb->busaddr; - frags[0].frag_len = 0; - frags[0].skb = skb; - frags[0].cmplt = true; - frag_cnt = 1; - } - - /* Found one */ - fpdu_info = kzalloc(sizeof(*fpdu_info), GFP_ATOMIC); - if (!fpdu_info) { - rc = -ENOMEM; - goto out; - } - - fpdu_info->cqp_request = nes_get_cqp_request(nesdev); - if (fpdu_info->cqp_request == NULL) { - nes_debug(NES_DBG_PAU, "Failed to get a cqp_request.\n"); - rc = -ENOMEM; - goto out; - } - - cb = (struct nes_rskb_cb *)&frags[0].skb->cb[0]; - iph = (struct iphdr *)(cb->data_start + ETH_HLEN); - tcph = (struct tcphdr *)(((char *)iph) + (4 * iph->ihl)); - fpdu_info->hdr_len = (((unsigned char *)tcph) + 4 * (tcph->doff)) - cb->data_start; - fpdu_info->data_len = fpdu_len; - tot_len = fpdu_info->hdr_len + fpdu_len - ETH_HLEN; - - if (frags[0].cmplt) { - fpdu_info->hdr_pbase = cb->busaddr; - fpdu_info->hdr_vbase = NULL; - } else { - fpdu_info->hdr_vbase = pci_alloc_consistent(nesdev->pcidev, - fpdu_info->hdr_len, &fpdu_info->hdr_pbase); - if (!fpdu_info->hdr_vbase) { - nes_debug(NES_DBG_PAU, "Unable to allocate memory for pau first frag\n"); - rc = -ENOMEM; - goto out; - } - - /* Copy hdrs, adjusting len and seqnum */ - memcpy(fpdu_info->hdr_vbase, cb->data_start, fpdu_info->hdr_len); - iph = (struct iphdr *)(fpdu_info->hdr_vbase + ETH_HLEN); - tcph = (struct tcphdr *)(((char *)iph) + (4 * iph->ihl)); - } - - iph->tot_len = cpu_to_be16(tot_len); - iph->saddr = cpu_to_be32(0x7f000001); - - tcph->seq = cpu_to_be32(nesqp->pau_rcv_nxt); - tcph->ack_seq = cpu_to_be32(ack); - tcph->window = cpu_to_be16(wnd); - - nesqp->pau_rcv_nxt += fpdu_len + fin_rcvd; - - memcpy(fpdu_info->frags, frags, sizeof(fpdu_info->frags)); - fpdu_info->frag_cnt = frag_cnt; - fpdu_info->nesqp = nesqp; - *pau_fpdu_info = fpdu_info; - - /* Update skb's for next pass */ - for (i = 0; i < frag_cnt; i++) { - cb = (struct nes_rskb_cb *)&frags[i].skb->cb[0]; - skb_pull(frags[i].skb, frags[i].frag_len); - - if (frags[i].skb->len == 0) { - /* Pull skb off the list - it will be freed in the callback */ - if (!skb_queue_empty(&nesqp->pau_list)) - skb_unlink(frags[i].skb, &nesqp->pau_list); - } else { - /* Last skb still has data so update the seq */ - iph = (struct iphdr *)(cb->data_start + ETH_HLEN); - tcph = (struct tcphdr *)(((char *)iph) + (4 * iph->ihl)); - tcph->seq = cpu_to_be32(nesqp->pau_rcv_nxt); - } - } - -out: - if (rc) { - if (fpdu_info) { - if (fpdu_info->cqp_request) - nes_put_cqp_request(nesdev, fpdu_info->cqp_request); - kfree(fpdu_info); - } - } - return rc; -} - -/** - * forward_fpdu - send complete fpdus, one at a time - */ -static int forward_fpdus(struct nes_vnic *nesvnic, struct nes_qp *nesqp) -{ - struct nes_device *nesdev = nesvnic->nesdev; - struct pau_fpdu_info *fpdu_info; - struct nes_hw_cqp_wqe *cqp_wqe; - struct nes_cqp_request *cqp_request; - unsigned long flags; - u64 u64tmp; - u32 u32tmp; - int rc; - - while (1) { - spin_lock_irqsave(&nesqp->pau_lock, flags); - rc = get_fpdu_info(nesdev, nesqp, &fpdu_info); - if (rc || (fpdu_info == NULL)) { - spin_unlock_irqrestore(&nesqp->pau_lock, flags); - return rc; - } - - cqp_request = fpdu_info->cqp_request; - cqp_wqe = &cqp_request->cqp_wqe; - nes_fill_init_cqp_wqe(cqp_wqe, nesdev); - set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_DL_OPCODE_IDX, - NES_CQP_DOWNLOAD_SEGMENT | - (((u32)nesvnic->logical_port) << NES_CQP_OP_LOGICAL_PORT_SHIFT)); - - u32tmp = fpdu_info->hdr_len << 16; - u32tmp |= fpdu_info->hdr_len + (u32)fpdu_info->data_len; - set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_DL_LENGTH_0_TOTAL_IDX, - u32tmp); - - u32tmp = (fpdu_info->frags[1].frag_len << 16) | fpdu_info->frags[0].frag_len; - set_wqe_32bit_value(cqp_wqe->wqe_words, NES_NIC_SQ_WQE_LENGTH_2_1_IDX, - u32tmp); - - u32tmp = (fpdu_info->frags[3].frag_len << 16) | fpdu_info->frags[2].frag_len; - set_wqe_32bit_value(cqp_wqe->wqe_words, NES_NIC_SQ_WQE_LENGTH_4_3_IDX, - u32tmp); - - u64tmp = (u64)fpdu_info->hdr_pbase; - set_wqe_32bit_value(cqp_wqe->wqe_words, NES_NIC_SQ_WQE_FRAG0_LOW_IDX, - lower_32_bits(u64tmp)); - set_wqe_32bit_value(cqp_wqe->wqe_words, NES_NIC_SQ_WQE_FRAG0_HIGH_IDX, - upper_32_bits(u64tmp)); - - set_wqe_32bit_value(cqp_wqe->wqe_words, NES_NIC_SQ_WQE_FRAG1_LOW_IDX, - lower_32_bits(fpdu_info->frags[0].physaddr)); - set_wqe_32bit_value(cqp_wqe->wqe_words, NES_NIC_SQ_WQE_FRAG1_HIGH_IDX, - upper_32_bits(fpdu_info->frags[0].physaddr)); - - set_wqe_32bit_value(cqp_wqe->wqe_words, NES_NIC_SQ_WQE_FRAG2_LOW_IDX, - lower_32_bits(fpdu_info->frags[1].physaddr)); - set_wqe_32bit_value(cqp_wqe->wqe_words, NES_NIC_SQ_WQE_FRAG2_HIGH_IDX, - upper_32_bits(fpdu_info->frags[1].physaddr)); - - set_wqe_32bit_value(cqp_wqe->wqe_words, NES_NIC_SQ_WQE_FRAG3_LOW_IDX, - lower_32_bits(fpdu_info->frags[2].physaddr)); - set_wqe_32bit_value(cqp_wqe->wqe_words, NES_NIC_SQ_WQE_FRAG3_HIGH_IDX, - upper_32_bits(fpdu_info->frags[2].physaddr)); - - set_wqe_32bit_value(cqp_wqe->wqe_words, NES_NIC_SQ_WQE_FRAG4_LOW_IDX, - lower_32_bits(fpdu_info->frags[3].physaddr)); - set_wqe_32bit_value(cqp_wqe->wqe_words, NES_NIC_SQ_WQE_FRAG4_HIGH_IDX, - upper_32_bits(fpdu_info->frags[3].physaddr)); - - cqp_request->cqp_callback_pointer = fpdu_info; - cqp_request->callback = 1; - cqp_request->cqp_callback = nes_download_callback; - - atomic_set(&cqp_request->refcount, 1); - nes_post_cqp_request(nesdev, cqp_request); - spin_unlock_irqrestore(&nesqp->pau_lock, flags); - } - - return 0; -} - -static void process_fpdus(struct nes_vnic *nesvnic, struct nes_qp *nesqp) -{ - int again = 1; - unsigned long flags; - - do { - /* Ignore rc - if it failed, tcp retries will cause it to try again */ - forward_fpdus(nesvnic, nesqp); - - spin_lock_irqsave(&nesqp->pau_lock, flags); - if (nesqp->pau_pending) { - nesqp->pau_pending = 0; - } else { - nesqp->pau_busy = 0; - again = 0; - } - - spin_unlock_irqrestore(&nesqp->pau_lock, flags); - } while (again); -} - -/** - * queue_fpdus - Handle fpdu's that hw passed up to sw - */ -static void queue_fpdus(struct sk_buff *skb, struct nes_vnic *nesvnic, struct nes_qp *nesqp) -{ - struct sk_buff *tmpskb; - struct nes_rskb_cb *cb; - struct iphdr *iph; - struct tcphdr *tcph; - unsigned char *tcph_end; - u32 rcv_nxt; - u32 rcv_wnd; - u32 seqnum; - u32 len; - bool process_it = false; - unsigned long flags; - - /* Move data ptr to after tcp header */ - iph = (struct iphdr *)skb->data; - tcph = (struct tcphdr *)(((char *)iph) + (4 * iph->ihl)); - seqnum = be32_to_cpu(tcph->seq); - tcph_end = (((char *)tcph) + (4 * tcph->doff)); - - len = be16_to_cpu(iph->tot_len); - if (skb->len > len) - skb_trim(skb, len); - skb_pull(skb, tcph_end - skb->data); - - /* Initialize tracking values */ - cb = (struct nes_rskb_cb *)&skb->cb[0]; - cb->seqnum = seqnum; - - /* Make sure data is in the receive window */ - rcv_nxt = nesqp->pau_rcv_nxt; - rcv_wnd = le32_to_cpu(nesqp->nesqp_context->rcv_wnd); - if (!between(seqnum, rcv_nxt, (rcv_nxt + rcv_wnd))) { - nes_mgt_free_skb(nesvnic->nesdev, skb, PCI_DMA_TODEVICE); - nes_rem_ref_cm_node(nesqp->cm_node); - return; - } - - spin_lock_irqsave(&nesqp->pau_lock, flags); - - if (nesqp->pau_busy) - nesqp->pau_pending = 1; - else - nesqp->pau_busy = 1; - - /* Queue skb by sequence number */ - if (skb_queue_len(&nesqp->pau_list) == 0) { - __skb_queue_head(&nesqp->pau_list, skb); - } else { - skb_queue_walk(&nesqp->pau_list, tmpskb) { - cb = (struct nes_rskb_cb *)&tmpskb->cb[0]; - if (before(seqnum, cb->seqnum)) - break; - } - __skb_insert(skb, tmpskb->prev, tmpskb, &nesqp->pau_list); - } - if (nesqp->pau_state == PAU_READY) - process_it = true; - spin_unlock_irqrestore(&nesqp->pau_lock, flags); - - if (process_it) - process_fpdus(nesvnic, nesqp); - - return; -} - -/** - * mgt_thread - Handle mgt skbs in a safe context - */ -static int mgt_thread(void *context) -{ - struct nes_vnic *nesvnic = context; - struct sk_buff *skb; - struct nes_rskb_cb *cb; - - while (!kthread_should_stop()) { - wait_event_interruptible(nesvnic->mgt_wait_queue, - skb_queue_len(&nesvnic->mgt_skb_list) || kthread_should_stop()); - while ((skb_queue_len(&nesvnic->mgt_skb_list)) && !kthread_should_stop()) { - skb = skb_dequeue(&nesvnic->mgt_skb_list); - cb = (struct nes_rskb_cb *)&skb->cb[0]; - cb->data_start = skb->data - ETH_HLEN; - cb->busaddr = pci_map_single(nesvnic->nesdev->pcidev, cb->data_start, - nesvnic->max_frame_size, PCI_DMA_TODEVICE); - queue_fpdus(skb, nesvnic, cb->nesqp); - } - } - - /* Closing down so delete any entries on the queue */ - while (skb_queue_len(&nesvnic->mgt_skb_list)) { - skb = skb_dequeue(&nesvnic->mgt_skb_list); - cb = (struct nes_rskb_cb *)&skb->cb[0]; - nes_rem_ref_cm_node(cb->nesqp->cm_node); - dev_kfree_skb_any(skb); - } - return 0; -} - -/** - * nes_queue_skbs - Queue skb so it can be handled in a thread context - */ -void nes_queue_mgt_skbs(struct sk_buff *skb, struct nes_vnic *nesvnic, struct nes_qp *nesqp) -{ - struct nes_rskb_cb *cb; - - cb = (struct nes_rskb_cb *)&skb->cb[0]; - cb->nesqp = nesqp; - skb_queue_tail(&nesvnic->mgt_skb_list, skb); - wake_up_interruptible(&nesvnic->mgt_wait_queue); -} - -void nes_destroy_pau_qp(struct nes_device *nesdev, struct nes_qp *nesqp) -{ - struct sk_buff *skb; - unsigned long flags; - atomic_inc(&pau_qps_destroyed); - - /* Free packets that have not yet been forwarded */ - /* Lock is acquired by skb_dequeue when removing the skb */ - spin_lock_irqsave(&nesqp->pau_lock, flags); - while (skb_queue_len(&nesqp->pau_list)) { - skb = skb_dequeue(&nesqp->pau_list); - nes_mgt_free_skb(nesdev, skb, PCI_DMA_TODEVICE); - nes_rem_ref_cm_node(nesqp->cm_node); - } - spin_unlock_irqrestore(&nesqp->pau_lock, flags); -} - -static void nes_chg_qh_handler(struct nes_device *nesdev, struct nes_cqp_request *cqp_request) -{ - struct pau_qh_chg *qh_chg = cqp_request->cqp_callback_pointer; - struct nes_cqp_request *new_request; - struct nes_hw_cqp_wqe *cqp_wqe; - struct nes_adapter *nesadapter; - struct nes_qp *nesqp; - struct nes_v4_quad nes_quad; - u32 crc_value; - u64 u64temp; - - nesadapter = nesdev->nesadapter; - nesqp = qh_chg->nesqp; - - /* Should we handle the bad completion */ - if (cqp_request->major_code) - WARN(1, PFX "Invalid cqp_request major_code=0x%x\n", - cqp_request->major_code); - - switch (nesqp->pau_state) { - case PAU_DEL_QH: - /* Old hash code deleted, now set the new one */ - nesqp->pau_state = PAU_ADD_LB_QH; - new_request = nes_get_cqp_request(nesdev); - if (new_request == NULL) { - nes_debug(NES_DBG_PAU, "Failed to get a new_request.\n"); - WARN_ON(1); - return; - } - - memset(&nes_quad, 0, sizeof(nes_quad)); - nes_quad.DstIpAdrIndex = - cpu_to_le32((u32)PCI_FUNC(nesdev->pcidev->devfn) << 24); - nes_quad.SrcIpadr = cpu_to_be32(0x7f000001); - nes_quad.TcpPorts[0] = swab16(nesqp->nesqp_context->tcpPorts[1]); - nes_quad.TcpPorts[1] = swab16(nesqp->nesqp_context->tcpPorts[0]); - - /* Produce hash key */ - crc_value = get_crc_value(&nes_quad); - nesqp->hte_index = cpu_to_be32(crc_value ^ 0xffffffff); - nes_debug(NES_DBG_PAU, "new HTE Index = 0x%08X, CRC = 0x%08X\n", - nesqp->hte_index, nesqp->hte_index & nesadapter->hte_index_mask); - - nesqp->hte_index &= nesadapter->hte_index_mask; - nesqp->nesqp_context->hte_index = cpu_to_le32(nesqp->hte_index); - nesqp->nesqp_context->ip0 = cpu_to_le32(0x7f000001); - nesqp->nesqp_context->rcv_nxt = cpu_to_le32(nesqp->pau_rcv_nxt); - - cqp_wqe = &new_request->cqp_wqe; - nes_fill_init_cqp_wqe(cqp_wqe, nesdev); - set_wqe_32bit_value(cqp_wqe->wqe_words, - NES_CQP_WQE_OPCODE_IDX, NES_CQP_MANAGE_QUAD_HASH | - NES_CQP_QP_TYPE_IWARP | NES_CQP_QP_CONTEXT_VALID | NES_CQP_QP_IWARP_STATE_RTS); - set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX, nesqp->hwqp.qp_id); - u64temp = (u64)nesqp->nesqp_context_pbase; - set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_QP_WQE_CONTEXT_LOW_IDX, u64temp); - - nes_debug(NES_DBG_PAU, "Waiting for CQP completion for adding the quad hash.\n"); - - new_request->cqp_callback_pointer = qh_chg; - new_request->callback = 1; - new_request->cqp_callback = nes_chg_qh_handler; - atomic_set(&new_request->refcount, 1); - nes_post_cqp_request(nesdev, new_request); - break; - - case PAU_ADD_LB_QH: - /* Start processing the queued fpdu's */ - nesqp->pau_state = PAU_READY; - process_fpdus(qh_chg->nesvnic, qh_chg->nesqp); - kfree(qh_chg); - break; - } -} - -/** - * nes_change_quad_hash - */ -static int nes_change_quad_hash(struct nes_device *nesdev, - struct nes_vnic *nesvnic, struct nes_qp *nesqp) -{ - struct nes_cqp_request *cqp_request = NULL; - struct pau_qh_chg *qh_chg = NULL; - u64 u64temp; - struct nes_hw_cqp_wqe *cqp_wqe; - int ret = 0; - - cqp_request = nes_get_cqp_request(nesdev); - if (cqp_request == NULL) { - nes_debug(NES_DBG_PAU, "Failed to get a cqp_request.\n"); - ret = -ENOMEM; - goto chg_qh_err; - } - - qh_chg = kmalloc(sizeof *qh_chg, GFP_ATOMIC); - if (!qh_chg) { - ret = -ENOMEM; - goto chg_qh_err; - } - qh_chg->nesdev = nesdev; - qh_chg->nesvnic = nesvnic; - qh_chg->nesqp = nesqp; - nesqp->pau_state = PAU_DEL_QH; - - cqp_wqe = &cqp_request->cqp_wqe; - nes_fill_init_cqp_wqe(cqp_wqe, nesdev); - set_wqe_32bit_value(cqp_wqe->wqe_words, - NES_CQP_WQE_OPCODE_IDX, NES_CQP_MANAGE_QUAD_HASH | NES_CQP_QP_DEL_HTE | - NES_CQP_QP_TYPE_IWARP | NES_CQP_QP_CONTEXT_VALID | NES_CQP_QP_IWARP_STATE_RTS); - set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX, nesqp->hwqp.qp_id); - u64temp = (u64)nesqp->nesqp_context_pbase; - set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_QP_WQE_CONTEXT_LOW_IDX, u64temp); - - nes_debug(NES_DBG_PAU, "Waiting for CQP completion for deleting the quad hash.\n"); - - cqp_request->cqp_callback_pointer = qh_chg; - cqp_request->callback = 1; - cqp_request->cqp_callback = nes_chg_qh_handler; - atomic_set(&cqp_request->refcount, 1); - nes_post_cqp_request(nesdev, cqp_request); - - return ret; - -chg_qh_err: - kfree(qh_chg); - if (cqp_request) - nes_put_cqp_request(nesdev, cqp_request); - return ret; -} - -/** - * nes_mgt_ce_handler - * This management code deals with any packed and unaligned (pau) fpdu's - * that the hardware cannot handle. - */ -static void nes_mgt_ce_handler(struct nes_device *nesdev, struct nes_hw_nic_cq *cq) -{ - struct nes_vnic_mgt *mgtvnic = container_of(cq, struct nes_vnic_mgt, mgt_cq); - struct nes_adapter *nesadapter = nesdev->nesadapter; - u32 head; - u32 cq_size; - u32 cqe_count = 0; - u32 cqe_misc; - u32 qp_id = 0; - u32 skbs_needed; - unsigned long context; - struct nes_qp *nesqp; - struct sk_buff *rx_skb; - struct nes_rskb_cb *cb; - - head = cq->cq_head; - cq_size = cq->cq_size; - - while (1) { - cqe_misc = le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_NIC_CQE_MISC_IDX]); - if (!(cqe_misc & NES_NIC_CQE_VALID)) - break; - - nesqp = NULL; - if (cqe_misc & NES_NIC_CQE_ACCQP_VALID) { - qp_id = le32_to_cpu(cq->cq_vbase[head].cqe_words[NES_NIC_CQE_ACCQP_ID_IDX]); - qp_id &= 0x001fffff; - if (qp_id < nesadapter->max_qp) { - context = (unsigned long)nesadapter->qp_table[qp_id - NES_FIRST_QPN]; - nesqp = (struct nes_qp *)context; - } - } - - if (nesqp) { - if (nesqp->pau_mode == false) { - nesqp->pau_mode = true; /* First time for this qp */ - nesqp->pau_rcv_nxt = le32_to_cpu( - cq->cq_vbase[head].cqe_words[NES_NIC_CQE_HASH_RCVNXT]); - skb_queue_head_init(&nesqp->pau_list); - spin_lock_init(&nesqp->pau_lock); - atomic_inc(&pau_qps_created); - nes_change_quad_hash(nesdev, mgtvnic->nesvnic, nesqp); - } - - rx_skb = mgtvnic->mgt.rx_skb[mgtvnic->mgt.rq_tail]; - rx_skb->len = 0; - skb_put(rx_skb, cqe_misc & 0x0000ffff); - rx_skb->protocol = eth_type_trans(rx_skb, mgtvnic->nesvnic->netdev); - cb = (struct nes_rskb_cb *)&rx_skb->cb[0]; - pci_unmap_single(nesdev->pcidev, cb->busaddr, cb->maplen, PCI_DMA_FROMDEVICE); - cb->busaddr = 0; - mgtvnic->mgt.rq_tail++; - mgtvnic->mgt.rq_tail &= mgtvnic->mgt.rq_size - 1; - - nes_add_ref_cm_node(nesqp->cm_node); - nes_queue_mgt_skbs(rx_skb, mgtvnic->nesvnic, nesqp); - } else { - printk(KERN_ERR PFX "Invalid QP %d for packed/unaligned handling\n", qp_id); - } - - cq->cq_vbase[head].cqe_words[NES_NIC_CQE_MISC_IDX] = 0; - cqe_count++; - if (++head >= cq_size) - head = 0; - - if (cqe_count == 255) { - /* Replenish mgt CQ */ - nes_write32(nesdev->regs + NES_CQE_ALLOC, cq->cq_number | (cqe_count << 16)); - nesdev->currcq_count += cqe_count; - cqe_count = 0; - } - - skbs_needed = atomic_inc_return(&mgtvnic->rx_skbs_needed); - if (skbs_needed > (mgtvnic->mgt.rq_size >> 1)) - nes_replenish_mgt_rq(mgtvnic); - } - - cq->cq_head = head; - nes_write32(nesdev->regs + NES_CQE_ALLOC, NES_CQE_ALLOC_NOTIFY_NEXT | - cq->cq_number | (cqe_count << 16)); - nes_read32(nesdev->regs + NES_CQE_ALLOC); - nesdev->currcq_count += cqe_count; -} - -/** - * nes_init_mgt_qp - */ -int nes_init_mgt_qp(struct nes_device *nesdev, struct net_device *netdev, struct nes_vnic *nesvnic) -{ - struct nes_vnic_mgt *mgtvnic; - u32 counter; - void *vmem; - dma_addr_t pmem; - struct nes_hw_cqp_wqe *cqp_wqe; - u32 cqp_head; - unsigned long flags; - struct nes_hw_nic_qp_context *mgt_context; - u64 u64temp; - struct nes_hw_nic_rq_wqe *mgt_rqe; - struct sk_buff *skb; - u32 wqe_count; - struct nes_rskb_cb *cb; - u32 mgt_mem_size; - void *mgt_vbase; - dma_addr_t mgt_pbase; - int i; - int ret; - - /* Allocate space the all mgt QPs once */ - mgtvnic = kcalloc(NES_MGT_QP_COUNT, sizeof(struct nes_vnic_mgt), - GFP_KERNEL); - if (!mgtvnic) - return -ENOMEM; - - /* Allocate fragment, RQ, and CQ; Reuse CEQ based on the PCI function */ - /* We are not sending from this NIC so sq is not allocated */ - mgt_mem_size = 256 + - (NES_MGT_WQ_COUNT * sizeof(struct nes_hw_nic_rq_wqe)) + - (NES_MGT_WQ_COUNT * sizeof(struct nes_hw_nic_cqe)) + - sizeof(struct nes_hw_nic_qp_context); - mgt_mem_size = (mgt_mem_size + PAGE_SIZE - 1) & ~(PAGE_SIZE - 1); - mgt_vbase = pci_alloc_consistent(nesdev->pcidev, NES_MGT_QP_COUNT * mgt_mem_size, &mgt_pbase); - if (!mgt_vbase) { - kfree(mgtvnic); - nes_debug(NES_DBG_INIT, "Unable to allocate memory for mgt host descriptor rings\n"); - return -ENOMEM; - } - - nesvnic->mgt_mem_size = NES_MGT_QP_COUNT * mgt_mem_size; - nesvnic->mgt_vbase = mgt_vbase; - nesvnic->mgt_pbase = mgt_pbase; - - skb_queue_head_init(&nesvnic->mgt_skb_list); - init_waitqueue_head(&nesvnic->mgt_wait_queue); - nesvnic->mgt_thread = kthread_run(mgt_thread, nesvnic, "nes_mgt_thread"); - - for (i = 0; i < NES_MGT_QP_COUNT; i++) { - mgtvnic->nesvnic = nesvnic; - mgtvnic->mgt.qp_id = nesdev->mac_index + NES_MGT_QP_OFFSET + i; - memset(mgt_vbase, 0, mgt_mem_size); - nes_debug(NES_DBG_INIT, "Allocated mgt QP structures at %p (phys = %016lX), size = %u.\n", - mgt_vbase, (unsigned long)mgt_pbase, mgt_mem_size); - - vmem = (void *)(((unsigned long)mgt_vbase + (256 - 1)) & - ~(unsigned long)(256 - 1)); - pmem = (dma_addr_t)(((unsigned long long)mgt_pbase + (256 - 1)) & - ~(unsigned long long)(256 - 1)); - - spin_lock_init(&mgtvnic->mgt.rq_lock); - - /* setup the RQ */ - mgtvnic->mgt.rq_vbase = vmem; - mgtvnic->mgt.rq_pbase = pmem; - mgtvnic->mgt.rq_head = 0; - mgtvnic->mgt.rq_tail = 0; - mgtvnic->mgt.rq_size = NES_MGT_WQ_COUNT; - - /* setup the CQ */ - vmem += (NES_MGT_WQ_COUNT * sizeof(struct nes_hw_nic_rq_wqe)); - pmem += (NES_MGT_WQ_COUNT * sizeof(struct nes_hw_nic_rq_wqe)); - - mgtvnic->mgt_cq.cq_number = mgtvnic->mgt.qp_id; - mgtvnic->mgt_cq.cq_vbase = vmem; - mgtvnic->mgt_cq.cq_pbase = pmem; - mgtvnic->mgt_cq.cq_head = 0; - mgtvnic->mgt_cq.cq_size = NES_MGT_WQ_COUNT; - - mgtvnic->mgt_cq.ce_handler = nes_mgt_ce_handler; - - /* Send CreateCQ request to CQP */ - spin_lock_irqsave(&nesdev->cqp.lock, flags); - cqp_head = nesdev->cqp.sq_head; - - cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head]; - nes_fill_init_cqp_wqe(cqp_wqe, nesdev); - - cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32( - NES_CQP_CREATE_CQ | NES_CQP_CQ_CEQ_VALID | - ((u32)mgtvnic->mgt_cq.cq_size << 16)); - cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32( - mgtvnic->mgt_cq.cq_number | ((u32)nesdev->ceq_index << 16)); - u64temp = (u64)mgtvnic->mgt_cq.cq_pbase; - set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp); - cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] = 0; - u64temp = (unsigned long)&mgtvnic->mgt_cq; - cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_LOW_IDX] = cpu_to_le32((u32)(u64temp >> 1)); - cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] = - cpu_to_le32(((u32)((u64temp) >> 33)) & 0x7FFFFFFF); - cqp_wqe->wqe_words[NES_CQP_CQ_WQE_DOORBELL_INDEX_HIGH_IDX] = 0; - - if (++cqp_head >= nesdev->cqp.sq_size) - cqp_head = 0; - cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head]; - nes_fill_init_cqp_wqe(cqp_wqe, nesdev); - - /* Send CreateQP request to CQP */ - mgt_context = (void *)(&mgtvnic->mgt_cq.cq_vbase[mgtvnic->mgt_cq.cq_size]); - mgt_context->context_words[NES_NIC_CTX_MISC_IDX] = - cpu_to_le32((u32)NES_MGT_CTX_SIZE | - ((u32)PCI_FUNC(nesdev->pcidev->devfn) << 12)); - nes_debug(NES_DBG_INIT, "RX_WINDOW_BUFFER_PAGE_TABLE_SIZE = 0x%08X, RX_WINDOW_BUFFER_SIZE = 0x%08X\n", - nes_read_indexed(nesdev, NES_IDX_RX_WINDOW_BUFFER_PAGE_TABLE_SIZE), - nes_read_indexed(nesdev, NES_IDX_RX_WINDOW_BUFFER_SIZE)); - if (nes_read_indexed(nesdev, NES_IDX_RX_WINDOW_BUFFER_SIZE) != 0) - mgt_context->context_words[NES_NIC_CTX_MISC_IDX] |= cpu_to_le32(NES_NIC_BACK_STORE); - - u64temp = (u64)mgtvnic->mgt.rq_pbase; - mgt_context->context_words[NES_NIC_CTX_SQ_LOW_IDX] = cpu_to_le32((u32)u64temp); - mgt_context->context_words[NES_NIC_CTX_SQ_HIGH_IDX] = cpu_to_le32((u32)(u64temp >> 32)); - u64temp = (u64)mgtvnic->mgt.rq_pbase; - mgt_context->context_words[NES_NIC_CTX_RQ_LOW_IDX] = cpu_to_le32((u32)u64temp); - mgt_context->context_words[NES_NIC_CTX_RQ_HIGH_IDX] = cpu_to_le32((u32)(u64temp >> 32)); - - cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = cpu_to_le32(NES_CQP_CREATE_QP | - NES_CQP_QP_TYPE_NIC); - cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX] = cpu_to_le32(mgtvnic->mgt.qp_id); - u64temp = (u64)mgtvnic->mgt_cq.cq_pbase + - (mgtvnic->mgt_cq.cq_size * sizeof(struct nes_hw_nic_cqe)); - set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_QP_WQE_CONTEXT_LOW_IDX, u64temp); - - if (++cqp_head >= nesdev->cqp.sq_size) - cqp_head = 0; - nesdev->cqp.sq_head = cqp_head; - - barrier(); - - /* Ring doorbell (2 WQEs) */ - nes_write32(nesdev->regs + NES_WQE_ALLOC, 0x02800000 | nesdev->cqp.qp_id); - - spin_unlock_irqrestore(&nesdev->cqp.lock, flags); - nes_debug(NES_DBG_INIT, "Waiting for create MGT QP%u to complete.\n", - mgtvnic->mgt.qp_id); - - ret = wait_event_timeout(nesdev->cqp.waitq, (nesdev->cqp.sq_tail == cqp_head), - NES_EVENT_TIMEOUT); - nes_debug(NES_DBG_INIT, "Create MGT QP%u completed, wait_event_timeout ret = %u.\n", - mgtvnic->mgt.qp_id, ret); - if (!ret) { - nes_debug(NES_DBG_INIT, "MGT QP%u create timeout expired\n", mgtvnic->mgt.qp_id); - if (i == 0) { - pci_free_consistent(nesdev->pcidev, nesvnic->mgt_mem_size, nesvnic->mgt_vbase, - nesvnic->mgt_pbase); - kfree(mgtvnic); - } else { - nes_destroy_mgt(nesvnic); - } - return -EIO; - } - - /* Populate the RQ */ - for (counter = 0; counter < (NES_MGT_WQ_COUNT - 1); counter++) { - skb = dev_alloc_skb(nesvnic->max_frame_size); - if (!skb) { - nes_debug(NES_DBG_INIT, "%s: out of memory for receive skb\n", netdev->name); - return -ENOMEM; - } - - skb->dev = netdev; - - pmem = pci_map_single(nesdev->pcidev, skb->data, - nesvnic->max_frame_size, PCI_DMA_FROMDEVICE); - cb = (struct nes_rskb_cb *)&skb->cb[0]; - cb->busaddr = pmem; - cb->maplen = nesvnic->max_frame_size; - - mgt_rqe = &mgtvnic->mgt.rq_vbase[counter]; - mgt_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_1_0_IDX] = cpu_to_le32((u32)nesvnic->max_frame_size); - mgt_rqe->wqe_words[NES_NIC_RQ_WQE_LENGTH_3_2_IDX] = 0; - mgt_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_LOW_IDX] = cpu_to_le32((u32)pmem); - mgt_rqe->wqe_words[NES_NIC_RQ_WQE_FRAG0_HIGH_IDX] = cpu_to_le32((u32)((u64)pmem >> 32)); - mgtvnic->mgt.rx_skb[counter] = skb; - } - - timer_setup(&mgtvnic->rq_wqes_timer, nes_mgt_rq_wqes_timeout, - 0); - - wqe_count = NES_MGT_WQ_COUNT - 1; - mgtvnic->mgt.rq_head = wqe_count; - barrier(); - do { - counter = min(wqe_count, ((u32)255)); - wqe_count -= counter; - nes_write32(nesdev->regs + NES_WQE_ALLOC, (counter << 24) | mgtvnic->mgt.qp_id); - } while (wqe_count); - - nes_write32(nesdev->regs + NES_CQE_ALLOC, NES_CQE_ALLOC_NOTIFY_NEXT | - mgtvnic->mgt_cq.cq_number); - nes_read32(nesdev->regs + NES_CQE_ALLOC); - - mgt_vbase += mgt_mem_size; - mgt_pbase += mgt_mem_size; - nesvnic->mgtvnic[i] = mgtvnic++; - } - return 0; -} - - -void nes_destroy_mgt(struct nes_vnic *nesvnic) -{ - struct nes_device *nesdev = nesvnic->nesdev; - struct nes_vnic_mgt *mgtvnic; - struct nes_vnic_mgt *first_mgtvnic; - unsigned long flags; - struct nes_hw_cqp_wqe *cqp_wqe; - u32 cqp_head; - struct sk_buff *rx_skb; - int i; - int ret; - - kthread_stop(nesvnic->mgt_thread); - - /* Free remaining NIC receive buffers */ - first_mgtvnic = nesvnic->mgtvnic[0]; - for (i = 0; i < NES_MGT_QP_COUNT; i++) { - mgtvnic = nesvnic->mgtvnic[i]; - if (mgtvnic == NULL) - continue; - - while (mgtvnic->mgt.rq_head != mgtvnic->mgt.rq_tail) { - rx_skb = mgtvnic->mgt.rx_skb[mgtvnic->mgt.rq_tail]; - nes_mgt_free_skb(nesdev, rx_skb, PCI_DMA_FROMDEVICE); - mgtvnic->mgt.rq_tail++; - mgtvnic->mgt.rq_tail &= (mgtvnic->mgt.rq_size - 1); - } - - spin_lock_irqsave(&nesdev->cqp.lock, flags); - - /* Destroy NIC QP */ - cqp_head = nesdev->cqp.sq_head; - cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head]; - nes_fill_init_cqp_wqe(cqp_wqe, nesdev); - - set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX, - (NES_CQP_DESTROY_QP | NES_CQP_QP_TYPE_NIC)); - set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX, - mgtvnic->mgt.qp_id); - - if (++cqp_head >= nesdev->cqp.sq_size) - cqp_head = 0; - - cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head]; - - /* Destroy NIC CQ */ - nes_fill_init_cqp_wqe(cqp_wqe, nesdev); - set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX, - (NES_CQP_DESTROY_CQ | ((u32)mgtvnic->mgt_cq.cq_size << 16))); - set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX, - (mgtvnic->mgt_cq.cq_number | ((u32)nesdev->ceq_index << 16))); - - if (++cqp_head >= nesdev->cqp.sq_size) - cqp_head = 0; - - nesdev->cqp.sq_head = cqp_head; - barrier(); - - /* Ring doorbell (2 WQEs) */ - nes_write32(nesdev->regs + NES_WQE_ALLOC, 0x02800000 | nesdev->cqp.qp_id); - - spin_unlock_irqrestore(&nesdev->cqp.lock, flags); - nes_debug(NES_DBG_SHUTDOWN, "Waiting for CQP, cqp_head=%u, cqp.sq_head=%u," - " cqp.sq_tail=%u, cqp.sq_size=%u\n", - cqp_head, nesdev->cqp.sq_head, - nesdev->cqp.sq_tail, nesdev->cqp.sq_size); - - ret = wait_event_timeout(nesdev->cqp.waitq, (nesdev->cqp.sq_tail == cqp_head), - NES_EVENT_TIMEOUT); - - nes_debug(NES_DBG_SHUTDOWN, "Destroy MGT QP returned, wait_event_timeout ret = %u, cqp_head=%u," - " cqp.sq_head=%u, cqp.sq_tail=%u\n", - ret, cqp_head, nesdev->cqp.sq_head, nesdev->cqp.sq_tail); - if (!ret) - nes_debug(NES_DBG_SHUTDOWN, "MGT QP%u destroy timeout expired\n", - mgtvnic->mgt.qp_id); - - nesvnic->mgtvnic[i] = NULL; - } - - if (nesvnic->mgt_vbase) { - pci_free_consistent(nesdev->pcidev, nesvnic->mgt_mem_size, nesvnic->mgt_vbase, - nesvnic->mgt_pbase); - nesvnic->mgt_vbase = NULL; - nesvnic->mgt_pbase = 0; - } - - kfree(first_mgtvnic); -} diff --git a/drivers/infiniband/hw/nes/nes_mgt.h b/drivers/infiniband/hw/nes/nes_mgt.h deleted file mode 100644 index 4f7f701c4a81..000000000000 --- a/drivers/infiniband/hw/nes/nes_mgt.h +++ /dev/null @@ -1,97 +0,0 @@ -/* -* Copyright (c) 2006 - 2011 Intel-NE, Inc. All rights reserved. -* -* This software is available to you under a choice of one of two -* licenses. You may choose to be licensed under the terms of the GNU -* General Public License (GPL) Version 2, available from the file -* COPYING in the main directory of this source tree, or the -* OpenIB.org BSD license below: -* -* Redistribution and use in source and binary forms, with or -* without modification, are permitted provided that the following -* conditions are met: -* -* - Redistributions of source code must retain the above -* copyright notice, this list of conditions and the following -* disclaimer. -* -* - Redistributions in binary form must reproduce the above -* copyright notice, this list of conditions and the following -* disclaimer in the documentation and/or other materials -* provided with the distribution. -* -* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS -* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN -* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN -* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -* SOFTWARE. -*/ - -#ifndef __NES_MGT_H -#define __NES_MGT_H - -#define MPA_FRAMING 6 /* length is 2 bytes, crc is 4 bytes */ - -int nes_init_mgt_qp(struct nes_device *nesdev, struct net_device *netdev, struct nes_vnic *nesvnic); -void nes_queue_mgt_skbs(struct sk_buff *skb, struct nes_vnic *nesvnic, struct nes_qp *nesqp); -void nes_destroy_mgt(struct nes_vnic *nesvnic); -void nes_destroy_pau_qp(struct nes_device *nesdev, struct nes_qp *nesqp); - -struct nes_hw_mgt { - struct nes_hw_nic_rq_wqe *rq_vbase; /* virtual address of rq */ - dma_addr_t rq_pbase; /* PCI memory for host rings */ - struct sk_buff *rx_skb[NES_NIC_WQ_SIZE]; - u16 qp_id; - u16 sq_head; - u16 rq_head; - u16 rq_tail; - u16 rq_size; - u8 replenishing_rq; - u8 reserved; - spinlock_t rq_lock; -}; - -struct nes_vnic_mgt { - struct nes_vnic *nesvnic; - struct nes_hw_mgt mgt; - struct nes_hw_nic_cq mgt_cq; - atomic_t rx_skbs_needed; - struct timer_list rq_wqes_timer; - atomic_t rx_skb_timer_running; -}; - -#define MAX_FPDU_FRAGS 4 -struct pau_fpdu_frag { - struct sk_buff *skb; - u64 physaddr; - u32 frag_len; - bool cmplt; -}; - -struct pau_fpdu_info { - struct nes_qp *nesqp; - struct nes_cqp_request *cqp_request; - void *hdr_vbase; - dma_addr_t hdr_pbase; - int hdr_len; - u16 data_len; - u16 frag_cnt; - struct pau_fpdu_frag frags[MAX_FPDU_FRAGS]; -}; - -enum pau_qh_state { - PAU_DEL_QH, - PAU_ADD_LB_QH, - PAU_READY -}; - -struct pau_qh_chg { - struct nes_device *nesdev; - struct nes_vnic *nesvnic; - struct nes_qp *nesqp; -}; - -#endif /* __NES_MGT_H */ diff --git a/drivers/infiniband/hw/nes/nes_nic.c b/drivers/infiniband/hw/nes/nes_nic.c deleted file mode 100644 index 16f33454c198..000000000000 --- a/drivers/infiniband/hw/nes/nes_nic.c +++ /dev/null @@ -1,1870 +0,0 @@ -/* - * Copyright (c) 2006 - 2011 Intel Corporation. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - */ - -#include <linux/module.h> -#include <linux/moduleparam.h> -#include <linux/netdevice.h> -#include <linux/etherdevice.h> -#include <linux/ip.h> -#include <linux/tcp.h> -#include <linux/if_arp.h> -#include <linux/if_vlan.h> -#include <linux/ethtool.h> -#include <linux/slab.h> -#include <net/tcp.h> - -#include <net/inet_common.h> -#include <linux/inet.h> - -#include "nes.h" - -static struct nic_qp_map nic_qp_mapping_0[] = { - {16,0,0,1},{24,4,0,0},{28,8,0,0},{32,12,0,0}, - {20,2,2,1},{26,6,2,0},{30,10,2,0},{34,14,2,0}, - {18,1,1,1},{25,5,1,0},{29,9,1,0},{33,13,1,0}, - {22,3,3,1},{27,7,3,0},{31,11,3,0},{35,15,3,0} -}; - -static struct nic_qp_map nic_qp_mapping_1[] = { - {18,1,1,1},{25,5,1,0},{29,9,1,0},{33,13,1,0}, - {22,3,3,1},{27,7,3,0},{31,11,3,0},{35,15,3,0} -}; - -static struct nic_qp_map nic_qp_mapping_2[] = { - {20,2,2,1},{26,6,2,0},{30,10,2,0},{34,14,2,0} -}; - -static struct nic_qp_map nic_qp_mapping_3[] = { - {22,3,3,1},{27,7,3,0},{31,11,3,0},{35,15,3,0} -}; - -static struct nic_qp_map nic_qp_mapping_4[] = { - {28,8,0,0},{32,12,0,0} -}; - -static struct nic_qp_map nic_qp_mapping_5[] = { - {29,9,1,0},{33,13,1,0} -}; - -static struct nic_qp_map nic_qp_mapping_6[] = { - {30,10,2,0},{34,14,2,0} -}; - -static struct nic_qp_map nic_qp_mapping_7[] = { - {31,11,3,0},{35,15,3,0} -}; - -static struct nic_qp_map *nic_qp_mapping_per_function[] = { - nic_qp_mapping_0, nic_qp_mapping_1, nic_qp_mapping_2, nic_qp_mapping_3, - nic_qp_mapping_4, nic_qp_mapping_5, nic_qp_mapping_6, nic_qp_mapping_7 -}; - -static const u32 default_msg = NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_LINK - | NETIF_MSG_IFUP | NETIF_MSG_IFDOWN; -static int debug = -1; -static int nics_per_function = 1; - -/** - * nes_netdev_poll - */ -static int nes_netdev_poll(struct napi_struct *napi, int budget) -{ - struct nes_vnic *nesvnic = container_of(napi, struct nes_vnic, napi); - struct nes_device *nesdev = nesvnic->nesdev; - struct nes_hw_nic_cq *nescq = &nesvnic->nic_cq; - - nesvnic->budget = budget; - nescq->cqes_pending = 0; - nescq->rx_cqes_completed = 0; - nescq->cqe_allocs_pending = 0; - nescq->rx_pkts_indicated = 0; - - nes_nic_ce_handler(nesdev, nescq); - - if (nescq->cqes_pending == 0) { - napi_complete(napi); - /* clear out completed cqes and arm */ - nes_write32(nesdev->regs+NES_CQE_ALLOC, NES_CQE_ALLOC_NOTIFY_NEXT | - nescq->cq_number | (nescq->cqe_allocs_pending << 16)); - nes_read32(nesdev->regs+NES_CQE_ALLOC); - } else { - /* clear out completed cqes but don't arm */ - nes_write32(nesdev->regs+NES_CQE_ALLOC, - nescq->cq_number | (nescq->cqe_allocs_pending << 16)); - nes_debug(NES_DBG_NETDEV, "%s: exiting with work pending\n", - nesvnic->netdev->name); - } - return nescq->rx_pkts_indicated; -} - - -/** - * nes_netdev_open - Activate the network interface; ifconfig - * ethx up. - */ -static int nes_netdev_open(struct net_device *netdev) -{ - u32 macaddr_low; - u16 macaddr_high; - struct nes_vnic *nesvnic = netdev_priv(netdev); - struct nes_device *nesdev = nesvnic->nesdev; - int ret; - int i; - struct nes_vnic *first_nesvnic = NULL; - u32 nic_active_bit; - u32 nic_active; - struct list_head *list_pos, *list_temp; - unsigned long flags; - - if (nesvnic->netdev_open == 1) - return 0; - - if (netif_msg_ifup(nesvnic)) - printk(KERN_INFO PFX "%s: enabling interface\n", netdev->name); - - ret = nes_init_nic_qp(nesdev, netdev); - if (ret) { - return ret; - } - - netif_carrier_off(netdev); - netif_stop_queue(netdev); - - if ((!nesvnic->of_device_registered) && (nesvnic->rdma_enabled)) { - nesvnic->nesibdev = nes_init_ofa_device(netdev); - if (nesvnic->nesibdev == NULL) { - printk(KERN_ERR PFX "%s: nesvnic->nesibdev alloc failed", netdev->name); - } else { - nesvnic->nesibdev->nesvnic = nesvnic; - ret = nes_register_ofa_device(nesvnic->nesibdev); - if (ret) { - printk(KERN_ERR PFX "%s: Unable to register RDMA device, ret = %d\n", - netdev->name, ret); - } - } - } - /* Set packet filters */ - nic_active_bit = 1 << nesvnic->nic_index; - nic_active = nes_read_indexed(nesdev, NES_IDX_NIC_ACTIVE); - nic_active |= nic_active_bit; - nes_write_indexed(nesdev, NES_IDX_NIC_ACTIVE, nic_active); - nic_active = nes_read_indexed(nesdev, NES_IDX_NIC_MULTICAST_ENABLE); - nic_active |= nic_active_bit; - nes_write_indexed(nesdev, NES_IDX_NIC_MULTICAST_ENABLE, nic_active); - nic_active = nes_read_indexed(nesdev, NES_IDX_NIC_BROADCAST_ON); - nic_active |= nic_active_bit; - nes_write_indexed(nesdev, NES_IDX_NIC_BROADCAST_ON, nic_active); - - macaddr_high = ((u16)netdev->dev_addr[0]) << 8; - macaddr_high += (u16)netdev->dev_addr[1]; - - macaddr_low = ((u32)netdev->dev_addr[2]) << 24; - macaddr_low += ((u32)netdev->dev_addr[3]) << 16; - macaddr_low += ((u32)netdev->dev_addr[4]) << 8; - macaddr_low += (u32)netdev->dev_addr[5]; - - /* Program the various MAC regs */ - for (i = 0; i < NES_MAX_PORT_COUNT; i++) { - if (nesvnic->qp_nic_index[i] == 0xf) { - break; - } - nes_debug(NES_DBG_NETDEV, "i=%d, perfect filter table index= %d, PERF FILTER LOW" - " (Addr:%08X) = %08X, HIGH = %08X.\n", - i, nesvnic->qp_nic_index[i], - NES_IDX_PERFECT_FILTER_LOW+ - (nesvnic->qp_nic_index[i] * 8), - macaddr_low, - (u32)macaddr_high | NES_MAC_ADDR_VALID | - ((((u32)nesvnic->nic_index) << 16))); - nes_write_indexed(nesdev, - NES_IDX_PERFECT_FILTER_LOW + (nesvnic->qp_nic_index[i] * 8), - macaddr_low); - nes_write_indexed(nesdev, - NES_IDX_PERFECT_FILTER_HIGH + (nesvnic->qp_nic_index[i] * 8), - (u32)macaddr_high | NES_MAC_ADDR_VALID | - ((((u32)nesvnic->nic_index) << 16))); - } - - - nes_write32(nesdev->regs+NES_CQE_ALLOC, NES_CQE_ALLOC_NOTIFY_NEXT | - nesvnic->nic_cq.cq_number); - nes_read32(nesdev->regs+NES_CQE_ALLOC); - list_for_each_safe(list_pos, list_temp, &nesdev->nesadapter->nesvnic_list[nesdev->mac_index]) { - first_nesvnic = container_of(list_pos, struct nes_vnic, list); - if (first_nesvnic->netdev_open == 1) - break; - } - if (first_nesvnic->netdev_open == 0) { - nes_debug(NES_DBG_INIT, "Setting up MAC interrupt mask.\n"); - nes_write_indexed(nesdev, NES_IDX_MAC_INT_MASK + (0x200 * nesdev->mac_index), - ~(NES_MAC_INT_LINK_STAT_CHG | NES_MAC_INT_XGMII_EXT | - NES_MAC_INT_TX_UNDERFLOW | NES_MAC_INT_TX_ERROR)); - first_nesvnic = nesvnic; - } - - if (first_nesvnic->linkup) { - /* Enable network packets */ - nesvnic->linkup = 1; - netif_start_queue(netdev); - netif_carrier_on(netdev); - } - - spin_lock_irqsave(&nesdev->nesadapter->phy_lock, flags); - if (nesdev->nesadapter->phy_type[nesdev->mac_index] == NES_PHY_TYPE_SFP_D) { - nesdev->link_recheck = 1; - mod_delayed_work(system_wq, &nesdev->work, - NES_LINK_RECHECK_DELAY); - } - spin_unlock_irqrestore(&nesdev->nesadapter->phy_lock, flags); - - spin_lock_irqsave(&nesvnic->port_ibevent_lock, flags); - if (nesvnic->of_device_registered) { - nesdev->nesadapter->send_term_ok = 1; - if (nesvnic->linkup == 1) { - if (nesdev->iw_status == 0) { - nesdev->iw_status = 1; - nes_port_ibevent(nesvnic); - } - } else { - nesdev->iw_status = 0; - } - } - spin_unlock_irqrestore(&nesvnic->port_ibevent_lock, flags); - - napi_enable(&nesvnic->napi); - nesvnic->netdev_open = 1; - - return 0; -} - - -/** - * nes_netdev_stop - */ -static int nes_netdev_stop(struct net_device *netdev) -{ - struct nes_vnic *nesvnic = netdev_priv(netdev); - struct nes_device *nesdev = nesvnic->nesdev; - u32 nic_active_mask; - u32 nic_active; - struct nes_vnic *first_nesvnic = NULL; - struct list_head *list_pos, *list_temp; - unsigned long flags; - - nes_debug(NES_DBG_SHUTDOWN, "nesvnic=%p, nesdev=%p, netdev=%p %s\n", - nesvnic, nesdev, netdev, netdev->name); - if (nesvnic->netdev_open == 0) - return 0; - - if (netif_msg_ifdown(nesvnic)) - printk(KERN_INFO PFX "%s: disabling interface\n", netdev->name); - netif_carrier_off(netdev); - - /* Disable network packets */ - napi_disable(&nesvnic->napi); - netif_stop_queue(netdev); - list_for_each_safe(list_pos, list_temp, &nesdev->nesadapter->nesvnic_list[nesdev->mac_index]) { - first_nesvnic = container_of(list_pos, struct nes_vnic, list); - if ((first_nesvnic->netdev_open == 1) && (first_nesvnic != nesvnic)) - break; - } - - if ((first_nesvnic->netdev_open == 1) && (first_nesvnic != nesvnic) && - (PCI_FUNC(first_nesvnic->nesdev->pcidev->devfn) != - PCI_FUNC(nesvnic->nesdev->pcidev->devfn))) { - nes_write_indexed(nesdev, NES_IDX_MAC_INT_MASK+ - (0x200*nesdev->mac_index), 0xffffffff); - nes_write_indexed(first_nesvnic->nesdev, - NES_IDX_MAC_INT_MASK+ - (0x200*first_nesvnic->nesdev->mac_index), - ~(NES_MAC_INT_LINK_STAT_CHG | NES_MAC_INT_XGMII_EXT | - NES_MAC_INT_TX_UNDERFLOW | NES_MAC_INT_TX_ERROR)); - } else { - nes_write_indexed(nesdev, NES_IDX_MAC_INT_MASK+(0x200*nesdev->mac_index), 0xffffffff); - } - - nic_active_mask = ~((u32)(1 << nesvnic->nic_index)); - nes_write_indexed(nesdev, NES_IDX_PERFECT_FILTER_HIGH+ - (nesvnic->perfect_filter_index*8), 0); - nic_active = nes_read_indexed(nesdev, NES_IDX_NIC_ACTIVE); - nic_active &= nic_active_mask; - nes_write_indexed(nesdev, NES_IDX_NIC_ACTIVE, nic_active); - nic_active = nes_read_indexed(nesdev, NES_IDX_NIC_MULTICAST_ALL); - nic_active &= nic_active_mask; - nes_write_indexed(nesdev, NES_IDX_NIC_MULTICAST_ALL, nic_active); - nic_active = nes_read_indexed(nesdev, NES_IDX_NIC_MULTICAST_ENABLE); - nic_active &= nic_active_mask; - nes_write_indexed(nesdev, NES_IDX_NIC_MULTICAST_ENABLE, nic_active); - nic_active = nes_read_indexed(nesdev, NES_IDX_NIC_UNICAST_ALL); - nic_active &= nic_active_mask; - nes_write_indexed(nesdev, NES_IDX_NIC_UNICAST_ALL, nic_active); - nic_active = nes_read_indexed(nesdev, NES_IDX_NIC_BROADCAST_ON); - nic_active &= nic_active_mask; - nes_write_indexed(nesdev, NES_IDX_NIC_BROADCAST_ON, nic_active); - - spin_lock_irqsave(&nesvnic->port_ibevent_lock, flags); - if (nesvnic->of_device_registered) { - nesdev->nesadapter->send_term_ok = 0; - nesdev->iw_status = 0; - if (nesvnic->linkup == 1) - nes_port_ibevent(nesvnic); - } - del_timer_sync(&nesvnic->event_timer); - nesvnic->event_timer.function = NULL; - spin_unlock_irqrestore(&nesvnic->port_ibevent_lock, flags); - - nes_destroy_nic_qp(nesvnic); - - nesvnic->netdev_open = 0; - - return 0; -} - - -/** - * nes_nic_send - */ -static bool nes_nic_send(struct sk_buff *skb, struct net_device *netdev) -{ - struct nes_vnic *nesvnic = netdev_priv(netdev); - struct nes_device *nesdev = nesvnic->nesdev; - struct nes_hw_nic *nesnic = &nesvnic->nic; - struct nes_hw_nic_sq_wqe *nic_sqe; - struct tcphdr *tcph; - __le16 *wqe_fragment_length; - u32 wqe_misc; - u16 wqe_fragment_index = 1; /* first fragment (0) is used by copy buffer */ - u16 skb_fragment_index; - dma_addr_t bus_address; - - nic_sqe = &nesnic->sq_vbase[nesnic->sq_head]; - wqe_fragment_length = (__le16 *)&nic_sqe->wqe_words[NES_NIC_SQ_WQE_LENGTH_0_TAG_IDX]; - - /* setup the VLAN tag if present */ - if (skb_vlan_tag_present(skb)) { - nes_debug(NES_DBG_NIC_TX, "%s: VLAN packet to send... VLAN = %08X\n", - netdev->name, skb_vlan_tag_get(skb)); - wqe_misc = NES_NIC_SQ_WQE_TAGVALUE_ENABLE; - wqe_fragment_length[0] = (__force __le16) skb_vlan_tag_get(skb); - } else - wqe_misc = 0; - - /* bump past the vlan tag */ - wqe_fragment_length++; - /* wqe_fragment_address = (u64 *)&nic_sqe->wqe_words[NES_NIC_SQ_WQE_FRAG0_LOW_IDX]; */ - wqe_misc |= NES_NIC_SQ_WQE_COMPLETION; - - if (skb->ip_summed == CHECKSUM_PARTIAL) { - if (skb_is_gso(skb)) { - tcph = tcp_hdr(skb); - /* nes_debug(NES_DBG_NIC_TX, "%s: TSO request... is_gso = %u seg size = %u\n", - netdev->name, skb_is_gso(skb), skb_shinfo(skb)->gso_size); */ - wqe_misc |= NES_NIC_SQ_WQE_LSO_ENABLE | (u16)skb_shinfo(skb)->gso_size; - set_wqe_32bit_value(nic_sqe->wqe_words, NES_NIC_SQ_WQE_LSO_INFO_IDX, - ((u32)tcph->doff) | - (((u32)(((unsigned char *)tcph) - skb->data)) << 4)); - } - } else { /* CHECKSUM_HW */ - wqe_misc |= NES_NIC_SQ_WQE_DISABLE_CHKSUM; - } - - set_wqe_32bit_value(nic_sqe->wqe_words, NES_NIC_SQ_WQE_TOTAL_LENGTH_IDX, - skb->len); - memcpy(&nesnic->first_frag_vbase[nesnic->sq_head].buffer, - skb->data, min(((unsigned int)NES_FIRST_FRAG_SIZE), skb_headlen(skb))); - wqe_fragment_length[0] = cpu_to_le16(min(((unsigned int)NES_FIRST_FRAG_SIZE), - skb_headlen(skb))); - wqe_fragment_length[1] = 0; - if (skb_headlen(skb) > NES_FIRST_FRAG_SIZE) { - if ((skb_shinfo(skb)->nr_frags + 1) > 4) { - nes_debug(NES_DBG_NIC_TX, "%s: Packet with %u fragments not sent, skb_headlen=%u\n", - netdev->name, skb_shinfo(skb)->nr_frags + 2, skb_headlen(skb)); - kfree_skb(skb); - nesvnic->tx_sw_dropped++; - return false; - } - set_bit(nesnic->sq_head, nesnic->first_frag_overflow); - bus_address = pci_map_single(nesdev->pcidev, skb->data + NES_FIRST_FRAG_SIZE, - skb_headlen(skb) - NES_FIRST_FRAG_SIZE, PCI_DMA_TODEVICE); - wqe_fragment_length[wqe_fragment_index++] = - cpu_to_le16(skb_headlen(skb) - NES_FIRST_FRAG_SIZE); - wqe_fragment_length[wqe_fragment_index] = 0; - set_wqe_64bit_value(nic_sqe->wqe_words, NES_NIC_SQ_WQE_FRAG1_LOW_IDX, - ((u64)(bus_address))); - nesnic->tx_skb[nesnic->sq_head] = skb; - } - - if (skb_headlen(skb) == skb->len) { - if (skb_headlen(skb) <= NES_FIRST_FRAG_SIZE) { - nic_sqe->wqe_words[NES_NIC_SQ_WQE_LENGTH_2_1_IDX] = 0; - nesnic->tx_skb[nesnic->sq_head] = skb; - } - } else { - /* Deal with Fragments */ - nesnic->tx_skb[nesnic->sq_head] = skb; - for (skb_fragment_index = 0; skb_fragment_index < skb_shinfo(skb)->nr_frags; - skb_fragment_index++) { - skb_frag_t *frag = - &skb_shinfo(skb)->frags[skb_fragment_index]; - bus_address = skb_frag_dma_map(&nesdev->pcidev->dev, - frag, 0, skb_frag_size(frag), - DMA_TO_DEVICE); - wqe_fragment_length[wqe_fragment_index] = - cpu_to_le16(skb_frag_size(&skb_shinfo(skb)->frags[skb_fragment_index])); - set_wqe_64bit_value(nic_sqe->wqe_words, NES_NIC_SQ_WQE_FRAG0_LOW_IDX+(2*wqe_fragment_index), - bus_address); - wqe_fragment_index++; - if (wqe_fragment_index < 5) - wqe_fragment_length[wqe_fragment_index] = 0; - } - } - - set_wqe_32bit_value(nic_sqe->wqe_words, NES_NIC_SQ_WQE_MISC_IDX, wqe_misc); - nesnic->sq_head++; - nesnic->sq_head &= nesnic->sq_size - 1; - return true; -} - - -/** - * nes_netdev_start_xmit - */ -static netdev_tx_t nes_netdev_start_xmit(struct sk_buff *skb, struct net_device *netdev) -{ - struct nes_vnic *nesvnic = netdev_priv(netdev); - struct nes_device *nesdev = nesvnic->nesdev; - struct nes_hw_nic *nesnic = &nesvnic->nic; - struct nes_hw_nic_sq_wqe *nic_sqe; - struct tcphdr *tcph; - /* struct udphdr *udph; */ -#define NES_MAX_TSO_FRAGS MAX_SKB_FRAGS - /* 64K segment plus overflow on each side */ - dma_addr_t tso_bus_address[NES_MAX_TSO_FRAGS]; - dma_addr_t bus_address; - u32 tso_frag_index; - u32 tso_frag_count; - u32 tso_wqe_length; - u32 curr_tcp_seq; - u32 wqe_count=1; - struct iphdr *iph; - __le16 *wqe_fragment_length; - u32 nr_frags; - u32 original_first_length; - /* u64 *wqe_fragment_address; */ - /* first fragment (0) is used by copy buffer */ - u16 wqe_fragment_index=1; - u16 hoffset; - u16 nhoffset; - u16 wqes_needed; - u16 wqes_available; - u32 wqe_misc; - - /* - * nes_debug(NES_DBG_NIC_TX, "%s Request to tx NIC packet length %u, headlen %u," - * " (%u frags), tso_size=%u\n", - * netdev->name, skb->len, skb_headlen(skb), - * skb_shinfo(skb)->nr_frags, skb_is_gso(skb)); - */ - - if (netif_queue_stopped(netdev)) - return NETDEV_TX_BUSY; - - /* Check if SQ is full */ - if ((((nesnic->sq_tail+(nesnic->sq_size*2))-nesnic->sq_head) & (nesnic->sq_size - 1)) == 1) { - if (!netif_queue_stopped(netdev)) { - netif_stop_queue(netdev); - barrier(); - if ((((((volatile u16)nesnic->sq_tail)+(nesnic->sq_size*2))-nesnic->sq_head) & (nesnic->sq_size - 1)) != 1) { - netif_start_queue(netdev); - goto sq_no_longer_full; - } - } - nesvnic->sq_full++; - return NETDEV_TX_BUSY; - } - -sq_no_longer_full: - nr_frags = skb_shinfo(skb)->nr_frags; - if (skb_headlen(skb) > NES_FIRST_FRAG_SIZE) { - nr_frags++; - } - /* Check if too many fragments */ - if (unlikely((nr_frags > 4))) { - if (skb_is_gso(skb)) { - nesvnic->segmented_tso_requests++; - nesvnic->tso_requests++; - /* Basically 4 fragments available per WQE with extended fragments */ - wqes_needed = nr_frags >> 2; - wqes_needed += (nr_frags&3)?1:0; - wqes_available = (((nesnic->sq_tail+nesnic->sq_size)-nesnic->sq_head) - 1) & - (nesnic->sq_size - 1); - - if (unlikely(wqes_needed > wqes_available)) { - if (!netif_queue_stopped(netdev)) { - netif_stop_queue(netdev); - barrier(); - wqes_available = (((((volatile u16)nesnic->sq_tail)+nesnic->sq_size)-nesnic->sq_head) - 1) & - (nesnic->sq_size - 1); - if (wqes_needed <= wqes_available) { - netif_start_queue(netdev); - goto tso_sq_no_longer_full; - } - } - nesvnic->sq_full++; - nes_debug(NES_DBG_NIC_TX, "%s: HNIC SQ full- TSO request has too many frags!\n", - netdev->name); - return NETDEV_TX_BUSY; - } -tso_sq_no_longer_full: - /* Map all the buffers */ - for (tso_frag_count=0; tso_frag_count < skb_shinfo(skb)->nr_frags; - tso_frag_count++) { - skb_frag_t *frag = - &skb_shinfo(skb)->frags[tso_frag_count]; - tso_bus_address[tso_frag_count] = - skb_frag_dma_map(&nesdev->pcidev->dev, - frag, 0, skb_frag_size(frag), - DMA_TO_DEVICE); - } - - tso_frag_index = 0; - curr_tcp_seq = ntohl(tcp_hdr(skb)->seq); - hoffset = skb_transport_header(skb) - skb->data; - nhoffset = skb_network_header(skb) - skb->data; - original_first_length = hoffset + ((((struct tcphdr *)skb_transport_header(skb))->doff)<<2); - - for (wqe_count=0; wqe_count<((u32)wqes_needed); wqe_count++) { - tso_wqe_length = 0; - nic_sqe = &nesnic->sq_vbase[nesnic->sq_head]; - wqe_fragment_length = - (__le16 *)&nic_sqe->wqe_words[NES_NIC_SQ_WQE_LENGTH_0_TAG_IDX]; - /* setup the VLAN tag if present */ - if (skb_vlan_tag_present(skb)) { - nes_debug(NES_DBG_NIC_TX, "%s: VLAN packet to send... VLAN = %08X\n", - netdev->name, - skb_vlan_tag_get(skb)); - wqe_misc = NES_NIC_SQ_WQE_TAGVALUE_ENABLE; - wqe_fragment_length[0] = (__force __le16) skb_vlan_tag_get(skb); - } else - wqe_misc = 0; - - /* bump past the vlan tag */ - wqe_fragment_length++; - - /* Assumes header totally fits in allocated buffer and is in first fragment */ - if (original_first_length > NES_FIRST_FRAG_SIZE) { - nes_debug(NES_DBG_NIC_TX, "ERROR: SKB header too big, headlen=%u, FIRST_FRAG_SIZE=%u\n", - original_first_length, NES_FIRST_FRAG_SIZE); - nes_debug(NES_DBG_NIC_TX, "%s Request to tx NIC packet length %u, headlen %u," - " (%u frags), is_gso = %u tso_size=%u\n", - netdev->name, - skb->len, skb_headlen(skb), - skb_shinfo(skb)->nr_frags, skb_is_gso(skb), skb_shinfo(skb)->gso_size); - } - memcpy(&nesnic->first_frag_vbase[nesnic->sq_head].buffer, - skb->data, min(((unsigned int)NES_FIRST_FRAG_SIZE), - original_first_length)); - iph = (struct iphdr *) - (&nesnic->first_frag_vbase[nesnic->sq_head].buffer[nhoffset]); - tcph = (struct tcphdr *) - (&nesnic->first_frag_vbase[nesnic->sq_head].buffer[hoffset]); - if ((wqe_count+1)!=(u32)wqes_needed) { - tcph->fin = 0; - tcph->psh = 0; - tcph->rst = 0; - tcph->urg = 0; - } - if (wqe_count) { - tcph->syn = 0; - } - tcph->seq = htonl(curr_tcp_seq); - wqe_fragment_length[0] = cpu_to_le16(min(((unsigned int)NES_FIRST_FRAG_SIZE), - original_first_length)); - - wqe_fragment_index = 1; - if ((wqe_count==0) && (skb_headlen(skb) > original_first_length)) { - set_bit(nesnic->sq_head, nesnic->first_frag_overflow); - bus_address = pci_map_single(nesdev->pcidev, skb->data + original_first_length, - skb_headlen(skb) - original_first_length, PCI_DMA_TODEVICE); - wqe_fragment_length[wqe_fragment_index++] = - cpu_to_le16(skb_headlen(skb) - original_first_length); - wqe_fragment_length[wqe_fragment_index] = 0; - set_wqe_64bit_value(nic_sqe->wqe_words, NES_NIC_SQ_WQE_FRAG1_LOW_IDX, - bus_address); - tso_wqe_length += skb_headlen(skb) - - original_first_length; - } - while (wqe_fragment_index < 5) { - wqe_fragment_length[wqe_fragment_index] = - cpu_to_le16(skb_frag_size(&skb_shinfo(skb)->frags[tso_frag_index])); - set_wqe_64bit_value(nic_sqe->wqe_words, NES_NIC_SQ_WQE_FRAG0_LOW_IDX+(2*wqe_fragment_index), - (u64)tso_bus_address[tso_frag_index]); - wqe_fragment_index++; - tso_wqe_length += skb_frag_size(&skb_shinfo(skb)->frags[tso_frag_index++]); - if (wqe_fragment_index < 5) - wqe_fragment_length[wqe_fragment_index] = 0; - if (tso_frag_index == tso_frag_count) - break; - } - if ((wqe_count+1) == (u32)wqes_needed) { - nesnic->tx_skb[nesnic->sq_head] = skb; - } else { - nesnic->tx_skb[nesnic->sq_head] = NULL; - } - wqe_misc |= NES_NIC_SQ_WQE_COMPLETION | (u16)skb_shinfo(skb)->gso_size; - if ((tso_wqe_length + original_first_length) > skb_shinfo(skb)->gso_size) { - wqe_misc |= NES_NIC_SQ_WQE_LSO_ENABLE; - } else { - iph->tot_len = htons(tso_wqe_length + original_first_length - nhoffset); - } - - set_wqe_32bit_value(nic_sqe->wqe_words, NES_NIC_SQ_WQE_MISC_IDX, - wqe_misc); - set_wqe_32bit_value(nic_sqe->wqe_words, NES_NIC_SQ_WQE_LSO_INFO_IDX, - ((u32)tcph->doff) | (((u32)hoffset) << 4)); - - set_wqe_32bit_value(nic_sqe->wqe_words, NES_NIC_SQ_WQE_TOTAL_LENGTH_IDX, - tso_wqe_length + original_first_length); - curr_tcp_seq += tso_wqe_length; - nesnic->sq_head++; - nesnic->sq_head &= nesnic->sq_size-1; - } - } else { - hoffset = skb_transport_header(skb) - skb->data; - nhoffset = skb_network_header(skb) - skb->data; - if (skb_linearize(skb)) { - nesvnic->tx_sw_dropped++; - kfree_skb(skb); - return NETDEV_TX_OK; - } - nesvnic->linearized_skbs++; - skb_set_transport_header(skb, hoffset); - skb_set_network_header(skb, nhoffset); - if (!nes_nic_send(skb, netdev)) - return NETDEV_TX_OK; - } - } else { - if (!nes_nic_send(skb, netdev)) - return NETDEV_TX_OK; - } - - barrier(); - - if (wqe_count) - nes_write32(nesdev->regs+NES_WQE_ALLOC, - (wqe_count << 24) | (1 << 23) | nesvnic->nic.qp_id); - - netif_trans_update(netdev); - - return NETDEV_TX_OK; -} - - -/** - * nes_netdev_get_stats - */ -static struct net_device_stats *nes_netdev_get_stats(struct net_device *netdev) -{ - struct nes_vnic *nesvnic = netdev_priv(netdev); - struct nes_device *nesdev = nesvnic->nesdev; - u64 u64temp; - u32 u32temp; - - u32temp = nes_read_indexed(nesdev, - NES_IDX_ENDNODE0_NSTAT_RX_DISCARD + (nesvnic->nic_index*0x200)); - nesvnic->netstats.rx_dropped += u32temp; - nesvnic->endnode_nstat_rx_discard += u32temp; - - u64temp = (u64)nes_read_indexed(nesdev, - NES_IDX_ENDNODE0_NSTAT_RX_OCTETS_LO + (nesvnic->nic_index*0x200)); - u64temp += ((u64)nes_read_indexed(nesdev, - NES_IDX_ENDNODE0_NSTAT_RX_OCTETS_HI + (nesvnic->nic_index*0x200))) << 32; - - nesvnic->endnode_nstat_rx_octets += u64temp; - nesvnic->netstats.rx_bytes += u64temp; - - u64temp = (u64)nes_read_indexed(nesdev, - NES_IDX_ENDNODE0_NSTAT_RX_FRAMES_LO + (nesvnic->nic_index*0x200)); - u64temp += ((u64)nes_read_indexed(nesdev, - NES_IDX_ENDNODE0_NSTAT_RX_FRAMES_HI + (nesvnic->nic_index*0x200))) << 32; - - nesvnic->endnode_nstat_rx_frames += u64temp; - nesvnic->netstats.rx_packets += u64temp; - - u64temp = (u64)nes_read_indexed(nesdev, - NES_IDX_ENDNODE0_NSTAT_TX_OCTETS_LO + (nesvnic->nic_index*0x200)); - u64temp += ((u64)nes_read_indexed(nesdev, - NES_IDX_ENDNODE0_NSTAT_TX_OCTETS_HI + (nesvnic->nic_index*0x200))) << 32; - - nesvnic->endnode_nstat_tx_octets += u64temp; - nesvnic->netstats.tx_bytes += u64temp; - - u64temp = (u64)nes_read_indexed(nesdev, - NES_IDX_ENDNODE0_NSTAT_TX_FRAMES_LO + (nesvnic->nic_index*0x200)); - u64temp += ((u64)nes_read_indexed(nesdev, - NES_IDX_ENDNODE0_NSTAT_TX_FRAMES_HI + (nesvnic->nic_index*0x200))) << 32; - - nesvnic->endnode_nstat_tx_frames += u64temp; - nesvnic->netstats.tx_packets += u64temp; - - u32temp = nes_read_indexed(nesdev, - NES_IDX_MAC_RX_SHORT_FRAMES + (nesvnic->nesdev->mac_index*0x200)); - nesvnic->netstats.rx_dropped += u32temp; - nesvnic->nesdev->mac_rx_errors += u32temp; - nesvnic->nesdev->mac_rx_short_frames += u32temp; - - u32temp = nes_read_indexed(nesdev, - NES_IDX_MAC_RX_OVERSIZED_FRAMES + (nesvnic->nesdev->mac_index*0x200)); - nesvnic->netstats.rx_dropped += u32temp; - nesvnic->nesdev->mac_rx_errors += u32temp; - nesvnic->nesdev->mac_rx_oversized_frames += u32temp; - - u32temp = nes_read_indexed(nesdev, - NES_IDX_MAC_RX_JABBER_FRAMES + (nesvnic->nesdev->mac_index*0x200)); - nesvnic->netstats.rx_dropped += u32temp; - nesvnic->nesdev->mac_rx_errors += u32temp; - nesvnic->nesdev->mac_rx_jabber_frames += u32temp; - - u32temp = nes_read_indexed(nesdev, - NES_IDX_MAC_RX_SYMBOL_ERR_FRAMES + (nesvnic->nesdev->mac_index*0x200)); - nesvnic->netstats.rx_dropped += u32temp; - nesvnic->nesdev->mac_rx_errors += u32temp; - nesvnic->nesdev->mac_rx_symbol_err_frames += u32temp; - - u32temp = nes_read_indexed(nesdev, - NES_IDX_MAC_RX_LENGTH_ERR_FRAMES + (nesvnic->nesdev->mac_index*0x200)); - nesvnic->netstats.rx_length_errors += u32temp; - nesvnic->nesdev->mac_rx_errors += u32temp; - - u32temp = nes_read_indexed(nesdev, - NES_IDX_MAC_RX_CRC_ERR_FRAMES + (nesvnic->nesdev->mac_index*0x200)); - nesvnic->nesdev->mac_rx_errors += u32temp; - nesvnic->nesdev->mac_rx_crc_errors += u32temp; - nesvnic->netstats.rx_crc_errors += u32temp; - - u32temp = nes_read_indexed(nesdev, - NES_IDX_MAC_TX_ERRORS + (nesvnic->nesdev->mac_index*0x200)); - nesvnic->nesdev->mac_tx_errors += u32temp; - nesvnic->netstats.tx_errors += u32temp; - - return &nesvnic->netstats; -} - - -/** - * nes_netdev_tx_timeout - */ -static void nes_netdev_tx_timeout(struct net_device *netdev) -{ - struct nes_vnic *nesvnic = netdev_priv(netdev); - - if (netif_msg_timer(nesvnic)) - nes_debug(NES_DBG_NIC_TX, "%s: tx timeout\n", netdev->name); -} - - -/** - * nes_netdev_set_mac_address - */ -static int nes_netdev_set_mac_address(struct net_device *netdev, void *p) -{ - struct nes_vnic *nesvnic = netdev_priv(netdev); - struct nes_device *nesdev = nesvnic->nesdev; - struct sockaddr *mac_addr = p; - int i; - u32 macaddr_low; - u16 macaddr_high; - - if (!is_valid_ether_addr(mac_addr->sa_data)) - return -EADDRNOTAVAIL; - - memcpy(netdev->dev_addr, mac_addr->sa_data, netdev->addr_len); - printk(PFX "%s: Address length = %d, Address = %pM\n", - __func__, netdev->addr_len, mac_addr->sa_data); - macaddr_high = ((u16)netdev->dev_addr[0]) << 8; - macaddr_high += (u16)netdev->dev_addr[1]; - macaddr_low = ((u32)netdev->dev_addr[2]) << 24; - macaddr_low += ((u32)netdev->dev_addr[3]) << 16; - macaddr_low += ((u32)netdev->dev_addr[4]) << 8; - macaddr_low += (u32)netdev->dev_addr[5]; - - for (i = 0; i < NES_MAX_PORT_COUNT; i++) { - if (nesvnic->qp_nic_index[i] == 0xf) { - break; - } - nes_write_indexed(nesdev, - NES_IDX_PERFECT_FILTER_LOW + (nesvnic->qp_nic_index[i] * 8), - macaddr_low); - nes_write_indexed(nesdev, - NES_IDX_PERFECT_FILTER_HIGH + (nesvnic->qp_nic_index[i] * 8), - (u32)macaddr_high | NES_MAC_ADDR_VALID | - ((((u32)nesvnic->nic_index) << 16))); - } - return 0; -} - - -static void set_allmulti(struct nes_device *nesdev, u32 nic_active_bit) -{ - u32 nic_active; - - nic_active = nes_read_indexed(nesdev, NES_IDX_NIC_MULTICAST_ALL); - nic_active |= nic_active_bit; - nes_write_indexed(nesdev, NES_IDX_NIC_MULTICAST_ALL, nic_active); - nic_active = nes_read_indexed(nesdev, NES_IDX_NIC_UNICAST_ALL); - nic_active &= ~nic_active_bit; - nes_write_indexed(nesdev, NES_IDX_NIC_UNICAST_ALL, nic_active); -} - -#define get_addr(addrs, index) ((addrs) + (index) * ETH_ALEN) - -/** - * nes_netdev_set_multicast_list - */ -static void nes_netdev_set_multicast_list(struct net_device *netdev) -{ - struct nes_vnic *nesvnic = netdev_priv(netdev); - struct nes_device *nesdev = nesvnic->nesdev; - struct nes_adapter *nesadapter = nesvnic->nesdev->nesadapter; - u32 nic_active_bit; - u32 nic_active; - u32 perfect_filter_register_address; - u32 macaddr_low; - u16 macaddr_high; - u8 mc_all_on = 0; - u8 mc_index; - int mc_nic_index = -1; - u8 pft_entries_preallocated = max(nesadapter->adapter_fcn_count * - nics_per_function, 4); - u8 max_pft_entries_avaiable = NES_PFT_SIZE - pft_entries_preallocated; - unsigned long flags; - int mc_count = netdev_mc_count(netdev); - - spin_lock_irqsave(&nesadapter->resource_lock, flags); - nic_active_bit = 1 << nesvnic->nic_index; - - if (netdev->flags & IFF_PROMISC) { - nic_active = nes_read_indexed(nesdev, NES_IDX_NIC_MULTICAST_ALL); - nic_active |= nic_active_bit; - nes_write_indexed(nesdev, NES_IDX_NIC_MULTICAST_ALL, nic_active); - nic_active = nes_read_indexed(nesdev, NES_IDX_NIC_UNICAST_ALL); - nic_active |= nic_active_bit; - nes_write_indexed(nesdev, NES_IDX_NIC_UNICAST_ALL, nic_active); - mc_all_on = 1; - } else if ((netdev->flags & IFF_ALLMULTI) || - (nesvnic->nic_index > 3)) { - set_allmulti(nesdev, nic_active_bit); - mc_all_on = 1; - } else { - nic_active = nes_read_indexed(nesdev, NES_IDX_NIC_MULTICAST_ALL); - nic_active &= ~nic_active_bit; - nes_write_indexed(nesdev, NES_IDX_NIC_MULTICAST_ALL, nic_active); - nic_active = nes_read_indexed(nesdev, NES_IDX_NIC_UNICAST_ALL); - nic_active &= ~nic_active_bit; - nes_write_indexed(nesdev, NES_IDX_NIC_UNICAST_ALL, nic_active); - } - - nes_debug(NES_DBG_NIC_RX, "Number of MC entries = %d, Promiscuous = %d, All Multicast = %d.\n", - mc_count, !!(netdev->flags & IFF_PROMISC), - !!(netdev->flags & IFF_ALLMULTI)); - if (!mc_all_on) { - char *addrs; - int i; - struct netdev_hw_addr *ha; - - addrs = kmalloc_array(mc_count, ETH_ALEN, GFP_ATOMIC); - if (!addrs) { - set_allmulti(nesdev, nic_active_bit); - goto unlock; - } - i = 0; - netdev_for_each_mc_addr(ha, netdev) - memcpy(get_addr(addrs, i++), ha->addr, ETH_ALEN); - - perfect_filter_register_address = NES_IDX_PERFECT_FILTER_LOW + - pft_entries_preallocated * 0x8; - for (i = 0, mc_index = 0; mc_index < max_pft_entries_avaiable; - mc_index++) { - while (i < mc_count && nesvnic->mcrq_mcast_filter && - ((mc_nic_index = nesvnic->mcrq_mcast_filter(nesvnic, - get_addr(addrs, i++))) == 0)); - if (mc_nic_index < 0) - mc_nic_index = nesvnic->nic_index; - while (nesadapter->pft_mcast_map[mc_index] < 16 && - nesadapter->pft_mcast_map[mc_index] != - nesvnic->nic_index && - mc_index < max_pft_entries_avaiable) { - nes_debug(NES_DBG_NIC_RX, - "mc_index=%d skipping nic_index=%d, used for=%d\n", - mc_index, nesvnic->nic_index, - nesadapter->pft_mcast_map[mc_index]); - mc_index++; - } - if (mc_index >= max_pft_entries_avaiable) - break; - if (i < mc_count) { - char *addr = get_addr(addrs, i++); - - nes_debug(NES_DBG_NIC_RX, "Assigning MC Address %pM to register 0x%04X nic_idx=%d\n", - addr, - perfect_filter_register_address+(mc_index * 8), - mc_nic_index); - macaddr_high = ((u8) addr[0]) << 8; - macaddr_high += (u8) addr[1]; - macaddr_low = ((u8) addr[2]) << 24; - macaddr_low += ((u8) addr[3]) << 16; - macaddr_low += ((u8) addr[4]) << 8; - macaddr_low += (u8) addr[5]; - - nes_write_indexed(nesdev, - perfect_filter_register_address+(mc_index * 8), - macaddr_low); - nes_write_indexed(nesdev, - perfect_filter_register_address+4+(mc_index * 8), - (u32)macaddr_high | NES_MAC_ADDR_VALID | - ((((u32)(1<<mc_nic_index)) << 16))); - nesadapter->pft_mcast_map[mc_index] = - nesvnic->nic_index; - } else { - nes_debug(NES_DBG_NIC_RX, "Clearing MC Address at register 0x%04X\n", - perfect_filter_register_address+(mc_index * 8)); - nes_write_indexed(nesdev, - perfect_filter_register_address+4+(mc_index * 8), - 0); - nesadapter->pft_mcast_map[mc_index] = 255; - } - } - kfree(addrs); - /* PFT is not large enough */ - if (i < mc_count) - set_allmulti(nesdev, nic_active_bit); - } - -unlock: - spin_unlock_irqrestore(&nesadapter->resource_lock, flags); -} - - -/** - * nes_netdev_change_mtu - */ -static int nes_netdev_change_mtu(struct net_device *netdev, int new_mtu) -{ - struct nes_vnic *nesvnic = netdev_priv(netdev); - struct nes_device *nesdev = nesvnic->nesdev; - u8 jumbomode = 0; - u32 nic_active; - u32 nic_active_bit; - u32 uc_all_active; - u32 mc_all_active; - - netdev->mtu = new_mtu; - nesvnic->max_frame_size = new_mtu + VLAN_ETH_HLEN; - - if (netdev->mtu > ETH_DATA_LEN) { - jumbomode=1; - } - nes_nic_init_timer_defaults(nesdev, jumbomode); - - if (netif_running(netdev)) { - nic_active_bit = 1 << nesvnic->nic_index; - mc_all_active = nes_read_indexed(nesdev, - NES_IDX_NIC_MULTICAST_ALL) & nic_active_bit; - uc_all_active = nes_read_indexed(nesdev, - NES_IDX_NIC_UNICAST_ALL) & nic_active_bit; - - nes_netdev_stop(netdev); - nes_netdev_open(netdev); - - nic_active = nes_read_indexed(nesdev, - NES_IDX_NIC_MULTICAST_ALL); - nic_active |= mc_all_active; - nes_write_indexed(nesdev, NES_IDX_NIC_MULTICAST_ALL, - nic_active); - - nic_active = nes_read_indexed(nesdev, NES_IDX_NIC_UNICAST_ALL); - nic_active |= uc_all_active; - nes_write_indexed(nesdev, NES_IDX_NIC_UNICAST_ALL, nic_active); - } - - return 0; -} - - -static const char nes_ethtool_stringset[][ETH_GSTRING_LEN] = { - "Link Change Interrupts", - "Linearized SKBs", - "T/GSO Requests", - "Pause Frames Sent", - "Pause Frames Received", - "Internal Routing Errors", - "SQ SW Dropped SKBs", - "SQ Full", - "Segmented TSO Requests", - "Rx Symbol Errors", - "Rx Jabber Errors", - "Rx Oversized Frames", - "Rx Short Frames", - "Rx Length Errors", - "Rx CRC Errors", - "Rx Port Discard", - "Endnode Rx Discards", - "Endnode Rx Octets", - "Endnode Rx Frames", - "Endnode Tx Octets", - "Endnode Tx Frames", - "Tx Errors", - "mh detected", - "mh pauses", - "Retransmission Count", - "CM Connects", - "CM Accepts", - "Disconnects", - "Connected Events", - "Connect Requests", - "CM Rejects", - "ModifyQP Timeouts", - "CreateQPs", - "SW DestroyQPs", - "DestroyQPs", - "CM Closes", - "CM Packets Sent", - "CM Packets Bounced", - "CM Packets Created", - "CM Packets Rcvd", - "CM Packets Dropped", - "CM Packets Retrans", - "CM Listens Created", - "CM Listens Destroyed", - "CM Backlog Drops", - "CM Loopbacks", - "CM Nodes Created", - "CM Nodes Destroyed", - "CM Accel Drops", - "CM Resets Received", - "Free 4Kpbls", - "Free 256pbls", - "Timer Inits", - "PAU CreateQPs", - "PAU DestroyQPs", -}; -#define NES_ETHTOOL_STAT_COUNT ARRAY_SIZE(nes_ethtool_stringset) - - -/** - * nes_netdev_get_sset_count - */ -static int nes_netdev_get_sset_count(struct net_device *netdev, int stringset) -{ - if (stringset == ETH_SS_STATS) - return NES_ETHTOOL_STAT_COUNT; - else - return -EINVAL; -} - - -/** - * nes_netdev_get_strings - */ -static void nes_netdev_get_strings(struct net_device *netdev, u32 stringset, - u8 *ethtool_strings) -{ - if (stringset == ETH_SS_STATS) - memcpy(ethtool_strings, - &nes_ethtool_stringset, - sizeof(nes_ethtool_stringset)); -} - - -/** - * nes_netdev_get_ethtool_stats - */ - -static void nes_netdev_get_ethtool_stats(struct net_device *netdev, - struct ethtool_stats *target_ethtool_stats, u64 *target_stat_values) -{ - u64 u64temp; - struct nes_vnic *nesvnic = netdev_priv(netdev); - struct nes_device *nesdev = nesvnic->nesdev; - struct nes_adapter *nesadapter = nesdev->nesadapter; - u32 nic_count; - u32 u32temp; - u32 index = 0; - - target_ethtool_stats->n_stats = NES_ETHTOOL_STAT_COUNT; - target_stat_values[index] = nesvnic->nesdev->link_status_interrupts; - target_stat_values[++index] = nesvnic->linearized_skbs; - target_stat_values[++index] = nesvnic->tso_requests; - - u32temp = nes_read_indexed(nesdev, - NES_IDX_MAC_TX_PAUSE_FRAMES + (nesvnic->nesdev->mac_index*0x200)); - nesvnic->nesdev->mac_pause_frames_sent += u32temp; - target_stat_values[++index] = nesvnic->nesdev->mac_pause_frames_sent; - - u32temp = nes_read_indexed(nesdev, - NES_IDX_MAC_RX_PAUSE_FRAMES + (nesvnic->nesdev->mac_index*0x200)); - nesvnic->nesdev->mac_pause_frames_received += u32temp; - - u32temp = nes_read_indexed(nesdev, - NES_IDX_PORT_RX_DISCARDS + (nesvnic->nesdev->mac_index*0x40)); - nesvnic->nesdev->port_rx_discards += u32temp; - nesvnic->netstats.rx_dropped += u32temp; - - u32temp = nes_read_indexed(nesdev, - NES_IDX_PORT_TX_DISCARDS + (nesvnic->nesdev->mac_index*0x40)); - nesvnic->nesdev->port_tx_discards += u32temp; - nesvnic->netstats.tx_dropped += u32temp; - - u32temp = nes_read_indexed(nesdev, - NES_IDX_MAC_RX_SHORT_FRAMES + (nesvnic->nesdev->mac_index*0x200)); - nesvnic->netstats.rx_dropped += u32temp; - nesvnic->nesdev->mac_rx_errors += u32temp; - nesvnic->nesdev->mac_rx_short_frames += u32temp; - - u32temp = nes_read_indexed(nesdev, - NES_IDX_MAC_RX_OVERSIZED_FRAMES + (nesvnic->nesdev->mac_index*0x200)); - nesvnic->netstats.rx_dropped += u32temp; - nesvnic->nesdev->mac_rx_errors += u32temp; - nesvnic->nesdev->mac_rx_oversized_frames += u32temp; - - u32temp = nes_read_indexed(nesdev, - NES_IDX_MAC_RX_JABBER_FRAMES + (nesvnic->nesdev->mac_index*0x200)); - nesvnic->netstats.rx_dropped += u32temp; - nesvnic->nesdev->mac_rx_errors += u32temp; - nesvnic->nesdev->mac_rx_jabber_frames += u32temp; - - u32temp = nes_read_indexed(nesdev, - NES_IDX_MAC_RX_SYMBOL_ERR_FRAMES + (nesvnic->nesdev->mac_index*0x200)); - nesvnic->netstats.rx_dropped += u32temp; - nesvnic->nesdev->mac_rx_errors += u32temp; - nesvnic->nesdev->mac_rx_symbol_err_frames += u32temp; - - u32temp = nes_read_indexed(nesdev, - NES_IDX_MAC_RX_LENGTH_ERR_FRAMES + (nesvnic->nesdev->mac_index*0x200)); - nesvnic->netstats.rx_length_errors += u32temp; - nesvnic->nesdev->mac_rx_errors += u32temp; - - u32temp = nes_read_indexed(nesdev, - NES_IDX_MAC_RX_CRC_ERR_FRAMES + (nesvnic->nesdev->mac_index*0x200)); - nesvnic->nesdev->mac_rx_errors += u32temp; - nesvnic->nesdev->mac_rx_crc_errors += u32temp; - nesvnic->netstats.rx_crc_errors += u32temp; - - u32temp = nes_read_indexed(nesdev, - NES_IDX_MAC_TX_ERRORS + (nesvnic->nesdev->mac_index*0x200)); - nesvnic->nesdev->mac_tx_errors += u32temp; - nesvnic->netstats.tx_errors += u32temp; - - for (nic_count = 0; nic_count < NES_MAX_PORT_COUNT; nic_count++) { - if (nesvnic->qp_nic_index[nic_count] == 0xf) - break; - - u32temp = nes_read_indexed(nesdev, - NES_IDX_ENDNODE0_NSTAT_RX_DISCARD + - (nesvnic->qp_nic_index[nic_count]*0x200)); - nesvnic->netstats.rx_dropped += u32temp; - nesvnic->endnode_nstat_rx_discard += u32temp; - - u64temp = (u64)nes_read_indexed(nesdev, - NES_IDX_ENDNODE0_NSTAT_RX_OCTETS_LO + - (nesvnic->qp_nic_index[nic_count]*0x200)); - u64temp += ((u64)nes_read_indexed(nesdev, - NES_IDX_ENDNODE0_NSTAT_RX_OCTETS_HI + - (nesvnic->qp_nic_index[nic_count]*0x200))) << 32; - - nesvnic->endnode_nstat_rx_octets += u64temp; - nesvnic->netstats.rx_bytes += u64temp; - - u64temp = (u64)nes_read_indexed(nesdev, - NES_IDX_ENDNODE0_NSTAT_RX_FRAMES_LO + - (nesvnic->qp_nic_index[nic_count]*0x200)); - u64temp += ((u64)nes_read_indexed(nesdev, - NES_IDX_ENDNODE0_NSTAT_RX_FRAMES_HI + - (nesvnic->qp_nic_index[nic_count]*0x200))) << 32; - - nesvnic->endnode_nstat_rx_frames += u64temp; - nesvnic->netstats.rx_packets += u64temp; - - u64temp = (u64)nes_read_indexed(nesdev, - NES_IDX_ENDNODE0_NSTAT_TX_OCTETS_LO + - (nesvnic->qp_nic_index[nic_count]*0x200)); - u64temp += ((u64)nes_read_indexed(nesdev, - NES_IDX_ENDNODE0_NSTAT_TX_OCTETS_HI + - (nesvnic->qp_nic_index[nic_count]*0x200))) << 32; - - nesvnic->endnode_nstat_tx_octets += u64temp; - nesvnic->netstats.tx_bytes += u64temp; - - u64temp = (u64)nes_read_indexed(nesdev, - NES_IDX_ENDNODE0_NSTAT_TX_FRAMES_LO + - (nesvnic->qp_nic_index[nic_count]*0x200)); - u64temp += ((u64)nes_read_indexed(nesdev, - NES_IDX_ENDNODE0_NSTAT_TX_FRAMES_HI + - (nesvnic->qp_nic_index[nic_count]*0x200))) << 32; - - nesvnic->endnode_nstat_tx_frames += u64temp; - nesvnic->netstats.tx_packets += u64temp; - - u32temp = nes_read_indexed(nesdev, - NES_IDX_IPV4_TCP_REXMITS + (nesvnic->qp_nic_index[nic_count]*0x200)); - nesvnic->endnode_ipv4_tcp_retransmits += u32temp; - } - - target_stat_values[++index] = nesvnic->nesdev->mac_pause_frames_received; - target_stat_values[++index] = nesdev->nesadapter->nic_rx_eth_route_err; - target_stat_values[++index] = nesvnic->tx_sw_dropped; - target_stat_values[++index] = nesvnic->sq_full; - target_stat_values[++index] = nesvnic->segmented_tso_requests; - target_stat_values[++index] = nesvnic->nesdev->mac_rx_symbol_err_frames; - target_stat_values[++index] = nesvnic->nesdev->mac_rx_jabber_frames; - target_stat_values[++index] = nesvnic->nesdev->mac_rx_oversized_frames; - target_stat_values[++index] = nesvnic->nesdev->mac_rx_short_frames; - target_stat_values[++index] = nesvnic->netstats.rx_length_errors; - target_stat_values[++index] = nesvnic->nesdev->mac_rx_crc_errors; - target_stat_values[++index] = nesvnic->nesdev->port_rx_discards; - target_stat_values[++index] = nesvnic->endnode_nstat_rx_discard; - target_stat_values[++index] = nesvnic->endnode_nstat_rx_octets; - target_stat_values[++index] = nesvnic->endnode_nstat_rx_frames; - target_stat_values[++index] = nesvnic->endnode_nstat_tx_octets; - target_stat_values[++index] = nesvnic->endnode_nstat_tx_frames; - target_stat_values[++index] = nesvnic->nesdev->mac_tx_errors; - target_stat_values[++index] = mh_detected; - target_stat_values[++index] = mh_pauses_sent; - target_stat_values[++index] = nesvnic->endnode_ipv4_tcp_retransmits; - target_stat_values[++index] = atomic_read(&cm_connects); - target_stat_values[++index] = atomic_read(&cm_accepts); - target_stat_values[++index] = atomic_read(&cm_disconnects); - target_stat_values[++index] = atomic_read(&cm_connecteds); - target_stat_values[++index] = atomic_read(&cm_connect_reqs); - target_stat_values[++index] = atomic_read(&cm_rejects); - target_stat_values[++index] = atomic_read(&mod_qp_timouts); - target_stat_values[++index] = atomic_read(&qps_created); - target_stat_values[++index] = atomic_read(&sw_qps_destroyed); - target_stat_values[++index] = atomic_read(&qps_destroyed); - target_stat_values[++index] = atomic_read(&cm_closes); - target_stat_values[++index] = cm_packets_sent; - target_stat_values[++index] = cm_packets_bounced; - target_stat_values[++index] = cm_packets_created; - target_stat_values[++index] = cm_packets_received; - target_stat_values[++index] = cm_packets_dropped; - target_stat_values[++index] = cm_packets_retrans; - target_stat_values[++index] = atomic_read(&cm_listens_created); - target_stat_values[++index] = atomic_read(&cm_listens_destroyed); - target_stat_values[++index] = cm_backlog_drops; - target_stat_values[++index] = atomic_read(&cm_loopbacks); - target_stat_values[++index] = atomic_read(&cm_nodes_created); - target_stat_values[++index] = atomic_read(&cm_nodes_destroyed); - target_stat_values[++index] = atomic_read(&cm_accel_dropped_pkts); - target_stat_values[++index] = atomic_read(&cm_resets_recvd); - target_stat_values[++index] = nesadapter->free_4kpbl; - target_stat_values[++index] = nesadapter->free_256pbl; - target_stat_values[++index] = int_mod_timer_init; - target_stat_values[++index] = atomic_read(&pau_qps_created); - target_stat_values[++index] = atomic_read(&pau_qps_destroyed); -} - -/** - * nes_netdev_get_drvinfo - */ -static void nes_netdev_get_drvinfo(struct net_device *netdev, - struct ethtool_drvinfo *drvinfo) -{ - struct nes_vnic *nesvnic = netdev_priv(netdev); - struct nes_adapter *nesadapter = nesvnic->nesdev->nesadapter; - - strlcpy(drvinfo->driver, DRV_NAME, sizeof(drvinfo->driver)); - strlcpy(drvinfo->bus_info, pci_name(nesvnic->nesdev->pcidev), - sizeof(drvinfo->bus_info)); - snprintf(drvinfo->fw_version, sizeof(drvinfo->fw_version), - "%u.%u", nesadapter->firmware_version >> 16, - nesadapter->firmware_version & 0x000000ff); - strlcpy(drvinfo->version, DRV_VERSION, sizeof(drvinfo->version)); -} - - -/** - * nes_netdev_set_coalesce - */ -static int nes_netdev_set_coalesce(struct net_device *netdev, - struct ethtool_coalesce *et_coalesce) -{ - struct nes_vnic *nesvnic = netdev_priv(netdev); - struct nes_device *nesdev = nesvnic->nesdev; - struct nes_adapter *nesadapter = nesdev->nesadapter; - struct nes_hw_tune_timer *shared_timer = &nesadapter->tune_timer; - unsigned long flags; - - spin_lock_irqsave(&nesadapter->periodic_timer_lock, flags); - if (et_coalesce->rx_max_coalesced_frames_low) { - shared_timer->threshold_low = et_coalesce->rx_max_coalesced_frames_low; - } - if (et_coalesce->rx_max_coalesced_frames_irq) { - shared_timer->threshold_target = et_coalesce->rx_max_coalesced_frames_irq; - } - if (et_coalesce->rx_max_coalesced_frames_high) { - shared_timer->threshold_high = et_coalesce->rx_max_coalesced_frames_high; - } - if (et_coalesce->rx_coalesce_usecs_low) { - shared_timer->timer_in_use_min = et_coalesce->rx_coalesce_usecs_low; - } - if (et_coalesce->rx_coalesce_usecs_high) { - shared_timer->timer_in_use_max = et_coalesce->rx_coalesce_usecs_high; - } - spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags); - - /* using this to drive total interrupt moderation */ - nesadapter->et_rx_coalesce_usecs_irq = et_coalesce->rx_coalesce_usecs_irq; - if (et_coalesce->use_adaptive_rx_coalesce) { - nesadapter->et_use_adaptive_rx_coalesce = 1; - nesadapter->timer_int_limit = NES_TIMER_INT_LIMIT_DYNAMIC; - nesadapter->et_rx_coalesce_usecs_irq = 0; - if (et_coalesce->pkt_rate_low) { - nesadapter->et_pkt_rate_low = et_coalesce->pkt_rate_low; - } - } else { - nesadapter->et_use_adaptive_rx_coalesce = 0; - nesadapter->timer_int_limit = NES_TIMER_INT_LIMIT; - if (nesadapter->et_rx_coalesce_usecs_irq) { - nes_write32(nesdev->regs+NES_PERIODIC_CONTROL, - 0x80000000 | ((u32)(nesadapter->et_rx_coalesce_usecs_irq*8))); - } - } - return 0; -} - - -/** - * nes_netdev_get_coalesce - */ -static int nes_netdev_get_coalesce(struct net_device *netdev, - struct ethtool_coalesce *et_coalesce) -{ - struct nes_vnic *nesvnic = netdev_priv(netdev); - struct nes_device *nesdev = nesvnic->nesdev; - struct nes_adapter *nesadapter = nesdev->nesadapter; - struct ethtool_coalesce temp_et_coalesce; - struct nes_hw_tune_timer *shared_timer = &nesadapter->tune_timer; - unsigned long flags; - - memset(&temp_et_coalesce, 0, sizeof(temp_et_coalesce)); - temp_et_coalesce.rx_coalesce_usecs_irq = nesadapter->et_rx_coalesce_usecs_irq; - temp_et_coalesce.use_adaptive_rx_coalesce = nesadapter->et_use_adaptive_rx_coalesce; - temp_et_coalesce.rate_sample_interval = nesadapter->et_rate_sample_interval; - temp_et_coalesce.pkt_rate_low = nesadapter->et_pkt_rate_low; - spin_lock_irqsave(&nesadapter->periodic_timer_lock, flags); - temp_et_coalesce.rx_max_coalesced_frames_low = shared_timer->threshold_low; - temp_et_coalesce.rx_max_coalesced_frames_irq = shared_timer->threshold_target; - temp_et_coalesce.rx_max_coalesced_frames_high = shared_timer->threshold_high; - temp_et_coalesce.rx_coalesce_usecs_low = shared_timer->timer_in_use_min; - temp_et_coalesce.rx_coalesce_usecs_high = shared_timer->timer_in_use_max; - if (nesadapter->et_use_adaptive_rx_coalesce) { - temp_et_coalesce.rx_coalesce_usecs_irq = shared_timer->timer_in_use; - } - spin_unlock_irqrestore(&nesadapter->periodic_timer_lock, flags); - memcpy(et_coalesce, &temp_et_coalesce, sizeof(*et_coalesce)); - return 0; -} - - -/** - * nes_netdev_get_pauseparam - */ -static void nes_netdev_get_pauseparam(struct net_device *netdev, - struct ethtool_pauseparam *et_pauseparam) -{ - struct nes_vnic *nesvnic = netdev_priv(netdev); - - et_pauseparam->autoneg = 0; - et_pauseparam->rx_pause = (nesvnic->nesdev->disable_rx_flow_control == 0) ? 1:0; - et_pauseparam->tx_pause = (nesvnic->nesdev->disable_tx_flow_control == 0) ? 1:0; -} - - -/** - * nes_netdev_set_pauseparam - */ -static int nes_netdev_set_pauseparam(struct net_device *netdev, - struct ethtool_pauseparam *et_pauseparam) -{ - struct nes_vnic *nesvnic = netdev_priv(netdev); - struct nes_device *nesdev = nesvnic->nesdev; - u32 u32temp; - - if (et_pauseparam->autoneg) { - /* TODO: should return unsupported */ - return 0; - } - if ((et_pauseparam->tx_pause == 1) && (nesdev->disable_tx_flow_control == 1)) { - u32temp = nes_read_indexed(nesdev, - NES_IDX_MAC_TX_CONFIG + (nesdev->mac_index*0x200)); - u32temp |= NES_IDX_MAC_TX_CONFIG_ENABLE_PAUSE; - nes_write_indexed(nesdev, - NES_IDX_MAC_TX_CONFIG + (nesdev->mac_index*0x200), u32temp); - nesdev->disable_tx_flow_control = 0; - } else if ((et_pauseparam->tx_pause == 0) && (nesdev->disable_tx_flow_control == 0)) { - u32temp = nes_read_indexed(nesdev, - NES_IDX_MAC_TX_CONFIG + (nesdev->mac_index*0x200)); - u32temp &= ~NES_IDX_MAC_TX_CONFIG_ENABLE_PAUSE; - nes_write_indexed(nesdev, - NES_IDX_MAC_TX_CONFIG + (nesdev->mac_index*0x200), u32temp); - nesdev->disable_tx_flow_control = 1; - } - if ((et_pauseparam->rx_pause == 1) && (nesdev->disable_rx_flow_control == 1)) { - u32temp = nes_read_indexed(nesdev, - NES_IDX_MPP_DEBUG + (nesdev->mac_index*0x40)); - u32temp &= ~NES_IDX_MPP_DEBUG_PORT_DISABLE_PAUSE; - nes_write_indexed(nesdev, - NES_IDX_MPP_DEBUG + (nesdev->mac_index*0x40), u32temp); - nesdev->disable_rx_flow_control = 0; - } else if ((et_pauseparam->rx_pause == 0) && (nesdev->disable_rx_flow_control == 0)) { - u32temp = nes_read_indexed(nesdev, - NES_IDX_MPP_DEBUG + (nesdev->mac_index*0x40)); - u32temp |= NES_IDX_MPP_DEBUG_PORT_DISABLE_PAUSE; - nes_write_indexed(nesdev, - NES_IDX_MPP_DEBUG + (nesdev->mac_index*0x40), u32temp); - nesdev->disable_rx_flow_control = 1; - } - - return 0; -} - - -/** - * nes_netdev_get_settings - */ -static int nes_netdev_get_link_ksettings(struct net_device *netdev, - struct ethtool_link_ksettings *cmd) -{ - struct nes_vnic *nesvnic = netdev_priv(netdev); - struct nes_device *nesdev = nesvnic->nesdev; - struct nes_adapter *nesadapter = nesdev->nesadapter; - u32 mac_index = nesdev->mac_index; - u8 phy_type = nesadapter->phy_type[mac_index]; - u8 phy_index = nesadapter->phy_index[mac_index]; - u16 phy_data; - u32 supported, advertising; - - cmd->base.duplex = DUPLEX_FULL; - cmd->base.port = PORT_MII; - - if (nesadapter->OneG_Mode) { - cmd->base.speed = SPEED_1000; - if (phy_type == NES_PHY_TYPE_PUMA_1G) { - supported = SUPPORTED_1000baseT_Full; - advertising = ADVERTISED_1000baseT_Full; - cmd->base.autoneg = AUTONEG_DISABLE; - cmd->base.phy_address = mac_index; - } else { - unsigned long flags; - - supported = SUPPORTED_1000baseT_Full - | SUPPORTED_Autoneg; - advertising = ADVERTISED_1000baseT_Full - | ADVERTISED_Autoneg; - spin_lock_irqsave(&nesadapter->phy_lock, flags); - nes_read_1G_phy_reg(nesdev, 0, phy_index, &phy_data); - spin_unlock_irqrestore(&nesadapter->phy_lock, flags); - if (phy_data & 0x1000) - cmd->base.autoneg = AUTONEG_ENABLE; - else - cmd->base.autoneg = AUTONEG_DISABLE; - cmd->base.phy_address = phy_index; - } - ethtool_convert_legacy_u32_to_link_mode( - cmd->link_modes.supported, supported); - ethtool_convert_legacy_u32_to_link_mode( - cmd->link_modes.advertising, advertising); - return 0; - } - if ((phy_type == NES_PHY_TYPE_ARGUS) || - (phy_type == NES_PHY_TYPE_SFP_D) || - (phy_type == NES_PHY_TYPE_KR)) { - cmd->base.port = PORT_FIBRE; - supported = SUPPORTED_FIBRE; - advertising = ADVERTISED_FIBRE; - cmd->base.phy_address = phy_index; - } else { - supported = SUPPORTED_10000baseT_Full; - advertising = ADVERTISED_10000baseT_Full; - cmd->base.phy_address = mac_index; - } - cmd->base.speed = SPEED_10000; - cmd->base.autoneg = AUTONEG_DISABLE; - ethtool_convert_legacy_u32_to_link_mode(cmd->link_modes.supported, - supported); - ethtool_convert_legacy_u32_to_link_mode(cmd->link_modes.advertising, - advertising); - - return 0; -} - - -/** - * nes_netdev_set_settings - */ -static int -nes_netdev_set_link_ksettings(struct net_device *netdev, - const struct ethtool_link_ksettings *cmd) -{ - struct nes_vnic *nesvnic = netdev_priv(netdev); - struct nes_device *nesdev = nesvnic->nesdev; - struct nes_adapter *nesadapter = nesdev->nesadapter; - - if ((nesadapter->OneG_Mode) && - (nesadapter->phy_type[nesdev->mac_index] != NES_PHY_TYPE_PUMA_1G)) { - unsigned long flags; - u16 phy_data; - u8 phy_index = nesadapter->phy_index[nesdev->mac_index]; - - spin_lock_irqsave(&nesadapter->phy_lock, flags); - nes_read_1G_phy_reg(nesdev, 0, phy_index, &phy_data); - if (cmd->base.autoneg) { - /* Turn on Full duplex, Autoneg, and restart autonegotiation */ - phy_data |= 0x1300; - } else { - /* Turn off autoneg */ - phy_data &= ~0x1000; - } - nes_write_1G_phy_reg(nesdev, 0, phy_index, phy_data); - spin_unlock_irqrestore(&nesadapter->phy_lock, flags); - } - - return 0; -} - - -static const struct ethtool_ops nes_ethtool_ops = { - .get_link = ethtool_op_get_link, - .get_strings = nes_netdev_get_strings, - .get_sset_count = nes_netdev_get_sset_count, - .get_ethtool_stats = nes_netdev_get_ethtool_stats, - .get_drvinfo = nes_netdev_get_drvinfo, - .get_coalesce = nes_netdev_get_coalesce, - .set_coalesce = nes_netdev_set_coalesce, - .get_pauseparam = nes_netdev_get_pauseparam, - .set_pauseparam = nes_netdev_set_pauseparam, - .get_link_ksettings = nes_netdev_get_link_ksettings, - .set_link_ksettings = nes_netdev_set_link_ksettings, -}; - -static void nes_vlan_mode(struct net_device *netdev, struct nes_device *nesdev, netdev_features_t features) -{ - struct nes_adapter *nesadapter = nesdev->nesadapter; - u32 u32temp; - unsigned long flags; - - spin_lock_irqsave(&nesadapter->phy_lock, flags); - - nes_debug(NES_DBG_NETDEV, "%s: %s\n", __func__, netdev->name); - - /* Enable/Disable VLAN Stripping */ - u32temp = nes_read_indexed(nesdev, NES_IDX_PCIX_DIAG); - if (features & NETIF_F_HW_VLAN_CTAG_RX) - u32temp &= 0xfdffffff; - else - u32temp |= 0x02000000; - - nes_write_indexed(nesdev, NES_IDX_PCIX_DIAG, u32temp); - spin_unlock_irqrestore(&nesadapter->phy_lock, flags); -} - -static netdev_features_t nes_fix_features(struct net_device *netdev, netdev_features_t features) -{ - /* - * Since there is no support for separate rx/tx vlan accel - * enable/disable make sure tx flag is always in same state as rx. - */ - if (features & NETIF_F_HW_VLAN_CTAG_RX) - features |= NETIF_F_HW_VLAN_CTAG_TX; - else - features &= ~NETIF_F_HW_VLAN_CTAG_TX; - - return features; -} - -static int nes_set_features(struct net_device *netdev, netdev_features_t features) -{ - struct nes_vnic *nesvnic = netdev_priv(netdev); - struct nes_device *nesdev = nesvnic->nesdev; - u32 changed = netdev->features ^ features; - - if (changed & NETIF_F_HW_VLAN_CTAG_RX) - nes_vlan_mode(netdev, nesdev, features); - - return 0; -} - -static const struct net_device_ops nes_netdev_ops = { - .ndo_open = nes_netdev_open, - .ndo_stop = nes_netdev_stop, - .ndo_start_xmit = nes_netdev_start_xmit, - .ndo_get_stats = nes_netdev_get_stats, - .ndo_tx_timeout = nes_netdev_tx_timeout, - .ndo_set_mac_address = nes_netdev_set_mac_address, - .ndo_set_rx_mode = nes_netdev_set_multicast_list, - .ndo_change_mtu = nes_netdev_change_mtu, - .ndo_validate_addr = eth_validate_addr, - .ndo_fix_features = nes_fix_features, - .ndo_set_features = nes_set_features, -}; - -/** - * nes_netdev_init - initialize network device - */ -struct net_device *nes_netdev_init(struct nes_device *nesdev, - void __iomem *mmio_addr) -{ - u64 u64temp; - struct nes_vnic *nesvnic; - struct net_device *netdev; - struct nic_qp_map *curr_qp_map; - u8 phy_type = nesdev->nesadapter->phy_type[nesdev->mac_index]; - - netdev = alloc_etherdev(sizeof(struct nes_vnic)); - if (!netdev) { - printk(KERN_ERR PFX "nesvnic etherdev alloc failed"); - return NULL; - } - nesvnic = netdev_priv(netdev); - - nes_debug(NES_DBG_INIT, "netdev = %p, %s\n", netdev, netdev->name); - - SET_NETDEV_DEV(netdev, &nesdev->pcidev->dev); - - netdev->watchdog_timeo = NES_TX_TIMEOUT; - netdev->irq = nesdev->pcidev->irq; - netdev->max_mtu = NES_MAX_MTU; - netdev->hard_header_len = ETH_HLEN; - netdev->addr_len = ETH_ALEN; - netdev->type = ARPHRD_ETHER; - netdev->netdev_ops = &nes_netdev_ops; - netdev->ethtool_ops = &nes_ethtool_ops; - netif_napi_add(netdev, &nesvnic->napi, nes_netdev_poll, 128); - nes_debug(NES_DBG_INIT, "Enabling VLAN Insert/Delete.\n"); - - /* Fill in the port structure */ - nesvnic->netdev = netdev; - nesvnic->nesdev = nesdev; - nesvnic->msg_enable = netif_msg_init(debug, default_msg); - nesvnic->netdev_index = nesdev->netdev_count; - nesvnic->perfect_filter_index = nesdev->nesadapter->netdev_count; - nesvnic->max_frame_size = netdev->mtu + netdev->hard_header_len + VLAN_HLEN; - - curr_qp_map = nic_qp_mapping_per_function[PCI_FUNC(nesdev->pcidev->devfn)]; - nesvnic->nic.qp_id = curr_qp_map[nesdev->netdev_count].qpid; - nesvnic->nic_index = curr_qp_map[nesdev->netdev_count].nic_index; - nesvnic->logical_port = curr_qp_map[nesdev->netdev_count].logical_port; - - /* Setup the burned in MAC address */ - u64temp = (u64)nesdev->nesadapter->mac_addr_low; - u64temp += ((u64)nesdev->nesadapter->mac_addr_high) << 32; - u64temp += nesvnic->nic_index; - netdev->dev_addr[0] = (u8)(u64temp>>40); - netdev->dev_addr[1] = (u8)(u64temp>>32); - netdev->dev_addr[2] = (u8)(u64temp>>24); - netdev->dev_addr[3] = (u8)(u64temp>>16); - netdev->dev_addr[4] = (u8)(u64temp>>8); - netdev->dev_addr[5] = (u8)u64temp; - - netdev->hw_features = NETIF_F_SG | NETIF_F_IP_CSUM | NETIF_F_RXCSUM | NETIF_F_HW_VLAN_CTAG_RX; - if ((nesvnic->logical_port < 2) || (nesdev->nesadapter->hw_rev != NE020_REV)) - netdev->hw_features |= NETIF_F_TSO; - - netdev->features = netdev->hw_features | NETIF_F_HIGHDMA | NETIF_F_HW_VLAN_CTAG_TX; - - nes_debug(NES_DBG_INIT, "nesvnic = %p, reported features = 0x%lX, QPid = %d," - " nic_index = %d, logical_port = %d, mac_index = %d.\n", - nesvnic, (unsigned long)netdev->features, nesvnic->nic.qp_id, - nesvnic->nic_index, nesvnic->logical_port, nesdev->mac_index); - - if (nesvnic->nesdev->nesadapter->port_count == 1 && - nesvnic->nesdev->nesadapter->adapter_fcn_count == 1) { - - nesvnic->qp_nic_index[0] = nesvnic->nic_index; - nesvnic->qp_nic_index[1] = nesvnic->nic_index + 1; - if (nes_drv_opt & NES_DRV_OPT_DUAL_LOGICAL_PORT) { - nesvnic->qp_nic_index[2] = 0xf; - nesvnic->qp_nic_index[3] = 0xf; - } else { - nesvnic->qp_nic_index[2] = nesvnic->nic_index + 2; - nesvnic->qp_nic_index[3] = nesvnic->nic_index + 3; - } - } else { - if (nesvnic->nesdev->nesadapter->port_count == 2 || - (nesvnic->nesdev->nesadapter->port_count == 1 && - nesvnic->nesdev->nesadapter->adapter_fcn_count == 2)) { - nesvnic->qp_nic_index[0] = nesvnic->nic_index; - nesvnic->qp_nic_index[1] = nesvnic->nic_index - + 2; - nesvnic->qp_nic_index[2] = 0xf; - nesvnic->qp_nic_index[3] = 0xf; - } else { - nesvnic->qp_nic_index[0] = nesvnic->nic_index; - nesvnic->qp_nic_index[1] = 0xf; - nesvnic->qp_nic_index[2] = 0xf; - nesvnic->qp_nic_index[3] = 0xf; - } - } - nesvnic->next_qp_nic_index = 0; - - if (nesdev->netdev_count == 0) { - nesvnic->rdma_enabled = 1; - } else { - nesvnic->rdma_enabled = 0; - } - nesvnic->nic_cq.cq_number = nesvnic->nic.qp_id; - timer_setup(&nesvnic->event_timer, NULL, 0); - spin_lock_init(&nesvnic->tx_lock); - spin_lock_init(&nesvnic->port_ibevent_lock); - nesdev->netdev[nesdev->netdev_count] = netdev; - - nes_debug(NES_DBG_INIT, "Adding nesvnic (%p) to the adapters nesvnic_list for MAC%d.\n", - nesvnic, nesdev->mac_index); - list_add_tail(&nesvnic->list, &nesdev->nesadapter->nesvnic_list[nesdev->mac_index]); - - if ((nesdev->netdev_count == 0) && - ((PCI_FUNC(nesdev->pcidev->devfn) == nesdev->mac_index) || - ((phy_type == NES_PHY_TYPE_PUMA_1G) && - (((PCI_FUNC(nesdev->pcidev->devfn) == 1) && (nesdev->mac_index == 2)) || - ((PCI_FUNC(nesdev->pcidev->devfn) == 2) && (nesdev->mac_index == 1)))))) { - u32 u32temp; - u32 link_mask = 0; - u32 link_val = 0; - u16 temp_phy_data; - u16 phy_data = 0; - unsigned long flags; - - u32temp = nes_read_indexed(nesdev, NES_IDX_PHY_PCS_CONTROL_STATUS0 + - (0x200 * (nesdev->mac_index & 1))); - if (phy_type != NES_PHY_TYPE_PUMA_1G) { - u32temp |= 0x00200000; - nes_write_indexed(nesdev, NES_IDX_PHY_PCS_CONTROL_STATUS0 + - (0x200 * (nesdev->mac_index & 1)), u32temp); - } - - /* Check and set linkup here. This is for back to back */ - /* configuration where second port won't get link interrupt */ - switch (phy_type) { - case NES_PHY_TYPE_PUMA_1G: - if (nesdev->mac_index < 2) { - link_mask = 0x01010000; - link_val = 0x01010000; - } else { - link_mask = 0x02020000; - link_val = 0x02020000; - } - break; - case NES_PHY_TYPE_SFP_D: - spin_lock_irqsave(&nesdev->nesadapter->phy_lock, flags); - nes_read_10G_phy_reg(nesdev, - nesdev->nesadapter->phy_index[nesdev->mac_index], - 1, 0x9003); - temp_phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL); - nes_read_10G_phy_reg(nesdev, - nesdev->nesadapter->phy_index[nesdev->mac_index], - 3, 0x0021); - nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL); - nes_read_10G_phy_reg(nesdev, - nesdev->nesadapter->phy_index[nesdev->mac_index], - 3, 0x0021); - phy_data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL); - spin_unlock_irqrestore(&nesdev->nesadapter->phy_lock, flags); - phy_data = (!temp_phy_data && (phy_data == 0x8000)) ? 0x4 : 0x0; - break; - default: - link_mask = 0x0f1f0000; - link_val = 0x0f0f0000; - break; - } - - u32temp = nes_read_indexed(nesdev, - NES_IDX_PHY_PCS_CONTROL_STATUS0 + - (0x200 * (nesdev->mac_index & 1))); - - if (phy_type == NES_PHY_TYPE_SFP_D) { - if (phy_data & 0x0004) - nesvnic->linkup = 1; - } else { - if ((u32temp & link_mask) == link_val) - nesvnic->linkup = 1; - } - - /* clear the MAC interrupt status, assumes direct logical to physical mapping */ - u32temp = nes_read_indexed(nesdev, NES_IDX_MAC_INT_STATUS + (0x200 * nesdev->mac_index)); - nes_debug(NES_DBG_INIT, "Phy interrupt status = 0x%X.\n", u32temp); - nes_write_indexed(nesdev, NES_IDX_MAC_INT_STATUS + (0x200 * nesdev->mac_index), u32temp); - - nes_init_phy(nesdev); - } - - nes_vlan_mode(netdev, nesdev, netdev->features); - - return netdev; -} - - -/** - * nes_netdev_destroy - destroy network device structure - */ -void nes_netdev_destroy(struct net_device *netdev) -{ - struct nes_vnic *nesvnic = netdev_priv(netdev); - - /* make sure 'stop' method is called by Linux stack */ - /* nes_netdev_stop(netdev); */ - - list_del(&nesvnic->list); - - if (nesvnic->of_device_registered) { - nes_destroy_ofa_device(nesvnic->nesibdev); - } - - free_netdev(netdev); -} - - -/** - * nes_nic_cm_xmit -- CM calls this to send out pkts - */ -int nes_nic_cm_xmit(struct sk_buff *skb, struct net_device *netdev) -{ - int ret; - - skb->dev = netdev; - ret = dev_queue_xmit(skb); - if (ret) { - nes_debug(NES_DBG_CM, "Bad return code from dev_queue_xmit %d\n", ret); - } - - return ret; -} diff --git a/drivers/infiniband/hw/nes/nes_utils.c b/drivers/infiniband/hw/nes/nes_utils.c deleted file mode 100644 index 21b4a8373acf..000000000000 --- a/drivers/infiniband/hw/nes/nes_utils.c +++ /dev/null @@ -1,916 +0,0 @@ -/* - * Copyright (c) 2006 - 2011 Intel Corporation. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - */ - -#include <linux/module.h> -#include <linux/moduleparam.h> -#include <linux/netdevice.h> -#include <linux/etherdevice.h> -#include <linux/ethtool.h> -#include <linux/mii.h> -#include <linux/if_vlan.h> -#include <linux/slab.h> -#include <linux/crc32.h> -#include <linux/in.h> -#include <linux/ip.h> -#include <linux/tcp.h> -#include <linux/init.h> -#include <linux/kernel.h> - -#include <asm/io.h> -#include <asm/irq.h> -#include <asm/byteorder.h> - -#include "nes.h" - -static u16 nes_read16_eeprom(void __iomem *addr, u16 offset); - -u32 mh_detected; -u32 mh_pauses_sent; - -static u32 nes_set_pau(struct nes_device *nesdev) -{ - u32 ret = 0; - u32 counter; - - nes_write_indexed(nesdev, NES_IDX_GPR2, NES_ENABLE_PAU); - nes_write_indexed(nesdev, NES_IDX_GPR_TRIGGER, 1); - - for (counter = 0; counter < NES_PAU_COUNTER; counter++) { - udelay(30); - if (!nes_read_indexed(nesdev, NES_IDX_GPR2)) { - printk(KERN_INFO PFX "PAU is supported.\n"); - break; - } - nes_write_indexed(nesdev, NES_IDX_GPR_TRIGGER, 1); - } - if (counter == NES_PAU_COUNTER) { - printk(KERN_INFO PFX "PAU is not supported.\n"); - return -EPERM; - } - return ret; -} - -/** - * nes_read_eeprom_values - - */ -int nes_read_eeprom_values(struct nes_device *nesdev, struct nes_adapter *nesadapter) -{ - u32 mac_addr_low; - u16 mac_addr_high; - u16 eeprom_data; - u16 eeprom_offset; - u16 next_section_address; - u16 sw_section_ver; - u8 major_ver = 0; - u8 minor_ver = 0; - - /* TODO: deal with EEPROM endian issues */ - if (nesadapter->firmware_eeprom_offset == 0) { - /* Read the EEPROM Parameters */ - eeprom_data = nes_read16_eeprom(nesdev->regs, 0); - nes_debug(NES_DBG_HW, "EEPROM Offset 0 = 0x%04X\n", eeprom_data); - eeprom_offset = 2 + (((eeprom_data & 0x007f) << 3) << - ((eeprom_data & 0x0080) >> 7)); - nes_debug(NES_DBG_HW, "Firmware Offset = 0x%04X\n", eeprom_offset); - nesadapter->firmware_eeprom_offset = eeprom_offset; - eeprom_data = nes_read16_eeprom(nesdev->regs, eeprom_offset + 4); - if (eeprom_data != 0x5746) { - nes_debug(NES_DBG_HW, "Not a valid Firmware Image = 0x%04X\n", eeprom_data); - return -1; - } - - eeprom_data = nes_read16_eeprom(nesdev->regs, eeprom_offset + 2); - nes_debug(NES_DBG_HW, "EEPROM Offset %u = 0x%04X\n", - eeprom_offset + 2, eeprom_data); - eeprom_offset += ((eeprom_data & 0x00ff) << 3) << ((eeprom_data & 0x0100) >> 8); - nes_debug(NES_DBG_HW, "Software Offset = 0x%04X\n", eeprom_offset); - nesadapter->software_eeprom_offset = eeprom_offset; - eeprom_data = nes_read16_eeprom(nesdev->regs, eeprom_offset + 4); - if (eeprom_data != 0x5753) { - printk("Not a valid Software Image = 0x%04X\n", eeprom_data); - return -1; - } - sw_section_ver = nes_read16_eeprom(nesdev->regs, nesadapter->software_eeprom_offset + 6); - nes_debug(NES_DBG_HW, "Software section version number = 0x%04X\n", - sw_section_ver); - - eeprom_data = nes_read16_eeprom(nesdev->regs, eeprom_offset + 2); - nes_debug(NES_DBG_HW, "EEPROM Offset %u (next section) = 0x%04X\n", - eeprom_offset + 2, eeprom_data); - next_section_address = eeprom_offset + (((eeprom_data & 0x00ff) << 3) << - ((eeprom_data & 0x0100) >> 8)); - eeprom_data = nes_read16_eeprom(nesdev->regs, next_section_address + 4); - if (eeprom_data != 0x414d) { - nes_debug(NES_DBG_HW, "EEPROM Changed offset should be 0x414d but was 0x%04X\n", - eeprom_data); - goto no_fw_rev; - } - eeprom_offset = next_section_address; - - eeprom_data = nes_read16_eeprom(nesdev->regs, eeprom_offset + 2); - nes_debug(NES_DBG_HW, "EEPROM Offset %u (next section) = 0x%04X\n", - eeprom_offset + 2, eeprom_data); - next_section_address = eeprom_offset + (((eeprom_data & 0x00ff) << 3) << - ((eeprom_data & 0x0100) >> 8)); - eeprom_data = nes_read16_eeprom(nesdev->regs, next_section_address + 4); - if (eeprom_data != 0x4f52) { - nes_debug(NES_DBG_HW, "EEPROM Changed offset should be 0x4f52 but was 0x%04X\n", - eeprom_data); - goto no_fw_rev; - } - eeprom_offset = next_section_address; - - eeprom_data = nes_read16_eeprom(nesdev->regs, eeprom_offset + 2); - nes_debug(NES_DBG_HW, "EEPROM Offset %u (next section) = 0x%04X\n", - eeprom_offset + 2, eeprom_data); - next_section_address = eeprom_offset + ((eeprom_data & 0x00ff) << 3); - eeprom_data = nes_read16_eeprom(nesdev->regs, next_section_address + 4); - if (eeprom_data != 0x5746) { - nes_debug(NES_DBG_HW, "EEPROM Changed offset should be 0x5746 but was 0x%04X\n", - eeprom_data); - goto no_fw_rev; - } - eeprom_offset = next_section_address; - - eeprom_data = nes_read16_eeprom(nesdev->regs, eeprom_offset + 2); - nes_debug(NES_DBG_HW, "EEPROM Offset %u (next section) = 0x%04X\n", - eeprom_offset + 2, eeprom_data); - next_section_address = eeprom_offset + ((eeprom_data & 0x00ff) << 3); - eeprom_data = nes_read16_eeprom(nesdev->regs, next_section_address + 4); - if (eeprom_data != 0x5753) { - nes_debug(NES_DBG_HW, "EEPROM Changed offset should be 0x5753 but was 0x%04X\n", - eeprom_data); - goto no_fw_rev; - } - eeprom_offset = next_section_address; - - eeprom_data = nes_read16_eeprom(nesdev->regs, eeprom_offset + 2); - nes_debug(NES_DBG_HW, "EEPROM Offset %u (next section) = 0x%04X\n", - eeprom_offset + 2, eeprom_data); - next_section_address = eeprom_offset + ((eeprom_data & 0x00ff) << 3); - eeprom_data = nes_read16_eeprom(nesdev->regs, next_section_address + 4); - if (eeprom_data != 0x414d) { - nes_debug(NES_DBG_HW, "EEPROM Changed offset should be 0x414d but was 0x%04X\n", - eeprom_data); - goto no_fw_rev; - } - eeprom_offset = next_section_address; - - eeprom_data = nes_read16_eeprom(nesdev->regs, eeprom_offset + 2); - nes_debug(NES_DBG_HW, "EEPROM Offset %u (next section) = 0x%04X\n", - eeprom_offset + 2, eeprom_data); - next_section_address = eeprom_offset + ((eeprom_data & 0x00ff) << 3); - eeprom_data = nes_read16_eeprom(nesdev->regs, next_section_address + 4); - if (eeprom_data != 0x464e) { - nes_debug(NES_DBG_HW, "EEPROM Changed offset should be 0x464e but was 0x%04X\n", - eeprom_data); - goto no_fw_rev; - } - eeprom_data = nes_read16_eeprom(nesdev->regs, next_section_address + 8); - printk(PFX "Firmware version %u.%u\n", (u8)(eeprom_data>>8), (u8)eeprom_data); - major_ver = (u8)(eeprom_data >> 8); - minor_ver = (u8)(eeprom_data); - - if (nes_drv_opt & NES_DRV_OPT_DISABLE_VIRT_WQ) { - nes_debug(NES_DBG_HW, "Virtual WQs have been disabled\n"); - } else if (((major_ver == 2) && (minor_ver > 21)) || ((major_ver > 2) && (major_ver != 255))) { - nesadapter->virtwq = 1; - } - if (((major_ver == 3) && (minor_ver >= 16)) || (major_ver > 3)) - nesadapter->send_term_ok = 1; - - if (nes_drv_opt & NES_DRV_OPT_ENABLE_PAU) { - if (!nes_set_pau(nesdev)) - nesadapter->allow_unaligned_fpdus = 1; - } - - nesadapter->firmware_version = (((u32)(u8)(eeprom_data>>8)) << 16) + - (u32)((u8)eeprom_data); - - eeprom_data = nes_read16_eeprom(nesdev->regs, next_section_address + 10); - printk(PFX "EEPROM version %u.%u\n", (u8)(eeprom_data>>8), (u8)eeprom_data); - nesadapter->eeprom_version = (((u32)(u8)(eeprom_data>>8)) << 16) + - (u32)((u8)eeprom_data); - -no_fw_rev: - /* eeprom is valid */ - eeprom_offset = nesadapter->software_eeprom_offset; - eeprom_offset += 8; - nesadapter->netdev_max = (u8)nes_read16_eeprom(nesdev->regs, eeprom_offset); - eeprom_offset += 2; - mac_addr_high = nes_read16_eeprom(nesdev->regs, eeprom_offset); - eeprom_offset += 2; - mac_addr_low = (u32)nes_read16_eeprom(nesdev->regs, eeprom_offset); - eeprom_offset += 2; - mac_addr_low <<= 16; - mac_addr_low += (u32)nes_read16_eeprom(nesdev->regs, eeprom_offset); - nes_debug(NES_DBG_HW, "Base MAC Address = 0x%04X%08X\n", - mac_addr_high, mac_addr_low); - nes_debug(NES_DBG_HW, "MAC Address count = %u\n", nesadapter->netdev_max); - - nesadapter->mac_addr_low = mac_addr_low; - nesadapter->mac_addr_high = mac_addr_high; - - /* Read the Phy Type array */ - eeprom_offset += 10; - eeprom_data = nes_read16_eeprom(nesdev->regs, eeprom_offset); - nesadapter->phy_type[0] = (u8)(eeprom_data >> 8); - nesadapter->phy_type[1] = (u8)eeprom_data; - - /* Read the port array */ - eeprom_offset += 2; - eeprom_data = nes_read16_eeprom(nesdev->regs, eeprom_offset); - nesadapter->phy_type[2] = (u8)(eeprom_data >> 8); - nesadapter->phy_type[3] = (u8)eeprom_data; - /* port_count is set by soft reset reg */ - nes_debug(NES_DBG_HW, "port_count = %u, port 0 -> %u, port 1 -> %u," - " port 2 -> %u, port 3 -> %u\n", - nesadapter->port_count, - nesadapter->phy_type[0], nesadapter->phy_type[1], - nesadapter->phy_type[2], nesadapter->phy_type[3]); - - /* Read PD config array */ - eeprom_offset += 10; - eeprom_data = nes_read16_eeprom(nesdev->regs, eeprom_offset); - nesadapter->pd_config_size[0] = eeprom_data; - eeprom_offset += 2; - eeprom_data = nes_read16_eeprom(nesdev->regs, eeprom_offset); - nesadapter->pd_config_base[0] = eeprom_data; - nes_debug(NES_DBG_HW, "PD0 config, size=0x%04x, base=0x%04x\n", - nesadapter->pd_config_size[0], nesadapter->pd_config_base[0]); - - eeprom_offset += 2; - eeprom_data = nes_read16_eeprom(nesdev->regs, eeprom_offset); - nesadapter->pd_config_size[1] = eeprom_data; - eeprom_offset += 2; - eeprom_data = nes_read16_eeprom(nesdev->regs, eeprom_offset); - nesadapter->pd_config_base[1] = eeprom_data; - nes_debug(NES_DBG_HW, "PD1 config, size=0x%04x, base=0x%04x\n", - nesadapter->pd_config_size[1], nesadapter->pd_config_base[1]); - - eeprom_offset += 2; - eeprom_data = nes_read16_eeprom(nesdev->regs, eeprom_offset); - nesadapter->pd_config_size[2] = eeprom_data; - eeprom_offset += 2; - eeprom_data = nes_read16_eeprom(nesdev->regs, eeprom_offset); - nesadapter->pd_config_base[2] = eeprom_data; - nes_debug(NES_DBG_HW, "PD2 config, size=0x%04x, base=0x%04x\n", - nesadapter->pd_config_size[2], nesadapter->pd_config_base[2]); - - eeprom_offset += 2; - eeprom_data = nes_read16_eeprom(nesdev->regs, eeprom_offset); - nesadapter->pd_config_size[3] = eeprom_data; - eeprom_offset += 2; - eeprom_data = nes_read16_eeprom(nesdev->regs, eeprom_offset); - nesadapter->pd_config_base[3] = eeprom_data; - nes_debug(NES_DBG_HW, "PD3 config, size=0x%04x, base=0x%04x\n", - nesadapter->pd_config_size[3], nesadapter->pd_config_base[3]); - - /* Read Rx Pool Size */ - eeprom_offset += 22; /* 46 */ - eeprom_data = nes_read16_eeprom(nesdev->regs, eeprom_offset); - eeprom_offset += 2; - nesadapter->rx_pool_size = (((u32)eeprom_data) << 16) + - nes_read16_eeprom(nesdev->regs, eeprom_offset); - nes_debug(NES_DBG_HW, "rx_pool_size = 0x%08X\n", nesadapter->rx_pool_size); - - eeprom_offset += 2; - eeprom_data = nes_read16_eeprom(nesdev->regs, eeprom_offset); - eeprom_offset += 2; - nesadapter->tx_pool_size = (((u32)eeprom_data) << 16) + - nes_read16_eeprom(nesdev->regs, eeprom_offset); - nes_debug(NES_DBG_HW, "tx_pool_size = 0x%08X\n", nesadapter->tx_pool_size); - - eeprom_offset += 2; - eeprom_data = nes_read16_eeprom(nesdev->regs, eeprom_offset); - eeprom_offset += 2; - nesadapter->rx_threshold = (((u32)eeprom_data) << 16) + - nes_read16_eeprom(nesdev->regs, eeprom_offset); - nes_debug(NES_DBG_HW, "rx_threshold = 0x%08X\n", nesadapter->rx_threshold); - - eeprom_offset += 2; - eeprom_data = nes_read16_eeprom(nesdev->regs, eeprom_offset); - eeprom_offset += 2; - nesadapter->tcp_timer_core_clk_divisor = (((u32)eeprom_data) << 16) + - nes_read16_eeprom(nesdev->regs, eeprom_offset); - nes_debug(NES_DBG_HW, "tcp_timer_core_clk_divisor = 0x%08X\n", - nesadapter->tcp_timer_core_clk_divisor); - - eeprom_offset += 2; - eeprom_data = nes_read16_eeprom(nesdev->regs, eeprom_offset); - eeprom_offset += 2; - nesadapter->iwarp_config = (((u32)eeprom_data) << 16) + - nes_read16_eeprom(nesdev->regs, eeprom_offset); - nes_debug(NES_DBG_HW, "iwarp_config = 0x%08X\n", nesadapter->iwarp_config); - - eeprom_offset += 2; - eeprom_data = nes_read16_eeprom(nesdev->regs, eeprom_offset); - eeprom_offset += 2; - nesadapter->cm_config = (((u32)eeprom_data) << 16) + - nes_read16_eeprom(nesdev->regs, eeprom_offset); - nes_debug(NES_DBG_HW, "cm_config = 0x%08X\n", nesadapter->cm_config); - - eeprom_offset += 2; - eeprom_data = nes_read16_eeprom(nesdev->regs, eeprom_offset); - eeprom_offset += 2; - nesadapter->sws_timer_config = (((u32)eeprom_data) << 16) + - nes_read16_eeprom(nesdev->regs, eeprom_offset); - nes_debug(NES_DBG_HW, "sws_timer_config = 0x%08X\n", nesadapter->sws_timer_config); - - eeprom_offset += 2; - eeprom_data = nes_read16_eeprom(nesdev->regs, eeprom_offset); - eeprom_offset += 2; - nesadapter->tcp_config1 = (((u32)eeprom_data) << 16) + - nes_read16_eeprom(nesdev->regs, eeprom_offset); - nes_debug(NES_DBG_HW, "tcp_config1 = 0x%08X\n", nesadapter->tcp_config1); - - eeprom_offset += 2; - eeprom_data = nes_read16_eeprom(nesdev->regs, eeprom_offset); - eeprom_offset += 2; - nesadapter->wqm_wat = (((u32)eeprom_data) << 16) + - nes_read16_eeprom(nesdev->regs, eeprom_offset); - nes_debug(NES_DBG_HW, "wqm_wat = 0x%08X\n", nesadapter->wqm_wat); - - eeprom_offset += 2; - eeprom_data = nes_read16_eeprom(nesdev->regs, eeprom_offset); - eeprom_offset += 2; - nesadapter->core_clock = (((u32)eeprom_data) << 16) + - nes_read16_eeprom(nesdev->regs, eeprom_offset); - nes_debug(NES_DBG_HW, "core_clock = 0x%08X\n", nesadapter->core_clock); - - if ((sw_section_ver) && (nesadapter->hw_rev != NE020_REV)) { - eeprom_offset += 2; - eeprom_data = nes_read16_eeprom(nesdev->regs, eeprom_offset); - nesadapter->phy_index[0] = (eeprom_data & 0xff00)>>8; - nesadapter->phy_index[1] = eeprom_data & 0x00ff; - eeprom_offset += 2; - eeprom_data = nes_read16_eeprom(nesdev->regs, eeprom_offset); - nesadapter->phy_index[2] = (eeprom_data & 0xff00)>>8; - nesadapter->phy_index[3] = eeprom_data & 0x00ff; - } else { - nesadapter->phy_index[0] = 4; - nesadapter->phy_index[1] = 5; - nesadapter->phy_index[2] = 6; - nesadapter->phy_index[3] = 7; - } - nes_debug(NES_DBG_HW, "Phy address map = 0 > %u, 1 > %u, 2 > %u, 3 > %u\n", - nesadapter->phy_index[0],nesadapter->phy_index[1], - nesadapter->phy_index[2],nesadapter->phy_index[3]); - } - - return 0; -} - - -/** - * nes_read16_eeprom - */ -static u16 nes_read16_eeprom(void __iomem *addr, u16 offset) -{ - writel(NES_EEPROM_READ_REQUEST + (offset >> 1), - (void __iomem *)addr + NES_EEPROM_COMMAND); - - do { - } while (readl((void __iomem *)addr + NES_EEPROM_COMMAND) & - NES_EEPROM_READ_REQUEST); - - return readw((void __iomem *)addr + NES_EEPROM_DATA); -} - - -/** - * nes_write_1G_phy_reg - */ -void nes_write_1G_phy_reg(struct nes_device *nesdev, u8 phy_reg, u8 phy_addr, u16 data) -{ - u32 u32temp; - u32 counter; - - nes_write_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL, - 0x50020000 | data | ((u32)phy_reg << 18) | ((u32)phy_addr << 23)); - for (counter = 0; counter < 100 ; counter++) { - udelay(30); - u32temp = nes_read_indexed(nesdev, NES_IDX_MAC_INT_STATUS); - if (u32temp & 1) { - /* nes_debug(NES_DBG_PHY, "Phy interrupt status = 0x%X.\n", u32temp); */ - nes_write_indexed(nesdev, NES_IDX_MAC_INT_STATUS, 1); - break; - } - } - if (!(u32temp & 1)) - nes_debug(NES_DBG_PHY, "Phy is not responding. interrupt status = 0x%X.\n", - u32temp); -} - - -/** - * nes_read_1G_phy_reg - * This routine only issues the read, the data must be read - * separately. - */ -void nes_read_1G_phy_reg(struct nes_device *nesdev, u8 phy_reg, u8 phy_addr, u16 *data) -{ - u32 u32temp; - u32 counter; - - /* nes_debug(NES_DBG_PHY, "phy addr = %d, mac_index = %d\n", - phy_addr, nesdev->mac_index); */ - - nes_write_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL, - 0x60020000 | ((u32)phy_reg << 18) | ((u32)phy_addr << 23)); - for (counter = 0; counter < 100 ; counter++) { - udelay(30); - u32temp = nes_read_indexed(nesdev, NES_IDX_MAC_INT_STATUS); - if (u32temp & 1) { - /* nes_debug(NES_DBG_PHY, "Phy interrupt status = 0x%X.\n", u32temp); */ - nes_write_indexed(nesdev, NES_IDX_MAC_INT_STATUS, 1); - break; - } - } - if (!(u32temp & 1)) { - nes_debug(NES_DBG_PHY, "Phy is not responding. interrupt status = 0x%X.\n", - u32temp); - *data = 0xffff; - } else { - *data = (u16)nes_read_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL); - } -} - - -/** - * nes_write_10G_phy_reg - */ -void nes_write_10G_phy_reg(struct nes_device *nesdev, u16 phy_addr, u8 dev_addr, u16 phy_reg, - u16 data) -{ - u32 port_addr; - u32 u32temp; - u32 counter; - - port_addr = phy_addr; - - /* set address */ - nes_write_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL, - 0x00020000 | (u32)phy_reg | (((u32)dev_addr) << 18) | (((u32)port_addr) << 23)); - for (counter = 0; counter < 100 ; counter++) { - udelay(30); - u32temp = nes_read_indexed(nesdev, NES_IDX_MAC_INT_STATUS); - if (u32temp & 1) { - nes_write_indexed(nesdev, NES_IDX_MAC_INT_STATUS, 1); - break; - } - } - if (!(u32temp & 1)) - nes_debug(NES_DBG_PHY, "Phy is not responding. interrupt status = 0x%X.\n", - u32temp); - - /* set data */ - nes_write_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL, - 0x10020000 | (u32)data | (((u32)dev_addr) << 18) | (((u32)port_addr) << 23)); - for (counter = 0; counter < 100 ; counter++) { - udelay(30); - u32temp = nes_read_indexed(nesdev, NES_IDX_MAC_INT_STATUS); - if (u32temp & 1) { - nes_write_indexed(nesdev, NES_IDX_MAC_INT_STATUS, 1); - break; - } - } - if (!(u32temp & 1)) - nes_debug(NES_DBG_PHY, "Phy is not responding. interrupt status = 0x%X.\n", - u32temp); -} - - -/** - * nes_read_10G_phy_reg - * This routine only issues the read, the data must be read - * separately. - */ -void nes_read_10G_phy_reg(struct nes_device *nesdev, u8 phy_addr, u8 dev_addr, u16 phy_reg) -{ - u32 port_addr; - u32 u32temp; - u32 counter; - - port_addr = phy_addr; - - /* set address */ - nes_write_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL, - 0x00020000 | (u32)phy_reg | (((u32)dev_addr) << 18) | (((u32)port_addr) << 23)); - for (counter = 0; counter < 100 ; counter++) { - udelay(30); - u32temp = nes_read_indexed(nesdev, NES_IDX_MAC_INT_STATUS); - if (u32temp & 1) { - nes_write_indexed(nesdev, NES_IDX_MAC_INT_STATUS, 1); - break; - } - } - if (!(u32temp & 1)) - nes_debug(NES_DBG_PHY, "Phy is not responding. interrupt status = 0x%X.\n", - u32temp); - - /* issue read */ - nes_write_indexed(nesdev, NES_IDX_MAC_MDIO_CONTROL, - 0x30020000 | (((u32)dev_addr) << 18) | (((u32)port_addr) << 23)); - for (counter = 0; counter < 100 ; counter++) { - udelay(30); - u32temp = nes_read_indexed(nesdev, NES_IDX_MAC_INT_STATUS); - if (u32temp & 1) { - nes_write_indexed(nesdev, NES_IDX_MAC_INT_STATUS, 1); - break; - } - } - if (!(u32temp & 1)) - nes_debug(NES_DBG_PHY, "Phy is not responding. interrupt status = 0x%X.\n", - u32temp); -} - - -/** - * nes_get_cqp_request - */ -struct nes_cqp_request *nes_get_cqp_request(struct nes_device *nesdev) -{ - unsigned long flags; - struct nes_cqp_request *cqp_request = NULL; - - if (!list_empty(&nesdev->cqp_avail_reqs)) { - spin_lock_irqsave(&nesdev->cqp.lock, flags); - if (!list_empty(&nesdev->cqp_avail_reqs)) { - cqp_request = list_entry(nesdev->cqp_avail_reqs.next, - struct nes_cqp_request, list); - list_del_init(&cqp_request->list); - } - spin_unlock_irqrestore(&nesdev->cqp.lock, flags); - } - if (cqp_request == NULL) { - cqp_request = kzalloc(sizeof(struct nes_cqp_request), GFP_ATOMIC); - if (cqp_request) { - cqp_request->dynamic = 1; - INIT_LIST_HEAD(&cqp_request->list); - } - } - - if (cqp_request) { - init_waitqueue_head(&cqp_request->waitq); - cqp_request->waiting = 0; - cqp_request->request_done = 0; - cqp_request->callback = 0; - init_waitqueue_head(&cqp_request->waitq); - nes_debug(NES_DBG_CQP, "Got cqp request %p from the available list \n", - cqp_request); - } else - printk(KERN_ERR PFX "%s: Could not allocated a CQP request.\n", - __func__); - - return cqp_request; -} - -void nes_free_cqp_request(struct nes_device *nesdev, - struct nes_cqp_request *cqp_request) -{ - unsigned long flags; - - nes_debug(NES_DBG_CQP, "CQP request %p (opcode 0x%02X) freed.\n", - cqp_request, - le32_to_cpu(cqp_request->cqp_wqe.wqe_words[NES_CQP_WQE_OPCODE_IDX]) & 0x3f); - - if (cqp_request->dynamic) { - kfree(cqp_request); - } else { - spin_lock_irqsave(&nesdev->cqp.lock, flags); - list_add_tail(&cqp_request->list, &nesdev->cqp_avail_reqs); - spin_unlock_irqrestore(&nesdev->cqp.lock, flags); - } -} - -void nes_put_cqp_request(struct nes_device *nesdev, - struct nes_cqp_request *cqp_request) -{ - if (atomic_dec_and_test(&cqp_request->refcount)) - nes_free_cqp_request(nesdev, cqp_request); -} - - -/** - * nes_post_cqp_request - */ -void nes_post_cqp_request(struct nes_device *nesdev, - struct nes_cqp_request *cqp_request) -{ - struct nes_hw_cqp_wqe *cqp_wqe; - unsigned long flags; - u32 cqp_head; - u64 u64temp; - u32 opcode; - int ctx_index = NES_CQP_WQE_COMP_CTX_LOW_IDX; - - spin_lock_irqsave(&nesdev->cqp.lock, flags); - - if (((((nesdev->cqp.sq_tail+(nesdev->cqp.sq_size*2))-nesdev->cqp.sq_head) & - (nesdev->cqp.sq_size - 1)) != 1) - && (list_empty(&nesdev->cqp_pending_reqs))) { - cqp_head = nesdev->cqp.sq_head++; - nesdev->cqp.sq_head &= nesdev->cqp.sq_size-1; - cqp_wqe = &nesdev->cqp.sq_vbase[cqp_head]; - memcpy(cqp_wqe, &cqp_request->cqp_wqe, sizeof(*cqp_wqe)); - opcode = le32_to_cpu(cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX]); - if ((opcode & NES_CQP_OPCODE_MASK) == NES_CQP_DOWNLOAD_SEGMENT) - ctx_index = NES_CQP_WQE_DL_COMP_CTX_LOW_IDX; - barrier(); - u64temp = (unsigned long)cqp_request; - set_wqe_64bit_value(cqp_wqe->wqe_words, ctx_index, u64temp); - nes_debug(NES_DBG_CQP, "CQP request (opcode 0x%02X), line 1 = 0x%08X put on CQPs SQ," - " request = %p, cqp_head = %u, cqp_tail = %u, cqp_size = %u," - " waiting = %d, refcount = %d.\n", - opcode & NES_CQP_OPCODE_MASK, - le32_to_cpu(cqp_wqe->wqe_words[NES_CQP_WQE_ID_IDX]), cqp_request, - nesdev->cqp.sq_head, nesdev->cqp.sq_tail, nesdev->cqp.sq_size, - cqp_request->waiting, atomic_read(&cqp_request->refcount)); - - barrier(); - - /* Ring doorbell (1 WQEs) */ - nes_write32(nesdev->regs+NES_WQE_ALLOC, 0x01800000 | nesdev->cqp.qp_id); - - barrier(); - } else { - nes_debug(NES_DBG_CQP, "CQP request %p (opcode 0x%02X), line 1 = 0x%08X" - " put on the pending queue.\n", - cqp_request, - le32_to_cpu(cqp_request->cqp_wqe.wqe_words[NES_CQP_WQE_OPCODE_IDX])&0x3f, - le32_to_cpu(cqp_request->cqp_wqe.wqe_words[NES_CQP_WQE_ID_IDX])); - list_add_tail(&cqp_request->list, &nesdev->cqp_pending_reqs); - } - - spin_unlock_irqrestore(&nesdev->cqp.lock, flags); - - return; -} - -/** - * nes_arp_table - */ -int nes_arp_table(struct nes_device *nesdev, u32 ip_addr, u8 *mac_addr, u32 action) -{ - struct nes_adapter *nesadapter = nesdev->nesadapter; - int arp_index; - int err = 0; - __be32 tmp_addr; - - for (arp_index = 0; (u32) arp_index < nesadapter->arp_table_size; arp_index++) { - if (nesadapter->arp_table[arp_index].ip_addr == ip_addr) - break; - } - - if (action == NES_ARP_ADD) { - if (arp_index != nesadapter->arp_table_size) { - return -1; - } - - arp_index = 0; - err = nes_alloc_resource(nesadapter, nesadapter->allocated_arps, - nesadapter->arp_table_size, (u32 *)&arp_index, &nesadapter->next_arp_index, NES_RESOURCE_ARP); - if (err) { - nes_debug(NES_DBG_NETDEV, "nes_alloc_resource returned error = %u\n", err); - return err; - } - nes_debug(NES_DBG_NETDEV, "ADD, arp_index=%d\n", arp_index); - - nesadapter->arp_table[arp_index].ip_addr = ip_addr; - memcpy(nesadapter->arp_table[arp_index].mac_addr, mac_addr, ETH_ALEN); - return arp_index; - } - - /* DELETE or RESOLVE */ - if (arp_index == nesadapter->arp_table_size) { - tmp_addr = cpu_to_be32(ip_addr); - nes_debug(NES_DBG_NETDEV, "MAC for %pI4 not in ARP table - cannot %s\n", - &tmp_addr, action == NES_ARP_RESOLVE ? "resolve" : "delete"); - return -1; - } - - if (action == NES_ARP_RESOLVE) { - nes_debug(NES_DBG_NETDEV, "RESOLVE, arp_index=%d\n", arp_index); - return arp_index; - } - - if (action == NES_ARP_DELETE) { - nes_debug(NES_DBG_NETDEV, "DELETE, arp_index=%d\n", arp_index); - nesadapter->arp_table[arp_index].ip_addr = 0; - eth_zero_addr(nesadapter->arp_table[arp_index].mac_addr); - nes_free_resource(nesadapter, nesadapter->allocated_arps, arp_index); - return arp_index; - } - - return -1; -} - - -/** - * nes_mh_fix - */ -void nes_mh_fix(struct timer_list *t) -{ - struct nes_adapter *nesadapter = from_timer(nesadapter, t, mh_timer); - struct nes_device *nesdev = nesadapter->nesdev; - unsigned long flags; - struct nes_vnic *nesvnic; - u32 used_chunks_tx; - u32 temp_used_chunks_tx; - u32 temp_last_used_chunks_tx; - u32 used_chunks_mask; - u32 mac_tx_frames_low; - u32 mac_tx_frames_high; - u32 mac_tx_pauses; - u32 reset_value; - u32 tx_control; - u32 tx_config; - u32 tx_pause_quanta; - u32 rx_control; - u32 rx_config; - u32 mac_exact_match; - u32 mpp_debug; - u32 i=0; - u32 chunks_tx_progress = 0; - - spin_lock_irqsave(&nesadapter->phy_lock, flags); - if ((nesadapter->mac_sw_state[0] != NES_MAC_SW_IDLE) || (nesadapter->mac_link_down[0])) { - spin_unlock_irqrestore(&nesadapter->phy_lock, flags); - goto no_mh_work; - } - nesadapter->mac_sw_state[0] = NES_MAC_SW_MH; - spin_unlock_irqrestore(&nesadapter->phy_lock, flags); - do { - mac_tx_frames_low = nes_read_indexed(nesdev, NES_IDX_MAC_TX_FRAMES_LOW); - mac_tx_frames_high = nes_read_indexed(nesdev, NES_IDX_MAC_TX_FRAMES_HIGH); - mac_tx_pauses = nes_read_indexed(nesdev, NES_IDX_MAC_TX_PAUSE_FRAMES); - used_chunks_tx = nes_read_indexed(nesdev, NES_IDX_USED_CHUNKS_TX); - nesdev->mac_pause_frames_sent += mac_tx_pauses; - used_chunks_mask = 0; - temp_used_chunks_tx = used_chunks_tx; - temp_last_used_chunks_tx = nesdev->last_used_chunks_tx; - - if (nesdev->netdev[0]) { - nesvnic = netdev_priv(nesdev->netdev[0]); - } else { - break; - } - - for (i=0; i<4; i++) { - used_chunks_mask <<= 8; - if (nesvnic->qp_nic_index[i] != 0xff) { - used_chunks_mask |= 0xff; - if ((temp_used_chunks_tx&0xff)<(temp_last_used_chunks_tx&0xff)) { - chunks_tx_progress = 1; - } - } - temp_used_chunks_tx >>= 8; - temp_last_used_chunks_tx >>= 8; - } - if ((mac_tx_frames_low) || (mac_tx_frames_high) || - (!(used_chunks_tx&used_chunks_mask)) || - (!(nesdev->last_used_chunks_tx&used_chunks_mask)) || - (chunks_tx_progress) ) { - nesdev->last_used_chunks_tx = used_chunks_tx; - break; - } - nesdev->last_used_chunks_tx = used_chunks_tx; - barrier(); - - nes_write_indexed(nesdev, NES_IDX_MAC_TX_CONTROL, 0x00000005); - mh_pauses_sent++; - mac_tx_pauses = nes_read_indexed(nesdev, NES_IDX_MAC_TX_PAUSE_FRAMES); - if (mac_tx_pauses) { - nesdev->mac_pause_frames_sent += mac_tx_pauses; - break; - } - - tx_control = nes_read_indexed(nesdev, NES_IDX_MAC_TX_CONTROL); - tx_config = nes_read_indexed(nesdev, NES_IDX_MAC_TX_CONFIG); - tx_pause_quanta = nes_read_indexed(nesdev, NES_IDX_MAC_TX_PAUSE_QUANTA); - rx_control = nes_read_indexed(nesdev, NES_IDX_MAC_RX_CONTROL); - rx_config = nes_read_indexed(nesdev, NES_IDX_MAC_RX_CONFIG); - mac_exact_match = nes_read_indexed(nesdev, NES_IDX_MAC_EXACT_MATCH_BOTTOM); - mpp_debug = nes_read_indexed(nesdev, NES_IDX_MPP_DEBUG); - - /* one last ditch effort to avoid a false positive */ - mac_tx_pauses = nes_read_indexed(nesdev, NES_IDX_MAC_TX_PAUSE_FRAMES); - if (mac_tx_pauses) { - nesdev->last_mac_tx_pauses = nesdev->mac_pause_frames_sent; - nes_debug(NES_DBG_HW, "failsafe caught slow outbound pause\n"); - break; - } - mh_detected++; - - nes_write_indexed(nesdev, NES_IDX_MAC_TX_CONTROL, 0x00000000); - nes_write_indexed(nesdev, NES_IDX_MAC_TX_CONFIG, 0x00000000); - reset_value = nes_read32(nesdev->regs+NES_SOFTWARE_RESET); - - nes_write32(nesdev->regs+NES_SOFTWARE_RESET, reset_value | 0x0000001d); - - while (((nes_read32(nesdev->regs+NES_SOFTWARE_RESET) - & 0x00000040) != 0x00000040) && (i++ < 5000)) { - /* mdelay(1); */ - } - - nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_CONTROL0, 0x00000008); - nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_COMMON_STATUS0); - - nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_EMP0, 0x000bdef7); - nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_TX_DRIVE0, 0x9ce73000); - nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_MODE0, 0x0ff00000); - nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_SIGDET0, 0x00000000); - nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_BYPASS0, 0x00000000); - nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_LOOPBACK_CONTROL0, 0x00000000); - if (nesadapter->OneG_Mode) { - nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_EQ_CONTROL0, 0xf0182222); - } else { - nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_RX_EQ_CONTROL0, 0xf0042222); - } - nes_read_indexed(nesdev, NES_IDX_ETH_SERDES_RX_EQ_STATUS0); - nes_write_indexed(nesdev, NES_IDX_ETH_SERDES_CDR_CONTROL0, 0x000000ff); - - nes_write_indexed(nesdev, NES_IDX_MAC_TX_CONTROL, tx_control); - nes_write_indexed(nesdev, NES_IDX_MAC_TX_CONFIG, tx_config); - nes_write_indexed(nesdev, NES_IDX_MAC_TX_PAUSE_QUANTA, tx_pause_quanta); - nes_write_indexed(nesdev, NES_IDX_MAC_RX_CONTROL, rx_control); - nes_write_indexed(nesdev, NES_IDX_MAC_RX_CONFIG, rx_config); - nes_write_indexed(nesdev, NES_IDX_MAC_EXACT_MATCH_BOTTOM, mac_exact_match); - nes_write_indexed(nesdev, NES_IDX_MPP_DEBUG, mpp_debug); - - } while (0); - - nesadapter->mac_sw_state[0] = NES_MAC_SW_IDLE; -no_mh_work: - nesdev->nesadapter->mh_timer.expires = jiffies + (HZ/5); - add_timer(&nesdev->nesadapter->mh_timer); -} - -/** - * nes_clc - */ -void nes_clc(struct timer_list *t) -{ - struct nes_adapter *nesadapter = from_timer(nesadapter, t, lc_timer); - unsigned long flags; - - spin_lock_irqsave(&nesadapter->phy_lock, flags); - nesadapter->link_interrupt_count[0] = 0; - nesadapter->link_interrupt_count[1] = 0; - nesadapter->link_interrupt_count[2] = 0; - nesadapter->link_interrupt_count[3] = 0; - spin_unlock_irqrestore(&nesadapter->phy_lock, flags); - - nesadapter->lc_timer.expires = jiffies + 3600 * HZ; /* 1 hour */ - add_timer(&nesadapter->lc_timer); -} - - -/** - * nes_dump_mem - */ -void nes_dump_mem(unsigned int dump_debug_level, void *addr, int length) -{ - if (!(nes_debug_level & dump_debug_level)) { - return; - } - - if (length > 0x100) { - nes_debug(dump_debug_level, "Length truncated from %x to %x\n", length, 0x100); - length = 0x100; - } - nes_debug(dump_debug_level, "Address=0x%p, length=0x%x (%d)\n", addr, length, length); - - print_hex_dump(KERN_ERR, PFX, DUMP_PREFIX_NONE, 16, 1, addr, length, true); -} diff --git a/drivers/infiniband/hw/nes/nes_verbs.c b/drivers/infiniband/hw/nes/nes_verbs.c deleted file mode 100644 index 49024326a518..000000000000 --- a/drivers/infiniband/hw/nes/nes_verbs.c +++ /dev/null @@ -1,3759 +0,0 @@ -/* - * Copyright (c) 2006 - 2011 Intel Corporation. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - */ - -#include <linux/module.h> -#include <linux/moduleparam.h> -#include <linux/random.h> -#include <linux/highmem.h> -#include <linux/slab.h> -#include <asm/byteorder.h> - -#include <rdma/ib_verbs.h> -#include <rdma/iw_cm.h> -#include <rdma/ib_user_verbs.h> -#include <rdma/uverbs_ioctl.h> - -#include "nes.h" - -#include <rdma/ib_umem.h> - -atomic_t mod_qp_timouts; -atomic_t qps_created; -atomic_t sw_qps_destroyed; - -static void nes_unregister_ofa_device(struct nes_ib_device *nesibdev); -static int nes_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata); - -/** - * nes_alloc_mw - */ -static struct ib_mw *nes_alloc_mw(struct ib_pd *ibpd, enum ib_mw_type type, - struct ib_udata *udata) -{ - struct nes_pd *nespd = to_nespd(ibpd); - struct nes_vnic *nesvnic = to_nesvnic(ibpd->device); - struct nes_device *nesdev = nesvnic->nesdev; - struct nes_adapter *nesadapter = nesdev->nesadapter; - struct nes_cqp_request *cqp_request; - struct nes_mr *nesmr; - struct ib_mw *ibmw; - struct nes_hw_cqp_wqe *cqp_wqe; - int ret; - u32 stag; - u32 stag_index = 0; - u32 next_stag_index = 0; - u32 driver_key = 0; - u8 stag_key = 0; - - if (type != IB_MW_TYPE_1) - return ERR_PTR(-EINVAL); - - get_random_bytes(&next_stag_index, sizeof(next_stag_index)); - stag_key = (u8)next_stag_index; - - driver_key = 0; - - next_stag_index >>= 8; - next_stag_index %= nesadapter->max_mr; - - ret = nes_alloc_resource(nesadapter, nesadapter->allocated_mrs, - nesadapter->max_mr, &stag_index, &next_stag_index, NES_RESOURCE_MW); - if (ret) { - return ERR_PTR(ret); - } - - nesmr = kzalloc(sizeof(*nesmr), GFP_KERNEL); - if (!nesmr) { - nes_free_resource(nesadapter, nesadapter->allocated_mrs, stag_index); - return ERR_PTR(-ENOMEM); - } - - stag = stag_index << 8; - stag |= driver_key; - stag += (u32)stag_key; - - nes_debug(NES_DBG_MR, "Registering STag 0x%08X, index = 0x%08X\n", - stag, stag_index); - - /* Register the region with the adapter */ - cqp_request = nes_get_cqp_request(nesdev); - if (cqp_request == NULL) { - kfree(nesmr); - nes_free_resource(nesadapter, nesadapter->allocated_mrs, stag_index); - return ERR_PTR(-ENOMEM); - } - - cqp_request->waiting = 1; - cqp_wqe = &cqp_request->cqp_wqe; - - cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] = - cpu_to_le32( NES_CQP_ALLOCATE_STAG | NES_CQP_STAG_RIGHTS_REMOTE_READ | - NES_CQP_STAG_RIGHTS_REMOTE_WRITE | NES_CQP_STAG_VA_TO | - NES_CQP_STAG_REM_ACC_EN); - - nes_fill_init_cqp_wqe(cqp_wqe, nesdev); - set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_STAG_WQE_LEN_HIGH_PD_IDX, (nespd->pd_id & 0x00007fff)); - set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_STAG_WQE_STAG_IDX, stag); - - atomic_set(&cqp_request->refcount, 2); - nes_post_cqp_request(nesdev, cqp_request); - - /* Wait for CQP */ - ret = wait_event_timeout(cqp_request->waitq, (cqp_request->request_done != 0), - NES_EVENT_TIMEOUT); - nes_debug(NES_DBG_MR, "Register STag 0x%08X completed, wait_event_timeout ret = %u," - " CQP Major:Minor codes = 0x%04X:0x%04X.\n", - stag, ret, cqp_request->major_code, cqp_request->minor_code); - if ((!ret) || (cqp_request->major_code)) { - nes_put_cqp_request(nesdev, cqp_request); - kfree(nesmr); - nes_free_resource(nesadapter, nesadapter->allocated_mrs, stag_index); - if (!ret) { - return ERR_PTR(-ETIME); - } else { - return ERR_PTR(-ENOMEM); - } - } - nes_put_cqp_request(nesdev, cqp_request); - - nesmr->ibmw.rkey = stag; - nesmr->mode = IWNES_MEMREG_TYPE_MW; - ibmw = &nesmr->ibmw; - nesmr->pbl_4k = 0; - nesmr->pbls_used = 0; - - return ibmw; -} - - -/** - * nes_dealloc_mw - */ -static int nes_dealloc_mw(struct ib_mw *ibmw) -{ - struct nes_mr *nesmr = to_nesmw(ibmw); - struct nes_vnic *nesvnic = to_nesvnic(ibmw->device); - struct nes_device *nesdev = nesvnic->nesdev; - struct nes_adapter *nesadapter = nesdev->nesadapter; - struct nes_hw_cqp_wqe *cqp_wqe; - struct nes_cqp_request *cqp_request; - int err = 0; - int ret; - - /* Deallocate the window with the adapter */ - cqp_request = nes_get_cqp_request(nesdev); - if (cqp_request == NULL) { - nes_debug(NES_DBG_MR, "Failed to get a cqp_request.\n"); - return -ENOMEM; - } - cqp_request->waiting = 1; - cqp_wqe = &cqp_request->cqp_wqe; - nes_fill_init_cqp_wqe(cqp_wqe, nesdev); - set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX, NES_CQP_DEALLOCATE_STAG); - set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_STAG_WQE_STAG_IDX, ibmw->rkey); - - atomic_set(&cqp_request->refcount, 2); - nes_post_cqp_request(nesdev, cqp_request); - - /* Wait for CQP */ - nes_debug(NES_DBG_MR, "Waiting for deallocate STag 0x%08X to complete.\n", - ibmw->rkey); - ret = wait_event_timeout(cqp_request->waitq, (0 != cqp_request->request_done), - NES_EVENT_TIMEOUT); - nes_debug(NES_DBG_MR, "Deallocate STag completed, wait_event_timeout ret = %u," - " CQP Major:Minor codes = 0x%04X:0x%04X.\n", - ret, cqp_request->major_code, cqp_request->minor_code); - if (!ret) - err = -ETIME; - else if (cqp_request->major_code) - err = -EIO; - - nes_put_cqp_request(nesdev, cqp_request); - - nes_free_resource(nesadapter, nesadapter->allocated_mrs, - (ibmw->rkey & 0x0fffff00) >> 8); - kfree(nesmr); - - return err; -} - - -/* - * nes_alloc_fast_mr - */ -static int alloc_fast_reg_mr(struct nes_device *nesdev, struct nes_pd *nespd, - u32 stag, u32 page_count) -{ - struct nes_hw_cqp_wqe *cqp_wqe; - struct nes_cqp_request *cqp_request; - unsigned long flags; - int ret; - struct nes_adapter *nesadapter = nesdev->nesadapter; - u32 opcode = 0; - u16 major_code; - u64 region_length = page_count * PAGE_SIZE; - - - cqp_request = nes_get_cqp_request(nesdev); - if (cqp_request == NULL) { - nes_debug(NES_DBG_MR, "Failed to get a cqp_request.\n"); - return -ENOMEM; - } - nes_debug(NES_DBG_MR, "alloc_fast_reg_mr: page_count = %d, " - "region_length = %llu\n", - page_count, region_length); - cqp_request->waiting = 1; - cqp_wqe = &cqp_request->cqp_wqe; - - spin_lock_irqsave(&nesadapter->pbl_lock, flags); - if (nesadapter->free_4kpbl > 0) { - nesadapter->free_4kpbl--; - spin_unlock_irqrestore(&nesadapter->pbl_lock, flags); - } else { - /* No 4kpbl's available: */ - spin_unlock_irqrestore(&nesadapter->pbl_lock, flags); - nes_debug(NES_DBG_MR, "Out of Pbls\n"); - nes_free_cqp_request(nesdev, cqp_request); - return -ENOMEM; - } - - opcode = NES_CQP_ALLOCATE_STAG | NES_CQP_STAG_MR | - NES_CQP_STAG_PBL_BLK_SIZE | NES_CQP_STAG_VA_TO | - NES_CQP_STAG_REM_ACC_EN; - /* - * The current OFED API does not support the zero based TO option. - * If added then need to changed the NES_CQP_STAG_VA* option. Also, - * the API does not support that ability to have the MR set for local - * access only when created and not allow the SQ op to override. Given - * this the remote enable must be set here. - */ - - nes_fill_init_cqp_wqe(cqp_wqe, nesdev); - set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX, opcode); - set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_STAG_WQE_PBL_BLK_COUNT_IDX, 1); - - cqp_wqe->wqe_words[NES_CQP_STAG_WQE_LEN_HIGH_PD_IDX] = - cpu_to_le32((u32)(region_length >> 8) & 0xff000000); - cqp_wqe->wqe_words[NES_CQP_STAG_WQE_LEN_HIGH_PD_IDX] |= - cpu_to_le32(nespd->pd_id & 0x00007fff); - - set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_STAG_WQE_STAG_IDX, stag); - set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_STAG_WQE_VA_LOW_IDX, 0); - set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_STAG_WQE_LEN_LOW_IDX, 0); - set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_STAG_WQE_PA_LOW_IDX, 0); - set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_STAG_WQE_PBL_LEN_IDX, (page_count * 8)); - cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] |= cpu_to_le32(NES_CQP_STAG_PBL_BLK_SIZE); - barrier(); - - atomic_set(&cqp_request->refcount, 2); - nes_post_cqp_request(nesdev, cqp_request); - - /* Wait for CQP */ - ret = wait_event_timeout(cqp_request->waitq, - (0 != cqp_request->request_done), - NES_EVENT_TIMEOUT); - - nes_debug(NES_DBG_MR, "Allocate STag 0x%08X completed, " - "wait_event_timeout ret = %u, CQP Major:Minor codes = " - "0x%04X:0x%04X.\n", stag, ret, cqp_request->major_code, - cqp_request->minor_code); - major_code = cqp_request->major_code; - nes_put_cqp_request(nesdev, cqp_request); - - if (!ret || major_code) { - spin_lock_irqsave(&nesadapter->pbl_lock, flags); - nesadapter->free_4kpbl++; - spin_unlock_irqrestore(&nesadapter->pbl_lock, flags); - } - - if (!ret) - return -ETIME; - else if (major_code) - return -EIO; - return 0; -} - -/* - * nes_alloc_mr - */ -static struct ib_mr *nes_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type mr_type, - u32 max_num_sg, struct ib_udata *udata) -{ - struct nes_pd *nespd = to_nespd(ibpd); - struct nes_vnic *nesvnic = to_nesvnic(ibpd->device); - struct nes_device *nesdev = nesvnic->nesdev; - struct nes_adapter *nesadapter = nesdev->nesadapter; - - u32 next_stag_index; - u8 stag_key = 0; - u32 driver_key = 0; - int err = 0; - u32 stag_index = 0; - struct nes_mr *nesmr; - u32 stag; - int ret; - struct ib_mr *ibmr; - - if (mr_type != IB_MR_TYPE_MEM_REG) - return ERR_PTR(-EINVAL); - - if (max_num_sg > (NES_4K_PBL_CHUNK_SIZE / sizeof(u64))) - return ERR_PTR(-E2BIG); - -/* - * Note: Set to always use a fixed length single page entry PBL. This is to allow - * for the fast_reg_mr operation to always know the size of the PBL. - */ - if (max_num_sg > (NES_4K_PBL_CHUNK_SIZE / sizeof(u64))) - return ERR_PTR(-E2BIG); - - get_random_bytes(&next_stag_index, sizeof(next_stag_index)); - stag_key = (u8)next_stag_index; - next_stag_index >>= 8; - next_stag_index %= nesadapter->max_mr; - - err = nes_alloc_resource(nesadapter, nesadapter->allocated_mrs, - nesadapter->max_mr, &stag_index, - &next_stag_index, NES_RESOURCE_FAST_MR); - if (err) - return ERR_PTR(err); - - nesmr = kzalloc(sizeof(*nesmr), GFP_KERNEL); - if (!nesmr) { - nes_free_resource(nesadapter, nesadapter->allocated_mrs, stag_index); - return ERR_PTR(-ENOMEM); - } - - stag = stag_index << 8; - stag |= driver_key; - stag += (u32)stag_key; - - nes_debug(NES_DBG_MR, "Allocating STag 0x%08X index = 0x%08X\n", - stag, stag_index); - - ret = alloc_fast_reg_mr(nesdev, nespd, stag, max_num_sg); - - if (ret == 0) { - nesmr->ibmr.rkey = stag; - nesmr->ibmr.lkey = stag; - nesmr->mode = IWNES_MEMREG_TYPE_FMEM; - ibmr = &nesmr->ibmr; - } else { - kfree(nesmr); - nes_free_resource(nesadapter, nesadapter->allocated_mrs, stag_index); - return ERR_PTR(-ENOMEM); - } - - nesmr->pages = pci_alloc_consistent(nesdev->pcidev, - max_num_sg * sizeof(u64), - &nesmr->paddr); - if (!nesmr->paddr) - goto err; - - nesmr->max_pages = max_num_sg; - - return ibmr; - -err: - nes_dereg_mr(ibmr, udata); - - return ERR_PTR(-ENOMEM); -} - -static int nes_set_page(struct ib_mr *ibmr, u64 addr) -{ - struct nes_mr *nesmr = to_nesmr(ibmr); - - if (unlikely(nesmr->npages == nesmr->max_pages)) - return -ENOMEM; - - nesmr->pages[nesmr->npages++] = cpu_to_le64(addr); - - return 0; -} - -static int nes_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, - int sg_nents, unsigned int *sg_offset) -{ - struct nes_mr *nesmr = to_nesmr(ibmr); - - nesmr->npages = 0; - - return ib_sg_to_pages(ibmr, sg, sg_nents, sg_offset, nes_set_page); -} - -/** - * nes_query_device - */ -static int nes_query_device(struct ib_device *ibdev, struct ib_device_attr *props, - struct ib_udata *uhw) -{ - struct nes_vnic *nesvnic = to_nesvnic(ibdev); - struct nes_device *nesdev = nesvnic->nesdev; - struct nes_ib_device *nesibdev = nesvnic->nesibdev; - - if (uhw->inlen || uhw->outlen) - return -EINVAL; - - memset(props, 0, sizeof(*props)); - memcpy(&props->sys_image_guid, nesvnic->netdev->dev_addr, 6); - - props->fw_ver = nesdev->nesadapter->firmware_version; - props->device_cap_flags = nesdev->nesadapter->device_cap_flags; - props->vendor_id = nesdev->nesadapter->vendor_id; - props->vendor_part_id = nesdev->nesadapter->vendor_part_id; - props->hw_ver = nesdev->nesadapter->hw_rev; - props->max_mr_size = 0x80000000; - props->max_qp = nesibdev->max_qp; - props->max_qp_wr = nesdev->nesadapter->max_qp_wr - 2; - props->max_send_sge = nesdev->nesadapter->max_sge; - props->max_recv_sge = nesdev->nesadapter->max_sge; - props->max_cq = nesibdev->max_cq; - props->max_cqe = nesdev->nesadapter->max_cqe; - props->max_mr = nesibdev->max_mr; - props->max_mw = nesibdev->max_mr; - props->max_pd = nesibdev->max_pd; - props->max_sge_rd = 1; - switch (nesdev->nesadapter->max_irrq_wr) { - case 0: - props->max_qp_rd_atom = 2; - break; - case 1: - props->max_qp_rd_atom = 8; - break; - case 2: - props->max_qp_rd_atom = 32; - break; - case 3: - props->max_qp_rd_atom = 64; - break; - default: - props->max_qp_rd_atom = 0; - } - props->max_qp_init_rd_atom = props->max_qp_rd_atom; - props->atomic_cap = IB_ATOMIC_NONE; - props->max_map_per_fmr = 1; - - return 0; -} - - -/** - * nes_query_port - */ -static int nes_query_port(struct ib_device *ibdev, u8 port, struct ib_port_attr *props) -{ - struct nes_vnic *nesvnic = to_nesvnic(ibdev); - struct net_device *netdev = nesvnic->netdev; - - /* props being zeroed by the caller, avoid zeroing it here */ - - props->max_mtu = IB_MTU_4096; - props->active_mtu = ib_mtu_int_to_enum(netdev->mtu); - - props->lid = 1; - if (netif_queue_stopped(netdev)) - props->state = IB_PORT_DOWN; - else if (nesvnic->linkup) - props->state = IB_PORT_ACTIVE; - else - props->state = IB_PORT_DOWN; - props->port_cap_flags = IB_PORT_CM_SUP | IB_PORT_REINIT_SUP | - IB_PORT_VENDOR_CLASS_SUP | IB_PORT_BOOT_MGMT_SUP; - props->gid_tbl_len = 1; - props->pkey_tbl_len = 1; - props->active_width = IB_WIDTH_4X; - props->active_speed = IB_SPEED_SDR; - props->max_msg_sz = 0x80000000; - - return 0; -} - -/** - * nes_query_pkey - */ -static int nes_query_pkey(struct ib_device *ibdev, u8 port, u16 index, u16 *pkey) -{ - *pkey = 0; - return 0; -} - - -/** - * nes_query_gid - */ -static int nes_query_gid(struct ib_device *ibdev, u8 port, - int index, union ib_gid *gid) -{ - struct nes_vnic *nesvnic = to_nesvnic(ibdev); - - memset(&(gid->raw[0]), 0, sizeof(gid->raw)); - memcpy(&(gid->raw[0]), nesvnic->netdev->dev_addr, 6); - - return 0; -} - - -/** - * nes_alloc_ucontext - Allocate the user context data structure. This keeps track - * of all objects associated with a particular user-mode client. - */ -static int nes_alloc_ucontext(struct ib_ucontext *uctx, struct ib_udata *udata) -{ - struct ib_device *ibdev = uctx->device; - struct nes_vnic *nesvnic = to_nesvnic(ibdev); - struct nes_device *nesdev = nesvnic->nesdev; - struct nes_adapter *nesadapter = nesdev->nesadapter; - struct nes_alloc_ucontext_req req; - struct nes_alloc_ucontext_resp uresp = {}; - struct nes_ucontext *nes_ucontext = to_nesucontext(uctx); - struct nes_ib_device *nesibdev = nesvnic->nesibdev; - - - if (ib_copy_from_udata(&req, udata, sizeof(struct nes_alloc_ucontext_req))) { - printk(KERN_ERR PFX "Invalid structure size on allocate user context.\n"); - return -EINVAL; - } - - if (req.userspace_ver != NES_ABI_USERSPACE_VER) { - printk(KERN_ERR PFX "Invalid userspace driver version detected. Detected version %d, should be %d\n", - req.userspace_ver, NES_ABI_USERSPACE_VER); - return -EINVAL; - } - - - uresp.max_qps = nesibdev->max_qp; - uresp.max_pds = nesibdev->max_pd; - uresp.wq_size = nesdev->nesadapter->max_qp_wr * 2; - uresp.virtwq = nesadapter->virtwq; - uresp.kernel_ver = NES_ABI_KERNEL_VER; - - nes_ucontext->nesdev = nesdev; - nes_ucontext->mmap_wq_offset = uresp.max_pds; - nes_ucontext->mmap_cq_offset = nes_ucontext->mmap_wq_offset + - ((sizeof(struct nes_hw_qp_wqe) * uresp.max_qps * 2) + PAGE_SIZE-1) / - PAGE_SIZE; - - - if (ib_copy_to_udata(udata, &uresp, sizeof(uresp))) - return -EFAULT; - - INIT_LIST_HEAD(&nes_ucontext->cq_reg_mem_list); - INIT_LIST_HEAD(&nes_ucontext->qp_reg_mem_list); - return 0; -} - -/** - * nes_dealloc_ucontext - */ -static void nes_dealloc_ucontext(struct ib_ucontext *context) -{ - return; -} - -/** - * nes_mmap - */ -static int nes_mmap(struct ib_ucontext *context, struct vm_area_struct *vma) -{ - unsigned long index; - struct nes_vnic *nesvnic = to_nesvnic(context->device); - struct nes_device *nesdev = nesvnic->nesdev; - /* struct nes_adapter *nesadapter = nesdev->nesadapter; */ - struct nes_ucontext *nes_ucontext; - struct nes_qp *nesqp; - - nes_ucontext = to_nesucontext(context); - - - if (vma->vm_pgoff >= nes_ucontext->mmap_wq_offset) { - index = (vma->vm_pgoff - nes_ucontext->mmap_wq_offset) * PAGE_SIZE; - index /= ((sizeof(struct nes_hw_qp_wqe) * nesdev->nesadapter->max_qp_wr * 2) + - PAGE_SIZE-1) & (~(PAGE_SIZE-1)); - if (!test_bit(index, nes_ucontext->allocated_wqs)) { - nes_debug(NES_DBG_MMAP, "wq %lu not allocated\n", index); - return -EFAULT; - } - nesqp = nes_ucontext->mmap_nesqp[index]; - if (nesqp == NULL) { - nes_debug(NES_DBG_MMAP, "wq %lu has a NULL QP base.\n", index); - return -EFAULT; - } - if (remap_pfn_range(vma, vma->vm_start, - virt_to_phys(nesqp->hwqp.sq_vbase) >> PAGE_SHIFT, - vma->vm_end - vma->vm_start, - vma->vm_page_prot)) { - nes_debug(NES_DBG_MMAP, "remap_pfn_range failed.\n"); - return -EAGAIN; - } - vma->vm_private_data = nesqp; - return 0; - } else { - index = vma->vm_pgoff; - if (!test_bit(index, nes_ucontext->allocated_doorbells)) - return -EFAULT; - - vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); - if (io_remap_pfn_range(vma, vma->vm_start, - (nesdev->doorbell_start + - ((nes_ucontext->mmap_db_index[index] - nesdev->base_doorbell_index) * 4096)) - >> PAGE_SHIFT, PAGE_SIZE, vma->vm_page_prot)) - return -EAGAIN; - vma->vm_private_data = nes_ucontext; - return 0; - } - - return -ENOSYS; -} - - -/** - * nes_alloc_pd - */ -static int nes_alloc_pd(struct ib_pd *pd, struct ib_udata *udata) -{ - struct ib_device *ibdev = pd->device; - struct nes_pd *nespd = to_nespd(pd); - struct nes_vnic *nesvnic = to_nesvnic(ibdev); - struct nes_device *nesdev = nesvnic->nesdev; - struct nes_adapter *nesadapter = nesdev->nesadapter; - struct nes_alloc_pd_resp uresp; - u32 pd_num = 0; - int err; - struct nes_ucontext *nesucontext = rdma_udata_to_drv_context( - udata, struct nes_ucontext, ibucontext); - - nes_debug( - NES_DBG_PD, - "nesvnic=%p, netdev=%p %s, ibdev=%p, context=%p, netdev refcnt=%u\n", - nesvnic, nesdev->netdev[0], nesdev->netdev[0]->name, ibdev, - &nesucontext->ibucontext, netdev_refcnt_read(nesvnic->netdev)); - - err = nes_alloc_resource(nesadapter, nesadapter->allocated_pds, - nesadapter->max_pd, &pd_num, &nesadapter->next_pd, NES_RESOURCE_PD); - if (err) - return err; - - nes_debug(NES_DBG_PD, "Allocating PD (%p) for ib device %s\n", - nespd, dev_name(&nesvnic->nesibdev->ibdev.dev)); - - nespd->pd_id = (pd_num << (PAGE_SHIFT-12)) + nesadapter->base_pd; - - if (udata) { - nespd->mmap_db_index = find_next_zero_bit(nesucontext->allocated_doorbells, - NES_MAX_USER_DB_REGIONS, nesucontext->first_free_db); - nes_debug(NES_DBG_PD, "find_first_zero_biton doorbells returned %u, mapping pd_id %u.\n", - nespd->mmap_db_index, nespd->pd_id); - if (nespd->mmap_db_index >= NES_MAX_USER_DB_REGIONS) { - nes_debug(NES_DBG_PD, "mmap_db_index > MAX\n"); - nes_free_resource(nesadapter, nesadapter->allocated_pds, pd_num); - return -ENOMEM; - } - - uresp.pd_id = nespd->pd_id; - uresp.mmap_db_index = nespd->mmap_db_index; - if (ib_copy_to_udata(udata, &uresp, sizeof (struct nes_alloc_pd_resp))) { - nes_free_resource(nesadapter, nesadapter->allocated_pds, pd_num); - return -EFAULT; - } - - set_bit(nespd->mmap_db_index, nesucontext->allocated_doorbells); - nesucontext->mmap_db_index[nespd->mmap_db_index] = nespd->pd_id; - nesucontext->first_free_db = nespd->mmap_db_index + 1; - } - - nes_debug(NES_DBG_PD, "PD%u structure located @%p.\n", nespd->pd_id, nespd); - return 0; -} - - -/** - * nes_dealloc_pd - */ -static void nes_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata) -{ - struct nes_ucontext *nesucontext; - struct nes_pd *nespd = to_nespd(ibpd); - struct nes_vnic *nesvnic = to_nesvnic(ibpd->device); - struct nes_device *nesdev = nesvnic->nesdev; - struct nes_adapter *nesadapter = nesdev->nesadapter; - - if (udata) { - nesucontext = - rdma_udata_to_drv_context( - udata, - struct nes_ucontext, - ibucontext); - nes_debug(NES_DBG_PD, "Clearing bit %u from allocated doorbells\n", - nespd->mmap_db_index); - clear_bit(nespd->mmap_db_index, nesucontext->allocated_doorbells); - nesucontext->mmap_db_index[nespd->mmap_db_index] = 0; - if (nesucontext->first_free_db > nespd->mmap_db_index) { - nesucontext->first_free_db = nespd->mmap_db_index; - } - } - - nes_debug(NES_DBG_PD, "Deallocating PD%u structure located @%p.\n", - nespd->pd_id, nespd); - nes_free_resource(nesadapter, nesadapter->allocated_pds, - (nespd->pd_id-nesadapter->base_pd)>>(PAGE_SHIFT-12)); -} - - -/** - * nes_get_encoded_size - */ -static inline u8 nes_get_encoded_size(int *size) -{ - u8 encoded_size = 0; - if (*size <= 32) { - *size = 32; - encoded_size = 1; - } else if (*size <= 128) { - *size = 128; - encoded_size = 2; - } else if (*size <= 512) { - *size = 512; - encoded_size = 3; - } - return (encoded_size); -} - - - -/** - * nes_setup_virt_qp - */ -static int nes_setup_virt_qp(struct nes_qp *nesqp, struct nes_pbl *nespbl, - struct nes_vnic *nesvnic, int sq_size, int rq_size) -{ - unsigned long flags; - void *mem; - __le64 *pbl = NULL; - __le64 *tpbl; - __le64 *pblbuffer; - struct nes_device *nesdev = nesvnic->nesdev; - struct nes_adapter *nesadapter = nesdev->nesadapter; - u32 pbl_entries; - u8 rq_pbl_entries; - u8 sq_pbl_entries; - - pbl_entries = nespbl->pbl_size >> 3; - nes_debug(NES_DBG_QP, "Userspace PBL, pbl_size=%u, pbl_entries = %d pbl_vbase=%p, pbl_pbase=%lx\n", - nespbl->pbl_size, pbl_entries, - (void *)nespbl->pbl_vbase, - (unsigned long) nespbl->pbl_pbase); - pbl = (__le64 *) nespbl->pbl_vbase; /* points to first pbl entry */ - /* now lets set the sq_vbase as well as rq_vbase addrs we will assign */ - /* the first pbl to be fro the rq_vbase... */ - rq_pbl_entries = (rq_size * sizeof(struct nes_hw_qp_wqe)) >> 12; - sq_pbl_entries = (sq_size * sizeof(struct nes_hw_qp_wqe)) >> 12; - nesqp->hwqp.sq_pbase = (le32_to_cpu(((__le32 *)pbl)[0])) | ((u64)((le32_to_cpu(((__le32 *)pbl)[1]))) << 32); - if (!nespbl->page) { - nes_debug(NES_DBG_QP, "QP nespbl->page is NULL \n"); - kfree(nespbl); - return -ENOMEM; - } - - nesqp->hwqp.sq_vbase = kmap(nespbl->page); - nesqp->page = nespbl->page; - if (!nesqp->hwqp.sq_vbase) { - nes_debug(NES_DBG_QP, "QP sq_vbase kmap failed\n"); - kfree(nespbl); - return -ENOMEM; - } - - /* Now to get to sq.. we need to calculate how many */ - /* PBL entries were used by the rq.. */ - pbl += sq_pbl_entries; - nesqp->hwqp.rq_pbase = (le32_to_cpu(((__le32 *)pbl)[0])) | ((u64)((le32_to_cpu(((__le32 *)pbl)[1]))) << 32); - /* nesqp->hwqp.rq_vbase = bus_to_virt(*pbl); */ - /*nesqp->hwqp.rq_vbase = phys_to_virt(*pbl); */ - - nes_debug(NES_DBG_QP, "QP sq_vbase= %p sq_pbase=%lx rq_vbase=%p rq_pbase=%lx\n", - nesqp->hwqp.sq_vbase, (unsigned long) nesqp->hwqp.sq_pbase, - nesqp->hwqp.rq_vbase, (unsigned long) nesqp->hwqp.rq_pbase); - spin_lock_irqsave(&nesadapter->pbl_lock, flags); - if (!nesadapter->free_256pbl) { - pci_free_consistent(nesdev->pcidev, nespbl->pbl_size, nespbl->pbl_vbase, - nespbl->pbl_pbase); - spin_unlock_irqrestore(&nesadapter->pbl_lock, flags); - kunmap(nesqp->page); - kfree(nespbl); - return -ENOMEM; - } - nesadapter->free_256pbl--; - spin_unlock_irqrestore(&nesadapter->pbl_lock, flags); - - nesqp->pbl_vbase = pci_alloc_consistent(nesdev->pcidev, 256, &nesqp->pbl_pbase); - pblbuffer = nesqp->pbl_vbase; - if (!nesqp->pbl_vbase) { - /* memory allocated during nes_reg_user_mr() */ - pci_free_consistent(nesdev->pcidev, nespbl->pbl_size, nespbl->pbl_vbase, - nespbl->pbl_pbase); - kfree(nespbl); - spin_lock_irqsave(&nesadapter->pbl_lock, flags); - nesadapter->free_256pbl++; - spin_unlock_irqrestore(&nesadapter->pbl_lock, flags); - kunmap(nesqp->page); - return -ENOMEM; - } - memset(nesqp->pbl_vbase, 0, 256); - /* fill in the page address in the pbl buffer.. */ - tpbl = pblbuffer + 16; - pbl = (__le64 *)nespbl->pbl_vbase; - while (sq_pbl_entries--) - *tpbl++ = *pbl++; - tpbl = pblbuffer; - while (rq_pbl_entries--) - *tpbl++ = *pbl++; - - /* done with memory allocated during nes_reg_user_mr() */ - pci_free_consistent(nesdev->pcidev, nespbl->pbl_size, nespbl->pbl_vbase, - nespbl->pbl_pbase); - kfree(nespbl); - - nesqp->qp_mem_size = - max((u32)sizeof(struct nes_qp_context), ((u32)256)) + 256; /* this is Q2 */ - /* Round up to a multiple of a page */ - nesqp->qp_mem_size += PAGE_SIZE - 1; - nesqp->qp_mem_size &= ~(PAGE_SIZE - 1); - - mem = pci_alloc_consistent(nesdev->pcidev, nesqp->qp_mem_size, - &nesqp->hwqp.q2_pbase); - - if (!mem) { - pci_free_consistent(nesdev->pcidev, 256, nesqp->pbl_vbase, nesqp->pbl_pbase); - nesqp->pbl_vbase = NULL; - spin_lock_irqsave(&nesadapter->pbl_lock, flags); - nesadapter->free_256pbl++; - spin_unlock_irqrestore(&nesadapter->pbl_lock, flags); - kunmap(nesqp->page); - return -ENOMEM; - } - nesqp->sq_kmapped = 1; - nesqp->hwqp.q2_vbase = mem; - mem += 256; - memset(nesqp->hwqp.q2_vbase, 0, 256); - nesqp->nesqp_context = mem; - memset(nesqp->nesqp_context, 0, sizeof(*nesqp->nesqp_context)); - nesqp->nesqp_context_pbase = nesqp->hwqp.q2_pbase + 256; - - return 0; -} - - -/** - * nes_setup_mmap_qp - */ -static int nes_setup_mmap_qp(struct nes_qp *nesqp, struct nes_vnic *nesvnic, - int sq_size, int rq_size) -{ - void *mem; - struct nes_device *nesdev = nesvnic->nesdev; - - nesqp->qp_mem_size = (sizeof(struct nes_hw_qp_wqe) * sq_size) + - (sizeof(struct nes_hw_qp_wqe) * rq_size) + - max((u32)sizeof(struct nes_qp_context), ((u32)256)) + - 256; /* this is Q2 */ - /* Round up to a multiple of a page */ - nesqp->qp_mem_size += PAGE_SIZE - 1; - nesqp->qp_mem_size &= ~(PAGE_SIZE - 1); - - mem = pci_alloc_consistent(nesdev->pcidev, nesqp->qp_mem_size, - &nesqp->hwqp.sq_pbase); - if (!mem) - return -ENOMEM; - nes_debug(NES_DBG_QP, "PCI consistent memory for " - "host descriptor rings located @ %p (pa = 0x%08lX.) size = %u.\n", - mem, (unsigned long)nesqp->hwqp.sq_pbase, nesqp->qp_mem_size); - - memset(mem, 0, nesqp->qp_mem_size); - - nesqp->hwqp.sq_vbase = mem; - mem += sizeof(struct nes_hw_qp_wqe) * sq_size; - - nesqp->hwqp.rq_vbase = mem; - nesqp->hwqp.rq_pbase = nesqp->hwqp.sq_pbase + - sizeof(struct nes_hw_qp_wqe) * sq_size; - mem += sizeof(struct nes_hw_qp_wqe) * rq_size; - - nesqp->hwqp.q2_vbase = mem; - nesqp->hwqp.q2_pbase = nesqp->hwqp.rq_pbase + - sizeof(struct nes_hw_qp_wqe) * rq_size; - mem += 256; - memset(nesqp->hwqp.q2_vbase, 0, 256); - - nesqp->nesqp_context = mem; - nesqp->nesqp_context_pbase = nesqp->hwqp.q2_pbase + 256; - memset(nesqp->nesqp_context, 0, sizeof(*nesqp->nesqp_context)); - return 0; -} - - -/** - * nes_free_qp_mem() is to free up the qp's pci_alloc_consistent() memory. - */ -static void nes_free_qp_mem(struct nes_device *nesdev, - struct nes_qp *nesqp, int virt_wqs) -{ - unsigned long flags; - struct nes_adapter *nesadapter = nesdev->nesadapter; - if (!virt_wqs) { - pci_free_consistent(nesdev->pcidev, nesqp->qp_mem_size, - nesqp->hwqp.sq_vbase, nesqp->hwqp.sq_pbase); - }else { - spin_lock_irqsave(&nesadapter->pbl_lock, flags); - nesadapter->free_256pbl++; - spin_unlock_irqrestore(&nesadapter->pbl_lock, flags); - pci_free_consistent(nesdev->pcidev, nesqp->qp_mem_size, nesqp->hwqp.q2_vbase, nesqp->hwqp.q2_pbase); - pci_free_consistent(nesdev->pcidev, 256, nesqp->pbl_vbase, nesqp->pbl_pbase ); - nesqp->pbl_vbase = NULL; - if (nesqp->sq_kmapped) { - nesqp->sq_kmapped = 0; - kunmap(nesqp->page); - } - } -} - - -/** - * nes_create_qp - */ -static struct ib_qp *nes_create_qp(struct ib_pd *ibpd, - struct ib_qp_init_attr *init_attr, struct ib_udata *udata) -{ - u64 u64temp= 0; - u64 u64nesqp = 0; - struct nes_pd *nespd = to_nespd(ibpd); - struct nes_vnic *nesvnic = to_nesvnic(ibpd->device); - struct nes_device *nesdev = nesvnic->nesdev; - struct nes_adapter *nesadapter = nesdev->nesadapter; - struct nes_qp *nesqp; - struct nes_cq *nescq; - struct nes_ucontext *nes_ucontext = rdma_udata_to_drv_context( - udata, struct nes_ucontext, ibucontext); - struct nes_hw_cqp_wqe *cqp_wqe; - struct nes_cqp_request *cqp_request; - struct nes_create_qp_req req; - struct nes_create_qp_resp uresp; - struct nes_pbl *nespbl = NULL; - u32 qp_num = 0; - u32 opcode = 0; - /* u32 counter = 0; */ - void *mem; - unsigned long flags; - int ret; - int err; - int virt_wqs = 0; - int sq_size; - int rq_size; - u8 sq_encoded_size; - u8 rq_encoded_size; - /* int counter; */ - - if (init_attr->create_flags) - return ERR_PTR(-EINVAL); - - atomic_inc(&qps_created); - switch (init_attr->qp_type) { - case IB_QPT_RC: - if (nes_drv_opt & NES_DRV_OPT_NO_INLINE_DATA) { - init_attr->cap.max_inline_data = 0; - } else { - init_attr->cap.max_inline_data = 64; - } - sq_size = init_attr->cap.max_send_wr; - rq_size = init_attr->cap.max_recv_wr; - - /* check if the encoded sizes are OK or not... */ - sq_encoded_size = nes_get_encoded_size(&sq_size); - rq_encoded_size = nes_get_encoded_size(&rq_size); - - if ((!sq_encoded_size) || (!rq_encoded_size)) { - nes_debug(NES_DBG_QP, "ERROR bad rq (%u) or sq (%u) size\n", - rq_size, sq_size); - return ERR_PTR(-EINVAL); - } - - init_attr->cap.max_send_wr = sq_size -2; - init_attr->cap.max_recv_wr = rq_size -1; - nes_debug(NES_DBG_QP, "RQ size=%u, SQ Size=%u\n", rq_size, sq_size); - - ret = nes_alloc_resource(nesadapter, nesadapter->allocated_qps, - nesadapter->max_qp, &qp_num, &nesadapter->next_qp, NES_RESOURCE_QP); - if (ret) { - return ERR_PTR(ret); - } - - /* Need 512 (actually now 1024) byte alignment on this structure */ - mem = kzalloc(sizeof(*nesqp)+NES_SW_CONTEXT_ALIGN-1, GFP_KERNEL); - if (!mem) { - nes_free_resource(nesadapter, nesadapter->allocated_qps, qp_num); - return ERR_PTR(-ENOMEM); - } - u64nesqp = (unsigned long)mem; - u64nesqp += ((u64)NES_SW_CONTEXT_ALIGN) - 1; - u64temp = ((u64)NES_SW_CONTEXT_ALIGN) - 1; - u64nesqp &= ~u64temp; - nesqp = (struct nes_qp *)(unsigned long)u64nesqp; - /* nes_debug(NES_DBG_QP, "nesqp=%p, allocated buffer=%p. Rounded to closest %u\n", - nesqp, mem, NES_SW_CONTEXT_ALIGN); */ - nesqp->allocated_buffer = mem; - - if (udata) { - if (ib_copy_from_udata(&req, udata, sizeof(struct nes_create_qp_req))) { - nes_free_resource(nesadapter, nesadapter->allocated_qps, qp_num); - kfree(nesqp->allocated_buffer); - nes_debug(NES_DBG_QP, "ib_copy_from_udata() Failed \n"); - return ERR_PTR(-EFAULT); - } - if (req.user_wqe_buffers) { - virt_wqs = 1; - } - if (req.user_qp_buffer) - nesqp->nesuqp_addr = req.user_qp_buffer; - - nesqp->user_mode = 1; - if (virt_wqs) { - err = 1; - list_for_each_entry(nespbl, &nes_ucontext->qp_reg_mem_list, list) { - if (nespbl->user_base == (unsigned long )req.user_wqe_buffers) { - list_del(&nespbl->list); - err = 0; - nes_debug(NES_DBG_QP, "Found PBL for virtual QP. nespbl=%p. user_base=0x%lx\n", - nespbl, nespbl->user_base); - break; - } - } - if (err) { - nes_debug(NES_DBG_QP, "Didn't Find PBL for virtual QP. address = %llx.\n", - (long long unsigned int)req.user_wqe_buffers); - nes_free_resource(nesadapter, nesadapter->allocated_qps, qp_num); - kfree(nesqp->allocated_buffer); - return ERR_PTR(-EFAULT); - } - } - - nesqp->mmap_sq_db_index = - find_next_zero_bit(nes_ucontext->allocated_wqs, - NES_MAX_USER_WQ_REGIONS, nes_ucontext->first_free_wq); - /* nes_debug(NES_DBG_QP, "find_first_zero_biton wqs returned %u\n", - nespd->mmap_db_index); */ - if (nesqp->mmap_sq_db_index >= NES_MAX_USER_WQ_REGIONS) { - nes_debug(NES_DBG_QP, - "db index > max user regions, failing create QP\n"); - nes_free_resource(nesadapter, nesadapter->allocated_qps, qp_num); - if (virt_wqs) { - pci_free_consistent(nesdev->pcidev, nespbl->pbl_size, nespbl->pbl_vbase, - nespbl->pbl_pbase); - kfree(nespbl); - } - kfree(nesqp->allocated_buffer); - return ERR_PTR(-ENOMEM); - } - set_bit(nesqp->mmap_sq_db_index, nes_ucontext->allocated_wqs); - nes_ucontext->mmap_nesqp[nesqp->mmap_sq_db_index] = nesqp; - nes_ucontext->first_free_wq = nesqp->mmap_sq_db_index + 1; - } - err = (!virt_wqs) ? nes_setup_mmap_qp(nesqp, nesvnic, sq_size, rq_size) : - nes_setup_virt_qp(nesqp, nespbl, nesvnic, sq_size, rq_size); - if (err) { - nes_debug(NES_DBG_QP, - "error geting qp mem code = %d\n", err); - nes_free_resource(nesadapter, nesadapter->allocated_qps, qp_num); - kfree(nesqp->allocated_buffer); - return ERR_PTR(-ENOMEM); - } - - nesqp->hwqp.sq_size = sq_size; - nesqp->hwqp.sq_encoded_size = sq_encoded_size; - nesqp->hwqp.sq_head = 1; - nesqp->hwqp.rq_size = rq_size; - nesqp->hwqp.rq_encoded_size = rq_encoded_size; - /* nes_debug(NES_DBG_QP, "nesqp->nesqp_context_pbase = %p\n", - (void *)nesqp->nesqp_context_pbase); - */ - nesqp->hwqp.qp_id = qp_num; - nesqp->ibqp.qp_num = nesqp->hwqp.qp_id; - nesqp->nespd = nespd; - - nescq = to_nescq(init_attr->send_cq); - nesqp->nesscq = nescq; - nescq = to_nescq(init_attr->recv_cq); - nesqp->nesrcq = nescq; - - nesqp->nesqp_context->misc |= cpu_to_le32((u32)PCI_FUNC(nesdev->pcidev->devfn) << - NES_QPCONTEXT_MISC_PCI_FCN_SHIFT); - nesqp->nesqp_context->misc |= cpu_to_le32((u32)nesqp->hwqp.rq_encoded_size << - NES_QPCONTEXT_MISC_RQ_SIZE_SHIFT); - nesqp->nesqp_context->misc |= cpu_to_le32((u32)nesqp->hwqp.sq_encoded_size << - NES_QPCONTEXT_MISC_SQ_SIZE_SHIFT); - if (!udata) { - nesqp->nesqp_context->misc |= cpu_to_le32(NES_QPCONTEXT_MISC_PRIV_EN); - nesqp->nesqp_context->misc |= cpu_to_le32(NES_QPCONTEXT_MISC_FAST_REGISTER_EN); - } - nesqp->nesqp_context->cqs = cpu_to_le32(nesqp->nesscq->hw_cq.cq_number + - ((u32)nesqp->nesrcq->hw_cq.cq_number << 16)); - u64temp = (u64)nesqp->hwqp.sq_pbase; - nesqp->nesqp_context->sq_addr_low = cpu_to_le32((u32)u64temp); - nesqp->nesqp_context->sq_addr_high = cpu_to_le32((u32)(u64temp >> 32)); - - - if (!virt_wqs) { - u64temp = (u64)nesqp->hwqp.sq_pbase; - nesqp->nesqp_context->sq_addr_low = cpu_to_le32((u32)u64temp); - nesqp->nesqp_context->sq_addr_high = cpu_to_le32((u32)(u64temp >> 32)); - u64temp = (u64)nesqp->hwqp.rq_pbase; - nesqp->nesqp_context->rq_addr_low = cpu_to_le32((u32)u64temp); - nesqp->nesqp_context->rq_addr_high = cpu_to_le32((u32)(u64temp >> 32)); - } else { - u64temp = (u64)nesqp->pbl_pbase; - nesqp->nesqp_context->rq_addr_low = cpu_to_le32((u32)u64temp); - nesqp->nesqp_context->rq_addr_high = cpu_to_le32((u32)(u64temp >> 32)); - } - - /* nes_debug(NES_DBG_QP, "next_qp_nic_index=%u, using nic_index=%d\n", - nesvnic->next_qp_nic_index, - nesvnic->qp_nic_index[nesvnic->next_qp_nic_index]); */ - spin_lock_irqsave(&nesdev->cqp.lock, flags); - nesqp->nesqp_context->misc2 |= cpu_to_le32( - (u32)nesvnic->qp_nic_index[nesvnic->next_qp_nic_index] << - NES_QPCONTEXT_MISC2_NIC_INDEX_SHIFT); - nesvnic->next_qp_nic_index++; - if ((nesvnic->next_qp_nic_index > 3) || - (nesvnic->qp_nic_index[nesvnic->next_qp_nic_index] == 0xf)) { - nesvnic->next_qp_nic_index = 0; - } - spin_unlock_irqrestore(&nesdev->cqp.lock, flags); - - nesqp->nesqp_context->pd_index_wscale |= cpu_to_le32((u32)nesqp->nespd->pd_id << 16); - u64temp = (u64)nesqp->hwqp.q2_pbase; - nesqp->nesqp_context->q2_addr_low = cpu_to_le32((u32)u64temp); - nesqp->nesqp_context->q2_addr_high = cpu_to_le32((u32)(u64temp >> 32)); - nesqp->nesqp_context->aeq_token_low = cpu_to_le32((u32)((unsigned long)(nesqp))); - nesqp->nesqp_context->aeq_token_high = cpu_to_le32((u32)(upper_32_bits((unsigned long)(nesqp)))); - nesqp->nesqp_context->ird_ord_sizes = cpu_to_le32(NES_QPCONTEXT_ORDIRD_ALSMM | - NES_QPCONTEXT_ORDIRD_AAH | - ((((u32)nesadapter->max_irrq_wr) << - NES_QPCONTEXT_ORDIRD_IRDSIZE_SHIFT) & NES_QPCONTEXT_ORDIRD_IRDSIZE_MASK)); - if (disable_mpa_crc) { - nes_debug(NES_DBG_QP, "Disabling MPA crc checking due to module option.\n"); - nesqp->nesqp_context->ird_ord_sizes |= cpu_to_le32(NES_QPCONTEXT_ORDIRD_RNMC); - } - - - /* Create the QP */ - cqp_request = nes_get_cqp_request(nesdev); - if (cqp_request == NULL) { - nes_debug(NES_DBG_QP, "Failed to get a cqp_request\n"); - nes_free_resource(nesadapter, nesadapter->allocated_qps, qp_num); - nes_free_qp_mem(nesdev, nesqp,virt_wqs); - kfree(nesqp->allocated_buffer); - return ERR_PTR(-ENOMEM); - } - cqp_request->waiting = 1; - cqp_wqe = &cqp_request->cqp_wqe; - - if (!virt_wqs) { - opcode = NES_CQP_CREATE_QP | NES_CQP_QP_TYPE_IWARP | - NES_CQP_QP_IWARP_STATE_IDLE; - } else { - opcode = NES_CQP_CREATE_QP | NES_CQP_QP_TYPE_IWARP | NES_CQP_QP_VIRT_WQS | - NES_CQP_QP_IWARP_STATE_IDLE; - } - opcode |= NES_CQP_QP_CQS_VALID; - nes_fill_init_cqp_wqe(cqp_wqe, nesdev); - set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX, opcode); - set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX, nesqp->hwqp.qp_id); - - u64temp = (u64)nesqp->nesqp_context_pbase; - set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_QP_WQE_CONTEXT_LOW_IDX, u64temp); - - atomic_set(&cqp_request->refcount, 2); - nes_post_cqp_request(nesdev, cqp_request); - - /* Wait for CQP */ - nes_debug(NES_DBG_QP, "Waiting for create iWARP QP%u to complete.\n", - nesqp->hwqp.qp_id); - ret = wait_event_timeout(cqp_request->waitq, - (cqp_request->request_done != 0), NES_EVENT_TIMEOUT); - nes_debug(NES_DBG_QP, "Create iwarp QP%u completed, wait_event_timeout ret=%u," - " nesdev->cqp_head = %u, nesdev->cqp.sq_tail = %u," - " CQP Major:Minor codes = 0x%04X:0x%04X.\n", - nesqp->hwqp.qp_id, ret, nesdev->cqp.sq_head, nesdev->cqp.sq_tail, - cqp_request->major_code, cqp_request->minor_code); - if ((!ret) || (cqp_request->major_code)) { - nes_put_cqp_request(nesdev, cqp_request); - nes_free_resource(nesadapter, nesadapter->allocated_qps, qp_num); - nes_free_qp_mem(nesdev, nesqp,virt_wqs); - kfree(nesqp->allocated_buffer); - if (!ret) { - return ERR_PTR(-ETIME); - } else { - return ERR_PTR(-EIO); - } - } - - nes_put_cqp_request(nesdev, cqp_request); - - if (udata) { - uresp.mmap_sq_db_index = nesqp->mmap_sq_db_index; - uresp.mmap_rq_db_index = 0; - uresp.actual_sq_size = sq_size; - uresp.actual_rq_size = rq_size; - uresp.qp_id = nesqp->hwqp.qp_id; - uresp.nes_drv_opt = nes_drv_opt; - if (ib_copy_to_udata(udata, &uresp, sizeof uresp)) { - nes_free_resource(nesadapter, nesadapter->allocated_qps, qp_num); - nes_free_qp_mem(nesdev, nesqp,virt_wqs); - kfree(nesqp->allocated_buffer); - return ERR_PTR(-EFAULT); - } - } - - nes_debug(NES_DBG_QP, "QP%u structure located @%p.Size = %u.\n", - nesqp->hwqp.qp_id, nesqp, (u32)sizeof(*nesqp)); - spin_lock_init(&nesqp->lock); - nes_add_ref(&nesqp->ibqp); - break; - default: - nes_debug(NES_DBG_QP, "Invalid QP type: %d\n", init_attr->qp_type); - return ERR_PTR(-EINVAL); - } - init_completion(&nesqp->sq_drained); - init_completion(&nesqp->rq_drained); - - nesqp->sig_all = (init_attr->sq_sig_type == IB_SIGNAL_ALL_WR); - timer_setup(&nesqp->terminate_timer, nes_terminate_timeout, 0); - - /* update the QP table */ - nesdev->nesadapter->qp_table[nesqp->hwqp.qp_id-NES_FIRST_QPN] = nesqp; - nes_debug(NES_DBG_QP, "netdev refcnt=%u\n", - netdev_refcnt_read(nesvnic->netdev)); - - return &nesqp->ibqp; -} - -/** - * nes_clean_cq - */ -static void nes_clean_cq(struct nes_qp *nesqp, struct nes_cq *nescq) -{ - u32 cq_head; - u32 lo; - u32 hi; - u64 u64temp; - unsigned long flags = 0; - - spin_lock_irqsave(&nescq->lock, flags); - - cq_head = nescq->hw_cq.cq_head; - while (le32_to_cpu(nescq->hw_cq.cq_vbase[cq_head].cqe_words[NES_CQE_OPCODE_IDX]) & NES_CQE_VALID) { - rmb(); - lo = le32_to_cpu(nescq->hw_cq.cq_vbase[cq_head].cqe_words[NES_CQE_COMP_COMP_CTX_LOW_IDX]); - hi = le32_to_cpu(nescq->hw_cq.cq_vbase[cq_head].cqe_words[NES_CQE_COMP_COMP_CTX_HIGH_IDX]); - u64temp = (((u64)hi) << 32) | ((u64)lo); - u64temp &= ~(NES_SW_CONTEXT_ALIGN-1); - if (u64temp == (u64)(unsigned long)nesqp) { - /* Zero the context value so cqe will be ignored */ - nescq->hw_cq.cq_vbase[cq_head].cqe_words[NES_CQE_COMP_COMP_CTX_LOW_IDX] = 0; - nescq->hw_cq.cq_vbase[cq_head].cqe_words[NES_CQE_COMP_COMP_CTX_HIGH_IDX] = 0; - } - - if (++cq_head >= nescq->hw_cq.cq_size) - cq_head = 0; - } - - spin_unlock_irqrestore(&nescq->lock, flags); -} - - -/** - * nes_destroy_qp - */ -static int nes_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata) -{ - struct nes_qp *nesqp = to_nesqp(ibqp); - struct nes_ucontext *nes_ucontext; - struct ib_qp_attr attr; - struct iw_cm_id *cm_id; - struct iw_cm_event cm_event; - int ret = 0; - - atomic_inc(&sw_qps_destroyed); - nesqp->destroyed = 1; - - /* Blow away the connection if it exists. */ - if (nesqp->ibqp_state >= IB_QPS_INIT && nesqp->ibqp_state <= IB_QPS_RTS) { - /* if (nesqp->ibqp_state == IB_QPS_RTS) { */ - attr.qp_state = IB_QPS_ERR; - nes_modify_qp(&nesqp->ibqp, &attr, IB_QP_STATE, NULL); - } - - if (((nesqp->ibqp_state == IB_QPS_INIT) || - (nesqp->ibqp_state == IB_QPS_RTR)) && (nesqp->cm_id)) { - cm_id = nesqp->cm_id; - cm_event.event = IW_CM_EVENT_CONNECT_REPLY; - cm_event.status = -ETIMEDOUT; - cm_event.local_addr = cm_id->local_addr; - cm_event.remote_addr = cm_id->remote_addr; - cm_event.private_data = NULL; - cm_event.private_data_len = 0; - - nes_debug(NES_DBG_QP, "Generating a CM Timeout Event for " - "QP%u. cm_id = %p, refcount = %u. \n", - nesqp->hwqp.qp_id, cm_id, atomic_read(&nesqp->refcount)); - - cm_id->rem_ref(cm_id); - ret = cm_id->event_handler(cm_id, &cm_event); - if (ret) - nes_debug(NES_DBG_QP, "OFA CM event_handler returned, ret=%d\n", ret); - } - - if (nesqp->user_mode) { - if (udata) { - nes_ucontext = - rdma_udata_to_drv_context( - udata, - struct nes_ucontext, - ibucontext); - clear_bit(nesqp->mmap_sq_db_index, nes_ucontext->allocated_wqs); - nes_ucontext->mmap_nesqp[nesqp->mmap_sq_db_index] = NULL; - if (nes_ucontext->first_free_wq > nesqp->mmap_sq_db_index) { - nes_ucontext->first_free_wq = nesqp->mmap_sq_db_index; - } - } - if (nesqp->pbl_pbase && nesqp->sq_kmapped) { - nesqp->sq_kmapped = 0; - kunmap(nesqp->page); - } - } else { - /* Clean any pending completions from the cq(s) */ - if (nesqp->nesscq) - nes_clean_cq(nesqp, nesqp->nesscq); - - if ((nesqp->nesrcq) && (nesqp->nesrcq != nesqp->nesscq)) - nes_clean_cq(nesqp, nesqp->nesrcq); - } - nes_rem_ref(&nesqp->ibqp); - return 0; -} - - -/** - * nes_create_cq - */ -static struct ib_cq *nes_create_cq(struct ib_device *ibdev, - const struct ib_cq_init_attr *attr, - struct ib_udata *udata) -{ - int entries = attr->cqe; - u64 u64temp; - struct nes_vnic *nesvnic = to_nesvnic(ibdev); - struct nes_device *nesdev = nesvnic->nesdev; - struct nes_adapter *nesadapter = nesdev->nesadapter; - struct nes_cq *nescq; - struct nes_ucontext *nes_ucontext = NULL; - struct nes_cqp_request *cqp_request; - void *mem = NULL; - struct nes_hw_cqp_wqe *cqp_wqe; - struct nes_pbl *nespbl = NULL; - struct nes_create_cq_req req; - struct nes_create_cq_resp resp; - u32 cq_num = 0; - u32 opcode = 0; - u32 pbl_entries = 1; - int err; - unsigned long flags; - int ret; - - if (attr->flags) - return ERR_PTR(-EINVAL); - - if (entries > nesadapter->max_cqe) - return ERR_PTR(-EINVAL); - - err = nes_alloc_resource(nesadapter, nesadapter->allocated_cqs, - nesadapter->max_cq, &cq_num, &nesadapter->next_cq, NES_RESOURCE_CQ); - if (err) { - return ERR_PTR(err); - } - - nescq = kzalloc(sizeof(struct nes_cq), GFP_KERNEL); - if (!nescq) { - nes_free_resource(nesadapter, nesadapter->allocated_cqs, cq_num); - return ERR_PTR(-ENOMEM); - } - - nescq->hw_cq.cq_size = max(entries + 1, 5); - nescq->hw_cq.cq_number = cq_num; - nescq->ibcq.cqe = nescq->hw_cq.cq_size - 1; - - if (udata) { - struct nes_ucontext *nes_ucontext = rdma_udata_to_drv_context( - udata, struct nes_ucontext, ibucontext); - - if (ib_copy_from_udata(&req, udata, sizeof (struct nes_create_cq_req))) { - nes_free_resource(nesadapter, nesadapter->allocated_cqs, cq_num); - kfree(nescq); - return ERR_PTR(-EFAULT); - } - nesvnic->mcrq_ucontext = nes_ucontext; - nes_ucontext->mcrqf = req.mcrqf; - if (nes_ucontext->mcrqf) { - if (nes_ucontext->mcrqf & 0x80000000) - nescq->hw_cq.cq_number = nesvnic->nic.qp_id + 28 + 2 * ((nes_ucontext->mcrqf & 0xf) - 1); - else if (nes_ucontext->mcrqf & 0x40000000) - nescq->hw_cq.cq_number = nes_ucontext->mcrqf & 0xffff; - else - nescq->hw_cq.cq_number = nesvnic->mcrq_qp_id + nes_ucontext->mcrqf-1; - nescq->mcrqf = nes_ucontext->mcrqf; - nes_free_resource(nesadapter, nesadapter->allocated_cqs, cq_num); - } - nes_debug(NES_DBG_CQ, "CQ Virtual Address = %08lX, size = %u.\n", - (unsigned long)req.user_cq_buffer, entries); - err = 1; - list_for_each_entry(nespbl, &nes_ucontext->cq_reg_mem_list, list) { - if (nespbl->user_base == (unsigned long )req.user_cq_buffer) { - list_del(&nespbl->list); - err = 0; - nes_debug(NES_DBG_CQ, "Found PBL for virtual CQ. nespbl=%p.\n", - nespbl); - break; - } - } - if (err) { - nes_free_resource(nesadapter, nesadapter->allocated_cqs, cq_num); - kfree(nescq); - return ERR_PTR(-EFAULT); - } - - pbl_entries = nespbl->pbl_size >> 3; - nescq->cq_mem_size = 0; - } else { - nescq->cq_mem_size = nescq->hw_cq.cq_size * sizeof(struct nes_hw_cqe); - nes_debug(NES_DBG_CQ, "Attempting to allocate pci memory (%u entries, %u bytes) for CQ%u.\n", - entries, nescq->cq_mem_size, nescq->hw_cq.cq_number); - - /* allocate the physical buffer space */ - mem = pci_zalloc_consistent(nesdev->pcidev, nescq->cq_mem_size, - &nescq->hw_cq.cq_pbase); - if (!mem) { - printk(KERN_ERR PFX "Unable to allocate pci memory for cq\n"); - nes_free_resource(nesadapter, nesadapter->allocated_cqs, cq_num); - kfree(nescq); - return ERR_PTR(-ENOMEM); - } - - nescq->hw_cq.cq_vbase = mem; - nescq->hw_cq.cq_head = 0; - nes_debug(NES_DBG_CQ, "CQ%u virtual address @ %p, phys = 0x%08X\n", - nescq->hw_cq.cq_number, nescq->hw_cq.cq_vbase, - (u32)nescq->hw_cq.cq_pbase); - } - - nescq->hw_cq.ce_handler = nes_iwarp_ce_handler; - spin_lock_init(&nescq->lock); - - /* send CreateCQ request to CQP */ - cqp_request = nes_get_cqp_request(nesdev); - if (cqp_request == NULL) { - nes_debug(NES_DBG_CQ, "Failed to get a cqp_request.\n"); - if (!udata) - pci_free_consistent(nesdev->pcidev, nescq->cq_mem_size, mem, - nescq->hw_cq.cq_pbase); - else { - pci_free_consistent(nesdev->pcidev, nespbl->pbl_size, - nespbl->pbl_vbase, nespbl->pbl_pbase); - kfree(nespbl); - } - - nes_free_resource(nesadapter, nesadapter->allocated_cqs, cq_num); - kfree(nescq); - return ERR_PTR(-ENOMEM); - } - cqp_request->waiting = 1; - cqp_wqe = &cqp_request->cqp_wqe; - - opcode = NES_CQP_CREATE_CQ | NES_CQP_CQ_CEQ_VALID | - NES_CQP_CQ_CHK_OVERFLOW | - NES_CQP_CQ_CEQE_MASK | ((u32)nescq->hw_cq.cq_size << 16); - - spin_lock_irqsave(&nesadapter->pbl_lock, flags); - - if (pbl_entries != 1) { - if (pbl_entries > 32) { - /* use 4k pbl */ - nes_debug(NES_DBG_CQ, "pbl_entries=%u, use a 4k PBL\n", pbl_entries); - if (nesadapter->free_4kpbl == 0) { - spin_unlock_irqrestore(&nesadapter->pbl_lock, flags); - nes_free_cqp_request(nesdev, cqp_request); - if (!udata) - pci_free_consistent(nesdev->pcidev, nescq->cq_mem_size, mem, - nescq->hw_cq.cq_pbase); - else { - pci_free_consistent(nesdev->pcidev, nespbl->pbl_size, - nespbl->pbl_vbase, nespbl->pbl_pbase); - kfree(nespbl); - } - nes_free_resource(nesadapter, nesadapter->allocated_cqs, cq_num); - kfree(nescq); - return ERR_PTR(-ENOMEM); - } else { - opcode |= (NES_CQP_CQ_VIRT | NES_CQP_CQ_4KB_CHUNK); - nescq->virtual_cq = 2; - nesadapter->free_4kpbl--; - } - } else { - /* use 256 byte pbl */ - nes_debug(NES_DBG_CQ, "pbl_entries=%u, use a 256 byte PBL\n", pbl_entries); - if (nesadapter->free_256pbl == 0) { - spin_unlock_irqrestore(&nesadapter->pbl_lock, flags); - nes_free_cqp_request(nesdev, cqp_request); - if (!udata) - pci_free_consistent(nesdev->pcidev, nescq->cq_mem_size, mem, - nescq->hw_cq.cq_pbase); - else { - pci_free_consistent(nesdev->pcidev, nespbl->pbl_size, - nespbl->pbl_vbase, nespbl->pbl_pbase); - kfree(nespbl); - } - nes_free_resource(nesadapter, nesadapter->allocated_cqs, cq_num); - kfree(nescq); - return ERR_PTR(-ENOMEM); - } else { - opcode |= NES_CQP_CQ_VIRT; - nescq->virtual_cq = 1; - nesadapter->free_256pbl--; - } - } - } - - spin_unlock_irqrestore(&nesadapter->pbl_lock, flags); - - nes_fill_init_cqp_wqe(cqp_wqe, nesdev); - set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX, opcode); - set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX, - (nescq->hw_cq.cq_number | ((u32)nesdev->ceq_index << 16))); - - if (udata) { - if (pbl_entries != 1) - u64temp = (u64)nespbl->pbl_pbase; - else - u64temp = le64_to_cpu(nespbl->pbl_vbase[0]); - set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_DOORBELL_INDEX_HIGH_IDX, - nes_ucontext->mmap_db_index[0]); - } else { - u64temp = (u64)nescq->hw_cq.cq_pbase; - cqp_wqe->wqe_words[NES_CQP_CQ_WQE_DOORBELL_INDEX_HIGH_IDX] = 0; - } - set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_CQ_WQE_PBL_LOW_IDX, u64temp); - cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] = 0; - u64temp = (u64)(unsigned long)&nescq->hw_cq; - cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_LOW_IDX] = - cpu_to_le32((u32)(u64temp >> 1)); - cqp_wqe->wqe_words[NES_CQP_CQ_WQE_CQ_CONTEXT_HIGH_IDX] = - cpu_to_le32(((u32)((u64temp) >> 33)) & 0x7FFFFFFF); - - atomic_set(&cqp_request->refcount, 2); - nes_post_cqp_request(nesdev, cqp_request); - - /* Wait for CQP */ - nes_debug(NES_DBG_CQ, "Waiting for create iWARP CQ%u to complete.\n", - nescq->hw_cq.cq_number); - ret = wait_event_timeout(cqp_request->waitq, (0 != cqp_request->request_done), - NES_EVENT_TIMEOUT * 2); - nes_debug(NES_DBG_CQ, "Create iWARP CQ%u completed, wait_event_timeout ret = %d.\n", - nescq->hw_cq.cq_number, ret); - if ((!ret) || (cqp_request->major_code)) { - nes_put_cqp_request(nesdev, cqp_request); - if (!udata) - pci_free_consistent(nesdev->pcidev, nescq->cq_mem_size, mem, - nescq->hw_cq.cq_pbase); - else { - pci_free_consistent(nesdev->pcidev, nespbl->pbl_size, - nespbl->pbl_vbase, nespbl->pbl_pbase); - kfree(nespbl); - } - nes_free_resource(nesadapter, nesadapter->allocated_cqs, cq_num); - kfree(nescq); - return ERR_PTR(-EIO); - } - nes_put_cqp_request(nesdev, cqp_request); - - if (udata) { - /* free the nespbl */ - pci_free_consistent(nesdev->pcidev, nespbl->pbl_size, nespbl->pbl_vbase, - nespbl->pbl_pbase); - kfree(nespbl); - resp.cq_id = nescq->hw_cq.cq_number; - resp.cq_size = nescq->hw_cq.cq_size; - resp.mmap_db_index = 0; - if (ib_copy_to_udata(udata, &resp, sizeof resp - sizeof resp.reserved)) { - nes_free_resource(nesadapter, nesadapter->allocated_cqs, cq_num); - kfree(nescq); - return ERR_PTR(-EFAULT); - } - } - - return &nescq->ibcq; -} - - -/** - * nes_destroy_cq - */ -static int nes_destroy_cq(struct ib_cq *ib_cq, struct ib_udata *udata) -{ - struct nes_cq *nescq; - struct nes_device *nesdev; - struct nes_vnic *nesvnic; - struct nes_adapter *nesadapter; - struct nes_hw_cqp_wqe *cqp_wqe; - struct nes_cqp_request *cqp_request; - unsigned long flags; - u32 opcode = 0; - int ret; - - if (ib_cq == NULL) - return 0; - - nescq = to_nescq(ib_cq); - nesvnic = to_nesvnic(ib_cq->device); - nesdev = nesvnic->nesdev; - nesadapter = nesdev->nesadapter; - - nes_debug(NES_DBG_CQ, "Destroy CQ%u\n", nescq->hw_cq.cq_number); - - /* Send DestroyCQ request to CQP */ - cqp_request = nes_get_cqp_request(nesdev); - if (cqp_request == NULL) { - nes_debug(NES_DBG_CQ, "Failed to get a cqp_request.\n"); - return -ENOMEM; - } - cqp_request->waiting = 1; - cqp_wqe = &cqp_request->cqp_wqe; - opcode = NES_CQP_DESTROY_CQ | (nescq->hw_cq.cq_size << 16); - spin_lock_irqsave(&nesadapter->pbl_lock, flags); - if (nescq->virtual_cq == 1) { - nesadapter->free_256pbl++; - if (nesadapter->free_256pbl > nesadapter->max_256pbl) { - printk(KERN_ERR PFX "%s: free 256B PBLs(%u) has exceeded the max(%u)\n", - __func__, nesadapter->free_256pbl, nesadapter->max_256pbl); - } - } else if (nescq->virtual_cq == 2) { - nesadapter->free_4kpbl++; - if (nesadapter->free_4kpbl > nesadapter->max_4kpbl) { - printk(KERN_ERR PFX "%s: free 4K PBLs(%u) has exceeded the max(%u)\n", - __func__, nesadapter->free_4kpbl, nesadapter->max_4kpbl); - } - opcode |= NES_CQP_CQ_4KB_CHUNK; - } - - spin_unlock_irqrestore(&nesadapter->pbl_lock, flags); - - nes_fill_init_cqp_wqe(cqp_wqe, nesdev); - set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX, opcode); - set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX, - (nescq->hw_cq.cq_number | ((u32)PCI_FUNC(nesdev->pcidev->devfn) << 16))); - if (!nescq->mcrqf) - nes_free_resource(nesadapter, nesadapter->allocated_cqs, nescq->hw_cq.cq_number); - - atomic_set(&cqp_request->refcount, 2); - nes_post_cqp_request(nesdev, cqp_request); - - /* Wait for CQP */ - nes_debug(NES_DBG_CQ, "Waiting for destroy iWARP CQ%u to complete.\n", - nescq->hw_cq.cq_number); - ret = wait_event_timeout(cqp_request->waitq, (0 != cqp_request->request_done), - NES_EVENT_TIMEOUT); - nes_debug(NES_DBG_CQ, "Destroy iWARP CQ%u completed, wait_event_timeout ret = %u," - " CQP Major:Minor codes = 0x%04X:0x%04X.\n", - nescq->hw_cq.cq_number, ret, cqp_request->major_code, - cqp_request->minor_code); - if (!ret) { - nes_debug(NES_DBG_CQ, "iWARP CQ%u destroy timeout expired\n", - nescq->hw_cq.cq_number); - ret = -ETIME; - } else if (cqp_request->major_code) { - nes_debug(NES_DBG_CQ, "iWARP CQ%u destroy failed\n", - nescq->hw_cq.cq_number); - ret = -EIO; - } else { - ret = 0; - } - nes_put_cqp_request(nesdev, cqp_request); - - if (nescq->cq_mem_size) - pci_free_consistent(nesdev->pcidev, nescq->cq_mem_size, - nescq->hw_cq.cq_vbase, nescq->hw_cq.cq_pbase); - kfree(nescq); - - return ret; -} - -/** - * root_256 - */ -static u32 root_256(struct nes_device *nesdev, - struct nes_root_vpbl *root_vpbl, - struct nes_root_vpbl *new_root, - u16 pbl_count_4k) -{ - u64 leaf_pbl; - int i, j, k; - - if (pbl_count_4k == 1) { - new_root->pbl_vbase = pci_alloc_consistent(nesdev->pcidev, - 512, &new_root->pbl_pbase); - - if (new_root->pbl_vbase == NULL) - return 0; - - leaf_pbl = (u64)root_vpbl->pbl_pbase; - for (i = 0; i < 16; i++) { - new_root->pbl_vbase[i].pa_low = - cpu_to_le32((u32)leaf_pbl); - new_root->pbl_vbase[i].pa_high = - cpu_to_le32((u32)((((u64)leaf_pbl) >> 32))); - leaf_pbl += 256; - } - } else { - for (i = 3; i >= 0; i--) { - j = i * 16; - root_vpbl->pbl_vbase[j] = root_vpbl->pbl_vbase[i]; - leaf_pbl = le32_to_cpu(root_vpbl->pbl_vbase[j].pa_low) + - (((u64)le32_to_cpu(root_vpbl->pbl_vbase[j].pa_high)) - << 32); - for (k = 1; k < 16; k++) { - leaf_pbl += 256; - root_vpbl->pbl_vbase[j + k].pa_low = - cpu_to_le32((u32)leaf_pbl); - root_vpbl->pbl_vbase[j + k].pa_high = - cpu_to_le32((u32)((((u64)leaf_pbl) >> 32))); - } - } - } - - return 1; -} - - -/** - * nes_reg_mr - */ -static int nes_reg_mr(struct nes_device *nesdev, struct nes_pd *nespd, - u32 stag, u64 region_length, struct nes_root_vpbl *root_vpbl, - dma_addr_t single_buffer, u16 pbl_count_4k, - u16 residual_page_count_4k, int acc, u64 *iova_start, - u16 *actual_pbl_cnt, u8 *used_4k_pbls) -{ - struct nes_hw_cqp_wqe *cqp_wqe; - struct nes_cqp_request *cqp_request; - unsigned long flags; - int ret; - struct nes_adapter *nesadapter = nesdev->nesadapter; - uint pg_cnt = 0; - u16 pbl_count_256 = 0; - u16 pbl_count = 0; - u8 use_256_pbls = 0; - u8 use_4k_pbls = 0; - u16 use_two_level = (pbl_count_4k > 1) ? 1 : 0; - struct nes_root_vpbl new_root = { 0, NULL, NULL }; - u32 opcode = 0; - u16 major_code; - - /* Register the region with the adapter */ - cqp_request = nes_get_cqp_request(nesdev); - if (cqp_request == NULL) { - nes_debug(NES_DBG_MR, "Failed to get a cqp_request.\n"); - return -ENOMEM; - } - cqp_request->waiting = 1; - cqp_wqe = &cqp_request->cqp_wqe; - - if (pbl_count_4k) { - spin_lock_irqsave(&nesadapter->pbl_lock, flags); - - pg_cnt = ((pbl_count_4k - 1) * 512) + residual_page_count_4k; - pbl_count_256 = (pg_cnt + 31) / 32; - if (pg_cnt <= 32) { - if (pbl_count_256 <= nesadapter->free_256pbl) - use_256_pbls = 1; - else if (pbl_count_4k <= nesadapter->free_4kpbl) - use_4k_pbls = 1; - } else if (pg_cnt <= 2048) { - if (((pbl_count_4k + use_two_level) <= nesadapter->free_4kpbl) && - (nesadapter->free_4kpbl > (nesadapter->max_4kpbl >> 1))) { - use_4k_pbls = 1; - } else if ((pbl_count_256 + 1) <= nesadapter->free_256pbl) { - use_256_pbls = 1; - use_two_level = 1; - } else if ((pbl_count_4k + use_two_level) <= nesadapter->free_4kpbl) { - use_4k_pbls = 1; - } - } else { - if ((pbl_count_4k + 1) <= nesadapter->free_4kpbl) - use_4k_pbls = 1; - } - - if (use_256_pbls) { - pbl_count = pbl_count_256; - nesadapter->free_256pbl -= pbl_count + use_two_level; - } else if (use_4k_pbls) { - pbl_count = pbl_count_4k; - nesadapter->free_4kpbl -= pbl_count + use_two_level; - } else { - spin_unlock_irqrestore(&nesadapter->pbl_lock, flags); - nes_debug(NES_DBG_MR, "Out of Pbls\n"); - nes_free_cqp_request(nesdev, cqp_request); - return -ENOMEM; - } - - spin_unlock_irqrestore(&nesadapter->pbl_lock, flags); - } - - if (use_256_pbls && use_two_level) { - if (root_256(nesdev, root_vpbl, &new_root, pbl_count_4k) == 1) { - if (new_root.pbl_pbase != 0) - root_vpbl = &new_root; - } else { - spin_lock_irqsave(&nesadapter->pbl_lock, flags); - nesadapter->free_256pbl += pbl_count_256 + use_two_level; - use_256_pbls = 0; - - if (pbl_count_4k == 1) - use_two_level = 0; - pbl_count = pbl_count_4k; - - if ((pbl_count_4k + use_two_level) <= nesadapter->free_4kpbl) { - nesadapter->free_4kpbl -= pbl_count + use_two_level; - use_4k_pbls = 1; - } - spin_unlock_irqrestore(&nesadapter->pbl_lock, flags); - - if (use_4k_pbls == 0) - return -ENOMEM; - } - } - - opcode = NES_CQP_REGISTER_STAG | NES_CQP_STAG_RIGHTS_LOCAL_READ | - NES_CQP_STAG_VA_TO | NES_CQP_STAG_MR; - if (acc & IB_ACCESS_LOCAL_WRITE) - opcode |= NES_CQP_STAG_RIGHTS_LOCAL_WRITE; - if (acc & IB_ACCESS_REMOTE_WRITE) - opcode |= NES_CQP_STAG_RIGHTS_REMOTE_WRITE | NES_CQP_STAG_REM_ACC_EN; - if (acc & IB_ACCESS_REMOTE_READ) - opcode |= NES_CQP_STAG_RIGHTS_REMOTE_READ | NES_CQP_STAG_REM_ACC_EN; - if (acc & IB_ACCESS_MW_BIND) - opcode |= NES_CQP_STAG_RIGHTS_WINDOW_BIND | NES_CQP_STAG_REM_ACC_EN; - - nes_fill_init_cqp_wqe(cqp_wqe, nesdev); - set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX, opcode); - set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_STAG_WQE_VA_LOW_IDX, *iova_start); - set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_STAG_WQE_LEN_LOW_IDX, region_length); - - cqp_wqe->wqe_words[NES_CQP_STAG_WQE_LEN_HIGH_PD_IDX] = - cpu_to_le32((u32)(region_length >> 8) & 0xff000000); - cqp_wqe->wqe_words[NES_CQP_STAG_WQE_LEN_HIGH_PD_IDX] |= - cpu_to_le32(nespd->pd_id & 0x00007fff); - set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_STAG_WQE_STAG_IDX, stag); - - if (pbl_count == 0) { - set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_STAG_WQE_PA_LOW_IDX, single_buffer); - } else { - set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_STAG_WQE_PA_LOW_IDX, root_vpbl->pbl_pbase); - set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_STAG_WQE_PBL_BLK_COUNT_IDX, pbl_count); - set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_STAG_WQE_PBL_LEN_IDX, (pg_cnt * 8)); - - if (use_4k_pbls) - cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX] |= cpu_to_le32(NES_CQP_STAG_PBL_BLK_SIZE); - } - barrier(); - - atomic_set(&cqp_request->refcount, 2); - nes_post_cqp_request(nesdev, cqp_request); - - /* Wait for CQP */ - ret = wait_event_timeout(cqp_request->waitq, (0 != cqp_request->request_done), - NES_EVENT_TIMEOUT); - nes_debug(NES_DBG_MR, "Register STag 0x%08X completed, wait_event_timeout ret = %u," - " CQP Major:Minor codes = 0x%04X:0x%04X.\n", - stag, ret, cqp_request->major_code, cqp_request->minor_code); - major_code = cqp_request->major_code; - nes_put_cqp_request(nesdev, cqp_request); - - if ((!ret || major_code) && pbl_count != 0) { - spin_lock_irqsave(&nesadapter->pbl_lock, flags); - if (use_256_pbls) - nesadapter->free_256pbl += pbl_count + use_two_level; - else if (use_4k_pbls) - nesadapter->free_4kpbl += pbl_count + use_two_level; - spin_unlock_irqrestore(&nesadapter->pbl_lock, flags); - } - if (new_root.pbl_pbase) - pci_free_consistent(nesdev->pcidev, 512, new_root.pbl_vbase, - new_root.pbl_pbase); - - if (!ret) - return -ETIME; - else if (major_code) - return -EIO; - - *actual_pbl_cnt = pbl_count + use_two_level; - *used_4k_pbls = use_4k_pbls; - return 0; -} - - -/** - * nes_reg_phys_mr - */ -struct ib_mr *nes_reg_phys_mr(struct ib_pd *ib_pd, u64 addr, u64 size, - int acc, u64 *iova_start) -{ - u64 region_length; - struct nes_pd *nespd = to_nespd(ib_pd); - struct nes_vnic *nesvnic = to_nesvnic(ib_pd->device); - struct nes_device *nesdev = nesvnic->nesdev; - struct nes_adapter *nesadapter = nesdev->nesadapter; - struct nes_mr *nesmr; - struct ib_mr *ibmr; - struct nes_vpbl vpbl; - struct nes_root_vpbl root_vpbl; - u32 stag; - unsigned long mask; - u32 stag_index = 0; - u32 next_stag_index = 0; - u32 driver_key = 0; - int err = 0; - int ret = 0; - u16 pbl_count = 0; - u8 single_page = 1; - u8 stag_key = 0; - - region_length = 0; - vpbl.pbl_vbase = NULL; - root_vpbl.pbl_vbase = NULL; - root_vpbl.pbl_pbase = 0; - - get_random_bytes(&next_stag_index, sizeof(next_stag_index)); - stag_key = (u8)next_stag_index; - - driver_key = 0; - - next_stag_index >>= 8; - next_stag_index %= nesadapter->max_mr; - - if ((addr ^ *iova_start) & ~PAGE_MASK) - return ERR_PTR(-EINVAL); - - err = nes_alloc_resource(nesadapter, nesadapter->allocated_mrs, nesadapter->max_mr, - &stag_index, &next_stag_index, NES_RESOURCE_PHYS_MR); - if (err) { - return ERR_PTR(err); - } - - nesmr = kzalloc(sizeof(*nesmr), GFP_KERNEL); - if (!nesmr) { - nes_free_resource(nesadapter, nesadapter->allocated_mrs, stag_index); - return ERR_PTR(-ENOMEM); - } - - /* Allocate a 4K buffer for the PBL */ - vpbl.pbl_vbase = pci_alloc_consistent(nesdev->pcidev, 4096, - &vpbl.pbl_pbase); - nes_debug(NES_DBG_MR, "Allocating leaf PBL, va = %p, pa = 0x%016lX\n", - vpbl.pbl_vbase, (unsigned long)vpbl.pbl_pbase); - if (!vpbl.pbl_vbase) { - nes_free_resource(nesadapter, nesadapter->allocated_mrs, stag_index); - ibmr = ERR_PTR(-ENOMEM); - kfree(nesmr); - goto reg_phys_err; - } - - - mask = !size; - - if (mask & ~PAGE_MASK) { - nes_free_resource(nesadapter, nesadapter->allocated_mrs, stag_index); - nes_debug(NES_DBG_MR, "Invalid buffer addr or size\n"); - ibmr = ERR_PTR(-EINVAL); - kfree(nesmr); - goto reg_phys_err; - } - - region_length += size; - vpbl.pbl_vbase[0].pa_low = cpu_to_le32((u32)addr & PAGE_MASK); - vpbl.pbl_vbase[0].pa_high = cpu_to_le32((u32)((((u64)addr) >> 32))); - - stag = stag_index << 8; - stag |= driver_key; - stag += (u32)stag_key; - - nes_debug(NES_DBG_MR, "Registering STag 0x%08X, VA = 0x%016lX," - " length = 0x%016lX, index = 0x%08X\n", - stag, (unsigned long)*iova_start, (unsigned long)region_length, stag_index); - - /* Make the leaf PBL the root if only one PBL */ - root_vpbl.pbl_pbase = vpbl.pbl_pbase; - - if (single_page) { - pbl_count = 0; - } else { - pbl_count = 1; - } - ret = nes_reg_mr(nesdev, nespd, stag, region_length, &root_vpbl, - addr, pbl_count, 1, acc, iova_start, - &nesmr->pbls_used, &nesmr->pbl_4k); - - if (ret == 0) { - nesmr->ibmr.rkey = stag; - nesmr->ibmr.lkey = stag; - nesmr->mode = IWNES_MEMREG_TYPE_MEM; - ibmr = &nesmr->ibmr; - } else { - kfree(nesmr); - ibmr = ERR_PTR(-ENOMEM); - } - -reg_phys_err: - /* single PBL case */ - pci_free_consistent(nesdev->pcidev, 4096, vpbl.pbl_vbase, vpbl.pbl_pbase); - return ibmr; -} - - -/** - * nes_get_dma_mr - */ -static struct ib_mr *nes_get_dma_mr(struct ib_pd *pd, int acc) -{ - u64 kva = 0; - - nes_debug(NES_DBG_MR, "\n"); - - return nes_reg_phys_mr(pd, 0, 0xffffffffffULL, acc, &kva); -} - -/** - * nes_reg_user_mr - */ -static struct ib_mr *nes_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, - u64 virt, int acc, struct ib_udata *udata) -{ - u64 iova_start; - __le64 *pbl; - u64 region_length; - dma_addr_t last_dma_addr = 0; - dma_addr_t first_dma_addr = 0; - struct nes_pd *nespd = to_nespd(pd); - struct nes_vnic *nesvnic = to_nesvnic(pd->device); - struct nes_device *nesdev = nesvnic->nesdev; - struct nes_adapter *nesadapter = nesdev->nesadapter; - struct ib_mr *ibmr = ERR_PTR(-EINVAL); - struct sg_dma_page_iter dma_iter; - struct nes_ucontext *nes_ucontext = rdma_udata_to_drv_context( - udata, struct nes_ucontext, ibucontext); - struct nes_pbl *nespbl; - struct nes_mr *nesmr; - struct ib_umem *region; - struct nes_mem_reg_req req; - struct nes_vpbl vpbl; - struct nes_root_vpbl root_vpbl; - int page_index; - int page_count = 0; - int err, pbl_depth = 0; - int ret; - u32 stag; - u32 stag_index = 0; - u32 next_stag_index; - u32 driver_key; - u32 root_pbl_index = 0; - u32 cur_pbl_index = 0; - u32 skip_pages; - u16 pbl_count; - u8 single_page = 1; - u8 stag_key; - - region = ib_umem_get(udata, start, length, acc, 0); - if (IS_ERR(region)) { - return (struct ib_mr *)region; - } - - nes_debug(NES_DBG_MR, "User base = 0x%lX, Virt base = 0x%lX, length = %u," - " offset = %u, page size = %lu.\n", - (unsigned long int)start, (unsigned long int)virt, (u32)length, - ib_umem_offset(region), BIT(region->page_shift)); - - skip_pages = ((u32)ib_umem_offset(region)) >> 12; - - if (ib_copy_from_udata(&req, udata, sizeof(req))) { - ib_umem_release(region); - return ERR_PTR(-EFAULT); - } - nes_debug(NES_DBG_MR, "Memory Registration type = %08X.\n", req.reg_type); - - switch (req.reg_type) { - case IWNES_MEMREG_TYPE_MEM: - pbl_depth = 0; - region_length = 0; - vpbl.pbl_vbase = NULL; - root_vpbl.pbl_vbase = NULL; - root_vpbl.pbl_pbase = 0; - - get_random_bytes(&next_stag_index, sizeof(next_stag_index)); - stag_key = (u8)next_stag_index; - - driver_key = next_stag_index & 0x70000000; - - next_stag_index >>= 8; - next_stag_index %= nesadapter->max_mr; - - err = nes_alloc_resource(nesadapter, nesadapter->allocated_mrs, - nesadapter->max_mr, &stag_index, &next_stag_index, NES_RESOURCE_USER_MR); - if (err) { - ib_umem_release(region); - return ERR_PTR(err); - } - - nesmr = kzalloc(sizeof(*nesmr), GFP_KERNEL); - if (!nesmr) { - ib_umem_release(region); - nes_free_resource(nesadapter, nesadapter->allocated_mrs, stag_index); - return ERR_PTR(-ENOMEM); - } - nesmr->region = region; - - for_each_sg_dma_page (region->sg_head.sgl, &dma_iter, region->nmap, 0) { - - region_length += PAGE_SIZE; - region_length -= skip_pages << 12; - skip_pages = 0; - if ((page_count != 0) && (page_count << 12) - (ib_umem_offset(region) & (4096 - 1)) >= region->length) - goto enough_pages; - if ((page_count & 0x01FF) == 0) { - if (page_count >= 1024 * 512) { - ib_umem_release(region); - nes_free_resource(nesadapter, - nesadapter->allocated_mrs, stag_index); - kfree(nesmr); - ibmr = ERR_PTR(-E2BIG); - goto reg_user_mr_err; - } - if (root_pbl_index == 1) { - root_vpbl.pbl_vbase = pci_alloc_consistent(nesdev->pcidev, - 8192, &root_vpbl.pbl_pbase); - nes_debug(NES_DBG_MR, "Allocating root PBL, va = %p, pa = 0x%08X\n", - root_vpbl.pbl_vbase, (unsigned int)root_vpbl.pbl_pbase); - if (!root_vpbl.pbl_vbase) { - ib_umem_release(region); - pci_free_consistent(nesdev->pcidev, 4096, vpbl.pbl_vbase, - vpbl.pbl_pbase); - nes_free_resource(nesadapter, nesadapter->allocated_mrs, - stag_index); - kfree(nesmr); - ibmr = ERR_PTR(-ENOMEM); - goto reg_user_mr_err; - } - root_vpbl.leaf_vpbl = kcalloc(1024, - sizeof(*root_vpbl.leaf_vpbl), - GFP_KERNEL); - if (!root_vpbl.leaf_vpbl) { - ib_umem_release(region); - pci_free_consistent(nesdev->pcidev, 8192, root_vpbl.pbl_vbase, - root_vpbl.pbl_pbase); - pci_free_consistent(nesdev->pcidev, 4096, vpbl.pbl_vbase, - vpbl.pbl_pbase); - nes_free_resource(nesadapter, nesadapter->allocated_mrs, - stag_index); - kfree(nesmr); - ibmr = ERR_PTR(-ENOMEM); - goto reg_user_mr_err; - } - root_vpbl.pbl_vbase[0].pa_low = - cpu_to_le32((u32)vpbl.pbl_pbase); - root_vpbl.pbl_vbase[0].pa_high = - cpu_to_le32((u32)((((u64)vpbl.pbl_pbase) >> 32))); - root_vpbl.leaf_vpbl[0] = vpbl; - } - vpbl.pbl_vbase = pci_alloc_consistent(nesdev->pcidev, 4096, - &vpbl.pbl_pbase); - nes_debug(NES_DBG_MR, "Allocating leaf PBL, va = %p, pa = 0x%08X\n", - vpbl.pbl_vbase, (unsigned int)vpbl.pbl_pbase); - if (!vpbl.pbl_vbase) { - ib_umem_release(region); - nes_free_resource(nesadapter, nesadapter->allocated_mrs, stag_index); - ibmr = ERR_PTR(-ENOMEM); - kfree(nesmr); - goto reg_user_mr_err; - } - if (1 <= root_pbl_index) { - root_vpbl.pbl_vbase[root_pbl_index].pa_low = - cpu_to_le32((u32)vpbl.pbl_pbase); - root_vpbl.pbl_vbase[root_pbl_index].pa_high = - cpu_to_le32((u32)((((u64)vpbl.pbl_pbase) >> 32))); - root_vpbl.leaf_vpbl[root_pbl_index] = vpbl; - } - root_pbl_index++; - cur_pbl_index = 0; - } - if (single_page) { - if (page_count != 0) { - if ((last_dma_addr + 4096) != sg_page_iter_dma_address(&dma_iter)) - single_page = 0; - last_dma_addr = sg_page_iter_dma_address(&dma_iter); - } else { - first_dma_addr = sg_page_iter_dma_address(&dma_iter); - last_dma_addr = first_dma_addr; - } - } - - vpbl.pbl_vbase[cur_pbl_index].pa_low = - cpu_to_le32((u32)(sg_page_iter_dma_address(&dma_iter))); - vpbl.pbl_vbase[cur_pbl_index].pa_high = - cpu_to_le32((u32)((u64)(sg_page_iter_dma_address(&dma_iter)))); - cur_pbl_index++; - page_count++; - } - -enough_pages: - nes_debug(NES_DBG_MR, "calculating stag, stag_index=0x%08x, driver_key=0x%08x," - " stag_key=0x%08x\n", - stag_index, driver_key, stag_key); - stag = stag_index << 8; - stag |= driver_key; - stag += (u32)stag_key; - - iova_start = virt; - /* Make the leaf PBL the root if only one PBL */ - if (root_pbl_index == 1) { - root_vpbl.pbl_pbase = vpbl.pbl_pbase; - } - - if (single_page) { - pbl_count = 0; - } else { - pbl_count = root_pbl_index; - first_dma_addr = 0; - } - nes_debug(NES_DBG_MR, "Registering STag 0x%08X, VA = 0x%08X, length = 0x%08X," - " index = 0x%08X, region->length=0x%08llx, pbl_count = %u\n", - stag, (unsigned int)iova_start, - (unsigned int)region_length, stag_index, - (unsigned long long)region->length, pbl_count); - ret = nes_reg_mr(nesdev, nespd, stag, region->length, &root_vpbl, - first_dma_addr, pbl_count, (u16)cur_pbl_index, acc, - &iova_start, &nesmr->pbls_used, &nesmr->pbl_4k); - - nes_debug(NES_DBG_MR, "ret=%d\n", ret); - - if (ret == 0) { - nesmr->ibmr.rkey = stag; - nesmr->ibmr.lkey = stag; - nesmr->mode = IWNES_MEMREG_TYPE_MEM; - ibmr = &nesmr->ibmr; - } else { - ib_umem_release(region); - kfree(nesmr); - ibmr = ERR_PTR(-ENOMEM); - } - -reg_user_mr_err: - /* free the resources */ - if (root_pbl_index == 1) { - pci_free_consistent(nesdev->pcidev, 4096, vpbl.pbl_vbase, - vpbl.pbl_pbase); - } else { - for (page_index=0; page_index<root_pbl_index; page_index++) { - pci_free_consistent(nesdev->pcidev, 4096, - root_vpbl.leaf_vpbl[page_index].pbl_vbase, - root_vpbl.leaf_vpbl[page_index].pbl_pbase); - } - kfree(root_vpbl.leaf_vpbl); - pci_free_consistent(nesdev->pcidev, 8192, root_vpbl.pbl_vbase, - root_vpbl.pbl_pbase); - } - - nes_debug(NES_DBG_MR, "Leaving, ibmr=%p", ibmr); - - return ibmr; - case IWNES_MEMREG_TYPE_QP: - case IWNES_MEMREG_TYPE_CQ: - if (!region->length) { - nes_debug(NES_DBG_MR, "Unable to register zero length region for CQ\n"); - ib_umem_release(region); - return ERR_PTR(-EINVAL); - } - nespbl = kzalloc(sizeof(*nespbl), GFP_KERNEL); - if (!nespbl) { - ib_umem_release(region); - return ERR_PTR(-ENOMEM); - } - nesmr = kzalloc(sizeof(*nesmr), GFP_KERNEL); - if (!nesmr) { - ib_umem_release(region); - kfree(nespbl); - return ERR_PTR(-ENOMEM); - } - nesmr->region = region; - pbl_depth = region->length >> 12; - pbl_depth += (region->length & (4096-1)) ? 1 : 0; - nespbl->pbl_size = pbl_depth*sizeof(u64); - if (req.reg_type == IWNES_MEMREG_TYPE_QP) { - nes_debug(NES_DBG_MR, "Attempting to allocate QP PBL memory"); - } else { - nes_debug(NES_DBG_MR, "Attempting to allocate CP PBL memory"); - } - - nes_debug(NES_DBG_MR, " %u bytes, %u entries.\n", - nespbl->pbl_size, pbl_depth); - pbl = pci_alloc_consistent(nesdev->pcidev, nespbl->pbl_size, - &nespbl->pbl_pbase); - if (!pbl) { - ib_umem_release(region); - kfree(nesmr); - kfree(nespbl); - nes_debug(NES_DBG_MR, "Unable to allocate PBL memory\n"); - return ERR_PTR(-ENOMEM); - } - - nespbl->pbl_vbase = (u64 *)pbl; - nespbl->user_base = start; - nes_debug(NES_DBG_MR, "Allocated PBL memory, %u bytes, pbl_pbase=%lx," - " pbl_vbase=%p user_base=0x%lx\n", - nespbl->pbl_size, (unsigned long) nespbl->pbl_pbase, - (void *) nespbl->pbl_vbase, nespbl->user_base); - - nespbl->page = sg_page(region->sg_head.sgl); - for_each_sg_dma_page(region->sg_head.sgl, &dma_iter, region->nmap, 0) { - ((__le32 *)pbl)[0] = cpu_to_le32((u32)(sg_page_iter_dma_address(&dma_iter))); - ((__le32 *)pbl)[1] = cpu_to_le32(((u64)(sg_page_iter_dma_address(&dma_iter)))>>32); - nes_debug(NES_DBG_MR, "pbl=%p, *pbl=0x%016llx, 0x%08x%08x\n", pbl, - (unsigned long long)*pbl, - le32_to_cpu(((__le32 *)pbl)[1]), le32_to_cpu(((__le32 *)pbl)[0])); - pbl++; - } - - if (req.reg_type == IWNES_MEMREG_TYPE_QP) { - list_add_tail(&nespbl->list, &nes_ucontext->qp_reg_mem_list); - } else { - list_add_tail(&nespbl->list, &nes_ucontext->cq_reg_mem_list); - } - nesmr->ibmr.rkey = -1; - nesmr->ibmr.lkey = -1; - nesmr->mode = req.reg_type; - return &nesmr->ibmr; - } - - ib_umem_release(region); - return ERR_PTR(-ENOSYS); -} - - -/** - * nes_dereg_mr - */ -static int nes_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata) -{ - struct nes_mr *nesmr = to_nesmr(ib_mr); - struct nes_vnic *nesvnic = to_nesvnic(ib_mr->device); - struct nes_device *nesdev = nesvnic->nesdev; - struct nes_adapter *nesadapter = nesdev->nesadapter; - struct nes_hw_cqp_wqe *cqp_wqe; - struct nes_cqp_request *cqp_request; - unsigned long flags; - int ret; - u16 major_code; - u16 minor_code; - - - if (nesmr->pages) - pci_free_consistent(nesdev->pcidev, - nesmr->max_pages * sizeof(u64), - nesmr->pages, - nesmr->paddr); - - if (nesmr->region) { - ib_umem_release(nesmr->region); - } - if (nesmr->mode != IWNES_MEMREG_TYPE_MEM) { - kfree(nesmr); - return 0; - } - - /* Deallocate the region with the adapter */ - - cqp_request = nes_get_cqp_request(nesdev); - if (cqp_request == NULL) { - nes_debug(NES_DBG_MR, "Failed to get a cqp_request.\n"); - return -ENOMEM; - } - cqp_request->waiting = 1; - cqp_wqe = &cqp_request->cqp_wqe; - - nes_fill_init_cqp_wqe(cqp_wqe, nesdev); - set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX, - NES_CQP_DEALLOCATE_STAG | NES_CQP_STAG_VA_TO | - NES_CQP_STAG_DEALLOC_PBLS | NES_CQP_STAG_MR); - set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_STAG_WQE_STAG_IDX, ib_mr->rkey); - - atomic_set(&cqp_request->refcount, 2); - nes_post_cqp_request(nesdev, cqp_request); - - /* Wait for CQP */ - nes_debug(NES_DBG_MR, "Waiting for deallocate STag 0x%08X completed\n", ib_mr->rkey); - ret = wait_event_timeout(cqp_request->waitq, (cqp_request->request_done != 0), - NES_EVENT_TIMEOUT); - nes_debug(NES_DBG_MR, "Deallocate STag 0x%08X completed, wait_event_timeout ret = %u," - " CQP Major:Minor codes = 0x%04X:0x%04X\n", - ib_mr->rkey, ret, cqp_request->major_code, cqp_request->minor_code); - - major_code = cqp_request->major_code; - minor_code = cqp_request->minor_code; - - nes_put_cqp_request(nesdev, cqp_request); - - if (!ret) { - nes_debug(NES_DBG_MR, "Timeout waiting to destroy STag," - " ib_mr=%p, rkey = 0x%08X\n", - ib_mr, ib_mr->rkey); - return -ETIME; - } else if (major_code) { - nes_debug(NES_DBG_MR, "Error (0x%04X:0x%04X) while attempting" - " to destroy STag, ib_mr=%p, rkey = 0x%08X\n", - major_code, minor_code, ib_mr, ib_mr->rkey); - return -EIO; - } - - if (nesmr->pbls_used != 0) { - spin_lock_irqsave(&nesadapter->pbl_lock, flags); - if (nesmr->pbl_4k) { - nesadapter->free_4kpbl += nesmr->pbls_used; - if (nesadapter->free_4kpbl > nesadapter->max_4kpbl) - printk(KERN_ERR PFX "free 4KB PBLs(%u) has " - "exceeded the max(%u)\n", - nesadapter->free_4kpbl, - nesadapter->max_4kpbl); - } else { - nesadapter->free_256pbl += nesmr->pbls_used; - if (nesadapter->free_256pbl > nesadapter->max_256pbl) - printk(KERN_ERR PFX "free 256B PBLs(%u) has " - "exceeded the max(%u)\n", - nesadapter->free_256pbl, - nesadapter->max_256pbl); - } - spin_unlock_irqrestore(&nesadapter->pbl_lock, flags); - } - nes_free_resource(nesadapter, nesadapter->allocated_mrs, - (ib_mr->rkey & 0x0fffff00) >> 8); - - kfree(nesmr); - - return 0; -} - - -/** - * show_rev - */ -static ssize_t hw_rev_show(struct device *dev, - struct device_attribute *attr, char *buf) -{ - struct nes_ib_device *nesibdev = - rdma_device_to_drv_device(dev, struct nes_ib_device, ibdev); - struct nes_vnic *nesvnic = nesibdev->nesvnic; - - nes_debug(NES_DBG_INIT, "\n"); - return sprintf(buf, "%x\n", nesvnic->nesdev->nesadapter->hw_rev); -} -static DEVICE_ATTR_RO(hw_rev); - -/** - * show_hca - */ -static ssize_t hca_type_show(struct device *dev, - struct device_attribute *attr, char *buf) -{ - nes_debug(NES_DBG_INIT, "\n"); - return sprintf(buf, "NES020\n"); -} -static DEVICE_ATTR_RO(hca_type); - -/** - * show_board - */ -static ssize_t board_id_show(struct device *dev, - struct device_attribute *attr, char *buf) -{ - nes_debug(NES_DBG_INIT, "\n"); - return sprintf(buf, "%.*s\n", 32, "NES020 Board ID"); -} -static DEVICE_ATTR_RO(board_id); - -static struct attribute *nes_dev_attributes[] = { - &dev_attr_hw_rev.attr, - &dev_attr_hca_type.attr, - &dev_attr_board_id.attr, - NULL -}; - -static const struct attribute_group nes_attr_group = { - .attrs = nes_dev_attributes, -}; - -/** - * nes_query_qp - */ -static int nes_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, - int attr_mask, struct ib_qp_init_attr *init_attr) -{ - struct nes_qp *nesqp = to_nesqp(ibqp); - - nes_debug(NES_DBG_QP, "\n"); - - attr->qp_access_flags = 0; - attr->cap.max_send_wr = nesqp->hwqp.sq_size; - attr->cap.max_recv_wr = nesqp->hwqp.rq_size; - attr->cap.max_recv_sge = 1; - if (nes_drv_opt & NES_DRV_OPT_NO_INLINE_DATA) - attr->cap.max_inline_data = 0; - else - attr->cap.max_inline_data = 64; - - init_attr->event_handler = nesqp->ibqp.event_handler; - init_attr->qp_context = nesqp->ibqp.qp_context; - init_attr->send_cq = nesqp->ibqp.send_cq; - init_attr->recv_cq = nesqp->ibqp.recv_cq; - init_attr->srq = nesqp->ibqp.srq; - init_attr->cap = attr->cap; - - return 0; -} - - -/** - * nes_hw_modify_qp - */ -int nes_hw_modify_qp(struct nes_device *nesdev, struct nes_qp *nesqp, - u32 next_iwarp_state, u32 termlen, u32 wait_completion) -{ - struct nes_hw_cqp_wqe *cqp_wqe; - /* struct iw_cm_id *cm_id = nesqp->cm_id; */ - /* struct iw_cm_event cm_event; */ - struct nes_cqp_request *cqp_request; - int ret; - u16 major_code; - - nes_debug(NES_DBG_MOD_QP, "QP%u, refcount=%d\n", - nesqp->hwqp.qp_id, atomic_read(&nesqp->refcount)); - - cqp_request = nes_get_cqp_request(nesdev); - if (cqp_request == NULL) { - nes_debug(NES_DBG_MOD_QP, "Failed to get a cqp_request.\n"); - return -ENOMEM; - } - if (wait_completion) { - cqp_request->waiting = 1; - } else { - cqp_request->waiting = 0; - } - cqp_wqe = &cqp_request->cqp_wqe; - - set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_OPCODE_IDX, - NES_CQP_MODIFY_QP | NES_CQP_QP_TYPE_IWARP | next_iwarp_state); - nes_debug(NES_DBG_MOD_QP, "using next_iwarp_state=%08x, wqe_words=%08x\n", - next_iwarp_state, le32_to_cpu(cqp_wqe->wqe_words[NES_CQP_WQE_OPCODE_IDX])); - nes_fill_init_cqp_wqe(cqp_wqe, nesdev); - set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_WQE_ID_IDX, nesqp->hwqp.qp_id); - set_wqe_64bit_value(cqp_wqe->wqe_words, NES_CQP_QP_WQE_CONTEXT_LOW_IDX, (u64)nesqp->nesqp_context_pbase); - - /* If sending a terminate message, fill in the length (in words) */ - if (((next_iwarp_state & NES_CQP_QP_IWARP_STATE_MASK) == NES_CQP_QP_IWARP_STATE_TERMINATE) && - !(next_iwarp_state & NES_CQP_QP_TERM_DONT_SEND_TERM_MSG)) { - termlen = ((termlen + 3) >> 2) << NES_CQP_OP_TERMLEN_SHIFT; - set_wqe_32bit_value(cqp_wqe->wqe_words, NES_CQP_QP_WQE_NEW_MSS_IDX, termlen); - } - - atomic_set(&cqp_request->refcount, 2); - nes_post_cqp_request(nesdev, cqp_request); - - /* Wait for CQP */ - if (wait_completion) { - /* nes_debug(NES_DBG_MOD_QP, "Waiting for modify iWARP QP%u to complete.\n", - nesqp->hwqp.qp_id); */ - ret = wait_event_timeout(cqp_request->waitq, (cqp_request->request_done != 0), - NES_EVENT_TIMEOUT); - nes_debug(NES_DBG_MOD_QP, "Modify iwarp QP%u completed, wait_event_timeout ret=%u, " - "CQP Major:Minor codes = 0x%04X:0x%04X.\n", - nesqp->hwqp.qp_id, ret, cqp_request->major_code, cqp_request->minor_code); - major_code = cqp_request->major_code; - if (major_code) { - nes_debug(NES_DBG_MOD_QP, "Modify iwarp QP%u failed" - "CQP Major:Minor codes = 0x%04X:0x%04X, intended next state = 0x%08X.\n", - nesqp->hwqp.qp_id, cqp_request->major_code, - cqp_request->minor_code, next_iwarp_state); - } - - nes_put_cqp_request(nesdev, cqp_request); - - if (!ret) - return -ETIME; - else if (major_code) - return -EIO; - else - return 0; - } else { - return 0; - } -} - - -/** - * nes_modify_qp - */ -int nes_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, - int attr_mask, struct ib_udata *udata) -{ - struct nes_qp *nesqp = to_nesqp(ibqp); - struct nes_vnic *nesvnic = to_nesvnic(ibqp->device); - struct nes_device *nesdev = nesvnic->nesdev; - /* u32 cqp_head; */ - /* u32 counter; */ - u32 next_iwarp_state = 0; - int err; - unsigned long qplockflags; - int ret; - u16 original_last_aeq; - u8 issue_modify_qp = 0; - u8 dont_wait = 0; - - nes_debug(NES_DBG_MOD_QP, "QP%u: QP State=%u, cur QP State=%u," - " iwarp_state=0x%X, refcount=%d\n", - nesqp->hwqp.qp_id, attr->qp_state, nesqp->ibqp_state, - nesqp->iwarp_state, atomic_read(&nesqp->refcount)); - - spin_lock_irqsave(&nesqp->lock, qplockflags); - - nes_debug(NES_DBG_MOD_QP, "QP%u: hw_iwarp_state=0x%X, hw_tcp_state=0x%X," - " QP Access Flags=0x%X, attr_mask = 0x%0x\n", - nesqp->hwqp.qp_id, nesqp->hw_iwarp_state, - nesqp->hw_tcp_state, attr->qp_access_flags, attr_mask); - - if (attr_mask & IB_QP_STATE) { - switch (attr->qp_state) { - case IB_QPS_INIT: - nes_debug(NES_DBG_MOD_QP, "QP%u: new state = init\n", - nesqp->hwqp.qp_id); - if (nesqp->iwarp_state > (u32)NES_CQP_QP_IWARP_STATE_IDLE) { - spin_unlock_irqrestore(&nesqp->lock, qplockflags); - return -EINVAL; - } - next_iwarp_state = NES_CQP_QP_IWARP_STATE_IDLE; - issue_modify_qp = 1; - break; - case IB_QPS_RTR: - nes_debug(NES_DBG_MOD_QP, "QP%u: new state = rtr\n", - nesqp->hwqp.qp_id); - if (nesqp->iwarp_state>(u32)NES_CQP_QP_IWARP_STATE_IDLE) { - spin_unlock_irqrestore(&nesqp->lock, qplockflags); - return -EINVAL; - } - next_iwarp_state = NES_CQP_QP_IWARP_STATE_IDLE; - issue_modify_qp = 1; - break; - case IB_QPS_RTS: - nes_debug(NES_DBG_MOD_QP, "QP%u: new state = rts\n", - nesqp->hwqp.qp_id); - if (nesqp->iwarp_state>(u32)NES_CQP_QP_IWARP_STATE_RTS) { - spin_unlock_irqrestore(&nesqp->lock, qplockflags); - return -EINVAL; - } - if (nesqp->cm_id == NULL) { - nes_debug(NES_DBG_MOD_QP, "QP%u: Failing attempt to move QP to RTS without a CM_ID. \n", - nesqp->hwqp.qp_id ); - spin_unlock_irqrestore(&nesqp->lock, qplockflags); - return -EINVAL; - } - next_iwarp_state = NES_CQP_QP_IWARP_STATE_RTS; - if (nesqp->iwarp_state != NES_CQP_QP_IWARP_STATE_RTS) - next_iwarp_state |= NES_CQP_QP_CONTEXT_VALID | - NES_CQP_QP_ARP_VALID | NES_CQP_QP_ORD_VALID; - issue_modify_qp = 1; - nesqp->hw_tcp_state = NES_AEQE_TCP_STATE_ESTABLISHED; - nesqp->hw_iwarp_state = NES_AEQE_IWARP_STATE_RTS; - nesqp->hte_added = 1; - break; - case IB_QPS_SQD: - issue_modify_qp = 1; - nes_debug(NES_DBG_MOD_QP, "QP%u: new state=closing. SQ head=%u, SQ tail=%u\n", - nesqp->hwqp.qp_id, nesqp->hwqp.sq_head, nesqp->hwqp.sq_tail); - if (nesqp->iwarp_state == (u32)NES_CQP_QP_IWARP_STATE_CLOSING) { - spin_unlock_irqrestore(&nesqp->lock, qplockflags); - return 0; - } else { - if (nesqp->iwarp_state > (u32)NES_CQP_QP_IWARP_STATE_CLOSING) { - nes_debug(NES_DBG_MOD_QP, "QP%u: State change to closing" - " ignored due to current iWARP state\n", - nesqp->hwqp.qp_id); - spin_unlock_irqrestore(&nesqp->lock, qplockflags); - return -EINVAL; - } - if (nesqp->hw_iwarp_state != NES_AEQE_IWARP_STATE_RTS) { - nes_debug(NES_DBG_MOD_QP, "QP%u: State change to closing" - " already done based on hw state.\n", - nesqp->hwqp.qp_id); - issue_modify_qp = 0; - } - switch (nesqp->hw_iwarp_state) { - case NES_AEQE_IWARP_STATE_CLOSING: - next_iwarp_state = NES_CQP_QP_IWARP_STATE_CLOSING; - break; - case NES_AEQE_IWARP_STATE_TERMINATE: - next_iwarp_state = NES_CQP_QP_IWARP_STATE_TERMINATE; - break; - case NES_AEQE_IWARP_STATE_ERROR: - next_iwarp_state = NES_CQP_QP_IWARP_STATE_ERROR; - break; - default: - next_iwarp_state = NES_CQP_QP_IWARP_STATE_CLOSING; - nesqp->hw_iwarp_state = NES_AEQE_IWARP_STATE_CLOSING; - break; - } - } - break; - case IB_QPS_SQE: - nes_debug(NES_DBG_MOD_QP, "QP%u: new state = terminate\n", - nesqp->hwqp.qp_id); - if (nesqp->iwarp_state>=(u32)NES_CQP_QP_IWARP_STATE_TERMINATE) { - spin_unlock_irqrestore(&nesqp->lock, qplockflags); - return -EINVAL; - } - /* next_iwarp_state = (NES_CQP_QP_IWARP_STATE_TERMINATE | 0x02000000); */ - next_iwarp_state = NES_CQP_QP_IWARP_STATE_TERMINATE; - nesqp->hw_iwarp_state = NES_AEQE_IWARP_STATE_TERMINATE; - issue_modify_qp = 1; - break; - case IB_QPS_ERR: - case IB_QPS_RESET: - if (nesqp->iwarp_state == (u32)NES_CQP_QP_IWARP_STATE_ERROR) { - spin_unlock_irqrestore(&nesqp->lock, qplockflags); - return -EINVAL; - } - nes_debug(NES_DBG_MOD_QP, "QP%u: new state = error\n", - nesqp->hwqp.qp_id); - if (nesqp->term_flags) - del_timer(&nesqp->terminate_timer); - - next_iwarp_state = NES_CQP_QP_IWARP_STATE_ERROR; - /* next_iwarp_state = (NES_CQP_QP_IWARP_STATE_TERMINATE | 0x02000000); */ - if (nesqp->hte_added) { - nes_debug(NES_DBG_MOD_QP, "set CQP_QP_DEL_HTE\n"); - next_iwarp_state |= NES_CQP_QP_DEL_HTE; - nesqp->hte_added = 0; - } - if ((nesqp->hw_tcp_state > NES_AEQE_TCP_STATE_CLOSED) && - (nesdev->iw_status) && - (nesqp->hw_tcp_state != NES_AEQE_TCP_STATE_TIME_WAIT)) { - next_iwarp_state |= NES_CQP_QP_RESET; - } else { - nes_debug(NES_DBG_MOD_QP, "QP%u NOT setting NES_CQP_QP_RESET since TCP state = %u\n", - nesqp->hwqp.qp_id, nesqp->hw_tcp_state); - dont_wait = 1; - } - issue_modify_qp = 1; - nesqp->hw_iwarp_state = NES_AEQE_IWARP_STATE_ERROR; - break; - default: - spin_unlock_irqrestore(&nesqp->lock, qplockflags); - return -EINVAL; - break; - } - - nesqp->ibqp_state = attr->qp_state; - nesqp->iwarp_state = next_iwarp_state & NES_CQP_QP_IWARP_STATE_MASK; - nes_debug(NES_DBG_MOD_QP, "Change nesqp->iwarp_state=%08x\n", - nesqp->iwarp_state); - } - - if (attr_mask & IB_QP_ACCESS_FLAGS) { - if (attr->qp_access_flags & IB_ACCESS_LOCAL_WRITE) { - nesqp->nesqp_context->misc |= cpu_to_le32(NES_QPCONTEXT_MISC_RDMA_WRITE_EN | - NES_QPCONTEXT_MISC_RDMA_READ_EN); - issue_modify_qp = 1; - } - if (attr->qp_access_flags & IB_ACCESS_REMOTE_WRITE) { - nesqp->nesqp_context->misc |= cpu_to_le32(NES_QPCONTEXT_MISC_RDMA_WRITE_EN); - issue_modify_qp = 1; - } - if (attr->qp_access_flags & IB_ACCESS_REMOTE_READ) { - nesqp->nesqp_context->misc |= cpu_to_le32(NES_QPCONTEXT_MISC_RDMA_READ_EN); - issue_modify_qp = 1; - } - if (attr->qp_access_flags & IB_ACCESS_MW_BIND) { - nesqp->nesqp_context->misc |= cpu_to_le32(NES_QPCONTEXT_MISC_WBIND_EN); - issue_modify_qp = 1; - } - - if (nesqp->user_mode) { - nesqp->nesqp_context->misc |= cpu_to_le32(NES_QPCONTEXT_MISC_RDMA_WRITE_EN | - NES_QPCONTEXT_MISC_RDMA_READ_EN); - issue_modify_qp = 1; - } - } - - original_last_aeq = nesqp->last_aeq; - spin_unlock_irqrestore(&nesqp->lock, qplockflags); - - nes_debug(NES_DBG_MOD_QP, "issue_modify_qp=%u\n", issue_modify_qp); - - ret = 0; - - - if (issue_modify_qp) { - nes_debug(NES_DBG_MOD_QP, "call nes_hw_modify_qp\n"); - ret = nes_hw_modify_qp(nesdev, nesqp, next_iwarp_state, 0, 1); - if (ret) - nes_debug(NES_DBG_MOD_QP, "nes_hw_modify_qp (next_iwarp_state = 0x%08X)" - " failed for QP%u.\n", - next_iwarp_state, nesqp->hwqp.qp_id); - - } - - if ((issue_modify_qp) && (nesqp->ibqp_state > IB_QPS_RTS)) { - nes_debug(NES_DBG_MOD_QP, "QP%u Issued ModifyQP refcount (%d)," - " original_last_aeq = 0x%04X. last_aeq = 0x%04X.\n", - nesqp->hwqp.qp_id, atomic_read(&nesqp->refcount), - original_last_aeq, nesqp->last_aeq); - if (!ret || original_last_aeq != NES_AEQE_AEID_RDMAP_ROE_BAD_LLP_CLOSE) { - if (dont_wait) { - if (nesqp->cm_id && nesqp->hw_tcp_state != 0) { - nes_debug(NES_DBG_MOD_QP, "QP%u Queuing fake disconnect for QP refcount (%d)," - " original_last_aeq = 0x%04X. last_aeq = 0x%04X.\n", - nesqp->hwqp.qp_id, atomic_read(&nesqp->refcount), - original_last_aeq, nesqp->last_aeq); - /* this one is for the cm_disconnect thread */ - spin_lock_irqsave(&nesqp->lock, qplockflags); - nesqp->hw_tcp_state = NES_AEQE_TCP_STATE_CLOSED; - nesqp->last_aeq = NES_AEQE_AEID_RESET_SENT; - spin_unlock_irqrestore(&nesqp->lock, qplockflags); - nes_cm_disconn(nesqp); - } else { - nes_debug(NES_DBG_MOD_QP, "QP%u No fake disconnect, QP refcount=%d\n", - nesqp->hwqp.qp_id, atomic_read(&nesqp->refcount)); - } - } else { - spin_lock_irqsave(&nesqp->lock, qplockflags); - if (nesqp->cm_id) { - /* These two are for the timer thread */ - if (atomic_inc_return(&nesqp->close_timer_started) == 1) { - nesqp->cm_id->add_ref(nesqp->cm_id); - nes_debug(NES_DBG_MOD_QP, "QP%u Not decrementing QP refcount (%d)," - " need ae to finish up, original_last_aeq = 0x%04X." - " last_aeq = 0x%04X, scheduling timer.\n", - nesqp->hwqp.qp_id, atomic_read(&nesqp->refcount), - original_last_aeq, nesqp->last_aeq); - schedule_nes_timer(nesqp->cm_node, (struct sk_buff *) nesqp, NES_TIMER_TYPE_CLOSE, 1, 0); - } - spin_unlock_irqrestore(&nesqp->lock, qplockflags); - } else { - spin_unlock_irqrestore(&nesqp->lock, qplockflags); - nes_debug(NES_DBG_MOD_QP, "QP%u Not decrementing QP refcount (%d)," - " need ae to finish up, original_last_aeq = 0x%04X." - " last_aeq = 0x%04X.\n", - nesqp->hwqp.qp_id, atomic_read(&nesqp->refcount), - original_last_aeq, nesqp->last_aeq); - } - } - } else { - nes_debug(NES_DBG_MOD_QP, "QP%u Decrementing QP refcount (%d), No ae to finish up," - " original_last_aeq = 0x%04X. last_aeq = 0x%04X.\n", - nesqp->hwqp.qp_id, atomic_read(&nesqp->refcount), - original_last_aeq, nesqp->last_aeq); - } - } else { - nes_debug(NES_DBG_MOD_QP, "QP%u Decrementing QP refcount (%d), No ae to finish up," - " original_last_aeq = 0x%04X. last_aeq = 0x%04X.\n", - nesqp->hwqp.qp_id, atomic_read(&nesqp->refcount), - original_last_aeq, nesqp->last_aeq); - } - - err = 0; - - nes_debug(NES_DBG_MOD_QP, "QP%u Leaving, refcount=%d\n", - nesqp->hwqp.qp_id, atomic_read(&nesqp->refcount)); - - return err; -} - -static inline void -fill_wqe_sg_send(struct nes_hw_qp_wqe *wqe, const struct ib_send_wr *ib_wr, - u32 uselkey) -{ - int sge_index; - int total_payload_length = 0; - for (sge_index = 0; sge_index < ib_wr->num_sge; sge_index++) { - set_wqe_64bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_FRAG0_LOW_IDX+(sge_index*4), - ib_wr->sg_list[sge_index].addr); - set_wqe_32bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_LENGTH0_IDX + (sge_index*4), - ib_wr->sg_list[sge_index].length); - if (uselkey) - set_wqe_32bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_STAG0_IDX + (sge_index*4), - (ib_wr->sg_list[sge_index].lkey)); - else - set_wqe_32bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_STAG0_IDX + (sge_index*4), 0); - - total_payload_length += ib_wr->sg_list[sge_index].length; - } - nes_debug(NES_DBG_IW_TX, "UC UC UC, sending total_payload_length=%u \n", - total_payload_length); - set_wqe_32bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_TOTAL_PAYLOAD_IDX, - total_payload_length); -} - -/** - * nes_post_send - */ -static int nes_post_send(struct ib_qp *ibqp, const struct ib_send_wr *ib_wr, - const struct ib_send_wr **bad_wr) -{ - u64 u64temp; - unsigned long flags = 0; - struct nes_vnic *nesvnic = to_nesvnic(ibqp->device); - struct nes_device *nesdev = nesvnic->nesdev; - struct nes_qp *nesqp = to_nesqp(ibqp); - struct nes_hw_qp_wqe *wqe; - int err = 0; - u32 qsize = nesqp->hwqp.sq_size; - u32 head; - u32 wqe_misc = 0; - u32 wqe_count = 0; - u32 counter; - - if (nesqp->ibqp_state > IB_QPS_RTS) { - err = -EINVAL; - goto out; - } - - spin_lock_irqsave(&nesqp->lock, flags); - - head = nesqp->hwqp.sq_head; - - while (ib_wr) { - /* Check for QP error */ - if (nesqp->term_flags) { - err = -EINVAL; - break; - } - - /* Check for SQ overflow */ - if (((head + (2 * qsize) - nesqp->hwqp.sq_tail) % qsize) == (qsize - 1)) { - err = -ENOMEM; - break; - } - - wqe = &nesqp->hwqp.sq_vbase[head]; - /* nes_debug(NES_DBG_IW_TX, "processing sq wqe for QP%u at %p, head = %u.\n", - nesqp->hwqp.qp_id, wqe, head); */ - nes_fill_init_qp_wqe(wqe, nesqp, head); - u64temp = (u64)(ib_wr->wr_id); - set_wqe_64bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_COMP_SCRATCH_LOW_IDX, - u64temp); - switch (ib_wr->opcode) { - case IB_WR_SEND: - case IB_WR_SEND_WITH_INV: - if (IB_WR_SEND == ib_wr->opcode) { - if (ib_wr->send_flags & IB_SEND_SOLICITED) - wqe_misc = NES_IWARP_SQ_OP_SENDSE; - else - wqe_misc = NES_IWARP_SQ_OP_SEND; - } else { - if (ib_wr->send_flags & IB_SEND_SOLICITED) - wqe_misc = NES_IWARP_SQ_OP_SENDSEINV; - else - wqe_misc = NES_IWARP_SQ_OP_SENDINV; - - set_wqe_32bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_INV_STAG_LOW_IDX, - ib_wr->ex.invalidate_rkey); - } - - if (ib_wr->num_sge > nesdev->nesadapter->max_sge) { - err = -EINVAL; - break; - } - - if (ib_wr->send_flags & IB_SEND_FENCE) - wqe_misc |= NES_IWARP_SQ_WQE_LOCAL_FENCE; - - if ((ib_wr->send_flags & IB_SEND_INLINE) && - ((nes_drv_opt & NES_DRV_OPT_NO_INLINE_DATA) == 0) && - (ib_wr->sg_list[0].length <= 64)) { - memcpy(&wqe->wqe_words[NES_IWARP_SQ_WQE_IMM_DATA_START_IDX], - (void *)(unsigned long)ib_wr->sg_list[0].addr, ib_wr->sg_list[0].length); - set_wqe_32bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_TOTAL_PAYLOAD_IDX, - ib_wr->sg_list[0].length); - wqe_misc |= NES_IWARP_SQ_WQE_IMM_DATA; - } else { - fill_wqe_sg_send(wqe, ib_wr, 1); - } - - break; - case IB_WR_RDMA_WRITE: - wqe_misc = NES_IWARP_SQ_OP_RDMAW; - if (ib_wr->num_sge > nesdev->nesadapter->max_sge) { - nes_debug(NES_DBG_IW_TX, "Exceeded max sge, ib_wr=%u, max=%u\n", - ib_wr->num_sge, nesdev->nesadapter->max_sge); - err = -EINVAL; - break; - } - - if (ib_wr->send_flags & IB_SEND_FENCE) - wqe_misc |= NES_IWARP_SQ_WQE_LOCAL_FENCE; - - set_wqe_32bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_RDMA_STAG_IDX, - rdma_wr(ib_wr)->rkey); - set_wqe_64bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_RDMA_TO_LOW_IDX, - rdma_wr(ib_wr)->remote_addr); - - if ((ib_wr->send_flags & IB_SEND_INLINE) && - ((nes_drv_opt & NES_DRV_OPT_NO_INLINE_DATA) == 0) && - (ib_wr->sg_list[0].length <= 64)) { - memcpy(&wqe->wqe_words[NES_IWARP_SQ_WQE_IMM_DATA_START_IDX], - (void *)(unsigned long)ib_wr->sg_list[0].addr, ib_wr->sg_list[0].length); - set_wqe_32bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_TOTAL_PAYLOAD_IDX, - ib_wr->sg_list[0].length); - wqe_misc |= NES_IWARP_SQ_WQE_IMM_DATA; - } else { - fill_wqe_sg_send(wqe, ib_wr, 1); - } - - wqe->wqe_words[NES_IWARP_SQ_WQE_RDMA_LENGTH_IDX] = - wqe->wqe_words[NES_IWARP_SQ_WQE_TOTAL_PAYLOAD_IDX]; - break; - case IB_WR_RDMA_READ: - case IB_WR_RDMA_READ_WITH_INV: - /* iWARP only supports 1 sge for RDMA reads */ - if (ib_wr->num_sge > 1) { - nes_debug(NES_DBG_IW_TX, "Exceeded max sge, ib_wr=%u, max=1\n", - ib_wr->num_sge); - err = -EINVAL; - break; - } - if (ib_wr->opcode == IB_WR_RDMA_READ) { - wqe_misc = NES_IWARP_SQ_OP_RDMAR; - } else { - wqe_misc = NES_IWARP_SQ_OP_RDMAR_LOCINV; - set_wqe_32bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_INV_STAG_LOW_IDX, - ib_wr->ex.invalidate_rkey); - } - - set_wqe_64bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_RDMA_TO_LOW_IDX, - rdma_wr(ib_wr)->remote_addr); - set_wqe_32bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_RDMA_STAG_IDX, - rdma_wr(ib_wr)->rkey); - set_wqe_32bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_RDMA_LENGTH_IDX, - ib_wr->sg_list->length); - set_wqe_64bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_FRAG0_LOW_IDX, - ib_wr->sg_list->addr); - set_wqe_32bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_STAG0_IDX, - ib_wr->sg_list->lkey); - break; - case IB_WR_LOCAL_INV: - wqe_misc = NES_IWARP_SQ_OP_LOCINV; - set_wqe_32bit_value(wqe->wqe_words, - NES_IWARP_SQ_LOCINV_WQE_INV_STAG_IDX, - ib_wr->ex.invalidate_rkey); - break; - case IB_WR_REG_MR: - { - struct nes_mr *mr = to_nesmr(reg_wr(ib_wr)->mr); - int page_shift = ilog2(reg_wr(ib_wr)->mr->page_size); - int flags = reg_wr(ib_wr)->access; - - if (mr->npages > (NES_4K_PBL_CHUNK_SIZE / sizeof(u64))) { - nes_debug(NES_DBG_IW_TX, "SQ_FMR: bad page_list_len\n"); - err = -EINVAL; - break; - } - wqe_misc = NES_IWARP_SQ_OP_FAST_REG; - set_wqe_64bit_value(wqe->wqe_words, - NES_IWARP_SQ_FMR_WQE_VA_FBO_LOW_IDX, - mr->ibmr.iova); - set_wqe_32bit_value(wqe->wqe_words, - NES_IWARP_SQ_FMR_WQE_LENGTH_LOW_IDX, - lower_32_bits(mr->ibmr.length)); - set_wqe_32bit_value(wqe->wqe_words, - NES_IWARP_SQ_FMR_WQE_LENGTH_HIGH_IDX, 0); - set_wqe_32bit_value(wqe->wqe_words, - NES_IWARP_SQ_FMR_WQE_MR_STAG_IDX, - reg_wr(ib_wr)->key); - - if (page_shift == 12) { - wqe_misc |= NES_IWARP_SQ_FMR_WQE_PAGE_SIZE_4K; - } else if (page_shift == 21) { - wqe_misc |= NES_IWARP_SQ_FMR_WQE_PAGE_SIZE_2M; - } else { - nes_debug(NES_DBG_IW_TX, "Invalid page shift," - " ib_wr=%u, max=1\n", ib_wr->num_sge); - err = -EINVAL; - break; - } - - /* Set access_flags */ - wqe_misc |= NES_IWARP_SQ_FMR_WQE_RIGHTS_ENABLE_LOCAL_READ; - if (flags & IB_ACCESS_LOCAL_WRITE) - wqe_misc |= NES_IWARP_SQ_FMR_WQE_RIGHTS_ENABLE_LOCAL_WRITE; - - if (flags & IB_ACCESS_REMOTE_WRITE) - wqe_misc |= NES_IWARP_SQ_FMR_WQE_RIGHTS_ENABLE_REMOTE_WRITE; - - if (flags & IB_ACCESS_REMOTE_READ) - wqe_misc |= NES_IWARP_SQ_FMR_WQE_RIGHTS_ENABLE_REMOTE_READ; - - if (flags & IB_ACCESS_MW_BIND) - wqe_misc |= NES_IWARP_SQ_FMR_WQE_RIGHTS_ENABLE_WINDOW_BIND; - - /* Fill in PBL info: */ - set_wqe_64bit_value(wqe->wqe_words, - NES_IWARP_SQ_FMR_WQE_PBL_ADDR_LOW_IDX, - mr->paddr); - - set_wqe_32bit_value(wqe->wqe_words, - NES_IWARP_SQ_FMR_WQE_PBL_LENGTH_IDX, - mr->npages * 8); - - nes_debug(NES_DBG_IW_TX, "SQ_REG_MR: iova_start: %llx, " - "length: %lld, rkey: %0x, pgl_paddr: %llx, " - "page_list_len: %u, wqe_misc: %x\n", - (unsigned long long) mr->ibmr.iova, - mr->ibmr.length, - reg_wr(ib_wr)->key, - (unsigned long long) mr->paddr, - mr->npages, - wqe_misc); - break; - } - default: - /* error */ - err = -EINVAL; - break; - } - - if (err) - break; - - if ((ib_wr->send_flags & IB_SEND_SIGNALED) || nesqp->sig_all) - wqe_misc |= NES_IWARP_SQ_WQE_SIGNALED_COMPL; - - wqe->wqe_words[NES_IWARP_SQ_WQE_MISC_IDX] = cpu_to_le32(wqe_misc); - - ib_wr = ib_wr->next; - head++; - wqe_count++; - if (head >= qsize) - head = 0; - - } - - nesqp->hwqp.sq_head = head; - barrier(); - while (wqe_count) { - counter = min(wqe_count, ((u32)255)); - wqe_count -= counter; - nes_write32(nesdev->regs + NES_WQE_ALLOC, - (counter << 24) | 0x00800000 | nesqp->hwqp.qp_id); - } - - spin_unlock_irqrestore(&nesqp->lock, flags); - -out: - if (err) - *bad_wr = ib_wr; - return err; -} - - -/** - * nes_post_recv - */ -static int nes_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *ib_wr, - const struct ib_recv_wr **bad_wr) -{ - u64 u64temp; - unsigned long flags = 0; - struct nes_vnic *nesvnic = to_nesvnic(ibqp->device); - struct nes_device *nesdev = nesvnic->nesdev; - struct nes_qp *nesqp = to_nesqp(ibqp); - struct nes_hw_qp_wqe *wqe; - int err = 0; - int sge_index; - u32 qsize = nesqp->hwqp.rq_size; - u32 head; - u32 wqe_count = 0; - u32 counter; - u32 total_payload_length; - - if (nesqp->ibqp_state > IB_QPS_RTS) { - err = -EINVAL; - goto out; - } - - spin_lock_irqsave(&nesqp->lock, flags); - - head = nesqp->hwqp.rq_head; - - while (ib_wr) { - /* Check for QP error */ - if (nesqp->term_flags) { - err = -EINVAL; - break; - } - - if (ib_wr->num_sge > nesdev->nesadapter->max_sge) { - err = -EINVAL; - break; - } - /* Check for RQ overflow */ - if (((head + (2 * qsize) - nesqp->hwqp.rq_tail) % qsize) == (qsize - 1)) { - err = -ENOMEM; - break; - } - - nes_debug(NES_DBG_IW_RX, "ibwr sge count = %u.\n", ib_wr->num_sge); - wqe = &nesqp->hwqp.rq_vbase[head]; - - /* nes_debug(NES_DBG_IW_RX, "QP%u:processing rq wqe at %p, head = %u.\n", - nesqp->hwqp.qp_id, wqe, head); */ - nes_fill_init_qp_wqe(wqe, nesqp, head); - u64temp = (u64)(ib_wr->wr_id); - set_wqe_64bit_value(wqe->wqe_words, NES_IWARP_SQ_WQE_COMP_SCRATCH_LOW_IDX, - u64temp); - total_payload_length = 0; - for (sge_index=0; sge_index < ib_wr->num_sge; sge_index++) { - set_wqe_64bit_value(wqe->wqe_words, NES_IWARP_RQ_WQE_FRAG0_LOW_IDX+(sge_index*4), - ib_wr->sg_list[sge_index].addr); - set_wqe_32bit_value(wqe->wqe_words, NES_IWARP_RQ_WQE_LENGTH0_IDX+(sge_index*4), - ib_wr->sg_list[sge_index].length); - set_wqe_32bit_value(wqe->wqe_words,NES_IWARP_RQ_WQE_STAG0_IDX+(sge_index*4), - ib_wr->sg_list[sge_index].lkey); - - total_payload_length += ib_wr->sg_list[sge_index].length; - } - set_wqe_32bit_value(wqe->wqe_words, NES_IWARP_RQ_WQE_TOTAL_PAYLOAD_IDX, - total_payload_length); - - ib_wr = ib_wr->next; - head++; - wqe_count++; - if (head >= qsize) - head = 0; - } - - nesqp->hwqp.rq_head = head; - barrier(); - while (wqe_count) { - counter = min(wqe_count, ((u32)255)); - wqe_count -= counter; - nes_write32(nesdev->regs+NES_WQE_ALLOC, (counter<<24) | nesqp->hwqp.qp_id); - } - - spin_unlock_irqrestore(&nesqp->lock, flags); - -out: - if (err) - *bad_wr = ib_wr; - return err; -} - -/** - * nes_drain_sq - drain sq - * @ibqp: pointer to ibqp - */ -static void nes_drain_sq(struct ib_qp *ibqp) -{ - struct nes_qp *nesqp = to_nesqp(ibqp); - - if (nesqp->hwqp.sq_tail != nesqp->hwqp.sq_head) - wait_for_completion(&nesqp->sq_drained); -} - -/** - * nes_drain_rq - drain rq - * @ibqp: pointer to ibqp - */ -static void nes_drain_rq(struct ib_qp *ibqp) -{ - struct nes_qp *nesqp = to_nesqp(ibqp); - - if (nesqp->hwqp.rq_tail != nesqp->hwqp.rq_head) - wait_for_completion(&nesqp->rq_drained); -} - -/** - * nes_poll_cq - */ -static int nes_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *entry) -{ - u64 u64temp; - u64 wrid; - unsigned long flags = 0; - struct nes_vnic *nesvnic = to_nesvnic(ibcq->device); - struct nes_device *nesdev = nesvnic->nesdev; - struct nes_cq *nescq = to_nescq(ibcq); - struct nes_qp *nesqp; - struct nes_hw_cqe cqe; - u32 head; - u32 wq_tail = 0; - u32 cq_size; - u32 cqe_count = 0; - u32 wqe_index; - u32 u32temp; - u32 move_cq_head = 1; - u32 err_code; - - nes_debug(NES_DBG_CQ, "\n"); - - spin_lock_irqsave(&nescq->lock, flags); - - head = nescq->hw_cq.cq_head; - cq_size = nescq->hw_cq.cq_size; - - while (cqe_count < num_entries) { - if ((le32_to_cpu(nescq->hw_cq.cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX]) & - NES_CQE_VALID) == 0) - break; - - /* - * Make sure we read CQ entry contents *after* - * we've checked the valid bit. - */ - rmb(); - - cqe = nescq->hw_cq.cq_vbase[head]; - u32temp = le32_to_cpu(cqe.cqe_words[NES_CQE_COMP_COMP_CTX_LOW_IDX]); - wqe_index = u32temp & (nesdev->nesadapter->max_qp_wr - 1); - u32temp &= ~(NES_SW_CONTEXT_ALIGN-1); - /* parse CQE, get completion context from WQE (either rq or sq) */ - u64temp = (((u64)(le32_to_cpu(cqe.cqe_words[NES_CQE_COMP_COMP_CTX_HIGH_IDX])))<<32) | - ((u64)u32temp); - - if (u64temp) { - nesqp = (struct nes_qp *)(unsigned long)u64temp; - memset(entry, 0, sizeof *entry); - if (cqe.cqe_words[NES_CQE_ERROR_CODE_IDX] == 0) { - entry->status = IB_WC_SUCCESS; - } else { - err_code = le32_to_cpu(cqe.cqe_words[NES_CQE_ERROR_CODE_IDX]); - if (NES_IWARP_CQE_MAJOR_DRV == (err_code >> 16)) { - entry->status = err_code & 0x0000ffff; - - /* The rest of the cqe's will be marked as flushed */ - nescq->hw_cq.cq_vbase[head].cqe_words[NES_CQE_ERROR_CODE_IDX] = - cpu_to_le32((NES_IWARP_CQE_MAJOR_FLUSH << 16) | - NES_IWARP_CQE_MINOR_FLUSH); - } else - entry->status = IB_WC_WR_FLUSH_ERR; - } - - entry->qp = &nesqp->ibqp; - entry->src_qp = nesqp->hwqp.qp_id; - - if (le32_to_cpu(cqe.cqe_words[NES_CQE_OPCODE_IDX]) & NES_CQE_SQ) { - if (nesqp->skip_lsmm) { - nesqp->skip_lsmm = 0; - nesqp->hwqp.sq_tail++; - } - - /* Working on a SQ Completion*/ - wrid = (((u64)(cpu_to_le32((u32)nesqp->hwqp.sq_vbase[wqe_index]. - wqe_words[NES_IWARP_SQ_WQE_COMP_SCRATCH_HIGH_IDX]))) << 32) | - ((u64)(cpu_to_le32((u32)nesqp->hwqp.sq_vbase[wqe_index]. - wqe_words[NES_IWARP_SQ_WQE_COMP_SCRATCH_LOW_IDX]))); - entry->byte_len = le32_to_cpu(nesqp->hwqp.sq_vbase[wqe_index]. - wqe_words[NES_IWARP_SQ_WQE_TOTAL_PAYLOAD_IDX]); - - switch (le32_to_cpu(nesqp->hwqp.sq_vbase[wqe_index]. - wqe_words[NES_IWARP_SQ_WQE_MISC_IDX]) & 0x3f) { - case NES_IWARP_SQ_OP_RDMAW: - nes_debug(NES_DBG_CQ, "Operation = RDMA WRITE.\n"); - entry->opcode = IB_WC_RDMA_WRITE; - break; - case NES_IWARP_SQ_OP_RDMAR: - nes_debug(NES_DBG_CQ, "Operation = RDMA READ.\n"); - entry->opcode = IB_WC_RDMA_READ; - entry->byte_len = le32_to_cpu(nesqp->hwqp.sq_vbase[wqe_index]. - wqe_words[NES_IWARP_SQ_WQE_RDMA_LENGTH_IDX]); - break; - case NES_IWARP_SQ_OP_SENDINV: - case NES_IWARP_SQ_OP_SENDSEINV: - case NES_IWARP_SQ_OP_SEND: - case NES_IWARP_SQ_OP_SENDSE: - nes_debug(NES_DBG_CQ, "Operation = Send.\n"); - entry->opcode = IB_WC_SEND; - break; - case NES_IWARP_SQ_OP_LOCINV: - entry->opcode = IB_WC_LOCAL_INV; - break; - case NES_IWARP_SQ_OP_FAST_REG: - entry->opcode = IB_WC_REG_MR; - break; - } - - nesqp->hwqp.sq_tail = (wqe_index+1)&(nesqp->hwqp.sq_size - 1); - if ((entry->status != IB_WC_SUCCESS) && (nesqp->hwqp.sq_tail != nesqp->hwqp.sq_head)) { - move_cq_head = 0; - wq_tail = nesqp->hwqp.sq_tail; - } - } else { - /* Working on a RQ Completion*/ - entry->byte_len = le32_to_cpu(cqe.cqe_words[NES_CQE_PAYLOAD_LENGTH_IDX]); - wrid = ((u64)(le32_to_cpu(nesqp->hwqp.rq_vbase[wqe_index].wqe_words[NES_IWARP_RQ_WQE_COMP_SCRATCH_LOW_IDX]))) | - ((u64)(le32_to_cpu(nesqp->hwqp.rq_vbase[wqe_index].wqe_words[NES_IWARP_RQ_WQE_COMP_SCRATCH_HIGH_IDX]))<<32); - entry->opcode = IB_WC_RECV; - - nesqp->hwqp.rq_tail = (wqe_index+1)&(nesqp->hwqp.rq_size - 1); - if ((entry->status != IB_WC_SUCCESS) && (nesqp->hwqp.rq_tail != nesqp->hwqp.rq_head)) { - move_cq_head = 0; - wq_tail = nesqp->hwqp.rq_tail; - } - } - - if (nesqp->iwarp_state > NES_CQP_QP_IWARP_STATE_RTS) { - if (nesqp->hwqp.sq_tail == nesqp->hwqp.sq_head) - complete(&nesqp->sq_drained); - if (nesqp->hwqp.rq_tail == nesqp->hwqp.rq_head) - complete(&nesqp->rq_drained); - } - - entry->wr_id = wrid; - entry++; - cqe_count++; - } - - if (move_cq_head) { - nescq->hw_cq.cq_vbase[head].cqe_words[NES_CQE_OPCODE_IDX] = 0; - if (++head >= cq_size) - head = 0; - nescq->polled_completions++; - - if ((nescq->polled_completions > (cq_size / 2)) || - (nescq->polled_completions == 255)) { - nes_debug(NES_DBG_CQ, "CQ%u Issuing CQE Allocate since more than half of cqes" - " are pending %u of %u.\n", - nescq->hw_cq.cq_number, nescq->polled_completions, cq_size); - nes_write32(nesdev->regs+NES_CQE_ALLOC, - nescq->hw_cq.cq_number | (nescq->polled_completions << 16)); - nescq->polled_completions = 0; - } - } else { - /* Update the wqe index and set status to flush */ - wqe_index = le32_to_cpu(cqe.cqe_words[NES_CQE_COMP_COMP_CTX_LOW_IDX]); - wqe_index = (wqe_index & (~(nesdev->nesadapter->max_qp_wr - 1))) | wq_tail; - nescq->hw_cq.cq_vbase[head].cqe_words[NES_CQE_COMP_COMP_CTX_LOW_IDX] = - cpu_to_le32(wqe_index); - move_cq_head = 1; /* ready for next pass */ - } - } - - if (nescq->polled_completions) { - nes_write32(nesdev->regs+NES_CQE_ALLOC, - nescq->hw_cq.cq_number | (nescq->polled_completions << 16)); - nescq->polled_completions = 0; - } - - nescq->hw_cq.cq_head = head; - nes_debug(NES_DBG_CQ, "Reporting %u completions for CQ%u.\n", - cqe_count, nescq->hw_cq.cq_number); - - spin_unlock_irqrestore(&nescq->lock, flags); - - return cqe_count; -} - - -/** - * nes_req_notify_cq - */ -static int nes_req_notify_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags notify_flags) - { - struct nes_vnic *nesvnic = to_nesvnic(ibcq->device); - struct nes_device *nesdev = nesvnic->nesdev; - struct nes_cq *nescq = to_nescq(ibcq); - u32 cq_arm; - - nes_debug(NES_DBG_CQ, "Requesting notification for CQ%u.\n", - nescq->hw_cq.cq_number); - - cq_arm = nescq->hw_cq.cq_number; - if ((notify_flags & IB_CQ_SOLICITED_MASK) == IB_CQ_NEXT_COMP) - cq_arm |= NES_CQE_ALLOC_NOTIFY_NEXT; - else if ((notify_flags & IB_CQ_SOLICITED_MASK) == IB_CQ_SOLICITED) - cq_arm |= NES_CQE_ALLOC_NOTIFY_SE; - else - return -EINVAL; - - nes_write32(nesdev->regs+NES_CQE_ALLOC, cq_arm); - nes_read32(nesdev->regs+NES_CQE_ALLOC); - - return 0; -} - -static int nes_port_immutable(struct ib_device *ibdev, u8 port_num, - struct ib_port_immutable *immutable) -{ - struct ib_port_attr attr; - int err; - - immutable->core_cap_flags = RDMA_CORE_PORT_IWARP; - - err = nes_query_port(ibdev, port_num, &attr); - if (err) - return err; - - immutable->pkey_tbl_len = attr.pkey_tbl_len; - immutable->gid_tbl_len = attr.gid_tbl_len; - - return 0; -} - -static void get_dev_fw_str(struct ib_device *dev, char *str) -{ - struct nes_ib_device *nesibdev = - container_of(dev, struct nes_ib_device, ibdev); - struct nes_vnic *nesvnic = nesibdev->nesvnic; - - nes_debug(NES_DBG_INIT, "\n"); - snprintf(str, IB_FW_VERSION_NAME_MAX, "%u.%u", - (nesvnic->nesdev->nesadapter->firmware_version >> 16), - (nesvnic->nesdev->nesadapter->firmware_version & 0x000000ff)); -} - -static const struct ib_device_ops nes_dev_ops = { - .alloc_mr = nes_alloc_mr, - .alloc_mw = nes_alloc_mw, - .alloc_pd = nes_alloc_pd, - .alloc_ucontext = nes_alloc_ucontext, - .create_cq = nes_create_cq, - .create_qp = nes_create_qp, - .dealloc_mw = nes_dealloc_mw, - .dealloc_pd = nes_dealloc_pd, - .dealloc_ucontext = nes_dealloc_ucontext, - .dereg_mr = nes_dereg_mr, - .destroy_cq = nes_destroy_cq, - .destroy_qp = nes_destroy_qp, - .drain_rq = nes_drain_rq, - .drain_sq = nes_drain_sq, - .get_dev_fw_str = get_dev_fw_str, - .get_dma_mr = nes_get_dma_mr, - .get_port_immutable = nes_port_immutable, - .iw_accept = nes_accept, - .iw_add_ref = nes_add_ref, - .iw_connect = nes_connect, - .iw_create_listen = nes_create_listen, - .iw_destroy_listen = nes_destroy_listen, - .iw_get_qp = nes_get_qp, - .iw_reject = nes_reject, - .iw_rem_ref = nes_rem_ref, - .map_mr_sg = nes_map_mr_sg, - .mmap = nes_mmap, - .modify_qp = nes_modify_qp, - .poll_cq = nes_poll_cq, - .post_recv = nes_post_recv, - .post_send = nes_post_send, - .query_device = nes_query_device, - .query_gid = nes_query_gid, - .query_pkey = nes_query_pkey, - .query_port = nes_query_port, - .query_qp = nes_query_qp, - .reg_user_mr = nes_reg_user_mr, - .req_notify_cq = nes_req_notify_cq, - INIT_RDMA_OBJ_SIZE(ib_pd, nes_pd, ibpd), - INIT_RDMA_OBJ_SIZE(ib_ucontext, nes_ucontext, ibucontext), -}; - -/** - * nes_init_ofa_device - */ -struct nes_ib_device *nes_init_ofa_device(struct net_device *netdev) -{ - struct nes_ib_device *nesibdev; - struct nes_vnic *nesvnic = netdev_priv(netdev); - struct nes_device *nesdev = nesvnic->nesdev; - - nesibdev = ib_alloc_device(nes_ib_device, ibdev); - if (nesibdev == NULL) { - return NULL; - } - nesibdev->ibdev.owner = THIS_MODULE; - - nesibdev->ibdev.node_type = RDMA_NODE_RNIC; - memset(&nesibdev->ibdev.node_guid, 0, sizeof(nesibdev->ibdev.node_guid)); - memcpy(&nesibdev->ibdev.node_guid, netdev->dev_addr, 6); - - nesibdev->ibdev.uverbs_cmd_mask = - (1ull << IB_USER_VERBS_CMD_GET_CONTEXT) | - (1ull << IB_USER_VERBS_CMD_QUERY_DEVICE) | - (1ull << IB_USER_VERBS_CMD_QUERY_PORT) | - (1ull << IB_USER_VERBS_CMD_ALLOC_PD) | - (1ull << IB_USER_VERBS_CMD_DEALLOC_PD) | - (1ull << IB_USER_VERBS_CMD_REG_MR) | - (1ull << IB_USER_VERBS_CMD_DEREG_MR) | - (1ull << IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL) | - (1ull << IB_USER_VERBS_CMD_CREATE_CQ) | - (1ull << IB_USER_VERBS_CMD_DESTROY_CQ) | - (1ull << IB_USER_VERBS_CMD_CREATE_AH) | - (1ull << IB_USER_VERBS_CMD_DESTROY_AH) | - (1ull << IB_USER_VERBS_CMD_REQ_NOTIFY_CQ) | - (1ull << IB_USER_VERBS_CMD_CREATE_QP) | - (1ull << IB_USER_VERBS_CMD_MODIFY_QP) | - (1ull << IB_USER_VERBS_CMD_POLL_CQ) | - (1ull << IB_USER_VERBS_CMD_DESTROY_QP) | - (1ull << IB_USER_VERBS_CMD_ALLOC_MW) | - (1ull << IB_USER_VERBS_CMD_BIND_MW) | - (1ull << IB_USER_VERBS_CMD_DEALLOC_MW) | - (1ull << IB_USER_VERBS_CMD_POST_RECV) | - (1ull << IB_USER_VERBS_CMD_POST_SEND); - - nesibdev->ibdev.phys_port_cnt = 1; - nesibdev->ibdev.num_comp_vectors = 1; - nesibdev->ibdev.dev.parent = &nesdev->pcidev->dev; - - ib_set_device_ops(&nesibdev->ibdev, &nes_dev_ops); - memcpy(nesibdev->ibdev.iw_ifname, netdev->name, - sizeof(nesibdev->ibdev.iw_ifname)); - - return nesibdev; -} - - -/** - * nes_handle_delayed_event - */ -static void nes_handle_delayed_event(struct timer_list *t) -{ - struct nes_vnic *nesvnic = from_timer(nesvnic, t, event_timer); - - if (nesvnic->delayed_event != nesvnic->last_dispatched_event) { - struct ib_event event; - - event.device = &nesvnic->nesibdev->ibdev; - if (!event.device) - goto stop_timer; - event.event = nesvnic->delayed_event; - event.element.port_num = nesvnic->logical_port + 1; - ib_dispatch_event(&event); - } - -stop_timer: - nesvnic->event_timer.function = NULL; -} - - -void nes_port_ibevent(struct nes_vnic *nesvnic) -{ - struct nes_ib_device *nesibdev = nesvnic->nesibdev; - struct nes_device *nesdev = nesvnic->nesdev; - struct ib_event event; - event.device = &nesibdev->ibdev; - event.element.port_num = nesvnic->logical_port + 1; - event.event = nesdev->iw_status ? IB_EVENT_PORT_ACTIVE : IB_EVENT_PORT_ERR; - - if (!nesvnic->event_timer.function) { - ib_dispatch_event(&event); - nesvnic->last_dispatched_event = event.event; - nesvnic->event_timer.function = nes_handle_delayed_event; - nesvnic->event_timer.expires = jiffies + NES_EVENT_DELAY; - add_timer(&nesvnic->event_timer); - } else { - mod_timer(&nesvnic->event_timer, jiffies + NES_EVENT_DELAY); - } - nesvnic->delayed_event = event.event; -} - - -/** - * nes_destroy_ofa_device - */ -void nes_destroy_ofa_device(struct nes_ib_device *nesibdev) -{ - if (nesibdev == NULL) - return; - - nes_unregister_ofa_device(nesibdev); - - ib_dealloc_device(&nesibdev->ibdev); -} - - -/** - * nes_register_ofa_device - */ -int nes_register_ofa_device(struct nes_ib_device *nesibdev) -{ - struct nes_vnic *nesvnic = nesibdev->nesvnic; - struct nes_device *nesdev = nesvnic->nesdev; - struct nes_adapter *nesadapter = nesdev->nesadapter; - int ret; - - rdma_set_device_sysfs_group(&nesvnic->nesibdev->ibdev, &nes_attr_group); - nesvnic->nesibdev->ibdev.driver_id = RDMA_DRIVER_NES; - ret = ib_register_device(&nesvnic->nesibdev->ibdev, "nes%d"); - if (ret) { - return ret; - } - - /* Get the resources allocated to this device */ - nesibdev->max_cq = (nesadapter->max_cq-NES_FIRST_QPN) / nesadapter->port_count; - nesibdev->max_mr = nesadapter->max_mr / nesadapter->port_count; - nesibdev->max_qp = (nesadapter->max_qp-NES_FIRST_QPN) / nesadapter->port_count; - nesibdev->max_pd = nesadapter->max_pd / nesadapter->port_count; - - nesvnic->of_device_registered = 1; - - return 0; -} - - -/** - * nes_unregister_ofa_device - */ -static void nes_unregister_ofa_device(struct nes_ib_device *nesibdev) -{ - struct nes_vnic *nesvnic = nesibdev->nesvnic; - - if (nesvnic->of_device_registered) - ib_unregister_device(&nesibdev->ibdev); - - nesvnic->of_device_registered = 0; -} diff --git a/drivers/infiniband/hw/nes/nes_verbs.h b/drivers/infiniband/hw/nes/nes_verbs.h deleted file mode 100644 index 114a9b59fefd..000000000000 --- a/drivers/infiniband/hw/nes/nes_verbs.h +++ /dev/null @@ -1,198 +0,0 @@ -/* - * Copyright (c) 2006 - 2011 Intel Corporation. All rights reserved. - * Copyright (c) 2005 Open Grid Computing, Inc. All rights reserved. - * - * This software is available to you under a choice of one of two - * licenses. You may choose to be licensed under the terms of the GNU - * General Public License (GPL) Version 2, available from the file - * COPYING in the main directory of this source tree, or the - * OpenIB.org BSD license below: - * - * Redistribution and use in source and binary forms, with or - * without modification, are permitted provided that the following - * conditions are met: - * - * - Redistributions of source code must retain the above - * copyright notice, this list of conditions and the following - * disclaimer. - * - * - Redistributions in binary form must reproduce the above - * copyright notice, this list of conditions and the following - * disclaimer in the documentation and/or other materials - * provided with the distribution. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, - * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF - * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND - * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS - * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN - * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN - * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - * - */ - -#ifndef NES_VERBS_H -#define NES_VERBS_H - -struct nes_device; - -#define NES_MAX_USER_DB_REGIONS 4096 -#define NES_MAX_USER_WQ_REGIONS 4096 - -#define NES_TERM_SENT 0x01 -#define NES_TERM_RCVD 0x02 -#define NES_TERM_DONE 0x04 - -struct nes_ucontext { - struct ib_ucontext ibucontext; - struct nes_device *nesdev; - unsigned long mmap_wq_offset; - unsigned long mmap_cq_offset; /* to be removed */ - int index; /* rnic index (minor) */ - unsigned long allocated_doorbells[BITS_TO_LONGS(NES_MAX_USER_DB_REGIONS)]; - u16 mmap_db_index[NES_MAX_USER_DB_REGIONS]; - u16 first_free_db; - unsigned long allocated_wqs[BITS_TO_LONGS(NES_MAX_USER_WQ_REGIONS)]; - struct nes_qp *mmap_nesqp[NES_MAX_USER_WQ_REGIONS]; - u16 first_free_wq; - struct list_head cq_reg_mem_list; - struct list_head qp_reg_mem_list; - u32 mcrqf; -}; - -struct nes_pd { - struct ib_pd ibpd; - u16 pd_id; - atomic_t sqp_count; - u16 mmap_db_index; -}; - -struct nes_mr { - union { - struct ib_mr ibmr; - struct ib_mw ibmw; - struct ib_fmr ibfmr; - }; - struct ib_umem *region; - u16 pbls_used; - u8 mode; - u8 pbl_4k; - __le64 *pages; - dma_addr_t paddr; - u32 max_pages; - u32 npages; -}; - -struct nes_hw_pb { - __le32 pa_low; - __le32 pa_high; -}; - -struct nes_vpbl { - dma_addr_t pbl_pbase; - struct nes_hw_pb *pbl_vbase; -}; - -struct nes_root_vpbl { - dma_addr_t pbl_pbase; - struct nes_hw_pb *pbl_vbase; - struct nes_vpbl *leaf_vpbl; -}; - -struct nes_fmr { - struct nes_mr nesmr; - u32 leaf_pbl_cnt; - struct nes_root_vpbl root_vpbl; - struct ib_qp *ib_qp; - int access_rights; - struct ib_fmr_attr attr; -}; - -struct nes_av; - -struct nes_cq { - struct ib_cq ibcq; - struct nes_hw_cq hw_cq; - u32 polled_completions; - u32 cq_mem_size; - spinlock_t lock; - u8 virtual_cq; - u8 pad[3]; - u32 mcrqf; -}; - -struct nes_wq { - spinlock_t lock; -}; - -struct disconn_work { - struct work_struct work; - struct nes_qp *nesqp; -}; - -struct iw_cm_id; -struct ietf_mpa_frame; - -struct nes_qp { - struct ib_qp ibqp; - void *allocated_buffer; - struct iw_cm_id *cm_id; - struct nes_cq *nesscq; - struct nes_cq *nesrcq; - struct nes_pd *nespd; - void *cm_node; /* handle of the node this QP is associated with */ - void *ietf_frame; - u8 ietf_frame_size; - dma_addr_t ietf_frame_pbase; - struct ib_mr *lsmm_mr; - struct nes_hw_qp hwqp; - struct work_struct work; - enum ib_qp_state ibqp_state; - u32 iwarp_state; - u32 hte_index; - u32 last_aeq; - u32 qp_mem_size; - atomic_t refcount; - atomic_t close_timer_started; - u32 mmap_sq_db_index; - u32 mmap_rq_db_index; - spinlock_t lock; - spinlock_t pau_lock; - struct nes_qp_context *nesqp_context; - dma_addr_t nesqp_context_pbase; - void *pbl_vbase; - dma_addr_t pbl_pbase; - struct page *page; - struct timer_list terminate_timer; - enum ib_event_type terminate_eventtype; - struct sk_buff_head pau_list; - u32 pau_rcv_nxt; - u16 active_conn:1; - u16 skip_lsmm:1; - u16 user_mode:1; - u16 hte_added:1; - u16 flush_issued:1; - u16 destroyed:1; - u16 sig_all:1; - u16 pau_mode:1; - u16 rsvd:8; - u16 private_data_len; - u16 term_sq_flush_code; - u16 term_rq_flush_code; - u8 hw_iwarp_state; - u8 hw_tcp_state; - u8 term_flags; - u8 sq_kmapped; - u8 pau_busy; - u8 pau_pending; - u8 pau_state; - __u64 nesuqp_addr; - struct completion sq_drained; - struct completion rq_drained; -}; - -struct ib_mr *nes_reg_phys_mr(struct ib_pd *ib_pd, - u64 addr, u64 size, int acc, u64 *iova_start); - -#endif /* NES_VERBS_H */ diff --git a/drivers/infiniband/hw/ocrdma/ocrdma_hw.c b/drivers/infiniband/hw/ocrdma/ocrdma_hw.c index 5127e2ea4bdd..d82d3ec3649e 100644 --- a/drivers/infiniband/hw/ocrdma/ocrdma_hw.c +++ b/drivers/infiniband/hw/ocrdma/ocrdma_hw.c @@ -1351,7 +1351,6 @@ static int ocrdma_mbx_get_ctrl_attribs(struct ocrdma_dev *dev) mqe->u.nonemb_req.sge[0].pa_hi = (u32) upper_32_bits(dma.pa); mqe->u.nonemb_req.sge[0].len = dma.size; - memset(dma.va, 0, dma.size); ocrdma_init_mch((struct ocrdma_mbx_hdr *)dma.va, OCRDMA_CMD_GET_CTRL_ATTRIBUTES, OCRDMA_SUBSYS_COMMON, @@ -1690,7 +1689,6 @@ static int ocrdma_mbx_create_ah_tbl(struct ocrdma_dev *dev) goto mem_err_ah; dev->av_tbl.pa = pa; dev->av_tbl.num_ah = max_ah; - memset(dev->av_tbl.va, 0, dev->av_tbl.size); pbes = (struct ocrdma_pbe *)dev->av_tbl.pbl.va; for (i = 0; i < dev->av_tbl.size / OCRDMA_MIN_Q_PAGE_SIZE; i++) { @@ -1888,14 +1886,13 @@ mem_err: return status; } -int ocrdma_mbx_destroy_cq(struct ocrdma_dev *dev, struct ocrdma_cq *cq) +void ocrdma_mbx_destroy_cq(struct ocrdma_dev *dev, struct ocrdma_cq *cq) { - int status = -ENOMEM; struct ocrdma_destroy_cq *cmd; cmd = ocrdma_init_emb_mqe(OCRDMA_CMD_DELETE_CQ, sizeof(*cmd)); if (!cmd) - return status; + return; ocrdma_init_mch(&cmd->req, OCRDMA_CMD_DELETE_CQ, OCRDMA_SUBSYS_COMMON, sizeof(*cmd)); @@ -1903,11 +1900,10 @@ int ocrdma_mbx_destroy_cq(struct ocrdma_dev *dev, struct ocrdma_cq *cq) (cq->id << OCRDMA_DESTROY_CQ_QID_SHIFT) & OCRDMA_DESTROY_CQ_QID_MASK; - status = ocrdma_mbx_cmd(dev, (struct ocrdma_mqe *)cmd); + ocrdma_mbx_cmd(dev, (struct ocrdma_mqe *)cmd); ocrdma_unbind_eq(dev, cq->eqn); dma_free_coherent(&dev->nic_info.pdev->dev, cq->len, cq->va, cq->pa); kfree(cmd); - return status; } int ocrdma_mbx_alloc_lkey(struct ocrdma_dev *dev, struct ocrdma_hw_mr *hwmr, @@ -2905,7 +2901,6 @@ static int ocrdma_mbx_get_dcbx_config(struct ocrdma_dev *dev, u32 ptype, mqe_sge->pa_hi = (u32) upper_32_bits(pa); mqe_sge->len = cmd.hdr.pyld_len; - memset(req, 0, sizeof(struct ocrdma_get_dcbx_cfg_req)); ocrdma_init_mch(&req->hdr, OCRDMA_CMD_GET_DCBX_CONFIG, OCRDMA_SUBSYS_DCBX, cmd.hdr.pyld_len); req->param_type = ptype; diff --git a/drivers/infiniband/hw/ocrdma/ocrdma_hw.h b/drivers/infiniband/hw/ocrdma/ocrdma_hw.h index 06ec59326a90..12c23a7652b9 100644 --- a/drivers/infiniband/hw/ocrdma/ocrdma_hw.h +++ b/drivers/infiniband/hw/ocrdma/ocrdma_hw.h @@ -122,7 +122,7 @@ int ocrdma_reg_mr(struct ocrdma_dev *, struct ocrdma_hw_mr *hwmr, u32 pd_id, int acc); int ocrdma_mbx_create_cq(struct ocrdma_dev *, struct ocrdma_cq *, int entries, int dpp_cq, u16 pd_id); -int ocrdma_mbx_destroy_cq(struct ocrdma_dev *, struct ocrdma_cq *); +void ocrdma_mbx_destroy_cq(struct ocrdma_dev *dev, struct ocrdma_cq *cq); int ocrdma_mbx_create_qp(struct ocrdma_qp *, struct ib_qp_init_attr *attrs, u8 enable_dpp_cq, u16 dpp_cq_id, u16 *dpp_offset, diff --git a/drivers/infiniband/hw/ocrdma/ocrdma_main.c b/drivers/infiniband/hw/ocrdma/ocrdma_main.c index fc6c0962dea9..c15cfc6cef81 100644 --- a/drivers/infiniband/hw/ocrdma/ocrdma_main.c +++ b/drivers/infiniband/hw/ocrdma/ocrdma_main.c @@ -144,6 +144,10 @@ static const struct attribute_group ocrdma_attr_group = { }; static const struct ib_device_ops ocrdma_dev_ops = { + .owner = THIS_MODULE, + .driver_id = RDMA_DRIVER_OCRDMA, + .uverbs_abi_ver = OCRDMA_ABI_VERSION, + .alloc_mr = ocrdma_alloc_mr, .alloc_pd = ocrdma_alloc_pd, .alloc_ucontext = ocrdma_alloc_ucontext, @@ -178,6 +182,7 @@ static const struct ib_device_ops ocrdma_dev_ops = { .resize_cq = ocrdma_resize_cq, INIT_RDMA_OBJ_SIZE(ib_ah, ocrdma_ah, ibah), + INIT_RDMA_OBJ_SIZE(ib_cq, ocrdma_cq, ibcq), INIT_RDMA_OBJ_SIZE(ib_pd, ocrdma_pd, ibpd), INIT_RDMA_OBJ_SIZE(ib_ucontext, ocrdma_ucontext, ibucontext), }; @@ -200,8 +205,6 @@ static int ocrdma_register_device(struct ocrdma_dev *dev) BUILD_BUG_ON(sizeof(OCRDMA_NODE_DESC) > IB_DEVICE_NODE_DESC_MAX); memcpy(dev->ibdev.node_desc, OCRDMA_NODE_DESC, sizeof(OCRDMA_NODE_DESC)); - dev->ibdev.owner = THIS_MODULE; - dev->ibdev.uverbs_abi_ver = OCRDMA_ABI_VERSION; dev->ibdev.uverbs_cmd_mask = OCRDMA_UVERBS(GET_CONTEXT) | OCRDMA_UVERBS(QUERY_DEVICE) | @@ -249,7 +252,6 @@ static int ocrdma_register_device(struct ocrdma_dev *dev) ib_set_device_ops(&dev->ibdev, &ocrdma_dev_srq_ops); } rdma_set_device_sysfs_group(&dev->ibdev, &ocrdma_attr_group); - dev->ibdev.driver_id = RDMA_DRIVER_OCRDMA; ret = ib_device_set_netdev(&dev->ibdev, dev->nic_info.netdev, 1); if (ret) return ret; diff --git a/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c b/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c index 35ec87015792..bccc11378109 100644 --- a/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c +++ b/drivers/infiniband/hw/ocrdma/ocrdma_verbs.c @@ -925,8 +925,7 @@ int ocrdma_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata) ocrdma_free_mr_pbl_tbl(dev, &mr->hwmr); /* it could be user registered memory. */ - if (mr->umem) - ib_umem_release(mr->umem); + ib_umem_release(mr->umem); kfree(mr); /* Don't stop cleanup, in case FW is unresponsive */ @@ -977,12 +976,12 @@ err: return status; } -struct ib_cq *ocrdma_create_cq(struct ib_device *ibdev, - const struct ib_cq_init_attr *attr, - struct ib_udata *udata) +int ocrdma_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, + struct ib_udata *udata) { + struct ib_device *ibdev = ibcq->device; int entries = attr->cqe; - struct ocrdma_cq *cq; + struct ocrdma_cq *cq = get_ocrdma_cq(ibcq); struct ocrdma_dev *dev = get_ocrdma_dev(ibdev); struct ocrdma_ucontext *uctx = rdma_udata_to_drv_context( udata, struct ocrdma_ucontext, ibucontext); @@ -991,16 +990,13 @@ struct ib_cq *ocrdma_create_cq(struct ib_device *ibdev, struct ocrdma_create_cq_ureq ureq; if (attr->flags) - return ERR_PTR(-EINVAL); + return -EINVAL; if (udata) { if (ib_copy_from_udata(&ureq, udata, sizeof(ureq))) - return ERR_PTR(-EFAULT); + return -EFAULT; } else ureq.dpp_cq = 0; - cq = kzalloc(sizeof(*cq), GFP_KERNEL); - if (!cq) - return ERR_PTR(-ENOMEM); spin_lock_init(&cq->cq_lock); spin_lock_init(&cq->comp_handler_lock); @@ -1011,10 +1007,9 @@ struct ib_cq *ocrdma_create_cq(struct ib_device *ibdev, pd_id = uctx->cntxt_pd->id; status = ocrdma_mbx_create_cq(dev, cq, entries, ureq.dpp_cq, pd_id); - if (status) { - kfree(cq); - return ERR_PTR(status); - } + if (status) + return status; + if (udata) { status = ocrdma_copy_cq_uresp(dev, cq, udata); if (status) @@ -1022,12 +1017,11 @@ struct ib_cq *ocrdma_create_cq(struct ib_device *ibdev, } cq->phase = OCRDMA_CQE_VALID; dev->cq_tbl[cq->id] = cq; - return &cq->ibcq; + return 0; ctx_err: ocrdma_mbx_destroy_cq(dev, cq); - kfree(cq); - return ERR_PTR(status); + return status; } int ocrdma_resize_cq(struct ib_cq *ibcq, int new_cnt, @@ -1070,7 +1064,7 @@ static void ocrdma_flush_cq(struct ocrdma_cq *cq) spin_unlock_irqrestore(&cq->cq_lock, flags); } -int ocrdma_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata) +void ocrdma_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata) { struct ocrdma_cq *cq = get_ocrdma_cq(ibcq); struct ocrdma_eq *eq = NULL; @@ -1080,14 +1074,13 @@ int ocrdma_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata) dev->cq_tbl[cq->id] = NULL; indx = ocrdma_get_eq_table_index(dev, cq->eqn); - BUG_ON(indx == -EINVAL); eq = &dev->eq_tbl[indx]; irq = ocrdma_get_irq(dev, eq); synchronize_irq(irq); ocrdma_flush_cq(cq); - (void)ocrdma_mbx_destroy_cq(dev, cq); + ocrdma_mbx_destroy_cq(dev, cq); if (cq->ucontext) { pdid = cq->ucontext->cntxt_pd->id; ocrdma_del_mmap(cq->ucontext, (u64) cq->pa, @@ -1096,9 +1089,6 @@ int ocrdma_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata) ocrdma_get_db_addr(dev, pdid), dev->nic_info.db_page_size); } - - kfree(cq); - return 0; } static int ocrdma_add_qpn_map(struct ocrdma_dev *dev, struct ocrdma_qp *qp) diff --git a/drivers/infiniband/hw/ocrdma/ocrdma_verbs.h b/drivers/infiniband/hw/ocrdma/ocrdma_verbs.h index d76aae7ed0d3..32488da1b752 100644 --- a/drivers/infiniband/hw/ocrdma/ocrdma_verbs.h +++ b/drivers/infiniband/hw/ocrdma/ocrdma_verbs.h @@ -71,11 +71,10 @@ int ocrdma_mmap(struct ib_ucontext *, struct vm_area_struct *vma); int ocrdma_alloc_pd(struct ib_pd *pd, struct ib_udata *udata); void ocrdma_dealloc_pd(struct ib_pd *pd, struct ib_udata *udata); -struct ib_cq *ocrdma_create_cq(struct ib_device *ibdev, - const struct ib_cq_init_attr *attr, - struct ib_udata *udata); +int ocrdma_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, + struct ib_udata *udata); int ocrdma_resize_cq(struct ib_cq *, int cqe, struct ib_udata *); -int ocrdma_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata); +void ocrdma_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata); struct ib_qp *ocrdma_create_qp(struct ib_pd *, struct ib_qp_init_attr *attrs, diff --git a/drivers/infiniband/hw/qedr/main.c b/drivers/infiniband/hw/qedr/main.c index 5ebf3c53b3fb..533157a2a3be 100644 --- a/drivers/infiniband/hw/qedr/main.c +++ b/drivers/infiniband/hw/qedr/main.c @@ -183,6 +183,10 @@ static void qedr_roce_register_device(struct qedr_dev *dev) } static const struct ib_device_ops qedr_dev_ops = { + .owner = THIS_MODULE, + .driver_id = RDMA_DRIVER_QEDR, + .uverbs_abi_ver = QEDR_ABI_VERSION, + .alloc_mr = qedr_alloc_mr, .alloc_pd = qedr_alloc_pd, .alloc_ucontext = qedr_alloc_ucontext, @@ -220,6 +224,7 @@ static const struct ib_device_ops qedr_dev_ops = { .resize_cq = qedr_resize_cq, INIT_RDMA_OBJ_SIZE(ib_ah, qedr_ah, ibah), + INIT_RDMA_OBJ_SIZE(ib_cq, qedr_cq, ibcq), INIT_RDMA_OBJ_SIZE(ib_pd, qedr_pd, ibpd), INIT_RDMA_OBJ_SIZE(ib_srq, qedr_srq, ibsrq), INIT_RDMA_OBJ_SIZE(ib_ucontext, qedr_ucontext, ibucontext), @@ -231,8 +236,6 @@ static int qedr_register_device(struct qedr_dev *dev) dev->ibdev.node_guid = dev->attr.node_guid; memcpy(dev->ibdev.node_desc, QEDR_NODE_DESC, sizeof(QEDR_NODE_DESC)); - dev->ibdev.owner = THIS_MODULE; - dev->ibdev.uverbs_abi_ver = QEDR_ABI_VERSION; dev->ibdev.uverbs_cmd_mask = QEDR_UVERBS(GET_CONTEXT) | QEDR_UVERBS(QUERY_DEVICE) | @@ -274,7 +277,6 @@ static int qedr_register_device(struct qedr_dev *dev) rdma_set_device_sysfs_group(&dev->ibdev, &qedr_attr_group); ib_set_device_ops(&dev->ibdev, &qedr_dev_ops); - dev->ibdev.driver_id = RDMA_DRIVER_QEDR; rc = ib_device_set_netdev(&dev->ibdev, dev->ndev, 1); if (rc) return rc; diff --git a/drivers/infiniband/hw/qedr/verbs.c b/drivers/infiniband/hw/qedr/verbs.c index 3d7bde19838e..27d90a84ea01 100644 --- a/drivers/infiniband/hw/qedr/verbs.c +++ b/drivers/infiniband/hw/qedr/verbs.c @@ -159,54 +159,47 @@ int qedr_query_device(struct ib_device *ibdev, return 0; } -#define QEDR_SPEED_SDR (1) -#define QEDR_SPEED_DDR (2) -#define QEDR_SPEED_QDR (4) -#define QEDR_SPEED_FDR10 (8) -#define QEDR_SPEED_FDR (16) -#define QEDR_SPEED_EDR (32) - static inline void get_link_speed_and_width(int speed, u8 *ib_speed, u8 *ib_width) { switch (speed) { case 1000: - *ib_speed = QEDR_SPEED_SDR; + *ib_speed = IB_SPEED_SDR; *ib_width = IB_WIDTH_1X; break; case 10000: - *ib_speed = QEDR_SPEED_QDR; + *ib_speed = IB_SPEED_QDR; *ib_width = IB_WIDTH_1X; break; case 20000: - *ib_speed = QEDR_SPEED_DDR; + *ib_speed = IB_SPEED_DDR; *ib_width = IB_WIDTH_4X; break; case 25000: - *ib_speed = QEDR_SPEED_EDR; + *ib_speed = IB_SPEED_EDR; *ib_width = IB_WIDTH_1X; break; case 40000: - *ib_speed = QEDR_SPEED_QDR; + *ib_speed = IB_SPEED_QDR; *ib_width = IB_WIDTH_4X; break; case 50000: - *ib_speed = QEDR_SPEED_QDR; - *ib_width = IB_WIDTH_4X; + *ib_speed = IB_SPEED_HDR; + *ib_width = IB_WIDTH_1X; break; case 100000: - *ib_speed = QEDR_SPEED_EDR; + *ib_speed = IB_SPEED_EDR; *ib_width = IB_WIDTH_4X; break; default: /* Unsupported */ - *ib_speed = QEDR_SPEED_SDR; + *ib_speed = IB_SPEED_SDR; *ib_width = IB_WIDTH_1X; } } @@ -813,20 +806,20 @@ int qedr_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags) return 0; } -struct ib_cq *qedr_create_cq(struct ib_device *ibdev, - const struct ib_cq_init_attr *attr, - struct ib_udata *udata) +int qedr_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, + struct ib_udata *udata) { + struct ib_device *ibdev = ibcq->device; struct qedr_ucontext *ctx = rdma_udata_to_drv_context( udata, struct qedr_ucontext, ibucontext); struct qed_rdma_destroy_cq_out_params destroy_oparams; struct qed_rdma_destroy_cq_in_params destroy_iparams; struct qedr_dev *dev = get_qedr_dev(ibdev); struct qed_rdma_create_cq_in_params params; - struct qedr_create_cq_ureq ureq; + struct qedr_create_cq_ureq ureq = {}; int vector = attr->comp_vector; int entries = attr->cqe; - struct qedr_cq *cq; + struct qedr_cq *cq = get_qedr_cq(ibcq); int chain_entries; int page_cnt; u64 pbl_ptr; @@ -841,18 +834,13 @@ struct ib_cq *qedr_create_cq(struct ib_device *ibdev, DP_ERR(dev, "create cq: the number of entries %d is too high. Must be equal or below %d.\n", entries, QEDR_MAX_CQES); - return ERR_PTR(-EINVAL); + return -EINVAL; } chain_entries = qedr_align_cq_entries(entries); chain_entries = min_t(int, chain_entries, QEDR_MAX_CQES); - cq = kzalloc(sizeof(*cq), GFP_KERNEL); - if (!cq) - return ERR_PTR(-ENOMEM); - if (udata) { - memset(&ureq, 0, sizeof(ureq)); if (ib_copy_from_udata(&ureq, udata, sizeof(ureq))) { DP_ERR(dev, "create cq: problem copying data from user space\n"); @@ -930,7 +918,7 @@ struct ib_cq *qedr_create_cq(struct ib_device *ibdev, "create cq: icid=0x%0x, addr=%p, size(entries)=0x%0x\n", cq->icid, cq, params.cq_size); - return &cq->ibcq; + return 0; err3: destroy_iparams.icid = cq->icid; @@ -945,8 +933,7 @@ err1: if (udata) ib_umem_release(cq->q.umem); err0: - kfree(cq); - return ERR_PTR(-EINVAL); + return -EINVAL; } int qedr_resize_cq(struct ib_cq *ibcq, int new_cnt, struct ib_udata *udata) @@ -962,14 +949,13 @@ int qedr_resize_cq(struct ib_cq *ibcq, int new_cnt, struct ib_udata *udata) #define QEDR_DESTROY_CQ_MAX_ITERATIONS (10) #define QEDR_DESTROY_CQ_ITER_DURATION (10) -int qedr_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata) +void qedr_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata) { struct qedr_dev *dev = get_qedr_dev(ibcq->device); struct qed_rdma_destroy_cq_out_params oparams; struct qed_rdma_destroy_cq_in_params iparams; struct qedr_cq *cq = get_qedr_cq(ibcq); int iter; - int rc; DP_DEBUG(dev, QEDR_MSG_CQ, "destroy cq %p (icid=%d)\n", cq, cq->icid); @@ -977,13 +963,10 @@ int qedr_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata) /* GSIs CQs are handled by driver, so they don't exist in the FW */ if (cq->cq_type == QEDR_CQ_TYPE_GSI) - goto done; + return; iparams.icid = cq->icid; - rc = dev->ops->rdma_destroy_cq(dev->rdma_ctx, &iparams, &oparams); - if (rc) - return rc; - + dev->ops->rdma_destroy_cq(dev->rdma_ctx, &iparams, &oparams); dev->ops->common->chain_free(dev->cdev, &cq->pbl); if (udata) { @@ -1014,27 +997,11 @@ int qedr_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata) iter--; } - if (oparams.num_cq_notif != cq->cnq_notif) - goto err; - /* Note that we don't need to have explicit code to wait for the * completion of the event handler because it is invoked from the EQ. * Since the destroy CQ ramrod has also been received on the EQ we can * be certain that there's no event handler in process. */ -done: - cq->sig = ~cq->sig; - - kfree(cq); - - return 0; - -err: - DP_ERR(dev, - "CQ %p (icid=%d) not freed, expecting %d ints but got %d ints\n", - cq, cq->icid, oparams.num_cq_notif, cq->cnq_notif); - - return -EINVAL; } static inline int get_gid_info_from_table(struct ib_qp *ibqp, @@ -1605,12 +1572,10 @@ qedr_iwarp_populate_user_qp(struct qedr_dev *dev, static void qedr_cleanup_user(struct qedr_dev *dev, struct qedr_qp *qp) { - if (qp->usq.umem) - ib_umem_release(qp->usq.umem); + ib_umem_release(qp->usq.umem); qp->usq.umem = NULL; - if (qp->urq.umem) - ib_umem_release(qp->urq.umem); + ib_umem_release(qp->urq.umem); qp->urq.umem = NULL; } @@ -2713,8 +2678,7 @@ int qedr_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata) qedr_free_pbl(dev, &mr->info.pbl_info, mr->info.pbl_table); /* it could be user registered memory. */ - if (mr->umem) - ib_umem_release(mr->umem); + ib_umem_release(mr->umem); kfree(mr); diff --git a/drivers/infiniband/hw/qedr/verbs.h b/drivers/infiniband/hw/qedr/verbs.h index 9328c80375ef..9aaa90283d6e 100644 --- a/drivers/infiniband/hw/qedr/verbs.h +++ b/drivers/infiniband/hw/qedr/verbs.h @@ -50,11 +50,10 @@ int qedr_mmap(struct ib_ucontext *, struct vm_area_struct *vma); int qedr_alloc_pd(struct ib_pd *pd, struct ib_udata *udata); void qedr_dealloc_pd(struct ib_pd *pd, struct ib_udata *udata); -struct ib_cq *qedr_create_cq(struct ib_device *ibdev, - const struct ib_cq_init_attr *attr, - struct ib_udata *udata); +int qedr_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, + struct ib_udata *udata); int qedr_resize_cq(struct ib_cq *, int cqe, struct ib_udata *); -int qedr_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata); +void qedr_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata); int qedr_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags); struct ib_qp *qedr_create_qp(struct ib_pd *, struct ib_qp_init_attr *attrs, struct ib_udata *); diff --git a/drivers/infiniband/hw/qib/qib_qp.c b/drivers/infiniband/hw/qib/qib_qp.c index a81905df2d0f..8d0563ef5be1 100644 --- a/drivers/infiniband/hw/qib/qib_qp.c +++ b/drivers/infiniband/hw/qib/qib_qp.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2012 - 2017 Intel Corporation. All rights reserved. + * Copyright (c) 2012 - 2019 Intel Corporation. All rights reserved. * Copyright (c) 2006 - 2012 QLogic Corporation. * All rights reserved. * Copyright (c) 2005, 2006 PathScale, Inc. All rights reserved. * @@ -398,7 +398,7 @@ int qib_check_send_wqe(struct rvt_qp *qp, case IB_QPT_SMI: case IB_QPT_GSI: case IB_QPT_UD: - ah = ibah_to_rvtah(wqe->ud_wr.ah); + ah = rvt_get_swqe_ah(wqe); if (wqe->length > (1 << ah->log_pmtu)) return -EINVAL; /* progress hint */ diff --git a/drivers/infiniband/hw/qib/qib_rc.c b/drivers/infiniband/hw/qib/qib_rc.c index 2ac4c67f5ba1..1d5e2d4ee257 100644 --- a/drivers/infiniband/hw/qib/qib_rc.c +++ b/drivers/infiniband/hw/qib/qib_rc.c @@ -921,20 +921,11 @@ void qib_rc_send_complete(struct rvt_qp *qp, struct ib_header *hdr) rvt_add_retry_timer(qp); while (qp->s_last != qp->s_acked) { - u32 s_last; - wqe = rvt_get_swqe_ptr(qp, qp->s_last); if (qib_cmp24(wqe->lpsn, qp->s_sending_psn) >= 0 && qib_cmp24(qp->s_sending_psn, qp->s_sending_hpsn) <= 0) break; - s_last = qp->s_last; - if (++s_last >= qp->s_size) - s_last = 0; - qp->s_last = s_last; - /* see post_send() */ - barrier(); - rvt_put_qp_swqe(qp, wqe); - rvt_qp_swqe_complete(qp, + rvt_qp_complete_swqe(qp, wqe, ib_qib_wc_opcode[wqe->wr.opcode], IB_WC_SUCCESS); @@ -972,21 +963,12 @@ static struct rvt_swqe *do_rc_completion(struct rvt_qp *qp, * is finished. */ if (qib_cmp24(wqe->lpsn, qp->s_sending_psn) < 0 || - qib_cmp24(qp->s_sending_psn, qp->s_sending_hpsn) > 0) { - u32 s_last; - - rvt_put_qp_swqe(qp, wqe); - s_last = qp->s_last; - if (++s_last >= qp->s_size) - s_last = 0; - qp->s_last = s_last; - /* see post_send() */ - barrier(); - rvt_qp_swqe_complete(qp, + qib_cmp24(qp->s_sending_psn, qp->s_sending_hpsn) > 0) + rvt_qp_complete_swqe(qp, wqe, ib_qib_wc_opcode[wqe->wr.opcode], IB_WC_SUCCESS); - } else + else this_cpu_inc(*ibp->rvp.rc_delayed_comp); qp->s_retry = qp->s_retry_cnt; @@ -1909,8 +1891,7 @@ send_last: wc.dlid_path_bits = 0; wc.port_num = 0; /* Signal completion event if the solicited bit is set. */ - rvt_cq_enter(ibcq_to_rvtcq(qp->ibqp.recv_cq), &wc, - ib_bth_is_solicited(ohdr)); + rvt_recv_cq(qp, &wc, ib_bth_is_solicited(ohdr)); break; case OP(RDMA_WRITE_FIRST): diff --git a/drivers/infiniband/hw/qib/qib_uc.c b/drivers/infiniband/hw/qib/qib_uc.c index 30c70ad0f4bf..e17b91e2c22a 100644 --- a/drivers/infiniband/hw/qib/qib_uc.c +++ b/drivers/infiniband/hw/qib/qib_uc.c @@ -400,8 +400,7 @@ last_imm: wc.dlid_path_bits = 0; wc.port_num = 0; /* Signal completion event if the solicited bit is set. */ - rvt_cq_enter(ibcq_to_rvtcq(qp->ibqp.recv_cq), &wc, - ib_bth_is_solicited(ohdr)); + rvt_recv_cq(qp, &wc, ib_bth_is_solicited(ohdr)); break; case OP(RDMA_WRITE_FIRST): diff --git a/drivers/infiniband/hw/qib/qib_ud.c b/drivers/infiniband/hw/qib/qib_ud.c index 5cdedba2d164..93ca21347959 100644 --- a/drivers/infiniband/hw/qib/qib_ud.c +++ b/drivers/infiniband/hw/qib/qib_ud.c @@ -1,4 +1,5 @@ /* + * Copyright (c) 2012 - 2019 Intel Corporation. All rights reserved. * Copyright (c) 2006, 2007, 2008, 2009 QLogic Corporation. All rights reserved. * Copyright (c) 2005, 2006 PathScale, Inc. All rights reserved. * @@ -63,7 +64,7 @@ static void qib_ud_loopback(struct rvt_qp *sqp, struct rvt_swqe *swqe) enum ib_qp_type sqptype, dqptype; rcu_read_lock(); - qp = rvt_lookup_qpn(rdi, &ibp->rvp, swqe->ud_wr.remote_qpn); + qp = rvt_lookup_qpn(rdi, &ibp->rvp, rvt_get_swqe_remote_qpn(swqe)); if (!qp) { ibp->rvp.n_pkt_drops++; goto drop; @@ -80,7 +81,7 @@ static void qib_ud_loopback(struct rvt_qp *sqp, struct rvt_swqe *swqe) goto drop; } - ah_attr = &ibah_to_rvtah(swqe->ud_wr.ah)->attr; + ah_attr = rvt_get_swqe_ah_attr(swqe); ppd = ppd_from_ibp(ibp); if (qp->ibqp.qp_num > 1) { @@ -110,8 +111,8 @@ static void qib_ud_loopback(struct rvt_qp *sqp, struct rvt_swqe *swqe) if (qp->ibqp.qp_num) { u32 qkey; - qkey = (int)swqe->ud_wr.remote_qkey < 0 ? - sqp->qkey : swqe->ud_wr.remote_qkey; + qkey = (int)rvt_get_swqe_remote_qkey(swqe) < 0 ? + sqp->qkey : rvt_get_swqe_remote_qkey(swqe); if (unlikely(qkey != qp->qkey)) goto drop; } @@ -203,15 +204,14 @@ static void qib_ud_loopback(struct rvt_qp *sqp, struct rvt_swqe *swqe) wc.qp = &qp->ibqp; wc.src_qp = sqp->ibqp.qp_num; wc.pkey_index = qp->ibqp.qp_type == IB_QPT_GSI ? - swqe->ud_wr.pkey_index : 0; + rvt_get_swqe_pkey_index(swqe) : 0; wc.slid = ppd->lid | (rdma_ah_get_path_bits(ah_attr) & ((1 << ppd->lmc) - 1)); wc.sl = rdma_ah_get_sl(ah_attr); wc.dlid_path_bits = rdma_ah_get_dlid(ah_attr) & ((1 << ppd->lmc) - 1); wc.port_num = qp->port_num; /* Signal completion event if the solicited bit is set. */ - rvt_cq_enter(ibcq_to_rvtcq(qp->ibqp.recv_cq), &wc, - swqe->wr.send_flags & IB_SEND_SOLICITED); + rvt_recv_cq(qp, &wc, swqe->wr.send_flags & IB_SEND_SOLICITED); ibp->rvp.n_loop_pkts++; bail_unlock: spin_unlock_irqrestore(&qp->r_lock, flags); @@ -271,7 +271,7 @@ int qib_make_ud_req(struct rvt_qp *qp, unsigned long *flags) /* Construct the header. */ ibp = to_iport(qp->ibqp.device, qp->port_num); ppd = ppd_from_ibp(ibp); - ah_attr = &ibah_to_rvtah(wqe->ud_wr.ah)->attr; + ah_attr = rvt_get_swqe_ah_attr(wqe); if (rdma_ah_get_dlid(ah_attr) >= be16_to_cpu(IB_MULTICAST_LID_BASE)) { if (rdma_ah_get_dlid(ah_attr) != be16_to_cpu(IB_LID_PERMISSIVE)) @@ -363,7 +363,7 @@ int qib_make_ud_req(struct rvt_qp *qp, unsigned long *flags) bth0 |= extra_bytes << 20; bth0 |= qp->ibqp.qp_type == IB_QPT_SMI ? QIB_DEFAULT_P_KEY : qib_get_pkey(ibp, qp->ibqp.qp_type == IB_QPT_GSI ? - wqe->ud_wr.pkey_index : qp->s_pkey_index); + rvt_get_swqe_pkey_index(wqe) : qp->s_pkey_index); ohdr->bth[0] = cpu_to_be32(bth0); /* * Use the multicast QP if the destination LID is a multicast LID. @@ -372,14 +372,15 @@ int qib_make_ud_req(struct rvt_qp *qp, unsigned long *flags) be16_to_cpu(IB_MULTICAST_LID_BASE) && rdma_ah_get_dlid(ah_attr) != be16_to_cpu(IB_LID_PERMISSIVE) ? cpu_to_be32(QIB_MULTICAST_QPN) : - cpu_to_be32(wqe->ud_wr.remote_qpn); + cpu_to_be32(rvt_get_swqe_remote_qpn(wqe)); ohdr->bth[2] = cpu_to_be32(wqe->psn & QIB_PSN_MASK); /* * Qkeys with the high order bit set mean use the * qkey from the QP context instead of the WR (see 10.2.5). */ - ohdr->u.ud.deth[0] = cpu_to_be32((int)wqe->ud_wr.remote_qkey < 0 ? - qp->qkey : wqe->ud_wr.remote_qkey); + ohdr->u.ud.deth[0] = + cpu_to_be32((int)rvt_get_swqe_remote_qkey(wqe) < 0 ? qp->qkey : + rvt_get_swqe_remote_qkey(wqe)); ohdr->u.ud.deth[1] = cpu_to_be32(qp->ibqp.qp_num); done: @@ -573,8 +574,7 @@ void qib_ud_rcv(struct qib_ibport *ibp, struct ib_header *hdr, dlid & ((1 << ppd_from_ibp(ibp)->lmc) - 1); wc.port_num = qp->port_num; /* Signal completion event if the solicited bit is set. */ - rvt_cq_enter(ibcq_to_rvtcq(qp->ibqp.recv_cq), &wc, - ib_bth_is_solicited(ohdr)); + rvt_recv_cq(qp, &wc, ib_bth_is_solicited(ohdr)); return; drop: diff --git a/drivers/infiniband/hw/qib/qib_user_pages.c b/drivers/infiniband/hw/qib/qib_user_pages.c index f712fb7fa82f..bfbfbb7e0ff4 100644 --- a/drivers/infiniband/hw/qib/qib_user_pages.c +++ b/drivers/infiniband/hw/qib/qib_user_pages.c @@ -40,13 +40,10 @@ static void __qib_release_user_pages(struct page **p, size_t num_pages, int dirty) { - size_t i; - - for (i = 0; i < num_pages; i++) { - if (dirty) - set_page_dirty_lock(p[i]); - put_page(p[i]); - } + if (dirty) + put_user_pages_dirty_lock(p, num_pages); + else + put_user_pages(p, num_pages); } /** diff --git a/drivers/infiniband/hw/qib/qib_user_sdma.c b/drivers/infiniband/hw/qib/qib_user_sdma.c index 0c204776263f..05190edc2611 100644 --- a/drivers/infiniband/hw/qib/qib_user_sdma.c +++ b/drivers/infiniband/hw/qib/qib_user_sdma.c @@ -317,7 +317,7 @@ static int qib_user_sdma_page_to_frags(const struct qib_devdata *dd, * the caller can ignore this page. */ if (put) { - put_page(page); + put_user_page(page); } else { /* coalesce case */ kunmap(page); @@ -631,7 +631,7 @@ static void qib_user_sdma_free_pkt_frag(struct device *dev, kunmap(pkt->addr[i].page); if (pkt->addr[i].put_page) - put_page(pkt->addr[i].page); + put_user_page(pkt->addr[i].page); else __free_page(pkt->addr[i].page); } else if (pkt->addr[i].kvaddr) { @@ -706,7 +706,7 @@ static int qib_user_sdma_pin_pages(const struct qib_devdata *dd, /* if error, return all pages not managed by pkt */ free_pages: while (i < j) - put_page(pages[i++]); + put_user_page(pages[i++]); done: return ret; @@ -904,10 +904,11 @@ static int qib_user_sdma_queue_pkts(const struct qib_devdata *dd, } if (frag_size) { - int pktsize, tidsmsize, n; + int tidsmsize, n; + size_t pktsize; n = npages*((2*PAGE_SIZE/frag_size)+1); - pktsize = sizeof(*pkt) + sizeof(pkt->addr[0])*n; + pktsize = struct_size(pkt, addr, n); /* * Determine if this is tid-sdma or just sdma. diff --git a/drivers/infiniband/hw/qib/qib_verbs.c b/drivers/infiniband/hw/qib/qib_verbs.c index 2c4e569ce438..33778d451b82 100644 --- a/drivers/infiniband/hw/qib/qib_verbs.c +++ b/drivers/infiniband/hw/qib/qib_verbs.c @@ -1480,6 +1480,9 @@ static void qib_fill_device_attr(struct qib_devdata *dd) } static const struct ib_device_ops qib_dev_ops = { + .owner = THIS_MODULE, + .driver_id = RDMA_DRIVER_QIB, + .init_port = qib_create_port_files, .modify_device = qib_modify_device, .process_mad = qib_process_mad, @@ -1543,7 +1546,6 @@ int qib_register_ib_device(struct qib_devdata *dd) if (!ib_qib_sys_image_guid) ib_qib_sys_image_guid = ppd->guid; - ibdev->owner = THIS_MODULE; ibdev->node_guid = ppd->guid; ibdev->phys_port_cnt = dd->num_pports; ibdev->dev.parent = &dd->pcidev->dev; @@ -1614,7 +1616,7 @@ int qib_register_ib_device(struct qib_devdata *dd) rdma_set_device_sysfs_group(&dd->verbs_dev.rdi.ibdev, &qib_attr_group); ib_set_device_ops(ibdev, &qib_dev_ops); - ret = rvt_register_device(&dd->verbs_dev.rdi, RDMA_DRIVER_QIB); + ret = rvt_register_device(&dd->verbs_dev.rdi); if (ret) goto err_tx; diff --git a/drivers/infiniband/hw/usnic/usnic_ib.h b/drivers/infiniband/hw/usnic/usnic_ib.h index 525bf272671e..84dd682d2334 100644 --- a/drivers/infiniband/hw/usnic/usnic_ib.h +++ b/drivers/infiniband/hw/usnic/usnic_ib.h @@ -61,6 +61,10 @@ struct usnic_ib_pd { struct usnic_uiom_pd *umem_pd; }; +struct usnic_ib_cq { + struct ib_cq ibcq; +}; + struct usnic_ib_mr { struct ib_mr ibmr; struct usnic_uiom_reg *umem; diff --git a/drivers/infiniband/hw/usnic/usnic_ib_main.c b/drivers/infiniband/hw/usnic/usnic_ib_main.c index 34c1f9d6c915..03f54eb9404b 100644 --- a/drivers/infiniband/hw/usnic/usnic_ib_main.c +++ b/drivers/infiniband/hw/usnic/usnic_ib_main.c @@ -329,6 +329,10 @@ static void usnic_get_dev_fw_str(struct ib_device *device, char *str) } static const struct ib_device_ops usnic_dev_ops = { + .owner = THIS_MODULE, + .driver_id = RDMA_DRIVER_USNIC, + .uverbs_abi_ver = USNIC_UVERBS_ABI_VERSION, + .alloc_pd = usnic_ib_alloc_pd, .alloc_ucontext = usnic_ib_alloc_ucontext, .create_cq = usnic_ib_create_cq, @@ -350,6 +354,7 @@ static const struct ib_device_ops usnic_dev_ops = { .query_qp = usnic_ib_query_qp, .reg_user_mr = usnic_ib_reg_mr, INIT_RDMA_OBJ_SIZE(ib_pd, usnic_ib_pd, ibpd), + INIT_RDMA_OBJ_SIZE(ib_cq, usnic_ib_cq, ibcq), INIT_RDMA_OBJ_SIZE(ib_ucontext, usnic_ib_ucontext, ibucontext), }; @@ -384,12 +389,10 @@ static void *usnic_ib_device_add(struct pci_dev *dev) us_ibdev->pdev = dev; us_ibdev->netdev = pci_get_drvdata(dev); - us_ibdev->ib_dev.owner = THIS_MODULE; us_ibdev->ib_dev.node_type = RDMA_NODE_USNIC_UDP; us_ibdev->ib_dev.phys_port_cnt = USNIC_IB_PORT_CNT; us_ibdev->ib_dev.num_comp_vectors = USNIC_IB_NUM_COMP_VECTORS; us_ibdev->ib_dev.dev.parent = &dev->dev; - us_ibdev->ib_dev.uverbs_abi_ver = USNIC_UVERBS_ABI_VERSION; us_ibdev->ib_dev.uverbs_cmd_mask = (1ull << IB_USER_VERBS_CMD_GET_CONTEXT) | @@ -412,7 +415,6 @@ static void *usnic_ib_device_add(struct pci_dev *dev) ib_set_device_ops(&us_ibdev->ib_dev, &usnic_dev_ops); - us_ibdev->ib_dev.driver_id = RDMA_DRIVER_USNIC; rdma_set_device_sysfs_group(&us_ibdev->ib_dev, &usnic_attr_group); ret = ib_device_set_netdev(&us_ibdev->ib_dev, us_ibdev->netdev, 1); diff --git a/drivers/infiniband/hw/usnic/usnic_ib_verbs.c b/drivers/infiniband/hw/usnic/usnic_ib_verbs.c index e9352750e029..eeb07b245ef9 100644 --- a/drivers/infiniband/hw/usnic/usnic_ib_verbs.c +++ b/drivers/infiniband/hw/usnic/usnic_ib_verbs.c @@ -587,28 +587,18 @@ out_unlock: return status; } -struct ib_cq *usnic_ib_create_cq(struct ib_device *ibdev, - const struct ib_cq_init_attr *attr, - struct ib_udata *udata) +int usnic_ib_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, + struct ib_udata *udata) { - struct ib_cq *cq; - - usnic_dbg("\n"); if (attr->flags) - return ERR_PTR(-EINVAL); - - cq = kzalloc(sizeof(*cq), GFP_KERNEL); - if (!cq) - return ERR_PTR(-EBUSY); + return -EINVAL; - return cq; + return 0; } -int usnic_ib_destroy_cq(struct ib_cq *cq, struct ib_udata *udata) +void usnic_ib_destroy_cq(struct ib_cq *cq, struct ib_udata *udata) { - usnic_dbg("\n"); - kfree(cq); - return 0; + return; } struct ib_mr *usnic_ib_reg_mr(struct ib_pd *pd, u64 start, u64 length, diff --git a/drivers/infiniband/hw/usnic/usnic_ib_verbs.h b/drivers/infiniband/hw/usnic/usnic_ib_verbs.h index 028f322f8e9b..2aedf78c13cf 100644 --- a/drivers/infiniband/hw/usnic/usnic_ib_verbs.h +++ b/drivers/infiniband/hw/usnic/usnic_ib_verbs.h @@ -58,10 +58,9 @@ struct ib_qp *usnic_ib_create_qp(struct ib_pd *pd, int usnic_ib_destroy_qp(struct ib_qp *qp, struct ib_udata *udata); int usnic_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr, int attr_mask, struct ib_udata *udata); -struct ib_cq *usnic_ib_create_cq(struct ib_device *ibdev, - const struct ib_cq_init_attr *attr, - struct ib_udata *udata); -int usnic_ib_destroy_cq(struct ib_cq *cq, struct ib_udata *udata); +int usnic_ib_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, + struct ib_udata *udata); +void usnic_ib_destroy_cq(struct ib_cq *cq, struct ib_udata *udata); struct ib_mr *usnic_ib_reg_mr(struct ib_pd *pd, u64 start, u64 length, u64 virt_addr, int access_flags, struct ib_udata *udata); diff --git a/drivers/infiniband/hw/usnic/usnic_uiom.c b/drivers/infiniband/hw/usnic/usnic_uiom.c index e312f522a66d..0b0237d41613 100644 --- a/drivers/infiniband/hw/usnic/usnic_uiom.c +++ b/drivers/infiniband/hw/usnic/usnic_uiom.c @@ -75,9 +75,10 @@ static void usnic_uiom_put_pages(struct list_head *chunk_list, int dirty) for_each_sg(chunk->page_list, sg, chunk->nents, i) { page = sg_page(sg); pa = sg_phys(sg); - if (!PageDirty(page) && dirty) - set_page_dirty_lock(page); - put_page(page); + if (dirty) + put_user_pages_dirty_lock(&page, 1); + else + put_user_page(page); usnic_dbg("pa: %pa\n", &pa); } kfree(chunk); diff --git a/drivers/infiniband/hw/vmw_pvrdma/pvrdma.h b/drivers/infiniband/hw/vmw_pvrdma/pvrdma.h index 3c633ab58052..c142f5e7f25f 100644 --- a/drivers/infiniband/hw/vmw_pvrdma/pvrdma.h +++ b/drivers/infiniband/hw/vmw_pvrdma/pvrdma.h @@ -456,7 +456,7 @@ static inline enum pvrdma_wr_opcode ib_wr_opcode_to_pvrdma(enum ib_wr_opcode op) return PVRDMA_WR_MASKED_ATOMIC_CMP_AND_SWP; case IB_WR_MASKED_ATOMIC_FETCH_AND_ADD: return PVRDMA_WR_MASKED_ATOMIC_FETCH_AND_ADD; - case IB_WR_REG_SIG_MR: + case IB_WR_REG_MR_INTEGRITY: return PVRDMA_WR_REG_SIG_MR; default: return PVRDMA_WR_ERROR; diff --git a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_cq.c b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_cq.c index d7deb19a2800..7800e6930502 100644 --- a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_cq.c +++ b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_cq.c @@ -92,20 +92,19 @@ int pvrdma_req_notify_cq(struct ib_cq *ibcq, /** * pvrdma_create_cq - create completion queue - * @ibdev: the device + * @ibcq: Allocated CQ * @attr: completion queue attributes * @udata: user data * - * @return: ib_cq completion queue pointer on success, - * otherwise returns negative errno. + * @return: 0 on success */ -struct ib_cq *pvrdma_create_cq(struct ib_device *ibdev, - const struct ib_cq_init_attr *attr, - struct ib_udata *udata) +int pvrdma_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, + struct ib_udata *udata) { + struct ib_device *ibdev = ibcq->device; int entries = attr->cqe; struct pvrdma_dev *dev = to_vdev(ibdev); - struct pvrdma_cq *cq; + struct pvrdma_cq *cq = to_vcq(ibcq); int ret; int npages; unsigned long flags; @@ -113,7 +112,7 @@ struct ib_cq *pvrdma_create_cq(struct ib_device *ibdev, union pvrdma_cmd_resp rsp; struct pvrdma_cmd_create_cq *cmd = &req.create_cq; struct pvrdma_cmd_create_cq_resp *resp = &rsp.create_cq_resp; - struct pvrdma_create_cq_resp cq_resp = {0}; + struct pvrdma_create_cq_resp cq_resp = {}; struct pvrdma_create_cq ucmd; struct pvrdma_ucontext *context = rdma_udata_to_drv_context( udata, struct pvrdma_ucontext, ibucontext); @@ -122,16 +121,10 @@ struct ib_cq *pvrdma_create_cq(struct ib_device *ibdev, entries = roundup_pow_of_two(entries); if (entries < 1 || entries > dev->dsr->caps.max_cqe) - return ERR_PTR(-EINVAL); + return -EINVAL; if (!atomic_add_unless(&dev->num_cqs, 1, dev->dsr->caps.max_cq)) - return ERR_PTR(-ENOMEM); - - cq = kzalloc(sizeof(*cq), GFP_KERNEL); - if (!cq) { - atomic_dec(&dev->num_cqs); - return ERR_PTR(-ENOMEM); - } + return -ENOMEM; cq->ibcq.cqe = entries; cq->is_kernel = !udata; @@ -211,22 +204,19 @@ struct ib_cq *pvrdma_create_cq(struct ib_device *ibdev, dev_warn(&dev->pdev->dev, "failed to copy back udata\n"); pvrdma_destroy_cq(&cq->ibcq, udata); - return ERR_PTR(-EINVAL); + return -EINVAL; } } - return &cq->ibcq; + return 0; err_page_dir: pvrdma_page_dir_cleanup(dev, &cq->pdir); err_umem: - if (!cq->is_kernel) - ib_umem_release(cq->umem); + ib_umem_release(cq->umem); err_cq: atomic_dec(&dev->num_cqs); - kfree(cq); - - return ERR_PTR(ret); + return ret; } static void pvrdma_free_cq(struct pvrdma_dev *dev, struct pvrdma_cq *cq) @@ -235,21 +225,17 @@ static void pvrdma_free_cq(struct pvrdma_dev *dev, struct pvrdma_cq *cq) complete(&cq->free); wait_for_completion(&cq->free); - if (!cq->is_kernel) - ib_umem_release(cq->umem); + ib_umem_release(cq->umem); pvrdma_page_dir_cleanup(dev, &cq->pdir); - kfree(cq); } /** * pvrdma_destroy_cq - destroy completion queue * @cq: the completion queue to destroy. * @udata: user data or null for kernel object - * - * @return: 0 for success. */ -int pvrdma_destroy_cq(struct ib_cq *cq, struct ib_udata *udata) +void pvrdma_destroy_cq(struct ib_cq *cq, struct ib_udata *udata) { struct pvrdma_cq *vcq = to_vcq(cq); union pvrdma_cmd_req req; @@ -275,8 +261,6 @@ int pvrdma_destroy_cq(struct ib_cq *cq, struct ib_udata *udata) pvrdma_free_cq(dev, vcq); atomic_dec(&dev->num_cqs); - - return ret; } static inline struct pvrdma_cqe *get_cqe(struct pvrdma_cq *cq, int i) diff --git a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_main.c b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_main.c index 40182297f87f..e580ae9cc55a 100644 --- a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_main.c +++ b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_main.c @@ -144,6 +144,10 @@ static int pvrdma_port_immutable(struct ib_device *ibdev, u8 port_num, } static const struct ib_device_ops pvrdma_dev_ops = { + .owner = THIS_MODULE, + .driver_id = RDMA_DRIVER_VMW_PVRDMA, + .uverbs_abi_ver = PVRDMA_UVERBS_ABI_VERSION, + .add_gid = pvrdma_add_gid, .alloc_mr = pvrdma_alloc_mr, .alloc_pd = pvrdma_alloc_pd, @@ -178,6 +182,7 @@ static const struct ib_device_ops pvrdma_dev_ops = { .req_notify_cq = pvrdma_req_notify_cq, INIT_RDMA_OBJ_SIZE(ib_ah, pvrdma_ah, ibah), + INIT_RDMA_OBJ_SIZE(ib_cq, pvrdma_cq, ibcq), INIT_RDMA_OBJ_SIZE(ib_pd, pvrdma_pd, ibpd), INIT_RDMA_OBJ_SIZE(ib_ucontext, pvrdma_ucontext, ibucontext), }; @@ -198,10 +203,8 @@ static int pvrdma_register_device(struct pvrdma_dev *dev) dev->ib_dev.node_guid = dev->dsr->caps.node_guid; dev->sys_image_guid = dev->dsr->caps.sys_image_guid; dev->flags = 0; - dev->ib_dev.owner = THIS_MODULE; dev->ib_dev.num_comp_vectors = 1; dev->ib_dev.dev.parent = &dev->pdev->dev; - dev->ib_dev.uverbs_abi_ver = PVRDMA_UVERBS_ABI_VERSION; dev->ib_dev.uverbs_cmd_mask = (1ull << IB_USER_VERBS_CMD_GET_CONTEXT) | (1ull << IB_USER_VERBS_CMD_QUERY_DEVICE) | @@ -261,7 +264,6 @@ static int pvrdma_register_device(struct pvrdma_dev *dev) if (!dev->srq_tbl) goto err_qp_free; } - dev->ib_dev.driver_id = RDMA_DRIVER_VMW_PVRDMA; ret = ib_device_set_netdev(&dev->ib_dev, dev->netdev, 1); if (ret) return ret; diff --git a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_mr.c b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_mr.c index 65dc47ffb8f3..f3a3d22ee8d7 100644 --- a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_mr.c +++ b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_mr.c @@ -290,8 +290,7 @@ int pvrdma_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata) "could not deregister mem region, error: %d\n", ret); pvrdma_page_dir_cleanup(dev, &mr->pdir); - if (mr->umem) - ib_umem_release(mr->umem); + ib_umem_release(mr->umem); kfree(mr->pages); kfree(mr); diff --git a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c index 0eaaead5baec..bca6a58a442e 100644 --- a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c +++ b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_qp.c @@ -391,12 +391,8 @@ struct ib_qp *pvrdma_create_qp(struct ib_pd *pd, err_pdir: pvrdma_page_dir_cleanup(dev, &qp->pdir); err_umem: - if (!qp->is_kernel) { - if (qp->rumem) - ib_umem_release(qp->rumem); - if (qp->sumem) - ib_umem_release(qp->sumem); - } + ib_umem_release(qp->rumem); + ib_umem_release(qp->sumem); err_qp: kfree(qp); atomic_dec(&dev->num_qps); @@ -429,12 +425,8 @@ static void pvrdma_free_qp(struct pvrdma_qp *qp) complete(&qp->free); wait_for_completion(&qp->free); - if (!qp->is_kernel) { - if (qp->rumem) - ib_umem_release(qp->rumem); - if (qp->sumem) - ib_umem_release(qp->sumem); - } + ib_umem_release(qp->rumem); + ib_umem_release(qp->sumem); pvrdma_page_dir_cleanup(dev, &qp->pdir); diff --git a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.h b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.h index 9d7b021e1c59..e4a48f5c0c85 100644 --- a/drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.h +++ b/drivers/infiniband/hw/vmw_pvrdma/pvrdma_verbs.h @@ -409,10 +409,9 @@ struct ib_mr *pvrdma_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type, u32 max_num_sg, struct ib_udata *udata); int pvrdma_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents, unsigned int *sg_offset); -struct ib_cq *pvrdma_create_cq(struct ib_device *ibdev, - const struct ib_cq_init_attr *attr, - struct ib_udata *udata); -int pvrdma_destroy_cq(struct ib_cq *cq, struct ib_udata *udata); +int pvrdma_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr, + struct ib_udata *udata); +void pvrdma_destroy_cq(struct ib_cq *cq, struct ib_udata *udata); int pvrdma_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc); int pvrdma_req_notify_cq(struct ib_cq *cq, enum ib_cq_notify_flags flags); int pvrdma_create_ah(struct ib_ah *ah, struct rdma_ah_attr *ah_attr, u32 flags, |