diff options
Diffstat (limited to 'drivers/net/ethernet/pensando')
| -rw-r--r-- | drivers/net/ethernet/pensando/Kconfig | 1 | ||||
| -rw-r--r-- | drivers/net/ethernet/pensando/ionic/ionic_debugfs.c | 2 | ||||
| -rw-r--r-- | drivers/net/ethernet/pensando/ionic/ionic_dev.h | 25 | ||||
| -rw-r--r-- | drivers/net/ethernet/pensando/ionic/ionic_ethtool.c | 2 | ||||
| -rw-r--r-- | drivers/net/ethernet/pensando/ionic/ionic_lif.c | 161 | ||||
| -rw-r--r-- | drivers/net/ethernet/pensando/ionic/ionic_lif.h | 2 | ||||
| -rw-r--r-- | drivers/net/ethernet/pensando/ionic/ionic_rx_filter.c | 4 | ||||
| -rw-r--r-- | drivers/net/ethernet/pensando/ionic/ionic_txrx.c | 420 | ||||
| -rw-r--r-- | drivers/net/ethernet/pensando/ionic/ionic_txrx.h | 4 |
9 files changed, 335 insertions, 286 deletions
diff --git a/drivers/net/ethernet/pensando/Kconfig b/drivers/net/ethernet/pensando/Kconfig index 3f7519e435b8..01fe76786f77 100644 --- a/drivers/net/ethernet/pensando/Kconfig +++ b/drivers/net/ethernet/pensando/Kconfig @@ -23,6 +23,7 @@ config IONIC depends on PTP_1588_CLOCK_OPTIONAL select NET_DEVLINK select DIMLIB + select PAGE_POOL help This enables the support for the Pensando family of Ethernet adapters. More specific information on this driver can be diff --git a/drivers/net/ethernet/pensando/ionic/ionic_debugfs.c b/drivers/net/ethernet/pensando/ionic/ionic_debugfs.c index 59e5a9f21105..c98b4e75e288 100644 --- a/drivers/net/ethernet/pensando/ionic/ionic_debugfs.c +++ b/drivers/net/ethernet/pensando/ionic/ionic_debugfs.c @@ -123,7 +123,7 @@ void ionic_debugfs_add_qcq(struct ionic_lif *lif, struct ionic_qcq *qcq) struct ionic_cq *cq = &qcq->cq; qcq_dentry = debugfs_create_dir(q->name, lif->dentry); - if (IS_ERR_OR_NULL(qcq_dentry)) + if (IS_ERR(qcq_dentry)) return; qcq->dentry = qcq_dentry; diff --git a/drivers/net/ethernet/pensando/ionic/ionic_dev.h b/drivers/net/ethernet/pensando/ionic/ionic_dev.h index c647033f3ad2..c8c710cfe70c 100644 --- a/drivers/net/ethernet/pensando/ionic/ionic_dev.h +++ b/drivers/net/ethernet/pensando/ionic/ionic_dev.h @@ -32,7 +32,7 @@ #define IONIC_ADMIN_DOORBELL_DEADLINE (HZ / 2) /* 500ms */ #define IONIC_TX_DOORBELL_DEADLINE (HZ / 100) /* 10ms */ #define IONIC_RX_MIN_DOORBELL_DEADLINE (HZ / 100) /* 10ms */ -#define IONIC_RX_MAX_DOORBELL_DEADLINE (HZ * 5) /* 5s */ +#define IONIC_RX_MAX_DOORBELL_DEADLINE (HZ * 4) /* 4s */ struct ionic_dev_bar { void __iomem *vaddr; @@ -181,10 +181,7 @@ struct ionic_queue; struct ionic_qcq; #define IONIC_MAX_BUF_LEN ((u16)-1) -#define IONIC_PAGE_SIZE PAGE_SIZE -#define IONIC_PAGE_SPLIT_SZ (PAGE_SIZE / 2) -#define IONIC_PAGE_GFP_MASK (GFP_ATOMIC | __GFP_NOWARN |\ - __GFP_COMP | __GFP_MEMALLOC) +#define IONIC_PAGE_SIZE MIN(PAGE_SIZE, IONIC_MAX_BUF_LEN) #define IONIC_XDP_MAX_LINEAR_MTU (IONIC_PAGE_SIZE - \ (VLAN_ETH_HLEN + \ @@ -238,9 +235,8 @@ struct ionic_queue { unsigned int index; unsigned int num_descs; unsigned int max_sg_elems; + u64 features; - unsigned int type; - unsigned int hw_index; unsigned int hw_type; bool xdp_flush; union { @@ -250,18 +246,23 @@ struct ionic_queue { struct ionic_admin_cmd *adminq; }; union { - void __iomem *cmb_base; - struct ionic_txq_desc __iomem *cmb_txq; - struct ionic_rxq_desc __iomem *cmb_rxq; - }; - union { void *sg_base; struct ionic_txq_sg_desc *txq_sgl; struct ionic_txq_sg_desc_v1 *txq_sgl_v1; struct ionic_rxq_sg_desc *rxq_sgl; }; struct xdp_rxq_info *xdp_rxq_info; + struct bpf_prog *xdp_prog; + struct page_pool *page_pool; struct ionic_queue *partner; + + union { + void __iomem *cmb_base; + struct ionic_txq_desc __iomem *cmb_txq; + struct ionic_rxq_desc __iomem *cmb_rxq; + }; + unsigned int type; + unsigned int hw_index; dma_addr_t base_pa; dma_addr_t cmb_base_pa; dma_addr_t sg_base_pa; diff --git a/drivers/net/ethernet/pensando/ionic/ionic_ethtool.c b/drivers/net/ethernet/pensando/ionic/ionic_ethtool.c index 4619fd74f3e3..dda22fa4448c 100644 --- a/drivers/net/ethernet/pensando/ionic/ionic_ethtool.c +++ b/drivers/net/ethernet/pensando/ionic/ionic_ethtool.c @@ -989,8 +989,6 @@ static int ionic_get_ts_info(struct net_device *netdev, info->phc_index = ptp_clock_index(lif->phc->ptp); info->so_timestamping = SOF_TIMESTAMPING_TX_SOFTWARE | - SOF_TIMESTAMPING_RX_SOFTWARE | - SOF_TIMESTAMPING_SOFTWARE | SOF_TIMESTAMPING_TX_HARDWARE | SOF_TIMESTAMPING_RX_HARDWARE | SOF_TIMESTAMPING_RAW_HARDWARE; diff --git a/drivers/net/ethernet/pensando/ionic/ionic_lif.c b/drivers/net/ethernet/pensando/ionic/ionic_lif.c index aa0cc31dfe6e..40496587b2b3 100644 --- a/drivers/net/ethernet/pensando/ionic/ionic_lif.c +++ b/drivers/net/ethernet/pensando/ionic/ionic_lif.c @@ -13,6 +13,7 @@ #include <linux/cpumask.h> #include <linux/crash_dump.h> #include <linux/vmalloc.h> +#include <net/page_pool/helpers.h> #include "ionic.h" #include "ionic_bus.h" @@ -46,8 +47,9 @@ static int ionic_start_queues(struct ionic_lif *lif); static void ionic_stop_queues(struct ionic_lif *lif); static void ionic_lif_queue_identify(struct ionic_lif *lif); -static int ionic_xdp_queues_config(struct ionic_lif *lif); -static void ionic_xdp_unregister_rxq_info(struct ionic_queue *q); +static void ionic_xdp_rxqs_prog_update(struct ionic_lif *lif); +static void ionic_unregister_rxq_info(struct ionic_queue *q); +static int ionic_register_rxq_info(struct ionic_queue *q, unsigned int napi_id); static void ionic_dim_work(struct work_struct *work) { @@ -380,6 +382,7 @@ static void ionic_lif_qcq_deinit(struct ionic_lif *lif, struct ionic_qcq *qcq) if (!(qcq->flags & IONIC_QCQ_F_INITED)) return; + ionic_unregister_rxq_info(&qcq->q); if (qcq->flags & IONIC_QCQ_F_INTR) { ionic_intr_mask(idev->intr_ctrl, qcq->intr.index, IONIC_INTR_MASK_SET); @@ -437,9 +440,10 @@ static void ionic_qcq_free(struct ionic_lif *lif, struct ionic_qcq *qcq) qcq->sg_base_pa = 0; } - ionic_xdp_unregister_rxq_info(&qcq->q); - ionic_qcq_intr_free(lif, qcq); + page_pool_destroy(qcq->q.page_pool); + qcq->q.page_pool = NULL; + ionic_qcq_intr_free(lif, qcq); vfree(qcq->q.info); qcq->q.info = NULL; } @@ -553,7 +557,8 @@ static int ionic_qcq_alloc(struct ionic_lif *lif, unsigned int type, unsigned int cq_desc_size, unsigned int sg_desc_size, unsigned int desc_info_size, - unsigned int pid, struct ionic_qcq **qcq) + unsigned int pid, struct bpf_prog *xdp_prog, + struct ionic_qcq **qcq) { struct ionic_dev *idev = &lif->ionic->idev; struct device *dev = lif->ionic->dev; @@ -579,6 +584,31 @@ static int ionic_qcq_alloc(struct ionic_lif *lif, unsigned int type, goto err_out_free_qcq; } + if (type == IONIC_QTYPE_RXQ) { + struct page_pool_params pp_params = { + .flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV, + .order = 0, + .pool_size = num_descs, + .nid = NUMA_NO_NODE, + .dev = lif->ionic->dev, + .napi = &new->napi, + .dma_dir = DMA_FROM_DEVICE, + .max_len = PAGE_SIZE, + .netdev = lif->netdev, + }; + + if (xdp_prog) + pp_params.dma_dir = DMA_BIDIRECTIONAL; + + new->q.page_pool = page_pool_create(&pp_params); + if (IS_ERR(new->q.page_pool)) { + netdev_err(lif->netdev, "Cannot create page_pool\n"); + err = PTR_ERR(new->q.page_pool); + new->q.page_pool = NULL; + goto err_out_free_q_info; + } + } + new->q.type = type; new->q.max_sg_elems = lif->qtype_info[type].max_sg_elems; @@ -586,12 +616,12 @@ static int ionic_qcq_alloc(struct ionic_lif *lif, unsigned int type, desc_size, sg_desc_size, pid); if (err) { netdev_err(lif->netdev, "Cannot initialize queue\n"); - goto err_out_free_q_info; + goto err_out_free_page_pool; } err = ionic_alloc_qcq_interrupt(lif, new); if (err) - goto err_out_free_q_info; + goto err_out_free_page_pool; err = ionic_cq_init(lif, &new->cq, &new->intr, num_descs, cq_desc_size); if (err) { @@ -712,6 +742,8 @@ err_out_free_irq: devm_free_irq(dev, new->intr.vector, &new->napi); ionic_intr_free(lif->ionic, new->intr.index); } +err_out_free_page_pool: + page_pool_destroy(new->q.page_pool); err_out_free_q_info: vfree(new->q.info); err_out_free_qcq: @@ -734,7 +766,7 @@ static int ionic_qcqs_alloc(struct ionic_lif *lif) sizeof(struct ionic_admin_comp), 0, sizeof(struct ionic_admin_desc_info), - lif->kern_pid, &lif->adminqcq); + lif->kern_pid, NULL, &lif->adminqcq); if (err) return err; ionic_debugfs_add_qcq(lif, lif->adminqcq); @@ -747,7 +779,7 @@ static int ionic_qcqs_alloc(struct ionic_lif *lif) sizeof(union ionic_notifyq_comp), 0, sizeof(struct ionic_admin_desc_info), - lif->kern_pid, &lif->notifyqcq); + lif->kern_pid, NULL, &lif->notifyqcq); if (err) goto err_out; ionic_debugfs_add_qcq(lif, lif->notifyqcq); @@ -925,6 +957,11 @@ static int ionic_lif_rxq_init(struct ionic_lif *lif, struct ionic_qcq *qcq) netif_napi_add(lif->netdev, &qcq->napi, ionic_rx_napi); else netif_napi_add(lif->netdev, &qcq->napi, ionic_txrx_napi); + err = ionic_register_rxq_info(q, qcq->napi.napi_id); + if (err) { + netif_napi_del(&qcq->napi); + return err; + } qcq->flags |= IONIC_QCQ_F_INITED; @@ -960,7 +997,7 @@ int ionic_lif_create_hwstamp_txq(struct ionic_lif *lif) err = ionic_qcq_alloc(lif, IONIC_QTYPE_TXQ, txq_i, "hwstamp_tx", flags, num_desc, desc_sz, comp_sz, sg_desc_sz, sizeof(struct ionic_tx_desc_info), - lif->kern_pid, &txq); + lif->kern_pid, NULL, &txq); if (err) goto err_qcq_alloc; @@ -1020,7 +1057,7 @@ int ionic_lif_create_hwstamp_rxq(struct ionic_lif *lif) err = ionic_qcq_alloc(lif, IONIC_QTYPE_RXQ, rxq_i, "hwstamp_rx", flags, num_desc, desc_sz, comp_sz, sg_desc_sz, sizeof(struct ionic_rx_desc_info), - lif->kern_pid, &rxq); + lif->kern_pid, NULL, &rxq); if (err) goto err_qcq_alloc; @@ -1037,7 +1074,7 @@ int ionic_lif_create_hwstamp_rxq(struct ionic_lif *lif) goto err_qcq_init; if (test_bit(IONIC_LIF_F_UP, lif->state)) { - ionic_rx_fill(&rxq->q); + ionic_rx_fill(&rxq->q, NULL); err = ionic_qcq_enable(rxq); if (err) goto err_qcq_enable; @@ -2046,7 +2083,7 @@ static int ionic_txrx_alloc(struct ionic_lif *lif) err = ionic_qcq_alloc(lif, IONIC_QTYPE_TXQ, i, "tx", flags, num_desc, desc_sz, comp_sz, sg_desc_sz, sizeof(struct ionic_tx_desc_info), - lif->kern_pid, &lif->txqcqs[i]); + lif->kern_pid, NULL, &lif->txqcqs[i]); if (err) goto err_out; @@ -2078,7 +2115,8 @@ static int ionic_txrx_alloc(struct ionic_lif *lif) err = ionic_qcq_alloc(lif, IONIC_QTYPE_RXQ, i, "rx", flags, num_desc, desc_sz, comp_sz, sg_desc_sz, sizeof(struct ionic_rx_desc_info), - lif->kern_pid, &lif->rxqcqs[i]); + lif->kern_pid, lif->xdp_prog, + &lif->rxqcqs[i]); if (err) goto err_out; @@ -2143,9 +2181,7 @@ static int ionic_txrx_enable(struct ionic_lif *lif) int derr = 0; int i, err; - err = ionic_xdp_queues_config(lif); - if (err) - return err; + ionic_xdp_rxqs_prog_update(lif); for (i = 0; i < lif->nxqs; i++) { if (!(lif->rxqcqs[i] && lif->txqcqs[i])) { @@ -2154,7 +2190,8 @@ static int ionic_txrx_enable(struct ionic_lif *lif) goto err_out; } - ionic_rx_fill(&lif->rxqcqs[i]->q); + ionic_rx_fill(&lif->rxqcqs[i]->q, + READ_ONCE(lif->rxqcqs[i]->q.xdp_prog)); err = ionic_qcq_enable(lif->rxqcqs[i]); if (err) goto err_out; @@ -2167,7 +2204,7 @@ static int ionic_txrx_enable(struct ionic_lif *lif) } if (lif->hwstamp_rxq) { - ionic_rx_fill(&lif->hwstamp_rxq->q); + ionic_rx_fill(&lif->hwstamp_rxq->q, NULL); err = ionic_qcq_enable(lif->hwstamp_rxq); if (err) goto err_out_hwstamp_rx; @@ -2192,7 +2229,7 @@ err_out: derr = ionic_qcq_disable(lif, lif->rxqcqs[i], derr); } - ionic_xdp_queues_config(lif); + ionic_xdp_rxqs_prog_update(lif); return err; } @@ -2651,7 +2688,7 @@ static void ionic_vf_attr_replay(struct ionic_lif *lif) ionic_vf_start(ionic); } -static void ionic_xdp_unregister_rxq_info(struct ionic_queue *q) +static void ionic_unregister_rxq_info(struct ionic_queue *q) { struct xdp_rxq_info *xi; @@ -2665,7 +2702,7 @@ static void ionic_xdp_unregister_rxq_info(struct ionic_queue *q) kfree(xi); } -static int ionic_xdp_register_rxq_info(struct ionic_queue *q, unsigned int napi_id) +static int ionic_register_rxq_info(struct ionic_queue *q, unsigned int napi_id) { struct xdp_rxq_info *rxq_info; int err; @@ -2676,15 +2713,15 @@ static int ionic_xdp_register_rxq_info(struct ionic_queue *q, unsigned int napi_ err = xdp_rxq_info_reg(rxq_info, q->lif->netdev, q->index, napi_id); if (err) { - dev_err(q->dev, "Queue %d xdp_rxq_info_reg failed, err %d\n", - q->index, err); + netdev_err(q->lif->netdev, "q%d xdp_rxq_info_reg failed, err %d\n", + q->index, err); goto err_out; } - err = xdp_rxq_info_reg_mem_model(rxq_info, MEM_TYPE_PAGE_ORDER0, NULL); + err = xdp_rxq_info_reg_mem_model(rxq_info, MEM_TYPE_PAGE_POOL, q->page_pool); if (err) { - dev_err(q->dev, "Queue %d xdp_rxq_info_reg_mem_model failed, err %d\n", - q->index, err); + netdev_err(q->lif->netdev, "q%d xdp_rxq_info_reg_mem_model failed, err %d\n", + q->index, err); xdp_rxq_info_unreg(rxq_info); goto err_out; } @@ -2698,44 +2735,20 @@ err_out: return err; } -static int ionic_xdp_queues_config(struct ionic_lif *lif) +static void ionic_xdp_rxqs_prog_update(struct ionic_lif *lif) { + struct bpf_prog *xdp_prog; unsigned int i; - int err; if (!lif->rxqcqs) - return 0; - - /* There's no need to rework memory if not going to/from NULL program. - * If there is no lif->xdp_prog, there should also be no q.xdp_rxq_info - * This way we don't need to keep an *xdp_prog in every queue struct. - */ - if (!lif->xdp_prog == !lif->rxqcqs[0]->q.xdp_rxq_info) - return 0; + return; + xdp_prog = READ_ONCE(lif->xdp_prog); for (i = 0; i < lif->ionic->nrxqs_per_lif && lif->rxqcqs[i]; i++) { struct ionic_queue *q = &lif->rxqcqs[i]->q; - if (q->xdp_rxq_info) { - ionic_xdp_unregister_rxq_info(q); - continue; - } - - err = ionic_xdp_register_rxq_info(q, lif->rxqcqs[i]->napi.napi_id); - if (err) { - dev_err(lif->ionic->dev, "failed to register RX queue %d info for XDP, err %d\n", - i, err); - goto err_out; - } + WRITE_ONCE(q->xdp_prog, xdp_prog); } - - return 0; - -err_out: - for (i = 0; i < lif->ionic->nrxqs_per_lif && lif->rxqcqs[i]; i++) - ionic_xdp_unregister_rxq_info(&lif->rxqcqs[i]->q); - - return err; } static int ionic_xdp_config(struct net_device *netdev, struct netdev_bpf *bpf) @@ -2765,11 +2778,17 @@ static int ionic_xdp_config(struct net_device *netdev, struct netdev_bpf *bpf) if (!netif_running(netdev)) { old_prog = xchg(&lif->xdp_prog, bpf->prog); + } else if (lif->xdp_prog && bpf->prog) { + old_prog = xchg(&lif->xdp_prog, bpf->prog); + ionic_xdp_rxqs_prog_update(lif); } else { + struct ionic_queue_params qparams; + + ionic_init_queue_params(lif, &qparams); + qparams.xdp_prog = bpf->prog; mutex_lock(&lif->queue_lock); - ionic_stop_queues_reconfig(lif); + ionic_reconfigure_queues(lif, &qparams); old_prog = xchg(&lif->xdp_prog, bpf->prog); - ionic_start_queues_reconfig(lif); mutex_unlock(&lif->queue_lock); } @@ -2871,13 +2890,23 @@ err_out: static void ionic_swap_queues(struct ionic_qcq *a, struct ionic_qcq *b) { - /* only swapping the queues, not the napi, flags, or other stuff */ + /* only swapping the queues and napi, not flags or other stuff */ + swap(a->napi, b->napi); + + if (a->q.type == IONIC_QTYPE_RXQ) { + swap(a->q.page_pool, b->q.page_pool); + a->q.page_pool->p.napi = &a->napi; + if (b->q.page_pool) /* is NULL when increasing queue count */ + b->q.page_pool->p.napi = &b->napi; + } + swap(a->q.features, b->q.features); swap(a->q.num_descs, b->q.num_descs); swap(a->q.desc_size, b->q.desc_size); swap(a->q.base, b->q.base); swap(a->q.base_pa, b->q.base_pa); swap(a->q.info, b->q.info); + swap(a->q.xdp_prog, b->q.xdp_prog); swap(a->q.xdp_rxq_info, b->q.xdp_rxq_info); swap(a->q.partner, b->q.partner); swap(a->q_base, b->q_base); @@ -2928,7 +2957,8 @@ int ionic_reconfigure_queues(struct ionic_lif *lif, } if (qparam->nxqs != lif->nxqs || qparam->nrxq_descs != lif->nrxq_descs || - qparam->rxq_features != lif->rxq_features) { + qparam->rxq_features != lif->rxq_features || + qparam->xdp_prog != lif->xdp_prog) { rx_qcqs = devm_kcalloc(lif->ionic->dev, lif->ionic->nrxqs_per_lif, sizeof(struct ionic_qcq *), GFP_KERNEL); if (!rx_qcqs) { @@ -2959,7 +2989,7 @@ int ionic_reconfigure_queues(struct ionic_lif *lif, err = ionic_qcq_alloc(lif, IONIC_QTYPE_TXQ, i, "tx", flags, 4, desc_sz, comp_sz, sg_desc_sz, sizeof(struct ionic_tx_desc_info), - lif->kern_pid, &lif->txqcqs[i]); + lif->kern_pid, NULL, &lif->txqcqs[i]); if (err) goto err_out; } @@ -2968,7 +2998,7 @@ int ionic_reconfigure_queues(struct ionic_lif *lif, err = ionic_qcq_alloc(lif, IONIC_QTYPE_TXQ, i, "tx", flags, num_desc, desc_sz, comp_sz, sg_desc_sz, sizeof(struct ionic_tx_desc_info), - lif->kern_pid, &tx_qcqs[i]); + lif->kern_pid, NULL, &tx_qcqs[i]); if (err) goto err_out; } @@ -2990,7 +3020,7 @@ int ionic_reconfigure_queues(struct ionic_lif *lif, err = ionic_qcq_alloc(lif, IONIC_QTYPE_RXQ, i, "rx", flags, 4, desc_sz, comp_sz, sg_desc_sz, sizeof(struct ionic_rx_desc_info), - lif->kern_pid, &lif->rxqcqs[i]); + lif->kern_pid, NULL, &lif->rxqcqs[i]); if (err) goto err_out; } @@ -2999,11 +3029,12 @@ int ionic_reconfigure_queues(struct ionic_lif *lif, err = ionic_qcq_alloc(lif, IONIC_QTYPE_RXQ, i, "rx", flags, num_desc, desc_sz, comp_sz, sg_desc_sz, sizeof(struct ionic_rx_desc_info), - lif->kern_pid, &rx_qcqs[i]); + lif->kern_pid, qparam->xdp_prog, &rx_qcqs[i]); if (err) goto err_out; rx_qcqs[i]->q.features = qparam->rxq_features; + rx_qcqs[i]->q.xdp_prog = qparam->xdp_prog; } } @@ -3220,7 +3251,7 @@ int ionic_lif_alloc(struct ionic *ionic) netdev->netdev_ops = &ionic_netdev_ops; ionic_ethtool_set_ops(netdev); - netdev->watchdog_timeo = 2 * HZ; + netdev->watchdog_timeo = 5 * HZ; netif_carrier_off(netdev); lif->identity = lid; diff --git a/drivers/net/ethernet/pensando/ionic/ionic_lif.h b/drivers/net/ethernet/pensando/ionic/ionic_lif.h index 3e1005293c4a..e01756fb7fdd 100644 --- a/drivers/net/ethernet/pensando/ionic/ionic_lif.h +++ b/drivers/net/ethernet/pensando/ionic/ionic_lif.h @@ -268,6 +268,7 @@ struct ionic_queue_params { unsigned int ntxq_descs; unsigned int nrxq_descs; u64 rxq_features; + struct bpf_prog *xdp_prog; bool intr_split; bool cmb_tx; bool cmb_rx; @@ -280,6 +281,7 @@ static inline void ionic_init_queue_params(struct ionic_lif *lif, qparam->ntxq_descs = lif->ntxq_descs; qparam->nrxq_descs = lif->nrxq_descs; qparam->rxq_features = lif->rxq_features; + qparam->xdp_prog = lif->xdp_prog; qparam->intr_split = test_bit(IONIC_LIF_F_SPLIT_INTR, lif->state); qparam->cmb_tx = test_bit(IONIC_LIF_F_CMB_TX_RINGS, lif->state); qparam->cmb_rx = test_bit(IONIC_LIF_F_CMB_RX_RINGS, lif->state); diff --git a/drivers/net/ethernet/pensando/ionic/ionic_rx_filter.c b/drivers/net/ethernet/pensando/ionic/ionic_rx_filter.c index 1ee2f285cb42..528114877677 100644 --- a/drivers/net/ethernet/pensando/ionic/ionic_rx_filter.c +++ b/drivers/net/ethernet/pensando/ionic/ionic_rx_filter.c @@ -312,8 +312,8 @@ static int ionic_lif_filter_add(struct ionic_lif *lif, int err = 0; ctx.cmd.rx_filter_add = *ac; - ctx.cmd.rx_filter_add.opcode = IONIC_CMD_RX_FILTER_ADD, - ctx.cmd.rx_filter_add.lif_index = cpu_to_le16(lif->index), + ctx.cmd.rx_filter_add.opcode = IONIC_CMD_RX_FILTER_ADD; + ctx.cmd.rx_filter_add.lif_index = cpu_to_le16(lif->index); spin_lock_bh(&lif->rx_filters.lock); f = ionic_rx_filter_find(lif, &ctx.cmd.rx_filter_add); diff --git a/drivers/net/ethernet/pensando/ionic/ionic_txrx.c b/drivers/net/ethernet/pensando/ionic/ionic_txrx.c index fc79baad4561..0eeda7e502db 100644 --- a/drivers/net/ethernet/pensando/ionic/ionic_txrx.c +++ b/drivers/net/ethernet/pensando/ionic/ionic_txrx.c @@ -6,6 +6,7 @@ #include <linux/if_vlan.h> #include <net/ip6_checksum.h> #include <net/netdev_queues.h> +#include <net/page_pool/helpers.h> #include "ionic.h" #include "ionic_lif.h" @@ -118,108 +119,57 @@ static void *ionic_rx_buf_va(struct ionic_buf_info *buf_info) static dma_addr_t ionic_rx_buf_pa(struct ionic_buf_info *buf_info) { - return buf_info->dma_addr + buf_info->page_offset; + return page_pool_get_dma_addr(buf_info->page) + buf_info->page_offset; } -static unsigned int ionic_rx_buf_size(struct ionic_buf_info *buf_info) +static void __ionic_rx_put_buf(struct ionic_queue *q, + struct ionic_buf_info *buf_info, + bool recycle_direct) { - return min_t(u32, IONIC_MAX_BUF_LEN, IONIC_PAGE_SIZE - buf_info->page_offset); -} - -static int ionic_rx_page_alloc(struct ionic_queue *q, - struct ionic_buf_info *buf_info) -{ - struct device *dev = q->dev; - dma_addr_t dma_addr; - struct page *page; - - page = alloc_pages(IONIC_PAGE_GFP_MASK, 0); - if (unlikely(!page)) { - net_err_ratelimited("%s: %s page alloc failed\n", - dev_name(dev), q->name); - q_to_rx_stats(q)->alloc_err++; - return -ENOMEM; - } - - dma_addr = dma_map_page(dev, page, 0, - IONIC_PAGE_SIZE, DMA_FROM_DEVICE); - if (unlikely(dma_mapping_error(dev, dma_addr))) { - __free_pages(page, 0); - net_err_ratelimited("%s: %s dma map failed\n", - dev_name(dev), q->name); - q_to_rx_stats(q)->dma_map_err++; - return -EIO; - } - - buf_info->dma_addr = dma_addr; - buf_info->page = page; - buf_info->page_offset = 0; - - return 0; -} - -static void ionic_rx_page_free(struct ionic_queue *q, - struct ionic_buf_info *buf_info) -{ - struct device *dev = q->dev; - - if (unlikely(!buf_info)) { - net_err_ratelimited("%s: %s invalid buf_info in free\n", - dev_name(dev), q->name); - return; - } - if (!buf_info->page) return; - dma_unmap_page(dev, buf_info->dma_addr, IONIC_PAGE_SIZE, DMA_FROM_DEVICE); - __free_pages(buf_info->page, 0); + page_pool_put_full_page(q->page_pool, buf_info->page, recycle_direct); buf_info->page = NULL; + buf_info->len = 0; + buf_info->page_offset = 0; } -static bool ionic_rx_buf_recycle(struct ionic_queue *q, - struct ionic_buf_info *buf_info, u32 len) -{ - u32 size; - - /* don't re-use pages allocated in low-mem condition */ - if (page_is_pfmemalloc(buf_info->page)) - return false; - - /* don't re-use buffers from non-local numa nodes */ - if (page_to_nid(buf_info->page) != numa_mem_id()) - return false; - - size = ALIGN(len, q->xdp_rxq_info ? IONIC_PAGE_SIZE : IONIC_PAGE_SPLIT_SZ); - buf_info->page_offset += size; - if (buf_info->page_offset >= IONIC_PAGE_SIZE) - return false; - get_page(buf_info->page); +static void ionic_rx_put_buf(struct ionic_queue *q, + struct ionic_buf_info *buf_info) +{ + __ionic_rx_put_buf(q, buf_info, false); +} - return true; +static void ionic_rx_put_buf_direct(struct ionic_queue *q, + struct ionic_buf_info *buf_info) +{ + __ionic_rx_put_buf(q, buf_info, true); } static void ionic_rx_add_skb_frag(struct ionic_queue *q, struct sk_buff *skb, struct ionic_buf_info *buf_info, - u32 off, u32 len, + u32 headroom, u32 len, bool synced) { if (!synced) - dma_sync_single_range_for_cpu(q->dev, ionic_rx_buf_pa(buf_info), - off, len, DMA_FROM_DEVICE); + page_pool_dma_sync_for_cpu(q->page_pool, + buf_info->page, + buf_info->page_offset + headroom, + len); skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, - buf_info->page, buf_info->page_offset + off, - len, - IONIC_PAGE_SIZE); + buf_info->page, buf_info->page_offset + headroom, + len, buf_info->len); - if (!ionic_rx_buf_recycle(q, buf_info, len)) { - dma_unmap_page(q->dev, buf_info->dma_addr, - IONIC_PAGE_SIZE, DMA_FROM_DEVICE); - buf_info->page = NULL; - } + /* napi_gro_frags() will release/recycle the + * page_pool buffers from the frags list + */ + buf_info->page = NULL; + buf_info->len = 0; + buf_info->page_offset = 0; } static struct sk_buff *ionic_rx_build_skb(struct ionic_queue *q, @@ -244,12 +194,13 @@ static struct sk_buff *ionic_rx_build_skb(struct ionic_queue *q, q_to_rx_stats(q)->alloc_err++; return NULL; } + skb_mark_for_recycle(skb); if (headroom) frag_len = min_t(u16, len, IONIC_XDP_MAX_LINEAR_MTU + VLAN_ETH_HLEN); else - frag_len = min_t(u16, len, ionic_rx_buf_size(buf_info)); + frag_len = min_t(u16, len, IONIC_PAGE_SIZE); if (unlikely(!buf_info->page)) goto err_bad_buf_page; @@ -260,7 +211,7 @@ static struct sk_buff *ionic_rx_build_skb(struct ionic_queue *q, for (i = 0; i < num_sg_elems; i++, buf_info++) { if (unlikely(!buf_info->page)) goto err_bad_buf_page; - frag_len = min_t(u16, len, ionic_rx_buf_size(buf_info)); + frag_len = min_t(u16, len, buf_info->len); ionic_rx_add_skb_frag(q, skb, buf_info, 0, frag_len, synced); len -= frag_len; } @@ -277,11 +228,13 @@ static struct sk_buff *ionic_rx_copybreak(struct net_device *netdev, struct ionic_rx_desc_info *desc_info, unsigned int headroom, unsigned int len, + unsigned int num_sg_elems, bool synced) { struct ionic_buf_info *buf_info; struct device *dev = q->dev; struct sk_buff *skb; + int i; buf_info = &desc_info->bufs[0]; @@ -292,54 +245,52 @@ static struct sk_buff *ionic_rx_copybreak(struct net_device *netdev, q_to_rx_stats(q)->alloc_err++; return NULL; } - - if (unlikely(!buf_info->page)) { - dev_kfree_skb(skb); - return NULL; - } + skb_mark_for_recycle(skb); if (!synced) - dma_sync_single_range_for_cpu(dev, ionic_rx_buf_pa(buf_info), - headroom, len, DMA_FROM_DEVICE); + page_pool_dma_sync_for_cpu(q->page_pool, + buf_info->page, + buf_info->page_offset + headroom, + len); + skb_copy_to_linear_data(skb, ionic_rx_buf_va(buf_info) + headroom, len); - dma_sync_single_range_for_device(dev, ionic_rx_buf_pa(buf_info), - headroom, len, DMA_FROM_DEVICE); skb_put(skb, len); skb->protocol = eth_type_trans(skb, netdev); + /* recycle the Rx buffer now that we're done with it */ + ionic_rx_put_buf_direct(q, buf_info); + buf_info++; + for (i = 0; i < num_sg_elems; i++, buf_info++) + ionic_rx_put_buf_direct(q, buf_info); + return skb; } static void ionic_xdp_tx_desc_clean(struct ionic_queue *q, - struct ionic_tx_desc_info *desc_info) + struct ionic_tx_desc_info *desc_info, + bool in_napi) { - unsigned int nbufs = desc_info->nbufs; - struct ionic_buf_info *buf_info; - struct device *dev = q->dev; - int i; + struct xdp_frame_bulk bq; - if (!nbufs) + if (!desc_info->nbufs) return; - buf_info = desc_info->bufs; - dma_unmap_single(dev, buf_info->dma_addr, - buf_info->len, DMA_TO_DEVICE); - if (desc_info->act == XDP_TX) - __free_pages(buf_info->page, 0); - buf_info->page = NULL; + xdp_frame_bulk_init(&bq); + rcu_read_lock(); /* need for xdp_return_frame_bulk */ - buf_info++; - for (i = 1; i < nbufs + 1 && buf_info->page; i++, buf_info++) { - dma_unmap_page(dev, buf_info->dma_addr, - buf_info->len, DMA_TO_DEVICE); - if (desc_info->act == XDP_TX) - __free_pages(buf_info->page, 0); - buf_info->page = NULL; + if (desc_info->act == XDP_TX) { + if (likely(in_napi)) + xdp_return_frame_rx_napi(desc_info->xdpf); + else + xdp_return_frame(desc_info->xdpf); + } else if (desc_info->act == XDP_REDIRECT) { + ionic_tx_desc_unmap_bufs(q, desc_info); + xdp_return_frame_bulk(desc_info->xdpf, &bq); } - if (desc_info->act == XDP_REDIRECT) - xdp_return_frame(desc_info->xdpf); + xdp_flush_frame_bulk(&bq); + rcu_read_unlock(); desc_info->nbufs = 0; desc_info->xdpf = NULL; @@ -363,9 +314,17 @@ static int ionic_xdp_post_frame(struct ionic_queue *q, struct xdp_frame *frame, buf_info = desc_info->bufs; stats = q_to_tx_stats(q); - dma_addr = ionic_tx_map_single(q, frame->data, len); - if (!dma_addr) - return -EIO; + if (act == XDP_TX) { + dma_addr = page_pool_get_dma_addr(page) + + off + XDP_PACKET_HEADROOM; + dma_sync_single_for_device(q->dev, dma_addr, + len, DMA_TO_DEVICE); + } else /* XDP_REDIRECT */ { + dma_addr = ionic_tx_map_single(q, frame->data, len); + if (!dma_addr) + return -EIO; + } + buf_info->dma_addr = dma_addr; buf_info->len = len; buf_info->page = page; @@ -387,10 +346,21 @@ static int ionic_xdp_post_frame(struct ionic_queue *q, struct xdp_frame *frame, frag = sinfo->frags; elem = ionic_tx_sg_elems(q); for (i = 0; i < sinfo->nr_frags; i++, frag++, bi++) { - dma_addr = ionic_tx_map_frag(q, frag, 0, skb_frag_size(frag)); - if (!dma_addr) { - ionic_tx_desc_unmap_bufs(q, desc_info); - return -EIO; + if (act == XDP_TX) { + struct page *pg = skb_frag_page(frag); + + dma_addr = page_pool_get_dma_addr(pg) + + skb_frag_off(frag); + dma_sync_single_for_device(q->dev, dma_addr, + skb_frag_size(frag), + DMA_TO_DEVICE); + } else { + dma_addr = ionic_tx_map_frag(q, frag, 0, + skb_frag_size(frag)); + if (dma_mapping_error(q->dev, dma_addr)) { + ionic_tx_desc_unmap_bufs(q, desc_info); + return -EIO; + } } bi->dma_addr = dma_addr; bi->len = skb_frag_size(frag); @@ -481,15 +451,13 @@ int ionic_xdp_xmit(struct net_device *netdev, int n, return nxmit; } -static void ionic_xdp_rx_put_bufs(struct ionic_queue *q, - struct ionic_buf_info *buf_info, - int nbufs) +static void ionic_xdp_rx_unlink_bufs(struct ionic_queue *q, + struct ionic_buf_info *buf_info, + int nbufs) { int i; for (i = 0; i < nbufs; i++) { - dma_unmap_page(q->dev, buf_info->dma_addr, - IONIC_PAGE_SIZE, DMA_FROM_DEVICE); buf_info->page = NULL; buf_info++; } @@ -516,11 +484,9 @@ static bool ionic_run_xdp(struct ionic_rx_stats *stats, frag_len = min_t(u16, len, IONIC_XDP_MAX_LINEAR_MTU + VLAN_ETH_HLEN); xdp_prepare_buff(&xdp_buf, ionic_rx_buf_va(buf_info), XDP_PACKET_HEADROOM, frag_len, false); - - dma_sync_single_range_for_cpu(rxq->dev, ionic_rx_buf_pa(buf_info), - XDP_PACKET_HEADROOM, frag_len, - DMA_FROM_DEVICE); - + page_pool_dma_sync_for_cpu(rxq->page_pool, buf_info->page, + buf_info->page_offset + XDP_PACKET_HEADROOM, + frag_len); prefetchw(&xdp_buf.data_hard_start); /* We limit MTU size to one buffer if !xdp_has_frags, so @@ -542,15 +508,16 @@ static bool ionic_run_xdp(struct ionic_rx_stats *stats, do { if (unlikely(sinfo->nr_frags >= MAX_SKB_FRAGS)) { err = -ENOSPC; - goto out_xdp_abort; + break; } frag = &sinfo->frags[sinfo->nr_frags]; sinfo->nr_frags++; bi++; - frag_len = min_t(u16, remain_len, ionic_rx_buf_size(bi)); - dma_sync_single_range_for_cpu(rxq->dev, ionic_rx_buf_pa(bi), - 0, frag_len, DMA_FROM_DEVICE); + frag_len = min_t(u16, remain_len, bi->len); + page_pool_dma_sync_for_cpu(rxq->page_pool, bi->page, + buf_info->page_offset, + frag_len); skb_frag_fill_page_desc(frag, bi->page, 0, frag_len); sinfo->xdp_frags_size += frag_len; remain_len -= frag_len; @@ -569,14 +536,16 @@ static bool ionic_run_xdp(struct ionic_rx_stats *stats, return false; /* false = we didn't consume the packet */ case XDP_DROP: - ionic_rx_page_free(rxq, buf_info); + ionic_rx_put_buf_direct(rxq, buf_info); stats->xdp_drop++; break; case XDP_TX: xdpf = xdp_convert_buff_to_frame(&xdp_buf); - if (!xdpf) - goto out_xdp_abort; + if (!xdpf) { + err = -ENOSPC; + break; + } txq = rxq->partner; nq = netdev_get_tx_queue(netdev, txq->index); @@ -588,7 +557,8 @@ static bool ionic_run_xdp(struct ionic_rx_stats *stats, ionic_q_space_avail(txq), 1, 1)) { __netif_tx_unlock(nq); - goto out_xdp_abort; + err = -EIO; + break; } err = ionic_xdp_post_frame(txq, xdpf, XDP_TX, @@ -598,49 +568,47 @@ static bool ionic_run_xdp(struct ionic_rx_stats *stats, __netif_tx_unlock(nq); if (unlikely(err)) { netdev_dbg(netdev, "tx ionic_xdp_post_frame err %d\n", err); - goto out_xdp_abort; + break; } - ionic_xdp_rx_put_bufs(rxq, buf_info, nbufs); + ionic_xdp_rx_unlink_bufs(rxq, buf_info, nbufs); stats->xdp_tx++; - - /* the Tx completion will free the buffers */ break; case XDP_REDIRECT: err = xdp_do_redirect(netdev, &xdp_buf, xdp_prog); if (unlikely(err)) { netdev_dbg(netdev, "xdp_do_redirect err %d\n", err); - goto out_xdp_abort; + break; } - ionic_xdp_rx_put_bufs(rxq, buf_info, nbufs); + ionic_xdp_rx_unlink_bufs(rxq, buf_info, nbufs); rxq->xdp_flush = true; stats->xdp_redirect++; break; case XDP_ABORTED: default: - goto out_xdp_abort; + err = -EIO; + break; } - return true; - -out_xdp_abort: - trace_xdp_exception(netdev, xdp_prog, xdp_action); - ionic_rx_page_free(rxq, buf_info); - stats->xdp_aborted++; + if (err) { + ionic_rx_put_buf_direct(rxq, buf_info); + trace_xdp_exception(netdev, xdp_prog, xdp_action); + stats->xdp_aborted++; + } return true; } static void ionic_rx_clean(struct ionic_queue *q, struct ionic_rx_desc_info *desc_info, - struct ionic_rxq_comp *comp) + struct ionic_rxq_comp *comp, + struct bpf_prog *xdp_prog) { struct net_device *netdev = q->lif->netdev; struct ionic_qcq *qcq = q_to_qcq(q); struct ionic_rx_stats *stats; - struct bpf_prog *xdp_prog; - unsigned int headroom; + unsigned int headroom = 0; struct sk_buff *skb; bool synced = false; bool use_copybreak; @@ -648,7 +616,14 @@ static void ionic_rx_clean(struct ionic_queue *q, stats = q_to_rx_stats(q); - if (comp->status) { + if (unlikely(comp->status)) { + /* Most likely status==2 and the pkt received was bigger + * than the buffer available: comp->len will show the + * pkt size received that didn't fit the advertised desc.len + */ + dev_dbg(q->dev, "q%d drop comp->status %d comp->len %d desc->len %d\n", + q->index, comp->status, comp->len, q->rxq[q->head_idx].len); + stats->dropped++; return; } @@ -657,18 +632,18 @@ static void ionic_rx_clean(struct ionic_queue *q, stats->pkts++; stats->bytes += len; - xdp_prog = READ_ONCE(q->lif->xdp_prog); if (xdp_prog) { if (ionic_run_xdp(stats, netdev, xdp_prog, q, desc_info->bufs, len)) return; synced = true; + headroom = XDP_PACKET_HEADROOM; } - headroom = q->xdp_rxq_info ? XDP_PACKET_HEADROOM : 0; use_copybreak = len <= q->lif->rx_copybreak; if (use_copybreak) skb = ionic_rx_copybreak(netdev, q, desc_info, - headroom, len, synced); + headroom, len, + comp->num_sg_elems, synced); else skb = ionic_rx_build_skb(q, desc_info, headroom, len, comp->num_sg_elems, synced); @@ -744,7 +719,7 @@ static void ionic_rx_clean(struct ionic_queue *q, napi_gro_frags(&qcq->napi); } -bool ionic_rx_service(struct ionic_cq *cq) +static bool __ionic_rx_service(struct ionic_cq *cq, struct bpf_prog *xdp_prog) { struct ionic_rx_desc_info *desc_info; struct ionic_queue *q = cq->bound_q; @@ -766,11 +741,16 @@ bool ionic_rx_service(struct ionic_cq *cq) q->tail_idx = (q->tail_idx + 1) & (q->num_descs - 1); /* clean the related q entry, only one per qc completion */ - ionic_rx_clean(q, desc_info, comp); + ionic_rx_clean(q, desc_info, comp, xdp_prog); return true; } +bool ionic_rx_service(struct ionic_cq *cq) +{ + return __ionic_rx_service(cq, NULL); +} + static inline void ionic_write_cmb_desc(struct ionic_queue *q, void *desc) { @@ -781,7 +761,7 @@ static inline void ionic_write_cmb_desc(struct ionic_queue *q, memcpy_toio(&q->cmb_txq[q->head_idx], desc, sizeof(q->cmb_txq[0])); } -void ionic_rx_fill(struct ionic_queue *q) +void ionic_rx_fill(struct ionic_queue *q, struct bpf_prog *xdp_prog) { struct net_device *netdev = q->lif->netdev; struct ionic_rx_desc_info *desc_info; @@ -789,6 +769,9 @@ void ionic_rx_fill(struct ionic_queue *q) struct ionic_buf_info *buf_info; unsigned int fill_threshold; struct ionic_rxq_desc *desc; + unsigned int first_frag_len; + unsigned int first_buf_len; + unsigned int headroom = 0; unsigned int remain_len; unsigned int frag_len; unsigned int nfrags; @@ -806,35 +789,43 @@ void ionic_rx_fill(struct ionic_queue *q) len = netdev->mtu + VLAN_ETH_HLEN; - for (i = n_fill; i; i--) { - unsigned int headroom; - unsigned int buf_len; + if (xdp_prog) { + /* Always alloc the full size buffer, but only need + * the actual frag_len in the descriptor + * XDP uses space in the first buffer, so account for + * head room, tail room, and ip header in the first frag size. + */ + headroom = XDP_PACKET_HEADROOM; + first_buf_len = IONIC_XDP_MAX_LINEAR_MTU + VLAN_ETH_HLEN + headroom; + first_frag_len = min_t(u16, len + headroom, first_buf_len); + } else { + /* Use MTU size if smaller than max buffer size */ + first_frag_len = min_t(u16, len, IONIC_PAGE_SIZE); + first_buf_len = first_frag_len; + } + for (i = n_fill; i; i--) { + /* fill main descriptor - buf[0] */ nfrags = 0; remain_len = len; desc = &q->rxq[q->head_idx]; desc_info = &q->rx_info[q->head_idx]; buf_info = &desc_info->bufs[0]; - if (!buf_info->page) { /* alloc a new buffer? */ - if (unlikely(ionic_rx_page_alloc(q, buf_info))) { - desc->addr = 0; - desc->len = 0; - return; - } + buf_info->len = first_buf_len; + frag_len = first_frag_len - headroom; + + /* get a new buffer if we can't reuse one */ + if (!buf_info->page) + buf_info->page = page_pool_alloc(q->page_pool, + &buf_info->page_offset, + &buf_info->len, + GFP_ATOMIC); + if (unlikely(!buf_info->page)) { + buf_info->len = 0; + return; } - /* fill main descriptor - buf[0] - * XDP uses space in the first buffer, so account for - * head room, tail room, and ip header in the first frag size. - */ - headroom = q->xdp_rxq_info ? XDP_PACKET_HEADROOM : 0; - if (q->xdp_rxq_info) - buf_len = IONIC_XDP_MAX_LINEAR_MTU + VLAN_ETH_HLEN; - else - buf_len = ionic_rx_buf_size(buf_info); - frag_len = min_t(u16, len, buf_len); - desc->addr = cpu_to_le64(ionic_rx_buf_pa(buf_info) + headroom); desc->len = cpu_to_le16(frag_len); remain_len -= frag_len; @@ -844,16 +835,26 @@ void ionic_rx_fill(struct ionic_queue *q) /* fill sg descriptors - buf[1..n] */ sg_elem = q->rxq_sgl[q->head_idx].elems; for (j = 0; remain_len > 0 && j < q->max_sg_elems; j++, sg_elem++) { - if (!buf_info->page) { /* alloc a new sg buffer? */ - if (unlikely(ionic_rx_page_alloc(q, buf_info))) { - sg_elem->addr = 0; - sg_elem->len = 0; + frag_len = min_t(u16, remain_len, IONIC_PAGE_SIZE); + + /* Recycle any leftover buffers that are too small to reuse */ + if (unlikely(buf_info->page && buf_info->len < frag_len)) + ionic_rx_put_buf_direct(q, buf_info); + + /* Get new buffer if needed */ + if (!buf_info->page) { + buf_info->len = frag_len; + buf_info->page = page_pool_alloc(q->page_pool, + &buf_info->page_offset, + &buf_info->len, + GFP_ATOMIC); + if (unlikely(!buf_info->page)) { + buf_info->len = 0; return; } } sg_elem->addr = cpu_to_le64(ionic_rx_buf_pa(buf_info)); - frag_len = min_t(u16, remain_len, ionic_rx_buf_size(buf_info)); sg_elem->len = cpu_to_le16(frag_len); remain_len -= frag_len; buf_info++; @@ -883,17 +884,12 @@ void ionic_rx_fill(struct ionic_queue *q) void ionic_rx_empty(struct ionic_queue *q) { struct ionic_rx_desc_info *desc_info; - struct ionic_buf_info *buf_info; unsigned int i, j; for (i = 0; i < q->num_descs; i++) { desc_info = &q->rx_info[i]; - for (j = 0; j < ARRAY_SIZE(desc_info->bufs); j++) { - buf_info = &desc_info->bufs[j]; - if (buf_info->page) - ionic_rx_page_free(q, buf_info); - } - + for (j = 0; j < ARRAY_SIZE(desc_info->bufs); j++) + ionic_rx_put_buf(q, &desc_info->bufs[j]); desc_info->nbufs = 0; } @@ -974,6 +970,32 @@ static void ionic_xdp_do_flush(struct ionic_cq *cq) } } +static unsigned int ionic_rx_cq_service(struct ionic_cq *cq, + unsigned int work_to_do) +{ + struct ionic_queue *q = cq->bound_q; + unsigned int work_done = 0; + struct bpf_prog *xdp_prog; + + if (work_to_do == 0) + return 0; + + xdp_prog = READ_ONCE(q->xdp_prog); + while (__ionic_rx_service(cq, xdp_prog)) { + if (cq->tail_idx == cq->num_descs - 1) + cq->done_color = !cq->done_color; + + cq->tail_idx = (cq->tail_idx + 1) & (cq->num_descs - 1); + + if (++work_done >= work_to_do) + break; + } + ionic_rx_fill(q, xdp_prog); + ionic_xdp_do_flush(cq); + + return work_done; +} + int ionic_rx_napi(struct napi_struct *napi, int budget) { struct ionic_qcq *qcq = napi_to_qcq(napi); @@ -984,12 +1006,8 @@ int ionic_rx_napi(struct napi_struct *napi, int budget) if (unlikely(!budget)) return budget; - work_done = ionic_cq_service(cq, budget, - ionic_rx_service, NULL, NULL); - - ionic_rx_fill(cq->bound_q); + work_done = ionic_rx_cq_service(cq, budget); - ionic_xdp_do_flush(cq); if (work_done < budget && napi_complete_done(napi, work_done)) { ionic_dim_update(qcq, IONIC_LIF_F_RX_DIM_INTR); flags |= IONIC_INTR_CRED_UNMASK; @@ -1030,12 +1048,8 @@ int ionic_txrx_napi(struct napi_struct *napi, int budget) if (unlikely(!budget)) return budget; - rx_work_done = ionic_cq_service(rxcq, budget, - ionic_rx_service, NULL, NULL); - - ionic_rx_fill(rxcq->bound_q); + rx_work_done = ionic_rx_cq_service(rxcq, budget); - ionic_xdp_do_flush(rxcq); if (rx_work_done < budget && napi_complete_done(napi, rx_work_done)) { ionic_dim_update(rxqcq, 0); flags |= IONIC_INTR_CRED_UNMASK; @@ -1166,7 +1180,7 @@ static void ionic_tx_clean(struct ionic_queue *q, struct sk_buff *skb; if (desc_info->xdpf) { - ionic_xdp_tx_desc_clean(q->partner, desc_info); + ionic_xdp_tx_desc_clean(q->partner, desc_info, in_napi); stats->clean++; if (unlikely(__netif_subqueue_stopped(q->lif->netdev, q->index))) diff --git a/drivers/net/ethernet/pensando/ionic/ionic_txrx.h b/drivers/net/ethernet/pensando/ionic/ionic_txrx.h index 9e73e324e7a1..b2b9a2dc9eb8 100644 --- a/drivers/net/ethernet/pensando/ionic/ionic_txrx.h +++ b/drivers/net/ethernet/pensando/ionic/ionic_txrx.h @@ -4,9 +4,11 @@ #ifndef _IONIC_TXRX_H_ #define _IONIC_TXRX_H_ +struct bpf_prog; + void ionic_tx_flush(struct ionic_cq *cq); -void ionic_rx_fill(struct ionic_queue *q); +void ionic_rx_fill(struct ionic_queue *q, struct bpf_prog *xdp_prog); void ionic_rx_empty(struct ionic_queue *q); void ionic_tx_empty(struct ionic_queue *q); int ionic_rx_napi(struct napi_struct *napi, int budget); |