diff options
Diffstat (limited to 'drivers/net/ethernet/broadcom')
26 files changed, 395 insertions, 223 deletions
diff --git a/drivers/net/ethernet/broadcom/Kconfig b/drivers/net/ethernet/broadcom/Kconfig index b123509d385f..e9017caf024d 100644 --- a/drivers/net/ethernet/broadcom/Kconfig +++ b/drivers/net/ethernet/broadcom/Kconfig @@ -8,6 +8,7 @@ config NET_VENDOR_BROADCOM default y depends on (SSB_POSSIBLE && HAS_DMA) || PCI || BCM63XX || \ SIBYTE_SB1xxx_SOC + select DIMLIB ---help--- If you have a network (Ethernet) chipset belonging to this class, say Y. @@ -198,6 +199,7 @@ config BNXT select FW_LOADER select LIBCRC32C select NET_DEVLINK + select PAGE_POOL ---help--- This driver supports Broadcom NetXtreme-C/E 10/25/40/50 gigabit Ethernet cards. To compile this driver as a module, choose M here: diff --git a/drivers/net/ethernet/broadcom/bcm63xx_enet.c b/drivers/net/ethernet/broadcom/bcm63xx_enet.c index 85e610210477..291e4afd4a1a 100644 --- a/drivers/net/ethernet/broadcom/bcm63xx_enet.c +++ b/drivers/net/ethernet/broadcom/bcm63xx_enet.c @@ -2659,7 +2659,6 @@ static int bcm_enetsw_probe(struct platform_device *pdev) if (!dev) return -ENOMEM; priv = netdev_priv(dev); - memset(priv, 0, sizeof(*priv)); /* initialize default and fetch platform data */ priv->enet_is_sw = true; diff --git a/drivers/net/ethernet/broadcom/bcmsysport.c b/drivers/net/ethernet/broadcom/bcmsysport.c index c623896e3ccb..b9c5cea8db16 100644 --- a/drivers/net/ethernet/broadcom/bcmsysport.c +++ b/drivers/net/ethernet/broadcom/bcmsysport.c @@ -1,11 +1,8 @@ +// SPDX-License-Identifier: GPL-2.0-only /* * Broadcom BCM7xxx System Port Ethernet MAC driver * * Copyright (C) 2014 Broadcom Corporation - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. */ #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt @@ -612,7 +609,7 @@ static int bcm_sysport_set_coalesce(struct net_device *dev, struct ethtool_coalesce *ec) { struct bcm_sysport_priv *priv = netdev_priv(dev); - struct net_dim_cq_moder moder; + struct dim_cq_moder moder; u32 usecs, pkts; unsigned int i; @@ -995,7 +992,7 @@ static int bcm_sysport_poll(struct napi_struct *napi, int budget) { struct bcm_sysport_priv *priv = container_of(napi, struct bcm_sysport_priv, napi); - struct net_dim_sample dim_sample; + struct dim_sample dim_sample; unsigned int work_done = 0; work_done = bcm_sysport_desc_rx(priv, budget); @@ -1019,8 +1016,8 @@ static int bcm_sysport_poll(struct napi_struct *napi, int budget) } if (priv->dim.use_dim) { - net_dim_sample(priv->dim.event_ctr, priv->dim.packets, - priv->dim.bytes, &dim_sample); + dim_update_sample(priv->dim.event_ctr, priv->dim.packets, + priv->dim.bytes, &dim_sample); net_dim(&priv->dim.dim, dim_sample); } @@ -1090,16 +1087,16 @@ static void bcm_sysport_resume_from_wol(struct bcm_sysport_priv *priv) static void bcm_sysport_dim_work(struct work_struct *work) { - struct net_dim *dim = container_of(work, struct net_dim, work); + struct dim *dim = container_of(work, struct dim, work); struct bcm_sysport_net_dim *ndim = container_of(dim, struct bcm_sysport_net_dim, dim); struct bcm_sysport_priv *priv = container_of(ndim, struct bcm_sysport_priv, dim); - struct net_dim_cq_moder cur_profile = - net_dim_get_rx_moderation(dim->mode, dim->profile_ix); + struct dim_cq_moder cur_profile = net_dim_get_rx_moderation(dim->mode, + dim->profile_ix); bcm_sysport_set_rx_coalesce(priv, cur_profile.usec, cur_profile.pkts); - dim->state = NET_DIM_START_MEASURE; + dim->state = DIM_START_MEASURE; } /* RX and misc interrupt routine */ @@ -1440,7 +1437,7 @@ static void bcm_sysport_init_dim(struct bcm_sysport_priv *priv, struct bcm_sysport_net_dim *dim = &priv->dim; INIT_WORK(&dim->dim.work, cb); - dim->dim.mode = NET_DIM_CQ_PERIOD_MODE_START_FROM_EQE; + dim->dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_EQE; dim->event_ctr = 0; dim->packets = 0; dim->bytes = 0; @@ -1449,7 +1446,7 @@ static void bcm_sysport_init_dim(struct bcm_sysport_priv *priv, static void bcm_sysport_init_rx_coalesce(struct bcm_sysport_priv *priv) { struct bcm_sysport_net_dim *dim = &priv->dim; - struct net_dim_cq_moder moder; + struct dim_cq_moder moder; u32 usecs, pkts; usecs = priv->rx_coalesce_usecs; diff --git a/drivers/net/ethernet/broadcom/bcmsysport.h b/drivers/net/ethernet/broadcom/bcmsysport.h index 6f3141c86436..6d80735fbc7f 100644 --- a/drivers/net/ethernet/broadcom/bcmsysport.h +++ b/drivers/net/ethernet/broadcom/bcmsysport.h @@ -1,11 +1,8 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ /* * Broadcom BCM7xxx System Port Ethernet MAC driver * * Copyright (C) 2014 Broadcom Corporation - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. */ #ifndef __BCM_SYSPORT_H @@ -14,7 +11,7 @@ #include <linux/bitmap.h> #include <linux/ethtool.h> #include <linux/if_vlan.h> -#include <linux/net_dim.h> +#include <linux/dim.h> /* Receive/transmit descriptor format */ #define DESC_ADDR_HI_STATUS_LEN 0x00 @@ -705,7 +702,7 @@ struct bcm_sysport_net_dim { u16 event_ctr; unsigned long packets; unsigned long bytes; - struct net_dim dim; + struct dim dim; }; /* Software view of the TX ring */ diff --git a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_cmn.c b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_cmn.c index 008ad0ca89ba..e2be5a685130 100644 --- a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_cmn.c +++ b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_cmn.c @@ -285,6 +285,9 @@ int bnx2x_tx_int(struct bnx2x *bp, struct bnx2x_fp_txdata *txdata) hw_cons = le16_to_cpu(*txdata->tx_cons_sb); sw_cons = txdata->tx_pkt_cons; + /* Ensure subsequent loads occur after hw_cons */ + smp_rmb(); + while (sw_cons != hw_cons) { u16 pkt_cons; @@ -684,7 +687,7 @@ static void *bnx2x_frag_alloc(const struct bnx2x_fastpath *fp, gfp_t gfp_mask) if (unlikely(gfpflags_allow_blocking(gfp_mask))) return (void *)__get_free_page(gfp_mask); - return netdev_alloc_frag(fp->rx_frag_size); + return napi_alloc_frag(fp->rx_frag_size); } return kmalloc(fp->rx_buf_size + NET_SKB_PAD, gfp_mask); @@ -3857,9 +3860,12 @@ netdev_tx_t bnx2x_start_xmit(struct sk_buff *skb, struct net_device *dev) if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) { if (!(bp->flags & TX_TIMESTAMPING_EN)) { + bp->eth_stats.ptp_skip_tx_ts++; BNX2X_ERR("Tx timestamping was not enabled, this packet will not be timestamped\n"); } else if (bp->ptp_tx_skb) { - BNX2X_ERR("The device supports only a single outstanding packet to timestamp, this packet will not be timestamped\n"); + bp->eth_stats.ptp_skip_tx_ts++; + netdev_err_once(bp->dev, + "Device supports only a single outstanding packet to timestamp, this packet won't be timestamped\n"); } else { skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; /* schedule check for Tx timestamp */ diff --git a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_ethtool.c b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_ethtool.c index 0745cccd416d..4a0ba6801c9e 100644 --- a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_ethtool.c +++ b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_ethtool.c @@ -182,7 +182,9 @@ static const struct { { STATS_OFFSET32(driver_filtered_tx_pkt), 4, false, "driver_filtered_tx_pkt" }, { STATS_OFFSET32(eee_tx_lpi), - 4, true, "Tx LPI entry count"} + 4, true, "Tx LPI entry count"}, + { STATS_OFFSET32(ptp_skip_tx_ts), + 4, false, "ptp_skipped_tx_tstamp" }, }; #define BNX2X_NUM_STATS ARRAY_SIZE(bnx2x_stats_arr) @@ -1609,7 +1611,8 @@ static int bnx2x_get_module_info(struct net_device *dev, } if (!sff8472_comp || - (diag_type & SFP_EEPROM_DIAG_ADDR_CHANGE_REQ)) { + (diag_type & SFP_EEPROM_DIAG_ADDR_CHANGE_REQ) || + !(diag_type & SFP_EEPROM_DDM_IMPLEMENTED)) { modinfo->type = ETH_MODULE_SFF_8079; modinfo->eeprom_len = ETH_MODULE_SFF_8079_LEN; } else { diff --git a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_link.h b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_link.h index b7d251108c19..7115f5025664 100644 --- a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_link.h +++ b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_link.h @@ -62,6 +62,7 @@ #define SFP_EEPROM_DIAG_TYPE_ADDR 0x5c #define SFP_EEPROM_DIAG_TYPE_SIZE 1 #define SFP_EEPROM_DIAG_ADDR_CHANGE_REQ (1<<2) +#define SFP_EEPROM_DDM_IMPLEMENTED (1<<6) #define SFP_EEPROM_SFF_8472_COMP_ADDR 0x5e #define SFP_EEPROM_SFF_8472_COMP_SIZE 1 diff --git a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_main.c b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_main.c index 03ac10b1cd1e..2cc14db8f0ec 100644 --- a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_main.c +++ b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_main.c @@ -15214,11 +15214,24 @@ static void bnx2x_ptp_task(struct work_struct *work) u32 val_seq; u64 timestamp, ns; struct skb_shared_hwtstamps shhwtstamps; + bool bail = true; + int i; + + /* FW may take a while to complete timestamping; try a bit and if it's + * still not complete, may indicate an error state - bail out then. + */ + for (i = 0; i < 10; i++) { + /* Read Tx timestamp registers */ + val_seq = REG_RD(bp, port ? NIG_REG_P1_TLLH_PTP_BUF_SEQID : + NIG_REG_P0_TLLH_PTP_BUF_SEQID); + if (val_seq & 0x10000) { + bail = false; + break; + } + msleep(1 << i); + } - /* Read Tx timestamp registers */ - val_seq = REG_RD(bp, port ? NIG_REG_P1_TLLH_PTP_BUF_SEQID : - NIG_REG_P0_TLLH_PTP_BUF_SEQID); - if (val_seq & 0x10000) { + if (!bail) { /* There is a valid timestamp value */ timestamp = REG_RD(bp, port ? NIG_REG_P1_TLLH_PTP_BUF_TS_MSB : NIG_REG_P0_TLLH_PTP_BUF_TS_MSB); @@ -15233,16 +15246,18 @@ static void bnx2x_ptp_task(struct work_struct *work) memset(&shhwtstamps, 0, sizeof(shhwtstamps)); shhwtstamps.hwtstamp = ns_to_ktime(ns); skb_tstamp_tx(bp->ptp_tx_skb, &shhwtstamps); - dev_kfree_skb_any(bp->ptp_tx_skb); - bp->ptp_tx_skb = NULL; DP(BNX2X_MSG_PTP, "Tx timestamp, timestamp cycles = %llu, ns = %llu\n", timestamp, ns); } else { - DP(BNX2X_MSG_PTP, "There is no valid Tx timestamp yet\n"); - /* Reschedule to keep checking for a valid timestamp value */ - schedule_work(&bp->ptp_task); + DP(BNX2X_MSG_PTP, + "Tx timestamp is not recorded (register read=%u)\n", + val_seq); + bp->eth_stats.ptp_skip_tx_ts++; } + + dev_kfree_skb_any(bp->ptp_tx_skb); + bp->ptp_tx_skb = NULL; } void bnx2x_set_rx_ts(struct bnx2x *bp, struct sk_buff *skb) diff --git a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_stats.h b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_stats.h index b2644ed13d06..d55e63692cf3 100644 --- a/drivers/net/ethernet/broadcom/bnx2x/bnx2x_stats.h +++ b/drivers/net/ethernet/broadcom/bnx2x/bnx2x_stats.h @@ -207,6 +207,9 @@ struct bnx2x_eth_stats { u32 driver_filtered_tx_pkt; /* src: Clear-on-Read register; Will not survive PMF Migration */ u32 eee_tx_lpi; + + /* PTP */ + u32 ptp_skip_tx_ts; }; struct bnx2x_eth_q_stats { diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt.c b/drivers/net/ethernet/broadcom/bnxt/bnxt.c index 8314c00d7537..7134d2c3eb1c 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt.c +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt.c @@ -54,6 +54,7 @@ #include <net/pkt_cls.h> #include <linux/hwmon.h> #include <linux/hwmon-sysfs.h> +#include <net/page_pool.h> #include "bnxt_hsi.h" #include "bnxt.h" @@ -668,19 +669,20 @@ next_tx_int: } static struct page *__bnxt_alloc_rx_page(struct bnxt *bp, dma_addr_t *mapping, + struct bnxt_rx_ring_info *rxr, gfp_t gfp) { struct device *dev = &bp->pdev->dev; struct page *page; - page = alloc_page(gfp); + page = page_pool_dev_alloc_pages(rxr->page_pool); if (!page) return NULL; *mapping = dma_map_page_attrs(dev, page, 0, PAGE_SIZE, bp->rx_dir, DMA_ATTR_WEAK_ORDERING); if (dma_mapping_error(dev, *mapping)) { - __free_page(page); + page_pool_recycle_direct(rxr->page_pool, page); return NULL; } *mapping += bp->rx_dma_offset; @@ -716,7 +718,8 @@ int bnxt_alloc_rx_data(struct bnxt *bp, struct bnxt_rx_ring_info *rxr, dma_addr_t mapping; if (BNXT_RX_PAGE_MODE(bp)) { - struct page *page = __bnxt_alloc_rx_page(bp, &mapping, gfp); + struct page *page = + __bnxt_alloc_rx_page(bp, &mapping, rxr, gfp); if (!page) return -ENOMEM; @@ -1642,6 +1645,8 @@ static int bnxt_rx_pkt(struct bnxt *bp, struct bnxt_cp_ring_info *cpr, skb = bnxt_copy_skb(bnapi, data_ptr, len, dma_addr); bnxt_reuse_rx_data(rxr, cons, data); if (!skb) { + if (agg_bufs) + bnxt_reuse_rx_agg_bufs(cpr, cp_cons, agg_bufs); rc = -ENOMEM; goto next_rx; } @@ -1987,6 +1992,9 @@ static int __bnxt_poll_work(struct bnxt *bp, struct bnxt_cp_ring_info *cpr, } } + if (event & BNXT_REDIRECT_EVENT) + xdp_do_flush_map(); + if (event & BNXT_TX_EVENT) { struct bnxt_tx_ring_info *txr = bnapi->tx_ring; u16 prod = txr->tx_prod; @@ -2128,12 +2136,12 @@ static int bnxt_poll(struct napi_struct *napi, int budget) } } if (bp->flags & BNXT_FLAG_DIM) { - struct net_dim_sample dim_sample; + struct dim_sample dim_sample; - net_dim_sample(cpr->event_ctr, - cpr->rx_packets, - cpr->rx_bytes, - &dim_sample); + dim_update_sample(cpr->event_ctr, + cpr->rx_packets, + cpr->rx_bytes, + &dim_sample); net_dim(&cpr->dim, dim_sample); } return work_done; @@ -2252,9 +2260,23 @@ static void bnxt_free_tx_skbs(struct bnxt *bp) for (j = 0; j < max_idx;) { struct bnxt_sw_tx_bd *tx_buf = &txr->tx_buf_ring[j]; - struct sk_buff *skb = tx_buf->skb; + struct sk_buff *skb; int k, last; + if (i < bp->tx_nr_rings_xdp && + tx_buf->action == XDP_REDIRECT) { + dma_unmap_single(&pdev->dev, + dma_unmap_addr(tx_buf, mapping), + dma_unmap_len(tx_buf, len), + PCI_DMA_TODEVICE); + xdp_return_frame(tx_buf->xdpf); + tx_buf->action = 0; + tx_buf->xdpf = NULL; + j++; + continue; + } + + skb = tx_buf->skb; if (!skb) { j++; continue; @@ -2341,7 +2363,7 @@ static void bnxt_free_rx_skbs(struct bnxt *bp) dma_unmap_page_attrs(&pdev->dev, mapping, PAGE_SIZE, bp->rx_dir, DMA_ATTR_WEAK_ORDERING); - __free_page(data); + page_pool_recycle_direct(rxr->page_pool, data); } else { dma_unmap_single_attrs(&pdev->dev, mapping, bp->rx_buf_use_size, @@ -2478,6 +2500,9 @@ static void bnxt_free_rx_rings(struct bnxt *bp) if (xdp_rxq_info_is_reg(&rxr->xdp_rxq)) xdp_rxq_info_unreg(&rxr->xdp_rxq); + page_pool_destroy(rxr->page_pool); + rxr->page_pool = NULL; + kfree(rxr->rx_tpa); rxr->rx_tpa = NULL; @@ -2492,6 +2517,26 @@ static void bnxt_free_rx_rings(struct bnxt *bp) } } +static int bnxt_alloc_rx_page_pool(struct bnxt *bp, + struct bnxt_rx_ring_info *rxr) +{ + struct page_pool_params pp = { 0 }; + + pp.pool_size = bp->rx_ring_size; + pp.nid = dev_to_node(&bp->pdev->dev); + pp.dev = &bp->pdev->dev; + pp.dma_dir = DMA_BIDIRECTIONAL; + + rxr->page_pool = page_pool_create(&pp); + if (IS_ERR(rxr->page_pool)) { + int err = PTR_ERR(rxr->page_pool); + + rxr->page_pool = NULL; + return err; + } + return 0; +} + static int bnxt_alloc_rx_rings(struct bnxt *bp) { int i, rc, agg_rings = 0, tpa_rings = 0; @@ -2511,10 +2556,22 @@ static int bnxt_alloc_rx_rings(struct bnxt *bp) ring = &rxr->rx_ring_struct; + rc = bnxt_alloc_rx_page_pool(bp, rxr); + if (rc) + return rc; + rc = xdp_rxq_info_reg(&rxr->xdp_rxq, bp->dev, i); if (rc < 0) return rc; + rc = xdp_rxq_info_reg_mem_model(&rxr->xdp_rxq, + MEM_TYPE_PAGE_POOL, + rxr->page_pool); + if (rc) { + xdp_rxq_info_unreg(&rxr->xdp_rxq); + return rc; + } + rc = bnxt_alloc_ring(bp, &ring->ring_mem); if (rc) return rc; @@ -2620,8 +2677,6 @@ static int bnxt_alloc_tx_rings(struct bnxt *bp) mapping = txr->tx_push_mapping + sizeof(struct tx_push_bd); txr->data_mapping = cpu_to_le64(mapping); - - memset(txr->tx_push, 0, sizeof(struct tx_push_bd)); } qidx = bp->tc_to_qidx[j]; ring->queue_id = bp->q_info[qidx].queue_id; @@ -3020,7 +3075,7 @@ static int bnxt_alloc_vnics(struct bnxt *bp) int num_vnics = 1; #ifdef CONFIG_RFS_ACCEL - if (bp->flags & BNXT_FLAG_RFS) + if ((bp->flags & (BNXT_FLAG_RFS | BNXT_FLAG_CHIP_P5)) == BNXT_FLAG_RFS) num_vnics += bp->rx_nr_rings; #endif @@ -5506,7 +5561,16 @@ static int bnxt_cp_rings_in_use(struct bnxt *bp) static int bnxt_get_func_stat_ctxs(struct bnxt *bp) { - return bp->cp_nr_rings + bnxt_get_ulp_stat_ctxs(bp); + int ulp_stat = bnxt_get_ulp_stat_ctxs(bp); + int cp = bp->cp_nr_rings; + + if (!ulp_stat) + return cp; + + if (bnxt_nq_rings_in_use(bp) > cp + bnxt_get_ulp_msix_num(bp)) + return bnxt_get_ulp_msix_base(bp) + ulp_stat; + + return cp + ulp_stat; } static bool bnxt_need_reserve_rings(struct bnxt *bp) @@ -6377,7 +6441,7 @@ static int bnxt_alloc_ctx_mem(struct bnxt *bp) if (!ctx || (ctx->flags & BNXT_CTX_FLAG_INITED)) return 0; - if (bp->flags & BNXT_FLAG_ROCE_CAP) { + if ((bp->flags & BNXT_FLAG_ROCE_CAP) && !is_kdump_kernel()) { pg_lvl = 2; extra_qps = 65536; extra_srqs = 8192; @@ -7122,6 +7186,9 @@ static int bnxt_alloc_rfs_vnics(struct bnxt *bp) #ifdef CONFIG_RFS_ACCEL int i, rc = 0; + if (bp->flags & BNXT_FLAG_CHIP_P5) + return 0; + for (i = 0; i < bp->rx_nr_rings; i++) { struct bnxt_vnic_info *vnic; u16 vnic_id = i + 1; @@ -7475,11 +7542,7 @@ unsigned int bnxt_get_avail_cp_rings_for_en(struct bnxt *bp) unsigned int bnxt_get_avail_stat_ctxs_for_en(struct bnxt *bp) { - unsigned int stat; - - stat = bnxt_get_max_func_stat_ctxs(bp) - bnxt_get_ulp_stat_ctxs(bp); - stat -= bp->cp_nr_rings; - return stat; + return bnxt_get_max_func_stat_ctxs(bp) - bnxt_get_func_stat_ctxs(bp); } int bnxt_get_avail_msix(struct bnxt *bp, int num) @@ -7616,22 +7679,23 @@ static void bnxt_clear_int_mode(struct bnxt *bp) bp->flags &= ~BNXT_FLAG_USING_MSIX; } -int bnxt_reserve_rings(struct bnxt *bp) +int bnxt_reserve_rings(struct bnxt *bp, bool irq_re_init) { int tcs = netdev_get_num_tc(bp->dev); - bool reinit_irq = false; + bool irq_cleared = false; int rc; if (!bnxt_need_reserve_rings(bp)) return 0; - if (BNXT_NEW_RM(bp) && (bnxt_get_num_msix(bp) != bp->total_irqs)) { + if (irq_re_init && BNXT_NEW_RM(bp) && + bnxt_get_num_msix(bp) != bp->total_irqs) { bnxt_ulp_irq_stop(bp); bnxt_clear_int_mode(bp); - reinit_irq = true; + irq_cleared = true; } rc = __bnxt_reserve_rings(bp); - if (reinit_irq) { + if (irq_cleared) { if (!rc) rc = bnxt_init_int_mode(bp); bnxt_ulp_irq_restart(bp, rc); @@ -7810,7 +7874,7 @@ static void bnxt_enable_napi(struct bnxt *bp) if (bp->bnapi[i]->rx_ring) { INIT_WORK(&cpr->dim.work, bnxt_dim_work); - cpr->dim.mode = NET_DIM_CQ_PERIOD_MODE_START_FROM_EQE; + cpr->dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_EQE; } napi_enable(&bp->bnapi[i]->napi); } @@ -8530,7 +8594,7 @@ static int __bnxt_open_nic(struct bnxt *bp, bool irq_re_init, bool link_re_init) return rc; } } - rc = bnxt_reserve_rings(bp); + rc = bnxt_reserve_rings(bp, irq_re_init); if (rc) return rc; if ((bp->flags & BNXT_FLAG_RFS) && @@ -9584,7 +9648,7 @@ int bnxt_check_rings(struct bnxt *bp, int tx, int rx, bool sh, int tcs, return -ENOMEM; vnics = 1; - if (bp->flags & BNXT_FLAG_RFS) + if ((bp->flags & (BNXT_FLAG_RFS | BNXT_FLAG_CHIP_P5)) == BNXT_FLAG_RFS) vnics += rx_rings; if (bp->flags & BNXT_FLAG_AGG_RINGS) @@ -9844,32 +9908,19 @@ static int bnxt_setup_tc_block_cb(enum tc_setup_type type, void *type_data, } } -static int bnxt_setup_tc_block(struct net_device *dev, - struct tc_block_offload *f) -{ - struct bnxt *bp = netdev_priv(dev); - - if (f->binder_type != TCF_BLOCK_BINDER_TYPE_CLSACT_INGRESS) - return -EOPNOTSUPP; - - switch (f->command) { - case TC_BLOCK_BIND: - return tcf_block_cb_register(f->block, bnxt_setup_tc_block_cb, - bp, bp, f->extack); - case TC_BLOCK_UNBIND: - tcf_block_cb_unregister(f->block, bnxt_setup_tc_block_cb, bp); - return 0; - default: - return -EOPNOTSUPP; - } -} +static LIST_HEAD(bnxt_block_cb_list); static int bnxt_setup_tc(struct net_device *dev, enum tc_setup_type type, void *type_data) { + struct bnxt *bp = netdev_priv(dev); + switch (type) { case TC_SETUP_BLOCK: - return bnxt_setup_tc_block(dev, type_data); + return flow_block_cb_setup_simple(type_data, + &bnxt_block_cb_list, + bnxt_setup_tc_block_cb, + bp, bp, true); case TC_SETUP_QDISC_MQPRIO: { struct tc_mqprio_qopt *mqprio = type_data; @@ -10230,6 +10281,7 @@ static const struct net_device_ops bnxt_netdev_ops = { .ndo_udp_tunnel_add = bnxt_udp_tunnel_add, .ndo_udp_tunnel_del = bnxt_udp_tunnel_del, .ndo_bpf = bnxt_xdp, + .ndo_xdp_xmit = bnxt_xdp_xmit, .ndo_bridge_getlink = bnxt_bridge_getlink, .ndo_bridge_setlink = bnxt_bridge_setlink, .ndo_get_devlink_port = bnxt_get_devlink_port, @@ -10259,10 +10311,10 @@ static void bnxt_remove_one(struct pci_dev *pdev) bnxt_dcb_free(bp); kfree(bp->edev); bp->edev = NULL; + bnxt_cleanup_pci(bp); bnxt_free_ctx_mem(bp); kfree(bp->ctx); bp->ctx = NULL; - bnxt_cleanup_pci(bp); bnxt_free_port_stats(bp); free_netdev(dev); } @@ -10434,7 +10486,7 @@ static int bnxt_set_dflt_rings(struct bnxt *bp, bool sh) if (sh) bp->flags |= BNXT_FLAG_SHARED_RINGS; - dflt_rings = netif_get_num_default_rss_queues(); + dflt_rings = is_kdump_kernel() ? 1 : netif_get_num_default_rss_queues(); /* Reduce default rings on multi-port cards so that total default * rings do not exceed CPU count. */ @@ -10722,11 +10774,12 @@ static int bnxt_init_one(struct pci_dev *pdev, const struct pci_device_id *ent) goto init_err_pci_clean; } - /* Read the adapter's DSN to use as the eswitch switch_id */ - rc = bnxt_pcie_dsn_get(bp, bp->switch_id); - if (rc) - goto init_err_pci_clean; - + if (BNXT_PF(bp)) { + /* Read the adapter's DSN to use as the eswitch switch_id */ + rc = bnxt_pcie_dsn_get(bp, bp->switch_id); + if (rc) + goto init_err_pci_clean; + } bnxt_hwrm_func_qcfg(bp); bnxt_hwrm_vnic_qcaps(bp); bnxt_hwrm_port_led_qcaps(bp); @@ -10855,6 +10908,7 @@ static void bnxt_shutdown(struct pci_dev *pdev) if (system_state == SYSTEM_POWER_OFF) { bnxt_clear_int_mode(bp); + pci_disable_device(pdev); pci_wake_from_d3(pdev, bp->wol); pci_set_power_state(pdev, PCI_D3hot); } diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt.h b/drivers/net/ethernet/broadcom/bnxt/bnxt.h index eca36dd6b751..16694b704d15 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt.h +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt.h @@ -20,10 +20,13 @@ #include <linux/interrupt.h> #include <linux/rhashtable.h> +#include <linux/crash_dump.h> #include <net/devlink.h> #include <net/dst_metadata.h> #include <net/xdp.h> -#include <linux/net_dim.h> +#include <linux/dim.h> + +struct page_pool; struct tx_bd { __le32 tx_bd_len_flags_type; @@ -586,15 +589,21 @@ struct nqe_cn { #define BNXT_HWRM_CHNL_CHIMP 0 #define BNXT_HWRM_CHNL_KONG 1 -#define BNXT_RX_EVENT 1 -#define BNXT_AGG_EVENT 2 -#define BNXT_TX_EVENT 4 +#define BNXT_RX_EVENT 1 +#define BNXT_AGG_EVENT 2 +#define BNXT_TX_EVENT 4 +#define BNXT_REDIRECT_EVENT 8 struct bnxt_sw_tx_bd { - struct sk_buff *skb; + union { + struct sk_buff *skb; + struct xdp_frame *xdpf; + }; DEFINE_DMA_UNMAP_ADDR(mapping); + DEFINE_DMA_UNMAP_LEN(len); u8 is_gso; u8 is_push; + u8 action; union { unsigned short nr_frags; u16 rx_prod; @@ -792,6 +801,7 @@ struct bnxt_rx_ring_info { struct bnxt_ring_struct rx_ring_struct; struct bnxt_ring_struct rx_agg_ring_struct; struct xdp_rxq_info xdp_rxq; + struct page_pool *page_pool; }; struct bnxt_cp_ring_info { @@ -809,7 +819,7 @@ struct bnxt_cp_ring_info { u64 rx_bytes; u64 event_ctr; - struct net_dim dim; + struct dim dim; union { struct tx_cmp *cp_desc_ring[MAX_CP_PAGES]; @@ -1369,7 +1379,8 @@ struct bnxt { #define BNXT_CHIP_TYPE_NITRO_A0(bp) ((bp)->flags & BNXT_FLAG_CHIP_NITRO_A0) #define BNXT_RX_PAGE_MODE(bp) ((bp)->flags & BNXT_FLAG_RX_PAGE_MODE) #define BNXT_SUPPORTS_TPA(bp) (!BNXT_CHIP_TYPE_NITRO_A0(bp) && \ - !(bp->flags & BNXT_FLAG_CHIP_P5)) + !(bp->flags & BNXT_FLAG_CHIP_P5) && \ + !is_kdump_kernel()) /* Chip class phase 5 */ #define BNXT_CHIP_P5(bp) \ @@ -1790,7 +1801,7 @@ unsigned int bnxt_get_avail_stat_ctxs_for_en(struct bnxt *bp); unsigned int bnxt_get_max_func_cp_rings(struct bnxt *bp); unsigned int bnxt_get_avail_cp_rings_for_en(struct bnxt *bp); int bnxt_get_avail_msix(struct bnxt *bp, int num); -int bnxt_reserve_rings(struct bnxt *bp); +int bnxt_reserve_rings(struct bnxt *bp, bool irq_re_init); void bnxt_tx_disable(struct bnxt *bp); void bnxt_tx_enable(struct bnxt *bp); int bnxt_hwrm_set_pause(struct bnxt *); diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_dcb.c b/drivers/net/ethernet/broadcom/bnxt/bnxt_dcb.c index 70775158c8c4..07301cb87c03 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt_dcb.c +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_dcb.c @@ -396,7 +396,7 @@ static int bnxt_hwrm_queue_dscp_qcaps(struct bnxt *bp) bnxt_hwrm_cmd_hdr_init(bp, &req, HWRM_QUEUE_DSCP_QCAPS, -1, -1); mutex_lock(&bp->hwrm_cmd_lock); - rc = _hwrm_send_message(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); + rc = _hwrm_send_message_silent(bp, &req, sizeof(req), HWRM_CMD_TIMEOUT); if (!rc) { bp->max_dscp_value = (1 << resp->num_dscp_bits) - 1; if (bp->max_dscp_value < 0x3f) diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_debugfs.c b/drivers/net/ethernet/broadcom/bnxt/bnxt_debugfs.c index 94e208e9789f..61393f351a77 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt_debugfs.c +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_debugfs.c @@ -11,7 +11,7 @@ #include <linux/module.h> #include <linux/pci.h> #include "bnxt_hsi.h" -#include <linux/net_dim.h> +#include <linux/dim.h> #include "bnxt.h" #include "bnxt_debugfs.h" @@ -21,7 +21,7 @@ static ssize_t debugfs_dim_read(struct file *filep, char __user *buffer, size_t count, loff_t *ppos) { - struct net_dim *dim = filep->private_data; + struct dim *dim = filep->private_data; int len; char *buf; @@ -61,7 +61,7 @@ static const struct file_operations debugfs_dim_fops = { .read = debugfs_dim_read, }; -static struct dentry *debugfs_dim_ring_init(struct net_dim *dim, int ring_idx, +static struct dentry *debugfs_dim_ring_init(struct dim *dim, int ring_idx, struct dentry *dd) { static char qname[16]; diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_dim.c b/drivers/net/ethernet/broadcom/bnxt/bnxt_dim.c index afa97c8bb081..6f6576dc417a 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt_dim.c +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_dim.c @@ -7,26 +7,25 @@ * the Free Software Foundation. */ -#include <linux/net_dim.h> +#include <linux/dim.h> #include "bnxt_hsi.h" #include "bnxt.h" void bnxt_dim_work(struct work_struct *work) { - struct net_dim *dim = container_of(work, struct net_dim, - work); + struct dim *dim = container_of(work, struct dim, work); struct bnxt_cp_ring_info *cpr = container_of(dim, struct bnxt_cp_ring_info, dim); struct bnxt_napi *bnapi = container_of(cpr, struct bnxt_napi, cp_ring); - struct net_dim_cq_moder cur_moder = + struct dim_cq_moder cur_moder = net_dim_get_rx_moderation(dim->mode, dim->profile_ix); cpr->rx_ring_coal.coal_ticks = cur_moder.usec; cpr->rx_ring_coal.coal_bufs = cur_moder.pkts; bnxt_hwrm_set_ring_coal(bnapi->bp, bnapi); - dim->state = NET_DIM_START_MEASURE; + dim->state = DIM_START_MEASURE; } diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c b/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c index b1263821a6e9..c7ee63d69679 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_ethtool.c @@ -831,7 +831,7 @@ static int bnxt_set_channels(struct net_device *dev, */ } } else { - rc = bnxt_reserve_rings(bp); + rc = bnxt_reserve_rings(bp, true); } return rc; @@ -2799,7 +2799,7 @@ static int bnxt_run_loopback(struct bnxt *bp) dev_kfree_skb(skb); return -EIO; } - bnxt_xmit_xdp(bp, txr, map, pkt_size, 0); + bnxt_xmit_bd(bp, txr, map, pkt_size); /* Sync BD data before updating doorbell */ wmb(); @@ -2842,7 +2842,7 @@ static void bnxt_self_test(struct net_device *dev, struct ethtool_test *etest, bool offline = false; u8 test_results = 0; u8 test_mask = 0; - int rc, i; + int rc = 0, i; if (!bp->num_tests || !BNXT_SINGLE_PF(bp)) return; @@ -2913,9 +2913,9 @@ static void bnxt_self_test(struct net_device *dev, struct ethtool_test *etest, } bnxt_hwrm_phy_loopback(bp, false, false); bnxt_half_close_nic(bp); - bnxt_open_nic(bp, false, true); + rc = bnxt_open_nic(bp, false, true); } - if (bnxt_test_irq(bp)) { + if (rc || bnxt_test_irq(bp)) { buf[BNXT_IRQ_TEST_IDX] = 1; etest->flags |= ETH_TEST_FL_FAILED; } diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_tc.c b/drivers/net/ethernet/broadcom/bnxt/bnxt_tc.c index 44d6c5743fb9..6fe4a7174271 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt_tc.c +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_tc.c @@ -170,10 +170,10 @@ static int bnxt_tc_parse_actions(struct bnxt *bp, } static int bnxt_tc_parse_flow(struct bnxt *bp, - struct tc_cls_flower_offload *tc_flow_cmd, + struct flow_cls_offload *tc_flow_cmd, struct bnxt_tc_flow *flow) { - struct flow_rule *rule = tc_cls_flower_offload_flow_rule(tc_flow_cmd); + struct flow_rule *rule = flow_cls_offload_flow_rule(tc_flow_cmd); struct flow_dissector *dissector = rule->match.dissector; /* KEY_CONTROL and KEY_BASIC are needed for forming a meaningful key */ @@ -1262,7 +1262,7 @@ static void bnxt_tc_set_src_fid(struct bnxt *bp, struct bnxt_tc_flow *flow, * The hash-tables are already protected by the rhashtable API. */ static int bnxt_tc_add_flow(struct bnxt *bp, u16 src_fid, - struct tc_cls_flower_offload *tc_flow_cmd) + struct flow_cls_offload *tc_flow_cmd) { struct bnxt_tc_flow_node *new_node, *old_node; struct bnxt_tc_info *tc_info = bp->tc_info; @@ -1348,7 +1348,7 @@ done: } static int bnxt_tc_del_flow(struct bnxt *bp, - struct tc_cls_flower_offload *tc_flow_cmd) + struct flow_cls_offload *tc_flow_cmd) { struct bnxt_tc_info *tc_info = bp->tc_info; struct bnxt_tc_flow_node *flow_node; @@ -1363,7 +1363,7 @@ static int bnxt_tc_del_flow(struct bnxt *bp, } static int bnxt_tc_get_flow_stats(struct bnxt *bp, - struct tc_cls_flower_offload *tc_flow_cmd) + struct flow_cls_offload *tc_flow_cmd) { struct bnxt_tc_flow_stats stats, *curr_stats, *prev_stats; struct bnxt_tc_info *tc_info = bp->tc_info; @@ -1585,14 +1585,14 @@ void bnxt_tc_flow_stats_work(struct bnxt *bp) } int bnxt_tc_setup_flower(struct bnxt *bp, u16 src_fid, - struct tc_cls_flower_offload *cls_flower) + struct flow_cls_offload *cls_flower) { switch (cls_flower->command) { - case TC_CLSFLOWER_REPLACE: + case FLOW_CLS_REPLACE: return bnxt_tc_add_flow(bp, src_fid, cls_flower); - case TC_CLSFLOWER_DESTROY: + case FLOW_CLS_DESTROY: return bnxt_tc_del_flow(bp, cls_flower); - case TC_CLSFLOWER_STATS: + case FLOW_CLS_STATS: return bnxt_tc_get_flow_stats(bp, cls_flower); default: return -EOPNOTSUPP; diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_tc.h b/drivers/net/ethernet/broadcom/bnxt/bnxt_tc.h index 8a0968967bc5..ffec57d1a5ec 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt_tc.h +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_tc.h @@ -196,7 +196,7 @@ struct bnxt_tc_flow_node { }; int bnxt_tc_setup_flower(struct bnxt *bp, u16 src_fid, - struct tc_cls_flower_offload *cls_flower); + struct flow_cls_offload *cls_flower); int bnxt_init_tc(struct bnxt *bp); void bnxt_shutdown_tc(struct bnxt *bp); void bnxt_tc_flow_stats_work(struct bnxt *bp); @@ -209,7 +209,7 @@ static inline bool bnxt_tc_flower_enabled(struct bnxt *bp) #else /* CONFIG_BNXT_FLOWER_OFFLOAD */ static inline int bnxt_tc_setup_flower(struct bnxt *bp, u16 src_fid, - struct tc_cls_flower_offload *cls_flower) + struct flow_cls_offload *cls_flower) { return -EOPNOTSUPP; } diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_ulp.c b/drivers/net/ethernet/broadcom/bnxt/bnxt_ulp.c index cf475873ce81..fc77caf0a076 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt_ulp.c +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_ulp.c @@ -147,7 +147,7 @@ static int bnxt_req_msix_vecs(struct bnxt_en_dev *edev, int ulp_id, bnxt_close_nic(bp, true, false); rc = bnxt_open_nic(bp, true, false); } else { - rc = bnxt_reserve_rings(bp); + rc = bnxt_reserve_rings(bp, true); } } if (rc) { @@ -157,8 +157,10 @@ static int bnxt_req_msix_vecs(struct bnxt_en_dev *edev, int ulp_id, if (BNXT_NEW_RM(bp)) { struct bnxt_hw_resc *hw_resc = &bp->hw_resc; + int resv_msix; - avail_msix = hw_resc->resv_irqs - bp->cp_nr_rings; + resv_msix = hw_resc->resv_irqs - bp->cp_nr_rings; + avail_msix = min_t(int, resv_msix, avail_msix); edev->ulp_tbl[ulp_id].msix_requested = avail_msix; } bnxt_fill_msix_vecs(bp, ent); diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_vfr.c b/drivers/net/ethernet/broadcom/bnxt/bnxt_vfr.c index f760921389a3..f9bf7d7250ab 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt_vfr.c +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_vfr.c @@ -161,34 +161,19 @@ static int bnxt_vf_rep_setup_tc_block_cb(enum tc_setup_type type, } } -static int bnxt_vf_rep_setup_tc_block(struct net_device *dev, - struct tc_block_offload *f) -{ - struct bnxt_vf_rep *vf_rep = netdev_priv(dev); - - if (f->binder_type != TCF_BLOCK_BINDER_TYPE_CLSACT_INGRESS) - return -EOPNOTSUPP; - - switch (f->command) { - case TC_BLOCK_BIND: - return tcf_block_cb_register(f->block, - bnxt_vf_rep_setup_tc_block_cb, - vf_rep, vf_rep, f->extack); - case TC_BLOCK_UNBIND: - tcf_block_cb_unregister(f->block, - bnxt_vf_rep_setup_tc_block_cb, vf_rep); - return 0; - default: - return -EOPNOTSUPP; - } -} +static LIST_HEAD(bnxt_vf_block_cb_list); static int bnxt_vf_rep_setup_tc(struct net_device *dev, enum tc_setup_type type, void *type_data) { + struct bnxt_vf_rep *vf_rep = netdev_priv(dev); + switch (type) { case TC_SETUP_BLOCK: - return bnxt_vf_rep_setup_tc_block(dev, type_data); + return flow_block_cb_setup_simple(type_data, + &bnxt_vf_block_cb_list, + bnxt_vf_rep_setup_tc_block_cb, + vf_rep, vf_rep, true); default: return -EOPNOTSUPP; } diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.c b/drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.c index 0184ef6f05a7..c6f6f2033880 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.c +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.c @@ -15,12 +15,14 @@ #include <linux/bpf.h> #include <linux/bpf_trace.h> #include <linux/filter.h> +#include <net/page_pool.h> #include "bnxt_hsi.h" #include "bnxt.h" #include "bnxt_xdp.h" -void bnxt_xmit_xdp(struct bnxt *bp, struct bnxt_tx_ring_info *txr, - dma_addr_t mapping, u32 len, u16 rx_prod) +struct bnxt_sw_tx_bd *bnxt_xmit_bd(struct bnxt *bp, + struct bnxt_tx_ring_info *txr, + dma_addr_t mapping, u32 len) { struct bnxt_sw_tx_bd *tx_buf; struct tx_bd *txbd; @@ -29,7 +31,6 @@ void bnxt_xmit_xdp(struct bnxt *bp, struct bnxt_tx_ring_info *txr, prod = txr->tx_prod; tx_buf = &txr->tx_buf_ring[prod]; - tx_buf->rx_prod = rx_prod; txbd = &txr->tx_desc_ring[TX_RING(prod)][TX_IDX(prod)]; flags = (len << TX_BD_LEN_SHIFT) | (1 << TX_BD_FLAGS_BD_CNT_SHIFT) | @@ -40,30 +41,67 @@ void bnxt_xmit_xdp(struct bnxt *bp, struct bnxt_tx_ring_info *txr, prod = NEXT_TX(prod); txr->tx_prod = prod; + return tx_buf; +} + +static void __bnxt_xmit_xdp(struct bnxt *bp, struct bnxt_tx_ring_info *txr, + dma_addr_t mapping, u32 len, u16 rx_prod) +{ + struct bnxt_sw_tx_bd *tx_buf; + + tx_buf = bnxt_xmit_bd(bp, txr, mapping, len); + tx_buf->rx_prod = rx_prod; + tx_buf->action = XDP_TX; +} + +static void __bnxt_xmit_xdp_redirect(struct bnxt *bp, + struct bnxt_tx_ring_info *txr, + dma_addr_t mapping, u32 len, + struct xdp_frame *xdpf) +{ + struct bnxt_sw_tx_bd *tx_buf; + + tx_buf = bnxt_xmit_bd(bp, txr, mapping, len); + tx_buf->action = XDP_REDIRECT; + tx_buf->xdpf = xdpf; + dma_unmap_addr_set(tx_buf, mapping, mapping); + dma_unmap_len_set(tx_buf, len, 0); } void bnxt_tx_int_xdp(struct bnxt *bp, struct bnxt_napi *bnapi, int nr_pkts) { struct bnxt_tx_ring_info *txr = bnapi->tx_ring; struct bnxt_rx_ring_info *rxr = bnapi->rx_ring; + bool rx_doorbell_needed = false; struct bnxt_sw_tx_bd *tx_buf; u16 tx_cons = txr->tx_cons; u16 last_tx_cons = tx_cons; - u16 rx_prod; int i; for (i = 0; i < nr_pkts; i++) { - last_tx_cons = tx_cons; + tx_buf = &txr->tx_buf_ring[tx_cons]; + + if (tx_buf->action == XDP_REDIRECT) { + struct pci_dev *pdev = bp->pdev; + + dma_unmap_single(&pdev->dev, + dma_unmap_addr(tx_buf, mapping), + dma_unmap_len(tx_buf, len), + PCI_DMA_TODEVICE); + xdp_return_frame(tx_buf->xdpf); + tx_buf->action = 0; + tx_buf->xdpf = NULL; + } else if (tx_buf->action == XDP_TX) { + rx_doorbell_needed = true; + last_tx_cons = tx_cons; + } tx_cons = NEXT_TX(tx_cons); } txr->tx_cons = tx_cons; - if (bnxt_tx_avail(bp, txr) == bp->tx_ring_size) { - rx_prod = rxr->rx_prod; - } else { + if (rx_doorbell_needed) { tx_buf = &txr->tx_buf_ring[last_tx_cons]; - rx_prod = tx_buf->rx_prod; + bnxt_db_write(bp, &rxr->rx_db, tx_buf->rx_prod); } - bnxt_db_write(bp, &rxr->rx_db, rx_prod); } /* returns the following: @@ -88,19 +126,19 @@ bool bnxt_rx_xdp(struct bnxt *bp, struct bnxt_rx_ring_info *rxr, u16 cons, return false; pdev = bp->pdev; - txr = rxr->bnapi->tx_ring; rx_buf = &rxr->rx_buf_ring[cons]; offset = bp->rx_offset; + mapping = rx_buf->mapping - bp->rx_dma_offset; + dma_sync_single_for_cpu(&pdev->dev, mapping + offset, *len, bp->rx_dir); + + txr = rxr->bnapi->tx_ring; xdp.data_hard_start = *data_ptr - offset; xdp.data = *data_ptr; xdp_set_data_meta_invalid(&xdp); xdp.data_end = *data_ptr + *len; xdp.rxq = &rxr->xdp_rxq; orig_data = xdp.data; - mapping = rx_buf->mapping - bp->rx_dma_offset; - - dma_sync_single_for_cpu(&pdev->dev, mapping + offset, *len, bp->rx_dir); rcu_read_lock(); act = bpf_prog_run_xdp(xdp_prog, &xdp); @@ -132,10 +170,34 @@ bool bnxt_rx_xdp(struct bnxt *bp, struct bnxt_rx_ring_info *rxr, u16 cons, *event = BNXT_TX_EVENT; dma_sync_single_for_device(&pdev->dev, mapping + offset, *len, bp->rx_dir); - bnxt_xmit_xdp(bp, txr, mapping + offset, *len, - NEXT_RX(rxr->rx_prod)); + __bnxt_xmit_xdp(bp, txr, mapping + offset, *len, + NEXT_RX(rxr->rx_prod)); bnxt_reuse_rx_data(rxr, cons, page); return true; + case XDP_REDIRECT: + /* if we are calling this here then we know that the + * redirect is coming from a frame received by the + * bnxt_en driver. + */ + dma_unmap_page_attrs(&pdev->dev, mapping, + PAGE_SIZE, bp->rx_dir, + DMA_ATTR_WEAK_ORDERING); + + /* if we are unable to allocate a new buffer, abort and reuse */ + if (bnxt_alloc_rx_data(bp, rxr, rxr->rx_prod, GFP_ATOMIC)) { + trace_xdp_exception(bp->dev, xdp_prog, act); + bnxt_reuse_rx_data(rxr, cons, page); + return true; + } + + if (xdp_do_redirect(bp->dev, &xdp, xdp_prog)) { + trace_xdp_exception(bp->dev, xdp_prog, act); + page_pool_recycle_direct(rxr->page_pool, page); + return true; + } + + *event |= BNXT_REDIRECT_EVENT; + break; default: bpf_warn_invalid_xdp_action(act); /* Fall thru */ @@ -149,6 +211,56 @@ bool bnxt_rx_xdp(struct bnxt *bp, struct bnxt_rx_ring_info *rxr, u16 cons, return true; } +int bnxt_xdp_xmit(struct net_device *dev, int num_frames, + struct xdp_frame **frames, u32 flags) +{ + struct bnxt *bp = netdev_priv(dev); + struct bpf_prog *xdp_prog = READ_ONCE(bp->xdp_prog); + struct pci_dev *pdev = bp->pdev; + struct bnxt_tx_ring_info *txr; + dma_addr_t mapping; + int drops = 0; + int ring; + int i; + + if (!test_bit(BNXT_STATE_OPEN, &bp->state) || + !bp->tx_nr_rings_xdp || + !xdp_prog) + return -EINVAL; + + ring = smp_processor_id() % bp->tx_nr_rings_xdp; + txr = &bp->tx_ring[ring]; + + for (i = 0; i < num_frames; i++) { + struct xdp_frame *xdp = frames[i]; + + if (!txr || !bnxt_tx_avail(bp, txr) || + !(bp->bnapi[ring]->flags & BNXT_NAPI_FLAG_XDP)) { + xdp_return_frame_rx_napi(xdp); + drops++; + continue; + } + + mapping = dma_map_single(&pdev->dev, xdp->data, xdp->len, + DMA_TO_DEVICE); + + if (dma_mapping_error(&pdev->dev, mapping)) { + xdp_return_frame_rx_napi(xdp); + drops++; + continue; + } + __bnxt_xmit_xdp_redirect(bp, txr, mapping, xdp->len, xdp); + } + + if (flags & XDP_XMIT_FLUSH) { + /* Sync BD data before updating doorbell */ + wmb(); + bnxt_db_write(bp, &txr->tx_db, txr->tx_prod); + } + + return num_frames - drops; +} + /* Under rtnl_lock */ static int bnxt_xdp_set(struct bnxt *bp, struct bpf_prog *prog) { diff --git a/drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.h b/drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.h index 414b748038ca..0df40c3beb05 100644 --- a/drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.h +++ b/drivers/net/ethernet/broadcom/bnxt/bnxt_xdp.h @@ -10,12 +10,15 @@ #ifndef BNXT_XDP_H #define BNXT_XDP_H -void bnxt_xmit_xdp(struct bnxt *bp, struct bnxt_tx_ring_info *txr, - dma_addr_t mapping, u32 len, u16 rx_prod); +struct bnxt_sw_tx_bd *bnxt_xmit_bd(struct bnxt *bp, + struct bnxt_tx_ring_info *txr, + dma_addr_t mapping, u32 len); void bnxt_tx_int_xdp(struct bnxt *bp, struct bnxt_napi *bnapi, int nr_pkts); bool bnxt_rx_xdp(struct bnxt *bp, struct bnxt_rx_ring_info *rxr, u16 cons, struct page *page, u8 **data_ptr, unsigned int *len, u8 *event); int bnxt_xdp(struct net_device *dev, struct netdev_bpf *xdp); +int bnxt_xdp_xmit(struct net_device *dev, int num_frames, + struct xdp_frame **frames, u32 flags); #endif diff --git a/drivers/net/ethernet/broadcom/genet/bcmgenet.c b/drivers/net/ethernet/broadcom/genet/bcmgenet.c index 374b9ff05c88..a2b57807453b 100644 --- a/drivers/net/ethernet/broadcom/genet/bcmgenet.c +++ b/drivers/net/ethernet/broadcom/genet/bcmgenet.c @@ -1,11 +1,8 @@ +// SPDX-License-Identifier: GPL-2.0-only /* * Broadcom GENET (Gigabit Ethernet) controller driver * * Copyright (c) 2014-2017 Broadcom - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. */ #define pr_fmt(fmt) "bcmgenet: " fmt @@ -643,7 +640,7 @@ static void bcmgenet_set_rx_coalesce(struct bcmgenet_rx_ring *ring, static void bcmgenet_set_ring_rx_coalesce(struct bcmgenet_rx_ring *ring, struct ethtool_coalesce *ec) { - struct net_dim_cq_moder moder; + struct dim_cq_moder moder; u32 usecs, pkts; ring->rx_coalesce_usecs = ec->rx_coalesce_usecs; @@ -1898,7 +1895,7 @@ static int bcmgenet_rx_poll(struct napi_struct *napi, int budget) { struct bcmgenet_rx_ring *ring = container_of(napi, struct bcmgenet_rx_ring, napi); - struct net_dim_sample dim_sample; + struct dim_sample dim_sample; unsigned int work_done; work_done = bcmgenet_desc_rx(ring, budget); @@ -1909,8 +1906,8 @@ static int bcmgenet_rx_poll(struct napi_struct *napi, int budget) } if (ring->dim.use_dim) { - net_dim_sample(ring->dim.event_ctr, ring->dim.packets, - ring->dim.bytes, &dim_sample); + dim_update_sample(ring->dim.event_ctr, ring->dim.packets, + ring->dim.bytes, &dim_sample); net_dim(&ring->dim.dim, dim_sample); } @@ -1919,16 +1916,16 @@ static int bcmgenet_rx_poll(struct napi_struct *napi, int budget) static void bcmgenet_dim_work(struct work_struct *work) { - struct net_dim *dim = container_of(work, struct net_dim, work); + struct dim *dim = container_of(work, struct dim, work); struct bcmgenet_net_dim *ndim = container_of(dim, struct bcmgenet_net_dim, dim); struct bcmgenet_rx_ring *ring = container_of(ndim, struct bcmgenet_rx_ring, dim); - struct net_dim_cq_moder cur_profile = + struct dim_cq_moder cur_profile = net_dim_get_rx_moderation(dim->mode, dim->profile_ix); bcmgenet_set_rx_coalesce(ring, cur_profile.usec, cur_profile.pkts); - dim->state = NET_DIM_START_MEASURE; + dim->state = DIM_START_MEASURE; } /* Assign skb to RX DMA descriptor. */ @@ -2085,7 +2082,7 @@ static void bcmgenet_init_dim(struct bcmgenet_rx_ring *ring, struct bcmgenet_net_dim *dim = &ring->dim; INIT_WORK(&dim->dim.work, cb); - dim->dim.mode = NET_DIM_CQ_PERIOD_MODE_START_FROM_EQE; + dim->dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_EQE; dim->event_ctr = 0; dim->packets = 0; dim->bytes = 0; @@ -2094,7 +2091,7 @@ static void bcmgenet_init_dim(struct bcmgenet_rx_ring *ring, static void bcmgenet_init_rx_coalesce(struct bcmgenet_rx_ring *ring) { struct bcmgenet_net_dim *dim = &ring->dim; - struct net_dim_cq_moder moder; + struct dim_cq_moder moder; u32 usecs, pkts; usecs = ring->rx_coalesce_usecs; @@ -3086,39 +3083,42 @@ static void bcmgenet_timeout(struct net_device *dev) netif_tx_wake_all_queues(dev); } -#define MAX_MC_COUNT 16 +#define MAX_MDF_FILTER 17 static inline void bcmgenet_set_mdf_addr(struct bcmgenet_priv *priv, unsigned char *addr, - int *i, - int *mc) + int *i) { - u32 reg; - bcmgenet_umac_writel(priv, addr[0] << 8 | addr[1], UMAC_MDF_ADDR + (*i * 4)); bcmgenet_umac_writel(priv, addr[2] << 24 | addr[3] << 16 | addr[4] << 8 | addr[5], UMAC_MDF_ADDR + ((*i + 1) * 4)); - reg = bcmgenet_umac_readl(priv, UMAC_MDF_CTRL); - reg |= (1 << (MAX_MC_COUNT - *mc)); - bcmgenet_umac_writel(priv, reg, UMAC_MDF_CTRL); *i += 2; - (*mc)++; } static void bcmgenet_set_rx_mode(struct net_device *dev) { struct bcmgenet_priv *priv = netdev_priv(dev); struct netdev_hw_addr *ha; - int i, mc; + int i, nfilter; u32 reg; netif_dbg(priv, hw, dev, "%s: %08X\n", __func__, dev->flags); - /* Promiscuous mode */ + /* Number of filters needed */ + nfilter = netdev_uc_count(dev) + netdev_mc_count(dev) + 2; + + /* + * Turn on promicuous mode for three scenarios + * 1. IFF_PROMISC flag is set + * 2. IFF_ALLMULTI flag is set + * 3. The number of filters needed exceeds the number filters + * supported by the hardware. + */ reg = bcmgenet_umac_readl(priv, UMAC_CMD); - if (dev->flags & IFF_PROMISC) { + if ((dev->flags & (IFF_PROMISC | IFF_ALLMULTI)) || + (nfilter > MAX_MDF_FILTER)) { reg |= CMD_PROMISC; bcmgenet_umac_writel(priv, reg, UMAC_CMD); bcmgenet_umac_writel(priv, 0, UMAC_MDF_CTRL); @@ -3128,32 +3128,24 @@ static void bcmgenet_set_rx_mode(struct net_device *dev) bcmgenet_umac_writel(priv, reg, UMAC_CMD); } - /* UniMac doesn't support ALLMULTI */ - if (dev->flags & IFF_ALLMULTI) { - netdev_warn(dev, "ALLMULTI is not supported\n"); - return; - } - /* update MDF filter */ i = 0; - mc = 0; /* Broadcast */ - bcmgenet_set_mdf_addr(priv, dev->broadcast, &i, &mc); + bcmgenet_set_mdf_addr(priv, dev->broadcast, &i); /* my own address.*/ - bcmgenet_set_mdf_addr(priv, dev->dev_addr, &i, &mc); - /* Unicast list*/ - if (netdev_uc_count(dev) > (MAX_MC_COUNT - mc)) - return; + bcmgenet_set_mdf_addr(priv, dev->dev_addr, &i); - if (!netdev_uc_empty(dev)) - netdev_for_each_uc_addr(ha, dev) - bcmgenet_set_mdf_addr(priv, ha->addr, &i, &mc); - /* Multicast */ - if (netdev_mc_empty(dev) || netdev_mc_count(dev) >= (MAX_MC_COUNT - mc)) - return; + /* Unicast */ + netdev_for_each_uc_addr(ha, dev) + bcmgenet_set_mdf_addr(priv, ha->addr, &i); + /* Multicast */ netdev_for_each_mc_addr(ha, dev) - bcmgenet_set_mdf_addr(priv, ha->addr, &i, &mc); + bcmgenet_set_mdf_addr(priv, ha->addr, &i); + + /* Enable filters */ + reg = GENMASK(MAX_MDF_FILTER - 1, MAX_MDF_FILTER - nfilter); + bcmgenet_umac_writel(priv, reg, UMAC_MDF_CTRL); } /* Set the hardware MAC address. */ diff --git a/drivers/net/ethernet/broadcom/genet/bcmgenet.h b/drivers/net/ethernet/broadcom/genet/bcmgenet.h index 14b49612aa86..4a8fc03d82fd 100644 --- a/drivers/net/ethernet/broadcom/genet/bcmgenet.h +++ b/drivers/net/ethernet/broadcom/genet/bcmgenet.h @@ -1,9 +1,6 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ /* * Copyright (c) 2014-2017 Broadcom - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. */ #ifndef __BCMGENET_H__ @@ -16,7 +13,7 @@ #include <linux/mii.h> #include <linux/if_vlan.h> #include <linux/phy.h> -#include <linux/net_dim.h> +#include <linux/dim.h> /* total number of Buffer Descriptors, same for Rx/Tx */ #define TOTAL_DESC 256 @@ -581,7 +578,7 @@ struct bcmgenet_net_dim { u16 event_ctr; unsigned long packets; unsigned long bytes; - struct net_dim dim; + struct dim dim; }; struct bcmgenet_rx_ring { diff --git a/drivers/net/ethernet/broadcom/genet/bcmgenet_wol.c b/drivers/net/ethernet/broadcom/genet/bcmgenet_wol.c index 57582efa362d..ea20d94bd050 100644 --- a/drivers/net/ethernet/broadcom/genet/bcmgenet_wol.c +++ b/drivers/net/ethernet/broadcom/genet/bcmgenet_wol.c @@ -1,11 +1,8 @@ +// SPDX-License-Identifier: GPL-2.0-only /* * Broadcom GENET (Gigabit Ethernet) Wake-on-LAN support * * Copyright (c) 2014-2017 Broadcom - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. */ #define pr_fmt(fmt) "bcmgenet_wol: " fmt diff --git a/drivers/net/ethernet/broadcom/genet/bcmmii.c b/drivers/net/ethernet/broadcom/genet/bcmmii.c index 51880d83131a..970e478a9017 100644 --- a/drivers/net/ethernet/broadcom/genet/bcmmii.c +++ b/drivers/net/ethernet/broadcom/genet/bcmmii.c @@ -1,11 +1,8 @@ +// SPDX-License-Identifier: GPL-2.0-only /* * Broadcom GENET MDIO routines * * Copyright (c) 2014-2017 Broadcom - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License version 2 as - * published by the Free Software Foundation. */ diff --git a/drivers/net/ethernet/broadcom/tg3.c b/drivers/net/ethernet/broadcom/tg3.c index 6d1f9c822548..4c404d2213f9 100644 --- a/drivers/net/ethernet/broadcom/tg3.c +++ b/drivers/net/ethernet/broadcom/tg3.c @@ -6710,7 +6710,7 @@ static int tg3_alloc_rx_data(struct tg3 *tp, struct tg3_rx_prodring_set *tpr, skb_size = SKB_DATA_ALIGN(data_size + TG3_RX_OFFSET(tp)) + SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); if (skb_size <= PAGE_SIZE) { - data = netdev_alloc_frag(skb_size); + data = napi_alloc_frag(skb_size); *frag_size = skb_size; } else { data = kmalloc(skb_size, GFP_ATOMIC); |