diff options
Diffstat (limited to 'drivers/net/ethernet/intel/i40e')
| -rw-r--r-- | drivers/net/ethernet/intel/i40e/i40e.h | 31 | ||||
| -rw-r--r-- | drivers/net/ethernet/intel/i40e/i40e_ethtool.c | 103 | ||||
| -rw-r--r-- | drivers/net/ethernet/intel/i40e/i40e_main.c | 249 | ||||
| -rw-r--r-- | drivers/net/ethernet/intel/i40e/i40e_ptp.c | 36 | ||||
| -rw-r--r-- | drivers/net/ethernet/intel/i40e/i40e_register.h | 13 | ||||
| -rw-r--r-- | drivers/net/ethernet/intel/i40e/i40e_txrx.c | 105 | ||||
| -rw-r--r-- | drivers/net/ethernet/intel/i40e/i40e_type.h | 1 | ||||
| -rw-r--r-- | drivers/net/ethernet/intel/i40e/i40e_virtchnl_pf.c | 12 | ||||
| -rw-r--r-- | drivers/net/ethernet/intel/i40e/i40e_xsk.c | 17 |
9 files changed, 473 insertions, 94 deletions
diff --git a/drivers/net/ethernet/intel/i40e/i40e.h b/drivers/net/ethernet/intel/i40e/i40e.h index 18558a019353..d86b6d349ea9 100644 --- a/drivers/net/ethernet/intel/i40e/i40e.h +++ b/drivers/net/ethernet/intel/i40e/i40e.h @@ -37,6 +37,7 @@ #include <net/tc_act/tc_mirred.h> #include <net/udp_tunnel.h> #include <net/xdp_sock.h> +#include <linux/bitfield.h> #include "i40e_type.h" #include "i40e_prototype.h" #include <linux/net/intel/i40e_client.h> @@ -565,6 +566,7 @@ struct i40e_pf { #define I40E_FLAG_DISABLE_FW_LLDP BIT(24) #define I40E_FLAG_RS_FEC BIT(25) #define I40E_FLAG_BASE_R_FEC BIT(26) +#define I40E_FLAG_VF_VLAN_PRUNING BIT(27) /* TOTAL_PORT_SHUTDOWN * Allows to physically disable the link on the NIC's port. * If enabled, (after link down request from the OS) @@ -1092,6 +1094,21 @@ static inline void i40e_write_fd_input_set(struct i40e_pf *pf, (u32)(val & 0xFFFFFFFFULL)); } +/** + * i40e_get_pf_count - get PCI PF count. + * @hw: pointer to a hw. + * + * Reports the function number of the highest PCI physical + * function plus 1 as it is loaded from the NVM. + * + * Return: PCI PF count. + **/ +static inline u32 i40e_get_pf_count(struct i40e_hw *hw) +{ + return FIELD_GET(I40E_GLGEN_PCIFCNCNT_PCIPFCNT_MASK, + rd32(hw, I40E_GLGEN_PCIFCNCNT)); +} + /* needed by i40e_ethtool.c */ int i40e_up(struct i40e_vsi *vsi); void i40e_down(struct i40e_vsi *vsi); @@ -1275,4 +1292,18 @@ int i40e_add_del_cloud_filter(struct i40e_vsi *vsi, int i40e_add_del_cloud_filter_big_buf(struct i40e_vsi *vsi, struct i40e_cloud_filter *filter, bool add); + +/** + * i40e_is_tc_mqprio_enabled - check if TC MQPRIO is enabled on PF + * @pf: pointer to a pf. + * + * Check and return value of flag I40E_FLAG_TC_MQPRIO. + * + * Return: I40E_FLAG_TC_MQPRIO set state. + **/ +static inline u32 i40e_is_tc_mqprio_enabled(struct i40e_pf *pf) +{ + return pf->flags & I40E_FLAG_TC_MQPRIO; +} + #endif /* _I40E_H_ */ diff --git a/drivers/net/ethernet/intel/i40e/i40e_ethtool.c b/drivers/net/ethernet/intel/i40e/i40e_ethtool.c index 19704f5c8291..156e92c43780 100644 --- a/drivers/net/ethernet/intel/i40e/i40e_ethtool.c +++ b/drivers/net/ethernet/intel/i40e/i40e_ethtool.c @@ -236,8 +236,6 @@ static void __i40e_add_stat_strings(u8 **p, const struct i40e_stats stats[], I40E_STAT(struct i40e_cp_veb_tc_stats, _name, _stat) #define I40E_PFC_STAT(_name, _stat) \ I40E_STAT(struct i40e_pfc_stats, _name, _stat) -#define I40E_QUEUE_STAT(_name, _stat) \ - I40E_STAT(struct i40e_ring, _name, _stat) static const struct i40e_stats i40e_gstrings_net_stats[] = { I40E_NETDEV_STAT(rx_packets), @@ -457,6 +455,8 @@ static const struct i40e_priv_flags i40e_gstrings_priv_flags[] = { I40E_PRIV_FLAG("disable-fw-lldp", I40E_FLAG_DISABLE_FW_LLDP, 0), I40E_PRIV_FLAG("rs-fec", I40E_FLAG_RS_FEC, 0), I40E_PRIV_FLAG("base-r-fec", I40E_FLAG_BASE_R_FEC, 0), + I40E_PRIV_FLAG("vf-vlan-pruning", + I40E_FLAG_VF_VLAN_PRUNING, 0), }; #define I40E_PRIV_FLAGS_STR_LEN ARRAY_SIZE(i40e_gstrings_priv_flags) @@ -1141,6 +1141,71 @@ static int i40e_get_link_ksettings(struct net_device *netdev, return 0; } +#define I40E_LBIT_SIZE 8 +/** + * i40e_speed_to_link_speed - Translate decimal speed to i40e_aq_link_speed + * @speed: speed in decimal + * @ks: ethtool ksettings + * + * Return i40e_aq_link_speed based on speed + **/ +static enum i40e_aq_link_speed +i40e_speed_to_link_speed(__u32 speed, const struct ethtool_link_ksettings *ks) +{ + enum i40e_aq_link_speed link_speed = I40E_LINK_SPEED_UNKNOWN; + bool speed_changed = false; + int i, j; + + static const struct { + __u32 speed; + enum i40e_aq_link_speed link_speed; + __u8 bit[I40E_LBIT_SIZE]; + } i40e_speed_lut[] = { +#define I40E_LBIT(mode) ETHTOOL_LINK_MODE_ ## mode ##_Full_BIT + {SPEED_100, I40E_LINK_SPEED_100MB, {I40E_LBIT(100baseT)} }, + {SPEED_1000, I40E_LINK_SPEED_1GB, + {I40E_LBIT(1000baseT), I40E_LBIT(1000baseX), + I40E_LBIT(1000baseKX)} }, + {SPEED_10000, I40E_LINK_SPEED_10GB, + {I40E_LBIT(10000baseT), I40E_LBIT(10000baseKR), + I40E_LBIT(10000baseLR), I40E_LBIT(10000baseCR), + I40E_LBIT(10000baseSR), I40E_LBIT(10000baseKX4)} }, + + {SPEED_25000, I40E_LINK_SPEED_25GB, + {I40E_LBIT(25000baseCR), I40E_LBIT(25000baseKR), + I40E_LBIT(25000baseSR)} }, + {SPEED_40000, I40E_LINK_SPEED_40GB, + {I40E_LBIT(40000baseKR4), I40E_LBIT(40000baseCR4), + I40E_LBIT(40000baseSR4), I40E_LBIT(40000baseLR4)} }, + {SPEED_20000, I40E_LINK_SPEED_20GB, + {I40E_LBIT(20000baseKR2)} }, + {SPEED_2500, I40E_LINK_SPEED_2_5GB, {I40E_LBIT(2500baseT)} }, + {SPEED_5000, I40E_LINK_SPEED_5GB, {I40E_LBIT(2500baseT)} } +#undef I40E_LBIT +}; + + for (i = 0; i < ARRAY_SIZE(i40e_speed_lut); i++) { + if (i40e_speed_lut[i].speed == speed) { + for (j = 0; j < I40E_LBIT_SIZE; j++) { + if (test_bit(i40e_speed_lut[i].bit[j], + ks->link_modes.supported)) { + speed_changed = true; + break; + } + if (!i40e_speed_lut[i].bit[j]) + break; + } + if (speed_changed) { + link_speed = i40e_speed_lut[i].link_speed; + break; + } + } + } + return link_speed; +} + +#undef I40E_LBIT_SIZE + /** * i40e_set_link_ksettings - Set Speed and Duplex * @netdev: network interface device structure @@ -1157,12 +1222,14 @@ static int i40e_set_link_ksettings(struct net_device *netdev, struct ethtool_link_ksettings copy_ks; struct i40e_aq_set_phy_config config; struct i40e_pf *pf = np->vsi->back; + enum i40e_aq_link_speed link_speed; struct i40e_vsi *vsi = np->vsi; struct i40e_hw *hw = &pf->hw; bool autoneg_changed = false; i40e_status status = 0; int timeout = 50; int err = 0; + __u32 speed; u8 autoneg; /* Changing port settings is not supported if this isn't the @@ -1195,6 +1262,7 @@ static int i40e_set_link_ksettings(struct net_device *netdev, /* save autoneg out of ksettings */ autoneg = copy_ks.base.autoneg; + speed = copy_ks.base.speed; /* get our own copy of the bits to check against */ memset(&safe_ks, 0, sizeof(struct ethtool_link_ksettings)); @@ -1213,6 +1281,7 @@ static int i40e_set_link_ksettings(struct net_device *netdev, /* set autoneg back to what it currently is */ copy_ks.base.autoneg = safe_ks.base.autoneg; + copy_ks.base.speed = safe_ks.base.speed; /* If copy_ks.base and safe_ks.base are not the same now, then they are * trying to set something that we do not support. @@ -1329,6 +1398,27 @@ static int i40e_set_link_ksettings(struct net_device *netdev, 40000baseLR4_Full)) config.link_speed |= I40E_LINK_SPEED_40GB; + /* Autonegotiation must be disabled to change speed */ + if ((speed != SPEED_UNKNOWN && safe_ks.base.speed != speed) && + (autoneg == AUTONEG_DISABLE || + (safe_ks.base.autoneg == AUTONEG_DISABLE && !autoneg_changed))) { + link_speed = i40e_speed_to_link_speed(speed, ks); + if (link_speed == I40E_LINK_SPEED_UNKNOWN) { + netdev_info(netdev, "Given speed is not supported\n"); + err = -EOPNOTSUPP; + goto done; + } else { + config.link_speed = link_speed; + } + } else { + if (safe_ks.base.speed != speed) { + netdev_info(netdev, + "Unable to set speed, disable autoneg\n"); + err = -EOPNOTSUPP; + goto done; + } + } + /* If speed didn't get set, set it to what it currently is. * This is needed because if advertise is 0 (as it is when autoneg * is disabled) then speed won't get set. @@ -4931,7 +5021,7 @@ static int i40e_set_channels(struct net_device *dev, /* We do not support setting channels via ethtool when TCs are * configured through mqprio */ - if (pf->flags & I40E_FLAG_TC_MQPRIO) + if (i40e_is_tc_mqprio_enabled(pf)) return -EINVAL; /* verify they are not requesting separate vectors */ @@ -5294,6 +5384,13 @@ flags_complete: return -EOPNOTSUPP; } + if ((changed_flags & I40E_FLAG_VF_VLAN_PRUNING) && + pf->num_alloc_vfs) { + dev_warn(&pf->pdev->dev, + "Changing vf-vlan-pruning flag while VF(s) are active is not supported\n"); + return -EOPNOTSUPP; + } + if ((changed_flags & new_flags & I40E_FLAG_LINK_DOWN_ON_CLOSE_ENABLED) && (new_flags & I40E_FLAG_MFP_ENABLED)) diff --git a/drivers/net/ethernet/intel/i40e/i40e_main.c b/drivers/net/ethernet/intel/i40e/i40e_main.c index 72576bb3e94d..b36bf9c3e1e4 100644 --- a/drivers/net/ethernet/intel/i40e/i40e_main.c +++ b/drivers/net/ethernet/intel/i40e/i40e_main.c @@ -551,6 +551,47 @@ void i40e_pf_reset_stats(struct i40e_pf *pf) } /** + * i40e_compute_pci_to_hw_id - compute index form PCI function. + * @vsi: ptr to the VSI to read from. + * @hw: ptr to the hardware info. + **/ +static u32 i40e_compute_pci_to_hw_id(struct i40e_vsi *vsi, struct i40e_hw *hw) +{ + int pf_count = i40e_get_pf_count(hw); + + if (vsi->type == I40E_VSI_SRIOV) + return (hw->port * BIT(7)) / pf_count + vsi->vf_id; + + return hw->port + BIT(7); +} + +/** + * i40e_stat_update64 - read and update a 64 bit stat from the chip. + * @hw: ptr to the hardware info. + * @hireg: the high 32 bit reg to read. + * @loreg: the low 32 bit reg to read. + * @offset_loaded: has the initial offset been loaded yet. + * @offset: ptr to current offset value. + * @stat: ptr to the stat. + * + * Since the device stats are not reset at PFReset, they will not + * be zeroed when the driver starts. We'll save the first values read + * and use them as offsets to be subtracted from the raw values in order + * to report stats that count from zero. + **/ +static void i40e_stat_update64(struct i40e_hw *hw, u32 hireg, u32 loreg, + bool offset_loaded, u64 *offset, u64 *stat) +{ + u64 new_data; + + new_data = rd64(hw, loreg); + + if (!offset_loaded || new_data < *offset) + *offset = new_data; + *stat = new_data - *offset; +} + +/** * i40e_stat_update48 - read and update a 48 bit stat from the chip * @hw: ptr to the hardware info * @hireg: the high 32 bit reg to read @@ -622,6 +663,34 @@ static void i40e_stat_update_and_clear32(struct i40e_hw *hw, u32 reg, u64 *stat) } /** + * i40e_stats_update_rx_discards - update rx_discards. + * @vsi: ptr to the VSI to be updated. + * @hw: ptr to the hardware info. + * @stat_idx: VSI's stat_counter_idx. + * @offset_loaded: ptr to the VSI's stat_offsets_loaded. + * @stat_offset: ptr to stat_offset to store first read of specific register. + * @stat: ptr to VSI's stat to be updated. + **/ +static void +i40e_stats_update_rx_discards(struct i40e_vsi *vsi, struct i40e_hw *hw, + int stat_idx, bool offset_loaded, + struct i40e_eth_stats *stat_offset, + struct i40e_eth_stats *stat) +{ + u64 rx_rdpc, rx_rxerr; + + i40e_stat_update32(hw, I40E_GLV_RDPC(stat_idx), offset_loaded, + &stat_offset->rx_discards, &rx_rdpc); + i40e_stat_update64(hw, + I40E_GL_RXERR1H(i40e_compute_pci_to_hw_id(vsi, hw)), + I40E_GL_RXERR1L(i40e_compute_pci_to_hw_id(vsi, hw)), + offset_loaded, &stat_offset->rx_discards_other, + &rx_rxerr); + + stat->rx_discards = rx_rdpc + rx_rxerr; +} + +/** * i40e_update_eth_stats - Update VSI-specific ethernet statistics counters. * @vsi: the VSI to be updated **/ @@ -680,6 +749,10 @@ void i40e_update_eth_stats(struct i40e_vsi *vsi) I40E_GLV_BPTCL(stat_idx), vsi->stat_offsets_loaded, &oes->tx_broadcast, &es->tx_broadcast); + + i40e_stats_update_rx_discards(vsi, hw, stat_idx, + vsi->stat_offsets_loaded, oes, es); + vsi->stat_offsets_loaded = true; } @@ -1369,6 +1442,114 @@ static int i40e_correct_mac_vlan_filters(struct i40e_vsi *vsi, } /** + * i40e_get_vf_new_vlan - Get new vlan id on a vf + * @vsi: the vsi to configure + * @new_mac: new mac filter to be added + * @f: existing mac filter, replaced with new_mac->f if new_mac is not NULL + * @vlan_filters: the number of active VLAN filters + * @trusted: flag if the VF is trusted + * + * Get new VLAN id based on current VLAN filters, trust, PVID + * and vf-vlan-prune-disable flag. + * + * Returns the value of the new vlan filter or + * the old value if no new filter is needed. + */ +static s16 i40e_get_vf_new_vlan(struct i40e_vsi *vsi, + struct i40e_new_mac_filter *new_mac, + struct i40e_mac_filter *f, + int vlan_filters, + bool trusted) +{ + s16 pvid = le16_to_cpu(vsi->info.pvid); + struct i40e_pf *pf = vsi->back; + bool is_any; + + if (new_mac) + f = new_mac->f; + + if (pvid && f->vlan != pvid) + return pvid; + + is_any = (trusted || + !(pf->flags & I40E_FLAG_VF_VLAN_PRUNING)); + + if ((vlan_filters && f->vlan == I40E_VLAN_ANY) || + (!is_any && !vlan_filters && f->vlan == I40E_VLAN_ANY) || + (is_any && !vlan_filters && f->vlan == 0)) { + if (is_any) + return I40E_VLAN_ANY; + else + return 0; + } + + return f->vlan; +} + +/** + * i40e_correct_vf_mac_vlan_filters - Correct non-VLAN VF filters if necessary + * @vsi: the vsi to configure + * @tmp_add_list: list of filters ready to be added + * @tmp_del_list: list of filters ready to be deleted + * @vlan_filters: the number of active VLAN filters + * @trusted: flag if the VF is trusted + * + * Correct VF VLAN filters based on current VLAN filters, trust, PVID + * and vf-vlan-prune-disable flag. + * + * In case of memory allocation failure return -ENOMEM. Otherwise, return 0. + * + * This function is only expected to be called from within + * i40e_sync_vsi_filters. + * + * NOTE: This function expects to be called while under the + * mac_filter_hash_lock + */ +static int i40e_correct_vf_mac_vlan_filters(struct i40e_vsi *vsi, + struct hlist_head *tmp_add_list, + struct hlist_head *tmp_del_list, + int vlan_filters, + bool trusted) +{ + struct i40e_mac_filter *f, *add_head; + struct i40e_new_mac_filter *new_mac; + struct hlist_node *h; + int bkt, new_vlan; + + hlist_for_each_entry(new_mac, tmp_add_list, hlist) { + new_mac->f->vlan = i40e_get_vf_new_vlan(vsi, new_mac, NULL, + vlan_filters, trusted); + } + + hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) { + new_vlan = i40e_get_vf_new_vlan(vsi, NULL, f, vlan_filters, + trusted); + if (new_vlan != f->vlan) { + add_head = i40e_add_filter(vsi, f->macaddr, new_vlan); + if (!add_head) + return -ENOMEM; + /* Create a temporary i40e_new_mac_filter */ + new_mac = kzalloc(sizeof(*new_mac), GFP_ATOMIC); + if (!new_mac) + return -ENOMEM; + new_mac->f = add_head; + new_mac->state = add_head->state; + + /* Add the new filter to the tmp list */ + hlist_add_head(&new_mac->hlist, tmp_add_list); + + /* Put the original filter into the delete list */ + f->state = I40E_FILTER_REMOVE; + hash_del(&f->hlist); + hlist_add_head(&f->hlist, tmp_del_list); + } + } + + vsi->has_vlan_filter = !!vlan_filters; + return 0; +} + +/** * i40e_rm_default_mac_filter - Remove the default MAC filter set by NVM * @vsi: the PF Main VSI - inappropriate for any other VSI * @macaddr: the MAC address @@ -1852,11 +2033,15 @@ static void i40e_vsi_setup_queue_map(struct i40e_vsi *vsi, * non-zero req_queue_pairs says that user requested a new * queue count via ethtool's set_channels, so use this * value for queues distribution across traffic classes + * We need at least one queue pair for the interface + * to be usable as we see in else statement. */ if (vsi->req_queue_pairs > 0) vsi->num_queue_pairs = vsi->req_queue_pairs; else if (pf->flags & I40E_FLAG_MSIX_ENABLED) vsi->num_queue_pairs = pf->num_lan_msix; + else + vsi->num_queue_pairs = 1; } /* Number of queues per enabled TC */ @@ -2423,10 +2608,14 @@ int i40e_sync_vsi_filters(struct i40e_vsi *vsi) vlan_filters++; } - retval = i40e_correct_mac_vlan_filters(vsi, - &tmp_add_list, - &tmp_del_list, - vlan_filters); + if (vsi->type != I40E_VSI_SRIOV) + retval = i40e_correct_mac_vlan_filters + (vsi, &tmp_add_list, &tmp_del_list, + vlan_filters); + else + retval = i40e_correct_vf_mac_vlan_filters + (vsi, &tmp_add_list, &tmp_del_list, + vlan_filters, pf->vf[vsi->vf_id].trusted); hlist_for_each_entry(new, &tmp_add_list, hlist) netdev_hw_addr_refcnt(new->f, vsi->netdev, 1); @@ -2855,8 +3044,21 @@ int i40e_add_vlan_all_mac(struct i40e_vsi *vsi, s16 vid) int bkt; hash_for_each_safe(vsi->mac_filter_hash, bkt, h, f, hlist) { - if (f->state == I40E_FILTER_REMOVE) + /* If we're asked to add a filter that has been marked for + * removal, it is safe to simply restore it to active state. + * __i40e_del_filter will have simply deleted any filters which + * were previously marked NEW or FAILED, so if it is currently + * marked REMOVE it must have previously been ACTIVE. Since we + * haven't yet run the sync filters task, just restore this + * filter to the ACTIVE state so that the sync task leaves it + * in place. + */ + if (f->state == I40E_FILTER_REMOVE && f->vlan == vid) { + f->state = I40E_FILTER_ACTIVE; + continue; + } else if (f->state == I40E_FILTER_REMOVE) { continue; + } add_f = i40e_add_filter(vsi, f->macaddr, vid); if (!add_f) { dev_info(&vsi->back->pdev->dev, @@ -4037,7 +4239,6 @@ static void i40e_free_misc_vector(struct i40e_pf *pf) i40e_flush(&pf->hw); if (pf->flags & I40E_FLAG_MSIX_ENABLED && pf->msix_entries) { - synchronize_irq(pf->msix_entries[0].vector); free_irq(pf->msix_entries[0].vector, pf); clear_bit(__I40E_MISC_IRQ_REQUESTED, pf->state); } @@ -4776,7 +4977,6 @@ static void i40e_vsi_free_irq(struct i40e_vsi *vsi) irq_set_affinity_notifier(irq_num, NULL); /* remove our suggested affinity mask for this IRQ */ irq_update_affinity_hint(irq_num, NULL); - synchronize_irq(irq_num); free_irq(irq_num, vsi->q_vectors[i]); /* Tear down the interrupt queue link list @@ -5260,7 +5460,7 @@ static u8 i40e_pf_get_num_tc(struct i40e_pf *pf) u8 num_tc = 0; struct i40e_dcbx_config *dcbcfg = &hw->local_dcbx_config; - if (pf->flags & I40E_FLAG_TC_MQPRIO) + if (i40e_is_tc_mqprio_enabled(pf)) return pf->vsi[pf->lan_vsi]->mqprio_qopt.qopt.num_tc; /* If neither MQPRIO nor DCB is enabled, then always use single TC */ @@ -5292,7 +5492,7 @@ static u8 i40e_pf_get_num_tc(struct i40e_pf *pf) **/ static u8 i40e_pf_get_tc_map(struct i40e_pf *pf) { - if (pf->flags & I40E_FLAG_TC_MQPRIO) + if (i40e_is_tc_mqprio_enabled(pf)) return i40e_mqprio_get_enabled_tc(pf); /* If neither MQPRIO nor DCB is enabled for this PF then just return @@ -5389,7 +5589,7 @@ static int i40e_vsi_configure_bw_alloc(struct i40e_vsi *vsi, u8 enabled_tc, int i; /* There is no need to reset BW when mqprio mode is on. */ - if (pf->flags & I40E_FLAG_TC_MQPRIO) + if (i40e_is_tc_mqprio_enabled(pf)) return 0; if (!vsi->mqprio_qopt.qopt.hw && !(pf->flags & I40E_FLAG_DCB_ENABLED)) { ret = i40e_set_bw_limit(vsi, vsi->seid, 0); @@ -5461,7 +5661,7 @@ static void i40e_vsi_config_netdev_tc(struct i40e_vsi *vsi, u8 enabled_tc) vsi->tc_config.tc_info[i].qoffset); } - if (pf->flags & I40E_FLAG_TC_MQPRIO) + if (i40e_is_tc_mqprio_enabled(pf)) return; /* Assign UP2TC map for the VSI */ @@ -5622,7 +5822,7 @@ static int i40e_vsi_config_tc(struct i40e_vsi *vsi, u8 enabled_tc) ctxt.vf_num = 0; ctxt.uplink_seid = vsi->uplink_seid; ctxt.info = vsi->info; - if (vsi->back->flags & I40E_FLAG_TC_MQPRIO) { + if (i40e_is_tc_mqprio_enabled(pf)) { ret = i40e_vsi_setup_queue_map_mqprio(vsi, &ctxt, enabled_tc); if (ret) goto out; @@ -6346,7 +6546,7 @@ int i40e_create_queue_channel(struct i40e_vsi *vsi, pf->flags |= I40E_FLAG_VEB_MODE_ENABLED; if (vsi->type == I40E_VSI_MAIN) { - if (pf->flags & I40E_FLAG_TC_MQPRIO) + if (i40e_is_tc_mqprio_enabled(pf)) i40e_do_reset(pf, I40E_PF_RESET_FLAG, true); else i40e_do_reset_safe(pf, I40E_PF_RESET_FLAG); @@ -7740,7 +7940,7 @@ static void *i40e_fwd_add(struct net_device *netdev, struct net_device *vdev) netdev_info(netdev, "Macvlans are not supported when DCB is enabled\n"); return ERR_PTR(-EINVAL); } - if ((pf->flags & I40E_FLAG_TC_MQPRIO)) { + if (i40e_is_tc_mqprio_enabled(pf)) { netdev_info(netdev, "Macvlans are not supported when HW TC offload is on\n"); return ERR_PTR(-EINVAL); } @@ -7993,7 +8193,7 @@ config_tc: /* Quiesce VSI queues */ i40e_quiesce_vsi(vsi); - if (!hw && !(pf->flags & I40E_FLAG_TC_MQPRIO)) + if (!hw && !i40e_is_tc_mqprio_enabled(pf)) i40e_remove_queue_channels(vsi); /* Configure VSI for enabled TCs */ @@ -8017,7 +8217,7 @@ config_tc: "Setup channel (id:%u) utilizing num_queues %d\n", vsi->seid, vsi->tc_config.tc_info[0].qcount); - if (pf->flags & I40E_FLAG_TC_MQPRIO) { + if (i40e_is_tc_mqprio_enabled(pf)) { if (vsi->mqprio_qopt.max_rate[0]) { u64 max_tx_rate = vsi->mqprio_qopt.max_rate[0]; @@ -10577,7 +10777,7 @@ static int i40e_reset(struct i40e_pf *pf) **/ static void i40e_rebuild(struct i40e_pf *pf, bool reinit, bool lock_acquired) { - int old_recovery_mode_bit = test_bit(__I40E_RECOVERY_MODE, pf->state); + const bool is_recovery_mode_reported = i40e_check_recovery_mode(pf); struct i40e_vsi *vsi = pf->vsi[pf->lan_vsi]; struct i40e_hw *hw = &pf->hw; i40e_status ret; @@ -10585,13 +10785,11 @@ static void i40e_rebuild(struct i40e_pf *pf, bool reinit, bool lock_acquired) int v; if (test_bit(__I40E_EMP_RESET_INTR_RECEIVED, pf->state) && - i40e_check_recovery_mode(pf)) { + is_recovery_mode_reported) i40e_set_ethtool_ops(pf->vsi[pf->lan_vsi]->netdev); - } if (test_bit(__I40E_DOWN, pf->state) && - !test_bit(__I40E_RECOVERY_MODE, pf->state) && - !old_recovery_mode_bit) + !test_bit(__I40E_RECOVERY_MODE, pf->state)) goto clear_recovery; dev_dbg(&pf->pdev->dev, "Rebuilding internal switch\n"); @@ -10618,13 +10816,12 @@ static void i40e_rebuild(struct i40e_pf *pf, bool reinit, bool lock_acquired) * accordingly with regard to resources initialization * and deinitialization */ - if (test_bit(__I40E_RECOVERY_MODE, pf->state) || - old_recovery_mode_bit) { + if (test_bit(__I40E_RECOVERY_MODE, pf->state)) { if (i40e_get_capabilities(pf, i40e_aqc_opc_list_func_capabilities)) goto end_unlock; - if (test_bit(__I40E_RECOVERY_MODE, pf->state)) { + if (is_recovery_mode_reported) { /* we're staying in recovery mode so we'll reinitialize * misc vector here */ @@ -10674,7 +10871,7 @@ static void i40e_rebuild(struct i40e_pf *pf, bool reinit, bool lock_acquired) * unless I40E_FLAG_TC_MQPRIO was enabled or DCB * is not supported with new link speed */ - if (pf->flags & I40E_FLAG_TC_MQPRIO) { + if (i40e_is_tc_mqprio_enabled(pf)) { i40e_aq_set_dcb_parameters(hw, false, NULL); } else { if (I40E_IS_X710TL_DEVICE(hw->device_id) && @@ -13033,7 +13230,7 @@ static netdev_features_t i40e_features_check(struct sk_buff *skb, } /* No need to validate L4LEN as TCP is the only protocol with a - * a flexible value and we support all possible values supported + * flexible value and we support all possible values supported * by TCP, which is at most 15 dwords */ diff --git a/drivers/net/ethernet/intel/i40e/i40e_ptp.c b/drivers/net/ethernet/intel/i40e/i40e_ptp.c index 61e5789d78db..2d3533f38d7b 100644 --- a/drivers/net/ethernet/intel/i40e/i40e_ptp.c +++ b/drivers/net/ethernet/intel/i40e/i40e_ptp.c @@ -27,7 +27,6 @@ #define I40E_PRTTSYN_CTL1_TSYNTYPE_V2 (2 << \ I40E_PRTTSYN_CTL1_TSYNTYPE_SHIFT) #define I40E_SUBDEV_ID_25G_PTP_PIN 0xB -#define to_dev(obj) container_of(obj, struct device, kobj) enum i40e_ptp_pin { SDP3_2 = 0, @@ -335,44 +334,37 @@ static void i40e_ptp_convert_to_hwtstamp(struct skb_shared_hwtstamps *hwtstamps, } /** - * i40e_ptp_adjfreq - Adjust the PHC frequency + * i40e_ptp_adjfine - Adjust the PHC frequency * @ptp: The PTP clock structure - * @ppb: Parts per billion adjustment from the base + * @scaled_ppm: Scaled parts per million adjustment from base * - * Adjust the frequency of the PHC by the indicated parts per billion from the - * base frequency. + * Adjust the frequency of the PHC by the indicated delta from the base + * frequency. + * + * Scaled parts per million is ppm with a 16 bit binary fractional field. **/ -static int i40e_ptp_adjfreq(struct ptp_clock_info *ptp, s32 ppb) +static int i40e_ptp_adjfine(struct ptp_clock_info *ptp, long scaled_ppm) { struct i40e_pf *pf = container_of(ptp, struct i40e_pf, ptp_caps); struct i40e_hw *hw = &pf->hw; u64 adj, freq, diff; int neg_adj = 0; - if (ppb < 0) { + if (scaled_ppm < 0) { neg_adj = 1; - ppb = -ppb; + scaled_ppm = -scaled_ppm; } - freq = I40E_PTP_40GB_INCVAL; - freq *= ppb; - diff = div_u64(freq, 1000000000ULL); + smp_mb(); /* Force any pending update before accessing. */ + freq = I40E_PTP_40GB_INCVAL * READ_ONCE(pf->ptp_adj_mult); + diff = mul_u64_u64_div_u64(freq, (u64)scaled_ppm, + 1000000ULL << 16); if (neg_adj) adj = I40E_PTP_40GB_INCVAL - diff; else adj = I40E_PTP_40GB_INCVAL + diff; - /* At some link speeds, the base incval is so large that directly - * multiplying by ppb would result in arithmetic overflow even when - * using a u64. Avoid this by instead calculating the new incval - * always in terms of the 40GbE clock rate and then multiplying by the - * link speed factor afterwards. This does result in slightly lower - * precision at lower link speeds, but it is fairly minor. - */ - smp_mb(); /* Force any pending update before accessing. */ - adj *= READ_ONCE(pf->ptp_adj_mult); - wr32(hw, I40E_PRTTSYN_INC_L, adj & 0xFFFFFFFF); wr32(hw, I40E_PRTTSYN_INC_H, adj >> 32); @@ -1402,7 +1394,7 @@ static long i40e_ptp_create_clock(struct i40e_pf *pf) sizeof(pf->ptp_caps.name) - 1); pf->ptp_caps.owner = THIS_MODULE; pf->ptp_caps.max_adj = 999999999; - pf->ptp_caps.adjfreq = i40e_ptp_adjfreq; + pf->ptp_caps.adjfine = i40e_ptp_adjfine; pf->ptp_caps.adjtime = i40e_ptp_adjtime; pf->ptp_caps.gettimex64 = i40e_ptp_gettimex; pf->ptp_caps.settime64 = i40e_ptp_settime; diff --git a/drivers/net/ethernet/intel/i40e/i40e_register.h b/drivers/net/ethernet/intel/i40e/i40e_register.h index 1908eed4fa5e..7339003aa17c 100644 --- a/drivers/net/ethernet/intel/i40e/i40e_register.h +++ b/drivers/net/ethernet/intel/i40e/i40e_register.h @@ -211,6 +211,11 @@ #define I40E_GLGEN_MSRWD_MDIWRDATA_SHIFT 0 #define I40E_GLGEN_MSRWD_MDIRDDATA_SHIFT 16 #define I40E_GLGEN_MSRWD_MDIRDDATA_MASK I40E_MASK(0xFFFF, I40E_GLGEN_MSRWD_MDIRDDATA_SHIFT) +#define I40E_GLGEN_PCIFCNCNT 0x001C0AB4 /* Reset: PCIR */ +#define I40E_GLGEN_PCIFCNCNT_PCIPFCNT_SHIFT 0 +#define I40E_GLGEN_PCIFCNCNT_PCIPFCNT_MASK I40E_MASK(0x1F, I40E_GLGEN_PCIFCNCNT_PCIPFCNT_SHIFT) +#define I40E_GLGEN_PCIFCNCNT_PCIVFCNT_SHIFT 16 +#define I40E_GLGEN_PCIFCNCNT_PCIVFCNT_MASK I40E_MASK(0xFF, I40E_GLGEN_PCIFCNCNT_PCIVFCNT_SHIFT) #define I40E_GLGEN_RSTAT 0x000B8188 /* Reset: POR */ #define I40E_GLGEN_RSTAT_DEVSTATE_SHIFT 0 #define I40E_GLGEN_RSTAT_DEVSTATE_MASK I40E_MASK(0x3, I40E_GLGEN_RSTAT_DEVSTATE_SHIFT) @@ -643,6 +648,14 @@ #define I40E_VFQF_HKEY1_MAX_INDEX 12 #define I40E_VFQF_HLUT1(_i, _VF) (0x00220000 + ((_i) * 1024 + (_VF) * 4)) /* _i=0...15, _VF=0...127 */ /* Reset: CORER */ #define I40E_VFQF_HLUT1_MAX_INDEX 15 +#define I40E_GL_RXERR1H(_i) (0x00318004 + ((_i) * 8)) /* _i=0...143 */ /* Reset: CORER */ +#define I40E_GL_RXERR1H_MAX_INDEX 143 +#define I40E_GL_RXERR1H_RXERR1H_SHIFT 0 +#define I40E_GL_RXERR1H_RXERR1H_MASK I40E_MASK(0xFFFFFFFF, I40E_GL_RXERR1H_RXERR1H_SHIFT) +#define I40E_GL_RXERR1L(_i) (0x00318000 + ((_i) * 8)) /* _i=0...143 */ /* Reset: CORER */ +#define I40E_GL_RXERR1L_MAX_INDEX 143 +#define I40E_GL_RXERR1L_RXERR1L_SHIFT 0 +#define I40E_GL_RXERR1L_RXERR1L_MASK I40E_MASK(0xFFFFFFFF, I40E_GL_RXERR1L_RXERR1L_SHIFT) #define I40E_GLPRT_BPRCH(_i) (0x003005E4 + ((_i) * 8)) /* _i=0...3 */ /* Reset: CORER */ #define I40E_GLPRT_BPRCL(_i) (0x003005E0 + ((_i) * 8)) /* _i=0...3 */ /* Reset: CORER */ #define I40E_GLPRT_BPTCH(_i) (0x00300A04 + ((_i) * 8)) /* _i=0...3 */ /* Reset: CORER */ diff --git a/drivers/net/ethernet/intel/i40e/i40e_txrx.c b/drivers/net/ethernet/intel/i40e/i40e_txrx.c index 7bc1174edf6b..f6ba97a0166e 100644 --- a/drivers/net/ethernet/intel/i40e/i40e_txrx.c +++ b/drivers/net/ethernet/intel/i40e/i40e_txrx.c @@ -372,7 +372,6 @@ static void i40e_change_filter_num(bool ipv4, bool add, u16 *ipv4_filter_num, } } -#define IP_HEADER_OFFSET 14 #define I40E_UDPIP_DUMMY_PACKET_LEN 42 #define I40E_UDPIP6_DUMMY_PACKET_LEN 62 /** @@ -1483,10 +1482,8 @@ void i40e_clean_rx_ring(struct i40e_ring *rx_ring) if (!rx_ring->rx_bi) return; - if (rx_ring->skb) { - dev_kfree_skb(rx_ring->skb); - rx_ring->skb = NULL; - } + dev_kfree_skb(rx_ring->skb); + rx_ring->skb = NULL; if (rx_ring->xsk_pool) { i40e_xsk_clean_rx_ring(rx_ring); @@ -2291,16 +2288,14 @@ int i40e_xmit_xdp_tx_ring(struct xdp_buff *xdp, struct i40e_ring *xdp_ring) * i40e_run_xdp - run an XDP program * @rx_ring: Rx ring being processed * @xdp: XDP buffer containing the frame + * @xdp_prog: XDP program to run **/ -static int i40e_run_xdp(struct i40e_ring *rx_ring, struct xdp_buff *xdp) +static int i40e_run_xdp(struct i40e_ring *rx_ring, struct xdp_buff *xdp, struct bpf_prog *xdp_prog) { int err, result = I40E_XDP_PASS; struct i40e_ring *xdp_ring; - struct bpf_prog *xdp_prog; u32 act; - xdp_prog = READ_ONCE(rx_ring->xdp_prog); - if (!xdp_prog) goto xdp_out; @@ -2445,6 +2440,7 @@ static int i40e_clean_rx_irq(struct i40e_ring *rx_ring, int budget) unsigned int offset = rx_ring->rx_offset; struct sk_buff *skb = rx_ring->skb; unsigned int xdp_xmit = 0; + struct bpf_prog *xdp_prog; bool failure = false; struct xdp_buff xdp; int xdp_res = 0; @@ -2454,6 +2450,8 @@ static int i40e_clean_rx_irq(struct i40e_ring *rx_ring, int budget) #endif xdp_init_buff(&xdp, frame_sz, &rx_ring->xdp_rxq); + xdp_prog = READ_ONCE(rx_ring->xdp_prog); + while (likely(total_rx_packets < (unsigned int)budget)) { struct i40e_rx_buffer *rx_buffer; union i40e_rx_desc *rx_desc; @@ -2509,11 +2507,12 @@ static int i40e_clean_rx_irq(struct i40e_ring *rx_ring, int budget) hard_start = page_address(rx_buffer->page) + rx_buffer->page_offset - offset; xdp_prepare_buff(&xdp, hard_start, offset, size, true); + xdp_buff_clear_frags_flag(&xdp); #if (PAGE_SIZE > 4096) /* At larger PAGE_SIZE, frame_sz depend on len size */ xdp.frame_sz = i40e_rx_frame_truesize(rx_ring, size); #endif - xdp_res = i40e_run_xdp(rx_ring, &xdp); + xdp_res = i40e_run_xdp(rx_ring, &xdp, xdp_prog); } if (xdp_res) { @@ -3713,35 +3712,55 @@ u16 i40e_lan_select_queue(struct net_device *netdev, static int i40e_xmit_xdp_ring(struct xdp_frame *xdpf, struct i40e_ring *xdp_ring) { - u16 i = xdp_ring->next_to_use; - struct i40e_tx_buffer *tx_bi; - struct i40e_tx_desc *tx_desc; + struct skb_shared_info *sinfo = xdp_get_shared_info_from_frame(xdpf); + u8 nr_frags = unlikely(xdp_frame_has_frags(xdpf)) ? sinfo->nr_frags : 0; + u16 i = 0, index = xdp_ring->next_to_use; + struct i40e_tx_buffer *tx_head = &xdp_ring->tx_bi[index]; + struct i40e_tx_buffer *tx_bi = tx_head; + struct i40e_tx_desc *tx_desc = I40E_TX_DESC(xdp_ring, index); void *data = xdpf->data; u32 size = xdpf->len; - dma_addr_t dma; - if (!unlikely(I40E_DESC_UNUSED(xdp_ring))) { + if (unlikely(I40E_DESC_UNUSED(xdp_ring) < 1 + nr_frags)) { xdp_ring->tx_stats.tx_busy++; return I40E_XDP_CONSUMED; } - dma = dma_map_single(xdp_ring->dev, data, size, DMA_TO_DEVICE); - if (dma_mapping_error(xdp_ring->dev, dma)) - return I40E_XDP_CONSUMED; - tx_bi = &xdp_ring->tx_bi[i]; - tx_bi->bytecount = size; - tx_bi->gso_segs = 1; - tx_bi->xdpf = xdpf; + tx_head->bytecount = xdp_get_frame_len(xdpf); + tx_head->gso_segs = 1; + tx_head->xdpf = xdpf; - /* record length, and DMA address */ - dma_unmap_len_set(tx_bi, len, size); - dma_unmap_addr_set(tx_bi, dma, dma); + for (;;) { + dma_addr_t dma; - tx_desc = I40E_TX_DESC(xdp_ring, i); - tx_desc->buffer_addr = cpu_to_le64(dma); - tx_desc->cmd_type_offset_bsz = build_ctob(I40E_TX_DESC_CMD_ICRC - | I40E_TXD_CMD, - 0, size, 0); + dma = dma_map_single(xdp_ring->dev, data, size, DMA_TO_DEVICE); + if (dma_mapping_error(xdp_ring->dev, dma)) + goto unmap; + + /* record length, and DMA address */ + dma_unmap_len_set(tx_bi, len, size); + dma_unmap_addr_set(tx_bi, dma, dma); + + tx_desc->buffer_addr = cpu_to_le64(dma); + tx_desc->cmd_type_offset_bsz = + build_ctob(I40E_TX_DESC_CMD_ICRC, 0, size, 0); + + if (++index == xdp_ring->count) + index = 0; + + if (i == nr_frags) + break; + + tx_bi = &xdp_ring->tx_bi[index]; + tx_desc = I40E_TX_DESC(xdp_ring, index); + + data = skb_frag_address(&sinfo->frags[i]); + size = skb_frag_size(&sinfo->frags[i]); + i++; + } + + tx_desc->cmd_type_offset_bsz |= + cpu_to_le64(I40E_TXD_CMD << I40E_TXD_QW1_CMD_SHIFT); /* Make certain all of the status bits have been updated * before next_to_watch is written. @@ -3749,14 +3768,30 @@ static int i40e_xmit_xdp_ring(struct xdp_frame *xdpf, smp_wmb(); xdp_ring->xdp_tx_active++; - i++; - if (i == xdp_ring->count) - i = 0; - tx_bi->next_to_watch = tx_desc; - xdp_ring->next_to_use = i; + tx_head->next_to_watch = tx_desc; + xdp_ring->next_to_use = index; return I40E_XDP_TX; + +unmap: + for (;;) { + tx_bi = &xdp_ring->tx_bi[index]; + if (dma_unmap_len(tx_bi, len)) + dma_unmap_page(xdp_ring->dev, + dma_unmap_addr(tx_bi, dma), + dma_unmap_len(tx_bi, len), + DMA_TO_DEVICE); + dma_unmap_len_set(tx_bi, len, 0); + if (tx_bi == tx_head) + break; + + if (!index) + index += xdp_ring->count; + index--; + } + + return I40E_XDP_CONSUMED; } /** diff --git a/drivers/net/ethernet/intel/i40e/i40e_type.h b/drivers/net/ethernet/intel/i40e/i40e_type.h index 36a4ca1ffb1a..7b3f30beb757 100644 --- a/drivers/net/ethernet/intel/i40e/i40e_type.h +++ b/drivers/net/ethernet/intel/i40e/i40e_type.h @@ -1172,6 +1172,7 @@ struct i40e_eth_stats { u64 tx_broadcast; /* bptc */ u64 tx_discards; /* tdpc */ u64 tx_errors; /* tepc */ + u64 rx_discards_other; /* rxerr1 */ }; /* Statistics collected per VEB per TC */ diff --git a/drivers/net/ethernet/intel/i40e/i40e_virtchnl_pf.c b/drivers/net/ethernet/intel/i40e/i40e_virtchnl_pf.c index 033ea71763e3..4f184c50f6e8 100644 --- a/drivers/net/ethernet/intel/i40e/i40e_virtchnl_pf.c +++ b/drivers/net/ethernet/intel/i40e/i40e_virtchnl_pf.c @@ -2147,6 +2147,10 @@ static int i40e_vc_get_vf_resources_msg(struct i40e_vf *vf, u8 *msg) /* VFs only use TC 0 */ vfres->vsi_res[0].qset_handle = le16_to_cpu(vsi->info.qs_handle[0]); + if (!(vf->driver_caps & VIRTCHNL_VF_OFFLOAD_USO) && !vf->pf_set_mac) { + i40e_del_mac_filter(vsi, vf->default_lan_addr.addr); + eth_zero_addr(vf->default_lan_addr.addr); + } ether_addr_copy(vfres->vsi_res[0].default_mac_addr, vf->default_lan_addr.addr); } @@ -4349,6 +4353,7 @@ int i40e_ndo_set_vf_port_vlan(struct net_device *netdev, int vf_id, /* duplicate request, so just return success */ goto error_pvid; + i40e_vlan_stripping_enable(vsi); i40e_vc_reset_vf(vf, true); /* During reset the VF got a new VSI, so refresh a pointer. */ vsi = pf->vsi[vf->lan_vsi_idx]; @@ -4364,7 +4369,7 @@ int i40e_ndo_set_vf_port_vlan(struct net_device *netdev, int vf_id, * MAC addresses deleted. */ if ((!(vlan_id || qos) || - vlanprio != le16_to_cpu(vsi->info.pvid)) && + vlanprio != le16_to_cpu(vsi->info.pvid)) && vsi->info.pvid) { ret = i40e_add_vlan_all_mac(vsi, I40E_VLAN_ANY); if (ret) { @@ -4727,6 +4732,11 @@ int i40e_ndo_set_vf_trust(struct net_device *netdev, int vf_id, bool setting) goto out; vf->trusted = setting; + + /* request PF to sync mac/vlan filters for the VF */ + set_bit(__I40E_MACVLAN_SYNC_PENDING, pf->state); + pf->vsi[vf->lan_vsi_idx]->flags |= I40E_VSI_FLAG_FILTER_CHANGED; + i40e_vc_reset_vf(vf, true); dev_info(&pf->pdev->dev, "VF %u is now %strusted\n", vf_id, setting ? "" : "un"); diff --git a/drivers/net/ethernet/intel/i40e/i40e_xsk.c b/drivers/net/ethernet/intel/i40e/i40e_xsk.c index af3e7e6afc85..6d4009e0cbd6 100644 --- a/drivers/net/ethernet/intel/i40e/i40e_xsk.c +++ b/drivers/net/ethernet/intel/i40e/i40e_xsk.c @@ -143,20 +143,17 @@ int i40e_xsk_pool_setup(struct i40e_vsi *vsi, struct xsk_buff_pool *pool, * i40e_run_xdp_zc - Executes an XDP program on an xdp_buff * @rx_ring: Rx ring * @xdp: xdp_buff used as input to the XDP program + * @xdp_prog: XDP program to run * * Returns any of I40E_XDP_{PASS, CONSUMED, TX, REDIR} **/ -static int i40e_run_xdp_zc(struct i40e_ring *rx_ring, struct xdp_buff *xdp) +static int i40e_run_xdp_zc(struct i40e_ring *rx_ring, struct xdp_buff *xdp, + struct bpf_prog *xdp_prog) { int err, result = I40E_XDP_PASS; struct i40e_ring *xdp_ring; - struct bpf_prog *xdp_prog; u32 act; - /* NB! xdp_prog will always be !NULL, due to the fact that - * this path is enabled by setting an XDP program. - */ - xdp_prog = READ_ONCE(rx_ring->xdp_prog); act = bpf_prog_run_xdp(xdp_prog, xdp); if (likely(act == XDP_REDIRECT)) { @@ -339,9 +336,15 @@ int i40e_clean_rx_irq_zc(struct i40e_ring *rx_ring, int budget) u16 next_to_clean = rx_ring->next_to_clean; u16 count_mask = rx_ring->count - 1; unsigned int xdp_res, xdp_xmit = 0; + struct bpf_prog *xdp_prog; bool failure = false; u16 cleaned_count; + /* NB! xdp_prog will always be !NULL, due to the fact that + * this path is enabled by setting an XDP program. + */ + xdp_prog = READ_ONCE(rx_ring->xdp_prog); + while (likely(total_rx_packets < (unsigned int)budget)) { union i40e_rx_desc *rx_desc; unsigned int rx_packets; @@ -378,7 +381,7 @@ int i40e_clean_rx_irq_zc(struct i40e_ring *rx_ring, int budget) xsk_buff_set_size(bi, size); xsk_buff_dma_sync_for_cpu(bi, rx_ring->xsk_pool); - xdp_res = i40e_run_xdp_zc(rx_ring, bi); + xdp_res = i40e_run_xdp_zc(rx_ring, bi, xdp_prog); i40e_handle_xdp_result_zc(rx_ring, bi, rx_desc, &rx_packets, &rx_bytes, size, xdp_res, &failure); if (failure) |