diff options
Diffstat (limited to 'drivers/net/ethernet/intel')
80 files changed, 7293 insertions, 901 deletions
diff --git a/drivers/net/ethernet/intel/e1000e/netdev.c b/drivers/net/ethernet/intel/e1000e/netdev.c index 360ee26557f7..f103249b12fa 100644 --- a/drivers/net/ethernet/intel/e1000e/netdev.c +++ b/drivers/net/ethernet/intel/e1000e/netdev.c @@ -6671,8 +6671,10 @@ static int __e1000_shutdown(struct pci_dev *pdev, bool runtime) if (adapter->flags2 & FLAG2_HAS_PHY_WAKEUP) { /* enable wakeup by the PHY */ retval = e1000_init_phy_wakeup(adapter, wufc); - if (retval) - return retval; + if (retval) { + e_err("Failed to enable wakeup\n"); + goto skip_phy_configurations; + } } else { /* enable wakeup by the MAC */ ew32(WUFC, wufc); @@ -6693,8 +6695,10 @@ static int __e1000_shutdown(struct pci_dev *pdev, bool runtime) * or broadcast. */ retval = e1000_enable_ulp_lpt_lp(hw, !runtime); - if (retval) - return retval; + if (retval) { + e_err("Failed to enable ULP\n"); + goto skip_phy_configurations; + } } } @@ -6726,6 +6730,7 @@ static int __e1000_shutdown(struct pci_dev *pdev, bool runtime) hw->phy.ops.release(hw); } +skip_phy_configurations: /* Release control of h/w to f/w. If f/w is AMT enabled, this * would have already happened in close and is redundant. */ @@ -6968,15 +6973,13 @@ static int e1000e_pm_suspend(struct device *dev) e1000e_pm_freeze(dev); rc = __e1000_shutdown(pdev, false); - if (rc) { - e1000e_pm_thaw(dev); - } else { + if (!rc) { /* Introduce S0ix implementation */ if (adapter->flags2 & FLAG2_ENABLE_S0IX_FLOWS) e1000e_s0ix_entry_flow(adapter); } - return rc; + return 0; } static int e1000e_pm_resume(struct device *dev) diff --git a/drivers/net/ethernet/intel/i40e/i40e.h b/drivers/net/ethernet/intel/i40e/i40e.h index d546567e0286..2089a0e172bf 100644 --- a/drivers/net/ethernet/intel/i40e/i40e.h +++ b/drivers/net/ethernet/intel/i40e/i40e.h @@ -4,6 +4,7 @@ #ifndef _I40E_H_ #define _I40E_H_ +#include <linux/linkmode.h> #include <linux/pci.h> #include <linux/ptp_clock_kernel.h> #include <linux/types.h> diff --git a/drivers/net/ethernet/intel/i40e/i40e_ethtool.c b/drivers/net/ethernet/intel/i40e/i40e_ethtool.c index 1d0d2e526adb..f2506511bbff 100644 --- a/drivers/net/ethernet/intel/i40e/i40e_ethtool.c +++ b/drivers/net/ethernet/intel/i40e/i40e_ethtool.c @@ -2555,16 +2555,12 @@ static int i40e_get_ts_info(struct net_device *dev, return ethtool_op_get_ts_info(dev, info); info->so_timestamping = SOF_TIMESTAMPING_TX_SOFTWARE | - SOF_TIMESTAMPING_RX_SOFTWARE | - SOF_TIMESTAMPING_SOFTWARE | SOF_TIMESTAMPING_TX_HARDWARE | SOF_TIMESTAMPING_RX_HARDWARE | SOF_TIMESTAMPING_RAW_HARDWARE; if (pf->ptp_clock) info->phc_index = ptp_clock_index(pf->ptp_clock); - else - info->phc_index = -1; info->tx_types = BIT(HWTSTAMP_TX_OFF) | BIT(HWTSTAMP_TX_ON); @@ -5641,6 +5637,26 @@ static int i40e_get_module_eeprom(struct net_device *netdev, return 0; } +static void i40e_eee_capability_to_kedata_supported(__le16 eee_capability_, + unsigned long *supported) +{ + const int eee_capability = le16_to_cpu(eee_capability_); + static const int lut[] = { + ETHTOOL_LINK_MODE_100baseT_Full_BIT, + ETHTOOL_LINK_MODE_1000baseT_Full_BIT, + ETHTOOL_LINK_MODE_10000baseT_Full_BIT, + ETHTOOL_LINK_MODE_1000baseKX_Full_BIT, + ETHTOOL_LINK_MODE_10000baseKX4_Full_BIT, + ETHTOOL_LINK_MODE_10000baseKR_Full_BIT, + ETHTOOL_LINK_MODE_40000baseKR4_Full_BIT, + }; + + linkmode_zero(supported); + for (unsigned int i = ARRAY_SIZE(lut); i--; ) + if (eee_capability & BIT(i + 1)) + linkmode_set_bit(lut[i], supported); +} + static int i40e_get_eee(struct net_device *netdev, struct ethtool_keee *edata) { struct i40e_netdev_priv *np = netdev_priv(netdev); @@ -5648,7 +5664,7 @@ static int i40e_get_eee(struct net_device *netdev, struct ethtool_keee *edata) struct i40e_vsi *vsi = np->vsi; struct i40e_pf *pf = vsi->back; struct i40e_hw *hw = &pf->hw; - int status = 0; + int status; /* Get initial PHY capabilities */ status = i40e_aq_get_phy_capabilities(hw, false, true, &phy_cfg, NULL); @@ -5661,11 +5677,18 @@ static int i40e_get_eee(struct net_device *netdev, struct ethtool_keee *edata) if (phy_cfg.eee_capability == 0) return -EOPNOTSUPP; + i40e_eee_capability_to_kedata_supported(phy_cfg.eee_capability, + edata->supported); + linkmode_copy(edata->lp_advertised, edata->supported); + /* Get current configuration */ status = i40e_aq_get_phy_capabilities(hw, false, false, &phy_cfg, NULL); if (status) return -EAGAIN; + linkmode_zero(edata->advertised); + if (phy_cfg.eee_capability) + linkmode_copy(edata->advertised, edata->supported); edata->eee_enabled = !!phy_cfg.eee_capability; edata->tx_lpi_enabled = pf->stats.tx_lpi_status; @@ -5681,10 +5704,11 @@ static int i40e_is_eee_param_supported(struct net_device *netdev, struct i40e_vsi *vsi = np->vsi; struct i40e_pf *pf = vsi->back; struct i40e_ethtool_not_used { - u32 value; + bool value; const char *name; } param[] = { - {edata->tx_lpi_timer, "tx-timer"}, + {!!(edata->advertised[0] & ~edata->supported[0]), "advertise"}, + {!!edata->tx_lpi_timer, "tx-timer"}, {edata->tx_lpi_enabled != pf->stats.tx_lpi_status, "tx-lpi"} }; int i; @@ -5710,7 +5734,7 @@ static int i40e_set_eee(struct net_device *netdev, struct ethtool_keee *edata) struct i40e_pf *pf = vsi->back; struct i40e_hw *hw = &pf->hw; __le16 eee_capability; - int status = 0; + int status; /* Deny parameters we don't support */ if (i40e_is_eee_param_supported(netdev, edata)) diff --git a/drivers/net/ethernet/intel/i40e/i40e_main.c b/drivers/net/ethernet/intel/i40e/i40e_main.c index cbcfada7b357..03205eb9f925 100644 --- a/drivers/net/ethernet/intel/i40e/i40e_main.c +++ b/drivers/net/ethernet/intel/i40e/i40e_main.c @@ -7264,6 +7264,26 @@ out: } #endif /* CONFIG_I40E_DCB */ +static void i40e_print_link_message_eee(struct i40e_vsi *vsi, + const char *speed, const char *fc) +{ + struct ethtool_keee kedata; + + memzero_explicit(&kedata, sizeof(kedata)); + if (vsi->netdev->ethtool_ops->get_eee) + vsi->netdev->ethtool_ops->get_eee(vsi->netdev, &kedata); + + if (!linkmode_empty(kedata.supported)) + netdev_info(vsi->netdev, + "NIC Link is Up, %sbps Full Duplex, Flow Control: %s, EEE: %s\n", + speed, fc, + kedata.eee_enabled ? "Enabled" : "Disabled"); + else + netdev_info(vsi->netdev, + "NIC Link is Up, %sbps Full Duplex, Flow Control: %s\n", + speed, fc); +} + /** * i40e_print_link_message - print link up or down * @vsi: the VSI for which link needs a message @@ -7395,9 +7415,7 @@ void i40e_print_link_message(struct i40e_vsi *vsi, bool isup) "NIC Link is Up, %sbps Full Duplex, Requested FEC: %s, Negotiated FEC: %s, Autoneg: %s, Flow Control: %s\n", speed, req_fec, fec, an, fc); } else { - netdev_info(vsi->netdev, - "NIC Link is Up, %sbps Full Duplex, Flow Control: %s\n", - speed, fc); + i40e_print_link_message_eee(vsi, speed, fc); } } diff --git a/drivers/net/ethernet/intel/iavf/iavf.h b/drivers/net/ethernet/intel/iavf/iavf.h index 23a6557fc3db..48cd1d06761c 100644 --- a/drivers/net/ethernet/intel/iavf/iavf.h +++ b/drivers/net/ethernet/intel/iavf/iavf.h @@ -33,6 +33,7 @@ #include <net/udp.h> #include <net/tc_act/tc_gact.h> #include <net/tc_act/tc_mirred.h> +#include <net/tc_act/tc_skbedit.h> #include "iavf_type.h" #include <linux/avf/virtchnl.h> @@ -393,6 +394,8 @@ struct iavf_adapter { VIRTCHNL_VF_OFFLOAD_VLAN_V2) #define CRC_OFFLOAD_ALLOWED(_a) ((_a)->vf_res->vf_cap_flags & \ VIRTCHNL_VF_OFFLOAD_CRC) +#define TC_U32_SUPPORT(_a) ((_a)->vf_res->vf_cap_flags & \ + VIRTCHNL_VF_OFFLOAD_TC_U32) #define VLAN_V2_FILTERING_ALLOWED(_a) \ (VLAN_V2_ALLOWED((_a)) && \ ((_a)->vlan_v2_caps.filtering.filtering_support.outer || \ @@ -437,6 +440,7 @@ struct iavf_adapter { #define IAVF_MAX_FDIR_FILTERS 128 /* max allowed Flow Director filters */ u16 fdir_active_fltr; + u16 raw_fdir_active_fltr; struct list_head fdir_list_head; spinlock_t fdir_fltr_lock; /* protect the Flow Director filter list */ @@ -444,6 +448,32 @@ struct iavf_adapter { spinlock_t adv_rss_lock; /* protect the RSS management list */ }; +/* Must be called with fdir_fltr_lock lock held */ +static inline bool iavf_fdir_max_reached(struct iavf_adapter *adapter) +{ + return adapter->fdir_active_fltr + adapter->raw_fdir_active_fltr >= + IAVF_MAX_FDIR_FILTERS; +} + +static inline void +iavf_inc_fdir_active_fltr(struct iavf_adapter *adapter, + struct iavf_fdir_fltr *fltr) +{ + if (iavf_is_raw_fdir(fltr)) + adapter->raw_fdir_active_fltr++; + else + adapter->fdir_active_fltr++; +} + +static inline void +iavf_dec_fdir_active_fltr(struct iavf_adapter *adapter, + struct iavf_fdir_fltr *fltr) +{ + if (iavf_is_raw_fdir(fltr)) + adapter->raw_fdir_active_fltr--; + else + adapter->fdir_active_fltr--; +} /* Ethtool Private Flags */ diff --git a/drivers/net/ethernet/intel/iavf/iavf_ethtool.c b/drivers/net/ethernet/intel/iavf/iavf_ethtool.c index 52273f7eab2c..74a1e9fe1821 100644 --- a/drivers/net/ethernet/intel/iavf/iavf_ethtool.c +++ b/drivers/net/ethernet/intel/iavf/iavf_ethtool.c @@ -927,7 +927,7 @@ iavf_get_ethtool_fdir_entry(struct iavf_adapter *adapter, spin_lock_bh(&adapter->fdir_fltr_lock); - rule = iavf_find_fdir_fltr_by_loc(adapter, fsp->location); + rule = iavf_find_fdir_fltr(adapter, false, fsp->location); if (!rule) { ret = -EINVAL; goto release_lock; @@ -1072,6 +1072,9 @@ iavf_get_fdir_fltr_ids(struct iavf_adapter *adapter, struct ethtool_rxnfc *cmd, spin_lock_bh(&adapter->fdir_fltr_lock); list_for_each_entry(fltr, &adapter->fdir_list_head, list) { + if (iavf_is_raw_fdir(fltr)) + continue; + if (cnt == cmd->rule_cnt) { val = -EMSGSIZE; goto release_lock; @@ -1263,15 +1266,7 @@ static int iavf_add_fdir_ethtool(struct iavf_adapter *adapter, struct ethtool_rx return -EINVAL; spin_lock_bh(&adapter->fdir_fltr_lock); - if (adapter->fdir_active_fltr >= IAVF_MAX_FDIR_FILTERS) { - spin_unlock_bh(&adapter->fdir_fltr_lock); - dev_err(&adapter->pdev->dev, - "Unable to add Flow Director filter because VF reached the limit of max allowed filters (%u)\n", - IAVF_MAX_FDIR_FILTERS); - return -ENOSPC; - } - - if (iavf_find_fdir_fltr_by_loc(adapter, fsp->location)) { + if (iavf_find_fdir_fltr(adapter, false, fsp->location)) { dev_err(&adapter->pdev->dev, "Failed to add Flow Director filter, it already exists\n"); spin_unlock_bh(&adapter->fdir_fltr_lock); return -EEXIST; @@ -1291,23 +1286,10 @@ static int iavf_add_fdir_ethtool(struct iavf_adapter *adapter, struct ethtool_rx } err = iavf_add_fdir_fltr_info(adapter, fsp, fltr); - if (err) - goto ret; - - spin_lock_bh(&adapter->fdir_fltr_lock); - iavf_fdir_list_add_fltr(adapter, fltr); - adapter->fdir_active_fltr++; - - if (adapter->link_up) - fltr->state = IAVF_FDIR_FLTR_ADD_REQUEST; - else - fltr->state = IAVF_FDIR_FLTR_INACTIVE; - spin_unlock_bh(&adapter->fdir_fltr_lock); + if (!err) + err = iavf_fdir_add_fltr(adapter, fltr); - if (adapter->link_up) - iavf_schedule_aq_request(adapter, IAVF_FLAG_AQ_ADD_FDIR_FILTER); -ret: - if (err && fltr) + if (err) kfree(fltr); mutex_unlock(&adapter->crit_lock); @@ -1324,34 +1306,11 @@ ret: static int iavf_del_fdir_ethtool(struct iavf_adapter *adapter, struct ethtool_rxnfc *cmd) { struct ethtool_rx_flow_spec *fsp = (struct ethtool_rx_flow_spec *)&cmd->fs; - struct iavf_fdir_fltr *fltr = NULL; - int err = 0; if (!(adapter->flags & IAVF_FLAG_FDIR_ENABLED)) return -EOPNOTSUPP; - spin_lock_bh(&adapter->fdir_fltr_lock); - fltr = iavf_find_fdir_fltr_by_loc(adapter, fsp->location); - if (fltr) { - if (fltr->state == IAVF_FDIR_FLTR_ACTIVE) { - fltr->state = IAVF_FDIR_FLTR_DEL_REQUEST; - } else if (fltr->state == IAVF_FDIR_FLTR_INACTIVE) { - list_del(&fltr->list); - kfree(fltr); - adapter->fdir_active_fltr--; - fltr = NULL; - } else { - err = -EBUSY; - } - } else if (adapter->fdir_active_fltr) { - err = -EINVAL; - } - spin_unlock_bh(&adapter->fdir_fltr_lock); - - if (fltr && fltr->state == IAVF_FDIR_FLTR_DEL_REQUEST) - iavf_schedule_aq_request(adapter, IAVF_FLAG_AQ_DEL_FDIR_FILTER); - - return err; + return iavf_fdir_del_fltr(adapter, false, fsp->location); } /** diff --git a/drivers/net/ethernet/intel/iavf/iavf_fdir.c b/drivers/net/ethernet/intel/iavf/iavf_fdir.c index 2d47b0b4640e..a1b3b44cc14a 100644 --- a/drivers/net/ethernet/intel/iavf/iavf_fdir.c +++ b/drivers/net/ethernet/intel/iavf/iavf_fdir.c @@ -796,6 +796,9 @@ bool iavf_fdir_is_dup_fltr(struct iavf_adapter *adapter, struct iavf_fdir_fltr * spin_lock_bh(&adapter->fdir_fltr_lock); list_for_each_entry(tmp, &adapter->fdir_list_head, list) { + if (iavf_is_raw_fdir(fltr)) + continue; + if (tmp->flow_type != fltr->flow_type) continue; @@ -815,33 +818,52 @@ bool iavf_fdir_is_dup_fltr(struct iavf_adapter *adapter, struct iavf_fdir_fltr * } /** - * iavf_find_fdir_fltr_by_loc - find filter with location + * iavf_find_fdir_fltr - find FDIR filter * @adapter: pointer to the VF adapter structure - * @loc: location to find. + * @is_raw: filter type, is raw (tc u32) or not (ethtool) + * @data: data to ID the filter, type dependent * - * Returns pointer to Flow Director filter if found or null + * Returns: pointer to Flow Director filter if found or NULL. Lock must be held. */ -struct iavf_fdir_fltr *iavf_find_fdir_fltr_by_loc(struct iavf_adapter *adapter, u32 loc) +struct iavf_fdir_fltr *iavf_find_fdir_fltr(struct iavf_adapter *adapter, + bool is_raw, u32 data) { struct iavf_fdir_fltr *rule; - list_for_each_entry(rule, &adapter->fdir_list_head, list) - if (rule->loc == loc) + list_for_each_entry(rule, &adapter->fdir_list_head, list) { + if ((is_raw && rule->cls_u32_handle == data) || + (!is_raw && rule->loc == data)) return rule; + } return NULL; } /** - * iavf_fdir_list_add_fltr - add a new node to the flow director filter list + * iavf_fdir_add_fltr - add a new node to the flow director filter list * @adapter: pointer to the VF adapter structure * @fltr: filter node to add to structure + * + * Return: 0 on success or negative errno on failure. */ -void iavf_fdir_list_add_fltr(struct iavf_adapter *adapter, struct iavf_fdir_fltr *fltr) +int iavf_fdir_add_fltr(struct iavf_adapter *adapter, + struct iavf_fdir_fltr *fltr) { struct iavf_fdir_fltr *rule, *parent = NULL; + spin_lock_bh(&adapter->fdir_fltr_lock); + if (iavf_fdir_max_reached(adapter)) { + spin_unlock_bh(&adapter->fdir_fltr_lock); + dev_err(&adapter->pdev->dev, + "Unable to add Flow Director filter (limit (%u) reached)\n", + IAVF_MAX_FDIR_FILTERS); + return -ENOSPC; + } + list_for_each_entry(rule, &adapter->fdir_list_head, list) { + if (iavf_is_raw_fdir(fltr)) + break; + if (rule->loc >= fltr->loc) break; parent = rule; @@ -851,4 +873,55 @@ void iavf_fdir_list_add_fltr(struct iavf_adapter *adapter, struct iavf_fdir_fltr list_add(&fltr->list, &parent->list); else list_add(&fltr->list, &adapter->fdir_list_head); + + iavf_inc_fdir_active_fltr(adapter, fltr); + + if (adapter->link_up) + fltr->state = IAVF_FDIR_FLTR_ADD_REQUEST; + else + fltr->state = IAVF_FDIR_FLTR_INACTIVE; + spin_unlock_bh(&adapter->fdir_fltr_lock); + + if (adapter->link_up) + iavf_schedule_aq_request(adapter, IAVF_FLAG_AQ_ADD_FDIR_FILTER); + + return 0; +} + +/** + * iavf_fdir_del_fltr - delete a flow director filter from the list + * @adapter: pointer to the VF adapter structure + * @is_raw: filter type, is raw (tc u32) or not (ethtool) + * @data: data to ID the filter, type dependent + * + * Return: 0 on success or negative errno on failure. + */ +int iavf_fdir_del_fltr(struct iavf_adapter *adapter, bool is_raw, u32 data) +{ + struct iavf_fdir_fltr *fltr = NULL; + int err = 0; + + spin_lock_bh(&adapter->fdir_fltr_lock); + fltr = iavf_find_fdir_fltr(adapter, is_raw, data); + + if (fltr) { + if (fltr->state == IAVF_FDIR_FLTR_ACTIVE) { + fltr->state = IAVF_FDIR_FLTR_DEL_REQUEST; + } else if (fltr->state == IAVF_FDIR_FLTR_INACTIVE) { + list_del(&fltr->list); + iavf_dec_fdir_active_fltr(adapter, fltr); + kfree(fltr); + fltr = NULL; + } else { + err = -EBUSY; + } + } else if (adapter->fdir_active_fltr) { + err = -EINVAL; + } + + if (fltr && fltr->state == IAVF_FDIR_FLTR_DEL_REQUEST) + iavf_schedule_aq_request(adapter, IAVF_FLAG_AQ_DEL_FDIR_FILTER); + + spin_unlock_bh(&adapter->fdir_fltr_lock); + return err; } diff --git a/drivers/net/ethernet/intel/iavf/iavf_fdir.h b/drivers/net/ethernet/intel/iavf/iavf_fdir.h index d31bd923ba8c..e84a5351162f 100644 --- a/drivers/net/ethernet/intel/iavf/iavf_fdir.h +++ b/drivers/net/ethernet/intel/iavf/iavf_fdir.h @@ -117,17 +117,26 @@ struct iavf_fdir_fltr { u32 flow_id; + u32 cls_u32_handle; /* for FDIR added via tc u32 */ u32 loc; /* Rule location inside the flow table */ u32 q_index; struct virtchnl_fdir_add vc_add_msg; }; +static inline bool iavf_is_raw_fdir(struct iavf_fdir_fltr *fltr) +{ + return !fltr->vc_add_msg.rule_cfg.proto_hdrs.count; +} + int iavf_validate_fdir_fltr_masks(struct iavf_adapter *adapter, struct iavf_fdir_fltr *fltr); int iavf_fill_fdir_add_msg(struct iavf_adapter *adapter, struct iavf_fdir_fltr *fltr); void iavf_print_fdir_fltr(struct iavf_adapter *adapter, struct iavf_fdir_fltr *fltr); bool iavf_fdir_is_dup_fltr(struct iavf_adapter *adapter, struct iavf_fdir_fltr *fltr); -void iavf_fdir_list_add_fltr(struct iavf_adapter *adapter, struct iavf_fdir_fltr *fltr); -struct iavf_fdir_fltr *iavf_find_fdir_fltr_by_loc(struct iavf_adapter *adapter, u32 loc); +int iavf_fdir_add_fltr(struct iavf_adapter *adapter, + struct iavf_fdir_fltr *fltr); +int iavf_fdir_del_fltr(struct iavf_adapter *adapter, bool is_raw, u32 data); +struct iavf_fdir_fltr *iavf_find_fdir_fltr(struct iavf_adapter *adapter, + bool is_raw, u32 data); #endif /* _IAVF_FDIR_H_ */ diff --git a/drivers/net/ethernet/intel/iavf/iavf_main.c b/drivers/net/ethernet/intel/iavf/iavf_main.c index ff11bafb3b4f..f782402cd789 100644 --- a/drivers/net/ethernet/intel/iavf/iavf_main.c +++ b/drivers/net/ethernet/intel/iavf/iavf_main.c @@ -4013,7 +4013,7 @@ static int iavf_delete_clsflower(struct iavf_adapter *adapter, /** * iavf_setup_tc_cls_flower - flower classifier offloads - * @adapter: board private structure + * @adapter: pointer to iavf adapter structure * @cls_flower: pointer to flow_cls_offload struct with flow info */ static int iavf_setup_tc_cls_flower(struct iavf_adapter *adapter, @@ -4032,6 +4032,154 @@ static int iavf_setup_tc_cls_flower(struct iavf_adapter *adapter, } /** + * iavf_add_cls_u32 - Add U32 classifier offloads + * @adapter: pointer to iavf adapter structure + * @cls_u32: pointer to tc_cls_u32_offload struct with flow info + * + * Return: 0 on success or negative errno on failure. + */ +static int iavf_add_cls_u32(struct iavf_adapter *adapter, + struct tc_cls_u32_offload *cls_u32) +{ + struct netlink_ext_ack *extack = cls_u32->common.extack; + struct virtchnl_fdir_rule *rule_cfg; + struct virtchnl_filter_action *vact; + struct virtchnl_proto_hdrs *hdrs; + struct ethhdr *spec_h, *mask_h; + const struct tc_action *act; + struct iavf_fdir_fltr *fltr; + struct tcf_exts *exts; + unsigned int q_index; + int i, status = 0; + int off_base = 0; + + if (cls_u32->knode.link_handle) { + NL_SET_ERR_MSG_MOD(extack, "Linking not supported"); + return -EOPNOTSUPP; + } + + fltr = kzalloc(sizeof(*fltr), GFP_KERNEL); + if (!fltr) + return -ENOMEM; + + rule_cfg = &fltr->vc_add_msg.rule_cfg; + hdrs = &rule_cfg->proto_hdrs; + hdrs->count = 0; + + /* The parser lib at the PF expects the packet starting with MAC hdr */ + switch (ntohs(cls_u32->common.protocol)) { + case ETH_P_802_3: + break; + case ETH_P_IP: + spec_h = (struct ethhdr *)hdrs->raw.spec; + mask_h = (struct ethhdr *)hdrs->raw.mask; + spec_h->h_proto = htons(ETH_P_IP); + mask_h->h_proto = htons(0xFFFF); + off_base += ETH_HLEN; + break; + default: + NL_SET_ERR_MSG_MOD(extack, "Only 802_3 and ip filter protocols are supported"); + status = -EOPNOTSUPP; + goto free_alloc; + } + + for (i = 0; i < cls_u32->knode.sel->nkeys; i++) { + __be32 val, mask; + int off; + + off = off_base + cls_u32->knode.sel->keys[i].off; + val = cls_u32->knode.sel->keys[i].val; + mask = cls_u32->knode.sel->keys[i].mask; + + if (off >= sizeof(hdrs->raw.spec)) { + NL_SET_ERR_MSG_MOD(extack, "Input exceeds maximum allowed."); + status = -EINVAL; + goto free_alloc; + } + + memcpy(&hdrs->raw.spec[off], &val, sizeof(val)); + memcpy(&hdrs->raw.mask[off], &mask, sizeof(mask)); + hdrs->raw.pkt_len = off + sizeof(val); + } + + /* Only one action is allowed */ + rule_cfg->action_set.count = 1; + vact = &rule_cfg->action_set.actions[0]; + exts = cls_u32->knode.exts; + + tcf_exts_for_each_action(i, act, exts) { + /* FDIR queue */ + if (is_tcf_skbedit_rx_queue_mapping(act)) { + q_index = tcf_skbedit_rx_queue_mapping(act); + if (q_index >= adapter->num_active_queues) { + status = -EINVAL; + goto free_alloc; + } + + vact->type = VIRTCHNL_ACTION_QUEUE; + vact->act_conf.queue.index = q_index; + break; + } + + /* Drop */ + if (is_tcf_gact_shot(act)) { + vact->type = VIRTCHNL_ACTION_DROP; + break; + } + + /* Unsupported action */ + NL_SET_ERR_MSG_MOD(extack, "Unsupported action."); + status = -EOPNOTSUPP; + goto free_alloc; + } + + fltr->vc_add_msg.vsi_id = adapter->vsi.id; + fltr->cls_u32_handle = cls_u32->knode.handle; + return iavf_fdir_add_fltr(adapter, fltr); + +free_alloc: + kfree(fltr); + return status; +} + +/** + * iavf_del_cls_u32 - Delete U32 classifier offloads + * @adapter: pointer to iavf adapter structure + * @cls_u32: pointer to tc_cls_u32_offload struct with flow info + * + * Return: 0 on success or negative errno on failure. + */ +static int iavf_del_cls_u32(struct iavf_adapter *adapter, + struct tc_cls_u32_offload *cls_u32) +{ + return iavf_fdir_del_fltr(adapter, true, cls_u32->knode.handle); +} + +/** + * iavf_setup_tc_cls_u32 - U32 filter offloads + * @adapter: pointer to iavf adapter structure + * @cls_u32: pointer to tc_cls_u32_offload struct with flow info + * + * Return: 0 on success or negative errno on failure. + */ +static int iavf_setup_tc_cls_u32(struct iavf_adapter *adapter, + struct tc_cls_u32_offload *cls_u32) +{ + if (!TC_U32_SUPPORT(adapter) || !FDIR_FLTR_SUPPORT(adapter)) + return -EOPNOTSUPP; + + switch (cls_u32->command) { + case TC_CLSU32_NEW_KNODE: + case TC_CLSU32_REPLACE_KNODE: + return iavf_add_cls_u32(adapter, cls_u32); + case TC_CLSU32_DELETE_KNODE: + return iavf_del_cls_u32(adapter, cls_u32); + default: + return -EOPNOTSUPP; + } +} + +/** * iavf_setup_tc_block_cb - block callback for tc * @type: type of offload * @type_data: offload data @@ -4050,6 +4198,8 @@ static int iavf_setup_tc_block_cb(enum tc_setup_type type, void *type_data, switch (type) { case TC_SETUP_CLSFLOWER: return iavf_setup_tc_cls_flower(cb_priv, type_data); + case TC_SETUP_CLSU32: + return iavf_setup_tc_cls_u32(cb_priv, type_data); default: return -EOPNOTSUPP; } @@ -4332,8 +4482,8 @@ static void iavf_disable_fdir(struct iavf_adapter *adapter) fdir->state == IAVF_FDIR_FLTR_INACTIVE) { /* Delete filters not registered in PF */ list_del(&fdir->list); + iavf_dec_fdir_active_fltr(adapter, fdir); kfree(fdir); - adapter->fdir_active_fltr--; } else if (fdir->state == IAVF_FDIR_FLTR_ADD_PENDING || fdir->state == IAVF_FDIR_FLTR_DIS_REQUEST || fdir->state == IAVF_FDIR_FLTR_ACTIVE) { @@ -4843,9 +4993,11 @@ int iavf_process_config(struct iavf_adapter *adapter) /* get HW VLAN features that can be toggled */ hw_vlan_features = iavf_get_netdev_vlan_hw_features(adapter); - /* Enable cloud filter if ADQ is supported */ - if (vfres->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_ADQ) + /* Enable HW TC offload if ADQ or tc U32 is supported */ + if (vfres->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_ADQ || + TC_U32_SUPPORT(adapter)) hw_features |= NETIF_F_HW_TC; + if (vfres->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_USO) hw_features |= NETIF_F_GSO_UDP_L4; diff --git a/drivers/net/ethernet/intel/iavf/iavf_virtchnl.c b/drivers/net/ethernet/intel/iavf/iavf_virtchnl.c index 1e543f6a7c30..7e810b65380c 100644 --- a/drivers/net/ethernet/intel/iavf/iavf_virtchnl.c +++ b/drivers/net/ethernet/intel/iavf/iavf_virtchnl.c @@ -142,6 +142,7 @@ int iavf_send_vf_config_msg(struct iavf_adapter *adapter) VIRTCHNL_VF_OFFLOAD_WB_ON_ITR | VIRTCHNL_VF_OFFLOAD_RSS_PCTYPE_V2 | VIRTCHNL_VF_OFFLOAD_ENCAP | + VIRTCHNL_VF_OFFLOAD_TC_U32 | VIRTCHNL_VF_OFFLOAD_VLAN_V2 | VIRTCHNL_VF_OFFLOAD_CRC | VIRTCHNL_VF_OFFLOAD_ENCAP_CSUM | @@ -1961,8 +1962,8 @@ static void iavf_activate_fdir_filters(struct iavf_adapter *adapter) * list on PF is already cleared after a reset */ list_del(&f->list); + iavf_dec_fdir_active_fltr(adapter, f); kfree(f); - adapter->fdir_active_fltr--; } } spin_unlock_bh(&adapter->fdir_fltr_lock); @@ -2135,8 +2136,8 @@ void iavf_virtchnl_completion(struct iavf_adapter *adapter, dev_err(&adapter->pdev->dev, "%s\n", msg); list_del(&fdir->list); + iavf_dec_fdir_active_fltr(adapter, fdir); kfree(fdir); - adapter->fdir_active_fltr--; } } spin_unlock_bh(&adapter->fdir_fltr_lock); @@ -2451,8 +2452,12 @@ void iavf_virtchnl_completion(struct iavf_adapter *adapter, list) { if (fdir->state == IAVF_FDIR_FLTR_ADD_PENDING) { if (add_fltr->status == VIRTCHNL_FDIR_SUCCESS) { - dev_info(&adapter->pdev->dev, "Flow Director filter with location %u is added\n", - fdir->loc); + if (!iavf_is_raw_fdir(fdir)) + dev_info(&adapter->pdev->dev, "Flow Director filter with location %u is added\n", + fdir->loc); + else + dev_info(&adapter->pdev->dev, "Flow Director filter (raw) for TC handle %x is added\n", + TC_U32_USERHTID(fdir->cls_u32_handle)); fdir->state = IAVF_FDIR_FLTR_ACTIVE; fdir->flow_id = add_fltr->flow_id; } else { @@ -2460,8 +2465,8 @@ void iavf_virtchnl_completion(struct iavf_adapter *adapter, add_fltr->status); iavf_print_fdir_fltr(adapter, fdir); list_del(&fdir->list); + iavf_dec_fdir_active_fltr(adapter, fdir); kfree(fdir); - adapter->fdir_active_fltr--; } } } @@ -2479,11 +2484,15 @@ void iavf_virtchnl_completion(struct iavf_adapter *adapter, if (del_fltr->status == VIRTCHNL_FDIR_SUCCESS || del_fltr->status == VIRTCHNL_FDIR_FAILURE_RULE_NONEXIST) { - dev_info(&adapter->pdev->dev, "Flow Director filter with location %u is deleted\n", - fdir->loc); + if (!iavf_is_raw_fdir(fdir)) + dev_info(&adapter->pdev->dev, "Flow Director filter with location %u is deleted\n", + fdir->loc); + else + dev_info(&adapter->pdev->dev, "Flow Director filter (raw) for TC handle %x is deleted\n", + TC_U32_USERHTID(fdir->cls_u32_handle)); list_del(&fdir->list); + iavf_dec_fdir_active_fltr(adapter, fdir); kfree(fdir); - adapter->fdir_active_fltr--; } else { fdir->state = IAVF_FDIR_FLTR_ACTIVE; dev_info(&adapter->pdev->dev, "Failed to delete Flow Director filter with status: %d\n", diff --git a/drivers/net/ethernet/intel/ice/Makefile b/drivers/net/ethernet/intel/ice/Makefile index 03500e28ac99..3307d551f431 100644 --- a/drivers/net/ethernet/intel/ice/Makefile +++ b/drivers/net/ethernet/intel/ice/Makefile @@ -28,9 +28,13 @@ ice-y := ice_main.o \ ice_vlan_mode.o \ ice_flex_pipe.o \ ice_flow.o \ + ice_parser.o \ + ice_parser_rt.o \ ice_idc.o \ devlink/devlink.o \ devlink/devlink_port.o \ + ice_sf_eth.o \ + ice_sf_vsi_vlan_ops.o \ ice_ddp.o \ ice_fw_update.o \ ice_lag.o \ diff --git a/drivers/net/ethernet/intel/ice/devlink/devlink.c b/drivers/net/ethernet/intel/ice/devlink/devlink.c index 810a901d7afd..415445cefdb2 100644 --- a/drivers/net/ethernet/intel/ice/devlink/devlink.c +++ b/drivers/net/ethernet/intel/ice/devlink/devlink.c @@ -6,9 +6,11 @@ #include "ice.h" #include "ice_lib.h" #include "devlink.h" +#include "devlink_port.h" #include "ice_eswitch.h" #include "ice_fw_update.h" #include "ice_dcb_lib.h" +#include "ice_sf_eth.h" /* context for devlink info version reporting */ struct ice_info_ctx { @@ -744,6 +746,7 @@ static void ice_traverse_tx_tree(struct devlink *devlink, struct ice_sched_node struct ice_sched_node *tc_node, struct ice_pf *pf) { struct devlink_rate *rate_node = NULL; + struct ice_dynamic_port *sf; struct ice_vf *vf; int i; @@ -755,6 +758,7 @@ static void ice_traverse_tx_tree(struct devlink *devlink, struct ice_sched_node /* create root node */ rate_node = devl_rate_node_create(devlink, node, node->name, NULL); } else if (node->vsi_handle && + pf->vsi[node->vsi_handle]->type == ICE_VSI_VF && pf->vsi[node->vsi_handle]->vf) { vf = pf->vsi[node->vsi_handle]->vf; if (!vf->devlink_port.devlink_rate) @@ -763,6 +767,16 @@ static void ice_traverse_tx_tree(struct devlink *devlink, struct ice_sched_node */ devl_rate_leaf_create(&vf->devlink_port, node, node->parent->rate_node); + } else if (node->vsi_handle && + pf->vsi[node->vsi_handle]->type == ICE_VSI_SF && + pf->vsi[node->vsi_handle]->sf) { + sf = pf->vsi[node->vsi_handle]->sf; + if (!sf->devlink_port.devlink_rate) + /* leaf nodes doesn't have children + * so we don't set rate_node + */ + devl_rate_leaf_create(&sf->devlink_port, node, + node->parent->rate_node); } else if (node->info.data.elem_type != ICE_AQC_ELEM_TYPE_LEAF && node->parent->rate_node) { rate_node = devl_rate_node_create(devlink, node, node->name, @@ -1277,8 +1291,12 @@ static const struct devlink_ops ice_devlink_ops = { .rate_leaf_parent_set = ice_devlink_set_parent, .rate_node_parent_set = ice_devlink_set_parent, + + .port_new = ice_devlink_port_new, }; +static const struct devlink_ops ice_sf_devlink_ops; + static int ice_devlink_enable_roce_get(struct devlink *devlink, u32 id, struct devlink_param_gset_ctx *ctx) @@ -1562,6 +1580,34 @@ struct ice_pf *ice_allocate_pf(struct device *dev) } /** + * ice_allocate_sf - Allocate devlink and return SF structure pointer + * @dev: the device to allocate for + * @pf: pointer to the PF structure + * + * Allocate a devlink instance for SF. + * + * Return: ice_sf_priv pointer to allocated memory or ERR_PTR in case of error + */ +struct ice_sf_priv *ice_allocate_sf(struct device *dev, struct ice_pf *pf) +{ + struct devlink *devlink; + int err; + + devlink = devlink_alloc(&ice_sf_devlink_ops, sizeof(struct ice_sf_priv), + dev); + if (!devlink) + return ERR_PTR(-ENOMEM); + + err = devl_nested_devlink_set(priv_to_devlink(pf), devlink); + if (err) { + devlink_free(devlink); + return ERR_PTR(err); + } + + return devlink_priv(devlink); +} + +/** * ice_devlink_register - Register devlink interface for this PF * @pf: the PF to register the devlink for. * diff --git a/drivers/net/ethernet/intel/ice/devlink/devlink.h b/drivers/net/ethernet/intel/ice/devlink/devlink.h index d291c0e2e17b..1af3b0763fbb 100644 --- a/drivers/net/ethernet/intel/ice/devlink/devlink.h +++ b/drivers/net/ethernet/intel/ice/devlink/devlink.h @@ -5,6 +5,7 @@ #define _ICE_DEVLINK_H_ struct ice_pf *ice_allocate_pf(struct device *dev); +struct ice_sf_priv *ice_allocate_sf(struct device *dev, struct ice_pf *pf); void ice_devlink_register(struct ice_pf *pf); void ice_devlink_unregister(struct ice_pf *pf); diff --git a/drivers/net/ethernet/intel/ice/devlink/devlink_port.c b/drivers/net/ethernet/intel/ice/devlink/devlink_port.c index 62ef8e2fb5f1..928c8bdb6649 100644 --- a/drivers/net/ethernet/intel/ice/devlink/devlink_port.c +++ b/drivers/net/ethernet/intel/ice/devlink/devlink_port.c @@ -5,6 +5,9 @@ #include "ice.h" #include "devlink.h" +#include "devlink_port.h" +#include "ice_lib.h" +#include "ice_fltr.h" static int ice_active_port_option = -1; @@ -485,3 +488,506 @@ void ice_devlink_destroy_vf_port(struct ice_vf *vf) devl_rate_leaf_destroy(&vf->devlink_port); devl_port_unregister(&vf->devlink_port); } + +/** + * ice_devlink_create_sf_dev_port - Register virtual port for a subfunction + * @sf_dev: the subfunction device to create a devlink port for + * + * Register virtual flavour devlink port for the subfunction auxiliary device + * created after activating a dynamically added devlink port. + * + * Return: zero on success or an error code on failure. + */ +int ice_devlink_create_sf_dev_port(struct ice_sf_dev *sf_dev) +{ + struct devlink_port_attrs attrs = {}; + struct ice_dynamic_port *dyn_port; + struct devlink_port *devlink_port; + struct devlink *devlink; + struct ice_vsi *vsi; + + dyn_port = sf_dev->dyn_port; + vsi = dyn_port->vsi; + + devlink_port = &sf_dev->priv->devlink_port; + + attrs.flavour = DEVLINK_PORT_FLAVOUR_VIRTUAL; + + devlink_port_attrs_set(devlink_port, &attrs); + devlink = priv_to_devlink(sf_dev->priv); + + return devl_port_register(devlink, devlink_port, vsi->idx); +} + +/** + * ice_devlink_destroy_sf_dev_port - Destroy virtual port for a subfunction + * @sf_dev: the subfunction device to create a devlink port for + * + * Unregisters the virtual port associated with this subfunction. + */ +void ice_devlink_destroy_sf_dev_port(struct ice_sf_dev *sf_dev) +{ + devl_port_unregister(&sf_dev->priv->devlink_port); +} + +/** + * ice_activate_dynamic_port - Activate a dynamic port + * @dyn_port: dynamic port instance to activate + * @extack: extack for reporting error messages + * + * Activate the dynamic port based on its flavour. + * + * Return: zero on success or an error code on failure. + */ +static int +ice_activate_dynamic_port(struct ice_dynamic_port *dyn_port, + struct netlink_ext_ack *extack) +{ + int err; + + if (dyn_port->active) + return 0; + + err = ice_sf_eth_activate(dyn_port, extack); + if (err) + return err; + + dyn_port->active = true; + + return 0; +} + +/** + * ice_deactivate_dynamic_port - Deactivate a dynamic port + * @dyn_port: dynamic port instance to deactivate + * + * Undo activation of a dynamic port. + */ +static void ice_deactivate_dynamic_port(struct ice_dynamic_port *dyn_port) +{ + if (!dyn_port->active) + return; + + ice_sf_eth_deactivate(dyn_port); + dyn_port->active = false; +} + +/** + * ice_dealloc_dynamic_port - Deallocate and remove a dynamic port + * @dyn_port: dynamic port instance to deallocate + * + * Free resources associated with a dynamically added devlink port. Will + * deactivate the port if its currently active. + */ +static void ice_dealloc_dynamic_port(struct ice_dynamic_port *dyn_port) +{ + struct devlink_port *devlink_port = &dyn_port->devlink_port; + struct ice_pf *pf = dyn_port->pf; + + ice_deactivate_dynamic_port(dyn_port); + + xa_erase(&pf->sf_nums, devlink_port->attrs.pci_sf.sf); + ice_eswitch_detach_sf(pf, dyn_port); + ice_vsi_free(dyn_port->vsi); + xa_erase(&pf->dyn_ports, dyn_port->vsi->idx); + kfree(dyn_port); +} + +/** + * ice_dealloc_all_dynamic_ports - Deallocate all dynamic devlink ports + * @pf: pointer to the pf structure + */ +void ice_dealloc_all_dynamic_ports(struct ice_pf *pf) +{ + struct ice_dynamic_port *dyn_port; + unsigned long index; + + xa_for_each(&pf->dyn_ports, index, dyn_port) + ice_dealloc_dynamic_port(dyn_port); +} + +/** + * ice_devlink_port_new_check_attr - Check that new port attributes are valid + * @pf: pointer to the PF structure + * @new_attr: the attributes for the new port + * @extack: extack for reporting error messages + * + * Check that the attributes for the new port are valid before continuing to + * allocate the devlink port. + * + * Return: zero on success or an error code on failure. + */ +static int +ice_devlink_port_new_check_attr(struct ice_pf *pf, + const struct devlink_port_new_attrs *new_attr, + struct netlink_ext_ack *extack) +{ + if (new_attr->flavour != DEVLINK_PORT_FLAVOUR_PCI_SF) { + NL_SET_ERR_MSG_MOD(extack, "Flavour other than pcisf is not supported"); + return -EOPNOTSUPP; + } + + if (new_attr->controller_valid) { + NL_SET_ERR_MSG_MOD(extack, "Setting controller is not supported"); + return -EOPNOTSUPP; + } + + if (new_attr->port_index_valid) { + NL_SET_ERR_MSG_MOD(extack, "Driver does not support user defined port index assignment"); + return -EOPNOTSUPP; + } + + if (new_attr->pfnum != pf->hw.pf_id) { + NL_SET_ERR_MSG_MOD(extack, "Incorrect pfnum supplied"); + return -EINVAL; + } + + if (!pci_msix_can_alloc_dyn(pf->pdev)) { + NL_SET_ERR_MSG_MOD(extack, "Dynamic MSIX-X interrupt allocation is not supported"); + return -EOPNOTSUPP; + } + + return 0; +} + +/** + * ice_devlink_port_del - devlink handler for port delete + * @devlink: pointer to devlink + * @port: devlink port to be deleted + * @extack: pointer to extack + * + * Deletes devlink port and deallocates all resources associated with + * created subfunction. + * + * Return: zero on success or an error code on failure. + */ +static int +ice_devlink_port_del(struct devlink *devlink, struct devlink_port *port, + struct netlink_ext_ack *extack) +{ + struct ice_dynamic_port *dyn_port; + + dyn_port = ice_devlink_port_to_dyn(port); + ice_dealloc_dynamic_port(dyn_port); + + return 0; +} + +/** + * ice_devlink_port_fn_hw_addr_set - devlink handler for mac address set + * @port: pointer to devlink port + * @hw_addr: hw address to set + * @hw_addr_len: hw address length + * @extack: extack for reporting error messages + * + * Sets mac address for the port, verifies arguments and copies address + * to the subfunction structure. + * + * Return: zero on success or an error code on failure. + */ +static int +ice_devlink_port_fn_hw_addr_set(struct devlink_port *port, const u8 *hw_addr, + int hw_addr_len, + struct netlink_ext_ack *extack) +{ + struct ice_dynamic_port *dyn_port; + + dyn_port = ice_devlink_port_to_dyn(port); + + if (dyn_port->attached) { + NL_SET_ERR_MSG_MOD(extack, + "Ethernet address can be change only in detached state"); + return -EBUSY; + } + + if (hw_addr_len != ETH_ALEN || !is_valid_ether_addr(hw_addr)) { + NL_SET_ERR_MSG_MOD(extack, "Invalid ethernet address"); + return -EADDRNOTAVAIL; + } + + ether_addr_copy(dyn_port->hw_addr, hw_addr); + + return 0; +} + +/** + * ice_devlink_port_fn_hw_addr_get - devlink handler for mac address get + * @port: pointer to devlink port + * @hw_addr: hw address to set + * @hw_addr_len: hw address length + * @extack: extack for reporting error messages + * + * Returns mac address for the port. + * + * Return: zero on success or an error code on failure. + */ +static int +ice_devlink_port_fn_hw_addr_get(struct devlink_port *port, u8 *hw_addr, + int *hw_addr_len, + struct netlink_ext_ack *extack) +{ + struct ice_dynamic_port *dyn_port; + + dyn_port = ice_devlink_port_to_dyn(port); + + ether_addr_copy(hw_addr, dyn_port->hw_addr); + *hw_addr_len = ETH_ALEN; + + return 0; +} + +/** + * ice_devlink_port_fn_state_set - devlink handler for port state set + * @port: pointer to devlink port + * @state: state to set + * @extack: extack for reporting error messages + * + * Activates or deactivates the port. + * + * Return: zero on success or an error code on failure. + */ +static int +ice_devlink_port_fn_state_set(struct devlink_port *port, + enum devlink_port_fn_state state, + struct netlink_ext_ack *extack) +{ + struct ice_dynamic_port *dyn_port; + + dyn_port = ice_devlink_port_to_dyn(port); + + switch (state) { + case DEVLINK_PORT_FN_STATE_ACTIVE: + return ice_activate_dynamic_port(dyn_port, extack); + + case DEVLINK_PORT_FN_STATE_INACTIVE: + ice_deactivate_dynamic_port(dyn_port); + break; + } + + return 0; +} + +/** + * ice_devlink_port_fn_state_get - devlink handler for port state get + * @port: pointer to devlink port + * @state: admin configured state of the port + * @opstate: current port operational state + * @extack: extack for reporting error messages + * + * Gets port state. + * + * Return: zero on success or an error code on failure. + */ +static int +ice_devlink_port_fn_state_get(struct devlink_port *port, + enum devlink_port_fn_state *state, + enum devlink_port_fn_opstate *opstate, + struct netlink_ext_ack *extack) +{ + struct ice_dynamic_port *dyn_port; + + dyn_port = ice_devlink_port_to_dyn(port); + + if (dyn_port->active) + *state = DEVLINK_PORT_FN_STATE_ACTIVE; + else + *state = DEVLINK_PORT_FN_STATE_INACTIVE; + + if (dyn_port->attached) + *opstate = DEVLINK_PORT_FN_OPSTATE_ATTACHED; + else + *opstate = DEVLINK_PORT_FN_OPSTATE_DETACHED; + + return 0; +} + +static const struct devlink_port_ops ice_devlink_port_sf_ops = { + .port_del = ice_devlink_port_del, + .port_fn_hw_addr_get = ice_devlink_port_fn_hw_addr_get, + .port_fn_hw_addr_set = ice_devlink_port_fn_hw_addr_set, + .port_fn_state_get = ice_devlink_port_fn_state_get, + .port_fn_state_set = ice_devlink_port_fn_state_set, +}; + +/** + * ice_reserve_sf_num - Reserve a subfunction number for this port + * @pf: pointer to the pf structure + * @new_attr: devlink port attributes requested + * @extack: extack for reporting error messages + * @sfnum: on success, the sf number reserved + * + * Reserve a subfunction number for this port. Only called for + * DEVLINK_PORT_FLAVOUR_PCI_SF ports. + * + * Return: zero on success or an error code on failure. + */ +static int +ice_reserve_sf_num(struct ice_pf *pf, + const struct devlink_port_new_attrs *new_attr, + struct netlink_ext_ack *extack, u32 *sfnum) +{ + int err; + + /* If user didn't request an explicit number, pick one */ + if (!new_attr->sfnum_valid) + return xa_alloc(&pf->sf_nums, sfnum, NULL, xa_limit_32b, + GFP_KERNEL); + + /* Otherwise, check and use the number provided */ + err = xa_insert(&pf->sf_nums, new_attr->sfnum, NULL, GFP_KERNEL); + if (err) { + if (err == -EBUSY) + NL_SET_ERR_MSG_MOD(extack, "Subfunction with given sfnum already exists"); + return err; + } + + *sfnum = new_attr->sfnum; + + return 0; +} + +/** + * ice_devlink_create_sf_port - Register PCI subfunction devlink port + * @dyn_port: the dynamic port instance structure for this subfunction + * + * Register PCI subfunction flavour devlink port for a dynamically added + * subfunction port. + * + * Return: zero on success or an error code on failure. + */ +int ice_devlink_create_sf_port(struct ice_dynamic_port *dyn_port) +{ + struct devlink_port_attrs attrs = {}; + struct devlink_port *devlink_port; + struct devlink *devlink; + struct ice_vsi *vsi; + struct ice_pf *pf; + + vsi = dyn_port->vsi; + pf = dyn_port->pf; + + devlink_port = &dyn_port->devlink_port; + + attrs.flavour = DEVLINK_PORT_FLAVOUR_PCI_SF; + attrs.pci_sf.pf = pf->hw.pf_id; + attrs.pci_sf.sf = dyn_port->sfnum; + + devlink_port_attrs_set(devlink_port, &attrs); + devlink = priv_to_devlink(pf); + + return devl_port_register_with_ops(devlink, devlink_port, vsi->idx, + &ice_devlink_port_sf_ops); +} + +/** + * ice_devlink_destroy_sf_port - Destroy the devlink_port for this SF + * @dyn_port: the dynamic port instance structure for this subfunction + * + * Unregisters the devlink_port structure associated with this SF. + */ +void ice_devlink_destroy_sf_port(struct ice_dynamic_port *dyn_port) +{ + devl_rate_leaf_destroy(&dyn_port->devlink_port); + devl_port_unregister(&dyn_port->devlink_port); +} + +/** + * ice_alloc_dynamic_port - Allocate new dynamic port + * @pf: pointer to the pf structure + * @new_attr: devlink port attributes requested + * @extack: extack for reporting error messages + * @devlink_port: index of newly created devlink port + * + * Allocate a new dynamic port instance and prepare it for configuration + * with devlink. + * + * Return: zero on success or an error code on failure. + */ +static int +ice_alloc_dynamic_port(struct ice_pf *pf, + const struct devlink_port_new_attrs *new_attr, + struct netlink_ext_ack *extack, + struct devlink_port **devlink_port) +{ + struct ice_dynamic_port *dyn_port; + struct ice_vsi *vsi; + u32 sfnum; + int err; + + err = ice_reserve_sf_num(pf, new_attr, extack, &sfnum); + if (err) + return err; + + dyn_port = kzalloc(sizeof(*dyn_port), GFP_KERNEL); + if (!dyn_port) { + err = -ENOMEM; + goto unroll_reserve_sf_num; + } + + vsi = ice_vsi_alloc(pf); + if (!vsi) { + NL_SET_ERR_MSG_MOD(extack, "Unable to allocate VSI"); + err = -ENOMEM; + goto unroll_dyn_port_alloc; + } + + dyn_port->vsi = vsi; + dyn_port->pf = pf; + dyn_port->sfnum = sfnum; + eth_random_addr(dyn_port->hw_addr); + + err = xa_insert(&pf->dyn_ports, vsi->idx, dyn_port, GFP_KERNEL); + if (err) { + NL_SET_ERR_MSG_MOD(extack, "Port index reservation failed"); + goto unroll_vsi_alloc; + } + + err = ice_eswitch_attach_sf(pf, dyn_port); + if (err) { + NL_SET_ERR_MSG_MOD(extack, "Failed to attach SF to eswitch"); + goto unroll_xa_insert; + } + + *devlink_port = &dyn_port->devlink_port; + + return 0; + +unroll_xa_insert: + xa_erase(&pf->dyn_ports, vsi->idx); +unroll_vsi_alloc: + ice_vsi_free(vsi); +unroll_dyn_port_alloc: + kfree(dyn_port); +unroll_reserve_sf_num: + xa_erase(&pf->sf_nums, sfnum); + + return err; +} + +/** + * ice_devlink_port_new - devlink handler for the new port + * @devlink: pointer to devlink + * @new_attr: pointer to the port new attributes + * @extack: extack for reporting error messages + * @devlink_port: pointer to a new port + * + * Creates new devlink port, checks new port attributes and reject + * any unsupported parameters, allocates new subfunction for that port. + * + * Return: zero on success or an error code on failure. + */ +int +ice_devlink_port_new(struct devlink *devlink, + const struct devlink_port_new_attrs *new_attr, + struct netlink_ext_ack *extack, + struct devlink_port **devlink_port) +{ + struct ice_pf *pf = devlink_priv(devlink); + int err; + + err = ice_devlink_port_new_check_attr(pf, new_attr, extack); + if (err) + return err; + + return ice_alloc_dynamic_port(pf, new_attr, extack, devlink_port); +} diff --git a/drivers/net/ethernet/intel/ice/devlink/devlink_port.h b/drivers/net/ethernet/intel/ice/devlink/devlink_port.h index 9223bcdb6444..d60efc340945 100644 --- a/drivers/net/ethernet/intel/ice/devlink/devlink_port.h +++ b/drivers/net/ethernet/intel/ice/devlink/devlink_port.h @@ -4,9 +4,55 @@ #ifndef _DEVLINK_PORT_H_ #define _DEVLINK_PORT_H_ +#include "../ice.h" +#include "../ice_sf_eth.h" + +/** + * struct ice_dynamic_port - Track dynamically added devlink port instance + * @hw_addr: the HW address for this port + * @active: true if the port has been activated + * @attached: true it the prot is attached + * @devlink_port: the associated devlink port structure + * @pf: pointer to the PF private structure + * @vsi: the VSI associated with this port + * @repr_id: the representor ID + * @sfnum: the subfunction ID + * @sf_dev: pointer to the subfunction device + * + * An instance of a dynamically added devlink port. Each port flavour + */ +struct ice_dynamic_port { + u8 hw_addr[ETH_ALEN]; + u8 active: 1; + u8 attached: 1; + struct devlink_port devlink_port; + struct ice_pf *pf; + struct ice_vsi *vsi; + unsigned long repr_id; + u32 sfnum; + /* Flavour-specific implementation data */ + union { + struct ice_sf_dev *sf_dev; + }; +}; + +void ice_dealloc_all_dynamic_ports(struct ice_pf *pf); + int ice_devlink_create_pf_port(struct ice_pf *pf); void ice_devlink_destroy_pf_port(struct ice_pf *pf); int ice_devlink_create_vf_port(struct ice_vf *vf); void ice_devlink_destroy_vf_port(struct ice_vf *vf); +int ice_devlink_create_sf_port(struct ice_dynamic_port *dyn_port); +void ice_devlink_destroy_sf_port(struct ice_dynamic_port *dyn_port); +int ice_devlink_create_sf_dev_port(struct ice_sf_dev *sf_dev); +void ice_devlink_destroy_sf_dev_port(struct ice_sf_dev *sf_dev); + +#define ice_devlink_port_to_dyn(port) \ + container_of(port, struct ice_dynamic_port, devlink_port) +int +ice_devlink_port_new(struct devlink *devlink, + const struct devlink_port_new_attrs *new_attr, + struct netlink_ext_ack *extack, + struct devlink_port **devlink_port); #endif /* _DEVLINK_PORT_H_ */ diff --git a/drivers/net/ethernet/intel/ice/ice.h b/drivers/net/ethernet/intel/ice/ice.h index caaa10157909..d6f80da30dec 100644 --- a/drivers/net/ethernet/intel/ice/ice.h +++ b/drivers/net/ethernet/intel/ice/ice.h @@ -318,6 +318,7 @@ enum ice_vsi_state { ICE_VSI_UMAC_FLTR_CHANGED, ICE_VSI_MMAC_FLTR_CHANGED, ICE_VSI_PROMISC_CHANGED, + ICE_VSI_REBUILD_PENDING, ICE_VSI_STATE_NBITS /* must be last */ }; @@ -411,6 +412,7 @@ struct ice_vsi { struct ice_tx_ring **xdp_rings; /* XDP ring array */ u16 num_xdp_txq; /* Used XDP queues */ u8 xdp_mapping_mode; /* ICE_MAP_MODE_[CONTIG|SCATTER] */ + struct mutex xdp_state_lock; struct net_device **target_netdevs; @@ -449,7 +451,12 @@ struct ice_vsi { struct_group_tagged(ice_vsi_cfg_params, params, struct ice_port_info *port_info; /* back pointer to port_info */ struct ice_channel *ch; /* VSI's channel structure, may be NULL */ - struct ice_vf *vf; /* VF associated with this VSI, may be NULL */ + union { + /* VF associated with this VSI, may be NULL */ + struct ice_vf *vf; + /* SF associated with this VSI, may be NULL */ + struct ice_dynamic_port *sf; + }; u32 flags; /* VSI flags used for rebuild and configuration */ enum ice_vsi_type type; /* the type of the VSI */ ); @@ -650,6 +657,9 @@ struct ice_pf { struct ice_eswitch eswitch; struct ice_esw_br_port *br_port; + struct xarray dyn_ports; + struct xarray sf_nums; + #define ICE_INVALID_AGG_NODE_ID 0 #define ICE_PF_AGG_NODE_ID_START 1 #define ICE_MAX_PF_AGG_NODES 32 @@ -916,6 +926,7 @@ int ice_vsi_open(struct ice_vsi *vsi); void ice_set_ethtool_ops(struct net_device *netdev); void ice_set_ethtool_repr_ops(struct net_device *netdev); void ice_set_ethtool_safe_mode_ops(struct net_device *netdev); +void ice_set_ethtool_sf_ops(struct net_device *netdev); u16 ice_get_avail_txq_count(struct ice_pf *pf); u16 ice_get_avail_rxq_count(struct ice_pf *pf); int ice_vsi_recfg_qs(struct ice_vsi *vsi, int new_rx, int new_tx, bool locked); @@ -1001,6 +1012,14 @@ void ice_unload(struct ice_pf *pf); void ice_adv_lnk_speed_maps_init(void); int ice_init_dev(struct ice_pf *pf); void ice_deinit_dev(struct ice_pf *pf); +int ice_change_mtu(struct net_device *netdev, int new_mtu); +void ice_tx_timeout(struct net_device *netdev, unsigned int txqueue); +int ice_xdp(struct net_device *dev, struct netdev_bpf *xdp); +void ice_set_netdev_features(struct net_device *netdev); +int ice_vlan_rx_add_vid(struct net_device *netdev, __be16 proto, u16 vid); +int ice_vlan_rx_kill_vid(struct net_device *netdev, __be16 proto, u16 vid); +void ice_get_stats64(struct net_device *netdev, + struct rtnl_link_stats64 *stats); /** * ice_set_rdma_cap - enable RDMA support diff --git a/drivers/net/ethernet/intel/ice/ice_adminq_cmd.h b/drivers/net/ethernet/intel/ice/ice_adminq_cmd.h index 66f02988d549..0be1a98d7cc1 100644 --- a/drivers/net/ethernet/intel/ice/ice_adminq_cmd.h +++ b/drivers/net/ethernet/intel/ice/ice_adminq_cmd.h @@ -2632,12 +2632,16 @@ struct ice_aq_desc { /* FW defined boundary for a large buffer, 4k >= Large buffer > 512 bytes */ #define ICE_AQ_LG_BUF 512 +#define ICE_AQ_FLAG_DD_S 0 +#define ICE_AQ_FLAG_CMP_S 1 #define ICE_AQ_FLAG_ERR_S 2 #define ICE_AQ_FLAG_LB_S 9 #define ICE_AQ_FLAG_RD_S 10 #define ICE_AQ_FLAG_BUF_S 12 #define ICE_AQ_FLAG_SI_S 13 +#define ICE_AQ_FLAG_DD BIT(ICE_AQ_FLAG_DD_S) /* 0x1 */ +#define ICE_AQ_FLAG_CMP BIT(ICE_AQ_FLAG_CMP_S) /* 0x2 */ #define ICE_AQ_FLAG_ERR BIT(ICE_AQ_FLAG_ERR_S) /* 0x4 */ #define ICE_AQ_FLAG_LB BIT(ICE_AQ_FLAG_LB_S) /* 0x200 */ #define ICE_AQ_FLAG_RD BIT(ICE_AQ_FLAG_RD_S) /* 0x400 */ diff --git a/drivers/net/ethernet/intel/ice/ice_base.c b/drivers/net/ethernet/intel/ice/ice_base.c index f448d3a84564..4a9a6899fc45 100644 --- a/drivers/net/ethernet/intel/ice/ice_base.c +++ b/drivers/net/ethernet/intel/ice/ice_base.c @@ -190,16 +190,11 @@ static void ice_free_q_vector(struct ice_vsi *vsi, int v_idx) } q_vector = vsi->q_vectors[v_idx]; - ice_for_each_tx_ring(tx_ring, q_vector->tx) { - ice_queue_set_napi(vsi, tx_ring->q_index, NETDEV_QUEUE_TYPE_TX, - NULL); + ice_for_each_tx_ring(tx_ring, vsi->q_vectors[v_idx]->tx) tx_ring->q_vector = NULL; - } - ice_for_each_rx_ring(rx_ring, q_vector->rx) { - ice_queue_set_napi(vsi, rx_ring->q_index, NETDEV_QUEUE_TYPE_RX, - NULL); + + ice_for_each_rx_ring(rx_ring, vsi->q_vectors[v_idx]->rx) rx_ring->q_vector = NULL; - } /* only VSI with an associated netdev is set up with NAPI */ if (vsi->netdev) @@ -330,6 +325,9 @@ ice_setup_tx_ctx(struct ice_tx_ring *ring, struct ice_tlan_ctx *tlan_ctx, u16 pf tlan_ctx->vmvf_num = hw->func_caps.vf_base_id + vsi->vf->vf_id; tlan_ctx->vmvf_type = ICE_TLAN_CTX_VMVF_TYPE_VF; break; + case ICE_VSI_SF: + tlan_ctx->vmvf_type = ICE_TLAN_CTX_VMVF_TYPE_VMQ; + break; default: return; } @@ -545,7 +543,7 @@ static int ice_vsi_cfg_rxq(struct ice_rx_ring *ring) ring->rx_buf_len = ring->vsi->rx_buf_len; - if (ring->vsi->type == ICE_VSI_PF) { + if (ring->vsi->type == ICE_VSI_PF || ring->vsi->type == ICE_VSI_SF) { if (!xdp_rxq_info_is_reg(&ring->xdp_rxq)) { err = __xdp_rxq_info_reg(&ring->xdp_rxq, ring->netdev, ring->q_index, diff --git a/drivers/net/ethernet/intel/ice/ice_common.h b/drivers/net/ethernet/intel/ice/ice_common.h index 66f29bac783a..27208a60cece 100644 --- a/drivers/net/ethernet/intel/ice/ice_common.h +++ b/drivers/net/ethernet/intel/ice/ice_common.h @@ -10,6 +10,7 @@ #include "ice_type.h" #include "ice_nvm.h" #include "ice_flex_pipe.h" +#include "ice_parser.h" #include <linux/avf/virtchnl.h> #include "ice_switch.h" #include "ice_fdir.h" diff --git a/drivers/net/ethernet/intel/ice/ice_controlq.c b/drivers/net/ethernet/intel/ice/ice_controlq.c index ffaa6511c455..e3959ad442a2 100644 --- a/drivers/net/ethernet/intel/ice/ice_controlq.c +++ b/drivers/net/ethernet/intel/ice/ice_controlq.c @@ -99,17 +99,6 @@ ice_alloc_ctrlq_sq_ring(struct ice_hw *hw, struct ice_ctl_q_info *cq) return -ENOMEM; cq->sq.desc_buf.size = size; - cq->sq.cmd_buf = devm_kcalloc(ice_hw_to_dev(hw), cq->num_sq_entries, - sizeof(struct ice_sq_cd), GFP_KERNEL); - if (!cq->sq.cmd_buf) { - dmam_free_coherent(ice_hw_to_dev(hw), cq->sq.desc_buf.size, - cq->sq.desc_buf.va, cq->sq.desc_buf.pa); - cq->sq.desc_buf.va = NULL; - cq->sq.desc_buf.pa = 0; - cq->sq.desc_buf.size = 0; - return -ENOMEM; - } - return 0; } @@ -188,7 +177,7 @@ ice_alloc_rq_bufs(struct ice_hw *hw, struct ice_ctl_q_info *cq) if (cq->rq_buf_size > ICE_AQ_LG_BUF) desc->flags |= cpu_to_le16(ICE_AQ_FLAG_LB); desc->opcode = 0; - /* This is in accordance with Admin queue design, there is no + /* This is in accordance with control queue design, there is no * register for buffer size configuration */ desc->datalen = cpu_to_le16(bi->size); @@ -338,8 +327,6 @@ do { \ (qi)->ring.r.ring##_bi[i].size = 0;\ } \ } \ - /* free the buffer info list */ \ - devm_kfree(ice_hw_to_dev(hw), (qi)->ring.cmd_buf); \ /* free DMA head */ \ devm_kfree(ice_hw_to_dev(hw), (qi)->ring.dma_head); \ } while (0) @@ -405,11 +392,11 @@ init_ctrlq_exit: } /** - * ice_init_rq - initialize ARQ + * ice_init_rq - initialize receive side of a control queue * @hw: pointer to the hardware structure * @cq: pointer to the specific Control queue * - * The main initialization routine for the Admin Receive (Event) Queue. + * The main initialization routine for Receive side of a control queue. * Prior to calling this function, the driver *MUST* set the following fields * in the cq->structure: * - cq->num_rq_entries @@ -465,7 +452,7 @@ init_ctrlq_exit: } /** - * ice_shutdown_sq - shutdown the Control ATQ + * ice_shutdown_sq - shutdown the transmit side of a control queue * @hw: pointer to the hardware structure * @cq: pointer to the specific Control queue * @@ -482,7 +469,7 @@ static int ice_shutdown_sq(struct ice_hw *hw, struct ice_ctl_q_info *cq) goto shutdown_sq_out; } - /* Stop firmware AdminQ processing */ + /* Stop processing of the control queue */ wr32(hw, cq->sq.head, 0); wr32(hw, cq->sq.tail, 0); wr32(hw, cq->sq.len, 0); @@ -501,7 +488,7 @@ shutdown_sq_out: } /** - * ice_aq_ver_check - Check the reported AQ API version. + * ice_aq_ver_check - Check the reported AQ API version * @hw: pointer to the hardware structure * * Checks if the driver should load on a given AQ API version. @@ -521,14 +508,20 @@ static bool ice_aq_ver_check(struct ice_hw *hw) } else if (hw->api_maj_ver == exp_fw_api_ver_major) { if (hw->api_min_ver > (exp_fw_api_ver_minor + 2)) dev_info(ice_hw_to_dev(hw), - "The driver for the device detected a newer version of the NVM image than expected. Please install the most recent version of the network driver.\n"); + "The driver for the device detected a newer version (%u.%u) of the NVM image than expected (%u.%u). Please install the most recent version of the network driver.\n", + hw->api_maj_ver, hw->api_min_ver, + exp_fw_api_ver_major, exp_fw_api_ver_minor); else if ((hw->api_min_ver + 2) < exp_fw_api_ver_minor) dev_info(ice_hw_to_dev(hw), - "The driver for the device detected an older version of the NVM image than expected. Please update the NVM image.\n"); + "The driver for the device detected an older version (%u.%u) of the NVM image than expected (%u.%u). Please update the NVM image.\n", + hw->api_maj_ver, hw->api_min_ver, + exp_fw_api_ver_major, exp_fw_api_ver_minor); } else { /* Major API version is older than expected, log a warning */ dev_info(ice_hw_to_dev(hw), - "The driver for the device detected an older version of the NVM image than expected. Please update the NVM image.\n"); + "The driver for the device detected an older version (%u.%u) of the NVM image than expected (%u.%u). Please update the NVM image.\n", + hw->api_maj_ver, hw->api_min_ver, + exp_fw_api_ver_major, exp_fw_api_ver_minor); } return true; } @@ -855,7 +848,7 @@ void ice_destroy_all_ctrlq(struct ice_hw *hw) } /** - * ice_clean_sq - cleans Admin send queue (ATQ) + * ice_clean_sq - cleans send side of a control queue * @hw: pointer to the hardware structure * @cq: pointer to the specific Control queue * @@ -865,21 +858,17 @@ static u16 ice_clean_sq(struct ice_hw *hw, struct ice_ctl_q_info *cq) { struct ice_ctl_q_ring *sq = &cq->sq; u16 ntc = sq->next_to_clean; - struct ice_sq_cd *details; struct ice_aq_desc *desc; desc = ICE_CTL_Q_DESC(*sq, ntc); - details = ICE_CTL_Q_DETAILS(*sq, ntc); while (rd32(hw, cq->sq.head) != ntc) { ice_debug(hw, ICE_DBG_AQ_MSG, "ntc %d head %d.\n", ntc, rd32(hw, cq->sq.head)); memset(desc, 0, sizeof(*desc)); - memset(details, 0, sizeof(*details)); ntc++; if (ntc == sq->count) ntc = 0; desc = ICE_CTL_Q_DESC(*sq, ntc); - details = ICE_CTL_Q_DETAILS(*sq, ntc); } sq->next_to_clean = ntc; @@ -888,18 +877,43 @@ static u16 ice_clean_sq(struct ice_hw *hw, struct ice_ctl_q_info *cq) } /** + * ice_ctl_q_str - Convert control queue type to string + * @qtype: the control queue type + * + * Return: A string name for the given control queue type. + */ +static const char *ice_ctl_q_str(enum ice_ctl_q qtype) +{ + switch (qtype) { + case ICE_CTL_Q_UNKNOWN: + return "Unknown CQ"; + case ICE_CTL_Q_ADMIN: + return "AQ"; + case ICE_CTL_Q_MAILBOX: + return "MBXQ"; + case ICE_CTL_Q_SB: + return "SBQ"; + default: + return "Unrecognized CQ"; + } +} + +/** * ice_debug_cq * @hw: pointer to the hardware structure + * @cq: pointer to the specific Control queue * @desc: pointer to control queue descriptor * @buf: pointer to command buffer * @buf_len: max length of buf + * @response: true if this is the writeback response * * Dumps debug log about control command with descriptor contents. */ -static void ice_debug_cq(struct ice_hw *hw, void *desc, void *buf, u16 buf_len) +static void ice_debug_cq(struct ice_hw *hw, struct ice_ctl_q_info *cq, + void *desc, void *buf, u16 buf_len, bool response) { struct ice_aq_desc *cq_desc = desc; - u16 len; + u16 datalen, flags; if (!IS_ENABLED(CONFIG_DYNAMIC_DEBUG) && !((ICE_DBG_AQ_DESC | ICE_DBG_AQ_DESC_BUF) & hw->debug_mask)) @@ -908,48 +922,63 @@ static void ice_debug_cq(struct ice_hw *hw, void *desc, void *buf, u16 buf_len) if (!desc) return; - len = le16_to_cpu(cq_desc->datalen); + datalen = le16_to_cpu(cq_desc->datalen); + flags = le16_to_cpu(cq_desc->flags); - ice_debug(hw, ICE_DBG_AQ_DESC, "CQ CMD: opcode 0x%04X, flags 0x%04X, datalen 0x%04X, retval 0x%04X\n", - le16_to_cpu(cq_desc->opcode), - le16_to_cpu(cq_desc->flags), - le16_to_cpu(cq_desc->datalen), le16_to_cpu(cq_desc->retval)); - ice_debug(hw, ICE_DBG_AQ_DESC, "\tcookie (h,l) 0x%08X 0x%08X\n", + ice_debug(hw, ICE_DBG_AQ_DESC, "%s %s: opcode 0x%04X, flags 0x%04X, datalen 0x%04X, retval 0x%04X\n\tcookie (h,l) 0x%08X 0x%08X\n\tparam (0,1) 0x%08X 0x%08X\n\taddr (h,l) 0x%08X 0x%08X\n", + ice_ctl_q_str(cq->qtype), response ? "Response" : "Command", + le16_to_cpu(cq_desc->opcode), flags, datalen, + le16_to_cpu(cq_desc->retval), le32_to_cpu(cq_desc->cookie_high), - le32_to_cpu(cq_desc->cookie_low)); - ice_debug(hw, ICE_DBG_AQ_DESC, "\tparam (0,1) 0x%08X 0x%08X\n", + le32_to_cpu(cq_desc->cookie_low), le32_to_cpu(cq_desc->params.generic.param0), - le32_to_cpu(cq_desc->params.generic.param1)); - ice_debug(hw, ICE_DBG_AQ_DESC, "\taddr (h,l) 0x%08X 0x%08X\n", + le32_to_cpu(cq_desc->params.generic.param1), le32_to_cpu(cq_desc->params.generic.addr_high), le32_to_cpu(cq_desc->params.generic.addr_low)); - if (buf && cq_desc->datalen != 0) { - ice_debug(hw, ICE_DBG_AQ_DESC_BUF, "Buffer:\n"); - if (buf_len < len) - len = buf_len; - - ice_debug_array(hw, ICE_DBG_AQ_DESC_BUF, 16, 1, buf, len); + /* Dump buffer iff 1) one exists and 2) is either a response indicated + * by the DD and/or CMP flag set or a command with the RD flag set. + */ + if (buf && cq_desc->datalen && + (flags & (ICE_AQ_FLAG_DD | ICE_AQ_FLAG_CMP | ICE_AQ_FLAG_RD))) { + char prefix[] = KBUILD_MODNAME " 0x12341234 0x12341234 "; + + sprintf(prefix, KBUILD_MODNAME " 0x%08X 0x%08X ", + le32_to_cpu(cq_desc->params.generic.addr_high), + le32_to_cpu(cq_desc->params.generic.addr_low)); + ice_debug_array_w_prefix(hw, ICE_DBG_AQ_DESC_BUF, prefix, + buf, + min_t(u16, buf_len, datalen)); } } /** - * ice_sq_done - check if FW has processed the Admin Send Queue (ATQ) + * ice_sq_done - poll until the last send on a control queue has completed * @hw: pointer to the HW struct * @cq: pointer to the specific Control queue * - * Returns true if the firmware has processed all descriptors on the - * admin send queue. Returns false if there are still requests pending. + * Use read_poll_timeout to poll the control queue head, checking until it + * matches next_to_use. According to the control queue designers, this has + * better timing reliability than the DD bit. + * + * Return: true if all the descriptors on the send side of a control queue + * are finished processing, false otherwise. */ static bool ice_sq_done(struct ice_hw *hw, struct ice_ctl_q_info *cq) { - /* AQ designers suggest use of head for better - * timing reliability than DD bit + u32 head; + + /* Wait a short time before the initial check, to allow hardware time + * for completion. */ - return rd32(hw, cq->sq.head) == cq->sq.next_to_use; + udelay(5); + + return !rd32_poll_timeout(hw, cq->sq.head, + head, head == cq->sq.next_to_use, + 20, ICE_CTL_Q_SQ_CMD_TIMEOUT); } /** - * ice_sq_send_cmd - send command to Control Queue (ATQ) + * ice_sq_send_cmd - send command to a control queue * @hw: pointer to the HW struct * @cq: pointer to the specific Control queue * @desc: prefilled descriptor describing the command @@ -957,8 +986,9 @@ static bool ice_sq_done(struct ice_hw *hw, struct ice_ctl_q_info *cq) * @buf_size: size of buffer for indirect commands (or 0 for direct commands) * @cd: pointer to command details structure * - * This is the main send command routine for the ATQ. It runs the queue, - * cleans the queue, etc. + * Main command for the transmit side of a control queue. It puts the command + * on the queue, bumps the tail, waits for processing of the command, captures + * command status and results, etc. */ int ice_sq_send_cmd(struct ice_hw *hw, struct ice_ctl_q_info *cq, @@ -968,8 +998,6 @@ ice_sq_send_cmd(struct ice_hw *hw, struct ice_ctl_q_info *cq, struct ice_dma_mem *dma_buf = NULL; struct ice_aq_desc *desc_on_ring; bool cmd_completed = false; - struct ice_sq_cd *details; - unsigned long timeout; int status = 0; u16 retval = 0; u32 val = 0; @@ -1013,12 +1041,6 @@ ice_sq_send_cmd(struct ice_hw *hw, struct ice_ctl_q_info *cq, goto sq_send_command_error; } - details = ICE_CTL_Q_DETAILS(cq->sq, cq->sq.next_to_use); - if (cd) - *details = *cd; - else - memset(details, 0, sizeof(*details)); - /* Call clean and check queue available function to reclaim the * descriptors that were processed by FW/MBX; the function returns the * number of desc available. The clean function called here could be @@ -1055,7 +1077,7 @@ ice_sq_send_cmd(struct ice_hw *hw, struct ice_ctl_q_info *cq, /* Debug desc and buffer */ ice_debug(hw, ICE_DBG_AQ_DESC, "ATQ: Control Send queue desc and buffer:\n"); - ice_debug_cq(hw, (void *)desc_on_ring, buf, buf_size); + ice_debug_cq(hw, cq, (void *)desc_on_ring, buf, buf_size, false); (cq->sq.next_to_use)++; if (cq->sq.next_to_use == cq->sq.count) @@ -1063,20 +1085,9 @@ ice_sq_send_cmd(struct ice_hw *hw, struct ice_ctl_q_info *cq, wr32(hw, cq->sq.tail, cq->sq.next_to_use); ice_flush(hw); - /* Wait a short time before initial ice_sq_done() check, to allow - * hardware time for completion. + /* Wait for the command to complete. If it finishes within the + * timeout, copy the descriptor back to temp. */ - udelay(5); - - timeout = jiffies + ICE_CTL_Q_SQ_CMD_TIMEOUT; - do { - if (ice_sq_done(hw, cq)) - break; - - usleep_range(100, 150); - } while (time_before(jiffies, timeout)); - - /* if ready, copy the desc back to temp */ if (ice_sq_done(hw, cq)) { memcpy(desc, desc_on_ring, sizeof(*desc)); if (buf) { @@ -1108,12 +1119,11 @@ ice_sq_send_cmd(struct ice_hw *hw, struct ice_ctl_q_info *cq, ice_debug(hw, ICE_DBG_AQ_MSG, "ATQ: desc and buffer writeback:\n"); - ice_debug_cq(hw, (void *)desc, buf, buf_size); + ice_debug_cq(hw, cq, (void *)desc, buf, buf_size, true); /* save writeback AQ if requested */ - if (details->wb_desc) - memcpy(details->wb_desc, desc_on_ring, - sizeof(*details->wb_desc)); + if (cd && cd->wb_desc) + memcpy(cd->wb_desc, desc_on_ring, sizeof(*cd->wb_desc)); /* update the error if time out occurred */ if (!cmd_completed) { @@ -1154,9 +1164,9 @@ void ice_fill_dflt_direct_cmd_desc(struct ice_aq_desc *desc, u16 opcode) * @e: event info from the receive descriptor, includes any buffers * @pending: number of events that could be left to process * - * This function cleans one Admin Receive Queue element and returns - * the contents through e. It can also return how many events are - * left to process through 'pending'. + * Clean one element from the receive side of a control queue. On return 'e' + * contains contents of the message, and 'pending' contains the number of + * events left to process. */ int ice_clean_rq_elem(struct ice_hw *hw, struct ice_ctl_q_info *cq, @@ -1212,7 +1222,7 @@ ice_clean_rq_elem(struct ice_hw *hw, struct ice_ctl_q_info *cq, ice_debug(hw, ICE_DBG_AQ_DESC, "ARQ: desc and buffer:\n"); - ice_debug_cq(hw, (void *)desc, e->msg_buf, cq->rq_buf_size); + ice_debug_cq(hw, cq, (void *)desc, e->msg_buf, cq->rq_buf_size, true); /* Restore the original datalen and buffer address in the desc, * FW updates datalen to indicate the event message size diff --git a/drivers/net/ethernet/intel/ice/ice_controlq.h b/drivers/net/ethernet/intel/ice/ice_controlq.h index 1d54b1cdb1c5..ca97b7365a1b 100644 --- a/drivers/net/ethernet/intel/ice/ice_controlq.h +++ b/drivers/net/ethernet/intel/ice/ice_controlq.h @@ -43,14 +43,13 @@ enum ice_ctl_q { }; /* Control Queue timeout settings - max delay 1s */ -#define ICE_CTL_Q_SQ_CMD_TIMEOUT HZ /* Wait max 1s */ +#define ICE_CTL_Q_SQ_CMD_TIMEOUT USEC_PER_SEC #define ICE_CTL_Q_ADMIN_INIT_TIMEOUT 10 /* Count 10 times */ #define ICE_CTL_Q_ADMIN_INIT_MSEC 100 /* Check every 100msec */ struct ice_ctl_q_ring { void *dma_head; /* Virtual address to DMA head */ struct ice_dma_mem desc_buf; /* descriptor ring memory */ - void *cmd_buf; /* command buffer memory */ union { struct ice_dma_mem *sq_bi; @@ -80,8 +79,6 @@ struct ice_sq_cd { struct ice_aq_desc *wb_desc; }; -#define ICE_CTL_Q_DETAILS(R, i) (&(((struct ice_sq_cd *)((R).cmd_buf))[i])) - /* rq event information */ struct ice_rq_event_info { struct ice_aq_desc desc; diff --git a/drivers/net/ethernet/intel/ice/ice_dcb_lib.c b/drivers/net/ethernet/intel/ice/ice_dcb_lib.c index a94e7072b570..a7c510832824 100644 --- a/drivers/net/ethernet/intel/ice/ice_dcb_lib.c +++ b/drivers/net/ethernet/intel/ice/ice_dcb_lib.c @@ -187,6 +187,7 @@ void ice_vsi_set_dcb_tc_cfg(struct ice_vsi *vsi) vsi->tc_cfg.numtc = ice_dcb_get_num_tc(cfg); break; case ICE_VSI_CHNL: + case ICE_VSI_SF: vsi->tc_cfg.ena_tc = BIT(ice_get_first_droptc(vsi)); vsi->tc_cfg.numtc = 1; break; diff --git a/drivers/net/ethernet/intel/ice/ice_ddp.c b/drivers/net/ethernet/intel/ice/ice_ddp.c index f182179529b7..953262b88a58 100644 --- a/drivers/net/ethernet/intel/ice/ice_ddp.c +++ b/drivers/net/ethernet/intel/ice/ice_ddp.c @@ -289,11 +289,11 @@ void *ice_pkg_enum_section(struct ice_seg *ice_seg, struct ice_pkg_enum *state, * indicates a base offset of 10, and the index for the entry is 2, then * section handler function should set the offset to 10 + 2 = 12. */ -static void *ice_pkg_enum_entry(struct ice_seg *ice_seg, - struct ice_pkg_enum *state, u32 sect_type, - u32 *offset, - void *(*handler)(u32 sect_type, void *section, - u32 index, u32 *offset)) +void *ice_pkg_enum_entry(struct ice_seg *ice_seg, + struct ice_pkg_enum *state, u32 sect_type, + u32 *offset, + void *(*handler)(u32 sect_type, void *section, + u32 index, u32 *offset)) { void *entry; diff --git a/drivers/net/ethernet/intel/ice/ice_ddp.h b/drivers/net/ethernet/intel/ice/ice_ddp.h index 622543f08b43..97f272317475 100644 --- a/drivers/net/ethernet/intel/ice/ice_ddp.h +++ b/drivers/net/ethernet/intel/ice/ice_ddp.h @@ -261,10 +261,17 @@ struct ice_meta_sect { #define ICE_SID_CDID_KEY_BUILDER_RSS 47 #define ICE_SID_CDID_REDIR_RSS 48 +#define ICE_SID_RXPARSER_CAM 50 +#define ICE_SID_RXPARSER_NOMATCH_CAM 51 +#define ICE_SID_RXPARSER_IMEM 52 #define ICE_SID_RXPARSER_MARKER_PTYPE 55 #define ICE_SID_RXPARSER_BOOST_TCAM 56 +#define ICE_SID_RXPARSER_PROTO_GRP 57 #define ICE_SID_RXPARSER_METADATA_INIT 58 #define ICE_SID_TXPARSER_BOOST_TCAM 66 +#define ICE_SID_RXPARSER_MARKER_GRP 72 +#define ICE_SID_RXPARSER_PG_SPILL 76 +#define ICE_SID_RXPARSER_NOMATCH_SPILL 78 #define ICE_SID_XLT0_PE 80 #define ICE_SID_XLT_KEY_BUILDER_PE 81 @@ -276,6 +283,7 @@ struct ice_meta_sect { #define ICE_SID_CDID_KEY_BUILDER_PE 87 #define ICE_SID_CDID_REDIR_PE 88 +#define ICE_SID_RXPARSER_FLAG_REDIR 97 /* Label Metadata section IDs */ #define ICE_SID_LBL_FIRST 0x80000010 #define ICE_SID_LBL_RXPARSER_TMEM 0x80000018 @@ -451,6 +459,11 @@ int ice_update_pkg(struct ice_hw *hw, struct ice_buf *bufs, u32 count); int ice_pkg_buf_reserve_section(struct ice_buf_build *bld, u16 count); u16 ice_pkg_buf_get_active_sections(struct ice_buf_build *bld); +void * +ice_pkg_enum_entry(struct ice_seg *ice_seg, struct ice_pkg_enum *state, + u32 sect_type, u32 *offset, + void *(*handler)(u32 sect_type, void *section, + u32 index, u32 *offset)); void *ice_pkg_enum_section(struct ice_seg *ice_seg, struct ice_pkg_enum *state, u32 sect_type); diff --git a/drivers/net/ethernet/intel/ice/ice_dpll.c b/drivers/net/ethernet/intel/ice/ice_dpll.c index e92be6f130a3..cd95705d1e7f 100644 --- a/drivers/net/ethernet/intel/ice/ice_dpll.c +++ b/drivers/net/ethernet/intel/ice/ice_dpll.c @@ -9,6 +9,7 @@ #define ICE_CGU_STATE_ACQ_ERR_THRESHOLD 50 #define ICE_DPLL_PIN_IDX_INVALID 0xff #define ICE_DPLL_RCLK_NUM_PER_PF 1 +#define ICE_DPLL_PIN_ESYNC_PULSE_HIGH_PERCENT 25 /** * enum ice_dpll_pin_type - enumerate ice pin types: @@ -30,6 +31,10 @@ static const char * const pin_type_name[] = { [ICE_DPLL_PIN_TYPE_RCLK_INPUT] = "rclk-input", }; +static const struct dpll_pin_frequency ice_esync_range[] = { + DPLL_PIN_FREQUENCY_RANGE(0, DPLL_PIN_FREQUENCY_1_HZ), +}; + /** * ice_dpll_is_reset - check if reset is in progress * @pf: private board structure @@ -394,8 +399,8 @@ ice_dpll_pin_state_update(struct ice_pf *pf, struct ice_dpll_pin *pin, switch (pin_type) { case ICE_DPLL_PIN_TYPE_INPUT: - ret = ice_aq_get_input_pin_cfg(&pf->hw, pin->idx, NULL, NULL, - NULL, &pin->flags[0], + ret = ice_aq_get_input_pin_cfg(&pf->hw, pin->idx, &pin->status, + NULL, NULL, &pin->flags[0], &pin->freq, &pin->phase_adjust); if (ret) goto err; @@ -430,7 +435,7 @@ ice_dpll_pin_state_update(struct ice_pf *pf, struct ice_dpll_pin *pin, goto err; parent &= ICE_AQC_GET_CGU_OUT_CFG_DPLL_SRC_SEL; - if (ICE_AQC_SET_CGU_OUT_CFG_OUT_EN & pin->flags[0]) { + if (ICE_AQC_GET_CGU_OUT_CFG_OUT_EN & pin->flags[0]) { pin->state[pf->dplls.eec.dpll_idx] = parent == pf->dplls.eec.dpll_idx ? DPLL_PIN_STATE_CONNECTED : @@ -1099,6 +1104,214 @@ ice_dpll_phase_offset_get(const struct dpll_pin *pin, void *pin_priv, } /** + * ice_dpll_output_esync_set - callback for setting embedded sync + * @pin: pointer to a pin + * @pin_priv: private data pointer passed on pin registration + * @dpll: registered dpll pointer + * @dpll_priv: private data pointer passed on dpll registration + * @freq: requested embedded sync frequency + * @extack: error reporting + * + * Dpll subsystem callback. Handler for setting embedded sync frequency value + * on output pin. + * + * Context: Acquires pf->dplls.lock + * Return: + * * 0 - success + * * negative - error + */ +static int +ice_dpll_output_esync_set(const struct dpll_pin *pin, void *pin_priv, + const struct dpll_device *dpll, void *dpll_priv, + u64 freq, struct netlink_ext_ack *extack) +{ + struct ice_dpll_pin *p = pin_priv; + struct ice_dpll *d = dpll_priv; + struct ice_pf *pf = d->pf; + u8 flags = 0; + int ret; + + if (ice_dpll_is_reset(pf, extack)) + return -EBUSY; + mutex_lock(&pf->dplls.lock); + if (p->flags[0] & ICE_AQC_GET_CGU_OUT_CFG_OUT_EN) + flags = ICE_AQC_SET_CGU_OUT_CFG_OUT_EN; + if (freq == DPLL_PIN_FREQUENCY_1_HZ) { + if (p->flags[0] & ICE_AQC_GET_CGU_OUT_CFG_ESYNC_EN) { + ret = 0; + } else { + flags |= ICE_AQC_SET_CGU_OUT_CFG_ESYNC_EN; + ret = ice_aq_set_output_pin_cfg(&pf->hw, p->idx, flags, + 0, 0, 0); + } + } else { + if (!(p->flags[0] & ICE_AQC_GET_CGU_OUT_CFG_ESYNC_EN)) { + ret = 0; + } else { + flags &= ~ICE_AQC_SET_CGU_OUT_CFG_ESYNC_EN; + ret = ice_aq_set_output_pin_cfg(&pf->hw, p->idx, flags, + 0, 0, 0); + } + } + mutex_unlock(&pf->dplls.lock); + + return ret; +} + +/** + * ice_dpll_output_esync_get - callback for getting embedded sync config + * @pin: pointer to a pin + * @pin_priv: private data pointer passed on pin registration + * @dpll: registered dpll pointer + * @dpll_priv: private data pointer passed on dpll registration + * @esync: on success holds embedded sync pin properties + * @extack: error reporting + * + * Dpll subsystem callback. Handler for getting embedded sync frequency value + * and capabilities on output pin. + * + * Context: Acquires pf->dplls.lock + * Return: + * * 0 - success + * * negative - error + */ +static int +ice_dpll_output_esync_get(const struct dpll_pin *pin, void *pin_priv, + const struct dpll_device *dpll, void *dpll_priv, + struct dpll_pin_esync *esync, + struct netlink_ext_ack *extack) +{ + struct ice_dpll_pin *p = pin_priv; + struct ice_dpll *d = dpll_priv; + struct ice_pf *pf = d->pf; + + if (ice_dpll_is_reset(pf, extack)) + return -EBUSY; + mutex_lock(&pf->dplls.lock); + if (!(p->flags[0] & ICE_AQC_GET_CGU_OUT_CFG_ESYNC_ABILITY) || + p->freq != DPLL_PIN_FREQUENCY_10_MHZ) { + mutex_unlock(&pf->dplls.lock); + return -EOPNOTSUPP; + } + esync->range = ice_esync_range; + esync->range_num = ARRAY_SIZE(ice_esync_range); + if (p->flags[0] & ICE_AQC_GET_CGU_OUT_CFG_ESYNC_EN) { + esync->freq = DPLL_PIN_FREQUENCY_1_HZ; + esync->pulse = ICE_DPLL_PIN_ESYNC_PULSE_HIGH_PERCENT; + } else { + esync->freq = 0; + esync->pulse = 0; + } + mutex_unlock(&pf->dplls.lock); + + return 0; +} + +/** + * ice_dpll_input_esync_set - callback for setting embedded sync + * @pin: pointer to a pin + * @pin_priv: private data pointer passed on pin registration + * @dpll: registered dpll pointer + * @dpll_priv: private data pointer passed on dpll registration + * @freq: requested embedded sync frequency + * @extack: error reporting + * + * Dpll subsystem callback. Handler for setting embedded sync frequency value + * on input pin. + * + * Context: Acquires pf->dplls.lock + * Return: + * * 0 - success + * * negative - error + */ +static int +ice_dpll_input_esync_set(const struct dpll_pin *pin, void *pin_priv, + const struct dpll_device *dpll, void *dpll_priv, + u64 freq, struct netlink_ext_ack *extack) +{ + struct ice_dpll_pin *p = pin_priv; + struct ice_dpll *d = dpll_priv; + struct ice_pf *pf = d->pf; + u8 flags_en = 0; + int ret; + + if (ice_dpll_is_reset(pf, extack)) + return -EBUSY; + mutex_lock(&pf->dplls.lock); + if (p->flags[0] & ICE_AQC_GET_CGU_IN_CFG_FLG2_INPUT_EN) + flags_en = ICE_AQC_SET_CGU_IN_CFG_FLG2_INPUT_EN; + if (freq == DPLL_PIN_FREQUENCY_1_HZ) { + if (p->flags[0] & ICE_AQC_GET_CGU_IN_CFG_FLG2_ESYNC_EN) { + ret = 0; + } else { + flags_en |= ICE_AQC_SET_CGU_IN_CFG_FLG2_ESYNC_EN; + ret = ice_aq_set_input_pin_cfg(&pf->hw, p->idx, 0, + flags_en, 0, 0); + } + } else { + if (!(p->flags[0] & ICE_AQC_GET_CGU_IN_CFG_FLG2_ESYNC_EN)) { + ret = 0; + } else { + flags_en &= ~ICE_AQC_SET_CGU_IN_CFG_FLG2_ESYNC_EN; + ret = ice_aq_set_input_pin_cfg(&pf->hw, p->idx, 0, + flags_en, 0, 0); + } + } + mutex_unlock(&pf->dplls.lock); + + return ret; +} + +/** + * ice_dpll_input_esync_get - callback for getting embedded sync config + * @pin: pointer to a pin + * @pin_priv: private data pointer passed on pin registration + * @dpll: registered dpll pointer + * @dpll_priv: private data pointer passed on dpll registration + * @esync: on success holds embedded sync pin properties + * @extack: error reporting + * + * Dpll subsystem callback. Handler for getting embedded sync frequency value + * and capabilities on input pin. + * + * Context: Acquires pf->dplls.lock + * Return: + * * 0 - success + * * negative - error + */ +static int +ice_dpll_input_esync_get(const struct dpll_pin *pin, void *pin_priv, + const struct dpll_device *dpll, void *dpll_priv, + struct dpll_pin_esync *esync, + struct netlink_ext_ack *extack) +{ + struct ice_dpll_pin *p = pin_priv; + struct ice_dpll *d = dpll_priv; + struct ice_pf *pf = d->pf; + + if (ice_dpll_is_reset(pf, extack)) + return -EBUSY; + mutex_lock(&pf->dplls.lock); + if (!(p->status & ICE_AQC_GET_CGU_IN_CFG_STATUS_ESYNC_CAP) || + p->freq != DPLL_PIN_FREQUENCY_10_MHZ) { + mutex_unlock(&pf->dplls.lock); + return -EOPNOTSUPP; + } + esync->range = ice_esync_range; + esync->range_num = ARRAY_SIZE(ice_esync_range); + if (p->flags[0] & ICE_AQC_GET_CGU_IN_CFG_FLG2_ESYNC_EN) { + esync->freq = DPLL_PIN_FREQUENCY_1_HZ; + esync->pulse = ICE_DPLL_PIN_ESYNC_PULSE_HIGH_PERCENT; + } else { + esync->freq = 0; + esync->pulse = 0; + } + mutex_unlock(&pf->dplls.lock); + + return 0; +} + +/** * ice_dpll_rclk_state_on_pin_set - set a state on rclk pin * @pin: pointer to a pin * @pin_priv: private data pointer passed on pin registration @@ -1222,6 +1435,8 @@ static const struct dpll_pin_ops ice_dpll_input_ops = { .phase_adjust_get = ice_dpll_pin_phase_adjust_get, .phase_adjust_set = ice_dpll_input_phase_adjust_set, .phase_offset_get = ice_dpll_phase_offset_get, + .esync_set = ice_dpll_input_esync_set, + .esync_get = ice_dpll_input_esync_get, }; static const struct dpll_pin_ops ice_dpll_output_ops = { @@ -1232,6 +1447,8 @@ static const struct dpll_pin_ops ice_dpll_output_ops = { .direction_get = ice_dpll_output_direction, .phase_adjust_get = ice_dpll_pin_phase_adjust_get, .phase_adjust_set = ice_dpll_output_phase_adjust_set, + .esync_set = ice_dpll_output_esync_set, + .esync_get = ice_dpll_output_esync_get, }; static const struct dpll_device_ops ice_dpll_ops = { diff --git a/drivers/net/ethernet/intel/ice/ice_dpll.h b/drivers/net/ethernet/intel/ice/ice_dpll.h index 93172e93995b..c320f1bf7d6d 100644 --- a/drivers/net/ethernet/intel/ice/ice_dpll.h +++ b/drivers/net/ethernet/intel/ice/ice_dpll.h @@ -31,6 +31,7 @@ struct ice_dpll_pin { struct dpll_pin_properties prop; u32 freq; s32 phase_adjust; + u8 status; }; /** ice_dpll - store info required for DPLL control diff --git a/drivers/net/ethernet/intel/ice/ice_eswitch.c b/drivers/net/ethernet/intel/ice/ice_eswitch.c index 3cfa071e3718..c0b3e70a7ea3 100644 --- a/drivers/net/ethernet/intel/ice/ice_eswitch.c +++ b/drivers/net/ethernet/intel/ice/ice_eswitch.c @@ -452,11 +452,9 @@ static void ice_eswitch_start_reprs(struct ice_pf *pf) ice_eswitch_start_all_tx_queues(pf); } -int -ice_eswitch_attach(struct ice_pf *pf, struct ice_vf *vf) +static int +ice_eswitch_attach(struct ice_pf *pf, struct ice_repr *repr, unsigned long *id) { - struct devlink *devlink = priv_to_devlink(pf); - struct ice_repr *repr; int err; if (pf->eswitch_mode == DEVLINK_ESWITCH_MODE_LEGACY) @@ -470,13 +468,9 @@ ice_eswitch_attach(struct ice_pf *pf, struct ice_vf *vf) ice_eswitch_stop_reprs(pf); - devl_lock(devlink); - repr = ice_repr_add_vf(vf); - devl_unlock(devlink); - if (IS_ERR(repr)) { - err = PTR_ERR(repr); + err = repr->ops.add(repr); + if (err) goto err_create_repr; - } err = ice_eswitch_setup_repr(pf, repr); if (err) @@ -486,7 +480,7 @@ ice_eswitch_attach(struct ice_pf *pf, struct ice_vf *vf) if (err) goto err_xa_alloc; - vf->repr_id = repr->id; + *id = repr->id; ice_eswitch_start_reprs(pf); @@ -495,9 +489,7 @@ ice_eswitch_attach(struct ice_pf *pf, struct ice_vf *vf) err_xa_alloc: ice_eswitch_release_repr(pf, repr); err_setup_repr: - devl_lock(devlink); - ice_repr_rem_vf(repr); - devl_unlock(devlink); + repr->ops.rem(repr); err_create_repr: if (xa_empty(&pf->eswitch.reprs)) ice_eswitch_disable_switchdev(pf); @@ -506,14 +498,59 @@ err_create_repr: return err; } -void ice_eswitch_detach(struct ice_pf *pf, struct ice_vf *vf) +/** + * ice_eswitch_attach_vf - attach VF to a eswitch + * @pf: pointer to PF structure + * @vf: pointer to VF structure to be attached + * + * During attaching port representor for VF is created. + * + * Return: zero on success or an error code on failure. + */ +int ice_eswitch_attach_vf(struct ice_pf *pf, struct ice_vf *vf) { - struct ice_repr *repr = xa_load(&pf->eswitch.reprs, vf->repr_id); + struct ice_repr *repr = ice_repr_create_vf(vf); struct devlink *devlink = priv_to_devlink(pf); + int err; - if (!repr) - return; + if (IS_ERR(repr)) + return PTR_ERR(repr); + + devl_lock(devlink); + err = ice_eswitch_attach(pf, repr, &vf->repr_id); + if (err) + ice_repr_destroy(repr); + devl_unlock(devlink); + + return err; +} + +/** + * ice_eswitch_attach_sf - attach SF to a eswitch + * @pf: pointer to PF structure + * @sf: pointer to SF structure to be attached + * + * During attaching port representor for SF is created. + * + * Return: zero on success or an error code on failure. + */ +int ice_eswitch_attach_sf(struct ice_pf *pf, struct ice_dynamic_port *sf) +{ + struct ice_repr *repr = ice_repr_create_sf(sf); + int err; + if (IS_ERR(repr)) + return PTR_ERR(repr); + + err = ice_eswitch_attach(pf, repr, &sf->repr_id); + if (err) + ice_repr_destroy(repr); + + return err; +} + +static void ice_eswitch_detach(struct ice_pf *pf, struct ice_repr *repr) +{ ice_eswitch_stop_reprs(pf); xa_erase(&pf->eswitch.reprs, repr->id); @@ -521,10 +558,12 @@ void ice_eswitch_detach(struct ice_pf *pf, struct ice_vf *vf) ice_eswitch_disable_switchdev(pf); ice_eswitch_release_repr(pf, repr); - devl_lock(devlink); - ice_repr_rem_vf(repr); + repr->ops.rem(repr); + ice_repr_destroy(repr); if (xa_empty(&pf->eswitch.reprs)) { + struct devlink *devlink = priv_to_devlink(pf); + /* since all port representors are destroyed, there is * no point in keeping the nodes */ @@ -533,10 +572,42 @@ void ice_eswitch_detach(struct ice_pf *pf, struct ice_vf *vf) } else { ice_eswitch_start_reprs(pf); } +} + +/** + * ice_eswitch_detach_vf - detach VF from a eswitch + * @pf: pointer to PF structure + * @vf: pointer to VF structure to be detached + */ +void ice_eswitch_detach_vf(struct ice_pf *pf, struct ice_vf *vf) +{ + struct ice_repr *repr = xa_load(&pf->eswitch.reprs, vf->repr_id); + struct devlink *devlink = priv_to_devlink(pf); + + if (!repr) + return; + + devl_lock(devlink); + ice_eswitch_detach(pf, repr); devl_unlock(devlink); } /** + * ice_eswitch_detach_sf - detach SF from a eswitch + * @pf: pointer to PF structure + * @sf: pointer to SF structure to be detached + */ +void ice_eswitch_detach_sf(struct ice_pf *pf, struct ice_dynamic_port *sf) +{ + struct ice_repr *repr = xa_load(&pf->eswitch.reprs, sf->repr_id); + + if (!repr) + return; + + ice_eswitch_detach(pf, repr); +} + +/** * ice_eswitch_get_target - get netdev based on src_vsi from descriptor * @rx_ring: ring used to receive the packet * @rx_desc: descriptor used to get src_vsi value diff --git a/drivers/net/ethernet/intel/ice/ice_eswitch.h b/drivers/net/ethernet/intel/ice/ice_eswitch.h index 78fd39a6935d..20ce32dda69c 100644 --- a/drivers/net/ethernet/intel/ice/ice_eswitch.h +++ b/drivers/net/ethernet/intel/ice/ice_eswitch.h @@ -5,11 +5,13 @@ #define _ICE_ESWITCH_H_ #include <net/devlink.h> +#include "devlink/devlink_port.h" #ifdef CONFIG_ICE_SWITCHDEV -void ice_eswitch_detach(struct ice_pf *pf, struct ice_vf *vf); -int -ice_eswitch_attach(struct ice_pf *pf, struct ice_vf *vf); +void ice_eswitch_detach_vf(struct ice_pf *pf, struct ice_vf *vf); +void ice_eswitch_detach_sf(struct ice_pf *pf, struct ice_dynamic_port *sf); +int ice_eswitch_attach_vf(struct ice_pf *pf, struct ice_vf *vf); +int ice_eswitch_attach_sf(struct ice_pf *pf, struct ice_dynamic_port *sf); int ice_eswitch_mode_get(struct devlink *devlink, u16 *mode); int @@ -31,10 +33,20 @@ struct net_device *ice_eswitch_get_target(struct ice_rx_ring *rx_ring, int ice_eswitch_cfg_vsi(struct ice_vsi *vsi, const u8 *mac); void ice_eswitch_decfg_vsi(struct ice_vsi *vsi, const u8 *mac); #else /* CONFIG_ICE_SWITCHDEV */ -static inline void ice_eswitch_detach(struct ice_pf *pf, struct ice_vf *vf) { } +static inline void +ice_eswitch_detach_vf(struct ice_pf *pf, struct ice_vf *vf) { } + +static inline void +ice_eswitch_detach_sf(struct ice_pf *pf, struct ice_dynamic_port *sf) { } + +static inline int +ice_eswitch_attach_vf(struct ice_pf *pf, struct ice_vf *vf) +{ + return -EOPNOTSUPP; +} static inline int -ice_eswitch_attach(struct ice_pf *pf, struct ice_vf *vf) +ice_eswitch_attach_sf(struct ice_pf *pf, struct ice_dynamic_port *sf) { return -EOPNOTSUPP; } diff --git a/drivers/net/ethernet/intel/ice/ice_ethtool.c b/drivers/net/ethernet/intel/ice/ice_ethtool.c index bc79ba974e49..d5cc934d1359 100644 --- a/drivers/net/ethernet/intel/ice/ice_ethtool.c +++ b/drivers/net/ethernet/intel/ice/ice_ethtool.c @@ -3792,8 +3792,6 @@ ice_get_ts_info(struct net_device *dev, struct kernel_ethtool_ts_info *info) return ethtool_op_get_ts_info(dev, info); info->so_timestamping = SOF_TIMESTAMPING_TX_SOFTWARE | - SOF_TIMESTAMPING_RX_SOFTWARE | - SOF_TIMESTAMPING_SOFTWARE | SOF_TIMESTAMPING_TX_HARDWARE | SOF_TIMESTAMPING_RX_HARDWARE | SOF_TIMESTAMPING_RAW_HARDWARE; @@ -4414,7 +4412,7 @@ ice_repr_get_drvinfo(struct net_device *netdev, { struct ice_repr *repr = ice_netdev_to_repr(netdev); - if (ice_check_vf_ready_for_cfg(repr->vf)) + if (repr->ops.ready(repr)) return; __ice_get_drvinfo(netdev, drvinfo, repr->src_vsi); @@ -4426,8 +4424,7 @@ ice_repr_get_strings(struct net_device *netdev, u32 stringset, u8 *data) struct ice_repr *repr = ice_netdev_to_repr(netdev); /* for port representors only ETH_SS_STATS is supported */ - if (ice_check_vf_ready_for_cfg(repr->vf) || - stringset != ETH_SS_STATS) + if (repr->ops.ready(repr) || stringset != ETH_SS_STATS) return; __ice_get_strings(netdev, stringset, data, repr->src_vsi); @@ -4440,7 +4437,7 @@ ice_repr_get_ethtool_stats(struct net_device *netdev, { struct ice_repr *repr = ice_netdev_to_repr(netdev); - if (ice_check_vf_ready_for_cfg(repr->vf)) + if (repr->ops.ready(repr)) return; __ice_get_ethtool_stats(netdev, stats, data, repr->src_vsi); @@ -4725,6 +4722,7 @@ static const struct ethtool_ops ice_ethtool_ops = { ETHTOOL_COALESCE_USE_ADAPTIVE | ETHTOOL_COALESCE_RX_USECS_HIGH, .cap_rss_sym_xor_supported = true, + .rxfh_per_ctx_key = true, .get_link_ksettings = ice_get_link_ksettings, .set_link_ksettings = ice_set_link_ksettings, .get_fec_stats = ice_get_fec_stats, diff --git a/drivers/net/ethernet/intel/ice/ice_flex_pipe.c b/drivers/net/ethernet/intel/ice/ice_flex_pipe.c index 20d5db88c99f..ed95072ca6e3 100644 --- a/drivers/net/ethernet/intel/ice/ice_flex_pipe.c +++ b/drivers/net/ethernet/intel/ice/ice_flex_pipe.c @@ -2981,6 +2981,50 @@ ice_add_prof_attrib(struct ice_prof_map *prof, u8 ptg, u16 ptype, } /** + * ice_disable_fd_swap - set register appropriately to disable FD SWAP + * @hw: pointer to the HW struct + * @prof_id: profile ID + */ +static void +ice_disable_fd_swap(struct ice_hw *hw, u8 prof_id) +{ + u16 swap_val, fvw_num; + unsigned int i; + + swap_val = ICE_SWAP_VALID; + fvw_num = hw->blk[ICE_BLK_FD].es.fvw / ICE_FDIR_REG_SET_SIZE; + + /* Since the SWAP Flag in the Programming Desc doesn't work, + * here add method to disable the SWAP Option via setting + * certain SWAP and INSET register sets. + */ + for (i = 0; i < fvw_num ; i++) { + u32 raw_swap, raw_in; + unsigned int j; + + raw_swap = 0; + raw_in = 0; + + for (j = 0; j < ICE_FDIR_REG_SET_SIZE; j++) { + raw_swap |= (swap_val++) << (j * BITS_PER_BYTE); + raw_in |= ICE_INSET_DFLT << (j * BITS_PER_BYTE); + } + + /* write the FDIR swap register set */ + wr32(hw, GLQF_FDSWAP(prof_id, i), raw_swap); + + ice_debug(hw, ICE_DBG_INIT, "swap wr(%d, %d): 0x%x = 0x%08x\n", + prof_id, i, GLQF_FDSWAP(prof_id, i), raw_swap); + + /* write the FDIR inset register set */ + wr32(hw, GLQF_FDINSET(prof_id, i), raw_in); + + ice_debug(hw, ICE_DBG_INIT, "inset wr(%d, %d): 0x%x = 0x%08x\n", + prof_id, i, GLQF_FDINSET(prof_id, i), raw_in); + } +} + +/* * ice_add_prof - add profile * @hw: pointer to the HW struct * @blk: hardware block @@ -2991,6 +3035,7 @@ ice_add_prof_attrib(struct ice_prof_map *prof, u8 ptg, u16 ptype, * @es: extraction sequence (length of array is determined by the block) * @masks: mask for extraction sequence * @symm: symmetric setting for RSS profiles + * @fd_swap: enable/disable FDIR paired src/dst fields swap option * * This function registers a profile, which matches a set of PTYPES with a * particular extraction sequence. While the hardware profile is allocated @@ -3000,7 +3045,7 @@ ice_add_prof_attrib(struct ice_prof_map *prof, u8 ptg, u16 ptype, int ice_add_prof(struct ice_hw *hw, enum ice_block blk, u64 id, u8 ptypes[], const struct ice_ptype_attributes *attr, u16 attr_cnt, - struct ice_fv_word *es, u16 *masks, bool symm) + struct ice_fv_word *es, u16 *masks, bool symm, bool fd_swap) { u32 bytes = DIV_ROUND_UP(ICE_FLOW_PTYPE_MAX, BITS_PER_BYTE); DECLARE_BITMAP(ptgs_used, ICE_XLT1_CNT); @@ -3020,7 +3065,7 @@ ice_add_prof(struct ice_hw *hw, enum ice_block blk, u64 id, u8 ptypes[], status = ice_alloc_prof_id(hw, blk, &prof_id); if (status) goto err_ice_add_prof; - if (blk == ICE_BLK_FD) { + if (blk == ICE_BLK_FD && fd_swap) { /* For Flow Director block, the extraction sequence may * need to be altered in the case where there are paired * fields that have no match. This is necessary because @@ -3031,6 +3076,8 @@ ice_add_prof(struct ice_hw *hw, enum ice_block blk, u64 id, u8 ptypes[], status = ice_update_fd_swap(hw, prof_id, es); if (status) goto err_ice_add_prof; + } else if (blk == ICE_BLK_FD) { + ice_disable_fd_swap(hw, prof_id); } status = ice_update_prof_masking(hw, blk, prof_id, masks); if (status) @@ -4099,6 +4146,54 @@ err_ice_add_prof_id_flow: } /** + * ice_flow_assoc_fdir_prof - add an FDIR profile for main/ctrl VSI + * @hw: pointer to the HW struct + * @blk: HW block + * @dest_vsi: dest VSI + * @fdir_vsi: fdir programming VSI + * @hdl: profile handle + * + * Update the hardware tables to enable the FDIR profile indicated by @hdl for + * the VSI specified by @dest_vsi. On success, the flow will be enabled. + * + * Return: 0 on success or negative errno on failure. + */ +int +ice_flow_assoc_fdir_prof(struct ice_hw *hw, enum ice_block blk, + u16 dest_vsi, u16 fdir_vsi, u64 hdl) +{ + u16 vsi_num; + int status; + + if (blk != ICE_BLK_FD) + return -EINVAL; + + vsi_num = ice_get_hw_vsi_num(hw, dest_vsi); + status = ice_add_prof_id_flow(hw, blk, vsi_num, hdl); + if (status) { + ice_debug(hw, ICE_DBG_FLOW, "Adding HW profile failed for main VSI flow entry: %d\n", + status); + return status; + } + + vsi_num = ice_get_hw_vsi_num(hw, fdir_vsi); + status = ice_add_prof_id_flow(hw, blk, vsi_num, hdl); + if (status) { + ice_debug(hw, ICE_DBG_FLOW, "Adding HW profile failed for ctrl VSI flow entry: %d\n", + status); + goto err; + } + + return 0; + +err: + vsi_num = ice_get_hw_vsi_num(hw, dest_vsi); + ice_rem_prof_id_flow(hw, blk, vsi_num, hdl); + + return status; +} + +/** * ice_rem_prof_from_list - remove a profile from list * @hw: pointer to the HW struct * @lst: list to remove the profile from diff --git a/drivers/net/ethernet/intel/ice/ice_flex_pipe.h b/drivers/net/ethernet/intel/ice/ice_flex_pipe.h index b39d7cdc381f..90b9b0993122 100644 --- a/drivers/net/ethernet/intel/ice/ice_flex_pipe.h +++ b/drivers/net/ethernet/intel/ice/ice_flex_pipe.h @@ -6,6 +6,8 @@ #include "ice_type.h" +#define ICE_FDIR_REG_SET_SIZE 4 + int ice_acquire_change_lock(struct ice_hw *hw, enum ice_aq_res_access_type access); void ice_release_change_lock(struct ice_hw *hw); @@ -42,13 +44,16 @@ bool ice_hw_ptype_ena(struct ice_hw *hw, u16 ptype); int ice_add_prof(struct ice_hw *hw, enum ice_block blk, u64 id, u8 ptypes[], const struct ice_ptype_attributes *attr, u16 attr_cnt, - struct ice_fv_word *es, u16 *masks, bool symm); + struct ice_fv_word *es, u16 *masks, bool symm, bool fd_swap); struct ice_prof_map * ice_search_prof_id(struct ice_hw *hw, enum ice_block blk, u64 id); int ice_add_prof_id_flow(struct ice_hw *hw, enum ice_block blk, u16 vsi, u64 hdl); int ice_rem_prof_id_flow(struct ice_hw *hw, enum ice_block blk, u16 vsi, u64 hdl); +int +ice_flow_assoc_fdir_prof(struct ice_hw *hw, enum ice_block blk, + u16 dest_vsi, u16 fdir_vsi, u64 hdl); enum ice_ddp_state ice_init_pkg(struct ice_hw *hw, u8 *buff, u32 len); enum ice_ddp_state ice_copy_and_init_pkg(struct ice_hw *hw, const u8 *buf, u32 len); diff --git a/drivers/net/ethernet/intel/ice/ice_flow.c b/drivers/net/ethernet/intel/ice/ice_flow.c index fc2b58f56279..d97b751052f2 100644 --- a/drivers/net/ethernet/intel/ice/ice_flow.c +++ b/drivers/net/ethernet/intel/ice/ice_flow.c @@ -409,6 +409,29 @@ static const u32 ice_ptypes_gtpc_tid[] = { }; /* Packet types for GTPU */ +static const struct ice_ptype_attributes ice_attr_gtpu_session[] = { + { ICE_MAC_IPV4_GTPU_IPV4_FRAG, ICE_PTYPE_ATTR_GTP_SESSION }, + { ICE_MAC_IPV4_GTPU_IPV4_PAY, ICE_PTYPE_ATTR_GTP_SESSION }, + { ICE_MAC_IPV4_GTPU_IPV4_UDP_PAY, ICE_PTYPE_ATTR_GTP_SESSION }, + { ICE_MAC_IPV4_GTPU_IPV4_TCP, ICE_PTYPE_ATTR_GTP_SESSION }, + { ICE_MAC_IPV4_GTPU_IPV4_ICMP, ICE_PTYPE_ATTR_GTP_SESSION }, + { ICE_MAC_IPV6_GTPU_IPV4_FRAG, ICE_PTYPE_ATTR_GTP_SESSION }, + { ICE_MAC_IPV6_GTPU_IPV4_PAY, ICE_PTYPE_ATTR_GTP_SESSION }, + { ICE_MAC_IPV6_GTPU_IPV4_UDP_PAY, ICE_PTYPE_ATTR_GTP_SESSION }, + { ICE_MAC_IPV6_GTPU_IPV4_TCP, ICE_PTYPE_ATTR_GTP_SESSION }, + { ICE_MAC_IPV6_GTPU_IPV4_ICMP, ICE_PTYPE_ATTR_GTP_SESSION }, + { ICE_MAC_IPV4_GTPU_IPV6_FRAG, ICE_PTYPE_ATTR_GTP_SESSION }, + { ICE_MAC_IPV4_GTPU_IPV6_PAY, ICE_PTYPE_ATTR_GTP_SESSION }, + { ICE_MAC_IPV4_GTPU_IPV6_UDP_PAY, ICE_PTYPE_ATTR_GTP_SESSION }, + { ICE_MAC_IPV4_GTPU_IPV6_TCP, ICE_PTYPE_ATTR_GTP_SESSION }, + { ICE_MAC_IPV4_GTPU_IPV6_ICMPV6, ICE_PTYPE_ATTR_GTP_SESSION }, + { ICE_MAC_IPV6_GTPU_IPV6_FRAG, ICE_PTYPE_ATTR_GTP_SESSION }, + { ICE_MAC_IPV6_GTPU_IPV6_PAY, ICE_PTYPE_ATTR_GTP_SESSION }, + { ICE_MAC_IPV6_GTPU_IPV6_UDP_PAY, ICE_PTYPE_ATTR_GTP_SESSION }, + { ICE_MAC_IPV6_GTPU_IPV6_TCP, ICE_PTYPE_ATTR_GTP_SESSION }, + { ICE_MAC_IPV6_GTPU_IPV6_ICMPV6, ICE_PTYPE_ATTR_GTP_SESSION }, +}; + static const struct ice_ptype_attributes ice_attr_gtpu_eh[] = { { ICE_MAC_IPV4_GTPU_IPV4_FRAG, ICE_PTYPE_ATTR_GTP_PDU_EH }, { ICE_MAC_IPV4_GTPU_IPV4_PAY, ICE_PTYPE_ATTR_GTP_PDU_EH }, @@ -1400,7 +1423,7 @@ ice_flow_add_prof_sync(struct ice_hw *hw, enum ice_block blk, /* Add a HW profile for this flow profile */ status = ice_add_prof(hw, blk, prof_id, (u8 *)params->ptypes, params->attr, params->attr_cnt, params->es, - params->mask, symm); + params->mask, symm, true); if (status) { ice_debug(hw, ICE_DBG_FLOW, "Error adding a HW flow profile\n"); goto out; @@ -1523,6 +1546,90 @@ ice_flow_disassoc_prof(struct ice_hw *hw, enum ice_block blk, return status; } +#define FLAG_GTP_EH_PDU_LINK BIT_ULL(13) +#define FLAG_GTP_EH_PDU BIT_ULL(14) + +#define HI_BYTE_IN_WORD GENMASK(15, 8) +#define LO_BYTE_IN_WORD GENMASK(7, 0) + +#define FLAG_GTPU_MSK \ + (FLAG_GTP_EH_PDU | FLAG_GTP_EH_PDU_LINK) +#define FLAG_GTPU_UP \ + (FLAG_GTP_EH_PDU | FLAG_GTP_EH_PDU_LINK) +#define FLAG_GTPU_DW FLAG_GTP_EH_PDU + +/** + * ice_flow_set_parser_prof - Set flow profile based on the parsed profile info + * @hw: pointer to the HW struct + * @dest_vsi: dest VSI + * @fdir_vsi: fdir programming VSI + * @prof: stores parsed profile info from raw flow + * @blk: classification blk + * + * Return: 0 on success or negative errno on failure. + */ +int +ice_flow_set_parser_prof(struct ice_hw *hw, u16 dest_vsi, u16 fdir_vsi, + struct ice_parser_profile *prof, enum ice_block blk) +{ + u64 id = find_first_bit(prof->ptypes, ICE_FLOW_PTYPE_MAX); + struct ice_flow_prof_params *params __free(kfree); + u8 fv_words = hw->blk[blk].es.fvw; + int status; + int i, idx; + + params = kzalloc(sizeof(*params), GFP_KERNEL); + if (!params) + return -ENOMEM; + + for (i = 0; i < ICE_MAX_FV_WORDS; i++) { + params->es[i].prot_id = ICE_PROT_INVALID; + params->es[i].off = ICE_FV_OFFSET_INVAL; + } + + for (i = 0; i < prof->fv_num; i++) { + if (hw->blk[blk].es.reverse) + idx = fv_words - i - 1; + else + idx = i; + params->es[idx].prot_id = prof->fv[i].proto_id; + params->es[idx].off = prof->fv[i].offset; + params->mask[idx] = (((prof->fv[i].msk) << BITS_PER_BYTE) & + HI_BYTE_IN_WORD) | + (((prof->fv[i].msk) >> BITS_PER_BYTE) & + LO_BYTE_IN_WORD); + } + + switch (prof->flags) { + case FLAG_GTPU_DW: + params->attr = ice_attr_gtpu_down; + params->attr_cnt = ARRAY_SIZE(ice_attr_gtpu_down); + break; + case FLAG_GTPU_UP: + params->attr = ice_attr_gtpu_up; + params->attr_cnt = ARRAY_SIZE(ice_attr_gtpu_up); + break; + default: + if (prof->flags_msk & FLAG_GTPU_MSK) { + params->attr = ice_attr_gtpu_session; + params->attr_cnt = ARRAY_SIZE(ice_attr_gtpu_session); + } + break; + } + + status = ice_add_prof(hw, blk, id, (u8 *)prof->ptypes, + params->attr, params->attr_cnt, + params->es, params->mask, false, false); + if (status) + return status; + + status = ice_flow_assoc_fdir_prof(hw, blk, dest_vsi, fdir_vsi, id); + if (status) + ice_rem_prof(hw, blk, id); + + return status; +} + /** * ice_flow_add_prof - Add a flow profile for packet segments and matched fields * @hw: pointer to the HW struct diff --git a/drivers/net/ethernet/intel/ice/ice_flow.h b/drivers/net/ethernet/intel/ice/ice_flow.h index 2fd2e0cb483d..6cb7bb879c98 100644 --- a/drivers/net/ethernet/intel/ice/ice_flow.h +++ b/drivers/net/ethernet/intel/ice/ice_flow.h @@ -5,6 +5,7 @@ #define _ICE_FLOW_H_ #include "ice_flex_type.h" +#include "ice_parser.h" #define ICE_FLOW_ENTRY_HANDLE_INVAL 0 #define ICE_FLOW_FLD_OFF_INVAL 0xffff @@ -326,6 +327,7 @@ enum ice_rss_cfg_hdr_type { ICE_RSS_ANY_HEADERS }; +struct ice_vsi; struct ice_rss_hash_cfg { u32 addl_hdrs; /* protocol header fields */ u64 hash_flds; /* hash bit field (ICE_FLOW_HASH_*) to configure */ @@ -445,6 +447,9 @@ ice_flow_add_prof(struct ice_hw *hw, enum ice_block blk, enum ice_flow_dir dir, bool symm, struct ice_flow_prof **prof); int ice_flow_rem_prof(struct ice_hw *hw, enum ice_block blk, u64 prof_id); int +ice_flow_set_parser_prof(struct ice_hw *hw, u16 dest_vsi, u16 fdir_vsi, + struct ice_parser_profile *prof, enum ice_block blk); +int ice_flow_add_entry(struct ice_hw *hw, enum ice_block blk, u64 prof_id, u64 entry_id, u16 vsi, enum ice_flow_priority prio, void *data, u64 *entry_h); diff --git a/drivers/net/ethernet/intel/ice/ice_lib.c b/drivers/net/ethernet/intel/ice/ice_lib.c index f559e60992fa..06e712cdc3d9 100644 --- a/drivers/net/ethernet/intel/ice/ice_lib.c +++ b/drivers/net/ethernet/intel/ice/ice_lib.c @@ -7,6 +7,7 @@ #include "ice_lib.h" #include "ice_fltr.h" #include "ice_dcb_lib.h" +#include "ice_type.h" #include "ice_vsi_vlan_ops.h" /** @@ -20,6 +21,8 @@ const char *ice_vsi_type_str(enum ice_vsi_type vsi_type) return "ICE_VSI_PF"; case ICE_VSI_VF: return "ICE_VSI_VF"; + case ICE_VSI_SF: + return "ICE_VSI_SF"; case ICE_VSI_CTRL: return "ICE_VSI_CTRL"; case ICE_VSI_CHNL: @@ -135,6 +138,7 @@ static void ice_vsi_set_num_desc(struct ice_vsi *vsi) { switch (vsi->type) { case ICE_VSI_PF: + case ICE_VSI_SF: case ICE_VSI_CTRL: case ICE_VSI_LB: /* a user could change the values of num_[tr]x_desc using @@ -201,6 +205,12 @@ static void ice_vsi_set_num_qs(struct ice_vsi *vsi) max_t(int, vsi->alloc_rxq, vsi->alloc_txq)); break; + case ICE_VSI_SF: + vsi->alloc_txq = 1; + vsi->alloc_rxq = 1; + vsi->num_q_vectors = 1; + vsi->irq_dyn_alloc = true; + break; case ICE_VSI_VF: if (vf->num_req_qs) vf->num_vf_qs = vf->num_req_qs; @@ -423,7 +433,7 @@ err_out: * This deallocates the VSI's queue resources, removes it from the PF's * VSI array if necessary, and deallocates the VSI */ -static void ice_vsi_free(struct ice_vsi *vsi) +void ice_vsi_free(struct ice_vsi *vsi) { struct ice_pf *pf = NULL; struct device *dev; @@ -447,6 +457,7 @@ static void ice_vsi_free(struct ice_vsi *vsi) ice_vsi_free_stats(vsi); ice_vsi_free_arrays(vsi); + mutex_destroy(&vsi->xdp_state_lock); mutex_unlock(&pf->sw_mutex); devm_kfree(dev, vsi); } @@ -558,6 +569,7 @@ ice_vsi_alloc_def(struct ice_vsi *vsi, struct ice_channel *ch) switch (vsi->type) { case ICE_VSI_PF: + case ICE_VSI_SF: /* Setup default MSIX irq handler for VSI */ vsi->irq_handler = ice_msix_clean_rings; break; @@ -594,7 +606,7 @@ ice_vsi_alloc_def(struct ice_vsi *vsi, struct ice_channel *ch) * * returns a pointer to a VSI on success, NULL on failure. */ -static struct ice_vsi *ice_vsi_alloc(struct ice_pf *pf) +struct ice_vsi *ice_vsi_alloc(struct ice_pf *pf) { struct device *dev = ice_pf_to_dev(pf); struct ice_vsi *vsi = NULL; @@ -626,6 +638,8 @@ static struct ice_vsi *ice_vsi_alloc(struct ice_pf *pf) pf->next_vsi = ice_get_free_slot(pf->vsi, pf->num_alloc_vsi, pf->next_vsi); + mutex_init(&vsi->xdp_state_lock); + unlock_pf: mutex_unlock(&pf->sw_mutex); return vsi; @@ -886,6 +900,11 @@ static void ice_vsi_set_rss_params(struct ice_vsi *vsi) max_rss_size); vsi->rss_lut_type = ICE_LUT_PF; break; + case ICE_VSI_SF: + vsi->rss_table_size = ICE_LUT_VSI_SIZE; + vsi->rss_size = min_t(u16, num_online_cpus(), max_rss_size); + vsi->rss_lut_type = ICE_LUT_VSI; + break; case ICE_VSI_VF: /* VF VSI will get a small RSS table. * For VSI_LUT, LUT size should be set to 64 bytes. @@ -1133,6 +1152,7 @@ static void ice_set_rss_vsi_ctx(struct ice_vsi_ctx *ctxt, struct ice_vsi *vsi) lut_type = ICE_AQ_VSI_Q_OPT_RSS_LUT_PF; break; case ICE_VSI_VF: + case ICE_VSI_SF: /* VF VSI will gets a small RSS table which is a VSI LUT type */ lut_type = ICE_AQ_VSI_Q_OPT_RSS_LUT_VSI; break; @@ -1211,6 +1231,7 @@ static int ice_vsi_init(struct ice_vsi *vsi, u32 vsi_flags) case ICE_VSI_PF: ctxt->flags = ICE_AQ_VSI_TYPE_PF; break; + case ICE_VSI_SF: case ICE_VSI_CHNL: ctxt->flags = ICE_AQ_VSI_TYPE_VMDQ2; break; @@ -2092,6 +2113,7 @@ static void ice_set_agg_vsi(struct ice_vsi *vsi) case ICE_VSI_CHNL: case ICE_VSI_LB: case ICE_VSI_PF: + case ICE_VSI_SF: max_agg_nodes = ICE_MAX_PF_AGG_NODES; agg_node_id_start = ICE_PF_AGG_NODE_ID_START; agg_node_iter = &pf->pf_agg_node[0]; @@ -2261,6 +2283,7 @@ static int ice_vsi_cfg_def(struct ice_vsi *vsi) switch (vsi->type) { case ICE_VSI_CTRL: + case ICE_VSI_SF: case ICE_VSI_PF: ret = ice_vsi_alloc_q_vectors(vsi); if (ret) @@ -2286,9 +2309,6 @@ static int ice_vsi_cfg_def(struct ice_vsi *vsi) ice_vsi_map_rings_to_vectors(vsi); - /* Associate q_vector rings to napi */ - ice_vsi_set_napi_queues(vsi); - vsi->stat_offsets_loaded = false; /* ICE_VSI_CTRL does not need RSS so skip RSS processing */ @@ -2413,20 +2433,13 @@ void ice_vsi_decfg(struct ice_vsi *vsi) struct ice_pf *pf = vsi->back; int err; - /* The Rx rule will only exist to remove if the LLDP FW - * engine is currently stopped - */ - if (!ice_is_safe_mode(pf) && vsi->type == ICE_VSI_PF && - !test_bit(ICE_FLAG_FW_LLDP_AGENT, pf->flags)) - ice_cfg_sw_lldp(vsi, false, false); - ice_rm_vsi_lan_cfg(vsi->port_info, vsi->idx); err = ice_rm_vsi_rdma_cfg(vsi->port_info, vsi->idx); if (err) dev_err(ice_pf_to_dev(pf), "Failed to remove RDMA scheduler config for VSI %u, err %d\n", vsi->vsi_num, err); - if (ice_is_xdp_ena_vsi(vsi)) + if (vsi->xdp_rings) /* return value check can be skipped here, it always returns * 0 if reset is in progress */ @@ -2528,7 +2541,7 @@ static void ice_vsi_release_msix(struct ice_vsi *vsi) for (q = 0; q < q_vector->num_ring_tx; q++) { ice_write_itr(&q_vector->tx, 0); wr32(hw, QINT_TQCTL(vsi->txq_map[txq]), 0); - if (ice_is_xdp_ena_vsi(vsi)) { + if (vsi->xdp_rings) { u32 xdp_txq = txq + vsi->num_xdp_txq; wr32(hw, QINT_TQCTL(vsi->txq_map[xdp_txq]), 0); @@ -2628,6 +2641,7 @@ void ice_vsi_close(struct ice_vsi *vsi) if (!test_and_set_bit(ICE_VSI_DOWN, vsi->state)) ice_down(vsi); + ice_vsi_clear_napi_queues(vsi); ice_vsi_free_irq(vsi); ice_vsi_free_tx_rings(vsi); ice_vsi_free_rx_rings(vsi); @@ -2647,7 +2661,8 @@ int ice_ena_vsi(struct ice_vsi *vsi, bool locked) clear_bit(ICE_VSI_NEEDS_RESTART, vsi->state); - if (vsi->netdev && vsi->type == ICE_VSI_PF) { + if (vsi->netdev && (vsi->type == ICE_VSI_PF || + vsi->type == ICE_VSI_SF)) { if (netif_running(vsi->netdev)) { if (!locked) rtnl_lock(); @@ -2671,143 +2686,99 @@ int ice_ena_vsi(struct ice_vsi *vsi, bool locked) */ void ice_dis_vsi(struct ice_vsi *vsi, bool locked) { - if (test_bit(ICE_VSI_DOWN, vsi->state)) - return; + bool already_down = test_bit(ICE_VSI_DOWN, vsi->state); set_bit(ICE_VSI_NEEDS_RESTART, vsi->state); - if (vsi->type == ICE_VSI_PF && vsi->netdev) { + if (vsi->netdev && (vsi->type == ICE_VSI_PF || + vsi->type == ICE_VSI_SF)) { if (netif_running(vsi->netdev)) { if (!locked) rtnl_lock(); - - ice_vsi_close(vsi); + already_down = test_bit(ICE_VSI_DOWN, vsi->state); + if (!already_down) + ice_vsi_close(vsi); if (!locked) rtnl_unlock(); - } else { + } else if (!already_down) { ice_vsi_close(vsi); } - } else if (vsi->type == ICE_VSI_CTRL) { + } else if (vsi->type == ICE_VSI_CTRL && !already_down) { ice_vsi_close(vsi); } } /** - * __ice_queue_set_napi - Set the napi instance for the queue - * @dev: device to which NAPI and queue belong - * @queue_index: Index of queue - * @type: queue type as RX or TX - * @napi: NAPI context - * @locked: is the rtnl_lock already held - * - * Set the napi instance for the queue. Caller indicates the lock status. - */ -static void -__ice_queue_set_napi(struct net_device *dev, unsigned int queue_index, - enum netdev_queue_type type, struct napi_struct *napi, - bool locked) -{ - if (!locked) - rtnl_lock(); - netif_queue_set_napi(dev, queue_index, type, napi); - if (!locked) - rtnl_unlock(); -} - -/** - * ice_queue_set_napi - Set the napi instance for the queue - * @vsi: VSI being configured - * @queue_index: Index of queue - * @type: queue type as RX or TX - * @napi: NAPI context + * ice_vsi_set_napi_queues - associate netdev queues with napi + * @vsi: VSI pointer * - * Set the napi instance for the queue. The rtnl lock state is derived from the - * execution path. + * Associate queue[s] with napi for all vectors. + * The caller must hold rtnl_lock. */ -void -ice_queue_set_napi(struct ice_vsi *vsi, unsigned int queue_index, - enum netdev_queue_type type, struct napi_struct *napi) +void ice_vsi_set_napi_queues(struct ice_vsi *vsi) { - struct ice_pf *pf = vsi->back; + struct net_device *netdev = vsi->netdev; + int q_idx, v_idx; - if (!vsi->netdev) + if (!netdev) return; - if (current_work() == &pf->serv_task || - test_bit(ICE_PREPARED_FOR_RESET, pf->state) || - test_bit(ICE_DOWN, pf->state) || - test_bit(ICE_SUSPENDED, pf->state)) - __ice_queue_set_napi(vsi->netdev, queue_index, type, napi, - false); - else - __ice_queue_set_napi(vsi->netdev, queue_index, type, napi, - true); -} + ice_for_each_rxq(vsi, q_idx) + netif_queue_set_napi(netdev, q_idx, NETDEV_QUEUE_TYPE_RX, + &vsi->rx_rings[q_idx]->q_vector->napi); -/** - * __ice_q_vector_set_napi_queues - Map queue[s] associated with the napi - * @q_vector: q_vector pointer - * @locked: is the rtnl_lock already held - * - * Associate the q_vector napi with all the queue[s] on the vector. - * Caller indicates the lock status. - */ -void __ice_q_vector_set_napi_queues(struct ice_q_vector *q_vector, bool locked) -{ - struct ice_rx_ring *rx_ring; - struct ice_tx_ring *tx_ring; - - ice_for_each_rx_ring(rx_ring, q_vector->rx) - __ice_queue_set_napi(q_vector->vsi->netdev, rx_ring->q_index, - NETDEV_QUEUE_TYPE_RX, &q_vector->napi, - locked); - - ice_for_each_tx_ring(tx_ring, q_vector->tx) - __ice_queue_set_napi(q_vector->vsi->netdev, tx_ring->q_index, - NETDEV_QUEUE_TYPE_TX, &q_vector->napi, - locked); + ice_for_each_txq(vsi, q_idx) + netif_queue_set_napi(netdev, q_idx, NETDEV_QUEUE_TYPE_TX, + &vsi->tx_rings[q_idx]->q_vector->napi); /* Also set the interrupt number for the NAPI */ - netif_napi_set_irq(&q_vector->napi, q_vector->irq.virq); + ice_for_each_q_vector(vsi, v_idx) { + struct ice_q_vector *q_vector = vsi->q_vectors[v_idx]; + + netif_napi_set_irq(&q_vector->napi, q_vector->irq.virq); + } } /** - * ice_q_vector_set_napi_queues - Map queue[s] associated with the napi - * @q_vector: q_vector pointer + * ice_vsi_clear_napi_queues - dissociate netdev queues from napi + * @vsi: VSI pointer * - * Associate the q_vector napi with all the queue[s] on the vector + * Clear the association between all VSI queues queue[s] and napi. + * The caller must hold rtnl_lock. */ -void ice_q_vector_set_napi_queues(struct ice_q_vector *q_vector) +void ice_vsi_clear_napi_queues(struct ice_vsi *vsi) { - struct ice_rx_ring *rx_ring; - struct ice_tx_ring *tx_ring; + struct net_device *netdev = vsi->netdev; + int q_idx; - ice_for_each_rx_ring(rx_ring, q_vector->rx) - ice_queue_set_napi(q_vector->vsi, rx_ring->q_index, - NETDEV_QUEUE_TYPE_RX, &q_vector->napi); + if (!netdev) + return; - ice_for_each_tx_ring(tx_ring, q_vector->tx) - ice_queue_set_napi(q_vector->vsi, tx_ring->q_index, - NETDEV_QUEUE_TYPE_TX, &q_vector->napi); - /* Also set the interrupt number for the NAPI */ - netif_napi_set_irq(&q_vector->napi, q_vector->irq.virq); + ice_for_each_txq(vsi, q_idx) + netif_queue_set_napi(netdev, q_idx, NETDEV_QUEUE_TYPE_TX, NULL); + + ice_for_each_rxq(vsi, q_idx) + netif_queue_set_napi(netdev, q_idx, NETDEV_QUEUE_TYPE_RX, NULL); } /** - * ice_vsi_set_napi_queues - * @vsi: VSI pointer + * ice_napi_add - register NAPI handler for the VSI + * @vsi: VSI for which NAPI handler is to be registered * - * Associate queue[s] with napi for all vectors + * This function is only called in the driver's load path. Registering the NAPI + * handler is done in ice_vsi_alloc_q_vector() for all other cases (i.e. resume, + * reset/rebuild, etc.) */ -void ice_vsi_set_napi_queues(struct ice_vsi *vsi) +void ice_napi_add(struct ice_vsi *vsi) { - int i; + int v_idx; if (!vsi->netdev) return; - ice_for_each_q_vector(vsi, i) - ice_q_vector_set_napi_queues(vsi->q_vectors[i]); + ice_for_each_q_vector(vsi, v_idx) + netif_napi_add(vsi->netdev, &vsi->q_vectors[v_idx]->napi, + ice_napi_poll); } /** @@ -2828,6 +2799,14 @@ int ice_vsi_release(struct ice_vsi *vsi) ice_rss_clean(vsi); ice_vsi_close(vsi); + + /* The Rx rule will only exist to remove if the LLDP FW + * engine is currently stopped + */ + if (!ice_is_safe_mode(pf) && vsi->type == ICE_VSI_PF && + !test_bit(ICE_FLAG_FW_LLDP_AGENT, pf->flags)) + ice_cfg_sw_lldp(vsi, false, false); + ice_vsi_decfg(vsi); /* retain SW VSI data structure since it is needed to unregister and @@ -3039,19 +3018,23 @@ int ice_vsi_rebuild(struct ice_vsi *vsi, u32 vsi_flags) if (WARN_ON(vsi->type == ICE_VSI_VF && !vsi->vf)) return -EINVAL; + mutex_lock(&vsi->xdp_state_lock); + ret = ice_vsi_realloc_stat_arrays(vsi); if (ret) - goto err_vsi_cfg; + goto unlock; ice_vsi_decfg(vsi); ret = ice_vsi_cfg_def(vsi); if (ret) - goto err_vsi_cfg; + goto unlock; coalesce = kcalloc(vsi->num_q_vectors, sizeof(struct ice_coalesce_stored), GFP_KERNEL); - if (!coalesce) - return -ENOMEM; + if (!coalesce) { + ret = -ENOMEM; + goto decfg; + } prev_num_q_vectors = ice_vsi_rebuild_get_coalesce(vsi, coalesce); @@ -3059,22 +3042,23 @@ int ice_vsi_rebuild(struct ice_vsi *vsi, u32 vsi_flags) if (ret) { if (vsi_flags & ICE_VSI_FLAG_INIT) { ret = -EIO; - goto err_vsi_cfg_tc_lan; + goto free_coalesce; } - kfree(coalesce); - return ice_schedule_reset(pf, ICE_RESET_PFR); + ret = ice_schedule_reset(pf, ICE_RESET_PFR); + goto free_coalesce; } ice_vsi_rebuild_set_coalesce(vsi, coalesce, prev_num_q_vectors); - kfree(coalesce); - - return 0; + clear_bit(ICE_VSI_REBUILD_PENDING, vsi->state); -err_vsi_cfg_tc_lan: - ice_vsi_decfg(vsi); +free_coalesce: kfree(coalesce); -err_vsi_cfg: +decfg: + if (ret) + ice_vsi_decfg(vsi); +unlock: + mutex_unlock(&vsi->xdp_state_lock); return ret; } diff --git a/drivers/net/ethernet/intel/ice/ice_lib.h b/drivers/net/ethernet/intel/ice/ice_lib.h index 94ce8964dda6..1a6cfc8693ce 100644 --- a/drivers/net/ethernet/intel/ice/ice_lib.h +++ b/drivers/net/ethernet/intel/ice/ice_lib.h @@ -44,15 +44,10 @@ void ice_vsi_cfg_netdev_tc(struct ice_vsi *vsi, u8 ena_tc); struct ice_vsi * ice_vsi_setup(struct ice_pf *pf, struct ice_vsi_cfg_params *params); -void -ice_queue_set_napi(struct ice_vsi *vsi, unsigned int queue_index, - enum netdev_queue_type type, struct napi_struct *napi); - -void __ice_q_vector_set_napi_queues(struct ice_q_vector *q_vector, bool locked); - -void ice_q_vector_set_napi_queues(struct ice_q_vector *q_vector); - void ice_vsi_set_napi_queues(struct ice_vsi *vsi); +void ice_napi_add(struct ice_vsi *vsi); + +void ice_vsi_clear_napi_queues(struct ice_vsi *vsi); int ice_vsi_release(struct ice_vsi *vsi); @@ -65,6 +60,8 @@ void ice_dis_vsi(struct ice_vsi *vsi, bool locked); int ice_vsi_rebuild(struct ice_vsi *vsi, u32 vsi_flags); int ice_vsi_cfg(struct ice_vsi *vsi); +struct ice_vsi *ice_vsi_alloc(struct ice_pf *pf); +void ice_vsi_free(struct ice_vsi *vsi); bool ice_is_reset_in_progress(unsigned long *state); int ice_wait_for_reset(struct ice_pf *pf, unsigned long timeout); diff --git a/drivers/net/ethernet/intel/ice/ice_main.c b/drivers/net/ethernet/intel/ice/ice_main.c index 6f97ed471fe9..eeb48cc48e08 100644 --- a/drivers/net/ethernet/intel/ice/ice_main.c +++ b/drivers/net/ethernet/intel/ice/ice_main.c @@ -15,6 +15,7 @@ #include "ice_dcb_nl.h" #include "devlink/devlink.h" #include "devlink/devlink_port.h" +#include "ice_sf_eth.h" #include "ice_hwmon.h" /* Including ice_trace.h with CREATE_TRACE_POINTS defined will generate the * ice tracepoint functions. This must be done exactly once across the @@ -608,11 +609,15 @@ ice_prepare_for_reset(struct ice_pf *pf, enum ice_reset_req reset_type) memset(&vsi->mqprio_qopt, 0, sizeof(vsi->mqprio_qopt)); } } + + if (vsi->netdev) + netif_device_detach(vsi->netdev); skip: /* clear SW filtering DB */ ice_clear_hw_tbls(hw); /* disable the VSIs and their queues that are not already DOWN */ + set_bit(ICE_VSI_REBUILD_PENDING, ice_get_main_vsi(pf)->state); ice_pf_dis_all_vsi(pf, false); if (test_bit(ICE_FLAG_PTP_SUPPORTED, pf->flags)) @@ -2970,6 +2975,9 @@ int ice_vsi_determine_xdp_res(struct ice_vsi *vsi) if (avail < cpus / 2) return -ENOMEM; + if (vsi->type == ICE_VSI_SF) + avail = vsi->alloc_txq; + vsi->num_xdp_txq = min_t(u16, avail, cpus); if (vsi->num_xdp_txq < cpus) @@ -3001,8 +3009,8 @@ ice_xdp_setup_prog(struct ice_vsi *vsi, struct bpf_prog *prog, struct netlink_ext_ack *extack) { unsigned int frame_size = vsi->netdev->mtu + ICE_ETH_PKT_HDR_PAD; - bool if_running = netif_running(vsi->netdev); int ret = 0, xdp_ring_err = 0; + bool if_running; if (prog && !prog->aux->xdp_has_frags) { if (frame_size > ice_max_xdp_frame_size(vsi)) { @@ -3013,13 +3021,17 @@ ice_xdp_setup_prog(struct ice_vsi *vsi, struct bpf_prog *prog, } /* hot swap progs and avoid toggling link */ - if (ice_is_xdp_ena_vsi(vsi) == !!prog) { + if (ice_is_xdp_ena_vsi(vsi) == !!prog || + test_bit(ICE_VSI_REBUILD_PENDING, vsi->state)) { ice_vsi_assign_bpf_prog(vsi, prog); return 0; } + if_running = netif_running(vsi->netdev) && + !test_and_set_bit(ICE_VSI_DOWN, vsi->state); + /* need to stop netdev while setting up the program for Rx rings */ - if (if_running && !test_and_set_bit(ICE_VSI_DOWN, vsi->state)) { + if (if_running) { ret = ice_down(vsi); if (ret) { NL_SET_ERR_MSG_MOD(extack, "Preparing device for XDP attach failed"); @@ -3081,25 +3093,32 @@ static int ice_xdp_safe_mode(struct net_device __always_unused *dev, * @dev: netdevice * @xdp: XDP command */ -static int ice_xdp(struct net_device *dev, struct netdev_bpf *xdp) +int ice_xdp(struct net_device *dev, struct netdev_bpf *xdp) { struct ice_netdev_priv *np = netdev_priv(dev); struct ice_vsi *vsi = np->vsi; + int ret; - if (vsi->type != ICE_VSI_PF) { - NL_SET_ERR_MSG_MOD(xdp->extack, "XDP can be loaded only on PF VSI"); + if (vsi->type != ICE_VSI_PF && vsi->type != ICE_VSI_SF) { + NL_SET_ERR_MSG_MOD(xdp->extack, "XDP can be loaded only on PF or SF VSI"); return -EINVAL; } + mutex_lock(&vsi->xdp_state_lock); + switch (xdp->command) { case XDP_SETUP_PROG: - return ice_xdp_setup_prog(vsi, xdp->prog, xdp->extack); + ret = ice_xdp_setup_prog(vsi, xdp->prog, xdp->extack); + break; case XDP_SETUP_XSK_POOL: - return ice_xsk_pool_setup(vsi, xdp->xsk.pool, - xdp->xsk.queue_id); + ret = ice_xsk_pool_setup(vsi, xdp->xsk.pool, xdp->xsk.queue_id); + break; default: - return -EINVAL; + ret = -EINVAL; } + + mutex_unlock(&vsi->xdp_state_lock); + return ret; } /** @@ -3541,28 +3560,6 @@ skip_req_irq: } /** - * ice_napi_add - register NAPI handler for the VSI - * @vsi: VSI for which NAPI handler is to be registered - * - * This function is only called in the driver's load path. Registering the NAPI - * handler is done in ice_vsi_alloc_q_vector() for all other cases (i.e. resume, - * reset/rebuild, etc.) - */ -static void ice_napi_add(struct ice_vsi *vsi) -{ - int v_idx; - - if (!vsi->netdev) - return; - - ice_for_each_q_vector(vsi, v_idx) { - netif_napi_add(vsi->netdev, &vsi->q_vectors[v_idx]->napi, - ice_napi_poll); - __ice_q_vector_set_napi_queues(vsi->q_vectors[v_idx], false); - } -} - -/** * ice_set_ops - set netdev and ethtools ops for the given netdev * @vsi: the VSI associated with the new netdev */ @@ -3595,7 +3592,7 @@ static void ice_set_ops(struct ice_vsi *vsi) * ice_set_netdev_features - set features for the given netdev * @netdev: netdev instance */ -static void ice_set_netdev_features(struct net_device *netdev) +void ice_set_netdev_features(struct net_device *netdev) { struct ice_pf *pf = ice_netdev_to_pf(netdev); bool is_dvm_ena = ice_is_dvm_ena(&pf->hw); @@ -3777,8 +3774,7 @@ ice_lb_vsi_setup(struct ice_pf *pf, struct ice_port_info *pi) * * net_device_ops implementation for adding VLAN IDs */ -static int -ice_vlan_rx_add_vid(struct net_device *netdev, __be16 proto, u16 vid) +int ice_vlan_rx_add_vid(struct net_device *netdev, __be16 proto, u16 vid) { struct ice_netdev_priv *np = netdev_priv(netdev); struct ice_vsi_vlan_ops *vlan_ops; @@ -3840,8 +3836,7 @@ finish: * * net_device_ops implementation for removing VLAN IDs */ -static int -ice_vlan_rx_kill_vid(struct net_device *netdev, __be16 proto, u16 vid) +int ice_vlan_rx_kill_vid(struct net_device *netdev, __be16 proto, u16 vid) { struct ice_netdev_priv *np = netdev_priv(netdev); struct ice_vsi_vlan_ops *vlan_ops; @@ -4010,6 +4005,9 @@ static void ice_deinit_pf(struct ice_pf *pf) if (pf->ptp.clock) ptp_clock_unregister(pf->ptp.clock); + + xa_destroy(&pf->dyn_ports); + xa_destroy(&pf->sf_nums); } /** @@ -4103,6 +4101,9 @@ static int ice_init_pf(struct ice_pf *pf) hash_init(pf->vfs.table); ice_mbx_init_snapshot(&pf->hw); + xa_init(&pf->dyn_ports); + xa_init(&pf->sf_nums); + return 0; } @@ -5350,7 +5351,6 @@ err_load: ice_deinit(pf); err_init: ice_adapter_put(pdev); - pci_disable_device(pdev); return err; } @@ -5445,6 +5445,7 @@ static void ice_remove(struct pci_dev *pdev) ice_remove_arfs(pf); devl_lock(priv_to_devlink(pf)); + ice_dealloc_all_dynamic_ports(pf); ice_deinit_devlink(pf); ice_unload(pf); @@ -5457,7 +5458,6 @@ static void ice_remove(struct pci_dev *pdev) ice_set_wake(pf); ice_adapter_put(pdev); - pci_disable_device(pdev); } /** @@ -5537,7 +5537,9 @@ static int ice_reinit_interrupt_scheme(struct ice_pf *pf) if (ret) goto err_reinit; ice_vsi_map_rings_to_vectors(pf->vsi[v]); + rtnl_lock(); ice_vsi_set_napi_queues(pf->vsi[v]); + rtnl_unlock(); } ret = ice_req_irq_msix_misc(pf); @@ -5551,8 +5553,12 @@ static int ice_reinit_interrupt_scheme(struct ice_pf *pf) err_reinit: while (v--) - if (pf->vsi[v]) + if (pf->vsi[v]) { + rtnl_lock(); + ice_vsi_clear_napi_queues(pf->vsi[v]); + rtnl_unlock(); ice_vsi_free_q_vectors(pf->vsi[v]); + } return ret; } @@ -5617,6 +5623,9 @@ static int ice_suspend(struct device *dev) ice_for_each_vsi(pf, v) { if (!pf->vsi[v]) continue; + rtnl_lock(); + ice_vsi_clear_napi_queues(pf->vsi[v]); + rtnl_unlock(); ice_vsi_free_q_vectors(pf->vsi[v]); } ice_clear_interrupt_scheme(pf); @@ -5924,8 +5933,16 @@ static int __init ice_module_init(void) goto err_dest_lag_wq; } + status = ice_sf_driver_register(); + if (status) { + pr_err("Failed to register SF driver, err %d\n", status); + goto err_sf_driver; + } + return 0; +err_sf_driver: + pci_unregister_driver(&ice_driver); err_dest_lag_wq: destroy_workqueue(ice_lag_wq); ice_debugfs_exit(); @@ -5943,6 +5960,7 @@ module_init(ice_module_init); */ static void __exit ice_module_exit(void) { + ice_sf_driver_unregister(); pci_unregister_driver(&ice_driver); ice_debugfs_exit(); destroy_workqueue(ice_wq); @@ -6744,7 +6762,8 @@ static int ice_up_complete(struct ice_vsi *vsi) if (vsi->port_info && (vsi->port_info->phy.link_info.link_info & ICE_AQ_LINK_UP) && - vsi->netdev && vsi->type == ICE_VSI_PF) { + ((vsi->netdev && (vsi->type == ICE_VSI_PF || + vsi->type == ICE_VSI_SF)))) { ice_print_link_msg(vsi, true); netif_tx_start_all_queues(vsi->netdev); netif_carrier_on(vsi->netdev); @@ -7102,7 +7121,6 @@ void ice_update_pf_stats(struct ice_pf *pf) * @netdev: network interface device structure * @stats: main device statistics structure */ -static void ice_get_stats64(struct net_device *netdev, struct rtnl_link_stats64 *stats) { struct ice_netdev_priv *np = netdev_priv(netdev); @@ -7230,7 +7248,7 @@ int ice_down(struct ice_vsi *vsi) if (tx_err) netdev_err(vsi->netdev, "Failed stop Tx rings, VSI %d error %d\n", vsi->vsi_num, tx_err); - if (!tx_err && ice_is_xdp_ena_vsi(vsi)) { + if (!tx_err && vsi->xdp_rings) { tx_err = ice_vsi_stop_xdp_tx_rings(vsi); if (tx_err) netdev_err(vsi->netdev, "Failed stop XDP rings, VSI %d error %d\n", @@ -7247,7 +7265,7 @@ int ice_down(struct ice_vsi *vsi) ice_for_each_txq(vsi, i) ice_clean_tx_ring(vsi->tx_rings[i]); - if (ice_is_xdp_ena_vsi(vsi)) + if (vsi->xdp_rings) ice_for_each_xdp_txq(vsi, i) ice_clean_tx_ring(vsi->xdp_rings[i]); @@ -7443,7 +7461,7 @@ int ice_vsi_open(struct ice_vsi *vsi) ice_vsi_cfg_netdev_tc(vsi, vsi->tc_cfg.ena_tc); - if (vsi->type == ICE_VSI_PF) { + if (vsi->type == ICE_VSI_PF || vsi->type == ICE_VSI_SF) { /* Notify the stack of the actual queue counts. */ err = netif_set_real_num_tx_queues(vsi->netdev, vsi->num_txq); if (err) @@ -7452,6 +7470,8 @@ int ice_vsi_open(struct ice_vsi *vsi) err = netif_set_real_num_rx_queues(vsi->netdev, vsi->num_rxq); if (err) goto err_set_qs; + + ice_vsi_set_napi_queues(vsi); } err = ice_up_complete(vsi); @@ -7589,6 +7609,7 @@ static void ice_update_pf_netdev_link(struct ice_pf *pf) */ static void ice_rebuild(struct ice_pf *pf, enum ice_reset_req reset_type) { + struct ice_vsi *vsi = ice_get_main_vsi(pf); struct device *dev = ice_pf_to_dev(pf); struct ice_hw *hw = &pf->hw; bool dvm; @@ -7731,6 +7752,9 @@ static void ice_rebuild(struct ice_pf *pf, enum ice_reset_req reset_type) ice_rebuild_arfs(pf); } + if (vsi && vsi->netdev) + netif_device_attach(vsi->netdev); + ice_update_pf_netdev_link(pf); /* tell the firmware we are up */ @@ -7773,7 +7797,7 @@ clear_recovery: * * Returns 0 on success, negative on failure */ -static int ice_change_mtu(struct net_device *netdev, int new_mtu) +int ice_change_mtu(struct net_device *netdev, int new_mtu) { struct ice_netdev_priv *np = netdev_priv(netdev); struct ice_vsi *vsi = np->vsi; @@ -8197,7 +8221,7 @@ ice_bridge_setlink(struct net_device *dev, struct nlmsghdr *nlh, * @netdev: network interface device structure * @txqueue: Tx queue */ -static void ice_tx_timeout(struct net_device *netdev, unsigned int txqueue) +void ice_tx_timeout(struct net_device *netdev, unsigned int txqueue) { struct ice_netdev_priv *np = netdev_priv(netdev); struct ice_tx_ring *tx_ring = NULL; diff --git a/drivers/net/ethernet/intel/ice/ice_osdep.h b/drivers/net/ethernet/intel/ice/ice_osdep.h index a2562f04267f..b9f383494b3f 100644 --- a/drivers/net/ethernet/intel/ice/ice_osdep.h +++ b/drivers/net/ethernet/intel/ice/ice_osdep.h @@ -12,6 +12,7 @@ #include <linux/ethtool.h> #include <linux/etherdevice.h> #include <linux/if_ether.h> +#include <linux/iopoll.h> #include <linux/pci_ids.h> #ifndef CONFIG_64BIT #include <linux/io-64-nonatomic-lo-hi.h> @@ -23,6 +24,9 @@ #define wr64(a, reg, value) writeq((value), ((a)->hw_addr + (reg))) #define rd64(a, reg) readq((a)->hw_addr + (reg)) +#define rd32_poll_timeout(a, addr, val, cond, delay_us, timeout_us) \ + read_poll_timeout(rd32, val, cond, delay_us, timeout_us, false, a, addr) + #define ice_flush(a) rd32((a), GLGEN_STAT) #define ICE_M(m, s) ((m ## U) << (s)) @@ -39,11 +43,10 @@ struct device *ice_hw_to_dev(struct ice_hw *hw); #define ice_debug(hw, type, fmt, args...) \ dev_dbg(ice_hw_to_dev(hw), fmt, ##args) -#define ice_debug_array(hw, type, rowsize, groupsize, buf, len) \ - print_hex_dump_debug(KBUILD_MODNAME " ", \ - DUMP_PREFIX_OFFSET, rowsize, \ - groupsize, buf, len, false) -#else +#define _ice_debug_array(hw, type, prefix, rowsize, groupsize, buf, len) \ + print_hex_dump_debug(prefix, DUMP_PREFIX_OFFSET, \ + rowsize, groupsize, buf, len, false) +#else /* CONFIG_DYNAMIC_DEBUG */ #define ice_debug(hw, type, fmt, args...) \ do { \ if ((type) & (hw)->debug_mask) \ @@ -51,16 +54,15 @@ do { \ } while (0) #ifdef DEBUG -#define ice_debug_array(hw, type, rowsize, groupsize, buf, len) \ +#define _ice_debug_array(hw, type, prefix, rowsize, groupsize, buf, len) \ do { \ if ((type) & (hw)->debug_mask) \ - print_hex_dump_debug(KBUILD_MODNAME, \ - DUMP_PREFIX_OFFSET, \ + print_hex_dump_debug(prefix, DUMP_PREFIX_OFFSET,\ rowsize, groupsize, buf, \ len, false); \ } while (0) -#else -#define ice_debug_array(hw, type, rowsize, groupsize, buf, len) \ +#else /* DEBUG */ +#define _ice_debug_array(hw, type, prefix, rowsize, groupsize, buf, len) \ do { \ struct ice_hw *hw_l = hw; \ if ((type) & (hw_l)->debug_mask) { \ @@ -78,4 +80,10 @@ do { \ #endif /* DEBUG */ #endif /* CONFIG_DYNAMIC_DEBUG */ +#define ice_debug_array(hw, type, rowsize, groupsize, buf, len) \ + _ice_debug_array(hw, type, KBUILD_MODNAME, rowsize, groupsize, buf, len) + +#define ice_debug_array_w_prefix(hw, type, prefix, buf, len) \ + _ice_debug_array(hw, type, prefix, 16, 1, buf, len) + #endif /* _ICE_OSDEP_H_ */ diff --git a/drivers/net/ethernet/intel/ice/ice_parser.c b/drivers/net/ethernet/intel/ice/ice_parser.c new file mode 100644 index 000000000000..664beb64f557 --- /dev/null +++ b/drivers/net/ethernet/intel/ice/ice_parser.c @@ -0,0 +1,2430 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (C) 2024 Intel Corporation */ + +#include "ice_common.h" + +struct ice_pkg_sect_hdr { + __le16 count; + __le16 offset; +}; + +/** + * ice_parser_sect_item_get - parse an item from a section + * @sect_type: section type + * @section: section object + * @index: index of the item to get + * @offset: dummy as prototype of ice_pkg_enum_entry's last parameter + * + * Return: a pointer to the item or NULL. + */ +static void *ice_parser_sect_item_get(u32 sect_type, void *section, + u32 index, u32 __maybe_unused *offset) +{ + size_t data_off = ICE_SEC_DATA_OFFSET; + struct ice_pkg_sect_hdr *hdr; + size_t size; + + if (!section) + return NULL; + + switch (sect_type) { + case ICE_SID_RXPARSER_IMEM: + size = ICE_SID_RXPARSER_IMEM_ENTRY_SIZE; + break; + case ICE_SID_RXPARSER_METADATA_INIT: + size = ICE_SID_RXPARSER_METADATA_INIT_ENTRY_SIZE; + break; + case ICE_SID_RXPARSER_CAM: + size = ICE_SID_RXPARSER_CAM_ENTRY_SIZE; + break; + case ICE_SID_RXPARSER_PG_SPILL: + size = ICE_SID_RXPARSER_PG_SPILL_ENTRY_SIZE; + break; + case ICE_SID_RXPARSER_NOMATCH_CAM: + size = ICE_SID_RXPARSER_NOMATCH_CAM_ENTRY_SIZE; + break; + case ICE_SID_RXPARSER_NOMATCH_SPILL: + size = ICE_SID_RXPARSER_NOMATCH_SPILL_ENTRY_SIZE; + break; + case ICE_SID_RXPARSER_BOOST_TCAM: + size = ICE_SID_RXPARSER_BOOST_TCAM_ENTRY_SIZE; + break; + case ICE_SID_LBL_RXPARSER_TMEM: + data_off = ICE_SEC_LBL_DATA_OFFSET; + size = ICE_SID_LBL_ENTRY_SIZE; + break; + case ICE_SID_RXPARSER_MARKER_PTYPE: + size = ICE_SID_RXPARSER_MARKER_TYPE_ENTRY_SIZE; + break; + case ICE_SID_RXPARSER_MARKER_GRP: + size = ICE_SID_RXPARSER_MARKER_GRP_ENTRY_SIZE; + break; + case ICE_SID_RXPARSER_PROTO_GRP: + size = ICE_SID_RXPARSER_PROTO_GRP_ENTRY_SIZE; + break; + case ICE_SID_RXPARSER_FLAG_REDIR: + size = ICE_SID_RXPARSER_FLAG_REDIR_ENTRY_SIZE; + break; + default: + return NULL; + } + + hdr = section; + if (index >= le16_to_cpu(hdr->count)) + return NULL; + + return section + data_off + index * size; +} + +/** + * ice_parser_create_table - create an item table from a section + * @hw: pointer to the hardware structure + * @sect_type: section type + * @item_size: item size in bytes + * @length: number of items in the table to create + * @parse_item: the function to parse the item + * @no_offset: ignore header offset, calculate index from 0 + * + * Return: a pointer to the allocated table or ERR_PTR. + */ +static void * +ice_parser_create_table(struct ice_hw *hw, u32 sect_type, + u32 item_size, u32 length, + void (*parse_item)(struct ice_hw *hw, u16 idx, + void *item, void *data, + int size), bool no_offset) +{ + struct ice_pkg_enum state = {}; + struct ice_seg *seg = hw->seg; + void *table, *data, *item; + u16 idx = 0; + + if (!seg) + return ERR_PTR(-EINVAL); + + table = kzalloc(item_size * length, GFP_KERNEL); + if (!table) + return ERR_PTR(-ENOMEM); + + do { + data = ice_pkg_enum_entry(seg, &state, sect_type, NULL, + ice_parser_sect_item_get); + seg = NULL; + if (data) { + struct ice_pkg_sect_hdr *hdr = state.sect; + + if (!no_offset) + idx = le16_to_cpu(hdr->offset) + + state.entry_idx; + + item = (void *)((uintptr_t)table + idx * item_size); + parse_item(hw, idx, item, data, item_size); + + if (no_offset) + idx++; + } + } while (data); + + return table; +} + +/*** ICE_SID_RXPARSER_IMEM section ***/ +static void ice_imem_bst_bm_dump(struct ice_hw *hw, struct ice_bst_main *bm) +{ + struct device *dev = ice_hw_to_dev(hw); + + dev_info(dev, "boost main:\n"); + dev_info(dev, "\talu0 = %d\n", bm->alu0); + dev_info(dev, "\talu1 = %d\n", bm->alu1); + dev_info(dev, "\talu2 = %d\n", bm->alu2); + dev_info(dev, "\tpg = %d\n", bm->pg); +} + +static void ice_imem_bst_kb_dump(struct ice_hw *hw, + struct ice_bst_keybuilder *kb) +{ + struct device *dev = ice_hw_to_dev(hw); + + dev_info(dev, "boost key builder:\n"); + dev_info(dev, "\tpriority = %d\n", kb->prio); + dev_info(dev, "\ttsr_ctrl = %d\n", kb->tsr_ctrl); +} + +static void ice_imem_np_kb_dump(struct ice_hw *hw, + struct ice_np_keybuilder *kb) +{ + struct device *dev = ice_hw_to_dev(hw); + + dev_info(dev, "next proto key builder:\n"); + dev_info(dev, "\topc = %d\n", kb->opc); + dev_info(dev, "\tstart_or_reg0 = %d\n", kb->start_reg0); + dev_info(dev, "\tlen_or_reg1 = %d\n", kb->len_reg1); +} + +static void ice_imem_pg_kb_dump(struct ice_hw *hw, + struct ice_pg_keybuilder *kb) +{ + struct device *dev = ice_hw_to_dev(hw); + + dev_info(dev, "parse graph key builder:\n"); + dev_info(dev, "\tflag0_ena = %d\n", kb->flag0_ena); + dev_info(dev, "\tflag1_ena = %d\n", kb->flag1_ena); + dev_info(dev, "\tflag2_ena = %d\n", kb->flag2_ena); + dev_info(dev, "\tflag3_ena = %d\n", kb->flag3_ena); + dev_info(dev, "\tflag0_idx = %d\n", kb->flag0_idx); + dev_info(dev, "\tflag1_idx = %d\n", kb->flag1_idx); + dev_info(dev, "\tflag2_idx = %d\n", kb->flag2_idx); + dev_info(dev, "\tflag3_idx = %d\n", kb->flag3_idx); + dev_info(dev, "\talu_reg_idx = %d\n", kb->alu_reg_idx); +} + +static void ice_imem_alu_dump(struct ice_hw *hw, + struct ice_alu *alu, int index) +{ + struct device *dev = ice_hw_to_dev(hw); + + dev_info(dev, "alu%d:\n", index); + dev_info(dev, "\topc = %d\n", alu->opc); + dev_info(dev, "\tsrc_start = %d\n", alu->src_start); + dev_info(dev, "\tsrc_len = %d\n", alu->src_len); + dev_info(dev, "\tshift_xlate_sel = %d\n", alu->shift_xlate_sel); + dev_info(dev, "\tshift_xlate_key = %d\n", alu->shift_xlate_key); + dev_info(dev, "\tsrc_reg_id = %d\n", alu->src_reg_id); + dev_info(dev, "\tdst_reg_id = %d\n", alu->dst_reg_id); + dev_info(dev, "\tinc0 = %d\n", alu->inc0); + dev_info(dev, "\tinc1 = %d\n", alu->inc1); + dev_info(dev, "\tproto_offset_opc = %d\n", alu->proto_offset_opc); + dev_info(dev, "\tproto_offset = %d\n", alu->proto_offset); + dev_info(dev, "\tbranch_addr = %d\n", alu->branch_addr); + dev_info(dev, "\timm = %d\n", alu->imm); + dev_info(dev, "\tdst_start = %d\n", alu->dst_start); + dev_info(dev, "\tdst_len = %d\n", alu->dst_len); + dev_info(dev, "\tflags_extr_imm = %d\n", alu->flags_extr_imm); + dev_info(dev, "\tflags_start_imm= %d\n", alu->flags_start_imm); +} + +/** + * ice_imem_dump - dump an imem item info + * @hw: pointer to the hardware structure + * @item: imem item to dump + */ +static void ice_imem_dump(struct ice_hw *hw, struct ice_imem_item *item) +{ + struct device *dev = ice_hw_to_dev(hw); + + dev_info(dev, "index = %d\n", item->idx); + ice_imem_bst_bm_dump(hw, &item->b_m); + ice_imem_bst_kb_dump(hw, &item->b_kb); + dev_info(dev, "pg priority = %d\n", item->pg_prio); + ice_imem_np_kb_dump(hw, &item->np_kb); + ice_imem_pg_kb_dump(hw, &item->pg_kb); + ice_imem_alu_dump(hw, &item->alu0, 0); + ice_imem_alu_dump(hw, &item->alu1, 1); + ice_imem_alu_dump(hw, &item->alu2, 2); +} + +#define ICE_IM_BM_ALU0 BIT(0) +#define ICE_IM_BM_ALU1 BIT(1) +#define ICE_IM_BM_ALU2 BIT(2) +#define ICE_IM_BM_PG BIT(3) + +/** + * ice_imem_bm_init - parse 4 bits of Boost Main + * @bm: pointer to the Boost Main structure + * @data: Boost Main data to be parsed + */ +static void ice_imem_bm_init(struct ice_bst_main *bm, u8 data) +{ + bm->alu0 = FIELD_GET(ICE_IM_BM_ALU0, data); + bm->alu1 = FIELD_GET(ICE_IM_BM_ALU1, data); + bm->alu2 = FIELD_GET(ICE_IM_BM_ALU2, data); + bm->pg = FIELD_GET(ICE_IM_BM_PG, data); +} + +#define ICE_IM_BKB_PRIO GENMASK(7, 0) +#define ICE_IM_BKB_TSR_CTRL BIT(8) + +/** + * ice_imem_bkb_init - parse 10 bits of Boost Main Build + * @bkb: pointer to the Boost Main Build structure + * @data: Boost Main Build data to be parsed + */ +static void ice_imem_bkb_init(struct ice_bst_keybuilder *bkb, u16 data) +{ + bkb->prio = FIELD_GET(ICE_IM_BKB_PRIO, data); + bkb->tsr_ctrl = FIELD_GET(ICE_IM_BKB_TSR_CTRL, data); +} + +#define ICE_IM_NPKB_OPC GENMASK(1, 0) +#define ICE_IM_NPKB_S_R0 GENMASK(9, 2) +#define ICE_IM_NPKB_L_R1 GENMASK(17, 10) + +/** + * ice_imem_npkb_init - parse 18 bits of Next Protocol Key Build + * @kb: pointer to the Next Protocol Key Build structure + * @data: Next Protocol Key Build data to be parsed + */ +static void ice_imem_npkb_init(struct ice_np_keybuilder *kb, u32 data) +{ + kb->opc = FIELD_GET(ICE_IM_NPKB_OPC, data); + kb->start_reg0 = FIELD_GET(ICE_IM_NPKB_S_R0, data); + kb->len_reg1 = FIELD_GET(ICE_IM_NPKB_L_R1, data); +} + +#define ICE_IM_PGKB_F0_ENA BIT_ULL(0) +#define ICE_IM_PGKB_F0_IDX GENMASK_ULL(6, 1) +#define ICE_IM_PGKB_F1_ENA BIT_ULL(7) +#define ICE_IM_PGKB_F1_IDX GENMASK_ULL(13, 8) +#define ICE_IM_PGKB_F2_ENA BIT_ULL(14) +#define ICE_IM_PGKB_F2_IDX GENMASK_ULL(20, 15) +#define ICE_IM_PGKB_F3_ENA BIT_ULL(21) +#define ICE_IM_PGKB_F3_IDX GENMASK_ULL(27, 22) +#define ICE_IM_PGKB_AR_IDX GENMASK_ULL(34, 28) + +/** + * ice_imem_pgkb_init - parse 35 bits of Parse Graph Key Build + * @kb: pointer to the Parse Graph Key Build structure + * @data: Parse Graph Key Build data to be parsed + */ +static void ice_imem_pgkb_init(struct ice_pg_keybuilder *kb, u64 data) +{ + kb->flag0_ena = FIELD_GET(ICE_IM_PGKB_F0_ENA, data); + kb->flag0_idx = FIELD_GET(ICE_IM_PGKB_F0_IDX, data); + kb->flag1_ena = FIELD_GET(ICE_IM_PGKB_F1_ENA, data); + kb->flag1_idx = FIELD_GET(ICE_IM_PGKB_F1_IDX, data); + kb->flag2_ena = FIELD_GET(ICE_IM_PGKB_F2_ENA, data); + kb->flag2_idx = FIELD_GET(ICE_IM_PGKB_F2_IDX, data); + kb->flag3_ena = FIELD_GET(ICE_IM_PGKB_F3_ENA, data); + kb->flag3_idx = FIELD_GET(ICE_IM_PGKB_F3_IDX, data); + kb->alu_reg_idx = FIELD_GET(ICE_IM_PGKB_AR_IDX, data); +} + +#define ICE_IM_ALU_OPC GENMASK_ULL(5, 0) +#define ICE_IM_ALU_SS GENMASK_ULL(13, 6) +#define ICE_IM_ALU_SL GENMASK_ULL(18, 14) +#define ICE_IM_ALU_SXS BIT_ULL(19) +#define ICE_IM_ALU_SXK GENMASK_ULL(23, 20) +#define ICE_IM_ALU_SRID GENMASK_ULL(30, 24) +#define ICE_IM_ALU_DRID GENMASK_ULL(37, 31) +#define ICE_IM_ALU_INC0 BIT_ULL(38) +#define ICE_IM_ALU_INC1 BIT_ULL(39) +#define ICE_IM_ALU_POO GENMASK_ULL(41, 40) +#define ICE_IM_ALU_PO GENMASK_ULL(49, 42) +#define ICE_IM_ALU_BA_S 50 /* offset for the 2nd 64-bits field */ +#define ICE_IM_ALU_BA GENMASK_ULL(57 - ICE_IM_ALU_BA_S, \ + 50 - ICE_IM_ALU_BA_S) +#define ICE_IM_ALU_IMM GENMASK_ULL(73 - ICE_IM_ALU_BA_S, \ + 58 - ICE_IM_ALU_BA_S) +#define ICE_IM_ALU_DFE BIT_ULL(74 - ICE_IM_ALU_BA_S) +#define ICE_IM_ALU_DS GENMASK_ULL(80 - ICE_IM_ALU_BA_S, \ + 75 - ICE_IM_ALU_BA_S) +#define ICE_IM_ALU_DL GENMASK_ULL(86 - ICE_IM_ALU_BA_S, \ + 81 - ICE_IM_ALU_BA_S) +#define ICE_IM_ALU_FEI BIT_ULL(87 - ICE_IM_ALU_BA_S) +#define ICE_IM_ALU_FSI GENMASK_ULL(95 - ICE_IM_ALU_BA_S, \ + 88 - ICE_IM_ALU_BA_S) + +/** + * ice_imem_alu_init - parse 96 bits of ALU entry + * @alu: pointer to the ALU entry structure + * @data: ALU entry data to be parsed + * @off: offset of the ALU entry data + */ +static void ice_imem_alu_init(struct ice_alu *alu, u8 *data, u8 off) +{ + u64 d64; + u8 idd; + + d64 = *((u64 *)data) >> off; + + alu->opc = FIELD_GET(ICE_IM_ALU_OPC, d64); + alu->src_start = FIELD_GET(ICE_IM_ALU_SS, d64); + alu->src_len = FIELD_GET(ICE_IM_ALU_SL, d64); + alu->shift_xlate_sel = FIELD_GET(ICE_IM_ALU_SXS, d64); + alu->shift_xlate_key = FIELD_GET(ICE_IM_ALU_SXK, d64); + alu->src_reg_id = FIELD_GET(ICE_IM_ALU_SRID, d64); + alu->dst_reg_id = FIELD_GET(ICE_IM_ALU_DRID, d64); + alu->inc0 = FIELD_GET(ICE_IM_ALU_INC0, d64); + alu->inc1 = FIELD_GET(ICE_IM_ALU_INC1, d64); + alu->proto_offset_opc = FIELD_GET(ICE_IM_ALU_POO, d64); + alu->proto_offset = FIELD_GET(ICE_IM_ALU_PO, d64); + + idd = (ICE_IM_ALU_BA_S + off) / BITS_PER_BYTE; + off = (ICE_IM_ALU_BA_S + off) % BITS_PER_BYTE; + d64 = *((u64 *)(&data[idd])) >> off; + + alu->branch_addr = FIELD_GET(ICE_IM_ALU_BA, d64); + alu->imm = FIELD_GET(ICE_IM_ALU_IMM, d64); + alu->dedicate_flags_ena = FIELD_GET(ICE_IM_ALU_DFE, d64); + alu->dst_start = FIELD_GET(ICE_IM_ALU_DS, d64); + alu->dst_len = FIELD_GET(ICE_IM_ALU_DL, d64); + alu->flags_extr_imm = FIELD_GET(ICE_IM_ALU_FEI, d64); + alu->flags_start_imm = FIELD_GET(ICE_IM_ALU_FSI, d64); +} + +#define ICE_IMEM_BM_S 0 +#define ICE_IMEM_BKB_S 4 +#define ICE_IMEM_BKB_IDD (ICE_IMEM_BKB_S / BITS_PER_BYTE) +#define ICE_IMEM_BKB_OFF (ICE_IMEM_BKB_S % BITS_PER_BYTE) +#define ICE_IMEM_PGP GENMASK(15, 14) +#define ICE_IMEM_NPKB_S 16 +#define ICE_IMEM_NPKB_IDD (ICE_IMEM_NPKB_S / BITS_PER_BYTE) +#define ICE_IMEM_NPKB_OFF (ICE_IMEM_NPKB_S % BITS_PER_BYTE) +#define ICE_IMEM_PGKB_S 34 +#define ICE_IMEM_PGKB_IDD (ICE_IMEM_PGKB_S / BITS_PER_BYTE) +#define ICE_IMEM_PGKB_OFF (ICE_IMEM_PGKB_S % BITS_PER_BYTE) +#define ICE_IMEM_ALU0_S 69 +#define ICE_IMEM_ALU0_IDD (ICE_IMEM_ALU0_S / BITS_PER_BYTE) +#define ICE_IMEM_ALU0_OFF (ICE_IMEM_ALU0_S % BITS_PER_BYTE) +#define ICE_IMEM_ALU1_S 165 +#define ICE_IMEM_ALU1_IDD (ICE_IMEM_ALU1_S / BITS_PER_BYTE) +#define ICE_IMEM_ALU1_OFF (ICE_IMEM_ALU1_S % BITS_PER_BYTE) +#define ICE_IMEM_ALU2_S 357 +#define ICE_IMEM_ALU2_IDD (ICE_IMEM_ALU2_S / BITS_PER_BYTE) +#define ICE_IMEM_ALU2_OFF (ICE_IMEM_ALU2_S % BITS_PER_BYTE) + +/** + * ice_imem_parse_item - parse 384 bits of IMEM entry + * @hw: pointer to the hardware structure + * @idx: index of IMEM entry + * @item: item of IMEM entry + * @data: IMEM entry data to be parsed + * @size: size of IMEM entry + */ +static void ice_imem_parse_item(struct ice_hw *hw, u16 idx, void *item, + void *data, int __maybe_unused size) +{ + struct ice_imem_item *ii = item; + u8 *buf = data; + + ii->idx = idx; + + ice_imem_bm_init(&ii->b_m, *(u8 *)buf); + ice_imem_bkb_init(&ii->b_kb, + *((u16 *)(&buf[ICE_IMEM_BKB_IDD])) >> + ICE_IMEM_BKB_OFF); + + ii->pg_prio = FIELD_GET(ICE_IMEM_PGP, *(u16 *)buf); + + ice_imem_npkb_init(&ii->np_kb, + *((u32 *)(&buf[ICE_IMEM_NPKB_IDD])) >> + ICE_IMEM_NPKB_OFF); + ice_imem_pgkb_init(&ii->pg_kb, + *((u64 *)(&buf[ICE_IMEM_PGKB_IDD])) >> + ICE_IMEM_PGKB_OFF); + + ice_imem_alu_init(&ii->alu0, + &buf[ICE_IMEM_ALU0_IDD], + ICE_IMEM_ALU0_OFF); + ice_imem_alu_init(&ii->alu1, + &buf[ICE_IMEM_ALU1_IDD], + ICE_IMEM_ALU1_OFF); + ice_imem_alu_init(&ii->alu2, + &buf[ICE_IMEM_ALU2_IDD], + ICE_IMEM_ALU2_OFF); + + if (hw->debug_mask & ICE_DBG_PARSER) + ice_imem_dump(hw, ii); +} + +/** + * ice_imem_table_get - create an imem table + * @hw: pointer to the hardware structure + * + * Return: a pointer to the allocated IMEM table. + */ +static struct ice_imem_item *ice_imem_table_get(struct ice_hw *hw) +{ + return ice_parser_create_table(hw, ICE_SID_RXPARSER_IMEM, + sizeof(struct ice_imem_item), + ICE_IMEM_TABLE_SIZE, + ice_imem_parse_item, false); +} + +/*** ICE_SID_RXPARSER_METADATA_INIT section ***/ +/** + * ice_metainit_dump - dump an metainit item info + * @hw: pointer to the hardware structure + * @item: metainit item to dump + */ +static void ice_metainit_dump(struct ice_hw *hw, struct ice_metainit_item *item) +{ + struct device *dev = ice_hw_to_dev(hw); + + dev_info(dev, "index = %d\n", item->idx); + + dev_info(dev, "tsr = %d\n", item->tsr); + dev_info(dev, "ho = %d\n", item->ho); + dev_info(dev, "pc = %d\n", item->pc); + dev_info(dev, "pg_rn = %d\n", item->pg_rn); + dev_info(dev, "cd = %d\n", item->cd); + + dev_info(dev, "gpr_a_ctrl = %d\n", item->gpr_a_ctrl); + dev_info(dev, "gpr_a_data_mdid = %d\n", item->gpr_a_data_mdid); + dev_info(dev, "gpr_a_data_start = %d\n", item->gpr_a_data_start); + dev_info(dev, "gpr_a_data_len = %d\n", item->gpr_a_data_len); + dev_info(dev, "gpr_a_id = %d\n", item->gpr_a_id); + + dev_info(dev, "gpr_b_ctrl = %d\n", item->gpr_b_ctrl); + dev_info(dev, "gpr_b_data_mdid = %d\n", item->gpr_b_data_mdid); + dev_info(dev, "gpr_b_data_start = %d\n", item->gpr_b_data_start); + dev_info(dev, "gpr_b_data_len = %d\n", item->gpr_b_data_len); + dev_info(dev, "gpr_b_id = %d\n", item->gpr_b_id); + + dev_info(dev, "gpr_c_ctrl = %d\n", item->gpr_c_ctrl); + dev_info(dev, "gpr_c_data_mdid = %d\n", item->gpr_c_data_mdid); + dev_info(dev, "gpr_c_data_start = %d\n", item->gpr_c_data_start); + dev_info(dev, "gpr_c_data_len = %d\n", item->gpr_c_data_len); + dev_info(dev, "gpr_c_id = %d\n", item->gpr_c_id); + + dev_info(dev, "gpr_d_ctrl = %d\n", item->gpr_d_ctrl); + dev_info(dev, "gpr_d_data_mdid = %d\n", item->gpr_d_data_mdid); + dev_info(dev, "gpr_d_data_start = %d\n", item->gpr_d_data_start); + dev_info(dev, "gpr_d_data_len = %d\n", item->gpr_d_data_len); + dev_info(dev, "gpr_d_id = %d\n", item->gpr_d_id); + + dev_info(dev, "flags = 0x%llx\n", (unsigned long long)(item->flags)); +} + +#define ICE_MI_TSR GENMASK_ULL(7, 0) +#define ICE_MI_HO GENMASK_ULL(16, 8) +#define ICE_MI_PC GENMASK_ULL(24, 17) +#define ICE_MI_PGRN GENMASK_ULL(35, 25) +#define ICE_MI_CD GENMASK_ULL(38, 36) +#define ICE_MI_GAC BIT_ULL(39) +#define ICE_MI_GADM GENMASK_ULL(44, 40) +#define ICE_MI_GADS GENMASK_ULL(48, 45) +#define ICE_MI_GADL GENMASK_ULL(53, 49) +#define ICE_MI_GAI GENMASK_ULL(59, 56) +#define ICE_MI_GBC BIT_ULL(60) +#define ICE_MI_GBDM_S 61 /* offset for the 2nd 64-bits field */ +#define ICE_MI_GBDM_IDD (ICE_MI_GBDM_S / BITS_PER_BYTE) +#define ICE_MI_GBDM_OFF (ICE_MI_GBDM_S % BITS_PER_BYTE) + +#define ICE_MI_GBDM_GENMASK_ULL(high, low) \ + GENMASK_ULL((high) - ICE_MI_GBDM_S, (low) - ICE_MI_GBDM_S) +#define ICE_MI_GBDM ICE_MI_GBDM_GENMASK_ULL(65, 61) +#define ICE_MI_GBDS ICE_MI_GBDM_GENMASK_ULL(69, 66) +#define ICE_MI_GBDL ICE_MI_GBDM_GENMASK_ULL(74, 70) +#define ICE_MI_GBI ICE_MI_GBDM_GENMASK_ULL(80, 77) +#define ICE_MI_GCC BIT_ULL(81 - ICE_MI_GBDM_S) +#define ICE_MI_GCDM ICE_MI_GBDM_GENMASK_ULL(86, 82) +#define ICE_MI_GCDS ICE_MI_GBDM_GENMASK_ULL(90, 87) +#define ICE_MI_GCDL ICE_MI_GBDM_GENMASK_ULL(95, 91) +#define ICE_MI_GCI ICE_MI_GBDM_GENMASK_ULL(101, 98) +#define ICE_MI_GDC BIT_ULL(102 - ICE_MI_GBDM_S) +#define ICE_MI_GDDM ICE_MI_GBDM_GENMASK_ULL(107, 103) +#define ICE_MI_GDDS ICE_MI_GBDM_GENMASK_ULL(111, 108) +#define ICE_MI_GDDL ICE_MI_GBDM_GENMASK_ULL(116, 112) +#define ICE_MI_GDI ICE_MI_GBDM_GENMASK_ULL(122, 119) +#define ICE_MI_FLAG_S 123 /* offset for the 3rd 64-bits field */ +#define ICE_MI_FLAG_IDD (ICE_MI_FLAG_S / BITS_PER_BYTE) +#define ICE_MI_FLAG_OFF (ICE_MI_FLAG_S % BITS_PER_BYTE) +#define ICE_MI_FLAG GENMASK_ULL(186 - ICE_MI_FLAG_S, \ + 123 - ICE_MI_FLAG_S) + +/** + * ice_metainit_parse_item - parse 192 bits of Metadata Init entry + * @hw: pointer to the hardware structure + * @idx: index of Metadata Init entry + * @item: item of Metadata Init entry + * @data: Metadata Init entry data to be parsed + * @size: size of Metadata Init entry + */ +static void ice_metainit_parse_item(struct ice_hw *hw, u16 idx, void *item, + void *data, int __maybe_unused size) +{ + struct ice_metainit_item *mi = item; + u8 *buf = data; + u64 d64; + + mi->idx = idx; + + d64 = *(u64 *)buf; + + mi->tsr = FIELD_GET(ICE_MI_TSR, d64); + mi->ho = FIELD_GET(ICE_MI_HO, d64); + mi->pc = FIELD_GET(ICE_MI_PC, d64); + mi->pg_rn = FIELD_GET(ICE_MI_PGRN, d64); + mi->cd = FIELD_GET(ICE_MI_CD, d64); + + mi->gpr_a_ctrl = FIELD_GET(ICE_MI_GAC, d64); + mi->gpr_a_data_mdid = FIELD_GET(ICE_MI_GADM, d64); + mi->gpr_a_data_start = FIELD_GET(ICE_MI_GADS, d64); + mi->gpr_a_data_len = FIELD_GET(ICE_MI_GADL, d64); + mi->gpr_a_id = FIELD_GET(ICE_MI_GAI, d64); + + mi->gpr_b_ctrl = FIELD_GET(ICE_MI_GBC, d64); + + d64 = *((u64 *)&buf[ICE_MI_GBDM_IDD]) >> ICE_MI_GBDM_OFF; + + mi->gpr_b_data_mdid = FIELD_GET(ICE_MI_GBDM, d64); + mi->gpr_b_data_start = FIELD_GET(ICE_MI_GBDS, d64); + mi->gpr_b_data_len = FIELD_GET(ICE_MI_GBDL, d64); + mi->gpr_b_id = FIELD_GET(ICE_MI_GBI, d64); + + mi->gpr_c_ctrl = FIELD_GET(ICE_MI_GCC, d64); + mi->gpr_c_data_mdid = FIELD_GET(ICE_MI_GCDM, d64); + mi->gpr_c_data_start = FIELD_GET(ICE_MI_GCDS, d64); + mi->gpr_c_data_len = FIELD_GET(ICE_MI_GCDL, d64); + mi->gpr_c_id = FIELD_GET(ICE_MI_GCI, d64); + + mi->gpr_d_ctrl = FIELD_GET(ICE_MI_GDC, d64); + mi->gpr_d_data_mdid = FIELD_GET(ICE_MI_GDDM, d64); + mi->gpr_d_data_start = FIELD_GET(ICE_MI_GDDS, d64); + mi->gpr_d_data_len = FIELD_GET(ICE_MI_GDDL, d64); + mi->gpr_d_id = FIELD_GET(ICE_MI_GDI, d64); + + d64 = *((u64 *)&buf[ICE_MI_FLAG_IDD]) >> ICE_MI_FLAG_OFF; + + mi->flags = FIELD_GET(ICE_MI_FLAG, d64); + + if (hw->debug_mask & ICE_DBG_PARSER) + ice_metainit_dump(hw, mi); +} + +/** + * ice_metainit_table_get - create a metainit table + * @hw: pointer to the hardware structure + * + * Return: a pointer to the allocated Metadata initialization table. + */ +static struct ice_metainit_item *ice_metainit_table_get(struct ice_hw *hw) +{ + return ice_parser_create_table(hw, ICE_SID_RXPARSER_METADATA_INIT, + sizeof(struct ice_metainit_item), + ICE_METAINIT_TABLE_SIZE, + ice_metainit_parse_item, false); +} + +/** + * ice_bst_tcam_search - find a TCAM item with specific type + * @tcam_table: the TCAM table + * @lbl_table: the lbl table to search + * @type: the type we need to match against + * @start: start searching from this index + * + * Return: a pointer to the matching BOOST TCAM item or NULL. + */ +struct ice_bst_tcam_item * +ice_bst_tcam_search(struct ice_bst_tcam_item *tcam_table, + struct ice_lbl_item *lbl_table, + enum ice_lbl_type type, u16 *start) +{ + u16 i = *start; + + for (; i < ICE_BST_TCAM_TABLE_SIZE; i++) { + if (lbl_table[i].type == type) { + *start = i; + return &tcam_table[lbl_table[i].idx]; + } + } + + return NULL; +} + +/*** ICE_SID_RXPARSER_CAM, ICE_SID_RXPARSER_PG_SPILL, + * ICE_SID_RXPARSER_NOMATCH_CAM and ICE_SID_RXPARSER_NOMATCH_CAM + * sections ***/ +static void ice_pg_cam_key_dump(struct ice_hw *hw, struct ice_pg_cam_key *key) +{ + struct device *dev = ice_hw_to_dev(hw); + + dev_info(dev, "key:\n"); + dev_info(dev, "\tvalid = %d\n", key->valid); + dev_info(dev, "\tnode_id = %d\n", key->node_id); + dev_info(dev, "\tflag0 = %d\n", key->flag0); + dev_info(dev, "\tflag1 = %d\n", key->flag1); + dev_info(dev, "\tflag2 = %d\n", key->flag2); + dev_info(dev, "\tflag3 = %d\n", key->flag3); + dev_info(dev, "\tboost_idx = %d\n", key->boost_idx); + dev_info(dev, "\talu_reg = 0x%04x\n", key->alu_reg); + dev_info(dev, "\tnext_proto = 0x%08x\n", key->next_proto); +} + +static void ice_pg_nm_cam_key_dump(struct ice_hw *hw, + struct ice_pg_nm_cam_key *key) +{ + struct device *dev = ice_hw_to_dev(hw); + + dev_info(dev, "key:\n"); + dev_info(dev, "\tvalid = %d\n", key->valid); + dev_info(dev, "\tnode_id = %d\n", key->node_id); + dev_info(dev, "\tflag0 = %d\n", key->flag0); + dev_info(dev, "\tflag1 = %d\n", key->flag1); + dev_info(dev, "\tflag2 = %d\n", key->flag2); + dev_info(dev, "\tflag3 = %d\n", key->flag3); + dev_info(dev, "\tboost_idx = %d\n", key->boost_idx); + dev_info(dev, "\talu_reg = 0x%04x\n", key->alu_reg); +} + +static void ice_pg_cam_action_dump(struct ice_hw *hw, + struct ice_pg_cam_action *action) +{ + struct device *dev = ice_hw_to_dev(hw); + + dev_info(dev, "action:\n"); + dev_info(dev, "\tnext_node = %d\n", action->next_node); + dev_info(dev, "\tnext_pc = %d\n", action->next_pc); + dev_info(dev, "\tis_pg = %d\n", action->is_pg); + dev_info(dev, "\tproto_id = %d\n", action->proto_id); + dev_info(dev, "\tis_mg = %d\n", action->is_mg); + dev_info(dev, "\tmarker_id = %d\n", action->marker_id); + dev_info(dev, "\tis_last_round = %d\n", action->is_last_round); + dev_info(dev, "\tho_polarity = %d\n", action->ho_polarity); + dev_info(dev, "\tho_inc = %d\n", action->ho_inc); +} + +/** + * ice_pg_cam_dump - dump an parse graph cam info + * @hw: pointer to the hardware structure + * @item: parse graph cam to dump + */ +static void ice_pg_cam_dump(struct ice_hw *hw, struct ice_pg_cam_item *item) +{ + dev_info(ice_hw_to_dev(hw), "index = %d\n", item->idx); + ice_pg_cam_key_dump(hw, &item->key); + ice_pg_cam_action_dump(hw, &item->action); +} + +/** + * ice_pg_nm_cam_dump - dump an parse graph no match cam info + * @hw: pointer to the hardware structure + * @item: parse graph no match cam to dump + */ +static void ice_pg_nm_cam_dump(struct ice_hw *hw, + struct ice_pg_nm_cam_item *item) +{ + dev_info(ice_hw_to_dev(hw), "index = %d\n", item->idx); + ice_pg_nm_cam_key_dump(hw, &item->key); + ice_pg_cam_action_dump(hw, &item->action); +} + +#define ICE_PGCA_NN GENMASK_ULL(10, 0) +#define ICE_PGCA_NPC GENMASK_ULL(18, 11) +#define ICE_PGCA_IPG BIT_ULL(19) +#define ICE_PGCA_PID GENMASK_ULL(30, 23) +#define ICE_PGCA_IMG BIT_ULL(31) +#define ICE_PGCA_MID GENMASK_ULL(39, 32) +#define ICE_PGCA_ILR BIT_ULL(40) +#define ICE_PGCA_HOP BIT_ULL(41) +#define ICE_PGCA_HOI GENMASK_ULL(50, 42) + +/** + * ice_pg_cam_action_init - parse 55 bits of Parse Graph CAM Action + * @action: pointer to the Parse Graph CAM Action structure + * @data: Parse Graph CAM Action data to be parsed + */ +static void ice_pg_cam_action_init(struct ice_pg_cam_action *action, u64 data) +{ + action->next_node = FIELD_GET(ICE_PGCA_NN, data); + action->next_pc = FIELD_GET(ICE_PGCA_NPC, data); + action->is_pg = FIELD_GET(ICE_PGCA_IPG, data); + action->proto_id = FIELD_GET(ICE_PGCA_PID, data); + action->is_mg = FIELD_GET(ICE_PGCA_IMG, data); + action->marker_id = FIELD_GET(ICE_PGCA_MID, data); + action->is_last_round = FIELD_GET(ICE_PGCA_ILR, data); + action->ho_polarity = FIELD_GET(ICE_PGCA_HOP, data); + action->ho_inc = FIELD_GET(ICE_PGCA_HOI, data); +} + +#define ICE_PGNCK_VLD BIT_ULL(0) +#define ICE_PGNCK_NID GENMASK_ULL(11, 1) +#define ICE_PGNCK_F0 BIT_ULL(12) +#define ICE_PGNCK_F1 BIT_ULL(13) +#define ICE_PGNCK_F2 BIT_ULL(14) +#define ICE_PGNCK_F3 BIT_ULL(15) +#define ICE_PGNCK_BH BIT_ULL(16) +#define ICE_PGNCK_BI GENMASK_ULL(24, 17) +#define ICE_PGNCK_AR GENMASK_ULL(40, 25) + +/** + * ice_pg_nm_cam_key_init - parse 41 bits of Parse Graph NoMatch CAM Key + * @key: pointer to the Parse Graph NoMatch CAM Key structure + * @data: Parse Graph NoMatch CAM Key data to be parsed + */ +static void ice_pg_nm_cam_key_init(struct ice_pg_nm_cam_key *key, u64 data) +{ + key->valid = FIELD_GET(ICE_PGNCK_VLD, data); + key->node_id = FIELD_GET(ICE_PGNCK_NID, data); + key->flag0 = FIELD_GET(ICE_PGNCK_F0, data); + key->flag1 = FIELD_GET(ICE_PGNCK_F1, data); + key->flag2 = FIELD_GET(ICE_PGNCK_F2, data); + key->flag3 = FIELD_GET(ICE_PGNCK_F3, data); + + if (FIELD_GET(ICE_PGNCK_BH, data)) + key->boost_idx = FIELD_GET(ICE_PGNCK_BI, data); + else + key->boost_idx = 0; + + key->alu_reg = FIELD_GET(ICE_PGNCK_AR, data); +} + +#define ICE_PGCK_VLD BIT_ULL(0) +#define ICE_PGCK_NID GENMASK_ULL(11, 1) +#define ICE_PGCK_F0 BIT_ULL(12) +#define ICE_PGCK_F1 BIT_ULL(13) +#define ICE_PGCK_F2 BIT_ULL(14) +#define ICE_PGCK_F3 BIT_ULL(15) +#define ICE_PGCK_BH BIT_ULL(16) +#define ICE_PGCK_BI GENMASK_ULL(24, 17) +#define ICE_PGCK_AR GENMASK_ULL(40, 25) +#define ICE_PGCK_NPK_S 41 /* offset for the 2nd 64-bits field */ +#define ICE_PGCK_NPK_IDD (ICE_PGCK_NPK_S / BITS_PER_BYTE) +#define ICE_PGCK_NPK_OFF (ICE_PGCK_NPK_S % BITS_PER_BYTE) +#define ICE_PGCK_NPK GENMASK_ULL(72 - ICE_PGCK_NPK_S, \ + 41 - ICE_PGCK_NPK_S) + +/** + * ice_pg_cam_key_init - parse 73 bits of Parse Graph CAM Key + * @key: pointer to the Parse Graph CAM Key structure + * @data: Parse Graph CAM Key data to be parsed + */ +static void ice_pg_cam_key_init(struct ice_pg_cam_key *key, u8 *data) +{ + u64 d64 = *(u64 *)data; + + key->valid = FIELD_GET(ICE_PGCK_VLD, d64); + key->node_id = FIELD_GET(ICE_PGCK_NID, d64); + key->flag0 = FIELD_GET(ICE_PGCK_F0, d64); + key->flag1 = FIELD_GET(ICE_PGCK_F1, d64); + key->flag2 = FIELD_GET(ICE_PGCK_F2, d64); + key->flag3 = FIELD_GET(ICE_PGCK_F3, d64); + + if (FIELD_GET(ICE_PGCK_BH, d64)) + key->boost_idx = FIELD_GET(ICE_PGCK_BI, d64); + else + key->boost_idx = 0; + + key->alu_reg = FIELD_GET(ICE_PGCK_AR, d64); + + d64 = *((u64 *)&data[ICE_PGCK_NPK_IDD]) >> ICE_PGCK_NPK_OFF; + + key->next_proto = FIELD_GET(ICE_PGCK_NPK, d64); +} + +#define ICE_PG_CAM_ACT_S 73 +#define ICE_PG_CAM_ACT_IDD (ICE_PG_CAM_ACT_S / BITS_PER_BYTE) +#define ICE_PG_CAM_ACT_OFF (ICE_PG_CAM_ACT_S % BITS_PER_BYTE) + +/** + * ice_pg_cam_parse_item - parse 128 bits of Parse Graph CAM Entry + * @hw: pointer to the hardware structure + * @idx: index of Parse Graph CAM Entry + * @item: item of Parse Graph CAM Entry + * @data: Parse Graph CAM Entry data to be parsed + * @size: size of Parse Graph CAM Entry + */ +static void ice_pg_cam_parse_item(struct ice_hw *hw, u16 idx, void *item, + void *data, int __maybe_unused size) +{ + struct ice_pg_cam_item *ci = item; + u8 *buf = data; + u64 d64; + + ci->idx = idx; + + ice_pg_cam_key_init(&ci->key, buf); + + d64 = *((u64 *)&buf[ICE_PG_CAM_ACT_IDD]) >> ICE_PG_CAM_ACT_OFF; + ice_pg_cam_action_init(&ci->action, d64); + + if (hw->debug_mask & ICE_DBG_PARSER) + ice_pg_cam_dump(hw, ci); +} + +#define ICE_PG_SP_CAM_KEY_S 56 +#define ICE_PG_SP_CAM_KEY_IDD (ICE_PG_SP_CAM_KEY_S / BITS_PER_BYTE) + +/** + * ice_pg_sp_cam_parse_item - parse 136 bits of Parse Graph Spill CAM Entry + * @hw: pointer to the hardware structure + * @idx: index of Parse Graph Spill CAM Entry + * @item: item of Parse Graph Spill CAM Entry + * @data: Parse Graph Spill CAM Entry data to be parsed + * @size: size of Parse Graph Spill CAM Entry + */ +static void ice_pg_sp_cam_parse_item(struct ice_hw *hw, u16 idx, void *item, + void *data, int __maybe_unused size) +{ + struct ice_pg_cam_item *ci = item; + u8 *buf = data; + u64 d64; + + ci->idx = idx; + + d64 = *(u64 *)buf; + ice_pg_cam_action_init(&ci->action, d64); + + ice_pg_cam_key_init(&ci->key, &buf[ICE_PG_SP_CAM_KEY_IDD]); + + if (hw->debug_mask & ICE_DBG_PARSER) + ice_pg_cam_dump(hw, ci); +} + +#define ICE_PG_NM_CAM_ACT_S 41 +#define ICE_PG_NM_CAM_ACT_IDD (ICE_PG_NM_CAM_ACT_S / BITS_PER_BYTE) +#define ICE_PG_NM_CAM_ACT_OFF (ICE_PG_NM_CAM_ACT_S % BITS_PER_BYTE) + +/** + * ice_pg_nm_cam_parse_item - parse 96 bits of Parse Graph NoMatch CAM Entry + * @hw: pointer to the hardware structure + * @idx: index of Parse Graph NoMatch CAM Entry + * @item: item of Parse Graph NoMatch CAM Entry + * @data: Parse Graph NoMatch CAM Entry data to be parsed + * @size: size of Parse Graph NoMatch CAM Entry + */ +static void ice_pg_nm_cam_parse_item(struct ice_hw *hw, u16 idx, void *item, + void *data, int __maybe_unused size) +{ + struct ice_pg_nm_cam_item *ci = item; + u8 *buf = data; + u64 d64; + + ci->idx = idx; + + d64 = *(u64 *)buf; + ice_pg_nm_cam_key_init(&ci->key, d64); + + d64 = *((u64 *)&buf[ICE_PG_NM_CAM_ACT_IDD]) >> ICE_PG_NM_CAM_ACT_OFF; + ice_pg_cam_action_init(&ci->action, d64); + + if (hw->debug_mask & ICE_DBG_PARSER) + ice_pg_nm_cam_dump(hw, ci); +} + +#define ICE_PG_NM_SP_CAM_ACT_S 56 +#define ICE_PG_NM_SP_CAM_ACT_IDD (ICE_PG_NM_SP_CAM_ACT_S / BITS_PER_BYTE) +#define ICE_PG_NM_SP_CAM_ACT_OFF (ICE_PG_NM_SP_CAM_ACT_S % BITS_PER_BYTE) + +/** + * ice_pg_nm_sp_cam_parse_item - parse 104 bits of Parse Graph NoMatch Spill + * CAM Entry + * @hw: pointer to the hardware structure + * @idx: index of Parse Graph NoMatch Spill CAM Entry + * @item: item of Parse Graph NoMatch Spill CAM Entry + * @data: Parse Graph NoMatch Spill CAM Entry data to be parsed + * @size: size of Parse Graph NoMatch Spill CAM Entry + */ +static void ice_pg_nm_sp_cam_parse_item(struct ice_hw *hw, u16 idx, + void *item, void *data, + int __maybe_unused size) +{ + struct ice_pg_nm_cam_item *ci = item; + u8 *buf = data; + u64 d64; + + ci->idx = idx; + + d64 = *(u64 *)buf; + ice_pg_cam_action_init(&ci->action, d64); + + d64 = *((u64 *)&buf[ICE_PG_NM_SP_CAM_ACT_IDD]) >> + ICE_PG_NM_SP_CAM_ACT_OFF; + ice_pg_nm_cam_key_init(&ci->key, d64); + + if (hw->debug_mask & ICE_DBG_PARSER) + ice_pg_nm_cam_dump(hw, ci); +} + +/** + * ice_pg_cam_table_get - create a parse graph cam table + * @hw: pointer to the hardware structure + * + * Return: a pointer to the allocated Parse Graph CAM table. + */ +static struct ice_pg_cam_item *ice_pg_cam_table_get(struct ice_hw *hw) +{ + return ice_parser_create_table(hw, ICE_SID_RXPARSER_CAM, + sizeof(struct ice_pg_cam_item), + ICE_PG_CAM_TABLE_SIZE, + ice_pg_cam_parse_item, false); +} + +/** + * ice_pg_sp_cam_table_get - create a parse graph spill cam table + * @hw: pointer to the hardware structure + * + * Return: a pointer to the allocated Parse Graph Spill CAM table. + */ +static struct ice_pg_cam_item *ice_pg_sp_cam_table_get(struct ice_hw *hw) +{ + return ice_parser_create_table(hw, ICE_SID_RXPARSER_PG_SPILL, + sizeof(struct ice_pg_cam_item), + ICE_PG_SP_CAM_TABLE_SIZE, + ice_pg_sp_cam_parse_item, false); +} + +/** + * ice_pg_nm_cam_table_get - create a parse graph no match cam table + * @hw: pointer to the hardware structure + * + * Return: a pointer to the allocated Parse Graph No Match CAM table. + */ +static struct ice_pg_nm_cam_item *ice_pg_nm_cam_table_get(struct ice_hw *hw) +{ + return ice_parser_create_table(hw, ICE_SID_RXPARSER_NOMATCH_CAM, + sizeof(struct ice_pg_nm_cam_item), + ICE_PG_NM_CAM_TABLE_SIZE, + ice_pg_nm_cam_parse_item, false); +} + +/** + * ice_pg_nm_sp_cam_table_get - create a parse graph no match spill cam table + * @hw: pointer to the hardware structure + * + * Return: a pointer to the allocated Parse Graph No Match Spill CAM table. + */ +static struct ice_pg_nm_cam_item *ice_pg_nm_sp_cam_table_get(struct ice_hw *hw) +{ + return ice_parser_create_table(hw, ICE_SID_RXPARSER_NOMATCH_SPILL, + sizeof(struct ice_pg_nm_cam_item), + ICE_PG_NM_SP_CAM_TABLE_SIZE, + ice_pg_nm_sp_cam_parse_item, false); +} + +static bool __ice_pg_cam_match(struct ice_pg_cam_item *item, + struct ice_pg_cam_key *key) +{ + return (item->key.valid && + !memcmp(&item->key.val, &key->val, sizeof(key->val))); +} + +static bool __ice_pg_nm_cam_match(struct ice_pg_nm_cam_item *item, + struct ice_pg_cam_key *key) +{ + return (item->key.valid && + !memcmp(&item->key.val, &key->val, sizeof(item->key.val))); +} + +/** + * ice_pg_cam_match - search parse graph cam table by key + * @table: parse graph cam table to search + * @size: cam table size + * @key: search key + * + * Return: a pointer to the matching PG CAM item or NULL. + */ +struct ice_pg_cam_item *ice_pg_cam_match(struct ice_pg_cam_item *table, + int size, struct ice_pg_cam_key *key) +{ + int i; + + for (i = 0; i < size; i++) { + struct ice_pg_cam_item *item = &table[i]; + + if (__ice_pg_cam_match(item, key)) + return item; + } + + return NULL; +} + +/** + * ice_pg_nm_cam_match - search parse graph no match cam table by key + * @table: parse graph no match cam table to search + * @size: cam table size + * @key: search key + * + * Return: a pointer to the matching PG No Match CAM item or NULL. + */ +struct ice_pg_nm_cam_item * +ice_pg_nm_cam_match(struct ice_pg_nm_cam_item *table, int size, + struct ice_pg_cam_key *key) +{ + int i; + + for (i = 0; i < size; i++) { + struct ice_pg_nm_cam_item *item = &table[i]; + + if (__ice_pg_nm_cam_match(item, key)) + return item; + } + + return NULL; +} + +/*** Ternary match ***/ +/* Perform a ternary match on a 1-byte pattern (@pat) given @key and @key_inv + * Rules (per bit): + * Key == 0 and Key_inv == 0 : Never match (Don't care) + * Key == 0 and Key_inv == 1 : Match on bit == 1 + * Key == 1 and Key_inv == 0 : Match on bit == 0 + * Key == 1 and Key_inv == 1 : Always match (Don't care) + * + * Return: true if all bits match, false otherwise. + */ +static bool ice_ternary_match_byte(u8 key, u8 key_inv, u8 pat) +{ + u8 bit_key, bit_key_inv, bit_pat; + int i; + + for (i = 0; i < BITS_PER_BYTE; i++) { + bit_key = key & BIT(i); + bit_key_inv = key_inv & BIT(i); + bit_pat = pat & BIT(i); + + if (bit_key != 0 && bit_key_inv != 0) + continue; + + if ((bit_key == 0 && bit_key_inv == 0) || bit_key == bit_pat) + return false; + } + + return true; +} + +static bool ice_ternary_match(const u8 *key, const u8 *key_inv, + const u8 *pat, int len) +{ + int i; + + for (i = 0; i < len; i++) + if (!ice_ternary_match_byte(key[i], key_inv[i], pat[i])) + return false; + + return true; +} + +/*** ICE_SID_RXPARSER_BOOST_TCAM and ICE_SID_LBL_RXPARSER_TMEM sections ***/ +static void ice_bst_np_kb_dump(struct ice_hw *hw, struct ice_np_keybuilder *kb) +{ + struct device *dev = ice_hw_to_dev(hw); + + dev_info(dev, "next proto key builder:\n"); + dev_info(dev, "\topc = %d\n", kb->opc); + dev_info(dev, "\tstart_reg0 = %d\n", kb->start_reg0); + dev_info(dev, "\tlen_reg1 = %d\n", kb->len_reg1); +} + +static void ice_bst_pg_kb_dump(struct ice_hw *hw, struct ice_pg_keybuilder *kb) +{ + struct device *dev = ice_hw_to_dev(hw); + + dev_info(dev, "parse graph key builder:\n"); + dev_info(dev, "\tflag0_ena = %d\n", kb->flag0_ena); + dev_info(dev, "\tflag1_ena = %d\n", kb->flag1_ena); + dev_info(dev, "\tflag2_ena = %d\n", kb->flag2_ena); + dev_info(dev, "\tflag3_ena = %d\n", kb->flag3_ena); + dev_info(dev, "\tflag0_idx = %d\n", kb->flag0_idx); + dev_info(dev, "\tflag1_idx = %d\n", kb->flag1_idx); + dev_info(dev, "\tflag2_idx = %d\n", kb->flag2_idx); + dev_info(dev, "\tflag3_idx = %d\n", kb->flag3_idx); + dev_info(dev, "\talu_reg_idx = %d\n", kb->alu_reg_idx); +} + +static void ice_bst_alu_dump(struct ice_hw *hw, struct ice_alu *alu, int idx) +{ + struct device *dev = ice_hw_to_dev(hw); + + dev_info(dev, "alu%d:\n", idx); + dev_info(dev, "\topc = %d\n", alu->opc); + dev_info(dev, "\tsrc_start = %d\n", alu->src_start); + dev_info(dev, "\tsrc_len = %d\n", alu->src_len); + dev_info(dev, "\tshift_xlate_sel = %d\n", alu->shift_xlate_sel); + dev_info(dev, "\tshift_xlate_key = %d\n", alu->shift_xlate_key); + dev_info(dev, "\tsrc_reg_id = %d\n", alu->src_reg_id); + dev_info(dev, "\tdst_reg_id = %d\n", alu->dst_reg_id); + dev_info(dev, "\tinc0 = %d\n", alu->inc0); + dev_info(dev, "\tinc1 = %d\n", alu->inc1); + dev_info(dev, "\tproto_offset_opc = %d\n", alu->proto_offset_opc); + dev_info(dev, "\tproto_offset = %d\n", alu->proto_offset); + dev_info(dev, "\tbranch_addr = %d\n", alu->branch_addr); + dev_info(dev, "\timm = %d\n", alu->imm); + dev_info(dev, "\tdst_start = %d\n", alu->dst_start); + dev_info(dev, "\tdst_len = %d\n", alu->dst_len); + dev_info(dev, "\tflags_extr_imm = %d\n", alu->flags_extr_imm); + dev_info(dev, "\tflags_start_imm= %d\n", alu->flags_start_imm); +} + +/** + * ice_bst_tcam_dump - dump a boost tcam info + * @hw: pointer to the hardware structure + * @item: boost tcam to dump + */ +static void ice_bst_tcam_dump(struct ice_hw *hw, struct ice_bst_tcam_item *item) +{ + struct device *dev = ice_hw_to_dev(hw); + int i; + + dev_info(dev, "addr = %d\n", item->addr); + + dev_info(dev, "key : "); + for (i = 0; i < ICE_BST_TCAM_KEY_SIZE; i++) + dev_info(dev, "%02x ", item->key[i]); + + dev_info(dev, "\n"); + + dev_info(dev, "key_inv: "); + for (i = 0; i < ICE_BST_TCAM_KEY_SIZE; i++) + dev_info(dev, "%02x ", item->key_inv[i]); + + dev_info(dev, "\n"); + + dev_info(dev, "hit_idx_grp = %d\n", item->hit_idx_grp); + dev_info(dev, "pg_prio = %d\n", item->pg_prio); + + ice_bst_np_kb_dump(hw, &item->np_kb); + ice_bst_pg_kb_dump(hw, &item->pg_kb); + + ice_bst_alu_dump(hw, &item->alu0, ICE_ALU0_IDX); + ice_bst_alu_dump(hw, &item->alu1, ICE_ALU1_IDX); + ice_bst_alu_dump(hw, &item->alu2, ICE_ALU2_IDX); +} + +static void ice_lbl_dump(struct ice_hw *hw, struct ice_lbl_item *item) +{ + struct device *dev = ice_hw_to_dev(hw); + + dev_info(dev, "index = %u\n", item->idx); + dev_info(dev, "type = %u\n", item->type); + dev_info(dev, "label = %s\n", item->label); +} + +#define ICE_BST_ALU_OPC GENMASK_ULL(5, 0) +#define ICE_BST_ALU_SS GENMASK_ULL(13, 6) +#define ICE_BST_ALU_SL GENMASK_ULL(18, 14) +#define ICE_BST_ALU_SXS BIT_ULL(19) +#define ICE_BST_ALU_SXK GENMASK_ULL(23, 20) +#define ICE_BST_ALU_SRID GENMASK_ULL(30, 24) +#define ICE_BST_ALU_DRID GENMASK_ULL(37, 31) +#define ICE_BST_ALU_INC0 BIT_ULL(38) +#define ICE_BST_ALU_INC1 BIT_ULL(39) +#define ICE_BST_ALU_POO GENMASK_ULL(41, 40) +#define ICE_BST_ALU_PO GENMASK_ULL(49, 42) +#define ICE_BST_ALU_BA_S 50 /* offset for the 2nd 64-bits field */ +#define ICE_BST_ALU_BA GENMASK_ULL(57 - ICE_BST_ALU_BA_S, \ + 50 - ICE_BST_ALU_BA_S) +#define ICE_BST_ALU_IMM GENMASK_ULL(73 - ICE_BST_ALU_BA_S, \ + 58 - ICE_BST_ALU_BA_S) +#define ICE_BST_ALU_DFE BIT_ULL(74 - ICE_BST_ALU_BA_S) +#define ICE_BST_ALU_DS GENMASK_ULL(80 - ICE_BST_ALU_BA_S, \ + 75 - ICE_BST_ALU_BA_S) +#define ICE_BST_ALU_DL GENMASK_ULL(86 - ICE_BST_ALU_BA_S, \ + 81 - ICE_BST_ALU_BA_S) +#define ICE_BST_ALU_FEI BIT_ULL(87 - ICE_BST_ALU_BA_S) +#define ICE_BST_ALU_FSI GENMASK_ULL(95 - ICE_BST_ALU_BA_S, \ + 88 - ICE_BST_ALU_BA_S) + +/** + * ice_bst_alu_init - parse 96 bits of ALU entry + * @alu: pointer to the ALU entry structure + * @data: ALU entry data to be parsed + * @off: offset of the ALU entry data + */ +static void ice_bst_alu_init(struct ice_alu *alu, u8 *data, u8 off) +{ + u64 d64; + u8 idd; + + d64 = *((u64 *)data) >> off; + + alu->opc = FIELD_GET(ICE_BST_ALU_OPC, d64); + alu->src_start = FIELD_GET(ICE_BST_ALU_SS, d64); + alu->src_len = FIELD_GET(ICE_BST_ALU_SL, d64); + alu->shift_xlate_sel = FIELD_GET(ICE_BST_ALU_SXS, d64); + alu->shift_xlate_key = FIELD_GET(ICE_BST_ALU_SXK, d64); + alu->src_reg_id = FIELD_GET(ICE_BST_ALU_SRID, d64); + alu->dst_reg_id = FIELD_GET(ICE_BST_ALU_DRID, d64); + alu->inc0 = FIELD_GET(ICE_BST_ALU_INC0, d64); + alu->inc1 = FIELD_GET(ICE_BST_ALU_INC1, d64); + alu->proto_offset_opc = FIELD_GET(ICE_BST_ALU_POO, d64); + alu->proto_offset = FIELD_GET(ICE_BST_ALU_PO, d64); + + idd = (ICE_BST_ALU_BA_S + off) / BITS_PER_BYTE; + off = (ICE_BST_ALU_BA_S + off) % BITS_PER_BYTE; + d64 = *((u64 *)(&data[idd])) >> off; + + alu->branch_addr = FIELD_GET(ICE_BST_ALU_BA, d64); + alu->imm = FIELD_GET(ICE_BST_ALU_IMM, d64); + alu->dedicate_flags_ena = FIELD_GET(ICE_BST_ALU_DFE, d64); + alu->dst_start = FIELD_GET(ICE_BST_ALU_DS, d64); + alu->dst_len = FIELD_GET(ICE_BST_ALU_DL, d64); + alu->flags_extr_imm = FIELD_GET(ICE_BST_ALU_FEI, d64); + alu->flags_start_imm = FIELD_GET(ICE_BST_ALU_FSI, d64); +} + +#define ICE_BST_PGKB_F0_ENA BIT_ULL(0) +#define ICE_BST_PGKB_F0_IDX GENMASK_ULL(6, 1) +#define ICE_BST_PGKB_F1_ENA BIT_ULL(7) +#define ICE_BST_PGKB_F1_IDX GENMASK_ULL(13, 8) +#define ICE_BST_PGKB_F2_ENA BIT_ULL(14) +#define ICE_BST_PGKB_F2_IDX GENMASK_ULL(20, 15) +#define ICE_BST_PGKB_F3_ENA BIT_ULL(21) +#define ICE_BST_PGKB_F3_IDX GENMASK_ULL(27, 22) +#define ICE_BST_PGKB_AR_IDX GENMASK_ULL(34, 28) + +/** + * ice_bst_pgkb_init - parse 35 bits of Parse Graph Key Build + * @kb: pointer to the Parse Graph Key Build structure + * @data: Parse Graph Key Build data to be parsed + */ +static void ice_bst_pgkb_init(struct ice_pg_keybuilder *kb, u64 data) +{ + kb->flag0_ena = FIELD_GET(ICE_BST_PGKB_F0_ENA, data); + kb->flag0_idx = FIELD_GET(ICE_BST_PGKB_F0_IDX, data); + kb->flag1_ena = FIELD_GET(ICE_BST_PGKB_F1_ENA, data); + kb->flag1_idx = FIELD_GET(ICE_BST_PGKB_F1_IDX, data); + kb->flag2_ena = FIELD_GET(ICE_BST_PGKB_F2_ENA, data); + kb->flag2_idx = FIELD_GET(ICE_BST_PGKB_F2_IDX, data); + kb->flag3_ena = FIELD_GET(ICE_BST_PGKB_F3_ENA, data); + kb->flag3_idx = FIELD_GET(ICE_BST_PGKB_F3_IDX, data); + kb->alu_reg_idx = FIELD_GET(ICE_BST_PGKB_AR_IDX, data); +} + +#define ICE_BST_NPKB_OPC GENMASK(1, 0) +#define ICE_BST_NPKB_S_R0 GENMASK(9, 2) +#define ICE_BST_NPKB_L_R1 GENMASK(17, 10) + +/** + * ice_bst_npkb_init - parse 18 bits of Next Protocol Key Build + * @kb: pointer to the Next Protocol Key Build structure + * @data: Next Protocol Key Build data to be parsed + */ +static void ice_bst_npkb_init(struct ice_np_keybuilder *kb, u32 data) +{ + kb->opc = FIELD_GET(ICE_BST_NPKB_OPC, data); + kb->start_reg0 = FIELD_GET(ICE_BST_NPKB_S_R0, data); + kb->len_reg1 = FIELD_GET(ICE_BST_NPKB_L_R1, data); +} + +#define ICE_BT_KEY_S 32 +#define ICE_BT_KEY_IDD (ICE_BT_KEY_S / BITS_PER_BYTE) +#define ICE_BT_KIV_S 192 +#define ICE_BT_KIV_IDD (ICE_BT_KIV_S / BITS_PER_BYTE) +#define ICE_BT_HIG_S 352 +#define ICE_BT_HIG_IDD (ICE_BT_HIG_S / BITS_PER_BYTE) +#define ICE_BT_PGP_S 360 +#define ICE_BT_PGP_IDD (ICE_BT_PGP_S / BITS_PER_BYTE) +#define ICE_BT_PGP_M GENMASK(361 - ICE_BT_PGP_S, 360 - ICE_BT_PGP_S) +#define ICE_BT_NPKB_S 362 +#define ICE_BT_NPKB_IDD (ICE_BT_NPKB_S / BITS_PER_BYTE) +#define ICE_BT_NPKB_OFF (ICE_BT_NPKB_S % BITS_PER_BYTE) +#define ICE_BT_PGKB_S 380 +#define ICE_BT_PGKB_IDD (ICE_BT_PGKB_S / BITS_PER_BYTE) +#define ICE_BT_PGKB_OFF (ICE_BT_PGKB_S % BITS_PER_BYTE) +#define ICE_BT_ALU0_S 415 +#define ICE_BT_ALU0_IDD (ICE_BT_ALU0_S / BITS_PER_BYTE) +#define ICE_BT_ALU0_OFF (ICE_BT_ALU0_S % BITS_PER_BYTE) +#define ICE_BT_ALU1_S 511 +#define ICE_BT_ALU1_IDD (ICE_BT_ALU1_S / BITS_PER_BYTE) +#define ICE_BT_ALU1_OFF (ICE_BT_ALU1_S % BITS_PER_BYTE) +#define ICE_BT_ALU2_S 607 +#define ICE_BT_ALU2_IDD (ICE_BT_ALU2_S / BITS_PER_BYTE) +#define ICE_BT_ALU2_OFF (ICE_BT_ALU2_S % BITS_PER_BYTE) + +/** + * ice_bst_parse_item - parse 704 bits of Boost TCAM entry + * @hw: pointer to the hardware structure + * @idx: index of Boost TCAM entry + * @item: item of Boost TCAM entry + * @data: Boost TCAM entry data to be parsed + * @size: size of Boost TCAM entry + */ +static void ice_bst_parse_item(struct ice_hw *hw, u16 idx, void *item, + void *data, int __maybe_unused size) +{ + struct ice_bst_tcam_item *ti = item; + u8 *buf = (u8 *)data; + int i; + + ti->addr = *(u16 *)buf; + + for (i = 0; i < ICE_BST_TCAM_KEY_SIZE; i++) { + ti->key[i] = buf[ICE_BT_KEY_IDD + i]; + ti->key_inv[i] = buf[ICE_BT_KIV_IDD + i]; + } + ti->hit_idx_grp = buf[ICE_BT_HIG_IDD]; + ti->pg_prio = buf[ICE_BT_PGP_IDD] & ICE_BT_PGP_M; + + ice_bst_npkb_init(&ti->np_kb, + *((u32 *)(&buf[ICE_BT_NPKB_IDD])) >> + ICE_BT_NPKB_OFF); + ice_bst_pgkb_init(&ti->pg_kb, + *((u64 *)(&buf[ICE_BT_PGKB_IDD])) >> + ICE_BT_PGKB_OFF); + + ice_bst_alu_init(&ti->alu0, &buf[ICE_BT_ALU0_IDD], ICE_BT_ALU0_OFF); + ice_bst_alu_init(&ti->alu1, &buf[ICE_BT_ALU1_IDD], ICE_BT_ALU1_OFF); + ice_bst_alu_init(&ti->alu2, &buf[ICE_BT_ALU2_IDD], ICE_BT_ALU2_OFF); + + if (hw->debug_mask & ICE_DBG_PARSER) + ice_bst_tcam_dump(hw, ti); +} + +/** + * ice_bst_tcam_table_get - create a boost tcam table + * @hw: pointer to the hardware structure + * + * Return: a pointer to the allocated Boost TCAM table. + */ +static struct ice_bst_tcam_item *ice_bst_tcam_table_get(struct ice_hw *hw) +{ + return ice_parser_create_table(hw, ICE_SID_RXPARSER_BOOST_TCAM, + sizeof(struct ice_bst_tcam_item), + ICE_BST_TCAM_TABLE_SIZE, + ice_bst_parse_item, true); +} + +static void ice_parse_lbl_item(struct ice_hw *hw, u16 idx, void *item, + void *data, int __maybe_unused size) +{ + struct ice_lbl_item *lbl_item = item; + struct ice_lbl_item *lbl_data = data; + + lbl_item->idx = lbl_data->idx; + memcpy(lbl_item->label, lbl_data->label, sizeof(lbl_item->label)); + + if (strstarts(lbl_item->label, ICE_LBL_BST_DVM)) + lbl_item->type = ICE_LBL_BST_TYPE_DVM; + else if (strstarts(lbl_item->label, ICE_LBL_BST_SVM)) + lbl_item->type = ICE_LBL_BST_TYPE_SVM; + else if (strstarts(lbl_item->label, ICE_LBL_TNL_VXLAN)) + lbl_item->type = ICE_LBL_BST_TYPE_VXLAN; + else if (strstarts(lbl_item->label, ICE_LBL_TNL_GENEVE)) + lbl_item->type = ICE_LBL_BST_TYPE_GENEVE; + else if (strstarts(lbl_item->label, ICE_LBL_TNL_UDP_ECPRI)) + lbl_item->type = ICE_LBL_BST_TYPE_UDP_ECPRI; + + if (hw->debug_mask & ICE_DBG_PARSER) + ice_lbl_dump(hw, lbl_item); +} + +/** + * ice_bst_lbl_table_get - create a boost label table + * @hw: pointer to the hardware structure + * + * Return: a pointer to the allocated Boost label table. + */ +static struct ice_lbl_item *ice_bst_lbl_table_get(struct ice_hw *hw) +{ + return ice_parser_create_table(hw, ICE_SID_LBL_RXPARSER_TMEM, + sizeof(struct ice_lbl_item), + ICE_BST_TCAM_TABLE_SIZE, + ice_parse_lbl_item, true); +} + +/** + * ice_bst_tcam_match - match a pattern on the boost tcam table + * @tcam_table: boost tcam table to search + * @pat: pattern to match + * + * Return: a pointer to the matching Boost TCAM item or NULL. + */ +struct ice_bst_tcam_item * +ice_bst_tcam_match(struct ice_bst_tcam_item *tcam_table, u8 *pat) +{ + int i; + + for (i = 0; i < ICE_BST_TCAM_TABLE_SIZE; i++) { + struct ice_bst_tcam_item *item = &tcam_table[i]; + + if (item->hit_idx_grp == 0) + continue; + if (ice_ternary_match(item->key, item->key_inv, pat, + ICE_BST_TCAM_KEY_SIZE)) + return item; + } + + return NULL; +} + +/*** ICE_SID_RXPARSER_MARKER_PTYPE section ***/ +/** + * ice_ptype_mk_tcam_dump - dump an ptype marker tcam info + * @hw: pointer to the hardware structure + * @item: ptype marker tcam to dump + */ +static void ice_ptype_mk_tcam_dump(struct ice_hw *hw, + struct ice_ptype_mk_tcam_item *item) +{ + struct device *dev = ice_hw_to_dev(hw); + int i; + + dev_info(dev, "address = %d\n", item->address); + dev_info(dev, "ptype = %d\n", item->ptype); + + dev_info(dev, "key :"); + for (i = 0; i < ICE_PTYPE_MK_TCAM_KEY_SIZE; i++) + dev_info(dev, "%02x ", item->key[i]); + + dev_info(dev, "\n"); + + dev_info(dev, "key_inv:"); + for (i = 0; i < ICE_PTYPE_MK_TCAM_KEY_SIZE; i++) + dev_info(dev, "%02x ", item->key_inv[i]); + + dev_info(dev, "\n"); +} + +static void ice_parse_ptype_mk_tcam_item(struct ice_hw *hw, u16 idx, + void *item, void *data, int size) +{ + memcpy(item, data, size); + + if (hw->debug_mask & ICE_DBG_PARSER) + ice_ptype_mk_tcam_dump(hw, + (struct ice_ptype_mk_tcam_item *)item); +} + +/** + * ice_ptype_mk_tcam_table_get - create a ptype marker tcam table + * @hw: pointer to the hardware structure + * + * Return: a pointer to the allocated Marker PType TCAM table. + */ +static +struct ice_ptype_mk_tcam_item *ice_ptype_mk_tcam_table_get(struct ice_hw *hw) +{ + return ice_parser_create_table(hw, ICE_SID_RXPARSER_MARKER_PTYPE, + sizeof(struct ice_ptype_mk_tcam_item), + ICE_PTYPE_MK_TCAM_TABLE_SIZE, + ice_parse_ptype_mk_tcam_item, true); +} + +/** + * ice_ptype_mk_tcam_match - match a pattern on a ptype marker tcam table + * @table: ptype marker tcam table to search + * @pat: pattern to match + * @len: length of the pattern + * + * Return: a pointer to the matching Marker PType item or NULL. + */ +struct ice_ptype_mk_tcam_item * +ice_ptype_mk_tcam_match(struct ice_ptype_mk_tcam_item *table, + u8 *pat, int len) +{ + int i; + + for (i = 0; i < ICE_PTYPE_MK_TCAM_TABLE_SIZE; i++) { + struct ice_ptype_mk_tcam_item *item = &table[i]; + + if (ice_ternary_match(item->key, item->key_inv, pat, len)) + return item; + } + + return NULL; +} + +/*** ICE_SID_RXPARSER_MARKER_GRP section ***/ +/** + * ice_mk_grp_dump - dump an marker group item info + * @hw: pointer to the hardware structure + * @item: marker group item to dump + */ +static void ice_mk_grp_dump(struct ice_hw *hw, struct ice_mk_grp_item *item) +{ + struct device *dev = ice_hw_to_dev(hw); + int i; + + dev_info(dev, "index = %d\n", item->idx); + + dev_info(dev, "markers: "); + for (i = 0; i < ICE_MK_COUNT_PER_GRP; i++) + dev_info(dev, "%d ", item->markers[i]); + + dev_info(dev, "\n"); +} + +static void ice_mk_grp_parse_item(struct ice_hw *hw, u16 idx, void *item, + void *data, int __maybe_unused size) +{ + struct ice_mk_grp_item *grp = item; + u8 *buf = data; + int i; + + grp->idx = idx; + + for (i = 0; i < ICE_MK_COUNT_PER_GRP; i++) + grp->markers[i] = buf[i]; + + if (hw->debug_mask & ICE_DBG_PARSER) + ice_mk_grp_dump(hw, grp); +} + +/** + * ice_mk_grp_table_get - create a marker group table + * @hw: pointer to the hardware structure + * + * Return: a pointer to the allocated Marker Group ID table. + */ +static struct ice_mk_grp_item *ice_mk_grp_table_get(struct ice_hw *hw) +{ + return ice_parser_create_table(hw, ICE_SID_RXPARSER_MARKER_GRP, + sizeof(struct ice_mk_grp_item), + ICE_MK_GRP_TABLE_SIZE, + ice_mk_grp_parse_item, false); +} + +/*** ICE_SID_RXPARSER_PROTO_GRP section ***/ +static void ice_proto_off_dump(struct ice_hw *hw, + struct ice_proto_off *po, int idx) +{ + struct device *dev = ice_hw_to_dev(hw); + + dev_info(dev, "proto %d\n", idx); + dev_info(dev, "\tpolarity = %d\n", po->polarity); + dev_info(dev, "\tproto_id = %d\n", po->proto_id); + dev_info(dev, "\toffset = %d\n", po->offset); +} + +/** + * ice_proto_grp_dump - dump a proto group item info + * @hw: pointer to the hardware structure + * @item: proto group item to dump + */ +static void ice_proto_grp_dump(struct ice_hw *hw, + struct ice_proto_grp_item *item) +{ + int i; + + dev_info(ice_hw_to_dev(hw), "index = %d\n", item->idx); + + for (i = 0; i < ICE_PROTO_COUNT_PER_GRP; i++) + ice_proto_off_dump(hw, &item->po[i], i); +} + +#define ICE_PO_POL BIT(0) +#define ICE_PO_PID GENMASK(8, 1) +#define ICE_PO_OFF GENMASK(21, 12) + +/** + * ice_proto_off_parse - parse 22 bits of Protocol entry + * @po: pointer to the Protocol entry structure + * @data: Protocol entry data to be parsed + */ +static void ice_proto_off_parse(struct ice_proto_off *po, u32 data) +{ + po->polarity = FIELD_GET(ICE_PO_POL, data); + po->proto_id = FIELD_GET(ICE_PO_PID, data); + po->offset = FIELD_GET(ICE_PO_OFF, data); +} + +/** + * ice_proto_grp_parse_item - parse 192 bits of Protocol Group Table entry + * @hw: pointer to the hardware structure + * @idx: index of Protocol Group Table entry + * @item: item of Protocol Group Table entry + * @data: Protocol Group Table entry data to be parsed + * @size: size of Protocol Group Table entry + */ +static void ice_proto_grp_parse_item(struct ice_hw *hw, u16 idx, void *item, + void *data, int __maybe_unused size) +{ + struct ice_proto_grp_item *grp = item; + u8 *buf = (u8 *)data; + u8 idd, off; + u32 d32; + int i; + + grp->idx = idx; + + for (i = 0; i < ICE_PROTO_COUNT_PER_GRP; i++) { + idd = (ICE_PROTO_GRP_ITEM_SIZE * i) / BITS_PER_BYTE; + off = (ICE_PROTO_GRP_ITEM_SIZE * i) % BITS_PER_BYTE; + d32 = *((u32 *)&buf[idd]) >> off; + ice_proto_off_parse(&grp->po[i], d32); + } + + if (hw->debug_mask & ICE_DBG_PARSER) + ice_proto_grp_dump(hw, grp); +} + +/** + * ice_proto_grp_table_get - create a proto group table + * @hw: pointer to the hardware structure + * + * Return: a pointer to the allocated Protocol Group table. + */ +static struct ice_proto_grp_item *ice_proto_grp_table_get(struct ice_hw *hw) +{ + return ice_parser_create_table(hw, ICE_SID_RXPARSER_PROTO_GRP, + sizeof(struct ice_proto_grp_item), + ICE_PROTO_GRP_TABLE_SIZE, + ice_proto_grp_parse_item, false); +} + +/*** ICE_SID_RXPARSER_FLAG_REDIR section ***/ +/** + * ice_flg_rd_dump - dump a flag redirect item info + * @hw: pointer to the hardware structure + * @item: flag redirect item to dump + */ +static void ice_flg_rd_dump(struct ice_hw *hw, struct ice_flg_rd_item *item) +{ + struct device *dev = ice_hw_to_dev(hw); + + dev_info(dev, "index = %d\n", item->idx); + dev_info(dev, "expose = %d\n", item->expose); + dev_info(dev, "intr_flg_id = %d\n", item->intr_flg_id); +} + +#define ICE_FRT_EXPO BIT(0) +#define ICE_FRT_IFID GENMASK(6, 1) + +/** + * ice_flg_rd_parse_item - parse 8 bits of Flag Redirect Table entry + * @hw: pointer to the hardware structure + * @idx: index of Flag Redirect Table entry + * @item: item of Flag Redirect Table entry + * @data: Flag Redirect Table entry data to be parsed + * @size: size of Flag Redirect Table entry + */ +static void ice_flg_rd_parse_item(struct ice_hw *hw, u16 idx, void *item, + void *data, int __maybe_unused size) +{ + struct ice_flg_rd_item *rdi = item; + u8 d8 = *(u8 *)data; + + rdi->idx = idx; + rdi->expose = FIELD_GET(ICE_FRT_EXPO, d8); + rdi->intr_flg_id = FIELD_GET(ICE_FRT_IFID, d8); + + if (hw->debug_mask & ICE_DBG_PARSER) + ice_flg_rd_dump(hw, rdi); +} + +/** + * ice_flg_rd_table_get - create a flag redirect table + * @hw: pointer to the hardware structure + * + * Return: a pointer to the allocated Flags Redirection table. + */ +static struct ice_flg_rd_item *ice_flg_rd_table_get(struct ice_hw *hw) +{ + return ice_parser_create_table(hw, ICE_SID_RXPARSER_FLAG_REDIR, + sizeof(struct ice_flg_rd_item), + ICE_FLG_RD_TABLE_SIZE, + ice_flg_rd_parse_item, false); +} + +/** + * ice_flg_redirect - redirect a parser flag to packet flag + * @table: flag redirect table + * @psr_flg: parser flag to redirect + * + * Return: flag or 0 if @psr_flag = 0. + */ +u64 ice_flg_redirect(struct ice_flg_rd_item *table, u64 psr_flg) +{ + u64 flg = 0; + int i; + + for (i = 0; i < ICE_FLG_RDT_SIZE; i++) { + struct ice_flg_rd_item *item = &table[i]; + + if (!item->expose) + continue; + + if (psr_flg & BIT(item->intr_flg_id)) + flg |= BIT(i); + } + + return flg; +} + +/*** ICE_SID_XLT_KEY_BUILDER_SW, ICE_SID_XLT_KEY_BUILDER_ACL, + * ICE_SID_XLT_KEY_BUILDER_FD and ICE_SID_XLT_KEY_BUILDER_RSS + * sections ***/ +static void ice_xlt_kb_entry_dump(struct ice_hw *hw, + struct ice_xlt_kb_entry *entry, int idx) +{ + struct device *dev = ice_hw_to_dev(hw); + int i; + + dev_info(dev, "key builder entry %d\n", idx); + dev_info(dev, "\txlt1_ad_sel = %d\n", entry->xlt1_ad_sel); + dev_info(dev, "\txlt2_ad_sel = %d\n", entry->xlt2_ad_sel); + + for (i = 0; i < ICE_XLT_KB_FLAG0_14_CNT; i++) + dev_info(dev, "\tflg%d_sel = %d\n", i, entry->flg0_14_sel[i]); + + dev_info(dev, "\txlt1_md_sel = %d\n", entry->xlt1_md_sel); + dev_info(dev, "\txlt2_md_sel = %d\n", entry->xlt2_md_sel); +} + +/** + * ice_xlt_kb_dump - dump a xlt key build info + * @hw: pointer to the hardware structure + * @kb: key build to dump + */ +static void ice_xlt_kb_dump(struct ice_hw *hw, struct ice_xlt_kb *kb) +{ + struct device *dev = ice_hw_to_dev(hw); + int i; + + dev_info(dev, "xlt1_pm = %d\n", kb->xlt1_pm); + dev_info(dev, "xlt2_pm = %d\n", kb->xlt2_pm); + dev_info(dev, "prof_id_pm = %d\n", kb->prof_id_pm); + dev_info(dev, "flag15 lo = 0x%08x\n", (u32)kb->flag15); + dev_info(dev, "flag15 hi = 0x%08x\n", + (u32)(kb->flag15 >> (sizeof(u32) * BITS_PER_BYTE))); + + for (i = 0; i < ICE_XLT_KB_TBL_CNT; i++) + ice_xlt_kb_entry_dump(hw, &kb->entries[i], i); +} + +#define ICE_XLT_KB_X1AS_S 32 /* offset for the 1st 64-bits field */ +#define ICE_XLT_KB_X1AS_IDD (ICE_XLT_KB_X1AS_S / BITS_PER_BYTE) +#define ICE_XLT_KB_X1AS_OFF (ICE_XLT_KB_X1AS_S % BITS_PER_BYTE) +#define ICE_XLT_KB_X1AS GENMASK_ULL(34 - ICE_XLT_KB_X1AS_S, \ + 32 - ICE_XLT_KB_X1AS_S) +#define ICE_XLT_KB_X2AS GENMASK_ULL(37 - ICE_XLT_KB_X1AS_S, \ + 35 - ICE_XLT_KB_X1AS_S) +#define ICE_XLT_KB_FL00 GENMASK_ULL(46 - ICE_XLT_KB_X1AS_S, \ + 38 - ICE_XLT_KB_X1AS_S) +#define ICE_XLT_KB_FL01 GENMASK_ULL(55 - ICE_XLT_KB_X1AS_S, \ + 47 - ICE_XLT_KB_X1AS_S) +#define ICE_XLT_KB_FL02 GENMASK_ULL(64 - ICE_XLT_KB_X1AS_S, \ + 56 - ICE_XLT_KB_X1AS_S) +#define ICE_XLT_KB_FL03 GENMASK_ULL(73 - ICE_XLT_KB_X1AS_S, \ + 65 - ICE_XLT_KB_X1AS_S) +#define ICE_XLT_KB_FL04 GENMASK_ULL(82 - ICE_XLT_KB_X1AS_S, \ + 74 - ICE_XLT_KB_X1AS_S) +#define ICE_XLT_KB_FL05 GENMASK_ULL(91 - ICE_XLT_KB_X1AS_S, \ + 83 - ICE_XLT_KB_X1AS_S) +#define ICE_XLT_KB_FL06_S 92 /* offset for the 2nd 64-bits field */ +#define ICE_XLT_KB_FL06_IDD (ICE_XLT_KB_FL06_S / BITS_PER_BYTE) +#define ICE_XLT_KB_FL06_OFF (ICE_XLT_KB_FL06_S % BITS_PER_BYTE) +#define ICE_XLT_KB_FL06 GENMASK_ULL(100 - ICE_XLT_KB_FL06_S, \ + 92 - ICE_XLT_KB_FL06_S) +#define ICE_XLT_KB_FL07 GENMASK_ULL(109 - ICE_XLT_KB_FL06_S, \ + 101 - ICE_XLT_KB_FL06_S) +#define ICE_XLT_KB_FL08 GENMASK_ULL(118 - ICE_XLT_KB_FL06_S, \ + 110 - ICE_XLT_KB_FL06_S) +#define ICE_XLT_KB_FL09 GENMASK_ULL(127 - ICE_XLT_KB_FL06_S, \ + 119 - ICE_XLT_KB_FL06_S) +#define ICE_XLT_KB_FL10 GENMASK_ULL(136 - ICE_XLT_KB_FL06_S, \ + 128 - ICE_XLT_KB_FL06_S) +#define ICE_XLT_KB_FL11 GENMASK_ULL(145 - ICE_XLT_KB_FL06_S, \ + 137 - ICE_XLT_KB_FL06_S) +#define ICE_XLT_KB_FL12_S 146 /* offset for the 3rd 64-bits field */ +#define ICE_XLT_KB_FL12_IDD (ICE_XLT_KB_FL12_S / BITS_PER_BYTE) +#define ICE_XLT_KB_FL12_OFF (ICE_XLT_KB_FL12_S % BITS_PER_BYTE) +#define ICE_XLT_KB_FL12 GENMASK_ULL(154 - ICE_XLT_KB_FL12_S, \ + 146 - ICE_XLT_KB_FL12_S) +#define ICE_XLT_KB_FL13 GENMASK_ULL(163 - ICE_XLT_KB_FL12_S, \ + 155 - ICE_XLT_KB_FL12_S) +#define ICE_XLT_KB_FL14 GENMASK_ULL(181 - ICE_XLT_KB_FL12_S, \ + 164 - ICE_XLT_KB_FL12_S) +#define ICE_XLT_KB_X1MS GENMASK_ULL(186 - ICE_XLT_KB_FL12_S, \ + 182 - ICE_XLT_KB_FL12_S) +#define ICE_XLT_KB_X2MS GENMASK_ULL(191 - ICE_XLT_KB_FL12_S, \ + 187 - ICE_XLT_KB_FL12_S) + +/** + * ice_kb_entry_init - parse 192 bits of XLT Key Builder entry + * @entry: pointer to the XLT Key Builder entry structure + * @data: XLT Key Builder entry data to be parsed + */ +static void ice_kb_entry_init(struct ice_xlt_kb_entry *entry, u8 *data) +{ + u8 i = 0; + u64 d64; + + d64 = *((u64 *)&data[ICE_XLT_KB_X1AS_IDD]) >> ICE_XLT_KB_X1AS_OFF; + + entry->xlt1_ad_sel = FIELD_GET(ICE_XLT_KB_X1AS, d64); + entry->xlt2_ad_sel = FIELD_GET(ICE_XLT_KB_X2AS, d64); + + entry->flg0_14_sel[i++] = FIELD_GET(ICE_XLT_KB_FL00, d64); + entry->flg0_14_sel[i++] = FIELD_GET(ICE_XLT_KB_FL01, d64); + entry->flg0_14_sel[i++] = FIELD_GET(ICE_XLT_KB_FL02, d64); + entry->flg0_14_sel[i++] = FIELD_GET(ICE_XLT_KB_FL03, d64); + entry->flg0_14_sel[i++] = FIELD_GET(ICE_XLT_KB_FL04, d64); + entry->flg0_14_sel[i++] = FIELD_GET(ICE_XLT_KB_FL05, d64); + + d64 = *((u64 *)&data[ICE_XLT_KB_FL06_IDD]) >> ICE_XLT_KB_FL06_OFF; + + entry->flg0_14_sel[i++] = FIELD_GET(ICE_XLT_KB_FL06, d64); + entry->flg0_14_sel[i++] = FIELD_GET(ICE_XLT_KB_FL07, d64); + entry->flg0_14_sel[i++] = FIELD_GET(ICE_XLT_KB_FL08, d64); + entry->flg0_14_sel[i++] = FIELD_GET(ICE_XLT_KB_FL09, d64); + entry->flg0_14_sel[i++] = FIELD_GET(ICE_XLT_KB_FL10, d64); + entry->flg0_14_sel[i++] = FIELD_GET(ICE_XLT_KB_FL11, d64); + + d64 = *((u64 *)&data[ICE_XLT_KB_FL12_IDD]) >> ICE_XLT_KB_FL12_OFF; + + entry->flg0_14_sel[i++] = FIELD_GET(ICE_XLT_KB_FL12, d64); + entry->flg0_14_sel[i++] = FIELD_GET(ICE_XLT_KB_FL13, d64); + entry->flg0_14_sel[i] = FIELD_GET(ICE_XLT_KB_FL14, d64); + + entry->xlt1_md_sel = FIELD_GET(ICE_XLT_KB_X1MS, d64); + entry->xlt2_md_sel = FIELD_GET(ICE_XLT_KB_X2MS, d64); +} + +#define ICE_XLT_KB_X1PM_OFF 0 +#define ICE_XLT_KB_X2PM_OFF 1 +#define ICE_XLT_KB_PIPM_OFF 2 +#define ICE_XLT_KB_FL15_OFF 4 +#define ICE_XLT_KB_TBL_OFF 12 + +/** + * ice_parse_kb_data - parse 204 bits of XLT Key Build Table + * @hw: pointer to the hardware structure + * @kb: pointer to the XLT Key Build Table structure + * @data: XLT Key Build Table data to be parsed + */ +static void ice_parse_kb_data(struct ice_hw *hw, struct ice_xlt_kb *kb, + void *data) +{ + u8 *buf = data; + int i; + + kb->xlt1_pm = buf[ICE_XLT_KB_X1PM_OFF]; + kb->xlt2_pm = buf[ICE_XLT_KB_X2PM_OFF]; + kb->prof_id_pm = buf[ICE_XLT_KB_PIPM_OFF]; + + kb->flag15 = *(u64 *)&buf[ICE_XLT_KB_FL15_OFF]; + for (i = 0; i < ICE_XLT_KB_TBL_CNT; i++) + ice_kb_entry_init(&kb->entries[i], + &buf[ICE_XLT_KB_TBL_OFF + + i * ICE_XLT_KB_TBL_ENTRY_SIZE]); + + if (hw->debug_mask & ICE_DBG_PARSER) + ice_xlt_kb_dump(hw, kb); +} + +static struct ice_xlt_kb *ice_xlt_kb_get(struct ice_hw *hw, u32 sect_type) +{ + struct ice_pkg_enum state = {}; + struct ice_seg *seg = hw->seg; + struct ice_xlt_kb *kb; + void *data; + + if (!seg) + return ERR_PTR(-EINVAL); + + kb = kzalloc(sizeof(*kb), GFP_KERNEL); + if (!kb) + return ERR_PTR(-ENOMEM); + + data = ice_pkg_enum_section(seg, &state, sect_type); + if (!data) { + ice_debug(hw, ICE_DBG_PARSER, "failed to find section type %d.\n", + sect_type); + kfree(kb); + return ERR_PTR(-EINVAL); + } + + ice_parse_kb_data(hw, kb, data); + + return kb; +} + +/** + * ice_xlt_kb_get_sw - create switch xlt key build + * @hw: pointer to the hardware structure + * + * Return: a pointer to the allocated Key Builder table for Switch. + */ +static struct ice_xlt_kb *ice_xlt_kb_get_sw(struct ice_hw *hw) +{ + return ice_xlt_kb_get(hw, ICE_SID_XLT_KEY_BUILDER_SW); +} + +/** + * ice_xlt_kb_get_acl - create acl xlt key build + * @hw: pointer to the hardware structure + * + * Return: a pointer to the allocated Key Builder table for ACL. + */ +static struct ice_xlt_kb *ice_xlt_kb_get_acl(struct ice_hw *hw) +{ + return ice_xlt_kb_get(hw, ICE_SID_XLT_KEY_BUILDER_ACL); +} + +/** + * ice_xlt_kb_get_fd - create fdir xlt key build + * @hw: pointer to the hardware structure + * + * Return: a pointer to the allocated Key Builder table for Flow Director. + */ +static struct ice_xlt_kb *ice_xlt_kb_get_fd(struct ice_hw *hw) +{ + return ice_xlt_kb_get(hw, ICE_SID_XLT_KEY_BUILDER_FD); +} + +/** + * ice_xlt_kb_get_rss - create rss xlt key build + * @hw: pointer to the hardware structure + * + * Return: a pointer to the allocated Key Builder table for RSS. + */ +static struct ice_xlt_kb *ice_xlt_kb_get_rss(struct ice_hw *hw) +{ + return ice_xlt_kb_get(hw, ICE_SID_XLT_KEY_BUILDER_RSS); +} + +#define ICE_XLT_KB_MASK GENMASK_ULL(5, 0) + +/** + * ice_xlt_kb_flag_get - aggregate 64 bits packet flag into 16 bits xlt flag + * @kb: xlt key build + * @pkt_flag: 64 bits packet flag + * + * Return: XLT flag or 0 if @pkt_flag = 0. + */ +u16 ice_xlt_kb_flag_get(struct ice_xlt_kb *kb, u64 pkt_flag) +{ + struct ice_xlt_kb_entry *entry = &kb->entries[0]; + u16 flag = 0; + int i; + + /* check flag 15 */ + if (kb->flag15 & pkt_flag) + flag = BIT(ICE_XLT_KB_FLAG0_14_CNT); + + /* check flag 0 - 14 */ + for (i = 0; i < ICE_XLT_KB_FLAG0_14_CNT; i++) { + /* only check first entry */ + u16 idx = entry->flg0_14_sel[i] & ICE_XLT_KB_MASK; + + if (pkt_flag & BIT(idx)) + flag |= (u16)BIT(i); + } + + return flag; +} + +/*** Parser API ***/ +/** + * ice_parser_create - create a parser instance + * @hw: pointer to the hardware structure + * + * Return: a pointer to the allocated parser instance or ERR_PTR + * in case of error. + */ +struct ice_parser *ice_parser_create(struct ice_hw *hw) +{ + struct ice_parser *p; + void *err; + + p = kzalloc(sizeof(*p), GFP_KERNEL); + if (!p) + return ERR_PTR(-ENOMEM); + + p->hw = hw; + p->rt.psr = p; + + p->imem_table = ice_imem_table_get(hw); + if (IS_ERR(p->imem_table)) { + err = p->imem_table; + goto err; + } + + p->mi_table = ice_metainit_table_get(hw); + if (IS_ERR(p->mi_table)) { + err = p->mi_table; + goto err; + } + + p->pg_cam_table = ice_pg_cam_table_get(hw); + if (IS_ERR(p->pg_cam_table)) { + err = p->pg_cam_table; + goto err; + } + + p->pg_sp_cam_table = ice_pg_sp_cam_table_get(hw); + if (IS_ERR(p->pg_sp_cam_table)) { + err = p->pg_sp_cam_table; + goto err; + } + + p->pg_nm_cam_table = ice_pg_nm_cam_table_get(hw); + if (IS_ERR(p->pg_nm_cam_table)) { + err = p->pg_nm_cam_table; + goto err; + } + + p->pg_nm_sp_cam_table = ice_pg_nm_sp_cam_table_get(hw); + if (IS_ERR(p->pg_nm_sp_cam_table)) { + err = p->pg_nm_sp_cam_table; + goto err; + } + + p->bst_tcam_table = ice_bst_tcam_table_get(hw); + if (IS_ERR(p->bst_tcam_table)) { + err = p->bst_tcam_table; + goto err; + } + + p->bst_lbl_table = ice_bst_lbl_table_get(hw); + if (IS_ERR(p->bst_lbl_table)) { + err = p->bst_lbl_table; + goto err; + } + + p->ptype_mk_tcam_table = ice_ptype_mk_tcam_table_get(hw); + if (IS_ERR(p->ptype_mk_tcam_table)) { + err = p->ptype_mk_tcam_table; + goto err; + } + + p->mk_grp_table = ice_mk_grp_table_get(hw); + if (IS_ERR(p->mk_grp_table)) { + err = p->mk_grp_table; + goto err; + } + + p->proto_grp_table = ice_proto_grp_table_get(hw); + if (IS_ERR(p->proto_grp_table)) { + err = p->proto_grp_table; + goto err; + } + + p->flg_rd_table = ice_flg_rd_table_get(hw); + if (IS_ERR(p->flg_rd_table)) { + err = p->flg_rd_table; + goto err; + } + + p->xlt_kb_sw = ice_xlt_kb_get_sw(hw); + if (IS_ERR(p->xlt_kb_sw)) { + err = p->xlt_kb_sw; + goto err; + } + + p->xlt_kb_acl = ice_xlt_kb_get_acl(hw); + if (IS_ERR(p->xlt_kb_acl)) { + err = p->xlt_kb_acl; + goto err; + } + + p->xlt_kb_fd = ice_xlt_kb_get_fd(hw); + if (IS_ERR(p->xlt_kb_fd)) { + err = p->xlt_kb_fd; + goto err; + } + + p->xlt_kb_rss = ice_xlt_kb_get_rss(hw); + if (IS_ERR(p->xlt_kb_rss)) { + err = p->xlt_kb_rss; + goto err; + } + + return p; +err: + ice_parser_destroy(p); + return err; +} + +/** + * ice_parser_destroy - destroy a parser instance + * @psr: pointer to a parser instance + */ +void ice_parser_destroy(struct ice_parser *psr) +{ + kfree(psr->imem_table); + kfree(psr->mi_table); + kfree(psr->pg_cam_table); + kfree(psr->pg_sp_cam_table); + kfree(psr->pg_nm_cam_table); + kfree(psr->pg_nm_sp_cam_table); + kfree(psr->bst_tcam_table); + kfree(psr->bst_lbl_table); + kfree(psr->ptype_mk_tcam_table); + kfree(psr->mk_grp_table); + kfree(psr->proto_grp_table); + kfree(psr->flg_rd_table); + kfree(psr->xlt_kb_sw); + kfree(psr->xlt_kb_acl); + kfree(psr->xlt_kb_fd); + kfree(psr->xlt_kb_rss); + + kfree(psr); +} + +/** + * ice_parser_run - parse on a packet in binary and return the result + * @psr: pointer to a parser instance + * @pkt_buf: packet data + * @pkt_len: packet length + * @rslt: input/output parameter to save parser result. + * + * Return: 0 on success or errno. + */ +int ice_parser_run(struct ice_parser *psr, const u8 *pkt_buf, + int pkt_len, struct ice_parser_result *rslt) +{ + ice_parser_rt_reset(&psr->rt); + ice_parser_rt_pktbuf_set(&psr->rt, pkt_buf, pkt_len); + + return ice_parser_rt_execute(&psr->rt, rslt); +} + +/** + * ice_parser_result_dump - dump a parser result info + * @hw: pointer to the hardware structure + * @rslt: parser result info to dump + */ +void ice_parser_result_dump(struct ice_hw *hw, struct ice_parser_result *rslt) +{ + struct device *dev = ice_hw_to_dev(hw); + int i; + + dev_info(dev, "ptype = %d\n", rslt->ptype); + for (i = 0; i < rslt->po_num; i++) + dev_info(dev, "proto = %d, offset = %d\n", + rslt->po[i].proto_id, rslt->po[i].offset); + + dev_info(dev, "flags_psr = 0x%016llx\n", rslt->flags_psr); + dev_info(dev, "flags_pkt = 0x%016llx\n", rslt->flags_pkt); + dev_info(dev, "flags_sw = 0x%04x\n", rslt->flags_sw); + dev_info(dev, "flags_fd = 0x%04x\n", rslt->flags_fd); + dev_info(dev, "flags_rss = 0x%04x\n", rslt->flags_rss); +} + +#define ICE_BT_VLD_KEY 0xFF +#define ICE_BT_INV_KEY 0xFE + +static void ice_bst_dvm_set(struct ice_parser *psr, enum ice_lbl_type type, + bool on) +{ + u16 i = 0; + + while (true) { + struct ice_bst_tcam_item *item; + u8 key; + + item = ice_bst_tcam_search(psr->bst_tcam_table, + psr->bst_lbl_table, + type, &i); + if (!item) + break; + + key = on ? ICE_BT_VLD_KEY : ICE_BT_INV_KEY; + item->key[ICE_BT_VM_OFF] = key; + item->key_inv[ICE_BT_VM_OFF] = key; + i++; + } +} + +/** + * ice_parser_dvm_set - configure double vlan mode for parser + * @psr: pointer to a parser instance + * @on: true to turn on; false to turn off + */ +void ice_parser_dvm_set(struct ice_parser *psr, bool on) +{ + ice_bst_dvm_set(psr, ICE_LBL_BST_TYPE_DVM, on); + ice_bst_dvm_set(psr, ICE_LBL_BST_TYPE_SVM, !on); +} + +static int ice_tunnel_port_set(struct ice_parser *psr, enum ice_lbl_type type, + u16 udp_port, bool on) +{ + u8 *buf = (u8 *)&udp_port; + u16 i = 0; + + while (true) { + struct ice_bst_tcam_item *item; + + item = ice_bst_tcam_search(psr->bst_tcam_table, + psr->bst_lbl_table, + type, &i); + if (!item) + break; + + /* found empty slot to add */ + if (on && item->key[ICE_BT_TUN_PORT_OFF_H] == ICE_BT_INV_KEY && + item->key_inv[ICE_BT_TUN_PORT_OFF_H] == ICE_BT_INV_KEY) { + item->key_inv[ICE_BT_TUN_PORT_OFF_L] = + buf[ICE_UDP_PORT_OFF_L]; + item->key_inv[ICE_BT_TUN_PORT_OFF_H] = + buf[ICE_UDP_PORT_OFF_H]; + + item->key[ICE_BT_TUN_PORT_OFF_L] = + ICE_BT_VLD_KEY - buf[ICE_UDP_PORT_OFF_L]; + item->key[ICE_BT_TUN_PORT_OFF_H] = + ICE_BT_VLD_KEY - buf[ICE_UDP_PORT_OFF_H]; + + return 0; + /* found a matched slot to delete */ + } else if (!on && + (item->key_inv[ICE_BT_TUN_PORT_OFF_L] == + buf[ICE_UDP_PORT_OFF_L] || + item->key_inv[ICE_BT_TUN_PORT_OFF_H] == + buf[ICE_UDP_PORT_OFF_H])) { + item->key_inv[ICE_BT_TUN_PORT_OFF_L] = ICE_BT_VLD_KEY; + item->key_inv[ICE_BT_TUN_PORT_OFF_H] = ICE_BT_INV_KEY; + + item->key[ICE_BT_TUN_PORT_OFF_L] = ICE_BT_VLD_KEY; + item->key[ICE_BT_TUN_PORT_OFF_H] = ICE_BT_INV_KEY; + + return 0; + } + i++; + } + + return -EINVAL; +} + +/** + * ice_parser_vxlan_tunnel_set - configure vxlan tunnel for parser + * @psr: pointer to a parser instance + * @udp_port: vxlan tunnel port in UDP header + * @on: true to turn on; false to turn off + * + * Return: 0 on success or errno on failure. + */ +int ice_parser_vxlan_tunnel_set(struct ice_parser *psr, + u16 udp_port, bool on) +{ + return ice_tunnel_port_set(psr, ICE_LBL_BST_TYPE_VXLAN, udp_port, on); +} + +/** + * ice_parser_geneve_tunnel_set - configure geneve tunnel for parser + * @psr: pointer to a parser instance + * @udp_port: geneve tunnel port in UDP header + * @on: true to turn on; false to turn off + * + * Return: 0 on success or errno on failure. + */ +int ice_parser_geneve_tunnel_set(struct ice_parser *psr, + u16 udp_port, bool on) +{ + return ice_tunnel_port_set(psr, ICE_LBL_BST_TYPE_GENEVE, udp_port, on); +} + +/** + * ice_parser_ecpri_tunnel_set - configure ecpri tunnel for parser + * @psr: pointer to a parser instance + * @udp_port: ecpri tunnel port in UDP header + * @on: true to turn on; false to turn off + * + * Return: 0 on success or errno on failure. + */ +int ice_parser_ecpri_tunnel_set(struct ice_parser *psr, + u16 udp_port, bool on) +{ + return ice_tunnel_port_set(psr, ICE_LBL_BST_TYPE_UDP_ECPRI, + udp_port, on); +} + +/** + * ice_nearest_proto_id - find nearest protocol ID + * @rslt: pointer to a parser result instance + * @offset: a min value for the protocol offset + * @proto_id: the protocol ID (output) + * @proto_off: the protocol offset (output) + * + * From the protocols in @rslt, find the nearest protocol that has offset + * larger than @offset. + * + * Return: if true, the protocol's ID and offset + */ +static bool ice_nearest_proto_id(struct ice_parser_result *rslt, u16 offset, + u8 *proto_id, u16 *proto_off) +{ + u16 dist = U16_MAX; + u8 proto = 0; + int i; + + for (i = 0; i < rslt->po_num; i++) { + if (offset < rslt->po[i].offset) + continue; + if (offset - rslt->po[i].offset < dist) { + proto = rslt->po[i].proto_id; + dist = offset - rslt->po[i].offset; + } + } + + if (dist % 2) + return false; + + *proto_id = proto; + *proto_off = dist; + + return true; +} + +/* default flag mask to cover GTP_EH_PDU, GTP_EH_PDU_LINK and TUN2 + * In future, the flag masks should learn from DDP + */ +#define ICE_KEYBUILD_FLAG_MASK_DEFAULT_SW 0x4002 +#define ICE_KEYBUILD_FLAG_MASK_DEFAULT_ACL 0x0000 +#define ICE_KEYBUILD_FLAG_MASK_DEFAULT_FD 0x6080 +#define ICE_KEYBUILD_FLAG_MASK_DEFAULT_RSS 0x6010 + +/** + * ice_parser_profile_init - initialize a FXP profile based on parser result + * @rslt: a instance of a parser result + * @pkt_buf: packet data buffer + * @msk_buf: packet mask buffer + * @buf_len: packet length + * @blk: FXP pipeline stage + * @prof: input/output parameter to save the profile + * + * Return: 0 on success or errno on failure. + */ +int ice_parser_profile_init(struct ice_parser_result *rslt, + const u8 *pkt_buf, const u8 *msk_buf, + int buf_len, enum ice_block blk, + struct ice_parser_profile *prof) +{ + u8 proto_id = U8_MAX; + u16 proto_off = 0; + u16 off; + + memset(prof, 0, sizeof(*prof)); + set_bit(rslt->ptype, prof->ptypes); + if (blk == ICE_BLK_SW) { + prof->flags = rslt->flags_sw; + prof->flags_msk = ICE_KEYBUILD_FLAG_MASK_DEFAULT_SW; + } else if (blk == ICE_BLK_ACL) { + prof->flags = rslt->flags_acl; + prof->flags_msk = ICE_KEYBUILD_FLAG_MASK_DEFAULT_ACL; + } else if (blk == ICE_BLK_FD) { + prof->flags = rslt->flags_fd; + prof->flags_msk = ICE_KEYBUILD_FLAG_MASK_DEFAULT_FD; + } else if (blk == ICE_BLK_RSS) { + prof->flags = rslt->flags_rss; + prof->flags_msk = ICE_KEYBUILD_FLAG_MASK_DEFAULT_RSS; + } else { + return -EINVAL; + } + + for (off = 0; off < buf_len - 1; off++) { + if (msk_buf[off] == 0 && msk_buf[off + 1] == 0) + continue; + if (!ice_nearest_proto_id(rslt, off, &proto_id, &proto_off)) + continue; + if (prof->fv_num >= ICE_PARSER_FV_MAX) + return -EINVAL; + + prof->fv[prof->fv_num].proto_id = proto_id; + prof->fv[prof->fv_num].offset = proto_off; + prof->fv[prof->fv_num].spec = *(const u16 *)&pkt_buf[off]; + prof->fv[prof->fv_num].msk = *(const u16 *)&msk_buf[off]; + prof->fv_num++; + } + + return 0; +} + +/** + * ice_parser_profile_dump - dump an FXP profile info + * @hw: pointer to the hardware structure + * @prof: profile info to dump + */ +void ice_parser_profile_dump(struct ice_hw *hw, + struct ice_parser_profile *prof) +{ + struct device *dev = ice_hw_to_dev(hw); + u16 i; + + dev_info(dev, "ptypes:\n"); + for (i = 0; i < ICE_FLOW_PTYPE_MAX; i++) + if (test_bit(i, prof->ptypes)) + dev_info(dev, "\t%u\n", i); + + for (i = 0; i < prof->fv_num; i++) + dev_info(dev, "proto = %u, offset = %2u, spec = 0x%04x, mask = 0x%04x\n", + prof->fv[i].proto_id, prof->fv[i].offset, + prof->fv[i].spec, prof->fv[i].msk); + + dev_info(dev, "flags = 0x%04x\n", prof->flags); + dev_info(dev, "flags_msk = 0x%04x\n", prof->flags_msk); +} diff --git a/drivers/net/ethernet/intel/ice/ice_parser.h b/drivers/net/ethernet/intel/ice/ice_parser.h new file mode 100644 index 000000000000..6509d807627c --- /dev/null +++ b/drivers/net/ethernet/intel/ice/ice_parser.h @@ -0,0 +1,540 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright (C) 2024 Intel Corporation */ + +#ifndef _ICE_PARSER_H_ +#define _ICE_PARSER_H_ + +#define ICE_SEC_DATA_OFFSET 4 +#define ICE_SID_RXPARSER_IMEM_ENTRY_SIZE 48 +#define ICE_SID_RXPARSER_METADATA_INIT_ENTRY_SIZE 24 +#define ICE_SID_RXPARSER_CAM_ENTRY_SIZE 16 +#define ICE_SID_RXPARSER_PG_SPILL_ENTRY_SIZE 17 +#define ICE_SID_RXPARSER_NOMATCH_CAM_ENTRY_SIZE 12 +#define ICE_SID_RXPARSER_NOMATCH_SPILL_ENTRY_SIZE 13 +#define ICE_SID_RXPARSER_BOOST_TCAM_ENTRY_SIZE 88 +#define ICE_SID_RXPARSER_MARKER_TYPE_ENTRY_SIZE 24 +#define ICE_SID_RXPARSER_MARKER_GRP_ENTRY_SIZE 8 +#define ICE_SID_RXPARSER_PROTO_GRP_ENTRY_SIZE 24 +#define ICE_SID_RXPARSER_FLAG_REDIR_ENTRY_SIZE 1 + +#define ICE_SEC_LBL_DATA_OFFSET 2 +#define ICE_SID_LBL_ENTRY_SIZE 66 + +/*** ICE_SID_RXPARSER_IMEM section ***/ +#define ICE_IMEM_TABLE_SIZE 192 + +/* TCAM boost Master; if bit is set, and TCAM hit, TCAM output overrides iMEM + * output. + */ +struct ice_bst_main { + bool alu0; + bool alu1; + bool alu2; + bool pg; +}; + +struct ice_bst_keybuilder { + u8 prio; /* 0-3: PG precedence within ALUs (3 highest) */ + bool tsr_ctrl; /* TCAM Search Register control */ +}; + +/* Next protocol Key builder */ +struct ice_np_keybuilder { + u8 opc; + u8 start_reg0; + u8 len_reg1; +}; + +enum ice_np_keybuilder_opcode { + ICE_NPKB_OPC_EXTRACT = 0, + ICE_NPKB_OPC_BUILD = 1, + ICE_NPKB_OPC_BYPASS = 2, +}; + +/* Parse Graph Key builder */ +struct ice_pg_keybuilder { + bool flag0_ena; + bool flag1_ena; + bool flag2_ena; + bool flag3_ena; + u8 flag0_idx; + u8 flag1_idx; + u8 flag2_idx; + u8 flag3_idx; + u8 alu_reg_idx; +}; + +enum ice_alu_idx { + ICE_ALU0_IDX = 0, + ICE_ALU1_IDX = 1, + ICE_ALU2_IDX = 2, +}; + +enum ice_alu_opcode { + ICE_ALU_PARK = 0, + ICE_ALU_MOV_ADD = 1, + ICE_ALU_ADD = 2, + ICE_ALU_MOV_AND = 4, + ICE_ALU_AND = 5, + ICE_ALU_AND_IMM = 6, + ICE_ALU_MOV_OR = 7, + ICE_ALU_OR = 8, + ICE_ALU_MOV_XOR = 9, + ICE_ALU_XOR = 10, + ICE_ALU_NOP = 11, + ICE_ALU_BR = 12, + ICE_ALU_BREQ = 13, + ICE_ALU_BRNEQ = 14, + ICE_ALU_BRGT = 15, + ICE_ALU_BRLT = 16, + ICE_ALU_BRGEQ = 17, + ICE_ALU_BRLEG = 18, + ICE_ALU_SETEQ = 19, + ICE_ALU_ANDEQ = 20, + ICE_ALU_OREQ = 21, + ICE_ALU_SETNEQ = 22, + ICE_ALU_ANDNEQ = 23, + ICE_ALU_ORNEQ = 24, + ICE_ALU_SETGT = 25, + ICE_ALU_ANDGT = 26, + ICE_ALU_ORGT = 27, + ICE_ALU_SETLT = 28, + ICE_ALU_ANDLT = 29, + ICE_ALU_ORLT = 30, + ICE_ALU_MOV_SUB = 31, + ICE_ALU_SUB = 32, + ICE_ALU_INVALID = 64, +}; + +enum ice_proto_off_opcode { + ICE_PO_OFF_REMAIN = 0, + ICE_PO_OFF_HDR_ADD = 1, + ICE_PO_OFF_HDR_SUB = 2, +}; + +struct ice_alu { + enum ice_alu_opcode opc; + u8 src_start; + u8 src_len; + bool shift_xlate_sel; + u8 shift_xlate_key; + u8 src_reg_id; + u8 dst_reg_id; + bool inc0; + bool inc1; + u8 proto_offset_opc; + u8 proto_offset; + u8 branch_addr; + u16 imm; + bool dedicate_flags_ena; + u8 dst_start; + u8 dst_len; + bool flags_extr_imm; + u8 flags_start_imm; +}; + +/* Parser program code (iMEM) */ +struct ice_imem_item { + u16 idx; + struct ice_bst_main b_m; + struct ice_bst_keybuilder b_kb; + u8 pg_prio; + struct ice_np_keybuilder np_kb; + struct ice_pg_keybuilder pg_kb; + struct ice_alu alu0; + struct ice_alu alu1; + struct ice_alu alu2; +}; + +/*** ICE_SID_RXPARSER_METADATA_INIT section ***/ +#define ICE_METAINIT_TABLE_SIZE 16 + +/* Metadata Initialization item */ +struct ice_metainit_item { + u16 idx; + + u8 tsr; /* TCAM Search key Register */ + u16 ho; /* Header Offset register */ + u16 pc; /* Program Counter register */ + u16 pg_rn; /* Parse Graph Root Node */ + u8 cd; /* Control Domain ID */ + + /* General Purpose Registers */ + bool gpr_a_ctrl; + u8 gpr_a_data_mdid; + u8 gpr_a_data_start; + u8 gpr_a_data_len; + u8 gpr_a_id; + + bool gpr_b_ctrl; + u8 gpr_b_data_mdid; + u8 gpr_b_data_start; + u8 gpr_b_data_len; + u8 gpr_b_id; + + bool gpr_c_ctrl; + u8 gpr_c_data_mdid; + u8 gpr_c_data_start; + u8 gpr_c_data_len; + u8 gpr_c_id; + + bool gpr_d_ctrl; + u8 gpr_d_data_mdid; + u8 gpr_d_data_start; + u8 gpr_d_data_len; + u8 gpr_d_id; + + u64 flags; /* Initial value for all flags */ +}; + +/*** ICE_SID_RXPARSER_CAM, ICE_SID_RXPARSER_PG_SPILL, + * ICE_SID_RXPARSER_NOMATCH_CAM and ICE_SID_RXPARSER_NOMATCH_CAM + * sections ***/ +#define ICE_PG_CAM_TABLE_SIZE 2048 +#define ICE_PG_SP_CAM_TABLE_SIZE 128 +#define ICE_PG_NM_CAM_TABLE_SIZE 1024 +#define ICE_PG_NM_SP_CAM_TABLE_SIZE 64 + +struct ice_pg_cam_key { + bool valid; + struct_group_attr(val, __packed, + u16 node_id; /* Node ID of protocol in parse graph */ + bool flag0; + bool flag1; + bool flag2; + bool flag3; + u8 boost_idx; /* Boost TCAM match index */ + u16 alu_reg; + u32 next_proto; /* next Protocol value (must be last) */ + ); +}; + +struct ice_pg_nm_cam_key { + bool valid; + struct_group_attr(val, __packed, + u16 node_id; + bool flag0; + bool flag1; + bool flag2; + bool flag3; + u8 boost_idx; + u16 alu_reg; + ); +}; + +struct ice_pg_cam_action { + u16 next_node; /* Parser Node ID for the next round */ + u8 next_pc; /* next Program Counter */ + bool is_pg; /* is protocol group */ + u8 proto_id; /* protocol ID or proto group ID */ + bool is_mg; /* is marker group */ + u8 marker_id; /* marker ID or marker group ID */ + bool is_last_round; + bool ho_polarity; /* header offset polarity */ + u16 ho_inc; +}; + +/* Parse Graph item */ +struct ice_pg_cam_item { + u16 idx; + struct ice_pg_cam_key key; + struct ice_pg_cam_action action; +}; + +/* Parse Graph No Match item */ +struct ice_pg_nm_cam_item { + u16 idx; + struct ice_pg_nm_cam_key key; + struct ice_pg_cam_action action; +}; + +struct ice_pg_cam_item *ice_pg_cam_match(struct ice_pg_cam_item *table, + int size, struct ice_pg_cam_key *key); +struct ice_pg_nm_cam_item * +ice_pg_nm_cam_match(struct ice_pg_nm_cam_item *table, int size, + struct ice_pg_cam_key *key); + +/*** ICE_SID_RXPARSER_BOOST_TCAM and ICE_SID_LBL_RXPARSER_TMEM sections ***/ +#define ICE_BST_TCAM_TABLE_SIZE 256 +#define ICE_BST_TCAM_KEY_SIZE 20 +#define ICE_BST_KEY_TCAM_SIZE 19 + +/* Boost TCAM item */ +struct ice_bst_tcam_item { + u16 addr; + u8 key[ICE_BST_TCAM_KEY_SIZE]; + u8 key_inv[ICE_BST_TCAM_KEY_SIZE]; + u8 hit_idx_grp; + u8 pg_prio; + struct ice_np_keybuilder np_kb; + struct ice_pg_keybuilder pg_kb; + struct ice_alu alu0; + struct ice_alu alu1; + struct ice_alu alu2; +}; + +#define ICE_LBL_LEN 64 +#define ICE_LBL_BST_DVM "BOOST_MAC_VLAN_DVM" +#define ICE_LBL_BST_SVM "BOOST_MAC_VLAN_SVM" +#define ICE_LBL_TNL_VXLAN "TNL_VXLAN" +#define ICE_LBL_TNL_GENEVE "TNL_GENEVE" +#define ICE_LBL_TNL_UDP_ECPRI "TNL_UDP_ECPRI" + +enum ice_lbl_type { + ICE_LBL_BST_TYPE_UNKNOWN, + ICE_LBL_BST_TYPE_DVM, + ICE_LBL_BST_TYPE_SVM, + ICE_LBL_BST_TYPE_VXLAN, + ICE_LBL_BST_TYPE_GENEVE, + ICE_LBL_BST_TYPE_UDP_ECPRI, +}; + +struct ice_lbl_item { + u16 idx; + char label[ICE_LBL_LEN]; + + /* must be at the end, not part of the DDP section */ + enum ice_lbl_type type; +}; + +struct ice_bst_tcam_item * +ice_bst_tcam_match(struct ice_bst_tcam_item *tcam_table, u8 *pat); +struct ice_bst_tcam_item * +ice_bst_tcam_search(struct ice_bst_tcam_item *tcam_table, + struct ice_lbl_item *lbl_table, + enum ice_lbl_type type, u16 *start); + +/*** ICE_SID_RXPARSER_MARKER_PTYPE section ***/ +#define ICE_PTYPE_MK_TCAM_TABLE_SIZE 1024 +#define ICE_PTYPE_MK_TCAM_KEY_SIZE 10 + +struct ice_ptype_mk_tcam_item { + u16 address; + u16 ptype; + u8 key[ICE_PTYPE_MK_TCAM_KEY_SIZE]; + u8 key_inv[ICE_PTYPE_MK_TCAM_KEY_SIZE]; +} __packed; + +struct ice_ptype_mk_tcam_item * +ice_ptype_mk_tcam_match(struct ice_ptype_mk_tcam_item *table, + u8 *pat, int len); +/*** ICE_SID_RXPARSER_MARKER_GRP section ***/ +#define ICE_MK_GRP_TABLE_SIZE 128 +#define ICE_MK_COUNT_PER_GRP 8 + +/* Marker Group item */ +struct ice_mk_grp_item { + int idx; + u8 markers[ICE_MK_COUNT_PER_GRP]; +}; + +/*** ICE_SID_RXPARSER_PROTO_GRP section ***/ +#define ICE_PROTO_COUNT_PER_GRP 8 +#define ICE_PROTO_GRP_TABLE_SIZE 192 +#define ICE_PROTO_GRP_ITEM_SIZE 22 +struct ice_proto_off { + bool polarity; /* true: positive, false: negative */ + u8 proto_id; + u16 offset; /* 10 bit protocol offset */ +}; + +/* Protocol Group item */ +struct ice_proto_grp_item { + u16 idx; + struct ice_proto_off po[ICE_PROTO_COUNT_PER_GRP]; +}; + +/*** ICE_SID_RXPARSER_FLAG_REDIR section ***/ +#define ICE_FLG_RD_TABLE_SIZE 64 +#define ICE_FLG_RDT_SIZE 64 + +/* Flags Redirection item */ +struct ice_flg_rd_item { + u16 idx; + bool expose; + u8 intr_flg_id; /* Internal Flag ID */ +}; + +u64 ice_flg_redirect(struct ice_flg_rd_item *table, u64 psr_flg); + +/*** ICE_SID_XLT_KEY_BUILDER_SW, ICE_SID_XLT_KEY_BUILDER_ACL, + * ICE_SID_XLT_KEY_BUILDER_FD and ICE_SID_XLT_KEY_BUILDER_RSS + * sections ***/ +#define ICE_XLT_KB_FLAG0_14_CNT 15 +#define ICE_XLT_KB_TBL_CNT 8 +#define ICE_XLT_KB_TBL_ENTRY_SIZE 24 + +struct ice_xlt_kb_entry { + u8 xlt1_ad_sel; + u8 xlt2_ad_sel; + u16 flg0_14_sel[ICE_XLT_KB_FLAG0_14_CNT]; + u8 xlt1_md_sel; + u8 xlt2_md_sel; +}; + +/* XLT Key Builder */ +struct ice_xlt_kb { + u8 xlt1_pm; /* XLT1 Partition Mode */ + u8 xlt2_pm; /* XLT2 Partition Mode */ + u8 prof_id_pm; /* Profile ID Partition Mode */ + u64 flag15; + + struct ice_xlt_kb_entry entries[ICE_XLT_KB_TBL_CNT]; +}; + +u16 ice_xlt_kb_flag_get(struct ice_xlt_kb *kb, u64 pkt_flag); + +/*** Parser API ***/ +#define ICE_GPR_HV_IDX 64 +#define ICE_GPR_HV_SIZE 32 +#define ICE_GPR_ERR_IDX 84 +#define ICE_GPR_FLG_IDX 104 +#define ICE_GPR_FLG_SIZE 16 + +#define ICE_GPR_TSR_IDX 108 /* TSR: TCAM Search Register */ +#define ICE_GPR_NN_IDX 109 /* NN: Next Parsing Cycle Node ID */ +#define ICE_GPR_HO_IDX 110 /* HO: Next Parsing Cycle hdr Offset */ +#define ICE_GPR_NP_IDX 111 /* NP: Next Parsing Cycle */ + +#define ICE_PARSER_MAX_PKT_LEN 504 +#define ICE_PARSER_PKT_REV 32 +#define ICE_PARSER_GPR_NUM 128 +#define ICE_PARSER_FLG_NUM 64 +#define ICE_PARSER_ERR_NUM 16 +#define ICE_BST_KEY_SIZE 10 +#define ICE_MARKER_ID_SIZE 9 +#define ICE_MARKER_MAX_SIZE \ + (ICE_MARKER_ID_SIZE * BITS_PER_BYTE - 1) +#define ICE_MARKER_ID_NUM 8 +#define ICE_PO_PAIR_SIZE 256 + +struct ice_gpr_pu { + /* array of flags to indicate if GRP needs to be updated */ + bool gpr_val_upd[ICE_PARSER_GPR_NUM]; + u16 gpr_val[ICE_PARSER_GPR_NUM]; + u64 flg_msk; + u64 flg_val; + u16 err_msk; + u16 err_val; +}; + +enum ice_pg_prio { + ICE_PG_P0 = 0, + ICE_PG_P1 = 1, + ICE_PG_P2 = 2, + ICE_PG_P3 = 3, +}; + +struct ice_parser_rt { + struct ice_parser *psr; + u16 gpr[ICE_PARSER_GPR_NUM]; + u8 pkt_buf[ICE_PARSER_MAX_PKT_LEN + ICE_PARSER_PKT_REV]; + u16 pkt_len; + u16 po; + u8 bst_key[ICE_BST_KEY_SIZE]; + struct ice_pg_cam_key pg_key; + struct ice_alu *alu0; + struct ice_alu *alu1; + struct ice_alu *alu2; + struct ice_pg_cam_action *action; + u8 pg_prio; + struct ice_gpr_pu pu; + u8 markers[ICE_MARKER_ID_SIZE]; + bool protocols[ICE_PO_PAIR_SIZE]; + u16 offsets[ICE_PO_PAIR_SIZE]; +}; + +struct ice_parser_proto_off { + u8 proto_id; /* hardware protocol ID */ + u16 offset; /* offset from the start of the protocol header */ +}; + +#define ICE_PARSER_PROTO_OFF_PAIR_SIZE 16 +#define ICE_PARSER_FLAG_PSR_SIZE 8 +#define ICE_PARSER_FV_SIZE 48 +#define ICE_PARSER_FV_MAX 24 +#define ICE_BT_TUN_PORT_OFF_H 16 +#define ICE_BT_TUN_PORT_OFF_L 15 +#define ICE_BT_VM_OFF 0 +#define ICE_UDP_PORT_OFF_H 1 +#define ICE_UDP_PORT_OFF_L 0 + +struct ice_parser_result { + u16 ptype; /* 16 bits hardware PTYPE */ + /* array of protocol and header offset pairs */ + struct ice_parser_proto_off po[ICE_PARSER_PROTO_OFF_PAIR_SIZE]; + int po_num; /* # of protocol-offset pairs must <= 16 */ + u64 flags_psr; /* parser flags */ + u64 flags_pkt; /* packet flags */ + u16 flags_sw; /* key builder flags for SW */ + u16 flags_acl; /* key builder flags for ACL */ + u16 flags_fd; /* key builder flags for FD */ + u16 flags_rss; /* key builder flags for RSS */ +}; + +void ice_parser_rt_reset(struct ice_parser_rt *rt); +void ice_parser_rt_pktbuf_set(struct ice_parser_rt *rt, const u8 *pkt_buf, + int pkt_len); +int ice_parser_rt_execute(struct ice_parser_rt *rt, + struct ice_parser_result *rslt); + +struct ice_parser { + struct ice_hw *hw; /* pointer to the hardware structure */ + + struct ice_imem_item *imem_table; + struct ice_metainit_item *mi_table; + + struct ice_pg_cam_item *pg_cam_table; + struct ice_pg_cam_item *pg_sp_cam_table; + struct ice_pg_nm_cam_item *pg_nm_cam_table; + struct ice_pg_nm_cam_item *pg_nm_sp_cam_table; + + struct ice_bst_tcam_item *bst_tcam_table; + struct ice_lbl_item *bst_lbl_table; + struct ice_ptype_mk_tcam_item *ptype_mk_tcam_table; + struct ice_mk_grp_item *mk_grp_table; + struct ice_proto_grp_item *proto_grp_table; + struct ice_flg_rd_item *flg_rd_table; + + struct ice_xlt_kb *xlt_kb_sw; + struct ice_xlt_kb *xlt_kb_acl; + struct ice_xlt_kb *xlt_kb_fd; + struct ice_xlt_kb *xlt_kb_rss; + + struct ice_parser_rt rt; +}; + +struct ice_parser *ice_parser_create(struct ice_hw *hw); +void ice_parser_destroy(struct ice_parser *psr); +void ice_parser_dvm_set(struct ice_parser *psr, bool on); +int ice_parser_vxlan_tunnel_set(struct ice_parser *psr, u16 udp_port, bool on); +int ice_parser_geneve_tunnel_set(struct ice_parser *psr, u16 udp_port, bool on); +int ice_parser_ecpri_tunnel_set(struct ice_parser *psr, u16 udp_port, bool on); +int ice_parser_run(struct ice_parser *psr, const u8 *pkt_buf, + int pkt_len, struct ice_parser_result *rslt); +void ice_parser_result_dump(struct ice_hw *hw, struct ice_parser_result *rslt); + +struct ice_parser_fv { + u8 proto_id; /* hardware protocol ID */ + u16 offset; /* offset from the start of the protocol header */ + u16 spec; /* pattern to match */ + u16 msk; /* pattern mask */ +}; + +struct ice_parser_profile { + /* array of field vectors */ + struct ice_parser_fv fv[ICE_PARSER_FV_SIZE]; + int fv_num; /* # of field vectors must <= 48 */ + u16 flags; /* key builder flags */ + u16 flags_msk; /* key builder flag mask */ + + DECLARE_BITMAP(ptypes, ICE_FLOW_PTYPE_MAX); /* PTYPE bitmap */ +}; + +int ice_parser_profile_init(struct ice_parser_result *rslt, + const u8 *pkt_buf, const u8 *msk_buf, + int buf_len, enum ice_block blk, + struct ice_parser_profile *prof); +void ice_parser_profile_dump(struct ice_hw *hw, + struct ice_parser_profile *prof); +#endif /* _ICE_PARSER_H_ */ diff --git a/drivers/net/ethernet/intel/ice/ice_parser_rt.c b/drivers/net/ethernet/intel/ice/ice_parser_rt.c new file mode 100644 index 000000000000..dedf5e854e4b --- /dev/null +++ b/drivers/net/ethernet/intel/ice/ice_parser_rt.c @@ -0,0 +1,861 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (C) 2024 Intel Corporation */ + +#include "ice_common.h" + +static void ice_rt_tsr_set(struct ice_parser_rt *rt, u16 tsr) +{ + rt->gpr[ICE_GPR_TSR_IDX] = tsr; +} + +static void ice_rt_ho_set(struct ice_parser_rt *rt, u16 ho) +{ + rt->gpr[ICE_GPR_HO_IDX] = ho; + memcpy(&rt->gpr[ICE_GPR_HV_IDX], &rt->pkt_buf[ho], ICE_GPR_HV_SIZE); +} + +static void ice_rt_np_set(struct ice_parser_rt *rt, u16 pc) +{ + rt->gpr[ICE_GPR_NP_IDX] = pc; +} + +static void ice_rt_nn_set(struct ice_parser_rt *rt, u16 node) +{ + rt->gpr[ICE_GPR_NN_IDX] = node; +} + +static void +ice_rt_flag_set(struct ice_parser_rt *rt, unsigned int idx, bool set) +{ + struct ice_hw *hw = rt->psr->hw; + unsigned int word, id; + + word = idx / ICE_GPR_FLG_SIZE; + id = idx % ICE_GPR_FLG_SIZE; + + if (set) { + rt->gpr[ICE_GPR_FLG_IDX + word] |= (u16)BIT(id); + ice_debug(hw, ICE_DBG_PARSER, "Set parser flag %u\n", idx); + } else { + rt->gpr[ICE_GPR_FLG_IDX + word] &= ~(u16)BIT(id); + ice_debug(hw, ICE_DBG_PARSER, "Clear parser flag %u\n", idx); + } +} + +static void ice_rt_gpr_set(struct ice_parser_rt *rt, int idx, u16 val) +{ + struct ice_hw *hw = rt->psr->hw; + + if (idx == ICE_GPR_HO_IDX) + ice_rt_ho_set(rt, val); + else + rt->gpr[idx] = val; + + ice_debug(hw, ICE_DBG_PARSER, "Set GPR %d value %d\n", idx, val); +} + +static void ice_rt_err_set(struct ice_parser_rt *rt, unsigned int idx, bool set) +{ + struct ice_hw *hw = rt->psr->hw; + + if (set) { + rt->gpr[ICE_GPR_ERR_IDX] |= (u16)BIT(idx); + ice_debug(hw, ICE_DBG_PARSER, "Set parser error %u\n", idx); + } else { + rt->gpr[ICE_GPR_ERR_IDX] &= ~(u16)BIT(idx); + ice_debug(hw, ICE_DBG_PARSER, "Reset parser error %u\n", idx); + } +} + +/** + * ice_parser_rt_reset - reset the parser runtime + * @rt: pointer to the parser runtime + */ +void ice_parser_rt_reset(struct ice_parser_rt *rt) +{ + struct ice_parser *psr = rt->psr; + struct ice_metainit_item *mi; + unsigned int i; + + mi = &psr->mi_table[0]; + + memset(rt, 0, sizeof(*rt)); + rt->psr = psr; + + ice_rt_tsr_set(rt, mi->tsr); + ice_rt_ho_set(rt, mi->ho); + ice_rt_np_set(rt, mi->pc); + ice_rt_nn_set(rt, mi->pg_rn); + + for (i = 0; i < ICE_PARSER_FLG_NUM; i++) { + if (mi->flags & BIT(i)) + ice_rt_flag_set(rt, i, true); + } +} + +/** + * ice_parser_rt_pktbuf_set - set a packet into parser runtime + * @rt: pointer to the parser runtime + * @pkt_buf: buffer with packet data + * @pkt_len: packet buffer length + */ +void ice_parser_rt_pktbuf_set(struct ice_parser_rt *rt, const u8 *pkt_buf, + int pkt_len) +{ + int len = min(ICE_PARSER_MAX_PKT_LEN, pkt_len); + u16 ho = rt->gpr[ICE_GPR_HO_IDX]; + + memcpy(rt->pkt_buf, pkt_buf, len); + rt->pkt_len = pkt_len; + + memcpy(&rt->gpr[ICE_GPR_HV_IDX], &rt->pkt_buf[ho], ICE_GPR_HV_SIZE); +} + +static void ice_bst_key_init(struct ice_parser_rt *rt, + struct ice_imem_item *imem) +{ + u8 tsr = (u8)rt->gpr[ICE_GPR_TSR_IDX]; + u16 ho = rt->gpr[ICE_GPR_HO_IDX]; + u8 *key = rt->bst_key; + int idd, i; + + idd = ICE_BST_TCAM_KEY_SIZE - 1; + if (imem->b_kb.tsr_ctrl) + key[idd] = tsr; + else + key[idd] = imem->b_kb.prio; + + idd = ICE_BST_KEY_TCAM_SIZE - 1; + for (i = idd; i >= 0; i--) { + int j; + + j = ho + idd - i; + if (j < ICE_PARSER_MAX_PKT_LEN) + key[i] = rt->pkt_buf[ho + idd - i]; + else + key[i] = 0; + } + + ice_debug(rt->psr->hw, ICE_DBG_PARSER, "Generated Boost TCAM Key:\n"); + ice_debug(rt->psr->hw, ICE_DBG_PARSER, "%02X %02X %02X %02X %02X %02X %02X %02X %02X %02X\n", + key[0], key[1], key[2], key[3], key[4], + key[5], key[6], key[7], key[8], key[9]); + ice_debug(rt->psr->hw, ICE_DBG_PARSER, "\n"); +} + +static u16 ice_bit_rev_u16(u16 v, int len) +{ + return bitrev16(v) >> (BITS_PER_TYPE(v) - len); +} + +static u32 ice_bit_rev_u32(u32 v, int len) +{ + return bitrev32(v) >> (BITS_PER_TYPE(v) - len); +} + +static u32 ice_hv_bit_sel(struct ice_parser_rt *rt, int start, int len) +{ + int offset; + u32 buf[2]; + u64 val; + + offset = ICE_GPR_HV_IDX + (start / BITS_PER_TYPE(u16)); + + memcpy(buf, &rt->gpr[offset], sizeof(buf)); + + buf[0] = bitrev8x4(buf[0]); + buf[1] = bitrev8x4(buf[1]); + + val = *(u64 *)buf; + val >>= start % BITS_PER_TYPE(u16); + + return ice_bit_rev_u32(val, len); +} + +static u32 ice_pk_build(struct ice_parser_rt *rt, + struct ice_np_keybuilder *kb) +{ + if (kb->opc == ICE_NPKB_OPC_EXTRACT) + return ice_hv_bit_sel(rt, kb->start_reg0, kb->len_reg1); + else if (kb->opc == ICE_NPKB_OPC_BUILD) + return rt->gpr[kb->start_reg0] | + ((u32)rt->gpr[kb->len_reg1] << BITS_PER_TYPE(u16)); + else if (kb->opc == ICE_NPKB_OPC_BYPASS) + return 0; + + ice_debug(rt->psr->hw, ICE_DBG_PARSER, "Unsupported OP Code %u\n", + kb->opc); + return U32_MAX; +} + +static bool ice_flag_get(struct ice_parser_rt *rt, unsigned int index) +{ + int word = index / ICE_GPR_FLG_SIZE; + int id = index % ICE_GPR_FLG_SIZE; + + return !!(rt->gpr[ICE_GPR_FLG_IDX + word] & (u16)BIT(id)); +} + +static int ice_imem_pgk_init(struct ice_parser_rt *rt, + struct ice_imem_item *imem) +{ + memset(&rt->pg_key, 0, sizeof(rt->pg_key)); + rt->pg_key.next_proto = ice_pk_build(rt, &imem->np_kb); + if (rt->pg_key.next_proto == U32_MAX) + return -EINVAL; + + if (imem->pg_kb.flag0_ena) + rt->pg_key.flag0 = ice_flag_get(rt, imem->pg_kb.flag0_idx); + if (imem->pg_kb.flag1_ena) + rt->pg_key.flag1 = ice_flag_get(rt, imem->pg_kb.flag1_idx); + if (imem->pg_kb.flag2_ena) + rt->pg_key.flag2 = ice_flag_get(rt, imem->pg_kb.flag2_idx); + if (imem->pg_kb.flag3_ena) + rt->pg_key.flag3 = ice_flag_get(rt, imem->pg_kb.flag3_idx); + + rt->pg_key.alu_reg = rt->gpr[imem->pg_kb.alu_reg_idx]; + rt->pg_key.node_id = rt->gpr[ICE_GPR_NN_IDX]; + + ice_debug(rt->psr->hw, ICE_DBG_PARSER, "Generate Parse Graph Key: node_id(%d), flag0-3(%d,%d,%d,%d), boost_idx(%d), alu_reg(0x%04x), next_proto(0x%08x)\n", + rt->pg_key.node_id, + rt->pg_key.flag0, + rt->pg_key.flag1, + rt->pg_key.flag2, + rt->pg_key.flag3, + rt->pg_key.boost_idx, + rt->pg_key.alu_reg, + rt->pg_key.next_proto); + + return 0; +} + +static void ice_imem_alu0_set(struct ice_parser_rt *rt, + struct ice_imem_item *imem) +{ + rt->alu0 = &imem->alu0; + ice_debug(rt->psr->hw, ICE_DBG_PARSER, "Load ALU0 from imem pc %d\n", + imem->idx); +} + +static void ice_imem_alu1_set(struct ice_parser_rt *rt, + struct ice_imem_item *imem) +{ + rt->alu1 = &imem->alu1; + ice_debug(rt->psr->hw, ICE_DBG_PARSER, "Load ALU1 from imem pc %d\n", + imem->idx); +} + +static void ice_imem_alu2_set(struct ice_parser_rt *rt, + struct ice_imem_item *imem) +{ + rt->alu2 = &imem->alu2; + ice_debug(rt->psr->hw, ICE_DBG_PARSER, "Load ALU2 from imem pc %d\n", + imem->idx); +} + +static void ice_imem_pgp_set(struct ice_parser_rt *rt, + struct ice_imem_item *imem) +{ + rt->pg_prio = imem->pg_prio; + ice_debug(rt->psr->hw, ICE_DBG_PARSER, "Load PG priority %d from imem pc %d\n", + rt->pg_prio, imem->idx); +} + +static int ice_bst_pgk_init(struct ice_parser_rt *rt, + struct ice_bst_tcam_item *bst) +{ + memset(&rt->pg_key, 0, sizeof(rt->pg_key)); + rt->pg_key.boost_idx = bst->hit_idx_grp; + rt->pg_key.next_proto = ice_pk_build(rt, &bst->np_kb); + if (rt->pg_key.next_proto == U32_MAX) + return -EINVAL; + + if (bst->pg_kb.flag0_ena) + rt->pg_key.flag0 = ice_flag_get(rt, bst->pg_kb.flag0_idx); + if (bst->pg_kb.flag1_ena) + rt->pg_key.flag1 = ice_flag_get(rt, bst->pg_kb.flag1_idx); + if (bst->pg_kb.flag2_ena) + rt->pg_key.flag2 = ice_flag_get(rt, bst->pg_kb.flag2_idx); + if (bst->pg_kb.flag3_ena) + rt->pg_key.flag3 = ice_flag_get(rt, bst->pg_kb.flag3_idx); + + rt->pg_key.alu_reg = rt->gpr[bst->pg_kb.alu_reg_idx]; + rt->pg_key.node_id = rt->gpr[ICE_GPR_NN_IDX]; + + ice_debug(rt->psr->hw, ICE_DBG_PARSER, "Generate Parse Graph Key: node_id(%d), flag0-3(%d,%d,%d,%d), boost_idx(%d), alu_reg(0x%04x), next_proto(0x%08x)\n", + rt->pg_key.node_id, + rt->pg_key.flag0, + rt->pg_key.flag1, + rt->pg_key.flag2, + rt->pg_key.flag3, + rt->pg_key.boost_idx, + rt->pg_key.alu_reg, + rt->pg_key.next_proto); + + return 0; +} + +static void ice_bst_alu0_set(struct ice_parser_rt *rt, + struct ice_bst_tcam_item *bst) +{ + rt->alu0 = &bst->alu0; + ice_debug(rt->psr->hw, ICE_DBG_PARSER, "Load ALU0 from boost address %d\n", + bst->addr); +} + +static void ice_bst_alu1_set(struct ice_parser_rt *rt, + struct ice_bst_tcam_item *bst) +{ + rt->alu1 = &bst->alu1; + ice_debug(rt->psr->hw, ICE_DBG_PARSER, "Load ALU1 from boost address %d\n", + bst->addr); +} + +static void ice_bst_alu2_set(struct ice_parser_rt *rt, + struct ice_bst_tcam_item *bst) +{ + rt->alu2 = &bst->alu2; + ice_debug(rt->psr->hw, ICE_DBG_PARSER, "Load ALU2 from boost address %d\n", + bst->addr); +} + +static void ice_bst_pgp_set(struct ice_parser_rt *rt, + struct ice_bst_tcam_item *bst) +{ + rt->pg_prio = bst->pg_prio; + ice_debug(rt->psr->hw, ICE_DBG_PARSER, "Load PG priority %d from boost address %d\n", + rt->pg_prio, bst->addr); +} + +static struct ice_pg_cam_item *ice_rt_pg_cam_match(struct ice_parser_rt *rt) +{ + struct ice_parser *psr = rt->psr; + struct ice_pg_cam_item *item; + + item = ice_pg_cam_match(psr->pg_cam_table, ICE_PG_CAM_TABLE_SIZE, + &rt->pg_key); + if (!item) + item = ice_pg_cam_match(psr->pg_sp_cam_table, + ICE_PG_SP_CAM_TABLE_SIZE, &rt->pg_key); + return item; +} + +static +struct ice_pg_nm_cam_item *ice_rt_pg_nm_cam_match(struct ice_parser_rt *rt) +{ + struct ice_parser *psr = rt->psr; + struct ice_pg_nm_cam_item *item; + + item = ice_pg_nm_cam_match(psr->pg_nm_cam_table, + ICE_PG_NM_CAM_TABLE_SIZE, &rt->pg_key); + + if (!item) + item = ice_pg_nm_cam_match(psr->pg_nm_sp_cam_table, + ICE_PG_NM_SP_CAM_TABLE_SIZE, + &rt->pg_key); + return item; +} + +static void ice_gpr_add(struct ice_parser_rt *rt, int idx, u16 val) +{ + rt->pu.gpr_val_upd[idx] = true; + rt->pu.gpr_val[idx] = val; + + ice_debug(rt->psr->hw, ICE_DBG_PARSER, "Pending update for register %d value %d\n", + idx, val); +} + +static void ice_pg_exe(struct ice_parser_rt *rt) +{ + ice_debug(rt->psr->hw, ICE_DBG_PARSER, "Executing ParseGraph action ...\n"); + + ice_gpr_add(rt, ICE_GPR_NP_IDX, rt->action->next_pc); + ice_gpr_add(rt, ICE_GPR_NN_IDX, rt->action->next_node); + + ice_debug(rt->psr->hw, ICE_DBG_PARSER, "Executing ParseGraph action done.\n"); +} + +static void ice_flg_add(struct ice_parser_rt *rt, int idx, bool val) +{ + rt->pu.flg_msk |= BIT_ULL(idx); + if (val) + rt->pu.flg_val |= BIT_ULL(idx); + else + rt->pu.flg_val &= ~BIT_ULL(idx); + + ice_debug(rt->psr->hw, ICE_DBG_PARSER, "Pending update for flag %d value %d\n", + idx, val); +} + +static void ice_flg_update(struct ice_parser_rt *rt, struct ice_alu *alu) +{ + u32 hv_bit_sel; + int i; + + if (!alu->dedicate_flags_ena) + return; + + if (alu->flags_extr_imm) { + for (i = 0; i < alu->dst_len; i++) + ice_flg_add(rt, alu->dst_start + i, + !!(alu->flags_start_imm & BIT(i))); + } else { + for (i = 0; i < alu->dst_len; i++) { + hv_bit_sel = ice_hv_bit_sel(rt, + alu->flags_start_imm + i, + 1); + ice_flg_add(rt, alu->dst_start + i, !!hv_bit_sel); + } + } +} + +static void ice_po_update(struct ice_parser_rt *rt, struct ice_alu *alu) +{ + if (alu->proto_offset_opc == ICE_PO_OFF_HDR_ADD) + rt->po = (u16)(rt->gpr[ICE_GPR_HO_IDX] + alu->proto_offset); + else if (alu->proto_offset_opc == ICE_PO_OFF_HDR_SUB) + rt->po = (u16)(rt->gpr[ICE_GPR_HO_IDX] - alu->proto_offset); + else if (alu->proto_offset_opc == ICE_PO_OFF_REMAIN) + rt->po = rt->gpr[ICE_GPR_HO_IDX]; + + ice_debug(rt->psr->hw, ICE_DBG_PARSER, "Update Protocol Offset = %d\n", + rt->po); +} + +static u16 ice_reg_bit_sel(struct ice_parser_rt *rt, int reg_idx, + int start, int len) +{ + int offset; + u32 val; + + offset = ICE_GPR_HV_IDX + (start / BITS_PER_TYPE(u16)); + + memcpy(&val, &rt->gpr[offset], sizeof(val)); + + val = bitrev8x4(val); + val >>= start % BITS_PER_TYPE(u16); + + return ice_bit_rev_u16(val, len); +} + +static void ice_err_add(struct ice_parser_rt *rt, int idx, bool val) +{ + rt->pu.err_msk |= (u16)BIT(idx); + if (val) + rt->pu.flg_val |= (u64)BIT_ULL(idx); + else + rt->pu.flg_val &= ~(u64)BIT_ULL(idx); + + ice_debug(rt->psr->hw, ICE_DBG_PARSER, "Pending update for error %d value %d\n", + idx, val); +} + +static void ice_dst_reg_bit_set(struct ice_parser_rt *rt, struct ice_alu *alu, + bool val) +{ + u16 flg_idx; + + if (alu->dedicate_flags_ena) { + ice_debug(rt->psr->hw, ICE_DBG_PARSER, "DedicatedFlagsEnable should not be enabled in opcode %d\n", + alu->opc); + return; + } + + if (alu->dst_reg_id == ICE_GPR_ERR_IDX) { + if (alu->dst_start >= ICE_PARSER_ERR_NUM) { + ice_debug(rt->psr->hw, ICE_DBG_PARSER, "Invalid error %d\n", + alu->dst_start); + return; + } + ice_err_add(rt, alu->dst_start, val); + } else if (alu->dst_reg_id >= ICE_GPR_FLG_IDX) { + flg_idx = (u16)(((alu->dst_reg_id - ICE_GPR_FLG_IDX) << 4) + + alu->dst_start); + + if (flg_idx >= ICE_PARSER_FLG_NUM) { + ice_debug(rt->psr->hw, ICE_DBG_PARSER, "Invalid flag %d\n", + flg_idx); + return; + } + ice_flg_add(rt, flg_idx, val); + } else { + ice_debug(rt->psr->hw, ICE_DBG_PARSER, "Unexpected Dest Register Bit set, RegisterID %d Start %d\n", + alu->dst_reg_id, alu->dst_start); + } +} + +static void ice_alu_exe(struct ice_parser_rt *rt, struct ice_alu *alu) +{ + u16 dst, src, shift, imm; + + if (alu->shift_xlate_sel) { + ice_debug(rt->psr->hw, ICE_DBG_PARSER, "shift_xlate_sel != 0 is not expected\n"); + return; + } + + ice_po_update(rt, alu); + ice_flg_update(rt, alu); + + dst = rt->gpr[alu->dst_reg_id]; + src = ice_reg_bit_sel(rt, alu->src_reg_id, + alu->src_start, alu->src_len); + shift = alu->shift_xlate_key; + imm = alu->imm; + + switch (alu->opc) { + case ICE_ALU_PARK: + break; + case ICE_ALU_MOV_ADD: + dst = (src << shift) + imm; + ice_gpr_add(rt, alu->dst_reg_id, dst); + break; + case ICE_ALU_ADD: + dst += (src << shift) + imm; + ice_gpr_add(rt, alu->dst_reg_id, dst); + break; + case ICE_ALU_ORLT: + if (src < imm) + ice_dst_reg_bit_set(rt, alu, true); + ice_gpr_add(rt, ICE_GPR_NP_IDX, alu->branch_addr); + break; + case ICE_ALU_OREQ: + if (src == imm) + ice_dst_reg_bit_set(rt, alu, true); + ice_gpr_add(rt, ICE_GPR_NP_IDX, alu->branch_addr); + break; + case ICE_ALU_SETEQ: + ice_dst_reg_bit_set(rt, alu, src == imm); + ice_gpr_add(rt, ICE_GPR_NP_IDX, alu->branch_addr); + break; + case ICE_ALU_MOV_XOR: + dst = (src << shift) ^ imm; + ice_gpr_add(rt, alu->dst_reg_id, dst); + break; + default: + ice_debug(rt->psr->hw, ICE_DBG_PARSER, "Unsupported ALU instruction %d\n", + alu->opc); + break; + } +} + +static void ice_alu0_exe(struct ice_parser_rt *rt) +{ + ice_debug(rt->psr->hw, ICE_DBG_PARSER, "Executing ALU0 ...\n"); + ice_alu_exe(rt, rt->alu0); + ice_debug(rt->psr->hw, ICE_DBG_PARSER, "Executing ALU0 done.\n"); +} + +static void ice_alu1_exe(struct ice_parser_rt *rt) +{ + ice_debug(rt->psr->hw, ICE_DBG_PARSER, "Executing ALU1 ...\n"); + ice_alu_exe(rt, rt->alu1); + ice_debug(rt->psr->hw, ICE_DBG_PARSER, "Executing ALU1 done.\n"); +} + +static void ice_alu2_exe(struct ice_parser_rt *rt) +{ + ice_debug(rt->psr->hw, ICE_DBG_PARSER, "Executing ALU2 ...\n"); + ice_alu_exe(rt, rt->alu2); + ice_debug(rt->psr->hw, ICE_DBG_PARSER, "Executing ALU2 done.\n"); +} + +static void ice_pu_exe(struct ice_parser_rt *rt) +{ + struct ice_gpr_pu *pu = &rt->pu; + unsigned int i; + + ice_debug(rt->psr->hw, ICE_DBG_PARSER, "Updating Registers ...\n"); + + for (i = 0; i < ICE_PARSER_GPR_NUM; i++) { + if (pu->gpr_val_upd[i]) + ice_rt_gpr_set(rt, i, pu->gpr_val[i]); + } + + for (i = 0; i < ICE_PARSER_FLG_NUM; i++) { + if (pu->flg_msk & BIT(i)) + ice_rt_flag_set(rt, i, pu->flg_val & BIT(i)); + } + + for (i = 0; i < ICE_PARSER_ERR_NUM; i++) { + if (pu->err_msk & BIT(i)) + ice_rt_err_set(rt, i, pu->err_val & BIT(i)); + } + + ice_debug(rt->psr->hw, ICE_DBG_PARSER, "Updating Registers done.\n"); +} + +static void ice_alu_pg_exe(struct ice_parser_rt *rt) +{ + memset(&rt->pu, 0, sizeof(rt->pu)); + + switch (rt->pg_prio) { + case (ICE_PG_P0): + ice_pg_exe(rt); + ice_alu0_exe(rt); + ice_alu1_exe(rt); + ice_alu2_exe(rt); + break; + case (ICE_PG_P1): + ice_alu0_exe(rt); + ice_pg_exe(rt); + ice_alu1_exe(rt); + ice_alu2_exe(rt); + break; + case (ICE_PG_P2): + ice_alu0_exe(rt); + ice_alu1_exe(rt); + ice_pg_exe(rt); + ice_alu2_exe(rt); + break; + case (ICE_PG_P3): + ice_alu0_exe(rt); + ice_alu1_exe(rt); + ice_alu2_exe(rt); + ice_pg_exe(rt); + break; + } + + ice_pu_exe(rt); + + if (rt->action->ho_inc == 0) + return; + + if (rt->action->ho_polarity) + ice_rt_ho_set(rt, rt->gpr[ICE_GPR_HO_IDX] + rt->action->ho_inc); + else + ice_rt_ho_set(rt, rt->gpr[ICE_GPR_HO_IDX] - rt->action->ho_inc); +} + +static void ice_proto_off_update(struct ice_parser_rt *rt) +{ + struct ice_parser *psr = rt->psr; + + if (rt->action->is_pg) { + struct ice_proto_grp_item *proto_grp = + &psr->proto_grp_table[rt->action->proto_id]; + u16 po; + int i; + + for (i = 0; i < ICE_PROTO_COUNT_PER_GRP; i++) { + struct ice_proto_off *entry = &proto_grp->po[i]; + + if (entry->proto_id == U8_MAX) + break; + + if (!entry->polarity) + po = rt->po + entry->offset; + else + po = rt->po - entry->offset; + + rt->protocols[entry->proto_id] = true; + rt->offsets[entry->proto_id] = po; + + ice_debug(rt->psr->hw, ICE_DBG_PARSER, "Set Protocol %d at offset %d\n", + entry->proto_id, po); + } + } else { + rt->protocols[rt->action->proto_id] = true; + rt->offsets[rt->action->proto_id] = rt->po; + + ice_debug(rt->psr->hw, ICE_DBG_PARSER, "Set Protocol %d at offset %d\n", + rt->action->proto_id, rt->po); + } +} + +static void ice_marker_set(struct ice_parser_rt *rt, int idx) +{ + unsigned int byte = idx / BITS_PER_BYTE; + unsigned int bit = idx % BITS_PER_BYTE; + + rt->markers[byte] |= (u8)BIT(bit); +} + +static void ice_marker_update(struct ice_parser_rt *rt) +{ + struct ice_parser *psr = rt->psr; + + if (rt->action->is_mg) { + struct ice_mk_grp_item *mk_grp = + &psr->mk_grp_table[rt->action->marker_id]; + int i; + + for (i = 0; i < ICE_MARKER_ID_NUM; i++) { + u8 marker = mk_grp->markers[i]; + + if (marker == ICE_MARKER_MAX_SIZE) + break; + + ice_marker_set(rt, marker); + ice_debug(rt->psr->hw, ICE_DBG_PARSER, "Set Marker %d\n", + marker); + } + } else { + if (rt->action->marker_id != ICE_MARKER_MAX_SIZE) + ice_marker_set(rt, rt->action->marker_id); + + ice_debug(rt->psr->hw, ICE_DBG_PARSER, "Set Marker %d\n", + rt->action->marker_id); + } +} + +static u16 ice_ptype_resolve(struct ice_parser_rt *rt) +{ + struct ice_ptype_mk_tcam_item *item; + struct ice_parser *psr = rt->psr; + + item = ice_ptype_mk_tcam_match(psr->ptype_mk_tcam_table, + rt->markers, ICE_MARKER_ID_SIZE); + if (item) + return item->ptype; + + ice_debug(rt->psr->hw, ICE_DBG_PARSER, "Could not resolve PTYPE\n"); + return U16_MAX; +} + +static void ice_proto_off_resolve(struct ice_parser_rt *rt, + struct ice_parser_result *rslt) +{ + int i; + + for (i = 0; i < ICE_PO_PAIR_SIZE - 1; i++) { + if (rt->protocols[i]) { + rslt->po[rslt->po_num].proto_id = (u8)i; + rslt->po[rslt->po_num].offset = rt->offsets[i]; + rslt->po_num++; + } + } +} + +static void ice_result_resolve(struct ice_parser_rt *rt, + struct ice_parser_result *rslt) +{ + struct ice_parser *psr = rt->psr; + + memset(rslt, 0, sizeof(*rslt)); + + memcpy(&rslt->flags_psr, &rt->gpr[ICE_GPR_FLG_IDX], + ICE_PARSER_FLAG_PSR_SIZE); + rslt->flags_pkt = ice_flg_redirect(psr->flg_rd_table, rslt->flags_psr); + rslt->flags_sw = ice_xlt_kb_flag_get(psr->xlt_kb_sw, rslt->flags_pkt); + rslt->flags_fd = ice_xlt_kb_flag_get(psr->xlt_kb_fd, rslt->flags_pkt); + rslt->flags_rss = ice_xlt_kb_flag_get(psr->xlt_kb_rss, rslt->flags_pkt); + + ice_proto_off_resolve(rt, rslt); + rslt->ptype = ice_ptype_resolve(rt); +} + +/** + * ice_parser_rt_execute - parser execution routine + * @rt: pointer to the parser runtime + * @rslt: input/output parameter to save parser result + * + * Return: 0 on success or errno. + */ +int ice_parser_rt_execute(struct ice_parser_rt *rt, + struct ice_parser_result *rslt) +{ + struct ice_pg_nm_cam_item *pg_nm_cam; + struct ice_parser *psr = rt->psr; + struct ice_pg_cam_item *pg_cam; + int status = 0; + u16 node; + u16 pc; + + node = rt->gpr[ICE_GPR_NN_IDX]; + ice_debug(rt->psr->hw, ICE_DBG_PARSER, "Start with Node: %u\n", node); + + while (true) { + struct ice_bst_tcam_item *bst; + struct ice_imem_item *imem; + + pc = rt->gpr[ICE_GPR_NP_IDX]; + imem = &psr->imem_table[pc]; + ice_debug(rt->psr->hw, ICE_DBG_PARSER, "Load imem at pc: %u\n", + pc); + + ice_bst_key_init(rt, imem); + bst = ice_bst_tcam_match(psr->bst_tcam_table, rt->bst_key); + if (!bst) { + ice_debug(rt->psr->hw, ICE_DBG_PARSER, "No Boost TCAM Match\n"); + status = ice_imem_pgk_init(rt, imem); + if (status) + break; + ice_imem_alu0_set(rt, imem); + ice_imem_alu1_set(rt, imem); + ice_imem_alu2_set(rt, imem); + ice_imem_pgp_set(rt, imem); + } else { + ice_debug(rt->psr->hw, ICE_DBG_PARSER, "Boost TCAM Match address: %u\n", + bst->addr); + if (imem->b_m.pg) { + status = ice_bst_pgk_init(rt, bst); + if (status) + break; + ice_bst_pgp_set(rt, bst); + } else { + status = ice_imem_pgk_init(rt, imem); + if (status) + break; + ice_imem_pgp_set(rt, imem); + } + + if (imem->b_m.alu0) + ice_bst_alu0_set(rt, bst); + else + ice_imem_alu0_set(rt, imem); + + if (imem->b_m.alu1) + ice_bst_alu1_set(rt, bst); + else + ice_imem_alu1_set(rt, imem); + + if (imem->b_m.alu2) + ice_bst_alu2_set(rt, bst); + else + ice_imem_alu2_set(rt, imem); + } + + rt->action = NULL; + pg_cam = ice_rt_pg_cam_match(rt); + if (!pg_cam) { + pg_nm_cam = ice_rt_pg_nm_cam_match(rt); + if (pg_nm_cam) { + ice_debug(rt->psr->hw, ICE_DBG_PARSER, "Match ParseGraph Nomatch CAM Address %u\n", + pg_nm_cam->idx); + rt->action = &pg_nm_cam->action; + } + } else { + ice_debug(rt->psr->hw, ICE_DBG_PARSER, "Match ParseGraph CAM Address %u\n", + pg_cam->idx); + rt->action = &pg_cam->action; + } + + if (!rt->action) { + ice_debug(rt->psr->hw, ICE_DBG_PARSER, "Failed to match ParseGraph CAM, stop parsing.\n"); + status = -EINVAL; + break; + } + + ice_alu_pg_exe(rt); + ice_marker_update(rt); + ice_proto_off_update(rt); + + ice_debug(rt->psr->hw, ICE_DBG_PARSER, "Go to node %u\n", + rt->action->next_node); + + if (rt->action->is_last_round) { + ice_debug(rt->psr->hw, ICE_DBG_PARSER, "Last Round in ParseGraph Action, stop parsing.\n"); + break; + } + + if (rt->gpr[ICE_GPR_HO_IDX] >= rt->pkt_len) { + ice_debug(rt->psr->hw, ICE_DBG_PARSER, "Header Offset (%u) is larger than packet len (%u), stop parsing\n", + rt->gpr[ICE_GPR_HO_IDX], rt->pkt_len); + break; + } + } + + ice_result_resolve(rt, rslt); + + return status; +} diff --git a/drivers/net/ethernet/intel/ice/ice_repr.c b/drivers/net/ethernet/intel/ice/ice_repr.c index bdda3401e343..970a99a52bf1 100644 --- a/drivers/net/ethernet/intel/ice/ice_repr.c +++ b/drivers/net/ethernet/intel/ice/ice_repr.c @@ -59,12 +59,13 @@ static void ice_repr_get_stats64(struct net_device *netdev, struct rtnl_link_stats64 *stats) { struct ice_netdev_priv *np = netdev_priv(netdev); + struct ice_repr *repr = np->repr; struct ice_eth_stats *eth_stats; struct ice_vsi *vsi; - if (ice_is_vf_disabled(np->repr->vf)) + if (repr->ops.ready(repr)) return; - vsi = np->repr->src_vsi; + vsi = repr->src_vsi; ice_update_vsi_stats(vsi); eth_stats = &vsi->eth_stats; @@ -93,7 +94,7 @@ struct ice_repr *ice_netdev_to_repr(const struct net_device *netdev) } /** - * ice_repr_open - Enable port representor's network interface + * ice_repr_vf_open - Enable port representor's network interface * @netdev: network interface device structure * * The open entry point is called when a port representor's network @@ -102,7 +103,7 @@ struct ice_repr *ice_netdev_to_repr(const struct net_device *netdev) * * Returns 0 on success */ -static int ice_repr_open(struct net_device *netdev) +static int ice_repr_vf_open(struct net_device *netdev) { struct ice_repr *repr = ice_netdev_to_repr(netdev); struct ice_vf *vf; @@ -118,8 +119,16 @@ static int ice_repr_open(struct net_device *netdev) return 0; } +static int ice_repr_sf_open(struct net_device *netdev) +{ + netif_carrier_on(netdev); + netif_tx_start_all_queues(netdev); + + return 0; +} + /** - * ice_repr_stop - Disable port representor's network interface + * ice_repr_vf_stop - Disable port representor's network interface * @netdev: network interface device structure * * The stop entry point is called when a port representor's network @@ -128,7 +137,7 @@ static int ice_repr_open(struct net_device *netdev) * * Returns 0 on success */ -static int ice_repr_stop(struct net_device *netdev) +static int ice_repr_vf_stop(struct net_device *netdev) { struct ice_repr *repr = ice_netdev_to_repr(netdev); struct ice_vf *vf; @@ -144,6 +153,14 @@ static int ice_repr_stop(struct net_device *netdev) return 0; } +static int ice_repr_sf_stop(struct net_device *netdev) +{ + netif_carrier_off(netdev); + netif_tx_stop_all_queues(netdev); + + return 0; +} + /** * ice_repr_sp_stats64 - get slow path stats for port representor * @dev: network interface device structure @@ -245,10 +262,20 @@ ice_repr_setup_tc(struct net_device *netdev, enum tc_setup_type type, } } -static const struct net_device_ops ice_repr_netdev_ops = { +static const struct net_device_ops ice_repr_vf_netdev_ops = { .ndo_get_stats64 = ice_repr_get_stats64, - .ndo_open = ice_repr_open, - .ndo_stop = ice_repr_stop, + .ndo_open = ice_repr_vf_open, + .ndo_stop = ice_repr_vf_stop, + .ndo_start_xmit = ice_eswitch_port_start_xmit, + .ndo_setup_tc = ice_repr_setup_tc, + .ndo_has_offload_stats = ice_repr_ndo_has_offload_stats, + .ndo_get_offload_stats = ice_repr_ndo_get_offload_stats, +}; + +static const struct net_device_ops ice_repr_sf_netdev_ops = { + .ndo_get_stats64 = ice_repr_get_stats64, + .ndo_open = ice_repr_sf_open, + .ndo_stop = ice_repr_sf_stop, .ndo_start_xmit = ice_eswitch_port_start_xmit, .ndo_setup_tc = ice_repr_setup_tc, .ndo_has_offload_stats = ice_repr_ndo_has_offload_stats, @@ -261,18 +288,20 @@ static const struct net_device_ops ice_repr_netdev_ops = { */ bool ice_is_port_repr_netdev(const struct net_device *netdev) { - return netdev && (netdev->netdev_ops == &ice_repr_netdev_ops); + return netdev && (netdev->netdev_ops == &ice_repr_vf_netdev_ops || + netdev->netdev_ops == &ice_repr_sf_netdev_ops); } /** * ice_repr_reg_netdev - register port representor netdev * @netdev: pointer to port representor netdev + * @ops: new ops for netdev */ static int -ice_repr_reg_netdev(struct net_device *netdev) +ice_repr_reg_netdev(struct net_device *netdev, const struct net_device_ops *ops) { eth_hw_addr_random(netdev); - netdev->netdev_ops = &ice_repr_netdev_ops; + netdev->netdev_ops = ops; ice_set_ethtool_repr_ops(netdev); netdev->hw_features |= NETIF_F_HW_TC; @@ -283,57 +312,56 @@ ice_repr_reg_netdev(struct net_device *netdev) return register_netdev(netdev); } -static void ice_repr_remove_node(struct devlink_port *devlink_port) +static int ice_repr_ready_vf(struct ice_repr *repr) +{ + return !ice_check_vf_ready_for_cfg(repr->vf); +} + +static int ice_repr_ready_sf(struct ice_repr *repr) { - devl_rate_leaf_destroy(devlink_port); + return !repr->sf->active; } /** - * ice_repr_rem - remove representor from VF + * ice_repr_destroy - remove representor from VF * @repr: pointer to representor structure */ -static void ice_repr_rem(struct ice_repr *repr) +void ice_repr_destroy(struct ice_repr *repr) { free_percpu(repr->stats); free_netdev(repr->netdev); kfree(repr); } -/** - * ice_repr_rem_vf - remove representor from VF - * @repr: pointer to representor structure - */ -void ice_repr_rem_vf(struct ice_repr *repr) +static void ice_repr_rem_vf(struct ice_repr *repr) { - ice_repr_remove_node(&repr->vf->devlink_port); ice_eswitch_decfg_vsi(repr->src_vsi, repr->parent_mac); unregister_netdev(repr->netdev); ice_devlink_destroy_vf_port(repr->vf); ice_virtchnl_set_dflt_ops(repr->vf); - ice_repr_rem(repr); } -static void ice_repr_set_tx_topology(struct ice_pf *pf) +static void ice_repr_rem_sf(struct ice_repr *repr) { - struct devlink *devlink; + unregister_netdev(repr->netdev); + ice_devlink_destroy_sf_port(repr->sf); +} +static void ice_repr_set_tx_topology(struct ice_pf *pf, struct devlink *devlink) +{ /* only export if ADQ and DCB disabled and eswitch enabled*/ if (ice_is_adq_active(pf) || ice_is_dcb_active(pf) || !ice_is_switchdev_running(pf)) return; - devlink = priv_to_devlink(pf); ice_devlink_rate_init_tx_topology(devlink, ice_get_main_vsi(pf)); } /** - * ice_repr_add - add representor for generic VSI - * @pf: pointer to PF structure + * ice_repr_create - add representor for generic VSI * @src_vsi: pointer to VSI structure of device to represent - * @parent_mac: device MAC address */ -static struct ice_repr * -ice_repr_add(struct ice_pf *pf, struct ice_vsi *src_vsi, const u8 *parent_mac) +static struct ice_repr *ice_repr_create(struct ice_vsi *src_vsi) { struct ice_netdev_priv *np; struct ice_repr *repr; @@ -360,7 +388,10 @@ ice_repr_add(struct ice_pf *pf, struct ice_vsi *src_vsi, const u8 *parent_mac) np = netdev_priv(repr->netdev); np->repr = repr; - ether_addr_copy(repr->parent_mac, parent_mac); + repr->netdev->min_mtu = ETH_MIN_MTU; + repr->netdev->max_mtu = ICE_MAX_MTU; + + SET_NETDEV_DEV(repr->netdev, ice_pf_to_dev(src_vsi->back)); return repr; @@ -371,34 +402,18 @@ err_alloc: return ERR_PTR(err); } -struct ice_repr *ice_repr_add_vf(struct ice_vf *vf) +static int ice_repr_add_vf(struct ice_repr *repr) { - struct ice_repr *repr; - struct ice_vsi *vsi; + struct ice_vf *vf = repr->vf; + struct devlink *devlink; int err; - vsi = ice_get_vf_vsi(vf); - if (!vsi) - return ERR_PTR(-ENOENT); - err = ice_devlink_create_vf_port(vf); if (err) - return ERR_PTR(err); - - repr = ice_repr_add(vf->pf, vsi, vf->hw_lan_addr); - if (IS_ERR(repr)) { - err = PTR_ERR(repr); - goto err_repr_add; - } - - repr->vf = vf; - - repr->netdev->min_mtu = ETH_MIN_MTU; - repr->netdev->max_mtu = ICE_MAX_MTU; + return err; - SET_NETDEV_DEV(repr->netdev, ice_pf_to_dev(vf->pf)); SET_NETDEV_DEVLINK_PORT(repr->netdev, &vf->devlink_port); - err = ice_repr_reg_netdev(repr->netdev); + err = ice_repr_reg_netdev(repr->netdev, &ice_repr_vf_netdev_ops); if (err) goto err_netdev; @@ -407,17 +422,97 @@ struct ice_repr *ice_repr_add_vf(struct ice_vf *vf) goto err_cfg_vsi; ice_virtchnl_set_repr_ops(vf); - ice_repr_set_tx_topology(vf->pf); - return repr; + devlink = priv_to_devlink(vf->pf); + ice_repr_set_tx_topology(vf->pf, devlink); + + return 0; err_cfg_vsi: unregister_netdev(repr->netdev); err_netdev: - ice_repr_rem(repr); -err_repr_add: ice_devlink_destroy_vf_port(vf); - return ERR_PTR(err); + return err; +} + +/** + * ice_repr_create_vf - add representor for VF VSI + * @vf: VF to create port representor on + * + * Set correct representor type for VF and functions pointer. + * + * Return: created port representor on success, error otherwise + */ +struct ice_repr *ice_repr_create_vf(struct ice_vf *vf) +{ + struct ice_vsi *vsi = ice_get_vf_vsi(vf); + struct ice_repr *repr; + + if (!vsi) + return ERR_PTR(-EINVAL); + + repr = ice_repr_create(vsi); + if (IS_ERR(repr)) + return repr; + + repr->type = ICE_REPR_TYPE_VF; + repr->vf = vf; + repr->ops.add = ice_repr_add_vf; + repr->ops.rem = ice_repr_rem_vf; + repr->ops.ready = ice_repr_ready_vf; + + ether_addr_copy(repr->parent_mac, vf->hw_lan_addr); + + return repr; +} + +static int ice_repr_add_sf(struct ice_repr *repr) +{ + struct ice_dynamic_port *sf = repr->sf; + int err; + + err = ice_devlink_create_sf_port(sf); + if (err) + return err; + + SET_NETDEV_DEVLINK_PORT(repr->netdev, &sf->devlink_port); + err = ice_repr_reg_netdev(repr->netdev, &ice_repr_sf_netdev_ops); + if (err) + goto err_netdev; + + ice_repr_set_tx_topology(sf->vsi->back, priv_to_devlink(sf->vsi->back)); + + return 0; + +err_netdev: + ice_devlink_destroy_sf_port(sf); + return err; +} + +/** + * ice_repr_create_sf - add representor for SF VSI + * @sf: SF to create port representor on + * + * Set correct representor type for SF and functions pointer. + * + * Return: created port representor on success, error otherwise + */ +struct ice_repr *ice_repr_create_sf(struct ice_dynamic_port *sf) +{ + struct ice_repr *repr = ice_repr_create(sf->vsi); + + if (IS_ERR(repr)) + return repr; + + repr->type = ICE_REPR_TYPE_SF; + repr->sf = sf; + repr->ops.add = ice_repr_add_sf; + repr->ops.rem = ice_repr_rem_sf; + repr->ops.ready = ice_repr_ready_sf; + + ether_addr_copy(repr->parent_mac, sf->hw_addr); + + return repr; } struct ice_repr *ice_repr_get(struct ice_pf *pf, u32 id) diff --git a/drivers/net/ethernet/intel/ice/ice_repr.h b/drivers/net/ethernet/intel/ice/ice_repr.h index 488661b2900b..35bd93165e1e 100644 --- a/drivers/net/ethernet/intel/ice/ice_repr.h +++ b/drivers/net/ethernet/intel/ice/ice_repr.h @@ -15,19 +15,35 @@ struct ice_repr_pcpu_stats { u64 tx_drops; }; +enum ice_repr_type { + ICE_REPR_TYPE_VF, + ICE_REPR_TYPE_SF, +}; + struct ice_repr { struct ice_vsi *src_vsi; - struct ice_vf *vf; struct net_device *netdev; struct metadata_dst *dst; struct ice_esw_br_port *br_port; struct ice_repr_pcpu_stats __percpu *stats; u32 id; u8 parent_mac[ETH_ALEN]; + enum ice_repr_type type; + union { + struct ice_vf *vf; + struct ice_dynamic_port *sf; + }; + struct { + int (*add)(struct ice_repr *repr); + void (*rem)(struct ice_repr *repr); + int (*ready)(struct ice_repr *repr); + } ops; }; -struct ice_repr *ice_repr_add_vf(struct ice_vf *vf); -void ice_repr_rem_vf(struct ice_repr *repr); +struct ice_repr *ice_repr_create_vf(struct ice_vf *vf); +struct ice_repr *ice_repr_create_sf(struct ice_dynamic_port *sf); + +void ice_repr_destroy(struct ice_repr *repr); void ice_repr_start_tx_queues(struct ice_repr *repr); void ice_repr_stop_tx_queues(struct ice_repr *repr); diff --git a/drivers/net/ethernet/intel/ice/ice_sched.c b/drivers/net/ethernet/intel/ice/ice_sched.c index ecf8f5d60292..6ca13c5dcb14 100644 --- a/drivers/net/ethernet/intel/ice/ice_sched.c +++ b/drivers/net/ethernet/intel/ice/ice_sched.c @@ -28,9 +28,8 @@ ice_sched_add_root_node(struct ice_port_info *pi, if (!root) return -ENOMEM; - /* coverity[suspicious_sizeof] */ root->children = devm_kcalloc(ice_hw_to_dev(hw), hw->max_children[0], - sizeof(*root), GFP_KERNEL); + sizeof(*root->children), GFP_KERNEL); if (!root->children) { devm_kfree(ice_hw_to_dev(hw), root); return -ENOMEM; @@ -186,10 +185,9 @@ ice_sched_add_node(struct ice_port_info *pi, u8 layer, if (!node) return -ENOMEM; if (hw->max_children[layer]) { - /* coverity[suspicious_sizeof] */ node->children = devm_kcalloc(ice_hw_to_dev(hw), hw->max_children[layer], - sizeof(*node), GFP_KERNEL); + sizeof(*node->children), GFP_KERNEL); if (!node->children) { devm_kfree(ice_hw_to_dev(hw), node); return -ENOMEM; diff --git a/drivers/net/ethernet/intel/ice/ice_sf_eth.c b/drivers/net/ethernet/intel/ice/ice_sf_eth.c new file mode 100644 index 000000000000..75d7147e1c01 --- /dev/null +++ b/drivers/net/ethernet/intel/ice/ice_sf_eth.c @@ -0,0 +1,329 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2024, Intel Corporation. */ +#include "ice.h" +#include "ice_lib.h" +#include "ice_txrx.h" +#include "ice_fltr.h" +#include "ice_sf_eth.h" +#include "devlink/devlink_port.h" +#include "devlink/devlink.h" + +static const struct net_device_ops ice_sf_netdev_ops = { + .ndo_open = ice_open, + .ndo_stop = ice_stop, + .ndo_start_xmit = ice_start_xmit, + .ndo_vlan_rx_add_vid = ice_vlan_rx_add_vid, + .ndo_vlan_rx_kill_vid = ice_vlan_rx_kill_vid, + .ndo_change_mtu = ice_change_mtu, + .ndo_get_stats64 = ice_get_stats64, + .ndo_tx_timeout = ice_tx_timeout, + .ndo_bpf = ice_xdp, + .ndo_xdp_xmit = ice_xdp_xmit, + .ndo_xsk_wakeup = ice_xsk_wakeup, +}; + +/** + * ice_sf_cfg_netdev - Allocate, configure and register a netdev + * @dyn_port: subfunction associated with configured netdev + * @devlink_port: subfunction devlink port to be linked with netdev + * + * Return: 0 on success, negative value on failure + */ +static int ice_sf_cfg_netdev(struct ice_dynamic_port *dyn_port, + struct devlink_port *devlink_port) +{ + struct ice_vsi *vsi = dyn_port->vsi; + struct ice_netdev_priv *np; + struct net_device *netdev; + int err; + + netdev = alloc_etherdev_mqs(sizeof(*np), vsi->alloc_txq, + vsi->alloc_rxq); + if (!netdev) + return -ENOMEM; + + SET_NETDEV_DEV(netdev, &vsi->back->pdev->dev); + set_bit(ICE_VSI_NETDEV_ALLOCD, vsi->state); + vsi->netdev = netdev; + np = netdev_priv(netdev); + np->vsi = vsi; + + ice_set_netdev_features(netdev); + + netdev->xdp_features = NETDEV_XDP_ACT_BASIC | NETDEV_XDP_ACT_REDIRECT | + NETDEV_XDP_ACT_XSK_ZEROCOPY | + NETDEV_XDP_ACT_RX_SG; + netdev->xdp_zc_max_segs = ICE_MAX_BUF_TXD; + + eth_hw_addr_set(netdev, dyn_port->hw_addr); + ether_addr_copy(netdev->perm_addr, dyn_port->hw_addr); + netdev->netdev_ops = &ice_sf_netdev_ops; + SET_NETDEV_DEVLINK_PORT(netdev, devlink_port); + + err = register_netdev(netdev); + if (err) { + free_netdev(netdev); + vsi->netdev = NULL; + return -ENOMEM; + } + set_bit(ICE_VSI_NETDEV_REGISTERED, vsi->state); + netif_carrier_off(netdev); + netif_tx_stop_all_queues(netdev); + + return 0; +} + +static void ice_sf_decfg_netdev(struct ice_vsi *vsi) +{ + unregister_netdev(vsi->netdev); + clear_bit(ICE_VSI_NETDEV_REGISTERED, vsi->state); + free_netdev(vsi->netdev); + vsi->netdev = NULL; + clear_bit(ICE_VSI_NETDEV_ALLOCD, vsi->state); +} + +/** + * ice_sf_dev_probe - subfunction driver probe function + * @adev: pointer to the auxiliary device + * @id: pointer to the auxiliary_device id + * + * Configure VSI and netdev resources for the subfunction device. + * + * Return: zero on success or an error code on failure. + */ +static int ice_sf_dev_probe(struct auxiliary_device *adev, + const struct auxiliary_device_id *id) +{ + struct ice_sf_dev *sf_dev = ice_adev_to_sf_dev(adev); + struct ice_dynamic_port *dyn_port = sf_dev->dyn_port; + struct ice_vsi *vsi = dyn_port->vsi; + struct ice_pf *pf = dyn_port->pf; + struct device *dev = &adev->dev; + struct ice_sf_priv *priv; + struct devlink *devlink; + int err; + + vsi->type = ICE_VSI_SF; + vsi->port_info = pf->hw.port_info; + vsi->flags = ICE_VSI_FLAG_INIT; + + priv = ice_allocate_sf(&adev->dev, pf); + if (IS_ERR(priv)) { + dev_err(dev, "Subfunction devlink alloc failed"); + return PTR_ERR(priv); + } + + priv->dev = sf_dev; + sf_dev->priv = priv; + devlink = priv_to_devlink(priv); + + devl_lock(devlink); + + err = ice_vsi_cfg(vsi); + if (err) { + dev_err(dev, "Subfunction vsi config failed"); + goto err_free_devlink; + } + vsi->sf = dyn_port; + + ice_eswitch_update_repr(&dyn_port->repr_id, vsi); + + err = ice_devlink_create_sf_dev_port(sf_dev); + if (err) { + dev_err(dev, "Cannot add ice virtual devlink port for subfunction"); + goto err_vsi_decfg; + } + + err = ice_sf_cfg_netdev(dyn_port, &sf_dev->priv->devlink_port); + if (err) { + dev_err(dev, "Subfunction netdev config failed"); + goto err_devlink_destroy; + } + + err = devl_port_fn_devlink_set(&dyn_port->devlink_port, devlink); + if (err) { + dev_err(dev, "Can't link devlink instance to SF devlink port"); + goto err_netdev_decfg; + } + + ice_napi_add(vsi); + + devl_register(devlink); + devl_unlock(devlink); + + dyn_port->attached = true; + + return 0; + +err_netdev_decfg: + ice_sf_decfg_netdev(vsi); +err_devlink_destroy: + ice_devlink_destroy_sf_dev_port(sf_dev); +err_vsi_decfg: + ice_vsi_decfg(vsi); +err_free_devlink: + devl_unlock(devlink); + devlink_free(devlink); + return err; +} + +/** + * ice_sf_dev_remove - subfunction driver remove function + * @adev: pointer to the auxiliary device + * + * Deinitalize VSI and netdev resources for the subfunction device. + */ +static void ice_sf_dev_remove(struct auxiliary_device *adev) +{ + struct ice_sf_dev *sf_dev = ice_adev_to_sf_dev(adev); + struct ice_dynamic_port *dyn_port = sf_dev->dyn_port; + struct ice_vsi *vsi = dyn_port->vsi; + struct devlink *devlink; + + devlink = priv_to_devlink(sf_dev->priv); + devl_lock(devlink); + + ice_vsi_close(vsi); + + ice_sf_decfg_netdev(vsi); + ice_devlink_destroy_sf_dev_port(sf_dev); + devl_unregister(devlink); + devl_unlock(devlink); + devlink_free(devlink); + ice_vsi_decfg(vsi); + + dyn_port->attached = false; +} + +static const struct auxiliary_device_id ice_sf_dev_id_table[] = { + { .name = "ice.sf", }, + { }, +}; + +MODULE_DEVICE_TABLE(auxiliary, ice_sf_dev_id_table); + +static struct auxiliary_driver ice_sf_driver = { + .name = "sf", + .probe = ice_sf_dev_probe, + .remove = ice_sf_dev_remove, + .id_table = ice_sf_dev_id_table +}; + +static DEFINE_XARRAY_ALLOC1(ice_sf_aux_id); + +/** + * ice_sf_driver_register - Register new auxiliary subfunction driver + * + * Return: zero on success or an error code on failure. + */ +int ice_sf_driver_register(void) +{ + return auxiliary_driver_register(&ice_sf_driver); +} + +/** + * ice_sf_driver_unregister - Unregister new auxiliary subfunction driver + * + */ +void ice_sf_driver_unregister(void) +{ + auxiliary_driver_unregister(&ice_sf_driver); +} + +/** + * ice_sf_dev_release - Release device associated with auxiliary device + * @device: pointer to the device + * + * Since most of the code for subfunction deactivation is handled in + * the remove handler, here just free tracking resources. + */ +static void ice_sf_dev_release(struct device *device) +{ + struct auxiliary_device *adev = to_auxiliary_dev(device); + struct ice_sf_dev *sf_dev = ice_adev_to_sf_dev(adev); + + xa_erase(&ice_sf_aux_id, adev->id); + kfree(sf_dev); +} + +/** + * ice_sf_eth_activate - Activate Ethernet subfunction port + * @dyn_port: the dynamic port instance for this subfunction + * @extack: extack for reporting error messages + * + * Activate the dynamic port as an Ethernet subfunction. Setup the netdev + * resources associated and initialize the auxiliary device. + * + * Return: zero on success or an error code on failure. + */ +int +ice_sf_eth_activate(struct ice_dynamic_port *dyn_port, + struct netlink_ext_ack *extack) +{ + struct ice_pf *pf = dyn_port->pf; + struct ice_sf_dev *sf_dev; + struct pci_dev *pdev; + int err; + u32 id; + + err = xa_alloc(&ice_sf_aux_id, &id, NULL, xa_limit_32b, + GFP_KERNEL); + if (err) { + NL_SET_ERR_MSG_MOD(extack, "Could not allocate SF ID"); + return err; + } + + sf_dev = kzalloc(sizeof(*sf_dev), GFP_KERNEL); + if (!sf_dev) { + err = -ENOMEM; + NL_SET_ERR_MSG_MOD(extack, "Could not allocate SF memory"); + goto xa_erase; + } + pdev = pf->pdev; + + sf_dev->dyn_port = dyn_port; + sf_dev->adev.id = id; + sf_dev->adev.name = "sf"; + sf_dev->adev.dev.release = ice_sf_dev_release; + sf_dev->adev.dev.parent = &pdev->dev; + + err = auxiliary_device_init(&sf_dev->adev); + if (err) { + NL_SET_ERR_MSG_MOD(extack, "Failed to initialize SF device"); + goto sf_dev_free; + } + + err = auxiliary_device_add(&sf_dev->adev); + if (err) { + NL_SET_ERR_MSG_MOD(extack, "Failed to add SF device"); + goto aux_dev_uninit; + } + + dyn_port->sf_dev = sf_dev; + + return 0; + +aux_dev_uninit: + auxiliary_device_uninit(&sf_dev->adev); +sf_dev_free: + kfree(sf_dev); +xa_erase: + xa_erase(&ice_sf_aux_id, id); + + return err; +} + +/** + * ice_sf_eth_deactivate - Deactivate Ethernet subfunction port + * @dyn_port: the dynamic port instance for this subfunction + * + * Deactivate the Ethernet subfunction, removing its auxiliary device and the + * associated resources. + */ +void ice_sf_eth_deactivate(struct ice_dynamic_port *dyn_port) +{ + struct ice_sf_dev *sf_dev = dyn_port->sf_dev; + + auxiliary_device_delete(&sf_dev->adev); + auxiliary_device_uninit(&sf_dev->adev); +} diff --git a/drivers/net/ethernet/intel/ice/ice_sf_eth.h b/drivers/net/ethernet/intel/ice/ice_sf_eth.h new file mode 100644 index 000000000000..c558cad0a183 --- /dev/null +++ b/drivers/net/ethernet/intel/ice/ice_sf_eth.h @@ -0,0 +1,33 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright (c) 2024, Intel Corporation. */ + +#ifndef _ICE_SF_ETH_H_ +#define _ICE_SF_ETH_H_ + +#include <linux/auxiliary_bus.h> +#include "ice.h" + +struct ice_sf_dev { + struct auxiliary_device adev; + struct ice_dynamic_port *dyn_port; + struct ice_sf_priv *priv; +}; + +struct ice_sf_priv { + struct ice_sf_dev *dev; + struct devlink_port devlink_port; +}; + +static inline struct +ice_sf_dev *ice_adev_to_sf_dev(struct auxiliary_device *adev) +{ + return container_of(adev, struct ice_sf_dev, adev); +} + +int ice_sf_driver_register(void); +void ice_sf_driver_unregister(void); + +int ice_sf_eth_activate(struct ice_dynamic_port *dyn_port, + struct netlink_ext_ack *extack); +void ice_sf_eth_deactivate(struct ice_dynamic_port *dyn_port); +#endif /* _ICE_SF_ETH_H_ */ diff --git a/drivers/net/ethernet/intel/ice/ice_sf_vsi_vlan_ops.c b/drivers/net/ethernet/intel/ice/ice_sf_vsi_vlan_ops.c new file mode 100644 index 000000000000..3d7e96721cf9 --- /dev/null +++ b/drivers/net/ethernet/intel/ice/ice_sf_vsi_vlan_ops.c @@ -0,0 +1,21 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2023, Intel Corporation. */ + +#include "ice_vsi_vlan_ops.h" +#include "ice_vsi_vlan_lib.h" +#include "ice_vlan_mode.h" +#include "ice.h" +#include "ice_sf_vsi_vlan_ops.h" + +void ice_sf_vsi_init_vlan_ops(struct ice_vsi *vsi) +{ + struct ice_vsi_vlan_ops *vlan_ops; + + if (ice_is_dvm_ena(&vsi->back->hw)) + vlan_ops = &vsi->outer_vlan_ops; + else + vlan_ops = &vsi->inner_vlan_ops; + + vlan_ops->add_vlan = ice_vsi_add_vlan; + vlan_ops->del_vlan = ice_vsi_del_vlan; +} diff --git a/drivers/net/ethernet/intel/ice/ice_sf_vsi_vlan_ops.h b/drivers/net/ethernet/intel/ice/ice_sf_vsi_vlan_ops.h new file mode 100644 index 000000000000..8c44eafceea0 --- /dev/null +++ b/drivers/net/ethernet/intel/ice/ice_sf_vsi_vlan_ops.h @@ -0,0 +1,13 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright (c) 2023, Intel Corporation. */ + +#ifndef _ICE_SF_VSI_VLAN_OPS_H_ +#define _ICE_SF_VSI_VLAN_OPS_H_ + +#include "ice_vsi_vlan_ops.h" + +struct ice_vsi; + +void ice_sf_vsi_init_vlan_ops(struct ice_vsi *vsi); + +#endif /* _ICE_SF_VSI_VLAN_OPS_H_ */ diff --git a/drivers/net/ethernet/intel/ice/ice_sriov.c b/drivers/net/ethernet/intel/ice/ice_sriov.c index 55ef33208456..e34fe2516ccc 100644 --- a/drivers/net/ethernet/intel/ice/ice_sriov.c +++ b/drivers/net/ethernet/intel/ice/ice_sriov.c @@ -175,7 +175,7 @@ void ice_free_vfs(struct ice_pf *pf) ice_for_each_vf(pf, bkt, vf) { mutex_lock(&vf->cfg_lock); - ice_eswitch_detach(pf, vf); + ice_eswitch_detach_vf(pf, vf); ice_dis_vf_qs(vf); if (test_bit(ICE_VF_STATE_INIT, vf->vf_states)) { @@ -598,7 +598,7 @@ static int ice_start_vfs(struct ice_pf *pf) goto teardown; } - retval = ice_eswitch_attach(pf, vf); + retval = ice_eswitch_attach_vf(pf, vf); if (retval) { dev_err(ice_pf_to_dev(pf), "Failed to attach VF %d to eswitch, error %d", vf->vf_id, retval); diff --git a/drivers/net/ethernet/intel/ice/ice_switch.c b/drivers/net/ethernet/intel/ice/ice_switch.c index fe8847184cb1..79d91e95358c 100644 --- a/drivers/net/ethernet/intel/ice/ice_switch.c +++ b/drivers/net/ethernet/intel/ice/ice_switch.c @@ -3194,7 +3194,7 @@ ice_add_update_vsi_list(struct ice_hw *hw, /* A rule already exists with the new VSI being added */ if (test_bit(vsi_handle, m_entry->vsi_list_info->vsi_map)) - return 0; + return -EEXIST; /* Update the previously created VSI list set with * the new VSI ID passed in @@ -3264,7 +3264,7 @@ ice_find_vsi_list_entry(struct ice_hw *hw, u8 recp_id, u16 vsi_handle, list_head = &sw->recp_list[recp_id].filt_rules; list_for_each_entry(list_itr, list_head, list_entry) { - if (list_itr->vsi_list_info) { + if (list_itr->vsi_count == 1 && list_itr->vsi_list_info) { map_info = list_itr->vsi_list_info; if (test_bit(vsi_handle, map_info->vsi_map)) { *vsi_list_id = map_info->vsi_list_id; diff --git a/drivers/net/ethernet/intel/ice/ice_txrx.c b/drivers/net/ethernet/intel/ice/ice_txrx.c index c9bc3f1add5d..8208055d6e7f 100644 --- a/drivers/net/ethernet/intel/ice/ice_txrx.c +++ b/drivers/net/ethernet/intel/ice/ice_txrx.c @@ -2368,7 +2368,7 @@ ice_xmit_frame_ring(struct sk_buff *skb, struct ice_tx_ring *tx_ring) ICE_TXD_CTX_QW1_CMD_S); ice_tstamp(tx_ring, skb, first, &offload); - if (ice_is_switchdev_running(vsi->back)) + if (ice_is_switchdev_running(vsi->back) && vsi->type != ICE_VSI_SF) ice_eswitch_set_target_vsi(skb, &offload); if (offload.cd_qw1 & ICE_TX_DESC_DTYPE_CTX) { diff --git a/drivers/net/ethernet/intel/ice/ice_type.h b/drivers/net/ethernet/intel/ice/ice_type.h index 96037bef3e78..45768796691f 100644 --- a/drivers/net/ethernet/intel/ice/ice_type.h +++ b/drivers/net/ethernet/intel/ice/ice_type.h @@ -61,6 +61,7 @@ static inline u32 ice_round_to_num(u32 N, u32 R) ICE_DBG_AQ_DESC | \ ICE_DBG_AQ_DESC_BUF | \ ICE_DBG_AQ_CMD) +#define ICE_DBG_PARSER BIT_ULL(28) #define ICE_DBG_USER BIT_ULL(31) @@ -158,6 +159,7 @@ enum ice_vsi_type { ICE_VSI_CTRL = 3, /* equates to ICE_VSI_PF with 1 queue pair */ ICE_VSI_CHNL = 4, ICE_VSI_LB = 6, + ICE_VSI_SF = 9, }; struct ice_link_status { diff --git a/drivers/net/ethernet/intel/ice/ice_vf_lib.c b/drivers/net/ethernet/intel/ice/ice_vf_lib.c index 5635e9da2212..a69e91f88d81 100644 --- a/drivers/net/ethernet/intel/ice/ice_vf_lib.c +++ b/drivers/net/ethernet/intel/ice/ice_vf_lib.c @@ -766,7 +766,7 @@ void ice_reset_all_vfs(struct ice_pf *pf) ice_for_each_vf(pf, bkt, vf) { mutex_lock(&vf->cfg_lock); - ice_eswitch_detach(pf, vf); + ice_eswitch_detach_vf(pf, vf); vf->driver_caps = 0; ice_vc_set_default_allowlist(vf); @@ -782,7 +782,7 @@ void ice_reset_all_vfs(struct ice_pf *pf) ice_vf_rebuild_vsi(vf); ice_vf_post_vsi_rebuild(vf); - ice_eswitch_attach(pf, vf); + ice_eswitch_attach_vf(pf, vf); mutex_unlock(&vf->cfg_lock); } diff --git a/drivers/net/ethernet/intel/ice/ice_vf_lib.h b/drivers/net/ethernet/intel/ice/ice_vf_lib.h index fec16919ec19..be4266899690 100644 --- a/drivers/net/ethernet/intel/ice/ice_vf_lib.h +++ b/drivers/net/ethernet/intel/ice/ice_vf_lib.h @@ -12,6 +12,7 @@ #include <net/devlink.h> #include <linux/avf/virtchnl.h> #include "ice_type.h" +#include "ice_flow.h" #include "ice_virtchnl_fdir.h" #include "ice_vsi_vlan_ops.h" @@ -52,6 +53,12 @@ struct ice_mdd_vf_events { u16 last_printed; }; +/* Structure to store fdir fv entry */ +struct ice_fdir_prof_info { + struct ice_parser_profile prof; + u64 fdir_active_cnt; +}; + /* VF operations */ struct ice_vf_ops { enum ice_disq_rst_src reset_type; @@ -91,6 +98,7 @@ struct ice_vf { u16 lan_vsi_idx; /* index into PF struct */ u16 ctrl_vsi_idx; struct ice_vf_fdir fdir; + struct ice_fdir_prof_info fdir_prof_info[ICE_MAX_PTGS]; /* first vector index of this VF in the PF space */ int first_vector_idx; struct ice_sw *vf_sw_id; /* switch ID the VF VSIs connect to */ diff --git a/drivers/net/ethernet/intel/ice/ice_virtchnl.c b/drivers/net/ethernet/intel/ice/ice_virtchnl.c index 1c6ce0c4ed4e..59f62306b9cb 100644 --- a/drivers/net/ethernet/intel/ice/ice_virtchnl.c +++ b/drivers/net/ethernet/intel/ice/ice_virtchnl.c @@ -461,6 +461,10 @@ static int ice_vc_get_vf_res_msg(struct ice_vf *vf, u8 *msg) if (vf->driver_caps & VIRTCHNL_VF_OFFLOAD_FDIR_PF) vfres->vf_cap_flags |= VIRTCHNL_VF_OFFLOAD_FDIR_PF; + if (vf->driver_caps & VIRTCHNL_VF_OFFLOAD_TC_U32 && + vfres->vf_cap_flags & VIRTCHNL_VF_OFFLOAD_FDIR_PF) + vfres->vf_cap_flags |= VIRTCHNL_VF_OFFLOAD_TC_U32; + if (vf->driver_caps & VIRTCHNL_VF_OFFLOAD_RSS_PCTYPE_V2) vfres->vf_cap_flags |= VIRTCHNL_VF_OFFLOAD_RSS_PCTYPE_V2; diff --git a/drivers/net/ethernet/intel/ice/ice_virtchnl_fdir.c b/drivers/net/ethernet/intel/ice/ice_virtchnl_fdir.c index b4feb0927687..14e3f0f89c78 100644 --- a/drivers/net/ethernet/intel/ice/ice_virtchnl_fdir.c +++ b/drivers/net/ethernet/intel/ice/ice_virtchnl_fdir.c @@ -26,6 +26,15 @@ enum ice_fdir_tunnel_type { ICE_FDIR_TUNNEL_TYPE_NONE = 0, ICE_FDIR_TUNNEL_TYPE_GTPU, ICE_FDIR_TUNNEL_TYPE_GTPU_EH, + ICE_FDIR_TUNNEL_TYPE_ECPRI, + ICE_FDIR_TUNNEL_TYPE_GTPU_INNER, + ICE_FDIR_TUNNEL_TYPE_GTPU_EH_INNER, + ICE_FDIR_TUNNEL_TYPE_GRE, + ICE_FDIR_TUNNEL_TYPE_GTPOGRE, + ICE_FDIR_TUNNEL_TYPE_GTPOGRE_INNER, + ICE_FDIR_TUNNEL_TYPE_GRE_INNER, + ICE_FDIR_TUNNEL_TYPE_L2TPV2, + ICE_FDIR_TUNNEL_TYPE_L2TPV2_INNER, }; struct virtchnl_fdir_fltr_conf { @@ -33,6 +42,11 @@ struct virtchnl_fdir_fltr_conf { enum ice_fdir_tunnel_type ttype; u64 inset_flag; u32 flow_id; + + struct ice_parser_profile *prof; + bool parser_ena; + u8 *pkt_buf; + u8 pkt_len; }; struct virtchnl_fdir_inset_map { @@ -787,6 +801,107 @@ err_exit: } /** + * ice_vc_fdir_is_raw_flow - check if FDIR flow is raw (binary) + * @proto: virtchnl protocol headers + * + * Check if the FDIR rule is raw flow (protocol agnostic flow) or not. Note + * that common FDIR rule must have non-zero proto->count. Thus, we choose the + * tunnel_level and count of proto as the indicators. If both tunnel_level and + * count of proto are zero, this FDIR rule will be regarded as raw flow. + * + * Returns: true if headers describe raw flow, false otherwise. + */ +static bool +ice_vc_fdir_is_raw_flow(struct virtchnl_proto_hdrs *proto) +{ + return (proto->tunnel_level == 0 && proto->count == 0); +} + +/** + * ice_vc_fdir_parse_raw - parse a virtchnl raw FDIR rule + * @vf: pointer to the VF info + * @proto: virtchnl protocol headers + * @conf: FDIR configuration for each filter + * + * Parse the virtual channel filter's raw flow and store it in @conf + * + * Return: 0 on success or negative errno on failure. + */ +static int +ice_vc_fdir_parse_raw(struct ice_vf *vf, + struct virtchnl_proto_hdrs *proto, + struct virtchnl_fdir_fltr_conf *conf) +{ + u8 *pkt_buf, *msk_buf __free(kfree); + struct ice_parser_result rslt; + struct ice_pf *pf = vf->pf; + struct ice_parser *psr; + int status = -ENOMEM; + struct ice_hw *hw; + u16 udp_port = 0; + + pkt_buf = kzalloc(proto->raw.pkt_len, GFP_KERNEL); + msk_buf = kzalloc(proto->raw.pkt_len, GFP_KERNEL); + if (!pkt_buf || !msk_buf) + goto err_mem_alloc; + + memcpy(pkt_buf, proto->raw.spec, proto->raw.pkt_len); + memcpy(msk_buf, proto->raw.mask, proto->raw.pkt_len); + + hw = &pf->hw; + + /* Get raw profile info via Parser Lib */ + psr = ice_parser_create(hw); + if (IS_ERR(psr)) { + status = PTR_ERR(psr); + goto err_mem_alloc; + } + + ice_parser_dvm_set(psr, ice_is_dvm_ena(hw)); + + if (ice_get_open_tunnel_port(hw, &udp_port, TNL_VXLAN)) + ice_parser_vxlan_tunnel_set(psr, udp_port, true); + + status = ice_parser_run(psr, pkt_buf, proto->raw.pkt_len, &rslt); + if (status) + goto err_parser_destroy; + + if (hw->debug_mask & ICE_DBG_PARSER) + ice_parser_result_dump(hw, &rslt); + + conf->prof = kzalloc(sizeof(*conf->prof), GFP_KERNEL); + if (!conf->prof) { + status = -ENOMEM; + goto err_parser_destroy; + } + + status = ice_parser_profile_init(&rslt, pkt_buf, msk_buf, + proto->raw.pkt_len, ICE_BLK_FD, + conf->prof); + if (status) + goto err_parser_profile_init; + + if (hw->debug_mask & ICE_DBG_PARSER) + ice_parser_profile_dump(hw, conf->prof); + + /* Store raw flow info into @conf */ + conf->pkt_len = proto->raw.pkt_len; + conf->pkt_buf = pkt_buf; + conf->parser_ena = true; + + ice_parser_destroy(psr); + return 0; + +err_parser_profile_init: + kfree(conf->prof); +err_parser_destroy: + ice_parser_destroy(psr); +err_mem_alloc: + kfree(pkt_buf); + return status; +} + +/** * ice_vc_fdir_parse_pattern * @vf: pointer to the VF info * @fltr: virtual channel add cmd buffer @@ -813,6 +928,10 @@ ice_vc_fdir_parse_pattern(struct ice_vf *vf, struct virtchnl_fdir_add *fltr, return -EINVAL; } + /* For raw FDIR filters created by the parser */ + if (ice_vc_fdir_is_raw_flow(proto)) + return ice_vc_fdir_parse_raw(vf, proto, conf); + for (i = 0; i < proto->count; i++) { struct virtchnl_proto_hdr *hdr = &proto->proto_hdr[i]; struct ip_esp_hdr *esph; @@ -1101,8 +1220,10 @@ ice_vc_validate_fdir_fltr(struct ice_vf *vf, struct virtchnl_fdir_add *fltr, struct virtchnl_proto_hdrs *proto = &fltr->rule_cfg.proto_hdrs; int ret; - if (!ice_vc_validate_pattern(vf, proto)) - return -EINVAL; + /* For raw FDIR filters created by the parser */ + if (!ice_vc_fdir_is_raw_flow(proto)) + if (!ice_vc_validate_pattern(vf, proto)) + return -EINVAL; ret = ice_vc_fdir_parse_pattern(vf, fltr, conf); if (ret) @@ -1295,11 +1416,15 @@ static int ice_vc_fdir_write_fltr(struct ice_vf *vf, return -ENOMEM; ice_fdir_get_prgm_desc(hw, input, &desc, add); - ret = ice_fdir_get_gen_prgm_pkt(hw, input, pkt, false, is_tun); - if (ret) { - dev_dbg(dev, "Gen training pkt for VF %d ptype %d failed\n", - vf->vf_id, input->flow_type); - goto err_free_pkt; + if (conf->parser_ena) { + memcpy(pkt, conf->pkt_buf, conf->pkt_len); + } else { + ret = ice_fdir_get_gen_prgm_pkt(hw, input, pkt, false, is_tun); + if (ret) { + dev_dbg(dev, "Gen training pkt for VF %d ptype %d failed\n", + vf->vf_id, input->flow_type); + goto err_free_pkt; + } } ret = ice_prgm_fdir_fltr(ctrl_vsi, &desc, pkt); @@ -1521,6 +1646,16 @@ err_exit: return ret; } +static int ice_fdir_is_tunnel(enum ice_fdir_tunnel_type ttype) +{ + return (ttype == ICE_FDIR_TUNNEL_TYPE_GRE_INNER || + ttype == ICE_FDIR_TUNNEL_TYPE_GTPU_INNER || + ttype == ICE_FDIR_TUNNEL_TYPE_GTPU_EH_INNER || + ttype == ICE_FDIR_TUNNEL_TYPE_GTPOGRE_INNER || + ttype == ICE_FDIR_TUNNEL_TYPE_ECPRI || + ttype == ICE_FDIR_TUNNEL_TYPE_L2TPV2_INNER); +} + /** * ice_vc_add_fdir_fltr_post * @vf: pointer to the VF structure @@ -1782,6 +1917,158 @@ static void ice_vc_fdir_clear_irq_ctx(struct ice_vf *vf) } /** + * ice_vc_parser_fv_check_diff - check two parsed FDIR profile fv context + * @fv_a: struct of parsed FDIR profile field vector + * @fv_b: struct of parsed FDIR profile field vector + * + * Check if the two parsed FDIR profile field vector context are different, + * including proto_id, offset and mask. + * + * Return: true on different, false on otherwise. + */ +static bool ice_vc_parser_fv_check_diff(struct ice_parser_fv *fv_a, + struct ice_parser_fv *fv_b) +{ + return (fv_a->proto_id != fv_b->proto_id || + fv_a->offset != fv_b->offset || + fv_a->msk != fv_b->msk); +} + +/** + * ice_vc_parser_fv_save - save parsed FDIR profile fv context + * @fv: struct of parsed FDIR profile field vector + * @fv_src: parsed FDIR profile field vector context to save + * + * Save the parsed FDIR profile field vector context, including proto_id, + * offset and mask. + * + * Return: Void. + */ +static void ice_vc_parser_fv_save(struct ice_parser_fv *fv, + struct ice_parser_fv *fv_src) +{ + fv->proto_id = fv_src->proto_id; + fv->offset = fv_src->offset; + fv->msk = fv_src->msk; + fv->spec = 0; +} + +/** + * ice_vc_add_fdir_raw - add a raw FDIR filter for VF + * @vf: pointer to the VF info + * @conf: FDIR configuration for each filter + * @v_ret: the final VIRTCHNL code + * @stat: pointer to the VIRTCHNL_OP_ADD_FDIR_FILTER + * @len: length of the stat + * + * Return: 0 on success or negative errno on failure. + */ +static int +ice_vc_add_fdir_raw(struct ice_vf *vf, + struct virtchnl_fdir_fltr_conf *conf, + enum virtchnl_status_code *v_ret, + struct virtchnl_fdir_add *stat, int len) +{ + struct ice_vsi *vf_vsi, *ctrl_vsi; + struct ice_fdir_prof_info *pi; + struct ice_pf *pf = vf->pf; + int ret, ptg, id, i; + struct device *dev; + struct ice_hw *hw; + bool fv_found; + + dev = ice_pf_to_dev(pf); + hw = &pf->hw; + *v_ret = VIRTCHNL_STATUS_ERR_PARAM; + stat->status = VIRTCHNL_FDIR_FAILURE_RULE_NORESOURCE; + + id = find_first_bit(conf->prof->ptypes, ICE_FLOW_PTYPE_MAX); + ptg = hw->blk[ICE_BLK_FD].xlt1.t[id]; + + vf_vsi = ice_get_vf_vsi(vf); + if (!vf_vsi) { + dev_err(dev, "Can not get FDIR vf_vsi for VF %d\n", vf->vf_id); + return -ENODEV; + } + + ctrl_vsi = pf->vsi[vf->ctrl_vsi_idx]; + if (!ctrl_vsi) { + dev_err(dev, "Can not get FDIR ctrl_vsi for VF %d\n", + vf->vf_id); + return -ENODEV; + } + + fv_found = false; + + /* Check if profile info already exists, then update the counter */ + pi = &vf->fdir_prof_info[ptg]; + if (pi->fdir_active_cnt != 0) { + for (i = 0; i < ICE_MAX_FV_WORDS; i++) + if (ice_vc_parser_fv_check_diff(&pi->prof.fv[i], + &conf->prof->fv[i])) + break; + if (i == ICE_MAX_FV_WORDS) { + fv_found = true; + pi->fdir_active_cnt++; + } + } + + /* HW profile setting is only required for the first time */ + if (!fv_found) { + ret = ice_flow_set_parser_prof(hw, vf_vsi->idx, + ctrl_vsi->idx, conf->prof, + ICE_BLK_FD); + + if (ret) { + *v_ret = VIRTCHNL_STATUS_ERR_NO_MEMORY; + dev_dbg(dev, "VF %d: insert hw prof failed\n", + vf->vf_id); + return ret; + } + } + + ret = ice_vc_fdir_insert_entry(vf, conf, &conf->flow_id); + if (ret) { + *v_ret = VIRTCHNL_STATUS_ERR_NO_MEMORY; + dev_dbg(dev, "VF %d: insert FDIR list failed\n", + vf->vf_id); + return ret; + } + + ret = ice_vc_fdir_set_irq_ctx(vf, conf, + VIRTCHNL_OP_ADD_FDIR_FILTER); + if (ret) { + dev_dbg(dev, "VF %d: set FDIR context failed\n", + vf->vf_id); + goto err_rem_entry; + } + + ret = ice_vc_fdir_write_fltr(vf, conf, true, false); + if (ret) { + dev_err(dev, "VF %d: adding FDIR raw flow rule failed, ret:%d\n", + vf->vf_id, ret); + goto err_clr_irq; + } + + /* Save parsed profile fv info of the FDIR rule for the first time */ + if (!fv_found) { + for (i = 0; i < conf->prof->fv_num; i++) + ice_vc_parser_fv_save(&pi->prof.fv[i], + &conf->prof->fv[i]); + pi->prof.fv_num = conf->prof->fv_num; + pi->fdir_active_cnt = 1; + } + + return 0; + +err_clr_irq: + ice_vc_fdir_clear_irq_ctx(vf); +err_rem_entry: + ice_vc_fdir_remove_entry(vf, conf, conf->flow_id); + return ret; +} + +/** * ice_vc_add_fdir_fltr - add a FDIR filter for VF by the msg buffer * @vf: pointer to the VF info * @msg: pointer to the msg buffer @@ -1846,7 +2133,7 @@ int ice_vc_add_fdir_fltr(struct ice_vf *vf, u8 *msg) len = sizeof(*stat); ret = ice_vc_validate_fdir_fltr(vf, fltr, conf); if (ret) { - v_ret = VIRTCHNL_STATUS_SUCCESS; + v_ret = VIRTCHNL_STATUS_ERR_PARAM; stat->status = VIRTCHNL_FDIR_FAILURE_RULE_INVALID; dev_dbg(dev, "Invalid FDIR filter from VF %d\n", vf->vf_id); goto err_free_conf; @@ -1861,6 +2148,15 @@ int ice_vc_add_fdir_fltr(struct ice_vf *vf, u8 *msg) goto exit; } + /* For raw FDIR filters created by the parser */ + if (conf->parser_ena) { + ret = ice_vc_add_fdir_raw(vf, conf, &v_ret, stat, len); + if (ret) + goto err_free_conf; + goto exit; + } + + is_tun = ice_fdir_is_tunnel(conf->ttype); ret = ice_vc_fdir_config_input_set(vf, fltr, conf, is_tun); if (ret) { v_ret = VIRTCHNL_STATUS_SUCCESS; @@ -1922,6 +2218,78 @@ err_exit: } /** + * ice_vc_del_fdir_raw - delete a raw FDIR filter for VF + * @vf: pointer to the VF info + * @conf: FDIR configuration for each filter + * @v_ret: the final VIRTCHNL code + * @stat: pointer to the VIRTCHNL_OP_DEL_FDIR_FILTER + * @len: length of the stat + * + * Return: 0 on success or negative errno on failure. + */ +static int +ice_vc_del_fdir_raw(struct ice_vf *vf, + struct virtchnl_fdir_fltr_conf *conf, + enum virtchnl_status_code *v_ret, + struct virtchnl_fdir_del *stat, int len) +{ + struct ice_vsi *vf_vsi, *ctrl_vsi; + enum ice_block blk = ICE_BLK_FD; + struct ice_fdir_prof_info *pi; + struct ice_pf *pf = vf->pf; + struct device *dev; + struct ice_hw *hw; + unsigned long id; + u16 vsi_num; + int ptg; + int ret; + + dev = ice_pf_to_dev(pf); + hw = &pf->hw; + *v_ret = VIRTCHNL_STATUS_ERR_PARAM; + stat->status = VIRTCHNL_FDIR_FAILURE_RULE_NORESOURCE; + + id = find_first_bit(conf->prof->ptypes, ICE_FLOW_PTYPE_MAX); + ptg = hw->blk[ICE_BLK_FD].xlt1.t[id]; + + ret = ice_vc_fdir_write_fltr(vf, conf, false, false); + if (ret) { + dev_err(dev, "VF %u: deleting FDIR raw flow rule failed: %d\n", + vf->vf_id, ret); + return ret; + } + + vf_vsi = ice_get_vf_vsi(vf); + if (!vf_vsi) { + dev_err(dev, "Can not get FDIR vf_vsi for VF %u\n", vf->vf_id); + return -ENODEV; + } + + ctrl_vsi = pf->vsi[vf->ctrl_vsi_idx]; + if (!ctrl_vsi) { + dev_err(dev, "Can not get FDIR ctrl_vsi for VF %u\n", + vf->vf_id); + return -ENODEV; + } + + pi = &vf->fdir_prof_info[ptg]; + if (pi->fdir_active_cnt != 0) { + pi->fdir_active_cnt--; + /* Remove the profile id flow if no active FDIR rule left */ + if (!pi->fdir_active_cnt) { + vsi_num = ice_get_hw_vsi_num(hw, ctrl_vsi->idx); + ice_rem_prof_id_flow(hw, blk, vsi_num, id); + + vsi_num = ice_get_hw_vsi_num(hw, vf_vsi->idx); + ice_rem_prof_id_flow(hw, blk, vsi_num, id); + } + } + + conf->parser_ena = false; + return 0; +} + +/** * ice_vc_del_fdir_fltr - delete a FDIR filter for VF by the msg buffer * @vf: pointer to the VF info * @msg: pointer to the msg buffer @@ -1933,7 +2301,10 @@ int ice_vc_del_fdir_fltr(struct ice_vf *vf, u8 *msg) struct virtchnl_fdir_del *fltr = (struct virtchnl_fdir_del *)msg; struct virtchnl_fdir_del *stat = NULL; struct virtchnl_fdir_fltr_conf *conf; + struct ice_vf_fdir *fdir = &vf->fdir; enum virtchnl_status_code v_ret; + struct ice_fdir_fltr *input; + enum ice_fltr_ptype flow; struct device *dev; struct ice_pf *pf; int is_tun = 0; @@ -1983,6 +2354,15 @@ int ice_vc_del_fdir_fltr(struct ice_vf *vf, u8 *msg) goto err_exit; } + /* For raw FDIR filters created by the parser */ + if (conf->parser_ena) { + ret = ice_vc_del_fdir_raw(vf, conf, &v_ret, stat, len); + if (ret) + goto err_del_tmr; + goto exit; + } + + is_tun = ice_fdir_is_tunnel(conf->ttype); ret = ice_vc_fdir_write_fltr(vf, conf, false, is_tun); if (ret) { v_ret = VIRTCHNL_STATUS_SUCCESS; @@ -1992,6 +2372,13 @@ int ice_vc_del_fdir_fltr(struct ice_vf *vf, u8 *msg) goto err_del_tmr; } + /* Remove unused profiles to avoid unexpected behaviors */ + input = &conf->input; + flow = input->flow_type; + if (fdir->fdir_fltr_cnt[flow][is_tun] == 1) + ice_vc_fdir_rem_prof(vf, flow, is_tun); + +exit: kfree(stat); return ret; diff --git a/drivers/net/ethernet/intel/ice/ice_vsi_vlan_ops.c b/drivers/net/ethernet/intel/ice/ice_vsi_vlan_ops.c index 7aae7fdcfcdb..8c7a9b41fb63 100644 --- a/drivers/net/ethernet/intel/ice/ice_vsi_vlan_ops.c +++ b/drivers/net/ethernet/intel/ice/ice_vsi_vlan_ops.c @@ -3,6 +3,7 @@ #include "ice_pf_vsi_vlan_ops.h" #include "ice_vf_vsi_vlan_ops.h" +#include "ice_sf_vsi_vlan_ops.h" #include "ice_lib.h" #include "ice.h" @@ -77,6 +78,9 @@ void ice_vsi_init_vlan_ops(struct ice_vsi *vsi) case ICE_VSI_VF: ice_vf_vsi_init_vlan_ops(vsi); break; + case ICE_VSI_SF: + ice_sf_vsi_init_vlan_ops(vsi); + break; default: dev_dbg(ice_pf_to_dev(vsi->back), "%s does not support VLAN operations\n", ice_vsi_type_str(vsi->type)); diff --git a/drivers/net/ethernet/intel/ice/ice_xsk.c b/drivers/net/ethernet/intel/ice/ice_xsk.c index 240a7bec242b..334ae945d640 100644 --- a/drivers/net/ethernet/intel/ice/ice_xsk.c +++ b/drivers/net/ethernet/intel/ice/ice_xsk.c @@ -39,7 +39,7 @@ static void ice_qp_reset_stats(struct ice_vsi *vsi, u16 q_idx) sizeof(vsi_stat->rx_ring_stats[q_idx]->rx_stats)); memset(&vsi_stat->tx_ring_stats[q_idx]->stats, 0, sizeof(vsi_stat->tx_ring_stats[q_idx]->stats)); - if (ice_is_xdp_ena_vsi(vsi)) + if (vsi->xdp_rings) memset(&vsi->xdp_rings[q_idx]->ring_stats->stats, 0, sizeof(vsi->xdp_rings[q_idx]->ring_stats->stats)); } @@ -52,7 +52,7 @@ static void ice_qp_reset_stats(struct ice_vsi *vsi, u16 q_idx) static void ice_qp_clean_rings(struct ice_vsi *vsi, u16 q_idx) { ice_clean_tx_ring(vsi->tx_rings[q_idx]); - if (ice_is_xdp_ena_vsi(vsi)) + if (vsi->xdp_rings) ice_clean_tx_ring(vsi->xdp_rings[q_idx]); ice_clean_rx_ring(vsi->rx_rings[q_idx]); } @@ -165,7 +165,6 @@ static int ice_qp_dis(struct ice_vsi *vsi, u16 q_idx) struct ice_q_vector *q_vector; struct ice_tx_ring *tx_ring; struct ice_rx_ring *rx_ring; - int timeout = 50; int fail = 0; int err; @@ -176,13 +175,6 @@ static int ice_qp_dis(struct ice_vsi *vsi, u16 q_idx) rx_ring = vsi->rx_rings[q_idx]; q_vector = rx_ring->q_vector; - while (test_and_set_bit(ICE_CFG_BUSY, vsi->state)) { - timeout--; - if (!timeout) - return -EBUSY; - usleep_range(1000, 2000); - } - synchronize_net(); netif_carrier_off(vsi->netdev); netif_tx_stop_queue(netdev_get_tx_queue(vsi->netdev, q_idx)); @@ -194,7 +186,7 @@ static int ice_qp_dis(struct ice_vsi *vsi, u16 q_idx) err = ice_vsi_stop_tx_ring(vsi, ICE_NO_RESET, 0, tx_ring, &txq_meta); if (!fail) fail = err; - if (ice_is_xdp_ena_vsi(vsi)) { + if (vsi->xdp_rings) { struct ice_tx_ring *xdp_ring = vsi->xdp_rings[q_idx]; memset(&txq_meta, 0, sizeof(txq_meta)); @@ -261,7 +253,6 @@ static int ice_qp_ena(struct ice_vsi *vsi, u16 q_idx) netif_tx_start_queue(netdev_get_tx_queue(vsi->netdev, q_idx)); netif_carrier_on(vsi->netdev); } - clear_bit(ICE_CFG_BUSY, vsi->state); return fail; } @@ -298,7 +289,7 @@ ice_xsk_pool_enable(struct ice_vsi *vsi, struct xsk_buff_pool *pool, u16 qid) { int err; - if (vsi->type != ICE_VSI_PF) + if (vsi->type != ICE_VSI_PF && vsi->type != ICE_VSI_SF) return -EINVAL; if (qid >= vsi->netdev->real_num_rx_queues || @@ -390,7 +381,8 @@ int ice_xsk_pool_setup(struct ice_vsi *vsi, struct xsk_buff_pool *pool, u16 qid) goto failure; } - if_running = netif_running(vsi->netdev) && ice_is_xdp_ena_vsi(vsi); + if_running = !test_bit(ICE_VSI_DOWN, vsi->state) && + ice_is_xdp_ena_vsi(vsi); if (if_running) { struct ice_rx_ring *rx_ring = vsi->rx_rings[qid]; diff --git a/drivers/net/ethernet/intel/idpf/idpf_dev.c b/drivers/net/ethernet/intel/idpf/idpf_dev.c index 3df9935685e9..6c913a703df6 100644 --- a/drivers/net/ethernet/intel/idpf/idpf_dev.c +++ b/drivers/net/ethernet/intel/idpf/idpf_dev.c @@ -97,8 +97,10 @@ static int idpf_intr_reg_init(struct idpf_vport *vport) intr->dyn_ctl = idpf_get_reg_addr(adapter, reg_vals[vec_id].dyn_ctl_reg); intr->dyn_ctl_intena_m = PF_GLINT_DYN_CTL_INTENA_M; + intr->dyn_ctl_intena_msk_m = PF_GLINT_DYN_CTL_INTENA_MSK_M; intr->dyn_ctl_itridx_s = PF_GLINT_DYN_CTL_ITR_INDX_S; intr->dyn_ctl_intrvl_s = PF_GLINT_DYN_CTL_INTERVAL_S; + intr->dyn_ctl_wb_on_itr_m = PF_GLINT_DYN_CTL_WB_ON_ITR_M; spacing = IDPF_ITR_IDX_SPACING(reg_vals[vec_id].itrn_index_spacing, IDPF_PF_ITR_IDX_SPACING); diff --git a/drivers/net/ethernet/intel/idpf/idpf_lib.c b/drivers/net/ethernet/intel/idpf/idpf_lib.c index 0b6c8fd5bc90..4f20343e49a9 100644 --- a/drivers/net/ethernet/intel/idpf/idpf_lib.c +++ b/drivers/net/ethernet/intel/idpf/idpf_lib.c @@ -357,24 +357,11 @@ int idpf_intr_req(struct idpf_adapter *adapter) goto free_msix; } - if (adapter->req_vec_chunks) { - struct virtchnl2_vector_chunks *vchunks; - struct virtchnl2_alloc_vectors *ac; - - ac = adapter->req_vec_chunks; - vchunks = &ac->vchunks; - - num_vec_ids = idpf_get_vec_ids(adapter, vecids, total_vecs, - vchunks); - if (num_vec_ids < v_actual) { - err = -EINVAL; - goto free_vecids; - } - } else { - int i; - - for (i = 0; i < v_actual; i++) - vecids[i] = i; + num_vec_ids = idpf_get_vec_ids(adapter, vecids, total_vecs, + &adapter->req_vec_chunks->vchunks); + if (num_vec_ids < v_actual) { + err = -EINVAL; + goto free_vecids; } for (vector = 0; vector < v_actual; vector++) { diff --git a/drivers/net/ethernet/intel/idpf/idpf_singleq_txrx.c b/drivers/net/ethernet/intel/idpf/idpf_singleq_txrx.c index fe64febf7436..dfd7cf1d9aa0 100644 --- a/drivers/net/ethernet/intel/idpf/idpf_singleq_txrx.c +++ b/drivers/net/ethernet/intel/idpf/idpf_singleq_txrx.c @@ -2,6 +2,7 @@ /* Copyright (C) 2023 Intel Corporation */ #include <net/libeth/rx.h> +#include <net/libeth/tx.h> #include "idpf.h" @@ -224,6 +225,7 @@ static void idpf_tx_singleq_map(struct idpf_tx_queue *tx_q, /* record length, and DMA address */ dma_unmap_len_set(tx_buf, len, size); dma_unmap_addr_set(tx_buf, dma, dma); + tx_buf->type = LIBETH_SQE_FRAG; /* align size to end of page */ max_data += -dma & (IDPF_TX_MAX_READ_REQ_SIZE - 1); @@ -237,14 +239,17 @@ static void idpf_tx_singleq_map(struct idpf_tx_queue *tx_q, offsets, max_data, td_tag); - tx_desc++; - i++; - - if (i == tx_q->desc_count) { + if (unlikely(++i == tx_q->desc_count)) { + tx_buf = &tx_q->tx_buf[0]; tx_desc = &tx_q->base_tx[0]; i = 0; + } else { + tx_buf++; + tx_desc++; } + tx_buf->type = LIBETH_SQE_EMPTY; + dma += max_data; size -= max_data; @@ -257,12 +262,14 @@ static void idpf_tx_singleq_map(struct idpf_tx_queue *tx_q, tx_desc->qw1 = idpf_tx_singleq_build_ctob(td_cmd, offsets, size, td_tag); - tx_desc++; - i++; - if (i == tx_q->desc_count) { + if (unlikely(++i == tx_q->desc_count)) { + tx_buf = &tx_q->tx_buf[0]; tx_desc = &tx_q->base_tx[0]; i = 0; + } else { + tx_buf++; + tx_desc++; } size = skb_frag_size(frag); @@ -270,8 +277,6 @@ static void idpf_tx_singleq_map(struct idpf_tx_queue *tx_q, dma = skb_frag_dma_map(tx_q->dev, frag, 0, size, DMA_TO_DEVICE); - - tx_buf = &tx_q->tx_buf[i]; } skb_tx_timestamp(first->skb); @@ -282,13 +287,13 @@ static void idpf_tx_singleq_map(struct idpf_tx_queue *tx_q, tx_desc->qw1 = idpf_tx_singleq_build_ctob(td_cmd, offsets, size, td_tag); - IDPF_SINGLEQ_BUMP_RING_IDX(tx_q, i); + first->type = LIBETH_SQE_SKB; + first->rs_idx = i; - /* set next_to_watch value indicating a packet is present */ - first->next_to_watch = tx_desc; + IDPF_SINGLEQ_BUMP_RING_IDX(tx_q, i); nq = netdev_get_tx_queue(tx_q->netdev, tx_q->idx); - netdev_tx_sent_queue(nq, first->bytecount); + netdev_tx_sent_queue(nq, first->bytes); idpf_tx_buf_hw_update(tx_q, i, netdev_xmit_more()); } @@ -306,8 +311,7 @@ idpf_tx_singleq_get_ctx_desc(struct idpf_tx_queue *txq) struct idpf_base_tx_ctx_desc *ctx_desc; int ntu = txq->next_to_use; - memset(&txq->tx_buf[ntu], 0, sizeof(struct idpf_tx_buf)); - txq->tx_buf[ntu].ctx_entry = true; + txq->tx_buf[ntu].type = LIBETH_SQE_CTX; ctx_desc = &txq->base_ctx[ntu]; @@ -371,6 +375,10 @@ netdev_tx_t idpf_tx_singleq_frame(struct sk_buff *skb, IDPF_TX_DESCS_FOR_CTX)) { idpf_tx_buf_hw_update(tx_q, tx_q->next_to_use, false); + u64_stats_update_begin(&tx_q->stats_sync); + u64_stats_inc(&tx_q->q_stats.q_busy); + u64_stats_update_end(&tx_q->stats_sync); + return NETDEV_TX_BUSY; } @@ -396,11 +404,11 @@ netdev_tx_t idpf_tx_singleq_frame(struct sk_buff *skb, first->skb = skb; if (tso) { - first->gso_segs = offload.tso_segs; - first->bytecount = skb->len + ((first->gso_segs - 1) * offload.tso_hdr_len); + first->packets = offload.tso_segs; + first->bytes = skb->len + ((first->packets - 1) * offload.tso_hdr_len); } else { - first->bytecount = max_t(unsigned int, skb->len, ETH_ZLEN); - first->gso_segs = 1; + first->bytes = max_t(unsigned int, skb->len, ETH_ZLEN); + first->packets = 1; } idpf_tx_singleq_map(tx_q, first, &offload); @@ -420,10 +428,15 @@ out_drop: static bool idpf_tx_singleq_clean(struct idpf_tx_queue *tx_q, int napi_budget, int *cleaned) { - unsigned int total_bytes = 0, total_pkts = 0; + struct libeth_sq_napi_stats ss = { }; struct idpf_base_tx_desc *tx_desc; u32 budget = tx_q->clean_budget; s16 ntc = tx_q->next_to_clean; + struct libeth_cq_pp cp = { + .dev = tx_q->dev, + .ss = &ss, + .napi = napi_budget, + }; struct idpf_netdev_priv *np; struct idpf_tx_buf *tx_buf; struct netdev_queue *nq; @@ -441,47 +454,26 @@ static bool idpf_tx_singleq_clean(struct idpf_tx_queue *tx_q, int napi_budget, * such. We can skip this descriptor since there is no buffer * to clean. */ - if (tx_buf->ctx_entry) { - /* Clear this flag here to avoid stale flag values when - * this buffer is used for actual data in the future. - * There are cases where the tx_buf struct / the flags - * field will not be cleared before being reused. - */ - tx_buf->ctx_entry = false; + if (unlikely(tx_buf->type <= LIBETH_SQE_CTX)) { + tx_buf->type = LIBETH_SQE_EMPTY; goto fetch_next_txq_desc; } - /* if next_to_watch is not set then no work pending */ - eop_desc = (struct idpf_base_tx_desc *)tx_buf->next_to_watch; - if (!eop_desc) + if (unlikely(tx_buf->type != LIBETH_SQE_SKB)) break; - /* prevent any other reads prior to eop_desc */ + /* prevent any other reads prior to type */ smp_rmb(); + eop_desc = &tx_q->base_tx[tx_buf->rs_idx]; + /* if the descriptor isn't done, no work yet to do */ if (!(eop_desc->qw1 & cpu_to_le64(IDPF_TX_DESC_DTYPE_DESC_DONE))) break; - /* clear next_to_watch to prevent false hangs */ - tx_buf->next_to_watch = NULL; - /* update the statistics for this packet */ - total_bytes += tx_buf->bytecount; - total_pkts += tx_buf->gso_segs; - - napi_consume_skb(tx_buf->skb, napi_budget); - - /* unmap skb header data */ - dma_unmap_single(tx_q->dev, - dma_unmap_addr(tx_buf, dma), - dma_unmap_len(tx_buf, len), - DMA_TO_DEVICE); - - /* clear tx_buf data */ - tx_buf->skb = NULL; - dma_unmap_len_set(tx_buf, len, 0); + libeth_tx_complete(tx_buf, &cp); /* unmap remaining buffers */ while (tx_desc != eop_desc) { @@ -495,13 +487,7 @@ static bool idpf_tx_singleq_clean(struct idpf_tx_queue *tx_q, int napi_budget, } /* unmap any remaining paged data */ - if (dma_unmap_len(tx_buf, len)) { - dma_unmap_page(tx_q->dev, - dma_unmap_addr(tx_buf, dma), - dma_unmap_len(tx_buf, len), - DMA_TO_DEVICE); - dma_unmap_len_set(tx_buf, len, 0); - } + libeth_tx_complete(tx_buf, &cp); } /* update budget only if we did something */ @@ -521,11 +507,11 @@ fetch_next_txq_desc: ntc += tx_q->desc_count; tx_q->next_to_clean = ntc; - *cleaned += total_pkts; + *cleaned += ss.packets; u64_stats_update_begin(&tx_q->stats_sync); - u64_stats_add(&tx_q->q_stats.packets, total_pkts); - u64_stats_add(&tx_q->q_stats.bytes, total_bytes); + u64_stats_add(&tx_q->q_stats.packets, ss.packets); + u64_stats_add(&tx_q->q_stats.bytes, ss.bytes); u64_stats_update_end(&tx_q->stats_sync); np = netdev_priv(tx_q->netdev); @@ -533,7 +519,7 @@ fetch_next_txq_desc: dont_wake = np->state != __IDPF_VPORT_UP || !netif_carrier_ok(tx_q->netdev); - __netif_txq_completed_wake(nq, total_pkts, total_bytes, + __netif_txq_completed_wake(nq, ss.packets, ss.bytes, IDPF_DESC_UNUSED(tx_q), IDPF_TX_WAKE_THRESH, dont_wake); @@ -1134,8 +1120,10 @@ int idpf_vport_singleq_napi_poll(struct napi_struct *napi, int budget) &work_done); /* If work not completed, return budget and polling will return */ - if (!clean_complete) + if (!clean_complete) { + idpf_vport_intr_set_wb_on_itr(q_vector); return budget; + } work_done = min_t(int, work_done, budget - 1); @@ -1144,6 +1132,8 @@ int idpf_vport_singleq_napi_poll(struct napi_struct *napi, int budget) */ if (likely(napi_complete_done(napi, work_done))) idpf_vport_intr_update_itr_ena_irq(q_vector); + else + idpf_vport_intr_set_wb_on_itr(q_vector); return work_done; } diff --git a/drivers/net/ethernet/intel/idpf/idpf_txrx.c b/drivers/net/ethernet/intel/idpf/idpf_txrx.c index 585c3dadd9bf..d4e6f0e10487 100644 --- a/drivers/net/ethernet/intel/idpf/idpf_txrx.c +++ b/drivers/net/ethernet/intel/idpf/idpf_txrx.c @@ -2,10 +2,19 @@ /* Copyright (C) 2023 Intel Corporation */ #include <net/libeth/rx.h> +#include <net/libeth/tx.h> #include "idpf.h" #include "idpf_virtchnl.h" +struct idpf_tx_stash { + struct hlist_node hlist; + struct libeth_sqe buf; +}; + +#define idpf_tx_buf_compl_tag(buf) (*(u32 *)&(buf)->priv) +LIBETH_SQE_CHECK_PRIV(u32); + static bool idpf_chk_linearize(struct sk_buff *skb, unsigned int max_bufs, unsigned int count); @@ -61,41 +70,20 @@ void idpf_tx_timeout(struct net_device *netdev, unsigned int txqueue) } /** - * idpf_tx_buf_rel - Release a Tx buffer - * @tx_q: the queue that owns the buffer - * @tx_buf: the buffer to free - */ -static void idpf_tx_buf_rel(struct idpf_tx_queue *tx_q, - struct idpf_tx_buf *tx_buf) -{ - if (tx_buf->skb) { - if (dma_unmap_len(tx_buf, len)) - dma_unmap_single(tx_q->dev, - dma_unmap_addr(tx_buf, dma), - dma_unmap_len(tx_buf, len), - DMA_TO_DEVICE); - dev_kfree_skb_any(tx_buf->skb); - } else if (dma_unmap_len(tx_buf, len)) { - dma_unmap_page(tx_q->dev, - dma_unmap_addr(tx_buf, dma), - dma_unmap_len(tx_buf, len), - DMA_TO_DEVICE); - } - - tx_buf->next_to_watch = NULL; - tx_buf->skb = NULL; - tx_buf->compl_tag = IDPF_SPLITQ_TX_INVAL_COMPL_TAG; - dma_unmap_len_set(tx_buf, len, 0); -} - -/** * idpf_tx_buf_rel_all - Free any empty Tx buffers * @txq: queue to be cleaned */ static void idpf_tx_buf_rel_all(struct idpf_tx_queue *txq) { + struct libeth_sq_napi_stats ss = { }; struct idpf_buf_lifo *buf_stack; - u16 i; + struct idpf_tx_stash *stash; + struct libeth_cq_pp cp = { + .dev = txq->dev, + .ss = &ss, + }; + struct hlist_node *tmp; + u32 i, tag; /* Buffers already cleared, nothing to do */ if (!txq->tx_buf) @@ -103,7 +91,7 @@ static void idpf_tx_buf_rel_all(struct idpf_tx_queue *txq) /* Free all the Tx buffer sk_buffs */ for (i = 0; i < txq->desc_count; i++) - idpf_tx_buf_rel(txq, &txq->tx_buf[i]); + libeth_tx_complete(&txq->tx_buf[i], &cp); kfree(txq->tx_buf); txq->tx_buf = NULL; @@ -115,6 +103,20 @@ static void idpf_tx_buf_rel_all(struct idpf_tx_queue *txq) if (!buf_stack->bufs) return; + /* + * If a Tx timeout occurred, there are potentially still bufs in the + * hash table, free them here. + */ + hash_for_each_safe(txq->stash->sched_buf_hash, tag, tmp, stash, + hlist) { + if (!stash) + continue; + + libeth_tx_complete(&stash->buf, &cp); + hash_del(&stash->hlist); + idpf_buf_lifo_push(buf_stack, stash); + } + for (i = 0; i < buf_stack->size; i++) kfree(buf_stack->bufs[i]); @@ -131,6 +133,7 @@ static void idpf_tx_buf_rel_all(struct idpf_tx_queue *txq) static void idpf_tx_desc_rel(struct idpf_tx_queue *txq) { idpf_tx_buf_rel_all(txq); + netdev_tx_reset_subqueue(txq->netdev, txq->idx); if (!txq->desc_ring) return; @@ -203,10 +206,6 @@ static int idpf_tx_buf_alloc_all(struct idpf_tx_queue *tx_q) if (!tx_q->tx_buf) return -ENOMEM; - /* Initialize tx_bufs with invalid completion tags */ - for (i = 0; i < tx_q->desc_count; i++) - tx_q->tx_buf[i].compl_tag = IDPF_SPLITQ_TX_INVAL_COMPL_TAG; - if (!idpf_queue_has(FLOW_SCH_EN, tx_q)) return 0; @@ -1656,37 +1655,6 @@ static void idpf_tx_handle_sw_marker(struct idpf_tx_queue *tx_q) } /** - * idpf_tx_splitq_clean_hdr - Clean TX buffer resources for header portion of - * packet - * @tx_q: tx queue to clean buffer from - * @tx_buf: buffer to be cleaned - * @cleaned: pointer to stats struct to track cleaned packets/bytes - * @napi_budget: Used to determine if we are in netpoll - */ -static void idpf_tx_splitq_clean_hdr(struct idpf_tx_queue *tx_q, - struct idpf_tx_buf *tx_buf, - struct idpf_cleaned_stats *cleaned, - int napi_budget) -{ - napi_consume_skb(tx_buf->skb, napi_budget); - - if (dma_unmap_len(tx_buf, len)) { - dma_unmap_single(tx_q->dev, - dma_unmap_addr(tx_buf, dma), - dma_unmap_len(tx_buf, len), - DMA_TO_DEVICE); - - dma_unmap_len_set(tx_buf, len, 0); - } - - /* clear tx_buf data */ - tx_buf->skb = NULL; - - cleaned->bytes += tx_buf->bytecount; - cleaned->packets += tx_buf->gso_segs; -} - -/** * idpf_tx_clean_stashed_bufs - clean bufs that were stored for * out of order completions * @txq: queue to clean @@ -1696,33 +1664,28 @@ static void idpf_tx_splitq_clean_hdr(struct idpf_tx_queue *tx_q, */ static void idpf_tx_clean_stashed_bufs(struct idpf_tx_queue *txq, u16 compl_tag, - struct idpf_cleaned_stats *cleaned, + struct libeth_sq_napi_stats *cleaned, int budget) { struct idpf_tx_stash *stash; struct hlist_node *tmp_buf; + struct libeth_cq_pp cp = { + .dev = txq->dev, + .ss = cleaned, + .napi = budget, + }; /* Buffer completion */ hash_for_each_possible_safe(txq->stash->sched_buf_hash, stash, tmp_buf, hlist, compl_tag) { - if (unlikely(stash->buf.compl_tag != (int)compl_tag)) + if (unlikely(idpf_tx_buf_compl_tag(&stash->buf) != compl_tag)) continue; - if (stash->buf.skb) { - idpf_tx_splitq_clean_hdr(txq, &stash->buf, cleaned, - budget); - } else if (dma_unmap_len(&stash->buf, len)) { - dma_unmap_page(txq->dev, - dma_unmap_addr(&stash->buf, dma), - dma_unmap_len(&stash->buf, len), - DMA_TO_DEVICE); - dma_unmap_len_set(&stash->buf, len, 0); - } + hash_del(&stash->hlist); + libeth_tx_complete(&stash->buf, &cp); /* Push shadow buf back onto stack */ idpf_buf_lifo_push(&txq->stash->buf_stack, stash); - - hash_del(&stash->hlist); } } @@ -1737,8 +1700,7 @@ static int idpf_stash_flow_sch_buffers(struct idpf_tx_queue *txq, { struct idpf_tx_stash *stash; - if (unlikely(!dma_unmap_addr(tx_buf, dma) && - !dma_unmap_len(tx_buf, len))) + if (unlikely(tx_buf->type <= LIBETH_SQE_CTX)) return 0; stash = idpf_buf_lifo_pop(&txq->stash->buf_stack); @@ -1751,29 +1713,27 @@ static int idpf_stash_flow_sch_buffers(struct idpf_tx_queue *txq, /* Store buffer params in shadow buffer */ stash->buf.skb = tx_buf->skb; - stash->buf.bytecount = tx_buf->bytecount; - stash->buf.gso_segs = tx_buf->gso_segs; + stash->buf.bytes = tx_buf->bytes; + stash->buf.packets = tx_buf->packets; + stash->buf.type = tx_buf->type; + stash->buf.nr_frags = tx_buf->nr_frags; dma_unmap_addr_set(&stash->buf, dma, dma_unmap_addr(tx_buf, dma)); dma_unmap_len_set(&stash->buf, len, dma_unmap_len(tx_buf, len)); - stash->buf.compl_tag = tx_buf->compl_tag; + idpf_tx_buf_compl_tag(&stash->buf) = idpf_tx_buf_compl_tag(tx_buf); /* Add buffer to buf_hash table to be freed later */ hash_add(txq->stash->sched_buf_hash, &stash->hlist, - stash->buf.compl_tag); - - memset(tx_buf, 0, sizeof(struct idpf_tx_buf)); + idpf_tx_buf_compl_tag(&stash->buf)); - /* Reinitialize buf_id portion of tag */ - tx_buf->compl_tag = IDPF_SPLITQ_TX_INVAL_COMPL_TAG; + tx_buf->type = LIBETH_SQE_EMPTY; return 0; } #define idpf_tx_splitq_clean_bump_ntc(txq, ntc, desc, buf) \ do { \ - (ntc)++; \ - if (unlikely(!(ntc))) { \ - ntc -= (txq)->desc_count; \ + if (unlikely(++(ntc) == (txq)->desc_count)) { \ + ntc = 0; \ buf = (txq)->tx_buf; \ desc = &(txq)->flex_tx[0]; \ } else { \ @@ -1797,69 +1757,71 @@ do { \ * Separate packet completion events will be reported on the completion queue, * and the buffers will be cleaned separately. The stats are not updated from * this function when using flow-based scheduling. + * + * Furthermore, in flow scheduling mode, check to make sure there are enough + * reserve buffers to stash the packet. If there are not, return early, which + * will leave next_to_clean pointing to the packet that failed to be stashed. + * + * Return: false in the scenario above, true otherwise. */ -static void idpf_tx_splitq_clean(struct idpf_tx_queue *tx_q, u16 end, +static bool idpf_tx_splitq_clean(struct idpf_tx_queue *tx_q, u16 end, int napi_budget, - struct idpf_cleaned_stats *cleaned, + struct libeth_sq_napi_stats *cleaned, bool descs_only) { union idpf_tx_flex_desc *next_pending_desc = NULL; union idpf_tx_flex_desc *tx_desc; - s16 ntc = tx_q->next_to_clean; + u32 ntc = tx_q->next_to_clean; + struct libeth_cq_pp cp = { + .dev = tx_q->dev, + .ss = cleaned, + .napi = napi_budget, + }; struct idpf_tx_buf *tx_buf; + bool clean_complete = true; tx_desc = &tx_q->flex_tx[ntc]; next_pending_desc = &tx_q->flex_tx[end]; tx_buf = &tx_q->tx_buf[ntc]; - ntc -= tx_q->desc_count; while (tx_desc != next_pending_desc) { - union idpf_tx_flex_desc *eop_desc; + u32 eop_idx; /* If this entry in the ring was used as a context descriptor, - * it's corresponding entry in the buffer ring will have an - * invalid completion tag since no buffer was used. We can - * skip this descriptor since there is no buffer to clean. + * it's corresponding entry in the buffer ring is reserved. We + * can skip this descriptor since there is no buffer to clean. */ - if (unlikely(tx_buf->compl_tag == IDPF_SPLITQ_TX_INVAL_COMPL_TAG)) + if (tx_buf->type <= LIBETH_SQE_CTX) goto fetch_next_txq_desc; - eop_desc = (union idpf_tx_flex_desc *)tx_buf->next_to_watch; + if (unlikely(tx_buf->type != LIBETH_SQE_SKB)) + break; - /* clear next_to_watch to prevent false hangs */ - tx_buf->next_to_watch = NULL; + eop_idx = tx_buf->rs_idx; if (descs_only) { - if (idpf_stash_flow_sch_buffers(tx_q, tx_buf)) + if (IDPF_TX_BUF_RSV_UNUSED(tx_q) < tx_buf->nr_frags) { + clean_complete = false; goto tx_splitq_clean_out; + } + + idpf_stash_flow_sch_buffers(tx_q, tx_buf); - while (tx_desc != eop_desc) { + while (ntc != eop_idx) { idpf_tx_splitq_clean_bump_ntc(tx_q, ntc, tx_desc, tx_buf); - - if (dma_unmap_len(tx_buf, len)) { - if (idpf_stash_flow_sch_buffers(tx_q, - tx_buf)) - goto tx_splitq_clean_out; - } + idpf_stash_flow_sch_buffers(tx_q, tx_buf); } } else { - idpf_tx_splitq_clean_hdr(tx_q, tx_buf, cleaned, - napi_budget); + libeth_tx_complete(tx_buf, &cp); /* unmap remaining buffers */ - while (tx_desc != eop_desc) { + while (ntc != eop_idx) { idpf_tx_splitq_clean_bump_ntc(tx_q, ntc, tx_desc, tx_buf); /* unmap any remaining paged data */ - if (dma_unmap_len(tx_buf, len)) { - dma_unmap_page(tx_q->dev, - dma_unmap_addr(tx_buf, dma), - dma_unmap_len(tx_buf, len), - DMA_TO_DEVICE); - dma_unmap_len_set(tx_buf, len, 0); - } + libeth_tx_complete(tx_buf, &cp); } } @@ -1868,8 +1830,9 @@ fetch_next_txq_desc: } tx_splitq_clean_out: - ntc += tx_q->desc_count; tx_q->next_to_clean = ntc; + + return clean_complete; } #define idpf_tx_clean_buf_ring_bump_ntc(txq, ntc, buf) \ @@ -1895,57 +1858,68 @@ do { \ * this completion tag. */ static bool idpf_tx_clean_buf_ring(struct idpf_tx_queue *txq, u16 compl_tag, - struct idpf_cleaned_stats *cleaned, + struct libeth_sq_napi_stats *cleaned, int budget) { u16 idx = compl_tag & txq->compl_tag_bufid_m; struct idpf_tx_buf *tx_buf = NULL; - u16 ntc = txq->next_to_clean; - u16 num_descs_cleaned = 0; - u16 orig_idx = idx; + struct libeth_cq_pp cp = { + .dev = txq->dev, + .ss = cleaned, + .napi = budget, + }; + u16 ntc, orig_idx = idx; tx_buf = &txq->tx_buf[idx]; - while (tx_buf->compl_tag == (int)compl_tag) { - if (tx_buf->skb) { - idpf_tx_splitq_clean_hdr(txq, tx_buf, cleaned, budget); - } else if (dma_unmap_len(tx_buf, len)) { - dma_unmap_page(txq->dev, - dma_unmap_addr(tx_buf, dma), - dma_unmap_len(tx_buf, len), - DMA_TO_DEVICE); - dma_unmap_len_set(tx_buf, len, 0); - } + if (unlikely(tx_buf->type <= LIBETH_SQE_CTX || + idpf_tx_buf_compl_tag(tx_buf) != compl_tag)) + return false; + + if (tx_buf->type == LIBETH_SQE_SKB) + libeth_tx_complete(tx_buf, &cp); - memset(tx_buf, 0, sizeof(struct idpf_tx_buf)); - tx_buf->compl_tag = IDPF_SPLITQ_TX_INVAL_COMPL_TAG; + idpf_tx_clean_buf_ring_bump_ntc(txq, idx, tx_buf); - num_descs_cleaned++; + while (idpf_tx_buf_compl_tag(tx_buf) == compl_tag) { + libeth_tx_complete(tx_buf, &cp); idpf_tx_clean_buf_ring_bump_ntc(txq, idx, tx_buf); } - /* If we didn't clean anything on the ring for this completion, there's - * nothing more to do. - */ - if (unlikely(!num_descs_cleaned)) - return false; - - /* Otherwise, if we did clean a packet on the ring directly, it's safe - * to assume that the descriptors starting from the original - * next_to_clean up until the previously cleaned packet can be reused. - * Therefore, we will go back in the ring and stash any buffers still - * in the ring into the hash table to be cleaned later. + /* + * It's possible the packet we just cleaned was an out of order + * completion, which means we can stash the buffers starting from + * the original next_to_clean and reuse the descriptors. We need + * to compare the descriptor ring next_to_clean packet's "first" buffer + * to the "first" buffer of the packet we just cleaned to determine if + * this is the case. Howevever, next_to_clean can point to either a + * reserved buffer that corresponds to a context descriptor used for the + * next_to_clean packet (TSO packet) or the "first" buffer (single + * packet). The orig_idx from the packet we just cleaned will always + * point to the "first" buffer. If next_to_clean points to a reserved + * buffer, let's bump ntc once and start the comparison from there. */ + ntc = txq->next_to_clean; tx_buf = &txq->tx_buf[ntc]; - while (tx_buf != &txq->tx_buf[orig_idx]) { - idpf_stash_flow_sch_buffers(txq, tx_buf); + + if (tx_buf->type == LIBETH_SQE_CTX) idpf_tx_clean_buf_ring_bump_ntc(txq, ntc, tx_buf); - } - /* Finally, update next_to_clean to reflect the work that was just done - * on the ring, if any. If the packet was only cleaned from the hash - * table, the ring will not be impacted, therefore we should not touch - * next_to_clean. The updated idx is used here + /* + * If ntc still points to a different "first" buffer, clean the + * descriptor ring and stash all of the buffers for later cleaning. If + * we cannot stash all of the buffers, next_to_clean will point to the + * "first" buffer of the packet that could not be stashed and cleaning + * will start there next time. + */ + if (unlikely(tx_buf != &txq->tx_buf[orig_idx] && + !idpf_tx_splitq_clean(txq, orig_idx, budget, cleaned, + true))) + return true; + + /* + * Otherwise, update next_to_clean to reflect the cleaning that was + * done above. */ txq->next_to_clean = idx; @@ -1965,7 +1939,7 @@ static bool idpf_tx_clean_buf_ring(struct idpf_tx_queue *txq, u16 compl_tag, */ static void idpf_tx_handle_rs_completion(struct idpf_tx_queue *txq, struct idpf_splitq_tx_compl_desc *desc, - struct idpf_cleaned_stats *cleaned, + struct libeth_sq_napi_stats *cleaned, int budget) { u16 compl_tag; @@ -1973,7 +1947,8 @@ static void idpf_tx_handle_rs_completion(struct idpf_tx_queue *txq, if (!idpf_queue_has(FLOW_SCH_EN, txq)) { u16 head = le16_to_cpu(desc->q_head_compl_tag.q_head); - return idpf_tx_splitq_clean(txq, head, budget, cleaned, false); + idpf_tx_splitq_clean(txq, head, budget, cleaned, false); + return; } compl_tag = le16_to_cpu(desc->q_head_compl_tag.compl_tag); @@ -2008,7 +1983,7 @@ static bool idpf_tx_clean_complq(struct idpf_compl_queue *complq, int budget, ntc -= complq->desc_count; do { - struct idpf_cleaned_stats cleaned_stats = { }; + struct libeth_sq_napi_stats cleaned_stats = { }; struct idpf_tx_queue *tx_q; int rel_tx_qid; u16 hw_head; @@ -2158,29 +2133,6 @@ void idpf_tx_splitq_build_flow_desc(union idpf_tx_flex_desc *desc, } /** - * idpf_tx_maybe_stop_common - 1st level check for common Tx stop conditions - * @tx_q: the queue to be checked - * @size: number of descriptors we want to assure is available - * - * Returns 0 if stop is not needed - */ -int idpf_tx_maybe_stop_common(struct idpf_tx_queue *tx_q, unsigned int size) -{ - struct netdev_queue *nq; - - if (likely(IDPF_DESC_UNUSED(tx_q) >= size)) - return 0; - - u64_stats_update_begin(&tx_q->stats_sync); - u64_stats_inc(&tx_q->q_stats.q_busy); - u64_stats_update_end(&tx_q->stats_sync); - - nq = netdev_get_tx_queue(tx_q->netdev, tx_q->idx); - - return netif_txq_maybe_stop(nq, IDPF_DESC_UNUSED(tx_q), size, size); -} - -/** * idpf_tx_maybe_stop_splitq - 1st level check for Tx splitq stop conditions * @tx_q: the queue to be checked * @descs_needed: number of descriptors required for this packet @@ -2191,7 +2143,7 @@ static int idpf_tx_maybe_stop_splitq(struct idpf_tx_queue *tx_q, unsigned int descs_needed) { if (idpf_tx_maybe_stop_common(tx_q, descs_needed)) - goto splitq_stop; + goto out; /* If there are too many outstanding completions expected on the * completion queue, stop the TX queue to give the device some time to @@ -2210,10 +2162,12 @@ static int idpf_tx_maybe_stop_splitq(struct idpf_tx_queue *tx_q, return 0; splitq_stop: + netif_stop_subqueue(tx_q->netdev, tx_q->idx); + +out: u64_stats_update_begin(&tx_q->stats_sync); u64_stats_inc(&tx_q->q_stats.q_busy); u64_stats_update_end(&tx_q->stats_sync); - netif_stop_subqueue(tx_q->netdev, tx_q->idx); return -EBUSY; } @@ -2236,7 +2190,11 @@ void idpf_tx_buf_hw_update(struct idpf_tx_queue *tx_q, u32 val, nq = netdev_get_tx_queue(tx_q->netdev, tx_q->idx); tx_q->next_to_use = val; - idpf_tx_maybe_stop_common(tx_q, IDPF_TX_DESC_NEEDED); + if (idpf_tx_maybe_stop_common(tx_q, IDPF_TX_DESC_NEEDED)) { + u64_stats_update_begin(&tx_q->stats_sync); + u64_stats_inc(&tx_q->q_stats.q_busy); + u64_stats_update_end(&tx_q->stats_sync); + } /* Force memory writes to complete before letting h/w * know there are new descriptors to fetch. (Only @@ -2307,6 +2265,12 @@ unsigned int idpf_tx_desc_count_required(struct idpf_tx_queue *txq, void idpf_tx_dma_map_error(struct idpf_tx_queue *txq, struct sk_buff *skb, struct idpf_tx_buf *first, u16 idx) { + struct libeth_sq_napi_stats ss = { }; + struct libeth_cq_pp cp = { + .dev = txq->dev, + .ss = &ss, + }; + u64_stats_update_begin(&txq->stats_sync); u64_stats_inc(&txq->q_stats.dma_map_errs); u64_stats_update_end(&txq->stats_sync); @@ -2316,7 +2280,7 @@ void idpf_tx_dma_map_error(struct idpf_tx_queue *txq, struct sk_buff *skb, struct idpf_tx_buf *tx_buf; tx_buf = &txq->tx_buf[idx]; - idpf_tx_buf_rel(txq, tx_buf); + libeth_tx_complete(tx_buf, &cp); if (tx_buf == first) break; if (idx == 0) @@ -2395,6 +2359,7 @@ static void idpf_tx_splitq_map(struct idpf_tx_queue *tx_q, dma = dma_map_single(tx_q->dev, skb->data, size, DMA_TO_DEVICE); tx_buf = first; + first->nr_frags = 0; params->compl_tag = (tx_q->compl_tag_cur_gen << tx_q->compl_tag_gen_s) | i; @@ -2405,7 +2370,9 @@ static void idpf_tx_splitq_map(struct idpf_tx_queue *tx_q, if (dma_mapping_error(tx_q->dev, dma)) return idpf_tx_dma_map_error(tx_q, skb, first, i); - tx_buf->compl_tag = params->compl_tag; + first->nr_frags++; + idpf_tx_buf_compl_tag(tx_buf) = params->compl_tag; + tx_buf->type = LIBETH_SQE_FRAG; /* record length, and DMA address */ dma_unmap_len_set(tx_buf, len, size); @@ -2459,14 +2426,15 @@ static void idpf_tx_splitq_map(struct idpf_tx_queue *tx_q, idpf_tx_splitq_build_desc(tx_desc, params, td_cmd, max_data); - tx_desc++; - i++; - - if (i == tx_q->desc_count) { + if (unlikely(++i == tx_q->desc_count)) { + tx_buf = tx_q->tx_buf; tx_desc = &tx_q->flex_tx[0]; i = 0; tx_q->compl_tag_cur_gen = IDPF_TX_ADJ_COMPL_TAG_GEN(tx_q); + } else { + tx_buf++; + tx_desc++; } /* Since this packet has a buffer that is going to span @@ -2479,8 +2447,7 @@ static void idpf_tx_splitq_map(struct idpf_tx_queue *tx_q, * simply pass over these holes and finish cleaning the * rest of the packet. */ - memset(&tx_q->tx_buf[i], 0, sizeof(struct idpf_tx_buf)); - tx_q->tx_buf[i].compl_tag = params->compl_tag; + tx_buf->type = LIBETH_SQE_EMPTY; /* Adjust the DMA offset and the remaining size of the * fragment. On the first iteration of this loop, @@ -2504,13 +2471,15 @@ static void idpf_tx_splitq_map(struct idpf_tx_queue *tx_q, break; idpf_tx_splitq_build_desc(tx_desc, params, td_cmd, size); - tx_desc++; - i++; - if (i == tx_q->desc_count) { + if (unlikely(++i == tx_q->desc_count)) { + tx_buf = tx_q->tx_buf; tx_desc = &tx_q->flex_tx[0]; i = 0; tx_q->compl_tag_cur_gen = IDPF_TX_ADJ_COMPL_TAG_GEN(tx_q); + } else { + tx_buf++; + tx_desc++; } size = skb_frag_size(frag); @@ -2518,26 +2487,24 @@ static void idpf_tx_splitq_map(struct idpf_tx_queue *tx_q, dma = skb_frag_dma_map(tx_q->dev, frag, 0, size, DMA_TO_DEVICE); - - tx_buf = &tx_q->tx_buf[i]; } /* record SW timestamp if HW timestamp is not available */ skb_tx_timestamp(skb); + first->type = LIBETH_SQE_SKB; + /* write last descriptor with RS and EOP bits */ + first->rs_idx = i; td_cmd |= params->eop_cmd; idpf_tx_splitq_build_desc(tx_desc, params, td_cmd, size); i = idpf_tx_splitq_bump_ntu(tx_q, i); - /* set next_to_watch value indicating a packet is present */ - first->next_to_watch = tx_desc; - tx_q->txq_grp->num_completions_pending++; /* record bytecount for BQL */ nq = netdev_get_tx_queue(tx_q->netdev, tx_q->idx); - netdev_tx_sent_queue(nq, first->bytecount); + netdev_tx_sent_queue(nq, first->bytes); idpf_tx_buf_hw_update(tx_q, i, netdev_xmit_more()); } @@ -2737,8 +2704,7 @@ idpf_tx_splitq_get_ctx_desc(struct idpf_tx_queue *txq) struct idpf_flex_tx_ctx_desc *desc; int i = txq->next_to_use; - memset(&txq->tx_buf[i], 0, sizeof(struct idpf_tx_buf)); - txq->tx_buf[i].compl_tag = IDPF_SPLITQ_TX_INVAL_COMPL_TAG; + txq->tx_buf[i].type = LIBETH_SQE_CTX; /* grab the next descriptor */ desc = &txq->flex_ctx[i]; @@ -2822,12 +2788,12 @@ static netdev_tx_t idpf_tx_splitq_frame(struct sk_buff *skb, first->skb = skb; if (tso) { - first->gso_segs = tx_params.offload.tso_segs; - first->bytecount = skb->len + - ((first->gso_segs - 1) * tx_params.offload.tso_hdr_len); + first->packets = tx_params.offload.tso_segs; + first->bytes = skb->len + + ((first->packets - 1) * tx_params.offload.tso_hdr_len); } else { - first->gso_segs = 1; - first->bytecount = max_t(unsigned int, skb->len, ETH_ZLEN); + first->packets = 1; + first->bytes = max_t(unsigned int, skb->len, ETH_ZLEN); } if (idpf_queue_has(FLOW_SCH_EN, tx_q)) { @@ -3749,6 +3715,7 @@ void idpf_vport_intr_update_itr_ena_irq(struct idpf_q_vector *q_vector) /* net_dim() updates ITR out-of-band using a work item */ idpf_net_dim(q_vector); + q_vector->wb_on_itr = false; intval = idpf_vport_intr_buildreg_itr(q_vector, IDPF_NO_ITR_UPDATE_IDX, 0); @@ -4051,8 +4018,10 @@ static int idpf_vport_splitq_napi_poll(struct napi_struct *napi, int budget) clean_complete &= idpf_tx_splitq_clean_all(q_vector, budget, &work_done); /* If work not completed, return budget and polling will return */ - if (!clean_complete) + if (!clean_complete) { + idpf_vport_intr_set_wb_on_itr(q_vector); return budget; + } work_done = min_t(int, work_done, budget - 1); @@ -4061,6 +4030,8 @@ static int idpf_vport_splitq_napi_poll(struct napi_struct *napi, int budget) */ if (likely(napi_complete_done(napi, work_done))) idpf_vport_intr_update_itr_ena_irq(q_vector); + else + idpf_vport_intr_set_wb_on_itr(q_vector); /* Switch to poll mode in the tear-down path after sending disable * queues virtchnl message, as the interrupts will be disabled after diff --git a/drivers/net/ethernet/intel/idpf/idpf_txrx.h b/drivers/net/ethernet/intel/idpf/idpf_txrx.h index 6215dbee5546..f0537826f840 100644 --- a/drivers/net/ethernet/intel/idpf/idpf_txrx.h +++ b/drivers/net/ethernet/intel/idpf/idpf_txrx.h @@ -127,11 +127,10 @@ do { \ */ #define IDPF_TX_COMPLQ_PENDING(txq) \ (((txq)->num_completions_pending >= (txq)->complq->num_completions ? \ - 0 : U64_MAX) + \ + 0 : U32_MAX) + \ (txq)->num_completions_pending - (txq)->complq->num_completions) #define IDPF_TX_SPLITQ_COMPL_TAG_WIDTH 16 -#define IDPF_SPLITQ_TX_INVAL_COMPL_TAG -1 /* Adjust the generation for the completion tag and wrap if necessary */ #define IDPF_TX_ADJ_COMPL_TAG_GEN(txq) \ ((++(txq)->compl_tag_cur_gen) >= (txq)->compl_tag_gen_max ? \ @@ -149,47 +148,7 @@ union idpf_tx_flex_desc { struct idpf_flex_tx_sched_desc flow; /* flow based scheduling */ }; -/** - * struct idpf_tx_buf - * @next_to_watch: Next descriptor to clean - * @skb: Pointer to the skb - * @dma: DMA address - * @len: DMA length - * @bytecount: Number of bytes - * @gso_segs: Number of GSO segments - * @compl_tag: Splitq only, unique identifier for a buffer. Used to compare - * with completion tag returned in buffer completion event. - * Because the completion tag is expected to be the same in all - * data descriptors for a given packet, and a single packet can - * span multiple buffers, we need this field to track all - * buffers associated with this completion tag independently of - * the buf_id. The tag consists of a N bit buf_id and M upper - * order "generation bits". See compl_tag_bufid_m and - * compl_tag_gen_s in struct idpf_queue. We'll use a value of -1 - * to indicate the tag is not valid. - * @ctx_entry: Singleq only. Used to indicate the corresponding entry - * in the descriptor ring was used for a context descriptor and - * this buffer entry should be skipped. - */ -struct idpf_tx_buf { - void *next_to_watch; - struct sk_buff *skb; - DEFINE_DMA_UNMAP_ADDR(dma); - DEFINE_DMA_UNMAP_LEN(len); - unsigned int bytecount; - unsigned short gso_segs; - - union { - int compl_tag; - - bool ctx_entry; - }; -}; - -struct idpf_tx_stash { - struct hlist_node hlist; - struct idpf_tx_buf buf; -}; +#define idpf_tx_buf libeth_sqe /** * struct idpf_buf_lifo - LIFO for managing OOO completions @@ -390,9 +349,11 @@ struct idpf_vec_regs { * struct idpf_intr_reg * @dyn_ctl: Dynamic control interrupt register * @dyn_ctl_intena_m: Mask for dyn_ctl interrupt enable + * @dyn_ctl_intena_msk_m: Mask for dyn_ctl interrupt enable mask * @dyn_ctl_itridx_s: Register bit offset for ITR index * @dyn_ctl_itridx_m: Mask for ITR index * @dyn_ctl_intrvl_s: Register bit offset for ITR interval + * @dyn_ctl_wb_on_itr_m: Mask for WB on ITR feature * @rx_itr: RX ITR register * @tx_itr: TX ITR register * @icr_ena: Interrupt cause register offset @@ -401,9 +362,11 @@ struct idpf_vec_regs { struct idpf_intr_reg { void __iomem *dyn_ctl; u32 dyn_ctl_intena_m; + u32 dyn_ctl_intena_msk_m; u32 dyn_ctl_itridx_s; u32 dyn_ctl_itridx_m; u32 dyn_ctl_intrvl_s; + u32 dyn_ctl_wb_on_itr_m; void __iomem *rx_itr; void __iomem *tx_itr; void __iomem *icr_ena; @@ -424,6 +387,7 @@ struct idpf_intr_reg { * @intr_reg: See struct idpf_intr_reg * @napi: napi handler * @total_events: Number of interrupts processed + * @wb_on_itr: whether WB on ITR is enabled * @tx_dim: Data for TX net_dim algorithm * @tx_itr_value: TX interrupt throttling rate * @tx_intr_mode: Dynamic ITR or not @@ -454,6 +418,7 @@ struct idpf_q_vector { __cacheline_group_begin_aligned(read_write); struct napi_struct napi; u16 total_events; + bool wb_on_itr; struct dim tx_dim; u16 tx_itr_value; @@ -472,7 +437,7 @@ struct idpf_q_vector { cpumask_var_t affinity_mask; __cacheline_group_end_aligned(cold); }; -libeth_cacheline_set_assert(struct idpf_q_vector, 104, +libeth_cacheline_set_assert(struct idpf_q_vector, 112, 424 + 2 * sizeof(struct dim), 8 + sizeof(cpumask_var_t)); @@ -496,11 +461,6 @@ struct idpf_tx_queue_stats { u64_stats_t dma_map_errs; }; -struct idpf_cleaned_stats { - u32 packets; - u32 bytes; -}; - #define IDPF_ITR_DYNAMIC 1 #define IDPF_ITR_MAX 0x1FE0 #define IDPF_ITR_20K 0x0032 @@ -688,7 +648,7 @@ struct idpf_tx_queue { void *desc_ring; }; - struct idpf_tx_buf *tx_buf; + struct libeth_sqe *tx_buf; struct idpf_txq_group *txq_grp; struct device *dev; void __iomem *tail; @@ -831,7 +791,7 @@ struct idpf_compl_queue { u32 next_to_use; u32 next_to_clean; - u32 num_completions; + aligned_u64 num_completions; __cacheline_group_end_aligned(read_write); __cacheline_group_begin_aligned(cold); @@ -963,7 +923,7 @@ struct idpf_txq_group { struct idpf_compl_queue *complq; - u32 num_completions_pending; + aligned_u64 num_completions_pending; }; static inline int idpf_q_vector_to_mem(const struct idpf_q_vector *q_vector) @@ -1033,6 +993,25 @@ static inline void idpf_tx_splitq_build_desc(union idpf_tx_flex_desc *desc, idpf_tx_splitq_build_flow_desc(desc, params, td_cmd, size); } +/** + * idpf_vport_intr_set_wb_on_itr - enable descriptor writeback on disabled interrupts + * @q_vector: pointer to queue vector struct + */ +static inline void idpf_vport_intr_set_wb_on_itr(struct idpf_q_vector *q_vector) +{ + struct idpf_intr_reg *reg; + + if (q_vector->wb_on_itr) + return; + + q_vector->wb_on_itr = true; + reg = &q_vector->intr_reg; + + writel(reg->dyn_ctl_wb_on_itr_m | reg->dyn_ctl_intena_msk_m | + (IDPF_NO_ITR_UPDATE_IDX << reg->dyn_ctl_itridx_s), + reg->dyn_ctl); +} + int idpf_vport_singleq_napi_poll(struct napi_struct *napi, int budget); void idpf_vport_init_num_qs(struct idpf_vport *vport, struct virtchnl2_create_vport *vport_msg); @@ -1064,7 +1043,6 @@ void idpf_tx_dma_map_error(struct idpf_tx_queue *txq, struct sk_buff *skb, struct idpf_tx_buf *first, u16 ring_idx); unsigned int idpf_tx_desc_count_required(struct idpf_tx_queue *txq, struct sk_buff *skb); -int idpf_tx_maybe_stop_common(struct idpf_tx_queue *tx_q, unsigned int size); void idpf_tx_timeout(struct net_device *netdev, unsigned int txqueue); netdev_tx_t idpf_tx_singleq_frame(struct sk_buff *skb, struct idpf_tx_queue *tx_q); @@ -1073,4 +1051,12 @@ bool idpf_rx_singleq_buf_hw_alloc_all(struct idpf_rx_queue *rxq, u16 cleaned_count); int idpf_tso(struct sk_buff *skb, struct idpf_tx_offload_params *off); +static inline bool idpf_tx_maybe_stop_common(struct idpf_tx_queue *tx_q, + u32 needed) +{ + return !netif_subqueue_maybe_stop(tx_q->netdev, tx_q->idx, + IDPF_DESC_UNUSED(tx_q), + needed, needed); +} + #endif /* !_IDPF_TXRX_H_ */ diff --git a/drivers/net/ethernet/intel/idpf/idpf_vf_dev.c b/drivers/net/ethernet/intel/idpf/idpf_vf_dev.c index 629cb5cb7c9f..99b8dbaf4225 100644 --- a/drivers/net/ethernet/intel/idpf/idpf_vf_dev.c +++ b/drivers/net/ethernet/intel/idpf/idpf_vf_dev.c @@ -97,7 +97,9 @@ static int idpf_vf_intr_reg_init(struct idpf_vport *vport) intr->dyn_ctl = idpf_get_reg_addr(adapter, reg_vals[vec_id].dyn_ctl_reg); intr->dyn_ctl_intena_m = VF_INT_DYN_CTLN_INTENA_M; + intr->dyn_ctl_intena_msk_m = VF_INT_DYN_CTLN_INTENA_MSK_M; intr->dyn_ctl_itridx_s = VF_INT_DYN_CTLN_ITR_INDX_S; + intr->dyn_ctl_wb_on_itr_m = VF_INT_DYN_CTLN_WB_ON_ITR_M; spacing = IDPF_ITR_IDX_SPACING(reg_vals[vec_id].itrn_index_spacing, IDPF_VF_ITR_IDX_SPACING); diff --git a/drivers/net/ethernet/intel/igb/igb_ethtool.c b/drivers/net/ethernet/intel/igb/igb_ethtool.c index 06b9970dffad..ca6ccbc13954 100644 --- a/drivers/net/ethernet/intel/igb/igb_ethtool.c +++ b/drivers/net/ethernet/intel/igb/igb_ethtool.c @@ -2387,15 +2387,11 @@ static int igb_get_ts_info(struct net_device *dev, if (adapter->ptp_clock) info->phc_index = ptp_clock_index(adapter->ptp_clock); - else - info->phc_index = -1; switch (adapter->hw.mac.type) { case e1000_82575: info->so_timestamping = - SOF_TIMESTAMPING_TX_SOFTWARE | - SOF_TIMESTAMPING_RX_SOFTWARE | - SOF_TIMESTAMPING_SOFTWARE; + SOF_TIMESTAMPING_TX_SOFTWARE; return 0; case e1000_82576: case e1000_82580: @@ -2405,8 +2401,6 @@ static int igb_get_ts_info(struct net_device *dev, case e1000_i211: info->so_timestamping = SOF_TIMESTAMPING_TX_SOFTWARE | - SOF_TIMESTAMPING_RX_SOFTWARE | - SOF_TIMESTAMPING_SOFTWARE | SOF_TIMESTAMPING_TX_HARDWARE | SOF_TIMESTAMPING_RX_HARDWARE | SOF_TIMESTAMPING_RAW_HARDWARE; diff --git a/drivers/net/ethernet/intel/igb/igb_main.c b/drivers/net/ethernet/intel/igb/igb_main.c index 33a42b4c21e0..1ef4cb871452 100644 --- a/drivers/net/ethernet/intel/igb/igb_main.c +++ b/drivers/net/ethernet/intel/igb/igb_main.c @@ -33,6 +33,7 @@ #include <linux/bpf_trace.h> #include <linux/pm_runtime.h> #include <linux/etherdevice.h> +#include <linux/lockdep.h> #ifdef CONFIG_IGB_DCA #include <linux/dca.h> #endif @@ -2914,8 +2915,11 @@ static int igb_xdp(struct net_device *dev, struct netdev_bpf *xdp) } } +/* This function assumes __netif_tx_lock is held by the caller. */ static void igb_xdp_ring_update_tail(struct igb_ring *ring) { + lockdep_assert_held(&txring_txq(ring)->_xmit_lock); + /* Force memory writes to complete before letting h/w know there * are new descriptors to fetch. */ @@ -3000,11 +3004,11 @@ static int igb_xdp_xmit(struct net_device *dev, int n, nxmit++; } - __netif_tx_unlock(nq); - if (unlikely(flags & XDP_XMIT_FLUSH)) igb_xdp_ring_update_tail(tx_ring); + __netif_tx_unlock(nq); + return nxmit; } @@ -6960,10 +6964,20 @@ static void igb_extts(struct igb_adapter *adapter, int tsintr_tt) static void igb_tsync_interrupt(struct igb_adapter *adapter) { + const u32 mask = (TSINTR_SYS_WRAP | E1000_TSICR_TXTS | + TSINTR_TT0 | TSINTR_TT1 | + TSINTR_AUTT0 | TSINTR_AUTT1); struct e1000_hw *hw = &adapter->hw; u32 tsicr = rd32(E1000_TSICR); struct ptp_clock_event event; + if (hw->mac.type == e1000_82580) { + /* 82580 has a hardware bug that requires an explicit + * write to clear the TimeSync interrupt cause. + */ + wr32(E1000_TSICR, tsicr & mask); + } + if (tsicr & TSINTR_SYS_WRAP) { event.type = PTP_CLOCK_PPS; if (adapter->ptp_caps.pps) @@ -8854,12 +8868,14 @@ static void igb_put_rx_buffer(struct igb_ring *rx_ring, static int igb_clean_rx_irq(struct igb_q_vector *q_vector, const int budget) { + unsigned int total_bytes = 0, total_packets = 0; struct igb_adapter *adapter = q_vector->adapter; struct igb_ring *rx_ring = q_vector->rx.ring; - struct sk_buff *skb = rx_ring->skb; - unsigned int total_bytes = 0, total_packets = 0; u16 cleaned_count = igb_desc_unused(rx_ring); + struct sk_buff *skb = rx_ring->skb; + int cpu = smp_processor_id(); unsigned int xdp_xmit = 0; + struct netdev_queue *nq; struct xdp_buff xdp; u32 frame_sz = 0; int rx_buf_pgcnt; @@ -8987,7 +9003,10 @@ static int igb_clean_rx_irq(struct igb_q_vector *q_vector, const int budget) if (xdp_xmit & IGB_XDP_TX) { struct igb_ring *tx_ring = igb_xdp_tx_queue_mapping(adapter); + nq = txring_txq(tx_ring); + __netif_tx_lock(nq, cpu); igb_xdp_ring_update_tail(tx_ring); + __netif_tx_unlock(nq); } u64_stats_update_begin(&rx_ring->rx_syncp); diff --git a/drivers/net/ethernet/intel/igbvf/igbvf.h b/drivers/net/ethernet/intel/igbvf/igbvf.h index 7b83678ba83a..6ad35a00a287 100644 --- a/drivers/net/ethernet/intel/igbvf/igbvf.h +++ b/drivers/net/ethernet/intel/igbvf/igbvf.h @@ -282,7 +282,6 @@ enum igbvf_state_t { extern char igbvf_driver_name[]; -void igbvf_check_options(struct igbvf_adapter *); void igbvf_set_ethtool_ops(struct net_device *); int igbvf_up(struct igbvf_adapter *); diff --git a/drivers/net/ethernet/intel/igbvf/mbx.h b/drivers/net/ethernet/intel/igbvf/mbx.h index e5b31818d565..7637d21445bf 100644 --- a/drivers/net/ethernet/intel/igbvf/mbx.h +++ b/drivers/net/ethernet/intel/igbvf/mbx.h @@ -49,7 +49,6 @@ #define E1000_PF_CONTROL_MSG 0x0100 /* PF control message */ -void e1000_init_mbx_ops_generic(struct e1000_hw *hw); s32 e1000_init_mbx_params_vf(struct e1000_hw *); #endif /* _E1000_MBX_H_ */ diff --git a/drivers/net/ethernet/intel/igc/igc.h b/drivers/net/ethernet/intel/igc/igc.h index c38b4d0f00ce..eac0f966e0e4 100644 --- a/drivers/net/ethernet/intel/igc/igc.h +++ b/drivers/net/ethernet/intel/igc/igc.h @@ -259,6 +259,10 @@ struct igc_adapter { */ spinlock_t qbv_tx_lock; + bool strict_priority_enable; + u8 num_tc; + u16 queue_per_tc[IGC_MAX_TX_QUEUES]; + /* OS defined structs */ struct pci_dev *pdev; /* lock for statistics */ @@ -382,9 +386,11 @@ extern char igc_driver_name[]; #define IGC_FLAG_RX_LEGACY BIT(16) #define IGC_FLAG_TSN_QBV_ENABLED BIT(17) #define IGC_FLAG_TSN_QAV_ENABLED BIT(18) +#define IGC_FLAG_TSN_LEGACY_ENABLED BIT(19) -#define IGC_FLAG_TSN_ANY_ENABLED \ - (IGC_FLAG_TSN_QBV_ENABLED | IGC_FLAG_TSN_QAV_ENABLED) +#define IGC_FLAG_TSN_ANY_ENABLED \ + (IGC_FLAG_TSN_QBV_ENABLED | IGC_FLAG_TSN_QAV_ENABLED | \ + IGC_FLAG_TSN_LEGACY_ENABLED) #define IGC_FLAG_RSS_FIELD_IPV4_UDP BIT(6) #define IGC_FLAG_RSS_FIELD_IPV6_UDP BIT(7) @@ -681,6 +687,7 @@ enum igc_ring_flags_t { IGC_RING_FLAG_TX_DETECT_HANG, IGC_RING_FLAG_AF_XDP_ZC, IGC_RING_FLAG_TX_HWTSTAMP, + IGC_RING_FLAG_RX_ALLOC_FAILED, }; #define ring_uses_large_buffer(ring) \ diff --git a/drivers/net/ethernet/intel/igc/igc_defines.h b/drivers/net/ethernet/intel/igc/igc_defines.h index 511384f3ec5c..8e449904aa7d 100644 --- a/drivers/net/ethernet/intel/igc/igc_defines.h +++ b/drivers/net/ethernet/intel/igc/igc_defines.h @@ -4,6 +4,8 @@ #ifndef _IGC_DEFINES_H_ #define _IGC_DEFINES_H_ +#include <linux/bitfield.h> + /* Number of Transmit and Receive Descriptors must be a multiple of 8 */ #define REQ_TX_DESCRIPTOR_MULTIPLE 8 #define REQ_RX_DESCRIPTOR_MULTIPLE 8 @@ -176,7 +178,6 @@ /* PHY GPY 211 registers */ #define STANDARD_AN_REG_MASK 0x0007 /* MMD */ -#define ANEG_MULTIGBT_AN_CTRL 0x0020 /* MULTI GBT AN Control Register */ #define MMD_DEVADDR_SHIFT 16 /* Shift MMD to higher bits */ #define CR_2500T_FD_CAPS 0x0080 /* Advertise 2500T FD capability */ @@ -553,6 +554,15 @@ #define IGC_MAX_SR_QUEUES 2 +#define IGC_TXARB_TXQ_PRIO_0_MASK GENMASK(1, 0) +#define IGC_TXARB_TXQ_PRIO_1_MASK GENMASK(3, 2) +#define IGC_TXARB_TXQ_PRIO_2_MASK GENMASK(5, 4) +#define IGC_TXARB_TXQ_PRIO_3_MASK GENMASK(7, 6) +#define IGC_TXARB_TXQ_PRIO_0(x) FIELD_PREP(IGC_TXARB_TXQ_PRIO_0_MASK, (x)) +#define IGC_TXARB_TXQ_PRIO_1(x) FIELD_PREP(IGC_TXARB_TXQ_PRIO_1_MASK, (x)) +#define IGC_TXARB_TXQ_PRIO_2(x) FIELD_PREP(IGC_TXARB_TXQ_PRIO_2_MASK, (x)) +#define IGC_TXARB_TXQ_PRIO_3(x) FIELD_PREP(IGC_TXARB_TXQ_PRIO_3_MASK, (x)) + /* Receive Checksum Control */ #define IGC_RXCSUM_CRCOFL 0x00000800 /* CRC32 offload enable */ #define IGC_RXCSUM_PCSD 0x00002000 /* packet checksum disabled */ @@ -641,6 +651,16 @@ #define IGC_MDIC_READY 0x10000000 #define IGC_MDIC_ERROR 0x40000000 +/* EEE Link Ability */ +#define IGC_EEE_2500BT_MASK BIT(0) +#define IGC_EEE_1000BT_MASK BIT(2) +#define IGC_EEE_100BT_MASK BIT(1) + +/* EEE Link-Partner Ability */ +#define IGC_LP_EEE_2500BT_MASK BIT(0) +#define IGC_LP_EEE_1000BT_MASK BIT(2) +#define IGC_LP_EEE_100BT_MASK BIT(1) + #define IGC_N0_QUEUE -1 #define IGC_MAX_MAC_HDR_LEN 127 diff --git a/drivers/net/ethernet/intel/igc/igc_ethtool.c b/drivers/net/ethernet/intel/igc/igc_ethtool.c index 3d3ef4e1547c..5b0c6f433767 100644 --- a/drivers/net/ethernet/intel/igc/igc_ethtool.c +++ b/drivers/net/ethernet/intel/igc/igc_ethtool.c @@ -1540,6 +1540,10 @@ static int igc_ethtool_set_channels(struct net_device *netdev, if (ch->other_count != NON_Q_VECTORS) return -EINVAL; + /* Do not allow channel reconfiguration when mqprio is enabled */ + if (adapter->strict_priority_enable) + return -EINVAL; + /* Verify the number of channels doesn't exceed hw limits */ max_combined = igc_get_max_rss_queues(adapter); if (count > max_combined) @@ -1565,15 +1569,11 @@ static int igc_ethtool_get_ts_info(struct net_device *dev, if (adapter->ptp_clock) info->phc_index = ptp_clock_index(adapter->ptp_clock); - else - info->phc_index = -1; switch (adapter->hw.mac.type) { case igc_i225: info->so_timestamping = SOF_TIMESTAMPING_TX_SOFTWARE | - SOF_TIMESTAMPING_RX_SOFTWARE | - SOF_TIMESTAMPING_SOFTWARE | SOF_TIMESTAMPING_TX_HARDWARE | SOF_TIMESTAMPING_RX_HARDWARE | SOF_TIMESTAMPING_RAW_HARDWARE; @@ -1627,8 +1627,11 @@ static int igc_ethtool_get_eee(struct net_device *netdev, { struct igc_adapter *adapter = netdev_priv(netdev); struct igc_hw *hw = &adapter->hw; - u32 eeer; + struct igc_phy_info *phy = &hw->phy; + u16 eee_advert, eee_lp_advert; + u32 eeer, ret_val; + /* EEE supported */ linkmode_set_bit(ETHTOOL_LINK_MODE_2500baseT_Full_BIT, edata->supported); linkmode_set_bit(ETHTOOL_LINK_MODE_1000baseT_Full_BIT, @@ -1636,6 +1639,74 @@ static int igc_ethtool_get_eee(struct net_device *netdev, linkmode_set_bit(ETHTOOL_LINK_MODE_100baseT_Full_BIT, edata->supported); + /* EEE Advertisement 1 - reg 7.60 */ + ret_val = phy->ops.read_reg(hw, (STANDARD_AN_REG_MASK << + MMD_DEVADDR_SHIFT) | + IGC_ANEG_EEE_AB1, + &eee_advert); + if (ret_val) { + netdev_err(adapter->netdev, + "Failed to read IEEE 7.60 register\n"); + return -EINVAL; + } + + if (eee_advert & IGC_EEE_1000BT_MASK) + linkmode_set_bit(ETHTOOL_LINK_MODE_1000baseT_Full_BIT, + edata->advertised); + + if (eee_advert & IGC_EEE_100BT_MASK) + linkmode_set_bit(ETHTOOL_LINK_MODE_100baseT_Full_BIT, + edata->advertised); + + /* EEE Advertisement 2 - reg 7.62 */ + ret_val = phy->ops.read_reg(hw, (STANDARD_AN_REG_MASK << + MMD_DEVADDR_SHIFT) | + IGC_ANEG_EEE_AB2, + &eee_advert); + if (ret_val) { + netdev_err(adapter->netdev, + "Failed to read IEEE 7.62 register\n"); + return -EINVAL; + } + + if (eee_advert & IGC_EEE_2500BT_MASK) + linkmode_set_bit(ETHTOOL_LINK_MODE_2500baseT_Full_BIT, + edata->advertised); + + /* EEE Link-Partner Ability 1 - reg 7.61 */ + ret_val = phy->ops.read_reg(hw, (STANDARD_AN_REG_MASK << + MMD_DEVADDR_SHIFT) | + IGC_ANEG_EEE_LP_AB1, + &eee_lp_advert); + if (ret_val) { + netdev_err(adapter->netdev, + "Failed to read IEEE 7.61 register\n"); + return -EINVAL; + } + + if (eee_lp_advert & IGC_LP_EEE_1000BT_MASK) + linkmode_set_bit(ETHTOOL_LINK_MODE_1000baseT_Full_BIT, + edata->lp_advertised); + + if (eee_lp_advert & IGC_LP_EEE_100BT_MASK) + linkmode_set_bit(ETHTOOL_LINK_MODE_100baseT_Full_BIT, + edata->lp_advertised); + + /* EEE Link-Partner Ability 2 - reg 7.63 */ + ret_val = phy->ops.read_reg(hw, (STANDARD_AN_REG_MASK << + MMD_DEVADDR_SHIFT) | + IGC_ANEG_EEE_LP_AB2, + &eee_lp_advert); + if (ret_val) { + netdev_err(adapter->netdev, + "Failed to read IEEE 7.63 register\n"); + return -EINVAL; + } + + if (eee_lp_advert & IGC_LP_EEE_2500BT_MASK) + linkmode_set_bit(ETHTOOL_LINK_MODE_2500baseT_Full_BIT, + edata->lp_advertised); + eeer = rd32(IGC_EEER); /* EEE status on negotiated link */ diff --git a/drivers/net/ethernet/intel/igc/igc_main.c b/drivers/net/ethernet/intel/igc/igc_main.c index dfd6c00b4205..6e70bca15db1 100644 --- a/drivers/net/ethernet/intel/igc/igc_main.c +++ b/drivers/net/ethernet/intel/igc/igc_main.c @@ -2191,6 +2191,7 @@ static bool igc_alloc_mapped_page(struct igc_ring *rx_ring, page = dev_alloc_pages(igc_rx_pg_order(rx_ring)); if (unlikely(!page)) { rx_ring->rx_stats.alloc_failed++; + set_bit(IGC_RING_FLAG_RX_ALLOC_FAILED, &rx_ring->flags); return false; } @@ -2207,6 +2208,7 @@ static bool igc_alloc_mapped_page(struct igc_ring *rx_ring, __free_page(page); rx_ring->rx_stats.alloc_failed++; + set_bit(IGC_RING_FLAG_RX_ALLOC_FAILED, &rx_ring->flags); return false; } @@ -2658,6 +2660,7 @@ static int igc_clean_rx_irq(struct igc_q_vector *q_vector, const int budget) if (!skb) { rx_ring->rx_stats.alloc_failed++; rx_buffer->pagecnt_bias++; + set_bit(IGC_RING_FLAG_RX_ALLOC_FAILED, &rx_ring->flags); break; } @@ -2738,6 +2741,7 @@ static void igc_dispatch_skb_zc(struct igc_q_vector *q_vector, skb = igc_construct_skb_zc(ring, xdp); if (!skb) { ring->rx_stats.alloc_failed++; + set_bit(IGC_RING_FLAG_RX_ALLOC_FAILED, &ring->flags); return; } @@ -5807,11 +5811,29 @@ no_wait: if (adapter->flags & IGC_FLAG_HAS_MSIX) { u32 eics = 0; - for (i = 0; i < adapter->num_q_vectors; i++) - eics |= adapter->q_vector[i]->eims_value; - wr32(IGC_EICS, eics); + for (i = 0; i < adapter->num_q_vectors; i++) { + struct igc_q_vector *q_vector = adapter->q_vector[i]; + struct igc_ring *rx_ring; + + if (!q_vector->rx.ring) + continue; + + rx_ring = adapter->rx_ring[q_vector->rx.ring->queue_index]; + + if (test_bit(IGC_RING_FLAG_RX_ALLOC_FAILED, &rx_ring->flags)) { + eics |= q_vector->eims_value; + clear_bit(IGC_RING_FLAG_RX_ALLOC_FAILED, &rx_ring->flags); + } + } + if (eics) + wr32(IGC_EICS, eics); } else { - wr32(IGC_ICS, IGC_ICS_RXDMT0); + struct igc_ring *rx_ring = adapter->rx_ring[0]; + + if (test_bit(IGC_RING_FLAG_RX_ALLOC_FAILED, &rx_ring->flags)) { + clear_bit(IGC_RING_FLAG_RX_ALLOC_FAILED, &rx_ring->flags); + wr32(IGC_ICS, IGC_ICS_RXDMT0); + } } igc_ptp_tx_hang(adapter); @@ -6515,6 +6537,13 @@ static int igc_tc_query_caps(struct igc_adapter *adapter, struct igc_hw *hw = &adapter->hw; switch (base->type) { + case TC_SETUP_QDISC_MQPRIO: { + struct tc_mqprio_caps *caps = base->caps; + + caps->validate_queue_counts = true; + + return 0; + } case TC_SETUP_QDISC_TAPRIO: { struct tc_taprio_caps *caps = base->caps; @@ -6532,6 +6561,65 @@ static int igc_tc_query_caps(struct igc_adapter *adapter, } } +static void igc_save_mqprio_params(struct igc_adapter *adapter, u8 num_tc, + u16 *offset) +{ + int i; + + adapter->strict_priority_enable = true; + adapter->num_tc = num_tc; + + for (i = 0; i < num_tc; i++) + adapter->queue_per_tc[i] = offset[i]; +} + +static int igc_tsn_enable_mqprio(struct igc_adapter *adapter, + struct tc_mqprio_qopt_offload *mqprio) +{ + struct igc_hw *hw = &adapter->hw; + int i; + + if (hw->mac.type != igc_i225) + return -EOPNOTSUPP; + + if (!mqprio->qopt.num_tc) { + adapter->strict_priority_enable = false; + goto apply; + } + + /* There are as many TCs as Tx queues. */ + if (mqprio->qopt.num_tc != adapter->num_tx_queues) { + NL_SET_ERR_MSG_FMT_MOD(mqprio->extack, + "Only %d traffic classes supported", + adapter->num_tx_queues); + return -EOPNOTSUPP; + } + + /* Only one queue per TC is supported. */ + for (i = 0; i < mqprio->qopt.num_tc; i++) { + if (mqprio->qopt.count[i] != 1) { + NL_SET_ERR_MSG_MOD(mqprio->extack, + "Only one queue per TC supported"); + return -EOPNOTSUPP; + } + } + + /* Preemption is not supported yet. */ + if (mqprio->preemptible_tcs) { + NL_SET_ERR_MSG_MOD(mqprio->extack, + "Preemption is not supported yet"); + return -EOPNOTSUPP; + } + + igc_save_mqprio_params(adapter, mqprio->qopt.num_tc, + mqprio->qopt.offset); + + mqprio->qopt.hw = TC_MQPRIO_HW_OFFLOAD_TCS; + +apply: + return igc_tsn_offload_apply(adapter); +} + static int igc_setup_tc(struct net_device *dev, enum tc_setup_type type, void *type_data) { @@ -6551,6 +6639,9 @@ static int igc_setup_tc(struct net_device *dev, enum tc_setup_type type, case TC_SETUP_QDISC_CBS: return igc_tsn_enable_cbs(adapter, type_data); + case TC_SETUP_QDISC_MQPRIO: + return igc_tsn_enable_mqprio(adapter, type_data); + default: return -EOPNOTSUPP; } @@ -7413,6 +7504,7 @@ static void igc_io_resume(struct pci_dev *pdev) rtnl_lock(); if (netif_running(netdev)) { if (igc_open(netdev)) { + rtnl_unlock(); netdev_err(netdev, "igc_open failed after reset\n"); return; } diff --git a/drivers/net/ethernet/intel/igc/igc_phy.c b/drivers/net/ethernet/intel/igc/igc_phy.c index 861f37076861..2801e5f24df9 100644 --- a/drivers/net/ethernet/intel/igc/igc_phy.c +++ b/drivers/net/ethernet/intel/igc/igc_phy.c @@ -240,7 +240,7 @@ static s32 igc_phy_setup_autoneg(struct igc_hw *hw) /* Read the MULTI GBT AN Control Register - reg 7.32 */ ret_val = phy->ops.read_reg(hw, (STANDARD_AN_REG_MASK << MMD_DEVADDR_SHIFT) | - ANEG_MULTIGBT_AN_CTRL, + IGC_ANEG_MULTIGBT_AN_CTRL, &aneg_multigbt_an_ctrl); if (ret_val) @@ -380,7 +380,7 @@ static s32 igc_phy_setup_autoneg(struct igc_hw *hw) ret_val = phy->ops.write_reg(hw, (STANDARD_AN_REG_MASK << MMD_DEVADDR_SHIFT) | - ANEG_MULTIGBT_AN_CTRL, + IGC_ANEG_MULTIGBT_AN_CTRL, aneg_multigbt_an_ctrl); return ret_val; diff --git a/drivers/net/ethernet/intel/igc/igc_regs.h b/drivers/net/ethernet/intel/igc/igc_regs.h index e5b893fc5b66..12ddc5793651 100644 --- a/drivers/net/ethernet/intel/igc/igc_regs.h +++ b/drivers/net/ethernet/intel/igc/igc_regs.h @@ -238,6 +238,8 @@ #define IGC_TQAVCC(_n) (0x3004 + ((_n) * 0x40)) #define IGC_TQAVHC(_n) (0x300C + ((_n) * 0x40)) +#define IGC_TXARB 0x3354 /* Tx Arbitration Control TxARB - RW */ + /* System Time Registers */ #define IGC_SYSTIML 0x0B600 /* System time register Low - RO */ #define IGC_SYSTIMH 0x0B604 /* System time register High - RO */ @@ -308,6 +310,16 @@ #define IGC_IPCNFG 0x0E38 /* Internal PHY Configuration */ #define IGC_EEE_SU 0x0E34 /* EEE Setup */ +/* MULTI GBT AN Control Register - reg. 7.32 */ +#define IGC_ANEG_MULTIGBT_AN_CTRL 0x0020 + +/* EEE ANeg Advertisement Register - reg 7.60 and reg 7.62 */ +#define IGC_ANEG_EEE_AB1 0x003c +#define IGC_ANEG_EEE_AB2 0x003e +/* EEE ANeg Link-Partner Advertisement Register - reg 7.61 and reg 7.63 */ +#define IGC_ANEG_EEE_LP_AB1 0x003d +#define IGC_ANEG_EEE_LP_AB2 0x003f + /* LTR registers */ #define IGC_LTRC 0x01A0 /* Latency Tolerance Reporting Control */ #define IGC_LTRMINV 0x5BB0 /* LTR Minimum Value */ diff --git a/drivers/net/ethernet/intel/igc/igc_tsn.c b/drivers/net/ethernet/intel/igc/igc_tsn.c index d68fa7f3d5f0..1e44374ca1ff 100644 --- a/drivers/net/ethernet/intel/igc/igc_tsn.c +++ b/drivers/net/ethernet/intel/igc/igc_tsn.c @@ -46,6 +46,9 @@ static unsigned int igc_tsn_new_flags(struct igc_adapter *adapter) if (is_cbs_enabled(adapter)) new_flags |= IGC_FLAG_TSN_QAV_ENABLED; + if (adapter->strict_priority_enable) + new_flags |= IGC_FLAG_TSN_LEGACY_ENABLED; + return new_flags; } @@ -102,11 +105,32 @@ bool igc_tsn_is_taprio_activated_by_user(struct igc_adapter *adapter) adapter->taprio_offload_enable; } +static void igc_tsn_tx_arb(struct igc_adapter *adapter, u16 *queue_per_tc) +{ + struct igc_hw *hw = &adapter->hw; + u32 txarb; + + txarb = rd32(IGC_TXARB); + + txarb &= ~(IGC_TXARB_TXQ_PRIO_0_MASK | + IGC_TXARB_TXQ_PRIO_1_MASK | + IGC_TXARB_TXQ_PRIO_2_MASK | + IGC_TXARB_TXQ_PRIO_3_MASK); + + txarb |= IGC_TXARB_TXQ_PRIO_0(queue_per_tc[3]); + txarb |= IGC_TXARB_TXQ_PRIO_1(queue_per_tc[2]); + txarb |= IGC_TXARB_TXQ_PRIO_2(queue_per_tc[1]); + txarb |= IGC_TXARB_TXQ_PRIO_3(queue_per_tc[0]); + + wr32(IGC_TXARB, txarb); +} + /* Returns the TSN specific registers to their default values after * the adapter is reset. */ static int igc_tsn_disable_offload(struct igc_adapter *adapter) { + u16 queue_per_tc[4] = { 3, 2, 1, 0 }; struct igc_hw *hw = &adapter->hw; u32 tqavctrl; int i; @@ -133,7 +157,16 @@ static int igc_tsn_disable_offload(struct igc_adapter *adapter) wr32(IGC_QBVCYCLET_S, 0); wr32(IGC_QBVCYCLET, NSEC_PER_SEC); + /* Reset mqprio TC configuration. */ + netdev_reset_tc(adapter->netdev); + + /* Restore the default Tx arbitration: Priority 0 has the highest + * priority and is assigned to queue 0 and so on and so forth. + */ + igc_tsn_tx_arb(adapter, queue_per_tc); + adapter->flags &= ~IGC_FLAG_TSN_QBV_ENABLED; + adapter->flags &= ~IGC_FLAG_TSN_LEGACY_ENABLED; return 0; } @@ -172,6 +205,40 @@ static int igc_tsn_enable_offload(struct igc_adapter *adapter) if (igc_is_device_id_i226(hw)) igc_tsn_set_retx_qbvfullthreshold(adapter); + if (adapter->strict_priority_enable) { + int err; + + err = netdev_set_num_tc(adapter->netdev, adapter->num_tc); + if (err) + return err; + + for (i = 0; i < adapter->num_tc; i++) { + err = netdev_set_tc_queue(adapter->netdev, i, 1, + adapter->queue_per_tc[i]); + if (err) + return err; + } + + /* In case the card is configured with less than four queues. */ + for (; i < IGC_MAX_TX_QUEUES; i++) + adapter->queue_per_tc[i] = i; + + /* Configure queue priorities according to the user provided + * mapping. + */ + igc_tsn_tx_arb(adapter, adapter->queue_per_tc); + + /* Enable legacy TSN mode which will do strict priority without + * any other TSN features. + */ + tqavctrl = rd32(IGC_TQAVCTRL); + tqavctrl |= IGC_TQAVCTRL_TRANSMIT_MODE_TSN; + tqavctrl &= ~IGC_TQAVCTRL_ENHANCED_QAV; + wr32(IGC_TQAVCTRL, tqavctrl); + + return 0; + } + for (i = 0; i < adapter->num_tx_queues; i++) { struct igc_ring *ring = adapter->tx_ring[i]; u32 txqctl = 0; diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_dcb_nl.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_dcb_nl.c index e85f7d2e8810..f2709b10c2e5 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_dcb_nl.c +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_dcb_nl.c @@ -317,7 +317,7 @@ static u8 ixgbe_dcbnl_set_all(struct net_device *netdev) int max_frame = adapter->netdev->mtu + ETH_HLEN + ETH_FCS_LEN; #ifdef IXGBE_FCOE - if (adapter->netdev->features & NETIF_F_FCOE_MTU) + if (adapter->netdev->fcoe_mtu) max_frame = max(max_frame, IXGBE_FCOE_JUMBO_FRAME_SIZE); #endif diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.c index 4cac76254966..9482e0cca8b7 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.c +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_ethtool.c @@ -3196,16 +3196,12 @@ static int ixgbe_get_ts_info(struct net_device *dev, info->so_timestamping = SOF_TIMESTAMPING_TX_SOFTWARE | - SOF_TIMESTAMPING_RX_SOFTWARE | - SOF_TIMESTAMPING_SOFTWARE | SOF_TIMESTAMPING_TX_HARDWARE | SOF_TIMESTAMPING_RX_HARDWARE | SOF_TIMESTAMPING_RAW_HARDWARE; if (adapter->ptp_clock) info->phc_index = ptp_clock_index(adapter->ptp_clock); - else - info->phc_index = -1; info->tx_types = BIT(HWTSTAMP_TX_OFF) | diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_fcoe.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_fcoe.c index 18d63c8c2ff4..955dced844a9 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_fcoe.c +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_fcoe.c @@ -858,7 +858,7 @@ int ixgbe_fcoe_enable(struct net_device *netdev) /* enable FCoE and notify stack */ adapter->flags |= IXGBE_FLAG_FCOE_ENABLED; - netdev->features |= NETIF_F_FCOE_MTU; + netdev->fcoe_mtu = true; netdev_features_change(netdev); /* release existing queues and reallocate them */ @@ -898,7 +898,7 @@ int ixgbe_fcoe_disable(struct net_device *netdev) /* disable FCoE and notify stack */ adapter->flags &= ~IXGBE_FLAG_FCOE_ENABLED; - netdev->features &= ~NETIF_F_FCOE_MTU; + netdev->fcoe_mtu = false; netdev_features_change(netdev); diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_lib.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_lib.c index 0ee943db3dc9..16fa621ce0ff 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_lib.c +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_lib.c @@ -981,7 +981,7 @@ static int ixgbe_alloc_q_vector(struct ixgbe_adapter *adapter, set_bit(__IXGBE_RX_CSUM_UDP_ZERO_ERR, &ring->state); #ifdef IXGBE_FCOE - if (adapter->netdev->features & NETIF_F_FCOE_MTU) { + if (adapter->netdev->fcoe_mtu) { struct ixgbe_ring_feature *f; f = &adapter->ring_feature[RING_F_FCOE]; if ((rxr_idx >= f->offset) && diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c index 8057cef61f39..8b8404d8c946 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_main.c @@ -5079,7 +5079,7 @@ static void ixgbe_configure_dcb(struct ixgbe_adapter *adapter) netif_set_tso_max_size(adapter->netdev, 32768); #ifdef IXGBE_FCOE - if (adapter->netdev->features & NETIF_F_FCOE_MTU) + if (adapter->netdev->fcoe_mtu) max_frame = max(max_frame, IXGBE_FCOE_JUMBO_FRAME_SIZE); #endif @@ -5136,8 +5136,7 @@ static int ixgbe_hpbthresh(struct ixgbe_adapter *adapter, int pb) #ifdef IXGBE_FCOE /* FCoE traffic class uses FCOE jumbo frames */ - if ((dev->features & NETIF_F_FCOE_MTU) && - (tc < IXGBE_FCOE_JUMBO_FRAME_SIZE) && + if (dev->fcoe_mtu && tc < IXGBE_FCOE_JUMBO_FRAME_SIZE && (pb == ixgbe_fcoe_get_tc(adapter))) tc = IXGBE_FCOE_JUMBO_FRAME_SIZE; #endif @@ -5197,8 +5196,7 @@ static int ixgbe_lpbthresh(struct ixgbe_adapter *adapter, int pb) #ifdef IXGBE_FCOE /* FCoE traffic class uses FCOE jumbo frames */ - if ((dev->features & NETIF_F_FCOE_MTU) && - (tc < IXGBE_FCOE_JUMBO_FRAME_SIZE) && + if (dev->fcoe_mtu && tc < IXGBE_FCOE_JUMBO_FRAME_SIZE && (pb == netdev_get_prio_tc_map(dev, adapter->fcoe.up))) tc = IXGBE_FCOE_JUMBO_FRAME_SIZE; #endif @@ -11096,8 +11094,7 @@ skip_sriov: NETIF_F_FCOE_CRC; netdev->vlan_features |= NETIF_F_FSO | - NETIF_F_FCOE_CRC | - NETIF_F_FCOE_MTU; + NETIF_F_FCOE_CRC; } #endif /* IXGBE_FCOE */ if (adapter->flags2 & IXGBE_FLAG2_RSC_CAPABLE) diff --git a/drivers/net/ethernet/intel/ixgbe/ixgbe_sriov.c b/drivers/net/ethernet/intel/ixgbe/ixgbe_sriov.c index fcfd0a075eee..e71715f5da22 100644 --- a/drivers/net/ethernet/intel/ixgbe/ixgbe_sriov.c +++ b/drivers/net/ethernet/intel/ixgbe/ixgbe_sriov.c @@ -495,7 +495,7 @@ static int ixgbe_set_vf_lpe(struct ixgbe_adapter *adapter, u32 max_frame, u32 vf int err = 0; #ifdef CONFIG_FCOE - if (dev->features & NETIF_F_FCOE_MTU) + if (dev->fcoe_mtu) pf_max_frame = max_t(int, pf_max_frame, IXGBE_FCOE_JUMBO_FRAME_SIZE); @@ -857,7 +857,7 @@ static void ixgbe_set_vf_rx_tx(struct ixgbe_adapter *adapter, int vf) int pf_max_frame = dev->mtu + ETH_HLEN; #if IS_ENABLED(CONFIG_FCOE) - if (dev->features & NETIF_F_FCOE_MTU) + if (dev->fcoe_mtu) pf_max_frame = max_t(int, pf_max_frame, IXGBE_FCOE_JUMBO_FRAME_SIZE); #endif /* CONFIG_FCOE */ |