From b7b462454a5c2aed1d8aecd165d1dae9017bd5fd Mon Sep 17 00:00:00 2001 From: Sasha Neftin Date: Thu, 31 Oct 2019 08:06:07 +0200 Subject: igc: Add scatter gather support Scatter gather is used to do DMA data transfers of data that is written to noncontiguous areas of memory. This patch enables scatter gather support. Signed-off-by: Sasha Neftin Tested-by: Aaron Brown Signed-off-by: Jeff Kirsher --- drivers/net/ethernet/intel/igc/igc_main.c | 1 + 1 file changed, 1 insertion(+) (limited to 'drivers/net/ethernet/intel/igc/igc_main.c') diff --git a/drivers/net/ethernet/intel/igc/igc_main.c b/drivers/net/ethernet/intel/igc/igc_main.c index 9700527dd797..75dcaa87d06b 100644 --- a/drivers/net/ethernet/intel/igc/igc_main.c +++ b/drivers/net/ethernet/intel/igc/igc_main.c @@ -4433,6 +4433,7 @@ static int igc_probe(struct pci_dev *pdev, goto err_sw_init; /* Add supported features to the features list*/ + netdev->features |= NETIF_F_SG; netdev->features |= NETIF_F_RXCSUM; netdev->features |= NETIF_F_HW_CSUM; netdev->features |= NETIF_F_SCTP_CRC; -- cgit From 21da01fd3bb1124b46e6b15657de36dffcc58af9 Mon Sep 17 00:00:00 2001 From: Sasha Neftin Date: Tue, 5 Nov 2019 11:44:13 +0200 Subject: igc: Improve the DMA mapping flow Improve the probe flow and set both the DMA mask and the coherent to the same thing. Make the flow optimized and cleared. Signed-off-by: Sasha Neftin Tested-by: Aaron Brown Signed-off-by: Jeff Kirsher --- drivers/net/ethernet/intel/igc/igc_main.c | 24 +++++++++--------------- 1 file changed, 9 insertions(+), 15 deletions(-) (limited to 'drivers/net/ethernet/intel/igc/igc_main.c') diff --git a/drivers/net/ethernet/intel/igc/igc_main.c b/drivers/net/ethernet/intel/igc/igc_main.c index 75dcaa87d06b..91acce2e1d1c 100644 --- a/drivers/net/ethernet/intel/igc/igc_main.c +++ b/drivers/net/ethernet/intel/igc/igc_main.c @@ -4345,32 +4345,26 @@ static int igc_probe(struct pci_dev *pdev, struct net_device *netdev; struct igc_hw *hw; const struct igc_info *ei = igc_info_tbl[ent->driver_data]; - int err; + int err, pci_using_dac; err = pci_enable_device_mem(pdev); if (err) return err; - err = dma_set_mask(&pdev->dev, DMA_BIT_MASK(64)); + pci_using_dac = 0; + err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); if (!err) { - err = dma_set_coherent_mask(&pdev->dev, - DMA_BIT_MASK(64)); + pci_using_dac = 1; } else { - err = dma_set_mask(&pdev->dev, DMA_BIT_MASK(32)); + err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)); if (err) { - err = dma_set_coherent_mask(&pdev->dev, - DMA_BIT_MASK(32)); - if (err) { - dev_err(&pdev->dev, "igc: Wrong DMA config\n"); - goto err_dma; - } + dev_err(&pdev->dev, + "No usable DMA configuration, aborting\n"); + goto err_dma; } } - err = pci_request_selected_regions(pdev, - pci_select_bars(pdev, - IORESOURCE_MEM), - igc_driver_name); + err = pci_request_mem_regions(pdev, igc_driver_name); if (err) goto err_pci_reg; -- cgit From faf4dd52e9e34a36254a6ad43369064b6928d504 Mon Sep 17 00:00:00 2001 From: Sasha Neftin Date: Sun, 10 Nov 2019 16:05:20 +0200 Subject: igc: Prefer to use the pci_release_mem_regions method Use the pci_release_mem_regions method instead of the pci_release_selected_regions method Signed-off-by: Sasha Neftin Tested-by: Aaron Brown Signed-off-by: Jeff Kirsher --- drivers/net/ethernet/intel/igc/igc_main.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'drivers/net/ethernet/intel/igc/igc_main.c') diff --git a/drivers/net/ethernet/intel/igc/igc_main.c b/drivers/net/ethernet/intel/igc/igc_main.c index 91acce2e1d1c..de8e4b570fa6 100644 --- a/drivers/net/ethernet/intel/igc/igc_main.c +++ b/drivers/net/ethernet/intel/igc/igc_main.c @@ -4527,8 +4527,7 @@ err_sw_init: err_ioremap: free_netdev(netdev); err_alloc_etherdev: - pci_release_selected_regions(pdev, - pci_select_bars(pdev, IORESOURCE_MEM)); + pci_release_mem_regions(pdev); err_pci_reg: err_dma: pci_disable_device(pdev); -- cgit From 57cd472c2baa1d1dc7f037d2c6d6d7d29be29d5d Mon Sep 17 00:00:00 2001 From: Sasha Neftin Date: Sun, 10 Nov 2019 17:57:44 +0200 Subject: igc: Remove excess parameter description from igc_is_non_eop The function description for igc_is_non_eop includes an extra @skb parameter description. This parameter doesn't exist on the function, so remove it. Suggested-by: Jacob Keller Signed-off-by: Sasha Neftin Tested-by: Aaron Brown Signed-off-by: Jeff Kirsher --- drivers/net/ethernet/intel/igc/igc_main.c | 1 - 1 file changed, 1 deletion(-) (limited to 'drivers/net/ethernet/intel/igc/igc_main.c') diff --git a/drivers/net/ethernet/intel/igc/igc_main.c b/drivers/net/ethernet/intel/igc/igc_main.c index de8e4b570fa6..ac420a9b4d2b 100644 --- a/drivers/net/ethernet/intel/igc/igc_main.c +++ b/drivers/net/ethernet/intel/igc/igc_main.c @@ -1485,7 +1485,6 @@ static bool igc_can_reuse_rx_page(struct igc_rx_buffer *rx_buffer) * igc_is_non_eop - process handling of non-EOP buffers * @rx_ring: Rx ring being processed * @rx_desc: Rx descriptor for current buffer - * @skb: current socket buffer containing buffer in progress * * This function updates next to clean. If the buffer is an EOP buffer * this function exits returning false, otherwise it will place the -- cgit From 085c858950419f5a7eea351c2e5cb999b4caad2d Mon Sep 17 00:00:00 2001 From: Sasha Neftin Date: Mon, 11 Nov 2019 09:04:12 +0200 Subject: igc: Fix the parameter description for igc_alloc_rx_buffers The function description for igc_alloc_rx_buffers has not reflected the function meaning. Add meaningful description. Signed-off-by: Sasha Neftin Tested-by: Aaron Brown Signed-off-by: Jeff Kirsher --- drivers/net/ethernet/intel/igc/igc_main.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'drivers/net/ethernet/intel/igc/igc_main.c') diff --git a/drivers/net/ethernet/intel/igc/igc_main.c b/drivers/net/ethernet/intel/igc/igc_main.c index ac420a9b4d2b..833770fd16d2 100644 --- a/drivers/net/ethernet/intel/igc/igc_main.c +++ b/drivers/net/ethernet/intel/igc/igc_main.c @@ -1566,7 +1566,8 @@ static void igc_put_rx_buffer(struct igc_ring *rx_ring, /** * igc_alloc_rx_buffers - Replace used receive buffers; packet split - * @adapter: address of board private structure + * @rx_ring: rx descriptor ring + * @cleaned_count: number of buffers to clean */ static void igc_alloc_rx_buffers(struct igc_ring *rx_ring, u16 cleaned_count) { -- cgit From 86efeccd5a7adcfbe12c0ba558615e53a33b67fc Mon Sep 17 00:00:00 2001 From: Sasha Neftin Date: Tue, 12 Nov 2019 17:13:32 +0200 Subject: igc: Fix parameter descriptions for a several functions igc_watchdog, igc_set_interrupt_capability, igc_init_interrupt_scheme, __igc_open and __igc_close parameter descriptions has not reflected functions meaning. Add meaningful description. Signed-off-by: Sasha Neftin Tested-by: Aaron Brown Signed-off-by: Jeff Kirsher --- drivers/net/ethernet/intel/igc/igc_main.c | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) (limited to 'drivers/net/ethernet/intel/igc/igc_main.c') diff --git a/drivers/net/ethernet/intel/igc/igc_main.c b/drivers/net/ethernet/intel/igc/igc_main.c index 833770fd16d2..d3a45d3cbcd9 100644 --- a/drivers/net/ethernet/intel/igc/igc_main.c +++ b/drivers/net/ethernet/intel/igc/igc_main.c @@ -3109,7 +3109,7 @@ bool igc_has_link(struct igc_adapter *adapter) /** * igc_watchdog - Timer Call-back - * @data: pointer to adapter cast into an unsigned long + * @t: timer for the watchdog */ static void igc_watchdog(struct timer_list *t) { @@ -3638,6 +3638,7 @@ static int igc_poll(struct napi_struct *napi, int budget) /** * igc_set_interrupt_capability - set MSI or MSI-X if supported * @adapter: Pointer to adapter structure + * @msix: boolean value for MSI-X capability * * Attempt to configure interrupts using the best available * capabilities of the hardware and kernel. @@ -3906,6 +3907,7 @@ static void igc_cache_ring_register(struct igc_adapter *adapter) /** * igc_init_interrupt_scheme - initialize interrupts, allocate queues/vectors * @adapter: Pointer to adapter structure + * @msix: boolean for MSI-X capability * * This function initializes the interrupts and allocates all of the queues. */ @@ -4073,8 +4075,9 @@ static void igc_write_itr(struct igc_q_vector *q_vector) } /** - * igc_open - Called when a network interface is made active + * __igc_open - Called when a network interface is made active * @netdev: network interface device structure + * @resuming: boolean indicating if the device is resuming * * Returns 0 on success, negative value on failure * @@ -4164,8 +4167,9 @@ static int igc_open(struct net_device *netdev) } /** - * igc_close - Disables a network interface + * __igc_close - Disables a network interface * @netdev: network interface device structure + * @suspending: boolean indicating the device is suspending * * Returns 0, this is not allowed to fail * -- cgit From 4439dc427d83c67aaa8447091c479c482ff85af9 Mon Sep 17 00:00:00 2001 From: Sasha Neftin Date: Wed, 13 Nov 2019 11:27:29 +0200 Subject: igc: Add 64 bit DMA access support On relevant platforms ndo_start_xmit can handle socket buffer fragments in high memory Signed-off-by: Sasha Neftin Tested-by: Aaron Brown Signed-off-by: Jeff Kirsher --- drivers/net/ethernet/intel/igc/igc_main.c | 3 +++ 1 file changed, 3 insertions(+) (limited to 'drivers/net/ethernet/intel/igc/igc_main.c') diff --git a/drivers/net/ethernet/intel/igc/igc_main.c b/drivers/net/ethernet/intel/igc/igc_main.c index d3a45d3cbcd9..b85009837cdf 100644 --- a/drivers/net/ethernet/intel/igc/igc_main.c +++ b/drivers/net/ethernet/intel/igc/igc_main.c @@ -4445,6 +4445,9 @@ static int igc_probe(struct pci_dev *pdev, netdev->hw_features |= NETIF_F_NTUPLE; netdev->hw_features |= netdev->features; + if (pci_using_dac) + netdev->features |= NETIF_F_HIGHDMA; + /* MTU range: 68 - 9216 */ netdev->min_mtu = ETH_MIN_MTU; netdev->max_mtu = MAX_STD_JUMBO_FRAME_SIZE; -- cgit From 9513d2a5dc7f3f2c037bfd2ea35264cc783b954e Mon Sep 17 00:00:00 2001 From: Sasha Neftin Date: Thu, 14 Nov 2019 09:54:46 +0200 Subject: igc: Add legacy power management support Add suspend, resume, runtime_suspend, runtime_resume and runtime_idle callbacks implementation. Reported-by: kbuild test robot Signed-off-by: Sasha Neftin Tested-by: Aaron Brown Signed-off-by: Jeff Kirsher --- drivers/net/ethernet/intel/igc/igc.h | 2 + drivers/net/ethernet/intel/igc/igc_defines.h | 31 ++++ drivers/net/ethernet/intel/igc/igc_main.c | 204 +++++++++++++++++++++++++++ drivers/net/ethernet/intel/igc/igc_regs.h | 9 ++ 4 files changed, 246 insertions(+) (limited to 'drivers/net/ethernet/intel/igc/igc_main.c') diff --git a/drivers/net/ethernet/intel/igc/igc.h b/drivers/net/ethernet/intel/igc/igc.h index 0868677d43ed..612fe9ec81a4 100644 --- a/drivers/net/ethernet/intel/igc/igc.h +++ b/drivers/net/ethernet/intel/igc/igc.h @@ -370,6 +370,8 @@ struct igc_adapter { struct timer_list dma_err_timer; struct timer_list phy_info_timer; + u32 wol; + u32 en_mng_pt; u16 link_speed; u16 link_duplex; diff --git a/drivers/net/ethernet/intel/igc/igc_defines.h b/drivers/net/ethernet/intel/igc/igc_defines.h index f3788f0b95b4..50dffd5db606 100644 --- a/drivers/net/ethernet/intel/igc/igc_defines.h +++ b/drivers/net/ethernet/intel/igc/igc_defines.h @@ -10,6 +10,37 @@ #define IGC_CTRL_EXT_DRV_LOAD 0x10000000 /* Drv loaded bit for FW */ +/* Definitions for power management and wakeup registers */ +/* Wake Up Control */ +#define IGC_WUC_PME_EN 0x00000002 /* PME Enable */ + +/* Wake Up Filter Control */ +#define IGC_WUFC_LNKC 0x00000001 /* Link Status Change Wakeup Enable */ +#define IGC_WUFC_MC 0x00000008 /* Directed Multicast Wakeup Enable */ + +#define IGC_CTRL_ADVD3WUC 0x00100000 /* D3 WUC */ + +/* Wake Up Status */ +#define IGC_WUS_EX 0x00000004 /* Directed Exact */ +#define IGC_WUS_ARPD 0x00000020 /* Directed ARP Request */ +#define IGC_WUS_IPV4 0x00000040 /* Directed IPv4 */ +#define IGC_WUS_IPV6 0x00000080 /* Directed IPv6 */ +#define IGC_WUS_NSD 0x00000400 /* Directed IPv6 Neighbor Solicitation */ + +/* Packet types that are enabled for wake packet delivery */ +#define WAKE_PKT_WUS ( \ + IGC_WUS_EX | \ + IGC_WUS_ARPD | \ + IGC_WUS_IPV4 | \ + IGC_WUS_IPV6 | \ + IGC_WUS_NSD) + +/* Wake Up Packet Length */ +#define IGC_WUPL_MASK 0x00000FFF + +/* Wake Up Packet Memory stores the first 128 bytes of the wake up packet */ +#define IGC_WUPM_BYTES 128 + /* Physical Func Reset Done Indication */ #define IGC_CTRL_EXT_LINK_MODE_MASK 0x00C00000 diff --git a/drivers/net/ethernet/intel/igc/igc_main.c b/drivers/net/ethernet/intel/igc/igc_main.c index b85009837cdf..ca7b8d6791f1 100644 --- a/drivers/net/ethernet/intel/igc/igc_main.c +++ b/drivers/net/ethernet/intel/igc/igc_main.c @@ -8,6 +8,7 @@ #include #include #include +#include #include @@ -4581,11 +4582,214 @@ static void igc_remove(struct pci_dev *pdev) pci_disable_device(pdev); } +static int __igc_shutdown(struct pci_dev *pdev, bool *enable_wake, + bool runtime) +{ + struct net_device *netdev = pci_get_drvdata(pdev); + struct igc_adapter *adapter = netdev_priv(netdev); + u32 wufc = runtime ? IGC_WUFC_LNKC : adapter->wol; + struct igc_hw *hw = &adapter->hw; + u32 ctrl, rctl, status; + bool wake; + + rtnl_lock(); + netif_device_detach(netdev); + + if (netif_running(netdev)) + __igc_close(netdev, true); + + igc_clear_interrupt_scheme(adapter); + rtnl_unlock(); + + status = rd32(IGC_STATUS); + if (status & IGC_STATUS_LU) + wufc &= ~IGC_WUFC_LNKC; + + if (wufc) { + igc_setup_rctl(adapter); + igc_set_rx_mode(netdev); + + /* turn on all-multi mode if wake on multicast is enabled */ + if (wufc & IGC_WUFC_MC) { + rctl = rd32(IGC_RCTL); + rctl |= IGC_RCTL_MPE; + wr32(IGC_RCTL, rctl); + } + + ctrl = rd32(IGC_CTRL); + ctrl |= IGC_CTRL_ADVD3WUC; + wr32(IGC_CTRL, ctrl); + + /* Allow time for pending master requests to run */ + igc_disable_pcie_master(hw); + + wr32(IGC_WUC, IGC_WUC_PME_EN); + wr32(IGC_WUFC, wufc); + } else { + wr32(IGC_WUC, 0); + wr32(IGC_WUFC, 0); + } + + wake = wufc || adapter->en_mng_pt; + if (!wake) + igc_power_down_link(adapter); + else + igc_power_up_link(adapter); + + if (enable_wake) + *enable_wake = wake; + + /* Release control of h/w to f/w. If f/w is AMT enabled, this + * would have already happened in close and is redundant. + */ + igc_release_hw_control(adapter); + + pci_disable_device(pdev); + + return 0; +} + +#ifdef CONFIG_PM +static int __maybe_unused igc_runtime_suspend(struct device *dev) +{ + return __igc_shutdown(to_pci_dev(dev), NULL, 1); +} + +static void igc_deliver_wake_packet(struct net_device *netdev) +{ + struct igc_adapter *adapter = netdev_priv(netdev); + struct igc_hw *hw = &adapter->hw; + struct sk_buff *skb; + u32 wupl; + + wupl = rd32(IGC_WUPL) & IGC_WUPL_MASK; + + /* WUPM stores only the first 128 bytes of the wake packet. + * Read the packet only if we have the whole thing. + */ + if (wupl == 0 || wupl > IGC_WUPM_BYTES) + return; + + skb = netdev_alloc_skb_ip_align(netdev, IGC_WUPM_BYTES); + if (!skb) + return; + + skb_put(skb, wupl); + + /* Ensure reads are 32-bit aligned */ + wupl = roundup(wupl, 4); + + memcpy_fromio(skb->data, hw->hw_addr + IGC_WUPM_REG(0), wupl); + + skb->protocol = eth_type_trans(skb, netdev); + netif_rx(skb); +} + +static int __maybe_unused igc_resume(struct device *dev) +{ + struct pci_dev *pdev = to_pci_dev(dev); + struct net_device *netdev = pci_get_drvdata(pdev); + struct igc_adapter *adapter = netdev_priv(netdev); + struct igc_hw *hw = &adapter->hw; + u32 err, val; + + pci_set_power_state(pdev, PCI_D0); + pci_restore_state(pdev); + pci_save_state(pdev); + + if (!pci_device_is_present(pdev)) + return -ENODEV; + err = pci_enable_device_mem(pdev); + if (err) { + dev_err(&pdev->dev, + "igc: Cannot enable PCI device from suspend\n"); + return err; + } + pci_set_master(pdev); + + pci_enable_wake(pdev, PCI_D3hot, 0); + pci_enable_wake(pdev, PCI_D3cold, 0); + + if (igc_init_interrupt_scheme(adapter, true)) { + dev_err(&pdev->dev, "Unable to allocate memory for queues\n"); + return -ENOMEM; + } + + igc_reset(adapter); + + /* let the f/w know that the h/w is now under the control of the + * driver. + */ + igc_get_hw_control(adapter); + + val = rd32(IGC_WUS); + if (val & WAKE_PKT_WUS) + igc_deliver_wake_packet(netdev); + + wr32(IGC_WUS, ~0); + + rtnl_lock(); + if (!err && netif_running(netdev)) + err = __igc_open(netdev, true); + + if (!err) + netif_device_attach(netdev); + rtnl_unlock(); + + return err; +} + +static int __maybe_unused igc_runtime_resume(struct device *dev) +{ + return igc_resume(dev); +} + +static int __maybe_unused igc_suspend(struct device *dev) +{ + return __igc_shutdown(to_pci_dev(dev), NULL, 0); +} + +static int __maybe_unused igc_runtime_idle(struct device *dev) +{ + struct net_device *netdev = dev_get_drvdata(dev); + struct igc_adapter *adapter = netdev_priv(netdev); + + if (!igc_has_link(adapter)) + pm_schedule_suspend(dev, MSEC_PER_SEC * 5); + + return -EBUSY; +} +#endif /* CONFIG_PM */ + +static void igc_shutdown(struct pci_dev *pdev) +{ + bool wake; + + __igc_shutdown(pdev, &wake, 0); + + if (system_state == SYSTEM_POWER_OFF) { + pci_wake_from_d3(pdev, wake); + pci_set_power_state(pdev, PCI_D3hot); + } +} + +#ifdef CONFIG_PM +static const struct dev_pm_ops igc_pm_ops = { + SET_SYSTEM_SLEEP_PM_OPS(igc_suspend, igc_resume) + SET_RUNTIME_PM_OPS(igc_runtime_suspend, igc_runtime_resume, + igc_runtime_idle) +}; +#endif + static struct pci_driver igc_driver = { .name = igc_driver_name, .id_table = igc_pci_tbl, .probe = igc_probe, .remove = igc_remove, +#ifdef CONFIG_PM + .driver.pm = &igc_pm_ops, +#endif + .shutdown = igc_shutdown, }; void igc_set_flag_queue_pairs(struct igc_adapter *adapter, diff --git a/drivers/net/ethernet/intel/igc/igc_regs.h b/drivers/net/ethernet/intel/igc/igc_regs.h index 50d7c04dccf5..93a9139f08c5 100644 --- a/drivers/net/ethernet/intel/igc/igc_regs.h +++ b/drivers/net/ethernet/intel/igc/igc_regs.h @@ -215,6 +215,15 @@ /* Shadow Ram Write Register - RW */ #define IGC_SRWR 0x12018 +/* Wake Up registers */ +#define IGC_WUC 0x05800 /* Wakeup Control - RW */ +#define IGC_WUFC 0x05808 /* Wakeup Filter Control - RW */ +#define IGC_WUS 0x05810 /* Wakeup Status - R/W1C */ +#define IGC_WUPL 0x05900 /* Wakeup Packet Length - RW */ + +/* Wake Up packet memory */ +#define IGC_WUPM_REG(_i) (0x05A00 + ((_i) * 4)) + /* forward declaration */ struct igc_hw; u32 igc_rd32(struct igc_hw *hw, u32 reg); -- cgit From 684ea87cc312c98386c667d1046c61eb92ea8379 Mon Sep 17 00:00:00 2001 From: Sasha Neftin Date: Mon, 2 Dec 2019 09:56:26 +0200 Subject: igc: Remove serdes comments from a description of methods Serdes interface is not applicable for i225 devices. Remove this from comments and make comments more clearly. Signed-off-by: Sasha Neftin Tested-by: Aaron Brown Signed-off-by: Jeff Kirsher --- drivers/net/ethernet/intel/igc/igc_main.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'drivers/net/ethernet/intel/igc/igc_main.c') diff --git a/drivers/net/ethernet/intel/igc/igc_main.c b/drivers/net/ethernet/intel/igc/igc_main.c index ca7b8d6791f1..25989c087aca 100644 --- a/drivers/net/ethernet/intel/igc/igc_main.c +++ b/drivers/net/ethernet/intel/igc/igc_main.c @@ -115,7 +115,7 @@ void igc_reset(struct igc_adapter *adapter) } /** - * igc_power_up_link - Power up the phy/serdes link + * igc_power_up_link - Power up the phy link * @adapter: address of board private structure */ static void igc_power_up_link(struct igc_adapter *adapter) @@ -129,7 +129,7 @@ static void igc_power_up_link(struct igc_adapter *adapter) } /** - * igc_power_down_link - Power down the phy/serdes link + * igc_power_down_link - Power down the phy link * @adapter: address of board private structure */ static void igc_power_down_link(struct igc_adapter *adapter) -- cgit From 14504ac5048e1773da16e741b2e659c8b7257592 Mon Sep 17 00:00:00 2001 From: Sasha Neftin Date: Tue, 19 Nov 2019 13:43:32 +0200 Subject: igc: Remove no need declaration of the igc_clean_tx_ring We want to avoid forward-declarations of function if possible. Rearrange the igc_clean_tx_ring function implementation. Signed-off-by: Sasha Neftin Tested-by: Aaron Brown Signed-off-by: Jeff Kirsher --- drivers/net/ethernet/intel/igc/igc_main.c | 75 +++++++++++++++---------------- 1 file changed, 37 insertions(+), 38 deletions(-) (limited to 'drivers/net/ethernet/intel/igc/igc_main.c') diff --git a/drivers/net/ethernet/intel/igc/igc_main.c b/drivers/net/ethernet/intel/igc/igc_main.c index 25989c087aca..c156cd68532b 100644 --- a/drivers/net/ethernet/intel/igc/igc_main.c +++ b/drivers/net/ethernet/intel/igc/igc_main.c @@ -52,7 +52,6 @@ static const struct pci_device_id igc_pci_tbl[] = { MODULE_DEVICE_TABLE(pci, igc_pci_tbl); /* forward declaration */ -static void igc_clean_tx_ring(struct igc_ring *tx_ring); static int igc_sw_init(struct igc_adapter *); static void igc_configure(struct igc_adapter *adapter); static void igc_power_down_link(struct igc_adapter *adapter); @@ -176,43 +175,6 @@ static void igc_get_hw_control(struct igc_adapter *adapter) ctrl_ext | IGC_CTRL_EXT_DRV_LOAD); } -/** - * igc_free_tx_resources - Free Tx Resources per Queue - * @tx_ring: Tx descriptor ring for a specific queue - * - * Free all transmit software resources - */ -void igc_free_tx_resources(struct igc_ring *tx_ring) -{ - igc_clean_tx_ring(tx_ring); - - vfree(tx_ring->tx_buffer_info); - tx_ring->tx_buffer_info = NULL; - - /* if not set, then don't free */ - if (!tx_ring->desc) - return; - - dma_free_coherent(tx_ring->dev, tx_ring->size, - tx_ring->desc, tx_ring->dma); - - tx_ring->desc = NULL; -} - -/** - * igc_free_all_tx_resources - Free Tx Resources for All Queues - * @adapter: board private structure - * - * Free all transmit software resources - */ -static void igc_free_all_tx_resources(struct igc_adapter *adapter) -{ - int i; - - for (i = 0; i < adapter->num_tx_queues; i++) - igc_free_tx_resources(adapter->tx_ring[i]); -} - /** * igc_clean_tx_ring - Free Tx Buffers * @tx_ring: ring to be cleaned @@ -274,6 +236,43 @@ static void igc_clean_tx_ring(struct igc_ring *tx_ring) tx_ring->next_to_clean = 0; } +/** + * igc_free_tx_resources - Free Tx Resources per Queue + * @tx_ring: Tx descriptor ring for a specific queue + * + * Free all transmit software resources + */ +void igc_free_tx_resources(struct igc_ring *tx_ring) +{ + igc_clean_tx_ring(tx_ring); + + vfree(tx_ring->tx_buffer_info); + tx_ring->tx_buffer_info = NULL; + + /* if not set, then don't free */ + if (!tx_ring->desc) + return; + + dma_free_coherent(tx_ring->dev, tx_ring->size, + tx_ring->desc, tx_ring->dma); + + tx_ring->desc = NULL; +} + +/** + * igc_free_all_tx_resources - Free Tx Resources for All Queues + * @adapter: board private structure + * + * Free all transmit software resources + */ +static void igc_free_all_tx_resources(struct igc_adapter *adapter) +{ + int i; + + for (i = 0; i < adapter->num_tx_queues; i++) + igc_free_tx_resources(adapter->tx_ring[i]); +} + /** * igc_clean_all_tx_rings - Free Tx Buffers for all queues * @adapter: board private structure -- cgit From 1ab69fb357099d7bd869cf9332f0cb137640c6a4 Mon Sep 17 00:00:00 2001 From: Sasha Neftin Date: Tue, 19 Nov 2019 13:43:41 +0200 Subject: igc: Remove no need declaration of the igc_power_down_link We want to avoid forward-declarations of function if possible. Rearrange the igc_power_down_link function implementation. Signed-off-by: Sasha Neftin Tested-by: Aaron Brown Signed-off-by: Jeff Kirsher --- drivers/net/ethernet/intel/igc/igc_main.c | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) (limited to 'drivers/net/ethernet/intel/igc/igc_main.c') diff --git a/drivers/net/ethernet/intel/igc/igc_main.c b/drivers/net/ethernet/intel/igc/igc_main.c index c156cd68532b..17b83638efc9 100644 --- a/drivers/net/ethernet/intel/igc/igc_main.c +++ b/drivers/net/ethernet/intel/igc/igc_main.c @@ -54,7 +54,6 @@ MODULE_DEVICE_TABLE(pci, igc_pci_tbl); /* forward declaration */ static int igc_sw_init(struct igc_adapter *); static void igc_configure(struct igc_adapter *adapter); -static void igc_power_down_link(struct igc_adapter *adapter); static void igc_set_default_mac_filter(struct igc_adapter *adapter); static void igc_set_rx_mode(struct net_device *netdev); static void igc_write_itr(struct igc_q_vector *q_vector); @@ -76,6 +75,16 @@ enum latency_range { latency_invalid = 255 }; +/** + * igc_power_down_link - Power down the phy/serdes link + * @adapter: address of board private structure + */ +static void igc_power_down_link(struct igc_adapter *adapter) +{ + if (adapter->hw.phy.media_type == igc_media_type_copper) + igc_power_down_phy_copper_base(&adapter->hw); +} + void igc_reset(struct igc_adapter *adapter) { struct pci_dev *pdev = adapter->pdev; @@ -127,16 +136,6 @@ static void igc_power_up_link(struct igc_adapter *adapter) igc_setup_link(&adapter->hw); } -/** - * igc_power_down_link - Power down the phy link - * @adapter: address of board private structure - */ -static void igc_power_down_link(struct igc_adapter *adapter) -{ - if (adapter->hw.phy.media_type == igc_media_type_copper) - igc_power_down_phy_copper_base(&adapter->hw); -} - /** * igc_release_hw_control - release control of the h/w to f/w * @adapter: address of board private structure -- cgit From 3988d8bfd71395b838ed0a887f7b8c2392fe6082 Mon Sep 17 00:00:00 2001 From: Sasha Neftin Date: Tue, 19 Nov 2019 13:43:51 +0200 Subject: igc: Remove no need declaration of the igc_set_default_mac_filter We want to avoid forward-declarations of function if possible. Rearrange the igc_set_default_mac_filter function implementation. Signed-off-by: Sasha Neftin Tested-by: Aaron Brown Signed-off-by: Jeff Kirsher --- drivers/net/ethernet/intel/igc/igc_main.c | 91 +++++++++++++++---------------- 1 file changed, 45 insertions(+), 46 deletions(-) (limited to 'drivers/net/ethernet/intel/igc/igc_main.c') diff --git a/drivers/net/ethernet/intel/igc/igc_main.c b/drivers/net/ethernet/intel/igc/igc_main.c index 17b83638efc9..95a62f838668 100644 --- a/drivers/net/ethernet/intel/igc/igc_main.c +++ b/drivers/net/ethernet/intel/igc/igc_main.c @@ -54,7 +54,6 @@ MODULE_DEVICE_TABLE(pci, igc_pci_tbl); /* forward declaration */ static int igc_sw_init(struct igc_adapter *); static void igc_configure(struct igc_adapter *adapter); -static void igc_set_default_mac_filter(struct igc_adapter *adapter); static void igc_set_rx_mode(struct net_device *netdev); static void igc_write_itr(struct igc_q_vector *q_vector); static void igc_assign_vector(struct igc_q_vector *q_vector, int msix_vector); @@ -769,6 +768,51 @@ static void igc_setup_tctl(struct igc_adapter *adapter) wr32(IGC_TCTL, tctl); } +/** + * igc_rar_set_index - Sync RAL[index] and RAH[index] registers with MAC table + * @adapter: address of board private structure + * @index: Index of the RAR entry which need to be synced with MAC table + */ +static void igc_rar_set_index(struct igc_adapter *adapter, u32 index) +{ + u8 *addr = adapter->mac_table[index].addr; + struct igc_hw *hw = &adapter->hw; + u32 rar_low, rar_high; + + /* HW expects these to be in network order when they are plugged + * into the registers which are little endian. In order to guarantee + * that ordering we need to do an leXX_to_cpup here in order to be + * ready for the byteswap that occurs with writel + */ + rar_low = le32_to_cpup((__le32 *)(addr)); + rar_high = le16_to_cpup((__le16 *)(addr + 4)); + + /* Indicate to hardware the Address is Valid. */ + if (adapter->mac_table[index].state & IGC_MAC_STATE_IN_USE) { + if (is_valid_ether_addr(addr)) + rar_high |= IGC_RAH_AV; + + rar_high |= IGC_RAH_POOL_1 << + adapter->mac_table[index].queue; + } + + wr32(IGC_RAL(index), rar_low); + wrfl(); + wr32(IGC_RAH(index), rar_high); + wrfl(); +} + +/* Set default MAC address for the PF in the first RAR entry */ +static void igc_set_default_mac_filter(struct igc_adapter *adapter) +{ + struct igc_mac_addr *mac_table = &adapter->mac_table[0]; + + ether_addr_copy(mac_table->addr, adapter->hw.mac.addr); + mac_table->state = IGC_MAC_STATE_DEFAULT | IGC_MAC_STATE_IN_USE; + + igc_rar_set_index(adapter, 0); +} + /** * igc_set_mac - Change the Ethernet Address of the NIC * @netdev: network interface device structure @@ -2430,51 +2474,6 @@ static void igc_configure(struct igc_adapter *adapter) } } -/** - * igc_rar_set_index - Sync RAL[index] and RAH[index] registers with MAC table - * @adapter: address of board private structure - * @index: Index of the RAR entry which need to be synced with MAC table - */ -static void igc_rar_set_index(struct igc_adapter *adapter, u32 index) -{ - u8 *addr = adapter->mac_table[index].addr; - struct igc_hw *hw = &adapter->hw; - u32 rar_low, rar_high; - - /* HW expects these to be in network order when they are plugged - * into the registers which are little endian. In order to guarantee - * that ordering we need to do an leXX_to_cpup here in order to be - * ready for the byteswap that occurs with writel - */ - rar_low = le32_to_cpup((__le32 *)(addr)); - rar_high = le16_to_cpup((__le16 *)(addr + 4)); - - /* Indicate to hardware the Address is Valid. */ - if (adapter->mac_table[index].state & IGC_MAC_STATE_IN_USE) { - if (is_valid_ether_addr(addr)) - rar_high |= IGC_RAH_AV; - - rar_high |= IGC_RAH_POOL_1 << - adapter->mac_table[index].queue; - } - - wr32(IGC_RAL(index), rar_low); - wrfl(); - wr32(IGC_RAH(index), rar_high); - wrfl(); -} - -/* Set default MAC address for the PF in the first RAR entry */ -static void igc_set_default_mac_filter(struct igc_adapter *adapter) -{ - struct igc_mac_addr *mac_table = &adapter->mac_table[0]; - - ether_addr_copy(mac_table->addr, adapter->hw.mac.addr); - mac_table->state = IGC_MAC_STATE_DEFAULT | IGC_MAC_STATE_IN_USE; - - igc_rar_set_index(adapter, 0); -} - /* If the filter to be added and an already existing filter express * the same address and address type, it should be possible to only * override the other configurations, for example the queue to steer -- cgit From 1a7c0f2efd5559408c01c22474127d95274637a8 Mon Sep 17 00:00:00 2001 From: Sasha Neftin Date: Tue, 19 Nov 2019 13:44:01 +0200 Subject: igc: Remove no need declaration of the igc_configure We want to avoid forward-declarations of function if possible. Rearrange the igc_configure function implementation. Signed-off-by: Sasha Neftin Tested-by: Aaron Brown Signed-off-by: Jeff Kirsher --- drivers/net/ethernet/intel/igc/igc_main.c | 91 +++++++++++++++---------------- 1 file changed, 45 insertions(+), 46 deletions(-) (limited to 'drivers/net/ethernet/intel/igc/igc_main.c') diff --git a/drivers/net/ethernet/intel/igc/igc_main.c b/drivers/net/ethernet/intel/igc/igc_main.c index 95a62f838668..1023c9226a8e 100644 --- a/drivers/net/ethernet/intel/igc/igc_main.c +++ b/drivers/net/ethernet/intel/igc/igc_main.c @@ -53,7 +53,6 @@ MODULE_DEVICE_TABLE(pci, igc_pci_tbl); /* forward declaration */ static int igc_sw_init(struct igc_adapter *); -static void igc_configure(struct igc_adapter *adapter); static void igc_set_rx_mode(struct net_device *netdev); static void igc_write_itr(struct igc_q_vector *q_vector); static void igc_assign_vector(struct igc_q_vector *q_vector, int msix_vector); @@ -1985,6 +1984,51 @@ static bool igc_clean_tx_irq(struct igc_q_vector *q_vector, int napi_budget) return !!budget; } +static void igc_nfc_filter_restore(struct igc_adapter *adapter) +{ + struct igc_nfc_filter *rule; + + spin_lock(&adapter->nfc_lock); + + hlist_for_each_entry(rule, &adapter->nfc_filter_list, nfc_node) + igc_add_filter(adapter, rule); + + spin_unlock(&adapter->nfc_lock); +} + +/** + * igc_configure - configure the hardware for RX and TX + * @adapter: private board structure + */ +static void igc_configure(struct igc_adapter *adapter) +{ + struct net_device *netdev = adapter->netdev; + int i = 0; + + igc_get_hw_control(adapter); + igc_set_rx_mode(netdev); + + igc_setup_tctl(adapter); + igc_setup_mrqc(adapter); + igc_setup_rctl(adapter); + + igc_nfc_filter_restore(adapter); + igc_configure_tx(adapter); + igc_configure_rx(adapter); + + igc_rx_fifo_flush_base(&adapter->hw); + + /* call igc_desc_unused which always leaves + * at least 1 descriptor unused to make sure + * next_to_use != next_to_clean + */ + for (i = 0; i < adapter->num_rx_queues; i++) { + struct igc_ring *ring = adapter->rx_ring[i]; + + igc_alloc_rx_buffers(ring, igc_desc_unused(ring)); + } +} + /** * igc_up - Open the interface and prepare it to handle traffic * @adapter: board private structure @@ -2206,18 +2250,6 @@ static void igc_nfc_filter_exit(struct igc_adapter *adapter) spin_unlock(&adapter->nfc_lock); } -static void igc_nfc_filter_restore(struct igc_adapter *adapter) -{ - struct igc_nfc_filter *rule; - - spin_lock(&adapter->nfc_lock); - - hlist_for_each_entry(rule, &adapter->nfc_filter_list, nfc_node) - igc_add_filter(adapter, rule); - - spin_unlock(&adapter->nfc_lock); -} - /** * igc_down - Close the interface * @adapter: board private structure @@ -2441,39 +2473,6 @@ igc_features_check(struct sk_buff *skb, struct net_device *dev, return features; } -/** - * igc_configure - configure the hardware for RX and TX - * @adapter: private board structure - */ -static void igc_configure(struct igc_adapter *adapter) -{ - struct net_device *netdev = adapter->netdev; - int i = 0; - - igc_get_hw_control(adapter); - igc_set_rx_mode(netdev); - - igc_setup_tctl(adapter); - igc_setup_mrqc(adapter); - igc_setup_rctl(adapter); - - igc_nfc_filter_restore(adapter); - igc_configure_tx(adapter); - igc_configure_rx(adapter); - - igc_rx_fifo_flush_base(&adapter->hw); - - /* call igc_desc_unused which always leaves - * at least 1 descriptor unused to make sure - * next_to_use != next_to_clean - */ - for (i = 0; i < adapter->num_rx_queues; i++) { - struct igc_ring *ring = adapter->rx_ring[i]; - - igc_alloc_rx_buffers(ring, igc_desc_unused(ring)); - } -} - /* If the filter to be added and an already existing filter express * the same address and address type, it should be possible to only * override the other configurations, for example the queue to steer -- cgit From aac8f68c5687c3b3a592ae05f66a78837430330e Mon Sep 17 00:00:00 2001 From: Sasha Neftin Date: Tue, 19 Nov 2019 13:44:11 +0200 Subject: igc: Remove no need declaration of the igc_alloc_mapped_page We want to avoid forward-declarations of function if possible. Rearrange the igc_alloc_mapped_page function implementation. Signed-off-by: Sasha Neftin Tested-by: Aaron Brown Signed-off-by: Jeff Kirsher --- drivers/net/ethernet/intel/igc/igc_main.c | 94 +++++++++++++++---------------- 1 file changed, 46 insertions(+), 48 deletions(-) (limited to 'drivers/net/ethernet/intel/igc/igc_main.c') diff --git a/drivers/net/ethernet/intel/igc/igc_main.c b/drivers/net/ethernet/intel/igc/igc_main.c index 1023c9226a8e..4ad06952056b 100644 --- a/drivers/net/ethernet/intel/igc/igc_main.c +++ b/drivers/net/ethernet/intel/igc/igc_main.c @@ -63,8 +63,6 @@ static void igc_free_q_vectors(struct igc_adapter *adapter); static void igc_irq_disable(struct igc_adapter *adapter); static void igc_irq_enable(struct igc_adapter *adapter); static void igc_configure_msix(struct igc_adapter *adapter); -static bool igc_alloc_mapped_page(struct igc_ring *rx_ring, - struct igc_rx_buffer *bi); enum latency_range { lowest_latency = 0, @@ -1606,6 +1604,52 @@ static void igc_put_rx_buffer(struct igc_ring *rx_ring, rx_buffer->page = NULL; } +static inline unsigned int igc_rx_offset(struct igc_ring *rx_ring) +{ + return ring_uses_build_skb(rx_ring) ? IGC_SKB_PAD : 0; +} + +static bool igc_alloc_mapped_page(struct igc_ring *rx_ring, + struct igc_rx_buffer *bi) +{ + struct page *page = bi->page; + dma_addr_t dma; + + /* since we are recycling buffers we should seldom need to alloc */ + if (likely(page)) + return true; + + /* alloc new page for storage */ + page = dev_alloc_pages(igc_rx_pg_order(rx_ring)); + if (unlikely(!page)) { + rx_ring->rx_stats.alloc_failed++; + return false; + } + + /* map page for use */ + dma = dma_map_page_attrs(rx_ring->dev, page, 0, + igc_rx_pg_size(rx_ring), + DMA_FROM_DEVICE, + IGC_RX_DMA_ATTR); + + /* if mapping failed free memory back to system since + * there isn't much point in holding memory we can't use + */ + if (dma_mapping_error(rx_ring->dev, dma)) { + __free_page(page); + + rx_ring->rx_stats.alloc_failed++; + return false; + } + + bi->dma = dma; + bi->page = page; + bi->page_offset = igc_rx_offset(rx_ring); + bi->pagecnt_bias = 1; + + return true; +} + /** * igc_alloc_rx_buffers - Replace used receive buffers; packet split * @rx_ring: rx descriptor ring @@ -1767,52 +1811,6 @@ static int igc_clean_rx_irq(struct igc_q_vector *q_vector, const int budget) return total_packets; } -static inline unsigned int igc_rx_offset(struct igc_ring *rx_ring) -{ - return ring_uses_build_skb(rx_ring) ? IGC_SKB_PAD : 0; -} - -static bool igc_alloc_mapped_page(struct igc_ring *rx_ring, - struct igc_rx_buffer *bi) -{ - struct page *page = bi->page; - dma_addr_t dma; - - /* since we are recycling buffers we should seldom need to alloc */ - if (likely(page)) - return true; - - /* alloc new page for storage */ - page = dev_alloc_pages(igc_rx_pg_order(rx_ring)); - if (unlikely(!page)) { - rx_ring->rx_stats.alloc_failed++; - return false; - } - - /* map page for use */ - dma = dma_map_page_attrs(rx_ring->dev, page, 0, - igc_rx_pg_size(rx_ring), - DMA_FROM_DEVICE, - IGC_RX_DMA_ATTR); - - /* if mapping failed free memory back to system since - * there isn't much point in holding memory we can't use - */ - if (dma_mapping_error(rx_ring->dev, dma)) { - __free_page(page); - - rx_ring->rx_stats.alloc_failed++; - return false; - } - - bi->dma = dma; - bi->page = page; - bi->page_offset = igc_rx_offset(rx_ring); - bi->pagecnt_bias = 1; - - return true; -} - /** * igc_clean_tx_irq - Reclaim resources after transmit completes * @q_vector: pointer to q_vector containing needed info -- cgit From 0411d368bc8a43f33f2e012b19304cc9df8f03ba Mon Sep 17 00:00:00 2001 From: Sasha Neftin Date: Tue, 19 Nov 2019 13:44:19 +0200 Subject: igc: Remove no need declaration of the igc_set_interrupt_capability We want to avoid forward-declarations of function if possible. Rearrange the igc_set_interrupt_capability function implementation. Signed-off-by: Sasha Neftin Tested-by: Aaron Brown Signed-off-by: Jeff Kirsher --- drivers/net/ethernet/intel/igc/igc_main.c | 142 +++++++++++++++--------------- 1 file changed, 70 insertions(+), 72 deletions(-) (limited to 'drivers/net/ethernet/intel/igc/igc_main.c') diff --git a/drivers/net/ethernet/intel/igc/igc_main.c b/drivers/net/ethernet/intel/igc/igc_main.c index 4ad06952056b..452e26202c9e 100644 --- a/drivers/net/ethernet/intel/igc/igc_main.c +++ b/drivers/net/ethernet/intel/igc/igc_main.c @@ -57,8 +57,6 @@ static void igc_set_rx_mode(struct net_device *netdev); static void igc_write_itr(struct igc_q_vector *q_vector); static void igc_assign_vector(struct igc_q_vector *q_vector, int msix_vector); static void igc_free_q_vector(struct igc_adapter *adapter, int v_idx); -static void igc_set_interrupt_capability(struct igc_adapter *adapter, - bool msix); static void igc_free_q_vectors(struct igc_adapter *adapter); static void igc_irq_disable(struct igc_adapter *adapter); static void igc_irq_enable(struct igc_adapter *adapter); @@ -2998,6 +2996,76 @@ static void igc_reset_interrupt_capability(struct igc_adapter *adapter) igc_reset_q_vector(adapter, v_idx); } +/** + * igc_set_interrupt_capability - set MSI or MSI-X if supported + * @adapter: Pointer to adapter structure + * @msix: boolean value for MSI-X capability + * + * Attempt to configure interrupts using the best available + * capabilities of the hardware and kernel. + */ +static void igc_set_interrupt_capability(struct igc_adapter *adapter, + bool msix) +{ + int numvecs, i; + int err; + + if (!msix) + goto msi_only; + adapter->flags |= IGC_FLAG_HAS_MSIX; + + /* Number of supported queues. */ + adapter->num_rx_queues = adapter->rss_queues; + + adapter->num_tx_queues = adapter->rss_queues; + + /* start with one vector for every Rx queue */ + numvecs = adapter->num_rx_queues; + + /* if Tx handler is separate add 1 for every Tx queue */ + if (!(adapter->flags & IGC_FLAG_QUEUE_PAIRS)) + numvecs += adapter->num_tx_queues; + + /* store the number of vectors reserved for queues */ + adapter->num_q_vectors = numvecs; + + /* add 1 vector for link status interrupts */ + numvecs++; + + adapter->msix_entries = kcalloc(numvecs, sizeof(struct msix_entry), + GFP_KERNEL); + + if (!adapter->msix_entries) + return; + + /* populate entry values */ + for (i = 0; i < numvecs; i++) + adapter->msix_entries[i].entry = i; + + err = pci_enable_msix_range(adapter->pdev, + adapter->msix_entries, + numvecs, + numvecs); + if (err > 0) + return; + + kfree(adapter->msix_entries); + adapter->msix_entries = NULL; + + igc_reset_interrupt_capability(adapter); + +msi_only: + adapter->flags &= ~IGC_FLAG_HAS_MSIX; + + adapter->rss_queues = 1; + adapter->flags |= IGC_FLAG_QUEUE_PAIRS; + adapter->num_rx_queues = 1; + adapter->num_tx_queues = 1; + adapter->num_q_vectors = 1; + if (!pci_enable_msi(adapter->pdev)) + adapter->flags |= IGC_FLAG_HAS_MSI; +} + /** * igc_clear_interrupt_scheme - reset the device to a state of no interrupts * @adapter: Pointer to adapter structure @@ -3630,76 +3698,6 @@ static int igc_poll(struct napi_struct *napi, int budget) return min(work_done, budget - 1); } -/** - * igc_set_interrupt_capability - set MSI or MSI-X if supported - * @adapter: Pointer to adapter structure - * @msix: boolean value for MSI-X capability - * - * Attempt to configure interrupts using the best available - * capabilities of the hardware and kernel. - */ -static void igc_set_interrupt_capability(struct igc_adapter *adapter, - bool msix) -{ - int numvecs, i; - int err; - - if (!msix) - goto msi_only; - adapter->flags |= IGC_FLAG_HAS_MSIX; - - /* Number of supported queues. */ - adapter->num_rx_queues = adapter->rss_queues; - - adapter->num_tx_queues = adapter->rss_queues; - - /* start with one vector for every Rx queue */ - numvecs = adapter->num_rx_queues; - - /* if Tx handler is separate add 1 for every Tx queue */ - if (!(adapter->flags & IGC_FLAG_QUEUE_PAIRS)) - numvecs += adapter->num_tx_queues; - - /* store the number of vectors reserved for queues */ - adapter->num_q_vectors = numvecs; - - /* add 1 vector for link status interrupts */ - numvecs++; - - adapter->msix_entries = kcalloc(numvecs, sizeof(struct msix_entry), - GFP_KERNEL); - - if (!adapter->msix_entries) - return; - - /* populate entry values */ - for (i = 0; i < numvecs; i++) - adapter->msix_entries[i].entry = i; - - err = pci_enable_msix_range(adapter->pdev, - adapter->msix_entries, - numvecs, - numvecs); - if (err > 0) - return; - - kfree(adapter->msix_entries); - adapter->msix_entries = NULL; - - igc_reset_interrupt_capability(adapter); - -msi_only: - adapter->flags &= ~IGC_FLAG_HAS_MSIX; - - adapter->rss_queues = 1; - adapter->flags |= IGC_FLAG_QUEUE_PAIRS; - adapter->num_rx_queues = 1; - adapter->num_tx_queues = 1; - adapter->num_q_vectors = 1; - if (!pci_enable_msi(adapter->pdev)) - adapter->flags |= IGC_FLAG_HAS_MSI; -} - static void igc_add_ring(struct igc_ring *ring, struct igc_ring_container *head) { -- cgit From 86a4de661b7bb27f208c6c327870bcdbbe9cb152 Mon Sep 17 00:00:00 2001 From: Sasha Neftin Date: Tue, 19 Nov 2019 13:44:29 +0200 Subject: igc: Remove no need declaration of the igc_set_rx_mode We want to avoid forward-declarations of function if possible. Rearrange the igc_set_rx_mode function implementation. Signed-off-by: Sasha Neftin Tested-by: Aaron Brown Signed-off-by: Jeff Kirsher --- drivers/net/ethernet/intel/igc/igc_main.c | 353 +++++++++++++++--------------- 1 file changed, 176 insertions(+), 177 deletions(-) (limited to 'drivers/net/ethernet/intel/igc/igc_main.c') diff --git a/drivers/net/ethernet/intel/igc/igc_main.c b/drivers/net/ethernet/intel/igc/igc_main.c index 452e26202c9e..13491ad7d251 100644 --- a/drivers/net/ethernet/intel/igc/igc_main.c +++ b/drivers/net/ethernet/intel/igc/igc_main.c @@ -53,7 +53,6 @@ MODULE_DEVICE_TABLE(pci, igc_pci_tbl); /* forward declaration */ static int igc_sw_init(struct igc_adapter *); -static void igc_set_rx_mode(struct net_device *netdev); static void igc_write_itr(struct igc_q_vector *q_vector); static void igc_assign_vector(struct igc_q_vector *q_vector, int msix_vector); static void igc_free_q_vector(struct igc_adapter *adapter, int v_idx); @@ -1992,6 +1991,182 @@ static void igc_nfc_filter_restore(struct igc_adapter *adapter) spin_unlock(&adapter->nfc_lock); } +/* If the filter to be added and an already existing filter express + * the same address and address type, it should be possible to only + * override the other configurations, for example the queue to steer + * traffic. + */ +static bool igc_mac_entry_can_be_used(const struct igc_mac_addr *entry, + const u8 *addr, const u8 flags) +{ + if (!(entry->state & IGC_MAC_STATE_IN_USE)) + return true; + + if ((entry->state & IGC_MAC_STATE_SRC_ADDR) != + (flags & IGC_MAC_STATE_SRC_ADDR)) + return false; + + if (!ether_addr_equal(addr, entry->addr)) + return false; + + return true; +} + +/* Add a MAC filter for 'addr' directing matching traffic to 'queue', + * 'flags' is used to indicate what kind of match is made, match is by + * default for the destination address, if matching by source address + * is desired the flag IGC_MAC_STATE_SRC_ADDR can be used. + */ +static int igc_add_mac_filter(struct igc_adapter *adapter, + const u8 *addr, const u8 queue) +{ + struct igc_hw *hw = &adapter->hw; + int rar_entries = hw->mac.rar_entry_count; + int i; + + if (is_zero_ether_addr(addr)) + return -EINVAL; + + /* Search for the first empty entry in the MAC table. + * Do not touch entries at the end of the table reserved for the VF MAC + * addresses. + */ + for (i = 0; i < rar_entries; i++) { + if (!igc_mac_entry_can_be_used(&adapter->mac_table[i], + addr, 0)) + continue; + + ether_addr_copy(adapter->mac_table[i].addr, addr); + adapter->mac_table[i].queue = queue; + adapter->mac_table[i].state |= IGC_MAC_STATE_IN_USE; + + igc_rar_set_index(adapter, i); + return i; + } + + return -ENOSPC; +} + +/* Remove a MAC filter for 'addr' directing matching traffic to + * 'queue', 'flags' is used to indicate what kind of match need to be + * removed, match is by default for the destination address, if + * matching by source address is to be removed the flag + * IGC_MAC_STATE_SRC_ADDR can be used. + */ +static int igc_del_mac_filter(struct igc_adapter *adapter, + const u8 *addr, const u8 queue) +{ + struct igc_hw *hw = &adapter->hw; + int rar_entries = hw->mac.rar_entry_count; + int i; + + if (is_zero_ether_addr(addr)) + return -EINVAL; + + /* Search for matching entry in the MAC table based on given address + * and queue. Do not touch entries at the end of the table reserved + * for the VF MAC addresses. + */ + for (i = 0; i < rar_entries; i++) { + if (!(adapter->mac_table[i].state & IGC_MAC_STATE_IN_USE)) + continue; + if (adapter->mac_table[i].state != 0) + continue; + if (adapter->mac_table[i].queue != queue) + continue; + if (!ether_addr_equal(adapter->mac_table[i].addr, addr)) + continue; + + /* When a filter for the default address is "deleted", + * we return it to its initial configuration + */ + if (adapter->mac_table[i].state & IGC_MAC_STATE_DEFAULT) { + adapter->mac_table[i].state = + IGC_MAC_STATE_DEFAULT | IGC_MAC_STATE_IN_USE; + adapter->mac_table[i].queue = 0; + } else { + adapter->mac_table[i].state = 0; + adapter->mac_table[i].queue = 0; + memset(adapter->mac_table[i].addr, 0, ETH_ALEN); + } + + igc_rar_set_index(adapter, i); + return 0; + } + + return -ENOENT; +} + +static int igc_uc_sync(struct net_device *netdev, const unsigned char *addr) +{ + struct igc_adapter *adapter = netdev_priv(netdev); + int ret; + + ret = igc_add_mac_filter(adapter, addr, adapter->num_rx_queues); + + return min_t(int, ret, 0); +} + +static int igc_uc_unsync(struct net_device *netdev, const unsigned char *addr) +{ + struct igc_adapter *adapter = netdev_priv(netdev); + + igc_del_mac_filter(adapter, addr, adapter->num_rx_queues); + + return 0; +} + +/** + * igc_set_rx_mode - Secondary Unicast, Multicast and Promiscuous mode set + * @netdev: network interface device structure + * + * The set_rx_mode entry point is called whenever the unicast or multicast + * address lists or the network interface flags are updated. This routine is + * responsible for configuring the hardware for proper unicast, multicast, + * promiscuous mode, and all-multi behavior. + */ +static void igc_set_rx_mode(struct net_device *netdev) +{ + struct igc_adapter *adapter = netdev_priv(netdev); + struct igc_hw *hw = &adapter->hw; + u32 rctl = 0, rlpml = MAX_JUMBO_FRAME_SIZE; + int count; + + /* Check for Promiscuous and All Multicast modes */ + if (netdev->flags & IFF_PROMISC) { + rctl |= IGC_RCTL_UPE | IGC_RCTL_MPE; + } else { + if (netdev->flags & IFF_ALLMULTI) { + rctl |= IGC_RCTL_MPE; + } else { + /* Write addresses to the MTA, if the attempt fails + * then we should just turn on promiscuous mode so + * that we can at least receive multicast traffic + */ + count = igc_write_mc_addr_list(netdev); + if (count < 0) + rctl |= IGC_RCTL_MPE; + } + } + + /* Write addresses to available RAR registers, if there is not + * sufficient space to store all the addresses then enable + * unicast promiscuous mode + */ + if (__dev_uc_sync(netdev, igc_uc_sync, igc_uc_unsync)) + rctl |= IGC_RCTL_UPE; + + /* update state of unicast and multicast */ + rctl |= rd32(IGC_RCTL) & ~(IGC_RCTL_UPE | IGC_RCTL_MPE); + wr32(IGC_RCTL, rctl); + +#if (PAGE_SIZE < 8192) + if (adapter->max_frame_size <= IGC_MAX_FRAME_BUILD_SKB) + rlpml = IGC_MAX_FRAME_BUILD_SKB; +#endif + wr32(IGC_RLPML, rlpml); +} + /** * igc_configure - configure the hardware for RX and TX * @adapter: private board structure @@ -2469,27 +2644,6 @@ igc_features_check(struct sk_buff *skb, struct net_device *dev, return features; } -/* If the filter to be added and an already existing filter express - * the same address and address type, it should be possible to only - * override the other configurations, for example the queue to steer - * traffic. - */ -static bool igc_mac_entry_can_be_used(const struct igc_mac_addr *entry, - const u8 *addr, const u8 flags) -{ - if (!(entry->state & IGC_MAC_STATE_IN_USE)) - return true; - - if ((entry->state & IGC_MAC_STATE_SRC_ADDR) != - (flags & IGC_MAC_STATE_SRC_ADDR)) - return false; - - if (!ether_addr_equal(addr, entry->addr)) - return false; - - return true; -} - /* Add a MAC filter for 'addr' directing matching traffic to 'queue', * 'flags' is used to indicate what kind of match is made, match is by * default for the destination address, if matching by source address @@ -2590,161 +2744,6 @@ int igc_del_mac_steering_filter(struct igc_adapter *adapter, IGC_MAC_STATE_QUEUE_STEERING | flags); } -/* Add a MAC filter for 'addr' directing matching traffic to 'queue', - * 'flags' is used to indicate what kind of match is made, match is by - * default for the destination address, if matching by source address - * is desired the flag IGC_MAC_STATE_SRC_ADDR can be used. - */ -static int igc_add_mac_filter(struct igc_adapter *adapter, - const u8 *addr, const u8 queue) -{ - struct igc_hw *hw = &adapter->hw; - int rar_entries = hw->mac.rar_entry_count; - int i; - - if (is_zero_ether_addr(addr)) - return -EINVAL; - - /* Search for the first empty entry in the MAC table. - * Do not touch entries at the end of the table reserved for the VF MAC - * addresses. - */ - for (i = 0; i < rar_entries; i++) { - if (!igc_mac_entry_can_be_used(&adapter->mac_table[i], - addr, 0)) - continue; - - ether_addr_copy(adapter->mac_table[i].addr, addr); - adapter->mac_table[i].queue = queue; - adapter->mac_table[i].state |= IGC_MAC_STATE_IN_USE; - - igc_rar_set_index(adapter, i); - return i; - } - - return -ENOSPC; -} - -/* Remove a MAC filter for 'addr' directing matching traffic to - * 'queue', 'flags' is used to indicate what kind of match need to be - * removed, match is by default for the destination address, if - * matching by source address is to be removed the flag - * IGC_MAC_STATE_SRC_ADDR can be used. - */ -static int igc_del_mac_filter(struct igc_adapter *adapter, - const u8 *addr, const u8 queue) -{ - struct igc_hw *hw = &adapter->hw; - int rar_entries = hw->mac.rar_entry_count; - int i; - - if (is_zero_ether_addr(addr)) - return -EINVAL; - - /* Search for matching entry in the MAC table based on given address - * and queue. Do not touch entries at the end of the table reserved - * for the VF MAC addresses. - */ - for (i = 0; i < rar_entries; i++) { - if (!(adapter->mac_table[i].state & IGC_MAC_STATE_IN_USE)) - continue; - if (adapter->mac_table[i].state != 0) - continue; - if (adapter->mac_table[i].queue != queue) - continue; - if (!ether_addr_equal(adapter->mac_table[i].addr, addr)) - continue; - - /* When a filter for the default address is "deleted", - * we return it to its initial configuration - */ - if (adapter->mac_table[i].state & IGC_MAC_STATE_DEFAULT) { - adapter->mac_table[i].state = - IGC_MAC_STATE_DEFAULT | IGC_MAC_STATE_IN_USE; - adapter->mac_table[i].queue = 0; - } else { - adapter->mac_table[i].state = 0; - adapter->mac_table[i].queue = 0; - memset(adapter->mac_table[i].addr, 0, ETH_ALEN); - } - - igc_rar_set_index(adapter, i); - return 0; - } - - return -ENOENT; -} - -static int igc_uc_sync(struct net_device *netdev, const unsigned char *addr) -{ - struct igc_adapter *adapter = netdev_priv(netdev); - int ret; - - ret = igc_add_mac_filter(adapter, addr, adapter->num_rx_queues); - - return min_t(int, ret, 0); -} - -static int igc_uc_unsync(struct net_device *netdev, const unsigned char *addr) -{ - struct igc_adapter *adapter = netdev_priv(netdev); - - igc_del_mac_filter(adapter, addr, adapter->num_rx_queues); - - return 0; -} - -/** - * igc_set_rx_mode - Secondary Unicast, Multicast and Promiscuous mode set - * @netdev: network interface device structure - * - * The set_rx_mode entry point is called whenever the unicast or multicast - * address lists or the network interface flags are updated. This routine is - * responsible for configuring the hardware for proper unicast, multicast, - * promiscuous mode, and all-multi behavior. - */ -static void igc_set_rx_mode(struct net_device *netdev) -{ - struct igc_adapter *adapter = netdev_priv(netdev); - struct igc_hw *hw = &adapter->hw; - u32 rctl = 0, rlpml = MAX_JUMBO_FRAME_SIZE; - int count; - - /* Check for Promiscuous and All Multicast modes */ - if (netdev->flags & IFF_PROMISC) { - rctl |= IGC_RCTL_UPE | IGC_RCTL_MPE; - } else { - if (netdev->flags & IFF_ALLMULTI) { - rctl |= IGC_RCTL_MPE; - } else { - /* Write addresses to the MTA, if the attempt fails - * then we should just turn on promiscuous mode so - * that we can at least receive multicast traffic - */ - count = igc_write_mc_addr_list(netdev); - if (count < 0) - rctl |= IGC_RCTL_MPE; - } - } - - /* Write addresses to available RAR registers, if there is not - * sufficient space to store all the addresses then enable - * unicast promiscuous mode - */ - if (__dev_uc_sync(netdev, igc_uc_sync, igc_uc_unsync)) - rctl |= IGC_RCTL_UPE; - - /* update state of unicast and multicast */ - rctl |= rd32(IGC_RCTL) & ~(IGC_RCTL_UPE | IGC_RCTL_MPE); - wr32(IGC_RCTL, rctl); - -#if (PAGE_SIZE < 8192) - if (adapter->max_frame_size <= IGC_MAX_FRAME_BUILD_SKB) - rlpml = IGC_MAX_FRAME_BUILD_SKB; -#endif - wr32(IGC_RLPML, rlpml); -} - /** * igc_msix_other - msix other interrupt handler * @irq: interrupt number -- cgit From a146ea022f1d02df0116fae9d5d334b75f022a24 Mon Sep 17 00:00:00 2001 From: Sasha Neftin Date: Tue, 19 Nov 2019 13:44:38 +0200 Subject: igc: Remove no need declaration of the igc_configure_msix We want to avoid forward-declarations of function if possible. Rearrange the igc_configure_msix function implementation. Signed-off-by: Sasha Neftin Tested-by: Aaron Brown Signed-off-by: Jeff Kirsher --- drivers/net/ethernet/intel/igc/igc_main.c | 89 +++++++++++++++---------------- 1 file changed, 44 insertions(+), 45 deletions(-) (limited to 'drivers/net/ethernet/intel/igc/igc_main.c') diff --git a/drivers/net/ethernet/intel/igc/igc_main.c b/drivers/net/ethernet/intel/igc/igc_main.c index 13491ad7d251..6e35ac5bd2fc 100644 --- a/drivers/net/ethernet/intel/igc/igc_main.c +++ b/drivers/net/ethernet/intel/igc/igc_main.c @@ -59,7 +59,6 @@ static void igc_free_q_vector(struct igc_adapter *adapter, int v_idx); static void igc_free_q_vectors(struct igc_adapter *adapter); static void igc_irq_disable(struct igc_adapter *adapter); static void igc_irq_enable(struct igc_adapter *adapter); -static void igc_configure_msix(struct igc_adapter *adapter); enum latency_range { lowest_latency = 0, @@ -2200,6 +2199,50 @@ static void igc_configure(struct igc_adapter *adapter) } } +/** + * igc_configure_msix - Configure MSI-X hardware + * @adapter: Pointer to adapter structure + * + * igc_configure_msix sets up the hardware to properly + * generate MSI-X interrupts. + */ +static void igc_configure_msix(struct igc_adapter *adapter) +{ + struct igc_hw *hw = &adapter->hw; + int i, vector = 0; + u32 tmp; + + adapter->eims_enable_mask = 0; + + /* set vector for other causes, i.e. link changes */ + switch (hw->mac.type) { + case igc_i225: + /* Turn on MSI-X capability first, or our settings + * won't stick. And it will take days to debug. + */ + wr32(IGC_GPIE, IGC_GPIE_MSIX_MODE | + IGC_GPIE_PBA | IGC_GPIE_EIAME | + IGC_GPIE_NSICR); + + /* enable msix_other interrupt */ + adapter->eims_other = BIT(vector); + tmp = (vector++ | IGC_IVAR_VALID) << 8; + + wr32(IGC_IVAR_MISC, tmp); + break; + default: + /* do nothing, since nothing else supports MSI-X */ + break; + } /* switch (hw->mac.type) */ + + adapter->eims_enable_mask |= adapter->eims_other; + + for (i = 0; i < adapter->num_q_vectors; i++) + igc_assign_vector(adapter->q_vector[i], vector++); + + wrfl(); +} + /** * igc_up - Open the interface and prepare it to handle traffic * @adapter: board private structure @@ -2837,50 +2880,6 @@ static void igc_assign_vector(struct igc_q_vector *q_vector, int msix_vector) q_vector->set_itr = 1; } -/** - * igc_configure_msix - Configure MSI-X hardware - * @adapter: Pointer to adapter structure - * - * igc_configure_msix sets up the hardware to properly - * generate MSI-X interrupts. - */ -static void igc_configure_msix(struct igc_adapter *adapter) -{ - struct igc_hw *hw = &adapter->hw; - int i, vector = 0; - u32 tmp; - - adapter->eims_enable_mask = 0; - - /* set vector for other causes, i.e. link changes */ - switch (hw->mac.type) { - case igc_i225: - /* Turn on MSI-X capability first, or our settings - * won't stick. And it will take days to debug. - */ - wr32(IGC_GPIE, IGC_GPIE_MSIX_MODE | - IGC_GPIE_PBA | IGC_GPIE_EIAME | - IGC_GPIE_NSICR); - - /* enable msix_other interrupt */ - adapter->eims_other = BIT(vector); - tmp = (vector++ | IGC_IVAR_VALID) << 8; - - wr32(IGC_IVAR_MISC, tmp); - break; - default: - /* do nothing, since nothing else supports MSI-X */ - break; - } /* switch (hw->mac.type) */ - - adapter->eims_enable_mask |= adapter->eims_other; - - for (i = 0; i < adapter->num_q_vectors; i++) - igc_assign_vector(adapter->q_vector[i], vector++); - - wrfl(); -} - static irqreturn_t igc_msix_ring(int irq, void *data) { struct igc_q_vector *q_vector = data; -- cgit From fccf939e2709bf82fd58df73061a63e5016eecbe Mon Sep 17 00:00:00 2001 From: Sasha Neftin Date: Tue, 19 Nov 2019 13:44:47 +0200 Subject: igc: Remove no need declaration of the igc_irq_enable We want to avoid forward-declarations of function if possible. Rearrange the igc_irq_enable function implementation. Signed-off-by: Sasha Neftin Tested-by: Aaron Brown Signed-off-by: Jeff Kirsher --- drivers/net/ethernet/intel/igc/igc_main.c | 47 +++++++++++++++---------------- 1 file changed, 23 insertions(+), 24 deletions(-) (limited to 'drivers/net/ethernet/intel/igc/igc_main.c') diff --git a/drivers/net/ethernet/intel/igc/igc_main.c b/drivers/net/ethernet/intel/igc/igc_main.c index 6e35ac5bd2fc..9597c5753d6b 100644 --- a/drivers/net/ethernet/intel/igc/igc_main.c +++ b/drivers/net/ethernet/intel/igc/igc_main.c @@ -58,7 +58,6 @@ static void igc_assign_vector(struct igc_q_vector *q_vector, int msix_vector); static void igc_free_q_vector(struct igc_adapter *adapter, int v_idx); static void igc_free_q_vectors(struct igc_adapter *adapter); static void igc_irq_disable(struct igc_adapter *adapter); -static void igc_irq_enable(struct igc_adapter *adapter); enum latency_range { lowest_latency = 0, @@ -2243,6 +2242,29 @@ static void igc_configure_msix(struct igc_adapter *adapter) wrfl(); } +/** + * igc_irq_enable - Enable default interrupt generation settings + * @adapter: board private structure + */ +static void igc_irq_enable(struct igc_adapter *adapter) +{ + struct igc_hw *hw = &adapter->hw; + + if (adapter->msix_entries) { + u32 ims = IGC_IMS_LSC | IGC_IMS_DOUTSYNC | IGC_IMS_DRSTA; + u32 regval = rd32(IGC_EIAC); + + wr32(IGC_EIAC, regval | adapter->eims_enable_mask); + regval = rd32(IGC_EIAM); + wr32(IGC_EIAM, regval | adapter->eims_enable_mask); + wr32(IGC_EIMS, adapter->eims_enable_mask); + wr32(IGC_IMS, ims); + } else { + wr32(IGC_IMS, IMS_ENABLE_MASK | IGC_IMS_DRSTA); + wr32(IGC_IAM, IMS_ENABLE_MASK | IGC_IMS_DRSTA); + } +} + /** * igc_up - Open the interface and prepare it to handle traffic * @adapter: board private structure @@ -3972,29 +3994,6 @@ static void igc_irq_disable(struct igc_adapter *adapter) } } -/** - * igc_irq_enable - Enable default interrupt generation settings - * @adapter: board private structure - */ -static void igc_irq_enable(struct igc_adapter *adapter) -{ - struct igc_hw *hw = &adapter->hw; - - if (adapter->msix_entries) { - u32 ims = IGC_IMS_LSC | IGC_IMS_DOUTSYNC | IGC_IMS_DRSTA; - u32 regval = rd32(IGC_EIAC); - - wr32(IGC_EIAC, regval | adapter->eims_enable_mask); - regval = rd32(IGC_EIAM); - wr32(IGC_EIAM, regval | adapter->eims_enable_mask); - wr32(IGC_EIMS, adapter->eims_enable_mask); - wr32(IGC_IMS, ims); - } else { - wr32(IGC_IMS, IMS_ENABLE_MASK | IGC_IMS_DRSTA); - wr32(IGC_IAM, IMS_ENABLE_MASK | IGC_IMS_DRSTA); - } -} - /** * igc_request_irq - initialize interrupts * @adapter: Pointer to adapter structure -- cgit From 35f9a78ac93361c2992719f9e00d60c158a32874 Mon Sep 17 00:00:00 2001 From: Sasha Neftin Date: Tue, 19 Nov 2019 13:44:58 +0200 Subject: igc: Remove no need declaration of the igc_irq_disable We want to avoid forward-declarations of function if possible. Rearrange the igc_irq_disable function implementation. Signed-off-by: Sasha Neftin Tested-by: Aaron Brown Signed-off-by: Jeff Kirsher --- drivers/net/ethernet/intel/igc/igc_main.c | 67 +++++++++++++++---------------- 1 file changed, 33 insertions(+), 34 deletions(-) (limited to 'drivers/net/ethernet/intel/igc/igc_main.c') diff --git a/drivers/net/ethernet/intel/igc/igc_main.c b/drivers/net/ethernet/intel/igc/igc_main.c index 9597c5753d6b..4cfb9856ee13 100644 --- a/drivers/net/ethernet/intel/igc/igc_main.c +++ b/drivers/net/ethernet/intel/igc/igc_main.c @@ -57,7 +57,6 @@ static void igc_write_itr(struct igc_q_vector *q_vector); static void igc_assign_vector(struct igc_q_vector *q_vector, int msix_vector); static void igc_free_q_vector(struct igc_adapter *adapter, int v_idx); static void igc_free_q_vectors(struct igc_adapter *adapter); -static void igc_irq_disable(struct igc_adapter *adapter); enum latency_range { lowest_latency = 0, @@ -2265,6 +2264,39 @@ static void igc_irq_enable(struct igc_adapter *adapter) } } +/** + * igc_irq_disable - Mask off interrupt generation on the NIC + * @adapter: board private structure + */ +static void igc_irq_disable(struct igc_adapter *adapter) +{ + struct igc_hw *hw = &adapter->hw; + + if (adapter->msix_entries) { + u32 regval = rd32(IGC_EIAM); + + wr32(IGC_EIAM, regval & ~adapter->eims_enable_mask); + wr32(IGC_EIMC, adapter->eims_enable_mask); + regval = rd32(IGC_EIAC); + wr32(IGC_EIAC, regval & ~adapter->eims_enable_mask); + } + + wr32(IGC_IAM, 0); + wr32(IGC_IMC, ~0); + wrfl(); + + if (adapter->msix_entries) { + int vector = 0, i; + + synchronize_irq(adapter->msix_entries[vector++].vector); + + for (i = 0; i < adapter->num_q_vectors; i++) + synchronize_irq(adapter->msix_entries[vector++].vector); + } else { + synchronize_irq(adapter->pdev->irq); + } +} + /** * igc_up - Open the interface and prepare it to handle traffic * @adapter: board private structure @@ -3961,39 +3993,6 @@ static void igc_free_irq(struct igc_adapter *adapter) } } -/** - * igc_irq_disable - Mask off interrupt generation on the NIC - * @adapter: board private structure - */ -static void igc_irq_disable(struct igc_adapter *adapter) -{ - struct igc_hw *hw = &adapter->hw; - - if (adapter->msix_entries) { - u32 regval = rd32(IGC_EIAM); - - wr32(IGC_EIAM, regval & ~adapter->eims_enable_mask); - wr32(IGC_EIMC, adapter->eims_enable_mask); - regval = rd32(IGC_EIAC); - wr32(IGC_EIAC, regval & ~adapter->eims_enable_mask); - } - - wr32(IGC_IAM, 0); - wr32(IGC_IMC, ~0); - wrfl(); - - if (adapter->msix_entries) { - int vector = 0, i; - - synchronize_irq(adapter->msix_entries[vector++].vector); - - for (i = 0; i < adapter->num_q_vectors; i++) - synchronize_irq(adapter->msix_entries[vector++].vector); - } else { - synchronize_irq(adapter->pdev->irq); - } -} - /** * igc_request_irq - initialize interrupts * @adapter: Pointer to adapter structure -- cgit From a8c4873b566bcbf024988334bcbfd0a31667064f Mon Sep 17 00:00:00 2001 From: Sasha Neftin Date: Tue, 19 Nov 2019 13:45:07 +0200 Subject: igc: Remove no need declaration of the igc_free_q_vectors We want to avoid forward-declarations of function if possible. Rearrange the igc_free_q_vectors function implementation. Signed-off-by: Sasha Neftin Tested-by: Aaron Brown Signed-off-by: Jeff Kirsher --- drivers/net/ethernet/intel/igc/igc_main.c | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) (limited to 'drivers/net/ethernet/intel/igc/igc_main.c') diff --git a/drivers/net/ethernet/intel/igc/igc_main.c b/drivers/net/ethernet/intel/igc/igc_main.c index 4cfb9856ee13..58347135b0fc 100644 --- a/drivers/net/ethernet/intel/igc/igc_main.c +++ b/drivers/net/ethernet/intel/igc/igc_main.c @@ -56,7 +56,6 @@ static int igc_sw_init(struct igc_adapter *); static void igc_write_itr(struct igc_q_vector *q_vector); static void igc_assign_vector(struct igc_q_vector *q_vector, int msix_vector); static void igc_free_q_vector(struct igc_adapter *adapter, int v_idx); -static void igc_free_q_vectors(struct igc_adapter *adapter); enum latency_range { lowest_latency = 0, @@ -3118,19 +3117,6 @@ msi_only: adapter->flags |= IGC_FLAG_HAS_MSI; } -/** - * igc_clear_interrupt_scheme - reset the device to a state of no interrupts - * @adapter: Pointer to adapter structure - * - * This function resets the device so that it has 0 rx queues, tx queues, and - * MSI-X interrupts allocated. - */ -static void igc_clear_interrupt_scheme(struct igc_adapter *adapter) -{ - igc_free_q_vectors(adapter); - igc_reset_interrupt_capability(adapter); -} - /** * igc_free_q_vectors - Free memory allocated for interrupt vectors * @adapter: board private structure to initialize @@ -3153,6 +3139,19 @@ static void igc_free_q_vectors(struct igc_adapter *adapter) } } +/** + * igc_clear_interrupt_scheme - reset the device to a state of no interrupts + * @adapter: Pointer to adapter structure + * + * This function resets the device so that it has 0 rx queues, tx queues, and + * MSI-X interrupts allocated. + */ +static void igc_clear_interrupt_scheme(struct igc_adapter *adapter) +{ + igc_free_q_vectors(adapter); + igc_reset_interrupt_capability(adapter); +} + /** * igc_free_q_vector - Free memory allocated for specific interrupt vector * @adapter: board private structure to initialize -- cgit From f7bcca5d9a8cf087b3f202ad45907b02437faf77 Mon Sep 17 00:00:00 2001 From: Sasha Neftin Date: Tue, 19 Nov 2019 13:45:16 +0200 Subject: igc: Remove no need declaration of the igc_free_q_vector We want to avoid forward-declarations of function if possible. Rearrange the igc_free_q_vector function implementation. Signed-off-by: Sasha Neftin Tested-by: Aaron Brown Signed-off-by: Jeff Kirsher --- drivers/net/ethernet/intel/igc/igc_main.c | 41 +++++++++++++++---------------- 1 file changed, 20 insertions(+), 21 deletions(-) (limited to 'drivers/net/ethernet/intel/igc/igc_main.c') diff --git a/drivers/net/ethernet/intel/igc/igc_main.c b/drivers/net/ethernet/intel/igc/igc_main.c index 58347135b0fc..368e537d1be2 100644 --- a/drivers/net/ethernet/intel/igc/igc_main.c +++ b/drivers/net/ethernet/intel/igc/igc_main.c @@ -55,7 +55,6 @@ MODULE_DEVICE_TABLE(pci, igc_pci_tbl); static int igc_sw_init(struct igc_adapter *); static void igc_write_itr(struct igc_q_vector *q_vector); static void igc_assign_vector(struct igc_q_vector *q_vector, int msix_vector); -static void igc_free_q_vector(struct igc_adapter *adapter, int v_idx); enum latency_range { lowest_latency = 0, @@ -3117,6 +3116,26 @@ msi_only: adapter->flags |= IGC_FLAG_HAS_MSI; } +/** + * igc_free_q_vector - Free memory allocated for specific interrupt vector + * @adapter: board private structure to initialize + * @v_idx: Index of vector to be freed + * + * This function frees the memory allocated to the q_vector. + */ +static void igc_free_q_vector(struct igc_adapter *adapter, int v_idx) +{ + struct igc_q_vector *q_vector = adapter->q_vector[v_idx]; + + adapter->q_vector[v_idx] = NULL; + + /* igc_get_stats64() might access the rings on this vector, + * we must wait a grace period before freeing it. + */ + if (q_vector) + kfree_rcu(q_vector, rcu); +} + /** * igc_free_q_vectors - Free memory allocated for interrupt vectors * @adapter: board private structure to initialize @@ -3152,26 +3171,6 @@ static void igc_clear_interrupt_scheme(struct igc_adapter *adapter) igc_reset_interrupt_capability(adapter); } -/** - * igc_free_q_vector - Free memory allocated for specific interrupt vector - * @adapter: board private structure to initialize - * @v_idx: Index of vector to be freed - * - * This function frees the memory allocated to the q_vector. - */ -static void igc_free_q_vector(struct igc_adapter *adapter, int v_idx) -{ - struct igc_q_vector *q_vector = adapter->q_vector[v_idx]; - - adapter->q_vector[v_idx] = NULL; - - /* igc_get_stats64() might access the rings on this vector, - * we must wait a grace period before freeing it. - */ - if (q_vector) - kfree_rcu(q_vector, rcu); -} - /* Need to wait a few seconds after link up to get diagnostic information from * the phy */ -- cgit From f817fa0555c8885060801b645885bcaf44c76b35 Mon Sep 17 00:00:00 2001 From: Sasha Neftin Date: Tue, 19 Nov 2019 13:45:23 +0200 Subject: igc: Remove no need declaration of the igc_assign_vector We want to avoid forward-declarations of function if possible. Rearrange the igc_assign_vector function implementation. Signed-off-by: Sasha Neftin Tested-by: Aaron Brown Signed-off-by: Jeff Kirsher --- drivers/net/ethernet/intel/igc/igc_main.c | 123 +++++++++++++++--------------- 1 file changed, 61 insertions(+), 62 deletions(-) (limited to 'drivers/net/ethernet/intel/igc/igc_main.c') diff --git a/drivers/net/ethernet/intel/igc/igc_main.c b/drivers/net/ethernet/intel/igc/igc_main.c index 368e537d1be2..4fb40c6aa583 100644 --- a/drivers/net/ethernet/intel/igc/igc_main.c +++ b/drivers/net/ethernet/intel/igc/igc_main.c @@ -54,7 +54,6 @@ MODULE_DEVICE_TABLE(pci, igc_pci_tbl); /* forward declaration */ static int igc_sw_init(struct igc_adapter *); static void igc_write_itr(struct igc_q_vector *q_vector); -static void igc_assign_vector(struct igc_q_vector *q_vector, int msix_vector); enum latency_range { lowest_latency = 0, @@ -2195,6 +2194,67 @@ static void igc_configure(struct igc_adapter *adapter) } } +/** + * igc_write_ivar - configure ivar for given MSI-X vector + * @hw: pointer to the HW structure + * @msix_vector: vector number we are allocating to a given ring + * @index: row index of IVAR register to write within IVAR table + * @offset: column offset of in IVAR, should be multiple of 8 + * + * The IVAR table consists of 2 columns, + * each containing an cause allocation for an Rx and Tx ring, and a + * variable number of rows depending on the number of queues supported. + */ +static void igc_write_ivar(struct igc_hw *hw, int msix_vector, + int index, int offset) +{ + u32 ivar = array_rd32(IGC_IVAR0, index); + + /* clear any bits that are currently set */ + ivar &= ~((u32)0xFF << offset); + + /* write vector and valid bit */ + ivar |= (msix_vector | IGC_IVAR_VALID) << offset; + + array_wr32(IGC_IVAR0, index, ivar); +} + +static void igc_assign_vector(struct igc_q_vector *q_vector, int msix_vector) +{ + struct igc_adapter *adapter = q_vector->adapter; + struct igc_hw *hw = &adapter->hw; + int rx_queue = IGC_N0_QUEUE; + int tx_queue = IGC_N0_QUEUE; + + if (q_vector->rx.ring) + rx_queue = q_vector->rx.ring->reg_idx; + if (q_vector->tx.ring) + tx_queue = q_vector->tx.ring->reg_idx; + + switch (hw->mac.type) { + case igc_i225: + if (rx_queue > IGC_N0_QUEUE) + igc_write_ivar(hw, msix_vector, + rx_queue >> 1, + (rx_queue & 0x1) << 4); + if (tx_queue > IGC_N0_QUEUE) + igc_write_ivar(hw, msix_vector, + tx_queue >> 1, + ((tx_queue & 0x1) << 4) + 8); + q_vector->eims_value = BIT(msix_vector); + break; + default: + WARN_ONCE(hw->mac.type != igc_i225, "Wrong MAC type\n"); + break; + } + + /* add q_vector eims value to global eims_enable_mask */ + adapter->eims_enable_mask |= q_vector->eims_value; + + /* configure q_vector to set itr on first interrupt */ + q_vector->set_itr = 1; +} + /** * igc_configure_msix - Configure MSI-X hardware * @adapter: Pointer to adapter structure @@ -2871,67 +2931,6 @@ static irqreturn_t igc_msix_other(int irq, void *data) return IRQ_HANDLED; } -/** - * igc_write_ivar - configure ivar for given MSI-X vector - * @hw: pointer to the HW structure - * @msix_vector: vector number we are allocating to a given ring - * @index: row index of IVAR register to write within IVAR table - * @offset: column offset of in IVAR, should be multiple of 8 - * - * The IVAR table consists of 2 columns, - * each containing an cause allocation for an Rx and Tx ring, and a - * variable number of rows depending on the number of queues supported. - */ -static void igc_write_ivar(struct igc_hw *hw, int msix_vector, - int index, int offset) -{ - u32 ivar = array_rd32(IGC_IVAR0, index); - - /* clear any bits that are currently set */ - ivar &= ~((u32)0xFF << offset); - - /* write vector and valid bit */ - ivar |= (msix_vector | IGC_IVAR_VALID) << offset; - - array_wr32(IGC_IVAR0, index, ivar); -} - -static void igc_assign_vector(struct igc_q_vector *q_vector, int msix_vector) -{ - struct igc_adapter *adapter = q_vector->adapter; - struct igc_hw *hw = &adapter->hw; - int rx_queue = IGC_N0_QUEUE; - int tx_queue = IGC_N0_QUEUE; - - if (q_vector->rx.ring) - rx_queue = q_vector->rx.ring->reg_idx; - if (q_vector->tx.ring) - tx_queue = q_vector->tx.ring->reg_idx; - - switch (hw->mac.type) { - case igc_i225: - if (rx_queue > IGC_N0_QUEUE) - igc_write_ivar(hw, msix_vector, - rx_queue >> 1, - (rx_queue & 0x1) << 4); - if (tx_queue > IGC_N0_QUEUE) - igc_write_ivar(hw, msix_vector, - tx_queue >> 1, - ((tx_queue & 0x1) << 4) + 8); - q_vector->eims_value = BIT(msix_vector); - break; - default: - WARN_ONCE(hw->mac.type != igc_i225, "Wrong MAC type\n"); - break; - } - - /* add q_vector eims value to global eims_enable_mask */ - adapter->eims_enable_mask |= q_vector->eims_value; - - /* configure q_vector to set itr on first interrupt */ - q_vector->set_itr = 1; -} - static irqreturn_t igc_msix_ring(int irq, void *data) { struct igc_q_vector *q_vector = data; -- cgit From 55cd7386c5b759ba4a736dee85cbad825013fd20 Mon Sep 17 00:00:00 2001 From: Sasha Neftin Date: Tue, 19 Nov 2019 13:45:32 +0200 Subject: igc: Remove no need declaration of the igc_write_itr We want to avoid forward-declarations of function if possible. Rearrange the igc_write_itr function implementation. Signed-off-by: Sasha Neftin Tested-by: Aaron Brown Signed-off-by: Jeff Kirsher --- drivers/net/ethernet/intel/igc/igc_main.c | 33 +++++++++++++++---------------- 1 file changed, 16 insertions(+), 17 deletions(-) (limited to 'drivers/net/ethernet/intel/igc/igc_main.c') diff --git a/drivers/net/ethernet/intel/igc/igc_main.c b/drivers/net/ethernet/intel/igc/igc_main.c index 4fb40c6aa583..c7b9bbbef3aa 100644 --- a/drivers/net/ethernet/intel/igc/igc_main.c +++ b/drivers/net/ethernet/intel/igc/igc_main.c @@ -53,7 +53,6 @@ MODULE_DEVICE_TABLE(pci, igc_pci_tbl); /* forward declaration */ static int igc_sw_init(struct igc_adapter *); -static void igc_write_itr(struct igc_q_vector *q_vector); enum latency_range { lowest_latency = 0, @@ -2931,6 +2930,22 @@ static irqreturn_t igc_msix_other(int irq, void *data) return IRQ_HANDLED; } +static void igc_write_itr(struct igc_q_vector *q_vector) +{ + u32 itr_val = q_vector->itr_val & IGC_QVECTOR_MASK; + + if (!q_vector->set_itr) + return; + + if (!itr_val) + itr_val = IGC_ITR_VAL_MASK; + + itr_val |= IGC_EITR_CNT_IGNR; + + writel(itr_val, q_vector->itr_register); + q_vector->set_itr = 0; +} + static irqreturn_t igc_msix_ring(int irq, void *data) { struct igc_q_vector *q_vector = data; @@ -4044,22 +4059,6 @@ request_done: return err; } -static void igc_write_itr(struct igc_q_vector *q_vector) -{ - u32 itr_val = q_vector->itr_val & IGC_QVECTOR_MASK; - - if (!q_vector->set_itr) - return; - - if (!itr_val) - itr_val = IGC_ITR_VAL_MASK; - - itr_val |= IGC_EITR_CNT_IGNR; - - writel(itr_val, q_vector->itr_register); - q_vector->set_itr = 0; -} - /** * __igc_open - Called when a network interface is made active * @netdev: network interface device structure -- cgit From 63c92c9d2e536c91ace17dec7534883f40421753 Mon Sep 17 00:00:00 2001 From: Sasha Neftin Date: Tue, 19 Nov 2019 13:45:40 +0200 Subject: igc: Remove no need declaration of the igc_sw_init We want to avoid forward-declarations of function if possible. Rearrange the igc_sw_init function implementation. Signed-off-by: Sasha Neftin Tested-by: Aaron Brown Signed-off-by: Jeff Kirsher --- drivers/net/ethernet/intel/igc/igc_main.c | 2733 ++++++++++++++--------------- 1 file changed, 1365 insertions(+), 1368 deletions(-) (limited to 'drivers/net/ethernet/intel/igc/igc_main.c') diff --git a/drivers/net/ethernet/intel/igc/igc_main.c b/drivers/net/ethernet/intel/igc/igc_main.c index c7b9bbbef3aa..24fbbc928d66 100644 --- a/drivers/net/ethernet/intel/igc/igc_main.c +++ b/drivers/net/ethernet/intel/igc/igc_main.c @@ -51,9 +51,6 @@ static const struct pci_device_id igc_pci_tbl[] = { MODULE_DEVICE_TABLE(pci, igc_pci_tbl); -/* forward declaration */ -static int igc_sw_init(struct igc_adapter *); - enum latency_range { lowest_latency = 0, low_latency = 1, @@ -2354,1230 +2351,1302 @@ static void igc_irq_disable(struct igc_adapter *adapter) } } +void igc_set_flag_queue_pairs(struct igc_adapter *adapter, + const u32 max_rss_queues) +{ + /* Determine if we need to pair queues. */ + /* If rss_queues > half of max_rss_queues, pair the queues in + * order to conserve interrupts due to limited supply. + */ + if (adapter->rss_queues > (max_rss_queues / 2)) + adapter->flags |= IGC_FLAG_QUEUE_PAIRS; + else + adapter->flags &= ~IGC_FLAG_QUEUE_PAIRS; +} + +unsigned int igc_get_max_rss_queues(struct igc_adapter *adapter) +{ + unsigned int max_rss_queues; + + /* Determine the maximum number of RSS queues supported. */ + max_rss_queues = IGC_MAX_RX_QUEUES; + + return max_rss_queues; +} + +static void igc_init_queue_configuration(struct igc_adapter *adapter) +{ + u32 max_rss_queues; + + max_rss_queues = igc_get_max_rss_queues(adapter); + adapter->rss_queues = min_t(u32, max_rss_queues, num_online_cpus()); + + igc_set_flag_queue_pairs(adapter, max_rss_queues); +} + /** - * igc_up - Open the interface and prepare it to handle traffic - * @adapter: board private structure + * igc_reset_q_vector - Reset config for interrupt vector + * @adapter: board private structure to initialize + * @v_idx: Index of vector to be reset + * + * If NAPI is enabled it will delete any references to the + * NAPI struct. This is preparation for igc_free_q_vector. */ -void igc_up(struct igc_adapter *adapter) +static void igc_reset_q_vector(struct igc_adapter *adapter, int v_idx) { - struct igc_hw *hw = &adapter->hw; - int i = 0; + struct igc_q_vector *q_vector = adapter->q_vector[v_idx]; - /* hardware has been reset, we need to reload some things */ - igc_configure(adapter); + /* if we're coming from igc_set_interrupt_capability, the vectors are + * not yet allocated + */ + if (!q_vector) + return; - clear_bit(__IGC_DOWN, &adapter->state); + if (q_vector->tx.ring) + adapter->tx_ring[q_vector->tx.ring->queue_index] = NULL; - for (i = 0; i < adapter->num_q_vectors; i++) - napi_enable(&adapter->q_vector[i]->napi); + if (q_vector->rx.ring) + adapter->rx_ring[q_vector->rx.ring->queue_index] = NULL; - if (adapter->msix_entries) - igc_configure_msix(adapter); - else - igc_assign_vector(adapter->q_vector[0], 0); + netif_napi_del(&q_vector->napi); +} - /* Clear any pending interrupts. */ - rd32(IGC_ICR); - igc_irq_enable(adapter); +/** + * igc_free_q_vector - Free memory allocated for specific interrupt vector + * @adapter: board private structure to initialize + * @v_idx: Index of vector to be freed + * + * This function frees the memory allocated to the q_vector. + */ +static void igc_free_q_vector(struct igc_adapter *adapter, int v_idx) +{ + struct igc_q_vector *q_vector = adapter->q_vector[v_idx]; - netif_tx_start_all_queues(adapter->netdev); + adapter->q_vector[v_idx] = NULL; - /* start the watchdog. */ - hw->mac.get_link_status = 1; - schedule_work(&adapter->watchdog_task); + /* igc_get_stats64() might access the rings on this vector, + * we must wait a grace period before freeing it. + */ + if (q_vector) + kfree_rcu(q_vector, rcu); } /** - * igc_update_stats - Update the board statistics counters - * @adapter: board private structure + * igc_free_q_vectors - Free memory allocated for interrupt vectors + * @adapter: board private structure to initialize + * + * This function frees the memory allocated to the q_vectors. In addition if + * NAPI is enabled it will delete any references to the NAPI struct prior + * to freeing the q_vector. */ -void igc_update_stats(struct igc_adapter *adapter) +static void igc_free_q_vectors(struct igc_adapter *adapter) { - struct rtnl_link_stats64 *net_stats = &adapter->stats64; - struct pci_dev *pdev = adapter->pdev; - struct igc_hw *hw = &adapter->hw; - u64 _bytes, _packets; - u64 bytes, packets; - unsigned int start; - u32 mpc; - int i; + int v_idx = adapter->num_q_vectors; - /* Prevent stats update while adapter is being reset, or if the pci - * connection is down. - */ - if (adapter->link_speed == 0) - return; - if (pci_channel_offline(pdev)) - return; + adapter->num_tx_queues = 0; + adapter->num_rx_queues = 0; + adapter->num_q_vectors = 0; - packets = 0; - bytes = 0; + while (v_idx--) { + igc_reset_q_vector(adapter, v_idx); + igc_free_q_vector(adapter, v_idx); + } +} - rcu_read_lock(); - for (i = 0; i < adapter->num_rx_queues; i++) { - struct igc_ring *ring = adapter->rx_ring[i]; - u32 rqdpc = rd32(IGC_RQDPC(i)); +/** + * igc_update_itr - update the dynamic ITR value based on statistics + * @q_vector: pointer to q_vector + * @ring_container: ring info to update the itr for + * + * Stores a new ITR value based on packets and byte + * counts during the last interrupt. The advantage of per interrupt + * computation is faster updates and more accurate ITR for the current + * traffic pattern. Constants in this function were computed + * based on theoretical maximum wire speed and thresholds were set based + * on testing data as well as attempting to minimize response time + * while increasing bulk throughput. + * NOTE: These calculations are only valid when operating in a single- + * queue environment. + */ +static void igc_update_itr(struct igc_q_vector *q_vector, + struct igc_ring_container *ring_container) +{ + unsigned int packets = ring_container->total_packets; + unsigned int bytes = ring_container->total_bytes; + u8 itrval = ring_container->itr; - if (hw->mac.type >= igc_i225) - wr32(IGC_RQDPC(i), 0); + /* no packets, exit with status unchanged */ + if (packets == 0) + return; - if (rqdpc) { - ring->rx_stats.drops += rqdpc; - net_stats->rx_fifo_errors += rqdpc; + switch (itrval) { + case lowest_latency: + /* handle TSO and jumbo frames */ + if (bytes / packets > 8000) + itrval = bulk_latency; + else if ((packets < 5) && (bytes > 512)) + itrval = low_latency; + break; + case low_latency: /* 50 usec aka 20000 ints/s */ + if (bytes > 10000) { + /* this if handles the TSO accounting */ + if (bytes / packets > 8000) + itrval = bulk_latency; + else if ((packets < 10) || ((bytes / packets) > 1200)) + itrval = bulk_latency; + else if ((packets > 35)) + itrval = lowest_latency; + } else if (bytes / packets > 2000) { + itrval = bulk_latency; + } else if (packets <= 2 && bytes < 512) { + itrval = lowest_latency; } - - do { - start = u64_stats_fetch_begin_irq(&ring->rx_syncp); - _bytes = ring->rx_stats.bytes; - _packets = ring->rx_stats.packets; - } while (u64_stats_fetch_retry_irq(&ring->rx_syncp, start)); - bytes += _bytes; - packets += _packets; + break; + case bulk_latency: /* 250 usec aka 4000 ints/s */ + if (bytes > 25000) { + if (packets > 35) + itrval = low_latency; + } else if (bytes < 1500) { + itrval = low_latency; + } + break; } - net_stats->rx_bytes = bytes; - net_stats->rx_packets = packets; + /* clear work counters since we have the values we need */ + ring_container->total_bytes = 0; + ring_container->total_packets = 0; - packets = 0; - bytes = 0; - for (i = 0; i < adapter->num_tx_queues; i++) { - struct igc_ring *ring = adapter->tx_ring[i]; + /* write updated itr to ring container */ + ring_container->itr = itrval; +} - do { - start = u64_stats_fetch_begin_irq(&ring->tx_syncp); - _bytes = ring->tx_stats.bytes; - _packets = ring->tx_stats.packets; - } while (u64_stats_fetch_retry_irq(&ring->tx_syncp, start)); - bytes += _bytes; - packets += _packets; +static void igc_set_itr(struct igc_q_vector *q_vector) +{ + struct igc_adapter *adapter = q_vector->adapter; + u32 new_itr = q_vector->itr_val; + u8 current_itr = 0; + + /* for non-gigabit speeds, just fix the interrupt rate at 4000 */ + switch (adapter->link_speed) { + case SPEED_10: + case SPEED_100: + current_itr = 0; + new_itr = IGC_4K_ITR; + goto set_itr_now; + default: + break; } - net_stats->tx_bytes = bytes; - net_stats->tx_packets = packets; - rcu_read_unlock(); - /* read stats registers */ - adapter->stats.crcerrs += rd32(IGC_CRCERRS); - adapter->stats.gprc += rd32(IGC_GPRC); - adapter->stats.gorc += rd32(IGC_GORCL); - rd32(IGC_GORCH); /* clear GORCL */ - adapter->stats.bprc += rd32(IGC_BPRC); - adapter->stats.mprc += rd32(IGC_MPRC); - adapter->stats.roc += rd32(IGC_ROC); + igc_update_itr(q_vector, &q_vector->tx); + igc_update_itr(q_vector, &q_vector->rx); - adapter->stats.prc64 += rd32(IGC_PRC64); - adapter->stats.prc127 += rd32(IGC_PRC127); - adapter->stats.prc255 += rd32(IGC_PRC255); - adapter->stats.prc511 += rd32(IGC_PRC511); - adapter->stats.prc1023 += rd32(IGC_PRC1023); - adapter->stats.prc1522 += rd32(IGC_PRC1522); - adapter->stats.symerrs += rd32(IGC_SYMERRS); - adapter->stats.sec += rd32(IGC_SEC); + current_itr = max(q_vector->rx.itr, q_vector->tx.itr); - mpc = rd32(IGC_MPC); - adapter->stats.mpc += mpc; - net_stats->rx_fifo_errors += mpc; - adapter->stats.scc += rd32(IGC_SCC); - adapter->stats.ecol += rd32(IGC_ECOL); - adapter->stats.mcc += rd32(IGC_MCC); - adapter->stats.latecol += rd32(IGC_LATECOL); - adapter->stats.dc += rd32(IGC_DC); - adapter->stats.rlec += rd32(IGC_RLEC); - adapter->stats.xonrxc += rd32(IGC_XONRXC); - adapter->stats.xontxc += rd32(IGC_XONTXC); - adapter->stats.xoffrxc += rd32(IGC_XOFFRXC); - adapter->stats.xofftxc += rd32(IGC_XOFFTXC); - adapter->stats.fcruc += rd32(IGC_FCRUC); - adapter->stats.gptc += rd32(IGC_GPTC); - adapter->stats.gotc += rd32(IGC_GOTCL); - rd32(IGC_GOTCH); /* clear GOTCL */ - adapter->stats.rnbc += rd32(IGC_RNBC); - adapter->stats.ruc += rd32(IGC_RUC); - adapter->stats.rfc += rd32(IGC_RFC); - adapter->stats.rjc += rd32(IGC_RJC); - adapter->stats.tor += rd32(IGC_TORH); - adapter->stats.tot += rd32(IGC_TOTH); - adapter->stats.tpr += rd32(IGC_TPR); - - adapter->stats.ptc64 += rd32(IGC_PTC64); - adapter->stats.ptc127 += rd32(IGC_PTC127); - adapter->stats.ptc255 += rd32(IGC_PTC255); - adapter->stats.ptc511 += rd32(IGC_PTC511); - adapter->stats.ptc1023 += rd32(IGC_PTC1023); - adapter->stats.ptc1522 += rd32(IGC_PTC1522); - - adapter->stats.mptc += rd32(IGC_MPTC); - adapter->stats.bptc += rd32(IGC_BPTC); - - adapter->stats.tpt += rd32(IGC_TPT); - adapter->stats.colc += rd32(IGC_COLC); - - adapter->stats.algnerrc += rd32(IGC_ALGNERRC); - - adapter->stats.tsctc += rd32(IGC_TSCTC); - adapter->stats.tsctfc += rd32(IGC_TSCTFC); - - adapter->stats.iac += rd32(IGC_IAC); - adapter->stats.icrxoc += rd32(IGC_ICRXOC); - adapter->stats.icrxptc += rd32(IGC_ICRXPTC); - adapter->stats.icrxatc += rd32(IGC_ICRXATC); - adapter->stats.ictxptc += rd32(IGC_ICTXPTC); - adapter->stats.ictxatc += rd32(IGC_ICTXATC); - adapter->stats.ictxqec += rd32(IGC_ICTXQEC); - adapter->stats.ictxqmtc += rd32(IGC_ICTXQMTC); - adapter->stats.icrxdmtc += rd32(IGC_ICRXDMTC); - - /* Fill out the OS statistics structure */ - net_stats->multicast = adapter->stats.mprc; - net_stats->collisions = adapter->stats.colc; - - /* Rx Errors */ - - /* RLEC on some newer hardware can be incorrect so build - * our own version based on RUC and ROC - */ - net_stats->rx_errors = adapter->stats.rxerrc + - adapter->stats.crcerrs + adapter->stats.algnerrc + - adapter->stats.ruc + adapter->stats.roc + - adapter->stats.cexterr; - net_stats->rx_length_errors = adapter->stats.ruc + - adapter->stats.roc; - net_stats->rx_crc_errors = adapter->stats.crcerrs; - net_stats->rx_frame_errors = adapter->stats.algnerrc; - net_stats->rx_missed_errors = adapter->stats.mpc; - - /* Tx Errors */ - net_stats->tx_errors = adapter->stats.ecol + - adapter->stats.latecol; - net_stats->tx_aborted_errors = adapter->stats.ecol; - net_stats->tx_window_errors = adapter->stats.latecol; - net_stats->tx_carrier_errors = adapter->stats.tncrs; + /* conservative mode (itr 3) eliminates the lowest_latency setting */ + if (current_itr == lowest_latency && + ((q_vector->rx.ring && adapter->rx_itr_setting == 3) || + (!q_vector->rx.ring && adapter->tx_itr_setting == 3))) + current_itr = low_latency; - /* Tx Dropped needs to be maintained elsewhere */ + switch (current_itr) { + /* counts and packets in update_itr are dependent on these numbers */ + case lowest_latency: + new_itr = IGC_70K_ITR; /* 70,000 ints/sec */ + break; + case low_latency: + new_itr = IGC_20K_ITR; /* 20,000 ints/sec */ + break; + case bulk_latency: + new_itr = IGC_4K_ITR; /* 4,000 ints/sec */ + break; + default: + break; + } - /* Management Stats */ - adapter->stats.mgptc += rd32(IGC_MGTPTC); - adapter->stats.mgprc += rd32(IGC_MGTPRC); - adapter->stats.mgpdc += rd32(IGC_MGTPDC); +set_itr_now: + if (new_itr != q_vector->itr_val) { + /* this attempts to bias the interrupt rate towards Bulk + * by adding intermediate steps when interrupt rate is + * increasing + */ + new_itr = new_itr > q_vector->itr_val ? + max((new_itr * q_vector->itr_val) / + (new_itr + (q_vector->itr_val >> 2)), + new_itr) : new_itr; + /* Don't write the value here; it resets the adapter's + * internal timer, and causes us to delay far longer than + * we should between interrupts. Instead, we write the ITR + * value at the beginning of the next interrupt so the timing + * ends up being correct. + */ + q_vector->itr_val = new_itr; + q_vector->set_itr = 1; + } } -static void igc_nfc_filter_exit(struct igc_adapter *adapter) +static void igc_reset_interrupt_capability(struct igc_adapter *adapter) { - struct igc_nfc_filter *rule; - - spin_lock(&adapter->nfc_lock); - - hlist_for_each_entry(rule, &adapter->nfc_filter_list, nfc_node) - igc_erase_filter(adapter, rule); + int v_idx = adapter->num_q_vectors; - hlist_for_each_entry(rule, &adapter->cls_flower_list, nfc_node) - igc_erase_filter(adapter, rule); + if (adapter->msix_entries) { + pci_disable_msix(adapter->pdev); + kfree(adapter->msix_entries); + adapter->msix_entries = NULL; + } else if (adapter->flags & IGC_FLAG_HAS_MSI) { + pci_disable_msi(adapter->pdev); + } - spin_unlock(&adapter->nfc_lock); + while (v_idx--) + igc_reset_q_vector(adapter, v_idx); } /** - * igc_down - Close the interface - * @adapter: board private structure + * igc_set_interrupt_capability - set MSI or MSI-X if supported + * @adapter: Pointer to adapter structure + * @msix: boolean value for MSI-X capability + * + * Attempt to configure interrupts using the best available + * capabilities of the hardware and kernel. */ -void igc_down(struct igc_adapter *adapter) +static void igc_set_interrupt_capability(struct igc_adapter *adapter, + bool msix) { - struct net_device *netdev = adapter->netdev; - struct igc_hw *hw = &adapter->hw; - u32 tctl, rctl; - int i = 0; - - set_bit(__IGC_DOWN, &adapter->state); - - /* disable receives in the hardware */ - rctl = rd32(IGC_RCTL); - wr32(IGC_RCTL, rctl & ~IGC_RCTL_EN); - /* flush and sleep below */ - - igc_nfc_filter_exit(adapter); - - /* set trans_start so we don't get spurious watchdogs during reset */ - netif_trans_update(netdev); + int numvecs, i; + int err; - netif_carrier_off(netdev); - netif_tx_stop_all_queues(netdev); + if (!msix) + goto msi_only; + adapter->flags |= IGC_FLAG_HAS_MSIX; - /* disable transmits in the hardware */ - tctl = rd32(IGC_TCTL); - tctl &= ~IGC_TCTL_EN; - wr32(IGC_TCTL, tctl); - /* flush both disables and wait for them to finish */ - wrfl(); - usleep_range(10000, 20000); + /* Number of supported queues. */ + adapter->num_rx_queues = adapter->rss_queues; - igc_irq_disable(adapter); + adapter->num_tx_queues = adapter->rss_queues; - adapter->flags &= ~IGC_FLAG_NEED_LINK_UPDATE; + /* start with one vector for every Rx queue */ + numvecs = adapter->num_rx_queues; - for (i = 0; i < adapter->num_q_vectors; i++) { - if (adapter->q_vector[i]) { - napi_synchronize(&adapter->q_vector[i]->napi); - napi_disable(&adapter->q_vector[i]->napi); - } - } + /* if Tx handler is separate add 1 for every Tx queue */ + if (!(adapter->flags & IGC_FLAG_QUEUE_PAIRS)) + numvecs += adapter->num_tx_queues; - del_timer_sync(&adapter->watchdog_timer); - del_timer_sync(&adapter->phy_info_timer); + /* store the number of vectors reserved for queues */ + adapter->num_q_vectors = numvecs; - /* record the stats before reset*/ - spin_lock(&adapter->stats64_lock); - igc_update_stats(adapter); - spin_unlock(&adapter->stats64_lock); + /* add 1 vector for link status interrupts */ + numvecs++; - adapter->link_speed = 0; - adapter->link_duplex = 0; + adapter->msix_entries = kcalloc(numvecs, sizeof(struct msix_entry), + GFP_KERNEL); - if (!pci_channel_offline(adapter->pdev)) - igc_reset(adapter); + if (!adapter->msix_entries) + return; - /* clear VLAN promisc flag so VFTA will be updated if necessary */ - adapter->flags &= ~IGC_FLAG_VLAN_PROMISC; + /* populate entry values */ + for (i = 0; i < numvecs; i++) + adapter->msix_entries[i].entry = i; - igc_clean_all_tx_rings(adapter); - igc_clean_all_rx_rings(adapter); -} + err = pci_enable_msix_range(adapter->pdev, + adapter->msix_entries, + numvecs, + numvecs); + if (err > 0) + return; -void igc_reinit_locked(struct igc_adapter *adapter) -{ - WARN_ON(in_interrupt()); - while (test_and_set_bit(__IGC_RESETTING, &adapter->state)) - usleep_range(1000, 2000); - igc_down(adapter); - igc_up(adapter); - clear_bit(__IGC_RESETTING, &adapter->state); -} + kfree(adapter->msix_entries); + adapter->msix_entries = NULL; -static void igc_reset_task(struct work_struct *work) -{ - struct igc_adapter *adapter; + igc_reset_interrupt_capability(adapter); - adapter = container_of(work, struct igc_adapter, reset_task); +msi_only: + adapter->flags &= ~IGC_FLAG_HAS_MSIX; - netdev_err(adapter->netdev, "Reset adapter\n"); - igc_reinit_locked(adapter); + adapter->rss_queues = 1; + adapter->flags |= IGC_FLAG_QUEUE_PAIRS; + adapter->num_rx_queues = 1; + adapter->num_tx_queues = 1; + adapter->num_q_vectors = 1; + if (!pci_enable_msi(adapter->pdev)) + adapter->flags |= IGC_FLAG_HAS_MSI; } /** - * igc_change_mtu - Change the Maximum Transfer Unit - * @netdev: network interface device structure - * @new_mtu: new value for maximum frame size + * igc_update_ring_itr - update the dynamic ITR value based on packet size + * @q_vector: pointer to q_vector * - * Returns 0 on success, negative on failure + * Stores a new ITR value based on strictly on packet size. This + * algorithm is less sophisticated than that used in igc_update_itr, + * due to the difficulty of synchronizing statistics across multiple + * receive rings. The divisors and thresholds used by this function + * were determined based on theoretical maximum wire speed and testing + * data, in order to minimize response time while increasing bulk + * throughput. + * NOTE: This function is called only when operating in a multiqueue + * receive environment. */ -static int igc_change_mtu(struct net_device *netdev, int new_mtu) +static void igc_update_ring_itr(struct igc_q_vector *q_vector) { - int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN; - struct igc_adapter *adapter = netdev_priv(netdev); + struct igc_adapter *adapter = q_vector->adapter; + int new_val = q_vector->itr_val; + int avg_wire_size = 0; + unsigned int packets; - /* adjust max frame to be at least the size of a standard frame */ - if (max_frame < (ETH_FRAME_LEN + ETH_FCS_LEN)) - max_frame = ETH_FRAME_LEN + ETH_FCS_LEN; + /* For non-gigabit speeds, just fix the interrupt rate at 4000 + * ints/sec - ITR timer value of 120 ticks. + */ + switch (adapter->link_speed) { + case SPEED_10: + case SPEED_100: + new_val = IGC_4K_ITR; + goto set_itr_val; + default: + break; + } - while (test_and_set_bit(__IGC_RESETTING, &adapter->state)) - usleep_range(1000, 2000); + packets = q_vector->rx.total_packets; + if (packets) + avg_wire_size = q_vector->rx.total_bytes / packets; - /* igc_down has a dependency on max_frame_size */ - adapter->max_frame_size = max_frame; + packets = q_vector->tx.total_packets; + if (packets) + avg_wire_size = max_t(u32, avg_wire_size, + q_vector->tx.total_bytes / packets); - if (netif_running(netdev)) - igc_down(adapter); + /* if avg_wire_size isn't set no work was done */ + if (!avg_wire_size) + goto clear_counts; - netdev_dbg(netdev, "changing MTU from %d to %d\n", - netdev->mtu, new_mtu); - netdev->mtu = new_mtu; + /* Add 24 bytes to size to account for CRC, preamble, and gap */ + avg_wire_size += 24; - if (netif_running(netdev)) - igc_up(adapter); + /* Don't starve jumbo frames */ + avg_wire_size = min(avg_wire_size, 3000); + + /* Give a little boost to mid-size frames */ + if (avg_wire_size > 300 && avg_wire_size < 1200) + new_val = avg_wire_size / 3; else - igc_reset(adapter); + new_val = avg_wire_size / 2; - clear_bit(__IGC_RESETTING, &adapter->state); + /* conservative mode (itr 3) eliminates the lowest_latency setting */ + if (new_val < IGC_20K_ITR && + ((q_vector->rx.ring && adapter->rx_itr_setting == 3) || + (!q_vector->rx.ring && adapter->tx_itr_setting == 3))) + new_val = IGC_20K_ITR; - return 0; +set_itr_val: + if (new_val != q_vector->itr_val) { + q_vector->itr_val = new_val; + q_vector->set_itr = 1; + } +clear_counts: + q_vector->rx.total_bytes = 0; + q_vector->rx.total_packets = 0; + q_vector->tx.total_bytes = 0; + q_vector->tx.total_packets = 0; } -/** - * igc_get_stats - Get System Network Statistics - * @netdev: network interface device structure - * - * Returns the address of the device statistics structure. - * The statistics are updated here and also from the timer callback. - */ -static struct net_device_stats *igc_get_stats(struct net_device *netdev) +static void igc_ring_irq_enable(struct igc_q_vector *q_vector) { - struct igc_adapter *adapter = netdev_priv(netdev); + struct igc_adapter *adapter = q_vector->adapter; + struct igc_hw *hw = &adapter->hw; - if (!test_bit(__IGC_RESETTING, &adapter->state)) - igc_update_stats(adapter); + if ((q_vector->rx.ring && (adapter->rx_itr_setting & 3)) || + (!q_vector->rx.ring && (adapter->tx_itr_setting & 3))) { + if (adapter->num_q_vectors == 1) + igc_set_itr(q_vector); + else + igc_update_ring_itr(q_vector); + } - /* only return the current stats */ - return &netdev->stats; + if (!test_bit(__IGC_DOWN, &adapter->state)) { + if (adapter->msix_entries) + wr32(IGC_EIMS, q_vector->eims_value); + else + igc_irq_enable(adapter); + } } -static netdev_features_t igc_fix_features(struct net_device *netdev, - netdev_features_t features) +static void igc_add_ring(struct igc_ring *ring, + struct igc_ring_container *head) { - /* Since there is no support for separate Rx/Tx vlan accel - * enable/disable make sure Tx flag is always in same state as Rx. - */ - if (features & NETIF_F_HW_VLAN_CTAG_RX) - features |= NETIF_F_HW_VLAN_CTAG_TX; - else - features &= ~NETIF_F_HW_VLAN_CTAG_TX; - - return features; + head->ring = ring; + head->count++; } -static int igc_set_features(struct net_device *netdev, - netdev_features_t features) +/** + * igc_cache_ring_register - Descriptor ring to register mapping + * @adapter: board private structure to initialize + * + * Once we know the feature-set enabled for the device, we'll cache + * the register offset the descriptor ring is assigned to. + */ +static void igc_cache_ring_register(struct igc_adapter *adapter) { - netdev_features_t changed = netdev->features ^ features; - struct igc_adapter *adapter = netdev_priv(netdev); - - /* Add VLAN support */ - if (!(changed & (NETIF_F_RXALL | NETIF_F_NTUPLE))) - return 0; - - if (!(features & NETIF_F_NTUPLE)) { - struct hlist_node *node2; - struct igc_nfc_filter *rule; + int i = 0, j = 0; - spin_lock(&adapter->nfc_lock); - hlist_for_each_entry_safe(rule, node2, - &adapter->nfc_filter_list, nfc_node) { - igc_erase_filter(adapter, rule); - hlist_del(&rule->nfc_node); - kfree(rule); - } - spin_unlock(&adapter->nfc_lock); - adapter->nfc_filter_count = 0; + switch (adapter->hw.mac.type) { + case igc_i225: + /* Fall through */ + default: + for (; i < adapter->num_rx_queues; i++) + adapter->rx_ring[i]->reg_idx = i; + for (; j < adapter->num_tx_queues; j++) + adapter->tx_ring[j]->reg_idx = j; + break; } - - netdev->features = features; - - if (netif_running(netdev)) - igc_reinit_locked(adapter); - else - igc_reset(adapter); - - return 1; } -static netdev_features_t -igc_features_check(struct sk_buff *skb, struct net_device *dev, - netdev_features_t features) +/** + * igc_poll - NAPI Rx polling callback + * @napi: napi polling structure + * @budget: count of how many packets we should handle + */ +static int igc_poll(struct napi_struct *napi, int budget) { - unsigned int network_hdr_len, mac_hdr_len; + struct igc_q_vector *q_vector = container_of(napi, + struct igc_q_vector, + napi); + bool clean_complete = true; + int work_done = 0; - /* Make certain the headers can be described by a context descriptor */ - mac_hdr_len = skb_network_header(skb) - skb->data; - if (unlikely(mac_hdr_len > IGC_MAX_MAC_HDR_LEN)) - return features & ~(NETIF_F_HW_CSUM | - NETIF_F_SCTP_CRC | - NETIF_F_HW_VLAN_CTAG_TX | - NETIF_F_TSO | - NETIF_F_TSO6); + if (q_vector->tx.ring) + clean_complete = igc_clean_tx_irq(q_vector, budget); - network_hdr_len = skb_checksum_start(skb) - skb_network_header(skb); - if (unlikely(network_hdr_len > IGC_MAX_NETWORK_HDR_LEN)) - return features & ~(NETIF_F_HW_CSUM | - NETIF_F_SCTP_CRC | - NETIF_F_TSO | - NETIF_F_TSO6); + if (q_vector->rx.ring) { + int cleaned = igc_clean_rx_irq(q_vector, budget); - /* We can only support IPv4 TSO in tunnels if we can mangle the - * inner IP ID field, so strip TSO if MANGLEID is not supported. + work_done += cleaned; + if (cleaned >= budget) + clean_complete = false; + } + + /* If all work not completed, return budget and keep polling */ + if (!clean_complete) + return budget; + + /* Exit the polling mode, but don't re-enable interrupts if stack might + * poll us due to busy-polling */ - if (skb->encapsulation && !(features & NETIF_F_TSO_MANGLEID)) - features &= ~NETIF_F_TSO; + if (likely(napi_complete_done(napi, work_done))) + igc_ring_irq_enable(q_vector); - return features; + return min(work_done, budget - 1); } -/* Add a MAC filter for 'addr' directing matching traffic to 'queue', - * 'flags' is used to indicate what kind of match is made, match is by - * default for the destination address, if matching by source address - * is desired the flag IGC_MAC_STATE_SRC_ADDR can be used. +/** + * igc_alloc_q_vector - Allocate memory for a single interrupt vector + * @adapter: board private structure to initialize + * @v_count: q_vectors allocated on adapter, used for ring interleaving + * @v_idx: index of vector in adapter struct + * @txr_count: total number of Tx rings to allocate + * @txr_idx: index of first Tx ring to allocate + * @rxr_count: total number of Rx rings to allocate + * @rxr_idx: index of first Rx ring to allocate + * + * We allocate one q_vector. If allocation fails we return -ENOMEM. */ -static int igc_add_mac_filter_flags(struct igc_adapter *adapter, - const u8 *addr, const u8 queue, - const u8 flags) +static int igc_alloc_q_vector(struct igc_adapter *adapter, + unsigned int v_count, unsigned int v_idx, + unsigned int txr_count, unsigned int txr_idx, + unsigned int rxr_count, unsigned int rxr_idx) { - struct igc_hw *hw = &adapter->hw; - int rar_entries = hw->mac.rar_entry_count; - int i; + struct igc_q_vector *q_vector; + struct igc_ring *ring; + int ring_count; - if (is_zero_ether_addr(addr)) - return -EINVAL; + /* igc only supports 1 Tx and/or 1 Rx queue per vector */ + if (txr_count > 1 || rxr_count > 1) + return -ENOMEM; - /* Search for the first empty entry in the MAC table. - * Do not touch entries at the end of the table reserved for the VF MAC - * addresses. - */ - for (i = 0; i < rar_entries; i++) { - if (!igc_mac_entry_can_be_used(&adapter->mac_table[i], - addr, flags)) - continue; + ring_count = txr_count + rxr_count; - ether_addr_copy(adapter->mac_table[i].addr, addr); - adapter->mac_table[i].queue = queue; - adapter->mac_table[i].state |= IGC_MAC_STATE_IN_USE | flags; + /* allocate q_vector and rings */ + q_vector = adapter->q_vector[v_idx]; + if (!q_vector) + q_vector = kzalloc(struct_size(q_vector, ring, ring_count), + GFP_KERNEL); + else + memset(q_vector, 0, struct_size(q_vector, ring, ring_count)); + if (!q_vector) + return -ENOMEM; - igc_rar_set_index(adapter, i); - return i; - } + /* initialize NAPI */ + netif_napi_add(adapter->netdev, &q_vector->napi, + igc_poll, 64); - return -ENOSPC; -} + /* tie q_vector and adapter together */ + adapter->q_vector[v_idx] = q_vector; + q_vector->adapter = adapter; -int igc_add_mac_steering_filter(struct igc_adapter *adapter, - const u8 *addr, u8 queue, u8 flags) -{ - return igc_add_mac_filter_flags(adapter, addr, queue, - IGC_MAC_STATE_QUEUE_STEERING | flags); -} + /* initialize work limits */ + q_vector->tx.work_limit = adapter->tx_work_limit; -/* Remove a MAC filter for 'addr' directing matching traffic to - * 'queue', 'flags' is used to indicate what kind of match need to be - * removed, match is by default for the destination address, if - * matching by source address is to be removed the flag - * IGC_MAC_STATE_SRC_ADDR can be used. - */ -static int igc_del_mac_filter_flags(struct igc_adapter *adapter, - const u8 *addr, const u8 queue, - const u8 flags) -{ - struct igc_hw *hw = &adapter->hw; - int rar_entries = hw->mac.rar_entry_count; - int i; + /* initialize ITR configuration */ + q_vector->itr_register = adapter->io_addr + IGC_EITR(0); + q_vector->itr_val = IGC_START_ITR; - if (is_zero_ether_addr(addr)) - return -EINVAL; + /* initialize pointer to rings */ + ring = q_vector->ring; - /* Search for matching entry in the MAC table based on given address - * and queue. Do not touch entries at the end of the table reserved - * for the VF MAC addresses. - */ - for (i = 0; i < rar_entries; i++) { - if (!(adapter->mac_table[i].state & IGC_MAC_STATE_IN_USE)) - continue; - if ((adapter->mac_table[i].state & flags) != flags) - continue; - if (adapter->mac_table[i].queue != queue) - continue; - if (!ether_addr_equal(adapter->mac_table[i].addr, addr)) - continue; + /* initialize ITR */ + if (rxr_count) { + /* rx or rx/tx vector */ + if (!adapter->rx_itr_setting || adapter->rx_itr_setting > 3) + q_vector->itr_val = adapter->rx_itr_setting; + } else { + /* tx only vector */ + if (!adapter->tx_itr_setting || adapter->tx_itr_setting > 3) + q_vector->itr_val = adapter->tx_itr_setting; + } - /* When a filter for the default address is "deleted", - * we return it to its initial configuration - */ - if (adapter->mac_table[i].state & IGC_MAC_STATE_DEFAULT) { - adapter->mac_table[i].state = - IGC_MAC_STATE_DEFAULT | IGC_MAC_STATE_IN_USE; - } else { - adapter->mac_table[i].state = 0; - adapter->mac_table[i].queue = 0; - memset(adapter->mac_table[i].addr, 0, ETH_ALEN); - } + if (txr_count) { + /* assign generic ring traits */ + ring->dev = &adapter->pdev->dev; + ring->netdev = adapter->netdev; - igc_rar_set_index(adapter, i); - return 0; + /* configure backlink on ring */ + ring->q_vector = q_vector; + + /* update q_vector Tx values */ + igc_add_ring(ring, &q_vector->tx); + + /* apply Tx specific ring traits */ + ring->count = adapter->tx_ring_count; + ring->queue_index = txr_idx; + + /* assign ring to adapter */ + adapter->tx_ring[txr_idx] = ring; + + /* push pointer to next ring */ + ring++; } - return -ENOENT; -} + if (rxr_count) { + /* assign generic ring traits */ + ring->dev = &adapter->pdev->dev; + ring->netdev = adapter->netdev; -int igc_del_mac_steering_filter(struct igc_adapter *adapter, - const u8 *addr, u8 queue, u8 flags) -{ - return igc_del_mac_filter_flags(adapter, addr, queue, - IGC_MAC_STATE_QUEUE_STEERING | flags); + /* configure backlink on ring */ + ring->q_vector = q_vector; + + /* update q_vector Rx values */ + igc_add_ring(ring, &q_vector->rx); + + /* apply Rx specific ring traits */ + ring->count = adapter->rx_ring_count; + ring->queue_index = rxr_idx; + + /* assign ring to adapter */ + adapter->rx_ring[rxr_idx] = ring; + } + + return 0; } /** - * igc_msix_other - msix other interrupt handler - * @irq: interrupt number - * @data: pointer to a q_vector + * igc_alloc_q_vectors - Allocate memory for interrupt vectors + * @adapter: board private structure to initialize + * + * We allocate one q_vector per queue interrupt. If allocation fails we + * return -ENOMEM. */ -static irqreturn_t igc_msix_other(int irq, void *data) +static int igc_alloc_q_vectors(struct igc_adapter *adapter) { - struct igc_adapter *adapter = data; - struct igc_hw *hw = &adapter->hw; - u32 icr = rd32(IGC_ICR); + int rxr_remaining = adapter->num_rx_queues; + int txr_remaining = adapter->num_tx_queues; + int rxr_idx = 0, txr_idx = 0, v_idx = 0; + int q_vectors = adapter->num_q_vectors; + int err; - /* reading ICR causes bit 31 of EICR to be cleared */ - if (icr & IGC_ICR_DRSTA) - schedule_work(&adapter->reset_task); + if (q_vectors >= (rxr_remaining + txr_remaining)) { + for (; rxr_remaining; v_idx++) { + err = igc_alloc_q_vector(adapter, q_vectors, v_idx, + 0, 0, 1, rxr_idx); - if (icr & IGC_ICR_DOUTSYNC) { - /* HW is reporting DMA is out of sync */ - adapter->stats.doosync++; - } + if (err) + goto err_out; - if (icr & IGC_ICR_LSC) { - hw->mac.get_link_status = 1; - /* guard against interrupt when we're going down */ - if (!test_bit(__IGC_DOWN, &adapter->state)) - mod_timer(&adapter->watchdog_timer, jiffies + 1); + /* update counts and index */ + rxr_remaining--; + rxr_idx++; + } } - wr32(IGC_EIMS, adapter->eims_other); - - return IRQ_HANDLED; -} - -static void igc_write_itr(struct igc_q_vector *q_vector) -{ - u32 itr_val = q_vector->itr_val & IGC_QVECTOR_MASK; - - if (!q_vector->set_itr) - return; + for (; v_idx < q_vectors; v_idx++) { + int rqpv = DIV_ROUND_UP(rxr_remaining, q_vectors - v_idx); + int tqpv = DIV_ROUND_UP(txr_remaining, q_vectors - v_idx); - if (!itr_val) - itr_val = IGC_ITR_VAL_MASK; + err = igc_alloc_q_vector(adapter, q_vectors, v_idx, + tqpv, txr_idx, rqpv, rxr_idx); - itr_val |= IGC_EITR_CNT_IGNR; + if (err) + goto err_out; - writel(itr_val, q_vector->itr_register); - q_vector->set_itr = 0; -} + /* update counts and index */ + rxr_remaining -= rqpv; + txr_remaining -= tqpv; + rxr_idx++; + txr_idx++; + } -static irqreturn_t igc_msix_ring(int irq, void *data) -{ - struct igc_q_vector *q_vector = data; + return 0; - /* Write the ITR value calculated from the previous interrupt. */ - igc_write_itr(q_vector); +err_out: + adapter->num_tx_queues = 0; + adapter->num_rx_queues = 0; + adapter->num_q_vectors = 0; - napi_schedule(&q_vector->napi); + while (v_idx--) + igc_free_q_vector(adapter, v_idx); - return IRQ_HANDLED; + return -ENOMEM; } /** - * igc_request_msix - Initialize MSI-X interrupts + * igc_init_interrupt_scheme - initialize interrupts, allocate queues/vectors * @adapter: Pointer to adapter structure + * @msix: boolean for MSI-X capability * - * igc_request_msix allocates MSI-X vectors and requests interrupts from the - * kernel. + * This function initializes the interrupts and allocates all of the queues. */ -static int igc_request_msix(struct igc_adapter *adapter) +static int igc_init_interrupt_scheme(struct igc_adapter *adapter, bool msix) { - int i = 0, err = 0, vector = 0, free_vector = 0; - struct net_device *netdev = adapter->netdev; + struct pci_dev *pdev = adapter->pdev; + int err = 0; - err = request_irq(adapter->msix_entries[vector].vector, - &igc_msix_other, 0, netdev->name, adapter); - if (err) - goto err_out; + igc_set_interrupt_capability(adapter, msix); - for (i = 0; i < adapter->num_q_vectors; i++) { - struct igc_q_vector *q_vector = adapter->q_vector[i]; + err = igc_alloc_q_vectors(adapter); + if (err) { + dev_err(&pdev->dev, "Unable to allocate memory for vectors\n"); + goto err_alloc_q_vectors; + } - vector++; - - q_vector->itr_register = adapter->io_addr + IGC_EITR(vector); - - if (q_vector->rx.ring && q_vector->tx.ring) - sprintf(q_vector->name, "%s-TxRx-%u", netdev->name, - q_vector->rx.ring->queue_index); - else if (q_vector->tx.ring) - sprintf(q_vector->name, "%s-tx-%u", netdev->name, - q_vector->tx.ring->queue_index); - else if (q_vector->rx.ring) - sprintf(q_vector->name, "%s-rx-%u", netdev->name, - q_vector->rx.ring->queue_index); - else - sprintf(q_vector->name, "%s-unused", netdev->name); - - err = request_irq(adapter->msix_entries[vector].vector, - igc_msix_ring, 0, q_vector->name, - q_vector); - if (err) - goto err_free; - } + igc_cache_ring_register(adapter); - igc_configure_msix(adapter); return 0; -err_free: - /* free already assigned IRQs */ - free_irq(adapter->msix_entries[free_vector++].vector, adapter); - - vector--; - for (i = 0; i < vector; i++) { - free_irq(adapter->msix_entries[free_vector++].vector, - adapter->q_vector[i]); - } -err_out: +err_alloc_q_vectors: + igc_reset_interrupt_capability(adapter); return err; } /** - * igc_reset_q_vector - Reset config for interrupt vector + * igc_sw_init - Initialize general software structures (struct igc_adapter) * @adapter: board private structure to initialize - * @v_idx: Index of vector to be reset * - * If NAPI is enabled it will delete any references to the - * NAPI struct. This is preparation for igc_free_q_vector. + * igc_sw_init initializes the Adapter private data structure. + * Fields are initialized based on PCI device information and + * OS network device settings (MTU size). */ -static void igc_reset_q_vector(struct igc_adapter *adapter, int v_idx) +static int igc_sw_init(struct igc_adapter *adapter) { - struct igc_q_vector *q_vector = adapter->q_vector[v_idx]; - - /* if we're coming from igc_set_interrupt_capability, the vectors are - * not yet allocated - */ - if (!q_vector) - return; - - if (q_vector->tx.ring) - adapter->tx_ring[q_vector->tx.ring->queue_index] = NULL; + struct net_device *netdev = adapter->netdev; + struct pci_dev *pdev = adapter->pdev; + struct igc_hw *hw = &adapter->hw; - if (q_vector->rx.ring) - adapter->rx_ring[q_vector->rx.ring->queue_index] = NULL; + int size = sizeof(struct igc_mac_addr) * hw->mac.rar_entry_count; - netif_napi_del(&q_vector->napi); -} + pci_read_config_word(pdev, PCI_COMMAND, &hw->bus.pci_cmd_word); -static void igc_reset_interrupt_capability(struct igc_adapter *adapter) -{ - int v_idx = adapter->num_q_vectors; + /* set default ring sizes */ + adapter->tx_ring_count = IGC_DEFAULT_TXD; + adapter->rx_ring_count = IGC_DEFAULT_RXD; - if (adapter->msix_entries) { - pci_disable_msix(adapter->pdev); - kfree(adapter->msix_entries); - adapter->msix_entries = NULL; - } else if (adapter->flags & IGC_FLAG_HAS_MSI) { - pci_disable_msi(adapter->pdev); - } + /* set default ITR values */ + adapter->rx_itr_setting = IGC_DEFAULT_ITR; + adapter->tx_itr_setting = IGC_DEFAULT_ITR; - while (v_idx--) - igc_reset_q_vector(adapter, v_idx); -} + /* set default work limits */ + adapter->tx_work_limit = IGC_DEFAULT_TX_WORK; -/** - * igc_set_interrupt_capability - set MSI or MSI-X if supported - * @adapter: Pointer to adapter structure - * @msix: boolean value for MSI-X capability - * - * Attempt to configure interrupts using the best available - * capabilities of the hardware and kernel. - */ -static void igc_set_interrupt_capability(struct igc_adapter *adapter, - bool msix) -{ - int numvecs, i; - int err; + /* adjust max frame to be at least the size of a standard frame */ + adapter->max_frame_size = netdev->mtu + ETH_HLEN + ETH_FCS_LEN + + VLAN_HLEN; + adapter->min_frame_size = ETH_ZLEN + ETH_FCS_LEN; - if (!msix) - goto msi_only; + spin_lock_init(&adapter->nfc_lock); + spin_lock_init(&adapter->stats64_lock); + /* Assume MSI-X interrupts, will be checked during IRQ allocation */ adapter->flags |= IGC_FLAG_HAS_MSIX; - /* Number of supported queues. */ - adapter->num_rx_queues = adapter->rss_queues; + adapter->mac_table = kzalloc(size, GFP_ATOMIC); + if (!adapter->mac_table) + return -ENOMEM; - adapter->num_tx_queues = adapter->rss_queues; + igc_init_queue_configuration(adapter); - /* start with one vector for every Rx queue */ - numvecs = adapter->num_rx_queues; + /* This call may decrease the number of queues */ + if (igc_init_interrupt_scheme(adapter, true)) { + dev_err(&pdev->dev, "Unable to allocate memory for queues\n"); + return -ENOMEM; + } - /* if Tx handler is separate add 1 for every Tx queue */ - if (!(adapter->flags & IGC_FLAG_QUEUE_PAIRS)) - numvecs += adapter->num_tx_queues; + /* Explicitly disable IRQ since the NIC can be in any state. */ + igc_irq_disable(adapter); - /* store the number of vectors reserved for queues */ - adapter->num_q_vectors = numvecs; + set_bit(__IGC_DOWN, &adapter->state); - /* add 1 vector for link status interrupts */ - numvecs++; + return 0; +} - adapter->msix_entries = kcalloc(numvecs, sizeof(struct msix_entry), - GFP_KERNEL); +/** + * igc_up - Open the interface and prepare it to handle traffic + * @adapter: board private structure + */ +void igc_up(struct igc_adapter *adapter) +{ + struct igc_hw *hw = &adapter->hw; + int i = 0; - if (!adapter->msix_entries) - return; + /* hardware has been reset, we need to reload some things */ + igc_configure(adapter); - /* populate entry values */ - for (i = 0; i < numvecs; i++) - adapter->msix_entries[i].entry = i; + clear_bit(__IGC_DOWN, &adapter->state); - err = pci_enable_msix_range(adapter->pdev, - adapter->msix_entries, - numvecs, - numvecs); - if (err > 0) - return; + for (i = 0; i < adapter->num_q_vectors; i++) + napi_enable(&adapter->q_vector[i]->napi); - kfree(adapter->msix_entries); - adapter->msix_entries = NULL; + if (adapter->msix_entries) + igc_configure_msix(adapter); + else + igc_assign_vector(adapter->q_vector[0], 0); - igc_reset_interrupt_capability(adapter); + /* Clear any pending interrupts. */ + rd32(IGC_ICR); + igc_irq_enable(adapter); -msi_only: - adapter->flags &= ~IGC_FLAG_HAS_MSIX; + netif_tx_start_all_queues(adapter->netdev); - adapter->rss_queues = 1; - adapter->flags |= IGC_FLAG_QUEUE_PAIRS; - adapter->num_rx_queues = 1; - adapter->num_tx_queues = 1; - adapter->num_q_vectors = 1; - if (!pci_enable_msi(adapter->pdev)) - adapter->flags |= IGC_FLAG_HAS_MSI; + /* start the watchdog. */ + hw->mac.get_link_status = 1; + schedule_work(&adapter->watchdog_task); } /** - * igc_free_q_vector - Free memory allocated for specific interrupt vector - * @adapter: board private structure to initialize - * @v_idx: Index of vector to be freed - * - * This function frees the memory allocated to the q_vector. + * igc_update_stats - Update the board statistics counters + * @adapter: board private structure */ -static void igc_free_q_vector(struct igc_adapter *adapter, int v_idx) +void igc_update_stats(struct igc_adapter *adapter) { - struct igc_q_vector *q_vector = adapter->q_vector[v_idx]; - - adapter->q_vector[v_idx] = NULL; + struct rtnl_link_stats64 *net_stats = &adapter->stats64; + struct pci_dev *pdev = adapter->pdev; + struct igc_hw *hw = &adapter->hw; + u64 _bytes, _packets; + u64 bytes, packets; + unsigned int start; + u32 mpc; + int i; - /* igc_get_stats64() might access the rings on this vector, - * we must wait a grace period before freeing it. + /* Prevent stats update while adapter is being reset, or if the pci + * connection is down. */ - if (q_vector) - kfree_rcu(q_vector, rcu); -} + if (adapter->link_speed == 0) + return; + if (pci_channel_offline(pdev)) + return; -/** - * igc_free_q_vectors - Free memory allocated for interrupt vectors - * @adapter: board private structure to initialize - * - * This function frees the memory allocated to the q_vectors. In addition if - * NAPI is enabled it will delete any references to the NAPI struct prior - * to freeing the q_vector. - */ -static void igc_free_q_vectors(struct igc_adapter *adapter) -{ - int v_idx = adapter->num_q_vectors; + packets = 0; + bytes = 0; - adapter->num_tx_queues = 0; - adapter->num_rx_queues = 0; - adapter->num_q_vectors = 0; + rcu_read_lock(); + for (i = 0; i < adapter->num_rx_queues; i++) { + struct igc_ring *ring = adapter->rx_ring[i]; + u32 rqdpc = rd32(IGC_RQDPC(i)); - while (v_idx--) { - igc_reset_q_vector(adapter, v_idx); - igc_free_q_vector(adapter, v_idx); - } -} + if (hw->mac.type >= igc_i225) + wr32(IGC_RQDPC(i), 0); -/** - * igc_clear_interrupt_scheme - reset the device to a state of no interrupts - * @adapter: Pointer to adapter structure - * - * This function resets the device so that it has 0 rx queues, tx queues, and - * MSI-X interrupts allocated. - */ -static void igc_clear_interrupt_scheme(struct igc_adapter *adapter) -{ - igc_free_q_vectors(adapter); - igc_reset_interrupt_capability(adapter); -} + if (rqdpc) { + ring->rx_stats.drops += rqdpc; + net_stats->rx_fifo_errors += rqdpc; + } -/* Need to wait a few seconds after link up to get diagnostic information from - * the phy - */ -static void igc_update_phy_info(struct timer_list *t) + do { + start = u64_stats_fetch_begin_irq(&ring->rx_syncp); + _bytes = ring->rx_stats.bytes; + _packets = ring->rx_stats.packets; + } while (u64_stats_fetch_retry_irq(&ring->rx_syncp, start)); + bytes += _bytes; + packets += _packets; + } + + net_stats->rx_bytes = bytes; + net_stats->rx_packets = packets; + + packets = 0; + bytes = 0; + for (i = 0; i < adapter->num_tx_queues; i++) { + struct igc_ring *ring = adapter->tx_ring[i]; + + do { + start = u64_stats_fetch_begin_irq(&ring->tx_syncp); + _bytes = ring->tx_stats.bytes; + _packets = ring->tx_stats.packets; + } while (u64_stats_fetch_retry_irq(&ring->tx_syncp, start)); + bytes += _bytes; + packets += _packets; + } + net_stats->tx_bytes = bytes; + net_stats->tx_packets = packets; + rcu_read_unlock(); + + /* read stats registers */ + adapter->stats.crcerrs += rd32(IGC_CRCERRS); + adapter->stats.gprc += rd32(IGC_GPRC); + adapter->stats.gorc += rd32(IGC_GORCL); + rd32(IGC_GORCH); /* clear GORCL */ + adapter->stats.bprc += rd32(IGC_BPRC); + adapter->stats.mprc += rd32(IGC_MPRC); + adapter->stats.roc += rd32(IGC_ROC); + + adapter->stats.prc64 += rd32(IGC_PRC64); + adapter->stats.prc127 += rd32(IGC_PRC127); + adapter->stats.prc255 += rd32(IGC_PRC255); + adapter->stats.prc511 += rd32(IGC_PRC511); + adapter->stats.prc1023 += rd32(IGC_PRC1023); + adapter->stats.prc1522 += rd32(IGC_PRC1522); + adapter->stats.symerrs += rd32(IGC_SYMERRS); + adapter->stats.sec += rd32(IGC_SEC); + + mpc = rd32(IGC_MPC); + adapter->stats.mpc += mpc; + net_stats->rx_fifo_errors += mpc; + adapter->stats.scc += rd32(IGC_SCC); + adapter->stats.ecol += rd32(IGC_ECOL); + adapter->stats.mcc += rd32(IGC_MCC); + adapter->stats.latecol += rd32(IGC_LATECOL); + adapter->stats.dc += rd32(IGC_DC); + adapter->stats.rlec += rd32(IGC_RLEC); + adapter->stats.xonrxc += rd32(IGC_XONRXC); + adapter->stats.xontxc += rd32(IGC_XONTXC); + adapter->stats.xoffrxc += rd32(IGC_XOFFRXC); + adapter->stats.xofftxc += rd32(IGC_XOFFTXC); + adapter->stats.fcruc += rd32(IGC_FCRUC); + adapter->stats.gptc += rd32(IGC_GPTC); + adapter->stats.gotc += rd32(IGC_GOTCL); + rd32(IGC_GOTCH); /* clear GOTCL */ + adapter->stats.rnbc += rd32(IGC_RNBC); + adapter->stats.ruc += rd32(IGC_RUC); + adapter->stats.rfc += rd32(IGC_RFC); + adapter->stats.rjc += rd32(IGC_RJC); + adapter->stats.tor += rd32(IGC_TORH); + adapter->stats.tot += rd32(IGC_TOTH); + adapter->stats.tpr += rd32(IGC_TPR); + + adapter->stats.ptc64 += rd32(IGC_PTC64); + adapter->stats.ptc127 += rd32(IGC_PTC127); + adapter->stats.ptc255 += rd32(IGC_PTC255); + adapter->stats.ptc511 += rd32(IGC_PTC511); + adapter->stats.ptc1023 += rd32(IGC_PTC1023); + adapter->stats.ptc1522 += rd32(IGC_PTC1522); + + adapter->stats.mptc += rd32(IGC_MPTC); + adapter->stats.bptc += rd32(IGC_BPTC); + + adapter->stats.tpt += rd32(IGC_TPT); + adapter->stats.colc += rd32(IGC_COLC); + + adapter->stats.algnerrc += rd32(IGC_ALGNERRC); + + adapter->stats.tsctc += rd32(IGC_TSCTC); + adapter->stats.tsctfc += rd32(IGC_TSCTFC); + + adapter->stats.iac += rd32(IGC_IAC); + adapter->stats.icrxoc += rd32(IGC_ICRXOC); + adapter->stats.icrxptc += rd32(IGC_ICRXPTC); + adapter->stats.icrxatc += rd32(IGC_ICRXATC); + adapter->stats.ictxptc += rd32(IGC_ICTXPTC); + adapter->stats.ictxatc += rd32(IGC_ICTXATC); + adapter->stats.ictxqec += rd32(IGC_ICTXQEC); + adapter->stats.ictxqmtc += rd32(IGC_ICTXQMTC); + adapter->stats.icrxdmtc += rd32(IGC_ICRXDMTC); + + /* Fill out the OS statistics structure */ + net_stats->multicast = adapter->stats.mprc; + net_stats->collisions = adapter->stats.colc; + + /* Rx Errors */ + + /* RLEC on some newer hardware can be incorrect so build + * our own version based on RUC and ROC + */ + net_stats->rx_errors = adapter->stats.rxerrc + + adapter->stats.crcerrs + adapter->stats.algnerrc + + adapter->stats.ruc + adapter->stats.roc + + adapter->stats.cexterr; + net_stats->rx_length_errors = adapter->stats.ruc + + adapter->stats.roc; + net_stats->rx_crc_errors = adapter->stats.crcerrs; + net_stats->rx_frame_errors = adapter->stats.algnerrc; + net_stats->rx_missed_errors = adapter->stats.mpc; + + /* Tx Errors */ + net_stats->tx_errors = adapter->stats.ecol + + adapter->stats.latecol; + net_stats->tx_aborted_errors = adapter->stats.ecol; + net_stats->tx_window_errors = adapter->stats.latecol; + net_stats->tx_carrier_errors = adapter->stats.tncrs; + + /* Tx Dropped needs to be maintained elsewhere */ + + /* Management Stats */ + adapter->stats.mgptc += rd32(IGC_MGTPTC); + adapter->stats.mgprc += rd32(IGC_MGTPRC); + adapter->stats.mgpdc += rd32(IGC_MGTPDC); +} + +static void igc_nfc_filter_exit(struct igc_adapter *adapter) { - struct igc_adapter *adapter = from_timer(adapter, t, phy_info_timer); + struct igc_nfc_filter *rule; - igc_get_phy_info(&adapter->hw); + spin_lock(&adapter->nfc_lock); + + hlist_for_each_entry(rule, &adapter->nfc_filter_list, nfc_node) + igc_erase_filter(adapter, rule); + + hlist_for_each_entry(rule, &adapter->cls_flower_list, nfc_node) + igc_erase_filter(adapter, rule); + + spin_unlock(&adapter->nfc_lock); } /** - * igc_has_link - check shared code for link and determine up/down - * @adapter: pointer to driver private info + * igc_down - Close the interface + * @adapter: board private structure */ -bool igc_has_link(struct igc_adapter *adapter) +void igc_down(struct igc_adapter *adapter) { + struct net_device *netdev = adapter->netdev; struct igc_hw *hw = &adapter->hw; - bool link_active = false; + u32 tctl, rctl; + int i = 0; - /* get_link_status is set on LSC (link status) interrupt or - * rx sequence error interrupt. get_link_status will stay - * false until the igc_check_for_link establishes link - * for copper adapters ONLY - */ - switch (hw->phy.media_type) { - case igc_media_type_copper: - if (!hw->mac.get_link_status) - return true; - hw->mac.ops.check_for_link(hw); - link_active = !hw->mac.get_link_status; - break; - default: - case igc_media_type_unknown: - break; - } + set_bit(__IGC_DOWN, &adapter->state); - if (hw->mac.type == igc_i225 && - hw->phy.id == I225_I_PHY_ID) { - if (!netif_carrier_ok(adapter->netdev)) { - adapter->flags &= ~IGC_FLAG_NEED_LINK_UPDATE; - } else if (!(adapter->flags & IGC_FLAG_NEED_LINK_UPDATE)) { - adapter->flags |= IGC_FLAG_NEED_LINK_UPDATE; - adapter->link_check_timeout = jiffies; + /* disable receives in the hardware */ + rctl = rd32(IGC_RCTL); + wr32(IGC_RCTL, rctl & ~IGC_RCTL_EN); + /* flush and sleep below */ + + igc_nfc_filter_exit(adapter); + + /* set trans_start so we don't get spurious watchdogs during reset */ + netif_trans_update(netdev); + + netif_carrier_off(netdev); + netif_tx_stop_all_queues(netdev); + + /* disable transmits in the hardware */ + tctl = rd32(IGC_TCTL); + tctl &= ~IGC_TCTL_EN; + wr32(IGC_TCTL, tctl); + /* flush both disables and wait for them to finish */ + wrfl(); + usleep_range(10000, 20000); + + igc_irq_disable(adapter); + + adapter->flags &= ~IGC_FLAG_NEED_LINK_UPDATE; + + for (i = 0; i < adapter->num_q_vectors; i++) { + if (adapter->q_vector[i]) { + napi_synchronize(&adapter->q_vector[i]->napi); + napi_disable(&adapter->q_vector[i]->napi); } } - return link_active; + del_timer_sync(&adapter->watchdog_timer); + del_timer_sync(&adapter->phy_info_timer); + + /* record the stats before reset*/ + spin_lock(&adapter->stats64_lock); + igc_update_stats(adapter); + spin_unlock(&adapter->stats64_lock); + + adapter->link_speed = 0; + adapter->link_duplex = 0; + + if (!pci_channel_offline(adapter->pdev)) + igc_reset(adapter); + + /* clear VLAN promisc flag so VFTA will be updated if necessary */ + adapter->flags &= ~IGC_FLAG_VLAN_PROMISC; + + igc_clean_all_tx_rings(adapter); + igc_clean_all_rx_rings(adapter); } -/** - * igc_watchdog - Timer Call-back - * @t: timer for the watchdog - */ -static void igc_watchdog(struct timer_list *t) +void igc_reinit_locked(struct igc_adapter *adapter) { - struct igc_adapter *adapter = from_timer(adapter, t, watchdog_timer); - /* Do the rest outside of interrupt context */ - schedule_work(&adapter->watchdog_task); + WARN_ON(in_interrupt()); + while (test_and_set_bit(__IGC_RESETTING, &adapter->state)) + usleep_range(1000, 2000); + igc_down(adapter); + igc_up(adapter); + clear_bit(__IGC_RESETTING, &adapter->state); } -static void igc_watchdog_task(struct work_struct *work) +static void igc_reset_task(struct work_struct *work) { - struct igc_adapter *adapter = container_of(work, - struct igc_adapter, - watchdog_task); - struct net_device *netdev = adapter->netdev; - struct igc_hw *hw = &adapter->hw; - struct igc_phy_info *phy = &hw->phy; - u16 phy_data, retry_count = 20; - u32 connsw; - u32 link; - int i; + struct igc_adapter *adapter; - link = igc_has_link(adapter); + adapter = container_of(work, struct igc_adapter, reset_task); - if (adapter->flags & IGC_FLAG_NEED_LINK_UPDATE) { - if (time_after(jiffies, (adapter->link_check_timeout + HZ))) - adapter->flags &= ~IGC_FLAG_NEED_LINK_UPDATE; - else - link = false; - } + netdev_err(adapter->netdev, "Reset adapter\n"); + igc_reinit_locked(adapter); +} - /* Force link down if we have fiber to swap to */ - if (adapter->flags & IGC_FLAG_MAS_ENABLE) { - if (hw->phy.media_type == igc_media_type_copper) { - connsw = rd32(IGC_CONNSW); - if (!(connsw & IGC_CONNSW_AUTOSENSE_EN)) - link = 0; - } - } - if (link) { - if (!netif_carrier_ok(netdev)) { - u32 ctrl; +/** + * igc_change_mtu - Change the Maximum Transfer Unit + * @netdev: network interface device structure + * @new_mtu: new value for maximum frame size + * + * Returns 0 on success, negative on failure + */ +static int igc_change_mtu(struct net_device *netdev, int new_mtu) +{ + int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN; + struct igc_adapter *adapter = netdev_priv(netdev); - hw->mac.ops.get_speed_and_duplex(hw, - &adapter->link_speed, - &adapter->link_duplex); + /* adjust max frame to be at least the size of a standard frame */ + if (max_frame < (ETH_FRAME_LEN + ETH_FCS_LEN)) + max_frame = ETH_FRAME_LEN + ETH_FCS_LEN; - ctrl = rd32(IGC_CTRL); - /* Link status message must follow this format */ - netdev_info(netdev, - "igc: %s NIC Link is Up %d Mbps %s Duplex, Flow Control: %s\n", - netdev->name, - adapter->link_speed, - adapter->link_duplex == FULL_DUPLEX ? - "Full" : "Half", - (ctrl & IGC_CTRL_TFCE) && - (ctrl & IGC_CTRL_RFCE) ? "RX/TX" : - (ctrl & IGC_CTRL_RFCE) ? "RX" : - (ctrl & IGC_CTRL_TFCE) ? "TX" : "None"); + while (test_and_set_bit(__IGC_RESETTING, &adapter->state)) + usleep_range(1000, 2000); - /* check if SmartSpeed worked */ - igc_check_downshift(hw); - if (phy->speed_downgraded) - netdev_warn(netdev, "Link Speed was downgraded by SmartSpeed\n"); + /* igc_down has a dependency on max_frame_size */ + adapter->max_frame_size = max_frame; - /* adjust timeout factor according to speed/duplex */ - adapter->tx_timeout_factor = 1; - switch (adapter->link_speed) { - case SPEED_10: - adapter->tx_timeout_factor = 14; - break; - case SPEED_100: - /* maybe add some timeout factor ? */ - break; - } + if (netif_running(netdev)) + igc_down(adapter); - if (adapter->link_speed != SPEED_1000) - goto no_wait; + netdev_dbg(netdev, "changing MTU from %d to %d\n", + netdev->mtu, new_mtu); + netdev->mtu = new_mtu; - /* wait for Remote receiver status OK */ -retry_read_status: - if (!igc_read_phy_reg(hw, PHY_1000T_STATUS, - &phy_data)) { - if (!(phy_data & SR_1000T_REMOTE_RX_STATUS) && - retry_count) { - msleep(100); - retry_count--; - goto retry_read_status; - } else if (!retry_count) { - dev_err(&adapter->pdev->dev, "exceed max 2 second\n"); - } - } else { - dev_err(&adapter->pdev->dev, "read 1000Base-T Status Reg\n"); - } -no_wait: - netif_carrier_on(netdev); + if (netif_running(netdev)) + igc_up(adapter); + else + igc_reset(adapter); - /* link state has changed, schedule phy info update */ - if (!test_bit(__IGC_DOWN, &adapter->state)) - mod_timer(&adapter->phy_info_timer, - round_jiffies(jiffies + 2 * HZ)); - } - } else { - if (netif_carrier_ok(netdev)) { - adapter->link_speed = 0; - adapter->link_duplex = 0; + clear_bit(__IGC_RESETTING, &adapter->state); - /* Links status message must follow this format */ - netdev_info(netdev, "igc: %s NIC Link is Down\n", - netdev->name); - netif_carrier_off(netdev); + return 0; +} - /* link state has changed, schedule phy info update */ - if (!test_bit(__IGC_DOWN, &adapter->state)) - mod_timer(&adapter->phy_info_timer, - round_jiffies(jiffies + 2 * HZ)); +/** + * igc_get_stats - Get System Network Statistics + * @netdev: network interface device structure + * + * Returns the address of the device statistics structure. + * The statistics are updated here and also from the timer callback. + */ +static struct net_device_stats *igc_get_stats(struct net_device *netdev) +{ + struct igc_adapter *adapter = netdev_priv(netdev); - /* link is down, time to check for alternate media */ - if (adapter->flags & IGC_FLAG_MAS_ENABLE) { - if (adapter->flags & IGC_FLAG_MEDIA_RESET) { - schedule_work(&adapter->reset_task); - /* return immediately */ - return; - } - } + if (!test_bit(__IGC_RESETTING, &adapter->state)) + igc_update_stats(adapter); - /* also check for alternate media here */ - } else if (!netif_carrier_ok(netdev) && - (adapter->flags & IGC_FLAG_MAS_ENABLE)) { - if (adapter->flags & IGC_FLAG_MEDIA_RESET) { - schedule_work(&adapter->reset_task); - /* return immediately */ - return; - } - } - } + /* only return the current stats */ + return &netdev->stats; +} + +static netdev_features_t igc_fix_features(struct net_device *netdev, + netdev_features_t features) +{ + /* Since there is no support for separate Rx/Tx vlan accel + * enable/disable make sure Tx flag is always in same state as Rx. + */ + if (features & NETIF_F_HW_VLAN_CTAG_RX) + features |= NETIF_F_HW_VLAN_CTAG_TX; + else + features &= ~NETIF_F_HW_VLAN_CTAG_TX; + + return features; +} + +static int igc_set_features(struct net_device *netdev, + netdev_features_t features) +{ + netdev_features_t changed = netdev->features ^ features; + struct igc_adapter *adapter = netdev_priv(netdev); - spin_lock(&adapter->stats64_lock); - igc_update_stats(adapter); - spin_unlock(&adapter->stats64_lock); + /* Add VLAN support */ + if (!(changed & (NETIF_F_RXALL | NETIF_F_NTUPLE))) + return 0; - for (i = 0; i < adapter->num_tx_queues; i++) { - struct igc_ring *tx_ring = adapter->tx_ring[i]; + if (!(features & NETIF_F_NTUPLE)) { + struct hlist_node *node2; + struct igc_nfc_filter *rule; - if (!netif_carrier_ok(netdev)) { - /* We've lost link, so the controller stops DMA, - * but we've got queued Tx work that's never going - * to get done, so reset controller to flush Tx. - * (Do the reset outside of interrupt context). - */ - if (igc_desc_unused(tx_ring) + 1 < tx_ring->count) { - adapter->tx_timeout_count++; - schedule_work(&adapter->reset_task); - /* return immediately since reset is imminent */ - return; - } + spin_lock(&adapter->nfc_lock); + hlist_for_each_entry_safe(rule, node2, + &adapter->nfc_filter_list, nfc_node) { + igc_erase_filter(adapter, rule); + hlist_del(&rule->nfc_node); + kfree(rule); } - - /* Force detection of hung controller every watchdog period */ - set_bit(IGC_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags); + spin_unlock(&adapter->nfc_lock); + adapter->nfc_filter_count = 0; } - /* Cause software interrupt to ensure Rx ring is cleaned */ - if (adapter->flags & IGC_FLAG_HAS_MSIX) { - u32 eics = 0; + netdev->features = features; - for (i = 0; i < adapter->num_q_vectors; i++) - eics |= adapter->q_vector[i]->eims_value; - wr32(IGC_EICS, eics); - } else { - wr32(IGC_ICS, IGC_ICS_RXDMT0); - } + if (netif_running(netdev)) + igc_reinit_locked(adapter); + else + igc_reset(adapter); - /* Reset the timer */ - if (!test_bit(__IGC_DOWN, &adapter->state)) { - if (adapter->flags & IGC_FLAG_NEED_LINK_UPDATE) - mod_timer(&adapter->watchdog_timer, - round_jiffies(jiffies + HZ)); - else - mod_timer(&adapter->watchdog_timer, - round_jiffies(jiffies + 2 * HZ)); - } + return 1; } -/** - * igc_update_ring_itr - update the dynamic ITR value based on packet size - * @q_vector: pointer to q_vector - * - * Stores a new ITR value based on strictly on packet size. This - * algorithm is less sophisticated than that used in igc_update_itr, - * due to the difficulty of synchronizing statistics across multiple - * receive rings. The divisors and thresholds used by this function - * were determined based on theoretical maximum wire speed and testing - * data, in order to minimize response time while increasing bulk - * throughput. - * NOTE: This function is called only when operating in a multiqueue - * receive environment. - */ -static void igc_update_ring_itr(struct igc_q_vector *q_vector) +static netdev_features_t +igc_features_check(struct sk_buff *skb, struct net_device *dev, + netdev_features_t features) { - struct igc_adapter *adapter = q_vector->adapter; - int new_val = q_vector->itr_val; - int avg_wire_size = 0; - unsigned int packets; + unsigned int network_hdr_len, mac_hdr_len; - /* For non-gigabit speeds, just fix the interrupt rate at 4000 - * ints/sec - ITR timer value of 120 ticks. - */ - switch (adapter->link_speed) { - case SPEED_10: - case SPEED_100: - new_val = IGC_4K_ITR; - goto set_itr_val; - default: - break; - } + /* Make certain the headers can be described by a context descriptor */ + mac_hdr_len = skb_network_header(skb) - skb->data; + if (unlikely(mac_hdr_len > IGC_MAX_MAC_HDR_LEN)) + return features & ~(NETIF_F_HW_CSUM | + NETIF_F_SCTP_CRC | + NETIF_F_HW_VLAN_CTAG_TX | + NETIF_F_TSO | + NETIF_F_TSO6); - packets = q_vector->rx.total_packets; - if (packets) - avg_wire_size = q_vector->rx.total_bytes / packets; + network_hdr_len = skb_checksum_start(skb) - skb_network_header(skb); + if (unlikely(network_hdr_len > IGC_MAX_NETWORK_HDR_LEN)) + return features & ~(NETIF_F_HW_CSUM | + NETIF_F_SCTP_CRC | + NETIF_F_TSO | + NETIF_F_TSO6); - packets = q_vector->tx.total_packets; - if (packets) - avg_wire_size = max_t(u32, avg_wire_size, - q_vector->tx.total_bytes / packets); + /* We can only support IPv4 TSO in tunnels if we can mangle the + * inner IP ID field, so strip TSO if MANGLEID is not supported. + */ + if (skb->encapsulation && !(features & NETIF_F_TSO_MANGLEID)) + features &= ~NETIF_F_TSO; - /* if avg_wire_size isn't set no work was done */ - if (!avg_wire_size) - goto clear_counts; + return features; +} - /* Add 24 bytes to size to account for CRC, preamble, and gap */ - avg_wire_size += 24; +/* Add a MAC filter for 'addr' directing matching traffic to 'queue', + * 'flags' is used to indicate what kind of match is made, match is by + * default for the destination address, if matching by source address + * is desired the flag IGC_MAC_STATE_SRC_ADDR can be used. + */ +static int igc_add_mac_filter_flags(struct igc_adapter *adapter, + const u8 *addr, const u8 queue, + const u8 flags) +{ + struct igc_hw *hw = &adapter->hw; + int rar_entries = hw->mac.rar_entry_count; + int i; - /* Don't starve jumbo frames */ - avg_wire_size = min(avg_wire_size, 3000); + if (is_zero_ether_addr(addr)) + return -EINVAL; - /* Give a little boost to mid-size frames */ - if (avg_wire_size > 300 && avg_wire_size < 1200) - new_val = avg_wire_size / 3; - else - new_val = avg_wire_size / 2; + /* Search for the first empty entry in the MAC table. + * Do not touch entries at the end of the table reserved for the VF MAC + * addresses. + */ + for (i = 0; i < rar_entries; i++) { + if (!igc_mac_entry_can_be_used(&adapter->mac_table[i], + addr, flags)) + continue; - /* conservative mode (itr 3) eliminates the lowest_latency setting */ - if (new_val < IGC_20K_ITR && - ((q_vector->rx.ring && adapter->rx_itr_setting == 3) || - (!q_vector->rx.ring && adapter->tx_itr_setting == 3))) - new_val = IGC_20K_ITR; + ether_addr_copy(adapter->mac_table[i].addr, addr); + adapter->mac_table[i].queue = queue; + adapter->mac_table[i].state |= IGC_MAC_STATE_IN_USE | flags; -set_itr_val: - if (new_val != q_vector->itr_val) { - q_vector->itr_val = new_val; - q_vector->set_itr = 1; + igc_rar_set_index(adapter, i); + return i; } -clear_counts: - q_vector->rx.total_bytes = 0; - q_vector->rx.total_packets = 0; - q_vector->tx.total_bytes = 0; - q_vector->tx.total_packets = 0; + + return -ENOSPC; } -/** - * igc_update_itr - update the dynamic ITR value based on statistics - * @q_vector: pointer to q_vector - * @ring_container: ring info to update the itr for - * - * Stores a new ITR value based on packets and byte - * counts during the last interrupt. The advantage of per interrupt - * computation is faster updates and more accurate ITR for the current - * traffic pattern. Constants in this function were computed - * based on theoretical maximum wire speed and thresholds were set based - * on testing data as well as attempting to minimize response time - * while increasing bulk throughput. - * NOTE: These calculations are only valid when operating in a single- - * queue environment. +int igc_add_mac_steering_filter(struct igc_adapter *adapter, + const u8 *addr, u8 queue, u8 flags) +{ + return igc_add_mac_filter_flags(adapter, addr, queue, + IGC_MAC_STATE_QUEUE_STEERING | flags); +} + +/* Remove a MAC filter for 'addr' directing matching traffic to + * 'queue', 'flags' is used to indicate what kind of match need to be + * removed, match is by default for the destination address, if + * matching by source address is to be removed the flag + * IGC_MAC_STATE_SRC_ADDR can be used. */ -static void igc_update_itr(struct igc_q_vector *q_vector, - struct igc_ring_container *ring_container) +static int igc_del_mac_filter_flags(struct igc_adapter *adapter, + const u8 *addr, const u8 queue, + const u8 flags) { - unsigned int packets = ring_container->total_packets; - unsigned int bytes = ring_container->total_bytes; - u8 itrval = ring_container->itr; + struct igc_hw *hw = &adapter->hw; + int rar_entries = hw->mac.rar_entry_count; + int i; - /* no packets, exit with status unchanged */ - if (packets == 0) - return; + if (is_zero_ether_addr(addr)) + return -EINVAL; - switch (itrval) { - case lowest_latency: - /* handle TSO and jumbo frames */ - if (bytes / packets > 8000) - itrval = bulk_latency; - else if ((packets < 5) && (bytes > 512)) - itrval = low_latency; - break; - case low_latency: /* 50 usec aka 20000 ints/s */ - if (bytes > 10000) { - /* this if handles the TSO accounting */ - if (bytes / packets > 8000) - itrval = bulk_latency; - else if ((packets < 10) || ((bytes / packets) > 1200)) - itrval = bulk_latency; - else if ((packets > 35)) - itrval = lowest_latency; - } else if (bytes / packets > 2000) { - itrval = bulk_latency; - } else if (packets <= 2 && bytes < 512) { - itrval = lowest_latency; - } - break; - case bulk_latency: /* 250 usec aka 4000 ints/s */ - if (bytes > 25000) { - if (packets > 35) - itrval = low_latency; - } else if (bytes < 1500) { - itrval = low_latency; + /* Search for matching entry in the MAC table based on given address + * and queue. Do not touch entries at the end of the table reserved + * for the VF MAC addresses. + */ + for (i = 0; i < rar_entries; i++) { + if (!(adapter->mac_table[i].state & IGC_MAC_STATE_IN_USE)) + continue; + if ((adapter->mac_table[i].state & flags) != flags) + continue; + if (adapter->mac_table[i].queue != queue) + continue; + if (!ether_addr_equal(adapter->mac_table[i].addr, addr)) + continue; + + /* When a filter for the default address is "deleted", + * we return it to its initial configuration + */ + if (adapter->mac_table[i].state & IGC_MAC_STATE_DEFAULT) { + adapter->mac_table[i].state = + IGC_MAC_STATE_DEFAULT | IGC_MAC_STATE_IN_USE; + } else { + adapter->mac_table[i].state = 0; + adapter->mac_table[i].queue = 0; + memset(adapter->mac_table[i].addr, 0, ETH_ALEN); } - break; + + igc_rar_set_index(adapter, i); + return 0; } - /* clear work counters since we have the values we need */ - ring_container->total_bytes = 0; - ring_container->total_packets = 0; + return -ENOENT; +} - /* write updated itr to ring container */ - ring_container->itr = itrval; +int igc_del_mac_steering_filter(struct igc_adapter *adapter, + const u8 *addr, u8 queue, u8 flags) +{ + return igc_del_mac_filter_flags(adapter, addr, queue, + IGC_MAC_STATE_QUEUE_STEERING | flags); } /** - * igc_intr_msi - Interrupt Handler + * igc_msix_other - msix other interrupt handler * @irq: interrupt number - * @data: pointer to a network interface device structure + * @data: pointer to a q_vector */ -static irqreturn_t igc_intr_msi(int irq, void *data) +static irqreturn_t igc_msix_other(int irq, void *data) { struct igc_adapter *adapter = data; - struct igc_q_vector *q_vector = adapter->q_vector[0]; struct igc_hw *hw = &adapter->hw; - /* read ICR disables interrupts using IAM */ u32 icr = rd32(IGC_ICR); - igc_write_itr(q_vector); - + /* reading ICR causes bit 31 of EICR to be cleared */ if (icr & IGC_ICR_DRSTA) schedule_work(&adapter->reset_task); @@ -3586,408 +3655,428 @@ static irqreturn_t igc_intr_msi(int irq, void *data) adapter->stats.doosync++; } - if (icr & (IGC_ICR_RXSEQ | IGC_ICR_LSC)) { + if (icr & IGC_ICR_LSC) { hw->mac.get_link_status = 1; + /* guard against interrupt when we're going down */ if (!test_bit(__IGC_DOWN, &adapter->state)) mod_timer(&adapter->watchdog_timer, jiffies + 1); } - napi_schedule(&q_vector->napi); + wr32(IGC_EIMS, adapter->eims_other); return IRQ_HANDLED; } -/** - * igc_intr - Legacy Interrupt Handler - * @irq: interrupt number - * @data: pointer to a network interface device structure - */ -static irqreturn_t igc_intr(int irq, void *data) +static void igc_write_itr(struct igc_q_vector *q_vector) { - struct igc_adapter *adapter = data; - struct igc_q_vector *q_vector = adapter->q_vector[0]; - struct igc_hw *hw = &adapter->hw; - /* Interrupt Auto-Mask...upon reading ICR, interrupts are masked. No - * need for the IMC write - */ - u32 icr = rd32(IGC_ICR); + u32 itr_val = q_vector->itr_val & IGC_QVECTOR_MASK; - /* IMS will not auto-mask if INT_ASSERTED is not set, and if it is - * not set, then the adapter didn't send an interrupt - */ - if (!(icr & IGC_ICR_INT_ASSERTED)) - return IRQ_NONE; + if (!q_vector->set_itr) + return; - igc_write_itr(q_vector); + if (!itr_val) + itr_val = IGC_ITR_VAL_MASK; - if (icr & IGC_ICR_DRSTA) - schedule_work(&adapter->reset_task); + itr_val |= IGC_EITR_CNT_IGNR; - if (icr & IGC_ICR_DOUTSYNC) { - /* HW is reporting DMA is out of sync */ - adapter->stats.doosync++; - } + writel(itr_val, q_vector->itr_register); + q_vector->set_itr = 0; +} - if (icr & (IGC_ICR_RXSEQ | IGC_ICR_LSC)) { - hw->mac.get_link_status = 1; - /* guard against interrupt when we're going down */ - if (!test_bit(__IGC_DOWN, &adapter->state)) - mod_timer(&adapter->watchdog_timer, jiffies + 1); - } +static irqreturn_t igc_msix_ring(int irq, void *data) +{ + struct igc_q_vector *q_vector = data; + + /* Write the ITR value calculated from the previous interrupt. */ + igc_write_itr(q_vector); napi_schedule(&q_vector->napi); return IRQ_HANDLED; } -static void igc_set_itr(struct igc_q_vector *q_vector) +/** + * igc_request_msix - Initialize MSI-X interrupts + * @adapter: Pointer to adapter structure + * + * igc_request_msix allocates MSI-X vectors and requests interrupts from the + * kernel. + */ +static int igc_request_msix(struct igc_adapter *adapter) { - struct igc_adapter *adapter = q_vector->adapter; - u32 new_itr = q_vector->itr_val; - u8 current_itr = 0; + int i = 0, err = 0, vector = 0, free_vector = 0; + struct net_device *netdev = adapter->netdev; - /* for non-gigabit speeds, just fix the interrupt rate at 4000 */ - switch (adapter->link_speed) { - case SPEED_10: - case SPEED_100: - current_itr = 0; - new_itr = IGC_4K_ITR; - goto set_itr_now; - default: - break; - } + err = request_irq(adapter->msix_entries[vector].vector, + &igc_msix_other, 0, netdev->name, adapter); + if (err) + goto err_out; - igc_update_itr(q_vector, &q_vector->tx); - igc_update_itr(q_vector, &q_vector->rx); + for (i = 0; i < adapter->num_q_vectors; i++) { + struct igc_q_vector *q_vector = adapter->q_vector[i]; - current_itr = max(q_vector->rx.itr, q_vector->tx.itr); + vector++; - /* conservative mode (itr 3) eliminates the lowest_latency setting */ - if (current_itr == lowest_latency && - ((q_vector->rx.ring && adapter->rx_itr_setting == 3) || - (!q_vector->rx.ring && adapter->tx_itr_setting == 3))) - current_itr = low_latency; + q_vector->itr_register = adapter->io_addr + IGC_EITR(vector); - switch (current_itr) { - /* counts and packets in update_itr are dependent on these numbers */ - case lowest_latency: - new_itr = IGC_70K_ITR; /* 70,000 ints/sec */ - break; - case low_latency: - new_itr = IGC_20K_ITR; /* 20,000 ints/sec */ - break; - case bulk_latency: - new_itr = IGC_4K_ITR; /* 4,000 ints/sec */ - break; - default: - break; - } + if (q_vector->rx.ring && q_vector->tx.ring) + sprintf(q_vector->name, "%s-TxRx-%u", netdev->name, + q_vector->rx.ring->queue_index); + else if (q_vector->tx.ring) + sprintf(q_vector->name, "%s-tx-%u", netdev->name, + q_vector->tx.ring->queue_index); + else if (q_vector->rx.ring) + sprintf(q_vector->name, "%s-rx-%u", netdev->name, + q_vector->rx.ring->queue_index); + else + sprintf(q_vector->name, "%s-unused", netdev->name); -set_itr_now: - if (new_itr != q_vector->itr_val) { - /* this attempts to bias the interrupt rate towards Bulk - * by adding intermediate steps when interrupt rate is - * increasing - */ - new_itr = new_itr > q_vector->itr_val ? - max((new_itr * q_vector->itr_val) / - (new_itr + (q_vector->itr_val >> 2)), - new_itr) : new_itr; - /* Don't write the value here; it resets the adapter's - * internal timer, and causes us to delay far longer than - * we should between interrupts. Instead, we write the ITR - * value at the beginning of the next interrupt so the timing - * ends up being correct. - */ - q_vector->itr_val = new_itr; - q_vector->set_itr = 1; + err = request_irq(adapter->msix_entries[vector].vector, + igc_msix_ring, 0, q_vector->name, + q_vector); + if (err) + goto err_free; } -} -static void igc_ring_irq_enable(struct igc_q_vector *q_vector) -{ - struct igc_adapter *adapter = q_vector->adapter; - struct igc_hw *hw = &adapter->hw; + igc_configure_msix(adapter); + return 0; - if ((q_vector->rx.ring && (adapter->rx_itr_setting & 3)) || - (!q_vector->rx.ring && (adapter->tx_itr_setting & 3))) { - if (adapter->num_q_vectors == 1) - igc_set_itr(q_vector); - else - igc_update_ring_itr(q_vector); - } +err_free: + /* free already assigned IRQs */ + free_irq(adapter->msix_entries[free_vector++].vector, adapter); - if (!test_bit(__IGC_DOWN, &adapter->state)) { - if (adapter->msix_entries) - wr32(IGC_EIMS, q_vector->eims_value); - else - igc_irq_enable(adapter); + vector--; + for (i = 0; i < vector; i++) { + free_irq(adapter->msix_entries[free_vector++].vector, + adapter->q_vector[i]); } +err_out: + return err; } /** - * igc_poll - NAPI Rx polling callback - * @napi: napi polling structure - * @budget: count of how many packets we should handle + * igc_clear_interrupt_scheme - reset the device to a state of no interrupts + * @adapter: Pointer to adapter structure + * + * This function resets the device so that it has 0 rx queues, tx queues, and + * MSI-X interrupts allocated. */ -static int igc_poll(struct napi_struct *napi, int budget) +static void igc_clear_interrupt_scheme(struct igc_adapter *adapter) { - struct igc_q_vector *q_vector = container_of(napi, - struct igc_q_vector, - napi); - bool clean_complete = true; - int work_done = 0; - - if (q_vector->tx.ring) - clean_complete = igc_clean_tx_irq(q_vector, budget); - - if (q_vector->rx.ring) { - int cleaned = igc_clean_rx_irq(q_vector, budget); - - work_done += cleaned; - if (cleaned >= budget) - clean_complete = false; - } - - /* If all work not completed, return budget and keep polling */ - if (!clean_complete) - return budget; + igc_free_q_vectors(adapter); + igc_reset_interrupt_capability(adapter); +} - /* Exit the polling mode, but don't re-enable interrupts if stack might - * poll us due to busy-polling - */ - if (likely(napi_complete_done(napi, work_done))) - igc_ring_irq_enable(q_vector); +/* Need to wait a few seconds after link up to get diagnostic information from + * the phy + */ +static void igc_update_phy_info(struct timer_list *t) +{ + struct igc_adapter *adapter = from_timer(adapter, t, phy_info_timer); - return min(work_done, budget - 1); + igc_get_phy_info(&adapter->hw); } -static void igc_add_ring(struct igc_ring *ring, - struct igc_ring_container *head) +/** + * igc_has_link - check shared code for link and determine up/down + * @adapter: pointer to driver private info + */ +bool igc_has_link(struct igc_adapter *adapter) { - head->ring = ring; - head->count++; + struct igc_hw *hw = &adapter->hw; + bool link_active = false; + + /* get_link_status is set on LSC (link status) interrupt or + * rx sequence error interrupt. get_link_status will stay + * false until the igc_check_for_link establishes link + * for copper adapters ONLY + */ + switch (hw->phy.media_type) { + case igc_media_type_copper: + if (!hw->mac.get_link_status) + return true; + hw->mac.ops.check_for_link(hw); + link_active = !hw->mac.get_link_status; + break; + default: + case igc_media_type_unknown: + break; + } + + if (hw->mac.type == igc_i225 && + hw->phy.id == I225_I_PHY_ID) { + if (!netif_carrier_ok(adapter->netdev)) { + adapter->flags &= ~IGC_FLAG_NEED_LINK_UPDATE; + } else if (!(adapter->flags & IGC_FLAG_NEED_LINK_UPDATE)) { + adapter->flags |= IGC_FLAG_NEED_LINK_UPDATE; + adapter->link_check_timeout = jiffies; + } + } + + return link_active; } /** - * igc_alloc_q_vector - Allocate memory for a single interrupt vector - * @adapter: board private structure to initialize - * @v_count: q_vectors allocated on adapter, used for ring interleaving - * @v_idx: index of vector in adapter struct - * @txr_count: total number of Tx rings to allocate - * @txr_idx: index of first Tx ring to allocate - * @rxr_count: total number of Rx rings to allocate - * @rxr_idx: index of first Rx ring to allocate - * - * We allocate one q_vector. If allocation fails we return -ENOMEM. + * igc_watchdog - Timer Call-back + * @t: timer for the watchdog */ -static int igc_alloc_q_vector(struct igc_adapter *adapter, - unsigned int v_count, unsigned int v_idx, - unsigned int txr_count, unsigned int txr_idx, - unsigned int rxr_count, unsigned int rxr_idx) +static void igc_watchdog(struct timer_list *t) { - struct igc_q_vector *q_vector; - struct igc_ring *ring; - int ring_count; + struct igc_adapter *adapter = from_timer(adapter, t, watchdog_timer); + /* Do the rest outside of interrupt context */ + schedule_work(&adapter->watchdog_task); +} - /* igc only supports 1 Tx and/or 1 Rx queue per vector */ - if (txr_count > 1 || rxr_count > 1) - return -ENOMEM; +static void igc_watchdog_task(struct work_struct *work) +{ + struct igc_adapter *adapter = container_of(work, + struct igc_adapter, + watchdog_task); + struct net_device *netdev = adapter->netdev; + struct igc_hw *hw = &adapter->hw; + struct igc_phy_info *phy = &hw->phy; + u16 phy_data, retry_count = 20; + u32 connsw; + u32 link; + int i; - ring_count = txr_count + rxr_count; + link = igc_has_link(adapter); - /* allocate q_vector and rings */ - q_vector = adapter->q_vector[v_idx]; - if (!q_vector) - q_vector = kzalloc(struct_size(q_vector, ring, ring_count), - GFP_KERNEL); - else - memset(q_vector, 0, struct_size(q_vector, ring, ring_count)); - if (!q_vector) - return -ENOMEM; + if (adapter->flags & IGC_FLAG_NEED_LINK_UPDATE) { + if (time_after(jiffies, (adapter->link_check_timeout + HZ))) + adapter->flags &= ~IGC_FLAG_NEED_LINK_UPDATE; + else + link = false; + } - /* initialize NAPI */ - netif_napi_add(adapter->netdev, &q_vector->napi, - igc_poll, 64); + /* Force link down if we have fiber to swap to */ + if (adapter->flags & IGC_FLAG_MAS_ENABLE) { + if (hw->phy.media_type == igc_media_type_copper) { + connsw = rd32(IGC_CONNSW); + if (!(connsw & IGC_CONNSW_AUTOSENSE_EN)) + link = 0; + } + } + if (link) { + if (!netif_carrier_ok(netdev)) { + u32 ctrl; - /* tie q_vector and adapter together */ - adapter->q_vector[v_idx] = q_vector; - q_vector->adapter = adapter; + hw->mac.ops.get_speed_and_duplex(hw, + &adapter->link_speed, + &adapter->link_duplex); - /* initialize work limits */ - q_vector->tx.work_limit = adapter->tx_work_limit; + ctrl = rd32(IGC_CTRL); + /* Link status message must follow this format */ + netdev_info(netdev, + "igc: %s NIC Link is Up %d Mbps %s Duplex, Flow Control: %s\n", + netdev->name, + adapter->link_speed, + adapter->link_duplex == FULL_DUPLEX ? + "Full" : "Half", + (ctrl & IGC_CTRL_TFCE) && + (ctrl & IGC_CTRL_RFCE) ? "RX/TX" : + (ctrl & IGC_CTRL_RFCE) ? "RX" : + (ctrl & IGC_CTRL_TFCE) ? "TX" : "None"); - /* initialize ITR configuration */ - q_vector->itr_register = adapter->io_addr + IGC_EITR(0); - q_vector->itr_val = IGC_START_ITR; + /* check if SmartSpeed worked */ + igc_check_downshift(hw); + if (phy->speed_downgraded) + netdev_warn(netdev, "Link Speed was downgraded by SmartSpeed\n"); - /* initialize pointer to rings */ - ring = q_vector->ring; + /* adjust timeout factor according to speed/duplex */ + adapter->tx_timeout_factor = 1; + switch (adapter->link_speed) { + case SPEED_10: + adapter->tx_timeout_factor = 14; + break; + case SPEED_100: + /* maybe add some timeout factor ? */ + break; + } - /* initialize ITR */ - if (rxr_count) { - /* rx or rx/tx vector */ - if (!adapter->rx_itr_setting || adapter->rx_itr_setting > 3) - q_vector->itr_val = adapter->rx_itr_setting; - } else { - /* tx only vector */ - if (!adapter->tx_itr_setting || adapter->tx_itr_setting > 3) - q_vector->itr_val = adapter->tx_itr_setting; - } + if (adapter->link_speed != SPEED_1000) + goto no_wait; - if (txr_count) { - /* assign generic ring traits */ - ring->dev = &adapter->pdev->dev; - ring->netdev = adapter->netdev; + /* wait for Remote receiver status OK */ +retry_read_status: + if (!igc_read_phy_reg(hw, PHY_1000T_STATUS, + &phy_data)) { + if (!(phy_data & SR_1000T_REMOTE_RX_STATUS) && + retry_count) { + msleep(100); + retry_count--; + goto retry_read_status; + } else if (!retry_count) { + dev_err(&adapter->pdev->dev, "exceed max 2 second\n"); + } + } else { + dev_err(&adapter->pdev->dev, "read 1000Base-T Status Reg\n"); + } +no_wait: + netif_carrier_on(netdev); - /* configure backlink on ring */ - ring->q_vector = q_vector; + /* link state has changed, schedule phy info update */ + if (!test_bit(__IGC_DOWN, &adapter->state)) + mod_timer(&adapter->phy_info_timer, + round_jiffies(jiffies + 2 * HZ)); + } + } else { + if (netif_carrier_ok(netdev)) { + adapter->link_speed = 0; + adapter->link_duplex = 0; - /* update q_vector Tx values */ - igc_add_ring(ring, &q_vector->tx); + /* Links status message must follow this format */ + netdev_info(netdev, "igc: %s NIC Link is Down\n", + netdev->name); + netif_carrier_off(netdev); - /* apply Tx specific ring traits */ - ring->count = adapter->tx_ring_count; - ring->queue_index = txr_idx; + /* link state has changed, schedule phy info update */ + if (!test_bit(__IGC_DOWN, &adapter->state)) + mod_timer(&adapter->phy_info_timer, + round_jiffies(jiffies + 2 * HZ)); - /* assign ring to adapter */ - adapter->tx_ring[txr_idx] = ring; + /* link is down, time to check for alternate media */ + if (adapter->flags & IGC_FLAG_MAS_ENABLE) { + if (adapter->flags & IGC_FLAG_MEDIA_RESET) { + schedule_work(&adapter->reset_task); + /* return immediately */ + return; + } + } - /* push pointer to next ring */ - ring++; + /* also check for alternate media here */ + } else if (!netif_carrier_ok(netdev) && + (adapter->flags & IGC_FLAG_MAS_ENABLE)) { + if (adapter->flags & IGC_FLAG_MEDIA_RESET) { + schedule_work(&adapter->reset_task); + /* return immediately */ + return; + } + } } - if (rxr_count) { - /* assign generic ring traits */ - ring->dev = &adapter->pdev->dev; - ring->netdev = adapter->netdev; + spin_lock(&adapter->stats64_lock); + igc_update_stats(adapter); + spin_unlock(&adapter->stats64_lock); - /* configure backlink on ring */ - ring->q_vector = q_vector; + for (i = 0; i < adapter->num_tx_queues; i++) { + struct igc_ring *tx_ring = adapter->tx_ring[i]; + + if (!netif_carrier_ok(netdev)) { + /* We've lost link, so the controller stops DMA, + * but we've got queued Tx work that's never going + * to get done, so reset controller to flush Tx. + * (Do the reset outside of interrupt context). + */ + if (igc_desc_unused(tx_ring) + 1 < tx_ring->count) { + adapter->tx_timeout_count++; + schedule_work(&adapter->reset_task); + /* return immediately since reset is imminent */ + return; + } + } - /* update q_vector Rx values */ - igc_add_ring(ring, &q_vector->rx); + /* Force detection of hung controller every watchdog period */ + set_bit(IGC_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags); + } - /* apply Rx specific ring traits */ - ring->count = adapter->rx_ring_count; - ring->queue_index = rxr_idx; + /* Cause software interrupt to ensure Rx ring is cleaned */ + if (adapter->flags & IGC_FLAG_HAS_MSIX) { + u32 eics = 0; - /* assign ring to adapter */ - adapter->rx_ring[rxr_idx] = ring; + for (i = 0; i < adapter->num_q_vectors; i++) + eics |= adapter->q_vector[i]->eims_value; + wr32(IGC_EICS, eics); + } else { + wr32(IGC_ICS, IGC_ICS_RXDMT0); } - return 0; + /* Reset the timer */ + if (!test_bit(__IGC_DOWN, &adapter->state)) { + if (adapter->flags & IGC_FLAG_NEED_LINK_UPDATE) + mod_timer(&adapter->watchdog_timer, + round_jiffies(jiffies + HZ)); + else + mod_timer(&adapter->watchdog_timer, + round_jiffies(jiffies + 2 * HZ)); + } } /** - * igc_alloc_q_vectors - Allocate memory for interrupt vectors - * @adapter: board private structure to initialize - * - * We allocate one q_vector per queue interrupt. If allocation fails we - * return -ENOMEM. + * igc_intr_msi - Interrupt Handler + * @irq: interrupt number + * @data: pointer to a network interface device structure */ -static int igc_alloc_q_vectors(struct igc_adapter *adapter) +static irqreturn_t igc_intr_msi(int irq, void *data) { - int rxr_remaining = adapter->num_rx_queues; - int txr_remaining = adapter->num_tx_queues; - int rxr_idx = 0, txr_idx = 0, v_idx = 0; - int q_vectors = adapter->num_q_vectors; - int err; + struct igc_adapter *adapter = data; + struct igc_q_vector *q_vector = adapter->q_vector[0]; + struct igc_hw *hw = &adapter->hw; + /* read ICR disables interrupts using IAM */ + u32 icr = rd32(IGC_ICR); - if (q_vectors >= (rxr_remaining + txr_remaining)) { - for (; rxr_remaining; v_idx++) { - err = igc_alloc_q_vector(adapter, q_vectors, v_idx, - 0, 0, 1, rxr_idx); + igc_write_itr(q_vector); - if (err) - goto err_out; + if (icr & IGC_ICR_DRSTA) + schedule_work(&adapter->reset_task); - /* update counts and index */ - rxr_remaining--; - rxr_idx++; - } + if (icr & IGC_ICR_DOUTSYNC) { + /* HW is reporting DMA is out of sync */ + adapter->stats.doosync++; } - for (; v_idx < q_vectors; v_idx++) { - int rqpv = DIV_ROUND_UP(rxr_remaining, q_vectors - v_idx); - int tqpv = DIV_ROUND_UP(txr_remaining, q_vectors - v_idx); - - err = igc_alloc_q_vector(adapter, q_vectors, v_idx, - tqpv, txr_idx, rqpv, rxr_idx); - - if (err) - goto err_out; - - /* update counts and index */ - rxr_remaining -= rqpv; - txr_remaining -= tqpv; - rxr_idx++; - txr_idx++; + if (icr & (IGC_ICR_RXSEQ | IGC_ICR_LSC)) { + hw->mac.get_link_status = 1; + if (!test_bit(__IGC_DOWN, &adapter->state)) + mod_timer(&adapter->watchdog_timer, jiffies + 1); } - return 0; - -err_out: - adapter->num_tx_queues = 0; - adapter->num_rx_queues = 0; - adapter->num_q_vectors = 0; - - while (v_idx--) - igc_free_q_vector(adapter, v_idx); + napi_schedule(&q_vector->napi); - return -ENOMEM; + return IRQ_HANDLED; } /** - * igc_cache_ring_register - Descriptor ring to register mapping - * @adapter: board private structure to initialize - * - * Once we know the feature-set enabled for the device, we'll cache - * the register offset the descriptor ring is assigned to. + * igc_intr - Legacy Interrupt Handler + * @irq: interrupt number + * @data: pointer to a network interface device structure */ -static void igc_cache_ring_register(struct igc_adapter *adapter) +static irqreturn_t igc_intr(int irq, void *data) { - int i = 0, j = 0; + struct igc_adapter *adapter = data; + struct igc_q_vector *q_vector = adapter->q_vector[0]; + struct igc_hw *hw = &adapter->hw; + /* Interrupt Auto-Mask...upon reading ICR, interrupts are masked. No + * need for the IMC write + */ + u32 icr = rd32(IGC_ICR); - switch (adapter->hw.mac.type) { - case igc_i225: - /* Fall through */ - default: - for (; i < adapter->num_rx_queues; i++) - adapter->rx_ring[i]->reg_idx = i; - for (; j < adapter->num_tx_queues; j++) - adapter->tx_ring[j]->reg_idx = j; - break; - } -} + /* IMS will not auto-mask if INT_ASSERTED is not set, and if it is + * not set, then the adapter didn't send an interrupt + */ + if (!(icr & IGC_ICR_INT_ASSERTED)) + return IRQ_NONE; -/** - * igc_init_interrupt_scheme - initialize interrupts, allocate queues/vectors - * @adapter: Pointer to adapter structure - * @msix: boolean for MSI-X capability - * - * This function initializes the interrupts and allocates all of the queues. - */ -static int igc_init_interrupt_scheme(struct igc_adapter *adapter, bool msix) -{ - struct pci_dev *pdev = adapter->pdev; - int err = 0; + igc_write_itr(q_vector); - igc_set_interrupt_capability(adapter, msix); + if (icr & IGC_ICR_DRSTA) + schedule_work(&adapter->reset_task); - err = igc_alloc_q_vectors(adapter); - if (err) { - dev_err(&pdev->dev, "Unable to allocate memory for vectors\n"); - goto err_alloc_q_vectors; + if (icr & IGC_ICR_DOUTSYNC) { + /* HW is reporting DMA is out of sync */ + adapter->stats.doosync++; } - igc_cache_ring_register(adapter); + if (icr & (IGC_ICR_RXSEQ | IGC_ICR_LSC)) { + hw->mac.get_link_status = 1; + /* guard against interrupt when we're going down */ + if (!test_bit(__IGC_DOWN, &adapter->state)) + mod_timer(&adapter->watchdog_timer, jiffies + 1); + } - return 0; + napi_schedule(&q_vector->napi); -err_alloc_q_vectors: - igc_reset_interrupt_capability(adapter); - return err; + return IRQ_HANDLED; } static void igc_free_irq(struct igc_adapter *adapter) @@ -4776,98 +4865,6 @@ static struct pci_driver igc_driver = { .shutdown = igc_shutdown, }; -void igc_set_flag_queue_pairs(struct igc_adapter *adapter, - const u32 max_rss_queues) -{ - /* Determine if we need to pair queues. */ - /* If rss_queues > half of max_rss_queues, pair the queues in - * order to conserve interrupts due to limited supply. - */ - if (adapter->rss_queues > (max_rss_queues / 2)) - adapter->flags |= IGC_FLAG_QUEUE_PAIRS; - else - adapter->flags &= ~IGC_FLAG_QUEUE_PAIRS; -} - -unsigned int igc_get_max_rss_queues(struct igc_adapter *adapter) -{ - unsigned int max_rss_queues; - - /* Determine the maximum number of RSS queues supported. */ - max_rss_queues = IGC_MAX_RX_QUEUES; - - return max_rss_queues; -} - -static void igc_init_queue_configuration(struct igc_adapter *adapter) -{ - u32 max_rss_queues; - - max_rss_queues = igc_get_max_rss_queues(adapter); - adapter->rss_queues = min_t(u32, max_rss_queues, num_online_cpus()); - - igc_set_flag_queue_pairs(adapter, max_rss_queues); -} - -/** - * igc_sw_init - Initialize general software structures (struct igc_adapter) - * @adapter: board private structure to initialize - * - * igc_sw_init initializes the Adapter private data structure. - * Fields are initialized based on PCI device information and - * OS network device settings (MTU size). - */ -static int igc_sw_init(struct igc_adapter *adapter) -{ - struct net_device *netdev = adapter->netdev; - struct pci_dev *pdev = adapter->pdev; - struct igc_hw *hw = &adapter->hw; - - int size = sizeof(struct igc_mac_addr) * hw->mac.rar_entry_count; - - pci_read_config_word(pdev, PCI_COMMAND, &hw->bus.pci_cmd_word); - - /* set default ring sizes */ - adapter->tx_ring_count = IGC_DEFAULT_TXD; - adapter->rx_ring_count = IGC_DEFAULT_RXD; - - /* set default ITR values */ - adapter->rx_itr_setting = IGC_DEFAULT_ITR; - adapter->tx_itr_setting = IGC_DEFAULT_ITR; - - /* set default work limits */ - adapter->tx_work_limit = IGC_DEFAULT_TX_WORK; - - /* adjust max frame to be at least the size of a standard frame */ - adapter->max_frame_size = netdev->mtu + ETH_HLEN + ETH_FCS_LEN + - VLAN_HLEN; - adapter->min_frame_size = ETH_ZLEN + ETH_FCS_LEN; - - spin_lock_init(&adapter->nfc_lock); - spin_lock_init(&adapter->stats64_lock); - /* Assume MSI-X interrupts, will be checked during IRQ allocation */ - adapter->flags |= IGC_FLAG_HAS_MSIX; - - adapter->mac_table = kzalloc(size, GFP_ATOMIC); - if (!adapter->mac_table) - return -ENOMEM; - - igc_init_queue_configuration(adapter); - - /* This call may decrease the number of queues */ - if (igc_init_interrupt_scheme(adapter, true)) { - dev_err(&pdev->dev, "Unable to allocate memory for queues\n"); - return -ENOMEM; - } - - /* Explicitly disable IRQ since the NIC can be in any state. */ - igc_irq_disable(adapter); - - set_bit(__IGC_DOWN, &adapter->state); - - return 0; -} - /** * igc_reinit_queues - return error * @adapter: pointer to adapter structure -- cgit From 5f2958052c5820d0795616fcc299097390cf92a8 Mon Sep 17 00:00:00 2001 From: Vinicius Costa Gomes Date: Mon, 2 Dec 2019 15:19:49 -0800 Subject: igc: Add basic skeleton for PTP This allows the creation of the /dev/ptpX device for i225, and reading and writing the time. Signed-off-by: Vinicius Costa Gomes Tested-by: Aaron Brown Signed-off-by: Jeff Kirsher --- drivers/net/ethernet/intel/igc/Makefile | 2 +- drivers/net/ethernet/intel/igc/igc.h | 26 ++ drivers/net/ethernet/intel/igc/igc_defines.h | 12 + drivers/net/ethernet/intel/igc/igc_main.c | 27 +++ drivers/net/ethernet/intel/igc/igc_ptp.c | 349 +++++++++++++++++++++++++++ drivers/net/ethernet/intel/igc/igc_regs.h | 24 ++ 6 files changed, 439 insertions(+), 1 deletion(-) create mode 100644 drivers/net/ethernet/intel/igc/igc_ptp.c (limited to 'drivers/net/ethernet/intel/igc/igc_main.c') diff --git a/drivers/net/ethernet/intel/igc/Makefile b/drivers/net/ethernet/intel/igc/Makefile index 88c6f88baac5..49fb1e1965cd 100644 --- a/drivers/net/ethernet/intel/igc/Makefile +++ b/drivers/net/ethernet/intel/igc/Makefile @@ -8,4 +8,4 @@ obj-$(CONFIG_IGC) += igc.o igc-objs := igc_main.o igc_mac.o igc_i225.o igc_base.o igc_nvm.o igc_phy.o \ -igc_ethtool.o +igc_ethtool.o igc_ptp.o diff --git a/drivers/net/ethernet/intel/igc/igc.h b/drivers/net/ethernet/intel/igc/igc.h index 612fe9ec81a4..7586f237747a 100644 --- a/drivers/net/ethernet/intel/igc/igc.h +++ b/drivers/net/ethernet/intel/igc/igc.h @@ -10,6 +10,9 @@ #include #include #include +#include +#include +#include #include "igc_hw.h" @@ -45,11 +48,15 @@ extern char igc_driver_version[]; #define IGC_REGS_LEN 740 #define IGC_RETA_SIZE 128 +/* flags controlling PTP/1588 function */ +#define IGC_PTP_ENABLED BIT(0) + /* Interrupt defines */ #define IGC_START_ITR 648 /* ~6000 ints/sec */ #define IGC_FLAG_HAS_MSI BIT(0) #define IGC_FLAG_QUEUE_PAIRS BIT(3) #define IGC_FLAG_DMAC BIT(4) +#define IGC_FLAG_PTP BIT(8) #define IGC_FLAG_NEED_LINK_UPDATE BIT(9) #define IGC_FLAG_MEDIA_RESET BIT(10) #define IGC_FLAG_MAS_ENABLE BIT(12) @@ -432,6 +439,20 @@ struct igc_adapter { unsigned long link_check_timeout; struct igc_info ei; + + struct ptp_clock *ptp_clock; + struct ptp_clock_info ptp_caps; + struct work_struct ptp_tx_work; + struct sk_buff *ptp_tx_skb; + struct hwtstamp_config tstamp_config; + unsigned long ptp_tx_start; + unsigned long last_rx_ptp_check; + unsigned long last_rx_timestamp; + unsigned int ptp_flags; + /* System time value lock */ + spinlock_t tmreg_lock; + struct cyclecounter cc; + struct timecounter tc; }; /* igc_desc_unused - calculate if we have unused descriptors */ @@ -515,6 +536,11 @@ int igc_add_filter(struct igc_adapter *adapter, int igc_erase_filter(struct igc_adapter *adapter, struct igc_nfc_filter *input); +void igc_ptp_init(struct igc_adapter *adapter); +void igc_ptp_reset(struct igc_adapter *adapter); +void igc_ptp_stop(struct igc_adapter *adapter); +int igc_ptp_set_ts_config(struct net_device *netdev, struct ifreq *ifr); +int igc_ptp_get_ts_config(struct net_device *netdev, struct ifreq *ifr); #define igc_rx_pg_size(_ring) (PAGE_SIZE << igc_rx_pg_order(_ring)) #define IGC_TXD_DCMD (IGC_ADVTXD_DCMD_EOP | IGC_ADVTXD_DCMD_RS) diff --git a/drivers/net/ethernet/intel/igc/igc_defines.h b/drivers/net/ethernet/intel/igc/igc_defines.h index 50dffd5db606..12f9127e40b7 100644 --- a/drivers/net/ethernet/intel/igc/igc_defines.h +++ b/drivers/net/ethernet/intel/igc/igc_defines.h @@ -218,6 +218,7 @@ #define IGC_ICR_RXDMT0 BIT(4) /* Rx desc min. threshold (0) */ #define IGC_ICR_RXO BIT(6) /* Rx overrun */ #define IGC_ICR_RXT0 BIT(7) /* Rx timer intr (ring 0) */ +#define IGC_ICR_TS BIT(19) /* Time Sync Interrupt */ #define IGC_ICR_DRSTA BIT(30) /* Device Reset Asserted */ /* If this bit asserted, the driver should claim the interrupt */ @@ -240,6 +241,7 @@ #define IGC_IMS_DRSTA IGC_ICR_DRSTA /* Device Reset Asserted */ #define IGC_IMS_RXT0 IGC_ICR_RXT0 /* Rx timer intr */ #define IGC_IMS_RXDMT0 IGC_ICR_RXDMT0 /* Rx desc min. threshold */ +#define IGC_IMS_TS IGC_ICR_TS /* Time Sync Interrupt */ #define IGC_QVECTOR_MASK 0x7FFC /* Q-vector mask */ #define IGC_ITR_VAL_MASK 0x04 /* ITR value mask */ @@ -355,6 +357,16 @@ #define I225_RXPBSIZE_DEFAULT 0x000000A2 /* RXPBSIZE default */ #define I225_TXPBSIZE_DEFAULT 0x04000014 /* TXPBSIZE default */ +/* Time Sync Interrupt Causes */ +#define IGC_TSICR_SYS_WRAP BIT(0) /* SYSTIM Wrap around. */ +#define IGC_TSICR_TXTS BIT(1) /* Transmit Timestamp. */ +#define IGC_TSICR_TT0 BIT(3) /* Target Time 0 Trigger. */ +#define IGC_TSICR_TT1 BIT(4) /* Target Time 1 Trigger. */ +#define IGC_TSICR_AUTT0 BIT(5) /* Auxiliary Timestamp 0 Taken. */ +#define IGC_TSICR_AUTT1 BIT(6) /* Auxiliary Timestamp 1 Taken. */ + +#define IGC_TSICR_INTERRUPTS IGC_TSICR_TXTS + /* Receive Checksum Control */ #define IGC_RXCSUM_CRCOFL 0x00000800 /* CRC32 offload enable */ #define IGC_RXCSUM_PCSD 0x00002000 /* packet checksum disabled */ diff --git a/drivers/net/ethernet/intel/igc/igc_main.c b/drivers/net/ethernet/intel/igc/igc_main.c index 24fbbc928d66..f242205792e4 100644 --- a/drivers/net/ethernet/intel/igc/igc_main.c +++ b/drivers/net/ethernet/intel/igc/igc_main.c @@ -102,6 +102,9 @@ void igc_reset(struct igc_adapter *adapter) if (!netif_running(adapter->netdev)) igc_power_down_link(adapter); + /* Re-enable PTP, where applicable. */ + igc_ptp_reset(adapter); + igc_get_phy_info(hw); } @@ -4277,6 +4280,24 @@ static int igc_close(struct net_device *netdev) return 0; } +/** + * igc_ioctl - Access the hwtstamp interface + * @netdev: network interface device structure + * @ifreq: interface request data + * @cmd: ioctl command + **/ +static int igc_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd) +{ + switch (cmd) { + case SIOCGHWTSTAMP: + return igc_ptp_get_ts_config(netdev, ifr); + case SIOCSHWTSTAMP: + return igc_ptp_set_ts_config(netdev, ifr); + default: + return -EOPNOTSUPP; + } +} + static const struct net_device_ops igc_netdev_ops = { .ndo_open = igc_open, .ndo_stop = igc_close, @@ -4288,6 +4309,7 @@ static const struct net_device_ops igc_netdev_ops = { .ndo_fix_features = igc_fix_features, .ndo_set_features = igc_set_features, .ndo_features_check = igc_features_check, + .ndo_do_ioctl = igc_ioctl, }; /* PCIe configuration access */ @@ -4588,6 +4610,9 @@ static int igc_probe(struct pci_dev *pdev, /* carrier off reporting is important to ethtool even BEFORE open */ netif_carrier_off(netdev); + /* do hw tstamp init after resetting */ + igc_ptp_init(adapter); + /* Check if Media Autosense is enabled */ adapter->ei = *ei; @@ -4629,6 +4654,8 @@ static void igc_remove(struct pci_dev *pdev) struct net_device *netdev = pci_get_drvdata(pdev); struct igc_adapter *adapter = netdev_priv(netdev); + igc_ptp_stop(adapter); + set_bit(__IGC_DOWN, &adapter->state); del_timer_sync(&adapter->watchdog_timer); diff --git a/drivers/net/ethernet/intel/igc/igc_ptp.c b/drivers/net/ethernet/intel/igc/igc_ptp.c new file mode 100644 index 000000000000..bab384880f96 --- /dev/null +++ b/drivers/net/ethernet/intel/igc/igc_ptp.c @@ -0,0 +1,349 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2019 Intel Corporation */ + +#include "igc.h" + +#include +#include +#include +#include +#include + +#define INCVALUE_MASK 0x7fffffff +#define ISGN 0x80000000 + +#define IGC_SYSTIM_OVERFLOW_PERIOD (HZ * 60 * 9) +#define IGC_PTP_TX_TIMEOUT (HZ * 15) + +/* SYSTIM read access for I225 */ +static void igc_ptp_read_i225(struct igc_adapter *adapter, + struct timespec64 *ts) +{ + struct igc_hw *hw = &adapter->hw; + u32 sec, nsec; + + /* The timestamp latches on lowest register read. For I210/I211, the + * lowest register is SYSTIMR. Since we only need to provide nanosecond + * resolution, we can ignore it. + */ + rd32(IGC_SYSTIMR); + nsec = rd32(IGC_SYSTIML); + sec = rd32(IGC_SYSTIMH); + + ts->tv_sec = sec; + ts->tv_nsec = nsec; +} + +static void igc_ptp_write_i225(struct igc_adapter *adapter, + const struct timespec64 *ts) +{ + struct igc_hw *hw = &adapter->hw; + + /* Writing the SYSTIMR register is not necessary as it only + * provides sub-nanosecond resolution. + */ + wr32(IGC_SYSTIML, ts->tv_nsec); + wr32(IGC_SYSTIMH, ts->tv_sec); +} + +static int igc_ptp_adjfine_i225(struct ptp_clock_info *ptp, long scaled_ppm) +{ + struct igc_adapter *igc = container_of(ptp, struct igc_adapter, + ptp_caps); + struct igc_hw *hw = &igc->hw; + int neg_adj = 0; + u64 rate; + u32 inca; + + if (scaled_ppm < 0) { + neg_adj = 1; + scaled_ppm = -scaled_ppm; + } + rate = scaled_ppm; + rate <<= 14; + rate = div_u64(rate, 78125); + + inca = rate & INCVALUE_MASK; + if (neg_adj) + inca |= ISGN; + + wr32(IGC_TIMINCA, inca); + + return 0; +} + +static int igc_ptp_adjtime_i225(struct ptp_clock_info *ptp, s64 delta) +{ + struct igc_adapter *igc = container_of(ptp, struct igc_adapter, + ptp_caps); + struct timespec64 now, then = ns_to_timespec64(delta); + unsigned long flags; + + spin_lock_irqsave(&igc->tmreg_lock, flags); + + igc_ptp_read_i225(igc, &now); + now = timespec64_add(now, then); + igc_ptp_write_i225(igc, (const struct timespec64 *)&now); + + spin_unlock_irqrestore(&igc->tmreg_lock, flags); + + return 0; +} + +static int igc_ptp_gettimex64_i225(struct ptp_clock_info *ptp, + struct timespec64 *ts, + struct ptp_system_timestamp *sts) +{ + struct igc_adapter *igc = container_of(ptp, struct igc_adapter, + ptp_caps); + struct igc_hw *hw = &igc->hw; + unsigned long flags; + + spin_lock_irqsave(&igc->tmreg_lock, flags); + + ptp_read_system_prets(sts); + rd32(IGC_SYSTIMR); + ptp_read_system_postts(sts); + ts->tv_nsec = rd32(IGC_SYSTIML); + ts->tv_sec = rd32(IGC_SYSTIMH); + + spin_unlock_irqrestore(&igc->tmreg_lock, flags); + + return 0; +} + +static int igc_ptp_settime_i225(struct ptp_clock_info *ptp, + const struct timespec64 *ts) +{ + struct igc_adapter *igc = container_of(ptp, struct igc_adapter, + ptp_caps); + unsigned long flags; + + spin_lock_irqsave(&igc->tmreg_lock, flags); + + igc_ptp_write_i225(igc, ts); + + spin_unlock_irqrestore(&igc->tmreg_lock, flags); + + return 0; +} + +static int igc_ptp_feature_enable_i225(struct ptp_clock_info *ptp, + struct ptp_clock_request *rq, int on) +{ + return -EOPNOTSUPP; +} + +static int igc_ptp_set_timestamp_mode(struct igc_adapter *adapter, + struct hwtstamp_config *config) +{ + return 0; +} + +void igc_ptp_tx_hang(struct igc_adapter *adapter) +{ + bool timeout = time_is_before_jiffies(adapter->ptp_tx_start + + IGC_PTP_TX_TIMEOUT); + struct igc_hw *hw = &adapter->hw; + + if (!adapter->ptp_tx_skb) + return; + + if (!test_bit(__IGC_PTP_TX_IN_PROGRESS, &adapter->state)) + return; + + /* If we haven't received a timestamp within the timeout, it is + * reasonable to assume that it will never occur, so we can unlock the + * timestamp bit when this occurs. + */ + if (timeout) { + cancel_work_sync(&adapter->ptp_tx_work); + dev_kfree_skb_any(adapter->ptp_tx_skb); + adapter->ptp_tx_skb = NULL; + clear_bit_unlock(__IGC_PTP_TX_IN_PROGRESS, &adapter->state); + adapter->tx_hwtstamp_timeouts++; + /* Clear the Tx valid bit in TSYNCTXCTL register to enable + * interrupt + */ + rd32(IGC_TXSTMPH); + dev_warn(&adapter->pdev->dev, "clearing Tx timestamp hang\n"); + } +} + +void igc_ptp_tx_work(struct work_struct *work) +{ +} + +/** + * igc_ptp_set_ts_config - set hardware time stamping config + * @netdev: network interface device structure + * @ifreq: interface request data + * + **/ +int igc_ptp_set_ts_config(struct net_device *netdev, struct ifreq *ifr) +{ + struct igc_adapter *adapter = netdev_priv(netdev); + struct hwtstamp_config config; + int err; + + if (copy_from_user(&config, ifr->ifr_data, sizeof(config))) + return -EFAULT; + + err = igc_ptp_set_timestamp_mode(adapter, &config); + if (err) + return err; + + /* save these settings for future reference */ + memcpy(&adapter->tstamp_config, &config, + sizeof(adapter->tstamp_config)); + + return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ? + -EFAULT : 0; +} + +/** + * igc_ptp_get_ts_config - get hardware time stamping config + * @netdev: network interface device structure + * @ifreq: interface request data + * + * Get the hwtstamp_config settings to return to the user. Rather than attempt + * to deconstruct the settings from the registers, just return a shadow copy + * of the last known settings. + **/ +int igc_ptp_get_ts_config(struct net_device *netdev, struct ifreq *ifr) +{ + struct igc_adapter *adapter = netdev_priv(netdev); + struct hwtstamp_config *config = &adapter->tstamp_config; + + return copy_to_user(ifr->ifr_data, config, sizeof(*config)) ? + -EFAULT : 0; +} + +/** + * igc_ptp_init - Initialize PTP functionality + * @adapter: Board private structure + * + * This function is called at device probe to initialize the PTP + * functionality. + */ +void igc_ptp_init(struct igc_adapter *adapter) +{ + struct net_device *netdev = adapter->netdev; + struct igc_hw *hw = &adapter->hw; + + switch (hw->mac.type) { + case igc_i225: + snprintf(adapter->ptp_caps.name, 16, "%pm", netdev->dev_addr); + adapter->ptp_caps.owner = THIS_MODULE; + adapter->ptp_caps.max_adj = 62499999; + adapter->ptp_caps.adjfine = igc_ptp_adjfine_i225; + adapter->ptp_caps.adjtime = igc_ptp_adjtime_i225; + adapter->ptp_caps.gettimex64 = igc_ptp_gettimex64_i225; + adapter->ptp_caps.settime64 = igc_ptp_settime_i225; + adapter->ptp_caps.enable = igc_ptp_feature_enable_i225; + break; + default: + adapter->ptp_clock = NULL; + return; + } + + spin_lock_init(&adapter->tmreg_lock); + INIT_WORK(&adapter->ptp_tx_work, igc_ptp_tx_work); + + adapter->tstamp_config.rx_filter = HWTSTAMP_FILTER_NONE; + adapter->tstamp_config.tx_type = HWTSTAMP_TX_OFF; + + igc_ptp_reset(adapter); + + adapter->ptp_clock = ptp_clock_register(&adapter->ptp_caps, + &adapter->pdev->dev); + if (IS_ERR(adapter->ptp_clock)) { + adapter->ptp_clock = NULL; + dev_err(&adapter->pdev->dev, "ptp_clock_register failed\n"); + } else if (adapter->ptp_clock) { + dev_info(&adapter->pdev->dev, "added PHC on %s\n", + adapter->netdev->name); + adapter->ptp_flags |= IGC_PTP_ENABLED; + } +} + +/** + * igc_ptp_suspend - Disable PTP work items and prepare for suspend + * @adapter: Board private structure + * + * This function stops the overflow check work and PTP Tx timestamp work, and + * will prepare the device for OS suspend. + */ +void igc_ptp_suspend(struct igc_adapter *adapter) +{ + if (!(adapter->ptp_flags & IGC_PTP_ENABLED)) + return; + + cancel_work_sync(&adapter->ptp_tx_work); + if (adapter->ptp_tx_skb) { + dev_kfree_skb_any(adapter->ptp_tx_skb); + adapter->ptp_tx_skb = NULL; + clear_bit_unlock(__IGC_PTP_TX_IN_PROGRESS, &adapter->state); + } +} + +/** + * igc_ptp_stop - Disable PTP device and stop the overflow check. + * @adapter: Board private structure. + * + * This function stops the PTP support and cancels the delayed work. + **/ +void igc_ptp_stop(struct igc_adapter *adapter) +{ + igc_ptp_suspend(adapter); + + if (adapter->ptp_clock) { + ptp_clock_unregister(adapter->ptp_clock); + dev_info(&adapter->pdev->dev, "removed PHC on %s\n", + adapter->netdev->name); + adapter->ptp_flags &= ~IGC_PTP_ENABLED; + } +} + +/** + * igc_ptp_reset - Re-enable the adapter for PTP following a reset. + * @adapter: Board private structure. + * + * This function handles the reset work required to re-enable the PTP device. + **/ +void igc_ptp_reset(struct igc_adapter *adapter) +{ + struct igc_hw *hw = &adapter->hw; + unsigned long flags; + + /* reset the tstamp_config */ + igc_ptp_set_timestamp_mode(adapter, &adapter->tstamp_config); + + spin_lock_irqsave(&adapter->tmreg_lock, flags); + + switch (adapter->hw.mac.type) { + case igc_i225: + wr32(IGC_TSAUXC, 0x0); + wr32(IGC_TSSDP, 0x0); + wr32(IGC_TSIM, IGC_TSICR_INTERRUPTS); + wr32(IGC_IMS, IGC_IMS_TS); + break; + default: + /* No work to do. */ + goto out; + } + + /* Re-initialize the timer. */ + if (hw->mac.type == igc_i225) { + struct timespec64 ts64 = ktime_to_timespec64(ktime_get_real()); + + igc_ptp_write_i225(adapter, &ts64); + } else { + timecounter_init(&adapter->tc, &adapter->cc, + ktime_to_ns(ktime_get_real())); + } +out: + spin_unlock_irqrestore(&adapter->tmreg_lock, flags); + + wrfl(); +} diff --git a/drivers/net/ethernet/intel/igc/igc_regs.h b/drivers/net/ethernet/intel/igc/igc_regs.h index 93a9139f08c5..b5c72430a18b 100644 --- a/drivers/net/ethernet/intel/igc/igc_regs.h +++ b/drivers/net/ethernet/intel/igc/igc_regs.h @@ -209,6 +209,30 @@ #define IGC_LENERRS 0x04138 /* Length Errors Count */ #define IGC_HRMPC 0x0A018 /* Header Redirection Missed Packet Count */ +/* Time sync registers */ +#define IGC_TSICR 0x0B66C /* Time Sync Interrupt Cause */ +#define IGC_TSIM 0x0B674 /* Time Sync Interrupt Mask Register */ +#define IGC_TSAUXC 0x0B640 /* Timesync Auxiliary Control register */ +#define IGC_TSYNCRXCTL 0x0B620 /* Rx Time Sync Control register - RW */ +#define IGC_TSYNCTXCTL 0x0B614 /* Tx Time Sync Control register - RW */ +#define IGC_TSYNCRXCFG 0x05F50 /* Time Sync Rx Configuration - RW */ +#define IGC_TSSDP 0x0003C /* Time Sync SDP Configuration Register - RW */ + +#define IGC_IMIR(_i) (0x05A80 + ((_i) * 4)) /* Immediate Interrupt */ +#define IGC_IMIREXT(_i) (0x05AA0 + ((_i) * 4)) /* Immediate INTR Ext*/ + +#define IGC_FTQF(_n) (0x059E0 + (4 * (_n))) /* 5-tuple Queue Fltr */ +/* System Time Registers */ +#define IGC_SYSTIML 0x0B600 /* System time register Low - RO */ +#define IGC_SYSTIMH 0x0B604 /* System time register High - RO */ +#define IGC_SYSTIMR 0x0B6F8 /* System time register Residue */ +#define IGC_TIMINCA 0x0B608 /* Increment attributes register - RW */ + +#define IGC_RXSTMPL 0x0B624 /* Rx timestamp Low - RO */ +#define IGC_RXSTMPH 0x0B628 /* Rx timestamp High - RO */ +#define IGC_TXSTMPL 0x0B618 /* Tx timestamp value Low - RO */ +#define IGC_TXSTMPH 0x0B61C /* Tx timestamp value High - RO */ + /* Management registers */ #define IGC_MANC 0x05820 /* Management Control - RW */ -- cgit From 81b055205e8ba2d400c8fa5845ba540a4a880a3a Mon Sep 17 00:00:00 2001 From: Vinicius Costa Gomes Date: Mon, 2 Dec 2019 15:19:50 -0800 Subject: igc: Add support for RX timestamping This adds support for timestamping received packets. It is based on the i210, as many features of i225 work the same way. The main difference from i210 is that i225 has support for choosing the timer register to use when timestamping packets. Right now, we only support using timer 0. The other difference is that i225 stores two timestamps in the receive descriptor, right now, we only retrieve one. Signed-off-by: Vinicius Costa Gomes Tested-by: Aaron Brown Signed-off-by: Jeff Kirsher --- drivers/net/ethernet/intel/igc/igc.h | 17 ++ drivers/net/ethernet/intel/igc/igc_defines.h | 38 ++++ drivers/net/ethernet/intel/igc/igc_main.c | 10 + drivers/net/ethernet/intel/igc/igc_ptp.c | 272 +++++++++++++++++++++++++++ drivers/net/ethernet/intel/igc/igc_regs.h | 3 + 5 files changed, 340 insertions(+) (limited to 'drivers/net/ethernet/intel/igc/igc_main.c') diff --git a/drivers/net/ethernet/intel/igc/igc.h b/drivers/net/ethernet/intel/igc/igc.h index 7586f237747a..9b0d1cbf941f 100644 --- a/drivers/net/ethernet/intel/igc/igc.h +++ b/drivers/net/ethernet/intel/igc/igc.h @@ -107,6 +107,20 @@ extern char igc_driver_version[]; #define AUTO_ALL_MODES 0 #define IGC_RX_HDR_LEN IGC_RXBUFFER_256 +/* Transmit and receive latency (for PTP timestamps) */ +/* FIXME: These values were estimated using the ones that i210 has as + * basis, they seem to provide good numbers with ptp4l/phc2sys, but we + * need to confirm them. + */ +#define IGC_I225_TX_LATENCY_10 9542 +#define IGC_I225_TX_LATENCY_100 1024 +#define IGC_I225_TX_LATENCY_1000 178 +#define IGC_I225_TX_LATENCY_2500 64 +#define IGC_I225_RX_LATENCY_10 20662 +#define IGC_I225_RX_LATENCY_100 2213 +#define IGC_I225_RX_LATENCY_1000 448 +#define IGC_I225_RX_LATENCY_2500 160 + /* RX and TX descriptor control thresholds. * PTHRESH - MAC will consider prefetch if it has fewer than this number of * descriptors available in its onboard memory. @@ -539,6 +553,9 @@ int igc_erase_filter(struct igc_adapter *adapter, void igc_ptp_init(struct igc_adapter *adapter); void igc_ptp_reset(struct igc_adapter *adapter); void igc_ptp_stop(struct igc_adapter *adapter); +void igc_ptp_rx_rgtstamp(struct igc_q_vector *q_vector, struct sk_buff *skb); +void igc_ptp_rx_pktstamp(struct igc_q_vector *q_vector, void *va, + struct sk_buff *skb); int igc_ptp_set_ts_config(struct net_device *netdev, struct ifreq *ifr); int igc_ptp_get_ts_config(struct net_device *netdev, struct ifreq *ifr); #define igc_rx_pg_size(_ring) (PAGE_SIZE << igc_rx_pg_order(_ring)) diff --git a/drivers/net/ethernet/intel/igc/igc_defines.h b/drivers/net/ethernet/intel/igc/igc_defines.h index 12f9127e40b7..9dede618362f 100644 --- a/drivers/net/ethernet/intel/igc/igc_defines.h +++ b/drivers/net/ethernet/intel/igc/igc_defines.h @@ -314,12 +314,21 @@ #define IGC_RCTL_RDMTS_HALF 0x00000000 /* Rx desc min thresh size */ #define IGC_RCTL_BAM 0x00008000 /* broadcast enable */ +/* Split Replication Receive Control */ +#define IGC_SRRCTL_TIMESTAMP 0x40000000 +#define IGC_SRRCTL_TIMER1SEL(timer) (((timer) & 0x3) << 14) +#define IGC_SRRCTL_TIMER0SEL(timer) (((timer) & 0x3) << 17) + /* Receive Descriptor bit definitions */ #define IGC_RXD_STAT_EOP 0x02 /* End of Packet */ #define IGC_RXD_STAT_IXSM 0x04 /* Ignore checksum */ #define IGC_RXD_STAT_UDPCS 0x10 /* UDP xsum calculated */ #define IGC_RXD_STAT_TCPCS 0x20 /* TCP xsum calculated */ +/* Advanced Receive Descriptor bit definitions */ +#define IGC_RXDADV_STAT_TSIP 0x08000 /* timestamp in packet */ +#define IGC_RXDADV_STAT_TS 0x10000 /* Pkt was time stamped */ + #define IGC_RXDEXT_STATERR_CE 0x01000000 #define IGC_RXDEXT_STATERR_SE 0x02000000 #define IGC_RXDEXT_STATERR_SEQ 0x04000000 @@ -356,6 +365,7 @@ #define I225_RXPBSIZE_DEFAULT 0x000000A2 /* RXPBSIZE default */ #define I225_TXPBSIZE_DEFAULT 0x04000014 /* TXPBSIZE default */ +#define IGC_RXPBS_CFG_TS_EN 0x80000000 /* Timestamp in Rx buffer */ /* Time Sync Interrupt Causes */ #define IGC_TSICR_SYS_WRAP BIT(0) /* SYSTIM Wrap around. */ @@ -367,6 +377,34 @@ #define IGC_TSICR_INTERRUPTS IGC_TSICR_TXTS +/* PTP Queue Filter */ +#define IGC_ETQF_1588 BIT(30) + +#define IGC_FTQF_VF_BP 0x00008000 +#define IGC_FTQF_1588_TIME_STAMP 0x08000000 +#define IGC_FTQF_MASK 0xF0000000 +#define IGC_FTQF_MASK_PROTO_BP 0x10000000 + +/* Time Sync Receive Control bit definitions */ +#define IGC_TSYNCRXCTL_VALID 0x00000001 /* Rx timestamp valid */ +#define IGC_TSYNCRXCTL_TYPE_MASK 0x0000000E /* Rx type mask */ +#define IGC_TSYNCRXCTL_TYPE_L2_V2 0x00 +#define IGC_TSYNCRXCTL_TYPE_L4_V1 0x02 +#define IGC_TSYNCRXCTL_TYPE_L2_L4_V2 0x04 +#define IGC_TSYNCRXCTL_TYPE_ALL 0x08 +#define IGC_TSYNCRXCTL_TYPE_EVENT_V2 0x0A +#define IGC_TSYNCRXCTL_ENABLED 0x00000010 /* enable Rx timestamping */ +#define IGC_TSYNCRXCTL_SYSCFI 0x00000020 /* Sys clock frequency */ + +/* Time Sync Receive Configuration */ +#define IGC_TSYNCRXCFG_PTP_V1_CTRLT_MASK 0x000000FF +#define IGC_TSYNCRXCFG_PTP_V1_SYNC_MESSAGE 0x00 +#define IGC_TSYNCRXCFG_PTP_V1_DELAY_REQ_MESSAGE 0x01 + +/* Immediate Interrupt Receive Extended */ +#define IGC_IMIREXT_CTRL_BP 0x00080000 /* Bypass check of ctrl bits */ +#define IGC_IMIREXT_SIZE_BP 0x00001000 /* Packet size bypass */ + /* Receive Checksum Control */ #define IGC_RXCSUM_CRCOFL 0x00000800 /* CRC32 offload enable */ #define IGC_RXCSUM_PCSD 0x00002000 /* packet checksum disabled */ diff --git a/drivers/net/ethernet/intel/igc/igc_main.c b/drivers/net/ethernet/intel/igc/igc_main.c index f242205792e4..504dbb74ca75 100644 --- a/drivers/net/ethernet/intel/igc/igc_main.c +++ b/drivers/net/ethernet/intel/igc/igc_main.c @@ -1299,6 +1299,10 @@ static void igc_process_skb_fields(struct igc_ring *rx_ring, igc_rx_checksum(rx_ring, rx_desc, skb); + if (igc_test_staterr(rx_desc, IGC_RXDADV_STAT_TS) && + !igc_test_staterr(rx_desc, IGC_RXDADV_STAT_TSIP)) + igc_ptp_rx_rgtstamp(rx_ring->q_vector, skb); + skb_record_rx_queue(skb, rx_ring->queue_index); skb->protocol = eth_type_trans(skb, rx_ring->netdev); @@ -1418,6 +1422,12 @@ static struct sk_buff *igc_construct_skb(struct igc_ring *rx_ring, if (unlikely(!skb)) return NULL; + if (unlikely(igc_test_staterr(rx_desc, IGC_RXDADV_STAT_TSIP))) { + igc_ptp_rx_pktstamp(rx_ring->q_vector, va, skb); + va += IGC_TS_HDR_LEN; + size -= IGC_TS_HDR_LEN; + } + /* Determine available headroom for copy */ headlen = size; if (headlen > IGC_RX_HDR_LEN) diff --git a/drivers/net/ethernet/intel/igc/igc_ptp.c b/drivers/net/ethernet/intel/igc/igc_ptp.c index bab384880f96..093ffc9312df 100644 --- a/drivers/net/ethernet/intel/igc/igc_ptp.c +++ b/drivers/net/ethernet/intel/igc/igc_ptp.c @@ -134,9 +134,281 @@ static int igc_ptp_feature_enable_i225(struct ptp_clock_info *ptp, return -EOPNOTSUPP; } +/** + * igc_ptp_systim_to_hwtstamp - convert system time value to HW timestamp + * @adapter: board private structure + * @hwtstamps: timestamp structure to update + * @systim: unsigned 64bit system time value + * + * We need to convert the system time value stored in the RX/TXSTMP registers + * into a hwtstamp which can be used by the upper level timestamping functions. + **/ +static void igc_ptp_systim_to_hwtstamp(struct igc_adapter *adapter, + struct skb_shared_hwtstamps *hwtstamps, + u64 systim) +{ + switch (adapter->hw.mac.type) { + case igc_i225: + memset(hwtstamps, 0, sizeof(*hwtstamps)); + /* Upper 32 bits contain s, lower 32 bits contain ns. */ + hwtstamps->hwtstamp = ktime_set(systim >> 32, + systim & 0xFFFFFFFF); + break; + default: + break; + } +} + +/** + * igc_ptp_rx_pktstamp - retrieve Rx per packet timestamp + * @q_vector: Pointer to interrupt specific structure + * @va: Pointer to address containing Rx buffer + * @skb: Buffer containing timestamp and packet + * + * This function is meant to retrieve the first timestamp from the + * first buffer of an incoming frame. The value is stored in little + * endian format starting on byte 0. There's a second timestamp + * starting on byte 8. + **/ +void igc_ptp_rx_pktstamp(struct igc_q_vector *q_vector, void *va, + struct sk_buff *skb) +{ + struct igc_adapter *adapter = q_vector->adapter; + __le64 *regval = (__le64 *)va; + int adjust = 0; + + /* The timestamp is recorded in little endian format. + * DWORD: | 0 | 1 | 2 | 3 + * Field: | Timer0 Low | Timer0 High | Timer1 Low | Timer1 High + */ + igc_ptp_systim_to_hwtstamp(adapter, skb_hwtstamps(skb), + le64_to_cpu(regval[0])); + + /* adjust timestamp for the RX latency based on link speed */ + if (adapter->hw.mac.type == igc_i225) { + switch (adapter->link_speed) { + case SPEED_10: + adjust = IGC_I225_RX_LATENCY_10; + break; + case SPEED_100: + adjust = IGC_I225_RX_LATENCY_100; + break; + case SPEED_1000: + adjust = IGC_I225_RX_LATENCY_1000; + break; + case SPEED_2500: + adjust = IGC_I225_RX_LATENCY_2500; + break; + } + } + skb_hwtstamps(skb)->hwtstamp = + ktime_sub_ns(skb_hwtstamps(skb)->hwtstamp, adjust); +} + +/** + * igc_ptp_rx_rgtstamp - retrieve Rx timestamp stored in register + * @q_vector: Pointer to interrupt specific structure + * @skb: Buffer containing timestamp and packet + * + * This function is meant to retrieve a timestamp from the internal registers + * of the adapter and store it in the skb. + */ +void igc_ptp_rx_rgtstamp(struct igc_q_vector *q_vector, + struct sk_buff *skb) +{ + struct igc_adapter *adapter = q_vector->adapter; + struct igc_hw *hw = &adapter->hw; + u64 regval; + + /* If this bit is set, then the RX registers contain the time + * stamp. No other packet will be time stamped until we read + * these registers, so read the registers to make them + * available again. Because only one packet can be time + * stamped at a time, we know that the register values must + * belong to this one here and therefore we don't need to + * compare any of the additional attributes stored for it. + * + * If nothing went wrong, then it should have a shared + * tx_flags that we can turn into a skb_shared_hwtstamps. + */ + if (!(rd32(IGC_TSYNCRXCTL) & IGC_TSYNCRXCTL_VALID)) + return; + + regval = rd32(IGC_RXSTMPL); + regval |= (u64)rd32(IGC_RXSTMPH) << 32; + + igc_ptp_systim_to_hwtstamp(adapter, skb_hwtstamps(skb), regval); + + /* Update the last_rx_timestamp timer in order to enable watchdog check + * for error case of latched timestamp on a dropped packet. + */ + adapter->last_rx_timestamp = jiffies; +} + +/** + * igc_ptp_enable_tstamp_rxqueue - Enable RX timestamp for a queue + * @rx_ring: Pointer to RX queue + * @timer: Index for timer + * + * This function enables RX timestamping for a queue, and selects + * which 1588 timer will provide the timestamp. + */ +static void igc_ptp_enable_tstamp_rxqueue(struct igc_adapter *adapter, + struct igc_ring *rx_ring, u8 timer) +{ + struct igc_hw *hw = &adapter->hw; + int reg_idx = rx_ring->reg_idx; + u32 srrctl = rd32(IGC_SRRCTL(reg_idx)); + + srrctl |= IGC_SRRCTL_TIMESTAMP; + srrctl |= IGC_SRRCTL_TIMER1SEL(timer); + srrctl |= IGC_SRRCTL_TIMER0SEL(timer); + + wr32(IGC_SRRCTL(reg_idx), srrctl); +} + +static void igc_ptp_enable_tstamp_all_rxqueues(struct igc_adapter *adapter, + u8 timer) +{ + int i; + + for (i = 0; i < adapter->num_rx_queues; i++) { + struct igc_ring *ring = adapter->rx_ring[i]; + + igc_ptp_enable_tstamp_rxqueue(adapter, ring, timer); + } +} + +/** + * igc_ptp_set_timestamp_mode - setup hardware for timestamping + * @adapter: networking device structure + * @config: hwtstamp configuration + * + * Incoming time stamping has to be configured via the hardware + * filters. Not all combinations are supported, in particular event + * type has to be specified. Matching the kind of event packet is + * not supported, with the exception of "all V2 events regardless of + * level 2 or 4". + * + */ static int igc_ptp_set_timestamp_mode(struct igc_adapter *adapter, struct hwtstamp_config *config) { + u32 tsync_rx_ctl = IGC_TSYNCRXCTL_ENABLED; + struct igc_hw *hw = &adapter->hw; + u32 tsync_rx_cfg = 0; + bool is_l4 = false; + bool is_l2 = false; + u32 regval; + + /* reserved for future extensions */ + if (config->flags) + return -EINVAL; + + switch (config->rx_filter) { + case HWTSTAMP_FILTER_NONE: + tsync_rx_ctl = 0; + break; + case HWTSTAMP_FILTER_PTP_V1_L4_SYNC: + tsync_rx_ctl |= IGC_TSYNCRXCTL_TYPE_L4_V1; + tsync_rx_cfg = IGC_TSYNCRXCFG_PTP_V1_SYNC_MESSAGE; + is_l4 = true; + break; + case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ: + tsync_rx_ctl |= IGC_TSYNCRXCTL_TYPE_L4_V1; + tsync_rx_cfg = IGC_TSYNCRXCFG_PTP_V1_DELAY_REQ_MESSAGE; + is_l4 = true; + break; + case HWTSTAMP_FILTER_PTP_V2_EVENT: + case HWTSTAMP_FILTER_PTP_V2_L2_EVENT: + case HWTSTAMP_FILTER_PTP_V2_L4_EVENT: + case HWTSTAMP_FILTER_PTP_V2_SYNC: + case HWTSTAMP_FILTER_PTP_V2_L2_SYNC: + case HWTSTAMP_FILTER_PTP_V2_L4_SYNC: + case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ: + case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ: + case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ: + tsync_rx_ctl |= IGC_TSYNCRXCTL_TYPE_EVENT_V2; + config->rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT; + is_l2 = true; + is_l4 = true; + break; + case HWTSTAMP_FILTER_PTP_V1_L4_EVENT: + case HWTSTAMP_FILTER_NTP_ALL: + case HWTSTAMP_FILTER_ALL: + tsync_rx_ctl |= IGC_TSYNCRXCTL_TYPE_ALL; + config->rx_filter = HWTSTAMP_FILTER_ALL; + break; + /* fall through */ + default: + config->rx_filter = HWTSTAMP_FILTER_NONE; + return -ERANGE; + } + + /* Per-packet timestamping only works if all packets are + * timestamped, so enable timestamping in all packets as long + * as one Rx filter was configured. + */ + if (tsync_rx_ctl) { + tsync_rx_ctl = IGC_TSYNCRXCTL_ENABLED; + tsync_rx_ctl |= IGC_TSYNCRXCTL_TYPE_ALL; + config->rx_filter = HWTSTAMP_FILTER_ALL; + is_l2 = true; + is_l4 = true; + + if (hw->mac.type == igc_i225) { + regval = rd32(IGC_RXPBS); + regval |= IGC_RXPBS_CFG_TS_EN; + wr32(IGC_RXPBS, regval); + + /* FIXME: For now, only support retrieving RX + * timestamps from timer 0 + */ + igc_ptp_enable_tstamp_all_rxqueues(adapter, 0); + } + } + + /* enable/disable RX */ + regval = rd32(IGC_TSYNCRXCTL); + regval &= ~(IGC_TSYNCRXCTL_ENABLED | IGC_TSYNCRXCTL_TYPE_MASK); + regval |= tsync_rx_ctl; + wr32(IGC_TSYNCRXCTL, regval); + + /* define which PTP packets are time stamped */ + wr32(IGC_TSYNCRXCFG, tsync_rx_cfg); + + /* define ethertype filter for timestamped packets */ + if (is_l2) + wr32(IGC_ETQF(3), + (IGC_ETQF_FILTER_ENABLE | /* enable filter */ + IGC_ETQF_1588 | /* enable timestamping */ + ETH_P_1588)); /* 1588 eth protocol type */ + else + wr32(IGC_ETQF(3), 0); + + /* L4 Queue Filter[3]: filter by destination port and protocol */ + if (is_l4) { + u32 ftqf = (IPPROTO_UDP /* UDP */ + | IGC_FTQF_VF_BP /* VF not compared */ + | IGC_FTQF_1588_TIME_STAMP /* Enable Timestamp */ + | IGC_FTQF_MASK); /* mask all inputs */ + ftqf &= ~IGC_FTQF_MASK_PROTO_BP; /* enable protocol check */ + + wr32(IGC_IMIR(3), htons(PTP_EV_PORT)); + wr32(IGC_IMIREXT(3), + (IGC_IMIREXT_SIZE_BP | IGC_IMIREXT_CTRL_BP)); + wr32(IGC_FTQF(3), ftqf); + } else { + wr32(IGC_FTQF(3), IGC_FTQF_MASK); + } + wrfl(); + + /* clear TX/RX time stamp registers, just to be sure */ + regval = rd32(IGC_TXSTMPL); + regval = rd32(IGC_TXSTMPH); + regval = rd32(IGC_RXSTMPL); + regval = rd32(IGC_RXSTMPH); + return 0; } diff --git a/drivers/net/ethernet/intel/igc/igc_regs.h b/drivers/net/ethernet/intel/igc/igc_regs.h index b5c72430a18b..c82111051898 100644 --- a/drivers/net/ethernet/intel/igc/igc_regs.h +++ b/drivers/net/ethernet/intel/igc/igc_regs.h @@ -222,6 +222,9 @@ #define IGC_IMIREXT(_i) (0x05AA0 + ((_i) * 4)) /* Immediate INTR Ext*/ #define IGC_FTQF(_n) (0x059E0 + (4 * (_n))) /* 5-tuple Queue Fltr */ + +#define IGC_RXPBS 0x02404 /* Rx Packet Buffer Size - RW */ + /* System Time Registers */ #define IGC_SYSTIML 0x0B600 /* System time register Low - RO */ #define IGC_SYSTIMH 0x0B604 /* System time register High - RO */ -- cgit From 2c344ae24501558b14341247abef41e0b420c090 Mon Sep 17 00:00:00 2001 From: Vinicius Costa Gomes Date: Mon, 2 Dec 2019 15:19:51 -0800 Subject: igc: Add support for TX timestamping This adds support for timestamping packets being transmitted. Based on the code from i210. The basic differences is that i225 has 4 registers to store the transmit timestamps (i210 has one). Right now, we only support retrieving from one register, support for using the other registers will be added later. Signed-off-by: Vinicius Costa Gomes Tested-by: Aaron Brown Signed-off-by: Jeff Kirsher --- drivers/net/ethernet/intel/igc/igc.h | 2 + drivers/net/ethernet/intel/igc/igc_defines.h | 14 +++++ drivers/net/ethernet/intel/igc/igc_main.c | 50 +++++++++++++++ drivers/net/ethernet/intel/igc/igc_ptp.c | 92 ++++++++++++++++++++++++++++ 4 files changed, 158 insertions(+) (limited to 'drivers/net/ethernet/intel/igc/igc_main.c') diff --git a/drivers/net/ethernet/intel/igc/igc.h b/drivers/net/ethernet/intel/igc/igc.h index 9b0d1cbf941f..52066bdbbad0 100644 --- a/drivers/net/ethernet/intel/igc/igc.h +++ b/drivers/net/ethernet/intel/igc/igc.h @@ -558,6 +558,8 @@ void igc_ptp_rx_pktstamp(struct igc_q_vector *q_vector, void *va, struct sk_buff *skb); int igc_ptp_set_ts_config(struct net_device *netdev, struct ifreq *ifr); int igc_ptp_get_ts_config(struct net_device *netdev, struct ifreq *ifr); +void igc_ptp_tx_hang(struct igc_adapter *adapter); + #define igc_rx_pg_size(_ring) (PAGE_SIZE << igc_rx_pg_order(_ring)) #define IGC_TXD_DCMD (IGC_ADVTXD_DCMD_EOP | IGC_ADVTXD_DCMD_RS) diff --git a/drivers/net/ethernet/intel/igc/igc_defines.h b/drivers/net/ethernet/intel/igc/igc_defines.h index 9dede618362f..586fa14098eb 100644 --- a/drivers/net/ethernet/intel/igc/igc_defines.h +++ b/drivers/net/ethernet/intel/igc/igc_defines.h @@ -401,10 +401,24 @@ #define IGC_TSYNCRXCFG_PTP_V1_SYNC_MESSAGE 0x00 #define IGC_TSYNCRXCFG_PTP_V1_DELAY_REQ_MESSAGE 0x01 +/* Immediate Interrupt Receive */ +#define IGC_IMIR_CLEAR_MASK 0xF001FFFF /* IMIR Reg Clear Mask */ +#define IGC_IMIR_PORT_BYPASS 0x20000 /* IMIR Port Bypass Bit */ +#define IGC_IMIR_PRIORITY_SHIFT 29 /* IMIR Priority Shift */ +#define IGC_IMIREXT_CLEAR_MASK 0x7FFFF /* IMIREXT Reg Clear Mask */ + /* Immediate Interrupt Receive Extended */ #define IGC_IMIREXT_CTRL_BP 0x00080000 /* Bypass check of ctrl bits */ #define IGC_IMIREXT_SIZE_BP 0x00001000 /* Packet size bypass */ +/* Time Sync Transmit Control bit definitions */ +#define IGC_TSYNCTXCTL_VALID 0x00000001 /* Tx timestamp valid */ +#define IGC_TSYNCTXCTL_ENABLED 0x00000010 /* enable Tx timestamping */ +#define IGC_TSYNCTXCTL_MAX_ALLOWED_DLY_MASK 0x0000F000 /* max delay */ +#define IGC_TSYNCTXCTL_SYNC_COMP_ERR 0x20000000 /* sync err */ +#define IGC_TSYNCTXCTL_SYNC_COMP 0x40000000 /* sync complete */ +#define IGC_TSYNCTXCTL_START_SYNC 0x80000000 /* initiate sync */ + /* Receive Checksum Control */ #define IGC_RXCSUM_CRCOFL 0x00000800 /* CRC32 offload enable */ #define IGC_RXCSUM_PCSD 0x00002000 /* packet checksum disabled */ diff --git a/drivers/net/ethernet/intel/igc/igc_main.c b/drivers/net/ethernet/intel/igc/igc_main.c index 504dbb74ca75..c359f3d9fb25 100644 --- a/drivers/net/ethernet/intel/igc/igc_main.c +++ b/drivers/net/ethernet/intel/igc/igc_main.c @@ -987,6 +987,11 @@ static inline int igc_maybe_stop_tx(struct igc_ring *tx_ring, const u16 size) return __igc_maybe_stop_tx(tx_ring, size); } +#define IGC_SET_FLAG(_input, _flag, _result) \ + (((_flag) <= (_result)) ? \ + ((u32)((_input) & (_flag)) * ((_result) / (_flag))) : \ + ((u32)((_input) & (_flag)) / ((_flag) / (_result)))) + static u32 igc_tx_cmd_type(struct sk_buff *skb, u32 tx_flags) { /* set type for advanced descriptor with frame checksum insertion */ @@ -994,6 +999,10 @@ static u32 igc_tx_cmd_type(struct sk_buff *skb, u32 tx_flags) IGC_ADVTXD_DCMD_DEXT | IGC_ADVTXD_DCMD_IFCS; + /* set timestamp bit if present */ + cmd_type |= IGC_SET_FLAG(tx_flags, IGC_TX_FLAGS_TSTAMP, + (IGC_ADVTXD_MAC_TSTAMP)); + return cmd_type; } @@ -1192,6 +1201,26 @@ static netdev_tx_t igc_xmit_frame_ring(struct sk_buff *skb, first->bytecount = skb->len; first->gso_segs = 1; + if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) { + struct igc_adapter *adapter = netdev_priv(tx_ring->netdev); + + /* FIXME: add support for retrieving timestamps from + * the other timer registers before skipping the + * timestamping request. + */ + if (adapter->tstamp_config.tx_type == HWTSTAMP_TX_ON && + !test_and_set_bit_lock(__IGC_PTP_TX_IN_PROGRESS, + &adapter->state)) { + skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS; + tx_flags |= IGC_TX_FLAGS_TSTAMP; + + adapter->ptp_tx_skb = skb_get(skb); + adapter->ptp_tx_start = jiffies; + } else { + adapter->tx_hwtstamp_skipped++; + } + } + /* record initial flags and protocol */ first->tx_flags = tx_flags; first->protocol = protocol; @@ -3648,6 +3677,22 @@ int igc_del_mac_steering_filter(struct igc_adapter *adapter, IGC_MAC_STATE_QUEUE_STEERING | flags); } +static void igc_tsync_interrupt(struct igc_adapter *adapter) +{ + struct igc_hw *hw = &adapter->hw; + u32 tsicr = rd32(IGC_TSICR); + u32 ack = 0; + + if (tsicr & IGC_TSICR_TXTS) { + /* retrieve hardware timestamp */ + schedule_work(&adapter->ptp_tx_work); + ack |= IGC_TSICR_TXTS; + } + + /* acknowledge the interrupts */ + wr32(IGC_TSICR, ack); +} + /** * igc_msix_other - msix other interrupt handler * @irq: interrupt number @@ -3675,6 +3720,9 @@ static irqreturn_t igc_msix_other(int irq, void *data) mod_timer(&adapter->watchdog_timer, jiffies + 1); } + if (icr & IGC_ICR_TS) + igc_tsync_interrupt(adapter); + wr32(IGC_EIMS, adapter->eims_other); return IRQ_HANDLED; @@ -4004,6 +4052,8 @@ no_wait: wr32(IGC_ICS, IGC_ICS_RXDMT0); } + igc_ptp_tx_hang(adapter); + /* Reset the timer */ if (!test_bit(__IGC_DOWN, &adapter->state)) { if (adapter->flags & IGC_FLAG_NEED_LINK_UPDATE) diff --git a/drivers/net/ethernet/intel/igc/igc_ptp.c b/drivers/net/ethernet/intel/igc/igc_ptp.c index 093ffc9312df..79ffb833aa80 100644 --- a/drivers/net/ethernet/intel/igc/igc_ptp.c +++ b/drivers/net/ethernet/intel/igc/igc_ptp.c @@ -284,6 +284,12 @@ static void igc_ptp_enable_tstamp_all_rxqueues(struct igc_adapter *adapter, * @adapter: networking device structure * @config: hwtstamp configuration * + * Outgoing time stamping can be enabled and disabled. Play nice and + * disable it when requested, although it shouldn't case any overhead + * when no packet needs it. At most one packet in the queue may be + * marked for time stamping, otherwise it would be impossible to tell + * for sure to which packet the hardware time stamp belongs. + * * Incoming time stamping has to be configured via the hardware * filters. Not all combinations are supported, in particular event * type has to be specified. Matching the kind of event packet is @@ -294,6 +300,7 @@ static void igc_ptp_enable_tstamp_all_rxqueues(struct igc_adapter *adapter, static int igc_ptp_set_timestamp_mode(struct igc_adapter *adapter, struct hwtstamp_config *config) { + u32 tsync_tx_ctl = IGC_TSYNCTXCTL_ENABLED; u32 tsync_rx_ctl = IGC_TSYNCRXCTL_ENABLED; struct igc_hw *hw = &adapter->hw; u32 tsync_rx_cfg = 0; @@ -305,6 +312,15 @@ static int igc_ptp_set_timestamp_mode(struct igc_adapter *adapter, if (config->flags) return -EINVAL; + switch (config->tx_type) { + case HWTSTAMP_TX_OFF: + tsync_tx_ctl = 0; + case HWTSTAMP_TX_ON: + break; + default: + return -ERANGE; + } + switch (config->rx_filter) { case HWTSTAMP_FILTER_NONE: tsync_rx_ctl = 0; @@ -368,6 +384,15 @@ static int igc_ptp_set_timestamp_mode(struct igc_adapter *adapter, } } + if (tsync_tx_ctl) + tsync_tx_ctl = IGC_TSYNCTXCTL_ENABLED; + + /* enable/disable TX */ + regval = rd32(IGC_TSYNCTXCTL); + regval &= ~IGC_TSYNCTXCTL_ENABLED; + regval |= tsync_tx_ctl; + wr32(IGC_TSYNCTXCTL, regval); + /* enable/disable RX */ regval = rd32(IGC_TSYNCRXCTL); regval &= ~(IGC_TSYNCRXCTL_ENABLED | IGC_TSYNCRXCTL_TYPE_MASK); @@ -442,8 +467,75 @@ void igc_ptp_tx_hang(struct igc_adapter *adapter) } } +/** + * igc_ptp_tx_hwtstamp - utility function which checks for TX time stamp + * @adapter: Board private structure + * + * If we were asked to do hardware stamping and such a time stamp is + * available, then it must have been for this skb here because we only + * allow only one such packet into the queue. + */ +static void igc_ptp_tx_hwtstamp(struct igc_adapter *adapter) +{ + struct sk_buff *skb = adapter->ptp_tx_skb; + struct skb_shared_hwtstamps shhwtstamps; + struct igc_hw *hw = &adapter->hw; + u64 regval; + + regval = rd32(IGC_TXSTMPL); + regval |= (u64)rd32(IGC_TXSTMPH) << 32; + igc_ptp_systim_to_hwtstamp(adapter, &shhwtstamps, regval); + + /* Clear the lock early before calling skb_tstamp_tx so that + * applications are not woken up before the lock bit is clear. We use + * a copy of the skb pointer to ensure other threads can't change it + * while we're notifying the stack. + */ + adapter->ptp_tx_skb = NULL; + clear_bit_unlock(__IGC_PTP_TX_IN_PROGRESS, &adapter->state); + + /* Notify the stack and free the skb after we've unlocked */ + skb_tstamp_tx(skb, &shhwtstamps); + dev_kfree_skb_any(skb); +} + +/** + * igc_ptp_tx_work + * @work: pointer to work struct + * + * This work function polls the TSYNCTXCTL valid bit to determine when a + * timestamp has been taken for the current stored skb. + */ void igc_ptp_tx_work(struct work_struct *work) { + struct igc_adapter *adapter = container_of(work, struct igc_adapter, + ptp_tx_work); + struct igc_hw *hw = &adapter->hw; + u32 tsynctxctl; + + if (!adapter->ptp_tx_skb) + return; + + if (time_is_before_jiffies(adapter->ptp_tx_start + + IGC_PTP_TX_TIMEOUT)) { + dev_kfree_skb_any(adapter->ptp_tx_skb); + adapter->ptp_tx_skb = NULL; + clear_bit_unlock(__IGC_PTP_TX_IN_PROGRESS, &adapter->state); + adapter->tx_hwtstamp_timeouts++; + /* Clear the tx valid bit in TSYNCTXCTL register to enable + * interrupt + */ + rd32(IGC_TXSTMPH); + dev_warn(&adapter->pdev->dev, "clearing Tx timestamp hang\n"); + return; + } + + tsynctxctl = rd32(IGC_TSYNCTXCTL); + if (tsynctxctl & IGC_TSYNCTXCTL_VALID) + igc_ptp_tx_hwtstamp(adapter); + else + /* reschedule to check later */ + schedule_work(&adapter->ptp_tx_work); } /** -- cgit From 93d85dc5651d8337e3681441df1d5dd4184784ab Mon Sep 17 00:00:00 2001 From: Sasha Neftin Date: Sun, 29 Dec 2019 16:09:09 +0200 Subject: igc: Fix typo in a comment Fix typo in a context descriptor comment Signed-off-by: Sasha Neftin Tested-by: Aaron Brown Signed-off-by: Jeff Kirsher --- drivers/net/ethernet/intel/igc/igc_main.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'drivers/net/ethernet/intel/igc/igc_main.c') diff --git a/drivers/net/ethernet/intel/igc/igc_main.c b/drivers/net/ethernet/intel/igc/igc_main.c index c359f3d9fb25..1e27389bbda8 100644 --- a/drivers/net/ethernet/intel/igc/igc_main.c +++ b/drivers/net/ethernet/intel/igc/igc_main.c @@ -880,7 +880,7 @@ static void igc_tx_ctxtdesc(struct igc_ring *tx_ring, /* set bits to identify this as an advanced context descriptor */ type_tucmd |= IGC_TXD_CMD_DEXT | IGC_ADVTXD_DTYP_CTXT; - /* For 82575, context index must be unique per ring. */ + /* For i225, context index must be unique per ring. */ if (test_bit(IGC_RING_FLAG_TX_CTX_IDX, &tx_ring->flags)) mss_l4len_idx |= tx_ring->reg_idx << 4; -- cgit From 0e7d4b93b0de94386be44d2a7ffa417a5c4adc1b Mon Sep 17 00:00:00 2001 From: Sasha Neftin Date: Wed, 1 Jan 2020 09:14:17 +0200 Subject: igc: Add SKU for i225 device Add support for blank NVM SKU Signed-off-by: Sasha Neftin Tested-by: Aaron Brown Signed-off-by: Jeff Kirsher --- drivers/net/ethernet/intel/igc/igc_base.c | 1 + drivers/net/ethernet/intel/igc/igc_hw.h | 1 + drivers/net/ethernet/intel/igc/igc_main.c | 1 + 3 files changed, 3 insertions(+) (limited to 'drivers/net/ethernet/intel/igc/igc_main.c') diff --git a/drivers/net/ethernet/intel/igc/igc_base.c b/drivers/net/ethernet/intel/igc/igc_base.c index db289bcce21d..5a506440560a 100644 --- a/drivers/net/ethernet/intel/igc/igc_base.c +++ b/drivers/net/ethernet/intel/igc/igc_base.c @@ -212,6 +212,7 @@ static s32 igc_get_invariants_base(struct igc_hw *hw) case IGC_DEV_ID_I225_I: case IGC_DEV_ID_I220_V: case IGC_DEV_ID_I225_K: + case IGC_DEV_ID_I225_BLANK_NVM: mac->type = igc_i225; break; default: diff --git a/drivers/net/ethernet/intel/igc/igc_hw.h b/drivers/net/ethernet/intel/igc/igc_hw.h index be7273c4b53f..90ac0e0144d8 100644 --- a/drivers/net/ethernet/intel/igc/igc_hw.h +++ b/drivers/net/ethernet/intel/igc/igc_hw.h @@ -21,6 +21,7 @@ #define IGC_DEV_ID_I225_I 0x15F8 #define IGC_DEV_ID_I220_V 0x15F7 #define IGC_DEV_ID_I225_K 0x3100 +#define IGC_DEV_ID_I225_BLANK_NVM 0x15FD /* Function pointers for the MAC. */ struct igc_mac_operations { diff --git a/drivers/net/ethernet/intel/igc/igc_main.c b/drivers/net/ethernet/intel/igc/igc_main.c index 1e27389bbda8..a8f8d279f3b1 100644 --- a/drivers/net/ethernet/intel/igc/igc_main.c +++ b/drivers/net/ethernet/intel/igc/igc_main.c @@ -45,6 +45,7 @@ static const struct pci_device_id igc_pci_tbl[] = { { PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_I), board_base }, { PCI_VDEVICE(INTEL, IGC_DEV_ID_I220_V), board_base }, { PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_K), board_base }, + { PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_BLANK_NVM), board_base }, /* required last entry */ {0, } }; -- cgit From f38b782dccabb2ada5c5b3c0dfb5ef0758a67ce0 Mon Sep 17 00:00:00 2001 From: Sasha Neftin Date: Sun, 5 Jan 2020 14:20:53 +0200 Subject: igc: Add support for TSO TCP segmentation offload allows a device to segment a single frame into multiple frames with a data payload size specified in socket buffer. As a result we can now send data approximately up to seven percents fast than was previously possible on my system. Signed-off-by: Sasha Neftin Tested-by: Aaron Brown Signed-off-by: Jeff Kirsher --- drivers/net/ethernet/intel/igc/igc_defines.h | 4 + drivers/net/ethernet/intel/igc/igc_main.c | 113 ++++++++++++++++++++++++++- 2 files changed, 116 insertions(+), 1 deletion(-) (limited to 'drivers/net/ethernet/intel/igc/igc_main.c') diff --git a/drivers/net/ethernet/intel/igc/igc_defines.h b/drivers/net/ethernet/intel/igc/igc_defines.h index 2121fc34e300..9e34b0969322 100644 --- a/drivers/net/ethernet/intel/igc/igc_defines.h +++ b/drivers/net/ethernet/intel/igc/igc_defines.h @@ -282,6 +282,10 @@ #define IGC_TXD_STAT_TC 0x00000004 /* Tx Underrun */ #define IGC_TXD_EXTCMD_TSTAMP 0x00000010 /* IEEE1588 Timestamp packet */ +/* IPSec Encrypt Enable */ +#define IGC_ADVTXD_L4LEN_SHIFT 8 /* Adv ctxt L4LEN shift */ +#define IGC_ADVTXD_MSS_SHIFT 16 /* Adv ctxt MSS shift */ + /* Transmit Control */ #define IGC_TCTL_EN 0x00000002 /* enable Tx */ #define IGC_TCTL_PSP 0x00000008 /* pad short packets */ diff --git a/drivers/net/ethernet/intel/igc/igc_main.c b/drivers/net/ethernet/intel/igc/igc_main.c index a8f8d279f3b1..d9d5425fe8d9 100644 --- a/drivers/net/ethernet/intel/igc/igc_main.c +++ b/drivers/net/ethernet/intel/igc/igc_main.c @@ -1000,6 +1000,10 @@ static u32 igc_tx_cmd_type(struct sk_buff *skb, u32 tx_flags) IGC_ADVTXD_DCMD_DEXT | IGC_ADVTXD_DCMD_IFCS; + /* set segmentation bits for TSO */ + cmd_type |= IGC_SET_FLAG(tx_flags, IGC_TX_FLAGS_TSO, + (IGC_ADVTXD_DCMD_TSE)); + /* set timestamp bit if present */ cmd_type |= IGC_SET_FLAG(tx_flags, IGC_TX_FLAGS_TSTAMP, (IGC_ADVTXD_MAC_TSTAMP)); @@ -1171,6 +1175,100 @@ dma_error: return -1; } +static int igc_tso(struct igc_ring *tx_ring, + struct igc_tx_buffer *first, + u8 *hdr_len) +{ + u32 vlan_macip_lens, type_tucmd, mss_l4len_idx; + struct sk_buff *skb = first->skb; + union { + struct iphdr *v4; + struct ipv6hdr *v6; + unsigned char *hdr; + } ip; + union { + struct tcphdr *tcp; + struct udphdr *udp; + unsigned char *hdr; + } l4; + u32 paylen, l4_offset; + int err; + + if (skb->ip_summed != CHECKSUM_PARTIAL) + return 0; + + if (!skb_is_gso(skb)) + return 0; + + err = skb_cow_head(skb, 0); + if (err < 0) + return err; + + ip.hdr = skb_network_header(skb); + l4.hdr = skb_checksum_start(skb); + + /* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */ + type_tucmd = IGC_ADVTXD_TUCMD_L4T_TCP; + + /* initialize outer IP header fields */ + if (ip.v4->version == 4) { + unsigned char *csum_start = skb_checksum_start(skb); + unsigned char *trans_start = ip.hdr + (ip.v4->ihl * 4); + + /* IP header will have to cancel out any data that + * is not a part of the outer IP header + */ + ip.v4->check = csum_fold(csum_partial(trans_start, + csum_start - trans_start, + 0)); + type_tucmd |= IGC_ADVTXD_TUCMD_IPV4; + + ip.v4->tot_len = 0; + first->tx_flags |= IGC_TX_FLAGS_TSO | + IGC_TX_FLAGS_CSUM | + IGC_TX_FLAGS_IPV4; + } else { + ip.v6->payload_len = 0; + first->tx_flags |= IGC_TX_FLAGS_TSO | + IGC_TX_FLAGS_CSUM; + } + + /* determine offset of inner transport header */ + l4_offset = l4.hdr - skb->data; + + /* remove payload length from inner checksum */ + paylen = skb->len - l4_offset; + if (type_tucmd & IGC_ADVTXD_TUCMD_L4T_TCP) { + /* compute length of segmentation header */ + *hdr_len = (l4.tcp->doff * 4) + l4_offset; + csum_replace_by_diff(&l4.tcp->check, + (__force __wsum)htonl(paylen)); + } else { + /* compute length of segmentation header */ + *hdr_len = sizeof(*l4.udp) + l4_offset; + csum_replace_by_diff(&l4.udp->check, + (__force __wsum)htonl(paylen)); + } + + /* update gso size and bytecount with header size */ + first->gso_segs = skb_shinfo(skb)->gso_segs; + first->bytecount += (first->gso_segs - 1) * *hdr_len; + + /* MSS L4LEN IDX */ + mss_l4len_idx = (*hdr_len - l4_offset) << IGC_ADVTXD_L4LEN_SHIFT; + mss_l4len_idx |= skb_shinfo(skb)->gso_size << IGC_ADVTXD_MSS_SHIFT; + + /* VLAN MACLEN IPLEN */ + vlan_macip_lens = l4.hdr - ip.hdr; + vlan_macip_lens |= (ip.hdr - skb->data) << IGC_ADVTXD_MACLEN_SHIFT; + vlan_macip_lens |= first->tx_flags & IGC_TX_FLAGS_VLAN_MASK; + + igc_tx_ctxtdesc(tx_ring, first, vlan_macip_lens, + type_tucmd, mss_l4len_idx); + + return 1; +} + static netdev_tx_t igc_xmit_frame_ring(struct sk_buff *skb, struct igc_ring *tx_ring) { @@ -1180,6 +1278,7 @@ static netdev_tx_t igc_xmit_frame_ring(struct sk_buff *skb, u32 tx_flags = 0; unsigned short f; u8 hdr_len = 0; + int tso = 0; /* need: 1 descriptor per page * PAGE_SIZE/IGC_MAX_DATA_PER_TXD, * + 1 desc for skb_headlen/IGC_MAX_DATA_PER_TXD, @@ -1226,10 +1325,20 @@ static netdev_tx_t igc_xmit_frame_ring(struct sk_buff *skb, first->tx_flags = tx_flags; first->protocol = protocol; - igc_tx_csum(tx_ring, first); + tso = igc_tso(tx_ring, first, &hdr_len); + if (tso < 0) + goto out_drop; + else if (!tso) + igc_tx_csum(tx_ring, first); igc_tx_map(tx_ring, first, hdr_len); + return NETDEV_TX_OK; + +out_drop: + dev_kfree_skb_any(first->skb); + first->skb = NULL; + return NETDEV_TX_OK; } @@ -4589,6 +4698,8 @@ static int igc_probe(struct pci_dev *pdev, /* Add supported features to the features list*/ netdev->features |= NETIF_F_SG; + netdev->features |= NETIF_F_TSO; + netdev->features |= NETIF_F_TSO6; netdev->features |= NETIF_F_RXCSUM; netdev->features |= NETIF_F_HW_CSUM; netdev->features |= NETIF_F_SCTP_CRC; -- cgit