aboutsummaryrefslogtreecommitdiff
path: root/drivers/net/ethernet/marvell/mvpp2/mvpp2_main.c
diff options
context:
space:
mode:
Diffstat (limited to 'drivers/net/ethernet/marvell/mvpp2/mvpp2_main.c')
-rw-r--r--drivers/net/ethernet/marvell/mvpp2/mvpp2_main.c1791
1 files changed, 1418 insertions, 373 deletions
diff --git a/drivers/net/ethernet/marvell/mvpp2/mvpp2_main.c b/drivers/net/ethernet/marvell/mvpp2/mvpp2_main.c
index 2b5dad2ec650..f6616c8933ca 100644
--- a/drivers/net/ethernet/marvell/mvpp2/mvpp2_main.c
+++ b/drivers/net/ethernet/marvell/mvpp2/mvpp2_main.c
@@ -28,6 +28,7 @@
#include <linux/phy.h>
#include <linux/phylink.h>
#include <linux/phy/phy.h>
+#include <linux/ptp_classify.h>
#include <linux/clk.h>
#include <linux/hrtimer.h>
#include <linux/ktime.h>
@@ -36,6 +37,7 @@
#include <net/ip.h>
#include <net/ipv6.h>
#include <net/tso.h>
+#include <linux/bpf_trace.h>
#include "mvpp2.h"
#include "mvpp2_prs.h"
@@ -56,13 +58,7 @@ static struct {
/* The prototype is added here to be used in start_dev when using ACPI. This
* will be removed once phylink is used for all modes (dt+ACPI).
*/
-static void mvpp2_mac_config(struct phylink_config *config, unsigned int mode,
- const struct phylink_link_state *state);
-static void mvpp2_mac_link_up(struct phylink_config *config,
- struct phy_device *phy,
- unsigned int mode, phy_interface_t interface,
- int speed, int duplex,
- bool tx_pause, bool rx_pause);
+static void mvpp2_acpi_start(struct mvpp2_port *port);
/* Queue modes */
#define MVPP2_QDIST_SINGLE_MODE 0
@@ -95,6 +91,24 @@ static inline u32 mvpp2_cpu_to_thread(struct mvpp2 *priv, int cpu)
return cpu % priv->nthreads;
}
+static struct page_pool *
+mvpp2_create_page_pool(struct device *dev, int num, int len,
+ enum dma_data_direction dma_dir)
+{
+ struct page_pool_params pp_params = {
+ /* internal DMA mapping in page_pool */
+ .flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV,
+ .pool_size = num,
+ .nid = NUMA_NO_NODE,
+ .dev = dev,
+ .dma_dir = dma_dir,
+ .offset = MVPP2_SKB_HEADROOM,
+ .max_len = len,
+ };
+
+ return page_pool_create(&pp_params);
+}
+
/* These accessors should be used to access:
*
* - per-thread registers, where each thread has its own copy of the
@@ -281,12 +295,17 @@ static void mvpp2_txq_inc_get(struct mvpp2_txq_pcpu *txq_pcpu)
static void mvpp2_txq_inc_put(struct mvpp2_port *port,
struct mvpp2_txq_pcpu *txq_pcpu,
- struct sk_buff *skb,
- struct mvpp2_tx_desc *tx_desc)
+ void *data,
+ struct mvpp2_tx_desc *tx_desc,
+ enum mvpp2_tx_buf_type buf_type)
{
struct mvpp2_txq_pcpu_buf *tx_buf =
txq_pcpu->buffs + txq_pcpu->txq_put_index;
- tx_buf->skb = skb;
+ tx_buf->type = buf_type;
+ if (buf_type == MVPP2_TYPE_SKB)
+ tx_buf->skb = data;
+ else
+ tx_buf->xdpf = data;
tx_buf->size = mvpp2_txdesc_size_get(port, tx_desc);
tx_buf->dma = mvpp2_txdesc_dma_addr_get(port, tx_desc) +
mvpp2_txdesc_offset_get(port, tx_desc);
@@ -327,17 +346,25 @@ static inline int mvpp2_txq_phys(int port, int txq)
return (MVPP2_MAX_TCONT + port) * MVPP2_MAX_TXQ + txq;
}
-static void *mvpp2_frag_alloc(const struct mvpp2_bm_pool *pool)
+/* Returns a struct page if page_pool is set, otherwise a buffer */
+static void *mvpp2_frag_alloc(const struct mvpp2_bm_pool *pool,
+ struct page_pool *page_pool)
{
+ if (page_pool)
+ return page_pool_dev_alloc_pages(page_pool);
+
if (likely(pool->frag_size <= PAGE_SIZE))
return netdev_alloc_frag(pool->frag_size);
- else
- return kmalloc(pool->frag_size, GFP_ATOMIC);
+
+ return kmalloc(pool->frag_size, GFP_ATOMIC);
}
-static void mvpp2_frag_free(const struct mvpp2_bm_pool *pool, void *data)
+static void mvpp2_frag_free(const struct mvpp2_bm_pool *pool,
+ struct page_pool *page_pool, void *data)
{
- if (likely(pool->frag_size <= PAGE_SIZE))
+ if (page_pool)
+ page_pool_put_full_page(page_pool, virt_to_head_page(data), false);
+ else if (likely(pool->frag_size <= PAGE_SIZE))
skb_free_frag(data);
else
kfree(data);
@@ -442,6 +469,7 @@ static void mvpp2_bm_bufs_get_addrs(struct device *dev, struct mvpp2 *priv,
static void mvpp2_bm_bufs_free(struct device *dev, struct mvpp2 *priv,
struct mvpp2_bm_pool *bm_pool, int buf_num)
{
+ struct page_pool *pp = NULL;
int i;
if (buf_num > bm_pool->buf_num) {
@@ -450,6 +478,9 @@ static void mvpp2_bm_bufs_free(struct device *dev, struct mvpp2 *priv,
buf_num = bm_pool->buf_num;
}
+ if (priv->percpu_pools)
+ pp = priv->page_pool[bm_pool->id];
+
for (i = 0; i < buf_num; i++) {
dma_addr_t buf_dma_addr;
phys_addr_t buf_phys_addr;
@@ -458,14 +489,15 @@ static void mvpp2_bm_bufs_free(struct device *dev, struct mvpp2 *priv,
mvpp2_bm_bufs_get_addrs(dev, priv, bm_pool,
&buf_dma_addr, &buf_phys_addr);
- dma_unmap_single(dev, buf_dma_addr,
- bm_pool->buf_size, DMA_FROM_DEVICE);
+ if (!pp)
+ dma_unmap_single(dev, buf_dma_addr,
+ bm_pool->buf_size, DMA_FROM_DEVICE);
data = (void *)phys_to_virt(buf_phys_addr);
if (!data)
break;
- mvpp2_frag_free(bm_pool, data);
+ mvpp2_frag_free(bm_pool, pp, data);
}
/* Update BM driver with number of buffers removed from pool */
@@ -511,6 +543,11 @@ static int mvpp2_bm_pool_destroy(struct device *dev, struct mvpp2 *priv,
val |= MVPP2_BM_STOP_MASK;
mvpp2_write(priv, MVPP2_BM_POOL_CTRL_REG(bm_pool->id), val);
+ if (priv->percpu_pools) {
+ page_pool_destroy(priv->page_pool[bm_pool->id]);
+ priv->page_pool[bm_pool->id] = NULL;
+ }
+
dma_free_coherent(dev, bm_pool->size_bytes,
bm_pool->virt_addr,
bm_pool->dma_addr);
@@ -546,10 +583,40 @@ err_unroll_pools:
static int mvpp2_bm_init(struct device *dev, struct mvpp2 *priv)
{
+ enum dma_data_direction dma_dir = DMA_FROM_DEVICE;
int i, err, poolnum = MVPP2_BM_POOLS_NUM;
+ struct mvpp2_port *port;
+
+ if (priv->percpu_pools) {
+ for (i = 0; i < priv->port_count; i++) {
+ port = priv->port_list[i];
+ if (port->xdp_prog) {
+ dma_dir = DMA_BIDIRECTIONAL;
+ break;
+ }
+ }
- if (priv->percpu_pools)
poolnum = mvpp2_get_nrxqs(priv) * 2;
+ for (i = 0; i < poolnum; i++) {
+ /* the pool in use */
+ int pn = i / (poolnum / 2);
+
+ priv->page_pool[i] =
+ mvpp2_create_page_pool(dev,
+ mvpp2_pools[pn].buf_num,
+ mvpp2_pools[pn].pkt_size,
+ dma_dir);
+ if (IS_ERR(priv->page_pool[i])) {
+ int j;
+
+ for (j = 0; j < i; j++) {
+ page_pool_destroy(priv->page_pool[j]);
+ priv->page_pool[j] = NULL;
+ }
+ return PTR_ERR(priv->page_pool[i]);
+ }
+ }
+ }
dev_info(dev, "using %d %s buffers\n", poolnum,
priv->percpu_pools ? "per-cpu" : "shared");
@@ -632,23 +699,31 @@ static void mvpp2_rxq_short_pool_set(struct mvpp2_port *port,
static void *mvpp2_buf_alloc(struct mvpp2_port *port,
struct mvpp2_bm_pool *bm_pool,
+ struct page_pool *page_pool,
dma_addr_t *buf_dma_addr,
phys_addr_t *buf_phys_addr,
gfp_t gfp_mask)
{
dma_addr_t dma_addr;
+ struct page *page;
void *data;
- data = mvpp2_frag_alloc(bm_pool);
+ data = mvpp2_frag_alloc(bm_pool, page_pool);
if (!data)
return NULL;
- dma_addr = dma_map_single(port->dev->dev.parent, data,
- MVPP2_RX_BUF_SIZE(bm_pool->pkt_size),
- DMA_FROM_DEVICE);
- if (unlikely(dma_mapping_error(port->dev->dev.parent, dma_addr))) {
- mvpp2_frag_free(bm_pool, data);
- return NULL;
+ if (page_pool) {
+ page = (struct page *)data;
+ dma_addr = page_pool_get_dma_addr(page);
+ data = page_to_virt(page);
+ } else {
+ dma_addr = dma_map_single(port->dev->dev.parent, data,
+ MVPP2_RX_BUF_SIZE(bm_pool->pkt_size),
+ DMA_FROM_DEVICE);
+ if (unlikely(dma_mapping_error(port->dev->dev.parent, dma_addr))) {
+ mvpp2_frag_free(bm_pool, NULL, data);
+ return NULL;
+ }
}
*buf_dma_addr = dma_addr;
*buf_phys_addr = virt_to_phys(data);
@@ -706,6 +781,7 @@ static int mvpp2_bm_bufs_add(struct mvpp2_port *port,
int i, buf_size, total_size;
dma_addr_t dma_addr;
phys_addr_t phys_addr;
+ struct page_pool *pp = NULL;
void *buf;
if (port->priv->percpu_pools &&
@@ -726,8 +802,10 @@ static int mvpp2_bm_bufs_add(struct mvpp2_port *port,
return 0;
}
+ if (port->priv->percpu_pools)
+ pp = port->priv->page_pool[bm_pool->id];
for (i = 0; i < buf_num; i++) {
- buf = mvpp2_buf_alloc(port, bm_pool, &dma_addr,
+ buf = mvpp2_buf_alloc(port, bm_pool, pp, &dma_addr,
&phys_addr, GFP_KERNEL);
if (!buf)
break;
@@ -907,28 +985,27 @@ static int mvpp2_swf_bm_pool_init_shared(struct mvpp2_port *port)
/* Initialize pools for swf, percpu buffers variant */
static int mvpp2_swf_bm_pool_init_percpu(struct mvpp2_port *port)
{
- struct mvpp2_bm_pool *p;
+ struct mvpp2_bm_pool *bm_pool;
int i;
for (i = 0; i < port->nrxqs; i++) {
- p = mvpp2_bm_pool_use_percpu(port, MVPP2_BM_SHORT, i,
- mvpp2_pools[MVPP2_BM_SHORT].pkt_size);
- if (!p)
+ bm_pool = mvpp2_bm_pool_use_percpu(port, MVPP2_BM_SHORT, i,
+ mvpp2_pools[MVPP2_BM_SHORT].pkt_size);
+ if (!bm_pool)
return -ENOMEM;
- port->priv->bm_pools[i].port_map |= BIT(port->id);
- mvpp2_rxq_short_pool_set(port, i, port->priv->bm_pools[i].id);
+ bm_pool->port_map |= BIT(port->id);
+ mvpp2_rxq_short_pool_set(port, i, bm_pool->id);
}
for (i = 0; i < port->nrxqs; i++) {
- p = mvpp2_bm_pool_use_percpu(port, MVPP2_BM_LONG, i + port->nrxqs,
- mvpp2_pools[MVPP2_BM_LONG].pkt_size);
- if (!p)
+ bm_pool = mvpp2_bm_pool_use_percpu(port, MVPP2_BM_LONG, i + port->nrxqs,
+ mvpp2_pools[MVPP2_BM_LONG].pkt_size);
+ if (!bm_pool)
return -ENOMEM;
- port->priv->bm_pools[i + port->nrxqs].port_map |= BIT(port->id);
- mvpp2_rxq_long_pool_set(port, i,
- port->priv->bm_pools[i + port->nrxqs].id);
+ bm_pool->port_map |= BIT(port->id);
+ mvpp2_rxq_long_pool_set(port, i, bm_pool->id);
}
port->pool_long = NULL;
@@ -1114,6 +1191,17 @@ mvpp2_shared_interrupt_mask_unmask(struct mvpp2_port *port, bool mask)
}
}
+/* Only GOP port 0 has an XLG MAC */
+static bool mvpp2_port_supports_xlg(struct mvpp2_port *port)
+{
+ return port->gop_id == 0;
+}
+
+static bool mvpp2_port_supports_rgmii(struct mvpp2_port *port)
+{
+ return !(port->priv->hw_version == MVPP22 && port->gop_id == 0);
+}
+
/* Port configuration routines */
static bool mvpp2_is_xlg(phy_interface_t interface)
{
@@ -1121,6 +1209,17 @@ static bool mvpp2_is_xlg(phy_interface_t interface)
interface == PHY_INTERFACE_MODE_XAUI;
}
+static void mvpp2_modify(void __iomem *ptr, u32 mask, u32 set)
+{
+ u32 old, val;
+
+ old = val = readl(ptr);
+ val &= ~mask;
+ val |= set;
+ if (old != val)
+ writel(val, ptr);
+}
+
static void mvpp22_gop_init_rgmii(struct mvpp2_port *port)
{
struct mvpp2 *priv = port->priv;
@@ -1194,7 +1293,7 @@ static int mvpp22_gop_init(struct mvpp2_port *port)
case PHY_INTERFACE_MODE_RGMII_ID:
case PHY_INTERFACE_MODE_RGMII_RXID:
case PHY_INTERFACE_MODE_RGMII_TXID:
- if (port->gop_id == 0)
+ if (!mvpp2_port_supports_rgmii(port))
goto invalid_conf;
mvpp22_gop_init_rgmii(port);
break;
@@ -1204,7 +1303,7 @@ static int mvpp22_gop_init(struct mvpp2_port *port)
mvpp22_gop_init_sgmii(port);
break;
case PHY_INTERFACE_MODE_10GBASER:
- if (port->gop_id != 0)
+ if (!mvpp2_port_supports_xlg(port))
goto invalid_conf;
mvpp22_gop_init_10gkr(port);
break;
@@ -1246,7 +1345,7 @@ static void mvpp22_gop_unmask_irq(struct mvpp2_port *port)
writel(val, port->base + MVPP22_GMAC_INT_SUM_MASK);
}
- if (port->gop_id == 0) {
+ if (mvpp2_port_supports_xlg(port)) {
/* Enable the XLG/GIG irqs for this port */
val = readl(port->base + MVPP22_XLG_EXT_INT_MASK);
if (mvpp2_is_xlg(port->phy_interface))
@@ -1261,7 +1360,7 @@ static void mvpp22_gop_mask_irq(struct mvpp2_port *port)
{
u32 val;
- if (port->gop_id == 0) {
+ if (mvpp2_port_supports_xlg(port)) {
val = readl(port->base + MVPP22_XLG_EXT_INT_MASK);
val &= ~(MVPP22_XLG_EXT_INT_MASK_XLG |
MVPP22_XLG_EXT_INT_MASK_GIG);
@@ -1281,6 +1380,10 @@ static void mvpp22_gop_setup_irq(struct mvpp2_port *port)
{
u32 val;
+ mvpp2_modify(port->base + MVPP22_GMAC_INT_SUM_MASK,
+ MVPP22_GMAC_INT_SUM_MASK_PTP,
+ MVPP22_GMAC_INT_SUM_MASK_PTP);
+
if (port->phylink ||
phy_interface_mode_is_rgmii(port->phy_interface) ||
phy_interface_mode_is_8023z(port->phy_interface) ||
@@ -1290,10 +1393,14 @@ static void mvpp22_gop_setup_irq(struct mvpp2_port *port)
writel(val, port->base + MVPP22_GMAC_INT_MASK);
}
- if (port->gop_id == 0) {
+ if (mvpp2_port_supports_xlg(port)) {
val = readl(port->base + MVPP22_XLG_INT_MASK);
val |= MVPP22_XLG_INT_MASK_LINK;
writel(val, port->base + MVPP22_XLG_INT_MASK);
+
+ mvpp2_modify(port->base + MVPP22_XLG_EXT_INT_MASK,
+ MVPP22_XLG_EXT_INT_MASK_PTP,
+ MVPP22_XLG_EXT_INT_MASK_PTP);
}
mvpp22_gop_unmask_irq(port);
@@ -1328,8 +1435,8 @@ static void mvpp2_port_enable(struct mvpp2_port *port)
{
u32 val;
- /* Only GOP port 0 has an XLG MAC */
- if (port->gop_id == 0 && mvpp2_is_xlg(port->phy_interface)) {
+ if (mvpp2_port_supports_xlg(port) &&
+ mvpp2_is_xlg(port->phy_interface)) {
val = readl(port->base + MVPP22_XLG_CTRL0_REG);
val |= MVPP22_XLG_CTRL0_PORT_EN;
val &= ~MVPP22_XLG_CTRL0_MIB_CNT_DIS;
@@ -1346,8 +1453,8 @@ static void mvpp2_port_disable(struct mvpp2_port *port)
{
u32 val;
- /* Only GOP port 0 has an XLG MAC */
- if (port->gop_id == 0 && mvpp2_is_xlg(port->phy_interface)) {
+ if (mvpp2_port_supports_xlg(port) &&
+ mvpp2_is_xlg(port->phy_interface)) {
val = readl(port->base + MVPP22_XLG_CTRL0_REG);
val &= ~MVPP22_XLG_CTRL0_PORT_EN;
writel(val, port->base + MVPP22_XLG_CTRL0_REG);
@@ -1381,8 +1488,8 @@ static void mvpp2_port_loopback_set(struct mvpp2_port *port,
else
val &= ~MVPP2_GMAC_GMII_LB_EN_MASK;
- if (phy_interface_mode_is_8023z(port->phy_interface) ||
- port->phy_interface == PHY_INTERFACE_MODE_SGMII)
+ if (phy_interface_mode_is_8023z(state->interface) ||
+ state->interface == PHY_INTERFACE_MODE_SGMII)
val |= MVPP2_GMAC_PCS_LB_EN_MASK;
else
val &= ~MVPP2_GMAC_PCS_LB_EN_MASK;
@@ -1390,6 +1497,16 @@ static void mvpp2_port_loopback_set(struct mvpp2_port *port,
writel(val, port->base + MVPP2_GMAC_CTRL_1_REG);
}
+enum {
+ ETHTOOL_XDP_REDIRECT,
+ ETHTOOL_XDP_PASS,
+ ETHTOOL_XDP_DROP,
+ ETHTOOL_XDP_TX,
+ ETHTOOL_XDP_TX_ERR,
+ ETHTOOL_XDP_XMIT,
+ ETHTOOL_XDP_XMIT_ERR,
+};
+
struct mvpp2_ethtool_counter {
unsigned int offset;
const char string[ETH_GSTRING_LEN];
@@ -1482,10 +1599,21 @@ static const struct mvpp2_ethtool_counter mvpp2_ethtool_rxq_regs[] = {
{ MVPP2_RX_PKTS_BM_DROP_CTR, "rxq_%d_packets_bm_drops" },
};
+static const struct mvpp2_ethtool_counter mvpp2_ethtool_xdp[] = {
+ { ETHTOOL_XDP_REDIRECT, "rx_xdp_redirect", },
+ { ETHTOOL_XDP_PASS, "rx_xdp_pass", },
+ { ETHTOOL_XDP_DROP, "rx_xdp_drop", },
+ { ETHTOOL_XDP_TX, "rx_xdp_tx", },
+ { ETHTOOL_XDP_TX_ERR, "rx_xdp_tx_errors", },
+ { ETHTOOL_XDP_XMIT, "tx_xdp_xmit", },
+ { ETHTOOL_XDP_XMIT_ERR, "tx_xdp_xmit_errors", },
+};
+
#define MVPP2_N_ETHTOOL_STATS(ntxqs, nrxqs) (ARRAY_SIZE(mvpp2_ethtool_mib_regs) + \
ARRAY_SIZE(mvpp2_ethtool_port_regs) + \
(ARRAY_SIZE(mvpp2_ethtool_txq_regs) * (ntxqs)) + \
- (ARRAY_SIZE(mvpp2_ethtool_rxq_regs) * (nrxqs)))
+ (ARRAY_SIZE(mvpp2_ethtool_rxq_regs) * (nrxqs)) + \
+ ARRAY_SIZE(mvpp2_ethtool_xdp))
static void mvpp2_ethtool_get_strings(struct net_device *netdev, u32 sset,
u8 *data)
@@ -1524,10 +1652,57 @@ static void mvpp2_ethtool_get_strings(struct net_device *netdev, u32 sset,
data += ETH_GSTRING_LEN;
}
}
+
+ for (i = 0; i < ARRAY_SIZE(mvpp2_ethtool_xdp); i++) {
+ strscpy(data, mvpp2_ethtool_xdp[i].string,
+ ETH_GSTRING_LEN);
+ data += ETH_GSTRING_LEN;
+ }
+}
+
+static void
+mvpp2_get_xdp_stats(struct mvpp2_port *port, struct mvpp2_pcpu_stats *xdp_stats)
+{
+ unsigned int start;
+ unsigned int cpu;
+
+ /* Gather XDP Statistics */
+ for_each_possible_cpu(cpu) {
+ struct mvpp2_pcpu_stats *cpu_stats;
+ u64 xdp_redirect;
+ u64 xdp_pass;
+ u64 xdp_drop;
+ u64 xdp_xmit;
+ u64 xdp_xmit_err;
+ u64 xdp_tx;
+ u64 xdp_tx_err;
+
+ cpu_stats = per_cpu_ptr(port->stats, cpu);
+ do {
+ start = u64_stats_fetch_begin_irq(&cpu_stats->syncp);
+ xdp_redirect = cpu_stats->xdp_redirect;
+ xdp_pass = cpu_stats->xdp_pass;
+ xdp_drop = cpu_stats->xdp_drop;
+ xdp_xmit = cpu_stats->xdp_xmit;
+ xdp_xmit_err = cpu_stats->xdp_xmit_err;
+ xdp_tx = cpu_stats->xdp_tx;
+ xdp_tx_err = cpu_stats->xdp_tx_err;
+ } while (u64_stats_fetch_retry_irq(&cpu_stats->syncp, start));
+
+ xdp_stats->xdp_redirect += xdp_redirect;
+ xdp_stats->xdp_pass += xdp_pass;
+ xdp_stats->xdp_drop += xdp_drop;
+ xdp_stats->xdp_xmit += xdp_xmit;
+ xdp_stats->xdp_xmit_err += xdp_xmit_err;
+ xdp_stats->xdp_tx += xdp_tx;
+ xdp_stats->xdp_tx_err += xdp_tx_err;
+ }
}
static void mvpp2_read_stats(struct mvpp2_port *port)
{
+ struct mvpp2_pcpu_stats xdp_stats = {};
+ const struct mvpp2_ethtool_counter *s;
u64 *pstats;
int i, q;
@@ -1544,7 +1719,7 @@ static void mvpp2_read_stats(struct mvpp2_port *port)
for (q = 0; q < port->ntxqs; q++)
for (i = 0; i < ARRAY_SIZE(mvpp2_ethtool_txq_regs); i++)
*pstats++ += mvpp2_read_index(port->priv,
- MVPP22_CTRS_TX_CTR(port->id, i),
+ MVPP22_CTRS_TX_CTR(port->id, q),
mvpp2_ethtool_txq_regs[i].offset);
/* Rxqs are numbered from 0 from the user standpoint, but not from the
@@ -1553,8 +1728,39 @@ static void mvpp2_read_stats(struct mvpp2_port *port)
for (q = 0; q < port->nrxqs; q++)
for (i = 0; i < ARRAY_SIZE(mvpp2_ethtool_rxq_regs); i++)
*pstats++ += mvpp2_read_index(port->priv,
- port->first_rxq + i,
+ port->first_rxq + q,
mvpp2_ethtool_rxq_regs[i].offset);
+
+ /* Gather XDP Statistics */
+ mvpp2_get_xdp_stats(port, &xdp_stats);
+
+ for (i = 0, s = mvpp2_ethtool_xdp;
+ s < mvpp2_ethtool_xdp + ARRAY_SIZE(mvpp2_ethtool_xdp);
+ s++, i++) {
+ switch (s->offset) {
+ case ETHTOOL_XDP_REDIRECT:
+ *pstats++ = xdp_stats.xdp_redirect;
+ break;
+ case ETHTOOL_XDP_PASS:
+ *pstats++ = xdp_stats.xdp_pass;
+ break;
+ case ETHTOOL_XDP_DROP:
+ *pstats++ = xdp_stats.xdp_drop;
+ break;
+ case ETHTOOL_XDP_TX:
+ *pstats++ = xdp_stats.xdp_tx;
+ break;
+ case ETHTOOL_XDP_TX_ERR:
+ *pstats++ = xdp_stats.xdp_tx_err;
+ break;
+ case ETHTOOL_XDP_XMIT:
+ *pstats++ = xdp_stats.xdp_xmit;
+ break;
+ case ETHTOOL_XDP_XMIT_ERR:
+ *pstats++ = xdp_stats.xdp_xmit_err;
+ break;
+ }
+ }
}
static void mvpp2_gather_hw_statistics(struct work_struct *work)
@@ -2240,11 +2446,15 @@ static void mvpp2_txq_bufs_free(struct mvpp2_port *port,
struct mvpp2_txq_pcpu_buf *tx_buf =
txq_pcpu->buffs + txq_pcpu->txq_get_index;
- if (!IS_TSO_HEADER(txq_pcpu, tx_buf->dma))
+ if (!IS_TSO_HEADER(txq_pcpu, tx_buf->dma) &&
+ tx_buf->type != MVPP2_TYPE_XDP_TX)
dma_unmap_single(port->dev->dev.parent, tx_buf->dma,
tx_buf->size, DMA_TO_DEVICE);
- if (tx_buf->skb)
+ if (tx_buf->type == MVPP2_TYPE_SKB && tx_buf->skb)
dev_kfree_skb_any(tx_buf->skb);
+ else if (tx_buf->type == MVPP2_TYPE_XDP_TX ||
+ tx_buf->type == MVPP2_TYPE_XDP_NDO)
+ xdp_return_frame(tx_buf->xdpf);
mvpp2_txq_inc_get(txq_pcpu);
}
@@ -2353,10 +2563,11 @@ static int mvpp2_aggr_txq_init(struct platform_device *pdev,
/* Create a specified Rx queue */
static int mvpp2_rxq_init(struct mvpp2_port *port,
struct mvpp2_rx_queue *rxq)
-
{
+ struct mvpp2 *priv = port->priv;
unsigned int thread;
u32 rxq_dma;
+ int err;
rxq->size = port->rx_ring_size;
@@ -2385,7 +2596,7 @@ static int mvpp2_rxq_init(struct mvpp2_port *port,
put_cpu();
/* Set Offset */
- mvpp2_rxq_offset_set(port, rxq->id, NET_SKB_PAD);
+ mvpp2_rxq_offset_set(port, rxq->id, MVPP2_SKB_HEADROOM);
/* Set coalescing pkts and time */
mvpp2_rx_pkts_coal_set(port, rxq);
@@ -2394,7 +2605,43 @@ static int mvpp2_rxq_init(struct mvpp2_port *port,
/* Add number of descriptors ready for receiving packets */
mvpp2_rxq_status_update(port, rxq->id, 0, rxq->size);
+ if (priv->percpu_pools) {
+ err = xdp_rxq_info_reg(&rxq->xdp_rxq_short, port->dev, rxq->id);
+ if (err < 0)
+ goto err_free_dma;
+
+ err = xdp_rxq_info_reg(&rxq->xdp_rxq_long, port->dev, rxq->id);
+ if (err < 0)
+ goto err_unregister_rxq_short;
+
+ /* Every RXQ has a pool for short and another for long packets */
+ err = xdp_rxq_info_reg_mem_model(&rxq->xdp_rxq_short,
+ MEM_TYPE_PAGE_POOL,
+ priv->page_pool[rxq->logic_rxq]);
+ if (err < 0)
+ goto err_unregister_rxq_long;
+
+ err = xdp_rxq_info_reg_mem_model(&rxq->xdp_rxq_long,
+ MEM_TYPE_PAGE_POOL,
+ priv->page_pool[rxq->logic_rxq +
+ port->nrxqs]);
+ if (err < 0)
+ goto err_unregister_mem_rxq_short;
+ }
+
return 0;
+
+err_unregister_mem_rxq_short:
+ xdp_rxq_info_unreg_mem_model(&rxq->xdp_rxq_short);
+err_unregister_rxq_long:
+ xdp_rxq_info_unreg(&rxq->xdp_rxq_long);
+err_unregister_rxq_short:
+ xdp_rxq_info_unreg(&rxq->xdp_rxq_short);
+err_free_dma:
+ dma_free_coherent(port->dev->dev.parent,
+ rxq->size * MVPP2_DESC_ALIGNED_SIZE,
+ rxq->descs, rxq->descs_dma);
+ return err;
}
/* Push packets received by the RXQ to BM pool */
@@ -2428,6 +2675,12 @@ static void mvpp2_rxq_deinit(struct mvpp2_port *port,
{
unsigned int thread;
+ if (xdp_rxq_info_is_reg(&rxq->xdp_rxq_short))
+ xdp_rxq_info_unreg(&rxq->xdp_rxq_short);
+
+ if (xdp_rxq_info_is_reg(&rxq->xdp_rxq_long))
+ xdp_rxq_info_unreg(&rxq->xdp_rxq_long);
+
mvpp2_rxq_drop_pkts(port, rxq);
if (rxq->descs)
@@ -2689,7 +2942,7 @@ err_cleanup:
static int mvpp2_setup_txqs(struct mvpp2_port *port)
{
struct mvpp2_tx_queue *txq;
- int queue, err, cpu;
+ int queue, err;
for (queue = 0; queue < port->ntxqs; queue++) {
txq = port->txqs[queue];
@@ -2698,8 +2951,8 @@ static int mvpp2_setup_txqs(struct mvpp2_port *port)
goto err_cleanup;
/* Assign this queue to a CPU */
- cpu = queue % num_present_cpus();
- netif_set_xps_queue(port->dev, cpumask_of(cpu), queue);
+ if (queue < num_possible_cpus())
+ netif_set_xps_queue(port->dev, cpumask_of(queue), queue);
}
if (port->has_tx_irqs) {
@@ -2730,43 +2983,67 @@ static irqreturn_t mvpp2_isr(int irq, void *dev_id)
return IRQ_HANDLED;
}
-/* Per-port interrupt for link status changes */
-static irqreturn_t mvpp2_link_status_isr(int irq, void *dev_id)
+static void mvpp2_isr_handle_ptp_queue(struct mvpp2_port *port, int nq)
{
- struct mvpp2_port *port = (struct mvpp2_port *)dev_id;
- struct net_device *dev = port->dev;
- bool event = false, link = false;
- u32 val;
+ struct skb_shared_hwtstamps shhwtstamps;
+ struct mvpp2_hwtstamp_queue *queue;
+ struct sk_buff *skb;
+ void __iomem *ptp_q;
+ unsigned int id;
+ u32 r0, r1, r2;
- mvpp22_gop_mask_irq(port);
+ ptp_q = port->priv->iface_base + MVPP22_PTP_BASE(port->gop_id);
+ if (nq)
+ ptp_q += MVPP22_PTP_TX_Q1_R0 - MVPP22_PTP_TX_Q0_R0;
- if (port->gop_id == 0 && mvpp2_is_xlg(port->phy_interface)) {
- val = readl(port->base + MVPP22_XLG_INT_STAT);
- if (val & MVPP22_XLG_INT_STAT_LINK) {
- event = true;
- val = readl(port->base + MVPP22_XLG_STATUS);
- if (val & MVPP22_XLG_STATUS_LINK_UP)
- link = true;
- }
- } else if (phy_interface_mode_is_rgmii(port->phy_interface) ||
- phy_interface_mode_is_8023z(port->phy_interface) ||
- port->phy_interface == PHY_INTERFACE_MODE_SGMII) {
- val = readl(port->base + MVPP22_GMAC_INT_STAT);
- if (val & MVPP22_GMAC_INT_STAT_LINK) {
- event = true;
- val = readl(port->base + MVPP2_GMAC_STATUS0);
- if (val & MVPP2_GMAC_STATUS0_LINK_UP)
- link = true;
+ queue = &port->tx_hwtstamp_queue[nq];
+
+ while (1) {
+ r0 = readl_relaxed(ptp_q + MVPP22_PTP_TX_Q0_R0) & 0xffff;
+ if (!r0)
+ break;
+
+ r1 = readl_relaxed(ptp_q + MVPP22_PTP_TX_Q0_R1) & 0xffff;
+ r2 = readl_relaxed(ptp_q + MVPP22_PTP_TX_Q0_R2) & 0xffff;
+
+ id = (r0 >> 1) & 31;
+
+ skb = queue->skb[id];
+ queue->skb[id] = NULL;
+ if (skb) {
+ u32 ts = r2 << 19 | r1 << 3 | r0 >> 13;
+
+ mvpp22_tai_tstamp(port->priv->tai, ts, &shhwtstamps);
+ skb_tstamp_tx(skb, &shhwtstamps);
+ dev_kfree_skb_any(skb);
}
}
+}
+
+static void mvpp2_isr_handle_ptp(struct mvpp2_port *port)
+{
+ void __iomem *ptp;
+ u32 val;
+
+ ptp = port->priv->iface_base + MVPP22_PTP_BASE(port->gop_id);
+ val = readl(ptp + MVPP22_PTP_INT_CAUSE);
+ if (val & MVPP22_PTP_INT_CAUSE_QUEUE0)
+ mvpp2_isr_handle_ptp_queue(port, 0);
+ if (val & MVPP22_PTP_INT_CAUSE_QUEUE1)
+ mvpp2_isr_handle_ptp_queue(port, 1);
+}
+
+static void mvpp2_isr_handle_link(struct mvpp2_port *port, bool link)
+{
+ struct net_device *dev = port->dev;
if (port->phylink) {
phylink_mac_change(port->phylink, link);
- goto handled;
+ return;
}
- if (!netif_running(dev) || !event)
- goto handled;
+ if (!netif_running(dev))
+ return;
if (link) {
mvpp2_interrupts_enable(port);
@@ -2783,8 +3060,65 @@ static irqreturn_t mvpp2_link_status_isr(int irq, void *dev_id)
mvpp2_interrupts_disable(port);
}
+}
+
+static void mvpp2_isr_handle_xlg(struct mvpp2_port *port)
+{
+ bool link;
+ u32 val;
+
+ val = readl(port->base + MVPP22_XLG_INT_STAT);
+ if (val & MVPP22_XLG_INT_STAT_LINK) {
+ val = readl(port->base + MVPP22_XLG_STATUS);
+ link = (val & MVPP22_XLG_STATUS_LINK_UP);
+ mvpp2_isr_handle_link(port, link);
+ }
+}
+
+static void mvpp2_isr_handle_gmac_internal(struct mvpp2_port *port)
+{
+ bool link;
+ u32 val;
+
+ if (phy_interface_mode_is_rgmii(port->phy_interface) ||
+ phy_interface_mode_is_8023z(port->phy_interface) ||
+ port->phy_interface == PHY_INTERFACE_MODE_SGMII) {
+ val = readl(port->base + MVPP22_GMAC_INT_STAT);
+ if (val & MVPP22_GMAC_INT_STAT_LINK) {
+ val = readl(port->base + MVPP2_GMAC_STATUS0);
+ link = (val & MVPP2_GMAC_STATUS0_LINK_UP);
+ mvpp2_isr_handle_link(port, link);
+ }
+ }
+}
+
+/* Per-port interrupt for link status changes */
+static irqreturn_t mvpp2_port_isr(int irq, void *dev_id)
+{
+ struct mvpp2_port *port = (struct mvpp2_port *)dev_id;
+ u32 val;
+
+ mvpp22_gop_mask_irq(port);
+
+ if (mvpp2_port_supports_xlg(port) &&
+ mvpp2_is_xlg(port->phy_interface)) {
+ /* Check the external status register */
+ val = readl(port->base + MVPP22_XLG_EXT_INT_STAT);
+ if (val & MVPP22_XLG_EXT_INT_STAT_XLG)
+ mvpp2_isr_handle_xlg(port);
+ if (val & MVPP22_XLG_EXT_INT_STAT_PTP)
+ mvpp2_isr_handle_ptp(port);
+ } else {
+ /* If it's not the XLG, we must be using the GMAC.
+ * Check the summary status.
+ */
+ val = readl(port->base + MVPP22_GMAC_INT_SUM_STAT);
+ if (val & MVPP22_GMAC_INT_SUM_STAT_INTERNAL)
+ mvpp2_isr_handle_gmac_internal(port);
+ if (val & MVPP22_GMAC_INT_SUM_STAT_PTP)
+ mvpp2_isr_handle_ptp(port);
+ }
-handled:
mvpp22_gop_unmask_irq(port);
return IRQ_HANDLED;
}
@@ -2868,14 +3202,15 @@ static void mvpp2_rx_csum(struct mvpp2_port *port, u32 status,
/* Allocate a new skb and add it to BM pool */
static int mvpp2_rx_refill(struct mvpp2_port *port,
- struct mvpp2_bm_pool *bm_pool, int pool)
+ struct mvpp2_bm_pool *bm_pool,
+ struct page_pool *page_pool, int pool)
{
dma_addr_t dma_addr;
phys_addr_t phys_addr;
void *buf;
- buf = mvpp2_buf_alloc(port, bm_pool, &dma_addr, &phys_addr,
- GFP_ATOMIC);
+ buf = mvpp2_buf_alloc(port, bm_pool, page_pool,
+ &dma_addr, &phys_addr, GFP_ATOMIC);
if (!buf)
return -ENOMEM;
@@ -2916,15 +3251,251 @@ static u32 mvpp2_skb_tx_csum(struct mvpp2_port *port, struct sk_buff *skb)
return MVPP2_TXD_L4_CSUM_NOT | MVPP2_TXD_IP_CSUM_DISABLE;
}
+static void mvpp2_xdp_finish_tx(struct mvpp2_port *port, u16 txq_id, int nxmit, int nxmit_byte)
+{
+ unsigned int thread = mvpp2_cpu_to_thread(port->priv, smp_processor_id());
+ struct mvpp2_tx_queue *aggr_txq;
+ struct mvpp2_txq_pcpu *txq_pcpu;
+ struct mvpp2_tx_queue *txq;
+ struct netdev_queue *nq;
+
+ txq = port->txqs[txq_id];
+ txq_pcpu = per_cpu_ptr(txq->pcpu, thread);
+ nq = netdev_get_tx_queue(port->dev, txq_id);
+ aggr_txq = &port->priv->aggr_txqs[thread];
+
+ txq_pcpu->reserved_num -= nxmit;
+ txq_pcpu->count += nxmit;
+ aggr_txq->count += nxmit;
+
+ /* Enable transmit */
+ wmb();
+ mvpp2_aggr_txq_pend_desc_add(port, nxmit);
+
+ if (txq_pcpu->count >= txq_pcpu->stop_threshold)
+ netif_tx_stop_queue(nq);
+
+ /* Finalize TX processing */
+ if (!port->has_tx_irqs && txq_pcpu->count >= txq->done_pkts_coal)
+ mvpp2_txq_done(port, txq, txq_pcpu);
+}
+
+static int
+mvpp2_xdp_submit_frame(struct mvpp2_port *port, u16 txq_id,
+ struct xdp_frame *xdpf, bool dma_map)
+{
+ unsigned int thread = mvpp2_cpu_to_thread(port->priv, smp_processor_id());
+ u32 tx_cmd = MVPP2_TXD_L4_CSUM_NOT | MVPP2_TXD_IP_CSUM_DISABLE |
+ MVPP2_TXD_F_DESC | MVPP2_TXD_L_DESC;
+ enum mvpp2_tx_buf_type buf_type;
+ struct mvpp2_txq_pcpu *txq_pcpu;
+ struct mvpp2_tx_queue *aggr_txq;
+ struct mvpp2_tx_desc *tx_desc;
+ struct mvpp2_tx_queue *txq;
+ int ret = MVPP2_XDP_TX;
+ dma_addr_t dma_addr;
+
+ txq = port->txqs[txq_id];
+ txq_pcpu = per_cpu_ptr(txq->pcpu, thread);
+ aggr_txq = &port->priv->aggr_txqs[thread];
+
+ /* Check number of available descriptors */
+ if (mvpp2_aggr_desc_num_check(port, aggr_txq, 1) ||
+ mvpp2_txq_reserved_desc_num_proc(port, txq, txq_pcpu, 1)) {
+ ret = MVPP2_XDP_DROPPED;
+ goto out;
+ }
+
+ /* Get a descriptor for the first part of the packet */
+ tx_desc = mvpp2_txq_next_desc_get(aggr_txq);
+ mvpp2_txdesc_txq_set(port, tx_desc, txq->id);
+ mvpp2_txdesc_size_set(port, tx_desc, xdpf->len);
+
+ if (dma_map) {
+ /* XDP_REDIRECT or AF_XDP */
+ dma_addr = dma_map_single(port->dev->dev.parent, xdpf->data,
+ xdpf->len, DMA_TO_DEVICE);
+
+ if (unlikely(dma_mapping_error(port->dev->dev.parent, dma_addr))) {
+ mvpp2_txq_desc_put(txq);
+ ret = MVPP2_XDP_DROPPED;
+ goto out;
+ }
+
+ buf_type = MVPP2_TYPE_XDP_NDO;
+ } else {
+ /* XDP_TX */
+ struct page *page = virt_to_page(xdpf->data);
+
+ dma_addr = page_pool_get_dma_addr(page) +
+ sizeof(*xdpf) + xdpf->headroom;
+ dma_sync_single_for_device(port->dev->dev.parent, dma_addr,
+ xdpf->len, DMA_BIDIRECTIONAL);
+
+ buf_type = MVPP2_TYPE_XDP_TX;
+ }
+
+ mvpp2_txdesc_dma_addr_set(port, tx_desc, dma_addr);
+
+ mvpp2_txdesc_cmd_set(port, tx_desc, tx_cmd);
+ mvpp2_txq_inc_put(port, txq_pcpu, xdpf, tx_desc, buf_type);
+
+out:
+ return ret;
+}
+
+static int
+mvpp2_xdp_xmit_back(struct mvpp2_port *port, struct xdp_buff *xdp)
+{
+ struct mvpp2_pcpu_stats *stats = this_cpu_ptr(port->stats);
+ struct xdp_frame *xdpf;
+ u16 txq_id;
+ int ret;
+
+ xdpf = xdp_convert_buff_to_frame(xdp);
+ if (unlikely(!xdpf))
+ return MVPP2_XDP_DROPPED;
+
+ /* The first of the TX queues are used for XPS,
+ * the second half for XDP_TX
+ */
+ txq_id = mvpp2_cpu_to_thread(port->priv, smp_processor_id()) + (port->ntxqs / 2);
+
+ ret = mvpp2_xdp_submit_frame(port, txq_id, xdpf, false);
+ if (ret == MVPP2_XDP_TX) {
+ u64_stats_update_begin(&stats->syncp);
+ stats->tx_bytes += xdpf->len;
+ stats->tx_packets++;
+ stats->xdp_tx++;
+ u64_stats_update_end(&stats->syncp);
+
+ mvpp2_xdp_finish_tx(port, txq_id, 1, xdpf->len);
+ } else {
+ u64_stats_update_begin(&stats->syncp);
+ stats->xdp_tx_err++;
+ u64_stats_update_end(&stats->syncp);
+ }
+
+ return ret;
+}
+
+static int
+mvpp2_xdp_xmit(struct net_device *dev, int num_frame,
+ struct xdp_frame **frames, u32 flags)
+{
+ struct mvpp2_port *port = netdev_priv(dev);
+ int i, nxmit_byte = 0, nxmit = num_frame;
+ struct mvpp2_pcpu_stats *stats;
+ u16 txq_id;
+ u32 ret;
+
+ if (unlikely(test_bit(0, &port->state)))
+ return -ENETDOWN;
+
+ if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK))
+ return -EINVAL;
+
+ /* The first of the TX queues are used for XPS,
+ * the second half for XDP_TX
+ */
+ txq_id = mvpp2_cpu_to_thread(port->priv, smp_processor_id()) + (port->ntxqs / 2);
+
+ for (i = 0; i < num_frame; i++) {
+ ret = mvpp2_xdp_submit_frame(port, txq_id, frames[i], true);
+ if (ret == MVPP2_XDP_TX) {
+ nxmit_byte += frames[i]->len;
+ } else {
+ xdp_return_frame_rx_napi(frames[i]);
+ nxmit--;
+ }
+ }
+
+ if (likely(nxmit > 0))
+ mvpp2_xdp_finish_tx(port, txq_id, nxmit, nxmit_byte);
+
+ stats = this_cpu_ptr(port->stats);
+ u64_stats_update_begin(&stats->syncp);
+ stats->tx_bytes += nxmit_byte;
+ stats->tx_packets += nxmit;
+ stats->xdp_xmit += nxmit;
+ stats->xdp_xmit_err += num_frame - nxmit;
+ u64_stats_update_end(&stats->syncp);
+
+ return nxmit;
+}
+
+static int
+mvpp2_run_xdp(struct mvpp2_port *port, struct mvpp2_rx_queue *rxq,
+ struct bpf_prog *prog, struct xdp_buff *xdp,
+ struct page_pool *pp, struct mvpp2_pcpu_stats *stats)
+{
+ unsigned int len, sync, err;
+ struct page *page;
+ u32 ret, act;
+
+ len = xdp->data_end - xdp->data_hard_start - MVPP2_SKB_HEADROOM;
+ act = bpf_prog_run_xdp(prog, xdp);
+
+ /* Due xdp_adjust_tail: DMA sync for_device cover max len CPU touch */
+ sync = xdp->data_end - xdp->data_hard_start - MVPP2_SKB_HEADROOM;
+ sync = max(sync, len);
+
+ switch (act) {
+ case XDP_PASS:
+ stats->xdp_pass++;
+ ret = MVPP2_XDP_PASS;
+ break;
+ case XDP_REDIRECT:
+ err = xdp_do_redirect(port->dev, xdp, prog);
+ if (unlikely(err)) {
+ ret = MVPP2_XDP_DROPPED;
+ page = virt_to_head_page(xdp->data);
+ page_pool_put_page(pp, page, sync, true);
+ } else {
+ ret = MVPP2_XDP_REDIR;
+ stats->xdp_redirect++;
+ }
+ break;
+ case XDP_TX:
+ ret = mvpp2_xdp_xmit_back(port, xdp);
+ if (ret != MVPP2_XDP_TX) {
+ page = virt_to_head_page(xdp->data);
+ page_pool_put_page(pp, page, sync, true);
+ }
+ break;
+ default:
+ bpf_warn_invalid_xdp_action(act);
+ fallthrough;
+ case XDP_ABORTED:
+ trace_xdp_exception(port->dev, prog, act);
+ fallthrough;
+ case XDP_DROP:
+ page = virt_to_head_page(xdp->data);
+ page_pool_put_page(pp, page, sync, true);
+ ret = MVPP2_XDP_DROPPED;
+ stats->xdp_drop++;
+ break;
+ }
+
+ return ret;
+}
+
/* Main rx processing */
static int mvpp2_rx(struct mvpp2_port *port, struct napi_struct *napi,
int rx_todo, struct mvpp2_rx_queue *rxq)
{
struct net_device *dev = port->dev;
+ struct mvpp2_pcpu_stats ps = {};
+ enum dma_data_direction dma_dir;
+ struct bpf_prog *xdp_prog;
+ struct xdp_buff xdp;
int rx_received;
int rx_done = 0;
- u32 rcvd_pkts = 0;
- u32 rcvd_bytes = 0;
+ u32 xdp_ret = 0;
+
+ rcu_read_lock();
+
+ xdp_prog = READ_ONCE(port->xdp_prog);
/* Get number of received packets and clamp the to-do */
rx_received = mvpp2_rxq_received(port, rxq->id);
@@ -2934,12 +3505,13 @@ static int mvpp2_rx(struct mvpp2_port *port, struct napi_struct *napi,
while (rx_done < rx_todo) {
struct mvpp2_rx_desc *rx_desc = mvpp2_rxq_next_desc_get(rxq);
struct mvpp2_bm_pool *bm_pool;
+ struct page_pool *pp = NULL;
struct sk_buff *skb;
unsigned int frag_size;
dma_addr_t dma_addr;
phys_addr_t phys_addr;
- u32 rx_status;
- int pool, rx_bytes, err;
+ u32 rx_status, timestamp;
+ int pool, rx_bytes, err, ret;
void *data;
rx_done++;
@@ -2962,9 +3534,18 @@ static int mvpp2_rx(struct mvpp2_port *port, struct napi_struct *napi,
if (rx_status & MVPP2_RXD_ERR_SUMMARY)
goto err_drop_frame;
+ if (port->priv->percpu_pools) {
+ pp = port->priv->page_pool[pool];
+ dma_dir = page_pool_get_dma_dir(pp);
+ } else {
+ dma_dir = DMA_FROM_DEVICE;
+ }
+
dma_sync_single_for_cpu(dev->dev.parent, dma_addr,
rx_bytes + MVPP2_MH_SIZE,
- DMA_FROM_DEVICE);
+ dma_dir);
+
+ /* Prefetch header */
prefetch(data);
if (bm_pool->frag_size > PAGE_SIZE)
@@ -2972,26 +3553,68 @@ static int mvpp2_rx(struct mvpp2_port *port, struct napi_struct *napi,
else
frag_size = bm_pool->frag_size;
+ if (xdp_prog) {
+ xdp.data_hard_start = data;
+ xdp.data = data + MVPP2_MH_SIZE + MVPP2_SKB_HEADROOM;
+ xdp.data_end = xdp.data + rx_bytes;
+ xdp.frame_sz = PAGE_SIZE;
+
+ if (bm_pool->pkt_size == MVPP2_BM_SHORT_PKT_SIZE)
+ xdp.rxq = &rxq->xdp_rxq_short;
+ else
+ xdp.rxq = &rxq->xdp_rxq_long;
+
+ xdp_set_data_meta_invalid(&xdp);
+
+ ret = mvpp2_run_xdp(port, rxq, xdp_prog, &xdp, pp, &ps);
+
+ if (ret) {
+ xdp_ret |= ret;
+ err = mvpp2_rx_refill(port, bm_pool, pp, pool);
+ if (err) {
+ netdev_err(port->dev, "failed to refill BM pools\n");
+ goto err_drop_frame;
+ }
+
+ ps.rx_packets++;
+ ps.rx_bytes += rx_bytes;
+ continue;
+ }
+ }
+
skb = build_skb(data, frag_size);
if (!skb) {
netdev_warn(port->dev, "skb build failed\n");
goto err_drop_frame;
}
- err = mvpp2_rx_refill(port, bm_pool, pool);
+ /* If we have RX hardware timestamping enabled, grab the
+ * timestamp from the queue and convert.
+ */
+ if (mvpp22_rx_hwtstamping(port)) {
+ timestamp = le32_to_cpu(rx_desc->pp22.timestamp);
+ mvpp22_tai_tstamp(port->priv->tai, timestamp,
+ skb_hwtstamps(skb));
+ }
+
+ err = mvpp2_rx_refill(port, bm_pool, pp, pool);
if (err) {
netdev_err(port->dev, "failed to refill BM pools\n");
+ dev_kfree_skb_any(skb);
goto err_drop_frame;
}
- dma_unmap_single_attrs(dev->dev.parent, dma_addr,
- bm_pool->buf_size, DMA_FROM_DEVICE,
- DMA_ATTR_SKIP_CPU_SYNC);
+ if (pp)
+ page_pool_release_page(pp, virt_to_page(data));
+ else
+ dma_unmap_single_attrs(dev->dev.parent, dma_addr,
+ bm_pool->buf_size, DMA_FROM_DEVICE,
+ DMA_ATTR_SKIP_CPU_SYNC);
- rcvd_pkts++;
- rcvd_bytes += rx_bytes;
+ ps.rx_packets++;
+ ps.rx_bytes += rx_bytes;
- skb_reserve(skb, MVPP2_MH_SIZE + NET_SKB_PAD);
+ skb_reserve(skb, MVPP2_MH_SIZE + MVPP2_SKB_HEADROOM);
skb_put(skb, rx_bytes);
skb->protocol = eth_type_trans(skb, dev);
mvpp2_rx_csum(port, rx_status, skb);
@@ -3006,12 +3629,21 @@ err_drop_frame:
mvpp2_bm_pool_put(port, pool, dma_addr, phys_addr);
}
- if (rcvd_pkts) {
+ rcu_read_unlock();
+
+ if (xdp_ret & MVPP2_XDP_REDIR)
+ xdp_do_flush_map();
+
+ if (ps.rx_packets) {
struct mvpp2_pcpu_stats *stats = this_cpu_ptr(port->stats);
u64_stats_update_begin(&stats->syncp);
- stats->rx_packets += rcvd_pkts;
- stats->rx_bytes += rcvd_bytes;
+ stats->rx_packets += ps.rx_packets;
+ stats->rx_bytes += ps.rx_bytes;
+ /* xdp */
+ stats->xdp_redirect += ps.xdp_redirect;
+ stats->xdp_pass += ps.xdp_pass;
+ stats->xdp_drop += ps.xdp_drop;
u64_stats_update_end(&stats->syncp);
}
@@ -3039,6 +3671,94 @@ tx_desc_unmap_put(struct mvpp2_port *port, struct mvpp2_tx_queue *txq,
mvpp2_txq_desc_put(txq);
}
+static void mvpp2_txdesc_clear_ptp(struct mvpp2_port *port,
+ struct mvpp2_tx_desc *desc)
+{
+ /* We only need to clear the low bits */
+ if (port->priv->hw_version != MVPP21)
+ desc->pp22.ptp_descriptor &=
+ cpu_to_le32(~MVPP22_PTP_DESC_MASK_LOW);
+}
+
+static bool mvpp2_tx_hw_tstamp(struct mvpp2_port *port,
+ struct mvpp2_tx_desc *tx_desc,
+ struct sk_buff *skb)
+{
+ struct mvpp2_hwtstamp_queue *queue;
+ unsigned int mtype, type, i;
+ struct ptp_header *hdr;
+ u64 ptpdesc;
+
+ if (port->priv->hw_version == MVPP21 ||
+ port->tx_hwtstamp_type == HWTSTAMP_TX_OFF)
+ return false;
+
+ type = ptp_classify_raw(skb);
+ if (!type)
+ return false;
+
+ hdr = ptp_parse_header(skb, type);
+ if (!hdr)
+ return false;
+
+ skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
+
+ ptpdesc = MVPP22_PTP_MACTIMESTAMPINGEN |
+ MVPP22_PTP_ACTION_CAPTURE;
+ queue = &port->tx_hwtstamp_queue[0];
+
+ switch (type & PTP_CLASS_VMASK) {
+ case PTP_CLASS_V1:
+ ptpdesc |= MVPP22_PTP_PACKETFORMAT(MVPP22_PTP_PKT_FMT_PTPV1);
+ break;
+
+ case PTP_CLASS_V2:
+ ptpdesc |= MVPP22_PTP_PACKETFORMAT(MVPP22_PTP_PKT_FMT_PTPV2);
+ mtype = hdr->tsmt & 15;
+ /* Direct PTP Sync messages to queue 1 */
+ if (mtype == 0) {
+ ptpdesc |= MVPP22_PTP_TIMESTAMPQUEUESELECT;
+ queue = &port->tx_hwtstamp_queue[1];
+ }
+ break;
+ }
+
+ /* Take a reference on the skb and insert into our queue */
+ i = queue->next;
+ queue->next = (i + 1) & 31;
+ if (queue->skb[i])
+ dev_kfree_skb_any(queue->skb[i]);
+ queue->skb[i] = skb_get(skb);
+
+ ptpdesc |= MVPP22_PTP_TIMESTAMPENTRYID(i);
+
+ /*
+ * 3:0 - PTPAction
+ * 6:4 - PTPPacketFormat
+ * 7 - PTP_CF_WraparoundCheckEn
+ * 9:8 - IngressTimestampSeconds[1:0]
+ * 10 - Reserved
+ * 11 - MACTimestampingEn
+ * 17:12 - PTP_TimestampQueueEntryID[5:0]
+ * 18 - PTPTimestampQueueSelect
+ * 19 - UDPChecksumUpdateEn
+ * 27:20 - TimestampOffset
+ * PTP, NTPTransmit, OWAMP/TWAMP - L3 to PTP header
+ * NTPTs, Y.1731 - L3 to timestamp entry
+ * 35:28 - UDP Checksum Offset
+ *
+ * stored in tx descriptor bits 75:64 (11:0) and 191:168 (35:12)
+ */
+ tx_desc->pp22.ptp_descriptor &=
+ cpu_to_le32(~MVPP22_PTP_DESC_MASK_LOW);
+ tx_desc->pp22.ptp_descriptor |=
+ cpu_to_le32(ptpdesc & MVPP22_PTP_DESC_MASK_LOW);
+ tx_desc->pp22.buf_dma_addr_ptp &= cpu_to_le64(~0xffffff0000000000ULL);
+ tx_desc->pp22.buf_dma_addr_ptp |= cpu_to_le64((ptpdesc >> 12) << 40);
+
+ return true;
+}
+
/* Handle tx fragmentation processing */
static int mvpp2_tx_frag_process(struct mvpp2_port *port, struct sk_buff *skb,
struct mvpp2_tx_queue *aggr_txq,
@@ -3055,6 +3775,7 @@ static int mvpp2_tx_frag_process(struct mvpp2_port *port, struct sk_buff *skb,
void *addr = skb_frag_address(frag);
tx_desc = mvpp2_txq_next_desc_get(aggr_txq);
+ mvpp2_txdesc_clear_ptp(port, tx_desc);
mvpp2_txdesc_txq_set(port, tx_desc, txq->id);
mvpp2_txdesc_size_set(port, tx_desc, skb_frag_size(frag));
@@ -3072,11 +3793,11 @@ static int mvpp2_tx_frag_process(struct mvpp2_port *port, struct sk_buff *skb,
/* Last descriptor */
mvpp2_txdesc_cmd_set(port, tx_desc,
MVPP2_TXD_L_DESC);
- mvpp2_txq_inc_put(port, txq_pcpu, skb, tx_desc);
+ mvpp2_txq_inc_put(port, txq_pcpu, skb, tx_desc, MVPP2_TYPE_SKB);
} else {
/* Descriptor in the middle: Not First, Not Last */
mvpp2_txdesc_cmd_set(port, tx_desc, 0);
- mvpp2_txq_inc_put(port, txq_pcpu, NULL, tx_desc);
+ mvpp2_txq_inc_put(port, txq_pcpu, NULL, tx_desc, MVPP2_TYPE_SKB);
}
}
@@ -3104,6 +3825,7 @@ static inline void mvpp2_tso_put_hdr(struct sk_buff *skb,
struct mvpp2_tx_desc *tx_desc = mvpp2_txq_next_desc_get(aggr_txq);
dma_addr_t addr;
+ mvpp2_txdesc_clear_ptp(port, tx_desc);
mvpp2_txdesc_txq_set(port, tx_desc, txq->id);
mvpp2_txdesc_size_set(port, tx_desc, hdr_sz);
@@ -3114,7 +3836,7 @@ static inline void mvpp2_tso_put_hdr(struct sk_buff *skb,
mvpp2_txdesc_cmd_set(port, tx_desc, mvpp2_skb_tx_csum(port, skb) |
MVPP2_TXD_F_DESC |
MVPP2_TXD_PADDING_DISABLE);
- mvpp2_txq_inc_put(port, txq_pcpu, NULL, tx_desc);
+ mvpp2_txq_inc_put(port, txq_pcpu, NULL, tx_desc, MVPP2_TYPE_SKB);
}
static inline int mvpp2_tso_put_data(struct sk_buff *skb,
@@ -3128,6 +3850,7 @@ static inline int mvpp2_tso_put_data(struct sk_buff *skb,
struct mvpp2_tx_desc *tx_desc = mvpp2_txq_next_desc_get(aggr_txq);
dma_addr_t buf_dma_addr;
+ mvpp2_txdesc_clear_ptp(port, tx_desc);
mvpp2_txdesc_txq_set(port, tx_desc, txq->id);
mvpp2_txdesc_size_set(port, tx_desc, sz);
@@ -3143,14 +3866,14 @@ static inline int mvpp2_tso_put_data(struct sk_buff *skb,
if (!left) {
mvpp2_txdesc_cmd_set(port, tx_desc, MVPP2_TXD_L_DESC);
if (last) {
- mvpp2_txq_inc_put(port, txq_pcpu, skb, tx_desc);
+ mvpp2_txq_inc_put(port, txq_pcpu, skb, tx_desc, MVPP2_TYPE_SKB);
return 0;
}
} else {
mvpp2_txdesc_cmd_set(port, tx_desc, 0);
}
- mvpp2_txq_inc_put(port, txq_pcpu, NULL, tx_desc);
+ mvpp2_txq_inc_put(port, txq_pcpu, NULL, tx_desc, MVPP2_TYPE_SKB);
return 0;
}
@@ -3160,9 +3883,8 @@ static int mvpp2_tx_tso(struct sk_buff *skb, struct net_device *dev,
struct mvpp2_txq_pcpu *txq_pcpu)
{
struct mvpp2_port *port = netdev_priv(dev);
+ int hdr_sz, i, len, descs = 0;
struct tso_t tso;
- int hdr_sz = skb_transport_offset(skb) + tcp_hdrlen(skb);
- int i, len, descs = 0;
/* Check number of available descriptors */
if (mvpp2_aggr_desc_num_check(port, aggr_txq, tso_count_descs(skb)) ||
@@ -3170,7 +3892,8 @@ static int mvpp2_tx_tso(struct sk_buff *skb, struct net_device *dev,
tso_count_descs(skb)))
return 0;
- tso_start(skb, &tso);
+ hdr_sz = tso_start(skb, &tso);
+
len = skb->len - hdr_sz;
while (len > 0) {
int left = min_t(int, skb_shinfo(skb)->gso_size, len);
@@ -3244,6 +3967,9 @@ static netdev_tx_t mvpp2_tx(struct sk_buff *skb, struct net_device *dev)
/* Get a descriptor for the first part of the packet */
tx_desc = mvpp2_txq_next_desc_get(aggr_txq);
+ if (!(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) ||
+ !mvpp2_tx_hw_tstamp(port, tx_desc, skb))
+ mvpp2_txdesc_clear_ptp(port, tx_desc);
mvpp2_txdesc_txq_set(port, tx_desc, txq->id);
mvpp2_txdesc_size_set(port, tx_desc, skb_headlen(skb));
@@ -3263,12 +3989,12 @@ static netdev_tx_t mvpp2_tx(struct sk_buff *skb, struct net_device *dev)
/* First and Last descriptor */
tx_cmd |= MVPP2_TXD_F_DESC | MVPP2_TXD_L_DESC;
mvpp2_txdesc_cmd_set(port, tx_desc, tx_cmd);
- mvpp2_txq_inc_put(port, txq_pcpu, skb, tx_desc);
+ mvpp2_txq_inc_put(port, txq_pcpu, skb, tx_desc, MVPP2_TYPE_SKB);
} else {
/* First but not Last */
tx_cmd |= MVPP2_TXD_F_DESC | MVPP2_TXD_PADDING_DISABLE;
mvpp2_txdesc_cmd_set(port, tx_desc, tx_cmd);
- mvpp2_txq_inc_put(port, txq_pcpu, NULL, tx_desc);
+ mvpp2_txq_inc_put(port, txq_pcpu, NULL, tx_desc, MVPP2_TYPE_SKB);
/* Continue with other skb fragments */
if (mvpp2_tx_frag_process(port, skb, aggr_txq, txq)) {
@@ -3430,8 +4156,7 @@ static void mvpp22_mode_reconfigure(struct mvpp2_port *port)
mvpp22_pcs_reset_deassert(port);
- /* Only GOP port 0 has an XLG MAC */
- if (port->gop_id == 0) {
+ if (mvpp2_port_supports_xlg(port)) {
ctrl3 = readl(port->base + MVPP22_XLG_CTRL3_REG);
ctrl3 &= ~MVPP22_XLG_CTRL3_MACMODESELECT_MASK;
@@ -3443,7 +4168,7 @@ static void mvpp22_mode_reconfigure(struct mvpp2_port *port)
writel(ctrl3, port->base + MVPP22_XLG_CTRL3_REG);
}
- if (port->gop_id == 0 && mvpp2_is_xlg(port->phy_interface))
+ if (mvpp2_port_supports_xlg(port) && mvpp2_is_xlg(port->phy_interface))
mvpp2_xlg_max_rx_size_set(port);
else
mvpp2_gmac_max_rx_size_set(port);
@@ -3468,20 +4193,12 @@ static void mvpp2_start_dev(struct mvpp2_port *port)
if (port->phylink) {
phylink_start(port->phylink);
} else {
- /* Phylink isn't used as of now for ACPI, so the MAC has to be
- * configured manually when the interface is started. This will
- * be removed as soon as the phylink ACPI support lands in.
- */
- struct phylink_link_state state = {
- .interface = port->phy_interface,
- };
- mvpp2_mac_config(&port->phylink_config, MLO_AN_INBAND, &state);
- mvpp2_mac_link_up(&port->phylink_config, NULL,
- MLO_AN_INBAND, port->phy_interface,
- SPEED_UNKNOWN, DUPLEX_UNKNOWN, false, false);
+ mvpp2_acpi_start(port);
}
netif_tx_start_all_queues(port->dev);
+
+ clear_bit(0, &port->state);
}
/* Set hw internals when stopping port */
@@ -3489,6 +4206,8 @@ static void mvpp2_stop_dev(struct mvpp2_port *port)
{
int i;
+ set_bit(0, &port->state);
+
/* Disable interrupts on all threads */
mvpp2_interrupts_disable(port);
@@ -3684,12 +4403,13 @@ static int mvpp2_open(struct net_device *dev)
valid = true;
}
- if (priv->hw_version == MVPP22 && port->link_irq) {
- err = request_irq(port->link_irq, mvpp2_link_status_isr, 0,
+ if (priv->hw_version == MVPP22 && port->port_irq) {
+ err = request_irq(port->port_irq, mvpp2_port_isr, 0,
dev->name, port);
if (err) {
- netdev_err(port->dev, "cannot request link IRQ %d\n",
- port->link_irq);
+ netdev_err(port->dev,
+ "cannot request port link/ptp IRQ %d\n",
+ port->port_irq);
goto err_free_irq;
}
@@ -3700,7 +4420,7 @@ static int mvpp2_open(struct net_device *dev)
valid = true;
} else {
- port->link_irq = 0;
+ port->port_irq = 0;
}
if (!valid) {
@@ -3744,8 +4464,8 @@ static int mvpp2_stop(struct net_device *dev)
if (port->phylink)
phylink_disconnect_phy(port->phylink);
- if (port->link_irq)
- free_irq(port->link_irq, port);
+ if (port->port_irq)
+ free_irq(port->port_irq, port);
mvpp2_irqs_deinit(port);
if (!port->has_tx_irqs) {
@@ -3895,6 +4615,10 @@ static int mvpp2_change_mtu(struct net_device *dev, int mtu)
}
if (MVPP2_RX_PKT_SIZE(mtu) > MVPP2_BM_LONG_PKT_SIZE) {
+ if (port->xdp_prog) {
+ netdev_err(dev, "Jumbo frames are not supported with XDP\n");
+ return -EINVAL;
+ }
if (priv->percpu_pools) {
netdev_warn(dev, "mtu %d too high, switching to shared buffers", mtu);
mvpp2_bm_switch_buffers(priv, false);
@@ -3940,6 +4664,33 @@ static int mvpp2_change_mtu(struct net_device *dev, int mtu)
return err;
}
+static int mvpp2_check_pagepool_dma(struct mvpp2_port *port)
+{
+ enum dma_data_direction dma_dir = DMA_FROM_DEVICE;
+ struct mvpp2 *priv = port->priv;
+ int err = -1, i;
+
+ if (!priv->percpu_pools)
+ return err;
+
+ if (!priv->page_pool[0])
+ return -ENOMEM;
+
+ for (i = 0; i < priv->port_count; i++) {
+ port = priv->port_list[i];
+ if (port->xdp_prog) {
+ dma_dir = DMA_BIDIRECTIONAL;
+ break;
+ }
+ }
+
+ /* All pools are equal in terms of DMA direction */
+ if (priv->page_pool[0]->p.dma_dir != dma_dir)
+ err = mvpp2_bm_switch_buffers(priv, true);
+
+ return err;
+}
+
static void
mvpp2_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats)
{
@@ -3974,10 +4725,124 @@ mvpp2_get_stats64(struct net_device *dev, struct rtnl_link_stats64 *stats)
stats->tx_dropped = dev->stats.tx_dropped;
}
+static int mvpp2_set_ts_config(struct mvpp2_port *port, struct ifreq *ifr)
+{
+ struct hwtstamp_config config;
+ void __iomem *ptp;
+ u32 gcr, int_mask;
+
+ if (copy_from_user(&config, ifr->ifr_data, sizeof(config)))
+ return -EFAULT;
+
+ if (config.flags)
+ return -EINVAL;
+
+ if (config.tx_type != HWTSTAMP_TX_OFF &&
+ config.tx_type != HWTSTAMP_TX_ON)
+ return -ERANGE;
+
+ ptp = port->priv->iface_base + MVPP22_PTP_BASE(port->gop_id);
+
+ int_mask = gcr = 0;
+ if (config.tx_type != HWTSTAMP_TX_OFF) {
+ gcr |= MVPP22_PTP_GCR_TSU_ENABLE | MVPP22_PTP_GCR_TX_RESET;
+ int_mask |= MVPP22_PTP_INT_MASK_QUEUE1 |
+ MVPP22_PTP_INT_MASK_QUEUE0;
+ }
+
+ /* It seems we must also release the TX reset when enabling the TSU */
+ if (config.rx_filter != HWTSTAMP_FILTER_NONE)
+ gcr |= MVPP22_PTP_GCR_TSU_ENABLE | MVPP22_PTP_GCR_RX_RESET |
+ MVPP22_PTP_GCR_TX_RESET;
+
+ if (gcr & MVPP22_PTP_GCR_TSU_ENABLE)
+ mvpp22_tai_start(port->priv->tai);
+
+ if (config.rx_filter != HWTSTAMP_FILTER_NONE) {
+ config.rx_filter = HWTSTAMP_FILTER_ALL;
+ mvpp2_modify(ptp + MVPP22_PTP_GCR,
+ MVPP22_PTP_GCR_RX_RESET |
+ MVPP22_PTP_GCR_TX_RESET |
+ MVPP22_PTP_GCR_TSU_ENABLE, gcr);
+ port->rx_hwtstamp = true;
+ } else {
+ port->rx_hwtstamp = false;
+ mvpp2_modify(ptp + MVPP22_PTP_GCR,
+ MVPP22_PTP_GCR_RX_RESET |
+ MVPP22_PTP_GCR_TX_RESET |
+ MVPP22_PTP_GCR_TSU_ENABLE, gcr);
+ }
+
+ mvpp2_modify(ptp + MVPP22_PTP_INT_MASK,
+ MVPP22_PTP_INT_MASK_QUEUE1 |
+ MVPP22_PTP_INT_MASK_QUEUE0, int_mask);
+
+ if (!(gcr & MVPP22_PTP_GCR_TSU_ENABLE))
+ mvpp22_tai_stop(port->priv->tai);
+
+ port->tx_hwtstamp_type = config.tx_type;
+
+ if (copy_to_user(ifr->ifr_data, &config, sizeof(config)))
+ return -EFAULT;
+
+ return 0;
+}
+
+static int mvpp2_get_ts_config(struct mvpp2_port *port, struct ifreq *ifr)
+{
+ struct hwtstamp_config config;
+
+ memset(&config, 0, sizeof(config));
+
+ config.tx_type = port->tx_hwtstamp_type;
+ config.rx_filter = port->rx_hwtstamp ?
+ HWTSTAMP_FILTER_ALL : HWTSTAMP_FILTER_NONE;
+
+ if (copy_to_user(ifr->ifr_data, &config, sizeof(config)))
+ return -EFAULT;
+
+ return 0;
+}
+
+static int mvpp2_ethtool_get_ts_info(struct net_device *dev,
+ struct ethtool_ts_info *info)
+{
+ struct mvpp2_port *port = netdev_priv(dev);
+
+ if (!port->hwtstamp)
+ return -EOPNOTSUPP;
+
+ info->phc_index = mvpp22_tai_ptp_clock_index(port->priv->tai);
+ info->so_timestamping = SOF_TIMESTAMPING_TX_SOFTWARE |
+ SOF_TIMESTAMPING_RX_SOFTWARE |
+ SOF_TIMESTAMPING_SOFTWARE |
+ SOF_TIMESTAMPING_TX_HARDWARE |
+ SOF_TIMESTAMPING_RX_HARDWARE |
+ SOF_TIMESTAMPING_RAW_HARDWARE;
+ info->tx_types = BIT(HWTSTAMP_TX_OFF) |
+ BIT(HWTSTAMP_TX_ON);
+ info->rx_filters = BIT(HWTSTAMP_FILTER_NONE) |
+ BIT(HWTSTAMP_FILTER_ALL);
+
+ return 0;
+}
+
static int mvpp2_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd)
{
struct mvpp2_port *port = netdev_priv(dev);
+ switch (cmd) {
+ case SIOCSHWTSTAMP:
+ if (port->hwtstamp)
+ return mvpp2_set_ts_config(port, ifr);
+ break;
+
+ case SIOCGHWTSTAMP:
+ if (port->hwtstamp)
+ return mvpp2_get_ts_config(port, ifr);
+ break;
+ }
+
if (!port->phylink)
return -ENOTSUPP;
@@ -4033,6 +4898,61 @@ static int mvpp2_set_features(struct net_device *dev,
return 0;
}
+static int mvpp2_xdp_setup(struct mvpp2_port *port, struct netdev_bpf *bpf)
+{
+ struct bpf_prog *prog = bpf->prog, *old_prog;
+ bool running = netif_running(port->dev);
+ bool reset = !prog != !port->xdp_prog;
+
+ if (port->dev->mtu > ETH_DATA_LEN) {
+ NL_SET_ERR_MSG_MOD(bpf->extack, "XDP is not supported with jumbo frames enabled");
+ return -EOPNOTSUPP;
+ }
+
+ if (!port->priv->percpu_pools) {
+ NL_SET_ERR_MSG_MOD(bpf->extack, "Per CPU Pools required for XDP");
+ return -EOPNOTSUPP;
+ }
+
+ if (port->ntxqs < num_possible_cpus() * 2) {
+ NL_SET_ERR_MSG_MOD(bpf->extack, "XDP_TX needs two TX queues per CPU");
+ return -EOPNOTSUPP;
+ }
+
+ /* device is up and bpf is added/removed, must setup the RX queues */
+ if (running && reset)
+ mvpp2_stop(port->dev);
+
+ old_prog = xchg(&port->xdp_prog, prog);
+ if (old_prog)
+ bpf_prog_put(old_prog);
+
+ /* bpf is just replaced, RXQ and MTU are already setup */
+ if (!reset)
+ return 0;
+
+ /* device was up, restore the link */
+ if (running)
+ mvpp2_open(port->dev);
+
+ /* Check Page Pool DMA Direction */
+ mvpp2_check_pagepool_dma(port);
+
+ return 0;
+}
+
+static int mvpp2_xdp(struct net_device *dev, struct netdev_bpf *xdp)
+{
+ struct mvpp2_port *port = netdev_priv(dev);
+
+ switch (xdp->command) {
+ case XDP_SETUP_PROG:
+ return mvpp2_xdp_setup(port, xdp);
+ default:
+ return -EINVAL;
+ }
+}
+
/* Ethtool methods */
static int mvpp2_ethtool_nway_reset(struct net_device *dev)
@@ -4383,6 +5303,8 @@ static const struct net_device_ops mvpp2_netdev_ops = {
.ndo_vlan_rx_add_vid = mvpp2_vlan_rx_add_vid,
.ndo_vlan_rx_kill_vid = mvpp2_vlan_rx_kill_vid,
.ndo_set_features = mvpp2_set_features,
+ .ndo_bpf = mvpp2_xdp,
+ .ndo_xdp_xmit = mvpp2_xdp_xmit,
};
static const struct ethtool_ops mvpp2_eth_tool_ops = {
@@ -4390,6 +5312,7 @@ static const struct ethtool_ops mvpp2_eth_tool_ops = {
ETHTOOL_COALESCE_MAX_FRAMES,
.nway_reset = mvpp2_ethtool_nway_reset,
.get_link = ethtool_op_get_link,
+ .get_ts_info = mvpp2_ethtool_get_ts_info,
.set_coalesce = mvpp2_ethtool_set_coalesce,
.get_coalesce = mvpp2_ethtool_get_coalesce,
.get_drvinfo = mvpp2_ethtool_get_drvinfo,
@@ -4756,26 +5679,179 @@ static void mvpp2_port_copy_mac_addr(struct net_device *dev, struct mvpp2 *priv,
eth_hw_addr_random(dev);
}
+static struct mvpp2_port *mvpp2_phylink_to_port(struct phylink_config *config)
+{
+ return container_of(config, struct mvpp2_port, phylink_config);
+}
+
+static struct mvpp2_port *mvpp2_pcs_to_port(struct phylink_pcs *pcs)
+{
+ return container_of(pcs, struct mvpp2_port, phylink_pcs);
+}
+
+static void mvpp2_xlg_pcs_get_state(struct phylink_pcs *pcs,
+ struct phylink_link_state *state)
+{
+ struct mvpp2_port *port = mvpp2_pcs_to_port(pcs);
+ u32 val;
+
+ state->speed = SPEED_10000;
+ state->duplex = 1;
+ state->an_complete = 1;
+
+ val = readl(port->base + MVPP22_XLG_STATUS);
+ state->link = !!(val & MVPP22_XLG_STATUS_LINK_UP);
+
+ state->pause = 0;
+ val = readl(port->base + MVPP22_XLG_CTRL0_REG);
+ if (val & MVPP22_XLG_CTRL0_TX_FLOW_CTRL_EN)
+ state->pause |= MLO_PAUSE_TX;
+ if (val & MVPP22_XLG_CTRL0_RX_FLOW_CTRL_EN)
+ state->pause |= MLO_PAUSE_RX;
+}
+
+static int mvpp2_xlg_pcs_config(struct phylink_pcs *pcs,
+ unsigned int mode,
+ phy_interface_t interface,
+ const unsigned long *advertising,
+ bool permit_pause_to_mac)
+{
+ return 0;
+}
+
+static const struct phylink_pcs_ops mvpp2_phylink_xlg_pcs_ops = {
+ .pcs_get_state = mvpp2_xlg_pcs_get_state,
+ .pcs_config = mvpp2_xlg_pcs_config,
+};
+
+static void mvpp2_gmac_pcs_get_state(struct phylink_pcs *pcs,
+ struct phylink_link_state *state)
+{
+ struct mvpp2_port *port = mvpp2_pcs_to_port(pcs);
+ u32 val;
+
+ val = readl(port->base + MVPP2_GMAC_STATUS0);
+
+ state->an_complete = !!(val & MVPP2_GMAC_STATUS0_AN_COMPLETE);
+ state->link = !!(val & MVPP2_GMAC_STATUS0_LINK_UP);
+ state->duplex = !!(val & MVPP2_GMAC_STATUS0_FULL_DUPLEX);
+
+ switch (port->phy_interface) {
+ case PHY_INTERFACE_MODE_1000BASEX:
+ state->speed = SPEED_1000;
+ break;
+ case PHY_INTERFACE_MODE_2500BASEX:
+ state->speed = SPEED_2500;
+ break;
+ default:
+ if (val & MVPP2_GMAC_STATUS0_GMII_SPEED)
+ state->speed = SPEED_1000;
+ else if (val & MVPP2_GMAC_STATUS0_MII_SPEED)
+ state->speed = SPEED_100;
+ else
+ state->speed = SPEED_10;
+ }
+
+ state->pause = 0;
+ if (val & MVPP2_GMAC_STATUS0_RX_PAUSE)
+ state->pause |= MLO_PAUSE_RX;
+ if (val & MVPP2_GMAC_STATUS0_TX_PAUSE)
+ state->pause |= MLO_PAUSE_TX;
+}
+
+static int mvpp2_gmac_pcs_config(struct phylink_pcs *pcs, unsigned int mode,
+ phy_interface_t interface,
+ const unsigned long *advertising,
+ bool permit_pause_to_mac)
+{
+ struct mvpp2_port *port = mvpp2_pcs_to_port(pcs);
+ u32 mask, val, an, old_an, changed;
+
+ mask = MVPP2_GMAC_IN_BAND_AUTONEG_BYPASS |
+ MVPP2_GMAC_IN_BAND_AUTONEG |
+ MVPP2_GMAC_AN_SPEED_EN |
+ MVPP2_GMAC_FLOW_CTRL_AUTONEG |
+ MVPP2_GMAC_AN_DUPLEX_EN;
+
+ if (phylink_autoneg_inband(mode)) {
+ mask |= MVPP2_GMAC_CONFIG_MII_SPEED |
+ MVPP2_GMAC_CONFIG_GMII_SPEED |
+ MVPP2_GMAC_CONFIG_FULL_DUPLEX;
+ val = MVPP2_GMAC_IN_BAND_AUTONEG;
+
+ if (interface == PHY_INTERFACE_MODE_SGMII) {
+ /* SGMII mode receives the speed and duplex from PHY */
+ val |= MVPP2_GMAC_AN_SPEED_EN |
+ MVPP2_GMAC_AN_DUPLEX_EN;
+ } else {
+ /* 802.3z mode has fixed speed and duplex */
+ val |= MVPP2_GMAC_CONFIG_GMII_SPEED |
+ MVPP2_GMAC_CONFIG_FULL_DUPLEX;
+
+ /* The FLOW_CTRL_AUTONEG bit selects either the hardware
+ * automatically or the bits in MVPP22_GMAC_CTRL_4_REG
+ * manually controls the GMAC pause modes.
+ */
+ if (permit_pause_to_mac)
+ val |= MVPP2_GMAC_FLOW_CTRL_AUTONEG;
+
+ /* Configure advertisement bits */
+ mask |= MVPP2_GMAC_FC_ADV_EN | MVPP2_GMAC_FC_ADV_ASM_EN;
+ if (phylink_test(advertising, Pause))
+ val |= MVPP2_GMAC_FC_ADV_EN;
+ if (phylink_test(advertising, Asym_Pause))
+ val |= MVPP2_GMAC_FC_ADV_ASM_EN;
+ }
+ } else {
+ val = 0;
+ }
+
+ old_an = an = readl(port->base + MVPP2_GMAC_AUTONEG_CONFIG);
+ an = (an & ~mask) | val;
+ changed = an ^ old_an;
+ if (changed)
+ writel(an, port->base + MVPP2_GMAC_AUTONEG_CONFIG);
+
+ /* We are only interested in the advertisement bits changing */
+ return changed & (MVPP2_GMAC_FC_ADV_EN | MVPP2_GMAC_FC_ADV_ASM_EN);
+}
+
+static void mvpp2_gmac_pcs_an_restart(struct phylink_pcs *pcs)
+{
+ struct mvpp2_port *port = mvpp2_pcs_to_port(pcs);
+ u32 val = readl(port->base + MVPP2_GMAC_AUTONEG_CONFIG);
+
+ writel(val | MVPP2_GMAC_IN_BAND_RESTART_AN,
+ port->base + MVPP2_GMAC_AUTONEG_CONFIG);
+ writel(val & ~MVPP2_GMAC_IN_BAND_RESTART_AN,
+ port->base + MVPP2_GMAC_AUTONEG_CONFIG);
+}
+
+static const struct phylink_pcs_ops mvpp2_phylink_gmac_pcs_ops = {
+ .pcs_get_state = mvpp2_gmac_pcs_get_state,
+ .pcs_config = mvpp2_gmac_pcs_config,
+ .pcs_an_restart = mvpp2_gmac_pcs_an_restart,
+};
+
static void mvpp2_phylink_validate(struct phylink_config *config,
unsigned long *supported,
struct phylink_link_state *state)
{
- struct mvpp2_port *port = container_of(config, struct mvpp2_port,
- phylink_config);
+ struct mvpp2_port *port = mvpp2_phylink_to_port(config);
__ETHTOOL_DECLARE_LINK_MODE_MASK(mask) = { 0, };
/* Invalid combinations */
switch (state->interface) {
case PHY_INTERFACE_MODE_10GBASER:
case PHY_INTERFACE_MODE_XAUI:
- if (port->gop_id != 0)
+ if (!mvpp2_port_supports_xlg(port))
goto empty_set;
break;
case PHY_INTERFACE_MODE_RGMII:
case PHY_INTERFACE_MODE_RGMII_ID:
case PHY_INTERFACE_MODE_RGMII_RXID:
case PHY_INTERFACE_MODE_RGMII_TXID:
- if (port->priv->hw_version == MVPP22 && port->gop_id == 0)
+ if (!mvpp2_port_supports_rgmii(port))
goto empty_set;
break;
default:
@@ -4791,7 +5867,7 @@ static void mvpp2_phylink_validate(struct phylink_config *config,
case PHY_INTERFACE_MODE_10GBASER:
case PHY_INTERFACE_MODE_XAUI:
case PHY_INTERFACE_MODE_NA:
- if (port->gop_id == 0) {
+ if (mvpp2_port_supports_xlg(port)) {
phylink_set(mask, 10000baseT_Full);
phylink_set(mask, 10000baseCR_Full);
phylink_set(mask, 10000baseSR_Full);
@@ -4802,7 +5878,7 @@ static void mvpp2_phylink_validate(struct phylink_config *config,
}
if (state->interface != PHY_INTERFACE_MODE_NA)
break;
- /* Fall-through */
+ fallthrough;
case PHY_INTERFACE_MODE_RGMII:
case PHY_INTERFACE_MODE_RGMII_ID:
case PHY_INTERFACE_MODE_RGMII_RXID:
@@ -4816,7 +5892,7 @@ static void mvpp2_phylink_validate(struct phylink_config *config,
phylink_set(mask, 1000baseX_Full);
if (state->interface != PHY_INTERFACE_MODE_NA)
break;
- /* Fall-through */
+ fallthrough;
case PHY_INTERFACE_MODE_1000BASEX:
case PHY_INTERFACE_MODE_2500BASEX:
if (port->comphy ||
@@ -4845,148 +5921,39 @@ empty_set:
bitmap_zero(supported, __ETHTOOL_LINK_MODE_MASK_NBITS);
}
-static void mvpp22_xlg_pcs_get_state(struct mvpp2_port *port,
- struct phylink_link_state *state)
-{
- u32 val;
-
- state->speed = SPEED_10000;
- state->duplex = 1;
- state->an_complete = 1;
-
- val = readl(port->base + MVPP22_XLG_STATUS);
- state->link = !!(val & MVPP22_XLG_STATUS_LINK_UP);
-
- state->pause = 0;
- val = readl(port->base + MVPP22_XLG_CTRL0_REG);
- if (val & MVPP22_XLG_CTRL0_TX_FLOW_CTRL_EN)
- state->pause |= MLO_PAUSE_TX;
- if (val & MVPP22_XLG_CTRL0_RX_FLOW_CTRL_EN)
- state->pause |= MLO_PAUSE_RX;
-}
-
-static void mvpp2_gmac_pcs_get_state(struct mvpp2_port *port,
- struct phylink_link_state *state)
-{
- u32 val;
-
- val = readl(port->base + MVPP2_GMAC_STATUS0);
-
- state->an_complete = !!(val & MVPP2_GMAC_STATUS0_AN_COMPLETE);
- state->link = !!(val & MVPP2_GMAC_STATUS0_LINK_UP);
- state->duplex = !!(val & MVPP2_GMAC_STATUS0_FULL_DUPLEX);
-
- switch (port->phy_interface) {
- case PHY_INTERFACE_MODE_1000BASEX:
- state->speed = SPEED_1000;
- break;
- case PHY_INTERFACE_MODE_2500BASEX:
- state->speed = SPEED_2500;
- break;
- default:
- if (val & MVPP2_GMAC_STATUS0_GMII_SPEED)
- state->speed = SPEED_1000;
- else if (val & MVPP2_GMAC_STATUS0_MII_SPEED)
- state->speed = SPEED_100;
- else
- state->speed = SPEED_10;
- }
-
- state->pause = 0;
- if (val & MVPP2_GMAC_STATUS0_RX_PAUSE)
- state->pause |= MLO_PAUSE_RX;
- if (val & MVPP2_GMAC_STATUS0_TX_PAUSE)
- state->pause |= MLO_PAUSE_TX;
-}
-
-static void mvpp2_phylink_mac_pcs_get_state(struct phylink_config *config,
- struct phylink_link_state *state)
-{
- struct mvpp2_port *port = container_of(config, struct mvpp2_port,
- phylink_config);
-
- if (port->priv->hw_version == MVPP22 && port->gop_id == 0) {
- u32 mode = readl(port->base + MVPP22_XLG_CTRL3_REG);
- mode &= MVPP22_XLG_CTRL3_MACMODESELECT_MASK;
-
- if (mode == MVPP22_XLG_CTRL3_MACMODESELECT_10G) {
- mvpp22_xlg_pcs_get_state(port, state);
- return;
- }
- }
-
- mvpp2_gmac_pcs_get_state(port, state);
-}
-
-static void mvpp2_mac_an_restart(struct phylink_config *config)
-{
- struct mvpp2_port *port = container_of(config, struct mvpp2_port,
- phylink_config);
- u32 val = readl(port->base + MVPP2_GMAC_AUTONEG_CONFIG);
-
- writel(val | MVPP2_GMAC_IN_BAND_RESTART_AN,
- port->base + MVPP2_GMAC_AUTONEG_CONFIG);
- writel(val & ~MVPP2_GMAC_IN_BAND_RESTART_AN,
- port->base + MVPP2_GMAC_AUTONEG_CONFIG);
-}
-
static void mvpp2_xlg_config(struct mvpp2_port *port, unsigned int mode,
const struct phylink_link_state *state)
{
- u32 old_ctrl0, ctrl0;
- u32 old_ctrl4, ctrl4;
-
- old_ctrl0 = ctrl0 = readl(port->base + MVPP22_XLG_CTRL0_REG);
- old_ctrl4 = ctrl4 = readl(port->base + MVPP22_XLG_CTRL4_REG);
-
- ctrl0 |= MVPP22_XLG_CTRL0_MAC_RESET_DIS;
-
- if (state->pause & MLO_PAUSE_TX)
- ctrl0 |= MVPP22_XLG_CTRL0_TX_FLOW_CTRL_EN;
- else
- ctrl0 &= ~MVPP22_XLG_CTRL0_TX_FLOW_CTRL_EN;
-
- if (state->pause & MLO_PAUSE_RX)
- ctrl0 |= MVPP22_XLG_CTRL0_RX_FLOW_CTRL_EN;
- else
- ctrl0 &= ~MVPP22_XLG_CTRL0_RX_FLOW_CTRL_EN;
-
- ctrl4 &= ~(MVPP22_XLG_CTRL4_MACMODSELECT_GMAC |
- MVPP22_XLG_CTRL4_EN_IDLE_CHECK);
- ctrl4 |= MVPP22_XLG_CTRL4_FWD_FC | MVPP22_XLG_CTRL4_FWD_PFC;
+ u32 val;
- if (old_ctrl0 != ctrl0)
- writel(ctrl0, port->base + MVPP22_XLG_CTRL0_REG);
- if (old_ctrl4 != ctrl4)
- writel(ctrl4, port->base + MVPP22_XLG_CTRL4_REG);
+ mvpp2_modify(port->base + MVPP22_XLG_CTRL0_REG,
+ MVPP22_XLG_CTRL0_MAC_RESET_DIS,
+ MVPP22_XLG_CTRL0_MAC_RESET_DIS);
+ mvpp2_modify(port->base + MVPP22_XLG_CTRL4_REG,
+ MVPP22_XLG_CTRL4_MACMODSELECT_GMAC |
+ MVPP22_XLG_CTRL4_EN_IDLE_CHECK |
+ MVPP22_XLG_CTRL4_FWD_FC | MVPP22_XLG_CTRL4_FWD_PFC,
+ MVPP22_XLG_CTRL4_FWD_FC | MVPP22_XLG_CTRL4_FWD_PFC);
- if (!(old_ctrl0 & MVPP22_XLG_CTRL0_MAC_RESET_DIS)) {
- while (!(readl(port->base + MVPP22_XLG_CTRL0_REG) &
- MVPP22_XLG_CTRL0_MAC_RESET_DIS))
- continue;
- }
+ /* Wait for reset to deassert */
+ do {
+ val = readl(port->base + MVPP22_XLG_CTRL0_REG);
+ } while (!(val & MVPP22_XLG_CTRL0_MAC_RESET_DIS));
}
static void mvpp2_gmac_config(struct mvpp2_port *port, unsigned int mode,
const struct phylink_link_state *state)
{
- u32 old_an, an;
u32 old_ctrl0, ctrl0;
u32 old_ctrl2, ctrl2;
u32 old_ctrl4, ctrl4;
- old_an = an = readl(port->base + MVPP2_GMAC_AUTONEG_CONFIG);
old_ctrl0 = ctrl0 = readl(port->base + MVPP2_GMAC_CTRL_0_REG);
old_ctrl2 = ctrl2 = readl(port->base + MVPP2_GMAC_CTRL_2_REG);
old_ctrl4 = ctrl4 = readl(port->base + MVPP22_GMAC_CTRL_4_REG);
- an &= ~(MVPP2_GMAC_AN_SPEED_EN | MVPP2_GMAC_FC_ADV_EN |
- MVPP2_GMAC_FC_ADV_ASM_EN | MVPP2_GMAC_FLOW_CTRL_AUTONEG |
- MVPP2_GMAC_AN_DUPLEX_EN | MVPP2_GMAC_IN_BAND_AUTONEG |
- MVPP2_GMAC_IN_BAND_AUTONEG_BYPASS);
ctrl0 &= ~MVPP2_GMAC_PORT_TYPE_MASK;
- ctrl2 &= ~(MVPP2_GMAC_INBAND_AN_MASK | MVPP2_GMAC_PORT_RESET_MASK |
- MVPP2_GMAC_PCS_ENABLE_MASK);
+ ctrl2 &= ~(MVPP2_GMAC_INBAND_AN_MASK | MVPP2_GMAC_PCS_ENABLE_MASK);
/* Configure port type */
if (phy_interface_mode_is_8023z(state->interface)) {
@@ -5008,12 +5975,6 @@ static void mvpp2_gmac_config(struct mvpp2_port *port, unsigned int mode,
MVPP22_CTRL4_QSGMII_BYPASS_ACTIVE;
}
- /* Configure advertisement bits */
- if (phylink_test(state->advertising, Pause))
- an |= MVPP2_GMAC_FC_ADV_EN;
- if (phylink_test(state->advertising, Asym_Pause))
- an |= MVPP2_GMAC_FC_ADV_ASM_EN;
-
/* Configure negotiation style */
if (!phylink_autoneg_inband(mode)) {
/* Phy or fixed speed - no in-band AN, nothing to do, leave the
@@ -5022,14 +5983,6 @@ static void mvpp2_gmac_config(struct mvpp2_port *port, unsigned int mode,
} else if (state->interface == PHY_INTERFACE_MODE_SGMII) {
/* SGMII in-band mode receives the speed and duplex from
* the PHY. Flow control information is not received. */
- an &= ~(MVPP2_GMAC_FORCE_LINK_DOWN |
- MVPP2_GMAC_FORCE_LINK_PASS |
- MVPP2_GMAC_CONFIG_MII_SPEED |
- MVPP2_GMAC_CONFIG_GMII_SPEED |
- MVPP2_GMAC_CONFIG_FULL_DUPLEX);
- an |= MVPP2_GMAC_IN_BAND_AUTONEG |
- MVPP2_GMAC_AN_SPEED_EN |
- MVPP2_GMAC_AN_DUPLEX_EN;
} else if (phy_interface_mode_is_8023z(state->interface)) {
/* 1000BaseX and 2500BaseX ports cannot negotiate speed nor can
* they negotiate duplex: they are always operating with a fixed
@@ -5037,42 +5990,6 @@ static void mvpp2_gmac_config(struct mvpp2_port *port, unsigned int mode,
* speed and full duplex here.
*/
ctrl0 |= MVPP2_GMAC_PORT_TYPE_MASK;
- an &= ~(MVPP2_GMAC_FORCE_LINK_DOWN |
- MVPP2_GMAC_FORCE_LINK_PASS |
- MVPP2_GMAC_CONFIG_MII_SPEED |
- MVPP2_GMAC_CONFIG_GMII_SPEED |
- MVPP2_GMAC_CONFIG_FULL_DUPLEX);
- an |= MVPP2_GMAC_IN_BAND_AUTONEG |
- MVPP2_GMAC_CONFIG_GMII_SPEED |
- MVPP2_GMAC_CONFIG_FULL_DUPLEX;
-
- if (state->pause & MLO_PAUSE_AN && state->an_enabled)
- an |= MVPP2_GMAC_FLOW_CTRL_AUTONEG;
- }
-
-/* Some fields of the auto-negotiation register require the port to be down when
- * their value is updated.
- */
-#define MVPP2_GMAC_AN_PORT_DOWN_MASK \
- (MVPP2_GMAC_IN_BAND_AUTONEG | \
- MVPP2_GMAC_IN_BAND_AUTONEG_BYPASS | \
- MVPP2_GMAC_CONFIG_MII_SPEED | MVPP2_GMAC_CONFIG_GMII_SPEED | \
- MVPP2_GMAC_AN_SPEED_EN | MVPP2_GMAC_CONFIG_FULL_DUPLEX | \
- MVPP2_GMAC_AN_DUPLEX_EN)
-
- if ((old_ctrl0 ^ ctrl0) & MVPP2_GMAC_PORT_TYPE_MASK ||
- (old_ctrl2 ^ ctrl2) & MVPP2_GMAC_INBAND_AN_MASK ||
- (old_an ^ an) & MVPP2_GMAC_AN_PORT_DOWN_MASK) {
- /* Force link down */
- old_an &= ~MVPP2_GMAC_FORCE_LINK_PASS;
- old_an |= MVPP2_GMAC_FORCE_LINK_DOWN;
- writel(old_an, port->base + MVPP2_GMAC_AUTONEG_CONFIG);
-
- /* Set the GMAC in a reset state - do this in a way that
- * ensures we clear it below.
- */
- old_ctrl2 |= MVPP2_GMAC_PORT_RESET_MASK;
- writel(old_ctrl2, port->base + MVPP2_GMAC_CTRL_2_REG);
}
if (old_ctrl0 != ctrl0)
@@ -5081,42 +5998,85 @@ static void mvpp2_gmac_config(struct mvpp2_port *port, unsigned int mode,
writel(ctrl2, port->base + MVPP2_GMAC_CTRL_2_REG);
if (old_ctrl4 != ctrl4)
writel(ctrl4, port->base + MVPP22_GMAC_CTRL_4_REG);
- if (old_an != an)
- writel(an, port->base + MVPP2_GMAC_AUTONEG_CONFIG);
-
- if (old_ctrl2 & MVPP2_GMAC_PORT_RESET_MASK) {
- while (readl(port->base + MVPP2_GMAC_CTRL_2_REG) &
- MVPP2_GMAC_PORT_RESET_MASK)
- continue;
- }
}
-static void mvpp2_mac_config(struct phylink_config *config, unsigned int mode,
- const struct phylink_link_state *state)
+static int mvpp2__mac_prepare(struct phylink_config *config, unsigned int mode,
+ phy_interface_t interface)
{
- struct net_device *dev = to_net_dev(config->dev);
- struct mvpp2_port *port = netdev_priv(dev);
- bool change_interface = port->phy_interface != state->interface;
+ struct mvpp2_port *port = mvpp2_phylink_to_port(config);
/* Check for invalid configuration */
- if (mvpp2_is_xlg(state->interface) && port->gop_id != 0) {
- netdev_err(dev, "Invalid mode on %s\n", dev->name);
- return;
+ if (mvpp2_is_xlg(interface) && port->gop_id != 0) {
+ netdev_err(port->dev, "Invalid mode on %s\n", port->dev->name);
+ return -EINVAL;
+ }
+
+ if (port->phy_interface != interface ||
+ phylink_autoneg_inband(mode)) {
+ /* Force the link down when changing the interface or if in
+ * in-band mode to ensure we do not change the configuration
+ * while the hardware is indicating link is up. We force both
+ * XLG and GMAC down to ensure that they're both in a known
+ * state.
+ */
+ mvpp2_modify(port->base + MVPP2_GMAC_AUTONEG_CONFIG,
+ MVPP2_GMAC_FORCE_LINK_PASS |
+ MVPP2_GMAC_FORCE_LINK_DOWN,
+ MVPP2_GMAC_FORCE_LINK_DOWN);
+
+ if (mvpp2_port_supports_xlg(port))
+ mvpp2_modify(port->base + MVPP22_XLG_CTRL0_REG,
+ MVPP22_XLG_CTRL0_FORCE_LINK_PASS |
+ MVPP22_XLG_CTRL0_FORCE_LINK_DOWN,
+ MVPP22_XLG_CTRL0_FORCE_LINK_DOWN);
}
/* Make sure the port is disabled when reconfiguring the mode */
mvpp2_port_disable(port);
- if (port->priv->hw_version == MVPP22 && change_interface) {
- mvpp22_gop_mask_irq(port);
+ if (port->phy_interface != interface) {
+ /* Place GMAC into reset */
+ mvpp2_modify(port->base + MVPP2_GMAC_CTRL_2_REG,
+ MVPP2_GMAC_PORT_RESET_MASK,
+ MVPP2_GMAC_PORT_RESET_MASK);
- port->phy_interface = state->interface;
+ if (port->priv->hw_version == MVPP22) {
+ mvpp22_gop_mask_irq(port);
- /* Reconfigure the serdes lanes */
- phy_power_off(port->comphy);
- mvpp22_mode_reconfigure(port);
+ phy_power_off(port->comphy);
+ }
}
+ /* Select the appropriate PCS operations depending on the
+ * configured interface mode. We will only switch to a mode
+ * that the validate() checks have already passed.
+ */
+ if (mvpp2_is_xlg(interface))
+ port->phylink_pcs.ops = &mvpp2_phylink_xlg_pcs_ops;
+ else
+ port->phylink_pcs.ops = &mvpp2_phylink_gmac_pcs_ops;
+
+ return 0;
+}
+
+static int mvpp2_mac_prepare(struct phylink_config *config, unsigned int mode,
+ phy_interface_t interface)
+{
+ struct mvpp2_port *port = mvpp2_phylink_to_port(config);
+ int ret;
+
+ ret = mvpp2__mac_prepare(config, mode, interface);
+ if (ret == 0)
+ phylink_set_pcs(port->phylink, &port->phylink_pcs);
+
+ return ret;
+}
+
+static void mvpp2_mac_config(struct phylink_config *config, unsigned int mode,
+ const struct phylink_link_state *state)
+{
+ struct mvpp2_port *port = mvpp2_phylink_to_port(config);
+
/* mac (re)configuration */
if (mvpp2_is_xlg(state->interface))
mvpp2_xlg_config(port, mode, state);
@@ -5127,11 +6087,51 @@ static void mvpp2_mac_config(struct phylink_config *config, unsigned int mode,
if (port->priv->hw_version == MVPP21 && port->flags & MVPP2_F_LOOPBACK)
mvpp2_port_loopback_set(port, state);
+}
+
+static int mvpp2_mac_finish(struct phylink_config *config, unsigned int mode,
+ phy_interface_t interface)
+{
+ struct mvpp2_port *port = mvpp2_phylink_to_port(config);
- if (port->priv->hw_version == MVPP22 && change_interface)
+ if (port->priv->hw_version == MVPP22 &&
+ port->phy_interface != interface) {
+ port->phy_interface = interface;
+
+ /* Reconfigure the serdes lanes */
+ mvpp22_mode_reconfigure(port);
+
+ /* Unmask interrupts */
mvpp22_gop_unmask_irq(port);
+ }
+
+ if (!mvpp2_is_xlg(interface)) {
+ /* Release GMAC reset and wait */
+ mvpp2_modify(port->base + MVPP2_GMAC_CTRL_2_REG,
+ MVPP2_GMAC_PORT_RESET_MASK, 0);
+
+ while (readl(port->base + MVPP2_GMAC_CTRL_2_REG) &
+ MVPP2_GMAC_PORT_RESET_MASK)
+ continue;
+ }
mvpp2_port_enable(port);
+
+ /* Allow the link to come up if in in-band mode, otherwise the
+ * link is forced via mac_link_down()/mac_link_up()
+ */
+ if (phylink_autoneg_inband(mode)) {
+ if (mvpp2_is_xlg(interface))
+ mvpp2_modify(port->base + MVPP22_XLG_CTRL0_REG,
+ MVPP22_XLG_CTRL0_FORCE_LINK_PASS |
+ MVPP22_XLG_CTRL0_FORCE_LINK_DOWN, 0);
+ else
+ mvpp2_modify(port->base + MVPP2_GMAC_AUTONEG_CONFIG,
+ MVPP2_GMAC_FORCE_LINK_PASS |
+ MVPP2_GMAC_FORCE_LINK_DOWN, 0);
+ }
+
+ return 0;
}
static void mvpp2_mac_link_up(struct phylink_config *config,
@@ -5140,25 +6140,26 @@ static void mvpp2_mac_link_up(struct phylink_config *config,
int speed, int duplex,
bool tx_pause, bool rx_pause)
{
- struct net_device *dev = to_net_dev(config->dev);
- struct mvpp2_port *port = netdev_priv(dev);
+ struct mvpp2_port *port = mvpp2_phylink_to_port(config);
u32 val;
if (mvpp2_is_xlg(interface)) {
if (!phylink_autoneg_inband(mode)) {
- val = readl(port->base + MVPP22_XLG_CTRL0_REG);
- val &= ~MVPP22_XLG_CTRL0_FORCE_LINK_DOWN;
- val |= MVPP22_XLG_CTRL0_FORCE_LINK_PASS;
- writel(val, port->base + MVPP22_XLG_CTRL0_REG);
+ val = MVPP22_XLG_CTRL0_FORCE_LINK_PASS;
+ if (tx_pause)
+ val |= MVPP22_XLG_CTRL0_TX_FLOW_CTRL_EN;
+ if (rx_pause)
+ val |= MVPP22_XLG_CTRL0_RX_FLOW_CTRL_EN;
+
+ mvpp2_modify(port->base + MVPP22_XLG_CTRL0_REG,
+ MVPP22_XLG_CTRL0_FORCE_LINK_DOWN |
+ MVPP22_XLG_CTRL0_FORCE_LINK_PASS |
+ MVPP22_XLG_CTRL0_TX_FLOW_CTRL_EN |
+ MVPP22_XLG_CTRL0_RX_FLOW_CTRL_EN, val);
}
} else {
if (!phylink_autoneg_inband(mode)) {
- val = readl(port->base + MVPP2_GMAC_AUTONEG_CONFIG);
- val &= ~(MVPP2_GMAC_FORCE_LINK_DOWN |
- MVPP2_GMAC_CONFIG_MII_SPEED |
- MVPP2_GMAC_CONFIG_GMII_SPEED |
- MVPP2_GMAC_CONFIG_FULL_DUPLEX);
- val |= MVPP2_GMAC_FORCE_LINK_PASS;
+ val = MVPP2_GMAC_FORCE_LINK_PASS;
if (speed == SPEED_1000 || speed == SPEED_2500)
val |= MVPP2_GMAC_CONFIG_GMII_SPEED;
@@ -5168,34 +6169,40 @@ static void mvpp2_mac_link_up(struct phylink_config *config,
if (duplex == DUPLEX_FULL)
val |= MVPP2_GMAC_CONFIG_FULL_DUPLEX;
- writel(val, port->base + MVPP2_GMAC_AUTONEG_CONFIG);
+ mvpp2_modify(port->base + MVPP2_GMAC_AUTONEG_CONFIG,
+ MVPP2_GMAC_FORCE_LINK_DOWN |
+ MVPP2_GMAC_FORCE_LINK_PASS |
+ MVPP2_GMAC_CONFIG_MII_SPEED |
+ MVPP2_GMAC_CONFIG_GMII_SPEED |
+ MVPP2_GMAC_CONFIG_FULL_DUPLEX, val);
}
/* We can always update the flow control enable bits;
* these will only be effective if flow control AN
* (MVPP2_GMAC_FLOW_CTRL_AUTONEG) is disabled.
*/
- val = readl(port->base + MVPP22_GMAC_CTRL_4_REG);
- val &= ~(MVPP22_CTRL4_RX_FC_EN | MVPP22_CTRL4_TX_FC_EN);
+ val = 0;
if (tx_pause)
val |= MVPP22_CTRL4_TX_FC_EN;
if (rx_pause)
val |= MVPP22_CTRL4_RX_FC_EN;
- writel(val, port->base + MVPP22_GMAC_CTRL_4_REG);
+
+ mvpp2_modify(port->base + MVPP22_GMAC_CTRL_4_REG,
+ MVPP22_CTRL4_RX_FC_EN | MVPP22_CTRL4_TX_FC_EN,
+ val);
}
mvpp2_port_enable(port);
mvpp2_egress_enable(port);
mvpp2_ingress_enable(port);
- netif_tx_wake_all_queues(dev);
+ netif_tx_wake_all_queues(port->dev);
}
static void mvpp2_mac_link_down(struct phylink_config *config,
unsigned int mode, phy_interface_t interface)
{
- struct net_device *dev = to_net_dev(config->dev);
- struct mvpp2_port *port = netdev_priv(dev);
+ struct mvpp2_port *port = mvpp2_phylink_to_port(config);
u32 val;
if (!phylink_autoneg_inband(mode)) {
@@ -5212,7 +6219,7 @@ static void mvpp2_mac_link_down(struct phylink_config *config,
}
}
- netif_tx_stop_all_queues(dev);
+ netif_tx_stop_all_queues(port->dev);
mvpp2_egress_disable(port);
mvpp2_ingress_disable(port);
@@ -5221,13 +6228,36 @@ static void mvpp2_mac_link_down(struct phylink_config *config,
static const struct phylink_mac_ops mvpp2_phylink_ops = {
.validate = mvpp2_phylink_validate,
- .mac_pcs_get_state = mvpp2_phylink_mac_pcs_get_state,
- .mac_an_restart = mvpp2_mac_an_restart,
+ .mac_prepare = mvpp2_mac_prepare,
.mac_config = mvpp2_mac_config,
+ .mac_finish = mvpp2_mac_finish,
.mac_link_up = mvpp2_mac_link_up,
.mac_link_down = mvpp2_mac_link_down,
};
+/* Work-around for ACPI */
+static void mvpp2_acpi_start(struct mvpp2_port *port)
+{
+ /* Phylink isn't used as of now for ACPI, so the MAC has to be
+ * configured manually when the interface is started. This will
+ * be removed as soon as the phylink ACPI support lands in.
+ */
+ struct phylink_link_state state = {
+ .interface = port->phy_interface,
+ };
+ mvpp2__mac_prepare(&port->phylink_config, MLO_AN_INBAND,
+ port->phy_interface);
+ mvpp2_mac_config(&port->phylink_config, MLO_AN_INBAND, &state);
+ port->phylink_pcs.ops->pcs_config(&port->phylink_pcs, MLO_AN_INBAND,
+ port->phy_interface,
+ state.advertising, false);
+ mvpp2_mac_finish(&port->phylink_config, MLO_AN_INBAND,
+ port->phy_interface);
+ mvpp2_mac_link_up(&port->phylink_config, NULL,
+ MLO_AN_INBAND, port->phy_interface,
+ SPEED_UNKNOWN, DUPLEX_UNKNOWN, false, false);
+}
+
/* Ports initialization */
static int mvpp2_port_probe(struct platform_device *pdev,
struct fwnode_handle *port_fwnode,
@@ -5315,16 +6345,16 @@ static int mvpp2_port_probe(struct platform_device *pdev,
goto err_free_netdev;
if (port_node)
- port->link_irq = of_irq_get_byname(port_node, "link");
+ port->port_irq = of_irq_get_byname(port_node, "link");
else
- port->link_irq = fwnode_irq_get(port_fwnode, port->nqvecs + 1);
- if (port->link_irq == -EPROBE_DEFER) {
+ port->port_irq = fwnode_irq_get(port_fwnode, port->nqvecs + 1);
+ if (port->port_irq == -EPROBE_DEFER) {
err = -EPROBE_DEFER;
goto err_deinit_qvecs;
}
- if (port->link_irq <= 0)
+ if (port->port_irq <= 0)
/* the link irq is optional */
- port->link_irq = 0;
+ port->port_irq = 0;
if (fwnode_property_read_bool(port_fwnode, "marvell,loopback"))
port->flags |= MVPP2_F_LOOPBACK;
@@ -5361,6 +6391,12 @@ static int mvpp2_port_probe(struct platform_device *pdev,
port->stats_base = port->priv->iface_base +
MVPP22_MIB_COUNTERS_OFFSET +
port->gop_id * MVPP22_MIB_COUNTERS_PORT_SZ;
+
+ /* We may want a property to describe whether we should use
+ * MAC hardware timestamping.
+ */
+ if (priv->tai)
+ port->hwtstamp = true;
}
/* Alloc per-cpu and ethtool stats */
@@ -5488,8 +6524,8 @@ err_free_txq_pcpu:
err_free_stats:
free_percpu(port->stats);
err_free_irq:
- if (port->link_irq)
- irq_dispose_mapping(port->link_irq);
+ if (port->port_irq)
+ irq_dispose_mapping(port->port_irq);
err_deinit_qvecs:
mvpp2_queue_vectors_deinit(port);
err_free_netdev:
@@ -5510,8 +6546,8 @@ static void mvpp2_port_remove(struct mvpp2_port *port)
for (i = 0; i < port->ntxqs; i++)
free_percpu(port->txqs[i]->pcpu);
mvpp2_queue_vectors_deinit(port);
- if (port->link_irq)
- irq_dispose_mapping(port->link_irq);
+ if (port->port_irq)
+ irq_dispose_mapping(port->port_irq);
free_netdev(port->dev);
}
@@ -5923,6 +6959,10 @@ static int mvpp2_probe(struct platform_device *pdev)
goto err_axi_clk;
}
+ err = mvpp22_tai_probe(&pdev->dev, priv);
+ if (err < 0)
+ goto err_axi_clk;
+
/* Initialize ports */
fwnode_for_each_available_child_node(fwnode, port_fwnode) {
err = mvpp2_port_probe(pdev, port_fwnode, priv);
@@ -5983,8 +7023,8 @@ static int mvpp2_remove(struct platform_device *pdev)
{
struct mvpp2 *priv = platform_get_drvdata(pdev);
struct fwnode_handle *fwnode = pdev->dev.fwnode;
+ int i = 0, poolnum = MVPP2_BM_POOLS_NUM;
struct fwnode_handle *port_fwnode;
- int i = 0;
mvpp2_dbgfs_cleanup(priv);
@@ -5998,7 +7038,10 @@ static int mvpp2_remove(struct platform_device *pdev)
destroy_workqueue(priv->stats_queue);
- for (i = 0; i < MVPP2_BM_POOLS_NUM; i++) {
+ if (priv->percpu_pools)
+ poolnum = mvpp2_get_nrxqs(priv) * 2;
+
+ for (i = 0; i < poolnum; i++) {
struct mvpp2_bm_pool *bm_pool = &priv->bm_pools[i];
mvpp2_bm_pool_destroy(&pdev->dev, priv, bm_pool);
@@ -6038,11 +7081,13 @@ static const struct of_device_id mvpp2_match[] = {
};
MODULE_DEVICE_TABLE(of, mvpp2_match);
+#ifdef CONFIG_ACPI
static const struct acpi_device_id mvpp2_acpi_match[] = {
{ "MRVL0110", MVPP22 },
{ },
};
MODULE_DEVICE_TABLE(acpi, mvpp2_acpi_match);
+#endif
static struct platform_driver mvpp2_driver = {
.probe = mvpp2_probe,
Powered by cgit, Gruvboxed by Blaster4385.