X-Git-Url: http://git.openwrt.org/?a=blobdiff_plain;f=target%2Flinux%2Framips%2Ffiles%2Fdrivers%2Fnet%2Fethernet%2Fralink%2Fralink_soc_eth.c;h=2691cfb71051aa230fc1ec680e750d1e4748b251;hb=6b4985b105d39ca22bc2de9da6889157cdac97a8;hp=5a681f8e43e5e685ac434cfa069ebb95bf8f84b8;hpb=b3ca42ec8f861922f4e0da8d745811b052252ae3;p=openwrt%2Fopenwrt.git diff --git a/target/linux/ramips/files/drivers/net/ethernet/ralink/ralink_soc_eth.c b/target/linux/ramips/files/drivers/net/ethernet/ralink/ralink_soc_eth.c index 5a681f8e43..2691cfb710 100644 --- a/target/linux/ramips/files/drivers/net/ethernet/ralink/ralink_soc_eth.c +++ b/target/linux/ramips/files/drivers/net/ethernet/ralink/ralink_soc_eth.c @@ -56,8 +56,8 @@ #define TX_DMA_DESP2_DEF (TX_DMA_LS0 | TX_DMA_DONE) #define TX_DMA_DESP4_DEF (TX_DMA_QN(3) | TX_DMA_PN(1)) -#define NEXT_TX_DESP_IDX(X) (((X) + 1) & (NUM_DMA_DESC - 1)) -#define NEXT_RX_DESP_IDX(X) (((X) + 1) & (NUM_DMA_DESC - 1)) +#define NEXT_TX_DESP_IDX(X) (((X) + 1) & (ring->tx_ring_size - 1)) +#define NEXT_RX_DESP_IDX(X) (((X) + 1) & (ring->rx_ring_size - 1)) #define SYSC_REG_RSTCTRL 0x34 @@ -65,7 +65,7 @@ static int fe_msg_level = -1; module_param_named(msg_level, fe_msg_level, int, 0); MODULE_PARM_DESC(msg_level, "Message level (-1=defaults,0=none,...,16=all)"); -static const u32 fe_reg_table_default[FE_REG_COUNT] = { +static const u16 fe_reg_table_default[FE_REG_COUNT] = { [FE_REG_PDMA_GLO_CFG] = FE_PDMA_GLO_CFG, [FE_REG_PDMA_RST_CFG] = FE_PDMA_RST_CFG, [FE_REG_DLY_INT_CFG] = FE_DLY_INT_CFG, @@ -84,7 +84,7 @@ static const u32 fe_reg_table_default[FE_REG_COUNT] = { [FE_REG_FE_RST_GL] = FE_FE_RST_GL, }; -static const u32 *fe_reg_table = fe_reg_table_default; +static const u16 *fe_reg_table = fe_reg_table_default; struct fe_work_t { int bitnr; @@ -178,7 +178,7 @@ static inline int fe_max_frag_size(int mtu) static inline int fe_max_buf_size(int frag_size) { - return frag_size - FE_RX_HLEN - + return frag_size - NET_SKB_PAD - NET_IP_ALIGN - SKB_DATA_ALIGN(sizeof(struct skb_shared_info)); } @@ -190,14 +190,6 @@ static inline void fe_get_rxd(struct fe_rx_dma *rxd, struct fe_rx_dma *dma_rxd) rxd->rxd4 = dma_rxd->rxd4; } -static inline void fe_get_txd(struct fe_tx_dma *txd, struct fe_tx_dma *dma_txd) -{ - txd->txd1 = dma_txd->txd1; - txd->txd2 = dma_txd->txd2; - txd->txd3 = dma_txd->txd3; - txd->txd4 = dma_txd->txd4; -} - static inline void fe_set_txd(struct fe_tx_dma *txd, struct fe_tx_dma *dma_txd) { dma_txd->txd1 = txd->txd1; @@ -210,77 +202,80 @@ static inline void fe_set_txd(struct fe_tx_dma *txd, struct fe_tx_dma *dma_txd) static void fe_clean_rx(struct fe_priv *priv) { int i; + struct fe_rx_ring *ring = &priv->rx_ring; - if (priv->rx_data) { - for (i = 0; i < NUM_DMA_DESC; i++) - if (priv->rx_data[i]) { - if (priv->rx_dma && priv->rx_dma[i].rxd1) + if (ring->rx_data) { + for (i = 0; i < ring->rx_ring_size; i++) + if (ring->rx_data[i]) { + if (ring->rx_dma && ring->rx_dma[i].rxd1) dma_unmap_single(&priv->netdev->dev, - priv->rx_dma[i].rxd1, - priv->rx_buf_size, + ring->rx_dma[i].rxd1, + ring->rx_buf_size, DMA_FROM_DEVICE); - put_page(virt_to_head_page(priv->rx_data[i])); + put_page(virt_to_head_page(ring->rx_data[i])); } - kfree(priv->rx_data); - priv->rx_data = NULL; + kfree(ring->rx_data); + ring->rx_data = NULL; } - if (priv->rx_dma) { + if (ring->rx_dma) { dma_free_coherent(&priv->netdev->dev, - NUM_DMA_DESC * sizeof(*priv->rx_dma), - priv->rx_dma, - priv->rx_phys); - priv->rx_dma = NULL; + ring->rx_ring_size * sizeof(*ring->rx_dma), + ring->rx_dma, + ring->rx_phys); + ring->rx_dma = NULL; } } static int fe_alloc_rx(struct fe_priv *priv) { struct net_device *netdev = priv->netdev; + struct fe_rx_ring *ring = &priv->rx_ring; int i, pad; - priv->rx_data = kcalloc(NUM_DMA_DESC, sizeof(*priv->rx_data), + ring->rx_data = kcalloc(ring->rx_ring_size, sizeof(*ring->rx_data), GFP_KERNEL); - if (!priv->rx_data) + if (!ring->rx_data) goto no_rx_mem; - for (i = 0; i < NUM_DMA_DESC; i++) { - priv->rx_data[i] = netdev_alloc_frag(priv->frag_size); - if (!priv->rx_data[i]) + for (i = 0; i < ring->rx_ring_size; i++) { + ring->rx_data[i] = netdev_alloc_frag(ring->frag_size); + if (!ring->rx_data[i]) goto no_rx_mem; } - priv->rx_dma = dma_alloc_coherent(&netdev->dev, - NUM_DMA_DESC * sizeof(*priv->rx_dma), - &priv->rx_phys, + ring->rx_dma = dma_alloc_coherent(&netdev->dev, + ring->rx_ring_size * sizeof(*ring->rx_dma), + &ring->rx_phys, GFP_ATOMIC | __GFP_ZERO); - if (!priv->rx_dma) + if (!ring->rx_dma) goto no_rx_mem; if (priv->flags & FE_FLAG_RX_2B_OFFSET) pad = 0; else pad = NET_IP_ALIGN; - for (i = 0; i < NUM_DMA_DESC; i++) { + for (i = 0; i < ring->rx_ring_size; i++) { dma_addr_t dma_addr = dma_map_single(&netdev->dev, - priv->rx_data[i] + NET_SKB_PAD + pad, - priv->rx_buf_size, + ring->rx_data[i] + NET_SKB_PAD + pad, + ring->rx_buf_size, DMA_FROM_DEVICE); if (unlikely(dma_mapping_error(&netdev->dev, dma_addr))) goto no_rx_mem; - priv->rx_dma[i].rxd1 = (unsigned int) dma_addr; + ring->rx_dma[i].rxd1 = (unsigned int) dma_addr; - if (priv->soc->rx_dma) - priv->soc->rx_dma(&priv->rx_dma[i], priv->rx_buf_size); + if (priv->flags & FE_FLAG_RX_SG_DMA) + ring->rx_dma[i].rxd2 = RX_DMA_PLEN0(ring->rx_buf_size); else - priv->rx_dma[i].rxd2 = RX_DMA_LSO; + ring->rx_dma[i].rxd2 = RX_DMA_LSO; } + ring->rx_calc_idx = ring->rx_ring_size - 1; wmb(); - fe_reg_w32(priv->rx_phys, FE_REG_RX_BASE_PTR0); - fe_reg_w32(NUM_DMA_DESC, FE_REG_RX_MAX_CNT0); - fe_reg_w32((NUM_DMA_DESC - 1), FE_REG_RX_CALC_IDX0); + fe_reg_w32(ring->rx_phys, FE_REG_RX_BASE_PTR0); + fe_reg_w32(ring->rx_ring_size, FE_REG_RX_MAX_CNT0); + fe_reg_w32(ring->rx_calc_idx, FE_REG_RX_CALC_IDX0); fe_reg_w32(FE_PST_DRX_IDX0, FE_REG_PDMA_RST_CFG); return 0; @@ -289,57 +284,87 @@ no_rx_mem: return -ENOMEM; } +static void fe_txd_unmap(struct device *dev, struct fe_tx_buf *tx_buf) +{ + if (tx_buf->flags & FE_TX_FLAGS_SINGLE0) { + dma_unmap_single(dev, + dma_unmap_addr(tx_buf, dma_addr0), + dma_unmap_len(tx_buf, dma_len0), + DMA_TO_DEVICE); + } else if (tx_buf->flags & FE_TX_FLAGS_PAGE0) { + dma_unmap_page(dev, + dma_unmap_addr(tx_buf, dma_addr0), + dma_unmap_len(tx_buf, dma_len0), + DMA_TO_DEVICE); + } + if (tx_buf->flags & FE_TX_FLAGS_PAGE1) + dma_unmap_page(dev, + dma_unmap_addr(tx_buf, dma_addr1), + dma_unmap_len(tx_buf, dma_len1), + DMA_TO_DEVICE); + + tx_buf->flags = 0; + if (tx_buf->skb && (tx_buf->skb != (struct sk_buff *) DMA_DUMMY_DESC)) { + dev_kfree_skb_any(tx_buf->skb); + } + tx_buf->skb = NULL; +} + static void fe_clean_tx(struct fe_priv *priv) { int i; - - if (priv->tx_skb) { - for (i = 0; i < NUM_DMA_DESC; i++) { - if (priv->tx_skb[i]) - dev_kfree_skb_any(priv->tx_skb[i]); - } - kfree(priv->tx_skb); - priv->tx_skb = NULL; + struct device *dev = &priv->netdev->dev; + struct fe_tx_ring *ring = &priv->tx_ring; + + if (ring->tx_buf) { + for (i = 0; i < ring->tx_ring_size; i++) + fe_txd_unmap(dev, &ring->tx_buf[i]); + kfree(ring->tx_buf); + ring->tx_buf = NULL; } - if (priv->tx_dma) { - dma_free_coherent(&priv->netdev->dev, - NUM_DMA_DESC * sizeof(*priv->tx_dma), - priv->tx_dma, - priv->tx_phys); - priv->tx_dma = NULL; + if (ring->tx_dma) { + dma_free_coherent(dev, + ring->tx_ring_size * sizeof(*ring->tx_dma), + ring->tx_dma, + ring->tx_phys); + ring->tx_dma = NULL; } + + netdev_reset_queue(priv->netdev); } static int fe_alloc_tx(struct fe_priv *priv) { int i; + struct fe_tx_ring *ring = &priv->tx_ring; - priv->tx_free_idx = 0; + ring->tx_free_idx = 0; + ring->tx_next_idx = 0; + ring->tx_thresh = max((unsigned long)ring->tx_ring_size >> 2, MAX_SKB_FRAGS); - priv->tx_skb = kcalloc(NUM_DMA_DESC, sizeof(*priv->tx_skb), + ring->tx_buf = kcalloc(ring->tx_ring_size, sizeof(*ring->tx_buf), GFP_KERNEL); - if (!priv->tx_skb) + if (!ring->tx_buf) goto no_tx_mem; - priv->tx_dma = dma_alloc_coherent(&priv->netdev->dev, - NUM_DMA_DESC * sizeof(*priv->tx_dma), - &priv->tx_phys, + ring->tx_dma = dma_alloc_coherent(&priv->netdev->dev, + ring->tx_ring_size * sizeof(*ring->tx_dma), + &ring->tx_phys, GFP_ATOMIC | __GFP_ZERO); - if (!priv->tx_dma) + if (!ring->tx_dma) goto no_tx_mem; - for (i = 0; i < NUM_DMA_DESC; i++) { + for (i = 0; i < ring->tx_ring_size; i++) { if (priv->soc->tx_dma) { - priv->soc->tx_dma(&priv->tx_dma[i]); - continue; + priv->soc->tx_dma(&ring->tx_dma[i]); } - priv->tx_dma[i].txd2 = TX_DMA_DESP2_DEF; + ring->tx_dma[i].txd2 = TX_DMA_DESP2_DEF; } wmb(); - fe_reg_w32(priv->tx_phys, FE_REG_TX_BASE_PTR0); - fe_reg_w32(NUM_DMA_DESC, FE_REG_TX_MAX_CNT0); + fe_reg_w32(ring->tx_phys, FE_REG_TX_BASE_PTR0); + fe_reg_w32(ring->tx_ring_size, FE_REG_TX_MAX_CNT0); fe_reg_w32(0, FE_REG_TX_CTX_IDX0); fe_reg_w32(FE_PST_DTX_IDX0, FE_REG_PDMA_RST_CFG); @@ -368,43 +393,21 @@ static void fe_free_dma(struct fe_priv *priv) { fe_clean_tx(priv); fe_clean_rx(priv); - - netdev_reset_queue(priv->netdev); -} - -static inline void txd_unmap_single(struct device *dev, struct fe_tx_dma *txd) -{ - if (txd->txd1 && TX_DMA_GET_PLEN0(txd->txd2)) - dma_unmap_single(dev, txd->txd1, - TX_DMA_GET_PLEN0(txd->txd2), - DMA_TO_DEVICE); -} - -static inline void txd_unmap_page0(struct device *dev, struct fe_tx_dma *txd) -{ - if (txd->txd1 && TX_DMA_GET_PLEN0(txd->txd2)) - dma_unmap_page(dev, txd->txd1, - TX_DMA_GET_PLEN0(txd->txd2), - DMA_TO_DEVICE); -} - -static inline void txd_unmap_page1(struct device *dev, struct fe_tx_dma *txd) -{ - if (txd->txd3 && TX_DMA_GET_PLEN1(txd->txd2)) - dma_unmap_page(dev, txd->txd3, - TX_DMA_GET_PLEN1(txd->txd2), - DMA_TO_DEVICE); } void fe_stats_update(struct fe_priv *priv) { struct fe_hw_stats *hwstats = priv->hw_stats; unsigned int base = fe_reg_table[FE_REG_FE_COUNTER_BASE]; + u64 stats; u64_stats_update_begin(&hwstats->syncp); if (IS_ENABLED(CONFIG_SOC_MT7621)) { hwstats->rx_bytes += fe_r32(base); + stats = fe_r32(base + 0x04); + if (stats) + hwstats->rx_bytes += (stats << 32); hwstats->rx_packets += fe_r32(base + 0x08); hwstats->rx_overflow += fe_r32(base + 0x10); hwstats->rx_fcs_errors += fe_r32(base + 0x14); @@ -415,6 +418,9 @@ void fe_stats_update(struct fe_priv *priv) hwstats->tx_skip += fe_r32(base + 0x28); hwstats->tx_collisions += fe_r32(base + 0x2c); hwstats->tx_bytes += fe_r32(base + 0x30); + stats = fe_r32(base + 0x34); + if (stats) + hwstats->tx_bytes += (stats << 32); hwstats->tx_packets += fe_r32(base + 0x38); } else { hwstats->tx_bytes += fe_r32(base); @@ -455,7 +461,7 @@ static struct rtnl_link_stats64 *fe_get_stats64(struct net_device *dev, } do { - start = u64_stats_fetch_begin_bh(&hwstats->syncp); + start = u64_stats_fetch_begin_irq(&hwstats->syncp); storage->rx_packets = hwstats->rx_packets; storage->tx_packets = hwstats->tx_packets; storage->rx_bytes = hwstats->rx_bytes; @@ -467,7 +473,7 @@ static struct rtnl_link_stats64 *fe_get_stats64(struct net_device *dev, storage->rx_crc_errors = hwstats->rx_fcs_errors; storage->rx_errors = hwstats->rx_checksum_errors; storage->tx_aborted_errors = hwstats->tx_skip; - } while (u64_stats_fetch_retry_bh(&hwstats->syncp, start)); + } while (u64_stats_fetch_retry_irq(&hwstats->syncp, start)); storage->tx_errors = priv->netdev->stats.tx_errors; storage->rx_dropped = priv->netdev->stats.rx_dropped; @@ -484,7 +490,7 @@ static int fe_vlan_rx_add_vid(struct net_device *dev, u32 vlan_cfg; if (!((fe_reg_table[FE_REG_FE_DMA_VID_BASE]) && - (dev->features | NETIF_F_HW_VLAN_CTAG_TX))) + (dev->features & NETIF_F_HW_VLAN_CTAG_TX))) return 0; if (test_bit(idx, &priv->vlan_map)) { @@ -516,7 +522,7 @@ static int fe_vlan_rx_kill_vid(struct net_device *dev, u32 idx = (vid & 0xf); if (!((fe_reg_table[FE_REG_FE_DMA_VID_BASE]) && - (dev->features | NETIF_F_HW_VLAN_CTAG_TX))) + (dev->features & NETIF_F_HW_VLAN_CTAG_TX))) return 0; clear_bit(idx, &priv->vlan_map); @@ -524,20 +530,30 @@ static int fe_vlan_rx_kill_vid(struct net_device *dev, return 0; } +static inline u32 fe_empty_txd(struct fe_tx_ring *ring) +{ + barrier(); + return (u32)(ring->tx_ring_size - + ((ring->tx_next_idx - ring->tx_free_idx) & + (ring->tx_ring_size - 1))); +} + static int fe_tx_map_dma(struct sk_buff *skb, struct net_device *dev, - int idx) + int tx_num, struct fe_tx_ring *ring) { struct fe_priv *priv = netdev_priv(dev); struct skb_frag_struct *frag; struct fe_tx_dma txd, *ptxd; + struct fe_tx_buf *tx_buf; dma_addr_t mapped_addr; unsigned int nr_frags; u32 def_txd4; - int i, j, unmap_idx, tx_num; + int i, j, k, frag_size, frag_map_size, offset; + tx_buf = &ring->tx_buf[ring->tx_next_idx]; + memset(tx_buf, 0, sizeof(*tx_buf)); memset(&txd, 0, sizeof(txd)); nr_frags = skb_shinfo(skb)->nr_frags; - tx_num = 1 + (nr_frags >> 1); /* init tx descriptor */ if (priv->soc->tx_dma) @@ -546,9 +562,6 @@ static int fe_tx_map_dma(struct sk_buff *skb, struct net_device *dev, txd.txd4 = TX_DMA_DESP4_DEF; def_txd4 = txd.txd4; - /* use dma_unmap_single to free it */ - txd.txd4 |= priv->soc->tx_udf_bit; - /* TX Checksum offload */ if (skb->ip_summed == CHECKSUM_PARTIAL) txd.txd4 |= TX_DMA_CHKSUM; @@ -584,78 +597,100 @@ static int fe_tx_map_dma(struct sk_buff *skb, struct net_device *dev, txd.txd1 = mapped_addr; txd.txd2 = TX_DMA_PLEN0(skb_headlen(skb)); + tx_buf->flags |= FE_TX_FLAGS_SINGLE0; + dma_unmap_addr_set(tx_buf, dma_addr0, mapped_addr); + dma_unmap_len_set(tx_buf, dma_len0, skb_headlen(skb)); + /* TX SG offload */ - j = idx; + j = ring->tx_next_idx; + k = 0; for (i = 0; i < nr_frags; i++) { - + offset = 0; frag = &skb_shinfo(skb)->frags[i]; - mapped_addr = skb_frag_dma_map(&dev->dev, frag, 0, - skb_frag_size(frag), DMA_TO_DEVICE); - if (unlikely(dma_mapping_error(&dev->dev, mapped_addr))) - goto err_dma; - - if (i & 0x1) { - j = NEXT_TX_DESP_IDX(j); - txd.txd1 = mapped_addr; - txd.txd2 = TX_DMA_PLEN0(frag->size); - txd.txd4 = def_txd4; - } else { - txd.txd3 = mapped_addr; - txd.txd2 |= TX_DMA_PLEN1(frag->size); - if (i != (nr_frags -1)) { - fe_set_txd(&txd, &priv->tx_dma[j]); - memset(&txd, 0, sizeof(txd)); + frag_size = skb_frag_size(frag); + + while (frag_size > 0) { + frag_map_size = min(frag_size, TX_DMA_BUF_LEN); + mapped_addr = skb_frag_dma_map(&dev->dev, frag, offset, + frag_map_size, DMA_TO_DEVICE); + if (unlikely(dma_mapping_error(&dev->dev, mapped_addr))) + goto err_dma; + + if (k & 0x1) { + j = NEXT_TX_DESP_IDX(j); + txd.txd1 = mapped_addr; + txd.txd2 = TX_DMA_PLEN0(frag_map_size); + txd.txd4 = def_txd4; + + tx_buf = &ring->tx_buf[j]; + memset(tx_buf, 0, sizeof(*tx_buf)); + + tx_buf->flags |= FE_TX_FLAGS_PAGE0; + dma_unmap_addr_set(tx_buf, dma_addr0, mapped_addr); + dma_unmap_len_set(tx_buf, dma_len0, frag_map_size); + } else { + txd.txd3 = mapped_addr; + txd.txd2 |= TX_DMA_PLEN1(frag_map_size); + + tx_buf->skb = (struct sk_buff *) DMA_DUMMY_DESC; + tx_buf->flags |= FE_TX_FLAGS_PAGE1; + dma_unmap_addr_set(tx_buf, dma_addr1, mapped_addr); + dma_unmap_len_set(tx_buf, dma_len1, frag_map_size); + + if (!((i == (nr_frags -1)) && + (frag_map_size == frag_size))) { + fe_set_txd(&txd, &ring->tx_dma[j]); + memset(&txd, 0, sizeof(txd)); + } } - priv->tx_skb[j] = (struct sk_buff *) DMA_DUMMY_DESC; + frag_size -= frag_map_size; + offset += frag_map_size; + k++; } } /* set last segment */ - if (nr_frags & 0x1) + if (k & 0x1) txd.txd2 |= TX_DMA_LS1; else txd.txd2 |= TX_DMA_LS0; - fe_set_txd(&txd, &priv->tx_dma[j]); + fe_set_txd(&txd, &ring->tx_dma[j]); /* store skb to cleanup */ - priv->tx_skb[j] = skb; + tx_buf->skb = skb; netdev_sent_queue(dev, skb->len); skb_tx_timestamp(skb); + ring->tx_next_idx = NEXT_TX_DESP_IDX(j); wmb(); - j = NEXT_TX_DESP_IDX(j); - fe_reg_w32(j, FE_REG_TX_CTX_IDX0); + if (unlikely(fe_empty_txd(ring) <= ring->tx_thresh)) { + netif_stop_queue(dev); + smp_mb(); + if (unlikely(fe_empty_txd(ring) > ring->tx_thresh)) + netif_wake_queue(dev); + } + + if (netif_xmit_stopped(netdev_get_tx_queue(dev, 0)) || !skb->xmit_more) + fe_reg_w32(ring->tx_next_idx, FE_REG_TX_CTX_IDX0); return 0; err_dma: - /* unmap dma */ - ptxd = &priv->tx_dma[idx]; - txd_unmap_single(&dev->dev, ptxd); - - j = idx; - unmap_idx = i; - for (i = 0; i < unmap_idx; i++) { - if (i & 0x1) { - j = NEXT_TX_DESP_IDX(j); - ptxd = &priv->tx_dma[j]; - txd_unmap_page0(&dev->dev, ptxd); - } else { - txd_unmap_page1(&dev->dev, ptxd); - } - } - -err_out: - /* reinit descriptors and skb */ - j = idx; + j = ring->tx_next_idx; for (i = 0; i < tx_num; i++) { - priv->tx_dma[j].txd2 = TX_DMA_DESP2_DEF; - priv->tx_skb[j] = NULL; + ptxd = &ring->tx_dma[j]; + tx_buf = &ring->tx_buf[j]; + + /* unmap dma */ + fe_txd_unmap(&dev->dev, tx_buf); + + ptxd->txd2 = TX_DMA_DESP2_DEF; j = NEXT_TX_DESP_IDX(j); } wmb(); +err_out: return -1; } @@ -689,17 +724,29 @@ static inline int fe_skb_padto(struct sk_buff *skb, struct fe_priv *priv) { return ret; } -static inline u32 fe_empty_txd(struct fe_priv *priv, u32 tx_fill_idx) +static inline int fe_cal_txd_req(struct sk_buff *skb) { - return (u32)(NUM_DMA_DESC - ((tx_fill_idx - priv->tx_free_idx) & - (NUM_DMA_DESC - 1))); + int i, nfrags; + struct skb_frag_struct *frag; + + nfrags = 1; + if (skb_is_gso(skb)) { + for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { + frag = &skb_shinfo(skb)->frags[i]; + nfrags += DIV_ROUND_UP(frag->size, TX_DMA_BUF_LEN); + } + } else { + nfrags += skb_shinfo(skb)->nr_frags; + } + + return DIV_ROUND_UP(nfrags, 2); } static int fe_start_xmit(struct sk_buff *skb, struct net_device *dev) { struct fe_priv *priv = netdev_priv(dev); + struct fe_tx_ring *ring = &priv->tx_ring; struct net_device_stats *stats = &dev->stats; - u32 tx; int tx_num; int len = skb->len; @@ -708,9 +755,8 @@ static int fe_start_xmit(struct sk_buff *skb, struct net_device *dev) return NETDEV_TX_OK; } - tx_num = 1 + (skb_shinfo(skb)->nr_frags >> 1); - tx = fe_reg_r32(FE_REG_TX_CTX_IDX0); - if (unlikely(fe_empty_txd(priv, tx) <= tx_num)) + tx_num = fe_cal_txd_req(skb); + if (unlikely(fe_empty_txd(ring) <= tx_num)) { netif_stop_queue(dev); netif_err(priv, tx_queued,dev, @@ -718,9 +764,7 @@ static int fe_start_xmit(struct sk_buff *skb, struct net_device *dev) return NETDEV_TX_BUSY; } - if (fe_tx_map_dma(skb, dev, tx) < 0) { - kfree_skb(skb); - + if (fe_tx_map_dma(skb, dev, tx_num, ring) < 0) { stats->tx_dropped++; } else { stats->tx_packets++; @@ -745,13 +789,14 @@ static inline void fe_rx_vlan(struct sk_buff *skb) } static int fe_poll_rx(struct napi_struct *napi, int budget, - struct fe_priv *priv) + struct fe_priv *priv, u32 rx_intr) { struct net_device *netdev = priv->netdev; struct net_device_stats *stats = &netdev->stats; struct fe_soc_data *soc = priv->soc; + struct fe_rx_ring *ring = &priv->rx_ring; + int idx = ring->rx_calc_idx; u32 checksum_bit; - int idx = fe_reg_r32(FE_REG_RX_CALC_IDX0); struct sk_buff *skb; u8 *data, *new_data; struct fe_rx_dma *rxd, trxd; @@ -767,26 +812,27 @@ static int fe_poll_rx(struct napi_struct *napi, int budget, pad = 0; else pad = NET_IP_ALIGN; + while (done < budget) { unsigned int pktlen; dma_addr_t dma_addr; idx = NEXT_RX_DESP_IDX(idx); - rxd = &priv->rx_dma[idx]; - data = priv->rx_data[idx]; + rxd = &ring->rx_dma[idx]; + data = ring->rx_data[idx]; fe_get_rxd(&trxd, rxd); if (!(trxd.rxd2 & RX_DMA_DONE)) break; /* alloc new buffer */ - new_data = netdev_alloc_frag(priv->frag_size); + new_data = netdev_alloc_frag(ring->frag_size); if (unlikely(!new_data)) { stats->rx_dropped++; goto release_desc; } dma_addr = dma_map_single(&netdev->dev, new_data + NET_SKB_PAD + pad, - priv->rx_buf_size, + ring->rx_buf_size, DMA_FROM_DEVICE); if (unlikely(dma_mapping_error(&netdev->dev, dma_addr))) { put_page(virt_to_head_page(new_data)); @@ -794,7 +840,7 @@ static int fe_poll_rx(struct napi_struct *napi, int budget, } /* receive data */ - skb = build_skb(data, priv->frag_size); + skb = build_skb(data, ring->frag_size); if (unlikely(!skb)) { put_page(virt_to_head_page(new_data)); goto release_desc; @@ -802,8 +848,8 @@ static int fe_poll_rx(struct napi_struct *napi, int budget, skb_reserve(skb, NET_SKB_PAD + NET_IP_ALIGN); dma_unmap_single(&netdev->dev, trxd.rxd1, - priv->rx_buf_size, DMA_FROM_DEVICE); - pktlen = RX_DMA_PLEN0(trxd.rxd2); + ring->rx_buf_size, DMA_FROM_DEVICE); + pktlen = RX_DMA_GET_PLEN0(trxd.rxd2); skb->dev = netdev; skb_put(skb, pktlen); if (trxd.rxd4 & checksum_bit) { @@ -818,70 +864,77 @@ static int fe_poll_rx(struct napi_struct *napi, int budget, stats->rx_packets++; stats->rx_bytes += pktlen; - if (skb->ip_summed == CHECKSUM_NONE) - netif_receive_skb(skb); - else - napi_gro_receive(napi, skb); + napi_gro_receive(napi, skb); - priv->rx_data[idx] = new_data; + ring->rx_data[idx] = new_data; rxd->rxd1 = (unsigned int) dma_addr; release_desc: - if (soc->rx_dma) - soc->rx_dma(rxd, priv->rx_buf_size); + if (priv->flags & FE_FLAG_RX_SG_DMA) + rxd->rxd2 = RX_DMA_PLEN0(ring->rx_buf_size); else rxd->rxd2 = RX_DMA_LSO; + ring->rx_calc_idx = idx; wmb(); - fe_reg_w32(idx, FE_REG_RX_CALC_IDX0); + fe_reg_w32(ring->rx_calc_idx, FE_REG_RX_CALC_IDX0); done++; } + if (done < budget) + fe_reg_w32(rx_intr, FE_REG_FE_INT_STATUS); + return done; } -static int fe_poll_tx(struct fe_priv *priv, int budget) +static int fe_poll_tx(struct fe_priv *priv, int budget, u32 tx_intr, + int *tx_again) { struct net_device *netdev = priv->netdev; struct device *dev = &netdev->dev; unsigned int bytes_compl = 0; struct sk_buff *skb; - struct fe_tx_dma txd; - int done = 0, idx; - u32 udf_bit = priv->soc->tx_udf_bit; + struct fe_tx_buf *tx_buf; + int done = 0; + u32 idx, hwidx; + struct fe_tx_ring *ring = &priv->tx_ring; - idx = priv->tx_free_idx; - while (done < budget) { - fe_get_txd(&txd, &priv->tx_dma[idx]); - skb = priv->tx_skb[idx]; - - if (!(txd.txd2 & TX_DMA_DONE) || !skb) - break; + idx = ring->tx_free_idx; + hwidx = fe_reg_r32(FE_REG_TX_DTX_IDX0); - txd_unmap_page1(dev, &txd); + while ((idx != hwidx) && budget) { + tx_buf = &ring->tx_buf[idx]; + skb = tx_buf->skb; - if (txd.txd4 & udf_bit) - txd_unmap_single(dev, &txd); - else - txd_unmap_page0(dev, &txd); + if (!skb) + break; if (skb != (struct sk_buff *) DMA_DUMMY_DESC) { bytes_compl += skb->len; - dev_kfree_skb_any(skb); done++; + budget--; } - priv->tx_skb[idx] = NULL; + fe_txd_unmap(dev, tx_buf); idx = NEXT_TX_DESP_IDX(idx); } - priv->tx_free_idx = idx; - - if (!done) - return 0; + ring->tx_free_idx = idx; - netdev_completed_queue(netdev, done, bytes_compl); - if (unlikely(netif_queue_stopped(netdev) && - netif_carrier_ok(netdev))) { - netif_wake_queue(netdev); + if (idx == hwidx) { + /* read hw index again make sure no new tx packet */ + hwidx = fe_reg_r32(FE_REG_TX_DTX_IDX0); + if (idx == hwidx) + fe_reg_w32(tx_intr, FE_REG_FE_INT_STATUS); + else + *tx_again = 1; + } else + *tx_again = 1; + + if (done) { + netdev_completed_queue(netdev, done, bytes_compl); + smp_mb(); + if (unlikely(netif_queue_stopped(netdev) && + (fe_empty_txd(ring) > ring->tx_thresh))) + netif_wake_queue(netdev); } return done; @@ -891,37 +944,34 @@ static int fe_poll(struct napi_struct *napi, int budget) { struct fe_priv *priv = container_of(napi, struct fe_priv, rx_napi); struct fe_hw_stats *hwstat = priv->hw_stats; - int tx_done, rx_done; - u32 status, mask; - u32 tx_intr, rx_intr; + int tx_done, rx_done, tx_again; + u32 status, fe_status, status_reg, mask; + u32 tx_intr, rx_intr, status_intr; - status = fe_reg_r32(FE_REG_FE_INT_STATUS); + fe_status = status = fe_reg_r32(FE_REG_FE_INT_STATUS); tx_intr = priv->soc->tx_int; rx_intr = priv->soc->rx_int; - tx_done = rx_done = 0; + status_intr = priv->soc->status_int; + tx_done = rx_done = tx_again = 0; -poll_again: - if (status & tx_intr) { - tx_done += fe_poll_tx(priv, budget - tx_done); - if (tx_done < budget) { - fe_reg_w32(tx_intr, FE_REG_FE_INT_STATUS); - } - status = fe_reg_r32(FE_REG_FE_INT_STATUS); - } + if (fe_reg_table[FE_REG_FE_INT_STATUS2]) { + fe_status = fe_reg_r32(FE_REG_FE_INT_STATUS2); + status_reg = FE_REG_FE_INT_STATUS2; + } else + status_reg = FE_REG_FE_INT_STATUS; - if (status & rx_intr) { - rx_done += fe_poll_rx(napi, budget - rx_done, priv); - if (rx_done < budget) { - fe_reg_w32(rx_intr, FE_REG_FE_INT_STATUS); - } - } + if (status & tx_intr) + tx_done = fe_poll_tx(priv, budget, tx_intr, &tx_again); + + if (status & rx_intr) + rx_done = fe_poll_rx(napi, budget, priv, rx_intr); - if (unlikely(hwstat && (status & FE_CNT_GDM_AF))) { - if (spin_trylock(&hwstat->stats_lock)) { + if (unlikely(fe_status & status_intr)) { + if (hwstat && spin_trylock(&hwstat->stats_lock)) { fe_stats_update(priv); spin_unlock(&hwstat->stats_lock); } - fe_reg_w32(FE_CNT_GDM_AF, FE_REG_FE_INT_STATUS); + fe_reg_w32(status_intr, status_reg); } if (unlikely(netif_msg_intr(priv))) { @@ -931,21 +981,23 @@ poll_again: tx_done, rx_done, status, mask); } - if ((tx_done < budget) && (rx_done < budget)) { + if (!tx_again && (rx_done < budget)) { status = fe_reg_r32(FE_REG_FE_INT_STATUS); - if (status & (tx_intr | rx_intr )) { + if (status & (tx_intr | rx_intr )) goto poll_again; - } + napi_complete(napi); fe_int_enable(tx_intr | rx_intr); } +poll_again: return rx_done; } static void fe_tx_timeout(struct net_device *dev) { struct fe_priv *priv = netdev_priv(dev); + struct fe_tx_ring *ring = &priv->tx_ring; priv->netdev->stats.tx_errors++; netif_err(priv, tx_err, dev, @@ -953,12 +1005,13 @@ static void fe_tx_timeout(struct net_device *dev) netif_info(priv, drv, dev, "dma_cfg:%08x\n", fe_reg_r32(FE_REG_PDMA_GLO_CFG)); netif_info(priv, drv, dev, "tx_ring=%d, " \ - "base=%08x, max=%u, ctx=%u, dtx=%u, fdx=%d\n", 0, + "base=%08x, max=%u, ctx=%u, dtx=%u, fdx=%hu, next=%hu\n", 0, fe_reg_r32(FE_REG_TX_BASE_PTR0), fe_reg_r32(FE_REG_TX_MAX_CNT0), fe_reg_r32(FE_REG_TX_CTX_IDX0), fe_reg_r32(FE_REG_TX_DTX_IDX0), - priv->tx_free_idx + ring->tx_free_idx, + ring->tx_next_idx ); netif_info(priv, drv, dev, "rx_ring=%d, " \ "base=%08x, max=%u, calc=%u, drx=%u\n", 0, @@ -984,8 +1037,10 @@ static irqreturn_t fe_handle_irq(int irq, void *dev) int_mask = (priv->soc->rx_int | priv->soc->tx_int); if (likely(status & int_mask)) { - fe_int_disable(int_mask); - napi_schedule(&priv->rx_napi); + if (likely(napi_schedule_prep(&priv->rx_napi))) { + fe_int_disable(int_mask); + __napi_schedule(&priv->rx_napi); + } } else { fe_reg_w32(status, FE_REG_FE_INT_STATUS); } @@ -1086,6 +1141,9 @@ static int fe_hw_init(struct net_device *dev) else fe_hw_set_macaddr(priv, dev->dev_addr); + /* disable delay interrupt */ + fe_reg_w32(0, FE_REG_DLY_INT_CFG); + fe_int_disable(priv->soc->tx_int | priv->soc->rx_int); /* frame engine will push VLAN tag regarding to VIDX feild in Tx desc. */ @@ -1119,7 +1177,6 @@ static int fe_open(struct net_device *dev) goto err_out; spin_lock_irqsave(&priv->page_lock, flags); - napi_enable(&priv->rx_napi); val = FE_TX_WB_DDONE | FE_RX_DMA_EN | FE_TX_DMA_EN; if (priv->flags & FE_FLAG_RX_2B_OFFSET) @@ -1135,8 +1192,9 @@ static int fe_open(struct net_device *dev) if (priv->soc->has_carrier && priv->soc->has_carrier(priv)) netif_carrier_on(dev); - netif_start_queue(dev); + napi_enable(&priv->rx_napi); fe_int_enable(priv->soc->tx_int | priv->soc->rx_int); + netif_start_queue(dev); return 0; @@ -1151,15 +1209,14 @@ static int fe_stop(struct net_device *dev) unsigned long flags; int i; - fe_int_disable(priv->soc->tx_int | priv->soc->rx_int); - netif_tx_disable(dev); + fe_int_disable(priv->soc->tx_int | priv->soc->rx_int); + napi_disable(&priv->rx_napi); if (priv->phy) priv->phy->stop(priv); spin_lock_irqsave(&priv->page_lock, flags); - napi_disable(&priv->rx_napi); fe_reg_w32(fe_reg_r32(FE_REG_PDMA_GLO_CFG) & ~(FE_TX_WB_DDONE | FE_RX_DMA_EN | FE_TX_DMA_EN), @@ -1193,8 +1250,13 @@ static int __init fe_init(struct net_device *dev) if (priv->soc->switch_init) priv->soc->switch_init(priv); - memcpy(dev->dev_addr, priv->soc->mac, ETH_ALEN); of_get_mac_address_mtd(priv->device->of_node, dev->dev_addr); + /*If the mac address is invalid, use random mac address */ + if (!is_valid_ether_addr(dev->dev_addr)) { + random_ether_addr(dev->dev_addr); + dev_err(priv->device, "generated random MAC address %pM\n", + dev->dev_addr); + } err = fe_mdio_init(priv); if (err) @@ -1284,13 +1346,11 @@ static int fe_change_mtu(struct net_device *dev, int new_mtu) if (old_mtu > ETH_DATA_LEN && new_mtu > ETH_DATA_LEN) return 0; - if (new_mtu <= ETH_DATA_LEN) { - priv->frag_size = fe_max_frag_size(ETH_DATA_LEN); - priv->rx_buf_size = fe_max_buf_size(ETH_DATA_LEN); - } else { - priv->frag_size = PAGE_SIZE; - priv->rx_buf_size = fe_max_buf_size(PAGE_SIZE); - } + if (new_mtu <= ETH_DATA_LEN) + priv->rx_ring.frag_size = fe_max_frag_size(ETH_DATA_LEN); + else + priv->rx_ring.frag_size = PAGE_SIZE; + priv->rx_ring.rx_buf_size = fe_max_buf_size(priv->rx_ring.frag_size); if (!netif_running(dev)) return 0; @@ -1375,7 +1435,7 @@ static int fe_probe(struct platform_device *pdev) struct net_device *netdev; struct fe_priv *priv; struct clk *sysclk; - int err; + int err, napi_weight; device_reset(&pdev->dev); @@ -1387,7 +1447,7 @@ static int fe_probe(struct platform_device *pdev) else soc->reg_table = fe_reg_table; - fe_base = devm_request_and_ioremap(&pdev->dev, res); + fe_base = devm_ioremap_resource(&pdev->dev, res); if (!fe_base) { err = -EADDRNOTAVAIL; goto err_out; @@ -1442,16 +1502,18 @@ static int fe_probe(struct platform_device *pdev) priv->device = &pdev->dev; priv->soc = soc; priv->msg_enable = netif_msg_init(fe_msg_level, FE_DEFAULT_MSG_ENABLE); - priv->frag_size = fe_max_frag_size(ETH_DATA_LEN); - priv->rx_buf_size = fe_max_buf_size(ETH_DATA_LEN); - if (priv->frag_size > PAGE_SIZE) { - dev_err(&pdev->dev, "error frag size.\n"); - err = -EINVAL; - goto err_free_dev; - } + priv->rx_ring.frag_size = fe_max_frag_size(ETH_DATA_LEN); + priv->rx_ring.rx_buf_size = fe_max_buf_size(priv->rx_ring.frag_size); + priv->tx_ring.tx_ring_size = priv->rx_ring.rx_ring_size = NUM_DMA_DESC; INIT_WORK(&priv->pending_work, fe_pending_work); - netif_napi_add(netdev, &priv->rx_napi, fe_poll, 32); + napi_weight = 32; + if (priv->flags & FE_FLAG_NAPI_WEIGHT) { + napi_weight *= 4; + priv->tx_ring.tx_ring_size *= 4; + priv->rx_ring.rx_ring_size *= 4; + } + netif_napi_add(netdev, &priv->rx_napi, fe_poll, napi_weight); fe_set_ethtool_ops(netdev); err = register_netdev(netdev);