projects / openwrt / staging / yousong.git / blob
? search:


83455c400319f8662667d6357447eeedacd3993b
[openwrt/staging/yousong.git] / target / linux / gemini / files-4.4 / drivers / net / ethernet / gemini / sl351x.c
1 /*
2 * Ethernet device driver for Gemini SoC (SL351x GMAC).
3 *
4 * Copyright (C) 2011, Tobias Waldvogel <tobias.waldvogel@gmail.com>
5 *
6 * Based on work by Michał Mirosław <mirq-linux@rere.qmqm.pl> and
7 * Paulius Zaleckas <paulius.zaleckas@gmail.com> and
8 * Giuseppe De Robertis <Giuseppe.DeRobertis@ba.infn.it> and
9 * GPLd spaghetti code from Raidsonic and other Gemini-based NAS vendors.
10 *
11 * This program is free software; you can redistribute it and/or modify
12 * it under the terms of the GNU General Public License as published by
13 * the Free Software Foundation; either version 2 of the License, or
14 * (at your option) any later version.
15 */
16
17 #include <linux/module.h>
18 #include <linux/kernel.h>
19 #include <linux/init.h>
20
21 #include <linux/spinlock.h>
22 #include <linux/slab.h>
23 #include <linux/dma-mapping.h>
24 #include <linux/cache.h>
25 #include <linux/interrupt.h>
26
27 #include <linux/platform_device.h>
28 #include <linux/etherdevice.h>
29 #include <linux/if_vlan.h>
30 #include <linux/skbuff.h>
31 #include <linux/phy.h>
32 #include <linux/crc32.h>
33 #include <linux/ethtool.h>
34 #include <linux/tcp.h>
35 #include <linux/u64_stats_sync.h>
36
37 #include <linux/in.h>
38 #include <linux/ip.h>
39 #include <linux/ipv6.h>
40
41 #include <mach/hardware.h>
42 #include <mach/global_reg.h>
43
44 #include <mach/gmac.h>
45 #include "sl351x_hw.h"
46
47 #define DRV_NAME "gmac-gemini"
48 #define DRV_VERSION "1.0"
49
50 #define HSIZE_8 0b00
51 #define HSIZE_16 0b01
52 #define HSIZE_32 0b10
53
54 #define HBURST_SINGLE 0b00
55 #define HBURST_INCR 0b01
56 #define HBURST_INCR4 0b10
57 #define HBURST_INCR8 0b11
58
59 #define HPROT_DATA_CACHE BIT(0)
60 #define HPROT_PRIVILIGED BIT(1)
61 #define HPROT_BUFFERABLE BIT(2)
62 #define HPROT_CACHABLE BIT(3)
63
64 #define DEFAULT_RX_COALESCE_NSECS 0
65 #define DEFAULT_GMAC_RXQ_ORDER 9
66 #define DEFAULT_GMAC_TXQ_ORDER 8
67 #define DEFAULT_RX_BUF_ORDER 11
68 #define DEFAULT_NAPI_WEIGHT 64
69 #define TX_MAX_FRAGS 16
70 #define TX_QUEUE_NUM 1 /* max: 6 */
71 #define RX_MAX_ALLOC_ORDER 2
72
73 #define GMAC0_IRQ0_2 (GMAC0_TXDERR_INT_BIT|GMAC0_TXPERR_INT_BIT| \
74 GMAC0_RXDERR_INT_BIT|GMAC0_RXPERR_INT_BIT)
75 #define GMAC0_IRQ0_TXQ0_INTS (GMAC0_SWTQ00_EOF_INT_BIT| \
76 GMAC0_SWTQ00_FIN_INT_BIT)
77 #define GMAC0_IRQ4_8 (GMAC0_MIB_INT_BIT|GMAC0_RX_OVERRUN_INT_BIT)
78
79 #define GMAC_OFFLOAD_FEATURES (NETIF_F_SG | NETIF_F_IP_CSUM | \
80 NETIF_F_IPV6_CSUM | NETIF_F_RXCSUM | \
81 NETIF_F_TSO | NETIF_F_TSO_ECN | NETIF_F_TSO6)
82
83 MODULE_AUTHOR("Tobias Waldvogel");
84 MODULE_DESCRIPTION("StorLink SL351x (Gemini) ethernet driver");
85 MODULE_LICENSE("GPL");
86 MODULE_ALIAS("platform:" DRV_NAME);
87
88 struct toe_private {
89 void __iomem *iomem;
90 spinlock_t irq_lock;
91
92 struct net_device *netdev[2];
93 __le32 mac_addr[2][3];
94
95 struct device *dev;
96 int irq;
97
98 unsigned int freeq_order;
99 unsigned int freeq_frag_order;
100 GMAC_RXDESC_T *freeq_ring;
101 dma_addr_t freeq_dma_base;
102 struct page **freeq_page_tab;
103 spinlock_t freeq_lock;
104 };
105
106 struct gmac_txq {
107 GMAC_TXDESC_T *ring;
108 struct sk_buff **skb;
109 unsigned int cptr;
110 unsigned int noirq_packets;
111 };
112
113 struct gmac_private {
114 unsigned int num;
115 struct toe_private *toe;
116 void __iomem *ctl_iomem;
117 void __iomem *dma_iomem;
118
119 void __iomem *rxq_rwptr;
120 GMAC_RXDESC_T *rxq_ring;
121 unsigned int rxq_order;
122
123 struct napi_struct napi;
124 struct hrtimer rx_coalesce_timer;
125 unsigned int rx_coalesce_nsecs;
126 unsigned int freeq_refill;
127 struct gmac_txq txq[TX_QUEUE_NUM];
128 unsigned int txq_order;
129 unsigned int irq_every_tx_packets;
130
131 dma_addr_t rxq_dma_base;
132 dma_addr_t txq_dma_base;
133
134 unsigned int msg_enable;
135 spinlock_t config_lock;
136
137 struct u64_stats_sync tx_stats_syncp;
138 struct u64_stats_sync rx_stats_syncp;
139 struct u64_stats_sync ir_stats_syncp;
140
141 struct rtnl_link_stats64 stats;
142 u64 hw_stats[RX_STATS_NUM];
143 u64 rx_stats[RX_STATUS_NUM];
144 u64 rx_csum_stats[RX_CHKSUM_NUM];
145 u64 rx_napi_exits;
146 u64 tx_frag_stats[TX_MAX_FRAGS];
147 u64 tx_frags_linearized;
148 u64 tx_hw_csummed;
149 };
150
151 #define GMAC_STATS_NUM ( \
152 RX_STATS_NUM + RX_STATUS_NUM + RX_CHKSUM_NUM + 1 + \
153 TX_MAX_FRAGS + 2)
154
155 static const char gmac_stats_strings[GMAC_STATS_NUM][ETH_GSTRING_LEN] = {
156 "GMAC_IN_DISCARDS",
157 "GMAC_IN_ERRORS",
158 "GMAC_IN_MCAST",
159 "GMAC_IN_BCAST",
160 "GMAC_IN_MAC1",
161 "GMAC_IN_MAC2",
162 "RX_STATUS_GOOD_FRAME",
163 "RX_STATUS_TOO_LONG_GOOD_CRC",
164 "RX_STATUS_RUNT_FRAME",
165 "RX_STATUS_SFD_NOT_FOUND",
166 "RX_STATUS_CRC_ERROR",
167 "RX_STATUS_TOO_LONG_BAD_CRC",
168 "RX_STATUS_ALIGNMENT_ERROR",
169 "RX_STATUS_TOO_LONG_BAD_ALIGN",
170 "RX_STATUS_RX_ERR",
171 "RX_STATUS_DA_FILTERED",
172 "RX_STATUS_BUFFER_FULL",
173 "RX_STATUS_11",
174 "RX_STATUS_12",
175 "RX_STATUS_13",
176 "RX_STATUS_14",
177 "RX_STATUS_15",
178 "RX_CHKSUM_IP_UDP_TCP_OK",
179 "RX_CHKSUM_IP_OK_ONLY",
180 "RX_CHKSUM_NONE",
181 "RX_CHKSUM_3",
182 "RX_CHKSUM_IP_ERR_UNKNOWN",
183 "RX_CHKSUM_IP_ERR",
184 "RX_CHKSUM_TCP_UDP_ERR",
185 "RX_CHKSUM_7",
186 "RX_NAPI_EXITS",
187 "TX_FRAGS[1]",
188 "TX_FRAGS[2]",
189 "TX_FRAGS[3]",
190 "TX_FRAGS[4]",
191 "TX_FRAGS[5]",
192 "TX_FRAGS[6]",
193 "TX_FRAGS[7]",
194 "TX_FRAGS[8]",
195 "TX_FRAGS[9]",
196 "TX_FRAGS[10]",
197 "TX_FRAGS[11]",
198 "TX_FRAGS[12]",
199 "TX_FRAGS[13]",
200 "TX_FRAGS[14]",
201 "TX_FRAGS[15]",
202 "TX_FRAGS[16+]",
203 "TX_FRAGS_LINEARIZED",
204 "TX_HW_CSUMMED",
205 };
206
207 static void gmac_dump_dma_state(struct net_device *dev);
208
209 static void gmac_update_config0_reg(struct net_device *dev, u32 val, u32 vmask)
210 {
211 struct gmac_private *gmac = netdev_priv(dev);
212 unsigned long flags;
213 u32 reg;
214
215 spin_lock_irqsave(&gmac->config_lock, flags);
216
217 reg = readl(gmac->ctl_iomem + GMAC_CONFIG0);
218 reg = (reg & ~vmask) | val;
219 writel(reg, gmac->ctl_iomem + GMAC_CONFIG0);
220
221 spin_unlock_irqrestore(&gmac->config_lock, flags);
222 }
223
224 static void gmac_enable_tx_rx(struct net_device *dev)
225 {
226 struct gmac_private *gmac = netdev_priv(dev);
227 void __iomem *config0 = gmac->ctl_iomem + GMAC_CONFIG0;
228 unsigned long flags;
229 u32 reg;
230
231 spin_lock_irqsave(&gmac->config_lock, flags);
232
233 reg = readl(config0);
234 reg &= ~CONFIG0_TX_RX_DISABLE;
235 writel(reg, config0);
236
237 spin_unlock_irqrestore(&gmac->config_lock, flags);
238 }
239
240 static void gmac_disable_tx_rx(struct net_device *dev)
241 {
242 struct gmac_private *gmac = netdev_priv(dev);
243 void __iomem *config0 = gmac->ctl_iomem + GMAC_CONFIG0;
244 unsigned long flags;
245 u32 reg;
246
247 spin_lock_irqsave(&gmac->config_lock, flags);
248
249 reg = readl(config0);
250 reg |= CONFIG0_TX_RX_DISABLE;
251 writel(reg, config0);
252
253 spin_unlock_irqrestore(&gmac->config_lock, flags);
254
255 mdelay(10); /* let GMAC consume packet */
256 }
257
258 static void gmac_set_flow_control(struct net_device *dev, bool tx, bool rx)
259 {
260 struct gmac_private *gmac = netdev_priv(dev);
261 void __iomem *config0 = gmac->ctl_iomem + GMAC_CONFIG0;
262 unsigned long flags;
263 u32 reg;
264
265 spin_lock_irqsave(&gmac->config_lock, flags);
266
267 reg = readl(config0);
268 reg &= ~CONFIG0_FLOW_CTL;
269 if (tx)
270 reg |= CONFIG0_FLOW_TX;
271 if (rx)
272 reg |= CONFIG0_FLOW_RX;
273 writel(reg, config0);
274
275 spin_unlock_irqrestore(&gmac->config_lock, flags);
276 }
277
278 static void gmac_update_link_state(struct net_device *dev)
279 {
280 struct gmac_private *gmac = netdev_priv(dev);
281 void __iomem *status_reg = gmac->ctl_iomem + GMAC_STATUS;
282 struct phy_device *phydev = dev->phydev;
283 GMAC_STATUS_T status, old_status;
284 int pause_tx=0, pause_rx=0;
285
286 old_status.bits32 = status.bits32 = readl(status_reg);
287
288 status.bits.link = phydev->link;
289 status.bits.duplex = phydev->duplex;
290
291 switch (phydev->speed) {
292 case 1000:
293 status.bits.speed = GMAC_SPEED_1000;
294 if (phydev->interface == PHY_INTERFACE_MODE_RGMII)
295 status.bits.mii_rmii = GMAC_PHY_RGMII_1000;
296 break;
297 case 100:
298 status.bits.speed = GMAC_SPEED_100;
299 if (phydev->interface == PHY_INTERFACE_MODE_RGMII)
300 status.bits.mii_rmii = GMAC_PHY_RGMII_100_10;
301 break;
302 case 10:
303 status.bits.speed = GMAC_SPEED_10;
304 if (phydev->interface == PHY_INTERFACE_MODE_RGMII)
305 status.bits.mii_rmii = GMAC_PHY_RGMII_100_10;
306 break;
307 default:
308 netdev_warn(dev, "Not supported PHY speed (%d)\n",
309 phydev->speed);
310 }
311
312 if (phydev->duplex == DUPLEX_FULL) {
313 u16 lcladv = phy_read(phydev, MII_ADVERTISE);
314 u16 rmtadv = phy_read(phydev, MII_LPA);
315 u8 cap = mii_resolve_flowctrl_fdx(lcladv, rmtadv);
316
317 if (cap & FLOW_CTRL_RX)
318 pause_rx=1;
319 if (cap & FLOW_CTRL_TX)
320 pause_tx=1;
321 }
322
323 gmac_set_flow_control(dev, pause_tx, pause_rx);
324
325 if (old_status.bits32 == status.bits32)
326 return;
327
328 if (netif_msg_link(gmac)) {
329 phy_print_status(phydev);
330 netdev_info(dev, "link flow control: %s\n",
331 phydev->pause
332 ? (phydev->asym_pause ? "tx" : "both")
333 : (phydev->asym_pause ? "rx" : "none")
334 );
335 }
336
337 gmac_disable_tx_rx(dev);
338 writel(status.bits32, status_reg);
339 gmac_enable_tx_rx(dev);
340 }
341
342 static int gmac_setup_phy(struct net_device *dev)
343 {
344 struct gmac_private *gmac = netdev_priv(dev);
345 struct toe_private *toe = gmac->toe;
346 struct gemini_gmac_platform_data *pdata = toe->dev->platform_data;
347 GMAC_STATUS_T status = { .bits32 = 0 };
348 int num = dev->dev_id;
349
350 dev->phydev = phy_connect(dev, pdata->bus_id[num],
351 &gmac_update_link_state, pdata->interface[num]);
352
353 if (IS_ERR(dev->phydev)) {
354 int err = PTR_ERR(dev->phydev);
355 dev->phydev = NULL;
356 return err;
357 }
358
359 dev->phydev->supported &= PHY_GBIT_FEATURES;
360 dev->phydev->supported |= SUPPORTED_Asym_Pause | SUPPORTED_Pause;
361 dev->phydev->advertising = dev->phydev->supported;
362
363 /* set PHY interface type */
364 switch (dev->phydev->interface) {
365 case PHY_INTERFACE_MODE_MII:
366 status.bits.mii_rmii = GMAC_PHY_MII;
367 break;
368 case PHY_INTERFACE_MODE_GMII:
369 status.bits.mii_rmii = GMAC_PHY_GMII;
370 break;
371 case PHY_INTERFACE_MODE_RGMII:
372 status.bits.mii_rmii = GMAC_PHY_RGMII_100_10;
373 break;
374 default:
375 netdev_err(dev, "Unsupported MII interface\n");
376 phy_disconnect(dev->phydev);
377 dev->phydev = NULL;
378 return -EINVAL;
379 }
380 writel(status.bits32, gmac->ctl_iomem + GMAC_STATUS);
381
382 return 0;
383 }
384
385 static int gmac_pick_rx_max_len(int max_l3_len)
386 {
387 /* index = CONFIG_MAXLEN_XXX values */
388 static const int max_len[8] = {
389 1536, 1518, 1522, 1542,
390 9212, 10236, 1518, 1518
391 };
392 int i, n = 5;
393
394 max_l3_len += ETH_HLEN + VLAN_HLEN;
395
396 if (max_l3_len > max_len[n])
397 return -1;
398
399 for (i = 0; i < 5; ++i) {
400 if (max_len[i] >= max_l3_len && max_len[i] < max_len[n])
401 n = i;
402 }
403
404 return n;
405 }
406
407 static int gmac_init(struct net_device *dev)
408 {
409 struct gmac_private *gmac = netdev_priv(dev);
410 u32 val;
411
412 GMAC_CONFIG0_T config0 = { .bits = {
413 .dis_tx = 1,
414 .dis_rx = 1,
415 .ipv4_rx_chksum = 1,
416 .ipv6_rx_chksum = 1,
417 .rx_err_detect = 1,
418 .rgmm_edge = 1,
419 .port0_chk_hwq = 1,
420 .port1_chk_hwq = 1,
421 .port0_chk_toeq = 1,
422 .port1_chk_toeq = 1,
423 .port0_chk_classq = 1,
424 .port1_chk_classq = 1,
425 } };
426 GMAC_AHB_WEIGHT_T ahb_weight = { .bits = {
427 .rx_weight = 1,
428 .tx_weight = 1,
429 .hash_weight = 1,
430 .pre_req = 0x1f,
431 .tqDV_threshold = 0,
432 } };
433 GMAC_TX_WCR0_T hw_weigh = { .bits = {
434 .hw_tq3 = 1,
435 .hw_tq2 = 1,
436 .hw_tq1 = 1,
437 .hw_tq0 = 1,
438 } };
439 GMAC_TX_WCR1_T sw_weigh = { .bits = {
440 .sw_tq5 = 1,
441 .sw_tq4 = 1,
442 .sw_tq3 = 1,
443 .sw_tq2 = 1,
444 .sw_tq1 = 1,
445 .sw_tq0 = 1,
446 } };
447 GMAC_CONFIG1_T config1 = { .bits = {
448 .set_threshold = 16,
449 .rel_threshold = 24,
450 } };
451 GMAC_CONFIG2_T config2 = { .bits = {
452 .set_threshold = 16,
453 .rel_threshold = 32,
454 } };
455 GMAC_CONFIG3_T config3 = { .bits = {
456 .set_threshold = 0,
457 .rel_threshold = 0,
458 } };
459
460 config0.bits.max_len = gmac_pick_rx_max_len(dev->mtu);
461
462 val = readl(gmac->ctl_iomem + GMAC_CONFIG0);
463 config0.bits.reserved = ((GMAC_CONFIG0_T)val).bits.reserved;
464 writel(config0.bits32, gmac->ctl_iomem + GMAC_CONFIG0);
465 writel(config1.bits32, gmac->ctl_iomem + GMAC_CONFIG1);
466 writel(config2.bits32, gmac->ctl_iomem + GMAC_CONFIG2);
467 writel(config3.bits32, gmac->ctl_iomem + GMAC_CONFIG3);
468
469 val = readl(gmac->dma_iomem + GMAC_AHB_WEIGHT_REG);
470 writel(ahb_weight.bits32, gmac->dma_iomem + GMAC_AHB_WEIGHT_REG);
471
472 writel(hw_weigh.bits32,
473 gmac->dma_iomem + GMAC_TX_WEIGHTING_CTRL_0_REG);
474 writel(sw_weigh.bits32,
475 gmac->dma_iomem + GMAC_TX_WEIGHTING_CTRL_1_REG);
476
477 gmac->rxq_order = DEFAULT_GMAC_RXQ_ORDER;
478 gmac->txq_order = DEFAULT_GMAC_TXQ_ORDER;
479 gmac->rx_coalesce_nsecs = DEFAULT_RX_COALESCE_NSECS;
480
481 /* Mark every quarter of the queue a packet for interrupt
482 in order to be able to wake up the queue if it was stopped */
483 gmac->irq_every_tx_packets = 1 << (gmac->txq_order - 2);
484
485 return 0;
486 }
487
488 static void gmac_uninit(struct net_device *dev)
489 {
490 if (dev->phydev)
491 phy_disconnect(dev->phydev);
492 }
493
494 static int gmac_setup_txqs(struct net_device *dev)
495 {
496 struct gmac_private *gmac = netdev_priv(dev);
497 struct toe_private *toe = gmac->toe;
498 void __iomem *rwptr_reg = gmac->dma_iomem + GMAC_SW_TX_QUEUE0_PTR_REG;
499 void __iomem *base_reg = gmac->dma_iomem + GMAC_SW_TX_QUEUE_BASE_REG;
500
501 unsigned int n_txq = dev->num_tx_queues;
502 size_t entries = 1 <<gmac->txq_order;
503 size_t len = n_txq * entries;
504 struct gmac_txq *txq = gmac->txq;
505 GMAC_TXDESC_T *desc_ring;
506 struct sk_buff **skb_tab;
507 unsigned int r;
508 int i;
509
510 skb_tab = kzalloc(len * sizeof(*skb_tab), GFP_KERNEL);
511 if (!skb_tab)
512 return -ENOMEM;
513
514 desc_ring = dma_alloc_coherent(toe->dev, len * sizeof(*desc_ring),
515 &gmac->txq_dma_base, GFP_KERNEL);
516
517 if (!desc_ring) {
518 kfree(skb_tab);
519 return -ENOMEM;
520 }
521
522 BUG_ON(gmac->txq_dma_base & ~DMA_Q_BASE_MASK);
523
524 writel(gmac->txq_dma_base | gmac->txq_order, base_reg);
525
526 for (i = 0; i < n_txq; i++) {
527 txq->ring = desc_ring;
528 txq->skb = skb_tab;
529 txq->noirq_packets = 0;
530
531 r = readw(rwptr_reg);
532 rwptr_reg += 2;
533 writew(r, rwptr_reg);
534 rwptr_reg +=2;
535 txq->cptr = r;
536
537 txq++;
538 desc_ring += entries;
539 skb_tab += entries;
540 }
541
542 return 0;
543 }
544
545 static void gmac_clean_txq(struct net_device *dev, struct gmac_txq *txq,
546 unsigned int r)
547 {
548 struct gmac_private *gmac = netdev_priv(dev);
549 struct toe_private *toe = gmac->toe;
550 unsigned int errs = 0;
551 unsigned int pkts = 0;
552 unsigned int hwchksum = 0;
553 unsigned long bytes = 0;
554 unsigned int m = (1 << gmac->txq_order) - 1;
555 unsigned int c = txq->cptr;
556 GMAC_TXDESC_0_T word0;
557 GMAC_TXDESC_1_T word1;
558 unsigned int word3;
559 dma_addr_t mapping;
560 GMAC_TXDESC_T *txd;
561 unsigned short nfrags;
562
563 if (unlikely(c == r))
564 return;
565
566 rmb();
567 while (c != r) {
568 txd = txq->ring + c;
569 word0 = txd->word0;
570 word1 = txd->word1;
571 mapping = txd->word2.buf_adr;
572 word3 = txd->word3.bits32;
573
574 dma_unmap_single(toe->dev, mapping, word0.bits.buffer_size, DMA_TO_DEVICE);
575
576 if (word3 & EOF_BIT)
577 dev_kfree_skb(txq->skb[c]);
578
579 c++;
580 c &= m;
581
582 if (!(word3 & SOF_BIT))
583 continue;
584
585 if (!word0.bits.status_tx_ok) {
586 errs++;
587 continue;
588 }
589
590 pkts++;
591 bytes += txd->word1.bits.byte_count;
592
593 if (word1.bits32 & TSS_CHECKUM_ENABLE)
594 hwchksum++;
595
596 nfrags = word0.bits.desc_count - 1;
597 if (nfrags) {
598 if (nfrags >= TX_MAX_FRAGS)
599 nfrags = TX_MAX_FRAGS - 1;
600
601 u64_stats_update_begin(&gmac->tx_stats_syncp);
602 gmac->tx_frag_stats[nfrags]++;
603 u64_stats_update_end(&gmac->ir_stats_syncp);
604 }
605 }
606
607 u64_stats_update_begin(&gmac->ir_stats_syncp);
608 gmac->stats.tx_errors += errs;
609 gmac->stats.tx_packets += pkts;
610 gmac->stats.tx_bytes += bytes;
611 gmac->tx_hw_csummed += hwchksum;
612 u64_stats_update_end(&gmac->ir_stats_syncp);
613
614 txq->cptr = c;
615 }
616
617 static void gmac_cleanup_txqs(struct net_device *dev)
618 {
619 struct gmac_private *gmac = netdev_priv(dev);
620 struct toe_private *toe = gmac->toe;
621 void __iomem *rwptr_reg = gmac->dma_iomem + GMAC_SW_TX_QUEUE0_PTR_REG;
622 void __iomem *base_reg = gmac->dma_iomem + GMAC_SW_TX_QUEUE_BASE_REG;
623
624 unsigned n_txq = dev->num_tx_queues;
625 unsigned int r, i;
626
627 for (i = 0; i < n_txq; i++) {
628 r = readw(rwptr_reg);
629 rwptr_reg += 2;
630 writew(r, rwptr_reg);
631 rwptr_reg += 2;
632
633 gmac_clean_txq(dev, gmac->txq + i, r);
634 }
635 writel(0, base_reg);
636
637 kfree(gmac->txq->skb);
638 dma_free_coherent(toe->dev,
639 n_txq * sizeof(*gmac->txq->ring) << gmac->txq_order,
640 gmac->txq->ring, gmac->txq_dma_base);
641 }
642
643 static int gmac_setup_rxq(struct net_device *dev)
644 {
645 struct gmac_private *gmac = netdev_priv(dev);
646 struct toe_private *toe = gmac->toe;
647 NONTOE_QHDR_T __iomem *qhdr = toe->iomem + TOE_DEFAULT_Q_HDR_BASE(dev->dev_id);
648
649 gmac->rxq_rwptr = &qhdr->word1;
650 gmac->rxq_ring = dma_alloc_coherent(toe->dev,
651 sizeof(*gmac->rxq_ring) << gmac->rxq_order,
652 &gmac->rxq_dma_base, GFP_KERNEL);
653 if (!gmac->rxq_ring)
654 return -ENOMEM;
655
656 BUG_ON(gmac->rxq_dma_base & ~NONTOE_QHDR0_BASE_MASK);
657
658 writel(gmac->rxq_dma_base | gmac->rxq_order, &qhdr->word0);
659 writel(0, gmac->rxq_rwptr);
660 return 0;
661 }
662
663 static void gmac_cleanup_rxq(struct net_device *dev)
664 {
665 struct gmac_private *gmac = netdev_priv(dev);
666 struct toe_private *toe = gmac->toe;
667
668 NONTOE_QHDR_T __iomem *qhdr = toe->iomem + TOE_DEFAULT_Q_HDR_BASE(dev->dev_id);
669 void __iomem *dma_reg = &qhdr->word0;
670 void __iomem *ptr_reg = &qhdr->word1;
671 GMAC_RXDESC_T *rxd = gmac->rxq_ring;
672 DMA_RWPTR_T rw;
673 unsigned int r, w;
674 unsigned int m = (1 <<gmac->rxq_order) - 1;
675 struct page *page;
676 dma_addr_t mapping;
677
678 rw.bits32 = readl(ptr_reg);
679 r = rw.bits.rptr;
680 w = rw.bits.wptr;
681 writew(r, ptr_reg + 2);
682
683 writel(0, dma_reg);
684
685 rmb();
686 while (r != w) {
687 mapping = rxd[r].word2.buf_adr;
688 r++;
689 r &= m;
690
691 if (!mapping)
692 continue;
693
694 page = pfn_to_page(dma_to_pfn(toe->dev, mapping));
695 put_page(page);
696 }
697
698 dma_free_coherent(toe->dev, sizeof(*gmac->rxq_ring) << gmac->rxq_order,
699 gmac->rxq_ring, gmac->rxq_dma_base);
700 }
701
702 static struct page *toe_freeq_alloc_map_page(struct toe_private *toe, int pn)
703 {
704 unsigned int fpp_order = PAGE_SHIFT - toe->freeq_frag_order;
705 unsigned int frag_len = 1 << toe->freeq_frag_order;
706 GMAC_RXDESC_T *freeq_entry;
707 dma_addr_t mapping;
708 struct page *page;
709 int i;
710
711 page = alloc_page(__GFP_COLD | GFP_ATOMIC);
712 if (!page)
713 return NULL;
714
715 mapping = dma_map_single(toe->dev, page_address(page),
716 PAGE_SIZE, DMA_FROM_DEVICE);
717
718 if (unlikely(dma_mapping_error(toe->dev, mapping) || !mapping)) {
719 put_page(page);
720 return NULL;
721 }
722
723 freeq_entry = toe->freeq_ring + (pn << fpp_order);
724 for (i = 1 << fpp_order; i > 0; --i) {
725 freeq_entry->word2.buf_adr = mapping;
726 freeq_entry++;
727 mapping += frag_len;
728 }
729
730 if (toe->freeq_page_tab[pn]) {
731 mapping = toe->freeq_ring[pn << fpp_order].word2.buf_adr;
732 dma_unmap_single(toe->dev, mapping, frag_len, DMA_FROM_DEVICE);
733 put_page(toe->freeq_page_tab[pn]);
734 }
735
736 toe->freeq_page_tab[pn] = page;
737 return page;
738 }
739
740 static unsigned int toe_fill_freeq(struct toe_private *toe, int reset)
741 {
742 void __iomem *rwptr_reg = toe->iomem + GLOBAL_SWFQ_RWPTR_REG;
743
744 DMA_RWPTR_T rw;
745 unsigned int pn, epn;
746 unsigned int fpp_order = PAGE_SHIFT - toe->freeq_frag_order;
747 unsigned int m_pn = (1 << (toe->freeq_order - fpp_order)) - 1;
748 struct page *page;
749 unsigned int count = 0;
750 unsigned long flags;
751
752 spin_lock_irqsave(&toe->freeq_lock, flags);
753
754 rw.bits32 = readl(rwptr_reg);
755 pn = (reset ? rw.bits.rptr : rw.bits.wptr) >> fpp_order;
756 epn = (rw.bits.rptr >> fpp_order) - 1;
757 epn &= m_pn;
758
759 while (pn != epn) {
760 page = toe->freeq_page_tab[pn];
761
762 if (atomic_read(&page->_count) > 1) {
763 unsigned int fl = (pn -epn) & m_pn;
764
765 if (fl > 64 >> fpp_order)
766 break;
767
768 page = toe_freeq_alloc_map_page(toe, pn);
769 if (!page)
770 break;
771 }
772
773 atomic_add(1 << fpp_order, &page->_count);
774 count += 1 << fpp_order;
775 pn++;
776 pn &= m_pn;
777 }
778
779 wmb();
780 writew(pn << fpp_order, rwptr_reg+2);
781
782 spin_unlock_irqrestore(&toe->freeq_lock, flags);
783 return count;
784 }
785
786 static int toe_setup_freeq(struct toe_private *toe)
787 {
788 void __iomem *dma_reg = toe->iomem + GLOBAL_SW_FREEQ_BASE_SIZE_REG;
789 QUEUE_THRESHOLD_T qt;
790 DMA_SKB_SIZE_T skbsz;
791 unsigned int filled;
792 unsigned int frag_len = 1 << toe->freeq_frag_order;
793 unsigned int len = 1 << toe->freeq_order;
794 unsigned int fpp_order = PAGE_SHIFT - toe->freeq_frag_order;
795 unsigned int pages = len >> fpp_order;
796 dma_addr_t mapping;
797 unsigned int pn;
798
799 toe->freeq_ring = dma_alloc_coherent(toe->dev,
800 sizeof(*toe->freeq_ring) << toe->freeq_order,
801 &toe->freeq_dma_base, GFP_KERNEL);
802 if (!toe->freeq_ring)
803 return -ENOMEM;
804
805 BUG_ON(toe->freeq_dma_base & ~DMA_Q_BASE_MASK);
806
807 toe->freeq_page_tab = kzalloc(pages * sizeof(*toe->freeq_page_tab),
808 GFP_KERNEL);
809 if (!toe->freeq_page_tab)
810 goto err_freeq;
811
812 for (pn = 0; pn < pages; pn++)
813 if (!toe_freeq_alloc_map_page(toe, pn))
814 goto err_freeq_alloc;
815
816 filled = toe_fill_freeq(toe, 1);
817 if (!filled)
818 goto err_freeq_alloc;
819
820 qt.bits32 = readl(toe->iomem + GLOBAL_QUEUE_THRESHOLD_REG);
821 qt.bits.swfq_empty = 32;
822 writel(qt.bits32, toe->iomem + GLOBAL_QUEUE_THRESHOLD_REG);
823
824 skbsz.bits.sw_skb_size = 1 << toe->freeq_frag_order;
825 writel(skbsz.bits32, toe->iomem + GLOBAL_DMA_SKB_SIZE_REG);
826 writel(toe->freeq_dma_base | toe->freeq_order, dma_reg);
827
828 return 0;
829
830 err_freeq_alloc:
831 while (pn > 0) {
832 --pn;
833 mapping = toe->freeq_ring[pn << fpp_order].word2.buf_adr;
834 dma_unmap_single(toe->dev, mapping, frag_len, DMA_FROM_DEVICE);
835 put_page(toe->freeq_page_tab[pn]);
836 }
837
838 err_freeq:
839 dma_free_coherent(toe->dev,
840 sizeof(*toe->freeq_ring) << toe->freeq_order,
841 toe->freeq_ring, toe->freeq_dma_base);
842 toe->freeq_ring = NULL;
843 return -ENOMEM;
844 }
845
846 static void toe_cleanup_freeq(struct toe_private *toe)
847 {
848 void __iomem *dma_reg = toe->iomem + GLOBAL_SW_FREEQ_BASE_SIZE_REG;
849 void __iomem *ptr_reg = toe->iomem + GLOBAL_SWFQ_RWPTR_REG;
850
851 unsigned int frag_len = 1 << toe->freeq_frag_order;
852 unsigned int len = 1 << toe->freeq_order;
853 unsigned int fpp_order = PAGE_SHIFT - toe->freeq_frag_order;
854 unsigned int pages = len >> fpp_order;
855 struct page *page;
856 dma_addr_t mapping;
857 unsigned int pn;
858
859 writew(readw(ptr_reg), ptr_reg + 2);
860 writel(0, dma_reg);
861
862 for (pn = 0; pn < pages; pn++) {
863 mapping = toe->freeq_ring[pn << fpp_order].word2.buf_adr;
864 dma_unmap_single(toe->dev, mapping, frag_len, DMA_FROM_DEVICE);
865
866 page = toe->freeq_page_tab[pn];
867 while (atomic_read(&page->_count) > 0)
868 put_page(page);
869 }
870
871 kfree(toe->freeq_page_tab);
872
873 dma_free_coherent(toe->dev,
874 sizeof(*toe->freeq_ring) << toe->freeq_order,
875 toe->freeq_ring, toe->freeq_dma_base);
876 }
877
878 static int toe_resize_freeq(struct toe_private *toe, int changing_dev_id)
879 {
880 void __iomem *irqen_reg = toe->iomem + GLOBAL_INTERRUPT_ENABLE_4_REG;
881 struct gmac_private *gmac;
882 struct net_device *other = toe->netdev[1 - changing_dev_id];
883 unsigned new_size = 0;
884 unsigned new_order;
885 int err;
886 unsigned long flags;
887 unsigned en;
888
889 if (other && netif_running(other))
890 return -EBUSY;
891
892 if (toe->netdev[0]) {
893 gmac = netdev_priv(toe->netdev[0]);
894 new_size = 1 << (gmac->rxq_order + 1);
895 }
896
897 if (toe->netdev[1]) {
898 gmac = netdev_priv(toe->netdev[1]);
899 new_size += 1 << (gmac->rxq_order + 1);
900 }
901
902 new_order = min(15, ilog2(new_size - 1) + 1);
903 if (toe->freeq_order == new_order)
904 return 0;
905
906 spin_lock_irqsave(&toe->irq_lock, flags);
907 en = readl(irqen_reg);
908 en &= ~SWFQ_EMPTY_INT_BIT;
909 writel(en, irqen_reg);
910
911 if (toe->freeq_ring)
912 toe_cleanup_freeq(toe);
913
914 toe->freeq_order = new_order;
915 err = toe_setup_freeq(toe);
916
917 en |= SWFQ_EMPTY_INT_BIT;
918 writel(en, irqen_reg);
919 spin_unlock_irqrestore(&toe->irq_lock, flags);
920
921 return err;
922 }
923
924 static void gmac_tx_irq_enable(struct net_device *dev, unsigned txq, int en)
925 {
926 struct gmac_private *gmac = netdev_priv(dev);
927 struct toe_private *toe = gmac->toe;
928 unsigned val, mask;
929
930 mask = GMAC0_IRQ0_TXQ0_INTS << (6 * dev->dev_id + txq);
931
932 if (en)
933 writel(mask, toe->iomem + GLOBAL_INTERRUPT_STATUS_0_REG);
934
935 val = readl(toe->iomem + GLOBAL_INTERRUPT_ENABLE_0_REG);
936 val = en ? val | mask : val & ~mask;
937 writel(val, toe->iomem + GLOBAL_INTERRUPT_ENABLE_0_REG);
938 }
939
940
941 static void gmac_tx_irq(struct net_device *dev, unsigned txq_num)
942 {
943 struct netdev_queue *ntxq = netdev_get_tx_queue(dev, txq_num);
944
945 gmac_tx_irq_enable(dev, txq_num, 0);
946 netif_tx_wake_queue(ntxq);
947 }
948
949 static int gmac_map_tx_bufs(struct net_device *dev, struct sk_buff *skb,
950 struct gmac_txq *txq, unsigned short *desc)
951 {
952 struct gmac_private *gmac = netdev_priv(dev);
953 struct toe_private *toe = gmac->toe;
954 struct skb_shared_info *skb_si = skb_shinfo(skb);
955 skb_frag_t *skb_frag;
956 short frag, last_frag = skb_si->nr_frags - 1;
957 unsigned short m = (1 << gmac->txq_order) -1;
958 unsigned short w = *desc;
959 unsigned word1, word3, buflen;
960 dma_addr_t mapping;
961 void *buffer;
962 unsigned short mtu;
963 GMAC_TXDESC_T *txd;
964
965 mtu = ETH_HLEN;
966 mtu += dev->mtu;
967 if (skb->protocol == htons(ETH_P_8021Q))
968 mtu += VLAN_HLEN;
969
970 word1 = skb->len;
971 word3 = SOF_BIT;
972
973 if (word1 > mtu) {
974 word1 |= TSS_MTU_ENABLE_BIT;
975 word3 += mtu;
976 }
977
978 if (skb->ip_summed != CHECKSUM_NONE) {
979 int tcp = 0;
980 if (skb->protocol == htons(ETH_P_IP)) {
981 word1 |= TSS_IP_CHKSUM_BIT;
982 tcp = ip_hdr(skb)->protocol == IPPROTO_TCP;
983 } else { /* IPv6 */
984 word1 |= TSS_IPV6_ENABLE_BIT;
985 tcp = ipv6_hdr(skb)->nexthdr == IPPROTO_TCP;
986 }
987
988 word1 |= tcp ? TSS_TCP_CHKSUM_BIT : TSS_UDP_CHKSUM_BIT;
989 }
990
991 frag = -1;
992 while (frag <= last_frag) {
993 if (frag == -1) {
994 buffer = skb->data;
995 buflen = skb_headlen(skb);
996 } else {
997 skb_frag = skb_si->frags + frag;
998 buffer = page_address(skb_frag_page(skb_frag)) +
999 skb_frag->page_offset;
1000 buflen = skb_frag->size;
1001 }
1002
1003 if (frag == last_frag) {
1004 word3 |= EOF_BIT;
1005 txq->skb[w] = skb;
1006 }
1007
1008 mapping = dma_map_single(toe->dev, buffer, buflen,
1009 DMA_TO_DEVICE);
1010 if (dma_mapping_error(toe->dev, mapping) ||
1011 !(mapping & PAGE_MASK))
1012 goto map_error;
1013
1014 txd = txq->ring + w;
1015 txd->word0.bits32 = buflen;
1016 txd->word1.bits32 = word1;
1017 txd->word2.buf_adr = mapping;
1018 txd->word3.bits32 = word3;
1019
1020 word3 &= MTU_SIZE_BIT_MASK;
1021 w++;
1022 w &= m;
1023 frag++;
1024 }
1025
1026 *desc = w;
1027 return 0;
1028
1029 map_error:
1030 while (w != *desc) {
1031 w--;
1032 w &= m;
1033
1034 dma_unmap_page(toe->dev, txq->ring[w].word2.buf_adr,
1035 txq->ring[w].word0.bits.buffer_size, DMA_TO_DEVICE);
1036 }
1037 return ENOMEM;
1038 }
1039
1040 static int gmac_start_xmit(struct sk_buff *skb, struct net_device *dev)
1041 {
1042 struct gmac_private *gmac = netdev_priv(dev);
1043
1044 void __iomem *ptr_reg;
1045 struct gmac_txq *txq;
1046 struct netdev_queue *ntxq;
1047 int txq_num, nfrags;
1048 DMA_RWPTR_T rw;
1049 unsigned short r, w, d;
1050 unsigned short m = (1 << gmac->txq_order) - 1;
1051
1052 SKB_FRAG_ASSERT(skb);
1053
1054 if (unlikely(skb->len >= 0x10000))
1055 goto out_drop_free;
1056
1057 txq_num = skb_get_queue_mapping(skb);
1058 ptr_reg = gmac->dma_iomem + GMAC_SW_TX_QUEUE_PTR_REG(txq_num);
1059 txq = &gmac->txq[txq_num];
1060 ntxq = netdev_get_tx_queue(dev, txq_num);
1061 nfrags = skb_shinfo(skb)->nr_frags;
1062
1063 rw.bits32 = readl(ptr_reg);
1064 r = rw.bits.rptr;
1065 w = rw.bits.wptr;
1066
1067 d = txq->cptr - w - 1;
1068 d &= m;
1069
1070 if (unlikely(d < nfrags+2))
1071 {
1072 gmac_clean_txq(dev, txq, r);
1073 d = txq->cptr - w - 1;
1074 d &= m;
1075
1076 if (unlikely(d < nfrags+2)) {
1077 netif_tx_stop_queue(ntxq);
1078
1079 d = txq->cptr + nfrags + 16;
1080 d &= m;
1081 txq->ring[d].word3.bits.eofie = 1;
1082 gmac_tx_irq_enable(dev, txq_num, 1);
1083
1084 u64_stats_update_begin(&gmac->tx_stats_syncp);
1085 dev->stats.tx_fifo_errors++;
1086 u64_stats_update_end(&gmac->tx_stats_syncp);
1087 return NETDEV_TX_BUSY;
1088 }
1089 }
1090
1091 if (unlikely(gmac_map_tx_bufs(dev, skb, txq, &w))) {
1092 if (skb_linearize(skb))
1093 goto out_drop;
1094
1095 if (unlikely(gmac_map_tx_bufs(dev, skb, txq, &w)))
1096 goto out_drop_free;
1097
1098 u64_stats_update_begin(&gmac->tx_stats_syncp);
1099 gmac->tx_frags_linearized++;
1100 u64_stats_update_end(&gmac->tx_stats_syncp);
1101 }
1102
1103 writew(w, ptr_reg+2);
1104
1105 gmac_clean_txq(dev, txq, r);
1106 return NETDEV_TX_OK;
1107
1108 out_drop_free:
1109 dev_kfree_skb(skb);
1110 out_drop:
1111 u64_stats_update_begin(&gmac->tx_stats_syncp);
1112 gmac->stats.tx_dropped++;
1113 u64_stats_update_end(&gmac->tx_stats_syncp);
1114 return NETDEV_TX_OK;
1115 }
1116
1117 static void gmac_tx_timeout(struct net_device *dev)
1118 {
1119 netdev_err(dev, "Tx timeout\n");
1120 gmac_dump_dma_state(dev);
1121 }
1122
1123 static void gmac_enable_irq(struct net_device *dev, int enable)
1124 {
1125 struct gmac_private *gmac = netdev_priv(dev);
1126 struct toe_private *toe = gmac->toe;
1127 unsigned long flags;
1128 unsigned val, mask;
1129
1130 spin_lock_irqsave(&toe->irq_lock, flags);
1131
1132 mask = GMAC0_IRQ0_2 << (dev->dev_id * 2);
1133 val = readl(toe->iomem + GLOBAL_INTERRUPT_ENABLE_0_REG);
1134 val = enable ? (val | mask) : (val & ~mask);
1135 writel(val, toe->iomem + GLOBAL_INTERRUPT_ENABLE_0_REG);
1136
1137 mask = DEFAULT_Q0_INT_BIT << dev->dev_id;
1138 val = readl(toe->iomem + GLOBAL_INTERRUPT_ENABLE_1_REG);
1139 val = enable ? (val | mask) : (val & ~mask);
1140 writel(val, toe->iomem + GLOBAL_INTERRUPT_ENABLE_1_REG);
1141
1142 mask = GMAC0_IRQ4_8 << (dev->dev_id * 8);
1143 val = readl(toe->iomem + GLOBAL_INTERRUPT_ENABLE_4_REG);
1144 val = enable ? (val | mask) : (val & ~mask);
1145 writel(val, toe->iomem + GLOBAL_INTERRUPT_ENABLE_4_REG);
1146
1147 spin_unlock_irqrestore(&toe->irq_lock, flags);
1148 }
1149
1150 static void gmac_enable_rx_irq(struct net_device *dev, int enable)
1151 {
1152 struct gmac_private *gmac = netdev_priv(dev);
1153 struct toe_private *toe = gmac->toe;
1154 unsigned long flags;
1155 unsigned val, mask;
1156
1157 spin_lock_irqsave(&toe->irq_lock, flags);
1158 mask = DEFAULT_Q0_INT_BIT << dev->dev_id;
1159
1160 val = readl(toe->iomem + GLOBAL_INTERRUPT_ENABLE_1_REG);
1161 val = enable ? (val | mask) : (val & ~mask);
1162 writel(val, toe->iomem + GLOBAL_INTERRUPT_ENABLE_1_REG);
1163
1164 spin_unlock_irqrestore(&toe->irq_lock, flags);
1165 }
1166
1167 static struct sk_buff *gmac_skb_if_good_frame(struct gmac_private *gmac,
1168 GMAC_RXDESC_0_T word0, unsigned frame_len)
1169 {
1170 struct sk_buff *skb = NULL;
1171 unsigned rx_status = word0.bits.status;
1172 unsigned rx_csum = word0.bits.chksum_status;
1173
1174 gmac->rx_stats[rx_status]++;
1175 gmac->rx_csum_stats[rx_csum]++;
1176
1177 if (word0.bits.derr || word0.bits.perr ||
1178 rx_status || frame_len < ETH_ZLEN ||
1179 rx_csum >= RX_CHKSUM_IP_ERR_UNKNOWN) {
1180 gmac->stats.rx_errors++;
1181
1182 if (frame_len < ETH_ZLEN || RX_ERROR_LENGTH(rx_status))
1183 gmac->stats.rx_length_errors++;
1184 if (RX_ERROR_OVER(rx_status))
1185 gmac->stats.rx_over_errors++;
1186 if (RX_ERROR_CRC(rx_status))
1187 gmac->stats.rx_crc_errors++;
1188 if (RX_ERROR_FRAME(rx_status))
1189 gmac->stats.rx_frame_errors++;
1190
1191 return NULL;
1192 }
1193
1194 skb = napi_get_frags(&gmac->napi);
1195 if (!skb)
1196 return NULL;
1197
1198 if (rx_csum == RX_CHKSUM_IP_UDP_TCP_OK)
1199 skb->ip_summed = CHECKSUM_UNNECESSARY;
1200
1201 gmac->stats.rx_bytes += frame_len;
1202 gmac->stats.rx_packets++;
1203 return skb;
1204 }
1205
1206 static unsigned gmac_rx(struct net_device *dev, unsigned budget)
1207 {
1208 struct gmac_private *gmac = netdev_priv(dev);
1209 struct toe_private *toe = gmac->toe;
1210 void __iomem *ptr_reg = gmac->rxq_rwptr;
1211
1212 static struct sk_buff *skb;
1213
1214 DMA_RWPTR_T rw;
1215 unsigned short r, w;
1216 unsigned short m = (1 << gmac->rxq_order) -1;
1217 GMAC_RXDESC_T *rx = NULL;
1218 struct page* page = NULL;
1219 unsigned page_offs;
1220 unsigned int frame_len, frag_len;
1221 int frag_nr = 0;
1222
1223 GMAC_RXDESC_0_T word0;
1224 GMAC_RXDESC_1_T word1;
1225 dma_addr_t mapping;
1226 GMAC_RXDESC_3_T word3;
1227
1228 rw.bits32 = readl(ptr_reg);
1229 /* Reset interrupt as all packages until here are taken into account */
1230 writel(DEFAULT_Q0_INT_BIT << dev->dev_id,
1231 toe->iomem + GLOBAL_INTERRUPT_STATUS_1_REG);
1232 r = rw.bits.rptr;
1233 w = rw.bits.wptr;
1234
1235 while (budget && w != r) {
1236 rx = gmac->rxq_ring + r;
1237 word0 = rx->word0;
1238 word1 = rx->word1;
1239 mapping = rx->word2.buf_adr;
1240 word3 = rx->word3;
1241
1242 r++;
1243 r &= m;
1244
1245 frag_len = word0.bits.buffer_size;
1246 frame_len =word1.bits.byte_count;
1247 page_offs = mapping & ~PAGE_MASK;
1248
1249 if (unlikely(!mapping)) {
1250 netdev_err(dev, "rxq[%u]: HW BUG: zero DMA desc\n", r);
1251 goto err_drop;
1252 }
1253
1254 page = pfn_to_page(dma_to_pfn(toe->dev, mapping));
1255
1256 if (word3.bits32 & SOF_BIT) {
1257 if (unlikely(skb)) {
1258 napi_free_frags(&gmac->napi);
1259 gmac->stats.rx_dropped++;
1260 }
1261
1262 skb = gmac_skb_if_good_frame(gmac, word0, frame_len);
1263 if (unlikely(!skb))
1264 goto err_drop;
1265
1266 page_offs += NET_IP_ALIGN;
1267 frag_len -= NET_IP_ALIGN;
1268 frag_nr = 0;
1269
1270 } else if (!skb) {
1271 put_page(page);
1272 continue;
1273 }
1274
1275 if (word3.bits32 & EOF_BIT)
1276 frag_len = frame_len - skb->len;
1277
1278 /* append page frag to skb */
1279 if (unlikely(frag_nr == MAX_SKB_FRAGS))
1280 goto err_drop;
1281
1282 if (frag_len == 0)
1283 netdev_err(dev, "Received fragment with len = 0\n");
1284
1285 skb_fill_page_desc(skb, frag_nr, page, page_offs, frag_len);
1286 skb->len += frag_len;
1287 skb->data_len += frag_len;
1288 skb->truesize += frag_len;
1289 frag_nr++;
1290
1291 if (word3.bits32 & EOF_BIT) {
1292 napi_gro_frags(&gmac->napi);
1293 skb = NULL;
1294 --budget;
1295 }
1296 continue;
1297
1298 err_drop:
1299 if (skb) {
1300 napi_free_frags(&gmac->napi);
1301 skb = NULL;
1302 }
1303
1304 if (mapping)
1305 put_page(page);
1306
1307 gmac->stats.rx_dropped++;
1308 }
1309
1310 writew(r, ptr_reg);
1311 return budget;
1312 }
1313
1314 static int gmac_napi_poll(struct napi_struct *napi, int budget)
1315 {
1316 struct gmac_private *gmac = netdev_priv(napi->dev);
1317 struct toe_private *toe = gmac->toe;
1318 unsigned rx;
1319 unsigned freeq_threshold = 1 << (toe->freeq_order - 1);
1320
1321 u64_stats_update_begin(&gmac->rx_stats_syncp);
1322
1323 rx = budget - gmac_rx(napi->dev, budget);
1324
1325 if (rx == 0) {
1326 napi_gro_flush(napi, false);
1327 __napi_complete(napi);
1328 gmac_enable_rx_irq(napi->dev, 1);
1329 ++gmac->rx_napi_exits;
1330 }
1331
1332 gmac->freeq_refill += rx;
1333 if (gmac->freeq_refill > freeq_threshold) {
1334 gmac->freeq_refill -= freeq_threshold;
1335 toe_fill_freeq(toe, 0);
1336 }
1337
1338 u64_stats_update_end(&gmac->rx_stats_syncp);
1339 return budget;
1340 }
1341
1342 static void gmac_dump_dma_state(struct net_device *dev)
1343 {
1344 struct gmac_private *gmac = netdev_priv(dev);
1345 struct toe_private *toe = gmac->toe;
1346 void __iomem *ptr_reg;
1347 unsigned reg[5];
1348
1349 /* Interrupt status */
1350 reg[0] = readl(toe->iomem + GLOBAL_INTERRUPT_STATUS_0_REG);
1351 reg[1] = readl(toe->iomem + GLOBAL_INTERRUPT_STATUS_1_REG);
1352 reg[2] = readl(toe->iomem + GLOBAL_INTERRUPT_STATUS_2_REG);
1353 reg[3] = readl(toe->iomem + GLOBAL_INTERRUPT_STATUS_3_REG);
1354 reg[4] = readl(toe->iomem + GLOBAL_INTERRUPT_STATUS_4_REG);
1355 netdev_err(dev, "IRQ status: 0x%08x 0x%08x 0x%08x 0x%08x 0x%08x\n",
1356 reg[0], reg[1], reg[2], reg[3], reg[4]);
1357
1358 /* Interrupt enable */
1359 reg[0] = readl(toe->iomem + GLOBAL_INTERRUPT_ENABLE_0_REG);
1360 reg[1] = readl(toe->iomem + GLOBAL_INTERRUPT_ENABLE_1_REG);
1361 reg[2] = readl(toe->iomem + GLOBAL_INTERRUPT_ENABLE_2_REG);
1362 reg[3] = readl(toe->iomem + GLOBAL_INTERRUPT_ENABLE_3_REG);
1363 reg[4] = readl(toe->iomem + GLOBAL_INTERRUPT_ENABLE_4_REG);
1364 netdev_err(dev, "IRQ enable: 0x%08x 0x%08x 0x%08x 0x%08x 0x%08x\n",
1365 reg[0], reg[1], reg[2], reg[3], reg[4]);
1366
1367 /* RX DMA status */
1368 reg[0] = readl(gmac->dma_iomem + GMAC_DMA_RX_FIRST_DESC_REG);
1369 reg[1] = readl(gmac->dma_iomem + GMAC_DMA_RX_CURR_DESC_REG);
1370 reg[2] = GET_RPTR(gmac->rxq_rwptr);
1371 reg[3] = GET_WPTR(gmac->rxq_rwptr);
1372 netdev_err(dev, "RX DMA regs: 0x%08x 0x%08x, ptr: %u %u\n",
1373 reg[0], reg[1], reg[2], reg[3]);
1374
1375 reg[0] = readl(gmac->dma_iomem + GMAC_DMA_RX_DESC_WORD0_REG);
1376 reg[1] = readl(gmac->dma_iomem + GMAC_DMA_RX_DESC_WORD1_REG);
1377 reg[2] = readl(gmac->dma_iomem + GMAC_DMA_RX_DESC_WORD2_REG);
1378 reg[3] = readl(gmac->dma_iomem + GMAC_DMA_RX_DESC_WORD3_REG);
1379 netdev_err(dev, "RX DMA descriptor: 0x%08x 0x%08x 0x%08x 0x%08x\n",
1380 reg[0], reg[1], reg[2], reg[3]);
1381
1382 /* TX DMA status */
1383 ptr_reg = gmac->dma_iomem + GMAC_SW_TX_QUEUE0_PTR_REG;
1384
1385 reg[0] = readl(gmac->dma_iomem + GMAC_DMA_TX_FIRST_DESC_REG);
1386 reg[1] = readl(gmac->dma_iomem + GMAC_DMA_TX_CURR_DESC_REG);
1387 reg[2] = GET_RPTR(ptr_reg);
1388 reg[3] = GET_WPTR(ptr_reg);
1389 netdev_err(dev, "TX DMA regs: 0x%08x 0x%08x, ptr: %u %u\n",
1390 reg[0], reg[1], reg[2], reg[3]);
1391
1392 reg[0] = readl(gmac->dma_iomem + GMAC_DMA_TX_DESC_WORD0_REG);
1393 reg[1] = readl(gmac->dma_iomem + GMAC_DMA_TX_DESC_WORD1_REG);
1394 reg[2] = readl(gmac->dma_iomem + GMAC_DMA_TX_DESC_WORD2_REG);
1395 reg[3] = readl(gmac->dma_iomem + GMAC_DMA_TX_DESC_WORD3_REG);
1396 netdev_err(dev, "TX DMA descriptor: 0x%08x 0x%08x 0x%08x 0x%08x\n",
1397 reg[0], reg[1], reg[2], reg[3]);
1398
1399 /* FREE queues status */
1400 ptr_reg = toe->iomem + GLOBAL_SWFQ_RWPTR_REG;
1401
1402 reg[0] = GET_RPTR(ptr_reg);
1403 reg[1] = GET_WPTR(ptr_reg);
1404
1405 ptr_reg = toe->iomem + GLOBAL_HWFQ_RWPTR_REG;
1406
1407 reg[2] = GET_RPTR(ptr_reg);
1408 reg[3] = GET_WPTR(ptr_reg);
1409 netdev_err(dev, "FQ SW ptr: %u %u, HW ptr: %u %u\n",
1410 reg[0], reg[1], reg[2], reg[3]);
1411 }
1412
1413 static void gmac_update_hw_stats(struct net_device *dev)
1414 {
1415 struct gmac_private *gmac = netdev_priv(dev);
1416 struct toe_private *toe = gmac->toe;
1417 unsigned long flags;
1418 unsigned int rx_discards, rx_mcast, rx_bcast;
1419
1420 spin_lock_irqsave(&toe->irq_lock, flags);
1421 u64_stats_update_begin(&gmac->ir_stats_syncp);
1422
1423 gmac->hw_stats[0] += rx_discards = readl(gmac->ctl_iomem + GMAC_IN_DISCARDS);
1424 gmac->hw_stats[1] += readl(gmac->ctl_iomem + GMAC_IN_ERRORS);
1425 gmac->hw_stats[2] += rx_mcast = readl(gmac->ctl_iomem + GMAC_IN_MCAST);
1426 gmac->hw_stats[3] += rx_bcast = readl(gmac->ctl_iomem + GMAC_IN_BCAST);
1427 gmac->hw_stats[4] += readl(gmac->ctl_iomem + GMAC_IN_MAC1);
1428 gmac->hw_stats[5] += readl(gmac->ctl_iomem + GMAC_IN_MAC2);
1429
1430 gmac->stats.rx_missed_errors += rx_discards;
1431 gmac->stats.multicast += rx_mcast;
1432 gmac->stats.multicast += rx_bcast;
1433
1434 writel(GMAC0_MIB_INT_BIT << (dev->dev_id * 8),
1435 toe->iomem + GLOBAL_INTERRUPT_STATUS_4_REG);
1436
1437 u64_stats_update_end(&gmac->ir_stats_syncp);
1438 spin_unlock_irqrestore(&toe->irq_lock, flags);
1439 }
1440
1441 static inline unsigned gmac_get_intr_flags(struct net_device *dev, int i)
1442 {
1443 struct gmac_private *gmac = netdev_priv(dev);
1444 struct toe_private *toe = gmac->toe;
1445 void __iomem *irqif_reg, *irqen_reg;
1446 unsigned offs, val;
1447
1448 offs = i * (GLOBAL_INTERRUPT_STATUS_1_REG - GLOBAL_INTERRUPT_STATUS_0_REG);
1449
1450 irqif_reg = toe->iomem + GLOBAL_INTERRUPT_STATUS_0_REG + offs;
1451 irqen_reg = toe->iomem + GLOBAL_INTERRUPT_ENABLE_0_REG + offs;
1452
1453 val = readl(irqif_reg) & readl(irqen_reg);
1454 return val;
1455 }
1456
1457 enum hrtimer_restart gmac_coalesce_delay_expired( struct hrtimer *timer )
1458 {
1459 struct gmac_private *gmac = container_of(timer, struct gmac_private, rx_coalesce_timer);
1460
1461 napi_schedule(&gmac->napi);
1462 return HRTIMER_NORESTART;
1463 }
1464
1465 static irqreturn_t gmac_irq(int irq, void *data)
1466 {
1467 struct net_device *dev = data;
1468 struct gmac_private *gmac = netdev_priv(dev);
1469 struct toe_private *toe = gmac->toe;
1470 unsigned val, orr = 0;
1471
1472 orr |= val = gmac_get_intr_flags(dev, 0);
1473
1474 if (unlikely(val & (GMAC0_IRQ0_2 << (dev->dev_id * 2)))) {
1475 /* oh, crap. */
1476 netdev_err(dev, "hw failure/sw bug\n");
1477 gmac_dump_dma_state(dev);
1478
1479 /* don't know how to recover, just reduce losses */
1480 gmac_enable_irq(dev, 0);
1481 return IRQ_HANDLED;
1482 }
1483
1484 if (val & (GMAC0_IRQ0_TXQ0_INTS << (dev->dev_id * 6)))
1485 gmac_tx_irq(dev, 0);
1486
1487 orr |= val = gmac_get_intr_flags(dev, 1);
1488
1489 if (val & (DEFAULT_Q0_INT_BIT << dev->dev_id)) {
1490
1491 gmac_enable_rx_irq(dev, 0);
1492
1493 if (!gmac->rx_coalesce_nsecs)
1494 napi_schedule(&gmac->napi);
1495 else {
1496 ktime_t ktime;
1497 ktime = ktime_set(0, gmac->rx_coalesce_nsecs);
1498 hrtimer_start(&gmac->rx_coalesce_timer, ktime, HRTIMER_MODE_REL);
1499 }
1500 }
1501
1502 orr |= val = gmac_get_intr_flags(dev, 4);
1503
1504 if (unlikely(val & (GMAC0_MIB_INT_BIT << (dev->dev_id * 8))))
1505 gmac_update_hw_stats(dev);
1506
1507 if (unlikely(val & (GMAC0_RX_OVERRUN_INT_BIT << (dev->dev_id * 8)))) {
1508 writel(GMAC0_RXDERR_INT_BIT << (dev->dev_id * 8),
1509 toe->iomem + GLOBAL_INTERRUPT_STATUS_4_REG);
1510
1511 spin_lock(&toe->irq_lock);
1512 u64_stats_update_begin(&gmac->ir_stats_syncp);
1513 ++gmac->stats.rx_fifo_errors;
1514 u64_stats_update_end(&gmac->ir_stats_syncp);
1515 spin_unlock(&toe->irq_lock);
1516 }
1517
1518 return orr ? IRQ_HANDLED : IRQ_NONE;
1519 }
1520
1521 static void gmac_start_dma(struct gmac_private *gmac)
1522 {
1523 void __iomem *dma_ctrl_reg = gmac->dma_iomem + GMAC_DMA_CTRL_REG;
1524 GMAC_DMA_CTRL_T dma_ctrl;
1525
1526 dma_ctrl.bits32 = readl(dma_ctrl_reg);
1527 dma_ctrl.bits.rd_enable = 1;
1528 dma_ctrl.bits.td_enable = 1;
1529 dma_ctrl.bits.loopback = 0;
1530 dma_ctrl.bits.drop_small_ack = 0;
1531 dma_ctrl.bits.rd_insert_bytes = NET_IP_ALIGN;
1532 dma_ctrl.bits.rd_prot = HPROT_DATA_CACHE | HPROT_PRIVILIGED;
1533 dma_ctrl.bits.rd_burst_size = HBURST_INCR8;
1534 dma_ctrl.bits.rd_bus = HSIZE_8;
1535 dma_ctrl.bits.td_prot = HPROT_DATA_CACHE;
1536 dma_ctrl.bits.td_burst_size = HBURST_INCR8;
1537 dma_ctrl.bits.td_bus = HSIZE_8;
1538
1539 writel(dma_ctrl.bits32, dma_ctrl_reg);
1540 }
1541
1542 static void gmac_stop_dma(struct gmac_private *gmac)
1543 {
1544 void __iomem *dma_ctrl_reg = gmac->dma_iomem + GMAC_DMA_CTRL_REG;
1545 GMAC_DMA_CTRL_T dma_ctrl;
1546
1547 dma_ctrl.bits32 = readl(dma_ctrl_reg);
1548 dma_ctrl.bits.rd_enable = 0;
1549 dma_ctrl.bits.td_enable = 0;
1550 writel(dma_ctrl.bits32, dma_ctrl_reg);
1551 }
1552
1553 static int gmac_open(struct net_device *dev)
1554 {
1555 struct gmac_private *gmac = netdev_priv(dev);
1556 int err;
1557
1558 if (!dev->phydev) {
1559 err = gmac_setup_phy(dev);
1560 if (err) {
1561 netif_err(gmac, ifup, dev,
1562 "PHY init failed: %d\n", err);
1563 return err;
1564 }
1565 }
1566
1567 err = request_irq(dev->irq, gmac_irq,
1568 IRQF_SHARED, dev->name, dev);
1569 if (unlikely(err))
1570 return err;
1571
1572 netif_carrier_off(dev);
1573 phy_start(dev->phydev);
1574
1575 err = toe_resize_freeq(gmac->toe, dev->dev_id);
1576 if (unlikely(err))
1577 goto err_stop_phy;
1578
1579 err = gmac_setup_rxq(dev);
1580 if (unlikely(err))
1581 goto err_stop_phy;
1582
1583 err = gmac_setup_txqs(dev);
1584 if (unlikely(err)) {
1585 gmac_cleanup_rxq(dev);
1586 goto err_stop_phy;
1587 }
1588
1589 napi_enable(&gmac->napi);
1590
1591 gmac_start_dma(gmac);
1592 gmac_enable_irq(dev, 1);
1593 gmac_enable_tx_rx(dev);
1594 netif_tx_start_all_queues(dev);
1595
1596 hrtimer_init(&gmac->rx_coalesce_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
1597 gmac->rx_coalesce_timer.function = &gmac_coalesce_delay_expired;
1598 return 0;
1599
1600 err_stop_phy:
1601 phy_stop(dev->phydev);
1602 free_irq(dev->irq, dev);
1603 return err;
1604 }
1605
1606 static int gmac_stop(struct net_device *dev)
1607 {
1608 struct gmac_private *gmac = netdev_priv(dev);
1609
1610 hrtimer_cancel(&gmac->rx_coalesce_timer);
1611 netif_tx_stop_all_queues(dev);
1612 gmac_disable_tx_rx(dev);
1613 gmac_stop_dma(gmac);
1614 napi_disable(&gmac->napi);
1615
1616 gmac_enable_irq(dev, 0);
1617 gmac_cleanup_rxq(dev);
1618 gmac_cleanup_txqs(dev);
1619
1620 phy_stop(dev->phydev);
1621 free_irq(dev->irq, dev);
1622
1623 gmac_update_hw_stats(dev);
1624 return 0;
1625 }
1626
1627 static void gmac_set_rx_mode(struct net_device *dev)
1628 {
1629 struct gmac_private *gmac = netdev_priv(dev);
1630 struct netdev_hw_addr *ha;
1631 __u32 mc_filter[2];
1632 unsigned bit_nr;
1633 GMAC_RX_FLTR_T filter = { .bits = {
1634 .broadcast = 1,
1635 .multicast = 1,
1636 .unicast = 1,
1637 } };
1638
1639 mc_filter[1] = mc_filter[0] = 0;
1640
1641 if (dev->flags & IFF_PROMISC) {
1642 filter.bits.error = 1;
1643 filter.bits.promiscuous = 1;
1644 } else if (!(dev->flags & IFF_ALLMULTI)) {
1645 mc_filter[1] = mc_filter[0] = 0;
1646 netdev_for_each_mc_addr(ha, dev) {
1647 bit_nr = ~crc32_le(~0, ha->addr, ETH_ALEN) & 0x3f;
1648 mc_filter[bit_nr >> 5] |= 1 << (bit_nr & 0x1f);
1649 }
1650 }
1651
1652 writel(mc_filter[0], gmac->ctl_iomem + GMAC_MCAST_FIL0);
1653 writel(mc_filter[1], gmac->ctl_iomem + GMAC_MCAST_FIL1);
1654 writel(filter.bits32, gmac->ctl_iomem + GMAC_RX_FLTR);
1655 }
1656
1657 static void __gmac_set_mac_address(struct net_device *dev)
1658 {
1659 struct gmac_private *gmac = netdev_priv(dev);
1660 __le32 addr[3];
1661
1662 memset(addr, 0, sizeof(addr));
1663 memcpy(addr, dev->dev_addr, ETH_ALEN);
1664
1665 writel(le32_to_cpu(addr[0]), gmac->ctl_iomem + GMAC_STA_ADD0);
1666 writel(le32_to_cpu(addr[1]), gmac->ctl_iomem + GMAC_STA_ADD1);
1667 writel(le32_to_cpu(addr[2]), gmac->ctl_iomem + GMAC_STA_ADD2);
1668 }
1669
1670 static int gmac_set_mac_address(struct net_device *dev, void *addr)
1671 {
1672 struct sockaddr *sa = addr;
1673
1674 memcpy(dev->dev_addr, sa->sa_data, ETH_ALEN);
1675 __gmac_set_mac_address(dev);
1676
1677 return 0;
1678 }
1679
1680 static void gmac_clear_hw_stats(struct net_device *dev)
1681 {
1682 struct gmac_private *gmac = netdev_priv(dev);
1683
1684 readl(gmac->ctl_iomem + GMAC_IN_DISCARDS);
1685 readl(gmac->ctl_iomem + GMAC_IN_ERRORS);
1686 readl(gmac->ctl_iomem + GMAC_IN_MCAST);
1687 readl(gmac->ctl_iomem + GMAC_IN_BCAST);
1688 readl(gmac->ctl_iomem + GMAC_IN_MAC1);
1689 readl(gmac->ctl_iomem + GMAC_IN_MAC2);
1690 }
1691
1692 static struct rtnl_link_stats64 *gmac_get_stats64(struct net_device *dev,
1693 struct rtnl_link_stats64 *storage)
1694 {
1695 struct gmac_private *gmac = netdev_priv(dev);
1696 unsigned int start;
1697
1698 gmac_update_hw_stats(dev);
1699
1700 /* racing with RX NAPI */
1701 do {
1702 start = u64_stats_fetch_begin(&gmac->rx_stats_syncp);
1703
1704 storage->rx_packets = gmac->stats.rx_packets;
1705 storage->rx_bytes = gmac->stats.rx_bytes;
1706 storage->rx_errors = gmac->stats.rx_errors;
1707 storage->rx_dropped = gmac->stats.rx_dropped;
1708
1709 storage->rx_length_errors = gmac->stats.rx_length_errors;
1710 storage->rx_over_errors = gmac->stats.rx_over_errors;
1711 storage->rx_crc_errors = gmac->stats.rx_crc_errors;
1712 storage->rx_frame_errors = gmac->stats.rx_frame_errors;
1713
1714 } while (u64_stats_fetch_retry(&gmac->rx_stats_syncp, start));
1715
1716 /* racing with MIB and TX completion interrupts */
1717 do {
1718 start = u64_stats_fetch_begin(&gmac->ir_stats_syncp);
1719
1720 storage->tx_errors = gmac->stats.tx_errors;
1721 storage->tx_packets = gmac->stats.tx_packets;
1722 storage->tx_bytes = gmac->stats.tx_bytes;
1723
1724 storage->multicast = gmac->stats.multicast;
1725 storage->rx_missed_errors = gmac->stats.rx_missed_errors;
1726 storage->rx_fifo_errors = gmac->stats.rx_fifo_errors;
1727
1728 } while (u64_stats_fetch_retry(&gmac->ir_stats_syncp, start));
1729
1730 /* racing with hard_start_xmit */
1731 do {
1732 start = u64_stats_fetch_begin(&gmac->tx_stats_syncp);
1733
1734 storage->tx_dropped = gmac->stats.tx_dropped;
1735
1736 } while (u64_stats_fetch_retry(&gmac->tx_stats_syncp, start));
1737
1738 storage->rx_dropped += storage->rx_missed_errors;
1739
1740 return storage;
1741 }
1742
1743 static int gmac_change_mtu(struct net_device *dev, int new_mtu)
1744 {
1745 int max_len = gmac_pick_rx_max_len(new_mtu);
1746
1747 if (max_len < 0)
1748 return -EINVAL;
1749
1750 gmac_disable_tx_rx(dev);
1751
1752 dev->mtu = new_mtu;
1753 gmac_update_config0_reg(dev,
1754 max_len << CONFIG0_MAXLEN_SHIFT,
1755 CONFIG0_MAXLEN_MASK);
1756
1757 netdev_update_features(dev);
1758
1759 gmac_enable_tx_rx(dev);
1760
1761 return 0;
1762 }
1763
1764 static netdev_features_t gmac_fix_features(struct net_device *dev, netdev_features_t features)
1765 {
1766 if (dev->mtu + ETH_HLEN + VLAN_HLEN > MTU_SIZE_BIT_MASK)
1767 features &= ~GMAC_OFFLOAD_FEATURES;
1768
1769 return features;
1770 }
1771
1772 static int gmac_set_features(struct net_device *dev, netdev_features_t features)
1773 {
1774 struct gmac_private *gmac = netdev_priv(dev);
1775 int enable = features & NETIF_F_RXCSUM;
1776 unsigned long flags;
1777 u32 reg;
1778
1779 spin_lock_irqsave(&gmac->config_lock, flags);
1780
1781 reg = readl(gmac->ctl_iomem + GMAC_CONFIG0);
1782 reg = enable ? reg | CONFIG0_RX_CHKSUM : reg & ~CONFIG0_RX_CHKSUM;
1783 writel(reg, gmac->ctl_iomem + GMAC_CONFIG0);
1784
1785 spin_unlock_irqrestore(&gmac->config_lock, flags);
1786 return 0;
1787 }
1788
1789 static int gmac_get_sset_count(struct net_device *dev, int sset)
1790 {
1791 return sset == ETH_SS_STATS ? GMAC_STATS_NUM : 0;
1792 }
1793
1794 static void gmac_get_strings(struct net_device *dev, u32 stringset, u8 *data)
1795 {
1796 if (stringset != ETH_SS_STATS)
1797 return;
1798
1799 memcpy(data, gmac_stats_strings, sizeof(gmac_stats_strings));
1800 }
1801
1802 static void gmac_get_ethtool_stats(struct net_device *dev,
1803 struct ethtool_stats *estats, u64 *values)
1804 {
1805 struct gmac_private *gmac = netdev_priv(dev);
1806 unsigned int start;
1807 u64 *p;
1808 int i;
1809
1810 gmac_update_hw_stats(dev);
1811
1812 /* racing with MIB interrupt */
1813 do {
1814 p = values;
1815 start = u64_stats_fetch_begin(&gmac->ir_stats_syncp);
1816
1817 for (i = 0; i < RX_STATS_NUM; ++i)
1818 *p++ = gmac->hw_stats[i];
1819
1820 } while (u64_stats_fetch_retry(&gmac->ir_stats_syncp, start));
1821 values = p;
1822
1823 /* racing with RX NAPI */
1824 do {
1825 p = values;
1826 start = u64_stats_fetch_begin(&gmac->rx_stats_syncp);
1827
1828 for (i = 0; i < RX_STATUS_NUM; ++i)
1829 *p++ = gmac->rx_stats[i];
1830 for (i = 0; i < RX_CHKSUM_NUM; ++i)
1831 *p++ = gmac->rx_csum_stats[i];
1832 *p++ = gmac->rx_napi_exits;
1833
1834 } while (u64_stats_fetch_retry(&gmac->rx_stats_syncp, start));
1835 values = p;
1836
1837 /* racing with TX start_xmit */
1838 do {
1839 p = values;
1840 start = u64_stats_fetch_begin(&gmac->tx_stats_syncp);
1841
1842 for (i = 0; i < TX_MAX_FRAGS; ++i) {
1843 *values++ = gmac->tx_frag_stats[i];
1844 gmac->tx_frag_stats[i] = 0;
1845 }
1846 *values++ = gmac->tx_frags_linearized;
1847 *values++ = gmac->tx_hw_csummed;
1848
1849 } while (u64_stats_fetch_retry(&gmac->tx_stats_syncp, start));
1850 }
1851
1852 static int gmac_get_settings(struct net_device *dev, struct ethtool_cmd *cmd)
1853 {
1854 if (!dev->phydev)
1855 return -ENXIO;
1856 return phy_ethtool_gset(dev->phydev, cmd);
1857 }
1858
1859 static int gmac_set_settings(struct net_device *dev, struct ethtool_cmd *cmd)
1860 {
1861 if (!dev->phydev)
1862 return -ENXIO;
1863 return phy_ethtool_sset(dev->phydev, cmd);
1864 }
1865
1866 static int gmac_nway_reset(struct net_device *dev)
1867 {
1868 if (!dev->phydev)
1869 return -ENXIO;
1870 return phy_start_aneg(dev->phydev);
1871 }
1872
1873 static void gmac_get_pauseparam(struct net_device *dev,
1874 struct ethtool_pauseparam *pparam)
1875 {
1876 struct gmac_private *gmac = netdev_priv(dev);
1877 GMAC_CONFIG0_T config0;
1878
1879 config0.bits32 = readl(gmac->ctl_iomem + GMAC_CONFIG0);
1880
1881 pparam->rx_pause = config0.bits.rx_fc_en;
1882 pparam->tx_pause = config0.bits.tx_fc_en;
1883 pparam->autoneg = true;
1884 }
1885
1886 static void gmac_get_ringparam(struct net_device *dev,
1887 struct ethtool_ringparam *rp)
1888 {
1889 struct gmac_private *gmac = netdev_priv(dev);
1890 GMAC_CONFIG0_T config0;
1891
1892 config0.bits32 = readl(gmac->ctl_iomem + GMAC_CONFIG0);
1893
1894 rp->rx_max_pending = 1 << 15;
1895 rp->rx_mini_max_pending = 0;
1896 rp->rx_jumbo_max_pending = 0;
1897 rp->tx_max_pending = 1 << 15;
1898
1899 rp->rx_pending = 1 << gmac->rxq_order;
1900 rp->rx_mini_pending = 0;
1901 rp->rx_jumbo_pending = 0;
1902 rp->tx_pending = 1 << gmac->txq_order;
1903 }
1904
1905 static int toe_resize_freeq(struct toe_private *toe, int changing_dev_id);
1906
1907 static int gmac_set_ringparam(struct net_device *dev,
1908 struct ethtool_ringparam *rp)
1909 {
1910 struct gmac_private *gmac = netdev_priv(dev);
1911 struct toe_private *toe = gmac->toe;
1912 int err = 0;
1913
1914 if (netif_running(dev))
1915 return -EBUSY;
1916
1917 if (rp->rx_pending) {
1918 gmac->rxq_order = min(15, ilog2(rp->rx_pending - 1) + 1);
1919 err = toe_resize_freeq(toe, dev->dev_id);
1920 }
1921
1922 if (rp->tx_pending)
1923 {
1924 gmac->txq_order = min(15, ilog2(rp->tx_pending - 1) + 1);
1925 gmac->irq_every_tx_packets = 1 << (gmac->txq_order - 2);
1926 }
1927
1928 return err;
1929 }
1930
1931 static int gmac_get_coalesce(struct net_device *dev,
1932 struct ethtool_coalesce *ecmd)
1933 {
1934 struct gmac_private *gmac = netdev_priv(dev);
1935
1936 ecmd->rx_max_coalesced_frames = 1;
1937 ecmd->tx_max_coalesced_frames = gmac->irq_every_tx_packets;
1938 ecmd->rx_coalesce_usecs = gmac->rx_coalesce_nsecs/1000;
1939
1940 return 0;
1941 }
1942
1943 static int gmac_set_coalesce(struct net_device *dev,
1944 struct ethtool_coalesce *ecmd)
1945 {
1946 struct gmac_private *gmac = netdev_priv(dev);
1947
1948 if (ecmd->tx_max_coalesced_frames < 1)
1949 return -EINVAL;
1950 if (ecmd->tx_max_coalesced_frames >= 1 << gmac->txq_order)
1951 return -EINVAL;
1952
1953 gmac->irq_every_tx_packets = ecmd->tx_max_coalesced_frames;
1954 gmac->rx_coalesce_nsecs = ecmd->rx_coalesce_usecs * 1000;
1955
1956 return 0;
1957 }
1958
1959 static u32 gmac_get_msglevel(struct net_device *dev)
1960 {
1961 struct gmac_private *gmac = netdev_priv(dev);
1962 return gmac->msg_enable;
1963 }
1964
1965 static void gmac_set_msglevel(struct net_device *dev, u32 level)
1966 {
1967 struct gmac_private *gmac = netdev_priv(dev);
1968 gmac->msg_enable = level;
1969 }
1970
1971 static void gmac_get_drvinfo(struct net_device *dev,
1972 struct ethtool_drvinfo *info)
1973 {
1974 strcpy(info->driver, DRV_NAME);
1975 strcpy(info->version, DRV_VERSION);
1976 strcpy(info->bus_info, dev->dev_id ? "1" : "0");
1977 }
1978
1979 static const struct net_device_ops gmac_351x_ops = {
1980 .ndo_init = gmac_init,
1981 .ndo_uninit = gmac_uninit,
1982 .ndo_open = gmac_open,
1983 .ndo_stop = gmac_stop,
1984 .ndo_start_xmit = gmac_start_xmit,
1985 .ndo_tx_timeout = gmac_tx_timeout,
1986 .ndo_set_rx_mode = gmac_set_rx_mode,
1987 .ndo_set_mac_address = gmac_set_mac_address,
1988 .ndo_get_stats64 = gmac_get_stats64,
1989 .ndo_change_mtu = gmac_change_mtu,
1990 .ndo_fix_features = gmac_fix_features,
1991 .ndo_set_features = gmac_set_features,
1992 };
1993
1994 static const struct ethtool_ops gmac_351x_ethtool_ops = {
1995 .get_sset_count = gmac_get_sset_count,
1996 .get_strings = gmac_get_strings,
1997 .get_ethtool_stats = gmac_get_ethtool_stats,
1998 .get_settings = gmac_get_settings,
1999 .set_settings = gmac_set_settings,
2000 .get_link = ethtool_op_get_link,
2001 .nway_reset = gmac_nway_reset,
2002 .get_pauseparam = gmac_get_pauseparam,
2003 .get_ringparam = gmac_get_ringparam,
2004 .set_ringparam = gmac_set_ringparam,
2005 .get_coalesce = gmac_get_coalesce,
2006 .set_coalesce = gmac_set_coalesce,
2007 .get_msglevel = gmac_get_msglevel,
2008 .set_msglevel = gmac_set_msglevel,
2009 .get_drvinfo = gmac_get_drvinfo,
2010 };
2011
2012 static int gmac_init_netdev(struct toe_private *toe, int num,
2013 struct platform_device *pdev)
2014 {
2015 struct gemini_gmac_platform_data *pdata = pdev->dev.platform_data;
2016 struct gmac_private *gmac;
2017 struct net_device *dev;
2018 int irq, err;
2019
2020 if (!pdata->bus_id[num])
2021 return 0;
2022
2023 irq = platform_get_irq(pdev, num);
2024 if (irq < 0) {
2025 dev_err(toe->dev, "No IRQ for ethernet device #%d\n", num);
2026 return irq;
2027 }
2028
2029 dev = alloc_etherdev_mq(sizeof(*gmac), TX_QUEUE_NUM);
2030 if (!dev) {
2031 dev_err(toe->dev, "Can't allocate ethernet device #%d\n", num);
2032 return -ENOMEM;
2033 }
2034
2035 gmac = netdev_priv(dev);
2036 gmac->num = num;
2037 gmac->toe = toe;
2038 SET_NETDEV_DEV(dev, toe->dev);
2039
2040 toe->netdev[num] = dev;
2041 dev->dev_id = num;
2042
2043 gmac->ctl_iomem = toe->iomem + TOE_GMAC_BASE(num);
2044 gmac->dma_iomem = toe->iomem + TOE_GMAC_DMA_BASE(num);
2045 dev->irq = irq;
2046
2047 dev->netdev_ops = &gmac_351x_ops;
2048 dev->ethtool_ops = &gmac_351x_ethtool_ops;
2049
2050 spin_lock_init(&gmac->config_lock);
2051 gmac_clear_hw_stats(dev);
2052
2053 dev->hw_features = GMAC_OFFLOAD_FEATURES;
2054 dev->features |= GMAC_OFFLOAD_FEATURES | NETIF_F_GRO;
2055
2056 gmac->freeq_refill = 0;
2057 netif_napi_add(dev, &gmac->napi, gmac_napi_poll, DEFAULT_NAPI_WEIGHT);
2058
2059 if (is_valid_ether_addr((void *)toe->mac_addr[num]))
2060 memcpy(dev->dev_addr, toe->mac_addr[num], ETH_ALEN);
2061 else
2062 random_ether_addr(dev->dev_addr);
2063 __gmac_set_mac_address(dev);
2064
2065 err = gmac_setup_phy(dev);
2066 if (err)
2067 netif_warn(gmac, probe, dev,
2068 "PHY init failed: %d, deferring to ifup time\n", err);
2069
2070 err = register_netdev(dev);
2071 if (!err)
2072 {
2073 pr_info(DRV_NAME " %s: irq %d, dma base 0x%p, io base 0x%p\n",
2074 dev->name, irq, gmac->dma_iomem, gmac->ctl_iomem);
2075 return 0;
2076 }
2077
2078 toe->netdev[num] = NULL;
2079 free_netdev(dev);
2080 return err;
2081 }
2082
2083 static irqreturn_t toe_irq_thread(int irq, void *data)
2084 {
2085 struct toe_private *toe = data;
2086 void __iomem *irqen_reg = toe->iomem + GLOBAL_INTERRUPT_ENABLE_4_REG;
2087 void __iomem *irqif_reg = toe->iomem + GLOBAL_INTERRUPT_STATUS_4_REG;
2088 unsigned long irqmask = SWFQ_EMPTY_INT_BIT;
2089 unsigned long flags;
2090
2091 toe_fill_freeq(toe, 0);
2092
2093 /* Ack and enable interrupt */
2094 spin_lock_irqsave(&toe->irq_lock, flags);
2095 writel(irqmask, irqif_reg);
2096 irqmask |= readl(irqen_reg);
2097 writel(irqmask, irqen_reg);
2098 spin_unlock_irqrestore(&toe->irq_lock, flags);
2099
2100 return IRQ_HANDLED;
2101 }
2102
2103 static irqreturn_t toe_irq(int irq, void *data)
2104 {
2105 struct toe_private *toe = data;
2106 void __iomem *irqif_reg = toe->iomem + GLOBAL_INTERRUPT_STATUS_4_REG;
2107 void __iomem *irqen_reg = toe->iomem + GLOBAL_INTERRUPT_ENABLE_4_REG;
2108 unsigned long val, en;
2109 irqreturn_t ret = IRQ_NONE;
2110
2111 spin_lock(&toe->irq_lock);
2112
2113 val = readl(irqif_reg);
2114 en = readl(irqen_reg);
2115
2116 if (val & en & SWFQ_EMPTY_INT_BIT) {
2117 en &= ~(SWFQ_EMPTY_INT_BIT | GMAC0_RX_OVERRUN_INT_BIT
2118 | GMAC1_RX_OVERRUN_INT_BIT);
2119 writel(en, irqen_reg);
2120 ret = IRQ_WAKE_THREAD;
2121 }
2122
2123 spin_unlock(&toe->irq_lock);
2124 return ret;
2125 }
2126
2127 static int toe_init(struct toe_private *toe,
2128 struct platform_device *pdev)
2129 {
2130 int err;
2131
2132 writel(0, toe->iomem + GLOBAL_SW_FREEQ_BASE_SIZE_REG);
2133 writel(0, toe->iomem + GLOBAL_HW_FREEQ_BASE_SIZE_REG);
2134 writel(0, toe->iomem + GLOBAL_SWFQ_RWPTR_REG);
2135 writel(0, toe->iomem + GLOBAL_HWFQ_RWPTR_REG);
2136
2137 toe->freeq_frag_order = DEFAULT_RX_BUF_ORDER;
2138 toe->freeq_order = ~0;
2139
2140 err = request_threaded_irq(toe->irq, toe_irq,
2141 toe_irq_thread, IRQF_SHARED, DRV_NAME " toe", toe);
2142 if (err)
2143 goto err_freeq;
2144
2145 return 0;
2146
2147 err_freeq:
2148 toe_cleanup_freeq(toe);
2149 return err;
2150 }
2151
2152 static void toe_deinit(struct toe_private *toe)
2153 {
2154 free_irq(toe->irq, toe);
2155 toe_cleanup_freeq(toe);
2156 }
2157
2158 static int toe_reset(struct toe_private *toe)
2159 {
2160 unsigned int reg = 0, retry = 5;
2161
2162 reg = readl((void __iomem*)(IO_ADDRESS(GEMINI_GLOBAL_BASE) +
2163 GLOBAL_RESET));
2164 reg |= RESET_GMAC1 | RESET_GMAC0;
2165 writel(reg, (void __iomem*)(IO_ADDRESS(GEMINI_GLOBAL_BASE) +
2166 GLOBAL_RESET));
2167
2168 do {
2169 udelay(2);
2170 reg = readl((void __iomem*)(toe->iomem +
2171 GLOBAL_TOE_VERSION_REG));
2172 barrier();
2173 } while (!reg && --retry);
2174
2175 return reg ? 0 : -EIO;
2176 }
2177
2178 /*
2179 * Interrupt config:
2180 *
2181 * GMAC0 intr bits ------> int0 ----> eth0
2182 * GMAC1 intr bits ------> int1 ----> eth1
2183 * TOE intr -------------> int1 ----> eth1
2184 * Classification Intr --> int0 ----> eth0
2185 * Default Q0 -----------> int0 ----> eth0
2186 * Default Q1 -----------> int1 ----> eth1
2187 * FreeQ intr -----------> int1 ----> eth1
2188 */
2189 static void toe_init_irq(struct toe_private *toe)
2190 {
2191 writel(0, toe->iomem + GLOBAL_INTERRUPT_ENABLE_0_REG);
2192 writel(0, toe->iomem + GLOBAL_INTERRUPT_ENABLE_1_REG);
2193 writel(0, toe->iomem + GLOBAL_INTERRUPT_ENABLE_2_REG);
2194 writel(0, toe->iomem + GLOBAL_INTERRUPT_ENABLE_3_REG);
2195 writel(0, toe->iomem + GLOBAL_INTERRUPT_ENABLE_4_REG);
2196
2197 writel(0xCCFC0FC0, toe->iomem + GLOBAL_INTERRUPT_SELECT_0_REG);
2198 writel(0x00F00002, toe->iomem + GLOBAL_INTERRUPT_SELECT_1_REG);
2199 writel(0xFFFFFFFF, toe->iomem + GLOBAL_INTERRUPT_SELECT_2_REG);
2200 writel(0xFFFFFFFF, toe->iomem + GLOBAL_INTERRUPT_SELECT_3_REG);
2201 writel(0xFF000003, toe->iomem + GLOBAL_INTERRUPT_SELECT_4_REG);
2202
2203 /* edge-triggered interrupts packed to level-triggered one... */
2204 writel(~0, toe->iomem + GLOBAL_INTERRUPT_STATUS_0_REG);
2205 writel(~0, toe->iomem + GLOBAL_INTERRUPT_STATUS_1_REG);
2206 writel(~0, toe->iomem + GLOBAL_INTERRUPT_STATUS_2_REG);
2207 writel(~0, toe->iomem + GLOBAL_INTERRUPT_STATUS_3_REG);
2208 writel(~0, toe->iomem + GLOBAL_INTERRUPT_STATUS_4_REG);
2209 }
2210
2211 static void toe_save_mac_addr(struct toe_private *toe,
2212 struct platform_device *pdev)
2213 {
2214 struct gemini_gmac_platform_data *pdata = pdev->dev.platform_data;
2215 void __iomem *ctl;
2216 int i;
2217
2218 for (i = 0; i < 2; i++) {
2219 if (pdata->bus_id[i]) {
2220 ctl = toe->iomem + TOE_GMAC_BASE(i);
2221 toe->mac_addr[i][0] = cpu_to_le32(readl(ctl + GMAC_STA_ADD0));
2222 toe->mac_addr[i][1] = cpu_to_le32(readl(ctl + GMAC_STA_ADD1));
2223 toe->mac_addr[i][2] = cpu_to_le32(readl(ctl + GMAC_STA_ADD2));
2224 }
2225 }
2226 }
2227
2228 static int gemini_gmac_probe(struct platform_device *pdev)
2229 {
2230 struct resource *res;
2231 struct toe_private *toe;
2232 int irq, retval;
2233
2234 if (!pdev->dev.platform_data)
2235 return -EINVAL;
2236
2237 irq = platform_get_irq(pdev, 1);
2238 if (irq < 0)
2239 return irq;
2240
2241 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
2242 if (!res) {
2243 dev_err(&pdev->dev, "can't get device resources\n");
2244 return -ENODEV;
2245 }
2246
2247 toe = kzalloc(sizeof(*toe), GFP_KERNEL);
2248 if (!toe)
2249 return -ENOMEM;
2250
2251 platform_set_drvdata(pdev, toe);
2252 toe->dev = &pdev->dev;
2253 toe->irq = irq;
2254
2255 toe->iomem = ioremap(res->start, resource_size(res));
2256 if (!toe->iomem) {
2257 dev_err(toe->dev, "ioremap failed\n");
2258 retval = -EIO;
2259 goto err_data;
2260 }
2261
2262 toe_save_mac_addr(toe, pdev);
2263
2264 retval = toe_reset(toe);
2265 if (retval < 0)
2266 goto err_unmap;
2267
2268 pr_info(DRV_NAME " toe: irq %d, io base 0x%08x, version %d\n",
2269 irq, res->start, retval);
2270
2271 spin_lock_init(&toe->irq_lock);
2272 spin_lock_init(&toe->freeq_lock);
2273
2274 toe_init_irq(toe);
2275
2276 retval = toe_init(toe, pdev);
2277 if (retval)
2278 goto err_unmap;
2279
2280 retval = gmac_init_netdev(toe, 0, pdev);
2281 if (retval)
2282 goto err_uninit;
2283
2284 retval = gmac_init_netdev(toe, 1, pdev);
2285 if (retval)
2286 goto err_uninit;
2287
2288 return 0;
2289
2290 err_uninit:
2291 if (toe->netdev[0])
2292 unregister_netdev(toe->netdev[0]);
2293 toe_deinit(toe);
2294 err_unmap:
2295 iounmap(toe->iomem);
2296 err_data:
2297 kfree(toe);
2298 return retval;
2299 }
2300
2301 static int gemini_gmac_remove(struct platform_device *pdev)
2302 {
2303 struct toe_private *toe = platform_get_drvdata(pdev);
2304 int i;
2305
2306 for (i = 0; i < 2; i++)
2307 if (toe->netdev[i])
2308 unregister_netdev(toe->netdev[i]);
2309
2310 toe_init_irq(toe);
2311 toe_deinit(toe);
2312
2313 iounmap(toe->iomem);
2314 kfree(toe);
2315
2316 return 0;
2317 }
2318
2319 static struct platform_driver gemini_gmac_driver = {
2320 .probe = gemini_gmac_probe,
2321 .remove = gemini_gmac_remove,
2322 .driver.name = DRV_NAME,
2323 .driver.owner = THIS_MODULE,
2324 };
2325
2326 static int __init gemini_gmac_init(void)
2327 {
2328 #ifdef CONFIG_MDIO_GPIO_MODULE
2329 request_module("mdio-gpio");
2330 #endif
2331 return platform_driver_register(&gemini_gmac_driver);
2332 }
2333
2334 static void __exit gemini_gmac_exit(void)
2335 {
2336 platform_driver_unregister(&gemini_gmac_driver);
2337 }
2338
2339 module_init(gemini_gmac_init);
2340 module_exit(gemini_gmac_exit);
Yousong Zhou staging tree