1 From: Felix Fietkau <nbd@nbd.name>
2 Date: Tue, 23 Apr 2024 11:23:03 +0200
3 Subject: [PATCH] net: add TCP fraglist GRO support
5 When forwarding TCP after GRO, software segmentation is very expensive,
6 especially when the checksum needs to be recalculated.
7 One case where that's currently unavoidable is when routing packets over
8 PPPoE. Performance improves significantly when using fraglist GRO
9 implemented in the same way as for UDP.
11 Here's a measurement of running 2 TCP streams through a MediaTek MT7622
12 device (2-core Cortex-A53), which runs NAT with flow offload enabled from
13 one ethernet port to PPPoE on another ethernet port + cake qdisc set to
16 rx-gro-list off: 630 Mbit/s, CPU 35% idle
17 rx-gro-list on: 770 Mbit/s, CPU 40% idle
19 Signe-off-by: Felix Fietkau <nbd@nbd.name>
22 --- a/include/net/gro.h
23 +++ b/include/net/gro.h
24 @@ -439,6 +439,7 @@ static inline __wsum ip6_gro_compute_pse
27 int skb_gro_receive(struct sk_buff *p, struct sk_buff *skb);
28 +int skb_gro_receive_list(struct sk_buff *p, struct sk_buff *skb);
30 /* Pass the currently batched GRO_NORMAL SKBs up to the stack. */
31 static inline void gro_normal_list(struct napi_struct *napi)
32 --- a/include/net/tcp.h
33 +++ b/include/net/tcp.h
34 @@ -2082,7 +2082,10 @@ void tcp_v4_destroy_sock(struct sock *sk
36 struct sk_buff *tcp_gso_segment(struct sk_buff *skb,
37 netdev_features_t features);
38 -struct sk_buff *tcp_gro_receive(struct list_head *head, struct sk_buff *skb);
39 +struct tcphdr *tcp_gro_pull_header(struct sk_buff *skb);
40 +struct sk_buff *tcp_gro_lookup(struct list_head *head, struct tcphdr *th);
41 +struct sk_buff *tcp_gro_receive(struct list_head *head, struct sk_buff *skb,
43 INDIRECT_CALLABLE_DECLARE(int tcp4_gro_complete(struct sk_buff *skb, int thoff));
44 INDIRECT_CALLABLE_DECLARE(struct sk_buff *tcp4_gro_receive(struct list_head *head, struct sk_buff *skb));
45 INDIRECT_CALLABLE_DECLARE(int tcp6_gro_complete(struct sk_buff *skb, int thoff));
48 @@ -233,6 +233,33 @@ done:
52 +int skb_gro_receive_list(struct sk_buff *p, struct sk_buff *skb)
54 + if (unlikely(p->len + skb->len >= 65536))
57 + if (NAPI_GRO_CB(p)->last == p)
58 + skb_shinfo(p)->frag_list = skb;
60 + NAPI_GRO_CB(p)->last->next = skb;
62 + skb_pull(skb, skb_gro_offset(skb));
64 + NAPI_GRO_CB(p)->last = skb;
65 + NAPI_GRO_CB(p)->count++;
66 + p->data_len += skb->len;
68 + /* sk ownership - if any - completely transferred to the aggregated packet */
69 + skb->destructor = NULL;
71 + p->truesize += skb->truesize;
74 + NAPI_GRO_CB(skb)->same_flow = 1;
80 static void napi_gro_complete(struct napi_struct *napi, struct sk_buff *skb)
82 --- a/net/ipv4/tcp_offload.c
83 +++ b/net/ipv4/tcp_offload.c
84 @@ -28,6 +28,70 @@ static void tcp_gso_tstamp(struct sk_buf
88 +static void __tcpv4_gso_segment_csum(struct sk_buff *seg,
89 + __be32 *oldip, __be32 newip,
90 + __be16 *oldport, __be16 newport)
95 + if (*oldip == newip && *oldport == newport)
101 + inet_proto_csum_replace4(&th->check, seg, *oldip, newip, true);
102 + inet_proto_csum_replace2(&th->check, seg, *oldport, newport, false);
103 + *oldport = newport;
105 + csum_replace4(&iph->check, *oldip, newip);
109 +static struct sk_buff *__tcpv4_gso_segment_list_csum(struct sk_buff *segs)
111 + const struct tcphdr *th;
112 + const struct iphdr *iph;
113 + struct sk_buff *seg;
114 + struct tcphdr *th2;
115 + struct iphdr *iph2;
120 + th2 = tcp_hdr(seg->next);
121 + iph2 = ip_hdr(seg->next);
123 + if (!(*(const u32 *)&th->source ^ *(const u32 *)&th2->source) &&
124 + iph->daddr == iph2->daddr && iph->saddr == iph2->saddr)
127 + while ((seg = seg->next)) {
128 + th2 = tcp_hdr(seg);
129 + iph2 = ip_hdr(seg);
131 + __tcpv4_gso_segment_csum(seg,
132 + &iph2->saddr, iph->saddr,
133 + &th2->source, th->source);
134 + __tcpv4_gso_segment_csum(seg,
135 + &iph2->daddr, iph->daddr,
136 + &th2->dest, th->dest);
142 +static struct sk_buff *__tcp4_gso_segment_list(struct sk_buff *skb,
143 + netdev_features_t features)
145 + skb = skb_segment_list(skb, features, skb_mac_header_len(skb));
149 + return __tcpv4_gso_segment_list_csum(skb);
152 static struct sk_buff *tcp4_gso_segment(struct sk_buff *skb,
153 netdev_features_t features)
155 @@ -37,6 +101,9 @@ static struct sk_buff *tcp4_gso_segment(
156 if (!pskb_may_pull(skb, sizeof(struct tcphdr)))
157 return ERR_PTR(-EINVAL);
159 + if (skb_shinfo(skb)->gso_type & SKB_GSO_FRAGLIST)
160 + return __tcp4_gso_segment_list(skb, features);
162 if (unlikely(skb->ip_summed != CHECKSUM_PARTIAL)) {
163 const struct iphdr *iph = ip_hdr(skb);
164 struct tcphdr *th = tcp_hdr(skb);
165 @@ -178,61 +245,76 @@ out:
169 -struct sk_buff *tcp_gro_receive(struct list_head *head, struct sk_buff *skb)
170 +struct sk_buff *tcp_gro_lookup(struct list_head *head, struct tcphdr *th)
172 - struct sk_buff *pp = NULL;
173 + struct tcphdr *th2;
176 + list_for_each_entry(p, head, list) {
177 + if (!NAPI_GRO_CB(p)->same_flow)
181 + if (*(u32 *)&th->source ^ *(u32 *)&th2->source) {
182 + NAPI_GRO_CB(p)->same_flow = 0;
192 +struct tcphdr *tcp_gro_pull_header(struct sk_buff *skb)
194 + unsigned int thlen, hlen, off;
196 - struct tcphdr *th2;
198 - unsigned int thlen;
200 - unsigned int mss = 1;
206 off = skb_gro_offset(skb);
207 hlen = off + sizeof(*th);
208 th = skb_gro_header(skb, hlen, off);
213 thlen = th->doff * 4;
214 if (thlen < sizeof(*th))
219 if (skb_gro_header_hard(skb, hlen)) {
220 th = skb_gro_header_slow(skb, hlen, off);
226 skb_gro_pull(skb, thlen);
228 - len = skb_gro_len(skb);
229 - flags = tcp_flag_word(th);
231 - list_for_each_entry(p, head, list) {
232 - if (!NAPI_GRO_CB(p)->same_flow)
238 +struct sk_buff *tcp_gro_receive(struct list_head *head, struct sk_buff *skb,
241 + unsigned int thlen = th->doff * 4;
242 + struct sk_buff *pp = NULL;
244 + struct tcphdr *th2;
247 + unsigned int mss = 1;
251 - if (*(u32 *)&th->source ^ *(u32 *)&th2->source) {
252 - NAPI_GRO_CB(p)->same_flow = 0;
255 + len = skb_gro_len(skb);
256 + flags = tcp_flag_word(th);
261 - goto out_check_final;
262 + p = tcp_gro_lookup(head, th);
264 + goto out_check_final;
267 /* Include the IP ID check below from the inner most IP hdr */
269 flush = NAPI_GRO_CB(p)->flush;
270 flush |= (__force int)(flags & TCP_FLAG_CWR);
271 flush |= (__force int)((flags ^ tcp_flag_word(th2)) &
272 @@ -269,6 +351,18 @@ found:
273 flush |= p->decrypted ^ skb->decrypted;
276 + if (unlikely(NAPI_GRO_CB(p)->is_flist)) {
277 + flush |= (__force int)(flags ^ tcp_flag_word(th2));
278 + flush |= skb->ip_summed != p->ip_summed;
279 + flush |= skb->csum_level != p->csum_level;
280 + flush |= NAPI_GRO_CB(p)->count >= 64;
282 + if (flush || skb_gro_receive_list(p, skb))
285 + goto out_check_final;
288 if (flush || skb_gro_receive(p, skb)) {
290 goto out_check_final;
291 @@ -290,7 +384,6 @@ out_check_final:
292 if (p && (!NAPI_GRO_CB(skb)->same_flow || flush))
296 NAPI_GRO_CB(skb)->flush |= (flush != 0);
299 @@ -314,18 +407,58 @@ void tcp_gro_complete(struct sk_buff *sk
301 EXPORT_SYMBOL(tcp_gro_complete);
303 +static void tcp4_check_fraglist_gro(struct list_head *head, struct sk_buff *skb,
306 + const struct iphdr *iph;
312 + if (!(skb->dev->features & NETIF_F_GRO_FRAGLIST))
315 + p = tcp_gro_lookup(head, th);
317 + NAPI_GRO_CB(skb)->is_flist = NAPI_GRO_CB(p)->is_flist;
321 + inet_get_iif_sdif(skb, &iif, &sdif);
322 + iph = skb_gro_network_header(skb);
323 + net = dev_net(skb->dev);
324 + sk = __inet_lookup_established(net, net->ipv4.tcp_death_row.hashinfo,
325 + iph->saddr, th->source,
326 + iph->daddr, ntohs(th->dest),
328 + NAPI_GRO_CB(skb)->is_flist = !sk;
333 INDIRECT_CALLABLE_SCOPE
334 struct sk_buff *tcp4_gro_receive(struct list_head *head, struct sk_buff *skb)
338 /* Don't bother verifying checksum if we're going to flush anyway. */
339 if (!NAPI_GRO_CB(skb)->flush &&
340 skb_gro_checksum_validate(skb, IPPROTO_TCP,
341 - inet_gro_compute_pseudo)) {
342 - NAPI_GRO_CB(skb)->flush = 1;
345 + inet_gro_compute_pseudo))
348 + th = tcp_gro_pull_header(skb);
352 - return tcp_gro_receive(head, skb);
353 + tcp4_check_fraglist_gro(head, skb, th);
355 + return tcp_gro_receive(head, skb, th);
358 + NAPI_GRO_CB(skb)->flush = 1;
362 INDIRECT_CALLABLE_SCOPE int tcp4_gro_complete(struct sk_buff *skb, int thoff)
363 @@ -333,6 +466,15 @@ INDIRECT_CALLABLE_SCOPE int tcp4_gro_com
364 const struct iphdr *iph = ip_hdr(skb);
365 struct tcphdr *th = tcp_hdr(skb);
367 + if (unlikely(NAPI_GRO_CB(skb)->is_flist)) {
368 + skb_shinfo(skb)->gso_type |= SKB_GSO_FRAGLIST | SKB_GSO_TCPV4;
369 + skb_shinfo(skb)->gso_segs = NAPI_GRO_CB(skb)->count;
371 + __skb_incr_checksum_unnecessary(skb);
376 th->check = ~tcp_v4_check(skb->len - thoff, iph->saddr,
378 skb_shinfo(skb)->gso_type |= SKB_GSO_TCPV4;
379 --- a/net/ipv4/udp_offload.c
380 +++ b/net/ipv4/udp_offload.c
381 @@ -433,33 +433,6 @@ out:
385 -static int skb_gro_receive_list(struct sk_buff *p, struct sk_buff *skb)
387 - if (unlikely(p->len + skb->len >= 65536))
390 - if (NAPI_GRO_CB(p)->last == p)
391 - skb_shinfo(p)->frag_list = skb;
393 - NAPI_GRO_CB(p)->last->next = skb;
395 - skb_pull(skb, skb_gro_offset(skb));
397 - NAPI_GRO_CB(p)->last = skb;
398 - NAPI_GRO_CB(p)->count++;
399 - p->data_len += skb->len;
401 - /* sk ownership - if any - completely transferred to the aggregated packet */
402 - skb->destructor = NULL;
404 - p->truesize += skb->truesize;
405 - p->len += skb->len;
407 - NAPI_GRO_CB(skb)->same_flow = 1;
413 #define UDP_GRO_CNT_MAX 64
414 static struct sk_buff *udp_gro_receive_segment(struct list_head *head,
415 --- a/net/ipv6/tcpv6_offload.c
416 +++ b/net/ipv6/tcpv6_offload.c
419 #include <linux/indirect_call_wrapper.h>
420 #include <linux/skbuff.h>
421 +#include <net/inet6_hashtables.h>
423 #include <net/protocol.h>
425 #include <net/ip6_checksum.h>
426 #include "ip6_offload.h"
428 +static void tcp6_check_fraglist_gro(struct list_head *head, struct sk_buff *skb,
431 +#if IS_ENABLED(CONFIG_IPV6)
432 + const struct ipv6hdr *hdr;
438 + if (!(skb->dev->features & NETIF_F_GRO_FRAGLIST))
441 + p = tcp_gro_lookup(head, th);
443 + NAPI_GRO_CB(skb)->is_flist = NAPI_GRO_CB(p)->is_flist;
447 + inet6_get_iif_sdif(skb, &iif, &sdif);
448 + hdr = skb_gro_network_header(skb);
449 + net = dev_net(skb->dev);
450 + sk = __inet6_lookup_established(net, net->ipv4.tcp_death_row.hashinfo,
451 + &hdr->saddr, th->source,
452 + &hdr->daddr, ntohs(th->dest),
454 + NAPI_GRO_CB(skb)->is_flist = !sk;
457 +#endif /* IS_ENABLED(CONFIG_IPV6) */
460 INDIRECT_CALLABLE_SCOPE
461 struct sk_buff *tcp6_gro_receive(struct list_head *head, struct sk_buff *skb)
465 /* Don't bother verifying checksum if we're going to flush anyway. */
466 if (!NAPI_GRO_CB(skb)->flush &&
467 skb_gro_checksum_validate(skb, IPPROTO_TCP,
468 - ip6_gro_compute_pseudo)) {
469 - NAPI_GRO_CB(skb)->flush = 1;
472 + ip6_gro_compute_pseudo))
475 - return tcp_gro_receive(head, skb);
476 + th = tcp_gro_pull_header(skb);
480 + tcp6_check_fraglist_gro(head, skb, th);
482 + return tcp_gro_receive(head, skb, th);
485 + NAPI_GRO_CB(skb)->flush = 1;
489 INDIRECT_CALLABLE_SCOPE int tcp6_gro_complete(struct sk_buff *skb, int thoff)
490 @@ -32,6 +75,15 @@ INDIRECT_CALLABLE_SCOPE int tcp6_gro_com
491 const struct ipv6hdr *iph = ipv6_hdr(skb);
492 struct tcphdr *th = tcp_hdr(skb);
494 + if (unlikely(NAPI_GRO_CB(skb)->is_flist)) {
495 + skb_shinfo(skb)->gso_type |= SKB_GSO_FRAGLIST | SKB_GSO_TCPV6;
496 + skb_shinfo(skb)->gso_segs = NAPI_GRO_CB(skb)->count;
498 + __skb_incr_checksum_unnecessary(skb);
503 th->check = ~tcp_v6_check(skb->len - thoff, &iph->saddr,
505 skb_shinfo(skb)->gso_type |= SKB_GSO_TCPV6;
506 @@ -40,6 +92,61 @@ INDIRECT_CALLABLE_SCOPE int tcp6_gro_com
510 +static void __tcpv6_gso_segment_csum(struct sk_buff *seg,
511 + __be16 *oldport, __be16 newport)
515 + if (*oldport == newport)
519 + inet_proto_csum_replace2(&th->check, seg, *oldport, newport, false);
520 + *oldport = newport;
523 +static struct sk_buff *__tcpv6_gso_segment_list_csum(struct sk_buff *segs)
525 + const struct tcphdr *th;
526 + const struct ipv6hdr *iph;
527 + struct sk_buff *seg;
528 + struct tcphdr *th2;
529 + struct ipv6hdr *iph2;
533 + iph = ipv6_hdr(seg);
534 + th2 = tcp_hdr(seg->next);
535 + iph2 = ipv6_hdr(seg->next);
537 + if (!(*(const u32 *)&th->source ^ *(const u32 *)&th2->source) &&
538 + ipv6_addr_equal(&iph->saddr, &iph2->saddr) &&
539 + ipv6_addr_equal(&iph->daddr, &iph2->daddr))
542 + while ((seg = seg->next)) {
543 + th2 = tcp_hdr(seg);
544 + iph2 = ipv6_hdr(seg);
546 + iph2->saddr = iph->saddr;
547 + iph2->daddr = iph->daddr;
548 + __tcpv6_gso_segment_csum(seg, &th2->source, th->source);
549 + __tcpv6_gso_segment_csum(seg, &th2->dest, th->dest);
555 +static struct sk_buff *__tcp6_gso_segment_list(struct sk_buff *skb,
556 + netdev_features_t features)
558 + skb = skb_segment_list(skb, features, skb_mac_header_len(skb));
562 + return __tcpv6_gso_segment_list_csum(skb);
565 static struct sk_buff *tcp6_gso_segment(struct sk_buff *skb,
566 netdev_features_t features)
568 @@ -51,6 +158,9 @@ static struct sk_buff *tcp6_gso_segment(
569 if (!pskb_may_pull(skb, sizeof(*th)))
570 return ERR_PTR(-EINVAL);
572 + if (skb_shinfo(skb)->gso_type & SKB_GSO_FRAGLIST)
573 + return __tcp6_gso_segment_list(skb, features);
575 if (unlikely(skb->ip_summed != CHECKSUM_PARTIAL)) {
576 const struct ipv6hdr *ipv6h = ipv6_hdr(skb);
577 struct tcphdr *th = tcp_hdr(skb);