1 From 9115e8cd2a0c6eaaa900c462721f12e1d45f326c Mon Sep 17 00:00:00 2001
2 From: Eric Dumazet <edumazet@google.com>
3 Date: Sat, 3 Dec 2016 11:14:56 -0800
4 Subject: [PATCH 07/10] net: reorganize struct sock for better data locality
6 Group fields used in TX path, and keep some cache lines mostly read
7 to permit sharing among cpus.
9 Gained two 4 bytes holes on 64bit arches.
11 Added a place holder for tcp tsq_flags, next to sk_wmem_alloc
12 to speed up tcp_wfree() in the following patch.
14 I have not added ____cacheline_aligned_in_smp, this might be done later.
15 I prefer doing this once inet and tcp/udp sockets reorg is also done.
17 Tested with both TCP and UDP.
19 UDP receiver performance under flood increased by ~20 % :
20 Accessing sk_filter/sk_wq/sk_napi_id no longer stalls because sk_drops
21 was moved away from a critical cache line, now mostly read and shared.
23 /* --- cacheline 4 boundary (256 bytes) --- */
24 unsigned int sk_napi_id; /* 0x100 0x4 */
25 int sk_rcvbuf; /* 0x104 0x4 */
26 struct sk_filter * sk_filter; /* 0x108 0x8 */
28 struct socket_wq * sk_wq; /* 0x8 */
29 struct socket_wq * sk_wq_raw; /* 0x8 */
31 struct xfrm_policy * sk_policy[2]; /* 0x118 0x10 */
32 struct dst_entry * sk_rx_dst; /* 0x128 0x8 */
33 struct dst_entry * sk_dst_cache; /* 0x130 0x8 */
34 atomic_t sk_omem_alloc; /* 0x138 0x4 */
35 int sk_sndbuf; /* 0x13c 0x4 */
36 /* --- cacheline 5 boundary (320 bytes) --- */
37 int sk_wmem_queued; /* 0x140 0x4 */
38 atomic_t sk_wmem_alloc; /* 0x144 0x4 */
39 long unsigned int sk_tsq_flags; /* 0x148 0x8 */
40 struct sk_buff * sk_send_head; /* 0x150 0x8 */
41 struct sk_buff_head sk_write_queue; /* 0x158 0x18 */
42 __s32 sk_peek_off; /* 0x170 0x4 */
43 int sk_write_pending; /* 0x174 0x4 */
44 long int sk_sndtimeo; /* 0x178 0x8 */
46 Signed-off-by: Eric Dumazet <edumazet@google.com>
47 Tested-by: Paolo Abeni <pabeni@redhat.com>
48 Signed-off-by: David S. Miller <davem@davemloft.net>
50 include/net/sock.h | 51 +++++++++++++++++++++++++++------------------------
51 1 file changed, 27 insertions(+), 24 deletions(-)
53 --- a/include/net/sock.h
54 +++ b/include/net/sock.h
55 @@ -343,6 +343,9 @@ struct sock {
56 #define sk_rxhash __sk_common.skc_rxhash
58 socket_lock_t sk_lock;
61 + struct sk_buff_head sk_error_queue;
62 struct sk_buff_head sk_receive_queue;
64 * The backlog queue is special, it is always used with
65 @@ -359,14 +362,13 @@ struct sock {
68 #define sk_rmem_alloc sk_backlog.rmem_alloc
69 - int sk_forward_alloc;
72 + int sk_forward_alloc;
73 #ifdef CONFIG_NET_RX_BUSY_POLL
74 - unsigned int sk_napi_id;
75 unsigned int sk_ll_usec;
76 + /* ===== mostly read cache line ===== */
77 + unsigned int sk_napi_id;
82 struct sk_filter __rcu *sk_filter;
83 @@ -379,11 +381,30 @@ struct sock {
85 struct dst_entry *sk_rx_dst;
86 struct dst_entry __rcu *sk_dst_cache;
87 - /* Note: 32bit hole on 64bit arches */
88 - atomic_t sk_wmem_alloc;
89 atomic_t sk_omem_alloc;
92 + /* ===== cache line for TX ===== */
94 + atomic_t sk_wmem_alloc;
95 + unsigned long sk_tsq_flags;
96 + struct sk_buff *sk_send_head;
97 struct sk_buff_head sk_write_queue;
99 + int sk_write_pending;
101 + struct timer_list sk_timer;
104 + u32 sk_pacing_rate; /* bytes per second */
105 + u32 sk_max_pacing_rate;
106 + struct page_frag sk_frag;
107 + netdev_features_t sk_route_caps;
108 + netdev_features_t sk_route_nocaps;
110 + unsigned int sk_gso_max_size;
111 + gfp_t sk_allocation;
115 * Because of non atomicity rules, all
116 @@ -399,41 +420,23 @@ struct sock {
117 #define SK_PROTOCOL_MAX U8_MAX
118 kmemcheck_bitfield_end(flags);
120 - int sk_wmem_queued;
121 - gfp_t sk_allocation;
122 - u32 sk_pacing_rate; /* bytes per second */
123 - u32 sk_max_pacing_rate;
124 - netdev_features_t sk_route_caps;
125 - netdev_features_t sk_route_nocaps;
127 - unsigned int sk_gso_max_size;
130 unsigned long sk_lingertime;
131 - struct sk_buff_head sk_error_queue;
132 struct proto *sk_prot_creator;
133 rwlock_t sk_callback_lock;
137 u32 sk_max_ack_backlog;
140 struct pid *sk_peer_pid;
141 const struct cred *sk_peer_cred;
144 - struct timer_list sk_timer;
149 struct socket *sk_socket;
151 - struct page_frag sk_frag;
152 - struct sk_buff *sk_send_head;
154 - int sk_write_pending;
155 #ifdef CONFIG_SECURITY