1 From: Alexander Duyck <alexander.h.duyck@redhat.com>
2 Date: Wed, 31 Dec 2014 10:55:29 -0800
3 Subject: [PATCH] fib_trie: Update usage stats to be percpu instead of
6 The trie usage stats were currently being shared by all threads that were
7 calling fib_table_lookup. As a result when multiple threads were
8 performing lookups simultaneously the trie would begin to cache bounce
11 In order to prevent this I have updated the usage stats to use a set of
12 percpu variables. By doing this we should be able to avoid the cache
13 bouncing and still make use of these stats.
15 Signed-off-by: Alexander Duyck <alexander.h.duyck@redhat.com>
16 Signed-off-by: David S. Miller <davem@davemloft.net>
19 --- a/net/ipv4/fib_frontend.c
20 +++ b/net/ipv4/fib_frontend.c
21 @@ -67,7 +67,7 @@ static int __net_init fib4_rules_init(st
26 + fib_free_table(local_table);
30 --- a/net/ipv4/fib_trie.c
31 +++ b/net/ipv4/fib_trie.c
32 @@ -153,7 +153,7 @@ struct trie_stat {
34 struct rt_trie_node __rcu *trie;
35 #ifdef CONFIG_IP_FIB_TRIE_STATS
36 - struct trie_use_stats stats;
37 + struct trie_use_stats __percpu *stats;
41 @@ -631,7 +631,7 @@ static struct rt_trie_node *resize(struc
44 #ifdef CONFIG_IP_FIB_TRIE_STATS
45 - t->stats.resize_node_skipped++;
46 + this_cpu_inc(t->stats->resize_node_skipped);
50 @@ -658,7 +658,7 @@ static struct rt_trie_node *resize(struc
53 #ifdef CONFIG_IP_FIB_TRIE_STATS
54 - t->stats.resize_node_skipped++;
55 + this_cpu_inc(t->stats->resize_node_skipped);
59 @@ -1357,7 +1357,7 @@ static int check_leaf(struct fib_table *
60 err = fib_props[fa->fa_type].error;
62 #ifdef CONFIG_IP_FIB_TRIE_STATS
63 - t->stats.semantic_match_passed++;
64 + this_cpu_inc(t->stats->semantic_match_passed);
68 @@ -1372,7 +1372,7 @@ static int check_leaf(struct fib_table *
71 #ifdef CONFIG_IP_FIB_TRIE_STATS
72 - t->stats.semantic_match_passed++;
73 + this_cpu_inc(t->stats->semantic_match_passed);
75 res->prefixlen = li->plen;
77 @@ -1388,7 +1388,7 @@ static int check_leaf(struct fib_table *
80 #ifdef CONFIG_IP_FIB_TRIE_STATS
81 - t->stats.semantic_match_miss++;
82 + this_cpu_inc(t->stats->semantic_match_miss);
86 @@ -1399,6 +1399,9 @@ int fib_table_lookup(struct fib_table *t
87 struct fib_result *res, int fib_flags)
89 struct trie *t = (struct trie *) tb->tb_data;
90 +#ifdef CONFIG_IP_FIB_TRIE_STATS
91 + struct trie_use_stats __percpu *stats = t->stats;
94 struct rt_trie_node *n;
96 @@ -1417,7 +1420,7 @@ int fib_table_lookup(struct fib_table *t
99 #ifdef CONFIG_IP_FIB_TRIE_STATS
101 + this_cpu_inc(stats->gets);
105 @@ -1441,7 +1444,7 @@ int fib_table_lookup(struct fib_table *t
108 #ifdef CONFIG_IP_FIB_TRIE_STATS
109 - t->stats.null_node_hit++;
110 + this_cpu_inc(stats->null_node_hit);
114 @@ -1576,7 +1579,7 @@ backtrace:
117 #ifdef CONFIG_IP_FIB_TRIE_STATS
118 - t->stats.backtrack++;
119 + this_cpu_inc(stats->backtrack);
123 @@ -1830,6 +1833,11 @@ int fib_table_flush(struct fib_table *tb
125 void fib_free_table(struct fib_table *tb)
127 +#ifdef CONFIG_IP_FIB_TRIE_STATS
128 + struct trie *t = (struct trie *)tb->tb_data;
130 + free_percpu(t->stats);
131 +#endif /* CONFIG_IP_FIB_TRIE_STATS */
135 @@ -1973,7 +1981,14 @@ struct fib_table *fib_trie_table(u32 id)
136 tb->tb_num_default = 0;
138 t = (struct trie *) tb->tb_data;
139 - memset(t, 0, sizeof(*t));
140 + RCU_INIT_POINTER(t->trie, NULL);
141 +#ifdef CONFIG_IP_FIB_TRIE_STATS
142 + t->stats = alloc_percpu(struct trie_use_stats);
151 @@ -2139,18 +2154,31 @@ static void trie_show_stats(struct seq_f
153 #ifdef CONFIG_IP_FIB_TRIE_STATS
154 static void trie_show_usage(struct seq_file *seq,
155 - const struct trie_use_stats *stats)
156 + const struct trie_use_stats __percpu *stats)
158 + struct trie_use_stats s = { 0 };
161 + /* loop through all of the CPUs and gather up the stats */
162 + for_each_possible_cpu(cpu) {
163 + const struct trie_use_stats *pcpu = per_cpu_ptr(stats, cpu);
165 + s.gets += pcpu->gets;
166 + s.backtrack += pcpu->backtrack;
167 + s.semantic_match_passed += pcpu->semantic_match_passed;
168 + s.semantic_match_miss += pcpu->semantic_match_miss;
169 + s.null_node_hit += pcpu->null_node_hit;
170 + s.resize_node_skipped += pcpu->resize_node_skipped;
173 seq_printf(seq, "\nCounters:\n---------\n");
174 - seq_printf(seq, "gets = %u\n", stats->gets);
175 - seq_printf(seq, "backtracks = %u\n", stats->backtrack);
176 + seq_printf(seq, "gets = %u\n", s.gets);
177 + seq_printf(seq, "backtracks = %u\n", s.backtrack);
178 seq_printf(seq, "semantic match passed = %u\n",
179 - stats->semantic_match_passed);
180 - seq_printf(seq, "semantic match miss = %u\n",
181 - stats->semantic_match_miss);
182 - seq_printf(seq, "null node hit= %u\n", stats->null_node_hit);
183 - seq_printf(seq, "skipped node resize = %u\n\n",
184 - stats->resize_node_skipped);
185 + s.semantic_match_passed);
186 + seq_printf(seq, "semantic match miss = %u\n", s.semantic_match_miss);
187 + seq_printf(seq, "null node hit= %u\n", s.null_node_hit);
188 + seq_printf(seq, "skipped node resize = %u\n\n", s.resize_node_skipped);
190 #endif /* CONFIG_IP_FIB_TRIE_STATS */
192 @@ -2191,7 +2219,7 @@ static int fib_triestat_seq_show(struct
193 trie_collect_stats(t, &stat);
194 trie_show_stats(seq, &stat);
195 #ifdef CONFIG_IP_FIB_TRIE_STATS
196 - trie_show_usage(seq, &t->stats);
197 + trie_show_usage(seq, t->stats);