target/linux/generic/backport-4.14/292-v4.16-netfilter-core-free-hooks-with-call_rcu.patch

   1 From 8c873e2199700c2de7dbd5eedb9d90d5f109462b Mon Sep 17 00:00:00 2001
   2 From: Florian Westphal <fw@strlen.de>
   3 Date: Fri, 1 Dec 2017 00:21:04 +0100
   4 Subject: [PATCH 04/11] netfilter: core: free hooks with call_rcu
   5
   6 Giuseppe Scrivano says:
   7   "SELinux, if enabled, registers for each new network namespace 6
   8     netfilter hooks."
   9
  10 Cost for this is high.  With synchronize_net() removed:
  11    "The net benefit on an SMP machine with two cores is that creating a
  12    new network namespace takes -40% of the original time."
  13
  14 This patch replaces synchronize_net+kvfree with call_rcu().
  15 We store rcu_head at the tail of a structure that has no fixed layout,
  16 i.e. we cannot use offsetof() to compute the start of the original
  17 allocation.  Thus store this information right after the rcu head.
  18
  19 We could simplify this by just placing the rcu_head at the start
  20 of struct nf_hook_entries.  However, this structure is used in
  21 packet processing hotpath, so only place what is needed for that
  22 at the beginning of the struct.
  23
  24 Reported-by: Giuseppe Scrivano <gscrivan@redhat.com>
  25 Signed-off-by: Florian Westphal <fw@strlen.de>
  26 Signed-off-by: Pablo Neira Ayuso <pablo@netfilter.org>
  27 ---
  28  include/linux/netfilter.h | 19 +++++++++++++++----
  29  net/netfilter/core.c      | 34 ++++++++++++++++++++++++++++------
  30  2 files changed, 43 insertions(+), 10 deletions(-)
  31
  32 --- a/include/linux/netfilter.h
  33 +++ b/include/linux/netfilter.h
  34 @@ -77,17 +77,28 @@ struct nf_hook_entry {
  35         void                            *priv;
  36  };
  37
  38 +struct nf_hook_entries_rcu_head {
  39 +       struct rcu_head head;
  40 +       void    *allocation;
  41 +};
  42 +
  43  struct nf_hook_entries {
  44         u16                             num_hook_entries;
  45         /* padding */
  46         struct nf_hook_entry            hooks[];
  47
  48 -       /* trailer: pointers to original orig_ops of each hook.
  49 -        *
  50 -        * This is not part of struct nf_hook_entry since its only
  51 -        * needed in slow path (hook register/unregister).
  52 +       /* trailer: pointers to original orig_ops of each hook,
  53 +        * followed by rcu_head and scratch space used for freeing
  54 +        * the structure via call_rcu.
  55          *
  56 +        *   This is not part of struct nf_hook_entry since its only
  57 +        *   needed in slow path (hook register/unregister):
  58          * const struct nf_hook_ops     *orig_ops[]
  59 +        *
  60 +        *   For the same reason, we store this at end -- its
  61 +        *   only needed when a hook is deleted, not during
  62 +        *   packet path processing:
  63 +        * struct nf_hook_entries_rcu_head     head
  64          */
  65  };
  66
  67 --- a/net/netfilter/core.c
  68 +++ b/net/netfilter/core.c
  69 @@ -74,7 +74,8 @@ static struct nf_hook_entries *allocate_
  70         struct nf_hook_entries *e;
  71         size_t alloc = sizeof(*e) +
  72                        sizeof(struct nf_hook_entry) * num +
  73 -                      sizeof(struct nf_hook_ops *) * num;
  74 +                      sizeof(struct nf_hook_ops *) * num +
  75 +                      sizeof(struct nf_hook_entries_rcu_head);
  76
  77         if (num == 0)
  78                 return NULL;
  79 @@ -85,6 +86,30 @@ static struct nf_hook_entries *allocate_
  80         return e;
  81  }
  82
  83 +static void __nf_hook_entries_free(struct rcu_head *h)
  84 +{
  85 +       struct nf_hook_entries_rcu_head *head;
  86 +
  87 +       head = container_of(h, struct nf_hook_entries_rcu_head, head);
  88 +       kvfree(head->allocation);
  89 +}
  90 +
  91 +static void nf_hook_entries_free(struct nf_hook_entries *e)
  92 +{
  93 +       struct nf_hook_entries_rcu_head *head;
  94 +       struct nf_hook_ops **ops;
  95 +       unsigned int num;
  96 +
  97 +       if (!e)
  98 +               return;
  99 +
 100 +       num = e->num_hook_entries;
 101 +       ops = nf_hook_entries_get_hook_ops(e);
 102 +       head = (void *)&ops[num];
 103 +       head->allocation = e;
 104 +       call_rcu(&head->head, __nf_hook_entries_free);
 105 +}
 106 +
 107  static unsigned int accept_all(void *priv,
 108                                struct sk_buff *skb,
 109                                const struct nf_hook_state *state)
 110 @@ -291,9 +316,8 @@ int nf_register_net_hook(struct net *net
 111  #ifdef HAVE_JUMP_LABEL
 112         static_key_slow_inc(&nf_hooks_needed[reg->pf][reg->hooknum]);
 113  #endif
 114 -       synchronize_net();
 115         BUG_ON(p == new_hooks);
 116 -       kvfree(p);
 117 +       nf_hook_entries_free(p);
 118         return 0;
 119  }
 120  EXPORT_SYMBOL(nf_register_net_hook);
 121 @@ -361,10 +385,8 @@ void nf_unregister_net_hook(struct net *
 122         if (!p)
 123                 return;
 124
 125 -       synchronize_net();
 126 -
 127         nf_queue_nf_hook_drop(net);
 128 -       kvfree(p);
 129 +       nf_hook_entries_free(p);
 130  }
 131  EXPORT_SYMBOL(nf_unregister_net_hook);
 132