target/linux/generic/pending-4.4/090-MIPS-c-r4k-Use-IPI-calls-for-CM-indexed-cache-ops.patch

   1 From: James Hogan <james.hogan@imgtec.com>
   2 Date: Mon, 25 Jan 2016 21:30:00 +0000
   3 Subject: [PATCH] MIPS: c-r4k: Use IPI calls for CM indexed cache ops
   4
   5 The Coherence Manager (CM) can propagate address-based ("hit") cache
   6 operations to other cores in the coherent system, alleviating software
   7 of the need to use IPI calls, however indexed cache operations are not
   8 propagated since doing so makes no sense for separate caches.
   9
  10 r4k_on_each_cpu() previously had a special case for CONFIG_MIPS_MT_SMP,
  11 intended to avoid the IPIs when the only other CPUs in the system were
  12 other VPEs in the same core, and hence sharing the same caches. This was
  13 changed by commit cccf34e9411c ("MIPS: c-r4k: Fix cache flushing for MT
  14 cores") to apparently handle multi-core multi-VPE systems, but it
  15 focussed mainly on hit cache ops, so the IPI calls were still disabled
  16 entirely for CM systems.
  17
  18 This doesn't normally cause problems, but tests can be written to hit
  19 these corner cases by using multiple threads, or changing task
  20 affinities to force the process to migrate cores. For example the
  21 failure of mprotect RW->RX to globally sync icaches (via
  22 flush_cache_range) can be detected by modifying and mprotecting a code
  23 page on one core, and migrating to a different core to execute from it.
  24
  25 Most of the functions called by r4k_on_each_cpu() perform cache
  26 operations exclusively with a single addressing-type (virtual address vs
  27 indexed), so add a type argument and modify the callers to pass in
  28 R4K_USER (user virtual addressing), R4K_KERN (global kernel virtual
  29 addressing) or R4K_INDEX (index into cache).
  30
  31 local_r4k_flush_icache_range() is split up, to allow it to be called
  32 from the rest of the kernel, or from r4k_flush_icache_range() where it
  33 will choose either indexed or hit cache operations based on the size of
  34 the range and the cache sizes.
  35
  36 local_r4k_flush_kernel_vmap_range() is split into two functions, each of
  37 which uses cache operations with a single addressing-type, with
  38 r4k_flush_kernel_vmap_range() making the decision whether to use indexed
  39 cache ops or not.
  40
  41 Signed-off-by: James Hogan <james.hogan@imgtec.com>
  42 Cc: Ralf Baechle <ralf@linux-mips.org>
  43 Cc: Paul Burton <paul.burton@imgtec.com>
  44 Cc: Leonid Yegoshin <leonid.yegoshin@imgtec.com>
  45 Cc: linux-mips@linux-mips.org
  46 ---
  47
  48 --- a/arch/mips/mm/c-r4k.c
  49 +++ b/arch/mips/mm/c-r4k.c
  50 @@ -40,6 +40,50 @@
  51  #include <asm/mips-cm.h>
  52
  53  /*
  54 + * Bits describing what cache ops an IPI callback function may perform.
  55 + *
  56 + * R4K_USER   -        Virtual user address based cache operations.
  57 + *             Ineffective on other CPUs.
  58 + * R4K_KERN   -        Virtual kernel address based cache operations (including kmap).
  59 + *             Effective on other CPUs.
  60 + * R4K_INDEX - Index based cache operations.
  61 + *             Effective on other CPUs.
  62 + */
  63 +
  64 +#define R4K_USER       BIT(0)
  65 +#define R4K_KERN       BIT(1)
  66 +#define R4K_INDEX      BIT(2)
  67 +
  68 +#ifdef CONFIG_SMP
  69 +/* The Coherence manager propagates address-based cache ops to other cores */
  70 +#define r4k_hit_globalized     mips_cm_present()
  71 +#define r4k_index_globalized   0
  72 +#else
  73 +/* If there's only 1 CPU, then all cache ops are globalized to that 1 CPU */
  74 +#define r4k_hit_globalized     1
  75 +#define r4k_index_globalized   1
  76 +#endif
  77 +
  78 +/**
  79 + * r4k_op_needs_ipi() - Decide if a cache op needs to be done on every core.
  80 + * @type:      Type of cache operations (R4K_USER, R4K_KERN or R4K_INDEX).
  81 + *
  82 + * Returns:    1 if the cache operation @type should be done on every core in
  83 + *             the system.
  84 + *             0 if the cache operation @type is globalized and only needs to
  85 + *             be performed on a simple CPU.
  86 + */
  87 +static inline bool r4k_op_needs_ipi(unsigned int type)
  88 +{
  89 +       /*
  90 +        * If hardware doesn't globalize the required cache ops we must use IPIs
  91 +        * to do so.
  92 +        */
  93 +       return (type & R4K_KERN  && !r4k_hit_globalized) ||
  94 +              (type & R4K_INDEX && !r4k_index_globalized);
  95 +}
  96 +
  97 +/*
  98   * Special Variant of smp_call_function for use by cache functions:
  99   *
 100   *  o No return value
 101 @@ -48,19 +92,11 @@
 102   *    primary cache.
 103   *  o doesn't disable interrupts on the local CPU
 104   */
 105 -static inline void r4k_on_each_cpu(void (*func) (void *info), void *info)
 106 +static inline void r4k_on_each_cpu(unsigned int type,
 107 +                                  void (*func) (void *info), void *info)
 108  {
 109         preempt_disable();
 110 -
 111 -       /*
 112 -        * The Coherent Manager propagates address-based cache ops to other
 113 -        * cores but not index-based ops. However, r4k_on_each_cpu is used
 114 -        * in both cases so there is no easy way to tell what kind of op is
 115 -        * executed to the other cores. The best we can probably do is
 116 -        * to restrict that call when a CM is not present because both
 117 -        * CM-based SMP protocols (CMP & CPS) restrict index-based cache ops.
 118 -        */
 119 -       if (!mips_cm_present())
 120 +       if (r4k_op_needs_ipi(type))
 121                 smp_call_function_many(&cpu_foreign_map, func, info, 1);
 122         func(info);
 123         preempt_enable();
 124 @@ -456,7 +492,7 @@ static inline void local_r4k___flush_cac
 125
 126  static void r4k___flush_cache_all(void)
 127  {
 128 -       r4k_on_each_cpu(local_r4k___flush_cache_all, NULL);
 129 +       r4k_on_each_cpu(R4K_INDEX, local_r4k___flush_cache_all, NULL);
 130  }
 131
 132  static inline int has_valid_asid(const struct mm_struct *mm)
 133 @@ -503,7 +539,7 @@ static void r4k_flush_cache_range(struct
 134         int exec = vma->vm_flags & VM_EXEC;
 135
 136         if (cpu_has_dc_aliases || (exec && !cpu_has_ic_fills_f_dc))
 137 -               r4k_on_each_cpu(local_r4k_flush_cache_range, vma);
 138 +               r4k_on_each_cpu(R4K_INDEX, local_r4k_flush_cache_range, vma);
 139  }
 140
 141  static inline void local_r4k_flush_cache_mm(void * args)
 142 @@ -535,7 +571,7 @@ static void r4k_flush_cache_mm(struct mm
 143         if (!cpu_has_dc_aliases)
 144                 return;
 145
 146 -       r4k_on_each_cpu(local_r4k_flush_cache_mm, mm);
 147 +       r4k_on_each_cpu(R4K_INDEX, local_r4k_flush_cache_mm, mm);
 148  }
 149
 150  struct flush_cache_page_args {
 151 @@ -629,7 +665,7 @@ static void r4k_flush_cache_page(struct
 152         args.addr = addr;
 153         args.pfn = pfn;
 154
 155 -       r4k_on_each_cpu(local_r4k_flush_cache_page, &args);
 156 +       r4k_on_each_cpu(R4K_KERN, local_r4k_flush_cache_page, &args);
 157  }
 158
 159  static inline void local_r4k_flush_data_cache_page(void * addr)
 160 @@ -642,18 +678,23 @@ static void r4k_flush_data_cache_page(un
 161         if (in_atomic())
 162                 local_r4k_flush_data_cache_page((void *)addr);
 163         else
 164 -               r4k_on_each_cpu(local_r4k_flush_data_cache_page, (void *) addr);
 165 +               r4k_on_each_cpu(R4K_KERN, local_r4k_flush_data_cache_page,
 166 +                               (void *) addr);
 167  }
 168
 169  struct flush_icache_range_args {
 170         unsigned long start;
 171         unsigned long end;
 172 +       unsigned int type;
 173  };
 174
 175 -static inline void local_r4k_flush_icache_range(unsigned long start, unsigned long end)
 176 +static inline void __local_r4k_flush_icache_range(unsigned long start,
 177 +                                                 unsigned long end,
 178 +                                                 unsigned int type)
 179  {
 180         if (!cpu_has_ic_fills_f_dc) {
 181 -               if (end - start >= dcache_size) {
 182 +               if (type == R4K_INDEX ||
 183 +                   (type & R4K_INDEX && end - start >= dcache_size)) {
 184                         r4k_blast_dcache();
 185                 } else {
 186                         R4600_HIT_CACHEOP_WAR_IMPL;
 187 @@ -661,7 +702,8 @@ static inline void local_r4k_flush_icach
 188                 }
 189         }
 190
 191 -       if (end - start > icache_size)
 192 +       if (type == R4K_INDEX ||
 193 +           (type & R4K_INDEX && end - start > icache_size))
 194                 r4k_blast_icache();
 195         else {
 196                 switch (boot_cpu_type()) {
 197 @@ -687,23 +729,59 @@ static inline void local_r4k_flush_icach
 198  #endif
 199  }
 200
 201 +static inline void local_r4k_flush_icache_range(unsigned long start,
 202 +                                               unsigned long end)
 203 +{
 204 +       __local_r4k_flush_icache_range(start, end, R4K_KERN | R4K_INDEX);
 205 +}
 206 +
 207  static inline void local_r4k_flush_icache_range_ipi(void *args)
 208  {
 209         struct flush_icache_range_args *fir_args = args;
 210         unsigned long start = fir_args->start;
 211         unsigned long end = fir_args->end;
 212 +       unsigned int type = fir_args->type;
 213
 214 -       local_r4k_flush_icache_range(start, end);
 215 +       __local_r4k_flush_icache_range(start, end, type);
 216  }
 217
 218  static void r4k_flush_icache_range(unsigned long start, unsigned long end)
 219  {
 220         struct flush_icache_range_args args;
 221 +       unsigned long size, cache_size;
 222
 223         args.start = start;
 224         args.end = end;
 225 +       args.type = R4K_KERN | R4K_INDEX;
 226
 227 -       r4k_on_each_cpu(local_r4k_flush_icache_range_ipi, &args);
 228 +       if (in_atomic()) {
 229 +               /*
 230 +                * We can't do blocking IPI calls from atomic context, so fall
 231 +                * back to pure address-based cache ops if they globalize.
 232 +                */
 233 +               if (!r4k_index_globalized && r4k_hit_globalized) {
 234 +                       args.type &= ~R4K_INDEX;
 235 +               } else {
 236 +                       /* Just do it locally instead. */
 237 +                       local_r4k_flush_icache_range(start, end);
 238 +                       instruction_hazard();
 239 +                       return;
 240 +               }
 241 +       } else if (!r4k_index_globalized && r4k_hit_globalized) {
 242 +               /*
 243 +                * If address-based cache ops are globalized, then we may be
 244 +                * able to avoid the IPI for small flushes.
 245 +                */
 246 +               size = start - end;
 247 +               cache_size = icache_size;
 248 +               if (!cpu_has_ic_fills_f_dc) {
 249 +                       size *= 2;
 250 +                       cache_size += dcache_size;
 251 +               }
 252 +               if (size <= cache_size)
 253 +                       args.type &= ~R4K_INDEX;
 254 +       }
 255 +       r4k_on_each_cpu(args.type, local_r4k_flush_icache_range_ipi, &args);
 256         instruction_hazard();
 257  }
 258
 259 @@ -823,7 +901,12 @@ static void local_r4k_flush_cache_sigtra
 260
 261  static void r4k_flush_cache_sigtramp(unsigned long addr)
 262  {
 263 -       r4k_on_each_cpu(local_r4k_flush_cache_sigtramp, (void *) addr);
 264 +       /*
 265 +        * FIXME this is a bit broken when !r4k_hit_globalized, since the user
 266 +        * code probably won't be mapped on other CPUs, so if the process is
 267 +        * migrated, it could end up hitting stale icache lines.
 268 +        */
 269 +       r4k_on_each_cpu(R4K_USER, local_r4k_flush_cache_sigtramp, (void *)addr);
 270  }
 271
 272  static void r4k_flush_icache_all(void)
 273 @@ -837,6 +920,15 @@ struct flush_kernel_vmap_range_args {
 274         int             size;
 275  };
 276
 277 +static inline void local_r4k_flush_kernel_vmap_range_index(void *args)
 278 +{
 279 +       /*
 280 +        * Aliases only affect the primary caches so don't bother with
 281 +        * S-caches or T-caches.
 282 +        */
 283 +       r4k_blast_dcache();
 284 +}
 285 +
 286  static inline void local_r4k_flush_kernel_vmap_range(void *args)
 287  {
 288         struct flush_kernel_vmap_range_args *vmra = args;
 289 @@ -847,12 +939,8 @@ static inline void local_r4k_flush_kerne
 290          * Aliases only affect the primary caches so don't bother with
 291          * S-caches or T-caches.
 292          */
 293 -       if (cpu_has_safe_index_cacheops && size >= dcache_size)
 294 -               r4k_blast_dcache();
 295 -       else {
 296 -               R4600_HIT_CACHEOP_WAR_IMPL;
 297 -               blast_dcache_range(vaddr, vaddr + size);
 298 -       }
 299 +       R4600_HIT_CACHEOP_WAR_IMPL;
 300 +       blast_dcache_range(vaddr, vaddr + size);
 301  }
 302
 303  static void r4k_flush_kernel_vmap_range(unsigned long vaddr, int size)
 304 @@ -862,7 +950,12 @@ static void r4k_flush_kernel_vmap_range(
 305         args.vaddr = (unsigned long) vaddr;
 306         args.size = size;
 307
 308 -       r4k_on_each_cpu(local_r4k_flush_kernel_vmap_range, &args);
 309 +       if (cpu_has_safe_index_cacheops && size >= dcache_size)
 310 +               r4k_on_each_cpu(R4K_INDEX,
 311 +                               local_r4k_flush_kernel_vmap_range_index, NULL);
 312 +       else
 313 +               r4k_on_each_cpu(R4K_KERN, local_r4k_flush_kernel_vmap_range,
 314 +                               &args);
 315  }
 316
 317  static inline void rm7k_erratum31(void)