target/linux/generic/patches-4.1/132-mips_inline_dma_ops.patch

   1 From 2c58080407554e1bac8fd50d23cb02420524caed Mon Sep 17 00:00:00 2001
   2 From: Felix Fietkau <nbd@openwrt.org>
   3 Date: Mon, 12 Aug 2013 12:50:22 +0200
   4 Subject: [PATCH] MIPS: partially inline dma ops
   5
   6 Several DMA ops are no-op on many platforms, and the indirection through
   7 the mips_dma_map_ops function table is causing the compiler to emit
   8 unnecessary code.
   9
  10 Inlining visibly improves network performance in my tests (on a 24Kc
  11 based system), and also slightly reduces code size of a few drivers.
  12
  13 Signed-off-by: Felix Fietkau <nbd@openwrt.org>
  14 ---
  15  arch/mips/Kconfig                   |   4 +
  16  arch/mips/include/asm/dma-mapping.h | 360 +++++++++++++++++++++++++++++++++++-
  17  arch/mips/mm/dma-default.c          | 163 ++--------------
  18  3 files changed, 373 insertions(+), 154 deletions(-)
  19
  20 --- a/arch/mips/Kconfig
  21 +++ b/arch/mips/Kconfig
  22 @@ -1571,6 +1571,7 @@ config CPU_CAVIUM_OCTEON
  23         select CPU_SUPPORTS_HUGEPAGES
  24         select USB_EHCI_BIG_ENDIAN_MMIO
  25         select MIPS_L1_CACHE_SHIFT_7
  26 +       select SYS_HAS_DMA_OPS
  27         help
  28           The Cavium Octeon processor is a highly integrated chip containing
  29           many ethernet hardware widgets for networking tasks. The processor
  30 @@ -1866,6 +1867,9 @@ config MIPS_MALTA_PM
  31         bool
  32         default y
  33
  34 +config SYS_HAS_DMA_OPS
  35 +       bool
  36 +
  37  #
  38  # CPU may reorder R->R, R->W, W->R, W->W
  39  # Reordering beyond LL and SC is handled in WEAK_REORDERING_BEYOND_LLSC
  40 --- a/arch/mips/include/asm/dma-mapping.h
  41 +++ b/arch/mips/include/asm/dma-mapping.h
  42 @@ -1,9 +1,16 @@
  43  #ifndef _ASM_DMA_MAPPING_H
  44  #define _ASM_DMA_MAPPING_H
  45
  46 +#include <linux/kmemcheck.h>
  47 +#include <linux/bug.h>
  48 +#include <linux/scatterlist.h>
  49 +#include <linux/dma-debug.h>
  50 +#include <linux/dma-attrs.h>
  51 +
  52  #include <asm/scatterlist.h>
  53  #include <asm/dma-coherence.h>
  54  #include <asm/cache.h>
  55 +#include <asm/cpu-type.h>
  56  #include <asm-generic/dma-coherent.h>
  57
  58  #ifndef CONFIG_SGI_IP27 /* Kludge to fix 2.6.39 build for IP27 */
  59 @@ -12,12 +19,48 @@
  60
  61  extern struct dma_map_ops *mips_dma_map_ops;
  62
  63 +void __dma_sync(struct page *page, unsigned long offset, size_t size,
  64 +               enum dma_data_direction direction);
  65 +void *mips_dma_alloc_coherent(struct device *dev, size_t size,
  66 +                             dma_addr_t *dma_handle, gfp_t gfp,
  67 +                             struct dma_attrs *attrs);
  68 +void mips_dma_free_coherent(struct device *dev, size_t size, void *vaddr,
  69 +                           dma_addr_t dma_handle, struct dma_attrs *attrs);
  70 +
  71  static inline struct dma_map_ops *get_dma_ops(struct device *dev)
  72  {
  73 +#ifdef CONFIG_SYS_HAS_DMA_OPS
  74         if (dev && dev->archdata.dma_ops)
  75                 return dev->archdata.dma_ops;
  76         else
  77                 return mips_dma_map_ops;
  78 +#else
  79 +       return NULL;
  80 +#endif
  81 +}
  82 +
  83 +/*
  84 + * Warning on the terminology - Linux calls an uncached area coherent;
  85 + * MIPS terminology calls memory areas with hardware maintained coherency
  86 + * coherent.
  87 + */
  88 +
  89 +static inline int cpu_needs_post_dma_flush(struct device *dev)
  90 +{
  91 +#ifndef CONFIG_SYS_HAS_CPU_R10000
  92 +       return 0;
  93 +#endif
  94 +       return !plat_device_is_coherent(dev) &&
  95 +              (boot_cpu_type() == CPU_R10000 ||
  96 +               boot_cpu_type() == CPU_R12000 ||
  97 +               boot_cpu_type() == CPU_BMIPS5000);
  98 +}
  99 +
 100 +static inline struct page *dma_addr_to_page(struct device *dev,
 101 +       dma_addr_t dma_addr)
 102 +{
 103 +       return pfn_to_page(
 104 +               plat_dma_addr_to_phys(dev, dma_addr) >> PAGE_SHIFT);
 105  }
 106
 107  static inline bool dma_capable(struct device *dev, dma_addr_t addr, size_t size)
 108 @@ -30,12 +73,306 @@ static inline bool dma_capable(struct de
 109
 110  static inline void dma_mark_clean(void *addr, size_t size) {}
 111
 112 -#include <asm-generic/dma-mapping-common.h>
 113 +static inline dma_addr_t dma_map_single_attrs(struct device *dev, void *ptr,
 114 +                                             size_t size,
 115 +                                             enum dma_data_direction dir,
 116 +                                             struct dma_attrs *attrs)
 117 +{
 118 +       struct dma_map_ops *ops = get_dma_ops(dev);
 119 +       unsigned long offset = (unsigned long)ptr & ~PAGE_MASK;
 120 +       struct page *page = virt_to_page(ptr);
 121 +       dma_addr_t addr;
 122 +
 123 +       kmemcheck_mark_initialized(ptr, size);
 124 +       BUG_ON(!valid_dma_direction(dir));
 125 +       if (ops) {
 126 +               addr = ops->map_page(dev, page, offset, size, dir, attrs);
 127 +       } else {
 128 +               if (!plat_device_is_coherent(dev))
 129 +                       __dma_sync(page, offset, size, dir);
 130 +
 131 +               addr = plat_map_dma_mem_page(dev, page) + offset;
 132 +       }
 133 +       debug_dma_map_page(dev, page, offset, size, dir, addr, true);
 134 +       return addr;
 135 +}
 136 +
 137 +static inline void dma_unmap_single_attrs(struct device *dev, dma_addr_t addr,
 138 +                                         size_t size,
 139 +                                         enum dma_data_direction dir,
 140 +                                         struct dma_attrs *attrs)
 141 +{
 142 +       struct dma_map_ops *ops = get_dma_ops(dev);
 143 +
 144 +       BUG_ON(!valid_dma_direction(dir));
 145 +       if (ops) {
 146 +               ops->unmap_page(dev, addr, size, dir, attrs);
 147 +       } else {
 148 +               if (cpu_needs_post_dma_flush(dev))
 149 +                       __dma_sync(dma_addr_to_page(dev, addr),
 150 +                                  addr & ~PAGE_MASK, size, dir);
 151 +
 152 +               plat_unmap_dma_mem(dev, addr, size, dir);
 153 +       }
 154 +       debug_dma_unmap_page(dev, addr, size, dir, true);
 155 +}
 156 +
 157 +static inline int dma_map_sg_attrs(struct device *dev, struct scatterlist *sg,
 158 +                                  int nents, enum dma_data_direction dir,
 159 +                                  struct dma_attrs *attrs)
 160 +{
 161 +       struct dma_map_ops *ops = get_dma_ops(dev);
 162 +       int i, ents;
 163 +       struct scatterlist *s;
 164 +
 165 +       for_each_sg(sg, s, nents, i)
 166 +               kmemcheck_mark_initialized(sg_virt(s), s->length);
 167 +       BUG_ON(!valid_dma_direction(dir));
 168 +       if (ops) {
 169 +               ents = ops->map_sg(dev, sg, nents, dir, attrs);
 170 +       } else {
 171 +               for_each_sg(sg, s, nents, i) {
 172 +                       struct page *page = sg_page(s);
 173 +
 174 +                       if (!plat_device_is_coherent(dev))
 175 +                               __dma_sync(page, s->offset, s->length, dir);
 176 +#ifdef CONFIG_NEED_SG_DMA_LENGTH
 177 +                       s->dma_length = s->length;
 178 +#endif
 179 +                       s->dma_address =
 180 +                               plat_map_dma_mem_page(dev, page) + s->offset;
 181 +               }
 182 +               ents = nents;
 183 +       }
 184 +       debug_dma_map_sg(dev, sg, nents, ents, dir);
 185 +
 186 +       return ents;
 187 +}
 188 +
 189 +static inline void dma_unmap_sg_attrs(struct device *dev, struct scatterlist *sg,
 190 +                                     int nents, enum dma_data_direction dir,
 191 +                                     struct dma_attrs *attrs)
 192 +{
 193 +       struct dma_map_ops *ops = get_dma_ops(dev);
 194 +       struct scatterlist *s;
 195 +       int i;
 196 +
 197 +       BUG_ON(!valid_dma_direction(dir));
 198 +       debug_dma_unmap_sg(dev, sg, nents, dir);
 199 +       if (ops) {
 200 +               ops->unmap_sg(dev, sg, nents, dir, attrs);
 201 +               return;
 202 +       }
 203 +
 204 +       for_each_sg(sg, s, nents, i) {
 205 +               if (!plat_device_is_coherent(dev) && dir != DMA_TO_DEVICE)
 206 +                       __dma_sync(sg_page(s), s->offset, s->length, dir);
 207 +               plat_unmap_dma_mem(dev, s->dma_address, s->length, dir);
 208 +       }
 209 +}
 210 +
 211 +static inline dma_addr_t dma_map_page(struct device *dev, struct page *page,
 212 +                                     size_t offset, size_t size,
 213 +                                     enum dma_data_direction dir)
 214 +{
 215 +       struct dma_map_ops *ops = get_dma_ops(dev);
 216 +       dma_addr_t addr;
 217 +
 218 +       kmemcheck_mark_initialized(page_address(page) + offset, size);
 219 +       BUG_ON(!valid_dma_direction(dir));
 220 +       if (ops) {
 221 +               addr = ops->map_page(dev, page, offset, size, dir, NULL);
 222 +       } else {
 223 +               if (!plat_device_is_coherent(dev))
 224 +                       __dma_sync(page, offset, size, dir);
 225 +
 226 +               addr = plat_map_dma_mem_page(dev, page) + offset;
 227 +       }
 228 +       debug_dma_map_page(dev, page, offset, size, dir, addr, false);
 229 +
 230 +       return addr;
 231 +}
 232 +
 233 +static inline void dma_unmap_page(struct device *dev, dma_addr_t addr,
 234 +                                 size_t size, enum dma_data_direction dir)
 235 +{
 236 +       struct dma_map_ops *ops = get_dma_ops(dev);
 237 +
 238 +       BUG_ON(!valid_dma_direction(dir));
 239 +       if (ops) {
 240 +               ops->unmap_page(dev, addr, size, dir, NULL);
 241 +       } else {
 242 +               if (cpu_needs_post_dma_flush(dev))
 243 +                       __dma_sync(dma_addr_to_page(dev, addr),
 244 +                                  addr & ~PAGE_MASK, size, dir);
 245 +               plat_post_dma_flush(dev);
 246 +               plat_unmap_dma_mem(dev, addr, size, dir);
 247 +       }
 248 +       debug_dma_unmap_page(dev, addr, size, dir, false);
 249 +}
 250 +
 251 +static inline void dma_sync_single_for_cpu(struct device *dev, dma_addr_t addr,
 252 +                                          size_t size,
 253 +                                          enum dma_data_direction dir)
 254 +{
 255 +       struct dma_map_ops *ops = get_dma_ops(dev);
 256 +
 257 +       BUG_ON(!valid_dma_direction(dir));
 258 +       if (ops)
 259 +               ops->sync_single_for_cpu(dev, addr, size, dir);
 260 +       else if (cpu_needs_post_dma_flush(dev))
 261 +               __dma_sync(dma_addr_to_page(dev, addr),
 262 +                          addr & ~PAGE_MASK, size, dir);
 263 +       plat_post_dma_flush(dev);
 264 +       debug_dma_sync_single_for_cpu(dev, addr, size, dir);
 265 +}
 266 +
 267 +static inline void dma_sync_single_for_device(struct device *dev,
 268 +                                             dma_addr_t addr, size_t size,
 269 +                                             enum dma_data_direction dir)
 270 +{
 271 +       struct dma_map_ops *ops = get_dma_ops(dev);
 272 +
 273 +       BUG_ON(!valid_dma_direction(dir));
 274 +       if (ops)
 275 +               ops->sync_single_for_device(dev, addr, size, dir);
 276 +       else if (!plat_device_is_coherent(dev))
 277 +               __dma_sync(dma_addr_to_page(dev, addr),
 278 +                          addr & ~PAGE_MASK, size, dir);
 279 +       debug_dma_sync_single_for_device(dev, addr, size, dir);
 280 +}
 281 +
 282 +static inline void dma_sync_single_range_for_cpu(struct device *dev,
 283 +                                                dma_addr_t addr,
 284 +                                                unsigned long offset,
 285 +                                                size_t size,
 286 +                                                enum dma_data_direction dir)
 287 +{
 288 +       const struct dma_map_ops *ops = get_dma_ops(dev);
 289 +
 290 +       BUG_ON(!valid_dma_direction(dir));
 291 +       if (ops)
 292 +               ops->sync_single_for_cpu(dev, addr + offset, size, dir);
 293 +       else if (cpu_needs_post_dma_flush(dev))
 294 +               __dma_sync(dma_addr_to_page(dev, addr + offset),
 295 +                          (addr + offset) & ~PAGE_MASK, size, dir);
 296 +       debug_dma_sync_single_range_for_cpu(dev, addr, offset, size, dir);
 297 +}
 298 +
 299 +static inline void dma_sync_single_range_for_device(struct device *dev,
 300 +                                                   dma_addr_t addr,
 301 +                                                   unsigned long offset,
 302 +                                                   size_t size,
 303 +                                                   enum dma_data_direction dir)
 304 +{
 305 +       const struct dma_map_ops *ops = get_dma_ops(dev);
 306 +
 307 +       BUG_ON(!valid_dma_direction(dir));
 308 +       if (ops)
 309 +               ops->sync_single_for_device(dev, addr + offset, size, dir);
 310 +       else if (!plat_device_is_coherent(dev))
 311 +               __dma_sync(dma_addr_to_page(dev, addr + offset),
 312 +                          (addr + offset) & ~PAGE_MASK, size, dir);
 313 +       debug_dma_sync_single_range_for_device(dev, addr, offset, size, dir);
 314 +}
 315 +
 316 +static inline void
 317 +dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg,
 318 +                   int nelems, enum dma_data_direction dir)
 319 +{
 320 +       struct dma_map_ops *ops = get_dma_ops(dev);
 321 +       struct scatterlist *s;
 322 +       int i;
 323 +
 324 +       BUG_ON(!valid_dma_direction(dir));
 325 +       if (ops)
 326 +               ops->sync_sg_for_cpu(dev, sg, nelems, dir);
 327 +       else if (cpu_needs_post_dma_flush(dev)) {
 328 +               for_each_sg(sg, s, nelems, i)
 329 +                       __dma_sync(sg_page(s), s->offset, s->length, dir);
 330 +       }
 331 +       plat_post_dma_flush(dev);
 332 +       debug_dma_sync_sg_for_cpu(dev, sg, nelems, dir);
 333 +}
 334 +
 335 +static inline void
 336 +dma_sync_sg_for_device(struct device *dev, struct scatterlist *sg,
 337 +                      int nelems, enum dma_data_direction dir)
 338 +{
 339 +       struct dma_map_ops *ops = get_dma_ops(dev);
 340 +       struct scatterlist *s;
 341 +       int i;
 342 +
 343 +       BUG_ON(!valid_dma_direction(dir));
 344 +       if (ops)
 345 +               ops->sync_sg_for_device(dev, sg, nelems, dir);
 346 +       else if (!plat_device_is_coherent(dev)) {
 347 +               for_each_sg(sg, s, nelems, i)
 348 +                       __dma_sync(sg_page(s), s->offset, s->length, dir);
 349 +       }
 350 +       debug_dma_sync_sg_for_device(dev, sg, nelems, dir);
 351 +
 352 +}
 353 +
 354 +#define dma_map_single(d, a, s, r) dma_map_single_attrs(d, a, s, r, NULL)
 355 +#define dma_unmap_single(d, a, s, r) dma_unmap_single_attrs(d, a, s, r, NULL)
 356 +#define dma_map_sg(d, s, n, r) dma_map_sg_attrs(d, s, n, r, NULL)
 357 +#define dma_unmap_sg(d, s, n, r) dma_unmap_sg_attrs(d, s, n, r, NULL)
 358 +
 359 +extern int dma_common_mmap(struct device *dev, struct vm_area_struct *vma,
 360 +                          void *cpu_addr, dma_addr_t dma_addr, size_t size);
 361 +
 362 +/**
 363 + * dma_mmap_attrs - map a coherent DMA allocation into user space
 364 + * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices
 365 + * @vma: vm_area_struct describing requested user mapping
 366 + * @cpu_addr: kernel CPU-view address returned from dma_alloc_attrs
 367 + * @handle: device-view address returned from dma_alloc_attrs
 368 + * @size: size of memory originally requested in dma_alloc_attrs
 369 + * @attrs: attributes of mapping properties requested in dma_alloc_attrs
 370 + *
 371 + * Map a coherent DMA buffer previously allocated by dma_alloc_attrs
 372 + * into user space.  The coherent DMA buffer must not be freed by the
 373 + * driver until the user space mapping has been released.
 374 + */
 375 +static inline int
 376 +dma_mmap_attrs(struct device *dev, struct vm_area_struct *vma, void *cpu_addr,
 377 +              dma_addr_t dma_addr, size_t size, struct dma_attrs *attrs)
 378 +{
 379 +       struct dma_map_ops *ops = get_dma_ops(dev);
 380 +       BUG_ON(!ops);
 381 +       if (ops && ops->mmap)
 382 +               return ops->mmap(dev, vma, cpu_addr, dma_addr, size, attrs);
 383 +       return dma_common_mmap(dev, vma, cpu_addr, dma_addr, size);
 384 +}
 385 +
 386 +#define dma_mmap_coherent(d, v, c, h, s) dma_mmap_attrs(d, v, c, h, s, NULL)
 387 +
 388 +int
 389 +dma_common_get_sgtable(struct device *dev, struct sg_table *sgt,
 390 +                      void *cpu_addr, dma_addr_t dma_addr, size_t size);
 391 +
 392 +static inline int
 393 +dma_get_sgtable_attrs(struct device *dev, struct sg_table *sgt, void *cpu_addr,
 394 +                     dma_addr_t dma_addr, size_t size, struct dma_attrs *attrs)
 395 +{
 396 +       struct dma_map_ops *ops = get_dma_ops(dev);
 397 +       BUG_ON(!ops);
 398 +       if (ops && ops->get_sgtable)
 399 +               return ops->get_sgtable(dev, sgt, cpu_addr, dma_addr, size,
 400 +                                       attrs);
 401 +       return dma_common_get_sgtable(dev, sgt, cpu_addr, dma_addr, size);
 402 +}
 403 +
 404 +#define dma_get_sgtable(d, t, v, h, s) dma_get_sgtable_attrs(d, t, v, h, s, NULL)
 405 +
 406
 407  static inline int dma_supported(struct device *dev, u64 mask)
 408  {
 409         struct dma_map_ops *ops = get_dma_ops(dev);
 410 -       return ops->dma_supported(dev, mask);
 411 +       if (ops)
 412 +               return ops->dma_supported(dev, mask);
 413 +       return plat_dma_supported(dev, mask);
 414  }
 415
 416  static inline int dma_mapping_error(struct device *dev, u64 mask)
 417 @@ -43,7 +380,9 @@ static inline int dma_mapping_error(stru
 418         struct dma_map_ops *ops = get_dma_ops(dev);
 419
 420         debug_dma_mapping_error(dev, mask);
 421 -       return ops->mapping_error(dev, mask);
 422 +       if (ops)
 423 +               return ops->mapping_error(dev, mask);
 424 +       return 0;
 425  }
 426
 427  static inline int
 428 @@ -54,7 +393,7 @@ dma_set_mask(struct device *dev, u64 mas
 429         if(!dev->dma_mask || !dma_supported(dev, mask))
 430                 return -EIO;
 431
 432 -       if (ops->set_dma_mask)
 433 +       if (ops && ops->set_dma_mask)
 434                 return ops->set_dma_mask(dev, mask);
 435
 436         *dev->dma_mask = mask;
 437 @@ -74,7 +413,11 @@ static inline void *dma_alloc_attrs(stru
 438         void *ret;
 439         struct dma_map_ops *ops = get_dma_ops(dev);
 440
 441 -       ret = ops->alloc(dev, size, dma_handle, gfp, attrs);
 442 +       if (ops)
 443 +               ret = ops->alloc(dev, size, dma_handle, gfp, attrs);
 444 +       else
 445 +               ret = mips_dma_alloc_coherent(dev, size, dma_handle, gfp,
 446 +                                             attrs);
 447
 448         debug_dma_alloc_coherent(dev, size, *dma_handle, ret);
 449
 450 @@ -89,7 +432,10 @@ static inline void dma_free_attrs(struct
 451  {
 452         struct dma_map_ops *ops = get_dma_ops(dev);
 453
 454 -       ops->free(dev, size, vaddr, dma_handle, attrs);
 455 +       if (ops)
 456 +               ops->free(dev, size, vaddr, dma_handle, attrs);
 457 +       else
 458 +               mips_dma_free_coherent(dev, size, vaddr, dma_handle, attrs);
 459
 460         debug_dma_free_coherent(dev, size, vaddr, dma_handle);
 461  }
 462 --- a/arch/mips/mm/dma-default.c
 463 +++ b/arch/mips/mm/dma-default.c
 464 @@ -26,7 +26,7 @@
 465
 466  #ifdef CONFIG_DMA_MAYBE_COHERENT
 467  int coherentio = 0;    /* User defined DMA coherency from command line. */
 468 -EXPORT_SYMBOL_GPL(coherentio);
 469 +EXPORT_SYMBOL(coherentio);
 470  int hw_coherentio = 0; /* Actual hardware supported DMA coherency setting. */
 471
 472  static int __init setcoherentio(char *str)
 473 @@ -46,35 +46,6 @@ static int __init setnocoherentio(char *
 474  early_param("nocoherentio", setnocoherentio);
 475  #endif
 476
 477 -static inline struct page *dma_addr_to_page(struct device *dev,
 478 -       dma_addr_t dma_addr)
 479 -{
 480 -       return pfn_to_page(
 481 -               plat_dma_addr_to_phys(dev, dma_addr) >> PAGE_SHIFT);
 482 -}
 483 -
 484 -/*
 485 - * The affected CPUs below in 'cpu_needs_post_dma_flush()' can
 486 - * speculatively fill random cachelines with stale data at any time,
 487 - * requiring an extra flush post-DMA.
 488 - *
 489 - * Warning on the terminology - Linux calls an uncached area coherent;
 490 - * MIPS terminology calls memory areas with hardware maintained coherency
 491 - * coherent.
 492 - *
 493 - * Note that the R14000 and R16000 should also be checked for in this
 494 - * condition.  However this function is only called on non-I/O-coherent
 495 - * systems and only the R10000 and R12000 are used in such systems, the
 496 - * SGI IP28 Indigo² rsp. SGI IP32 aka O2.
 497 - */
 498 -static inline int cpu_needs_post_dma_flush(struct device *dev)
 499 -{
 500 -       return !plat_device_is_coherent(dev) &&
 501 -              (boot_cpu_type() == CPU_R10000 ||
 502 -               boot_cpu_type() == CPU_R12000 ||
 503 -               boot_cpu_type() == CPU_BMIPS5000);
 504 -}
 505 -
 506  static gfp_t massage_gfp_flags(const struct device *dev, gfp_t gfp)
 507  {
 508         gfp_t dma_flag;
 509 @@ -130,8 +101,9 @@ void *dma_alloc_noncoherent(struct devic
 510  }
 511  EXPORT_SYMBOL(dma_alloc_noncoherent);
 512
 513 -static void *mips_dma_alloc_coherent(struct device *dev, size_t size,
 514 -       dma_addr_t * dma_handle, gfp_t gfp, struct dma_attrs *attrs)
 515 +void *mips_dma_alloc_coherent(struct device *dev, size_t size,
 516 +                             dma_addr_t *dma_handle, gfp_t gfp,
 517 +                             struct dma_attrs *attrs)
 518  {
 519         void *ret;
 520         struct page *page = NULL;
 521 @@ -162,6 +134,7 @@ static void *mips_dma_alloc_coherent(str
 522
 523         return ret;
 524  }
 525 +EXPORT_SYMBOL(mips_dma_alloc_coherent);
 526
 527
 528  void dma_free_noncoherent(struct device *dev, size_t size, void *vaddr,
 529 @@ -172,8 +145,8 @@ void dma_free_noncoherent(struct device
 530  }
 531  EXPORT_SYMBOL(dma_free_noncoherent);
 532
 533 -static void mips_dma_free_coherent(struct device *dev, size_t size, void *vaddr,
 534 -       dma_addr_t dma_handle, struct dma_attrs *attrs)
 535 +void mips_dma_free_coherent(struct device *dev, size_t size, void *vaddr,
 536 +                           dma_addr_t dma_handle, struct dma_attrs *attrs)
 537  {
 538         unsigned long addr = (unsigned long) vaddr;
 539         int order = get_order(size);
 540 @@ -193,6 +166,7 @@ static void mips_dma_free_coherent(struc
 541         if (!dma_release_from_contiguous(dev, page, count))
 542                 __free_pages(page, get_order(size));
 543  }
 544 +EXPORT_SYMBOL(mips_dma_free_coherent);
 545
 546  static inline void __dma_sync_virtual(void *addr, size_t size,
 547         enum dma_data_direction direction)
 548 @@ -221,8 +195,8 @@ static inline void __dma_sync_virtual(vo
 549   * If highmem is not configured then the bulk of this loop gets
 550   * optimized out.
 551   */
 552 -static inline void __dma_sync(struct page *page,
 553 -       unsigned long offset, size_t size, enum dma_data_direction direction)
 554 +void __dma_sync(struct page *page, unsigned long offset, size_t size,
 555 +               enum dma_data_direction direction)
 556  {
 557         size_t left = size;
 558
 559 @@ -251,110 +225,7 @@ static inline void __dma_sync(struct pag
 560                 left -= len;
 561         } while (left);
 562  }
 563 -
 564 -static void mips_dma_unmap_page(struct device *dev, dma_addr_t dma_addr,
 565 -       size_t size, enum dma_data_direction direction, struct dma_attrs *attrs)
 566 -{
 567 -       if (cpu_needs_post_dma_flush(dev))
 568 -               __dma_sync(dma_addr_to_page(dev, dma_addr),
 569 -                          dma_addr & ~PAGE_MASK, size, direction);
 570 -       plat_post_dma_flush(dev);
 571 -       plat_unmap_dma_mem(dev, dma_addr, size, direction);
 572 -}
 573 -
 574 -static int mips_dma_map_sg(struct device *dev, struct scatterlist *sg,
 575 -       int nents, enum dma_data_direction direction, struct dma_attrs *attrs)
 576 -{
 577 -       int i;
 578 -
 579 -       for (i = 0; i < nents; i++, sg++) {
 580 -               if (!plat_device_is_coherent(dev))
 581 -                       __dma_sync(sg_page(sg), sg->offset, sg->length,
 582 -                                  direction);
 583 -#ifdef CONFIG_NEED_SG_DMA_LENGTH
 584 -               sg->dma_length = sg->length;
 585 -#endif
 586 -               sg->dma_address = plat_map_dma_mem_page(dev, sg_page(sg)) +
 587 -                                 sg->offset;
 588 -       }
 589 -
 590 -       return nents;
 591 -}
 592 -
 593 -static dma_addr_t mips_dma_map_page(struct device *dev, struct page *page,
 594 -       unsigned long offset, size_t size, enum dma_data_direction direction,
 595 -       struct dma_attrs *attrs)
 596 -{
 597 -       if (!plat_device_is_coherent(dev))
 598 -               __dma_sync(page, offset, size, direction);
 599 -
 600 -       return plat_map_dma_mem_page(dev, page) + offset;
 601 -}
 602 -
 603 -static void mips_dma_unmap_sg(struct device *dev, struct scatterlist *sg,
 604 -       int nhwentries, enum dma_data_direction direction,
 605 -       struct dma_attrs *attrs)
 606 -{
 607 -       int i;
 608 -
 609 -       for (i = 0; i < nhwentries; i++, sg++) {
 610 -               if (!plat_device_is_coherent(dev) &&
 611 -                   direction != DMA_TO_DEVICE)
 612 -                       __dma_sync(sg_page(sg), sg->offset, sg->length,
 613 -                                  direction);
 614 -               plat_unmap_dma_mem(dev, sg->dma_address, sg->length, direction);
 615 -       }
 616 -}
 617 -
 618 -static void mips_dma_sync_single_for_cpu(struct device *dev,
 619 -       dma_addr_t dma_handle, size_t size, enum dma_data_direction direction)
 620 -{
 621 -       if (cpu_needs_post_dma_flush(dev))
 622 -               __dma_sync(dma_addr_to_page(dev, dma_handle),
 623 -                          dma_handle & ~PAGE_MASK, size, direction);
 624 -       plat_post_dma_flush(dev);
 625 -}
 626 -
 627 -static void mips_dma_sync_single_for_device(struct device *dev,
 628 -       dma_addr_t dma_handle, size_t size, enum dma_data_direction direction)
 629 -{
 630 -       if (!plat_device_is_coherent(dev))
 631 -               __dma_sync(dma_addr_to_page(dev, dma_handle),
 632 -                          dma_handle & ~PAGE_MASK, size, direction);
 633 -}
 634 -
 635 -static void mips_dma_sync_sg_for_cpu(struct device *dev,
 636 -       struct scatterlist *sg, int nelems, enum dma_data_direction direction)
 637 -{
 638 -       int i;
 639 -
 640 -       if (cpu_needs_post_dma_flush(dev))
 641 -               for (i = 0; i < nelems; i++, sg++)
 642 -                       __dma_sync(sg_page(sg), sg->offset, sg->length,
 643 -                                  direction);
 644 -       plat_post_dma_flush(dev);
 645 -}
 646 -
 647 -static void mips_dma_sync_sg_for_device(struct device *dev,
 648 -       struct scatterlist *sg, int nelems, enum dma_data_direction direction)
 649 -{
 650 -       int i;
 651 -
 652 -       if (!plat_device_is_coherent(dev))
 653 -               for (i = 0; i < nelems; i++, sg++)
 654 -                       __dma_sync(sg_page(sg), sg->offset, sg->length,
 655 -                                  direction);
 656 -}
 657 -
 658 -int mips_dma_mapping_error(struct device *dev, dma_addr_t dma_addr)
 659 -{
 660 -       return 0;
 661 -}
 662 -
 663 -int mips_dma_supported(struct device *dev, u64 mask)
 664 -{
 665 -       return plat_dma_supported(dev, mask);
 666 -}
 667 +EXPORT_SYMBOL(__dma_sync);
 668
 669  void dma_cache_sync(struct device *dev, void *vaddr, size_t size,
 670                          enum dma_data_direction direction)
 671 @@ -367,23 +238,10 @@ void dma_cache_sync(struct device *dev,
 672
 673  EXPORT_SYMBOL(dma_cache_sync);
 674
 675 -static struct dma_map_ops mips_default_dma_map_ops = {
 676 -       .alloc = mips_dma_alloc_coherent,
 677 -       .free = mips_dma_free_coherent,
 678 -       .map_page = mips_dma_map_page,
 679 -       .unmap_page = mips_dma_unmap_page,
 680 -       .map_sg = mips_dma_map_sg,
 681 -       .unmap_sg = mips_dma_unmap_sg,
 682 -       .sync_single_for_cpu = mips_dma_sync_single_for_cpu,
 683 -       .sync_single_for_device = mips_dma_sync_single_for_device,
 684 -       .sync_sg_for_cpu = mips_dma_sync_sg_for_cpu,
 685 -       .sync_sg_for_device = mips_dma_sync_sg_for_device,
 686 -       .mapping_error = mips_dma_mapping_error,
 687 -       .dma_supported = mips_dma_supported
 688 -};
 689 -
 690 -struct dma_map_ops *mips_dma_map_ops = &mips_default_dma_map_ops;
 691 +#ifdef CONFIG_SYS_HAS_DMA_OPS
 692 +struct dma_map_ops *mips_dma_map_ops = NULL;
 693  EXPORT_SYMBOL(mips_dma_map_ops);
 694 +#endif
 695
 696  #define PREALLOC_DMA_DEBUG_ENTRIES (1 << 16)
 697