target/linux/generic/patches-3.14/132-mips_inline_dma_ops.patch

   1 From 2c58080407554e1bac8fd50d23cb02420524caed Mon Sep 17 00:00:00 2001
   2 From: Felix Fietkau <nbd@openwrt.org>
   3 Date: Mon, 12 Aug 2013 12:50:22 +0200
   4 Subject: [PATCH] MIPS: partially inline dma ops
   5
   6 Several DMA ops are no-op on many platforms, and the indirection through
   7 the mips_dma_map_ops function table is causing the compiler to emit
   8 unnecessary code.
   9
  10 Inlining visibly improves network performance in my tests (on a 24Kc
  11 based system), and also slightly reduces code size of a few drivers.
  12
  13 Signed-off-by: Felix Fietkau <nbd@openwrt.org>
  14 ---
  15  arch/mips/Kconfig                   |   4 +
  16  arch/mips/include/asm/dma-mapping.h | 360 +++++++++++++++++++++++++++++++++++-
  17  arch/mips/mm/dma-default.c          | 163 ++--------------
  18  3 files changed, 373 insertions(+), 154 deletions(-)
  19
  20 --- a/arch/mips/Kconfig
  21 +++ b/arch/mips/Kconfig
  22 @@ -1621,6 +1621,9 @@ config SYS_HAS_CPU_XLR
  23  config SYS_HAS_CPU_XLP
  24         bool
  25
  26 +config SYS_HAS_DMA_OPS
  27 +       bool
  28 +
  29  #
  30  # CPU may reorder R->R, R->W, W->R, W->W
  31  # Reordering beyond LL and SC is handled in WEAK_REORDERING_BEYOND_LLSC
  32 --- a/arch/mips/include/asm/dma-mapping.h
  33 +++ b/arch/mips/include/asm/dma-mapping.h
  34 @@ -1,9 +1,16 @@
  35  #ifndef _ASM_DMA_MAPPING_H
  36  #define _ASM_DMA_MAPPING_H
  37
  38 +#include <linux/kmemcheck.h>
  39 +#include <linux/bug.h>
  40 +#include <linux/scatterlist.h>
  41 +#include <linux/dma-debug.h>
  42 +#include <linux/dma-attrs.h>
  43 +
  44  #include <asm/scatterlist.h>
  45  #include <asm/dma-coherence.h>
  46  #include <asm/cache.h>
  47 +#include <asm/cpu-type.h>
  48  #include <asm-generic/dma-coherent.h>
  49
  50  #ifndef CONFIG_SGI_IP27 /* Kludge to fix 2.6.39 build for IP27 */
  51 @@ -12,12 +19,48 @@
  52
  53  extern struct dma_map_ops *mips_dma_map_ops;
  54
  55 +void __dma_sync(struct page *page, unsigned long offset, size_t size,
  56 +               enum dma_data_direction direction);
  57 +void *mips_dma_alloc_coherent(struct device *dev, size_t size,
  58 +                             dma_addr_t *dma_handle, gfp_t gfp,
  59 +                             struct dma_attrs *attrs);
  60 +void mips_dma_free_coherent(struct device *dev, size_t size, void *vaddr,
  61 +                           dma_addr_t dma_handle, struct dma_attrs *attrs);
  62 +
  63  static inline struct dma_map_ops *get_dma_ops(struct device *dev)
  64  {
  65 +#ifdef CONFIG_SYS_HAS_DMA_OPS
  66         if (dev && dev->archdata.dma_ops)
  67                 return dev->archdata.dma_ops;
  68         else
  69                 return mips_dma_map_ops;
  70 +#else
  71 +       return NULL;
  72 +#endif
  73 +}
  74 +
  75 +/*
  76 + * Warning on the terminology - Linux calls an uncached area coherent;
  77 + * MIPS terminology calls memory areas with hardware maintained coherency
  78 + * coherent.
  79 + */
  80 +
  81 +static inline int cpu_needs_post_dma_flush(struct device *dev)
  82 +{
  83 +#ifndef CONFIG_SYS_HAS_CPU_R10000
  84 +       return 0;
  85 +#endif
  86 +       return !plat_device_is_coherent(dev) &&
  87 +              (boot_cpu_type() == CPU_R10000 ||
  88 +               boot_cpu_type() == CPU_R12000 ||
  89 +               boot_cpu_type() == CPU_BMIPS5000);
  90 +}
  91 +
  92 +static inline struct page *dma_addr_to_page(struct device *dev,
  93 +       dma_addr_t dma_addr)
  94 +{
  95 +       return pfn_to_page(
  96 +               plat_dma_addr_to_phys(dev, dma_addr) >> PAGE_SHIFT);
  97  }
  98
  99  static inline bool dma_capable(struct device *dev, dma_addr_t addr, size_t size)
 100 @@ -30,12 +73,312 @@ static inline bool dma_capable(struct de
 101
 102  static inline void dma_mark_clean(void *addr, size_t size) {}
 103
 104 -#include <asm-generic/dma-mapping-common.h>
 105 +static inline dma_addr_t dma_map_single_attrs(struct device *dev, void *ptr,
 106 +                                             size_t size,
 107 +                                             enum dma_data_direction dir,
 108 +                                             struct dma_attrs *attrs)
 109 +{
 110 +       struct dma_map_ops *ops = get_dma_ops(dev);
 111 +       unsigned long offset = (unsigned long)ptr & ~PAGE_MASK;
 112 +       struct page *page = virt_to_page(ptr);
 113 +       dma_addr_t addr;
 114 +
 115 +       kmemcheck_mark_initialized(ptr, size);
 116 +       BUG_ON(!valid_dma_direction(dir));
 117 +       if (ops) {
 118 +               addr = ops->map_page(dev, page, offset, size, dir, attrs);
 119 +       } else {
 120 +               if (!plat_device_is_coherent(dev))
 121 +                       __dma_sync(page, offset, size, dir);
 122 +
 123 +               addr = plat_map_dma_mem_page(dev, page) + offset;
 124 +       }
 125 +       debug_dma_map_page(dev, page, offset, size, dir, addr, true);
 126 +       return addr;
 127 +}
 128 +
 129 +static inline void dma_unmap_single_attrs(struct device *dev, dma_addr_t addr,
 130 +                                         size_t size,
 131 +                                         enum dma_data_direction dir,
 132 +                                         struct dma_attrs *attrs)
 133 +{
 134 +       struct dma_map_ops *ops = get_dma_ops(dev);
 135 +
 136 +       BUG_ON(!valid_dma_direction(dir));
 137 +       if (ops) {
 138 +               ops->unmap_page(dev, addr, size, dir, attrs);
 139 +       } else {
 140 +               if (cpu_needs_post_dma_flush(dev))
 141 +                       __dma_sync(dma_addr_to_page(dev, addr),
 142 +                                  addr & ~PAGE_MASK, size, dir);
 143 +
 144 +               plat_unmap_dma_mem(dev, addr, size, dir);
 145 +       }
 146 +       debug_dma_unmap_page(dev, addr, size, dir, true);
 147 +}
 148 +
 149 +static inline int dma_map_sg_attrs(struct device *dev, struct scatterlist *sg,
 150 +                                  int nents, enum dma_data_direction dir,
 151 +                                  struct dma_attrs *attrs)
 152 +{
 153 +       struct dma_map_ops *ops = get_dma_ops(dev);
 154 +       int i, ents;
 155 +       struct scatterlist *s;
 156 +
 157 +       for_each_sg(sg, s, nents, i)
 158 +               kmemcheck_mark_initialized(sg_virt(s), s->length);
 159 +       BUG_ON(!valid_dma_direction(dir));
 160 +       if (ops) {
 161 +               ents = ops->map_sg(dev, sg, nents, dir, attrs);
 162 +       } else {
 163 +               for_each_sg(sg, s, nents, i) {
 164 +                       struct page *page = sg_page(s);
 165 +
 166 +                       if (!plat_device_is_coherent(dev))
 167 +                               __dma_sync(page, s->offset, s->length, dir);
 168 +#ifdef CONFIG_NEED_SG_DMA_LENGTH
 169 +                       s->dma_length = s->length;
 170 +#endif
 171 +                       s->dma_address =
 172 +                               plat_map_dma_mem_page(dev, page) + s->offset;
 173 +               }
 174 +               ents = nents;
 175 +       }
 176 +       debug_dma_map_sg(dev, sg, nents, ents, dir);
 177 +
 178 +       return ents;
 179 +}
 180 +
 181 +static inline void dma_unmap_sg_attrs(struct device *dev, struct scatterlist *sg,
 182 +                                     int nents, enum dma_data_direction dir,
 183 +                                     struct dma_attrs *attrs)
 184 +{
 185 +       struct dma_map_ops *ops = get_dma_ops(dev);
 186 +       struct scatterlist *s;
 187 +       int i;
 188 +
 189 +       BUG_ON(!valid_dma_direction(dir));
 190 +       debug_dma_unmap_sg(dev, sg, nents, dir);
 191 +       if (ops) {
 192 +               ops->unmap_sg(dev, sg, nents, dir, attrs);
 193 +               return;
 194 +       }
 195 +
 196 +       for_each_sg(sg, s, nents, i) {
 197 +               if (!plat_device_is_coherent(dev) && dir != DMA_TO_DEVICE)
 198 +                       __dma_sync(sg_page(s), s->offset, s->length, dir);
 199 +               plat_unmap_dma_mem(dev, s->dma_address, s->length, dir);
 200 +       }
 201 +}
 202 +
 203 +static inline dma_addr_t dma_map_page(struct device *dev, struct page *page,
 204 +                                     size_t offset, size_t size,
 205 +                                     enum dma_data_direction dir)
 206 +{
 207 +       struct dma_map_ops *ops = get_dma_ops(dev);
 208 +       dma_addr_t addr;
 209 +
 210 +       kmemcheck_mark_initialized(page_address(page) + offset, size);
 211 +       BUG_ON(!valid_dma_direction(dir));
 212 +       if (ops) {
 213 +               addr = ops->map_page(dev, page, offset, size, dir, NULL);
 214 +       } else {
 215 +               if (!plat_device_is_coherent(dev))
 216 +                       __dma_sync(page, offset, size, dir);
 217 +
 218 +               addr = plat_map_dma_mem_page(dev, page) + offset;
 219 +       }
 220 +       debug_dma_map_page(dev, page, offset, size, dir, addr, false);
 221 +
 222 +       return addr;
 223 +}
 224 +
 225 +static inline void dma_unmap_page(struct device *dev, dma_addr_t addr,
 226 +                                 size_t size, enum dma_data_direction dir)
 227 +{
 228 +       struct dma_map_ops *ops = get_dma_ops(dev);
 229 +
 230 +       BUG_ON(!valid_dma_direction(dir));
 231 +       if (ops) {
 232 +               ops->unmap_page(dev, addr, size, dir, NULL);
 233 +       } else {
 234 +               if (cpu_needs_post_dma_flush(dev))
 235 +                       __dma_sync(dma_addr_to_page(dev, addr),
 236 +                                  addr & ~PAGE_MASK, size, dir);
 237 +
 238 +               plat_unmap_dma_mem(dev, addr, size, dir);
 239 +       }
 240 +       debug_dma_unmap_page(dev, addr, size, dir, false);
 241 +}
 242 +
 243 +static inline void dma_sync_single_for_cpu(struct device *dev, dma_addr_t addr,
 244 +                                          size_t size,
 245 +                                          enum dma_data_direction dir)
 246 +{
 247 +       struct dma_map_ops *ops = get_dma_ops(dev);
 248 +
 249 +       BUG_ON(!valid_dma_direction(dir));
 250 +       if (ops)
 251 +               ops->sync_single_for_cpu(dev, addr, size, dir);
 252 +       else if (cpu_needs_post_dma_flush(dev))
 253 +               __dma_sync(dma_addr_to_page(dev, addr),
 254 +                          addr & ~PAGE_MASK, size, dir);
 255 +       debug_dma_sync_single_for_cpu(dev, addr, size, dir);
 256 +}
 257 +
 258 +static inline void dma_sync_single_for_device(struct device *dev,
 259 +                                             dma_addr_t addr, size_t size,
 260 +                                             enum dma_data_direction dir)
 261 +{
 262 +       struct dma_map_ops *ops = get_dma_ops(dev);
 263 +
 264 +       BUG_ON(!valid_dma_direction(dir));
 265 +       if (ops)
 266 +               ops->sync_single_for_device(dev, addr, size, dir);
 267 +       else if (!plat_device_is_coherent(dev))
 268 +               __dma_sync(dma_addr_to_page(dev, addr),
 269 +                          addr & ~PAGE_MASK, size, dir);
 270 +       debug_dma_sync_single_for_device(dev, addr, size, dir);
 271 +}
 272 +
 273 +static inline void dma_sync_single_range_for_cpu(struct device *dev,
 274 +                                                dma_addr_t addr,
 275 +                                                unsigned long offset,
 276 +                                                size_t size,
 277 +                                                enum dma_data_direction dir)
 278 +{
 279 +       const struct dma_map_ops *ops = get_dma_ops(dev);
 280 +
 281 +       BUG_ON(!valid_dma_direction(dir));
 282 +       if (ops)
 283 +               ops->sync_single_for_cpu(dev, addr + offset, size, dir);
 284 +       else if (cpu_needs_post_dma_flush(dev))
 285 +               __dma_sync(dma_addr_to_page(dev, addr + offset),
 286 +                          (addr + offset) & ~PAGE_MASK, size, dir);
 287 +       debug_dma_sync_single_range_for_cpu(dev, addr, offset, size, dir);
 288 +}
 289 +
 290 +static inline void dma_sync_single_range_for_device(struct device *dev,
 291 +                                                   dma_addr_t addr,
 292 +                                                   unsigned long offset,
 293 +                                                   size_t size,
 294 +                                                   enum dma_data_direction dir)
 295 +{
 296 +       const struct dma_map_ops *ops = get_dma_ops(dev);
 297 +
 298 +       BUG_ON(!valid_dma_direction(dir));
 299 +       if (ops)
 300 +               ops->sync_single_for_device(dev, addr + offset, size, dir);
 301 +       else if (!plat_device_is_coherent(dev))
 302 +               __dma_sync(dma_addr_to_page(dev, addr + offset),
 303 +                          (addr + offset) & ~PAGE_MASK, size, dir);
 304 +       debug_dma_sync_single_range_for_device(dev, addr, offset, size, dir);
 305 +}
 306 +
 307 +static inline void
 308 +dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg,
 309 +                   int nelems, enum dma_data_direction dir)
 310 +{
 311 +       struct dma_map_ops *ops = get_dma_ops(dev);
 312 +       struct scatterlist *s;
 313 +       int i;
 314 +
 315 +       BUG_ON(!valid_dma_direction(dir));
 316 +       if (ops)
 317 +               ops->sync_sg_for_cpu(dev, sg, nelems, dir);
 318 +       else if (cpu_needs_post_dma_flush(dev)) {
 319 +               for_each_sg(sg, s, nelems, i)
 320 +                       __dma_sync(sg_page(s), s->offset, s->length, dir);
 321 +       }
 322 +       debug_dma_sync_sg_for_cpu(dev, sg, nelems, dir);
 323 +}
 324 +
 325 +static inline void
 326 +dma_sync_sg_for_device(struct device *dev, struct scatterlist *sg,
 327 +                      int nelems, enum dma_data_direction dir)
 328 +{
 329 +       struct dma_map_ops *ops = get_dma_ops(dev);
 330 +       struct scatterlist *s;
 331 +       int i;
 332 +
 333 +       BUG_ON(!valid_dma_direction(dir));
 334 +       if (ops)
 335 +               ops->sync_sg_for_device(dev, sg, nelems, dir);
 336 +       else if (!plat_device_is_coherent(dev)) {
 337 +               for_each_sg(sg, s, nelems, i)
 338 +                       __dma_sync(sg_page(s), s->offset, s->length, dir);
 339 +       }
 340 +       debug_dma_sync_sg_for_device(dev, sg, nelems, dir);
 341 +
 342 +}
 343 +
 344 +#define dma_map_single(d, a, s, r) dma_map_single_attrs(d, a, s, r, NULL)
 345 +#define dma_unmap_single(d, a, s, r) dma_unmap_single_attrs(d, a, s, r, NULL)
 346 +#define dma_map_sg(d, s, n, r) dma_map_sg_attrs(d, s, n, r, NULL)
 347 +#define dma_unmap_sg(d, s, n, r) dma_unmap_sg_attrs(d, s, n, r, NULL)
 348 +
 349 +extern int dma_common_mmap(struct device *dev, struct vm_area_struct *vma,
 350 +                          void *cpu_addr, dma_addr_t dma_addr, size_t size);
 351 +
 352 +/**
 353 + * dma_mmap_attrs - map a coherent DMA allocation into user space
 354 + * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices
 355 + * @vma: vm_area_struct describing requested user mapping
 356 + * @cpu_addr: kernel CPU-view address returned from dma_alloc_attrs
 357 + * @handle: device-view address returned from dma_alloc_attrs
 358 + * @size: size of memory originally requested in dma_alloc_attrs
 359 + * @attrs: attributes of mapping properties requested in dma_alloc_attrs
 360 + *
 361 + * Map a coherent DMA buffer previously allocated by dma_alloc_attrs
 362 + * into user space.  The coherent DMA buffer must not be freed by the
 363 + * driver until the user space mapping has been released.
 364 + */
 365 +static inline int
 366 +dma_mmap_attrs(struct device *dev, struct vm_area_struct *vma, void *cpu_addr,
 367 +              dma_addr_t dma_addr, size_t size, struct dma_attrs *attrs)
 368 +{
 369 +       struct dma_map_ops *ops = get_dma_ops(dev);
 370 +       BUG_ON(!ops);
 371 +       if (ops && ops->mmap)
 372 +               return ops->mmap(dev, vma, cpu_addr, dma_addr, size, attrs);
 373 +       return dma_common_mmap(dev, vma, cpu_addr, dma_addr, size);
 374 +}
 375 +
 376 +#define dma_mmap_coherent(d, v, c, h, s) dma_mmap_attrs(d, v, c, h, s, NULL)
 377 +
 378 +static inline int dma_mmap_writecombine(struct device *dev, struct vm_area_struct *vma,
 379 +                     void *cpu_addr, dma_addr_t dma_addr, size_t size)
 380 +{
 381 +       DEFINE_DMA_ATTRS(attrs);
 382 +       dma_set_attr(DMA_ATTR_WRITE_COMBINE, &attrs);
 383 +       return dma_mmap_attrs(dev, vma, cpu_addr, dma_addr, size, &attrs);
 384 +}
 385 +
 386 +int
 387 +dma_common_get_sgtable(struct device *dev, struct sg_table *sgt,
 388 +                      void *cpu_addr, dma_addr_t dma_addr, size_t size);
 389 +
 390 +static inline int
 391 +dma_get_sgtable_attrs(struct device *dev, struct sg_table *sgt, void *cpu_addr,
 392 +                     dma_addr_t dma_addr, size_t size, struct dma_attrs *attrs)
 393 +{
 394 +       struct dma_map_ops *ops = get_dma_ops(dev);
 395 +       BUG_ON(!ops);
 396 +       if (ops && ops->get_sgtable)
 397 +               return ops->get_sgtable(dev, sgt, cpu_addr, dma_addr, size,
 398 +                                       attrs);
 399 +       return dma_common_get_sgtable(dev, sgt, cpu_addr, dma_addr, size);
 400 +}
 401 +
 402 +#define dma_get_sgtable(d, t, v, h, s) dma_get_sgtable_attrs(d, t, v, h, s, NULL)
 403 +
 404
 405  static inline int dma_supported(struct device *dev, u64 mask)
 406  {
 407         struct dma_map_ops *ops = get_dma_ops(dev);
 408 -       return ops->dma_supported(dev, mask);
 409 +       if (ops)
 410 +               return ops->dma_supported(dev, mask);
 411 +       return plat_dma_supported(dev, mask);
 412  }
 413
 414  static inline int dma_mapping_error(struct device *dev, u64 mask)
 415 @@ -43,7 +386,9 @@ static inline int dma_mapping_error(stru
 416         struct dma_map_ops *ops = get_dma_ops(dev);
 417
 418         debug_dma_mapping_error(dev, mask);
 419 -       return ops->mapping_error(dev, mask);
 420 +       if (ops)
 421 +               return ops->mapping_error(dev, mask);
 422 +       return 0;
 423  }
 424
 425  static inline int
 426 @@ -69,7 +414,11 @@ static inline void *dma_alloc_attrs(stru
 427         void *ret;
 428         struct dma_map_ops *ops = get_dma_ops(dev);
 429
 430 -       ret = ops->alloc(dev, size, dma_handle, gfp, attrs);
 431 +       if (ops)
 432 +               ret = ops->alloc(dev, size, dma_handle, gfp, attrs);
 433 +       else
 434 +               ret = mips_dma_alloc_coherent(dev, size, dma_handle, gfp,
 435 +                                             attrs);
 436
 437         debug_dma_alloc_coherent(dev, size, *dma_handle, ret);
 438
 439 @@ -84,7 +433,10 @@ static inline void dma_free_attrs(struct
 440  {
 441         struct dma_map_ops *ops = get_dma_ops(dev);
 442
 443 -       ops->free(dev, size, vaddr, dma_handle, attrs);
 444 +       if (ops)
 445 +               ops->free(dev, size, vaddr, dma_handle, attrs);
 446 +       else
 447 +               mips_dma_free_coherent(dev, size, vaddr, dma_handle, attrs);
 448
 449         debug_dma_free_coherent(dev, size, vaddr, dma_handle);
 450  }
 451 --- a/arch/mips/mm/dma-default.c
 452 +++ b/arch/mips/mm/dma-default.c
 453 @@ -25,7 +25,7 @@
 454
 455  #ifdef CONFIG_DMA_MAYBE_COHERENT
 456  int coherentio = 0;    /* User defined DMA coherency from command line. */
 457 -EXPORT_SYMBOL_GPL(coherentio);
 458 +EXPORT_SYMBOL(coherentio);
 459  int hw_coherentio = 0; /* Actual hardware supported DMA coherency setting. */
 460
 461  static int __init setcoherentio(char *str)
 462 @@ -45,30 +45,6 @@ static int __init setnocoherentio(char *
 463  early_param("nocoherentio", setnocoherentio);
 464  #endif
 465
 466 -static inline struct page *dma_addr_to_page(struct device *dev,
 467 -       dma_addr_t dma_addr)
 468 -{
 469 -       return pfn_to_page(
 470 -               plat_dma_addr_to_phys(dev, dma_addr) >> PAGE_SHIFT);
 471 -}
 472 -
 473 -/*
 474 - * The affected CPUs below in 'cpu_needs_post_dma_flush()' can
 475 - * speculatively fill random cachelines with stale data at any time,
 476 - * requiring an extra flush post-DMA.
 477 - *
 478 - * Warning on the terminology - Linux calls an uncached area coherent;
 479 - * MIPS terminology calls memory areas with hardware maintained coherency
 480 - * coherent.
 481 - */
 482 -static inline int cpu_needs_post_dma_flush(struct device *dev)
 483 -{
 484 -       return !plat_device_is_coherent(dev) &&
 485 -              (boot_cpu_type() == CPU_R10000 ||
 486 -               boot_cpu_type() == CPU_R12000 ||
 487 -               boot_cpu_type() == CPU_BMIPS5000);
 488 -}
 489 -
 490  static gfp_t massage_gfp_flags(const struct device *dev, gfp_t gfp)
 491  {
 492         gfp_t dma_flag;
 493 @@ -124,8 +100,9 @@ void *dma_alloc_noncoherent(struct devic
 494  }
 495  EXPORT_SYMBOL(dma_alloc_noncoherent);
 496
 497 -static void *mips_dma_alloc_coherent(struct device *dev, size_t size,
 498 -       dma_addr_t * dma_handle, gfp_t gfp, struct dma_attrs *attrs)
 499 +void *mips_dma_alloc_coherent(struct device *dev, size_t size,
 500 +                             dma_addr_t *dma_handle, gfp_t gfp,
 501 +                             struct dma_attrs *attrs)
 502  {
 503         void *ret;
 504
 505 @@ -149,6 +126,7 @@ static void *mips_dma_alloc_coherent(str
 506
 507         return ret;
 508  }
 509 +EXPORT_SYMBOL(mips_dma_alloc_coherent);
 510
 511
 512  void dma_free_noncoherent(struct device *dev, size_t size, void *vaddr,
 513 @@ -159,8 +137,8 @@ void dma_free_noncoherent(struct device
 514  }
 515  EXPORT_SYMBOL(dma_free_noncoherent);
 516
 517 -static void mips_dma_free_coherent(struct device *dev, size_t size, void *vaddr,
 518 -       dma_addr_t dma_handle, struct dma_attrs *attrs)
 519 +void mips_dma_free_coherent(struct device *dev, size_t size, void *vaddr,
 520 +                           dma_addr_t dma_handle, struct dma_attrs *attrs)
 521  {
 522         unsigned long addr = (unsigned long) vaddr;
 523         int order = get_order(size);
 524 @@ -175,6 +153,7 @@ static void mips_dma_free_coherent(struc
 525
 526         free_pages(addr, get_order(size));
 527  }
 528 +EXPORT_SYMBOL(mips_dma_free_coherent);
 529
 530  static inline void __dma_sync_virtual(void *addr, size_t size,
 531         enum dma_data_direction direction)
 532 @@ -203,8 +182,8 @@ static inline void __dma_sync_virtual(vo
 533   * If highmem is not configured then the bulk of this loop gets
 534   * optimized out.
 535   */
 536 -static inline void __dma_sync(struct page *page,
 537 -       unsigned long offset, size_t size, enum dma_data_direction direction)
 538 +void __dma_sync(struct page *page, unsigned long offset, size_t size,
 539 +               enum dma_data_direction direction)
 540  {
 541         size_t left = size;
 542
 543 @@ -233,108 +212,7 @@ static inline void __dma_sync(struct pag
 544                 left -= len;
 545         } while (left);
 546  }
 547 -
 548 -static void mips_dma_unmap_page(struct device *dev, dma_addr_t dma_addr,
 549 -       size_t size, enum dma_data_direction direction, struct dma_attrs *attrs)
 550 -{
 551 -       if (cpu_needs_post_dma_flush(dev))
 552 -               __dma_sync(dma_addr_to_page(dev, dma_addr),
 553 -                          dma_addr & ~PAGE_MASK, size, direction);
 554 -
 555 -       plat_unmap_dma_mem(dev, dma_addr, size, direction);
 556 -}
 557 -
 558 -static int mips_dma_map_sg(struct device *dev, struct scatterlist *sg,
 559 -       int nents, enum dma_data_direction direction, struct dma_attrs *attrs)
 560 -{
 561 -       int i;
 562 -
 563 -       for (i = 0; i < nents; i++, sg++) {
 564 -               if (!plat_device_is_coherent(dev))
 565 -                       __dma_sync(sg_page(sg), sg->offset, sg->length,
 566 -                                  direction);
 567 -#ifdef CONFIG_NEED_SG_DMA_LENGTH
 568 -               sg->dma_length = sg->length;
 569 -#endif
 570 -               sg->dma_address = plat_map_dma_mem_page(dev, sg_page(sg)) +
 571 -                                 sg->offset;
 572 -       }
 573 -
 574 -       return nents;
 575 -}
 576 -
 577 -static dma_addr_t mips_dma_map_page(struct device *dev, struct page *page,
 578 -       unsigned long offset, size_t size, enum dma_data_direction direction,
 579 -       struct dma_attrs *attrs)
 580 -{
 581 -       if (!plat_device_is_coherent(dev))
 582 -               __dma_sync(page, offset, size, direction);
 583 -
 584 -       return plat_map_dma_mem_page(dev, page) + offset;
 585 -}
 586 -
 587 -static void mips_dma_unmap_sg(struct device *dev, struct scatterlist *sg,
 588 -       int nhwentries, enum dma_data_direction direction,
 589 -       struct dma_attrs *attrs)
 590 -{
 591 -       int i;
 592 -
 593 -       for (i = 0; i < nhwentries; i++, sg++) {
 594 -               if (!plat_device_is_coherent(dev) &&
 595 -                   direction != DMA_TO_DEVICE)
 596 -                       __dma_sync(sg_page(sg), sg->offset, sg->length,
 597 -                                  direction);
 598 -               plat_unmap_dma_mem(dev, sg->dma_address, sg->length, direction);
 599 -       }
 600 -}
 601 -
 602 -static void mips_dma_sync_single_for_cpu(struct device *dev,
 603 -       dma_addr_t dma_handle, size_t size, enum dma_data_direction direction)
 604 -{
 605 -       if (cpu_needs_post_dma_flush(dev))
 606 -               __dma_sync(dma_addr_to_page(dev, dma_handle),
 607 -                          dma_handle & ~PAGE_MASK, size, direction);
 608 -}
 609 -
 610 -static void mips_dma_sync_single_for_device(struct device *dev,
 611 -       dma_addr_t dma_handle, size_t size, enum dma_data_direction direction)
 612 -{
 613 -       if (!plat_device_is_coherent(dev))
 614 -               __dma_sync(dma_addr_to_page(dev, dma_handle),
 615 -                          dma_handle & ~PAGE_MASK, size, direction);
 616 -}
 617 -
 618 -static void mips_dma_sync_sg_for_cpu(struct device *dev,
 619 -       struct scatterlist *sg, int nelems, enum dma_data_direction direction)
 620 -{
 621 -       int i;
 622 -
 623 -       if (cpu_needs_post_dma_flush(dev))
 624 -               for (i = 0; i < nelems; i++, sg++)
 625 -                       __dma_sync(sg_page(sg), sg->offset, sg->length,
 626 -                                  direction);
 627 -}
 628 -
 629 -static void mips_dma_sync_sg_for_device(struct device *dev,
 630 -       struct scatterlist *sg, int nelems, enum dma_data_direction direction)
 631 -{
 632 -       int i;
 633 -
 634 -       if (!plat_device_is_coherent(dev))
 635 -               for (i = 0; i < nelems; i++, sg++)
 636 -                       __dma_sync(sg_page(sg), sg->offset, sg->length,
 637 -                                  direction);
 638 -}
 639 -
 640 -int mips_dma_mapping_error(struct device *dev, dma_addr_t dma_addr)
 641 -{
 642 -       return 0;
 643 -}
 644 -
 645 -int mips_dma_supported(struct device *dev, u64 mask)
 646 -{
 647 -       return plat_dma_supported(dev, mask);
 648 -}
 649 +EXPORT_SYMBOL(__dma_sync);
 650
 651  void dma_cache_sync(struct device *dev, void *vaddr, size_t size,
 652                          enum dma_data_direction direction)
 653 @@ -347,23 +225,10 @@ void dma_cache_sync(struct device *dev,
 654
 655  EXPORT_SYMBOL(dma_cache_sync);
 656
 657 -static struct dma_map_ops mips_default_dma_map_ops = {
 658 -       .alloc = mips_dma_alloc_coherent,
 659 -       .free = mips_dma_free_coherent,
 660 -       .map_page = mips_dma_map_page,
 661 -       .unmap_page = mips_dma_unmap_page,
 662 -       .map_sg = mips_dma_map_sg,
 663 -       .unmap_sg = mips_dma_unmap_sg,
 664 -       .sync_single_for_cpu = mips_dma_sync_single_for_cpu,
 665 -       .sync_single_for_device = mips_dma_sync_single_for_device,
 666 -       .sync_sg_for_cpu = mips_dma_sync_sg_for_cpu,
 667 -       .sync_sg_for_device = mips_dma_sync_sg_for_device,
 668 -       .mapping_error = mips_dma_mapping_error,
 669 -       .dma_supported = mips_dma_supported
 670 -};
 671 -
 672 -struct dma_map_ops *mips_dma_map_ops = &mips_default_dma_map_ops;
 673 +#ifdef CONFIG_SYS_HAS_DMA_OPS
 674 +struct dma_map_ops *mips_dma_map_ops = NULL;
 675  EXPORT_SYMBOL(mips_dma_map_ops);
 676 +#endif
 677
 678  #define PREALLOC_DMA_DEBUG_ENTRIES (1 << 16)
 679