target/linux/generic/patches-3.10/132-mips_inline_dma_ops.patch

   1 From 2c58080407554e1bac8fd50d23cb02420524caed Mon Sep 17 00:00:00 2001
   2 From: Felix Fietkau <nbd@openwrt.org>
   3 Date: Mon, 12 Aug 2013 12:50:22 +0200
   4 Subject: [PATCH] MIPS: partially inline dma ops
   5
   6 Several DMA ops are no-op on many platforms, and the indirection through
   7 the mips_dma_map_ops function table is causing the compiler to emit
   8 unnecessary code.
   9
  10 Inlining visibly improves network performance in my tests (on a 24Kc
  11 based system), and also slightly reduces code size of a few drivers.
  12
  13 Signed-off-by: Felix Fietkau <nbd@openwrt.org>
  14 ---
  15  arch/mips/Kconfig                   |   4 +
  16  arch/mips/include/asm/dma-mapping.h | 360 +++++++++++++++++++++++++++++++++++-
  17  arch/mips/mm/dma-default.c          | 163 ++--------------
  18  3 files changed, 373 insertions(+), 154 deletions(-)
  19
  20 --- a/arch/mips/Kconfig
  21 +++ b/arch/mips/Kconfig
  22 @@ -1430,6 +1430,7 @@ config CPU_CAVIUM_OCTEON
  23         select LIBFDT
  24         select USE_OF
  25         select USB_EHCI_BIG_ENDIAN_MMIO
  26 +       select SYS_HAS_DMA_OPS
  27         help
  28           The Cavium Octeon processor is a highly integrated chip containing
  29           many ethernet hardware widgets for networking tasks. The processor
  30 @@ -1650,6 +1651,9 @@ config SYS_HAS_CPU_XLR
  31  config SYS_HAS_CPU_XLP
  32         bool
  33
  34 +config SYS_HAS_DMA_OPS
  35 +       bool
  36 +
  37  #
  38  # CPU may reorder R->R, R->W, W->R, W->W
  39  # Reordering beyond LL and SC is handled in WEAK_REORDERING_BEYOND_LLSC
  40 --- a/arch/mips/include/asm/dma-mapping.h
  41 +++ b/arch/mips/include/asm/dma-mapping.h
  42 @@ -1,6 +1,12 @@
  43  #ifndef _ASM_DMA_MAPPING_H
  44  #define _ASM_DMA_MAPPING_H
  45
  46 +#include <linux/kmemcheck.h>
  47 +#include <linux/bug.h>
  48 +#include <linux/scatterlist.h>
  49 +#include <linux/dma-debug.h>
  50 +#include <linux/dma-attrs.h>
  51 +
  52  #include <asm/scatterlist.h>
  53  #include <asm/dma-coherence.h>
  54  #include <asm/cache.h>
  55 @@ -10,14 +16,47 @@
  56  #include <dma-coherence.h>
  57  #endif
  58
  59 -extern struct dma_map_ops *mips_dma_map_ops;
  60 +void __dma_sync(struct page *page, unsigned long offset, size_t size,
  61 +               enum dma_data_direction direction);
  62 +void *mips_dma_alloc_coherent(struct device *dev, size_t size,
  63 +                             dma_addr_t *dma_handle, gfp_t gfp,
  64 +                             struct dma_attrs *attrs);
  65 +void mips_dma_free_coherent(struct device *dev, size_t size, void *vaddr,
  66 +                           dma_addr_t dma_handle, struct dma_attrs *attrs);
  67
  68  static inline struct dma_map_ops *get_dma_ops(struct device *dev)
  69  {
  70 +#ifdef CONFIG_SYS_HAS_DMA_OPS
  71         if (dev && dev->archdata.dma_ops)
  72                 return dev->archdata.dma_ops;
  73         else
  74                 return mips_dma_map_ops;
  75 +#else
  76 +       return NULL;
  77 +#endif
  78 +}
  79 +
  80 +/*
  81 + * Warning on the terminology - Linux calls an uncached area coherent;
  82 + * MIPS terminology calls memory areas with hardware maintained coherency
  83 + * coherent.
  84 + */
  85 +
  86 +static inline int cpu_is_noncoherent_r10000(struct device *dev)
  87 +{
  88 +#ifndef CONFIG_SYS_HAS_CPU_R10000
  89 +       return 0;
  90 +#endif
  91 +       return !plat_device_is_coherent(dev) &&
  92 +              (current_cpu_type() == CPU_R10000 ||
  93 +              current_cpu_type() == CPU_R12000);
  94 +}
  95 +
  96 +static inline struct page *dma_addr_to_page(struct device *dev,
  97 +       dma_addr_t dma_addr)
  98 +{
  99 +       return pfn_to_page(
 100 +               plat_dma_addr_to_phys(dev, dma_addr) >> PAGE_SHIFT);
 101  }
 102
 103  static inline bool dma_capable(struct device *dev, dma_addr_t addr, size_t size)
 104 @@ -30,12 +69,309 @@ static inline bool dma_capable(struct de
 105
 106  static inline void dma_mark_clean(void *addr, size_t size) {}
 107
 108 -#include <asm-generic/dma-mapping-common.h>
 109 +static inline dma_addr_t dma_map_single_attrs(struct device *dev, void *ptr,
 110 +                                             size_t size,
 111 +                                             enum dma_data_direction dir,
 112 +                                             struct dma_attrs *attrs)
 113 +{
 114 +       struct dma_map_ops *ops = get_dma_ops(dev);
 115 +       unsigned long offset = (unsigned long)ptr & ~PAGE_MASK;
 116 +       struct page *page = virt_to_page(ptr);
 117 +       dma_addr_t addr;
 118 +
 119 +       kmemcheck_mark_initialized(ptr, size);
 120 +       BUG_ON(!valid_dma_direction(dir));
 121 +       if (ops) {
 122 +               addr = ops->map_page(dev, page, offset, size, dir, attrs);
 123 +       } else {
 124 +               if (!plat_device_is_coherent(dev))
 125 +                       __dma_sync(page, offset, size, dir);
 126 +
 127 +               addr = plat_map_dma_mem_page(dev, page) + offset;
 128 +       }
 129 +       debug_dma_map_page(dev, page, offset, size, dir, addr, true);
 130 +       return addr;
 131 +}
 132 +
 133 +static inline void dma_unmap_single_attrs(struct device *dev, dma_addr_t addr,
 134 +                                         size_t size,
 135 +                                         enum dma_data_direction dir,
 136 +                                         struct dma_attrs *attrs)
 137 +{
 138 +       struct dma_map_ops *ops = get_dma_ops(dev);
 139 +
 140 +       BUG_ON(!valid_dma_direction(dir));
 141 +       if (ops) {
 142 +               ops->unmap_page(dev, addr, size, dir, attrs);
 143 +       } else {
 144 +               if (cpu_is_noncoherent_r10000(dev))
 145 +                       __dma_sync(dma_addr_to_page(dev, addr),
 146 +                                  addr & ~PAGE_MASK, size, dir);
 147 +
 148 +               plat_unmap_dma_mem(dev, addr, size, dir);
 149 +       }
 150 +       debug_dma_unmap_page(dev, addr, size, dir, true);
 151 +}
 152 +
 153 +static inline int dma_map_sg_attrs(struct device *dev, struct scatterlist *sg,
 154 +                                  int nents, enum dma_data_direction dir,
 155 +                                  struct dma_attrs *attrs)
 156 +{
 157 +       struct dma_map_ops *ops = get_dma_ops(dev);
 158 +       int i, ents;
 159 +       struct scatterlist *s;
 160 +
 161 +       for_each_sg(sg, s, nents, i)
 162 +               kmemcheck_mark_initialized(sg_virt(s), s->length);
 163 +       BUG_ON(!valid_dma_direction(dir));
 164 +       if (ops) {
 165 +               ents = ops->map_sg(dev, sg, nents, dir, attrs);
 166 +       } else {
 167 +               for_each_sg(sg, s, nents, i) {
 168 +                       struct page *page = sg_page(s);
 169 +
 170 +                       if (!plat_device_is_coherent(dev))
 171 +                               __dma_sync(page, s->offset, s->length, dir);
 172 +                       s->dma_address =
 173 +                               plat_map_dma_mem_page(dev, page) + s->offset;
 174 +               }
 175 +               ents = nents;
 176 +       }
 177 +       debug_dma_map_sg(dev, sg, nents, ents, dir);
 178 +
 179 +       return ents;
 180 +}
 181 +
 182 +static inline void dma_unmap_sg_attrs(struct device *dev, struct scatterlist *sg,
 183 +                                     int nents, enum dma_data_direction dir,
 184 +                                     struct dma_attrs *attrs)
 185 +{
 186 +       struct dma_map_ops *ops = get_dma_ops(dev);
 187 +       struct scatterlist *s;
 188 +       int i;
 189 +
 190 +       BUG_ON(!valid_dma_direction(dir));
 191 +       debug_dma_unmap_sg(dev, sg, nents, dir);
 192 +       if (ops) {
 193 +               ops->unmap_sg(dev, sg, nents, dir, attrs);
 194 +               return;
 195 +       }
 196 +
 197 +       for_each_sg(sg, s, nents, i) {
 198 +               if (!plat_device_is_coherent(dev) && dir != DMA_TO_DEVICE)
 199 +                       __dma_sync(sg_page(s), s->offset, s->length, dir);
 200 +               plat_unmap_dma_mem(dev, s->dma_address, s->length, dir);
 201 +       }
 202 +}
 203 +
 204 +static inline dma_addr_t dma_map_page(struct device *dev, struct page *page,
 205 +                                     size_t offset, size_t size,
 206 +                                     enum dma_data_direction dir)
 207 +{
 208 +       struct dma_map_ops *ops = get_dma_ops(dev);
 209 +       dma_addr_t addr;
 210 +
 211 +       kmemcheck_mark_initialized(page_address(page) + offset, size);
 212 +       BUG_ON(!valid_dma_direction(dir));
 213 +       if (ops) {
 214 +               addr = ops->map_page(dev, page, offset, size, dir, NULL);
 215 +       } else {
 216 +               if (!plat_device_is_coherent(dev))
 217 +                       __dma_sync(page, offset, size, dir);
 218 +
 219 +               addr = plat_map_dma_mem_page(dev, page) + offset;
 220 +       }
 221 +       debug_dma_map_page(dev, page, offset, size, dir, addr, false);
 222 +
 223 +       return addr;
 224 +}
 225 +
 226 +static inline void dma_unmap_page(struct device *dev, dma_addr_t addr,
 227 +                                 size_t size, enum dma_data_direction dir)
 228 +{
 229 +       struct dma_map_ops *ops = get_dma_ops(dev);
 230 +
 231 +       BUG_ON(!valid_dma_direction(dir));
 232 +       if (ops) {
 233 +               ops->unmap_page(dev, addr, size, dir, NULL);
 234 +       } else {
 235 +               if (cpu_is_noncoherent_r10000(dev))
 236 +                       __dma_sync(dma_addr_to_page(dev, addr),
 237 +                                  addr & ~PAGE_MASK, size, dir);
 238 +
 239 +               plat_unmap_dma_mem(dev, addr, size, dir);
 240 +       }
 241 +       debug_dma_unmap_page(dev, addr, size, dir, false);
 242 +}
 243 +
 244 +static inline void dma_sync_single_for_cpu(struct device *dev, dma_addr_t addr,
 245 +                                          size_t size,
 246 +                                          enum dma_data_direction dir)
 247 +{
 248 +       struct dma_map_ops *ops = get_dma_ops(dev);
 249 +
 250 +       BUG_ON(!valid_dma_direction(dir));
 251 +       if (ops)
 252 +               ops->sync_single_for_cpu(dev, addr, size, dir);
 253 +       else if (cpu_is_noncoherent_r10000(dev))
 254 +               __dma_sync(dma_addr_to_page(dev, addr),
 255 +                          addr & ~PAGE_MASK, size, dir);
 256 +       debug_dma_sync_single_for_cpu(dev, addr, size, dir);
 257 +}
 258 +
 259 +static inline void dma_sync_single_for_device(struct device *dev,
 260 +                                             dma_addr_t addr, size_t size,
 261 +                                             enum dma_data_direction dir)
 262 +{
 263 +       struct dma_map_ops *ops = get_dma_ops(dev);
 264 +
 265 +       BUG_ON(!valid_dma_direction(dir));
 266 +       if (ops)
 267 +               ops->sync_single_for_device(dev, addr, size, dir);
 268 +       else if (!plat_device_is_coherent(dev))
 269 +               __dma_sync(dma_addr_to_page(dev, addr),
 270 +                          addr & ~PAGE_MASK, size, dir);
 271 +       debug_dma_sync_single_for_device(dev, addr, size, dir);
 272 +}
 273 +
 274 +static inline void dma_sync_single_range_for_cpu(struct device *dev,
 275 +                                                dma_addr_t addr,
 276 +                                                unsigned long offset,
 277 +                                                size_t size,
 278 +                                                enum dma_data_direction dir)
 279 +{
 280 +       const struct dma_map_ops *ops = get_dma_ops(dev);
 281 +
 282 +       BUG_ON(!valid_dma_direction(dir));
 283 +       if (ops)
 284 +               ops->sync_single_for_cpu(dev, addr + offset, size, dir);
 285 +       else if (cpu_is_noncoherent_r10000(dev))
 286 +               __dma_sync(dma_addr_to_page(dev, addr + offset),
 287 +                          (addr + offset) & ~PAGE_MASK, size, dir);
 288 +       debug_dma_sync_single_range_for_cpu(dev, addr, offset, size, dir);
 289 +}
 290 +
 291 +static inline void dma_sync_single_range_for_device(struct device *dev,
 292 +                                                   dma_addr_t addr,
 293 +                                                   unsigned long offset,
 294 +                                                   size_t size,
 295 +                                                   enum dma_data_direction dir)
 296 +{
 297 +       const struct dma_map_ops *ops = get_dma_ops(dev);
 298 +
 299 +       BUG_ON(!valid_dma_direction(dir));
 300 +       if (ops)
 301 +               ops->sync_single_for_device(dev, addr + offset, size, dir);
 302 +       else if (!plat_device_is_coherent(dev))
 303 +               __dma_sync(dma_addr_to_page(dev, addr + offset),
 304 +                          (addr + offset) & ~PAGE_MASK, size, dir);
 305 +       debug_dma_sync_single_range_for_device(dev, addr, offset, size, dir);
 306 +}
 307 +
 308 +static inline void
 309 +dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg,
 310 +                   int nelems, enum dma_data_direction dir)
 311 +{
 312 +       struct dma_map_ops *ops = get_dma_ops(dev);
 313 +       struct scatterlist *s;
 314 +       int i;
 315 +
 316 +       BUG_ON(!valid_dma_direction(dir));
 317 +       if (ops)
 318 +               ops->sync_sg_for_cpu(dev, sg, nelems, dir);
 319 +       else if (cpu_is_noncoherent_r10000(dev)) {
 320 +               for_each_sg(sg, s, nelems, i)
 321 +                       __dma_sync(sg_page(s), s->offset, s->length, dir);
 322 +       }
 323 +       debug_dma_sync_sg_for_cpu(dev, sg, nelems, dir);
 324 +}
 325 +
 326 +static inline void
 327 +dma_sync_sg_for_device(struct device *dev, struct scatterlist *sg,
 328 +                      int nelems, enum dma_data_direction dir)
 329 +{
 330 +       struct dma_map_ops *ops = get_dma_ops(dev);
 331 +       struct scatterlist *s;
 332 +       int i;
 333 +
 334 +       BUG_ON(!valid_dma_direction(dir));
 335 +       if (ops)
 336 +               ops->sync_sg_for_device(dev, sg, nelems, dir);
 337 +       else if (!plat_device_is_coherent(dev)) {
 338 +               for_each_sg(sg, s, nelems, i)
 339 +                       __dma_sync(sg_page(s), s->offset, s->length, dir);
 340 +       }
 341 +       debug_dma_sync_sg_for_device(dev, sg, nelems, dir);
 342 +
 343 +}
 344 +
 345 +#define dma_map_single(d, a, s, r) dma_map_single_attrs(d, a, s, r, NULL)
 346 +#define dma_unmap_single(d, a, s, r) dma_unmap_single_attrs(d, a, s, r, NULL)
 347 +#define dma_map_sg(d, s, n, r) dma_map_sg_attrs(d, s, n, r, NULL)
 348 +#define dma_unmap_sg(d, s, n, r) dma_unmap_sg_attrs(d, s, n, r, NULL)
 349 +
 350 +extern int dma_common_mmap(struct device *dev, struct vm_area_struct *vma,
 351 +                          void *cpu_addr, dma_addr_t dma_addr, size_t size);
 352 +
 353 +/**
 354 + * dma_mmap_attrs - map a coherent DMA allocation into user space
 355 + * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices
 356 + * @vma: vm_area_struct describing requested user mapping
 357 + * @cpu_addr: kernel CPU-view address returned from dma_alloc_attrs
 358 + * @handle: device-view address returned from dma_alloc_attrs
 359 + * @size: size of memory originally requested in dma_alloc_attrs
 360 + * @attrs: attributes of mapping properties requested in dma_alloc_attrs
 361 + *
 362 + * Map a coherent DMA buffer previously allocated by dma_alloc_attrs
 363 + * into user space.  The coherent DMA buffer must not be freed by the
 364 + * driver until the user space mapping has been released.
 365 + */
 366 +static inline int
 367 +dma_mmap_attrs(struct device *dev, struct vm_area_struct *vma, void *cpu_addr,
 368 +              dma_addr_t dma_addr, size_t size, struct dma_attrs *attrs)
 369 +{
 370 +       struct dma_map_ops *ops = get_dma_ops(dev);
 371 +       BUG_ON(!ops);
 372 +       if (ops && ops->mmap)
 373 +               return ops->mmap(dev, vma, cpu_addr, dma_addr, size, attrs);
 374 +       return dma_common_mmap(dev, vma, cpu_addr, dma_addr, size);
 375 +}
 376 +
 377 +#define dma_mmap_coherent(d, v, c, h, s) dma_mmap_attrs(d, v, c, h, s, NULL)
 378 +
 379 +static inline int dma_mmap_writecombine(struct device *dev, struct vm_area_struct *vma,
 380 +                     void *cpu_addr, dma_addr_t dma_addr, size_t size)
 381 +{
 382 +       DEFINE_DMA_ATTRS(attrs);
 383 +       dma_set_attr(DMA_ATTR_WRITE_COMBINE, &attrs);
 384 +       return dma_mmap_attrs(dev, vma, cpu_addr, dma_addr, size, &attrs);
 385 +}
 386 +
 387 +int
 388 +dma_common_get_sgtable(struct device *dev, struct sg_table *sgt,
 389 +                      void *cpu_addr, dma_addr_t dma_addr, size_t size);
 390 +
 391 +static inline int
 392 +dma_get_sgtable_attrs(struct device *dev, struct sg_table *sgt, void *cpu_addr,
 393 +                     dma_addr_t dma_addr, size_t size, struct dma_attrs *attrs)
 394 +{
 395 +       struct dma_map_ops *ops = get_dma_ops(dev);
 396 +       BUG_ON(!ops);
 397 +       if (ops && ops->get_sgtable)
 398 +               return ops->get_sgtable(dev, sgt, cpu_addr, dma_addr, size,
 399 +                                       attrs);
 400 +       return dma_common_get_sgtable(dev, sgt, cpu_addr, dma_addr, size);
 401 +}
 402 +
 403 +#define dma_get_sgtable(d, t, v, h, s) dma_get_sgtable_attrs(d, t, v, h, s, NULL)
 404 +
 405
 406  static inline int dma_supported(struct device *dev, u64 mask)
 407  {
 408         struct dma_map_ops *ops = get_dma_ops(dev);
 409 -       return ops->dma_supported(dev, mask);
 410 +       if (ops)
 411 +               return ops->dma_supported(dev, mask);
 412 +       return plat_dma_supported(dev, mask);
 413  }
 414
 415  static inline int dma_mapping_error(struct device *dev, u64 mask)
 416 @@ -43,7 +379,9 @@ static inline int dma_mapping_error(stru
 417         struct dma_map_ops *ops = get_dma_ops(dev);
 418
 419         debug_dma_mapping_error(dev, mask);
 420 -       return ops->mapping_error(dev, mask);
 421 +       if (ops)
 422 +               return ops->mapping_error(dev, mask);
 423 +       return 0;
 424  }
 425
 426  static inline int
 427 @@ -69,7 +407,11 @@ static inline void *dma_alloc_attrs(stru
 428         void *ret;
 429         struct dma_map_ops *ops = get_dma_ops(dev);
 430
 431 -       ret = ops->alloc(dev, size, dma_handle, gfp, attrs);
 432 +       if (ops)
 433 +               ret = ops->alloc(dev, size, dma_handle, gfp, attrs);
 434 +       else
 435 +               ret = mips_dma_alloc_coherent(dev, size, dma_handle, gfp,
 436 +                                             attrs);
 437
 438         debug_dma_alloc_coherent(dev, size, *dma_handle, ret);
 439
 440 @@ -84,7 +426,10 @@ static inline void dma_free_attrs(struct
 441  {
 442         struct dma_map_ops *ops = get_dma_ops(dev);
 443
 444 -       ops->free(dev, size, vaddr, dma_handle, attrs);
 445 +       if (ops)
 446 +               ops->free(dev, size, vaddr, dma_handle, attrs);
 447 +       else
 448 +               mips_dma_free_coherent(dev, size, vaddr, dma_handle, attrs);
 449
 450         debug_dma_free_coherent(dev, size, vaddr, dma_handle);
 451  }
 452 --- a/arch/mips/mm/dma-default.c
 453 +++ b/arch/mips/mm/dma-default.c
 454 @@ -24,7 +24,7 @@
 455
 456  #ifdef CONFIG_DMA_MAYBE_COHERENT
 457  int coherentio = 0;    /* User defined DMA coherency from command line. */
 458 -EXPORT_SYMBOL_GPL(coherentio);
 459 +EXPORT_SYMBOL(coherentio);
 460  int hw_coherentio = 0; /* Actual hardware supported DMA coherency setting. */
 461
 462  static int __init setcoherentio(char *str)
 463 @@ -44,26 +44,6 @@ static int __init setnocoherentio(char *
 464  early_param("nocoherentio", setnocoherentio);
 465  #endif
 466
 467 -static inline struct page *dma_addr_to_page(struct device *dev,
 468 -       dma_addr_t dma_addr)
 469 -{
 470 -       return pfn_to_page(
 471 -               plat_dma_addr_to_phys(dev, dma_addr) >> PAGE_SHIFT);
 472 -}
 473 -
 474 -/*
 475 - * Warning on the terminology - Linux calls an uncached area coherent;
 476 - * MIPS terminology calls memory areas with hardware maintained coherency
 477 - * coherent.
 478 - */
 479 -
 480 -static inline int cpu_is_noncoherent_r10000(struct device *dev)
 481 -{
 482 -       return !plat_device_is_coherent(dev) &&
 483 -              (current_cpu_type() == CPU_R10000 ||
 484 -              current_cpu_type() == CPU_R12000);
 485 -}
 486 -
 487  static gfp_t massage_gfp_flags(const struct device *dev, gfp_t gfp)
 488  {
 489         gfp_t dma_flag;
 490 @@ -119,8 +99,9 @@ void *dma_alloc_noncoherent(struct devic
 491  }
 492  EXPORT_SYMBOL(dma_alloc_noncoherent);
 493
 494 -static void *mips_dma_alloc_coherent(struct device *dev, size_t size,
 495 -       dma_addr_t * dma_handle, gfp_t gfp, struct dma_attrs *attrs)
 496 +void *mips_dma_alloc_coherent(struct device *dev, size_t size,
 497 +                             dma_addr_t *dma_handle, gfp_t gfp,
 498 +                             struct dma_attrs *attrs)
 499  {
 500         void *ret;
 501
 502 @@ -144,6 +125,7 @@ static void *mips_dma_alloc_coherent(str
 503
 504         return ret;
 505  }
 506 +EXPORT_SYMBOL(mips_dma_alloc_coherent);
 507
 508
 509  void dma_free_noncoherent(struct device *dev, size_t size, void *vaddr,
 510 @@ -154,8 +136,8 @@ void dma_free_noncoherent(struct device
 511  }
 512  EXPORT_SYMBOL(dma_free_noncoherent);
 513
 514 -static void mips_dma_free_coherent(struct device *dev, size_t size, void *vaddr,
 515 -       dma_addr_t dma_handle, struct dma_attrs *attrs)
 516 +void mips_dma_free_coherent(struct device *dev, size_t size, void *vaddr,
 517 +                           dma_addr_t dma_handle, struct dma_attrs *attrs)
 518  {
 519         unsigned long addr = (unsigned long) vaddr;
 520         int order = get_order(size);
 521 @@ -170,6 +152,7 @@ static void mips_dma_free_coherent(struc
 522
 523         free_pages(addr, get_order(size));
 524  }
 525 +EXPORT_SYMBOL(mips_dma_free_coherent);
 526
 527  static inline void __dma_sync_virtual(void *addr, size_t size,
 528         enum dma_data_direction direction)
 529 @@ -198,8 +181,8 @@ static inline void __dma_sync_virtual(vo
 530   * If highmem is not configured then the bulk of this loop gets
 531   * optimized out.
 532   */
 533 -static inline void __dma_sync(struct page *page,
 534 -       unsigned long offset, size_t size, enum dma_data_direction direction)
 535 +void __dma_sync(struct page *page, unsigned long offset, size_t size,
 536 +               enum dma_data_direction direction)
 537  {
 538         size_t left = size;
 539
 540 @@ -228,109 +211,7 @@ static inline void __dma_sync(struct pag
 541                 left -= len;
 542         } while (left);
 543  }
 544 -
 545 -static void mips_dma_unmap_page(struct device *dev, dma_addr_t dma_addr,
 546 -       size_t size, enum dma_data_direction direction, struct dma_attrs *attrs)
 547 -{
 548 -       if (cpu_is_noncoherent_r10000(dev))
 549 -               __dma_sync(dma_addr_to_page(dev, dma_addr),
 550 -                          dma_addr & ~PAGE_MASK, size, direction);
 551 -
 552 -       plat_unmap_dma_mem(dev, dma_addr, size, direction);
 553 -}
 554 -
 555 -static int mips_dma_map_sg(struct device *dev, struct scatterlist *sg,
 556 -       int nents, enum dma_data_direction direction, struct dma_attrs *attrs)
 557 -{
 558 -       int i;
 559 -
 560 -       for (i = 0; i < nents; i++, sg++) {
 561 -               if (!plat_device_is_coherent(dev))
 562 -                       __dma_sync(sg_page(sg), sg->offset, sg->length,
 563 -                                  direction);
 564 -               sg->dma_address = plat_map_dma_mem_page(dev, sg_page(sg)) +
 565 -                                 sg->offset;
 566 -       }
 567 -
 568 -       return nents;
 569 -}
 570 -
 571 -static dma_addr_t mips_dma_map_page(struct device *dev, struct page *page,
 572 -       unsigned long offset, size_t size, enum dma_data_direction direction,
 573 -       struct dma_attrs *attrs)
 574 -{
 575 -       if (!plat_device_is_coherent(dev))
 576 -               __dma_sync(page, offset, size, direction);
 577 -
 578 -       return plat_map_dma_mem_page(dev, page) + offset;
 579 -}
 580 -
 581 -static void mips_dma_unmap_sg(struct device *dev, struct scatterlist *sg,
 582 -       int nhwentries, enum dma_data_direction direction,
 583 -       struct dma_attrs *attrs)
 584 -{
 585 -       int i;
 586 -
 587 -       for (i = 0; i < nhwentries; i++, sg++) {
 588 -               if (!plat_device_is_coherent(dev) &&
 589 -                   direction != DMA_TO_DEVICE)
 590 -                       __dma_sync(sg_page(sg), sg->offset, sg->length,
 591 -                                  direction);
 592 -               plat_unmap_dma_mem(dev, sg->dma_address, sg->length, direction);
 593 -       }
 594 -}
 595 -
 596 -static void mips_dma_sync_single_for_cpu(struct device *dev,
 597 -       dma_addr_t dma_handle, size_t size, enum dma_data_direction direction)
 598 -{
 599 -       if (cpu_is_noncoherent_r10000(dev))
 600 -               __dma_sync(dma_addr_to_page(dev, dma_handle),
 601 -                          dma_handle & ~PAGE_MASK, size, direction);
 602 -}
 603 -
 604 -static void mips_dma_sync_single_for_device(struct device *dev,
 605 -       dma_addr_t dma_handle, size_t size, enum dma_data_direction direction)
 606 -{
 607 -       if (!plat_device_is_coherent(dev))
 608 -               __dma_sync(dma_addr_to_page(dev, dma_handle),
 609 -                          dma_handle & ~PAGE_MASK, size, direction);
 610 -}
 611 -
 612 -static void mips_dma_sync_sg_for_cpu(struct device *dev,
 613 -       struct scatterlist *sg, int nelems, enum dma_data_direction direction)
 614 -{
 615 -       int i;
 616 -
 617 -       /* Make sure that gcc doesn't leave the empty loop body.  */
 618 -       for (i = 0; i < nelems; i++, sg++) {
 619 -               if (cpu_is_noncoherent_r10000(dev))
 620 -                       __dma_sync(sg_page(sg), sg->offset, sg->length,
 621 -                                  direction);
 622 -       }
 623 -}
 624 -
 625 -static void mips_dma_sync_sg_for_device(struct device *dev,
 626 -       struct scatterlist *sg, int nelems, enum dma_data_direction direction)
 627 -{
 628 -       int i;
 629 -
 630 -       /* Make sure that gcc doesn't leave the empty loop body.  */
 631 -       for (i = 0; i < nelems; i++, sg++) {
 632 -               if (!plat_device_is_coherent(dev))
 633 -                       __dma_sync(sg_page(sg), sg->offset, sg->length,
 634 -                                  direction);
 635 -       }
 636 -}
 637 -
 638 -int mips_dma_mapping_error(struct device *dev, dma_addr_t dma_addr)
 639 -{
 640 -       return 0;
 641 -}
 642 -
 643 -int mips_dma_supported(struct device *dev, u64 mask)
 644 -{
 645 -       return plat_dma_supported(dev, mask);
 646 -}
 647 +EXPORT_SYMBOL(__dma_sync);
 648
 649  void dma_cache_sync(struct device *dev, void *vaddr, size_t size,
 650                          enum dma_data_direction direction)
 651 @@ -343,23 +224,10 @@ void dma_cache_sync(struct device *dev,
 652
 653  EXPORT_SYMBOL(dma_cache_sync);
 654
 655 -static struct dma_map_ops mips_default_dma_map_ops = {
 656 -       .alloc = mips_dma_alloc_coherent,
 657 -       .free = mips_dma_free_coherent,
 658 -       .map_page = mips_dma_map_page,
 659 -       .unmap_page = mips_dma_unmap_page,
 660 -       .map_sg = mips_dma_map_sg,
 661 -       .unmap_sg = mips_dma_unmap_sg,
 662 -       .sync_single_for_cpu = mips_dma_sync_single_for_cpu,
 663 -       .sync_single_for_device = mips_dma_sync_single_for_device,
 664 -       .sync_sg_for_cpu = mips_dma_sync_sg_for_cpu,
 665 -       .sync_sg_for_device = mips_dma_sync_sg_for_device,
 666 -       .mapping_error = mips_dma_mapping_error,
 667 -       .dma_supported = mips_dma_supported
 668 -};
 669 -
 670 -struct dma_map_ops *mips_dma_map_ops = &mips_default_dma_map_ops;
 671 +#ifdef CONFIG_SYS_HAS_DMA_OPS
 672 +struct dma_map_ops *mips_dma_map_ops = NULL;
 673  EXPORT_SYMBOL(mips_dma_map_ops);
 674 +#endif
 675
 676  #define PREALLOC_DMA_DEBUG_ENTRIES (1 << 16)
 677