MIPS: Add implementation of dma_map_ops.mmap()
[linux-drm-fsl-dcu.git] / arch / mips / mm / dma-default.c
1 /*
2  * This file is subject to the terms and conditions of the GNU General Public
3  * License.  See the file "COPYING" in the main directory of this archive
4  * for more details.
5  *
6  * Copyright (C) 2000  Ani Joshi <ajoshi@unixbox.com>
7  * Copyright (C) 2000, 2001, 06  Ralf Baechle <ralf@linux-mips.org>
8  * swiped from i386, and cloned for MIPS by Geert, polished by Ralf.
9  */
10
11 #include <linux/types.h>
12 #include <linux/dma-mapping.h>
13 #include <linux/mm.h>
14 #include <linux/module.h>
15 #include <linux/scatterlist.h>
16 #include <linux/string.h>
17 #include <linux/gfp.h>
18 #include <linux/highmem.h>
19 #include <linux/dma-contiguous.h>
20
21 #include <asm/cache.h>
22 #include <asm/cpu-type.h>
23 #include <asm/io.h>
24
25 #include <dma-coherence.h>
26
27 #ifdef CONFIG_DMA_MAYBE_COHERENT
28 int coherentio = 0;     /* User defined DMA coherency from command line. */
29 EXPORT_SYMBOL_GPL(coherentio);
30 int hw_coherentio = 0;  /* Actual hardware supported DMA coherency setting. */
31
32 static int __init setcoherentio(char *str)
33 {
34         coherentio = 1;
35         pr_info("Hardware DMA cache coherency (command line)\n");
36         return 0;
37 }
38 early_param("coherentio", setcoherentio);
39
40 static int __init setnocoherentio(char *str)
41 {
42         coherentio = 0;
43         pr_info("Software DMA cache coherency (command line)\n");
44         return 0;
45 }
46 early_param("nocoherentio", setnocoherentio);
47 #endif
48
49 static inline struct page *dma_addr_to_page(struct device *dev,
50         dma_addr_t dma_addr)
51 {
52         return pfn_to_page(
53                 plat_dma_addr_to_phys(dev, dma_addr) >> PAGE_SHIFT);
54 }
55
56 /*
57  * The affected CPUs below in 'cpu_needs_post_dma_flush()' can
58  * speculatively fill random cachelines with stale data at any time,
59  * requiring an extra flush post-DMA.
60  *
61  * Warning on the terminology - Linux calls an uncached area coherent;
62  * MIPS terminology calls memory areas with hardware maintained coherency
63  * coherent.
64  *
65  * Note that the R14000 and R16000 should also be checked for in this
66  * condition.  However this function is only called on non-I/O-coherent
67  * systems and only the R10000 and R12000 are used in such systems, the
68  * SGI IP28 Indigo¬≤ rsp. SGI IP32 aka O2.
69  */
70 static inline int cpu_needs_post_dma_flush(struct device *dev)
71 {
72         return !plat_device_is_coherent(dev) &&
73                (boot_cpu_type() == CPU_R10000 ||
74                 boot_cpu_type() == CPU_R12000 ||
75                 boot_cpu_type() == CPU_BMIPS5000);
76 }
77
78 static gfp_t massage_gfp_flags(const struct device *dev, gfp_t gfp)
79 {
80         gfp_t dma_flag;
81
82         /* ignore region specifiers */
83         gfp &= ~(__GFP_DMA | __GFP_DMA32 | __GFP_HIGHMEM);
84
85 #ifdef CONFIG_ISA
86         if (dev == NULL)
87                 dma_flag = __GFP_DMA;
88         else
89 #endif
90 #if defined(CONFIG_ZONE_DMA32) && defined(CONFIG_ZONE_DMA)
91              if (dev->coherent_dma_mask < DMA_BIT_MASK(32))
92                         dma_flag = __GFP_DMA;
93         else if (dev->coherent_dma_mask < DMA_BIT_MASK(64))
94                         dma_flag = __GFP_DMA32;
95         else
96 #endif
97 #if defined(CONFIG_ZONE_DMA32) && !defined(CONFIG_ZONE_DMA)
98              if (dev->coherent_dma_mask < DMA_BIT_MASK(64))
99                 dma_flag = __GFP_DMA32;
100         else
101 #endif
102 #if defined(CONFIG_ZONE_DMA) && !defined(CONFIG_ZONE_DMA32)
103              if (dev->coherent_dma_mask < DMA_BIT_MASK(64))
104                 dma_flag = __GFP_DMA;
105         else
106 #endif
107                 dma_flag = 0;
108
109         /* Don't invoke OOM killer */
110         gfp |= __GFP_NORETRY;
111
112         return gfp | dma_flag;
113 }
114
115 void *dma_alloc_noncoherent(struct device *dev, size_t size,
116         dma_addr_t * dma_handle, gfp_t gfp)
117 {
118         void *ret;
119
120         gfp = massage_gfp_flags(dev, gfp);
121
122         ret = (void *) __get_free_pages(gfp, get_order(size));
123
124         if (ret != NULL) {
125                 memset(ret, 0, size);
126                 *dma_handle = plat_map_dma_mem(dev, ret, size);
127         }
128
129         return ret;
130 }
131 EXPORT_SYMBOL(dma_alloc_noncoherent);
132
133 static void *mips_dma_alloc_coherent(struct device *dev, size_t size,
134         dma_addr_t * dma_handle, gfp_t gfp, struct dma_attrs *attrs)
135 {
136         void *ret;
137         struct page *page = NULL;
138         unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT;
139
140         if (dma_alloc_from_coherent(dev, size, dma_handle, &ret))
141                 return ret;
142
143         gfp = massage_gfp_flags(dev, gfp);
144
145         if (IS_ENABLED(CONFIG_DMA_CMA) && !(gfp & GFP_ATOMIC))
146                 page = dma_alloc_from_contiguous(dev,
147                                         count, get_order(size));
148         if (!page)
149                 page = alloc_pages(gfp, get_order(size));
150
151         if (!page)
152                 return NULL;
153
154         ret = page_address(page);
155         memset(ret, 0, size);
156         *dma_handle = plat_map_dma_mem(dev, ret, size);
157         if (!plat_device_is_coherent(dev)) {
158                 dma_cache_wback_inv((unsigned long) ret, size);
159                 if (!hw_coherentio)
160                         ret = UNCAC_ADDR(ret);
161         }
162
163         return ret;
164 }
165
166
167 void dma_free_noncoherent(struct device *dev, size_t size, void *vaddr,
168         dma_addr_t dma_handle)
169 {
170         plat_unmap_dma_mem(dev, dma_handle, size, DMA_BIDIRECTIONAL);
171         free_pages((unsigned long) vaddr, get_order(size));
172 }
173 EXPORT_SYMBOL(dma_free_noncoherent);
174
175 static void mips_dma_free_coherent(struct device *dev, size_t size, void *vaddr,
176         dma_addr_t dma_handle, struct dma_attrs *attrs)
177 {
178         unsigned long addr = (unsigned long) vaddr;
179         int order = get_order(size);
180         unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT;
181         struct page *page = NULL;
182
183         if (dma_release_from_coherent(dev, order, vaddr))
184                 return;
185
186         plat_unmap_dma_mem(dev, dma_handle, size, DMA_BIDIRECTIONAL);
187
188         if (!plat_device_is_coherent(dev) && !hw_coherentio)
189                 addr = CAC_ADDR(addr);
190
191         page = virt_to_page((void *) addr);
192
193         if (!dma_release_from_contiguous(dev, page, count))
194                 __free_pages(page, get_order(size));
195 }
196
197 static int mips_dma_mmap(struct device *dev, struct vm_area_struct *vma,
198         void *cpu_addr, dma_addr_t dma_addr, size_t size,
199         struct dma_attrs *attrs)
200 {
201         unsigned long user_count = (vma->vm_end - vma->vm_start) >> PAGE_SHIFT;
202         unsigned long count = PAGE_ALIGN(size) >> PAGE_SHIFT;
203         unsigned long addr = (unsigned long)cpu_addr;
204         unsigned long off = vma->vm_pgoff;
205         unsigned long pfn;
206         int ret = -ENXIO;
207
208         if (!plat_device_is_coherent(dev) && !hw_coherentio)
209                 addr = CAC_ADDR(addr);
210
211         pfn = page_to_pfn(virt_to_page((void *)addr));
212
213         if (dma_get_attr(DMA_ATTR_WRITE_COMBINE, attrs))
214                 vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot);
215         else
216                 vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
217
218         if (dma_mmap_from_coherent(dev, vma, cpu_addr, size, &ret))
219                 return ret;
220
221         if (off < count && user_count <= (count - off)) {
222                 ret = remap_pfn_range(vma, vma->vm_start,
223                                       pfn + off,
224                                       user_count << PAGE_SHIFT,
225                                       vma->vm_page_prot);
226         }
227
228         return ret;
229 }
230
231 static inline void __dma_sync_virtual(void *addr, size_t size,
232         enum dma_data_direction direction)
233 {
234         switch (direction) {
235         case DMA_TO_DEVICE:
236                 dma_cache_wback((unsigned long)addr, size);
237                 break;
238
239         case DMA_FROM_DEVICE:
240                 dma_cache_inv((unsigned long)addr, size);
241                 break;
242
243         case DMA_BIDIRECTIONAL:
244                 dma_cache_wback_inv((unsigned long)addr, size);
245                 break;
246
247         default:
248                 BUG();
249         }
250 }
251
252 /*
253  * A single sg entry may refer to multiple physically contiguous
254  * pages. But we still need to process highmem pages individually.
255  * If highmem is not configured then the bulk of this loop gets
256  * optimized out.
257  */
258 static inline void __dma_sync(struct page *page,
259         unsigned long offset, size_t size, enum dma_data_direction direction)
260 {
261         size_t left = size;
262
263         do {
264                 size_t len = left;
265
266                 if (PageHighMem(page)) {
267                         void *addr;
268
269                         if (offset + len > PAGE_SIZE) {
270                                 if (offset >= PAGE_SIZE) {
271                                         page += offset >> PAGE_SHIFT;
272                                         offset &= ~PAGE_MASK;
273                                 }
274                                 len = PAGE_SIZE - offset;
275                         }
276
277                         addr = kmap_atomic(page);
278                         __dma_sync_virtual(addr + offset, len, direction);
279                         kunmap_atomic(addr);
280                 } else
281                         __dma_sync_virtual(page_address(page) + offset,
282                                            size, direction);
283                 offset = 0;
284                 page++;
285                 left -= len;
286         } while (left);
287 }
288
289 static void mips_dma_unmap_page(struct device *dev, dma_addr_t dma_addr,
290         size_t size, enum dma_data_direction direction, struct dma_attrs *attrs)
291 {
292         if (cpu_needs_post_dma_flush(dev))
293                 __dma_sync(dma_addr_to_page(dev, dma_addr),
294                            dma_addr & ~PAGE_MASK, size, direction);
295         plat_post_dma_flush(dev);
296         plat_unmap_dma_mem(dev, dma_addr, size, direction);
297 }
298
299 static int mips_dma_map_sg(struct device *dev, struct scatterlist *sglist,
300         int nents, enum dma_data_direction direction, struct dma_attrs *attrs)
301 {
302         int i;
303         struct scatterlist *sg;
304
305         for_each_sg(sglist, sg, nents, i) {
306                 if (!plat_device_is_coherent(dev))
307                         __dma_sync(sg_page(sg), sg->offset, sg->length,
308                                    direction);
309 #ifdef CONFIG_NEED_SG_DMA_LENGTH
310                 sg->dma_length = sg->length;
311 #endif
312                 sg->dma_address = plat_map_dma_mem_page(dev, sg_page(sg)) +
313                                   sg->offset;
314         }
315
316         return nents;
317 }
318
319 static dma_addr_t mips_dma_map_page(struct device *dev, struct page *page,
320         unsigned long offset, size_t size, enum dma_data_direction direction,
321         struct dma_attrs *attrs)
322 {
323         if (!plat_device_is_coherent(dev))
324                 __dma_sync(page, offset, size, direction);
325
326         return plat_map_dma_mem_page(dev, page) + offset;
327 }
328
329 static void mips_dma_unmap_sg(struct device *dev, struct scatterlist *sglist,
330         int nhwentries, enum dma_data_direction direction,
331         struct dma_attrs *attrs)
332 {
333         int i;
334         struct scatterlist *sg;
335
336         for_each_sg(sglist, sg, nhwentries, i) {
337                 if (!plat_device_is_coherent(dev) &&
338                     direction != DMA_TO_DEVICE)
339                         __dma_sync(sg_page(sg), sg->offset, sg->length,
340                                    direction);
341                 plat_unmap_dma_mem(dev, sg->dma_address, sg->length, direction);
342         }
343 }
344
345 static void mips_dma_sync_single_for_cpu(struct device *dev,
346         dma_addr_t dma_handle, size_t size, enum dma_data_direction direction)
347 {
348         if (cpu_needs_post_dma_flush(dev))
349                 __dma_sync(dma_addr_to_page(dev, dma_handle),
350                            dma_handle & ~PAGE_MASK, size, direction);
351         plat_post_dma_flush(dev);
352 }
353
354 static void mips_dma_sync_single_for_device(struct device *dev,
355         dma_addr_t dma_handle, size_t size, enum dma_data_direction direction)
356 {
357         if (!plat_device_is_coherent(dev))
358                 __dma_sync(dma_addr_to_page(dev, dma_handle),
359                            dma_handle & ~PAGE_MASK, size, direction);
360 }
361
362 static void mips_dma_sync_sg_for_cpu(struct device *dev,
363         struct scatterlist *sglist, int nelems,
364         enum dma_data_direction direction)
365 {
366         int i;
367         struct scatterlist *sg;
368
369         if (cpu_needs_post_dma_flush(dev)) {
370                 for_each_sg(sglist, sg, nelems, i) {
371                         __dma_sync(sg_page(sg), sg->offset, sg->length,
372                                    direction);
373                 }
374         }
375         plat_post_dma_flush(dev);
376 }
377
378 static void mips_dma_sync_sg_for_device(struct device *dev,
379         struct scatterlist *sglist, int nelems,
380         enum dma_data_direction direction)
381 {
382         int i;
383         struct scatterlist *sg;
384
385         if (!plat_device_is_coherent(dev)) {
386                 for_each_sg(sglist, sg, nelems, i) {
387                         __dma_sync(sg_page(sg), sg->offset, sg->length,
388                                    direction);
389                 }
390         }
391 }
392
393 int mips_dma_mapping_error(struct device *dev, dma_addr_t dma_addr)
394 {
395         return 0;
396 }
397
398 int mips_dma_supported(struct device *dev, u64 mask)
399 {
400         return plat_dma_supported(dev, mask);
401 }
402
403 void dma_cache_sync(struct device *dev, void *vaddr, size_t size,
404                          enum dma_data_direction direction)
405 {
406         BUG_ON(direction == DMA_NONE);
407
408         if (!plat_device_is_coherent(dev))
409                 __dma_sync_virtual(vaddr, size, direction);
410 }
411
412 EXPORT_SYMBOL(dma_cache_sync);
413
414 static struct dma_map_ops mips_default_dma_map_ops = {
415         .alloc = mips_dma_alloc_coherent,
416         .free = mips_dma_free_coherent,
417         .mmap = mips_dma_mmap,
418         .map_page = mips_dma_map_page,
419         .unmap_page = mips_dma_unmap_page,
420         .map_sg = mips_dma_map_sg,
421         .unmap_sg = mips_dma_unmap_sg,
422         .sync_single_for_cpu = mips_dma_sync_single_for_cpu,
423         .sync_single_for_device = mips_dma_sync_single_for_device,
424         .sync_sg_for_cpu = mips_dma_sync_sg_for_cpu,
425         .sync_sg_for_device = mips_dma_sync_sg_for_device,
426         .mapping_error = mips_dma_mapping_error,
427         .dma_supported = mips_dma_supported
428 };
429
430 struct dma_map_ops *mips_dma_map_ops = &mips_default_dma_map_ops;
431 EXPORT_SYMBOL(mips_dma_map_ops);
432
433 #define PREALLOC_DMA_DEBUG_ENTRIES (1 << 16)
434
435 static int __init mips_dma_init(void)
436 {
437         dma_debug_init(PREALLOC_DMA_DEBUG_ENTRIES);
438
439         return 0;
440 }
441 fs_initcall(mips_dma_init);