1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright(c) 2017 IBM Corporation. All rights reserved.
4 */
5
6 #include <linux/string.h>
7 #include <linux/export.h>
8 #include <linux/uaccess.h>
9
10 #include <asm/cacheflush.h>
11
__clean_pmem_range(unsigned long start,unsigned long stop)12 static inline void __clean_pmem_range(unsigned long start, unsigned long stop)
13 {
14 unsigned long shift = l1_dcache_shift();
15 unsigned long bytes = l1_dcache_bytes();
16 void *addr = (void *)(start & ~(bytes - 1));
17 unsigned long size = stop - (unsigned long)addr + (bytes - 1);
18 unsigned long i;
19
20 for (i = 0; i < size >> shift; i++, addr += bytes)
21 asm volatile(PPC_DCBSTPS(%0, %1): :"i"(0), "r"(addr): "memory");
22 }
23
__flush_pmem_range(unsigned long start,unsigned long stop)24 static inline void __flush_pmem_range(unsigned long start, unsigned long stop)
25 {
26 unsigned long shift = l1_dcache_shift();
27 unsigned long bytes = l1_dcache_bytes();
28 void *addr = (void *)(start & ~(bytes - 1));
29 unsigned long size = stop - (unsigned long)addr + (bytes - 1);
30 unsigned long i;
31
32 for (i = 0; i < size >> shift; i++, addr += bytes)
33 asm volatile(PPC_DCBFPS(%0, %1): :"i"(0), "r"(addr): "memory");
34 }
35
clean_pmem_range(unsigned long start,unsigned long stop)36 static inline void clean_pmem_range(unsigned long start, unsigned long stop)
37 {
38 if (cpu_has_feature(CPU_FTR_ARCH_207S))
39 return __clean_pmem_range(start, stop);
40 }
41
flush_pmem_range(unsigned long start,unsigned long stop)42 static inline void flush_pmem_range(unsigned long start, unsigned long stop)
43 {
44 if (cpu_has_feature(CPU_FTR_ARCH_207S))
45 return __flush_pmem_range(start, stop);
46 }
47
48 /*
49 * CONFIG_ARCH_HAS_PMEM_API symbols
50 */
arch_wb_cache_pmem(void * addr,size_t size)51 void arch_wb_cache_pmem(void *addr, size_t size)
52 {
53 unsigned long start = (unsigned long) addr;
54 clean_pmem_range(start, start + size);
55 }
56 EXPORT_SYMBOL_GPL(arch_wb_cache_pmem);
57
arch_invalidate_pmem(void * addr,size_t size)58 void arch_invalidate_pmem(void *addr, size_t size)
59 {
60 unsigned long start = (unsigned long) addr;
61 flush_pmem_range(start, start + size);
62 }
63 EXPORT_SYMBOL_GPL(arch_invalidate_pmem);
64
65 /*
66 * CONFIG_ARCH_HAS_UACCESS_FLUSHCACHE symbols
67 */
__copy_from_user_flushcache(void * dest,const void __user * src,unsigned size)68 long __copy_from_user_flushcache(void *dest, const void __user *src,
69 unsigned size)
70 {
71 unsigned long copied, start = (unsigned long) dest;
72
73 copied = __copy_from_user(dest, src, size);
74 clean_pmem_range(start, start + size);
75
76 return copied;
77 }
78
memcpy_flushcache(void * dest,const void * src,size_t size)79 void memcpy_flushcache(void *dest, const void *src, size_t size)
80 {
81 unsigned long start = (unsigned long) dest;
82
83 memcpy(dest, src, size);
84 clean_pmem_range(start, start + size);
85 }
86 EXPORT_SYMBOL(memcpy_flushcache);
87
memcpy_page_flushcache(char * to,struct page * page,size_t offset,size_t len)88 void memcpy_page_flushcache(char *to, struct page *page, size_t offset,
89 size_t len)
90 {
91 memcpy_flushcache(to, page_to_virt(page) + offset, len);
92 }
93 EXPORT_SYMBOL(memcpy_page_flushcache);
94