1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright(c) 2017 IBM Corporation. All rights reserved. 4 */ 5 6 #include <linux/string.h> 7 #include <linux/export.h> 8 #include <linux/uaccess.h> 9 #include <linux/libnvdimm.h> 10 11 #include <asm/cacheflush.h> 12 __clean_pmem_range(unsigned long start,unsigned long stop)13 static inline void __clean_pmem_range(unsigned long start, unsigned long stop) 14 { 15 unsigned long shift = l1_dcache_shift(); 16 unsigned long bytes = l1_dcache_bytes(); 17 void *addr = (void *)(start & ~(bytes - 1)); 18 unsigned long size = stop - (unsigned long)addr + (bytes - 1); 19 unsigned long i; 20 21 for (i = 0; i < size >> shift; i++, addr += bytes) 22 asm volatile(PPC_DCBSTPS(%0, %1): :"i"(0), "r"(addr): "memory"); 23 } 24 __flush_pmem_range(unsigned long start,unsigned long stop)25 static inline void __flush_pmem_range(unsigned long start, unsigned long stop) 26 { 27 unsigned long shift = l1_dcache_shift(); 28 unsigned long bytes = l1_dcache_bytes(); 29 void *addr = (void *)(start & ~(bytes - 1)); 30 unsigned long size = stop - (unsigned long)addr + (bytes - 1); 31 unsigned long i; 32 33 for (i = 0; i < size >> shift; i++, addr += bytes) 34 asm volatile(PPC_DCBFPS(%0, %1): :"i"(0), "r"(addr): "memory"); 35 } 36 clean_pmem_range(unsigned long start,unsigned long stop)37 static inline void clean_pmem_range(unsigned long start, unsigned long stop) 38 { 39 if (cpu_has_feature(CPU_FTR_ARCH_207S)) 40 return __clean_pmem_range(start, stop); 41 } 42 flush_pmem_range(unsigned long start,unsigned long stop)43 static inline void flush_pmem_range(unsigned long start, unsigned long stop) 44 { 45 if (cpu_has_feature(CPU_FTR_ARCH_207S)) 46 return __flush_pmem_range(start, stop); 47 } 48 49 /* 50 * CONFIG_ARCH_HAS_PMEM_API symbols 51 */ arch_wb_cache_pmem(void * addr,size_t size)52 void arch_wb_cache_pmem(void *addr, size_t size) 53 { 54 unsigned long start = (unsigned long) addr; 55 clean_pmem_range(start, start + size); 56 } 57 EXPORT_SYMBOL_GPL(arch_wb_cache_pmem); 58 arch_invalidate_pmem(void * addr,size_t size)59 void arch_invalidate_pmem(void *addr, size_t size) 60 { 61 unsigned long start = (unsigned long) addr; 62 flush_pmem_range(start, start + size); 63 } 64 EXPORT_SYMBOL_GPL(arch_invalidate_pmem); 65 66 /* 67 * CONFIG_ARCH_HAS_UACCESS_FLUSHCACHE symbols 68 */ __copy_from_user_flushcache(void * dest,const void __user * src,unsigned size)69 long __copy_from_user_flushcache(void *dest, const void __user *src, 70 unsigned size) 71 { 72 unsigned long copied, start = (unsigned long) dest; 73 74 copied = __copy_from_user(dest, src, size); 75 clean_pmem_range(start, start + size); 76 77 return copied; 78 } 79 memcpy_flushcache(void * dest,const void * src,size_t size)80 void memcpy_flushcache(void *dest, const void *src, size_t size) 81 { 82 unsigned long start = (unsigned long) dest; 83 84 memcpy(dest, src, size); 85 clean_pmem_range(start, start + size); 86 } 87 EXPORT_SYMBOL(memcpy_flushcache); 88 memcpy_page_flushcache(char * to,struct page * page,size_t offset,size_t len)89 void memcpy_page_flushcache(char *to, struct page *page, size_t offset, 90 size_t len) 91 { 92 memcpy_flushcache(to, page_to_virt(page) + offset, len); 93 } 94 EXPORT_SYMBOL(memcpy_page_flushcache); 95