1/* 2 * Cache maintenance 3 * 4 * Copyright (C) 2001 Deep Blue Solutions Ltd. 5 * Copyright (C) 2012 ARM Ltd. 6 * 7 * This program is free software; you can redistribute it and/or modify 8 * it under the terms of the GNU General Public License version 2 as 9 * published by the Free Software Foundation. 10 * 11 * This program is distributed in the hope that it will be useful, 12 * but WITHOUT ANY WARRANTY; without even the implied warranty of 13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 14 * GNU General Public License for more details. 15 * 16 * You should have received a copy of the GNU General Public License 17 * along with this program. If not, see <http://www.gnu.org/licenses/>. 18 */ 19 20#include <linux/linkage.h> 21#include <linux/init.h> 22#include <asm/assembler.h> 23#include <asm/cpufeature.h> 24#include <asm/alternative.h> 25#include <asm/uaccess.h> 26 27/* 28 * flush_icache_range(start,end) 29 * 30 * Ensure that the I and D caches are coherent within specified region. 31 * This is typically used when code has been written to a memory region, 32 * and will be executed. 33 * 34 * - start - virtual start address of region 35 * - end - virtual end address of region 36 */ 37ENTRY(flush_icache_range) 38 /* FALLTHROUGH */ 39 40/* 41 * __flush_cache_user_range(start,end) 42 * 43 * Ensure that the I and D caches are coherent within specified region. 44 * This is typically used when code has been written to a memory region, 45 * and will be executed. 46 * 47 * - start - virtual start address of region 48 * - end - virtual end address of region 49 */ 50ENTRY(__flush_cache_user_range) 51 uaccess_ttbr0_enable x2, x3, x4 52 dcache_line_size x2, x3 53 sub x3, x2, #1 54 bic x4, x0, x3 551: 56USER(9f, dc cvau, x4 ) // clean D line to PoU 57 add x4, x4, x2 58 cmp x4, x1 59 b.lo 1b 60 dsb ish 61 62 icache_line_size x2, x3 63 sub x3, x2, #1 64 bic x4, x0, x3 651: 66USER(9f, ic ivau, x4 ) // invalidate I line PoU 67 add x4, x4, x2 68 cmp x4, x1 69 b.lo 1b 709: // ignore any faulting cache operation 71 dsb ish 72 isb 73 uaccess_ttbr0_disable x1, x2 74 ret 75ENDPROC(flush_icache_range) 76ENDPROC(__flush_cache_user_range) 77 78/* 79 * __flush_dcache_area(kaddr, size) 80 * 81 * Ensure that the data held in the page kaddr is written back to the 82 * page in question. 83 * 84 * - kaddr - kernel address 85 * - size - size in question 86 */ 87ENTRY(__flush_dcache_area) 88 dcache_line_size x2, x3 89 add x1, x0, x1 90 sub x3, x2, #1 91 bic x0, x0, x3 921: dc civac, x0 // clean & invalidate D line / unified line 93 add x0, x0, x2 94 cmp x0, x1 95 b.lo 1b 96 dsb sy 97 ret 98ENDPIPROC(__flush_dcache_area) 99 100/* 101 * __inval_cache_range(start, end) 102 * - start - start address of region 103 * - end - end address of region 104 */ 105ENTRY(__inval_cache_range) 106 /* FALLTHROUGH */ 107 108/* 109 * __dma_inv_range(start, end) 110 * - start - virtual start address of region 111 * - end - virtual end address of region 112 */ 113__dma_inv_range: 114 dcache_line_size x2, x3 115 sub x3, x2, #1 116 tst x1, x3 // end cache line aligned? 117 bic x1, x1, x3 118 b.eq 1f 119 dc civac, x1 // clean & invalidate D / U line 1201: tst x0, x3 // start cache line aligned? 121 bic x0, x0, x3 122 b.eq 2f 123 dc civac, x0 // clean & invalidate D / U line 124 b 3f 1252: dc ivac, x0 // invalidate D / U line 1263: add x0, x0, x2 127 cmp x0, x1 128 b.lo 2b 129 dsb sy 130 ret 131ENDPIPROC(__inval_cache_range) 132ENDPROC(__dma_inv_range) 133 134/* 135 * __dma_clean_range(start, end) 136 * - start - virtual start address of region 137 * - end - virtual end address of region 138 */ 139__dma_clean_range: 140 dcache_line_size x2, x3 141 sub x3, x2, #1 142 bic x0, x0, x3 1431: 144alternative_if_not ARM64_WORKAROUND_CLEAN_CACHE 145 dc cvac, x0 146alternative_else 147 dc civac, x0 148alternative_endif 149 add x0, x0, x2 150 cmp x0, x1 151 b.lo 1b 152 dsb sy 153 ret 154ENDPROC(__dma_clean_range) 155 156/* 157 * __dma_flush_range(start, end) 158 * - start - virtual start address of region 159 * - end - virtual end address of region 160 */ 161ENTRY(__dma_flush_range) 162 dcache_line_size x2, x3 163 sub x3, x2, #1 164 bic x0, x0, x3 1651: dc civac, x0 // clean & invalidate D / U line 166 add x0, x0, x2 167 cmp x0, x1 168 b.lo 1b 169 dsb sy 170 ret 171ENDPIPROC(__dma_flush_range) 172 173/* 174 * __dma_map_area(start, size, dir) 175 * - start - kernel virtual start address 176 * - size - size of region 177 * - dir - DMA direction 178 */ 179ENTRY(__dma_map_area) 180 add x1, x1, x0 181 cmp w2, #DMA_FROM_DEVICE 182 b.eq __dma_inv_range 183 b __dma_clean_range 184ENDPIPROC(__dma_map_area) 185 186/* 187 * __dma_unmap_area(start, size, dir) 188 * - start - kernel virtual start address 189 * - size - size of region 190 * - dir - DMA direction 191 */ 192ENTRY(__dma_unmap_area) 193 add x1, x1, x0 194 cmp w2, #DMA_TO_DEVICE 195 b.ne __dma_inv_range 196 ret 197ENDPIPROC(__dma_unmap_area) 198