1/* SPDX-License-Identifier: GPL-2.0-only */ 2/* 3 * Cache maintenance 4 * 5 * Copyright (C) 2001 Deep Blue Solutions Ltd. 6 * Copyright (C) 2012 ARM Ltd. 7 */ 8 9#include <linux/errno.h> 10#include <linux/linkage.h> 11#include <linux/init.h> 12#include <asm/assembler.h> 13#include <asm/cpufeature.h> 14#include <asm/alternative.h> 15#include <asm/asm-uaccess.h> 16 17/* 18 * caches_clean_inval_pou_macro(start,end) [fixup] 19 * 20 * Ensure that the I and D caches are coherent within specified region. 21 * This is typically used when code has been written to a memory region, 22 * and will be executed. 23 * 24 * - start - virtual start address of region 25 * - end - virtual end address of region 26 * - fixup - optional label to branch to on user fault 27 */ 28.macro caches_clean_inval_pou_macro, fixup 29alternative_if ARM64_HAS_CACHE_IDC 30 dsb ishst 31 b .Ldc_skip_\@ 32alternative_else_nop_endif 33 mov x2, x0 34 mov x3, x1 35 dcache_by_line_op cvau, ish, x2, x3, x4, x5, \fixup 36.Ldc_skip_\@: 37alternative_if ARM64_HAS_CACHE_DIC 38 isb 39 b .Lic_skip_\@ 40alternative_else_nop_endif 41 invalidate_icache_by_line x0, x1, x2, x3, \fixup 42.Lic_skip_\@: 43.endm 44 45/* 46 * caches_clean_inval_pou(start,end) 47 * 48 * Ensure that the I and D caches are coherent within specified region. 49 * This is typically used when code has been written to a memory region, 50 * and will be executed. 51 * 52 * - start - virtual start address of region 53 * - end - virtual end address of region 54 */ 55SYM_FUNC_START(caches_clean_inval_pou) 56 caches_clean_inval_pou_macro 57 ret 58SYM_FUNC_END(caches_clean_inval_pou) 59 60/* 61 * caches_clean_inval_user_pou(start,end) 62 * 63 * Ensure that the I and D caches are coherent within specified region. 64 * This is typically used when code has been written to a memory region, 65 * and will be executed. 66 * 67 * - start - virtual start address of region 68 * - end - virtual end address of region 69 */ 70SYM_FUNC_START(caches_clean_inval_user_pou) 71 uaccess_ttbr0_enable x2, x3, x4 72 73 caches_clean_inval_pou_macro 2f 74 mov x0, xzr 751: 76 uaccess_ttbr0_disable x1, x2 77 ret 782: 79 mov x0, #-EFAULT 80 b 1b 81SYM_FUNC_END(caches_clean_inval_user_pou) 82 83/* 84 * icache_inval_pou(start,end) 85 * 86 * Ensure that the I cache is invalid within specified region. 87 * 88 * - start - virtual start address of region 89 * - end - virtual end address of region 90 */ 91SYM_FUNC_START_PI(icache_inval_pou) 92alternative_if ARM64_HAS_CACHE_DIC 93 isb 94 ret 95alternative_else_nop_endif 96 97 invalidate_icache_by_line x0, x1, x2, x3 98 ret 99SYM_FUNC_END_PI(icache_inval_pou) 100 101/* 102 * dcache_clean_inval_poc(start, end) 103 * 104 * Ensure that any D-cache lines for the interval [start, end) 105 * are cleaned and invalidated to the PoC. 106 * 107 * - start - virtual start address of region 108 * - end - virtual end address of region 109 */ 110SYM_FUNC_START_PI(dcache_clean_inval_poc) 111 dcache_by_line_op civac, sy, x0, x1, x2, x3 112 ret 113SYM_FUNC_END_PI(dcache_clean_inval_poc) 114 115/* 116 * dcache_clean_pou(start, end) 117 * 118 * Ensure that any D-cache lines for the interval [start, end) 119 * are cleaned to the PoU. 120 * 121 * - start - virtual start address of region 122 * - end - virtual end address of region 123 */ 124SYM_FUNC_START(dcache_clean_pou) 125alternative_if ARM64_HAS_CACHE_IDC 126 dsb ishst 127 ret 128alternative_else_nop_endif 129 dcache_by_line_op cvau, ish, x0, x1, x2, x3 130 ret 131SYM_FUNC_END(dcache_clean_pou) 132 133/* 134 * dcache_inval_poc(start, end) 135 * 136 * Ensure that any D-cache lines for the interval [start, end) 137 * are invalidated. Any partial lines at the ends of the interval are 138 * also cleaned to PoC to prevent data loss. 139 * 140 * - start - kernel start address of region 141 * - end - kernel end address of region 142 */ 143SYM_FUNC_START_LOCAL(__dma_inv_area) 144SYM_FUNC_START_PI(dcache_inval_poc) 145 /* FALLTHROUGH */ 146 147/* 148 * __dma_inv_area(start, end) 149 * - start - virtual start address of region 150 * - end - virtual end address of region 151 */ 152 dcache_line_size x2, x3 153 sub x3, x2, #1 154 tst x1, x3 // end cache line aligned? 155 bic x1, x1, x3 156 b.eq 1f 157 dc civac, x1 // clean & invalidate D / U line 1581: tst x0, x3 // start cache line aligned? 159 bic x0, x0, x3 160 b.eq 2f 161 dc civac, x0 // clean & invalidate D / U line 162 b 3f 1632: dc ivac, x0 // invalidate D / U line 1643: add x0, x0, x2 165 cmp x0, x1 166 b.lo 2b 167 dsb sy 168 ret 169SYM_FUNC_END_PI(dcache_inval_poc) 170SYM_FUNC_END(__dma_inv_area) 171 172/* 173 * dcache_clean_poc(start, end) 174 * 175 * Ensure that any D-cache lines for the interval [start, end) 176 * are cleaned to the PoC. 177 * 178 * - start - virtual start address of region 179 * - end - virtual end address of region 180 */ 181SYM_FUNC_START_LOCAL(__dma_clean_area) 182SYM_FUNC_START_PI(dcache_clean_poc) 183 /* FALLTHROUGH */ 184 185/* 186 * __dma_clean_area(start, end) 187 * - start - virtual start address of region 188 * - end - virtual end address of region 189 */ 190 dcache_by_line_op cvac, sy, x0, x1, x2, x3 191 ret 192SYM_FUNC_END_PI(dcache_clean_poc) 193SYM_FUNC_END(__dma_clean_area) 194 195/* 196 * dcache_clean_pop(start, end) 197 * 198 * Ensure that any D-cache lines for the interval [start, end) 199 * are cleaned to the PoP. 200 * 201 * - start - virtual start address of region 202 * - end - virtual end address of region 203 */ 204SYM_FUNC_START_PI(dcache_clean_pop) 205 alternative_if_not ARM64_HAS_DCPOP 206 b dcache_clean_poc 207 alternative_else_nop_endif 208 dcache_by_line_op cvap, sy, x0, x1, x2, x3 209 ret 210SYM_FUNC_END_PI(dcache_clean_pop) 211 212/* 213 * __dma_flush_area(start, size) 214 * 215 * clean & invalidate D / U line 216 * 217 * - start - virtual start address of region 218 * - size - size in question 219 */ 220SYM_FUNC_START_PI(__dma_flush_area) 221 add x1, x0, x1 222 dcache_by_line_op civac, sy, x0, x1, x2, x3 223 ret 224SYM_FUNC_END_PI(__dma_flush_area) 225 226/* 227 * __dma_map_area(start, size, dir) 228 * - start - kernel virtual start address 229 * - size - size of region 230 * - dir - DMA direction 231 */ 232SYM_FUNC_START_PI(__dma_map_area) 233 cmp w2, #DMA_FROM_DEVICE 234 b.eq __dma_flush_area 235 add x1, x0, x1 236 b __dma_clean_area 237SYM_FUNC_END_PI(__dma_map_area) 238 239/* 240 * __dma_unmap_area(start, size, dir) 241 * - start - kernel virtual start address 242 * - size - size of region 243 * - dir - DMA direction 244 */ 245SYM_FUNC_START_PI(__dma_unmap_area) 246 add x1, x0, x1 247 cmp w2, #DMA_TO_DEVICE 248 b.ne __dma_inv_area 249 ret 250SYM_FUNC_END_PI(__dma_unmap_area) 251