1/* 2 * Copyright (c) 2013-2019, ARM Limited and Contributors. All rights reserved. 3 * 4 * SPDX-License-Identifier: BSD-3-Clause 5 */ 6 7#include <arch.h> 8#include <asm_macros.S> 9 10 .globl flush_dcache_range 11 .globl clean_dcache_range 12 .globl inv_dcache_range 13 .globl dcsw_op_louis 14 .globl dcsw_op_all 15 .globl dcsw_op_level1 16 .globl dcsw_op_level2 17 .globl dcsw_op_level3 18 19/* 20 * This macro can be used for implementing various data cache operations `op` 21 */ 22.macro do_dcache_maintenance_by_mva op 23 /* Exit early if size is zero */ 24 cbz x1, exit_loop_\op 25 dcache_line_size x2, x3 26 add x1, x0, x1 27 sub x3, x2, #1 28 bic x0, x0, x3 29loop_\op: 30 dc \op, x0 31 add x0, x0, x2 32 cmp x0, x1 33 b.lo loop_\op 34 dsb sy 35exit_loop_\op: 36 ret 37.endm 38 /* ------------------------------------------ 39 * Clean+Invalidate from base address till 40 * size. 'x0' = addr, 'x1' = size 41 * ------------------------------------------ 42 */ 43func flush_dcache_range 44 do_dcache_maintenance_by_mva civac 45endfunc flush_dcache_range 46 47 /* ------------------------------------------ 48 * Clean from base address till size. 49 * 'x0' = addr, 'x1' = size 50 * ------------------------------------------ 51 */ 52func clean_dcache_range 53 do_dcache_maintenance_by_mva cvac 54endfunc clean_dcache_range 55 56 /* ------------------------------------------ 57 * Invalidate from base address till 58 * size. 'x0' = addr, 'x1' = size 59 * ------------------------------------------ 60 */ 61func inv_dcache_range 62 do_dcache_maintenance_by_mva ivac 63endfunc inv_dcache_range 64 65 66 /* --------------------------------------------------------------- 67 * Data cache operations by set/way to the level specified 68 * 69 * The main function, do_dcsw_op requires: 70 * x0: The operation type (0-2), as defined in arch.h 71 * x3: The last cache level to operate on 72 * x9: clidr_el1 73 * x10: The cache level to begin operation from 74 * and will carry out the operation on each data cache from level 0 75 * to the level in x3 in sequence 76 * 77 * The dcsw_op macro sets up the x3 and x9 parameters based on 78 * clidr_el1 cache information before invoking the main function 79 * --------------------------------------------------------------- 80 */ 81 82 .macro dcsw_op shift, fw, ls 83 mrs x9, clidr_el1 84 ubfx x3, x9, \shift, \fw 85 lsl x3, x3, \ls 86 mov x10, xzr 87 b do_dcsw_op 88 .endm 89 90func do_dcsw_op 91 cbz x3, exit 92 adr x14, dcsw_loop_table // compute inner loop address 93 add x14, x14, x0, lsl #5 // inner loop is 8x32-bit instructions 94#if ENABLE_BTI 95 add x14, x14, x0, lsl #2 // inner loop is + "bti j" instruction 96#endif 97 mov x0, x9 98 mov w8, #1 99loop1: 100 add x2, x10, x10, lsr #1 // work out 3x current cache level 101 lsr x1, x0, x2 // extract cache type bits from clidr 102 and x1, x1, #7 // mask the bits for current cache only 103 cmp x1, #2 // see what cache we have at this level 104 b.lo level_done // nothing to do if no cache or icache 105 106 msr csselr_el1, x10 // select current cache level in csselr 107 isb // isb to sych the new cssr&csidr 108 mrs x1, ccsidr_el1 // read the new ccsidr 109 and x2, x1, #7 // extract the length of the cache lines 110 add x2, x2, #4 // add 4 (line length offset) 111 ubfx x4, x1, #3, #10 // maximum way number 112 clz w5, w4 // bit position of way size increment 113 lsl w9, w4, w5 // w9 = aligned max way number 114 lsl w16, w8, w5 // w16 = way number loop decrement 115 orr w9, w10, w9 // w9 = combine way and cache number 116 ubfx w6, w1, #13, #15 // w6 = max set number 117 lsl w17, w8, w2 // w17 = set number loop decrement 118 dsb sy // barrier before we start this level 119 br x14 // jump to DC operation specific loop 120 121 .macro dcsw_loop _op 122#if ENABLE_BTI 123 bti j 124#endif 125loop2_\_op: 126 lsl w7, w6, w2 // w7 = aligned max set number 127 128loop3_\_op: 129 orr w11, w9, w7 // combine cache, way and set number 130 dc \_op, x11 131 subs w7, w7, w17 // decrement set number 132 b.hs loop3_\_op 133 134 subs x9, x9, x16 // decrement way number 135 b.hs loop2_\_op 136 137 b level_done 138 .endm 139 140level_done: 141 add x10, x10, #2 // increment cache number 142 cmp x3, x10 143 b.hi loop1 144 msr csselr_el1, xzr // select cache level 0 in csselr 145 dsb sy // barrier to complete final cache operation 146 isb 147exit: 148 ret 149endfunc do_dcsw_op 150 151dcsw_loop_table: 152 dcsw_loop isw 153 dcsw_loop cisw 154 dcsw_loop csw 155 156 157func dcsw_op_louis 158 dcsw_op #LOUIS_SHIFT, #CLIDR_FIELD_WIDTH, #LEVEL_SHIFT 159endfunc dcsw_op_louis 160 161 162func dcsw_op_all 163 dcsw_op #LOC_SHIFT, #CLIDR_FIELD_WIDTH, #LEVEL_SHIFT 164endfunc dcsw_op_all 165 166 /* --------------------------------------------------------------- 167 * Helper macro for data cache operations by set/way for the 168 * level specified 169 * --------------------------------------------------------------- 170 */ 171 .macro dcsw_op_level level 172 mrs x9, clidr_el1 173 mov x3, \level 174 sub x10, x3, #2 175 b do_dcsw_op 176 .endm 177 178 /* --------------------------------------------------------------- 179 * Data cache operations by set/way for level 1 cache 180 * 181 * The main function, do_dcsw_op requires: 182 * x0: The operation type (0-2), as defined in arch.h 183 * --------------------------------------------------------------- 184 */ 185func dcsw_op_level1 186 dcsw_op_level #(1 << LEVEL_SHIFT) 187endfunc dcsw_op_level1 188 189 /* --------------------------------------------------------------- 190 * Data cache operations by set/way for level 2 cache 191 * 192 * The main function, do_dcsw_op requires: 193 * x0: The operation type (0-2), as defined in arch.h 194 * --------------------------------------------------------------- 195 */ 196func dcsw_op_level2 197 dcsw_op_level #(2 << LEVEL_SHIFT) 198endfunc dcsw_op_level2 199 200 /* --------------------------------------------------------------- 201 * Data cache operations by set/way for level 3 cache 202 * 203 * The main function, do_dcsw_op requires: 204 * x0: The operation type (0-2), as defined in arch.h 205 * --------------------------------------------------------------- 206 */ 207func dcsw_op_level3 208 dcsw_op_level #(3 << LEVEL_SHIFT) 209endfunc dcsw_op_level3 210