1 /* SPDX-License-Identifier: GPL-2.0-only */ 2 /* 3 * FP/SIMD state saving and restoring macros 4 * 5 * Copyright (C) 2012 ARM Ltd. 6 * Author: Catalin Marinas <catalin.marinas@arm.com> 7 */ 8 9 .macro fpsimd_save state, tmpnr 10 stp q0, q1, [\state, #16 * 0] 11 stp q2, q3, [\state, #16 * 2] 12 stp q4, q5, [\state, #16 * 4] 13 stp q6, q7, [\state, #16 * 6] 14 stp q8, q9, [\state, #16 * 8] 15 stp q10, q11, [\state, #16 * 10] 16 stp q12, q13, [\state, #16 * 12] 17 stp q14, q15, [\state, #16 * 14] 18 stp q16, q17, [\state, #16 * 16] 19 stp q18, q19, [\state, #16 * 18] 20 stp q20, q21, [\state, #16 * 20] 21 stp q22, q23, [\state, #16 * 22] 22 stp q24, q25, [\state, #16 * 24] 23 stp q26, q27, [\state, #16 * 26] 24 stp q28, q29, [\state, #16 * 28] 25 stp q30, q31, [\state, #16 * 30]! 26 mrs x\tmpnr, fpsr 27 str w\tmpnr, [\state, #16 * 2] 28 mrs x\tmpnr, fpcr 29 str w\tmpnr, [\state, #16 * 2 + 4] 30 .endm 31 32 .macro fpsimd_restore_fpcr state, tmp 33 /* 34 * Writes to fpcr may be self-synchronising, so avoid restoring 35 * the register if it hasn't changed. 36 */ 37 mrs \tmp, fpcr 38 cmp \tmp, \state 39 b.eq 9999f 40 msr fpcr, \state 41 9999: 42 .endm 43 44 /* Clobbers \state */ 45 .macro fpsimd_restore state, tmpnr 46 ldp q0, q1, [\state, #16 * 0] 47 ldp q2, q3, [\state, #16 * 2] 48 ldp q4, q5, [\state, #16 * 4] 49 ldp q6, q7, [\state, #16 * 6] 50 ldp q8, q9, [\state, #16 * 8] 51 ldp q10, q11, [\state, #16 * 10] 52 ldp q12, q13, [\state, #16 * 12] 53 ldp q14, q15, [\state, #16 * 14] 54 ldp q16, q17, [\state, #16 * 16] 55 ldp q18, q19, [\state, #16 * 18] 56 ldp q20, q21, [\state, #16 * 20] 57 ldp q22, q23, [\state, #16 * 22] 58 ldp q24, q25, [\state, #16 * 24] 59 ldp q26, q27, [\state, #16 * 26] 60 ldp q28, q29, [\state, #16 * 28] 61 ldp q30, q31, [\state, #16 * 30]! 62 ldr w\tmpnr, [\state, #16 * 2] 63 msr fpsr, x\tmpnr 64 ldr w\tmpnr, [\state, #16 * 2 + 4] 65 fpsimd_restore_fpcr x\tmpnr, \state 66 .endm 67 68 /* Sanity-check macros to help avoid encoding garbage instructions */ 69 70 .macro _check_general_reg nr 71 .if (\nr) < 0 || (\nr) > 30 72 .error "Bad register number \nr." 73 .endif 74 .endm 75 76 .macro _sve_check_zreg znr 77 .if (\znr) < 0 || (\znr) > 31 78 .error "Bad Scalable Vector Extension vector register number \znr." 79 .endif 80 .endm 81 82 .macro _sve_check_preg pnr 83 .if (\pnr) < 0 || (\pnr) > 15 84 .error "Bad Scalable Vector Extension predicate register number \pnr." 85 .endif 86 .endm 87 88 .macro _check_num n, min, max 89 .if (\n) < (\min) || (\n) > (\max) 90 .error "Number \n out of range [\min,\max]" 91 .endif 92 .endm 93 94 /* SVE instruction encodings for non-SVE-capable assemblers */ 95 96 /* STR (vector): STR Z\nz, [X\nxbase, #\offset, MUL VL] */ 97 .macro _sve_str_v nz, nxbase, offset=0 98 _sve_check_zreg \nz 99 _check_general_reg \nxbase 100 _check_num (\offset), -0x100, 0xff 101 .inst 0xe5804000 \ 102 | (\nz) \ 103 | ((\nxbase) << 5) \ 104 | (((\offset) & 7) << 10) \ 105 | (((\offset) & 0x1f8) << 13) 106 .endm 107 108 /* LDR (vector): LDR Z\nz, [X\nxbase, #\offset, MUL VL] */ 109 .macro _sve_ldr_v nz, nxbase, offset=0 110 _sve_check_zreg \nz 111 _check_general_reg \nxbase 112 _check_num (\offset), -0x100, 0xff 113 .inst 0x85804000 \ 114 | (\nz) \ 115 | ((\nxbase) << 5) \ 116 | (((\offset) & 7) << 10) \ 117 | (((\offset) & 0x1f8) << 13) 118 .endm 119 120 /* STR (predicate): STR P\np, [X\nxbase, #\offset, MUL VL] */ 121 .macro _sve_str_p np, nxbase, offset=0 122 _sve_check_preg \np 123 _check_general_reg \nxbase 124 _check_num (\offset), -0x100, 0xff 125 .inst 0xe5800000 \ 126 | (\np) \ 127 | ((\nxbase) << 5) \ 128 | (((\offset) & 7) << 10) \ 129 | (((\offset) & 0x1f8) << 13) 130 .endm 131 132 /* LDR (predicate): LDR P\np, [X\nxbase, #\offset, MUL VL] */ 133 .macro _sve_ldr_p np, nxbase, offset=0 134 _sve_check_preg \np 135 _check_general_reg \nxbase 136 _check_num (\offset), -0x100, 0xff 137 .inst 0x85800000 \ 138 | (\np) \ 139 | ((\nxbase) << 5) \ 140 | (((\offset) & 7) << 10) \ 141 | (((\offset) & 0x1f8) << 13) 142 .endm 143 144 /* RDVL X\nx, #\imm */ 145 .macro _sve_rdvl nx, imm 146 _check_general_reg \nx 147 _check_num (\imm), -0x20, 0x1f 148 .inst 0x04bf5000 \ 149 | (\nx) \ 150 | (((\imm) & 0x3f) << 5) 151 .endm 152 153 /* RDFFR (unpredicated): RDFFR P\np.B */ 154 .macro _sve_rdffr np 155 _sve_check_preg \np 156 .inst 0x2519f000 \ 157 | (\np) 158 .endm 159 160 /* WRFFR P\np.B */ 161 .macro _sve_wrffr np 162 _sve_check_preg \np 163 .inst 0x25289000 \ 164 | ((\np) << 5) 165 .endm 166 167 /* PFALSE P\np.B */ 168 .macro _sve_pfalse np 169 _sve_check_preg \np 170 .inst 0x2518e400 \ 171 | (\np) 172 .endm 173 174 .macro __for from:req, to:req 175 .if (\from) == (\to) 176 _for__body %\from 177 .else 178 __for %\from, %((\from) + ((\to) - (\from)) / 2) 179 __for %((\from) + ((\to) - (\from)) / 2 + 1), %\to 180 .endif 181 .endm 182 183 .macro _for var:req, from:req, to:req, insn:vararg 184 .macro _for__body \var:req 185 .noaltmacro 186 \insn 187 .altmacro 188 .endm 189 190 .altmacro 191 __for \from, \to 192 .noaltmacro 193 194 .purgem _for__body 195 .endm 196 197 /* Update ZCR_EL1.LEN with the new VQ */ 198 .macro sve_load_vq xvqminus1, xtmp, xtmp2 199 mrs_s \xtmp, SYS_ZCR_EL1 200 bic \xtmp2, \xtmp, ZCR_ELx_LEN_MASK 201 orr \xtmp2, \xtmp2, \xvqminus1 202 cmp \xtmp2, \xtmp 203 b.eq 921f 204 msr_s SYS_ZCR_EL1, \xtmp2 //self-synchronising 205 921: 206 .endm 207 208 /* Preserve the first 128-bits of Znz and zero the rest. */ 209 .macro _sve_flush_z nz 210 _sve_check_zreg \nz 211 mov v\nz\().16b, v\nz\().16b 212 .endm 213 214 .macro sve_flush 215 _for n, 0, 31, _sve_flush_z \n 216 _for n, 0, 15, _sve_pfalse \n 217 _sve_wrffr 0 218 .endm 219 220 .macro sve_save nxbase, xpfpsr, nxtmp 221 _for n, 0, 31, _sve_str_v \n, \nxbase, \n - 34 222 _for n, 0, 15, _sve_str_p \n, \nxbase, \n - 16 223 _sve_rdffr 0 224 _sve_str_p 0, \nxbase 225 _sve_ldr_p 0, \nxbase, -16 226 227 mrs x\nxtmp, fpsr 228 str w\nxtmp, [\xpfpsr] 229 mrs x\nxtmp, fpcr 230 str w\nxtmp, [\xpfpsr, #4] 231 .endm 232 233 .macro sve_load nxbase, xpfpsr, xvqminus1, nxtmp, xtmp2 234 sve_load_vq \xvqminus1, x\nxtmp, \xtmp2 235 _for n, 0, 31, _sve_ldr_v \n, \nxbase, \n - 34 236 _sve_ldr_p 0, \nxbase 237 _sve_wrffr 0 238 _for n, 0, 15, _sve_ldr_p \n, \nxbase, \n - 16 239 240 ldr w\nxtmp, [\xpfpsr] 241 msr fpsr, x\nxtmp 242 ldr w\nxtmp, [\xpfpsr, #4] 243 msr fpcr, x\nxtmp 244 .endm 245