1 /* SPDX-License-Identifier: GPL-2.0-only */ 2 /* 3 * Based on arch/arm/include/asm/assembler.h, arch/arm/mm/proc-macros.S 4 * 5 * Copyright (C) 1996-2000 Russell King 6 * Copyright (C) 2012 ARM Ltd. 7 */ 8 #ifndef __ASSEMBLY__ 9 #error "Only include this from assembly code" 10 #endif 11 12 #ifndef __ASM_ASSEMBLER_H 13 #define __ASM_ASSEMBLER_H 14 15 #include <asm-generic/export.h> 16 17 #include <asm/asm-offsets.h> 18 #include <asm/asm-bug.h> 19 #include <asm/alternative.h> 20 #include <asm/cpufeature.h> 21 #include <asm/cputype.h> 22 #include <asm/debug-monitors.h> 23 #include <asm/page.h> 24 #include <asm/pgtable-hwdef.h> 25 #include <asm/ptrace.h> 26 #include <asm/thread_info.h> 27 28 /* 29 * Provide a wxN alias for each wN register so what we can paste a xN 30 * reference after a 'w' to obtain the 32-bit version. 31 */ 32 .irp n,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30 33 wx\n .req w\n 34 .endr 35 36 .macro save_and_disable_daif, flags 37 mrs \flags, daif 38 msr daifset, #0xf 39 .endm 40 41 .macro disable_daif 42 msr daifset, #0xf 43 .endm 44 45 .macro enable_daif 46 msr daifclr, #0xf 47 .endm 48 49 .macro restore_daif, flags:req 50 msr daif, \flags 51 .endm 52 53 /* IRQ is the lowest priority flag, unconditionally unmask the rest. */ 54 .macro enable_da_f 55 msr daifclr, #(8 | 4 | 1) 56 .endm 57 58 /* 59 * Save/restore interrupts. 60 */ 61 .macro save_and_disable_irq, flags 62 mrs \flags, daif 63 msr daifset, #2 64 .endm 65 66 .macro restore_irq, flags 67 msr daif, \flags 68 .endm 69 70 .macro enable_dbg 71 msr daifclr, #8 72 .endm 73 74 .macro disable_step_tsk, flgs, tmp 75 tbz \flgs, #TIF_SINGLESTEP, 9990f 76 mrs \tmp, mdscr_el1 77 bic \tmp, \tmp, #DBG_MDSCR_SS 78 msr mdscr_el1, \tmp 79 isb // Synchronise with enable_dbg 80 9990: 81 .endm 82 83 /* call with daif masked */ 84 .macro enable_step_tsk, flgs, tmp 85 tbz \flgs, #TIF_SINGLESTEP, 9990f 86 mrs \tmp, mdscr_el1 87 orr \tmp, \tmp, #DBG_MDSCR_SS 88 msr mdscr_el1, \tmp 89 9990: 90 .endm 91 92 /* 93 * RAS Error Synchronization barrier 94 */ 95 .macro esb 96 #ifdef CONFIG_ARM64_RAS_EXTN 97 hint #16 98 #else 99 nop 100 #endif 101 .endm 102 103 /* 104 * Value prediction barrier 105 */ 106 .macro csdb 107 hint #20 108 .endm 109 110 /* 111 * Clear Branch History instruction 112 */ 113 .macro clearbhb 114 hint #22 115 .endm 116 117 /* 118 * Speculation barrier 119 */ 120 .macro sb 121 alternative_if_not ARM64_HAS_SB 122 dsb nsh 123 isb 124 alternative_else 125 SB_BARRIER_INSN 126 nop 127 alternative_endif 128 .endm 129 130 /* 131 * NOP sequence 132 */ 133 .macro nops, num 134 .rept \num 135 nop 136 .endr 137 .endm 138 139 /* 140 * Emit an entry into the exception table 141 */ 142 .macro _asm_extable, from, to 143 .pushsection __ex_table, "a" 144 .align 3 145 .long (\from - .), (\to - .) 146 .popsection 147 .endm 148 149 #define USER(l, x...) \ 150 9999: x; \ 151 _asm_extable 9999b, l 152 153 /* 154 * Register aliases. 155 */ 156 lr .req x30 // link register 157 158 /* 159 * Vector entry 160 */ 161 .macro ventry label 162 .align 7 163 b \label 164 .endm 165 166 /* 167 * Select code when configured for BE. 168 */ 169 #ifdef CONFIG_CPU_BIG_ENDIAN 170 #define CPU_BE(code...) code 171 #else 172 #define CPU_BE(code...) 173 #endif 174 175 /* 176 * Select code when configured for LE. 177 */ 178 #ifdef CONFIG_CPU_BIG_ENDIAN 179 #define CPU_LE(code...) 180 #else 181 #define CPU_LE(code...) code 182 #endif 183 184 /* 185 * Define a macro that constructs a 64-bit value by concatenating two 186 * 32-bit registers. Note that on big endian systems the order of the 187 * registers is swapped. 188 */ 189 #ifndef CONFIG_CPU_BIG_ENDIAN 190 .macro regs_to_64, rd, lbits, hbits 191 #else 192 .macro regs_to_64, rd, hbits, lbits 193 #endif 194 orr \rd, \lbits, \hbits, lsl #32 195 .endm 196 197 /* 198 * Pseudo-ops for PC-relative adr/ldr/str <reg>, <symbol> where 199 * <symbol> is within the range +/- 4 GB of the PC. 200 */ 201 /* 202 * @dst: destination register (64 bit wide) 203 * @sym: name of the symbol 204 */ 205 .macro adr_l, dst, sym 206 adrp \dst, \sym 207 add \dst, \dst, :lo12:\sym 208 .endm 209 210 /* 211 * @dst: destination register (32 or 64 bit wide) 212 * @sym: name of the symbol 213 * @tmp: optional 64-bit scratch register to be used if <dst> is a 214 * 32-bit wide register, in which case it cannot be used to hold 215 * the address 216 */ 217 .macro ldr_l, dst, sym, tmp= 218 .ifb \tmp 219 adrp \dst, \sym 220 ldr \dst, [\dst, :lo12:\sym] 221 .else 222 adrp \tmp, \sym 223 ldr \dst, [\tmp, :lo12:\sym] 224 .endif 225 .endm 226 227 /* 228 * @src: source register (32 or 64 bit wide) 229 * @sym: name of the symbol 230 * @tmp: mandatory 64-bit scratch register to calculate the address 231 * while <src> needs to be preserved. 232 */ 233 .macro str_l, src, sym, tmp 234 adrp \tmp, \sym 235 str \src, [\tmp, :lo12:\sym] 236 .endm 237 238 /* 239 * @dst: destination register 240 */ 241 #if defined(__KVM_NVHE_HYPERVISOR__) || defined(__KVM_VHE_HYPERVISOR__) 242 .macro this_cpu_offset, dst 243 mrs \dst, tpidr_el2 244 .endm 245 #else 246 .macro this_cpu_offset, dst 247 alternative_if_not ARM64_HAS_VIRT_HOST_EXTN 248 mrs \dst, tpidr_el1 249 alternative_else 250 mrs \dst, tpidr_el2 251 alternative_endif 252 .endm 253 #endif 254 255 /* 256 * @dst: Result of per_cpu(sym, smp_processor_id()) (can be SP) 257 * @sym: The name of the per-cpu variable 258 * @tmp: scratch register 259 */ 260 .macro adr_this_cpu, dst, sym, tmp 261 adrp \tmp, \sym 262 add \dst, \tmp, #:lo12:\sym 263 this_cpu_offset \tmp 264 add \dst, \dst, \tmp 265 .endm 266 267 /* 268 * @dst: Result of READ_ONCE(per_cpu(sym, smp_processor_id())) 269 * @sym: The name of the per-cpu variable 270 * @tmp: scratch register 271 */ 272 .macro ldr_this_cpu dst, sym, tmp 273 adr_l \dst, \sym 274 this_cpu_offset \tmp 275 ldr \dst, [\dst, \tmp] 276 .endm 277 278 /* 279 * vma_vm_mm - get mm pointer from vma pointer (vma->vm_mm) 280 */ 281 .macro vma_vm_mm, rd, rn 282 ldr \rd, [\rn, #VMA_VM_MM] 283 .endm 284 285 /* 286 * read_ctr - read CTR_EL0. If the system has mismatched register fields, 287 * provide the system wide safe value from arm64_ftr_reg_ctrel0.sys_val 288 */ 289 .macro read_ctr, reg 290 #ifndef __KVM_NVHE_HYPERVISOR__ 291 alternative_if_not ARM64_MISMATCHED_CACHE_TYPE 292 mrs \reg, ctr_el0 // read CTR 293 nop 294 alternative_else 295 ldr_l \reg, arm64_ftr_reg_ctrel0 + ARM64_FTR_SYSVAL 296 alternative_endif 297 #else 298 alternative_if_not ARM64_KVM_PROTECTED_MODE 299 ASM_BUG() 300 alternative_else_nop_endif 301 alternative_cb kvm_compute_final_ctr_el0 302 movz \reg, #0 303 movk \reg, #0, lsl #16 304 movk \reg, #0, lsl #32 305 movk \reg, #0, lsl #48 306 alternative_cb_end 307 #endif 308 .endm 309 310 311 /* 312 * raw_dcache_line_size - get the minimum D-cache line size on this CPU 313 * from the CTR register. 314 */ 315 .macro raw_dcache_line_size, reg, tmp 316 mrs \tmp, ctr_el0 // read CTR 317 ubfm \tmp, \tmp, #16, #19 // cache line size encoding 318 mov \reg, #4 // bytes per word 319 lsl \reg, \reg, \tmp // actual cache line size 320 .endm 321 322 /* 323 * dcache_line_size - get the safe D-cache line size across all CPUs 324 */ 325 .macro dcache_line_size, reg, tmp 326 read_ctr \tmp 327 ubfm \tmp, \tmp, #16, #19 // cache line size encoding 328 mov \reg, #4 // bytes per word 329 lsl \reg, \reg, \tmp // actual cache line size 330 .endm 331 332 /* 333 * raw_icache_line_size - get the minimum I-cache line size on this CPU 334 * from the CTR register. 335 */ 336 .macro raw_icache_line_size, reg, tmp 337 mrs \tmp, ctr_el0 // read CTR 338 and \tmp, \tmp, #0xf // cache line size encoding 339 mov \reg, #4 // bytes per word 340 lsl \reg, \reg, \tmp // actual cache line size 341 .endm 342 343 /* 344 * icache_line_size - get the safe I-cache line size across all CPUs 345 */ 346 .macro icache_line_size, reg, tmp 347 read_ctr \tmp 348 and \tmp, \tmp, #0xf // cache line size encoding 349 mov \reg, #4 // bytes per word 350 lsl \reg, \reg, \tmp // actual cache line size 351 .endm 352 353 /* 354 * tcr_set_t0sz - update TCR.T0SZ so that we can load the ID map 355 */ 356 .macro tcr_set_t0sz, valreg, t0sz 357 bfi \valreg, \t0sz, #TCR_T0SZ_OFFSET, #TCR_TxSZ_WIDTH 358 .endm 359 360 /* 361 * tcr_set_t1sz - update TCR.T1SZ 362 */ 363 .macro tcr_set_t1sz, valreg, t1sz 364 bfi \valreg, \t1sz, #TCR_T1SZ_OFFSET, #TCR_TxSZ_WIDTH 365 .endm 366 367 /* 368 * tcr_compute_pa_size - set TCR.(I)PS to the highest supported 369 * ID_AA64MMFR0_EL1.PARange value 370 * 371 * tcr: register with the TCR_ELx value to be updated 372 * pos: IPS or PS bitfield position 373 * tmp{0,1}: temporary registers 374 */ 375 .macro tcr_compute_pa_size, tcr, pos, tmp0, tmp1 376 mrs \tmp0, ID_AA64MMFR0_EL1 377 // Narrow PARange to fit the PS field in TCR_ELx 378 ubfx \tmp0, \tmp0, #ID_AA64MMFR0_PARANGE_SHIFT, #3 379 mov \tmp1, #ID_AA64MMFR0_PARANGE_MAX 380 cmp \tmp0, \tmp1 381 csel \tmp0, \tmp1, \tmp0, hi 382 bfi \tcr, \tmp0, \pos, #3 383 .endm 384 385 /* 386 * Macro to perform a data cache maintenance for the interval 387 * [kaddr, kaddr + size) 388 * 389 * op: operation passed to dc instruction 390 * domain: domain used in dsb instruciton 391 * kaddr: starting virtual address of the region 392 * size: size of the region 393 * Corrupts: kaddr, size, tmp1, tmp2 394 */ 395 .macro __dcache_op_workaround_clean_cache, op, kaddr 396 alternative_if_not ARM64_WORKAROUND_CLEAN_CACHE 397 dc \op, \kaddr 398 alternative_else 399 dc civac, \kaddr 400 alternative_endif 401 .endm 402 403 .macro dcache_by_line_op op, domain, kaddr, size, tmp1, tmp2 404 dcache_line_size \tmp1, \tmp2 405 add \size, \kaddr, \size 406 sub \tmp2, \tmp1, #1 407 bic \kaddr, \kaddr, \tmp2 408 9998: 409 .ifc \op, cvau 410 __dcache_op_workaround_clean_cache \op, \kaddr 411 .else 412 .ifc \op, cvac 413 __dcache_op_workaround_clean_cache \op, \kaddr 414 .else 415 .ifc \op, cvap 416 sys 3, c7, c12, 1, \kaddr // dc cvap 417 .else 418 .ifc \op, cvadp 419 sys 3, c7, c13, 1, \kaddr // dc cvadp 420 .else 421 dc \op, \kaddr 422 .endif 423 .endif 424 .endif 425 .endif 426 add \kaddr, \kaddr, \tmp1 427 cmp \kaddr, \size 428 b.lo 9998b 429 dsb \domain 430 .endm 431 432 /* 433 * Macro to perform an instruction cache maintenance for the interval 434 * [start, end) 435 * 436 * start, end: virtual addresses describing the region 437 * label: A label to branch to on user fault. 438 * Corrupts: tmp1, tmp2 439 */ 440 .macro invalidate_icache_by_line start, end, tmp1, tmp2, label 441 icache_line_size \tmp1, \tmp2 442 sub \tmp2, \tmp1, #1 443 bic \tmp2, \start, \tmp2 444 9997: 445 USER(\label, ic ivau, \tmp2) // invalidate I line PoU 446 add \tmp2, \tmp2, \tmp1 447 cmp \tmp2, \end 448 b.lo 9997b 449 dsb ish 450 isb 451 .endm 452 453 /* 454 * reset_pmuserenr_el0 - reset PMUSERENR_EL0 if PMUv3 present 455 */ 456 .macro reset_pmuserenr_el0, tmpreg 457 mrs \tmpreg, id_aa64dfr0_el1 458 sbfx \tmpreg, \tmpreg, #ID_AA64DFR0_PMUVER_SHIFT, #4 459 cmp \tmpreg, #1 // Skip if no PMU present 460 b.lt 9000f 461 msr pmuserenr_el0, xzr // Disable PMU access from EL0 462 9000: 463 .endm 464 465 /* 466 * reset_amuserenr_el0 - reset AMUSERENR_EL0 if AMUv1 present 467 */ 468 .macro reset_amuserenr_el0, tmpreg 469 mrs \tmpreg, id_aa64pfr0_el1 // Check ID_AA64PFR0_EL1 470 ubfx \tmpreg, \tmpreg, #ID_AA64PFR0_AMU_SHIFT, #4 471 cbz \tmpreg, .Lskip_\@ // Skip if no AMU present 472 msr_s SYS_AMUSERENR_EL0, xzr // Disable AMU access from EL0 473 .Lskip_\@: 474 .endm 475 /* 476 * copy_page - copy src to dest using temp registers t1-t8 477 */ 478 .macro copy_page dest:req src:req t1:req t2:req t3:req t4:req t5:req t6:req t7:req t8:req 479 9998: ldp \t1, \t2, [\src] 480 ldp \t3, \t4, [\src, #16] 481 ldp \t5, \t6, [\src, #32] 482 ldp \t7, \t8, [\src, #48] 483 add \src, \src, #64 484 stnp \t1, \t2, [\dest] 485 stnp \t3, \t4, [\dest, #16] 486 stnp \t5, \t6, [\dest, #32] 487 stnp \t7, \t8, [\dest, #48] 488 add \dest, \dest, #64 489 tst \src, #(PAGE_SIZE - 1) 490 b.ne 9998b 491 .endm 492 493 /* 494 * Annotate a function as being unsuitable for kprobes. 495 */ 496 #ifdef CONFIG_KPROBES 497 #define NOKPROBE(x) \ 498 .pushsection "_kprobe_blacklist", "aw"; \ 499 .quad x; \ 500 .popsection; 501 #else 502 #define NOKPROBE(x) 503 #endif 504 505 #if defined(CONFIG_KASAN_GENERIC) || defined(CONFIG_KASAN_SW_TAGS) 506 #define EXPORT_SYMBOL_NOKASAN(name) 507 #else 508 #define EXPORT_SYMBOL_NOKASAN(name) EXPORT_SYMBOL(name) 509 #endif 510 511 /* 512 * Emit a 64-bit absolute little endian symbol reference in a way that 513 * ensures that it will be resolved at build time, even when building a 514 * PIE binary. This requires cooperation from the linker script, which 515 * must emit the lo32/hi32 halves individually. 516 */ 517 .macro le64sym, sym 518 .long \sym\()_lo32 519 .long \sym\()_hi32 520 .endm 521 522 /* 523 * mov_q - move an immediate constant into a 64-bit register using 524 * between 2 and 4 movz/movk instructions (depending on the 525 * magnitude and sign of the operand) 526 */ 527 .macro mov_q, reg, val 528 .if (((\val) >> 31) == 0 || ((\val) >> 31) == 0x1ffffffff) 529 movz \reg, :abs_g1_s:\val 530 .else 531 .if (((\val) >> 47) == 0 || ((\val) >> 47) == 0x1ffff) 532 movz \reg, :abs_g2_s:\val 533 .else 534 movz \reg, :abs_g3:\val 535 movk \reg, :abs_g2_nc:\val 536 .endif 537 movk \reg, :abs_g1_nc:\val 538 .endif 539 movk \reg, :abs_g0_nc:\val 540 .endm 541 542 /* 543 * Return the current task_struct. 544 */ 545 .macro get_current_task, rd 546 mrs \rd, sp_el0 547 .endm 548 549 /* 550 * Offset ttbr1 to allow for 48-bit kernel VAs set with 52-bit PTRS_PER_PGD. 551 * orr is used as it can cover the immediate value (and is idempotent). 552 * In future this may be nop'ed out when dealing with 52-bit kernel VAs. 553 * ttbr: Value of ttbr to set, modified. 554 */ 555 .macro offset_ttbr1, ttbr, tmp 556 #ifdef CONFIG_ARM64_VA_BITS_52 557 mrs_s \tmp, SYS_ID_AA64MMFR2_EL1 558 and \tmp, \tmp, #(0xf << ID_AA64MMFR2_LVA_SHIFT) 559 cbnz \tmp, .Lskipoffs_\@ 560 orr \ttbr, \ttbr, #TTBR1_BADDR_4852_OFFSET 561 .Lskipoffs_\@ : 562 #endif 563 .endm 564 565 /* 566 * Perform the reverse of offset_ttbr1. 567 * bic is used as it can cover the immediate value and, in future, won't need 568 * to be nop'ed out when dealing with 52-bit kernel VAs. 569 */ 570 .macro restore_ttbr1, ttbr 571 #ifdef CONFIG_ARM64_VA_BITS_52 572 bic \ttbr, \ttbr, #TTBR1_BADDR_4852_OFFSET 573 #endif 574 .endm 575 576 /* 577 * Arrange a physical address in a TTBR register, taking care of 52-bit 578 * addresses. 579 * 580 * phys: physical address, preserved 581 * ttbr: returns the TTBR value 582 */ 583 .macro phys_to_ttbr, ttbr, phys 584 #ifdef CONFIG_ARM64_PA_BITS_52 585 orr \ttbr, \phys, \phys, lsr #46 586 and \ttbr, \ttbr, #TTBR_BADDR_MASK_52 587 #else 588 mov \ttbr, \phys 589 #endif 590 .endm 591 592 .macro phys_to_pte, pte, phys 593 #ifdef CONFIG_ARM64_PA_BITS_52 594 /* 595 * We assume \phys is 64K aligned and this is guaranteed by only 596 * supporting this configuration with 64K pages. 597 */ 598 orr \pte, \phys, \phys, lsr #36 599 and \pte, \pte, #PTE_ADDR_MASK 600 #else 601 mov \pte, \phys 602 #endif 603 .endm 604 605 .macro pte_to_phys, phys, pte 606 #ifdef CONFIG_ARM64_PA_BITS_52 607 ubfiz \phys, \pte, #(48 - 16 - 12), #16 608 bfxil \phys, \pte, #16, #32 609 lsl \phys, \phys, #16 610 #else 611 and \phys, \pte, #PTE_ADDR_MASK 612 #endif 613 .endm 614 615 /* 616 * tcr_clear_errata_bits - Clear TCR bits that trigger an errata on this CPU. 617 */ 618 .macro tcr_clear_errata_bits, tcr, tmp1, tmp2 619 #ifdef CONFIG_FUJITSU_ERRATUM_010001 620 mrs \tmp1, midr_el1 621 622 mov_q \tmp2, MIDR_FUJITSU_ERRATUM_010001_MASK 623 and \tmp1, \tmp1, \tmp2 624 mov_q \tmp2, MIDR_FUJITSU_ERRATUM_010001 625 cmp \tmp1, \tmp2 626 b.ne 10f 627 628 mov_q \tmp2, TCR_CLEAR_FUJITSU_ERRATUM_010001 629 bic \tcr, \tcr, \tmp2 630 10: 631 #endif /* CONFIG_FUJITSU_ERRATUM_010001 */ 632 .endm 633 634 /** 635 * Errata workaround prior to disable MMU. Insert an ISB immediately prior 636 * to executing the MSR that will change SCTLR_ELn[M] from a value of 1 to 0. 637 */ 638 .macro pre_disable_mmu_workaround 639 #ifdef CONFIG_QCOM_FALKOR_ERRATUM_E1041 640 isb 641 #endif 642 .endm 643 644 /* 645 * frame_push - Push @regcount callee saved registers to the stack, 646 * starting at x19, as well as x29/x30, and set x29 to 647 * the new value of sp. Add @extra bytes of stack space 648 * for locals. 649 */ 650 .macro frame_push, regcount:req, extra 651 __frame st, \regcount, \extra 652 .endm 653 654 /* 655 * frame_pop - Pop the callee saved registers from the stack that were 656 * pushed in the most recent call to frame_push, as well 657 * as x29/x30 and any extra stack space that may have been 658 * allocated. 659 */ 660 .macro frame_pop 661 __frame ld 662 .endm 663 664 .macro __frame_regs, reg1, reg2, op, num 665 .if .Lframe_regcount == \num 666 \op\()r \reg1, [sp, #(\num + 1) * 8] 667 .elseif .Lframe_regcount > \num 668 \op\()p \reg1, \reg2, [sp, #(\num + 1) * 8] 669 .endif 670 .endm 671 672 .macro __frame, op, regcount, extra=0 673 .ifc \op, st 674 .if (\regcount) < 0 || (\regcount) > 10 675 .error "regcount should be in the range [0 ... 10]" 676 .endif 677 .if ((\extra) % 16) != 0 678 .error "extra should be a multiple of 16 bytes" 679 .endif 680 .ifdef .Lframe_regcount 681 .if .Lframe_regcount != -1 682 .error "frame_push/frame_pop may not be nested" 683 .endif 684 .endif 685 .set .Lframe_regcount, \regcount 686 .set .Lframe_extra, \extra 687 .set .Lframe_local_offset, ((\regcount + 3) / 2) * 16 688 stp x29, x30, [sp, #-.Lframe_local_offset - .Lframe_extra]! 689 mov x29, sp 690 .endif 691 692 __frame_regs x19, x20, \op, 1 693 __frame_regs x21, x22, \op, 3 694 __frame_regs x23, x24, \op, 5 695 __frame_regs x25, x26, \op, 7 696 __frame_regs x27, x28, \op, 9 697 698 .ifc \op, ld 699 .if .Lframe_regcount == -1 700 .error "frame_push/frame_pop may not be nested" 701 .endif 702 ldp x29, x30, [sp], #.Lframe_local_offset + .Lframe_extra 703 .set .Lframe_regcount, -1 704 .endif 705 .endm 706 707 /* 708 * Set SCTLR_ELx to the @reg value, and invalidate the local icache 709 * in the process. This is called when setting the MMU on. 710 */ 711 .macro set_sctlr, sreg, reg 712 msr \sreg, \reg 713 isb 714 /* 715 * Invalidate the local I-cache so that any instructions fetched 716 * speculatively from the PoC are discarded, since they may have 717 * been dynamically patched at the PoU. 718 */ 719 ic iallu 720 dsb nsh 721 isb 722 .endm 723 724 .macro set_sctlr_el1, reg 725 set_sctlr sctlr_el1, \reg 726 .endm 727 728 .macro set_sctlr_el2, reg 729 set_sctlr sctlr_el2, \reg 730 .endm 731 732 /* 733 * Check whether preempt/bh-disabled asm code should yield as soon as 734 * it is able. This is the case if we are currently running in task 735 * context, and either a softirq is pending, or the TIF_NEED_RESCHED 736 * flag is set and re-enabling preemption a single time would result in 737 * a preempt count of zero. (Note that the TIF_NEED_RESCHED flag is 738 * stored negated in the top word of the thread_info::preempt_count 739 * field) 740 */ 741 .macro cond_yield, lbl:req, tmp:req, tmp2:req 742 get_current_task \tmp 743 ldr \tmp, [\tmp, #TSK_TI_PREEMPT] 744 /* 745 * If we are serving a softirq, there is no point in yielding: the 746 * softirq will not be preempted no matter what we do, so we should 747 * run to completion as quickly as we can. 748 */ 749 tbnz \tmp, #SOFTIRQ_SHIFT, .Lnoyield_\@ 750 #ifdef CONFIG_PREEMPTION 751 sub \tmp, \tmp, #PREEMPT_DISABLE_OFFSET 752 cbz \tmp, \lbl 753 #endif 754 adr_l \tmp, irq_stat + IRQ_CPUSTAT_SOFTIRQ_PENDING 755 this_cpu_offset \tmp2 756 ldr w\tmp, [\tmp, \tmp2] 757 cbnz w\tmp, \lbl // yield on pending softirq in task context 758 .Lnoyield_\@: 759 .endm 760 761 /* 762 * This macro emits a program property note section identifying 763 * architecture features which require special handling, mainly for 764 * use in assembly files included in the VDSO. 765 */ 766 767 #define NT_GNU_PROPERTY_TYPE_0 5 768 #define GNU_PROPERTY_AARCH64_FEATURE_1_AND 0xc0000000 769 770 #define GNU_PROPERTY_AARCH64_FEATURE_1_BTI (1U << 0) 771 #define GNU_PROPERTY_AARCH64_FEATURE_1_PAC (1U << 1) 772 773 #ifdef CONFIG_ARM64_BTI_KERNEL 774 #define GNU_PROPERTY_AARCH64_FEATURE_1_DEFAULT \ 775 ((GNU_PROPERTY_AARCH64_FEATURE_1_BTI | \ 776 GNU_PROPERTY_AARCH64_FEATURE_1_PAC)) 777 #endif 778 779 #ifdef GNU_PROPERTY_AARCH64_FEATURE_1_DEFAULT 780 .macro emit_aarch64_feature_1_and, feat=GNU_PROPERTY_AARCH64_FEATURE_1_DEFAULT 781 .pushsection .note.gnu.property, "a" 782 .align 3 783 .long 2f - 1f 784 .long 6f - 3f 785 .long NT_GNU_PROPERTY_TYPE_0 786 1: .string "GNU" 787 2: 788 .align 3 789 3: .long GNU_PROPERTY_AARCH64_FEATURE_1_AND 790 .long 5f - 4f 791 4: 792 /* 793 * This is described with an array of char in the Linux API 794 * spec but the text and all other usage (including binutils, 795 * clang and GCC) treat this as a 32 bit value so no swizzling 796 * is required for big endian. 797 */ 798 .long \feat 799 5: 800 .align 3 801 6: 802 .popsection 803 .endm 804 805 #else 806 .macro emit_aarch64_feature_1_and, feat=0 807 .endm 808 809 #endif /* GNU_PROPERTY_AARCH64_FEATURE_1_DEFAULT */ 810 811 .macro __mitigate_spectre_bhb_loop tmp 812 #ifdef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY 813 alternative_cb spectre_bhb_patch_loop_iter 814 mov \tmp, #32 // Patched to correct the immediate 815 alternative_cb_end 816 .Lspectre_bhb_loop\@: 817 b . + 4 818 subs \tmp, \tmp, #1 819 b.ne .Lspectre_bhb_loop\@ 820 sb 821 #endif /* CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY */ 822 .endm 823 824 .macro mitigate_spectre_bhb_loop tmp 825 #ifdef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY 826 alternative_cb spectre_bhb_patch_loop_mitigation_enable 827 b .L_spectre_bhb_loop_done\@ // Patched to NOP 828 alternative_cb_end 829 __mitigate_spectre_bhb_loop \tmp 830 .L_spectre_bhb_loop_done\@: 831 #endif /* CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY */ 832 .endm 833 834 /* Save/restores x0-x3 to the stack */ 835 .macro __mitigate_spectre_bhb_fw 836 #ifdef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY 837 stp x0, x1, [sp, #-16]! 838 stp x2, x3, [sp, #-16]! 839 mov w0, #ARM_SMCCC_ARCH_WORKAROUND_3 840 alternative_cb smccc_patch_fw_mitigation_conduit 841 nop // Patched to SMC/HVC #0 842 alternative_cb_end 843 ldp x2, x3, [sp], #16 844 ldp x0, x1, [sp], #16 845 #endif /* CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY */ 846 .endm 847 848 .macro mitigate_spectre_bhb_clear_insn 849 #ifdef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY 850 alternative_cb spectre_bhb_patch_clearbhb 851 /* Patched to NOP when not supported */ 852 clearbhb 853 isb 854 alternative_cb_end 855 #endif /* CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY */ 856 .endm 857 #endif /* __ASM_ASSEMBLER_H */ 858