1 /* SPDX-License-Identifier: GPL-2.0-only */ 2 /* 3 * Based on arch/arm/include/asm/assembler.h, arch/arm/mm/proc-macros.S 4 * 5 * Copyright (C) 1996-2000 Russell King 6 * Copyright (C) 2012 ARM Ltd. 7 */ 8 #ifndef __ASSEMBLY__ 9 #error "Only include this from assembly code" 10 #endif 11 12 #ifndef __ASM_ASSEMBLER_H 13 #define __ASM_ASSEMBLER_H 14 15 #include <asm-generic/export.h> 16 17 #include <asm/alternative.h> 18 #include <asm/asm-bug.h> 19 #include <asm/asm-extable.h> 20 #include <asm/asm-offsets.h> 21 #include <asm/cpufeature.h> 22 #include <asm/cputype.h> 23 #include <asm/debug-monitors.h> 24 #include <asm/page.h> 25 #include <asm/pgtable-hwdef.h> 26 #include <asm/ptrace.h> 27 #include <asm/thread_info.h> 28 29 /* 30 * Provide a wxN alias for each wN register so what we can paste a xN 31 * reference after a 'w' to obtain the 32-bit version. 32 */ 33 .irp n,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30 34 wx\n .req w\n 35 .endr 36 37 .macro disable_daif 38 msr daifset, #0xf 39 .endm 40 41 .macro enable_daif 42 msr daifclr, #0xf 43 .endm 44 45 /* 46 * Save/restore interrupts. 47 */ 48 .macro save_and_disable_daif, flags 49 mrs \flags, daif 50 msr daifset, #0xf 51 .endm 52 53 .macro save_and_disable_irq, flags 54 mrs \flags, daif 55 msr daifset, #3 56 .endm 57 58 .macro restore_irq, flags 59 msr daif, \flags 60 .endm 61 62 .macro enable_dbg 63 msr daifclr, #8 64 .endm 65 66 .macro disable_step_tsk, flgs, tmp 67 tbz \flgs, #TIF_SINGLESTEP, 9990f 68 mrs \tmp, mdscr_el1 69 bic \tmp, \tmp, #DBG_MDSCR_SS 70 msr mdscr_el1, \tmp 71 isb // Synchronise with enable_dbg 72 9990: 73 .endm 74 75 /* call with daif masked */ 76 .macro enable_step_tsk, flgs, tmp 77 tbz \flgs, #TIF_SINGLESTEP, 9990f 78 mrs \tmp, mdscr_el1 79 orr \tmp, \tmp, #DBG_MDSCR_SS 80 msr mdscr_el1, \tmp 81 9990: 82 .endm 83 84 /* 85 * RAS Error Synchronization barrier 86 */ 87 .macro esb 88 #ifdef CONFIG_ARM64_RAS_EXTN 89 hint #16 90 #else 91 nop 92 #endif 93 .endm 94 95 /* 96 * Value prediction barrier 97 */ 98 .macro csdb 99 hint #20 100 .endm 101 102 /* 103 * Clear Branch History instruction 104 */ 105 .macro clearbhb 106 hint #22 107 .endm 108 109 /* 110 * Speculation barrier 111 */ 112 .macro sb 113 alternative_if_not ARM64_HAS_SB 114 dsb nsh 115 isb 116 alternative_else 117 SB_BARRIER_INSN 118 nop 119 alternative_endif 120 .endm 121 122 /* 123 * NOP sequence 124 */ 125 .macro nops, num 126 .rept \num 127 nop 128 .endr 129 .endm 130 131 /* 132 * Register aliases. 133 */ 134 lr .req x30 // link register 135 136 /* 137 * Vector entry 138 */ 139 .macro ventry label 140 .align 7 141 b \label 142 .endm 143 144 /* 145 * Select code when configured for BE. 146 */ 147 #ifdef CONFIG_CPU_BIG_ENDIAN 148 #define CPU_BE(code...) code 149 #else 150 #define CPU_BE(code...) 151 #endif 152 153 /* 154 * Select code when configured for LE. 155 */ 156 #ifdef CONFIG_CPU_BIG_ENDIAN 157 #define CPU_LE(code...) 158 #else 159 #define CPU_LE(code...) code 160 #endif 161 162 /* 163 * Define a macro that constructs a 64-bit value by concatenating two 164 * 32-bit registers. Note that on big endian systems the order of the 165 * registers is swapped. 166 */ 167 #ifndef CONFIG_CPU_BIG_ENDIAN 168 .macro regs_to_64, rd, lbits, hbits 169 #else 170 .macro regs_to_64, rd, hbits, lbits 171 #endif 172 orr \rd, \lbits, \hbits, lsl #32 173 .endm 174 175 /* 176 * Pseudo-ops for PC-relative adr/ldr/str <reg>, <symbol> where 177 * <symbol> is within the range +/- 4 GB of the PC. 178 */ 179 /* 180 * @dst: destination register (64 bit wide) 181 * @sym: name of the symbol 182 */ 183 .macro adr_l, dst, sym 184 adrp \dst, \sym 185 add \dst, \dst, :lo12:\sym 186 .endm 187 188 /* 189 * @dst: destination register (32 or 64 bit wide) 190 * @sym: name of the symbol 191 * @tmp: optional 64-bit scratch register to be used if <dst> is a 192 * 32-bit wide register, in which case it cannot be used to hold 193 * the address 194 */ 195 .macro ldr_l, dst, sym, tmp= 196 .ifb \tmp 197 adrp \dst, \sym 198 ldr \dst, [\dst, :lo12:\sym] 199 .else 200 adrp \tmp, \sym 201 ldr \dst, [\tmp, :lo12:\sym] 202 .endif 203 .endm 204 205 /* 206 * @src: source register (32 or 64 bit wide) 207 * @sym: name of the symbol 208 * @tmp: mandatory 64-bit scratch register to calculate the address 209 * while <src> needs to be preserved. 210 */ 211 .macro str_l, src, sym, tmp 212 adrp \tmp, \sym 213 str \src, [\tmp, :lo12:\sym] 214 .endm 215 216 /* 217 * @dst: destination register 218 */ 219 #if defined(__KVM_NVHE_HYPERVISOR__) || defined(__KVM_VHE_HYPERVISOR__) 220 .macro get_this_cpu_offset, dst 221 mrs \dst, tpidr_el2 222 .endm 223 #else 224 .macro get_this_cpu_offset, dst 225 alternative_if_not ARM64_HAS_VIRT_HOST_EXTN 226 mrs \dst, tpidr_el1 227 alternative_else 228 mrs \dst, tpidr_el2 229 alternative_endif 230 .endm 231 232 .macro set_this_cpu_offset, src 233 alternative_if_not ARM64_HAS_VIRT_HOST_EXTN 234 msr tpidr_el1, \src 235 alternative_else 236 msr tpidr_el2, \src 237 alternative_endif 238 .endm 239 #endif 240 241 /* 242 * @dst: Result of per_cpu(sym, smp_processor_id()) (can be SP) 243 * @sym: The name of the per-cpu variable 244 * @tmp: scratch register 245 */ 246 .macro adr_this_cpu, dst, sym, tmp 247 adrp \tmp, \sym 248 add \dst, \tmp, #:lo12:\sym 249 get_this_cpu_offset \tmp 250 add \dst, \dst, \tmp 251 .endm 252 253 /* 254 * @dst: Result of READ_ONCE(per_cpu(sym, smp_processor_id())) 255 * @sym: The name of the per-cpu variable 256 * @tmp: scratch register 257 */ 258 .macro ldr_this_cpu dst, sym, tmp 259 adr_l \dst, \sym 260 get_this_cpu_offset \tmp 261 ldr \dst, [\dst, \tmp] 262 .endm 263 264 /* 265 * vma_vm_mm - get mm pointer from vma pointer (vma->vm_mm) 266 */ 267 .macro vma_vm_mm, rd, rn 268 ldr \rd, [\rn, #VMA_VM_MM] 269 .endm 270 271 /* 272 * read_ctr - read CTR_EL0. If the system has mismatched register fields, 273 * provide the system wide safe value from arm64_ftr_reg_ctrel0.sys_val 274 */ 275 .macro read_ctr, reg 276 #ifndef __KVM_NVHE_HYPERVISOR__ 277 alternative_if_not ARM64_MISMATCHED_CACHE_TYPE 278 mrs \reg, ctr_el0 // read CTR 279 nop 280 alternative_else 281 ldr_l \reg, arm64_ftr_reg_ctrel0 + ARM64_FTR_SYSVAL 282 alternative_endif 283 #else 284 alternative_if_not ARM64_KVM_PROTECTED_MODE 285 ASM_BUG() 286 alternative_else_nop_endif 287 alternative_cb ARM64_ALWAYS_SYSTEM, kvm_compute_final_ctr_el0 288 movz \reg, #0 289 movk \reg, #0, lsl #16 290 movk \reg, #0, lsl #32 291 movk \reg, #0, lsl #48 292 alternative_cb_end 293 #endif 294 .endm 295 296 297 /* 298 * raw_dcache_line_size - get the minimum D-cache line size on this CPU 299 * from the CTR register. 300 */ 301 .macro raw_dcache_line_size, reg, tmp 302 mrs \tmp, ctr_el0 // read CTR 303 ubfm \tmp, \tmp, #16, #19 // cache line size encoding 304 mov \reg, #4 // bytes per word 305 lsl \reg, \reg, \tmp // actual cache line size 306 .endm 307 308 /* 309 * dcache_line_size - get the safe D-cache line size across all CPUs 310 */ 311 .macro dcache_line_size, reg, tmp 312 read_ctr \tmp 313 ubfm \tmp, \tmp, #16, #19 // cache line size encoding 314 mov \reg, #4 // bytes per word 315 lsl \reg, \reg, \tmp // actual cache line size 316 .endm 317 318 /* 319 * raw_icache_line_size - get the minimum I-cache line size on this CPU 320 * from the CTR register. 321 */ 322 .macro raw_icache_line_size, reg, tmp 323 mrs \tmp, ctr_el0 // read CTR 324 and \tmp, \tmp, #0xf // cache line size encoding 325 mov \reg, #4 // bytes per word 326 lsl \reg, \reg, \tmp // actual cache line size 327 .endm 328 329 /* 330 * icache_line_size - get the safe I-cache line size across all CPUs 331 */ 332 .macro icache_line_size, reg, tmp 333 read_ctr \tmp 334 and \tmp, \tmp, #0xf // cache line size encoding 335 mov \reg, #4 // bytes per word 336 lsl \reg, \reg, \tmp // actual cache line size 337 .endm 338 339 /* 340 * tcr_set_t0sz - update TCR.T0SZ so that we can load the ID map 341 */ 342 .macro tcr_set_t0sz, valreg, t0sz 343 bfi \valreg, \t0sz, #TCR_T0SZ_OFFSET, #TCR_TxSZ_WIDTH 344 .endm 345 346 /* 347 * tcr_set_t1sz - update TCR.T1SZ 348 */ 349 .macro tcr_set_t1sz, valreg, t1sz 350 bfi \valreg, \t1sz, #TCR_T1SZ_OFFSET, #TCR_TxSZ_WIDTH 351 .endm 352 353 /* 354 * idmap_get_t0sz - get the T0SZ value needed to cover the ID map 355 * 356 * Calculate the maximum allowed value for TCR_EL1.T0SZ so that the 357 * entire ID map region can be mapped. As T0SZ == (64 - #bits used), 358 * this number conveniently equals the number of leading zeroes in 359 * the physical address of _end. 360 */ 361 .macro idmap_get_t0sz, reg 362 adrp \reg, _end 363 orr \reg, \reg, #(1 << VA_BITS_MIN) - 1 364 clz \reg, \reg 365 .endm 366 367 /* 368 * tcr_compute_pa_size - set TCR.(I)PS to the highest supported 369 * ID_AA64MMFR0_EL1.PARange value 370 * 371 * tcr: register with the TCR_ELx value to be updated 372 * pos: IPS or PS bitfield position 373 * tmp{0,1}: temporary registers 374 */ 375 .macro tcr_compute_pa_size, tcr, pos, tmp0, tmp1 376 mrs \tmp0, ID_AA64MMFR0_EL1 377 // Narrow PARange to fit the PS field in TCR_ELx 378 ubfx \tmp0, \tmp0, #ID_AA64MMFR0_EL1_PARANGE_SHIFT, #3 379 mov \tmp1, #ID_AA64MMFR0_EL1_PARANGE_MAX 380 cmp \tmp0, \tmp1 381 csel \tmp0, \tmp1, \tmp0, hi 382 bfi \tcr, \tmp0, \pos, #3 383 .endm 384 385 .macro __dcache_op_workaround_clean_cache, op, addr 386 alternative_if_not ARM64_WORKAROUND_CLEAN_CACHE 387 dc \op, \addr 388 alternative_else 389 dc civac, \addr 390 alternative_endif 391 .endm 392 393 /* 394 * Macro to perform a data cache maintenance for the interval 395 * [start, end) with dcache line size explicitly provided. 396 * 397 * op: operation passed to dc instruction 398 * domain: domain used in dsb instruciton 399 * start: starting virtual address of the region 400 * end: end virtual address of the region 401 * linesz: dcache line size 402 * fixup: optional label to branch to on user fault 403 * Corrupts: start, end, tmp 404 */ 405 .macro dcache_by_myline_op op, domain, start, end, linesz, tmp, fixup 406 sub \tmp, \linesz, #1 407 bic \start, \start, \tmp 408 .Ldcache_op\@: 409 .ifc \op, cvau 410 __dcache_op_workaround_clean_cache \op, \start 411 .else 412 .ifc \op, cvac 413 __dcache_op_workaround_clean_cache \op, \start 414 .else 415 .ifc \op, cvap 416 sys 3, c7, c12, 1, \start // dc cvap 417 .else 418 .ifc \op, cvadp 419 sys 3, c7, c13, 1, \start // dc cvadp 420 .else 421 dc \op, \start 422 .endif 423 .endif 424 .endif 425 .endif 426 add \start, \start, \linesz 427 cmp \start, \end 428 b.lo .Ldcache_op\@ 429 dsb \domain 430 431 _cond_uaccess_extable .Ldcache_op\@, \fixup 432 .endm 433 434 /* 435 * Macro to perform a data cache maintenance for the interval 436 * [start, end) 437 * 438 * op: operation passed to dc instruction 439 * domain: domain used in dsb instruciton 440 * start: starting virtual address of the region 441 * end: end virtual address of the region 442 * fixup: optional label to branch to on user fault 443 * Corrupts: start, end, tmp1, tmp2 444 */ 445 .macro dcache_by_line_op op, domain, start, end, tmp1, tmp2, fixup 446 dcache_line_size \tmp1, \tmp2 447 dcache_by_myline_op \op, \domain, \start, \end, \tmp1, \tmp2, \fixup 448 .endm 449 450 /* 451 * Macro to perform an instruction cache maintenance for the interval 452 * [start, end) 453 * 454 * start, end: virtual addresses describing the region 455 * fixup: optional label to branch to on user fault 456 * Corrupts: tmp1, tmp2 457 */ 458 .macro invalidate_icache_by_line start, end, tmp1, tmp2, fixup 459 icache_line_size \tmp1, \tmp2 460 sub \tmp2, \tmp1, #1 461 bic \tmp2, \start, \tmp2 462 .Licache_op\@: 463 ic ivau, \tmp2 // invalidate I line PoU 464 add \tmp2, \tmp2, \tmp1 465 cmp \tmp2, \end 466 b.lo .Licache_op\@ 467 dsb ish 468 isb 469 470 _cond_uaccess_extable .Licache_op\@, \fixup 471 .endm 472 473 /* 474 * load_ttbr1 - install @pgtbl as a TTBR1 page table 475 * pgtbl preserved 476 * tmp1/tmp2 clobbered, either may overlap with pgtbl 477 */ 478 .macro load_ttbr1, pgtbl, tmp1, tmp2 479 phys_to_ttbr \tmp1, \pgtbl 480 offset_ttbr1 \tmp1, \tmp2 481 msr ttbr1_el1, \tmp1 482 isb 483 .endm 484 485 /* 486 * To prevent the possibility of old and new partial table walks being visible 487 * in the tlb, switch the ttbr to a zero page when we invalidate the old 488 * records. D4.7.1 'General TLB maintenance requirements' in ARM DDI 0487A.i 489 * Even switching to our copied tables will cause a changed output address at 490 * each stage of the walk. 491 */ 492 .macro break_before_make_ttbr_switch zero_page, page_table, tmp, tmp2 493 phys_to_ttbr \tmp, \zero_page 494 msr ttbr1_el1, \tmp 495 isb 496 tlbi vmalle1 497 dsb nsh 498 load_ttbr1 \page_table, \tmp, \tmp2 499 .endm 500 501 /* 502 * reset_pmuserenr_el0 - reset PMUSERENR_EL0 if PMUv3 present 503 */ 504 .macro reset_pmuserenr_el0, tmpreg 505 mrs \tmpreg, id_aa64dfr0_el1 506 sbfx \tmpreg, \tmpreg, #ID_AA64DFR0_EL1_PMUVer_SHIFT, #4 507 cmp \tmpreg, #1 // Skip if no PMU present 508 b.lt 9000f 509 msr pmuserenr_el0, xzr // Disable PMU access from EL0 510 9000: 511 .endm 512 513 /* 514 * reset_amuserenr_el0 - reset AMUSERENR_EL0 if AMUv1 present 515 */ 516 .macro reset_amuserenr_el0, tmpreg 517 mrs \tmpreg, id_aa64pfr0_el1 // Check ID_AA64PFR0_EL1 518 ubfx \tmpreg, \tmpreg, #ID_AA64PFR0_EL1_AMU_SHIFT, #4 519 cbz \tmpreg, .Lskip_\@ // Skip if no AMU present 520 msr_s SYS_AMUSERENR_EL0, xzr // Disable AMU access from EL0 521 .Lskip_\@: 522 .endm 523 /* 524 * copy_page - copy src to dest using temp registers t1-t8 525 */ 526 .macro copy_page dest:req src:req t1:req t2:req t3:req t4:req t5:req t6:req t7:req t8:req 527 9998: ldp \t1, \t2, [\src] 528 ldp \t3, \t4, [\src, #16] 529 ldp \t5, \t6, [\src, #32] 530 ldp \t7, \t8, [\src, #48] 531 add \src, \src, #64 532 stnp \t1, \t2, [\dest] 533 stnp \t3, \t4, [\dest, #16] 534 stnp \t5, \t6, [\dest, #32] 535 stnp \t7, \t8, [\dest, #48] 536 add \dest, \dest, #64 537 tst \src, #(PAGE_SIZE - 1) 538 b.ne 9998b 539 .endm 540 541 /* 542 * Annotate a function as being unsuitable for kprobes. 543 */ 544 #ifdef CONFIG_KPROBES 545 #define NOKPROBE(x) \ 546 .pushsection "_kprobe_blacklist", "aw"; \ 547 .quad x; \ 548 .popsection; 549 #else 550 #define NOKPROBE(x) 551 #endif 552 553 #if defined(CONFIG_KASAN_GENERIC) || defined(CONFIG_KASAN_SW_TAGS) 554 #define EXPORT_SYMBOL_NOKASAN(name) 555 #else 556 #define EXPORT_SYMBOL_NOKASAN(name) EXPORT_SYMBOL(name) 557 #endif 558 559 /* 560 * Emit a 64-bit absolute little endian symbol reference in a way that 561 * ensures that it will be resolved at build time, even when building a 562 * PIE binary. This requires cooperation from the linker script, which 563 * must emit the lo32/hi32 halves individually. 564 */ 565 .macro le64sym, sym 566 .long \sym\()_lo32 567 .long \sym\()_hi32 568 .endm 569 570 /* 571 * mov_q - move an immediate constant into a 64-bit register using 572 * between 2 and 4 movz/movk instructions (depending on the 573 * magnitude and sign of the operand) 574 */ 575 .macro mov_q, reg, val 576 .if (((\val) >> 31) == 0 || ((\val) >> 31) == 0x1ffffffff) 577 movz \reg, :abs_g1_s:\val 578 .else 579 .if (((\val) >> 47) == 0 || ((\val) >> 47) == 0x1ffff) 580 movz \reg, :abs_g2_s:\val 581 .else 582 movz \reg, :abs_g3:\val 583 movk \reg, :abs_g2_nc:\val 584 .endif 585 movk \reg, :abs_g1_nc:\val 586 .endif 587 movk \reg, :abs_g0_nc:\val 588 .endm 589 590 /* 591 * Return the current task_struct. 592 */ 593 .macro get_current_task, rd 594 mrs \rd, sp_el0 595 .endm 596 597 /* 598 * Offset ttbr1 to allow for 48-bit kernel VAs set with 52-bit PTRS_PER_PGD. 599 * orr is used as it can cover the immediate value (and is idempotent). 600 * In future this may be nop'ed out when dealing with 52-bit kernel VAs. 601 * ttbr: Value of ttbr to set, modified. 602 */ 603 .macro offset_ttbr1, ttbr, tmp 604 #ifdef CONFIG_ARM64_VA_BITS_52 605 mrs_s \tmp, SYS_ID_AA64MMFR2_EL1 606 and \tmp, \tmp, #(0xf << ID_AA64MMFR2_EL1_VARange_SHIFT) 607 cbnz \tmp, .Lskipoffs_\@ 608 orr \ttbr, \ttbr, #TTBR1_BADDR_4852_OFFSET 609 .Lskipoffs_\@ : 610 #endif 611 .endm 612 613 /* 614 * Arrange a physical address in a TTBR register, taking care of 52-bit 615 * addresses. 616 * 617 * phys: physical address, preserved 618 * ttbr: returns the TTBR value 619 */ 620 .macro phys_to_ttbr, ttbr, phys 621 #ifdef CONFIG_ARM64_PA_BITS_52 622 orr \ttbr, \phys, \phys, lsr #46 623 and \ttbr, \ttbr, #TTBR_BADDR_MASK_52 624 #else 625 mov \ttbr, \phys 626 #endif 627 .endm 628 629 .macro phys_to_pte, pte, phys 630 #ifdef CONFIG_ARM64_PA_BITS_52 631 /* 632 * We assume \phys is 64K aligned and this is guaranteed by only 633 * supporting this configuration with 64K pages. 634 */ 635 orr \pte, \phys, \phys, lsr #36 636 and \pte, \pte, #PTE_ADDR_MASK 637 #else 638 mov \pte, \phys 639 #endif 640 .endm 641 642 .macro pte_to_phys, phys, pte 643 and \phys, \pte, #PTE_ADDR_MASK 644 #ifdef CONFIG_ARM64_PA_BITS_52 645 orr \phys, \phys, \phys, lsl #PTE_ADDR_HIGH_SHIFT 646 and \phys, \phys, GENMASK_ULL(PHYS_MASK_SHIFT - 1, PAGE_SHIFT) 647 #endif 648 .endm 649 650 /* 651 * tcr_clear_errata_bits - Clear TCR bits that trigger an errata on this CPU. 652 */ 653 .macro tcr_clear_errata_bits, tcr, tmp1, tmp2 654 #ifdef CONFIG_FUJITSU_ERRATUM_010001 655 mrs \tmp1, midr_el1 656 657 mov_q \tmp2, MIDR_FUJITSU_ERRATUM_010001_MASK 658 and \tmp1, \tmp1, \tmp2 659 mov_q \tmp2, MIDR_FUJITSU_ERRATUM_010001 660 cmp \tmp1, \tmp2 661 b.ne 10f 662 663 mov_q \tmp2, TCR_CLEAR_FUJITSU_ERRATUM_010001 664 bic \tcr, \tcr, \tmp2 665 10: 666 #endif /* CONFIG_FUJITSU_ERRATUM_010001 */ 667 .endm 668 669 /** 670 * Errata workaround prior to disable MMU. Insert an ISB immediately prior 671 * to executing the MSR that will change SCTLR_ELn[M] from a value of 1 to 0. 672 */ 673 .macro pre_disable_mmu_workaround 674 #ifdef CONFIG_QCOM_FALKOR_ERRATUM_E1041 675 isb 676 #endif 677 .endm 678 679 /* 680 * frame_push - Push @regcount callee saved registers to the stack, 681 * starting at x19, as well as x29/x30, and set x29 to 682 * the new value of sp. Add @extra bytes of stack space 683 * for locals. 684 */ 685 .macro frame_push, regcount:req, extra 686 __frame st, \regcount, \extra 687 .endm 688 689 /* 690 * frame_pop - Pop the callee saved registers from the stack that were 691 * pushed in the most recent call to frame_push, as well 692 * as x29/x30 and any extra stack space that may have been 693 * allocated. 694 */ 695 .macro frame_pop 696 __frame ld 697 .endm 698 699 .macro __frame_regs, reg1, reg2, op, num 700 .if .Lframe_regcount == \num 701 \op\()r \reg1, [sp, #(\num + 1) * 8] 702 .elseif .Lframe_regcount > \num 703 \op\()p \reg1, \reg2, [sp, #(\num + 1) * 8] 704 .endif 705 .endm 706 707 .macro __frame, op, regcount, extra=0 708 .ifc \op, st 709 .if (\regcount) < 0 || (\regcount) > 10 710 .error "regcount should be in the range [0 ... 10]" 711 .endif 712 .if ((\extra) % 16) != 0 713 .error "extra should be a multiple of 16 bytes" 714 .endif 715 .ifdef .Lframe_regcount 716 .if .Lframe_regcount != -1 717 .error "frame_push/frame_pop may not be nested" 718 .endif 719 .endif 720 .set .Lframe_regcount, \regcount 721 .set .Lframe_extra, \extra 722 .set .Lframe_local_offset, ((\regcount + 3) / 2) * 16 723 stp x29, x30, [sp, #-.Lframe_local_offset - .Lframe_extra]! 724 mov x29, sp 725 .endif 726 727 __frame_regs x19, x20, \op, 1 728 __frame_regs x21, x22, \op, 3 729 __frame_regs x23, x24, \op, 5 730 __frame_regs x25, x26, \op, 7 731 __frame_regs x27, x28, \op, 9 732 733 .ifc \op, ld 734 .if .Lframe_regcount == -1 735 .error "frame_push/frame_pop may not be nested" 736 .endif 737 ldp x29, x30, [sp], #.Lframe_local_offset + .Lframe_extra 738 .set .Lframe_regcount, -1 739 .endif 740 .endm 741 742 /* 743 * Set SCTLR_ELx to the @reg value, and invalidate the local icache 744 * in the process. This is called when setting the MMU on. 745 */ 746 .macro set_sctlr, sreg, reg 747 msr \sreg, \reg 748 isb 749 /* 750 * Invalidate the local I-cache so that any instructions fetched 751 * speculatively from the PoC are discarded, since they may have 752 * been dynamically patched at the PoU. 753 */ 754 ic iallu 755 dsb nsh 756 isb 757 .endm 758 759 .macro set_sctlr_el1, reg 760 set_sctlr sctlr_el1, \reg 761 .endm 762 763 .macro set_sctlr_el2, reg 764 set_sctlr sctlr_el2, \reg 765 .endm 766 767 /* 768 * Check whether preempt/bh-disabled asm code should yield as soon as 769 * it is able. This is the case if we are currently running in task 770 * context, and either a softirq is pending, or the TIF_NEED_RESCHED 771 * flag is set and re-enabling preemption a single time would result in 772 * a preempt count of zero. (Note that the TIF_NEED_RESCHED flag is 773 * stored negated in the top word of the thread_info::preempt_count 774 * field) 775 */ 776 .macro cond_yield, lbl:req, tmp:req, tmp2:req 777 get_current_task \tmp 778 ldr \tmp, [\tmp, #TSK_TI_PREEMPT] 779 /* 780 * If we are serving a softirq, there is no point in yielding: the 781 * softirq will not be preempted no matter what we do, so we should 782 * run to completion as quickly as we can. 783 */ 784 tbnz \tmp, #SOFTIRQ_SHIFT, .Lnoyield_\@ 785 #ifdef CONFIG_PREEMPTION 786 sub \tmp, \tmp, #PREEMPT_DISABLE_OFFSET 787 cbz \tmp, \lbl 788 #endif 789 adr_l \tmp, irq_stat + IRQ_CPUSTAT_SOFTIRQ_PENDING 790 get_this_cpu_offset \tmp2 791 ldr w\tmp, [\tmp, \tmp2] 792 cbnz w\tmp, \lbl // yield on pending softirq in task context 793 .Lnoyield_\@: 794 .endm 795 796 /* 797 * Branch Target Identifier (BTI) 798 */ 799 .macro bti, targets 800 .equ .L__bti_targets_c, 34 801 .equ .L__bti_targets_j, 36 802 .equ .L__bti_targets_jc,38 803 hint #.L__bti_targets_\targets 804 .endm 805 806 /* 807 * This macro emits a program property note section identifying 808 * architecture features which require special handling, mainly for 809 * use in assembly files included in the VDSO. 810 */ 811 812 #define NT_GNU_PROPERTY_TYPE_0 5 813 #define GNU_PROPERTY_AARCH64_FEATURE_1_AND 0xc0000000 814 815 #define GNU_PROPERTY_AARCH64_FEATURE_1_BTI (1U << 0) 816 #define GNU_PROPERTY_AARCH64_FEATURE_1_PAC (1U << 1) 817 818 #ifdef CONFIG_ARM64_BTI_KERNEL 819 #define GNU_PROPERTY_AARCH64_FEATURE_1_DEFAULT \ 820 ((GNU_PROPERTY_AARCH64_FEATURE_1_BTI | \ 821 GNU_PROPERTY_AARCH64_FEATURE_1_PAC)) 822 #endif 823 824 #ifdef GNU_PROPERTY_AARCH64_FEATURE_1_DEFAULT 825 .macro emit_aarch64_feature_1_and, feat=GNU_PROPERTY_AARCH64_FEATURE_1_DEFAULT 826 .pushsection .note.gnu.property, "a" 827 .align 3 828 .long 2f - 1f 829 .long 6f - 3f 830 .long NT_GNU_PROPERTY_TYPE_0 831 1: .string "GNU" 832 2: 833 .align 3 834 3: .long GNU_PROPERTY_AARCH64_FEATURE_1_AND 835 .long 5f - 4f 836 4: 837 /* 838 * This is described with an array of char in the Linux API 839 * spec but the text and all other usage (including binutils, 840 * clang and GCC) treat this as a 32 bit value so no swizzling 841 * is required for big endian. 842 */ 843 .long \feat 844 5: 845 .align 3 846 6: 847 .popsection 848 .endm 849 850 #else 851 .macro emit_aarch64_feature_1_and, feat=0 852 .endm 853 854 #endif /* GNU_PROPERTY_AARCH64_FEATURE_1_DEFAULT */ 855 856 .macro __mitigate_spectre_bhb_loop tmp 857 #ifdef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY 858 alternative_cb ARM64_ALWAYS_SYSTEM, spectre_bhb_patch_loop_iter 859 mov \tmp, #32 // Patched to correct the immediate 860 alternative_cb_end 861 .Lspectre_bhb_loop\@: 862 b . + 4 863 subs \tmp, \tmp, #1 864 b.ne .Lspectre_bhb_loop\@ 865 sb 866 #endif /* CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY */ 867 .endm 868 869 .macro mitigate_spectre_bhb_loop tmp 870 #ifdef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY 871 alternative_cb ARM64_ALWAYS_SYSTEM, spectre_bhb_patch_loop_mitigation_enable 872 b .L_spectre_bhb_loop_done\@ // Patched to NOP 873 alternative_cb_end 874 __mitigate_spectre_bhb_loop \tmp 875 .L_spectre_bhb_loop_done\@: 876 #endif /* CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY */ 877 .endm 878 879 /* Save/restores x0-x3 to the stack */ 880 .macro __mitigate_spectre_bhb_fw 881 #ifdef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY 882 stp x0, x1, [sp, #-16]! 883 stp x2, x3, [sp, #-16]! 884 mov w0, #ARM_SMCCC_ARCH_WORKAROUND_3 885 alternative_cb ARM64_ALWAYS_SYSTEM, smccc_patch_fw_mitigation_conduit 886 nop // Patched to SMC/HVC #0 887 alternative_cb_end 888 ldp x2, x3, [sp], #16 889 ldp x0, x1, [sp], #16 890 #endif /* CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY */ 891 .endm 892 893 .macro mitigate_spectre_bhb_clear_insn 894 #ifdef CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY 895 alternative_cb ARM64_ALWAYS_SYSTEM, spectre_bhb_patch_clearbhb 896 /* Patched to NOP when not supported */ 897 clearbhb 898 isb 899 alternative_cb_end 900 #endif /* CONFIG_MITIGATE_SPECTRE_BRANCH_HISTORY */ 901 .endm 902 #endif /* __ASM_ASSEMBLER_H */ 903