1/* SPDX-License-Identifier: GPL-2.0-only */ 2/* Copyright 2002 Andi Kleen */ 3 4#include <linux/linkage.h> 5#include <asm/errno.h> 6#include <asm/cpufeatures.h> 7#include <asm/mcsafe_test.h> 8#include <asm/alternative-asm.h> 9#include <asm/export.h> 10 11/* 12 * We build a jump to memcpy_orig by default which gets NOPped out on 13 * the majority of x86 CPUs which set REP_GOOD. In addition, CPUs which 14 * have the enhanced REP MOVSB/STOSB feature (ERMS), change those NOPs 15 * to a jmp to memcpy_erms which does the REP; MOVSB mem copy. 16 */ 17 18/* 19 * memcpy - Copy a memory block. 20 * 21 * Input: 22 * rdi destination 23 * rsi source 24 * rdx count 25 * 26 * Output: 27 * rax original destination 28 */ 29ENTRY(__memcpy) 30SYM_FUNC_START_WEAK(memcpy) 31 ALTERNATIVE_2 "jmp memcpy_orig", "", X86_FEATURE_REP_GOOD, \ 32 "jmp memcpy_erms", X86_FEATURE_ERMS 33 34 movq %rdi, %rax 35 movq %rdx, %rcx 36 shrq $3, %rcx 37 andl $7, %edx 38 rep movsq 39 movl %edx, %ecx 40 rep movsb 41 ret 42ENDPROC(memcpy) 43ENDPROC(__memcpy) 44EXPORT_SYMBOL(memcpy) 45EXPORT_SYMBOL(__memcpy) 46 47/* 48 * memcpy_erms() - enhanced fast string memcpy. This is faster and 49 * simpler than memcpy. Use memcpy_erms when possible. 50 */ 51ENTRY(memcpy_erms) 52 movq %rdi, %rax 53 movq %rdx, %rcx 54 rep movsb 55 ret 56ENDPROC(memcpy_erms) 57 58ENTRY(memcpy_orig) 59 movq %rdi, %rax 60 61 cmpq $0x20, %rdx 62 jb .Lhandle_tail 63 64 /* 65 * We check whether memory false dependence could occur, 66 * then jump to corresponding copy mode. 67 */ 68 cmp %dil, %sil 69 jl .Lcopy_backward 70 subq $0x20, %rdx 71.Lcopy_forward_loop: 72 subq $0x20, %rdx 73 74 /* 75 * Move in blocks of 4x8 bytes: 76 */ 77 movq 0*8(%rsi), %r8 78 movq 1*8(%rsi), %r9 79 movq 2*8(%rsi), %r10 80 movq 3*8(%rsi), %r11 81 leaq 4*8(%rsi), %rsi 82 83 movq %r8, 0*8(%rdi) 84 movq %r9, 1*8(%rdi) 85 movq %r10, 2*8(%rdi) 86 movq %r11, 3*8(%rdi) 87 leaq 4*8(%rdi), %rdi 88 jae .Lcopy_forward_loop 89 addl $0x20, %edx 90 jmp .Lhandle_tail 91 92.Lcopy_backward: 93 /* 94 * Calculate copy position to tail. 95 */ 96 addq %rdx, %rsi 97 addq %rdx, %rdi 98 subq $0x20, %rdx 99 /* 100 * At most 3 ALU operations in one cycle, 101 * so append NOPS in the same 16 bytes trunk. 102 */ 103 .p2align 4 104.Lcopy_backward_loop: 105 subq $0x20, %rdx 106 movq -1*8(%rsi), %r8 107 movq -2*8(%rsi), %r9 108 movq -3*8(%rsi), %r10 109 movq -4*8(%rsi), %r11 110 leaq -4*8(%rsi), %rsi 111 movq %r8, -1*8(%rdi) 112 movq %r9, -2*8(%rdi) 113 movq %r10, -3*8(%rdi) 114 movq %r11, -4*8(%rdi) 115 leaq -4*8(%rdi), %rdi 116 jae .Lcopy_backward_loop 117 118 /* 119 * Calculate copy position to head. 120 */ 121 addl $0x20, %edx 122 subq %rdx, %rsi 123 subq %rdx, %rdi 124.Lhandle_tail: 125 cmpl $16, %edx 126 jb .Lless_16bytes 127 128 /* 129 * Move data from 16 bytes to 31 bytes. 130 */ 131 movq 0*8(%rsi), %r8 132 movq 1*8(%rsi), %r9 133 movq -2*8(%rsi, %rdx), %r10 134 movq -1*8(%rsi, %rdx), %r11 135 movq %r8, 0*8(%rdi) 136 movq %r9, 1*8(%rdi) 137 movq %r10, -2*8(%rdi, %rdx) 138 movq %r11, -1*8(%rdi, %rdx) 139 retq 140 .p2align 4 141.Lless_16bytes: 142 cmpl $8, %edx 143 jb .Lless_8bytes 144 /* 145 * Move data from 8 bytes to 15 bytes. 146 */ 147 movq 0*8(%rsi), %r8 148 movq -1*8(%rsi, %rdx), %r9 149 movq %r8, 0*8(%rdi) 150 movq %r9, -1*8(%rdi, %rdx) 151 retq 152 .p2align 4 153.Lless_8bytes: 154 cmpl $4, %edx 155 jb .Lless_3bytes 156 157 /* 158 * Move data from 4 bytes to 7 bytes. 159 */ 160 movl (%rsi), %ecx 161 movl -4(%rsi, %rdx), %r8d 162 movl %ecx, (%rdi) 163 movl %r8d, -4(%rdi, %rdx) 164 retq 165 .p2align 4 166.Lless_3bytes: 167 subl $1, %edx 168 jb .Lend 169 /* 170 * Move data from 1 bytes to 3 bytes. 171 */ 172 movzbl (%rsi), %ecx 173 jz .Lstore_1byte 174 movzbq 1(%rsi), %r8 175 movzbq (%rsi, %rdx), %r9 176 movb %r8b, 1(%rdi) 177 movb %r9b, (%rdi, %rdx) 178.Lstore_1byte: 179 movb %cl, (%rdi) 180 181.Lend: 182 retq 183ENDPROC(memcpy_orig) 184 185#ifndef CONFIG_UML 186 187MCSAFE_TEST_CTL 188 189/* 190 * __memcpy_mcsafe - memory copy with machine check exception handling 191 * Note that we only catch machine checks when reading the source addresses. 192 * Writes to target are posted and don't generate machine checks. 193 */ 194ENTRY(__memcpy_mcsafe) 195 cmpl $8, %edx 196 /* Less than 8 bytes? Go to byte copy loop */ 197 jb .L_no_whole_words 198 199 /* Check for bad alignment of source */ 200 testl $7, %esi 201 /* Already aligned */ 202 jz .L_8byte_aligned 203 204 /* Copy one byte at a time until source is 8-byte aligned */ 205 movl %esi, %ecx 206 andl $7, %ecx 207 subl $8, %ecx 208 negl %ecx 209 subl %ecx, %edx 210.L_read_leading_bytes: 211 movb (%rsi), %al 212 MCSAFE_TEST_SRC %rsi 1 .E_leading_bytes 213 MCSAFE_TEST_DST %rdi 1 .E_leading_bytes 214.L_write_leading_bytes: 215 movb %al, (%rdi) 216 incq %rsi 217 incq %rdi 218 decl %ecx 219 jnz .L_read_leading_bytes 220 221.L_8byte_aligned: 222 movl %edx, %ecx 223 andl $7, %edx 224 shrl $3, %ecx 225 jz .L_no_whole_words 226 227.L_read_words: 228 movq (%rsi), %r8 229 MCSAFE_TEST_SRC %rsi 8 .E_read_words 230 MCSAFE_TEST_DST %rdi 8 .E_write_words 231.L_write_words: 232 movq %r8, (%rdi) 233 addq $8, %rsi 234 addq $8, %rdi 235 decl %ecx 236 jnz .L_read_words 237 238 /* Any trailing bytes? */ 239.L_no_whole_words: 240 andl %edx, %edx 241 jz .L_done_memcpy_trap 242 243 /* Copy trailing bytes */ 244 movl %edx, %ecx 245.L_read_trailing_bytes: 246 movb (%rsi), %al 247 MCSAFE_TEST_SRC %rsi 1 .E_trailing_bytes 248 MCSAFE_TEST_DST %rdi 1 .E_trailing_bytes 249.L_write_trailing_bytes: 250 movb %al, (%rdi) 251 incq %rsi 252 incq %rdi 253 decl %ecx 254 jnz .L_read_trailing_bytes 255 256 /* Copy successful. Return zero */ 257.L_done_memcpy_trap: 258 xorl %eax, %eax 259.L_done: 260 ret 261ENDPROC(__memcpy_mcsafe) 262EXPORT_SYMBOL_GPL(__memcpy_mcsafe) 263 264 .section .fixup, "ax" 265 /* 266 * Return number of bytes not copied for any failure. Note that 267 * there is no "tail" handling since the source buffer is 8-byte 268 * aligned and poison is cacheline aligned. 269 */ 270.E_read_words: 271 shll $3, %ecx 272.E_leading_bytes: 273 addl %edx, %ecx 274.E_trailing_bytes: 275 mov %ecx, %eax 276 jmp .L_done 277 278 /* 279 * For write fault handling, given the destination is unaligned, 280 * we handle faults on multi-byte writes with a byte-by-byte 281 * copy up to the write-protected page. 282 */ 283.E_write_words: 284 shll $3, %ecx 285 addl %edx, %ecx 286 movl %ecx, %edx 287 jmp mcsafe_handle_tail 288 289 .previous 290 291 _ASM_EXTABLE_FAULT(.L_read_leading_bytes, .E_leading_bytes) 292 _ASM_EXTABLE_FAULT(.L_read_words, .E_read_words) 293 _ASM_EXTABLE_FAULT(.L_read_trailing_bytes, .E_trailing_bytes) 294 _ASM_EXTABLE(.L_write_leading_bytes, .E_leading_bytes) 295 _ASM_EXTABLE(.L_write_words, .E_write_words) 296 _ASM_EXTABLE(.L_write_trailing_bytes, .E_trailing_bytes) 297#endif 298