1#! /usr/bin/env perl 2# Copyright 2011-2016 The OpenSSL Project Authors. All Rights Reserved. 3# 4# Licensed under the OpenSSL license (the "License"). You may not use 5# this file except in compliance with the License. You can obtain a copy 6# in the file LICENSE in the source distribution or at 7# https://www.openssl.org/source/license.html 8 9 10###################################################################### 11## Constant-time SSSE3 AES core implementation. 12## version 0.1 13## 14## By Mike Hamburg (Stanford University), 2009 15## Public domain. 16## 17## For details see http://shiftleft.org/papers/vector_aes/ and 18## http://crypto.stanford.edu/vpaes/. 19 20###################################################################### 21# September 2011. 22# 23# Interface to OpenSSL as "almost" drop-in replacement for 24# aes-x86_64.pl. "Almost" refers to the fact that AES_cbc_encrypt 25# doesn't handle partial vectors (doesn't have to if called from 26# EVP only). "Drop-in" implies that this module doesn't share key 27# schedule structure with the original nor does it make assumption 28# about its alignment... 29# 30# Performance summary. aes-x86_64.pl column lists large-block CBC 31# encrypt/decrypt/with-hyper-threading-off(*) results in cycles per 32# byte processed with 128-bit key, and vpaes-x86_64.pl column - 33# [also large-block CBC] encrypt/decrypt. 34# 35# aes-x86_64.pl vpaes-x86_64.pl 36# 37# Core 2(**) 29.6/41.1/14.3 21.9/25.2(***) 38# Nehalem 29.6/40.3/14.6 10.0/11.8 39# Atom 57.3/74.2/32.1 60.9/77.2(***) 40# Silvermont 52.7/64.0/19.5 48.8/60.8(***) 41# Goldmont 38.9/49.0/17.8 10.6/12.6 42# 43# (*) "Hyper-threading" in the context refers rather to cache shared 44# among multiple cores, than to specifically Intel HTT. As vast 45# majority of contemporary cores share cache, slower code path 46# is common place. In other words "with-hyper-threading-off" 47# results are presented mostly for reference purposes. 48# 49# (**) "Core 2" refers to initial 65nm design, a.k.a. Conroe. 50# 51# (***) Less impressive improvement on Core 2 and Atom is due to slow 52# pshufb, yet it's respectable +36%/62% improvement on Core 2 53# (as implied, over "hyper-threading-safe" code path). 54# 55# <appro@openssl.org> 56 57$flavour = shift; 58$output = shift; 59if ($flavour =~ /\./) { $output = $flavour; undef $flavour; } 60 61$win64=0; $win64=1 if ($flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/); 62 63$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1; 64( $xlate="${dir}x86_64-xlate.pl" and -f $xlate ) or 65( $xlate="${dir}../../../perlasm/x86_64-xlate.pl" and -f $xlate) or 66die "can't locate x86_64-xlate.pl"; 67 68open OUT,"| \"$^X\" \"$xlate\" $flavour \"$output\""; 69*STDOUT=*OUT; 70 71$PREFIX="vpaes"; 72 73$code.=<<___; 74.text 75 76## 77## _aes_encrypt_core 78## 79## AES-encrypt %xmm0. 80## 81## Inputs: 82## %xmm0 = input 83## %xmm9-%xmm15 as in _vpaes_preheat 84## (%rdx) = scheduled keys 85## 86## Output in %xmm0 87## Clobbers %xmm1-%xmm5, %r9, %r10, %r11, %rax 88## Preserves %xmm6 - %xmm8 so you get some local vectors 89## 90## 91.type _vpaes_encrypt_core,\@abi-omnipotent 92.align 16 93_vpaes_encrypt_core: 94.cfi_startproc 95 mov %rdx, %r9 96 mov \$16, %r11 97 mov 240(%rdx),%eax 98 movdqa %xmm9, %xmm1 99 movdqa .Lk_ipt(%rip), %xmm2 # iptlo 100 pandn %xmm0, %xmm1 101 movdqu (%r9), %xmm5 # round0 key 102 psrld \$4, %xmm1 103 pand %xmm9, %xmm0 104 pshufb %xmm0, %xmm2 105 movdqa .Lk_ipt+16(%rip), %xmm0 # ipthi 106 pshufb %xmm1, %xmm0 107 pxor %xmm5, %xmm2 108 add \$16, %r9 109 pxor %xmm2, %xmm0 110 lea .Lk_mc_backward(%rip),%r10 111 jmp .Lenc_entry 112 113.align 16 114.Lenc_loop: 115 # middle of middle round 116 movdqa %xmm13, %xmm4 # 4 : sb1u 117 movdqa %xmm12, %xmm0 # 0 : sb1t 118 pshufb %xmm2, %xmm4 # 4 = sb1u 119 pshufb %xmm3, %xmm0 # 0 = sb1t 120 pxor %xmm5, %xmm4 # 4 = sb1u + k 121 movdqa %xmm15, %xmm5 # 4 : sb2u 122 pxor %xmm4, %xmm0 # 0 = A 123 movdqa -0x40(%r11,%r10), %xmm1 # .Lk_mc_forward[] 124 pshufb %xmm2, %xmm5 # 4 = sb2u 125 movdqa (%r11,%r10), %xmm4 # .Lk_mc_backward[] 126 movdqa %xmm14, %xmm2 # 2 : sb2t 127 pshufb %xmm3, %xmm2 # 2 = sb2t 128 movdqa %xmm0, %xmm3 # 3 = A 129 pxor %xmm5, %xmm2 # 2 = 2A 130 pshufb %xmm1, %xmm0 # 0 = B 131 add \$16, %r9 # next key 132 pxor %xmm2, %xmm0 # 0 = 2A+B 133 pshufb %xmm4, %xmm3 # 3 = D 134 add \$16, %r11 # next mc 135 pxor %xmm0, %xmm3 # 3 = 2A+B+D 136 pshufb %xmm1, %xmm0 # 0 = 2B+C 137 and \$0x30, %r11 # ... mod 4 138 sub \$1,%rax # nr-- 139 pxor %xmm3, %xmm0 # 0 = 2A+3B+C+D 140 141.Lenc_entry: 142 # top of round 143 movdqa %xmm9, %xmm1 # 1 : i 144 movdqa %xmm11, %xmm5 # 2 : a/k 145 pandn %xmm0, %xmm1 # 1 = i<<4 146 psrld \$4, %xmm1 # 1 = i 147 pand %xmm9, %xmm0 # 0 = k 148 pshufb %xmm0, %xmm5 # 2 = a/k 149 movdqa %xmm10, %xmm3 # 3 : 1/i 150 pxor %xmm1, %xmm0 # 0 = j 151 pshufb %xmm1, %xmm3 # 3 = 1/i 152 movdqa %xmm10, %xmm4 # 4 : 1/j 153 pxor %xmm5, %xmm3 # 3 = iak = 1/i + a/k 154 pshufb %xmm0, %xmm4 # 4 = 1/j 155 movdqa %xmm10, %xmm2 # 2 : 1/iak 156 pxor %xmm5, %xmm4 # 4 = jak = 1/j + a/k 157 pshufb %xmm3, %xmm2 # 2 = 1/iak 158 movdqa %xmm10, %xmm3 # 3 : 1/jak 159 pxor %xmm0, %xmm2 # 2 = io 160 pshufb %xmm4, %xmm3 # 3 = 1/jak 161 movdqu (%r9), %xmm5 162 pxor %xmm1, %xmm3 # 3 = jo 163 jnz .Lenc_loop 164 165 # middle of last round 166 movdqa -0x60(%r10), %xmm4 # 3 : sbou .Lk_sbo 167 movdqa -0x50(%r10), %xmm0 # 0 : sbot .Lk_sbo+16 168 pshufb %xmm2, %xmm4 # 4 = sbou 169 pxor %xmm5, %xmm4 # 4 = sb1u + k 170 pshufb %xmm3, %xmm0 # 0 = sb1t 171 movdqa 0x40(%r11,%r10), %xmm1 # .Lk_sr[] 172 pxor %xmm4, %xmm0 # 0 = A 173 pshufb %xmm1, %xmm0 174 ret 175.cfi_endproc 176.size _vpaes_encrypt_core,.-_vpaes_encrypt_core 177 178## 179## _aes_encrypt_core_2x 180## 181## AES-encrypt %xmm0 and %xmm6 in parallel. 182## 183## Inputs: 184## %xmm0 and %xmm6 = input 185## %xmm9 and %xmm10 as in _vpaes_preheat 186## (%rdx) = scheduled keys 187## 188## Output in %xmm0 and %xmm6 189## Clobbers %xmm1-%xmm5, %xmm7, %xmm8, %xmm11-%xmm13, %r9, %r10, %r11, %rax 190## Preserves %xmm14 and %xmm15 191## 192## This function stitches two parallel instances of _vpaes_encrypt_core. x86_64 193## provides 16 XMM registers. _vpaes_encrypt_core computes over six registers 194## (%xmm0-%xmm5) and additionally uses seven registers with preloaded constants 195## from _vpaes_preheat (%xmm9-%xmm15). This does not quite fit two instances, 196## so we spill some of %xmm9 through %xmm15 back to memory. We keep %xmm9 and 197## %xmm10 in registers as these values are used several times in a row. The 198## remainder are read once per round and are spilled to memory. This leaves two 199## registers preserved for the caller. 200## 201## Thus, of the two _vpaes_encrypt_core instances, the first uses (%xmm0-%xmm5) 202## as before. The second uses %xmm6-%xmm8,%xmm11-%xmm13. (Add 6 to %xmm2 and 203## below. Add 8 to %xmm3 and up.) Instructions in the second instance are 204## indented by one space. 205## 206## 207.type _vpaes_encrypt_core_2x,\@abi-omnipotent 208.align 16 209_vpaes_encrypt_core_2x: 210.cfi_startproc 211 mov %rdx, %r9 212 mov \$16, %r11 213 mov 240(%rdx),%eax 214 movdqa %xmm9, %xmm1 215 movdqa %xmm9, %xmm7 216 movdqa .Lk_ipt(%rip), %xmm2 # iptlo 217 movdqa %xmm2, %xmm8 218 pandn %xmm0, %xmm1 219 pandn %xmm6, %xmm7 220 movdqu (%r9), %xmm5 # round0 key 221 # Also use %xmm5 in the second instance. 222 psrld \$4, %xmm1 223 psrld \$4, %xmm7 224 pand %xmm9, %xmm0 225 pand %xmm9, %xmm6 226 pshufb %xmm0, %xmm2 227 pshufb %xmm6, %xmm8 228 movdqa .Lk_ipt+16(%rip), %xmm0 # ipthi 229 movdqa %xmm0, %xmm6 230 pshufb %xmm1, %xmm0 231 pshufb %xmm7, %xmm6 232 pxor %xmm5, %xmm2 233 pxor %xmm5, %xmm8 234 add \$16, %r9 235 pxor %xmm2, %xmm0 236 pxor %xmm8, %xmm6 237 lea .Lk_mc_backward(%rip),%r10 238 jmp .Lenc2x_entry 239 240.align 16 241.Lenc2x_loop: 242 # middle of middle round 243 movdqa .Lk_sb1(%rip), %xmm4 # 4 : sb1u 244 movdqa .Lk_sb1+16(%rip),%xmm0 # 0 : sb1t 245 movdqa %xmm4, %xmm12 246 movdqa %xmm0, %xmm6 247 pshufb %xmm2, %xmm4 # 4 = sb1u 248 pshufb %xmm8, %xmm12 249 pshufb %xmm3, %xmm0 # 0 = sb1t 250 pshufb %xmm11, %xmm6 251 pxor %xmm5, %xmm4 # 4 = sb1u + k 252 pxor %xmm5, %xmm12 253 movdqa .Lk_sb2(%rip), %xmm5 # 4 : sb2u 254 movdqa %xmm5, %xmm13 255 pxor %xmm4, %xmm0 # 0 = A 256 pxor %xmm12, %xmm6 257 movdqa -0x40(%r11,%r10), %xmm1 # .Lk_mc_forward[] 258 # Also use %xmm1 in the second instance. 259 pshufb %xmm2, %xmm5 # 4 = sb2u 260 pshufb %xmm8, %xmm13 261 movdqa (%r11,%r10), %xmm4 # .Lk_mc_backward[] 262 # Also use %xmm4 in the second instance. 263 movdqa .Lk_sb2+16(%rip), %xmm2 # 2 : sb2t 264 movdqa %xmm2, %xmm8 265 pshufb %xmm3, %xmm2 # 2 = sb2t 266 pshufb %xmm11, %xmm8 267 movdqa %xmm0, %xmm3 # 3 = A 268 movdqa %xmm6, %xmm11 269 pxor %xmm5, %xmm2 # 2 = 2A 270 pxor %xmm13, %xmm8 271 pshufb %xmm1, %xmm0 # 0 = B 272 pshufb %xmm1, %xmm6 273 add \$16, %r9 # next key 274 pxor %xmm2, %xmm0 # 0 = 2A+B 275 pxor %xmm8, %xmm6 276 pshufb %xmm4, %xmm3 # 3 = D 277 pshufb %xmm4, %xmm11 278 add \$16, %r11 # next mc 279 pxor %xmm0, %xmm3 # 3 = 2A+B+D 280 pxor %xmm6, %xmm11 281 pshufb %xmm1, %xmm0 # 0 = 2B+C 282 pshufb %xmm1, %xmm6 283 and \$0x30, %r11 # ... mod 4 284 sub \$1,%rax # nr-- 285 pxor %xmm3, %xmm0 # 0 = 2A+3B+C+D 286 pxor %xmm11, %xmm6 287 288.Lenc2x_entry: 289 # top of round 290 movdqa %xmm9, %xmm1 # 1 : i 291 movdqa %xmm9, %xmm7 292 movdqa .Lk_inv+16(%rip), %xmm5 # 2 : a/k 293 movdqa %xmm5, %xmm13 294 pandn %xmm0, %xmm1 # 1 = i<<4 295 pandn %xmm6, %xmm7 296 psrld \$4, %xmm1 # 1 = i 297 psrld \$4, %xmm7 298 pand %xmm9, %xmm0 # 0 = k 299 pand %xmm9, %xmm6 300 pshufb %xmm0, %xmm5 # 2 = a/k 301 pshufb %xmm6, %xmm13 302 movdqa %xmm10, %xmm3 # 3 : 1/i 303 movdqa %xmm10, %xmm11 304 pxor %xmm1, %xmm0 # 0 = j 305 pxor %xmm7, %xmm6 306 pshufb %xmm1, %xmm3 # 3 = 1/i 307 pshufb %xmm7, %xmm11 308 movdqa %xmm10, %xmm4 # 4 : 1/j 309 movdqa %xmm10, %xmm12 310 pxor %xmm5, %xmm3 # 3 = iak = 1/i + a/k 311 pxor %xmm13, %xmm11 312 pshufb %xmm0, %xmm4 # 4 = 1/j 313 pshufb %xmm6, %xmm12 314 movdqa %xmm10, %xmm2 # 2 : 1/iak 315 movdqa %xmm10, %xmm8 316 pxor %xmm5, %xmm4 # 4 = jak = 1/j + a/k 317 pxor %xmm13, %xmm12 318 pshufb %xmm3, %xmm2 # 2 = 1/iak 319 pshufb %xmm11, %xmm8 320 movdqa %xmm10, %xmm3 # 3 : 1/jak 321 movdqa %xmm10, %xmm11 322 pxor %xmm0, %xmm2 # 2 = io 323 pxor %xmm6, %xmm8 324 pshufb %xmm4, %xmm3 # 3 = 1/jak 325 pshufb %xmm12, %xmm11 326 movdqu (%r9), %xmm5 327 # Also use %xmm5 in the second instance. 328 pxor %xmm1, %xmm3 # 3 = jo 329 pxor %xmm7, %xmm11 330 jnz .Lenc2x_loop 331 332 # middle of last round 333 movdqa -0x60(%r10), %xmm4 # 3 : sbou .Lk_sbo 334 movdqa -0x50(%r10), %xmm0 # 0 : sbot .Lk_sbo+16 335 movdqa %xmm4, %xmm12 336 movdqa %xmm0, %xmm6 337 pshufb %xmm2, %xmm4 # 4 = sbou 338 pshufb %xmm8, %xmm12 339 pxor %xmm5, %xmm4 # 4 = sb1u + k 340 pxor %xmm5, %xmm12 341 pshufb %xmm3, %xmm0 # 0 = sb1t 342 pshufb %xmm11, %xmm6 343 movdqa 0x40(%r11,%r10), %xmm1 # .Lk_sr[] 344 # Also use %xmm1 in the second instance. 345 pxor %xmm4, %xmm0 # 0 = A 346 pxor %xmm12, %xmm6 347 pshufb %xmm1, %xmm0 348 pshufb %xmm1, %xmm6 349 ret 350.cfi_endproc 351.size _vpaes_encrypt_core_2x,.-_vpaes_encrypt_core_2x 352 353######################################################## 354## ## 355## AES key schedule ## 356## ## 357######################################################## 358.type _vpaes_schedule_core,\@abi-omnipotent 359.align 16 360_vpaes_schedule_core: 361.cfi_startproc 362 # rdi = key 363 # rsi = size in bits 364 # rdx = buffer 365 # rcx = direction. 0=encrypt, 1=decrypt 366 367 call _vpaes_preheat # load the tables 368 movdqa .Lk_rcon(%rip), %xmm8 # load rcon 369 movdqu (%rdi), %xmm0 # load key (unaligned) 370 371 # input transform 372 movdqa %xmm0, %xmm3 373 lea .Lk_ipt(%rip), %r11 374 call _vpaes_schedule_transform 375 movdqa %xmm0, %xmm7 376 377 lea .Lk_sr(%rip),%r10 378 379 # encrypting, output zeroth round key after transform 380 movdqu %xmm0, (%rdx) 381 382.Lschedule_go: 383 cmp \$192, %esi 384 ja .Lschedule_256 385 # 192-bit key support was removed. 386 # 128: fall though 387 388## 389## .schedule_128 390## 391## 128-bit specific part of key schedule. 392## 393## This schedule is really simple, because all its parts 394## are accomplished by the subroutines. 395## 396.Lschedule_128: 397 mov \$10, %esi 398 399.Loop_schedule_128: 400 call _vpaes_schedule_round 401 dec %rsi 402 jz .Lschedule_mangle_last 403 call _vpaes_schedule_mangle # write output 404 jmp .Loop_schedule_128 405 406## 407## .aes_schedule_256 408## 409## 256-bit specific part of key schedule. 410## 411## The structure here is very similar to the 128-bit 412## schedule, but with an additional "low side" in 413## %xmm6. The low side's rounds are the same as the 414## high side's, except no rcon and no rotation. 415## 416.align 16 417.Lschedule_256: 418 movdqu 16(%rdi),%xmm0 # load key part 2 (unaligned) 419 call _vpaes_schedule_transform # input transform 420 mov \$7, %esi 421 422.Loop_schedule_256: 423 call _vpaes_schedule_mangle # output low result 424 movdqa %xmm0, %xmm6 # save cur_lo in xmm6 425 426 # high round 427 call _vpaes_schedule_round 428 dec %rsi 429 jz .Lschedule_mangle_last 430 call _vpaes_schedule_mangle 431 432 # low round. swap xmm7 and xmm6 433 pshufd \$0xFF, %xmm0, %xmm0 434 movdqa %xmm7, %xmm5 435 movdqa %xmm6, %xmm7 436 call _vpaes_schedule_low_round 437 movdqa %xmm5, %xmm7 438 439 jmp .Loop_schedule_256 440 441 442## 443## .aes_schedule_mangle_last 444## 445## Mangler for last round of key schedule 446## Mangles %xmm0 447## when encrypting, outputs out(%xmm0) ^ 63 448## when decrypting, outputs unskew(%xmm0) 449## 450## Always called right before return... jumps to cleanup and exits 451## 452.align 16 453.Lschedule_mangle_last: 454 # schedule last round key from xmm0 455 lea .Lk_deskew(%rip),%r11 # prepare to deskew 456 457 # encrypting 458 movdqa (%r8,%r10),%xmm1 459 pshufb %xmm1, %xmm0 # output permute 460 lea .Lk_opt(%rip), %r11 # prepare to output transform 461 add \$32, %rdx 462 463.Lschedule_mangle_last_dec: 464 add \$-16, %rdx 465 pxor .Lk_s63(%rip), %xmm0 466 call _vpaes_schedule_transform # output transform 467 movdqu %xmm0, (%rdx) # save last key 468 469 # cleanup 470 pxor %xmm0, %xmm0 471 pxor %xmm1, %xmm1 472 pxor %xmm2, %xmm2 473 pxor %xmm3, %xmm3 474 pxor %xmm4, %xmm4 475 pxor %xmm5, %xmm5 476 pxor %xmm6, %xmm6 477 pxor %xmm7, %xmm7 478 ret 479.cfi_endproc 480.size _vpaes_schedule_core,.-_vpaes_schedule_core 481 482## 483## .aes_schedule_round 484## 485## Runs one main round of the key schedule on %xmm0, %xmm7 486## 487## Specifically, runs subbytes on the high dword of %xmm0 488## then rotates it by one byte and xors into the low dword of 489## %xmm7. 490## 491## Adds rcon from low byte of %xmm8, then rotates %xmm8 for 492## next rcon. 493## 494## Smears the dwords of %xmm7 by xoring the low into the 495## second low, result into third, result into highest. 496## 497## Returns results in %xmm7 = %xmm0. 498## Clobbers %xmm1-%xmm4, %r11. 499## 500.type _vpaes_schedule_round,\@abi-omnipotent 501.align 16 502_vpaes_schedule_round: 503.cfi_startproc 504 # extract rcon from xmm8 505 pxor %xmm1, %xmm1 506 palignr \$15, %xmm8, %xmm1 507 palignr \$15, %xmm8, %xmm8 508 pxor %xmm1, %xmm7 509 510 # rotate 511 pshufd \$0xFF, %xmm0, %xmm0 512 palignr \$1, %xmm0, %xmm0 513 514 # fall through... 515 516 # low round: same as high round, but no rotation and no rcon. 517_vpaes_schedule_low_round: 518 # smear xmm7 519 movdqa %xmm7, %xmm1 520 pslldq \$4, %xmm7 521 pxor %xmm1, %xmm7 522 movdqa %xmm7, %xmm1 523 pslldq \$8, %xmm7 524 pxor %xmm1, %xmm7 525 pxor .Lk_s63(%rip), %xmm7 526 527 # subbytes 528 movdqa %xmm9, %xmm1 529 pandn %xmm0, %xmm1 530 psrld \$4, %xmm1 # 1 = i 531 pand %xmm9, %xmm0 # 0 = k 532 movdqa %xmm11, %xmm2 # 2 : a/k 533 pshufb %xmm0, %xmm2 # 2 = a/k 534 pxor %xmm1, %xmm0 # 0 = j 535 movdqa %xmm10, %xmm3 # 3 : 1/i 536 pshufb %xmm1, %xmm3 # 3 = 1/i 537 pxor %xmm2, %xmm3 # 3 = iak = 1/i + a/k 538 movdqa %xmm10, %xmm4 # 4 : 1/j 539 pshufb %xmm0, %xmm4 # 4 = 1/j 540 pxor %xmm2, %xmm4 # 4 = jak = 1/j + a/k 541 movdqa %xmm10, %xmm2 # 2 : 1/iak 542 pshufb %xmm3, %xmm2 # 2 = 1/iak 543 pxor %xmm0, %xmm2 # 2 = io 544 movdqa %xmm10, %xmm3 # 3 : 1/jak 545 pshufb %xmm4, %xmm3 # 3 = 1/jak 546 pxor %xmm1, %xmm3 # 3 = jo 547 movdqa %xmm13, %xmm4 # 4 : sbou 548 pshufb %xmm2, %xmm4 # 4 = sbou 549 movdqa %xmm12, %xmm0 # 0 : sbot 550 pshufb %xmm3, %xmm0 # 0 = sb1t 551 pxor %xmm4, %xmm0 # 0 = sbox output 552 553 # add in smeared stuff 554 pxor %xmm7, %xmm0 555 movdqa %xmm0, %xmm7 556 ret 557.cfi_endproc 558.size _vpaes_schedule_round,.-_vpaes_schedule_round 559 560## 561## .aes_schedule_transform 562## 563## Linear-transform %xmm0 according to tables at (%r11) 564## 565## Requires that %xmm9 = 0x0F0F... as in preheat 566## Output in %xmm0 567## Clobbers %xmm1, %xmm2 568## 569.type _vpaes_schedule_transform,\@abi-omnipotent 570.align 16 571_vpaes_schedule_transform: 572.cfi_startproc 573 movdqa %xmm9, %xmm1 574 pandn %xmm0, %xmm1 575 psrld \$4, %xmm1 576 pand %xmm9, %xmm0 577 movdqa (%r11), %xmm2 # lo 578 pshufb %xmm0, %xmm2 579 movdqa 16(%r11), %xmm0 # hi 580 pshufb %xmm1, %xmm0 581 pxor %xmm2, %xmm0 582 ret 583.cfi_endproc 584.size _vpaes_schedule_transform,.-_vpaes_schedule_transform 585 586## 587## .aes_schedule_mangle 588## 589## Mangle xmm0 from (basis-transformed) standard version 590## to our version. 591## 592## On encrypt, 593## xor with 0x63 594## multiply by circulant 0,1,1,1 595## apply shiftrows transform 596## 597## On decrypt, 598## xor with 0x63 599## multiply by "inverse mixcolumns" circulant E,B,D,9 600## deskew 601## apply shiftrows transform 602## 603## 604## Writes out to (%rdx), and increments or decrements it 605## Keeps track of round number mod 4 in %r8 606## Preserves xmm0 607## Clobbers xmm1-xmm5 608## 609.type _vpaes_schedule_mangle,\@abi-omnipotent 610.align 16 611_vpaes_schedule_mangle: 612.cfi_startproc 613 movdqa %xmm0, %xmm4 # save xmm0 for later 614 movdqa .Lk_mc_forward(%rip),%xmm5 615 616 # encrypting 617 add \$16, %rdx 618 pxor .Lk_s63(%rip),%xmm4 619 pshufb %xmm5, %xmm4 620 movdqa %xmm4, %xmm3 621 pshufb %xmm5, %xmm4 622 pxor %xmm4, %xmm3 623 pshufb %xmm5, %xmm4 624 pxor %xmm4, %xmm3 625 626.Lschedule_mangle_both: 627 movdqa (%r8,%r10),%xmm1 628 pshufb %xmm1,%xmm3 629 add \$-16, %r8 630 and \$0x30, %r8 631 movdqu %xmm3, (%rdx) 632 ret 633.cfi_endproc 634.size _vpaes_schedule_mangle,.-_vpaes_schedule_mangle 635 636# 637# Interface to OpenSSL 638# 639.globl ${PREFIX}_set_encrypt_key 640.type ${PREFIX}_set_encrypt_key,\@function,3 641.align 16 642${PREFIX}_set_encrypt_key: 643.cfi_startproc 644 _CET_ENDBR 645#ifdef BORINGSSL_DISPATCH_TEST 646.extern BORINGSSL_function_hit 647 movb \$1, BORINGSSL_function_hit+5(%rip) 648#endif 649 650___ 651$code.=<<___ if ($win64); 652 lea -0xb8(%rsp),%rsp 653 movaps %xmm6,0x10(%rsp) 654 movaps %xmm7,0x20(%rsp) 655 movaps %xmm8,0x30(%rsp) 656 movaps %xmm9,0x40(%rsp) 657 movaps %xmm10,0x50(%rsp) 658 movaps %xmm11,0x60(%rsp) 659 movaps %xmm12,0x70(%rsp) 660 movaps %xmm13,0x80(%rsp) 661 movaps %xmm14,0x90(%rsp) 662 movaps %xmm15,0xa0(%rsp) 663.Lenc_key_body: 664___ 665$code.=<<___; 666 mov %esi,%eax 667 shr \$5,%eax 668 add \$5,%eax 669 mov %eax,240(%rdx) # AES_KEY->rounds = nbits/32+5; 670 671 mov \$0,%ecx 672 mov \$0x30,%r8d 673 call _vpaes_schedule_core 674___ 675$code.=<<___ if ($win64); 676 movaps 0x10(%rsp),%xmm6 677 movaps 0x20(%rsp),%xmm7 678 movaps 0x30(%rsp),%xmm8 679 movaps 0x40(%rsp),%xmm9 680 movaps 0x50(%rsp),%xmm10 681 movaps 0x60(%rsp),%xmm11 682 movaps 0x70(%rsp),%xmm12 683 movaps 0x80(%rsp),%xmm13 684 movaps 0x90(%rsp),%xmm14 685 movaps 0xa0(%rsp),%xmm15 686 lea 0xb8(%rsp),%rsp 687.Lenc_key_epilogue: 688___ 689$code.=<<___; 690 xor %eax,%eax 691 ret 692.cfi_endproc 693.size ${PREFIX}_set_encrypt_key,.-${PREFIX}_set_encrypt_key 694 695.globl ${PREFIX}_encrypt 696.type ${PREFIX}_encrypt,\@function,3 697.align 16 698${PREFIX}_encrypt: 699.cfi_startproc 700 _CET_ENDBR 701#ifdef BORINGSSL_DISPATCH_TEST 702.extern BORINGSSL_function_hit 703 movb \$1, BORINGSSL_function_hit+4(%rip) 704#endif 705___ 706$code.=<<___ if ($win64); 707 lea -0xb8(%rsp),%rsp 708 movaps %xmm6,0x10(%rsp) 709 movaps %xmm7,0x20(%rsp) 710 movaps %xmm8,0x30(%rsp) 711 movaps %xmm9,0x40(%rsp) 712 movaps %xmm10,0x50(%rsp) 713 movaps %xmm11,0x60(%rsp) 714 movaps %xmm12,0x70(%rsp) 715 movaps %xmm13,0x80(%rsp) 716 movaps %xmm14,0x90(%rsp) 717 movaps %xmm15,0xa0(%rsp) 718.Lenc_body: 719___ 720$code.=<<___; 721 movdqu (%rdi),%xmm0 722 call _vpaes_preheat 723 call _vpaes_encrypt_core 724 movdqu %xmm0,(%rsi) 725___ 726$code.=<<___ if ($win64); 727 movaps 0x10(%rsp),%xmm6 728 movaps 0x20(%rsp),%xmm7 729 movaps 0x30(%rsp),%xmm8 730 movaps 0x40(%rsp),%xmm9 731 movaps 0x50(%rsp),%xmm10 732 movaps 0x60(%rsp),%xmm11 733 movaps 0x70(%rsp),%xmm12 734 movaps 0x80(%rsp),%xmm13 735 movaps 0x90(%rsp),%xmm14 736 movaps 0xa0(%rsp),%xmm15 737 lea 0xb8(%rsp),%rsp 738.Lenc_epilogue: 739___ 740$code.=<<___; 741 ret 742.cfi_endproc 743.size ${PREFIX}_encrypt,.-${PREFIX}_encrypt 744___ 745{ 746my ($inp,$out,$blocks,$key,$ivp)=("%rdi","%rsi","%rdx","%rcx","%r8"); 747# void vpaes_ctr32_encrypt_blocks(const uint8_t *inp, uint8_t *out, 748# size_t blocks, const AES_KEY *key, 749# const uint8_t ivp[16]); 750$code.=<<___; 751.globl ${PREFIX}_ctr32_encrypt_blocks 752.type ${PREFIX}_ctr32_encrypt_blocks,\@function,5 753.align 16 754${PREFIX}_ctr32_encrypt_blocks: 755.cfi_startproc 756 _CET_ENDBR 757 # _vpaes_encrypt_core and _vpaes_encrypt_core_2x expect the key in %rdx. 758 xchg $key, $blocks 759___ 760($blocks,$key)=($key,$blocks); 761$code.=<<___; 762 test $blocks, $blocks 763 jz .Lctr32_abort 764___ 765$code.=<<___ if ($win64); 766 lea -0xb8(%rsp),%rsp 767 movaps %xmm6,0x10(%rsp) 768 movaps %xmm7,0x20(%rsp) 769 movaps %xmm8,0x30(%rsp) 770 movaps %xmm9,0x40(%rsp) 771 movaps %xmm10,0x50(%rsp) 772 movaps %xmm11,0x60(%rsp) 773 movaps %xmm12,0x70(%rsp) 774 movaps %xmm13,0x80(%rsp) 775 movaps %xmm14,0x90(%rsp) 776 movaps %xmm15,0xa0(%rsp) 777.Lctr32_body: 778___ 779$code.=<<___; 780 movdqu ($ivp), %xmm0 # Load IV. 781 movdqa .Lctr_add_one(%rip), %xmm8 782 sub $inp, $out # This allows only incrementing $inp. 783 call _vpaes_preheat 784 movdqa %xmm0, %xmm6 785 pshufb .Lrev_ctr(%rip), %xmm6 786 787 test \$1, $blocks 788 jz .Lctr32_prep_loop 789 790 # Handle one block so the remaining block count is even for 791 # _vpaes_encrypt_core_2x. 792 movdqu ($inp), %xmm7 # Load input. 793 call _vpaes_encrypt_core 794 pxor %xmm7, %xmm0 795 paddd %xmm8, %xmm6 796 movdqu %xmm0, ($out,$inp) 797 sub \$1, $blocks 798 lea 16($inp), $inp 799 jz .Lctr32_done 800 801.Lctr32_prep_loop: 802 # _vpaes_encrypt_core_2x leaves only %xmm14 and %xmm15 as spare 803 # registers. We maintain two byte-swapped counters in them. 804 movdqa %xmm6, %xmm14 805 movdqa %xmm6, %xmm15 806 paddd %xmm8, %xmm15 807 808.Lctr32_loop: 809 movdqa .Lrev_ctr(%rip), %xmm1 # Set up counters. 810 movdqa %xmm14, %xmm0 811 movdqa %xmm15, %xmm6 812 pshufb %xmm1, %xmm0 813 pshufb %xmm1, %xmm6 814 call _vpaes_encrypt_core_2x 815 movdqu ($inp), %xmm1 # Load input. 816 movdqu 16($inp), %xmm2 817 movdqa .Lctr_add_two(%rip), %xmm3 818 pxor %xmm1, %xmm0 # XOR input. 819 pxor %xmm2, %xmm6 820 paddd %xmm3, %xmm14 # Increment counters. 821 paddd %xmm3, %xmm15 822 movdqu %xmm0, ($out,$inp) # Write output. 823 movdqu %xmm6, 16($out,$inp) 824 sub \$2, $blocks # Advance loop. 825 lea 32($inp), $inp 826 jnz .Lctr32_loop 827 828.Lctr32_done: 829___ 830$code.=<<___ if ($win64); 831 movaps 0x10(%rsp),%xmm6 832 movaps 0x20(%rsp),%xmm7 833 movaps 0x30(%rsp),%xmm8 834 movaps 0x40(%rsp),%xmm9 835 movaps 0x50(%rsp),%xmm10 836 movaps 0x60(%rsp),%xmm11 837 movaps 0x70(%rsp),%xmm12 838 movaps 0x80(%rsp),%xmm13 839 movaps 0x90(%rsp),%xmm14 840 movaps 0xa0(%rsp),%xmm15 841 lea 0xb8(%rsp),%rsp 842.Lctr32_epilogue: 843___ 844$code.=<<___; 845.Lctr32_abort: 846 ret 847.cfi_endproc 848.size ${PREFIX}_ctr32_encrypt_blocks,.-${PREFIX}_ctr32_encrypt_blocks 849___ 850} 851$code.=<<___; 852## 853## _aes_preheat 854## 855## Fills register %r10 -> .aes_consts (so you can -fPIC) 856## and %xmm9-%xmm15 as specified below. 857## 858.type _vpaes_preheat,\@abi-omnipotent 859.align 16 860_vpaes_preheat: 861.cfi_startproc 862 lea .Lk_s0F(%rip), %r10 863 movdqa -0x20(%r10), %xmm10 # .Lk_inv 864 movdqa -0x10(%r10), %xmm11 # .Lk_inv+16 865 movdqa 0x00(%r10), %xmm9 # .Lk_s0F 866 movdqa 0x30(%r10), %xmm13 # .Lk_sb1 867 movdqa 0x40(%r10), %xmm12 # .Lk_sb1+16 868 movdqa 0x50(%r10), %xmm15 # .Lk_sb2 869 movdqa 0x60(%r10), %xmm14 # .Lk_sb2+16 870 ret 871.cfi_endproc 872.size _vpaes_preheat,.-_vpaes_preheat 873######################################################## 874## ## 875## Constants ## 876## ## 877######################################################## 878.type _vpaes_consts,\@object 879.section .rodata 880.align 64 881_vpaes_consts: 882.Lk_inv: # inv, inva 883 .quad 0x0E05060F0D080180, 0x040703090A0B0C02 884 .quad 0x01040A060F0B0780, 0x030D0E0C02050809 885 886.Lk_s0F: # s0F 887 .quad 0x0F0F0F0F0F0F0F0F, 0x0F0F0F0F0F0F0F0F 888 889.Lk_ipt: # input transform (lo, hi) 890 .quad 0xC2B2E8985A2A7000, 0xCABAE09052227808 891 .quad 0x4C01307D317C4D00, 0xCD80B1FCB0FDCC81 892 893.Lk_sb1: # sb1u, sb1t 894 .quad 0xB19BE18FCB503E00, 0xA5DF7A6E142AF544 895 .quad 0x3618D415FAE22300, 0x3BF7CCC10D2ED9EF 896.Lk_sb2: # sb2u, sb2t 897 .quad 0xE27A93C60B712400, 0x5EB7E955BC982FCD 898 .quad 0x69EB88400AE12900, 0xC2A163C8AB82234A 899.Lk_sbo: # sbou, sbot 900 .quad 0xD0D26D176FBDC700, 0x15AABF7AC502A878 901 .quad 0xCFE474A55FBB6A00, 0x8E1E90D1412B35FA 902 903.Lk_mc_forward: # mc_forward 904 .quad 0x0407060500030201, 0x0C0F0E0D080B0A09 905 .quad 0x080B0A0904070605, 0x000302010C0F0E0D 906 .quad 0x0C0F0E0D080B0A09, 0x0407060500030201 907 .quad 0x000302010C0F0E0D, 0x080B0A0904070605 908 909.Lk_mc_backward:# mc_backward 910 .quad 0x0605040702010003, 0x0E0D0C0F0A09080B 911 .quad 0x020100030E0D0C0F, 0x0A09080B06050407 912 .quad 0x0E0D0C0F0A09080B, 0x0605040702010003 913 .quad 0x0A09080B06050407, 0x020100030E0D0C0F 914 915.Lk_sr: # sr 916 .quad 0x0706050403020100, 0x0F0E0D0C0B0A0908 917 .quad 0x030E09040F0A0500, 0x0B06010C07020D08 918 .quad 0x0F060D040B020900, 0x070E050C030A0108 919 .quad 0x0B0E0104070A0D00, 0x0306090C0F020508 920 921.Lk_rcon: # rcon 922 .quad 0x1F8391B9AF9DEEB6, 0x702A98084D7C7D81 923 924.Lk_s63: # s63: all equal to 0x63 transformed 925 .quad 0x5B5B5B5B5B5B5B5B, 0x5B5B5B5B5B5B5B5B 926 927.Lk_opt: # output transform 928 .quad 0xFF9F4929D6B66000, 0xF7974121DEBE6808 929 .quad 0x01EDBD5150BCEC00, 0xE10D5DB1B05C0CE0 930 931.Lk_deskew: # deskew tables: inverts the sbox's "skew" 932 .quad 0x07E4A34047A4E300, 0x1DFEB95A5DBEF91A 933 .quad 0x5F36B5DC83EA6900, 0x2841C2ABF49D1E77 934 935# .Lrev_ctr is a permutation which byte-swaps the counter portion of the IV. 936.Lrev_ctr: 937 .quad 0x0706050403020100, 0x0c0d0e0f0b0a0908 938# .Lctr_add_* may be added to a byte-swapped xmm register to increment the 939# counter. The register must be byte-swapped again to form the actual input. 940.Lctr_add_one: 941 .quad 0x0000000000000000, 0x0000000100000000 942.Lctr_add_two: 943 .quad 0x0000000000000000, 0x0000000200000000 944 945.asciz "Vector Permutation AES for x86_64/SSSE3, Mike Hamburg (Stanford University)" 946.align 64 947.size _vpaes_consts,.-_vpaes_consts 948.text 949___ 950 951if ($win64) { 952# EXCEPTION_DISPOSITION handler (EXCEPTION_RECORD *rec,ULONG64 frame, 953# CONTEXT *context,DISPATCHER_CONTEXT *disp) 954$rec="%rcx"; 955$frame="%rdx"; 956$context="%r8"; 957$disp="%r9"; 958 959$code.=<<___; 960.extern __imp_RtlVirtualUnwind 961.type se_handler,\@abi-omnipotent 962.align 16 963se_handler: 964 push %rsi 965 push %rdi 966 push %rbx 967 push %rbp 968 push %r12 969 push %r13 970 push %r14 971 push %r15 972 pushfq 973 sub \$64,%rsp 974 975 mov 120($context),%rax # pull context->Rax 976 mov 248($context),%rbx # pull context->Rip 977 978 mov 8($disp),%rsi # disp->ImageBase 979 mov 56($disp),%r11 # disp->HandlerData 980 981 mov 0(%r11),%r10d # HandlerData[0] 982 lea (%rsi,%r10),%r10 # prologue label 983 cmp %r10,%rbx # context->Rip<prologue label 984 jb .Lin_prologue 985 986 mov 152($context),%rax # pull context->Rsp 987 988 mov 4(%r11),%r10d # HandlerData[1] 989 lea (%rsi,%r10),%r10 # epilogue label 990 cmp %r10,%rbx # context->Rip>=epilogue label 991 jae .Lin_prologue 992 993 lea 16(%rax),%rsi # %xmm save area 994 lea 512($context),%rdi # &context.Xmm6 995 mov \$20,%ecx # 10*sizeof(%xmm0)/sizeof(%rax) 996 .long 0xa548f3fc # cld; rep movsq 997 lea 0xb8(%rax),%rax # adjust stack pointer 998 999.Lin_prologue: 1000 mov 8(%rax),%rdi 1001 mov 16(%rax),%rsi 1002 mov %rax,152($context) # restore context->Rsp 1003 mov %rsi,168($context) # restore context->Rsi 1004 mov %rdi,176($context) # restore context->Rdi 1005 1006 mov 40($disp),%rdi # disp->ContextRecord 1007 mov $context,%rsi # context 1008 mov \$`1232/8`,%ecx # sizeof(CONTEXT) 1009 .long 0xa548f3fc # cld; rep movsq 1010 1011 mov $disp,%rsi 1012 xor %rcx,%rcx # arg1, UNW_FLAG_NHANDLER 1013 mov 8(%rsi),%rdx # arg2, disp->ImageBase 1014 mov 0(%rsi),%r8 # arg3, disp->ControlPc 1015 mov 16(%rsi),%r9 # arg4, disp->FunctionEntry 1016 mov 40(%rsi),%r10 # disp->ContextRecord 1017 lea 56(%rsi),%r11 # &disp->HandlerData 1018 lea 24(%rsi),%r12 # &disp->EstablisherFrame 1019 mov %r10,32(%rsp) # arg5 1020 mov %r11,40(%rsp) # arg6 1021 mov %r12,48(%rsp) # arg7 1022 mov %rcx,56(%rsp) # arg8, (NULL) 1023 call *__imp_RtlVirtualUnwind(%rip) 1024 1025 mov \$1,%eax # ExceptionContinueSearch 1026 add \$64,%rsp 1027 popfq 1028 pop %r15 1029 pop %r14 1030 pop %r13 1031 pop %r12 1032 pop %rbp 1033 pop %rbx 1034 pop %rdi 1035 pop %rsi 1036 ret 1037.size se_handler,.-se_handler 1038 1039.section .pdata 1040.align 4 1041 .rva .LSEH_begin_${PREFIX}_set_encrypt_key 1042 .rva .LSEH_end_${PREFIX}_set_encrypt_key 1043 .rva .LSEH_info_${PREFIX}_set_encrypt_key 1044 1045 .rva .LSEH_begin_${PREFIX}_encrypt 1046 .rva .LSEH_end_${PREFIX}_encrypt 1047 .rva .LSEH_info_${PREFIX}_encrypt 1048 .rva .LSEH_begin_${PREFIX}_ctr32_encrypt_blocks 1049 .rva .LSEH_end_${PREFIX}_ctr32_encrypt_blocks 1050 .rva .LSEH_info_${PREFIX}_ctr32_encrypt_blocks 1051 1052.section .xdata 1053.align 8 1054.LSEH_info_${PREFIX}_set_encrypt_key: 1055 .byte 9,0,0,0 1056 .rva se_handler 1057 .rva .Lenc_key_body,.Lenc_key_epilogue # HandlerData[] 1058.LSEH_info_${PREFIX}_encrypt: 1059 .byte 9,0,0,0 1060 .rva se_handler 1061 .rva .Lenc_body,.Lenc_epilogue # HandlerData[] 1062.LSEH_info_${PREFIX}_ctr32_encrypt_blocks: 1063 .byte 9,0,0,0 1064 .rva se_handler 1065 .rva .Lctr32_body,.Lctr32_epilogue # HandlerData[] 1066___ 1067} 1068 1069$code =~ s/\`([^\`]*)\`/eval($1)/gem; 1070 1071print $code; 1072 1073close STDOUT or die "error closing STDOUT: $!"; 1074