• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1; This file is generated from a similarly-named Perl script in the BoringSSL
2; source tree. Do not edit by hand.
3
4%ifdef BORINGSSL_PREFIX
5%include "boringssl_prefix_symbols_nasm.inc"
6%endif
7%ifidn __OUTPUT_FORMAT__, win32
8%ifidn __OUTPUT_FORMAT__,obj
9section	code	use32 class=code align=64
10%elifidn __OUTPUT_FORMAT__,win32
11$@feat.00 equ 1
12section	.text	code align=64
13%else
14section	.text	code
15%endif
16;extern	_OPENSSL_ia32cap_P
17%ifdef BORINGSSL_DISPATCH_TEST
18extern	_BORINGSSL_function_hit
19%endif
20global	_aes_hw_encrypt
21align	16
22_aes_hw_encrypt:
23L$_aes_hw_encrypt_begin:
24%ifdef BORINGSSL_DISPATCH_TEST
25	push	ebx
26	push	edx
27	call	L$000pic
28L$000pic:
29	pop	ebx
30	lea	ebx,[(_BORINGSSL_function_hit+1-L$000pic)+ebx]
31	mov	edx,1
32	mov	BYTE [ebx],dl
33	pop	edx
34	pop	ebx
35%endif
36	mov	eax,DWORD [4+esp]
37	mov	edx,DWORD [12+esp]
38	movups	xmm2,[eax]
39	mov	ecx,DWORD [240+edx]
40	mov	eax,DWORD [8+esp]
41	movups	xmm0,[edx]
42	movups	xmm1,[16+edx]
43	lea	edx,[32+edx]
44	xorps	xmm2,xmm0
45L$001enc1_loop_1:
46db	102,15,56,220,209
47	dec	ecx
48	movups	xmm1,[edx]
49	lea	edx,[16+edx]
50	jnz	NEAR L$001enc1_loop_1
51db	102,15,56,221,209
52	pxor	xmm0,xmm0
53	pxor	xmm1,xmm1
54	movups	[eax],xmm2
55	pxor	xmm2,xmm2
56	ret
57global	_aes_hw_decrypt
58align	16
59_aes_hw_decrypt:
60L$_aes_hw_decrypt_begin:
61	mov	eax,DWORD [4+esp]
62	mov	edx,DWORD [12+esp]
63	movups	xmm2,[eax]
64	mov	ecx,DWORD [240+edx]
65	mov	eax,DWORD [8+esp]
66	movups	xmm0,[edx]
67	movups	xmm1,[16+edx]
68	lea	edx,[32+edx]
69	xorps	xmm2,xmm0
70L$002dec1_loop_2:
71db	102,15,56,222,209
72	dec	ecx
73	movups	xmm1,[edx]
74	lea	edx,[16+edx]
75	jnz	NEAR L$002dec1_loop_2
76db	102,15,56,223,209
77	pxor	xmm0,xmm0
78	pxor	xmm1,xmm1
79	movups	[eax],xmm2
80	pxor	xmm2,xmm2
81	ret
82align	16
83__aesni_encrypt2:
84	movups	xmm0,[edx]
85	shl	ecx,4
86	movups	xmm1,[16+edx]
87	xorps	xmm2,xmm0
88	pxor	xmm3,xmm0
89	movups	xmm0,[32+edx]
90	lea	edx,[32+ecx*1+edx]
91	neg	ecx
92	add	ecx,16
93L$003enc2_loop:
94db	102,15,56,220,209
95db	102,15,56,220,217
96	movups	xmm1,[ecx*1+edx]
97	add	ecx,32
98db	102,15,56,220,208
99db	102,15,56,220,216
100	movups	xmm0,[ecx*1+edx-16]
101	jnz	NEAR L$003enc2_loop
102db	102,15,56,220,209
103db	102,15,56,220,217
104db	102,15,56,221,208
105db	102,15,56,221,216
106	ret
107align	16
108__aesni_decrypt2:
109	movups	xmm0,[edx]
110	shl	ecx,4
111	movups	xmm1,[16+edx]
112	xorps	xmm2,xmm0
113	pxor	xmm3,xmm0
114	movups	xmm0,[32+edx]
115	lea	edx,[32+ecx*1+edx]
116	neg	ecx
117	add	ecx,16
118L$004dec2_loop:
119db	102,15,56,222,209
120db	102,15,56,222,217
121	movups	xmm1,[ecx*1+edx]
122	add	ecx,32
123db	102,15,56,222,208
124db	102,15,56,222,216
125	movups	xmm0,[ecx*1+edx-16]
126	jnz	NEAR L$004dec2_loop
127db	102,15,56,222,209
128db	102,15,56,222,217
129db	102,15,56,223,208
130db	102,15,56,223,216
131	ret
132align	16
133__aesni_encrypt3:
134	movups	xmm0,[edx]
135	shl	ecx,4
136	movups	xmm1,[16+edx]
137	xorps	xmm2,xmm0
138	pxor	xmm3,xmm0
139	pxor	xmm4,xmm0
140	movups	xmm0,[32+edx]
141	lea	edx,[32+ecx*1+edx]
142	neg	ecx
143	add	ecx,16
144L$005enc3_loop:
145db	102,15,56,220,209
146db	102,15,56,220,217
147db	102,15,56,220,225
148	movups	xmm1,[ecx*1+edx]
149	add	ecx,32
150db	102,15,56,220,208
151db	102,15,56,220,216
152db	102,15,56,220,224
153	movups	xmm0,[ecx*1+edx-16]
154	jnz	NEAR L$005enc3_loop
155db	102,15,56,220,209
156db	102,15,56,220,217
157db	102,15,56,220,225
158db	102,15,56,221,208
159db	102,15,56,221,216
160db	102,15,56,221,224
161	ret
162align	16
163__aesni_decrypt3:
164	movups	xmm0,[edx]
165	shl	ecx,4
166	movups	xmm1,[16+edx]
167	xorps	xmm2,xmm0
168	pxor	xmm3,xmm0
169	pxor	xmm4,xmm0
170	movups	xmm0,[32+edx]
171	lea	edx,[32+ecx*1+edx]
172	neg	ecx
173	add	ecx,16
174L$006dec3_loop:
175db	102,15,56,222,209
176db	102,15,56,222,217
177db	102,15,56,222,225
178	movups	xmm1,[ecx*1+edx]
179	add	ecx,32
180db	102,15,56,222,208
181db	102,15,56,222,216
182db	102,15,56,222,224
183	movups	xmm0,[ecx*1+edx-16]
184	jnz	NEAR L$006dec3_loop
185db	102,15,56,222,209
186db	102,15,56,222,217
187db	102,15,56,222,225
188db	102,15,56,223,208
189db	102,15,56,223,216
190db	102,15,56,223,224
191	ret
192align	16
193__aesni_encrypt4:
194	movups	xmm0,[edx]
195	movups	xmm1,[16+edx]
196	shl	ecx,4
197	xorps	xmm2,xmm0
198	pxor	xmm3,xmm0
199	pxor	xmm4,xmm0
200	pxor	xmm5,xmm0
201	movups	xmm0,[32+edx]
202	lea	edx,[32+ecx*1+edx]
203	neg	ecx
204db	15,31,64,0
205	add	ecx,16
206L$007enc4_loop:
207db	102,15,56,220,209
208db	102,15,56,220,217
209db	102,15,56,220,225
210db	102,15,56,220,233
211	movups	xmm1,[ecx*1+edx]
212	add	ecx,32
213db	102,15,56,220,208
214db	102,15,56,220,216
215db	102,15,56,220,224
216db	102,15,56,220,232
217	movups	xmm0,[ecx*1+edx-16]
218	jnz	NEAR L$007enc4_loop
219db	102,15,56,220,209
220db	102,15,56,220,217
221db	102,15,56,220,225
222db	102,15,56,220,233
223db	102,15,56,221,208
224db	102,15,56,221,216
225db	102,15,56,221,224
226db	102,15,56,221,232
227	ret
228align	16
229__aesni_decrypt4:
230	movups	xmm0,[edx]
231	movups	xmm1,[16+edx]
232	shl	ecx,4
233	xorps	xmm2,xmm0
234	pxor	xmm3,xmm0
235	pxor	xmm4,xmm0
236	pxor	xmm5,xmm0
237	movups	xmm0,[32+edx]
238	lea	edx,[32+ecx*1+edx]
239	neg	ecx
240db	15,31,64,0
241	add	ecx,16
242L$008dec4_loop:
243db	102,15,56,222,209
244db	102,15,56,222,217
245db	102,15,56,222,225
246db	102,15,56,222,233
247	movups	xmm1,[ecx*1+edx]
248	add	ecx,32
249db	102,15,56,222,208
250db	102,15,56,222,216
251db	102,15,56,222,224
252db	102,15,56,222,232
253	movups	xmm0,[ecx*1+edx-16]
254	jnz	NEAR L$008dec4_loop
255db	102,15,56,222,209
256db	102,15,56,222,217
257db	102,15,56,222,225
258db	102,15,56,222,233
259db	102,15,56,223,208
260db	102,15,56,223,216
261db	102,15,56,223,224
262db	102,15,56,223,232
263	ret
264align	16
265__aesni_encrypt6:
266	movups	xmm0,[edx]
267	shl	ecx,4
268	movups	xmm1,[16+edx]
269	xorps	xmm2,xmm0
270	pxor	xmm3,xmm0
271	pxor	xmm4,xmm0
272db	102,15,56,220,209
273	pxor	xmm5,xmm0
274	pxor	xmm6,xmm0
275db	102,15,56,220,217
276	lea	edx,[32+ecx*1+edx]
277	neg	ecx
278db	102,15,56,220,225
279	pxor	xmm7,xmm0
280	movups	xmm0,[ecx*1+edx]
281	add	ecx,16
282	jmp	NEAR L$009_aesni_encrypt6_inner
283align	16
284L$010enc6_loop:
285db	102,15,56,220,209
286db	102,15,56,220,217
287db	102,15,56,220,225
288L$009_aesni_encrypt6_inner:
289db	102,15,56,220,233
290db	102,15,56,220,241
291db	102,15,56,220,249
292L$_aesni_encrypt6_enter:
293	movups	xmm1,[ecx*1+edx]
294	add	ecx,32
295db	102,15,56,220,208
296db	102,15,56,220,216
297db	102,15,56,220,224
298db	102,15,56,220,232
299db	102,15,56,220,240
300db	102,15,56,220,248
301	movups	xmm0,[ecx*1+edx-16]
302	jnz	NEAR L$010enc6_loop
303db	102,15,56,220,209
304db	102,15,56,220,217
305db	102,15,56,220,225
306db	102,15,56,220,233
307db	102,15,56,220,241
308db	102,15,56,220,249
309db	102,15,56,221,208
310db	102,15,56,221,216
311db	102,15,56,221,224
312db	102,15,56,221,232
313db	102,15,56,221,240
314db	102,15,56,221,248
315	ret
316align	16
317__aesni_decrypt6:
318	movups	xmm0,[edx]
319	shl	ecx,4
320	movups	xmm1,[16+edx]
321	xorps	xmm2,xmm0
322	pxor	xmm3,xmm0
323	pxor	xmm4,xmm0
324db	102,15,56,222,209
325	pxor	xmm5,xmm0
326	pxor	xmm6,xmm0
327db	102,15,56,222,217
328	lea	edx,[32+ecx*1+edx]
329	neg	ecx
330db	102,15,56,222,225
331	pxor	xmm7,xmm0
332	movups	xmm0,[ecx*1+edx]
333	add	ecx,16
334	jmp	NEAR L$011_aesni_decrypt6_inner
335align	16
336L$012dec6_loop:
337db	102,15,56,222,209
338db	102,15,56,222,217
339db	102,15,56,222,225
340L$011_aesni_decrypt6_inner:
341db	102,15,56,222,233
342db	102,15,56,222,241
343db	102,15,56,222,249
344L$_aesni_decrypt6_enter:
345	movups	xmm1,[ecx*1+edx]
346	add	ecx,32
347db	102,15,56,222,208
348db	102,15,56,222,216
349db	102,15,56,222,224
350db	102,15,56,222,232
351db	102,15,56,222,240
352db	102,15,56,222,248
353	movups	xmm0,[ecx*1+edx-16]
354	jnz	NEAR L$012dec6_loop
355db	102,15,56,222,209
356db	102,15,56,222,217
357db	102,15,56,222,225
358db	102,15,56,222,233
359db	102,15,56,222,241
360db	102,15,56,222,249
361db	102,15,56,223,208
362db	102,15,56,223,216
363db	102,15,56,223,224
364db	102,15,56,223,232
365db	102,15,56,223,240
366db	102,15,56,223,248
367	ret
368global	_aes_hw_ecb_encrypt
369align	16
370_aes_hw_ecb_encrypt:
371L$_aes_hw_ecb_encrypt_begin:
372	push	ebp
373	push	ebx
374	push	esi
375	push	edi
376	mov	esi,DWORD [20+esp]
377	mov	edi,DWORD [24+esp]
378	mov	eax,DWORD [28+esp]
379	mov	edx,DWORD [32+esp]
380	mov	ebx,DWORD [36+esp]
381	and	eax,-16
382	jz	NEAR L$013ecb_ret
383	mov	ecx,DWORD [240+edx]
384	test	ebx,ebx
385	jz	NEAR L$014ecb_decrypt
386	mov	ebp,edx
387	mov	ebx,ecx
388	cmp	eax,96
389	jb	NEAR L$015ecb_enc_tail
390	movdqu	xmm2,[esi]
391	movdqu	xmm3,[16+esi]
392	movdqu	xmm4,[32+esi]
393	movdqu	xmm5,[48+esi]
394	movdqu	xmm6,[64+esi]
395	movdqu	xmm7,[80+esi]
396	lea	esi,[96+esi]
397	sub	eax,96
398	jmp	NEAR L$016ecb_enc_loop6_enter
399align	16
400L$017ecb_enc_loop6:
401	movups	[edi],xmm2
402	movdqu	xmm2,[esi]
403	movups	[16+edi],xmm3
404	movdqu	xmm3,[16+esi]
405	movups	[32+edi],xmm4
406	movdqu	xmm4,[32+esi]
407	movups	[48+edi],xmm5
408	movdqu	xmm5,[48+esi]
409	movups	[64+edi],xmm6
410	movdqu	xmm6,[64+esi]
411	movups	[80+edi],xmm7
412	lea	edi,[96+edi]
413	movdqu	xmm7,[80+esi]
414	lea	esi,[96+esi]
415L$016ecb_enc_loop6_enter:
416	call	__aesni_encrypt6
417	mov	edx,ebp
418	mov	ecx,ebx
419	sub	eax,96
420	jnc	NEAR L$017ecb_enc_loop6
421	movups	[edi],xmm2
422	movups	[16+edi],xmm3
423	movups	[32+edi],xmm4
424	movups	[48+edi],xmm5
425	movups	[64+edi],xmm6
426	movups	[80+edi],xmm7
427	lea	edi,[96+edi]
428	add	eax,96
429	jz	NEAR L$013ecb_ret
430L$015ecb_enc_tail:
431	movups	xmm2,[esi]
432	cmp	eax,32
433	jb	NEAR L$018ecb_enc_one
434	movups	xmm3,[16+esi]
435	je	NEAR L$019ecb_enc_two
436	movups	xmm4,[32+esi]
437	cmp	eax,64
438	jb	NEAR L$020ecb_enc_three
439	movups	xmm5,[48+esi]
440	je	NEAR L$021ecb_enc_four
441	movups	xmm6,[64+esi]
442	xorps	xmm7,xmm7
443	call	__aesni_encrypt6
444	movups	[edi],xmm2
445	movups	[16+edi],xmm3
446	movups	[32+edi],xmm4
447	movups	[48+edi],xmm5
448	movups	[64+edi],xmm6
449	jmp	NEAR L$013ecb_ret
450align	16
451L$018ecb_enc_one:
452	movups	xmm0,[edx]
453	movups	xmm1,[16+edx]
454	lea	edx,[32+edx]
455	xorps	xmm2,xmm0
456L$022enc1_loop_3:
457db	102,15,56,220,209
458	dec	ecx
459	movups	xmm1,[edx]
460	lea	edx,[16+edx]
461	jnz	NEAR L$022enc1_loop_3
462db	102,15,56,221,209
463	movups	[edi],xmm2
464	jmp	NEAR L$013ecb_ret
465align	16
466L$019ecb_enc_two:
467	call	__aesni_encrypt2
468	movups	[edi],xmm2
469	movups	[16+edi],xmm3
470	jmp	NEAR L$013ecb_ret
471align	16
472L$020ecb_enc_three:
473	call	__aesni_encrypt3
474	movups	[edi],xmm2
475	movups	[16+edi],xmm3
476	movups	[32+edi],xmm4
477	jmp	NEAR L$013ecb_ret
478align	16
479L$021ecb_enc_four:
480	call	__aesni_encrypt4
481	movups	[edi],xmm2
482	movups	[16+edi],xmm3
483	movups	[32+edi],xmm4
484	movups	[48+edi],xmm5
485	jmp	NEAR L$013ecb_ret
486align	16
487L$014ecb_decrypt:
488	mov	ebp,edx
489	mov	ebx,ecx
490	cmp	eax,96
491	jb	NEAR L$023ecb_dec_tail
492	movdqu	xmm2,[esi]
493	movdqu	xmm3,[16+esi]
494	movdqu	xmm4,[32+esi]
495	movdqu	xmm5,[48+esi]
496	movdqu	xmm6,[64+esi]
497	movdqu	xmm7,[80+esi]
498	lea	esi,[96+esi]
499	sub	eax,96
500	jmp	NEAR L$024ecb_dec_loop6_enter
501align	16
502L$025ecb_dec_loop6:
503	movups	[edi],xmm2
504	movdqu	xmm2,[esi]
505	movups	[16+edi],xmm3
506	movdqu	xmm3,[16+esi]
507	movups	[32+edi],xmm4
508	movdqu	xmm4,[32+esi]
509	movups	[48+edi],xmm5
510	movdqu	xmm5,[48+esi]
511	movups	[64+edi],xmm6
512	movdqu	xmm6,[64+esi]
513	movups	[80+edi],xmm7
514	lea	edi,[96+edi]
515	movdqu	xmm7,[80+esi]
516	lea	esi,[96+esi]
517L$024ecb_dec_loop6_enter:
518	call	__aesni_decrypt6
519	mov	edx,ebp
520	mov	ecx,ebx
521	sub	eax,96
522	jnc	NEAR L$025ecb_dec_loop6
523	movups	[edi],xmm2
524	movups	[16+edi],xmm3
525	movups	[32+edi],xmm4
526	movups	[48+edi],xmm5
527	movups	[64+edi],xmm6
528	movups	[80+edi],xmm7
529	lea	edi,[96+edi]
530	add	eax,96
531	jz	NEAR L$013ecb_ret
532L$023ecb_dec_tail:
533	movups	xmm2,[esi]
534	cmp	eax,32
535	jb	NEAR L$026ecb_dec_one
536	movups	xmm3,[16+esi]
537	je	NEAR L$027ecb_dec_two
538	movups	xmm4,[32+esi]
539	cmp	eax,64
540	jb	NEAR L$028ecb_dec_three
541	movups	xmm5,[48+esi]
542	je	NEAR L$029ecb_dec_four
543	movups	xmm6,[64+esi]
544	xorps	xmm7,xmm7
545	call	__aesni_decrypt6
546	movups	[edi],xmm2
547	movups	[16+edi],xmm3
548	movups	[32+edi],xmm4
549	movups	[48+edi],xmm5
550	movups	[64+edi],xmm6
551	jmp	NEAR L$013ecb_ret
552align	16
553L$026ecb_dec_one:
554	movups	xmm0,[edx]
555	movups	xmm1,[16+edx]
556	lea	edx,[32+edx]
557	xorps	xmm2,xmm0
558L$030dec1_loop_4:
559db	102,15,56,222,209
560	dec	ecx
561	movups	xmm1,[edx]
562	lea	edx,[16+edx]
563	jnz	NEAR L$030dec1_loop_4
564db	102,15,56,223,209
565	movups	[edi],xmm2
566	jmp	NEAR L$013ecb_ret
567align	16
568L$027ecb_dec_two:
569	call	__aesni_decrypt2
570	movups	[edi],xmm2
571	movups	[16+edi],xmm3
572	jmp	NEAR L$013ecb_ret
573align	16
574L$028ecb_dec_three:
575	call	__aesni_decrypt3
576	movups	[edi],xmm2
577	movups	[16+edi],xmm3
578	movups	[32+edi],xmm4
579	jmp	NEAR L$013ecb_ret
580align	16
581L$029ecb_dec_four:
582	call	__aesni_decrypt4
583	movups	[edi],xmm2
584	movups	[16+edi],xmm3
585	movups	[32+edi],xmm4
586	movups	[48+edi],xmm5
587L$013ecb_ret:
588	pxor	xmm0,xmm0
589	pxor	xmm1,xmm1
590	pxor	xmm2,xmm2
591	pxor	xmm3,xmm3
592	pxor	xmm4,xmm4
593	pxor	xmm5,xmm5
594	pxor	xmm6,xmm6
595	pxor	xmm7,xmm7
596	pop	edi
597	pop	esi
598	pop	ebx
599	pop	ebp
600	ret
601global	_aes_hw_ccm64_encrypt_blocks
602align	16
603_aes_hw_ccm64_encrypt_blocks:
604L$_aes_hw_ccm64_encrypt_blocks_begin:
605	push	ebp
606	push	ebx
607	push	esi
608	push	edi
609	mov	esi,DWORD [20+esp]
610	mov	edi,DWORD [24+esp]
611	mov	eax,DWORD [28+esp]
612	mov	edx,DWORD [32+esp]
613	mov	ebx,DWORD [36+esp]
614	mov	ecx,DWORD [40+esp]
615	mov	ebp,esp
616	sub	esp,60
617	and	esp,-16
618	mov	DWORD [48+esp],ebp
619	movdqu	xmm7,[ebx]
620	movdqu	xmm3,[ecx]
621	mov	ecx,DWORD [240+edx]
622	mov	DWORD [esp],202182159
623	mov	DWORD [4+esp],134810123
624	mov	DWORD [8+esp],67438087
625	mov	DWORD [12+esp],66051
626	mov	ebx,1
627	xor	ebp,ebp
628	mov	DWORD [16+esp],ebx
629	mov	DWORD [20+esp],ebp
630	mov	DWORD [24+esp],ebp
631	mov	DWORD [28+esp],ebp
632	shl	ecx,4
633	mov	ebx,16
634	lea	ebp,[edx]
635	movdqa	xmm5,[esp]
636	movdqa	xmm2,xmm7
637	lea	edx,[32+ecx*1+edx]
638	sub	ebx,ecx
639db	102,15,56,0,253
640L$031ccm64_enc_outer:
641	movups	xmm0,[ebp]
642	mov	ecx,ebx
643	movups	xmm6,[esi]
644	xorps	xmm2,xmm0
645	movups	xmm1,[16+ebp]
646	xorps	xmm0,xmm6
647	xorps	xmm3,xmm0
648	movups	xmm0,[32+ebp]
649L$032ccm64_enc2_loop:
650db	102,15,56,220,209
651db	102,15,56,220,217
652	movups	xmm1,[ecx*1+edx]
653	add	ecx,32
654db	102,15,56,220,208
655db	102,15,56,220,216
656	movups	xmm0,[ecx*1+edx-16]
657	jnz	NEAR L$032ccm64_enc2_loop
658db	102,15,56,220,209
659db	102,15,56,220,217
660	paddq	xmm7,[16+esp]
661	dec	eax
662db	102,15,56,221,208
663db	102,15,56,221,216
664	lea	esi,[16+esi]
665	xorps	xmm6,xmm2
666	movdqa	xmm2,xmm7
667	movups	[edi],xmm6
668db	102,15,56,0,213
669	lea	edi,[16+edi]
670	jnz	NEAR L$031ccm64_enc_outer
671	mov	esp,DWORD [48+esp]
672	mov	edi,DWORD [40+esp]
673	movups	[edi],xmm3
674	pxor	xmm0,xmm0
675	pxor	xmm1,xmm1
676	pxor	xmm2,xmm2
677	pxor	xmm3,xmm3
678	pxor	xmm4,xmm4
679	pxor	xmm5,xmm5
680	pxor	xmm6,xmm6
681	pxor	xmm7,xmm7
682	pop	edi
683	pop	esi
684	pop	ebx
685	pop	ebp
686	ret
687global	_aes_hw_ccm64_decrypt_blocks
688align	16
689_aes_hw_ccm64_decrypt_blocks:
690L$_aes_hw_ccm64_decrypt_blocks_begin:
691	push	ebp
692	push	ebx
693	push	esi
694	push	edi
695	mov	esi,DWORD [20+esp]
696	mov	edi,DWORD [24+esp]
697	mov	eax,DWORD [28+esp]
698	mov	edx,DWORD [32+esp]
699	mov	ebx,DWORD [36+esp]
700	mov	ecx,DWORD [40+esp]
701	mov	ebp,esp
702	sub	esp,60
703	and	esp,-16
704	mov	DWORD [48+esp],ebp
705	movdqu	xmm7,[ebx]
706	movdqu	xmm3,[ecx]
707	mov	ecx,DWORD [240+edx]
708	mov	DWORD [esp],202182159
709	mov	DWORD [4+esp],134810123
710	mov	DWORD [8+esp],67438087
711	mov	DWORD [12+esp],66051
712	mov	ebx,1
713	xor	ebp,ebp
714	mov	DWORD [16+esp],ebx
715	mov	DWORD [20+esp],ebp
716	mov	DWORD [24+esp],ebp
717	mov	DWORD [28+esp],ebp
718	movdqa	xmm5,[esp]
719	movdqa	xmm2,xmm7
720	mov	ebp,edx
721	mov	ebx,ecx
722db	102,15,56,0,253
723	movups	xmm0,[edx]
724	movups	xmm1,[16+edx]
725	lea	edx,[32+edx]
726	xorps	xmm2,xmm0
727L$033enc1_loop_5:
728db	102,15,56,220,209
729	dec	ecx
730	movups	xmm1,[edx]
731	lea	edx,[16+edx]
732	jnz	NEAR L$033enc1_loop_5
733db	102,15,56,221,209
734	shl	ebx,4
735	mov	ecx,16
736	movups	xmm6,[esi]
737	paddq	xmm7,[16+esp]
738	lea	esi,[16+esi]
739	sub	ecx,ebx
740	lea	edx,[32+ebx*1+ebp]
741	mov	ebx,ecx
742	jmp	NEAR L$034ccm64_dec_outer
743align	16
744L$034ccm64_dec_outer:
745	xorps	xmm6,xmm2
746	movdqa	xmm2,xmm7
747	movups	[edi],xmm6
748	lea	edi,[16+edi]
749db	102,15,56,0,213
750	sub	eax,1
751	jz	NEAR L$035ccm64_dec_break
752	movups	xmm0,[ebp]
753	mov	ecx,ebx
754	movups	xmm1,[16+ebp]
755	xorps	xmm6,xmm0
756	xorps	xmm2,xmm0
757	xorps	xmm3,xmm6
758	movups	xmm0,[32+ebp]
759L$036ccm64_dec2_loop:
760db	102,15,56,220,209
761db	102,15,56,220,217
762	movups	xmm1,[ecx*1+edx]
763	add	ecx,32
764db	102,15,56,220,208
765db	102,15,56,220,216
766	movups	xmm0,[ecx*1+edx-16]
767	jnz	NEAR L$036ccm64_dec2_loop
768	movups	xmm6,[esi]
769	paddq	xmm7,[16+esp]
770db	102,15,56,220,209
771db	102,15,56,220,217
772db	102,15,56,221,208
773db	102,15,56,221,216
774	lea	esi,[16+esi]
775	jmp	NEAR L$034ccm64_dec_outer
776align	16
777L$035ccm64_dec_break:
778	mov	ecx,DWORD [240+ebp]
779	mov	edx,ebp
780	movups	xmm0,[edx]
781	movups	xmm1,[16+edx]
782	xorps	xmm6,xmm0
783	lea	edx,[32+edx]
784	xorps	xmm3,xmm6
785L$037enc1_loop_6:
786db	102,15,56,220,217
787	dec	ecx
788	movups	xmm1,[edx]
789	lea	edx,[16+edx]
790	jnz	NEAR L$037enc1_loop_6
791db	102,15,56,221,217
792	mov	esp,DWORD [48+esp]
793	mov	edi,DWORD [40+esp]
794	movups	[edi],xmm3
795	pxor	xmm0,xmm0
796	pxor	xmm1,xmm1
797	pxor	xmm2,xmm2
798	pxor	xmm3,xmm3
799	pxor	xmm4,xmm4
800	pxor	xmm5,xmm5
801	pxor	xmm6,xmm6
802	pxor	xmm7,xmm7
803	pop	edi
804	pop	esi
805	pop	ebx
806	pop	ebp
807	ret
808global	_aes_hw_ctr32_encrypt_blocks
809align	16
810_aes_hw_ctr32_encrypt_blocks:
811L$_aes_hw_ctr32_encrypt_blocks_begin:
812	push	ebp
813	push	ebx
814	push	esi
815	push	edi
816%ifdef BORINGSSL_DISPATCH_TEST
817	push	ebx
818	push	edx
819	call	L$038pic
820L$038pic:
821	pop	ebx
822	lea	ebx,[(_BORINGSSL_function_hit+0-L$038pic)+ebx]
823	mov	edx,1
824	mov	BYTE [ebx],dl
825	pop	edx
826	pop	ebx
827%endif
828	mov	esi,DWORD [20+esp]
829	mov	edi,DWORD [24+esp]
830	mov	eax,DWORD [28+esp]
831	mov	edx,DWORD [32+esp]
832	mov	ebx,DWORD [36+esp]
833	mov	ebp,esp
834	sub	esp,88
835	and	esp,-16
836	mov	DWORD [80+esp],ebp
837	cmp	eax,1
838	je	NEAR L$039ctr32_one_shortcut
839	movdqu	xmm7,[ebx]
840	mov	DWORD [esp],202182159
841	mov	DWORD [4+esp],134810123
842	mov	DWORD [8+esp],67438087
843	mov	DWORD [12+esp],66051
844	mov	ecx,6
845	xor	ebp,ebp
846	mov	DWORD [16+esp],ecx
847	mov	DWORD [20+esp],ecx
848	mov	DWORD [24+esp],ecx
849	mov	DWORD [28+esp],ebp
850db	102,15,58,22,251,3
851db	102,15,58,34,253,3
852	mov	ecx,DWORD [240+edx]
853	bswap	ebx
854	pxor	xmm0,xmm0
855	pxor	xmm1,xmm1
856	movdqa	xmm2,[esp]
857db	102,15,58,34,195,0
858	lea	ebp,[3+ebx]
859db	102,15,58,34,205,0
860	inc	ebx
861db	102,15,58,34,195,1
862	inc	ebp
863db	102,15,58,34,205,1
864	inc	ebx
865db	102,15,58,34,195,2
866	inc	ebp
867db	102,15,58,34,205,2
868	movdqa	[48+esp],xmm0
869db	102,15,56,0,194
870	movdqu	xmm6,[edx]
871	movdqa	[64+esp],xmm1
872db	102,15,56,0,202
873	pshufd	xmm2,xmm0,192
874	pshufd	xmm3,xmm0,128
875	cmp	eax,6
876	jb	NEAR L$040ctr32_tail
877	pxor	xmm7,xmm6
878	shl	ecx,4
879	mov	ebx,16
880	movdqa	[32+esp],xmm7
881	mov	ebp,edx
882	sub	ebx,ecx
883	lea	edx,[32+ecx*1+edx]
884	sub	eax,6
885	jmp	NEAR L$041ctr32_loop6
886align	16
887L$041ctr32_loop6:
888	pshufd	xmm4,xmm0,64
889	movdqa	xmm0,[32+esp]
890	pshufd	xmm5,xmm1,192
891	pxor	xmm2,xmm0
892	pshufd	xmm6,xmm1,128
893	pxor	xmm3,xmm0
894	pshufd	xmm7,xmm1,64
895	movups	xmm1,[16+ebp]
896	pxor	xmm4,xmm0
897	pxor	xmm5,xmm0
898db	102,15,56,220,209
899	pxor	xmm6,xmm0
900	pxor	xmm7,xmm0
901db	102,15,56,220,217
902	movups	xmm0,[32+ebp]
903	mov	ecx,ebx
904db	102,15,56,220,225
905db	102,15,56,220,233
906db	102,15,56,220,241
907db	102,15,56,220,249
908	call	L$_aesni_encrypt6_enter
909	movups	xmm1,[esi]
910	movups	xmm0,[16+esi]
911	xorps	xmm2,xmm1
912	movups	xmm1,[32+esi]
913	xorps	xmm3,xmm0
914	movups	[edi],xmm2
915	movdqa	xmm0,[16+esp]
916	xorps	xmm4,xmm1
917	movdqa	xmm1,[64+esp]
918	movups	[16+edi],xmm3
919	movups	[32+edi],xmm4
920	paddd	xmm1,xmm0
921	paddd	xmm0,[48+esp]
922	movdqa	xmm2,[esp]
923	movups	xmm3,[48+esi]
924	movups	xmm4,[64+esi]
925	xorps	xmm5,xmm3
926	movups	xmm3,[80+esi]
927	lea	esi,[96+esi]
928	movdqa	[48+esp],xmm0
929db	102,15,56,0,194
930	xorps	xmm6,xmm4
931	movups	[48+edi],xmm5
932	xorps	xmm7,xmm3
933	movdqa	[64+esp],xmm1
934db	102,15,56,0,202
935	movups	[64+edi],xmm6
936	pshufd	xmm2,xmm0,192
937	movups	[80+edi],xmm7
938	lea	edi,[96+edi]
939	pshufd	xmm3,xmm0,128
940	sub	eax,6
941	jnc	NEAR L$041ctr32_loop6
942	add	eax,6
943	jz	NEAR L$042ctr32_ret
944	movdqu	xmm7,[ebp]
945	mov	edx,ebp
946	pxor	xmm7,[32+esp]
947	mov	ecx,DWORD [240+ebp]
948L$040ctr32_tail:
949	por	xmm2,xmm7
950	cmp	eax,2
951	jb	NEAR L$043ctr32_one
952	pshufd	xmm4,xmm0,64
953	por	xmm3,xmm7
954	je	NEAR L$044ctr32_two
955	pshufd	xmm5,xmm1,192
956	por	xmm4,xmm7
957	cmp	eax,4
958	jb	NEAR L$045ctr32_three
959	pshufd	xmm6,xmm1,128
960	por	xmm5,xmm7
961	je	NEAR L$046ctr32_four
962	por	xmm6,xmm7
963	call	__aesni_encrypt6
964	movups	xmm1,[esi]
965	movups	xmm0,[16+esi]
966	xorps	xmm2,xmm1
967	movups	xmm1,[32+esi]
968	xorps	xmm3,xmm0
969	movups	xmm0,[48+esi]
970	xorps	xmm4,xmm1
971	movups	xmm1,[64+esi]
972	xorps	xmm5,xmm0
973	movups	[edi],xmm2
974	xorps	xmm6,xmm1
975	movups	[16+edi],xmm3
976	movups	[32+edi],xmm4
977	movups	[48+edi],xmm5
978	movups	[64+edi],xmm6
979	jmp	NEAR L$042ctr32_ret
980align	16
981L$039ctr32_one_shortcut:
982	movups	xmm2,[ebx]
983	mov	ecx,DWORD [240+edx]
984L$043ctr32_one:
985	movups	xmm0,[edx]
986	movups	xmm1,[16+edx]
987	lea	edx,[32+edx]
988	xorps	xmm2,xmm0
989L$047enc1_loop_7:
990db	102,15,56,220,209
991	dec	ecx
992	movups	xmm1,[edx]
993	lea	edx,[16+edx]
994	jnz	NEAR L$047enc1_loop_7
995db	102,15,56,221,209
996	movups	xmm6,[esi]
997	xorps	xmm6,xmm2
998	movups	[edi],xmm6
999	jmp	NEAR L$042ctr32_ret
1000align	16
1001L$044ctr32_two:
1002	call	__aesni_encrypt2
1003	movups	xmm5,[esi]
1004	movups	xmm6,[16+esi]
1005	xorps	xmm2,xmm5
1006	xorps	xmm3,xmm6
1007	movups	[edi],xmm2
1008	movups	[16+edi],xmm3
1009	jmp	NEAR L$042ctr32_ret
1010align	16
1011L$045ctr32_three:
1012	call	__aesni_encrypt3
1013	movups	xmm5,[esi]
1014	movups	xmm6,[16+esi]
1015	xorps	xmm2,xmm5
1016	movups	xmm7,[32+esi]
1017	xorps	xmm3,xmm6
1018	movups	[edi],xmm2
1019	xorps	xmm4,xmm7
1020	movups	[16+edi],xmm3
1021	movups	[32+edi],xmm4
1022	jmp	NEAR L$042ctr32_ret
1023align	16
1024L$046ctr32_four:
1025	call	__aesni_encrypt4
1026	movups	xmm6,[esi]
1027	movups	xmm7,[16+esi]
1028	movups	xmm1,[32+esi]
1029	xorps	xmm2,xmm6
1030	movups	xmm0,[48+esi]
1031	xorps	xmm3,xmm7
1032	movups	[edi],xmm2
1033	xorps	xmm4,xmm1
1034	movups	[16+edi],xmm3
1035	xorps	xmm5,xmm0
1036	movups	[32+edi],xmm4
1037	movups	[48+edi],xmm5
1038L$042ctr32_ret:
1039	pxor	xmm0,xmm0
1040	pxor	xmm1,xmm1
1041	pxor	xmm2,xmm2
1042	pxor	xmm3,xmm3
1043	pxor	xmm4,xmm4
1044	movdqa	[32+esp],xmm0
1045	pxor	xmm5,xmm5
1046	movdqa	[48+esp],xmm0
1047	pxor	xmm6,xmm6
1048	movdqa	[64+esp],xmm0
1049	pxor	xmm7,xmm7
1050	mov	esp,DWORD [80+esp]
1051	pop	edi
1052	pop	esi
1053	pop	ebx
1054	pop	ebp
1055	ret
1056global	_aes_hw_xts_encrypt
1057align	16
1058_aes_hw_xts_encrypt:
1059L$_aes_hw_xts_encrypt_begin:
1060	push	ebp
1061	push	ebx
1062	push	esi
1063	push	edi
1064	mov	edx,DWORD [36+esp]
1065	mov	esi,DWORD [40+esp]
1066	mov	ecx,DWORD [240+edx]
1067	movups	xmm2,[esi]
1068	movups	xmm0,[edx]
1069	movups	xmm1,[16+edx]
1070	lea	edx,[32+edx]
1071	xorps	xmm2,xmm0
1072L$048enc1_loop_8:
1073db	102,15,56,220,209
1074	dec	ecx
1075	movups	xmm1,[edx]
1076	lea	edx,[16+edx]
1077	jnz	NEAR L$048enc1_loop_8
1078db	102,15,56,221,209
1079	mov	esi,DWORD [20+esp]
1080	mov	edi,DWORD [24+esp]
1081	mov	eax,DWORD [28+esp]
1082	mov	edx,DWORD [32+esp]
1083	mov	ebp,esp
1084	sub	esp,120
1085	mov	ecx,DWORD [240+edx]
1086	and	esp,-16
1087	mov	DWORD [96+esp],135
1088	mov	DWORD [100+esp],0
1089	mov	DWORD [104+esp],1
1090	mov	DWORD [108+esp],0
1091	mov	DWORD [112+esp],eax
1092	mov	DWORD [116+esp],ebp
1093	movdqa	xmm1,xmm2
1094	pxor	xmm0,xmm0
1095	movdqa	xmm3,[96+esp]
1096	pcmpgtd	xmm0,xmm1
1097	and	eax,-16
1098	mov	ebp,edx
1099	mov	ebx,ecx
1100	sub	eax,96
1101	jc	NEAR L$049xts_enc_short
1102	shl	ecx,4
1103	mov	ebx,16
1104	sub	ebx,ecx
1105	lea	edx,[32+ecx*1+edx]
1106	jmp	NEAR L$050xts_enc_loop6
1107align	16
1108L$050xts_enc_loop6:
1109	pshufd	xmm2,xmm0,19
1110	pxor	xmm0,xmm0
1111	movdqa	[esp],xmm1
1112	paddq	xmm1,xmm1
1113	pand	xmm2,xmm3
1114	pcmpgtd	xmm0,xmm1
1115	pxor	xmm1,xmm2
1116	pshufd	xmm2,xmm0,19
1117	pxor	xmm0,xmm0
1118	movdqa	[16+esp],xmm1
1119	paddq	xmm1,xmm1
1120	pand	xmm2,xmm3
1121	pcmpgtd	xmm0,xmm1
1122	pxor	xmm1,xmm2
1123	pshufd	xmm2,xmm0,19
1124	pxor	xmm0,xmm0
1125	movdqa	[32+esp],xmm1
1126	paddq	xmm1,xmm1
1127	pand	xmm2,xmm3
1128	pcmpgtd	xmm0,xmm1
1129	pxor	xmm1,xmm2
1130	pshufd	xmm2,xmm0,19
1131	pxor	xmm0,xmm0
1132	movdqa	[48+esp],xmm1
1133	paddq	xmm1,xmm1
1134	pand	xmm2,xmm3
1135	pcmpgtd	xmm0,xmm1
1136	pxor	xmm1,xmm2
1137	pshufd	xmm7,xmm0,19
1138	movdqa	[64+esp],xmm1
1139	paddq	xmm1,xmm1
1140	movups	xmm0,[ebp]
1141	pand	xmm7,xmm3
1142	movups	xmm2,[esi]
1143	pxor	xmm7,xmm1
1144	mov	ecx,ebx
1145	movdqu	xmm3,[16+esi]
1146	xorps	xmm2,xmm0
1147	movdqu	xmm4,[32+esi]
1148	pxor	xmm3,xmm0
1149	movdqu	xmm5,[48+esi]
1150	pxor	xmm4,xmm0
1151	movdqu	xmm6,[64+esi]
1152	pxor	xmm5,xmm0
1153	movdqu	xmm1,[80+esi]
1154	pxor	xmm6,xmm0
1155	lea	esi,[96+esi]
1156	pxor	xmm2,[esp]
1157	movdqa	[80+esp],xmm7
1158	pxor	xmm7,xmm1
1159	movups	xmm1,[16+ebp]
1160	pxor	xmm3,[16+esp]
1161	pxor	xmm4,[32+esp]
1162db	102,15,56,220,209
1163	pxor	xmm5,[48+esp]
1164	pxor	xmm6,[64+esp]
1165db	102,15,56,220,217
1166	pxor	xmm7,xmm0
1167	movups	xmm0,[32+ebp]
1168db	102,15,56,220,225
1169db	102,15,56,220,233
1170db	102,15,56,220,241
1171db	102,15,56,220,249
1172	call	L$_aesni_encrypt6_enter
1173	movdqa	xmm1,[80+esp]
1174	pxor	xmm0,xmm0
1175	xorps	xmm2,[esp]
1176	pcmpgtd	xmm0,xmm1
1177	xorps	xmm3,[16+esp]
1178	movups	[edi],xmm2
1179	xorps	xmm4,[32+esp]
1180	movups	[16+edi],xmm3
1181	xorps	xmm5,[48+esp]
1182	movups	[32+edi],xmm4
1183	xorps	xmm6,[64+esp]
1184	movups	[48+edi],xmm5
1185	xorps	xmm7,xmm1
1186	movups	[64+edi],xmm6
1187	pshufd	xmm2,xmm0,19
1188	movups	[80+edi],xmm7
1189	lea	edi,[96+edi]
1190	movdqa	xmm3,[96+esp]
1191	pxor	xmm0,xmm0
1192	paddq	xmm1,xmm1
1193	pand	xmm2,xmm3
1194	pcmpgtd	xmm0,xmm1
1195	pxor	xmm1,xmm2
1196	sub	eax,96
1197	jnc	NEAR L$050xts_enc_loop6
1198	mov	ecx,DWORD [240+ebp]
1199	mov	edx,ebp
1200	mov	ebx,ecx
1201L$049xts_enc_short:
1202	add	eax,96
1203	jz	NEAR L$051xts_enc_done6x
1204	movdqa	xmm5,xmm1
1205	cmp	eax,32
1206	jb	NEAR L$052xts_enc_one
1207	pshufd	xmm2,xmm0,19
1208	pxor	xmm0,xmm0
1209	paddq	xmm1,xmm1
1210	pand	xmm2,xmm3
1211	pcmpgtd	xmm0,xmm1
1212	pxor	xmm1,xmm2
1213	je	NEAR L$053xts_enc_two
1214	pshufd	xmm2,xmm0,19
1215	pxor	xmm0,xmm0
1216	movdqa	xmm6,xmm1
1217	paddq	xmm1,xmm1
1218	pand	xmm2,xmm3
1219	pcmpgtd	xmm0,xmm1
1220	pxor	xmm1,xmm2
1221	cmp	eax,64
1222	jb	NEAR L$054xts_enc_three
1223	pshufd	xmm2,xmm0,19
1224	pxor	xmm0,xmm0
1225	movdqa	xmm7,xmm1
1226	paddq	xmm1,xmm1
1227	pand	xmm2,xmm3
1228	pcmpgtd	xmm0,xmm1
1229	pxor	xmm1,xmm2
1230	movdqa	[esp],xmm5
1231	movdqa	[16+esp],xmm6
1232	je	NEAR L$055xts_enc_four
1233	movdqa	[32+esp],xmm7
1234	pshufd	xmm7,xmm0,19
1235	movdqa	[48+esp],xmm1
1236	paddq	xmm1,xmm1
1237	pand	xmm7,xmm3
1238	pxor	xmm7,xmm1
1239	movdqu	xmm2,[esi]
1240	movdqu	xmm3,[16+esi]
1241	movdqu	xmm4,[32+esi]
1242	pxor	xmm2,[esp]
1243	movdqu	xmm5,[48+esi]
1244	pxor	xmm3,[16+esp]
1245	movdqu	xmm6,[64+esi]
1246	pxor	xmm4,[32+esp]
1247	lea	esi,[80+esi]
1248	pxor	xmm5,[48+esp]
1249	movdqa	[64+esp],xmm7
1250	pxor	xmm6,xmm7
1251	call	__aesni_encrypt6
1252	movaps	xmm1,[64+esp]
1253	xorps	xmm2,[esp]
1254	xorps	xmm3,[16+esp]
1255	xorps	xmm4,[32+esp]
1256	movups	[edi],xmm2
1257	xorps	xmm5,[48+esp]
1258	movups	[16+edi],xmm3
1259	xorps	xmm6,xmm1
1260	movups	[32+edi],xmm4
1261	movups	[48+edi],xmm5
1262	movups	[64+edi],xmm6
1263	lea	edi,[80+edi]
1264	jmp	NEAR L$056xts_enc_done
1265align	16
1266L$052xts_enc_one:
1267	movups	xmm2,[esi]
1268	lea	esi,[16+esi]
1269	xorps	xmm2,xmm5
1270	movups	xmm0,[edx]
1271	movups	xmm1,[16+edx]
1272	lea	edx,[32+edx]
1273	xorps	xmm2,xmm0
1274L$057enc1_loop_9:
1275db	102,15,56,220,209
1276	dec	ecx
1277	movups	xmm1,[edx]
1278	lea	edx,[16+edx]
1279	jnz	NEAR L$057enc1_loop_9
1280db	102,15,56,221,209
1281	xorps	xmm2,xmm5
1282	movups	[edi],xmm2
1283	lea	edi,[16+edi]
1284	movdqa	xmm1,xmm5
1285	jmp	NEAR L$056xts_enc_done
1286align	16
1287L$053xts_enc_two:
1288	movaps	xmm6,xmm1
1289	movups	xmm2,[esi]
1290	movups	xmm3,[16+esi]
1291	lea	esi,[32+esi]
1292	xorps	xmm2,xmm5
1293	xorps	xmm3,xmm6
1294	call	__aesni_encrypt2
1295	xorps	xmm2,xmm5
1296	xorps	xmm3,xmm6
1297	movups	[edi],xmm2
1298	movups	[16+edi],xmm3
1299	lea	edi,[32+edi]
1300	movdqa	xmm1,xmm6
1301	jmp	NEAR L$056xts_enc_done
1302align	16
1303L$054xts_enc_three:
1304	movaps	xmm7,xmm1
1305	movups	xmm2,[esi]
1306	movups	xmm3,[16+esi]
1307	movups	xmm4,[32+esi]
1308	lea	esi,[48+esi]
1309	xorps	xmm2,xmm5
1310	xorps	xmm3,xmm6
1311	xorps	xmm4,xmm7
1312	call	__aesni_encrypt3
1313	xorps	xmm2,xmm5
1314	xorps	xmm3,xmm6
1315	xorps	xmm4,xmm7
1316	movups	[edi],xmm2
1317	movups	[16+edi],xmm3
1318	movups	[32+edi],xmm4
1319	lea	edi,[48+edi]
1320	movdqa	xmm1,xmm7
1321	jmp	NEAR L$056xts_enc_done
1322align	16
1323L$055xts_enc_four:
1324	movaps	xmm6,xmm1
1325	movups	xmm2,[esi]
1326	movups	xmm3,[16+esi]
1327	movups	xmm4,[32+esi]
1328	xorps	xmm2,[esp]
1329	movups	xmm5,[48+esi]
1330	lea	esi,[64+esi]
1331	xorps	xmm3,[16+esp]
1332	xorps	xmm4,xmm7
1333	xorps	xmm5,xmm6
1334	call	__aesni_encrypt4
1335	xorps	xmm2,[esp]
1336	xorps	xmm3,[16+esp]
1337	xorps	xmm4,xmm7
1338	movups	[edi],xmm2
1339	xorps	xmm5,xmm6
1340	movups	[16+edi],xmm3
1341	movups	[32+edi],xmm4
1342	movups	[48+edi],xmm5
1343	lea	edi,[64+edi]
1344	movdqa	xmm1,xmm6
1345	jmp	NEAR L$056xts_enc_done
1346align	16
1347L$051xts_enc_done6x:
1348	mov	eax,DWORD [112+esp]
1349	and	eax,15
1350	jz	NEAR L$058xts_enc_ret
1351	movdqa	xmm5,xmm1
1352	mov	DWORD [112+esp],eax
1353	jmp	NEAR L$059xts_enc_steal
1354align	16
1355L$056xts_enc_done:
1356	mov	eax,DWORD [112+esp]
1357	pxor	xmm0,xmm0
1358	and	eax,15
1359	jz	NEAR L$058xts_enc_ret
1360	pcmpgtd	xmm0,xmm1
1361	mov	DWORD [112+esp],eax
1362	pshufd	xmm5,xmm0,19
1363	paddq	xmm1,xmm1
1364	pand	xmm5,[96+esp]
1365	pxor	xmm5,xmm1
1366L$059xts_enc_steal:
1367	movzx	ecx,BYTE [esi]
1368	movzx	edx,BYTE [edi-16]
1369	lea	esi,[1+esi]
1370	mov	BYTE [edi-16],cl
1371	mov	BYTE [edi],dl
1372	lea	edi,[1+edi]
1373	sub	eax,1
1374	jnz	NEAR L$059xts_enc_steal
1375	sub	edi,DWORD [112+esp]
1376	mov	edx,ebp
1377	mov	ecx,ebx
1378	movups	xmm2,[edi-16]
1379	xorps	xmm2,xmm5
1380	movups	xmm0,[edx]
1381	movups	xmm1,[16+edx]
1382	lea	edx,[32+edx]
1383	xorps	xmm2,xmm0
1384L$060enc1_loop_10:
1385db	102,15,56,220,209
1386	dec	ecx
1387	movups	xmm1,[edx]
1388	lea	edx,[16+edx]
1389	jnz	NEAR L$060enc1_loop_10
1390db	102,15,56,221,209
1391	xorps	xmm2,xmm5
1392	movups	[edi-16],xmm2
1393L$058xts_enc_ret:
1394	pxor	xmm0,xmm0
1395	pxor	xmm1,xmm1
1396	pxor	xmm2,xmm2
1397	movdqa	[esp],xmm0
1398	pxor	xmm3,xmm3
1399	movdqa	[16+esp],xmm0
1400	pxor	xmm4,xmm4
1401	movdqa	[32+esp],xmm0
1402	pxor	xmm5,xmm5
1403	movdqa	[48+esp],xmm0
1404	pxor	xmm6,xmm6
1405	movdqa	[64+esp],xmm0
1406	pxor	xmm7,xmm7
1407	movdqa	[80+esp],xmm0
1408	mov	esp,DWORD [116+esp]
1409	pop	edi
1410	pop	esi
1411	pop	ebx
1412	pop	ebp
1413	ret
1414global	_aes_hw_xts_decrypt
1415align	16
1416_aes_hw_xts_decrypt:
1417L$_aes_hw_xts_decrypt_begin:
1418	push	ebp
1419	push	ebx
1420	push	esi
1421	push	edi
1422	mov	edx,DWORD [36+esp]
1423	mov	esi,DWORD [40+esp]
1424	mov	ecx,DWORD [240+edx]
1425	movups	xmm2,[esi]
1426	movups	xmm0,[edx]
1427	movups	xmm1,[16+edx]
1428	lea	edx,[32+edx]
1429	xorps	xmm2,xmm0
1430L$061enc1_loop_11:
1431db	102,15,56,220,209
1432	dec	ecx
1433	movups	xmm1,[edx]
1434	lea	edx,[16+edx]
1435	jnz	NEAR L$061enc1_loop_11
1436db	102,15,56,221,209
1437	mov	esi,DWORD [20+esp]
1438	mov	edi,DWORD [24+esp]
1439	mov	eax,DWORD [28+esp]
1440	mov	edx,DWORD [32+esp]
1441	mov	ebp,esp
1442	sub	esp,120
1443	and	esp,-16
1444	xor	ebx,ebx
1445	test	eax,15
1446	setnz	bl
1447	shl	ebx,4
1448	sub	eax,ebx
1449	mov	DWORD [96+esp],135
1450	mov	DWORD [100+esp],0
1451	mov	DWORD [104+esp],1
1452	mov	DWORD [108+esp],0
1453	mov	DWORD [112+esp],eax
1454	mov	DWORD [116+esp],ebp
1455	mov	ecx,DWORD [240+edx]
1456	mov	ebp,edx
1457	mov	ebx,ecx
1458	movdqa	xmm1,xmm2
1459	pxor	xmm0,xmm0
1460	movdqa	xmm3,[96+esp]
1461	pcmpgtd	xmm0,xmm1
1462	and	eax,-16
1463	sub	eax,96
1464	jc	NEAR L$062xts_dec_short
1465	shl	ecx,4
1466	mov	ebx,16
1467	sub	ebx,ecx
1468	lea	edx,[32+ecx*1+edx]
1469	jmp	NEAR L$063xts_dec_loop6
1470align	16
1471L$063xts_dec_loop6:
1472	pshufd	xmm2,xmm0,19
1473	pxor	xmm0,xmm0
1474	movdqa	[esp],xmm1
1475	paddq	xmm1,xmm1
1476	pand	xmm2,xmm3
1477	pcmpgtd	xmm0,xmm1
1478	pxor	xmm1,xmm2
1479	pshufd	xmm2,xmm0,19
1480	pxor	xmm0,xmm0
1481	movdqa	[16+esp],xmm1
1482	paddq	xmm1,xmm1
1483	pand	xmm2,xmm3
1484	pcmpgtd	xmm0,xmm1
1485	pxor	xmm1,xmm2
1486	pshufd	xmm2,xmm0,19
1487	pxor	xmm0,xmm0
1488	movdqa	[32+esp],xmm1
1489	paddq	xmm1,xmm1
1490	pand	xmm2,xmm3
1491	pcmpgtd	xmm0,xmm1
1492	pxor	xmm1,xmm2
1493	pshufd	xmm2,xmm0,19
1494	pxor	xmm0,xmm0
1495	movdqa	[48+esp],xmm1
1496	paddq	xmm1,xmm1
1497	pand	xmm2,xmm3
1498	pcmpgtd	xmm0,xmm1
1499	pxor	xmm1,xmm2
1500	pshufd	xmm7,xmm0,19
1501	movdqa	[64+esp],xmm1
1502	paddq	xmm1,xmm1
1503	movups	xmm0,[ebp]
1504	pand	xmm7,xmm3
1505	movups	xmm2,[esi]
1506	pxor	xmm7,xmm1
1507	mov	ecx,ebx
1508	movdqu	xmm3,[16+esi]
1509	xorps	xmm2,xmm0
1510	movdqu	xmm4,[32+esi]
1511	pxor	xmm3,xmm0
1512	movdqu	xmm5,[48+esi]
1513	pxor	xmm4,xmm0
1514	movdqu	xmm6,[64+esi]
1515	pxor	xmm5,xmm0
1516	movdqu	xmm1,[80+esi]
1517	pxor	xmm6,xmm0
1518	lea	esi,[96+esi]
1519	pxor	xmm2,[esp]
1520	movdqa	[80+esp],xmm7
1521	pxor	xmm7,xmm1
1522	movups	xmm1,[16+ebp]
1523	pxor	xmm3,[16+esp]
1524	pxor	xmm4,[32+esp]
1525db	102,15,56,222,209
1526	pxor	xmm5,[48+esp]
1527	pxor	xmm6,[64+esp]
1528db	102,15,56,222,217
1529	pxor	xmm7,xmm0
1530	movups	xmm0,[32+ebp]
1531db	102,15,56,222,225
1532db	102,15,56,222,233
1533db	102,15,56,222,241
1534db	102,15,56,222,249
1535	call	L$_aesni_decrypt6_enter
1536	movdqa	xmm1,[80+esp]
1537	pxor	xmm0,xmm0
1538	xorps	xmm2,[esp]
1539	pcmpgtd	xmm0,xmm1
1540	xorps	xmm3,[16+esp]
1541	movups	[edi],xmm2
1542	xorps	xmm4,[32+esp]
1543	movups	[16+edi],xmm3
1544	xorps	xmm5,[48+esp]
1545	movups	[32+edi],xmm4
1546	xorps	xmm6,[64+esp]
1547	movups	[48+edi],xmm5
1548	xorps	xmm7,xmm1
1549	movups	[64+edi],xmm6
1550	pshufd	xmm2,xmm0,19
1551	movups	[80+edi],xmm7
1552	lea	edi,[96+edi]
1553	movdqa	xmm3,[96+esp]
1554	pxor	xmm0,xmm0
1555	paddq	xmm1,xmm1
1556	pand	xmm2,xmm3
1557	pcmpgtd	xmm0,xmm1
1558	pxor	xmm1,xmm2
1559	sub	eax,96
1560	jnc	NEAR L$063xts_dec_loop6
1561	mov	ecx,DWORD [240+ebp]
1562	mov	edx,ebp
1563	mov	ebx,ecx
1564L$062xts_dec_short:
1565	add	eax,96
1566	jz	NEAR L$064xts_dec_done6x
1567	movdqa	xmm5,xmm1
1568	cmp	eax,32
1569	jb	NEAR L$065xts_dec_one
1570	pshufd	xmm2,xmm0,19
1571	pxor	xmm0,xmm0
1572	paddq	xmm1,xmm1
1573	pand	xmm2,xmm3
1574	pcmpgtd	xmm0,xmm1
1575	pxor	xmm1,xmm2
1576	je	NEAR L$066xts_dec_two
1577	pshufd	xmm2,xmm0,19
1578	pxor	xmm0,xmm0
1579	movdqa	xmm6,xmm1
1580	paddq	xmm1,xmm1
1581	pand	xmm2,xmm3
1582	pcmpgtd	xmm0,xmm1
1583	pxor	xmm1,xmm2
1584	cmp	eax,64
1585	jb	NEAR L$067xts_dec_three
1586	pshufd	xmm2,xmm0,19
1587	pxor	xmm0,xmm0
1588	movdqa	xmm7,xmm1
1589	paddq	xmm1,xmm1
1590	pand	xmm2,xmm3
1591	pcmpgtd	xmm0,xmm1
1592	pxor	xmm1,xmm2
1593	movdqa	[esp],xmm5
1594	movdqa	[16+esp],xmm6
1595	je	NEAR L$068xts_dec_four
1596	movdqa	[32+esp],xmm7
1597	pshufd	xmm7,xmm0,19
1598	movdqa	[48+esp],xmm1
1599	paddq	xmm1,xmm1
1600	pand	xmm7,xmm3
1601	pxor	xmm7,xmm1
1602	movdqu	xmm2,[esi]
1603	movdqu	xmm3,[16+esi]
1604	movdqu	xmm4,[32+esi]
1605	pxor	xmm2,[esp]
1606	movdqu	xmm5,[48+esi]
1607	pxor	xmm3,[16+esp]
1608	movdqu	xmm6,[64+esi]
1609	pxor	xmm4,[32+esp]
1610	lea	esi,[80+esi]
1611	pxor	xmm5,[48+esp]
1612	movdqa	[64+esp],xmm7
1613	pxor	xmm6,xmm7
1614	call	__aesni_decrypt6
1615	movaps	xmm1,[64+esp]
1616	xorps	xmm2,[esp]
1617	xorps	xmm3,[16+esp]
1618	xorps	xmm4,[32+esp]
1619	movups	[edi],xmm2
1620	xorps	xmm5,[48+esp]
1621	movups	[16+edi],xmm3
1622	xorps	xmm6,xmm1
1623	movups	[32+edi],xmm4
1624	movups	[48+edi],xmm5
1625	movups	[64+edi],xmm6
1626	lea	edi,[80+edi]
1627	jmp	NEAR L$069xts_dec_done
1628align	16
1629L$065xts_dec_one:
1630	movups	xmm2,[esi]
1631	lea	esi,[16+esi]
1632	xorps	xmm2,xmm5
1633	movups	xmm0,[edx]
1634	movups	xmm1,[16+edx]
1635	lea	edx,[32+edx]
1636	xorps	xmm2,xmm0
1637L$070dec1_loop_12:
1638db	102,15,56,222,209
1639	dec	ecx
1640	movups	xmm1,[edx]
1641	lea	edx,[16+edx]
1642	jnz	NEAR L$070dec1_loop_12
1643db	102,15,56,223,209
1644	xorps	xmm2,xmm5
1645	movups	[edi],xmm2
1646	lea	edi,[16+edi]
1647	movdqa	xmm1,xmm5
1648	jmp	NEAR L$069xts_dec_done
1649align	16
1650L$066xts_dec_two:
1651	movaps	xmm6,xmm1
1652	movups	xmm2,[esi]
1653	movups	xmm3,[16+esi]
1654	lea	esi,[32+esi]
1655	xorps	xmm2,xmm5
1656	xorps	xmm3,xmm6
1657	call	__aesni_decrypt2
1658	xorps	xmm2,xmm5
1659	xorps	xmm3,xmm6
1660	movups	[edi],xmm2
1661	movups	[16+edi],xmm3
1662	lea	edi,[32+edi]
1663	movdqa	xmm1,xmm6
1664	jmp	NEAR L$069xts_dec_done
1665align	16
1666L$067xts_dec_three:
1667	movaps	xmm7,xmm1
1668	movups	xmm2,[esi]
1669	movups	xmm3,[16+esi]
1670	movups	xmm4,[32+esi]
1671	lea	esi,[48+esi]
1672	xorps	xmm2,xmm5
1673	xorps	xmm3,xmm6
1674	xorps	xmm4,xmm7
1675	call	__aesni_decrypt3
1676	xorps	xmm2,xmm5
1677	xorps	xmm3,xmm6
1678	xorps	xmm4,xmm7
1679	movups	[edi],xmm2
1680	movups	[16+edi],xmm3
1681	movups	[32+edi],xmm4
1682	lea	edi,[48+edi]
1683	movdqa	xmm1,xmm7
1684	jmp	NEAR L$069xts_dec_done
1685align	16
1686L$068xts_dec_four:
1687	movaps	xmm6,xmm1
1688	movups	xmm2,[esi]
1689	movups	xmm3,[16+esi]
1690	movups	xmm4,[32+esi]
1691	xorps	xmm2,[esp]
1692	movups	xmm5,[48+esi]
1693	lea	esi,[64+esi]
1694	xorps	xmm3,[16+esp]
1695	xorps	xmm4,xmm7
1696	xorps	xmm5,xmm6
1697	call	__aesni_decrypt4
1698	xorps	xmm2,[esp]
1699	xorps	xmm3,[16+esp]
1700	xorps	xmm4,xmm7
1701	movups	[edi],xmm2
1702	xorps	xmm5,xmm6
1703	movups	[16+edi],xmm3
1704	movups	[32+edi],xmm4
1705	movups	[48+edi],xmm5
1706	lea	edi,[64+edi]
1707	movdqa	xmm1,xmm6
1708	jmp	NEAR L$069xts_dec_done
1709align	16
1710L$064xts_dec_done6x:
1711	mov	eax,DWORD [112+esp]
1712	and	eax,15
1713	jz	NEAR L$071xts_dec_ret
1714	mov	DWORD [112+esp],eax
1715	jmp	NEAR L$072xts_dec_only_one_more
1716align	16
1717L$069xts_dec_done:
1718	mov	eax,DWORD [112+esp]
1719	pxor	xmm0,xmm0
1720	and	eax,15
1721	jz	NEAR L$071xts_dec_ret
1722	pcmpgtd	xmm0,xmm1
1723	mov	DWORD [112+esp],eax
1724	pshufd	xmm2,xmm0,19
1725	pxor	xmm0,xmm0
1726	movdqa	xmm3,[96+esp]
1727	paddq	xmm1,xmm1
1728	pand	xmm2,xmm3
1729	pcmpgtd	xmm0,xmm1
1730	pxor	xmm1,xmm2
1731L$072xts_dec_only_one_more:
1732	pshufd	xmm5,xmm0,19
1733	movdqa	xmm6,xmm1
1734	paddq	xmm1,xmm1
1735	pand	xmm5,xmm3
1736	pxor	xmm5,xmm1
1737	mov	edx,ebp
1738	mov	ecx,ebx
1739	movups	xmm2,[esi]
1740	xorps	xmm2,xmm5
1741	movups	xmm0,[edx]
1742	movups	xmm1,[16+edx]
1743	lea	edx,[32+edx]
1744	xorps	xmm2,xmm0
1745L$073dec1_loop_13:
1746db	102,15,56,222,209
1747	dec	ecx
1748	movups	xmm1,[edx]
1749	lea	edx,[16+edx]
1750	jnz	NEAR L$073dec1_loop_13
1751db	102,15,56,223,209
1752	xorps	xmm2,xmm5
1753	movups	[edi],xmm2
1754L$074xts_dec_steal:
1755	movzx	ecx,BYTE [16+esi]
1756	movzx	edx,BYTE [edi]
1757	lea	esi,[1+esi]
1758	mov	BYTE [edi],cl
1759	mov	BYTE [16+edi],dl
1760	lea	edi,[1+edi]
1761	sub	eax,1
1762	jnz	NEAR L$074xts_dec_steal
1763	sub	edi,DWORD [112+esp]
1764	mov	edx,ebp
1765	mov	ecx,ebx
1766	movups	xmm2,[edi]
1767	xorps	xmm2,xmm6
1768	movups	xmm0,[edx]
1769	movups	xmm1,[16+edx]
1770	lea	edx,[32+edx]
1771	xorps	xmm2,xmm0
1772L$075dec1_loop_14:
1773db	102,15,56,222,209
1774	dec	ecx
1775	movups	xmm1,[edx]
1776	lea	edx,[16+edx]
1777	jnz	NEAR L$075dec1_loop_14
1778db	102,15,56,223,209
1779	xorps	xmm2,xmm6
1780	movups	[edi],xmm2
1781L$071xts_dec_ret:
1782	pxor	xmm0,xmm0
1783	pxor	xmm1,xmm1
1784	pxor	xmm2,xmm2
1785	movdqa	[esp],xmm0
1786	pxor	xmm3,xmm3
1787	movdqa	[16+esp],xmm0
1788	pxor	xmm4,xmm4
1789	movdqa	[32+esp],xmm0
1790	pxor	xmm5,xmm5
1791	movdqa	[48+esp],xmm0
1792	pxor	xmm6,xmm6
1793	movdqa	[64+esp],xmm0
1794	pxor	xmm7,xmm7
1795	movdqa	[80+esp],xmm0
1796	mov	esp,DWORD [116+esp]
1797	pop	edi
1798	pop	esi
1799	pop	ebx
1800	pop	ebp
1801	ret
1802global	_aes_hw_cbc_encrypt
1803align	16
1804_aes_hw_cbc_encrypt:
1805L$_aes_hw_cbc_encrypt_begin:
1806	push	ebp
1807	push	ebx
1808	push	esi
1809	push	edi
1810	mov	esi,DWORD [20+esp]
1811	mov	ebx,esp
1812	mov	edi,DWORD [24+esp]
1813	sub	ebx,24
1814	mov	eax,DWORD [28+esp]
1815	and	ebx,-16
1816	mov	edx,DWORD [32+esp]
1817	mov	ebp,DWORD [36+esp]
1818	test	eax,eax
1819	jz	NEAR L$076cbc_abort
1820	cmp	DWORD [40+esp],0
1821	xchg	ebx,esp
1822	movups	xmm7,[ebp]
1823	mov	ecx,DWORD [240+edx]
1824	mov	ebp,edx
1825	mov	DWORD [16+esp],ebx
1826	mov	ebx,ecx
1827	je	NEAR L$077cbc_decrypt
1828	movaps	xmm2,xmm7
1829	cmp	eax,16
1830	jb	NEAR L$078cbc_enc_tail
1831	sub	eax,16
1832	jmp	NEAR L$079cbc_enc_loop
1833align	16
1834L$079cbc_enc_loop:
1835	movups	xmm7,[esi]
1836	lea	esi,[16+esi]
1837	movups	xmm0,[edx]
1838	movups	xmm1,[16+edx]
1839	xorps	xmm7,xmm0
1840	lea	edx,[32+edx]
1841	xorps	xmm2,xmm7
1842L$080enc1_loop_15:
1843db	102,15,56,220,209
1844	dec	ecx
1845	movups	xmm1,[edx]
1846	lea	edx,[16+edx]
1847	jnz	NEAR L$080enc1_loop_15
1848db	102,15,56,221,209
1849	mov	ecx,ebx
1850	mov	edx,ebp
1851	movups	[edi],xmm2
1852	lea	edi,[16+edi]
1853	sub	eax,16
1854	jnc	NEAR L$079cbc_enc_loop
1855	add	eax,16
1856	jnz	NEAR L$078cbc_enc_tail
1857	movaps	xmm7,xmm2
1858	pxor	xmm2,xmm2
1859	jmp	NEAR L$081cbc_ret
1860L$078cbc_enc_tail:
1861	mov	ecx,eax
1862dd	2767451785
1863	mov	ecx,16
1864	sub	ecx,eax
1865	xor	eax,eax
1866dd	2868115081
1867	lea	edi,[edi-16]
1868	mov	ecx,ebx
1869	mov	esi,edi
1870	mov	edx,ebp
1871	jmp	NEAR L$079cbc_enc_loop
1872align	16
1873L$077cbc_decrypt:
1874	cmp	eax,80
1875	jbe	NEAR L$082cbc_dec_tail
1876	movaps	[esp],xmm7
1877	sub	eax,80
1878	jmp	NEAR L$083cbc_dec_loop6_enter
1879align	16
1880L$084cbc_dec_loop6:
1881	movaps	[esp],xmm0
1882	movups	[edi],xmm7
1883	lea	edi,[16+edi]
1884L$083cbc_dec_loop6_enter:
1885	movdqu	xmm2,[esi]
1886	movdqu	xmm3,[16+esi]
1887	movdqu	xmm4,[32+esi]
1888	movdqu	xmm5,[48+esi]
1889	movdqu	xmm6,[64+esi]
1890	movdqu	xmm7,[80+esi]
1891	call	__aesni_decrypt6
1892	movups	xmm1,[esi]
1893	movups	xmm0,[16+esi]
1894	xorps	xmm2,[esp]
1895	xorps	xmm3,xmm1
1896	movups	xmm1,[32+esi]
1897	xorps	xmm4,xmm0
1898	movups	xmm0,[48+esi]
1899	xorps	xmm5,xmm1
1900	movups	xmm1,[64+esi]
1901	xorps	xmm6,xmm0
1902	movups	xmm0,[80+esi]
1903	xorps	xmm7,xmm1
1904	movups	[edi],xmm2
1905	movups	[16+edi],xmm3
1906	lea	esi,[96+esi]
1907	movups	[32+edi],xmm4
1908	mov	ecx,ebx
1909	movups	[48+edi],xmm5
1910	mov	edx,ebp
1911	movups	[64+edi],xmm6
1912	lea	edi,[80+edi]
1913	sub	eax,96
1914	ja	NEAR L$084cbc_dec_loop6
1915	movaps	xmm2,xmm7
1916	movaps	xmm7,xmm0
1917	add	eax,80
1918	jle	NEAR L$085cbc_dec_clear_tail_collected
1919	movups	[edi],xmm2
1920	lea	edi,[16+edi]
1921L$082cbc_dec_tail:
1922	movups	xmm2,[esi]
1923	movaps	xmm6,xmm2
1924	cmp	eax,16
1925	jbe	NEAR L$086cbc_dec_one
1926	movups	xmm3,[16+esi]
1927	movaps	xmm5,xmm3
1928	cmp	eax,32
1929	jbe	NEAR L$087cbc_dec_two
1930	movups	xmm4,[32+esi]
1931	cmp	eax,48
1932	jbe	NEAR L$088cbc_dec_three
1933	movups	xmm5,[48+esi]
1934	cmp	eax,64
1935	jbe	NEAR L$089cbc_dec_four
1936	movups	xmm6,[64+esi]
1937	movaps	[esp],xmm7
1938	movups	xmm2,[esi]
1939	xorps	xmm7,xmm7
1940	call	__aesni_decrypt6
1941	movups	xmm1,[esi]
1942	movups	xmm0,[16+esi]
1943	xorps	xmm2,[esp]
1944	xorps	xmm3,xmm1
1945	movups	xmm1,[32+esi]
1946	xorps	xmm4,xmm0
1947	movups	xmm0,[48+esi]
1948	xorps	xmm5,xmm1
1949	movups	xmm7,[64+esi]
1950	xorps	xmm6,xmm0
1951	movups	[edi],xmm2
1952	movups	[16+edi],xmm3
1953	pxor	xmm3,xmm3
1954	movups	[32+edi],xmm4
1955	pxor	xmm4,xmm4
1956	movups	[48+edi],xmm5
1957	pxor	xmm5,xmm5
1958	lea	edi,[64+edi]
1959	movaps	xmm2,xmm6
1960	pxor	xmm6,xmm6
1961	sub	eax,80
1962	jmp	NEAR L$090cbc_dec_tail_collected
1963align	16
1964L$086cbc_dec_one:
1965	movups	xmm0,[edx]
1966	movups	xmm1,[16+edx]
1967	lea	edx,[32+edx]
1968	xorps	xmm2,xmm0
1969L$091dec1_loop_16:
1970db	102,15,56,222,209
1971	dec	ecx
1972	movups	xmm1,[edx]
1973	lea	edx,[16+edx]
1974	jnz	NEAR L$091dec1_loop_16
1975db	102,15,56,223,209
1976	xorps	xmm2,xmm7
1977	movaps	xmm7,xmm6
1978	sub	eax,16
1979	jmp	NEAR L$090cbc_dec_tail_collected
1980align	16
1981L$087cbc_dec_two:
1982	call	__aesni_decrypt2
1983	xorps	xmm2,xmm7
1984	xorps	xmm3,xmm6
1985	movups	[edi],xmm2
1986	movaps	xmm2,xmm3
1987	pxor	xmm3,xmm3
1988	lea	edi,[16+edi]
1989	movaps	xmm7,xmm5
1990	sub	eax,32
1991	jmp	NEAR L$090cbc_dec_tail_collected
1992align	16
1993L$088cbc_dec_three:
1994	call	__aesni_decrypt3
1995	xorps	xmm2,xmm7
1996	xorps	xmm3,xmm6
1997	xorps	xmm4,xmm5
1998	movups	[edi],xmm2
1999	movaps	xmm2,xmm4
2000	pxor	xmm4,xmm4
2001	movups	[16+edi],xmm3
2002	pxor	xmm3,xmm3
2003	lea	edi,[32+edi]
2004	movups	xmm7,[32+esi]
2005	sub	eax,48
2006	jmp	NEAR L$090cbc_dec_tail_collected
2007align	16
2008L$089cbc_dec_four:
2009	call	__aesni_decrypt4
2010	movups	xmm1,[16+esi]
2011	movups	xmm0,[32+esi]
2012	xorps	xmm2,xmm7
2013	movups	xmm7,[48+esi]
2014	xorps	xmm3,xmm6
2015	movups	[edi],xmm2
2016	xorps	xmm4,xmm1
2017	movups	[16+edi],xmm3
2018	pxor	xmm3,xmm3
2019	xorps	xmm5,xmm0
2020	movups	[32+edi],xmm4
2021	pxor	xmm4,xmm4
2022	lea	edi,[48+edi]
2023	movaps	xmm2,xmm5
2024	pxor	xmm5,xmm5
2025	sub	eax,64
2026	jmp	NEAR L$090cbc_dec_tail_collected
2027align	16
2028L$085cbc_dec_clear_tail_collected:
2029	pxor	xmm3,xmm3
2030	pxor	xmm4,xmm4
2031	pxor	xmm5,xmm5
2032	pxor	xmm6,xmm6
2033L$090cbc_dec_tail_collected:
2034	and	eax,15
2035	jnz	NEAR L$092cbc_dec_tail_partial
2036	movups	[edi],xmm2
2037	pxor	xmm0,xmm0
2038	jmp	NEAR L$081cbc_ret
2039align	16
2040L$092cbc_dec_tail_partial:
2041	movaps	[esp],xmm2
2042	pxor	xmm0,xmm0
2043	mov	ecx,16
2044	mov	esi,esp
2045	sub	ecx,eax
2046dd	2767451785
2047	movdqa	[esp],xmm2
2048L$081cbc_ret:
2049	mov	esp,DWORD [16+esp]
2050	mov	ebp,DWORD [36+esp]
2051	pxor	xmm2,xmm2
2052	pxor	xmm1,xmm1
2053	movups	[ebp],xmm7
2054	pxor	xmm7,xmm7
2055L$076cbc_abort:
2056	pop	edi
2057	pop	esi
2058	pop	ebx
2059	pop	ebp
2060	ret
2061align	16
2062__aesni_set_encrypt_key:
2063	push	ebp
2064	push	ebx
2065	test	eax,eax
2066	jz	NEAR L$093bad_pointer
2067	test	edx,edx
2068	jz	NEAR L$093bad_pointer
2069	call	L$094pic
2070L$094pic:
2071	pop	ebx
2072	lea	ebx,[(L$key_const-L$094pic)+ebx]
2073	lea	ebp,[_OPENSSL_ia32cap_P]
2074	movups	xmm0,[eax]
2075	xorps	xmm4,xmm4
2076	mov	ebp,DWORD [4+ebp]
2077	lea	edx,[16+edx]
2078	and	ebp,268437504
2079	cmp	ecx,256
2080	je	NEAR L$09514rounds
2081	cmp	ecx,192
2082	je	NEAR L$09612rounds
2083	cmp	ecx,128
2084	jne	NEAR L$097bad_keybits
2085align	16
2086L$09810rounds:
2087	cmp	ebp,268435456
2088	je	NEAR L$09910rounds_alt
2089	mov	ecx,9
2090	movups	[edx-16],xmm0
2091db	102,15,58,223,200,1
2092	call	L$100key_128_cold
2093db	102,15,58,223,200,2
2094	call	L$101key_128
2095db	102,15,58,223,200,4
2096	call	L$101key_128
2097db	102,15,58,223,200,8
2098	call	L$101key_128
2099db	102,15,58,223,200,16
2100	call	L$101key_128
2101db	102,15,58,223,200,32
2102	call	L$101key_128
2103db	102,15,58,223,200,64
2104	call	L$101key_128
2105db	102,15,58,223,200,128
2106	call	L$101key_128
2107db	102,15,58,223,200,27
2108	call	L$101key_128
2109db	102,15,58,223,200,54
2110	call	L$101key_128
2111	movups	[edx],xmm0
2112	mov	DWORD [80+edx],ecx
2113	jmp	NEAR L$102good_key
2114align	16
2115L$101key_128:
2116	movups	[edx],xmm0
2117	lea	edx,[16+edx]
2118L$100key_128_cold:
2119	shufps	xmm4,xmm0,16
2120	xorps	xmm0,xmm4
2121	shufps	xmm4,xmm0,140
2122	xorps	xmm0,xmm4
2123	shufps	xmm1,xmm1,255
2124	xorps	xmm0,xmm1
2125	ret
2126align	16
2127L$09910rounds_alt:
2128	movdqa	xmm5,[ebx]
2129	mov	ecx,8
2130	movdqa	xmm4,[32+ebx]
2131	movdqa	xmm2,xmm0
2132	movdqu	[edx-16],xmm0
2133L$103loop_key128:
2134db	102,15,56,0,197
2135db	102,15,56,221,196
2136	pslld	xmm4,1
2137	lea	edx,[16+edx]
2138	movdqa	xmm3,xmm2
2139	pslldq	xmm2,4
2140	pxor	xmm3,xmm2
2141	pslldq	xmm2,4
2142	pxor	xmm3,xmm2
2143	pslldq	xmm2,4
2144	pxor	xmm2,xmm3
2145	pxor	xmm0,xmm2
2146	movdqu	[edx-16],xmm0
2147	movdqa	xmm2,xmm0
2148	dec	ecx
2149	jnz	NEAR L$103loop_key128
2150	movdqa	xmm4,[48+ebx]
2151db	102,15,56,0,197
2152db	102,15,56,221,196
2153	pslld	xmm4,1
2154	movdqa	xmm3,xmm2
2155	pslldq	xmm2,4
2156	pxor	xmm3,xmm2
2157	pslldq	xmm2,4
2158	pxor	xmm3,xmm2
2159	pslldq	xmm2,4
2160	pxor	xmm2,xmm3
2161	pxor	xmm0,xmm2
2162	movdqu	[edx],xmm0
2163	movdqa	xmm2,xmm0
2164db	102,15,56,0,197
2165db	102,15,56,221,196
2166	movdqa	xmm3,xmm2
2167	pslldq	xmm2,4
2168	pxor	xmm3,xmm2
2169	pslldq	xmm2,4
2170	pxor	xmm3,xmm2
2171	pslldq	xmm2,4
2172	pxor	xmm2,xmm3
2173	pxor	xmm0,xmm2
2174	movdqu	[16+edx],xmm0
2175	mov	ecx,9
2176	mov	DWORD [96+edx],ecx
2177	jmp	NEAR L$102good_key
2178align	16
2179L$09612rounds:
2180	movq	xmm2,[16+eax]
2181	cmp	ebp,268435456
2182	je	NEAR L$10412rounds_alt
2183	mov	ecx,11
2184	movups	[edx-16],xmm0
2185db	102,15,58,223,202,1
2186	call	L$105key_192a_cold
2187db	102,15,58,223,202,2
2188	call	L$106key_192b
2189db	102,15,58,223,202,4
2190	call	L$107key_192a
2191db	102,15,58,223,202,8
2192	call	L$106key_192b
2193db	102,15,58,223,202,16
2194	call	L$107key_192a
2195db	102,15,58,223,202,32
2196	call	L$106key_192b
2197db	102,15,58,223,202,64
2198	call	L$107key_192a
2199db	102,15,58,223,202,128
2200	call	L$106key_192b
2201	movups	[edx],xmm0
2202	mov	DWORD [48+edx],ecx
2203	jmp	NEAR L$102good_key
2204align	16
2205L$107key_192a:
2206	movups	[edx],xmm0
2207	lea	edx,[16+edx]
2208align	16
2209L$105key_192a_cold:
2210	movaps	xmm5,xmm2
2211L$108key_192b_warm:
2212	shufps	xmm4,xmm0,16
2213	movdqa	xmm3,xmm2
2214	xorps	xmm0,xmm4
2215	shufps	xmm4,xmm0,140
2216	pslldq	xmm3,4
2217	xorps	xmm0,xmm4
2218	pshufd	xmm1,xmm1,85
2219	pxor	xmm2,xmm3
2220	pxor	xmm0,xmm1
2221	pshufd	xmm3,xmm0,255
2222	pxor	xmm2,xmm3
2223	ret
2224align	16
2225L$106key_192b:
2226	movaps	xmm3,xmm0
2227	shufps	xmm5,xmm0,68
2228	movups	[edx],xmm5
2229	shufps	xmm3,xmm2,78
2230	movups	[16+edx],xmm3
2231	lea	edx,[32+edx]
2232	jmp	NEAR L$108key_192b_warm
2233align	16
2234L$10412rounds_alt:
2235	movdqa	xmm5,[16+ebx]
2236	movdqa	xmm4,[32+ebx]
2237	mov	ecx,8
2238	movdqu	[edx-16],xmm0
2239L$109loop_key192:
2240	movq	[edx],xmm2
2241	movdqa	xmm1,xmm2
2242db	102,15,56,0,213
2243db	102,15,56,221,212
2244	pslld	xmm4,1
2245	lea	edx,[24+edx]
2246	movdqa	xmm3,xmm0
2247	pslldq	xmm0,4
2248	pxor	xmm3,xmm0
2249	pslldq	xmm0,4
2250	pxor	xmm3,xmm0
2251	pslldq	xmm0,4
2252	pxor	xmm0,xmm3
2253	pshufd	xmm3,xmm0,255
2254	pxor	xmm3,xmm1
2255	pslldq	xmm1,4
2256	pxor	xmm3,xmm1
2257	pxor	xmm0,xmm2
2258	pxor	xmm2,xmm3
2259	movdqu	[edx-16],xmm0
2260	dec	ecx
2261	jnz	NEAR L$109loop_key192
2262	mov	ecx,11
2263	mov	DWORD [32+edx],ecx
2264	jmp	NEAR L$102good_key
2265align	16
2266L$09514rounds:
2267	movups	xmm2,[16+eax]
2268	lea	edx,[16+edx]
2269	cmp	ebp,268435456
2270	je	NEAR L$11014rounds_alt
2271	mov	ecx,13
2272	movups	[edx-32],xmm0
2273	movups	[edx-16],xmm2
2274db	102,15,58,223,202,1
2275	call	L$111key_256a_cold
2276db	102,15,58,223,200,1
2277	call	L$112key_256b
2278db	102,15,58,223,202,2
2279	call	L$113key_256a
2280db	102,15,58,223,200,2
2281	call	L$112key_256b
2282db	102,15,58,223,202,4
2283	call	L$113key_256a
2284db	102,15,58,223,200,4
2285	call	L$112key_256b
2286db	102,15,58,223,202,8
2287	call	L$113key_256a
2288db	102,15,58,223,200,8
2289	call	L$112key_256b
2290db	102,15,58,223,202,16
2291	call	L$113key_256a
2292db	102,15,58,223,200,16
2293	call	L$112key_256b
2294db	102,15,58,223,202,32
2295	call	L$113key_256a
2296db	102,15,58,223,200,32
2297	call	L$112key_256b
2298db	102,15,58,223,202,64
2299	call	L$113key_256a
2300	movups	[edx],xmm0
2301	mov	DWORD [16+edx],ecx
2302	xor	eax,eax
2303	jmp	NEAR L$102good_key
2304align	16
2305L$113key_256a:
2306	movups	[edx],xmm2
2307	lea	edx,[16+edx]
2308L$111key_256a_cold:
2309	shufps	xmm4,xmm0,16
2310	xorps	xmm0,xmm4
2311	shufps	xmm4,xmm0,140
2312	xorps	xmm0,xmm4
2313	shufps	xmm1,xmm1,255
2314	xorps	xmm0,xmm1
2315	ret
2316align	16
2317L$112key_256b:
2318	movups	[edx],xmm0
2319	lea	edx,[16+edx]
2320	shufps	xmm4,xmm2,16
2321	xorps	xmm2,xmm4
2322	shufps	xmm4,xmm2,140
2323	xorps	xmm2,xmm4
2324	shufps	xmm1,xmm1,170
2325	xorps	xmm2,xmm1
2326	ret
2327align	16
2328L$11014rounds_alt:
2329	movdqa	xmm5,[ebx]
2330	movdqa	xmm4,[32+ebx]
2331	mov	ecx,7
2332	movdqu	[edx-32],xmm0
2333	movdqa	xmm1,xmm2
2334	movdqu	[edx-16],xmm2
2335L$114loop_key256:
2336db	102,15,56,0,213
2337db	102,15,56,221,212
2338	movdqa	xmm3,xmm0
2339	pslldq	xmm0,4
2340	pxor	xmm3,xmm0
2341	pslldq	xmm0,4
2342	pxor	xmm3,xmm0
2343	pslldq	xmm0,4
2344	pxor	xmm0,xmm3
2345	pslld	xmm4,1
2346	pxor	xmm0,xmm2
2347	movdqu	[edx],xmm0
2348	dec	ecx
2349	jz	NEAR L$115done_key256
2350	pshufd	xmm2,xmm0,255
2351	pxor	xmm3,xmm3
2352db	102,15,56,221,211
2353	movdqa	xmm3,xmm1
2354	pslldq	xmm1,4
2355	pxor	xmm3,xmm1
2356	pslldq	xmm1,4
2357	pxor	xmm3,xmm1
2358	pslldq	xmm1,4
2359	pxor	xmm1,xmm3
2360	pxor	xmm2,xmm1
2361	movdqu	[16+edx],xmm2
2362	lea	edx,[32+edx]
2363	movdqa	xmm1,xmm2
2364	jmp	NEAR L$114loop_key256
2365L$115done_key256:
2366	mov	ecx,13
2367	mov	DWORD [16+edx],ecx
2368L$102good_key:
2369	pxor	xmm0,xmm0
2370	pxor	xmm1,xmm1
2371	pxor	xmm2,xmm2
2372	pxor	xmm3,xmm3
2373	pxor	xmm4,xmm4
2374	pxor	xmm5,xmm5
2375	xor	eax,eax
2376	pop	ebx
2377	pop	ebp
2378	ret
2379align	4
2380L$093bad_pointer:
2381	mov	eax,-1
2382	pop	ebx
2383	pop	ebp
2384	ret
2385align	4
2386L$097bad_keybits:
2387	pxor	xmm0,xmm0
2388	mov	eax,-2
2389	pop	ebx
2390	pop	ebp
2391	ret
2392global	_aes_hw_set_encrypt_key
2393align	16
2394_aes_hw_set_encrypt_key:
2395L$_aes_hw_set_encrypt_key_begin:
2396%ifdef BORINGSSL_DISPATCH_TEST
2397	push	ebx
2398	push	edx
2399	call	L$116pic
2400L$116pic:
2401	pop	ebx
2402	lea	ebx,[(_BORINGSSL_function_hit+3-L$116pic)+ebx]
2403	mov	edx,1
2404	mov	BYTE [ebx],dl
2405	pop	edx
2406	pop	ebx
2407%endif
2408	mov	eax,DWORD [4+esp]
2409	mov	ecx,DWORD [8+esp]
2410	mov	edx,DWORD [12+esp]
2411	call	__aesni_set_encrypt_key
2412	ret
2413global	_aes_hw_set_decrypt_key
2414align	16
2415_aes_hw_set_decrypt_key:
2416L$_aes_hw_set_decrypt_key_begin:
2417	mov	eax,DWORD [4+esp]
2418	mov	ecx,DWORD [8+esp]
2419	mov	edx,DWORD [12+esp]
2420	call	__aesni_set_encrypt_key
2421	mov	edx,DWORD [12+esp]
2422	shl	ecx,4
2423	test	eax,eax
2424	jnz	NEAR L$117dec_key_ret
2425	lea	eax,[16+ecx*1+edx]
2426	movups	xmm0,[edx]
2427	movups	xmm1,[eax]
2428	movups	[eax],xmm0
2429	movups	[edx],xmm1
2430	lea	edx,[16+edx]
2431	lea	eax,[eax-16]
2432L$118dec_key_inverse:
2433	movups	xmm0,[edx]
2434	movups	xmm1,[eax]
2435db	102,15,56,219,192
2436db	102,15,56,219,201
2437	lea	edx,[16+edx]
2438	lea	eax,[eax-16]
2439	movups	[16+eax],xmm0
2440	movups	[edx-16],xmm1
2441	cmp	eax,edx
2442	ja	NEAR L$118dec_key_inverse
2443	movups	xmm0,[edx]
2444db	102,15,56,219,192
2445	movups	[edx],xmm0
2446	pxor	xmm0,xmm0
2447	pxor	xmm1,xmm1
2448	xor	eax,eax
2449L$117dec_key_ret:
2450	ret
2451align	64
2452L$key_const:
2453dd	202313229,202313229,202313229,202313229
2454dd	67569157,67569157,67569157,67569157
2455dd	1,1,1,1
2456dd	27,27,27,27
2457db	65,69,83,32,102,111,114,32,73,110,116,101,108,32,65,69
2458db	83,45,78,73,44,32,67,82,89,80,84,79,71,65,77,83
2459db	32,98,121,32,60,97,112,112,114,111,64,111,112,101,110,115
2460db	115,108,46,111,114,103,62,0
2461segment	.bss
2462common	_OPENSSL_ia32cap_P 16
2463%else
2464; Work around https://bugzilla.nasm.us/show_bug.cgi?id=3392738
2465ret
2466%endif
2467