; ; jchuff-sse2.asm - Huffman entropy encoding (64-bit SSE2) ; ; Copyright (C) 2009-2011, 2014-2016, D. R. Commander. ; Copyright (C) 2015, Matthieu Darbois. ; ; Based on the x86 SIMD extension for IJG JPEG library ; Copyright (C) 1999-2006, MIYASAKA Masaru. ; For conditions of distribution and use, see copyright notice in jsimdext.inc ; ; This file should be assembled with NASM (Netwide Assembler), ; can *not* be assembled with Microsoft's MASM or any compatible ; assembler (including Borland's Turbo Assembler). ; NASM is available from http://nasm.sourceforge.net/ or ; http://sourceforge.net/project/showfiles.php?group_id=6208 ; ; This file contains an SSE2 implementation for Huffman coding of one block. ; The following code is based directly on jchuff.c; see jchuff.c for more ; details. ; ; [TAB8] %include "jsimdext.inc" ; -------------------------------------------------------------------------- SECTION SEG_CONST alignz 32 GLOBAL_DATA(jconst_huff_encode_one_block) EXTN(jconst_huff_encode_one_block): %include "jpeg_nbits_table.inc" alignz 32 ; -------------------------------------------------------------------------- SECTION SEG_TEXT BITS 64 ; These macros perform the same task as the emit_bits() function in the ; original libjpeg code. In addition to reducing overhead by explicitly ; inlining the code, additional performance is achieved by taking into ; account the size of the bit buffer and waiting until it is almost full ; before emptying it. This mostly benefits 64-bit platforms, since 6 ; bytes can be stored in a 64-bit bit buffer before it has to be emptied. %macro EMIT_BYTE 0 sub put_bits, 8 ; put_bits -= 8; mov rdx, put_buffer mov ecx, put_bits shr rdx, cl ; c = (JOCTET)GETJOCTET(put_buffer >> put_bits); mov byte [buffer], dl ; *buffer++ = c; add buffer, 1 cmp dl, 0xFF ; need to stuff a zero byte? jne %%.EMIT_BYTE_END mov byte [buffer], 0 ; *buffer++ = 0; add buffer, 1 %%.EMIT_BYTE_END: %endmacro %macro PUT_BITS 1 add put_bits, ecx ; put_bits += size; shl put_buffer, cl ; put_buffer = (put_buffer << size); or put_buffer, %1 %endmacro %macro CHECKBUF31 0 cmp put_bits, 32 ; if (put_bits > 31) { jl %%.CHECKBUF31_END EMIT_BYTE EMIT_BYTE EMIT_BYTE EMIT_BYTE %%.CHECKBUF31_END: %endmacro %macro CHECKBUF47 0 cmp put_bits, 48 ; if (put_bits > 47) { jl %%.CHECKBUF47_END EMIT_BYTE EMIT_BYTE EMIT_BYTE EMIT_BYTE EMIT_BYTE EMIT_BYTE %%.CHECKBUF47_END: %endmacro %macro EMIT_BITS 2 CHECKBUF47 mov ecx, %2 PUT_BITS %1 %endmacro %macro kloop_prepare 37 ;(ko, jno0, ..., jno31, xmm0, xmm1, xmm2, xmm3) pxor xmm8, xmm8 ; __m128i neg = _mm_setzero_si128(); pxor xmm9, xmm9 ; __m128i neg = _mm_setzero_si128(); pxor xmm10, xmm10 ; __m128i neg = _mm_setzero_si128(); pxor xmm11, xmm11 ; __m128i neg = _mm_setzero_si128(); pinsrw %34, word [r12 + %2 * SIZEOF_WORD], 0 ; xmm_shadow[0] = block[jno0]; pinsrw %35, word [r12 + %10 * SIZEOF_WORD], 0 ; xmm_shadow[8] = block[jno8]; pinsrw %36, word [r12 + %18 * SIZEOF_WORD], 0 ; xmm_shadow[16] = block[jno16]; pinsrw %37, word [r12 + %26 * SIZEOF_WORD], 0 ; xmm_shadow[24] = block[jno24]; pinsrw %34, word [r12 + %3 * SIZEOF_WORD], 1 ; xmm_shadow[1] = block[jno1]; pinsrw %35, word [r12 + %11 * SIZEOF_WORD], 1 ; xmm_shadow[9] = block[jno9]; pinsrw %36, word [r12 + %19 * SIZEOF_WORD], 1 ; xmm_shadow[17] = block[jno17]; pinsrw %37, word [r12 + %27 * SIZEOF_WORD], 1 ; xmm_shadow[25] = block[jno25]; pinsrw %34, word [r12 + %4 * SIZEOF_WORD], 2 ; xmm_shadow[2] = block[jno2]; pinsrw %35, word [r12 + %12 * SIZEOF_WORD], 2 ; xmm_shadow[10] = block[jno10]; pinsrw %36, word [r12 + %20 * SIZEOF_WORD], 2 ; xmm_shadow[18] = block[jno18]; pinsrw %37, word [r12 + %28 * SIZEOF_WORD], 2 ; xmm_shadow[26] = block[jno26]; pinsrw %34, word [r12 + %5 * SIZEOF_WORD], 3 ; xmm_shadow[3] = block[jno3]; pinsrw %35, word [r12 + %13 * SIZEOF_WORD], 3 ; xmm_shadow[11] = block[jno11]; pinsrw %36, word [r12 + %21 * SIZEOF_WORD], 3 ; xmm_shadow[19] = block[jno19]; pinsrw %37, word [r12 + %29 * SIZEOF_WORD], 3 ; xmm_shadow[27] = block[jno27]; pinsrw %34, word [r12 + %6 * SIZEOF_WORD], 4 ; xmm_shadow[4] = block[jno4]; pinsrw %35, word [r12 + %14 * SIZEOF_WORD], 4 ; xmm_shadow[12] = block[jno12]; pinsrw %36, word [r12 + %22 * SIZEOF_WORD], 4 ; xmm_shadow[20] = block[jno20]; pinsrw %37, word [r12 + %30 * SIZEOF_WORD], 4 ; xmm_shadow[28] = block[jno28]; pinsrw %34, word [r12 + %7 * SIZEOF_WORD], 5 ; xmm_shadow[5] = block[jno5]; pinsrw %35, word [r12 + %15 * SIZEOF_WORD], 5 ; xmm_shadow[13] = block[jno13]; pinsrw %36, word [r12 + %23 * SIZEOF_WORD], 5 ; xmm_shadow[21] = block[jno21]; pinsrw %37, word [r12 + %31 * SIZEOF_WORD], 5 ; xmm_shadow[29] = block[jno29]; pinsrw %34, word [r12 + %8 * SIZEOF_WORD], 6 ; xmm_shadow[6] = block[jno6]; pinsrw %35, word [r12 + %16 * SIZEOF_WORD], 6 ; xmm_shadow[14] = block[jno14]; pinsrw %36, word [r12 + %24 * SIZEOF_WORD], 6 ; xmm_shadow[22] = block[jno22]; pinsrw %37, word [r12 + %32 * SIZEOF_WORD], 6 ; xmm_shadow[30] = block[jno30]; pinsrw %34, word [r12 + %9 * SIZEOF_WORD], 7 ; xmm_shadow[7] = block[jno7]; pinsrw %35, word [r12 + %17 * SIZEOF_WORD], 7 ; xmm_shadow[15] = block[jno15]; pinsrw %36, word [r12 + %25 * SIZEOF_WORD], 7 ; xmm_shadow[23] = block[jno23]; %if %1 != 32 pinsrw %37, word [r12 + %33 * SIZEOF_WORD], 7 ; xmm_shadow[31] = block[jno31]; %else pinsrw %37, ebx, 7 ; xmm_shadow[31] = block[jno31]; %endif pcmpgtw xmm8, %34 ; neg = _mm_cmpgt_epi16(neg, x1); pcmpgtw xmm9, %35 ; neg = _mm_cmpgt_epi16(neg, x1); pcmpgtw xmm10, %36 ; neg = _mm_cmpgt_epi16(neg, x1); pcmpgtw xmm11, %37 ; neg = _mm_cmpgt_epi16(neg, x1); paddw %34, xmm8 ; x1 = _mm_add_epi16(x1, neg); paddw %35, xmm9 ; x1 = _mm_add_epi16(x1, neg); paddw %36, xmm10 ; x1 = _mm_add_epi16(x1, neg); paddw %37, xmm11 ; x1 = _mm_add_epi16(x1, neg); pxor %34, xmm8 ; x1 = _mm_xor_si128(x1, neg); pxor %35, xmm9 ; x1 = _mm_xor_si128(x1, neg); pxor %36, xmm10 ; x1 = _mm_xor_si128(x1, neg); pxor %37, xmm11 ; x1 = _mm_xor_si128(x1, neg); pxor xmm8, %34 ; neg = _mm_xor_si128(neg, x1); pxor xmm9, %35 ; neg = _mm_xor_si128(neg, x1); pxor xmm10, %36 ; neg = _mm_xor_si128(neg, x1); pxor xmm11, %37 ; neg = _mm_xor_si128(neg, x1); movdqa XMMWORD [t1 + %1 * SIZEOF_WORD], %34 ; _mm_storeu_si128((__m128i *)(t1 + ko), x1); movdqa XMMWORD [t1 + (%1 + 8) * SIZEOF_WORD], %35 ; _mm_storeu_si128((__m128i *)(t1 + ko + 8), x1); movdqa XMMWORD [t1 + (%1 + 16) * SIZEOF_WORD], %36 ; _mm_storeu_si128((__m128i *)(t1 + ko + 16), x1); movdqa XMMWORD [t1 + (%1 + 24) * SIZEOF_WORD], %37 ; _mm_storeu_si128((__m128i *)(t1 + ko + 24), x1); movdqa XMMWORD [t2 + %1 * SIZEOF_WORD], xmm8 ; _mm_storeu_si128((__m128i *)(t2 + ko), neg); movdqa XMMWORD [t2 + (%1 + 8) * SIZEOF_WORD], xmm9 ; _mm_storeu_si128((__m128i *)(t2 + ko + 8), neg); movdqa XMMWORD [t2 + (%1 + 16) * SIZEOF_WORD], xmm10 ; _mm_storeu_si128((__m128i *)(t2 + ko + 16), neg); movdqa XMMWORD [t2 + (%1 + 24) * SIZEOF_WORD], xmm11 ; _mm_storeu_si128((__m128i *)(t2 + ko + 24), neg); %endmacro ; ; Encode a single block's worth of coefficients. ; ; GLOBAL(JOCTET *) ; jsimd_huff_encode_one_block_sse2(working_state *state, JOCTET *buffer, ; JCOEFPTR block, int last_dc_val, ; c_derived_tbl *dctbl, c_derived_tbl *actbl) ; ; r10 = working_state *state ; r11 = JOCTET *buffer ; r12 = JCOEFPTR block ; r13d = int last_dc_val ; r14 = c_derived_tbl *dctbl ; r15 = c_derived_tbl *actbl %define t1 rbp - (DCTSIZE2 * SIZEOF_WORD) %define t2 t1 - (DCTSIZE2 * SIZEOF_WORD) %define put_buffer r8 %define put_bits r9d %define buffer rax align 32 GLOBAL_FUNCTION(jsimd_huff_encode_one_block_sse2) EXTN(jsimd_huff_encode_one_block_sse2): push rbp mov rax, rsp ; rax = original rbp sub rsp, byte 4 and rsp, byte (-SIZEOF_XMMWORD) ; align to 128 bits mov [rsp], rax mov rbp, rsp ; rbp = aligned rbp lea rsp, [t2] push_xmm 4 collect_args 6 push rbx mov buffer, r11 ; r11 is now sratch mov put_buffer, MMWORD [r10+16] ; put_buffer = state->cur.put_buffer; mov put_bits, DWORD [r10+24] ; put_bits = state->cur.put_bits; push r10 ; r10 is now scratch ; Encode the DC coefficient difference per section F.1.2.1 movsx edi, word [r12] ; temp = temp2 = block[0] - last_dc_val; sub edi, r13d ; r13 is not used anymore mov ebx, edi ; This is a well-known technique for obtaining the absolute value ; without a branch. It is derived from an assembly language technique ; presented in "How to Optimize for the Pentium Processors", ; Copyright (c) 1996, 1997 by Agner Fog. mov esi, edi sar esi, 31 ; temp3 = temp >> (CHAR_BIT * sizeof(int) - 1); xor edi, esi ; temp ^= temp3; sub edi, esi ; temp -= temp3; ; For a negative input, want temp2 = bitwise complement of abs(input) ; This code assumes we are on a two's complement machine add ebx, esi ; temp2 += temp3; ; Find the number of bits needed for the magnitude of the coefficient lea r11, [rel jpeg_nbits_table] movzx rdi, byte [r11 + rdi] ; nbits = JPEG_NBITS(temp); ; Emit the Huffman-coded symbol for the number of bits mov r11d, INT [r14 + rdi * 4] ; code = dctbl->ehufco[nbits]; movzx esi, byte [r14 + rdi + 1024] ; size = dctbl->ehufsi[nbits]; EMIT_BITS r11, esi ; EMIT_BITS(code, size) ; Mask off any extra bits in code mov esi, 1 mov ecx, edi shl esi, cl dec esi and ebx, esi ; temp2 &= (((JLONG)1)<ehufco[0xf0]; movzx r14d, byte [r15 + 1024 + 240] ; size_0xf0 = actbl->ehufsi[0xf0]; lea rsi, [t1] .BLOOP: bsf r12, r11 ; r = __builtin_ctzl(index); jz .ELOOP mov rcx, r12 lea rsi, [rsi+r12*2] ; k += r; shr r11, cl ; index >>= r; movzx rdi, word [rsi] ; temp = t1[k]; lea rbx, [rel jpeg_nbits_table] movzx rdi, byte [rbx + rdi] ; nbits = JPEG_NBITS(temp); .BRLOOP: cmp r12, 16 ; while (r > 15) { jl .ERLOOP EMIT_BITS r13, r14d ; EMIT_BITS(code_0xf0, size_0xf0) sub r12, 16 ; r -= 16; jmp .BRLOOP .ERLOOP: ; Emit Huffman symbol for run length / number of bits CHECKBUF31 ; uses rcx, rdx shl r12, 4 ; temp3 = (r << 4) + nbits; add r12, rdi mov ebx, INT [r15 + r12 * 4] ; code = actbl->ehufco[temp3]; movzx ecx, byte [r15 + r12 + 1024] ; size = actbl->ehufsi[temp3]; PUT_BITS rbx ;EMIT_CODE(code, size) movsx ebx, word [rsi-DCTSIZE2*2] ; temp2 = t2[k]; ; Mask off any extra bits in code mov rcx, rdi mov rdx, 1 shl rdx, cl dec rdx and rbx, rdx ; temp2 &= (((JLONG)1)<>= 1; add rsi, 2 ; ++k; jmp .BLOOP .ELOOP: ; If the last coef(s) were zero, emit an end-of-block code lea rdi, [t1 + (DCTSIZE2-1) * 2] ; r = DCTSIZE2-1-k; cmp rdi, rsi ; if (r > 0) { je .EFN mov ebx, INT [r15] ; code = actbl->ehufco[0]; movzx r12d, byte [r15 + 1024] ; size = actbl->ehufsi[0]; EMIT_BITS rbx, r12d .EFN: pop r10 ; Save put_buffer & put_bits mov MMWORD [r10+16], put_buffer ; state->cur.put_buffer = put_buffer; mov DWORD [r10+24], put_bits ; state->cur.put_bits = put_bits; pop rbx uncollect_args 6 pop_xmm 4 mov rsp, rbp ; rsp <- aligned rbp pop rsp ; rsp <- original rbp pop rbp ret ; For some reason, the OS X linker does not honor the request to align the ; segment unless we do this. align 32